1 // SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause) 2 /* 3 * Copyright 2015-2016 Freescale Semiconductor Inc. 4 * Copyright 2017-2019 NXP 5 */ 6 7 #include "compat.h" 8 #include "regs.h" 9 #include "caamalg_qi2.h" 10 #include "dpseci_cmd.h" 11 #include "desc_constr.h" 12 #include "error.h" 13 #include "sg_sw_sec4.h" 14 #include "sg_sw_qm2.h" 15 #include "key_gen.h" 16 #include "caamalg_desc.h" 17 #include "caamhash_desc.h" 18 #include "dpseci-debugfs.h" 19 #include <linux/dma-mapping.h> 20 #include <linux/fsl/mc.h> 21 #include <linux/kernel.h> 22 #include <linux/string.h> 23 #include <linux/string_choices.h> 24 #include <soc/fsl/dpaa2-io.h> 25 #include <soc/fsl/dpaa2-fd.h> 26 #include <crypto/xts.h> 27 #include <linux/unaligned.h> 28 29 #define CAAM_CRA_PRIORITY 100 30 31 /* max key is sum of AES_MAX_KEY_SIZE, max split key size */ 32 #define CAAM_MAX_KEY_SIZE (AES_MAX_KEY_SIZE + CTR_RFC3686_NONCE_SIZE + \ 33 SHA512_DIGEST_SIZE * 2) 34 35 /* 36 * This is a cache of buffers, from which the users of CAAM QI driver 37 * can allocate short buffers. It's speedier than doing kmalloc on the hotpath. 38 * NOTE: A more elegant solution would be to have some headroom in the frames 39 * being processed. This can be added by the dpaa2-eth driver. This would 40 * pose a problem for userspace application processing which cannot 41 * know of this limitation. So for now, this will work. 42 * NOTE: The memcache is SMP-safe. No need to handle spinlocks in-here 43 */ 44 static struct kmem_cache *qi_cache; 45 46 struct caam_alg_entry { 47 struct device *dev; 48 int class1_alg_type; 49 int class2_alg_type; 50 bool rfc3686; 51 bool geniv; 52 bool nodkp; 53 }; 54 55 struct caam_aead_alg { 56 struct aead_alg aead; 57 struct caam_alg_entry caam; 58 bool registered; 59 }; 60 61 struct caam_skcipher_alg { 62 struct skcipher_alg skcipher; 63 struct caam_alg_entry caam; 64 bool registered; 65 }; 66 67 /** 68 * struct caam_ctx - per-session context 69 * @flc: Flow Contexts array 70 * @key: [authentication key], encryption key 71 * @flc_dma: I/O virtual addresses of the Flow Contexts 72 * @key_dma: I/O virtual address of the key 73 * @dir: DMA direction for mapping key and Flow Contexts 74 * @dev: dpseci device 75 * @adata: authentication algorithm details 76 * @cdata: encryption algorithm details 77 * @authsize: authentication tag (a.k.a. ICV / MAC) size 78 * @xts_key_fallback: true if fallback tfm needs to be used due 79 * to unsupported xts key lengths 80 * @fallback: xts fallback tfm 81 */ 82 struct caam_ctx { 83 struct caam_flc flc[NUM_OP]; 84 u8 key[CAAM_MAX_KEY_SIZE]; 85 dma_addr_t flc_dma[NUM_OP]; 86 dma_addr_t key_dma; 87 enum dma_data_direction dir; 88 struct device *dev; 89 struct alginfo adata; 90 struct alginfo cdata; 91 unsigned int authsize; 92 bool xts_key_fallback; 93 struct crypto_skcipher *fallback; 94 }; 95 96 static void *dpaa2_caam_iova_to_virt(struct dpaa2_caam_priv *priv, 97 dma_addr_t iova_addr) 98 { 99 phys_addr_t phys_addr; 100 101 phys_addr = priv->domain ? iommu_iova_to_phys(priv->domain, iova_addr) : 102 iova_addr; 103 104 return phys_to_virt(phys_addr); 105 } 106 107 /* 108 * qi_cache_zalloc - Allocate buffers from CAAM-QI cache 109 * 110 * Allocate data on the hotpath. Instead of using kzalloc, one can use the 111 * services of the CAAM QI memory cache (backed by kmem_cache). The buffers 112 * will have a size of CAAM_QI_MEMCACHE_SIZE, which should be sufficient for 113 * hosting 16 SG entries. 114 * 115 * @flags - flags that would be used for the equivalent kmalloc(..) call 116 * 117 * Returns a pointer to a retrieved buffer on success or NULL on failure. 118 */ 119 static inline void *qi_cache_zalloc(gfp_t flags) 120 { 121 return kmem_cache_zalloc(qi_cache, flags); 122 } 123 124 /* 125 * qi_cache_free - Frees buffers allocated from CAAM-QI cache 126 * 127 * @obj - buffer previously allocated by qi_cache_zalloc 128 * 129 * No checking is being done, the call is a passthrough call to 130 * kmem_cache_free(...) 131 */ 132 static inline void qi_cache_free(void *obj) 133 { 134 kmem_cache_free(qi_cache, obj); 135 } 136 137 static struct caam_request *to_caam_req(struct crypto_async_request *areq) 138 { 139 switch (crypto_tfm_alg_type(areq->tfm)) { 140 case CRYPTO_ALG_TYPE_SKCIPHER: 141 return skcipher_request_ctx_dma(skcipher_request_cast(areq)); 142 case CRYPTO_ALG_TYPE_AEAD: 143 return aead_request_ctx_dma( 144 container_of(areq, struct aead_request, base)); 145 case CRYPTO_ALG_TYPE_AHASH: 146 return ahash_request_ctx_dma(ahash_request_cast(areq)); 147 default: 148 return ERR_PTR(-EINVAL); 149 } 150 } 151 152 static void caam_unmap(struct device *dev, struct scatterlist *src, 153 struct scatterlist *dst, int src_nents, 154 int dst_nents, dma_addr_t iv_dma, int ivsize, 155 enum dma_data_direction iv_dir, dma_addr_t qm_sg_dma, 156 int qm_sg_bytes) 157 { 158 if (dst != src) { 159 if (src_nents) 160 dma_unmap_sg(dev, src, src_nents, DMA_TO_DEVICE); 161 if (dst_nents) 162 dma_unmap_sg(dev, dst, dst_nents, DMA_FROM_DEVICE); 163 } else { 164 dma_unmap_sg(dev, src, src_nents, DMA_BIDIRECTIONAL); 165 } 166 167 if (iv_dma) 168 dma_unmap_single(dev, iv_dma, ivsize, iv_dir); 169 170 if (qm_sg_bytes) 171 dma_unmap_single(dev, qm_sg_dma, qm_sg_bytes, DMA_TO_DEVICE); 172 } 173 174 static int aead_set_sh_desc(struct crypto_aead *aead) 175 { 176 struct caam_aead_alg *alg = container_of(crypto_aead_alg(aead), 177 typeof(*alg), aead); 178 struct caam_ctx *ctx = crypto_aead_ctx_dma(aead); 179 unsigned int ivsize = crypto_aead_ivsize(aead); 180 struct device *dev = ctx->dev; 181 struct dpaa2_caam_priv *priv = dev_get_drvdata(dev); 182 struct caam_flc *flc; 183 u32 *desc; 184 u32 ctx1_iv_off = 0; 185 u32 *nonce = NULL; 186 unsigned int data_len[2]; 187 u32 inl_mask; 188 const bool ctr_mode = ((ctx->cdata.algtype & OP_ALG_AAI_MASK) == 189 OP_ALG_AAI_CTR_MOD128); 190 const bool is_rfc3686 = alg->caam.rfc3686; 191 192 if (!ctx->cdata.keylen || !ctx->authsize) 193 return 0; 194 195 /* 196 * AES-CTR needs to load IV in CONTEXT1 reg 197 * at an offset of 128bits (16bytes) 198 * CONTEXT1[255:128] = IV 199 */ 200 if (ctr_mode) 201 ctx1_iv_off = 16; 202 203 /* 204 * RFC3686 specific: 205 * CONTEXT1[255:128] = {NONCE, IV, COUNTER} 206 */ 207 if (is_rfc3686) { 208 ctx1_iv_off = 16 + CTR_RFC3686_NONCE_SIZE; 209 nonce = (u32 *)((void *)ctx->key + ctx->adata.keylen_pad + 210 ctx->cdata.keylen - CTR_RFC3686_NONCE_SIZE); 211 } 212 213 /* 214 * In case |user key| > |derived key|, using DKP<imm,imm> would result 215 * in invalid opcodes (last bytes of user key) in the resulting 216 * descriptor. Use DKP<ptr,imm> instead => both virtual and dma key 217 * addresses are needed. 218 */ 219 ctx->adata.key_virt = ctx->key; 220 ctx->adata.key_dma = ctx->key_dma; 221 222 ctx->cdata.key_virt = ctx->key + ctx->adata.keylen_pad; 223 ctx->cdata.key_dma = ctx->key_dma + ctx->adata.keylen_pad; 224 225 data_len[0] = ctx->adata.keylen_pad; 226 data_len[1] = ctx->cdata.keylen; 227 228 /* aead_encrypt shared descriptor */ 229 if (desc_inline_query((alg->caam.geniv ? DESC_QI_AEAD_GIVENC_LEN : 230 DESC_QI_AEAD_ENC_LEN) + 231 (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0), 232 DESC_JOB_IO_LEN, data_len, &inl_mask, 233 ARRAY_SIZE(data_len)) < 0) 234 return -EINVAL; 235 236 ctx->adata.key_inline = !!(inl_mask & 1); 237 ctx->cdata.key_inline = !!(inl_mask & 2); 238 239 flc = &ctx->flc[ENCRYPT]; 240 desc = flc->sh_desc; 241 242 if (alg->caam.geniv) 243 cnstr_shdsc_aead_givencap(desc, &ctx->cdata, &ctx->adata, 244 ivsize, ctx->authsize, is_rfc3686, 245 nonce, ctx1_iv_off, true, 246 priv->sec_attr.era); 247 else 248 cnstr_shdsc_aead_encap(desc, &ctx->cdata, &ctx->adata, 249 ivsize, ctx->authsize, is_rfc3686, nonce, 250 ctx1_iv_off, true, priv->sec_attr.era); 251 252 flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ 253 dma_sync_single_for_device(dev, ctx->flc_dma[ENCRYPT], 254 sizeof(flc->flc) + desc_bytes(desc), 255 ctx->dir); 256 257 /* aead_decrypt shared descriptor */ 258 if (desc_inline_query(DESC_QI_AEAD_DEC_LEN + 259 (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0), 260 DESC_JOB_IO_LEN, data_len, &inl_mask, 261 ARRAY_SIZE(data_len)) < 0) 262 return -EINVAL; 263 264 ctx->adata.key_inline = !!(inl_mask & 1); 265 ctx->cdata.key_inline = !!(inl_mask & 2); 266 267 flc = &ctx->flc[DECRYPT]; 268 desc = flc->sh_desc; 269 cnstr_shdsc_aead_decap(desc, &ctx->cdata, &ctx->adata, 270 ivsize, ctx->authsize, alg->caam.geniv, 271 is_rfc3686, nonce, ctx1_iv_off, true, 272 priv->sec_attr.era); 273 flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ 274 dma_sync_single_for_device(dev, ctx->flc_dma[DECRYPT], 275 sizeof(flc->flc) + desc_bytes(desc), 276 ctx->dir); 277 278 return 0; 279 } 280 281 static int aead_setauthsize(struct crypto_aead *authenc, unsigned int authsize) 282 { 283 struct caam_ctx *ctx = crypto_aead_ctx_dma(authenc); 284 285 ctx->authsize = authsize; 286 aead_set_sh_desc(authenc); 287 288 return 0; 289 } 290 291 static int aead_setkey(struct crypto_aead *aead, const u8 *key, 292 unsigned int keylen) 293 { 294 struct caam_ctx *ctx = crypto_aead_ctx_dma(aead); 295 struct device *dev = ctx->dev; 296 struct crypto_authenc_keys keys; 297 298 if (crypto_authenc_extractkeys(&keys, key, keylen) != 0) 299 goto badkey; 300 301 dev_dbg(dev, "keylen %d enckeylen %d authkeylen %d\n", 302 keys.authkeylen + keys.enckeylen, keys.enckeylen, 303 keys.authkeylen); 304 print_hex_dump_devel("key in @" __stringify(__LINE__)": ", 305 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); 306 307 ctx->adata.keylen = keys.authkeylen; 308 ctx->adata.keylen_pad = split_key_len(ctx->adata.algtype & 309 OP_ALG_ALGSEL_MASK); 310 311 if (ctx->adata.keylen_pad + keys.enckeylen > CAAM_MAX_KEY_SIZE) 312 goto badkey; 313 314 memcpy(ctx->key, keys.authkey, keys.authkeylen); 315 memcpy(ctx->key + ctx->adata.keylen_pad, keys.enckey, keys.enckeylen); 316 dma_sync_single_for_device(dev, ctx->key_dma, ctx->adata.keylen_pad + 317 keys.enckeylen, ctx->dir); 318 print_hex_dump_devel("ctx.key@" __stringify(__LINE__)": ", 319 DUMP_PREFIX_ADDRESS, 16, 4, ctx->key, 320 ctx->adata.keylen_pad + keys.enckeylen, 1); 321 322 ctx->cdata.keylen = keys.enckeylen; 323 324 memzero_explicit(&keys, sizeof(keys)); 325 return aead_set_sh_desc(aead); 326 badkey: 327 memzero_explicit(&keys, sizeof(keys)); 328 return -EINVAL; 329 } 330 331 static int des3_aead_setkey(struct crypto_aead *aead, const u8 *key, 332 unsigned int keylen) 333 { 334 struct crypto_authenc_keys keys; 335 int err; 336 337 err = crypto_authenc_extractkeys(&keys, key, keylen); 338 if (unlikely(err)) 339 goto out; 340 341 err = -EINVAL; 342 if (keys.enckeylen != DES3_EDE_KEY_SIZE) 343 goto out; 344 345 err = crypto_des3_ede_verify_key(crypto_aead_tfm(aead), keys.enckey) ?: 346 aead_setkey(aead, key, keylen); 347 348 out: 349 memzero_explicit(&keys, sizeof(keys)); 350 return err; 351 } 352 353 static struct aead_edesc *aead_edesc_alloc(struct aead_request *req, 354 bool encrypt) 355 { 356 struct crypto_aead *aead = crypto_aead_reqtfm(req); 357 struct caam_request *req_ctx = aead_request_ctx_dma(req); 358 struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1]; 359 struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0]; 360 struct caam_ctx *ctx = crypto_aead_ctx_dma(aead); 361 struct caam_aead_alg *alg = container_of(crypto_aead_alg(aead), 362 typeof(*alg), aead); 363 struct device *dev = ctx->dev; 364 gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? 365 GFP_KERNEL : GFP_ATOMIC; 366 int src_nents, mapped_src_nents, dst_nents = 0, mapped_dst_nents = 0; 367 int src_len, dst_len = 0; 368 struct aead_edesc *edesc; 369 dma_addr_t qm_sg_dma, iv_dma = 0; 370 int ivsize = 0; 371 unsigned int authsize = ctx->authsize; 372 int qm_sg_index = 0, qm_sg_nents = 0, qm_sg_bytes; 373 int in_len, out_len; 374 struct dpaa2_sg_entry *sg_table; 375 376 /* allocate space for base edesc, link tables and IV */ 377 edesc = qi_cache_zalloc(flags); 378 if (unlikely(!edesc)) { 379 dev_err(dev, "could not allocate extended descriptor\n"); 380 return ERR_PTR(-ENOMEM); 381 } 382 383 if (unlikely(req->dst != req->src)) { 384 src_len = req->assoclen + req->cryptlen; 385 dst_len = src_len + (encrypt ? authsize : (-authsize)); 386 387 src_nents = sg_nents_for_len(req->src, src_len); 388 if (unlikely(src_nents < 0)) { 389 dev_err(dev, "Insufficient bytes (%d) in src S/G\n", 390 src_len); 391 qi_cache_free(edesc); 392 return ERR_PTR(src_nents); 393 } 394 395 dst_nents = sg_nents_for_len(req->dst, dst_len); 396 if (unlikely(dst_nents < 0)) { 397 dev_err(dev, "Insufficient bytes (%d) in dst S/G\n", 398 dst_len); 399 qi_cache_free(edesc); 400 return ERR_PTR(dst_nents); 401 } 402 403 if (src_nents) { 404 mapped_src_nents = dma_map_sg(dev, req->src, src_nents, 405 DMA_TO_DEVICE); 406 if (unlikely(!mapped_src_nents)) { 407 dev_err(dev, "unable to map source\n"); 408 qi_cache_free(edesc); 409 return ERR_PTR(-ENOMEM); 410 } 411 } else { 412 mapped_src_nents = 0; 413 } 414 415 if (dst_nents) { 416 mapped_dst_nents = dma_map_sg(dev, req->dst, dst_nents, 417 DMA_FROM_DEVICE); 418 if (unlikely(!mapped_dst_nents)) { 419 dev_err(dev, "unable to map destination\n"); 420 dma_unmap_sg(dev, req->src, src_nents, 421 DMA_TO_DEVICE); 422 qi_cache_free(edesc); 423 return ERR_PTR(-ENOMEM); 424 } 425 } else { 426 mapped_dst_nents = 0; 427 } 428 } else { 429 src_len = req->assoclen + req->cryptlen + 430 (encrypt ? authsize : 0); 431 432 src_nents = sg_nents_for_len(req->src, src_len); 433 if (unlikely(src_nents < 0)) { 434 dev_err(dev, "Insufficient bytes (%d) in src S/G\n", 435 src_len); 436 qi_cache_free(edesc); 437 return ERR_PTR(src_nents); 438 } 439 440 mapped_src_nents = dma_map_sg(dev, req->src, src_nents, 441 DMA_BIDIRECTIONAL); 442 if (unlikely(!mapped_src_nents)) { 443 dev_err(dev, "unable to map source\n"); 444 qi_cache_free(edesc); 445 return ERR_PTR(-ENOMEM); 446 } 447 } 448 449 if ((alg->caam.rfc3686 && encrypt) || !alg->caam.geniv) 450 ivsize = crypto_aead_ivsize(aead); 451 452 /* 453 * Create S/G table: req->assoclen, [IV,] req->src [, req->dst]. 454 * Input is not contiguous. 455 * HW reads 4 S/G entries at a time; make sure the reads don't go beyond 456 * the end of the table by allocating more S/G entries. Logic: 457 * if (src != dst && output S/G) 458 * pad output S/G, if needed 459 * else if (src == dst && S/G) 460 * overlapping S/Gs; pad one of them 461 * else if (input S/G) ... 462 * pad input S/G, if needed 463 */ 464 qm_sg_nents = 1 + !!ivsize + mapped_src_nents; 465 if (mapped_dst_nents > 1) 466 qm_sg_nents += pad_sg_nents(mapped_dst_nents); 467 else if ((req->src == req->dst) && (mapped_src_nents > 1)) 468 qm_sg_nents = max(pad_sg_nents(qm_sg_nents), 469 1 + !!ivsize + 470 pad_sg_nents(mapped_src_nents)); 471 else 472 qm_sg_nents = pad_sg_nents(qm_sg_nents); 473 474 sg_table = &edesc->sgt[0]; 475 qm_sg_bytes = qm_sg_nents * sizeof(*sg_table); 476 if (unlikely(offsetof(struct aead_edesc, sgt) + qm_sg_bytes + ivsize > 477 CAAM_QI_MEMCACHE_SIZE)) { 478 dev_err(dev, "No space for %d S/G entries and/or %dB IV\n", 479 qm_sg_nents, ivsize); 480 caam_unmap(dev, req->src, req->dst, src_nents, dst_nents, 0, 481 0, DMA_NONE, 0, 0); 482 qi_cache_free(edesc); 483 return ERR_PTR(-ENOMEM); 484 } 485 486 if (ivsize) { 487 u8 *iv = (u8 *)(sg_table + qm_sg_nents); 488 489 /* Make sure IV is located in a DMAable area */ 490 memcpy(iv, req->iv, ivsize); 491 492 iv_dma = dma_map_single(dev, iv, ivsize, DMA_TO_DEVICE); 493 if (dma_mapping_error(dev, iv_dma)) { 494 dev_err(dev, "unable to map IV\n"); 495 caam_unmap(dev, req->src, req->dst, src_nents, 496 dst_nents, 0, 0, DMA_NONE, 0, 0); 497 qi_cache_free(edesc); 498 return ERR_PTR(-ENOMEM); 499 } 500 } 501 502 edesc->src_nents = src_nents; 503 edesc->dst_nents = dst_nents; 504 edesc->iv_dma = iv_dma; 505 506 if ((alg->caam.class1_alg_type & OP_ALG_ALGSEL_MASK) == 507 OP_ALG_ALGSEL_CHACHA20 && ivsize != CHACHAPOLY_IV_SIZE) 508 /* 509 * The associated data comes already with the IV but we need 510 * to skip it when we authenticate or encrypt... 511 */ 512 edesc->assoclen = cpu_to_caam32(req->assoclen - ivsize); 513 else 514 edesc->assoclen = cpu_to_caam32(req->assoclen); 515 edesc->assoclen_dma = dma_map_single(dev, &edesc->assoclen, 4, 516 DMA_TO_DEVICE); 517 if (dma_mapping_error(dev, edesc->assoclen_dma)) { 518 dev_err(dev, "unable to map assoclen\n"); 519 caam_unmap(dev, req->src, req->dst, src_nents, dst_nents, 520 iv_dma, ivsize, DMA_TO_DEVICE, 0, 0); 521 qi_cache_free(edesc); 522 return ERR_PTR(-ENOMEM); 523 } 524 525 dma_to_qm_sg_one(sg_table, edesc->assoclen_dma, 4, 0); 526 qm_sg_index++; 527 if (ivsize) { 528 dma_to_qm_sg_one(sg_table + qm_sg_index, iv_dma, ivsize, 0); 529 qm_sg_index++; 530 } 531 sg_to_qm_sg_last(req->src, src_len, sg_table + qm_sg_index, 0); 532 qm_sg_index += mapped_src_nents; 533 534 if (mapped_dst_nents > 1) 535 sg_to_qm_sg_last(req->dst, dst_len, sg_table + qm_sg_index, 0); 536 537 qm_sg_dma = dma_map_single(dev, sg_table, qm_sg_bytes, DMA_TO_DEVICE); 538 if (dma_mapping_error(dev, qm_sg_dma)) { 539 dev_err(dev, "unable to map S/G table\n"); 540 dma_unmap_single(dev, edesc->assoclen_dma, 4, DMA_TO_DEVICE); 541 caam_unmap(dev, req->src, req->dst, src_nents, dst_nents, 542 iv_dma, ivsize, DMA_TO_DEVICE, 0, 0); 543 qi_cache_free(edesc); 544 return ERR_PTR(-ENOMEM); 545 } 546 547 edesc->qm_sg_dma = qm_sg_dma; 548 edesc->qm_sg_bytes = qm_sg_bytes; 549 550 out_len = req->assoclen + req->cryptlen + 551 (encrypt ? ctx->authsize : (-ctx->authsize)); 552 in_len = 4 + ivsize + req->assoclen + req->cryptlen; 553 554 memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt)); 555 dpaa2_fl_set_final(in_fle, true); 556 dpaa2_fl_set_format(in_fle, dpaa2_fl_sg); 557 dpaa2_fl_set_addr(in_fle, qm_sg_dma); 558 dpaa2_fl_set_len(in_fle, in_len); 559 560 if (req->dst == req->src) { 561 if (mapped_src_nents == 1) { 562 dpaa2_fl_set_format(out_fle, dpaa2_fl_single); 563 dpaa2_fl_set_addr(out_fle, sg_dma_address(req->src)); 564 } else { 565 dpaa2_fl_set_format(out_fle, dpaa2_fl_sg); 566 dpaa2_fl_set_addr(out_fle, qm_sg_dma + 567 (1 + !!ivsize) * sizeof(*sg_table)); 568 } 569 } else if (!mapped_dst_nents) { 570 /* 571 * crypto engine requires the output entry to be present when 572 * "frame list" FD is used. 573 * Since engine does not support FMT=2'b11 (unused entry type), 574 * leaving out_fle zeroized is the best option. 575 */ 576 goto skip_out_fle; 577 } else if (mapped_dst_nents == 1) { 578 dpaa2_fl_set_format(out_fle, dpaa2_fl_single); 579 dpaa2_fl_set_addr(out_fle, sg_dma_address(req->dst)); 580 } else { 581 dpaa2_fl_set_format(out_fle, dpaa2_fl_sg); 582 dpaa2_fl_set_addr(out_fle, qm_sg_dma + qm_sg_index * 583 sizeof(*sg_table)); 584 } 585 586 dpaa2_fl_set_len(out_fle, out_len); 587 588 skip_out_fle: 589 return edesc; 590 } 591 592 static int chachapoly_set_sh_desc(struct crypto_aead *aead) 593 { 594 struct caam_ctx *ctx = crypto_aead_ctx_dma(aead); 595 unsigned int ivsize = crypto_aead_ivsize(aead); 596 struct device *dev = ctx->dev; 597 struct caam_flc *flc; 598 u32 *desc; 599 600 if (!ctx->cdata.keylen || !ctx->authsize) 601 return 0; 602 603 flc = &ctx->flc[ENCRYPT]; 604 desc = flc->sh_desc; 605 cnstr_shdsc_chachapoly(desc, &ctx->cdata, &ctx->adata, ivsize, 606 ctx->authsize, true, true); 607 flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ 608 dma_sync_single_for_device(dev, ctx->flc_dma[ENCRYPT], 609 sizeof(flc->flc) + desc_bytes(desc), 610 ctx->dir); 611 612 flc = &ctx->flc[DECRYPT]; 613 desc = flc->sh_desc; 614 cnstr_shdsc_chachapoly(desc, &ctx->cdata, &ctx->adata, ivsize, 615 ctx->authsize, false, true); 616 flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ 617 dma_sync_single_for_device(dev, ctx->flc_dma[DECRYPT], 618 sizeof(flc->flc) + desc_bytes(desc), 619 ctx->dir); 620 621 return 0; 622 } 623 624 static int chachapoly_setauthsize(struct crypto_aead *aead, 625 unsigned int authsize) 626 { 627 struct caam_ctx *ctx = crypto_aead_ctx_dma(aead); 628 629 if (authsize != POLY1305_DIGEST_SIZE) 630 return -EINVAL; 631 632 ctx->authsize = authsize; 633 return chachapoly_set_sh_desc(aead); 634 } 635 636 static int chachapoly_setkey(struct crypto_aead *aead, const u8 *key, 637 unsigned int keylen) 638 { 639 struct caam_ctx *ctx = crypto_aead_ctx_dma(aead); 640 unsigned int ivsize = crypto_aead_ivsize(aead); 641 unsigned int saltlen = CHACHAPOLY_IV_SIZE - ivsize; 642 643 if (keylen != CHACHA_KEY_SIZE + saltlen) 644 return -EINVAL; 645 646 memcpy(ctx->key, key, keylen); 647 ctx->cdata.key_virt = ctx->key; 648 ctx->cdata.keylen = keylen - saltlen; 649 650 return chachapoly_set_sh_desc(aead); 651 } 652 653 static int gcm_set_sh_desc(struct crypto_aead *aead) 654 { 655 struct caam_ctx *ctx = crypto_aead_ctx_dma(aead); 656 struct device *dev = ctx->dev; 657 unsigned int ivsize = crypto_aead_ivsize(aead); 658 struct caam_flc *flc; 659 u32 *desc; 660 int rem_bytes = CAAM_DESC_BYTES_MAX - DESC_JOB_IO_LEN - 661 ctx->cdata.keylen; 662 663 if (!ctx->cdata.keylen || !ctx->authsize) 664 return 0; 665 666 /* 667 * AES GCM encrypt shared descriptor 668 * Job Descriptor and Shared Descriptor 669 * must fit into the 64-word Descriptor h/w Buffer 670 */ 671 if (rem_bytes >= DESC_QI_GCM_ENC_LEN) { 672 ctx->cdata.key_inline = true; 673 ctx->cdata.key_virt = ctx->key; 674 } else { 675 ctx->cdata.key_inline = false; 676 ctx->cdata.key_dma = ctx->key_dma; 677 } 678 679 flc = &ctx->flc[ENCRYPT]; 680 desc = flc->sh_desc; 681 cnstr_shdsc_gcm_encap(desc, &ctx->cdata, ivsize, ctx->authsize, true); 682 flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ 683 dma_sync_single_for_device(dev, ctx->flc_dma[ENCRYPT], 684 sizeof(flc->flc) + desc_bytes(desc), 685 ctx->dir); 686 687 /* 688 * Job Descriptor and Shared Descriptors 689 * must all fit into the 64-word Descriptor h/w Buffer 690 */ 691 if (rem_bytes >= DESC_QI_GCM_DEC_LEN) { 692 ctx->cdata.key_inline = true; 693 ctx->cdata.key_virt = ctx->key; 694 } else { 695 ctx->cdata.key_inline = false; 696 ctx->cdata.key_dma = ctx->key_dma; 697 } 698 699 flc = &ctx->flc[DECRYPT]; 700 desc = flc->sh_desc; 701 cnstr_shdsc_gcm_decap(desc, &ctx->cdata, ivsize, ctx->authsize, true); 702 flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ 703 dma_sync_single_for_device(dev, ctx->flc_dma[DECRYPT], 704 sizeof(flc->flc) + desc_bytes(desc), 705 ctx->dir); 706 707 return 0; 708 } 709 710 static int gcm_setauthsize(struct crypto_aead *authenc, unsigned int authsize) 711 { 712 struct caam_ctx *ctx = crypto_aead_ctx_dma(authenc); 713 int err; 714 715 err = crypto_gcm_check_authsize(authsize); 716 if (err) 717 return err; 718 719 ctx->authsize = authsize; 720 gcm_set_sh_desc(authenc); 721 722 return 0; 723 } 724 725 static int gcm_setkey(struct crypto_aead *aead, 726 const u8 *key, unsigned int keylen) 727 { 728 struct caam_ctx *ctx = crypto_aead_ctx_dma(aead); 729 struct device *dev = ctx->dev; 730 int ret; 731 732 ret = aes_check_keylen(keylen); 733 if (ret) 734 return ret; 735 print_hex_dump_devel("key in @" __stringify(__LINE__)": ", 736 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); 737 738 memcpy(ctx->key, key, keylen); 739 dma_sync_single_for_device(dev, ctx->key_dma, keylen, ctx->dir); 740 ctx->cdata.keylen = keylen; 741 742 return gcm_set_sh_desc(aead); 743 } 744 745 static int rfc4106_set_sh_desc(struct crypto_aead *aead) 746 { 747 struct caam_ctx *ctx = crypto_aead_ctx_dma(aead); 748 struct device *dev = ctx->dev; 749 unsigned int ivsize = crypto_aead_ivsize(aead); 750 struct caam_flc *flc; 751 u32 *desc; 752 int rem_bytes = CAAM_DESC_BYTES_MAX - DESC_JOB_IO_LEN - 753 ctx->cdata.keylen; 754 755 if (!ctx->cdata.keylen || !ctx->authsize) 756 return 0; 757 758 ctx->cdata.key_virt = ctx->key; 759 760 /* 761 * RFC4106 encrypt shared descriptor 762 * Job Descriptor and Shared Descriptor 763 * must fit into the 64-word Descriptor h/w Buffer 764 */ 765 if (rem_bytes >= DESC_QI_RFC4106_ENC_LEN) { 766 ctx->cdata.key_inline = true; 767 } else { 768 ctx->cdata.key_inline = false; 769 ctx->cdata.key_dma = ctx->key_dma; 770 } 771 772 flc = &ctx->flc[ENCRYPT]; 773 desc = flc->sh_desc; 774 cnstr_shdsc_rfc4106_encap(desc, &ctx->cdata, ivsize, ctx->authsize, 775 true); 776 flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ 777 dma_sync_single_for_device(dev, ctx->flc_dma[ENCRYPT], 778 sizeof(flc->flc) + desc_bytes(desc), 779 ctx->dir); 780 781 /* 782 * Job Descriptor and Shared Descriptors 783 * must all fit into the 64-word Descriptor h/w Buffer 784 */ 785 if (rem_bytes >= DESC_QI_RFC4106_DEC_LEN) { 786 ctx->cdata.key_inline = true; 787 } else { 788 ctx->cdata.key_inline = false; 789 ctx->cdata.key_dma = ctx->key_dma; 790 } 791 792 flc = &ctx->flc[DECRYPT]; 793 desc = flc->sh_desc; 794 cnstr_shdsc_rfc4106_decap(desc, &ctx->cdata, ivsize, ctx->authsize, 795 true); 796 flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ 797 dma_sync_single_for_device(dev, ctx->flc_dma[DECRYPT], 798 sizeof(flc->flc) + desc_bytes(desc), 799 ctx->dir); 800 801 return 0; 802 } 803 804 static int rfc4106_setauthsize(struct crypto_aead *authenc, 805 unsigned int authsize) 806 { 807 struct caam_ctx *ctx = crypto_aead_ctx_dma(authenc); 808 int err; 809 810 err = crypto_rfc4106_check_authsize(authsize); 811 if (err) 812 return err; 813 814 ctx->authsize = authsize; 815 rfc4106_set_sh_desc(authenc); 816 817 return 0; 818 } 819 820 static int rfc4106_setkey(struct crypto_aead *aead, 821 const u8 *key, unsigned int keylen) 822 { 823 struct caam_ctx *ctx = crypto_aead_ctx_dma(aead); 824 struct device *dev = ctx->dev; 825 int ret; 826 827 ret = aes_check_keylen(keylen - 4); 828 if (ret) 829 return ret; 830 831 print_hex_dump_devel("key in @" __stringify(__LINE__)": ", 832 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); 833 834 memcpy(ctx->key, key, keylen); 835 /* 836 * The last four bytes of the key material are used as the salt value 837 * in the nonce. Update the AES key length. 838 */ 839 ctx->cdata.keylen = keylen - 4; 840 dma_sync_single_for_device(dev, ctx->key_dma, ctx->cdata.keylen, 841 ctx->dir); 842 843 return rfc4106_set_sh_desc(aead); 844 } 845 846 static int rfc4543_set_sh_desc(struct crypto_aead *aead) 847 { 848 struct caam_ctx *ctx = crypto_aead_ctx_dma(aead); 849 struct device *dev = ctx->dev; 850 unsigned int ivsize = crypto_aead_ivsize(aead); 851 struct caam_flc *flc; 852 u32 *desc; 853 int rem_bytes = CAAM_DESC_BYTES_MAX - DESC_JOB_IO_LEN - 854 ctx->cdata.keylen; 855 856 if (!ctx->cdata.keylen || !ctx->authsize) 857 return 0; 858 859 ctx->cdata.key_virt = ctx->key; 860 861 /* 862 * RFC4543 encrypt shared descriptor 863 * Job Descriptor and Shared Descriptor 864 * must fit into the 64-word Descriptor h/w Buffer 865 */ 866 if (rem_bytes >= DESC_QI_RFC4543_ENC_LEN) { 867 ctx->cdata.key_inline = true; 868 } else { 869 ctx->cdata.key_inline = false; 870 ctx->cdata.key_dma = ctx->key_dma; 871 } 872 873 flc = &ctx->flc[ENCRYPT]; 874 desc = flc->sh_desc; 875 cnstr_shdsc_rfc4543_encap(desc, &ctx->cdata, ivsize, ctx->authsize, 876 true); 877 flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ 878 dma_sync_single_for_device(dev, ctx->flc_dma[ENCRYPT], 879 sizeof(flc->flc) + desc_bytes(desc), 880 ctx->dir); 881 882 /* 883 * Job Descriptor and Shared Descriptors 884 * must all fit into the 64-word Descriptor h/w Buffer 885 */ 886 if (rem_bytes >= DESC_QI_RFC4543_DEC_LEN) { 887 ctx->cdata.key_inline = true; 888 } else { 889 ctx->cdata.key_inline = false; 890 ctx->cdata.key_dma = ctx->key_dma; 891 } 892 893 flc = &ctx->flc[DECRYPT]; 894 desc = flc->sh_desc; 895 cnstr_shdsc_rfc4543_decap(desc, &ctx->cdata, ivsize, ctx->authsize, 896 true); 897 flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ 898 dma_sync_single_for_device(dev, ctx->flc_dma[DECRYPT], 899 sizeof(flc->flc) + desc_bytes(desc), 900 ctx->dir); 901 902 return 0; 903 } 904 905 static int rfc4543_setauthsize(struct crypto_aead *authenc, 906 unsigned int authsize) 907 { 908 struct caam_ctx *ctx = crypto_aead_ctx_dma(authenc); 909 910 if (authsize != 16) 911 return -EINVAL; 912 913 ctx->authsize = authsize; 914 rfc4543_set_sh_desc(authenc); 915 916 return 0; 917 } 918 919 static int rfc4543_setkey(struct crypto_aead *aead, 920 const u8 *key, unsigned int keylen) 921 { 922 struct caam_ctx *ctx = crypto_aead_ctx_dma(aead); 923 struct device *dev = ctx->dev; 924 int ret; 925 926 ret = aes_check_keylen(keylen - 4); 927 if (ret) 928 return ret; 929 930 print_hex_dump_devel("key in @" __stringify(__LINE__)": ", 931 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); 932 933 memcpy(ctx->key, key, keylen); 934 /* 935 * The last four bytes of the key material are used as the salt value 936 * in the nonce. Update the AES key length. 937 */ 938 ctx->cdata.keylen = keylen - 4; 939 dma_sync_single_for_device(dev, ctx->key_dma, ctx->cdata.keylen, 940 ctx->dir); 941 942 return rfc4543_set_sh_desc(aead); 943 } 944 945 static int skcipher_setkey(struct crypto_skcipher *skcipher, const u8 *key, 946 unsigned int keylen, const u32 ctx1_iv_off) 947 { 948 struct caam_ctx *ctx = crypto_skcipher_ctx_dma(skcipher); 949 struct caam_skcipher_alg *alg = 950 container_of(crypto_skcipher_alg(skcipher), 951 struct caam_skcipher_alg, skcipher); 952 struct device *dev = ctx->dev; 953 struct caam_flc *flc; 954 unsigned int ivsize = crypto_skcipher_ivsize(skcipher); 955 u32 *desc; 956 const bool is_rfc3686 = alg->caam.rfc3686; 957 958 print_hex_dump_devel("key in @" __stringify(__LINE__)": ", 959 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); 960 961 ctx->cdata.keylen = keylen; 962 ctx->cdata.key_virt = key; 963 ctx->cdata.key_inline = true; 964 ctx->cdata.plain_keylen = keylen; 965 ctx->cdata.key_cmd_opt = 0; 966 967 /* skcipher_encrypt shared descriptor */ 968 flc = &ctx->flc[ENCRYPT]; 969 desc = flc->sh_desc; 970 cnstr_shdsc_skcipher_encap(desc, &ctx->cdata, ivsize, is_rfc3686, 971 ctx1_iv_off); 972 flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ 973 dma_sync_single_for_device(dev, ctx->flc_dma[ENCRYPT], 974 sizeof(flc->flc) + desc_bytes(desc), 975 ctx->dir); 976 977 /* skcipher_decrypt shared descriptor */ 978 flc = &ctx->flc[DECRYPT]; 979 desc = flc->sh_desc; 980 cnstr_shdsc_skcipher_decap(desc, &ctx->cdata, ivsize, is_rfc3686, 981 ctx1_iv_off); 982 flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ 983 dma_sync_single_for_device(dev, ctx->flc_dma[DECRYPT], 984 sizeof(flc->flc) + desc_bytes(desc), 985 ctx->dir); 986 987 return 0; 988 } 989 990 static int aes_skcipher_setkey(struct crypto_skcipher *skcipher, 991 const u8 *key, unsigned int keylen) 992 { 993 int err; 994 995 err = aes_check_keylen(keylen); 996 if (err) 997 return err; 998 999 return skcipher_setkey(skcipher, key, keylen, 0); 1000 } 1001 1002 static int rfc3686_skcipher_setkey(struct crypto_skcipher *skcipher, 1003 const u8 *key, unsigned int keylen) 1004 { 1005 u32 ctx1_iv_off; 1006 int err; 1007 1008 /* 1009 * RFC3686 specific: 1010 * | CONTEXT1[255:128] = {NONCE, IV, COUNTER} 1011 * | *key = {KEY, NONCE} 1012 */ 1013 ctx1_iv_off = 16 + CTR_RFC3686_NONCE_SIZE; 1014 keylen -= CTR_RFC3686_NONCE_SIZE; 1015 1016 err = aes_check_keylen(keylen); 1017 if (err) 1018 return err; 1019 1020 return skcipher_setkey(skcipher, key, keylen, ctx1_iv_off); 1021 } 1022 1023 static int ctr_skcipher_setkey(struct crypto_skcipher *skcipher, 1024 const u8 *key, unsigned int keylen) 1025 { 1026 u32 ctx1_iv_off; 1027 int err; 1028 1029 /* 1030 * AES-CTR needs to load IV in CONTEXT1 reg 1031 * at an offset of 128bits (16bytes) 1032 * CONTEXT1[255:128] = IV 1033 */ 1034 ctx1_iv_off = 16; 1035 1036 err = aes_check_keylen(keylen); 1037 if (err) 1038 return err; 1039 1040 return skcipher_setkey(skcipher, key, keylen, ctx1_iv_off); 1041 } 1042 1043 static int chacha20_skcipher_setkey(struct crypto_skcipher *skcipher, 1044 const u8 *key, unsigned int keylen) 1045 { 1046 if (keylen != CHACHA_KEY_SIZE) 1047 return -EINVAL; 1048 1049 return skcipher_setkey(skcipher, key, keylen, 0); 1050 } 1051 1052 static int des_skcipher_setkey(struct crypto_skcipher *skcipher, 1053 const u8 *key, unsigned int keylen) 1054 { 1055 return verify_skcipher_des_key(skcipher, key) ?: 1056 skcipher_setkey(skcipher, key, keylen, 0); 1057 } 1058 1059 static int des3_skcipher_setkey(struct crypto_skcipher *skcipher, 1060 const u8 *key, unsigned int keylen) 1061 { 1062 return verify_skcipher_des3_key(skcipher, key) ?: 1063 skcipher_setkey(skcipher, key, keylen, 0); 1064 } 1065 1066 static int xts_skcipher_setkey(struct crypto_skcipher *skcipher, const u8 *key, 1067 unsigned int keylen) 1068 { 1069 struct caam_ctx *ctx = crypto_skcipher_ctx_dma(skcipher); 1070 struct device *dev = ctx->dev; 1071 struct dpaa2_caam_priv *priv = dev_get_drvdata(dev); 1072 struct caam_flc *flc; 1073 u32 *desc; 1074 int err; 1075 1076 err = xts_verify_key(skcipher, key, keylen); 1077 if (err) { 1078 dev_dbg(dev, "key size mismatch\n"); 1079 return err; 1080 } 1081 1082 if (keylen != 2 * AES_KEYSIZE_128 && keylen != 2 * AES_KEYSIZE_256) 1083 ctx->xts_key_fallback = true; 1084 1085 if (priv->sec_attr.era <= 8 || ctx->xts_key_fallback) { 1086 err = crypto_skcipher_setkey(ctx->fallback, key, keylen); 1087 if (err) 1088 return err; 1089 } 1090 1091 ctx->cdata.keylen = keylen; 1092 ctx->cdata.key_virt = key; 1093 ctx->cdata.key_inline = true; 1094 1095 /* xts_skcipher_encrypt shared descriptor */ 1096 flc = &ctx->flc[ENCRYPT]; 1097 desc = flc->sh_desc; 1098 cnstr_shdsc_xts_skcipher_encap(desc, &ctx->cdata); 1099 flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ 1100 dma_sync_single_for_device(dev, ctx->flc_dma[ENCRYPT], 1101 sizeof(flc->flc) + desc_bytes(desc), 1102 ctx->dir); 1103 1104 /* xts_skcipher_decrypt shared descriptor */ 1105 flc = &ctx->flc[DECRYPT]; 1106 desc = flc->sh_desc; 1107 cnstr_shdsc_xts_skcipher_decap(desc, &ctx->cdata); 1108 flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ 1109 dma_sync_single_for_device(dev, ctx->flc_dma[DECRYPT], 1110 sizeof(flc->flc) + desc_bytes(desc), 1111 ctx->dir); 1112 1113 return 0; 1114 } 1115 1116 static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req) 1117 { 1118 struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); 1119 struct caam_request *req_ctx = skcipher_request_ctx_dma(req); 1120 struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1]; 1121 struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0]; 1122 struct caam_ctx *ctx = crypto_skcipher_ctx_dma(skcipher); 1123 struct device *dev = ctx->dev; 1124 gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? 1125 GFP_KERNEL : GFP_ATOMIC; 1126 int src_nents, mapped_src_nents, dst_nents = 0, mapped_dst_nents = 0; 1127 struct skcipher_edesc *edesc; 1128 dma_addr_t iv_dma; 1129 u8 *iv; 1130 int ivsize = crypto_skcipher_ivsize(skcipher); 1131 int dst_sg_idx, qm_sg_ents, qm_sg_bytes; 1132 struct dpaa2_sg_entry *sg_table; 1133 1134 src_nents = sg_nents_for_len(req->src, req->cryptlen); 1135 if (unlikely(src_nents < 0)) { 1136 dev_err(dev, "Insufficient bytes (%d) in src S/G\n", 1137 req->cryptlen); 1138 return ERR_PTR(src_nents); 1139 } 1140 1141 if (unlikely(req->dst != req->src)) { 1142 dst_nents = sg_nents_for_len(req->dst, req->cryptlen); 1143 if (unlikely(dst_nents < 0)) { 1144 dev_err(dev, "Insufficient bytes (%d) in dst S/G\n", 1145 req->cryptlen); 1146 return ERR_PTR(dst_nents); 1147 } 1148 1149 mapped_src_nents = dma_map_sg(dev, req->src, src_nents, 1150 DMA_TO_DEVICE); 1151 if (unlikely(!mapped_src_nents)) { 1152 dev_err(dev, "unable to map source\n"); 1153 return ERR_PTR(-ENOMEM); 1154 } 1155 1156 mapped_dst_nents = dma_map_sg(dev, req->dst, dst_nents, 1157 DMA_FROM_DEVICE); 1158 if (unlikely(!mapped_dst_nents)) { 1159 dev_err(dev, "unable to map destination\n"); 1160 dma_unmap_sg(dev, req->src, src_nents, DMA_TO_DEVICE); 1161 return ERR_PTR(-ENOMEM); 1162 } 1163 } else { 1164 mapped_src_nents = dma_map_sg(dev, req->src, src_nents, 1165 DMA_BIDIRECTIONAL); 1166 if (unlikely(!mapped_src_nents)) { 1167 dev_err(dev, "unable to map source\n"); 1168 return ERR_PTR(-ENOMEM); 1169 } 1170 } 1171 1172 qm_sg_ents = 1 + mapped_src_nents; 1173 dst_sg_idx = qm_sg_ents; 1174 1175 /* 1176 * Input, output HW S/G tables: [IV, src][dst, IV] 1177 * IV entries point to the same buffer 1178 * If src == dst, S/G entries are reused (S/G tables overlap) 1179 * 1180 * HW reads 4 S/G entries at a time; make sure the reads don't go beyond 1181 * the end of the table by allocating more S/G entries. 1182 */ 1183 if (req->src != req->dst) 1184 qm_sg_ents += pad_sg_nents(mapped_dst_nents + 1); 1185 else 1186 qm_sg_ents = 1 + pad_sg_nents(qm_sg_ents); 1187 1188 qm_sg_bytes = qm_sg_ents * sizeof(struct dpaa2_sg_entry); 1189 if (unlikely(offsetof(struct skcipher_edesc, sgt) + qm_sg_bytes + 1190 ivsize > CAAM_QI_MEMCACHE_SIZE)) { 1191 dev_err(dev, "No space for %d S/G entries and/or %dB IV\n", 1192 qm_sg_ents, ivsize); 1193 caam_unmap(dev, req->src, req->dst, src_nents, dst_nents, 0, 1194 0, DMA_NONE, 0, 0); 1195 return ERR_PTR(-ENOMEM); 1196 } 1197 1198 /* allocate space for base edesc, link tables and IV */ 1199 edesc = qi_cache_zalloc(flags); 1200 if (unlikely(!edesc)) { 1201 dev_err(dev, "could not allocate extended descriptor\n"); 1202 caam_unmap(dev, req->src, req->dst, src_nents, dst_nents, 0, 1203 0, DMA_NONE, 0, 0); 1204 return ERR_PTR(-ENOMEM); 1205 } 1206 1207 /* Make sure IV is located in a DMAable area */ 1208 sg_table = &edesc->sgt[0]; 1209 iv = (u8 *)(sg_table + qm_sg_ents); 1210 memcpy(iv, req->iv, ivsize); 1211 1212 iv_dma = dma_map_single(dev, iv, ivsize, DMA_BIDIRECTIONAL); 1213 if (dma_mapping_error(dev, iv_dma)) { 1214 dev_err(dev, "unable to map IV\n"); 1215 caam_unmap(dev, req->src, req->dst, src_nents, dst_nents, 0, 1216 0, DMA_NONE, 0, 0); 1217 qi_cache_free(edesc); 1218 return ERR_PTR(-ENOMEM); 1219 } 1220 1221 edesc->src_nents = src_nents; 1222 edesc->dst_nents = dst_nents; 1223 edesc->iv_dma = iv_dma; 1224 edesc->qm_sg_bytes = qm_sg_bytes; 1225 1226 dma_to_qm_sg_one(sg_table, iv_dma, ivsize, 0); 1227 sg_to_qm_sg(req->src, req->cryptlen, sg_table + 1, 0); 1228 1229 if (req->src != req->dst) 1230 sg_to_qm_sg(req->dst, req->cryptlen, sg_table + dst_sg_idx, 0); 1231 1232 dma_to_qm_sg_one(sg_table + dst_sg_idx + mapped_dst_nents, iv_dma, 1233 ivsize, 0); 1234 1235 edesc->qm_sg_dma = dma_map_single(dev, sg_table, edesc->qm_sg_bytes, 1236 DMA_TO_DEVICE); 1237 if (dma_mapping_error(dev, edesc->qm_sg_dma)) { 1238 dev_err(dev, "unable to map S/G table\n"); 1239 caam_unmap(dev, req->src, req->dst, src_nents, dst_nents, 1240 iv_dma, ivsize, DMA_BIDIRECTIONAL, 0, 0); 1241 qi_cache_free(edesc); 1242 return ERR_PTR(-ENOMEM); 1243 } 1244 1245 memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt)); 1246 dpaa2_fl_set_final(in_fle, true); 1247 dpaa2_fl_set_len(in_fle, req->cryptlen + ivsize); 1248 dpaa2_fl_set_len(out_fle, req->cryptlen + ivsize); 1249 1250 dpaa2_fl_set_format(in_fle, dpaa2_fl_sg); 1251 dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma); 1252 1253 dpaa2_fl_set_format(out_fle, dpaa2_fl_sg); 1254 1255 if (req->src == req->dst) 1256 dpaa2_fl_set_addr(out_fle, edesc->qm_sg_dma + 1257 sizeof(*sg_table)); 1258 else 1259 dpaa2_fl_set_addr(out_fle, edesc->qm_sg_dma + dst_sg_idx * 1260 sizeof(*sg_table)); 1261 1262 return edesc; 1263 } 1264 1265 static void aead_unmap(struct device *dev, struct aead_edesc *edesc, 1266 struct aead_request *req) 1267 { 1268 struct crypto_aead *aead = crypto_aead_reqtfm(req); 1269 int ivsize = crypto_aead_ivsize(aead); 1270 1271 caam_unmap(dev, req->src, req->dst, edesc->src_nents, edesc->dst_nents, 1272 edesc->iv_dma, ivsize, DMA_TO_DEVICE, edesc->qm_sg_dma, 1273 edesc->qm_sg_bytes); 1274 dma_unmap_single(dev, edesc->assoclen_dma, 4, DMA_TO_DEVICE); 1275 } 1276 1277 static void skcipher_unmap(struct device *dev, struct skcipher_edesc *edesc, 1278 struct skcipher_request *req) 1279 { 1280 struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); 1281 int ivsize = crypto_skcipher_ivsize(skcipher); 1282 1283 caam_unmap(dev, req->src, req->dst, edesc->src_nents, edesc->dst_nents, 1284 edesc->iv_dma, ivsize, DMA_BIDIRECTIONAL, edesc->qm_sg_dma, 1285 edesc->qm_sg_bytes); 1286 } 1287 1288 static void aead_encrypt_done(void *cbk_ctx, u32 status) 1289 { 1290 struct crypto_async_request *areq = cbk_ctx; 1291 struct aead_request *req = container_of(areq, struct aead_request, 1292 base); 1293 struct caam_request *req_ctx = to_caam_req(areq); 1294 struct aead_edesc *edesc = req_ctx->edesc; 1295 struct crypto_aead *aead = crypto_aead_reqtfm(req); 1296 struct caam_ctx *ctx = crypto_aead_ctx_dma(aead); 1297 int ecode = 0; 1298 1299 dev_dbg(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status); 1300 1301 if (unlikely(status)) 1302 ecode = caam_qi2_strstatus(ctx->dev, status); 1303 1304 aead_unmap(ctx->dev, edesc, req); 1305 qi_cache_free(edesc); 1306 aead_request_complete(req, ecode); 1307 } 1308 1309 static void aead_decrypt_done(void *cbk_ctx, u32 status) 1310 { 1311 struct crypto_async_request *areq = cbk_ctx; 1312 struct aead_request *req = container_of(areq, struct aead_request, 1313 base); 1314 struct caam_request *req_ctx = to_caam_req(areq); 1315 struct aead_edesc *edesc = req_ctx->edesc; 1316 struct crypto_aead *aead = crypto_aead_reqtfm(req); 1317 struct caam_ctx *ctx = crypto_aead_ctx_dma(aead); 1318 int ecode = 0; 1319 1320 dev_dbg(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status); 1321 1322 if (unlikely(status)) 1323 ecode = caam_qi2_strstatus(ctx->dev, status); 1324 1325 aead_unmap(ctx->dev, edesc, req); 1326 qi_cache_free(edesc); 1327 aead_request_complete(req, ecode); 1328 } 1329 1330 static int aead_encrypt(struct aead_request *req) 1331 { 1332 struct aead_edesc *edesc; 1333 struct crypto_aead *aead = crypto_aead_reqtfm(req); 1334 struct caam_ctx *ctx = crypto_aead_ctx_dma(aead); 1335 struct caam_request *caam_req = aead_request_ctx_dma(req); 1336 int ret; 1337 1338 /* allocate extended descriptor */ 1339 edesc = aead_edesc_alloc(req, true); 1340 if (IS_ERR(edesc)) 1341 return PTR_ERR(edesc); 1342 1343 caam_req->flc = &ctx->flc[ENCRYPT]; 1344 caam_req->flc_dma = ctx->flc_dma[ENCRYPT]; 1345 caam_req->cbk = aead_encrypt_done; 1346 caam_req->ctx = &req->base; 1347 caam_req->edesc = edesc; 1348 ret = dpaa2_caam_enqueue(ctx->dev, caam_req); 1349 if (ret != -EINPROGRESS && 1350 !(ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) { 1351 aead_unmap(ctx->dev, edesc, req); 1352 qi_cache_free(edesc); 1353 } 1354 1355 return ret; 1356 } 1357 1358 static int aead_decrypt(struct aead_request *req) 1359 { 1360 struct aead_edesc *edesc; 1361 struct crypto_aead *aead = crypto_aead_reqtfm(req); 1362 struct caam_ctx *ctx = crypto_aead_ctx_dma(aead); 1363 struct caam_request *caam_req = aead_request_ctx_dma(req); 1364 int ret; 1365 1366 /* allocate extended descriptor */ 1367 edesc = aead_edesc_alloc(req, false); 1368 if (IS_ERR(edesc)) 1369 return PTR_ERR(edesc); 1370 1371 caam_req->flc = &ctx->flc[DECRYPT]; 1372 caam_req->flc_dma = ctx->flc_dma[DECRYPT]; 1373 caam_req->cbk = aead_decrypt_done; 1374 caam_req->ctx = &req->base; 1375 caam_req->edesc = edesc; 1376 ret = dpaa2_caam_enqueue(ctx->dev, caam_req); 1377 if (ret != -EINPROGRESS && 1378 !(ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) { 1379 aead_unmap(ctx->dev, edesc, req); 1380 qi_cache_free(edesc); 1381 } 1382 1383 return ret; 1384 } 1385 1386 static int ipsec_gcm_encrypt(struct aead_request *req) 1387 { 1388 return crypto_ipsec_check_assoclen(req->assoclen) ? : aead_encrypt(req); 1389 } 1390 1391 static int ipsec_gcm_decrypt(struct aead_request *req) 1392 { 1393 return crypto_ipsec_check_assoclen(req->assoclen) ? : aead_decrypt(req); 1394 } 1395 1396 static void skcipher_encrypt_done(void *cbk_ctx, u32 status) 1397 { 1398 struct crypto_async_request *areq = cbk_ctx; 1399 struct skcipher_request *req = skcipher_request_cast(areq); 1400 struct caam_request *req_ctx = to_caam_req(areq); 1401 struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); 1402 struct caam_ctx *ctx = crypto_skcipher_ctx_dma(skcipher); 1403 struct skcipher_edesc *edesc = req_ctx->edesc; 1404 int ecode = 0; 1405 int ivsize = crypto_skcipher_ivsize(skcipher); 1406 1407 dev_dbg(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status); 1408 1409 if (unlikely(status)) 1410 ecode = caam_qi2_strstatus(ctx->dev, status); 1411 1412 print_hex_dump_debug("dstiv @" __stringify(__LINE__)": ", 1413 DUMP_PREFIX_ADDRESS, 16, 4, req->iv, 1414 edesc->src_nents > 1 ? 100 : ivsize, 1); 1415 caam_dump_sg("dst @" __stringify(__LINE__)": ", 1416 DUMP_PREFIX_ADDRESS, 16, 4, req->dst, 1417 edesc->dst_nents > 1 ? 100 : req->cryptlen, 1); 1418 1419 skcipher_unmap(ctx->dev, edesc, req); 1420 1421 /* 1422 * The crypto API expects us to set the IV (req->iv) to the last 1423 * ciphertext block (CBC mode) or last counter (CTR mode). 1424 * This is used e.g. by the CTS mode. 1425 */ 1426 if (!ecode) 1427 memcpy(req->iv, (u8 *)&edesc->sgt[0] + edesc->qm_sg_bytes, 1428 ivsize); 1429 1430 qi_cache_free(edesc); 1431 skcipher_request_complete(req, ecode); 1432 } 1433 1434 static void skcipher_decrypt_done(void *cbk_ctx, u32 status) 1435 { 1436 struct crypto_async_request *areq = cbk_ctx; 1437 struct skcipher_request *req = skcipher_request_cast(areq); 1438 struct caam_request *req_ctx = to_caam_req(areq); 1439 struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); 1440 struct caam_ctx *ctx = crypto_skcipher_ctx_dma(skcipher); 1441 struct skcipher_edesc *edesc = req_ctx->edesc; 1442 int ecode = 0; 1443 int ivsize = crypto_skcipher_ivsize(skcipher); 1444 1445 dev_dbg(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status); 1446 1447 if (unlikely(status)) 1448 ecode = caam_qi2_strstatus(ctx->dev, status); 1449 1450 print_hex_dump_debug("dstiv @" __stringify(__LINE__)": ", 1451 DUMP_PREFIX_ADDRESS, 16, 4, req->iv, 1452 edesc->src_nents > 1 ? 100 : ivsize, 1); 1453 caam_dump_sg("dst @" __stringify(__LINE__)": ", 1454 DUMP_PREFIX_ADDRESS, 16, 4, req->dst, 1455 edesc->dst_nents > 1 ? 100 : req->cryptlen, 1); 1456 1457 skcipher_unmap(ctx->dev, edesc, req); 1458 1459 /* 1460 * The crypto API expects us to set the IV (req->iv) to the last 1461 * ciphertext block (CBC mode) or last counter (CTR mode). 1462 * This is used e.g. by the CTS mode. 1463 */ 1464 if (!ecode) 1465 memcpy(req->iv, (u8 *)&edesc->sgt[0] + edesc->qm_sg_bytes, 1466 ivsize); 1467 1468 qi_cache_free(edesc); 1469 skcipher_request_complete(req, ecode); 1470 } 1471 1472 static inline bool xts_skcipher_ivsize(struct skcipher_request *req) 1473 { 1474 struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); 1475 unsigned int ivsize = crypto_skcipher_ivsize(skcipher); 1476 1477 return !!get_unaligned((u64 *)(req->iv + (ivsize / 2))); 1478 } 1479 1480 static int skcipher_encrypt(struct skcipher_request *req) 1481 { 1482 struct skcipher_edesc *edesc; 1483 struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); 1484 struct caam_ctx *ctx = crypto_skcipher_ctx_dma(skcipher); 1485 struct caam_request *caam_req = skcipher_request_ctx_dma(req); 1486 struct dpaa2_caam_priv *priv = dev_get_drvdata(ctx->dev); 1487 int ret; 1488 1489 /* 1490 * XTS is expected to return an error even for input length = 0 1491 * Note that the case input length < block size will be caught during 1492 * HW offloading and return an error. 1493 */ 1494 if (!req->cryptlen && !ctx->fallback) 1495 return 0; 1496 1497 if (ctx->fallback && ((priv->sec_attr.era <= 8 && xts_skcipher_ivsize(req)) || 1498 ctx->xts_key_fallback)) { 1499 skcipher_request_set_tfm(&caam_req->fallback_req, ctx->fallback); 1500 skcipher_request_set_callback(&caam_req->fallback_req, 1501 req->base.flags, 1502 req->base.complete, 1503 req->base.data); 1504 skcipher_request_set_crypt(&caam_req->fallback_req, req->src, 1505 req->dst, req->cryptlen, req->iv); 1506 1507 return crypto_skcipher_encrypt(&caam_req->fallback_req); 1508 } 1509 1510 /* allocate extended descriptor */ 1511 edesc = skcipher_edesc_alloc(req); 1512 if (IS_ERR(edesc)) 1513 return PTR_ERR(edesc); 1514 1515 caam_req->flc = &ctx->flc[ENCRYPT]; 1516 caam_req->flc_dma = ctx->flc_dma[ENCRYPT]; 1517 caam_req->cbk = skcipher_encrypt_done; 1518 caam_req->ctx = &req->base; 1519 caam_req->edesc = edesc; 1520 ret = dpaa2_caam_enqueue(ctx->dev, caam_req); 1521 if (ret != -EINPROGRESS && 1522 !(ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) { 1523 skcipher_unmap(ctx->dev, edesc, req); 1524 qi_cache_free(edesc); 1525 } 1526 1527 return ret; 1528 } 1529 1530 static int skcipher_decrypt(struct skcipher_request *req) 1531 { 1532 struct skcipher_edesc *edesc; 1533 struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); 1534 struct caam_ctx *ctx = crypto_skcipher_ctx_dma(skcipher); 1535 struct caam_request *caam_req = skcipher_request_ctx_dma(req); 1536 struct dpaa2_caam_priv *priv = dev_get_drvdata(ctx->dev); 1537 int ret; 1538 1539 /* 1540 * XTS is expected to return an error even for input length = 0 1541 * Note that the case input length < block size will be caught during 1542 * HW offloading and return an error. 1543 */ 1544 if (!req->cryptlen && !ctx->fallback) 1545 return 0; 1546 1547 if (ctx->fallback && ((priv->sec_attr.era <= 8 && xts_skcipher_ivsize(req)) || 1548 ctx->xts_key_fallback)) { 1549 skcipher_request_set_tfm(&caam_req->fallback_req, ctx->fallback); 1550 skcipher_request_set_callback(&caam_req->fallback_req, 1551 req->base.flags, 1552 req->base.complete, 1553 req->base.data); 1554 skcipher_request_set_crypt(&caam_req->fallback_req, req->src, 1555 req->dst, req->cryptlen, req->iv); 1556 1557 return crypto_skcipher_decrypt(&caam_req->fallback_req); 1558 } 1559 1560 /* allocate extended descriptor */ 1561 edesc = skcipher_edesc_alloc(req); 1562 if (IS_ERR(edesc)) 1563 return PTR_ERR(edesc); 1564 1565 caam_req->flc = &ctx->flc[DECRYPT]; 1566 caam_req->flc_dma = ctx->flc_dma[DECRYPT]; 1567 caam_req->cbk = skcipher_decrypt_done; 1568 caam_req->ctx = &req->base; 1569 caam_req->edesc = edesc; 1570 ret = dpaa2_caam_enqueue(ctx->dev, caam_req); 1571 if (ret != -EINPROGRESS && 1572 !(ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) { 1573 skcipher_unmap(ctx->dev, edesc, req); 1574 qi_cache_free(edesc); 1575 } 1576 1577 return ret; 1578 } 1579 1580 static int caam_cra_init(struct caam_ctx *ctx, struct caam_alg_entry *caam, 1581 bool uses_dkp) 1582 { 1583 dma_addr_t dma_addr; 1584 int i; 1585 1586 /* copy descriptor header template value */ 1587 ctx->cdata.algtype = OP_TYPE_CLASS1_ALG | caam->class1_alg_type; 1588 ctx->adata.algtype = OP_TYPE_CLASS2_ALG | caam->class2_alg_type; 1589 1590 ctx->dev = caam->dev; 1591 ctx->dir = uses_dkp ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE; 1592 1593 dma_addr = dma_map_single_attrs(ctx->dev, ctx->flc, 1594 offsetof(struct caam_ctx, flc_dma), 1595 ctx->dir, DMA_ATTR_SKIP_CPU_SYNC); 1596 if (dma_mapping_error(ctx->dev, dma_addr)) { 1597 dev_err(ctx->dev, "unable to map key, shared descriptors\n"); 1598 return -ENOMEM; 1599 } 1600 1601 for (i = 0; i < NUM_OP; i++) 1602 ctx->flc_dma[i] = dma_addr + i * sizeof(ctx->flc[i]); 1603 ctx->key_dma = dma_addr + NUM_OP * sizeof(ctx->flc[0]); 1604 1605 return 0; 1606 } 1607 1608 static int caam_cra_init_skcipher(struct crypto_skcipher *tfm) 1609 { 1610 struct skcipher_alg *alg = crypto_skcipher_alg(tfm); 1611 struct caam_skcipher_alg *caam_alg = 1612 container_of(alg, typeof(*caam_alg), skcipher); 1613 struct caam_ctx *ctx = crypto_skcipher_ctx_dma(tfm); 1614 u32 alg_aai = caam_alg->caam.class1_alg_type & OP_ALG_AAI_MASK; 1615 int ret = 0; 1616 1617 if (alg_aai == OP_ALG_AAI_XTS) { 1618 const char *tfm_name = crypto_tfm_alg_name(&tfm->base); 1619 struct crypto_skcipher *fallback; 1620 1621 fallback = crypto_alloc_skcipher(tfm_name, 0, 1622 CRYPTO_ALG_NEED_FALLBACK); 1623 if (IS_ERR(fallback)) { 1624 dev_err(caam_alg->caam.dev, 1625 "Failed to allocate %s fallback: %ld\n", 1626 tfm_name, PTR_ERR(fallback)); 1627 return PTR_ERR(fallback); 1628 } 1629 1630 ctx->fallback = fallback; 1631 crypto_skcipher_set_reqsize_dma( 1632 tfm, sizeof(struct caam_request) + 1633 crypto_skcipher_reqsize(fallback)); 1634 } else { 1635 crypto_skcipher_set_reqsize_dma(tfm, 1636 sizeof(struct caam_request)); 1637 } 1638 1639 ret = caam_cra_init(ctx, &caam_alg->caam, false); 1640 if (ret && ctx->fallback) 1641 crypto_free_skcipher(ctx->fallback); 1642 1643 return ret; 1644 } 1645 1646 static int caam_cra_init_aead(struct crypto_aead *tfm) 1647 { 1648 struct aead_alg *alg = crypto_aead_alg(tfm); 1649 struct caam_aead_alg *caam_alg = container_of(alg, typeof(*caam_alg), 1650 aead); 1651 1652 crypto_aead_set_reqsize_dma(tfm, sizeof(struct caam_request)); 1653 return caam_cra_init(crypto_aead_ctx_dma(tfm), &caam_alg->caam, 1654 !caam_alg->caam.nodkp); 1655 } 1656 1657 static void caam_exit_common(struct caam_ctx *ctx) 1658 { 1659 dma_unmap_single_attrs(ctx->dev, ctx->flc_dma[0], 1660 offsetof(struct caam_ctx, flc_dma), ctx->dir, 1661 DMA_ATTR_SKIP_CPU_SYNC); 1662 } 1663 1664 static void caam_cra_exit(struct crypto_skcipher *tfm) 1665 { 1666 struct caam_ctx *ctx = crypto_skcipher_ctx_dma(tfm); 1667 1668 if (ctx->fallback) 1669 crypto_free_skcipher(ctx->fallback); 1670 caam_exit_common(ctx); 1671 } 1672 1673 static void caam_cra_exit_aead(struct crypto_aead *tfm) 1674 { 1675 caam_exit_common(crypto_aead_ctx_dma(tfm)); 1676 } 1677 1678 static struct caam_skcipher_alg driver_algs[] = { 1679 { 1680 .skcipher = { 1681 .base = { 1682 .cra_name = "cbc(aes)", 1683 .cra_driver_name = "cbc-aes-caam-qi2", 1684 .cra_blocksize = AES_BLOCK_SIZE, 1685 }, 1686 .setkey = aes_skcipher_setkey, 1687 .encrypt = skcipher_encrypt, 1688 .decrypt = skcipher_decrypt, 1689 .min_keysize = AES_MIN_KEY_SIZE, 1690 .max_keysize = AES_MAX_KEY_SIZE, 1691 .ivsize = AES_BLOCK_SIZE, 1692 }, 1693 .caam.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1694 }, 1695 { 1696 .skcipher = { 1697 .base = { 1698 .cra_name = "cbc(des3_ede)", 1699 .cra_driver_name = "cbc-3des-caam-qi2", 1700 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 1701 }, 1702 .setkey = des3_skcipher_setkey, 1703 .encrypt = skcipher_encrypt, 1704 .decrypt = skcipher_decrypt, 1705 .min_keysize = DES3_EDE_KEY_SIZE, 1706 .max_keysize = DES3_EDE_KEY_SIZE, 1707 .ivsize = DES3_EDE_BLOCK_SIZE, 1708 }, 1709 .caam.class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 1710 }, 1711 { 1712 .skcipher = { 1713 .base = { 1714 .cra_name = "cbc(des)", 1715 .cra_driver_name = "cbc-des-caam-qi2", 1716 .cra_blocksize = DES_BLOCK_SIZE, 1717 }, 1718 .setkey = des_skcipher_setkey, 1719 .encrypt = skcipher_encrypt, 1720 .decrypt = skcipher_decrypt, 1721 .min_keysize = DES_KEY_SIZE, 1722 .max_keysize = DES_KEY_SIZE, 1723 .ivsize = DES_BLOCK_SIZE, 1724 }, 1725 .caam.class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 1726 }, 1727 { 1728 .skcipher = { 1729 .base = { 1730 .cra_name = "ctr(aes)", 1731 .cra_driver_name = "ctr-aes-caam-qi2", 1732 .cra_blocksize = 1, 1733 }, 1734 .setkey = ctr_skcipher_setkey, 1735 .encrypt = skcipher_encrypt, 1736 .decrypt = skcipher_decrypt, 1737 .min_keysize = AES_MIN_KEY_SIZE, 1738 .max_keysize = AES_MAX_KEY_SIZE, 1739 .ivsize = AES_BLOCK_SIZE, 1740 .chunksize = AES_BLOCK_SIZE, 1741 }, 1742 .caam.class1_alg_type = OP_ALG_ALGSEL_AES | 1743 OP_ALG_AAI_CTR_MOD128, 1744 }, 1745 { 1746 .skcipher = { 1747 .base = { 1748 .cra_name = "rfc3686(ctr(aes))", 1749 .cra_driver_name = "rfc3686-ctr-aes-caam-qi2", 1750 .cra_blocksize = 1, 1751 }, 1752 .setkey = rfc3686_skcipher_setkey, 1753 .encrypt = skcipher_encrypt, 1754 .decrypt = skcipher_decrypt, 1755 .min_keysize = AES_MIN_KEY_SIZE + 1756 CTR_RFC3686_NONCE_SIZE, 1757 .max_keysize = AES_MAX_KEY_SIZE + 1758 CTR_RFC3686_NONCE_SIZE, 1759 .ivsize = CTR_RFC3686_IV_SIZE, 1760 .chunksize = AES_BLOCK_SIZE, 1761 }, 1762 .caam = { 1763 .class1_alg_type = OP_ALG_ALGSEL_AES | 1764 OP_ALG_AAI_CTR_MOD128, 1765 .rfc3686 = true, 1766 }, 1767 }, 1768 { 1769 .skcipher = { 1770 .base = { 1771 .cra_name = "xts(aes)", 1772 .cra_driver_name = "xts-aes-caam-qi2", 1773 .cra_flags = CRYPTO_ALG_NEED_FALLBACK, 1774 .cra_blocksize = AES_BLOCK_SIZE, 1775 }, 1776 .setkey = xts_skcipher_setkey, 1777 .encrypt = skcipher_encrypt, 1778 .decrypt = skcipher_decrypt, 1779 .min_keysize = 2 * AES_MIN_KEY_SIZE, 1780 .max_keysize = 2 * AES_MAX_KEY_SIZE, 1781 .ivsize = AES_BLOCK_SIZE, 1782 }, 1783 .caam.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_XTS, 1784 }, 1785 { 1786 .skcipher = { 1787 .base = { 1788 .cra_name = "chacha20", 1789 .cra_driver_name = "chacha20-caam-qi2", 1790 .cra_blocksize = 1, 1791 }, 1792 .setkey = chacha20_skcipher_setkey, 1793 .encrypt = skcipher_encrypt, 1794 .decrypt = skcipher_decrypt, 1795 .min_keysize = CHACHA_KEY_SIZE, 1796 .max_keysize = CHACHA_KEY_SIZE, 1797 .ivsize = CHACHA_IV_SIZE, 1798 }, 1799 .caam.class1_alg_type = OP_ALG_ALGSEL_CHACHA20, 1800 }, 1801 }; 1802 1803 static struct caam_aead_alg driver_aeads[] = { 1804 { 1805 .aead = { 1806 .base = { 1807 .cra_name = "rfc4106(gcm(aes))", 1808 .cra_driver_name = "rfc4106-gcm-aes-caam-qi2", 1809 .cra_blocksize = 1, 1810 }, 1811 .setkey = rfc4106_setkey, 1812 .setauthsize = rfc4106_setauthsize, 1813 .encrypt = ipsec_gcm_encrypt, 1814 .decrypt = ipsec_gcm_decrypt, 1815 .ivsize = 8, 1816 .maxauthsize = AES_BLOCK_SIZE, 1817 }, 1818 .caam = { 1819 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM, 1820 .nodkp = true, 1821 }, 1822 }, 1823 { 1824 .aead = { 1825 .base = { 1826 .cra_name = "rfc4543(gcm(aes))", 1827 .cra_driver_name = "rfc4543-gcm-aes-caam-qi2", 1828 .cra_blocksize = 1, 1829 }, 1830 .setkey = rfc4543_setkey, 1831 .setauthsize = rfc4543_setauthsize, 1832 .encrypt = ipsec_gcm_encrypt, 1833 .decrypt = ipsec_gcm_decrypt, 1834 .ivsize = 8, 1835 .maxauthsize = AES_BLOCK_SIZE, 1836 }, 1837 .caam = { 1838 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM, 1839 .nodkp = true, 1840 }, 1841 }, 1842 /* Galois Counter Mode */ 1843 { 1844 .aead = { 1845 .base = { 1846 .cra_name = "gcm(aes)", 1847 .cra_driver_name = "gcm-aes-caam-qi2", 1848 .cra_blocksize = 1, 1849 }, 1850 .setkey = gcm_setkey, 1851 .setauthsize = gcm_setauthsize, 1852 .encrypt = aead_encrypt, 1853 .decrypt = aead_decrypt, 1854 .ivsize = 12, 1855 .maxauthsize = AES_BLOCK_SIZE, 1856 }, 1857 .caam = { 1858 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM, 1859 .nodkp = true, 1860 } 1861 }, 1862 /* single-pass ipsec_esp descriptor */ 1863 { 1864 .aead = { 1865 .base = { 1866 .cra_name = "authenc(hmac(md5),cbc(aes))", 1867 .cra_driver_name = "authenc-hmac-md5-" 1868 "cbc-aes-caam-qi2", 1869 .cra_blocksize = AES_BLOCK_SIZE, 1870 }, 1871 .setkey = aead_setkey, 1872 .setauthsize = aead_setauthsize, 1873 .encrypt = aead_encrypt, 1874 .decrypt = aead_decrypt, 1875 .ivsize = AES_BLOCK_SIZE, 1876 .maxauthsize = MD5_DIGEST_SIZE, 1877 }, 1878 .caam = { 1879 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1880 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 1881 OP_ALG_AAI_HMAC_PRECOMP, 1882 } 1883 }, 1884 { 1885 .aead = { 1886 .base = { 1887 .cra_name = "echainiv(authenc(hmac(md5)," 1888 "cbc(aes)))", 1889 .cra_driver_name = "echainiv-authenc-hmac-md5-" 1890 "cbc-aes-caam-qi2", 1891 .cra_blocksize = AES_BLOCK_SIZE, 1892 }, 1893 .setkey = aead_setkey, 1894 .setauthsize = aead_setauthsize, 1895 .encrypt = aead_encrypt, 1896 .decrypt = aead_decrypt, 1897 .ivsize = AES_BLOCK_SIZE, 1898 .maxauthsize = MD5_DIGEST_SIZE, 1899 }, 1900 .caam = { 1901 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1902 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 1903 OP_ALG_AAI_HMAC_PRECOMP, 1904 .geniv = true, 1905 } 1906 }, 1907 { 1908 .aead = { 1909 .base = { 1910 .cra_name = "authenc(hmac(sha1),cbc(aes))", 1911 .cra_driver_name = "authenc-hmac-sha1-" 1912 "cbc-aes-caam-qi2", 1913 .cra_blocksize = AES_BLOCK_SIZE, 1914 }, 1915 .setkey = aead_setkey, 1916 .setauthsize = aead_setauthsize, 1917 .encrypt = aead_encrypt, 1918 .decrypt = aead_decrypt, 1919 .ivsize = AES_BLOCK_SIZE, 1920 .maxauthsize = SHA1_DIGEST_SIZE, 1921 }, 1922 .caam = { 1923 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1924 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 1925 OP_ALG_AAI_HMAC_PRECOMP, 1926 } 1927 }, 1928 { 1929 .aead = { 1930 .base = { 1931 .cra_name = "echainiv(authenc(hmac(sha1)," 1932 "cbc(aes)))", 1933 .cra_driver_name = "echainiv-authenc-" 1934 "hmac-sha1-cbc-aes-caam-qi2", 1935 .cra_blocksize = AES_BLOCK_SIZE, 1936 }, 1937 .setkey = aead_setkey, 1938 .setauthsize = aead_setauthsize, 1939 .encrypt = aead_encrypt, 1940 .decrypt = aead_decrypt, 1941 .ivsize = AES_BLOCK_SIZE, 1942 .maxauthsize = SHA1_DIGEST_SIZE, 1943 }, 1944 .caam = { 1945 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1946 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 1947 OP_ALG_AAI_HMAC_PRECOMP, 1948 .geniv = true, 1949 }, 1950 }, 1951 { 1952 .aead = { 1953 .base = { 1954 .cra_name = "authenc(hmac(sha224),cbc(aes))", 1955 .cra_driver_name = "authenc-hmac-sha224-" 1956 "cbc-aes-caam-qi2", 1957 .cra_blocksize = AES_BLOCK_SIZE, 1958 }, 1959 .setkey = aead_setkey, 1960 .setauthsize = aead_setauthsize, 1961 .encrypt = aead_encrypt, 1962 .decrypt = aead_decrypt, 1963 .ivsize = AES_BLOCK_SIZE, 1964 .maxauthsize = SHA224_DIGEST_SIZE, 1965 }, 1966 .caam = { 1967 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1968 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 1969 OP_ALG_AAI_HMAC_PRECOMP, 1970 } 1971 }, 1972 { 1973 .aead = { 1974 .base = { 1975 .cra_name = "echainiv(authenc(hmac(sha224)," 1976 "cbc(aes)))", 1977 .cra_driver_name = "echainiv-authenc-" 1978 "hmac-sha224-cbc-aes-caam-qi2", 1979 .cra_blocksize = AES_BLOCK_SIZE, 1980 }, 1981 .setkey = aead_setkey, 1982 .setauthsize = aead_setauthsize, 1983 .encrypt = aead_encrypt, 1984 .decrypt = aead_decrypt, 1985 .ivsize = AES_BLOCK_SIZE, 1986 .maxauthsize = SHA224_DIGEST_SIZE, 1987 }, 1988 .caam = { 1989 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 1990 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 1991 OP_ALG_AAI_HMAC_PRECOMP, 1992 .geniv = true, 1993 } 1994 }, 1995 { 1996 .aead = { 1997 .base = { 1998 .cra_name = "authenc(hmac(sha256),cbc(aes))", 1999 .cra_driver_name = "authenc-hmac-sha256-" 2000 "cbc-aes-caam-qi2", 2001 .cra_blocksize = AES_BLOCK_SIZE, 2002 }, 2003 .setkey = aead_setkey, 2004 .setauthsize = aead_setauthsize, 2005 .encrypt = aead_encrypt, 2006 .decrypt = aead_decrypt, 2007 .ivsize = AES_BLOCK_SIZE, 2008 .maxauthsize = SHA256_DIGEST_SIZE, 2009 }, 2010 .caam = { 2011 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 2012 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 2013 OP_ALG_AAI_HMAC_PRECOMP, 2014 } 2015 }, 2016 { 2017 .aead = { 2018 .base = { 2019 .cra_name = "echainiv(authenc(hmac(sha256)," 2020 "cbc(aes)))", 2021 .cra_driver_name = "echainiv-authenc-" 2022 "hmac-sha256-cbc-aes-" 2023 "caam-qi2", 2024 .cra_blocksize = AES_BLOCK_SIZE, 2025 }, 2026 .setkey = aead_setkey, 2027 .setauthsize = aead_setauthsize, 2028 .encrypt = aead_encrypt, 2029 .decrypt = aead_decrypt, 2030 .ivsize = AES_BLOCK_SIZE, 2031 .maxauthsize = SHA256_DIGEST_SIZE, 2032 }, 2033 .caam = { 2034 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 2035 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 2036 OP_ALG_AAI_HMAC_PRECOMP, 2037 .geniv = true, 2038 } 2039 }, 2040 { 2041 .aead = { 2042 .base = { 2043 .cra_name = "authenc(hmac(sha384),cbc(aes))", 2044 .cra_driver_name = "authenc-hmac-sha384-" 2045 "cbc-aes-caam-qi2", 2046 .cra_blocksize = AES_BLOCK_SIZE, 2047 }, 2048 .setkey = aead_setkey, 2049 .setauthsize = aead_setauthsize, 2050 .encrypt = aead_encrypt, 2051 .decrypt = aead_decrypt, 2052 .ivsize = AES_BLOCK_SIZE, 2053 .maxauthsize = SHA384_DIGEST_SIZE, 2054 }, 2055 .caam = { 2056 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 2057 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 2058 OP_ALG_AAI_HMAC_PRECOMP, 2059 } 2060 }, 2061 { 2062 .aead = { 2063 .base = { 2064 .cra_name = "echainiv(authenc(hmac(sha384)," 2065 "cbc(aes)))", 2066 .cra_driver_name = "echainiv-authenc-" 2067 "hmac-sha384-cbc-aes-" 2068 "caam-qi2", 2069 .cra_blocksize = AES_BLOCK_SIZE, 2070 }, 2071 .setkey = aead_setkey, 2072 .setauthsize = aead_setauthsize, 2073 .encrypt = aead_encrypt, 2074 .decrypt = aead_decrypt, 2075 .ivsize = AES_BLOCK_SIZE, 2076 .maxauthsize = SHA384_DIGEST_SIZE, 2077 }, 2078 .caam = { 2079 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 2080 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 2081 OP_ALG_AAI_HMAC_PRECOMP, 2082 .geniv = true, 2083 } 2084 }, 2085 { 2086 .aead = { 2087 .base = { 2088 .cra_name = "authenc(hmac(sha512),cbc(aes))", 2089 .cra_driver_name = "authenc-hmac-sha512-" 2090 "cbc-aes-caam-qi2", 2091 .cra_blocksize = AES_BLOCK_SIZE, 2092 }, 2093 .setkey = aead_setkey, 2094 .setauthsize = aead_setauthsize, 2095 .encrypt = aead_encrypt, 2096 .decrypt = aead_decrypt, 2097 .ivsize = AES_BLOCK_SIZE, 2098 .maxauthsize = SHA512_DIGEST_SIZE, 2099 }, 2100 .caam = { 2101 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 2102 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 2103 OP_ALG_AAI_HMAC_PRECOMP, 2104 } 2105 }, 2106 { 2107 .aead = { 2108 .base = { 2109 .cra_name = "echainiv(authenc(hmac(sha512)," 2110 "cbc(aes)))", 2111 .cra_driver_name = "echainiv-authenc-" 2112 "hmac-sha512-cbc-aes-" 2113 "caam-qi2", 2114 .cra_blocksize = AES_BLOCK_SIZE, 2115 }, 2116 .setkey = aead_setkey, 2117 .setauthsize = aead_setauthsize, 2118 .encrypt = aead_encrypt, 2119 .decrypt = aead_decrypt, 2120 .ivsize = AES_BLOCK_SIZE, 2121 .maxauthsize = SHA512_DIGEST_SIZE, 2122 }, 2123 .caam = { 2124 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, 2125 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 2126 OP_ALG_AAI_HMAC_PRECOMP, 2127 .geniv = true, 2128 } 2129 }, 2130 { 2131 .aead = { 2132 .base = { 2133 .cra_name = "authenc(hmac(md5),cbc(des3_ede))", 2134 .cra_driver_name = "authenc-hmac-md5-" 2135 "cbc-des3_ede-caam-qi2", 2136 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2137 }, 2138 .setkey = des3_aead_setkey, 2139 .setauthsize = aead_setauthsize, 2140 .encrypt = aead_encrypt, 2141 .decrypt = aead_decrypt, 2142 .ivsize = DES3_EDE_BLOCK_SIZE, 2143 .maxauthsize = MD5_DIGEST_SIZE, 2144 }, 2145 .caam = { 2146 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2147 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 2148 OP_ALG_AAI_HMAC_PRECOMP, 2149 } 2150 }, 2151 { 2152 .aead = { 2153 .base = { 2154 .cra_name = "echainiv(authenc(hmac(md5)," 2155 "cbc(des3_ede)))", 2156 .cra_driver_name = "echainiv-authenc-hmac-md5-" 2157 "cbc-des3_ede-caam-qi2", 2158 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2159 }, 2160 .setkey = des3_aead_setkey, 2161 .setauthsize = aead_setauthsize, 2162 .encrypt = aead_encrypt, 2163 .decrypt = aead_decrypt, 2164 .ivsize = DES3_EDE_BLOCK_SIZE, 2165 .maxauthsize = MD5_DIGEST_SIZE, 2166 }, 2167 .caam = { 2168 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2169 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 2170 OP_ALG_AAI_HMAC_PRECOMP, 2171 .geniv = true, 2172 } 2173 }, 2174 { 2175 .aead = { 2176 .base = { 2177 .cra_name = "authenc(hmac(sha1)," 2178 "cbc(des3_ede))", 2179 .cra_driver_name = "authenc-hmac-sha1-" 2180 "cbc-des3_ede-caam-qi2", 2181 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2182 }, 2183 .setkey = des3_aead_setkey, 2184 .setauthsize = aead_setauthsize, 2185 .encrypt = aead_encrypt, 2186 .decrypt = aead_decrypt, 2187 .ivsize = DES3_EDE_BLOCK_SIZE, 2188 .maxauthsize = SHA1_DIGEST_SIZE, 2189 }, 2190 .caam = { 2191 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2192 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 2193 OP_ALG_AAI_HMAC_PRECOMP, 2194 }, 2195 }, 2196 { 2197 .aead = { 2198 .base = { 2199 .cra_name = "echainiv(authenc(hmac(sha1)," 2200 "cbc(des3_ede)))", 2201 .cra_driver_name = "echainiv-authenc-" 2202 "hmac-sha1-" 2203 "cbc-des3_ede-caam-qi2", 2204 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2205 }, 2206 .setkey = des3_aead_setkey, 2207 .setauthsize = aead_setauthsize, 2208 .encrypt = aead_encrypt, 2209 .decrypt = aead_decrypt, 2210 .ivsize = DES3_EDE_BLOCK_SIZE, 2211 .maxauthsize = SHA1_DIGEST_SIZE, 2212 }, 2213 .caam = { 2214 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2215 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 2216 OP_ALG_AAI_HMAC_PRECOMP, 2217 .geniv = true, 2218 } 2219 }, 2220 { 2221 .aead = { 2222 .base = { 2223 .cra_name = "authenc(hmac(sha224)," 2224 "cbc(des3_ede))", 2225 .cra_driver_name = "authenc-hmac-sha224-" 2226 "cbc-des3_ede-caam-qi2", 2227 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2228 }, 2229 .setkey = des3_aead_setkey, 2230 .setauthsize = aead_setauthsize, 2231 .encrypt = aead_encrypt, 2232 .decrypt = aead_decrypt, 2233 .ivsize = DES3_EDE_BLOCK_SIZE, 2234 .maxauthsize = SHA224_DIGEST_SIZE, 2235 }, 2236 .caam = { 2237 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2238 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 2239 OP_ALG_AAI_HMAC_PRECOMP, 2240 }, 2241 }, 2242 { 2243 .aead = { 2244 .base = { 2245 .cra_name = "echainiv(authenc(hmac(sha224)," 2246 "cbc(des3_ede)))", 2247 .cra_driver_name = "echainiv-authenc-" 2248 "hmac-sha224-" 2249 "cbc-des3_ede-caam-qi2", 2250 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2251 }, 2252 .setkey = des3_aead_setkey, 2253 .setauthsize = aead_setauthsize, 2254 .encrypt = aead_encrypt, 2255 .decrypt = aead_decrypt, 2256 .ivsize = DES3_EDE_BLOCK_SIZE, 2257 .maxauthsize = SHA224_DIGEST_SIZE, 2258 }, 2259 .caam = { 2260 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2261 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 2262 OP_ALG_AAI_HMAC_PRECOMP, 2263 .geniv = true, 2264 } 2265 }, 2266 { 2267 .aead = { 2268 .base = { 2269 .cra_name = "authenc(hmac(sha256)," 2270 "cbc(des3_ede))", 2271 .cra_driver_name = "authenc-hmac-sha256-" 2272 "cbc-des3_ede-caam-qi2", 2273 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2274 }, 2275 .setkey = des3_aead_setkey, 2276 .setauthsize = aead_setauthsize, 2277 .encrypt = aead_encrypt, 2278 .decrypt = aead_decrypt, 2279 .ivsize = DES3_EDE_BLOCK_SIZE, 2280 .maxauthsize = SHA256_DIGEST_SIZE, 2281 }, 2282 .caam = { 2283 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2284 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 2285 OP_ALG_AAI_HMAC_PRECOMP, 2286 }, 2287 }, 2288 { 2289 .aead = { 2290 .base = { 2291 .cra_name = "echainiv(authenc(hmac(sha256)," 2292 "cbc(des3_ede)))", 2293 .cra_driver_name = "echainiv-authenc-" 2294 "hmac-sha256-" 2295 "cbc-des3_ede-caam-qi2", 2296 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2297 }, 2298 .setkey = des3_aead_setkey, 2299 .setauthsize = aead_setauthsize, 2300 .encrypt = aead_encrypt, 2301 .decrypt = aead_decrypt, 2302 .ivsize = DES3_EDE_BLOCK_SIZE, 2303 .maxauthsize = SHA256_DIGEST_SIZE, 2304 }, 2305 .caam = { 2306 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2307 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 2308 OP_ALG_AAI_HMAC_PRECOMP, 2309 .geniv = true, 2310 } 2311 }, 2312 { 2313 .aead = { 2314 .base = { 2315 .cra_name = "authenc(hmac(sha384)," 2316 "cbc(des3_ede))", 2317 .cra_driver_name = "authenc-hmac-sha384-" 2318 "cbc-des3_ede-caam-qi2", 2319 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2320 }, 2321 .setkey = des3_aead_setkey, 2322 .setauthsize = aead_setauthsize, 2323 .encrypt = aead_encrypt, 2324 .decrypt = aead_decrypt, 2325 .ivsize = DES3_EDE_BLOCK_SIZE, 2326 .maxauthsize = SHA384_DIGEST_SIZE, 2327 }, 2328 .caam = { 2329 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2330 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 2331 OP_ALG_AAI_HMAC_PRECOMP, 2332 }, 2333 }, 2334 { 2335 .aead = { 2336 .base = { 2337 .cra_name = "echainiv(authenc(hmac(sha384)," 2338 "cbc(des3_ede)))", 2339 .cra_driver_name = "echainiv-authenc-" 2340 "hmac-sha384-" 2341 "cbc-des3_ede-caam-qi2", 2342 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2343 }, 2344 .setkey = des3_aead_setkey, 2345 .setauthsize = aead_setauthsize, 2346 .encrypt = aead_encrypt, 2347 .decrypt = aead_decrypt, 2348 .ivsize = DES3_EDE_BLOCK_SIZE, 2349 .maxauthsize = SHA384_DIGEST_SIZE, 2350 }, 2351 .caam = { 2352 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2353 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 2354 OP_ALG_AAI_HMAC_PRECOMP, 2355 .geniv = true, 2356 } 2357 }, 2358 { 2359 .aead = { 2360 .base = { 2361 .cra_name = "authenc(hmac(sha512)," 2362 "cbc(des3_ede))", 2363 .cra_driver_name = "authenc-hmac-sha512-" 2364 "cbc-des3_ede-caam-qi2", 2365 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2366 }, 2367 .setkey = des3_aead_setkey, 2368 .setauthsize = aead_setauthsize, 2369 .encrypt = aead_encrypt, 2370 .decrypt = aead_decrypt, 2371 .ivsize = DES3_EDE_BLOCK_SIZE, 2372 .maxauthsize = SHA512_DIGEST_SIZE, 2373 }, 2374 .caam = { 2375 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2376 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 2377 OP_ALG_AAI_HMAC_PRECOMP, 2378 }, 2379 }, 2380 { 2381 .aead = { 2382 .base = { 2383 .cra_name = "echainiv(authenc(hmac(sha512)," 2384 "cbc(des3_ede)))", 2385 .cra_driver_name = "echainiv-authenc-" 2386 "hmac-sha512-" 2387 "cbc-des3_ede-caam-qi2", 2388 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2389 }, 2390 .setkey = des3_aead_setkey, 2391 .setauthsize = aead_setauthsize, 2392 .encrypt = aead_encrypt, 2393 .decrypt = aead_decrypt, 2394 .ivsize = DES3_EDE_BLOCK_SIZE, 2395 .maxauthsize = SHA512_DIGEST_SIZE, 2396 }, 2397 .caam = { 2398 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, 2399 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 2400 OP_ALG_AAI_HMAC_PRECOMP, 2401 .geniv = true, 2402 } 2403 }, 2404 { 2405 .aead = { 2406 .base = { 2407 .cra_name = "authenc(hmac(md5),cbc(des))", 2408 .cra_driver_name = "authenc-hmac-md5-" 2409 "cbc-des-caam-qi2", 2410 .cra_blocksize = DES_BLOCK_SIZE, 2411 }, 2412 .setkey = aead_setkey, 2413 .setauthsize = aead_setauthsize, 2414 .encrypt = aead_encrypt, 2415 .decrypt = aead_decrypt, 2416 .ivsize = DES_BLOCK_SIZE, 2417 .maxauthsize = MD5_DIGEST_SIZE, 2418 }, 2419 .caam = { 2420 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2421 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 2422 OP_ALG_AAI_HMAC_PRECOMP, 2423 }, 2424 }, 2425 { 2426 .aead = { 2427 .base = { 2428 .cra_name = "echainiv(authenc(hmac(md5)," 2429 "cbc(des)))", 2430 .cra_driver_name = "echainiv-authenc-hmac-md5-" 2431 "cbc-des-caam-qi2", 2432 .cra_blocksize = DES_BLOCK_SIZE, 2433 }, 2434 .setkey = aead_setkey, 2435 .setauthsize = aead_setauthsize, 2436 .encrypt = aead_encrypt, 2437 .decrypt = aead_decrypt, 2438 .ivsize = DES_BLOCK_SIZE, 2439 .maxauthsize = MD5_DIGEST_SIZE, 2440 }, 2441 .caam = { 2442 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2443 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 2444 OP_ALG_AAI_HMAC_PRECOMP, 2445 .geniv = true, 2446 } 2447 }, 2448 { 2449 .aead = { 2450 .base = { 2451 .cra_name = "authenc(hmac(sha1),cbc(des))", 2452 .cra_driver_name = "authenc-hmac-sha1-" 2453 "cbc-des-caam-qi2", 2454 .cra_blocksize = DES_BLOCK_SIZE, 2455 }, 2456 .setkey = aead_setkey, 2457 .setauthsize = aead_setauthsize, 2458 .encrypt = aead_encrypt, 2459 .decrypt = aead_decrypt, 2460 .ivsize = DES_BLOCK_SIZE, 2461 .maxauthsize = SHA1_DIGEST_SIZE, 2462 }, 2463 .caam = { 2464 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2465 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 2466 OP_ALG_AAI_HMAC_PRECOMP, 2467 }, 2468 }, 2469 { 2470 .aead = { 2471 .base = { 2472 .cra_name = "echainiv(authenc(hmac(sha1)," 2473 "cbc(des)))", 2474 .cra_driver_name = "echainiv-authenc-" 2475 "hmac-sha1-cbc-des-caam-qi2", 2476 .cra_blocksize = DES_BLOCK_SIZE, 2477 }, 2478 .setkey = aead_setkey, 2479 .setauthsize = aead_setauthsize, 2480 .encrypt = aead_encrypt, 2481 .decrypt = aead_decrypt, 2482 .ivsize = DES_BLOCK_SIZE, 2483 .maxauthsize = SHA1_DIGEST_SIZE, 2484 }, 2485 .caam = { 2486 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2487 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 2488 OP_ALG_AAI_HMAC_PRECOMP, 2489 .geniv = true, 2490 } 2491 }, 2492 { 2493 .aead = { 2494 .base = { 2495 .cra_name = "authenc(hmac(sha224),cbc(des))", 2496 .cra_driver_name = "authenc-hmac-sha224-" 2497 "cbc-des-caam-qi2", 2498 .cra_blocksize = DES_BLOCK_SIZE, 2499 }, 2500 .setkey = aead_setkey, 2501 .setauthsize = aead_setauthsize, 2502 .encrypt = aead_encrypt, 2503 .decrypt = aead_decrypt, 2504 .ivsize = DES_BLOCK_SIZE, 2505 .maxauthsize = SHA224_DIGEST_SIZE, 2506 }, 2507 .caam = { 2508 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2509 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 2510 OP_ALG_AAI_HMAC_PRECOMP, 2511 }, 2512 }, 2513 { 2514 .aead = { 2515 .base = { 2516 .cra_name = "echainiv(authenc(hmac(sha224)," 2517 "cbc(des)))", 2518 .cra_driver_name = "echainiv-authenc-" 2519 "hmac-sha224-cbc-des-" 2520 "caam-qi2", 2521 .cra_blocksize = DES_BLOCK_SIZE, 2522 }, 2523 .setkey = aead_setkey, 2524 .setauthsize = aead_setauthsize, 2525 .encrypt = aead_encrypt, 2526 .decrypt = aead_decrypt, 2527 .ivsize = DES_BLOCK_SIZE, 2528 .maxauthsize = SHA224_DIGEST_SIZE, 2529 }, 2530 .caam = { 2531 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2532 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 2533 OP_ALG_AAI_HMAC_PRECOMP, 2534 .geniv = true, 2535 } 2536 }, 2537 { 2538 .aead = { 2539 .base = { 2540 .cra_name = "authenc(hmac(sha256),cbc(des))", 2541 .cra_driver_name = "authenc-hmac-sha256-" 2542 "cbc-des-caam-qi2", 2543 .cra_blocksize = DES_BLOCK_SIZE, 2544 }, 2545 .setkey = aead_setkey, 2546 .setauthsize = aead_setauthsize, 2547 .encrypt = aead_encrypt, 2548 .decrypt = aead_decrypt, 2549 .ivsize = DES_BLOCK_SIZE, 2550 .maxauthsize = SHA256_DIGEST_SIZE, 2551 }, 2552 .caam = { 2553 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2554 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 2555 OP_ALG_AAI_HMAC_PRECOMP, 2556 }, 2557 }, 2558 { 2559 .aead = { 2560 .base = { 2561 .cra_name = "echainiv(authenc(hmac(sha256)," 2562 "cbc(des)))", 2563 .cra_driver_name = "echainiv-authenc-" 2564 "hmac-sha256-cbc-des-" 2565 "caam-qi2", 2566 .cra_blocksize = DES_BLOCK_SIZE, 2567 }, 2568 .setkey = aead_setkey, 2569 .setauthsize = aead_setauthsize, 2570 .encrypt = aead_encrypt, 2571 .decrypt = aead_decrypt, 2572 .ivsize = DES_BLOCK_SIZE, 2573 .maxauthsize = SHA256_DIGEST_SIZE, 2574 }, 2575 .caam = { 2576 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2577 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 2578 OP_ALG_AAI_HMAC_PRECOMP, 2579 .geniv = true, 2580 }, 2581 }, 2582 { 2583 .aead = { 2584 .base = { 2585 .cra_name = "authenc(hmac(sha384),cbc(des))", 2586 .cra_driver_name = "authenc-hmac-sha384-" 2587 "cbc-des-caam-qi2", 2588 .cra_blocksize = DES_BLOCK_SIZE, 2589 }, 2590 .setkey = aead_setkey, 2591 .setauthsize = aead_setauthsize, 2592 .encrypt = aead_encrypt, 2593 .decrypt = aead_decrypt, 2594 .ivsize = DES_BLOCK_SIZE, 2595 .maxauthsize = SHA384_DIGEST_SIZE, 2596 }, 2597 .caam = { 2598 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2599 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 2600 OP_ALG_AAI_HMAC_PRECOMP, 2601 }, 2602 }, 2603 { 2604 .aead = { 2605 .base = { 2606 .cra_name = "echainiv(authenc(hmac(sha384)," 2607 "cbc(des)))", 2608 .cra_driver_name = "echainiv-authenc-" 2609 "hmac-sha384-cbc-des-" 2610 "caam-qi2", 2611 .cra_blocksize = DES_BLOCK_SIZE, 2612 }, 2613 .setkey = aead_setkey, 2614 .setauthsize = aead_setauthsize, 2615 .encrypt = aead_encrypt, 2616 .decrypt = aead_decrypt, 2617 .ivsize = DES_BLOCK_SIZE, 2618 .maxauthsize = SHA384_DIGEST_SIZE, 2619 }, 2620 .caam = { 2621 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2622 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 2623 OP_ALG_AAI_HMAC_PRECOMP, 2624 .geniv = true, 2625 } 2626 }, 2627 { 2628 .aead = { 2629 .base = { 2630 .cra_name = "authenc(hmac(sha512),cbc(des))", 2631 .cra_driver_name = "authenc-hmac-sha512-" 2632 "cbc-des-caam-qi2", 2633 .cra_blocksize = DES_BLOCK_SIZE, 2634 }, 2635 .setkey = aead_setkey, 2636 .setauthsize = aead_setauthsize, 2637 .encrypt = aead_encrypt, 2638 .decrypt = aead_decrypt, 2639 .ivsize = DES_BLOCK_SIZE, 2640 .maxauthsize = SHA512_DIGEST_SIZE, 2641 }, 2642 .caam = { 2643 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2644 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 2645 OP_ALG_AAI_HMAC_PRECOMP, 2646 } 2647 }, 2648 { 2649 .aead = { 2650 .base = { 2651 .cra_name = "echainiv(authenc(hmac(sha512)," 2652 "cbc(des)))", 2653 .cra_driver_name = "echainiv-authenc-" 2654 "hmac-sha512-cbc-des-" 2655 "caam-qi2", 2656 .cra_blocksize = DES_BLOCK_SIZE, 2657 }, 2658 .setkey = aead_setkey, 2659 .setauthsize = aead_setauthsize, 2660 .encrypt = aead_encrypt, 2661 .decrypt = aead_decrypt, 2662 .ivsize = DES_BLOCK_SIZE, 2663 .maxauthsize = SHA512_DIGEST_SIZE, 2664 }, 2665 .caam = { 2666 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, 2667 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 2668 OP_ALG_AAI_HMAC_PRECOMP, 2669 .geniv = true, 2670 } 2671 }, 2672 { 2673 .aead = { 2674 .base = { 2675 .cra_name = "authenc(hmac(md5)," 2676 "rfc3686(ctr(aes)))", 2677 .cra_driver_name = "authenc-hmac-md5-" 2678 "rfc3686-ctr-aes-caam-qi2", 2679 .cra_blocksize = 1, 2680 }, 2681 .setkey = aead_setkey, 2682 .setauthsize = aead_setauthsize, 2683 .encrypt = aead_encrypt, 2684 .decrypt = aead_decrypt, 2685 .ivsize = CTR_RFC3686_IV_SIZE, 2686 .maxauthsize = MD5_DIGEST_SIZE, 2687 }, 2688 .caam = { 2689 .class1_alg_type = OP_ALG_ALGSEL_AES | 2690 OP_ALG_AAI_CTR_MOD128, 2691 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 2692 OP_ALG_AAI_HMAC_PRECOMP, 2693 .rfc3686 = true, 2694 }, 2695 }, 2696 { 2697 .aead = { 2698 .base = { 2699 .cra_name = "seqiv(authenc(" 2700 "hmac(md5),rfc3686(ctr(aes))))", 2701 .cra_driver_name = "seqiv-authenc-hmac-md5-" 2702 "rfc3686-ctr-aes-caam-qi2", 2703 .cra_blocksize = 1, 2704 }, 2705 .setkey = aead_setkey, 2706 .setauthsize = aead_setauthsize, 2707 .encrypt = aead_encrypt, 2708 .decrypt = aead_decrypt, 2709 .ivsize = CTR_RFC3686_IV_SIZE, 2710 .maxauthsize = MD5_DIGEST_SIZE, 2711 }, 2712 .caam = { 2713 .class1_alg_type = OP_ALG_ALGSEL_AES | 2714 OP_ALG_AAI_CTR_MOD128, 2715 .class2_alg_type = OP_ALG_ALGSEL_MD5 | 2716 OP_ALG_AAI_HMAC_PRECOMP, 2717 .rfc3686 = true, 2718 .geniv = true, 2719 }, 2720 }, 2721 { 2722 .aead = { 2723 .base = { 2724 .cra_name = "authenc(hmac(sha1)," 2725 "rfc3686(ctr(aes)))", 2726 .cra_driver_name = "authenc-hmac-sha1-" 2727 "rfc3686-ctr-aes-caam-qi2", 2728 .cra_blocksize = 1, 2729 }, 2730 .setkey = aead_setkey, 2731 .setauthsize = aead_setauthsize, 2732 .encrypt = aead_encrypt, 2733 .decrypt = aead_decrypt, 2734 .ivsize = CTR_RFC3686_IV_SIZE, 2735 .maxauthsize = SHA1_DIGEST_SIZE, 2736 }, 2737 .caam = { 2738 .class1_alg_type = OP_ALG_ALGSEL_AES | 2739 OP_ALG_AAI_CTR_MOD128, 2740 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 2741 OP_ALG_AAI_HMAC_PRECOMP, 2742 .rfc3686 = true, 2743 }, 2744 }, 2745 { 2746 .aead = { 2747 .base = { 2748 .cra_name = "seqiv(authenc(" 2749 "hmac(sha1),rfc3686(ctr(aes))))", 2750 .cra_driver_name = "seqiv-authenc-hmac-sha1-" 2751 "rfc3686-ctr-aes-caam-qi2", 2752 .cra_blocksize = 1, 2753 }, 2754 .setkey = aead_setkey, 2755 .setauthsize = aead_setauthsize, 2756 .encrypt = aead_encrypt, 2757 .decrypt = aead_decrypt, 2758 .ivsize = CTR_RFC3686_IV_SIZE, 2759 .maxauthsize = SHA1_DIGEST_SIZE, 2760 }, 2761 .caam = { 2762 .class1_alg_type = OP_ALG_ALGSEL_AES | 2763 OP_ALG_AAI_CTR_MOD128, 2764 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | 2765 OP_ALG_AAI_HMAC_PRECOMP, 2766 .rfc3686 = true, 2767 .geniv = true, 2768 }, 2769 }, 2770 { 2771 .aead = { 2772 .base = { 2773 .cra_name = "authenc(hmac(sha224)," 2774 "rfc3686(ctr(aes)))", 2775 .cra_driver_name = "authenc-hmac-sha224-" 2776 "rfc3686-ctr-aes-caam-qi2", 2777 .cra_blocksize = 1, 2778 }, 2779 .setkey = aead_setkey, 2780 .setauthsize = aead_setauthsize, 2781 .encrypt = aead_encrypt, 2782 .decrypt = aead_decrypt, 2783 .ivsize = CTR_RFC3686_IV_SIZE, 2784 .maxauthsize = SHA224_DIGEST_SIZE, 2785 }, 2786 .caam = { 2787 .class1_alg_type = OP_ALG_ALGSEL_AES | 2788 OP_ALG_AAI_CTR_MOD128, 2789 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 2790 OP_ALG_AAI_HMAC_PRECOMP, 2791 .rfc3686 = true, 2792 }, 2793 }, 2794 { 2795 .aead = { 2796 .base = { 2797 .cra_name = "seqiv(authenc(" 2798 "hmac(sha224),rfc3686(ctr(aes))))", 2799 .cra_driver_name = "seqiv-authenc-hmac-sha224-" 2800 "rfc3686-ctr-aes-caam-qi2", 2801 .cra_blocksize = 1, 2802 }, 2803 .setkey = aead_setkey, 2804 .setauthsize = aead_setauthsize, 2805 .encrypt = aead_encrypt, 2806 .decrypt = aead_decrypt, 2807 .ivsize = CTR_RFC3686_IV_SIZE, 2808 .maxauthsize = SHA224_DIGEST_SIZE, 2809 }, 2810 .caam = { 2811 .class1_alg_type = OP_ALG_ALGSEL_AES | 2812 OP_ALG_AAI_CTR_MOD128, 2813 .class2_alg_type = OP_ALG_ALGSEL_SHA224 | 2814 OP_ALG_AAI_HMAC_PRECOMP, 2815 .rfc3686 = true, 2816 .geniv = true, 2817 }, 2818 }, 2819 { 2820 .aead = { 2821 .base = { 2822 .cra_name = "authenc(hmac(sha256)," 2823 "rfc3686(ctr(aes)))", 2824 .cra_driver_name = "authenc-hmac-sha256-" 2825 "rfc3686-ctr-aes-caam-qi2", 2826 .cra_blocksize = 1, 2827 }, 2828 .setkey = aead_setkey, 2829 .setauthsize = aead_setauthsize, 2830 .encrypt = aead_encrypt, 2831 .decrypt = aead_decrypt, 2832 .ivsize = CTR_RFC3686_IV_SIZE, 2833 .maxauthsize = SHA256_DIGEST_SIZE, 2834 }, 2835 .caam = { 2836 .class1_alg_type = OP_ALG_ALGSEL_AES | 2837 OP_ALG_AAI_CTR_MOD128, 2838 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 2839 OP_ALG_AAI_HMAC_PRECOMP, 2840 .rfc3686 = true, 2841 }, 2842 }, 2843 { 2844 .aead = { 2845 .base = { 2846 .cra_name = "seqiv(authenc(hmac(sha256)," 2847 "rfc3686(ctr(aes))))", 2848 .cra_driver_name = "seqiv-authenc-hmac-sha256-" 2849 "rfc3686-ctr-aes-caam-qi2", 2850 .cra_blocksize = 1, 2851 }, 2852 .setkey = aead_setkey, 2853 .setauthsize = aead_setauthsize, 2854 .encrypt = aead_encrypt, 2855 .decrypt = aead_decrypt, 2856 .ivsize = CTR_RFC3686_IV_SIZE, 2857 .maxauthsize = SHA256_DIGEST_SIZE, 2858 }, 2859 .caam = { 2860 .class1_alg_type = OP_ALG_ALGSEL_AES | 2861 OP_ALG_AAI_CTR_MOD128, 2862 .class2_alg_type = OP_ALG_ALGSEL_SHA256 | 2863 OP_ALG_AAI_HMAC_PRECOMP, 2864 .rfc3686 = true, 2865 .geniv = true, 2866 }, 2867 }, 2868 { 2869 .aead = { 2870 .base = { 2871 .cra_name = "authenc(hmac(sha384)," 2872 "rfc3686(ctr(aes)))", 2873 .cra_driver_name = "authenc-hmac-sha384-" 2874 "rfc3686-ctr-aes-caam-qi2", 2875 .cra_blocksize = 1, 2876 }, 2877 .setkey = aead_setkey, 2878 .setauthsize = aead_setauthsize, 2879 .encrypt = aead_encrypt, 2880 .decrypt = aead_decrypt, 2881 .ivsize = CTR_RFC3686_IV_SIZE, 2882 .maxauthsize = SHA384_DIGEST_SIZE, 2883 }, 2884 .caam = { 2885 .class1_alg_type = OP_ALG_ALGSEL_AES | 2886 OP_ALG_AAI_CTR_MOD128, 2887 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 2888 OP_ALG_AAI_HMAC_PRECOMP, 2889 .rfc3686 = true, 2890 }, 2891 }, 2892 { 2893 .aead = { 2894 .base = { 2895 .cra_name = "seqiv(authenc(hmac(sha384)," 2896 "rfc3686(ctr(aes))))", 2897 .cra_driver_name = "seqiv-authenc-hmac-sha384-" 2898 "rfc3686-ctr-aes-caam-qi2", 2899 .cra_blocksize = 1, 2900 }, 2901 .setkey = aead_setkey, 2902 .setauthsize = aead_setauthsize, 2903 .encrypt = aead_encrypt, 2904 .decrypt = aead_decrypt, 2905 .ivsize = CTR_RFC3686_IV_SIZE, 2906 .maxauthsize = SHA384_DIGEST_SIZE, 2907 }, 2908 .caam = { 2909 .class1_alg_type = OP_ALG_ALGSEL_AES | 2910 OP_ALG_AAI_CTR_MOD128, 2911 .class2_alg_type = OP_ALG_ALGSEL_SHA384 | 2912 OP_ALG_AAI_HMAC_PRECOMP, 2913 .rfc3686 = true, 2914 .geniv = true, 2915 }, 2916 }, 2917 { 2918 .aead = { 2919 .base = { 2920 .cra_name = "rfc7539(chacha20,poly1305)", 2921 .cra_driver_name = "rfc7539-chacha20-poly1305-" 2922 "caam-qi2", 2923 .cra_blocksize = 1, 2924 }, 2925 .setkey = chachapoly_setkey, 2926 .setauthsize = chachapoly_setauthsize, 2927 .encrypt = aead_encrypt, 2928 .decrypt = aead_decrypt, 2929 .ivsize = CHACHAPOLY_IV_SIZE, 2930 .maxauthsize = POLY1305_DIGEST_SIZE, 2931 }, 2932 .caam = { 2933 .class1_alg_type = OP_ALG_ALGSEL_CHACHA20 | 2934 OP_ALG_AAI_AEAD, 2935 .class2_alg_type = OP_ALG_ALGSEL_POLY1305 | 2936 OP_ALG_AAI_AEAD, 2937 .nodkp = true, 2938 }, 2939 }, 2940 { 2941 .aead = { 2942 .base = { 2943 .cra_name = "rfc7539esp(chacha20,poly1305)", 2944 .cra_driver_name = "rfc7539esp-chacha20-" 2945 "poly1305-caam-qi2", 2946 .cra_blocksize = 1, 2947 }, 2948 .setkey = chachapoly_setkey, 2949 .setauthsize = chachapoly_setauthsize, 2950 .encrypt = aead_encrypt, 2951 .decrypt = aead_decrypt, 2952 .ivsize = 8, 2953 .maxauthsize = POLY1305_DIGEST_SIZE, 2954 }, 2955 .caam = { 2956 .class1_alg_type = OP_ALG_ALGSEL_CHACHA20 | 2957 OP_ALG_AAI_AEAD, 2958 .class2_alg_type = OP_ALG_ALGSEL_POLY1305 | 2959 OP_ALG_AAI_AEAD, 2960 .nodkp = true, 2961 }, 2962 }, 2963 { 2964 .aead = { 2965 .base = { 2966 .cra_name = "authenc(hmac(sha512)," 2967 "rfc3686(ctr(aes)))", 2968 .cra_driver_name = "authenc-hmac-sha512-" 2969 "rfc3686-ctr-aes-caam-qi2", 2970 .cra_blocksize = 1, 2971 }, 2972 .setkey = aead_setkey, 2973 .setauthsize = aead_setauthsize, 2974 .encrypt = aead_encrypt, 2975 .decrypt = aead_decrypt, 2976 .ivsize = CTR_RFC3686_IV_SIZE, 2977 .maxauthsize = SHA512_DIGEST_SIZE, 2978 }, 2979 .caam = { 2980 .class1_alg_type = OP_ALG_ALGSEL_AES | 2981 OP_ALG_AAI_CTR_MOD128, 2982 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 2983 OP_ALG_AAI_HMAC_PRECOMP, 2984 .rfc3686 = true, 2985 }, 2986 }, 2987 { 2988 .aead = { 2989 .base = { 2990 .cra_name = "seqiv(authenc(hmac(sha512)," 2991 "rfc3686(ctr(aes))))", 2992 .cra_driver_name = "seqiv-authenc-hmac-sha512-" 2993 "rfc3686-ctr-aes-caam-qi2", 2994 .cra_blocksize = 1, 2995 }, 2996 .setkey = aead_setkey, 2997 .setauthsize = aead_setauthsize, 2998 .encrypt = aead_encrypt, 2999 .decrypt = aead_decrypt, 3000 .ivsize = CTR_RFC3686_IV_SIZE, 3001 .maxauthsize = SHA512_DIGEST_SIZE, 3002 }, 3003 .caam = { 3004 .class1_alg_type = OP_ALG_ALGSEL_AES | 3005 OP_ALG_AAI_CTR_MOD128, 3006 .class2_alg_type = OP_ALG_ALGSEL_SHA512 | 3007 OP_ALG_AAI_HMAC_PRECOMP, 3008 .rfc3686 = true, 3009 .geniv = true, 3010 }, 3011 }, 3012 }; 3013 3014 static void caam_skcipher_alg_init(struct caam_skcipher_alg *t_alg) 3015 { 3016 struct skcipher_alg *alg = &t_alg->skcipher; 3017 3018 alg->base.cra_module = THIS_MODULE; 3019 alg->base.cra_priority = CAAM_CRA_PRIORITY; 3020 alg->base.cra_ctxsize = sizeof(struct caam_ctx) + crypto_dma_padding(); 3021 alg->base.cra_flags |= (CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY | 3022 CRYPTO_ALG_KERN_DRIVER_ONLY); 3023 3024 alg->init = caam_cra_init_skcipher; 3025 alg->exit = caam_cra_exit; 3026 } 3027 3028 static void caam_aead_alg_init(struct caam_aead_alg *t_alg) 3029 { 3030 struct aead_alg *alg = &t_alg->aead; 3031 3032 alg->base.cra_module = THIS_MODULE; 3033 alg->base.cra_priority = CAAM_CRA_PRIORITY; 3034 alg->base.cra_ctxsize = sizeof(struct caam_ctx) + crypto_dma_padding(); 3035 alg->base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY | 3036 CRYPTO_ALG_KERN_DRIVER_ONLY; 3037 3038 alg->init = caam_cra_init_aead; 3039 alg->exit = caam_cra_exit_aead; 3040 } 3041 3042 /* max hash key is max split key size */ 3043 #define CAAM_MAX_HASH_KEY_SIZE (SHA512_DIGEST_SIZE * 2) 3044 3045 #define CAAM_MAX_HASH_BLOCK_SIZE SHA512_BLOCK_SIZE 3046 3047 /* caam context sizes for hashes: running digest + 8 */ 3048 #define HASH_MSG_LEN 8 3049 #define MAX_CTX_LEN (HASH_MSG_LEN + SHA512_DIGEST_SIZE) 3050 3051 enum hash_optype { 3052 UPDATE = 0, 3053 UPDATE_FIRST, 3054 FINALIZE, 3055 DIGEST, 3056 HASH_NUM_OP 3057 }; 3058 3059 /** 3060 * struct caam_hash_ctx - ahash per-session context 3061 * @flc: Flow Contexts array 3062 * @key: authentication key 3063 * @flc_dma: I/O virtual addresses of the Flow Contexts 3064 * @dev: dpseci device 3065 * @ctx_len: size of Context Register 3066 * @adata: hashing algorithm details 3067 */ 3068 struct caam_hash_ctx { 3069 struct caam_flc flc[HASH_NUM_OP]; 3070 u8 key[CAAM_MAX_HASH_BLOCK_SIZE] ____cacheline_aligned; 3071 dma_addr_t flc_dma[HASH_NUM_OP]; 3072 struct device *dev; 3073 int ctx_len; 3074 struct alginfo adata; 3075 }; 3076 3077 /* ahash state */ 3078 struct caam_hash_state { 3079 struct caam_request caam_req; 3080 dma_addr_t buf_dma; 3081 dma_addr_t ctx_dma; 3082 int ctx_dma_len; 3083 u8 buf[CAAM_MAX_HASH_BLOCK_SIZE] ____cacheline_aligned; 3084 int buflen; 3085 int next_buflen; 3086 u8 caam_ctx[MAX_CTX_LEN] ____cacheline_aligned; 3087 int (*update)(struct ahash_request *req); 3088 int (*final)(struct ahash_request *req); 3089 int (*finup)(struct ahash_request *req); 3090 }; 3091 3092 struct caam_export_state { 3093 u8 buf[CAAM_MAX_HASH_BLOCK_SIZE]; 3094 u8 caam_ctx[MAX_CTX_LEN]; 3095 int buflen; 3096 int (*update)(struct ahash_request *req); 3097 int (*final)(struct ahash_request *req); 3098 int (*finup)(struct ahash_request *req); 3099 }; 3100 3101 /* Map current buffer in state (if length > 0) and put it in link table */ 3102 static inline int buf_map_to_qm_sg(struct device *dev, 3103 struct dpaa2_sg_entry *qm_sg, 3104 struct caam_hash_state *state) 3105 { 3106 int buflen = state->buflen; 3107 3108 if (!buflen) 3109 return 0; 3110 3111 state->buf_dma = dma_map_single(dev, state->buf, buflen, 3112 DMA_TO_DEVICE); 3113 if (dma_mapping_error(dev, state->buf_dma)) { 3114 dev_err(dev, "unable to map buf\n"); 3115 state->buf_dma = 0; 3116 return -ENOMEM; 3117 } 3118 3119 dma_to_qm_sg_one(qm_sg, state->buf_dma, buflen, 0); 3120 3121 return 0; 3122 } 3123 3124 /* Map state->caam_ctx, and add it to link table */ 3125 static inline int ctx_map_to_qm_sg(struct device *dev, 3126 struct caam_hash_state *state, int ctx_len, 3127 struct dpaa2_sg_entry *qm_sg, u32 flag) 3128 { 3129 state->ctx_dma_len = ctx_len; 3130 state->ctx_dma = dma_map_single(dev, state->caam_ctx, ctx_len, flag); 3131 if (dma_mapping_error(dev, state->ctx_dma)) { 3132 dev_err(dev, "unable to map ctx\n"); 3133 state->ctx_dma = 0; 3134 return -ENOMEM; 3135 } 3136 3137 dma_to_qm_sg_one(qm_sg, state->ctx_dma, ctx_len, 0); 3138 3139 return 0; 3140 } 3141 3142 static int ahash_set_sh_desc(struct crypto_ahash *ahash) 3143 { 3144 struct caam_hash_ctx *ctx = crypto_ahash_ctx_dma(ahash); 3145 int digestsize = crypto_ahash_digestsize(ahash); 3146 struct dpaa2_caam_priv *priv = dev_get_drvdata(ctx->dev); 3147 struct caam_flc *flc; 3148 u32 *desc; 3149 3150 /* ahash_update shared descriptor */ 3151 flc = &ctx->flc[UPDATE]; 3152 desc = flc->sh_desc; 3153 cnstr_shdsc_ahash(desc, &ctx->adata, OP_ALG_AS_UPDATE, ctx->ctx_len, 3154 ctx->ctx_len, true, priv->sec_attr.era); 3155 flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ 3156 dma_sync_single_for_device(ctx->dev, ctx->flc_dma[UPDATE], 3157 desc_bytes(desc), DMA_BIDIRECTIONAL); 3158 print_hex_dump_debug("ahash update shdesc@" __stringify(__LINE__)": ", 3159 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 3160 1); 3161 3162 /* ahash_update_first shared descriptor */ 3163 flc = &ctx->flc[UPDATE_FIRST]; 3164 desc = flc->sh_desc; 3165 cnstr_shdsc_ahash(desc, &ctx->adata, OP_ALG_AS_INIT, ctx->ctx_len, 3166 ctx->ctx_len, false, priv->sec_attr.era); 3167 flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ 3168 dma_sync_single_for_device(ctx->dev, ctx->flc_dma[UPDATE_FIRST], 3169 desc_bytes(desc), DMA_BIDIRECTIONAL); 3170 print_hex_dump_debug("ahash update first shdesc@" __stringify(__LINE__)": ", 3171 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 3172 1); 3173 3174 /* ahash_final shared descriptor */ 3175 flc = &ctx->flc[FINALIZE]; 3176 desc = flc->sh_desc; 3177 cnstr_shdsc_ahash(desc, &ctx->adata, OP_ALG_AS_FINALIZE, digestsize, 3178 ctx->ctx_len, true, priv->sec_attr.era); 3179 flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ 3180 dma_sync_single_for_device(ctx->dev, ctx->flc_dma[FINALIZE], 3181 desc_bytes(desc), DMA_BIDIRECTIONAL); 3182 print_hex_dump_debug("ahash final shdesc@" __stringify(__LINE__)": ", 3183 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 3184 1); 3185 3186 /* ahash_digest shared descriptor */ 3187 flc = &ctx->flc[DIGEST]; 3188 desc = flc->sh_desc; 3189 cnstr_shdsc_ahash(desc, &ctx->adata, OP_ALG_AS_INITFINAL, digestsize, 3190 ctx->ctx_len, false, priv->sec_attr.era); 3191 flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ 3192 dma_sync_single_for_device(ctx->dev, ctx->flc_dma[DIGEST], 3193 desc_bytes(desc), DMA_BIDIRECTIONAL); 3194 print_hex_dump_debug("ahash digest shdesc@" __stringify(__LINE__)": ", 3195 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 3196 1); 3197 3198 return 0; 3199 } 3200 3201 struct split_key_sh_result { 3202 struct completion completion; 3203 int err; 3204 struct device *dev; 3205 }; 3206 3207 static void split_key_sh_done(void *cbk_ctx, u32 err) 3208 { 3209 struct split_key_sh_result *res = cbk_ctx; 3210 3211 dev_dbg(res->dev, "%s %d: err 0x%x\n", __func__, __LINE__, err); 3212 3213 res->err = err ? caam_qi2_strstatus(res->dev, err) : 0; 3214 complete(&res->completion); 3215 } 3216 3217 /* Digest hash size if it is too large */ 3218 static int hash_digest_key(struct caam_hash_ctx *ctx, u32 *keylen, u8 *key, 3219 u32 digestsize) 3220 { 3221 struct caam_request *req_ctx; 3222 u32 *desc; 3223 struct split_key_sh_result result; 3224 dma_addr_t key_dma; 3225 struct caam_flc *flc; 3226 dma_addr_t flc_dma; 3227 int ret = -ENOMEM; 3228 struct dpaa2_fl_entry *in_fle, *out_fle; 3229 3230 req_ctx = kzalloc_obj(*req_ctx); 3231 if (!req_ctx) 3232 return -ENOMEM; 3233 3234 in_fle = &req_ctx->fd_flt[1]; 3235 out_fle = &req_ctx->fd_flt[0]; 3236 3237 flc = kzalloc_obj(*flc); 3238 if (!flc) 3239 goto err_flc; 3240 3241 key_dma = dma_map_single(ctx->dev, key, *keylen, DMA_BIDIRECTIONAL); 3242 if (dma_mapping_error(ctx->dev, key_dma)) { 3243 dev_err(ctx->dev, "unable to map key memory\n"); 3244 goto err_key_dma; 3245 } 3246 3247 desc = flc->sh_desc; 3248 3249 init_sh_desc(desc, 0); 3250 3251 /* descriptor to perform unkeyed hash on key_in */ 3252 append_operation(desc, ctx->adata.algtype | OP_ALG_ENCRYPT | 3253 OP_ALG_AS_INITFINAL); 3254 append_seq_fifo_load(desc, *keylen, FIFOLD_CLASS_CLASS2 | 3255 FIFOLD_TYPE_LAST2 | FIFOLD_TYPE_MSG); 3256 append_seq_store(desc, digestsize, LDST_CLASS_2_CCB | 3257 LDST_SRCDST_BYTE_CONTEXT); 3258 3259 flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ 3260 flc_dma = dma_map_single(ctx->dev, flc, sizeof(flc->flc) + 3261 desc_bytes(desc), DMA_TO_DEVICE); 3262 if (dma_mapping_error(ctx->dev, flc_dma)) { 3263 dev_err(ctx->dev, "unable to map shared descriptor\n"); 3264 goto err_flc_dma; 3265 } 3266 3267 dpaa2_fl_set_final(in_fle, true); 3268 dpaa2_fl_set_format(in_fle, dpaa2_fl_single); 3269 dpaa2_fl_set_addr(in_fle, key_dma); 3270 dpaa2_fl_set_len(in_fle, *keylen); 3271 dpaa2_fl_set_format(out_fle, dpaa2_fl_single); 3272 dpaa2_fl_set_addr(out_fle, key_dma); 3273 dpaa2_fl_set_len(out_fle, digestsize); 3274 3275 print_hex_dump_devel("key_in@" __stringify(__LINE__)": ", 3276 DUMP_PREFIX_ADDRESS, 16, 4, key, *keylen, 1); 3277 print_hex_dump_debug("shdesc@" __stringify(__LINE__)": ", 3278 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 3279 1); 3280 3281 result.err = 0; 3282 init_completion(&result.completion); 3283 result.dev = ctx->dev; 3284 3285 req_ctx->flc = flc; 3286 req_ctx->flc_dma = flc_dma; 3287 req_ctx->cbk = split_key_sh_done; 3288 req_ctx->ctx = &result; 3289 3290 ret = dpaa2_caam_enqueue(ctx->dev, req_ctx); 3291 if (ret == -EINPROGRESS) { 3292 /* in progress */ 3293 wait_for_completion(&result.completion); 3294 ret = result.err; 3295 print_hex_dump_devel("digested key@" __stringify(__LINE__)": ", 3296 DUMP_PREFIX_ADDRESS, 16, 4, key, 3297 digestsize, 1); 3298 } 3299 3300 dma_unmap_single(ctx->dev, flc_dma, sizeof(flc->flc) + desc_bytes(desc), 3301 DMA_TO_DEVICE); 3302 err_flc_dma: 3303 dma_unmap_single(ctx->dev, key_dma, *keylen, DMA_BIDIRECTIONAL); 3304 err_key_dma: 3305 kfree(flc); 3306 err_flc: 3307 kfree(req_ctx); 3308 3309 *keylen = digestsize; 3310 3311 return ret; 3312 } 3313 3314 static int ahash_setkey(struct crypto_ahash *ahash, const u8 *key, 3315 unsigned int keylen) 3316 { 3317 struct caam_hash_ctx *ctx = crypto_ahash_ctx_dma(ahash); 3318 unsigned int blocksize = crypto_tfm_alg_blocksize(&ahash->base); 3319 unsigned int digestsize = crypto_ahash_digestsize(ahash); 3320 int ret; 3321 u8 *hashed_key = NULL; 3322 3323 dev_dbg(ctx->dev, "keylen %d blocksize %d\n", keylen, blocksize); 3324 3325 if (keylen > blocksize) { 3326 unsigned int aligned_len = 3327 ALIGN(keylen, dma_get_cache_alignment()); 3328 3329 if (aligned_len < keylen) 3330 return -EOVERFLOW; 3331 3332 hashed_key = kmalloc(aligned_len, GFP_KERNEL); 3333 if (!hashed_key) 3334 return -ENOMEM; 3335 memcpy(hashed_key, key, keylen); 3336 ret = hash_digest_key(ctx, &keylen, hashed_key, digestsize); 3337 if (ret) 3338 goto bad_free_key; 3339 key = hashed_key; 3340 } 3341 3342 ctx->adata.keylen = keylen; 3343 ctx->adata.keylen_pad = split_key_len(ctx->adata.algtype & 3344 OP_ALG_ALGSEL_MASK); 3345 if (ctx->adata.keylen_pad > CAAM_MAX_HASH_KEY_SIZE) 3346 goto bad_free_key; 3347 3348 ctx->adata.key_virt = key; 3349 ctx->adata.key_inline = true; 3350 3351 /* 3352 * In case |user key| > |derived key|, using DKP<imm,imm> would result 3353 * in invalid opcodes (last bytes of user key) in the resulting 3354 * descriptor. Use DKP<ptr,imm> instead => both virtual and dma key 3355 * addresses are needed. 3356 */ 3357 if (keylen > ctx->adata.keylen_pad) { 3358 memcpy(ctx->key, key, keylen); 3359 dma_sync_single_for_device(ctx->dev, ctx->adata.key_dma, 3360 ctx->adata.keylen_pad, 3361 DMA_TO_DEVICE); 3362 } 3363 3364 ret = ahash_set_sh_desc(ahash); 3365 kfree(hashed_key); 3366 return ret; 3367 bad_free_key: 3368 kfree(hashed_key); 3369 return -EINVAL; 3370 } 3371 3372 static inline void ahash_unmap(struct device *dev, struct ahash_edesc *edesc, 3373 struct ahash_request *req) 3374 { 3375 struct caam_hash_state *state = ahash_request_ctx_dma(req); 3376 3377 if (edesc->src_nents) 3378 dma_unmap_sg(dev, req->src, edesc->src_nents, DMA_TO_DEVICE); 3379 3380 if (edesc->qm_sg_bytes) 3381 dma_unmap_single(dev, edesc->qm_sg_dma, edesc->qm_sg_bytes, 3382 DMA_TO_DEVICE); 3383 3384 if (state->buf_dma) { 3385 dma_unmap_single(dev, state->buf_dma, state->buflen, 3386 DMA_TO_DEVICE); 3387 state->buf_dma = 0; 3388 } 3389 } 3390 3391 static inline void ahash_unmap_ctx(struct device *dev, 3392 struct ahash_edesc *edesc, 3393 struct ahash_request *req, u32 flag) 3394 { 3395 struct caam_hash_state *state = ahash_request_ctx_dma(req); 3396 3397 if (state->ctx_dma) { 3398 dma_unmap_single(dev, state->ctx_dma, state->ctx_dma_len, flag); 3399 state->ctx_dma = 0; 3400 } 3401 ahash_unmap(dev, edesc, req); 3402 } 3403 3404 static void ahash_done(void *cbk_ctx, u32 status) 3405 { 3406 struct crypto_async_request *areq = cbk_ctx; 3407 struct ahash_request *req = ahash_request_cast(areq); 3408 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); 3409 struct caam_hash_state *state = ahash_request_ctx_dma(req); 3410 struct ahash_edesc *edesc = state->caam_req.edesc; 3411 struct caam_hash_ctx *ctx = crypto_ahash_ctx_dma(ahash); 3412 int digestsize = crypto_ahash_digestsize(ahash); 3413 int ecode = 0; 3414 3415 dev_dbg(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status); 3416 3417 if (unlikely(status)) 3418 ecode = caam_qi2_strstatus(ctx->dev, status); 3419 3420 ahash_unmap_ctx(ctx->dev, edesc, req, DMA_FROM_DEVICE); 3421 memcpy(req->result, state->caam_ctx, digestsize); 3422 qi_cache_free(edesc); 3423 3424 print_hex_dump_debug("ctx@" __stringify(__LINE__)": ", 3425 DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx, 3426 ctx->ctx_len, 1); 3427 3428 ahash_request_complete(req, ecode); 3429 } 3430 3431 static void ahash_done_bi(void *cbk_ctx, u32 status) 3432 { 3433 struct crypto_async_request *areq = cbk_ctx; 3434 struct ahash_request *req = ahash_request_cast(areq); 3435 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); 3436 struct caam_hash_state *state = ahash_request_ctx_dma(req); 3437 struct ahash_edesc *edesc = state->caam_req.edesc; 3438 struct caam_hash_ctx *ctx = crypto_ahash_ctx_dma(ahash); 3439 int ecode = 0; 3440 3441 dev_dbg(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status); 3442 3443 if (unlikely(status)) 3444 ecode = caam_qi2_strstatus(ctx->dev, status); 3445 3446 ahash_unmap_ctx(ctx->dev, edesc, req, DMA_BIDIRECTIONAL); 3447 qi_cache_free(edesc); 3448 3449 scatterwalk_map_and_copy(state->buf, req->src, 3450 req->nbytes - state->next_buflen, 3451 state->next_buflen, 0); 3452 state->buflen = state->next_buflen; 3453 3454 print_hex_dump_debug("buf@" __stringify(__LINE__)": ", 3455 DUMP_PREFIX_ADDRESS, 16, 4, state->buf, 3456 state->buflen, 1); 3457 3458 print_hex_dump_debug("ctx@" __stringify(__LINE__)": ", 3459 DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx, 3460 ctx->ctx_len, 1); 3461 if (req->result) 3462 print_hex_dump_debug("result@" __stringify(__LINE__)": ", 3463 DUMP_PREFIX_ADDRESS, 16, 4, req->result, 3464 crypto_ahash_digestsize(ahash), 1); 3465 3466 ahash_request_complete(req, ecode); 3467 } 3468 3469 static void ahash_done_ctx_src(void *cbk_ctx, u32 status) 3470 { 3471 struct crypto_async_request *areq = cbk_ctx; 3472 struct ahash_request *req = ahash_request_cast(areq); 3473 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); 3474 struct caam_hash_state *state = ahash_request_ctx_dma(req); 3475 struct ahash_edesc *edesc = state->caam_req.edesc; 3476 struct caam_hash_ctx *ctx = crypto_ahash_ctx_dma(ahash); 3477 int digestsize = crypto_ahash_digestsize(ahash); 3478 int ecode = 0; 3479 3480 dev_dbg(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status); 3481 3482 if (unlikely(status)) 3483 ecode = caam_qi2_strstatus(ctx->dev, status); 3484 3485 ahash_unmap_ctx(ctx->dev, edesc, req, DMA_BIDIRECTIONAL); 3486 memcpy(req->result, state->caam_ctx, digestsize); 3487 qi_cache_free(edesc); 3488 3489 print_hex_dump_debug("ctx@" __stringify(__LINE__)": ", 3490 DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx, 3491 ctx->ctx_len, 1); 3492 3493 ahash_request_complete(req, ecode); 3494 } 3495 3496 static void ahash_done_ctx_dst(void *cbk_ctx, u32 status) 3497 { 3498 struct crypto_async_request *areq = cbk_ctx; 3499 struct ahash_request *req = ahash_request_cast(areq); 3500 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); 3501 struct caam_hash_state *state = ahash_request_ctx_dma(req); 3502 struct ahash_edesc *edesc = state->caam_req.edesc; 3503 struct caam_hash_ctx *ctx = crypto_ahash_ctx_dma(ahash); 3504 int ecode = 0; 3505 3506 dev_dbg(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status); 3507 3508 if (unlikely(status)) 3509 ecode = caam_qi2_strstatus(ctx->dev, status); 3510 3511 ahash_unmap_ctx(ctx->dev, edesc, req, DMA_FROM_DEVICE); 3512 qi_cache_free(edesc); 3513 3514 scatterwalk_map_and_copy(state->buf, req->src, 3515 req->nbytes - state->next_buflen, 3516 state->next_buflen, 0); 3517 state->buflen = state->next_buflen; 3518 3519 print_hex_dump_debug("buf@" __stringify(__LINE__)": ", 3520 DUMP_PREFIX_ADDRESS, 16, 4, state->buf, 3521 state->buflen, 1); 3522 3523 print_hex_dump_debug("ctx@" __stringify(__LINE__)": ", 3524 DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx, 3525 ctx->ctx_len, 1); 3526 if (req->result) 3527 print_hex_dump_debug("result@" __stringify(__LINE__)": ", 3528 DUMP_PREFIX_ADDRESS, 16, 4, req->result, 3529 crypto_ahash_digestsize(ahash), 1); 3530 3531 ahash_request_complete(req, ecode); 3532 } 3533 3534 static int ahash_update_ctx(struct ahash_request *req) 3535 { 3536 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); 3537 struct caam_hash_ctx *ctx = crypto_ahash_ctx_dma(ahash); 3538 struct caam_hash_state *state = ahash_request_ctx_dma(req); 3539 struct caam_request *req_ctx = &state->caam_req; 3540 struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1]; 3541 struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0]; 3542 gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? 3543 GFP_KERNEL : GFP_ATOMIC; 3544 u8 *buf = state->buf; 3545 int *buflen = &state->buflen; 3546 int *next_buflen = &state->next_buflen; 3547 int in_len = *buflen + req->nbytes, to_hash; 3548 int src_nents, mapped_nents, qm_sg_bytes, qm_sg_src_index; 3549 struct ahash_edesc *edesc; 3550 int ret = 0; 3551 3552 *next_buflen = in_len & (crypto_tfm_alg_blocksize(&ahash->base) - 1); 3553 to_hash = in_len - *next_buflen; 3554 3555 if (to_hash) { 3556 struct dpaa2_sg_entry *sg_table; 3557 int src_len = req->nbytes - *next_buflen; 3558 3559 src_nents = sg_nents_for_len(req->src, src_len); 3560 if (src_nents < 0) { 3561 dev_err(ctx->dev, "Invalid number of src SG.\n"); 3562 return src_nents; 3563 } 3564 3565 if (src_nents) { 3566 mapped_nents = dma_map_sg(ctx->dev, req->src, src_nents, 3567 DMA_TO_DEVICE); 3568 if (!mapped_nents) { 3569 dev_err(ctx->dev, "unable to DMA map source\n"); 3570 return -ENOMEM; 3571 } 3572 } else { 3573 mapped_nents = 0; 3574 } 3575 3576 /* allocate space for base edesc and link tables */ 3577 edesc = qi_cache_zalloc(flags); 3578 if (!edesc) { 3579 dma_unmap_sg(ctx->dev, req->src, src_nents, 3580 DMA_TO_DEVICE); 3581 return -ENOMEM; 3582 } 3583 3584 edesc->src_nents = src_nents; 3585 qm_sg_src_index = 1 + (*buflen ? 1 : 0); 3586 qm_sg_bytes = pad_sg_nents(qm_sg_src_index + mapped_nents) * 3587 sizeof(*sg_table); 3588 sg_table = &edesc->sgt[0]; 3589 3590 ret = ctx_map_to_qm_sg(ctx->dev, state, ctx->ctx_len, sg_table, 3591 DMA_BIDIRECTIONAL); 3592 if (ret) 3593 goto unmap_ctx; 3594 3595 ret = buf_map_to_qm_sg(ctx->dev, sg_table + 1, state); 3596 if (ret) 3597 goto unmap_ctx; 3598 3599 if (mapped_nents) { 3600 sg_to_qm_sg_last(req->src, src_len, 3601 sg_table + qm_sg_src_index, 0); 3602 } else { 3603 dpaa2_sg_set_final(sg_table + qm_sg_src_index - 1, 3604 true); 3605 } 3606 3607 edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table, 3608 qm_sg_bytes, DMA_TO_DEVICE); 3609 if (dma_mapping_error(ctx->dev, edesc->qm_sg_dma)) { 3610 dev_err(ctx->dev, "unable to map S/G table\n"); 3611 ret = -ENOMEM; 3612 goto unmap_ctx; 3613 } 3614 edesc->qm_sg_bytes = qm_sg_bytes; 3615 3616 memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt)); 3617 dpaa2_fl_set_final(in_fle, true); 3618 dpaa2_fl_set_format(in_fle, dpaa2_fl_sg); 3619 dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma); 3620 dpaa2_fl_set_len(in_fle, ctx->ctx_len + to_hash); 3621 dpaa2_fl_set_format(out_fle, dpaa2_fl_single); 3622 dpaa2_fl_set_addr(out_fle, state->ctx_dma); 3623 dpaa2_fl_set_len(out_fle, ctx->ctx_len); 3624 3625 req_ctx->flc = &ctx->flc[UPDATE]; 3626 req_ctx->flc_dma = ctx->flc_dma[UPDATE]; 3627 req_ctx->cbk = ahash_done_bi; 3628 req_ctx->ctx = &req->base; 3629 req_ctx->edesc = edesc; 3630 3631 ret = dpaa2_caam_enqueue(ctx->dev, req_ctx); 3632 if (ret != -EINPROGRESS && 3633 !(ret == -EBUSY && 3634 req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) 3635 goto unmap_ctx; 3636 } else if (*next_buflen) { 3637 scatterwalk_map_and_copy(buf + *buflen, req->src, 0, 3638 req->nbytes, 0); 3639 *buflen = *next_buflen; 3640 3641 print_hex_dump_debug("buf@" __stringify(__LINE__)": ", 3642 DUMP_PREFIX_ADDRESS, 16, 4, buf, 3643 *buflen, 1); 3644 } 3645 3646 return ret; 3647 unmap_ctx: 3648 ahash_unmap_ctx(ctx->dev, edesc, req, DMA_BIDIRECTIONAL); 3649 qi_cache_free(edesc); 3650 return ret; 3651 } 3652 3653 static int ahash_final_ctx(struct ahash_request *req) 3654 { 3655 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); 3656 struct caam_hash_ctx *ctx = crypto_ahash_ctx_dma(ahash); 3657 struct caam_hash_state *state = ahash_request_ctx_dma(req); 3658 struct caam_request *req_ctx = &state->caam_req; 3659 struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1]; 3660 struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0]; 3661 gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? 3662 GFP_KERNEL : GFP_ATOMIC; 3663 int buflen = state->buflen; 3664 int qm_sg_bytes; 3665 int digestsize = crypto_ahash_digestsize(ahash); 3666 struct ahash_edesc *edesc; 3667 struct dpaa2_sg_entry *sg_table; 3668 int ret; 3669 3670 /* allocate space for base edesc and link tables */ 3671 edesc = qi_cache_zalloc(flags); 3672 if (!edesc) 3673 return -ENOMEM; 3674 3675 qm_sg_bytes = pad_sg_nents(1 + (buflen ? 1 : 0)) * sizeof(*sg_table); 3676 sg_table = &edesc->sgt[0]; 3677 3678 ret = ctx_map_to_qm_sg(ctx->dev, state, ctx->ctx_len, sg_table, 3679 DMA_BIDIRECTIONAL); 3680 if (ret) 3681 goto unmap_ctx; 3682 3683 ret = buf_map_to_qm_sg(ctx->dev, sg_table + 1, state); 3684 if (ret) 3685 goto unmap_ctx; 3686 3687 dpaa2_sg_set_final(sg_table + (buflen ? 1 : 0), true); 3688 3689 edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table, qm_sg_bytes, 3690 DMA_TO_DEVICE); 3691 if (dma_mapping_error(ctx->dev, edesc->qm_sg_dma)) { 3692 dev_err(ctx->dev, "unable to map S/G table\n"); 3693 ret = -ENOMEM; 3694 goto unmap_ctx; 3695 } 3696 edesc->qm_sg_bytes = qm_sg_bytes; 3697 3698 memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt)); 3699 dpaa2_fl_set_final(in_fle, true); 3700 dpaa2_fl_set_format(in_fle, dpaa2_fl_sg); 3701 dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma); 3702 dpaa2_fl_set_len(in_fle, ctx->ctx_len + buflen); 3703 dpaa2_fl_set_format(out_fle, dpaa2_fl_single); 3704 dpaa2_fl_set_addr(out_fle, state->ctx_dma); 3705 dpaa2_fl_set_len(out_fle, digestsize); 3706 3707 req_ctx->flc = &ctx->flc[FINALIZE]; 3708 req_ctx->flc_dma = ctx->flc_dma[FINALIZE]; 3709 req_ctx->cbk = ahash_done_ctx_src; 3710 req_ctx->ctx = &req->base; 3711 req_ctx->edesc = edesc; 3712 3713 ret = dpaa2_caam_enqueue(ctx->dev, req_ctx); 3714 if (ret == -EINPROGRESS || 3715 (ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) 3716 return ret; 3717 3718 unmap_ctx: 3719 ahash_unmap_ctx(ctx->dev, edesc, req, DMA_BIDIRECTIONAL); 3720 qi_cache_free(edesc); 3721 return ret; 3722 } 3723 3724 static int ahash_finup_ctx(struct ahash_request *req) 3725 { 3726 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); 3727 struct caam_hash_ctx *ctx = crypto_ahash_ctx_dma(ahash); 3728 struct caam_hash_state *state = ahash_request_ctx_dma(req); 3729 struct caam_request *req_ctx = &state->caam_req; 3730 struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1]; 3731 struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0]; 3732 gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? 3733 GFP_KERNEL : GFP_ATOMIC; 3734 int buflen = state->buflen; 3735 int qm_sg_bytes, qm_sg_src_index; 3736 int src_nents, mapped_nents; 3737 int digestsize = crypto_ahash_digestsize(ahash); 3738 struct ahash_edesc *edesc; 3739 struct dpaa2_sg_entry *sg_table; 3740 int ret; 3741 3742 src_nents = sg_nents_for_len(req->src, req->nbytes); 3743 if (src_nents < 0) { 3744 dev_err(ctx->dev, "Invalid number of src SG.\n"); 3745 return src_nents; 3746 } 3747 3748 if (src_nents) { 3749 mapped_nents = dma_map_sg(ctx->dev, req->src, src_nents, 3750 DMA_TO_DEVICE); 3751 if (!mapped_nents) { 3752 dev_err(ctx->dev, "unable to DMA map source\n"); 3753 return -ENOMEM; 3754 } 3755 } else { 3756 mapped_nents = 0; 3757 } 3758 3759 /* allocate space for base edesc and link tables */ 3760 edesc = qi_cache_zalloc(flags); 3761 if (!edesc) { 3762 dma_unmap_sg(ctx->dev, req->src, src_nents, DMA_TO_DEVICE); 3763 return -ENOMEM; 3764 } 3765 3766 edesc->src_nents = src_nents; 3767 qm_sg_src_index = 1 + (buflen ? 1 : 0); 3768 qm_sg_bytes = pad_sg_nents(qm_sg_src_index + mapped_nents) * 3769 sizeof(*sg_table); 3770 sg_table = &edesc->sgt[0]; 3771 3772 ret = ctx_map_to_qm_sg(ctx->dev, state, ctx->ctx_len, sg_table, 3773 DMA_BIDIRECTIONAL); 3774 if (ret) 3775 goto unmap_ctx; 3776 3777 ret = buf_map_to_qm_sg(ctx->dev, sg_table + 1, state); 3778 if (ret) 3779 goto unmap_ctx; 3780 3781 sg_to_qm_sg_last(req->src, req->nbytes, sg_table + qm_sg_src_index, 0); 3782 3783 edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table, qm_sg_bytes, 3784 DMA_TO_DEVICE); 3785 if (dma_mapping_error(ctx->dev, edesc->qm_sg_dma)) { 3786 dev_err(ctx->dev, "unable to map S/G table\n"); 3787 ret = -ENOMEM; 3788 goto unmap_ctx; 3789 } 3790 edesc->qm_sg_bytes = qm_sg_bytes; 3791 3792 memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt)); 3793 dpaa2_fl_set_final(in_fle, true); 3794 dpaa2_fl_set_format(in_fle, dpaa2_fl_sg); 3795 dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma); 3796 dpaa2_fl_set_len(in_fle, ctx->ctx_len + buflen + req->nbytes); 3797 dpaa2_fl_set_format(out_fle, dpaa2_fl_single); 3798 dpaa2_fl_set_addr(out_fle, state->ctx_dma); 3799 dpaa2_fl_set_len(out_fle, digestsize); 3800 3801 req_ctx->flc = &ctx->flc[FINALIZE]; 3802 req_ctx->flc_dma = ctx->flc_dma[FINALIZE]; 3803 req_ctx->cbk = ahash_done_ctx_src; 3804 req_ctx->ctx = &req->base; 3805 req_ctx->edesc = edesc; 3806 3807 ret = dpaa2_caam_enqueue(ctx->dev, req_ctx); 3808 if (ret == -EINPROGRESS || 3809 (ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) 3810 return ret; 3811 3812 unmap_ctx: 3813 ahash_unmap_ctx(ctx->dev, edesc, req, DMA_BIDIRECTIONAL); 3814 qi_cache_free(edesc); 3815 return ret; 3816 } 3817 3818 static int ahash_digest(struct ahash_request *req) 3819 { 3820 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); 3821 struct caam_hash_ctx *ctx = crypto_ahash_ctx_dma(ahash); 3822 struct caam_hash_state *state = ahash_request_ctx_dma(req); 3823 struct caam_request *req_ctx = &state->caam_req; 3824 struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1]; 3825 struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0]; 3826 gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? 3827 GFP_KERNEL : GFP_ATOMIC; 3828 int digestsize = crypto_ahash_digestsize(ahash); 3829 int src_nents, mapped_nents; 3830 struct ahash_edesc *edesc; 3831 int ret = -ENOMEM; 3832 3833 state->buf_dma = 0; 3834 3835 src_nents = sg_nents_for_len(req->src, req->nbytes); 3836 if (src_nents < 0) { 3837 dev_err(ctx->dev, "Invalid number of src SG.\n"); 3838 return src_nents; 3839 } 3840 3841 if (src_nents) { 3842 mapped_nents = dma_map_sg(ctx->dev, req->src, src_nents, 3843 DMA_TO_DEVICE); 3844 if (!mapped_nents) { 3845 dev_err(ctx->dev, "unable to map source for DMA\n"); 3846 return ret; 3847 } 3848 } else { 3849 mapped_nents = 0; 3850 } 3851 3852 /* allocate space for base edesc and link tables */ 3853 edesc = qi_cache_zalloc(flags); 3854 if (!edesc) { 3855 dma_unmap_sg(ctx->dev, req->src, src_nents, DMA_TO_DEVICE); 3856 return ret; 3857 } 3858 3859 edesc->src_nents = src_nents; 3860 memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt)); 3861 3862 if (mapped_nents > 1) { 3863 int qm_sg_bytes; 3864 struct dpaa2_sg_entry *sg_table = &edesc->sgt[0]; 3865 3866 qm_sg_bytes = pad_sg_nents(mapped_nents) * sizeof(*sg_table); 3867 sg_to_qm_sg_last(req->src, req->nbytes, sg_table, 0); 3868 edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table, 3869 qm_sg_bytes, DMA_TO_DEVICE); 3870 if (dma_mapping_error(ctx->dev, edesc->qm_sg_dma)) { 3871 dev_err(ctx->dev, "unable to map S/G table\n"); 3872 goto unmap; 3873 } 3874 edesc->qm_sg_bytes = qm_sg_bytes; 3875 dpaa2_fl_set_format(in_fle, dpaa2_fl_sg); 3876 dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma); 3877 } else { 3878 dpaa2_fl_set_format(in_fle, dpaa2_fl_single); 3879 dpaa2_fl_set_addr(in_fle, sg_dma_address(req->src)); 3880 } 3881 3882 state->ctx_dma_len = digestsize; 3883 state->ctx_dma = dma_map_single(ctx->dev, state->caam_ctx, digestsize, 3884 DMA_FROM_DEVICE); 3885 if (dma_mapping_error(ctx->dev, state->ctx_dma)) { 3886 dev_err(ctx->dev, "unable to map ctx\n"); 3887 state->ctx_dma = 0; 3888 goto unmap; 3889 } 3890 3891 dpaa2_fl_set_final(in_fle, true); 3892 dpaa2_fl_set_len(in_fle, req->nbytes); 3893 dpaa2_fl_set_format(out_fle, dpaa2_fl_single); 3894 dpaa2_fl_set_addr(out_fle, state->ctx_dma); 3895 dpaa2_fl_set_len(out_fle, digestsize); 3896 3897 req_ctx->flc = &ctx->flc[DIGEST]; 3898 req_ctx->flc_dma = ctx->flc_dma[DIGEST]; 3899 req_ctx->cbk = ahash_done; 3900 req_ctx->ctx = &req->base; 3901 req_ctx->edesc = edesc; 3902 ret = dpaa2_caam_enqueue(ctx->dev, req_ctx); 3903 if (ret == -EINPROGRESS || 3904 (ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) 3905 return ret; 3906 3907 unmap: 3908 ahash_unmap_ctx(ctx->dev, edesc, req, DMA_FROM_DEVICE); 3909 qi_cache_free(edesc); 3910 return ret; 3911 } 3912 3913 static int ahash_final_no_ctx(struct ahash_request *req) 3914 { 3915 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); 3916 struct caam_hash_ctx *ctx = crypto_ahash_ctx_dma(ahash); 3917 struct caam_hash_state *state = ahash_request_ctx_dma(req); 3918 struct caam_request *req_ctx = &state->caam_req; 3919 struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1]; 3920 struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0]; 3921 gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? 3922 GFP_KERNEL : GFP_ATOMIC; 3923 u8 *buf = state->buf; 3924 int buflen = state->buflen; 3925 int digestsize = crypto_ahash_digestsize(ahash); 3926 struct ahash_edesc *edesc; 3927 int ret = -ENOMEM; 3928 3929 /* allocate space for base edesc and link tables */ 3930 edesc = qi_cache_zalloc(flags); 3931 if (!edesc) 3932 return ret; 3933 3934 if (buflen) { 3935 state->buf_dma = dma_map_single(ctx->dev, buf, buflen, 3936 DMA_TO_DEVICE); 3937 if (dma_mapping_error(ctx->dev, state->buf_dma)) { 3938 dev_err(ctx->dev, "unable to map src\n"); 3939 goto unmap; 3940 } 3941 } 3942 3943 state->ctx_dma_len = digestsize; 3944 state->ctx_dma = dma_map_single(ctx->dev, state->caam_ctx, digestsize, 3945 DMA_FROM_DEVICE); 3946 if (dma_mapping_error(ctx->dev, state->ctx_dma)) { 3947 dev_err(ctx->dev, "unable to map ctx\n"); 3948 state->ctx_dma = 0; 3949 goto unmap; 3950 } 3951 3952 memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt)); 3953 dpaa2_fl_set_final(in_fle, true); 3954 /* 3955 * crypto engine requires the input entry to be present when 3956 * "frame list" FD is used. 3957 * Since engine does not support FMT=2'b11 (unused entry type), leaving 3958 * in_fle zeroized (except for "Final" flag) is the best option. 3959 */ 3960 if (buflen) { 3961 dpaa2_fl_set_format(in_fle, dpaa2_fl_single); 3962 dpaa2_fl_set_addr(in_fle, state->buf_dma); 3963 dpaa2_fl_set_len(in_fle, buflen); 3964 } 3965 dpaa2_fl_set_format(out_fle, dpaa2_fl_single); 3966 dpaa2_fl_set_addr(out_fle, state->ctx_dma); 3967 dpaa2_fl_set_len(out_fle, digestsize); 3968 3969 req_ctx->flc = &ctx->flc[DIGEST]; 3970 req_ctx->flc_dma = ctx->flc_dma[DIGEST]; 3971 req_ctx->cbk = ahash_done; 3972 req_ctx->ctx = &req->base; 3973 req_ctx->edesc = edesc; 3974 3975 ret = dpaa2_caam_enqueue(ctx->dev, req_ctx); 3976 if (ret == -EINPROGRESS || 3977 (ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) 3978 return ret; 3979 3980 unmap: 3981 ahash_unmap_ctx(ctx->dev, edesc, req, DMA_FROM_DEVICE); 3982 qi_cache_free(edesc); 3983 return ret; 3984 } 3985 3986 static int ahash_update_no_ctx(struct ahash_request *req) 3987 { 3988 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); 3989 struct caam_hash_ctx *ctx = crypto_ahash_ctx_dma(ahash); 3990 struct caam_hash_state *state = ahash_request_ctx_dma(req); 3991 struct caam_request *req_ctx = &state->caam_req; 3992 struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1]; 3993 struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0]; 3994 gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? 3995 GFP_KERNEL : GFP_ATOMIC; 3996 u8 *buf = state->buf; 3997 int *buflen = &state->buflen; 3998 int *next_buflen = &state->next_buflen; 3999 int in_len = *buflen + req->nbytes, to_hash; 4000 int qm_sg_bytes, src_nents, mapped_nents; 4001 struct ahash_edesc *edesc; 4002 int ret = 0; 4003 4004 *next_buflen = in_len & (crypto_tfm_alg_blocksize(&ahash->base) - 1); 4005 to_hash = in_len - *next_buflen; 4006 4007 if (to_hash) { 4008 struct dpaa2_sg_entry *sg_table; 4009 int src_len = req->nbytes - *next_buflen; 4010 4011 src_nents = sg_nents_for_len(req->src, src_len); 4012 if (src_nents < 0) { 4013 dev_err(ctx->dev, "Invalid number of src SG.\n"); 4014 return src_nents; 4015 } 4016 4017 if (src_nents) { 4018 mapped_nents = dma_map_sg(ctx->dev, req->src, src_nents, 4019 DMA_TO_DEVICE); 4020 if (!mapped_nents) { 4021 dev_err(ctx->dev, "unable to DMA map source\n"); 4022 return -ENOMEM; 4023 } 4024 } else { 4025 mapped_nents = 0; 4026 } 4027 4028 /* allocate space for base edesc and link tables */ 4029 edesc = qi_cache_zalloc(flags); 4030 if (!edesc) { 4031 dma_unmap_sg(ctx->dev, req->src, src_nents, 4032 DMA_TO_DEVICE); 4033 return -ENOMEM; 4034 } 4035 4036 edesc->src_nents = src_nents; 4037 qm_sg_bytes = pad_sg_nents(1 + mapped_nents) * 4038 sizeof(*sg_table); 4039 sg_table = &edesc->sgt[0]; 4040 4041 ret = buf_map_to_qm_sg(ctx->dev, sg_table, state); 4042 if (ret) 4043 goto unmap_ctx; 4044 4045 sg_to_qm_sg_last(req->src, src_len, sg_table + 1, 0); 4046 4047 edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table, 4048 qm_sg_bytes, DMA_TO_DEVICE); 4049 if (dma_mapping_error(ctx->dev, edesc->qm_sg_dma)) { 4050 dev_err(ctx->dev, "unable to map S/G table\n"); 4051 ret = -ENOMEM; 4052 goto unmap_ctx; 4053 } 4054 edesc->qm_sg_bytes = qm_sg_bytes; 4055 4056 state->ctx_dma_len = ctx->ctx_len; 4057 state->ctx_dma = dma_map_single(ctx->dev, state->caam_ctx, 4058 ctx->ctx_len, DMA_FROM_DEVICE); 4059 if (dma_mapping_error(ctx->dev, state->ctx_dma)) { 4060 dev_err(ctx->dev, "unable to map ctx\n"); 4061 state->ctx_dma = 0; 4062 ret = -ENOMEM; 4063 goto unmap_ctx; 4064 } 4065 4066 memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt)); 4067 dpaa2_fl_set_final(in_fle, true); 4068 dpaa2_fl_set_format(in_fle, dpaa2_fl_sg); 4069 dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma); 4070 dpaa2_fl_set_len(in_fle, to_hash); 4071 dpaa2_fl_set_format(out_fle, dpaa2_fl_single); 4072 dpaa2_fl_set_addr(out_fle, state->ctx_dma); 4073 dpaa2_fl_set_len(out_fle, ctx->ctx_len); 4074 4075 req_ctx->flc = &ctx->flc[UPDATE_FIRST]; 4076 req_ctx->flc_dma = ctx->flc_dma[UPDATE_FIRST]; 4077 req_ctx->cbk = ahash_done_ctx_dst; 4078 req_ctx->ctx = &req->base; 4079 req_ctx->edesc = edesc; 4080 4081 ret = dpaa2_caam_enqueue(ctx->dev, req_ctx); 4082 if (ret != -EINPROGRESS && 4083 !(ret == -EBUSY && 4084 req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) 4085 goto unmap_ctx; 4086 4087 state->update = ahash_update_ctx; 4088 state->finup = ahash_finup_ctx; 4089 state->final = ahash_final_ctx; 4090 } else if (*next_buflen) { 4091 scatterwalk_map_and_copy(buf + *buflen, req->src, 0, 4092 req->nbytes, 0); 4093 *buflen = *next_buflen; 4094 4095 print_hex_dump_debug("buf@" __stringify(__LINE__)": ", 4096 DUMP_PREFIX_ADDRESS, 16, 4, buf, 4097 *buflen, 1); 4098 } 4099 4100 return ret; 4101 unmap_ctx: 4102 ahash_unmap_ctx(ctx->dev, edesc, req, DMA_TO_DEVICE); 4103 qi_cache_free(edesc); 4104 return ret; 4105 } 4106 4107 static int ahash_finup_no_ctx(struct ahash_request *req) 4108 { 4109 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); 4110 struct caam_hash_ctx *ctx = crypto_ahash_ctx_dma(ahash); 4111 struct caam_hash_state *state = ahash_request_ctx_dma(req); 4112 struct caam_request *req_ctx = &state->caam_req; 4113 struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1]; 4114 struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0]; 4115 gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? 4116 GFP_KERNEL : GFP_ATOMIC; 4117 int buflen = state->buflen; 4118 int qm_sg_bytes, src_nents, mapped_nents; 4119 int digestsize = crypto_ahash_digestsize(ahash); 4120 struct ahash_edesc *edesc; 4121 struct dpaa2_sg_entry *sg_table; 4122 int ret = -ENOMEM; 4123 4124 src_nents = sg_nents_for_len(req->src, req->nbytes); 4125 if (src_nents < 0) { 4126 dev_err(ctx->dev, "Invalid number of src SG.\n"); 4127 return src_nents; 4128 } 4129 4130 if (src_nents) { 4131 mapped_nents = dma_map_sg(ctx->dev, req->src, src_nents, 4132 DMA_TO_DEVICE); 4133 if (!mapped_nents) { 4134 dev_err(ctx->dev, "unable to DMA map source\n"); 4135 return ret; 4136 } 4137 } else { 4138 mapped_nents = 0; 4139 } 4140 4141 /* allocate space for base edesc and link tables */ 4142 edesc = qi_cache_zalloc(flags); 4143 if (!edesc) { 4144 dma_unmap_sg(ctx->dev, req->src, src_nents, DMA_TO_DEVICE); 4145 return ret; 4146 } 4147 4148 edesc->src_nents = src_nents; 4149 qm_sg_bytes = pad_sg_nents(2 + mapped_nents) * sizeof(*sg_table); 4150 sg_table = &edesc->sgt[0]; 4151 4152 ret = buf_map_to_qm_sg(ctx->dev, sg_table, state); 4153 if (ret) 4154 goto unmap; 4155 4156 sg_to_qm_sg_last(req->src, req->nbytes, sg_table + 1, 0); 4157 4158 edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table, qm_sg_bytes, 4159 DMA_TO_DEVICE); 4160 if (dma_mapping_error(ctx->dev, edesc->qm_sg_dma)) { 4161 dev_err(ctx->dev, "unable to map S/G table\n"); 4162 ret = -ENOMEM; 4163 goto unmap; 4164 } 4165 edesc->qm_sg_bytes = qm_sg_bytes; 4166 4167 state->ctx_dma_len = digestsize; 4168 state->ctx_dma = dma_map_single(ctx->dev, state->caam_ctx, digestsize, 4169 DMA_FROM_DEVICE); 4170 if (dma_mapping_error(ctx->dev, state->ctx_dma)) { 4171 dev_err(ctx->dev, "unable to map ctx\n"); 4172 state->ctx_dma = 0; 4173 ret = -ENOMEM; 4174 goto unmap; 4175 } 4176 4177 memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt)); 4178 dpaa2_fl_set_final(in_fle, true); 4179 dpaa2_fl_set_format(in_fle, dpaa2_fl_sg); 4180 dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma); 4181 dpaa2_fl_set_len(in_fle, buflen + req->nbytes); 4182 dpaa2_fl_set_format(out_fle, dpaa2_fl_single); 4183 dpaa2_fl_set_addr(out_fle, state->ctx_dma); 4184 dpaa2_fl_set_len(out_fle, digestsize); 4185 4186 req_ctx->flc = &ctx->flc[DIGEST]; 4187 req_ctx->flc_dma = ctx->flc_dma[DIGEST]; 4188 req_ctx->cbk = ahash_done; 4189 req_ctx->ctx = &req->base; 4190 req_ctx->edesc = edesc; 4191 ret = dpaa2_caam_enqueue(ctx->dev, req_ctx); 4192 if (ret != -EINPROGRESS && 4193 !(ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) 4194 goto unmap; 4195 4196 return ret; 4197 unmap: 4198 ahash_unmap_ctx(ctx->dev, edesc, req, DMA_FROM_DEVICE); 4199 qi_cache_free(edesc); 4200 return ret; 4201 } 4202 4203 static int ahash_update_first(struct ahash_request *req) 4204 { 4205 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); 4206 struct caam_hash_ctx *ctx = crypto_ahash_ctx_dma(ahash); 4207 struct caam_hash_state *state = ahash_request_ctx_dma(req); 4208 struct caam_request *req_ctx = &state->caam_req; 4209 struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1]; 4210 struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0]; 4211 gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? 4212 GFP_KERNEL : GFP_ATOMIC; 4213 u8 *buf = state->buf; 4214 int *buflen = &state->buflen; 4215 int *next_buflen = &state->next_buflen; 4216 int to_hash; 4217 int src_nents, mapped_nents; 4218 struct ahash_edesc *edesc; 4219 int ret = 0; 4220 4221 *next_buflen = req->nbytes & (crypto_tfm_alg_blocksize(&ahash->base) - 4222 1); 4223 to_hash = req->nbytes - *next_buflen; 4224 4225 if (to_hash) { 4226 struct dpaa2_sg_entry *sg_table; 4227 int src_len = req->nbytes - *next_buflen; 4228 4229 src_nents = sg_nents_for_len(req->src, src_len); 4230 if (src_nents < 0) { 4231 dev_err(ctx->dev, "Invalid number of src SG.\n"); 4232 return src_nents; 4233 } 4234 4235 if (src_nents) { 4236 mapped_nents = dma_map_sg(ctx->dev, req->src, src_nents, 4237 DMA_TO_DEVICE); 4238 if (!mapped_nents) { 4239 dev_err(ctx->dev, "unable to map source for DMA\n"); 4240 return -ENOMEM; 4241 } 4242 } else { 4243 mapped_nents = 0; 4244 } 4245 4246 /* allocate space for base edesc and link tables */ 4247 edesc = qi_cache_zalloc(flags); 4248 if (!edesc) { 4249 dma_unmap_sg(ctx->dev, req->src, src_nents, 4250 DMA_TO_DEVICE); 4251 return -ENOMEM; 4252 } 4253 4254 edesc->src_nents = src_nents; 4255 sg_table = &edesc->sgt[0]; 4256 4257 memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt)); 4258 dpaa2_fl_set_final(in_fle, true); 4259 dpaa2_fl_set_len(in_fle, to_hash); 4260 4261 if (mapped_nents > 1) { 4262 int qm_sg_bytes; 4263 4264 sg_to_qm_sg_last(req->src, src_len, sg_table, 0); 4265 qm_sg_bytes = pad_sg_nents(mapped_nents) * 4266 sizeof(*sg_table); 4267 edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table, 4268 qm_sg_bytes, 4269 DMA_TO_DEVICE); 4270 if (dma_mapping_error(ctx->dev, edesc->qm_sg_dma)) { 4271 dev_err(ctx->dev, "unable to map S/G table\n"); 4272 ret = -ENOMEM; 4273 goto unmap_ctx; 4274 } 4275 edesc->qm_sg_bytes = qm_sg_bytes; 4276 dpaa2_fl_set_format(in_fle, dpaa2_fl_sg); 4277 dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma); 4278 } else { 4279 dpaa2_fl_set_format(in_fle, dpaa2_fl_single); 4280 dpaa2_fl_set_addr(in_fle, sg_dma_address(req->src)); 4281 } 4282 4283 state->ctx_dma_len = ctx->ctx_len; 4284 state->ctx_dma = dma_map_single(ctx->dev, state->caam_ctx, 4285 ctx->ctx_len, DMA_FROM_DEVICE); 4286 if (dma_mapping_error(ctx->dev, state->ctx_dma)) { 4287 dev_err(ctx->dev, "unable to map ctx\n"); 4288 state->ctx_dma = 0; 4289 ret = -ENOMEM; 4290 goto unmap_ctx; 4291 } 4292 4293 dpaa2_fl_set_format(out_fle, dpaa2_fl_single); 4294 dpaa2_fl_set_addr(out_fle, state->ctx_dma); 4295 dpaa2_fl_set_len(out_fle, ctx->ctx_len); 4296 4297 req_ctx->flc = &ctx->flc[UPDATE_FIRST]; 4298 req_ctx->flc_dma = ctx->flc_dma[UPDATE_FIRST]; 4299 req_ctx->cbk = ahash_done_ctx_dst; 4300 req_ctx->ctx = &req->base; 4301 req_ctx->edesc = edesc; 4302 4303 ret = dpaa2_caam_enqueue(ctx->dev, req_ctx); 4304 if (ret != -EINPROGRESS && 4305 !(ret == -EBUSY && req->base.flags & 4306 CRYPTO_TFM_REQ_MAY_BACKLOG)) 4307 goto unmap_ctx; 4308 4309 state->update = ahash_update_ctx; 4310 state->finup = ahash_finup_ctx; 4311 state->final = ahash_final_ctx; 4312 } else if (*next_buflen) { 4313 state->update = ahash_update_no_ctx; 4314 state->finup = ahash_finup_no_ctx; 4315 state->final = ahash_final_no_ctx; 4316 scatterwalk_map_and_copy(buf, req->src, 0, 4317 req->nbytes, 0); 4318 *buflen = *next_buflen; 4319 4320 print_hex_dump_debug("buf@" __stringify(__LINE__)": ", 4321 DUMP_PREFIX_ADDRESS, 16, 4, buf, 4322 *buflen, 1); 4323 } 4324 4325 return ret; 4326 unmap_ctx: 4327 ahash_unmap_ctx(ctx->dev, edesc, req, DMA_TO_DEVICE); 4328 qi_cache_free(edesc); 4329 return ret; 4330 } 4331 4332 static int ahash_finup_first(struct ahash_request *req) 4333 { 4334 return ahash_digest(req); 4335 } 4336 4337 static int ahash_init(struct ahash_request *req) 4338 { 4339 struct caam_hash_state *state = ahash_request_ctx_dma(req); 4340 4341 state->update = ahash_update_first; 4342 state->finup = ahash_finup_first; 4343 state->final = ahash_final_no_ctx; 4344 4345 state->ctx_dma = 0; 4346 state->ctx_dma_len = 0; 4347 state->buf_dma = 0; 4348 state->buflen = 0; 4349 state->next_buflen = 0; 4350 4351 return 0; 4352 } 4353 4354 static int ahash_update(struct ahash_request *req) 4355 { 4356 struct caam_hash_state *state = ahash_request_ctx_dma(req); 4357 4358 return state->update(req); 4359 } 4360 4361 static int ahash_finup(struct ahash_request *req) 4362 { 4363 struct caam_hash_state *state = ahash_request_ctx_dma(req); 4364 4365 return state->finup(req); 4366 } 4367 4368 static int ahash_final(struct ahash_request *req) 4369 { 4370 struct caam_hash_state *state = ahash_request_ctx_dma(req); 4371 4372 return state->final(req); 4373 } 4374 4375 static int ahash_export(struct ahash_request *req, void *out) 4376 { 4377 struct caam_hash_state *state = ahash_request_ctx_dma(req); 4378 struct caam_export_state *export = out; 4379 u8 *buf = state->buf; 4380 int len = state->buflen; 4381 4382 memcpy(export->buf, buf, len); 4383 memcpy(export->caam_ctx, state->caam_ctx, sizeof(export->caam_ctx)); 4384 export->buflen = len; 4385 export->update = state->update; 4386 export->final = state->final; 4387 export->finup = state->finup; 4388 4389 return 0; 4390 } 4391 4392 static int ahash_import(struct ahash_request *req, const void *in) 4393 { 4394 struct caam_hash_state *state = ahash_request_ctx_dma(req); 4395 const struct caam_export_state *export = in; 4396 4397 memset(state, 0, sizeof(*state)); 4398 memcpy(state->buf, export->buf, export->buflen); 4399 memcpy(state->caam_ctx, export->caam_ctx, sizeof(state->caam_ctx)); 4400 state->buflen = export->buflen; 4401 state->update = export->update; 4402 state->final = export->final; 4403 state->finup = export->finup; 4404 4405 return 0; 4406 } 4407 4408 struct caam_hash_template { 4409 char name[CRYPTO_MAX_ALG_NAME]; 4410 char driver_name[CRYPTO_MAX_ALG_NAME]; 4411 char hmac_name[CRYPTO_MAX_ALG_NAME]; 4412 char hmac_driver_name[CRYPTO_MAX_ALG_NAME]; 4413 unsigned int blocksize; 4414 struct ahash_alg template_ahash; 4415 u32 alg_type; 4416 }; 4417 4418 /* ahash descriptors */ 4419 static struct caam_hash_template driver_hash[] = { 4420 { 4421 .name = "sha1", 4422 .driver_name = "sha1-caam-qi2", 4423 .hmac_name = "hmac(sha1)", 4424 .hmac_driver_name = "hmac-sha1-caam-qi2", 4425 .blocksize = SHA1_BLOCK_SIZE, 4426 .template_ahash = { 4427 .init = ahash_init, 4428 .update = ahash_update, 4429 .final = ahash_final, 4430 .finup = ahash_finup, 4431 .digest = ahash_digest, 4432 .export = ahash_export, 4433 .import = ahash_import, 4434 .setkey = ahash_setkey, 4435 .halg = { 4436 .digestsize = SHA1_DIGEST_SIZE, 4437 .statesize = sizeof(struct caam_export_state), 4438 }, 4439 }, 4440 .alg_type = OP_ALG_ALGSEL_SHA1, 4441 }, { 4442 .name = "sha224", 4443 .driver_name = "sha224-caam-qi2", 4444 .hmac_name = "hmac(sha224)", 4445 .hmac_driver_name = "hmac-sha224-caam-qi2", 4446 .blocksize = SHA224_BLOCK_SIZE, 4447 .template_ahash = { 4448 .init = ahash_init, 4449 .update = ahash_update, 4450 .final = ahash_final, 4451 .finup = ahash_finup, 4452 .digest = ahash_digest, 4453 .export = ahash_export, 4454 .import = ahash_import, 4455 .setkey = ahash_setkey, 4456 .halg = { 4457 .digestsize = SHA224_DIGEST_SIZE, 4458 .statesize = sizeof(struct caam_export_state), 4459 }, 4460 }, 4461 .alg_type = OP_ALG_ALGSEL_SHA224, 4462 }, { 4463 .name = "sha256", 4464 .driver_name = "sha256-caam-qi2", 4465 .hmac_name = "hmac(sha256)", 4466 .hmac_driver_name = "hmac-sha256-caam-qi2", 4467 .blocksize = SHA256_BLOCK_SIZE, 4468 .template_ahash = { 4469 .init = ahash_init, 4470 .update = ahash_update, 4471 .final = ahash_final, 4472 .finup = ahash_finup, 4473 .digest = ahash_digest, 4474 .export = ahash_export, 4475 .import = ahash_import, 4476 .setkey = ahash_setkey, 4477 .halg = { 4478 .digestsize = SHA256_DIGEST_SIZE, 4479 .statesize = sizeof(struct caam_export_state), 4480 }, 4481 }, 4482 .alg_type = OP_ALG_ALGSEL_SHA256, 4483 }, { 4484 .name = "sha384", 4485 .driver_name = "sha384-caam-qi2", 4486 .hmac_name = "hmac(sha384)", 4487 .hmac_driver_name = "hmac-sha384-caam-qi2", 4488 .blocksize = SHA384_BLOCK_SIZE, 4489 .template_ahash = { 4490 .init = ahash_init, 4491 .update = ahash_update, 4492 .final = ahash_final, 4493 .finup = ahash_finup, 4494 .digest = ahash_digest, 4495 .export = ahash_export, 4496 .import = ahash_import, 4497 .setkey = ahash_setkey, 4498 .halg = { 4499 .digestsize = SHA384_DIGEST_SIZE, 4500 .statesize = sizeof(struct caam_export_state), 4501 }, 4502 }, 4503 .alg_type = OP_ALG_ALGSEL_SHA384, 4504 }, { 4505 .name = "sha512", 4506 .driver_name = "sha512-caam-qi2", 4507 .hmac_name = "hmac(sha512)", 4508 .hmac_driver_name = "hmac-sha512-caam-qi2", 4509 .blocksize = SHA512_BLOCK_SIZE, 4510 .template_ahash = { 4511 .init = ahash_init, 4512 .update = ahash_update, 4513 .final = ahash_final, 4514 .finup = ahash_finup, 4515 .digest = ahash_digest, 4516 .export = ahash_export, 4517 .import = ahash_import, 4518 .setkey = ahash_setkey, 4519 .halg = { 4520 .digestsize = SHA512_DIGEST_SIZE, 4521 .statesize = sizeof(struct caam_export_state), 4522 }, 4523 }, 4524 .alg_type = OP_ALG_ALGSEL_SHA512, 4525 }, { 4526 .name = "md5", 4527 .driver_name = "md5-caam-qi2", 4528 .hmac_name = "hmac(md5)", 4529 .hmac_driver_name = "hmac-md5-caam-qi2", 4530 .blocksize = MD5_BLOCK_WORDS * 4, 4531 .template_ahash = { 4532 .init = ahash_init, 4533 .update = ahash_update, 4534 .final = ahash_final, 4535 .finup = ahash_finup, 4536 .digest = ahash_digest, 4537 .export = ahash_export, 4538 .import = ahash_import, 4539 .setkey = ahash_setkey, 4540 .halg = { 4541 .digestsize = MD5_DIGEST_SIZE, 4542 .statesize = sizeof(struct caam_export_state), 4543 }, 4544 }, 4545 .alg_type = OP_ALG_ALGSEL_MD5, 4546 } 4547 }; 4548 4549 struct caam_hash_alg { 4550 struct list_head entry; 4551 struct device *dev; 4552 int alg_type; 4553 bool is_hmac; 4554 struct ahash_alg ahash_alg; 4555 }; 4556 4557 static int caam_hash_cra_init(struct crypto_tfm *tfm) 4558 { 4559 struct crypto_ahash *ahash = __crypto_ahash_cast(tfm); 4560 struct crypto_alg *base = tfm->__crt_alg; 4561 struct hash_alg_common *halg = 4562 container_of(base, struct hash_alg_common, base); 4563 struct ahash_alg *alg = 4564 container_of(halg, struct ahash_alg, halg); 4565 struct caam_hash_alg *caam_hash = 4566 container_of(alg, struct caam_hash_alg, ahash_alg); 4567 struct caam_hash_ctx *ctx = crypto_tfm_ctx_dma(tfm); 4568 /* Sizes for MDHA running digests: MD5, SHA1, 224, 256, 384, 512 */ 4569 static const u8 runninglen[] = { HASH_MSG_LEN + MD5_DIGEST_SIZE, 4570 HASH_MSG_LEN + SHA1_DIGEST_SIZE, 4571 HASH_MSG_LEN + 32, 4572 HASH_MSG_LEN + SHA256_DIGEST_SIZE, 4573 HASH_MSG_LEN + 64, 4574 HASH_MSG_LEN + SHA512_DIGEST_SIZE }; 4575 dma_addr_t dma_addr; 4576 int i; 4577 4578 ctx->dev = caam_hash->dev; 4579 4580 if (caam_hash->is_hmac) { 4581 ctx->adata.key_dma = dma_map_single_attrs(ctx->dev, ctx->key, 4582 ARRAY_SIZE(ctx->key), 4583 DMA_TO_DEVICE, 4584 DMA_ATTR_SKIP_CPU_SYNC); 4585 if (dma_mapping_error(ctx->dev, ctx->adata.key_dma)) { 4586 dev_err(ctx->dev, "unable to map key\n"); 4587 return -ENOMEM; 4588 } 4589 } 4590 4591 dma_addr = dma_map_single_attrs(ctx->dev, ctx->flc, sizeof(ctx->flc), 4592 DMA_BIDIRECTIONAL, 4593 DMA_ATTR_SKIP_CPU_SYNC); 4594 if (dma_mapping_error(ctx->dev, dma_addr)) { 4595 dev_err(ctx->dev, "unable to map shared descriptors\n"); 4596 if (ctx->adata.key_dma) 4597 dma_unmap_single_attrs(ctx->dev, ctx->adata.key_dma, 4598 ARRAY_SIZE(ctx->key), 4599 DMA_TO_DEVICE, 4600 DMA_ATTR_SKIP_CPU_SYNC); 4601 return -ENOMEM; 4602 } 4603 4604 for (i = 0; i < HASH_NUM_OP; i++) 4605 ctx->flc_dma[i] = dma_addr + i * sizeof(ctx->flc[i]); 4606 4607 /* copy descriptor header template value */ 4608 ctx->adata.algtype = OP_TYPE_CLASS2_ALG | caam_hash->alg_type; 4609 4610 ctx->ctx_len = runninglen[(ctx->adata.algtype & 4611 OP_ALG_ALGSEL_SUBMASK) >> 4612 OP_ALG_ALGSEL_SHIFT]; 4613 4614 crypto_ahash_set_reqsize_dma(ahash, sizeof(struct caam_hash_state)); 4615 4616 /* 4617 * For keyed hash algorithms shared descriptors 4618 * will be created later in setkey() callback 4619 */ 4620 return caam_hash->is_hmac ? 0 : ahash_set_sh_desc(ahash); 4621 } 4622 4623 static void caam_hash_cra_exit(struct crypto_tfm *tfm) 4624 { 4625 struct caam_hash_ctx *ctx = crypto_tfm_ctx_dma(tfm); 4626 4627 dma_unmap_single_attrs(ctx->dev, ctx->flc_dma[0], sizeof(ctx->flc), 4628 DMA_BIDIRECTIONAL, DMA_ATTR_SKIP_CPU_SYNC); 4629 if (ctx->adata.key_dma) 4630 dma_unmap_single_attrs(ctx->dev, ctx->adata.key_dma, 4631 ARRAY_SIZE(ctx->key), DMA_TO_DEVICE, 4632 DMA_ATTR_SKIP_CPU_SYNC); 4633 } 4634 4635 static struct caam_hash_alg *caam_hash_alloc(struct device *dev, 4636 struct caam_hash_template *template, bool keyed) 4637 { 4638 struct caam_hash_alg *t_alg; 4639 struct ahash_alg *halg; 4640 struct crypto_alg *alg; 4641 4642 t_alg = kzalloc_obj(*t_alg); 4643 if (!t_alg) 4644 return ERR_PTR(-ENOMEM); 4645 4646 t_alg->ahash_alg = template->template_ahash; 4647 halg = &t_alg->ahash_alg; 4648 alg = &halg->halg.base; 4649 4650 if (keyed) { 4651 strscpy(alg->cra_name, template->hmac_name); 4652 strscpy(alg->cra_driver_name, template->hmac_driver_name); 4653 t_alg->is_hmac = true; 4654 } else { 4655 strscpy(alg->cra_name, template->name); 4656 strscpy(alg->cra_driver_name, template->driver_name); 4657 t_alg->ahash_alg.setkey = NULL; 4658 t_alg->is_hmac = false; 4659 } 4660 alg->cra_module = THIS_MODULE; 4661 alg->cra_init = caam_hash_cra_init; 4662 alg->cra_exit = caam_hash_cra_exit; 4663 alg->cra_ctxsize = sizeof(struct caam_hash_ctx) + crypto_dma_padding(); 4664 alg->cra_priority = CAAM_CRA_PRIORITY; 4665 alg->cra_blocksize = template->blocksize; 4666 alg->cra_alignmask = 0; 4667 alg->cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY; 4668 4669 t_alg->alg_type = template->alg_type; 4670 t_alg->dev = dev; 4671 4672 return t_alg; 4673 } 4674 4675 static void dpaa2_caam_fqdan_cb(struct dpaa2_io_notification_ctx *nctx) 4676 { 4677 struct dpaa2_caam_priv_per_cpu *ppriv; 4678 4679 ppriv = container_of(nctx, struct dpaa2_caam_priv_per_cpu, nctx); 4680 napi_schedule_irqoff(&ppriv->napi); 4681 } 4682 4683 static int __cold dpaa2_dpseci_dpio_setup(struct dpaa2_caam_priv *priv) 4684 { 4685 struct device *dev = priv->dev; 4686 struct dpaa2_io_notification_ctx *nctx; 4687 struct dpaa2_caam_priv_per_cpu *ppriv; 4688 int err, i = 0, cpu; 4689 4690 for_each_online_cpu(cpu) { 4691 ppriv = per_cpu_ptr(priv->ppriv, cpu); 4692 ppriv->priv = priv; 4693 nctx = &ppriv->nctx; 4694 nctx->is_cdan = 0; 4695 nctx->id = ppriv->rsp_fqid; 4696 nctx->desired_cpu = cpu; 4697 nctx->cb = dpaa2_caam_fqdan_cb; 4698 4699 /* Register notification callbacks */ 4700 ppriv->dpio = dpaa2_io_service_select(cpu); 4701 err = dpaa2_io_service_register(ppriv->dpio, nctx, dev); 4702 if (unlikely(err)) { 4703 dev_dbg(dev, "No affine DPIO for cpu %d\n", cpu); 4704 nctx->cb = NULL; 4705 /* 4706 * If no affine DPIO for this core, there's probably 4707 * none available for next cores either. Signal we want 4708 * to retry later, in case the DPIO devices weren't 4709 * probed yet. 4710 */ 4711 err = -EPROBE_DEFER; 4712 goto err; 4713 } 4714 4715 ppriv->store = dpaa2_io_store_create(DPAA2_CAAM_STORE_SIZE, 4716 dev); 4717 if (unlikely(!ppriv->store)) { 4718 dev_err(dev, "dpaa2_io_store_create() failed\n"); 4719 err = -ENOMEM; 4720 goto err; 4721 } 4722 4723 if (++i == priv->num_pairs) 4724 break; 4725 } 4726 4727 return 0; 4728 4729 err: 4730 for_each_online_cpu(cpu) { 4731 ppriv = per_cpu_ptr(priv->ppriv, cpu); 4732 if (!ppriv->nctx.cb) 4733 break; 4734 dpaa2_io_service_deregister(ppriv->dpio, &ppriv->nctx, dev); 4735 } 4736 4737 for_each_online_cpu(cpu) { 4738 ppriv = per_cpu_ptr(priv->ppriv, cpu); 4739 if (!ppriv->store) 4740 break; 4741 dpaa2_io_store_destroy(ppriv->store); 4742 } 4743 4744 return err; 4745 } 4746 4747 static void __cold dpaa2_dpseci_dpio_free(struct dpaa2_caam_priv *priv) 4748 { 4749 struct dpaa2_caam_priv_per_cpu *ppriv; 4750 int i = 0, cpu; 4751 4752 for_each_online_cpu(cpu) { 4753 ppriv = per_cpu_ptr(priv->ppriv, cpu); 4754 dpaa2_io_service_deregister(ppriv->dpio, &ppriv->nctx, 4755 priv->dev); 4756 dpaa2_io_store_destroy(ppriv->store); 4757 4758 if (++i == priv->num_pairs) 4759 return; 4760 } 4761 } 4762 4763 static int dpaa2_dpseci_bind(struct dpaa2_caam_priv *priv) 4764 { 4765 struct dpseci_rx_queue_cfg rx_queue_cfg; 4766 struct device *dev = priv->dev; 4767 struct fsl_mc_device *ls_dev = to_fsl_mc_device(dev); 4768 struct dpaa2_caam_priv_per_cpu *ppriv; 4769 int err = 0, i = 0, cpu; 4770 4771 /* Configure Rx queues */ 4772 for_each_online_cpu(cpu) { 4773 ppriv = per_cpu_ptr(priv->ppriv, cpu); 4774 4775 rx_queue_cfg.options = DPSECI_QUEUE_OPT_DEST | 4776 DPSECI_QUEUE_OPT_USER_CTX; 4777 rx_queue_cfg.order_preservation_en = 0; 4778 rx_queue_cfg.dest_cfg.dest_type = DPSECI_DEST_DPIO; 4779 rx_queue_cfg.dest_cfg.dest_id = ppriv->nctx.dpio_id; 4780 /* 4781 * Rx priority (WQ) doesn't really matter, since we use 4782 * pull mode, i.e. volatile dequeues from specific FQs 4783 */ 4784 rx_queue_cfg.dest_cfg.priority = 0; 4785 rx_queue_cfg.user_ctx = ppriv->nctx.qman64; 4786 4787 err = dpseci_set_rx_queue(priv->mc_io, 0, ls_dev->mc_handle, i, 4788 &rx_queue_cfg); 4789 if (err) { 4790 dev_err(dev, "dpseci_set_rx_queue() failed with err %d\n", 4791 err); 4792 return err; 4793 } 4794 4795 if (++i == priv->num_pairs) 4796 break; 4797 } 4798 4799 return err; 4800 } 4801 4802 static void dpaa2_dpseci_congestion_free(struct dpaa2_caam_priv *priv) 4803 { 4804 struct device *dev = priv->dev; 4805 4806 if (!priv->cscn_mem) 4807 return; 4808 4809 dma_unmap_single(dev, priv->cscn_dma, DPAA2_CSCN_SIZE, DMA_FROM_DEVICE); 4810 kfree(priv->cscn_mem); 4811 } 4812 4813 static void dpaa2_dpseci_free(struct dpaa2_caam_priv *priv) 4814 { 4815 struct device *dev = priv->dev; 4816 struct fsl_mc_device *ls_dev = to_fsl_mc_device(dev); 4817 struct dpaa2_caam_priv_per_cpu *ppriv; 4818 int i, err; 4819 4820 if (DPSECI_VER(priv->major_ver, priv->minor_ver) > DPSECI_VER(5, 3)) { 4821 err = dpseci_reset(priv->mc_io, 0, ls_dev->mc_handle); 4822 if (err) 4823 dev_err(dev, "dpseci_reset() failed\n"); 4824 } 4825 4826 for_each_cpu(i, priv->clean_mask) { 4827 ppriv = per_cpu_ptr(priv->ppriv, i); 4828 free_netdev(ppriv->net_dev); 4829 } 4830 free_cpumask_var(priv->clean_mask); 4831 4832 dpaa2_dpseci_congestion_free(priv); 4833 dpseci_close(priv->mc_io, 0, ls_dev->mc_handle); 4834 } 4835 4836 static void dpaa2_caam_process_fd(struct dpaa2_caam_priv *priv, 4837 const struct dpaa2_fd *fd) 4838 { 4839 struct caam_request *req; 4840 u32 fd_err; 4841 4842 if (dpaa2_fd_get_format(fd) != dpaa2_fd_list) { 4843 dev_err(priv->dev, "Only Frame List FD format is supported!\n"); 4844 return; 4845 } 4846 4847 fd_err = dpaa2_fd_get_ctrl(fd) & FD_CTRL_ERR_MASK; 4848 if (unlikely(fd_err)) 4849 dev_err_ratelimited(priv->dev, "FD error: %08x\n", fd_err); 4850 4851 /* 4852 * FD[ADDR] is guaranteed to be valid, irrespective of errors reported 4853 * in FD[ERR] or FD[FRC]. 4854 */ 4855 req = dpaa2_caam_iova_to_virt(priv, dpaa2_fd_get_addr(fd)); 4856 dma_unmap_single(priv->dev, req->fd_flt_dma, sizeof(req->fd_flt), 4857 DMA_BIDIRECTIONAL); 4858 req->cbk(req->ctx, dpaa2_fd_get_frc(fd)); 4859 } 4860 4861 static int dpaa2_caam_pull_fq(struct dpaa2_caam_priv_per_cpu *ppriv) 4862 { 4863 int err; 4864 4865 /* Retry while portal is busy */ 4866 do { 4867 err = dpaa2_io_service_pull_fq(ppriv->dpio, ppriv->rsp_fqid, 4868 ppriv->store); 4869 } while (err == -EBUSY); 4870 4871 if (unlikely(err)) 4872 dev_err(ppriv->priv->dev, "dpaa2_io_service_pull err %d", err); 4873 4874 return err; 4875 } 4876 4877 static int dpaa2_caam_store_consume(struct dpaa2_caam_priv_per_cpu *ppriv) 4878 { 4879 struct dpaa2_dq *dq; 4880 int cleaned = 0, is_last; 4881 4882 do { 4883 dq = dpaa2_io_store_next(ppriv->store, &is_last); 4884 if (unlikely(!dq)) { 4885 if (unlikely(!is_last)) { 4886 dev_dbg(ppriv->priv->dev, 4887 "FQ %d returned no valid frames\n", 4888 ppriv->rsp_fqid); 4889 /* 4890 * MUST retry until we get some sort of 4891 * valid response token (be it "empty dequeue" 4892 * or a valid frame). 4893 */ 4894 continue; 4895 } 4896 break; 4897 } 4898 4899 /* Process FD */ 4900 dpaa2_caam_process_fd(ppriv->priv, dpaa2_dq_fd(dq)); 4901 cleaned++; 4902 } while (!is_last); 4903 4904 return cleaned; 4905 } 4906 4907 static int dpaa2_dpseci_poll(struct napi_struct *napi, int budget) 4908 { 4909 struct dpaa2_caam_priv_per_cpu *ppriv; 4910 struct dpaa2_caam_priv *priv; 4911 int err, cleaned = 0, store_cleaned; 4912 4913 ppriv = container_of(napi, struct dpaa2_caam_priv_per_cpu, napi); 4914 priv = ppriv->priv; 4915 4916 if (unlikely(dpaa2_caam_pull_fq(ppriv))) 4917 return 0; 4918 4919 do { 4920 store_cleaned = dpaa2_caam_store_consume(ppriv); 4921 cleaned += store_cleaned; 4922 4923 if (store_cleaned == 0 || 4924 cleaned > budget - DPAA2_CAAM_STORE_SIZE) 4925 break; 4926 4927 /* Try to dequeue some more */ 4928 err = dpaa2_caam_pull_fq(ppriv); 4929 if (unlikely(err)) 4930 break; 4931 } while (1); 4932 4933 if (cleaned < budget) { 4934 napi_complete_done(napi, cleaned); 4935 err = dpaa2_io_service_rearm(ppriv->dpio, &ppriv->nctx); 4936 if (unlikely(err)) 4937 dev_err(priv->dev, "Notification rearm failed: %d\n", 4938 err); 4939 } 4940 4941 return cleaned; 4942 } 4943 4944 static int dpaa2_dpseci_congestion_setup(struct dpaa2_caam_priv *priv, 4945 u16 token) 4946 { 4947 struct dpseci_congestion_notification_cfg cong_notif_cfg = { 0 }; 4948 struct device *dev = priv->dev; 4949 unsigned int alignmask; 4950 int err; 4951 4952 /* 4953 * Congestion group feature supported starting with DPSECI API v5.1 4954 * and only when object has been created with this capability. 4955 */ 4956 if ((DPSECI_VER(priv->major_ver, priv->minor_ver) < DPSECI_VER(5, 1)) || 4957 !(priv->dpseci_attr.options & DPSECI_OPT_HAS_CG)) 4958 return 0; 4959 4960 alignmask = DPAA2_CSCN_ALIGN - 1; 4961 alignmask |= dma_get_cache_alignment() - 1; 4962 priv->cscn_mem = kzalloc(ALIGN(DPAA2_CSCN_SIZE, alignmask + 1), 4963 GFP_KERNEL); 4964 if (!priv->cscn_mem) 4965 return -ENOMEM; 4966 4967 priv->cscn_dma = dma_map_single(dev, priv->cscn_mem, 4968 DPAA2_CSCN_SIZE, DMA_FROM_DEVICE); 4969 if (dma_mapping_error(dev, priv->cscn_dma)) { 4970 dev_err(dev, "Error mapping CSCN memory area\n"); 4971 err = -ENOMEM; 4972 goto err_dma_map; 4973 } 4974 4975 cong_notif_cfg.units = DPSECI_CONGESTION_UNIT_BYTES; 4976 cong_notif_cfg.threshold_entry = DPAA2_SEC_CONG_ENTRY_THRESH; 4977 cong_notif_cfg.threshold_exit = DPAA2_SEC_CONG_EXIT_THRESH; 4978 cong_notif_cfg.message_ctx = (uintptr_t)priv; 4979 cong_notif_cfg.message_iova = priv->cscn_dma; 4980 cong_notif_cfg.notification_mode = DPSECI_CGN_MODE_WRITE_MEM_ON_ENTER | 4981 DPSECI_CGN_MODE_WRITE_MEM_ON_EXIT | 4982 DPSECI_CGN_MODE_COHERENT_WRITE; 4983 4984 err = dpseci_set_congestion_notification(priv->mc_io, 0, token, 4985 &cong_notif_cfg); 4986 if (err) { 4987 dev_err(dev, "dpseci_set_congestion_notification failed\n"); 4988 goto err_set_cong; 4989 } 4990 4991 return 0; 4992 4993 err_set_cong: 4994 dma_unmap_single(dev, priv->cscn_dma, DPAA2_CSCN_SIZE, DMA_FROM_DEVICE); 4995 err_dma_map: 4996 kfree(priv->cscn_mem); 4997 4998 return err; 4999 } 5000 5001 static void free_dpaa2_pcpu_netdev(struct dpaa2_caam_priv *priv, const cpumask_t *cpus) 5002 { 5003 struct dpaa2_caam_priv_per_cpu *ppriv; 5004 int i; 5005 5006 for_each_cpu(i, cpus) { 5007 ppriv = per_cpu_ptr(priv->ppriv, i); 5008 free_netdev(ppriv->net_dev); 5009 } 5010 } 5011 5012 static int __cold dpaa2_dpseci_setup(struct fsl_mc_device *ls_dev) 5013 { 5014 struct device *dev = &ls_dev->dev; 5015 struct dpaa2_caam_priv *priv; 5016 struct dpaa2_caam_priv_per_cpu *ppriv; 5017 int err, cpu; 5018 u8 i; 5019 5020 err = -ENOMEM; 5021 priv = dev_get_drvdata(dev); 5022 5023 if (!zalloc_cpumask_var(&priv->clean_mask, GFP_KERNEL)) 5024 goto err_cpumask; 5025 5026 priv->dev = dev; 5027 priv->dpsec_id = ls_dev->obj_desc.id; 5028 5029 /* Get a handle for the DPSECI this interface is associate with */ 5030 err = dpseci_open(priv->mc_io, 0, priv->dpsec_id, &ls_dev->mc_handle); 5031 if (err) { 5032 dev_err(dev, "dpseci_open() failed: %d\n", err); 5033 goto err_open; 5034 } 5035 5036 err = dpseci_get_api_version(priv->mc_io, 0, &priv->major_ver, 5037 &priv->minor_ver); 5038 if (err) { 5039 dev_err(dev, "dpseci_get_api_version() failed\n"); 5040 goto err_get_vers; 5041 } 5042 5043 dev_info(dev, "dpseci v%d.%d\n", priv->major_ver, priv->minor_ver); 5044 5045 if (DPSECI_VER(priv->major_ver, priv->minor_ver) > DPSECI_VER(5, 3)) { 5046 int enabled; 5047 5048 err = dpseci_is_enabled(priv->mc_io, 0, ls_dev->mc_handle, 5049 &enabled); 5050 if (err) { 5051 dev_err(dev, "dpseci_is_enabled() failed\n"); 5052 goto err_get_vers; 5053 } 5054 5055 if (enabled) { 5056 err = dpseci_reset(priv->mc_io, 0, ls_dev->mc_handle); 5057 if (err) { 5058 dev_err(dev, "dpseci_reset() failed\n"); 5059 goto err_get_vers; 5060 } 5061 } 5062 } 5063 5064 err = dpseci_get_attributes(priv->mc_io, 0, ls_dev->mc_handle, 5065 &priv->dpseci_attr); 5066 if (err) { 5067 dev_err(dev, "dpseci_get_attributes() failed\n"); 5068 goto err_get_vers; 5069 } 5070 5071 err = dpseci_get_sec_attr(priv->mc_io, 0, ls_dev->mc_handle, 5072 &priv->sec_attr); 5073 if (err) { 5074 dev_err(dev, "dpseci_get_sec_attr() failed\n"); 5075 goto err_get_vers; 5076 } 5077 5078 err = dpaa2_dpseci_congestion_setup(priv, ls_dev->mc_handle); 5079 if (err) { 5080 dev_err(dev, "setup_congestion() failed\n"); 5081 goto err_get_vers; 5082 } 5083 5084 priv->num_pairs = min(priv->dpseci_attr.num_rx_queues, 5085 priv->dpseci_attr.num_tx_queues); 5086 if (priv->num_pairs > num_online_cpus()) { 5087 dev_warn(dev, "%d queues won't be used\n", 5088 priv->num_pairs - num_online_cpus()); 5089 priv->num_pairs = num_online_cpus(); 5090 } 5091 5092 for (i = 0; i < priv->dpseci_attr.num_rx_queues; i++) { 5093 err = dpseci_get_rx_queue(priv->mc_io, 0, ls_dev->mc_handle, i, 5094 &priv->rx_queue_attr[i]); 5095 if (err) { 5096 dev_err(dev, "dpseci_get_rx_queue() failed\n"); 5097 goto err_get_rx_queue; 5098 } 5099 } 5100 5101 for (i = 0; i < priv->dpseci_attr.num_tx_queues; i++) { 5102 err = dpseci_get_tx_queue(priv->mc_io, 0, ls_dev->mc_handle, i, 5103 &priv->tx_queue_attr[i]); 5104 if (err) { 5105 dev_err(dev, "dpseci_get_tx_queue() failed\n"); 5106 goto err_get_rx_queue; 5107 } 5108 } 5109 5110 i = 0; 5111 for_each_online_cpu(cpu) { 5112 u8 j; 5113 5114 j = i % priv->num_pairs; 5115 5116 ppriv = per_cpu_ptr(priv->ppriv, cpu); 5117 ppriv->req_fqid = priv->tx_queue_attr[j].fqid; 5118 5119 /* 5120 * Allow all cores to enqueue, while only some of them 5121 * will take part in dequeuing. 5122 */ 5123 if (++i > priv->num_pairs) 5124 continue; 5125 5126 ppriv->rsp_fqid = priv->rx_queue_attr[j].fqid; 5127 ppriv->prio = j; 5128 5129 dev_dbg(dev, "pair %d: rx queue %d, tx queue %d\n", j, 5130 priv->rx_queue_attr[j].fqid, 5131 priv->tx_queue_attr[j].fqid); 5132 5133 ppriv->net_dev = alloc_netdev_dummy(0); 5134 if (!ppriv->net_dev) { 5135 err = -ENOMEM; 5136 goto err_alloc_netdev; 5137 } 5138 cpumask_set_cpu(cpu, priv->clean_mask); 5139 ppriv->net_dev->dev = *dev; 5140 5141 netif_napi_add_tx_weight(ppriv->net_dev, &ppriv->napi, 5142 dpaa2_dpseci_poll, 5143 DPAA2_CAAM_NAPI_WEIGHT); 5144 } 5145 5146 return 0; 5147 5148 err_alloc_netdev: 5149 free_dpaa2_pcpu_netdev(priv, priv->clean_mask); 5150 err_get_rx_queue: 5151 dpaa2_dpseci_congestion_free(priv); 5152 err_get_vers: 5153 dpseci_close(priv->mc_io, 0, ls_dev->mc_handle); 5154 err_open: 5155 free_cpumask_var(priv->clean_mask); 5156 err_cpumask: 5157 return err; 5158 } 5159 5160 static int dpaa2_dpseci_enable(struct dpaa2_caam_priv *priv) 5161 { 5162 struct device *dev = priv->dev; 5163 struct fsl_mc_device *ls_dev = to_fsl_mc_device(dev); 5164 struct dpaa2_caam_priv_per_cpu *ppriv; 5165 int i; 5166 5167 for (i = 0; i < priv->num_pairs; i++) { 5168 ppriv = per_cpu_ptr(priv->ppriv, i); 5169 napi_enable(&ppriv->napi); 5170 } 5171 5172 return dpseci_enable(priv->mc_io, 0, ls_dev->mc_handle); 5173 } 5174 5175 static int __cold dpaa2_dpseci_disable(struct dpaa2_caam_priv *priv) 5176 { 5177 struct device *dev = priv->dev; 5178 struct dpaa2_caam_priv_per_cpu *ppriv; 5179 struct fsl_mc_device *ls_dev = to_fsl_mc_device(dev); 5180 int i, err = 0, enabled; 5181 5182 err = dpseci_disable(priv->mc_io, 0, ls_dev->mc_handle); 5183 if (err) { 5184 dev_err(dev, "dpseci_disable() failed\n"); 5185 return err; 5186 } 5187 5188 err = dpseci_is_enabled(priv->mc_io, 0, ls_dev->mc_handle, &enabled); 5189 if (err) { 5190 dev_err(dev, "dpseci_is_enabled() failed\n"); 5191 return err; 5192 } 5193 5194 dev_dbg(dev, "disable: %s\n", str_false_true(enabled)); 5195 5196 for (i = 0; i < priv->num_pairs; i++) { 5197 ppriv = per_cpu_ptr(priv->ppriv, i); 5198 napi_disable(&ppriv->napi); 5199 netif_napi_del(&ppriv->napi); 5200 } 5201 5202 return 0; 5203 } 5204 5205 static struct list_head hash_list; 5206 5207 static int dpaa2_caam_probe(struct fsl_mc_device *dpseci_dev) 5208 { 5209 struct device *dev; 5210 struct dpaa2_caam_priv *priv; 5211 int i, err = 0; 5212 bool registered = false; 5213 5214 /* 5215 * There is no way to get CAAM endianness - there is no direct register 5216 * space access and MC f/w does not provide this attribute. 5217 * All DPAA2-based SoCs have little endian CAAM, thus hard-code this 5218 * property. 5219 */ 5220 caam_little_end = true; 5221 5222 caam_imx = false; 5223 5224 dev = &dpseci_dev->dev; 5225 5226 priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL); 5227 if (!priv) 5228 return -ENOMEM; 5229 5230 dev_set_drvdata(dev, priv); 5231 5232 priv->domain = iommu_get_domain_for_dev(dev); 5233 5234 qi_cache = kmem_cache_create("dpaa2_caamqicache", CAAM_QI_MEMCACHE_SIZE, 5235 0, 0, NULL); 5236 if (!qi_cache) { 5237 dev_err(dev, "Can't allocate SEC cache\n"); 5238 return -ENOMEM; 5239 } 5240 5241 err = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(49)); 5242 if (err) { 5243 dev_err(dev, "dma_set_mask_and_coherent() failed\n"); 5244 goto err_dma_mask; 5245 } 5246 5247 /* Obtain a MC portal */ 5248 err = fsl_mc_portal_allocate(dpseci_dev, 0, &priv->mc_io); 5249 if (err) { 5250 if (err == -ENXIO) 5251 err = -EPROBE_DEFER; 5252 else 5253 dev_err(dev, "MC portal allocation failed\n"); 5254 5255 goto err_dma_mask; 5256 } 5257 5258 priv->ppriv = alloc_percpu(*priv->ppriv); 5259 if (!priv->ppriv) { 5260 dev_err(dev, "alloc_percpu() failed\n"); 5261 err = -ENOMEM; 5262 goto err_alloc_ppriv; 5263 } 5264 5265 /* DPSECI initialization */ 5266 err = dpaa2_dpseci_setup(dpseci_dev); 5267 if (err) { 5268 dev_err(dev, "dpaa2_dpseci_setup() failed\n"); 5269 goto err_dpseci_setup; 5270 } 5271 5272 /* DPIO */ 5273 err = dpaa2_dpseci_dpio_setup(priv); 5274 if (err) { 5275 dev_err_probe(dev, err, "dpaa2_dpseci_dpio_setup() failed\n"); 5276 goto err_dpio_setup; 5277 } 5278 5279 /* DPSECI binding to DPIO */ 5280 err = dpaa2_dpseci_bind(priv); 5281 if (err) { 5282 dev_err(dev, "dpaa2_dpseci_bind() failed\n"); 5283 goto err_bind; 5284 } 5285 5286 /* DPSECI enable */ 5287 err = dpaa2_dpseci_enable(priv); 5288 if (err) { 5289 dev_err(dev, "dpaa2_dpseci_enable() failed\n"); 5290 goto err_bind; 5291 } 5292 5293 dpaa2_dpseci_debugfs_init(priv); 5294 5295 /* register crypto algorithms the device supports */ 5296 for (i = 0; i < ARRAY_SIZE(driver_algs); i++) { 5297 struct caam_skcipher_alg *t_alg = driver_algs + i; 5298 u32 alg_sel = t_alg->caam.class1_alg_type & OP_ALG_ALGSEL_MASK; 5299 5300 /* Skip DES algorithms if not supported by device */ 5301 if (!priv->sec_attr.des_acc_num && 5302 (alg_sel == OP_ALG_ALGSEL_3DES || 5303 alg_sel == OP_ALG_ALGSEL_DES)) 5304 continue; 5305 5306 /* Skip AES algorithms if not supported by device */ 5307 if (!priv->sec_attr.aes_acc_num && 5308 alg_sel == OP_ALG_ALGSEL_AES) 5309 continue; 5310 5311 /* Skip CHACHA20 algorithms if not supported by device */ 5312 if (alg_sel == OP_ALG_ALGSEL_CHACHA20 && 5313 !priv->sec_attr.ccha_acc_num) 5314 continue; 5315 5316 t_alg->caam.dev = dev; 5317 caam_skcipher_alg_init(t_alg); 5318 5319 err = crypto_register_skcipher(&t_alg->skcipher); 5320 if (err) { 5321 dev_warn(dev, "%s alg registration failed: %d\n", 5322 t_alg->skcipher.base.cra_driver_name, err); 5323 continue; 5324 } 5325 5326 t_alg->registered = true; 5327 registered = true; 5328 } 5329 5330 for (i = 0; i < ARRAY_SIZE(driver_aeads); i++) { 5331 struct caam_aead_alg *t_alg = driver_aeads + i; 5332 u32 c1_alg_sel = t_alg->caam.class1_alg_type & 5333 OP_ALG_ALGSEL_MASK; 5334 u32 c2_alg_sel = t_alg->caam.class2_alg_type & 5335 OP_ALG_ALGSEL_MASK; 5336 5337 /* Skip DES algorithms if not supported by device */ 5338 if (!priv->sec_attr.des_acc_num && 5339 (c1_alg_sel == OP_ALG_ALGSEL_3DES || 5340 c1_alg_sel == OP_ALG_ALGSEL_DES)) 5341 continue; 5342 5343 /* Skip AES algorithms if not supported by device */ 5344 if (!priv->sec_attr.aes_acc_num && 5345 c1_alg_sel == OP_ALG_ALGSEL_AES) 5346 continue; 5347 5348 /* Skip CHACHA20 algorithms if not supported by device */ 5349 if (c1_alg_sel == OP_ALG_ALGSEL_CHACHA20 && 5350 !priv->sec_attr.ccha_acc_num) 5351 continue; 5352 5353 /* Skip POLY1305 algorithms if not supported by device */ 5354 if (c2_alg_sel == OP_ALG_ALGSEL_POLY1305 && 5355 !priv->sec_attr.ptha_acc_num) 5356 continue; 5357 5358 /* 5359 * Skip algorithms requiring message digests 5360 * if MD not supported by device. 5361 */ 5362 if ((c2_alg_sel & ~OP_ALG_ALGSEL_SUBMASK) == 0x40 && 5363 !priv->sec_attr.md_acc_num) 5364 continue; 5365 5366 t_alg->caam.dev = dev; 5367 caam_aead_alg_init(t_alg); 5368 5369 err = crypto_register_aead(&t_alg->aead); 5370 if (err) { 5371 dev_warn(dev, "%s alg registration failed: %d\n", 5372 t_alg->aead.base.cra_driver_name, err); 5373 continue; 5374 } 5375 5376 t_alg->registered = true; 5377 registered = true; 5378 } 5379 if (registered) 5380 dev_info(dev, "algorithms registered in /proc/crypto\n"); 5381 5382 /* register hash algorithms the device supports */ 5383 INIT_LIST_HEAD(&hash_list); 5384 5385 /* 5386 * Skip registration of any hashing algorithms if MD block 5387 * is not present. 5388 */ 5389 if (!priv->sec_attr.md_acc_num) 5390 return 0; 5391 5392 for (i = 0; i < ARRAY_SIZE(driver_hash); i++) { 5393 struct caam_hash_alg *t_alg; 5394 struct caam_hash_template *alg = driver_hash + i; 5395 5396 /* register hmac version */ 5397 t_alg = caam_hash_alloc(dev, alg, true); 5398 if (IS_ERR(t_alg)) { 5399 err = PTR_ERR(t_alg); 5400 dev_warn(dev, "%s hash alg allocation failed: %d\n", 5401 alg->hmac_driver_name, err); 5402 continue; 5403 } 5404 5405 err = crypto_register_ahash(&t_alg->ahash_alg); 5406 if (err) { 5407 dev_warn(dev, "%s alg registration failed: %d\n", 5408 t_alg->ahash_alg.halg.base.cra_driver_name, 5409 err); 5410 kfree(t_alg); 5411 } else { 5412 list_add_tail(&t_alg->entry, &hash_list); 5413 } 5414 5415 /* register unkeyed version */ 5416 t_alg = caam_hash_alloc(dev, alg, false); 5417 if (IS_ERR(t_alg)) { 5418 err = PTR_ERR(t_alg); 5419 dev_warn(dev, "%s alg allocation failed: %d\n", 5420 alg->driver_name, err); 5421 continue; 5422 } 5423 5424 err = crypto_register_ahash(&t_alg->ahash_alg); 5425 if (err) { 5426 dev_warn(dev, "%s alg registration failed: %d\n", 5427 t_alg->ahash_alg.halg.base.cra_driver_name, 5428 err); 5429 kfree(t_alg); 5430 } else { 5431 list_add_tail(&t_alg->entry, &hash_list); 5432 } 5433 } 5434 if (!list_empty(&hash_list)) 5435 dev_info(dev, "hash algorithms registered in /proc/crypto\n"); 5436 5437 return err; 5438 5439 err_bind: 5440 dpaa2_dpseci_dpio_free(priv); 5441 err_dpio_setup: 5442 dpaa2_dpseci_free(priv); 5443 err_dpseci_setup: 5444 free_percpu(priv->ppriv); 5445 err_alloc_ppriv: 5446 fsl_mc_portal_free(priv->mc_io); 5447 err_dma_mask: 5448 kmem_cache_destroy(qi_cache); 5449 5450 return err; 5451 } 5452 5453 static void __cold dpaa2_caam_remove(struct fsl_mc_device *ls_dev) 5454 { 5455 struct device *dev; 5456 struct dpaa2_caam_priv *priv; 5457 int i; 5458 5459 dev = &ls_dev->dev; 5460 priv = dev_get_drvdata(dev); 5461 5462 dpaa2_dpseci_debugfs_exit(priv); 5463 5464 for (i = 0; i < ARRAY_SIZE(driver_aeads); i++) { 5465 struct caam_aead_alg *t_alg = driver_aeads + i; 5466 5467 if (t_alg->registered) 5468 crypto_unregister_aead(&t_alg->aead); 5469 } 5470 5471 for (i = 0; i < ARRAY_SIZE(driver_algs); i++) { 5472 struct caam_skcipher_alg *t_alg = driver_algs + i; 5473 5474 if (t_alg->registered) 5475 crypto_unregister_skcipher(&t_alg->skcipher); 5476 } 5477 5478 if (hash_list.next) { 5479 struct caam_hash_alg *t_hash_alg, *p; 5480 5481 list_for_each_entry_safe(t_hash_alg, p, &hash_list, entry) { 5482 crypto_unregister_ahash(&t_hash_alg->ahash_alg); 5483 list_del(&t_hash_alg->entry); 5484 kfree(t_hash_alg); 5485 } 5486 } 5487 5488 dpaa2_dpseci_disable(priv); 5489 dpaa2_dpseci_dpio_free(priv); 5490 dpaa2_dpseci_free(priv); 5491 free_percpu(priv->ppriv); 5492 fsl_mc_portal_free(priv->mc_io); 5493 kmem_cache_destroy(qi_cache); 5494 } 5495 5496 int dpaa2_caam_enqueue(struct device *dev, struct caam_request *req) 5497 { 5498 struct dpaa2_fd fd; 5499 struct dpaa2_caam_priv *priv = dev_get_drvdata(dev); 5500 struct dpaa2_caam_priv_per_cpu *ppriv; 5501 int err = 0, i; 5502 5503 if (IS_ERR(req)) 5504 return PTR_ERR(req); 5505 5506 if (priv->cscn_mem) { 5507 dma_sync_single_for_cpu(priv->dev, priv->cscn_dma, 5508 DPAA2_CSCN_SIZE, 5509 DMA_FROM_DEVICE); 5510 if (unlikely(dpaa2_cscn_state_congested(priv->cscn_mem))) { 5511 dev_dbg_ratelimited(dev, "Dropping request\n"); 5512 return -EBUSY; 5513 } 5514 } 5515 5516 dpaa2_fl_set_flc(&req->fd_flt[1], req->flc_dma); 5517 5518 req->fd_flt_dma = dma_map_single(dev, req->fd_flt, sizeof(req->fd_flt), 5519 DMA_BIDIRECTIONAL); 5520 if (dma_mapping_error(dev, req->fd_flt_dma)) { 5521 dev_err(dev, "DMA mapping error for QI enqueue request\n"); 5522 goto err_out; 5523 } 5524 5525 memset(&fd, 0, sizeof(fd)); 5526 dpaa2_fd_set_format(&fd, dpaa2_fd_list); 5527 dpaa2_fd_set_addr(&fd, req->fd_flt_dma); 5528 dpaa2_fd_set_len(&fd, dpaa2_fl_get_len(&req->fd_flt[1])); 5529 dpaa2_fd_set_flc(&fd, req->flc_dma); 5530 5531 ppriv = raw_cpu_ptr(priv->ppriv); 5532 for (i = 0; i < (priv->dpseci_attr.num_tx_queues << 1); i++) { 5533 err = dpaa2_io_service_enqueue_fq(ppriv->dpio, ppriv->req_fqid, 5534 &fd); 5535 if (err != -EBUSY) 5536 break; 5537 5538 cpu_relax(); 5539 } 5540 5541 if (unlikely(err)) { 5542 dev_err_ratelimited(dev, "Error enqueuing frame: %d\n", err); 5543 goto err_out; 5544 } 5545 5546 return -EINPROGRESS; 5547 5548 err_out: 5549 dma_unmap_single(dev, req->fd_flt_dma, sizeof(req->fd_flt), 5550 DMA_BIDIRECTIONAL); 5551 return -EIO; 5552 } 5553 EXPORT_SYMBOL(dpaa2_caam_enqueue); 5554 5555 static const struct fsl_mc_device_id dpaa2_caam_match_id_table[] = { 5556 { 5557 .vendor = FSL_MC_VENDOR_FREESCALE, 5558 .obj_type = "dpseci", 5559 }, 5560 { .vendor = 0x0 } 5561 }; 5562 MODULE_DEVICE_TABLE(fslmc, dpaa2_caam_match_id_table); 5563 5564 static struct fsl_mc_driver dpaa2_caam_driver = { 5565 .driver = { 5566 .name = KBUILD_MODNAME, 5567 .owner = THIS_MODULE, 5568 }, 5569 .probe = dpaa2_caam_probe, 5570 .remove = dpaa2_caam_remove, 5571 .match_id_table = dpaa2_caam_match_id_table 5572 }; 5573 5574 MODULE_LICENSE("Dual BSD/GPL"); 5575 MODULE_AUTHOR("Freescale Semiconductor, Inc"); 5576 MODULE_DESCRIPTION("Freescale DPAA2 CAAM Driver"); 5577 5578 module_fsl_mc_driver(dpaa2_caam_driver); 5579