1 /* 2 * This file is part of the Chelsio T6 Crypto driver for Linux. 3 * 4 * Copyright (c) 2003-2016 Chelsio Communications, Inc. All rights reserved. 5 * 6 * This software is available to you under a choice of one of two 7 * licenses. You may choose to be licensed under the terms of the GNU 8 * General Public License (GPL) Version 2, available from the file 9 * COPYING in the main directory of this source tree, or the 10 * OpenIB.org BSD license below: 11 * 12 * Redistribution and use in source and binary forms, with or 13 * without modification, are permitted provided that the following 14 * conditions are met: 15 * 16 * - Redistributions of source code must retain the above 17 * copyright notice, this list of conditions and the following 18 * disclaimer. 19 * 20 * - Redistributions in binary form must reproduce the above 21 * copyright notice, this list of conditions and the following 22 * disclaimer in the documentation and/or other materials 23 * provided with the distribution. 24 * 25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 26 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 27 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 28 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 29 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 30 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 31 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 32 * SOFTWARE. 33 * 34 * Written and Maintained by: 35 * Manoj Malviya (manojmalviya@chelsio.com) 36 * Atul Gupta (atul.gupta@chelsio.com) 37 * Jitendra Lulla (jlulla@chelsio.com) 38 * Yeshaswi M R Gowda (yeshaswi@chelsio.com) 39 * Harsh Jain (harsh@chelsio.com) 40 */ 41 42 #define pr_fmt(fmt) "chcr:" fmt 43 44 #include <linux/kernel.h> 45 #include <linux/module.h> 46 #include <linux/crypto.h> 47 #include <linux/cryptohash.h> 48 #include <linux/skbuff.h> 49 #include <linux/rtnetlink.h> 50 #include <linux/highmem.h> 51 #include <linux/scatterlist.h> 52 53 #include <crypto/aes.h> 54 #include <crypto/algapi.h> 55 #include <crypto/hash.h> 56 #include <crypto/sha.h> 57 #include <crypto/authenc.h> 58 #include <crypto/ctr.h> 59 #include <crypto/gf128mul.h> 60 #include <crypto/internal/aead.h> 61 #include <crypto/null.h> 62 #include <crypto/internal/skcipher.h> 63 #include <crypto/aead.h> 64 #include <crypto/scatterwalk.h> 65 #include <crypto/internal/hash.h> 66 67 #include "t4fw_api.h" 68 #include "t4_msg.h" 69 #include "chcr_core.h" 70 #include "chcr_algo.h" 71 #include "chcr_crypto.h" 72 73 static inline struct chcr_aead_ctx *AEAD_CTX(struct chcr_context *ctx) 74 { 75 return ctx->crypto_ctx->aeadctx; 76 } 77 78 static inline struct ablk_ctx *ABLK_CTX(struct chcr_context *ctx) 79 { 80 return ctx->crypto_ctx->ablkctx; 81 } 82 83 static inline struct hmac_ctx *HMAC_CTX(struct chcr_context *ctx) 84 { 85 return ctx->crypto_ctx->hmacctx; 86 } 87 88 static inline struct chcr_gcm_ctx *GCM_CTX(struct chcr_aead_ctx *gctx) 89 { 90 return gctx->ctx->gcm; 91 } 92 93 static inline struct chcr_authenc_ctx *AUTHENC_CTX(struct chcr_aead_ctx *gctx) 94 { 95 return gctx->ctx->authenc; 96 } 97 98 static inline struct uld_ctx *ULD_CTX(struct chcr_context *ctx) 99 { 100 return ctx->dev->u_ctx; 101 } 102 103 static inline int is_ofld_imm(const struct sk_buff *skb) 104 { 105 return (skb->len <= CRYPTO_MAX_IMM_TX_PKT_LEN); 106 } 107 108 /* 109 * sgl_len - calculates the size of an SGL of the given capacity 110 * @n: the number of SGL entries 111 * Calculates the number of flits needed for a scatter/gather list that 112 * can hold the given number of entries. 113 */ 114 static inline unsigned int sgl_len(unsigned int n) 115 { 116 n--; 117 return (3 * n) / 2 + (n & 1) + 2; 118 } 119 120 static void chcr_verify_tag(struct aead_request *req, u8 *input, int *err) 121 { 122 u8 temp[SHA512_DIGEST_SIZE]; 123 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 124 int authsize = crypto_aead_authsize(tfm); 125 struct cpl_fw6_pld *fw6_pld; 126 int cmp = 0; 127 128 fw6_pld = (struct cpl_fw6_pld *)input; 129 if ((get_aead_subtype(tfm) == CRYPTO_ALG_SUB_TYPE_AEAD_RFC4106) || 130 (get_aead_subtype(tfm) == CRYPTO_ALG_SUB_TYPE_AEAD_GCM)) { 131 cmp = crypto_memneq(&fw6_pld->data[2], (fw6_pld + 1), authsize); 132 } else { 133 134 sg_pcopy_to_buffer(req->src, sg_nents(req->src), temp, 135 authsize, req->assoclen + 136 req->cryptlen - authsize); 137 cmp = crypto_memneq(temp, (fw6_pld + 1), authsize); 138 } 139 if (cmp) 140 *err = -EBADMSG; 141 else 142 *err = 0; 143 } 144 145 /* 146 * chcr_handle_resp - Unmap the DMA buffers associated with the request 147 * @req: crypto request 148 */ 149 int chcr_handle_resp(struct crypto_async_request *req, unsigned char *input, 150 int err) 151 { 152 struct crypto_tfm *tfm = req->tfm; 153 struct chcr_context *ctx = crypto_tfm_ctx(tfm); 154 struct uld_ctx *u_ctx = ULD_CTX(ctx); 155 struct chcr_req_ctx ctx_req; 156 unsigned int digestsize, updated_digestsize; 157 struct adapter *adap = padap(ctx->dev); 158 159 switch (tfm->__crt_alg->cra_flags & CRYPTO_ALG_TYPE_MASK) { 160 case CRYPTO_ALG_TYPE_AEAD: 161 ctx_req.req.aead_req = aead_request_cast(req); 162 ctx_req.ctx.reqctx = aead_request_ctx(ctx_req.req.aead_req); 163 dma_unmap_sg(&u_ctx->lldi.pdev->dev, ctx_req.ctx.reqctx->dst, 164 ctx_req.ctx.reqctx->dst_nents, DMA_FROM_DEVICE); 165 if (ctx_req.ctx.reqctx->skb) { 166 kfree_skb(ctx_req.ctx.reqctx->skb); 167 ctx_req.ctx.reqctx->skb = NULL; 168 } 169 free_new_sg(ctx_req.ctx.reqctx->newdstsg); 170 ctx_req.ctx.reqctx->newdstsg = NULL; 171 if (ctx_req.ctx.reqctx->verify == VERIFY_SW) { 172 chcr_verify_tag(ctx_req.req.aead_req, input, 173 &err); 174 ctx_req.ctx.reqctx->verify = VERIFY_HW; 175 } 176 ctx_req.req.aead_req->base.complete(req, err); 177 break; 178 179 case CRYPTO_ALG_TYPE_ABLKCIPHER: 180 err = chcr_handle_cipher_resp(ablkcipher_request_cast(req), 181 input, err); 182 break; 183 184 case CRYPTO_ALG_TYPE_AHASH: 185 ctx_req.req.ahash_req = ahash_request_cast(req); 186 ctx_req.ctx.ahash_ctx = 187 ahash_request_ctx(ctx_req.req.ahash_req); 188 digestsize = 189 crypto_ahash_digestsize(crypto_ahash_reqtfm( 190 ctx_req.req.ahash_req)); 191 updated_digestsize = digestsize; 192 if (digestsize == SHA224_DIGEST_SIZE) 193 updated_digestsize = SHA256_DIGEST_SIZE; 194 else if (digestsize == SHA384_DIGEST_SIZE) 195 updated_digestsize = SHA512_DIGEST_SIZE; 196 if (ctx_req.ctx.ahash_ctx->skb) { 197 kfree_skb(ctx_req.ctx.ahash_ctx->skb); 198 ctx_req.ctx.ahash_ctx->skb = NULL; 199 } 200 if (ctx_req.ctx.ahash_ctx->result == 1) { 201 ctx_req.ctx.ahash_ctx->result = 0; 202 memcpy(ctx_req.req.ahash_req->result, input + 203 sizeof(struct cpl_fw6_pld), 204 digestsize); 205 } else { 206 memcpy(ctx_req.ctx.ahash_ctx->partial_hash, input + 207 sizeof(struct cpl_fw6_pld), 208 updated_digestsize); 209 } 210 ctx_req.req.ahash_req->base.complete(req, err); 211 break; 212 } 213 atomic_inc(&adap->chcr_stats.complete); 214 return err; 215 } 216 217 /* 218 * calc_tx_flits_ofld - calculate # of flits for an offload packet 219 * @skb: the packet 220 * Returns the number of flits needed for the given offload packet. 221 * These packets are already fully constructed and no additional headers 222 * will be added. 223 */ 224 static inline unsigned int calc_tx_flits_ofld(const struct sk_buff *skb) 225 { 226 unsigned int flits, cnt; 227 228 if (is_ofld_imm(skb)) 229 return DIV_ROUND_UP(skb->len, 8); 230 231 flits = skb_transport_offset(skb) / 8; /* headers */ 232 cnt = skb_shinfo(skb)->nr_frags; 233 if (skb_tail_pointer(skb) != skb_transport_header(skb)) 234 cnt++; 235 return flits + sgl_len(cnt); 236 } 237 238 static inline void get_aes_decrypt_key(unsigned char *dec_key, 239 const unsigned char *key, 240 unsigned int keylength) 241 { 242 u32 temp; 243 u32 w_ring[MAX_NK]; 244 int i, j, k; 245 u8 nr, nk; 246 247 switch (keylength) { 248 case AES_KEYLENGTH_128BIT: 249 nk = KEYLENGTH_4BYTES; 250 nr = NUMBER_OF_ROUNDS_10; 251 break; 252 case AES_KEYLENGTH_192BIT: 253 nk = KEYLENGTH_6BYTES; 254 nr = NUMBER_OF_ROUNDS_12; 255 break; 256 case AES_KEYLENGTH_256BIT: 257 nk = KEYLENGTH_8BYTES; 258 nr = NUMBER_OF_ROUNDS_14; 259 break; 260 default: 261 return; 262 } 263 for (i = 0; i < nk; i++) 264 w_ring[i] = be32_to_cpu(*(u32 *)&key[4 * i]); 265 266 i = 0; 267 temp = w_ring[nk - 1]; 268 while (i + nk < (nr + 1) * 4) { 269 if (!(i % nk)) { 270 /* RotWord(temp) */ 271 temp = (temp << 8) | (temp >> 24); 272 temp = aes_ks_subword(temp); 273 temp ^= round_constant[i / nk]; 274 } else if (nk == 8 && (i % 4 == 0)) { 275 temp = aes_ks_subword(temp); 276 } 277 w_ring[i % nk] ^= temp; 278 temp = w_ring[i % nk]; 279 i++; 280 } 281 i--; 282 for (k = 0, j = i % nk; k < nk; k++) { 283 *((u32 *)dec_key + k) = htonl(w_ring[j]); 284 j--; 285 if (j < 0) 286 j += nk; 287 } 288 } 289 290 static struct crypto_shash *chcr_alloc_shash(unsigned int ds) 291 { 292 struct crypto_shash *base_hash = ERR_PTR(-EINVAL); 293 294 switch (ds) { 295 case SHA1_DIGEST_SIZE: 296 base_hash = crypto_alloc_shash("sha1", 0, 0); 297 break; 298 case SHA224_DIGEST_SIZE: 299 base_hash = crypto_alloc_shash("sha224", 0, 0); 300 break; 301 case SHA256_DIGEST_SIZE: 302 base_hash = crypto_alloc_shash("sha256", 0, 0); 303 break; 304 case SHA384_DIGEST_SIZE: 305 base_hash = crypto_alloc_shash("sha384", 0, 0); 306 break; 307 case SHA512_DIGEST_SIZE: 308 base_hash = crypto_alloc_shash("sha512", 0, 0); 309 break; 310 } 311 312 return base_hash; 313 } 314 315 static int chcr_compute_partial_hash(struct shash_desc *desc, 316 char *iopad, char *result_hash, 317 int digest_size) 318 { 319 struct sha1_state sha1_st; 320 struct sha256_state sha256_st; 321 struct sha512_state sha512_st; 322 int error; 323 324 if (digest_size == SHA1_DIGEST_SIZE) { 325 error = crypto_shash_init(desc) ?: 326 crypto_shash_update(desc, iopad, SHA1_BLOCK_SIZE) ?: 327 crypto_shash_export(desc, (void *)&sha1_st); 328 memcpy(result_hash, sha1_st.state, SHA1_DIGEST_SIZE); 329 } else if (digest_size == SHA224_DIGEST_SIZE) { 330 error = crypto_shash_init(desc) ?: 331 crypto_shash_update(desc, iopad, SHA256_BLOCK_SIZE) ?: 332 crypto_shash_export(desc, (void *)&sha256_st); 333 memcpy(result_hash, sha256_st.state, SHA256_DIGEST_SIZE); 334 335 } else if (digest_size == SHA256_DIGEST_SIZE) { 336 error = crypto_shash_init(desc) ?: 337 crypto_shash_update(desc, iopad, SHA256_BLOCK_SIZE) ?: 338 crypto_shash_export(desc, (void *)&sha256_st); 339 memcpy(result_hash, sha256_st.state, SHA256_DIGEST_SIZE); 340 341 } else if (digest_size == SHA384_DIGEST_SIZE) { 342 error = crypto_shash_init(desc) ?: 343 crypto_shash_update(desc, iopad, SHA512_BLOCK_SIZE) ?: 344 crypto_shash_export(desc, (void *)&sha512_st); 345 memcpy(result_hash, sha512_st.state, SHA512_DIGEST_SIZE); 346 347 } else if (digest_size == SHA512_DIGEST_SIZE) { 348 error = crypto_shash_init(desc) ?: 349 crypto_shash_update(desc, iopad, SHA512_BLOCK_SIZE) ?: 350 crypto_shash_export(desc, (void *)&sha512_st); 351 memcpy(result_hash, sha512_st.state, SHA512_DIGEST_SIZE); 352 } else { 353 error = -EINVAL; 354 pr_err("Unknown digest size %d\n", digest_size); 355 } 356 return error; 357 } 358 359 static void chcr_change_order(char *buf, int ds) 360 { 361 int i; 362 363 if (ds == SHA512_DIGEST_SIZE) { 364 for (i = 0; i < (ds / sizeof(u64)); i++) 365 *((__be64 *)buf + i) = 366 cpu_to_be64(*((u64 *)buf + i)); 367 } else { 368 for (i = 0; i < (ds / sizeof(u32)); i++) 369 *((__be32 *)buf + i) = 370 cpu_to_be32(*((u32 *)buf + i)); 371 } 372 } 373 374 static inline int is_hmac(struct crypto_tfm *tfm) 375 { 376 struct crypto_alg *alg = tfm->__crt_alg; 377 struct chcr_alg_template *chcr_crypto_alg = 378 container_of(__crypto_ahash_alg(alg), struct chcr_alg_template, 379 alg.hash); 380 if (chcr_crypto_alg->type == CRYPTO_ALG_TYPE_HMAC) 381 return 1; 382 return 0; 383 } 384 385 static void write_phys_cpl(struct cpl_rx_phys_dsgl *phys_cpl, 386 struct scatterlist *sg, 387 struct phys_sge_parm *sg_param) 388 { 389 struct phys_sge_pairs *to; 390 unsigned int len = 0, left_size = sg_param->obsize; 391 unsigned int nents = sg_param->nents, i, j = 0; 392 393 phys_cpl->op_to_tid = htonl(CPL_RX_PHYS_DSGL_OPCODE_V(CPL_RX_PHYS_DSGL) 394 | CPL_RX_PHYS_DSGL_ISRDMA_V(0)); 395 phys_cpl->pcirlxorder_to_noofsgentr = 396 htonl(CPL_RX_PHYS_DSGL_PCIRLXORDER_V(0) | 397 CPL_RX_PHYS_DSGL_PCINOSNOOP_V(0) | 398 CPL_RX_PHYS_DSGL_PCITPHNTENB_V(0) | 399 CPL_RX_PHYS_DSGL_PCITPHNT_V(0) | 400 CPL_RX_PHYS_DSGL_DCAID_V(0) | 401 CPL_RX_PHYS_DSGL_NOOFSGENTR_V(nents)); 402 phys_cpl->rss_hdr_int.opcode = CPL_RX_PHYS_ADDR; 403 phys_cpl->rss_hdr_int.qid = htons(sg_param->qid); 404 phys_cpl->rss_hdr_int.hash_val = 0; 405 to = (struct phys_sge_pairs *)((unsigned char *)phys_cpl + 406 sizeof(struct cpl_rx_phys_dsgl)); 407 for (i = 0; nents && left_size; to++) { 408 for (j = 0; j < 8 && nents && left_size; j++, nents--) { 409 len = min(left_size, sg_dma_len(sg)); 410 to->len[j] = htons(len); 411 to->addr[j] = cpu_to_be64(sg_dma_address(sg)); 412 left_size -= len; 413 sg = sg_next(sg); 414 } 415 } 416 } 417 418 static inline int map_writesg_phys_cpl(struct device *dev, 419 struct cpl_rx_phys_dsgl *phys_cpl, 420 struct scatterlist *sg, 421 struct phys_sge_parm *sg_param) 422 { 423 if (!sg || !sg_param->nents) 424 return -EINVAL; 425 426 sg_param->nents = dma_map_sg(dev, sg, sg_param->nents, DMA_FROM_DEVICE); 427 if (sg_param->nents == 0) { 428 pr_err("CHCR : DMA mapping failed\n"); 429 return -EINVAL; 430 } 431 write_phys_cpl(phys_cpl, sg, sg_param); 432 return 0; 433 } 434 435 static inline int get_aead_subtype(struct crypto_aead *aead) 436 { 437 struct aead_alg *alg = crypto_aead_alg(aead); 438 struct chcr_alg_template *chcr_crypto_alg = 439 container_of(alg, struct chcr_alg_template, alg.aead); 440 return chcr_crypto_alg->type & CRYPTO_ALG_SUB_TYPE_MASK; 441 } 442 443 static inline int get_cryptoalg_subtype(struct crypto_tfm *tfm) 444 { 445 struct crypto_alg *alg = tfm->__crt_alg; 446 struct chcr_alg_template *chcr_crypto_alg = 447 container_of(alg, struct chcr_alg_template, alg.crypto); 448 449 return chcr_crypto_alg->type & CRYPTO_ALG_SUB_TYPE_MASK; 450 } 451 452 static inline void write_buffer_to_skb(struct sk_buff *skb, 453 unsigned int *frags, 454 char *bfr, 455 u8 bfr_len) 456 { 457 skb->len += bfr_len; 458 skb->data_len += bfr_len; 459 skb->truesize += bfr_len; 460 get_page(virt_to_page(bfr)); 461 skb_fill_page_desc(skb, *frags, virt_to_page(bfr), 462 offset_in_page(bfr), bfr_len); 463 (*frags)++; 464 } 465 466 467 static inline void 468 write_sg_to_skb(struct sk_buff *skb, unsigned int *frags, 469 struct scatterlist *sg, unsigned int count) 470 { 471 struct page *spage; 472 unsigned int page_len; 473 474 skb->len += count; 475 skb->data_len += count; 476 skb->truesize += count; 477 478 while (count > 0) { 479 if (!sg || (!(sg->length))) 480 break; 481 spage = sg_page(sg); 482 get_page(spage); 483 page_len = min(sg->length, count); 484 skb_fill_page_desc(skb, *frags, spage, sg->offset, page_len); 485 (*frags)++; 486 count -= page_len; 487 sg = sg_next(sg); 488 } 489 } 490 491 static int cxgb4_is_crypto_q_full(struct net_device *dev, unsigned int idx) 492 { 493 struct adapter *adap = netdev2adap(dev); 494 struct sge_uld_txq_info *txq_info = 495 adap->sge.uld_txq_info[CXGB4_TX_CRYPTO]; 496 struct sge_uld_txq *txq; 497 int ret = 0; 498 499 local_bh_disable(); 500 txq = &txq_info->uldtxq[idx]; 501 spin_lock(&txq->sendq.lock); 502 if (txq->full) 503 ret = -1; 504 spin_unlock(&txq->sendq.lock); 505 local_bh_enable(); 506 return ret; 507 } 508 509 static int generate_copy_rrkey(struct ablk_ctx *ablkctx, 510 struct _key_ctx *key_ctx) 511 { 512 if (ablkctx->ciph_mode == CHCR_SCMD_CIPHER_MODE_AES_CBC) { 513 memcpy(key_ctx->key, ablkctx->rrkey, ablkctx->enckey_len); 514 } else { 515 memcpy(key_ctx->key, 516 ablkctx->key + (ablkctx->enckey_len >> 1), 517 ablkctx->enckey_len >> 1); 518 memcpy(key_ctx->key + (ablkctx->enckey_len >> 1), 519 ablkctx->rrkey, ablkctx->enckey_len >> 1); 520 } 521 return 0; 522 } 523 static int chcr_sg_ent_in_wr(struct scatterlist *src, 524 struct scatterlist *dst, 525 unsigned int minsg, 526 unsigned int space, 527 short int *sent, 528 short int *dent) 529 { 530 int srclen = 0, dstlen = 0; 531 int srcsg = minsg, dstsg = 0; 532 533 *sent = 0; 534 *dent = 0; 535 while (src && dst && ((srcsg + 1) <= MAX_SKB_FRAGS) && 536 space > (sgl_ent_len[srcsg + 1] + dsgl_ent_len[dstsg])) { 537 srclen += src->length; 538 srcsg++; 539 while (dst && ((dstsg + 1) <= MAX_DSGL_ENT) && 540 space > (sgl_ent_len[srcsg] + dsgl_ent_len[dstsg + 1])) { 541 if (srclen <= dstlen) 542 break; 543 dstlen += dst->length; 544 dst = sg_next(dst); 545 dstsg++; 546 } 547 src = sg_next(src); 548 } 549 *sent = srcsg - minsg; 550 *dent = dstsg; 551 return min(srclen, dstlen); 552 } 553 554 static int chcr_cipher_fallback(struct crypto_skcipher *cipher, 555 u32 flags, 556 struct scatterlist *src, 557 struct scatterlist *dst, 558 unsigned int nbytes, 559 u8 *iv, 560 unsigned short op_type) 561 { 562 int err; 563 564 SKCIPHER_REQUEST_ON_STACK(subreq, cipher); 565 skcipher_request_set_tfm(subreq, cipher); 566 skcipher_request_set_callback(subreq, flags, NULL, NULL); 567 skcipher_request_set_crypt(subreq, src, dst, 568 nbytes, iv); 569 570 err = op_type ? crypto_skcipher_decrypt(subreq) : 571 crypto_skcipher_encrypt(subreq); 572 skcipher_request_zero(subreq); 573 574 return err; 575 576 } 577 static inline void create_wreq(struct chcr_context *ctx, 578 struct chcr_wr *chcr_req, 579 void *req, struct sk_buff *skb, 580 int kctx_len, int hash_sz, 581 int is_iv, 582 unsigned int sc_len, 583 unsigned int lcb) 584 { 585 struct uld_ctx *u_ctx = ULD_CTX(ctx); 586 int iv_loc = IV_DSGL; 587 int qid = u_ctx->lldi.rxq_ids[ctx->rx_qidx]; 588 unsigned int immdatalen = 0, nr_frags = 0; 589 590 if (is_ofld_imm(skb)) { 591 immdatalen = skb->data_len; 592 iv_loc = IV_IMMEDIATE; 593 } else { 594 nr_frags = skb_shinfo(skb)->nr_frags; 595 } 596 597 chcr_req->wreq.op_to_cctx_size = FILL_WR_OP_CCTX_SIZE(immdatalen, 598 ((sizeof(chcr_req->key_ctx) + kctx_len) >> 4)); 599 chcr_req->wreq.pld_size_hash_size = 600 htonl(FW_CRYPTO_LOOKASIDE_WR_PLD_SIZE_V(sgl_lengths[nr_frags]) | 601 FW_CRYPTO_LOOKASIDE_WR_HASH_SIZE_V(hash_sz)); 602 chcr_req->wreq.len16_pkd = 603 htonl(FW_CRYPTO_LOOKASIDE_WR_LEN16_V(DIV_ROUND_UP( 604 (calc_tx_flits_ofld(skb) * 8), 16))); 605 chcr_req->wreq.cookie = cpu_to_be64((uintptr_t)req); 606 chcr_req->wreq.rx_chid_to_rx_q_id = 607 FILL_WR_RX_Q_ID(ctx->dev->rx_channel_id, qid, 608 is_iv ? iv_loc : IV_NOP, !!lcb, 609 ctx->tx_qidx); 610 611 chcr_req->ulptx.cmd_dest = FILL_ULPTX_CMD_DEST(ctx->dev->tx_channel_id, 612 qid); 613 chcr_req->ulptx.len = htonl((DIV_ROUND_UP((calc_tx_flits_ofld(skb) * 8), 614 16) - ((sizeof(chcr_req->wreq)) >> 4))); 615 616 chcr_req->sc_imm.cmd_more = FILL_CMD_MORE(immdatalen); 617 chcr_req->sc_imm.len = cpu_to_be32(sizeof(struct cpl_tx_sec_pdu) + 618 sizeof(chcr_req->key_ctx) + 619 kctx_len + sc_len + immdatalen); 620 } 621 622 /** 623 * create_cipher_wr - form the WR for cipher operations 624 * @req: cipher req. 625 * @ctx: crypto driver context of the request. 626 * @qid: ingress qid where response of this WR should be received. 627 * @op_type: encryption or decryption 628 */ 629 static struct sk_buff *create_cipher_wr(struct cipher_wr_param *wrparam) 630 { 631 struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(wrparam->req); 632 struct chcr_context *ctx = crypto_ablkcipher_ctx(tfm); 633 struct uld_ctx *u_ctx = ULD_CTX(ctx); 634 struct ablk_ctx *ablkctx = ABLK_CTX(ctx); 635 struct sk_buff *skb = NULL; 636 struct chcr_wr *chcr_req; 637 struct cpl_rx_phys_dsgl *phys_cpl; 638 struct chcr_blkcipher_req_ctx *reqctx = 639 ablkcipher_request_ctx(wrparam->req); 640 struct phys_sge_parm sg_param; 641 unsigned int frags = 0, transhdr_len, phys_dsgl; 642 int error; 643 unsigned int ivsize = AES_BLOCK_SIZE, kctx_len; 644 gfp_t flags = wrparam->req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? 645 GFP_KERNEL : GFP_ATOMIC; 646 struct adapter *adap = padap(ctx->dev); 647 648 phys_dsgl = get_space_for_phys_dsgl(reqctx->dst_nents); 649 650 kctx_len = (DIV_ROUND_UP(ablkctx->enckey_len, 16) * 16); 651 transhdr_len = CIPHER_TRANSHDR_SIZE(kctx_len, phys_dsgl); 652 skb = alloc_skb((transhdr_len + sizeof(struct sge_opaque_hdr)), flags); 653 if (!skb) { 654 error = -ENOMEM; 655 goto err; 656 } 657 skb_reserve(skb, sizeof(struct sge_opaque_hdr)); 658 chcr_req = __skb_put_zero(skb, transhdr_len); 659 chcr_req->sec_cpl.op_ivinsrtofst = 660 FILL_SEC_CPL_OP_IVINSR(ctx->dev->rx_channel_id, 2, 1); 661 662 chcr_req->sec_cpl.pldlen = htonl(ivsize + wrparam->bytes); 663 chcr_req->sec_cpl.aadstart_cipherstop_hi = 664 FILL_SEC_CPL_CIPHERSTOP_HI(0, 0, ivsize + 1, 0); 665 666 chcr_req->sec_cpl.cipherstop_lo_authinsert = 667 FILL_SEC_CPL_AUTHINSERT(0, 0, 0, 0); 668 chcr_req->sec_cpl.seqno_numivs = FILL_SEC_CPL_SCMD0_SEQNO(reqctx->op, 0, 669 ablkctx->ciph_mode, 670 0, 0, ivsize >> 1); 671 chcr_req->sec_cpl.ivgen_hdrlen = FILL_SEC_CPL_IVGEN_HDRLEN(0, 0, 0, 672 0, 1, phys_dsgl); 673 674 chcr_req->key_ctx.ctx_hdr = ablkctx->key_ctx_hdr; 675 if ((reqctx->op == CHCR_DECRYPT_OP) && 676 (!(get_cryptoalg_subtype(crypto_ablkcipher_tfm(tfm)) == 677 CRYPTO_ALG_SUB_TYPE_CTR)) && 678 (!(get_cryptoalg_subtype(crypto_ablkcipher_tfm(tfm)) == 679 CRYPTO_ALG_SUB_TYPE_CTR_RFC3686))) { 680 generate_copy_rrkey(ablkctx, &chcr_req->key_ctx); 681 } else { 682 if ((ablkctx->ciph_mode == CHCR_SCMD_CIPHER_MODE_AES_CBC) || 683 (ablkctx->ciph_mode == CHCR_SCMD_CIPHER_MODE_AES_CTR)) { 684 memcpy(chcr_req->key_ctx.key, ablkctx->key, 685 ablkctx->enckey_len); 686 } else { 687 memcpy(chcr_req->key_ctx.key, ablkctx->key + 688 (ablkctx->enckey_len >> 1), 689 ablkctx->enckey_len >> 1); 690 memcpy(chcr_req->key_ctx.key + 691 (ablkctx->enckey_len >> 1), 692 ablkctx->key, 693 ablkctx->enckey_len >> 1); 694 } 695 } 696 phys_cpl = (struct cpl_rx_phys_dsgl *)((u8 *)(chcr_req + 1) + kctx_len); 697 sg_param.nents = reqctx->dst_nents; 698 sg_param.obsize = wrparam->bytes; 699 sg_param.qid = wrparam->qid; 700 error = map_writesg_phys_cpl(&u_ctx->lldi.pdev->dev, phys_cpl, 701 reqctx->dst, &sg_param); 702 if (error) 703 goto map_fail1; 704 705 skb_set_transport_header(skb, transhdr_len); 706 write_buffer_to_skb(skb, &frags, reqctx->iv, ivsize); 707 write_sg_to_skb(skb, &frags, wrparam->srcsg, wrparam->bytes); 708 atomic_inc(&adap->chcr_stats.cipher_rqst); 709 create_wreq(ctx, chcr_req, &(wrparam->req->base), skb, kctx_len, 0, 1, 710 sizeof(struct cpl_rx_phys_dsgl) + phys_dsgl, 711 ablkctx->ciph_mode == CHCR_SCMD_CIPHER_MODE_AES_CBC); 712 reqctx->skb = skb; 713 skb_get(skb); 714 return skb; 715 map_fail1: 716 kfree_skb(skb); 717 err: 718 return ERR_PTR(error); 719 } 720 721 static inline int chcr_keyctx_ck_size(unsigned int keylen) 722 { 723 int ck_size = 0; 724 725 if (keylen == AES_KEYSIZE_128) 726 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_128; 727 else if (keylen == AES_KEYSIZE_192) 728 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_192; 729 else if (keylen == AES_KEYSIZE_256) 730 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_256; 731 else 732 ck_size = 0; 733 734 return ck_size; 735 } 736 static int chcr_cipher_fallback_setkey(struct crypto_ablkcipher *cipher, 737 const u8 *key, 738 unsigned int keylen) 739 { 740 struct crypto_tfm *tfm = crypto_ablkcipher_tfm(cipher); 741 struct chcr_context *ctx = crypto_ablkcipher_ctx(cipher); 742 struct ablk_ctx *ablkctx = ABLK_CTX(ctx); 743 int err = 0; 744 745 crypto_skcipher_clear_flags(ablkctx->sw_cipher, CRYPTO_TFM_REQ_MASK); 746 crypto_skcipher_set_flags(ablkctx->sw_cipher, cipher->base.crt_flags & 747 CRYPTO_TFM_REQ_MASK); 748 err = crypto_skcipher_setkey(ablkctx->sw_cipher, key, keylen); 749 tfm->crt_flags &= ~CRYPTO_TFM_RES_MASK; 750 tfm->crt_flags |= 751 crypto_skcipher_get_flags(ablkctx->sw_cipher) & 752 CRYPTO_TFM_RES_MASK; 753 return err; 754 } 755 756 static int chcr_aes_cbc_setkey(struct crypto_ablkcipher *cipher, 757 const u8 *key, 758 unsigned int keylen) 759 { 760 struct chcr_context *ctx = crypto_ablkcipher_ctx(cipher); 761 struct ablk_ctx *ablkctx = ABLK_CTX(ctx); 762 unsigned int ck_size, context_size; 763 u16 alignment = 0; 764 int err; 765 766 err = chcr_cipher_fallback_setkey(cipher, key, keylen); 767 if (err) 768 goto badkey_err; 769 770 ck_size = chcr_keyctx_ck_size(keylen); 771 alignment = ck_size == CHCR_KEYCTX_CIPHER_KEY_SIZE_192 ? 8 : 0; 772 memcpy(ablkctx->key, key, keylen); 773 ablkctx->enckey_len = keylen; 774 get_aes_decrypt_key(ablkctx->rrkey, ablkctx->key, keylen << 3); 775 context_size = (KEY_CONTEXT_HDR_SALT_AND_PAD + 776 keylen + alignment) >> 4; 777 778 ablkctx->key_ctx_hdr = FILL_KEY_CTX_HDR(ck_size, CHCR_KEYCTX_NO_KEY, 779 0, 0, context_size); 780 ablkctx->ciph_mode = CHCR_SCMD_CIPHER_MODE_AES_CBC; 781 return 0; 782 badkey_err: 783 crypto_ablkcipher_set_flags(cipher, CRYPTO_TFM_RES_BAD_KEY_LEN); 784 ablkctx->enckey_len = 0; 785 786 return err; 787 } 788 789 static int chcr_aes_ctr_setkey(struct crypto_ablkcipher *cipher, 790 const u8 *key, 791 unsigned int keylen) 792 { 793 struct chcr_context *ctx = crypto_ablkcipher_ctx(cipher); 794 struct ablk_ctx *ablkctx = ABLK_CTX(ctx); 795 unsigned int ck_size, context_size; 796 u16 alignment = 0; 797 int err; 798 799 err = chcr_cipher_fallback_setkey(cipher, key, keylen); 800 if (err) 801 goto badkey_err; 802 ck_size = chcr_keyctx_ck_size(keylen); 803 alignment = (ck_size == CHCR_KEYCTX_CIPHER_KEY_SIZE_192) ? 8 : 0; 804 memcpy(ablkctx->key, key, keylen); 805 ablkctx->enckey_len = keylen; 806 context_size = (KEY_CONTEXT_HDR_SALT_AND_PAD + 807 keylen + alignment) >> 4; 808 809 ablkctx->key_ctx_hdr = FILL_KEY_CTX_HDR(ck_size, CHCR_KEYCTX_NO_KEY, 810 0, 0, context_size); 811 ablkctx->ciph_mode = CHCR_SCMD_CIPHER_MODE_AES_CTR; 812 813 return 0; 814 badkey_err: 815 crypto_ablkcipher_set_flags(cipher, CRYPTO_TFM_RES_BAD_KEY_LEN); 816 ablkctx->enckey_len = 0; 817 818 return err; 819 } 820 821 static int chcr_aes_rfc3686_setkey(struct crypto_ablkcipher *cipher, 822 const u8 *key, 823 unsigned int keylen) 824 { 825 struct chcr_context *ctx = crypto_ablkcipher_ctx(cipher); 826 struct ablk_ctx *ablkctx = ABLK_CTX(ctx); 827 unsigned int ck_size, context_size; 828 u16 alignment = 0; 829 int err; 830 831 if (keylen < CTR_RFC3686_NONCE_SIZE) 832 return -EINVAL; 833 memcpy(ablkctx->nonce, key + (keylen - CTR_RFC3686_NONCE_SIZE), 834 CTR_RFC3686_NONCE_SIZE); 835 836 keylen -= CTR_RFC3686_NONCE_SIZE; 837 err = chcr_cipher_fallback_setkey(cipher, key, keylen); 838 if (err) 839 goto badkey_err; 840 841 ck_size = chcr_keyctx_ck_size(keylen); 842 alignment = (ck_size == CHCR_KEYCTX_CIPHER_KEY_SIZE_192) ? 8 : 0; 843 memcpy(ablkctx->key, key, keylen); 844 ablkctx->enckey_len = keylen; 845 context_size = (KEY_CONTEXT_HDR_SALT_AND_PAD + 846 keylen + alignment) >> 4; 847 848 ablkctx->key_ctx_hdr = FILL_KEY_CTX_HDR(ck_size, CHCR_KEYCTX_NO_KEY, 849 0, 0, context_size); 850 ablkctx->ciph_mode = CHCR_SCMD_CIPHER_MODE_AES_CTR; 851 852 return 0; 853 badkey_err: 854 crypto_ablkcipher_set_flags(cipher, CRYPTO_TFM_RES_BAD_KEY_LEN); 855 ablkctx->enckey_len = 0; 856 857 return err; 858 } 859 static void ctr_add_iv(u8 *dstiv, u8 *srciv, u32 add) 860 { 861 unsigned int size = AES_BLOCK_SIZE; 862 __be32 *b = (__be32 *)(dstiv + size); 863 u32 c, prev; 864 865 memcpy(dstiv, srciv, AES_BLOCK_SIZE); 866 for (; size >= 4; size -= 4) { 867 prev = be32_to_cpu(*--b); 868 c = prev + add; 869 *b = cpu_to_be32(c); 870 if (prev < c) 871 break; 872 add = 1; 873 } 874 875 } 876 877 static unsigned int adjust_ctr_overflow(u8 *iv, u32 bytes) 878 { 879 __be32 *b = (__be32 *)(iv + AES_BLOCK_SIZE); 880 u64 c; 881 u32 temp = be32_to_cpu(*--b); 882 883 temp = ~temp; 884 c = (u64)temp + 1; // No of block can processed withou overflow 885 if ((bytes / AES_BLOCK_SIZE) > c) 886 bytes = c * AES_BLOCK_SIZE; 887 return bytes; 888 } 889 890 static int chcr_update_tweak(struct ablkcipher_request *req, u8 *iv) 891 { 892 struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req); 893 struct chcr_context *ctx = crypto_ablkcipher_ctx(tfm); 894 struct ablk_ctx *ablkctx = ABLK_CTX(ctx); 895 struct chcr_blkcipher_req_ctx *reqctx = ablkcipher_request_ctx(req); 896 struct crypto_cipher *cipher; 897 int ret, i; 898 u8 *key; 899 unsigned int keylen; 900 901 cipher = crypto_alloc_cipher("aes-generic", 0, 0); 902 memcpy(iv, req->info, AES_BLOCK_SIZE); 903 904 if (IS_ERR(cipher)) { 905 ret = -ENOMEM; 906 goto out; 907 } 908 keylen = ablkctx->enckey_len / 2; 909 key = ablkctx->key + keylen; 910 ret = crypto_cipher_setkey(cipher, key, keylen); 911 if (ret) 912 goto out1; 913 914 crypto_cipher_encrypt_one(cipher, iv, iv); 915 for (i = 0; i < (reqctx->processed / AES_BLOCK_SIZE); i++) 916 gf128mul_x_ble((le128 *)iv, (le128 *)iv); 917 918 crypto_cipher_decrypt_one(cipher, iv, iv); 919 out1: 920 crypto_free_cipher(cipher); 921 out: 922 return ret; 923 } 924 925 static int chcr_update_cipher_iv(struct ablkcipher_request *req, 926 struct cpl_fw6_pld *fw6_pld, u8 *iv) 927 { 928 struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req); 929 struct chcr_blkcipher_req_ctx *reqctx = ablkcipher_request_ctx(req); 930 int subtype = get_cryptoalg_subtype(crypto_ablkcipher_tfm(tfm)); 931 int ret = 0; 932 933 if (subtype == CRYPTO_ALG_SUB_TYPE_CTR) 934 ctr_add_iv(iv, req->info, (reqctx->processed / 935 AES_BLOCK_SIZE)); 936 else if (subtype == CRYPTO_ALG_SUB_TYPE_CTR_RFC3686) 937 *(__be32 *)(reqctx->iv + CTR_RFC3686_NONCE_SIZE + 938 CTR_RFC3686_IV_SIZE) = cpu_to_be32((reqctx->processed / 939 AES_BLOCK_SIZE) + 1); 940 else if (subtype == CRYPTO_ALG_SUB_TYPE_XTS) 941 ret = chcr_update_tweak(req, iv); 942 else if (subtype == CRYPTO_ALG_SUB_TYPE_CBC) { 943 if (reqctx->op) 944 sg_pcopy_to_buffer(req->src, sg_nents(req->src), iv, 945 16, 946 reqctx->processed - AES_BLOCK_SIZE); 947 else 948 memcpy(iv, &fw6_pld->data[2], AES_BLOCK_SIZE); 949 } 950 951 return ret; 952 953 } 954 955 /* We need separate function for final iv because in rfc3686 Initial counter 956 * starts from 1 and buffer size of iv is 8 byte only which remains constant 957 * for subsequent update requests 958 */ 959 960 static int chcr_final_cipher_iv(struct ablkcipher_request *req, 961 struct cpl_fw6_pld *fw6_pld, u8 *iv) 962 { 963 struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req); 964 struct chcr_blkcipher_req_ctx *reqctx = ablkcipher_request_ctx(req); 965 int subtype = get_cryptoalg_subtype(crypto_ablkcipher_tfm(tfm)); 966 int ret = 0; 967 968 if (subtype == CRYPTO_ALG_SUB_TYPE_CTR) 969 ctr_add_iv(iv, req->info, (reqctx->processed / 970 AES_BLOCK_SIZE)); 971 else if (subtype == CRYPTO_ALG_SUB_TYPE_XTS) 972 ret = chcr_update_tweak(req, iv); 973 else if (subtype == CRYPTO_ALG_SUB_TYPE_CBC) { 974 if (reqctx->op) 975 sg_pcopy_to_buffer(req->src, sg_nents(req->src), iv, 976 16, 977 reqctx->processed - AES_BLOCK_SIZE); 978 else 979 memcpy(iv, &fw6_pld->data[2], AES_BLOCK_SIZE); 980 981 } 982 return ret; 983 984 } 985 986 987 static int chcr_handle_cipher_resp(struct ablkcipher_request *req, 988 unsigned char *input, int err) 989 { 990 struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req); 991 struct chcr_context *ctx = crypto_ablkcipher_ctx(tfm); 992 struct uld_ctx *u_ctx = ULD_CTX(ctx); 993 struct ablk_ctx *ablkctx = ABLK_CTX(ctx); 994 struct sk_buff *skb; 995 struct cpl_fw6_pld *fw6_pld = (struct cpl_fw6_pld *)input; 996 struct chcr_blkcipher_req_ctx *reqctx = ablkcipher_request_ctx(req); 997 struct cipher_wr_param wrparam; 998 int bytes; 999 1000 dma_unmap_sg(&u_ctx->lldi.pdev->dev, reqctx->dst, reqctx->dst_nents, 1001 DMA_FROM_DEVICE); 1002 1003 if (reqctx->skb) { 1004 kfree_skb(reqctx->skb); 1005 reqctx->skb = NULL; 1006 } 1007 if (err) 1008 goto complete; 1009 1010 if (req->nbytes == reqctx->processed) { 1011 err = chcr_final_cipher_iv(req, fw6_pld, req->info); 1012 goto complete; 1013 } 1014 1015 if (unlikely(cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0], 1016 ctx->tx_qidx))) { 1017 if (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) { 1018 err = -EBUSY; 1019 goto complete; 1020 } 1021 1022 } 1023 wrparam.srcsg = scatterwalk_ffwd(reqctx->srcffwd, req->src, 1024 reqctx->processed); 1025 reqctx->dst = scatterwalk_ffwd(reqctx->dstffwd, reqctx->dstsg, 1026 reqctx->processed); 1027 if (!wrparam.srcsg || !reqctx->dst) { 1028 pr_err("Input sg list length less that nbytes\n"); 1029 err = -EINVAL; 1030 goto complete; 1031 } 1032 bytes = chcr_sg_ent_in_wr(wrparam.srcsg, reqctx->dst, 1, 1033 SPACE_LEFT(ablkctx->enckey_len), 1034 &wrparam.snent, &reqctx->dst_nents); 1035 if ((bytes + reqctx->processed) >= req->nbytes) 1036 bytes = req->nbytes - reqctx->processed; 1037 else 1038 bytes = ROUND_16(bytes); 1039 err = chcr_update_cipher_iv(req, fw6_pld, reqctx->iv); 1040 if (err) 1041 goto complete; 1042 1043 if (unlikely(bytes == 0)) { 1044 err = chcr_cipher_fallback(ablkctx->sw_cipher, 1045 req->base.flags, 1046 wrparam.srcsg, 1047 reqctx->dst, 1048 req->nbytes - reqctx->processed, 1049 reqctx->iv, 1050 reqctx->op); 1051 goto complete; 1052 } 1053 1054 if (get_cryptoalg_subtype(crypto_ablkcipher_tfm(tfm)) == 1055 CRYPTO_ALG_SUB_TYPE_CTR) 1056 bytes = adjust_ctr_overflow(reqctx->iv, bytes); 1057 reqctx->processed += bytes; 1058 wrparam.qid = u_ctx->lldi.rxq_ids[ctx->rx_qidx]; 1059 wrparam.req = req; 1060 wrparam.bytes = bytes; 1061 skb = create_cipher_wr(&wrparam); 1062 if (IS_ERR(skb)) { 1063 pr_err("chcr : %s : Failed to form WR. No memory\n", __func__); 1064 err = PTR_ERR(skb); 1065 goto complete; 1066 } 1067 skb->dev = u_ctx->lldi.ports[0]; 1068 set_wr_txq(skb, CPL_PRIORITY_DATA, ctx->tx_qidx); 1069 chcr_send_wr(skb); 1070 return 0; 1071 complete: 1072 free_new_sg(reqctx->newdstsg); 1073 reqctx->newdstsg = NULL; 1074 req->base.complete(&req->base, err); 1075 return err; 1076 } 1077 1078 static int process_cipher(struct ablkcipher_request *req, 1079 unsigned short qid, 1080 struct sk_buff **skb, 1081 unsigned short op_type) 1082 { 1083 struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req); 1084 unsigned int ivsize = crypto_ablkcipher_ivsize(tfm); 1085 struct chcr_blkcipher_req_ctx *reqctx = ablkcipher_request_ctx(req); 1086 struct chcr_context *ctx = crypto_ablkcipher_ctx(tfm); 1087 struct ablk_ctx *ablkctx = ABLK_CTX(ctx); 1088 struct cipher_wr_param wrparam; 1089 int bytes, nents, err = -EINVAL; 1090 1091 reqctx->newdstsg = NULL; 1092 reqctx->processed = 0; 1093 if (!req->info) 1094 goto error; 1095 if ((ablkctx->enckey_len == 0) || (ivsize > AES_BLOCK_SIZE) || 1096 (req->nbytes == 0) || 1097 (req->nbytes % crypto_ablkcipher_blocksize(tfm))) { 1098 pr_err("AES: Invalid value of Key Len %d nbytes %d IV Len %d\n", 1099 ablkctx->enckey_len, req->nbytes, ivsize); 1100 goto error; 1101 } 1102 wrparam.srcsg = req->src; 1103 if (is_newsg(req->dst, &nents)) { 1104 reqctx->newdstsg = alloc_new_sg(req->dst, nents); 1105 if (IS_ERR(reqctx->newdstsg)) 1106 return PTR_ERR(reqctx->newdstsg); 1107 reqctx->dstsg = reqctx->newdstsg; 1108 } else { 1109 reqctx->dstsg = req->dst; 1110 } 1111 bytes = chcr_sg_ent_in_wr(wrparam.srcsg, reqctx->dstsg, MIN_CIPHER_SG, 1112 SPACE_LEFT(ablkctx->enckey_len), 1113 &wrparam.snent, 1114 &reqctx->dst_nents); 1115 if ((bytes + reqctx->processed) >= req->nbytes) 1116 bytes = req->nbytes - reqctx->processed; 1117 else 1118 bytes = ROUND_16(bytes); 1119 if (unlikely(bytes > req->nbytes)) 1120 bytes = req->nbytes; 1121 if (get_cryptoalg_subtype(crypto_ablkcipher_tfm(tfm)) == 1122 CRYPTO_ALG_SUB_TYPE_CTR) { 1123 bytes = adjust_ctr_overflow(req->info, bytes); 1124 } 1125 if (get_cryptoalg_subtype(crypto_ablkcipher_tfm(tfm)) == 1126 CRYPTO_ALG_SUB_TYPE_CTR_RFC3686) { 1127 memcpy(reqctx->iv, ablkctx->nonce, CTR_RFC3686_NONCE_SIZE); 1128 memcpy(reqctx->iv + CTR_RFC3686_NONCE_SIZE, req->info, 1129 CTR_RFC3686_IV_SIZE); 1130 1131 /* initialize counter portion of counter block */ 1132 *(__be32 *)(reqctx->iv + CTR_RFC3686_NONCE_SIZE + 1133 CTR_RFC3686_IV_SIZE) = cpu_to_be32(1); 1134 1135 } else { 1136 1137 memcpy(reqctx->iv, req->info, ivsize); 1138 } 1139 if (unlikely(bytes == 0)) { 1140 err = chcr_cipher_fallback(ablkctx->sw_cipher, 1141 req->base.flags, 1142 req->src, 1143 req->dst, 1144 req->nbytes, 1145 req->info, 1146 op_type); 1147 goto error; 1148 } 1149 reqctx->processed = bytes; 1150 reqctx->dst = reqctx->dstsg; 1151 reqctx->op = op_type; 1152 wrparam.qid = qid; 1153 wrparam.req = req; 1154 wrparam.bytes = bytes; 1155 *skb = create_cipher_wr(&wrparam); 1156 if (IS_ERR(*skb)) { 1157 err = PTR_ERR(*skb); 1158 goto error; 1159 } 1160 1161 return 0; 1162 error: 1163 free_new_sg(reqctx->newdstsg); 1164 reqctx->newdstsg = NULL; 1165 return err; 1166 } 1167 1168 static int chcr_aes_encrypt(struct ablkcipher_request *req) 1169 { 1170 struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req); 1171 struct chcr_context *ctx = crypto_ablkcipher_ctx(tfm); 1172 struct sk_buff *skb = NULL; 1173 int err; 1174 struct uld_ctx *u_ctx = ULD_CTX(ctx); 1175 1176 if (unlikely(cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0], 1177 ctx->tx_qidx))) { 1178 if (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) 1179 return -EBUSY; 1180 } 1181 1182 err = process_cipher(req, u_ctx->lldi.rxq_ids[ctx->rx_qidx], &skb, 1183 CHCR_ENCRYPT_OP); 1184 if (err || !skb) 1185 return err; 1186 skb->dev = u_ctx->lldi.ports[0]; 1187 set_wr_txq(skb, CPL_PRIORITY_DATA, ctx->tx_qidx); 1188 chcr_send_wr(skb); 1189 return -EINPROGRESS; 1190 } 1191 1192 static int chcr_aes_decrypt(struct ablkcipher_request *req) 1193 { 1194 struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req); 1195 struct chcr_context *ctx = crypto_ablkcipher_ctx(tfm); 1196 struct uld_ctx *u_ctx = ULD_CTX(ctx); 1197 struct sk_buff *skb = NULL; 1198 int err; 1199 1200 if (unlikely(cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0], 1201 ctx->tx_qidx))) { 1202 if (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) 1203 return -EBUSY; 1204 } 1205 1206 err = process_cipher(req, u_ctx->lldi.rxq_ids[ctx->rx_qidx], &skb, 1207 CHCR_DECRYPT_OP); 1208 if (err || !skb) 1209 return err; 1210 skb->dev = u_ctx->lldi.ports[0]; 1211 set_wr_txq(skb, CPL_PRIORITY_DATA, ctx->tx_qidx); 1212 chcr_send_wr(skb); 1213 return -EINPROGRESS; 1214 } 1215 1216 static int chcr_device_init(struct chcr_context *ctx) 1217 { 1218 struct uld_ctx *u_ctx = NULL; 1219 struct adapter *adap; 1220 unsigned int id; 1221 int txq_perchan, txq_idx, ntxq; 1222 int err = 0, rxq_perchan, rxq_idx; 1223 1224 id = smp_processor_id(); 1225 if (!ctx->dev) { 1226 u_ctx = assign_chcr_device(); 1227 if (!u_ctx) { 1228 pr_err("chcr device assignment fails\n"); 1229 goto out; 1230 } 1231 ctx->dev = u_ctx->dev; 1232 adap = padap(ctx->dev); 1233 ntxq = min_not_zero((unsigned int)u_ctx->lldi.nrxq, 1234 adap->vres.ncrypto_fc); 1235 rxq_perchan = u_ctx->lldi.nrxq / u_ctx->lldi.nchan; 1236 txq_perchan = ntxq / u_ctx->lldi.nchan; 1237 rxq_idx = ctx->dev->tx_channel_id * rxq_perchan; 1238 rxq_idx += id % rxq_perchan; 1239 txq_idx = ctx->dev->tx_channel_id * txq_perchan; 1240 txq_idx += id % txq_perchan; 1241 spin_lock(&ctx->dev->lock_chcr_dev); 1242 ctx->rx_qidx = rxq_idx; 1243 ctx->tx_qidx = txq_idx; 1244 ctx->dev->tx_channel_id = !ctx->dev->tx_channel_id; 1245 ctx->dev->rx_channel_id = 0; 1246 spin_unlock(&ctx->dev->lock_chcr_dev); 1247 } 1248 out: 1249 return err; 1250 } 1251 1252 static int chcr_cra_init(struct crypto_tfm *tfm) 1253 { 1254 struct crypto_alg *alg = tfm->__crt_alg; 1255 struct chcr_context *ctx = crypto_tfm_ctx(tfm); 1256 struct ablk_ctx *ablkctx = ABLK_CTX(ctx); 1257 1258 ablkctx->sw_cipher = crypto_alloc_skcipher(alg->cra_name, 0, 1259 CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK); 1260 if (IS_ERR(ablkctx->sw_cipher)) { 1261 pr_err("failed to allocate fallback for %s\n", alg->cra_name); 1262 return PTR_ERR(ablkctx->sw_cipher); 1263 } 1264 tfm->crt_ablkcipher.reqsize = sizeof(struct chcr_blkcipher_req_ctx); 1265 return chcr_device_init(crypto_tfm_ctx(tfm)); 1266 } 1267 1268 static int chcr_rfc3686_init(struct crypto_tfm *tfm) 1269 { 1270 struct crypto_alg *alg = tfm->__crt_alg; 1271 struct chcr_context *ctx = crypto_tfm_ctx(tfm); 1272 struct ablk_ctx *ablkctx = ABLK_CTX(ctx); 1273 1274 /*RFC3686 initialises IV counter value to 1, rfc3686(ctr(aes)) 1275 * cannot be used as fallback in chcr_handle_cipher_response 1276 */ 1277 ablkctx->sw_cipher = crypto_alloc_skcipher("ctr(aes)", 0, 1278 CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK); 1279 if (IS_ERR(ablkctx->sw_cipher)) { 1280 pr_err("failed to allocate fallback for %s\n", alg->cra_name); 1281 return PTR_ERR(ablkctx->sw_cipher); 1282 } 1283 tfm->crt_ablkcipher.reqsize = sizeof(struct chcr_blkcipher_req_ctx); 1284 return chcr_device_init(crypto_tfm_ctx(tfm)); 1285 } 1286 1287 1288 static void chcr_cra_exit(struct crypto_tfm *tfm) 1289 { 1290 struct chcr_context *ctx = crypto_tfm_ctx(tfm); 1291 struct ablk_ctx *ablkctx = ABLK_CTX(ctx); 1292 1293 crypto_free_skcipher(ablkctx->sw_cipher); 1294 } 1295 1296 static int get_alg_config(struct algo_param *params, 1297 unsigned int auth_size) 1298 { 1299 switch (auth_size) { 1300 case SHA1_DIGEST_SIZE: 1301 params->mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_160; 1302 params->auth_mode = CHCR_SCMD_AUTH_MODE_SHA1; 1303 params->result_size = SHA1_DIGEST_SIZE; 1304 break; 1305 case SHA224_DIGEST_SIZE: 1306 params->mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_256; 1307 params->auth_mode = CHCR_SCMD_AUTH_MODE_SHA224; 1308 params->result_size = SHA256_DIGEST_SIZE; 1309 break; 1310 case SHA256_DIGEST_SIZE: 1311 params->mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_256; 1312 params->auth_mode = CHCR_SCMD_AUTH_MODE_SHA256; 1313 params->result_size = SHA256_DIGEST_SIZE; 1314 break; 1315 case SHA384_DIGEST_SIZE: 1316 params->mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_512; 1317 params->auth_mode = CHCR_SCMD_AUTH_MODE_SHA512_384; 1318 params->result_size = SHA512_DIGEST_SIZE; 1319 break; 1320 case SHA512_DIGEST_SIZE: 1321 params->mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_512; 1322 params->auth_mode = CHCR_SCMD_AUTH_MODE_SHA512_512; 1323 params->result_size = SHA512_DIGEST_SIZE; 1324 break; 1325 default: 1326 pr_err("chcr : ERROR, unsupported digest size\n"); 1327 return -EINVAL; 1328 } 1329 return 0; 1330 } 1331 1332 static inline void chcr_free_shash(struct crypto_shash *base_hash) 1333 { 1334 crypto_free_shash(base_hash); 1335 } 1336 1337 /** 1338 * create_hash_wr - Create hash work request 1339 * @req - Cipher req base 1340 */ 1341 static struct sk_buff *create_hash_wr(struct ahash_request *req, 1342 struct hash_wr_param *param) 1343 { 1344 struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(req); 1345 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 1346 struct chcr_context *ctx = crypto_tfm_ctx(crypto_ahash_tfm(tfm)); 1347 struct hmac_ctx *hmacctx = HMAC_CTX(ctx); 1348 struct sk_buff *skb = NULL; 1349 struct chcr_wr *chcr_req; 1350 unsigned int frags = 0, transhdr_len, iopad_alignment = 0; 1351 unsigned int digestsize = crypto_ahash_digestsize(tfm); 1352 unsigned int kctx_len = 0; 1353 u8 hash_size_in_response = 0; 1354 gfp_t flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL : 1355 GFP_ATOMIC; 1356 struct adapter *adap = padap(ctx->dev); 1357 1358 iopad_alignment = KEYCTX_ALIGN_PAD(digestsize); 1359 kctx_len = param->alg_prm.result_size + iopad_alignment; 1360 if (param->opad_needed) 1361 kctx_len += param->alg_prm.result_size + iopad_alignment; 1362 1363 if (req_ctx->result) 1364 hash_size_in_response = digestsize; 1365 else 1366 hash_size_in_response = param->alg_prm.result_size; 1367 transhdr_len = HASH_TRANSHDR_SIZE(kctx_len); 1368 skb = alloc_skb((transhdr_len + sizeof(struct sge_opaque_hdr)), flags); 1369 if (!skb) 1370 return skb; 1371 1372 skb_reserve(skb, sizeof(struct sge_opaque_hdr)); 1373 chcr_req = __skb_put_zero(skb, transhdr_len); 1374 1375 chcr_req->sec_cpl.op_ivinsrtofst = 1376 FILL_SEC_CPL_OP_IVINSR(ctx->dev->rx_channel_id, 2, 0); 1377 chcr_req->sec_cpl.pldlen = htonl(param->bfr_len + param->sg_len); 1378 1379 chcr_req->sec_cpl.aadstart_cipherstop_hi = 1380 FILL_SEC_CPL_CIPHERSTOP_HI(0, 0, 0, 0); 1381 chcr_req->sec_cpl.cipherstop_lo_authinsert = 1382 FILL_SEC_CPL_AUTHINSERT(0, 1, 0, 0); 1383 chcr_req->sec_cpl.seqno_numivs = 1384 FILL_SEC_CPL_SCMD0_SEQNO(0, 0, 0, param->alg_prm.auth_mode, 1385 param->opad_needed, 0); 1386 1387 chcr_req->sec_cpl.ivgen_hdrlen = 1388 FILL_SEC_CPL_IVGEN_HDRLEN(param->last, param->more, 0, 1, 0, 0); 1389 1390 memcpy(chcr_req->key_ctx.key, req_ctx->partial_hash, 1391 param->alg_prm.result_size); 1392 1393 if (param->opad_needed) 1394 memcpy(chcr_req->key_ctx.key + 1395 ((param->alg_prm.result_size <= 32) ? 32 : 1396 CHCR_HASH_MAX_DIGEST_SIZE), 1397 hmacctx->opad, param->alg_prm.result_size); 1398 1399 chcr_req->key_ctx.ctx_hdr = FILL_KEY_CTX_HDR(CHCR_KEYCTX_NO_KEY, 1400 param->alg_prm.mk_size, 0, 1401 param->opad_needed, 1402 ((kctx_len + 1403 sizeof(chcr_req->key_ctx)) >> 4)); 1404 chcr_req->sec_cpl.scmd1 = cpu_to_be64((u64)param->scmd1); 1405 1406 skb_set_transport_header(skb, transhdr_len); 1407 if (param->bfr_len != 0) 1408 write_buffer_to_skb(skb, &frags, req_ctx->reqbfr, 1409 param->bfr_len); 1410 if (param->sg_len != 0) 1411 write_sg_to_skb(skb, &frags, req->src, param->sg_len); 1412 atomic_inc(&adap->chcr_stats.digest_rqst); 1413 create_wreq(ctx, chcr_req, &req->base, skb, kctx_len, 1414 hash_size_in_response, 0, DUMMY_BYTES, 0); 1415 req_ctx->skb = skb; 1416 skb_get(skb); 1417 return skb; 1418 } 1419 1420 static int chcr_ahash_update(struct ahash_request *req) 1421 { 1422 struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(req); 1423 struct crypto_ahash *rtfm = crypto_ahash_reqtfm(req); 1424 struct chcr_context *ctx = crypto_tfm_ctx(crypto_ahash_tfm(rtfm)); 1425 struct uld_ctx *u_ctx = NULL; 1426 struct sk_buff *skb; 1427 u8 remainder = 0, bs; 1428 unsigned int nbytes = req->nbytes; 1429 struct hash_wr_param params; 1430 1431 bs = crypto_tfm_alg_blocksize(crypto_ahash_tfm(rtfm)); 1432 1433 u_ctx = ULD_CTX(ctx); 1434 if (unlikely(cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0], 1435 ctx->tx_qidx))) { 1436 if (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) 1437 return -EBUSY; 1438 } 1439 1440 if (nbytes + req_ctx->reqlen >= bs) { 1441 remainder = (nbytes + req_ctx->reqlen) % bs; 1442 nbytes = nbytes + req_ctx->reqlen - remainder; 1443 } else { 1444 sg_pcopy_to_buffer(req->src, sg_nents(req->src), req_ctx->reqbfr 1445 + req_ctx->reqlen, nbytes, 0); 1446 req_ctx->reqlen += nbytes; 1447 return 0; 1448 } 1449 1450 params.opad_needed = 0; 1451 params.more = 1; 1452 params.last = 0; 1453 params.sg_len = nbytes - req_ctx->reqlen; 1454 params.bfr_len = req_ctx->reqlen; 1455 params.scmd1 = 0; 1456 get_alg_config(¶ms.alg_prm, crypto_ahash_digestsize(rtfm)); 1457 req_ctx->result = 0; 1458 req_ctx->data_len += params.sg_len + params.bfr_len; 1459 skb = create_hash_wr(req, ¶ms); 1460 if (!skb) 1461 return -ENOMEM; 1462 1463 if (remainder) { 1464 u8 *temp; 1465 /* Swap buffers */ 1466 temp = req_ctx->reqbfr; 1467 req_ctx->reqbfr = req_ctx->skbfr; 1468 req_ctx->skbfr = temp; 1469 sg_pcopy_to_buffer(req->src, sg_nents(req->src), 1470 req_ctx->reqbfr, remainder, req->nbytes - 1471 remainder); 1472 } 1473 req_ctx->reqlen = remainder; 1474 skb->dev = u_ctx->lldi.ports[0]; 1475 set_wr_txq(skb, CPL_PRIORITY_DATA, ctx->tx_qidx); 1476 chcr_send_wr(skb); 1477 1478 return -EINPROGRESS; 1479 } 1480 1481 static void create_last_hash_block(char *bfr_ptr, unsigned int bs, u64 scmd1) 1482 { 1483 memset(bfr_ptr, 0, bs); 1484 *bfr_ptr = 0x80; 1485 if (bs == 64) 1486 *(__be64 *)(bfr_ptr + 56) = cpu_to_be64(scmd1 << 3); 1487 else 1488 *(__be64 *)(bfr_ptr + 120) = cpu_to_be64(scmd1 << 3); 1489 } 1490 1491 static int chcr_ahash_final(struct ahash_request *req) 1492 { 1493 struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(req); 1494 struct crypto_ahash *rtfm = crypto_ahash_reqtfm(req); 1495 struct chcr_context *ctx = crypto_tfm_ctx(crypto_ahash_tfm(rtfm)); 1496 struct hash_wr_param params; 1497 struct sk_buff *skb; 1498 struct uld_ctx *u_ctx = NULL; 1499 u8 bs = crypto_tfm_alg_blocksize(crypto_ahash_tfm(rtfm)); 1500 1501 u_ctx = ULD_CTX(ctx); 1502 if (is_hmac(crypto_ahash_tfm(rtfm))) 1503 params.opad_needed = 1; 1504 else 1505 params.opad_needed = 0; 1506 params.sg_len = 0; 1507 get_alg_config(¶ms.alg_prm, crypto_ahash_digestsize(rtfm)); 1508 req_ctx->result = 1; 1509 params.bfr_len = req_ctx->reqlen; 1510 req_ctx->data_len += params.bfr_len + params.sg_len; 1511 if (req_ctx->reqlen == 0) { 1512 create_last_hash_block(req_ctx->reqbfr, bs, req_ctx->data_len); 1513 params.last = 0; 1514 params.more = 1; 1515 params.scmd1 = 0; 1516 params.bfr_len = bs; 1517 1518 } else { 1519 params.scmd1 = req_ctx->data_len; 1520 params.last = 1; 1521 params.more = 0; 1522 } 1523 skb = create_hash_wr(req, ¶ms); 1524 if (!skb) 1525 return -ENOMEM; 1526 1527 skb->dev = u_ctx->lldi.ports[0]; 1528 set_wr_txq(skb, CPL_PRIORITY_DATA, ctx->tx_qidx); 1529 chcr_send_wr(skb); 1530 return -EINPROGRESS; 1531 } 1532 1533 static int chcr_ahash_finup(struct ahash_request *req) 1534 { 1535 struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(req); 1536 struct crypto_ahash *rtfm = crypto_ahash_reqtfm(req); 1537 struct chcr_context *ctx = crypto_tfm_ctx(crypto_ahash_tfm(rtfm)); 1538 struct uld_ctx *u_ctx = NULL; 1539 struct sk_buff *skb; 1540 struct hash_wr_param params; 1541 u8 bs; 1542 1543 bs = crypto_tfm_alg_blocksize(crypto_ahash_tfm(rtfm)); 1544 u_ctx = ULD_CTX(ctx); 1545 1546 if (unlikely(cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0], 1547 ctx->tx_qidx))) { 1548 if (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) 1549 return -EBUSY; 1550 } 1551 1552 if (is_hmac(crypto_ahash_tfm(rtfm))) 1553 params.opad_needed = 1; 1554 else 1555 params.opad_needed = 0; 1556 1557 params.sg_len = req->nbytes; 1558 params.bfr_len = req_ctx->reqlen; 1559 get_alg_config(¶ms.alg_prm, crypto_ahash_digestsize(rtfm)); 1560 req_ctx->data_len += params.bfr_len + params.sg_len; 1561 req_ctx->result = 1; 1562 if ((req_ctx->reqlen + req->nbytes) == 0) { 1563 create_last_hash_block(req_ctx->reqbfr, bs, req_ctx->data_len); 1564 params.last = 0; 1565 params.more = 1; 1566 params.scmd1 = 0; 1567 params.bfr_len = bs; 1568 } else { 1569 params.scmd1 = req_ctx->data_len; 1570 params.last = 1; 1571 params.more = 0; 1572 } 1573 1574 skb = create_hash_wr(req, ¶ms); 1575 if (!skb) 1576 return -ENOMEM; 1577 1578 skb->dev = u_ctx->lldi.ports[0]; 1579 set_wr_txq(skb, CPL_PRIORITY_DATA, ctx->tx_qidx); 1580 chcr_send_wr(skb); 1581 1582 return -EINPROGRESS; 1583 } 1584 1585 static int chcr_ahash_digest(struct ahash_request *req) 1586 { 1587 struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(req); 1588 struct crypto_ahash *rtfm = crypto_ahash_reqtfm(req); 1589 struct chcr_context *ctx = crypto_tfm_ctx(crypto_ahash_tfm(rtfm)); 1590 struct uld_ctx *u_ctx = NULL; 1591 struct sk_buff *skb; 1592 struct hash_wr_param params; 1593 u8 bs; 1594 1595 rtfm->init(req); 1596 bs = crypto_tfm_alg_blocksize(crypto_ahash_tfm(rtfm)); 1597 1598 u_ctx = ULD_CTX(ctx); 1599 if (unlikely(cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0], 1600 ctx->tx_qidx))) { 1601 if (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) 1602 return -EBUSY; 1603 } 1604 1605 if (is_hmac(crypto_ahash_tfm(rtfm))) 1606 params.opad_needed = 1; 1607 else 1608 params.opad_needed = 0; 1609 1610 params.last = 0; 1611 params.more = 0; 1612 params.sg_len = req->nbytes; 1613 params.bfr_len = 0; 1614 params.scmd1 = 0; 1615 get_alg_config(¶ms.alg_prm, crypto_ahash_digestsize(rtfm)); 1616 req_ctx->result = 1; 1617 req_ctx->data_len += params.bfr_len + params.sg_len; 1618 1619 if (req->nbytes == 0) { 1620 create_last_hash_block(req_ctx->reqbfr, bs, 0); 1621 params.more = 1; 1622 params.bfr_len = bs; 1623 } 1624 1625 skb = create_hash_wr(req, ¶ms); 1626 if (!skb) 1627 return -ENOMEM; 1628 1629 skb->dev = u_ctx->lldi.ports[0]; 1630 set_wr_txq(skb, CPL_PRIORITY_DATA, ctx->tx_qidx); 1631 chcr_send_wr(skb); 1632 return -EINPROGRESS; 1633 } 1634 1635 static int chcr_ahash_export(struct ahash_request *areq, void *out) 1636 { 1637 struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(areq); 1638 struct chcr_ahash_req_ctx *state = out; 1639 1640 state->reqlen = req_ctx->reqlen; 1641 state->data_len = req_ctx->data_len; 1642 memcpy(state->bfr1, req_ctx->reqbfr, req_ctx->reqlen); 1643 memcpy(state->partial_hash, req_ctx->partial_hash, 1644 CHCR_HASH_MAX_DIGEST_SIZE); 1645 return 0; 1646 } 1647 1648 static int chcr_ahash_import(struct ahash_request *areq, const void *in) 1649 { 1650 struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(areq); 1651 struct chcr_ahash_req_ctx *state = (struct chcr_ahash_req_ctx *)in; 1652 1653 req_ctx->reqlen = state->reqlen; 1654 req_ctx->data_len = state->data_len; 1655 req_ctx->reqbfr = req_ctx->bfr1; 1656 req_ctx->skbfr = req_ctx->bfr2; 1657 memcpy(req_ctx->bfr1, state->bfr1, CHCR_HASH_MAX_BLOCK_SIZE_128); 1658 memcpy(req_ctx->partial_hash, state->partial_hash, 1659 CHCR_HASH_MAX_DIGEST_SIZE); 1660 return 0; 1661 } 1662 1663 static int chcr_ahash_setkey(struct crypto_ahash *tfm, const u8 *key, 1664 unsigned int keylen) 1665 { 1666 struct chcr_context *ctx = crypto_tfm_ctx(crypto_ahash_tfm(tfm)); 1667 struct hmac_ctx *hmacctx = HMAC_CTX(ctx); 1668 unsigned int digestsize = crypto_ahash_digestsize(tfm); 1669 unsigned int bs = crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm)); 1670 unsigned int i, err = 0, updated_digestsize; 1671 1672 SHASH_DESC_ON_STACK(shash, hmacctx->base_hash); 1673 1674 /* use the key to calculate the ipad and opad. ipad will sent with the 1675 * first request's data. opad will be sent with the final hash result 1676 * ipad in hmacctx->ipad and opad in hmacctx->opad location 1677 */ 1678 shash->tfm = hmacctx->base_hash; 1679 shash->flags = crypto_shash_get_flags(hmacctx->base_hash); 1680 if (keylen > bs) { 1681 err = crypto_shash_digest(shash, key, keylen, 1682 hmacctx->ipad); 1683 if (err) 1684 goto out; 1685 keylen = digestsize; 1686 } else { 1687 memcpy(hmacctx->ipad, key, keylen); 1688 } 1689 memset(hmacctx->ipad + keylen, 0, bs - keylen); 1690 memcpy(hmacctx->opad, hmacctx->ipad, bs); 1691 1692 for (i = 0; i < bs / sizeof(int); i++) { 1693 *((unsigned int *)(&hmacctx->ipad) + i) ^= IPAD_DATA; 1694 *((unsigned int *)(&hmacctx->opad) + i) ^= OPAD_DATA; 1695 } 1696 1697 updated_digestsize = digestsize; 1698 if (digestsize == SHA224_DIGEST_SIZE) 1699 updated_digestsize = SHA256_DIGEST_SIZE; 1700 else if (digestsize == SHA384_DIGEST_SIZE) 1701 updated_digestsize = SHA512_DIGEST_SIZE; 1702 err = chcr_compute_partial_hash(shash, hmacctx->ipad, 1703 hmacctx->ipad, digestsize); 1704 if (err) 1705 goto out; 1706 chcr_change_order(hmacctx->ipad, updated_digestsize); 1707 1708 err = chcr_compute_partial_hash(shash, hmacctx->opad, 1709 hmacctx->opad, digestsize); 1710 if (err) 1711 goto out; 1712 chcr_change_order(hmacctx->opad, updated_digestsize); 1713 out: 1714 return err; 1715 } 1716 1717 static int chcr_aes_xts_setkey(struct crypto_ablkcipher *cipher, const u8 *key, 1718 unsigned int key_len) 1719 { 1720 struct chcr_context *ctx = crypto_ablkcipher_ctx(cipher); 1721 struct ablk_ctx *ablkctx = ABLK_CTX(ctx); 1722 unsigned short context_size = 0; 1723 int err; 1724 1725 err = chcr_cipher_fallback_setkey(cipher, key, key_len); 1726 if (err) 1727 goto badkey_err; 1728 1729 memcpy(ablkctx->key, key, key_len); 1730 ablkctx->enckey_len = key_len; 1731 get_aes_decrypt_key(ablkctx->rrkey, ablkctx->key, key_len << 2); 1732 context_size = (KEY_CONTEXT_HDR_SALT_AND_PAD + key_len) >> 4; 1733 ablkctx->key_ctx_hdr = 1734 FILL_KEY_CTX_HDR((key_len == AES_KEYSIZE_256) ? 1735 CHCR_KEYCTX_CIPHER_KEY_SIZE_128 : 1736 CHCR_KEYCTX_CIPHER_KEY_SIZE_256, 1737 CHCR_KEYCTX_NO_KEY, 1, 1738 0, context_size); 1739 ablkctx->ciph_mode = CHCR_SCMD_CIPHER_MODE_AES_XTS; 1740 return 0; 1741 badkey_err: 1742 crypto_ablkcipher_set_flags(cipher, CRYPTO_TFM_RES_BAD_KEY_LEN); 1743 ablkctx->enckey_len = 0; 1744 1745 return err; 1746 } 1747 1748 static int chcr_sha_init(struct ahash_request *areq) 1749 { 1750 struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(areq); 1751 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); 1752 int digestsize = crypto_ahash_digestsize(tfm); 1753 1754 req_ctx->data_len = 0; 1755 req_ctx->reqlen = 0; 1756 req_ctx->reqbfr = req_ctx->bfr1; 1757 req_ctx->skbfr = req_ctx->bfr2; 1758 req_ctx->skb = NULL; 1759 req_ctx->result = 0; 1760 copy_hash_init_values(req_ctx->partial_hash, digestsize); 1761 return 0; 1762 } 1763 1764 static int chcr_sha_cra_init(struct crypto_tfm *tfm) 1765 { 1766 crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), 1767 sizeof(struct chcr_ahash_req_ctx)); 1768 return chcr_device_init(crypto_tfm_ctx(tfm)); 1769 } 1770 1771 static int chcr_hmac_init(struct ahash_request *areq) 1772 { 1773 struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(areq); 1774 struct crypto_ahash *rtfm = crypto_ahash_reqtfm(areq); 1775 struct chcr_context *ctx = crypto_tfm_ctx(crypto_ahash_tfm(rtfm)); 1776 struct hmac_ctx *hmacctx = HMAC_CTX(ctx); 1777 unsigned int digestsize = crypto_ahash_digestsize(rtfm); 1778 unsigned int bs = crypto_tfm_alg_blocksize(crypto_ahash_tfm(rtfm)); 1779 1780 chcr_sha_init(areq); 1781 req_ctx->data_len = bs; 1782 if (is_hmac(crypto_ahash_tfm(rtfm))) { 1783 if (digestsize == SHA224_DIGEST_SIZE) 1784 memcpy(req_ctx->partial_hash, hmacctx->ipad, 1785 SHA256_DIGEST_SIZE); 1786 else if (digestsize == SHA384_DIGEST_SIZE) 1787 memcpy(req_ctx->partial_hash, hmacctx->ipad, 1788 SHA512_DIGEST_SIZE); 1789 else 1790 memcpy(req_ctx->partial_hash, hmacctx->ipad, 1791 digestsize); 1792 } 1793 return 0; 1794 } 1795 1796 static int chcr_hmac_cra_init(struct crypto_tfm *tfm) 1797 { 1798 struct chcr_context *ctx = crypto_tfm_ctx(tfm); 1799 struct hmac_ctx *hmacctx = HMAC_CTX(ctx); 1800 unsigned int digestsize = 1801 crypto_ahash_digestsize(__crypto_ahash_cast(tfm)); 1802 1803 crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), 1804 sizeof(struct chcr_ahash_req_ctx)); 1805 hmacctx->base_hash = chcr_alloc_shash(digestsize); 1806 if (IS_ERR(hmacctx->base_hash)) 1807 return PTR_ERR(hmacctx->base_hash); 1808 return chcr_device_init(crypto_tfm_ctx(tfm)); 1809 } 1810 1811 static void chcr_hmac_cra_exit(struct crypto_tfm *tfm) 1812 { 1813 struct chcr_context *ctx = crypto_tfm_ctx(tfm); 1814 struct hmac_ctx *hmacctx = HMAC_CTX(ctx); 1815 1816 if (hmacctx->base_hash) { 1817 chcr_free_shash(hmacctx->base_hash); 1818 hmacctx->base_hash = NULL; 1819 } 1820 } 1821 1822 static int is_newsg(struct scatterlist *sgl, unsigned int *newents) 1823 { 1824 int nents = 0; 1825 int ret = 0; 1826 1827 while (sgl) { 1828 if (sgl->length > CHCR_SG_SIZE) 1829 ret = 1; 1830 nents += DIV_ROUND_UP(sgl->length, CHCR_SG_SIZE); 1831 sgl = sg_next(sgl); 1832 } 1833 *newents = nents; 1834 return ret; 1835 } 1836 1837 static inline void free_new_sg(struct scatterlist *sgl) 1838 { 1839 kfree(sgl); 1840 } 1841 1842 static struct scatterlist *alloc_new_sg(struct scatterlist *sgl, 1843 unsigned int nents) 1844 { 1845 struct scatterlist *newsg, *sg; 1846 int i, len, processed = 0; 1847 struct page *spage; 1848 int offset; 1849 1850 newsg = kmalloc_array(nents, sizeof(struct scatterlist), GFP_KERNEL); 1851 if (!newsg) 1852 return ERR_PTR(-ENOMEM); 1853 sg = newsg; 1854 sg_init_table(sg, nents); 1855 offset = sgl->offset; 1856 spage = sg_page(sgl); 1857 for (i = 0; i < nents; i++) { 1858 len = min_t(u32, sgl->length - processed, CHCR_SG_SIZE); 1859 sg_set_page(sg, spage, len, offset); 1860 processed += len; 1861 offset += len; 1862 if (offset >= PAGE_SIZE) { 1863 offset = offset % PAGE_SIZE; 1864 spage++; 1865 } 1866 if (processed == sgl->length) { 1867 processed = 0; 1868 sgl = sg_next(sgl); 1869 if (!sgl) 1870 break; 1871 spage = sg_page(sgl); 1872 offset = sgl->offset; 1873 } 1874 sg = sg_next(sg); 1875 } 1876 return newsg; 1877 } 1878 1879 static int chcr_copy_assoc(struct aead_request *req, 1880 struct chcr_aead_ctx *ctx) 1881 { 1882 SKCIPHER_REQUEST_ON_STACK(skreq, ctx->null); 1883 1884 skcipher_request_set_tfm(skreq, ctx->null); 1885 skcipher_request_set_callback(skreq, aead_request_flags(req), 1886 NULL, NULL); 1887 skcipher_request_set_crypt(skreq, req->src, req->dst, req->assoclen, 1888 NULL); 1889 1890 return crypto_skcipher_encrypt(skreq); 1891 } 1892 static int chcr_aead_need_fallback(struct aead_request *req, int src_nent, 1893 int aadmax, int wrlen, 1894 unsigned short op_type) 1895 { 1896 unsigned int authsize = crypto_aead_authsize(crypto_aead_reqtfm(req)); 1897 1898 if (((req->cryptlen - (op_type ? authsize : 0)) == 0) || 1899 (req->assoclen > aadmax) || 1900 (src_nent > MAX_SKB_FRAGS) || 1901 (wrlen > MAX_WR_SIZE)) 1902 return 1; 1903 return 0; 1904 } 1905 1906 static int chcr_aead_fallback(struct aead_request *req, unsigned short op_type) 1907 { 1908 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 1909 struct chcr_context *ctx = crypto_aead_ctx(tfm); 1910 struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx); 1911 struct aead_request *subreq = aead_request_ctx(req); 1912 1913 aead_request_set_tfm(subreq, aeadctx->sw_cipher); 1914 aead_request_set_callback(subreq, req->base.flags, 1915 req->base.complete, req->base.data); 1916 aead_request_set_crypt(subreq, req->src, req->dst, req->cryptlen, 1917 req->iv); 1918 aead_request_set_ad(subreq, req->assoclen); 1919 return op_type ? crypto_aead_decrypt(subreq) : 1920 crypto_aead_encrypt(subreq); 1921 } 1922 1923 static struct sk_buff *create_authenc_wr(struct aead_request *req, 1924 unsigned short qid, 1925 int size, 1926 unsigned short op_type) 1927 { 1928 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 1929 struct chcr_context *ctx = crypto_aead_ctx(tfm); 1930 struct uld_ctx *u_ctx = ULD_CTX(ctx); 1931 struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx); 1932 struct chcr_authenc_ctx *actx = AUTHENC_CTX(aeadctx); 1933 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req); 1934 struct sk_buff *skb = NULL; 1935 struct chcr_wr *chcr_req; 1936 struct cpl_rx_phys_dsgl *phys_cpl; 1937 struct phys_sge_parm sg_param; 1938 struct scatterlist *src; 1939 unsigned int frags = 0, transhdr_len; 1940 unsigned int ivsize = crypto_aead_ivsize(tfm), dst_size = 0; 1941 unsigned int kctx_len = 0, nents; 1942 unsigned short stop_offset = 0; 1943 unsigned int assoclen = req->assoclen; 1944 unsigned int authsize = crypto_aead_authsize(tfm); 1945 int error = -EINVAL, src_nent; 1946 int null = 0; 1947 gfp_t flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL : 1948 GFP_ATOMIC; 1949 struct adapter *adap = padap(ctx->dev); 1950 1951 reqctx->newdstsg = NULL; 1952 dst_size = req->assoclen + req->cryptlen + (op_type ? -authsize : 1953 authsize); 1954 if (aeadctx->enckey_len == 0 || (req->cryptlen <= 0)) 1955 goto err; 1956 1957 if (op_type && req->cryptlen < crypto_aead_authsize(tfm)) 1958 goto err; 1959 src_nent = sg_nents_for_len(req->src, req->assoclen + req->cryptlen); 1960 if (src_nent < 0) 1961 goto err; 1962 src = scatterwalk_ffwd(reqctx->srcffwd, req->src, req->assoclen); 1963 1964 if (req->src != req->dst) { 1965 error = chcr_copy_assoc(req, aeadctx); 1966 if (error) 1967 return ERR_PTR(error); 1968 } 1969 if (dst_size && is_newsg(req->dst, &nents)) { 1970 reqctx->newdstsg = alloc_new_sg(req->dst, nents); 1971 if (IS_ERR(reqctx->newdstsg)) 1972 return ERR_CAST(reqctx->newdstsg); 1973 reqctx->dst = scatterwalk_ffwd(reqctx->dstffwd, 1974 reqctx->newdstsg, req->assoclen); 1975 } else { 1976 if (req->src == req->dst) 1977 reqctx->dst = src; 1978 else 1979 reqctx->dst = scatterwalk_ffwd(reqctx->dstffwd, 1980 req->dst, req->assoclen); 1981 } 1982 if (get_aead_subtype(tfm) == CRYPTO_ALG_SUB_TYPE_AEAD_NULL) { 1983 null = 1; 1984 assoclen = 0; 1985 } 1986 reqctx->dst_nents = sg_nents_for_len(reqctx->dst, req->cryptlen + 1987 (op_type ? -authsize : authsize)); 1988 if (reqctx->dst_nents < 0) { 1989 pr_err("AUTHENC:Invalid Destination sg entries\n"); 1990 error = -EINVAL; 1991 goto err; 1992 } 1993 dst_size = get_space_for_phys_dsgl(reqctx->dst_nents); 1994 kctx_len = (ntohl(KEY_CONTEXT_CTX_LEN_V(aeadctx->key_ctx_hdr)) << 4) 1995 - sizeof(chcr_req->key_ctx); 1996 transhdr_len = CIPHER_TRANSHDR_SIZE(kctx_len, dst_size); 1997 if (chcr_aead_need_fallback(req, src_nent + MIN_AUTH_SG, 1998 T6_MAX_AAD_SIZE, 1999 transhdr_len + (sgl_len(src_nent + MIN_AUTH_SG) * 8), 2000 op_type)) { 2001 atomic_inc(&adap->chcr_stats.fallback); 2002 free_new_sg(reqctx->newdstsg); 2003 reqctx->newdstsg = NULL; 2004 return ERR_PTR(chcr_aead_fallback(req, op_type)); 2005 } 2006 skb = alloc_skb((transhdr_len + sizeof(struct sge_opaque_hdr)), flags); 2007 if (!skb) { 2008 error = -ENOMEM; 2009 goto err; 2010 } 2011 2012 /* LLD is going to write the sge hdr. */ 2013 skb_reserve(skb, sizeof(struct sge_opaque_hdr)); 2014 2015 /* Write WR */ 2016 chcr_req = __skb_put_zero(skb, transhdr_len); 2017 2018 stop_offset = (op_type == CHCR_ENCRYPT_OP) ? 0 : authsize; 2019 2020 /* 2021 * Input order is AAD,IV and Payload. where IV should be included as 2022 * the part of authdata. All other fields should be filled according 2023 * to the hardware spec 2024 */ 2025 chcr_req->sec_cpl.op_ivinsrtofst = 2026 FILL_SEC_CPL_OP_IVINSR(ctx->dev->rx_channel_id, 2, 2027 (ivsize ? (assoclen + 1) : 0)); 2028 chcr_req->sec_cpl.pldlen = htonl(assoclen + ivsize + req->cryptlen); 2029 chcr_req->sec_cpl.aadstart_cipherstop_hi = FILL_SEC_CPL_CIPHERSTOP_HI( 2030 assoclen ? 1 : 0, assoclen, 2031 assoclen + ivsize + 1, 2032 (stop_offset & 0x1F0) >> 4); 2033 chcr_req->sec_cpl.cipherstop_lo_authinsert = FILL_SEC_CPL_AUTHINSERT( 2034 stop_offset & 0xF, 2035 null ? 0 : assoclen + ivsize + 1, 2036 stop_offset, stop_offset); 2037 chcr_req->sec_cpl.seqno_numivs = FILL_SEC_CPL_SCMD0_SEQNO(op_type, 2038 (op_type == CHCR_ENCRYPT_OP) ? 1 : 0, 2039 CHCR_SCMD_CIPHER_MODE_AES_CBC, 2040 actx->auth_mode, aeadctx->hmac_ctrl, 2041 ivsize >> 1); 2042 chcr_req->sec_cpl.ivgen_hdrlen = FILL_SEC_CPL_IVGEN_HDRLEN(0, 0, 1, 2043 0, 1, dst_size); 2044 2045 chcr_req->key_ctx.ctx_hdr = aeadctx->key_ctx_hdr; 2046 if (op_type == CHCR_ENCRYPT_OP) 2047 memcpy(chcr_req->key_ctx.key, aeadctx->key, 2048 aeadctx->enckey_len); 2049 else 2050 memcpy(chcr_req->key_ctx.key, actx->dec_rrkey, 2051 aeadctx->enckey_len); 2052 2053 memcpy(chcr_req->key_ctx.key + (DIV_ROUND_UP(aeadctx->enckey_len, 16) << 2054 4), actx->h_iopad, kctx_len - 2055 (DIV_ROUND_UP(aeadctx->enckey_len, 16) << 4)); 2056 2057 phys_cpl = (struct cpl_rx_phys_dsgl *)((u8 *)(chcr_req + 1) + kctx_len); 2058 sg_param.nents = reqctx->dst_nents; 2059 sg_param.obsize = req->cryptlen + (op_type ? -authsize : authsize); 2060 sg_param.qid = qid; 2061 error = map_writesg_phys_cpl(&u_ctx->lldi.pdev->dev, phys_cpl, 2062 reqctx->dst, &sg_param); 2063 if (error) 2064 goto dstmap_fail; 2065 2066 skb_set_transport_header(skb, transhdr_len); 2067 2068 if (assoclen) { 2069 /* AAD buffer in */ 2070 write_sg_to_skb(skb, &frags, req->src, assoclen); 2071 2072 } 2073 write_buffer_to_skb(skb, &frags, req->iv, ivsize); 2074 write_sg_to_skb(skb, &frags, src, req->cryptlen); 2075 atomic_inc(&adap->chcr_stats.cipher_rqst); 2076 create_wreq(ctx, chcr_req, &req->base, skb, kctx_len, size, 1, 2077 sizeof(struct cpl_rx_phys_dsgl) + dst_size, 0); 2078 reqctx->skb = skb; 2079 skb_get(skb); 2080 2081 return skb; 2082 dstmap_fail: 2083 /* ivmap_fail: */ 2084 kfree_skb(skb); 2085 err: 2086 free_new_sg(reqctx->newdstsg); 2087 reqctx->newdstsg = NULL; 2088 return ERR_PTR(error); 2089 } 2090 2091 static int set_msg_len(u8 *block, unsigned int msglen, int csize) 2092 { 2093 __be32 data; 2094 2095 memset(block, 0, csize); 2096 block += csize; 2097 2098 if (csize >= 4) 2099 csize = 4; 2100 else if (msglen > (unsigned int)(1 << (8 * csize))) 2101 return -EOVERFLOW; 2102 2103 data = cpu_to_be32(msglen); 2104 memcpy(block - csize, (u8 *)&data + 4 - csize, csize); 2105 2106 return 0; 2107 } 2108 2109 static void generate_b0(struct aead_request *req, 2110 struct chcr_aead_ctx *aeadctx, 2111 unsigned short op_type) 2112 { 2113 unsigned int l, lp, m; 2114 int rc; 2115 struct crypto_aead *aead = crypto_aead_reqtfm(req); 2116 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req); 2117 u8 *b0 = reqctx->scratch_pad; 2118 2119 m = crypto_aead_authsize(aead); 2120 2121 memcpy(b0, reqctx->iv, 16); 2122 2123 lp = b0[0]; 2124 l = lp + 1; 2125 2126 /* set m, bits 3-5 */ 2127 *b0 |= (8 * ((m - 2) / 2)); 2128 2129 /* set adata, bit 6, if associated data is used */ 2130 if (req->assoclen) 2131 *b0 |= 64; 2132 rc = set_msg_len(b0 + 16 - l, 2133 (op_type == CHCR_DECRYPT_OP) ? 2134 req->cryptlen - m : req->cryptlen, l); 2135 } 2136 2137 static inline int crypto_ccm_check_iv(const u8 *iv) 2138 { 2139 /* 2 <= L <= 8, so 1 <= L' <= 7. */ 2140 if (iv[0] < 1 || iv[0] > 7) 2141 return -EINVAL; 2142 2143 return 0; 2144 } 2145 2146 static int ccm_format_packet(struct aead_request *req, 2147 struct chcr_aead_ctx *aeadctx, 2148 unsigned int sub_type, 2149 unsigned short op_type) 2150 { 2151 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req); 2152 int rc = 0; 2153 2154 if (sub_type == CRYPTO_ALG_SUB_TYPE_AEAD_RFC4309) { 2155 reqctx->iv[0] = 3; 2156 memcpy(reqctx->iv + 1, &aeadctx->salt[0], 3); 2157 memcpy(reqctx->iv + 4, req->iv, 8); 2158 memset(reqctx->iv + 12, 0, 4); 2159 *((unsigned short *)(reqctx->scratch_pad + 16)) = 2160 htons(req->assoclen - 8); 2161 } else { 2162 memcpy(reqctx->iv, req->iv, 16); 2163 *((unsigned short *)(reqctx->scratch_pad + 16)) = 2164 htons(req->assoclen); 2165 } 2166 generate_b0(req, aeadctx, op_type); 2167 /* zero the ctr value */ 2168 memset(reqctx->iv + 15 - reqctx->iv[0], 0, reqctx->iv[0] + 1); 2169 return rc; 2170 } 2171 2172 static void fill_sec_cpl_for_aead(struct cpl_tx_sec_pdu *sec_cpl, 2173 unsigned int dst_size, 2174 struct aead_request *req, 2175 unsigned short op_type, 2176 struct chcr_context *chcrctx) 2177 { 2178 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 2179 struct chcr_aead_ctx *aeadctx = AEAD_CTX(crypto_aead_ctx(tfm)); 2180 unsigned int ivsize = AES_BLOCK_SIZE; 2181 unsigned int cipher_mode = CHCR_SCMD_CIPHER_MODE_AES_CCM; 2182 unsigned int mac_mode = CHCR_SCMD_AUTH_MODE_CBCMAC; 2183 unsigned int c_id = chcrctx->dev->rx_channel_id; 2184 unsigned int ccm_xtra; 2185 unsigned char tag_offset = 0, auth_offset = 0; 2186 unsigned int assoclen; 2187 2188 if (get_aead_subtype(tfm) == CRYPTO_ALG_SUB_TYPE_AEAD_RFC4309) 2189 assoclen = req->assoclen - 8; 2190 else 2191 assoclen = req->assoclen; 2192 ccm_xtra = CCM_B0_SIZE + 2193 ((assoclen) ? CCM_AAD_FIELD_SIZE : 0); 2194 2195 auth_offset = req->cryptlen ? 2196 (assoclen + ivsize + 1 + ccm_xtra) : 0; 2197 if (op_type == CHCR_DECRYPT_OP) { 2198 if (crypto_aead_authsize(tfm) != req->cryptlen) 2199 tag_offset = crypto_aead_authsize(tfm); 2200 else 2201 auth_offset = 0; 2202 } 2203 2204 2205 sec_cpl->op_ivinsrtofst = FILL_SEC_CPL_OP_IVINSR(c_id, 2206 2, (ivsize ? (assoclen + 1) : 0) + 2207 ccm_xtra); 2208 sec_cpl->pldlen = 2209 htonl(assoclen + ivsize + req->cryptlen + ccm_xtra); 2210 /* For CCM there wil be b0 always. So AAD start will be 1 always */ 2211 sec_cpl->aadstart_cipherstop_hi = FILL_SEC_CPL_CIPHERSTOP_HI( 2212 1, assoclen + ccm_xtra, assoclen 2213 + ivsize + 1 + ccm_xtra, 0); 2214 2215 sec_cpl->cipherstop_lo_authinsert = FILL_SEC_CPL_AUTHINSERT(0, 2216 auth_offset, tag_offset, 2217 (op_type == CHCR_ENCRYPT_OP) ? 0 : 2218 crypto_aead_authsize(tfm)); 2219 sec_cpl->seqno_numivs = FILL_SEC_CPL_SCMD0_SEQNO(op_type, 2220 (op_type == CHCR_ENCRYPT_OP) ? 0 : 1, 2221 cipher_mode, mac_mode, 2222 aeadctx->hmac_ctrl, ivsize >> 1); 2223 2224 sec_cpl->ivgen_hdrlen = FILL_SEC_CPL_IVGEN_HDRLEN(0, 0, 1, 0, 2225 1, dst_size); 2226 } 2227 2228 int aead_ccm_validate_input(unsigned short op_type, 2229 struct aead_request *req, 2230 struct chcr_aead_ctx *aeadctx, 2231 unsigned int sub_type) 2232 { 2233 if (sub_type != CRYPTO_ALG_SUB_TYPE_AEAD_RFC4309) { 2234 if (crypto_ccm_check_iv(req->iv)) { 2235 pr_err("CCM: IV check fails\n"); 2236 return -EINVAL; 2237 } 2238 } else { 2239 if (req->assoclen != 16 && req->assoclen != 20) { 2240 pr_err("RFC4309: Invalid AAD length %d\n", 2241 req->assoclen); 2242 return -EINVAL; 2243 } 2244 } 2245 if (aeadctx->enckey_len == 0) { 2246 pr_err("CCM: Encryption key not set\n"); 2247 return -EINVAL; 2248 } 2249 return 0; 2250 } 2251 2252 unsigned int fill_aead_req_fields(struct sk_buff *skb, 2253 struct aead_request *req, 2254 struct scatterlist *src, 2255 unsigned int ivsize, 2256 struct chcr_aead_ctx *aeadctx) 2257 { 2258 unsigned int frags = 0; 2259 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 2260 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req); 2261 /* b0 and aad length(if available) */ 2262 2263 write_buffer_to_skb(skb, &frags, reqctx->scratch_pad, CCM_B0_SIZE + 2264 (req->assoclen ? CCM_AAD_FIELD_SIZE : 0)); 2265 if (req->assoclen) { 2266 if (get_aead_subtype(tfm) == CRYPTO_ALG_SUB_TYPE_AEAD_RFC4309) 2267 write_sg_to_skb(skb, &frags, req->src, 2268 req->assoclen - 8); 2269 else 2270 write_sg_to_skb(skb, &frags, req->src, req->assoclen); 2271 } 2272 write_buffer_to_skb(skb, &frags, reqctx->iv, ivsize); 2273 if (req->cryptlen) 2274 write_sg_to_skb(skb, &frags, src, req->cryptlen); 2275 2276 return frags; 2277 } 2278 2279 static struct sk_buff *create_aead_ccm_wr(struct aead_request *req, 2280 unsigned short qid, 2281 int size, 2282 unsigned short op_type) 2283 { 2284 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 2285 struct chcr_context *ctx = crypto_aead_ctx(tfm); 2286 struct uld_ctx *u_ctx = ULD_CTX(ctx); 2287 struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx); 2288 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req); 2289 struct sk_buff *skb = NULL; 2290 struct chcr_wr *chcr_req; 2291 struct cpl_rx_phys_dsgl *phys_cpl; 2292 struct phys_sge_parm sg_param; 2293 struct scatterlist *src; 2294 unsigned int frags = 0, transhdr_len, ivsize = AES_BLOCK_SIZE; 2295 unsigned int dst_size = 0, kctx_len, nents; 2296 unsigned int sub_type; 2297 unsigned int authsize = crypto_aead_authsize(tfm); 2298 int error = -EINVAL, src_nent; 2299 gfp_t flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL : 2300 GFP_ATOMIC; 2301 struct adapter *adap = padap(ctx->dev); 2302 2303 dst_size = req->assoclen + req->cryptlen + (op_type ? -authsize : 2304 authsize); 2305 reqctx->newdstsg = NULL; 2306 if (op_type && req->cryptlen < crypto_aead_authsize(tfm)) 2307 goto err; 2308 src_nent = sg_nents_for_len(req->src, req->assoclen + req->cryptlen); 2309 if (src_nent < 0) 2310 goto err; 2311 2312 sub_type = get_aead_subtype(tfm); 2313 src = scatterwalk_ffwd(reqctx->srcffwd, req->src, req->assoclen); 2314 if (req->src != req->dst) { 2315 error = chcr_copy_assoc(req, aeadctx); 2316 if (error) { 2317 pr_err("AAD copy to destination buffer fails\n"); 2318 return ERR_PTR(error); 2319 } 2320 } 2321 if (dst_size && is_newsg(req->dst, &nents)) { 2322 reqctx->newdstsg = alloc_new_sg(req->dst, nents); 2323 if (IS_ERR(reqctx->newdstsg)) 2324 return ERR_CAST(reqctx->newdstsg); 2325 reqctx->dst = scatterwalk_ffwd(reqctx->dstffwd, 2326 reqctx->newdstsg, req->assoclen); 2327 } else { 2328 if (req->src == req->dst) 2329 reqctx->dst = src; 2330 else 2331 reqctx->dst = scatterwalk_ffwd(reqctx->dstffwd, 2332 req->dst, req->assoclen); 2333 } 2334 reqctx->dst_nents = sg_nents_for_len(reqctx->dst, req->cryptlen + 2335 (op_type ? -authsize : authsize)); 2336 if (reqctx->dst_nents < 0) { 2337 pr_err("CCM:Invalid Destination sg entries\n"); 2338 error = -EINVAL; 2339 goto err; 2340 } 2341 error = aead_ccm_validate_input(op_type, req, aeadctx, sub_type); 2342 if (error) 2343 goto err; 2344 2345 dst_size = get_space_for_phys_dsgl(reqctx->dst_nents); 2346 kctx_len = ((DIV_ROUND_UP(aeadctx->enckey_len, 16)) << 4) * 2; 2347 transhdr_len = CIPHER_TRANSHDR_SIZE(kctx_len, dst_size); 2348 if (chcr_aead_need_fallback(req, src_nent + MIN_CCM_SG, 2349 T6_MAX_AAD_SIZE - 18, 2350 transhdr_len + (sgl_len(src_nent + MIN_CCM_SG) * 8), 2351 op_type)) { 2352 atomic_inc(&adap->chcr_stats.fallback); 2353 free_new_sg(reqctx->newdstsg); 2354 reqctx->newdstsg = NULL; 2355 return ERR_PTR(chcr_aead_fallback(req, op_type)); 2356 } 2357 2358 skb = alloc_skb((transhdr_len + sizeof(struct sge_opaque_hdr)), flags); 2359 2360 if (!skb) { 2361 error = -ENOMEM; 2362 goto err; 2363 } 2364 2365 skb_reserve(skb, sizeof(struct sge_opaque_hdr)); 2366 2367 chcr_req = __skb_put_zero(skb, transhdr_len); 2368 2369 fill_sec_cpl_for_aead(&chcr_req->sec_cpl, dst_size, req, op_type, ctx); 2370 2371 chcr_req->key_ctx.ctx_hdr = aeadctx->key_ctx_hdr; 2372 memcpy(chcr_req->key_ctx.key, aeadctx->key, aeadctx->enckey_len); 2373 memcpy(chcr_req->key_ctx.key + (DIV_ROUND_UP(aeadctx->enckey_len, 16) * 2374 16), aeadctx->key, aeadctx->enckey_len); 2375 2376 phys_cpl = (struct cpl_rx_phys_dsgl *)((u8 *)(chcr_req + 1) + kctx_len); 2377 error = ccm_format_packet(req, aeadctx, sub_type, op_type); 2378 if (error) 2379 goto dstmap_fail; 2380 2381 sg_param.nents = reqctx->dst_nents; 2382 sg_param.obsize = req->cryptlen + (op_type ? -authsize : authsize); 2383 sg_param.qid = qid; 2384 error = map_writesg_phys_cpl(&u_ctx->lldi.pdev->dev, phys_cpl, 2385 reqctx->dst, &sg_param); 2386 if (error) 2387 goto dstmap_fail; 2388 2389 skb_set_transport_header(skb, transhdr_len); 2390 frags = fill_aead_req_fields(skb, req, src, ivsize, aeadctx); 2391 atomic_inc(&adap->chcr_stats.aead_rqst); 2392 create_wreq(ctx, chcr_req, &req->base, skb, kctx_len, 0, 1, 2393 sizeof(struct cpl_rx_phys_dsgl) + dst_size, 0); 2394 reqctx->skb = skb; 2395 skb_get(skb); 2396 return skb; 2397 dstmap_fail: 2398 kfree_skb(skb); 2399 err: 2400 free_new_sg(reqctx->newdstsg); 2401 reqctx->newdstsg = NULL; 2402 return ERR_PTR(error); 2403 } 2404 2405 static struct sk_buff *create_gcm_wr(struct aead_request *req, 2406 unsigned short qid, 2407 int size, 2408 unsigned short op_type) 2409 { 2410 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 2411 struct chcr_context *ctx = crypto_aead_ctx(tfm); 2412 struct uld_ctx *u_ctx = ULD_CTX(ctx); 2413 struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx); 2414 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req); 2415 struct sk_buff *skb = NULL; 2416 struct chcr_wr *chcr_req; 2417 struct cpl_rx_phys_dsgl *phys_cpl; 2418 struct phys_sge_parm sg_param; 2419 struct scatterlist *src; 2420 unsigned int frags = 0, transhdr_len; 2421 unsigned int ivsize = AES_BLOCK_SIZE; 2422 unsigned int dst_size = 0, kctx_len, nents, assoclen = req->assoclen; 2423 unsigned char tag_offset = 0; 2424 unsigned int authsize = crypto_aead_authsize(tfm); 2425 int error = -EINVAL, src_nent; 2426 gfp_t flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL : 2427 GFP_ATOMIC; 2428 struct adapter *adap = padap(ctx->dev); 2429 2430 reqctx->newdstsg = NULL; 2431 dst_size = assoclen + req->cryptlen + (op_type ? -authsize : 2432 authsize); 2433 /* validate key size */ 2434 if (aeadctx->enckey_len == 0) 2435 goto err; 2436 2437 if (op_type && req->cryptlen < crypto_aead_authsize(tfm)) 2438 goto err; 2439 src_nent = sg_nents_for_len(req->src, assoclen + req->cryptlen); 2440 if (src_nent < 0) 2441 goto err; 2442 2443 src = scatterwalk_ffwd(reqctx->srcffwd, req->src, assoclen); 2444 if (req->src != req->dst) { 2445 error = chcr_copy_assoc(req, aeadctx); 2446 if (error) 2447 return ERR_PTR(error); 2448 } 2449 2450 if (dst_size && is_newsg(req->dst, &nents)) { 2451 reqctx->newdstsg = alloc_new_sg(req->dst, nents); 2452 if (IS_ERR(reqctx->newdstsg)) 2453 return ERR_CAST(reqctx->newdstsg); 2454 reqctx->dst = scatterwalk_ffwd(reqctx->dstffwd, 2455 reqctx->newdstsg, assoclen); 2456 } else { 2457 if (req->src == req->dst) 2458 reqctx->dst = src; 2459 else 2460 reqctx->dst = scatterwalk_ffwd(reqctx->dstffwd, 2461 req->dst, assoclen); 2462 } 2463 2464 reqctx->dst_nents = sg_nents_for_len(reqctx->dst, req->cryptlen + 2465 (op_type ? -authsize : authsize)); 2466 if (reqctx->dst_nents < 0) { 2467 pr_err("GCM:Invalid Destination sg entries\n"); 2468 error = -EINVAL; 2469 goto err; 2470 } 2471 2472 2473 dst_size = get_space_for_phys_dsgl(reqctx->dst_nents); 2474 kctx_len = ((DIV_ROUND_UP(aeadctx->enckey_len, 16)) << 4) + 2475 AEAD_H_SIZE; 2476 transhdr_len = CIPHER_TRANSHDR_SIZE(kctx_len, dst_size); 2477 if (chcr_aead_need_fallback(req, src_nent + MIN_GCM_SG, 2478 T6_MAX_AAD_SIZE, 2479 transhdr_len + (sgl_len(src_nent + MIN_GCM_SG) * 8), 2480 op_type)) { 2481 atomic_inc(&adap->chcr_stats.fallback); 2482 free_new_sg(reqctx->newdstsg); 2483 reqctx->newdstsg = NULL; 2484 return ERR_PTR(chcr_aead_fallback(req, op_type)); 2485 } 2486 skb = alloc_skb((transhdr_len + sizeof(struct sge_opaque_hdr)), flags); 2487 if (!skb) { 2488 error = -ENOMEM; 2489 goto err; 2490 } 2491 2492 /* NIC driver is going to write the sge hdr. */ 2493 skb_reserve(skb, sizeof(struct sge_opaque_hdr)); 2494 2495 chcr_req = __skb_put_zero(skb, transhdr_len); 2496 2497 if (get_aead_subtype(tfm) == CRYPTO_ALG_SUB_TYPE_AEAD_RFC4106) 2498 assoclen = req->assoclen - 8; 2499 2500 tag_offset = (op_type == CHCR_ENCRYPT_OP) ? 0 : authsize; 2501 chcr_req->sec_cpl.op_ivinsrtofst = FILL_SEC_CPL_OP_IVINSR( 2502 ctx->dev->rx_channel_id, 2, (ivsize ? 2503 (assoclen + 1) : 0)); 2504 chcr_req->sec_cpl.pldlen = 2505 htonl(assoclen + ivsize + req->cryptlen); 2506 chcr_req->sec_cpl.aadstart_cipherstop_hi = FILL_SEC_CPL_CIPHERSTOP_HI( 2507 assoclen ? 1 : 0, assoclen, 2508 assoclen + ivsize + 1, 0); 2509 chcr_req->sec_cpl.cipherstop_lo_authinsert = 2510 FILL_SEC_CPL_AUTHINSERT(0, assoclen + ivsize + 1, 2511 tag_offset, tag_offset); 2512 chcr_req->sec_cpl.seqno_numivs = 2513 FILL_SEC_CPL_SCMD0_SEQNO(op_type, (op_type == 2514 CHCR_ENCRYPT_OP) ? 1 : 0, 2515 CHCR_SCMD_CIPHER_MODE_AES_GCM, 2516 CHCR_SCMD_AUTH_MODE_GHASH, 2517 aeadctx->hmac_ctrl, ivsize >> 1); 2518 chcr_req->sec_cpl.ivgen_hdrlen = FILL_SEC_CPL_IVGEN_HDRLEN(0, 0, 1, 2519 0, 1, dst_size); 2520 chcr_req->key_ctx.ctx_hdr = aeadctx->key_ctx_hdr; 2521 memcpy(chcr_req->key_ctx.key, aeadctx->key, aeadctx->enckey_len); 2522 memcpy(chcr_req->key_ctx.key + (DIV_ROUND_UP(aeadctx->enckey_len, 16) * 2523 16), GCM_CTX(aeadctx)->ghash_h, AEAD_H_SIZE); 2524 2525 /* prepare a 16 byte iv */ 2526 /* S A L T | IV | 0x00000001 */ 2527 if (get_aead_subtype(tfm) == 2528 CRYPTO_ALG_SUB_TYPE_AEAD_RFC4106) { 2529 memcpy(reqctx->iv, aeadctx->salt, 4); 2530 memcpy(reqctx->iv + 4, req->iv, 8); 2531 } else { 2532 memcpy(reqctx->iv, req->iv, 12); 2533 } 2534 *((unsigned int *)(reqctx->iv + 12)) = htonl(0x01); 2535 2536 phys_cpl = (struct cpl_rx_phys_dsgl *)((u8 *)(chcr_req + 1) + kctx_len); 2537 sg_param.nents = reqctx->dst_nents; 2538 sg_param.obsize = req->cryptlen + (op_type ? -authsize : authsize); 2539 sg_param.qid = qid; 2540 error = map_writesg_phys_cpl(&u_ctx->lldi.pdev->dev, phys_cpl, 2541 reqctx->dst, &sg_param); 2542 if (error) 2543 goto dstmap_fail; 2544 2545 skb_set_transport_header(skb, transhdr_len); 2546 write_sg_to_skb(skb, &frags, req->src, assoclen); 2547 write_buffer_to_skb(skb, &frags, reqctx->iv, ivsize); 2548 write_sg_to_skb(skb, &frags, src, req->cryptlen); 2549 atomic_inc(&adap->chcr_stats.aead_rqst); 2550 create_wreq(ctx, chcr_req, &req->base, skb, kctx_len, size, 1, 2551 sizeof(struct cpl_rx_phys_dsgl) + dst_size, 2552 reqctx->verify); 2553 reqctx->skb = skb; 2554 skb_get(skb); 2555 return skb; 2556 2557 dstmap_fail: 2558 /* ivmap_fail: */ 2559 kfree_skb(skb); 2560 err: 2561 free_new_sg(reqctx->newdstsg); 2562 reqctx->newdstsg = NULL; 2563 return ERR_PTR(error); 2564 } 2565 2566 2567 2568 static int chcr_aead_cra_init(struct crypto_aead *tfm) 2569 { 2570 struct chcr_context *ctx = crypto_aead_ctx(tfm); 2571 struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx); 2572 struct aead_alg *alg = crypto_aead_alg(tfm); 2573 2574 aeadctx->sw_cipher = crypto_alloc_aead(alg->base.cra_name, 0, 2575 CRYPTO_ALG_NEED_FALLBACK | 2576 CRYPTO_ALG_ASYNC); 2577 if (IS_ERR(aeadctx->sw_cipher)) 2578 return PTR_ERR(aeadctx->sw_cipher); 2579 crypto_aead_set_reqsize(tfm, max(sizeof(struct chcr_aead_reqctx), 2580 sizeof(struct aead_request) + 2581 crypto_aead_reqsize(aeadctx->sw_cipher))); 2582 aeadctx->null = crypto_get_default_null_skcipher(); 2583 if (IS_ERR(aeadctx->null)) 2584 return PTR_ERR(aeadctx->null); 2585 return chcr_device_init(ctx); 2586 } 2587 2588 static void chcr_aead_cra_exit(struct crypto_aead *tfm) 2589 { 2590 struct chcr_context *ctx = crypto_aead_ctx(tfm); 2591 struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx); 2592 2593 crypto_put_default_null_skcipher(); 2594 crypto_free_aead(aeadctx->sw_cipher); 2595 } 2596 2597 static int chcr_authenc_null_setauthsize(struct crypto_aead *tfm, 2598 unsigned int authsize) 2599 { 2600 struct chcr_aead_ctx *aeadctx = AEAD_CTX(crypto_aead_ctx(tfm)); 2601 2602 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_NOP; 2603 aeadctx->mayverify = VERIFY_HW; 2604 return crypto_aead_setauthsize(aeadctx->sw_cipher, authsize); 2605 } 2606 static int chcr_authenc_setauthsize(struct crypto_aead *tfm, 2607 unsigned int authsize) 2608 { 2609 struct chcr_aead_ctx *aeadctx = AEAD_CTX(crypto_aead_ctx(tfm)); 2610 u32 maxauth = crypto_aead_maxauthsize(tfm); 2611 2612 /*SHA1 authsize in ipsec is 12 instead of 10 i.e maxauthsize / 2 is not 2613 * true for sha1. authsize == 12 condition should be before 2614 * authsize == (maxauth >> 1) 2615 */ 2616 if (authsize == ICV_4) { 2617 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_PL1; 2618 aeadctx->mayverify = VERIFY_HW; 2619 } else if (authsize == ICV_6) { 2620 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_PL2; 2621 aeadctx->mayverify = VERIFY_HW; 2622 } else if (authsize == ICV_10) { 2623 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_TRUNC_RFC4366; 2624 aeadctx->mayverify = VERIFY_HW; 2625 } else if (authsize == ICV_12) { 2626 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_IPSEC_96BIT; 2627 aeadctx->mayverify = VERIFY_HW; 2628 } else if (authsize == ICV_14) { 2629 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_PL3; 2630 aeadctx->mayverify = VERIFY_HW; 2631 } else if (authsize == (maxauth >> 1)) { 2632 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_DIV2; 2633 aeadctx->mayverify = VERIFY_HW; 2634 } else if (authsize == maxauth) { 2635 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_NO_TRUNC; 2636 aeadctx->mayverify = VERIFY_HW; 2637 } else { 2638 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_NO_TRUNC; 2639 aeadctx->mayverify = VERIFY_SW; 2640 } 2641 return crypto_aead_setauthsize(aeadctx->sw_cipher, authsize); 2642 } 2643 2644 2645 static int chcr_gcm_setauthsize(struct crypto_aead *tfm, unsigned int authsize) 2646 { 2647 struct chcr_aead_ctx *aeadctx = AEAD_CTX(crypto_aead_ctx(tfm)); 2648 2649 switch (authsize) { 2650 case ICV_4: 2651 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_PL1; 2652 aeadctx->mayverify = VERIFY_HW; 2653 break; 2654 case ICV_8: 2655 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_DIV2; 2656 aeadctx->mayverify = VERIFY_HW; 2657 break; 2658 case ICV_12: 2659 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_IPSEC_96BIT; 2660 aeadctx->mayverify = VERIFY_HW; 2661 break; 2662 case ICV_14: 2663 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_PL3; 2664 aeadctx->mayverify = VERIFY_HW; 2665 break; 2666 case ICV_16: 2667 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_NO_TRUNC; 2668 aeadctx->mayverify = VERIFY_HW; 2669 break; 2670 case ICV_13: 2671 case ICV_15: 2672 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_NO_TRUNC; 2673 aeadctx->mayverify = VERIFY_SW; 2674 break; 2675 default: 2676 2677 crypto_tfm_set_flags((struct crypto_tfm *) tfm, 2678 CRYPTO_TFM_RES_BAD_KEY_LEN); 2679 return -EINVAL; 2680 } 2681 return crypto_aead_setauthsize(aeadctx->sw_cipher, authsize); 2682 } 2683 2684 static int chcr_4106_4309_setauthsize(struct crypto_aead *tfm, 2685 unsigned int authsize) 2686 { 2687 struct chcr_aead_ctx *aeadctx = AEAD_CTX(crypto_aead_ctx(tfm)); 2688 2689 switch (authsize) { 2690 case ICV_8: 2691 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_DIV2; 2692 aeadctx->mayverify = VERIFY_HW; 2693 break; 2694 case ICV_12: 2695 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_IPSEC_96BIT; 2696 aeadctx->mayverify = VERIFY_HW; 2697 break; 2698 case ICV_16: 2699 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_NO_TRUNC; 2700 aeadctx->mayverify = VERIFY_HW; 2701 break; 2702 default: 2703 crypto_tfm_set_flags((struct crypto_tfm *)tfm, 2704 CRYPTO_TFM_RES_BAD_KEY_LEN); 2705 return -EINVAL; 2706 } 2707 return crypto_aead_setauthsize(aeadctx->sw_cipher, authsize); 2708 } 2709 2710 static int chcr_ccm_setauthsize(struct crypto_aead *tfm, 2711 unsigned int authsize) 2712 { 2713 struct chcr_aead_ctx *aeadctx = AEAD_CTX(crypto_aead_ctx(tfm)); 2714 2715 switch (authsize) { 2716 case ICV_4: 2717 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_PL1; 2718 aeadctx->mayverify = VERIFY_HW; 2719 break; 2720 case ICV_6: 2721 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_PL2; 2722 aeadctx->mayverify = VERIFY_HW; 2723 break; 2724 case ICV_8: 2725 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_DIV2; 2726 aeadctx->mayverify = VERIFY_HW; 2727 break; 2728 case ICV_10: 2729 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_TRUNC_RFC4366; 2730 aeadctx->mayverify = VERIFY_HW; 2731 break; 2732 case ICV_12: 2733 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_IPSEC_96BIT; 2734 aeadctx->mayverify = VERIFY_HW; 2735 break; 2736 case ICV_14: 2737 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_PL3; 2738 aeadctx->mayverify = VERIFY_HW; 2739 break; 2740 case ICV_16: 2741 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_NO_TRUNC; 2742 aeadctx->mayverify = VERIFY_HW; 2743 break; 2744 default: 2745 crypto_tfm_set_flags((struct crypto_tfm *)tfm, 2746 CRYPTO_TFM_RES_BAD_KEY_LEN); 2747 return -EINVAL; 2748 } 2749 return crypto_aead_setauthsize(aeadctx->sw_cipher, authsize); 2750 } 2751 2752 static int chcr_ccm_common_setkey(struct crypto_aead *aead, 2753 const u8 *key, 2754 unsigned int keylen) 2755 { 2756 struct chcr_context *ctx = crypto_aead_ctx(aead); 2757 struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx); 2758 unsigned char ck_size, mk_size; 2759 int key_ctx_size = 0; 2760 2761 key_ctx_size = sizeof(struct _key_ctx) + 2762 ((DIV_ROUND_UP(keylen, 16)) << 4) * 2; 2763 if (keylen == AES_KEYSIZE_128) { 2764 mk_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_128; 2765 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_128; 2766 } else if (keylen == AES_KEYSIZE_192) { 2767 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_192; 2768 mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_192; 2769 } else if (keylen == AES_KEYSIZE_256) { 2770 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_256; 2771 mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_256; 2772 } else { 2773 crypto_tfm_set_flags((struct crypto_tfm *)aead, 2774 CRYPTO_TFM_RES_BAD_KEY_LEN); 2775 aeadctx->enckey_len = 0; 2776 return -EINVAL; 2777 } 2778 aeadctx->key_ctx_hdr = FILL_KEY_CTX_HDR(ck_size, mk_size, 0, 0, 2779 key_ctx_size >> 4); 2780 memcpy(aeadctx->key, key, keylen); 2781 aeadctx->enckey_len = keylen; 2782 2783 return 0; 2784 } 2785 2786 static int chcr_aead_ccm_setkey(struct crypto_aead *aead, 2787 const u8 *key, 2788 unsigned int keylen) 2789 { 2790 struct chcr_context *ctx = crypto_aead_ctx(aead); 2791 struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx); 2792 int error; 2793 2794 crypto_aead_clear_flags(aeadctx->sw_cipher, CRYPTO_TFM_REQ_MASK); 2795 crypto_aead_set_flags(aeadctx->sw_cipher, crypto_aead_get_flags(aead) & 2796 CRYPTO_TFM_REQ_MASK); 2797 error = crypto_aead_setkey(aeadctx->sw_cipher, key, keylen); 2798 crypto_aead_clear_flags(aead, CRYPTO_TFM_RES_MASK); 2799 crypto_aead_set_flags(aead, crypto_aead_get_flags(aeadctx->sw_cipher) & 2800 CRYPTO_TFM_RES_MASK); 2801 if (error) 2802 return error; 2803 return chcr_ccm_common_setkey(aead, key, keylen); 2804 } 2805 2806 static int chcr_aead_rfc4309_setkey(struct crypto_aead *aead, const u8 *key, 2807 unsigned int keylen) 2808 { 2809 struct chcr_context *ctx = crypto_aead_ctx(aead); 2810 struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx); 2811 int error; 2812 2813 if (keylen < 3) { 2814 crypto_tfm_set_flags((struct crypto_tfm *)aead, 2815 CRYPTO_TFM_RES_BAD_KEY_LEN); 2816 aeadctx->enckey_len = 0; 2817 return -EINVAL; 2818 } 2819 crypto_aead_clear_flags(aeadctx->sw_cipher, CRYPTO_TFM_REQ_MASK); 2820 crypto_aead_set_flags(aeadctx->sw_cipher, crypto_aead_get_flags(aead) & 2821 CRYPTO_TFM_REQ_MASK); 2822 error = crypto_aead_setkey(aeadctx->sw_cipher, key, keylen); 2823 crypto_aead_clear_flags(aead, CRYPTO_TFM_RES_MASK); 2824 crypto_aead_set_flags(aead, crypto_aead_get_flags(aeadctx->sw_cipher) & 2825 CRYPTO_TFM_RES_MASK); 2826 if (error) 2827 return error; 2828 keylen -= 3; 2829 memcpy(aeadctx->salt, key + keylen, 3); 2830 return chcr_ccm_common_setkey(aead, key, keylen); 2831 } 2832 2833 static int chcr_gcm_setkey(struct crypto_aead *aead, const u8 *key, 2834 unsigned int keylen) 2835 { 2836 struct chcr_context *ctx = crypto_aead_ctx(aead); 2837 struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx); 2838 struct chcr_gcm_ctx *gctx = GCM_CTX(aeadctx); 2839 struct crypto_cipher *cipher; 2840 unsigned int ck_size; 2841 int ret = 0, key_ctx_size = 0; 2842 2843 aeadctx->enckey_len = 0; 2844 crypto_aead_clear_flags(aeadctx->sw_cipher, CRYPTO_TFM_REQ_MASK); 2845 crypto_aead_set_flags(aeadctx->sw_cipher, crypto_aead_get_flags(aead) 2846 & CRYPTO_TFM_REQ_MASK); 2847 ret = crypto_aead_setkey(aeadctx->sw_cipher, key, keylen); 2848 crypto_aead_clear_flags(aead, CRYPTO_TFM_RES_MASK); 2849 crypto_aead_set_flags(aead, crypto_aead_get_flags(aeadctx->sw_cipher) & 2850 CRYPTO_TFM_RES_MASK); 2851 if (ret) 2852 goto out; 2853 2854 if (get_aead_subtype(aead) == CRYPTO_ALG_SUB_TYPE_AEAD_RFC4106 && 2855 keylen > 3) { 2856 keylen -= 4; /* nonce/salt is present in the last 4 bytes */ 2857 memcpy(aeadctx->salt, key + keylen, 4); 2858 } 2859 if (keylen == AES_KEYSIZE_128) { 2860 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_128; 2861 } else if (keylen == AES_KEYSIZE_192) { 2862 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_192; 2863 } else if (keylen == AES_KEYSIZE_256) { 2864 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_256; 2865 } else { 2866 crypto_tfm_set_flags((struct crypto_tfm *)aead, 2867 CRYPTO_TFM_RES_BAD_KEY_LEN); 2868 pr_err("GCM: Invalid key length %d\n", keylen); 2869 ret = -EINVAL; 2870 goto out; 2871 } 2872 2873 memcpy(aeadctx->key, key, keylen); 2874 aeadctx->enckey_len = keylen; 2875 key_ctx_size = sizeof(struct _key_ctx) + 2876 ((DIV_ROUND_UP(keylen, 16)) << 4) + 2877 AEAD_H_SIZE; 2878 aeadctx->key_ctx_hdr = FILL_KEY_CTX_HDR(ck_size, 2879 CHCR_KEYCTX_MAC_KEY_SIZE_128, 2880 0, 0, 2881 key_ctx_size >> 4); 2882 /* Calculate the H = CIPH(K, 0 repeated 16 times). 2883 * It will go in key context 2884 */ 2885 cipher = crypto_alloc_cipher("aes-generic", 0, 0); 2886 if (IS_ERR(cipher)) { 2887 aeadctx->enckey_len = 0; 2888 ret = -ENOMEM; 2889 goto out; 2890 } 2891 2892 ret = crypto_cipher_setkey(cipher, key, keylen); 2893 if (ret) { 2894 aeadctx->enckey_len = 0; 2895 goto out1; 2896 } 2897 memset(gctx->ghash_h, 0, AEAD_H_SIZE); 2898 crypto_cipher_encrypt_one(cipher, gctx->ghash_h, gctx->ghash_h); 2899 2900 out1: 2901 crypto_free_cipher(cipher); 2902 out: 2903 return ret; 2904 } 2905 2906 static int chcr_authenc_setkey(struct crypto_aead *authenc, const u8 *key, 2907 unsigned int keylen) 2908 { 2909 struct chcr_context *ctx = crypto_aead_ctx(authenc); 2910 struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx); 2911 struct chcr_authenc_ctx *actx = AUTHENC_CTX(aeadctx); 2912 /* it contains auth and cipher key both*/ 2913 struct crypto_authenc_keys keys; 2914 unsigned int bs; 2915 unsigned int max_authsize = crypto_aead_alg(authenc)->maxauthsize; 2916 int err = 0, i, key_ctx_len = 0; 2917 unsigned char ck_size = 0; 2918 unsigned char pad[CHCR_HASH_MAX_BLOCK_SIZE_128] = { 0 }; 2919 struct crypto_shash *base_hash = ERR_PTR(-EINVAL); 2920 struct algo_param param; 2921 int align; 2922 u8 *o_ptr = NULL; 2923 2924 crypto_aead_clear_flags(aeadctx->sw_cipher, CRYPTO_TFM_REQ_MASK); 2925 crypto_aead_set_flags(aeadctx->sw_cipher, crypto_aead_get_flags(authenc) 2926 & CRYPTO_TFM_REQ_MASK); 2927 err = crypto_aead_setkey(aeadctx->sw_cipher, key, keylen); 2928 crypto_aead_clear_flags(authenc, CRYPTO_TFM_RES_MASK); 2929 crypto_aead_set_flags(authenc, crypto_aead_get_flags(aeadctx->sw_cipher) 2930 & CRYPTO_TFM_RES_MASK); 2931 if (err) 2932 goto out; 2933 2934 if (crypto_authenc_extractkeys(&keys, key, keylen) != 0) { 2935 crypto_aead_set_flags(authenc, CRYPTO_TFM_RES_BAD_KEY_LEN); 2936 goto out; 2937 } 2938 2939 if (get_alg_config(¶m, max_authsize)) { 2940 pr_err("chcr : Unsupported digest size\n"); 2941 goto out; 2942 } 2943 if (keys.enckeylen == AES_KEYSIZE_128) { 2944 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_128; 2945 } else if (keys.enckeylen == AES_KEYSIZE_192) { 2946 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_192; 2947 } else if (keys.enckeylen == AES_KEYSIZE_256) { 2948 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_256; 2949 } else { 2950 pr_err("chcr : Unsupported cipher key\n"); 2951 goto out; 2952 } 2953 2954 /* Copy only encryption key. We use authkey to generate h(ipad) and 2955 * h(opad) so authkey is not needed again. authkeylen size have the 2956 * size of the hash digest size. 2957 */ 2958 memcpy(aeadctx->key, keys.enckey, keys.enckeylen); 2959 aeadctx->enckey_len = keys.enckeylen; 2960 get_aes_decrypt_key(actx->dec_rrkey, aeadctx->key, 2961 aeadctx->enckey_len << 3); 2962 2963 base_hash = chcr_alloc_shash(max_authsize); 2964 if (IS_ERR(base_hash)) { 2965 pr_err("chcr : Base driver cannot be loaded\n"); 2966 aeadctx->enckey_len = 0; 2967 return -EINVAL; 2968 } 2969 { 2970 SHASH_DESC_ON_STACK(shash, base_hash); 2971 shash->tfm = base_hash; 2972 shash->flags = crypto_shash_get_flags(base_hash); 2973 bs = crypto_shash_blocksize(base_hash); 2974 align = KEYCTX_ALIGN_PAD(max_authsize); 2975 o_ptr = actx->h_iopad + param.result_size + align; 2976 2977 if (keys.authkeylen > bs) { 2978 err = crypto_shash_digest(shash, keys.authkey, 2979 keys.authkeylen, 2980 o_ptr); 2981 if (err) { 2982 pr_err("chcr : Base driver cannot be loaded\n"); 2983 goto out; 2984 } 2985 keys.authkeylen = max_authsize; 2986 } else 2987 memcpy(o_ptr, keys.authkey, keys.authkeylen); 2988 2989 /* Compute the ipad-digest*/ 2990 memset(pad + keys.authkeylen, 0, bs - keys.authkeylen); 2991 memcpy(pad, o_ptr, keys.authkeylen); 2992 for (i = 0; i < bs >> 2; i++) 2993 *((unsigned int *)pad + i) ^= IPAD_DATA; 2994 2995 if (chcr_compute_partial_hash(shash, pad, actx->h_iopad, 2996 max_authsize)) 2997 goto out; 2998 /* Compute the opad-digest */ 2999 memset(pad + keys.authkeylen, 0, bs - keys.authkeylen); 3000 memcpy(pad, o_ptr, keys.authkeylen); 3001 for (i = 0; i < bs >> 2; i++) 3002 *((unsigned int *)pad + i) ^= OPAD_DATA; 3003 3004 if (chcr_compute_partial_hash(shash, pad, o_ptr, max_authsize)) 3005 goto out; 3006 3007 /* convert the ipad and opad digest to network order */ 3008 chcr_change_order(actx->h_iopad, param.result_size); 3009 chcr_change_order(o_ptr, param.result_size); 3010 key_ctx_len = sizeof(struct _key_ctx) + 3011 ((DIV_ROUND_UP(keys.enckeylen, 16)) << 4) + 3012 (param.result_size + align) * 2; 3013 aeadctx->key_ctx_hdr = FILL_KEY_CTX_HDR(ck_size, param.mk_size, 3014 0, 1, key_ctx_len >> 4); 3015 actx->auth_mode = param.auth_mode; 3016 chcr_free_shash(base_hash); 3017 3018 return 0; 3019 } 3020 out: 3021 aeadctx->enckey_len = 0; 3022 if (!IS_ERR(base_hash)) 3023 chcr_free_shash(base_hash); 3024 return -EINVAL; 3025 } 3026 3027 static int chcr_aead_digest_null_setkey(struct crypto_aead *authenc, 3028 const u8 *key, unsigned int keylen) 3029 { 3030 struct chcr_context *ctx = crypto_aead_ctx(authenc); 3031 struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx); 3032 struct chcr_authenc_ctx *actx = AUTHENC_CTX(aeadctx); 3033 struct crypto_authenc_keys keys; 3034 int err; 3035 /* it contains auth and cipher key both*/ 3036 int key_ctx_len = 0; 3037 unsigned char ck_size = 0; 3038 3039 crypto_aead_clear_flags(aeadctx->sw_cipher, CRYPTO_TFM_REQ_MASK); 3040 crypto_aead_set_flags(aeadctx->sw_cipher, crypto_aead_get_flags(authenc) 3041 & CRYPTO_TFM_REQ_MASK); 3042 err = crypto_aead_setkey(aeadctx->sw_cipher, key, keylen); 3043 crypto_aead_clear_flags(authenc, CRYPTO_TFM_RES_MASK); 3044 crypto_aead_set_flags(authenc, crypto_aead_get_flags(aeadctx->sw_cipher) 3045 & CRYPTO_TFM_RES_MASK); 3046 if (err) 3047 goto out; 3048 3049 if (crypto_authenc_extractkeys(&keys, key, keylen) != 0) { 3050 crypto_aead_set_flags(authenc, CRYPTO_TFM_RES_BAD_KEY_LEN); 3051 goto out; 3052 } 3053 if (keys.enckeylen == AES_KEYSIZE_128) { 3054 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_128; 3055 } else if (keys.enckeylen == AES_KEYSIZE_192) { 3056 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_192; 3057 } else if (keys.enckeylen == AES_KEYSIZE_256) { 3058 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_256; 3059 } else { 3060 pr_err("chcr : Unsupported cipher key\n"); 3061 goto out; 3062 } 3063 memcpy(aeadctx->key, keys.enckey, keys.enckeylen); 3064 aeadctx->enckey_len = keys.enckeylen; 3065 get_aes_decrypt_key(actx->dec_rrkey, aeadctx->key, 3066 aeadctx->enckey_len << 3); 3067 key_ctx_len = sizeof(struct _key_ctx) 3068 + ((DIV_ROUND_UP(keys.enckeylen, 16)) << 4); 3069 3070 aeadctx->key_ctx_hdr = FILL_KEY_CTX_HDR(ck_size, CHCR_KEYCTX_NO_KEY, 0, 3071 0, key_ctx_len >> 4); 3072 actx->auth_mode = CHCR_SCMD_AUTH_MODE_NOP; 3073 return 0; 3074 out: 3075 aeadctx->enckey_len = 0; 3076 return -EINVAL; 3077 } 3078 static int chcr_aead_encrypt(struct aead_request *req) 3079 { 3080 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 3081 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req); 3082 3083 reqctx->verify = VERIFY_HW; 3084 3085 switch (get_aead_subtype(tfm)) { 3086 case CRYPTO_ALG_SUB_TYPE_AEAD_AUTHENC: 3087 case CRYPTO_ALG_SUB_TYPE_AEAD_NULL: 3088 return chcr_aead_op(req, CHCR_ENCRYPT_OP, 0, 3089 create_authenc_wr); 3090 case CRYPTO_ALG_SUB_TYPE_AEAD_CCM: 3091 case CRYPTO_ALG_SUB_TYPE_AEAD_RFC4309: 3092 return chcr_aead_op(req, CHCR_ENCRYPT_OP, 0, 3093 create_aead_ccm_wr); 3094 default: 3095 return chcr_aead_op(req, CHCR_ENCRYPT_OP, 0, 3096 create_gcm_wr); 3097 } 3098 } 3099 3100 static int chcr_aead_decrypt(struct aead_request *req) 3101 { 3102 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 3103 struct chcr_aead_ctx *aeadctx = AEAD_CTX(crypto_aead_ctx(tfm)); 3104 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req); 3105 int size; 3106 3107 if (aeadctx->mayverify == VERIFY_SW) { 3108 size = crypto_aead_maxauthsize(tfm); 3109 reqctx->verify = VERIFY_SW; 3110 } else { 3111 size = 0; 3112 reqctx->verify = VERIFY_HW; 3113 } 3114 3115 switch (get_aead_subtype(tfm)) { 3116 case CRYPTO_ALG_SUB_TYPE_AEAD_AUTHENC: 3117 case CRYPTO_ALG_SUB_TYPE_AEAD_NULL: 3118 return chcr_aead_op(req, CHCR_DECRYPT_OP, size, 3119 create_authenc_wr); 3120 case CRYPTO_ALG_SUB_TYPE_AEAD_CCM: 3121 case CRYPTO_ALG_SUB_TYPE_AEAD_RFC4309: 3122 return chcr_aead_op(req, CHCR_DECRYPT_OP, size, 3123 create_aead_ccm_wr); 3124 default: 3125 return chcr_aead_op(req, CHCR_DECRYPT_OP, size, 3126 create_gcm_wr); 3127 } 3128 } 3129 3130 static int chcr_aead_op(struct aead_request *req, 3131 unsigned short op_type, 3132 int size, 3133 create_wr_t create_wr_fn) 3134 { 3135 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 3136 struct chcr_context *ctx = crypto_aead_ctx(tfm); 3137 struct uld_ctx *u_ctx; 3138 struct sk_buff *skb; 3139 3140 if (!ctx->dev) { 3141 pr_err("chcr : %s : No crypto device.\n", __func__); 3142 return -ENXIO; 3143 } 3144 u_ctx = ULD_CTX(ctx); 3145 if (cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0], 3146 ctx->tx_qidx)) { 3147 if (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) 3148 return -EBUSY; 3149 } 3150 3151 /* Form a WR from req */ 3152 skb = create_wr_fn(req, u_ctx->lldi.rxq_ids[ctx->rx_qidx], size, 3153 op_type); 3154 3155 if (IS_ERR(skb) || !skb) 3156 return PTR_ERR(skb); 3157 3158 skb->dev = u_ctx->lldi.ports[0]; 3159 set_wr_txq(skb, CPL_PRIORITY_DATA, ctx->tx_qidx); 3160 chcr_send_wr(skb); 3161 return -EINPROGRESS; 3162 } 3163 static struct chcr_alg_template driver_algs[] = { 3164 /* AES-CBC */ 3165 { 3166 .type = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_SUB_TYPE_CBC, 3167 .is_registered = 0, 3168 .alg.crypto = { 3169 .cra_name = "cbc(aes)", 3170 .cra_driver_name = "cbc-aes-chcr", 3171 .cra_blocksize = AES_BLOCK_SIZE, 3172 .cra_init = chcr_cra_init, 3173 .cra_exit = chcr_cra_exit, 3174 .cra_u.ablkcipher = { 3175 .min_keysize = AES_MIN_KEY_SIZE, 3176 .max_keysize = AES_MAX_KEY_SIZE, 3177 .ivsize = AES_BLOCK_SIZE, 3178 .setkey = chcr_aes_cbc_setkey, 3179 .encrypt = chcr_aes_encrypt, 3180 .decrypt = chcr_aes_decrypt, 3181 } 3182 } 3183 }, 3184 { 3185 .type = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_SUB_TYPE_XTS, 3186 .is_registered = 0, 3187 .alg.crypto = { 3188 .cra_name = "xts(aes)", 3189 .cra_driver_name = "xts-aes-chcr", 3190 .cra_blocksize = AES_BLOCK_SIZE, 3191 .cra_init = chcr_cra_init, 3192 .cra_exit = NULL, 3193 .cra_u .ablkcipher = { 3194 .min_keysize = 2 * AES_MIN_KEY_SIZE, 3195 .max_keysize = 2 * AES_MAX_KEY_SIZE, 3196 .ivsize = AES_BLOCK_SIZE, 3197 .setkey = chcr_aes_xts_setkey, 3198 .encrypt = chcr_aes_encrypt, 3199 .decrypt = chcr_aes_decrypt, 3200 } 3201 } 3202 }, 3203 { 3204 .type = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_SUB_TYPE_CTR, 3205 .is_registered = 0, 3206 .alg.crypto = { 3207 .cra_name = "ctr(aes)", 3208 .cra_driver_name = "ctr-aes-chcr", 3209 .cra_blocksize = 1, 3210 .cra_init = chcr_cra_init, 3211 .cra_exit = chcr_cra_exit, 3212 .cra_u.ablkcipher = { 3213 .min_keysize = AES_MIN_KEY_SIZE, 3214 .max_keysize = AES_MAX_KEY_SIZE, 3215 .ivsize = AES_BLOCK_SIZE, 3216 .setkey = chcr_aes_ctr_setkey, 3217 .encrypt = chcr_aes_encrypt, 3218 .decrypt = chcr_aes_decrypt, 3219 } 3220 } 3221 }, 3222 { 3223 .type = CRYPTO_ALG_TYPE_ABLKCIPHER | 3224 CRYPTO_ALG_SUB_TYPE_CTR_RFC3686, 3225 .is_registered = 0, 3226 .alg.crypto = { 3227 .cra_name = "rfc3686(ctr(aes))", 3228 .cra_driver_name = "rfc3686-ctr-aes-chcr", 3229 .cra_blocksize = 1, 3230 .cra_init = chcr_rfc3686_init, 3231 .cra_exit = chcr_cra_exit, 3232 .cra_u.ablkcipher = { 3233 .min_keysize = AES_MIN_KEY_SIZE + 3234 CTR_RFC3686_NONCE_SIZE, 3235 .max_keysize = AES_MAX_KEY_SIZE + 3236 CTR_RFC3686_NONCE_SIZE, 3237 .ivsize = CTR_RFC3686_IV_SIZE, 3238 .setkey = chcr_aes_rfc3686_setkey, 3239 .encrypt = chcr_aes_encrypt, 3240 .decrypt = chcr_aes_decrypt, 3241 .geniv = "seqiv", 3242 } 3243 } 3244 }, 3245 /* SHA */ 3246 { 3247 .type = CRYPTO_ALG_TYPE_AHASH, 3248 .is_registered = 0, 3249 .alg.hash = { 3250 .halg.digestsize = SHA1_DIGEST_SIZE, 3251 .halg.base = { 3252 .cra_name = "sha1", 3253 .cra_driver_name = "sha1-chcr", 3254 .cra_blocksize = SHA1_BLOCK_SIZE, 3255 } 3256 } 3257 }, 3258 { 3259 .type = CRYPTO_ALG_TYPE_AHASH, 3260 .is_registered = 0, 3261 .alg.hash = { 3262 .halg.digestsize = SHA256_DIGEST_SIZE, 3263 .halg.base = { 3264 .cra_name = "sha256", 3265 .cra_driver_name = "sha256-chcr", 3266 .cra_blocksize = SHA256_BLOCK_SIZE, 3267 } 3268 } 3269 }, 3270 { 3271 .type = CRYPTO_ALG_TYPE_AHASH, 3272 .is_registered = 0, 3273 .alg.hash = { 3274 .halg.digestsize = SHA224_DIGEST_SIZE, 3275 .halg.base = { 3276 .cra_name = "sha224", 3277 .cra_driver_name = "sha224-chcr", 3278 .cra_blocksize = SHA224_BLOCK_SIZE, 3279 } 3280 } 3281 }, 3282 { 3283 .type = CRYPTO_ALG_TYPE_AHASH, 3284 .is_registered = 0, 3285 .alg.hash = { 3286 .halg.digestsize = SHA384_DIGEST_SIZE, 3287 .halg.base = { 3288 .cra_name = "sha384", 3289 .cra_driver_name = "sha384-chcr", 3290 .cra_blocksize = SHA384_BLOCK_SIZE, 3291 } 3292 } 3293 }, 3294 { 3295 .type = CRYPTO_ALG_TYPE_AHASH, 3296 .is_registered = 0, 3297 .alg.hash = { 3298 .halg.digestsize = SHA512_DIGEST_SIZE, 3299 .halg.base = { 3300 .cra_name = "sha512", 3301 .cra_driver_name = "sha512-chcr", 3302 .cra_blocksize = SHA512_BLOCK_SIZE, 3303 } 3304 } 3305 }, 3306 /* HMAC */ 3307 { 3308 .type = CRYPTO_ALG_TYPE_HMAC, 3309 .is_registered = 0, 3310 .alg.hash = { 3311 .halg.digestsize = SHA1_DIGEST_SIZE, 3312 .halg.base = { 3313 .cra_name = "hmac(sha1)", 3314 .cra_driver_name = "hmac-sha1-chcr", 3315 .cra_blocksize = SHA1_BLOCK_SIZE, 3316 } 3317 } 3318 }, 3319 { 3320 .type = CRYPTO_ALG_TYPE_HMAC, 3321 .is_registered = 0, 3322 .alg.hash = { 3323 .halg.digestsize = SHA224_DIGEST_SIZE, 3324 .halg.base = { 3325 .cra_name = "hmac(sha224)", 3326 .cra_driver_name = "hmac-sha224-chcr", 3327 .cra_blocksize = SHA224_BLOCK_SIZE, 3328 } 3329 } 3330 }, 3331 { 3332 .type = CRYPTO_ALG_TYPE_HMAC, 3333 .is_registered = 0, 3334 .alg.hash = { 3335 .halg.digestsize = SHA256_DIGEST_SIZE, 3336 .halg.base = { 3337 .cra_name = "hmac(sha256)", 3338 .cra_driver_name = "hmac-sha256-chcr", 3339 .cra_blocksize = SHA256_BLOCK_SIZE, 3340 } 3341 } 3342 }, 3343 { 3344 .type = CRYPTO_ALG_TYPE_HMAC, 3345 .is_registered = 0, 3346 .alg.hash = { 3347 .halg.digestsize = SHA384_DIGEST_SIZE, 3348 .halg.base = { 3349 .cra_name = "hmac(sha384)", 3350 .cra_driver_name = "hmac-sha384-chcr", 3351 .cra_blocksize = SHA384_BLOCK_SIZE, 3352 } 3353 } 3354 }, 3355 { 3356 .type = CRYPTO_ALG_TYPE_HMAC, 3357 .is_registered = 0, 3358 .alg.hash = { 3359 .halg.digestsize = SHA512_DIGEST_SIZE, 3360 .halg.base = { 3361 .cra_name = "hmac(sha512)", 3362 .cra_driver_name = "hmac-sha512-chcr", 3363 .cra_blocksize = SHA512_BLOCK_SIZE, 3364 } 3365 } 3366 }, 3367 /* Add AEAD Algorithms */ 3368 { 3369 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_AEAD_GCM, 3370 .is_registered = 0, 3371 .alg.aead = { 3372 .base = { 3373 .cra_name = "gcm(aes)", 3374 .cra_driver_name = "gcm-aes-chcr", 3375 .cra_blocksize = 1, 3376 .cra_priority = CHCR_AEAD_PRIORITY, 3377 .cra_ctxsize = sizeof(struct chcr_context) + 3378 sizeof(struct chcr_aead_ctx) + 3379 sizeof(struct chcr_gcm_ctx), 3380 }, 3381 .ivsize = 12, 3382 .maxauthsize = GHASH_DIGEST_SIZE, 3383 .setkey = chcr_gcm_setkey, 3384 .setauthsize = chcr_gcm_setauthsize, 3385 } 3386 }, 3387 { 3388 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_AEAD_RFC4106, 3389 .is_registered = 0, 3390 .alg.aead = { 3391 .base = { 3392 .cra_name = "rfc4106(gcm(aes))", 3393 .cra_driver_name = "rfc4106-gcm-aes-chcr", 3394 .cra_blocksize = 1, 3395 .cra_priority = CHCR_AEAD_PRIORITY + 1, 3396 .cra_ctxsize = sizeof(struct chcr_context) + 3397 sizeof(struct chcr_aead_ctx) + 3398 sizeof(struct chcr_gcm_ctx), 3399 3400 }, 3401 .ivsize = 8, 3402 .maxauthsize = GHASH_DIGEST_SIZE, 3403 .setkey = chcr_gcm_setkey, 3404 .setauthsize = chcr_4106_4309_setauthsize, 3405 } 3406 }, 3407 { 3408 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_AEAD_CCM, 3409 .is_registered = 0, 3410 .alg.aead = { 3411 .base = { 3412 .cra_name = "ccm(aes)", 3413 .cra_driver_name = "ccm-aes-chcr", 3414 .cra_blocksize = 1, 3415 .cra_priority = CHCR_AEAD_PRIORITY, 3416 .cra_ctxsize = sizeof(struct chcr_context) + 3417 sizeof(struct chcr_aead_ctx), 3418 3419 }, 3420 .ivsize = AES_BLOCK_SIZE, 3421 .maxauthsize = GHASH_DIGEST_SIZE, 3422 .setkey = chcr_aead_ccm_setkey, 3423 .setauthsize = chcr_ccm_setauthsize, 3424 } 3425 }, 3426 { 3427 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_AEAD_RFC4309, 3428 .is_registered = 0, 3429 .alg.aead = { 3430 .base = { 3431 .cra_name = "rfc4309(ccm(aes))", 3432 .cra_driver_name = "rfc4309-ccm-aes-chcr", 3433 .cra_blocksize = 1, 3434 .cra_priority = CHCR_AEAD_PRIORITY + 1, 3435 .cra_ctxsize = sizeof(struct chcr_context) + 3436 sizeof(struct chcr_aead_ctx), 3437 3438 }, 3439 .ivsize = 8, 3440 .maxauthsize = GHASH_DIGEST_SIZE, 3441 .setkey = chcr_aead_rfc4309_setkey, 3442 .setauthsize = chcr_4106_4309_setauthsize, 3443 } 3444 }, 3445 { 3446 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_AEAD_AUTHENC, 3447 .is_registered = 0, 3448 .alg.aead = { 3449 .base = { 3450 .cra_name = "authenc(hmac(sha1),cbc(aes))", 3451 .cra_driver_name = 3452 "authenc-hmac-sha1-cbc-aes-chcr", 3453 .cra_blocksize = AES_BLOCK_SIZE, 3454 .cra_priority = CHCR_AEAD_PRIORITY, 3455 .cra_ctxsize = sizeof(struct chcr_context) + 3456 sizeof(struct chcr_aead_ctx) + 3457 sizeof(struct chcr_authenc_ctx), 3458 3459 }, 3460 .ivsize = AES_BLOCK_SIZE, 3461 .maxauthsize = SHA1_DIGEST_SIZE, 3462 .setkey = chcr_authenc_setkey, 3463 .setauthsize = chcr_authenc_setauthsize, 3464 } 3465 }, 3466 { 3467 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_AEAD_AUTHENC, 3468 .is_registered = 0, 3469 .alg.aead = { 3470 .base = { 3471 3472 .cra_name = "authenc(hmac(sha256),cbc(aes))", 3473 .cra_driver_name = 3474 "authenc-hmac-sha256-cbc-aes-chcr", 3475 .cra_blocksize = AES_BLOCK_SIZE, 3476 .cra_priority = CHCR_AEAD_PRIORITY, 3477 .cra_ctxsize = sizeof(struct chcr_context) + 3478 sizeof(struct chcr_aead_ctx) + 3479 sizeof(struct chcr_authenc_ctx), 3480 3481 }, 3482 .ivsize = AES_BLOCK_SIZE, 3483 .maxauthsize = SHA256_DIGEST_SIZE, 3484 .setkey = chcr_authenc_setkey, 3485 .setauthsize = chcr_authenc_setauthsize, 3486 } 3487 }, 3488 { 3489 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_AEAD_AUTHENC, 3490 .is_registered = 0, 3491 .alg.aead = { 3492 .base = { 3493 .cra_name = "authenc(hmac(sha224),cbc(aes))", 3494 .cra_driver_name = 3495 "authenc-hmac-sha224-cbc-aes-chcr", 3496 .cra_blocksize = AES_BLOCK_SIZE, 3497 .cra_priority = CHCR_AEAD_PRIORITY, 3498 .cra_ctxsize = sizeof(struct chcr_context) + 3499 sizeof(struct chcr_aead_ctx) + 3500 sizeof(struct chcr_authenc_ctx), 3501 }, 3502 .ivsize = AES_BLOCK_SIZE, 3503 .maxauthsize = SHA224_DIGEST_SIZE, 3504 .setkey = chcr_authenc_setkey, 3505 .setauthsize = chcr_authenc_setauthsize, 3506 } 3507 }, 3508 { 3509 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_AEAD_AUTHENC, 3510 .is_registered = 0, 3511 .alg.aead = { 3512 .base = { 3513 .cra_name = "authenc(hmac(sha384),cbc(aes))", 3514 .cra_driver_name = 3515 "authenc-hmac-sha384-cbc-aes-chcr", 3516 .cra_blocksize = AES_BLOCK_SIZE, 3517 .cra_priority = CHCR_AEAD_PRIORITY, 3518 .cra_ctxsize = sizeof(struct chcr_context) + 3519 sizeof(struct chcr_aead_ctx) + 3520 sizeof(struct chcr_authenc_ctx), 3521 3522 }, 3523 .ivsize = AES_BLOCK_SIZE, 3524 .maxauthsize = SHA384_DIGEST_SIZE, 3525 .setkey = chcr_authenc_setkey, 3526 .setauthsize = chcr_authenc_setauthsize, 3527 } 3528 }, 3529 { 3530 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_AEAD_AUTHENC, 3531 .is_registered = 0, 3532 .alg.aead = { 3533 .base = { 3534 .cra_name = "authenc(hmac(sha512),cbc(aes))", 3535 .cra_driver_name = 3536 "authenc-hmac-sha512-cbc-aes-chcr", 3537 .cra_blocksize = AES_BLOCK_SIZE, 3538 .cra_priority = CHCR_AEAD_PRIORITY, 3539 .cra_ctxsize = sizeof(struct chcr_context) + 3540 sizeof(struct chcr_aead_ctx) + 3541 sizeof(struct chcr_authenc_ctx), 3542 3543 }, 3544 .ivsize = AES_BLOCK_SIZE, 3545 .maxauthsize = SHA512_DIGEST_SIZE, 3546 .setkey = chcr_authenc_setkey, 3547 .setauthsize = chcr_authenc_setauthsize, 3548 } 3549 }, 3550 { 3551 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_AEAD_NULL, 3552 .is_registered = 0, 3553 .alg.aead = { 3554 .base = { 3555 .cra_name = "authenc(digest_null,cbc(aes))", 3556 .cra_driver_name = 3557 "authenc-digest_null-cbc-aes-chcr", 3558 .cra_blocksize = AES_BLOCK_SIZE, 3559 .cra_priority = CHCR_AEAD_PRIORITY, 3560 .cra_ctxsize = sizeof(struct chcr_context) + 3561 sizeof(struct chcr_aead_ctx) + 3562 sizeof(struct chcr_authenc_ctx), 3563 3564 }, 3565 .ivsize = AES_BLOCK_SIZE, 3566 .maxauthsize = 0, 3567 .setkey = chcr_aead_digest_null_setkey, 3568 .setauthsize = chcr_authenc_null_setauthsize, 3569 } 3570 }, 3571 }; 3572 3573 /* 3574 * chcr_unregister_alg - Deregister crypto algorithms with 3575 * kernel framework. 3576 */ 3577 static int chcr_unregister_alg(void) 3578 { 3579 int i; 3580 3581 for (i = 0; i < ARRAY_SIZE(driver_algs); i++) { 3582 switch (driver_algs[i].type & CRYPTO_ALG_TYPE_MASK) { 3583 case CRYPTO_ALG_TYPE_ABLKCIPHER: 3584 if (driver_algs[i].is_registered) 3585 crypto_unregister_alg( 3586 &driver_algs[i].alg.crypto); 3587 break; 3588 case CRYPTO_ALG_TYPE_AEAD: 3589 if (driver_algs[i].is_registered) 3590 crypto_unregister_aead( 3591 &driver_algs[i].alg.aead); 3592 break; 3593 case CRYPTO_ALG_TYPE_AHASH: 3594 if (driver_algs[i].is_registered) 3595 crypto_unregister_ahash( 3596 &driver_algs[i].alg.hash); 3597 break; 3598 } 3599 driver_algs[i].is_registered = 0; 3600 } 3601 return 0; 3602 } 3603 3604 #define SZ_AHASH_CTX sizeof(struct chcr_context) 3605 #define SZ_AHASH_H_CTX (sizeof(struct chcr_context) + sizeof(struct hmac_ctx)) 3606 #define SZ_AHASH_REQ_CTX sizeof(struct chcr_ahash_req_ctx) 3607 #define AHASH_CRA_FLAGS (CRYPTO_ALG_TYPE_AHASH | CRYPTO_ALG_ASYNC) 3608 3609 /* 3610 * chcr_register_alg - Register crypto algorithms with kernel framework. 3611 */ 3612 static int chcr_register_alg(void) 3613 { 3614 struct crypto_alg ai; 3615 struct ahash_alg *a_hash; 3616 int err = 0, i; 3617 char *name = NULL; 3618 3619 for (i = 0; i < ARRAY_SIZE(driver_algs); i++) { 3620 if (driver_algs[i].is_registered) 3621 continue; 3622 switch (driver_algs[i].type & CRYPTO_ALG_TYPE_MASK) { 3623 case CRYPTO_ALG_TYPE_ABLKCIPHER: 3624 driver_algs[i].alg.crypto.cra_priority = 3625 CHCR_CRA_PRIORITY; 3626 driver_algs[i].alg.crypto.cra_module = THIS_MODULE; 3627 driver_algs[i].alg.crypto.cra_flags = 3628 CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC | 3629 CRYPTO_ALG_NEED_FALLBACK; 3630 driver_algs[i].alg.crypto.cra_ctxsize = 3631 sizeof(struct chcr_context) + 3632 sizeof(struct ablk_ctx); 3633 driver_algs[i].alg.crypto.cra_alignmask = 0; 3634 driver_algs[i].alg.crypto.cra_type = 3635 &crypto_ablkcipher_type; 3636 err = crypto_register_alg(&driver_algs[i].alg.crypto); 3637 name = driver_algs[i].alg.crypto.cra_driver_name; 3638 break; 3639 case CRYPTO_ALG_TYPE_AEAD: 3640 driver_algs[i].alg.aead.base.cra_flags = 3641 CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC | 3642 CRYPTO_ALG_NEED_FALLBACK; 3643 driver_algs[i].alg.aead.encrypt = chcr_aead_encrypt; 3644 driver_algs[i].alg.aead.decrypt = chcr_aead_decrypt; 3645 driver_algs[i].alg.aead.init = chcr_aead_cra_init; 3646 driver_algs[i].alg.aead.exit = chcr_aead_cra_exit; 3647 driver_algs[i].alg.aead.base.cra_module = THIS_MODULE; 3648 err = crypto_register_aead(&driver_algs[i].alg.aead); 3649 name = driver_algs[i].alg.aead.base.cra_driver_name; 3650 break; 3651 case CRYPTO_ALG_TYPE_AHASH: 3652 a_hash = &driver_algs[i].alg.hash; 3653 a_hash->update = chcr_ahash_update; 3654 a_hash->final = chcr_ahash_final; 3655 a_hash->finup = chcr_ahash_finup; 3656 a_hash->digest = chcr_ahash_digest; 3657 a_hash->export = chcr_ahash_export; 3658 a_hash->import = chcr_ahash_import; 3659 a_hash->halg.statesize = SZ_AHASH_REQ_CTX; 3660 a_hash->halg.base.cra_priority = CHCR_CRA_PRIORITY; 3661 a_hash->halg.base.cra_module = THIS_MODULE; 3662 a_hash->halg.base.cra_flags = AHASH_CRA_FLAGS; 3663 a_hash->halg.base.cra_alignmask = 0; 3664 a_hash->halg.base.cra_exit = NULL; 3665 a_hash->halg.base.cra_type = &crypto_ahash_type; 3666 3667 if (driver_algs[i].type == CRYPTO_ALG_TYPE_HMAC) { 3668 a_hash->halg.base.cra_init = chcr_hmac_cra_init; 3669 a_hash->halg.base.cra_exit = chcr_hmac_cra_exit; 3670 a_hash->init = chcr_hmac_init; 3671 a_hash->setkey = chcr_ahash_setkey; 3672 a_hash->halg.base.cra_ctxsize = SZ_AHASH_H_CTX; 3673 } else { 3674 a_hash->init = chcr_sha_init; 3675 a_hash->halg.base.cra_ctxsize = SZ_AHASH_CTX; 3676 a_hash->halg.base.cra_init = chcr_sha_cra_init; 3677 } 3678 err = crypto_register_ahash(&driver_algs[i].alg.hash); 3679 ai = driver_algs[i].alg.hash.halg.base; 3680 name = ai.cra_driver_name; 3681 break; 3682 } 3683 if (err) { 3684 pr_err("chcr : %s : Algorithm registration failed\n", 3685 name); 3686 goto register_err; 3687 } else { 3688 driver_algs[i].is_registered = 1; 3689 } 3690 } 3691 return 0; 3692 3693 register_err: 3694 chcr_unregister_alg(); 3695 return err; 3696 } 3697 3698 /* 3699 * start_crypto - Register the crypto algorithms. 3700 * This should called once when the first device comesup. After this 3701 * kernel will start calling driver APIs for crypto operations. 3702 */ 3703 int start_crypto(void) 3704 { 3705 return chcr_register_alg(); 3706 } 3707 3708 /* 3709 * stop_crypto - Deregister all the crypto algorithms with kernel. 3710 * This should be called once when the last device goes down. After this 3711 * kernel will not call the driver API for crypto operations. 3712 */ 3713 int stop_crypto(void) 3714 { 3715 chcr_unregister_alg(); 3716 return 0; 3717 } 3718