1 /* 2 * This file is part of the Chelsio T6 Crypto driver for Linux. 3 * 4 * Copyright (c) 2003-2016 Chelsio Communications, Inc. All rights reserved. 5 * 6 * This software is available to you under a choice of one of two 7 * licenses. You may choose to be licensed under the terms of the GNU 8 * General Public License (GPL) Version 2, available from the file 9 * COPYING in the main directory of this source tree, or the 10 * OpenIB.org BSD license below: 11 * 12 * Redistribution and use in source and binary forms, with or 13 * without modification, are permitted provided that the following 14 * conditions are met: 15 * 16 * - Redistributions of source code must retain the above 17 * copyright notice, this list of conditions and the following 18 * disclaimer. 19 * 20 * - Redistributions in binary form must reproduce the above 21 * copyright notice, this list of conditions and the following 22 * disclaimer in the documentation and/or other materials 23 * provided with the distribution. 24 * 25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 26 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 27 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 28 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 29 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 30 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 31 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 32 * SOFTWARE. 33 * 34 * Written and Maintained by: 35 * Manoj Malviya (manojmalviya@chelsio.com) 36 * Atul Gupta (atul.gupta@chelsio.com) 37 * Jitendra Lulla (jlulla@chelsio.com) 38 * Yeshaswi M R Gowda (yeshaswi@chelsio.com) 39 * Harsh Jain (harsh@chelsio.com) 40 */ 41 42 #define pr_fmt(fmt) "chcr:" fmt 43 44 #include <linux/kernel.h> 45 #include <linux/module.h> 46 #include <linux/crypto.h> 47 #include <linux/cryptohash.h> 48 #include <linux/skbuff.h> 49 #include <linux/rtnetlink.h> 50 #include <linux/highmem.h> 51 #include <linux/scatterlist.h> 52 53 #include <crypto/aes.h> 54 #include <crypto/algapi.h> 55 #include <crypto/hash.h> 56 #include <crypto/sha.h> 57 #include <crypto/authenc.h> 58 #include <crypto/internal/aead.h> 59 #include <crypto/null.h> 60 #include <crypto/internal/skcipher.h> 61 #include <crypto/aead.h> 62 #include <crypto/scatterwalk.h> 63 #include <crypto/internal/hash.h> 64 65 #include "t4fw_api.h" 66 #include "t4_msg.h" 67 #include "chcr_core.h" 68 #include "chcr_algo.h" 69 #include "chcr_crypto.h" 70 71 static inline struct chcr_aead_ctx *AEAD_CTX(struct chcr_context *ctx) 72 { 73 return ctx->crypto_ctx->aeadctx; 74 } 75 76 static inline struct ablk_ctx *ABLK_CTX(struct chcr_context *ctx) 77 { 78 return ctx->crypto_ctx->ablkctx; 79 } 80 81 static inline struct hmac_ctx *HMAC_CTX(struct chcr_context *ctx) 82 { 83 return ctx->crypto_ctx->hmacctx; 84 } 85 86 static inline struct chcr_gcm_ctx *GCM_CTX(struct chcr_aead_ctx *gctx) 87 { 88 return gctx->ctx->gcm; 89 } 90 91 static inline struct chcr_authenc_ctx *AUTHENC_CTX(struct chcr_aead_ctx *gctx) 92 { 93 return gctx->ctx->authenc; 94 } 95 96 static inline struct uld_ctx *ULD_CTX(struct chcr_context *ctx) 97 { 98 return ctx->dev->u_ctx; 99 } 100 101 static inline int is_ofld_imm(const struct sk_buff *skb) 102 { 103 return (skb->len <= CRYPTO_MAX_IMM_TX_PKT_LEN); 104 } 105 106 /* 107 * sgl_len - calculates the size of an SGL of the given capacity 108 * @n: the number of SGL entries 109 * Calculates the number of flits needed for a scatter/gather list that 110 * can hold the given number of entries. 111 */ 112 static inline unsigned int sgl_len(unsigned int n) 113 { 114 n--; 115 return (3 * n) / 2 + (n & 1) + 2; 116 } 117 118 static void chcr_verify_tag(struct aead_request *req, u8 *input, int *err) 119 { 120 u8 temp[SHA512_DIGEST_SIZE]; 121 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 122 int authsize = crypto_aead_authsize(tfm); 123 struct cpl_fw6_pld *fw6_pld; 124 int cmp = 0; 125 126 fw6_pld = (struct cpl_fw6_pld *)input; 127 if ((get_aead_subtype(tfm) == CRYPTO_ALG_SUB_TYPE_AEAD_RFC4106) || 128 (get_aead_subtype(tfm) == CRYPTO_ALG_SUB_TYPE_AEAD_GCM)) { 129 cmp = memcmp(&fw6_pld->data[2], (fw6_pld + 1), authsize); 130 } else { 131 132 sg_pcopy_to_buffer(req->src, sg_nents(req->src), temp, 133 authsize, req->assoclen + 134 req->cryptlen - authsize); 135 cmp = memcmp(temp, (fw6_pld + 1), authsize); 136 } 137 if (cmp) 138 *err = -EBADMSG; 139 else 140 *err = 0; 141 } 142 143 /* 144 * chcr_handle_resp - Unmap the DMA buffers associated with the request 145 * @req: crypto request 146 */ 147 int chcr_handle_resp(struct crypto_async_request *req, unsigned char *input, 148 int err) 149 { 150 struct crypto_tfm *tfm = req->tfm; 151 struct chcr_context *ctx = crypto_tfm_ctx(tfm); 152 struct uld_ctx *u_ctx = ULD_CTX(ctx); 153 struct chcr_req_ctx ctx_req; 154 struct cpl_fw6_pld *fw6_pld; 155 unsigned int digestsize, updated_digestsize; 156 157 switch (tfm->__crt_alg->cra_flags & CRYPTO_ALG_TYPE_MASK) { 158 case CRYPTO_ALG_TYPE_AEAD: 159 ctx_req.req.aead_req = (struct aead_request *)req; 160 ctx_req.ctx.reqctx = aead_request_ctx(ctx_req.req.aead_req); 161 dma_unmap_sg(&u_ctx->lldi.pdev->dev, ctx_req.ctx.reqctx->dst, 162 ctx_req.ctx.reqctx->dst_nents, DMA_FROM_DEVICE); 163 if (ctx_req.ctx.reqctx->skb) { 164 kfree_skb(ctx_req.ctx.reqctx->skb); 165 ctx_req.ctx.reqctx->skb = NULL; 166 } 167 if (ctx_req.ctx.reqctx->verify == VERIFY_SW) { 168 chcr_verify_tag(ctx_req.req.aead_req, input, 169 &err); 170 ctx_req.ctx.reqctx->verify = VERIFY_HW; 171 } 172 break; 173 174 case CRYPTO_ALG_TYPE_BLKCIPHER: 175 ctx_req.req.ablk_req = (struct ablkcipher_request *)req; 176 ctx_req.ctx.ablk_ctx = 177 ablkcipher_request_ctx(ctx_req.req.ablk_req); 178 if (!err) { 179 fw6_pld = (struct cpl_fw6_pld *)input; 180 memcpy(ctx_req.req.ablk_req->info, &fw6_pld->data[2], 181 AES_BLOCK_SIZE); 182 } 183 dma_unmap_sg(&u_ctx->lldi.pdev->dev, ctx_req.req.ablk_req->dst, 184 ctx_req.ctx.ablk_ctx->dst_nents, DMA_FROM_DEVICE); 185 if (ctx_req.ctx.ablk_ctx->skb) { 186 kfree_skb(ctx_req.ctx.ablk_ctx->skb); 187 ctx_req.ctx.ablk_ctx->skb = NULL; 188 } 189 break; 190 191 case CRYPTO_ALG_TYPE_AHASH: 192 ctx_req.req.ahash_req = (struct ahash_request *)req; 193 ctx_req.ctx.ahash_ctx = 194 ahash_request_ctx(ctx_req.req.ahash_req); 195 digestsize = 196 crypto_ahash_digestsize(crypto_ahash_reqtfm( 197 ctx_req.req.ahash_req)); 198 updated_digestsize = digestsize; 199 if (digestsize == SHA224_DIGEST_SIZE) 200 updated_digestsize = SHA256_DIGEST_SIZE; 201 else if (digestsize == SHA384_DIGEST_SIZE) 202 updated_digestsize = SHA512_DIGEST_SIZE; 203 if (ctx_req.ctx.ahash_ctx->skb) { 204 kfree_skb(ctx_req.ctx.ahash_ctx->skb); 205 ctx_req.ctx.ahash_ctx->skb = NULL; 206 } 207 if (ctx_req.ctx.ahash_ctx->result == 1) { 208 ctx_req.ctx.ahash_ctx->result = 0; 209 memcpy(ctx_req.req.ahash_req->result, input + 210 sizeof(struct cpl_fw6_pld), 211 digestsize); 212 } else { 213 memcpy(ctx_req.ctx.ahash_ctx->partial_hash, input + 214 sizeof(struct cpl_fw6_pld), 215 updated_digestsize); 216 } 217 break; 218 } 219 return err; 220 } 221 222 /* 223 * calc_tx_flits_ofld - calculate # of flits for an offload packet 224 * @skb: the packet 225 * Returns the number of flits needed for the given offload packet. 226 * These packets are already fully constructed and no additional headers 227 * will be added. 228 */ 229 static inline unsigned int calc_tx_flits_ofld(const struct sk_buff *skb) 230 { 231 unsigned int flits, cnt; 232 233 if (is_ofld_imm(skb)) 234 return DIV_ROUND_UP(skb->len, 8); 235 236 flits = skb_transport_offset(skb) / 8; /* headers */ 237 cnt = skb_shinfo(skb)->nr_frags; 238 if (skb_tail_pointer(skb) != skb_transport_header(skb)) 239 cnt++; 240 return flits + sgl_len(cnt); 241 } 242 243 static inline void get_aes_decrypt_key(unsigned char *dec_key, 244 const unsigned char *key, 245 unsigned int keylength) 246 { 247 u32 temp; 248 u32 w_ring[MAX_NK]; 249 int i, j, k; 250 u8 nr, nk; 251 252 switch (keylength) { 253 case AES_KEYLENGTH_128BIT: 254 nk = KEYLENGTH_4BYTES; 255 nr = NUMBER_OF_ROUNDS_10; 256 break; 257 case AES_KEYLENGTH_192BIT: 258 nk = KEYLENGTH_6BYTES; 259 nr = NUMBER_OF_ROUNDS_12; 260 break; 261 case AES_KEYLENGTH_256BIT: 262 nk = KEYLENGTH_8BYTES; 263 nr = NUMBER_OF_ROUNDS_14; 264 break; 265 default: 266 return; 267 } 268 for (i = 0; i < nk; i++) 269 w_ring[i] = be32_to_cpu(*(u32 *)&key[4 * i]); 270 271 i = 0; 272 temp = w_ring[nk - 1]; 273 while (i + nk < (nr + 1) * 4) { 274 if (!(i % nk)) { 275 /* RotWord(temp) */ 276 temp = (temp << 8) | (temp >> 24); 277 temp = aes_ks_subword(temp); 278 temp ^= round_constant[i / nk]; 279 } else if (nk == 8 && (i % 4 == 0)) { 280 temp = aes_ks_subword(temp); 281 } 282 w_ring[i % nk] ^= temp; 283 temp = w_ring[i % nk]; 284 i++; 285 } 286 i--; 287 for (k = 0, j = i % nk; k < nk; k++) { 288 *((u32 *)dec_key + k) = htonl(w_ring[j]); 289 j--; 290 if (j < 0) 291 j += nk; 292 } 293 } 294 295 static struct crypto_shash *chcr_alloc_shash(unsigned int ds) 296 { 297 struct crypto_shash *base_hash = NULL; 298 299 switch (ds) { 300 case SHA1_DIGEST_SIZE: 301 base_hash = crypto_alloc_shash("sha1", 0, 0); 302 break; 303 case SHA224_DIGEST_SIZE: 304 base_hash = crypto_alloc_shash("sha224", 0, 0); 305 break; 306 case SHA256_DIGEST_SIZE: 307 base_hash = crypto_alloc_shash("sha256", 0, 0); 308 break; 309 case SHA384_DIGEST_SIZE: 310 base_hash = crypto_alloc_shash("sha384", 0, 0); 311 break; 312 case SHA512_DIGEST_SIZE: 313 base_hash = crypto_alloc_shash("sha512", 0, 0); 314 break; 315 } 316 317 return base_hash; 318 } 319 320 static int chcr_compute_partial_hash(struct shash_desc *desc, 321 char *iopad, char *result_hash, 322 int digest_size) 323 { 324 struct sha1_state sha1_st; 325 struct sha256_state sha256_st; 326 struct sha512_state sha512_st; 327 int error; 328 329 if (digest_size == SHA1_DIGEST_SIZE) { 330 error = crypto_shash_init(desc) ?: 331 crypto_shash_update(desc, iopad, SHA1_BLOCK_SIZE) ?: 332 crypto_shash_export(desc, (void *)&sha1_st); 333 memcpy(result_hash, sha1_st.state, SHA1_DIGEST_SIZE); 334 } else if (digest_size == SHA224_DIGEST_SIZE) { 335 error = crypto_shash_init(desc) ?: 336 crypto_shash_update(desc, iopad, SHA256_BLOCK_SIZE) ?: 337 crypto_shash_export(desc, (void *)&sha256_st); 338 memcpy(result_hash, sha256_st.state, SHA256_DIGEST_SIZE); 339 340 } else if (digest_size == SHA256_DIGEST_SIZE) { 341 error = crypto_shash_init(desc) ?: 342 crypto_shash_update(desc, iopad, SHA256_BLOCK_SIZE) ?: 343 crypto_shash_export(desc, (void *)&sha256_st); 344 memcpy(result_hash, sha256_st.state, SHA256_DIGEST_SIZE); 345 346 } else if (digest_size == SHA384_DIGEST_SIZE) { 347 error = crypto_shash_init(desc) ?: 348 crypto_shash_update(desc, iopad, SHA512_BLOCK_SIZE) ?: 349 crypto_shash_export(desc, (void *)&sha512_st); 350 memcpy(result_hash, sha512_st.state, SHA512_DIGEST_SIZE); 351 352 } else if (digest_size == SHA512_DIGEST_SIZE) { 353 error = crypto_shash_init(desc) ?: 354 crypto_shash_update(desc, iopad, SHA512_BLOCK_SIZE) ?: 355 crypto_shash_export(desc, (void *)&sha512_st); 356 memcpy(result_hash, sha512_st.state, SHA512_DIGEST_SIZE); 357 } else { 358 error = -EINVAL; 359 pr_err("Unknown digest size %d\n", digest_size); 360 } 361 return error; 362 } 363 364 static void chcr_change_order(char *buf, int ds) 365 { 366 int i; 367 368 if (ds == SHA512_DIGEST_SIZE) { 369 for (i = 0; i < (ds / sizeof(u64)); i++) 370 *((__be64 *)buf + i) = 371 cpu_to_be64(*((u64 *)buf + i)); 372 } else { 373 for (i = 0; i < (ds / sizeof(u32)); i++) 374 *((__be32 *)buf + i) = 375 cpu_to_be32(*((u32 *)buf + i)); 376 } 377 } 378 379 static inline int is_hmac(struct crypto_tfm *tfm) 380 { 381 struct crypto_alg *alg = tfm->__crt_alg; 382 struct chcr_alg_template *chcr_crypto_alg = 383 container_of(__crypto_ahash_alg(alg), struct chcr_alg_template, 384 alg.hash); 385 if (chcr_crypto_alg->type == CRYPTO_ALG_TYPE_HMAC) 386 return 1; 387 return 0; 388 } 389 390 static void write_phys_cpl(struct cpl_rx_phys_dsgl *phys_cpl, 391 struct scatterlist *sg, 392 struct phys_sge_parm *sg_param) 393 { 394 struct phys_sge_pairs *to; 395 int out_buf_size = sg_param->obsize; 396 unsigned int nents = sg_param->nents, i, j = 0; 397 398 phys_cpl->op_to_tid = htonl(CPL_RX_PHYS_DSGL_OPCODE_V(CPL_RX_PHYS_DSGL) 399 | CPL_RX_PHYS_DSGL_ISRDMA_V(0)); 400 phys_cpl->pcirlxorder_to_noofsgentr = 401 htonl(CPL_RX_PHYS_DSGL_PCIRLXORDER_V(0) | 402 CPL_RX_PHYS_DSGL_PCINOSNOOP_V(0) | 403 CPL_RX_PHYS_DSGL_PCITPHNTENB_V(0) | 404 CPL_RX_PHYS_DSGL_PCITPHNT_V(0) | 405 CPL_RX_PHYS_DSGL_DCAID_V(0) | 406 CPL_RX_PHYS_DSGL_NOOFSGENTR_V(nents)); 407 phys_cpl->rss_hdr_int.opcode = CPL_RX_PHYS_ADDR; 408 phys_cpl->rss_hdr_int.qid = htons(sg_param->qid); 409 phys_cpl->rss_hdr_int.hash_val = 0; 410 to = (struct phys_sge_pairs *)((unsigned char *)phys_cpl + 411 sizeof(struct cpl_rx_phys_dsgl)); 412 413 for (i = 0; nents; to++) { 414 for (j = 0; j < 8 && nents; j++, nents--) { 415 out_buf_size -= sg_dma_len(sg); 416 to->len[j] = htons(sg_dma_len(sg)); 417 to->addr[j] = cpu_to_be64(sg_dma_address(sg)); 418 sg = sg_next(sg); 419 } 420 } 421 if (out_buf_size) { 422 j--; 423 to--; 424 to->len[j] = htons(ntohs(to->len[j]) + (out_buf_size)); 425 } 426 } 427 428 static inline int map_writesg_phys_cpl(struct device *dev, 429 struct cpl_rx_phys_dsgl *phys_cpl, 430 struct scatterlist *sg, 431 struct phys_sge_parm *sg_param) 432 { 433 if (!sg || !sg_param->nents) 434 return 0; 435 436 sg_param->nents = dma_map_sg(dev, sg, sg_param->nents, DMA_FROM_DEVICE); 437 if (sg_param->nents == 0) { 438 pr_err("CHCR : DMA mapping failed\n"); 439 return -EINVAL; 440 } 441 write_phys_cpl(phys_cpl, sg, sg_param); 442 return 0; 443 } 444 445 static inline int get_aead_subtype(struct crypto_aead *aead) 446 { 447 struct aead_alg *alg = crypto_aead_alg(aead); 448 struct chcr_alg_template *chcr_crypto_alg = 449 container_of(alg, struct chcr_alg_template, alg.aead); 450 return chcr_crypto_alg->type & CRYPTO_ALG_SUB_TYPE_MASK; 451 } 452 453 static inline int get_cryptoalg_subtype(struct crypto_tfm *tfm) 454 { 455 struct crypto_alg *alg = tfm->__crt_alg; 456 struct chcr_alg_template *chcr_crypto_alg = 457 container_of(alg, struct chcr_alg_template, alg.crypto); 458 459 return chcr_crypto_alg->type & CRYPTO_ALG_SUB_TYPE_MASK; 460 } 461 462 static inline void write_buffer_to_skb(struct sk_buff *skb, 463 unsigned int *frags, 464 char *bfr, 465 u8 bfr_len) 466 { 467 skb->len += bfr_len; 468 skb->data_len += bfr_len; 469 skb->truesize += bfr_len; 470 get_page(virt_to_page(bfr)); 471 skb_fill_page_desc(skb, *frags, virt_to_page(bfr), 472 offset_in_page(bfr), bfr_len); 473 (*frags)++; 474 } 475 476 477 static inline void 478 write_sg_to_skb(struct sk_buff *skb, unsigned int *frags, 479 struct scatterlist *sg, unsigned int count) 480 { 481 struct page *spage; 482 unsigned int page_len; 483 484 skb->len += count; 485 skb->data_len += count; 486 skb->truesize += count; 487 488 while (count > 0) { 489 if (!sg || (!(sg->length))) 490 break; 491 spage = sg_page(sg); 492 get_page(spage); 493 page_len = min(sg->length, count); 494 skb_fill_page_desc(skb, *frags, spage, sg->offset, page_len); 495 (*frags)++; 496 count -= page_len; 497 sg = sg_next(sg); 498 } 499 } 500 501 static int generate_copy_rrkey(struct ablk_ctx *ablkctx, 502 struct _key_ctx *key_ctx) 503 { 504 if (ablkctx->ciph_mode == CHCR_SCMD_CIPHER_MODE_AES_CBC) { 505 memcpy(key_ctx->key, ablkctx->rrkey, ablkctx->enckey_len); 506 } else { 507 memcpy(key_ctx->key, 508 ablkctx->key + (ablkctx->enckey_len >> 1), 509 ablkctx->enckey_len >> 1); 510 memcpy(key_ctx->key + (ablkctx->enckey_len >> 1), 511 ablkctx->rrkey, ablkctx->enckey_len >> 1); 512 } 513 return 0; 514 } 515 516 static inline void create_wreq(struct chcr_context *ctx, 517 struct chcr_wr *chcr_req, 518 void *req, struct sk_buff *skb, 519 int kctx_len, int hash_sz, 520 int is_iv, 521 unsigned int sc_len) 522 { 523 struct uld_ctx *u_ctx = ULD_CTX(ctx); 524 int iv_loc = IV_DSGL; 525 int qid = u_ctx->lldi.rxq_ids[ctx->tx_channel_id]; 526 unsigned int immdatalen = 0, nr_frags = 0; 527 528 if (is_ofld_imm(skb)) { 529 immdatalen = skb->data_len; 530 iv_loc = IV_IMMEDIATE; 531 } else { 532 nr_frags = skb_shinfo(skb)->nr_frags; 533 } 534 535 chcr_req->wreq.op_to_cctx_size = FILL_WR_OP_CCTX_SIZE(immdatalen, 536 ((sizeof(chcr_req->key_ctx) + kctx_len) >> 4)); 537 chcr_req->wreq.pld_size_hash_size = 538 htonl(FW_CRYPTO_LOOKASIDE_WR_PLD_SIZE_V(sgl_lengths[nr_frags]) | 539 FW_CRYPTO_LOOKASIDE_WR_HASH_SIZE_V(hash_sz)); 540 chcr_req->wreq.len16_pkd = 541 htonl(FW_CRYPTO_LOOKASIDE_WR_LEN16_V(DIV_ROUND_UP( 542 (calc_tx_flits_ofld(skb) * 8), 16))); 543 chcr_req->wreq.cookie = cpu_to_be64((uintptr_t)req); 544 chcr_req->wreq.rx_chid_to_rx_q_id = 545 FILL_WR_RX_Q_ID(ctx->dev->tx_channel_id, qid, 546 is_iv ? iv_loc : IV_NOP); 547 548 chcr_req->ulptx.cmd_dest = FILL_ULPTX_CMD_DEST(ctx->dev->tx_channel_id); 549 chcr_req->ulptx.len = htonl((DIV_ROUND_UP((calc_tx_flits_ofld(skb) * 8), 550 16) - ((sizeof(chcr_req->wreq)) >> 4))); 551 552 chcr_req->sc_imm.cmd_more = FILL_CMD_MORE(immdatalen); 553 chcr_req->sc_imm.len = cpu_to_be32(sizeof(struct cpl_tx_sec_pdu) + 554 sizeof(chcr_req->key_ctx) + 555 kctx_len + sc_len + immdatalen); 556 } 557 558 /** 559 * create_cipher_wr - form the WR for cipher operations 560 * @req: cipher req. 561 * @ctx: crypto driver context of the request. 562 * @qid: ingress qid where response of this WR should be received. 563 * @op_type: encryption or decryption 564 */ 565 static struct sk_buff 566 *create_cipher_wr(struct ablkcipher_request *req, 567 unsigned short qid, 568 unsigned short op_type) 569 { 570 struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req); 571 struct chcr_context *ctx = crypto_ablkcipher_ctx(tfm); 572 struct uld_ctx *u_ctx = ULD_CTX(ctx); 573 struct ablk_ctx *ablkctx = ABLK_CTX(ctx); 574 struct sk_buff *skb = NULL; 575 struct chcr_wr *chcr_req; 576 struct cpl_rx_phys_dsgl *phys_cpl; 577 struct chcr_blkcipher_req_ctx *reqctx = ablkcipher_request_ctx(req); 578 struct phys_sge_parm sg_param; 579 unsigned int frags = 0, transhdr_len, phys_dsgl; 580 unsigned int ivsize = crypto_ablkcipher_ivsize(tfm), kctx_len; 581 gfp_t flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL : 582 GFP_ATOMIC; 583 584 if (!req->info) 585 return ERR_PTR(-EINVAL); 586 reqctx->dst_nents = sg_nents_for_len(req->dst, req->nbytes); 587 if (reqctx->dst_nents <= 0) { 588 pr_err("AES:Invalid Destination sg lists\n"); 589 return ERR_PTR(-EINVAL); 590 } 591 if ((ablkctx->enckey_len == 0) || (ivsize > AES_BLOCK_SIZE) || 592 (req->nbytes <= 0) || (req->nbytes % AES_BLOCK_SIZE)) { 593 pr_err("AES: Invalid value of Key Len %d nbytes %d IV Len %d\n", 594 ablkctx->enckey_len, req->nbytes, ivsize); 595 return ERR_PTR(-EINVAL); 596 } 597 598 phys_dsgl = get_space_for_phys_dsgl(reqctx->dst_nents); 599 600 kctx_len = (DIV_ROUND_UP(ablkctx->enckey_len, 16) * 16); 601 transhdr_len = CIPHER_TRANSHDR_SIZE(kctx_len, phys_dsgl); 602 skb = alloc_skb((transhdr_len + sizeof(struct sge_opaque_hdr)), flags); 603 if (!skb) 604 return ERR_PTR(-ENOMEM); 605 skb_reserve(skb, sizeof(struct sge_opaque_hdr)); 606 chcr_req = (struct chcr_wr *)__skb_put(skb, transhdr_len); 607 memset(chcr_req, 0, transhdr_len); 608 chcr_req->sec_cpl.op_ivinsrtofst = 609 FILL_SEC_CPL_OP_IVINSR(ctx->dev->tx_channel_id, 2, 1); 610 611 chcr_req->sec_cpl.pldlen = htonl(ivsize + req->nbytes); 612 chcr_req->sec_cpl.aadstart_cipherstop_hi = 613 FILL_SEC_CPL_CIPHERSTOP_HI(0, 0, ivsize + 1, 0); 614 615 chcr_req->sec_cpl.cipherstop_lo_authinsert = 616 FILL_SEC_CPL_AUTHINSERT(0, 0, 0, 0); 617 chcr_req->sec_cpl.seqno_numivs = FILL_SEC_CPL_SCMD0_SEQNO(op_type, 0, 618 ablkctx->ciph_mode, 619 0, 0, ivsize >> 1); 620 chcr_req->sec_cpl.ivgen_hdrlen = FILL_SEC_CPL_IVGEN_HDRLEN(0, 0, 0, 621 0, 1, phys_dsgl); 622 623 chcr_req->key_ctx.ctx_hdr = ablkctx->key_ctx_hdr; 624 if (op_type == CHCR_DECRYPT_OP) { 625 generate_copy_rrkey(ablkctx, &chcr_req->key_ctx); 626 } else { 627 if (ablkctx->ciph_mode == CHCR_SCMD_CIPHER_MODE_AES_CBC) { 628 memcpy(chcr_req->key_ctx.key, ablkctx->key, 629 ablkctx->enckey_len); 630 } else { 631 memcpy(chcr_req->key_ctx.key, ablkctx->key + 632 (ablkctx->enckey_len >> 1), 633 ablkctx->enckey_len >> 1); 634 memcpy(chcr_req->key_ctx.key + 635 (ablkctx->enckey_len >> 1), 636 ablkctx->key, 637 ablkctx->enckey_len >> 1); 638 } 639 } 640 phys_cpl = (struct cpl_rx_phys_dsgl *)((u8 *)(chcr_req + 1) + kctx_len); 641 sg_param.nents = reqctx->dst_nents; 642 sg_param.obsize = req->nbytes; 643 sg_param.qid = qid; 644 sg_param.align = 1; 645 if (map_writesg_phys_cpl(&u_ctx->lldi.pdev->dev, phys_cpl, req->dst, 646 &sg_param)) 647 goto map_fail1; 648 649 skb_set_transport_header(skb, transhdr_len); 650 memcpy(reqctx->iv, req->info, ivsize); 651 write_buffer_to_skb(skb, &frags, reqctx->iv, ivsize); 652 write_sg_to_skb(skb, &frags, req->src, req->nbytes); 653 create_wreq(ctx, chcr_req, req, skb, kctx_len, 0, 1, 654 sizeof(struct cpl_rx_phys_dsgl) + phys_dsgl); 655 reqctx->skb = skb; 656 skb_get(skb); 657 return skb; 658 map_fail1: 659 kfree_skb(skb); 660 return ERR_PTR(-ENOMEM); 661 } 662 663 static int chcr_aes_cbc_setkey(struct crypto_ablkcipher *tfm, const u8 *key, 664 unsigned int keylen) 665 { 666 struct chcr_context *ctx = crypto_ablkcipher_ctx(tfm); 667 struct ablk_ctx *ablkctx = ABLK_CTX(ctx); 668 unsigned int ck_size, context_size; 669 u16 alignment = 0; 670 671 if (keylen == AES_KEYSIZE_128) { 672 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_128; 673 } else if (keylen == AES_KEYSIZE_192) { 674 alignment = 8; 675 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_192; 676 } else if (keylen == AES_KEYSIZE_256) { 677 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_256; 678 } else { 679 goto badkey_err; 680 } 681 memcpy(ablkctx->key, key, keylen); 682 ablkctx->enckey_len = keylen; 683 get_aes_decrypt_key(ablkctx->rrkey, ablkctx->key, keylen << 3); 684 context_size = (KEY_CONTEXT_HDR_SALT_AND_PAD + 685 keylen + alignment) >> 4; 686 687 ablkctx->key_ctx_hdr = FILL_KEY_CTX_HDR(ck_size, CHCR_KEYCTX_NO_KEY, 688 0, 0, context_size); 689 ablkctx->ciph_mode = CHCR_SCMD_CIPHER_MODE_AES_CBC; 690 return 0; 691 badkey_err: 692 crypto_ablkcipher_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN); 693 ablkctx->enckey_len = 0; 694 return -EINVAL; 695 } 696 697 static int cxgb4_is_crypto_q_full(struct net_device *dev, unsigned int idx) 698 { 699 struct adapter *adap = netdev2adap(dev); 700 struct sge_uld_txq_info *txq_info = 701 adap->sge.uld_txq_info[CXGB4_TX_CRYPTO]; 702 struct sge_uld_txq *txq; 703 int ret = 0; 704 705 local_bh_disable(); 706 txq = &txq_info->uldtxq[idx]; 707 spin_lock(&txq->sendq.lock); 708 if (txq->full) 709 ret = -1; 710 spin_unlock(&txq->sendq.lock); 711 local_bh_enable(); 712 return ret; 713 } 714 715 static int chcr_aes_encrypt(struct ablkcipher_request *req) 716 { 717 struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req); 718 struct chcr_context *ctx = crypto_ablkcipher_ctx(tfm); 719 struct uld_ctx *u_ctx = ULD_CTX(ctx); 720 struct sk_buff *skb; 721 722 if (unlikely(cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0], 723 ctx->tx_channel_id))) { 724 if (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) 725 return -EBUSY; 726 } 727 728 skb = create_cipher_wr(req, u_ctx->lldi.rxq_ids[ctx->tx_channel_id], 729 CHCR_ENCRYPT_OP); 730 if (IS_ERR(skb)) { 731 pr_err("chcr : %s : Failed to form WR. No memory\n", __func__); 732 return PTR_ERR(skb); 733 } 734 skb->dev = u_ctx->lldi.ports[0]; 735 set_wr_txq(skb, CPL_PRIORITY_DATA, ctx->tx_channel_id); 736 chcr_send_wr(skb); 737 return -EINPROGRESS; 738 } 739 740 static int chcr_aes_decrypt(struct ablkcipher_request *req) 741 { 742 struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req); 743 struct chcr_context *ctx = crypto_ablkcipher_ctx(tfm); 744 struct uld_ctx *u_ctx = ULD_CTX(ctx); 745 struct sk_buff *skb; 746 747 if (unlikely(cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0], 748 ctx->tx_channel_id))) { 749 if (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) 750 return -EBUSY; 751 } 752 753 skb = create_cipher_wr(req, u_ctx->lldi.rxq_ids[0], 754 CHCR_DECRYPT_OP); 755 if (IS_ERR(skb)) { 756 pr_err("chcr : %s : Failed to form WR. No memory\n", __func__); 757 return PTR_ERR(skb); 758 } 759 skb->dev = u_ctx->lldi.ports[0]; 760 set_wr_txq(skb, CPL_PRIORITY_DATA, ctx->tx_channel_id); 761 chcr_send_wr(skb); 762 return -EINPROGRESS; 763 } 764 765 static int chcr_device_init(struct chcr_context *ctx) 766 { 767 struct uld_ctx *u_ctx; 768 unsigned int id; 769 int err = 0, rxq_perchan, rxq_idx; 770 771 id = smp_processor_id(); 772 if (!ctx->dev) { 773 err = assign_chcr_device(&ctx->dev); 774 if (err) { 775 pr_err("chcr device assignment fails\n"); 776 goto out; 777 } 778 u_ctx = ULD_CTX(ctx); 779 rxq_perchan = u_ctx->lldi.nrxq / u_ctx->lldi.nchan; 780 rxq_idx = ctx->dev->tx_channel_id * rxq_perchan; 781 rxq_idx += id % rxq_perchan; 782 spin_lock(&ctx->dev->lock_chcr_dev); 783 ctx->tx_channel_id = rxq_idx; 784 ctx->dev->tx_channel_id = !ctx->dev->tx_channel_id; 785 spin_unlock(&ctx->dev->lock_chcr_dev); 786 } 787 out: 788 return err; 789 } 790 791 static int chcr_cra_init(struct crypto_tfm *tfm) 792 { 793 tfm->crt_ablkcipher.reqsize = sizeof(struct chcr_blkcipher_req_ctx); 794 return chcr_device_init(crypto_tfm_ctx(tfm)); 795 } 796 797 static int get_alg_config(struct algo_param *params, 798 unsigned int auth_size) 799 { 800 switch (auth_size) { 801 case SHA1_DIGEST_SIZE: 802 params->mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_160; 803 params->auth_mode = CHCR_SCMD_AUTH_MODE_SHA1; 804 params->result_size = SHA1_DIGEST_SIZE; 805 break; 806 case SHA224_DIGEST_SIZE: 807 params->mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_256; 808 params->auth_mode = CHCR_SCMD_AUTH_MODE_SHA224; 809 params->result_size = SHA256_DIGEST_SIZE; 810 break; 811 case SHA256_DIGEST_SIZE: 812 params->mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_256; 813 params->auth_mode = CHCR_SCMD_AUTH_MODE_SHA256; 814 params->result_size = SHA256_DIGEST_SIZE; 815 break; 816 case SHA384_DIGEST_SIZE: 817 params->mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_512; 818 params->auth_mode = CHCR_SCMD_AUTH_MODE_SHA512_384; 819 params->result_size = SHA512_DIGEST_SIZE; 820 break; 821 case SHA512_DIGEST_SIZE: 822 params->mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_512; 823 params->auth_mode = CHCR_SCMD_AUTH_MODE_SHA512_512; 824 params->result_size = SHA512_DIGEST_SIZE; 825 break; 826 default: 827 pr_err("chcr : ERROR, unsupported digest size\n"); 828 return -EINVAL; 829 } 830 return 0; 831 } 832 833 static inline void chcr_free_shash(struct crypto_shash *base_hash) 834 { 835 crypto_free_shash(base_hash); 836 } 837 838 /** 839 * create_hash_wr - Create hash work request 840 * @req - Cipher req base 841 */ 842 static struct sk_buff *create_hash_wr(struct ahash_request *req, 843 struct hash_wr_param *param) 844 { 845 struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(req); 846 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 847 struct chcr_context *ctx = crypto_tfm_ctx(crypto_ahash_tfm(tfm)); 848 struct hmac_ctx *hmacctx = HMAC_CTX(ctx); 849 struct sk_buff *skb = NULL; 850 struct chcr_wr *chcr_req; 851 unsigned int frags = 0, transhdr_len, iopad_alignment = 0; 852 unsigned int digestsize = crypto_ahash_digestsize(tfm); 853 unsigned int kctx_len = 0; 854 u8 hash_size_in_response = 0; 855 gfp_t flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL : 856 GFP_ATOMIC; 857 858 iopad_alignment = KEYCTX_ALIGN_PAD(digestsize); 859 kctx_len = param->alg_prm.result_size + iopad_alignment; 860 if (param->opad_needed) 861 kctx_len += param->alg_prm.result_size + iopad_alignment; 862 863 if (req_ctx->result) 864 hash_size_in_response = digestsize; 865 else 866 hash_size_in_response = param->alg_prm.result_size; 867 transhdr_len = HASH_TRANSHDR_SIZE(kctx_len); 868 skb = alloc_skb((transhdr_len + sizeof(struct sge_opaque_hdr)), flags); 869 if (!skb) 870 return skb; 871 872 skb_reserve(skb, sizeof(struct sge_opaque_hdr)); 873 chcr_req = (struct chcr_wr *)__skb_put(skb, transhdr_len); 874 memset(chcr_req, 0, transhdr_len); 875 876 chcr_req->sec_cpl.op_ivinsrtofst = 877 FILL_SEC_CPL_OP_IVINSR(ctx->dev->tx_channel_id, 2, 0); 878 chcr_req->sec_cpl.pldlen = htonl(param->bfr_len + param->sg_len); 879 880 chcr_req->sec_cpl.aadstart_cipherstop_hi = 881 FILL_SEC_CPL_CIPHERSTOP_HI(0, 0, 0, 0); 882 chcr_req->sec_cpl.cipherstop_lo_authinsert = 883 FILL_SEC_CPL_AUTHINSERT(0, 1, 0, 0); 884 chcr_req->sec_cpl.seqno_numivs = 885 FILL_SEC_CPL_SCMD0_SEQNO(0, 0, 0, param->alg_prm.auth_mode, 886 param->opad_needed, 0); 887 888 chcr_req->sec_cpl.ivgen_hdrlen = 889 FILL_SEC_CPL_IVGEN_HDRLEN(param->last, param->more, 0, 1, 0, 0); 890 891 memcpy(chcr_req->key_ctx.key, req_ctx->partial_hash, 892 param->alg_prm.result_size); 893 894 if (param->opad_needed) 895 memcpy(chcr_req->key_ctx.key + 896 ((param->alg_prm.result_size <= 32) ? 32 : 897 CHCR_HASH_MAX_DIGEST_SIZE), 898 hmacctx->opad, param->alg_prm.result_size); 899 900 chcr_req->key_ctx.ctx_hdr = FILL_KEY_CTX_HDR(CHCR_KEYCTX_NO_KEY, 901 param->alg_prm.mk_size, 0, 902 param->opad_needed, 903 ((kctx_len + 904 sizeof(chcr_req->key_ctx)) >> 4)); 905 chcr_req->sec_cpl.scmd1 = cpu_to_be64((u64)param->scmd1); 906 907 skb_set_transport_header(skb, transhdr_len); 908 if (param->bfr_len != 0) 909 write_buffer_to_skb(skb, &frags, req_ctx->reqbfr, 910 param->bfr_len); 911 if (param->sg_len != 0) 912 write_sg_to_skb(skb, &frags, req->src, param->sg_len); 913 914 create_wreq(ctx, chcr_req, req, skb, kctx_len, hash_size_in_response, 0, 915 DUMMY_BYTES); 916 req_ctx->skb = skb; 917 skb_get(skb); 918 return skb; 919 } 920 921 static int chcr_ahash_update(struct ahash_request *req) 922 { 923 struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(req); 924 struct crypto_ahash *rtfm = crypto_ahash_reqtfm(req); 925 struct chcr_context *ctx = crypto_tfm_ctx(crypto_ahash_tfm(rtfm)); 926 struct uld_ctx *u_ctx = NULL; 927 struct sk_buff *skb; 928 u8 remainder = 0, bs; 929 unsigned int nbytes = req->nbytes; 930 struct hash_wr_param params; 931 932 bs = crypto_tfm_alg_blocksize(crypto_ahash_tfm(rtfm)); 933 934 u_ctx = ULD_CTX(ctx); 935 if (unlikely(cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0], 936 ctx->tx_channel_id))) { 937 if (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) 938 return -EBUSY; 939 } 940 941 if (nbytes + req_ctx->reqlen >= bs) { 942 remainder = (nbytes + req_ctx->reqlen) % bs; 943 nbytes = nbytes + req_ctx->reqlen - remainder; 944 } else { 945 sg_pcopy_to_buffer(req->src, sg_nents(req->src), req_ctx->reqbfr 946 + req_ctx->reqlen, nbytes, 0); 947 req_ctx->reqlen += nbytes; 948 return 0; 949 } 950 951 params.opad_needed = 0; 952 params.more = 1; 953 params.last = 0; 954 params.sg_len = nbytes - req_ctx->reqlen; 955 params.bfr_len = req_ctx->reqlen; 956 params.scmd1 = 0; 957 get_alg_config(¶ms.alg_prm, crypto_ahash_digestsize(rtfm)); 958 req_ctx->result = 0; 959 req_ctx->data_len += params.sg_len + params.bfr_len; 960 skb = create_hash_wr(req, ¶ms); 961 if (!skb) 962 return -ENOMEM; 963 964 if (remainder) { 965 u8 *temp; 966 /* Swap buffers */ 967 temp = req_ctx->reqbfr; 968 req_ctx->reqbfr = req_ctx->skbfr; 969 req_ctx->skbfr = temp; 970 sg_pcopy_to_buffer(req->src, sg_nents(req->src), 971 req_ctx->reqbfr, remainder, req->nbytes - 972 remainder); 973 } 974 req_ctx->reqlen = remainder; 975 skb->dev = u_ctx->lldi.ports[0]; 976 set_wr_txq(skb, CPL_PRIORITY_DATA, ctx->tx_channel_id); 977 chcr_send_wr(skb); 978 979 return -EINPROGRESS; 980 } 981 982 static void create_last_hash_block(char *bfr_ptr, unsigned int bs, u64 scmd1) 983 { 984 memset(bfr_ptr, 0, bs); 985 *bfr_ptr = 0x80; 986 if (bs == 64) 987 *(__be64 *)(bfr_ptr + 56) = cpu_to_be64(scmd1 << 3); 988 else 989 *(__be64 *)(bfr_ptr + 120) = cpu_to_be64(scmd1 << 3); 990 } 991 992 static int chcr_ahash_final(struct ahash_request *req) 993 { 994 struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(req); 995 struct crypto_ahash *rtfm = crypto_ahash_reqtfm(req); 996 struct chcr_context *ctx = crypto_tfm_ctx(crypto_ahash_tfm(rtfm)); 997 struct hash_wr_param params; 998 struct sk_buff *skb; 999 struct uld_ctx *u_ctx = NULL; 1000 u8 bs = crypto_tfm_alg_blocksize(crypto_ahash_tfm(rtfm)); 1001 1002 u_ctx = ULD_CTX(ctx); 1003 if (is_hmac(crypto_ahash_tfm(rtfm))) 1004 params.opad_needed = 1; 1005 else 1006 params.opad_needed = 0; 1007 params.sg_len = 0; 1008 get_alg_config(¶ms.alg_prm, crypto_ahash_digestsize(rtfm)); 1009 req_ctx->result = 1; 1010 params.bfr_len = req_ctx->reqlen; 1011 req_ctx->data_len += params.bfr_len + params.sg_len; 1012 if (req_ctx->reqlen == 0) { 1013 create_last_hash_block(req_ctx->reqbfr, bs, req_ctx->data_len); 1014 params.last = 0; 1015 params.more = 1; 1016 params.scmd1 = 0; 1017 params.bfr_len = bs; 1018 1019 } else { 1020 params.scmd1 = req_ctx->data_len; 1021 params.last = 1; 1022 params.more = 0; 1023 } 1024 skb = create_hash_wr(req, ¶ms); 1025 if (!skb) 1026 return -ENOMEM; 1027 1028 skb->dev = u_ctx->lldi.ports[0]; 1029 set_wr_txq(skb, CPL_PRIORITY_DATA, ctx->tx_channel_id); 1030 chcr_send_wr(skb); 1031 return -EINPROGRESS; 1032 } 1033 1034 static int chcr_ahash_finup(struct ahash_request *req) 1035 { 1036 struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(req); 1037 struct crypto_ahash *rtfm = crypto_ahash_reqtfm(req); 1038 struct chcr_context *ctx = crypto_tfm_ctx(crypto_ahash_tfm(rtfm)); 1039 struct uld_ctx *u_ctx = NULL; 1040 struct sk_buff *skb; 1041 struct hash_wr_param params; 1042 u8 bs; 1043 1044 bs = crypto_tfm_alg_blocksize(crypto_ahash_tfm(rtfm)); 1045 u_ctx = ULD_CTX(ctx); 1046 1047 if (unlikely(cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0], 1048 ctx->tx_channel_id))) { 1049 if (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) 1050 return -EBUSY; 1051 } 1052 1053 if (is_hmac(crypto_ahash_tfm(rtfm))) 1054 params.opad_needed = 1; 1055 else 1056 params.opad_needed = 0; 1057 1058 params.sg_len = req->nbytes; 1059 params.bfr_len = req_ctx->reqlen; 1060 get_alg_config(¶ms.alg_prm, crypto_ahash_digestsize(rtfm)); 1061 req_ctx->data_len += params.bfr_len + params.sg_len; 1062 req_ctx->result = 1; 1063 if ((req_ctx->reqlen + req->nbytes) == 0) { 1064 create_last_hash_block(req_ctx->reqbfr, bs, req_ctx->data_len); 1065 params.last = 0; 1066 params.more = 1; 1067 params.scmd1 = 0; 1068 params.bfr_len = bs; 1069 } else { 1070 params.scmd1 = req_ctx->data_len; 1071 params.last = 1; 1072 params.more = 0; 1073 } 1074 1075 skb = create_hash_wr(req, ¶ms); 1076 if (!skb) 1077 return -ENOMEM; 1078 1079 skb->dev = u_ctx->lldi.ports[0]; 1080 set_wr_txq(skb, CPL_PRIORITY_DATA, ctx->tx_channel_id); 1081 chcr_send_wr(skb); 1082 1083 return -EINPROGRESS; 1084 } 1085 1086 static int chcr_ahash_digest(struct ahash_request *req) 1087 { 1088 struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(req); 1089 struct crypto_ahash *rtfm = crypto_ahash_reqtfm(req); 1090 struct chcr_context *ctx = crypto_tfm_ctx(crypto_ahash_tfm(rtfm)); 1091 struct uld_ctx *u_ctx = NULL; 1092 struct sk_buff *skb; 1093 struct hash_wr_param params; 1094 u8 bs; 1095 1096 rtfm->init(req); 1097 bs = crypto_tfm_alg_blocksize(crypto_ahash_tfm(rtfm)); 1098 1099 u_ctx = ULD_CTX(ctx); 1100 if (unlikely(cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0], 1101 ctx->tx_channel_id))) { 1102 if (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) 1103 return -EBUSY; 1104 } 1105 1106 if (is_hmac(crypto_ahash_tfm(rtfm))) 1107 params.opad_needed = 1; 1108 else 1109 params.opad_needed = 0; 1110 1111 params.last = 0; 1112 params.more = 0; 1113 params.sg_len = req->nbytes; 1114 params.bfr_len = 0; 1115 params.scmd1 = 0; 1116 get_alg_config(¶ms.alg_prm, crypto_ahash_digestsize(rtfm)); 1117 req_ctx->result = 1; 1118 req_ctx->data_len += params.bfr_len + params.sg_len; 1119 1120 if (req->nbytes == 0) { 1121 create_last_hash_block(req_ctx->reqbfr, bs, 0); 1122 params.more = 1; 1123 params.bfr_len = bs; 1124 } 1125 1126 skb = create_hash_wr(req, ¶ms); 1127 if (!skb) 1128 return -ENOMEM; 1129 1130 skb->dev = u_ctx->lldi.ports[0]; 1131 set_wr_txq(skb, CPL_PRIORITY_DATA, ctx->tx_channel_id); 1132 chcr_send_wr(skb); 1133 return -EINPROGRESS; 1134 } 1135 1136 static int chcr_ahash_export(struct ahash_request *areq, void *out) 1137 { 1138 struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(areq); 1139 struct chcr_ahash_req_ctx *state = out; 1140 1141 state->reqlen = req_ctx->reqlen; 1142 state->data_len = req_ctx->data_len; 1143 memcpy(state->bfr1, req_ctx->reqbfr, req_ctx->reqlen); 1144 memcpy(state->partial_hash, req_ctx->partial_hash, 1145 CHCR_HASH_MAX_DIGEST_SIZE); 1146 return 0; 1147 } 1148 1149 static int chcr_ahash_import(struct ahash_request *areq, const void *in) 1150 { 1151 struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(areq); 1152 struct chcr_ahash_req_ctx *state = (struct chcr_ahash_req_ctx *)in; 1153 1154 req_ctx->reqlen = state->reqlen; 1155 req_ctx->data_len = state->data_len; 1156 req_ctx->reqbfr = req_ctx->bfr1; 1157 req_ctx->skbfr = req_ctx->bfr2; 1158 memcpy(req_ctx->bfr1, state->bfr1, CHCR_HASH_MAX_BLOCK_SIZE_128); 1159 memcpy(req_ctx->partial_hash, state->partial_hash, 1160 CHCR_HASH_MAX_DIGEST_SIZE); 1161 return 0; 1162 } 1163 1164 static int chcr_ahash_setkey(struct crypto_ahash *tfm, const u8 *key, 1165 unsigned int keylen) 1166 { 1167 struct chcr_context *ctx = crypto_tfm_ctx(crypto_ahash_tfm(tfm)); 1168 struct hmac_ctx *hmacctx = HMAC_CTX(ctx); 1169 unsigned int digestsize = crypto_ahash_digestsize(tfm); 1170 unsigned int bs = crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm)); 1171 unsigned int i, err = 0, updated_digestsize; 1172 1173 SHASH_DESC_ON_STACK(shash, hmacctx->base_hash); 1174 1175 /* use the key to calculate the ipad and opad. ipad will sent with the 1176 * first request's data. opad will be sent with the final hash result 1177 * ipad in hmacctx->ipad and opad in hmacctx->opad location 1178 */ 1179 shash->tfm = hmacctx->base_hash; 1180 shash->flags = crypto_shash_get_flags(hmacctx->base_hash); 1181 if (keylen > bs) { 1182 err = crypto_shash_digest(shash, key, keylen, 1183 hmacctx->ipad); 1184 if (err) 1185 goto out; 1186 keylen = digestsize; 1187 } else { 1188 memcpy(hmacctx->ipad, key, keylen); 1189 } 1190 memset(hmacctx->ipad + keylen, 0, bs - keylen); 1191 memcpy(hmacctx->opad, hmacctx->ipad, bs); 1192 1193 for (i = 0; i < bs / sizeof(int); i++) { 1194 *((unsigned int *)(&hmacctx->ipad) + i) ^= IPAD_DATA; 1195 *((unsigned int *)(&hmacctx->opad) + i) ^= OPAD_DATA; 1196 } 1197 1198 updated_digestsize = digestsize; 1199 if (digestsize == SHA224_DIGEST_SIZE) 1200 updated_digestsize = SHA256_DIGEST_SIZE; 1201 else if (digestsize == SHA384_DIGEST_SIZE) 1202 updated_digestsize = SHA512_DIGEST_SIZE; 1203 err = chcr_compute_partial_hash(shash, hmacctx->ipad, 1204 hmacctx->ipad, digestsize); 1205 if (err) 1206 goto out; 1207 chcr_change_order(hmacctx->ipad, updated_digestsize); 1208 1209 err = chcr_compute_partial_hash(shash, hmacctx->opad, 1210 hmacctx->opad, digestsize); 1211 if (err) 1212 goto out; 1213 chcr_change_order(hmacctx->opad, updated_digestsize); 1214 out: 1215 return err; 1216 } 1217 1218 static int chcr_aes_xts_setkey(struct crypto_ablkcipher *tfm, const u8 *key, 1219 unsigned int key_len) 1220 { 1221 struct chcr_context *ctx = crypto_ablkcipher_ctx(tfm); 1222 struct ablk_ctx *ablkctx = ABLK_CTX(ctx); 1223 unsigned short context_size = 0; 1224 1225 if ((key_len != (AES_KEYSIZE_128 << 1)) && 1226 (key_len != (AES_KEYSIZE_256 << 1))) { 1227 crypto_tfm_set_flags((struct crypto_tfm *)tfm, 1228 CRYPTO_TFM_RES_BAD_KEY_LEN); 1229 ablkctx->enckey_len = 0; 1230 return -EINVAL; 1231 1232 } 1233 1234 memcpy(ablkctx->key, key, key_len); 1235 ablkctx->enckey_len = key_len; 1236 get_aes_decrypt_key(ablkctx->rrkey, ablkctx->key, key_len << 2); 1237 context_size = (KEY_CONTEXT_HDR_SALT_AND_PAD + key_len) >> 4; 1238 ablkctx->key_ctx_hdr = 1239 FILL_KEY_CTX_HDR((key_len == AES_KEYSIZE_256) ? 1240 CHCR_KEYCTX_CIPHER_KEY_SIZE_128 : 1241 CHCR_KEYCTX_CIPHER_KEY_SIZE_256, 1242 CHCR_KEYCTX_NO_KEY, 1, 1243 0, context_size); 1244 ablkctx->ciph_mode = CHCR_SCMD_CIPHER_MODE_AES_XTS; 1245 return 0; 1246 } 1247 1248 static int chcr_sha_init(struct ahash_request *areq) 1249 { 1250 struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(areq); 1251 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); 1252 int digestsize = crypto_ahash_digestsize(tfm); 1253 1254 req_ctx->data_len = 0; 1255 req_ctx->reqlen = 0; 1256 req_ctx->reqbfr = req_ctx->bfr1; 1257 req_ctx->skbfr = req_ctx->bfr2; 1258 req_ctx->skb = NULL; 1259 req_ctx->result = 0; 1260 copy_hash_init_values(req_ctx->partial_hash, digestsize); 1261 return 0; 1262 } 1263 1264 static int chcr_sha_cra_init(struct crypto_tfm *tfm) 1265 { 1266 crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), 1267 sizeof(struct chcr_ahash_req_ctx)); 1268 return chcr_device_init(crypto_tfm_ctx(tfm)); 1269 } 1270 1271 static int chcr_hmac_init(struct ahash_request *areq) 1272 { 1273 struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(areq); 1274 struct crypto_ahash *rtfm = crypto_ahash_reqtfm(areq); 1275 struct chcr_context *ctx = crypto_tfm_ctx(crypto_ahash_tfm(rtfm)); 1276 struct hmac_ctx *hmacctx = HMAC_CTX(ctx); 1277 unsigned int digestsize = crypto_ahash_digestsize(rtfm); 1278 unsigned int bs = crypto_tfm_alg_blocksize(crypto_ahash_tfm(rtfm)); 1279 1280 chcr_sha_init(areq); 1281 req_ctx->data_len = bs; 1282 if (is_hmac(crypto_ahash_tfm(rtfm))) { 1283 if (digestsize == SHA224_DIGEST_SIZE) 1284 memcpy(req_ctx->partial_hash, hmacctx->ipad, 1285 SHA256_DIGEST_SIZE); 1286 else if (digestsize == SHA384_DIGEST_SIZE) 1287 memcpy(req_ctx->partial_hash, hmacctx->ipad, 1288 SHA512_DIGEST_SIZE); 1289 else 1290 memcpy(req_ctx->partial_hash, hmacctx->ipad, 1291 digestsize); 1292 } 1293 return 0; 1294 } 1295 1296 static int chcr_hmac_cra_init(struct crypto_tfm *tfm) 1297 { 1298 struct chcr_context *ctx = crypto_tfm_ctx(tfm); 1299 struct hmac_ctx *hmacctx = HMAC_CTX(ctx); 1300 unsigned int digestsize = 1301 crypto_ahash_digestsize(__crypto_ahash_cast(tfm)); 1302 1303 crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), 1304 sizeof(struct chcr_ahash_req_ctx)); 1305 hmacctx->base_hash = chcr_alloc_shash(digestsize); 1306 if (IS_ERR(hmacctx->base_hash)) 1307 return PTR_ERR(hmacctx->base_hash); 1308 return chcr_device_init(crypto_tfm_ctx(tfm)); 1309 } 1310 1311 static void chcr_hmac_cra_exit(struct crypto_tfm *tfm) 1312 { 1313 struct chcr_context *ctx = crypto_tfm_ctx(tfm); 1314 struct hmac_ctx *hmacctx = HMAC_CTX(ctx); 1315 1316 if (hmacctx->base_hash) { 1317 chcr_free_shash(hmacctx->base_hash); 1318 hmacctx->base_hash = NULL; 1319 } 1320 } 1321 1322 static int chcr_copy_assoc(struct aead_request *req, 1323 struct chcr_aead_ctx *ctx) 1324 { 1325 SKCIPHER_REQUEST_ON_STACK(skreq, ctx->null); 1326 1327 skcipher_request_set_tfm(skreq, ctx->null); 1328 skcipher_request_set_callback(skreq, aead_request_flags(req), 1329 NULL, NULL); 1330 skcipher_request_set_crypt(skreq, req->src, req->dst, req->assoclen, 1331 NULL); 1332 1333 return crypto_skcipher_encrypt(skreq); 1334 } 1335 1336 static unsigned char get_hmac(unsigned int authsize) 1337 { 1338 switch (authsize) { 1339 case ICV_8: 1340 return CHCR_SCMD_HMAC_CTRL_PL1; 1341 case ICV_10: 1342 return CHCR_SCMD_HMAC_CTRL_TRUNC_RFC4366; 1343 case ICV_12: 1344 return CHCR_SCMD_HMAC_CTRL_IPSEC_96BIT; 1345 } 1346 return CHCR_SCMD_HMAC_CTRL_NO_TRUNC; 1347 } 1348 1349 1350 static struct sk_buff *create_authenc_wr(struct aead_request *req, 1351 unsigned short qid, 1352 int size, 1353 unsigned short op_type) 1354 { 1355 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 1356 struct chcr_context *ctx = crypto_aead_ctx(tfm); 1357 struct uld_ctx *u_ctx = ULD_CTX(ctx); 1358 struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx); 1359 struct chcr_authenc_ctx *actx = AUTHENC_CTX(aeadctx); 1360 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req); 1361 struct sk_buff *skb = NULL; 1362 struct chcr_wr *chcr_req; 1363 struct cpl_rx_phys_dsgl *phys_cpl; 1364 struct phys_sge_parm sg_param; 1365 struct scatterlist *src; 1366 unsigned int frags = 0, transhdr_len; 1367 unsigned int ivsize = crypto_aead_ivsize(tfm), dst_size = 0; 1368 unsigned int kctx_len = 0; 1369 unsigned short stop_offset = 0; 1370 unsigned int assoclen = req->assoclen; 1371 unsigned int authsize = crypto_aead_authsize(tfm); 1372 int err = 0; 1373 int null = 0; 1374 gfp_t flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL : 1375 GFP_ATOMIC; 1376 1377 if (aeadctx->enckey_len == 0 || (req->cryptlen == 0)) 1378 goto err; 1379 1380 if (op_type && req->cryptlen < crypto_aead_authsize(tfm)) 1381 goto err; 1382 1383 if (sg_nents_for_len(req->src, req->assoclen + req->cryptlen) < 0) 1384 goto err; 1385 src = scatterwalk_ffwd(reqctx->srcffwd, req->src, req->assoclen); 1386 reqctx->dst = src; 1387 1388 if (req->src != req->dst) { 1389 err = chcr_copy_assoc(req, aeadctx); 1390 if (err) 1391 return ERR_PTR(err); 1392 reqctx->dst = scatterwalk_ffwd(reqctx->dstffwd, req->dst, 1393 req->assoclen); 1394 } 1395 if (get_aead_subtype(tfm) == CRYPTO_ALG_SUB_TYPE_AEAD_NULL) { 1396 null = 1; 1397 assoclen = 0; 1398 } 1399 reqctx->dst_nents = sg_nents_for_len(reqctx->dst, req->cryptlen + 1400 (op_type ? -authsize : authsize)); 1401 if (reqctx->dst_nents <= 0) { 1402 pr_err("AUTHENC:Invalid Destination sg entries\n"); 1403 goto err; 1404 } 1405 dst_size = get_space_for_phys_dsgl(reqctx->dst_nents); 1406 kctx_len = (ntohl(KEY_CONTEXT_CTX_LEN_V(aeadctx->key_ctx_hdr)) << 4) 1407 - sizeof(chcr_req->key_ctx); 1408 transhdr_len = CIPHER_TRANSHDR_SIZE(kctx_len, dst_size); 1409 skb = alloc_skb((transhdr_len + sizeof(struct sge_opaque_hdr)), flags); 1410 if (!skb) 1411 goto err; 1412 1413 /* LLD is going to write the sge hdr. */ 1414 skb_reserve(skb, sizeof(struct sge_opaque_hdr)); 1415 1416 /* Write WR */ 1417 chcr_req = (struct chcr_wr *) __skb_put(skb, transhdr_len); 1418 memset(chcr_req, 0, transhdr_len); 1419 1420 stop_offset = (op_type == CHCR_ENCRYPT_OP) ? 0 : authsize; 1421 1422 /* 1423 * Input order is AAD,IV and Payload. where IV should be included as 1424 * the part of authdata. All other fields should be filled according 1425 * to the hardware spec 1426 */ 1427 chcr_req->sec_cpl.op_ivinsrtofst = 1428 FILL_SEC_CPL_OP_IVINSR(ctx->dev->tx_channel_id, 2, 1429 (ivsize ? (assoclen + 1) : 0)); 1430 chcr_req->sec_cpl.pldlen = htonl(assoclen + ivsize + req->cryptlen); 1431 chcr_req->sec_cpl.aadstart_cipherstop_hi = FILL_SEC_CPL_CIPHERSTOP_HI( 1432 assoclen ? 1 : 0, assoclen, 1433 assoclen + ivsize + 1, 1434 (stop_offset & 0x1F0) >> 4); 1435 chcr_req->sec_cpl.cipherstop_lo_authinsert = FILL_SEC_CPL_AUTHINSERT( 1436 stop_offset & 0xF, 1437 null ? 0 : assoclen + ivsize + 1, 1438 stop_offset, stop_offset); 1439 chcr_req->sec_cpl.seqno_numivs = FILL_SEC_CPL_SCMD0_SEQNO(op_type, 1440 (op_type == CHCR_ENCRYPT_OP) ? 1 : 0, 1441 CHCR_SCMD_CIPHER_MODE_AES_CBC, 1442 actx->auth_mode, aeadctx->hmac_ctrl, 1443 ivsize >> 1); 1444 chcr_req->sec_cpl.ivgen_hdrlen = FILL_SEC_CPL_IVGEN_HDRLEN(0, 0, 1, 1445 0, 1, dst_size); 1446 1447 chcr_req->key_ctx.ctx_hdr = aeadctx->key_ctx_hdr; 1448 if (op_type == CHCR_ENCRYPT_OP) 1449 memcpy(chcr_req->key_ctx.key, aeadctx->key, 1450 aeadctx->enckey_len); 1451 else 1452 memcpy(chcr_req->key_ctx.key, actx->dec_rrkey, 1453 aeadctx->enckey_len); 1454 1455 memcpy(chcr_req->key_ctx.key + (DIV_ROUND_UP(aeadctx->enckey_len, 16) << 1456 4), actx->h_iopad, kctx_len - 1457 (DIV_ROUND_UP(aeadctx->enckey_len, 16) << 4)); 1458 1459 phys_cpl = (struct cpl_rx_phys_dsgl *)((u8 *)(chcr_req + 1) + kctx_len); 1460 sg_param.nents = reqctx->dst_nents; 1461 sg_param.obsize = req->cryptlen + (op_type ? -authsize : authsize); 1462 sg_param.qid = qid; 1463 sg_param.align = 0; 1464 if (map_writesg_phys_cpl(&u_ctx->lldi.pdev->dev, phys_cpl, reqctx->dst, 1465 &sg_param)) 1466 goto dstmap_fail; 1467 1468 skb_set_transport_header(skb, transhdr_len); 1469 1470 if (assoclen) { 1471 /* AAD buffer in */ 1472 write_sg_to_skb(skb, &frags, req->src, assoclen); 1473 1474 } 1475 write_buffer_to_skb(skb, &frags, req->iv, ivsize); 1476 write_sg_to_skb(skb, &frags, src, req->cryptlen); 1477 create_wreq(ctx, chcr_req, req, skb, kctx_len, size, 1, 1478 sizeof(struct cpl_rx_phys_dsgl) + dst_size); 1479 reqctx->skb = skb; 1480 skb_get(skb); 1481 1482 return skb; 1483 dstmap_fail: 1484 /* ivmap_fail: */ 1485 kfree_skb(skb); 1486 err: 1487 return ERR_PTR(-EINVAL); 1488 } 1489 1490 static void aes_gcm_empty_pld_pad(struct scatterlist *sg, 1491 unsigned short offset) 1492 { 1493 struct page *spage; 1494 unsigned char *addr; 1495 1496 spage = sg_page(sg); 1497 get_page(spage); /* so that it is not freed by NIC */ 1498 #ifdef KMAP_ATOMIC_ARGS 1499 addr = kmap_atomic(spage, KM_SOFTIRQ0); 1500 #else 1501 addr = kmap_atomic(spage); 1502 #endif 1503 memset(addr + sg->offset, 0, offset + 1); 1504 1505 kunmap_atomic(addr); 1506 } 1507 1508 static int set_msg_len(u8 *block, unsigned int msglen, int csize) 1509 { 1510 __be32 data; 1511 1512 memset(block, 0, csize); 1513 block += csize; 1514 1515 if (csize >= 4) 1516 csize = 4; 1517 else if (msglen > (unsigned int)(1 << (8 * csize))) 1518 return -EOVERFLOW; 1519 1520 data = cpu_to_be32(msglen); 1521 memcpy(block - csize, (u8 *)&data + 4 - csize, csize); 1522 1523 return 0; 1524 } 1525 1526 static void generate_b0(struct aead_request *req, 1527 struct chcr_aead_ctx *aeadctx, 1528 unsigned short op_type) 1529 { 1530 unsigned int l, lp, m; 1531 int rc; 1532 struct crypto_aead *aead = crypto_aead_reqtfm(req); 1533 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req); 1534 u8 *b0 = reqctx->scratch_pad; 1535 1536 m = crypto_aead_authsize(aead); 1537 1538 memcpy(b0, reqctx->iv, 16); 1539 1540 lp = b0[0]; 1541 l = lp + 1; 1542 1543 /* set m, bits 3-5 */ 1544 *b0 |= (8 * ((m - 2) / 2)); 1545 1546 /* set adata, bit 6, if associated data is used */ 1547 if (req->assoclen) 1548 *b0 |= 64; 1549 rc = set_msg_len(b0 + 16 - l, 1550 (op_type == CHCR_DECRYPT_OP) ? 1551 req->cryptlen - m : req->cryptlen, l); 1552 } 1553 1554 static inline int crypto_ccm_check_iv(const u8 *iv) 1555 { 1556 /* 2 <= L <= 8, so 1 <= L' <= 7. */ 1557 if (iv[0] < 1 || iv[0] > 7) 1558 return -EINVAL; 1559 1560 return 0; 1561 } 1562 1563 static int ccm_format_packet(struct aead_request *req, 1564 struct chcr_aead_ctx *aeadctx, 1565 unsigned int sub_type, 1566 unsigned short op_type) 1567 { 1568 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req); 1569 int rc = 0; 1570 1571 if (req->assoclen > T5_MAX_AAD_SIZE) { 1572 pr_err("CCM: Unsupported AAD data. It should be < %d\n", 1573 T5_MAX_AAD_SIZE); 1574 return -EINVAL; 1575 } 1576 if (sub_type == CRYPTO_ALG_SUB_TYPE_AEAD_RFC4309) { 1577 reqctx->iv[0] = 3; 1578 memcpy(reqctx->iv + 1, &aeadctx->salt[0], 3); 1579 memcpy(reqctx->iv + 4, req->iv, 8); 1580 memset(reqctx->iv + 12, 0, 4); 1581 *((unsigned short *)(reqctx->scratch_pad + 16)) = 1582 htons(req->assoclen - 8); 1583 } else { 1584 memcpy(reqctx->iv, req->iv, 16); 1585 *((unsigned short *)(reqctx->scratch_pad + 16)) = 1586 htons(req->assoclen); 1587 } 1588 generate_b0(req, aeadctx, op_type); 1589 /* zero the ctr value */ 1590 memset(reqctx->iv + 15 - reqctx->iv[0], 0, reqctx->iv[0] + 1); 1591 return rc; 1592 } 1593 1594 static void fill_sec_cpl_for_aead(struct cpl_tx_sec_pdu *sec_cpl, 1595 unsigned int dst_size, 1596 struct aead_request *req, 1597 unsigned short op_type, 1598 struct chcr_context *chcrctx) 1599 { 1600 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 1601 unsigned int ivsize = AES_BLOCK_SIZE; 1602 unsigned int cipher_mode = CHCR_SCMD_CIPHER_MODE_AES_CCM; 1603 unsigned int mac_mode = CHCR_SCMD_AUTH_MODE_CBCMAC; 1604 unsigned int c_id = chcrctx->dev->tx_channel_id; 1605 unsigned int ccm_xtra; 1606 unsigned char tag_offset = 0, auth_offset = 0; 1607 unsigned char hmac_ctrl = get_hmac(crypto_aead_authsize(tfm)); 1608 unsigned int assoclen; 1609 1610 if (get_aead_subtype(tfm) == CRYPTO_ALG_SUB_TYPE_AEAD_RFC4309) 1611 assoclen = req->assoclen - 8; 1612 else 1613 assoclen = req->assoclen; 1614 ccm_xtra = CCM_B0_SIZE + 1615 ((assoclen) ? CCM_AAD_FIELD_SIZE : 0); 1616 1617 auth_offset = req->cryptlen ? 1618 (assoclen + ivsize + 1 + ccm_xtra) : 0; 1619 if (op_type == CHCR_DECRYPT_OP) { 1620 if (crypto_aead_authsize(tfm) != req->cryptlen) 1621 tag_offset = crypto_aead_authsize(tfm); 1622 else 1623 auth_offset = 0; 1624 } 1625 1626 1627 sec_cpl->op_ivinsrtofst = FILL_SEC_CPL_OP_IVINSR(c_id, 1628 2, (ivsize ? (assoclen + 1) : 0) + 1629 ccm_xtra); 1630 sec_cpl->pldlen = 1631 htonl(assoclen + ivsize + req->cryptlen + ccm_xtra); 1632 /* For CCM there wil be b0 always. So AAD start will be 1 always */ 1633 sec_cpl->aadstart_cipherstop_hi = FILL_SEC_CPL_CIPHERSTOP_HI( 1634 1, assoclen + ccm_xtra, assoclen 1635 + ivsize + 1 + ccm_xtra, 0); 1636 1637 sec_cpl->cipherstop_lo_authinsert = FILL_SEC_CPL_AUTHINSERT(0, 1638 auth_offset, tag_offset, 1639 (op_type == CHCR_ENCRYPT_OP) ? 0 : 1640 crypto_aead_authsize(tfm)); 1641 sec_cpl->seqno_numivs = FILL_SEC_CPL_SCMD0_SEQNO(op_type, 1642 (op_type == CHCR_ENCRYPT_OP) ? 0 : 1, 1643 cipher_mode, mac_mode, hmac_ctrl, 1644 ivsize >> 1); 1645 1646 sec_cpl->ivgen_hdrlen = FILL_SEC_CPL_IVGEN_HDRLEN(0, 0, 1, 0, 1647 1, dst_size); 1648 } 1649 1650 int aead_ccm_validate_input(unsigned short op_type, 1651 struct aead_request *req, 1652 struct chcr_aead_ctx *aeadctx, 1653 unsigned int sub_type) 1654 { 1655 if (sub_type != CRYPTO_ALG_SUB_TYPE_AEAD_RFC4309) { 1656 if (crypto_ccm_check_iv(req->iv)) { 1657 pr_err("CCM: IV check fails\n"); 1658 return -EINVAL; 1659 } 1660 } else { 1661 if (req->assoclen != 16 && req->assoclen != 20) { 1662 pr_err("RFC4309: Invalid AAD length %d\n", 1663 req->assoclen); 1664 return -EINVAL; 1665 } 1666 } 1667 if (aeadctx->enckey_len == 0) { 1668 pr_err("CCM: Encryption key not set\n"); 1669 return -EINVAL; 1670 } 1671 return 0; 1672 } 1673 1674 unsigned int fill_aead_req_fields(struct sk_buff *skb, 1675 struct aead_request *req, 1676 struct scatterlist *src, 1677 unsigned int ivsize, 1678 struct chcr_aead_ctx *aeadctx) 1679 { 1680 unsigned int frags = 0; 1681 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 1682 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req); 1683 /* b0 and aad length(if available) */ 1684 1685 write_buffer_to_skb(skb, &frags, reqctx->scratch_pad, CCM_B0_SIZE + 1686 (req->assoclen ? CCM_AAD_FIELD_SIZE : 0)); 1687 if (req->assoclen) { 1688 if (get_aead_subtype(tfm) == CRYPTO_ALG_SUB_TYPE_AEAD_RFC4309) 1689 write_sg_to_skb(skb, &frags, req->src, 1690 req->assoclen - 8); 1691 else 1692 write_sg_to_skb(skb, &frags, req->src, req->assoclen); 1693 } 1694 write_buffer_to_skb(skb, &frags, reqctx->iv, ivsize); 1695 if (req->cryptlen) 1696 write_sg_to_skb(skb, &frags, src, req->cryptlen); 1697 1698 return frags; 1699 } 1700 1701 static struct sk_buff *create_aead_ccm_wr(struct aead_request *req, 1702 unsigned short qid, 1703 int size, 1704 unsigned short op_type) 1705 { 1706 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 1707 struct chcr_context *ctx = crypto_aead_ctx(tfm); 1708 struct uld_ctx *u_ctx = ULD_CTX(ctx); 1709 struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx); 1710 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req); 1711 struct sk_buff *skb = NULL; 1712 struct chcr_wr *chcr_req; 1713 struct cpl_rx_phys_dsgl *phys_cpl; 1714 struct phys_sge_parm sg_param; 1715 struct scatterlist *src; 1716 unsigned int frags = 0, transhdr_len, ivsize = AES_BLOCK_SIZE; 1717 unsigned int dst_size = 0, kctx_len; 1718 unsigned int sub_type; 1719 unsigned int authsize = crypto_aead_authsize(tfm); 1720 int err = 0; 1721 gfp_t flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL : 1722 GFP_ATOMIC; 1723 1724 1725 if (op_type && req->cryptlen < crypto_aead_authsize(tfm)) 1726 goto err; 1727 1728 if (sg_nents_for_len(req->src, req->assoclen + req->cryptlen) < 0) 1729 goto err; 1730 sub_type = get_aead_subtype(tfm); 1731 src = scatterwalk_ffwd(reqctx->srcffwd, req->src, req->assoclen); 1732 reqctx->dst = src; 1733 1734 if (req->src != req->dst) { 1735 err = chcr_copy_assoc(req, aeadctx); 1736 if (err) { 1737 pr_err("AAD copy to destination buffer fails\n"); 1738 return ERR_PTR(err); 1739 } 1740 reqctx->dst = scatterwalk_ffwd(reqctx->dstffwd, req->dst, 1741 req->assoclen); 1742 } 1743 reqctx->dst_nents = sg_nents_for_len(reqctx->dst, req->cryptlen + 1744 (op_type ? -authsize : authsize)); 1745 if (reqctx->dst_nents <= 0) { 1746 pr_err("CCM:Invalid Destination sg entries\n"); 1747 goto err; 1748 } 1749 1750 1751 if (aead_ccm_validate_input(op_type, req, aeadctx, sub_type)) 1752 goto err; 1753 1754 dst_size = get_space_for_phys_dsgl(reqctx->dst_nents); 1755 kctx_len = ((DIV_ROUND_UP(aeadctx->enckey_len, 16)) << 4) * 2; 1756 transhdr_len = CIPHER_TRANSHDR_SIZE(kctx_len, dst_size); 1757 skb = alloc_skb((transhdr_len + sizeof(struct sge_opaque_hdr)), flags); 1758 1759 if (!skb) 1760 goto err; 1761 1762 skb_reserve(skb, sizeof(struct sge_opaque_hdr)); 1763 1764 chcr_req = (struct chcr_wr *) __skb_put(skb, transhdr_len); 1765 memset(chcr_req, 0, transhdr_len); 1766 1767 fill_sec_cpl_for_aead(&chcr_req->sec_cpl, dst_size, req, op_type, ctx); 1768 1769 chcr_req->key_ctx.ctx_hdr = aeadctx->key_ctx_hdr; 1770 memcpy(chcr_req->key_ctx.key, aeadctx->key, aeadctx->enckey_len); 1771 memcpy(chcr_req->key_ctx.key + (DIV_ROUND_UP(aeadctx->enckey_len, 16) * 1772 16), aeadctx->key, aeadctx->enckey_len); 1773 1774 phys_cpl = (struct cpl_rx_phys_dsgl *)((u8 *)(chcr_req + 1) + kctx_len); 1775 if (ccm_format_packet(req, aeadctx, sub_type, op_type)) 1776 goto dstmap_fail; 1777 1778 sg_param.nents = reqctx->dst_nents; 1779 sg_param.obsize = req->cryptlen + (op_type ? -authsize : authsize); 1780 sg_param.qid = qid; 1781 sg_param.align = 0; 1782 if (map_writesg_phys_cpl(&u_ctx->lldi.pdev->dev, phys_cpl, reqctx->dst, 1783 &sg_param)) 1784 goto dstmap_fail; 1785 1786 skb_set_transport_header(skb, transhdr_len); 1787 frags = fill_aead_req_fields(skb, req, src, ivsize, aeadctx); 1788 create_wreq(ctx, chcr_req, req, skb, kctx_len, 0, 1, 1789 sizeof(struct cpl_rx_phys_dsgl) + dst_size); 1790 reqctx->skb = skb; 1791 skb_get(skb); 1792 return skb; 1793 dstmap_fail: 1794 kfree_skb(skb); 1795 skb = NULL; 1796 err: 1797 return ERR_PTR(-EINVAL); 1798 } 1799 1800 static struct sk_buff *create_gcm_wr(struct aead_request *req, 1801 unsigned short qid, 1802 int size, 1803 unsigned short op_type) 1804 { 1805 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 1806 struct chcr_context *ctx = crypto_aead_ctx(tfm); 1807 struct uld_ctx *u_ctx = ULD_CTX(ctx); 1808 struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx); 1809 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req); 1810 struct sk_buff *skb = NULL; 1811 struct chcr_wr *chcr_req; 1812 struct cpl_rx_phys_dsgl *phys_cpl; 1813 struct phys_sge_parm sg_param; 1814 struct scatterlist *src; 1815 unsigned int frags = 0, transhdr_len; 1816 unsigned int ivsize = AES_BLOCK_SIZE; 1817 unsigned int dst_size = 0, kctx_len; 1818 unsigned char tag_offset = 0; 1819 unsigned int crypt_len = 0; 1820 unsigned int authsize = crypto_aead_authsize(tfm); 1821 unsigned char hmac_ctrl = get_hmac(authsize); 1822 int err = 0; 1823 gfp_t flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL : 1824 GFP_ATOMIC; 1825 1826 /* validate key size */ 1827 if (aeadctx->enckey_len == 0) 1828 goto err; 1829 1830 if (op_type && req->cryptlen < crypto_aead_authsize(tfm)) 1831 goto err; 1832 1833 if (sg_nents_for_len(req->src, req->assoclen + req->cryptlen) < 0) 1834 goto err; 1835 1836 src = scatterwalk_ffwd(reqctx->srcffwd, req->src, req->assoclen); 1837 reqctx->dst = src; 1838 if (req->src != req->dst) { 1839 err = chcr_copy_assoc(req, aeadctx); 1840 if (err) 1841 return ERR_PTR(err); 1842 reqctx->dst = scatterwalk_ffwd(reqctx->dstffwd, req->dst, 1843 req->assoclen); 1844 } 1845 1846 if (!req->cryptlen) 1847 /* null-payload is not supported in the hardware. 1848 * software is sending block size 1849 */ 1850 crypt_len = AES_BLOCK_SIZE; 1851 else 1852 crypt_len = req->cryptlen; 1853 reqctx->dst_nents = sg_nents_for_len(reqctx->dst, req->cryptlen + 1854 (op_type ? -authsize : authsize)); 1855 if (reqctx->dst_nents <= 0) { 1856 pr_err("GCM:Invalid Destination sg entries\n"); 1857 goto err; 1858 } 1859 1860 1861 dst_size = get_space_for_phys_dsgl(reqctx->dst_nents); 1862 kctx_len = ((DIV_ROUND_UP(aeadctx->enckey_len, 16)) << 4) + 1863 AEAD_H_SIZE; 1864 transhdr_len = CIPHER_TRANSHDR_SIZE(kctx_len, dst_size); 1865 skb = alloc_skb((transhdr_len + sizeof(struct sge_opaque_hdr)), flags); 1866 if (!skb) 1867 goto err; 1868 1869 /* NIC driver is going to write the sge hdr. */ 1870 skb_reserve(skb, sizeof(struct sge_opaque_hdr)); 1871 1872 chcr_req = (struct chcr_wr *)__skb_put(skb, transhdr_len); 1873 memset(chcr_req, 0, transhdr_len); 1874 1875 if (get_aead_subtype(tfm) == CRYPTO_ALG_SUB_TYPE_AEAD_RFC4106) 1876 req->assoclen -= 8; 1877 1878 tag_offset = (op_type == CHCR_ENCRYPT_OP) ? 0 : authsize; 1879 chcr_req->sec_cpl.op_ivinsrtofst = FILL_SEC_CPL_OP_IVINSR( 1880 ctx->dev->tx_channel_id, 2, (ivsize ? 1881 (req->assoclen + 1) : 0)); 1882 chcr_req->sec_cpl.pldlen = htonl(req->assoclen + ivsize + crypt_len); 1883 chcr_req->sec_cpl.aadstart_cipherstop_hi = FILL_SEC_CPL_CIPHERSTOP_HI( 1884 req->assoclen ? 1 : 0, req->assoclen, 1885 req->assoclen + ivsize + 1, 0); 1886 if (req->cryptlen) { 1887 chcr_req->sec_cpl.cipherstop_lo_authinsert = 1888 FILL_SEC_CPL_AUTHINSERT(0, req->assoclen + ivsize + 1, 1889 tag_offset, tag_offset); 1890 chcr_req->sec_cpl.seqno_numivs = 1891 FILL_SEC_CPL_SCMD0_SEQNO(op_type, (op_type == 1892 CHCR_ENCRYPT_OP) ? 1 : 0, 1893 CHCR_SCMD_CIPHER_MODE_AES_GCM, 1894 CHCR_SCMD_AUTH_MODE_GHASH, hmac_ctrl, 1895 ivsize >> 1); 1896 } else { 1897 chcr_req->sec_cpl.cipherstop_lo_authinsert = 1898 FILL_SEC_CPL_AUTHINSERT(0, 0, 0, 0); 1899 chcr_req->sec_cpl.seqno_numivs = 1900 FILL_SEC_CPL_SCMD0_SEQNO(op_type, 1901 (op_type == CHCR_ENCRYPT_OP) ? 1902 1 : 0, CHCR_SCMD_CIPHER_MODE_AES_CBC, 1903 0, 0, ivsize >> 1); 1904 } 1905 chcr_req->sec_cpl.ivgen_hdrlen = FILL_SEC_CPL_IVGEN_HDRLEN(0, 0, 1, 1906 0, 1, dst_size); 1907 chcr_req->key_ctx.ctx_hdr = aeadctx->key_ctx_hdr; 1908 memcpy(chcr_req->key_ctx.key, aeadctx->key, aeadctx->enckey_len); 1909 memcpy(chcr_req->key_ctx.key + (DIV_ROUND_UP(aeadctx->enckey_len, 16) * 1910 16), GCM_CTX(aeadctx)->ghash_h, AEAD_H_SIZE); 1911 1912 /* prepare a 16 byte iv */ 1913 /* S A L T | IV | 0x00000001 */ 1914 if (get_aead_subtype(tfm) == 1915 CRYPTO_ALG_SUB_TYPE_AEAD_RFC4106) { 1916 memcpy(reqctx->iv, aeadctx->salt, 4); 1917 memcpy(reqctx->iv + 4, req->iv, 8); 1918 } else { 1919 memcpy(reqctx->iv, req->iv, 12); 1920 } 1921 *((unsigned int *)(reqctx->iv + 12)) = htonl(0x01); 1922 1923 phys_cpl = (struct cpl_rx_phys_dsgl *)((u8 *)(chcr_req + 1) + kctx_len); 1924 sg_param.nents = reqctx->dst_nents; 1925 sg_param.obsize = req->cryptlen + (op_type ? -authsize : authsize); 1926 sg_param.qid = qid; 1927 sg_param.align = 0; 1928 if (map_writesg_phys_cpl(&u_ctx->lldi.pdev->dev, phys_cpl, reqctx->dst, 1929 &sg_param)) 1930 goto dstmap_fail; 1931 1932 skb_set_transport_header(skb, transhdr_len); 1933 1934 write_sg_to_skb(skb, &frags, req->src, req->assoclen); 1935 1936 write_buffer_to_skb(skb, &frags, reqctx->iv, ivsize); 1937 1938 if (req->cryptlen) { 1939 write_sg_to_skb(skb, &frags, src, req->cryptlen); 1940 } else { 1941 aes_gcm_empty_pld_pad(req->dst, authsize - 1); 1942 write_sg_to_skb(skb, &frags, reqctx->dst, crypt_len); 1943 1944 } 1945 1946 create_wreq(ctx, chcr_req, req, skb, kctx_len, size, 1, 1947 sizeof(struct cpl_rx_phys_dsgl) + dst_size); 1948 reqctx->skb = skb; 1949 skb_get(skb); 1950 return skb; 1951 1952 dstmap_fail: 1953 /* ivmap_fail: */ 1954 kfree_skb(skb); 1955 skb = NULL; 1956 err: 1957 return skb; 1958 } 1959 1960 1961 1962 static int chcr_aead_cra_init(struct crypto_aead *tfm) 1963 { 1964 struct chcr_context *ctx = crypto_aead_ctx(tfm); 1965 struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx); 1966 1967 crypto_aead_set_reqsize(tfm, sizeof(struct chcr_aead_reqctx)); 1968 aeadctx->null = crypto_get_default_null_skcipher(); 1969 if (IS_ERR(aeadctx->null)) 1970 return PTR_ERR(aeadctx->null); 1971 return chcr_device_init(ctx); 1972 } 1973 1974 static void chcr_aead_cra_exit(struct crypto_aead *tfm) 1975 { 1976 crypto_put_default_null_skcipher(); 1977 } 1978 1979 static int chcr_authenc_null_setauthsize(struct crypto_aead *tfm, 1980 unsigned int authsize) 1981 { 1982 struct chcr_aead_ctx *aeadctx = AEAD_CTX(crypto_aead_ctx(tfm)); 1983 1984 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_NOP; 1985 aeadctx->mayverify = VERIFY_HW; 1986 return 0; 1987 } 1988 static int chcr_authenc_setauthsize(struct crypto_aead *tfm, 1989 unsigned int authsize) 1990 { 1991 struct chcr_aead_ctx *aeadctx = AEAD_CTX(crypto_aead_ctx(tfm)); 1992 u32 maxauth = crypto_aead_maxauthsize(tfm); 1993 1994 /*SHA1 authsize in ipsec is 12 instead of 10 i.e maxauthsize / 2 is not 1995 * true for sha1. authsize == 12 condition should be before 1996 * authsize == (maxauth >> 1) 1997 */ 1998 if (authsize == ICV_4) { 1999 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_PL1; 2000 aeadctx->mayverify = VERIFY_HW; 2001 } else if (authsize == ICV_6) { 2002 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_PL2; 2003 aeadctx->mayverify = VERIFY_HW; 2004 } else if (authsize == ICV_10) { 2005 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_TRUNC_RFC4366; 2006 aeadctx->mayverify = VERIFY_HW; 2007 } else if (authsize == ICV_12) { 2008 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_IPSEC_96BIT; 2009 aeadctx->mayverify = VERIFY_HW; 2010 } else if (authsize == ICV_14) { 2011 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_PL3; 2012 aeadctx->mayverify = VERIFY_HW; 2013 } else if (authsize == (maxauth >> 1)) { 2014 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_DIV2; 2015 aeadctx->mayverify = VERIFY_HW; 2016 } else if (authsize == maxauth) { 2017 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_NO_TRUNC; 2018 aeadctx->mayverify = VERIFY_HW; 2019 } else { 2020 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_NO_TRUNC; 2021 aeadctx->mayverify = VERIFY_SW; 2022 } 2023 return 0; 2024 } 2025 2026 2027 static int chcr_gcm_setauthsize(struct crypto_aead *tfm, unsigned int authsize) 2028 { 2029 struct chcr_aead_ctx *aeadctx = AEAD_CTX(crypto_aead_ctx(tfm)); 2030 2031 switch (authsize) { 2032 case ICV_4: 2033 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_PL1; 2034 aeadctx->mayverify = VERIFY_HW; 2035 break; 2036 case ICV_8: 2037 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_DIV2; 2038 aeadctx->mayverify = VERIFY_HW; 2039 break; 2040 case ICV_12: 2041 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_IPSEC_96BIT; 2042 aeadctx->mayverify = VERIFY_HW; 2043 break; 2044 case ICV_14: 2045 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_PL3; 2046 aeadctx->mayverify = VERIFY_HW; 2047 break; 2048 case ICV_16: 2049 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_NO_TRUNC; 2050 aeadctx->mayverify = VERIFY_HW; 2051 break; 2052 case ICV_13: 2053 case ICV_15: 2054 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_NO_TRUNC; 2055 aeadctx->mayverify = VERIFY_SW; 2056 break; 2057 default: 2058 2059 crypto_tfm_set_flags((struct crypto_tfm *) tfm, 2060 CRYPTO_TFM_RES_BAD_KEY_LEN); 2061 return -EINVAL; 2062 } 2063 return 0; 2064 } 2065 2066 static int chcr_4106_4309_setauthsize(struct crypto_aead *tfm, 2067 unsigned int authsize) 2068 { 2069 struct chcr_aead_ctx *aeadctx = AEAD_CTX(crypto_aead_ctx(tfm)); 2070 2071 switch (authsize) { 2072 case ICV_8: 2073 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_DIV2; 2074 aeadctx->mayverify = VERIFY_HW; 2075 break; 2076 case ICV_12: 2077 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_IPSEC_96BIT; 2078 aeadctx->mayverify = VERIFY_HW; 2079 break; 2080 case ICV_16: 2081 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_NO_TRUNC; 2082 aeadctx->mayverify = VERIFY_HW; 2083 break; 2084 default: 2085 crypto_tfm_set_flags((struct crypto_tfm *)tfm, 2086 CRYPTO_TFM_RES_BAD_KEY_LEN); 2087 return -EINVAL; 2088 } 2089 return 0; 2090 } 2091 2092 static int chcr_ccm_setauthsize(struct crypto_aead *tfm, 2093 unsigned int authsize) 2094 { 2095 struct chcr_aead_ctx *aeadctx = AEAD_CTX(crypto_aead_ctx(tfm)); 2096 2097 switch (authsize) { 2098 case ICV_4: 2099 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_PL1; 2100 aeadctx->mayverify = VERIFY_HW; 2101 break; 2102 case ICV_6: 2103 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_PL2; 2104 aeadctx->mayverify = VERIFY_HW; 2105 break; 2106 case ICV_8: 2107 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_DIV2; 2108 aeadctx->mayverify = VERIFY_HW; 2109 break; 2110 case ICV_10: 2111 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_TRUNC_RFC4366; 2112 aeadctx->mayverify = VERIFY_HW; 2113 break; 2114 case ICV_12: 2115 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_IPSEC_96BIT; 2116 aeadctx->mayverify = VERIFY_HW; 2117 break; 2118 case ICV_14: 2119 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_PL3; 2120 aeadctx->mayverify = VERIFY_HW; 2121 break; 2122 case ICV_16: 2123 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_NO_TRUNC; 2124 aeadctx->mayverify = VERIFY_HW; 2125 break; 2126 default: 2127 crypto_tfm_set_flags((struct crypto_tfm *)tfm, 2128 CRYPTO_TFM_RES_BAD_KEY_LEN); 2129 return -EINVAL; 2130 } 2131 return 0; 2132 } 2133 2134 static int chcr_aead_ccm_setkey(struct crypto_aead *aead, 2135 const u8 *key, 2136 unsigned int keylen) 2137 { 2138 struct chcr_context *ctx = crypto_aead_ctx(aead); 2139 struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx); 2140 unsigned char ck_size, mk_size; 2141 int key_ctx_size = 0; 2142 2143 memcpy(aeadctx->key, key, keylen); 2144 aeadctx->enckey_len = keylen; 2145 key_ctx_size = sizeof(struct _key_ctx) + 2146 ((DIV_ROUND_UP(keylen, 16)) << 4) * 2; 2147 if (keylen == AES_KEYSIZE_128) { 2148 mk_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_128; 2149 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_128; 2150 } else if (keylen == AES_KEYSIZE_192) { 2151 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_192; 2152 mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_192; 2153 } else if (keylen == AES_KEYSIZE_256) { 2154 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_256; 2155 mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_256; 2156 } else { 2157 crypto_tfm_set_flags((struct crypto_tfm *)aead, 2158 CRYPTO_TFM_RES_BAD_KEY_LEN); 2159 aeadctx->enckey_len = 0; 2160 return -EINVAL; 2161 } 2162 aeadctx->key_ctx_hdr = FILL_KEY_CTX_HDR(ck_size, mk_size, 0, 0, 2163 key_ctx_size >> 4); 2164 return 0; 2165 } 2166 2167 static int chcr_aead_rfc4309_setkey(struct crypto_aead *aead, const u8 *key, 2168 unsigned int keylen) 2169 { 2170 struct chcr_context *ctx = crypto_aead_ctx(aead); 2171 struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx); 2172 2173 if (keylen < 3) { 2174 crypto_tfm_set_flags((struct crypto_tfm *)aead, 2175 CRYPTO_TFM_RES_BAD_KEY_LEN); 2176 aeadctx->enckey_len = 0; 2177 return -EINVAL; 2178 } 2179 keylen -= 3; 2180 memcpy(aeadctx->salt, key + keylen, 3); 2181 return chcr_aead_ccm_setkey(aead, key, keylen); 2182 } 2183 2184 static int chcr_gcm_setkey(struct crypto_aead *aead, const u8 *key, 2185 unsigned int keylen) 2186 { 2187 struct chcr_context *ctx = crypto_aead_ctx(aead); 2188 struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx); 2189 struct chcr_gcm_ctx *gctx = GCM_CTX(aeadctx); 2190 struct blkcipher_desc h_desc; 2191 struct scatterlist src[1]; 2192 unsigned int ck_size; 2193 int ret = 0, key_ctx_size = 0; 2194 2195 if (get_aead_subtype(aead) == CRYPTO_ALG_SUB_TYPE_AEAD_RFC4106 && 2196 keylen > 3) { 2197 keylen -= 4; /* nonce/salt is present in the last 4 bytes */ 2198 memcpy(aeadctx->salt, key + keylen, 4); 2199 } 2200 if (keylen == AES_KEYSIZE_128) { 2201 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_128; 2202 } else if (keylen == AES_KEYSIZE_192) { 2203 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_192; 2204 } else if (keylen == AES_KEYSIZE_256) { 2205 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_256; 2206 } else { 2207 crypto_tfm_set_flags((struct crypto_tfm *)aead, 2208 CRYPTO_TFM_RES_BAD_KEY_LEN); 2209 aeadctx->enckey_len = 0; 2210 pr_err("GCM: Invalid key length %d", keylen); 2211 ret = -EINVAL; 2212 goto out; 2213 } 2214 2215 memcpy(aeadctx->key, key, keylen); 2216 aeadctx->enckey_len = keylen; 2217 key_ctx_size = sizeof(struct _key_ctx) + 2218 ((DIV_ROUND_UP(keylen, 16)) << 4) + 2219 AEAD_H_SIZE; 2220 aeadctx->key_ctx_hdr = FILL_KEY_CTX_HDR(ck_size, 2221 CHCR_KEYCTX_MAC_KEY_SIZE_128, 2222 0, 0, 2223 key_ctx_size >> 4); 2224 /* Calculate the H = CIPH(K, 0 repeated 16 times) using sync aes 2225 * blkcipher It will go on key context 2226 */ 2227 h_desc.tfm = crypto_alloc_blkcipher("cbc(aes-generic)", 0, 0); 2228 if (IS_ERR(h_desc.tfm)) { 2229 aeadctx->enckey_len = 0; 2230 ret = -ENOMEM; 2231 goto out; 2232 } 2233 h_desc.flags = 0; 2234 ret = crypto_blkcipher_setkey(h_desc.tfm, key, keylen); 2235 if (ret) { 2236 aeadctx->enckey_len = 0; 2237 goto out1; 2238 } 2239 memset(gctx->ghash_h, 0, AEAD_H_SIZE); 2240 sg_init_one(&src[0], gctx->ghash_h, AEAD_H_SIZE); 2241 ret = crypto_blkcipher_encrypt(&h_desc, &src[0], &src[0], AEAD_H_SIZE); 2242 2243 out1: 2244 crypto_free_blkcipher(h_desc.tfm); 2245 out: 2246 return ret; 2247 } 2248 2249 static int chcr_authenc_setkey(struct crypto_aead *authenc, const u8 *key, 2250 unsigned int keylen) 2251 { 2252 struct chcr_context *ctx = crypto_aead_ctx(authenc); 2253 struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx); 2254 struct chcr_authenc_ctx *actx = AUTHENC_CTX(aeadctx); 2255 /* it contains auth and cipher key both*/ 2256 struct crypto_authenc_keys keys; 2257 unsigned int bs; 2258 unsigned int max_authsize = crypto_aead_alg(authenc)->maxauthsize; 2259 int err = 0, i, key_ctx_len = 0; 2260 unsigned char ck_size = 0; 2261 unsigned char pad[CHCR_HASH_MAX_BLOCK_SIZE_128] = { 0 }; 2262 struct crypto_shash *base_hash = NULL; 2263 struct algo_param param; 2264 int align; 2265 u8 *o_ptr = NULL; 2266 2267 if (crypto_authenc_extractkeys(&keys, key, keylen) != 0) { 2268 crypto_aead_set_flags(authenc, CRYPTO_TFM_RES_BAD_KEY_LEN); 2269 goto out; 2270 } 2271 2272 if (get_alg_config(¶m, max_authsize)) { 2273 pr_err("chcr : Unsupported digest size\n"); 2274 goto out; 2275 } 2276 if (keys.enckeylen == AES_KEYSIZE_128) { 2277 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_128; 2278 } else if (keys.enckeylen == AES_KEYSIZE_192) { 2279 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_192; 2280 } else if (keys.enckeylen == AES_KEYSIZE_256) { 2281 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_256; 2282 } else { 2283 pr_err("chcr : Unsupported cipher key\n"); 2284 goto out; 2285 } 2286 2287 /* Copy only encryption key. We use authkey to generate h(ipad) and 2288 * h(opad) so authkey is not needed again. authkeylen size have the 2289 * size of the hash digest size. 2290 */ 2291 memcpy(aeadctx->key, keys.enckey, keys.enckeylen); 2292 aeadctx->enckey_len = keys.enckeylen; 2293 get_aes_decrypt_key(actx->dec_rrkey, aeadctx->key, 2294 aeadctx->enckey_len << 3); 2295 2296 base_hash = chcr_alloc_shash(max_authsize); 2297 if (IS_ERR(base_hash)) { 2298 pr_err("chcr : Base driver cannot be loaded\n"); 2299 goto out; 2300 } 2301 { 2302 SHASH_DESC_ON_STACK(shash, base_hash); 2303 shash->tfm = base_hash; 2304 shash->flags = crypto_shash_get_flags(base_hash); 2305 bs = crypto_shash_blocksize(base_hash); 2306 align = KEYCTX_ALIGN_PAD(max_authsize); 2307 o_ptr = actx->h_iopad + param.result_size + align; 2308 2309 if (keys.authkeylen > bs) { 2310 err = crypto_shash_digest(shash, keys.authkey, 2311 keys.authkeylen, 2312 o_ptr); 2313 if (err) { 2314 pr_err("chcr : Base driver cannot be loaded\n"); 2315 goto out; 2316 } 2317 keys.authkeylen = max_authsize; 2318 } else 2319 memcpy(o_ptr, keys.authkey, keys.authkeylen); 2320 2321 /* Compute the ipad-digest*/ 2322 memset(pad + keys.authkeylen, 0, bs - keys.authkeylen); 2323 memcpy(pad, o_ptr, keys.authkeylen); 2324 for (i = 0; i < bs >> 2; i++) 2325 *((unsigned int *)pad + i) ^= IPAD_DATA; 2326 2327 if (chcr_compute_partial_hash(shash, pad, actx->h_iopad, 2328 max_authsize)) 2329 goto out; 2330 /* Compute the opad-digest */ 2331 memset(pad + keys.authkeylen, 0, bs - keys.authkeylen); 2332 memcpy(pad, o_ptr, keys.authkeylen); 2333 for (i = 0; i < bs >> 2; i++) 2334 *((unsigned int *)pad + i) ^= OPAD_DATA; 2335 2336 if (chcr_compute_partial_hash(shash, pad, o_ptr, max_authsize)) 2337 goto out; 2338 2339 /* convert the ipad and opad digest to network order */ 2340 chcr_change_order(actx->h_iopad, param.result_size); 2341 chcr_change_order(o_ptr, param.result_size); 2342 key_ctx_len = sizeof(struct _key_ctx) + 2343 ((DIV_ROUND_UP(keys.enckeylen, 16)) << 4) + 2344 (param.result_size + align) * 2; 2345 aeadctx->key_ctx_hdr = FILL_KEY_CTX_HDR(ck_size, param.mk_size, 2346 0, 1, key_ctx_len >> 4); 2347 actx->auth_mode = param.auth_mode; 2348 chcr_free_shash(base_hash); 2349 2350 return 0; 2351 } 2352 out: 2353 aeadctx->enckey_len = 0; 2354 if (base_hash) 2355 chcr_free_shash(base_hash); 2356 return -EINVAL; 2357 } 2358 2359 static int chcr_aead_digest_null_setkey(struct crypto_aead *authenc, 2360 const u8 *key, unsigned int keylen) 2361 { 2362 struct chcr_context *ctx = crypto_aead_ctx(authenc); 2363 struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx); 2364 struct chcr_authenc_ctx *actx = AUTHENC_CTX(aeadctx); 2365 struct crypto_authenc_keys keys; 2366 2367 /* it contains auth and cipher key both*/ 2368 int key_ctx_len = 0; 2369 unsigned char ck_size = 0; 2370 2371 if (crypto_authenc_extractkeys(&keys, key, keylen) != 0) { 2372 crypto_aead_set_flags(authenc, CRYPTO_TFM_RES_BAD_KEY_LEN); 2373 goto out; 2374 } 2375 if (keys.enckeylen == AES_KEYSIZE_128) { 2376 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_128; 2377 } else if (keys.enckeylen == AES_KEYSIZE_192) { 2378 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_192; 2379 } else if (keys.enckeylen == AES_KEYSIZE_256) { 2380 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_256; 2381 } else { 2382 pr_err("chcr : Unsupported cipher key\n"); 2383 goto out; 2384 } 2385 memcpy(aeadctx->key, keys.enckey, keys.enckeylen); 2386 aeadctx->enckey_len = keys.enckeylen; 2387 get_aes_decrypt_key(actx->dec_rrkey, aeadctx->key, 2388 aeadctx->enckey_len << 3); 2389 key_ctx_len = sizeof(struct _key_ctx) 2390 + ((DIV_ROUND_UP(keys.enckeylen, 16)) << 4); 2391 2392 aeadctx->key_ctx_hdr = FILL_KEY_CTX_HDR(ck_size, CHCR_KEYCTX_NO_KEY, 0, 2393 0, key_ctx_len >> 4); 2394 actx->auth_mode = CHCR_SCMD_AUTH_MODE_NOP; 2395 return 0; 2396 out: 2397 aeadctx->enckey_len = 0; 2398 return -EINVAL; 2399 } 2400 static int chcr_aead_encrypt(struct aead_request *req) 2401 { 2402 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 2403 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req); 2404 2405 reqctx->verify = VERIFY_HW; 2406 2407 switch (get_aead_subtype(tfm)) { 2408 case CRYPTO_ALG_SUB_TYPE_AEAD_AUTHENC: 2409 case CRYPTO_ALG_SUB_TYPE_AEAD_NULL: 2410 return chcr_aead_op(req, CHCR_ENCRYPT_OP, 0, 2411 create_authenc_wr); 2412 case CRYPTO_ALG_SUB_TYPE_AEAD_CCM: 2413 case CRYPTO_ALG_SUB_TYPE_AEAD_RFC4309: 2414 return chcr_aead_op(req, CHCR_ENCRYPT_OP, 0, 2415 create_aead_ccm_wr); 2416 default: 2417 return chcr_aead_op(req, CHCR_ENCRYPT_OP, 0, 2418 create_gcm_wr); 2419 } 2420 } 2421 2422 static int chcr_aead_decrypt(struct aead_request *req) 2423 { 2424 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 2425 struct chcr_aead_ctx *aeadctx = AEAD_CTX(crypto_aead_ctx(tfm)); 2426 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req); 2427 int size; 2428 2429 if (aeadctx->mayverify == VERIFY_SW) { 2430 size = crypto_aead_maxauthsize(tfm); 2431 reqctx->verify = VERIFY_SW; 2432 } else { 2433 size = 0; 2434 reqctx->verify = VERIFY_HW; 2435 } 2436 2437 switch (get_aead_subtype(tfm)) { 2438 case CRYPTO_ALG_SUB_TYPE_AEAD_AUTHENC: 2439 case CRYPTO_ALG_SUB_TYPE_AEAD_NULL: 2440 return chcr_aead_op(req, CHCR_DECRYPT_OP, size, 2441 create_authenc_wr); 2442 case CRYPTO_ALG_SUB_TYPE_AEAD_CCM: 2443 case CRYPTO_ALG_SUB_TYPE_AEAD_RFC4309: 2444 return chcr_aead_op(req, CHCR_DECRYPT_OP, size, 2445 create_aead_ccm_wr); 2446 default: 2447 return chcr_aead_op(req, CHCR_DECRYPT_OP, size, 2448 create_gcm_wr); 2449 } 2450 } 2451 2452 static int chcr_aead_op(struct aead_request *req, 2453 unsigned short op_type, 2454 int size, 2455 create_wr_t create_wr_fn) 2456 { 2457 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 2458 struct chcr_context *ctx = crypto_aead_ctx(tfm); 2459 struct uld_ctx *u_ctx = ULD_CTX(ctx); 2460 struct sk_buff *skb; 2461 2462 if (ctx && !ctx->dev) { 2463 pr_err("chcr : %s : No crypto device.\n", __func__); 2464 return -ENXIO; 2465 } 2466 if (cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0], 2467 ctx->tx_channel_id)) { 2468 if (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) 2469 return -EBUSY; 2470 } 2471 2472 /* Form a WR from req */ 2473 skb = create_wr_fn(req, u_ctx->lldi.rxq_ids[ctx->tx_channel_id], size, 2474 op_type); 2475 2476 if (IS_ERR(skb) || skb == NULL) { 2477 pr_err("chcr : %s : failed to form WR. No memory\n", __func__); 2478 return PTR_ERR(skb); 2479 } 2480 2481 skb->dev = u_ctx->lldi.ports[0]; 2482 set_wr_txq(skb, CPL_PRIORITY_DATA, ctx->tx_channel_id); 2483 chcr_send_wr(skb); 2484 return -EINPROGRESS; 2485 } 2486 static struct chcr_alg_template driver_algs[] = { 2487 /* AES-CBC */ 2488 { 2489 .type = CRYPTO_ALG_TYPE_ABLKCIPHER, 2490 .is_registered = 0, 2491 .alg.crypto = { 2492 .cra_name = "cbc(aes)", 2493 .cra_driver_name = "cbc-aes-chcr", 2494 .cra_priority = CHCR_CRA_PRIORITY, 2495 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER | 2496 CRYPTO_ALG_ASYNC, 2497 .cra_blocksize = AES_BLOCK_SIZE, 2498 .cra_ctxsize = sizeof(struct chcr_context) 2499 + sizeof(struct ablk_ctx), 2500 .cra_alignmask = 0, 2501 .cra_type = &crypto_ablkcipher_type, 2502 .cra_module = THIS_MODULE, 2503 .cra_init = chcr_cra_init, 2504 .cra_exit = NULL, 2505 .cra_u.ablkcipher = { 2506 .min_keysize = AES_MIN_KEY_SIZE, 2507 .max_keysize = AES_MAX_KEY_SIZE, 2508 .ivsize = AES_BLOCK_SIZE, 2509 .setkey = chcr_aes_cbc_setkey, 2510 .encrypt = chcr_aes_encrypt, 2511 .decrypt = chcr_aes_decrypt, 2512 } 2513 } 2514 }, 2515 { 2516 .type = CRYPTO_ALG_TYPE_ABLKCIPHER, 2517 .is_registered = 0, 2518 .alg.crypto = { 2519 .cra_name = "xts(aes)", 2520 .cra_driver_name = "xts-aes-chcr", 2521 .cra_priority = CHCR_CRA_PRIORITY, 2522 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER | 2523 CRYPTO_ALG_ASYNC, 2524 .cra_blocksize = AES_BLOCK_SIZE, 2525 .cra_ctxsize = sizeof(struct chcr_context) + 2526 sizeof(struct ablk_ctx), 2527 .cra_alignmask = 0, 2528 .cra_type = &crypto_ablkcipher_type, 2529 .cra_module = THIS_MODULE, 2530 .cra_init = chcr_cra_init, 2531 .cra_exit = NULL, 2532 .cra_u = { 2533 .ablkcipher = { 2534 .min_keysize = 2 * AES_MIN_KEY_SIZE, 2535 .max_keysize = 2 * AES_MAX_KEY_SIZE, 2536 .ivsize = AES_BLOCK_SIZE, 2537 .setkey = chcr_aes_xts_setkey, 2538 .encrypt = chcr_aes_encrypt, 2539 .decrypt = chcr_aes_decrypt, 2540 } 2541 } 2542 } 2543 }, 2544 /* SHA */ 2545 { 2546 .type = CRYPTO_ALG_TYPE_AHASH, 2547 .is_registered = 0, 2548 .alg.hash = { 2549 .halg.digestsize = SHA1_DIGEST_SIZE, 2550 .halg.base = { 2551 .cra_name = "sha1", 2552 .cra_driver_name = "sha1-chcr", 2553 .cra_blocksize = SHA1_BLOCK_SIZE, 2554 } 2555 } 2556 }, 2557 { 2558 .type = CRYPTO_ALG_TYPE_AHASH, 2559 .is_registered = 0, 2560 .alg.hash = { 2561 .halg.digestsize = SHA256_DIGEST_SIZE, 2562 .halg.base = { 2563 .cra_name = "sha256", 2564 .cra_driver_name = "sha256-chcr", 2565 .cra_blocksize = SHA256_BLOCK_SIZE, 2566 } 2567 } 2568 }, 2569 { 2570 .type = CRYPTO_ALG_TYPE_AHASH, 2571 .is_registered = 0, 2572 .alg.hash = { 2573 .halg.digestsize = SHA224_DIGEST_SIZE, 2574 .halg.base = { 2575 .cra_name = "sha224", 2576 .cra_driver_name = "sha224-chcr", 2577 .cra_blocksize = SHA224_BLOCK_SIZE, 2578 } 2579 } 2580 }, 2581 { 2582 .type = CRYPTO_ALG_TYPE_AHASH, 2583 .is_registered = 0, 2584 .alg.hash = { 2585 .halg.digestsize = SHA384_DIGEST_SIZE, 2586 .halg.base = { 2587 .cra_name = "sha384", 2588 .cra_driver_name = "sha384-chcr", 2589 .cra_blocksize = SHA384_BLOCK_SIZE, 2590 } 2591 } 2592 }, 2593 { 2594 .type = CRYPTO_ALG_TYPE_AHASH, 2595 .is_registered = 0, 2596 .alg.hash = { 2597 .halg.digestsize = SHA512_DIGEST_SIZE, 2598 .halg.base = { 2599 .cra_name = "sha512", 2600 .cra_driver_name = "sha512-chcr", 2601 .cra_blocksize = SHA512_BLOCK_SIZE, 2602 } 2603 } 2604 }, 2605 /* HMAC */ 2606 { 2607 .type = CRYPTO_ALG_TYPE_HMAC, 2608 .is_registered = 0, 2609 .alg.hash = { 2610 .halg.digestsize = SHA1_DIGEST_SIZE, 2611 .halg.base = { 2612 .cra_name = "hmac(sha1)", 2613 .cra_driver_name = "hmac-sha1-chcr", 2614 .cra_blocksize = SHA1_BLOCK_SIZE, 2615 } 2616 } 2617 }, 2618 { 2619 .type = CRYPTO_ALG_TYPE_HMAC, 2620 .is_registered = 0, 2621 .alg.hash = { 2622 .halg.digestsize = SHA224_DIGEST_SIZE, 2623 .halg.base = { 2624 .cra_name = "hmac(sha224)", 2625 .cra_driver_name = "hmac-sha224-chcr", 2626 .cra_blocksize = SHA224_BLOCK_SIZE, 2627 } 2628 } 2629 }, 2630 { 2631 .type = CRYPTO_ALG_TYPE_HMAC, 2632 .is_registered = 0, 2633 .alg.hash = { 2634 .halg.digestsize = SHA256_DIGEST_SIZE, 2635 .halg.base = { 2636 .cra_name = "hmac(sha256)", 2637 .cra_driver_name = "hmac-sha256-chcr", 2638 .cra_blocksize = SHA256_BLOCK_SIZE, 2639 } 2640 } 2641 }, 2642 { 2643 .type = CRYPTO_ALG_TYPE_HMAC, 2644 .is_registered = 0, 2645 .alg.hash = { 2646 .halg.digestsize = SHA384_DIGEST_SIZE, 2647 .halg.base = { 2648 .cra_name = "hmac(sha384)", 2649 .cra_driver_name = "hmac-sha384-chcr", 2650 .cra_blocksize = SHA384_BLOCK_SIZE, 2651 } 2652 } 2653 }, 2654 { 2655 .type = CRYPTO_ALG_TYPE_HMAC, 2656 .is_registered = 0, 2657 .alg.hash = { 2658 .halg.digestsize = SHA512_DIGEST_SIZE, 2659 .halg.base = { 2660 .cra_name = "hmac(sha512)", 2661 .cra_driver_name = "hmac-sha512-chcr", 2662 .cra_blocksize = SHA512_BLOCK_SIZE, 2663 } 2664 } 2665 }, 2666 /* Add AEAD Algorithms */ 2667 { 2668 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_AEAD_GCM, 2669 .is_registered = 0, 2670 .alg.aead = { 2671 .base = { 2672 .cra_name = "gcm(aes)", 2673 .cra_driver_name = "gcm-aes-chcr", 2674 .cra_blocksize = 1, 2675 .cra_ctxsize = sizeof(struct chcr_context) + 2676 sizeof(struct chcr_aead_ctx) + 2677 sizeof(struct chcr_gcm_ctx), 2678 }, 2679 .ivsize = 12, 2680 .maxauthsize = GHASH_DIGEST_SIZE, 2681 .setkey = chcr_gcm_setkey, 2682 .setauthsize = chcr_gcm_setauthsize, 2683 } 2684 }, 2685 { 2686 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_AEAD_RFC4106, 2687 .is_registered = 0, 2688 .alg.aead = { 2689 .base = { 2690 .cra_name = "rfc4106(gcm(aes))", 2691 .cra_driver_name = "rfc4106-gcm-aes-chcr", 2692 .cra_blocksize = 1, 2693 .cra_ctxsize = sizeof(struct chcr_context) + 2694 sizeof(struct chcr_aead_ctx) + 2695 sizeof(struct chcr_gcm_ctx), 2696 2697 }, 2698 .ivsize = 8, 2699 .maxauthsize = GHASH_DIGEST_SIZE, 2700 .setkey = chcr_gcm_setkey, 2701 .setauthsize = chcr_4106_4309_setauthsize, 2702 } 2703 }, 2704 { 2705 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_AEAD_CCM, 2706 .is_registered = 0, 2707 .alg.aead = { 2708 .base = { 2709 .cra_name = "ccm(aes)", 2710 .cra_driver_name = "ccm-aes-chcr", 2711 .cra_blocksize = 1, 2712 .cra_ctxsize = sizeof(struct chcr_context) + 2713 sizeof(struct chcr_aead_ctx), 2714 2715 }, 2716 .ivsize = AES_BLOCK_SIZE, 2717 .maxauthsize = GHASH_DIGEST_SIZE, 2718 .setkey = chcr_aead_ccm_setkey, 2719 .setauthsize = chcr_ccm_setauthsize, 2720 } 2721 }, 2722 { 2723 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_AEAD_RFC4309, 2724 .is_registered = 0, 2725 .alg.aead = { 2726 .base = { 2727 .cra_name = "rfc4309(ccm(aes))", 2728 .cra_driver_name = "rfc4309-ccm-aes-chcr", 2729 .cra_blocksize = 1, 2730 .cra_ctxsize = sizeof(struct chcr_context) + 2731 sizeof(struct chcr_aead_ctx), 2732 2733 }, 2734 .ivsize = 8, 2735 .maxauthsize = GHASH_DIGEST_SIZE, 2736 .setkey = chcr_aead_rfc4309_setkey, 2737 .setauthsize = chcr_4106_4309_setauthsize, 2738 } 2739 }, 2740 { 2741 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_AEAD_AUTHENC, 2742 .is_registered = 0, 2743 .alg.aead = { 2744 .base = { 2745 .cra_name = "authenc(hmac(sha1),cbc(aes))", 2746 .cra_driver_name = 2747 "authenc-hmac-sha1-cbc-aes-chcr", 2748 .cra_blocksize = AES_BLOCK_SIZE, 2749 .cra_ctxsize = sizeof(struct chcr_context) + 2750 sizeof(struct chcr_aead_ctx) + 2751 sizeof(struct chcr_authenc_ctx), 2752 2753 }, 2754 .ivsize = AES_BLOCK_SIZE, 2755 .maxauthsize = SHA1_DIGEST_SIZE, 2756 .setkey = chcr_authenc_setkey, 2757 .setauthsize = chcr_authenc_setauthsize, 2758 } 2759 }, 2760 { 2761 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_AEAD_AUTHENC, 2762 .is_registered = 0, 2763 .alg.aead = { 2764 .base = { 2765 2766 .cra_name = "authenc(hmac(sha256),cbc(aes))", 2767 .cra_driver_name = 2768 "authenc-hmac-sha256-cbc-aes-chcr", 2769 .cra_blocksize = AES_BLOCK_SIZE, 2770 .cra_ctxsize = sizeof(struct chcr_context) + 2771 sizeof(struct chcr_aead_ctx) + 2772 sizeof(struct chcr_authenc_ctx), 2773 2774 }, 2775 .ivsize = AES_BLOCK_SIZE, 2776 .maxauthsize = SHA256_DIGEST_SIZE, 2777 .setkey = chcr_authenc_setkey, 2778 .setauthsize = chcr_authenc_setauthsize, 2779 } 2780 }, 2781 { 2782 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_AEAD_AUTHENC, 2783 .is_registered = 0, 2784 .alg.aead = { 2785 .base = { 2786 .cra_name = "authenc(hmac(sha224),cbc(aes))", 2787 .cra_driver_name = 2788 "authenc-hmac-sha224-cbc-aes-chcr", 2789 .cra_blocksize = AES_BLOCK_SIZE, 2790 .cra_ctxsize = sizeof(struct chcr_context) + 2791 sizeof(struct chcr_aead_ctx) + 2792 sizeof(struct chcr_authenc_ctx), 2793 }, 2794 .ivsize = AES_BLOCK_SIZE, 2795 .maxauthsize = SHA224_DIGEST_SIZE, 2796 .setkey = chcr_authenc_setkey, 2797 .setauthsize = chcr_authenc_setauthsize, 2798 } 2799 }, 2800 { 2801 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_AEAD_AUTHENC, 2802 .is_registered = 0, 2803 .alg.aead = { 2804 .base = { 2805 .cra_name = "authenc(hmac(sha384),cbc(aes))", 2806 .cra_driver_name = 2807 "authenc-hmac-sha384-cbc-aes-chcr", 2808 .cra_blocksize = AES_BLOCK_SIZE, 2809 .cra_ctxsize = sizeof(struct chcr_context) + 2810 sizeof(struct chcr_aead_ctx) + 2811 sizeof(struct chcr_authenc_ctx), 2812 2813 }, 2814 .ivsize = AES_BLOCK_SIZE, 2815 .maxauthsize = SHA384_DIGEST_SIZE, 2816 .setkey = chcr_authenc_setkey, 2817 .setauthsize = chcr_authenc_setauthsize, 2818 } 2819 }, 2820 { 2821 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_AEAD_AUTHENC, 2822 .is_registered = 0, 2823 .alg.aead = { 2824 .base = { 2825 .cra_name = "authenc(hmac(sha512),cbc(aes))", 2826 .cra_driver_name = 2827 "authenc-hmac-sha512-cbc-aes-chcr", 2828 .cra_blocksize = AES_BLOCK_SIZE, 2829 .cra_ctxsize = sizeof(struct chcr_context) + 2830 sizeof(struct chcr_aead_ctx) + 2831 sizeof(struct chcr_authenc_ctx), 2832 2833 }, 2834 .ivsize = AES_BLOCK_SIZE, 2835 .maxauthsize = SHA512_DIGEST_SIZE, 2836 .setkey = chcr_authenc_setkey, 2837 .setauthsize = chcr_authenc_setauthsize, 2838 } 2839 }, 2840 { 2841 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_AEAD_NULL, 2842 .is_registered = 0, 2843 .alg.aead = { 2844 .base = { 2845 .cra_name = "authenc(digest_null,cbc(aes))", 2846 .cra_driver_name = 2847 "authenc-digest_null-cbc-aes-chcr", 2848 .cra_blocksize = AES_BLOCK_SIZE, 2849 .cra_ctxsize = sizeof(struct chcr_context) + 2850 sizeof(struct chcr_aead_ctx) + 2851 sizeof(struct chcr_authenc_ctx), 2852 2853 }, 2854 .ivsize = AES_BLOCK_SIZE, 2855 .maxauthsize = 0, 2856 .setkey = chcr_aead_digest_null_setkey, 2857 .setauthsize = chcr_authenc_null_setauthsize, 2858 } 2859 }, 2860 }; 2861 2862 /* 2863 * chcr_unregister_alg - Deregister crypto algorithms with 2864 * kernel framework. 2865 */ 2866 static int chcr_unregister_alg(void) 2867 { 2868 int i; 2869 2870 for (i = 0; i < ARRAY_SIZE(driver_algs); i++) { 2871 switch (driver_algs[i].type & CRYPTO_ALG_TYPE_MASK) { 2872 case CRYPTO_ALG_TYPE_ABLKCIPHER: 2873 if (driver_algs[i].is_registered) 2874 crypto_unregister_alg( 2875 &driver_algs[i].alg.crypto); 2876 break; 2877 case CRYPTO_ALG_TYPE_AEAD: 2878 if (driver_algs[i].is_registered) 2879 crypto_unregister_aead( 2880 &driver_algs[i].alg.aead); 2881 break; 2882 case CRYPTO_ALG_TYPE_AHASH: 2883 if (driver_algs[i].is_registered) 2884 crypto_unregister_ahash( 2885 &driver_algs[i].alg.hash); 2886 break; 2887 } 2888 driver_algs[i].is_registered = 0; 2889 } 2890 return 0; 2891 } 2892 2893 #define SZ_AHASH_CTX sizeof(struct chcr_context) 2894 #define SZ_AHASH_H_CTX (sizeof(struct chcr_context) + sizeof(struct hmac_ctx)) 2895 #define SZ_AHASH_REQ_CTX sizeof(struct chcr_ahash_req_ctx) 2896 #define AHASH_CRA_FLAGS (CRYPTO_ALG_TYPE_AHASH | CRYPTO_ALG_ASYNC) 2897 2898 /* 2899 * chcr_register_alg - Register crypto algorithms with kernel framework. 2900 */ 2901 static int chcr_register_alg(void) 2902 { 2903 struct crypto_alg ai; 2904 struct ahash_alg *a_hash; 2905 int err = 0, i; 2906 char *name = NULL; 2907 2908 for (i = 0; i < ARRAY_SIZE(driver_algs); i++) { 2909 if (driver_algs[i].is_registered) 2910 continue; 2911 switch (driver_algs[i].type & CRYPTO_ALG_TYPE_MASK) { 2912 case CRYPTO_ALG_TYPE_ABLKCIPHER: 2913 err = crypto_register_alg(&driver_algs[i].alg.crypto); 2914 name = driver_algs[i].alg.crypto.cra_driver_name; 2915 break; 2916 case CRYPTO_ALG_TYPE_AEAD: 2917 driver_algs[i].alg.aead.base.cra_priority = 2918 CHCR_CRA_PRIORITY; 2919 driver_algs[i].alg.aead.base.cra_flags = 2920 CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC; 2921 driver_algs[i].alg.aead.encrypt = chcr_aead_encrypt; 2922 driver_algs[i].alg.aead.decrypt = chcr_aead_decrypt; 2923 driver_algs[i].alg.aead.init = chcr_aead_cra_init; 2924 driver_algs[i].alg.aead.exit = chcr_aead_cra_exit; 2925 driver_algs[i].alg.aead.base.cra_module = THIS_MODULE; 2926 err = crypto_register_aead(&driver_algs[i].alg.aead); 2927 name = driver_algs[i].alg.aead.base.cra_driver_name; 2928 break; 2929 case CRYPTO_ALG_TYPE_AHASH: 2930 a_hash = &driver_algs[i].alg.hash; 2931 a_hash->update = chcr_ahash_update; 2932 a_hash->final = chcr_ahash_final; 2933 a_hash->finup = chcr_ahash_finup; 2934 a_hash->digest = chcr_ahash_digest; 2935 a_hash->export = chcr_ahash_export; 2936 a_hash->import = chcr_ahash_import; 2937 a_hash->halg.statesize = SZ_AHASH_REQ_CTX; 2938 a_hash->halg.base.cra_priority = CHCR_CRA_PRIORITY; 2939 a_hash->halg.base.cra_module = THIS_MODULE; 2940 a_hash->halg.base.cra_flags = AHASH_CRA_FLAGS; 2941 a_hash->halg.base.cra_alignmask = 0; 2942 a_hash->halg.base.cra_exit = NULL; 2943 a_hash->halg.base.cra_type = &crypto_ahash_type; 2944 2945 if (driver_algs[i].type == CRYPTO_ALG_TYPE_HMAC) { 2946 a_hash->halg.base.cra_init = chcr_hmac_cra_init; 2947 a_hash->halg.base.cra_exit = chcr_hmac_cra_exit; 2948 a_hash->init = chcr_hmac_init; 2949 a_hash->setkey = chcr_ahash_setkey; 2950 a_hash->halg.base.cra_ctxsize = SZ_AHASH_H_CTX; 2951 } else { 2952 a_hash->init = chcr_sha_init; 2953 a_hash->halg.base.cra_ctxsize = SZ_AHASH_CTX; 2954 a_hash->halg.base.cra_init = chcr_sha_cra_init; 2955 } 2956 err = crypto_register_ahash(&driver_algs[i].alg.hash); 2957 ai = driver_algs[i].alg.hash.halg.base; 2958 name = ai.cra_driver_name; 2959 break; 2960 } 2961 if (err) { 2962 pr_err("chcr : %s : Algorithm registration failed\n", 2963 name); 2964 goto register_err; 2965 } else { 2966 driver_algs[i].is_registered = 1; 2967 } 2968 } 2969 return 0; 2970 2971 register_err: 2972 chcr_unregister_alg(); 2973 return err; 2974 } 2975 2976 /* 2977 * start_crypto - Register the crypto algorithms. 2978 * This should called once when the first device comesup. After this 2979 * kernel will start calling driver APIs for crypto operations. 2980 */ 2981 int start_crypto(void) 2982 { 2983 return chcr_register_alg(); 2984 } 2985 2986 /* 2987 * stop_crypto - Deregister all the crypto algorithms with kernel. 2988 * This should be called once when the last device goes down. After this 2989 * kernel will not call the driver API for crypto operations. 2990 */ 2991 int stop_crypto(void) 2992 { 2993 chcr_unregister_alg(); 2994 return 0; 2995 } 2996