1 /* 2 * This file is part of the Chelsio T6 Crypto driver for Linux. 3 * 4 * Copyright (c) 2003-2016 Chelsio Communications, Inc. All rights reserved. 5 * 6 * This software is available to you under a choice of one of two 7 * licenses. You may choose to be licensed under the terms of the GNU 8 * General Public License (GPL) Version 2, available from the file 9 * COPYING in the main directory of this source tree, or the 10 * OpenIB.org BSD license below: 11 * 12 * Redistribution and use in source and binary forms, with or 13 * without modification, are permitted provided that the following 14 * conditions are met: 15 * 16 * - Redistributions of source code must retain the above 17 * copyright notice, this list of conditions and the following 18 * disclaimer. 19 * 20 * - Redistributions in binary form must reproduce the above 21 * copyright notice, this list of conditions and the following 22 * disclaimer in the documentation and/or other materials 23 * provided with the distribution. 24 * 25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 26 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 27 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 28 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 29 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 30 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 31 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 32 * SOFTWARE. 33 * 34 * Written and Maintained by: 35 * Manoj Malviya (manojmalviya@chelsio.com) 36 * Atul Gupta (atul.gupta@chelsio.com) 37 * Jitendra Lulla (jlulla@chelsio.com) 38 * Yeshaswi M R Gowda (yeshaswi@chelsio.com) 39 * Harsh Jain (harsh@chelsio.com) 40 */ 41 42 #define pr_fmt(fmt) "chcr:" fmt 43 44 #include <linux/kernel.h> 45 #include <linux/module.h> 46 #include <linux/crypto.h> 47 #include <linux/cryptohash.h> 48 #include <linux/skbuff.h> 49 #include <linux/rtnetlink.h> 50 #include <linux/highmem.h> 51 #include <linux/scatterlist.h> 52 53 #include <crypto/aes.h> 54 #include <crypto/algapi.h> 55 #include <crypto/hash.h> 56 #include <crypto/sha.h> 57 #include <crypto/authenc.h> 58 #include <crypto/internal/aead.h> 59 #include <crypto/null.h> 60 #include <crypto/internal/skcipher.h> 61 #include <crypto/aead.h> 62 #include <crypto/scatterwalk.h> 63 #include <crypto/internal/hash.h> 64 65 #include "t4fw_api.h" 66 #include "t4_msg.h" 67 #include "chcr_core.h" 68 #include "chcr_algo.h" 69 #include "chcr_crypto.h" 70 71 static inline struct chcr_aead_ctx *AEAD_CTX(struct chcr_context *ctx) 72 { 73 return ctx->crypto_ctx->aeadctx; 74 } 75 76 static inline struct ablk_ctx *ABLK_CTX(struct chcr_context *ctx) 77 { 78 return ctx->crypto_ctx->ablkctx; 79 } 80 81 static inline struct hmac_ctx *HMAC_CTX(struct chcr_context *ctx) 82 { 83 return ctx->crypto_ctx->hmacctx; 84 } 85 86 static inline struct chcr_gcm_ctx *GCM_CTX(struct chcr_aead_ctx *gctx) 87 { 88 return gctx->ctx->gcm; 89 } 90 91 static inline struct chcr_authenc_ctx *AUTHENC_CTX(struct chcr_aead_ctx *gctx) 92 { 93 return gctx->ctx->authenc; 94 } 95 96 static inline struct uld_ctx *ULD_CTX(struct chcr_context *ctx) 97 { 98 return ctx->dev->u_ctx; 99 } 100 101 static inline int is_ofld_imm(const struct sk_buff *skb) 102 { 103 return (skb->len <= CRYPTO_MAX_IMM_TX_PKT_LEN); 104 } 105 106 /* 107 * sgl_len - calculates the size of an SGL of the given capacity 108 * @n: the number of SGL entries 109 * Calculates the number of flits needed for a scatter/gather list that 110 * can hold the given number of entries. 111 */ 112 static inline unsigned int sgl_len(unsigned int n) 113 { 114 n--; 115 return (3 * n) / 2 + (n & 1) + 2; 116 } 117 118 static void chcr_verify_tag(struct aead_request *req, u8 *input, int *err) 119 { 120 u8 temp[SHA512_DIGEST_SIZE]; 121 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 122 int authsize = crypto_aead_authsize(tfm); 123 struct cpl_fw6_pld *fw6_pld; 124 int cmp = 0; 125 126 fw6_pld = (struct cpl_fw6_pld *)input; 127 if ((get_aead_subtype(tfm) == CRYPTO_ALG_SUB_TYPE_AEAD_RFC4106) || 128 (get_aead_subtype(tfm) == CRYPTO_ALG_SUB_TYPE_AEAD_GCM)) { 129 cmp = memcmp(&fw6_pld->data[2], (fw6_pld + 1), authsize); 130 } else { 131 132 sg_pcopy_to_buffer(req->src, sg_nents(req->src), temp, 133 authsize, req->assoclen + 134 req->cryptlen - authsize); 135 cmp = memcmp(temp, (fw6_pld + 1), authsize); 136 } 137 if (cmp) 138 *err = -EBADMSG; 139 else 140 *err = 0; 141 } 142 143 /* 144 * chcr_handle_resp - Unmap the DMA buffers associated with the request 145 * @req: crypto request 146 */ 147 int chcr_handle_resp(struct crypto_async_request *req, unsigned char *input, 148 int err) 149 { 150 struct crypto_tfm *tfm = req->tfm; 151 struct chcr_context *ctx = crypto_tfm_ctx(tfm); 152 struct uld_ctx *u_ctx = ULD_CTX(ctx); 153 struct chcr_req_ctx ctx_req; 154 struct cpl_fw6_pld *fw6_pld; 155 unsigned int digestsize, updated_digestsize; 156 157 switch (tfm->__crt_alg->cra_flags & CRYPTO_ALG_TYPE_MASK) { 158 case CRYPTO_ALG_TYPE_AEAD: 159 ctx_req.req.aead_req = (struct aead_request *)req; 160 ctx_req.ctx.reqctx = aead_request_ctx(ctx_req.req.aead_req); 161 dma_unmap_sg(&u_ctx->lldi.pdev->dev, ctx_req.req.aead_req->dst, 162 ctx_req.ctx.reqctx->dst_nents, DMA_FROM_DEVICE); 163 if (ctx_req.ctx.reqctx->skb) { 164 kfree_skb(ctx_req.ctx.reqctx->skb); 165 ctx_req.ctx.reqctx->skb = NULL; 166 } 167 if (ctx_req.ctx.reqctx->verify == VERIFY_SW) { 168 chcr_verify_tag(ctx_req.req.aead_req, input, 169 &err); 170 ctx_req.ctx.reqctx->verify = VERIFY_HW; 171 } 172 break; 173 174 case CRYPTO_ALG_TYPE_BLKCIPHER: 175 ctx_req.req.ablk_req = (struct ablkcipher_request *)req; 176 ctx_req.ctx.ablk_ctx = 177 ablkcipher_request_ctx(ctx_req.req.ablk_req); 178 if (!err) { 179 fw6_pld = (struct cpl_fw6_pld *)input; 180 memcpy(ctx_req.req.ablk_req->info, &fw6_pld->data[2], 181 AES_BLOCK_SIZE); 182 } 183 dma_unmap_sg(&u_ctx->lldi.pdev->dev, ctx_req.req.ablk_req->dst, 184 ctx_req.ctx.ablk_ctx->dst_nents, DMA_FROM_DEVICE); 185 if (ctx_req.ctx.ablk_ctx->skb) { 186 kfree_skb(ctx_req.ctx.ablk_ctx->skb); 187 ctx_req.ctx.ablk_ctx->skb = NULL; 188 } 189 break; 190 191 case CRYPTO_ALG_TYPE_AHASH: 192 ctx_req.req.ahash_req = (struct ahash_request *)req; 193 ctx_req.ctx.ahash_ctx = 194 ahash_request_ctx(ctx_req.req.ahash_req); 195 digestsize = 196 crypto_ahash_digestsize(crypto_ahash_reqtfm( 197 ctx_req.req.ahash_req)); 198 updated_digestsize = digestsize; 199 if (digestsize == SHA224_DIGEST_SIZE) 200 updated_digestsize = SHA256_DIGEST_SIZE; 201 else if (digestsize == SHA384_DIGEST_SIZE) 202 updated_digestsize = SHA512_DIGEST_SIZE; 203 if (ctx_req.ctx.ahash_ctx->skb) { 204 kfree_skb(ctx_req.ctx.ahash_ctx->skb); 205 ctx_req.ctx.ahash_ctx->skb = NULL; 206 } 207 if (ctx_req.ctx.ahash_ctx->result == 1) { 208 ctx_req.ctx.ahash_ctx->result = 0; 209 memcpy(ctx_req.req.ahash_req->result, input + 210 sizeof(struct cpl_fw6_pld), 211 digestsize); 212 } else { 213 memcpy(ctx_req.ctx.ahash_ctx->partial_hash, input + 214 sizeof(struct cpl_fw6_pld), 215 updated_digestsize); 216 } 217 break; 218 } 219 return err; 220 } 221 222 /* 223 * calc_tx_flits_ofld - calculate # of flits for an offload packet 224 * @skb: the packet 225 * Returns the number of flits needed for the given offload packet. 226 * These packets are already fully constructed and no additional headers 227 * will be added. 228 */ 229 static inline unsigned int calc_tx_flits_ofld(const struct sk_buff *skb) 230 { 231 unsigned int flits, cnt; 232 233 if (is_ofld_imm(skb)) 234 return DIV_ROUND_UP(skb->len, 8); 235 236 flits = skb_transport_offset(skb) / 8; /* headers */ 237 cnt = skb_shinfo(skb)->nr_frags; 238 if (skb_tail_pointer(skb) != skb_transport_header(skb)) 239 cnt++; 240 return flits + sgl_len(cnt); 241 } 242 243 static inline void get_aes_decrypt_key(unsigned char *dec_key, 244 const unsigned char *key, 245 unsigned int keylength) 246 { 247 u32 temp; 248 u32 w_ring[MAX_NK]; 249 int i, j, k; 250 u8 nr, nk; 251 252 switch (keylength) { 253 case AES_KEYLENGTH_128BIT: 254 nk = KEYLENGTH_4BYTES; 255 nr = NUMBER_OF_ROUNDS_10; 256 break; 257 case AES_KEYLENGTH_192BIT: 258 nk = KEYLENGTH_6BYTES; 259 nr = NUMBER_OF_ROUNDS_12; 260 break; 261 case AES_KEYLENGTH_256BIT: 262 nk = KEYLENGTH_8BYTES; 263 nr = NUMBER_OF_ROUNDS_14; 264 break; 265 default: 266 return; 267 } 268 for (i = 0; i < nk; i++) 269 w_ring[i] = be32_to_cpu(*(u32 *)&key[4 * i]); 270 271 i = 0; 272 temp = w_ring[nk - 1]; 273 while (i + nk < (nr + 1) * 4) { 274 if (!(i % nk)) { 275 /* RotWord(temp) */ 276 temp = (temp << 8) | (temp >> 24); 277 temp = aes_ks_subword(temp); 278 temp ^= round_constant[i / nk]; 279 } else if (nk == 8 && (i % 4 == 0)) { 280 temp = aes_ks_subword(temp); 281 } 282 w_ring[i % nk] ^= temp; 283 temp = w_ring[i % nk]; 284 i++; 285 } 286 i--; 287 for (k = 0, j = i % nk; k < nk; k++) { 288 *((u32 *)dec_key + k) = htonl(w_ring[j]); 289 j--; 290 if (j < 0) 291 j += nk; 292 } 293 } 294 295 static struct crypto_shash *chcr_alloc_shash(unsigned int ds) 296 { 297 struct crypto_shash *base_hash = NULL; 298 299 switch (ds) { 300 case SHA1_DIGEST_SIZE: 301 base_hash = crypto_alloc_shash("sha1", 0, 0); 302 break; 303 case SHA224_DIGEST_SIZE: 304 base_hash = crypto_alloc_shash("sha224", 0, 0); 305 break; 306 case SHA256_DIGEST_SIZE: 307 base_hash = crypto_alloc_shash("sha256", 0, 0); 308 break; 309 case SHA384_DIGEST_SIZE: 310 base_hash = crypto_alloc_shash("sha384", 0, 0); 311 break; 312 case SHA512_DIGEST_SIZE: 313 base_hash = crypto_alloc_shash("sha512", 0, 0); 314 break; 315 } 316 317 return base_hash; 318 } 319 320 static int chcr_compute_partial_hash(struct shash_desc *desc, 321 char *iopad, char *result_hash, 322 int digest_size) 323 { 324 struct sha1_state sha1_st; 325 struct sha256_state sha256_st; 326 struct sha512_state sha512_st; 327 int error; 328 329 if (digest_size == SHA1_DIGEST_SIZE) { 330 error = crypto_shash_init(desc) ?: 331 crypto_shash_update(desc, iopad, SHA1_BLOCK_SIZE) ?: 332 crypto_shash_export(desc, (void *)&sha1_st); 333 memcpy(result_hash, sha1_st.state, SHA1_DIGEST_SIZE); 334 } else if (digest_size == SHA224_DIGEST_SIZE) { 335 error = crypto_shash_init(desc) ?: 336 crypto_shash_update(desc, iopad, SHA256_BLOCK_SIZE) ?: 337 crypto_shash_export(desc, (void *)&sha256_st); 338 memcpy(result_hash, sha256_st.state, SHA256_DIGEST_SIZE); 339 340 } else if (digest_size == SHA256_DIGEST_SIZE) { 341 error = crypto_shash_init(desc) ?: 342 crypto_shash_update(desc, iopad, SHA256_BLOCK_SIZE) ?: 343 crypto_shash_export(desc, (void *)&sha256_st); 344 memcpy(result_hash, sha256_st.state, SHA256_DIGEST_SIZE); 345 346 } else if (digest_size == SHA384_DIGEST_SIZE) { 347 error = crypto_shash_init(desc) ?: 348 crypto_shash_update(desc, iopad, SHA512_BLOCK_SIZE) ?: 349 crypto_shash_export(desc, (void *)&sha512_st); 350 memcpy(result_hash, sha512_st.state, SHA512_DIGEST_SIZE); 351 352 } else if (digest_size == SHA512_DIGEST_SIZE) { 353 error = crypto_shash_init(desc) ?: 354 crypto_shash_update(desc, iopad, SHA512_BLOCK_SIZE) ?: 355 crypto_shash_export(desc, (void *)&sha512_st); 356 memcpy(result_hash, sha512_st.state, SHA512_DIGEST_SIZE); 357 } else { 358 error = -EINVAL; 359 pr_err("Unknown digest size %d\n", digest_size); 360 } 361 return error; 362 } 363 364 static void chcr_change_order(char *buf, int ds) 365 { 366 int i; 367 368 if (ds == SHA512_DIGEST_SIZE) { 369 for (i = 0; i < (ds / sizeof(u64)); i++) 370 *((__be64 *)buf + i) = 371 cpu_to_be64(*((u64 *)buf + i)); 372 } else { 373 for (i = 0; i < (ds / sizeof(u32)); i++) 374 *((__be32 *)buf + i) = 375 cpu_to_be32(*((u32 *)buf + i)); 376 } 377 } 378 379 static inline int is_hmac(struct crypto_tfm *tfm) 380 { 381 struct crypto_alg *alg = tfm->__crt_alg; 382 struct chcr_alg_template *chcr_crypto_alg = 383 container_of(__crypto_ahash_alg(alg), struct chcr_alg_template, 384 alg.hash); 385 if (chcr_crypto_alg->type == CRYPTO_ALG_TYPE_HMAC) 386 return 1; 387 return 0; 388 } 389 390 static void write_phys_cpl(struct cpl_rx_phys_dsgl *phys_cpl, 391 struct scatterlist *sg, 392 struct phys_sge_parm *sg_param) 393 { 394 struct phys_sge_pairs *to; 395 int out_buf_size = sg_param->obsize; 396 unsigned int nents = sg_param->nents, i, j = 0; 397 398 phys_cpl->op_to_tid = htonl(CPL_RX_PHYS_DSGL_OPCODE_V(CPL_RX_PHYS_DSGL) 399 | CPL_RX_PHYS_DSGL_ISRDMA_V(0)); 400 phys_cpl->pcirlxorder_to_noofsgentr = 401 htonl(CPL_RX_PHYS_DSGL_PCIRLXORDER_V(0) | 402 CPL_RX_PHYS_DSGL_PCINOSNOOP_V(0) | 403 CPL_RX_PHYS_DSGL_PCITPHNTENB_V(0) | 404 CPL_RX_PHYS_DSGL_PCITPHNT_V(0) | 405 CPL_RX_PHYS_DSGL_DCAID_V(0) | 406 CPL_RX_PHYS_DSGL_NOOFSGENTR_V(nents)); 407 phys_cpl->rss_hdr_int.opcode = CPL_RX_PHYS_ADDR; 408 phys_cpl->rss_hdr_int.qid = htons(sg_param->qid); 409 phys_cpl->rss_hdr_int.hash_val = 0; 410 to = (struct phys_sge_pairs *)((unsigned char *)phys_cpl + 411 sizeof(struct cpl_rx_phys_dsgl)); 412 413 for (i = 0; nents; to++) { 414 for (j = 0; j < 8 && nents; j++, nents--) { 415 out_buf_size -= sg_dma_len(sg); 416 to->len[j] = htons(sg_dma_len(sg)); 417 to->addr[j] = cpu_to_be64(sg_dma_address(sg)); 418 sg = sg_next(sg); 419 } 420 } 421 if (out_buf_size) { 422 j--; 423 to--; 424 to->len[j] = htons(ntohs(to->len[j]) + (out_buf_size)); 425 } 426 } 427 428 static inline int map_writesg_phys_cpl(struct device *dev, 429 struct cpl_rx_phys_dsgl *phys_cpl, 430 struct scatterlist *sg, 431 struct phys_sge_parm *sg_param) 432 { 433 if (!sg || !sg_param->nents) 434 return 0; 435 436 sg_param->nents = dma_map_sg(dev, sg, sg_param->nents, DMA_FROM_DEVICE); 437 if (sg_param->nents == 0) { 438 pr_err("CHCR : DMA mapping failed\n"); 439 return -EINVAL; 440 } 441 write_phys_cpl(phys_cpl, sg, sg_param); 442 return 0; 443 } 444 445 static inline int get_aead_subtype(struct crypto_aead *aead) 446 { 447 struct aead_alg *alg = crypto_aead_alg(aead); 448 struct chcr_alg_template *chcr_crypto_alg = 449 container_of(alg, struct chcr_alg_template, alg.aead); 450 return chcr_crypto_alg->type & CRYPTO_ALG_SUB_TYPE_MASK; 451 } 452 453 static inline int get_cryptoalg_subtype(struct crypto_tfm *tfm) 454 { 455 struct crypto_alg *alg = tfm->__crt_alg; 456 struct chcr_alg_template *chcr_crypto_alg = 457 container_of(alg, struct chcr_alg_template, alg.crypto); 458 459 return chcr_crypto_alg->type & CRYPTO_ALG_SUB_TYPE_MASK; 460 } 461 462 static inline void write_buffer_to_skb(struct sk_buff *skb, 463 unsigned int *frags, 464 char *bfr, 465 u8 bfr_len) 466 { 467 skb->len += bfr_len; 468 skb->data_len += bfr_len; 469 skb->truesize += bfr_len; 470 get_page(virt_to_page(bfr)); 471 skb_fill_page_desc(skb, *frags, virt_to_page(bfr), 472 offset_in_page(bfr), bfr_len); 473 (*frags)++; 474 } 475 476 477 static inline void 478 write_sg_to_skb(struct sk_buff *skb, unsigned int *frags, 479 struct scatterlist *sg, unsigned int count) 480 { 481 struct page *spage; 482 unsigned int page_len; 483 484 skb->len += count; 485 skb->data_len += count; 486 skb->truesize += count; 487 488 while (count > 0) { 489 if (!sg || (!(sg->length))) 490 break; 491 spage = sg_page(sg); 492 get_page(spage); 493 page_len = min(sg->length, count); 494 skb_fill_page_desc(skb, *frags, spage, sg->offset, page_len); 495 (*frags)++; 496 count -= page_len; 497 sg = sg_next(sg); 498 } 499 } 500 501 static int generate_copy_rrkey(struct ablk_ctx *ablkctx, 502 struct _key_ctx *key_ctx) 503 { 504 if (ablkctx->ciph_mode == CHCR_SCMD_CIPHER_MODE_AES_CBC) { 505 memcpy(key_ctx->key, ablkctx->rrkey, ablkctx->enckey_len); 506 } else { 507 memcpy(key_ctx->key, 508 ablkctx->key + (ablkctx->enckey_len >> 1), 509 ablkctx->enckey_len >> 1); 510 memcpy(key_ctx->key + (ablkctx->enckey_len >> 1), 511 ablkctx->rrkey, ablkctx->enckey_len >> 1); 512 } 513 return 0; 514 } 515 516 static inline void create_wreq(struct chcr_context *ctx, 517 struct chcr_wr *chcr_req, 518 void *req, struct sk_buff *skb, 519 int kctx_len, int hash_sz, 520 int is_iv, 521 unsigned int sc_len) 522 { 523 struct uld_ctx *u_ctx = ULD_CTX(ctx); 524 int iv_loc = IV_DSGL; 525 int qid = u_ctx->lldi.rxq_ids[ctx->tx_channel_id]; 526 unsigned int immdatalen = 0, nr_frags = 0; 527 528 if (is_ofld_imm(skb)) { 529 immdatalen = skb->data_len; 530 iv_loc = IV_IMMEDIATE; 531 } else { 532 nr_frags = skb_shinfo(skb)->nr_frags; 533 } 534 535 chcr_req->wreq.op_to_cctx_size = FILL_WR_OP_CCTX_SIZE(immdatalen, 536 ((sizeof(chcr_req->key_ctx) + kctx_len) >> 4)); 537 chcr_req->wreq.pld_size_hash_size = 538 htonl(FW_CRYPTO_LOOKASIDE_WR_PLD_SIZE_V(sgl_lengths[nr_frags]) | 539 FW_CRYPTO_LOOKASIDE_WR_HASH_SIZE_V(hash_sz)); 540 chcr_req->wreq.len16_pkd = 541 htonl(FW_CRYPTO_LOOKASIDE_WR_LEN16_V(DIV_ROUND_UP( 542 (calc_tx_flits_ofld(skb) * 8), 16))); 543 chcr_req->wreq.cookie = cpu_to_be64((uintptr_t)req); 544 chcr_req->wreq.rx_chid_to_rx_q_id = 545 FILL_WR_RX_Q_ID(ctx->dev->tx_channel_id, qid, 546 is_iv ? iv_loc : IV_NOP); 547 548 chcr_req->ulptx.cmd_dest = FILL_ULPTX_CMD_DEST(ctx->dev->tx_channel_id); 549 chcr_req->ulptx.len = htonl((DIV_ROUND_UP((calc_tx_flits_ofld(skb) * 8), 550 16) - ((sizeof(chcr_req->wreq)) >> 4))); 551 552 chcr_req->sc_imm.cmd_more = FILL_CMD_MORE(immdatalen); 553 chcr_req->sc_imm.len = cpu_to_be32(sizeof(struct cpl_tx_sec_pdu) + 554 sizeof(chcr_req->key_ctx) + 555 kctx_len + sc_len + immdatalen); 556 } 557 558 /** 559 * create_cipher_wr - form the WR for cipher operations 560 * @req: cipher req. 561 * @ctx: crypto driver context of the request. 562 * @qid: ingress qid where response of this WR should be received. 563 * @op_type: encryption or decryption 564 */ 565 static struct sk_buff 566 *create_cipher_wr(struct ablkcipher_request *req, 567 unsigned short qid, 568 unsigned short op_type) 569 { 570 struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req); 571 struct chcr_context *ctx = crypto_ablkcipher_ctx(tfm); 572 struct uld_ctx *u_ctx = ULD_CTX(ctx); 573 struct ablk_ctx *ablkctx = ABLK_CTX(ctx); 574 struct sk_buff *skb = NULL; 575 struct chcr_wr *chcr_req; 576 struct cpl_rx_phys_dsgl *phys_cpl; 577 struct chcr_blkcipher_req_ctx *reqctx = ablkcipher_request_ctx(req); 578 struct phys_sge_parm sg_param; 579 unsigned int frags = 0, transhdr_len, phys_dsgl; 580 unsigned int ivsize = crypto_ablkcipher_ivsize(tfm), kctx_len; 581 gfp_t flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL : 582 GFP_ATOMIC; 583 584 if (!req->info) 585 return ERR_PTR(-EINVAL); 586 reqctx->dst_nents = sg_nents_for_len(req->dst, req->nbytes); 587 if (reqctx->dst_nents <= 0) { 588 pr_err("AES:Invalid Destination sg lists\n"); 589 return ERR_PTR(-EINVAL); 590 } 591 if ((ablkctx->enckey_len == 0) || (ivsize > AES_BLOCK_SIZE) || 592 (req->nbytes <= 0) || (req->nbytes % AES_BLOCK_SIZE)) { 593 pr_err("AES: Invalid value of Key Len %d nbytes %d IV Len %d\n", 594 ablkctx->enckey_len, req->nbytes, ivsize); 595 return ERR_PTR(-EINVAL); 596 } 597 598 phys_dsgl = get_space_for_phys_dsgl(reqctx->dst_nents); 599 600 kctx_len = (DIV_ROUND_UP(ablkctx->enckey_len, 16) * 16); 601 transhdr_len = CIPHER_TRANSHDR_SIZE(kctx_len, phys_dsgl); 602 skb = alloc_skb((transhdr_len + sizeof(struct sge_opaque_hdr)), flags); 603 if (!skb) 604 return ERR_PTR(-ENOMEM); 605 skb_reserve(skb, sizeof(struct sge_opaque_hdr)); 606 chcr_req = (struct chcr_wr *)__skb_put(skb, transhdr_len); 607 memset(chcr_req, 0, transhdr_len); 608 chcr_req->sec_cpl.op_ivinsrtofst = 609 FILL_SEC_CPL_OP_IVINSR(ctx->dev->tx_channel_id, 2, 1); 610 611 chcr_req->sec_cpl.pldlen = htonl(ivsize + req->nbytes); 612 chcr_req->sec_cpl.aadstart_cipherstop_hi = 613 FILL_SEC_CPL_CIPHERSTOP_HI(0, 0, ivsize + 1, 0); 614 615 chcr_req->sec_cpl.cipherstop_lo_authinsert = 616 FILL_SEC_CPL_AUTHINSERT(0, 0, 0, 0); 617 chcr_req->sec_cpl.seqno_numivs = FILL_SEC_CPL_SCMD0_SEQNO(op_type, 0, 618 ablkctx->ciph_mode, 619 0, 0, ivsize >> 1); 620 chcr_req->sec_cpl.ivgen_hdrlen = FILL_SEC_CPL_IVGEN_HDRLEN(0, 0, 0, 621 0, 1, phys_dsgl); 622 623 chcr_req->key_ctx.ctx_hdr = ablkctx->key_ctx_hdr; 624 if (op_type == CHCR_DECRYPT_OP) { 625 generate_copy_rrkey(ablkctx, &chcr_req->key_ctx); 626 } else { 627 if (ablkctx->ciph_mode == CHCR_SCMD_CIPHER_MODE_AES_CBC) { 628 memcpy(chcr_req->key_ctx.key, ablkctx->key, 629 ablkctx->enckey_len); 630 } else { 631 memcpy(chcr_req->key_ctx.key, ablkctx->key + 632 (ablkctx->enckey_len >> 1), 633 ablkctx->enckey_len >> 1); 634 memcpy(chcr_req->key_ctx.key + 635 (ablkctx->enckey_len >> 1), 636 ablkctx->key, 637 ablkctx->enckey_len >> 1); 638 } 639 } 640 phys_cpl = (struct cpl_rx_phys_dsgl *)((u8 *)(chcr_req + 1) + kctx_len); 641 sg_param.nents = reqctx->dst_nents; 642 sg_param.obsize = req->nbytes; 643 sg_param.qid = qid; 644 sg_param.align = 1; 645 if (map_writesg_phys_cpl(&u_ctx->lldi.pdev->dev, phys_cpl, req->dst, 646 &sg_param)) 647 goto map_fail1; 648 649 skb_set_transport_header(skb, transhdr_len); 650 memcpy(reqctx->iv, req->info, ivsize); 651 write_buffer_to_skb(skb, &frags, reqctx->iv, ivsize); 652 write_sg_to_skb(skb, &frags, req->src, req->nbytes); 653 create_wreq(ctx, chcr_req, req, skb, kctx_len, 0, 1, 654 sizeof(struct cpl_rx_phys_dsgl) + phys_dsgl); 655 reqctx->skb = skb; 656 skb_get(skb); 657 return skb; 658 map_fail1: 659 kfree_skb(skb); 660 return ERR_PTR(-ENOMEM); 661 } 662 663 static int chcr_aes_cbc_setkey(struct crypto_ablkcipher *tfm, const u8 *key, 664 unsigned int keylen) 665 { 666 struct chcr_context *ctx = crypto_ablkcipher_ctx(tfm); 667 struct ablk_ctx *ablkctx = ABLK_CTX(ctx); 668 unsigned int ck_size, context_size; 669 u16 alignment = 0; 670 671 if (keylen == AES_KEYSIZE_128) { 672 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_128; 673 } else if (keylen == AES_KEYSIZE_192) { 674 alignment = 8; 675 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_192; 676 } else if (keylen == AES_KEYSIZE_256) { 677 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_256; 678 } else { 679 goto badkey_err; 680 } 681 memcpy(ablkctx->key, key, keylen); 682 ablkctx->enckey_len = keylen; 683 get_aes_decrypt_key(ablkctx->rrkey, ablkctx->key, keylen << 3); 684 context_size = (KEY_CONTEXT_HDR_SALT_AND_PAD + 685 keylen + alignment) >> 4; 686 687 ablkctx->key_ctx_hdr = FILL_KEY_CTX_HDR(ck_size, CHCR_KEYCTX_NO_KEY, 688 0, 0, context_size); 689 ablkctx->ciph_mode = CHCR_SCMD_CIPHER_MODE_AES_CBC; 690 return 0; 691 badkey_err: 692 crypto_ablkcipher_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN); 693 ablkctx->enckey_len = 0; 694 return -EINVAL; 695 } 696 697 static int cxgb4_is_crypto_q_full(struct net_device *dev, unsigned int idx) 698 { 699 struct adapter *adap = netdev2adap(dev); 700 struct sge_uld_txq_info *txq_info = 701 adap->sge.uld_txq_info[CXGB4_TX_CRYPTO]; 702 struct sge_uld_txq *txq; 703 int ret = 0; 704 705 local_bh_disable(); 706 txq = &txq_info->uldtxq[idx]; 707 spin_lock(&txq->sendq.lock); 708 if (txq->full) 709 ret = -1; 710 spin_unlock(&txq->sendq.lock); 711 local_bh_enable(); 712 return ret; 713 } 714 715 static int chcr_aes_encrypt(struct ablkcipher_request *req) 716 { 717 struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req); 718 struct chcr_context *ctx = crypto_ablkcipher_ctx(tfm); 719 struct uld_ctx *u_ctx = ULD_CTX(ctx); 720 struct sk_buff *skb; 721 722 if (unlikely(cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0], 723 ctx->tx_channel_id))) { 724 if (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) 725 return -EBUSY; 726 } 727 728 skb = create_cipher_wr(req, u_ctx->lldi.rxq_ids[ctx->tx_channel_id], 729 CHCR_ENCRYPT_OP); 730 if (IS_ERR(skb)) { 731 pr_err("chcr : %s : Failed to form WR. No memory\n", __func__); 732 return PTR_ERR(skb); 733 } 734 skb->dev = u_ctx->lldi.ports[0]; 735 set_wr_txq(skb, CPL_PRIORITY_DATA, ctx->tx_channel_id); 736 chcr_send_wr(skb); 737 return -EINPROGRESS; 738 } 739 740 static int chcr_aes_decrypt(struct ablkcipher_request *req) 741 { 742 struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req); 743 struct chcr_context *ctx = crypto_ablkcipher_ctx(tfm); 744 struct uld_ctx *u_ctx = ULD_CTX(ctx); 745 struct sk_buff *skb; 746 747 if (unlikely(cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0], 748 ctx->tx_channel_id))) { 749 if (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) 750 return -EBUSY; 751 } 752 753 skb = create_cipher_wr(req, u_ctx->lldi.rxq_ids[0], 754 CHCR_DECRYPT_OP); 755 if (IS_ERR(skb)) { 756 pr_err("chcr : %s : Failed to form WR. No memory\n", __func__); 757 return PTR_ERR(skb); 758 } 759 skb->dev = u_ctx->lldi.ports[0]; 760 set_wr_txq(skb, CPL_PRIORITY_DATA, ctx->tx_channel_id); 761 chcr_send_wr(skb); 762 return -EINPROGRESS; 763 } 764 765 static int chcr_device_init(struct chcr_context *ctx) 766 { 767 struct uld_ctx *u_ctx; 768 unsigned int id; 769 int err = 0, rxq_perchan, rxq_idx; 770 771 id = smp_processor_id(); 772 if (!ctx->dev) { 773 err = assign_chcr_device(&ctx->dev); 774 if (err) { 775 pr_err("chcr device assignment fails\n"); 776 goto out; 777 } 778 u_ctx = ULD_CTX(ctx); 779 rxq_perchan = u_ctx->lldi.nrxq / u_ctx->lldi.nchan; 780 rxq_idx = ctx->dev->tx_channel_id * rxq_perchan; 781 rxq_idx += id % rxq_perchan; 782 spin_lock(&ctx->dev->lock_chcr_dev); 783 ctx->tx_channel_id = rxq_idx; 784 ctx->dev->tx_channel_id = !ctx->dev->tx_channel_id; 785 spin_unlock(&ctx->dev->lock_chcr_dev); 786 } 787 out: 788 return err; 789 } 790 791 static int chcr_cra_init(struct crypto_tfm *tfm) 792 { 793 tfm->crt_ablkcipher.reqsize = sizeof(struct chcr_blkcipher_req_ctx); 794 return chcr_device_init(crypto_tfm_ctx(tfm)); 795 } 796 797 static int get_alg_config(struct algo_param *params, 798 unsigned int auth_size) 799 { 800 switch (auth_size) { 801 case SHA1_DIGEST_SIZE: 802 params->mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_160; 803 params->auth_mode = CHCR_SCMD_AUTH_MODE_SHA1; 804 params->result_size = SHA1_DIGEST_SIZE; 805 break; 806 case SHA224_DIGEST_SIZE: 807 params->mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_256; 808 params->auth_mode = CHCR_SCMD_AUTH_MODE_SHA224; 809 params->result_size = SHA256_DIGEST_SIZE; 810 break; 811 case SHA256_DIGEST_SIZE: 812 params->mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_256; 813 params->auth_mode = CHCR_SCMD_AUTH_MODE_SHA256; 814 params->result_size = SHA256_DIGEST_SIZE; 815 break; 816 case SHA384_DIGEST_SIZE: 817 params->mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_512; 818 params->auth_mode = CHCR_SCMD_AUTH_MODE_SHA512_384; 819 params->result_size = SHA512_DIGEST_SIZE; 820 break; 821 case SHA512_DIGEST_SIZE: 822 params->mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_512; 823 params->auth_mode = CHCR_SCMD_AUTH_MODE_SHA512_512; 824 params->result_size = SHA512_DIGEST_SIZE; 825 break; 826 default: 827 pr_err("chcr : ERROR, unsupported digest size\n"); 828 return -EINVAL; 829 } 830 return 0; 831 } 832 833 static inline void chcr_free_shash(struct crypto_shash *base_hash) 834 { 835 crypto_free_shash(base_hash); 836 } 837 838 /** 839 * create_hash_wr - Create hash work request 840 * @req - Cipher req base 841 */ 842 static struct sk_buff *create_hash_wr(struct ahash_request *req, 843 struct hash_wr_param *param) 844 { 845 struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(req); 846 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 847 struct chcr_context *ctx = crypto_tfm_ctx(crypto_ahash_tfm(tfm)); 848 struct hmac_ctx *hmacctx = HMAC_CTX(ctx); 849 struct sk_buff *skb = NULL; 850 struct chcr_wr *chcr_req; 851 unsigned int frags = 0, transhdr_len, iopad_alignment = 0; 852 unsigned int digestsize = crypto_ahash_digestsize(tfm); 853 unsigned int kctx_len = 0; 854 u8 hash_size_in_response = 0; 855 gfp_t flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL : 856 GFP_ATOMIC; 857 858 iopad_alignment = KEYCTX_ALIGN_PAD(digestsize); 859 kctx_len = param->alg_prm.result_size + iopad_alignment; 860 if (param->opad_needed) 861 kctx_len += param->alg_prm.result_size + iopad_alignment; 862 863 if (req_ctx->result) 864 hash_size_in_response = digestsize; 865 else 866 hash_size_in_response = param->alg_prm.result_size; 867 transhdr_len = HASH_TRANSHDR_SIZE(kctx_len); 868 skb = alloc_skb((transhdr_len + sizeof(struct sge_opaque_hdr)), flags); 869 if (!skb) 870 return skb; 871 872 skb_reserve(skb, sizeof(struct sge_opaque_hdr)); 873 chcr_req = (struct chcr_wr *)__skb_put(skb, transhdr_len); 874 memset(chcr_req, 0, transhdr_len); 875 876 chcr_req->sec_cpl.op_ivinsrtofst = 877 FILL_SEC_CPL_OP_IVINSR(ctx->dev->tx_channel_id, 2, 0); 878 chcr_req->sec_cpl.pldlen = htonl(param->bfr_len + param->sg_len); 879 880 chcr_req->sec_cpl.aadstart_cipherstop_hi = 881 FILL_SEC_CPL_CIPHERSTOP_HI(0, 0, 0, 0); 882 chcr_req->sec_cpl.cipherstop_lo_authinsert = 883 FILL_SEC_CPL_AUTHINSERT(0, 1, 0, 0); 884 chcr_req->sec_cpl.seqno_numivs = 885 FILL_SEC_CPL_SCMD0_SEQNO(0, 0, 0, param->alg_prm.auth_mode, 886 param->opad_needed, 0); 887 888 chcr_req->sec_cpl.ivgen_hdrlen = 889 FILL_SEC_CPL_IVGEN_HDRLEN(param->last, param->more, 0, 1, 0, 0); 890 891 memcpy(chcr_req->key_ctx.key, req_ctx->partial_hash, 892 param->alg_prm.result_size); 893 894 if (param->opad_needed) 895 memcpy(chcr_req->key_ctx.key + 896 ((param->alg_prm.result_size <= 32) ? 32 : 897 CHCR_HASH_MAX_DIGEST_SIZE), 898 hmacctx->opad, param->alg_prm.result_size); 899 900 chcr_req->key_ctx.ctx_hdr = FILL_KEY_CTX_HDR(CHCR_KEYCTX_NO_KEY, 901 param->alg_prm.mk_size, 0, 902 param->opad_needed, 903 ((kctx_len + 904 sizeof(chcr_req->key_ctx)) >> 4)); 905 chcr_req->sec_cpl.scmd1 = cpu_to_be64((u64)param->scmd1); 906 907 skb_set_transport_header(skb, transhdr_len); 908 if (param->bfr_len != 0) 909 write_buffer_to_skb(skb, &frags, req_ctx->reqbfr, 910 param->bfr_len); 911 if (param->sg_len != 0) 912 write_sg_to_skb(skb, &frags, req->src, param->sg_len); 913 914 create_wreq(ctx, chcr_req, req, skb, kctx_len, hash_size_in_response, 0, 915 DUMMY_BYTES); 916 req_ctx->skb = skb; 917 skb_get(skb); 918 return skb; 919 } 920 921 static int chcr_ahash_update(struct ahash_request *req) 922 { 923 struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(req); 924 struct crypto_ahash *rtfm = crypto_ahash_reqtfm(req); 925 struct chcr_context *ctx = crypto_tfm_ctx(crypto_ahash_tfm(rtfm)); 926 struct uld_ctx *u_ctx = NULL; 927 struct sk_buff *skb; 928 u8 remainder = 0, bs; 929 unsigned int nbytes = req->nbytes; 930 struct hash_wr_param params; 931 932 bs = crypto_tfm_alg_blocksize(crypto_ahash_tfm(rtfm)); 933 934 u_ctx = ULD_CTX(ctx); 935 if (unlikely(cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0], 936 ctx->tx_channel_id))) { 937 if (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) 938 return -EBUSY; 939 } 940 941 if (nbytes + req_ctx->reqlen >= bs) { 942 remainder = (nbytes + req_ctx->reqlen) % bs; 943 nbytes = nbytes + req_ctx->reqlen - remainder; 944 } else { 945 sg_pcopy_to_buffer(req->src, sg_nents(req->src), req_ctx->reqbfr 946 + req_ctx->reqlen, nbytes, 0); 947 req_ctx->reqlen += nbytes; 948 return 0; 949 } 950 951 params.opad_needed = 0; 952 params.more = 1; 953 params.last = 0; 954 params.sg_len = nbytes - req_ctx->reqlen; 955 params.bfr_len = req_ctx->reqlen; 956 params.scmd1 = 0; 957 get_alg_config(¶ms.alg_prm, crypto_ahash_digestsize(rtfm)); 958 req_ctx->result = 0; 959 req_ctx->data_len += params.sg_len + params.bfr_len; 960 skb = create_hash_wr(req, ¶ms); 961 if (!skb) 962 return -ENOMEM; 963 964 if (remainder) { 965 u8 *temp; 966 /* Swap buffers */ 967 temp = req_ctx->reqbfr; 968 req_ctx->reqbfr = req_ctx->skbfr; 969 req_ctx->skbfr = temp; 970 sg_pcopy_to_buffer(req->src, sg_nents(req->src), 971 req_ctx->reqbfr, remainder, req->nbytes - 972 remainder); 973 } 974 req_ctx->reqlen = remainder; 975 skb->dev = u_ctx->lldi.ports[0]; 976 set_wr_txq(skb, CPL_PRIORITY_DATA, ctx->tx_channel_id); 977 chcr_send_wr(skb); 978 979 return -EINPROGRESS; 980 } 981 982 static void create_last_hash_block(char *bfr_ptr, unsigned int bs, u64 scmd1) 983 { 984 memset(bfr_ptr, 0, bs); 985 *bfr_ptr = 0x80; 986 if (bs == 64) 987 *(__be64 *)(bfr_ptr + 56) = cpu_to_be64(scmd1 << 3); 988 else 989 *(__be64 *)(bfr_ptr + 120) = cpu_to_be64(scmd1 << 3); 990 } 991 992 static int chcr_ahash_final(struct ahash_request *req) 993 { 994 struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(req); 995 struct crypto_ahash *rtfm = crypto_ahash_reqtfm(req); 996 struct chcr_context *ctx = crypto_tfm_ctx(crypto_ahash_tfm(rtfm)); 997 struct hash_wr_param params; 998 struct sk_buff *skb; 999 struct uld_ctx *u_ctx = NULL; 1000 u8 bs = crypto_tfm_alg_blocksize(crypto_ahash_tfm(rtfm)); 1001 1002 u_ctx = ULD_CTX(ctx); 1003 if (is_hmac(crypto_ahash_tfm(rtfm))) 1004 params.opad_needed = 1; 1005 else 1006 params.opad_needed = 0; 1007 params.sg_len = 0; 1008 get_alg_config(¶ms.alg_prm, crypto_ahash_digestsize(rtfm)); 1009 req_ctx->result = 1; 1010 params.bfr_len = req_ctx->reqlen; 1011 req_ctx->data_len += params.bfr_len + params.sg_len; 1012 if (req_ctx->reqlen == 0) { 1013 create_last_hash_block(req_ctx->reqbfr, bs, req_ctx->data_len); 1014 params.last = 0; 1015 params.more = 1; 1016 params.scmd1 = 0; 1017 params.bfr_len = bs; 1018 1019 } else { 1020 params.scmd1 = req_ctx->data_len; 1021 params.last = 1; 1022 params.more = 0; 1023 } 1024 skb = create_hash_wr(req, ¶ms); 1025 if (!skb) 1026 return -ENOMEM; 1027 1028 skb->dev = u_ctx->lldi.ports[0]; 1029 set_wr_txq(skb, CPL_PRIORITY_DATA, ctx->tx_channel_id); 1030 chcr_send_wr(skb); 1031 return -EINPROGRESS; 1032 } 1033 1034 static int chcr_ahash_finup(struct ahash_request *req) 1035 { 1036 struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(req); 1037 struct crypto_ahash *rtfm = crypto_ahash_reqtfm(req); 1038 struct chcr_context *ctx = crypto_tfm_ctx(crypto_ahash_tfm(rtfm)); 1039 struct uld_ctx *u_ctx = NULL; 1040 struct sk_buff *skb; 1041 struct hash_wr_param params; 1042 u8 bs; 1043 1044 bs = crypto_tfm_alg_blocksize(crypto_ahash_tfm(rtfm)); 1045 u_ctx = ULD_CTX(ctx); 1046 1047 if (unlikely(cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0], 1048 ctx->tx_channel_id))) { 1049 if (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) 1050 return -EBUSY; 1051 } 1052 1053 if (is_hmac(crypto_ahash_tfm(rtfm))) 1054 params.opad_needed = 1; 1055 else 1056 params.opad_needed = 0; 1057 1058 params.sg_len = req->nbytes; 1059 params.bfr_len = req_ctx->reqlen; 1060 get_alg_config(¶ms.alg_prm, crypto_ahash_digestsize(rtfm)); 1061 req_ctx->data_len += params.bfr_len + params.sg_len; 1062 req_ctx->result = 1; 1063 if ((req_ctx->reqlen + req->nbytes) == 0) { 1064 create_last_hash_block(req_ctx->reqbfr, bs, req_ctx->data_len); 1065 params.last = 0; 1066 params.more = 1; 1067 params.scmd1 = 0; 1068 params.bfr_len = bs; 1069 } else { 1070 params.scmd1 = req_ctx->data_len; 1071 params.last = 1; 1072 params.more = 0; 1073 } 1074 1075 skb = create_hash_wr(req, ¶ms); 1076 if (!skb) 1077 return -ENOMEM; 1078 1079 skb->dev = u_ctx->lldi.ports[0]; 1080 set_wr_txq(skb, CPL_PRIORITY_DATA, ctx->tx_channel_id); 1081 chcr_send_wr(skb); 1082 1083 return -EINPROGRESS; 1084 } 1085 1086 static int chcr_ahash_digest(struct ahash_request *req) 1087 { 1088 struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(req); 1089 struct crypto_ahash *rtfm = crypto_ahash_reqtfm(req); 1090 struct chcr_context *ctx = crypto_tfm_ctx(crypto_ahash_tfm(rtfm)); 1091 struct uld_ctx *u_ctx = NULL; 1092 struct sk_buff *skb; 1093 struct hash_wr_param params; 1094 u8 bs; 1095 1096 rtfm->init(req); 1097 bs = crypto_tfm_alg_blocksize(crypto_ahash_tfm(rtfm)); 1098 1099 u_ctx = ULD_CTX(ctx); 1100 if (unlikely(cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0], 1101 ctx->tx_channel_id))) { 1102 if (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) 1103 return -EBUSY; 1104 } 1105 1106 if (is_hmac(crypto_ahash_tfm(rtfm))) 1107 params.opad_needed = 1; 1108 else 1109 params.opad_needed = 0; 1110 1111 params.last = 0; 1112 params.more = 0; 1113 params.sg_len = req->nbytes; 1114 params.bfr_len = 0; 1115 params.scmd1 = 0; 1116 get_alg_config(¶ms.alg_prm, crypto_ahash_digestsize(rtfm)); 1117 req_ctx->result = 1; 1118 req_ctx->data_len += params.bfr_len + params.sg_len; 1119 1120 if (req->nbytes == 0) { 1121 create_last_hash_block(req_ctx->reqbfr, bs, 0); 1122 params.more = 1; 1123 params.bfr_len = bs; 1124 } 1125 1126 skb = create_hash_wr(req, ¶ms); 1127 if (!skb) 1128 return -ENOMEM; 1129 1130 skb->dev = u_ctx->lldi.ports[0]; 1131 set_wr_txq(skb, CPL_PRIORITY_DATA, ctx->tx_channel_id); 1132 chcr_send_wr(skb); 1133 return -EINPROGRESS; 1134 } 1135 1136 static int chcr_ahash_export(struct ahash_request *areq, void *out) 1137 { 1138 struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(areq); 1139 struct chcr_ahash_req_ctx *state = out; 1140 1141 state->reqlen = req_ctx->reqlen; 1142 state->data_len = req_ctx->data_len; 1143 memcpy(state->bfr1, req_ctx->reqbfr, req_ctx->reqlen); 1144 memcpy(state->partial_hash, req_ctx->partial_hash, 1145 CHCR_HASH_MAX_DIGEST_SIZE); 1146 return 0; 1147 } 1148 1149 static int chcr_ahash_import(struct ahash_request *areq, const void *in) 1150 { 1151 struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(areq); 1152 struct chcr_ahash_req_ctx *state = (struct chcr_ahash_req_ctx *)in; 1153 1154 req_ctx->reqlen = state->reqlen; 1155 req_ctx->data_len = state->data_len; 1156 req_ctx->reqbfr = req_ctx->bfr1; 1157 req_ctx->skbfr = req_ctx->bfr2; 1158 memcpy(req_ctx->bfr1, state->bfr1, CHCR_HASH_MAX_BLOCK_SIZE_128); 1159 memcpy(req_ctx->partial_hash, state->partial_hash, 1160 CHCR_HASH_MAX_DIGEST_SIZE); 1161 return 0; 1162 } 1163 1164 static int chcr_ahash_setkey(struct crypto_ahash *tfm, const u8 *key, 1165 unsigned int keylen) 1166 { 1167 struct chcr_context *ctx = crypto_tfm_ctx(crypto_ahash_tfm(tfm)); 1168 struct hmac_ctx *hmacctx = HMAC_CTX(ctx); 1169 unsigned int digestsize = crypto_ahash_digestsize(tfm); 1170 unsigned int bs = crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm)); 1171 unsigned int i, err = 0, updated_digestsize; 1172 1173 SHASH_DESC_ON_STACK(shash, hmacctx->base_hash); 1174 1175 /* use the key to calculate the ipad and opad. ipad will sent with the 1176 * first request's data. opad will be sent with the final hash result 1177 * ipad in hmacctx->ipad and opad in hmacctx->opad location 1178 */ 1179 shash->tfm = hmacctx->base_hash; 1180 shash->flags = crypto_shash_get_flags(hmacctx->base_hash); 1181 if (keylen > bs) { 1182 err = crypto_shash_digest(shash, key, keylen, 1183 hmacctx->ipad); 1184 if (err) 1185 goto out; 1186 keylen = digestsize; 1187 } else { 1188 memcpy(hmacctx->ipad, key, keylen); 1189 } 1190 memset(hmacctx->ipad + keylen, 0, bs - keylen); 1191 memcpy(hmacctx->opad, hmacctx->ipad, bs); 1192 1193 for (i = 0; i < bs / sizeof(int); i++) { 1194 *((unsigned int *)(&hmacctx->ipad) + i) ^= IPAD_DATA; 1195 *((unsigned int *)(&hmacctx->opad) + i) ^= OPAD_DATA; 1196 } 1197 1198 updated_digestsize = digestsize; 1199 if (digestsize == SHA224_DIGEST_SIZE) 1200 updated_digestsize = SHA256_DIGEST_SIZE; 1201 else if (digestsize == SHA384_DIGEST_SIZE) 1202 updated_digestsize = SHA512_DIGEST_SIZE; 1203 err = chcr_compute_partial_hash(shash, hmacctx->ipad, 1204 hmacctx->ipad, digestsize); 1205 if (err) 1206 goto out; 1207 chcr_change_order(hmacctx->ipad, updated_digestsize); 1208 1209 err = chcr_compute_partial_hash(shash, hmacctx->opad, 1210 hmacctx->opad, digestsize); 1211 if (err) 1212 goto out; 1213 chcr_change_order(hmacctx->opad, updated_digestsize); 1214 out: 1215 return err; 1216 } 1217 1218 static int chcr_aes_xts_setkey(struct crypto_ablkcipher *tfm, const u8 *key, 1219 unsigned int key_len) 1220 { 1221 struct chcr_context *ctx = crypto_ablkcipher_ctx(tfm); 1222 struct ablk_ctx *ablkctx = ABLK_CTX(ctx); 1223 unsigned short context_size = 0; 1224 1225 if ((key_len != (AES_KEYSIZE_128 << 1)) && 1226 (key_len != (AES_KEYSIZE_256 << 1))) { 1227 crypto_tfm_set_flags((struct crypto_tfm *)tfm, 1228 CRYPTO_TFM_RES_BAD_KEY_LEN); 1229 ablkctx->enckey_len = 0; 1230 return -EINVAL; 1231 1232 } 1233 1234 memcpy(ablkctx->key, key, key_len); 1235 ablkctx->enckey_len = key_len; 1236 get_aes_decrypt_key(ablkctx->rrkey, ablkctx->key, key_len << 2); 1237 context_size = (KEY_CONTEXT_HDR_SALT_AND_PAD + key_len) >> 4; 1238 ablkctx->key_ctx_hdr = 1239 FILL_KEY_CTX_HDR((key_len == AES_KEYSIZE_256) ? 1240 CHCR_KEYCTX_CIPHER_KEY_SIZE_128 : 1241 CHCR_KEYCTX_CIPHER_KEY_SIZE_256, 1242 CHCR_KEYCTX_NO_KEY, 1, 1243 0, context_size); 1244 ablkctx->ciph_mode = CHCR_SCMD_CIPHER_MODE_AES_XTS; 1245 return 0; 1246 } 1247 1248 static int chcr_sha_init(struct ahash_request *areq) 1249 { 1250 struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(areq); 1251 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); 1252 int digestsize = crypto_ahash_digestsize(tfm); 1253 1254 req_ctx->data_len = 0; 1255 req_ctx->reqlen = 0; 1256 req_ctx->reqbfr = req_ctx->bfr1; 1257 req_ctx->skbfr = req_ctx->bfr2; 1258 req_ctx->skb = NULL; 1259 req_ctx->result = 0; 1260 copy_hash_init_values(req_ctx->partial_hash, digestsize); 1261 return 0; 1262 } 1263 1264 static int chcr_sha_cra_init(struct crypto_tfm *tfm) 1265 { 1266 crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), 1267 sizeof(struct chcr_ahash_req_ctx)); 1268 return chcr_device_init(crypto_tfm_ctx(tfm)); 1269 } 1270 1271 static int chcr_hmac_init(struct ahash_request *areq) 1272 { 1273 struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(areq); 1274 struct crypto_ahash *rtfm = crypto_ahash_reqtfm(areq); 1275 struct chcr_context *ctx = crypto_tfm_ctx(crypto_ahash_tfm(rtfm)); 1276 struct hmac_ctx *hmacctx = HMAC_CTX(ctx); 1277 unsigned int digestsize = crypto_ahash_digestsize(rtfm); 1278 unsigned int bs = crypto_tfm_alg_blocksize(crypto_ahash_tfm(rtfm)); 1279 1280 chcr_sha_init(areq); 1281 req_ctx->data_len = bs; 1282 if (is_hmac(crypto_ahash_tfm(rtfm))) { 1283 if (digestsize == SHA224_DIGEST_SIZE) 1284 memcpy(req_ctx->partial_hash, hmacctx->ipad, 1285 SHA256_DIGEST_SIZE); 1286 else if (digestsize == SHA384_DIGEST_SIZE) 1287 memcpy(req_ctx->partial_hash, hmacctx->ipad, 1288 SHA512_DIGEST_SIZE); 1289 else 1290 memcpy(req_ctx->partial_hash, hmacctx->ipad, 1291 digestsize); 1292 } 1293 return 0; 1294 } 1295 1296 static int chcr_hmac_cra_init(struct crypto_tfm *tfm) 1297 { 1298 struct chcr_context *ctx = crypto_tfm_ctx(tfm); 1299 struct hmac_ctx *hmacctx = HMAC_CTX(ctx); 1300 unsigned int digestsize = 1301 crypto_ahash_digestsize(__crypto_ahash_cast(tfm)); 1302 1303 crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), 1304 sizeof(struct chcr_ahash_req_ctx)); 1305 hmacctx->base_hash = chcr_alloc_shash(digestsize); 1306 if (IS_ERR(hmacctx->base_hash)) 1307 return PTR_ERR(hmacctx->base_hash); 1308 return chcr_device_init(crypto_tfm_ctx(tfm)); 1309 } 1310 1311 static void chcr_hmac_cra_exit(struct crypto_tfm *tfm) 1312 { 1313 struct chcr_context *ctx = crypto_tfm_ctx(tfm); 1314 struct hmac_ctx *hmacctx = HMAC_CTX(ctx); 1315 1316 if (hmacctx->base_hash) { 1317 chcr_free_shash(hmacctx->base_hash); 1318 hmacctx->base_hash = NULL; 1319 } 1320 } 1321 1322 static int chcr_copy_assoc(struct aead_request *req, 1323 struct chcr_aead_ctx *ctx) 1324 { 1325 SKCIPHER_REQUEST_ON_STACK(skreq, ctx->null); 1326 1327 skcipher_request_set_tfm(skreq, ctx->null); 1328 skcipher_request_set_callback(skreq, aead_request_flags(req), 1329 NULL, NULL); 1330 skcipher_request_set_crypt(skreq, req->src, req->dst, req->assoclen, 1331 NULL); 1332 1333 return crypto_skcipher_encrypt(skreq); 1334 } 1335 1336 static unsigned char get_hmac(unsigned int authsize) 1337 { 1338 switch (authsize) { 1339 case ICV_8: 1340 return CHCR_SCMD_HMAC_CTRL_PL1; 1341 case ICV_10: 1342 return CHCR_SCMD_HMAC_CTRL_TRUNC_RFC4366; 1343 case ICV_12: 1344 return CHCR_SCMD_HMAC_CTRL_IPSEC_96BIT; 1345 } 1346 return CHCR_SCMD_HMAC_CTRL_NO_TRUNC; 1347 } 1348 1349 1350 static struct sk_buff *create_authenc_wr(struct aead_request *req, 1351 unsigned short qid, 1352 int size, 1353 unsigned short op_type) 1354 { 1355 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 1356 struct chcr_context *ctx = crypto_aead_ctx(tfm); 1357 struct uld_ctx *u_ctx = ULD_CTX(ctx); 1358 struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx); 1359 struct chcr_authenc_ctx *actx = AUTHENC_CTX(aeadctx); 1360 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req); 1361 struct sk_buff *skb = NULL; 1362 struct chcr_wr *chcr_req; 1363 struct cpl_rx_phys_dsgl *phys_cpl; 1364 struct phys_sge_parm sg_param; 1365 struct scatterlist *src, *dst; 1366 struct scatterlist src_sg[2], dst_sg[2]; 1367 unsigned int frags = 0, transhdr_len; 1368 unsigned int ivsize = crypto_aead_ivsize(tfm), dst_size = 0; 1369 unsigned int kctx_len = 0; 1370 unsigned short stop_offset = 0; 1371 unsigned int assoclen = req->assoclen; 1372 unsigned int authsize = crypto_aead_authsize(tfm); 1373 int err = 0; 1374 int null = 0; 1375 gfp_t flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL : 1376 GFP_ATOMIC; 1377 1378 if (aeadctx->enckey_len == 0 || (req->cryptlen == 0)) 1379 goto err; 1380 1381 if (op_type && req->cryptlen < crypto_aead_authsize(tfm)) 1382 goto err; 1383 1384 if (sg_nents_for_len(req->src, req->assoclen + req->cryptlen) < 0) 1385 goto err; 1386 src = scatterwalk_ffwd(src_sg, req->src, req->assoclen); 1387 dst = src; 1388 if (req->src != req->dst) { 1389 err = chcr_copy_assoc(req, aeadctx); 1390 if (err) 1391 return ERR_PTR(err); 1392 dst = scatterwalk_ffwd(dst_sg, req->dst, req->assoclen); 1393 } 1394 if (get_aead_subtype(tfm) == CRYPTO_ALG_SUB_TYPE_AEAD_NULL) { 1395 null = 1; 1396 assoclen = 0; 1397 } 1398 reqctx->dst_nents = sg_nents_for_len(dst, req->cryptlen + 1399 (op_type ? -authsize : authsize)); 1400 if (reqctx->dst_nents <= 0) { 1401 pr_err("AUTHENC:Invalid Destination sg entries\n"); 1402 goto err; 1403 } 1404 dst_size = get_space_for_phys_dsgl(reqctx->dst_nents); 1405 kctx_len = (ntohl(KEY_CONTEXT_CTX_LEN_V(aeadctx->key_ctx_hdr)) << 4) 1406 - sizeof(chcr_req->key_ctx); 1407 transhdr_len = CIPHER_TRANSHDR_SIZE(kctx_len, dst_size); 1408 skb = alloc_skb((transhdr_len + sizeof(struct sge_opaque_hdr)), flags); 1409 if (!skb) 1410 goto err; 1411 1412 /* LLD is going to write the sge hdr. */ 1413 skb_reserve(skb, sizeof(struct sge_opaque_hdr)); 1414 1415 /* Write WR */ 1416 chcr_req = (struct chcr_wr *) __skb_put(skb, transhdr_len); 1417 memset(chcr_req, 0, transhdr_len); 1418 1419 stop_offset = (op_type == CHCR_ENCRYPT_OP) ? 0 : authsize; 1420 1421 /* 1422 * Input order is AAD,IV and Payload. where IV should be included as 1423 * the part of authdata. All other fields should be filled according 1424 * to the hardware spec 1425 */ 1426 chcr_req->sec_cpl.op_ivinsrtofst = 1427 FILL_SEC_CPL_OP_IVINSR(ctx->dev->tx_channel_id, 2, 1428 (ivsize ? (assoclen + 1) : 0)); 1429 chcr_req->sec_cpl.pldlen = htonl(assoclen + ivsize + req->cryptlen); 1430 chcr_req->sec_cpl.aadstart_cipherstop_hi = FILL_SEC_CPL_CIPHERSTOP_HI( 1431 assoclen ? 1 : 0, assoclen, 1432 assoclen + ivsize + 1, 1433 (stop_offset & 0x1F0) >> 4); 1434 chcr_req->sec_cpl.cipherstop_lo_authinsert = FILL_SEC_CPL_AUTHINSERT( 1435 stop_offset & 0xF, 1436 null ? 0 : assoclen + ivsize + 1, 1437 stop_offset, stop_offset); 1438 chcr_req->sec_cpl.seqno_numivs = FILL_SEC_CPL_SCMD0_SEQNO(op_type, 1439 (op_type == CHCR_ENCRYPT_OP) ? 1 : 0, 1440 CHCR_SCMD_CIPHER_MODE_AES_CBC, 1441 actx->auth_mode, aeadctx->hmac_ctrl, 1442 ivsize >> 1); 1443 chcr_req->sec_cpl.ivgen_hdrlen = FILL_SEC_CPL_IVGEN_HDRLEN(0, 0, 1, 1444 0, 1, dst_size); 1445 1446 chcr_req->key_ctx.ctx_hdr = aeadctx->key_ctx_hdr; 1447 if (op_type == CHCR_ENCRYPT_OP) 1448 memcpy(chcr_req->key_ctx.key, aeadctx->key, 1449 aeadctx->enckey_len); 1450 else 1451 memcpy(chcr_req->key_ctx.key, actx->dec_rrkey, 1452 aeadctx->enckey_len); 1453 1454 memcpy(chcr_req->key_ctx.key + (DIV_ROUND_UP(aeadctx->enckey_len, 16) << 1455 4), actx->h_iopad, kctx_len - 1456 (DIV_ROUND_UP(aeadctx->enckey_len, 16) << 4)); 1457 1458 phys_cpl = (struct cpl_rx_phys_dsgl *)((u8 *)(chcr_req + 1) + kctx_len); 1459 sg_param.nents = reqctx->dst_nents; 1460 sg_param.obsize = req->cryptlen + (op_type ? -authsize : authsize); 1461 sg_param.qid = qid; 1462 sg_param.align = 0; 1463 if (map_writesg_phys_cpl(&u_ctx->lldi.pdev->dev, phys_cpl, dst, 1464 &sg_param)) 1465 goto dstmap_fail; 1466 1467 skb_set_transport_header(skb, transhdr_len); 1468 1469 if (assoclen) { 1470 /* AAD buffer in */ 1471 write_sg_to_skb(skb, &frags, req->src, assoclen); 1472 1473 } 1474 write_buffer_to_skb(skb, &frags, req->iv, ivsize); 1475 write_sg_to_skb(skb, &frags, src, req->cryptlen); 1476 create_wreq(ctx, chcr_req, req, skb, kctx_len, size, 1, 1477 sizeof(struct cpl_rx_phys_dsgl) + dst_size); 1478 reqctx->skb = skb; 1479 skb_get(skb); 1480 1481 return skb; 1482 dstmap_fail: 1483 /* ivmap_fail: */ 1484 kfree_skb(skb); 1485 err: 1486 return ERR_PTR(-EINVAL); 1487 } 1488 1489 static void aes_gcm_empty_pld_pad(struct scatterlist *sg, 1490 unsigned short offset) 1491 { 1492 struct page *spage; 1493 unsigned char *addr; 1494 1495 spage = sg_page(sg); 1496 get_page(spage); /* so that it is not freed by NIC */ 1497 #ifdef KMAP_ATOMIC_ARGS 1498 addr = kmap_atomic(spage, KM_SOFTIRQ0); 1499 #else 1500 addr = kmap_atomic(spage); 1501 #endif 1502 memset(addr + sg->offset, 0, offset + 1); 1503 1504 kunmap_atomic(addr); 1505 } 1506 1507 static int set_msg_len(u8 *block, unsigned int msglen, int csize) 1508 { 1509 __be32 data; 1510 1511 memset(block, 0, csize); 1512 block += csize; 1513 1514 if (csize >= 4) 1515 csize = 4; 1516 else if (msglen > (unsigned int)(1 << (8 * csize))) 1517 return -EOVERFLOW; 1518 1519 data = cpu_to_be32(msglen); 1520 memcpy(block - csize, (u8 *)&data + 4 - csize, csize); 1521 1522 return 0; 1523 } 1524 1525 static void generate_b0(struct aead_request *req, 1526 struct chcr_aead_ctx *aeadctx, 1527 unsigned short op_type) 1528 { 1529 unsigned int l, lp, m; 1530 int rc; 1531 struct crypto_aead *aead = crypto_aead_reqtfm(req); 1532 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req); 1533 u8 *b0 = reqctx->scratch_pad; 1534 1535 m = crypto_aead_authsize(aead); 1536 1537 memcpy(b0, reqctx->iv, 16); 1538 1539 lp = b0[0]; 1540 l = lp + 1; 1541 1542 /* set m, bits 3-5 */ 1543 *b0 |= (8 * ((m - 2) / 2)); 1544 1545 /* set adata, bit 6, if associated data is used */ 1546 if (req->assoclen) 1547 *b0 |= 64; 1548 rc = set_msg_len(b0 + 16 - l, 1549 (op_type == CHCR_DECRYPT_OP) ? 1550 req->cryptlen - m : req->cryptlen, l); 1551 } 1552 1553 static inline int crypto_ccm_check_iv(const u8 *iv) 1554 { 1555 /* 2 <= L <= 8, so 1 <= L' <= 7. */ 1556 if (iv[0] < 1 || iv[0] > 7) 1557 return -EINVAL; 1558 1559 return 0; 1560 } 1561 1562 static int ccm_format_packet(struct aead_request *req, 1563 struct chcr_aead_ctx *aeadctx, 1564 unsigned int sub_type, 1565 unsigned short op_type) 1566 { 1567 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req); 1568 int rc = 0; 1569 1570 if (req->assoclen > T5_MAX_AAD_SIZE) { 1571 pr_err("CCM: Unsupported AAD data. It should be < %d\n", 1572 T5_MAX_AAD_SIZE); 1573 return -EINVAL; 1574 } 1575 if (sub_type == CRYPTO_ALG_SUB_TYPE_AEAD_RFC4309) { 1576 reqctx->iv[0] = 3; 1577 memcpy(reqctx->iv + 1, &aeadctx->salt[0], 3); 1578 memcpy(reqctx->iv + 4, req->iv, 8); 1579 memset(reqctx->iv + 12, 0, 4); 1580 *((unsigned short *)(reqctx->scratch_pad + 16)) = 1581 htons(req->assoclen - 8); 1582 } else { 1583 memcpy(reqctx->iv, req->iv, 16); 1584 *((unsigned short *)(reqctx->scratch_pad + 16)) = 1585 htons(req->assoclen); 1586 } 1587 generate_b0(req, aeadctx, op_type); 1588 /* zero the ctr value */ 1589 memset(reqctx->iv + 15 - reqctx->iv[0], 0, reqctx->iv[0] + 1); 1590 return rc; 1591 } 1592 1593 static void fill_sec_cpl_for_aead(struct cpl_tx_sec_pdu *sec_cpl, 1594 unsigned int dst_size, 1595 struct aead_request *req, 1596 unsigned short op_type, 1597 struct chcr_context *chcrctx) 1598 { 1599 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 1600 unsigned int ivsize = AES_BLOCK_SIZE; 1601 unsigned int cipher_mode = CHCR_SCMD_CIPHER_MODE_AES_CCM; 1602 unsigned int mac_mode = CHCR_SCMD_AUTH_MODE_CBCMAC; 1603 unsigned int c_id = chcrctx->dev->tx_channel_id; 1604 unsigned int ccm_xtra; 1605 unsigned char tag_offset = 0, auth_offset = 0; 1606 unsigned char hmac_ctrl = get_hmac(crypto_aead_authsize(tfm)); 1607 unsigned int assoclen; 1608 1609 if (get_aead_subtype(tfm) == CRYPTO_ALG_SUB_TYPE_AEAD_RFC4309) 1610 assoclen = req->assoclen - 8; 1611 else 1612 assoclen = req->assoclen; 1613 ccm_xtra = CCM_B0_SIZE + 1614 ((assoclen) ? CCM_AAD_FIELD_SIZE : 0); 1615 1616 auth_offset = req->cryptlen ? 1617 (assoclen + ivsize + 1 + ccm_xtra) : 0; 1618 if (op_type == CHCR_DECRYPT_OP) { 1619 if (crypto_aead_authsize(tfm) != req->cryptlen) 1620 tag_offset = crypto_aead_authsize(tfm); 1621 else 1622 auth_offset = 0; 1623 } 1624 1625 1626 sec_cpl->op_ivinsrtofst = FILL_SEC_CPL_OP_IVINSR(c_id, 1627 2, (ivsize ? (assoclen + 1) : 0) + 1628 ccm_xtra); 1629 sec_cpl->pldlen = 1630 htonl(assoclen + ivsize + req->cryptlen + ccm_xtra); 1631 /* For CCM there wil be b0 always. So AAD start will be 1 always */ 1632 sec_cpl->aadstart_cipherstop_hi = FILL_SEC_CPL_CIPHERSTOP_HI( 1633 1, assoclen + ccm_xtra, assoclen 1634 + ivsize + 1 + ccm_xtra, 0); 1635 1636 sec_cpl->cipherstop_lo_authinsert = FILL_SEC_CPL_AUTHINSERT(0, 1637 auth_offset, tag_offset, 1638 (op_type == CHCR_ENCRYPT_OP) ? 0 : 1639 crypto_aead_authsize(tfm)); 1640 sec_cpl->seqno_numivs = FILL_SEC_CPL_SCMD0_SEQNO(op_type, 1641 (op_type == CHCR_ENCRYPT_OP) ? 0 : 1, 1642 cipher_mode, mac_mode, hmac_ctrl, 1643 ivsize >> 1); 1644 1645 sec_cpl->ivgen_hdrlen = FILL_SEC_CPL_IVGEN_HDRLEN(0, 0, 1, 0, 1646 1, dst_size); 1647 } 1648 1649 int aead_ccm_validate_input(unsigned short op_type, 1650 struct aead_request *req, 1651 struct chcr_aead_ctx *aeadctx, 1652 unsigned int sub_type) 1653 { 1654 if (sub_type != CRYPTO_ALG_SUB_TYPE_AEAD_RFC4309) { 1655 if (crypto_ccm_check_iv(req->iv)) { 1656 pr_err("CCM: IV check fails\n"); 1657 return -EINVAL; 1658 } 1659 } else { 1660 if (req->assoclen != 16 && req->assoclen != 20) { 1661 pr_err("RFC4309: Invalid AAD length %d\n", 1662 req->assoclen); 1663 return -EINVAL; 1664 } 1665 } 1666 if (aeadctx->enckey_len == 0) { 1667 pr_err("CCM: Encryption key not set\n"); 1668 return -EINVAL; 1669 } 1670 return 0; 1671 } 1672 1673 unsigned int fill_aead_req_fields(struct sk_buff *skb, 1674 struct aead_request *req, 1675 struct scatterlist *src, 1676 unsigned int ivsize, 1677 struct chcr_aead_ctx *aeadctx) 1678 { 1679 unsigned int frags = 0; 1680 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 1681 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req); 1682 /* b0 and aad length(if available) */ 1683 1684 write_buffer_to_skb(skb, &frags, reqctx->scratch_pad, CCM_B0_SIZE + 1685 (req->assoclen ? CCM_AAD_FIELD_SIZE : 0)); 1686 if (req->assoclen) { 1687 if (get_aead_subtype(tfm) == CRYPTO_ALG_SUB_TYPE_AEAD_RFC4309) 1688 write_sg_to_skb(skb, &frags, req->src, 1689 req->assoclen - 8); 1690 else 1691 write_sg_to_skb(skb, &frags, req->src, req->assoclen); 1692 } 1693 write_buffer_to_skb(skb, &frags, reqctx->iv, ivsize); 1694 if (req->cryptlen) 1695 write_sg_to_skb(skb, &frags, src, req->cryptlen); 1696 1697 return frags; 1698 } 1699 1700 static struct sk_buff *create_aead_ccm_wr(struct aead_request *req, 1701 unsigned short qid, 1702 int size, 1703 unsigned short op_type) 1704 { 1705 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 1706 struct chcr_context *ctx = crypto_aead_ctx(tfm); 1707 struct uld_ctx *u_ctx = ULD_CTX(ctx); 1708 struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx); 1709 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req); 1710 struct sk_buff *skb = NULL; 1711 struct chcr_wr *chcr_req; 1712 struct cpl_rx_phys_dsgl *phys_cpl; 1713 struct phys_sge_parm sg_param; 1714 struct scatterlist *src, *dst; 1715 struct scatterlist src_sg[2], dst_sg[2]; 1716 unsigned int frags = 0, transhdr_len, ivsize = AES_BLOCK_SIZE; 1717 unsigned int dst_size = 0, kctx_len; 1718 unsigned int sub_type; 1719 unsigned int authsize = crypto_aead_authsize(tfm); 1720 int err = 0; 1721 gfp_t flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL : 1722 GFP_ATOMIC; 1723 1724 1725 if (op_type && req->cryptlen < crypto_aead_authsize(tfm)) 1726 goto err; 1727 1728 if (sg_nents_for_len(req->src, req->assoclen + req->cryptlen) < 0) 1729 goto err; 1730 sub_type = get_aead_subtype(tfm); 1731 src = scatterwalk_ffwd(src_sg, req->src, req->assoclen); 1732 dst = src; 1733 if (req->src != req->dst) { 1734 err = chcr_copy_assoc(req, aeadctx); 1735 if (err) { 1736 pr_err("AAD copy to destination buffer fails\n"); 1737 return ERR_PTR(err); 1738 } 1739 dst = scatterwalk_ffwd(dst_sg, req->dst, req->assoclen); 1740 } 1741 reqctx->dst_nents = sg_nents_for_len(dst, req->cryptlen + 1742 (op_type ? -authsize : authsize)); 1743 if (reqctx->dst_nents <= 0) { 1744 pr_err("CCM:Invalid Destination sg entries\n"); 1745 goto err; 1746 } 1747 1748 1749 if (aead_ccm_validate_input(op_type, req, aeadctx, sub_type)) 1750 goto err; 1751 1752 dst_size = get_space_for_phys_dsgl(reqctx->dst_nents); 1753 kctx_len = ((DIV_ROUND_UP(aeadctx->enckey_len, 16)) << 4) * 2; 1754 transhdr_len = CIPHER_TRANSHDR_SIZE(kctx_len, dst_size); 1755 skb = alloc_skb((transhdr_len + sizeof(struct sge_opaque_hdr)), flags); 1756 1757 if (!skb) 1758 goto err; 1759 1760 skb_reserve(skb, sizeof(struct sge_opaque_hdr)); 1761 1762 chcr_req = (struct chcr_wr *) __skb_put(skb, transhdr_len); 1763 memset(chcr_req, 0, transhdr_len); 1764 1765 fill_sec_cpl_for_aead(&chcr_req->sec_cpl, dst_size, req, op_type, ctx); 1766 1767 chcr_req->key_ctx.ctx_hdr = aeadctx->key_ctx_hdr; 1768 memcpy(chcr_req->key_ctx.key, aeadctx->key, aeadctx->enckey_len); 1769 memcpy(chcr_req->key_ctx.key + (DIV_ROUND_UP(aeadctx->enckey_len, 16) * 1770 16), aeadctx->key, aeadctx->enckey_len); 1771 1772 phys_cpl = (struct cpl_rx_phys_dsgl *)((u8 *)(chcr_req + 1) + kctx_len); 1773 if (ccm_format_packet(req, aeadctx, sub_type, op_type)) 1774 goto dstmap_fail; 1775 1776 sg_param.nents = reqctx->dst_nents; 1777 sg_param.obsize = req->cryptlen + (op_type ? -authsize : authsize); 1778 sg_param.qid = qid; 1779 sg_param.align = 0; 1780 if (map_writesg_phys_cpl(&u_ctx->lldi.pdev->dev, phys_cpl, dst, 1781 &sg_param)) 1782 goto dstmap_fail; 1783 1784 skb_set_transport_header(skb, transhdr_len); 1785 frags = fill_aead_req_fields(skb, req, src, ivsize, aeadctx); 1786 create_wreq(ctx, chcr_req, req, skb, kctx_len, 0, 1, 1787 sizeof(struct cpl_rx_phys_dsgl) + dst_size); 1788 reqctx->skb = skb; 1789 skb_get(skb); 1790 return skb; 1791 dstmap_fail: 1792 kfree_skb(skb); 1793 skb = NULL; 1794 err: 1795 return ERR_PTR(-EINVAL); 1796 } 1797 1798 static struct sk_buff *create_gcm_wr(struct aead_request *req, 1799 unsigned short qid, 1800 int size, 1801 unsigned short op_type) 1802 { 1803 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 1804 struct chcr_context *ctx = crypto_aead_ctx(tfm); 1805 struct uld_ctx *u_ctx = ULD_CTX(ctx); 1806 struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx); 1807 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req); 1808 struct sk_buff *skb = NULL; 1809 struct chcr_wr *chcr_req; 1810 struct cpl_rx_phys_dsgl *phys_cpl; 1811 struct phys_sge_parm sg_param; 1812 struct scatterlist *src, *dst; 1813 struct scatterlist src_sg[2], dst_sg[2]; 1814 unsigned int frags = 0, transhdr_len; 1815 unsigned int ivsize = AES_BLOCK_SIZE; 1816 unsigned int dst_size = 0, kctx_len; 1817 unsigned char tag_offset = 0; 1818 unsigned int crypt_len = 0; 1819 unsigned int authsize = crypto_aead_authsize(tfm); 1820 unsigned char hmac_ctrl = get_hmac(authsize); 1821 int err = 0; 1822 gfp_t flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL : 1823 GFP_ATOMIC; 1824 1825 /* validate key size */ 1826 if (aeadctx->enckey_len == 0) 1827 goto err; 1828 1829 if (op_type && req->cryptlen < crypto_aead_authsize(tfm)) 1830 goto err; 1831 1832 if (sg_nents_for_len(req->src, req->assoclen + req->cryptlen) < 0) 1833 goto err; 1834 1835 src = scatterwalk_ffwd(src_sg, req->src, req->assoclen); 1836 dst = src; 1837 if (req->src != req->dst) { 1838 err = chcr_copy_assoc(req, aeadctx); 1839 if (err) 1840 return ERR_PTR(err); 1841 dst = scatterwalk_ffwd(dst_sg, req->dst, req->assoclen); 1842 } 1843 1844 if (!req->cryptlen) 1845 /* null-payload is not supported in the hardware. 1846 * software is sending block size 1847 */ 1848 crypt_len = AES_BLOCK_SIZE; 1849 else 1850 crypt_len = req->cryptlen; 1851 reqctx->dst_nents = sg_nents_for_len(dst, req->cryptlen + 1852 (op_type ? -authsize : authsize)); 1853 if (reqctx->dst_nents <= 0) { 1854 pr_err("GCM:Invalid Destination sg entries\n"); 1855 goto err; 1856 } 1857 1858 1859 dst_size = get_space_for_phys_dsgl(reqctx->dst_nents); 1860 kctx_len = ((DIV_ROUND_UP(aeadctx->enckey_len, 16)) << 4) + 1861 AEAD_H_SIZE; 1862 transhdr_len = CIPHER_TRANSHDR_SIZE(kctx_len, dst_size); 1863 skb = alloc_skb((transhdr_len + sizeof(struct sge_opaque_hdr)), flags); 1864 if (!skb) 1865 goto err; 1866 1867 /* NIC driver is going to write the sge hdr. */ 1868 skb_reserve(skb, sizeof(struct sge_opaque_hdr)); 1869 1870 chcr_req = (struct chcr_wr *)__skb_put(skb, transhdr_len); 1871 memset(chcr_req, 0, transhdr_len); 1872 1873 if (get_aead_subtype(tfm) == CRYPTO_ALG_SUB_TYPE_AEAD_RFC4106) 1874 req->assoclen -= 8; 1875 1876 tag_offset = (op_type == CHCR_ENCRYPT_OP) ? 0 : authsize; 1877 chcr_req->sec_cpl.op_ivinsrtofst = FILL_SEC_CPL_OP_IVINSR( 1878 ctx->dev->tx_channel_id, 2, (ivsize ? 1879 (req->assoclen + 1) : 0)); 1880 chcr_req->sec_cpl.pldlen = htonl(req->assoclen + ivsize + crypt_len); 1881 chcr_req->sec_cpl.aadstart_cipherstop_hi = FILL_SEC_CPL_CIPHERSTOP_HI( 1882 req->assoclen ? 1 : 0, req->assoclen, 1883 req->assoclen + ivsize + 1, 0); 1884 if (req->cryptlen) { 1885 chcr_req->sec_cpl.cipherstop_lo_authinsert = 1886 FILL_SEC_CPL_AUTHINSERT(0, req->assoclen + ivsize + 1, 1887 tag_offset, tag_offset); 1888 chcr_req->sec_cpl.seqno_numivs = 1889 FILL_SEC_CPL_SCMD0_SEQNO(op_type, (op_type == 1890 CHCR_ENCRYPT_OP) ? 1 : 0, 1891 CHCR_SCMD_CIPHER_MODE_AES_GCM, 1892 CHCR_SCMD_AUTH_MODE_GHASH, hmac_ctrl, 1893 ivsize >> 1); 1894 } else { 1895 chcr_req->sec_cpl.cipherstop_lo_authinsert = 1896 FILL_SEC_CPL_AUTHINSERT(0, 0, 0, 0); 1897 chcr_req->sec_cpl.seqno_numivs = 1898 FILL_SEC_CPL_SCMD0_SEQNO(op_type, 1899 (op_type == CHCR_ENCRYPT_OP) ? 1900 1 : 0, CHCR_SCMD_CIPHER_MODE_AES_CBC, 1901 0, 0, ivsize >> 1); 1902 } 1903 chcr_req->sec_cpl.ivgen_hdrlen = FILL_SEC_CPL_IVGEN_HDRLEN(0, 0, 1, 1904 0, 1, dst_size); 1905 chcr_req->key_ctx.ctx_hdr = aeadctx->key_ctx_hdr; 1906 memcpy(chcr_req->key_ctx.key, aeadctx->key, aeadctx->enckey_len); 1907 memcpy(chcr_req->key_ctx.key + (DIV_ROUND_UP(aeadctx->enckey_len, 16) * 1908 16), GCM_CTX(aeadctx)->ghash_h, AEAD_H_SIZE); 1909 1910 /* prepare a 16 byte iv */ 1911 /* S A L T | IV | 0x00000001 */ 1912 if (get_aead_subtype(tfm) == 1913 CRYPTO_ALG_SUB_TYPE_AEAD_RFC4106) { 1914 memcpy(reqctx->iv, aeadctx->salt, 4); 1915 memcpy(reqctx->iv + 4, req->iv, 8); 1916 } else { 1917 memcpy(reqctx->iv, req->iv, 12); 1918 } 1919 *((unsigned int *)(reqctx->iv + 12)) = htonl(0x01); 1920 1921 phys_cpl = (struct cpl_rx_phys_dsgl *)((u8 *)(chcr_req + 1) + kctx_len); 1922 sg_param.nents = reqctx->dst_nents; 1923 sg_param.obsize = req->cryptlen + (op_type ? -authsize : authsize); 1924 sg_param.qid = qid; 1925 sg_param.align = 0; 1926 if (map_writesg_phys_cpl(&u_ctx->lldi.pdev->dev, phys_cpl, dst, 1927 &sg_param)) 1928 goto dstmap_fail; 1929 1930 skb_set_transport_header(skb, transhdr_len); 1931 1932 write_sg_to_skb(skb, &frags, req->src, req->assoclen); 1933 1934 write_buffer_to_skb(skb, &frags, reqctx->iv, ivsize); 1935 1936 if (req->cryptlen) { 1937 write_sg_to_skb(skb, &frags, src, req->cryptlen); 1938 } else { 1939 aes_gcm_empty_pld_pad(req->dst, authsize - 1); 1940 write_sg_to_skb(skb, &frags, dst, crypt_len); 1941 } 1942 1943 create_wreq(ctx, chcr_req, req, skb, kctx_len, size, 1, 1944 sizeof(struct cpl_rx_phys_dsgl) + dst_size); 1945 reqctx->skb = skb; 1946 skb_get(skb); 1947 return skb; 1948 1949 dstmap_fail: 1950 /* ivmap_fail: */ 1951 kfree_skb(skb); 1952 skb = NULL; 1953 err: 1954 return skb; 1955 } 1956 1957 1958 1959 static int chcr_aead_cra_init(struct crypto_aead *tfm) 1960 { 1961 struct chcr_context *ctx = crypto_aead_ctx(tfm); 1962 struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx); 1963 1964 crypto_aead_set_reqsize(tfm, sizeof(struct chcr_aead_reqctx)); 1965 aeadctx->null = crypto_get_default_null_skcipher(); 1966 if (IS_ERR(aeadctx->null)) 1967 return PTR_ERR(aeadctx->null); 1968 return chcr_device_init(ctx); 1969 } 1970 1971 static void chcr_aead_cra_exit(struct crypto_aead *tfm) 1972 { 1973 crypto_put_default_null_skcipher(); 1974 } 1975 1976 static int chcr_authenc_null_setauthsize(struct crypto_aead *tfm, 1977 unsigned int authsize) 1978 { 1979 struct chcr_aead_ctx *aeadctx = AEAD_CTX(crypto_aead_ctx(tfm)); 1980 1981 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_NOP; 1982 aeadctx->mayverify = VERIFY_HW; 1983 return 0; 1984 } 1985 static int chcr_authenc_setauthsize(struct crypto_aead *tfm, 1986 unsigned int authsize) 1987 { 1988 struct chcr_aead_ctx *aeadctx = AEAD_CTX(crypto_aead_ctx(tfm)); 1989 u32 maxauth = crypto_aead_maxauthsize(tfm); 1990 1991 /*SHA1 authsize in ipsec is 12 instead of 10 i.e maxauthsize / 2 is not 1992 * true for sha1. authsize == 12 condition should be before 1993 * authsize == (maxauth >> 1) 1994 */ 1995 if (authsize == ICV_4) { 1996 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_PL1; 1997 aeadctx->mayverify = VERIFY_HW; 1998 } else if (authsize == ICV_6) { 1999 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_PL2; 2000 aeadctx->mayverify = VERIFY_HW; 2001 } else if (authsize == ICV_10) { 2002 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_TRUNC_RFC4366; 2003 aeadctx->mayverify = VERIFY_HW; 2004 } else if (authsize == ICV_12) { 2005 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_IPSEC_96BIT; 2006 aeadctx->mayverify = VERIFY_HW; 2007 } else if (authsize == ICV_14) { 2008 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_PL3; 2009 aeadctx->mayverify = VERIFY_HW; 2010 } else if (authsize == (maxauth >> 1)) { 2011 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_DIV2; 2012 aeadctx->mayverify = VERIFY_HW; 2013 } else if (authsize == maxauth) { 2014 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_NO_TRUNC; 2015 aeadctx->mayverify = VERIFY_HW; 2016 } else { 2017 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_NO_TRUNC; 2018 aeadctx->mayverify = VERIFY_SW; 2019 } 2020 return 0; 2021 } 2022 2023 2024 static int chcr_gcm_setauthsize(struct crypto_aead *tfm, unsigned int authsize) 2025 { 2026 struct chcr_aead_ctx *aeadctx = AEAD_CTX(crypto_aead_ctx(tfm)); 2027 2028 switch (authsize) { 2029 case ICV_4: 2030 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_PL1; 2031 aeadctx->mayverify = VERIFY_HW; 2032 break; 2033 case ICV_8: 2034 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_DIV2; 2035 aeadctx->mayverify = VERIFY_HW; 2036 break; 2037 case ICV_12: 2038 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_IPSEC_96BIT; 2039 aeadctx->mayverify = VERIFY_HW; 2040 break; 2041 case ICV_14: 2042 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_PL3; 2043 aeadctx->mayverify = VERIFY_HW; 2044 break; 2045 case ICV_16: 2046 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_NO_TRUNC; 2047 aeadctx->mayverify = VERIFY_HW; 2048 break; 2049 case ICV_13: 2050 case ICV_15: 2051 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_NO_TRUNC; 2052 aeadctx->mayverify = VERIFY_SW; 2053 break; 2054 default: 2055 2056 crypto_tfm_set_flags((struct crypto_tfm *) tfm, 2057 CRYPTO_TFM_RES_BAD_KEY_LEN); 2058 return -EINVAL; 2059 } 2060 return 0; 2061 } 2062 2063 static int chcr_4106_4309_setauthsize(struct crypto_aead *tfm, 2064 unsigned int authsize) 2065 { 2066 struct chcr_aead_ctx *aeadctx = AEAD_CTX(crypto_aead_ctx(tfm)); 2067 2068 switch (authsize) { 2069 case ICV_8: 2070 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_DIV2; 2071 aeadctx->mayverify = VERIFY_HW; 2072 break; 2073 case ICV_12: 2074 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_IPSEC_96BIT; 2075 aeadctx->mayverify = VERIFY_HW; 2076 break; 2077 case ICV_16: 2078 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_NO_TRUNC; 2079 aeadctx->mayverify = VERIFY_HW; 2080 break; 2081 default: 2082 crypto_tfm_set_flags((struct crypto_tfm *)tfm, 2083 CRYPTO_TFM_RES_BAD_KEY_LEN); 2084 return -EINVAL; 2085 } 2086 return 0; 2087 } 2088 2089 static int chcr_ccm_setauthsize(struct crypto_aead *tfm, 2090 unsigned int authsize) 2091 { 2092 struct chcr_aead_ctx *aeadctx = AEAD_CTX(crypto_aead_ctx(tfm)); 2093 2094 switch (authsize) { 2095 case ICV_4: 2096 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_PL1; 2097 aeadctx->mayverify = VERIFY_HW; 2098 break; 2099 case ICV_6: 2100 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_PL2; 2101 aeadctx->mayverify = VERIFY_HW; 2102 break; 2103 case ICV_8: 2104 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_DIV2; 2105 aeadctx->mayverify = VERIFY_HW; 2106 break; 2107 case ICV_10: 2108 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_TRUNC_RFC4366; 2109 aeadctx->mayverify = VERIFY_HW; 2110 break; 2111 case ICV_12: 2112 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_IPSEC_96BIT; 2113 aeadctx->mayverify = VERIFY_HW; 2114 break; 2115 case ICV_14: 2116 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_PL3; 2117 aeadctx->mayverify = VERIFY_HW; 2118 break; 2119 case ICV_16: 2120 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_NO_TRUNC; 2121 aeadctx->mayverify = VERIFY_HW; 2122 break; 2123 default: 2124 crypto_tfm_set_flags((struct crypto_tfm *)tfm, 2125 CRYPTO_TFM_RES_BAD_KEY_LEN); 2126 return -EINVAL; 2127 } 2128 return 0; 2129 } 2130 2131 static int chcr_aead_ccm_setkey(struct crypto_aead *aead, 2132 const u8 *key, 2133 unsigned int keylen) 2134 { 2135 struct chcr_context *ctx = crypto_aead_ctx(aead); 2136 struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx); 2137 unsigned char ck_size, mk_size; 2138 int key_ctx_size = 0; 2139 2140 memcpy(aeadctx->key, key, keylen); 2141 aeadctx->enckey_len = keylen; 2142 key_ctx_size = sizeof(struct _key_ctx) + 2143 ((DIV_ROUND_UP(keylen, 16)) << 4) * 2; 2144 if (keylen == AES_KEYSIZE_128) { 2145 mk_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_128; 2146 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_128; 2147 } else if (keylen == AES_KEYSIZE_192) { 2148 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_192; 2149 mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_192; 2150 } else if (keylen == AES_KEYSIZE_256) { 2151 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_256; 2152 mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_256; 2153 } else { 2154 crypto_tfm_set_flags((struct crypto_tfm *)aead, 2155 CRYPTO_TFM_RES_BAD_KEY_LEN); 2156 aeadctx->enckey_len = 0; 2157 return -EINVAL; 2158 } 2159 aeadctx->key_ctx_hdr = FILL_KEY_CTX_HDR(ck_size, mk_size, 0, 0, 2160 key_ctx_size >> 4); 2161 return 0; 2162 } 2163 2164 static int chcr_aead_rfc4309_setkey(struct crypto_aead *aead, const u8 *key, 2165 unsigned int keylen) 2166 { 2167 struct chcr_context *ctx = crypto_aead_ctx(aead); 2168 struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx); 2169 2170 if (keylen < 3) { 2171 crypto_tfm_set_flags((struct crypto_tfm *)aead, 2172 CRYPTO_TFM_RES_BAD_KEY_LEN); 2173 aeadctx->enckey_len = 0; 2174 return -EINVAL; 2175 } 2176 keylen -= 3; 2177 memcpy(aeadctx->salt, key + keylen, 3); 2178 return chcr_aead_ccm_setkey(aead, key, keylen); 2179 } 2180 2181 static int chcr_gcm_setkey(struct crypto_aead *aead, const u8 *key, 2182 unsigned int keylen) 2183 { 2184 struct chcr_context *ctx = crypto_aead_ctx(aead); 2185 struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx); 2186 struct chcr_gcm_ctx *gctx = GCM_CTX(aeadctx); 2187 struct blkcipher_desc h_desc; 2188 struct scatterlist src[1]; 2189 unsigned int ck_size; 2190 int ret = 0, key_ctx_size = 0; 2191 2192 if (get_aead_subtype(aead) == 2193 CRYPTO_ALG_SUB_TYPE_AEAD_RFC4106) { 2194 keylen -= 4; /* nonce/salt is present in the last 4 bytes */ 2195 memcpy(aeadctx->salt, key + keylen, 4); 2196 } 2197 if (keylen == AES_KEYSIZE_128) { 2198 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_128; 2199 } else if (keylen == AES_KEYSIZE_192) { 2200 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_192; 2201 } else if (keylen == AES_KEYSIZE_256) { 2202 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_256; 2203 } else { 2204 crypto_tfm_set_flags((struct crypto_tfm *)aead, 2205 CRYPTO_TFM_RES_BAD_KEY_LEN); 2206 aeadctx->enckey_len = 0; 2207 pr_err("GCM: Invalid key length %d", keylen); 2208 ret = -EINVAL; 2209 goto out; 2210 } 2211 2212 memcpy(aeadctx->key, key, keylen); 2213 aeadctx->enckey_len = keylen; 2214 key_ctx_size = sizeof(struct _key_ctx) + 2215 ((DIV_ROUND_UP(keylen, 16)) << 4) + 2216 AEAD_H_SIZE; 2217 aeadctx->key_ctx_hdr = FILL_KEY_CTX_HDR(ck_size, 2218 CHCR_KEYCTX_MAC_KEY_SIZE_128, 2219 0, 0, 2220 key_ctx_size >> 4); 2221 /* Calculate the H = CIPH(K, 0 repeated 16 times) using sync aes 2222 * blkcipher It will go on key context 2223 */ 2224 h_desc.tfm = crypto_alloc_blkcipher("cbc(aes-generic)", 0, 0); 2225 if (IS_ERR(h_desc.tfm)) { 2226 aeadctx->enckey_len = 0; 2227 ret = -ENOMEM; 2228 goto out; 2229 } 2230 h_desc.flags = 0; 2231 ret = crypto_blkcipher_setkey(h_desc.tfm, key, keylen); 2232 if (ret) { 2233 aeadctx->enckey_len = 0; 2234 goto out1; 2235 } 2236 memset(gctx->ghash_h, 0, AEAD_H_SIZE); 2237 sg_init_one(&src[0], gctx->ghash_h, AEAD_H_SIZE); 2238 ret = crypto_blkcipher_encrypt(&h_desc, &src[0], &src[0], AEAD_H_SIZE); 2239 2240 out1: 2241 crypto_free_blkcipher(h_desc.tfm); 2242 out: 2243 return ret; 2244 } 2245 2246 static int chcr_authenc_setkey(struct crypto_aead *authenc, const u8 *key, 2247 unsigned int keylen) 2248 { 2249 struct chcr_context *ctx = crypto_aead_ctx(authenc); 2250 struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx); 2251 struct chcr_authenc_ctx *actx = AUTHENC_CTX(aeadctx); 2252 /* it contains auth and cipher key both*/ 2253 struct crypto_authenc_keys keys; 2254 unsigned int bs; 2255 unsigned int max_authsize = crypto_aead_alg(authenc)->maxauthsize; 2256 int err = 0, i, key_ctx_len = 0; 2257 unsigned char ck_size = 0; 2258 unsigned char pad[CHCR_HASH_MAX_BLOCK_SIZE_128] = { 0 }; 2259 struct crypto_shash *base_hash = NULL; 2260 struct algo_param param; 2261 int align; 2262 u8 *o_ptr = NULL; 2263 2264 if (crypto_authenc_extractkeys(&keys, key, keylen) != 0) { 2265 crypto_aead_set_flags(authenc, CRYPTO_TFM_RES_BAD_KEY_LEN); 2266 goto out; 2267 } 2268 2269 if (get_alg_config(¶m, max_authsize)) { 2270 pr_err("chcr : Unsupported digest size\n"); 2271 goto out; 2272 } 2273 if (keys.enckeylen == AES_KEYSIZE_128) { 2274 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_128; 2275 } else if (keys.enckeylen == AES_KEYSIZE_192) { 2276 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_192; 2277 } else if (keys.enckeylen == AES_KEYSIZE_256) { 2278 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_256; 2279 } else { 2280 pr_err("chcr : Unsupported cipher key\n"); 2281 goto out; 2282 } 2283 2284 /* Copy only encryption key. We use authkey to generate h(ipad) and 2285 * h(opad) so authkey is not needed again. authkeylen size have the 2286 * size of the hash digest size. 2287 */ 2288 memcpy(aeadctx->key, keys.enckey, keys.enckeylen); 2289 aeadctx->enckey_len = keys.enckeylen; 2290 get_aes_decrypt_key(actx->dec_rrkey, aeadctx->key, 2291 aeadctx->enckey_len << 3); 2292 2293 base_hash = chcr_alloc_shash(max_authsize); 2294 if (IS_ERR(base_hash)) { 2295 pr_err("chcr : Base driver cannot be loaded\n"); 2296 goto out; 2297 } 2298 { 2299 SHASH_DESC_ON_STACK(shash, base_hash); 2300 shash->tfm = base_hash; 2301 shash->flags = crypto_shash_get_flags(base_hash); 2302 bs = crypto_shash_blocksize(base_hash); 2303 align = KEYCTX_ALIGN_PAD(max_authsize); 2304 o_ptr = actx->h_iopad + param.result_size + align; 2305 2306 if (keys.authkeylen > bs) { 2307 err = crypto_shash_digest(shash, keys.authkey, 2308 keys.authkeylen, 2309 o_ptr); 2310 if (err) { 2311 pr_err("chcr : Base driver cannot be loaded\n"); 2312 goto out; 2313 } 2314 keys.authkeylen = max_authsize; 2315 } else 2316 memcpy(o_ptr, keys.authkey, keys.authkeylen); 2317 2318 /* Compute the ipad-digest*/ 2319 memset(pad + keys.authkeylen, 0, bs - keys.authkeylen); 2320 memcpy(pad, o_ptr, keys.authkeylen); 2321 for (i = 0; i < bs >> 2; i++) 2322 *((unsigned int *)pad + i) ^= IPAD_DATA; 2323 2324 if (chcr_compute_partial_hash(shash, pad, actx->h_iopad, 2325 max_authsize)) 2326 goto out; 2327 /* Compute the opad-digest */ 2328 memset(pad + keys.authkeylen, 0, bs - keys.authkeylen); 2329 memcpy(pad, o_ptr, keys.authkeylen); 2330 for (i = 0; i < bs >> 2; i++) 2331 *((unsigned int *)pad + i) ^= OPAD_DATA; 2332 2333 if (chcr_compute_partial_hash(shash, pad, o_ptr, max_authsize)) 2334 goto out; 2335 2336 /* convert the ipad and opad digest to network order */ 2337 chcr_change_order(actx->h_iopad, param.result_size); 2338 chcr_change_order(o_ptr, param.result_size); 2339 key_ctx_len = sizeof(struct _key_ctx) + 2340 ((DIV_ROUND_UP(keys.enckeylen, 16)) << 4) + 2341 (param.result_size + align) * 2; 2342 aeadctx->key_ctx_hdr = FILL_KEY_CTX_HDR(ck_size, param.mk_size, 2343 0, 1, key_ctx_len >> 4); 2344 actx->auth_mode = param.auth_mode; 2345 chcr_free_shash(base_hash); 2346 2347 return 0; 2348 } 2349 out: 2350 aeadctx->enckey_len = 0; 2351 if (base_hash) 2352 chcr_free_shash(base_hash); 2353 return -EINVAL; 2354 } 2355 2356 static int chcr_aead_digest_null_setkey(struct crypto_aead *authenc, 2357 const u8 *key, unsigned int keylen) 2358 { 2359 struct chcr_context *ctx = crypto_aead_ctx(authenc); 2360 struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx); 2361 struct chcr_authenc_ctx *actx = AUTHENC_CTX(aeadctx); 2362 struct crypto_authenc_keys keys; 2363 2364 /* it contains auth and cipher key both*/ 2365 int key_ctx_len = 0; 2366 unsigned char ck_size = 0; 2367 2368 if (crypto_authenc_extractkeys(&keys, key, keylen) != 0) { 2369 crypto_aead_set_flags(authenc, CRYPTO_TFM_RES_BAD_KEY_LEN); 2370 goto out; 2371 } 2372 if (keys.enckeylen == AES_KEYSIZE_128) { 2373 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_128; 2374 } else if (keys.enckeylen == AES_KEYSIZE_192) { 2375 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_192; 2376 } else if (keys.enckeylen == AES_KEYSIZE_256) { 2377 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_256; 2378 } else { 2379 pr_err("chcr : Unsupported cipher key\n"); 2380 goto out; 2381 } 2382 memcpy(aeadctx->key, keys.enckey, keys.enckeylen); 2383 aeadctx->enckey_len = keys.enckeylen; 2384 get_aes_decrypt_key(actx->dec_rrkey, aeadctx->key, 2385 aeadctx->enckey_len << 3); 2386 key_ctx_len = sizeof(struct _key_ctx) 2387 + ((DIV_ROUND_UP(keys.enckeylen, 16)) << 4); 2388 2389 aeadctx->key_ctx_hdr = FILL_KEY_CTX_HDR(ck_size, CHCR_KEYCTX_NO_KEY, 0, 2390 0, key_ctx_len >> 4); 2391 actx->auth_mode = CHCR_SCMD_AUTH_MODE_NOP; 2392 return 0; 2393 out: 2394 aeadctx->enckey_len = 0; 2395 return -EINVAL; 2396 } 2397 static int chcr_aead_encrypt(struct aead_request *req) 2398 { 2399 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 2400 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req); 2401 2402 reqctx->verify = VERIFY_HW; 2403 2404 switch (get_aead_subtype(tfm)) { 2405 case CRYPTO_ALG_SUB_TYPE_AEAD_AUTHENC: 2406 case CRYPTO_ALG_SUB_TYPE_AEAD_NULL: 2407 return chcr_aead_op(req, CHCR_ENCRYPT_OP, 0, 2408 create_authenc_wr); 2409 case CRYPTO_ALG_SUB_TYPE_AEAD_CCM: 2410 case CRYPTO_ALG_SUB_TYPE_AEAD_RFC4309: 2411 return chcr_aead_op(req, CHCR_ENCRYPT_OP, 0, 2412 create_aead_ccm_wr); 2413 default: 2414 return chcr_aead_op(req, CHCR_ENCRYPT_OP, 0, 2415 create_gcm_wr); 2416 } 2417 } 2418 2419 static int chcr_aead_decrypt(struct aead_request *req) 2420 { 2421 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 2422 struct chcr_aead_ctx *aeadctx = AEAD_CTX(crypto_aead_ctx(tfm)); 2423 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req); 2424 int size; 2425 2426 if (aeadctx->mayverify == VERIFY_SW) { 2427 size = crypto_aead_maxauthsize(tfm); 2428 reqctx->verify = VERIFY_SW; 2429 } else { 2430 size = 0; 2431 reqctx->verify = VERIFY_HW; 2432 } 2433 2434 switch (get_aead_subtype(tfm)) { 2435 case CRYPTO_ALG_SUB_TYPE_AEAD_AUTHENC: 2436 case CRYPTO_ALG_SUB_TYPE_AEAD_NULL: 2437 return chcr_aead_op(req, CHCR_DECRYPT_OP, size, 2438 create_authenc_wr); 2439 case CRYPTO_ALG_SUB_TYPE_AEAD_CCM: 2440 case CRYPTO_ALG_SUB_TYPE_AEAD_RFC4309: 2441 return chcr_aead_op(req, CHCR_DECRYPT_OP, size, 2442 create_aead_ccm_wr); 2443 default: 2444 return chcr_aead_op(req, CHCR_DECRYPT_OP, size, 2445 create_gcm_wr); 2446 } 2447 } 2448 2449 static int chcr_aead_op(struct aead_request *req, 2450 unsigned short op_type, 2451 int size, 2452 create_wr_t create_wr_fn) 2453 { 2454 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 2455 struct chcr_context *ctx = crypto_aead_ctx(tfm); 2456 struct uld_ctx *u_ctx = ULD_CTX(ctx); 2457 struct sk_buff *skb; 2458 2459 if (ctx && !ctx->dev) { 2460 pr_err("chcr : %s : No crypto device.\n", __func__); 2461 return -ENXIO; 2462 } 2463 if (cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0], 2464 ctx->tx_channel_id)) { 2465 if (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) 2466 return -EBUSY; 2467 } 2468 2469 /* Form a WR from req */ 2470 skb = create_wr_fn(req, u_ctx->lldi.rxq_ids[ctx->tx_channel_id], size, 2471 op_type); 2472 2473 if (IS_ERR(skb) || skb == NULL) { 2474 pr_err("chcr : %s : failed to form WR. No memory\n", __func__); 2475 return PTR_ERR(skb); 2476 } 2477 2478 skb->dev = u_ctx->lldi.ports[0]; 2479 set_wr_txq(skb, CPL_PRIORITY_DATA, ctx->tx_channel_id); 2480 chcr_send_wr(skb); 2481 return -EINPROGRESS; 2482 } 2483 static struct chcr_alg_template driver_algs[] = { 2484 /* AES-CBC */ 2485 { 2486 .type = CRYPTO_ALG_TYPE_ABLKCIPHER, 2487 .is_registered = 0, 2488 .alg.crypto = { 2489 .cra_name = "cbc(aes)", 2490 .cra_driver_name = "cbc-aes-chcr", 2491 .cra_priority = CHCR_CRA_PRIORITY, 2492 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER | 2493 CRYPTO_ALG_ASYNC, 2494 .cra_blocksize = AES_BLOCK_SIZE, 2495 .cra_ctxsize = sizeof(struct chcr_context) 2496 + sizeof(struct ablk_ctx), 2497 .cra_alignmask = 0, 2498 .cra_type = &crypto_ablkcipher_type, 2499 .cra_module = THIS_MODULE, 2500 .cra_init = chcr_cra_init, 2501 .cra_exit = NULL, 2502 .cra_u.ablkcipher = { 2503 .min_keysize = AES_MIN_KEY_SIZE, 2504 .max_keysize = AES_MAX_KEY_SIZE, 2505 .ivsize = AES_BLOCK_SIZE, 2506 .setkey = chcr_aes_cbc_setkey, 2507 .encrypt = chcr_aes_encrypt, 2508 .decrypt = chcr_aes_decrypt, 2509 } 2510 } 2511 }, 2512 { 2513 .type = CRYPTO_ALG_TYPE_ABLKCIPHER, 2514 .is_registered = 0, 2515 .alg.crypto = { 2516 .cra_name = "xts(aes)", 2517 .cra_driver_name = "xts-aes-chcr", 2518 .cra_priority = CHCR_CRA_PRIORITY, 2519 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER | 2520 CRYPTO_ALG_ASYNC, 2521 .cra_blocksize = AES_BLOCK_SIZE, 2522 .cra_ctxsize = sizeof(struct chcr_context) + 2523 sizeof(struct ablk_ctx), 2524 .cra_alignmask = 0, 2525 .cra_type = &crypto_ablkcipher_type, 2526 .cra_module = THIS_MODULE, 2527 .cra_init = chcr_cra_init, 2528 .cra_exit = NULL, 2529 .cra_u = { 2530 .ablkcipher = { 2531 .min_keysize = 2 * AES_MIN_KEY_SIZE, 2532 .max_keysize = 2 * AES_MAX_KEY_SIZE, 2533 .ivsize = AES_BLOCK_SIZE, 2534 .setkey = chcr_aes_xts_setkey, 2535 .encrypt = chcr_aes_encrypt, 2536 .decrypt = chcr_aes_decrypt, 2537 } 2538 } 2539 } 2540 }, 2541 /* SHA */ 2542 { 2543 .type = CRYPTO_ALG_TYPE_AHASH, 2544 .is_registered = 0, 2545 .alg.hash = { 2546 .halg.digestsize = SHA1_DIGEST_SIZE, 2547 .halg.base = { 2548 .cra_name = "sha1", 2549 .cra_driver_name = "sha1-chcr", 2550 .cra_blocksize = SHA1_BLOCK_SIZE, 2551 } 2552 } 2553 }, 2554 { 2555 .type = CRYPTO_ALG_TYPE_AHASH, 2556 .is_registered = 0, 2557 .alg.hash = { 2558 .halg.digestsize = SHA256_DIGEST_SIZE, 2559 .halg.base = { 2560 .cra_name = "sha256", 2561 .cra_driver_name = "sha256-chcr", 2562 .cra_blocksize = SHA256_BLOCK_SIZE, 2563 } 2564 } 2565 }, 2566 { 2567 .type = CRYPTO_ALG_TYPE_AHASH, 2568 .is_registered = 0, 2569 .alg.hash = { 2570 .halg.digestsize = SHA224_DIGEST_SIZE, 2571 .halg.base = { 2572 .cra_name = "sha224", 2573 .cra_driver_name = "sha224-chcr", 2574 .cra_blocksize = SHA224_BLOCK_SIZE, 2575 } 2576 } 2577 }, 2578 { 2579 .type = CRYPTO_ALG_TYPE_AHASH, 2580 .is_registered = 0, 2581 .alg.hash = { 2582 .halg.digestsize = SHA384_DIGEST_SIZE, 2583 .halg.base = { 2584 .cra_name = "sha384", 2585 .cra_driver_name = "sha384-chcr", 2586 .cra_blocksize = SHA384_BLOCK_SIZE, 2587 } 2588 } 2589 }, 2590 { 2591 .type = CRYPTO_ALG_TYPE_AHASH, 2592 .is_registered = 0, 2593 .alg.hash = { 2594 .halg.digestsize = SHA512_DIGEST_SIZE, 2595 .halg.base = { 2596 .cra_name = "sha512", 2597 .cra_driver_name = "sha512-chcr", 2598 .cra_blocksize = SHA512_BLOCK_SIZE, 2599 } 2600 } 2601 }, 2602 /* HMAC */ 2603 { 2604 .type = CRYPTO_ALG_TYPE_HMAC, 2605 .is_registered = 0, 2606 .alg.hash = { 2607 .halg.digestsize = SHA1_DIGEST_SIZE, 2608 .halg.base = { 2609 .cra_name = "hmac(sha1)", 2610 .cra_driver_name = "hmac-sha1-chcr", 2611 .cra_blocksize = SHA1_BLOCK_SIZE, 2612 } 2613 } 2614 }, 2615 { 2616 .type = CRYPTO_ALG_TYPE_HMAC, 2617 .is_registered = 0, 2618 .alg.hash = { 2619 .halg.digestsize = SHA224_DIGEST_SIZE, 2620 .halg.base = { 2621 .cra_name = "hmac(sha224)", 2622 .cra_driver_name = "hmac-sha224-chcr", 2623 .cra_blocksize = SHA224_BLOCK_SIZE, 2624 } 2625 } 2626 }, 2627 { 2628 .type = CRYPTO_ALG_TYPE_HMAC, 2629 .is_registered = 0, 2630 .alg.hash = { 2631 .halg.digestsize = SHA256_DIGEST_SIZE, 2632 .halg.base = { 2633 .cra_name = "hmac(sha256)", 2634 .cra_driver_name = "hmac-sha256-chcr", 2635 .cra_blocksize = SHA256_BLOCK_SIZE, 2636 } 2637 } 2638 }, 2639 { 2640 .type = CRYPTO_ALG_TYPE_HMAC, 2641 .is_registered = 0, 2642 .alg.hash = { 2643 .halg.digestsize = SHA384_DIGEST_SIZE, 2644 .halg.base = { 2645 .cra_name = "hmac(sha384)", 2646 .cra_driver_name = "hmac-sha384-chcr", 2647 .cra_blocksize = SHA384_BLOCK_SIZE, 2648 } 2649 } 2650 }, 2651 { 2652 .type = CRYPTO_ALG_TYPE_HMAC, 2653 .is_registered = 0, 2654 .alg.hash = { 2655 .halg.digestsize = SHA512_DIGEST_SIZE, 2656 .halg.base = { 2657 .cra_name = "hmac(sha512)", 2658 .cra_driver_name = "hmac-sha512-chcr", 2659 .cra_blocksize = SHA512_BLOCK_SIZE, 2660 } 2661 } 2662 }, 2663 /* Add AEAD Algorithms */ 2664 { 2665 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_AEAD_GCM, 2666 .is_registered = 0, 2667 .alg.aead = { 2668 .base = { 2669 .cra_name = "gcm(aes)", 2670 .cra_driver_name = "gcm-aes-chcr", 2671 .cra_blocksize = 1, 2672 .cra_ctxsize = sizeof(struct chcr_context) + 2673 sizeof(struct chcr_aead_ctx) + 2674 sizeof(struct chcr_gcm_ctx), 2675 }, 2676 .ivsize = 12, 2677 .maxauthsize = GHASH_DIGEST_SIZE, 2678 .setkey = chcr_gcm_setkey, 2679 .setauthsize = chcr_gcm_setauthsize, 2680 } 2681 }, 2682 { 2683 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_AEAD_RFC4106, 2684 .is_registered = 0, 2685 .alg.aead = { 2686 .base = { 2687 .cra_name = "rfc4106(gcm(aes))", 2688 .cra_driver_name = "rfc4106-gcm-aes-chcr", 2689 .cra_blocksize = 1, 2690 .cra_ctxsize = sizeof(struct chcr_context) + 2691 sizeof(struct chcr_aead_ctx) + 2692 sizeof(struct chcr_gcm_ctx), 2693 2694 }, 2695 .ivsize = 8, 2696 .maxauthsize = GHASH_DIGEST_SIZE, 2697 .setkey = chcr_gcm_setkey, 2698 .setauthsize = chcr_4106_4309_setauthsize, 2699 } 2700 }, 2701 { 2702 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_AEAD_CCM, 2703 .is_registered = 0, 2704 .alg.aead = { 2705 .base = { 2706 .cra_name = "ccm(aes)", 2707 .cra_driver_name = "ccm-aes-chcr", 2708 .cra_blocksize = 1, 2709 .cra_ctxsize = sizeof(struct chcr_context) + 2710 sizeof(struct chcr_aead_ctx), 2711 2712 }, 2713 .ivsize = AES_BLOCK_SIZE, 2714 .maxauthsize = GHASH_DIGEST_SIZE, 2715 .setkey = chcr_aead_ccm_setkey, 2716 .setauthsize = chcr_ccm_setauthsize, 2717 } 2718 }, 2719 { 2720 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_AEAD_RFC4309, 2721 .is_registered = 0, 2722 .alg.aead = { 2723 .base = { 2724 .cra_name = "rfc4309(ccm(aes))", 2725 .cra_driver_name = "rfc4309-ccm-aes-chcr", 2726 .cra_blocksize = 1, 2727 .cra_ctxsize = sizeof(struct chcr_context) + 2728 sizeof(struct chcr_aead_ctx), 2729 2730 }, 2731 .ivsize = 8, 2732 .maxauthsize = GHASH_DIGEST_SIZE, 2733 .setkey = chcr_aead_rfc4309_setkey, 2734 .setauthsize = chcr_4106_4309_setauthsize, 2735 } 2736 }, 2737 { 2738 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_AEAD_AUTHENC, 2739 .is_registered = 0, 2740 .alg.aead = { 2741 .base = { 2742 .cra_name = "authenc(hmac(sha1),cbc(aes))", 2743 .cra_driver_name = 2744 "authenc-hmac-sha1-cbc-aes-chcr", 2745 .cra_blocksize = AES_BLOCK_SIZE, 2746 .cra_ctxsize = sizeof(struct chcr_context) + 2747 sizeof(struct chcr_aead_ctx) + 2748 sizeof(struct chcr_authenc_ctx), 2749 2750 }, 2751 .ivsize = AES_BLOCK_SIZE, 2752 .maxauthsize = SHA1_DIGEST_SIZE, 2753 .setkey = chcr_authenc_setkey, 2754 .setauthsize = chcr_authenc_setauthsize, 2755 } 2756 }, 2757 { 2758 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_AEAD_AUTHENC, 2759 .is_registered = 0, 2760 .alg.aead = { 2761 .base = { 2762 2763 .cra_name = "authenc(hmac(sha256),cbc(aes))", 2764 .cra_driver_name = 2765 "authenc-hmac-sha256-cbc-aes-chcr", 2766 .cra_blocksize = AES_BLOCK_SIZE, 2767 .cra_ctxsize = sizeof(struct chcr_context) + 2768 sizeof(struct chcr_aead_ctx) + 2769 sizeof(struct chcr_authenc_ctx), 2770 2771 }, 2772 .ivsize = AES_BLOCK_SIZE, 2773 .maxauthsize = SHA256_DIGEST_SIZE, 2774 .setkey = chcr_authenc_setkey, 2775 .setauthsize = chcr_authenc_setauthsize, 2776 } 2777 }, 2778 { 2779 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_AEAD_AUTHENC, 2780 .is_registered = 0, 2781 .alg.aead = { 2782 .base = { 2783 .cra_name = "authenc(hmac(sha224),cbc(aes))", 2784 .cra_driver_name = 2785 "authenc-hmac-sha224-cbc-aes-chcr", 2786 .cra_blocksize = AES_BLOCK_SIZE, 2787 .cra_ctxsize = sizeof(struct chcr_context) + 2788 sizeof(struct chcr_aead_ctx) + 2789 sizeof(struct chcr_authenc_ctx), 2790 }, 2791 .ivsize = AES_BLOCK_SIZE, 2792 .maxauthsize = SHA224_DIGEST_SIZE, 2793 .setkey = chcr_authenc_setkey, 2794 .setauthsize = chcr_authenc_setauthsize, 2795 } 2796 }, 2797 { 2798 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_AEAD_AUTHENC, 2799 .is_registered = 0, 2800 .alg.aead = { 2801 .base = { 2802 .cra_name = "authenc(hmac(sha384),cbc(aes))", 2803 .cra_driver_name = 2804 "authenc-hmac-sha384-cbc-aes-chcr", 2805 .cra_blocksize = AES_BLOCK_SIZE, 2806 .cra_ctxsize = sizeof(struct chcr_context) + 2807 sizeof(struct chcr_aead_ctx) + 2808 sizeof(struct chcr_authenc_ctx), 2809 2810 }, 2811 .ivsize = AES_BLOCK_SIZE, 2812 .maxauthsize = SHA384_DIGEST_SIZE, 2813 .setkey = chcr_authenc_setkey, 2814 .setauthsize = chcr_authenc_setauthsize, 2815 } 2816 }, 2817 { 2818 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_AEAD_AUTHENC, 2819 .is_registered = 0, 2820 .alg.aead = { 2821 .base = { 2822 .cra_name = "authenc(hmac(sha512),cbc(aes))", 2823 .cra_driver_name = 2824 "authenc-hmac-sha512-cbc-aes-chcr", 2825 .cra_blocksize = AES_BLOCK_SIZE, 2826 .cra_ctxsize = sizeof(struct chcr_context) + 2827 sizeof(struct chcr_aead_ctx) + 2828 sizeof(struct chcr_authenc_ctx), 2829 2830 }, 2831 .ivsize = AES_BLOCK_SIZE, 2832 .maxauthsize = SHA512_DIGEST_SIZE, 2833 .setkey = chcr_authenc_setkey, 2834 .setauthsize = chcr_authenc_setauthsize, 2835 } 2836 }, 2837 { 2838 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_AEAD_NULL, 2839 .is_registered = 0, 2840 .alg.aead = { 2841 .base = { 2842 .cra_name = "authenc(digest_null,cbc(aes))", 2843 .cra_driver_name = 2844 "authenc-digest_null-cbc-aes-chcr", 2845 .cra_blocksize = AES_BLOCK_SIZE, 2846 .cra_ctxsize = sizeof(struct chcr_context) + 2847 sizeof(struct chcr_aead_ctx) + 2848 sizeof(struct chcr_authenc_ctx), 2849 2850 }, 2851 .ivsize = AES_BLOCK_SIZE, 2852 .maxauthsize = 0, 2853 .setkey = chcr_aead_digest_null_setkey, 2854 .setauthsize = chcr_authenc_null_setauthsize, 2855 } 2856 }, 2857 }; 2858 2859 /* 2860 * chcr_unregister_alg - Deregister crypto algorithms with 2861 * kernel framework. 2862 */ 2863 static int chcr_unregister_alg(void) 2864 { 2865 int i; 2866 2867 for (i = 0; i < ARRAY_SIZE(driver_algs); i++) { 2868 switch (driver_algs[i].type & CRYPTO_ALG_TYPE_MASK) { 2869 case CRYPTO_ALG_TYPE_ABLKCIPHER: 2870 if (driver_algs[i].is_registered) 2871 crypto_unregister_alg( 2872 &driver_algs[i].alg.crypto); 2873 break; 2874 case CRYPTO_ALG_TYPE_AEAD: 2875 if (driver_algs[i].is_registered) 2876 crypto_unregister_aead( 2877 &driver_algs[i].alg.aead); 2878 break; 2879 case CRYPTO_ALG_TYPE_AHASH: 2880 if (driver_algs[i].is_registered) 2881 crypto_unregister_ahash( 2882 &driver_algs[i].alg.hash); 2883 break; 2884 } 2885 driver_algs[i].is_registered = 0; 2886 } 2887 return 0; 2888 } 2889 2890 #define SZ_AHASH_CTX sizeof(struct chcr_context) 2891 #define SZ_AHASH_H_CTX (sizeof(struct chcr_context) + sizeof(struct hmac_ctx)) 2892 #define SZ_AHASH_REQ_CTX sizeof(struct chcr_ahash_req_ctx) 2893 #define AHASH_CRA_FLAGS (CRYPTO_ALG_TYPE_AHASH | CRYPTO_ALG_ASYNC) 2894 2895 /* 2896 * chcr_register_alg - Register crypto algorithms with kernel framework. 2897 */ 2898 static int chcr_register_alg(void) 2899 { 2900 struct crypto_alg ai; 2901 struct ahash_alg *a_hash; 2902 int err = 0, i; 2903 char *name = NULL; 2904 2905 for (i = 0; i < ARRAY_SIZE(driver_algs); i++) { 2906 if (driver_algs[i].is_registered) 2907 continue; 2908 switch (driver_algs[i].type & CRYPTO_ALG_TYPE_MASK) { 2909 case CRYPTO_ALG_TYPE_ABLKCIPHER: 2910 err = crypto_register_alg(&driver_algs[i].alg.crypto); 2911 name = driver_algs[i].alg.crypto.cra_driver_name; 2912 break; 2913 case CRYPTO_ALG_TYPE_AEAD: 2914 driver_algs[i].alg.aead.base.cra_priority = 2915 CHCR_CRA_PRIORITY; 2916 driver_algs[i].alg.aead.base.cra_flags = 2917 CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC; 2918 driver_algs[i].alg.aead.encrypt = chcr_aead_encrypt; 2919 driver_algs[i].alg.aead.decrypt = chcr_aead_decrypt; 2920 driver_algs[i].alg.aead.init = chcr_aead_cra_init; 2921 driver_algs[i].alg.aead.exit = chcr_aead_cra_exit; 2922 driver_algs[i].alg.aead.base.cra_module = THIS_MODULE; 2923 err = crypto_register_aead(&driver_algs[i].alg.aead); 2924 name = driver_algs[i].alg.aead.base.cra_driver_name; 2925 break; 2926 case CRYPTO_ALG_TYPE_AHASH: 2927 a_hash = &driver_algs[i].alg.hash; 2928 a_hash->update = chcr_ahash_update; 2929 a_hash->final = chcr_ahash_final; 2930 a_hash->finup = chcr_ahash_finup; 2931 a_hash->digest = chcr_ahash_digest; 2932 a_hash->export = chcr_ahash_export; 2933 a_hash->import = chcr_ahash_import; 2934 a_hash->halg.statesize = SZ_AHASH_REQ_CTX; 2935 a_hash->halg.base.cra_priority = CHCR_CRA_PRIORITY; 2936 a_hash->halg.base.cra_module = THIS_MODULE; 2937 a_hash->halg.base.cra_flags = AHASH_CRA_FLAGS; 2938 a_hash->halg.base.cra_alignmask = 0; 2939 a_hash->halg.base.cra_exit = NULL; 2940 a_hash->halg.base.cra_type = &crypto_ahash_type; 2941 2942 if (driver_algs[i].type == CRYPTO_ALG_TYPE_HMAC) { 2943 a_hash->halg.base.cra_init = chcr_hmac_cra_init; 2944 a_hash->halg.base.cra_exit = chcr_hmac_cra_exit; 2945 a_hash->init = chcr_hmac_init; 2946 a_hash->setkey = chcr_ahash_setkey; 2947 a_hash->halg.base.cra_ctxsize = SZ_AHASH_H_CTX; 2948 } else { 2949 a_hash->init = chcr_sha_init; 2950 a_hash->halg.base.cra_ctxsize = SZ_AHASH_CTX; 2951 a_hash->halg.base.cra_init = chcr_sha_cra_init; 2952 } 2953 err = crypto_register_ahash(&driver_algs[i].alg.hash); 2954 ai = driver_algs[i].alg.hash.halg.base; 2955 name = ai.cra_driver_name; 2956 break; 2957 } 2958 if (err) { 2959 pr_err("chcr : %s : Algorithm registration failed\n", 2960 name); 2961 goto register_err; 2962 } else { 2963 driver_algs[i].is_registered = 1; 2964 } 2965 } 2966 return 0; 2967 2968 register_err: 2969 chcr_unregister_alg(); 2970 return err; 2971 } 2972 2973 /* 2974 * start_crypto - Register the crypto algorithms. 2975 * This should called once when the first device comesup. After this 2976 * kernel will start calling driver APIs for crypto operations. 2977 */ 2978 int start_crypto(void) 2979 { 2980 return chcr_register_alg(); 2981 } 2982 2983 /* 2984 * stop_crypto - Deregister all the crypto algorithms with kernel. 2985 * This should be called once when the last device goes down. After this 2986 * kernel will not call the driver API for crypto operations. 2987 */ 2988 int stop_crypto(void) 2989 { 2990 chcr_unregister_alg(); 2991 return 0; 2992 } 2993