1 /* SPDX-License-Identifier: GPL-2.0 */ 2 #ifndef __NITROX_REQ_H 3 #define __NITROX_REQ_H 4 5 #include <linux/dma-mapping.h> 6 #include <crypto/aes.h> 7 8 #include "nitrox_dev.h" 9 10 #define PENDING_SIG 0xFFFFFFFFFFFFFFFFUL 11 #define PRIO 4001 12 13 typedef void (*sereq_completion_t)(void *req, int err); 14 15 /** 16 * struct gphdr - General purpose Header 17 * @param0: first parameter. 18 * @param1: second parameter. 19 * @param2: third parameter. 20 * @param3: fourth parameter. 21 * 22 * Params tell the iv and enc/dec data offsets. 23 */ 24 struct gphdr { 25 __be16 param0; 26 __be16 param1; 27 __be16 param2; 28 __be16 param3; 29 }; 30 31 /** 32 * struct se_req_ctrl - SE request information. 33 * @arg: Minor number of the opcode 34 * @ctxc: Context control. 35 * @unca: Uncertainity enabled. 36 * @info: Additional information for SE cores. 37 * @ctxl: Context length in bytes. 38 * @uddl: User defined data length 39 */ 40 union se_req_ctrl { 41 u64 value; 42 struct { 43 u64 raz : 22; 44 u64 arg : 8; 45 u64 ctxc : 2; 46 u64 unca : 1; 47 u64 info : 3; 48 u64 unc : 8; 49 u64 ctxl : 12; 50 u64 uddl : 8; 51 } s; 52 }; 53 54 #define MAX_IV_LEN 16 55 56 /** 57 * struct se_crypto_request - SE crypto request structure. 58 * @opcode: Request opcode (enc/dec) 59 * @flags: flags from crypto subsystem 60 * @ctx_handle: Crypto context handle. 61 * @gph: GP Header 62 * @ctrl: Request Information. 63 * @orh: ORH address 64 * @comp: completion address 65 * @src: Input sglist 66 * @dst: Output sglist 67 */ 68 struct se_crypto_request { 69 u8 opcode; 70 gfp_t gfp; 71 u32 flags; 72 u64 ctx_handle; 73 74 struct gphdr gph; 75 union se_req_ctrl ctrl; 76 u64 *orh; 77 u64 *comp; 78 79 struct scatterlist *src; 80 struct scatterlist *dst; 81 }; 82 83 /* Crypto opcodes */ 84 #define FLEXI_CRYPTO_ENCRYPT_HMAC 0x33 85 #define ENCRYPT 0 86 #define DECRYPT 1 87 88 /* IV from context */ 89 #define IV_FROM_CTX 0 90 /* IV from Input data */ 91 #define IV_FROM_DPTR 1 92 93 /** 94 * cipher opcodes for firmware 95 */ 96 enum flexi_cipher { 97 CIPHER_NULL = 0, 98 CIPHER_3DES_CBC, 99 CIPHER_3DES_ECB, 100 CIPHER_AES_CBC, 101 CIPHER_AES_ECB, 102 CIPHER_AES_CFB, 103 CIPHER_AES_CTR, 104 CIPHER_AES_GCM, 105 CIPHER_AES_XTS, 106 CIPHER_AES_CCM, 107 CIPHER_AES_CBC_CTS, 108 CIPHER_AES_ECB_CTS, 109 CIPHER_INVALID 110 }; 111 112 enum flexi_auth { 113 AUTH_NULL = 0, 114 AUTH_MD5, 115 AUTH_SHA1, 116 AUTH_SHA2_SHA224, 117 AUTH_SHA2_SHA256, 118 AUTH_SHA2_SHA384, 119 AUTH_SHA2_SHA512, 120 AUTH_GMAC, 121 AUTH_INVALID 122 }; 123 124 /** 125 * struct crypto_keys - Crypto keys 126 * @key: Encryption key or KEY1 for AES-XTS 127 * @iv: Encryption IV or Tweak for AES-XTS 128 */ 129 struct crypto_keys { 130 union { 131 u8 key[AES_MAX_KEY_SIZE]; 132 u8 key1[AES_MAX_KEY_SIZE]; 133 } u; 134 u8 iv[AES_BLOCK_SIZE]; 135 }; 136 137 /** 138 * struct auth_keys - Authentication keys 139 * @ipad: IPAD or KEY2 for AES-XTS 140 * @opad: OPAD or AUTH KEY if auth_input_type = 1 141 */ 142 struct auth_keys { 143 union { 144 u8 ipad[64]; 145 u8 key2[64]; 146 } u; 147 u8 opad[64]; 148 }; 149 150 union fc_ctx_flags { 151 __be64 f; 152 u64 fu; 153 struct { 154 #if defined(__BIG_ENDIAN_BITFIELD) 155 u64 cipher_type : 4; 156 u64 reserved_59 : 1; 157 u64 aes_keylen : 2; 158 u64 iv_source : 1; 159 u64 hash_type : 4; 160 u64 reserved_49_51 : 3; 161 u64 auth_input_type: 1; 162 u64 mac_len : 8; 163 u64 reserved_0_39 : 40; 164 #else 165 u64 reserved_0_39 : 40; 166 u64 mac_len : 8; 167 u64 auth_input_type: 1; 168 u64 reserved_49_51 : 3; 169 u64 hash_type : 4; 170 u64 iv_source : 1; 171 u64 aes_keylen : 2; 172 u64 reserved_59 : 1; 173 u64 cipher_type : 4; 174 #endif 175 } w0; 176 }; 177 /** 178 * struct flexi_crypto_context - Crypto context 179 * @cipher_type: Encryption cipher type 180 * @aes_keylen: AES key length 181 * @iv_source: Encryption IV source 182 * @hash_type: Authentication type 183 * @auth_input_type: Authentication input type 184 * 1 - Authentication IV and KEY, microcode calculates OPAD/IPAD 185 * 0 - Authentication OPAD/IPAD 186 * @mac_len: mac length 187 * @crypto: Crypto keys 188 * @auth: Authentication keys 189 */ 190 struct flexi_crypto_context { 191 union fc_ctx_flags flags; 192 struct crypto_keys crypto; 193 struct auth_keys auth; 194 }; 195 196 struct crypto_ctx_hdr { 197 struct dma_pool *pool; 198 dma_addr_t dma; 199 void *vaddr; 200 }; 201 202 struct nitrox_crypto_ctx { 203 struct nitrox_device *ndev; 204 union { 205 u64 ctx_handle; 206 struct flexi_crypto_context *fctx; 207 } u; 208 struct crypto_ctx_hdr *chdr; 209 sereq_completion_t callback; 210 }; 211 212 struct nitrox_kcrypt_request { 213 struct se_crypto_request creq; 214 u8 *src; 215 u8 *dst; 216 u8 *iv_out; 217 }; 218 219 /** 220 * struct nitrox_aead_rctx - AEAD request context 221 * @nkreq: Base request context 222 * @cryptlen: Encryption/Decryption data length 223 * @assoclen: AAD length 224 * @srclen: Input buffer length 225 * @dstlen: Output buffer length 226 * @iv: IV data 227 * @ivsize: IV data length 228 * @flags: AEAD req flags 229 * @ctx_handle: Device context handle 230 * @src: Source sglist 231 * @dst: Destination sglist 232 * @ctrl_arg: Identifies the request type (ENCRYPT/DECRYPT) 233 */ 234 struct nitrox_aead_rctx { 235 struct nitrox_kcrypt_request nkreq; 236 unsigned int cryptlen; 237 unsigned int assoclen; 238 unsigned int srclen; 239 unsigned int dstlen; 240 u8 *iv; 241 int ivsize; 242 u32 flags; 243 u64 ctx_handle; 244 struct scatterlist *src; 245 struct scatterlist *dst; 246 u8 ctrl_arg; 247 }; 248 249 /** 250 * struct nitrox_rfc4106_rctx - rfc4106 cipher request context 251 * @base: AEAD request context 252 * @src: Source sglist 253 * @dst: Destination sglist 254 * @assoc: AAD 255 */ 256 struct nitrox_rfc4106_rctx { 257 struct nitrox_aead_rctx base; 258 struct scatterlist src[3]; 259 struct scatterlist dst[3]; 260 u8 assoc[20]; 261 }; 262 263 /** 264 * struct pkt_instr_hdr - Packet Instruction Header 265 * @g: Gather used 266 * When [G] is set and [GSZ] != 0, the instruction is 267 * indirect gather instruction. 268 * When [G] is set and [GSZ] = 0, the instruction is 269 * direct gather instruction. 270 * @gsz: Number of pointers in the indirect gather list 271 * @ihi: When set hardware duplicates the 1st 8 bytes of pkt_instr_hdr 272 * and adds them to the packet after the pkt_instr_hdr but before any UDD 273 * @ssz: Not used by the input hardware. But can become slc_store_int[SSZ] 274 * when [IHI] is set. 275 * @fsz: The number of front data bytes directly included in the 276 * PCIe instruction. 277 * @tlen: The length of the input packet in bytes, include: 278 * - 16B pkt_hdr 279 * - Inline context bytes if any, 280 * - UDD if any, 281 * - packet payload bytes 282 */ 283 union pkt_instr_hdr { 284 __be64 bev; 285 u64 value; 286 struct { 287 #if defined(__BIG_ENDIAN_BITFIELD) 288 u64 raz_48_63 : 16; 289 u64 g : 1; 290 u64 gsz : 7; 291 u64 ihi : 1; 292 u64 ssz : 7; 293 u64 raz_30_31 : 2; 294 u64 fsz : 6; 295 u64 raz_16_23 : 8; 296 u64 tlen : 16; 297 #else 298 u64 tlen : 16; 299 u64 raz_16_23 : 8; 300 u64 fsz : 6; 301 u64 raz_30_31 : 2; 302 u64 ssz : 7; 303 u64 ihi : 1; 304 u64 gsz : 7; 305 u64 g : 1; 306 u64 raz_48_63 : 16; 307 #endif 308 } s; 309 }; 310 311 /** 312 * struct pkt_hdr - Packet Input Header 313 * @opcode: Request opcode (Major) 314 * @arg: Request opcode (Minor) 315 * @ctxc: Context control. 316 * @unca: When set [UNC] is the uncertainty count for an input packet. 317 * The hardware uses uncertainty counts to predict 318 * output buffer use and avoid deadlock. 319 * @info: Not used by input hardware. Available for use 320 * during SE processing. 321 * @destport: The expected destination port/ring/channel for the packet. 322 * @unc: Uncertainty count for an input packet. 323 * @grp: SE group that will process the input packet. 324 * @ctxl: Context Length in 64-bit words. 325 * @uddl: User-defined data (UDD) length in bytes. 326 * @ctxp: Context pointer. CTXP<63,2:0> must be zero in all cases. 327 */ 328 union pkt_hdr { 329 __be64 bev[2]; 330 u64 value[2]; 331 struct { 332 #if defined(__BIG_ENDIAN_BITFIELD) 333 u64 opcode : 8; 334 u64 arg : 8; 335 u64 ctxc : 2; 336 u64 unca : 1; 337 u64 raz_44 : 1; 338 u64 info : 3; 339 u64 destport : 9; 340 u64 unc : 8; 341 u64 raz_19_23 : 5; 342 u64 grp : 3; 343 u64 raz_15 : 1; 344 u64 ctxl : 7; 345 u64 uddl : 8; 346 #else 347 u64 uddl : 8; 348 u64 ctxl : 7; 349 u64 raz_15 : 1; 350 u64 grp : 3; 351 u64 raz_19_23 : 5; 352 u64 unc : 8; 353 u64 destport : 9; 354 u64 info : 3; 355 u64 raz_44 : 1; 356 u64 unca : 1; 357 u64 ctxc : 2; 358 u64 arg : 8; 359 u64 opcode : 8; 360 #endif 361 __be64 ctxp; 362 } s; 363 }; 364 365 /** 366 * struct slc_store_info - Solicited Paceket Output Store Information. 367 * @ssz: The number of scatterlist pointers for the solicited output port 368 * packet. 369 * @rptr: The result pointer for the solicited output port packet. 370 * If [SSZ]=0, [RPTR] must point directly to a buffer on the remote 371 * host that is large enough to hold the entire output packet. 372 * If [SSZ]!=0, [RPTR] must point to an array of ([SSZ]+3)/4 373 * sglist components at [RPTR] on the remote host. 374 */ 375 union slc_store_info { 376 __be64 bev[2]; 377 u64 value[2]; 378 struct { 379 #if defined(__BIG_ENDIAN_BITFIELD) 380 u64 raz_39_63 : 25; 381 u64 ssz : 7; 382 u64 raz_0_31 : 32; 383 #else 384 u64 raz_0_31 : 32; 385 u64 ssz : 7; 386 u64 raz_39_63 : 25; 387 #endif 388 __be64 rptr; 389 } s; 390 }; 391 392 /** 393 * struct nps_pkt_instr - NPS Packet Instruction of SE cores. 394 * @dptr0 : Input pointer points to buffer in remote host. 395 * @ih: Packet Instruction Header (8 bytes) 396 * @irh: Packet Input Header (16 bytes) 397 * @slc: Solicited Packet Output Store Information (16 bytes) 398 * @fdata: Front data 399 * 400 * 64-Byte Instruction Format 401 */ 402 struct nps_pkt_instr { 403 __be64 dptr0; 404 union pkt_instr_hdr ih; 405 union pkt_hdr irh; 406 union slc_store_info slc; 407 u64 fdata[2]; 408 }; 409 410 /** 411 * struct aqmq_command_s - The 32 byte command for AE processing. 412 * @opcode: Request opcode 413 * @param1: Request control parameter 1 414 * @param2: Request control parameter 2 415 * @dlen: Input length 416 * @dptr: Input pointer points to buffer in remote host 417 * @rptr: Result pointer points to buffer in remote host 418 * @grp: AQM Group (0..7) 419 * @cptr: Context pointer 420 */ 421 struct aqmq_command_s { 422 __be16 opcode; 423 __be16 param1; 424 __be16 param2; 425 __be16 dlen; 426 __be64 dptr; 427 __be64 rptr; 428 union { 429 __be64 word3; 430 #if defined(__BIG_ENDIAN_BITFIELD) 431 u64 grp : 3; 432 u64 cptr : 61; 433 #else 434 u64 cptr : 61; 435 u64 grp : 3; 436 #endif 437 }; 438 }; 439 440 /** 441 * struct ctx_hdr - Book keeping data about the crypto context 442 * @pool: Pool used to allocate crypto context 443 * @dma: Base DMA address of the cypto context 444 * @ctx_dma: Actual usable crypto context for NITROX 445 */ 446 struct ctx_hdr { 447 struct dma_pool *pool; 448 dma_addr_t dma; 449 dma_addr_t ctx_dma; 450 }; 451 452 /* 453 * struct sglist_component - SG list component format 454 * @len0: The number of bytes at [PTR0] on the remote host. 455 * @len1: The number of bytes at [PTR1] on the remote host. 456 * @len2: The number of bytes at [PTR2] on the remote host. 457 * @len3: The number of bytes at [PTR3] on the remote host. 458 * @dma0: First pointer point to buffer in remote host. 459 * @dma1: Second pointer point to buffer in remote host. 460 * @dma2: Third pointer point to buffer in remote host. 461 * @dma3: Fourth pointer point to buffer in remote host. 462 */ 463 struct nitrox_sgcomp { 464 __be16 len[4]; 465 __be64 dma[4]; 466 }; 467 468 /* 469 * strutct nitrox_sgtable - SG list information 470 * @sgmap_cnt: Number of buffers mapped 471 * @total_bytes: Total bytes in sglist. 472 * @sgcomp_len: Total sglist components length. 473 * @sgcomp_dma: DMA address of sglist component. 474 * @sg: crypto request buffer. 475 * @sgcomp: sglist component for NITROX. 476 */ 477 struct nitrox_sgtable { 478 u8 sgmap_cnt; 479 u16 total_bytes; 480 u32 sgcomp_len; 481 dma_addr_t sgcomp_dma; 482 struct scatterlist *sg; 483 struct nitrox_sgcomp *sgcomp; 484 }; 485 486 /* Response Header Length */ 487 #define ORH_HLEN 8 488 /* Completion bytes Length */ 489 #define COMP_HLEN 8 490 491 struct resp_hdr { 492 u64 *orh; 493 u64 *completion; 494 }; 495 496 typedef void (*completion_t)(void *arg, int err); 497 498 /** 499 * struct nitrox_softreq - Represents the NIROX Request. 500 * @response: response list entry 501 * @backlog: Backlog list entry 502 * @ndev: Device used to submit the request 503 * @cmdq: Command queue for submission 504 * @resp: Response headers 505 * @instr: 64B instruction 506 * @in: SG table for input 507 * @out SG table for output 508 * @tstamp: Request submitted time in jiffies 509 * @callback: callback after request completion/timeout 510 * @cb_arg: callback argument 511 */ 512 struct nitrox_softreq { 513 struct list_head response; 514 struct list_head backlog; 515 516 u32 flags; 517 gfp_t gfp; 518 atomic_t status; 519 520 struct nitrox_device *ndev; 521 struct nitrox_cmdq *cmdq; 522 523 struct nps_pkt_instr instr; 524 struct resp_hdr resp; 525 struct nitrox_sgtable in; 526 struct nitrox_sgtable out; 527 528 unsigned long tstamp; 529 530 completion_t callback; 531 void *cb_arg; 532 }; 533 534 static inline int flexi_aes_keylen(int keylen) 535 { 536 int aes_keylen; 537 538 switch (keylen) { 539 case AES_KEYSIZE_128: 540 aes_keylen = 1; 541 break; 542 case AES_KEYSIZE_192: 543 aes_keylen = 2; 544 break; 545 case AES_KEYSIZE_256: 546 aes_keylen = 3; 547 break; 548 default: 549 aes_keylen = -EINVAL; 550 break; 551 } 552 return aes_keylen; 553 } 554 555 static inline void *alloc_req_buf(int nents, int extralen, gfp_t gfp) 556 { 557 size_t size; 558 559 size = sizeof(struct scatterlist) * nents; 560 size += extralen; 561 562 return kzalloc(size, gfp); 563 } 564 565 /** 566 * create_single_sg - Point SG entry to the data 567 * @sg: Destination SG list 568 * @buf: Data 569 * @buflen: Data length 570 * 571 * Returns next free entry in the destination SG list 572 **/ 573 static inline struct scatterlist *create_single_sg(struct scatterlist *sg, 574 void *buf, int buflen) 575 { 576 sg_set_buf(sg, buf, buflen); 577 sg++; 578 return sg; 579 } 580 581 /** 582 * create_multi_sg - Create multiple sg entries with buflen data length from 583 * source sglist 584 * @to_sg: Destination SG list 585 * @from_sg: Source SG list 586 * @buflen: Data length 587 * 588 * Returns next free entry in the destination SG list 589 **/ 590 static inline struct scatterlist *create_multi_sg(struct scatterlist *to_sg, 591 struct scatterlist *from_sg, 592 int buflen) 593 { 594 struct scatterlist *sg = to_sg; 595 unsigned int sglen; 596 597 for (; buflen && from_sg; buflen -= sglen) { 598 sglen = from_sg->length; 599 if (sglen > buflen) 600 sglen = buflen; 601 602 sg_set_buf(sg, sg_virt(from_sg), sglen); 603 from_sg = sg_next(from_sg); 604 sg++; 605 } 606 607 return sg; 608 } 609 610 static inline void set_orh_value(u64 *orh) 611 { 612 WRITE_ONCE(*orh, PENDING_SIG); 613 } 614 615 static inline void set_comp_value(u64 *comp) 616 { 617 WRITE_ONCE(*comp, PENDING_SIG); 618 } 619 620 static inline int alloc_src_req_buf(struct nitrox_kcrypt_request *nkreq, 621 int nents, int ivsize) 622 { 623 struct se_crypto_request *creq = &nkreq->creq; 624 625 nkreq->src = alloc_req_buf(nents, ivsize, creq->gfp); 626 if (!nkreq->src) 627 return -ENOMEM; 628 629 return 0; 630 } 631 632 static inline void nitrox_creq_copy_iv(char *dst, char *src, int size) 633 { 634 memcpy(dst, src, size); 635 } 636 637 static inline struct scatterlist *nitrox_creq_src_sg(char *iv, int ivsize) 638 { 639 return (struct scatterlist *)(iv + ivsize); 640 } 641 642 static inline void nitrox_creq_set_src_sg(struct nitrox_kcrypt_request *nkreq, 643 int nents, int ivsize, 644 struct scatterlist *src, int buflen) 645 { 646 char *iv = nkreq->src; 647 struct scatterlist *sg; 648 struct se_crypto_request *creq = &nkreq->creq; 649 650 creq->src = nitrox_creq_src_sg(iv, ivsize); 651 sg = creq->src; 652 sg_init_table(sg, nents); 653 654 /* Input format: 655 * +----+----------------+ 656 * | IV | SRC sg entries | 657 * +----+----------------+ 658 */ 659 660 /* IV */ 661 sg = create_single_sg(sg, iv, ivsize); 662 /* SRC entries */ 663 create_multi_sg(sg, src, buflen); 664 } 665 666 static inline int alloc_dst_req_buf(struct nitrox_kcrypt_request *nkreq, 667 int nents) 668 { 669 int extralen = ORH_HLEN + COMP_HLEN; 670 struct se_crypto_request *creq = &nkreq->creq; 671 672 nkreq->dst = alloc_req_buf(nents, extralen, creq->gfp); 673 if (!nkreq->dst) 674 return -ENOMEM; 675 676 return 0; 677 } 678 679 static inline void nitrox_creq_set_orh(struct nitrox_kcrypt_request *nkreq) 680 { 681 struct se_crypto_request *creq = &nkreq->creq; 682 683 creq->orh = (u64 *)(nkreq->dst); 684 set_orh_value(creq->orh); 685 } 686 687 static inline void nitrox_creq_set_comp(struct nitrox_kcrypt_request *nkreq) 688 { 689 struct se_crypto_request *creq = &nkreq->creq; 690 691 creq->comp = (u64 *)(nkreq->dst + ORH_HLEN); 692 set_comp_value(creq->comp); 693 } 694 695 static inline struct scatterlist *nitrox_creq_dst_sg(char *dst) 696 { 697 return (struct scatterlist *)(dst + ORH_HLEN + COMP_HLEN); 698 } 699 700 static inline void nitrox_creq_set_dst_sg(struct nitrox_kcrypt_request *nkreq, 701 int nents, int ivsize, 702 struct scatterlist *dst, int buflen) 703 { 704 struct se_crypto_request *creq = &nkreq->creq; 705 struct scatterlist *sg; 706 char *iv = nkreq->src; 707 708 creq->dst = nitrox_creq_dst_sg(nkreq->dst); 709 sg = creq->dst; 710 sg_init_table(sg, nents); 711 712 /* Output format: 713 * +-----+----+----------------+-----------------+ 714 * | ORH | IV | DST sg entries | COMPLETION Bytes| 715 * +-----+----+----------------+-----------------+ 716 */ 717 718 /* ORH */ 719 sg = create_single_sg(sg, creq->orh, ORH_HLEN); 720 /* IV */ 721 sg = create_single_sg(sg, iv, ivsize); 722 /* DST entries */ 723 sg = create_multi_sg(sg, dst, buflen); 724 /* COMPLETION Bytes */ 725 create_single_sg(sg, creq->comp, COMP_HLEN); 726 } 727 728 #endif /* __NITROX_REQ_H */ 729