1 /* SPDX-License-Identifier: GPL-2.0 */ 2 #ifndef __NITROX_REQ_H 3 #define __NITROX_REQ_H 4 5 #include <linux/dma-mapping.h> 6 #include <crypto/aes.h> 7 8 #include "nitrox_dev.h" 9 10 #define PENDING_SIG 0xFFFFFFFFFFFFFFFFUL 11 #define PRIO 4001 12 13 typedef void (*sereq_completion_t)(void *req, int err); 14 15 /** 16 * struct gphdr - General purpose Header 17 * @param0: first parameter. 18 * @param1: second parameter. 19 * @param2: third parameter. 20 * @param3: fourth parameter. 21 * 22 * Params tell the iv and enc/dec data offsets. 23 */ 24 struct gphdr { 25 __be16 param0; 26 __be16 param1; 27 __be16 param2; 28 __be16 param3; 29 }; 30 31 /** 32 * struct se_req_ctrl - SE request information. 33 * @arg: Minor number of the opcode 34 * @ctxc: Context control. 35 * @unca: Uncertainity enabled. 36 * @info: Additional information for SE cores. 37 * @ctxl: Context length in bytes. 38 * @uddl: User defined data length 39 */ 40 union se_req_ctrl { 41 u64 value; 42 struct { 43 u64 raz : 22; 44 u64 arg : 8; 45 u64 ctxc : 2; 46 u64 unca : 1; 47 u64 info : 3; 48 u64 unc : 8; 49 u64 ctxl : 12; 50 u64 uddl : 8; 51 } s; 52 }; 53 54 #define MAX_IV_LEN 16 55 56 /** 57 * struct se_crypto_request - SE crypto request structure. 58 * @opcode: Request opcode (enc/dec) 59 * @flags: flags from crypto subsystem 60 * @ctx_handle: Crypto context handle. 61 * @gph: GP Header 62 * @ctrl: Request Information. 63 * @orh: ORH address 64 * @comp: completion address 65 * @src: Input sglist 66 * @dst: Output sglist 67 */ 68 struct se_crypto_request { 69 u8 opcode; 70 gfp_t gfp; 71 u32 flags; 72 u64 ctx_handle; 73 74 struct gphdr gph; 75 union se_req_ctrl ctrl; 76 u64 *orh; 77 u64 *comp; 78 79 struct scatterlist *src; 80 struct scatterlist *dst; 81 }; 82 83 /* Crypto opcodes */ 84 #define FLEXI_CRYPTO_ENCRYPT_HMAC 0x33 85 #define ENCRYPT 0 86 #define DECRYPT 1 87 88 /* IV from context */ 89 #define IV_FROM_CTX 0 90 /* IV from Input data */ 91 #define IV_FROM_DPTR 1 92 93 /** 94 * cipher opcodes for firmware 95 */ 96 enum flexi_cipher { 97 CIPHER_NULL = 0, 98 CIPHER_3DES_CBC, 99 CIPHER_3DES_ECB, 100 CIPHER_AES_CBC, 101 CIPHER_AES_ECB, 102 CIPHER_AES_CFB, 103 CIPHER_AES_CTR, 104 CIPHER_AES_GCM, 105 CIPHER_AES_XTS, 106 CIPHER_AES_CCM, 107 CIPHER_AES_CBC_CTS, 108 CIPHER_AES_ECB_CTS, 109 CIPHER_INVALID 110 }; 111 112 enum flexi_auth { 113 AUTH_NULL = 0, 114 AUTH_MD5, 115 AUTH_SHA1, 116 AUTH_SHA2_SHA224, 117 AUTH_SHA2_SHA256, 118 AUTH_SHA2_SHA384, 119 AUTH_SHA2_SHA512, 120 AUTH_GMAC, 121 AUTH_INVALID 122 }; 123 124 /** 125 * struct crypto_keys - Crypto keys 126 * @key: Encryption key or KEY1 for AES-XTS 127 * @iv: Encryption IV or Tweak for AES-XTS 128 */ 129 struct crypto_keys { 130 union { 131 u8 key[AES_MAX_KEY_SIZE]; 132 u8 key1[AES_MAX_KEY_SIZE]; 133 } u; 134 u8 iv[AES_BLOCK_SIZE]; 135 }; 136 137 /** 138 * struct auth_keys - Authentication keys 139 * @ipad: IPAD or KEY2 for AES-XTS 140 * @opad: OPAD or AUTH KEY if auth_input_type = 1 141 */ 142 struct auth_keys { 143 union { 144 u8 ipad[64]; 145 u8 key2[64]; 146 } u; 147 u8 opad[64]; 148 }; 149 150 union fc_ctx_flags { 151 __be64 f; 152 struct { 153 #if defined(__BIG_ENDIAN_BITFIELD) 154 u64 cipher_type : 4; 155 u64 reserved_59 : 1; 156 u64 aes_keylen : 2; 157 u64 iv_source : 1; 158 u64 hash_type : 4; 159 u64 reserved_49_51 : 3; 160 u64 auth_input_type: 1; 161 u64 mac_len : 8; 162 u64 reserved_0_39 : 40; 163 #else 164 u64 reserved_0_39 : 40; 165 u64 mac_len : 8; 166 u64 auth_input_type: 1; 167 u64 reserved_49_51 : 3; 168 u64 hash_type : 4; 169 u64 iv_source : 1; 170 u64 aes_keylen : 2; 171 u64 reserved_59 : 1; 172 u64 cipher_type : 4; 173 #endif 174 } w0; 175 }; 176 /** 177 * struct flexi_crypto_context - Crypto context 178 * @cipher_type: Encryption cipher type 179 * @aes_keylen: AES key length 180 * @iv_source: Encryption IV source 181 * @hash_type: Authentication type 182 * @auth_input_type: Authentication input type 183 * 1 - Authentication IV and KEY, microcode calculates OPAD/IPAD 184 * 0 - Authentication OPAD/IPAD 185 * @mac_len: mac length 186 * @crypto: Crypto keys 187 * @auth: Authentication keys 188 */ 189 struct flexi_crypto_context { 190 union fc_ctx_flags flags; 191 struct crypto_keys crypto; 192 struct auth_keys auth; 193 }; 194 195 struct crypto_ctx_hdr { 196 struct dma_pool *pool; 197 dma_addr_t dma; 198 void *vaddr; 199 }; 200 201 struct nitrox_crypto_ctx { 202 struct nitrox_device *ndev; 203 union { 204 u64 ctx_handle; 205 struct flexi_crypto_context *fctx; 206 } u; 207 struct crypto_ctx_hdr *chdr; 208 sereq_completion_t callback; 209 }; 210 211 struct nitrox_kcrypt_request { 212 struct se_crypto_request creq; 213 u8 *src; 214 u8 *dst; 215 u8 *iv_out; 216 }; 217 218 /** 219 * struct nitrox_aead_rctx - AEAD request context 220 * @nkreq: Base request context 221 * @cryptlen: Encryption/Decryption data length 222 * @assoclen: AAD length 223 * @srclen: Input buffer length 224 * @dstlen: Output buffer length 225 * @iv: IV data 226 * @ivsize: IV data length 227 * @flags: AEAD req flags 228 * @ctx_handle: Device context handle 229 * @src: Source sglist 230 * @dst: Destination sglist 231 * @ctrl_arg: Identifies the request type (ENCRYPT/DECRYPT) 232 */ 233 struct nitrox_aead_rctx { 234 struct nitrox_kcrypt_request nkreq; 235 unsigned int cryptlen; 236 unsigned int assoclen; 237 unsigned int srclen; 238 unsigned int dstlen; 239 u8 *iv; 240 int ivsize; 241 u32 flags; 242 u64 ctx_handle; 243 struct scatterlist *src; 244 struct scatterlist *dst; 245 u8 ctrl_arg; 246 }; 247 248 /** 249 * struct nitrox_rfc4106_rctx - rfc4106 cipher request context 250 * @base: AEAD request context 251 * @src: Source sglist 252 * @dst: Destination sglist 253 * @assoc: AAD 254 */ 255 struct nitrox_rfc4106_rctx { 256 struct nitrox_aead_rctx base; 257 struct scatterlist src[3]; 258 struct scatterlist dst[3]; 259 u8 assoc[20]; 260 }; 261 262 /** 263 * struct pkt_instr_hdr - Packet Instruction Header 264 * @g: Gather used 265 * When [G] is set and [GSZ] != 0, the instruction is 266 * indirect gather instruction. 267 * When [G] is set and [GSZ] = 0, the instruction is 268 * direct gather instruction. 269 * @gsz: Number of pointers in the indirect gather list 270 * @ihi: When set hardware duplicates the 1st 8 bytes of pkt_instr_hdr 271 * and adds them to the packet after the pkt_instr_hdr but before any UDD 272 * @ssz: Not used by the input hardware. But can become slc_store_int[SSZ] 273 * when [IHI] is set. 274 * @fsz: The number of front data bytes directly included in the 275 * PCIe instruction. 276 * @tlen: The length of the input packet in bytes, include: 277 * - 16B pkt_hdr 278 * - Inline context bytes if any, 279 * - UDD if any, 280 * - packet payload bytes 281 */ 282 union pkt_instr_hdr { 283 u64 value; 284 struct { 285 #if defined(__BIG_ENDIAN_BITFIELD) 286 u64 raz_48_63 : 16; 287 u64 g : 1; 288 u64 gsz : 7; 289 u64 ihi : 1; 290 u64 ssz : 7; 291 u64 raz_30_31 : 2; 292 u64 fsz : 6; 293 u64 raz_16_23 : 8; 294 u64 tlen : 16; 295 #else 296 u64 tlen : 16; 297 u64 raz_16_23 : 8; 298 u64 fsz : 6; 299 u64 raz_30_31 : 2; 300 u64 ssz : 7; 301 u64 ihi : 1; 302 u64 gsz : 7; 303 u64 g : 1; 304 u64 raz_48_63 : 16; 305 #endif 306 } s; 307 }; 308 309 /** 310 * struct pkt_hdr - Packet Input Header 311 * @opcode: Request opcode (Major) 312 * @arg: Request opcode (Minor) 313 * @ctxc: Context control. 314 * @unca: When set [UNC] is the uncertainty count for an input packet. 315 * The hardware uses uncertainty counts to predict 316 * output buffer use and avoid deadlock. 317 * @info: Not used by input hardware. Available for use 318 * during SE processing. 319 * @destport: The expected destination port/ring/channel for the packet. 320 * @unc: Uncertainty count for an input packet. 321 * @grp: SE group that will process the input packet. 322 * @ctxl: Context Length in 64-bit words. 323 * @uddl: User-defined data (UDD) length in bytes. 324 * @ctxp: Context pointer. CTXP<63,2:0> must be zero in all cases. 325 */ 326 union pkt_hdr { 327 u64 value[2]; 328 struct { 329 #if defined(__BIG_ENDIAN_BITFIELD) 330 u64 opcode : 8; 331 u64 arg : 8; 332 u64 ctxc : 2; 333 u64 unca : 1; 334 u64 raz_44 : 1; 335 u64 info : 3; 336 u64 destport : 9; 337 u64 unc : 8; 338 u64 raz_19_23 : 5; 339 u64 grp : 3; 340 u64 raz_15 : 1; 341 u64 ctxl : 7; 342 u64 uddl : 8; 343 #else 344 u64 uddl : 8; 345 u64 ctxl : 7; 346 u64 raz_15 : 1; 347 u64 grp : 3; 348 u64 raz_19_23 : 5; 349 u64 unc : 8; 350 u64 destport : 9; 351 u64 info : 3; 352 u64 raz_44 : 1; 353 u64 unca : 1; 354 u64 ctxc : 2; 355 u64 arg : 8; 356 u64 opcode : 8; 357 #endif 358 __be64 ctxp; 359 } s; 360 }; 361 362 /** 363 * struct slc_store_info - Solicited Paceket Output Store Information. 364 * @ssz: The number of scatterlist pointers for the solicited output port 365 * packet. 366 * @rptr: The result pointer for the solicited output port packet. 367 * If [SSZ]=0, [RPTR] must point directly to a buffer on the remote 368 * host that is large enough to hold the entire output packet. 369 * If [SSZ]!=0, [RPTR] must point to an array of ([SSZ]+3)/4 370 * sglist components at [RPTR] on the remote host. 371 */ 372 union slc_store_info { 373 u64 value[2]; 374 struct { 375 #if defined(__BIG_ENDIAN_BITFIELD) 376 u64 raz_39_63 : 25; 377 u64 ssz : 7; 378 u64 raz_0_31 : 32; 379 #else 380 u64 raz_0_31 : 32; 381 u64 ssz : 7; 382 u64 raz_39_63 : 25; 383 #endif 384 __be64 rptr; 385 } s; 386 }; 387 388 /** 389 * struct nps_pkt_instr - NPS Packet Instruction of SE cores. 390 * @dptr0 : Input pointer points to buffer in remote host. 391 * @ih: Packet Instruction Header (8 bytes) 392 * @irh: Packet Input Header (16 bytes) 393 * @slc: Solicited Packet Output Store Information (16 bytes) 394 * @fdata: Front data 395 * 396 * 64-Byte Instruction Format 397 */ 398 struct nps_pkt_instr { 399 __be64 dptr0; 400 union pkt_instr_hdr ih; 401 union pkt_hdr irh; 402 union slc_store_info slc; 403 u64 fdata[2]; 404 }; 405 406 /** 407 * struct aqmq_command_s - The 32 byte command for AE processing. 408 * @opcode: Request opcode 409 * @param1: Request control parameter 1 410 * @param2: Request control parameter 2 411 * @dlen: Input length 412 * @dptr: Input pointer points to buffer in remote host 413 * @rptr: Result pointer points to buffer in remote host 414 * @grp: AQM Group (0..7) 415 * @cptr: Context pointer 416 */ 417 struct aqmq_command_s { 418 __be16 opcode; 419 __be16 param1; 420 __be16 param2; 421 __be16 dlen; 422 __be64 dptr; 423 __be64 rptr; 424 union { 425 __be64 word3; 426 #if defined(__BIG_ENDIAN_BITFIELD) 427 u64 grp : 3; 428 u64 cptr : 61; 429 #else 430 u64 cptr : 61; 431 u64 grp : 3; 432 #endif 433 }; 434 }; 435 436 /** 437 * struct ctx_hdr - Book keeping data about the crypto context 438 * @pool: Pool used to allocate crypto context 439 * @dma: Base DMA address of the cypto context 440 * @ctx_dma: Actual usable crypto context for NITROX 441 */ 442 struct ctx_hdr { 443 struct dma_pool *pool; 444 dma_addr_t dma; 445 dma_addr_t ctx_dma; 446 }; 447 448 /* 449 * struct sglist_component - SG list component format 450 * @len0: The number of bytes at [PTR0] on the remote host. 451 * @len1: The number of bytes at [PTR1] on the remote host. 452 * @len2: The number of bytes at [PTR2] on the remote host. 453 * @len3: The number of bytes at [PTR3] on the remote host. 454 * @dma0: First pointer point to buffer in remote host. 455 * @dma1: Second pointer point to buffer in remote host. 456 * @dma2: Third pointer point to buffer in remote host. 457 * @dma3: Fourth pointer point to buffer in remote host. 458 */ 459 struct nitrox_sgcomp { 460 __be16 len[4]; 461 __be64 dma[4]; 462 }; 463 464 /* 465 * strutct nitrox_sgtable - SG list information 466 * @sgmap_cnt: Number of buffers mapped 467 * @total_bytes: Total bytes in sglist. 468 * @sgcomp_len: Total sglist components length. 469 * @sgcomp_dma: DMA address of sglist component. 470 * @sg: crypto request buffer. 471 * @sgcomp: sglist component for NITROX. 472 */ 473 struct nitrox_sgtable { 474 u8 sgmap_cnt; 475 u16 total_bytes; 476 u32 sgcomp_len; 477 dma_addr_t sgcomp_dma; 478 struct scatterlist *sg; 479 struct nitrox_sgcomp *sgcomp; 480 }; 481 482 /* Response Header Length */ 483 #define ORH_HLEN 8 484 /* Completion bytes Length */ 485 #define COMP_HLEN 8 486 487 struct resp_hdr { 488 u64 *orh; 489 u64 *completion; 490 }; 491 492 typedef void (*completion_t)(void *arg, int err); 493 494 /** 495 * struct nitrox_softreq - Represents the NIROX Request. 496 * @response: response list entry 497 * @backlog: Backlog list entry 498 * @ndev: Device used to submit the request 499 * @cmdq: Command queue for submission 500 * @resp: Response headers 501 * @instr: 64B instruction 502 * @in: SG table for input 503 * @out SG table for output 504 * @tstamp: Request submitted time in jiffies 505 * @callback: callback after request completion/timeout 506 * @cb_arg: callback argument 507 */ 508 struct nitrox_softreq { 509 struct list_head response; 510 struct list_head backlog; 511 512 u32 flags; 513 gfp_t gfp; 514 atomic_t status; 515 516 struct nitrox_device *ndev; 517 struct nitrox_cmdq *cmdq; 518 519 struct nps_pkt_instr instr; 520 struct resp_hdr resp; 521 struct nitrox_sgtable in; 522 struct nitrox_sgtable out; 523 524 unsigned long tstamp; 525 526 completion_t callback; 527 void *cb_arg; 528 }; 529 530 static inline int flexi_aes_keylen(int keylen) 531 { 532 int aes_keylen; 533 534 switch (keylen) { 535 case AES_KEYSIZE_128: 536 aes_keylen = 1; 537 break; 538 case AES_KEYSIZE_192: 539 aes_keylen = 2; 540 break; 541 case AES_KEYSIZE_256: 542 aes_keylen = 3; 543 break; 544 default: 545 aes_keylen = -EINVAL; 546 break; 547 } 548 return aes_keylen; 549 } 550 551 static inline void *alloc_req_buf(int nents, int extralen, gfp_t gfp) 552 { 553 size_t size; 554 555 size = sizeof(struct scatterlist) * nents; 556 size += extralen; 557 558 return kzalloc(size, gfp); 559 } 560 561 /** 562 * create_single_sg - Point SG entry to the data 563 * @sg: Destination SG list 564 * @buf: Data 565 * @buflen: Data length 566 * 567 * Returns next free entry in the destination SG list 568 **/ 569 static inline struct scatterlist *create_single_sg(struct scatterlist *sg, 570 void *buf, int buflen) 571 { 572 sg_set_buf(sg, buf, buflen); 573 sg++; 574 return sg; 575 } 576 577 /** 578 * create_multi_sg - Create multiple sg entries with buflen data length from 579 * source sglist 580 * @to_sg: Destination SG list 581 * @from_sg: Source SG list 582 * @buflen: Data length 583 * 584 * Returns next free entry in the destination SG list 585 **/ 586 static inline struct scatterlist *create_multi_sg(struct scatterlist *to_sg, 587 struct scatterlist *from_sg, 588 int buflen) 589 { 590 struct scatterlist *sg = to_sg; 591 unsigned int sglen; 592 593 for (; buflen && from_sg; buflen -= sglen) { 594 sglen = from_sg->length; 595 if (sglen > buflen) 596 sglen = buflen; 597 598 sg_set_buf(sg, sg_virt(from_sg), sglen); 599 from_sg = sg_next(from_sg); 600 sg++; 601 } 602 603 return sg; 604 } 605 606 static inline void set_orh_value(u64 *orh) 607 { 608 WRITE_ONCE(*orh, PENDING_SIG); 609 } 610 611 static inline void set_comp_value(u64 *comp) 612 { 613 WRITE_ONCE(*comp, PENDING_SIG); 614 } 615 616 static inline int alloc_src_req_buf(struct nitrox_kcrypt_request *nkreq, 617 int nents, int ivsize) 618 { 619 struct se_crypto_request *creq = &nkreq->creq; 620 621 nkreq->src = alloc_req_buf(nents, ivsize, creq->gfp); 622 if (!nkreq->src) 623 return -ENOMEM; 624 625 return 0; 626 } 627 628 static inline void nitrox_creq_copy_iv(char *dst, char *src, int size) 629 { 630 memcpy(dst, src, size); 631 } 632 633 static inline struct scatterlist *nitrox_creq_src_sg(char *iv, int ivsize) 634 { 635 return (struct scatterlist *)(iv + ivsize); 636 } 637 638 static inline void nitrox_creq_set_src_sg(struct nitrox_kcrypt_request *nkreq, 639 int nents, int ivsize, 640 struct scatterlist *src, int buflen) 641 { 642 char *iv = nkreq->src; 643 struct scatterlist *sg; 644 struct se_crypto_request *creq = &nkreq->creq; 645 646 creq->src = nitrox_creq_src_sg(iv, ivsize); 647 sg = creq->src; 648 sg_init_table(sg, nents); 649 650 /* Input format: 651 * +----+----------------+ 652 * | IV | SRC sg entries | 653 * +----+----------------+ 654 */ 655 656 /* IV */ 657 sg = create_single_sg(sg, iv, ivsize); 658 /* SRC entries */ 659 create_multi_sg(sg, src, buflen); 660 } 661 662 static inline int alloc_dst_req_buf(struct nitrox_kcrypt_request *nkreq, 663 int nents) 664 { 665 int extralen = ORH_HLEN + COMP_HLEN; 666 struct se_crypto_request *creq = &nkreq->creq; 667 668 nkreq->dst = alloc_req_buf(nents, extralen, creq->gfp); 669 if (!nkreq->dst) 670 return -ENOMEM; 671 672 return 0; 673 } 674 675 static inline void nitrox_creq_set_orh(struct nitrox_kcrypt_request *nkreq) 676 { 677 struct se_crypto_request *creq = &nkreq->creq; 678 679 creq->orh = (u64 *)(nkreq->dst); 680 set_orh_value(creq->orh); 681 } 682 683 static inline void nitrox_creq_set_comp(struct nitrox_kcrypt_request *nkreq) 684 { 685 struct se_crypto_request *creq = &nkreq->creq; 686 687 creq->comp = (u64 *)(nkreq->dst + ORH_HLEN); 688 set_comp_value(creq->comp); 689 } 690 691 static inline struct scatterlist *nitrox_creq_dst_sg(char *dst) 692 { 693 return (struct scatterlist *)(dst + ORH_HLEN + COMP_HLEN); 694 } 695 696 static inline void nitrox_creq_set_dst_sg(struct nitrox_kcrypt_request *nkreq, 697 int nents, int ivsize, 698 struct scatterlist *dst, int buflen) 699 { 700 struct se_crypto_request *creq = &nkreq->creq; 701 struct scatterlist *sg; 702 char *iv = nkreq->src; 703 704 creq->dst = nitrox_creq_dst_sg(nkreq->dst); 705 sg = creq->dst; 706 sg_init_table(sg, nents); 707 708 /* Output format: 709 * +-----+----+----------------+-----------------+ 710 * | ORH | IV | DST sg entries | COMPLETION Bytes| 711 * +-----+----+----------------+-----------------+ 712 */ 713 714 /* ORH */ 715 sg = create_single_sg(sg, creq->orh, ORH_HLEN); 716 /* IV */ 717 sg = create_single_sg(sg, iv, ivsize); 718 /* DST entries */ 719 sg = create_multi_sg(sg, dst, buflen); 720 /* COMPLETION Bytes */ 721 create_single_sg(sg, creq->comp, COMP_HLEN); 722 } 723 724 #endif /* __NITROX_REQ_H */ 725