1 /* SPDX-License-Identifier: GPL-2.0 */ 2 /* Copyright(c) 2019 Intel Corporation. All rights rsvd. */ 3 #ifndef _IDXD_REGISTERS_H_ 4 #define _IDXD_REGISTERS_H_ 5 6 #ifdef __KERNEL__ 7 #include <uapi/linux/idxd.h> 8 #else 9 #include <linux/idxd.h> 10 #endif 11 12 /* PCI Config */ 13 #define PCI_DEVICE_ID_INTEL_DSA_GNRD 0x11fb 14 #define PCI_DEVICE_ID_INTEL_DSA_DMR 0x1212 15 #define PCI_DEVICE_ID_INTEL_IAA_DMR 0x1216 16 #define PCI_DEVICE_ID_INTEL_IAA_PTL 0xb02d 17 #define PCI_DEVICE_ID_INTEL_IAA_WCL 0xfd2d 18 19 #define DEVICE_VERSION_1 0x100 20 #define DEVICE_VERSION_2 0x200 21 #define DEVICE_VERSION_3 0x300 22 23 #define IDXD_MMIO_BAR 0 24 #define IDXD_WQ_BAR 2 25 #define IDXD_PORTAL_SIZE PAGE_SIZE 26 27 /* MMIO Device BAR0 Registers */ 28 #define IDXD_VER_OFFSET 0x00 29 #define IDXD_VER_MAJOR_MASK 0xf0 30 #define IDXD_VER_MINOR_MASK 0x0f 31 #define GET_IDXD_VER_MAJOR(x) (((x) & IDXD_VER_MAJOR_MASK) >> 4) 32 #define GET_IDXD_VER_MINOR(x) ((x) & IDXD_VER_MINOR_MASK) 33 34 union gen_cap_reg { 35 struct { 36 u64 block_on_fault:1; 37 u64 overlap_copy:1; 38 u64 cache_control_mem:1; 39 u64 cache_control_cache:1; 40 u64 cmd_cap:1; 41 u64 rsvd:3; 42 u64 dest_readback:1; 43 u64 drain_readback:1; 44 u64 rsvd2:3; 45 u64 evl_support:2; 46 u64 batch_continuation:1; 47 u64 max_xfer_shift:5; 48 u64 max_batch_shift:4; 49 u64 max_ims_mult:6; 50 u64 config_en:1; 51 u64 rsvd3:32; 52 }; 53 u64 bits; 54 }; 55 #define IDXD_GENCAP_OFFSET 0x10 56 57 union wq_cap_reg { 58 struct { 59 u64 total_wq_size:16; 60 u64 num_wqs:8; 61 u64 wqcfg_size:4; 62 u64 rsvd:20; 63 u64 shared_mode:1; 64 u64 dedicated_mode:1; 65 u64 wq_ats_support:1; 66 u64 priority:1; 67 u64 occupancy:1; 68 u64 occupancy_int:1; 69 u64 op_config:1; 70 u64 wq_prs_support:1; 71 u64 rsvd4:8; 72 }; 73 u64 bits; 74 }; 75 #define IDXD_WQCAP_OFFSET 0x20 76 #define IDXD_WQCFG_MIN 5 77 78 union group_cap_reg { 79 struct { 80 u64 num_groups:8; 81 u64 total_rdbufs:8; /* formerly total_tokens */ 82 u64 rdbuf_ctrl:1; /* formerly token_en */ 83 u64 rdbuf_limit:1; /* formerly token_limit */ 84 u64 progress_limit:1; /* descriptor and batch descriptor */ 85 u64 rsvd:45; 86 }; 87 u64 bits; 88 }; 89 #define IDXD_GRPCAP_OFFSET 0x30 90 91 union engine_cap_reg { 92 struct { 93 u64 num_engines:8; 94 u64 rsvd:56; 95 }; 96 u64 bits; 97 }; 98 99 #define IDXD_ENGCAP_OFFSET 0x38 100 101 #define IDXD_OPCAP_NOOP 0x0001 102 #define IDXD_OPCAP_BATCH 0x0002 103 #define IDXD_OPCAP_MEMMOVE 0x0008 104 struct opcap { 105 u64 bits[4]; 106 }; 107 108 #define IDXD_MAX_OPCAP_BITS 256U 109 110 #define IDXD_OPCAP_OFFSET 0x40 111 112 #define IDXD_TABLE_OFFSET 0x60 113 union offsets_reg { 114 struct { 115 u64 grpcfg:16; 116 u64 wqcfg:16; 117 u64 msix_perm:16; 118 u64 ims:16; 119 u64 perfmon:16; 120 u64 rsvd:48; 121 }; 122 u64 bits[2]; 123 }; 124 125 #define IDXD_TABLE_MULT 0x100 126 127 #define IDXD_GENCFG_OFFSET 0x80 128 union gencfg_reg { 129 struct { 130 u32 rdbuf_limit:8; 131 u32 rsvd:4; 132 u32 user_int_en:1; 133 u32 evl_en:1; 134 u32 rsvd2:18; 135 }; 136 u32 bits; 137 }; 138 139 #define IDXD_GENCTRL_OFFSET 0x88 140 union genctrl_reg { 141 struct { 142 u32 softerr_int_en:1; 143 u32 halt_int_en:1; 144 u32 evl_int_en:1; 145 u32 rsvd:29; 146 }; 147 u32 bits; 148 }; 149 150 #define IDXD_GENSTATS_OFFSET 0x90 151 union gensts_reg { 152 struct { 153 u32 state:2; 154 u32 reset_type:2; 155 u32 rsvd:28; 156 }; 157 u32 bits; 158 }; 159 160 enum idxd_device_status_state { 161 IDXD_DEVICE_STATE_DISABLED = 0, 162 IDXD_DEVICE_STATE_ENABLED, 163 IDXD_DEVICE_STATE_DRAIN, 164 IDXD_DEVICE_STATE_HALT, 165 }; 166 167 enum idxd_device_reset_type { 168 IDXD_DEVICE_RESET_SOFTWARE = 0, 169 IDXD_DEVICE_RESET_FLR, 170 IDXD_DEVICE_RESET_WARM, 171 IDXD_DEVICE_RESET_COLD, 172 }; 173 174 #define IDXD_INTCAUSE_OFFSET 0x98 175 #define IDXD_INTC_ERR 0x01 176 #define IDXD_INTC_CMD 0x02 177 #define IDXD_INTC_OCCUPY 0x04 178 #define IDXD_INTC_PERFMON_OVFL 0x08 179 #define IDXD_INTC_HALT_STATE 0x10 180 #define IDXD_INTC_EVL 0x20 181 #define IDXD_INTC_INT_HANDLE_REVOKED 0x80000000 182 183 #define IDXD_CMD_OFFSET 0xa0 184 union idxd_command_reg { 185 struct { 186 u32 operand:20; 187 u32 cmd:5; 188 u32 rsvd:6; 189 u32 int_req:1; 190 }; 191 u32 bits; 192 }; 193 194 enum idxd_cmd { 195 IDXD_CMD_ENABLE_DEVICE = 1, 196 IDXD_CMD_DISABLE_DEVICE, 197 IDXD_CMD_DRAIN_ALL, 198 IDXD_CMD_ABORT_ALL, 199 IDXD_CMD_RESET_DEVICE, 200 IDXD_CMD_ENABLE_WQ, 201 IDXD_CMD_DISABLE_WQ, 202 IDXD_CMD_DRAIN_WQ, 203 IDXD_CMD_ABORT_WQ, 204 IDXD_CMD_RESET_WQ, 205 IDXD_CMD_DRAIN_PASID, 206 IDXD_CMD_ABORT_PASID, 207 IDXD_CMD_REQUEST_INT_HANDLE, 208 IDXD_CMD_RELEASE_INT_HANDLE, 209 }; 210 211 #define CMD_INT_HANDLE_IMS 0x10000 212 213 #define IDXD_CMDSTS_OFFSET 0xa8 214 union cmdsts_reg { 215 struct { 216 u8 err; 217 u16 result; 218 u8 rsvd:7; 219 u8 active:1; 220 }; 221 u32 bits; 222 }; 223 #define IDXD_CMDSTS_ACTIVE 0x80000000 224 #define IDXD_CMDSTS_ERR_MASK 0xff 225 #define IDXD_CMDSTS_RES_SHIFT 8 226 227 enum idxd_cmdsts_err { 228 IDXD_CMDSTS_SUCCESS = 0, 229 IDXD_CMDSTS_INVAL_CMD, 230 IDXD_CMDSTS_INVAL_WQIDX, 231 IDXD_CMDSTS_HW_ERR, 232 /* enable device errors */ 233 IDXD_CMDSTS_ERR_DEV_ENABLED = 0x10, 234 IDXD_CMDSTS_ERR_CONFIG, 235 IDXD_CMDSTS_ERR_BUSMASTER_EN, 236 IDXD_CMDSTS_ERR_PASID_INVAL, 237 IDXD_CMDSTS_ERR_WQ_SIZE_ERANGE, 238 IDXD_CMDSTS_ERR_GRP_CONFIG, 239 IDXD_CMDSTS_ERR_GRP_CONFIG2, 240 IDXD_CMDSTS_ERR_GRP_CONFIG3, 241 IDXD_CMDSTS_ERR_GRP_CONFIG4, 242 /* enable wq errors */ 243 IDXD_CMDSTS_ERR_DEV_NOTEN = 0x20, 244 IDXD_CMDSTS_ERR_WQ_ENABLED, 245 IDXD_CMDSTS_ERR_WQ_SIZE, 246 IDXD_CMDSTS_ERR_WQ_PRIOR, 247 IDXD_CMDSTS_ERR_WQ_MODE, 248 IDXD_CMDSTS_ERR_BOF_EN, 249 IDXD_CMDSTS_ERR_PASID_EN, 250 IDXD_CMDSTS_ERR_MAX_BATCH_SIZE, 251 IDXD_CMDSTS_ERR_MAX_XFER_SIZE, 252 /* disable device errors */ 253 IDXD_CMDSTS_ERR_DIS_DEV_EN = 0x31, 254 /* disable WQ, drain WQ, abort WQ, reset WQ */ 255 IDXD_CMDSTS_ERR_DEV_NOT_EN, 256 /* request interrupt handle */ 257 IDXD_CMDSTS_ERR_INVAL_INT_IDX = 0x41, 258 IDXD_CMDSTS_ERR_NO_HANDLE, 259 }; 260 261 #define IDXD_CMDCAP_OFFSET 0xb0 262 263 #define IDXD_SWERR_OFFSET 0xc0 264 #define IDXD_SWERR_VALID 0x00000001 265 #define IDXD_SWERR_OVERFLOW 0x00000002 266 #define IDXD_SWERR_ACK (IDXD_SWERR_VALID | IDXD_SWERR_OVERFLOW) 267 union sw_err_reg { 268 struct { 269 u64 valid:1; 270 u64 overflow:1; 271 u64 desc_valid:1; 272 u64 wq_idx_valid:1; 273 u64 batch:1; 274 u64 fault_rw:1; 275 u64 priv:1; 276 u64 rsvd:1; 277 u64 error:8; 278 u64 wq_idx:8; 279 u64 rsvd2:8; 280 u64 operation:8; 281 u64 pasid:20; 282 u64 rsvd3:4; 283 284 u64 batch_idx:16; 285 u64 rsvd4:16; 286 u64 invalid_flags:32; 287 288 u64 fault_addr; 289 290 u64 rsvd5; 291 }; 292 u64 bits[4]; 293 }; 294 295 union iaa_cap_reg { 296 struct { 297 u64 dec_aecs_format_ver:1; 298 u64 drop_init_bits:1; 299 u64 chaining:1; 300 u64 force_array_output_mod:1; 301 u64 load_part_aecs:1; 302 u64 comp_early_abort:1; 303 u64 nested_comp:1; 304 u64 diction_comp:1; 305 u64 header_gen:1; 306 u64 crypto_gcm:1; 307 u64 crypto_cfb:1; 308 u64 crypto_xts:1; 309 u64 rsvd:52; 310 }; 311 u64 bits; 312 }; 313 314 #define IDXD_IAACAP_OFFSET 0x180 315 316 #define IDXD_EVLCFG_OFFSET 0xe0 317 union evlcfg_reg { 318 struct { 319 u64 pasid_en:1; 320 u64 priv:1; 321 u64 rsvd:10; 322 u64 base_addr:52; 323 324 u64 size:16; 325 u64 pasid:20; 326 u64 rsvd2:28; 327 }; 328 u64 bits[2]; 329 }; 330 331 #define IDXD_EVL_SIZE_MIN 0x0040 332 #define IDXD_EVL_SIZE_MAX 0xffff 333 334 union msix_perm { 335 struct { 336 u32 rsvd:2; 337 u32 ignore:1; 338 u32 pasid_en:1; 339 u32 rsvd2:8; 340 u32 pasid:20; 341 }; 342 u32 bits; 343 }; 344 345 union group_flags { 346 struct { 347 u64 tc_a:3; 348 u64 tc_b:3; 349 u64 rsvd:1; 350 u64 use_rdbuf_limit:1; 351 u64 rdbufs_reserved:8; 352 u64 rsvd2:4; 353 u64 rdbufs_allowed:8; 354 u64 rsvd3:4; 355 u64 desc_progress_limit:2; 356 u64 rsvd4:2; 357 u64 batch_progress_limit:2; 358 u64 rsvd5:26; 359 }; 360 u64 bits; 361 }; 362 363 struct grpcfg { 364 u64 wqs[4]; 365 u64 engines; 366 union group_flags flags; 367 }; 368 369 union wqcfg { 370 struct { 371 /* bytes 0-3 */ 372 u16 wq_size; 373 u16 rsvd; 374 375 /* bytes 4-7 */ 376 u16 wq_thresh; 377 u16 rsvd1; 378 379 /* bytes 8-11 */ 380 u32 mode:1; /* shared or dedicated */ 381 u32 bof:1; /* block on fault */ 382 u32 wq_ats_disable:1; 383 u32 wq_prs_disable:1; 384 u32 priority:4; 385 u32 pasid:20; 386 u32 pasid_en:1; 387 u32 priv:1; 388 u32 rsvd3:2; 389 390 /* bytes 12-15 */ 391 u32 max_xfer_shift:5; 392 u32 max_batch_shift:4; 393 u32 max_sgl_shift:4; 394 u32 rsvd4:19; 395 396 /* bytes 16-19 */ 397 u16 occupancy_inth; 398 u16 occupancy_table_sel:1; 399 u16 rsvd5:15; 400 401 /* bytes 20-23 */ 402 u16 occupancy_limit; 403 u16 occupancy_int_en:1; 404 u16 rsvd6:15; 405 406 /* bytes 24-27 */ 407 u16 occupancy; 408 u16 occupancy_int:1; 409 u16 rsvd7:12; 410 u16 mode_support:1; 411 u16 wq_state:2; 412 413 /* bytes 28-31 */ 414 u32 rsvd8; 415 416 /* bytes 32-63 */ 417 u64 op_config[4]; 418 }; 419 u32 bits[16]; 420 }; 421 422 #define WQCFG_PASID_IDX 2 423 #define WQCFG_PRIVL_IDX 2 424 #define WQCFG_OCCUP_IDX 6 425 426 #define WQCFG_OCCUP_MASK 0xffff 427 428 /* 429 * This macro calculates the offset into the WQCFG register 430 * idxd - struct idxd * 431 * n - wq id 432 * ofs - the index of the 32b dword for the config register 433 * 434 * The WQCFG register block is divided into groups per each wq. The n index 435 * allows us to move to the register group that's for that particular wq. 436 * Each register is 32bits. The ofs gives us the number of register to access. 437 */ 438 #define WQCFG_OFFSET(_idxd_dev, n, ofs) \ 439 ({\ 440 typeof(_idxd_dev) __idxd_dev = (_idxd_dev); \ 441 (__idxd_dev)->wqcfg_offset + (n) * (__idxd_dev)->wqcfg_size + sizeof(u32) * (ofs); \ 442 }) 443 444 #define WQCFG_STRIDES(_idxd_dev) ((_idxd_dev)->wqcfg_size / sizeof(u32)) 445 446 #define GRPCFG_SIZE 64 447 #define GRPWQCFG_STRIDES 4 448 449 /* 450 * This macro calculates the offset into the GRPCFG register 451 * idxd - struct idxd * 452 * n - group id 453 * ofs - the index of the 64b qword for the config register 454 * 455 * The GRPCFG register block is divided into three sub-registers, which 456 * are GRPWQCFG, GRPENGCFG and GRPFLGCFG. The n index allows us to move 457 * to the register block that contains the three sub-registers. 458 * Each register block is 64bits. And the ofs gives us the offset 459 * within the GRPWQCFG register to access. 460 */ 461 #define GRPWQCFG_OFFSET(idxd_dev, n, ofs) ((idxd_dev)->grpcfg_offset +\ 462 (n) * GRPCFG_SIZE + sizeof(u64) * (ofs)) 463 #define GRPENGCFG_OFFSET(idxd_dev, n) ((idxd_dev)->grpcfg_offset + (n) * GRPCFG_SIZE + 32) 464 #define GRPFLGCFG_OFFSET(idxd_dev, n) ((idxd_dev)->grpcfg_offset + (n) * GRPCFG_SIZE + 40) 465 466 /* Following is performance monitor registers */ 467 #define IDXD_PERFCAP_OFFSET 0x0 468 union idxd_perfcap { 469 struct { 470 u64 num_perf_counter:6; 471 u64 rsvd1:2; 472 u64 counter_width:8; 473 u64 num_event_category:4; 474 u64 global_event_category:16; 475 u64 filter:8; 476 u64 rsvd2:8; 477 u64 cap_per_counter:1; 478 u64 writeable_counter:1; 479 u64 counter_freeze:1; 480 u64 overflow_interrupt:1; 481 u64 rsvd3:8; 482 }; 483 u64 bits; 484 }; 485 486 #define IDXD_EVNTCAP_OFFSET 0x80 487 union idxd_evntcap { 488 struct { 489 u64 events:28; 490 u64 rsvd:36; 491 }; 492 u64 bits; 493 }; 494 495 struct idxd_event { 496 union { 497 struct { 498 u32 event_category:4; 499 u32 events:28; 500 }; 501 u32 val; 502 }; 503 }; 504 505 #define IDXD_CNTRCAP_OFFSET 0x800 506 struct idxd_cntrcap { 507 union { 508 struct { 509 u32 counter_width:8; 510 u32 rsvd:20; 511 u32 num_events:4; 512 }; 513 u32 val; 514 }; 515 struct idxd_event events[]; 516 }; 517 518 #define IDXD_PERFRST_OFFSET 0x10 519 union idxd_perfrst { 520 struct { 521 u32 perfrst_config:1; 522 u32 perfrst_counter:1; 523 u32 rsvd:30; 524 }; 525 u32 val; 526 }; 527 528 #define IDXD_OVFSTATUS_OFFSET 0x30 529 #define IDXD_PERFFRZ_OFFSET 0x20 530 #define IDXD_CNTRCFG_OFFSET 0x100 531 union idxd_cntrcfg { 532 struct { 533 u64 enable:1; 534 u64 interrupt_ovf:1; 535 u64 global_freeze_ovf:1; 536 u64 rsvd1:5; 537 u64 event_category:4; 538 u64 rsvd2:20; 539 u64 events:28; 540 u64 rsvd3:4; 541 }; 542 u64 val; 543 }; 544 545 #define IDXD_FLTCFG_OFFSET 0x300 546 547 #define IDXD_CNTRDATA_OFFSET 0x200 548 union idxd_cntrdata { 549 struct { 550 u64 event_count_value; 551 }; 552 u64 val; 553 }; 554 555 union event_cfg { 556 struct { 557 u64 event_cat:4; 558 u64 event_enc:28; 559 }; 560 u64 val; 561 }; 562 563 union filter_cfg { 564 struct { 565 u64 wq:32; 566 u64 tc:8; 567 u64 pg_sz:4; 568 u64 xfer_sz:8; 569 u64 eng:8; 570 }; 571 u64 val; 572 }; 573 574 #define IDXD_EVLSTATUS_OFFSET 0xf0 575 576 union evl_status_reg { 577 struct { 578 u32 head:16; 579 u32 rsvd:16; 580 u32 tail:16; 581 u32 rsvd2:14; 582 u32 int_pending:1; 583 u32 rsvd3:1; 584 }; 585 struct { 586 u32 bits_lower32; 587 u32 bits_upper32; 588 }; 589 u64 bits; 590 }; 591 592 #define IDXD_DSACAP0_OFFSET 0x180 593 union dsacap0_reg { 594 u64 bits; 595 struct { 596 u64 max_sgl_shift:4; 597 u64 max_gr_block_shift:4; 598 u64 ops_inter_domain:7; 599 u64 rsvd1:17; 600 u64 sgl_formats:16; 601 u64 max_sg_process:8; 602 u64 rsvd2:8; 603 }; 604 }; 605 606 #define IDXD_DSACAP1_OFFSET 0x188 607 union dsacap1_reg { 608 u64 bits; 609 }; 610 611 #define IDXD_DSACAP2_OFFSET 0x190 612 union dsacap2_reg { 613 u64 bits; 614 }; 615 616 #define IDXD_MAX_BATCH_IDENT 256 617 618 struct __evl_entry { 619 u64 rsvd:2; 620 u64 desc_valid:1; 621 u64 wq_idx_valid:1; 622 u64 batch:1; 623 u64 fault_rw:1; 624 u64 priv:1; 625 u64 err_info_valid:1; 626 u64 error:8; 627 u64 wq_idx:8; 628 u64 batch_id:8; 629 u64 operation:8; 630 u64 pasid:20; 631 u64 rsvd2:4; 632 633 u16 batch_idx; 634 u16 rsvd3; 635 union { 636 /* Invalid Flags 0x11 */ 637 u32 invalid_flags; 638 /* Invalid Int Handle 0x19 */ 639 /* Page fault 0x1a */ 640 /* Page fault 0x06, 0x1f, only operand_id */ 641 /* Page fault before drain or in batch, 0x26, 0x27 */ 642 struct { 643 u16 int_handle; 644 u16 rci:1; 645 u16 ims:1; 646 u16 rcr:1; 647 u16 first_err_in_batch:1; 648 u16 rsvd4_2:9; 649 u16 operand_id:3; 650 }; 651 }; 652 u64 fault_addr; 653 u64 rsvd5; 654 }; 655 656 struct dsa_evl_entry { 657 struct __evl_entry e; 658 struct dsa_completion_record cr; 659 }; 660 661 struct iax_evl_entry { 662 struct __evl_entry e; 663 u64 rsvd[4]; 664 struct iax_completion_record cr; 665 }; 666 667 #endif 668