1 /* SPDX-License-Identifier: GPL-2.0 */ 2 /* Copyright(c) 2019 Intel Corporation. All rights rsvd. */ 3 #ifndef _IDXD_REGISTERS_H_ 4 #define _IDXD_REGISTERS_H_ 5 6 #ifdef __KERNEL__ 7 #include <uapi/linux/idxd.h> 8 #else 9 #include <linux/idxd.h> 10 #endif 11 12 /* PCI Config */ 13 #define PCI_DEVICE_ID_INTEL_IAA_PTL 0xb02d 14 #define PCI_DEVICE_ID_INTEL_IAA_WCL 0xfd2d 15 16 #define DEVICE_VERSION_1 0x100 17 #define DEVICE_VERSION_2 0x200 18 #define DEVICE_VERSION_3 0x300 19 20 #define IDXD_MMIO_BAR 0 21 #define IDXD_WQ_BAR 2 22 #define IDXD_PORTAL_SIZE PAGE_SIZE 23 24 /* MMIO Device BAR0 Registers */ 25 #define IDXD_VER_OFFSET 0x00 26 #define IDXD_VER_MAJOR_MASK 0xf0 27 #define IDXD_VER_MINOR_MASK 0x0f 28 #define GET_IDXD_VER_MAJOR(x) (((x) & IDXD_VER_MAJOR_MASK) >> 4) 29 #define GET_IDXD_VER_MINOR(x) ((x) & IDXD_VER_MINOR_MASK) 30 31 union gen_cap_reg { 32 struct { 33 u64 block_on_fault:1; 34 u64 overlap_copy:1; 35 u64 cache_control_mem:1; 36 u64 cache_control_cache:1; 37 u64 cmd_cap:1; 38 u64 rsvd:3; 39 u64 dest_readback:1; 40 u64 drain_readback:1; 41 u64 rsvd2:3; 42 u64 evl_support:2; 43 u64 batch_continuation:1; 44 u64 max_xfer_shift:5; 45 u64 max_batch_shift:4; 46 u64 max_ims_mult:6; 47 u64 config_en:1; 48 u64 rsvd3:32; 49 }; 50 u64 bits; 51 }; 52 #define IDXD_GENCAP_OFFSET 0x10 53 54 union wq_cap_reg { 55 struct { 56 u64 total_wq_size:16; 57 u64 num_wqs:8; 58 u64 wqcfg_size:4; 59 u64 rsvd:20; 60 u64 shared_mode:1; 61 u64 dedicated_mode:1; 62 u64 wq_ats_support:1; 63 u64 priority:1; 64 u64 occupancy:1; 65 u64 occupancy_int:1; 66 u64 op_config:1; 67 u64 wq_prs_support:1; 68 u64 rsvd4:8; 69 }; 70 u64 bits; 71 }; 72 #define IDXD_WQCAP_OFFSET 0x20 73 #define IDXD_WQCFG_MIN 5 74 75 union group_cap_reg { 76 struct { 77 u64 num_groups:8; 78 u64 total_rdbufs:8; /* formerly total_tokens */ 79 u64 rdbuf_ctrl:1; /* formerly token_en */ 80 u64 rdbuf_limit:1; /* formerly token_limit */ 81 u64 progress_limit:1; /* descriptor and batch descriptor */ 82 u64 rsvd:45; 83 }; 84 u64 bits; 85 }; 86 #define IDXD_GRPCAP_OFFSET 0x30 87 88 union engine_cap_reg { 89 struct { 90 u64 num_engines:8; 91 u64 rsvd:56; 92 }; 93 u64 bits; 94 }; 95 96 #define IDXD_ENGCAP_OFFSET 0x38 97 98 #define IDXD_OPCAP_NOOP 0x0001 99 #define IDXD_OPCAP_BATCH 0x0002 100 #define IDXD_OPCAP_MEMMOVE 0x0008 101 struct opcap { 102 u64 bits[4]; 103 }; 104 105 #define IDXD_MAX_OPCAP_BITS 256U 106 107 #define IDXD_OPCAP_OFFSET 0x40 108 109 #define IDXD_TABLE_OFFSET 0x60 110 union offsets_reg { 111 struct { 112 u64 grpcfg:16; 113 u64 wqcfg:16; 114 u64 msix_perm:16; 115 u64 ims:16; 116 u64 perfmon:16; 117 u64 rsvd:48; 118 }; 119 u64 bits[2]; 120 }; 121 122 #define IDXD_TABLE_MULT 0x100 123 124 #define IDXD_GENCFG_OFFSET 0x80 125 union gencfg_reg { 126 struct { 127 u32 rdbuf_limit:8; 128 u32 rsvd:4; 129 u32 user_int_en:1; 130 u32 evl_en:1; 131 u32 rsvd2:18; 132 }; 133 u32 bits; 134 }; 135 136 #define IDXD_GENCTRL_OFFSET 0x88 137 union genctrl_reg { 138 struct { 139 u32 softerr_int_en:1; 140 u32 halt_int_en:1; 141 u32 evl_int_en:1; 142 u32 rsvd:29; 143 }; 144 u32 bits; 145 }; 146 147 #define IDXD_GENSTATS_OFFSET 0x90 148 union gensts_reg { 149 struct { 150 u32 state:2; 151 u32 reset_type:2; 152 u32 rsvd:28; 153 }; 154 u32 bits; 155 }; 156 157 enum idxd_device_status_state { 158 IDXD_DEVICE_STATE_DISABLED = 0, 159 IDXD_DEVICE_STATE_ENABLED, 160 IDXD_DEVICE_STATE_DRAIN, 161 IDXD_DEVICE_STATE_HALT, 162 }; 163 164 enum idxd_device_reset_type { 165 IDXD_DEVICE_RESET_SOFTWARE = 0, 166 IDXD_DEVICE_RESET_FLR, 167 IDXD_DEVICE_RESET_WARM, 168 IDXD_DEVICE_RESET_COLD, 169 }; 170 171 #define IDXD_INTCAUSE_OFFSET 0x98 172 #define IDXD_INTC_ERR 0x01 173 #define IDXD_INTC_CMD 0x02 174 #define IDXD_INTC_OCCUPY 0x04 175 #define IDXD_INTC_PERFMON_OVFL 0x08 176 #define IDXD_INTC_HALT_STATE 0x10 177 #define IDXD_INTC_EVL 0x20 178 #define IDXD_INTC_INT_HANDLE_REVOKED 0x80000000 179 180 #define IDXD_CMD_OFFSET 0xa0 181 union idxd_command_reg { 182 struct { 183 u32 operand:20; 184 u32 cmd:5; 185 u32 rsvd:6; 186 u32 int_req:1; 187 }; 188 u32 bits; 189 }; 190 191 enum idxd_cmd { 192 IDXD_CMD_ENABLE_DEVICE = 1, 193 IDXD_CMD_DISABLE_DEVICE, 194 IDXD_CMD_DRAIN_ALL, 195 IDXD_CMD_ABORT_ALL, 196 IDXD_CMD_RESET_DEVICE, 197 IDXD_CMD_ENABLE_WQ, 198 IDXD_CMD_DISABLE_WQ, 199 IDXD_CMD_DRAIN_WQ, 200 IDXD_CMD_ABORT_WQ, 201 IDXD_CMD_RESET_WQ, 202 IDXD_CMD_DRAIN_PASID, 203 IDXD_CMD_ABORT_PASID, 204 IDXD_CMD_REQUEST_INT_HANDLE, 205 IDXD_CMD_RELEASE_INT_HANDLE, 206 }; 207 208 #define CMD_INT_HANDLE_IMS 0x10000 209 210 #define IDXD_CMDSTS_OFFSET 0xa8 211 union cmdsts_reg { 212 struct { 213 u8 err; 214 u16 result; 215 u8 rsvd:7; 216 u8 active:1; 217 }; 218 u32 bits; 219 }; 220 #define IDXD_CMDSTS_ACTIVE 0x80000000 221 #define IDXD_CMDSTS_ERR_MASK 0xff 222 #define IDXD_CMDSTS_RES_SHIFT 8 223 224 enum idxd_cmdsts_err { 225 IDXD_CMDSTS_SUCCESS = 0, 226 IDXD_CMDSTS_INVAL_CMD, 227 IDXD_CMDSTS_INVAL_WQIDX, 228 IDXD_CMDSTS_HW_ERR, 229 /* enable device errors */ 230 IDXD_CMDSTS_ERR_DEV_ENABLED = 0x10, 231 IDXD_CMDSTS_ERR_CONFIG, 232 IDXD_CMDSTS_ERR_BUSMASTER_EN, 233 IDXD_CMDSTS_ERR_PASID_INVAL, 234 IDXD_CMDSTS_ERR_WQ_SIZE_ERANGE, 235 IDXD_CMDSTS_ERR_GRP_CONFIG, 236 IDXD_CMDSTS_ERR_GRP_CONFIG2, 237 IDXD_CMDSTS_ERR_GRP_CONFIG3, 238 IDXD_CMDSTS_ERR_GRP_CONFIG4, 239 /* enable wq errors */ 240 IDXD_CMDSTS_ERR_DEV_NOTEN = 0x20, 241 IDXD_CMDSTS_ERR_WQ_ENABLED, 242 IDXD_CMDSTS_ERR_WQ_SIZE, 243 IDXD_CMDSTS_ERR_WQ_PRIOR, 244 IDXD_CMDSTS_ERR_WQ_MODE, 245 IDXD_CMDSTS_ERR_BOF_EN, 246 IDXD_CMDSTS_ERR_PASID_EN, 247 IDXD_CMDSTS_ERR_MAX_BATCH_SIZE, 248 IDXD_CMDSTS_ERR_MAX_XFER_SIZE, 249 /* disable device errors */ 250 IDXD_CMDSTS_ERR_DIS_DEV_EN = 0x31, 251 /* disable WQ, drain WQ, abort WQ, reset WQ */ 252 IDXD_CMDSTS_ERR_DEV_NOT_EN, 253 /* request interrupt handle */ 254 IDXD_CMDSTS_ERR_INVAL_INT_IDX = 0x41, 255 IDXD_CMDSTS_ERR_NO_HANDLE, 256 }; 257 258 #define IDXD_CMDCAP_OFFSET 0xb0 259 260 #define IDXD_SWERR_OFFSET 0xc0 261 #define IDXD_SWERR_VALID 0x00000001 262 #define IDXD_SWERR_OVERFLOW 0x00000002 263 #define IDXD_SWERR_ACK (IDXD_SWERR_VALID | IDXD_SWERR_OVERFLOW) 264 union sw_err_reg { 265 struct { 266 u64 valid:1; 267 u64 overflow:1; 268 u64 desc_valid:1; 269 u64 wq_idx_valid:1; 270 u64 batch:1; 271 u64 fault_rw:1; 272 u64 priv:1; 273 u64 rsvd:1; 274 u64 error:8; 275 u64 wq_idx:8; 276 u64 rsvd2:8; 277 u64 operation:8; 278 u64 pasid:20; 279 u64 rsvd3:4; 280 281 u64 batch_idx:16; 282 u64 rsvd4:16; 283 u64 invalid_flags:32; 284 285 u64 fault_addr; 286 287 u64 rsvd5; 288 }; 289 u64 bits[4]; 290 }; 291 292 union iaa_cap_reg { 293 struct { 294 u64 dec_aecs_format_ver:1; 295 u64 drop_init_bits:1; 296 u64 chaining:1; 297 u64 force_array_output_mod:1; 298 u64 load_part_aecs:1; 299 u64 comp_early_abort:1; 300 u64 nested_comp:1; 301 u64 diction_comp:1; 302 u64 header_gen:1; 303 u64 crypto_gcm:1; 304 u64 crypto_cfb:1; 305 u64 crypto_xts:1; 306 u64 rsvd:52; 307 }; 308 u64 bits; 309 }; 310 311 #define IDXD_IAACAP_OFFSET 0x180 312 313 #define IDXD_EVLCFG_OFFSET 0xe0 314 union evlcfg_reg { 315 struct { 316 u64 pasid_en:1; 317 u64 priv:1; 318 u64 rsvd:10; 319 u64 base_addr:52; 320 321 u64 size:16; 322 u64 pasid:20; 323 u64 rsvd2:28; 324 }; 325 u64 bits[2]; 326 }; 327 328 #define IDXD_EVL_SIZE_MIN 0x0040 329 #define IDXD_EVL_SIZE_MAX 0xffff 330 331 union msix_perm { 332 struct { 333 u32 rsvd:2; 334 u32 ignore:1; 335 u32 pasid_en:1; 336 u32 rsvd2:8; 337 u32 pasid:20; 338 }; 339 u32 bits; 340 }; 341 342 union group_flags { 343 struct { 344 u64 tc_a:3; 345 u64 tc_b:3; 346 u64 rsvd:1; 347 u64 use_rdbuf_limit:1; 348 u64 rdbufs_reserved:8; 349 u64 rsvd2:4; 350 u64 rdbufs_allowed:8; 351 u64 rsvd3:4; 352 u64 desc_progress_limit:2; 353 u64 rsvd4:2; 354 u64 batch_progress_limit:2; 355 u64 rsvd5:26; 356 }; 357 u64 bits; 358 }; 359 360 struct grpcfg { 361 u64 wqs[4]; 362 u64 engines; 363 union group_flags flags; 364 }; 365 366 union wqcfg { 367 struct { 368 /* bytes 0-3 */ 369 u16 wq_size; 370 u16 rsvd; 371 372 /* bytes 4-7 */ 373 u16 wq_thresh; 374 u16 rsvd1; 375 376 /* bytes 8-11 */ 377 u32 mode:1; /* shared or dedicated */ 378 u32 bof:1; /* block on fault */ 379 u32 wq_ats_disable:1; 380 u32 wq_prs_disable:1; 381 u32 priority:4; 382 u32 pasid:20; 383 u32 pasid_en:1; 384 u32 priv:1; 385 u32 rsvd3:2; 386 387 /* bytes 12-15 */ 388 u32 max_xfer_shift:5; 389 u32 max_batch_shift:4; 390 u32 max_sgl_shift:4; 391 u32 rsvd4:19; 392 393 /* bytes 16-19 */ 394 u16 occupancy_inth; 395 u16 occupancy_table_sel:1; 396 u16 rsvd5:15; 397 398 /* bytes 20-23 */ 399 u16 occupancy_limit; 400 u16 occupancy_int_en:1; 401 u16 rsvd6:15; 402 403 /* bytes 24-27 */ 404 u16 occupancy; 405 u16 occupancy_int:1; 406 u16 rsvd7:12; 407 u16 mode_support:1; 408 u16 wq_state:2; 409 410 /* bytes 28-31 */ 411 u32 rsvd8; 412 413 /* bytes 32-63 */ 414 u64 op_config[4]; 415 }; 416 u32 bits[16]; 417 }; 418 419 #define WQCFG_PASID_IDX 2 420 #define WQCFG_PRIVL_IDX 2 421 #define WQCFG_OCCUP_IDX 6 422 423 #define WQCFG_OCCUP_MASK 0xffff 424 425 /* 426 * This macro calculates the offset into the WQCFG register 427 * idxd - struct idxd * 428 * n - wq id 429 * ofs - the index of the 32b dword for the config register 430 * 431 * The WQCFG register block is divided into groups per each wq. The n index 432 * allows us to move to the register group that's for that particular wq. 433 * Each register is 32bits. The ofs gives us the number of register to access. 434 */ 435 #define WQCFG_OFFSET(_idxd_dev, n, ofs) \ 436 ({\ 437 typeof(_idxd_dev) __idxd_dev = (_idxd_dev); \ 438 (__idxd_dev)->wqcfg_offset + (n) * (__idxd_dev)->wqcfg_size + sizeof(u32) * (ofs); \ 439 }) 440 441 #define WQCFG_STRIDES(_idxd_dev) ((_idxd_dev)->wqcfg_size / sizeof(u32)) 442 443 #define GRPCFG_SIZE 64 444 #define GRPWQCFG_STRIDES 4 445 446 /* 447 * This macro calculates the offset into the GRPCFG register 448 * idxd - struct idxd * 449 * n - group id 450 * ofs - the index of the 64b qword for the config register 451 * 452 * The GRPCFG register block is divided into three sub-registers, which 453 * are GRPWQCFG, GRPENGCFG and GRPFLGCFG. The n index allows us to move 454 * to the register block that contains the three sub-registers. 455 * Each register block is 64bits. And the ofs gives us the offset 456 * within the GRPWQCFG register to access. 457 */ 458 #define GRPWQCFG_OFFSET(idxd_dev, n, ofs) ((idxd_dev)->grpcfg_offset +\ 459 (n) * GRPCFG_SIZE + sizeof(u64) * (ofs)) 460 #define GRPENGCFG_OFFSET(idxd_dev, n) ((idxd_dev)->grpcfg_offset + (n) * GRPCFG_SIZE + 32) 461 #define GRPFLGCFG_OFFSET(idxd_dev, n) ((idxd_dev)->grpcfg_offset + (n) * GRPCFG_SIZE + 40) 462 463 /* Following is performance monitor registers */ 464 #define IDXD_PERFCAP_OFFSET 0x0 465 union idxd_perfcap { 466 struct { 467 u64 num_perf_counter:6; 468 u64 rsvd1:2; 469 u64 counter_width:8; 470 u64 num_event_category:4; 471 u64 global_event_category:16; 472 u64 filter:8; 473 u64 rsvd2:8; 474 u64 cap_per_counter:1; 475 u64 writeable_counter:1; 476 u64 counter_freeze:1; 477 u64 overflow_interrupt:1; 478 u64 rsvd3:8; 479 }; 480 u64 bits; 481 }; 482 483 #define IDXD_EVNTCAP_OFFSET 0x80 484 union idxd_evntcap { 485 struct { 486 u64 events:28; 487 u64 rsvd:36; 488 }; 489 u64 bits; 490 }; 491 492 struct idxd_event { 493 union { 494 struct { 495 u32 event_category:4; 496 u32 events:28; 497 }; 498 u32 val; 499 }; 500 }; 501 502 #define IDXD_CNTRCAP_OFFSET 0x800 503 struct idxd_cntrcap { 504 union { 505 struct { 506 u32 counter_width:8; 507 u32 rsvd:20; 508 u32 num_events:4; 509 }; 510 u32 val; 511 }; 512 struct idxd_event events[]; 513 }; 514 515 #define IDXD_PERFRST_OFFSET 0x10 516 union idxd_perfrst { 517 struct { 518 u32 perfrst_config:1; 519 u32 perfrst_counter:1; 520 u32 rsvd:30; 521 }; 522 u32 val; 523 }; 524 525 #define IDXD_OVFSTATUS_OFFSET 0x30 526 #define IDXD_PERFFRZ_OFFSET 0x20 527 #define IDXD_CNTRCFG_OFFSET 0x100 528 union idxd_cntrcfg { 529 struct { 530 u64 enable:1; 531 u64 interrupt_ovf:1; 532 u64 global_freeze_ovf:1; 533 u64 rsvd1:5; 534 u64 event_category:4; 535 u64 rsvd2:20; 536 u64 events:28; 537 u64 rsvd3:4; 538 }; 539 u64 val; 540 }; 541 542 #define IDXD_FLTCFG_OFFSET 0x300 543 544 #define IDXD_CNTRDATA_OFFSET 0x200 545 union idxd_cntrdata { 546 struct { 547 u64 event_count_value; 548 }; 549 u64 val; 550 }; 551 552 union event_cfg { 553 struct { 554 u64 event_cat:4; 555 u64 event_enc:28; 556 }; 557 u64 val; 558 }; 559 560 union filter_cfg { 561 struct { 562 u64 wq:32; 563 u64 tc:8; 564 u64 pg_sz:4; 565 u64 xfer_sz:8; 566 u64 eng:8; 567 }; 568 u64 val; 569 }; 570 571 #define IDXD_EVLSTATUS_OFFSET 0xf0 572 573 union evl_status_reg { 574 struct { 575 u32 head:16; 576 u32 rsvd:16; 577 u32 tail:16; 578 u32 rsvd2:14; 579 u32 int_pending:1; 580 u32 rsvd3:1; 581 }; 582 struct { 583 u32 bits_lower32; 584 u32 bits_upper32; 585 }; 586 u64 bits; 587 }; 588 589 #define IDXD_DSACAP0_OFFSET 0x180 590 union dsacap0_reg { 591 u64 bits; 592 struct { 593 u64 max_sgl_shift:4; 594 u64 max_gr_block_shift:4; 595 u64 ops_inter_domain:7; 596 u64 rsvd1:17; 597 u64 sgl_formats:16; 598 u64 max_sg_process:8; 599 u64 rsvd2:8; 600 }; 601 }; 602 603 #define IDXD_DSACAP1_OFFSET 0x188 604 union dsacap1_reg { 605 u64 bits; 606 }; 607 608 #define IDXD_DSACAP2_OFFSET 0x190 609 union dsacap2_reg { 610 u64 bits; 611 }; 612 613 #define IDXD_MAX_BATCH_IDENT 256 614 615 struct __evl_entry { 616 u64 rsvd:2; 617 u64 desc_valid:1; 618 u64 wq_idx_valid:1; 619 u64 batch:1; 620 u64 fault_rw:1; 621 u64 priv:1; 622 u64 err_info_valid:1; 623 u64 error:8; 624 u64 wq_idx:8; 625 u64 batch_id:8; 626 u64 operation:8; 627 u64 pasid:20; 628 u64 rsvd2:4; 629 630 u16 batch_idx; 631 u16 rsvd3; 632 union { 633 /* Invalid Flags 0x11 */ 634 u32 invalid_flags; 635 /* Invalid Int Handle 0x19 */ 636 /* Page fault 0x1a */ 637 /* Page fault 0x06, 0x1f, only operand_id */ 638 /* Page fault before drain or in batch, 0x26, 0x27 */ 639 struct { 640 u16 int_handle; 641 u16 rci:1; 642 u16 ims:1; 643 u16 rcr:1; 644 u16 first_err_in_batch:1; 645 u16 rsvd4_2:9; 646 u16 operand_id:3; 647 }; 648 }; 649 u64 fault_addr; 650 u64 rsvd5; 651 }; 652 653 struct dsa_evl_entry { 654 struct __evl_entry e; 655 struct dsa_completion_record cr; 656 }; 657 658 struct iax_evl_entry { 659 struct __evl_entry e; 660 u64 rsvd[4]; 661 struct iax_completion_record cr; 662 }; 663 664 #endif 665