1 /* SPDX-License-Identifier: GPL-2.0 */ 2 /* Copyright(c) 2019 Intel Corporation. All rights rsvd. */ 3 #ifndef _IDXD_H_ 4 #define _IDXD_H_ 5 6 #include <linux/sbitmap.h> 7 #include <linux/dmaengine.h> 8 #include <linux/percpu-rwsem.h> 9 #include <linux/wait.h> 10 #include <linux/cdev.h> 11 #include <linux/idr.h> 12 #include <linux/pci.h> 13 #include <linux/bitmap.h> 14 #include <linux/perf_event.h> 15 #include <linux/iommu.h> 16 #include <linux/crypto.h> 17 #include <uapi/linux/idxd.h> 18 #include "registers.h" 19 20 #define IDXD_DRIVER_VERSION "1.00" 21 22 extern struct kmem_cache *idxd_desc_pool; 23 extern bool tc_override; 24 25 struct idxd_wq; 26 struct idxd_dev; 27 28 enum idxd_dev_type { 29 IDXD_DEV_NONE = -1, 30 IDXD_DEV_DSA = 0, 31 IDXD_DEV_IAX, 32 IDXD_DEV_WQ, 33 IDXD_DEV_GROUP, 34 IDXD_DEV_ENGINE, 35 IDXD_DEV_CDEV, 36 IDXD_DEV_CDEV_FILE, 37 IDXD_DEV_MAX_TYPE, 38 }; 39 40 struct idxd_dev { 41 struct device conf_dev; 42 enum idxd_dev_type type; 43 }; 44 45 #define IDXD_REG_TIMEOUT 50 46 #define IDXD_DRAIN_TIMEOUT 5000 47 48 enum idxd_type { 49 IDXD_TYPE_UNKNOWN = -1, 50 IDXD_TYPE_DSA = 0, 51 IDXD_TYPE_IAX, 52 IDXD_TYPE_MAX, 53 }; 54 55 #define IDXD_NAME_SIZE 128 56 #define IDXD_PMU_EVENT_MAX 64 57 58 #define IDXD_ENQCMDS_RETRIES 32 59 #define IDXD_ENQCMDS_MAX_RETRIES 64 60 61 enum idxd_complete_type { 62 IDXD_COMPLETE_NORMAL = 0, 63 IDXD_COMPLETE_ABORT, 64 IDXD_COMPLETE_DEV_FAIL, 65 }; 66 67 struct idxd_desc; 68 69 struct idxd_device_driver { 70 const char *name; 71 enum idxd_dev_type *type; 72 int (*probe)(struct idxd_dev *idxd_dev); 73 void (*remove)(struct idxd_dev *idxd_dev); 74 void (*desc_complete)(struct idxd_desc *desc, 75 enum idxd_complete_type comp_type, 76 bool free_desc, 77 void *ctx, u32 *status); 78 struct device_driver drv; 79 }; 80 81 extern struct idxd_device_driver dsa_drv; 82 extern struct idxd_device_driver idxd_drv; 83 extern struct idxd_device_driver idxd_dmaengine_drv; 84 extern struct idxd_device_driver idxd_user_drv; 85 86 #define INVALID_INT_HANDLE -1 87 struct idxd_irq_entry { 88 int id; 89 int vector; 90 struct llist_head pending_llist; 91 struct list_head work_list; 92 /* 93 * Lock to protect access between irq thread process descriptor 94 * and irq thread processing error descriptor. 95 */ 96 spinlock_t list_lock; 97 int int_handle; 98 ioasid_t pasid; 99 }; 100 101 struct idxd_group { 102 struct idxd_dev idxd_dev; 103 struct idxd_device *idxd; 104 struct grpcfg grpcfg; 105 int id; 106 int num_engines; 107 int num_wqs; 108 bool use_rdbuf_limit; 109 u8 rdbufs_allowed; 110 u8 rdbufs_reserved; 111 int tc_a; 112 int tc_b; 113 int desc_progress_limit; 114 int batch_progress_limit; 115 }; 116 117 struct idxd_pmu { 118 struct idxd_device *idxd; 119 120 struct perf_event *event_list[IDXD_PMU_EVENT_MAX]; 121 int n_events; 122 123 DECLARE_BITMAP(used_mask, IDXD_PMU_EVENT_MAX); 124 125 struct pmu pmu; 126 char name[IDXD_NAME_SIZE]; 127 128 int n_counters; 129 int counter_width; 130 int n_event_categories; 131 132 bool per_counter_caps_supported; 133 unsigned long supported_event_categories; 134 135 unsigned long supported_filters; 136 int n_filters; 137 }; 138 139 #define IDXD_MAX_PRIORITY 0xf 140 141 enum { 142 COUNTER_FAULTS = 0, 143 COUNTER_FAULT_FAILS, 144 COUNTER_MAX 145 }; 146 147 enum idxd_wq_state { 148 IDXD_WQ_DISABLED = 0, 149 IDXD_WQ_ENABLED, 150 }; 151 152 enum idxd_wq_flag { 153 WQ_FLAG_DEDICATED = 0, 154 WQ_FLAG_BLOCK_ON_FAULT, 155 WQ_FLAG_ATS_DISABLE, 156 WQ_FLAG_PRS_DISABLE, 157 }; 158 159 enum idxd_wq_type { 160 IDXD_WQT_NONE = 0, 161 IDXD_WQT_KERNEL, 162 IDXD_WQT_USER, 163 }; 164 165 struct idxd_cdev { 166 struct idxd_wq *wq; 167 struct cdev cdev; 168 struct idxd_dev idxd_dev; 169 int minor; 170 }; 171 172 #define DRIVER_NAME_SIZE 128 173 174 #define IDXD_ALLOCATED_BATCH_SIZE 128U 175 #define WQ_NAME_SIZE 1024 176 #define WQ_TYPE_SIZE 10 177 178 #define WQ_DEFAULT_QUEUE_DEPTH 16 179 #define WQ_DEFAULT_MAX_XFER SZ_2M 180 #define WQ_DEFAULT_MAX_BATCH 32 181 182 enum idxd_op_type { 183 IDXD_OP_BLOCK = 0, 184 IDXD_OP_NONBLOCK = 1, 185 }; 186 187 struct idxd_dma_chan { 188 struct dma_chan chan; 189 struct idxd_wq *wq; 190 }; 191 192 struct idxd_wq { 193 void __iomem *portal; 194 u32 portal_offset; 195 unsigned int enqcmds_retries; 196 struct percpu_ref wq_active; 197 struct completion wq_dead; 198 struct completion wq_resurrect; 199 struct idxd_dev idxd_dev; 200 struct idxd_cdev *idxd_cdev; 201 struct wait_queue_head err_queue; 202 struct workqueue_struct *wq; 203 struct idxd_device *idxd; 204 int id; 205 struct idxd_irq_entry ie; 206 enum idxd_wq_type type; 207 struct idxd_group *group; 208 int client_count; 209 struct mutex wq_lock; /* mutex for workqueue */ 210 u32 size; 211 u32 threshold; 212 u32 priority; 213 enum idxd_wq_state state; 214 unsigned long flags; 215 union wqcfg *wqcfg; 216 unsigned long *opcap_bmap; 217 218 struct dsa_hw_desc **hw_descs; 219 int num_descs; 220 union { 221 struct dsa_completion_record *compls; 222 struct iax_completion_record *iax_compls; 223 }; 224 dma_addr_t compls_addr; 225 int compls_size; 226 struct idxd_desc **descs; 227 struct sbitmap_queue sbq; 228 struct idxd_dma_chan *idxd_chan; 229 char name[WQ_NAME_SIZE + 1]; 230 u64 max_xfer_bytes; 231 u32 max_batch_size; 232 233 /* Lock to protect upasid_xa access. */ 234 struct mutex uc_lock; 235 struct xarray upasid_xa; 236 237 char driver_name[DRIVER_NAME_SIZE + 1]; 238 }; 239 240 struct idxd_engine { 241 struct idxd_dev idxd_dev; 242 int id; 243 struct idxd_group *group; 244 struct idxd_device *idxd; 245 }; 246 247 /* shadow registers */ 248 struct idxd_hw { 249 u32 version; 250 union gen_cap_reg gen_cap; 251 union wq_cap_reg wq_cap; 252 union group_cap_reg group_cap; 253 union engine_cap_reg engine_cap; 254 struct opcap opcap; 255 u32 cmd_cap; 256 union iaa_cap_reg iaa_cap; 257 }; 258 259 enum idxd_device_state { 260 IDXD_DEV_HALTED = -1, 261 IDXD_DEV_DISABLED = 0, 262 IDXD_DEV_ENABLED, 263 }; 264 265 enum idxd_device_flag { 266 IDXD_FLAG_CONFIGURABLE = 0, 267 IDXD_FLAG_CMD_RUNNING, 268 IDXD_FLAG_PASID_ENABLED, 269 IDXD_FLAG_USER_PASID_ENABLED, 270 }; 271 272 struct idxd_dma_dev { 273 struct idxd_device *idxd; 274 struct dma_device dma; 275 }; 276 277 typedef int (*load_device_defaults_fn_t) (struct idxd_device *idxd); 278 279 struct idxd_driver_data { 280 const char *name_prefix; 281 enum idxd_type type; 282 const struct device_type *dev_type; 283 int compl_size; 284 int align; 285 int evl_cr_off; 286 int cr_status_off; 287 int cr_result_off; 288 bool user_submission_safe; 289 load_device_defaults_fn_t load_device_defaults; 290 }; 291 292 struct idxd_evl { 293 /* Lock to protect event log access. */ 294 struct mutex lock; 295 void *log; 296 dma_addr_t dma; 297 /* Total size of event log = number of entries * entry size. */ 298 unsigned int log_size; 299 /* The number of entries in the event log. */ 300 u16 size; 301 unsigned long *bmap; 302 bool batch_fail[IDXD_MAX_BATCH_IDENT]; 303 }; 304 305 struct idxd_evl_fault { 306 struct work_struct work; 307 struct idxd_wq *wq; 308 u8 status; 309 310 /* make this last member always */ 311 struct __evl_entry entry[]; 312 }; 313 314 struct idxd_device { 315 struct idxd_dev idxd_dev; 316 struct idxd_driver_data *data; 317 struct list_head list; 318 struct idxd_hw hw; 319 enum idxd_device_state state; 320 unsigned long flags; 321 int id; 322 int major; 323 u32 cmd_status; 324 struct idxd_irq_entry ie; /* misc irq, msix 0 */ 325 326 struct pci_dev *pdev; 327 void __iomem *reg_base; 328 329 spinlock_t dev_lock; /* spinlock for device */ 330 spinlock_t cmd_lock; /* spinlock for device commands */ 331 struct completion *cmd_done; 332 struct idxd_group **groups; 333 struct idxd_wq **wqs; 334 struct idxd_engine **engines; 335 336 struct iommu_sva *sva; 337 unsigned int pasid; 338 339 int num_groups; 340 int irq_cnt; 341 bool request_int_handles; 342 343 u32 msix_perm_offset; 344 u32 wqcfg_offset; 345 u32 grpcfg_offset; 346 u32 perfmon_offset; 347 348 u64 max_xfer_bytes; 349 u32 max_batch_size; 350 int max_groups; 351 int max_engines; 352 int max_rdbufs; 353 int max_wqs; 354 int max_wq_size; 355 int rdbuf_limit; 356 int nr_rdbufs; /* non-reserved read buffers */ 357 unsigned int wqcfg_size; 358 unsigned long *wq_enable_map; 359 360 union sw_err_reg sw_err; 361 wait_queue_head_t cmd_waitq; 362 363 struct idxd_dma_dev *idxd_dma; 364 struct workqueue_struct *wq; 365 struct work_struct work; 366 367 struct idxd_pmu *idxd_pmu; 368 369 unsigned long *opcap_bmap; 370 struct idxd_evl *evl; 371 struct kmem_cache *evl_cache; 372 373 struct dentry *dbgfs_dir; 374 struct dentry *dbgfs_evl_file; 375 376 bool user_submission_safe; 377 378 struct idxd_saved_states *idxd_saved; 379 }; 380 381 struct idxd_saved_states { 382 struct idxd_device saved_idxd; 383 struct idxd_evl saved_evl; 384 struct idxd_engine **saved_engines; 385 struct idxd_wq **saved_wqs; 386 struct idxd_group **saved_groups; 387 unsigned long *saved_wq_enable_map; 388 }; 389 390 static inline unsigned int evl_ent_size(struct idxd_device *idxd) 391 { 392 return idxd->hw.gen_cap.evl_support ? 393 (32 * (1 << idxd->hw.gen_cap.evl_support)) : 0; 394 } 395 396 static inline unsigned int evl_size(struct idxd_device *idxd) 397 { 398 return idxd->evl->size * evl_ent_size(idxd); 399 } 400 401 struct crypto_ctx { 402 struct acomp_req *req; 403 struct crypto_tfm *tfm; 404 dma_addr_t src_addr; 405 dma_addr_t dst_addr; 406 bool compress; 407 }; 408 409 /* IDXD software descriptor */ 410 struct idxd_desc { 411 union { 412 struct dsa_hw_desc *hw; 413 struct iax_hw_desc *iax_hw; 414 }; 415 dma_addr_t desc_dma; 416 union { 417 struct dsa_completion_record *completion; 418 struct iax_completion_record *iax_completion; 419 }; 420 dma_addr_t compl_dma; 421 union { 422 struct dma_async_tx_descriptor txd; 423 struct crypto_ctx crypto; 424 }; 425 struct llist_node llnode; 426 struct list_head list; 427 int id; 428 int cpu; 429 struct idxd_wq *wq; 430 }; 431 432 /* 433 * This is software defined error for the completion status. We overload the error code 434 * that will never appear in completion status and only SWERR register. 435 */ 436 enum idxd_completion_status { 437 IDXD_COMP_DESC_ABORT = 0xff, 438 }; 439 440 #define idxd_confdev(idxd) &idxd->idxd_dev.conf_dev 441 #define wq_confdev(wq) &wq->idxd_dev.conf_dev 442 #define engine_confdev(engine) &engine->idxd_dev.conf_dev 443 #define group_confdev(group) &group->idxd_dev.conf_dev 444 #define cdev_dev(cdev) &cdev->idxd_dev.conf_dev 445 #define user_ctx_dev(ctx) (&(ctx)->idxd_dev.conf_dev) 446 447 #define confdev_to_idxd_dev(dev) container_of(dev, struct idxd_dev, conf_dev) 448 #define idxd_dev_to_idxd(idxd_dev) container_of(idxd_dev, struct idxd_device, idxd_dev) 449 #define idxd_dev_to_wq(idxd_dev) container_of(idxd_dev, struct idxd_wq, idxd_dev) 450 451 static inline struct idxd_device_driver *wq_to_idxd_drv(struct idxd_wq *wq) 452 { 453 struct device *dev = wq_confdev(wq); 454 struct idxd_device_driver *idxd_drv = 455 container_of(dev->driver, struct idxd_device_driver, drv); 456 457 return idxd_drv; 458 } 459 460 static inline struct idxd_device *confdev_to_idxd(struct device *dev) 461 { 462 struct idxd_dev *idxd_dev = confdev_to_idxd_dev(dev); 463 464 return idxd_dev_to_idxd(idxd_dev); 465 } 466 467 static inline struct idxd_wq *confdev_to_wq(struct device *dev) 468 { 469 struct idxd_dev *idxd_dev = confdev_to_idxd_dev(dev); 470 471 return idxd_dev_to_wq(idxd_dev); 472 } 473 474 static inline struct idxd_engine *confdev_to_engine(struct device *dev) 475 { 476 struct idxd_dev *idxd_dev = confdev_to_idxd_dev(dev); 477 478 return container_of(idxd_dev, struct idxd_engine, idxd_dev); 479 } 480 481 static inline struct idxd_group *confdev_to_group(struct device *dev) 482 { 483 struct idxd_dev *idxd_dev = confdev_to_idxd_dev(dev); 484 485 return container_of(idxd_dev, struct idxd_group, idxd_dev); 486 } 487 488 static inline struct idxd_cdev *dev_to_cdev(struct device *dev) 489 { 490 struct idxd_dev *idxd_dev = confdev_to_idxd_dev(dev); 491 492 return container_of(idxd_dev, struct idxd_cdev, idxd_dev); 493 } 494 495 static inline void idxd_dev_set_type(struct idxd_dev *idev, int type) 496 { 497 if (type >= IDXD_DEV_MAX_TYPE) { 498 idev->type = IDXD_DEV_NONE; 499 return; 500 } 501 502 idev->type = type; 503 } 504 505 static inline struct idxd_irq_entry *idxd_get_ie(struct idxd_device *idxd, int idx) 506 { 507 return (idx == 0) ? &idxd->ie : &idxd->wqs[idx - 1]->ie; 508 } 509 510 static inline struct idxd_wq *ie_to_wq(struct idxd_irq_entry *ie) 511 { 512 return container_of(ie, struct idxd_wq, ie); 513 } 514 515 static inline struct idxd_device *ie_to_idxd(struct idxd_irq_entry *ie) 516 { 517 return container_of(ie, struct idxd_device, ie); 518 } 519 520 static inline void idxd_set_user_intr(struct idxd_device *idxd, bool enable) 521 { 522 union gencfg_reg reg; 523 524 reg.bits = ioread32(idxd->reg_base + IDXD_GENCFG_OFFSET); 525 reg.user_int_en = enable; 526 iowrite32(reg.bits, idxd->reg_base + IDXD_GENCFG_OFFSET); 527 } 528 529 extern const struct bus_type dsa_bus_type; 530 531 extern bool support_enqcmd; 532 extern struct ida idxd_ida; 533 extern const struct device_type dsa_device_type; 534 extern const struct device_type iax_device_type; 535 extern const struct device_type idxd_wq_device_type; 536 extern const struct device_type idxd_engine_device_type; 537 extern const struct device_type idxd_group_device_type; 538 539 static inline bool is_dsa_dev(struct idxd_dev *idxd_dev) 540 { 541 return idxd_dev->type == IDXD_DEV_DSA; 542 } 543 544 static inline bool is_iax_dev(struct idxd_dev *idxd_dev) 545 { 546 return idxd_dev->type == IDXD_DEV_IAX; 547 } 548 549 static inline bool is_idxd_dev(struct idxd_dev *idxd_dev) 550 { 551 return is_dsa_dev(idxd_dev) || is_iax_dev(idxd_dev); 552 } 553 554 static inline bool is_idxd_wq_dev(struct idxd_dev *idxd_dev) 555 { 556 return idxd_dev->type == IDXD_DEV_WQ; 557 } 558 559 static inline bool is_idxd_wq_dmaengine(struct idxd_wq *wq) 560 { 561 if (wq->type == IDXD_WQT_KERNEL && strcmp(wq->name, "dmaengine") == 0) 562 return true; 563 return false; 564 } 565 566 static inline bool is_idxd_wq_user(struct idxd_wq *wq) 567 { 568 return wq->type == IDXD_WQT_USER; 569 } 570 571 static inline bool is_idxd_wq_kernel(struct idxd_wq *wq) 572 { 573 return wq->type == IDXD_WQT_KERNEL; 574 } 575 576 static inline bool wq_dedicated(struct idxd_wq *wq) 577 { 578 return test_bit(WQ_FLAG_DEDICATED, &wq->flags); 579 } 580 581 static inline bool wq_shared(struct idxd_wq *wq) 582 { 583 return !test_bit(WQ_FLAG_DEDICATED, &wq->flags); 584 } 585 586 static inline bool device_pasid_enabled(struct idxd_device *idxd) 587 { 588 return test_bit(IDXD_FLAG_PASID_ENABLED, &idxd->flags); 589 } 590 591 static inline bool device_user_pasid_enabled(struct idxd_device *idxd) 592 { 593 return test_bit(IDXD_FLAG_USER_PASID_ENABLED, &idxd->flags); 594 } 595 596 static inline bool wq_pasid_enabled(struct idxd_wq *wq) 597 { 598 return (is_idxd_wq_kernel(wq) && device_pasid_enabled(wq->idxd)) || 599 (is_idxd_wq_user(wq) && device_user_pasid_enabled(wq->idxd)); 600 } 601 602 static inline bool wq_shared_supported(struct idxd_wq *wq) 603 { 604 return (support_enqcmd && wq_pasid_enabled(wq)); 605 } 606 607 enum idxd_portal_prot { 608 IDXD_PORTAL_UNLIMITED = 0, 609 IDXD_PORTAL_LIMITED, 610 }; 611 612 enum idxd_interrupt_type { 613 IDXD_IRQ_MSIX = 0, 614 IDXD_IRQ_IMS, 615 }; 616 617 static inline int idxd_get_wq_portal_offset(enum idxd_portal_prot prot) 618 { 619 return prot * 0x1000; 620 } 621 622 static inline int idxd_get_wq_portal_full_offset(int wq_id, 623 enum idxd_portal_prot prot) 624 { 625 return ((wq_id * 4) << PAGE_SHIFT) + idxd_get_wq_portal_offset(prot); 626 } 627 628 #define IDXD_PORTAL_MASK (PAGE_SIZE - 1) 629 630 /* 631 * Even though this function can be accessed by multiple threads, it is safe to use. 632 * At worst the address gets used more than once before it gets incremented. We don't 633 * hit a threshold until iops becomes many million times a second. So the occasional 634 * reuse of the same address is tolerable compare to using an atomic variable. This is 635 * safe on a system that has atomic load/store for 32bit integers. Given that this is an 636 * Intel iEP device, that should not be a problem. 637 */ 638 static inline void __iomem *idxd_wq_portal_addr(struct idxd_wq *wq) 639 { 640 int ofs = wq->portal_offset; 641 642 wq->portal_offset = (ofs + sizeof(struct dsa_raw_desc)) & IDXD_PORTAL_MASK; 643 return wq->portal + ofs; 644 } 645 646 static inline void idxd_wq_get(struct idxd_wq *wq) 647 { 648 wq->client_count++; 649 } 650 651 static inline void idxd_wq_put(struct idxd_wq *wq) 652 { 653 wq->client_count--; 654 } 655 656 static inline int idxd_wq_refcount(struct idxd_wq *wq) 657 { 658 return wq->client_count; 659 }; 660 661 static inline void idxd_wq_set_private(struct idxd_wq *wq, void *private) 662 { 663 dev_set_drvdata(wq_confdev(wq), private); 664 } 665 666 static inline void *idxd_wq_get_private(struct idxd_wq *wq) 667 { 668 return dev_get_drvdata(wq_confdev(wq)); 669 } 670 671 /* 672 * Intel IAA does not support batch processing. 673 * The max batch size of device, max batch size of wq and 674 * max batch shift of wqcfg should be always 0 on IAA. 675 */ 676 static inline void idxd_set_max_batch_size(int idxd_type, struct idxd_device *idxd, 677 u32 max_batch_size) 678 { 679 if (idxd_type == IDXD_TYPE_IAX) 680 idxd->max_batch_size = 0; 681 else 682 idxd->max_batch_size = max_batch_size; 683 } 684 685 static inline void idxd_wq_set_max_batch_size(int idxd_type, struct idxd_wq *wq, 686 u32 max_batch_size) 687 { 688 if (idxd_type == IDXD_TYPE_IAX) 689 wq->max_batch_size = 0; 690 else 691 wq->max_batch_size = max_batch_size; 692 } 693 694 static inline void idxd_wqcfg_set_max_batch_shift(int idxd_type, union wqcfg *wqcfg, 695 u32 max_batch_shift) 696 { 697 if (idxd_type == IDXD_TYPE_IAX) 698 wqcfg->max_batch_shift = 0; 699 else 700 wqcfg->max_batch_shift = max_batch_shift; 701 } 702 703 static inline int idxd_wq_driver_name_match(struct idxd_wq *wq, struct device *dev) 704 { 705 return (strncmp(wq->driver_name, dev->driver->name, strlen(dev->driver->name)) == 0); 706 } 707 708 #define MODULE_ALIAS_IDXD_DEVICE(type) MODULE_ALIAS("idxd:t" __stringify(type) "*") 709 #define IDXD_DEVICES_MODALIAS_FMT "idxd:t%d" 710 711 int __must_check __idxd_driver_register(struct idxd_device_driver *idxd_drv, 712 struct module *module, const char *mod_name); 713 #define idxd_driver_register(driver) \ 714 __idxd_driver_register(driver, THIS_MODULE, KBUILD_MODNAME) 715 716 void idxd_driver_unregister(struct idxd_device_driver *idxd_drv); 717 718 #define module_idxd_driver(__idxd_driver) \ 719 module_driver(__idxd_driver, idxd_driver_register, idxd_driver_unregister) 720 721 void idxd_free_desc(struct idxd_wq *wq, struct idxd_desc *desc); 722 void idxd_dma_complete_txd(struct idxd_desc *desc, 723 enum idxd_complete_type comp_type, 724 bool free_desc, void *ctx, u32 *status); 725 726 static inline void idxd_desc_complete(struct idxd_desc *desc, 727 enum idxd_complete_type comp_type, 728 bool free_desc) 729 { 730 struct idxd_device_driver *drv; 731 u32 status; 732 733 drv = wq_to_idxd_drv(desc->wq); 734 if (drv->desc_complete) 735 drv->desc_complete(desc, comp_type, free_desc, 736 &desc->txd, &status); 737 } 738 739 int idxd_register_devices(struct idxd_device *idxd); 740 void idxd_unregister_devices(struct idxd_device *idxd); 741 void idxd_wqs_quiesce(struct idxd_device *idxd); 742 bool idxd_queue_int_handle_resubmit(struct idxd_desc *desc); 743 void multi_u64_to_bmap(unsigned long *bmap, u64 *val, int count); 744 int idxd_load_iaa_device_defaults(struct idxd_device *idxd); 745 746 /* device interrupt control */ 747 irqreturn_t idxd_misc_thread(int vec, void *data); 748 irqreturn_t idxd_wq_thread(int irq, void *data); 749 void idxd_mask_error_interrupts(struct idxd_device *idxd); 750 void idxd_unmask_error_interrupts(struct idxd_device *idxd); 751 752 /* device control */ 753 int idxd_device_drv_probe(struct idxd_dev *idxd_dev); 754 int idxd_pci_probe_alloc(struct idxd_device *idxd, struct pci_dev *pdev, 755 const struct pci_device_id *id); 756 void idxd_device_drv_remove(struct idxd_dev *idxd_dev); 757 int idxd_drv_enable_wq(struct idxd_wq *wq); 758 void idxd_drv_disable_wq(struct idxd_wq *wq); 759 int idxd_device_init_reset(struct idxd_device *idxd); 760 int idxd_device_enable(struct idxd_device *idxd); 761 int idxd_device_disable(struct idxd_device *idxd); 762 void idxd_device_reset(struct idxd_device *idxd); 763 void idxd_device_clear_state(struct idxd_device *idxd); 764 int idxd_device_config(struct idxd_device *idxd); 765 void idxd_device_drain_pasid(struct idxd_device *idxd, int pasid); 766 int idxd_device_load_config(struct idxd_device *idxd); 767 int idxd_device_request_int_handle(struct idxd_device *idxd, int idx, int *handle, 768 enum idxd_interrupt_type irq_type); 769 int idxd_device_release_int_handle(struct idxd_device *idxd, int handle, 770 enum idxd_interrupt_type irq_type); 771 772 /* work queue control */ 773 void idxd_wqs_unmap_portal(struct idxd_device *idxd); 774 int idxd_wq_alloc_resources(struct idxd_wq *wq); 775 void idxd_wq_free_resources(struct idxd_wq *wq); 776 int idxd_wq_enable(struct idxd_wq *wq); 777 int idxd_wq_disable(struct idxd_wq *wq, bool reset_config); 778 void idxd_wq_drain(struct idxd_wq *wq); 779 void idxd_wq_reset(struct idxd_wq *wq); 780 int idxd_wq_map_portal(struct idxd_wq *wq); 781 void idxd_wq_unmap_portal(struct idxd_wq *wq); 782 int idxd_wq_set_pasid(struct idxd_wq *wq, int pasid); 783 int idxd_wq_disable_pasid(struct idxd_wq *wq); 784 void __idxd_wq_quiesce(struct idxd_wq *wq); 785 void idxd_wq_quiesce(struct idxd_wq *wq); 786 int idxd_wq_init_percpu_ref(struct idxd_wq *wq); 787 void idxd_wq_free_irq(struct idxd_wq *wq); 788 int idxd_wq_request_irq(struct idxd_wq *wq); 789 790 /* submission */ 791 int idxd_submit_desc(struct idxd_wq *wq, struct idxd_desc *desc); 792 struct idxd_desc *idxd_alloc_desc(struct idxd_wq *wq, enum idxd_op_type optype); 793 int idxd_enqcmds(struct idxd_wq *wq, void __iomem *portal, const void *desc); 794 795 /* dmaengine */ 796 int idxd_register_dma_device(struct idxd_device *idxd); 797 void idxd_unregister_dma_device(struct idxd_device *idxd); 798 799 /* cdev */ 800 int idxd_cdev_register(void); 801 void idxd_cdev_remove(void); 802 int idxd_cdev_get_major(struct idxd_device *idxd); 803 int idxd_wq_add_cdev(struct idxd_wq *wq); 804 void idxd_wq_del_cdev(struct idxd_wq *wq); 805 int idxd_copy_cr(struct idxd_wq *wq, ioasid_t pasid, unsigned long addr, 806 void *buf, int len); 807 void idxd_user_counter_increment(struct idxd_wq *wq, u32 pasid, int index); 808 809 /* perfmon */ 810 #if IS_ENABLED(CONFIG_INTEL_IDXD_PERFMON) 811 int perfmon_pmu_init(struct idxd_device *idxd); 812 void perfmon_pmu_remove(struct idxd_device *idxd); 813 void perfmon_counter_overflow(struct idxd_device *idxd); 814 #else 815 static inline int perfmon_pmu_init(struct idxd_device *idxd) { return 0; } 816 static inline void perfmon_pmu_remove(struct idxd_device *idxd) {} 817 static inline void perfmon_counter_overflow(struct idxd_device *idxd) {} 818 #endif 819 820 /* debugfs */ 821 int idxd_device_init_debugfs(struct idxd_device *idxd); 822 void idxd_device_remove_debugfs(struct idxd_device *idxd); 823 int idxd_init_debugfs(void); 824 void idxd_remove_debugfs(void); 825 826 #endif 827