1 /* SPDX-License-Identifier: GPL-2.0 */ 2 /* Copyright(c) 2019 Intel Corporation. All rights rsvd. */ 3 #ifndef _IDXD_H_ 4 #define _IDXD_H_ 5 6 #include <linux/sbitmap.h> 7 #include <linux/dmaengine.h> 8 #include <linux/percpu-rwsem.h> 9 #include <linux/wait.h> 10 #include <linux/cdev.h> 11 #include <linux/idr.h> 12 #include <linux/pci.h> 13 #include <linux/bitmap.h> 14 #include <linux/perf_event.h> 15 #include <linux/iommu.h> 16 #include <uapi/linux/idxd.h> 17 #include "registers.h" 18 19 #define IDXD_DRIVER_VERSION "1.00" 20 21 extern struct kmem_cache *idxd_desc_pool; 22 extern bool tc_override; 23 24 struct idxd_wq; 25 struct idxd_dev; 26 27 enum idxd_dev_type { 28 IDXD_DEV_NONE = -1, 29 IDXD_DEV_DSA = 0, 30 IDXD_DEV_IAX, 31 IDXD_DEV_WQ, 32 IDXD_DEV_GROUP, 33 IDXD_DEV_ENGINE, 34 IDXD_DEV_CDEV, 35 IDXD_DEV_CDEV_FILE, 36 IDXD_DEV_MAX_TYPE, 37 }; 38 39 struct idxd_dev { 40 struct device conf_dev; 41 enum idxd_dev_type type; 42 }; 43 44 #define IDXD_REG_TIMEOUT 50 45 #define IDXD_DRAIN_TIMEOUT 5000 46 47 enum idxd_type { 48 IDXD_TYPE_UNKNOWN = -1, 49 IDXD_TYPE_DSA = 0, 50 IDXD_TYPE_IAX, 51 IDXD_TYPE_MAX, 52 }; 53 54 #define IDXD_NAME_SIZE 128 55 #define IDXD_PMU_EVENT_MAX 64 56 57 #define IDXD_ENQCMDS_RETRIES 32 58 #define IDXD_ENQCMDS_MAX_RETRIES 64 59 60 struct idxd_device_driver { 61 const char *name; 62 enum idxd_dev_type *type; 63 int (*probe)(struct idxd_dev *idxd_dev); 64 void (*remove)(struct idxd_dev *idxd_dev); 65 struct device_driver drv; 66 }; 67 68 extern struct idxd_device_driver dsa_drv; 69 extern struct idxd_device_driver idxd_drv; 70 extern struct idxd_device_driver idxd_dmaengine_drv; 71 extern struct idxd_device_driver idxd_user_drv; 72 73 #define INVALID_INT_HANDLE -1 74 struct idxd_irq_entry { 75 int id; 76 int vector; 77 struct llist_head pending_llist; 78 struct list_head work_list; 79 /* 80 * Lock to protect access between irq thread process descriptor 81 * and irq thread processing error descriptor. 82 */ 83 spinlock_t list_lock; 84 int int_handle; 85 ioasid_t pasid; 86 }; 87 88 struct idxd_group { 89 struct idxd_dev idxd_dev; 90 struct idxd_device *idxd; 91 struct grpcfg grpcfg; 92 int id; 93 int num_engines; 94 int num_wqs; 95 bool use_rdbuf_limit; 96 u8 rdbufs_allowed; 97 u8 rdbufs_reserved; 98 int tc_a; 99 int tc_b; 100 int desc_progress_limit; 101 int batch_progress_limit; 102 }; 103 104 struct idxd_pmu { 105 struct idxd_device *idxd; 106 107 struct perf_event *event_list[IDXD_PMU_EVENT_MAX]; 108 int n_events; 109 110 DECLARE_BITMAP(used_mask, IDXD_PMU_EVENT_MAX); 111 112 struct pmu pmu; 113 char name[IDXD_NAME_SIZE]; 114 int cpu; 115 116 int n_counters; 117 int counter_width; 118 int n_event_categories; 119 120 bool per_counter_caps_supported; 121 unsigned long supported_event_categories; 122 123 unsigned long supported_filters; 124 int n_filters; 125 126 struct hlist_node cpuhp_node; 127 }; 128 129 #define IDXD_MAX_PRIORITY 0xf 130 131 enum { 132 COUNTER_FAULTS = 0, 133 COUNTER_FAULT_FAILS, 134 COUNTER_MAX 135 }; 136 137 enum idxd_wq_state { 138 IDXD_WQ_DISABLED = 0, 139 IDXD_WQ_ENABLED, 140 }; 141 142 enum idxd_wq_flag { 143 WQ_FLAG_DEDICATED = 0, 144 WQ_FLAG_BLOCK_ON_FAULT, 145 WQ_FLAG_ATS_DISABLE, 146 WQ_FLAG_PRS_DISABLE, 147 }; 148 149 enum idxd_wq_type { 150 IDXD_WQT_NONE = 0, 151 IDXD_WQT_KERNEL, 152 IDXD_WQT_USER, 153 }; 154 155 struct idxd_cdev { 156 struct idxd_wq *wq; 157 struct cdev cdev; 158 struct idxd_dev idxd_dev; 159 int minor; 160 }; 161 162 #define DRIVER_NAME_SIZE 128 163 164 #define IDXD_ALLOCATED_BATCH_SIZE 128U 165 #define WQ_NAME_SIZE 1024 166 #define WQ_TYPE_SIZE 10 167 168 #define WQ_DEFAULT_QUEUE_DEPTH 16 169 #define WQ_DEFAULT_MAX_XFER SZ_2M 170 #define WQ_DEFAULT_MAX_BATCH 32 171 172 enum idxd_op_type { 173 IDXD_OP_BLOCK = 0, 174 IDXD_OP_NONBLOCK = 1, 175 }; 176 177 enum idxd_complete_type { 178 IDXD_COMPLETE_NORMAL = 0, 179 IDXD_COMPLETE_ABORT, 180 IDXD_COMPLETE_DEV_FAIL, 181 }; 182 183 struct idxd_dma_chan { 184 struct dma_chan chan; 185 struct idxd_wq *wq; 186 }; 187 188 struct idxd_wq { 189 void __iomem *portal; 190 u32 portal_offset; 191 unsigned int enqcmds_retries; 192 struct percpu_ref wq_active; 193 struct completion wq_dead; 194 struct completion wq_resurrect; 195 struct idxd_dev idxd_dev; 196 struct idxd_cdev *idxd_cdev; 197 struct wait_queue_head err_queue; 198 struct workqueue_struct *wq; 199 struct idxd_device *idxd; 200 int id; 201 struct idxd_irq_entry ie; 202 enum idxd_wq_type type; 203 struct idxd_group *group; 204 int client_count; 205 struct mutex wq_lock; /* mutex for workqueue */ 206 u32 size; 207 u32 threshold; 208 u32 priority; 209 enum idxd_wq_state state; 210 unsigned long flags; 211 union wqcfg *wqcfg; 212 unsigned long *opcap_bmap; 213 214 struct dsa_hw_desc **hw_descs; 215 int num_descs; 216 union { 217 struct dsa_completion_record *compls; 218 struct iax_completion_record *iax_compls; 219 }; 220 dma_addr_t compls_addr; 221 int compls_size; 222 struct idxd_desc **descs; 223 struct sbitmap_queue sbq; 224 struct idxd_dma_chan *idxd_chan; 225 char name[WQ_NAME_SIZE + 1]; 226 u64 max_xfer_bytes; 227 u32 max_batch_size; 228 229 /* Lock to protect upasid_xa access. */ 230 struct mutex uc_lock; 231 struct xarray upasid_xa; 232 233 char driver_name[DRIVER_NAME_SIZE + 1]; 234 }; 235 236 struct idxd_engine { 237 struct idxd_dev idxd_dev; 238 int id; 239 struct idxd_group *group; 240 struct idxd_device *idxd; 241 }; 242 243 /* shadow registers */ 244 struct idxd_hw { 245 u32 version; 246 union gen_cap_reg gen_cap; 247 union wq_cap_reg wq_cap; 248 union group_cap_reg group_cap; 249 union engine_cap_reg engine_cap; 250 struct opcap opcap; 251 u32 cmd_cap; 252 union iaa_cap_reg iaa_cap; 253 }; 254 255 enum idxd_device_state { 256 IDXD_DEV_HALTED = -1, 257 IDXD_DEV_DISABLED = 0, 258 IDXD_DEV_ENABLED, 259 }; 260 261 enum idxd_device_flag { 262 IDXD_FLAG_CONFIGURABLE = 0, 263 IDXD_FLAG_CMD_RUNNING, 264 IDXD_FLAG_PASID_ENABLED, 265 IDXD_FLAG_USER_PASID_ENABLED, 266 }; 267 268 struct idxd_dma_dev { 269 struct idxd_device *idxd; 270 struct dma_device dma; 271 }; 272 273 struct idxd_driver_data { 274 const char *name_prefix; 275 enum idxd_type type; 276 struct device_type *dev_type; 277 int compl_size; 278 int align; 279 int evl_cr_off; 280 int cr_status_off; 281 int cr_result_off; 282 }; 283 284 struct idxd_evl { 285 /* Lock to protect event log access. */ 286 spinlock_t lock; 287 void *log; 288 dma_addr_t dma; 289 /* Total size of event log = number of entries * entry size. */ 290 unsigned int log_size; 291 /* The number of entries in the event log. */ 292 u16 size; 293 u16 head; 294 unsigned long *bmap; 295 bool batch_fail[IDXD_MAX_BATCH_IDENT]; 296 }; 297 298 struct idxd_evl_fault { 299 struct work_struct work; 300 struct idxd_wq *wq; 301 u8 status; 302 303 /* make this last member always */ 304 struct __evl_entry entry[]; 305 }; 306 307 struct idxd_device { 308 struct idxd_dev idxd_dev; 309 struct idxd_driver_data *data; 310 struct list_head list; 311 struct idxd_hw hw; 312 enum idxd_device_state state; 313 unsigned long flags; 314 int id; 315 int major; 316 u32 cmd_status; 317 struct idxd_irq_entry ie; /* misc irq, msix 0 */ 318 319 struct pci_dev *pdev; 320 void __iomem *reg_base; 321 322 spinlock_t dev_lock; /* spinlock for device */ 323 spinlock_t cmd_lock; /* spinlock for device commands */ 324 struct completion *cmd_done; 325 struct idxd_group **groups; 326 struct idxd_wq **wqs; 327 struct idxd_engine **engines; 328 329 struct iommu_sva *sva; 330 unsigned int pasid; 331 332 int num_groups; 333 int irq_cnt; 334 bool request_int_handles; 335 336 u32 msix_perm_offset; 337 u32 wqcfg_offset; 338 u32 grpcfg_offset; 339 u32 perfmon_offset; 340 341 u64 max_xfer_bytes; 342 u32 max_batch_size; 343 int max_groups; 344 int max_engines; 345 int max_rdbufs; 346 int max_wqs; 347 int max_wq_size; 348 int rdbuf_limit; 349 int nr_rdbufs; /* non-reserved read buffers */ 350 unsigned int wqcfg_size; 351 unsigned long *wq_enable_map; 352 353 union sw_err_reg sw_err; 354 wait_queue_head_t cmd_waitq; 355 356 struct idxd_dma_dev *idxd_dma; 357 struct workqueue_struct *wq; 358 struct work_struct work; 359 360 struct idxd_pmu *idxd_pmu; 361 362 unsigned long *opcap_bmap; 363 struct idxd_evl *evl; 364 struct kmem_cache *evl_cache; 365 366 struct dentry *dbgfs_dir; 367 struct dentry *dbgfs_evl_file; 368 }; 369 370 static inline unsigned int evl_ent_size(struct idxd_device *idxd) 371 { 372 return idxd->hw.gen_cap.evl_support ? 373 (32 * (1 << idxd->hw.gen_cap.evl_support)) : 0; 374 } 375 376 static inline unsigned int evl_size(struct idxd_device *idxd) 377 { 378 return idxd->evl->size * evl_ent_size(idxd); 379 } 380 381 /* IDXD software descriptor */ 382 struct idxd_desc { 383 union { 384 struct dsa_hw_desc *hw; 385 struct iax_hw_desc *iax_hw; 386 }; 387 dma_addr_t desc_dma; 388 union { 389 struct dsa_completion_record *completion; 390 struct iax_completion_record *iax_completion; 391 }; 392 dma_addr_t compl_dma; 393 struct dma_async_tx_descriptor txd; 394 struct llist_node llnode; 395 struct list_head list; 396 int id; 397 int cpu; 398 struct idxd_wq *wq; 399 }; 400 401 /* 402 * This is software defined error for the completion status. We overload the error code 403 * that will never appear in completion status and only SWERR register. 404 */ 405 enum idxd_completion_status { 406 IDXD_COMP_DESC_ABORT = 0xff, 407 }; 408 409 #define idxd_confdev(idxd) &idxd->idxd_dev.conf_dev 410 #define wq_confdev(wq) &wq->idxd_dev.conf_dev 411 #define engine_confdev(engine) &engine->idxd_dev.conf_dev 412 #define group_confdev(group) &group->idxd_dev.conf_dev 413 #define cdev_dev(cdev) &cdev->idxd_dev.conf_dev 414 #define user_ctx_dev(ctx) (&(ctx)->idxd_dev.conf_dev) 415 416 #define confdev_to_idxd_dev(dev) container_of(dev, struct idxd_dev, conf_dev) 417 #define idxd_dev_to_idxd(idxd_dev) container_of(idxd_dev, struct idxd_device, idxd_dev) 418 #define idxd_dev_to_wq(idxd_dev) container_of(idxd_dev, struct idxd_wq, idxd_dev) 419 420 static inline struct idxd_device *confdev_to_idxd(struct device *dev) 421 { 422 struct idxd_dev *idxd_dev = confdev_to_idxd_dev(dev); 423 424 return idxd_dev_to_idxd(idxd_dev); 425 } 426 427 static inline struct idxd_wq *confdev_to_wq(struct device *dev) 428 { 429 struct idxd_dev *idxd_dev = confdev_to_idxd_dev(dev); 430 431 return idxd_dev_to_wq(idxd_dev); 432 } 433 434 static inline struct idxd_engine *confdev_to_engine(struct device *dev) 435 { 436 struct idxd_dev *idxd_dev = confdev_to_idxd_dev(dev); 437 438 return container_of(idxd_dev, struct idxd_engine, idxd_dev); 439 } 440 441 static inline struct idxd_group *confdev_to_group(struct device *dev) 442 { 443 struct idxd_dev *idxd_dev = confdev_to_idxd_dev(dev); 444 445 return container_of(idxd_dev, struct idxd_group, idxd_dev); 446 } 447 448 static inline struct idxd_cdev *dev_to_cdev(struct device *dev) 449 { 450 struct idxd_dev *idxd_dev = confdev_to_idxd_dev(dev); 451 452 return container_of(idxd_dev, struct idxd_cdev, idxd_dev); 453 } 454 455 static inline void idxd_dev_set_type(struct idxd_dev *idev, int type) 456 { 457 if (type >= IDXD_DEV_MAX_TYPE) { 458 idev->type = IDXD_DEV_NONE; 459 return; 460 } 461 462 idev->type = type; 463 } 464 465 static inline struct idxd_irq_entry *idxd_get_ie(struct idxd_device *idxd, int idx) 466 { 467 return (idx == 0) ? &idxd->ie : &idxd->wqs[idx - 1]->ie; 468 } 469 470 static inline struct idxd_wq *ie_to_wq(struct idxd_irq_entry *ie) 471 { 472 return container_of(ie, struct idxd_wq, ie); 473 } 474 475 static inline struct idxd_device *ie_to_idxd(struct idxd_irq_entry *ie) 476 { 477 return container_of(ie, struct idxd_device, ie); 478 } 479 480 static inline void idxd_set_user_intr(struct idxd_device *idxd, bool enable) 481 { 482 union gencfg_reg reg; 483 484 reg.bits = ioread32(idxd->reg_base + IDXD_GENCFG_OFFSET); 485 reg.user_int_en = enable; 486 iowrite32(reg.bits, idxd->reg_base + IDXD_GENCFG_OFFSET); 487 } 488 489 extern struct bus_type dsa_bus_type; 490 491 extern bool support_enqcmd; 492 extern struct ida idxd_ida; 493 extern struct device_type dsa_device_type; 494 extern struct device_type iax_device_type; 495 extern struct device_type idxd_wq_device_type; 496 extern struct device_type idxd_engine_device_type; 497 extern struct device_type idxd_group_device_type; 498 499 static inline bool is_dsa_dev(struct idxd_dev *idxd_dev) 500 { 501 return idxd_dev->type == IDXD_DEV_DSA; 502 } 503 504 static inline bool is_iax_dev(struct idxd_dev *idxd_dev) 505 { 506 return idxd_dev->type == IDXD_DEV_IAX; 507 } 508 509 static inline bool is_idxd_dev(struct idxd_dev *idxd_dev) 510 { 511 return is_dsa_dev(idxd_dev) || is_iax_dev(idxd_dev); 512 } 513 514 static inline bool is_idxd_wq_dev(struct idxd_dev *idxd_dev) 515 { 516 return idxd_dev->type == IDXD_DEV_WQ; 517 } 518 519 static inline bool is_idxd_wq_dmaengine(struct idxd_wq *wq) 520 { 521 if (wq->type == IDXD_WQT_KERNEL && strcmp(wq->name, "dmaengine") == 0) 522 return true; 523 return false; 524 } 525 526 static inline bool is_idxd_wq_user(struct idxd_wq *wq) 527 { 528 return wq->type == IDXD_WQT_USER; 529 } 530 531 static inline bool is_idxd_wq_kernel(struct idxd_wq *wq) 532 { 533 return wq->type == IDXD_WQT_KERNEL; 534 } 535 536 static inline bool wq_dedicated(struct idxd_wq *wq) 537 { 538 return test_bit(WQ_FLAG_DEDICATED, &wq->flags); 539 } 540 541 static inline bool wq_shared(struct idxd_wq *wq) 542 { 543 return !test_bit(WQ_FLAG_DEDICATED, &wq->flags); 544 } 545 546 static inline bool device_pasid_enabled(struct idxd_device *idxd) 547 { 548 return test_bit(IDXD_FLAG_PASID_ENABLED, &idxd->flags); 549 } 550 551 static inline bool device_user_pasid_enabled(struct idxd_device *idxd) 552 { 553 return test_bit(IDXD_FLAG_USER_PASID_ENABLED, &idxd->flags); 554 } 555 556 static inline bool wq_pasid_enabled(struct idxd_wq *wq) 557 { 558 return (is_idxd_wq_kernel(wq) && device_pasid_enabled(wq->idxd)) || 559 (is_idxd_wq_user(wq) && device_user_pasid_enabled(wq->idxd)); 560 } 561 562 static inline bool wq_shared_supported(struct idxd_wq *wq) 563 { 564 return (support_enqcmd && wq_pasid_enabled(wq)); 565 } 566 567 enum idxd_portal_prot { 568 IDXD_PORTAL_UNLIMITED = 0, 569 IDXD_PORTAL_LIMITED, 570 }; 571 572 enum idxd_interrupt_type { 573 IDXD_IRQ_MSIX = 0, 574 IDXD_IRQ_IMS, 575 }; 576 577 static inline int idxd_get_wq_portal_offset(enum idxd_portal_prot prot) 578 { 579 return prot * 0x1000; 580 } 581 582 static inline int idxd_get_wq_portal_full_offset(int wq_id, 583 enum idxd_portal_prot prot) 584 { 585 return ((wq_id * 4) << PAGE_SHIFT) + idxd_get_wq_portal_offset(prot); 586 } 587 588 #define IDXD_PORTAL_MASK (PAGE_SIZE - 1) 589 590 /* 591 * Even though this function can be accessed by multiple threads, it is safe to use. 592 * At worst the address gets used more than once before it gets incremented. We don't 593 * hit a threshold until iops becomes many million times a second. So the occasional 594 * reuse of the same address is tolerable compare to using an atomic variable. This is 595 * safe on a system that has atomic load/store for 32bit integers. Given that this is an 596 * Intel iEP device, that should not be a problem. 597 */ 598 static inline void __iomem *idxd_wq_portal_addr(struct idxd_wq *wq) 599 { 600 int ofs = wq->portal_offset; 601 602 wq->portal_offset = (ofs + sizeof(struct dsa_raw_desc)) & IDXD_PORTAL_MASK; 603 return wq->portal + ofs; 604 } 605 606 static inline void idxd_wq_get(struct idxd_wq *wq) 607 { 608 wq->client_count++; 609 } 610 611 static inline void idxd_wq_put(struct idxd_wq *wq) 612 { 613 wq->client_count--; 614 } 615 616 static inline int idxd_wq_refcount(struct idxd_wq *wq) 617 { 618 return wq->client_count; 619 }; 620 621 /* 622 * Intel IAA does not support batch processing. 623 * The max batch size of device, max batch size of wq and 624 * max batch shift of wqcfg should be always 0 on IAA. 625 */ 626 static inline void idxd_set_max_batch_size(int idxd_type, struct idxd_device *idxd, 627 u32 max_batch_size) 628 { 629 if (idxd_type == IDXD_TYPE_IAX) 630 idxd->max_batch_size = 0; 631 else 632 idxd->max_batch_size = max_batch_size; 633 } 634 635 static inline void idxd_wq_set_max_batch_size(int idxd_type, struct idxd_wq *wq, 636 u32 max_batch_size) 637 { 638 if (idxd_type == IDXD_TYPE_IAX) 639 wq->max_batch_size = 0; 640 else 641 wq->max_batch_size = max_batch_size; 642 } 643 644 static inline void idxd_wqcfg_set_max_batch_shift(int idxd_type, union wqcfg *wqcfg, 645 u32 max_batch_shift) 646 { 647 if (idxd_type == IDXD_TYPE_IAX) 648 wqcfg->max_batch_shift = 0; 649 else 650 wqcfg->max_batch_shift = max_batch_shift; 651 } 652 653 static inline int idxd_wq_driver_name_match(struct idxd_wq *wq, struct device *dev) 654 { 655 return (strncmp(wq->driver_name, dev->driver->name, strlen(dev->driver->name)) == 0); 656 } 657 658 int __must_check __idxd_driver_register(struct idxd_device_driver *idxd_drv, 659 struct module *module, const char *mod_name); 660 #define idxd_driver_register(driver) \ 661 __idxd_driver_register(driver, THIS_MODULE, KBUILD_MODNAME) 662 663 void idxd_driver_unregister(struct idxd_device_driver *idxd_drv); 664 665 #define module_idxd_driver(__idxd_driver) \ 666 module_driver(__idxd_driver, idxd_driver_register, idxd_driver_unregister) 667 668 int idxd_register_bus_type(void); 669 void idxd_unregister_bus_type(void); 670 int idxd_register_devices(struct idxd_device *idxd); 671 void idxd_unregister_devices(struct idxd_device *idxd); 672 void idxd_wqs_quiesce(struct idxd_device *idxd); 673 bool idxd_queue_int_handle_resubmit(struct idxd_desc *desc); 674 void multi_u64_to_bmap(unsigned long *bmap, u64 *val, int count); 675 676 /* device interrupt control */ 677 irqreturn_t idxd_misc_thread(int vec, void *data); 678 irqreturn_t idxd_wq_thread(int irq, void *data); 679 void idxd_mask_error_interrupts(struct idxd_device *idxd); 680 void idxd_unmask_error_interrupts(struct idxd_device *idxd); 681 682 /* device control */ 683 int idxd_device_drv_probe(struct idxd_dev *idxd_dev); 684 void idxd_device_drv_remove(struct idxd_dev *idxd_dev); 685 int drv_enable_wq(struct idxd_wq *wq); 686 void drv_disable_wq(struct idxd_wq *wq); 687 int idxd_device_init_reset(struct idxd_device *idxd); 688 int idxd_device_enable(struct idxd_device *idxd); 689 int idxd_device_disable(struct idxd_device *idxd); 690 void idxd_device_reset(struct idxd_device *idxd); 691 void idxd_device_clear_state(struct idxd_device *idxd); 692 int idxd_device_config(struct idxd_device *idxd); 693 void idxd_device_drain_pasid(struct idxd_device *idxd, int pasid); 694 int idxd_device_load_config(struct idxd_device *idxd); 695 int idxd_device_request_int_handle(struct idxd_device *idxd, int idx, int *handle, 696 enum idxd_interrupt_type irq_type); 697 int idxd_device_release_int_handle(struct idxd_device *idxd, int handle, 698 enum idxd_interrupt_type irq_type); 699 700 /* work queue control */ 701 void idxd_wqs_unmap_portal(struct idxd_device *idxd); 702 int idxd_wq_alloc_resources(struct idxd_wq *wq); 703 void idxd_wq_free_resources(struct idxd_wq *wq); 704 int idxd_wq_enable(struct idxd_wq *wq); 705 int idxd_wq_disable(struct idxd_wq *wq, bool reset_config); 706 void idxd_wq_drain(struct idxd_wq *wq); 707 void idxd_wq_reset(struct idxd_wq *wq); 708 int idxd_wq_map_portal(struct idxd_wq *wq); 709 void idxd_wq_unmap_portal(struct idxd_wq *wq); 710 int idxd_wq_set_pasid(struct idxd_wq *wq, int pasid); 711 int idxd_wq_disable_pasid(struct idxd_wq *wq); 712 void __idxd_wq_quiesce(struct idxd_wq *wq); 713 void idxd_wq_quiesce(struct idxd_wq *wq); 714 int idxd_wq_init_percpu_ref(struct idxd_wq *wq); 715 void idxd_wq_free_irq(struct idxd_wq *wq); 716 int idxd_wq_request_irq(struct idxd_wq *wq); 717 718 /* submission */ 719 int idxd_submit_desc(struct idxd_wq *wq, struct idxd_desc *desc); 720 struct idxd_desc *idxd_alloc_desc(struct idxd_wq *wq, enum idxd_op_type optype); 721 void idxd_free_desc(struct idxd_wq *wq, struct idxd_desc *desc); 722 int idxd_enqcmds(struct idxd_wq *wq, void __iomem *portal, const void *desc); 723 724 /* dmaengine */ 725 int idxd_register_dma_device(struct idxd_device *idxd); 726 void idxd_unregister_dma_device(struct idxd_device *idxd); 727 void idxd_dma_complete_txd(struct idxd_desc *desc, 728 enum idxd_complete_type comp_type, bool free_desc); 729 730 /* cdev */ 731 int idxd_cdev_register(void); 732 void idxd_cdev_remove(void); 733 int idxd_cdev_get_major(struct idxd_device *idxd); 734 int idxd_wq_add_cdev(struct idxd_wq *wq); 735 void idxd_wq_del_cdev(struct idxd_wq *wq); 736 int idxd_copy_cr(struct idxd_wq *wq, ioasid_t pasid, unsigned long addr, 737 void *buf, int len); 738 void idxd_user_counter_increment(struct idxd_wq *wq, u32 pasid, int index); 739 740 /* perfmon */ 741 #if IS_ENABLED(CONFIG_INTEL_IDXD_PERFMON) 742 int perfmon_pmu_init(struct idxd_device *idxd); 743 void perfmon_pmu_remove(struct idxd_device *idxd); 744 void perfmon_counter_overflow(struct idxd_device *idxd); 745 void perfmon_init(void); 746 void perfmon_exit(void); 747 #else 748 static inline int perfmon_pmu_init(struct idxd_device *idxd) { return 0; } 749 static inline void perfmon_pmu_remove(struct idxd_device *idxd) {} 750 static inline void perfmon_counter_overflow(struct idxd_device *idxd) {} 751 static inline void perfmon_init(void) {} 752 static inline void perfmon_exit(void) {} 753 #endif 754 755 /* debugfs */ 756 int idxd_device_init_debugfs(struct idxd_device *idxd); 757 void idxd_device_remove_debugfs(struct idxd_device *idxd); 758 int idxd_init_debugfs(void); 759 void idxd_remove_debugfs(void); 760 761 #endif 762