1 /* SPDX-License-Identifier: GPL-2.0 */ 2 /* Copyright(c) 2019 Intel Corporation. All rights rsvd. */ 3 #ifndef _IDXD_H_ 4 #define _IDXD_H_ 5 6 #include <linux/sbitmap.h> 7 #include <linux/dmaengine.h> 8 #include <linux/percpu-rwsem.h> 9 #include <linux/wait.h> 10 #include <linux/cdev.h> 11 #include <linux/idr.h> 12 #include <linux/pci.h> 13 #include <linux/perf_event.h> 14 #include "registers.h" 15 16 #define IDXD_DRIVER_VERSION "1.00" 17 18 extern struct kmem_cache *idxd_desc_pool; 19 20 struct idxd_wq; 21 struct idxd_dev; 22 23 enum idxd_dev_type { 24 IDXD_DEV_NONE = -1, 25 IDXD_DEV_DSA = 0, 26 IDXD_DEV_IAX, 27 IDXD_DEV_WQ, 28 IDXD_DEV_GROUP, 29 IDXD_DEV_ENGINE, 30 IDXD_DEV_CDEV, 31 IDXD_DEV_MAX_TYPE, 32 }; 33 34 struct idxd_dev { 35 struct device conf_dev; 36 enum idxd_dev_type type; 37 }; 38 39 #define IDXD_REG_TIMEOUT 50 40 #define IDXD_DRAIN_TIMEOUT 5000 41 42 enum idxd_type { 43 IDXD_TYPE_UNKNOWN = -1, 44 IDXD_TYPE_DSA = 0, 45 IDXD_TYPE_IAX, 46 IDXD_TYPE_MAX, 47 }; 48 49 #define IDXD_NAME_SIZE 128 50 #define IDXD_PMU_EVENT_MAX 64 51 52 struct idxd_device_driver { 53 const char *name; 54 enum idxd_dev_type *type; 55 int (*probe)(struct idxd_dev *idxd_dev); 56 void (*remove)(struct idxd_dev *idxd_dev); 57 struct device_driver drv; 58 }; 59 60 extern struct idxd_device_driver dsa_drv; 61 extern struct idxd_device_driver idxd_drv; 62 extern struct idxd_device_driver idxd_dmaengine_drv; 63 extern struct idxd_device_driver idxd_user_drv; 64 65 struct idxd_irq_entry { 66 struct idxd_device *idxd; 67 int id; 68 int vector; 69 struct llist_head pending_llist; 70 struct list_head work_list; 71 /* 72 * Lock to protect access between irq thread process descriptor 73 * and irq thread processing error descriptor. 74 */ 75 spinlock_t list_lock; 76 }; 77 78 struct idxd_group { 79 struct idxd_dev idxd_dev; 80 struct idxd_device *idxd; 81 struct grpcfg grpcfg; 82 int id; 83 int num_engines; 84 int num_wqs; 85 bool use_token_limit; 86 u8 tokens_allowed; 87 u8 tokens_reserved; 88 int tc_a; 89 int tc_b; 90 }; 91 92 struct idxd_pmu { 93 struct idxd_device *idxd; 94 95 struct perf_event *event_list[IDXD_PMU_EVENT_MAX]; 96 int n_events; 97 98 DECLARE_BITMAP(used_mask, IDXD_PMU_EVENT_MAX); 99 100 struct pmu pmu; 101 char name[IDXD_NAME_SIZE]; 102 int cpu; 103 104 int n_counters; 105 int counter_width; 106 int n_event_categories; 107 108 bool per_counter_caps_supported; 109 unsigned long supported_event_categories; 110 111 unsigned long supported_filters; 112 int n_filters; 113 114 struct hlist_node cpuhp_node; 115 }; 116 117 #define IDXD_MAX_PRIORITY 0xf 118 119 enum idxd_wq_state { 120 IDXD_WQ_DISABLED = 0, 121 IDXD_WQ_ENABLED, 122 }; 123 124 enum idxd_wq_flag { 125 WQ_FLAG_DEDICATED = 0, 126 WQ_FLAG_BLOCK_ON_FAULT, 127 }; 128 129 enum idxd_wq_type { 130 IDXD_WQT_NONE = 0, 131 IDXD_WQT_KERNEL, 132 IDXD_WQT_USER, 133 }; 134 135 struct idxd_cdev { 136 struct idxd_wq *wq; 137 struct cdev cdev; 138 struct idxd_dev idxd_dev; 139 int minor; 140 }; 141 142 #define IDXD_ALLOCATED_BATCH_SIZE 128U 143 #define WQ_NAME_SIZE 1024 144 #define WQ_TYPE_SIZE 10 145 146 enum idxd_op_type { 147 IDXD_OP_BLOCK = 0, 148 IDXD_OP_NONBLOCK = 1, 149 }; 150 151 enum idxd_complete_type { 152 IDXD_COMPLETE_NORMAL = 0, 153 IDXD_COMPLETE_ABORT, 154 IDXD_COMPLETE_DEV_FAIL, 155 }; 156 157 struct idxd_dma_chan { 158 struct dma_chan chan; 159 struct idxd_wq *wq; 160 }; 161 162 struct idxd_wq { 163 void __iomem *portal; 164 struct percpu_ref wq_active; 165 struct completion wq_dead; 166 struct idxd_dev idxd_dev; 167 struct idxd_cdev *idxd_cdev; 168 struct wait_queue_head err_queue; 169 struct idxd_device *idxd; 170 int id; 171 enum idxd_wq_type type; 172 struct idxd_group *group; 173 int client_count; 174 struct mutex wq_lock; /* mutex for workqueue */ 175 u32 size; 176 u32 threshold; 177 u32 priority; 178 enum idxd_wq_state state; 179 unsigned long flags; 180 union wqcfg *wqcfg; 181 struct dsa_hw_desc **hw_descs; 182 int num_descs; 183 union { 184 struct dsa_completion_record *compls; 185 struct iax_completion_record *iax_compls; 186 }; 187 void *compls_raw; 188 dma_addr_t compls_addr; 189 dma_addr_t compls_addr_raw; 190 int compls_size; 191 struct idxd_desc **descs; 192 struct sbitmap_queue sbq; 193 struct idxd_dma_chan *idxd_chan; 194 char name[WQ_NAME_SIZE + 1]; 195 u64 max_xfer_bytes; 196 u32 max_batch_size; 197 bool ats_dis; 198 }; 199 200 struct idxd_engine { 201 struct idxd_dev idxd_dev; 202 int id; 203 struct idxd_group *group; 204 struct idxd_device *idxd; 205 }; 206 207 /* shadow registers */ 208 struct idxd_hw { 209 u32 version; 210 union gen_cap_reg gen_cap; 211 union wq_cap_reg wq_cap; 212 union group_cap_reg group_cap; 213 union engine_cap_reg engine_cap; 214 struct opcap opcap; 215 u32 cmd_cap; 216 }; 217 218 enum idxd_device_state { 219 IDXD_DEV_HALTED = -1, 220 IDXD_DEV_DISABLED = 0, 221 IDXD_DEV_ENABLED, 222 }; 223 224 enum idxd_device_flag { 225 IDXD_FLAG_CONFIGURABLE = 0, 226 IDXD_FLAG_CMD_RUNNING, 227 IDXD_FLAG_PASID_ENABLED, 228 }; 229 230 struct idxd_dma_dev { 231 struct idxd_device *idxd; 232 struct dma_device dma; 233 }; 234 235 struct idxd_driver_data { 236 const char *name_prefix; 237 enum idxd_type type; 238 struct device_type *dev_type; 239 int compl_size; 240 int align; 241 }; 242 243 struct idxd_device { 244 struct idxd_dev idxd_dev; 245 struct idxd_driver_data *data; 246 struct list_head list; 247 struct idxd_hw hw; 248 enum idxd_device_state state; 249 unsigned long flags; 250 int id; 251 int major; 252 u8 cmd_status; 253 254 struct pci_dev *pdev; 255 void __iomem *reg_base; 256 257 spinlock_t dev_lock; /* spinlock for device */ 258 spinlock_t cmd_lock; /* spinlock for device commands */ 259 struct completion *cmd_done; 260 struct idxd_group **groups; 261 struct idxd_wq **wqs; 262 struct idxd_engine **engines; 263 264 struct iommu_sva *sva; 265 unsigned int pasid; 266 267 int num_groups; 268 269 u32 msix_perm_offset; 270 u32 wqcfg_offset; 271 u32 grpcfg_offset; 272 u32 perfmon_offset; 273 274 u64 max_xfer_bytes; 275 u32 max_batch_size; 276 int max_groups; 277 int max_engines; 278 int max_tokens; 279 int max_wqs; 280 int max_wq_size; 281 int token_limit; 282 int nr_tokens; /* non-reserved tokens */ 283 unsigned int wqcfg_size; 284 285 union sw_err_reg sw_err; 286 wait_queue_head_t cmd_waitq; 287 int num_wq_irqs; 288 struct idxd_irq_entry *irq_entries; 289 290 struct idxd_dma_dev *idxd_dma; 291 struct workqueue_struct *wq; 292 struct work_struct work; 293 294 int *int_handles; 295 296 struct idxd_pmu *idxd_pmu; 297 }; 298 299 /* IDXD software descriptor */ 300 struct idxd_desc { 301 union { 302 struct dsa_hw_desc *hw; 303 struct iax_hw_desc *iax_hw; 304 }; 305 dma_addr_t desc_dma; 306 union { 307 struct dsa_completion_record *completion; 308 struct iax_completion_record *iax_completion; 309 }; 310 dma_addr_t compl_dma; 311 struct dma_async_tx_descriptor txd; 312 struct llist_node llnode; 313 struct list_head list; 314 int id; 315 int cpu; 316 struct idxd_wq *wq; 317 }; 318 319 /* 320 * This is software defined error for the completion status. We overload the error code 321 * that will never appear in completion status and only SWERR register. 322 */ 323 enum idxd_completion_status { 324 IDXD_COMP_DESC_ABORT = 0xff, 325 }; 326 327 #define idxd_confdev(idxd) &idxd->idxd_dev.conf_dev 328 #define wq_confdev(wq) &wq->idxd_dev.conf_dev 329 #define engine_confdev(engine) &engine->idxd_dev.conf_dev 330 #define group_confdev(group) &group->idxd_dev.conf_dev 331 #define cdev_dev(cdev) &cdev->idxd_dev.conf_dev 332 333 #define confdev_to_idxd_dev(dev) container_of(dev, struct idxd_dev, conf_dev) 334 #define idxd_dev_to_idxd(idxd_dev) container_of(idxd_dev, struct idxd_device, idxd_dev) 335 #define idxd_dev_to_wq(idxd_dev) container_of(idxd_dev, struct idxd_wq, idxd_dev) 336 337 static inline struct idxd_device *confdev_to_idxd(struct device *dev) 338 { 339 struct idxd_dev *idxd_dev = confdev_to_idxd_dev(dev); 340 341 return idxd_dev_to_idxd(idxd_dev); 342 } 343 344 static inline struct idxd_wq *confdev_to_wq(struct device *dev) 345 { 346 struct idxd_dev *idxd_dev = confdev_to_idxd_dev(dev); 347 348 return idxd_dev_to_wq(idxd_dev); 349 } 350 351 static inline struct idxd_engine *confdev_to_engine(struct device *dev) 352 { 353 struct idxd_dev *idxd_dev = confdev_to_idxd_dev(dev); 354 355 return container_of(idxd_dev, struct idxd_engine, idxd_dev); 356 } 357 358 static inline struct idxd_group *confdev_to_group(struct device *dev) 359 { 360 struct idxd_dev *idxd_dev = confdev_to_idxd_dev(dev); 361 362 return container_of(idxd_dev, struct idxd_group, idxd_dev); 363 } 364 365 static inline struct idxd_cdev *dev_to_cdev(struct device *dev) 366 { 367 struct idxd_dev *idxd_dev = confdev_to_idxd_dev(dev); 368 369 return container_of(idxd_dev, struct idxd_cdev, idxd_dev); 370 } 371 372 static inline void idxd_dev_set_type(struct idxd_dev *idev, int type) 373 { 374 if (type >= IDXD_DEV_MAX_TYPE) { 375 idev->type = IDXD_DEV_NONE; 376 return; 377 } 378 379 idev->type = type; 380 } 381 382 extern struct bus_type dsa_bus_type; 383 384 extern bool support_enqcmd; 385 extern struct ida idxd_ida; 386 extern struct device_type dsa_device_type; 387 extern struct device_type iax_device_type; 388 extern struct device_type idxd_wq_device_type; 389 extern struct device_type idxd_engine_device_type; 390 extern struct device_type idxd_group_device_type; 391 392 static inline bool is_dsa_dev(struct idxd_dev *idxd_dev) 393 { 394 return idxd_dev->type == IDXD_DEV_DSA; 395 } 396 397 static inline bool is_iax_dev(struct idxd_dev *idxd_dev) 398 { 399 return idxd_dev->type == IDXD_DEV_IAX; 400 } 401 402 static inline bool is_idxd_dev(struct idxd_dev *idxd_dev) 403 { 404 return is_dsa_dev(idxd_dev) || is_iax_dev(idxd_dev); 405 } 406 407 static inline bool is_idxd_wq_dev(struct idxd_dev *idxd_dev) 408 { 409 return idxd_dev->type == IDXD_DEV_WQ; 410 } 411 412 static inline bool is_idxd_wq_dmaengine(struct idxd_wq *wq) 413 { 414 if (wq->type == IDXD_WQT_KERNEL && strcmp(wq->name, "dmaengine") == 0) 415 return true; 416 return false; 417 } 418 419 static inline bool is_idxd_wq_user(struct idxd_wq *wq) 420 { 421 return wq->type == IDXD_WQT_USER; 422 } 423 424 static inline bool is_idxd_wq_kernel(struct idxd_wq *wq) 425 { 426 return wq->type == IDXD_WQT_KERNEL; 427 } 428 429 static inline bool wq_dedicated(struct idxd_wq *wq) 430 { 431 return test_bit(WQ_FLAG_DEDICATED, &wq->flags); 432 } 433 434 static inline bool wq_shared(struct idxd_wq *wq) 435 { 436 return !test_bit(WQ_FLAG_DEDICATED, &wq->flags); 437 } 438 439 static inline bool device_pasid_enabled(struct idxd_device *idxd) 440 { 441 return test_bit(IDXD_FLAG_PASID_ENABLED, &idxd->flags); 442 } 443 444 static inline bool device_swq_supported(struct idxd_device *idxd) 445 { 446 return (support_enqcmd && device_pasid_enabled(idxd)); 447 } 448 449 enum idxd_portal_prot { 450 IDXD_PORTAL_UNLIMITED = 0, 451 IDXD_PORTAL_LIMITED, 452 }; 453 454 enum idxd_interrupt_type { 455 IDXD_IRQ_MSIX = 0, 456 IDXD_IRQ_IMS, 457 }; 458 459 static inline int idxd_get_wq_portal_offset(enum idxd_portal_prot prot) 460 { 461 return prot * 0x1000; 462 } 463 464 static inline int idxd_get_wq_portal_full_offset(int wq_id, 465 enum idxd_portal_prot prot) 466 { 467 return ((wq_id * 4) << PAGE_SHIFT) + idxd_get_wq_portal_offset(prot); 468 } 469 470 static inline void idxd_wq_get(struct idxd_wq *wq) 471 { 472 wq->client_count++; 473 } 474 475 static inline void idxd_wq_put(struct idxd_wq *wq) 476 { 477 wq->client_count--; 478 } 479 480 static inline int idxd_wq_refcount(struct idxd_wq *wq) 481 { 482 return wq->client_count; 483 }; 484 485 int __must_check __idxd_driver_register(struct idxd_device_driver *idxd_drv, 486 struct module *module, const char *mod_name); 487 #define idxd_driver_register(driver) \ 488 __idxd_driver_register(driver, THIS_MODULE, KBUILD_MODNAME) 489 490 void idxd_driver_unregister(struct idxd_device_driver *idxd_drv); 491 492 #define module_idxd_driver(__idxd_driver) \ 493 module_driver(__idxd_driver, idxd_driver_register, idxd_driver_unregister) 494 495 int idxd_register_bus_type(void); 496 void idxd_unregister_bus_type(void); 497 int idxd_register_devices(struct idxd_device *idxd); 498 void idxd_unregister_devices(struct idxd_device *idxd); 499 int idxd_register_driver(void); 500 void idxd_unregister_driver(void); 501 void idxd_wqs_quiesce(struct idxd_device *idxd); 502 503 /* device interrupt control */ 504 void idxd_msix_perm_setup(struct idxd_device *idxd); 505 void idxd_msix_perm_clear(struct idxd_device *idxd); 506 irqreturn_t idxd_misc_thread(int vec, void *data); 507 irqreturn_t idxd_wq_thread(int irq, void *data); 508 void idxd_mask_error_interrupts(struct idxd_device *idxd); 509 void idxd_unmask_error_interrupts(struct idxd_device *idxd); 510 void idxd_mask_msix_vectors(struct idxd_device *idxd); 511 void idxd_mask_msix_vector(struct idxd_device *idxd, int vec_id); 512 void idxd_unmask_msix_vector(struct idxd_device *idxd, int vec_id); 513 514 /* device control */ 515 int idxd_register_idxd_drv(void); 516 void idxd_unregister_idxd_drv(void); 517 int idxd_device_drv_probe(struct idxd_dev *idxd_dev); 518 void idxd_device_drv_remove(struct idxd_dev *idxd_dev); 519 int drv_enable_wq(struct idxd_wq *wq); 520 int __drv_enable_wq(struct idxd_wq *wq); 521 void drv_disable_wq(struct idxd_wq *wq); 522 void __drv_disable_wq(struct idxd_wq *wq); 523 int idxd_device_init_reset(struct idxd_device *idxd); 524 int idxd_device_enable(struct idxd_device *idxd); 525 int idxd_device_disable(struct idxd_device *idxd); 526 void idxd_device_reset(struct idxd_device *idxd); 527 void idxd_device_clear_state(struct idxd_device *idxd); 528 int idxd_device_config(struct idxd_device *idxd); 529 void idxd_device_drain_pasid(struct idxd_device *idxd, int pasid); 530 int idxd_device_load_config(struct idxd_device *idxd); 531 int idxd_device_request_int_handle(struct idxd_device *idxd, int idx, int *handle, 532 enum idxd_interrupt_type irq_type); 533 int idxd_device_release_int_handle(struct idxd_device *idxd, int handle, 534 enum idxd_interrupt_type irq_type); 535 536 /* work queue control */ 537 void idxd_wqs_unmap_portal(struct idxd_device *idxd); 538 int idxd_wq_alloc_resources(struct idxd_wq *wq); 539 void idxd_wq_free_resources(struct idxd_wq *wq); 540 int idxd_wq_enable(struct idxd_wq *wq); 541 int idxd_wq_disable(struct idxd_wq *wq, bool reset_config); 542 void idxd_wq_drain(struct idxd_wq *wq); 543 void idxd_wq_reset(struct idxd_wq *wq); 544 int idxd_wq_map_portal(struct idxd_wq *wq); 545 void idxd_wq_unmap_portal(struct idxd_wq *wq); 546 int idxd_wq_set_pasid(struct idxd_wq *wq, int pasid); 547 int idxd_wq_disable_pasid(struct idxd_wq *wq); 548 void idxd_wq_quiesce(struct idxd_wq *wq); 549 int idxd_wq_init_percpu_ref(struct idxd_wq *wq); 550 551 /* submission */ 552 int idxd_submit_desc(struct idxd_wq *wq, struct idxd_desc *desc); 553 struct idxd_desc *idxd_alloc_desc(struct idxd_wq *wq, enum idxd_op_type optype); 554 void idxd_free_desc(struct idxd_wq *wq, struct idxd_desc *desc); 555 556 /* dmaengine */ 557 int idxd_register_dma_device(struct idxd_device *idxd); 558 void idxd_unregister_dma_device(struct idxd_device *idxd); 559 int idxd_register_dma_channel(struct idxd_wq *wq); 560 void idxd_unregister_dma_channel(struct idxd_wq *wq); 561 void idxd_parse_completion_status(u8 status, enum dmaengine_tx_result *res); 562 void idxd_dma_complete_txd(struct idxd_desc *desc, 563 enum idxd_complete_type comp_type); 564 565 /* cdev */ 566 int idxd_cdev_register(void); 567 void idxd_cdev_remove(void); 568 int idxd_cdev_get_major(struct idxd_device *idxd); 569 int idxd_wq_add_cdev(struct idxd_wq *wq); 570 void idxd_wq_del_cdev(struct idxd_wq *wq); 571 572 /* perfmon */ 573 #if IS_ENABLED(CONFIG_INTEL_IDXD_PERFMON) 574 int perfmon_pmu_init(struct idxd_device *idxd); 575 void perfmon_pmu_remove(struct idxd_device *idxd); 576 void perfmon_counter_overflow(struct idxd_device *idxd); 577 void perfmon_init(void); 578 void perfmon_exit(void); 579 #else 580 static inline int perfmon_pmu_init(struct idxd_device *idxd) { return 0; } 581 static inline void perfmon_pmu_remove(struct idxd_device *idxd) {} 582 static inline void perfmon_counter_overflow(struct idxd_device *idxd) {} 583 static inline void perfmon_init(void) {} 584 static inline void perfmon_exit(void) {} 585 #endif 586 587 static inline void complete_desc(struct idxd_desc *desc, enum idxd_complete_type reason) 588 { 589 idxd_dma_complete_txd(desc, reason); 590 idxd_free_desc(desc->wq, desc); 591 } 592 593 #endif 594