1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright(c) 2019 Intel Corporation. All rights rsvd. */ 3 #include <linux/init.h> 4 #include <linux/kernel.h> 5 #include <linux/module.h> 6 #include <linux/slab.h> 7 #include <linux/pci.h> 8 #include <linux/interrupt.h> 9 #include <linux/delay.h> 10 #include <linux/dma-mapping.h> 11 #include <linux/workqueue.h> 12 #include <linux/aer.h> 13 #include <linux/fs.h> 14 #include <linux/io-64-nonatomic-lo-hi.h> 15 #include <linux/device.h> 16 #include <linux/idr.h> 17 #include <linux/intel-svm.h> 18 #include <linux/iommu.h> 19 #include <uapi/linux/idxd.h> 20 #include <linux/dmaengine.h> 21 #include "../dmaengine.h" 22 #include "registers.h" 23 #include "idxd.h" 24 #include "perfmon.h" 25 26 MODULE_VERSION(IDXD_DRIVER_VERSION); 27 MODULE_LICENSE("GPL v2"); 28 MODULE_AUTHOR("Intel Corporation"); 29 MODULE_IMPORT_NS(IDXD); 30 31 static bool sva = true; 32 module_param(sva, bool, 0644); 33 MODULE_PARM_DESC(sva, "Toggle SVA support on/off"); 34 35 #define DRV_NAME "idxd" 36 37 bool support_enqcmd; 38 DEFINE_IDA(idxd_ida); 39 40 static struct idxd_driver_data idxd_driver_data[] = { 41 [IDXD_TYPE_DSA] = { 42 .name_prefix = "dsa", 43 .type = IDXD_TYPE_DSA, 44 .compl_size = sizeof(struct dsa_completion_record), 45 .align = 32, 46 .dev_type = &dsa_device_type, 47 }, 48 [IDXD_TYPE_IAX] = { 49 .name_prefix = "iax", 50 .type = IDXD_TYPE_IAX, 51 .compl_size = sizeof(struct iax_completion_record), 52 .align = 64, 53 .dev_type = &iax_device_type, 54 }, 55 }; 56 57 static struct pci_device_id idxd_pci_tbl[] = { 58 /* DSA ver 1.0 platforms */ 59 { PCI_DEVICE_DATA(INTEL, DSA_SPR0, &idxd_driver_data[IDXD_TYPE_DSA]) }, 60 61 /* IAX ver 1.0 platforms */ 62 { PCI_DEVICE_DATA(INTEL, IAX_SPR0, &idxd_driver_data[IDXD_TYPE_IAX]) }, 63 { 0, } 64 }; 65 MODULE_DEVICE_TABLE(pci, idxd_pci_tbl); 66 67 static int idxd_setup_interrupts(struct idxd_device *idxd) 68 { 69 struct pci_dev *pdev = idxd->pdev; 70 struct device *dev = &pdev->dev; 71 struct idxd_irq_entry *irq_entry; 72 int i, msixcnt; 73 int rc = 0; 74 75 msixcnt = pci_msix_vec_count(pdev); 76 if (msixcnt < 0) { 77 dev_err(dev, "Not MSI-X interrupt capable.\n"); 78 return -ENOSPC; 79 } 80 81 rc = pci_alloc_irq_vectors(pdev, msixcnt, msixcnt, PCI_IRQ_MSIX); 82 if (rc != msixcnt) { 83 dev_err(dev, "Failed enabling %d MSIX entries: %d\n", msixcnt, rc); 84 return -ENOSPC; 85 } 86 dev_dbg(dev, "Enabled %d msix vectors\n", msixcnt); 87 88 /* 89 * We implement 1 completion list per MSI-X entry except for 90 * entry 0, which is for errors and others. 91 */ 92 idxd->irq_entries = kcalloc_node(msixcnt, sizeof(struct idxd_irq_entry), 93 GFP_KERNEL, dev_to_node(dev)); 94 if (!idxd->irq_entries) { 95 rc = -ENOMEM; 96 goto err_irq_entries; 97 } 98 99 for (i = 0; i < msixcnt; i++) { 100 idxd->irq_entries[i].id = i; 101 idxd->irq_entries[i].idxd = idxd; 102 idxd->irq_entries[i].vector = pci_irq_vector(pdev, i); 103 spin_lock_init(&idxd->irq_entries[i].list_lock); 104 } 105 106 idxd_msix_perm_setup(idxd); 107 108 irq_entry = &idxd->irq_entries[0]; 109 rc = request_threaded_irq(irq_entry->vector, NULL, idxd_misc_thread, 110 0, "idxd-misc", irq_entry); 111 if (rc < 0) { 112 dev_err(dev, "Failed to allocate misc interrupt.\n"); 113 goto err_misc_irq; 114 } 115 116 dev_dbg(dev, "Allocated idxd-misc handler on msix vector %d\n", irq_entry->vector); 117 118 /* first MSI-X entry is not for wq interrupts */ 119 idxd->num_wq_irqs = msixcnt - 1; 120 121 for (i = 1; i < msixcnt; i++) { 122 irq_entry = &idxd->irq_entries[i]; 123 124 init_llist_head(&idxd->irq_entries[i].pending_llist); 125 INIT_LIST_HEAD(&idxd->irq_entries[i].work_list); 126 rc = request_threaded_irq(irq_entry->vector, NULL, 127 idxd_wq_thread, 0, "idxd-portal", irq_entry); 128 if (rc < 0) { 129 dev_err(dev, "Failed to allocate irq %d.\n", irq_entry->vector); 130 goto err_wq_irqs; 131 } 132 133 dev_dbg(dev, "Allocated idxd-msix %d for vector %d\n", i, irq_entry->vector); 134 if (idxd->hw.cmd_cap & BIT(IDXD_CMD_REQUEST_INT_HANDLE)) { 135 /* 136 * The MSIX vector enumeration starts at 1 with vector 0 being the 137 * misc interrupt that handles non I/O completion events. The 138 * interrupt handles are for IMS enumeration on guest. The misc 139 * interrupt vector does not require a handle and therefore we start 140 * the int_handles at index 0. Since 'i' starts at 1, the first 141 * int_handles index will be 0. 142 */ 143 rc = idxd_device_request_int_handle(idxd, i, &idxd->int_handles[i - 1], 144 IDXD_IRQ_MSIX); 145 if (rc < 0) { 146 free_irq(irq_entry->vector, irq_entry); 147 goto err_wq_irqs; 148 } 149 dev_dbg(dev, "int handle requested: %u\n", idxd->int_handles[i - 1]); 150 } 151 } 152 153 idxd_unmask_error_interrupts(idxd); 154 return 0; 155 156 err_wq_irqs: 157 while (--i >= 0) { 158 irq_entry = &idxd->irq_entries[i]; 159 free_irq(irq_entry->vector, irq_entry); 160 if (i != 0) 161 idxd_device_release_int_handle(idxd, 162 idxd->int_handles[i], IDXD_IRQ_MSIX); 163 } 164 err_misc_irq: 165 /* Disable error interrupt generation */ 166 idxd_mask_error_interrupts(idxd); 167 idxd_msix_perm_clear(idxd); 168 err_irq_entries: 169 pci_free_irq_vectors(pdev); 170 dev_err(dev, "No usable interrupts\n"); 171 return rc; 172 } 173 174 static void idxd_cleanup_interrupts(struct idxd_device *idxd) 175 { 176 struct pci_dev *pdev = idxd->pdev; 177 struct idxd_irq_entry *irq_entry; 178 int i, msixcnt; 179 180 msixcnt = pci_msix_vec_count(pdev); 181 if (msixcnt <= 0) 182 return; 183 184 irq_entry = &idxd->irq_entries[0]; 185 free_irq(irq_entry->vector, irq_entry); 186 187 for (i = 1; i < msixcnt; i++) { 188 189 irq_entry = &idxd->irq_entries[i]; 190 if (idxd->hw.cmd_cap & BIT(IDXD_CMD_RELEASE_INT_HANDLE)) 191 idxd_device_release_int_handle(idxd, idxd->int_handles[i], 192 IDXD_IRQ_MSIX); 193 free_irq(irq_entry->vector, irq_entry); 194 } 195 196 idxd_mask_error_interrupts(idxd); 197 pci_free_irq_vectors(pdev); 198 } 199 200 static int idxd_setup_wqs(struct idxd_device *idxd) 201 { 202 struct device *dev = &idxd->pdev->dev; 203 struct idxd_wq *wq; 204 struct device *conf_dev; 205 int i, rc; 206 207 idxd->wqs = kcalloc_node(idxd->max_wqs, sizeof(struct idxd_wq *), 208 GFP_KERNEL, dev_to_node(dev)); 209 if (!idxd->wqs) 210 return -ENOMEM; 211 212 for (i = 0; i < idxd->max_wqs; i++) { 213 wq = kzalloc_node(sizeof(*wq), GFP_KERNEL, dev_to_node(dev)); 214 if (!wq) { 215 rc = -ENOMEM; 216 goto err; 217 } 218 219 idxd_dev_set_type(&wq->idxd_dev, IDXD_DEV_WQ); 220 conf_dev = wq_confdev(wq); 221 wq->id = i; 222 wq->idxd = idxd; 223 device_initialize(wq_confdev(wq)); 224 conf_dev->parent = idxd_confdev(idxd); 225 conf_dev->bus = &dsa_bus_type; 226 conf_dev->type = &idxd_wq_device_type; 227 rc = dev_set_name(conf_dev, "wq%d.%d", idxd->id, wq->id); 228 if (rc < 0) { 229 put_device(conf_dev); 230 goto err; 231 } 232 233 mutex_init(&wq->wq_lock); 234 init_waitqueue_head(&wq->err_queue); 235 init_completion(&wq->wq_dead); 236 wq->max_xfer_bytes = idxd->max_xfer_bytes; 237 wq->max_batch_size = idxd->max_batch_size; 238 wq->wqcfg = kzalloc_node(idxd->wqcfg_size, GFP_KERNEL, dev_to_node(dev)); 239 if (!wq->wqcfg) { 240 put_device(conf_dev); 241 rc = -ENOMEM; 242 goto err; 243 } 244 idxd->wqs[i] = wq; 245 } 246 247 return 0; 248 249 err: 250 while (--i >= 0) { 251 wq = idxd->wqs[i]; 252 conf_dev = wq_confdev(wq); 253 put_device(conf_dev); 254 } 255 return rc; 256 } 257 258 static int idxd_setup_engines(struct idxd_device *idxd) 259 { 260 struct idxd_engine *engine; 261 struct device *dev = &idxd->pdev->dev; 262 struct device *conf_dev; 263 int i, rc; 264 265 idxd->engines = kcalloc_node(idxd->max_engines, sizeof(struct idxd_engine *), 266 GFP_KERNEL, dev_to_node(dev)); 267 if (!idxd->engines) 268 return -ENOMEM; 269 270 for (i = 0; i < idxd->max_engines; i++) { 271 engine = kzalloc_node(sizeof(*engine), GFP_KERNEL, dev_to_node(dev)); 272 if (!engine) { 273 rc = -ENOMEM; 274 goto err; 275 } 276 277 idxd_dev_set_type(&engine->idxd_dev, IDXD_DEV_ENGINE); 278 conf_dev = engine_confdev(engine); 279 engine->id = i; 280 engine->idxd = idxd; 281 device_initialize(conf_dev); 282 conf_dev->parent = idxd_confdev(idxd); 283 conf_dev->bus = &dsa_bus_type; 284 conf_dev->type = &idxd_engine_device_type; 285 rc = dev_set_name(conf_dev, "engine%d.%d", idxd->id, engine->id); 286 if (rc < 0) { 287 put_device(conf_dev); 288 goto err; 289 } 290 291 idxd->engines[i] = engine; 292 } 293 294 return 0; 295 296 err: 297 while (--i >= 0) { 298 engine = idxd->engines[i]; 299 conf_dev = engine_confdev(engine); 300 put_device(conf_dev); 301 } 302 return rc; 303 } 304 305 static int idxd_setup_groups(struct idxd_device *idxd) 306 { 307 struct device *dev = &idxd->pdev->dev; 308 struct device *conf_dev; 309 struct idxd_group *group; 310 int i, rc; 311 312 idxd->groups = kcalloc_node(idxd->max_groups, sizeof(struct idxd_group *), 313 GFP_KERNEL, dev_to_node(dev)); 314 if (!idxd->groups) 315 return -ENOMEM; 316 317 for (i = 0; i < idxd->max_groups; i++) { 318 group = kzalloc_node(sizeof(*group), GFP_KERNEL, dev_to_node(dev)); 319 if (!group) { 320 rc = -ENOMEM; 321 goto err; 322 } 323 324 idxd_dev_set_type(&group->idxd_dev, IDXD_DEV_GROUP); 325 conf_dev = group_confdev(group); 326 group->id = i; 327 group->idxd = idxd; 328 device_initialize(conf_dev); 329 conf_dev->parent = idxd_confdev(idxd); 330 conf_dev->bus = &dsa_bus_type; 331 conf_dev->type = &idxd_group_device_type; 332 rc = dev_set_name(conf_dev, "group%d.%d", idxd->id, group->id); 333 if (rc < 0) { 334 put_device(conf_dev); 335 goto err; 336 } 337 338 idxd->groups[i] = group; 339 group->tc_a = -1; 340 group->tc_b = -1; 341 } 342 343 return 0; 344 345 err: 346 while (--i >= 0) { 347 group = idxd->groups[i]; 348 put_device(group_confdev(group)); 349 } 350 return rc; 351 } 352 353 static void idxd_cleanup_internals(struct idxd_device *idxd) 354 { 355 int i; 356 357 for (i = 0; i < idxd->max_groups; i++) 358 put_device(group_confdev(idxd->groups[i])); 359 for (i = 0; i < idxd->max_engines; i++) 360 put_device(engine_confdev(idxd->engines[i])); 361 for (i = 0; i < idxd->max_wqs; i++) 362 put_device(wq_confdev(idxd->wqs[i])); 363 destroy_workqueue(idxd->wq); 364 } 365 366 static int idxd_setup_internals(struct idxd_device *idxd) 367 { 368 struct device *dev = &idxd->pdev->dev; 369 int rc, i; 370 371 init_waitqueue_head(&idxd->cmd_waitq); 372 373 if (idxd->hw.cmd_cap & BIT(IDXD_CMD_REQUEST_INT_HANDLE)) { 374 idxd->int_handles = kcalloc_node(idxd->max_wqs, sizeof(int), GFP_KERNEL, 375 dev_to_node(dev)); 376 if (!idxd->int_handles) 377 return -ENOMEM; 378 } 379 380 rc = idxd_setup_wqs(idxd); 381 if (rc < 0) 382 goto err_wqs; 383 384 rc = idxd_setup_engines(idxd); 385 if (rc < 0) 386 goto err_engine; 387 388 rc = idxd_setup_groups(idxd); 389 if (rc < 0) 390 goto err_group; 391 392 idxd->wq = create_workqueue(dev_name(dev)); 393 if (!idxd->wq) { 394 rc = -ENOMEM; 395 goto err_wkq_create; 396 } 397 398 return 0; 399 400 err_wkq_create: 401 for (i = 0; i < idxd->max_groups; i++) 402 put_device(group_confdev(idxd->groups[i])); 403 err_group: 404 for (i = 0; i < idxd->max_engines; i++) 405 put_device(engine_confdev(idxd->engines[i])); 406 err_engine: 407 for (i = 0; i < idxd->max_wqs; i++) 408 put_device(wq_confdev(idxd->wqs[i])); 409 err_wqs: 410 kfree(idxd->int_handles); 411 return rc; 412 } 413 414 static void idxd_read_table_offsets(struct idxd_device *idxd) 415 { 416 union offsets_reg offsets; 417 struct device *dev = &idxd->pdev->dev; 418 419 offsets.bits[0] = ioread64(idxd->reg_base + IDXD_TABLE_OFFSET); 420 offsets.bits[1] = ioread64(idxd->reg_base + IDXD_TABLE_OFFSET + sizeof(u64)); 421 idxd->grpcfg_offset = offsets.grpcfg * IDXD_TABLE_MULT; 422 dev_dbg(dev, "IDXD Group Config Offset: %#x\n", idxd->grpcfg_offset); 423 idxd->wqcfg_offset = offsets.wqcfg * IDXD_TABLE_MULT; 424 dev_dbg(dev, "IDXD Work Queue Config Offset: %#x\n", idxd->wqcfg_offset); 425 idxd->msix_perm_offset = offsets.msix_perm * IDXD_TABLE_MULT; 426 dev_dbg(dev, "IDXD MSIX Permission Offset: %#x\n", idxd->msix_perm_offset); 427 idxd->perfmon_offset = offsets.perfmon * IDXD_TABLE_MULT; 428 dev_dbg(dev, "IDXD Perfmon Offset: %#x\n", idxd->perfmon_offset); 429 } 430 431 static void idxd_read_caps(struct idxd_device *idxd) 432 { 433 struct device *dev = &idxd->pdev->dev; 434 int i; 435 436 /* reading generic capabilities */ 437 idxd->hw.gen_cap.bits = ioread64(idxd->reg_base + IDXD_GENCAP_OFFSET); 438 dev_dbg(dev, "gen_cap: %#llx\n", idxd->hw.gen_cap.bits); 439 440 if (idxd->hw.gen_cap.cmd_cap) { 441 idxd->hw.cmd_cap = ioread32(idxd->reg_base + IDXD_CMDCAP_OFFSET); 442 dev_dbg(dev, "cmd_cap: %#x\n", idxd->hw.cmd_cap); 443 } 444 445 idxd->max_xfer_bytes = 1ULL << idxd->hw.gen_cap.max_xfer_shift; 446 dev_dbg(dev, "max xfer size: %llu bytes\n", idxd->max_xfer_bytes); 447 idxd->max_batch_size = 1U << idxd->hw.gen_cap.max_batch_shift; 448 dev_dbg(dev, "max batch size: %u\n", idxd->max_batch_size); 449 if (idxd->hw.gen_cap.config_en) 450 set_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags); 451 452 /* reading group capabilities */ 453 idxd->hw.group_cap.bits = 454 ioread64(idxd->reg_base + IDXD_GRPCAP_OFFSET); 455 dev_dbg(dev, "group_cap: %#llx\n", idxd->hw.group_cap.bits); 456 idxd->max_groups = idxd->hw.group_cap.num_groups; 457 dev_dbg(dev, "max groups: %u\n", idxd->max_groups); 458 idxd->max_tokens = idxd->hw.group_cap.total_tokens; 459 dev_dbg(dev, "max tokens: %u\n", idxd->max_tokens); 460 idxd->nr_tokens = idxd->max_tokens; 461 462 /* read engine capabilities */ 463 idxd->hw.engine_cap.bits = 464 ioread64(idxd->reg_base + IDXD_ENGCAP_OFFSET); 465 dev_dbg(dev, "engine_cap: %#llx\n", idxd->hw.engine_cap.bits); 466 idxd->max_engines = idxd->hw.engine_cap.num_engines; 467 dev_dbg(dev, "max engines: %u\n", idxd->max_engines); 468 469 /* read workqueue capabilities */ 470 idxd->hw.wq_cap.bits = ioread64(idxd->reg_base + IDXD_WQCAP_OFFSET); 471 dev_dbg(dev, "wq_cap: %#llx\n", idxd->hw.wq_cap.bits); 472 idxd->max_wq_size = idxd->hw.wq_cap.total_wq_size; 473 dev_dbg(dev, "total workqueue size: %u\n", idxd->max_wq_size); 474 idxd->max_wqs = idxd->hw.wq_cap.num_wqs; 475 dev_dbg(dev, "max workqueues: %u\n", idxd->max_wqs); 476 idxd->wqcfg_size = 1 << (idxd->hw.wq_cap.wqcfg_size + IDXD_WQCFG_MIN); 477 dev_dbg(dev, "wqcfg size: %u\n", idxd->wqcfg_size); 478 479 /* reading operation capabilities */ 480 for (i = 0; i < 4; i++) { 481 idxd->hw.opcap.bits[i] = ioread64(idxd->reg_base + 482 IDXD_OPCAP_OFFSET + i * sizeof(u64)); 483 dev_dbg(dev, "opcap[%d]: %#llx\n", i, idxd->hw.opcap.bits[i]); 484 } 485 } 486 487 static struct idxd_device *idxd_alloc(struct pci_dev *pdev, struct idxd_driver_data *data) 488 { 489 struct device *dev = &pdev->dev; 490 struct device *conf_dev; 491 struct idxd_device *idxd; 492 int rc; 493 494 idxd = kzalloc_node(sizeof(*idxd), GFP_KERNEL, dev_to_node(dev)); 495 if (!idxd) 496 return NULL; 497 498 conf_dev = idxd_confdev(idxd); 499 idxd->pdev = pdev; 500 idxd->data = data; 501 idxd_dev_set_type(&idxd->idxd_dev, idxd->data->type); 502 idxd->id = ida_alloc(&idxd_ida, GFP_KERNEL); 503 if (idxd->id < 0) 504 return NULL; 505 506 device_initialize(conf_dev); 507 conf_dev->parent = dev; 508 conf_dev->bus = &dsa_bus_type; 509 conf_dev->type = idxd->data->dev_type; 510 rc = dev_set_name(conf_dev, "%s%d", idxd->data->name_prefix, idxd->id); 511 if (rc < 0) { 512 put_device(conf_dev); 513 return NULL; 514 } 515 516 spin_lock_init(&idxd->dev_lock); 517 spin_lock_init(&idxd->cmd_lock); 518 519 return idxd; 520 } 521 522 static int idxd_enable_system_pasid(struct idxd_device *idxd) 523 { 524 int flags; 525 unsigned int pasid; 526 struct iommu_sva *sva; 527 528 flags = SVM_FLAG_SUPERVISOR_MODE; 529 530 sva = iommu_sva_bind_device(&idxd->pdev->dev, NULL, &flags); 531 if (IS_ERR(sva)) { 532 dev_warn(&idxd->pdev->dev, 533 "iommu sva bind failed: %ld\n", PTR_ERR(sva)); 534 return PTR_ERR(sva); 535 } 536 537 pasid = iommu_sva_get_pasid(sva); 538 if (pasid == IOMMU_PASID_INVALID) { 539 iommu_sva_unbind_device(sva); 540 return -ENODEV; 541 } 542 543 idxd->sva = sva; 544 idxd->pasid = pasid; 545 dev_dbg(&idxd->pdev->dev, "system pasid: %u\n", pasid); 546 return 0; 547 } 548 549 static void idxd_disable_system_pasid(struct idxd_device *idxd) 550 { 551 552 iommu_sva_unbind_device(idxd->sva); 553 idxd->sva = NULL; 554 } 555 556 static int idxd_probe(struct idxd_device *idxd) 557 { 558 struct pci_dev *pdev = idxd->pdev; 559 struct device *dev = &pdev->dev; 560 int rc; 561 562 dev_dbg(dev, "%s entered and resetting device\n", __func__); 563 rc = idxd_device_init_reset(idxd); 564 if (rc < 0) 565 return rc; 566 567 dev_dbg(dev, "IDXD reset complete\n"); 568 569 if (IS_ENABLED(CONFIG_INTEL_IDXD_SVM) && sva) { 570 rc = iommu_dev_enable_feature(dev, IOMMU_DEV_FEAT_SVA); 571 if (rc == 0) { 572 rc = idxd_enable_system_pasid(idxd); 573 if (rc < 0) { 574 iommu_dev_disable_feature(dev, IOMMU_DEV_FEAT_SVA); 575 dev_warn(dev, "Failed to enable PASID. No SVA support: %d\n", rc); 576 } else { 577 set_bit(IDXD_FLAG_PASID_ENABLED, &idxd->flags); 578 } 579 } else { 580 dev_warn(dev, "Unable to turn on SVA feature.\n"); 581 } 582 } else if (!sva) { 583 dev_warn(dev, "User forced SVA off via module param.\n"); 584 } 585 586 idxd_read_caps(idxd); 587 idxd_read_table_offsets(idxd); 588 589 rc = idxd_setup_internals(idxd); 590 if (rc) 591 goto err; 592 593 /* If the configs are readonly, then load them from device */ 594 if (!test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags)) { 595 dev_dbg(dev, "Loading RO device config\n"); 596 rc = idxd_device_load_config(idxd); 597 if (rc < 0) 598 goto err_config; 599 } 600 601 rc = idxd_setup_interrupts(idxd); 602 if (rc) 603 goto err_config; 604 605 dev_dbg(dev, "IDXD interrupt setup complete.\n"); 606 607 idxd->major = idxd_cdev_get_major(idxd); 608 609 rc = perfmon_pmu_init(idxd); 610 if (rc < 0) 611 dev_warn(dev, "Failed to initialize perfmon. No PMU support: %d\n", rc); 612 613 dev_dbg(dev, "IDXD device %d probed successfully\n", idxd->id); 614 return 0; 615 616 err_config: 617 idxd_cleanup_internals(idxd); 618 err: 619 if (device_pasid_enabled(idxd)) 620 idxd_disable_system_pasid(idxd); 621 iommu_dev_disable_feature(dev, IOMMU_DEV_FEAT_SVA); 622 return rc; 623 } 624 625 static void idxd_cleanup(struct idxd_device *idxd) 626 { 627 struct device *dev = &idxd->pdev->dev; 628 629 perfmon_pmu_remove(idxd); 630 idxd_cleanup_interrupts(idxd); 631 idxd_cleanup_internals(idxd); 632 if (device_pasid_enabled(idxd)) 633 idxd_disable_system_pasid(idxd); 634 iommu_dev_disable_feature(dev, IOMMU_DEV_FEAT_SVA); 635 } 636 637 static int idxd_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) 638 { 639 struct device *dev = &pdev->dev; 640 struct idxd_device *idxd; 641 struct idxd_driver_data *data = (struct idxd_driver_data *)id->driver_data; 642 int rc; 643 644 rc = pci_enable_device(pdev); 645 if (rc) 646 return rc; 647 648 dev_dbg(dev, "Alloc IDXD context\n"); 649 idxd = idxd_alloc(pdev, data); 650 if (!idxd) { 651 rc = -ENOMEM; 652 goto err_idxd_alloc; 653 } 654 655 dev_dbg(dev, "Mapping BARs\n"); 656 idxd->reg_base = pci_iomap(pdev, IDXD_MMIO_BAR, 0); 657 if (!idxd->reg_base) { 658 rc = -ENOMEM; 659 goto err_iomap; 660 } 661 662 dev_dbg(dev, "Set DMA masks\n"); 663 rc = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); 664 if (rc) 665 rc = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)); 666 if (rc) 667 goto err; 668 669 dev_dbg(dev, "Set PCI master\n"); 670 pci_set_master(pdev); 671 pci_set_drvdata(pdev, idxd); 672 673 idxd->hw.version = ioread32(idxd->reg_base + IDXD_VER_OFFSET); 674 rc = idxd_probe(idxd); 675 if (rc) { 676 dev_err(dev, "Intel(R) IDXD DMA Engine init failed\n"); 677 goto err; 678 } 679 680 rc = idxd_register_devices(idxd); 681 if (rc) { 682 dev_err(dev, "IDXD sysfs setup failed\n"); 683 goto err_dev_register; 684 } 685 686 dev_info(&pdev->dev, "Intel(R) Accelerator Device (v%x)\n", 687 idxd->hw.version); 688 689 return 0; 690 691 err_dev_register: 692 idxd_cleanup(idxd); 693 err: 694 pci_iounmap(pdev, idxd->reg_base); 695 err_iomap: 696 put_device(idxd_confdev(idxd)); 697 err_idxd_alloc: 698 pci_disable_device(pdev); 699 return rc; 700 } 701 702 static void idxd_flush_pending_llist(struct idxd_irq_entry *ie) 703 { 704 struct idxd_desc *desc, *itr; 705 struct llist_node *head; 706 707 head = llist_del_all(&ie->pending_llist); 708 if (!head) 709 return; 710 711 llist_for_each_entry_safe(desc, itr, head, llnode) { 712 idxd_dma_complete_txd(desc, IDXD_COMPLETE_ABORT); 713 idxd_free_desc(desc->wq, desc); 714 } 715 } 716 717 static void idxd_flush_work_list(struct idxd_irq_entry *ie) 718 { 719 struct idxd_desc *desc, *iter; 720 721 list_for_each_entry_safe(desc, iter, &ie->work_list, list) { 722 list_del(&desc->list); 723 idxd_dma_complete_txd(desc, IDXD_COMPLETE_ABORT); 724 idxd_free_desc(desc->wq, desc); 725 } 726 } 727 728 void idxd_wqs_quiesce(struct idxd_device *idxd) 729 { 730 struct idxd_wq *wq; 731 int i; 732 733 for (i = 0; i < idxd->max_wqs; i++) { 734 wq = idxd->wqs[i]; 735 if (wq->state == IDXD_WQ_ENABLED && wq->type == IDXD_WQT_KERNEL) 736 idxd_wq_quiesce(wq); 737 } 738 } 739 740 static void idxd_release_int_handles(struct idxd_device *idxd) 741 { 742 struct device *dev = &idxd->pdev->dev; 743 int i, rc; 744 745 for (i = 0; i < idxd->num_wq_irqs; i++) { 746 if (idxd->hw.cmd_cap & BIT(IDXD_CMD_RELEASE_INT_HANDLE)) { 747 rc = idxd_device_release_int_handle(idxd, idxd->int_handles[i], 748 IDXD_IRQ_MSIX); 749 if (rc < 0) 750 dev_warn(dev, "irq handle %d release failed\n", 751 idxd->int_handles[i]); 752 else 753 dev_dbg(dev, "int handle requested: %u\n", idxd->int_handles[i]); 754 } 755 } 756 } 757 758 static void idxd_shutdown(struct pci_dev *pdev) 759 { 760 struct idxd_device *idxd = pci_get_drvdata(pdev); 761 int rc, i; 762 struct idxd_irq_entry *irq_entry; 763 int msixcnt = pci_msix_vec_count(pdev); 764 765 rc = idxd_device_disable(idxd); 766 if (rc) 767 dev_err(&pdev->dev, "Disabling device failed\n"); 768 769 dev_dbg(&pdev->dev, "%s called\n", __func__); 770 idxd_mask_msix_vectors(idxd); 771 idxd_mask_error_interrupts(idxd); 772 773 for (i = 0; i < msixcnt; i++) { 774 irq_entry = &idxd->irq_entries[i]; 775 synchronize_irq(irq_entry->vector); 776 if (i == 0) 777 continue; 778 idxd_flush_pending_llist(irq_entry); 779 idxd_flush_work_list(irq_entry); 780 } 781 flush_workqueue(idxd->wq); 782 } 783 784 static void idxd_remove(struct pci_dev *pdev) 785 { 786 struct idxd_device *idxd = pci_get_drvdata(pdev); 787 struct idxd_irq_entry *irq_entry; 788 int msixcnt = pci_msix_vec_count(pdev); 789 int i; 790 791 dev_dbg(&pdev->dev, "%s called\n", __func__); 792 idxd_shutdown(pdev); 793 if (device_pasid_enabled(idxd)) 794 idxd_disable_system_pasid(idxd); 795 idxd_unregister_devices(idxd); 796 797 for (i = 0; i < msixcnt; i++) { 798 irq_entry = &idxd->irq_entries[i]; 799 free_irq(irq_entry->vector, irq_entry); 800 } 801 idxd_msix_perm_clear(idxd); 802 idxd_release_int_handles(idxd); 803 pci_free_irq_vectors(pdev); 804 pci_iounmap(pdev, idxd->reg_base); 805 iommu_dev_disable_feature(&pdev->dev, IOMMU_DEV_FEAT_SVA); 806 pci_disable_device(pdev); 807 destroy_workqueue(idxd->wq); 808 perfmon_pmu_remove(idxd); 809 device_unregister(idxd_confdev(idxd)); 810 } 811 812 static struct pci_driver idxd_pci_driver = { 813 .name = DRV_NAME, 814 .id_table = idxd_pci_tbl, 815 .probe = idxd_pci_probe, 816 .remove = idxd_remove, 817 .shutdown = idxd_shutdown, 818 }; 819 820 static int __init idxd_init_module(void) 821 { 822 int err; 823 824 /* 825 * If the CPU does not support MOVDIR64B or ENQCMDS, there's no point in 826 * enumerating the device. We can not utilize it. 827 */ 828 if (!cpu_feature_enabled(X86_FEATURE_MOVDIR64B)) { 829 pr_warn("idxd driver failed to load without MOVDIR64B.\n"); 830 return -ENODEV; 831 } 832 833 if (!cpu_feature_enabled(X86_FEATURE_ENQCMD)) 834 pr_warn("Platform does not have ENQCMD(S) support.\n"); 835 else 836 support_enqcmd = true; 837 838 perfmon_init(); 839 840 err = idxd_driver_register(&idxd_drv); 841 if (err < 0) 842 goto err_idxd_driver_register; 843 844 err = idxd_driver_register(&idxd_dmaengine_drv); 845 if (err < 0) 846 goto err_idxd_dmaengine_driver_register; 847 848 err = idxd_driver_register(&idxd_user_drv); 849 if (err < 0) 850 goto err_idxd_user_driver_register; 851 852 err = idxd_cdev_register(); 853 if (err) 854 goto err_cdev_register; 855 856 err = pci_register_driver(&idxd_pci_driver); 857 if (err) 858 goto err_pci_register; 859 860 return 0; 861 862 err_pci_register: 863 idxd_cdev_remove(); 864 err_cdev_register: 865 idxd_driver_unregister(&idxd_user_drv); 866 err_idxd_user_driver_register: 867 idxd_driver_unregister(&idxd_dmaengine_drv); 868 err_idxd_dmaengine_driver_register: 869 idxd_driver_unregister(&idxd_drv); 870 err_idxd_driver_register: 871 return err; 872 } 873 module_init(idxd_init_module); 874 875 static void __exit idxd_exit_module(void) 876 { 877 idxd_driver_unregister(&idxd_user_drv); 878 idxd_driver_unregister(&idxd_dmaengine_drv); 879 idxd_driver_unregister(&idxd_drv); 880 pci_unregister_driver(&idxd_pci_driver); 881 idxd_cdev_remove(); 882 perfmon_exit(); 883 } 884 module_exit(idxd_exit_module); 885