1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright(c) 2019 Intel Corporation. All rights rsvd. */ 3 #include <linux/init.h> 4 #include <linux/kernel.h> 5 #include <linux/module.h> 6 #include <linux/slab.h> 7 #include <linux/pci.h> 8 #include <linux/interrupt.h> 9 #include <linux/delay.h> 10 #include <linux/dma-mapping.h> 11 #include <linux/workqueue.h> 12 #include <linux/fs.h> 13 #include <linux/io-64-nonatomic-lo-hi.h> 14 #include <linux/device.h> 15 #include <linux/idr.h> 16 #include <linux/iommu.h> 17 #include <uapi/linux/idxd.h> 18 #include <linux/dmaengine.h> 19 #include "../dmaengine.h" 20 #include "registers.h" 21 #include "idxd.h" 22 #include "perfmon.h" 23 24 MODULE_VERSION(IDXD_DRIVER_VERSION); 25 MODULE_DESCRIPTION("Intel Data Streaming Accelerator and In-Memory Analytics Accelerator common driver"); 26 MODULE_LICENSE("GPL v2"); 27 MODULE_AUTHOR("Intel Corporation"); 28 MODULE_IMPORT_NS("IDXD"); 29 30 static bool sva = true; 31 module_param(sva, bool, 0644); 32 MODULE_PARM_DESC(sva, "Toggle SVA support on/off"); 33 34 bool tc_override; 35 module_param(tc_override, bool, 0644); 36 MODULE_PARM_DESC(tc_override, "Override traffic class defaults"); 37 38 #define DRV_NAME "idxd" 39 40 bool support_enqcmd; 41 DEFINE_IDA(idxd_ida); 42 43 static struct idxd_driver_data idxd_driver_data[] = { 44 [IDXD_TYPE_DSA] = { 45 .name_prefix = "dsa", 46 .type = IDXD_TYPE_DSA, 47 .compl_size = sizeof(struct dsa_completion_record), 48 .align = 32, 49 .dev_type = &dsa_device_type, 50 .evl_cr_off = offsetof(struct dsa_evl_entry, cr), 51 .user_submission_safe = false, /* See INTEL-SA-01084 security advisory */ 52 .cr_status_off = offsetof(struct dsa_completion_record, status), 53 .cr_result_off = offsetof(struct dsa_completion_record, result), 54 }, 55 [IDXD_TYPE_IAX] = { 56 .name_prefix = "iax", 57 .type = IDXD_TYPE_IAX, 58 .compl_size = sizeof(struct iax_completion_record), 59 .align = 64, 60 .dev_type = &iax_device_type, 61 .evl_cr_off = offsetof(struct iax_evl_entry, cr), 62 .user_submission_safe = false, /* See INTEL-SA-01084 security advisory */ 63 .cr_status_off = offsetof(struct iax_completion_record, status), 64 .cr_result_off = offsetof(struct iax_completion_record, error_code), 65 .load_device_defaults = idxd_load_iaa_device_defaults, 66 }, 67 }; 68 69 static struct pci_device_id idxd_pci_tbl[] = { 70 /* DSA ver 1.0 platforms */ 71 { PCI_DEVICE_DATA(INTEL, DSA_SPR0, &idxd_driver_data[IDXD_TYPE_DSA]) }, 72 /* DSA on GNR-D platforms */ 73 { PCI_DEVICE_DATA(INTEL, DSA_GNRD, &idxd_driver_data[IDXD_TYPE_DSA]) }, 74 /* DSA on DMR platforms */ 75 { PCI_DEVICE_DATA(INTEL, DSA_DMR, &idxd_driver_data[IDXD_TYPE_DSA]) }, 76 77 /* IAX ver 1.0 platforms */ 78 { PCI_DEVICE_DATA(INTEL, IAX_SPR0, &idxd_driver_data[IDXD_TYPE_IAX]) }, 79 /* IAA on DMR platforms */ 80 { PCI_DEVICE_DATA(INTEL, IAA_DMR, &idxd_driver_data[IDXD_TYPE_IAX]) }, 81 /* IAA PTL platforms */ 82 { PCI_DEVICE_DATA(INTEL, IAA_PTL, &idxd_driver_data[IDXD_TYPE_IAX]) }, 83 /* IAA WCL platforms */ 84 { PCI_DEVICE_DATA(INTEL, IAA_WCL, &idxd_driver_data[IDXD_TYPE_IAX]) }, 85 { 0, } 86 }; 87 MODULE_DEVICE_TABLE(pci, idxd_pci_tbl); 88 89 static int idxd_setup_interrupts(struct idxd_device *idxd) 90 { 91 struct pci_dev *pdev = idxd->pdev; 92 struct device *dev = &pdev->dev; 93 struct idxd_irq_entry *ie; 94 int i, msixcnt; 95 int rc = 0; 96 97 msixcnt = pci_msix_vec_count(pdev); 98 if (msixcnt < 0) { 99 dev_err(dev, "Not MSI-X interrupt capable.\n"); 100 return -ENOSPC; 101 } 102 idxd->irq_cnt = msixcnt; 103 104 rc = pci_alloc_irq_vectors(pdev, msixcnt, msixcnt, PCI_IRQ_MSIX); 105 if (rc != msixcnt) { 106 dev_err(dev, "Failed enabling %d MSIX entries: %d\n", msixcnt, rc); 107 return -ENOSPC; 108 } 109 dev_dbg(dev, "Enabled %d msix vectors\n", msixcnt); 110 111 112 ie = idxd_get_ie(idxd, 0); 113 ie->vector = pci_irq_vector(pdev, 0); 114 rc = request_threaded_irq(ie->vector, NULL, idxd_misc_thread, 0, "idxd-misc", ie); 115 if (rc < 0) { 116 dev_err(dev, "Failed to allocate misc interrupt.\n"); 117 goto err_misc_irq; 118 } 119 dev_dbg(dev, "Requested idxd-misc handler on msix vector %d\n", ie->vector); 120 121 for (i = 0; i < idxd->max_wqs; i++) { 122 int msix_idx = i + 1; 123 124 ie = idxd_get_ie(idxd, msix_idx); 125 ie->id = msix_idx; 126 ie->int_handle = INVALID_INT_HANDLE; 127 ie->pasid = IOMMU_PASID_INVALID; 128 129 spin_lock_init(&ie->list_lock); 130 init_llist_head(&ie->pending_llist); 131 INIT_LIST_HEAD(&ie->work_list); 132 } 133 134 idxd_unmask_error_interrupts(idxd); 135 return 0; 136 137 err_misc_irq: 138 idxd_mask_error_interrupts(idxd); 139 pci_free_irq_vectors(pdev); 140 dev_err(dev, "No usable interrupts\n"); 141 return rc; 142 } 143 144 static void idxd_cleanup_interrupts(struct idxd_device *idxd) 145 { 146 struct pci_dev *pdev = idxd->pdev; 147 struct idxd_irq_entry *ie; 148 int msixcnt; 149 150 msixcnt = pci_msix_vec_count(pdev); 151 if (msixcnt <= 0) 152 return; 153 154 ie = idxd_get_ie(idxd, 0); 155 idxd_mask_error_interrupts(idxd); 156 free_irq(ie->vector, ie); 157 pci_free_irq_vectors(pdev); 158 } 159 160 static void idxd_clean_wqs(struct idxd_device *idxd) 161 { 162 struct device *conf_dev; 163 int i; 164 165 for (i = 0; i < idxd->max_wqs; i++) { 166 conf_dev = wq_confdev(idxd->wqs[i]); 167 put_device(conf_dev); 168 } 169 bitmap_free(idxd->wq_enable_map); 170 kfree(idxd->wqs); 171 } 172 173 static int idxd_setup_wqs(struct idxd_device *idxd) 174 { 175 struct device *dev = &idxd->pdev->dev; 176 struct idxd_wq *wq; 177 struct device *conf_dev; 178 int i, rc; 179 180 idxd->wqs = kcalloc_node(idxd->max_wqs, sizeof(struct idxd_wq *), 181 GFP_KERNEL, dev_to_node(dev)); 182 if (!idxd->wqs) 183 return -ENOMEM; 184 185 idxd->wq_enable_map = bitmap_zalloc_node(idxd->max_wqs, GFP_KERNEL, dev_to_node(dev)); 186 if (!idxd->wq_enable_map) { 187 rc = -ENOMEM; 188 goto err_free_wqs; 189 } 190 191 for (i = 0; i < idxd->max_wqs; i++) { 192 wq = kzalloc_node(sizeof(*wq), GFP_KERNEL, dev_to_node(dev)); 193 if (!wq) { 194 rc = -ENOMEM; 195 goto err_unwind; 196 } 197 198 idxd_dev_set_type(&wq->idxd_dev, IDXD_DEV_WQ); 199 conf_dev = wq_confdev(wq); 200 wq->id = i; 201 wq->idxd = idxd; 202 device_initialize(conf_dev); 203 conf_dev->parent = idxd_confdev(idxd); 204 conf_dev->bus = &dsa_bus_type; 205 conf_dev->type = &idxd_wq_device_type; 206 rc = dev_set_name(conf_dev, "wq%d.%d", idxd->id, wq->id); 207 if (rc < 0) { 208 put_device(conf_dev); 209 goto err_unwind; 210 } 211 212 mutex_init(&wq->wq_lock); 213 init_waitqueue_head(&wq->err_queue); 214 init_completion(&wq->wq_dead); 215 init_completion(&wq->wq_resurrect); 216 wq->max_xfer_bytes = WQ_DEFAULT_MAX_XFER; 217 idxd_wq_set_max_batch_size(idxd->data->type, wq, WQ_DEFAULT_MAX_BATCH); 218 idxd_wq_set_init_max_sgl_size(idxd, wq); 219 wq->enqcmds_retries = IDXD_ENQCMDS_RETRIES; 220 wq->wqcfg = kzalloc_node(idxd->wqcfg_size, GFP_KERNEL, dev_to_node(dev)); 221 if (!wq->wqcfg) { 222 put_device(conf_dev); 223 rc = -ENOMEM; 224 goto err_unwind; 225 } 226 227 if (idxd->hw.wq_cap.op_config) { 228 wq->opcap_bmap = bitmap_zalloc(IDXD_MAX_OPCAP_BITS, GFP_KERNEL); 229 if (!wq->opcap_bmap) { 230 put_device(conf_dev); 231 rc = -ENOMEM; 232 goto err_unwind; 233 } 234 bitmap_copy(wq->opcap_bmap, idxd->opcap_bmap, IDXD_MAX_OPCAP_BITS); 235 } 236 mutex_init(&wq->uc_lock); 237 xa_init(&wq->upasid_xa); 238 idxd->wqs[i] = wq; 239 } 240 241 return 0; 242 243 err_unwind: 244 while (--i >= 0) { 245 conf_dev = wq_confdev(idxd->wqs[i]); 246 put_device(conf_dev); 247 } 248 bitmap_free(idxd->wq_enable_map); 249 250 err_free_wqs: 251 kfree(idxd->wqs); 252 253 return rc; 254 } 255 256 static void idxd_clean_engines(struct idxd_device *idxd) 257 { 258 struct device *conf_dev; 259 int i; 260 261 for (i = 0; i < idxd->max_engines; i++) { 262 conf_dev = engine_confdev(idxd->engines[i]); 263 put_device(conf_dev); 264 } 265 kfree(idxd->engines); 266 } 267 268 static int idxd_setup_engines(struct idxd_device *idxd) 269 { 270 struct idxd_engine *engine; 271 struct device *dev = &idxd->pdev->dev; 272 struct device *conf_dev; 273 int i, rc; 274 275 idxd->engines = kcalloc_node(idxd->max_engines, sizeof(struct idxd_engine *), 276 GFP_KERNEL, dev_to_node(dev)); 277 if (!idxd->engines) 278 return -ENOMEM; 279 280 for (i = 0; i < idxd->max_engines; i++) { 281 engine = kzalloc_node(sizeof(*engine), GFP_KERNEL, dev_to_node(dev)); 282 if (!engine) { 283 rc = -ENOMEM; 284 goto err; 285 } 286 287 idxd_dev_set_type(&engine->idxd_dev, IDXD_DEV_ENGINE); 288 conf_dev = engine_confdev(engine); 289 engine->id = i; 290 engine->idxd = idxd; 291 device_initialize(conf_dev); 292 conf_dev->parent = idxd_confdev(idxd); 293 conf_dev->bus = &dsa_bus_type; 294 conf_dev->type = &idxd_engine_device_type; 295 rc = dev_set_name(conf_dev, "engine%d.%d", idxd->id, engine->id); 296 if (rc < 0) { 297 put_device(conf_dev); 298 goto err; 299 } 300 301 idxd->engines[i] = engine; 302 } 303 304 return 0; 305 306 err: 307 while (--i >= 0) { 308 conf_dev = engine_confdev(idxd->engines[i]); 309 put_device(conf_dev); 310 } 311 kfree(idxd->engines); 312 313 return rc; 314 } 315 316 static void idxd_clean_groups(struct idxd_device *idxd) 317 { 318 int i; 319 320 for (i = 0; i < idxd->max_groups; i++) { 321 put_device(group_confdev(idxd->groups[i])); 322 } 323 kfree(idxd->groups); 324 } 325 326 static int idxd_setup_groups(struct idxd_device *idxd) 327 { 328 struct device *dev = &idxd->pdev->dev; 329 struct device *conf_dev; 330 struct idxd_group *group; 331 int i, rc; 332 333 idxd->groups = kcalloc_node(idxd->max_groups, sizeof(struct idxd_group *), 334 GFP_KERNEL, dev_to_node(dev)); 335 if (!idxd->groups) 336 return -ENOMEM; 337 338 for (i = 0; i < idxd->max_groups; i++) { 339 group = kzalloc_node(sizeof(*group), GFP_KERNEL, dev_to_node(dev)); 340 if (!group) { 341 rc = -ENOMEM; 342 goto err; 343 } 344 345 idxd_dev_set_type(&group->idxd_dev, IDXD_DEV_GROUP); 346 conf_dev = group_confdev(group); 347 group->id = i; 348 group->idxd = idxd; 349 device_initialize(conf_dev); 350 conf_dev->parent = idxd_confdev(idxd); 351 conf_dev->bus = &dsa_bus_type; 352 conf_dev->type = &idxd_group_device_type; 353 rc = dev_set_name(conf_dev, "group%d.%d", idxd->id, group->id); 354 if (rc < 0) { 355 put_device(conf_dev); 356 goto err; 357 } 358 359 idxd->groups[i] = group; 360 if (idxd->hw.version <= DEVICE_VERSION_2 && !tc_override) { 361 group->tc_a = 1; 362 group->tc_b = 1; 363 } else { 364 group->tc_a = -1; 365 group->tc_b = -1; 366 } 367 /* 368 * The default value is the same as the value of 369 * total read buffers in GRPCAP. 370 */ 371 group->rdbufs_allowed = idxd->max_rdbufs; 372 } 373 374 return 0; 375 376 err: 377 while (--i >= 0) { 378 group = idxd->groups[i]; 379 put_device(group_confdev(group)); 380 } 381 kfree(idxd->groups); 382 383 return rc; 384 } 385 386 static void idxd_cleanup_internals(struct idxd_device *idxd) 387 { 388 idxd_clean_groups(idxd); 389 idxd_clean_engines(idxd); 390 idxd_clean_wqs(idxd); 391 destroy_workqueue(idxd->wq); 392 } 393 394 static int idxd_init_evl(struct idxd_device *idxd) 395 { 396 struct device *dev = &idxd->pdev->dev; 397 unsigned int evl_cache_size; 398 struct idxd_evl *evl; 399 const char *idxd_name; 400 401 if (idxd->hw.gen_cap.evl_support == 0) 402 return 0; 403 404 evl = kzalloc_node(sizeof(*evl), GFP_KERNEL, dev_to_node(dev)); 405 if (!evl) 406 return -ENOMEM; 407 408 mutex_init(&evl->lock); 409 evl->size = IDXD_EVL_SIZE_MIN; 410 411 idxd_name = dev_name(idxd_confdev(idxd)); 412 evl_cache_size = sizeof(struct idxd_evl_fault) + evl_ent_size(idxd); 413 /* 414 * Since completion record in evl_cache will be copied to user 415 * when handling completion record page fault, need to create 416 * the cache suitable for user copy. 417 */ 418 idxd->evl_cache = kmem_cache_create_usercopy(idxd_name, evl_cache_size, 419 0, 0, 0, evl_cache_size, 420 NULL); 421 if (!idxd->evl_cache) { 422 kfree(evl); 423 return -ENOMEM; 424 } 425 426 idxd->evl = evl; 427 return 0; 428 } 429 430 static int idxd_setup_internals(struct idxd_device *idxd) 431 { 432 struct device *dev = &idxd->pdev->dev; 433 int rc; 434 435 init_waitqueue_head(&idxd->cmd_waitq); 436 437 rc = idxd_setup_wqs(idxd); 438 if (rc < 0) 439 goto err_wqs; 440 441 rc = idxd_setup_engines(idxd); 442 if (rc < 0) 443 goto err_engine; 444 445 rc = idxd_setup_groups(idxd); 446 if (rc < 0) 447 goto err_group; 448 449 idxd->wq = create_workqueue(dev_name(dev)); 450 if (!idxd->wq) { 451 rc = -ENOMEM; 452 goto err_wkq_create; 453 } 454 455 rc = idxd_init_evl(idxd); 456 if (rc < 0) 457 goto err_evl; 458 459 return 0; 460 461 err_evl: 462 destroy_workqueue(idxd->wq); 463 err_wkq_create: 464 idxd_clean_groups(idxd); 465 err_group: 466 idxd_clean_engines(idxd); 467 err_engine: 468 idxd_clean_wqs(idxd); 469 err_wqs: 470 return rc; 471 } 472 473 static void idxd_read_table_offsets(struct idxd_device *idxd) 474 { 475 union offsets_reg offsets; 476 struct device *dev = &idxd->pdev->dev; 477 478 offsets.bits[0] = ioread64(idxd->reg_base + IDXD_TABLE_OFFSET); 479 offsets.bits[1] = ioread64(idxd->reg_base + IDXD_TABLE_OFFSET + sizeof(u64)); 480 idxd->grpcfg_offset = offsets.grpcfg * IDXD_TABLE_MULT; 481 dev_dbg(dev, "IDXD Group Config Offset: %#x\n", idxd->grpcfg_offset); 482 idxd->wqcfg_offset = offsets.wqcfg * IDXD_TABLE_MULT; 483 dev_dbg(dev, "IDXD Work Queue Config Offset: %#x\n", idxd->wqcfg_offset); 484 idxd->msix_perm_offset = offsets.msix_perm * IDXD_TABLE_MULT; 485 dev_dbg(dev, "IDXD MSIX Permission Offset: %#x\n", idxd->msix_perm_offset); 486 idxd->perfmon_offset = offsets.perfmon * IDXD_TABLE_MULT; 487 dev_dbg(dev, "IDXD Perfmon Offset: %#x\n", idxd->perfmon_offset); 488 } 489 490 void multi_u64_to_bmap(unsigned long *bmap, u64 *val, int count) 491 { 492 int i, j, nr; 493 494 for (i = 0, nr = 0; i < count; i++) { 495 for (j = 0; j < BITS_PER_LONG_LONG; j++) { 496 if (val[i] & BIT(j)) 497 set_bit(nr, bmap); 498 nr++; 499 } 500 } 501 } 502 503 static void idxd_read_caps(struct idxd_device *idxd) 504 { 505 struct device *dev = &idxd->pdev->dev; 506 int i; 507 508 /* reading generic capabilities */ 509 idxd->hw.gen_cap.bits = ioread64(idxd->reg_base + IDXD_GENCAP_OFFSET); 510 dev_dbg(dev, "gen_cap: %#llx\n", idxd->hw.gen_cap.bits); 511 512 if (idxd->hw.gen_cap.cmd_cap) { 513 idxd->hw.cmd_cap = ioread32(idxd->reg_base + IDXD_CMDCAP_OFFSET); 514 dev_dbg(dev, "cmd_cap: %#x\n", idxd->hw.cmd_cap); 515 } 516 517 /* reading command capabilities */ 518 if (idxd->hw.cmd_cap & BIT(IDXD_CMD_REQUEST_INT_HANDLE)) 519 idxd->request_int_handles = true; 520 521 idxd->max_xfer_bytes = 1ULL << idxd->hw.gen_cap.max_xfer_shift; 522 dev_dbg(dev, "max xfer size: %llu bytes\n", idxd->max_xfer_bytes); 523 idxd_set_max_batch_size(idxd->data->type, idxd, 1U << idxd->hw.gen_cap.max_batch_shift); 524 dev_dbg(dev, "max batch size: %u\n", idxd->max_batch_size); 525 if (idxd->hw.gen_cap.config_en) 526 set_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags); 527 528 /* reading group capabilities */ 529 idxd->hw.group_cap.bits = 530 ioread64(idxd->reg_base + IDXD_GRPCAP_OFFSET); 531 dev_dbg(dev, "group_cap: %#llx\n", idxd->hw.group_cap.bits); 532 idxd->max_groups = idxd->hw.group_cap.num_groups; 533 dev_dbg(dev, "max groups: %u\n", idxd->max_groups); 534 idxd->max_rdbufs = idxd->hw.group_cap.total_rdbufs; 535 dev_dbg(dev, "max read buffers: %u\n", idxd->max_rdbufs); 536 idxd->nr_rdbufs = idxd->max_rdbufs; 537 538 /* read engine capabilities */ 539 idxd->hw.engine_cap.bits = 540 ioread64(idxd->reg_base + IDXD_ENGCAP_OFFSET); 541 dev_dbg(dev, "engine_cap: %#llx\n", idxd->hw.engine_cap.bits); 542 idxd->max_engines = idxd->hw.engine_cap.num_engines; 543 dev_dbg(dev, "max engines: %u\n", idxd->max_engines); 544 545 /* read workqueue capabilities */ 546 idxd->hw.wq_cap.bits = ioread64(idxd->reg_base + IDXD_WQCAP_OFFSET); 547 dev_dbg(dev, "wq_cap: %#llx\n", idxd->hw.wq_cap.bits); 548 idxd->max_wq_size = idxd->hw.wq_cap.total_wq_size; 549 dev_dbg(dev, "total workqueue size: %u\n", idxd->max_wq_size); 550 idxd->max_wqs = idxd->hw.wq_cap.num_wqs; 551 dev_dbg(dev, "max workqueues: %u\n", idxd->max_wqs); 552 idxd->wqcfg_size = 1 << (idxd->hw.wq_cap.wqcfg_size + IDXD_WQCFG_MIN); 553 dev_dbg(dev, "wqcfg size: %u\n", idxd->wqcfg_size); 554 555 /* reading operation capabilities */ 556 for (i = 0; i < 4; i++) { 557 idxd->hw.opcap.bits[i] = ioread64(idxd->reg_base + 558 IDXD_OPCAP_OFFSET + i * sizeof(u64)); 559 dev_dbg(dev, "opcap[%d]: %#llx\n", i, idxd->hw.opcap.bits[i]); 560 } 561 multi_u64_to_bmap(idxd->opcap_bmap, &idxd->hw.opcap.bits[0], 4); 562 563 if (idxd->hw.version >= DEVICE_VERSION_3) { 564 idxd->hw.dsacap0.bits = ioread64(idxd->reg_base + IDXD_DSACAP0_OFFSET); 565 idxd->hw.dsacap1.bits = ioread64(idxd->reg_base + IDXD_DSACAP1_OFFSET); 566 idxd->hw.dsacap2.bits = ioread64(idxd->reg_base + IDXD_DSACAP2_OFFSET); 567 } 568 if (idxd_sgl_supported(idxd)) { 569 idxd->max_sgl_size = 1U << idxd->hw.dsacap0.max_sgl_shift; 570 dev_dbg(dev, "max sgl size: %u\n", idxd->max_sgl_size); 571 } 572 573 /* read iaa cap */ 574 if (idxd->data->type == IDXD_TYPE_IAX && idxd->hw.version >= DEVICE_VERSION_2) 575 idxd->hw.iaa_cap.bits = ioread64(idxd->reg_base + IDXD_IAACAP_OFFSET); 576 } 577 578 static void idxd_free(struct idxd_device *idxd) 579 { 580 if (!idxd) 581 return; 582 583 put_device(idxd_confdev(idxd)); 584 bitmap_free(idxd->opcap_bmap); 585 ida_free(&idxd_ida, idxd->id); 586 kfree(idxd); 587 } 588 589 static struct idxd_device *idxd_alloc(struct pci_dev *pdev, struct idxd_driver_data *data) 590 { 591 struct device *dev = &pdev->dev; 592 struct device *conf_dev; 593 struct idxd_device *idxd; 594 int rc; 595 596 idxd = kzalloc_node(sizeof(*idxd), GFP_KERNEL, dev_to_node(dev)); 597 if (!idxd) 598 return NULL; 599 600 conf_dev = idxd_confdev(idxd); 601 idxd->pdev = pdev; 602 idxd->data = data; 603 idxd_dev_set_type(&idxd->idxd_dev, idxd->data->type); 604 idxd->id = ida_alloc(&idxd_ida, GFP_KERNEL); 605 if (idxd->id < 0) 606 goto err_ida; 607 608 idxd->opcap_bmap = bitmap_zalloc_node(IDXD_MAX_OPCAP_BITS, GFP_KERNEL, dev_to_node(dev)); 609 if (!idxd->opcap_bmap) 610 goto err_opcap; 611 612 device_initialize(conf_dev); 613 conf_dev->parent = dev; 614 conf_dev->bus = &dsa_bus_type; 615 conf_dev->type = idxd->data->dev_type; 616 rc = dev_set_name(conf_dev, "%s%d", idxd->data->name_prefix, idxd->id); 617 if (rc < 0) 618 goto err_name; 619 620 spin_lock_init(&idxd->dev_lock); 621 spin_lock_init(&idxd->cmd_lock); 622 623 return idxd; 624 625 err_name: 626 put_device(conf_dev); 627 bitmap_free(idxd->opcap_bmap); 628 err_opcap: 629 ida_free(&idxd_ida, idxd->id); 630 err_ida: 631 kfree(idxd); 632 633 return NULL; 634 } 635 636 static int idxd_enable_system_pasid(struct idxd_device *idxd) 637 { 638 struct pci_dev *pdev = idxd->pdev; 639 struct device *dev = &pdev->dev; 640 struct iommu_domain *domain; 641 ioasid_t pasid; 642 int ret; 643 644 /* 645 * Attach a global PASID to the DMA domain so that we can use ENQCMDS 646 * to submit work on buffers mapped by DMA API. 647 */ 648 domain = iommu_get_domain_for_dev(dev); 649 if (!domain) 650 return -EPERM; 651 652 pasid = iommu_alloc_global_pasid(dev); 653 if (pasid == IOMMU_PASID_INVALID) 654 return -ENOSPC; 655 656 /* 657 * DMA domain is owned by the driver, it should support all valid 658 * types such as DMA-FQ, identity, etc. 659 */ 660 ret = iommu_attach_device_pasid(domain, dev, pasid, NULL); 661 if (ret) { 662 dev_err(dev, "failed to attach device pasid %d, domain type %d", 663 pasid, domain->type); 664 iommu_free_global_pasid(pasid); 665 return ret; 666 } 667 668 /* Since we set user privilege for kernel DMA, enable completion IRQ */ 669 idxd_set_user_intr(idxd, 1); 670 idxd->pasid = pasid; 671 672 return ret; 673 } 674 675 static void idxd_disable_system_pasid(struct idxd_device *idxd) 676 { 677 struct pci_dev *pdev = idxd->pdev; 678 struct device *dev = &pdev->dev; 679 struct iommu_domain *domain; 680 681 domain = iommu_get_domain_for_dev(dev); 682 if (!domain) 683 return; 684 685 iommu_detach_device_pasid(domain, dev, idxd->pasid); 686 iommu_free_global_pasid(idxd->pasid); 687 688 idxd_set_user_intr(idxd, 0); 689 idxd->sva = NULL; 690 idxd->pasid = IOMMU_PASID_INVALID; 691 } 692 693 static int idxd_probe(struct idxd_device *idxd) 694 { 695 struct pci_dev *pdev = idxd->pdev; 696 struct device *dev = &pdev->dev; 697 int rc; 698 699 dev_dbg(dev, "%s entered and resetting device\n", __func__); 700 rc = idxd_device_init_reset(idxd); 701 if (rc < 0) 702 return rc; 703 704 dev_dbg(dev, "IDXD reset complete\n"); 705 706 if (IS_ENABLED(CONFIG_INTEL_IDXD_SVM) && sva) { 707 set_bit(IDXD_FLAG_USER_PASID_ENABLED, &idxd->flags); 708 709 rc = idxd_enable_system_pasid(idxd); 710 if (rc) 711 dev_warn(dev, "No in-kernel DMA with PASID. %d\n", rc); 712 else 713 set_bit(IDXD_FLAG_PASID_ENABLED, &idxd->flags); 714 } else if (!sva) { 715 dev_warn(dev, "User forced SVA off via module param.\n"); 716 } 717 718 idxd_read_caps(idxd); 719 idxd_read_table_offsets(idxd); 720 721 rc = idxd_setup_internals(idxd); 722 if (rc) 723 goto err; 724 725 /* If the configs are readonly, then load them from device */ 726 if (!test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags)) { 727 dev_dbg(dev, "Loading RO device config\n"); 728 rc = idxd_device_load_config(idxd); 729 if (rc < 0) 730 goto err_config; 731 } 732 733 rc = idxd_setup_interrupts(idxd); 734 if (rc) 735 goto err_config; 736 737 idxd->major = idxd_cdev_get_major(idxd); 738 739 rc = perfmon_pmu_init(idxd); 740 if (rc < 0) 741 dev_warn(dev, "Failed to initialize perfmon. No PMU support: %d\n", rc); 742 743 dev_dbg(dev, "IDXD device %d probed successfully\n", idxd->id); 744 return 0; 745 746 err_config: 747 idxd_cleanup_internals(idxd); 748 err: 749 if (device_pasid_enabled(idxd)) 750 idxd_disable_system_pasid(idxd); 751 return rc; 752 } 753 754 static void idxd_cleanup(struct idxd_device *idxd) 755 { 756 perfmon_pmu_remove(idxd); 757 idxd_cleanup_interrupts(idxd); 758 idxd_cleanup_internals(idxd); 759 if (device_pasid_enabled(idxd)) 760 idxd_disable_system_pasid(idxd); 761 } 762 763 /* 764 * Attach IDXD device to IDXD driver. 765 */ 766 static int idxd_bind(struct device_driver *drv, const char *buf) 767 { 768 const struct bus_type *bus = drv->bus; 769 struct device *dev; 770 int err = -ENODEV; 771 772 dev = bus_find_device_by_name(bus, NULL, buf); 773 if (dev) 774 err = device_driver_attach(drv, dev); 775 776 put_device(dev); 777 778 return err; 779 } 780 781 /* 782 * Detach IDXD device from driver. 783 */ 784 static void idxd_unbind(struct device_driver *drv, const char *buf) 785 { 786 const struct bus_type *bus = drv->bus; 787 struct device *dev; 788 789 dev = bus_find_device_by_name(bus, NULL, buf); 790 if (dev && dev->driver == drv) 791 device_release_driver(dev); 792 793 put_device(dev); 794 } 795 796 #define idxd_free_saved_configs(saved_configs, count) \ 797 do { \ 798 int i; \ 799 \ 800 for (i = 0; i < (count); i++) \ 801 kfree(saved_configs[i]); \ 802 } while (0) 803 804 static void idxd_free_saved(struct idxd_group **saved_groups, 805 struct idxd_engine **saved_engines, 806 struct idxd_wq **saved_wqs, 807 struct idxd_device *idxd) 808 { 809 if (saved_groups) 810 idxd_free_saved_configs(saved_groups, idxd->max_groups); 811 if (saved_engines) 812 idxd_free_saved_configs(saved_engines, idxd->max_engines); 813 if (saved_wqs) 814 idxd_free_saved_configs(saved_wqs, idxd->max_wqs); 815 } 816 817 /* 818 * Save IDXD device configurations including engines, groups, wqs etc. 819 * The saved configurations can be restored when needed. 820 */ 821 static int idxd_device_config_save(struct idxd_device *idxd, 822 struct idxd_saved_states *idxd_saved) 823 { 824 struct device *dev = &idxd->pdev->dev; 825 int i; 826 827 memcpy(&idxd_saved->saved_idxd, idxd, sizeof(*idxd)); 828 829 if (idxd->evl) { 830 memcpy(&idxd_saved->saved_evl, idxd->evl, 831 sizeof(struct idxd_evl)); 832 } 833 834 struct idxd_group **saved_groups __free(kfree) = 835 kcalloc_node(idxd->max_groups, 836 sizeof(struct idxd_group *), 837 GFP_KERNEL, dev_to_node(dev)); 838 if (!saved_groups) 839 return -ENOMEM; 840 841 for (i = 0; i < idxd->max_groups; i++) { 842 struct idxd_group *saved_group __free(kfree) = 843 kzalloc_node(sizeof(*saved_group), GFP_KERNEL, 844 dev_to_node(dev)); 845 846 if (!saved_group) { 847 /* Free saved groups */ 848 idxd_free_saved(saved_groups, NULL, NULL, idxd); 849 850 return -ENOMEM; 851 } 852 853 memcpy(saved_group, idxd->groups[i], sizeof(*saved_group)); 854 saved_groups[i] = no_free_ptr(saved_group); 855 } 856 857 struct idxd_engine **saved_engines = 858 kcalloc_node(idxd->max_engines, 859 sizeof(struct idxd_engine *), 860 GFP_KERNEL, dev_to_node(dev)); 861 if (!saved_engines) { 862 /* Free saved groups */ 863 idxd_free_saved(saved_groups, NULL, NULL, idxd); 864 865 return -ENOMEM; 866 } 867 for (i = 0; i < idxd->max_engines; i++) { 868 struct idxd_engine *saved_engine __free(kfree) = 869 kzalloc_node(sizeof(*saved_engine), GFP_KERNEL, 870 dev_to_node(dev)); 871 if (!saved_engine) { 872 /* Free saved groups and engines */ 873 idxd_free_saved(saved_groups, saved_engines, NULL, 874 idxd); 875 876 return -ENOMEM; 877 } 878 879 memcpy(saved_engine, idxd->engines[i], sizeof(*saved_engine)); 880 saved_engines[i] = no_free_ptr(saved_engine); 881 } 882 883 unsigned long *saved_wq_enable_map __free(bitmap) = 884 bitmap_zalloc_node(idxd->max_wqs, GFP_KERNEL, 885 dev_to_node(dev)); 886 if (!saved_wq_enable_map) { 887 /* Free saved groups and engines */ 888 idxd_free_saved(saved_groups, saved_engines, NULL, idxd); 889 890 return -ENOMEM; 891 } 892 893 bitmap_copy(saved_wq_enable_map, idxd->wq_enable_map, idxd->max_wqs); 894 895 struct idxd_wq **saved_wqs __free(kfree) = 896 kcalloc_node(idxd->max_wqs, sizeof(struct idxd_wq *), 897 GFP_KERNEL, dev_to_node(dev)); 898 if (!saved_wqs) { 899 /* Free saved groups and engines */ 900 idxd_free_saved(saved_groups, saved_engines, NULL, idxd); 901 902 return -ENOMEM; 903 } 904 905 for (i = 0; i < idxd->max_wqs; i++) { 906 struct idxd_wq *saved_wq __free(kfree) = 907 kzalloc_node(sizeof(*saved_wq), GFP_KERNEL, 908 dev_to_node(dev)); 909 struct idxd_wq *wq; 910 911 if (!saved_wq) { 912 /* Free saved groups, engines, and wqs */ 913 idxd_free_saved(saved_groups, saved_engines, saved_wqs, 914 idxd); 915 916 return -ENOMEM; 917 } 918 919 if (!test_bit(i, saved_wq_enable_map)) 920 continue; 921 922 wq = idxd->wqs[i]; 923 mutex_lock(&wq->wq_lock); 924 memcpy(saved_wq, wq, sizeof(*saved_wq)); 925 saved_wqs[i] = no_free_ptr(saved_wq); 926 mutex_unlock(&wq->wq_lock); 927 } 928 929 /* Save configurations */ 930 idxd_saved->saved_groups = no_free_ptr(saved_groups); 931 idxd_saved->saved_engines = no_free_ptr(saved_engines); 932 idxd_saved->saved_wq_enable_map = no_free_ptr(saved_wq_enable_map); 933 idxd_saved->saved_wqs = no_free_ptr(saved_wqs); 934 935 return 0; 936 } 937 938 /* 939 * Restore IDXD device configurations including engines, groups, wqs etc 940 * that were saved before. 941 */ 942 static void idxd_device_config_restore(struct idxd_device *idxd, 943 struct idxd_saved_states *idxd_saved) 944 { 945 struct idxd_evl *saved_evl = &idxd_saved->saved_evl; 946 int i; 947 948 idxd->rdbuf_limit = idxd_saved->saved_idxd.rdbuf_limit; 949 950 if (idxd->evl) 951 idxd->evl->size = saved_evl->size; 952 953 for (i = 0; i < idxd->max_groups; i++) { 954 struct idxd_group *saved_group, *group; 955 956 saved_group = idxd_saved->saved_groups[i]; 957 group = idxd->groups[i]; 958 959 group->rdbufs_allowed = saved_group->rdbufs_allowed; 960 group->rdbufs_reserved = saved_group->rdbufs_reserved; 961 group->tc_a = saved_group->tc_a; 962 group->tc_b = saved_group->tc_b; 963 group->use_rdbuf_limit = saved_group->use_rdbuf_limit; 964 965 kfree(saved_group); 966 } 967 kfree(idxd_saved->saved_groups); 968 969 for (i = 0; i < idxd->max_engines; i++) { 970 struct idxd_engine *saved_engine, *engine; 971 972 saved_engine = idxd_saved->saved_engines[i]; 973 engine = idxd->engines[i]; 974 975 engine->group = saved_engine->group; 976 977 kfree(saved_engine); 978 } 979 kfree(idxd_saved->saved_engines); 980 981 bitmap_copy(idxd->wq_enable_map, idxd_saved->saved_wq_enable_map, 982 idxd->max_wqs); 983 bitmap_free(idxd_saved->saved_wq_enable_map); 984 985 for (i = 0; i < idxd->max_wqs; i++) { 986 struct idxd_wq *saved_wq, *wq; 987 size_t len; 988 989 if (!test_bit(i, idxd->wq_enable_map)) 990 continue; 991 992 saved_wq = idxd_saved->saved_wqs[i]; 993 wq = idxd->wqs[i]; 994 995 mutex_lock(&wq->wq_lock); 996 997 wq->group = saved_wq->group; 998 wq->flags = saved_wq->flags; 999 wq->threshold = saved_wq->threshold; 1000 wq->size = saved_wq->size; 1001 wq->priority = saved_wq->priority; 1002 wq->type = saved_wq->type; 1003 len = strlen(saved_wq->name) + 1; 1004 strscpy(wq->name, saved_wq->name, len); 1005 wq->max_xfer_bytes = saved_wq->max_xfer_bytes; 1006 wq->max_batch_size = saved_wq->max_batch_size; 1007 wq->enqcmds_retries = saved_wq->enqcmds_retries; 1008 wq->descs = saved_wq->descs; 1009 wq->idxd_chan = saved_wq->idxd_chan; 1010 len = strlen(saved_wq->driver_name) + 1; 1011 strscpy(wq->driver_name, saved_wq->driver_name, len); 1012 1013 mutex_unlock(&wq->wq_lock); 1014 1015 kfree(saved_wq); 1016 } 1017 1018 kfree(idxd_saved->saved_wqs); 1019 } 1020 1021 static void idxd_reset_prepare(struct pci_dev *pdev) 1022 { 1023 struct idxd_device *idxd = pci_get_drvdata(pdev); 1024 struct device *dev = &idxd->pdev->dev; 1025 const char *idxd_name; 1026 int rc; 1027 1028 idxd_name = dev_name(idxd_confdev(idxd)); 1029 1030 struct idxd_saved_states *idxd_saved __free(kfree) = 1031 kzalloc_node(sizeof(*idxd_saved), GFP_KERNEL, 1032 dev_to_node(&pdev->dev)); 1033 if (!idxd_saved) { 1034 dev_err(dev, "HALT: no memory\n"); 1035 1036 return; 1037 } 1038 1039 /* Save IDXD configurations. */ 1040 rc = idxd_device_config_save(idxd, idxd_saved); 1041 if (rc < 0) { 1042 dev_err(dev, "HALT: cannot save %s configs\n", idxd_name); 1043 1044 return; 1045 } 1046 1047 idxd->idxd_saved = no_free_ptr(idxd_saved); 1048 1049 /* Save PCI device state. */ 1050 pci_save_state(idxd->pdev); 1051 } 1052 1053 static void idxd_reset_done(struct pci_dev *pdev) 1054 { 1055 struct idxd_device *idxd = pci_get_drvdata(pdev); 1056 const char *idxd_name; 1057 struct device *dev; 1058 int rc, i; 1059 1060 if (!idxd->idxd_saved) 1061 return; 1062 1063 dev = &idxd->pdev->dev; 1064 idxd_name = dev_name(idxd_confdev(idxd)); 1065 1066 /* Restore PCI device state. */ 1067 pci_restore_state(idxd->pdev); 1068 1069 /* Unbind idxd device from driver. */ 1070 idxd_unbind(&idxd_drv.drv, idxd_name); 1071 1072 /* 1073 * Probe PCI device without allocating or changing 1074 * idxd software data which keeps the same as before FLR. 1075 */ 1076 idxd_pci_probe_alloc(idxd, NULL, NULL); 1077 1078 /* Restore IDXD configurations. */ 1079 idxd_device_config_restore(idxd, idxd->idxd_saved); 1080 1081 /* Re-configure IDXD device if allowed. */ 1082 rc = idxd_device_config(idxd); 1083 if (rc < 0) { 1084 dev_err(dev, "HALT: %s config fails\n", idxd_name); 1085 goto out; 1086 } 1087 1088 /* Bind IDXD device to driver. */ 1089 rc = idxd_bind(&idxd_drv.drv, idxd_name); 1090 if (rc < 0) { 1091 dev_err(dev, "HALT: binding %s to driver fails\n", idxd_name); 1092 goto out; 1093 } 1094 1095 /* Bind enabled wq in the IDXD device to driver. */ 1096 for (i = 0; i < idxd->max_wqs; i++) { 1097 if (test_bit(i, idxd->wq_enable_map)) { 1098 struct idxd_wq *wq = idxd->wqs[i]; 1099 char wq_name[32]; 1100 1101 wq->state = IDXD_WQ_DISABLED; 1102 sprintf(wq_name, "wq%d.%d", idxd->id, wq->id); 1103 /* 1104 * Bind to user driver depending on wq type. 1105 * 1106 * Currently only support user type WQ. Will support 1107 * kernel type WQ in the future. 1108 */ 1109 if (wq->type == IDXD_WQT_USER) 1110 rc = idxd_bind(&idxd_user_drv.drv, wq_name); 1111 else 1112 rc = -EINVAL; 1113 if (rc < 0) { 1114 clear_bit(i, idxd->wq_enable_map); 1115 dev_err(dev, 1116 "HALT: unable to re-enable wq %s\n", 1117 dev_name(wq_confdev(wq))); 1118 } 1119 } 1120 } 1121 out: 1122 kfree(idxd->idxd_saved); 1123 idxd->idxd_saved = NULL; 1124 } 1125 1126 static const struct pci_error_handlers idxd_error_handler = { 1127 .reset_prepare = idxd_reset_prepare, 1128 .reset_done = idxd_reset_done, 1129 }; 1130 1131 /* 1132 * Probe idxd PCI device. 1133 * If idxd is not given, need to allocate idxd and set up its data. 1134 * 1135 * If idxd is given, idxd was allocated and setup already. Just need to 1136 * configure device without re-allocating and re-configuring idxd data. 1137 * This is useful for recovering from FLR. 1138 */ 1139 int idxd_pci_probe_alloc(struct idxd_device *idxd, struct pci_dev *pdev, 1140 const struct pci_device_id *id) 1141 { 1142 bool alloc_idxd = idxd ? false : true; 1143 struct idxd_driver_data *data; 1144 struct device *dev; 1145 int rc; 1146 1147 pdev = idxd ? idxd->pdev : pdev; 1148 dev = &pdev->dev; 1149 data = id ? (struct idxd_driver_data *)id->driver_data : NULL; 1150 rc = pci_enable_device(pdev); 1151 if (rc) 1152 return rc; 1153 1154 if (alloc_idxd) { 1155 dev_dbg(dev, "Alloc IDXD context\n"); 1156 idxd = idxd_alloc(pdev, data); 1157 if (!idxd) { 1158 rc = -ENOMEM; 1159 goto err_idxd_alloc; 1160 } 1161 1162 dev_dbg(dev, "Mapping BARs\n"); 1163 idxd->reg_base = pci_iomap(pdev, IDXD_MMIO_BAR, 0); 1164 if (!idxd->reg_base) { 1165 rc = -ENOMEM; 1166 goto err_iomap; 1167 } 1168 1169 dev_dbg(dev, "Set DMA masks\n"); 1170 rc = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); 1171 if (rc) 1172 goto err; 1173 } 1174 1175 dev_dbg(dev, "Set PCI master\n"); 1176 pci_set_master(pdev); 1177 pci_set_drvdata(pdev, idxd); 1178 1179 if (alloc_idxd) { 1180 idxd->hw.version = ioread32(idxd->reg_base + IDXD_VER_OFFSET); 1181 rc = idxd_probe(idxd); 1182 if (rc) { 1183 dev_err(dev, "Intel(R) IDXD DMA Engine init failed\n"); 1184 goto err; 1185 } 1186 1187 if (data->load_device_defaults) { 1188 rc = data->load_device_defaults(idxd); 1189 if (rc) 1190 dev_warn(dev, "IDXD loading device defaults failed\n"); 1191 } 1192 1193 rc = idxd_register_devices(idxd); 1194 if (rc) { 1195 dev_err(dev, "IDXD sysfs setup failed\n"); 1196 goto err_dev_register; 1197 } 1198 1199 rc = idxd_device_init_debugfs(idxd); 1200 if (rc) 1201 dev_warn(dev, "IDXD debugfs failed to setup\n"); 1202 } 1203 1204 if (!alloc_idxd) { 1205 /* Release interrupts in the IDXD device. */ 1206 idxd_cleanup_interrupts(idxd); 1207 1208 /* Re-enable interrupts in the IDXD device. */ 1209 rc = idxd_setup_interrupts(idxd); 1210 if (rc) 1211 dev_warn(dev, "IDXD interrupts failed to setup\n"); 1212 } 1213 1214 dev_info(&pdev->dev, "Intel(R) Accelerator Device (v%x)\n", 1215 idxd->hw.version); 1216 1217 if (data) 1218 idxd->user_submission_safe = data->user_submission_safe; 1219 1220 return 0; 1221 1222 err_dev_register: 1223 idxd_cleanup(idxd); 1224 err: 1225 pci_iounmap(pdev, idxd->reg_base); 1226 err_iomap: 1227 idxd_free(idxd); 1228 err_idxd_alloc: 1229 pci_disable_device(pdev); 1230 return rc; 1231 } 1232 1233 static int idxd_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) 1234 { 1235 return idxd_pci_probe_alloc(NULL, pdev, id); 1236 } 1237 1238 void idxd_wqs_quiesce(struct idxd_device *idxd) 1239 { 1240 struct idxd_wq *wq; 1241 int i; 1242 1243 for (i = 0; i < idxd->max_wqs; i++) { 1244 wq = idxd->wqs[i]; 1245 if (wq->state == IDXD_WQ_ENABLED && wq->type == IDXD_WQT_KERNEL) 1246 idxd_wq_quiesce(wq); 1247 } 1248 } 1249 1250 static void idxd_shutdown(struct pci_dev *pdev) 1251 { 1252 struct idxd_device *idxd = pci_get_drvdata(pdev); 1253 struct idxd_irq_entry *irq_entry; 1254 int rc; 1255 1256 rc = idxd_device_disable(idxd); 1257 if (rc) 1258 dev_err(&pdev->dev, "Disabling device failed\n"); 1259 1260 irq_entry = &idxd->ie; 1261 synchronize_irq(irq_entry->vector); 1262 idxd_mask_error_interrupts(idxd); 1263 flush_workqueue(idxd->wq); 1264 } 1265 1266 static void idxd_remove(struct pci_dev *pdev) 1267 { 1268 struct idxd_device *idxd = pci_get_drvdata(pdev); 1269 1270 idxd_unregister_devices(idxd); 1271 /* 1272 * When ->release() is called for the idxd->conf_dev, it frees all the memory related 1273 * to the idxd context. The driver still needs those bits in order to do the rest of 1274 * the cleanup. However, we do need to unbound the idxd sub-driver. So take a ref 1275 * on the device here to hold off the freeing while allowing the idxd sub-driver 1276 * to unbind. 1277 */ 1278 get_device(idxd_confdev(idxd)); 1279 device_unregister(idxd_confdev(idxd)); 1280 idxd_shutdown(pdev); 1281 idxd_device_remove_debugfs(idxd); 1282 perfmon_pmu_remove(idxd); 1283 idxd_cleanup_interrupts(idxd); 1284 if (device_pasid_enabled(idxd)) 1285 idxd_disable_system_pasid(idxd); 1286 pci_iounmap(pdev, idxd->reg_base); 1287 put_device(idxd_confdev(idxd)); 1288 pci_disable_device(pdev); 1289 } 1290 1291 static struct pci_driver idxd_pci_driver = { 1292 .name = DRV_NAME, 1293 .id_table = idxd_pci_tbl, 1294 .probe = idxd_pci_probe, 1295 .remove = idxd_remove, 1296 .shutdown = idxd_shutdown, 1297 .err_handler = &idxd_error_handler, 1298 }; 1299 1300 static int __init idxd_init_module(void) 1301 { 1302 int err; 1303 1304 /* 1305 * If the CPU does not support MOVDIR64B or ENQCMDS, there's no point in 1306 * enumerating the device. We can not utilize it. 1307 */ 1308 if (!cpu_feature_enabled(X86_FEATURE_MOVDIR64B)) { 1309 pr_warn("idxd driver failed to load without MOVDIR64B.\n"); 1310 return -ENODEV; 1311 } 1312 1313 if (!cpu_feature_enabled(X86_FEATURE_ENQCMD)) 1314 pr_warn("Platform does not have ENQCMD(S) support.\n"); 1315 else 1316 support_enqcmd = true; 1317 1318 err = idxd_driver_register(&idxd_drv); 1319 if (err < 0) 1320 goto err_idxd_driver_register; 1321 1322 err = idxd_driver_register(&idxd_dmaengine_drv); 1323 if (err < 0) 1324 goto err_idxd_dmaengine_driver_register; 1325 1326 err = idxd_driver_register(&idxd_user_drv); 1327 if (err < 0) 1328 goto err_idxd_user_driver_register; 1329 1330 err = idxd_cdev_register(); 1331 if (err) 1332 goto err_cdev_register; 1333 1334 err = idxd_init_debugfs(); 1335 if (err) 1336 goto err_debugfs; 1337 1338 err = pci_register_driver(&idxd_pci_driver); 1339 if (err) 1340 goto err_pci_register; 1341 1342 return 0; 1343 1344 err_pci_register: 1345 idxd_remove_debugfs(); 1346 err_debugfs: 1347 idxd_cdev_remove(); 1348 err_cdev_register: 1349 idxd_driver_unregister(&idxd_user_drv); 1350 err_idxd_user_driver_register: 1351 idxd_driver_unregister(&idxd_dmaengine_drv); 1352 err_idxd_dmaengine_driver_register: 1353 idxd_driver_unregister(&idxd_drv); 1354 err_idxd_driver_register: 1355 return err; 1356 } 1357 module_init(idxd_init_module); 1358 1359 static void __exit idxd_exit_module(void) 1360 { 1361 idxd_driver_unregister(&idxd_user_drv); 1362 idxd_driver_unregister(&idxd_dmaengine_drv); 1363 idxd_driver_unregister(&idxd_drv); 1364 pci_unregister_driver(&idxd_pci_driver); 1365 idxd_cdev_remove(); 1366 idxd_remove_debugfs(); 1367 } 1368 module_exit(idxd_exit_module); 1369