1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright(c) 2019 Intel Corporation. All rights rsvd. */ 3 #include <linux/init.h> 4 #include <linux/kernel.h> 5 #include <linux/module.h> 6 #include <linux/slab.h> 7 #include <linux/pci.h> 8 #include <linux/interrupt.h> 9 #include <linux/delay.h> 10 #include <linux/dma-mapping.h> 11 #include <linux/workqueue.h> 12 #include <linux/fs.h> 13 #include <linux/io-64-nonatomic-lo-hi.h> 14 #include <linux/device.h> 15 #include <linux/idr.h> 16 #include <linux/iommu.h> 17 #include <uapi/linux/idxd.h> 18 #include <linux/dmaengine.h> 19 #include "../dmaengine.h" 20 #include "registers.h" 21 #include "idxd.h" 22 #include "perfmon.h" 23 24 MODULE_VERSION(IDXD_DRIVER_VERSION); 25 MODULE_DESCRIPTION("Intel Data Streaming Accelerator and In-Memory Analytics Accelerator common driver"); 26 MODULE_LICENSE("GPL v2"); 27 MODULE_AUTHOR("Intel Corporation"); 28 MODULE_IMPORT_NS("IDXD"); 29 30 static bool sva = true; 31 module_param(sva, bool, 0644); 32 MODULE_PARM_DESC(sva, "Toggle SVA support on/off"); 33 34 bool tc_override; 35 module_param(tc_override, bool, 0644); 36 MODULE_PARM_DESC(tc_override, "Override traffic class defaults"); 37 38 #define DRV_NAME "idxd" 39 40 bool support_enqcmd; 41 DEFINE_IDA(idxd_ida); 42 43 static struct idxd_driver_data idxd_driver_data[] = { 44 [IDXD_TYPE_DSA] = { 45 .name_prefix = "dsa", 46 .type = IDXD_TYPE_DSA, 47 .compl_size = sizeof(struct dsa_completion_record), 48 .align = 32, 49 .dev_type = &dsa_device_type, 50 .evl_cr_off = offsetof(struct dsa_evl_entry, cr), 51 .user_submission_safe = false, /* See INTEL-SA-01084 security advisory */ 52 .cr_status_off = offsetof(struct dsa_completion_record, status), 53 .cr_result_off = offsetof(struct dsa_completion_record, result), 54 }, 55 [IDXD_TYPE_IAX] = { 56 .name_prefix = "iax", 57 .type = IDXD_TYPE_IAX, 58 .compl_size = sizeof(struct iax_completion_record), 59 .align = 64, 60 .dev_type = &iax_device_type, 61 .evl_cr_off = offsetof(struct iax_evl_entry, cr), 62 .user_submission_safe = false, /* See INTEL-SA-01084 security advisory */ 63 .cr_status_off = offsetof(struct iax_completion_record, status), 64 .cr_result_off = offsetof(struct iax_completion_record, error_code), 65 .load_device_defaults = idxd_load_iaa_device_defaults, 66 }, 67 }; 68 69 static struct pci_device_id idxd_pci_tbl[] = { 70 /* DSA ver 1.0 platforms */ 71 { PCI_DEVICE_DATA(INTEL, DSA_SPR0, &idxd_driver_data[IDXD_TYPE_DSA]) }, 72 /* DSA on GNR-D platforms */ 73 { PCI_DEVICE_DATA(INTEL, DSA_GNRD, &idxd_driver_data[IDXD_TYPE_DSA]) }, 74 /* DSA on DMR platforms */ 75 { PCI_DEVICE_DATA(INTEL, DSA_DMR, &idxd_driver_data[IDXD_TYPE_DSA]) }, 76 77 /* IAX ver 1.0 platforms */ 78 { PCI_DEVICE_DATA(INTEL, IAX_SPR0, &idxd_driver_data[IDXD_TYPE_IAX]) }, 79 /* IAA on DMR platforms */ 80 { PCI_DEVICE_DATA(INTEL, IAA_DMR, &idxd_driver_data[IDXD_TYPE_IAX]) }, 81 /* IAA PTL platforms */ 82 { PCI_DEVICE_DATA(INTEL, IAA_PTL, &idxd_driver_data[IDXD_TYPE_IAX]) }, 83 { 0, } 84 }; 85 MODULE_DEVICE_TABLE(pci, idxd_pci_tbl); 86 87 static int idxd_setup_interrupts(struct idxd_device *idxd) 88 { 89 struct pci_dev *pdev = idxd->pdev; 90 struct device *dev = &pdev->dev; 91 struct idxd_irq_entry *ie; 92 int i, msixcnt; 93 int rc = 0; 94 95 msixcnt = pci_msix_vec_count(pdev); 96 if (msixcnt < 0) { 97 dev_err(dev, "Not MSI-X interrupt capable.\n"); 98 return -ENOSPC; 99 } 100 idxd->irq_cnt = msixcnt; 101 102 rc = pci_alloc_irq_vectors(pdev, msixcnt, msixcnt, PCI_IRQ_MSIX); 103 if (rc != msixcnt) { 104 dev_err(dev, "Failed enabling %d MSIX entries: %d\n", msixcnt, rc); 105 return -ENOSPC; 106 } 107 dev_dbg(dev, "Enabled %d msix vectors\n", msixcnt); 108 109 110 ie = idxd_get_ie(idxd, 0); 111 ie->vector = pci_irq_vector(pdev, 0); 112 rc = request_threaded_irq(ie->vector, NULL, idxd_misc_thread, 0, "idxd-misc", ie); 113 if (rc < 0) { 114 dev_err(dev, "Failed to allocate misc interrupt.\n"); 115 goto err_misc_irq; 116 } 117 dev_dbg(dev, "Requested idxd-misc handler on msix vector %d\n", ie->vector); 118 119 for (i = 0; i < idxd->max_wqs; i++) { 120 int msix_idx = i + 1; 121 122 ie = idxd_get_ie(idxd, msix_idx); 123 ie->id = msix_idx; 124 ie->int_handle = INVALID_INT_HANDLE; 125 ie->pasid = IOMMU_PASID_INVALID; 126 127 spin_lock_init(&ie->list_lock); 128 init_llist_head(&ie->pending_llist); 129 INIT_LIST_HEAD(&ie->work_list); 130 } 131 132 idxd_unmask_error_interrupts(idxd); 133 return 0; 134 135 err_misc_irq: 136 idxd_mask_error_interrupts(idxd); 137 pci_free_irq_vectors(pdev); 138 dev_err(dev, "No usable interrupts\n"); 139 return rc; 140 } 141 142 static void idxd_cleanup_interrupts(struct idxd_device *idxd) 143 { 144 struct pci_dev *pdev = idxd->pdev; 145 struct idxd_irq_entry *ie; 146 int msixcnt; 147 148 msixcnt = pci_msix_vec_count(pdev); 149 if (msixcnt <= 0) 150 return; 151 152 ie = idxd_get_ie(idxd, 0); 153 idxd_mask_error_interrupts(idxd); 154 free_irq(ie->vector, ie); 155 pci_free_irq_vectors(pdev); 156 } 157 158 static void idxd_clean_wqs(struct idxd_device *idxd) 159 { 160 struct idxd_wq *wq; 161 struct device *conf_dev; 162 int i; 163 164 for (i = 0; i < idxd->max_wqs; i++) { 165 wq = idxd->wqs[i]; 166 if (idxd->hw.wq_cap.op_config) 167 bitmap_free(wq->opcap_bmap); 168 kfree(wq->wqcfg); 169 conf_dev = wq_confdev(wq); 170 put_device(conf_dev); 171 kfree(wq); 172 } 173 bitmap_free(idxd->wq_enable_map); 174 kfree(idxd->wqs); 175 } 176 177 static int idxd_setup_wqs(struct idxd_device *idxd) 178 { 179 struct device *dev = &idxd->pdev->dev; 180 struct idxd_wq *wq; 181 struct device *conf_dev; 182 int i, rc; 183 184 idxd->wqs = kcalloc_node(idxd->max_wqs, sizeof(struct idxd_wq *), 185 GFP_KERNEL, dev_to_node(dev)); 186 if (!idxd->wqs) 187 return -ENOMEM; 188 189 idxd->wq_enable_map = bitmap_zalloc_node(idxd->max_wqs, GFP_KERNEL, dev_to_node(dev)); 190 if (!idxd->wq_enable_map) { 191 rc = -ENOMEM; 192 goto err_bitmap; 193 } 194 195 for (i = 0; i < idxd->max_wqs; i++) { 196 wq = kzalloc_node(sizeof(*wq), GFP_KERNEL, dev_to_node(dev)); 197 if (!wq) { 198 rc = -ENOMEM; 199 goto err; 200 } 201 202 idxd_dev_set_type(&wq->idxd_dev, IDXD_DEV_WQ); 203 conf_dev = wq_confdev(wq); 204 wq->id = i; 205 wq->idxd = idxd; 206 device_initialize(wq_confdev(wq)); 207 conf_dev->parent = idxd_confdev(idxd); 208 conf_dev->bus = &dsa_bus_type; 209 conf_dev->type = &idxd_wq_device_type; 210 rc = dev_set_name(conf_dev, "wq%d.%d", idxd->id, wq->id); 211 if (rc < 0) 212 goto err; 213 214 mutex_init(&wq->wq_lock); 215 init_waitqueue_head(&wq->err_queue); 216 init_completion(&wq->wq_dead); 217 init_completion(&wq->wq_resurrect); 218 wq->max_xfer_bytes = WQ_DEFAULT_MAX_XFER; 219 idxd_wq_set_max_batch_size(idxd->data->type, wq, WQ_DEFAULT_MAX_BATCH); 220 wq->enqcmds_retries = IDXD_ENQCMDS_RETRIES; 221 wq->wqcfg = kzalloc_node(idxd->wqcfg_size, GFP_KERNEL, dev_to_node(dev)); 222 if (!wq->wqcfg) { 223 rc = -ENOMEM; 224 goto err; 225 } 226 227 if (idxd->hw.wq_cap.op_config) { 228 wq->opcap_bmap = bitmap_zalloc(IDXD_MAX_OPCAP_BITS, GFP_KERNEL); 229 if (!wq->opcap_bmap) { 230 rc = -ENOMEM; 231 goto err_opcap_bmap; 232 } 233 bitmap_copy(wq->opcap_bmap, idxd->opcap_bmap, IDXD_MAX_OPCAP_BITS); 234 } 235 mutex_init(&wq->uc_lock); 236 xa_init(&wq->upasid_xa); 237 idxd->wqs[i] = wq; 238 } 239 240 return 0; 241 242 err_opcap_bmap: 243 kfree(wq->wqcfg); 244 245 err: 246 put_device(conf_dev); 247 kfree(wq); 248 249 while (--i >= 0) { 250 wq = idxd->wqs[i]; 251 if (idxd->hw.wq_cap.op_config) 252 bitmap_free(wq->opcap_bmap); 253 kfree(wq->wqcfg); 254 conf_dev = wq_confdev(wq); 255 put_device(conf_dev); 256 kfree(wq); 257 258 } 259 bitmap_free(idxd->wq_enable_map); 260 261 err_bitmap: 262 kfree(idxd->wqs); 263 264 return rc; 265 } 266 267 static void idxd_clean_engines(struct idxd_device *idxd) 268 { 269 struct idxd_engine *engine; 270 struct device *conf_dev; 271 int i; 272 273 for (i = 0; i < idxd->max_engines; i++) { 274 engine = idxd->engines[i]; 275 conf_dev = engine_confdev(engine); 276 put_device(conf_dev); 277 kfree(engine); 278 } 279 kfree(idxd->engines); 280 } 281 282 static int idxd_setup_engines(struct idxd_device *idxd) 283 { 284 struct idxd_engine *engine; 285 struct device *dev = &idxd->pdev->dev; 286 struct device *conf_dev; 287 int i, rc; 288 289 idxd->engines = kcalloc_node(idxd->max_engines, sizeof(struct idxd_engine *), 290 GFP_KERNEL, dev_to_node(dev)); 291 if (!idxd->engines) 292 return -ENOMEM; 293 294 for (i = 0; i < idxd->max_engines; i++) { 295 engine = kzalloc_node(sizeof(*engine), GFP_KERNEL, dev_to_node(dev)); 296 if (!engine) { 297 rc = -ENOMEM; 298 goto err; 299 } 300 301 idxd_dev_set_type(&engine->idxd_dev, IDXD_DEV_ENGINE); 302 conf_dev = engine_confdev(engine); 303 engine->id = i; 304 engine->idxd = idxd; 305 device_initialize(conf_dev); 306 conf_dev->parent = idxd_confdev(idxd); 307 conf_dev->bus = &dsa_bus_type; 308 conf_dev->type = &idxd_engine_device_type; 309 rc = dev_set_name(conf_dev, "engine%d.%d", idxd->id, engine->id); 310 if (rc < 0) { 311 put_device(conf_dev); 312 kfree(engine); 313 goto err; 314 } 315 316 idxd->engines[i] = engine; 317 } 318 319 return 0; 320 321 err: 322 while (--i >= 0) { 323 engine = idxd->engines[i]; 324 conf_dev = engine_confdev(engine); 325 put_device(conf_dev); 326 kfree(engine); 327 } 328 kfree(idxd->engines); 329 330 return rc; 331 } 332 333 static void idxd_clean_groups(struct idxd_device *idxd) 334 { 335 struct idxd_group *group; 336 int i; 337 338 for (i = 0; i < idxd->max_groups; i++) { 339 group = idxd->groups[i]; 340 put_device(group_confdev(group)); 341 kfree(group); 342 } 343 kfree(idxd->groups); 344 } 345 346 static int idxd_setup_groups(struct idxd_device *idxd) 347 { 348 struct device *dev = &idxd->pdev->dev; 349 struct device *conf_dev; 350 struct idxd_group *group; 351 int i, rc; 352 353 idxd->groups = kcalloc_node(idxd->max_groups, sizeof(struct idxd_group *), 354 GFP_KERNEL, dev_to_node(dev)); 355 if (!idxd->groups) 356 return -ENOMEM; 357 358 for (i = 0; i < idxd->max_groups; i++) { 359 group = kzalloc_node(sizeof(*group), GFP_KERNEL, dev_to_node(dev)); 360 if (!group) { 361 rc = -ENOMEM; 362 goto err; 363 } 364 365 idxd_dev_set_type(&group->idxd_dev, IDXD_DEV_GROUP); 366 conf_dev = group_confdev(group); 367 group->id = i; 368 group->idxd = idxd; 369 device_initialize(conf_dev); 370 conf_dev->parent = idxd_confdev(idxd); 371 conf_dev->bus = &dsa_bus_type; 372 conf_dev->type = &idxd_group_device_type; 373 rc = dev_set_name(conf_dev, "group%d.%d", idxd->id, group->id); 374 if (rc < 0) { 375 put_device(conf_dev); 376 kfree(group); 377 goto err; 378 } 379 380 idxd->groups[i] = group; 381 if (idxd->hw.version <= DEVICE_VERSION_2 && !tc_override) { 382 group->tc_a = 1; 383 group->tc_b = 1; 384 } else { 385 group->tc_a = -1; 386 group->tc_b = -1; 387 } 388 /* 389 * The default value is the same as the value of 390 * total read buffers in GRPCAP. 391 */ 392 group->rdbufs_allowed = idxd->max_rdbufs; 393 } 394 395 return 0; 396 397 err: 398 while (--i >= 0) { 399 group = idxd->groups[i]; 400 put_device(group_confdev(group)); 401 kfree(group); 402 } 403 kfree(idxd->groups); 404 405 return rc; 406 } 407 408 static void idxd_cleanup_internals(struct idxd_device *idxd) 409 { 410 idxd_clean_groups(idxd); 411 idxd_clean_engines(idxd); 412 idxd_clean_wqs(idxd); 413 destroy_workqueue(idxd->wq); 414 } 415 416 static int idxd_init_evl(struct idxd_device *idxd) 417 { 418 struct device *dev = &idxd->pdev->dev; 419 unsigned int evl_cache_size; 420 struct idxd_evl *evl; 421 const char *idxd_name; 422 423 if (idxd->hw.gen_cap.evl_support == 0) 424 return 0; 425 426 evl = kzalloc_node(sizeof(*evl), GFP_KERNEL, dev_to_node(dev)); 427 if (!evl) 428 return -ENOMEM; 429 430 mutex_init(&evl->lock); 431 evl->size = IDXD_EVL_SIZE_MIN; 432 433 idxd_name = dev_name(idxd_confdev(idxd)); 434 evl_cache_size = sizeof(struct idxd_evl_fault) + evl_ent_size(idxd); 435 /* 436 * Since completion record in evl_cache will be copied to user 437 * when handling completion record page fault, need to create 438 * the cache suitable for user copy. 439 */ 440 idxd->evl_cache = kmem_cache_create_usercopy(idxd_name, evl_cache_size, 441 0, 0, 0, evl_cache_size, 442 NULL); 443 if (!idxd->evl_cache) { 444 kfree(evl); 445 return -ENOMEM; 446 } 447 448 idxd->evl = evl; 449 return 0; 450 } 451 452 static int idxd_setup_internals(struct idxd_device *idxd) 453 { 454 struct device *dev = &idxd->pdev->dev; 455 int rc; 456 457 init_waitqueue_head(&idxd->cmd_waitq); 458 459 rc = idxd_setup_wqs(idxd); 460 if (rc < 0) 461 goto err_wqs; 462 463 rc = idxd_setup_engines(idxd); 464 if (rc < 0) 465 goto err_engine; 466 467 rc = idxd_setup_groups(idxd); 468 if (rc < 0) 469 goto err_group; 470 471 idxd->wq = create_workqueue(dev_name(dev)); 472 if (!idxd->wq) { 473 rc = -ENOMEM; 474 goto err_wkq_create; 475 } 476 477 rc = idxd_init_evl(idxd); 478 if (rc < 0) 479 goto err_evl; 480 481 return 0; 482 483 err_evl: 484 destroy_workqueue(idxd->wq); 485 err_wkq_create: 486 idxd_clean_groups(idxd); 487 err_group: 488 idxd_clean_engines(idxd); 489 err_engine: 490 idxd_clean_wqs(idxd); 491 err_wqs: 492 return rc; 493 } 494 495 static void idxd_read_table_offsets(struct idxd_device *idxd) 496 { 497 union offsets_reg offsets; 498 struct device *dev = &idxd->pdev->dev; 499 500 offsets.bits[0] = ioread64(idxd->reg_base + IDXD_TABLE_OFFSET); 501 offsets.bits[1] = ioread64(idxd->reg_base + IDXD_TABLE_OFFSET + sizeof(u64)); 502 idxd->grpcfg_offset = offsets.grpcfg * IDXD_TABLE_MULT; 503 dev_dbg(dev, "IDXD Group Config Offset: %#x\n", idxd->grpcfg_offset); 504 idxd->wqcfg_offset = offsets.wqcfg * IDXD_TABLE_MULT; 505 dev_dbg(dev, "IDXD Work Queue Config Offset: %#x\n", idxd->wqcfg_offset); 506 idxd->msix_perm_offset = offsets.msix_perm * IDXD_TABLE_MULT; 507 dev_dbg(dev, "IDXD MSIX Permission Offset: %#x\n", idxd->msix_perm_offset); 508 idxd->perfmon_offset = offsets.perfmon * IDXD_TABLE_MULT; 509 dev_dbg(dev, "IDXD Perfmon Offset: %#x\n", idxd->perfmon_offset); 510 } 511 512 void multi_u64_to_bmap(unsigned long *bmap, u64 *val, int count) 513 { 514 int i, j, nr; 515 516 for (i = 0, nr = 0; i < count; i++) { 517 for (j = 0; j < BITS_PER_LONG_LONG; j++) { 518 if (val[i] & BIT(j)) 519 set_bit(nr, bmap); 520 nr++; 521 } 522 } 523 } 524 525 static void idxd_read_caps(struct idxd_device *idxd) 526 { 527 struct device *dev = &idxd->pdev->dev; 528 int i; 529 530 /* reading generic capabilities */ 531 idxd->hw.gen_cap.bits = ioread64(idxd->reg_base + IDXD_GENCAP_OFFSET); 532 dev_dbg(dev, "gen_cap: %#llx\n", idxd->hw.gen_cap.bits); 533 534 if (idxd->hw.gen_cap.cmd_cap) { 535 idxd->hw.cmd_cap = ioread32(idxd->reg_base + IDXD_CMDCAP_OFFSET); 536 dev_dbg(dev, "cmd_cap: %#x\n", idxd->hw.cmd_cap); 537 } 538 539 /* reading command capabilities */ 540 if (idxd->hw.cmd_cap & BIT(IDXD_CMD_REQUEST_INT_HANDLE)) 541 idxd->request_int_handles = true; 542 543 idxd->max_xfer_bytes = 1ULL << idxd->hw.gen_cap.max_xfer_shift; 544 dev_dbg(dev, "max xfer size: %llu bytes\n", idxd->max_xfer_bytes); 545 idxd_set_max_batch_size(idxd->data->type, idxd, 1U << idxd->hw.gen_cap.max_batch_shift); 546 dev_dbg(dev, "max batch size: %u\n", idxd->max_batch_size); 547 if (idxd->hw.gen_cap.config_en) 548 set_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags); 549 550 /* reading group capabilities */ 551 idxd->hw.group_cap.bits = 552 ioread64(idxd->reg_base + IDXD_GRPCAP_OFFSET); 553 dev_dbg(dev, "group_cap: %#llx\n", idxd->hw.group_cap.bits); 554 idxd->max_groups = idxd->hw.group_cap.num_groups; 555 dev_dbg(dev, "max groups: %u\n", idxd->max_groups); 556 idxd->max_rdbufs = idxd->hw.group_cap.total_rdbufs; 557 dev_dbg(dev, "max read buffers: %u\n", idxd->max_rdbufs); 558 idxd->nr_rdbufs = idxd->max_rdbufs; 559 560 /* read engine capabilities */ 561 idxd->hw.engine_cap.bits = 562 ioread64(idxd->reg_base + IDXD_ENGCAP_OFFSET); 563 dev_dbg(dev, "engine_cap: %#llx\n", idxd->hw.engine_cap.bits); 564 idxd->max_engines = idxd->hw.engine_cap.num_engines; 565 dev_dbg(dev, "max engines: %u\n", idxd->max_engines); 566 567 /* read workqueue capabilities */ 568 idxd->hw.wq_cap.bits = ioread64(idxd->reg_base + IDXD_WQCAP_OFFSET); 569 dev_dbg(dev, "wq_cap: %#llx\n", idxd->hw.wq_cap.bits); 570 idxd->max_wq_size = idxd->hw.wq_cap.total_wq_size; 571 dev_dbg(dev, "total workqueue size: %u\n", idxd->max_wq_size); 572 idxd->max_wqs = idxd->hw.wq_cap.num_wqs; 573 dev_dbg(dev, "max workqueues: %u\n", idxd->max_wqs); 574 idxd->wqcfg_size = 1 << (idxd->hw.wq_cap.wqcfg_size + IDXD_WQCFG_MIN); 575 dev_dbg(dev, "wqcfg size: %u\n", idxd->wqcfg_size); 576 577 /* reading operation capabilities */ 578 for (i = 0; i < 4; i++) { 579 idxd->hw.opcap.bits[i] = ioread64(idxd->reg_base + 580 IDXD_OPCAP_OFFSET + i * sizeof(u64)); 581 dev_dbg(dev, "opcap[%d]: %#llx\n", i, idxd->hw.opcap.bits[i]); 582 } 583 multi_u64_to_bmap(idxd->opcap_bmap, &idxd->hw.opcap.bits[0], 4); 584 585 /* read iaa cap */ 586 if (idxd->data->type == IDXD_TYPE_IAX && idxd->hw.version >= DEVICE_VERSION_2) 587 idxd->hw.iaa_cap.bits = ioread64(idxd->reg_base + IDXD_IAACAP_OFFSET); 588 } 589 590 static void idxd_free(struct idxd_device *idxd) 591 { 592 if (!idxd) 593 return; 594 595 put_device(idxd_confdev(idxd)); 596 bitmap_free(idxd->opcap_bmap); 597 ida_free(&idxd_ida, idxd->id); 598 kfree(idxd); 599 } 600 601 static struct idxd_device *idxd_alloc(struct pci_dev *pdev, struct idxd_driver_data *data) 602 { 603 struct device *dev = &pdev->dev; 604 struct device *conf_dev; 605 struct idxd_device *idxd; 606 int rc; 607 608 idxd = kzalloc_node(sizeof(*idxd), GFP_KERNEL, dev_to_node(dev)); 609 if (!idxd) 610 return NULL; 611 612 conf_dev = idxd_confdev(idxd); 613 idxd->pdev = pdev; 614 idxd->data = data; 615 idxd_dev_set_type(&idxd->idxd_dev, idxd->data->type); 616 idxd->id = ida_alloc(&idxd_ida, GFP_KERNEL); 617 if (idxd->id < 0) 618 goto err_ida; 619 620 idxd->opcap_bmap = bitmap_zalloc_node(IDXD_MAX_OPCAP_BITS, GFP_KERNEL, dev_to_node(dev)); 621 if (!idxd->opcap_bmap) 622 goto err_opcap; 623 624 device_initialize(conf_dev); 625 conf_dev->parent = dev; 626 conf_dev->bus = &dsa_bus_type; 627 conf_dev->type = idxd->data->dev_type; 628 rc = dev_set_name(conf_dev, "%s%d", idxd->data->name_prefix, idxd->id); 629 if (rc < 0) 630 goto err_name; 631 632 spin_lock_init(&idxd->dev_lock); 633 spin_lock_init(&idxd->cmd_lock); 634 635 return idxd; 636 637 err_name: 638 put_device(conf_dev); 639 bitmap_free(idxd->opcap_bmap); 640 err_opcap: 641 ida_free(&idxd_ida, idxd->id); 642 err_ida: 643 kfree(idxd); 644 645 return NULL; 646 } 647 648 static int idxd_enable_system_pasid(struct idxd_device *idxd) 649 { 650 struct pci_dev *pdev = idxd->pdev; 651 struct device *dev = &pdev->dev; 652 struct iommu_domain *domain; 653 ioasid_t pasid; 654 int ret; 655 656 /* 657 * Attach a global PASID to the DMA domain so that we can use ENQCMDS 658 * to submit work on buffers mapped by DMA API. 659 */ 660 domain = iommu_get_domain_for_dev(dev); 661 if (!domain) 662 return -EPERM; 663 664 pasid = iommu_alloc_global_pasid(dev); 665 if (pasid == IOMMU_PASID_INVALID) 666 return -ENOSPC; 667 668 /* 669 * DMA domain is owned by the driver, it should support all valid 670 * types such as DMA-FQ, identity, etc. 671 */ 672 ret = iommu_attach_device_pasid(domain, dev, pasid, NULL); 673 if (ret) { 674 dev_err(dev, "failed to attach device pasid %d, domain type %d", 675 pasid, domain->type); 676 iommu_free_global_pasid(pasid); 677 return ret; 678 } 679 680 /* Since we set user privilege for kernel DMA, enable completion IRQ */ 681 idxd_set_user_intr(idxd, 1); 682 idxd->pasid = pasid; 683 684 return ret; 685 } 686 687 static void idxd_disable_system_pasid(struct idxd_device *idxd) 688 { 689 struct pci_dev *pdev = idxd->pdev; 690 struct device *dev = &pdev->dev; 691 struct iommu_domain *domain; 692 693 domain = iommu_get_domain_for_dev(dev); 694 if (!domain) 695 return; 696 697 iommu_detach_device_pasid(domain, dev, idxd->pasid); 698 iommu_free_global_pasid(idxd->pasid); 699 700 idxd_set_user_intr(idxd, 0); 701 idxd->sva = NULL; 702 idxd->pasid = IOMMU_PASID_INVALID; 703 } 704 705 static int idxd_enable_sva(struct pci_dev *pdev) 706 { 707 int ret; 708 709 ret = iommu_dev_enable_feature(&pdev->dev, IOMMU_DEV_FEAT_IOPF); 710 if (ret) 711 return ret; 712 713 ret = iommu_dev_enable_feature(&pdev->dev, IOMMU_DEV_FEAT_SVA); 714 if (ret) 715 iommu_dev_disable_feature(&pdev->dev, IOMMU_DEV_FEAT_IOPF); 716 717 return ret; 718 } 719 720 static void idxd_disable_sva(struct pci_dev *pdev) 721 { 722 iommu_dev_disable_feature(&pdev->dev, IOMMU_DEV_FEAT_SVA); 723 iommu_dev_disable_feature(&pdev->dev, IOMMU_DEV_FEAT_IOPF); 724 } 725 726 static int idxd_probe(struct idxd_device *idxd) 727 { 728 struct pci_dev *pdev = idxd->pdev; 729 struct device *dev = &pdev->dev; 730 int rc; 731 732 dev_dbg(dev, "%s entered and resetting device\n", __func__); 733 rc = idxd_device_init_reset(idxd); 734 if (rc < 0) 735 return rc; 736 737 dev_dbg(dev, "IDXD reset complete\n"); 738 739 if (IS_ENABLED(CONFIG_INTEL_IDXD_SVM) && sva) { 740 if (idxd_enable_sva(pdev)) { 741 dev_warn(dev, "Unable to turn on user SVA feature.\n"); 742 } else { 743 set_bit(IDXD_FLAG_USER_PASID_ENABLED, &idxd->flags); 744 745 rc = idxd_enable_system_pasid(idxd); 746 if (rc) 747 dev_warn(dev, "No in-kernel DMA with PASID. %d\n", rc); 748 else 749 set_bit(IDXD_FLAG_PASID_ENABLED, &idxd->flags); 750 } 751 } else if (!sva) { 752 dev_warn(dev, "User forced SVA off via module param.\n"); 753 } 754 755 idxd_read_caps(idxd); 756 idxd_read_table_offsets(idxd); 757 758 rc = idxd_setup_internals(idxd); 759 if (rc) 760 goto err; 761 762 /* If the configs are readonly, then load them from device */ 763 if (!test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags)) { 764 dev_dbg(dev, "Loading RO device config\n"); 765 rc = idxd_device_load_config(idxd); 766 if (rc < 0) 767 goto err_config; 768 } 769 770 rc = idxd_setup_interrupts(idxd); 771 if (rc) 772 goto err_config; 773 774 idxd->major = idxd_cdev_get_major(idxd); 775 776 rc = perfmon_pmu_init(idxd); 777 if (rc < 0) 778 dev_warn(dev, "Failed to initialize perfmon. No PMU support: %d\n", rc); 779 780 dev_dbg(dev, "IDXD device %d probed successfully\n", idxd->id); 781 return 0; 782 783 err_config: 784 idxd_cleanup_internals(idxd); 785 err: 786 if (device_pasid_enabled(idxd)) 787 idxd_disable_system_pasid(idxd); 788 if (device_user_pasid_enabled(idxd)) 789 idxd_disable_sva(pdev); 790 return rc; 791 } 792 793 static void idxd_cleanup(struct idxd_device *idxd) 794 { 795 perfmon_pmu_remove(idxd); 796 idxd_cleanup_interrupts(idxd); 797 idxd_cleanup_internals(idxd); 798 if (device_pasid_enabled(idxd)) 799 idxd_disable_system_pasid(idxd); 800 if (device_user_pasid_enabled(idxd)) 801 idxd_disable_sva(idxd->pdev); 802 } 803 804 /* 805 * Attach IDXD device to IDXD driver. 806 */ 807 static int idxd_bind(struct device_driver *drv, const char *buf) 808 { 809 const struct bus_type *bus = drv->bus; 810 struct device *dev; 811 int err = -ENODEV; 812 813 dev = bus_find_device_by_name(bus, NULL, buf); 814 if (dev) 815 err = device_driver_attach(drv, dev); 816 817 put_device(dev); 818 819 return err; 820 } 821 822 /* 823 * Detach IDXD device from driver. 824 */ 825 static void idxd_unbind(struct device_driver *drv, const char *buf) 826 { 827 const struct bus_type *bus = drv->bus; 828 struct device *dev; 829 830 dev = bus_find_device_by_name(bus, NULL, buf); 831 if (dev && dev->driver == drv) 832 device_release_driver(dev); 833 834 put_device(dev); 835 } 836 837 #define idxd_free_saved_configs(saved_configs, count) \ 838 do { \ 839 int i; \ 840 \ 841 for (i = 0; i < (count); i++) \ 842 kfree(saved_configs[i]); \ 843 } while (0) 844 845 static void idxd_free_saved(struct idxd_group **saved_groups, 846 struct idxd_engine **saved_engines, 847 struct idxd_wq **saved_wqs, 848 struct idxd_device *idxd) 849 { 850 if (saved_groups) 851 idxd_free_saved_configs(saved_groups, idxd->max_groups); 852 if (saved_engines) 853 idxd_free_saved_configs(saved_engines, idxd->max_engines); 854 if (saved_wqs) 855 idxd_free_saved_configs(saved_wqs, idxd->max_wqs); 856 } 857 858 /* 859 * Save IDXD device configurations including engines, groups, wqs etc. 860 * The saved configurations can be restored when needed. 861 */ 862 static int idxd_device_config_save(struct idxd_device *idxd, 863 struct idxd_saved_states *idxd_saved) 864 { 865 struct device *dev = &idxd->pdev->dev; 866 int i; 867 868 memcpy(&idxd_saved->saved_idxd, idxd, sizeof(*idxd)); 869 870 if (idxd->evl) { 871 memcpy(&idxd_saved->saved_evl, idxd->evl, 872 sizeof(struct idxd_evl)); 873 } 874 875 struct idxd_group **saved_groups __free(kfree) = 876 kcalloc_node(idxd->max_groups, 877 sizeof(struct idxd_group *), 878 GFP_KERNEL, dev_to_node(dev)); 879 if (!saved_groups) 880 return -ENOMEM; 881 882 for (i = 0; i < idxd->max_groups; i++) { 883 struct idxd_group *saved_group __free(kfree) = 884 kzalloc_node(sizeof(*saved_group), GFP_KERNEL, 885 dev_to_node(dev)); 886 887 if (!saved_group) { 888 /* Free saved groups */ 889 idxd_free_saved(saved_groups, NULL, NULL, idxd); 890 891 return -ENOMEM; 892 } 893 894 memcpy(saved_group, idxd->groups[i], sizeof(*saved_group)); 895 saved_groups[i] = no_free_ptr(saved_group); 896 } 897 898 struct idxd_engine **saved_engines = 899 kcalloc_node(idxd->max_engines, 900 sizeof(struct idxd_engine *), 901 GFP_KERNEL, dev_to_node(dev)); 902 if (!saved_engines) { 903 /* Free saved groups */ 904 idxd_free_saved(saved_groups, NULL, NULL, idxd); 905 906 return -ENOMEM; 907 } 908 for (i = 0; i < idxd->max_engines; i++) { 909 struct idxd_engine *saved_engine __free(kfree) = 910 kzalloc_node(sizeof(*saved_engine), GFP_KERNEL, 911 dev_to_node(dev)); 912 if (!saved_engine) { 913 /* Free saved groups and engines */ 914 idxd_free_saved(saved_groups, saved_engines, NULL, 915 idxd); 916 917 return -ENOMEM; 918 } 919 920 memcpy(saved_engine, idxd->engines[i], sizeof(*saved_engine)); 921 saved_engines[i] = no_free_ptr(saved_engine); 922 } 923 924 unsigned long *saved_wq_enable_map __free(bitmap) = 925 bitmap_zalloc_node(idxd->max_wqs, GFP_KERNEL, 926 dev_to_node(dev)); 927 if (!saved_wq_enable_map) { 928 /* Free saved groups and engines */ 929 idxd_free_saved(saved_groups, saved_engines, NULL, idxd); 930 931 return -ENOMEM; 932 } 933 934 bitmap_copy(saved_wq_enable_map, idxd->wq_enable_map, idxd->max_wqs); 935 936 struct idxd_wq **saved_wqs __free(kfree) = 937 kcalloc_node(idxd->max_wqs, sizeof(struct idxd_wq *), 938 GFP_KERNEL, dev_to_node(dev)); 939 if (!saved_wqs) { 940 /* Free saved groups and engines */ 941 idxd_free_saved(saved_groups, saved_engines, NULL, idxd); 942 943 return -ENOMEM; 944 } 945 946 for (i = 0; i < idxd->max_wqs; i++) { 947 struct idxd_wq *saved_wq __free(kfree) = 948 kzalloc_node(sizeof(*saved_wq), GFP_KERNEL, 949 dev_to_node(dev)); 950 struct idxd_wq *wq; 951 952 if (!saved_wq) { 953 /* Free saved groups, engines, and wqs */ 954 idxd_free_saved(saved_groups, saved_engines, saved_wqs, 955 idxd); 956 957 return -ENOMEM; 958 } 959 960 if (!test_bit(i, saved_wq_enable_map)) 961 continue; 962 963 wq = idxd->wqs[i]; 964 mutex_lock(&wq->wq_lock); 965 memcpy(saved_wq, wq, sizeof(*saved_wq)); 966 saved_wqs[i] = no_free_ptr(saved_wq); 967 mutex_unlock(&wq->wq_lock); 968 } 969 970 /* Save configurations */ 971 idxd_saved->saved_groups = no_free_ptr(saved_groups); 972 idxd_saved->saved_engines = no_free_ptr(saved_engines); 973 idxd_saved->saved_wq_enable_map = no_free_ptr(saved_wq_enable_map); 974 idxd_saved->saved_wqs = no_free_ptr(saved_wqs); 975 976 return 0; 977 } 978 979 /* 980 * Restore IDXD device configurations including engines, groups, wqs etc 981 * that were saved before. 982 */ 983 static void idxd_device_config_restore(struct idxd_device *idxd, 984 struct idxd_saved_states *idxd_saved) 985 { 986 struct idxd_evl *saved_evl = &idxd_saved->saved_evl; 987 int i; 988 989 idxd->rdbuf_limit = idxd_saved->saved_idxd.rdbuf_limit; 990 991 idxd->evl->size = saved_evl->size; 992 993 for (i = 0; i < idxd->max_groups; i++) { 994 struct idxd_group *saved_group, *group; 995 996 saved_group = idxd_saved->saved_groups[i]; 997 group = idxd->groups[i]; 998 999 group->rdbufs_allowed = saved_group->rdbufs_allowed; 1000 group->rdbufs_reserved = saved_group->rdbufs_reserved; 1001 group->tc_a = saved_group->tc_a; 1002 group->tc_b = saved_group->tc_b; 1003 group->use_rdbuf_limit = saved_group->use_rdbuf_limit; 1004 1005 kfree(saved_group); 1006 } 1007 kfree(idxd_saved->saved_groups); 1008 1009 for (i = 0; i < idxd->max_engines; i++) { 1010 struct idxd_engine *saved_engine, *engine; 1011 1012 saved_engine = idxd_saved->saved_engines[i]; 1013 engine = idxd->engines[i]; 1014 1015 engine->group = saved_engine->group; 1016 1017 kfree(saved_engine); 1018 } 1019 kfree(idxd_saved->saved_engines); 1020 1021 bitmap_copy(idxd->wq_enable_map, idxd_saved->saved_wq_enable_map, 1022 idxd->max_wqs); 1023 bitmap_free(idxd_saved->saved_wq_enable_map); 1024 1025 for (i = 0; i < idxd->max_wqs; i++) { 1026 struct idxd_wq *saved_wq, *wq; 1027 size_t len; 1028 1029 if (!test_bit(i, idxd->wq_enable_map)) 1030 continue; 1031 1032 saved_wq = idxd_saved->saved_wqs[i]; 1033 wq = idxd->wqs[i]; 1034 1035 mutex_lock(&wq->wq_lock); 1036 1037 wq->group = saved_wq->group; 1038 wq->flags = saved_wq->flags; 1039 wq->threshold = saved_wq->threshold; 1040 wq->size = saved_wq->size; 1041 wq->priority = saved_wq->priority; 1042 wq->type = saved_wq->type; 1043 len = strlen(saved_wq->name) + 1; 1044 strscpy(wq->name, saved_wq->name, len); 1045 wq->max_xfer_bytes = saved_wq->max_xfer_bytes; 1046 wq->max_batch_size = saved_wq->max_batch_size; 1047 wq->enqcmds_retries = saved_wq->enqcmds_retries; 1048 wq->descs = saved_wq->descs; 1049 wq->idxd_chan = saved_wq->idxd_chan; 1050 len = strlen(saved_wq->driver_name) + 1; 1051 strscpy(wq->driver_name, saved_wq->driver_name, len); 1052 1053 mutex_unlock(&wq->wq_lock); 1054 1055 kfree(saved_wq); 1056 } 1057 1058 kfree(idxd_saved->saved_wqs); 1059 } 1060 1061 static void idxd_reset_prepare(struct pci_dev *pdev) 1062 { 1063 struct idxd_device *idxd = pci_get_drvdata(pdev); 1064 struct device *dev = &idxd->pdev->dev; 1065 const char *idxd_name; 1066 int rc; 1067 1068 dev = &idxd->pdev->dev; 1069 idxd_name = dev_name(idxd_confdev(idxd)); 1070 1071 struct idxd_saved_states *idxd_saved __free(kfree) = 1072 kzalloc_node(sizeof(*idxd_saved), GFP_KERNEL, 1073 dev_to_node(&pdev->dev)); 1074 if (!idxd_saved) { 1075 dev_err(dev, "HALT: no memory\n"); 1076 1077 return; 1078 } 1079 1080 /* Save IDXD configurations. */ 1081 rc = idxd_device_config_save(idxd, idxd_saved); 1082 if (rc < 0) { 1083 dev_err(dev, "HALT: cannot save %s configs\n", idxd_name); 1084 1085 return; 1086 } 1087 1088 idxd->idxd_saved = no_free_ptr(idxd_saved); 1089 1090 /* Save PCI device state. */ 1091 pci_save_state(idxd->pdev); 1092 } 1093 1094 static void idxd_reset_done(struct pci_dev *pdev) 1095 { 1096 struct idxd_device *idxd = pci_get_drvdata(pdev); 1097 const char *idxd_name; 1098 struct device *dev; 1099 int rc, i; 1100 1101 if (!idxd->idxd_saved) 1102 return; 1103 1104 dev = &idxd->pdev->dev; 1105 idxd_name = dev_name(idxd_confdev(idxd)); 1106 1107 /* Restore PCI device state. */ 1108 pci_restore_state(idxd->pdev); 1109 1110 /* Unbind idxd device from driver. */ 1111 idxd_unbind(&idxd_drv.drv, idxd_name); 1112 1113 /* 1114 * Probe PCI device without allocating or changing 1115 * idxd software data which keeps the same as before FLR. 1116 */ 1117 idxd_pci_probe_alloc(idxd, NULL, NULL); 1118 1119 /* Restore IDXD configurations. */ 1120 idxd_device_config_restore(idxd, idxd->idxd_saved); 1121 1122 /* Re-configure IDXD device if allowed. */ 1123 if (test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags)) { 1124 rc = idxd_device_config(idxd); 1125 if (rc < 0) { 1126 dev_err(dev, "HALT: %s config fails\n", idxd_name); 1127 goto out; 1128 } 1129 } 1130 1131 /* Bind IDXD device to driver. */ 1132 rc = idxd_bind(&idxd_drv.drv, idxd_name); 1133 if (rc < 0) { 1134 dev_err(dev, "HALT: binding %s to driver fails\n", idxd_name); 1135 goto out; 1136 } 1137 1138 /* Bind enabled wq in the IDXD device to driver. */ 1139 for (i = 0; i < idxd->max_wqs; i++) { 1140 if (test_bit(i, idxd->wq_enable_map)) { 1141 struct idxd_wq *wq = idxd->wqs[i]; 1142 char wq_name[32]; 1143 1144 wq->state = IDXD_WQ_DISABLED; 1145 sprintf(wq_name, "wq%d.%d", idxd->id, wq->id); 1146 /* 1147 * Bind to user driver depending on wq type. 1148 * 1149 * Currently only support user type WQ. Will support 1150 * kernel type WQ in the future. 1151 */ 1152 if (wq->type == IDXD_WQT_USER) 1153 rc = idxd_bind(&idxd_user_drv.drv, wq_name); 1154 else 1155 rc = -EINVAL; 1156 if (rc < 0) { 1157 clear_bit(i, idxd->wq_enable_map); 1158 dev_err(dev, 1159 "HALT: unable to re-enable wq %s\n", 1160 dev_name(wq_confdev(wq))); 1161 } 1162 } 1163 } 1164 out: 1165 kfree(idxd->idxd_saved); 1166 } 1167 1168 static const struct pci_error_handlers idxd_error_handler = { 1169 .reset_prepare = idxd_reset_prepare, 1170 .reset_done = idxd_reset_done, 1171 }; 1172 1173 /* 1174 * Probe idxd PCI device. 1175 * If idxd is not given, need to allocate idxd and set up its data. 1176 * 1177 * If idxd is given, idxd was allocated and setup already. Just need to 1178 * configure device without re-allocating and re-configuring idxd data. 1179 * This is useful for recovering from FLR. 1180 */ 1181 int idxd_pci_probe_alloc(struct idxd_device *idxd, struct pci_dev *pdev, 1182 const struct pci_device_id *id) 1183 { 1184 bool alloc_idxd = idxd ? false : true; 1185 struct idxd_driver_data *data; 1186 struct device *dev; 1187 int rc; 1188 1189 pdev = idxd ? idxd->pdev : pdev; 1190 dev = &pdev->dev; 1191 data = id ? (struct idxd_driver_data *)id->driver_data : NULL; 1192 rc = pci_enable_device(pdev); 1193 if (rc) 1194 return rc; 1195 1196 if (alloc_idxd) { 1197 dev_dbg(dev, "Alloc IDXD context\n"); 1198 idxd = idxd_alloc(pdev, data); 1199 if (!idxd) { 1200 rc = -ENOMEM; 1201 goto err_idxd_alloc; 1202 } 1203 1204 dev_dbg(dev, "Mapping BARs\n"); 1205 idxd->reg_base = pci_iomap(pdev, IDXD_MMIO_BAR, 0); 1206 if (!idxd->reg_base) { 1207 rc = -ENOMEM; 1208 goto err_iomap; 1209 } 1210 1211 dev_dbg(dev, "Set DMA masks\n"); 1212 rc = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); 1213 if (rc) 1214 goto err; 1215 } 1216 1217 dev_dbg(dev, "Set PCI master\n"); 1218 pci_set_master(pdev); 1219 pci_set_drvdata(pdev, idxd); 1220 1221 if (alloc_idxd) { 1222 idxd->hw.version = ioread32(idxd->reg_base + IDXD_VER_OFFSET); 1223 rc = idxd_probe(idxd); 1224 if (rc) { 1225 dev_err(dev, "Intel(R) IDXD DMA Engine init failed\n"); 1226 goto err; 1227 } 1228 1229 if (data->load_device_defaults) { 1230 rc = data->load_device_defaults(idxd); 1231 if (rc) 1232 dev_warn(dev, "IDXD loading device defaults failed\n"); 1233 } 1234 1235 rc = idxd_register_devices(idxd); 1236 if (rc) { 1237 dev_err(dev, "IDXD sysfs setup failed\n"); 1238 goto err_dev_register; 1239 } 1240 1241 rc = idxd_device_init_debugfs(idxd); 1242 if (rc) 1243 dev_warn(dev, "IDXD debugfs failed to setup\n"); 1244 } 1245 1246 if (!alloc_idxd) { 1247 /* Release interrupts in the IDXD device. */ 1248 idxd_cleanup_interrupts(idxd); 1249 1250 /* Re-enable interrupts in the IDXD device. */ 1251 rc = idxd_setup_interrupts(idxd); 1252 if (rc) 1253 dev_warn(dev, "IDXD interrupts failed to setup\n"); 1254 } 1255 1256 dev_info(&pdev->dev, "Intel(R) Accelerator Device (v%x)\n", 1257 idxd->hw.version); 1258 1259 if (data) 1260 idxd->user_submission_safe = data->user_submission_safe; 1261 1262 return 0; 1263 1264 err_dev_register: 1265 idxd_cleanup(idxd); 1266 err: 1267 pci_iounmap(pdev, idxd->reg_base); 1268 err_iomap: 1269 idxd_free(idxd); 1270 err_idxd_alloc: 1271 pci_disable_device(pdev); 1272 return rc; 1273 } 1274 1275 static int idxd_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) 1276 { 1277 return idxd_pci_probe_alloc(NULL, pdev, id); 1278 } 1279 1280 void idxd_wqs_quiesce(struct idxd_device *idxd) 1281 { 1282 struct idxd_wq *wq; 1283 int i; 1284 1285 for (i = 0; i < idxd->max_wqs; i++) { 1286 wq = idxd->wqs[i]; 1287 if (wq->state == IDXD_WQ_ENABLED && wq->type == IDXD_WQT_KERNEL) 1288 idxd_wq_quiesce(wq); 1289 } 1290 } 1291 1292 static void idxd_shutdown(struct pci_dev *pdev) 1293 { 1294 struct idxd_device *idxd = pci_get_drvdata(pdev); 1295 struct idxd_irq_entry *irq_entry; 1296 int rc; 1297 1298 rc = idxd_device_disable(idxd); 1299 if (rc) 1300 dev_err(&pdev->dev, "Disabling device failed\n"); 1301 1302 irq_entry = &idxd->ie; 1303 synchronize_irq(irq_entry->vector); 1304 idxd_mask_error_interrupts(idxd); 1305 flush_workqueue(idxd->wq); 1306 } 1307 1308 static void idxd_remove(struct pci_dev *pdev) 1309 { 1310 struct idxd_device *idxd = pci_get_drvdata(pdev); 1311 1312 idxd_unregister_devices(idxd); 1313 /* 1314 * When ->release() is called for the idxd->conf_dev, it frees all the memory related 1315 * to the idxd context. The driver still needs those bits in order to do the rest of 1316 * the cleanup. However, we do need to unbound the idxd sub-driver. So take a ref 1317 * on the device here to hold off the freeing while allowing the idxd sub-driver 1318 * to unbind. 1319 */ 1320 get_device(idxd_confdev(idxd)); 1321 device_unregister(idxd_confdev(idxd)); 1322 idxd_shutdown(pdev); 1323 idxd_device_remove_debugfs(idxd); 1324 idxd_cleanup(idxd); 1325 pci_iounmap(pdev, idxd->reg_base); 1326 put_device(idxd_confdev(idxd)); 1327 idxd_free(idxd); 1328 pci_disable_device(pdev); 1329 } 1330 1331 static struct pci_driver idxd_pci_driver = { 1332 .name = DRV_NAME, 1333 .id_table = idxd_pci_tbl, 1334 .probe = idxd_pci_probe, 1335 .remove = idxd_remove, 1336 .shutdown = idxd_shutdown, 1337 .err_handler = &idxd_error_handler, 1338 }; 1339 1340 static int __init idxd_init_module(void) 1341 { 1342 int err; 1343 1344 /* 1345 * If the CPU does not support MOVDIR64B or ENQCMDS, there's no point in 1346 * enumerating the device. We can not utilize it. 1347 */ 1348 if (!cpu_feature_enabled(X86_FEATURE_MOVDIR64B)) { 1349 pr_warn("idxd driver failed to load without MOVDIR64B.\n"); 1350 return -ENODEV; 1351 } 1352 1353 if (!cpu_feature_enabled(X86_FEATURE_ENQCMD)) 1354 pr_warn("Platform does not have ENQCMD(S) support.\n"); 1355 else 1356 support_enqcmd = true; 1357 1358 err = idxd_driver_register(&idxd_drv); 1359 if (err < 0) 1360 goto err_idxd_driver_register; 1361 1362 err = idxd_driver_register(&idxd_dmaengine_drv); 1363 if (err < 0) 1364 goto err_idxd_dmaengine_driver_register; 1365 1366 err = idxd_driver_register(&idxd_user_drv); 1367 if (err < 0) 1368 goto err_idxd_user_driver_register; 1369 1370 err = idxd_cdev_register(); 1371 if (err) 1372 goto err_cdev_register; 1373 1374 err = idxd_init_debugfs(); 1375 if (err) 1376 goto err_debugfs; 1377 1378 err = pci_register_driver(&idxd_pci_driver); 1379 if (err) 1380 goto err_pci_register; 1381 1382 return 0; 1383 1384 err_pci_register: 1385 idxd_remove_debugfs(); 1386 err_debugfs: 1387 idxd_cdev_remove(); 1388 err_cdev_register: 1389 idxd_driver_unregister(&idxd_user_drv); 1390 err_idxd_user_driver_register: 1391 idxd_driver_unregister(&idxd_dmaengine_drv); 1392 err_idxd_dmaengine_driver_register: 1393 idxd_driver_unregister(&idxd_drv); 1394 err_idxd_driver_register: 1395 return err; 1396 } 1397 module_init(idxd_init_module); 1398 1399 static void __exit idxd_exit_module(void) 1400 { 1401 idxd_driver_unregister(&idxd_user_drv); 1402 idxd_driver_unregister(&idxd_dmaengine_drv); 1403 idxd_driver_unregister(&idxd_drv); 1404 pci_unregister_driver(&idxd_pci_driver); 1405 idxd_cdev_remove(); 1406 idxd_remove_debugfs(); 1407 } 1408 module_exit(idxd_exit_module); 1409