1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright(c) 2019 Intel Corporation. All rights rsvd. */ 3 #include <linux/init.h> 4 #include <linux/kernel.h> 5 #include <linux/module.h> 6 #include <linux/slab.h> 7 #include <linux/pci.h> 8 #include <linux/interrupt.h> 9 #include <linux/delay.h> 10 #include <linux/dma-mapping.h> 11 #include <linux/workqueue.h> 12 #include <linux/fs.h> 13 #include <linux/io-64-nonatomic-lo-hi.h> 14 #include <linux/device.h> 15 #include <linux/idr.h> 16 #include <linux/iommu.h> 17 #include <uapi/linux/idxd.h> 18 #include <linux/dmaengine.h> 19 #include "../dmaengine.h" 20 #include "registers.h" 21 #include "idxd.h" 22 #include "perfmon.h" 23 24 MODULE_VERSION(IDXD_DRIVER_VERSION); 25 MODULE_LICENSE("GPL v2"); 26 MODULE_AUTHOR("Intel Corporation"); 27 MODULE_IMPORT_NS(IDXD); 28 29 static bool sva = true; 30 module_param(sva, bool, 0644); 31 MODULE_PARM_DESC(sva, "Toggle SVA support on/off"); 32 33 bool tc_override; 34 module_param(tc_override, bool, 0644); 35 MODULE_PARM_DESC(tc_override, "Override traffic class defaults"); 36 37 #define DRV_NAME "idxd" 38 39 bool support_enqcmd; 40 DEFINE_IDA(idxd_ida); 41 42 static struct idxd_driver_data idxd_driver_data[] = { 43 [IDXD_TYPE_DSA] = { 44 .name_prefix = "dsa", 45 .type = IDXD_TYPE_DSA, 46 .compl_size = sizeof(struct dsa_completion_record), 47 .align = 32, 48 .dev_type = &dsa_device_type, 49 .evl_cr_off = offsetof(struct dsa_evl_entry, cr), 50 .user_submission_safe = false, /* See INTEL-SA-01084 security advisory */ 51 .cr_status_off = offsetof(struct dsa_completion_record, status), 52 .cr_result_off = offsetof(struct dsa_completion_record, result), 53 }, 54 [IDXD_TYPE_IAX] = { 55 .name_prefix = "iax", 56 .type = IDXD_TYPE_IAX, 57 .compl_size = sizeof(struct iax_completion_record), 58 .align = 64, 59 .dev_type = &iax_device_type, 60 .evl_cr_off = offsetof(struct iax_evl_entry, cr), 61 .user_submission_safe = false, /* See INTEL-SA-01084 security advisory */ 62 .cr_status_off = offsetof(struct iax_completion_record, status), 63 .cr_result_off = offsetof(struct iax_completion_record, error_code), 64 .load_device_defaults = idxd_load_iaa_device_defaults, 65 }, 66 }; 67 68 static struct pci_device_id idxd_pci_tbl[] = { 69 /* DSA ver 1.0 platforms */ 70 { PCI_DEVICE_DATA(INTEL, DSA_SPR0, &idxd_driver_data[IDXD_TYPE_DSA]) }, 71 72 /* IAX ver 1.0 platforms */ 73 { PCI_DEVICE_DATA(INTEL, IAX_SPR0, &idxd_driver_data[IDXD_TYPE_IAX]) }, 74 { 0, } 75 }; 76 MODULE_DEVICE_TABLE(pci, idxd_pci_tbl); 77 78 static int idxd_setup_interrupts(struct idxd_device *idxd) 79 { 80 struct pci_dev *pdev = idxd->pdev; 81 struct device *dev = &pdev->dev; 82 struct idxd_irq_entry *ie; 83 int i, msixcnt; 84 int rc = 0; 85 86 msixcnt = pci_msix_vec_count(pdev); 87 if (msixcnt < 0) { 88 dev_err(dev, "Not MSI-X interrupt capable.\n"); 89 return -ENOSPC; 90 } 91 idxd->irq_cnt = msixcnt; 92 93 rc = pci_alloc_irq_vectors(pdev, msixcnt, msixcnt, PCI_IRQ_MSIX); 94 if (rc != msixcnt) { 95 dev_err(dev, "Failed enabling %d MSIX entries: %d\n", msixcnt, rc); 96 return -ENOSPC; 97 } 98 dev_dbg(dev, "Enabled %d msix vectors\n", msixcnt); 99 100 101 ie = idxd_get_ie(idxd, 0); 102 ie->vector = pci_irq_vector(pdev, 0); 103 rc = request_threaded_irq(ie->vector, NULL, idxd_misc_thread, 0, "idxd-misc", ie); 104 if (rc < 0) { 105 dev_err(dev, "Failed to allocate misc interrupt.\n"); 106 goto err_misc_irq; 107 } 108 dev_dbg(dev, "Requested idxd-misc handler on msix vector %d\n", ie->vector); 109 110 for (i = 0; i < idxd->max_wqs; i++) { 111 int msix_idx = i + 1; 112 113 ie = idxd_get_ie(idxd, msix_idx); 114 ie->id = msix_idx; 115 ie->int_handle = INVALID_INT_HANDLE; 116 ie->pasid = IOMMU_PASID_INVALID; 117 118 spin_lock_init(&ie->list_lock); 119 init_llist_head(&ie->pending_llist); 120 INIT_LIST_HEAD(&ie->work_list); 121 } 122 123 idxd_unmask_error_interrupts(idxd); 124 return 0; 125 126 err_misc_irq: 127 idxd_mask_error_interrupts(idxd); 128 pci_free_irq_vectors(pdev); 129 dev_err(dev, "No usable interrupts\n"); 130 return rc; 131 } 132 133 static void idxd_cleanup_interrupts(struct idxd_device *idxd) 134 { 135 struct pci_dev *pdev = idxd->pdev; 136 struct idxd_irq_entry *ie; 137 int msixcnt; 138 139 msixcnt = pci_msix_vec_count(pdev); 140 if (msixcnt <= 0) 141 return; 142 143 ie = idxd_get_ie(idxd, 0); 144 idxd_mask_error_interrupts(idxd); 145 free_irq(ie->vector, ie); 146 pci_free_irq_vectors(pdev); 147 } 148 149 static int idxd_setup_wqs(struct idxd_device *idxd) 150 { 151 struct device *dev = &idxd->pdev->dev; 152 struct idxd_wq *wq; 153 struct device *conf_dev; 154 int i, rc; 155 156 idxd->wqs = kcalloc_node(idxd->max_wqs, sizeof(struct idxd_wq *), 157 GFP_KERNEL, dev_to_node(dev)); 158 if (!idxd->wqs) 159 return -ENOMEM; 160 161 idxd->wq_enable_map = bitmap_zalloc_node(idxd->max_wqs, GFP_KERNEL, dev_to_node(dev)); 162 if (!idxd->wq_enable_map) { 163 kfree(idxd->wqs); 164 return -ENOMEM; 165 } 166 167 for (i = 0; i < idxd->max_wqs; i++) { 168 wq = kzalloc_node(sizeof(*wq), GFP_KERNEL, dev_to_node(dev)); 169 if (!wq) { 170 rc = -ENOMEM; 171 goto err; 172 } 173 174 idxd_dev_set_type(&wq->idxd_dev, IDXD_DEV_WQ); 175 conf_dev = wq_confdev(wq); 176 wq->id = i; 177 wq->idxd = idxd; 178 device_initialize(wq_confdev(wq)); 179 conf_dev->parent = idxd_confdev(idxd); 180 conf_dev->bus = &dsa_bus_type; 181 conf_dev->type = &idxd_wq_device_type; 182 rc = dev_set_name(conf_dev, "wq%d.%d", idxd->id, wq->id); 183 if (rc < 0) { 184 put_device(conf_dev); 185 goto err; 186 } 187 188 mutex_init(&wq->wq_lock); 189 init_waitqueue_head(&wq->err_queue); 190 init_completion(&wq->wq_dead); 191 init_completion(&wq->wq_resurrect); 192 wq->max_xfer_bytes = WQ_DEFAULT_MAX_XFER; 193 idxd_wq_set_max_batch_size(idxd->data->type, wq, WQ_DEFAULT_MAX_BATCH); 194 wq->enqcmds_retries = IDXD_ENQCMDS_RETRIES; 195 wq->wqcfg = kzalloc_node(idxd->wqcfg_size, GFP_KERNEL, dev_to_node(dev)); 196 if (!wq->wqcfg) { 197 put_device(conf_dev); 198 rc = -ENOMEM; 199 goto err; 200 } 201 202 if (idxd->hw.wq_cap.op_config) { 203 wq->opcap_bmap = bitmap_zalloc(IDXD_MAX_OPCAP_BITS, GFP_KERNEL); 204 if (!wq->opcap_bmap) { 205 put_device(conf_dev); 206 rc = -ENOMEM; 207 goto err; 208 } 209 bitmap_copy(wq->opcap_bmap, idxd->opcap_bmap, IDXD_MAX_OPCAP_BITS); 210 } 211 mutex_init(&wq->uc_lock); 212 xa_init(&wq->upasid_xa); 213 idxd->wqs[i] = wq; 214 } 215 216 return 0; 217 218 err: 219 while (--i >= 0) { 220 wq = idxd->wqs[i]; 221 conf_dev = wq_confdev(wq); 222 put_device(conf_dev); 223 } 224 return rc; 225 } 226 227 static int idxd_setup_engines(struct idxd_device *idxd) 228 { 229 struct idxd_engine *engine; 230 struct device *dev = &idxd->pdev->dev; 231 struct device *conf_dev; 232 int i, rc; 233 234 idxd->engines = kcalloc_node(idxd->max_engines, sizeof(struct idxd_engine *), 235 GFP_KERNEL, dev_to_node(dev)); 236 if (!idxd->engines) 237 return -ENOMEM; 238 239 for (i = 0; i < idxd->max_engines; i++) { 240 engine = kzalloc_node(sizeof(*engine), GFP_KERNEL, dev_to_node(dev)); 241 if (!engine) { 242 rc = -ENOMEM; 243 goto err; 244 } 245 246 idxd_dev_set_type(&engine->idxd_dev, IDXD_DEV_ENGINE); 247 conf_dev = engine_confdev(engine); 248 engine->id = i; 249 engine->idxd = idxd; 250 device_initialize(conf_dev); 251 conf_dev->parent = idxd_confdev(idxd); 252 conf_dev->bus = &dsa_bus_type; 253 conf_dev->type = &idxd_engine_device_type; 254 rc = dev_set_name(conf_dev, "engine%d.%d", idxd->id, engine->id); 255 if (rc < 0) { 256 put_device(conf_dev); 257 goto err; 258 } 259 260 idxd->engines[i] = engine; 261 } 262 263 return 0; 264 265 err: 266 while (--i >= 0) { 267 engine = idxd->engines[i]; 268 conf_dev = engine_confdev(engine); 269 put_device(conf_dev); 270 } 271 return rc; 272 } 273 274 static int idxd_setup_groups(struct idxd_device *idxd) 275 { 276 struct device *dev = &idxd->pdev->dev; 277 struct device *conf_dev; 278 struct idxd_group *group; 279 int i, rc; 280 281 idxd->groups = kcalloc_node(idxd->max_groups, sizeof(struct idxd_group *), 282 GFP_KERNEL, dev_to_node(dev)); 283 if (!idxd->groups) 284 return -ENOMEM; 285 286 for (i = 0; i < idxd->max_groups; i++) { 287 group = kzalloc_node(sizeof(*group), GFP_KERNEL, dev_to_node(dev)); 288 if (!group) { 289 rc = -ENOMEM; 290 goto err; 291 } 292 293 idxd_dev_set_type(&group->idxd_dev, IDXD_DEV_GROUP); 294 conf_dev = group_confdev(group); 295 group->id = i; 296 group->idxd = idxd; 297 device_initialize(conf_dev); 298 conf_dev->parent = idxd_confdev(idxd); 299 conf_dev->bus = &dsa_bus_type; 300 conf_dev->type = &idxd_group_device_type; 301 rc = dev_set_name(conf_dev, "group%d.%d", idxd->id, group->id); 302 if (rc < 0) { 303 put_device(conf_dev); 304 goto err; 305 } 306 307 idxd->groups[i] = group; 308 if (idxd->hw.version <= DEVICE_VERSION_2 && !tc_override) { 309 group->tc_a = 1; 310 group->tc_b = 1; 311 } else { 312 group->tc_a = -1; 313 group->tc_b = -1; 314 } 315 /* 316 * The default value is the same as the value of 317 * total read buffers in GRPCAP. 318 */ 319 group->rdbufs_allowed = idxd->max_rdbufs; 320 } 321 322 return 0; 323 324 err: 325 while (--i >= 0) { 326 group = idxd->groups[i]; 327 put_device(group_confdev(group)); 328 } 329 return rc; 330 } 331 332 static void idxd_cleanup_internals(struct idxd_device *idxd) 333 { 334 int i; 335 336 for (i = 0; i < idxd->max_groups; i++) 337 put_device(group_confdev(idxd->groups[i])); 338 for (i = 0; i < idxd->max_engines; i++) 339 put_device(engine_confdev(idxd->engines[i])); 340 for (i = 0; i < idxd->max_wqs; i++) 341 put_device(wq_confdev(idxd->wqs[i])); 342 destroy_workqueue(idxd->wq); 343 } 344 345 static int idxd_init_evl(struct idxd_device *idxd) 346 { 347 struct device *dev = &idxd->pdev->dev; 348 unsigned int evl_cache_size; 349 struct idxd_evl *evl; 350 const char *idxd_name; 351 352 if (idxd->hw.gen_cap.evl_support == 0) 353 return 0; 354 355 evl = kzalloc_node(sizeof(*evl), GFP_KERNEL, dev_to_node(dev)); 356 if (!evl) 357 return -ENOMEM; 358 359 mutex_init(&evl->lock); 360 evl->size = IDXD_EVL_SIZE_MIN; 361 362 idxd_name = dev_name(idxd_confdev(idxd)); 363 evl_cache_size = sizeof(struct idxd_evl_fault) + evl_ent_size(idxd); 364 /* 365 * Since completion record in evl_cache will be copied to user 366 * when handling completion record page fault, need to create 367 * the cache suitable for user copy. 368 */ 369 idxd->evl_cache = kmem_cache_create_usercopy(idxd_name, evl_cache_size, 370 0, 0, 0, evl_cache_size, 371 NULL); 372 if (!idxd->evl_cache) { 373 kfree(evl); 374 return -ENOMEM; 375 } 376 377 idxd->evl = evl; 378 return 0; 379 } 380 381 static int idxd_setup_internals(struct idxd_device *idxd) 382 { 383 struct device *dev = &idxd->pdev->dev; 384 int rc, i; 385 386 init_waitqueue_head(&idxd->cmd_waitq); 387 388 rc = idxd_setup_wqs(idxd); 389 if (rc < 0) 390 goto err_wqs; 391 392 rc = idxd_setup_engines(idxd); 393 if (rc < 0) 394 goto err_engine; 395 396 rc = idxd_setup_groups(idxd); 397 if (rc < 0) 398 goto err_group; 399 400 idxd->wq = create_workqueue(dev_name(dev)); 401 if (!idxd->wq) { 402 rc = -ENOMEM; 403 goto err_wkq_create; 404 } 405 406 rc = idxd_init_evl(idxd); 407 if (rc < 0) 408 goto err_evl; 409 410 return 0; 411 412 err_evl: 413 destroy_workqueue(idxd->wq); 414 err_wkq_create: 415 for (i = 0; i < idxd->max_groups; i++) 416 put_device(group_confdev(idxd->groups[i])); 417 err_group: 418 for (i = 0; i < idxd->max_engines; i++) 419 put_device(engine_confdev(idxd->engines[i])); 420 err_engine: 421 for (i = 0; i < idxd->max_wqs; i++) 422 put_device(wq_confdev(idxd->wqs[i])); 423 err_wqs: 424 return rc; 425 } 426 427 static void idxd_read_table_offsets(struct idxd_device *idxd) 428 { 429 union offsets_reg offsets; 430 struct device *dev = &idxd->pdev->dev; 431 432 offsets.bits[0] = ioread64(idxd->reg_base + IDXD_TABLE_OFFSET); 433 offsets.bits[1] = ioread64(idxd->reg_base + IDXD_TABLE_OFFSET + sizeof(u64)); 434 idxd->grpcfg_offset = offsets.grpcfg * IDXD_TABLE_MULT; 435 dev_dbg(dev, "IDXD Group Config Offset: %#x\n", idxd->grpcfg_offset); 436 idxd->wqcfg_offset = offsets.wqcfg * IDXD_TABLE_MULT; 437 dev_dbg(dev, "IDXD Work Queue Config Offset: %#x\n", idxd->wqcfg_offset); 438 idxd->msix_perm_offset = offsets.msix_perm * IDXD_TABLE_MULT; 439 dev_dbg(dev, "IDXD MSIX Permission Offset: %#x\n", idxd->msix_perm_offset); 440 idxd->perfmon_offset = offsets.perfmon * IDXD_TABLE_MULT; 441 dev_dbg(dev, "IDXD Perfmon Offset: %#x\n", idxd->perfmon_offset); 442 } 443 444 void multi_u64_to_bmap(unsigned long *bmap, u64 *val, int count) 445 { 446 int i, j, nr; 447 448 for (i = 0, nr = 0; i < count; i++) { 449 for (j = 0; j < BITS_PER_LONG_LONG; j++) { 450 if (val[i] & BIT(j)) 451 set_bit(nr, bmap); 452 nr++; 453 } 454 } 455 } 456 457 static void idxd_read_caps(struct idxd_device *idxd) 458 { 459 struct device *dev = &idxd->pdev->dev; 460 int i; 461 462 /* reading generic capabilities */ 463 idxd->hw.gen_cap.bits = ioread64(idxd->reg_base + IDXD_GENCAP_OFFSET); 464 dev_dbg(dev, "gen_cap: %#llx\n", idxd->hw.gen_cap.bits); 465 466 if (idxd->hw.gen_cap.cmd_cap) { 467 idxd->hw.cmd_cap = ioread32(idxd->reg_base + IDXD_CMDCAP_OFFSET); 468 dev_dbg(dev, "cmd_cap: %#x\n", idxd->hw.cmd_cap); 469 } 470 471 /* reading command capabilities */ 472 if (idxd->hw.cmd_cap & BIT(IDXD_CMD_REQUEST_INT_HANDLE)) 473 idxd->request_int_handles = true; 474 475 idxd->max_xfer_bytes = 1ULL << idxd->hw.gen_cap.max_xfer_shift; 476 dev_dbg(dev, "max xfer size: %llu bytes\n", idxd->max_xfer_bytes); 477 idxd_set_max_batch_size(idxd->data->type, idxd, 1U << idxd->hw.gen_cap.max_batch_shift); 478 dev_dbg(dev, "max batch size: %u\n", idxd->max_batch_size); 479 if (idxd->hw.gen_cap.config_en) 480 set_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags); 481 482 /* reading group capabilities */ 483 idxd->hw.group_cap.bits = 484 ioread64(idxd->reg_base + IDXD_GRPCAP_OFFSET); 485 dev_dbg(dev, "group_cap: %#llx\n", idxd->hw.group_cap.bits); 486 idxd->max_groups = idxd->hw.group_cap.num_groups; 487 dev_dbg(dev, "max groups: %u\n", idxd->max_groups); 488 idxd->max_rdbufs = idxd->hw.group_cap.total_rdbufs; 489 dev_dbg(dev, "max read buffers: %u\n", idxd->max_rdbufs); 490 idxd->nr_rdbufs = idxd->max_rdbufs; 491 492 /* read engine capabilities */ 493 idxd->hw.engine_cap.bits = 494 ioread64(idxd->reg_base + IDXD_ENGCAP_OFFSET); 495 dev_dbg(dev, "engine_cap: %#llx\n", idxd->hw.engine_cap.bits); 496 idxd->max_engines = idxd->hw.engine_cap.num_engines; 497 dev_dbg(dev, "max engines: %u\n", idxd->max_engines); 498 499 /* read workqueue capabilities */ 500 idxd->hw.wq_cap.bits = ioread64(idxd->reg_base + IDXD_WQCAP_OFFSET); 501 dev_dbg(dev, "wq_cap: %#llx\n", idxd->hw.wq_cap.bits); 502 idxd->max_wq_size = idxd->hw.wq_cap.total_wq_size; 503 dev_dbg(dev, "total workqueue size: %u\n", idxd->max_wq_size); 504 idxd->max_wqs = idxd->hw.wq_cap.num_wqs; 505 dev_dbg(dev, "max workqueues: %u\n", idxd->max_wqs); 506 idxd->wqcfg_size = 1 << (idxd->hw.wq_cap.wqcfg_size + IDXD_WQCFG_MIN); 507 dev_dbg(dev, "wqcfg size: %u\n", idxd->wqcfg_size); 508 509 /* reading operation capabilities */ 510 for (i = 0; i < 4; i++) { 511 idxd->hw.opcap.bits[i] = ioread64(idxd->reg_base + 512 IDXD_OPCAP_OFFSET + i * sizeof(u64)); 513 dev_dbg(dev, "opcap[%d]: %#llx\n", i, idxd->hw.opcap.bits[i]); 514 } 515 multi_u64_to_bmap(idxd->opcap_bmap, &idxd->hw.opcap.bits[0], 4); 516 517 /* read iaa cap */ 518 if (idxd->data->type == IDXD_TYPE_IAX && idxd->hw.version >= DEVICE_VERSION_2) 519 idxd->hw.iaa_cap.bits = ioread64(idxd->reg_base + IDXD_IAACAP_OFFSET); 520 } 521 522 static struct idxd_device *idxd_alloc(struct pci_dev *pdev, struct idxd_driver_data *data) 523 { 524 struct device *dev = &pdev->dev; 525 struct device *conf_dev; 526 struct idxd_device *idxd; 527 int rc; 528 529 idxd = kzalloc_node(sizeof(*idxd), GFP_KERNEL, dev_to_node(dev)); 530 if (!idxd) 531 return NULL; 532 533 conf_dev = idxd_confdev(idxd); 534 idxd->pdev = pdev; 535 idxd->data = data; 536 idxd_dev_set_type(&idxd->idxd_dev, idxd->data->type); 537 idxd->id = ida_alloc(&idxd_ida, GFP_KERNEL); 538 if (idxd->id < 0) 539 return NULL; 540 541 idxd->opcap_bmap = bitmap_zalloc_node(IDXD_MAX_OPCAP_BITS, GFP_KERNEL, dev_to_node(dev)); 542 if (!idxd->opcap_bmap) { 543 ida_free(&idxd_ida, idxd->id); 544 return NULL; 545 } 546 547 device_initialize(conf_dev); 548 conf_dev->parent = dev; 549 conf_dev->bus = &dsa_bus_type; 550 conf_dev->type = idxd->data->dev_type; 551 rc = dev_set_name(conf_dev, "%s%d", idxd->data->name_prefix, idxd->id); 552 if (rc < 0) { 553 put_device(conf_dev); 554 return NULL; 555 } 556 557 spin_lock_init(&idxd->dev_lock); 558 spin_lock_init(&idxd->cmd_lock); 559 560 return idxd; 561 } 562 563 static int idxd_enable_system_pasid(struct idxd_device *idxd) 564 { 565 struct pci_dev *pdev = idxd->pdev; 566 struct device *dev = &pdev->dev; 567 struct iommu_domain *domain; 568 ioasid_t pasid; 569 int ret; 570 571 /* 572 * Attach a global PASID to the DMA domain so that we can use ENQCMDS 573 * to submit work on buffers mapped by DMA API. 574 */ 575 domain = iommu_get_domain_for_dev(dev); 576 if (!domain) 577 return -EPERM; 578 579 pasid = iommu_alloc_global_pasid(dev); 580 if (pasid == IOMMU_PASID_INVALID) 581 return -ENOSPC; 582 583 /* 584 * DMA domain is owned by the driver, it should support all valid 585 * types such as DMA-FQ, identity, etc. 586 */ 587 ret = iommu_attach_device_pasid(domain, dev, pasid); 588 if (ret) { 589 dev_err(dev, "failed to attach device pasid %d, domain type %d", 590 pasid, domain->type); 591 iommu_free_global_pasid(pasid); 592 return ret; 593 } 594 595 /* Since we set user privilege for kernel DMA, enable completion IRQ */ 596 idxd_set_user_intr(idxd, 1); 597 idxd->pasid = pasid; 598 599 return ret; 600 } 601 602 static void idxd_disable_system_pasid(struct idxd_device *idxd) 603 { 604 struct pci_dev *pdev = idxd->pdev; 605 struct device *dev = &pdev->dev; 606 struct iommu_domain *domain; 607 608 domain = iommu_get_domain_for_dev(dev); 609 if (!domain) 610 return; 611 612 iommu_detach_device_pasid(domain, dev, idxd->pasid); 613 iommu_free_global_pasid(idxd->pasid); 614 615 idxd_set_user_intr(idxd, 0); 616 idxd->sva = NULL; 617 idxd->pasid = IOMMU_PASID_INVALID; 618 } 619 620 static int idxd_enable_sva(struct pci_dev *pdev) 621 { 622 int ret; 623 624 ret = iommu_dev_enable_feature(&pdev->dev, IOMMU_DEV_FEAT_IOPF); 625 if (ret) 626 return ret; 627 628 ret = iommu_dev_enable_feature(&pdev->dev, IOMMU_DEV_FEAT_SVA); 629 if (ret) 630 iommu_dev_disable_feature(&pdev->dev, IOMMU_DEV_FEAT_IOPF); 631 632 return ret; 633 } 634 635 static void idxd_disable_sva(struct pci_dev *pdev) 636 { 637 iommu_dev_disable_feature(&pdev->dev, IOMMU_DEV_FEAT_SVA); 638 iommu_dev_disable_feature(&pdev->dev, IOMMU_DEV_FEAT_IOPF); 639 } 640 641 static int idxd_probe(struct idxd_device *idxd) 642 { 643 struct pci_dev *pdev = idxd->pdev; 644 struct device *dev = &pdev->dev; 645 int rc; 646 647 dev_dbg(dev, "%s entered and resetting device\n", __func__); 648 rc = idxd_device_init_reset(idxd); 649 if (rc < 0) 650 return rc; 651 652 dev_dbg(dev, "IDXD reset complete\n"); 653 654 if (IS_ENABLED(CONFIG_INTEL_IDXD_SVM) && sva) { 655 if (idxd_enable_sva(pdev)) { 656 dev_warn(dev, "Unable to turn on user SVA feature.\n"); 657 } else { 658 set_bit(IDXD_FLAG_USER_PASID_ENABLED, &idxd->flags); 659 660 rc = idxd_enable_system_pasid(idxd); 661 if (rc) 662 dev_warn(dev, "No in-kernel DMA with PASID. %d\n", rc); 663 else 664 set_bit(IDXD_FLAG_PASID_ENABLED, &idxd->flags); 665 } 666 } else if (!sva) { 667 dev_warn(dev, "User forced SVA off via module param.\n"); 668 } 669 670 idxd_read_caps(idxd); 671 idxd_read_table_offsets(idxd); 672 673 rc = idxd_setup_internals(idxd); 674 if (rc) 675 goto err; 676 677 /* If the configs are readonly, then load them from device */ 678 if (!test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags)) { 679 dev_dbg(dev, "Loading RO device config\n"); 680 rc = idxd_device_load_config(idxd); 681 if (rc < 0) 682 goto err_config; 683 } 684 685 rc = idxd_setup_interrupts(idxd); 686 if (rc) 687 goto err_config; 688 689 idxd->major = idxd_cdev_get_major(idxd); 690 691 rc = perfmon_pmu_init(idxd); 692 if (rc < 0) 693 dev_warn(dev, "Failed to initialize perfmon. No PMU support: %d\n", rc); 694 695 dev_dbg(dev, "IDXD device %d probed successfully\n", idxd->id); 696 return 0; 697 698 err_config: 699 idxd_cleanup_internals(idxd); 700 err: 701 if (device_pasid_enabled(idxd)) 702 idxd_disable_system_pasid(idxd); 703 if (device_user_pasid_enabled(idxd)) 704 idxd_disable_sva(pdev); 705 return rc; 706 } 707 708 static void idxd_cleanup(struct idxd_device *idxd) 709 { 710 perfmon_pmu_remove(idxd); 711 idxd_cleanup_interrupts(idxd); 712 idxd_cleanup_internals(idxd); 713 if (device_pasid_enabled(idxd)) 714 idxd_disable_system_pasid(idxd); 715 if (device_user_pasid_enabled(idxd)) 716 idxd_disable_sva(idxd->pdev); 717 } 718 719 static int idxd_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) 720 { 721 struct device *dev = &pdev->dev; 722 struct idxd_device *idxd; 723 struct idxd_driver_data *data = (struct idxd_driver_data *)id->driver_data; 724 int rc; 725 726 rc = pci_enable_device(pdev); 727 if (rc) 728 return rc; 729 730 dev_dbg(dev, "Alloc IDXD context\n"); 731 idxd = idxd_alloc(pdev, data); 732 if (!idxd) { 733 rc = -ENOMEM; 734 goto err_idxd_alloc; 735 } 736 737 dev_dbg(dev, "Mapping BARs\n"); 738 idxd->reg_base = pci_iomap(pdev, IDXD_MMIO_BAR, 0); 739 if (!idxd->reg_base) { 740 rc = -ENOMEM; 741 goto err_iomap; 742 } 743 744 dev_dbg(dev, "Set DMA masks\n"); 745 rc = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); 746 if (rc) 747 goto err; 748 749 dev_dbg(dev, "Set PCI master\n"); 750 pci_set_master(pdev); 751 pci_set_drvdata(pdev, idxd); 752 753 idxd->hw.version = ioread32(idxd->reg_base + IDXD_VER_OFFSET); 754 rc = idxd_probe(idxd); 755 if (rc) { 756 dev_err(dev, "Intel(R) IDXD DMA Engine init failed\n"); 757 goto err; 758 } 759 760 if (data->load_device_defaults) { 761 rc = data->load_device_defaults(idxd); 762 if (rc) 763 dev_warn(dev, "IDXD loading device defaults failed\n"); 764 } 765 766 rc = idxd_register_devices(idxd); 767 if (rc) { 768 dev_err(dev, "IDXD sysfs setup failed\n"); 769 goto err_dev_register; 770 } 771 772 rc = idxd_device_init_debugfs(idxd); 773 if (rc) 774 dev_warn(dev, "IDXD debugfs failed to setup\n"); 775 776 dev_info(&pdev->dev, "Intel(R) Accelerator Device (v%x)\n", 777 idxd->hw.version); 778 779 idxd->user_submission_safe = data->user_submission_safe; 780 781 return 0; 782 783 err_dev_register: 784 idxd_cleanup(idxd); 785 err: 786 pci_iounmap(pdev, idxd->reg_base); 787 err_iomap: 788 put_device(idxd_confdev(idxd)); 789 err_idxd_alloc: 790 pci_disable_device(pdev); 791 return rc; 792 } 793 794 void idxd_wqs_quiesce(struct idxd_device *idxd) 795 { 796 struct idxd_wq *wq; 797 int i; 798 799 for (i = 0; i < idxd->max_wqs; i++) { 800 wq = idxd->wqs[i]; 801 if (wq->state == IDXD_WQ_ENABLED && wq->type == IDXD_WQT_KERNEL) 802 idxd_wq_quiesce(wq); 803 } 804 } 805 806 static void idxd_shutdown(struct pci_dev *pdev) 807 { 808 struct idxd_device *idxd = pci_get_drvdata(pdev); 809 struct idxd_irq_entry *irq_entry; 810 int rc; 811 812 rc = idxd_device_disable(idxd); 813 if (rc) 814 dev_err(&pdev->dev, "Disabling device failed\n"); 815 816 irq_entry = &idxd->ie; 817 synchronize_irq(irq_entry->vector); 818 idxd_mask_error_interrupts(idxd); 819 flush_workqueue(idxd->wq); 820 } 821 822 static void idxd_remove(struct pci_dev *pdev) 823 { 824 struct idxd_device *idxd = pci_get_drvdata(pdev); 825 struct idxd_irq_entry *irq_entry; 826 827 idxd_unregister_devices(idxd); 828 /* 829 * When ->release() is called for the idxd->conf_dev, it frees all the memory related 830 * to the idxd context. The driver still needs those bits in order to do the rest of 831 * the cleanup. However, we do need to unbound the idxd sub-driver. So take a ref 832 * on the device here to hold off the freeing while allowing the idxd sub-driver 833 * to unbind. 834 */ 835 get_device(idxd_confdev(idxd)); 836 device_unregister(idxd_confdev(idxd)); 837 idxd_shutdown(pdev); 838 if (device_pasid_enabled(idxd)) 839 idxd_disable_system_pasid(idxd); 840 idxd_device_remove_debugfs(idxd); 841 842 irq_entry = idxd_get_ie(idxd, 0); 843 free_irq(irq_entry->vector, irq_entry); 844 pci_free_irq_vectors(pdev); 845 pci_iounmap(pdev, idxd->reg_base); 846 if (device_user_pasid_enabled(idxd)) 847 idxd_disable_sva(pdev); 848 pci_disable_device(pdev); 849 destroy_workqueue(idxd->wq); 850 perfmon_pmu_remove(idxd); 851 put_device(idxd_confdev(idxd)); 852 } 853 854 static struct pci_driver idxd_pci_driver = { 855 .name = DRV_NAME, 856 .id_table = idxd_pci_tbl, 857 .probe = idxd_pci_probe, 858 .remove = idxd_remove, 859 .shutdown = idxd_shutdown, 860 }; 861 862 static int __init idxd_init_module(void) 863 { 864 int err; 865 866 /* 867 * If the CPU does not support MOVDIR64B or ENQCMDS, there's no point in 868 * enumerating the device. We can not utilize it. 869 */ 870 if (!cpu_feature_enabled(X86_FEATURE_MOVDIR64B)) { 871 pr_warn("idxd driver failed to load without MOVDIR64B.\n"); 872 return -ENODEV; 873 } 874 875 if (!cpu_feature_enabled(X86_FEATURE_ENQCMD)) 876 pr_warn("Platform does not have ENQCMD(S) support.\n"); 877 else 878 support_enqcmd = true; 879 880 perfmon_init(); 881 882 err = idxd_driver_register(&idxd_drv); 883 if (err < 0) 884 goto err_idxd_driver_register; 885 886 err = idxd_driver_register(&idxd_dmaengine_drv); 887 if (err < 0) 888 goto err_idxd_dmaengine_driver_register; 889 890 err = idxd_driver_register(&idxd_user_drv); 891 if (err < 0) 892 goto err_idxd_user_driver_register; 893 894 err = idxd_cdev_register(); 895 if (err) 896 goto err_cdev_register; 897 898 err = idxd_init_debugfs(); 899 if (err) 900 goto err_debugfs; 901 902 err = pci_register_driver(&idxd_pci_driver); 903 if (err) 904 goto err_pci_register; 905 906 return 0; 907 908 err_pci_register: 909 idxd_remove_debugfs(); 910 err_debugfs: 911 idxd_cdev_remove(); 912 err_cdev_register: 913 idxd_driver_unregister(&idxd_user_drv); 914 err_idxd_user_driver_register: 915 idxd_driver_unregister(&idxd_dmaengine_drv); 916 err_idxd_dmaengine_driver_register: 917 idxd_driver_unregister(&idxd_drv); 918 err_idxd_driver_register: 919 return err; 920 } 921 module_init(idxd_init_module); 922 923 static void __exit idxd_exit_module(void) 924 { 925 idxd_driver_unregister(&idxd_user_drv); 926 idxd_driver_unregister(&idxd_dmaengine_drv); 927 idxd_driver_unregister(&idxd_drv); 928 pci_unregister_driver(&idxd_pci_driver); 929 idxd_cdev_remove(); 930 perfmon_exit(); 931 idxd_remove_debugfs(); 932 } 933 module_exit(idxd_exit_module); 934