1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright(c) 2019 Intel Corporation. All rights rsvd. */ 3 #include <linux/init.h> 4 #include <linux/kernel.h> 5 #include <linux/module.h> 6 #include <linux/slab.h> 7 #include <linux/pci.h> 8 #include <linux/interrupt.h> 9 #include <linux/delay.h> 10 #include <linux/dma-mapping.h> 11 #include <linux/workqueue.h> 12 #include <linux/fs.h> 13 #include <linux/io-64-nonatomic-lo-hi.h> 14 #include <linux/device.h> 15 #include <linux/idr.h> 16 #include <linux/iommu.h> 17 #include <uapi/linux/idxd.h> 18 #include <linux/dmaengine.h> 19 #include "../dmaengine.h" 20 #include "registers.h" 21 #include "idxd.h" 22 #include "perfmon.h" 23 24 MODULE_VERSION(IDXD_DRIVER_VERSION); 25 MODULE_DESCRIPTION("Intel Data Streaming Accelerator and In-Memory Analytics Accelerator common driver"); 26 MODULE_LICENSE("GPL v2"); 27 MODULE_AUTHOR("Intel Corporation"); 28 MODULE_IMPORT_NS(IDXD); 29 30 static bool sva = true; 31 module_param(sva, bool, 0644); 32 MODULE_PARM_DESC(sva, "Toggle SVA support on/off"); 33 34 bool tc_override; 35 module_param(tc_override, bool, 0644); 36 MODULE_PARM_DESC(tc_override, "Override traffic class defaults"); 37 38 #define DRV_NAME "idxd" 39 40 bool support_enqcmd; 41 DEFINE_IDA(idxd_ida); 42 43 static struct idxd_driver_data idxd_driver_data[] = { 44 [IDXD_TYPE_DSA] = { 45 .name_prefix = "dsa", 46 .type = IDXD_TYPE_DSA, 47 .compl_size = sizeof(struct dsa_completion_record), 48 .align = 32, 49 .dev_type = &dsa_device_type, 50 .evl_cr_off = offsetof(struct dsa_evl_entry, cr), 51 .user_submission_safe = false, /* See INTEL-SA-01084 security advisory */ 52 .cr_status_off = offsetof(struct dsa_completion_record, status), 53 .cr_result_off = offsetof(struct dsa_completion_record, result), 54 }, 55 [IDXD_TYPE_IAX] = { 56 .name_prefix = "iax", 57 .type = IDXD_TYPE_IAX, 58 .compl_size = sizeof(struct iax_completion_record), 59 .align = 64, 60 .dev_type = &iax_device_type, 61 .evl_cr_off = offsetof(struct iax_evl_entry, cr), 62 .user_submission_safe = false, /* See INTEL-SA-01084 security advisory */ 63 .cr_status_off = offsetof(struct iax_completion_record, status), 64 .cr_result_off = offsetof(struct iax_completion_record, error_code), 65 .load_device_defaults = idxd_load_iaa_device_defaults, 66 }, 67 }; 68 69 static struct pci_device_id idxd_pci_tbl[] = { 70 /* DSA ver 1.0 platforms */ 71 { PCI_DEVICE_DATA(INTEL, DSA_SPR0, &idxd_driver_data[IDXD_TYPE_DSA]) }, 72 73 /* IAX ver 1.0 platforms */ 74 { PCI_DEVICE_DATA(INTEL, IAX_SPR0, &idxd_driver_data[IDXD_TYPE_IAX]) }, 75 { 0, } 76 }; 77 MODULE_DEVICE_TABLE(pci, idxd_pci_tbl); 78 79 static int idxd_setup_interrupts(struct idxd_device *idxd) 80 { 81 struct pci_dev *pdev = idxd->pdev; 82 struct device *dev = &pdev->dev; 83 struct idxd_irq_entry *ie; 84 int i, msixcnt; 85 int rc = 0; 86 87 msixcnt = pci_msix_vec_count(pdev); 88 if (msixcnt < 0) { 89 dev_err(dev, "Not MSI-X interrupt capable.\n"); 90 return -ENOSPC; 91 } 92 idxd->irq_cnt = msixcnt; 93 94 rc = pci_alloc_irq_vectors(pdev, msixcnt, msixcnt, PCI_IRQ_MSIX); 95 if (rc != msixcnt) { 96 dev_err(dev, "Failed enabling %d MSIX entries: %d\n", msixcnt, rc); 97 return -ENOSPC; 98 } 99 dev_dbg(dev, "Enabled %d msix vectors\n", msixcnt); 100 101 102 ie = idxd_get_ie(idxd, 0); 103 ie->vector = pci_irq_vector(pdev, 0); 104 rc = request_threaded_irq(ie->vector, NULL, idxd_misc_thread, 0, "idxd-misc", ie); 105 if (rc < 0) { 106 dev_err(dev, "Failed to allocate misc interrupt.\n"); 107 goto err_misc_irq; 108 } 109 dev_dbg(dev, "Requested idxd-misc handler on msix vector %d\n", ie->vector); 110 111 for (i = 0; i < idxd->max_wqs; i++) { 112 int msix_idx = i + 1; 113 114 ie = idxd_get_ie(idxd, msix_idx); 115 ie->id = msix_idx; 116 ie->int_handle = INVALID_INT_HANDLE; 117 ie->pasid = IOMMU_PASID_INVALID; 118 119 spin_lock_init(&ie->list_lock); 120 init_llist_head(&ie->pending_llist); 121 INIT_LIST_HEAD(&ie->work_list); 122 } 123 124 idxd_unmask_error_interrupts(idxd); 125 return 0; 126 127 err_misc_irq: 128 idxd_mask_error_interrupts(idxd); 129 pci_free_irq_vectors(pdev); 130 dev_err(dev, "No usable interrupts\n"); 131 return rc; 132 } 133 134 static void idxd_cleanup_interrupts(struct idxd_device *idxd) 135 { 136 struct pci_dev *pdev = idxd->pdev; 137 struct idxd_irq_entry *ie; 138 int msixcnt; 139 140 msixcnt = pci_msix_vec_count(pdev); 141 if (msixcnt <= 0) 142 return; 143 144 ie = idxd_get_ie(idxd, 0); 145 idxd_mask_error_interrupts(idxd); 146 free_irq(ie->vector, ie); 147 pci_free_irq_vectors(pdev); 148 } 149 150 static int idxd_setup_wqs(struct idxd_device *idxd) 151 { 152 struct device *dev = &idxd->pdev->dev; 153 struct idxd_wq *wq; 154 struct device *conf_dev; 155 int i, rc; 156 157 idxd->wqs = kcalloc_node(idxd->max_wqs, sizeof(struct idxd_wq *), 158 GFP_KERNEL, dev_to_node(dev)); 159 if (!idxd->wqs) 160 return -ENOMEM; 161 162 idxd->wq_enable_map = bitmap_zalloc_node(idxd->max_wqs, GFP_KERNEL, dev_to_node(dev)); 163 if (!idxd->wq_enable_map) { 164 kfree(idxd->wqs); 165 return -ENOMEM; 166 } 167 168 for (i = 0; i < idxd->max_wqs; i++) { 169 wq = kzalloc_node(sizeof(*wq), GFP_KERNEL, dev_to_node(dev)); 170 if (!wq) { 171 rc = -ENOMEM; 172 goto err; 173 } 174 175 idxd_dev_set_type(&wq->idxd_dev, IDXD_DEV_WQ); 176 conf_dev = wq_confdev(wq); 177 wq->id = i; 178 wq->idxd = idxd; 179 device_initialize(wq_confdev(wq)); 180 conf_dev->parent = idxd_confdev(idxd); 181 conf_dev->bus = &dsa_bus_type; 182 conf_dev->type = &idxd_wq_device_type; 183 rc = dev_set_name(conf_dev, "wq%d.%d", idxd->id, wq->id); 184 if (rc < 0) { 185 put_device(conf_dev); 186 goto err; 187 } 188 189 mutex_init(&wq->wq_lock); 190 init_waitqueue_head(&wq->err_queue); 191 init_completion(&wq->wq_dead); 192 init_completion(&wq->wq_resurrect); 193 wq->max_xfer_bytes = WQ_DEFAULT_MAX_XFER; 194 idxd_wq_set_max_batch_size(idxd->data->type, wq, WQ_DEFAULT_MAX_BATCH); 195 wq->enqcmds_retries = IDXD_ENQCMDS_RETRIES; 196 wq->wqcfg = kzalloc_node(idxd->wqcfg_size, GFP_KERNEL, dev_to_node(dev)); 197 if (!wq->wqcfg) { 198 put_device(conf_dev); 199 rc = -ENOMEM; 200 goto err; 201 } 202 203 if (idxd->hw.wq_cap.op_config) { 204 wq->opcap_bmap = bitmap_zalloc(IDXD_MAX_OPCAP_BITS, GFP_KERNEL); 205 if (!wq->opcap_bmap) { 206 put_device(conf_dev); 207 rc = -ENOMEM; 208 goto err; 209 } 210 bitmap_copy(wq->opcap_bmap, idxd->opcap_bmap, IDXD_MAX_OPCAP_BITS); 211 } 212 mutex_init(&wq->uc_lock); 213 xa_init(&wq->upasid_xa); 214 idxd->wqs[i] = wq; 215 } 216 217 return 0; 218 219 err: 220 while (--i >= 0) { 221 wq = idxd->wqs[i]; 222 conf_dev = wq_confdev(wq); 223 put_device(conf_dev); 224 } 225 return rc; 226 } 227 228 static int idxd_setup_engines(struct idxd_device *idxd) 229 { 230 struct idxd_engine *engine; 231 struct device *dev = &idxd->pdev->dev; 232 struct device *conf_dev; 233 int i, rc; 234 235 idxd->engines = kcalloc_node(idxd->max_engines, sizeof(struct idxd_engine *), 236 GFP_KERNEL, dev_to_node(dev)); 237 if (!idxd->engines) 238 return -ENOMEM; 239 240 for (i = 0; i < idxd->max_engines; i++) { 241 engine = kzalloc_node(sizeof(*engine), GFP_KERNEL, dev_to_node(dev)); 242 if (!engine) { 243 rc = -ENOMEM; 244 goto err; 245 } 246 247 idxd_dev_set_type(&engine->idxd_dev, IDXD_DEV_ENGINE); 248 conf_dev = engine_confdev(engine); 249 engine->id = i; 250 engine->idxd = idxd; 251 device_initialize(conf_dev); 252 conf_dev->parent = idxd_confdev(idxd); 253 conf_dev->bus = &dsa_bus_type; 254 conf_dev->type = &idxd_engine_device_type; 255 rc = dev_set_name(conf_dev, "engine%d.%d", idxd->id, engine->id); 256 if (rc < 0) { 257 put_device(conf_dev); 258 goto err; 259 } 260 261 idxd->engines[i] = engine; 262 } 263 264 return 0; 265 266 err: 267 while (--i >= 0) { 268 engine = idxd->engines[i]; 269 conf_dev = engine_confdev(engine); 270 put_device(conf_dev); 271 } 272 return rc; 273 } 274 275 static int idxd_setup_groups(struct idxd_device *idxd) 276 { 277 struct device *dev = &idxd->pdev->dev; 278 struct device *conf_dev; 279 struct idxd_group *group; 280 int i, rc; 281 282 idxd->groups = kcalloc_node(idxd->max_groups, sizeof(struct idxd_group *), 283 GFP_KERNEL, dev_to_node(dev)); 284 if (!idxd->groups) 285 return -ENOMEM; 286 287 for (i = 0; i < idxd->max_groups; i++) { 288 group = kzalloc_node(sizeof(*group), GFP_KERNEL, dev_to_node(dev)); 289 if (!group) { 290 rc = -ENOMEM; 291 goto err; 292 } 293 294 idxd_dev_set_type(&group->idxd_dev, IDXD_DEV_GROUP); 295 conf_dev = group_confdev(group); 296 group->id = i; 297 group->idxd = idxd; 298 device_initialize(conf_dev); 299 conf_dev->parent = idxd_confdev(idxd); 300 conf_dev->bus = &dsa_bus_type; 301 conf_dev->type = &idxd_group_device_type; 302 rc = dev_set_name(conf_dev, "group%d.%d", idxd->id, group->id); 303 if (rc < 0) { 304 put_device(conf_dev); 305 goto err; 306 } 307 308 idxd->groups[i] = group; 309 if (idxd->hw.version <= DEVICE_VERSION_2 && !tc_override) { 310 group->tc_a = 1; 311 group->tc_b = 1; 312 } else { 313 group->tc_a = -1; 314 group->tc_b = -1; 315 } 316 /* 317 * The default value is the same as the value of 318 * total read buffers in GRPCAP. 319 */ 320 group->rdbufs_allowed = idxd->max_rdbufs; 321 } 322 323 return 0; 324 325 err: 326 while (--i >= 0) { 327 group = idxd->groups[i]; 328 put_device(group_confdev(group)); 329 } 330 return rc; 331 } 332 333 static void idxd_cleanup_internals(struct idxd_device *idxd) 334 { 335 int i; 336 337 for (i = 0; i < idxd->max_groups; i++) 338 put_device(group_confdev(idxd->groups[i])); 339 for (i = 0; i < idxd->max_engines; i++) 340 put_device(engine_confdev(idxd->engines[i])); 341 for (i = 0; i < idxd->max_wqs; i++) 342 put_device(wq_confdev(idxd->wqs[i])); 343 destroy_workqueue(idxd->wq); 344 } 345 346 static int idxd_init_evl(struct idxd_device *idxd) 347 { 348 struct device *dev = &idxd->pdev->dev; 349 unsigned int evl_cache_size; 350 struct idxd_evl *evl; 351 const char *idxd_name; 352 353 if (idxd->hw.gen_cap.evl_support == 0) 354 return 0; 355 356 evl = kzalloc_node(sizeof(*evl), GFP_KERNEL, dev_to_node(dev)); 357 if (!evl) 358 return -ENOMEM; 359 360 mutex_init(&evl->lock); 361 evl->size = IDXD_EVL_SIZE_MIN; 362 363 idxd_name = dev_name(idxd_confdev(idxd)); 364 evl_cache_size = sizeof(struct idxd_evl_fault) + evl_ent_size(idxd); 365 /* 366 * Since completion record in evl_cache will be copied to user 367 * when handling completion record page fault, need to create 368 * the cache suitable for user copy. 369 */ 370 idxd->evl_cache = kmem_cache_create_usercopy(idxd_name, evl_cache_size, 371 0, 0, 0, evl_cache_size, 372 NULL); 373 if (!idxd->evl_cache) { 374 kfree(evl); 375 return -ENOMEM; 376 } 377 378 idxd->evl = evl; 379 return 0; 380 } 381 382 static int idxd_setup_internals(struct idxd_device *idxd) 383 { 384 struct device *dev = &idxd->pdev->dev; 385 int rc, i; 386 387 init_waitqueue_head(&idxd->cmd_waitq); 388 389 rc = idxd_setup_wqs(idxd); 390 if (rc < 0) 391 goto err_wqs; 392 393 rc = idxd_setup_engines(idxd); 394 if (rc < 0) 395 goto err_engine; 396 397 rc = idxd_setup_groups(idxd); 398 if (rc < 0) 399 goto err_group; 400 401 idxd->wq = create_workqueue(dev_name(dev)); 402 if (!idxd->wq) { 403 rc = -ENOMEM; 404 goto err_wkq_create; 405 } 406 407 rc = idxd_init_evl(idxd); 408 if (rc < 0) 409 goto err_evl; 410 411 return 0; 412 413 err_evl: 414 destroy_workqueue(idxd->wq); 415 err_wkq_create: 416 for (i = 0; i < idxd->max_groups; i++) 417 put_device(group_confdev(idxd->groups[i])); 418 err_group: 419 for (i = 0; i < idxd->max_engines; i++) 420 put_device(engine_confdev(idxd->engines[i])); 421 err_engine: 422 for (i = 0; i < idxd->max_wqs; i++) 423 put_device(wq_confdev(idxd->wqs[i])); 424 err_wqs: 425 return rc; 426 } 427 428 static void idxd_read_table_offsets(struct idxd_device *idxd) 429 { 430 union offsets_reg offsets; 431 struct device *dev = &idxd->pdev->dev; 432 433 offsets.bits[0] = ioread64(idxd->reg_base + IDXD_TABLE_OFFSET); 434 offsets.bits[1] = ioread64(idxd->reg_base + IDXD_TABLE_OFFSET + sizeof(u64)); 435 idxd->grpcfg_offset = offsets.grpcfg * IDXD_TABLE_MULT; 436 dev_dbg(dev, "IDXD Group Config Offset: %#x\n", idxd->grpcfg_offset); 437 idxd->wqcfg_offset = offsets.wqcfg * IDXD_TABLE_MULT; 438 dev_dbg(dev, "IDXD Work Queue Config Offset: %#x\n", idxd->wqcfg_offset); 439 idxd->msix_perm_offset = offsets.msix_perm * IDXD_TABLE_MULT; 440 dev_dbg(dev, "IDXD MSIX Permission Offset: %#x\n", idxd->msix_perm_offset); 441 idxd->perfmon_offset = offsets.perfmon * IDXD_TABLE_MULT; 442 dev_dbg(dev, "IDXD Perfmon Offset: %#x\n", idxd->perfmon_offset); 443 } 444 445 void multi_u64_to_bmap(unsigned long *bmap, u64 *val, int count) 446 { 447 int i, j, nr; 448 449 for (i = 0, nr = 0; i < count; i++) { 450 for (j = 0; j < BITS_PER_LONG_LONG; j++) { 451 if (val[i] & BIT(j)) 452 set_bit(nr, bmap); 453 nr++; 454 } 455 } 456 } 457 458 static void idxd_read_caps(struct idxd_device *idxd) 459 { 460 struct device *dev = &idxd->pdev->dev; 461 int i; 462 463 /* reading generic capabilities */ 464 idxd->hw.gen_cap.bits = ioread64(idxd->reg_base + IDXD_GENCAP_OFFSET); 465 dev_dbg(dev, "gen_cap: %#llx\n", idxd->hw.gen_cap.bits); 466 467 if (idxd->hw.gen_cap.cmd_cap) { 468 idxd->hw.cmd_cap = ioread32(idxd->reg_base + IDXD_CMDCAP_OFFSET); 469 dev_dbg(dev, "cmd_cap: %#x\n", idxd->hw.cmd_cap); 470 } 471 472 /* reading command capabilities */ 473 if (idxd->hw.cmd_cap & BIT(IDXD_CMD_REQUEST_INT_HANDLE)) 474 idxd->request_int_handles = true; 475 476 idxd->max_xfer_bytes = 1ULL << idxd->hw.gen_cap.max_xfer_shift; 477 dev_dbg(dev, "max xfer size: %llu bytes\n", idxd->max_xfer_bytes); 478 idxd_set_max_batch_size(idxd->data->type, idxd, 1U << idxd->hw.gen_cap.max_batch_shift); 479 dev_dbg(dev, "max batch size: %u\n", idxd->max_batch_size); 480 if (idxd->hw.gen_cap.config_en) 481 set_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags); 482 483 /* reading group capabilities */ 484 idxd->hw.group_cap.bits = 485 ioread64(idxd->reg_base + IDXD_GRPCAP_OFFSET); 486 dev_dbg(dev, "group_cap: %#llx\n", idxd->hw.group_cap.bits); 487 idxd->max_groups = idxd->hw.group_cap.num_groups; 488 dev_dbg(dev, "max groups: %u\n", idxd->max_groups); 489 idxd->max_rdbufs = idxd->hw.group_cap.total_rdbufs; 490 dev_dbg(dev, "max read buffers: %u\n", idxd->max_rdbufs); 491 idxd->nr_rdbufs = idxd->max_rdbufs; 492 493 /* read engine capabilities */ 494 idxd->hw.engine_cap.bits = 495 ioread64(idxd->reg_base + IDXD_ENGCAP_OFFSET); 496 dev_dbg(dev, "engine_cap: %#llx\n", idxd->hw.engine_cap.bits); 497 idxd->max_engines = idxd->hw.engine_cap.num_engines; 498 dev_dbg(dev, "max engines: %u\n", idxd->max_engines); 499 500 /* read workqueue capabilities */ 501 idxd->hw.wq_cap.bits = ioread64(idxd->reg_base + IDXD_WQCAP_OFFSET); 502 dev_dbg(dev, "wq_cap: %#llx\n", idxd->hw.wq_cap.bits); 503 idxd->max_wq_size = idxd->hw.wq_cap.total_wq_size; 504 dev_dbg(dev, "total workqueue size: %u\n", idxd->max_wq_size); 505 idxd->max_wqs = idxd->hw.wq_cap.num_wqs; 506 dev_dbg(dev, "max workqueues: %u\n", idxd->max_wqs); 507 idxd->wqcfg_size = 1 << (idxd->hw.wq_cap.wqcfg_size + IDXD_WQCFG_MIN); 508 dev_dbg(dev, "wqcfg size: %u\n", idxd->wqcfg_size); 509 510 /* reading operation capabilities */ 511 for (i = 0; i < 4; i++) { 512 idxd->hw.opcap.bits[i] = ioread64(idxd->reg_base + 513 IDXD_OPCAP_OFFSET + i * sizeof(u64)); 514 dev_dbg(dev, "opcap[%d]: %#llx\n", i, idxd->hw.opcap.bits[i]); 515 } 516 multi_u64_to_bmap(idxd->opcap_bmap, &idxd->hw.opcap.bits[0], 4); 517 518 /* read iaa cap */ 519 if (idxd->data->type == IDXD_TYPE_IAX && idxd->hw.version >= DEVICE_VERSION_2) 520 idxd->hw.iaa_cap.bits = ioread64(idxd->reg_base + IDXD_IAACAP_OFFSET); 521 } 522 523 static struct idxd_device *idxd_alloc(struct pci_dev *pdev, struct idxd_driver_data *data) 524 { 525 struct device *dev = &pdev->dev; 526 struct device *conf_dev; 527 struct idxd_device *idxd; 528 int rc; 529 530 idxd = kzalloc_node(sizeof(*idxd), GFP_KERNEL, dev_to_node(dev)); 531 if (!idxd) 532 return NULL; 533 534 conf_dev = idxd_confdev(idxd); 535 idxd->pdev = pdev; 536 idxd->data = data; 537 idxd_dev_set_type(&idxd->idxd_dev, idxd->data->type); 538 idxd->id = ida_alloc(&idxd_ida, GFP_KERNEL); 539 if (idxd->id < 0) 540 return NULL; 541 542 idxd->opcap_bmap = bitmap_zalloc_node(IDXD_MAX_OPCAP_BITS, GFP_KERNEL, dev_to_node(dev)); 543 if (!idxd->opcap_bmap) { 544 ida_free(&idxd_ida, idxd->id); 545 return NULL; 546 } 547 548 device_initialize(conf_dev); 549 conf_dev->parent = dev; 550 conf_dev->bus = &dsa_bus_type; 551 conf_dev->type = idxd->data->dev_type; 552 rc = dev_set_name(conf_dev, "%s%d", idxd->data->name_prefix, idxd->id); 553 if (rc < 0) { 554 put_device(conf_dev); 555 return NULL; 556 } 557 558 spin_lock_init(&idxd->dev_lock); 559 spin_lock_init(&idxd->cmd_lock); 560 561 return idxd; 562 } 563 564 static int idxd_enable_system_pasid(struct idxd_device *idxd) 565 { 566 struct pci_dev *pdev = idxd->pdev; 567 struct device *dev = &pdev->dev; 568 struct iommu_domain *domain; 569 ioasid_t pasid; 570 int ret; 571 572 /* 573 * Attach a global PASID to the DMA domain so that we can use ENQCMDS 574 * to submit work on buffers mapped by DMA API. 575 */ 576 domain = iommu_get_domain_for_dev(dev); 577 if (!domain) 578 return -EPERM; 579 580 pasid = iommu_alloc_global_pasid(dev); 581 if (pasid == IOMMU_PASID_INVALID) 582 return -ENOSPC; 583 584 /* 585 * DMA domain is owned by the driver, it should support all valid 586 * types such as DMA-FQ, identity, etc. 587 */ 588 ret = iommu_attach_device_pasid(domain, dev, pasid, NULL); 589 if (ret) { 590 dev_err(dev, "failed to attach device pasid %d, domain type %d", 591 pasid, domain->type); 592 iommu_free_global_pasid(pasid); 593 return ret; 594 } 595 596 /* Since we set user privilege for kernel DMA, enable completion IRQ */ 597 idxd_set_user_intr(idxd, 1); 598 idxd->pasid = pasid; 599 600 return ret; 601 } 602 603 static void idxd_disable_system_pasid(struct idxd_device *idxd) 604 { 605 struct pci_dev *pdev = idxd->pdev; 606 struct device *dev = &pdev->dev; 607 struct iommu_domain *domain; 608 609 domain = iommu_get_domain_for_dev(dev); 610 if (!domain) 611 return; 612 613 iommu_detach_device_pasid(domain, dev, idxd->pasid); 614 iommu_free_global_pasid(idxd->pasid); 615 616 idxd_set_user_intr(idxd, 0); 617 idxd->sva = NULL; 618 idxd->pasid = IOMMU_PASID_INVALID; 619 } 620 621 static int idxd_enable_sva(struct pci_dev *pdev) 622 { 623 int ret; 624 625 ret = iommu_dev_enable_feature(&pdev->dev, IOMMU_DEV_FEAT_IOPF); 626 if (ret) 627 return ret; 628 629 ret = iommu_dev_enable_feature(&pdev->dev, IOMMU_DEV_FEAT_SVA); 630 if (ret) 631 iommu_dev_disable_feature(&pdev->dev, IOMMU_DEV_FEAT_IOPF); 632 633 return ret; 634 } 635 636 static void idxd_disable_sva(struct pci_dev *pdev) 637 { 638 iommu_dev_disable_feature(&pdev->dev, IOMMU_DEV_FEAT_SVA); 639 iommu_dev_disable_feature(&pdev->dev, IOMMU_DEV_FEAT_IOPF); 640 } 641 642 static int idxd_probe(struct idxd_device *idxd) 643 { 644 struct pci_dev *pdev = idxd->pdev; 645 struct device *dev = &pdev->dev; 646 int rc; 647 648 dev_dbg(dev, "%s entered and resetting device\n", __func__); 649 rc = idxd_device_init_reset(idxd); 650 if (rc < 0) 651 return rc; 652 653 dev_dbg(dev, "IDXD reset complete\n"); 654 655 if (IS_ENABLED(CONFIG_INTEL_IDXD_SVM) && sva) { 656 if (idxd_enable_sva(pdev)) { 657 dev_warn(dev, "Unable to turn on user SVA feature.\n"); 658 } else { 659 set_bit(IDXD_FLAG_USER_PASID_ENABLED, &idxd->flags); 660 661 rc = idxd_enable_system_pasid(idxd); 662 if (rc) 663 dev_warn(dev, "No in-kernel DMA with PASID. %d\n", rc); 664 else 665 set_bit(IDXD_FLAG_PASID_ENABLED, &idxd->flags); 666 } 667 } else if (!sva) { 668 dev_warn(dev, "User forced SVA off via module param.\n"); 669 } 670 671 idxd_read_caps(idxd); 672 idxd_read_table_offsets(idxd); 673 674 rc = idxd_setup_internals(idxd); 675 if (rc) 676 goto err; 677 678 /* If the configs are readonly, then load them from device */ 679 if (!test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags)) { 680 dev_dbg(dev, "Loading RO device config\n"); 681 rc = idxd_device_load_config(idxd); 682 if (rc < 0) 683 goto err_config; 684 } 685 686 rc = idxd_setup_interrupts(idxd); 687 if (rc) 688 goto err_config; 689 690 idxd->major = idxd_cdev_get_major(idxd); 691 692 rc = perfmon_pmu_init(idxd); 693 if (rc < 0) 694 dev_warn(dev, "Failed to initialize perfmon. No PMU support: %d\n", rc); 695 696 dev_dbg(dev, "IDXD device %d probed successfully\n", idxd->id); 697 return 0; 698 699 err_config: 700 idxd_cleanup_internals(idxd); 701 err: 702 if (device_pasid_enabled(idxd)) 703 idxd_disable_system_pasid(idxd); 704 if (device_user_pasid_enabled(idxd)) 705 idxd_disable_sva(pdev); 706 return rc; 707 } 708 709 static void idxd_cleanup(struct idxd_device *idxd) 710 { 711 perfmon_pmu_remove(idxd); 712 idxd_cleanup_interrupts(idxd); 713 idxd_cleanup_internals(idxd); 714 if (device_pasid_enabled(idxd)) 715 idxd_disable_system_pasid(idxd); 716 if (device_user_pasid_enabled(idxd)) 717 idxd_disable_sva(idxd->pdev); 718 } 719 720 static int idxd_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) 721 { 722 struct device *dev = &pdev->dev; 723 struct idxd_device *idxd; 724 struct idxd_driver_data *data = (struct idxd_driver_data *)id->driver_data; 725 int rc; 726 727 rc = pci_enable_device(pdev); 728 if (rc) 729 return rc; 730 731 dev_dbg(dev, "Alloc IDXD context\n"); 732 idxd = idxd_alloc(pdev, data); 733 if (!idxd) { 734 rc = -ENOMEM; 735 goto err_idxd_alloc; 736 } 737 738 dev_dbg(dev, "Mapping BARs\n"); 739 idxd->reg_base = pci_iomap(pdev, IDXD_MMIO_BAR, 0); 740 if (!idxd->reg_base) { 741 rc = -ENOMEM; 742 goto err_iomap; 743 } 744 745 dev_dbg(dev, "Set DMA masks\n"); 746 rc = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); 747 if (rc) 748 goto err; 749 750 dev_dbg(dev, "Set PCI master\n"); 751 pci_set_master(pdev); 752 pci_set_drvdata(pdev, idxd); 753 754 idxd->hw.version = ioread32(idxd->reg_base + IDXD_VER_OFFSET); 755 rc = idxd_probe(idxd); 756 if (rc) { 757 dev_err(dev, "Intel(R) IDXD DMA Engine init failed\n"); 758 goto err; 759 } 760 761 if (data->load_device_defaults) { 762 rc = data->load_device_defaults(idxd); 763 if (rc) 764 dev_warn(dev, "IDXD loading device defaults failed\n"); 765 } 766 767 rc = idxd_register_devices(idxd); 768 if (rc) { 769 dev_err(dev, "IDXD sysfs setup failed\n"); 770 goto err_dev_register; 771 } 772 773 rc = idxd_device_init_debugfs(idxd); 774 if (rc) 775 dev_warn(dev, "IDXD debugfs failed to setup\n"); 776 777 dev_info(&pdev->dev, "Intel(R) Accelerator Device (v%x)\n", 778 idxd->hw.version); 779 780 idxd->user_submission_safe = data->user_submission_safe; 781 782 return 0; 783 784 err_dev_register: 785 idxd_cleanup(idxd); 786 err: 787 pci_iounmap(pdev, idxd->reg_base); 788 err_iomap: 789 put_device(idxd_confdev(idxd)); 790 err_idxd_alloc: 791 pci_disable_device(pdev); 792 return rc; 793 } 794 795 void idxd_wqs_quiesce(struct idxd_device *idxd) 796 { 797 struct idxd_wq *wq; 798 int i; 799 800 for (i = 0; i < idxd->max_wqs; i++) { 801 wq = idxd->wqs[i]; 802 if (wq->state == IDXD_WQ_ENABLED && wq->type == IDXD_WQT_KERNEL) 803 idxd_wq_quiesce(wq); 804 } 805 } 806 807 static void idxd_shutdown(struct pci_dev *pdev) 808 { 809 struct idxd_device *idxd = pci_get_drvdata(pdev); 810 struct idxd_irq_entry *irq_entry; 811 int rc; 812 813 rc = idxd_device_disable(idxd); 814 if (rc) 815 dev_err(&pdev->dev, "Disabling device failed\n"); 816 817 irq_entry = &idxd->ie; 818 synchronize_irq(irq_entry->vector); 819 idxd_mask_error_interrupts(idxd); 820 flush_workqueue(idxd->wq); 821 } 822 823 static void idxd_remove(struct pci_dev *pdev) 824 { 825 struct idxd_device *idxd = pci_get_drvdata(pdev); 826 struct idxd_irq_entry *irq_entry; 827 828 idxd_unregister_devices(idxd); 829 /* 830 * When ->release() is called for the idxd->conf_dev, it frees all the memory related 831 * to the idxd context. The driver still needs those bits in order to do the rest of 832 * the cleanup. However, we do need to unbound the idxd sub-driver. So take a ref 833 * on the device here to hold off the freeing while allowing the idxd sub-driver 834 * to unbind. 835 */ 836 get_device(idxd_confdev(idxd)); 837 device_unregister(idxd_confdev(idxd)); 838 idxd_shutdown(pdev); 839 if (device_pasid_enabled(idxd)) 840 idxd_disable_system_pasid(idxd); 841 idxd_device_remove_debugfs(idxd); 842 843 irq_entry = idxd_get_ie(idxd, 0); 844 free_irq(irq_entry->vector, irq_entry); 845 pci_free_irq_vectors(pdev); 846 pci_iounmap(pdev, idxd->reg_base); 847 if (device_user_pasid_enabled(idxd)) 848 idxd_disable_sva(pdev); 849 pci_disable_device(pdev); 850 destroy_workqueue(idxd->wq); 851 perfmon_pmu_remove(idxd); 852 put_device(idxd_confdev(idxd)); 853 } 854 855 static struct pci_driver idxd_pci_driver = { 856 .name = DRV_NAME, 857 .id_table = idxd_pci_tbl, 858 .probe = idxd_pci_probe, 859 .remove = idxd_remove, 860 .shutdown = idxd_shutdown, 861 }; 862 863 static int __init idxd_init_module(void) 864 { 865 int err; 866 867 /* 868 * If the CPU does not support MOVDIR64B or ENQCMDS, there's no point in 869 * enumerating the device. We can not utilize it. 870 */ 871 if (!cpu_feature_enabled(X86_FEATURE_MOVDIR64B)) { 872 pr_warn("idxd driver failed to load without MOVDIR64B.\n"); 873 return -ENODEV; 874 } 875 876 if (!cpu_feature_enabled(X86_FEATURE_ENQCMD)) 877 pr_warn("Platform does not have ENQCMD(S) support.\n"); 878 else 879 support_enqcmd = true; 880 881 perfmon_init(); 882 883 err = idxd_driver_register(&idxd_drv); 884 if (err < 0) 885 goto err_idxd_driver_register; 886 887 err = idxd_driver_register(&idxd_dmaengine_drv); 888 if (err < 0) 889 goto err_idxd_dmaengine_driver_register; 890 891 err = idxd_driver_register(&idxd_user_drv); 892 if (err < 0) 893 goto err_idxd_user_driver_register; 894 895 err = idxd_cdev_register(); 896 if (err) 897 goto err_cdev_register; 898 899 err = idxd_init_debugfs(); 900 if (err) 901 goto err_debugfs; 902 903 err = pci_register_driver(&idxd_pci_driver); 904 if (err) 905 goto err_pci_register; 906 907 return 0; 908 909 err_pci_register: 910 idxd_remove_debugfs(); 911 err_debugfs: 912 idxd_cdev_remove(); 913 err_cdev_register: 914 idxd_driver_unregister(&idxd_user_drv); 915 err_idxd_user_driver_register: 916 idxd_driver_unregister(&idxd_dmaengine_drv); 917 err_idxd_dmaengine_driver_register: 918 idxd_driver_unregister(&idxd_drv); 919 err_idxd_driver_register: 920 return err; 921 } 922 module_init(idxd_init_module); 923 924 static void __exit idxd_exit_module(void) 925 { 926 idxd_driver_unregister(&idxd_user_drv); 927 idxd_driver_unregister(&idxd_dmaengine_drv); 928 idxd_driver_unregister(&idxd_drv); 929 pci_unregister_driver(&idxd_pci_driver); 930 idxd_cdev_remove(); 931 perfmon_exit(); 932 idxd_remove_debugfs(); 933 } 934 module_exit(idxd_exit_module); 935