1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright(c) 2019 Intel Corporation. All rights rsvd. */ 3 #include <linux/init.h> 4 #include <linux/kernel.h> 5 #include <linux/module.h> 6 #include <linux/slab.h> 7 #include <linux/pci.h> 8 #include <linux/interrupt.h> 9 #include <linux/delay.h> 10 #include <linux/dma-mapping.h> 11 #include <linux/workqueue.h> 12 #include <linux/fs.h> 13 #include <linux/io-64-nonatomic-lo-hi.h> 14 #include <linux/device.h> 15 #include <linux/idr.h> 16 #include <linux/iommu.h> 17 #include <uapi/linux/idxd.h> 18 #include <linux/dmaengine.h> 19 #include "../dmaengine.h" 20 #include "registers.h" 21 #include "idxd.h" 22 #include "perfmon.h" 23 24 MODULE_VERSION(IDXD_DRIVER_VERSION); 25 MODULE_DESCRIPTION("Intel Data Streaming Accelerator and In-Memory Analytics Accelerator common driver"); 26 MODULE_LICENSE("GPL v2"); 27 MODULE_AUTHOR("Intel Corporation"); 28 MODULE_IMPORT_NS(IDXD); 29 30 static bool sva = true; 31 module_param(sva, bool, 0644); 32 MODULE_PARM_DESC(sva, "Toggle SVA support on/off"); 33 34 bool tc_override; 35 module_param(tc_override, bool, 0644); 36 MODULE_PARM_DESC(tc_override, "Override traffic class defaults"); 37 38 #define DRV_NAME "idxd" 39 40 bool support_enqcmd; 41 DEFINE_IDA(idxd_ida); 42 43 static struct idxd_driver_data idxd_driver_data[] = { 44 [IDXD_TYPE_DSA] = { 45 .name_prefix = "dsa", 46 .type = IDXD_TYPE_DSA, 47 .compl_size = sizeof(struct dsa_completion_record), 48 .align = 32, 49 .dev_type = &dsa_device_type, 50 .evl_cr_off = offsetof(struct dsa_evl_entry, cr), 51 .user_submission_safe = false, /* See INTEL-SA-01084 security advisory */ 52 .cr_status_off = offsetof(struct dsa_completion_record, status), 53 .cr_result_off = offsetof(struct dsa_completion_record, result), 54 }, 55 [IDXD_TYPE_IAX] = { 56 .name_prefix = "iax", 57 .type = IDXD_TYPE_IAX, 58 .compl_size = sizeof(struct iax_completion_record), 59 .align = 64, 60 .dev_type = &iax_device_type, 61 .evl_cr_off = offsetof(struct iax_evl_entry, cr), 62 .user_submission_safe = false, /* See INTEL-SA-01084 security advisory */ 63 .cr_status_off = offsetof(struct iax_completion_record, status), 64 .cr_result_off = offsetof(struct iax_completion_record, error_code), 65 .load_device_defaults = idxd_load_iaa_device_defaults, 66 }, 67 }; 68 69 static struct pci_device_id idxd_pci_tbl[] = { 70 /* DSA ver 1.0 platforms */ 71 { PCI_DEVICE_DATA(INTEL, DSA_SPR0, &idxd_driver_data[IDXD_TYPE_DSA]) }, 72 /* DSA on GNR-D platforms */ 73 { PCI_DEVICE_DATA(INTEL, DSA_GNRD, &idxd_driver_data[IDXD_TYPE_DSA]) }, 74 /* DSA on DMR platforms */ 75 { PCI_DEVICE_DATA(INTEL, DSA_DMR, &idxd_driver_data[IDXD_TYPE_DSA]) }, 76 77 /* IAX ver 1.0 platforms */ 78 { PCI_DEVICE_DATA(INTEL, IAX_SPR0, &idxd_driver_data[IDXD_TYPE_IAX]) }, 79 /* IAA on DMR platforms */ 80 { PCI_DEVICE_DATA(INTEL, IAA_DMR, &idxd_driver_data[IDXD_TYPE_IAX]) }, 81 { 0, } 82 }; 83 MODULE_DEVICE_TABLE(pci, idxd_pci_tbl); 84 85 static int idxd_setup_interrupts(struct idxd_device *idxd) 86 { 87 struct pci_dev *pdev = idxd->pdev; 88 struct device *dev = &pdev->dev; 89 struct idxd_irq_entry *ie; 90 int i, msixcnt; 91 int rc = 0; 92 93 msixcnt = pci_msix_vec_count(pdev); 94 if (msixcnt < 0) { 95 dev_err(dev, "Not MSI-X interrupt capable.\n"); 96 return -ENOSPC; 97 } 98 idxd->irq_cnt = msixcnt; 99 100 rc = pci_alloc_irq_vectors(pdev, msixcnt, msixcnt, PCI_IRQ_MSIX); 101 if (rc != msixcnt) { 102 dev_err(dev, "Failed enabling %d MSIX entries: %d\n", msixcnt, rc); 103 return -ENOSPC; 104 } 105 dev_dbg(dev, "Enabled %d msix vectors\n", msixcnt); 106 107 108 ie = idxd_get_ie(idxd, 0); 109 ie->vector = pci_irq_vector(pdev, 0); 110 rc = request_threaded_irq(ie->vector, NULL, idxd_misc_thread, 0, "idxd-misc", ie); 111 if (rc < 0) { 112 dev_err(dev, "Failed to allocate misc interrupt.\n"); 113 goto err_misc_irq; 114 } 115 dev_dbg(dev, "Requested idxd-misc handler on msix vector %d\n", ie->vector); 116 117 for (i = 0; i < idxd->max_wqs; i++) { 118 int msix_idx = i + 1; 119 120 ie = idxd_get_ie(idxd, msix_idx); 121 ie->id = msix_idx; 122 ie->int_handle = INVALID_INT_HANDLE; 123 ie->pasid = IOMMU_PASID_INVALID; 124 125 spin_lock_init(&ie->list_lock); 126 init_llist_head(&ie->pending_llist); 127 INIT_LIST_HEAD(&ie->work_list); 128 } 129 130 idxd_unmask_error_interrupts(idxd); 131 return 0; 132 133 err_misc_irq: 134 idxd_mask_error_interrupts(idxd); 135 pci_free_irq_vectors(pdev); 136 dev_err(dev, "No usable interrupts\n"); 137 return rc; 138 } 139 140 static void idxd_cleanup_interrupts(struct idxd_device *idxd) 141 { 142 struct pci_dev *pdev = idxd->pdev; 143 struct idxd_irq_entry *ie; 144 int msixcnt; 145 146 msixcnt = pci_msix_vec_count(pdev); 147 if (msixcnt <= 0) 148 return; 149 150 ie = idxd_get_ie(idxd, 0); 151 idxd_mask_error_interrupts(idxd); 152 free_irq(ie->vector, ie); 153 pci_free_irq_vectors(pdev); 154 } 155 156 static int idxd_setup_wqs(struct idxd_device *idxd) 157 { 158 struct device *dev = &idxd->pdev->dev; 159 struct idxd_wq *wq; 160 struct device *conf_dev; 161 int i, rc; 162 163 idxd->wqs = kcalloc_node(idxd->max_wqs, sizeof(struct idxd_wq *), 164 GFP_KERNEL, dev_to_node(dev)); 165 if (!idxd->wqs) 166 return -ENOMEM; 167 168 idxd->wq_enable_map = bitmap_zalloc_node(idxd->max_wqs, GFP_KERNEL, dev_to_node(dev)); 169 if (!idxd->wq_enable_map) { 170 kfree(idxd->wqs); 171 return -ENOMEM; 172 } 173 174 for (i = 0; i < idxd->max_wqs; i++) { 175 wq = kzalloc_node(sizeof(*wq), GFP_KERNEL, dev_to_node(dev)); 176 if (!wq) { 177 rc = -ENOMEM; 178 goto err; 179 } 180 181 idxd_dev_set_type(&wq->idxd_dev, IDXD_DEV_WQ); 182 conf_dev = wq_confdev(wq); 183 wq->id = i; 184 wq->idxd = idxd; 185 device_initialize(wq_confdev(wq)); 186 conf_dev->parent = idxd_confdev(idxd); 187 conf_dev->bus = &dsa_bus_type; 188 conf_dev->type = &idxd_wq_device_type; 189 rc = dev_set_name(conf_dev, "wq%d.%d", idxd->id, wq->id); 190 if (rc < 0) { 191 put_device(conf_dev); 192 goto err; 193 } 194 195 mutex_init(&wq->wq_lock); 196 init_waitqueue_head(&wq->err_queue); 197 init_completion(&wq->wq_dead); 198 init_completion(&wq->wq_resurrect); 199 wq->max_xfer_bytes = WQ_DEFAULT_MAX_XFER; 200 idxd_wq_set_max_batch_size(idxd->data->type, wq, WQ_DEFAULT_MAX_BATCH); 201 wq->enqcmds_retries = IDXD_ENQCMDS_RETRIES; 202 wq->wqcfg = kzalloc_node(idxd->wqcfg_size, GFP_KERNEL, dev_to_node(dev)); 203 if (!wq->wqcfg) { 204 put_device(conf_dev); 205 rc = -ENOMEM; 206 goto err; 207 } 208 209 if (idxd->hw.wq_cap.op_config) { 210 wq->opcap_bmap = bitmap_zalloc(IDXD_MAX_OPCAP_BITS, GFP_KERNEL); 211 if (!wq->opcap_bmap) { 212 put_device(conf_dev); 213 rc = -ENOMEM; 214 goto err; 215 } 216 bitmap_copy(wq->opcap_bmap, idxd->opcap_bmap, IDXD_MAX_OPCAP_BITS); 217 } 218 mutex_init(&wq->uc_lock); 219 xa_init(&wq->upasid_xa); 220 idxd->wqs[i] = wq; 221 } 222 223 return 0; 224 225 err: 226 while (--i >= 0) { 227 wq = idxd->wqs[i]; 228 conf_dev = wq_confdev(wq); 229 put_device(conf_dev); 230 } 231 return rc; 232 } 233 234 static int idxd_setup_engines(struct idxd_device *idxd) 235 { 236 struct idxd_engine *engine; 237 struct device *dev = &idxd->pdev->dev; 238 struct device *conf_dev; 239 int i, rc; 240 241 idxd->engines = kcalloc_node(idxd->max_engines, sizeof(struct idxd_engine *), 242 GFP_KERNEL, dev_to_node(dev)); 243 if (!idxd->engines) 244 return -ENOMEM; 245 246 for (i = 0; i < idxd->max_engines; i++) { 247 engine = kzalloc_node(sizeof(*engine), GFP_KERNEL, dev_to_node(dev)); 248 if (!engine) { 249 rc = -ENOMEM; 250 goto err; 251 } 252 253 idxd_dev_set_type(&engine->idxd_dev, IDXD_DEV_ENGINE); 254 conf_dev = engine_confdev(engine); 255 engine->id = i; 256 engine->idxd = idxd; 257 device_initialize(conf_dev); 258 conf_dev->parent = idxd_confdev(idxd); 259 conf_dev->bus = &dsa_bus_type; 260 conf_dev->type = &idxd_engine_device_type; 261 rc = dev_set_name(conf_dev, "engine%d.%d", idxd->id, engine->id); 262 if (rc < 0) { 263 put_device(conf_dev); 264 goto err; 265 } 266 267 idxd->engines[i] = engine; 268 } 269 270 return 0; 271 272 err: 273 while (--i >= 0) { 274 engine = idxd->engines[i]; 275 conf_dev = engine_confdev(engine); 276 put_device(conf_dev); 277 } 278 return rc; 279 } 280 281 static int idxd_setup_groups(struct idxd_device *idxd) 282 { 283 struct device *dev = &idxd->pdev->dev; 284 struct device *conf_dev; 285 struct idxd_group *group; 286 int i, rc; 287 288 idxd->groups = kcalloc_node(idxd->max_groups, sizeof(struct idxd_group *), 289 GFP_KERNEL, dev_to_node(dev)); 290 if (!idxd->groups) 291 return -ENOMEM; 292 293 for (i = 0; i < idxd->max_groups; i++) { 294 group = kzalloc_node(sizeof(*group), GFP_KERNEL, dev_to_node(dev)); 295 if (!group) { 296 rc = -ENOMEM; 297 goto err; 298 } 299 300 idxd_dev_set_type(&group->idxd_dev, IDXD_DEV_GROUP); 301 conf_dev = group_confdev(group); 302 group->id = i; 303 group->idxd = idxd; 304 device_initialize(conf_dev); 305 conf_dev->parent = idxd_confdev(idxd); 306 conf_dev->bus = &dsa_bus_type; 307 conf_dev->type = &idxd_group_device_type; 308 rc = dev_set_name(conf_dev, "group%d.%d", idxd->id, group->id); 309 if (rc < 0) { 310 put_device(conf_dev); 311 goto err; 312 } 313 314 idxd->groups[i] = group; 315 if (idxd->hw.version <= DEVICE_VERSION_2 && !tc_override) { 316 group->tc_a = 1; 317 group->tc_b = 1; 318 } else { 319 group->tc_a = -1; 320 group->tc_b = -1; 321 } 322 /* 323 * The default value is the same as the value of 324 * total read buffers in GRPCAP. 325 */ 326 group->rdbufs_allowed = idxd->max_rdbufs; 327 } 328 329 return 0; 330 331 err: 332 while (--i >= 0) { 333 group = idxd->groups[i]; 334 put_device(group_confdev(group)); 335 } 336 return rc; 337 } 338 339 static void idxd_cleanup_internals(struct idxd_device *idxd) 340 { 341 int i; 342 343 for (i = 0; i < idxd->max_groups; i++) 344 put_device(group_confdev(idxd->groups[i])); 345 for (i = 0; i < idxd->max_engines; i++) 346 put_device(engine_confdev(idxd->engines[i])); 347 for (i = 0; i < idxd->max_wqs; i++) 348 put_device(wq_confdev(idxd->wqs[i])); 349 destroy_workqueue(idxd->wq); 350 } 351 352 static int idxd_init_evl(struct idxd_device *idxd) 353 { 354 struct device *dev = &idxd->pdev->dev; 355 unsigned int evl_cache_size; 356 struct idxd_evl *evl; 357 const char *idxd_name; 358 359 if (idxd->hw.gen_cap.evl_support == 0) 360 return 0; 361 362 evl = kzalloc_node(sizeof(*evl), GFP_KERNEL, dev_to_node(dev)); 363 if (!evl) 364 return -ENOMEM; 365 366 mutex_init(&evl->lock); 367 evl->size = IDXD_EVL_SIZE_MIN; 368 369 idxd_name = dev_name(idxd_confdev(idxd)); 370 evl_cache_size = sizeof(struct idxd_evl_fault) + evl_ent_size(idxd); 371 /* 372 * Since completion record in evl_cache will be copied to user 373 * when handling completion record page fault, need to create 374 * the cache suitable for user copy. 375 */ 376 idxd->evl_cache = kmem_cache_create_usercopy(idxd_name, evl_cache_size, 377 0, 0, 0, evl_cache_size, 378 NULL); 379 if (!idxd->evl_cache) { 380 kfree(evl); 381 return -ENOMEM; 382 } 383 384 idxd->evl = evl; 385 return 0; 386 } 387 388 static int idxd_setup_internals(struct idxd_device *idxd) 389 { 390 struct device *dev = &idxd->pdev->dev; 391 int rc, i; 392 393 init_waitqueue_head(&idxd->cmd_waitq); 394 395 rc = idxd_setup_wqs(idxd); 396 if (rc < 0) 397 goto err_wqs; 398 399 rc = idxd_setup_engines(idxd); 400 if (rc < 0) 401 goto err_engine; 402 403 rc = idxd_setup_groups(idxd); 404 if (rc < 0) 405 goto err_group; 406 407 idxd->wq = create_workqueue(dev_name(dev)); 408 if (!idxd->wq) { 409 rc = -ENOMEM; 410 goto err_wkq_create; 411 } 412 413 rc = idxd_init_evl(idxd); 414 if (rc < 0) 415 goto err_evl; 416 417 return 0; 418 419 err_evl: 420 destroy_workqueue(idxd->wq); 421 err_wkq_create: 422 for (i = 0; i < idxd->max_groups; i++) 423 put_device(group_confdev(idxd->groups[i])); 424 err_group: 425 for (i = 0; i < idxd->max_engines; i++) 426 put_device(engine_confdev(idxd->engines[i])); 427 err_engine: 428 for (i = 0; i < idxd->max_wqs; i++) 429 put_device(wq_confdev(idxd->wqs[i])); 430 err_wqs: 431 return rc; 432 } 433 434 static void idxd_read_table_offsets(struct idxd_device *idxd) 435 { 436 union offsets_reg offsets; 437 struct device *dev = &idxd->pdev->dev; 438 439 offsets.bits[0] = ioread64(idxd->reg_base + IDXD_TABLE_OFFSET); 440 offsets.bits[1] = ioread64(idxd->reg_base + IDXD_TABLE_OFFSET + sizeof(u64)); 441 idxd->grpcfg_offset = offsets.grpcfg * IDXD_TABLE_MULT; 442 dev_dbg(dev, "IDXD Group Config Offset: %#x\n", idxd->grpcfg_offset); 443 idxd->wqcfg_offset = offsets.wqcfg * IDXD_TABLE_MULT; 444 dev_dbg(dev, "IDXD Work Queue Config Offset: %#x\n", idxd->wqcfg_offset); 445 idxd->msix_perm_offset = offsets.msix_perm * IDXD_TABLE_MULT; 446 dev_dbg(dev, "IDXD MSIX Permission Offset: %#x\n", idxd->msix_perm_offset); 447 idxd->perfmon_offset = offsets.perfmon * IDXD_TABLE_MULT; 448 dev_dbg(dev, "IDXD Perfmon Offset: %#x\n", idxd->perfmon_offset); 449 } 450 451 void multi_u64_to_bmap(unsigned long *bmap, u64 *val, int count) 452 { 453 int i, j, nr; 454 455 for (i = 0, nr = 0; i < count; i++) { 456 for (j = 0; j < BITS_PER_LONG_LONG; j++) { 457 if (val[i] & BIT(j)) 458 set_bit(nr, bmap); 459 nr++; 460 } 461 } 462 } 463 464 static void idxd_read_caps(struct idxd_device *idxd) 465 { 466 struct device *dev = &idxd->pdev->dev; 467 int i; 468 469 /* reading generic capabilities */ 470 idxd->hw.gen_cap.bits = ioread64(idxd->reg_base + IDXD_GENCAP_OFFSET); 471 dev_dbg(dev, "gen_cap: %#llx\n", idxd->hw.gen_cap.bits); 472 473 if (idxd->hw.gen_cap.cmd_cap) { 474 idxd->hw.cmd_cap = ioread32(idxd->reg_base + IDXD_CMDCAP_OFFSET); 475 dev_dbg(dev, "cmd_cap: %#x\n", idxd->hw.cmd_cap); 476 } 477 478 /* reading command capabilities */ 479 if (idxd->hw.cmd_cap & BIT(IDXD_CMD_REQUEST_INT_HANDLE)) 480 idxd->request_int_handles = true; 481 482 idxd->max_xfer_bytes = 1ULL << idxd->hw.gen_cap.max_xfer_shift; 483 dev_dbg(dev, "max xfer size: %llu bytes\n", idxd->max_xfer_bytes); 484 idxd_set_max_batch_size(idxd->data->type, idxd, 1U << idxd->hw.gen_cap.max_batch_shift); 485 dev_dbg(dev, "max batch size: %u\n", idxd->max_batch_size); 486 if (idxd->hw.gen_cap.config_en) 487 set_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags); 488 489 /* reading group capabilities */ 490 idxd->hw.group_cap.bits = 491 ioread64(idxd->reg_base + IDXD_GRPCAP_OFFSET); 492 dev_dbg(dev, "group_cap: %#llx\n", idxd->hw.group_cap.bits); 493 idxd->max_groups = idxd->hw.group_cap.num_groups; 494 dev_dbg(dev, "max groups: %u\n", idxd->max_groups); 495 idxd->max_rdbufs = idxd->hw.group_cap.total_rdbufs; 496 dev_dbg(dev, "max read buffers: %u\n", idxd->max_rdbufs); 497 idxd->nr_rdbufs = idxd->max_rdbufs; 498 499 /* read engine capabilities */ 500 idxd->hw.engine_cap.bits = 501 ioread64(idxd->reg_base + IDXD_ENGCAP_OFFSET); 502 dev_dbg(dev, "engine_cap: %#llx\n", idxd->hw.engine_cap.bits); 503 idxd->max_engines = idxd->hw.engine_cap.num_engines; 504 dev_dbg(dev, "max engines: %u\n", idxd->max_engines); 505 506 /* read workqueue capabilities */ 507 idxd->hw.wq_cap.bits = ioread64(idxd->reg_base + IDXD_WQCAP_OFFSET); 508 dev_dbg(dev, "wq_cap: %#llx\n", idxd->hw.wq_cap.bits); 509 idxd->max_wq_size = idxd->hw.wq_cap.total_wq_size; 510 dev_dbg(dev, "total workqueue size: %u\n", idxd->max_wq_size); 511 idxd->max_wqs = idxd->hw.wq_cap.num_wqs; 512 dev_dbg(dev, "max workqueues: %u\n", idxd->max_wqs); 513 idxd->wqcfg_size = 1 << (idxd->hw.wq_cap.wqcfg_size + IDXD_WQCFG_MIN); 514 dev_dbg(dev, "wqcfg size: %u\n", idxd->wqcfg_size); 515 516 /* reading operation capabilities */ 517 for (i = 0; i < 4; i++) { 518 idxd->hw.opcap.bits[i] = ioread64(idxd->reg_base + 519 IDXD_OPCAP_OFFSET + i * sizeof(u64)); 520 dev_dbg(dev, "opcap[%d]: %#llx\n", i, idxd->hw.opcap.bits[i]); 521 } 522 multi_u64_to_bmap(idxd->opcap_bmap, &idxd->hw.opcap.bits[0], 4); 523 524 /* read iaa cap */ 525 if (idxd->data->type == IDXD_TYPE_IAX && idxd->hw.version >= DEVICE_VERSION_2) 526 idxd->hw.iaa_cap.bits = ioread64(idxd->reg_base + IDXD_IAACAP_OFFSET); 527 } 528 529 static struct idxd_device *idxd_alloc(struct pci_dev *pdev, struct idxd_driver_data *data) 530 { 531 struct device *dev = &pdev->dev; 532 struct device *conf_dev; 533 struct idxd_device *idxd; 534 int rc; 535 536 idxd = kzalloc_node(sizeof(*idxd), GFP_KERNEL, dev_to_node(dev)); 537 if (!idxd) 538 return NULL; 539 540 conf_dev = idxd_confdev(idxd); 541 idxd->pdev = pdev; 542 idxd->data = data; 543 idxd_dev_set_type(&idxd->idxd_dev, idxd->data->type); 544 idxd->id = ida_alloc(&idxd_ida, GFP_KERNEL); 545 if (idxd->id < 0) 546 return NULL; 547 548 idxd->opcap_bmap = bitmap_zalloc_node(IDXD_MAX_OPCAP_BITS, GFP_KERNEL, dev_to_node(dev)); 549 if (!idxd->opcap_bmap) { 550 ida_free(&idxd_ida, idxd->id); 551 return NULL; 552 } 553 554 device_initialize(conf_dev); 555 conf_dev->parent = dev; 556 conf_dev->bus = &dsa_bus_type; 557 conf_dev->type = idxd->data->dev_type; 558 rc = dev_set_name(conf_dev, "%s%d", idxd->data->name_prefix, idxd->id); 559 if (rc < 0) { 560 put_device(conf_dev); 561 return NULL; 562 } 563 564 spin_lock_init(&idxd->dev_lock); 565 spin_lock_init(&idxd->cmd_lock); 566 567 return idxd; 568 } 569 570 static int idxd_enable_system_pasid(struct idxd_device *idxd) 571 { 572 struct pci_dev *pdev = idxd->pdev; 573 struct device *dev = &pdev->dev; 574 struct iommu_domain *domain; 575 ioasid_t pasid; 576 int ret; 577 578 /* 579 * Attach a global PASID to the DMA domain so that we can use ENQCMDS 580 * to submit work on buffers mapped by DMA API. 581 */ 582 domain = iommu_get_domain_for_dev(dev); 583 if (!domain) 584 return -EPERM; 585 586 pasid = iommu_alloc_global_pasid(dev); 587 if (pasid == IOMMU_PASID_INVALID) 588 return -ENOSPC; 589 590 /* 591 * DMA domain is owned by the driver, it should support all valid 592 * types such as DMA-FQ, identity, etc. 593 */ 594 ret = iommu_attach_device_pasid(domain, dev, pasid, NULL); 595 if (ret) { 596 dev_err(dev, "failed to attach device pasid %d, domain type %d", 597 pasid, domain->type); 598 iommu_free_global_pasid(pasid); 599 return ret; 600 } 601 602 /* Since we set user privilege for kernel DMA, enable completion IRQ */ 603 idxd_set_user_intr(idxd, 1); 604 idxd->pasid = pasid; 605 606 return ret; 607 } 608 609 static void idxd_disable_system_pasid(struct idxd_device *idxd) 610 { 611 struct pci_dev *pdev = idxd->pdev; 612 struct device *dev = &pdev->dev; 613 struct iommu_domain *domain; 614 615 domain = iommu_get_domain_for_dev(dev); 616 if (!domain) 617 return; 618 619 iommu_detach_device_pasid(domain, dev, idxd->pasid); 620 iommu_free_global_pasid(idxd->pasid); 621 622 idxd_set_user_intr(idxd, 0); 623 idxd->sva = NULL; 624 idxd->pasid = IOMMU_PASID_INVALID; 625 } 626 627 static int idxd_enable_sva(struct pci_dev *pdev) 628 { 629 int ret; 630 631 ret = iommu_dev_enable_feature(&pdev->dev, IOMMU_DEV_FEAT_IOPF); 632 if (ret) 633 return ret; 634 635 ret = iommu_dev_enable_feature(&pdev->dev, IOMMU_DEV_FEAT_SVA); 636 if (ret) 637 iommu_dev_disable_feature(&pdev->dev, IOMMU_DEV_FEAT_IOPF); 638 639 return ret; 640 } 641 642 static void idxd_disable_sva(struct pci_dev *pdev) 643 { 644 iommu_dev_disable_feature(&pdev->dev, IOMMU_DEV_FEAT_SVA); 645 iommu_dev_disable_feature(&pdev->dev, IOMMU_DEV_FEAT_IOPF); 646 } 647 648 static int idxd_probe(struct idxd_device *idxd) 649 { 650 struct pci_dev *pdev = idxd->pdev; 651 struct device *dev = &pdev->dev; 652 int rc; 653 654 dev_dbg(dev, "%s entered and resetting device\n", __func__); 655 rc = idxd_device_init_reset(idxd); 656 if (rc < 0) 657 return rc; 658 659 dev_dbg(dev, "IDXD reset complete\n"); 660 661 if (IS_ENABLED(CONFIG_INTEL_IDXD_SVM) && sva) { 662 if (idxd_enable_sva(pdev)) { 663 dev_warn(dev, "Unable to turn on user SVA feature.\n"); 664 } else { 665 set_bit(IDXD_FLAG_USER_PASID_ENABLED, &idxd->flags); 666 667 rc = idxd_enable_system_pasid(idxd); 668 if (rc) 669 dev_warn(dev, "No in-kernel DMA with PASID. %d\n", rc); 670 else 671 set_bit(IDXD_FLAG_PASID_ENABLED, &idxd->flags); 672 } 673 } else if (!sva) { 674 dev_warn(dev, "User forced SVA off via module param.\n"); 675 } 676 677 idxd_read_caps(idxd); 678 idxd_read_table_offsets(idxd); 679 680 rc = idxd_setup_internals(idxd); 681 if (rc) 682 goto err; 683 684 /* If the configs are readonly, then load them from device */ 685 if (!test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags)) { 686 dev_dbg(dev, "Loading RO device config\n"); 687 rc = idxd_device_load_config(idxd); 688 if (rc < 0) 689 goto err_config; 690 } 691 692 rc = idxd_setup_interrupts(idxd); 693 if (rc) 694 goto err_config; 695 696 idxd->major = idxd_cdev_get_major(idxd); 697 698 rc = perfmon_pmu_init(idxd); 699 if (rc < 0) 700 dev_warn(dev, "Failed to initialize perfmon. No PMU support: %d\n", rc); 701 702 dev_dbg(dev, "IDXD device %d probed successfully\n", idxd->id); 703 return 0; 704 705 err_config: 706 idxd_cleanup_internals(idxd); 707 err: 708 if (device_pasid_enabled(idxd)) 709 idxd_disable_system_pasid(idxd); 710 if (device_user_pasid_enabled(idxd)) 711 idxd_disable_sva(pdev); 712 return rc; 713 } 714 715 static void idxd_cleanup(struct idxd_device *idxd) 716 { 717 perfmon_pmu_remove(idxd); 718 idxd_cleanup_interrupts(idxd); 719 idxd_cleanup_internals(idxd); 720 if (device_pasid_enabled(idxd)) 721 idxd_disable_system_pasid(idxd); 722 if (device_user_pasid_enabled(idxd)) 723 idxd_disable_sva(idxd->pdev); 724 } 725 726 static int idxd_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) 727 { 728 struct device *dev = &pdev->dev; 729 struct idxd_device *idxd; 730 struct idxd_driver_data *data = (struct idxd_driver_data *)id->driver_data; 731 int rc; 732 733 rc = pci_enable_device(pdev); 734 if (rc) 735 return rc; 736 737 dev_dbg(dev, "Alloc IDXD context\n"); 738 idxd = idxd_alloc(pdev, data); 739 if (!idxd) { 740 rc = -ENOMEM; 741 goto err_idxd_alloc; 742 } 743 744 dev_dbg(dev, "Mapping BARs\n"); 745 idxd->reg_base = pci_iomap(pdev, IDXD_MMIO_BAR, 0); 746 if (!idxd->reg_base) { 747 rc = -ENOMEM; 748 goto err_iomap; 749 } 750 751 dev_dbg(dev, "Set DMA masks\n"); 752 rc = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); 753 if (rc) 754 goto err; 755 756 dev_dbg(dev, "Set PCI master\n"); 757 pci_set_master(pdev); 758 pci_set_drvdata(pdev, idxd); 759 760 idxd->hw.version = ioread32(idxd->reg_base + IDXD_VER_OFFSET); 761 rc = idxd_probe(idxd); 762 if (rc) { 763 dev_err(dev, "Intel(R) IDXD DMA Engine init failed\n"); 764 goto err; 765 } 766 767 if (data->load_device_defaults) { 768 rc = data->load_device_defaults(idxd); 769 if (rc) 770 dev_warn(dev, "IDXD loading device defaults failed\n"); 771 } 772 773 rc = idxd_register_devices(idxd); 774 if (rc) { 775 dev_err(dev, "IDXD sysfs setup failed\n"); 776 goto err_dev_register; 777 } 778 779 rc = idxd_device_init_debugfs(idxd); 780 if (rc) 781 dev_warn(dev, "IDXD debugfs failed to setup\n"); 782 783 dev_info(&pdev->dev, "Intel(R) Accelerator Device (v%x)\n", 784 idxd->hw.version); 785 786 idxd->user_submission_safe = data->user_submission_safe; 787 788 return 0; 789 790 err_dev_register: 791 idxd_cleanup(idxd); 792 err: 793 pci_iounmap(pdev, idxd->reg_base); 794 err_iomap: 795 put_device(idxd_confdev(idxd)); 796 err_idxd_alloc: 797 pci_disable_device(pdev); 798 return rc; 799 } 800 801 void idxd_wqs_quiesce(struct idxd_device *idxd) 802 { 803 struct idxd_wq *wq; 804 int i; 805 806 for (i = 0; i < idxd->max_wqs; i++) { 807 wq = idxd->wqs[i]; 808 if (wq->state == IDXD_WQ_ENABLED && wq->type == IDXD_WQT_KERNEL) 809 idxd_wq_quiesce(wq); 810 } 811 } 812 813 static void idxd_shutdown(struct pci_dev *pdev) 814 { 815 struct idxd_device *idxd = pci_get_drvdata(pdev); 816 struct idxd_irq_entry *irq_entry; 817 int rc; 818 819 rc = idxd_device_disable(idxd); 820 if (rc) 821 dev_err(&pdev->dev, "Disabling device failed\n"); 822 823 irq_entry = &idxd->ie; 824 synchronize_irq(irq_entry->vector); 825 idxd_mask_error_interrupts(idxd); 826 flush_workqueue(idxd->wq); 827 } 828 829 static void idxd_remove(struct pci_dev *pdev) 830 { 831 struct idxd_device *idxd = pci_get_drvdata(pdev); 832 struct idxd_irq_entry *irq_entry; 833 834 idxd_unregister_devices(idxd); 835 /* 836 * When ->release() is called for the idxd->conf_dev, it frees all the memory related 837 * to the idxd context. The driver still needs those bits in order to do the rest of 838 * the cleanup. However, we do need to unbound the idxd sub-driver. So take a ref 839 * on the device here to hold off the freeing while allowing the idxd sub-driver 840 * to unbind. 841 */ 842 get_device(idxd_confdev(idxd)); 843 device_unregister(idxd_confdev(idxd)); 844 idxd_shutdown(pdev); 845 if (device_pasid_enabled(idxd)) 846 idxd_disable_system_pasid(idxd); 847 idxd_device_remove_debugfs(idxd); 848 849 irq_entry = idxd_get_ie(idxd, 0); 850 free_irq(irq_entry->vector, irq_entry); 851 pci_free_irq_vectors(pdev); 852 pci_iounmap(pdev, idxd->reg_base); 853 if (device_user_pasid_enabled(idxd)) 854 idxd_disable_sva(pdev); 855 pci_disable_device(pdev); 856 destroy_workqueue(idxd->wq); 857 perfmon_pmu_remove(idxd); 858 put_device(idxd_confdev(idxd)); 859 } 860 861 static struct pci_driver idxd_pci_driver = { 862 .name = DRV_NAME, 863 .id_table = idxd_pci_tbl, 864 .probe = idxd_pci_probe, 865 .remove = idxd_remove, 866 .shutdown = idxd_shutdown, 867 }; 868 869 static int __init idxd_init_module(void) 870 { 871 int err; 872 873 /* 874 * If the CPU does not support MOVDIR64B or ENQCMDS, there's no point in 875 * enumerating the device. We can not utilize it. 876 */ 877 if (!cpu_feature_enabled(X86_FEATURE_MOVDIR64B)) { 878 pr_warn("idxd driver failed to load without MOVDIR64B.\n"); 879 return -ENODEV; 880 } 881 882 if (!cpu_feature_enabled(X86_FEATURE_ENQCMD)) 883 pr_warn("Platform does not have ENQCMD(S) support.\n"); 884 else 885 support_enqcmd = true; 886 887 err = idxd_driver_register(&idxd_drv); 888 if (err < 0) 889 goto err_idxd_driver_register; 890 891 err = idxd_driver_register(&idxd_dmaengine_drv); 892 if (err < 0) 893 goto err_idxd_dmaengine_driver_register; 894 895 err = idxd_driver_register(&idxd_user_drv); 896 if (err < 0) 897 goto err_idxd_user_driver_register; 898 899 err = idxd_cdev_register(); 900 if (err) 901 goto err_cdev_register; 902 903 err = idxd_init_debugfs(); 904 if (err) 905 goto err_debugfs; 906 907 err = pci_register_driver(&idxd_pci_driver); 908 if (err) 909 goto err_pci_register; 910 911 return 0; 912 913 err_pci_register: 914 idxd_remove_debugfs(); 915 err_debugfs: 916 idxd_cdev_remove(); 917 err_cdev_register: 918 idxd_driver_unregister(&idxd_user_drv); 919 err_idxd_user_driver_register: 920 idxd_driver_unregister(&idxd_dmaengine_drv); 921 err_idxd_dmaengine_driver_register: 922 idxd_driver_unregister(&idxd_drv); 923 err_idxd_driver_register: 924 return err; 925 } 926 module_init(idxd_init_module); 927 928 static void __exit idxd_exit_module(void) 929 { 930 idxd_driver_unregister(&idxd_user_drv); 931 idxd_driver_unregister(&idxd_dmaengine_drv); 932 idxd_driver_unregister(&idxd_drv); 933 pci_unregister_driver(&idxd_pci_driver); 934 idxd_cdev_remove(); 935 idxd_remove_debugfs(); 936 } 937 module_exit(idxd_exit_module); 938