1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright(c) 2019 Intel Corporation. All rights rsvd. */ 3 #include <linux/init.h> 4 #include <linux/kernel.h> 5 #include <linux/module.h> 6 #include <linux/slab.h> 7 #include <linux/pci.h> 8 #include <linux/interrupt.h> 9 #include <linux/delay.h> 10 #include <linux/dma-mapping.h> 11 #include <linux/workqueue.h> 12 #include <linux/fs.h> 13 #include <linux/io-64-nonatomic-lo-hi.h> 14 #include <linux/device.h> 15 #include <linux/idr.h> 16 #include <linux/iommu.h> 17 #include <uapi/linux/idxd.h> 18 #include <linux/dmaengine.h> 19 #include "../dmaengine.h" 20 #include "registers.h" 21 #include "idxd.h" 22 #include "perfmon.h" 23 24 MODULE_VERSION(IDXD_DRIVER_VERSION); 25 MODULE_DESCRIPTION("Intel Data Streaming Accelerator and In-Memory Analytics Accelerator common driver"); 26 MODULE_LICENSE("GPL v2"); 27 MODULE_AUTHOR("Intel Corporation"); 28 MODULE_IMPORT_NS("IDXD"); 29 30 static bool sva = true; 31 module_param(sva, bool, 0644); 32 MODULE_PARM_DESC(sva, "Toggle SVA support on/off"); 33 34 bool tc_override; 35 module_param(tc_override, bool, 0644); 36 MODULE_PARM_DESC(tc_override, "Override traffic class defaults"); 37 38 #define DRV_NAME "idxd" 39 40 bool support_enqcmd; 41 DEFINE_IDA(idxd_ida); 42 43 static struct idxd_driver_data idxd_driver_data[] = { 44 [IDXD_TYPE_DSA] = { 45 .name_prefix = "dsa", 46 .type = IDXD_TYPE_DSA, 47 .compl_size = sizeof(struct dsa_completion_record), 48 .align = 32, 49 .dev_type = &dsa_device_type, 50 .evl_cr_off = offsetof(struct dsa_evl_entry, cr), 51 .user_submission_safe = false, /* See INTEL-SA-01084 security advisory */ 52 .cr_status_off = offsetof(struct dsa_completion_record, status), 53 .cr_result_off = offsetof(struct dsa_completion_record, result), 54 }, 55 [IDXD_TYPE_IAX] = { 56 .name_prefix = "iax", 57 .type = IDXD_TYPE_IAX, 58 .compl_size = sizeof(struct iax_completion_record), 59 .align = 64, 60 .dev_type = &iax_device_type, 61 .evl_cr_off = offsetof(struct iax_evl_entry, cr), 62 .user_submission_safe = false, /* See INTEL-SA-01084 security advisory */ 63 .cr_status_off = offsetof(struct iax_completion_record, status), 64 .cr_result_off = offsetof(struct iax_completion_record, error_code), 65 .load_device_defaults = idxd_load_iaa_device_defaults, 66 }, 67 }; 68 69 static struct pci_device_id idxd_pci_tbl[] = { 70 /* DSA ver 1.0 platforms */ 71 { PCI_DEVICE_DATA(INTEL, DSA_SPR0, &idxd_driver_data[IDXD_TYPE_DSA]) }, 72 /* DSA on GNR-D platforms */ 73 { PCI_DEVICE_DATA(INTEL, DSA_GNRD, &idxd_driver_data[IDXD_TYPE_DSA]) }, 74 /* DSA on DMR platforms */ 75 { PCI_DEVICE_DATA(INTEL, DSA_DMR, &idxd_driver_data[IDXD_TYPE_DSA]) }, 76 77 /* IAX ver 1.0 platforms */ 78 { PCI_DEVICE_DATA(INTEL, IAX_SPR0, &idxd_driver_data[IDXD_TYPE_IAX]) }, 79 /* IAA on DMR platforms */ 80 { PCI_DEVICE_DATA(INTEL, IAA_DMR, &idxd_driver_data[IDXD_TYPE_IAX]) }, 81 /* IAA PTL platforms */ 82 { PCI_DEVICE_DATA(INTEL, IAA_PTL, &idxd_driver_data[IDXD_TYPE_IAX]) }, 83 { 0, } 84 }; 85 MODULE_DEVICE_TABLE(pci, idxd_pci_tbl); 86 87 static int idxd_setup_interrupts(struct idxd_device *idxd) 88 { 89 struct pci_dev *pdev = idxd->pdev; 90 struct device *dev = &pdev->dev; 91 struct idxd_irq_entry *ie; 92 int i, msixcnt; 93 int rc = 0; 94 95 msixcnt = pci_msix_vec_count(pdev); 96 if (msixcnt < 0) { 97 dev_err(dev, "Not MSI-X interrupt capable.\n"); 98 return -ENOSPC; 99 } 100 idxd->irq_cnt = msixcnt; 101 102 rc = pci_alloc_irq_vectors(pdev, msixcnt, msixcnt, PCI_IRQ_MSIX); 103 if (rc != msixcnt) { 104 dev_err(dev, "Failed enabling %d MSIX entries: %d\n", msixcnt, rc); 105 return -ENOSPC; 106 } 107 dev_dbg(dev, "Enabled %d msix vectors\n", msixcnt); 108 109 110 ie = idxd_get_ie(idxd, 0); 111 ie->vector = pci_irq_vector(pdev, 0); 112 rc = request_threaded_irq(ie->vector, NULL, idxd_misc_thread, 0, "idxd-misc", ie); 113 if (rc < 0) { 114 dev_err(dev, "Failed to allocate misc interrupt.\n"); 115 goto err_misc_irq; 116 } 117 dev_dbg(dev, "Requested idxd-misc handler on msix vector %d\n", ie->vector); 118 119 for (i = 0; i < idxd->max_wqs; i++) { 120 int msix_idx = i + 1; 121 122 ie = idxd_get_ie(idxd, msix_idx); 123 ie->id = msix_idx; 124 ie->int_handle = INVALID_INT_HANDLE; 125 ie->pasid = IOMMU_PASID_INVALID; 126 127 spin_lock_init(&ie->list_lock); 128 init_llist_head(&ie->pending_llist); 129 INIT_LIST_HEAD(&ie->work_list); 130 } 131 132 idxd_unmask_error_interrupts(idxd); 133 return 0; 134 135 err_misc_irq: 136 idxd_mask_error_interrupts(idxd); 137 pci_free_irq_vectors(pdev); 138 dev_err(dev, "No usable interrupts\n"); 139 return rc; 140 } 141 142 static void idxd_cleanup_interrupts(struct idxd_device *idxd) 143 { 144 struct pci_dev *pdev = idxd->pdev; 145 struct idxd_irq_entry *ie; 146 int msixcnt; 147 148 msixcnt = pci_msix_vec_count(pdev); 149 if (msixcnt <= 0) 150 return; 151 152 ie = idxd_get_ie(idxd, 0); 153 idxd_mask_error_interrupts(idxd); 154 free_irq(ie->vector, ie); 155 pci_free_irq_vectors(pdev); 156 } 157 158 static int idxd_setup_wqs(struct idxd_device *idxd) 159 { 160 struct device *dev = &idxd->pdev->dev; 161 struct idxd_wq *wq; 162 struct device *conf_dev; 163 int i, rc; 164 165 idxd->wqs = kcalloc_node(idxd->max_wqs, sizeof(struct idxd_wq *), 166 GFP_KERNEL, dev_to_node(dev)); 167 if (!idxd->wqs) 168 return -ENOMEM; 169 170 idxd->wq_enable_map = bitmap_zalloc_node(idxd->max_wqs, GFP_KERNEL, dev_to_node(dev)); 171 if (!idxd->wq_enable_map) { 172 kfree(idxd->wqs); 173 return -ENOMEM; 174 } 175 176 for (i = 0; i < idxd->max_wqs; i++) { 177 wq = kzalloc_node(sizeof(*wq), GFP_KERNEL, dev_to_node(dev)); 178 if (!wq) { 179 rc = -ENOMEM; 180 goto err; 181 } 182 183 idxd_dev_set_type(&wq->idxd_dev, IDXD_DEV_WQ); 184 conf_dev = wq_confdev(wq); 185 wq->id = i; 186 wq->idxd = idxd; 187 device_initialize(wq_confdev(wq)); 188 conf_dev->parent = idxd_confdev(idxd); 189 conf_dev->bus = &dsa_bus_type; 190 conf_dev->type = &idxd_wq_device_type; 191 rc = dev_set_name(conf_dev, "wq%d.%d", idxd->id, wq->id); 192 if (rc < 0) { 193 put_device(conf_dev); 194 goto err; 195 } 196 197 mutex_init(&wq->wq_lock); 198 init_waitqueue_head(&wq->err_queue); 199 init_completion(&wq->wq_dead); 200 init_completion(&wq->wq_resurrect); 201 wq->max_xfer_bytes = WQ_DEFAULT_MAX_XFER; 202 idxd_wq_set_max_batch_size(idxd->data->type, wq, WQ_DEFAULT_MAX_BATCH); 203 wq->enqcmds_retries = IDXD_ENQCMDS_RETRIES; 204 wq->wqcfg = kzalloc_node(idxd->wqcfg_size, GFP_KERNEL, dev_to_node(dev)); 205 if (!wq->wqcfg) { 206 put_device(conf_dev); 207 rc = -ENOMEM; 208 goto err; 209 } 210 211 if (idxd->hw.wq_cap.op_config) { 212 wq->opcap_bmap = bitmap_zalloc(IDXD_MAX_OPCAP_BITS, GFP_KERNEL); 213 if (!wq->opcap_bmap) { 214 put_device(conf_dev); 215 rc = -ENOMEM; 216 goto err; 217 } 218 bitmap_copy(wq->opcap_bmap, idxd->opcap_bmap, IDXD_MAX_OPCAP_BITS); 219 } 220 mutex_init(&wq->uc_lock); 221 xa_init(&wq->upasid_xa); 222 idxd->wqs[i] = wq; 223 } 224 225 return 0; 226 227 err: 228 while (--i >= 0) { 229 wq = idxd->wqs[i]; 230 conf_dev = wq_confdev(wq); 231 put_device(conf_dev); 232 } 233 return rc; 234 } 235 236 static int idxd_setup_engines(struct idxd_device *idxd) 237 { 238 struct idxd_engine *engine; 239 struct device *dev = &idxd->pdev->dev; 240 struct device *conf_dev; 241 int i, rc; 242 243 idxd->engines = kcalloc_node(idxd->max_engines, sizeof(struct idxd_engine *), 244 GFP_KERNEL, dev_to_node(dev)); 245 if (!idxd->engines) 246 return -ENOMEM; 247 248 for (i = 0; i < idxd->max_engines; i++) { 249 engine = kzalloc_node(sizeof(*engine), GFP_KERNEL, dev_to_node(dev)); 250 if (!engine) { 251 rc = -ENOMEM; 252 goto err; 253 } 254 255 idxd_dev_set_type(&engine->idxd_dev, IDXD_DEV_ENGINE); 256 conf_dev = engine_confdev(engine); 257 engine->id = i; 258 engine->idxd = idxd; 259 device_initialize(conf_dev); 260 conf_dev->parent = idxd_confdev(idxd); 261 conf_dev->bus = &dsa_bus_type; 262 conf_dev->type = &idxd_engine_device_type; 263 rc = dev_set_name(conf_dev, "engine%d.%d", idxd->id, engine->id); 264 if (rc < 0) { 265 put_device(conf_dev); 266 goto err; 267 } 268 269 idxd->engines[i] = engine; 270 } 271 272 return 0; 273 274 err: 275 while (--i >= 0) { 276 engine = idxd->engines[i]; 277 conf_dev = engine_confdev(engine); 278 put_device(conf_dev); 279 } 280 return rc; 281 } 282 283 static int idxd_setup_groups(struct idxd_device *idxd) 284 { 285 struct device *dev = &idxd->pdev->dev; 286 struct device *conf_dev; 287 struct idxd_group *group; 288 int i, rc; 289 290 idxd->groups = kcalloc_node(idxd->max_groups, sizeof(struct idxd_group *), 291 GFP_KERNEL, dev_to_node(dev)); 292 if (!idxd->groups) 293 return -ENOMEM; 294 295 for (i = 0; i < idxd->max_groups; i++) { 296 group = kzalloc_node(sizeof(*group), GFP_KERNEL, dev_to_node(dev)); 297 if (!group) { 298 rc = -ENOMEM; 299 goto err; 300 } 301 302 idxd_dev_set_type(&group->idxd_dev, IDXD_DEV_GROUP); 303 conf_dev = group_confdev(group); 304 group->id = i; 305 group->idxd = idxd; 306 device_initialize(conf_dev); 307 conf_dev->parent = idxd_confdev(idxd); 308 conf_dev->bus = &dsa_bus_type; 309 conf_dev->type = &idxd_group_device_type; 310 rc = dev_set_name(conf_dev, "group%d.%d", idxd->id, group->id); 311 if (rc < 0) { 312 put_device(conf_dev); 313 goto err; 314 } 315 316 idxd->groups[i] = group; 317 if (idxd->hw.version <= DEVICE_VERSION_2 && !tc_override) { 318 group->tc_a = 1; 319 group->tc_b = 1; 320 } else { 321 group->tc_a = -1; 322 group->tc_b = -1; 323 } 324 /* 325 * The default value is the same as the value of 326 * total read buffers in GRPCAP. 327 */ 328 group->rdbufs_allowed = idxd->max_rdbufs; 329 } 330 331 return 0; 332 333 err: 334 while (--i >= 0) { 335 group = idxd->groups[i]; 336 put_device(group_confdev(group)); 337 } 338 return rc; 339 } 340 341 static void idxd_cleanup_internals(struct idxd_device *idxd) 342 { 343 int i; 344 345 for (i = 0; i < idxd->max_groups; i++) 346 put_device(group_confdev(idxd->groups[i])); 347 for (i = 0; i < idxd->max_engines; i++) 348 put_device(engine_confdev(idxd->engines[i])); 349 for (i = 0; i < idxd->max_wqs; i++) 350 put_device(wq_confdev(idxd->wqs[i])); 351 destroy_workqueue(idxd->wq); 352 } 353 354 static int idxd_init_evl(struct idxd_device *idxd) 355 { 356 struct device *dev = &idxd->pdev->dev; 357 unsigned int evl_cache_size; 358 struct idxd_evl *evl; 359 const char *idxd_name; 360 361 if (idxd->hw.gen_cap.evl_support == 0) 362 return 0; 363 364 evl = kzalloc_node(sizeof(*evl), GFP_KERNEL, dev_to_node(dev)); 365 if (!evl) 366 return -ENOMEM; 367 368 mutex_init(&evl->lock); 369 evl->size = IDXD_EVL_SIZE_MIN; 370 371 idxd_name = dev_name(idxd_confdev(idxd)); 372 evl_cache_size = sizeof(struct idxd_evl_fault) + evl_ent_size(idxd); 373 /* 374 * Since completion record in evl_cache will be copied to user 375 * when handling completion record page fault, need to create 376 * the cache suitable for user copy. 377 */ 378 idxd->evl_cache = kmem_cache_create_usercopy(idxd_name, evl_cache_size, 379 0, 0, 0, evl_cache_size, 380 NULL); 381 if (!idxd->evl_cache) { 382 kfree(evl); 383 return -ENOMEM; 384 } 385 386 idxd->evl = evl; 387 return 0; 388 } 389 390 static int idxd_setup_internals(struct idxd_device *idxd) 391 { 392 struct device *dev = &idxd->pdev->dev; 393 int rc, i; 394 395 init_waitqueue_head(&idxd->cmd_waitq); 396 397 rc = idxd_setup_wqs(idxd); 398 if (rc < 0) 399 goto err_wqs; 400 401 rc = idxd_setup_engines(idxd); 402 if (rc < 0) 403 goto err_engine; 404 405 rc = idxd_setup_groups(idxd); 406 if (rc < 0) 407 goto err_group; 408 409 idxd->wq = create_workqueue(dev_name(dev)); 410 if (!idxd->wq) { 411 rc = -ENOMEM; 412 goto err_wkq_create; 413 } 414 415 rc = idxd_init_evl(idxd); 416 if (rc < 0) 417 goto err_evl; 418 419 return 0; 420 421 err_evl: 422 destroy_workqueue(idxd->wq); 423 err_wkq_create: 424 for (i = 0; i < idxd->max_groups; i++) 425 put_device(group_confdev(idxd->groups[i])); 426 err_group: 427 for (i = 0; i < idxd->max_engines; i++) 428 put_device(engine_confdev(idxd->engines[i])); 429 err_engine: 430 for (i = 0; i < idxd->max_wqs; i++) 431 put_device(wq_confdev(idxd->wqs[i])); 432 err_wqs: 433 return rc; 434 } 435 436 static void idxd_read_table_offsets(struct idxd_device *idxd) 437 { 438 union offsets_reg offsets; 439 struct device *dev = &idxd->pdev->dev; 440 441 offsets.bits[0] = ioread64(idxd->reg_base + IDXD_TABLE_OFFSET); 442 offsets.bits[1] = ioread64(idxd->reg_base + IDXD_TABLE_OFFSET + sizeof(u64)); 443 idxd->grpcfg_offset = offsets.grpcfg * IDXD_TABLE_MULT; 444 dev_dbg(dev, "IDXD Group Config Offset: %#x\n", idxd->grpcfg_offset); 445 idxd->wqcfg_offset = offsets.wqcfg * IDXD_TABLE_MULT; 446 dev_dbg(dev, "IDXD Work Queue Config Offset: %#x\n", idxd->wqcfg_offset); 447 idxd->msix_perm_offset = offsets.msix_perm * IDXD_TABLE_MULT; 448 dev_dbg(dev, "IDXD MSIX Permission Offset: %#x\n", idxd->msix_perm_offset); 449 idxd->perfmon_offset = offsets.perfmon * IDXD_TABLE_MULT; 450 dev_dbg(dev, "IDXD Perfmon Offset: %#x\n", idxd->perfmon_offset); 451 } 452 453 void multi_u64_to_bmap(unsigned long *bmap, u64 *val, int count) 454 { 455 int i, j, nr; 456 457 for (i = 0, nr = 0; i < count; i++) { 458 for (j = 0; j < BITS_PER_LONG_LONG; j++) { 459 if (val[i] & BIT(j)) 460 set_bit(nr, bmap); 461 nr++; 462 } 463 } 464 } 465 466 static void idxd_read_caps(struct idxd_device *idxd) 467 { 468 struct device *dev = &idxd->pdev->dev; 469 int i; 470 471 /* reading generic capabilities */ 472 idxd->hw.gen_cap.bits = ioread64(idxd->reg_base + IDXD_GENCAP_OFFSET); 473 dev_dbg(dev, "gen_cap: %#llx\n", idxd->hw.gen_cap.bits); 474 475 if (idxd->hw.gen_cap.cmd_cap) { 476 idxd->hw.cmd_cap = ioread32(idxd->reg_base + IDXD_CMDCAP_OFFSET); 477 dev_dbg(dev, "cmd_cap: %#x\n", idxd->hw.cmd_cap); 478 } 479 480 /* reading command capabilities */ 481 if (idxd->hw.cmd_cap & BIT(IDXD_CMD_REQUEST_INT_HANDLE)) 482 idxd->request_int_handles = true; 483 484 idxd->max_xfer_bytes = 1ULL << idxd->hw.gen_cap.max_xfer_shift; 485 dev_dbg(dev, "max xfer size: %llu bytes\n", idxd->max_xfer_bytes); 486 idxd_set_max_batch_size(idxd->data->type, idxd, 1U << idxd->hw.gen_cap.max_batch_shift); 487 dev_dbg(dev, "max batch size: %u\n", idxd->max_batch_size); 488 if (idxd->hw.gen_cap.config_en) 489 set_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags); 490 491 /* reading group capabilities */ 492 idxd->hw.group_cap.bits = 493 ioread64(idxd->reg_base + IDXD_GRPCAP_OFFSET); 494 dev_dbg(dev, "group_cap: %#llx\n", idxd->hw.group_cap.bits); 495 idxd->max_groups = idxd->hw.group_cap.num_groups; 496 dev_dbg(dev, "max groups: %u\n", idxd->max_groups); 497 idxd->max_rdbufs = idxd->hw.group_cap.total_rdbufs; 498 dev_dbg(dev, "max read buffers: %u\n", idxd->max_rdbufs); 499 idxd->nr_rdbufs = idxd->max_rdbufs; 500 501 /* read engine capabilities */ 502 idxd->hw.engine_cap.bits = 503 ioread64(idxd->reg_base + IDXD_ENGCAP_OFFSET); 504 dev_dbg(dev, "engine_cap: %#llx\n", idxd->hw.engine_cap.bits); 505 idxd->max_engines = idxd->hw.engine_cap.num_engines; 506 dev_dbg(dev, "max engines: %u\n", idxd->max_engines); 507 508 /* read workqueue capabilities */ 509 idxd->hw.wq_cap.bits = ioread64(idxd->reg_base + IDXD_WQCAP_OFFSET); 510 dev_dbg(dev, "wq_cap: %#llx\n", idxd->hw.wq_cap.bits); 511 idxd->max_wq_size = idxd->hw.wq_cap.total_wq_size; 512 dev_dbg(dev, "total workqueue size: %u\n", idxd->max_wq_size); 513 idxd->max_wqs = idxd->hw.wq_cap.num_wqs; 514 dev_dbg(dev, "max workqueues: %u\n", idxd->max_wqs); 515 idxd->wqcfg_size = 1 << (idxd->hw.wq_cap.wqcfg_size + IDXD_WQCFG_MIN); 516 dev_dbg(dev, "wqcfg size: %u\n", idxd->wqcfg_size); 517 518 /* reading operation capabilities */ 519 for (i = 0; i < 4; i++) { 520 idxd->hw.opcap.bits[i] = ioread64(idxd->reg_base + 521 IDXD_OPCAP_OFFSET + i * sizeof(u64)); 522 dev_dbg(dev, "opcap[%d]: %#llx\n", i, idxd->hw.opcap.bits[i]); 523 } 524 multi_u64_to_bmap(idxd->opcap_bmap, &idxd->hw.opcap.bits[0], 4); 525 526 /* read iaa cap */ 527 if (idxd->data->type == IDXD_TYPE_IAX && idxd->hw.version >= DEVICE_VERSION_2) 528 idxd->hw.iaa_cap.bits = ioread64(idxd->reg_base + IDXD_IAACAP_OFFSET); 529 } 530 531 static struct idxd_device *idxd_alloc(struct pci_dev *pdev, struct idxd_driver_data *data) 532 { 533 struct device *dev = &pdev->dev; 534 struct device *conf_dev; 535 struct idxd_device *idxd; 536 int rc; 537 538 idxd = kzalloc_node(sizeof(*idxd), GFP_KERNEL, dev_to_node(dev)); 539 if (!idxd) 540 return NULL; 541 542 conf_dev = idxd_confdev(idxd); 543 idxd->pdev = pdev; 544 idxd->data = data; 545 idxd_dev_set_type(&idxd->idxd_dev, idxd->data->type); 546 idxd->id = ida_alloc(&idxd_ida, GFP_KERNEL); 547 if (idxd->id < 0) 548 return NULL; 549 550 idxd->opcap_bmap = bitmap_zalloc_node(IDXD_MAX_OPCAP_BITS, GFP_KERNEL, dev_to_node(dev)); 551 if (!idxd->opcap_bmap) { 552 ida_free(&idxd_ida, idxd->id); 553 return NULL; 554 } 555 556 device_initialize(conf_dev); 557 conf_dev->parent = dev; 558 conf_dev->bus = &dsa_bus_type; 559 conf_dev->type = idxd->data->dev_type; 560 rc = dev_set_name(conf_dev, "%s%d", idxd->data->name_prefix, idxd->id); 561 if (rc < 0) { 562 put_device(conf_dev); 563 return NULL; 564 } 565 566 spin_lock_init(&idxd->dev_lock); 567 spin_lock_init(&idxd->cmd_lock); 568 569 return idxd; 570 } 571 572 static int idxd_enable_system_pasid(struct idxd_device *idxd) 573 { 574 struct pci_dev *pdev = idxd->pdev; 575 struct device *dev = &pdev->dev; 576 struct iommu_domain *domain; 577 ioasid_t pasid; 578 int ret; 579 580 /* 581 * Attach a global PASID to the DMA domain so that we can use ENQCMDS 582 * to submit work on buffers mapped by DMA API. 583 */ 584 domain = iommu_get_domain_for_dev(dev); 585 if (!domain) 586 return -EPERM; 587 588 pasid = iommu_alloc_global_pasid(dev); 589 if (pasid == IOMMU_PASID_INVALID) 590 return -ENOSPC; 591 592 /* 593 * DMA domain is owned by the driver, it should support all valid 594 * types such as DMA-FQ, identity, etc. 595 */ 596 ret = iommu_attach_device_pasid(domain, dev, pasid, NULL); 597 if (ret) { 598 dev_err(dev, "failed to attach device pasid %d, domain type %d", 599 pasid, domain->type); 600 iommu_free_global_pasid(pasid); 601 return ret; 602 } 603 604 /* Since we set user privilege for kernel DMA, enable completion IRQ */ 605 idxd_set_user_intr(idxd, 1); 606 idxd->pasid = pasid; 607 608 return ret; 609 } 610 611 static void idxd_disable_system_pasid(struct idxd_device *idxd) 612 { 613 struct pci_dev *pdev = idxd->pdev; 614 struct device *dev = &pdev->dev; 615 struct iommu_domain *domain; 616 617 domain = iommu_get_domain_for_dev(dev); 618 if (!domain) 619 return; 620 621 iommu_detach_device_pasid(domain, dev, idxd->pasid); 622 iommu_free_global_pasid(idxd->pasid); 623 624 idxd_set_user_intr(idxd, 0); 625 idxd->sva = NULL; 626 idxd->pasid = IOMMU_PASID_INVALID; 627 } 628 629 static int idxd_enable_sva(struct pci_dev *pdev) 630 { 631 int ret; 632 633 ret = iommu_dev_enable_feature(&pdev->dev, IOMMU_DEV_FEAT_IOPF); 634 if (ret) 635 return ret; 636 637 ret = iommu_dev_enable_feature(&pdev->dev, IOMMU_DEV_FEAT_SVA); 638 if (ret) 639 iommu_dev_disable_feature(&pdev->dev, IOMMU_DEV_FEAT_IOPF); 640 641 return ret; 642 } 643 644 static void idxd_disable_sva(struct pci_dev *pdev) 645 { 646 iommu_dev_disable_feature(&pdev->dev, IOMMU_DEV_FEAT_SVA); 647 iommu_dev_disable_feature(&pdev->dev, IOMMU_DEV_FEAT_IOPF); 648 } 649 650 static int idxd_probe(struct idxd_device *idxd) 651 { 652 struct pci_dev *pdev = idxd->pdev; 653 struct device *dev = &pdev->dev; 654 int rc; 655 656 dev_dbg(dev, "%s entered and resetting device\n", __func__); 657 rc = idxd_device_init_reset(idxd); 658 if (rc < 0) 659 return rc; 660 661 dev_dbg(dev, "IDXD reset complete\n"); 662 663 if (IS_ENABLED(CONFIG_INTEL_IDXD_SVM) && sva) { 664 if (idxd_enable_sva(pdev)) { 665 dev_warn(dev, "Unable to turn on user SVA feature.\n"); 666 } else { 667 set_bit(IDXD_FLAG_USER_PASID_ENABLED, &idxd->flags); 668 669 rc = idxd_enable_system_pasid(idxd); 670 if (rc) 671 dev_warn(dev, "No in-kernel DMA with PASID. %d\n", rc); 672 else 673 set_bit(IDXD_FLAG_PASID_ENABLED, &idxd->flags); 674 } 675 } else if (!sva) { 676 dev_warn(dev, "User forced SVA off via module param.\n"); 677 } 678 679 idxd_read_caps(idxd); 680 idxd_read_table_offsets(idxd); 681 682 rc = idxd_setup_internals(idxd); 683 if (rc) 684 goto err; 685 686 /* If the configs are readonly, then load them from device */ 687 if (!test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags)) { 688 dev_dbg(dev, "Loading RO device config\n"); 689 rc = idxd_device_load_config(idxd); 690 if (rc < 0) 691 goto err_config; 692 } 693 694 rc = idxd_setup_interrupts(idxd); 695 if (rc) 696 goto err_config; 697 698 idxd->major = idxd_cdev_get_major(idxd); 699 700 rc = perfmon_pmu_init(idxd); 701 if (rc < 0) 702 dev_warn(dev, "Failed to initialize perfmon. No PMU support: %d\n", rc); 703 704 dev_dbg(dev, "IDXD device %d probed successfully\n", idxd->id); 705 return 0; 706 707 err_config: 708 idxd_cleanup_internals(idxd); 709 err: 710 if (device_pasid_enabled(idxd)) 711 idxd_disable_system_pasid(idxd); 712 if (device_user_pasid_enabled(idxd)) 713 idxd_disable_sva(pdev); 714 return rc; 715 } 716 717 static void idxd_cleanup(struct idxd_device *idxd) 718 { 719 perfmon_pmu_remove(idxd); 720 idxd_cleanup_interrupts(idxd); 721 idxd_cleanup_internals(idxd); 722 if (device_pasid_enabled(idxd)) 723 idxd_disable_system_pasid(idxd); 724 if (device_user_pasid_enabled(idxd)) 725 idxd_disable_sva(idxd->pdev); 726 } 727 728 /* 729 * Attach IDXD device to IDXD driver. 730 */ 731 static int idxd_bind(struct device_driver *drv, const char *buf) 732 { 733 const struct bus_type *bus = drv->bus; 734 struct device *dev; 735 int err = -ENODEV; 736 737 dev = bus_find_device_by_name(bus, NULL, buf); 738 if (dev) 739 err = device_driver_attach(drv, dev); 740 741 put_device(dev); 742 743 return err; 744 } 745 746 /* 747 * Detach IDXD device from driver. 748 */ 749 static void idxd_unbind(struct device_driver *drv, const char *buf) 750 { 751 const struct bus_type *bus = drv->bus; 752 struct device *dev; 753 754 dev = bus_find_device_by_name(bus, NULL, buf); 755 if (dev && dev->driver == drv) 756 device_release_driver(dev); 757 758 put_device(dev); 759 } 760 761 #define idxd_free_saved_configs(saved_configs, count) \ 762 do { \ 763 int i; \ 764 \ 765 for (i = 0; i < (count); i++) \ 766 kfree(saved_configs[i]); \ 767 } while (0) 768 769 static void idxd_free_saved(struct idxd_group **saved_groups, 770 struct idxd_engine **saved_engines, 771 struct idxd_wq **saved_wqs, 772 struct idxd_device *idxd) 773 { 774 if (saved_groups) 775 idxd_free_saved_configs(saved_groups, idxd->max_groups); 776 if (saved_engines) 777 idxd_free_saved_configs(saved_engines, idxd->max_engines); 778 if (saved_wqs) 779 idxd_free_saved_configs(saved_wqs, idxd->max_wqs); 780 } 781 782 /* 783 * Save IDXD device configurations including engines, groups, wqs etc. 784 * The saved configurations can be restored when needed. 785 */ 786 static int idxd_device_config_save(struct idxd_device *idxd, 787 struct idxd_saved_states *idxd_saved) 788 { 789 struct device *dev = &idxd->pdev->dev; 790 int i; 791 792 memcpy(&idxd_saved->saved_idxd, idxd, sizeof(*idxd)); 793 794 if (idxd->evl) { 795 memcpy(&idxd_saved->saved_evl, idxd->evl, 796 sizeof(struct idxd_evl)); 797 } 798 799 struct idxd_group **saved_groups __free(kfree) = 800 kcalloc_node(idxd->max_groups, 801 sizeof(struct idxd_group *), 802 GFP_KERNEL, dev_to_node(dev)); 803 if (!saved_groups) 804 return -ENOMEM; 805 806 for (i = 0; i < idxd->max_groups; i++) { 807 struct idxd_group *saved_group __free(kfree) = 808 kzalloc_node(sizeof(*saved_group), GFP_KERNEL, 809 dev_to_node(dev)); 810 811 if (!saved_group) { 812 /* Free saved groups */ 813 idxd_free_saved(saved_groups, NULL, NULL, idxd); 814 815 return -ENOMEM; 816 } 817 818 memcpy(saved_group, idxd->groups[i], sizeof(*saved_group)); 819 saved_groups[i] = no_free_ptr(saved_group); 820 } 821 822 struct idxd_engine **saved_engines = 823 kcalloc_node(idxd->max_engines, 824 sizeof(struct idxd_engine *), 825 GFP_KERNEL, dev_to_node(dev)); 826 if (!saved_engines) { 827 /* Free saved groups */ 828 idxd_free_saved(saved_groups, NULL, NULL, idxd); 829 830 return -ENOMEM; 831 } 832 for (i = 0; i < idxd->max_engines; i++) { 833 struct idxd_engine *saved_engine __free(kfree) = 834 kzalloc_node(sizeof(*saved_engine), GFP_KERNEL, 835 dev_to_node(dev)); 836 if (!saved_engine) { 837 /* Free saved groups and engines */ 838 idxd_free_saved(saved_groups, saved_engines, NULL, 839 idxd); 840 841 return -ENOMEM; 842 } 843 844 memcpy(saved_engine, idxd->engines[i], sizeof(*saved_engine)); 845 saved_engines[i] = no_free_ptr(saved_engine); 846 } 847 848 unsigned long *saved_wq_enable_map __free(bitmap) = 849 bitmap_zalloc_node(idxd->max_wqs, GFP_KERNEL, 850 dev_to_node(dev)); 851 if (!saved_wq_enable_map) { 852 /* Free saved groups and engines */ 853 idxd_free_saved(saved_groups, saved_engines, NULL, idxd); 854 855 return -ENOMEM; 856 } 857 858 bitmap_copy(saved_wq_enable_map, idxd->wq_enable_map, idxd->max_wqs); 859 860 struct idxd_wq **saved_wqs __free(kfree) = 861 kcalloc_node(idxd->max_wqs, sizeof(struct idxd_wq *), 862 GFP_KERNEL, dev_to_node(dev)); 863 if (!saved_wqs) { 864 /* Free saved groups and engines */ 865 idxd_free_saved(saved_groups, saved_engines, NULL, idxd); 866 867 return -ENOMEM; 868 } 869 870 for (i = 0; i < idxd->max_wqs; i++) { 871 struct idxd_wq *saved_wq __free(kfree) = 872 kzalloc_node(sizeof(*saved_wq), GFP_KERNEL, 873 dev_to_node(dev)); 874 struct idxd_wq *wq; 875 876 if (!saved_wq) { 877 /* Free saved groups, engines, and wqs */ 878 idxd_free_saved(saved_groups, saved_engines, saved_wqs, 879 idxd); 880 881 return -ENOMEM; 882 } 883 884 if (!test_bit(i, saved_wq_enable_map)) 885 continue; 886 887 wq = idxd->wqs[i]; 888 mutex_lock(&wq->wq_lock); 889 memcpy(saved_wq, wq, sizeof(*saved_wq)); 890 saved_wqs[i] = no_free_ptr(saved_wq); 891 mutex_unlock(&wq->wq_lock); 892 } 893 894 /* Save configurations */ 895 idxd_saved->saved_groups = no_free_ptr(saved_groups); 896 idxd_saved->saved_engines = no_free_ptr(saved_engines); 897 idxd_saved->saved_wq_enable_map = no_free_ptr(saved_wq_enable_map); 898 idxd_saved->saved_wqs = no_free_ptr(saved_wqs); 899 900 return 0; 901 } 902 903 /* 904 * Restore IDXD device configurations including engines, groups, wqs etc 905 * that were saved before. 906 */ 907 static void idxd_device_config_restore(struct idxd_device *idxd, 908 struct idxd_saved_states *idxd_saved) 909 { 910 struct idxd_evl *saved_evl = &idxd_saved->saved_evl; 911 int i; 912 913 idxd->rdbuf_limit = idxd_saved->saved_idxd.rdbuf_limit; 914 915 if (saved_evl) 916 idxd->evl->size = saved_evl->size; 917 918 for (i = 0; i < idxd->max_groups; i++) { 919 struct idxd_group *saved_group, *group; 920 921 saved_group = idxd_saved->saved_groups[i]; 922 group = idxd->groups[i]; 923 924 group->rdbufs_allowed = saved_group->rdbufs_allowed; 925 group->rdbufs_reserved = saved_group->rdbufs_reserved; 926 group->tc_a = saved_group->tc_a; 927 group->tc_b = saved_group->tc_b; 928 group->use_rdbuf_limit = saved_group->use_rdbuf_limit; 929 930 kfree(saved_group); 931 } 932 kfree(idxd_saved->saved_groups); 933 934 for (i = 0; i < idxd->max_engines; i++) { 935 struct idxd_engine *saved_engine, *engine; 936 937 saved_engine = idxd_saved->saved_engines[i]; 938 engine = idxd->engines[i]; 939 940 engine->group = saved_engine->group; 941 942 kfree(saved_engine); 943 } 944 kfree(idxd_saved->saved_engines); 945 946 bitmap_copy(idxd->wq_enable_map, idxd_saved->saved_wq_enable_map, 947 idxd->max_wqs); 948 bitmap_free(idxd_saved->saved_wq_enable_map); 949 950 for (i = 0; i < idxd->max_wqs; i++) { 951 struct idxd_wq *saved_wq, *wq; 952 size_t len; 953 954 if (!test_bit(i, idxd->wq_enable_map)) 955 continue; 956 957 saved_wq = idxd_saved->saved_wqs[i]; 958 wq = idxd->wqs[i]; 959 960 mutex_lock(&wq->wq_lock); 961 962 wq->group = saved_wq->group; 963 wq->flags = saved_wq->flags; 964 wq->threshold = saved_wq->threshold; 965 wq->size = saved_wq->size; 966 wq->priority = saved_wq->priority; 967 wq->type = saved_wq->type; 968 len = strlen(saved_wq->name) + 1; 969 strscpy(wq->name, saved_wq->name, len); 970 wq->max_xfer_bytes = saved_wq->max_xfer_bytes; 971 wq->max_batch_size = saved_wq->max_batch_size; 972 wq->enqcmds_retries = saved_wq->enqcmds_retries; 973 wq->descs = saved_wq->descs; 974 wq->idxd_chan = saved_wq->idxd_chan; 975 len = strlen(saved_wq->driver_name) + 1; 976 strscpy(wq->driver_name, saved_wq->driver_name, len); 977 978 mutex_unlock(&wq->wq_lock); 979 980 kfree(saved_wq); 981 } 982 983 kfree(idxd_saved->saved_wqs); 984 } 985 986 static void idxd_reset_prepare(struct pci_dev *pdev) 987 { 988 struct idxd_device *idxd = pci_get_drvdata(pdev); 989 struct device *dev = &idxd->pdev->dev; 990 const char *idxd_name; 991 int rc; 992 993 dev = &idxd->pdev->dev; 994 idxd_name = dev_name(idxd_confdev(idxd)); 995 996 struct idxd_saved_states *idxd_saved __free(kfree) = 997 kzalloc_node(sizeof(*idxd_saved), GFP_KERNEL, 998 dev_to_node(&pdev->dev)); 999 if (!idxd_saved) { 1000 dev_err(dev, "HALT: no memory\n"); 1001 1002 return; 1003 } 1004 1005 /* Save IDXD configurations. */ 1006 rc = idxd_device_config_save(idxd, idxd_saved); 1007 if (rc < 0) { 1008 dev_err(dev, "HALT: cannot save %s configs\n", idxd_name); 1009 1010 return; 1011 } 1012 1013 idxd->idxd_saved = no_free_ptr(idxd_saved); 1014 1015 /* Save PCI device state. */ 1016 pci_save_state(idxd->pdev); 1017 } 1018 1019 static void idxd_reset_done(struct pci_dev *pdev) 1020 { 1021 struct idxd_device *idxd = pci_get_drvdata(pdev); 1022 const char *idxd_name; 1023 struct device *dev; 1024 int rc, i; 1025 1026 if (!idxd->idxd_saved) 1027 return; 1028 1029 dev = &idxd->pdev->dev; 1030 idxd_name = dev_name(idxd_confdev(idxd)); 1031 1032 /* Restore PCI device state. */ 1033 pci_restore_state(idxd->pdev); 1034 1035 /* Unbind idxd device from driver. */ 1036 idxd_unbind(&idxd_drv.drv, idxd_name); 1037 1038 /* 1039 * Probe PCI device without allocating or changing 1040 * idxd software data which keeps the same as before FLR. 1041 */ 1042 idxd_pci_probe_alloc(idxd, NULL, NULL); 1043 1044 /* Restore IDXD configurations. */ 1045 idxd_device_config_restore(idxd, idxd->idxd_saved); 1046 1047 /* Re-configure IDXD device if allowed. */ 1048 if (test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags)) { 1049 rc = idxd_device_config(idxd); 1050 if (rc < 0) { 1051 dev_err(dev, "HALT: %s config fails\n", idxd_name); 1052 goto out; 1053 } 1054 } 1055 1056 /* Bind IDXD device to driver. */ 1057 rc = idxd_bind(&idxd_drv.drv, idxd_name); 1058 if (rc < 0) { 1059 dev_err(dev, "HALT: binding %s to driver fails\n", idxd_name); 1060 goto out; 1061 } 1062 1063 /* Bind enabled wq in the IDXD device to driver. */ 1064 for (i = 0; i < idxd->max_wqs; i++) { 1065 if (test_bit(i, idxd->wq_enable_map)) { 1066 struct idxd_wq *wq = idxd->wqs[i]; 1067 char wq_name[32]; 1068 1069 wq->state = IDXD_WQ_DISABLED; 1070 sprintf(wq_name, "wq%d.%d", idxd->id, wq->id); 1071 /* 1072 * Bind to user driver depending on wq type. 1073 * 1074 * Currently only support user type WQ. Will support 1075 * kernel type WQ in the future. 1076 */ 1077 if (wq->type == IDXD_WQT_USER) 1078 rc = idxd_bind(&idxd_user_drv.drv, wq_name); 1079 else 1080 rc = -EINVAL; 1081 if (rc < 0) { 1082 clear_bit(i, idxd->wq_enable_map); 1083 dev_err(dev, 1084 "HALT: unable to re-enable wq %s\n", 1085 dev_name(wq_confdev(wq))); 1086 } 1087 } 1088 } 1089 out: 1090 kfree(idxd->idxd_saved); 1091 } 1092 1093 static const struct pci_error_handlers idxd_error_handler = { 1094 .reset_prepare = idxd_reset_prepare, 1095 .reset_done = idxd_reset_done, 1096 }; 1097 1098 /* 1099 * Probe idxd PCI device. 1100 * If idxd is not given, need to allocate idxd and set up its data. 1101 * 1102 * If idxd is given, idxd was allocated and setup already. Just need to 1103 * configure device without re-allocating and re-configuring idxd data. 1104 * This is useful for recovering from FLR. 1105 */ 1106 int idxd_pci_probe_alloc(struct idxd_device *idxd, struct pci_dev *pdev, 1107 const struct pci_device_id *id) 1108 { 1109 bool alloc_idxd = idxd ? false : true; 1110 struct idxd_driver_data *data; 1111 struct device *dev; 1112 int rc; 1113 1114 pdev = idxd ? idxd->pdev : pdev; 1115 dev = &pdev->dev; 1116 data = id ? (struct idxd_driver_data *)id->driver_data : NULL; 1117 rc = pci_enable_device(pdev); 1118 if (rc) 1119 return rc; 1120 1121 if (alloc_idxd) { 1122 dev_dbg(dev, "Alloc IDXD context\n"); 1123 idxd = idxd_alloc(pdev, data); 1124 if (!idxd) { 1125 rc = -ENOMEM; 1126 goto err_idxd_alloc; 1127 } 1128 1129 dev_dbg(dev, "Mapping BARs\n"); 1130 idxd->reg_base = pci_iomap(pdev, IDXD_MMIO_BAR, 0); 1131 if (!idxd->reg_base) { 1132 rc = -ENOMEM; 1133 goto err_iomap; 1134 } 1135 1136 dev_dbg(dev, "Set DMA masks\n"); 1137 rc = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); 1138 if (rc) 1139 goto err; 1140 } 1141 1142 dev_dbg(dev, "Set PCI master\n"); 1143 pci_set_master(pdev); 1144 pci_set_drvdata(pdev, idxd); 1145 1146 if (alloc_idxd) { 1147 idxd->hw.version = ioread32(idxd->reg_base + IDXD_VER_OFFSET); 1148 rc = idxd_probe(idxd); 1149 if (rc) { 1150 dev_err(dev, "Intel(R) IDXD DMA Engine init failed\n"); 1151 goto err; 1152 } 1153 1154 if (data->load_device_defaults) { 1155 rc = data->load_device_defaults(idxd); 1156 if (rc) 1157 dev_warn(dev, "IDXD loading device defaults failed\n"); 1158 } 1159 1160 rc = idxd_register_devices(idxd); 1161 if (rc) { 1162 dev_err(dev, "IDXD sysfs setup failed\n"); 1163 goto err_dev_register; 1164 } 1165 1166 rc = idxd_device_init_debugfs(idxd); 1167 if (rc) 1168 dev_warn(dev, "IDXD debugfs failed to setup\n"); 1169 } 1170 1171 if (!alloc_idxd) { 1172 /* Release interrupts in the IDXD device. */ 1173 idxd_cleanup_interrupts(idxd); 1174 1175 /* Re-enable interrupts in the IDXD device. */ 1176 rc = idxd_setup_interrupts(idxd); 1177 if (rc) 1178 dev_warn(dev, "IDXD interrupts failed to setup\n"); 1179 } 1180 1181 dev_info(&pdev->dev, "Intel(R) Accelerator Device (v%x)\n", 1182 idxd->hw.version); 1183 1184 if (data) 1185 idxd->user_submission_safe = data->user_submission_safe; 1186 1187 return 0; 1188 1189 err_dev_register: 1190 idxd_cleanup(idxd); 1191 err: 1192 pci_iounmap(pdev, idxd->reg_base); 1193 err_iomap: 1194 put_device(idxd_confdev(idxd)); 1195 err_idxd_alloc: 1196 pci_disable_device(pdev); 1197 return rc; 1198 } 1199 1200 static int idxd_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) 1201 { 1202 return idxd_pci_probe_alloc(NULL, pdev, id); 1203 } 1204 1205 void idxd_wqs_quiesce(struct idxd_device *idxd) 1206 { 1207 struct idxd_wq *wq; 1208 int i; 1209 1210 for (i = 0; i < idxd->max_wqs; i++) { 1211 wq = idxd->wqs[i]; 1212 if (wq->state == IDXD_WQ_ENABLED && wq->type == IDXD_WQT_KERNEL) 1213 idxd_wq_quiesce(wq); 1214 } 1215 } 1216 1217 static void idxd_shutdown(struct pci_dev *pdev) 1218 { 1219 struct idxd_device *idxd = pci_get_drvdata(pdev); 1220 struct idxd_irq_entry *irq_entry; 1221 int rc; 1222 1223 rc = idxd_device_disable(idxd); 1224 if (rc) 1225 dev_err(&pdev->dev, "Disabling device failed\n"); 1226 1227 irq_entry = &idxd->ie; 1228 synchronize_irq(irq_entry->vector); 1229 idxd_mask_error_interrupts(idxd); 1230 flush_workqueue(idxd->wq); 1231 } 1232 1233 static void idxd_remove(struct pci_dev *pdev) 1234 { 1235 struct idxd_device *idxd = pci_get_drvdata(pdev); 1236 struct idxd_irq_entry *irq_entry; 1237 1238 idxd_unregister_devices(idxd); 1239 /* 1240 * When ->release() is called for the idxd->conf_dev, it frees all the memory related 1241 * to the idxd context. The driver still needs those bits in order to do the rest of 1242 * the cleanup. However, we do need to unbound the idxd sub-driver. So take a ref 1243 * on the device here to hold off the freeing while allowing the idxd sub-driver 1244 * to unbind. 1245 */ 1246 get_device(idxd_confdev(idxd)); 1247 device_unregister(idxd_confdev(idxd)); 1248 idxd_shutdown(pdev); 1249 if (device_pasid_enabled(idxd)) 1250 idxd_disable_system_pasid(idxd); 1251 idxd_device_remove_debugfs(idxd); 1252 1253 irq_entry = idxd_get_ie(idxd, 0); 1254 free_irq(irq_entry->vector, irq_entry); 1255 pci_free_irq_vectors(pdev); 1256 pci_iounmap(pdev, idxd->reg_base); 1257 if (device_user_pasid_enabled(idxd)) 1258 idxd_disable_sva(pdev); 1259 pci_disable_device(pdev); 1260 destroy_workqueue(idxd->wq); 1261 perfmon_pmu_remove(idxd); 1262 put_device(idxd_confdev(idxd)); 1263 } 1264 1265 static struct pci_driver idxd_pci_driver = { 1266 .name = DRV_NAME, 1267 .id_table = idxd_pci_tbl, 1268 .probe = idxd_pci_probe, 1269 .remove = idxd_remove, 1270 .shutdown = idxd_shutdown, 1271 .err_handler = &idxd_error_handler, 1272 }; 1273 1274 static int __init idxd_init_module(void) 1275 { 1276 int err; 1277 1278 /* 1279 * If the CPU does not support MOVDIR64B or ENQCMDS, there's no point in 1280 * enumerating the device. We can not utilize it. 1281 */ 1282 if (!cpu_feature_enabled(X86_FEATURE_MOVDIR64B)) { 1283 pr_warn("idxd driver failed to load without MOVDIR64B.\n"); 1284 return -ENODEV; 1285 } 1286 1287 if (!cpu_feature_enabled(X86_FEATURE_ENQCMD)) 1288 pr_warn("Platform does not have ENQCMD(S) support.\n"); 1289 else 1290 support_enqcmd = true; 1291 1292 err = idxd_driver_register(&idxd_drv); 1293 if (err < 0) 1294 goto err_idxd_driver_register; 1295 1296 err = idxd_driver_register(&idxd_dmaengine_drv); 1297 if (err < 0) 1298 goto err_idxd_dmaengine_driver_register; 1299 1300 err = idxd_driver_register(&idxd_user_drv); 1301 if (err < 0) 1302 goto err_idxd_user_driver_register; 1303 1304 err = idxd_cdev_register(); 1305 if (err) 1306 goto err_cdev_register; 1307 1308 err = idxd_init_debugfs(); 1309 if (err) 1310 goto err_debugfs; 1311 1312 err = pci_register_driver(&idxd_pci_driver); 1313 if (err) 1314 goto err_pci_register; 1315 1316 return 0; 1317 1318 err_pci_register: 1319 idxd_remove_debugfs(); 1320 err_debugfs: 1321 idxd_cdev_remove(); 1322 err_cdev_register: 1323 idxd_driver_unregister(&idxd_user_drv); 1324 err_idxd_user_driver_register: 1325 idxd_driver_unregister(&idxd_dmaengine_drv); 1326 err_idxd_dmaengine_driver_register: 1327 idxd_driver_unregister(&idxd_drv); 1328 err_idxd_driver_register: 1329 return err; 1330 } 1331 module_init(idxd_init_module); 1332 1333 static void __exit idxd_exit_module(void) 1334 { 1335 idxd_driver_unregister(&idxd_user_drv); 1336 idxd_driver_unregister(&idxd_dmaengine_drv); 1337 idxd_driver_unregister(&idxd_drv); 1338 pci_unregister_driver(&idxd_pci_driver); 1339 idxd_cdev_remove(); 1340 idxd_remove_debugfs(); 1341 } 1342 module_exit(idxd_exit_module); 1343