1 // SPDX-License-Identifier: GPL-2.0-only 2 3 /* Copyright (c) 2019-2021, The Linux Foundation. All rights reserved. */ 4 /* Copyright (c) 2021-2023 Qualcomm Innovation Center, Inc. All rights reserved. */ 5 6 #include <linux/delay.h> 7 #include <linux/dma-mapping.h> 8 #include <linux/idr.h> 9 #include <linux/interrupt.h> 10 #include <linux/list.h> 11 #include <linux/kobject.h> 12 #include <linux/kref.h> 13 #include <linux/mhi.h> 14 #include <linux/module.h> 15 #include <linux/msi.h> 16 #include <linux/mutex.h> 17 #include <linux/pci.h> 18 #include <linux/spinlock.h> 19 #include <linux/workqueue.h> 20 #include <linux/wait.h> 21 #include <drm/drm_accel.h> 22 #include <drm/drm_drv.h> 23 #include <drm/drm_file.h> 24 #include <drm/drm_gem.h> 25 #include <drm/drm_ioctl.h> 26 #include <drm/drm_managed.h> 27 #include <uapi/drm/qaic_accel.h> 28 29 #include "mhi_controller.h" 30 #include "qaic.h" 31 #include "qaic_debugfs.h" 32 #include "qaic_ras.h" 33 #include "qaic_timesync.h" 34 #include "sahara.h" 35 36 MODULE_IMPORT_NS("DMA_BUF"); 37 38 #define PCI_DEVICE_ID_QCOM_AIC080 0xa080 39 #define PCI_DEVICE_ID_QCOM_AIC100 0xa100 40 #define PCI_DEVICE_ID_QCOM_AIC200 0xa110 41 #define QAIC_NAME "qaic" 42 #define QAIC_DESC "Qualcomm Cloud AI Accelerators" 43 #define CNTL_MAJOR 5 44 #define CNTL_MINOR 0 45 46 struct qaic_device_config { 47 /* Indicates the AIC family the device belongs to */ 48 int family; 49 /* A bitmask representing the available BARs */ 50 int bar_mask; 51 /* An index value used to identify the MHI controller BAR */ 52 unsigned int mhi_bar_idx; 53 /* An index value used to identify the DBCs BAR */ 54 unsigned int dbc_bar_idx; 55 }; 56 57 static const struct qaic_device_config aic080_config = { 58 .family = FAMILY_AIC100, 59 .bar_mask = BIT(0) | BIT(2) | BIT(4), 60 .mhi_bar_idx = 0, 61 .dbc_bar_idx = 2, 62 }; 63 64 static const struct qaic_device_config aic100_config = { 65 .family = FAMILY_AIC100, 66 .bar_mask = BIT(0) | BIT(2) | BIT(4), 67 .mhi_bar_idx = 0, 68 .dbc_bar_idx = 2, 69 }; 70 71 static const struct qaic_device_config aic200_config = { 72 .family = FAMILY_AIC200, 73 .bar_mask = BIT(0) | BIT(1) | BIT(2) | BIT(4), 74 .mhi_bar_idx = 1, 75 .dbc_bar_idx = 2, 76 }; 77 78 bool datapath_polling; 79 module_param(datapath_polling, bool, 0400); 80 MODULE_PARM_DESC(datapath_polling, "Operate the datapath in polling mode"); 81 static bool link_up; 82 static DEFINE_IDA(qaic_usrs); 83 84 static void qaicm_wq_release(struct drm_device *dev, void *res) 85 { 86 struct workqueue_struct *wq = res; 87 88 destroy_workqueue(wq); 89 } 90 91 static struct workqueue_struct *qaicm_wq_init(struct drm_device *dev, const char *name) 92 { 93 struct workqueue_struct *wq; 94 int ret; 95 96 wq = alloc_workqueue("%s", WQ_UNBOUND, 0, name); 97 if (!wq) 98 return ERR_PTR(-ENOMEM); 99 ret = drmm_add_action_or_reset(dev, qaicm_wq_release, wq); 100 if (ret) 101 return ERR_PTR(ret); 102 103 return wq; 104 } 105 106 static void qaicm_srcu_release(struct drm_device *dev, void *res) 107 { 108 struct srcu_struct *lock = res; 109 110 cleanup_srcu_struct(lock); 111 } 112 113 static int qaicm_srcu_init(struct drm_device *dev, struct srcu_struct *lock) 114 { 115 int ret; 116 117 ret = init_srcu_struct(lock); 118 if (ret) 119 return ret; 120 121 return drmm_add_action_or_reset(dev, qaicm_srcu_release, lock); 122 } 123 124 static void qaicm_pci_release(struct drm_device *dev, void *res) 125 { 126 struct qaic_device *qdev = to_qaic_device(dev); 127 128 pci_set_drvdata(qdev->pdev, NULL); 129 } 130 131 static void free_usr(struct kref *kref) 132 { 133 struct qaic_user *usr = container_of(kref, struct qaic_user, ref_count); 134 135 cleanup_srcu_struct(&usr->qddev_lock); 136 ida_free(&qaic_usrs, usr->handle); 137 kfree(usr); 138 } 139 140 static int qaic_open(struct drm_device *dev, struct drm_file *file) 141 { 142 struct qaic_drm_device *qddev = to_qaic_drm_device(dev); 143 struct qaic_device *qdev = qddev->qdev; 144 struct qaic_user *usr; 145 int rcu_id; 146 int ret; 147 148 rcu_id = srcu_read_lock(&qdev->dev_lock); 149 if (qdev->dev_state != QAIC_ONLINE) { 150 ret = -ENODEV; 151 goto dev_unlock; 152 } 153 154 usr = kmalloc(sizeof(*usr), GFP_KERNEL); 155 if (!usr) { 156 ret = -ENOMEM; 157 goto dev_unlock; 158 } 159 160 usr->handle = ida_alloc(&qaic_usrs, GFP_KERNEL); 161 if (usr->handle < 0) { 162 ret = usr->handle; 163 goto free_usr; 164 } 165 usr->qddev = qddev; 166 atomic_set(&usr->chunk_id, 0); 167 init_srcu_struct(&usr->qddev_lock); 168 kref_init(&usr->ref_count); 169 170 ret = mutex_lock_interruptible(&qddev->users_mutex); 171 if (ret) 172 goto cleanup_usr; 173 174 list_add(&usr->node, &qddev->users); 175 mutex_unlock(&qddev->users_mutex); 176 177 file->driver_priv = usr; 178 179 srcu_read_unlock(&qdev->dev_lock, rcu_id); 180 return 0; 181 182 cleanup_usr: 183 cleanup_srcu_struct(&usr->qddev_lock); 184 ida_free(&qaic_usrs, usr->handle); 185 free_usr: 186 kfree(usr); 187 dev_unlock: 188 srcu_read_unlock(&qdev->dev_lock, rcu_id); 189 return ret; 190 } 191 192 static void qaic_postclose(struct drm_device *dev, struct drm_file *file) 193 { 194 struct qaic_user *usr = file->driver_priv; 195 struct qaic_drm_device *qddev; 196 struct qaic_device *qdev; 197 int qdev_rcu_id; 198 int usr_rcu_id; 199 int i; 200 201 qddev = usr->qddev; 202 usr_rcu_id = srcu_read_lock(&usr->qddev_lock); 203 if (qddev) { 204 qdev = qddev->qdev; 205 qdev_rcu_id = srcu_read_lock(&qdev->dev_lock); 206 if (qdev->dev_state == QAIC_ONLINE) { 207 qaic_release_usr(qdev, usr); 208 for (i = 0; i < qdev->num_dbc; ++i) 209 if (qdev->dbc[i].usr && qdev->dbc[i].usr->handle == usr->handle) 210 release_dbc(qdev, i); 211 } 212 srcu_read_unlock(&qdev->dev_lock, qdev_rcu_id); 213 214 mutex_lock(&qddev->users_mutex); 215 if (!list_empty(&usr->node)) 216 list_del_init(&usr->node); 217 mutex_unlock(&qddev->users_mutex); 218 } 219 220 srcu_read_unlock(&usr->qddev_lock, usr_rcu_id); 221 kref_put(&usr->ref_count, free_usr); 222 223 file->driver_priv = NULL; 224 } 225 226 DEFINE_DRM_ACCEL_FOPS(qaic_accel_fops); 227 228 static const struct drm_ioctl_desc qaic_drm_ioctls[] = { 229 DRM_IOCTL_DEF_DRV(QAIC_MANAGE, qaic_manage_ioctl, 0), 230 DRM_IOCTL_DEF_DRV(QAIC_CREATE_BO, qaic_create_bo_ioctl, 0), 231 DRM_IOCTL_DEF_DRV(QAIC_MMAP_BO, qaic_mmap_bo_ioctl, 0), 232 DRM_IOCTL_DEF_DRV(QAIC_ATTACH_SLICE_BO, qaic_attach_slice_bo_ioctl, 0), 233 DRM_IOCTL_DEF_DRV(QAIC_EXECUTE_BO, qaic_execute_bo_ioctl, 0), 234 DRM_IOCTL_DEF_DRV(QAIC_PARTIAL_EXECUTE_BO, qaic_partial_execute_bo_ioctl, 0), 235 DRM_IOCTL_DEF_DRV(QAIC_WAIT_BO, qaic_wait_bo_ioctl, 0), 236 DRM_IOCTL_DEF_DRV(QAIC_PERF_STATS_BO, qaic_perf_stats_bo_ioctl, 0), 237 DRM_IOCTL_DEF_DRV(QAIC_DETACH_SLICE_BO, qaic_detach_slice_bo_ioctl, 0), 238 }; 239 240 static const struct drm_driver qaic_accel_driver = { 241 .driver_features = DRIVER_GEM | DRIVER_COMPUTE_ACCEL, 242 243 .name = QAIC_NAME, 244 .desc = QAIC_DESC, 245 246 .fops = &qaic_accel_fops, 247 .open = qaic_open, 248 .postclose = qaic_postclose, 249 250 .ioctls = qaic_drm_ioctls, 251 .num_ioctls = ARRAY_SIZE(qaic_drm_ioctls), 252 .gem_prime_import = qaic_gem_prime_import, 253 }; 254 255 static int qaic_create_drm_device(struct qaic_device *qdev, s32 partition_id) 256 { 257 struct qaic_drm_device *qddev = qdev->qddev; 258 struct drm_device *drm = to_drm(qddev); 259 int ret; 260 261 /* Hold off implementing partitions until the uapi is determined */ 262 if (partition_id != QAIC_NO_PARTITION) 263 return -EINVAL; 264 265 qddev->partition_id = partition_id; 266 267 ret = drm_dev_register(drm, 0); 268 if (ret) { 269 pci_dbg(qdev->pdev, "drm_dev_register failed %d\n", ret); 270 return ret; 271 } 272 273 qaic_debugfs_init(qddev); 274 275 return ret; 276 } 277 278 static void qaic_destroy_drm_device(struct qaic_device *qdev, s32 partition_id) 279 { 280 struct qaic_drm_device *qddev = qdev->qddev; 281 struct drm_device *drm = to_drm(qddev); 282 struct qaic_user *usr; 283 284 drm_dev_unregister(drm); 285 qddev->partition_id = 0; 286 /* 287 * Existing users get unresolvable errors till they close FDs. 288 * Need to sync carefully with users calling close(). The 289 * list of users can be modified elsewhere when the lock isn't 290 * held here, but the sync'ing the srcu with the mutex held 291 * could deadlock. Grab the mutex so that the list will be 292 * unmodified. The user we get will exist as long as the 293 * lock is held. Signal that the qcdev is going away, and 294 * grab a reference to the user so they don't go away for 295 * synchronize_srcu(). Then release the mutex to avoid 296 * deadlock and make sure the user has observed the signal. 297 * With the lock released, we cannot maintain any state of the 298 * user list. 299 */ 300 mutex_lock(&qddev->users_mutex); 301 while (!list_empty(&qddev->users)) { 302 usr = list_first_entry(&qddev->users, struct qaic_user, node); 303 list_del_init(&usr->node); 304 kref_get(&usr->ref_count); 305 usr->qddev = NULL; 306 mutex_unlock(&qddev->users_mutex); 307 synchronize_srcu(&usr->qddev_lock); 308 kref_put(&usr->ref_count, free_usr); 309 mutex_lock(&qddev->users_mutex); 310 } 311 mutex_unlock(&qddev->users_mutex); 312 } 313 314 static int qaic_mhi_probe(struct mhi_device *mhi_dev, const struct mhi_device_id *id) 315 { 316 u16 major = -1, minor = -1; 317 struct qaic_device *qdev; 318 int ret; 319 320 /* 321 * Invoking this function indicates that the control channel to the 322 * device is available. We use that as a signal to indicate that 323 * the device side firmware has booted. The device side firmware 324 * manages the device resources, so we need to communicate with it 325 * via the control channel in order to utilize the device. Therefore 326 * we wait until this signal to create the drm dev that userspace will 327 * use to control the device, because without the device side firmware, 328 * userspace can't do anything useful. 329 */ 330 331 qdev = pci_get_drvdata(to_pci_dev(mhi_dev->mhi_cntrl->cntrl_dev)); 332 333 dev_set_drvdata(&mhi_dev->dev, qdev); 334 qdev->cntl_ch = mhi_dev; 335 336 ret = qaic_control_open(qdev); 337 if (ret) { 338 pci_dbg(qdev->pdev, "%s: control_open failed %d\n", __func__, ret); 339 return ret; 340 } 341 342 qdev->dev_state = QAIC_BOOT; 343 ret = get_cntl_version(qdev, NULL, &major, &minor); 344 if (ret || major != CNTL_MAJOR || minor > CNTL_MINOR) { 345 pci_err(qdev->pdev, "%s: Control protocol version (%d.%d) not supported. Supported version is (%d.%d). Ret: %d\n", 346 __func__, major, minor, CNTL_MAJOR, CNTL_MINOR, ret); 347 ret = -EINVAL; 348 goto close_control; 349 } 350 qdev->dev_state = QAIC_ONLINE; 351 kobject_uevent(&(to_accel_kdev(qdev->qddev))->kobj, KOBJ_ONLINE); 352 353 return ret; 354 355 close_control: 356 qaic_control_close(qdev); 357 return ret; 358 } 359 360 static void qaic_mhi_remove(struct mhi_device *mhi_dev) 361 { 362 /* This is redundant since we have already observed the device crash */ 363 } 364 365 static void qaic_notify_reset(struct qaic_device *qdev) 366 { 367 int i; 368 369 kobject_uevent(&(to_accel_kdev(qdev->qddev))->kobj, KOBJ_OFFLINE); 370 qdev->dev_state = QAIC_OFFLINE; 371 /* wake up any waiters to avoid waiting for timeouts at sync */ 372 wake_all_cntl(qdev); 373 for (i = 0; i < qdev->num_dbc; ++i) 374 wakeup_dbc(qdev, i); 375 synchronize_srcu(&qdev->dev_lock); 376 } 377 378 void qaic_dev_reset_clean_local_state(struct qaic_device *qdev) 379 { 380 int i; 381 382 qaic_notify_reset(qdev); 383 384 /* start tearing things down */ 385 for (i = 0; i < qdev->num_dbc; ++i) 386 release_dbc(qdev, i); 387 } 388 389 static struct qaic_device *create_qdev(struct pci_dev *pdev, 390 const struct qaic_device_config *config) 391 { 392 struct device *dev = &pdev->dev; 393 struct qaic_drm_device *qddev; 394 struct qaic_device *qdev; 395 struct drm_device *drm; 396 int i, ret; 397 398 qdev = devm_kzalloc(dev, sizeof(*qdev), GFP_KERNEL); 399 if (!qdev) 400 return NULL; 401 402 qdev->dev_state = QAIC_OFFLINE; 403 qdev->num_dbc = 16; 404 qdev->dbc = devm_kcalloc(dev, qdev->num_dbc, sizeof(*qdev->dbc), GFP_KERNEL); 405 if (!qdev->dbc) 406 return NULL; 407 408 qddev = devm_drm_dev_alloc(&pdev->dev, &qaic_accel_driver, struct qaic_drm_device, drm); 409 if (IS_ERR(qddev)) 410 return NULL; 411 412 drm = to_drm(qddev); 413 pci_set_drvdata(pdev, qdev); 414 415 ret = drmm_mutex_init(drm, &qddev->users_mutex); 416 if (ret) 417 return NULL; 418 ret = drmm_add_action_or_reset(drm, qaicm_pci_release, NULL); 419 if (ret) 420 return NULL; 421 ret = drmm_mutex_init(drm, &qdev->cntl_mutex); 422 if (ret) 423 return NULL; 424 ret = drmm_mutex_init(drm, &qdev->bootlog_mutex); 425 if (ret) 426 return NULL; 427 428 qdev->cntl_wq = qaicm_wq_init(drm, "qaic_cntl"); 429 if (IS_ERR(qdev->cntl_wq)) 430 return NULL; 431 qdev->qts_wq = qaicm_wq_init(drm, "qaic_ts"); 432 if (IS_ERR(qdev->qts_wq)) 433 return NULL; 434 435 ret = qaicm_srcu_init(drm, &qdev->dev_lock); 436 if (ret) 437 return NULL; 438 439 qdev->qddev = qddev; 440 qdev->pdev = pdev; 441 qddev->qdev = qdev; 442 443 INIT_LIST_HEAD(&qdev->cntl_xfer_list); 444 INIT_LIST_HEAD(&qdev->bootlog); 445 INIT_LIST_HEAD(&qddev->users); 446 447 for (i = 0; i < qdev->num_dbc; ++i) { 448 spin_lock_init(&qdev->dbc[i].xfer_lock); 449 qdev->dbc[i].qdev = qdev; 450 qdev->dbc[i].id = i; 451 INIT_LIST_HEAD(&qdev->dbc[i].xfer_list); 452 ret = qaicm_srcu_init(drm, &qdev->dbc[i].ch_lock); 453 if (ret) 454 return NULL; 455 init_waitqueue_head(&qdev->dbc[i].dbc_release); 456 INIT_LIST_HEAD(&qdev->dbc[i].bo_lists); 457 ret = drmm_mutex_init(drm, &qdev->dbc[i].req_lock); 458 if (ret) 459 return NULL; 460 } 461 462 return qdev; 463 } 464 465 static int init_pci(struct qaic_device *qdev, struct pci_dev *pdev, 466 const struct qaic_device_config *config) 467 { 468 int bars; 469 int ret; 470 471 bars = pci_select_bars(pdev, IORESOURCE_MEM) & 0x3f; 472 473 /* make sure the device has the expected BARs */ 474 if (bars != config->bar_mask) { 475 pci_dbg(pdev, "%s: expected BARs %#x not found in device. Found %#x\n", 476 __func__, config->bar_mask, bars); 477 return -EINVAL; 478 } 479 480 ret = pcim_enable_device(pdev); 481 if (ret) 482 return ret; 483 484 ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); 485 if (ret) 486 return ret; 487 dma_set_max_seg_size(&pdev->dev, UINT_MAX); 488 489 qdev->bar_mhi = devm_ioremap_resource(&pdev->dev, &pdev->resource[config->mhi_bar_idx]); 490 if (IS_ERR(qdev->bar_mhi)) 491 return PTR_ERR(qdev->bar_mhi); 492 493 qdev->bar_dbc = devm_ioremap_resource(&pdev->dev, &pdev->resource[config->dbc_bar_idx]); 494 if (IS_ERR(qdev->bar_dbc)) 495 return PTR_ERR(qdev->bar_dbc); 496 497 /* Managed release since we use pcim_enable_device above */ 498 pci_set_master(pdev); 499 500 return 0; 501 } 502 503 static int init_msi(struct qaic_device *qdev, struct pci_dev *pdev) 504 { 505 int irq_count = qdev->num_dbc + 1; 506 int mhi_irq; 507 int ret; 508 int i; 509 510 /* Managed release since we use pcim_enable_device */ 511 ret = pci_alloc_irq_vectors(pdev, irq_count, irq_count, PCI_IRQ_MSI | PCI_IRQ_MSIX); 512 if (ret == -ENOSPC) { 513 ret = pci_alloc_irq_vectors(pdev, 1, 1, PCI_IRQ_MSI | PCI_IRQ_MSIX); 514 if (ret < 0) 515 return ret; 516 517 /* 518 * Operate in one MSI mode. All interrupts will be directed to 519 * MSI0; every interrupt will wake up all the interrupt handlers 520 * (MHI and DBC[0-15]). Since the interrupt is now shared, it is 521 * not disabled during DBC threaded handler, but only one thread 522 * will be allowed to run per DBC, so while it can be 523 * interrupted, it shouldn't race with itself. 524 */ 525 qdev->single_msi = true; 526 pci_info(pdev, "Allocating %d MSIs failed, operating in 1 MSI mode. Performance may be impacted.\n", 527 irq_count); 528 } else if (ret < 0) { 529 return ret; 530 } 531 532 mhi_irq = pci_irq_vector(pdev, 0); 533 if (mhi_irq < 0) 534 return mhi_irq; 535 536 for (i = 0; i < qdev->num_dbc; ++i) { 537 ret = devm_request_threaded_irq(&pdev->dev, 538 pci_irq_vector(pdev, qdev->single_msi ? 0 : i + 1), 539 dbc_irq_handler, dbc_irq_threaded_fn, IRQF_SHARED, 540 "qaic_dbc", &qdev->dbc[i]); 541 if (ret) 542 return ret; 543 544 if (datapath_polling) { 545 qdev->dbc[i].irq = pci_irq_vector(pdev, qdev->single_msi ? 0 : i + 1); 546 if (!qdev->single_msi) 547 disable_irq_nosync(qdev->dbc[i].irq); 548 INIT_WORK(&qdev->dbc[i].poll_work, irq_polling_work); 549 } 550 } 551 552 return mhi_irq; 553 } 554 555 static int qaic_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) 556 { 557 struct qaic_device_config *config = (struct qaic_device_config *)id->driver_data; 558 struct qaic_device *qdev; 559 int mhi_irq; 560 int ret; 561 int i; 562 563 qdev = create_qdev(pdev, config); 564 if (!qdev) 565 return -ENOMEM; 566 567 ret = init_pci(qdev, pdev, config); 568 if (ret) 569 return ret; 570 571 for (i = 0; i < qdev->num_dbc; ++i) 572 qdev->dbc[i].dbc_base = qdev->bar_dbc + QAIC_DBC_OFF(i); 573 574 mhi_irq = init_msi(qdev, pdev); 575 if (mhi_irq < 0) 576 return mhi_irq; 577 578 ret = qaic_create_drm_device(qdev, QAIC_NO_PARTITION); 579 if (ret) 580 return ret; 581 582 qdev->mhi_cntrl = qaic_mhi_register_controller(pdev, qdev->bar_mhi, mhi_irq, 583 qdev->single_msi, config->family); 584 if (IS_ERR(qdev->mhi_cntrl)) { 585 ret = PTR_ERR(qdev->mhi_cntrl); 586 qaic_destroy_drm_device(qdev, QAIC_NO_PARTITION); 587 return ret; 588 } 589 590 return 0; 591 } 592 593 static void qaic_pci_remove(struct pci_dev *pdev) 594 { 595 struct qaic_device *qdev = pci_get_drvdata(pdev); 596 597 if (!qdev) 598 return; 599 600 qaic_dev_reset_clean_local_state(qdev); 601 qaic_mhi_free_controller(qdev->mhi_cntrl, link_up); 602 qaic_destroy_drm_device(qdev, QAIC_NO_PARTITION); 603 } 604 605 static void qaic_pci_shutdown(struct pci_dev *pdev) 606 { 607 /* see qaic_exit for what link_up is doing */ 608 link_up = true; 609 qaic_pci_remove(pdev); 610 } 611 612 static pci_ers_result_t qaic_pci_error_detected(struct pci_dev *pdev, pci_channel_state_t error) 613 { 614 return PCI_ERS_RESULT_NEED_RESET; 615 } 616 617 static void qaic_pci_reset_prepare(struct pci_dev *pdev) 618 { 619 struct qaic_device *qdev = pci_get_drvdata(pdev); 620 621 qaic_notify_reset(qdev); 622 qaic_mhi_start_reset(qdev->mhi_cntrl); 623 qaic_dev_reset_clean_local_state(qdev); 624 } 625 626 static void qaic_pci_reset_done(struct pci_dev *pdev) 627 { 628 struct qaic_device *qdev = pci_get_drvdata(pdev); 629 630 qaic_mhi_reset_done(qdev->mhi_cntrl); 631 } 632 633 static const struct mhi_device_id qaic_mhi_match_table[] = { 634 { .chan = "QAIC_CONTROL", }, 635 {}, 636 }; 637 638 static struct mhi_driver qaic_mhi_driver = { 639 .id_table = qaic_mhi_match_table, 640 .remove = qaic_mhi_remove, 641 .probe = qaic_mhi_probe, 642 .ul_xfer_cb = qaic_mhi_ul_xfer_cb, 643 .dl_xfer_cb = qaic_mhi_dl_xfer_cb, 644 .driver = { 645 .name = "qaic_mhi", 646 }, 647 }; 648 649 static const struct pci_device_id qaic_ids[] = { 650 { PCI_DEVICE_DATA(QCOM, AIC080, (kernel_ulong_t)&aic080_config), }, 651 { PCI_DEVICE_DATA(QCOM, AIC100, (kernel_ulong_t)&aic100_config), }, 652 { PCI_DEVICE_DATA(QCOM, AIC200, (kernel_ulong_t)&aic200_config), }, 653 { } 654 }; 655 MODULE_DEVICE_TABLE(pci, qaic_ids); 656 657 static const struct pci_error_handlers qaic_pci_err_handler = { 658 .error_detected = qaic_pci_error_detected, 659 .reset_prepare = qaic_pci_reset_prepare, 660 .reset_done = qaic_pci_reset_done, 661 }; 662 663 static struct pci_driver qaic_pci_driver = { 664 .name = QAIC_NAME, 665 .id_table = qaic_ids, 666 .probe = qaic_pci_probe, 667 .remove = qaic_pci_remove, 668 .shutdown = qaic_pci_shutdown, 669 .err_handler = &qaic_pci_err_handler, 670 }; 671 672 static int __init qaic_init(void) 673 { 674 int ret; 675 676 ret = pci_register_driver(&qaic_pci_driver); 677 if (ret) { 678 pr_debug("qaic: pci_register_driver failed %d\n", ret); 679 return ret; 680 } 681 682 ret = mhi_driver_register(&qaic_mhi_driver); 683 if (ret) { 684 pr_debug("qaic: mhi_driver_register failed %d\n", ret); 685 goto free_pci; 686 } 687 688 ret = sahara_register(); 689 if (ret) { 690 pr_debug("qaic: sahara_register failed %d\n", ret); 691 goto free_mhi; 692 } 693 694 ret = qaic_timesync_init(); 695 if (ret) 696 pr_debug("qaic: qaic_timesync_init failed %d\n", ret); 697 698 ret = qaic_bootlog_register(); 699 if (ret) 700 pr_debug("qaic: qaic_bootlog_register failed %d\n", ret); 701 702 ret = qaic_ras_register(); 703 if (ret) 704 pr_debug("qaic: qaic_ras_register failed %d\n", ret); 705 706 return 0; 707 708 free_mhi: 709 mhi_driver_unregister(&qaic_mhi_driver); 710 free_pci: 711 pci_unregister_driver(&qaic_pci_driver); 712 return ret; 713 } 714 715 static void __exit qaic_exit(void) 716 { 717 /* 718 * We assume that qaic_pci_remove() is called due to a hotplug event 719 * which would mean that the link is down, and thus 720 * qaic_mhi_free_controller() should not try to access the device during 721 * cleanup. 722 * We call pci_unregister_driver() below, which also triggers 723 * qaic_pci_remove(), but since this is module exit, we expect the link 724 * to the device to be up, in which case qaic_mhi_free_controller() 725 * should try to access the device during cleanup to put the device in 726 * a sane state. 727 * For that reason, we set link_up here to let qaic_mhi_free_controller 728 * know the expected link state. Since the module is going to be 729 * removed at the end of this, we don't need to worry about 730 * reinitializing the link_up state after the cleanup is done. 731 */ 732 link_up = true; 733 qaic_ras_unregister(); 734 qaic_bootlog_unregister(); 735 qaic_timesync_deinit(); 736 sahara_unregister(); 737 mhi_driver_unregister(&qaic_mhi_driver); 738 pci_unregister_driver(&qaic_pci_driver); 739 } 740 741 module_init(qaic_init); 742 module_exit(qaic_exit); 743 744 MODULE_AUTHOR(QAIC_DESC " Kernel Driver Team"); 745 MODULE_DESCRIPTION(QAIC_DESC " Accel Driver"); 746 MODULE_LICENSE("GPL"); 747