1 // SPDX-License-Identifier: GPL-2.0-only 2 3 /* Copyright (c) 2019-2021, The Linux Foundation. All rights reserved. */ 4 /* Copyright (c) 2021-2023 Qualcomm Innovation Center, Inc. All rights reserved. */ 5 6 #include <linux/delay.h> 7 #include <linux/dma-mapping.h> 8 #include <linux/idr.h> 9 #include <linux/interrupt.h> 10 #include <linux/list.h> 11 #include <linux/kobject.h> 12 #include <linux/kref.h> 13 #include <linux/mhi.h> 14 #include <linux/module.h> 15 #include <linux/msi.h> 16 #include <linux/mutex.h> 17 #include <linux/pci.h> 18 #include <linux/spinlock.h> 19 #include <linux/workqueue.h> 20 #include <linux/wait.h> 21 #include <drm/drm_accel.h> 22 #include <drm/drm_drv.h> 23 #include <drm/drm_file.h> 24 #include <drm/drm_gem.h> 25 #include <drm/drm_ioctl.h> 26 #include <drm/drm_managed.h> 27 #include <uapi/drm/qaic_accel.h> 28 29 #include "mhi_controller.h" 30 #include "qaic.h" 31 #include "qaic_debugfs.h" 32 #include "qaic_timesync.h" 33 #include "sahara.h" 34 35 MODULE_IMPORT_NS("DMA_BUF"); 36 37 #define PCI_DEVICE_ID_QCOM_AIC080 0xa080 38 #define PCI_DEVICE_ID_QCOM_AIC100 0xa100 39 #define PCI_DEVICE_ID_QCOM_AIC200 0xa110 40 #define QAIC_NAME "qaic" 41 #define QAIC_DESC "Qualcomm Cloud AI Accelerators" 42 #define CNTL_MAJOR 5 43 #define CNTL_MINOR 0 44 45 struct qaic_device_config { 46 /* Indicates the AIC family the device belongs to */ 47 int family; 48 /* A bitmask representing the available BARs */ 49 int bar_mask; 50 /* An index value used to identify the MHI controller BAR */ 51 unsigned int mhi_bar_idx; 52 /* An index value used to identify the DBCs BAR */ 53 unsigned int dbc_bar_idx; 54 }; 55 56 static const struct qaic_device_config aic080_config = { 57 .family = FAMILY_AIC100, 58 .bar_mask = BIT(0) | BIT(2) | BIT(4), 59 .mhi_bar_idx = 0, 60 .dbc_bar_idx = 2, 61 }; 62 63 static const struct qaic_device_config aic100_config = { 64 .family = FAMILY_AIC100, 65 .bar_mask = BIT(0) | BIT(2) | BIT(4), 66 .mhi_bar_idx = 0, 67 .dbc_bar_idx = 2, 68 }; 69 70 static const struct qaic_device_config aic200_config = { 71 .family = FAMILY_AIC200, 72 .bar_mask = BIT(0) | BIT(1) | BIT(2) | BIT(4), 73 .mhi_bar_idx = 1, 74 .dbc_bar_idx = 2, 75 }; 76 77 bool datapath_polling; 78 module_param(datapath_polling, bool, 0400); 79 MODULE_PARM_DESC(datapath_polling, "Operate the datapath in polling mode"); 80 static bool link_up; 81 static DEFINE_IDA(qaic_usrs); 82 83 static void qaicm_wq_release(struct drm_device *dev, void *res) 84 { 85 struct workqueue_struct *wq = res; 86 87 destroy_workqueue(wq); 88 } 89 90 static struct workqueue_struct *qaicm_wq_init(struct drm_device *dev, const char *name) 91 { 92 struct workqueue_struct *wq; 93 int ret; 94 95 wq = alloc_workqueue("%s", WQ_UNBOUND, 0, name); 96 if (!wq) 97 return ERR_PTR(-ENOMEM); 98 ret = drmm_add_action_or_reset(dev, qaicm_wq_release, wq); 99 if (ret) 100 return ERR_PTR(ret); 101 102 return wq; 103 } 104 105 static void qaicm_srcu_release(struct drm_device *dev, void *res) 106 { 107 struct srcu_struct *lock = res; 108 109 cleanup_srcu_struct(lock); 110 } 111 112 static int qaicm_srcu_init(struct drm_device *dev, struct srcu_struct *lock) 113 { 114 int ret; 115 116 ret = init_srcu_struct(lock); 117 if (ret) 118 return ret; 119 120 return drmm_add_action_or_reset(dev, qaicm_srcu_release, lock); 121 } 122 123 static void qaicm_pci_release(struct drm_device *dev, void *res) 124 { 125 struct qaic_device *qdev = to_qaic_device(dev); 126 127 pci_set_drvdata(qdev->pdev, NULL); 128 } 129 130 static void free_usr(struct kref *kref) 131 { 132 struct qaic_user *usr = container_of(kref, struct qaic_user, ref_count); 133 134 cleanup_srcu_struct(&usr->qddev_lock); 135 ida_free(&qaic_usrs, usr->handle); 136 kfree(usr); 137 } 138 139 static int qaic_open(struct drm_device *dev, struct drm_file *file) 140 { 141 struct qaic_drm_device *qddev = to_qaic_drm_device(dev); 142 struct qaic_device *qdev = qddev->qdev; 143 struct qaic_user *usr; 144 int rcu_id; 145 int ret; 146 147 rcu_id = srcu_read_lock(&qdev->dev_lock); 148 if (qdev->dev_state != QAIC_ONLINE) { 149 ret = -ENODEV; 150 goto dev_unlock; 151 } 152 153 usr = kmalloc(sizeof(*usr), GFP_KERNEL); 154 if (!usr) { 155 ret = -ENOMEM; 156 goto dev_unlock; 157 } 158 159 usr->handle = ida_alloc(&qaic_usrs, GFP_KERNEL); 160 if (usr->handle < 0) { 161 ret = usr->handle; 162 goto free_usr; 163 } 164 usr->qddev = qddev; 165 atomic_set(&usr->chunk_id, 0); 166 init_srcu_struct(&usr->qddev_lock); 167 kref_init(&usr->ref_count); 168 169 ret = mutex_lock_interruptible(&qddev->users_mutex); 170 if (ret) 171 goto cleanup_usr; 172 173 list_add(&usr->node, &qddev->users); 174 mutex_unlock(&qddev->users_mutex); 175 176 file->driver_priv = usr; 177 178 srcu_read_unlock(&qdev->dev_lock, rcu_id); 179 return 0; 180 181 cleanup_usr: 182 cleanup_srcu_struct(&usr->qddev_lock); 183 ida_free(&qaic_usrs, usr->handle); 184 free_usr: 185 kfree(usr); 186 dev_unlock: 187 srcu_read_unlock(&qdev->dev_lock, rcu_id); 188 return ret; 189 } 190 191 static void qaic_postclose(struct drm_device *dev, struct drm_file *file) 192 { 193 struct qaic_user *usr = file->driver_priv; 194 struct qaic_drm_device *qddev; 195 struct qaic_device *qdev; 196 int qdev_rcu_id; 197 int usr_rcu_id; 198 int i; 199 200 qddev = usr->qddev; 201 usr_rcu_id = srcu_read_lock(&usr->qddev_lock); 202 if (qddev) { 203 qdev = qddev->qdev; 204 qdev_rcu_id = srcu_read_lock(&qdev->dev_lock); 205 if (qdev->dev_state == QAIC_ONLINE) { 206 qaic_release_usr(qdev, usr); 207 for (i = 0; i < qdev->num_dbc; ++i) 208 if (qdev->dbc[i].usr && qdev->dbc[i].usr->handle == usr->handle) 209 release_dbc(qdev, i); 210 } 211 srcu_read_unlock(&qdev->dev_lock, qdev_rcu_id); 212 213 mutex_lock(&qddev->users_mutex); 214 if (!list_empty(&usr->node)) 215 list_del_init(&usr->node); 216 mutex_unlock(&qddev->users_mutex); 217 } 218 219 srcu_read_unlock(&usr->qddev_lock, usr_rcu_id); 220 kref_put(&usr->ref_count, free_usr); 221 222 file->driver_priv = NULL; 223 } 224 225 DEFINE_DRM_ACCEL_FOPS(qaic_accel_fops); 226 227 static const struct drm_ioctl_desc qaic_drm_ioctls[] = { 228 DRM_IOCTL_DEF_DRV(QAIC_MANAGE, qaic_manage_ioctl, 0), 229 DRM_IOCTL_DEF_DRV(QAIC_CREATE_BO, qaic_create_bo_ioctl, 0), 230 DRM_IOCTL_DEF_DRV(QAIC_MMAP_BO, qaic_mmap_bo_ioctl, 0), 231 DRM_IOCTL_DEF_DRV(QAIC_ATTACH_SLICE_BO, qaic_attach_slice_bo_ioctl, 0), 232 DRM_IOCTL_DEF_DRV(QAIC_EXECUTE_BO, qaic_execute_bo_ioctl, 0), 233 DRM_IOCTL_DEF_DRV(QAIC_PARTIAL_EXECUTE_BO, qaic_partial_execute_bo_ioctl, 0), 234 DRM_IOCTL_DEF_DRV(QAIC_WAIT_BO, qaic_wait_bo_ioctl, 0), 235 DRM_IOCTL_DEF_DRV(QAIC_PERF_STATS_BO, qaic_perf_stats_bo_ioctl, 0), 236 DRM_IOCTL_DEF_DRV(QAIC_DETACH_SLICE_BO, qaic_detach_slice_bo_ioctl, 0), 237 }; 238 239 static const struct drm_driver qaic_accel_driver = { 240 .driver_features = DRIVER_GEM | DRIVER_COMPUTE_ACCEL, 241 242 .name = QAIC_NAME, 243 .desc = QAIC_DESC, 244 245 .fops = &qaic_accel_fops, 246 .open = qaic_open, 247 .postclose = qaic_postclose, 248 249 .ioctls = qaic_drm_ioctls, 250 .num_ioctls = ARRAY_SIZE(qaic_drm_ioctls), 251 .gem_prime_import = qaic_gem_prime_import, 252 }; 253 254 static int qaic_create_drm_device(struct qaic_device *qdev, s32 partition_id) 255 { 256 struct qaic_drm_device *qddev = qdev->qddev; 257 struct drm_device *drm = to_drm(qddev); 258 int ret; 259 260 /* Hold off implementing partitions until the uapi is determined */ 261 if (partition_id != QAIC_NO_PARTITION) 262 return -EINVAL; 263 264 qddev->partition_id = partition_id; 265 266 ret = drm_dev_register(drm, 0); 267 if (ret) { 268 pci_dbg(qdev->pdev, "drm_dev_register failed %d\n", ret); 269 return ret; 270 } 271 272 qaic_debugfs_init(qddev); 273 274 return ret; 275 } 276 277 static void qaic_destroy_drm_device(struct qaic_device *qdev, s32 partition_id) 278 { 279 struct qaic_drm_device *qddev = qdev->qddev; 280 struct drm_device *drm = to_drm(qddev); 281 struct qaic_user *usr; 282 283 drm_dev_unregister(drm); 284 qddev->partition_id = 0; 285 /* 286 * Existing users get unresolvable errors till they close FDs. 287 * Need to sync carefully with users calling close(). The 288 * list of users can be modified elsewhere when the lock isn't 289 * held here, but the sync'ing the srcu with the mutex held 290 * could deadlock. Grab the mutex so that the list will be 291 * unmodified. The user we get will exist as long as the 292 * lock is held. Signal that the qcdev is going away, and 293 * grab a reference to the user so they don't go away for 294 * synchronize_srcu(). Then release the mutex to avoid 295 * deadlock and make sure the user has observed the signal. 296 * With the lock released, we cannot maintain any state of the 297 * user list. 298 */ 299 mutex_lock(&qddev->users_mutex); 300 while (!list_empty(&qddev->users)) { 301 usr = list_first_entry(&qddev->users, struct qaic_user, node); 302 list_del_init(&usr->node); 303 kref_get(&usr->ref_count); 304 usr->qddev = NULL; 305 mutex_unlock(&qddev->users_mutex); 306 synchronize_srcu(&usr->qddev_lock); 307 kref_put(&usr->ref_count, free_usr); 308 mutex_lock(&qddev->users_mutex); 309 } 310 mutex_unlock(&qddev->users_mutex); 311 } 312 313 static int qaic_mhi_probe(struct mhi_device *mhi_dev, const struct mhi_device_id *id) 314 { 315 u16 major = -1, minor = -1; 316 struct qaic_device *qdev; 317 int ret; 318 319 /* 320 * Invoking this function indicates that the control channel to the 321 * device is available. We use that as a signal to indicate that 322 * the device side firmware has booted. The device side firmware 323 * manages the device resources, so we need to communicate with it 324 * via the control channel in order to utilize the device. Therefore 325 * we wait until this signal to create the drm dev that userspace will 326 * use to control the device, because without the device side firmware, 327 * userspace can't do anything useful. 328 */ 329 330 qdev = pci_get_drvdata(to_pci_dev(mhi_dev->mhi_cntrl->cntrl_dev)); 331 332 dev_set_drvdata(&mhi_dev->dev, qdev); 333 qdev->cntl_ch = mhi_dev; 334 335 ret = qaic_control_open(qdev); 336 if (ret) { 337 pci_dbg(qdev->pdev, "%s: control_open failed %d\n", __func__, ret); 338 return ret; 339 } 340 341 qdev->dev_state = QAIC_BOOT; 342 ret = get_cntl_version(qdev, NULL, &major, &minor); 343 if (ret || major != CNTL_MAJOR || minor > CNTL_MINOR) { 344 pci_err(qdev->pdev, "%s: Control protocol version (%d.%d) not supported. Supported version is (%d.%d). Ret: %d\n", 345 __func__, major, minor, CNTL_MAJOR, CNTL_MINOR, ret); 346 ret = -EINVAL; 347 goto close_control; 348 } 349 qdev->dev_state = QAIC_ONLINE; 350 kobject_uevent(&(to_accel_kdev(qdev->qddev))->kobj, KOBJ_ONLINE); 351 352 return ret; 353 354 close_control: 355 qaic_control_close(qdev); 356 return ret; 357 } 358 359 static void qaic_mhi_remove(struct mhi_device *mhi_dev) 360 { 361 /* This is redundant since we have already observed the device crash */ 362 } 363 364 static void qaic_notify_reset(struct qaic_device *qdev) 365 { 366 int i; 367 368 kobject_uevent(&(to_accel_kdev(qdev->qddev))->kobj, KOBJ_OFFLINE); 369 qdev->dev_state = QAIC_OFFLINE; 370 /* wake up any waiters to avoid waiting for timeouts at sync */ 371 wake_all_cntl(qdev); 372 for (i = 0; i < qdev->num_dbc; ++i) 373 wakeup_dbc(qdev, i); 374 synchronize_srcu(&qdev->dev_lock); 375 } 376 377 void qaic_dev_reset_clean_local_state(struct qaic_device *qdev) 378 { 379 int i; 380 381 qaic_notify_reset(qdev); 382 383 /* start tearing things down */ 384 for (i = 0; i < qdev->num_dbc; ++i) 385 release_dbc(qdev, i); 386 } 387 388 static struct qaic_device *create_qdev(struct pci_dev *pdev, 389 const struct qaic_device_config *config) 390 { 391 struct device *dev = &pdev->dev; 392 struct qaic_drm_device *qddev; 393 struct qaic_device *qdev; 394 struct drm_device *drm; 395 int i, ret; 396 397 qdev = devm_kzalloc(dev, sizeof(*qdev), GFP_KERNEL); 398 if (!qdev) 399 return NULL; 400 401 qdev->dev_state = QAIC_OFFLINE; 402 qdev->num_dbc = 16; 403 qdev->dbc = devm_kcalloc(dev, qdev->num_dbc, sizeof(*qdev->dbc), GFP_KERNEL); 404 if (!qdev->dbc) 405 return NULL; 406 407 qddev = devm_drm_dev_alloc(&pdev->dev, &qaic_accel_driver, struct qaic_drm_device, drm); 408 if (IS_ERR(qddev)) 409 return NULL; 410 411 drm = to_drm(qddev); 412 pci_set_drvdata(pdev, qdev); 413 414 ret = drmm_mutex_init(drm, &qddev->users_mutex); 415 if (ret) 416 return NULL; 417 ret = drmm_add_action_or_reset(drm, qaicm_pci_release, NULL); 418 if (ret) 419 return NULL; 420 ret = drmm_mutex_init(drm, &qdev->cntl_mutex); 421 if (ret) 422 return NULL; 423 ret = drmm_mutex_init(drm, &qdev->bootlog_mutex); 424 if (ret) 425 return NULL; 426 427 qdev->cntl_wq = qaicm_wq_init(drm, "qaic_cntl"); 428 if (IS_ERR(qdev->cntl_wq)) 429 return NULL; 430 qdev->qts_wq = qaicm_wq_init(drm, "qaic_ts"); 431 if (IS_ERR(qdev->qts_wq)) 432 return NULL; 433 434 ret = qaicm_srcu_init(drm, &qdev->dev_lock); 435 if (ret) 436 return NULL; 437 438 qdev->qddev = qddev; 439 qdev->pdev = pdev; 440 qddev->qdev = qdev; 441 442 INIT_LIST_HEAD(&qdev->cntl_xfer_list); 443 INIT_LIST_HEAD(&qdev->bootlog); 444 INIT_LIST_HEAD(&qddev->users); 445 446 for (i = 0; i < qdev->num_dbc; ++i) { 447 spin_lock_init(&qdev->dbc[i].xfer_lock); 448 qdev->dbc[i].qdev = qdev; 449 qdev->dbc[i].id = i; 450 INIT_LIST_HEAD(&qdev->dbc[i].xfer_list); 451 ret = qaicm_srcu_init(drm, &qdev->dbc[i].ch_lock); 452 if (ret) 453 return NULL; 454 init_waitqueue_head(&qdev->dbc[i].dbc_release); 455 INIT_LIST_HEAD(&qdev->dbc[i].bo_lists); 456 } 457 458 return qdev; 459 } 460 461 static int init_pci(struct qaic_device *qdev, struct pci_dev *pdev, 462 const struct qaic_device_config *config) 463 { 464 int bars; 465 int ret; 466 467 bars = pci_select_bars(pdev, IORESOURCE_MEM) & 0x3f; 468 469 /* make sure the device has the expected BARs */ 470 if (bars != config->bar_mask) { 471 pci_dbg(pdev, "%s: expected BARs %#x not found in device. Found %#x\n", 472 __func__, config->bar_mask, bars); 473 return -EINVAL; 474 } 475 476 ret = pcim_enable_device(pdev); 477 if (ret) 478 return ret; 479 480 ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); 481 if (ret) 482 return ret; 483 dma_set_max_seg_size(&pdev->dev, UINT_MAX); 484 485 qdev->bar_mhi = devm_ioremap_resource(&pdev->dev, &pdev->resource[config->mhi_bar_idx]); 486 if (IS_ERR(qdev->bar_mhi)) 487 return PTR_ERR(qdev->bar_mhi); 488 489 qdev->bar_dbc = devm_ioremap_resource(&pdev->dev, &pdev->resource[config->dbc_bar_idx]); 490 if (IS_ERR(qdev->bar_dbc)) 491 return PTR_ERR(qdev->bar_dbc); 492 493 /* Managed release since we use pcim_enable_device above */ 494 pci_set_master(pdev); 495 496 return 0; 497 } 498 499 static int init_msi(struct qaic_device *qdev, struct pci_dev *pdev) 500 { 501 int irq_count = qdev->num_dbc + 1; 502 int mhi_irq; 503 int ret; 504 int i; 505 506 /* Managed release since we use pcim_enable_device */ 507 ret = pci_alloc_irq_vectors(pdev, irq_count, irq_count, PCI_IRQ_MSI | PCI_IRQ_MSIX); 508 if (ret == -ENOSPC) { 509 ret = pci_alloc_irq_vectors(pdev, 1, 1, PCI_IRQ_MSI | PCI_IRQ_MSIX); 510 if (ret < 0) 511 return ret; 512 513 /* 514 * Operate in one MSI mode. All interrupts will be directed to 515 * MSI0; every interrupt will wake up all the interrupt handlers 516 * (MHI and DBC[0-15]). Since the interrupt is now shared, it is 517 * not disabled during DBC threaded handler, but only one thread 518 * will be allowed to run per DBC, so while it can be 519 * interrupted, it shouldn't race with itself. 520 */ 521 qdev->single_msi = true; 522 pci_info(pdev, "Allocating %d MSIs failed, operating in 1 MSI mode. Performance may be impacted.\n", 523 irq_count); 524 } else if (ret < 0) { 525 return ret; 526 } 527 528 mhi_irq = pci_irq_vector(pdev, 0); 529 if (mhi_irq < 0) 530 return mhi_irq; 531 532 for (i = 0; i < qdev->num_dbc; ++i) { 533 ret = devm_request_threaded_irq(&pdev->dev, 534 pci_irq_vector(pdev, qdev->single_msi ? 0 : i + 1), 535 dbc_irq_handler, dbc_irq_threaded_fn, IRQF_SHARED, 536 "qaic_dbc", &qdev->dbc[i]); 537 if (ret) 538 return ret; 539 540 if (datapath_polling) { 541 qdev->dbc[i].irq = pci_irq_vector(pdev, qdev->single_msi ? 0 : i + 1); 542 if (!qdev->single_msi) 543 disable_irq_nosync(qdev->dbc[i].irq); 544 INIT_WORK(&qdev->dbc[i].poll_work, irq_polling_work); 545 } 546 } 547 548 return mhi_irq; 549 } 550 551 static int qaic_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) 552 { 553 struct qaic_device_config *config = (struct qaic_device_config *)id->driver_data; 554 struct qaic_device *qdev; 555 int mhi_irq; 556 int ret; 557 int i; 558 559 qdev = create_qdev(pdev, config); 560 if (!qdev) 561 return -ENOMEM; 562 563 ret = init_pci(qdev, pdev, config); 564 if (ret) 565 return ret; 566 567 for (i = 0; i < qdev->num_dbc; ++i) 568 qdev->dbc[i].dbc_base = qdev->bar_dbc + QAIC_DBC_OFF(i); 569 570 mhi_irq = init_msi(qdev, pdev); 571 if (mhi_irq < 0) 572 return mhi_irq; 573 574 ret = qaic_create_drm_device(qdev, QAIC_NO_PARTITION); 575 if (ret) 576 return ret; 577 578 qdev->mhi_cntrl = qaic_mhi_register_controller(pdev, qdev->bar_mhi, mhi_irq, 579 qdev->single_msi, config->family); 580 if (IS_ERR(qdev->mhi_cntrl)) { 581 ret = PTR_ERR(qdev->mhi_cntrl); 582 qaic_destroy_drm_device(qdev, QAIC_NO_PARTITION); 583 return ret; 584 } 585 586 return 0; 587 } 588 589 static void qaic_pci_remove(struct pci_dev *pdev) 590 { 591 struct qaic_device *qdev = pci_get_drvdata(pdev); 592 593 if (!qdev) 594 return; 595 596 qaic_dev_reset_clean_local_state(qdev); 597 qaic_mhi_free_controller(qdev->mhi_cntrl, link_up); 598 qaic_destroy_drm_device(qdev, QAIC_NO_PARTITION); 599 } 600 601 static void qaic_pci_shutdown(struct pci_dev *pdev) 602 { 603 /* see qaic_exit for what link_up is doing */ 604 link_up = true; 605 qaic_pci_remove(pdev); 606 } 607 608 static pci_ers_result_t qaic_pci_error_detected(struct pci_dev *pdev, pci_channel_state_t error) 609 { 610 return PCI_ERS_RESULT_NEED_RESET; 611 } 612 613 static void qaic_pci_reset_prepare(struct pci_dev *pdev) 614 { 615 struct qaic_device *qdev = pci_get_drvdata(pdev); 616 617 qaic_notify_reset(qdev); 618 qaic_mhi_start_reset(qdev->mhi_cntrl); 619 qaic_dev_reset_clean_local_state(qdev); 620 } 621 622 static void qaic_pci_reset_done(struct pci_dev *pdev) 623 { 624 struct qaic_device *qdev = pci_get_drvdata(pdev); 625 626 qaic_mhi_reset_done(qdev->mhi_cntrl); 627 } 628 629 static const struct mhi_device_id qaic_mhi_match_table[] = { 630 { .chan = "QAIC_CONTROL", }, 631 {}, 632 }; 633 634 static struct mhi_driver qaic_mhi_driver = { 635 .id_table = qaic_mhi_match_table, 636 .remove = qaic_mhi_remove, 637 .probe = qaic_mhi_probe, 638 .ul_xfer_cb = qaic_mhi_ul_xfer_cb, 639 .dl_xfer_cb = qaic_mhi_dl_xfer_cb, 640 .driver = { 641 .name = "qaic_mhi", 642 }, 643 }; 644 645 static const struct pci_device_id qaic_ids[] = { 646 { PCI_DEVICE_DATA(QCOM, AIC080, (kernel_ulong_t)&aic080_config), }, 647 { PCI_DEVICE_DATA(QCOM, AIC100, (kernel_ulong_t)&aic100_config), }, 648 { PCI_DEVICE_DATA(QCOM, AIC200, (kernel_ulong_t)&aic200_config), }, 649 { } 650 }; 651 MODULE_DEVICE_TABLE(pci, qaic_ids); 652 653 static const struct pci_error_handlers qaic_pci_err_handler = { 654 .error_detected = qaic_pci_error_detected, 655 .reset_prepare = qaic_pci_reset_prepare, 656 .reset_done = qaic_pci_reset_done, 657 }; 658 659 static struct pci_driver qaic_pci_driver = { 660 .name = QAIC_NAME, 661 .id_table = qaic_ids, 662 .probe = qaic_pci_probe, 663 .remove = qaic_pci_remove, 664 .shutdown = qaic_pci_shutdown, 665 .err_handler = &qaic_pci_err_handler, 666 }; 667 668 static int __init qaic_init(void) 669 { 670 int ret; 671 672 ret = pci_register_driver(&qaic_pci_driver); 673 if (ret) { 674 pr_debug("qaic: pci_register_driver failed %d\n", ret); 675 return ret; 676 } 677 678 ret = mhi_driver_register(&qaic_mhi_driver); 679 if (ret) { 680 pr_debug("qaic: mhi_driver_register failed %d\n", ret); 681 goto free_pci; 682 } 683 684 ret = sahara_register(); 685 if (ret) { 686 pr_debug("qaic: sahara_register failed %d\n", ret); 687 goto free_mhi; 688 } 689 690 ret = qaic_timesync_init(); 691 if (ret) 692 pr_debug("qaic: qaic_timesync_init failed %d\n", ret); 693 694 ret = qaic_bootlog_register(); 695 if (ret) 696 pr_debug("qaic: qaic_bootlog_register failed %d\n", ret); 697 698 return 0; 699 700 free_mhi: 701 mhi_driver_unregister(&qaic_mhi_driver); 702 free_pci: 703 pci_unregister_driver(&qaic_pci_driver); 704 return ret; 705 } 706 707 static void __exit qaic_exit(void) 708 { 709 /* 710 * We assume that qaic_pci_remove() is called due to a hotplug event 711 * which would mean that the link is down, and thus 712 * qaic_mhi_free_controller() should not try to access the device during 713 * cleanup. 714 * We call pci_unregister_driver() below, which also triggers 715 * qaic_pci_remove(), but since this is module exit, we expect the link 716 * to the device to be up, in which case qaic_mhi_free_controller() 717 * should try to access the device during cleanup to put the device in 718 * a sane state. 719 * For that reason, we set link_up here to let qaic_mhi_free_controller 720 * know the expected link state. Since the module is going to be 721 * removed at the end of this, we don't need to worry about 722 * reinitializing the link_up state after the cleanup is done. 723 */ 724 link_up = true; 725 qaic_bootlog_unregister(); 726 qaic_timesync_deinit(); 727 sahara_unregister(); 728 mhi_driver_unregister(&qaic_mhi_driver); 729 pci_unregister_driver(&qaic_pci_driver); 730 } 731 732 module_init(qaic_init); 733 module_exit(qaic_exit); 734 735 MODULE_AUTHOR(QAIC_DESC " Kernel Driver Team"); 736 MODULE_DESCRIPTION(QAIC_DESC " Accel Driver"); 737 MODULE_LICENSE("GPL"); 738