1 // SPDX-License-Identifier: GPL-2.0-only 2 3 /* Copyright (c) 2019-2021, The Linux Foundation. All rights reserved. */ 4 /* Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. */ 5 6 #include <linux/delay.h> 7 #include <linux/dma-mapping.h> 8 #include <linux/idr.h> 9 #include <linux/interrupt.h> 10 #include <linux/list.h> 11 #include <linux/kobject.h> 12 #include <linux/kref.h> 13 #include <linux/mhi.h> 14 #include <linux/module.h> 15 #include <linux/msi.h> 16 #include <linux/mutex.h> 17 #include <linux/pci.h> 18 #include <linux/spinlock.h> 19 #include <linux/workqueue.h> 20 #include <linux/wait.h> 21 #include <drm/drm_accel.h> 22 #include <drm/drm_drv.h> 23 #include <drm/drm_file.h> 24 #include <drm/drm_gem.h> 25 #include <drm/drm_ioctl.h> 26 #include <drm/drm_managed.h> 27 #include <uapi/drm/qaic_accel.h> 28 29 #include "mhi_controller.h" 30 #include "qaic.h" 31 #include "qaic_debugfs.h" 32 #include "qaic_ras.h" 33 #include "qaic_ssr.h" 34 #include "qaic_timesync.h" 35 #include "sahara.h" 36 37 MODULE_IMPORT_NS("DMA_BUF"); 38 39 #define PCI_DEVICE_ID_QCOM_AIC080 0xa080 40 #define PCI_DEVICE_ID_QCOM_AIC100 0xa100 41 #define PCI_DEVICE_ID_QCOM_AIC200 0xa110 42 #define QAIC_NAME "qaic" 43 #define QAIC_DESC "Qualcomm Cloud AI Accelerators" 44 #define CNTL_MAJOR 5 45 #define CNTL_MINOR 0 46 #define DBC_NUM 16 47 48 struct qaic_device_config { 49 /* Indicates the AIC family the device belongs to */ 50 int family; 51 /* A bitmask representing the available BARs */ 52 int bar_mask; 53 /* An index value used to identify the MHI controller BAR */ 54 unsigned int mhi_bar_idx; 55 /* An index value used to identify the DBCs BAR */ 56 unsigned int dbc_bar_idx; 57 }; 58 59 static const struct qaic_device_config aic080_config = { 60 .family = FAMILY_AIC100, 61 .bar_mask = BIT(0) | BIT(2) | BIT(4), 62 .mhi_bar_idx = 0, 63 .dbc_bar_idx = 2, 64 }; 65 66 static const struct qaic_device_config aic100_config = { 67 .family = FAMILY_AIC100, 68 .bar_mask = BIT(0) | BIT(2) | BIT(4), 69 .mhi_bar_idx = 0, 70 .dbc_bar_idx = 2, 71 }; 72 73 static const struct qaic_device_config aic200_config = { 74 .family = FAMILY_AIC200, 75 .bar_mask = BIT(0) | BIT(1) | BIT(2) | BIT(4), 76 .mhi_bar_idx = 1, 77 .dbc_bar_idx = 2, 78 }; 79 80 bool datapath_polling; 81 module_param(datapath_polling, bool, 0400); 82 MODULE_PARM_DESC(datapath_polling, "Operate the datapath in polling mode"); 83 static bool link_up; 84 static DEFINE_IDA(qaic_usrs); 85 86 static void qaicm_wq_release(struct drm_device *dev, void *res) 87 { 88 struct workqueue_struct *wq = res; 89 90 destroy_workqueue(wq); 91 } 92 93 static struct workqueue_struct *qaicm_wq_init(struct drm_device *dev, const char *name) 94 { 95 struct workqueue_struct *wq; 96 int ret; 97 98 wq = alloc_workqueue("%s", WQ_UNBOUND, 0, name); 99 if (!wq) 100 return ERR_PTR(-ENOMEM); 101 ret = drmm_add_action_or_reset(dev, qaicm_wq_release, wq); 102 if (ret) 103 return ERR_PTR(ret); 104 105 return wq; 106 } 107 108 static void qaicm_srcu_release(struct drm_device *dev, void *res) 109 { 110 struct srcu_struct *lock = res; 111 112 cleanup_srcu_struct(lock); 113 } 114 115 static int qaicm_srcu_init(struct drm_device *dev, struct srcu_struct *lock) 116 { 117 int ret; 118 119 ret = init_srcu_struct(lock); 120 if (ret) 121 return ret; 122 123 return drmm_add_action_or_reset(dev, qaicm_srcu_release, lock); 124 } 125 126 static void qaicm_pci_release(struct drm_device *dev, void *res) 127 { 128 struct qaic_device *qdev = to_qaic_device(dev); 129 130 pci_set_drvdata(qdev->pdev, NULL); 131 } 132 133 static void free_usr(struct kref *kref) 134 { 135 struct qaic_user *usr = container_of(kref, struct qaic_user, ref_count); 136 137 cleanup_srcu_struct(&usr->qddev_lock); 138 ida_free(&qaic_usrs, usr->handle); 139 kfree(usr); 140 } 141 142 static int qaic_open(struct drm_device *dev, struct drm_file *file) 143 { 144 struct qaic_drm_device *qddev = to_qaic_drm_device(dev); 145 struct qaic_device *qdev = qddev->qdev; 146 struct qaic_user *usr; 147 int rcu_id; 148 int ret; 149 150 rcu_id = srcu_read_lock(&qdev->dev_lock); 151 if (qdev->dev_state != QAIC_ONLINE) { 152 ret = -ENODEV; 153 goto dev_unlock; 154 } 155 156 usr = kmalloc_obj(*usr); 157 if (!usr) { 158 ret = -ENOMEM; 159 goto dev_unlock; 160 } 161 162 usr->handle = ida_alloc(&qaic_usrs, GFP_KERNEL); 163 if (usr->handle < 0) { 164 ret = usr->handle; 165 goto free_usr; 166 } 167 usr->qddev = qddev; 168 atomic_set(&usr->chunk_id, 0); 169 init_srcu_struct(&usr->qddev_lock); 170 kref_init(&usr->ref_count); 171 172 ret = mutex_lock_interruptible(&qddev->users_mutex); 173 if (ret) 174 goto cleanup_usr; 175 176 list_add(&usr->node, &qddev->users); 177 mutex_unlock(&qddev->users_mutex); 178 179 file->driver_priv = usr; 180 181 srcu_read_unlock(&qdev->dev_lock, rcu_id); 182 return 0; 183 184 cleanup_usr: 185 cleanup_srcu_struct(&usr->qddev_lock); 186 ida_free(&qaic_usrs, usr->handle); 187 free_usr: 188 kfree(usr); 189 dev_unlock: 190 srcu_read_unlock(&qdev->dev_lock, rcu_id); 191 return ret; 192 } 193 194 static void qaic_postclose(struct drm_device *dev, struct drm_file *file) 195 { 196 struct qaic_user *usr = file->driver_priv; 197 struct qaic_drm_device *qddev; 198 struct qaic_device *qdev; 199 int qdev_rcu_id; 200 int usr_rcu_id; 201 int i; 202 203 qddev = usr->qddev; 204 usr_rcu_id = srcu_read_lock(&usr->qddev_lock); 205 if (qddev) { 206 qdev = qddev->qdev; 207 qdev_rcu_id = srcu_read_lock(&qdev->dev_lock); 208 if (qdev->dev_state == QAIC_ONLINE) { 209 qaic_release_usr(qdev, usr); 210 for (i = 0; i < qdev->num_dbc; ++i) 211 if (qdev->dbc[i].usr && qdev->dbc[i].usr->handle == usr->handle) 212 release_dbc(qdev, i); 213 } 214 srcu_read_unlock(&qdev->dev_lock, qdev_rcu_id); 215 216 mutex_lock(&qddev->users_mutex); 217 if (!list_empty(&usr->node)) 218 list_del_init(&usr->node); 219 mutex_unlock(&qddev->users_mutex); 220 } 221 222 srcu_read_unlock(&usr->qddev_lock, usr_rcu_id); 223 kref_put(&usr->ref_count, free_usr); 224 225 file->driver_priv = NULL; 226 } 227 228 DEFINE_DRM_ACCEL_FOPS(qaic_accel_fops); 229 230 static const struct drm_ioctl_desc qaic_drm_ioctls[] = { 231 DRM_IOCTL_DEF_DRV(QAIC_MANAGE, qaic_manage_ioctl, 0), 232 DRM_IOCTL_DEF_DRV(QAIC_CREATE_BO, qaic_create_bo_ioctl, 0), 233 DRM_IOCTL_DEF_DRV(QAIC_MMAP_BO, qaic_mmap_bo_ioctl, 0), 234 DRM_IOCTL_DEF_DRV(QAIC_ATTACH_SLICE_BO, qaic_attach_slice_bo_ioctl, 0), 235 DRM_IOCTL_DEF_DRV(QAIC_EXECUTE_BO, qaic_execute_bo_ioctl, 0), 236 DRM_IOCTL_DEF_DRV(QAIC_PARTIAL_EXECUTE_BO, qaic_partial_execute_bo_ioctl, 0), 237 DRM_IOCTL_DEF_DRV(QAIC_WAIT_BO, qaic_wait_bo_ioctl, 0), 238 DRM_IOCTL_DEF_DRV(QAIC_PERF_STATS_BO, qaic_perf_stats_bo_ioctl, 0), 239 DRM_IOCTL_DEF_DRV(QAIC_DETACH_SLICE_BO, qaic_detach_slice_bo_ioctl, 0), 240 }; 241 242 static const struct drm_driver qaic_accel_driver = { 243 .driver_features = DRIVER_GEM | DRIVER_COMPUTE_ACCEL, 244 245 .name = QAIC_NAME, 246 .desc = QAIC_DESC, 247 248 .fops = &qaic_accel_fops, 249 .open = qaic_open, 250 .postclose = qaic_postclose, 251 252 .ioctls = qaic_drm_ioctls, 253 .num_ioctls = ARRAY_SIZE(qaic_drm_ioctls), 254 .gem_prime_import = qaic_gem_prime_import, 255 }; 256 257 static int qaic_create_drm_device(struct qaic_device *qdev, s32 partition_id) 258 { 259 struct qaic_drm_device *qddev = qdev->qddev; 260 struct drm_device *drm = to_drm(qddev); 261 int ret; 262 263 /* Hold off implementing partitions until the uapi is determined */ 264 if (partition_id != QAIC_NO_PARTITION) 265 return -EINVAL; 266 267 qddev->partition_id = partition_id; 268 269 ret = drm_dev_register(drm, 0); 270 if (ret) { 271 pci_dbg(qdev->pdev, "drm_dev_register failed %d\n", ret); 272 return ret; 273 } 274 275 ret = qaic_sysfs_init(qddev); 276 if (ret) { 277 drm_dev_unregister(drm); 278 pci_dbg(qdev->pdev, "qaic_sysfs_init failed %d\n", ret); 279 return ret; 280 } 281 282 qaic_debugfs_init(qddev); 283 284 return ret; 285 } 286 287 static void qaic_destroy_drm_device(struct qaic_device *qdev, s32 partition_id) 288 { 289 struct qaic_drm_device *qddev = qdev->qddev; 290 struct drm_device *drm = to_drm(qddev); 291 struct qaic_user *usr; 292 293 qaic_sysfs_remove(qddev); 294 drm_dev_unregister(drm); 295 qddev->partition_id = 0; 296 /* 297 * Existing users get unresolvable errors till they close FDs. 298 * Need to sync carefully with users calling close(). The 299 * list of users can be modified elsewhere when the lock isn't 300 * held here, but the sync'ing the srcu with the mutex held 301 * could deadlock. Grab the mutex so that the list will be 302 * unmodified. The user we get will exist as long as the 303 * lock is held. Signal that the qcdev is going away, and 304 * grab a reference to the user so they don't go away for 305 * synchronize_srcu(). Then release the mutex to avoid 306 * deadlock and make sure the user has observed the signal. 307 * With the lock released, we cannot maintain any state of the 308 * user list. 309 */ 310 mutex_lock(&qddev->users_mutex); 311 while (!list_empty(&qddev->users)) { 312 usr = list_first_entry(&qddev->users, struct qaic_user, node); 313 list_del_init(&usr->node); 314 kref_get(&usr->ref_count); 315 usr->qddev = NULL; 316 mutex_unlock(&qddev->users_mutex); 317 synchronize_srcu(&usr->qddev_lock); 318 kref_put(&usr->ref_count, free_usr); 319 mutex_lock(&qddev->users_mutex); 320 } 321 mutex_unlock(&qddev->users_mutex); 322 } 323 324 static int qaic_mhi_probe(struct mhi_device *mhi_dev, const struct mhi_device_id *id) 325 { 326 u16 major = -1, minor = -1; 327 struct qaic_device *qdev; 328 int ret; 329 330 /* 331 * Invoking this function indicates that the control channel to the 332 * device is available. We use that as a signal to indicate that 333 * the device side firmware has booted. The device side firmware 334 * manages the device resources, so we need to communicate with it 335 * via the control channel in order to utilize the device. Therefore 336 * we wait until this signal to create the drm dev that userspace will 337 * use to control the device, because without the device side firmware, 338 * userspace can't do anything useful. 339 */ 340 341 qdev = pci_get_drvdata(to_pci_dev(mhi_dev->mhi_cntrl->cntrl_dev)); 342 343 dev_set_drvdata(&mhi_dev->dev, qdev); 344 qdev->cntl_ch = mhi_dev; 345 346 ret = qaic_control_open(qdev); 347 if (ret) { 348 pci_dbg(qdev->pdev, "%s: control_open failed %d\n", __func__, ret); 349 return ret; 350 } 351 352 qdev->dev_state = QAIC_BOOT; 353 ret = get_cntl_version(qdev, NULL, &major, &minor); 354 if (ret || major != CNTL_MAJOR || minor > CNTL_MINOR) { 355 pci_err(qdev->pdev, "%s: Control protocol version (%d.%d) not supported. Supported version is (%d.%d). Ret: %d\n", 356 __func__, major, minor, CNTL_MAJOR, CNTL_MINOR, ret); 357 ret = -EINVAL; 358 goto close_control; 359 } 360 qdev->dev_state = QAIC_ONLINE; 361 kobject_uevent(&(to_accel_kdev(qdev->qddev))->kobj, KOBJ_ONLINE); 362 363 return ret; 364 365 close_control: 366 qaic_control_close(qdev); 367 return ret; 368 } 369 370 static void qaic_mhi_remove(struct mhi_device *mhi_dev) 371 { 372 /* This is redundant since we have already observed the device crash */ 373 } 374 375 static void qaic_notify_reset(struct qaic_device *qdev) 376 { 377 int i; 378 379 kobject_uevent(&(to_accel_kdev(qdev->qddev))->kobj, KOBJ_OFFLINE); 380 qdev->dev_state = QAIC_OFFLINE; 381 /* wake up any waiters to avoid waiting for timeouts at sync */ 382 wake_all_cntl(qdev); 383 for (i = 0; i < qdev->num_dbc; ++i) 384 wakeup_dbc(qdev, i); 385 synchronize_srcu(&qdev->dev_lock); 386 } 387 388 void qaic_dev_reset_clean_local_state(struct qaic_device *qdev) 389 { 390 int i; 391 392 qaic_notify_reset(qdev); 393 394 /* start tearing things down */ 395 qaic_clean_up_ssr(qdev); 396 for (i = 0; i < qdev->num_dbc; ++i) 397 release_dbc(qdev, i); 398 } 399 400 static struct qaic_device *create_qdev(struct pci_dev *pdev, 401 const struct qaic_device_config *config) 402 { 403 struct device *dev = &pdev->dev; 404 struct qaic_drm_device *qddev; 405 struct qaic_device *qdev; 406 struct drm_device *drm; 407 int i, ret; 408 409 qdev = devm_kzalloc(dev, struct_size(qdev, dbc, DBC_NUM), GFP_KERNEL); 410 if (!qdev) 411 return NULL; 412 413 qdev->num_dbc = DBC_NUM; 414 qdev->dev_state = QAIC_OFFLINE; 415 416 qddev = devm_drm_dev_alloc(&pdev->dev, &qaic_accel_driver, struct qaic_drm_device, drm); 417 if (IS_ERR(qddev)) 418 return NULL; 419 420 drm = to_drm(qddev); 421 pci_set_drvdata(pdev, qdev); 422 423 ret = drmm_mutex_init(drm, &qddev->users_mutex); 424 if (ret) 425 return NULL; 426 ret = drmm_add_action_or_reset(drm, qaicm_pci_release, NULL); 427 if (ret) 428 return NULL; 429 ret = drmm_mutex_init(drm, &qdev->cntl_mutex); 430 if (ret) 431 return NULL; 432 ret = drmm_mutex_init(drm, &qdev->bootlog_mutex); 433 if (ret) 434 return NULL; 435 436 qdev->cntl_wq = qaicm_wq_init(drm, "qaic_cntl"); 437 if (IS_ERR(qdev->cntl_wq)) 438 return NULL; 439 qdev->qts_wq = qaicm_wq_init(drm, "qaic_ts"); 440 if (IS_ERR(qdev->qts_wq)) 441 return NULL; 442 qdev->ssr_wq = qaicm_wq_init(drm, "qaic_ssr"); 443 if (IS_ERR(qdev->ssr_wq)) 444 return NULL; 445 446 ret = qaicm_srcu_init(drm, &qdev->dev_lock); 447 if (ret) 448 return NULL; 449 450 ret = qaic_ssr_init(qdev, drm); 451 if (ret) 452 pci_info(pdev, "QAIC SSR crashdump collection not supported.\n"); 453 454 qdev->qddev = qddev; 455 qdev->pdev = pdev; 456 qddev->qdev = qdev; 457 458 INIT_LIST_HEAD(&qdev->cntl_xfer_list); 459 INIT_LIST_HEAD(&qdev->bootlog); 460 INIT_LIST_HEAD(&qddev->users); 461 462 for (i = 0; i < qdev->num_dbc; ++i) { 463 spin_lock_init(&qdev->dbc[i].xfer_lock); 464 qdev->dbc[i].qdev = qdev; 465 qdev->dbc[i].id = i; 466 INIT_LIST_HEAD(&qdev->dbc[i].xfer_list); 467 ret = qaicm_srcu_init(drm, &qdev->dbc[i].ch_lock); 468 if (ret) 469 return NULL; 470 init_waitqueue_head(&qdev->dbc[i].dbc_release); 471 INIT_LIST_HEAD(&qdev->dbc[i].bo_lists); 472 ret = drmm_mutex_init(drm, &qdev->dbc[i].req_lock); 473 if (ret) 474 return NULL; 475 } 476 477 return qdev; 478 } 479 480 static int init_pci(struct qaic_device *qdev, struct pci_dev *pdev, 481 const struct qaic_device_config *config) 482 { 483 int bars; 484 int ret; 485 486 bars = pci_select_bars(pdev, IORESOURCE_MEM) & 0x3f; 487 488 /* make sure the device has the expected BARs */ 489 if (bars != config->bar_mask) { 490 pci_dbg(pdev, "%s: expected BARs %#x not found in device. Found %#x\n", 491 __func__, config->bar_mask, bars); 492 return -EINVAL; 493 } 494 495 ret = pcim_enable_device(pdev); 496 if (ret) 497 return ret; 498 499 ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); 500 if (ret) 501 return ret; 502 dma_set_max_seg_size(&pdev->dev, UINT_MAX); 503 504 qdev->bar_mhi = devm_ioremap_resource(&pdev->dev, &pdev->resource[config->mhi_bar_idx]); 505 if (IS_ERR(qdev->bar_mhi)) 506 return PTR_ERR(qdev->bar_mhi); 507 508 qdev->bar_dbc = devm_ioremap_resource(&pdev->dev, &pdev->resource[config->dbc_bar_idx]); 509 if (IS_ERR(qdev->bar_dbc)) 510 return PTR_ERR(qdev->bar_dbc); 511 512 /* Managed release since we use pcim_enable_device above */ 513 pci_set_master(pdev); 514 515 return 0; 516 } 517 518 static int init_msi(struct qaic_device *qdev, struct pci_dev *pdev) 519 { 520 int irq_count = qdev->num_dbc + 1; 521 int mhi_irq; 522 int ret; 523 int i; 524 525 /* Managed release since we use pcim_enable_device */ 526 ret = pci_alloc_irq_vectors(pdev, irq_count, irq_count, PCI_IRQ_MSI | PCI_IRQ_MSIX); 527 if (ret == -ENOSPC) { 528 ret = pci_alloc_irq_vectors(pdev, 1, 1, PCI_IRQ_MSI | PCI_IRQ_MSIX); 529 if (ret < 0) 530 return ret; 531 532 /* 533 * Operate in one MSI mode. All interrupts will be directed to 534 * MSI0; every interrupt will wake up all the interrupt handlers 535 * (MHI and DBC[0-15]). Since the interrupt is now shared, it is 536 * not disabled during DBC threaded handler, but only one thread 537 * will be allowed to run per DBC, so while it can be 538 * interrupted, it shouldn't race with itself. 539 */ 540 qdev->single_msi = true; 541 pci_info(pdev, "Allocating %d MSIs failed, operating in 1 MSI mode. Performance may be impacted.\n", 542 irq_count); 543 } else if (ret < 0) { 544 return ret; 545 } 546 547 mhi_irq = pci_irq_vector(pdev, 0); 548 if (mhi_irq < 0) 549 return mhi_irq; 550 551 for (i = 0; i < qdev->num_dbc; ++i) { 552 ret = devm_request_threaded_irq(&pdev->dev, 553 pci_irq_vector(pdev, qdev->single_msi ? 0 : i + 1), 554 dbc_irq_handler, dbc_irq_threaded_fn, IRQF_SHARED, 555 "qaic_dbc", &qdev->dbc[i]); 556 if (ret) 557 return ret; 558 559 if (datapath_polling) { 560 qdev->dbc[i].irq = pci_irq_vector(pdev, qdev->single_msi ? 0 : i + 1); 561 if (!qdev->single_msi) 562 disable_irq_nosync(qdev->dbc[i].irq); 563 INIT_WORK(&qdev->dbc[i].poll_work, qaic_irq_polling_work); 564 } 565 } 566 567 return mhi_irq; 568 } 569 570 static int qaic_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) 571 { 572 struct qaic_device_config *config = (struct qaic_device_config *)id->driver_data; 573 struct qaic_device *qdev; 574 int mhi_irq; 575 int ret; 576 int i; 577 578 qdev = create_qdev(pdev, config); 579 if (!qdev) 580 return -ENOMEM; 581 582 ret = init_pci(qdev, pdev, config); 583 if (ret) 584 return ret; 585 586 for (i = 0; i < qdev->num_dbc; ++i) 587 qdev->dbc[i].dbc_base = qdev->bar_dbc + QAIC_DBC_OFF(i); 588 589 mhi_irq = init_msi(qdev, pdev); 590 if (mhi_irq < 0) 591 return mhi_irq; 592 593 ret = qaic_create_drm_device(qdev, QAIC_NO_PARTITION); 594 if (ret) 595 return ret; 596 597 qdev->mhi_cntrl = qaic_mhi_register_controller(pdev, qdev->bar_mhi, mhi_irq, 598 qdev->single_msi, config->family); 599 if (IS_ERR(qdev->mhi_cntrl)) { 600 ret = PTR_ERR(qdev->mhi_cntrl); 601 qaic_destroy_drm_device(qdev, QAIC_NO_PARTITION); 602 return ret; 603 } 604 605 return 0; 606 } 607 608 static void qaic_pci_remove(struct pci_dev *pdev) 609 { 610 struct qaic_device *qdev = pci_get_drvdata(pdev); 611 612 if (!qdev) 613 return; 614 615 qaic_dev_reset_clean_local_state(qdev); 616 qaic_mhi_free_controller(qdev->mhi_cntrl, link_up); 617 qaic_destroy_drm_device(qdev, QAIC_NO_PARTITION); 618 } 619 620 static void qaic_pci_shutdown(struct pci_dev *pdev) 621 { 622 /* see qaic_exit for what link_up is doing */ 623 link_up = true; 624 qaic_pci_remove(pdev); 625 } 626 627 static pci_ers_result_t qaic_pci_error_detected(struct pci_dev *pdev, pci_channel_state_t error) 628 { 629 return PCI_ERS_RESULT_NEED_RESET; 630 } 631 632 static void qaic_pci_reset_prepare(struct pci_dev *pdev) 633 { 634 struct qaic_device *qdev = pci_get_drvdata(pdev); 635 636 qaic_notify_reset(qdev); 637 qaic_mhi_start_reset(qdev->mhi_cntrl); 638 qaic_dev_reset_clean_local_state(qdev); 639 } 640 641 static void qaic_pci_reset_done(struct pci_dev *pdev) 642 { 643 struct qaic_device *qdev = pci_get_drvdata(pdev); 644 645 qaic_mhi_reset_done(qdev->mhi_cntrl); 646 } 647 648 static const struct mhi_device_id qaic_mhi_match_table[] = { 649 { .chan = "QAIC_CONTROL", }, 650 {}, 651 }; 652 653 static struct mhi_driver qaic_mhi_driver = { 654 .id_table = qaic_mhi_match_table, 655 .remove = qaic_mhi_remove, 656 .probe = qaic_mhi_probe, 657 .ul_xfer_cb = qaic_mhi_ul_xfer_cb, 658 .dl_xfer_cb = qaic_mhi_dl_xfer_cb, 659 .driver = { 660 .name = "qaic_mhi", 661 }, 662 }; 663 664 static const struct pci_device_id qaic_ids[] = { 665 { PCI_DEVICE_DATA(QCOM, AIC080, (kernel_ulong_t)&aic080_config), }, 666 { PCI_DEVICE_DATA(QCOM, AIC100, (kernel_ulong_t)&aic100_config), }, 667 { PCI_DEVICE_DATA(QCOM, AIC200, (kernel_ulong_t)&aic200_config), }, 668 { } 669 }; 670 MODULE_DEVICE_TABLE(pci, qaic_ids); 671 672 static const struct pci_error_handlers qaic_pci_err_handler = { 673 .error_detected = qaic_pci_error_detected, 674 .reset_prepare = qaic_pci_reset_prepare, 675 .reset_done = qaic_pci_reset_done, 676 }; 677 678 static bool qaic_is_under_reset(struct qaic_device *qdev) 679 { 680 int rcu_id; 681 bool ret; 682 683 rcu_id = srcu_read_lock(&qdev->dev_lock); 684 ret = qdev->dev_state != QAIC_ONLINE; 685 srcu_read_unlock(&qdev->dev_lock, rcu_id); 686 return ret; 687 } 688 689 static bool qaic_data_path_busy(struct qaic_device *qdev) 690 { 691 bool ret = false; 692 int dev_rcu_id; 693 int i; 694 695 dev_rcu_id = srcu_read_lock(&qdev->dev_lock); 696 if (qdev->dev_state != QAIC_ONLINE) { 697 srcu_read_unlock(&qdev->dev_lock, dev_rcu_id); 698 return false; 699 } 700 for (i = 0; i < qdev->num_dbc; i++) { 701 struct dma_bridge_chan *dbc = &qdev->dbc[i]; 702 unsigned long flags; 703 int ch_rcu_id; 704 705 ch_rcu_id = srcu_read_lock(&dbc->ch_lock); 706 if (!dbc->usr || !dbc->in_use) { 707 srcu_read_unlock(&dbc->ch_lock, ch_rcu_id); 708 continue; 709 } 710 spin_lock_irqsave(&dbc->xfer_lock, flags); 711 ret = !list_empty(&dbc->xfer_list); 712 spin_unlock_irqrestore(&dbc->xfer_lock, flags); 713 srcu_read_unlock(&dbc->ch_lock, ch_rcu_id); 714 if (ret) 715 break; 716 } 717 srcu_read_unlock(&qdev->dev_lock, dev_rcu_id); 718 return ret; 719 } 720 721 static int qaic_pm_suspend(struct device *dev) 722 { 723 struct qaic_device *qdev = pci_get_drvdata(to_pci_dev(dev)); 724 725 dev_dbg(dev, "Suspending..\n"); 726 if (qaic_data_path_busy(qdev)) { 727 dev_dbg(dev, "Device's datapath is busy. Aborting suspend..\n"); 728 return -EBUSY; 729 } 730 if (qaic_is_under_reset(qdev)) { 731 dev_dbg(dev, "Device is under reset. Aborting suspend..\n"); 732 return -EBUSY; 733 } 734 qaic_mqts_ch_stop_timer(qdev->mqts_ch); 735 qaic_pci_reset_prepare(qdev->pdev); 736 pci_save_state(qdev->pdev); 737 pci_disable_device(qdev->pdev); 738 pci_set_power_state(qdev->pdev, PCI_D3hot); 739 return 0; 740 } 741 742 static int qaic_pm_resume(struct device *dev) 743 { 744 struct qaic_device *qdev = pci_get_drvdata(to_pci_dev(dev)); 745 int ret; 746 747 dev_dbg(dev, "Resuming..\n"); 748 pci_set_power_state(qdev->pdev, PCI_D0); 749 pci_restore_state(qdev->pdev); 750 ret = pci_enable_device(qdev->pdev); 751 if (ret) { 752 dev_err(dev, "pci_enable_device failed on resume %d\n", ret); 753 return ret; 754 } 755 pci_set_master(qdev->pdev); 756 qaic_pci_reset_done(qdev->pdev); 757 return 0; 758 } 759 760 static const struct dev_pm_ops qaic_pm_ops = { 761 SYSTEM_SLEEP_PM_OPS(qaic_pm_suspend, qaic_pm_resume) 762 }; 763 764 static struct pci_driver qaic_pci_driver = { 765 .name = QAIC_NAME, 766 .id_table = qaic_ids, 767 .probe = qaic_pci_probe, 768 .remove = qaic_pci_remove, 769 .shutdown = qaic_pci_shutdown, 770 .err_handler = &qaic_pci_err_handler, 771 .driver = { 772 .pm = pm_sleep_ptr(&qaic_pm_ops), 773 }, 774 }; 775 776 static int __init qaic_init(void) 777 { 778 int ret; 779 780 ret = pci_register_driver(&qaic_pci_driver); 781 if (ret) { 782 pr_debug("qaic: pci_register_driver failed %d\n", ret); 783 return ret; 784 } 785 786 ret = mhi_driver_register(&qaic_mhi_driver); 787 if (ret) { 788 pr_debug("qaic: mhi_driver_register failed %d\n", ret); 789 goto free_pci; 790 } 791 792 ret = sahara_register(); 793 if (ret) { 794 pr_debug("qaic: sahara_register failed %d\n", ret); 795 goto free_mhi; 796 } 797 798 ret = qaic_timesync_init(); 799 if (ret) 800 pr_debug("qaic: qaic_timesync_init failed %d\n", ret); 801 802 ret = qaic_bootlog_register(); 803 if (ret) 804 pr_debug("qaic: qaic_bootlog_register failed %d\n", ret); 805 806 ret = qaic_ras_register(); 807 if (ret) 808 pr_debug("qaic: qaic_ras_register failed %d\n", ret); 809 ret = qaic_ssr_register(); 810 if (ret) { 811 pr_debug("qaic: qaic_ssr_register failed %d\n", ret); 812 goto free_bootlog; 813 } 814 815 return 0; 816 817 free_bootlog: 818 qaic_bootlog_unregister(); 819 free_mhi: 820 mhi_driver_unregister(&qaic_mhi_driver); 821 free_pci: 822 pci_unregister_driver(&qaic_pci_driver); 823 return ret; 824 } 825 826 static void __exit qaic_exit(void) 827 { 828 /* 829 * We assume that qaic_pci_remove() is called due to a hotplug event 830 * which would mean that the link is down, and thus 831 * qaic_mhi_free_controller() should not try to access the device during 832 * cleanup. 833 * We call pci_unregister_driver() below, which also triggers 834 * qaic_pci_remove(), but since this is module exit, we expect the link 835 * to the device to be up, in which case qaic_mhi_free_controller() 836 * should try to access the device during cleanup to put the device in 837 * a sane state. 838 * For that reason, we set link_up here to let qaic_mhi_free_controller 839 * know the expected link state. Since the module is going to be 840 * removed at the end of this, we don't need to worry about 841 * reinitializing the link_up state after the cleanup is done. 842 */ 843 link_up = true; 844 qaic_ssr_unregister(); 845 qaic_ras_unregister(); 846 qaic_bootlog_unregister(); 847 qaic_timesync_deinit(); 848 sahara_unregister(); 849 mhi_driver_unregister(&qaic_mhi_driver); 850 pci_unregister_driver(&qaic_pci_driver); 851 } 852 853 module_init(qaic_init); 854 module_exit(qaic_exit); 855 856 MODULE_AUTHOR(QAIC_DESC " Kernel Driver Team"); 857 MODULE_DESCRIPTION(QAIC_DESC " Accel Driver"); 858 MODULE_LICENSE("GPL"); 859