1 // SPDX-License-Identifier: GPL-2.0-only
2
3 /* Copyright (c) 2019-2021, The Linux Foundation. All rights reserved. */
4 /* Copyright (c) 2021-2023 Qualcomm Innovation Center, Inc. All rights reserved. */
5
6 #include <linux/delay.h>
7 #include <linux/dma-mapping.h>
8 #include <linux/idr.h>
9 #include <linux/interrupt.h>
10 #include <linux/list.h>
11 #include <linux/kobject.h>
12 #include <linux/kref.h>
13 #include <linux/mhi.h>
14 #include <linux/module.h>
15 #include <linux/msi.h>
16 #include <linux/mutex.h>
17 #include <linux/pci.h>
18 #include <linux/spinlock.h>
19 #include <linux/workqueue.h>
20 #include <linux/wait.h>
21 #include <drm/drm_accel.h>
22 #include <drm/drm_drv.h>
23 #include <drm/drm_file.h>
24 #include <drm/drm_gem.h>
25 #include <drm/drm_ioctl.h>
26 #include <drm/drm_managed.h>
27 #include <uapi/drm/qaic_accel.h>
28
29 #include "mhi_controller.h"
30 #include "qaic.h"
31 #include "qaic_debugfs.h"
32 #include "qaic_ras.h"
33 #include "qaic_timesync.h"
34 #include "sahara.h"
35
36 MODULE_IMPORT_NS("DMA_BUF");
37
38 #define PCI_DEVICE_ID_QCOM_AIC080 0xa080
39 #define PCI_DEVICE_ID_QCOM_AIC100 0xa100
40 #define PCI_DEVICE_ID_QCOM_AIC200 0xa110
41 #define QAIC_NAME "qaic"
42 #define QAIC_DESC "Qualcomm Cloud AI Accelerators"
43 #define CNTL_MAJOR 5
44 #define CNTL_MINOR 0
45
46 struct qaic_device_config {
47 /* Indicates the AIC family the device belongs to */
48 int family;
49 /* A bitmask representing the available BARs */
50 int bar_mask;
51 /* An index value used to identify the MHI controller BAR */
52 unsigned int mhi_bar_idx;
53 /* An index value used to identify the DBCs BAR */
54 unsigned int dbc_bar_idx;
55 };
56
57 static const struct qaic_device_config aic080_config = {
58 .family = FAMILY_AIC100,
59 .bar_mask = BIT(0) | BIT(2) | BIT(4),
60 .mhi_bar_idx = 0,
61 .dbc_bar_idx = 2,
62 };
63
64 static const struct qaic_device_config aic100_config = {
65 .family = FAMILY_AIC100,
66 .bar_mask = BIT(0) | BIT(2) | BIT(4),
67 .mhi_bar_idx = 0,
68 .dbc_bar_idx = 2,
69 };
70
71 static const struct qaic_device_config aic200_config = {
72 .family = FAMILY_AIC200,
73 .bar_mask = BIT(0) | BIT(1) | BIT(2) | BIT(4),
74 .mhi_bar_idx = 1,
75 .dbc_bar_idx = 2,
76 };
77
78 bool datapath_polling;
79 module_param(datapath_polling, bool, 0400);
80 MODULE_PARM_DESC(datapath_polling, "Operate the datapath in polling mode");
81 static bool link_up;
82 static DEFINE_IDA(qaic_usrs);
83
qaicm_wq_release(struct drm_device * dev,void * res)84 static void qaicm_wq_release(struct drm_device *dev, void *res)
85 {
86 struct workqueue_struct *wq = res;
87
88 destroy_workqueue(wq);
89 }
90
qaicm_wq_init(struct drm_device * dev,const char * name)91 static struct workqueue_struct *qaicm_wq_init(struct drm_device *dev, const char *name)
92 {
93 struct workqueue_struct *wq;
94 int ret;
95
96 wq = alloc_workqueue("%s", WQ_UNBOUND, 0, name);
97 if (!wq)
98 return ERR_PTR(-ENOMEM);
99 ret = drmm_add_action_or_reset(dev, qaicm_wq_release, wq);
100 if (ret)
101 return ERR_PTR(ret);
102
103 return wq;
104 }
105
qaicm_srcu_release(struct drm_device * dev,void * res)106 static void qaicm_srcu_release(struct drm_device *dev, void *res)
107 {
108 struct srcu_struct *lock = res;
109
110 cleanup_srcu_struct(lock);
111 }
112
qaicm_srcu_init(struct drm_device * dev,struct srcu_struct * lock)113 static int qaicm_srcu_init(struct drm_device *dev, struct srcu_struct *lock)
114 {
115 int ret;
116
117 ret = init_srcu_struct(lock);
118 if (ret)
119 return ret;
120
121 return drmm_add_action_or_reset(dev, qaicm_srcu_release, lock);
122 }
123
qaicm_pci_release(struct drm_device * dev,void * res)124 static void qaicm_pci_release(struct drm_device *dev, void *res)
125 {
126 struct qaic_device *qdev = to_qaic_device(dev);
127
128 pci_set_drvdata(qdev->pdev, NULL);
129 }
130
free_usr(struct kref * kref)131 static void free_usr(struct kref *kref)
132 {
133 struct qaic_user *usr = container_of(kref, struct qaic_user, ref_count);
134
135 cleanup_srcu_struct(&usr->qddev_lock);
136 ida_free(&qaic_usrs, usr->handle);
137 kfree(usr);
138 }
139
qaic_open(struct drm_device * dev,struct drm_file * file)140 static int qaic_open(struct drm_device *dev, struct drm_file *file)
141 {
142 struct qaic_drm_device *qddev = to_qaic_drm_device(dev);
143 struct qaic_device *qdev = qddev->qdev;
144 struct qaic_user *usr;
145 int rcu_id;
146 int ret;
147
148 rcu_id = srcu_read_lock(&qdev->dev_lock);
149 if (qdev->dev_state != QAIC_ONLINE) {
150 ret = -ENODEV;
151 goto dev_unlock;
152 }
153
154 usr = kmalloc(sizeof(*usr), GFP_KERNEL);
155 if (!usr) {
156 ret = -ENOMEM;
157 goto dev_unlock;
158 }
159
160 usr->handle = ida_alloc(&qaic_usrs, GFP_KERNEL);
161 if (usr->handle < 0) {
162 ret = usr->handle;
163 goto free_usr;
164 }
165 usr->qddev = qddev;
166 atomic_set(&usr->chunk_id, 0);
167 init_srcu_struct(&usr->qddev_lock);
168 kref_init(&usr->ref_count);
169
170 ret = mutex_lock_interruptible(&qddev->users_mutex);
171 if (ret)
172 goto cleanup_usr;
173
174 list_add(&usr->node, &qddev->users);
175 mutex_unlock(&qddev->users_mutex);
176
177 file->driver_priv = usr;
178
179 srcu_read_unlock(&qdev->dev_lock, rcu_id);
180 return 0;
181
182 cleanup_usr:
183 cleanup_srcu_struct(&usr->qddev_lock);
184 ida_free(&qaic_usrs, usr->handle);
185 free_usr:
186 kfree(usr);
187 dev_unlock:
188 srcu_read_unlock(&qdev->dev_lock, rcu_id);
189 return ret;
190 }
191
qaic_postclose(struct drm_device * dev,struct drm_file * file)192 static void qaic_postclose(struct drm_device *dev, struct drm_file *file)
193 {
194 struct qaic_user *usr = file->driver_priv;
195 struct qaic_drm_device *qddev;
196 struct qaic_device *qdev;
197 int qdev_rcu_id;
198 int usr_rcu_id;
199 int i;
200
201 qddev = usr->qddev;
202 usr_rcu_id = srcu_read_lock(&usr->qddev_lock);
203 if (qddev) {
204 qdev = qddev->qdev;
205 qdev_rcu_id = srcu_read_lock(&qdev->dev_lock);
206 if (qdev->dev_state == QAIC_ONLINE) {
207 qaic_release_usr(qdev, usr);
208 for (i = 0; i < qdev->num_dbc; ++i)
209 if (qdev->dbc[i].usr && qdev->dbc[i].usr->handle == usr->handle)
210 release_dbc(qdev, i);
211 }
212 srcu_read_unlock(&qdev->dev_lock, qdev_rcu_id);
213
214 mutex_lock(&qddev->users_mutex);
215 if (!list_empty(&usr->node))
216 list_del_init(&usr->node);
217 mutex_unlock(&qddev->users_mutex);
218 }
219
220 srcu_read_unlock(&usr->qddev_lock, usr_rcu_id);
221 kref_put(&usr->ref_count, free_usr);
222
223 file->driver_priv = NULL;
224 }
225
226 DEFINE_DRM_ACCEL_FOPS(qaic_accel_fops);
227
228 static const struct drm_ioctl_desc qaic_drm_ioctls[] = {
229 DRM_IOCTL_DEF_DRV(QAIC_MANAGE, qaic_manage_ioctl, 0),
230 DRM_IOCTL_DEF_DRV(QAIC_CREATE_BO, qaic_create_bo_ioctl, 0),
231 DRM_IOCTL_DEF_DRV(QAIC_MMAP_BO, qaic_mmap_bo_ioctl, 0),
232 DRM_IOCTL_DEF_DRV(QAIC_ATTACH_SLICE_BO, qaic_attach_slice_bo_ioctl, 0),
233 DRM_IOCTL_DEF_DRV(QAIC_EXECUTE_BO, qaic_execute_bo_ioctl, 0),
234 DRM_IOCTL_DEF_DRV(QAIC_PARTIAL_EXECUTE_BO, qaic_partial_execute_bo_ioctl, 0),
235 DRM_IOCTL_DEF_DRV(QAIC_WAIT_BO, qaic_wait_bo_ioctl, 0),
236 DRM_IOCTL_DEF_DRV(QAIC_PERF_STATS_BO, qaic_perf_stats_bo_ioctl, 0),
237 DRM_IOCTL_DEF_DRV(QAIC_DETACH_SLICE_BO, qaic_detach_slice_bo_ioctl, 0),
238 };
239
240 static const struct drm_driver qaic_accel_driver = {
241 .driver_features = DRIVER_GEM | DRIVER_COMPUTE_ACCEL,
242
243 .name = QAIC_NAME,
244 .desc = QAIC_DESC,
245
246 .fops = &qaic_accel_fops,
247 .open = qaic_open,
248 .postclose = qaic_postclose,
249
250 .ioctls = qaic_drm_ioctls,
251 .num_ioctls = ARRAY_SIZE(qaic_drm_ioctls),
252 .gem_prime_import = qaic_gem_prime_import,
253 };
254
qaic_create_drm_device(struct qaic_device * qdev,s32 partition_id)255 static int qaic_create_drm_device(struct qaic_device *qdev, s32 partition_id)
256 {
257 struct qaic_drm_device *qddev = qdev->qddev;
258 struct drm_device *drm = to_drm(qddev);
259 int ret;
260
261 /* Hold off implementing partitions until the uapi is determined */
262 if (partition_id != QAIC_NO_PARTITION)
263 return -EINVAL;
264
265 qddev->partition_id = partition_id;
266
267 ret = drm_dev_register(drm, 0);
268 if (ret) {
269 pci_dbg(qdev->pdev, "drm_dev_register failed %d\n", ret);
270 return ret;
271 }
272
273 qaic_debugfs_init(qddev);
274
275 return ret;
276 }
277
qaic_destroy_drm_device(struct qaic_device * qdev,s32 partition_id)278 static void qaic_destroy_drm_device(struct qaic_device *qdev, s32 partition_id)
279 {
280 struct qaic_drm_device *qddev = qdev->qddev;
281 struct drm_device *drm = to_drm(qddev);
282 struct qaic_user *usr;
283
284 drm_dev_unregister(drm);
285 qddev->partition_id = 0;
286 /*
287 * Existing users get unresolvable errors till they close FDs.
288 * Need to sync carefully with users calling close(). The
289 * list of users can be modified elsewhere when the lock isn't
290 * held here, but the sync'ing the srcu with the mutex held
291 * could deadlock. Grab the mutex so that the list will be
292 * unmodified. The user we get will exist as long as the
293 * lock is held. Signal that the qcdev is going away, and
294 * grab a reference to the user so they don't go away for
295 * synchronize_srcu(). Then release the mutex to avoid
296 * deadlock and make sure the user has observed the signal.
297 * With the lock released, we cannot maintain any state of the
298 * user list.
299 */
300 mutex_lock(&qddev->users_mutex);
301 while (!list_empty(&qddev->users)) {
302 usr = list_first_entry(&qddev->users, struct qaic_user, node);
303 list_del_init(&usr->node);
304 kref_get(&usr->ref_count);
305 usr->qddev = NULL;
306 mutex_unlock(&qddev->users_mutex);
307 synchronize_srcu(&usr->qddev_lock);
308 kref_put(&usr->ref_count, free_usr);
309 mutex_lock(&qddev->users_mutex);
310 }
311 mutex_unlock(&qddev->users_mutex);
312 }
313
qaic_mhi_probe(struct mhi_device * mhi_dev,const struct mhi_device_id * id)314 static int qaic_mhi_probe(struct mhi_device *mhi_dev, const struct mhi_device_id *id)
315 {
316 u16 major = -1, minor = -1;
317 struct qaic_device *qdev;
318 int ret;
319
320 /*
321 * Invoking this function indicates that the control channel to the
322 * device is available. We use that as a signal to indicate that
323 * the device side firmware has booted. The device side firmware
324 * manages the device resources, so we need to communicate with it
325 * via the control channel in order to utilize the device. Therefore
326 * we wait until this signal to create the drm dev that userspace will
327 * use to control the device, because without the device side firmware,
328 * userspace can't do anything useful.
329 */
330
331 qdev = pci_get_drvdata(to_pci_dev(mhi_dev->mhi_cntrl->cntrl_dev));
332
333 dev_set_drvdata(&mhi_dev->dev, qdev);
334 qdev->cntl_ch = mhi_dev;
335
336 ret = qaic_control_open(qdev);
337 if (ret) {
338 pci_dbg(qdev->pdev, "%s: control_open failed %d\n", __func__, ret);
339 return ret;
340 }
341
342 qdev->dev_state = QAIC_BOOT;
343 ret = get_cntl_version(qdev, NULL, &major, &minor);
344 if (ret || major != CNTL_MAJOR || minor > CNTL_MINOR) {
345 pci_err(qdev->pdev, "%s: Control protocol version (%d.%d) not supported. Supported version is (%d.%d). Ret: %d\n",
346 __func__, major, minor, CNTL_MAJOR, CNTL_MINOR, ret);
347 ret = -EINVAL;
348 goto close_control;
349 }
350 qdev->dev_state = QAIC_ONLINE;
351 kobject_uevent(&(to_accel_kdev(qdev->qddev))->kobj, KOBJ_ONLINE);
352
353 return ret;
354
355 close_control:
356 qaic_control_close(qdev);
357 return ret;
358 }
359
qaic_mhi_remove(struct mhi_device * mhi_dev)360 static void qaic_mhi_remove(struct mhi_device *mhi_dev)
361 {
362 /* This is redundant since we have already observed the device crash */
363 }
364
qaic_notify_reset(struct qaic_device * qdev)365 static void qaic_notify_reset(struct qaic_device *qdev)
366 {
367 int i;
368
369 kobject_uevent(&(to_accel_kdev(qdev->qddev))->kobj, KOBJ_OFFLINE);
370 qdev->dev_state = QAIC_OFFLINE;
371 /* wake up any waiters to avoid waiting for timeouts at sync */
372 wake_all_cntl(qdev);
373 for (i = 0; i < qdev->num_dbc; ++i)
374 wakeup_dbc(qdev, i);
375 synchronize_srcu(&qdev->dev_lock);
376 }
377
qaic_dev_reset_clean_local_state(struct qaic_device * qdev)378 void qaic_dev_reset_clean_local_state(struct qaic_device *qdev)
379 {
380 int i;
381
382 qaic_notify_reset(qdev);
383
384 /* start tearing things down */
385 for (i = 0; i < qdev->num_dbc; ++i)
386 release_dbc(qdev, i);
387 }
388
create_qdev(struct pci_dev * pdev,const struct qaic_device_config * config)389 static struct qaic_device *create_qdev(struct pci_dev *pdev,
390 const struct qaic_device_config *config)
391 {
392 struct device *dev = &pdev->dev;
393 struct qaic_drm_device *qddev;
394 struct qaic_device *qdev;
395 struct drm_device *drm;
396 int i, ret;
397
398 qdev = devm_kzalloc(dev, sizeof(*qdev), GFP_KERNEL);
399 if (!qdev)
400 return NULL;
401
402 qdev->dev_state = QAIC_OFFLINE;
403 qdev->num_dbc = 16;
404 qdev->dbc = devm_kcalloc(dev, qdev->num_dbc, sizeof(*qdev->dbc), GFP_KERNEL);
405 if (!qdev->dbc)
406 return NULL;
407
408 qddev = devm_drm_dev_alloc(&pdev->dev, &qaic_accel_driver, struct qaic_drm_device, drm);
409 if (IS_ERR(qddev))
410 return NULL;
411
412 drm = to_drm(qddev);
413 pci_set_drvdata(pdev, qdev);
414
415 ret = drmm_mutex_init(drm, &qddev->users_mutex);
416 if (ret)
417 return NULL;
418 ret = drmm_add_action_or_reset(drm, qaicm_pci_release, NULL);
419 if (ret)
420 return NULL;
421 ret = drmm_mutex_init(drm, &qdev->cntl_mutex);
422 if (ret)
423 return NULL;
424 ret = drmm_mutex_init(drm, &qdev->bootlog_mutex);
425 if (ret)
426 return NULL;
427
428 qdev->cntl_wq = qaicm_wq_init(drm, "qaic_cntl");
429 if (IS_ERR(qdev->cntl_wq))
430 return NULL;
431 qdev->qts_wq = qaicm_wq_init(drm, "qaic_ts");
432 if (IS_ERR(qdev->qts_wq))
433 return NULL;
434
435 ret = qaicm_srcu_init(drm, &qdev->dev_lock);
436 if (ret)
437 return NULL;
438
439 qdev->qddev = qddev;
440 qdev->pdev = pdev;
441 qddev->qdev = qdev;
442
443 INIT_LIST_HEAD(&qdev->cntl_xfer_list);
444 INIT_LIST_HEAD(&qdev->bootlog);
445 INIT_LIST_HEAD(&qddev->users);
446
447 for (i = 0; i < qdev->num_dbc; ++i) {
448 spin_lock_init(&qdev->dbc[i].xfer_lock);
449 qdev->dbc[i].qdev = qdev;
450 qdev->dbc[i].id = i;
451 INIT_LIST_HEAD(&qdev->dbc[i].xfer_list);
452 ret = qaicm_srcu_init(drm, &qdev->dbc[i].ch_lock);
453 if (ret)
454 return NULL;
455 init_waitqueue_head(&qdev->dbc[i].dbc_release);
456 INIT_LIST_HEAD(&qdev->dbc[i].bo_lists);
457 ret = drmm_mutex_init(drm, &qdev->dbc[i].req_lock);
458 if (ret)
459 return NULL;
460 }
461
462 return qdev;
463 }
464
init_pci(struct qaic_device * qdev,struct pci_dev * pdev,const struct qaic_device_config * config)465 static int init_pci(struct qaic_device *qdev, struct pci_dev *pdev,
466 const struct qaic_device_config *config)
467 {
468 int bars;
469 int ret;
470
471 bars = pci_select_bars(pdev, IORESOURCE_MEM) & 0x3f;
472
473 /* make sure the device has the expected BARs */
474 if (bars != config->bar_mask) {
475 pci_dbg(pdev, "%s: expected BARs %#x not found in device. Found %#x\n",
476 __func__, config->bar_mask, bars);
477 return -EINVAL;
478 }
479
480 ret = pcim_enable_device(pdev);
481 if (ret)
482 return ret;
483
484 ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
485 if (ret)
486 return ret;
487 dma_set_max_seg_size(&pdev->dev, UINT_MAX);
488
489 qdev->bar_mhi = devm_ioremap_resource(&pdev->dev, &pdev->resource[config->mhi_bar_idx]);
490 if (IS_ERR(qdev->bar_mhi))
491 return PTR_ERR(qdev->bar_mhi);
492
493 qdev->bar_dbc = devm_ioremap_resource(&pdev->dev, &pdev->resource[config->dbc_bar_idx]);
494 if (IS_ERR(qdev->bar_dbc))
495 return PTR_ERR(qdev->bar_dbc);
496
497 /* Managed release since we use pcim_enable_device above */
498 pci_set_master(pdev);
499
500 return 0;
501 }
502
init_msi(struct qaic_device * qdev,struct pci_dev * pdev)503 static int init_msi(struct qaic_device *qdev, struct pci_dev *pdev)
504 {
505 int irq_count = qdev->num_dbc + 1;
506 int mhi_irq;
507 int ret;
508 int i;
509
510 /* Managed release since we use pcim_enable_device */
511 ret = pci_alloc_irq_vectors(pdev, irq_count, irq_count, PCI_IRQ_MSI | PCI_IRQ_MSIX);
512 if (ret == -ENOSPC) {
513 ret = pci_alloc_irq_vectors(pdev, 1, 1, PCI_IRQ_MSI | PCI_IRQ_MSIX);
514 if (ret < 0)
515 return ret;
516
517 /*
518 * Operate in one MSI mode. All interrupts will be directed to
519 * MSI0; every interrupt will wake up all the interrupt handlers
520 * (MHI and DBC[0-15]). Since the interrupt is now shared, it is
521 * not disabled during DBC threaded handler, but only one thread
522 * will be allowed to run per DBC, so while it can be
523 * interrupted, it shouldn't race with itself.
524 */
525 qdev->single_msi = true;
526 pci_info(pdev, "Allocating %d MSIs failed, operating in 1 MSI mode. Performance may be impacted.\n",
527 irq_count);
528 } else if (ret < 0) {
529 return ret;
530 }
531
532 mhi_irq = pci_irq_vector(pdev, 0);
533 if (mhi_irq < 0)
534 return mhi_irq;
535
536 for (i = 0; i < qdev->num_dbc; ++i) {
537 ret = devm_request_threaded_irq(&pdev->dev,
538 pci_irq_vector(pdev, qdev->single_msi ? 0 : i + 1),
539 dbc_irq_handler, dbc_irq_threaded_fn, IRQF_SHARED,
540 "qaic_dbc", &qdev->dbc[i]);
541 if (ret)
542 return ret;
543
544 if (datapath_polling) {
545 qdev->dbc[i].irq = pci_irq_vector(pdev, qdev->single_msi ? 0 : i + 1);
546 if (!qdev->single_msi)
547 disable_irq_nosync(qdev->dbc[i].irq);
548 INIT_WORK(&qdev->dbc[i].poll_work, irq_polling_work);
549 }
550 }
551
552 return mhi_irq;
553 }
554
qaic_pci_probe(struct pci_dev * pdev,const struct pci_device_id * id)555 static int qaic_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
556 {
557 struct qaic_device_config *config = (struct qaic_device_config *)id->driver_data;
558 struct qaic_device *qdev;
559 int mhi_irq;
560 int ret;
561 int i;
562
563 qdev = create_qdev(pdev, config);
564 if (!qdev)
565 return -ENOMEM;
566
567 ret = init_pci(qdev, pdev, config);
568 if (ret)
569 return ret;
570
571 for (i = 0; i < qdev->num_dbc; ++i)
572 qdev->dbc[i].dbc_base = qdev->bar_dbc + QAIC_DBC_OFF(i);
573
574 mhi_irq = init_msi(qdev, pdev);
575 if (mhi_irq < 0)
576 return mhi_irq;
577
578 ret = qaic_create_drm_device(qdev, QAIC_NO_PARTITION);
579 if (ret)
580 return ret;
581
582 qdev->mhi_cntrl = qaic_mhi_register_controller(pdev, qdev->bar_mhi, mhi_irq,
583 qdev->single_msi, config->family);
584 if (IS_ERR(qdev->mhi_cntrl)) {
585 ret = PTR_ERR(qdev->mhi_cntrl);
586 qaic_destroy_drm_device(qdev, QAIC_NO_PARTITION);
587 return ret;
588 }
589
590 return 0;
591 }
592
qaic_pci_remove(struct pci_dev * pdev)593 static void qaic_pci_remove(struct pci_dev *pdev)
594 {
595 struct qaic_device *qdev = pci_get_drvdata(pdev);
596
597 if (!qdev)
598 return;
599
600 qaic_dev_reset_clean_local_state(qdev);
601 qaic_mhi_free_controller(qdev->mhi_cntrl, link_up);
602 qaic_destroy_drm_device(qdev, QAIC_NO_PARTITION);
603 }
604
qaic_pci_shutdown(struct pci_dev * pdev)605 static void qaic_pci_shutdown(struct pci_dev *pdev)
606 {
607 /* see qaic_exit for what link_up is doing */
608 link_up = true;
609 qaic_pci_remove(pdev);
610 }
611
qaic_pci_error_detected(struct pci_dev * pdev,pci_channel_state_t error)612 static pci_ers_result_t qaic_pci_error_detected(struct pci_dev *pdev, pci_channel_state_t error)
613 {
614 return PCI_ERS_RESULT_NEED_RESET;
615 }
616
qaic_pci_reset_prepare(struct pci_dev * pdev)617 static void qaic_pci_reset_prepare(struct pci_dev *pdev)
618 {
619 struct qaic_device *qdev = pci_get_drvdata(pdev);
620
621 qaic_notify_reset(qdev);
622 qaic_mhi_start_reset(qdev->mhi_cntrl);
623 qaic_dev_reset_clean_local_state(qdev);
624 }
625
qaic_pci_reset_done(struct pci_dev * pdev)626 static void qaic_pci_reset_done(struct pci_dev *pdev)
627 {
628 struct qaic_device *qdev = pci_get_drvdata(pdev);
629
630 qaic_mhi_reset_done(qdev->mhi_cntrl);
631 }
632
633 static const struct mhi_device_id qaic_mhi_match_table[] = {
634 { .chan = "QAIC_CONTROL", },
635 {},
636 };
637
638 static struct mhi_driver qaic_mhi_driver = {
639 .id_table = qaic_mhi_match_table,
640 .remove = qaic_mhi_remove,
641 .probe = qaic_mhi_probe,
642 .ul_xfer_cb = qaic_mhi_ul_xfer_cb,
643 .dl_xfer_cb = qaic_mhi_dl_xfer_cb,
644 .driver = {
645 .name = "qaic_mhi",
646 },
647 };
648
649 static const struct pci_device_id qaic_ids[] = {
650 { PCI_DEVICE_DATA(QCOM, AIC080, (kernel_ulong_t)&aic080_config), },
651 { PCI_DEVICE_DATA(QCOM, AIC100, (kernel_ulong_t)&aic100_config), },
652 { PCI_DEVICE_DATA(QCOM, AIC200, (kernel_ulong_t)&aic200_config), },
653 { }
654 };
655 MODULE_DEVICE_TABLE(pci, qaic_ids);
656
657 static const struct pci_error_handlers qaic_pci_err_handler = {
658 .error_detected = qaic_pci_error_detected,
659 .reset_prepare = qaic_pci_reset_prepare,
660 .reset_done = qaic_pci_reset_done,
661 };
662
663 static struct pci_driver qaic_pci_driver = {
664 .name = QAIC_NAME,
665 .id_table = qaic_ids,
666 .probe = qaic_pci_probe,
667 .remove = qaic_pci_remove,
668 .shutdown = qaic_pci_shutdown,
669 .err_handler = &qaic_pci_err_handler,
670 };
671
qaic_init(void)672 static int __init qaic_init(void)
673 {
674 int ret;
675
676 ret = pci_register_driver(&qaic_pci_driver);
677 if (ret) {
678 pr_debug("qaic: pci_register_driver failed %d\n", ret);
679 return ret;
680 }
681
682 ret = mhi_driver_register(&qaic_mhi_driver);
683 if (ret) {
684 pr_debug("qaic: mhi_driver_register failed %d\n", ret);
685 goto free_pci;
686 }
687
688 ret = sahara_register();
689 if (ret) {
690 pr_debug("qaic: sahara_register failed %d\n", ret);
691 goto free_mhi;
692 }
693
694 ret = qaic_timesync_init();
695 if (ret)
696 pr_debug("qaic: qaic_timesync_init failed %d\n", ret);
697
698 ret = qaic_bootlog_register();
699 if (ret)
700 pr_debug("qaic: qaic_bootlog_register failed %d\n", ret);
701
702 ret = qaic_ras_register();
703 if (ret)
704 pr_debug("qaic: qaic_ras_register failed %d\n", ret);
705
706 return 0;
707
708 free_mhi:
709 mhi_driver_unregister(&qaic_mhi_driver);
710 free_pci:
711 pci_unregister_driver(&qaic_pci_driver);
712 return ret;
713 }
714
qaic_exit(void)715 static void __exit qaic_exit(void)
716 {
717 /*
718 * We assume that qaic_pci_remove() is called due to a hotplug event
719 * which would mean that the link is down, and thus
720 * qaic_mhi_free_controller() should not try to access the device during
721 * cleanup.
722 * We call pci_unregister_driver() below, which also triggers
723 * qaic_pci_remove(), but since this is module exit, we expect the link
724 * to the device to be up, in which case qaic_mhi_free_controller()
725 * should try to access the device during cleanup to put the device in
726 * a sane state.
727 * For that reason, we set link_up here to let qaic_mhi_free_controller
728 * know the expected link state. Since the module is going to be
729 * removed at the end of this, we don't need to worry about
730 * reinitializing the link_up state after the cleanup is done.
731 */
732 link_up = true;
733 qaic_ras_unregister();
734 qaic_bootlog_unregister();
735 qaic_timesync_deinit();
736 sahara_unregister();
737 mhi_driver_unregister(&qaic_mhi_driver);
738 pci_unregister_driver(&qaic_pci_driver);
739 }
740
741 module_init(qaic_init);
742 module_exit(qaic_exit);
743
744 MODULE_AUTHOR(QAIC_DESC " Kernel Driver Team");
745 MODULE_DESCRIPTION(QAIC_DESC " Accel Driver");
746 MODULE_LICENSE("GPL");
747