1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright(c) 2023 Advanced Micro Devices, Inc */
3
4 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
5
6 #include <linux/pci.h>
7
8 #include <linux/pds/pds_common.h>
9
10 #include "core.h"
11
12 MODULE_DESCRIPTION(PDSC_DRV_DESCRIPTION);
13 MODULE_AUTHOR("Advanced Micro Devices, Inc");
14 MODULE_LICENSE("GPL");
15
16 /* Supported devices */
17 static const struct pci_device_id pdsc_id_table[] = {
18 { PCI_VDEVICE(PENSANDO, PCI_DEVICE_ID_PENSANDO_CORE_PF) },
19 { PCI_VDEVICE(PENSANDO, PCI_DEVICE_ID_PENSANDO_VDPA_VF) },
20 { 0, } /* end of table */
21 };
22 MODULE_DEVICE_TABLE(pci, pdsc_id_table);
23
24 static void pdsc_stop_health_thread(struct pdsc *pdsc);
25
pdsc_wdtimer_cb(struct timer_list * t)26 static void pdsc_wdtimer_cb(struct timer_list *t)
27 {
28 struct pdsc *pdsc = timer_container_of(pdsc, t, wdtimer);
29
30 dev_dbg(pdsc->dev, "%s: jiffies %ld\n", __func__, jiffies);
31 mod_timer(&pdsc->wdtimer,
32 round_jiffies(jiffies + pdsc->wdtimer_period));
33
34 queue_work(pdsc->wq, &pdsc->health_work);
35 }
36
pdsc_unmap_bars(struct pdsc * pdsc)37 static void pdsc_unmap_bars(struct pdsc *pdsc)
38 {
39 struct pdsc_dev_bar *bars = pdsc->bars;
40 unsigned int i;
41
42 pdsc->info_regs = NULL;
43 pdsc->cmd_regs = NULL;
44 pdsc->intr_status = NULL;
45 pdsc->intr_ctrl = NULL;
46
47 for (i = 0; i < PDS_CORE_BARS_MAX; i++) {
48 if (bars[i].vaddr)
49 pci_iounmap(pdsc->pdev, bars[i].vaddr);
50 bars[i].vaddr = NULL;
51 }
52 }
53
pdsc_map_bars(struct pdsc * pdsc)54 static int pdsc_map_bars(struct pdsc *pdsc)
55 {
56 struct pdsc_dev_bar *bar = pdsc->bars;
57 struct pci_dev *pdev = pdsc->pdev;
58 struct device *dev = pdsc->dev;
59 struct pdsc_dev_bar *bars;
60 unsigned int i, j;
61 int num_bars = 0;
62 int err;
63 u32 sig;
64
65 bars = pdsc->bars;
66
67 /* Since the PCI interface in the hardware is configurable,
68 * we need to poke into all the bars to find the set we're
69 * expecting.
70 */
71 for (i = 0, j = 0; i < PDS_CORE_BARS_MAX; i++) {
72 if (!(pci_resource_flags(pdev, i) & IORESOURCE_MEM))
73 continue;
74
75 bars[j].len = pci_resource_len(pdev, i);
76 bars[j].bus_addr = pci_resource_start(pdev, i);
77 bars[j].res_index = i;
78
79 /* only map the whole bar 0 */
80 if (j > 0) {
81 bars[j].vaddr = NULL;
82 } else {
83 bars[j].vaddr = pci_iomap(pdev, i, bars[j].len);
84 if (!bars[j].vaddr) {
85 dev_err(dev, "Cannot map BAR %d, aborting\n", i);
86 return -ENODEV;
87 }
88 }
89
90 j++;
91 }
92 num_bars = j;
93
94 /* BAR0: dev_cmd and interrupts */
95 if (num_bars < 1) {
96 dev_err(dev, "No bars found\n");
97 err = -EFAULT;
98 goto err_out;
99 }
100
101 if (bar->len < PDS_CORE_BAR0_SIZE) {
102 dev_err(dev, "Resource bar size %lu too small\n", bar->len);
103 err = -EFAULT;
104 goto err_out;
105 }
106
107 pdsc->info_regs = bar->vaddr + PDS_CORE_BAR0_DEV_INFO_REGS_OFFSET;
108 pdsc->cmd_regs = bar->vaddr + PDS_CORE_BAR0_DEV_CMD_REGS_OFFSET;
109 pdsc->intr_status = bar->vaddr + PDS_CORE_BAR0_INTR_STATUS_OFFSET;
110 pdsc->intr_ctrl = bar->vaddr + PDS_CORE_BAR0_INTR_CTRL_OFFSET;
111
112 sig = ioread32(&pdsc->info_regs->signature);
113 if (sig != PDS_CORE_DEV_INFO_SIGNATURE) {
114 dev_err(dev, "Incompatible firmware signature %x", sig);
115 err = -EFAULT;
116 goto err_out;
117 }
118
119 /* BAR1: doorbells */
120 bar++;
121 if (num_bars < 2) {
122 dev_err(dev, "Doorbell bar missing\n");
123 err = -EFAULT;
124 goto err_out;
125 }
126
127 pdsc->db_pages = bar->vaddr;
128 pdsc->phy_db_pages = bar->bus_addr;
129
130 return 0;
131
132 err_out:
133 pdsc_unmap_bars(pdsc);
134 return err;
135 }
136
pdsc_map_dbpage(struct pdsc * pdsc,int page_num)137 void __iomem *pdsc_map_dbpage(struct pdsc *pdsc, int page_num)
138 {
139 return pci_iomap_range(pdsc->pdev,
140 pdsc->bars[PDS_CORE_PCI_BAR_DBELL].res_index,
141 (u64)page_num << PAGE_SHIFT, PAGE_SIZE);
142 }
143
pdsc_sriov_configure(struct pci_dev * pdev,int num_vfs)144 static int pdsc_sriov_configure(struct pci_dev *pdev, int num_vfs)
145 {
146 struct pdsc *pdsc = pci_get_drvdata(pdev);
147 struct device *dev = pdsc->dev;
148 int ret = 0;
149
150 if (num_vfs > 0) {
151 pdsc->vfs = kzalloc_objs(struct pdsc_vf, num_vfs);
152 if (!pdsc->vfs)
153 return -ENOMEM;
154 pdsc->num_vfs = num_vfs;
155
156 ret = pci_enable_sriov(pdev, num_vfs);
157 if (ret) {
158 dev_err(dev, "Cannot enable SRIOV: %pe\n",
159 ERR_PTR(ret));
160 goto no_vfs;
161 }
162
163 return num_vfs;
164 }
165
166 no_vfs:
167 pci_disable_sriov(pdev);
168
169 kfree(pdsc->vfs);
170 pdsc->vfs = NULL;
171 pdsc->num_vfs = 0;
172
173 return ret;
174 }
175
pdsc_init_vf(struct pdsc * vf)176 static int pdsc_init_vf(struct pdsc *vf)
177 {
178 struct devlink *dl;
179 struct pdsc *pf;
180 int err;
181
182 pf = pdsc_get_pf_struct(vf->pdev);
183 if (IS_ERR_OR_NULL(pf))
184 return PTR_ERR(pf) ?: -1;
185
186 vf->vf_id = pci_iov_vf_id(vf->pdev);
187
188 dl = priv_to_devlink(vf);
189 devl_lock(dl);
190 devl_register(dl);
191 devl_unlock(dl);
192
193 pf->vfs[vf->vf_id].vf = vf;
194 err = pdsc_auxbus_dev_add(vf, pf, PDS_DEV_TYPE_VDPA,
195 &pf->vfs[vf->vf_id].padev);
196 if (err) {
197 devl_lock(dl);
198 devl_unregister(dl);
199 devl_unlock(dl);
200 }
201
202 return err;
203 }
204
205 static const struct devlink_health_reporter_ops pdsc_fw_reporter_ops = {
206 .name = "fw",
207 .diagnose = pdsc_fw_reporter_diagnose,
208 };
209
210 static const struct devlink_param pdsc_dl_params[] = {
211 DEVLINK_PARAM_GENERIC(ENABLE_VNET,
212 BIT(DEVLINK_PARAM_CMODE_RUNTIME),
213 pdsc_dl_enable_get,
214 pdsc_dl_enable_set,
215 pdsc_dl_enable_validate),
216 };
217
218 #define PDSC_WQ_NAME_LEN 24
219
pdsc_init_pf(struct pdsc * pdsc)220 static int pdsc_init_pf(struct pdsc *pdsc)
221 {
222 struct devlink_health_reporter *hr;
223 char wq_name[PDSC_WQ_NAME_LEN];
224 struct devlink *dl;
225 int err;
226
227 pcie_print_link_status(pdsc->pdev);
228
229 err = pci_request_regions(pdsc->pdev, PDS_CORE_DRV_NAME);
230 if (err) {
231 dev_err(pdsc->dev, "Cannot request PCI regions: %pe\n",
232 ERR_PTR(err));
233 return err;
234 }
235
236 err = pdsc_map_bars(pdsc);
237 if (err)
238 goto err_out_release_regions;
239
240 /* General workqueue and timer, but don't start timer yet */
241 snprintf(wq_name, sizeof(wq_name), "%s.%d", PDS_CORE_DRV_NAME, pdsc->uid);
242 pdsc->wq = create_singlethread_workqueue(wq_name);
243 if (!pdsc->wq) {
244 err = -ENOMEM;
245 goto err_out_unmap_bars;
246 }
247 INIT_WORK(&pdsc->health_work, pdsc_health_thread);
248 INIT_WORK(&pdsc->pci_reset_work, pdsc_pci_reset_thread);
249 timer_setup(&pdsc->wdtimer, pdsc_wdtimer_cb, 0);
250 pdsc->wdtimer_period = PDSC_WATCHDOG_SECS * HZ;
251
252 mutex_init(&pdsc->devcmd_lock);
253 mutex_init(&pdsc->config_lock);
254 spin_lock_init(&pdsc->adminq_lock);
255
256 mutex_lock(&pdsc->config_lock);
257 set_bit(PDSC_S_FW_DEAD, &pdsc->state);
258
259 err = pdsc_setup(pdsc, PDSC_SETUP_INIT);
260 if (err) {
261 mutex_unlock(&pdsc->config_lock);
262 goto err_out_shutdown_timer;
263 }
264
265 err = pdsc_start(pdsc);
266 if (err) {
267 mutex_unlock(&pdsc->config_lock);
268 goto err_out_teardown;
269 }
270
271 mutex_unlock(&pdsc->config_lock);
272
273 err = pdsc_auxbus_dev_add(pdsc, pdsc, PDS_DEV_TYPE_FWCTL, &pdsc->padev);
274 if (err)
275 goto err_out_stop;
276
277 dl = priv_to_devlink(pdsc);
278 devl_lock(dl);
279 err = devl_params_register(dl, pdsc_dl_params,
280 ARRAY_SIZE(pdsc_dl_params));
281 if (err) {
282 devl_unlock(dl);
283 dev_warn(pdsc->dev, "Failed to register devlink params: %pe\n",
284 ERR_PTR(err));
285 goto err_out_del_dev;
286 }
287
288 hr = devl_health_reporter_create(dl, &pdsc_fw_reporter_ops, pdsc);
289 if (IS_ERR(hr)) {
290 devl_unlock(dl);
291 dev_warn(pdsc->dev, "Failed to create fw reporter: %pe\n", hr);
292 err = PTR_ERR(hr);
293 goto err_out_unreg_params;
294 }
295 pdsc->fw_reporter = hr;
296
297 devl_register(dl);
298 devl_unlock(dl);
299
300 /* Lastly, start the health check timer */
301 mod_timer(&pdsc->wdtimer, round_jiffies(jiffies + pdsc->wdtimer_period));
302
303 return 0;
304
305 err_out_unreg_params:
306 devlink_params_unregister(dl, pdsc_dl_params,
307 ARRAY_SIZE(pdsc_dl_params));
308 err_out_del_dev:
309 pdsc_auxbus_dev_del(pdsc, pdsc, &pdsc->padev);
310 err_out_stop:
311 pdsc_stop(pdsc);
312 err_out_teardown:
313 pdsc_teardown(pdsc, PDSC_TEARDOWN_REMOVING);
314 err_out_shutdown_timer:
315 timer_shutdown_sync(&pdsc->wdtimer);
316 if (pdsc->wq)
317 destroy_workqueue(pdsc->wq);
318 mutex_destroy(&pdsc->config_lock);
319 mutex_destroy(&pdsc->devcmd_lock);
320 pdsc_deferred_dma_free(pdsc);
321 pci_free_irq_vectors(pdsc->pdev);
322 err_out_unmap_bars:
323 pdsc_unmap_bars(pdsc);
324 err_out_release_regions:
325 pci_release_regions(pdsc->pdev);
326
327 return err;
328 }
329
330 static const struct devlink_ops pdsc_dl_ops = {
331 .info_get = pdsc_dl_info_get,
332 .flash_update = pdsc_dl_flash_update,
333 };
334
335 static const struct devlink_ops pdsc_dl_vf_ops = {
336 };
337
338 static DEFINE_IDA(pdsc_ida);
339
pdsc_probe(struct pci_dev * pdev,const struct pci_device_id * ent)340 static int pdsc_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
341 {
342 struct device *dev = &pdev->dev;
343 const struct devlink_ops *ops;
344 struct devlink *dl;
345 struct pdsc *pdsc;
346 bool is_pf;
347 int err;
348
349 is_pf = !pdev->is_virtfn;
350 ops = is_pf ? &pdsc_dl_ops : &pdsc_dl_vf_ops;
351 dl = devlink_alloc(ops, sizeof(struct pdsc), dev);
352 if (!dl)
353 return -ENOMEM;
354 pdsc = devlink_priv(dl);
355
356 pdsc->pdev = pdev;
357 pdsc->dev = &pdev->dev;
358 set_bit(PDSC_S_INITING_DRIVER, &pdsc->state);
359 INIT_LIST_HEAD(&pdsc->deferred_dma_list);
360 spin_lock_init(&pdsc->deferred_dma_lock);
361 pci_set_drvdata(pdev, pdsc);
362 pdsc_debugfs_add_dev(pdsc);
363
364 err = ida_alloc(&pdsc_ida, GFP_KERNEL);
365 if (err < 0) {
366 dev_err(pdsc->dev, "%s: id alloc failed: %pe\n",
367 __func__, ERR_PTR(err));
368 goto err_out_free_devlink;
369 }
370 pdsc->uid = err;
371
372 /* Query system for DMA addressing limitation for the device. */
373 err = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(PDS_CORE_ADDR_LEN));
374 if (err) {
375 dev_err(dev, "Unable to obtain 64-bit DMA for consistent allocations, aborting: %pe\n",
376 ERR_PTR(err));
377 goto err_out_free_ida;
378 }
379
380 err = pci_enable_device(pdev);
381 if (err) {
382 dev_err(dev, "Cannot enable PCI device: %pe\n", ERR_PTR(err));
383 goto err_out_free_ida;
384 }
385 pci_set_master(pdev);
386
387 if (is_pf)
388 err = pdsc_init_pf(pdsc);
389 else
390 err = pdsc_init_vf(pdsc);
391 if (err) {
392 dev_err(dev, "Cannot init device: %pe\n", ERR_PTR(err));
393 goto err_out_disable_device;
394 }
395
396 clear_bit(PDSC_S_INITING_DRIVER, &pdsc->state);
397 return 0;
398
399 err_out_disable_device:
400 pci_disable_device(pdev);
401 err_out_free_ida:
402 ida_free(&pdsc_ida, pdsc->uid);
403 err_out_free_devlink:
404 pdsc_debugfs_del_dev(pdsc);
405 devlink_free(dl);
406
407 return err;
408 }
409
pdsc_remove(struct pci_dev * pdev)410 static void pdsc_remove(struct pci_dev *pdev)
411 {
412 struct pdsc *pdsc = pci_get_drvdata(pdev);
413 struct devlink *dl;
414
415 /* Unhook the registrations first to be sure there
416 * are no requests while we're stopping.
417 */
418 dl = priv_to_devlink(pdsc);
419 devl_lock(dl);
420 devl_unregister(dl);
421 if (!pdev->is_virtfn) {
422 if (pdsc->fw_reporter) {
423 devl_health_reporter_destroy(pdsc->fw_reporter);
424 pdsc->fw_reporter = NULL;
425 }
426 devl_params_unregister(dl, pdsc_dl_params,
427 ARRAY_SIZE(pdsc_dl_params));
428 }
429 devl_unlock(dl);
430
431 if (pdev->is_virtfn) {
432 struct pdsc *pf;
433
434 pf = pdsc_get_pf_struct(pdsc->pdev);
435 if (!IS_ERR(pf)) {
436 pdsc_auxbus_dev_del(pdsc, pf, &pf->vfs[pdsc->vf_id].padev);
437 pf->vfs[pdsc->vf_id].vf = NULL;
438 }
439 } else {
440 /* Remove the VFs and their aux_bus connections before other
441 * cleanup so that the clients can use the AdminQ to cleanly
442 * shut themselves down.
443 */
444 pdsc_sriov_configure(pdev, 0);
445 pdsc_auxbus_dev_del(pdsc, pdsc, &pdsc->padev);
446
447 timer_shutdown_sync(&pdsc->wdtimer);
448
449 mutex_lock(&pdsc->config_lock);
450 set_bit(PDSC_S_STOPPING_DRIVER, &pdsc->state);
451
452 pdsc_stop(pdsc);
453 pdsc_teardown(pdsc, PDSC_TEARDOWN_REMOVING);
454 mutex_unlock(&pdsc->config_lock);
455
456 if (pdsc->wq)
457 destroy_workqueue(pdsc->wq);
458 mutex_destroy(&pdsc->config_lock);
459 mutex_destroy(&pdsc->devcmd_lock);
460
461 pdsc_unmap_bars(pdsc);
462 pci_release_regions(pdev);
463 }
464
465 pci_disable_device(pdev);
466 pdsc_deferred_dma_free(pdsc);
467
468 ida_free(&pdsc_ida, pdsc->uid);
469 pdsc_debugfs_del_dev(pdsc);
470 devlink_free(dl);
471 }
472
pdsc_stop_health_thread(struct pdsc * pdsc)473 static void pdsc_stop_health_thread(struct pdsc *pdsc)
474 {
475 if (pdsc->pdev->is_virtfn)
476 return;
477
478 timer_shutdown_sync(&pdsc->wdtimer);
479 if (pdsc->health_work.func && !pdsc->health_stopped) {
480 disable_work_sync(&pdsc->health_work);
481 pdsc->health_stopped = true;
482 }
483 }
484
pdsc_restart_health_thread(struct pdsc * pdsc)485 static void pdsc_restart_health_thread(struct pdsc *pdsc)
486 {
487 if (pdsc->pdev->is_virtfn)
488 return;
489
490 if (pdsc->health_stopped) {
491 enable_work(&pdsc->health_work);
492 pdsc->health_stopped = false;
493 }
494 timer_setup(&pdsc->wdtimer, pdsc_wdtimer_cb, 0);
495 mod_timer(&pdsc->wdtimer, jiffies + 1);
496 }
497
pdsc_reset_prepare(struct pci_dev * pdev)498 static void pdsc_reset_prepare(struct pci_dev *pdev)
499 {
500 struct pdsc *pdsc = pci_get_drvdata(pdev);
501
502 pdsc_stop_health_thread(pdsc);
503 pdsc_fw_down(pdsc);
504
505 if (pdev->is_virtfn) {
506 struct pdsc *pf;
507
508 pf = pdsc_get_pf_struct(pdsc->pdev);
509 if (!IS_ERR(pf))
510 pdsc_auxbus_dev_del(pdsc, pf,
511 &pf->vfs[pdsc->vf_id].padev);
512 } else {
513 pdsc_auxbus_dev_del(pdsc, pdsc, &pdsc->padev);
514 }
515
516 pdsc_unmap_bars(pdsc);
517 pci_release_regions(pdev);
518 if (pci_is_enabled(pdev))
519 pci_disable_device(pdev);
520 pdsc_deferred_dma_free(pdsc);
521 }
522
pdsc_reset_done(struct pci_dev * pdev)523 static void pdsc_reset_done(struct pci_dev *pdev)
524 {
525 struct pdsc *pdsc = pci_get_drvdata(pdev);
526 struct device *dev = pdsc->dev;
527 int err;
528
529 err = pci_enable_device(pdev);
530 if (err) {
531 dev_err(dev, "Cannot enable PCI device: %pe\n", ERR_PTR(err));
532 return;
533 }
534 pci_set_master(pdev);
535
536 if (!pdev->is_virtfn) {
537 pcie_print_link_status(pdsc->pdev);
538
539 err = pci_request_regions(pdsc->pdev, PDS_CORE_DRV_NAME);
540 if (err) {
541 dev_err(pdsc->dev, "Cannot request PCI regions: %pe\n",
542 ERR_PTR(err));
543 return;
544 }
545
546 err = pdsc_map_bars(pdsc);
547 if (err)
548 return;
549 }
550
551 pdsc_fw_up(pdsc);
552 pdsc_restart_health_thread(pdsc);
553
554 if (pdev->is_virtfn) {
555 struct pdsc *pf;
556
557 pf = pdsc_get_pf_struct(pdsc->pdev);
558 if (!IS_ERR(pf))
559 pdsc_auxbus_dev_add(pdsc, pf, PDS_DEV_TYPE_VDPA,
560 &pf->vfs[pdsc->vf_id].padev);
561 } else {
562 pdsc_auxbus_dev_add(pdsc, pdsc, PDS_DEV_TYPE_FWCTL,
563 &pdsc->padev);
564 }
565 }
566
pdsc_pci_error_detected(struct pci_dev * pdev,pci_channel_state_t error)567 static pci_ers_result_t pdsc_pci_error_detected(struct pci_dev *pdev,
568 pci_channel_state_t error)
569 {
570 if (error == pci_channel_io_frozen) {
571 struct pdsc *pdsc = pci_get_drvdata(pdev);
572
573 pdsc_reset_prepare(pdev);
574 if (!pdev->is_virtfn)
575 cancel_work_sync(&pdsc->pci_reset_work);
576 return PCI_ERS_RESULT_NEED_RESET;
577 }
578
579 return PCI_ERS_RESULT_NONE;
580 }
581
pdsc_pci_error_resume(struct pci_dev * pdev)582 static void pdsc_pci_error_resume(struct pci_dev *pdev)
583 {
584 struct pdsc *pdsc = pci_get_drvdata(pdev);
585
586 if (test_bit(PDSC_S_FW_DEAD, &pdsc->state))
587 pci_reset_function_locked(pdev);
588 }
589
590 static const struct pci_error_handlers pdsc_err_handler = {
591 /* FLR handling */
592 .reset_prepare = pdsc_reset_prepare,
593 .reset_done = pdsc_reset_done,
594
595 /* AER handling */
596 .error_detected = pdsc_pci_error_detected,
597 .resume = pdsc_pci_error_resume,
598 };
599
600 static struct pci_driver pdsc_driver = {
601 .name = PDS_CORE_DRV_NAME,
602 .id_table = pdsc_id_table,
603 .probe = pdsc_probe,
604 .remove = pdsc_remove,
605 .sriov_configure = pdsc_sriov_configure,
606 .err_handler = &pdsc_err_handler,
607 };
608
pdsc_get_pf_struct(struct pci_dev * vf_pdev)609 void *pdsc_get_pf_struct(struct pci_dev *vf_pdev)
610 {
611 return pci_iov_get_pf_drvdata(vf_pdev, &pdsc_driver);
612 }
613 EXPORT_SYMBOL_GPL(pdsc_get_pf_struct);
614
pdsc_init_module(void)615 static int __init pdsc_init_module(void)
616 {
617 if (strcmp(KBUILD_MODNAME, PDS_CORE_DRV_NAME))
618 return -EINVAL;
619
620 pdsc_debugfs_create();
621 return pci_register_driver(&pdsc_driver);
622 }
623
pdsc_cleanup_module(void)624 static void __exit pdsc_cleanup_module(void)
625 {
626 pci_unregister_driver(&pdsc_driver);
627 pdsc_debugfs_destroy();
628 }
629
630 module_init(pdsc_init_module);
631 module_exit(pdsc_cleanup_module);
632