1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright(c) 2023 Advanced Micro Devices, Inc */
3
4 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
5
6 #include <linux/pci.h>
7
8 #include <linux/pds/pds_common.h>
9
10 #include "core.h"
11
12 MODULE_DESCRIPTION(PDSC_DRV_DESCRIPTION);
13 MODULE_AUTHOR("Advanced Micro Devices, Inc");
14 MODULE_LICENSE("GPL");
15
16 /* Supported devices */
17 static const struct pci_device_id pdsc_id_table[] = {
18 { PCI_VDEVICE(PENSANDO, PCI_DEVICE_ID_PENSANDO_CORE_PF) },
19 { PCI_VDEVICE(PENSANDO, PCI_DEVICE_ID_PENSANDO_VDPA_VF) },
20 { 0, } /* end of table */
21 };
22 MODULE_DEVICE_TABLE(pci, pdsc_id_table);
23
pdsc_wdtimer_cb(struct timer_list * t)24 static void pdsc_wdtimer_cb(struct timer_list *t)
25 {
26 struct pdsc *pdsc = timer_container_of(pdsc, t, wdtimer);
27
28 dev_dbg(pdsc->dev, "%s: jiffies %ld\n", __func__, jiffies);
29 mod_timer(&pdsc->wdtimer,
30 round_jiffies(jiffies + pdsc->wdtimer_period));
31
32 queue_work(pdsc->wq, &pdsc->health_work);
33 }
34
pdsc_unmap_bars(struct pdsc * pdsc)35 static void pdsc_unmap_bars(struct pdsc *pdsc)
36 {
37 struct pdsc_dev_bar *bars = pdsc->bars;
38 unsigned int i;
39
40 pdsc->info_regs = NULL;
41 pdsc->cmd_regs = NULL;
42 pdsc->intr_status = NULL;
43 pdsc->intr_ctrl = NULL;
44
45 for (i = 0; i < PDS_CORE_BARS_MAX; i++) {
46 if (bars[i].vaddr)
47 pci_iounmap(pdsc->pdev, bars[i].vaddr);
48 bars[i].vaddr = NULL;
49 }
50 }
51
pdsc_map_bars(struct pdsc * pdsc)52 static int pdsc_map_bars(struct pdsc *pdsc)
53 {
54 struct pdsc_dev_bar *bar = pdsc->bars;
55 struct pci_dev *pdev = pdsc->pdev;
56 struct device *dev = pdsc->dev;
57 struct pdsc_dev_bar *bars;
58 unsigned int i, j;
59 int num_bars = 0;
60 int err;
61 u32 sig;
62
63 bars = pdsc->bars;
64
65 /* Since the PCI interface in the hardware is configurable,
66 * we need to poke into all the bars to find the set we're
67 * expecting.
68 */
69 for (i = 0, j = 0; i < PDS_CORE_BARS_MAX; i++) {
70 if (!(pci_resource_flags(pdev, i) & IORESOURCE_MEM))
71 continue;
72
73 bars[j].len = pci_resource_len(pdev, i);
74 bars[j].bus_addr = pci_resource_start(pdev, i);
75 bars[j].res_index = i;
76
77 /* only map the whole bar 0 */
78 if (j > 0) {
79 bars[j].vaddr = NULL;
80 } else {
81 bars[j].vaddr = pci_iomap(pdev, i, bars[j].len);
82 if (!bars[j].vaddr) {
83 dev_err(dev, "Cannot map BAR %d, aborting\n", i);
84 return -ENODEV;
85 }
86 }
87
88 j++;
89 }
90 num_bars = j;
91
92 /* BAR0: dev_cmd and interrupts */
93 if (num_bars < 1) {
94 dev_err(dev, "No bars found\n");
95 err = -EFAULT;
96 goto err_out;
97 }
98
99 if (bar->len < PDS_CORE_BAR0_SIZE) {
100 dev_err(dev, "Resource bar size %lu too small\n", bar->len);
101 err = -EFAULT;
102 goto err_out;
103 }
104
105 pdsc->info_regs = bar->vaddr + PDS_CORE_BAR0_DEV_INFO_REGS_OFFSET;
106 pdsc->cmd_regs = bar->vaddr + PDS_CORE_BAR0_DEV_CMD_REGS_OFFSET;
107 pdsc->intr_status = bar->vaddr + PDS_CORE_BAR0_INTR_STATUS_OFFSET;
108 pdsc->intr_ctrl = bar->vaddr + PDS_CORE_BAR0_INTR_CTRL_OFFSET;
109
110 sig = ioread32(&pdsc->info_regs->signature);
111 if (sig != PDS_CORE_DEV_INFO_SIGNATURE) {
112 dev_err(dev, "Incompatible firmware signature %x", sig);
113 err = -EFAULT;
114 goto err_out;
115 }
116
117 /* BAR1: doorbells */
118 bar++;
119 if (num_bars < 2) {
120 dev_err(dev, "Doorbell bar missing\n");
121 err = -EFAULT;
122 goto err_out;
123 }
124
125 pdsc->db_pages = bar->vaddr;
126 pdsc->phy_db_pages = bar->bus_addr;
127
128 return 0;
129
130 err_out:
131 pdsc_unmap_bars(pdsc);
132 return err;
133 }
134
pdsc_map_dbpage(struct pdsc * pdsc,int page_num)135 void __iomem *pdsc_map_dbpage(struct pdsc *pdsc, int page_num)
136 {
137 return pci_iomap_range(pdsc->pdev,
138 pdsc->bars[PDS_CORE_PCI_BAR_DBELL].res_index,
139 (u64)page_num << PAGE_SHIFT, PAGE_SIZE);
140 }
141
pdsc_sriov_configure(struct pci_dev * pdev,int num_vfs)142 static int pdsc_sriov_configure(struct pci_dev *pdev, int num_vfs)
143 {
144 struct pdsc *pdsc = pci_get_drvdata(pdev);
145 struct device *dev = pdsc->dev;
146 int ret = 0;
147
148 if (num_vfs > 0) {
149 pdsc->vfs = kcalloc(num_vfs, sizeof(struct pdsc_vf),
150 GFP_KERNEL);
151 if (!pdsc->vfs)
152 return -ENOMEM;
153 pdsc->num_vfs = num_vfs;
154
155 ret = pci_enable_sriov(pdev, num_vfs);
156 if (ret) {
157 dev_err(dev, "Cannot enable SRIOV: %pe\n",
158 ERR_PTR(ret));
159 goto no_vfs;
160 }
161
162 return num_vfs;
163 }
164
165 no_vfs:
166 pci_disable_sriov(pdev);
167
168 kfree(pdsc->vfs);
169 pdsc->vfs = NULL;
170 pdsc->num_vfs = 0;
171
172 return ret;
173 }
174
pdsc_init_vf(struct pdsc * vf)175 static int pdsc_init_vf(struct pdsc *vf)
176 {
177 struct devlink *dl;
178 struct pdsc *pf;
179 int err;
180
181 pf = pdsc_get_pf_struct(vf->pdev);
182 if (IS_ERR_OR_NULL(pf))
183 return PTR_ERR(pf) ?: -1;
184
185 vf->vf_id = pci_iov_vf_id(vf->pdev);
186
187 dl = priv_to_devlink(vf);
188 devl_lock(dl);
189 devl_register(dl);
190 devl_unlock(dl);
191
192 pf->vfs[vf->vf_id].vf = vf;
193 err = pdsc_auxbus_dev_add(vf, pf, PDS_DEV_TYPE_VDPA,
194 &pf->vfs[vf->vf_id].padev);
195 if (err) {
196 devl_lock(dl);
197 devl_unregister(dl);
198 devl_unlock(dl);
199 }
200
201 return err;
202 }
203
204 static const struct devlink_health_reporter_ops pdsc_fw_reporter_ops = {
205 .name = "fw",
206 .diagnose = pdsc_fw_reporter_diagnose,
207 };
208
209 static const struct devlink_param pdsc_dl_params[] = {
210 DEVLINK_PARAM_GENERIC(ENABLE_VNET,
211 BIT(DEVLINK_PARAM_CMODE_RUNTIME),
212 pdsc_dl_enable_get,
213 pdsc_dl_enable_set,
214 pdsc_dl_enable_validate),
215 };
216
217 #define PDSC_WQ_NAME_LEN 24
218
pdsc_init_pf(struct pdsc * pdsc)219 static int pdsc_init_pf(struct pdsc *pdsc)
220 {
221 struct devlink_health_reporter *hr;
222 char wq_name[PDSC_WQ_NAME_LEN];
223 struct devlink *dl;
224 int err;
225
226 pcie_print_link_status(pdsc->pdev);
227
228 err = pci_request_regions(pdsc->pdev, PDS_CORE_DRV_NAME);
229 if (err) {
230 dev_err(pdsc->dev, "Cannot request PCI regions: %pe\n",
231 ERR_PTR(err));
232 return err;
233 }
234
235 err = pdsc_map_bars(pdsc);
236 if (err)
237 goto err_out_release_regions;
238
239 /* General workqueue and timer, but don't start timer yet */
240 snprintf(wq_name, sizeof(wq_name), "%s.%d", PDS_CORE_DRV_NAME, pdsc->uid);
241 pdsc->wq = create_singlethread_workqueue(wq_name);
242 INIT_WORK(&pdsc->health_work, pdsc_health_thread);
243 INIT_WORK(&pdsc->pci_reset_work, pdsc_pci_reset_thread);
244 timer_setup(&pdsc->wdtimer, pdsc_wdtimer_cb, 0);
245 pdsc->wdtimer_period = PDSC_WATCHDOG_SECS * HZ;
246
247 mutex_init(&pdsc->devcmd_lock);
248 mutex_init(&pdsc->config_lock);
249 spin_lock_init(&pdsc->adminq_lock);
250
251 mutex_lock(&pdsc->config_lock);
252 set_bit(PDSC_S_FW_DEAD, &pdsc->state);
253
254 err = pdsc_setup(pdsc, PDSC_SETUP_INIT);
255 if (err) {
256 mutex_unlock(&pdsc->config_lock);
257 goto err_out_unmap_bars;
258 }
259
260 err = pdsc_start(pdsc);
261 if (err) {
262 mutex_unlock(&pdsc->config_lock);
263 goto err_out_teardown;
264 }
265
266 mutex_unlock(&pdsc->config_lock);
267
268 err = pdsc_auxbus_dev_add(pdsc, pdsc, PDS_DEV_TYPE_FWCTL, &pdsc->padev);
269 if (err)
270 goto err_out_stop;
271
272 dl = priv_to_devlink(pdsc);
273 devl_lock(dl);
274 err = devl_params_register(dl, pdsc_dl_params,
275 ARRAY_SIZE(pdsc_dl_params));
276 if (err) {
277 devl_unlock(dl);
278 dev_warn(pdsc->dev, "Failed to register devlink params: %pe\n",
279 ERR_PTR(err));
280 goto err_out_del_dev;
281 }
282
283 hr = devl_health_reporter_create(dl, &pdsc_fw_reporter_ops, 0, pdsc);
284 if (IS_ERR(hr)) {
285 devl_unlock(dl);
286 dev_warn(pdsc->dev, "Failed to create fw reporter: %pe\n", hr);
287 err = PTR_ERR(hr);
288 goto err_out_unreg_params;
289 }
290 pdsc->fw_reporter = hr;
291
292 devl_register(dl);
293 devl_unlock(dl);
294
295 /* Lastly, start the health check timer */
296 mod_timer(&pdsc->wdtimer, round_jiffies(jiffies + pdsc->wdtimer_period));
297
298 return 0;
299
300 err_out_unreg_params:
301 devlink_params_unregister(dl, pdsc_dl_params,
302 ARRAY_SIZE(pdsc_dl_params));
303 err_out_del_dev:
304 pdsc_auxbus_dev_del(pdsc, pdsc, &pdsc->padev);
305 err_out_stop:
306 pdsc_stop(pdsc);
307 err_out_teardown:
308 pdsc_teardown(pdsc, PDSC_TEARDOWN_REMOVING);
309 err_out_unmap_bars:
310 timer_shutdown_sync(&pdsc->wdtimer);
311 if (pdsc->wq)
312 destroy_workqueue(pdsc->wq);
313 mutex_destroy(&pdsc->config_lock);
314 mutex_destroy(&pdsc->devcmd_lock);
315 pci_free_irq_vectors(pdsc->pdev);
316 pdsc_unmap_bars(pdsc);
317 err_out_release_regions:
318 pci_release_regions(pdsc->pdev);
319
320 return err;
321 }
322
323 static const struct devlink_ops pdsc_dl_ops = {
324 .info_get = pdsc_dl_info_get,
325 .flash_update = pdsc_dl_flash_update,
326 };
327
328 static const struct devlink_ops pdsc_dl_vf_ops = {
329 };
330
331 static DEFINE_IDA(pdsc_ida);
332
pdsc_probe(struct pci_dev * pdev,const struct pci_device_id * ent)333 static int pdsc_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
334 {
335 struct device *dev = &pdev->dev;
336 const struct devlink_ops *ops;
337 struct devlink *dl;
338 struct pdsc *pdsc;
339 bool is_pf;
340 int err;
341
342 is_pf = !pdev->is_virtfn;
343 ops = is_pf ? &pdsc_dl_ops : &pdsc_dl_vf_ops;
344 dl = devlink_alloc(ops, sizeof(struct pdsc), dev);
345 if (!dl)
346 return -ENOMEM;
347 pdsc = devlink_priv(dl);
348
349 pdsc->pdev = pdev;
350 pdsc->dev = &pdev->dev;
351 set_bit(PDSC_S_INITING_DRIVER, &pdsc->state);
352 pci_set_drvdata(pdev, pdsc);
353 pdsc_debugfs_add_dev(pdsc);
354
355 err = ida_alloc(&pdsc_ida, GFP_KERNEL);
356 if (err < 0) {
357 dev_err(pdsc->dev, "%s: id alloc failed: %pe\n",
358 __func__, ERR_PTR(err));
359 goto err_out_free_devlink;
360 }
361 pdsc->uid = err;
362
363 /* Query system for DMA addressing limitation for the device. */
364 err = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(PDS_CORE_ADDR_LEN));
365 if (err) {
366 dev_err(dev, "Unable to obtain 64-bit DMA for consistent allocations, aborting: %pe\n",
367 ERR_PTR(err));
368 goto err_out_free_ida;
369 }
370
371 err = pci_enable_device(pdev);
372 if (err) {
373 dev_err(dev, "Cannot enable PCI device: %pe\n", ERR_PTR(err));
374 goto err_out_free_ida;
375 }
376 pci_set_master(pdev);
377
378 if (is_pf)
379 err = pdsc_init_pf(pdsc);
380 else
381 err = pdsc_init_vf(pdsc);
382 if (err) {
383 dev_err(dev, "Cannot init device: %pe\n", ERR_PTR(err));
384 goto err_out_disable_device;
385 }
386
387 clear_bit(PDSC_S_INITING_DRIVER, &pdsc->state);
388 return 0;
389
390 err_out_disable_device:
391 pci_disable_device(pdev);
392 err_out_free_ida:
393 ida_free(&pdsc_ida, pdsc->uid);
394 err_out_free_devlink:
395 pdsc_debugfs_del_dev(pdsc);
396 devlink_free(dl);
397
398 return err;
399 }
400
pdsc_remove(struct pci_dev * pdev)401 static void pdsc_remove(struct pci_dev *pdev)
402 {
403 struct pdsc *pdsc = pci_get_drvdata(pdev);
404 struct devlink *dl;
405
406 /* Unhook the registrations first to be sure there
407 * are no requests while we're stopping.
408 */
409 dl = priv_to_devlink(pdsc);
410 devl_lock(dl);
411 devl_unregister(dl);
412 if (!pdev->is_virtfn) {
413 if (pdsc->fw_reporter) {
414 devl_health_reporter_destroy(pdsc->fw_reporter);
415 pdsc->fw_reporter = NULL;
416 }
417 devl_params_unregister(dl, pdsc_dl_params,
418 ARRAY_SIZE(pdsc_dl_params));
419 }
420 devl_unlock(dl);
421
422 if (pdev->is_virtfn) {
423 struct pdsc *pf;
424
425 pf = pdsc_get_pf_struct(pdsc->pdev);
426 if (!IS_ERR(pf)) {
427 pdsc_auxbus_dev_del(pdsc, pf, &pf->vfs[pdsc->vf_id].padev);
428 pf->vfs[pdsc->vf_id].vf = NULL;
429 }
430 } else {
431 /* Remove the VFs and their aux_bus connections before other
432 * cleanup so that the clients can use the AdminQ to cleanly
433 * shut themselves down.
434 */
435 pdsc_sriov_configure(pdev, 0);
436 pdsc_auxbus_dev_del(pdsc, pdsc, &pdsc->padev);
437
438 timer_shutdown_sync(&pdsc->wdtimer);
439 if (pdsc->wq)
440 destroy_workqueue(pdsc->wq);
441
442 mutex_lock(&pdsc->config_lock);
443 set_bit(PDSC_S_STOPPING_DRIVER, &pdsc->state);
444
445 pdsc_stop(pdsc);
446 pdsc_teardown(pdsc, PDSC_TEARDOWN_REMOVING);
447 mutex_unlock(&pdsc->config_lock);
448 mutex_destroy(&pdsc->config_lock);
449 mutex_destroy(&pdsc->devcmd_lock);
450
451 pdsc_unmap_bars(pdsc);
452 pci_release_regions(pdev);
453 }
454
455 pci_disable_device(pdev);
456
457 ida_free(&pdsc_ida, pdsc->uid);
458 pdsc_debugfs_del_dev(pdsc);
459 devlink_free(dl);
460 }
461
pdsc_stop_health_thread(struct pdsc * pdsc)462 static void pdsc_stop_health_thread(struct pdsc *pdsc)
463 {
464 if (pdsc->pdev->is_virtfn)
465 return;
466
467 timer_shutdown_sync(&pdsc->wdtimer);
468 if (pdsc->health_work.func)
469 cancel_work_sync(&pdsc->health_work);
470 }
471
pdsc_restart_health_thread(struct pdsc * pdsc)472 static void pdsc_restart_health_thread(struct pdsc *pdsc)
473 {
474 if (pdsc->pdev->is_virtfn)
475 return;
476
477 timer_setup(&pdsc->wdtimer, pdsc_wdtimer_cb, 0);
478 mod_timer(&pdsc->wdtimer, jiffies + 1);
479 }
480
pdsc_reset_prepare(struct pci_dev * pdev)481 static void pdsc_reset_prepare(struct pci_dev *pdev)
482 {
483 struct pdsc *pdsc = pci_get_drvdata(pdev);
484
485 pdsc_stop_health_thread(pdsc);
486 pdsc_fw_down(pdsc);
487
488 if (pdev->is_virtfn) {
489 struct pdsc *pf;
490
491 pf = pdsc_get_pf_struct(pdsc->pdev);
492 if (!IS_ERR(pf))
493 pdsc_auxbus_dev_del(pdsc, pf,
494 &pf->vfs[pdsc->vf_id].padev);
495 } else {
496 pdsc_auxbus_dev_del(pdsc, pdsc, &pdsc->padev);
497 }
498
499 pdsc_unmap_bars(pdsc);
500 pci_release_regions(pdev);
501 if (pci_is_enabled(pdev))
502 pci_disable_device(pdev);
503 }
504
pdsc_reset_done(struct pci_dev * pdev)505 static void pdsc_reset_done(struct pci_dev *pdev)
506 {
507 struct pdsc *pdsc = pci_get_drvdata(pdev);
508 struct device *dev = pdsc->dev;
509 int err;
510
511 err = pci_enable_device(pdev);
512 if (err) {
513 dev_err(dev, "Cannot enable PCI device: %pe\n", ERR_PTR(err));
514 return;
515 }
516 pci_set_master(pdev);
517
518 if (!pdev->is_virtfn) {
519 pcie_print_link_status(pdsc->pdev);
520
521 err = pci_request_regions(pdsc->pdev, PDS_CORE_DRV_NAME);
522 if (err) {
523 dev_err(pdsc->dev, "Cannot request PCI regions: %pe\n",
524 ERR_PTR(err));
525 return;
526 }
527
528 err = pdsc_map_bars(pdsc);
529 if (err)
530 return;
531 }
532
533 pdsc_fw_up(pdsc);
534 pdsc_restart_health_thread(pdsc);
535
536 if (pdev->is_virtfn) {
537 struct pdsc *pf;
538
539 pf = pdsc_get_pf_struct(pdsc->pdev);
540 if (!IS_ERR(pf))
541 pdsc_auxbus_dev_add(pdsc, pf, PDS_DEV_TYPE_VDPA,
542 &pf->vfs[pdsc->vf_id].padev);
543 } else {
544 pdsc_auxbus_dev_add(pdsc, pdsc, PDS_DEV_TYPE_FWCTL,
545 &pdsc->padev);
546 }
547 }
548
pdsc_pci_error_detected(struct pci_dev * pdev,pci_channel_state_t error)549 static pci_ers_result_t pdsc_pci_error_detected(struct pci_dev *pdev,
550 pci_channel_state_t error)
551 {
552 if (error == pci_channel_io_frozen) {
553 pdsc_reset_prepare(pdev);
554 return PCI_ERS_RESULT_NEED_RESET;
555 }
556
557 return PCI_ERS_RESULT_NONE;
558 }
559
pdsc_pci_error_resume(struct pci_dev * pdev)560 static void pdsc_pci_error_resume(struct pci_dev *pdev)
561 {
562 struct pdsc *pdsc = pci_get_drvdata(pdev);
563
564 if (test_bit(PDSC_S_FW_DEAD, &pdsc->state))
565 pci_reset_function_locked(pdev);
566 }
567
568 static const struct pci_error_handlers pdsc_err_handler = {
569 /* FLR handling */
570 .reset_prepare = pdsc_reset_prepare,
571 .reset_done = pdsc_reset_done,
572
573 /* AER handling */
574 .error_detected = pdsc_pci_error_detected,
575 .resume = pdsc_pci_error_resume,
576 };
577
578 static struct pci_driver pdsc_driver = {
579 .name = PDS_CORE_DRV_NAME,
580 .id_table = pdsc_id_table,
581 .probe = pdsc_probe,
582 .remove = pdsc_remove,
583 .sriov_configure = pdsc_sriov_configure,
584 .err_handler = &pdsc_err_handler,
585 };
586
pdsc_get_pf_struct(struct pci_dev * vf_pdev)587 void *pdsc_get_pf_struct(struct pci_dev *vf_pdev)
588 {
589 return pci_iov_get_pf_drvdata(vf_pdev, &pdsc_driver);
590 }
591 EXPORT_SYMBOL_GPL(pdsc_get_pf_struct);
592
pdsc_init_module(void)593 static int __init pdsc_init_module(void)
594 {
595 if (strcmp(KBUILD_MODNAME, PDS_CORE_DRV_NAME))
596 return -EINVAL;
597
598 pdsc_debugfs_create();
599 return pci_register_driver(&pdsc_driver);
600 }
601
pdsc_cleanup_module(void)602 static void __exit pdsc_cleanup_module(void)
603 {
604 pci_unregister_driver(&pdsc_driver);
605 pdsc_debugfs_destroy();
606 }
607
608 module_init(pdsc_init_module);
609 module_exit(pdsc_cleanup_module);
610