1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright(c) 2023 Advanced Micro Devices, Inc */
3
4 #include <linux/pci.h>
5 #include <linux/vmalloc.h>
6
7 #include "core.h"
8
9 static BLOCKING_NOTIFIER_HEAD(pds_notify_chain);
10
pdsc_register_notify(struct notifier_block * nb)11 int pdsc_register_notify(struct notifier_block *nb)
12 {
13 return blocking_notifier_chain_register(&pds_notify_chain, nb);
14 }
15 EXPORT_SYMBOL_GPL(pdsc_register_notify);
16
pdsc_unregister_notify(struct notifier_block * nb)17 void pdsc_unregister_notify(struct notifier_block *nb)
18 {
19 blocking_notifier_chain_unregister(&pds_notify_chain, nb);
20 }
21 EXPORT_SYMBOL_GPL(pdsc_unregister_notify);
22
pdsc_notify(unsigned long event,void * data)23 void pdsc_notify(unsigned long event, void *data)
24 {
25 blocking_notifier_call_chain(&pds_notify_chain, event, data);
26 }
27
pdsc_intr_free(struct pdsc * pdsc,int index)28 void pdsc_intr_free(struct pdsc *pdsc, int index)
29 {
30 struct pdsc_intr_info *intr_info;
31
32 if (index >= pdsc->nintrs || index < 0) {
33 WARN(true, "bad intr index %d\n", index);
34 return;
35 }
36
37 intr_info = &pdsc->intr_info[index];
38 if (!intr_info->vector)
39 return;
40 dev_dbg(pdsc->dev, "%s: idx %d vec %d name %s\n",
41 __func__, index, intr_info->vector, intr_info->name);
42
43 pds_core_intr_mask(&pdsc->intr_ctrl[index], PDS_CORE_INTR_MASK_SET);
44 pds_core_intr_clean(&pdsc->intr_ctrl[index]);
45
46 free_irq(intr_info->vector, intr_info->data);
47
48 memset(intr_info, 0, sizeof(*intr_info));
49 }
50
pdsc_intr_alloc(struct pdsc * pdsc,char * name,irq_handler_t handler,void * data)51 int pdsc_intr_alloc(struct pdsc *pdsc, char *name,
52 irq_handler_t handler, void *data)
53 {
54 struct pdsc_intr_info *intr_info;
55 unsigned int index;
56 int err;
57
58 /* Find the first available interrupt */
59 for (index = 0; index < pdsc->nintrs; index++)
60 if (!pdsc->intr_info[index].vector)
61 break;
62 if (index >= pdsc->nintrs) {
63 dev_warn(pdsc->dev, "%s: no intr, index=%d nintrs=%d\n",
64 __func__, index, pdsc->nintrs);
65 return -ENOSPC;
66 }
67
68 pds_core_intr_clean_flags(&pdsc->intr_ctrl[index],
69 PDS_CORE_INTR_CRED_RESET_COALESCE);
70
71 intr_info = &pdsc->intr_info[index];
72
73 intr_info->index = index;
74 intr_info->data = data;
75 strscpy(intr_info->name, name, sizeof(intr_info->name));
76
77 /* Get the OS vector number for the interrupt */
78 err = pci_irq_vector(pdsc->pdev, index);
79 if (err < 0) {
80 dev_err(pdsc->dev, "failed to get intr vector index %d: %pe\n",
81 index, ERR_PTR(err));
82 goto err_out_free_intr;
83 }
84 intr_info->vector = err;
85
86 /* Init the device's intr mask */
87 pds_core_intr_clean(&pdsc->intr_ctrl[index]);
88 pds_core_intr_mask_assert(&pdsc->intr_ctrl[index], 1);
89 pds_core_intr_mask(&pdsc->intr_ctrl[index], PDS_CORE_INTR_MASK_SET);
90
91 /* Register the isr with a name */
92 err = request_irq(intr_info->vector, handler, 0, intr_info->name, data);
93 if (err) {
94 dev_err(pdsc->dev, "failed to get intr irq vector %d: %pe\n",
95 intr_info->vector, ERR_PTR(err));
96 goto err_out_free_intr;
97 }
98
99 return index;
100
101 err_out_free_intr:
102 pdsc_intr_free(pdsc, index);
103 return err;
104 }
105
pdsc_qcq_intr_free(struct pdsc * pdsc,struct pdsc_qcq * qcq)106 static void pdsc_qcq_intr_free(struct pdsc *pdsc, struct pdsc_qcq *qcq)
107 {
108 if (!(qcq->flags & PDS_CORE_QCQ_F_INTR) ||
109 qcq->intx == PDS_CORE_INTR_INDEX_NOT_ASSIGNED)
110 return;
111
112 pdsc_intr_free(pdsc, qcq->intx);
113 }
114
pdsc_qcq_intr_alloc(struct pdsc * pdsc,struct pdsc_qcq * qcq)115 static int pdsc_qcq_intr_alloc(struct pdsc *pdsc, struct pdsc_qcq *qcq)
116 {
117 char name[PDSC_INTR_NAME_MAX_SZ];
118 int index;
119
120 if (!(qcq->flags & PDS_CORE_QCQ_F_INTR)) {
121 qcq->intx = PDS_CORE_INTR_INDEX_NOT_ASSIGNED;
122 return 0;
123 }
124
125 snprintf(name, sizeof(name), "%s-%d-%s",
126 PDS_CORE_DRV_NAME, pdsc->pdev->bus->number, qcq->q.name);
127 index = pdsc_intr_alloc(pdsc, name, pdsc_adminq_isr, pdsc);
128 if (index < 0)
129 return index;
130 qcq->intx = index;
131 qcq->cq.bound_intr = &pdsc->intr_info[index];
132
133 return 0;
134 }
135
pdsc_qcq_free(struct pdsc * pdsc,struct pdsc_qcq * qcq)136 void pdsc_qcq_free(struct pdsc *pdsc, struct pdsc_qcq *qcq)
137 {
138 struct device *dev = pdsc->dev;
139
140 if (!(qcq && qcq->pdsc))
141 return;
142
143 pdsc_debugfs_del_qcq(qcq);
144
145 pdsc_qcq_intr_free(pdsc, qcq);
146
147 /* Drain any work queued by ISR before it was freed above */
148 if (qcq->work.func)
149 cancel_work_sync(&qcq->work);
150
151 qcq->intx = PDS_CORE_INTR_INDEX_NOT_ASSIGNED;
152
153 if (qcq->q_base)
154 dma_free_coherent(dev, qcq->q_size,
155 qcq->q_base, qcq->q_base_pa);
156
157 if (qcq->cq_base)
158 dma_free_coherent(dev, qcq->cq_size,
159 qcq->cq_base, qcq->cq_base_pa);
160
161 vfree(qcq->cq.info);
162 vfree(qcq->q.info);
163
164 memset(qcq, 0, sizeof(*qcq));
165 }
166
pdsc_q_map(struct pdsc_queue * q,void * base,dma_addr_t base_pa)167 static void pdsc_q_map(struct pdsc_queue *q, void *base, dma_addr_t base_pa)
168 {
169 struct pdsc_q_info *cur;
170 unsigned int i;
171
172 q->base = base;
173 q->base_pa = base_pa;
174
175 for (i = 0, cur = q->info; i < q->num_descs; i++, cur++) {
176 cur->desc = base + (i * q->desc_size);
177 init_completion(&cur->completion);
178 }
179 }
180
pdsc_cq_map(struct pdsc_cq * cq,void * base,dma_addr_t base_pa)181 static void pdsc_cq_map(struct pdsc_cq *cq, void *base, dma_addr_t base_pa)
182 {
183 struct pdsc_cq_info *cur;
184 unsigned int i;
185
186 cq->base = base;
187 cq->base_pa = base_pa;
188
189 for (i = 0, cur = cq->info; i < cq->num_descs; i++, cur++)
190 cur->comp = base + (i * cq->desc_size);
191 }
192
pdsc_qcq_alloc(struct pdsc * pdsc,unsigned int type,unsigned int index,const char * name,unsigned int flags,unsigned int num_descs,unsigned int desc_size,unsigned int cq_desc_size,unsigned int pid,struct pdsc_qcq * qcq)193 int pdsc_qcq_alloc(struct pdsc *pdsc, unsigned int type, unsigned int index,
194 const char *name, unsigned int flags, unsigned int num_descs,
195 unsigned int desc_size, unsigned int cq_desc_size,
196 unsigned int pid, struct pdsc_qcq *qcq)
197 {
198 struct device *dev = pdsc->dev;
199 void *q_base, *cq_base;
200 dma_addr_t cq_base_pa;
201 dma_addr_t q_base_pa;
202 int err;
203
204 qcq->q.info = vcalloc(num_descs, sizeof(*qcq->q.info));
205 if (!qcq->q.info) {
206 err = -ENOMEM;
207 goto err_out;
208 }
209
210 qcq->pdsc = pdsc;
211 qcq->flags = flags;
212 INIT_WORK(&qcq->work, pdsc_work_thread);
213
214 qcq->q.type = type;
215 qcq->q.index = index;
216 qcq->q.num_descs = num_descs;
217 qcq->q.desc_size = desc_size;
218 qcq->q.tail_idx = 0;
219 qcq->q.head_idx = 0;
220 qcq->q.pid = pid;
221 snprintf(qcq->q.name, sizeof(qcq->q.name), "%s%u", name, index);
222
223 err = pdsc_qcq_intr_alloc(pdsc, qcq);
224 if (err)
225 goto err_out_free_q_info;
226
227 qcq->cq.info = vcalloc(num_descs, sizeof(*qcq->cq.info));
228 if (!qcq->cq.info) {
229 err = -ENOMEM;
230 goto err_out_free_irq;
231 }
232
233 qcq->cq.num_descs = num_descs;
234 qcq->cq.desc_size = cq_desc_size;
235 qcq->cq.tail_idx = 0;
236 qcq->cq.done_color = 1;
237
238 if (flags & PDS_CORE_QCQ_F_NOTIFYQ) {
239 /* q & cq need to be contiguous in case of notifyq */
240 qcq->q_size = PDS_PAGE_SIZE +
241 ALIGN(num_descs * desc_size, PDS_PAGE_SIZE) +
242 ALIGN(num_descs * cq_desc_size, PDS_PAGE_SIZE);
243 qcq->q_base = dma_alloc_coherent(dev,
244 qcq->q_size + qcq->cq_size,
245 &qcq->q_base_pa,
246 GFP_KERNEL);
247 if (!qcq->q_base) {
248 err = -ENOMEM;
249 goto err_out_free_cq_info;
250 }
251 q_base = PTR_ALIGN(qcq->q_base, PDS_PAGE_SIZE);
252 q_base_pa = ALIGN(qcq->q_base_pa, PDS_PAGE_SIZE);
253 pdsc_q_map(&qcq->q, q_base, q_base_pa);
254
255 cq_base = PTR_ALIGN(q_base +
256 ALIGN(num_descs * desc_size, PDS_PAGE_SIZE),
257 PDS_PAGE_SIZE);
258 cq_base_pa = ALIGN(qcq->q_base_pa +
259 ALIGN(num_descs * desc_size, PDS_PAGE_SIZE),
260 PDS_PAGE_SIZE);
261
262 } else {
263 /* q DMA descriptors */
264 qcq->q_size = PDS_PAGE_SIZE + (num_descs * desc_size);
265 qcq->q_base = dma_alloc_coherent(dev, qcq->q_size,
266 &qcq->q_base_pa,
267 GFP_KERNEL);
268 if (!qcq->q_base) {
269 err = -ENOMEM;
270 goto err_out_free_cq_info;
271 }
272 q_base = PTR_ALIGN(qcq->q_base, PDS_PAGE_SIZE);
273 q_base_pa = ALIGN(qcq->q_base_pa, PDS_PAGE_SIZE);
274 pdsc_q_map(&qcq->q, q_base, q_base_pa);
275
276 /* cq DMA descriptors */
277 qcq->cq_size = PDS_PAGE_SIZE + (num_descs * cq_desc_size);
278 qcq->cq_base = dma_alloc_coherent(dev, qcq->cq_size,
279 &qcq->cq_base_pa,
280 GFP_KERNEL);
281 if (!qcq->cq_base) {
282 err = -ENOMEM;
283 goto err_out_free_q;
284 }
285 cq_base = PTR_ALIGN(qcq->cq_base, PDS_PAGE_SIZE);
286 cq_base_pa = ALIGN(qcq->cq_base_pa, PDS_PAGE_SIZE);
287 }
288
289 pdsc_cq_map(&qcq->cq, cq_base, cq_base_pa);
290 qcq->cq.bound_q = &qcq->q;
291
292 pdsc_debugfs_add_qcq(pdsc, qcq);
293
294 return 0;
295
296 err_out_free_q:
297 dma_free_coherent(dev, qcq->q_size, qcq->q_base, qcq->q_base_pa);
298 err_out_free_cq_info:
299 vfree(qcq->cq.info);
300 err_out_free_irq:
301 pdsc_qcq_intr_free(pdsc, qcq);
302 err_out_free_q_info:
303 vfree(qcq->q.info);
304 memset(qcq, 0, sizeof(*qcq));
305 err_out:
306 dev_err(dev, "qcq alloc of %s%d failed %d\n", name, index, err);
307 return err;
308 }
309
pdsc_core_uninit(struct pdsc * pdsc)310 static void pdsc_core_uninit(struct pdsc *pdsc)
311 {
312 /* Free adminqcq first: its work accesses notifyqcq, so we must
313 * disable its IRQ and drain its work before freeing notifyqcq.
314 */
315 pdsc_qcq_free(pdsc, &pdsc->adminqcq);
316 pdsc_qcq_free(pdsc, &pdsc->notifyqcq);
317
318 if (pdsc->kern_dbpage) {
319 iounmap(pdsc->kern_dbpage);
320 pdsc->kern_dbpage = NULL;
321 }
322 }
323
pdsc_core_init(struct pdsc * pdsc)324 static int pdsc_core_init(struct pdsc *pdsc)
325 {
326 union pds_core_dev_comp comp = {};
327 union pds_core_dev_cmd cmd = {
328 .init.opcode = PDS_CORE_CMD_INIT,
329 };
330 struct pds_core_dev_init_data_out cido;
331 struct pds_core_dev_init_data_in cidi;
332 u32 dbid_count;
333 u32 dbpage_num;
334 int numdescs;
335 size_t sz;
336 int err;
337
338 numdescs = PDSC_ADMINQ_MAX_LENGTH;
339 err = pdsc_qcq_alloc(pdsc, PDS_CORE_QTYPE_ADMINQ, 0, "adminq",
340 PDS_CORE_QCQ_F_CORE | PDS_CORE_QCQ_F_INTR,
341 numdescs,
342 sizeof(union pds_core_adminq_cmd),
343 sizeof(union pds_core_adminq_comp),
344 0, &pdsc->adminqcq);
345 if (err)
346 return err;
347
348 err = pdsc_qcq_alloc(pdsc, PDS_CORE_QTYPE_NOTIFYQ, 0, "notifyq",
349 PDS_CORE_QCQ_F_NOTIFYQ,
350 PDSC_NOTIFYQ_LENGTH,
351 sizeof(struct pds_core_notifyq_cmd),
352 sizeof(union pds_core_notifyq_comp),
353 0, &pdsc->notifyqcq);
354 if (err)
355 goto err_out_uninit;
356
357 cidi.adminq_q_base = cpu_to_le64(pdsc->adminqcq.q_base_pa);
358 cidi.adminq_cq_base = cpu_to_le64(pdsc->adminqcq.cq_base_pa);
359 cidi.notifyq_cq_base = cpu_to_le64(pdsc->notifyqcq.cq.base_pa);
360 cidi.flags = cpu_to_le32(PDS_CORE_QINIT_F_IRQ | PDS_CORE_QINIT_F_ENA);
361 cidi.intr_index = cpu_to_le16(pdsc->adminqcq.intx);
362 cidi.adminq_ring_size = ilog2(pdsc->adminqcq.q.num_descs);
363 cidi.notifyq_ring_size = ilog2(pdsc->notifyqcq.q.num_descs);
364
365 mutex_lock(&pdsc->devcmd_lock);
366
367 sz = min_t(size_t, sizeof(cidi), sizeof(pdsc->cmd_regs->data));
368 memcpy_toio(&pdsc->cmd_regs->data, &cidi, sz);
369
370 err = pdsc_devcmd_locked(pdsc, &cmd, &comp, pdsc->devcmd_timeout);
371 if (!err) {
372 sz = min_t(size_t, sizeof(cido), sizeof(pdsc->cmd_regs->data));
373 memcpy_fromio(&cido, &pdsc->cmd_regs->data, sz);
374 }
375
376 mutex_unlock(&pdsc->devcmd_lock);
377 if (err) {
378 dev_err(pdsc->dev, "Device init command failed: %pe\n",
379 ERR_PTR(err));
380 goto err_out_uninit;
381 }
382
383 pdsc->hw_index = le32_to_cpu(cido.core_hw_index);
384
385 dbid_count = le32_to_cpu(pdsc->dev_ident.ndbpgs_per_lif);
386 dbpage_num = pdsc->hw_index * dbid_count;
387 pdsc->kern_dbpage = pdsc_map_dbpage(pdsc, dbpage_num);
388 if (!pdsc->kern_dbpage) {
389 dev_err(pdsc->dev, "Cannot map dbpage, aborting\n");
390 err = -ENOMEM;
391 goto err_out_uninit;
392 }
393
394 pdsc->adminqcq.q.hw_type = cido.adminq_hw_type;
395 pdsc->adminqcq.q.hw_index = le32_to_cpu(cido.adminq_hw_index);
396 pdsc->adminqcq.q.dbval = PDS_CORE_DBELL_QID(pdsc->adminqcq.q.hw_index);
397
398 pdsc->notifyqcq.q.hw_type = cido.notifyq_hw_type;
399 pdsc->notifyqcq.q.hw_index = le32_to_cpu(cido.notifyq_hw_index);
400 pdsc->notifyqcq.q.dbval = PDS_CORE_DBELL_QID(pdsc->notifyqcq.q.hw_index);
401
402 pdsc->last_eid = 0;
403
404 return 0;
405
406 err_out_uninit:
407 pdsc_core_uninit(pdsc);
408 return err;
409 }
410
411 static struct pdsc_viftype pdsc_viftype_defaults[] = {
412 [PDS_DEV_TYPE_FWCTL] = { .name = PDS_DEV_TYPE_FWCTL_STR,
413 .enabled = true,
414 .vif_id = PDS_DEV_TYPE_FWCTL,
415 .dl_id = -1 },
416 [PDS_DEV_TYPE_VDPA] = { .name = PDS_DEV_TYPE_VDPA_STR,
417 .vif_id = PDS_DEV_TYPE_VDPA,
418 .dl_id = DEVLINK_PARAM_GENERIC_ID_ENABLE_VNET },
419 [PDS_DEV_TYPE_MAX] = {}
420 };
421
pdsc_viftypes_init(struct pdsc * pdsc)422 static int pdsc_viftypes_init(struct pdsc *pdsc)
423 {
424 enum pds_core_vif_types vt;
425
426 pdsc->viftype_status = kzalloc_objs(*pdsc->viftype_status,
427 ARRAY_SIZE(pdsc_viftype_defaults));
428 if (!pdsc->viftype_status)
429 return -ENOMEM;
430
431 for (vt = 0; vt < PDS_DEV_TYPE_MAX; vt++) {
432 bool vt_support;
433
434 if (!pdsc_viftype_defaults[vt].name)
435 continue;
436
437 /* Grab the defaults */
438 pdsc->viftype_status[vt] = pdsc_viftype_defaults[vt];
439
440 /* See what the Core device has for support */
441 vt_support = !!le16_to_cpu(pdsc->dev_ident.vif_types[vt]);
442
443 dev_dbg(pdsc->dev, "VIF %s is %ssupported\n",
444 pdsc->viftype_status[vt].name,
445 vt_support ? "" : "not ");
446
447 pdsc->viftype_status[vt].supported = vt_support;
448 }
449
450 return 0;
451 }
452
pdsc_setup(struct pdsc * pdsc,bool init)453 int pdsc_setup(struct pdsc *pdsc, bool init)
454 {
455 int err;
456
457 pci_set_master(pdsc->pdev);
458
459 err = pdsc_dev_init(pdsc);
460 if (err)
461 return err;
462
463 /* Set up the Core with the AdminQ and NotifyQ info */
464 err = pdsc_core_init(pdsc);
465 if (err)
466 goto err_out_teardown;
467
468 /* Set up the VIFs */
469 if (init) {
470 err = pdsc_viftypes_init(pdsc);
471 if (err)
472 goto err_out_teardown;
473
474 pdsc_debugfs_add_viftype(pdsc);
475 }
476
477 refcount_set(&pdsc->adminq_refcnt, 1);
478 clear_bit(PDSC_S_FW_DEAD, &pdsc->state);
479 return 0;
480
481 err_out_teardown:
482 pdsc_teardown(pdsc, init);
483 return err;
484 }
485
pdsc_teardown(struct pdsc * pdsc,bool removing)486 void pdsc_teardown(struct pdsc *pdsc, bool removing)
487 {
488 if (!pdsc->pdev->is_virtfn)
489 pdsc_devcmd_reset(pdsc);
490
491 pci_clear_master(pdsc->pdev);
492 if (!pdsc->pdev->is_virtfn) {
493 u16 val;
494
495 /* Flush any in-flight DMA before freeing buffers.
496 * A config read completion cannot return until all prior
497 * device-initiated memory writes have completed.
498 */
499 pci_read_config_word(pdsc->pdev, PCI_VENDOR_ID, &val);
500 }
501
502 pdsc_core_uninit(pdsc);
503
504 if (removing) {
505 kfree(pdsc->viftype_status);
506 pdsc->viftype_status = NULL;
507 }
508
509 pdsc_debugfs_del_host_mem(pdsc);
510 pdsc_host_mem_free(pdsc);
511 pdsc_dev_uninit(pdsc);
512
513 set_bit(PDSC_S_FW_DEAD, &pdsc->state);
514 }
515
pdsc_start(struct pdsc * pdsc)516 int pdsc_start(struct pdsc *pdsc)
517 {
518 pds_core_intr_mask(&pdsc->intr_ctrl[pdsc->adminqcq.intx],
519 PDS_CORE_INTR_MASK_CLEAR);
520 pdsc_host_mem_add(pdsc);
521 pdsc_debugfs_add_host_mem(pdsc);
522
523 return 0;
524 }
525
pdsc_stop(struct pdsc * pdsc)526 void pdsc_stop(struct pdsc *pdsc)
527 {
528 int i;
529
530 if (!pdsc->intr_info)
531 return;
532
533 /* Mask interrupts that are in use */
534 for (i = 0; i < pdsc->nintrs; i++)
535 if (pdsc->intr_info[i].vector)
536 pds_core_intr_mask(&pdsc->intr_ctrl[i],
537 PDS_CORE_INTR_MASK_SET);
538 }
539
pdsc_adminq_wait_and_dec_once_unused(struct pdsc * pdsc)540 static void pdsc_adminq_wait_and_dec_once_unused(struct pdsc *pdsc)
541 {
542 /* The driver initializes the adminq_refcnt to 1 when the adminq is
543 * allocated and ready for use. Other users/requesters will increment
544 * the refcnt while in use. If the refcnt is down to 1 then the adminq
545 * is not in use and the refcnt can be cleared and adminq freed. Before
546 * calling this function the driver will set PDSC_S_FW_DEAD, which
547 * prevent subsequent attempts to use the adminq and increment the
548 * refcnt to fail. This guarantees that this function will eventually
549 * exit.
550 */
551 while (!refcount_dec_if_one(&pdsc->adminq_refcnt)) {
552 dev_dbg_ratelimited(pdsc->dev, "%s: adminq in use\n",
553 __func__);
554 cpu_relax();
555 cond_resched();
556 }
557 }
558
pdsc_fw_down(struct pdsc * pdsc)559 void pdsc_fw_down(struct pdsc *pdsc)
560 {
561 union pds_core_notifyq_comp reset_event = {
562 .reset.ecode = cpu_to_le16(PDS_EVENT_RESET),
563 .reset.state = 0,
564 };
565
566 if (test_and_set_bit(PDSC_S_FW_DEAD, &pdsc->state)) {
567 dev_warn(pdsc->dev, "%s: already happening\n", __func__);
568 return;
569 }
570
571 if (pdsc->pdev->is_virtfn)
572 return;
573
574 pdsc_adminq_wait_and_dec_once_unused(pdsc);
575
576 /* Notify clients of fw_down */
577 if (pdsc->fw_reporter)
578 devlink_health_report(pdsc->fw_reporter, "FW down reported", pdsc);
579 pdsc_notify(PDS_EVENT_RESET, &reset_event);
580
581 pdsc_stop(pdsc);
582 pdsc_teardown(pdsc, PDSC_TEARDOWN_RECOVERY);
583 }
584
pdsc_fw_up(struct pdsc * pdsc)585 void pdsc_fw_up(struct pdsc *pdsc)
586 {
587 union pds_core_notifyq_comp reset_event = {
588 .reset.ecode = cpu_to_le16(PDS_EVENT_RESET),
589 .reset.state = 1,
590 };
591 int err;
592
593 if (!test_bit(PDSC_S_FW_DEAD, &pdsc->state)) {
594 dev_err(pdsc->dev, "%s: fw not dead\n", __func__);
595 return;
596 }
597
598 if (pdsc->pdev->is_virtfn) {
599 clear_bit(PDSC_S_FW_DEAD, &pdsc->state);
600 return;
601 }
602
603 pdsc_fw_components_invalidate(pdsc);
604
605 err = pdsc_setup(pdsc, PDSC_SETUP_RECOVERY);
606 if (err)
607 goto err_out;
608
609 err = pdsc_start(pdsc);
610 if (err)
611 goto err_out;
612
613 /* Notify clients of fw_up */
614 pdsc->fw_recoveries++;
615 if (pdsc->fw_reporter)
616 devlink_health_reporter_state_update(pdsc->fw_reporter,
617 DEVLINK_HEALTH_REPORTER_STATE_HEALTHY);
618 pdsc_notify(PDS_EVENT_RESET, &reset_event);
619
620 return;
621
622 err_out:
623 pdsc_teardown(pdsc, PDSC_TEARDOWN_RECOVERY);
624 }
625
pdsc_pci_reset_thread(struct work_struct * work)626 void pdsc_pci_reset_thread(struct work_struct *work)
627 {
628 struct pdsc *pdsc = container_of(work, struct pdsc, pci_reset_work);
629 struct pci_dev *pdev = pdsc->pdev;
630
631 /* Use try variant to avoid deadlock with pdsc_remove().
632 * If lock is contended, the watchdog timer will retry.
633 */
634 pci_try_reset_function(pdev);
635 }
636
pdsc_check_pci_health(struct pdsc * pdsc)637 static void pdsc_check_pci_health(struct pdsc *pdsc)
638 {
639 u8 fw_status;
640
641 /* some sort of teardown already in progress */
642 if (!pdsc->info_regs)
643 return;
644
645 fw_status = ioread8(&pdsc->info_regs->fw_status);
646
647 /* is PCI broken? */
648 if (fw_status != PDS_RC_BAD_PCI)
649 return;
650
651 /* prevent deadlock between pdsc_reset_prepare and pdsc_health_thread */
652 queue_work(pdsc->wq, &pdsc->pci_reset_work);
653 }
654
pdsc_health_thread(struct work_struct * work)655 void pdsc_health_thread(struct work_struct *work)
656 {
657 struct pdsc *pdsc = container_of(work, struct pdsc, health_work);
658 unsigned long mask;
659 bool healthy;
660
661 mutex_lock(&pdsc->config_lock);
662
663 /* Don't do a check when in a transition state */
664 mask = BIT_ULL(PDSC_S_INITING_DRIVER) |
665 BIT_ULL(PDSC_S_STOPPING_DRIVER);
666 if (pdsc->state & mask)
667 goto out_unlock;
668
669 healthy = pdsc_is_fw_good(pdsc);
670 dev_dbg(pdsc->dev, "%s: health %d fw_status %#02x fw_heartbeat %d\n",
671 __func__, healthy, pdsc->fw_status, pdsc->last_hb);
672
673 if (test_bit(PDSC_S_FW_DEAD, &pdsc->state)) {
674 if (healthy)
675 pdsc_fw_up(pdsc);
676 } else {
677 if (!healthy)
678 pdsc_fw_down(pdsc);
679 }
680
681 pdsc_check_pci_health(pdsc);
682
683 pdsc->fw_generation = pdsc->fw_status & PDS_CORE_FW_STS_F_GENERATION;
684
685 out_unlock:
686 mutex_unlock(&pdsc->config_lock);
687 }
688
pdsc_host_mem_del_one(struct pdsc * pdsc,u16 tag,u8 reason)689 static void pdsc_host_mem_del_one(struct pdsc *pdsc, u16 tag, u8 reason)
690 {
691 union pds_core_dev_comp comp = {};
692 union pds_core_dev_cmd cmd = {
693 .host_mem.opcode = PDS_CORE_CMD_HOST_MEM,
694 .host_mem.oper = PDS_CORE_HOST_MEM_DEL,
695 .host_mem.tag = cpu_to_le16(tag),
696 .host_mem.reason = reason,
697 };
698
699 dev_dbg(pdsc->dev, "Sending devcmd for mem del tag %d\n", tag);
700 pdsc_devcmd(pdsc, &cmd, &comp, pdsc->devcmd_timeout);
701 }
702
pdsc_host_mem_add_one(struct pdsc * pdsc,int index)703 static int pdsc_host_mem_add_one(struct pdsc *pdsc, int index)
704 {
705 struct pdsc_host_mem *hm = &pdsc->host_mem_reqs[index];
706 union pds_core_dev_comp comp = {};
707 union pds_core_dev_cmd cmd = {};
708 int err;
709
710 cmd.host_mem.opcode = PDS_CORE_CMD_HOST_MEM;
711 cmd.host_mem.oper = PDS_CORE_HOST_MEM_QUERY;
712 cmd.host_mem.index = cpu_to_le16(index);
713 dev_dbg(pdsc->dev, "Sending devcmd for mem query index %d\n", index);
714 err = pdsc_devcmd(pdsc, &cmd, &comp, pdsc->devcmd_timeout);
715 if (err || comp.status != PDS_RC_SUCCESS) {
716 dev_err(pdsc->dev, "mem query failed err %d status %d\n",
717 err, comp.status);
718 return err ? err : -EIO;
719 }
720 hm->size = le32_to_cpu(comp.host_mem.size);
721 hm->tag = le16_to_cpu(comp.host_mem.tag);
722 dev_dbg(pdsc->dev, "mem query returned size %d tag %d\n",
723 hm->size, hm->tag);
724
725 if (!hm->size || hm->size > PDSC_HOST_MEM_MAX_CONTIG) {
726 dev_err(pdsc->dev, "invalid size %d for tag %d\n",
727 hm->size, hm->tag);
728 err = -EINVAL;
729 goto err_del;
730 }
731
732 hm->order = get_order(hm->size);
733 hm->pg = alloc_pages(GFP_KERNEL | __GFP_ZERO | __GFP_NOWARN, hm->order);
734 if (!hm->pg) {
735 dev_warn(pdsc->dev, "alloc order %d failed for tag %d\n",
736 hm->order, hm->tag);
737 err = -ENOMEM;
738 goto err_del;
739 }
740
741 hm->pa = dma_map_page(pdsc->dev, hm->pg, 0, hm->size,
742 DMA_BIDIRECTIONAL);
743 if (dma_mapping_error(pdsc->dev, hm->pa)) {
744 dev_err(pdsc->dev, "dma map failed for tag %d size %d\n",
745 hm->tag, hm->size);
746 __free_pages(hm->pg, hm->order);
747 hm->pg = NULL;
748 err = -EIO;
749 goto err_del;
750 }
751
752 /* Track this allocation so pdsc_host_mem_free() can clean it up */
753 pdsc->num_host_mem_reqs++;
754
755 memset(&cmd, 0, sizeof(cmd));
756 memset(&comp, 0, sizeof(comp));
757 cmd.host_mem.opcode = PDS_CORE_CMD_HOST_MEM;
758 cmd.host_mem.oper = PDS_CORE_HOST_MEM_ADD;
759 cmd.host_mem.tag = cpu_to_le16(hm->tag);
760 cmd.host_mem.size = cpu_to_le32(hm->size);
761 cmd.host_mem.buf_pa = cpu_to_le64(hm->pa);
762
763 dev_dbg(pdsc->dev, "Sending devcmd for mem add tag %d size %d pa %pad\n",
764 hm->tag, hm->size, &hm->pa);
765 err = pdsc_devcmd(pdsc, &cmd, &comp, pdsc->devcmd_timeout);
766 if (err || comp.status != PDS_RC_SUCCESS) {
767 dev_err(pdsc->dev, "mem add failed err %d status %d for tag %d\n",
768 err, comp.status, hm->tag);
769 err = err ? err : -EIO;
770 goto err_del;
771 }
772 dev_dbg(pdsc->dev, "mem add completed for tag %d\n", hm->tag);
773
774 return 0;
775
776 err_del:
777 /* After MEM_QUERY succeeds, firmware expects MEM_ADD or MEM_DEL */
778 pdsc_host_mem_del_one(pdsc, hm->tag, PDS_RC_ENOMEM);
779 return err;
780 }
781
pdsc_host_mem_add(struct pdsc * pdsc)782 void pdsc_host_mem_add(struct pdsc *pdsc)
783 {
784 union pds_core_dev_comp comp = {};
785 union pds_core_dev_cmd cmd = {};
786 u16 count;
787 int err;
788 int i;
789
790 if (!(pdsc->dev_ident.capabilities &
791 cpu_to_le64(PDS_CORE_DEV_CAP_HOST_MEM)))
792 return;
793
794 cmd.host_mem.opcode = PDS_CORE_CMD_HOST_MEM;
795 cmd.host_mem.oper = PDS_CORE_HOST_MEM_GET_COUNT;
796 cmd.host_mem.index = cpu_to_le16(PDSC_HOST_MEM_MAX_COUNT);
797 cmd.host_mem.max_contig = cpu_to_le32(PDSC_HOST_MEM_MAX_CONTIG);
798 dev_dbg(pdsc->dev, "Sending devcmd for mem get count max_contig %u\n",
799 PDSC_HOST_MEM_MAX_CONTIG);
800 err = pdsc_devcmd(pdsc, &cmd, &comp, pdsc->devcmd_timeout);
801 if (err || comp.status != PDS_RC_SUCCESS) {
802 dev_err(pdsc->dev, "mem get count failed err %d status %d\n",
803 err, comp.status);
804 return;
805 }
806
807 count = min(le16_to_cpu(comp.host_mem.count),
808 PDSC_HOST_MEM_MAX_COUNT);
809 dev_dbg(pdsc->dev, "mem get count returned count %d\n", count);
810 if (count == 0)
811 return;
812
813 pdsc->host_mem_reqs = kzalloc_objs(*pdsc->host_mem_reqs, count,
814 GFP_KERNEL);
815 if (!pdsc->host_mem_reqs) {
816 dev_err(pdsc->dev, "failed to alloc host_mem_reqs array\n");
817 return;
818 }
819
820 for (i = 0; i < count; i++) {
821 err = pdsc_host_mem_add_one(pdsc, i);
822 if (err)
823 break;
824 }
825 }
826
pdsc_host_mem_free(struct pdsc * pdsc)827 void pdsc_host_mem_free(struct pdsc *pdsc)
828 {
829 int i;
830
831 if (!pdsc->host_mem_reqs)
832 return;
833
834 for (i = 0; i < pdsc->num_host_mem_reqs; i++) {
835 dma_unmap_page(pdsc->dev, pdsc->host_mem_reqs[i].pa,
836 pdsc->host_mem_reqs[i].size,
837 DMA_BIDIRECTIONAL);
838 __free_pages(pdsc->host_mem_reqs[i].pg,
839 pdsc->host_mem_reqs[i].order);
840 }
841
842 kfree(pdsc->host_mem_reqs);
843 pdsc->host_mem_reqs = NULL;
844 pdsc->num_host_mem_reqs = 0;
845 }
846