1 // SPDX-License-Identifier: GPL-2.0
2 /* Marvell RVU Physical Function ethernet driver
3 *
4 * Copyright (C) 2020 Marvell.
5 *
6 */
7
8 #include <linux/module.h>
9 #include <linux/interrupt.h>
10 #include <linux/pci.h>
11 #include <linux/etherdevice.h>
12 #include <linux/of.h>
13 #include <linux/if_vlan.h>
14 #include <linux/iommu.h>
15 #include <net/ip.h>
16 #include <linux/bpf.h>
17 #include <linux/bpf_trace.h>
18 #include <linux/bitfield.h>
19 #include <net/page_pool/types.h>
20
21 #include "otx2_reg.h"
22 #include "otx2_common.h"
23 #include "otx2_txrx.h"
24 #include "otx2_struct.h"
25 #include "otx2_ptp.h"
26 #include "cn10k.h"
27 #include "qos.h"
28 #include <rvu_trace.h>
29 #include "cn10k_ipsec.h"
30 #include "otx2_xsk.h"
31
32 #define DRV_NAME "rvu_nicpf"
33 #define DRV_STRING "Marvell RVU NIC Physical Function Driver"
34
35 /* Supported devices */
36 static const struct pci_device_id otx2_pf_id_table[] = {
37 { PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, PCI_DEVID_OCTEONTX2_RVU_PF) },
38 { 0, } /* end of table */
39 };
40
41 MODULE_AUTHOR("Sunil Goutham <sgoutham@marvell.com>");
42 MODULE_DESCRIPTION(DRV_STRING);
43 MODULE_LICENSE("GPL v2");
44 MODULE_DEVICE_TABLE(pci, otx2_pf_id_table);
45
46 static void otx2_vf_link_event_task(struct work_struct *work);
47
48 enum {
49 TYPE_PFAF,
50 TYPE_PFVF,
51 };
52
53 static int otx2_config_hw_tx_tstamp(struct otx2_nic *pfvf, bool enable);
54 static int otx2_config_hw_rx_tstamp(struct otx2_nic *pfvf, bool enable);
55
otx2_change_mtu(struct net_device * netdev,int new_mtu)56 static int otx2_change_mtu(struct net_device *netdev, int new_mtu)
57 {
58 struct otx2_nic *pf = netdev_priv(netdev);
59 bool if_up = netif_running(netdev);
60 int err = 0;
61
62 if (pf->xdp_prog && new_mtu > MAX_XDP_MTU) {
63 netdev_warn(netdev, "Jumbo frames not yet supported with XDP, current MTU %d.\n",
64 netdev->mtu);
65 return -EINVAL;
66 }
67 if (if_up)
68 otx2_stop(netdev);
69
70 netdev_info(netdev, "Changing MTU from %d to %d\n",
71 netdev->mtu, new_mtu);
72 WRITE_ONCE(netdev->mtu, new_mtu);
73
74 if (if_up)
75 err = otx2_open(netdev);
76
77 return err;
78 }
79
otx2_disable_flr_me_intr(struct otx2_nic * pf)80 static void otx2_disable_flr_me_intr(struct otx2_nic *pf)
81 {
82 int irq, vfs = pf->total_vfs;
83
84 /* Disable VFs ME interrupts */
85 otx2_write64(pf, RVU_PF_VFME_INT_ENA_W1CX(0), INTR_MASK(vfs));
86 irq = pci_irq_vector(pf->pdev, RVU_PF_INT_VEC_VFME0);
87 free_irq(irq, pf);
88
89 /* Disable VFs FLR interrupts */
90 otx2_write64(pf, RVU_PF_VFFLR_INT_ENA_W1CX(0), INTR_MASK(vfs));
91 irq = pci_irq_vector(pf->pdev, RVU_PF_INT_VEC_VFFLR0);
92 free_irq(irq, pf);
93
94 if (vfs <= 64)
95 return;
96
97 otx2_write64(pf, RVU_PF_VFME_INT_ENA_W1CX(1), INTR_MASK(vfs - 64));
98 irq = pci_irq_vector(pf->pdev, RVU_PF_INT_VEC_VFME1);
99 free_irq(irq, pf);
100
101 otx2_write64(pf, RVU_PF_VFFLR_INT_ENA_W1CX(1), INTR_MASK(vfs - 64));
102 irq = pci_irq_vector(pf->pdev, RVU_PF_INT_VEC_VFFLR1);
103 free_irq(irq, pf);
104 }
105
otx2_flr_wq_destroy(struct otx2_nic * pf)106 static void otx2_flr_wq_destroy(struct otx2_nic *pf)
107 {
108 if (!pf->flr_wq)
109 return;
110 destroy_workqueue(pf->flr_wq);
111 pf->flr_wq = NULL;
112 devm_kfree(pf->dev, pf->flr_wrk);
113 }
114
otx2_flr_handler(struct work_struct * work)115 static void otx2_flr_handler(struct work_struct *work)
116 {
117 struct flr_work *flrwork = container_of(work, struct flr_work, work);
118 struct otx2_nic *pf = flrwork->pf;
119 struct mbox *mbox = &pf->mbox;
120 struct msg_req *req;
121 int vf, reg = 0;
122
123 vf = flrwork - pf->flr_wrk;
124
125 mutex_lock(&mbox->lock);
126 req = otx2_mbox_alloc_msg_vf_flr(mbox);
127 if (!req) {
128 mutex_unlock(&mbox->lock);
129 return;
130 }
131 req->hdr.pcifunc &= RVU_PFVF_FUNC_MASK;
132 req->hdr.pcifunc |= (vf + 1) & RVU_PFVF_FUNC_MASK;
133
134 if (!otx2_sync_mbox_msg(&pf->mbox)) {
135 if (vf >= 64) {
136 reg = 1;
137 vf = vf - 64;
138 }
139 /* clear transcation pending bit */
140 otx2_write64(pf, RVU_PF_VFTRPENDX(reg), BIT_ULL(vf));
141 otx2_write64(pf, RVU_PF_VFFLR_INT_ENA_W1SX(reg), BIT_ULL(vf));
142 }
143
144 mutex_unlock(&mbox->lock);
145 }
146
otx2_pf_flr_intr_handler(int irq,void * pf_irq)147 static irqreturn_t otx2_pf_flr_intr_handler(int irq, void *pf_irq)
148 {
149 struct otx2_nic *pf = (struct otx2_nic *)pf_irq;
150 int reg, dev, vf, start_vf, num_reg = 1;
151 u64 intr;
152
153 if (pf->total_vfs > 64)
154 num_reg = 2;
155
156 for (reg = 0; reg < num_reg; reg++) {
157 intr = otx2_read64(pf, RVU_PF_VFFLR_INTX(reg));
158 if (!intr)
159 continue;
160 start_vf = 64 * reg;
161 for (vf = 0; vf < 64; vf++) {
162 if (!(intr & BIT_ULL(vf)))
163 continue;
164 dev = vf + start_vf;
165 queue_work(pf->flr_wq, &pf->flr_wrk[dev].work);
166 /* Clear interrupt */
167 otx2_write64(pf, RVU_PF_VFFLR_INTX(reg), BIT_ULL(vf));
168 /* Disable the interrupt */
169 otx2_write64(pf, RVU_PF_VFFLR_INT_ENA_W1CX(reg),
170 BIT_ULL(vf));
171 }
172 }
173 return IRQ_HANDLED;
174 }
175
otx2_pf_me_intr_handler(int irq,void * pf_irq)176 static irqreturn_t otx2_pf_me_intr_handler(int irq, void *pf_irq)
177 {
178 struct otx2_nic *pf = (struct otx2_nic *)pf_irq;
179 int vf, reg, num_reg = 1;
180 u64 intr;
181
182 if (pf->total_vfs > 64)
183 num_reg = 2;
184
185 for (reg = 0; reg < num_reg; reg++) {
186 intr = otx2_read64(pf, RVU_PF_VFME_INTX(reg));
187 if (!intr)
188 continue;
189 for (vf = 0; vf < 64; vf++) {
190 if (!(intr & BIT_ULL(vf)))
191 continue;
192 /* clear trpend bit */
193 otx2_write64(pf, RVU_PF_VFTRPENDX(reg), BIT_ULL(vf));
194 /* clear interrupt */
195 otx2_write64(pf, RVU_PF_VFME_INTX(reg), BIT_ULL(vf));
196 }
197 }
198 return IRQ_HANDLED;
199 }
200
otx2_register_flr_me_intr(struct otx2_nic * pf,int numvfs)201 static int otx2_register_flr_me_intr(struct otx2_nic *pf, int numvfs)
202 {
203 struct otx2_hw *hw = &pf->hw;
204 char *irq_name;
205 int ret;
206
207 /* Register ME interrupt handler*/
208 irq_name = &hw->irq_name[RVU_PF_INT_VEC_VFME0 * NAME_SIZE];
209 snprintf(irq_name, NAME_SIZE, "RVUPF%d_ME0",
210 rvu_get_pf(pf->pdev, pf->pcifunc));
211 ret = request_irq(pci_irq_vector(pf->pdev, RVU_PF_INT_VEC_VFME0),
212 otx2_pf_me_intr_handler, 0, irq_name, pf);
213 if (ret) {
214 dev_err(pf->dev,
215 "RVUPF: IRQ registration failed for ME0\n");
216 }
217
218 /* Register FLR interrupt handler */
219 irq_name = &hw->irq_name[RVU_PF_INT_VEC_VFFLR0 * NAME_SIZE];
220 snprintf(irq_name, NAME_SIZE, "RVUPF%d_FLR0",
221 rvu_get_pf(pf->pdev, pf->pcifunc));
222 ret = request_irq(pci_irq_vector(pf->pdev, RVU_PF_INT_VEC_VFFLR0),
223 otx2_pf_flr_intr_handler, 0, irq_name, pf);
224 if (ret) {
225 dev_err(pf->dev,
226 "RVUPF: IRQ registration failed for FLR0\n");
227 return ret;
228 }
229
230 if (numvfs > 64) {
231 irq_name = &hw->irq_name[RVU_PF_INT_VEC_VFME1 * NAME_SIZE];
232 snprintf(irq_name, NAME_SIZE, "RVUPF%d_ME1",
233 rvu_get_pf(pf->pdev, pf->pcifunc));
234 ret = request_irq(pci_irq_vector
235 (pf->pdev, RVU_PF_INT_VEC_VFME1),
236 otx2_pf_me_intr_handler, 0, irq_name, pf);
237 if (ret) {
238 dev_err(pf->dev,
239 "RVUPF: IRQ registration failed for ME1\n");
240 }
241 irq_name = &hw->irq_name[RVU_PF_INT_VEC_VFFLR1 * NAME_SIZE];
242 snprintf(irq_name, NAME_SIZE, "RVUPF%d_FLR1",
243 rvu_get_pf(pf->pdev, pf->pcifunc));
244 ret = request_irq(pci_irq_vector
245 (pf->pdev, RVU_PF_INT_VEC_VFFLR1),
246 otx2_pf_flr_intr_handler, 0, irq_name, pf);
247 if (ret) {
248 dev_err(pf->dev,
249 "RVUPF: IRQ registration failed for FLR1\n");
250 return ret;
251 }
252 }
253
254 /* Enable ME interrupt for all VFs*/
255 otx2_write64(pf, RVU_PF_VFME_INTX(0), INTR_MASK(numvfs));
256 otx2_write64(pf, RVU_PF_VFME_INT_ENA_W1SX(0), INTR_MASK(numvfs));
257
258 /* Enable FLR interrupt for all VFs*/
259 otx2_write64(pf, RVU_PF_VFFLR_INTX(0), INTR_MASK(numvfs));
260 otx2_write64(pf, RVU_PF_VFFLR_INT_ENA_W1SX(0), INTR_MASK(numvfs));
261
262 if (numvfs > 64) {
263 numvfs -= 64;
264
265 otx2_write64(pf, RVU_PF_VFME_INTX(1), INTR_MASK(numvfs));
266 otx2_write64(pf, RVU_PF_VFME_INT_ENA_W1SX(1),
267 INTR_MASK(numvfs));
268
269 otx2_write64(pf, RVU_PF_VFFLR_INTX(1), INTR_MASK(numvfs));
270 otx2_write64(pf, RVU_PF_VFFLR_INT_ENA_W1SX(1),
271 INTR_MASK(numvfs));
272 }
273 return 0;
274 }
275
otx2_pf_flr_init(struct otx2_nic * pf,int num_vfs)276 static int otx2_pf_flr_init(struct otx2_nic *pf, int num_vfs)
277 {
278 int vf;
279
280 pf->flr_wq = alloc_ordered_workqueue("otx2_pf_flr_wq", WQ_HIGHPRI);
281 if (!pf->flr_wq)
282 return -ENOMEM;
283
284 pf->flr_wrk = devm_kcalloc(pf->dev, num_vfs,
285 sizeof(struct flr_work), GFP_KERNEL);
286 if (!pf->flr_wrk) {
287 destroy_workqueue(pf->flr_wq);
288 return -ENOMEM;
289 }
290
291 for (vf = 0; vf < num_vfs; vf++) {
292 pf->flr_wrk[vf].pf = pf;
293 INIT_WORK(&pf->flr_wrk[vf].work, otx2_flr_handler);
294 }
295
296 return 0;
297 }
298
otx2_queue_vf_work(struct mbox * mw,struct workqueue_struct * mbox_wq,int first,int mdevs,u64 intr)299 void otx2_queue_vf_work(struct mbox *mw, struct workqueue_struct *mbox_wq,
300 int first, int mdevs, u64 intr)
301 {
302 struct otx2_mbox_dev *mdev;
303 struct otx2_mbox *mbox;
304 struct mbox_hdr *hdr;
305 int i;
306
307 for (i = first; i < mdevs; i++) {
308 /* start from 0 */
309 if (!(intr & BIT_ULL(i - first)))
310 continue;
311
312 mbox = &mw->mbox;
313 mdev = &mbox->dev[i];
314 hdr = mdev->mbase + mbox->rx_start;
315 /* The hdr->num_msgs is set to zero immediately in the interrupt
316 * handler to ensure that it holds a correct value next time
317 * when the interrupt handler is called. pf->mw[i].num_msgs
318 * holds the data for use in otx2_pfvf_mbox_handler and
319 * pf->mw[i].up_num_msgs holds the data for use in
320 * otx2_pfvf_mbox_up_handler.
321 */
322 if (hdr->num_msgs) {
323 mw[i].num_msgs = hdr->num_msgs;
324 hdr->num_msgs = 0;
325 queue_work(mbox_wq, &mw[i].mbox_wrk);
326 }
327
328 mbox = &mw->mbox_up;
329 mdev = &mbox->dev[i];
330 hdr = mdev->mbase + mbox->rx_start;
331 if (hdr->num_msgs) {
332 mw[i].up_num_msgs = hdr->num_msgs;
333 hdr->num_msgs = 0;
334 queue_work(mbox_wq, &mw[i].mbox_up_wrk);
335 }
336 }
337 }
338
otx2_forward_msg_pfvf(struct otx2_mbox_dev * mdev,struct otx2_mbox * pfvf_mbox,void * bbuf_base,int devid)339 static void otx2_forward_msg_pfvf(struct otx2_mbox_dev *mdev,
340 struct otx2_mbox *pfvf_mbox, void *bbuf_base,
341 int devid)
342 {
343 struct otx2_mbox_dev *src_mdev = mdev;
344 int offset;
345
346 /* Msgs are already copied, trigger VF's mbox irq */
347 smp_wmb();
348
349 otx2_mbox_wait_for_zero(pfvf_mbox, devid);
350
351 offset = pfvf_mbox->trigger | (devid << pfvf_mbox->tr_shift);
352 writeq(MBOX_DOWN_MSG, (void __iomem *)pfvf_mbox->reg_base + offset);
353
354 /* Restore VF's mbox bounce buffer region address */
355 src_mdev->mbase = bbuf_base;
356 }
357
otx2_forward_vf_mbox_msgs(struct otx2_nic * pf,struct otx2_mbox * src_mbox,int dir,int vf,int num_msgs)358 static int otx2_forward_vf_mbox_msgs(struct otx2_nic *pf,
359 struct otx2_mbox *src_mbox,
360 int dir, int vf, int num_msgs)
361 {
362 struct otx2_mbox_dev *src_mdev, *dst_mdev;
363 struct mbox_hdr *mbox_hdr;
364 struct mbox_hdr *req_hdr;
365 struct mbox *dst_mbox;
366 int dst_size, err;
367
368 if (dir == MBOX_DIR_PFAF) {
369 /* Set VF's mailbox memory as PF's bounce buffer memory, so
370 * that explicit copying of VF's msgs to PF=>AF mbox region
371 * and AF=>PF responses to VF's mbox region can be avoided.
372 */
373 src_mdev = &src_mbox->dev[vf];
374 mbox_hdr = src_mbox->hwbase +
375 src_mbox->rx_start + (vf * MBOX_SIZE);
376
377 dst_mbox = &pf->mbox;
378 dst_size = dst_mbox->mbox.tx_size -
379 ALIGN(sizeof(*mbox_hdr), MBOX_MSG_ALIGN);
380 /* Check if msgs fit into destination area and has valid size */
381 if (mbox_hdr->msg_size > dst_size || !mbox_hdr->msg_size)
382 return -EINVAL;
383
384 dst_mdev = &dst_mbox->mbox.dev[0];
385
386 mutex_lock(&pf->mbox.lock);
387 dst_mdev->mbase = src_mdev->mbase;
388 dst_mdev->msg_size = mbox_hdr->msg_size;
389 dst_mdev->num_msgs = num_msgs;
390 err = otx2_sync_mbox_msg(dst_mbox);
391 /* Error code -EIO indicate there is a communication failure
392 * to the AF. Rest of the error codes indicate that AF processed
393 * VF messages and set the error codes in response messages
394 * (if any) so simply forward responses to VF.
395 */
396 if (err == -EIO) {
397 dev_warn(pf->dev,
398 "AF not responding to VF%d messages\n", vf);
399 /* restore PF mbase and exit */
400 dst_mdev->mbase = pf->mbox.bbuf_base;
401 mutex_unlock(&pf->mbox.lock);
402 return err;
403 }
404 /* At this point, all the VF messages sent to AF are acked
405 * with proper responses and responses are copied to VF
406 * mailbox hence raise interrupt to VF.
407 */
408 req_hdr = (struct mbox_hdr *)(dst_mdev->mbase +
409 dst_mbox->mbox.rx_start);
410 req_hdr->num_msgs = num_msgs;
411
412 otx2_forward_msg_pfvf(dst_mdev, &pf->mbox_pfvf[0].mbox,
413 pf->mbox.bbuf_base, vf);
414 mutex_unlock(&pf->mbox.lock);
415 } else if (dir == MBOX_DIR_PFVF_UP) {
416 src_mdev = &src_mbox->dev[0];
417 mbox_hdr = src_mbox->hwbase + src_mbox->rx_start;
418 req_hdr = (struct mbox_hdr *)(src_mdev->mbase +
419 src_mbox->rx_start);
420 req_hdr->num_msgs = num_msgs;
421
422 dst_mbox = &pf->mbox_pfvf[0];
423 dst_size = dst_mbox->mbox_up.tx_size -
424 ALIGN(sizeof(*mbox_hdr), MBOX_MSG_ALIGN);
425 /* Check if msgs fit into destination area */
426 if (mbox_hdr->msg_size > dst_size)
427 return -EINVAL;
428
429 dst_mdev = &dst_mbox->mbox_up.dev[vf];
430 dst_mdev->mbase = src_mdev->mbase;
431 dst_mdev->msg_size = mbox_hdr->msg_size;
432 dst_mdev->num_msgs = mbox_hdr->num_msgs;
433 err = otx2_sync_mbox_up_msg(dst_mbox, vf);
434 if (err) {
435 dev_warn(pf->dev,
436 "VF%d is not responding to mailbox\n", vf);
437 return err;
438 }
439 } else if (dir == MBOX_DIR_VFPF_UP) {
440 req_hdr = (struct mbox_hdr *)(src_mbox->dev[0].mbase +
441 src_mbox->rx_start);
442 req_hdr->num_msgs = num_msgs;
443 otx2_forward_msg_pfvf(&pf->mbox_pfvf->mbox_up.dev[vf],
444 &pf->mbox.mbox_up,
445 pf->mbox_pfvf[vf].bbuf_base,
446 0);
447 }
448
449 return 0;
450 }
451
otx2_pfvf_mbox_handler(struct work_struct * work)452 static void otx2_pfvf_mbox_handler(struct work_struct *work)
453 {
454 struct mbox_msghdr *msg = NULL;
455 int offset, vf_idx, id, err;
456 struct otx2_mbox_dev *mdev;
457 struct otx2_mbox *mbox;
458 struct mbox *vf_mbox;
459 struct otx2_nic *pf;
460
461 vf_mbox = container_of(work, struct mbox, mbox_wrk);
462 pf = vf_mbox->pfvf;
463 vf_idx = vf_mbox - pf->mbox_pfvf;
464
465 mbox = &pf->mbox_pfvf[0].mbox;
466 mdev = &mbox->dev[vf_idx];
467
468 offset = ALIGN(sizeof(struct mbox_hdr), MBOX_MSG_ALIGN);
469
470 trace_otx2_msg_status(pf->pdev, "PF-VF down queue handler(forwarding)",
471 vf_mbox->num_msgs);
472
473 for (id = 0; id < vf_mbox->num_msgs; id++) {
474 msg = (struct mbox_msghdr *)(mdev->mbase + mbox->rx_start +
475 offset);
476
477 if (msg->sig != OTX2_MBOX_REQ_SIG)
478 goto inval_msg;
479
480 /* Set VF's number in each of the msg */
481 msg->pcifunc &= ~RVU_PFVF_FUNC_MASK;
482 msg->pcifunc |= (vf_idx + 1) & RVU_PFVF_FUNC_MASK;
483 offset = msg->next_msgoff;
484 }
485 err = otx2_forward_vf_mbox_msgs(pf, mbox, MBOX_DIR_PFAF, vf_idx,
486 vf_mbox->num_msgs);
487 if (err)
488 goto inval_msg;
489 return;
490
491 inval_msg:
492 otx2_reply_invalid_msg(mbox, vf_idx, 0, msg->id);
493 otx2_mbox_msg_send(mbox, vf_idx);
494 }
495
otx2_pfvf_mbox_up_handler(struct work_struct * work)496 static void otx2_pfvf_mbox_up_handler(struct work_struct *work)
497 {
498 struct mbox *vf_mbox = container_of(work, struct mbox, mbox_up_wrk);
499 struct otx2_nic *pf = vf_mbox->pfvf;
500 struct otx2_mbox_dev *mdev;
501 int offset, id, vf_idx = 0;
502 struct mbox_msghdr *msg;
503 struct otx2_mbox *mbox;
504
505 vf_idx = vf_mbox - pf->mbox_pfvf;
506 mbox = &pf->mbox_pfvf[0].mbox_up;
507 mdev = &mbox->dev[vf_idx];
508
509 offset = mbox->rx_start + ALIGN(sizeof(struct mbox_hdr), MBOX_MSG_ALIGN);
510
511 trace_otx2_msg_status(pf->pdev, "PF-VF up queue handler(response)",
512 vf_mbox->up_num_msgs);
513
514 for (id = 0; id < vf_mbox->up_num_msgs; id++) {
515 msg = mdev->mbase + offset;
516
517 if (msg->id >= MBOX_MSG_MAX) {
518 dev_err(pf->dev,
519 "Mbox msg with unknown ID 0x%x\n", msg->id);
520 goto end;
521 }
522
523 if (msg->sig != OTX2_MBOX_RSP_SIG) {
524 dev_err(pf->dev,
525 "Mbox msg with wrong signature %x, ID 0x%x\n",
526 msg->sig, msg->id);
527 goto end;
528 }
529
530 switch (msg->id) {
531 case MBOX_MSG_CGX_LINK_EVENT:
532 case MBOX_MSG_REP_EVENT_UP_NOTIFY:
533 break;
534 default:
535 if (msg->rc)
536 dev_err(pf->dev,
537 "Mbox msg response has err %d, ID 0x%x\n",
538 msg->rc, msg->id);
539 break;
540 }
541
542 end:
543 offset = mbox->rx_start + msg->next_msgoff;
544 if (mdev->msgs_acked == (vf_mbox->up_num_msgs - 1))
545 __otx2_mbox_reset(mbox, vf_idx);
546 mdev->msgs_acked++;
547 }
548 }
549
otx2_pfvf_mbox_intr_handler(int irq,void * pf_irq)550 irqreturn_t otx2_pfvf_mbox_intr_handler(int irq, void *pf_irq)
551 {
552 struct otx2_nic *pf = (struct otx2_nic *)(pf_irq);
553 int vfs = pf->total_vfs;
554 struct mbox *mbox;
555 u64 intr;
556
557 mbox = pf->mbox_pfvf;
558 /* Handle VF interrupts */
559 if (vfs > 64) {
560 intr = otx2_read64(pf, RVU_PF_VFPF_MBOX_INTX(1));
561 otx2_write64(pf, RVU_PF_VFPF_MBOX_INTX(1), intr);
562 otx2_queue_vf_work(mbox, pf->mbox_pfvf_wq, 64, vfs, intr);
563 if (intr)
564 trace_otx2_msg_interrupt(mbox->mbox.pdev, "VF(s) to PF", intr);
565 vfs = 64;
566 }
567
568 intr = otx2_read64(pf, RVU_PF_VFPF_MBOX_INTX(0));
569 otx2_write64(pf, RVU_PF_VFPF_MBOX_INTX(0), intr);
570
571 otx2_queue_vf_work(mbox, pf->mbox_pfvf_wq, 0, vfs, intr);
572
573 if (intr)
574 trace_otx2_msg_interrupt(mbox->mbox.pdev, "VF(s) to PF", intr);
575
576 return IRQ_HANDLED;
577 }
578
cn20k_pfvf_mbox_alloc(struct otx2_nic * pf,int numvfs)579 static void *cn20k_pfvf_mbox_alloc(struct otx2_nic *pf, int numvfs)
580 {
581 struct qmem *mbox_addr;
582 int err;
583
584 err = qmem_alloc(&pf->pdev->dev, &mbox_addr, numvfs, MBOX_SIZE);
585 if (err) {
586 dev_err(pf->dev, "qmem alloc fail\n");
587 return ERR_PTR(-ENOMEM);
588 }
589
590 otx2_write64(pf, RVU_PF_VF_MBOX_ADDR, (u64)mbox_addr->iova);
591 pf->pfvf_mbox_addr = mbox_addr;
592
593 return mbox_addr->base;
594 }
595
otx2_pfvf_mbox_init(struct otx2_nic * pf,int numvfs)596 static int otx2_pfvf_mbox_init(struct otx2_nic *pf, int numvfs)
597 {
598 void __iomem *hwbase;
599 struct mbox *mbox;
600 int err, vf;
601 u64 base;
602
603 if (!numvfs)
604 return -EINVAL;
605
606 pf->mbox_pfvf = devm_kcalloc(&pf->pdev->dev, numvfs,
607 sizeof(struct mbox), GFP_KERNEL);
608 if (!pf->mbox_pfvf)
609 return -ENOMEM;
610
611 pf->mbox_pfvf_wq = alloc_workqueue("otx2_pfvf_mailbox",
612 WQ_UNBOUND | WQ_HIGHPRI |
613 WQ_MEM_RECLAIM, 0);
614 if (!pf->mbox_pfvf_wq)
615 return -ENOMEM;
616
617 /* For CN20K, PF allocates mbox memory in DRAM and writes PF/VF
618 * regions/offsets in RVU_PF_VF_MBOX_ADDR, the RVU_PFX_FUNC_PFAF_MBOX
619 * gives the aliased address to access PF/VF mailbox regions.
620 */
621 if (is_cn20k(pf->pdev)) {
622 hwbase = (void __iomem *)cn20k_pfvf_mbox_alloc(pf, numvfs);
623 } else {
624 /* On CN10K platform, PF <-> VF mailbox region follows after
625 * PF <-> AF mailbox region.
626 */
627 if (test_bit(CN10K_MBOX, &pf->hw.cap_flag))
628 base = pci_resource_start(pf->pdev, PCI_MBOX_BAR_NUM) +
629 MBOX_SIZE;
630 else
631 base = readq(pf->reg_base + RVU_PF_VF_BAR4_ADDR);
632
633 hwbase = ioremap_wc(base, MBOX_SIZE * pf->total_vfs);
634 if (!hwbase) {
635 err = -ENOMEM;
636 goto free_wq;
637 }
638 }
639
640 mbox = &pf->mbox_pfvf[0];
641 err = otx2_mbox_init(&mbox->mbox, hwbase, pf->pdev, pf->reg_base,
642 MBOX_DIR_PFVF, numvfs);
643 if (err)
644 goto free_iomem;
645
646 err = otx2_mbox_init(&mbox->mbox_up, hwbase, pf->pdev, pf->reg_base,
647 MBOX_DIR_PFVF_UP, numvfs);
648 if (err)
649 goto free_iomem;
650
651 for (vf = 0; vf < numvfs; vf++) {
652 mbox->pfvf = pf;
653 INIT_WORK(&mbox->mbox_wrk, otx2_pfvf_mbox_handler);
654 INIT_WORK(&mbox->mbox_up_wrk, otx2_pfvf_mbox_up_handler);
655 mbox++;
656 }
657
658 return 0;
659
660 free_iomem:
661 if (hwbase && !(is_cn20k(pf->pdev)))
662 iounmap(hwbase);
663 free_wq:
664 destroy_workqueue(pf->mbox_pfvf_wq);
665 return err;
666 }
667
otx2_pfvf_mbox_destroy(struct otx2_nic * pf)668 static void otx2_pfvf_mbox_destroy(struct otx2_nic *pf)
669 {
670 struct mbox *mbox = &pf->mbox_pfvf[0];
671
672 if (!mbox)
673 return;
674
675 if (pf->mbox_pfvf_wq) {
676 destroy_workqueue(pf->mbox_pfvf_wq);
677 pf->mbox_pfvf_wq = NULL;
678 }
679
680 if (mbox->mbox.hwbase && !is_cn20k(pf->pdev))
681 iounmap(mbox->mbox.hwbase);
682 else
683 qmem_free(&pf->pdev->dev, pf->pfvf_mbox_addr);
684
685 otx2_mbox_destroy(&mbox->mbox);
686 }
687
otx2_enable_pfvf_mbox_intr(struct otx2_nic * pf,int numvfs)688 static void otx2_enable_pfvf_mbox_intr(struct otx2_nic *pf, int numvfs)
689 {
690 /* Clear PF <=> VF mailbox IRQ */
691 otx2_write64(pf, RVU_PF_VFPF_MBOX_INTX(0), ~0ull);
692 otx2_write64(pf, RVU_PF_VFPF_MBOX_INTX(1), ~0ull);
693
694 /* Enable PF <=> VF mailbox IRQ */
695 otx2_write64(pf, RVU_PF_VFPF_MBOX_INT_ENA_W1SX(0), INTR_MASK(numvfs));
696 if (numvfs > 64) {
697 numvfs -= 64;
698 otx2_write64(pf, RVU_PF_VFPF_MBOX_INT_ENA_W1SX(1),
699 INTR_MASK(numvfs));
700 }
701 }
702
otx2_disable_pfvf_mbox_intr(struct otx2_nic * pf,int numvfs)703 static void otx2_disable_pfvf_mbox_intr(struct otx2_nic *pf, int numvfs)
704 {
705 int vector;
706
707 if (is_cn20k(pf->pdev))
708 return cn20k_disable_pfvf_mbox_intr(pf, numvfs);
709
710 /* Disable PF <=> VF mailbox IRQ */
711 otx2_write64(pf, RVU_PF_VFPF_MBOX_INT_ENA_W1CX(0), ~0ull);
712 otx2_write64(pf, RVU_PF_VFPF_MBOX_INT_ENA_W1CX(1), ~0ull);
713
714 otx2_write64(pf, RVU_PF_VFPF_MBOX_INTX(0), ~0ull);
715 vector = pci_irq_vector(pf->pdev, RVU_PF_INT_VEC_VFPF_MBOX0);
716 free_irq(vector, pf);
717
718 if (numvfs > 64) {
719 otx2_write64(pf, RVU_PF_VFPF_MBOX_INTX(1), ~0ull);
720 vector = pci_irq_vector(pf->pdev, RVU_PF_INT_VEC_VFPF_MBOX1);
721 free_irq(vector, pf);
722 }
723 }
724
otx2_register_pfvf_mbox_intr(struct otx2_nic * pf,int numvfs)725 static int otx2_register_pfvf_mbox_intr(struct otx2_nic *pf, int numvfs)
726 {
727 struct otx2_hw *hw = &pf->hw;
728 char *irq_name;
729 int err;
730
731 if (is_cn20k(pf->pdev))
732 return cn20k_register_pfvf_mbox_intr(pf, numvfs);
733
734 /* Register MBOX0 interrupt handler */
735 irq_name = &hw->irq_name[RVU_PF_INT_VEC_VFPF_MBOX0 * NAME_SIZE];
736 if (pf->pcifunc)
737 snprintf(irq_name, NAME_SIZE,
738 "RVUPF%d_VF Mbox0", rvu_get_pf(pf->pdev, pf->pcifunc));
739 else
740 snprintf(irq_name, NAME_SIZE, "RVUPF_VF Mbox0");
741 err = request_irq(pci_irq_vector(pf->pdev, RVU_PF_INT_VEC_VFPF_MBOX0),
742 otx2_pfvf_mbox_intr_handler, 0, irq_name, pf);
743 if (err) {
744 dev_err(pf->dev,
745 "RVUPF: IRQ registration failed for PFVF mbox0 irq\n");
746 return err;
747 }
748
749 if (numvfs > 64) {
750 /* Register MBOX1 interrupt handler */
751 irq_name = &hw->irq_name[RVU_PF_INT_VEC_VFPF_MBOX1 * NAME_SIZE];
752 if (pf->pcifunc)
753 snprintf(irq_name, NAME_SIZE,
754 "RVUPF%d_VF Mbox1",
755 rvu_get_pf(pf->pdev, pf->pcifunc));
756 else
757 snprintf(irq_name, NAME_SIZE, "RVUPF_VF Mbox1");
758 err = request_irq(pci_irq_vector(pf->pdev,
759 RVU_PF_INT_VEC_VFPF_MBOX1),
760 otx2_pfvf_mbox_intr_handler,
761 0, irq_name, pf);
762 if (err) {
763 dev_err(pf->dev,
764 "RVUPF: IRQ registration failed for PFVF mbox1 irq\n");
765 return err;
766 }
767 }
768
769 otx2_enable_pfvf_mbox_intr(pf, numvfs);
770
771 return 0;
772 }
773
otx2_process_pfaf_mbox_msg(struct otx2_nic * pf,struct mbox_msghdr * msg)774 static void otx2_process_pfaf_mbox_msg(struct otx2_nic *pf,
775 struct mbox_msghdr *msg)
776 {
777 int devid;
778
779 if (msg->id >= MBOX_MSG_MAX) {
780 dev_err(pf->dev,
781 "Mbox msg with unknown ID 0x%x\n", msg->id);
782 return;
783 }
784
785 if (msg->sig != OTX2_MBOX_RSP_SIG) {
786 dev_err(pf->dev,
787 "Mbox msg with wrong signature %x, ID 0x%x\n",
788 msg->sig, msg->id);
789 return;
790 }
791
792 /* message response heading VF */
793 devid = msg->pcifunc & RVU_PFVF_FUNC_MASK;
794 if (devid) {
795 struct otx2_vf_config *config = &pf->vf_configs[devid - 1];
796 struct delayed_work *dwork;
797
798 switch (msg->id) {
799 case MBOX_MSG_NIX_LF_START_RX:
800 config->intf_down = false;
801 dwork = &config->link_event_work;
802 schedule_delayed_work(dwork, msecs_to_jiffies(100));
803 break;
804 case MBOX_MSG_NIX_LF_STOP_RX:
805 config->intf_down = true;
806 break;
807 }
808
809 return;
810 }
811
812 switch (msg->id) {
813 case MBOX_MSG_READY:
814 pf->pcifunc = msg->pcifunc;
815 break;
816 case MBOX_MSG_MSIX_OFFSET:
817 mbox_handler_msix_offset(pf, (struct msix_offset_rsp *)msg);
818 break;
819 case MBOX_MSG_NPA_LF_ALLOC:
820 mbox_handler_npa_lf_alloc(pf, (struct npa_lf_alloc_rsp *)msg);
821 break;
822 case MBOX_MSG_NIX_LF_ALLOC:
823 mbox_handler_nix_lf_alloc(pf, (struct nix_lf_alloc_rsp *)msg);
824 break;
825 case MBOX_MSG_NIX_BP_ENABLE:
826 mbox_handler_nix_bp_enable(pf, (struct nix_bp_cfg_rsp *)msg);
827 break;
828 case MBOX_MSG_CGX_STATS:
829 mbox_handler_cgx_stats(pf, (struct cgx_stats_rsp *)msg);
830 break;
831 case MBOX_MSG_CGX_FEC_STATS:
832 mbox_handler_cgx_fec_stats(pf, (struct cgx_fec_stats_rsp *)msg);
833 break;
834 default:
835 if (msg->rc)
836 dev_err(pf->dev,
837 "Mbox msg response has err %d, ID 0x%x\n",
838 msg->rc, msg->id);
839 break;
840 }
841 }
842
otx2_pfaf_mbox_handler(struct work_struct * work)843 static void otx2_pfaf_mbox_handler(struct work_struct *work)
844 {
845 struct otx2_mbox_dev *mdev;
846 struct mbox_hdr *rsp_hdr;
847 struct mbox_msghdr *msg;
848 struct otx2_mbox *mbox;
849 struct mbox *af_mbox;
850 struct otx2_nic *pf;
851 int offset, id;
852 u16 num_msgs;
853
854 af_mbox = container_of(work, struct mbox, mbox_wrk);
855 mbox = &af_mbox->mbox;
856 mdev = &mbox->dev[0];
857 rsp_hdr = (struct mbox_hdr *)(mdev->mbase + mbox->rx_start);
858 num_msgs = rsp_hdr->num_msgs;
859
860 offset = mbox->rx_start + ALIGN(sizeof(*rsp_hdr), MBOX_MSG_ALIGN);
861 pf = af_mbox->pfvf;
862
863 trace_otx2_msg_status(pf->pdev, "PF-AF down queue handler(response)",
864 num_msgs);
865
866 for (id = 0; id < num_msgs; id++) {
867 msg = (struct mbox_msghdr *)(mdev->mbase + offset);
868 otx2_process_pfaf_mbox_msg(pf, msg);
869 offset = mbox->rx_start + msg->next_msgoff;
870 if (mdev->msgs_acked == (num_msgs - 1))
871 __otx2_mbox_reset(mbox, 0);
872 mdev->msgs_acked++;
873 }
874
875 }
876
otx2_handle_link_event(struct otx2_nic * pf)877 static void otx2_handle_link_event(struct otx2_nic *pf)
878 {
879 struct cgx_link_user_info *linfo = &pf->linfo;
880 struct net_device *netdev = pf->netdev;
881
882 if (pf->flags & OTX2_FLAG_PORT_UP)
883 return;
884
885 pr_info("%s NIC Link is %s %d Mbps %s duplex\n", netdev->name,
886 linfo->link_up ? "UP" : "DOWN", linfo->speed,
887 linfo->full_duplex ? "Full" : "Half");
888 if (linfo->link_up) {
889 netif_carrier_on(netdev);
890 netif_tx_start_all_queues(netdev);
891 } else {
892 netif_tx_stop_all_queues(netdev);
893 netif_carrier_off(netdev);
894 }
895 }
896
otx2_mbox_up_handler_rep_event_up_notify(struct otx2_nic * pf,struct rep_event * info,struct msg_rsp * rsp)897 static int otx2_mbox_up_handler_rep_event_up_notify(struct otx2_nic *pf,
898 struct rep_event *info,
899 struct msg_rsp *rsp)
900 {
901 struct net_device *netdev = pf->netdev;
902
903 if (info->event == RVU_EVENT_MTU_CHANGE) {
904 netdev->mtu = info->evt_data.mtu;
905 return 0;
906 }
907
908 if (info->event == RVU_EVENT_PORT_STATE) {
909 if (info->evt_data.port_state) {
910 pf->flags |= OTX2_FLAG_PORT_UP;
911 netif_carrier_on(netdev);
912 netif_tx_start_all_queues(netdev);
913 } else {
914 pf->flags &= ~OTX2_FLAG_PORT_UP;
915 netif_tx_stop_all_queues(netdev);
916 netif_carrier_off(netdev);
917 }
918 return 0;
919 }
920 #ifdef CONFIG_RVU_ESWITCH
921 rvu_event_up_notify(pf, info);
922 #endif
923 return 0;
924 }
925
otx2_mbox_up_handler_mcs_intr_notify(struct otx2_nic * pf,struct mcs_intr_info * event,struct msg_rsp * rsp)926 int otx2_mbox_up_handler_mcs_intr_notify(struct otx2_nic *pf,
927 struct mcs_intr_info *event,
928 struct msg_rsp *rsp)
929 {
930 cn10k_handle_mcs_event(pf, event);
931
932 return 0;
933 }
934
otx2_mbox_up_handler_cgx_link_event(struct otx2_nic * pf,struct cgx_link_info_msg * msg,struct msg_rsp * rsp)935 int otx2_mbox_up_handler_cgx_link_event(struct otx2_nic *pf,
936 struct cgx_link_info_msg *msg,
937 struct msg_rsp *rsp)
938 {
939 int i;
940
941 /* Copy the link info sent by AF */
942 pf->linfo = msg->link_info;
943
944 /* notify VFs about link event */
945 for (i = 0; i < pci_num_vf(pf->pdev); i++) {
946 struct otx2_vf_config *config = &pf->vf_configs[i];
947 struct delayed_work *dwork = &config->link_event_work;
948
949 if (config->intf_down)
950 continue;
951
952 schedule_delayed_work(dwork, msecs_to_jiffies(100));
953 }
954
955 /* interface has not been fully configured yet */
956 if (pf->flags & OTX2_FLAG_INTF_DOWN)
957 return 0;
958
959 otx2_handle_link_event(pf);
960 return 0;
961 }
962
otx2_process_mbox_msg_up(struct otx2_nic * pf,struct mbox_msghdr * req)963 static int otx2_process_mbox_msg_up(struct otx2_nic *pf,
964 struct mbox_msghdr *req)
965 {
966 /* Check if valid, if not reply with a invalid msg */
967 if (req->sig != OTX2_MBOX_REQ_SIG) {
968 otx2_reply_invalid_msg(&pf->mbox.mbox_up, 0, 0, req->id);
969 return -ENODEV;
970 }
971
972 switch (req->id) {
973 #define M(_name, _id, _fn_name, _req_type, _rsp_type) \
974 case _id: { \
975 struct _rsp_type *rsp; \
976 int err; \
977 \
978 rsp = (struct _rsp_type *)otx2_mbox_alloc_msg( \
979 &pf->mbox.mbox_up, 0, \
980 sizeof(struct _rsp_type)); \
981 if (!rsp) \
982 return -ENOMEM; \
983 \
984 rsp->hdr.id = _id; \
985 rsp->hdr.sig = OTX2_MBOX_RSP_SIG; \
986 rsp->hdr.pcifunc = 0; \
987 rsp->hdr.rc = 0; \
988 \
989 err = otx2_mbox_up_handler_ ## _fn_name( \
990 pf, (struct _req_type *)req, rsp); \
991 return err; \
992 }
993 MBOX_UP_CGX_MESSAGES
994 MBOX_UP_MCS_MESSAGES
995 MBOX_UP_REP_MESSAGES
996 #undef M
997 break;
998 default:
999 otx2_reply_invalid_msg(&pf->mbox.mbox_up, 0, 0, req->id);
1000 return -ENODEV;
1001 }
1002 return 0;
1003 }
1004
otx2_pfaf_mbox_up_handler(struct work_struct * work)1005 static void otx2_pfaf_mbox_up_handler(struct work_struct *work)
1006 {
1007 struct mbox *af_mbox = container_of(work, struct mbox, mbox_up_wrk);
1008 struct otx2_mbox *mbox = &af_mbox->mbox_up;
1009 struct otx2_mbox_dev *mdev = &mbox->dev[0];
1010 struct otx2_nic *pf = af_mbox->pfvf;
1011 int offset, id, devid = 0;
1012 struct mbox_hdr *rsp_hdr;
1013 struct mbox_msghdr *msg;
1014 u16 num_msgs;
1015
1016 rsp_hdr = (struct mbox_hdr *)(mdev->mbase + mbox->rx_start);
1017 num_msgs = rsp_hdr->num_msgs;
1018
1019 offset = mbox->rx_start + ALIGN(sizeof(*rsp_hdr), MBOX_MSG_ALIGN);
1020
1021 trace_otx2_msg_status(pf->pdev, "PF-AF up queue handler(notification)",
1022 num_msgs);
1023
1024 for (id = 0; id < num_msgs; id++) {
1025 msg = (struct mbox_msghdr *)(mdev->mbase + offset);
1026
1027 devid = msg->pcifunc & RVU_PFVF_FUNC_MASK;
1028 /* Skip processing VF's messages */
1029 if (!devid)
1030 otx2_process_mbox_msg_up(pf, msg);
1031 offset = mbox->rx_start + msg->next_msgoff;
1032 }
1033 /* Forward to VF iff VFs are really present */
1034 if (devid && pci_num_vf(pf->pdev)) {
1035 otx2_forward_vf_mbox_msgs(pf, &pf->mbox.mbox_up,
1036 MBOX_DIR_PFVF_UP, devid - 1,
1037 num_msgs);
1038 return;
1039 }
1040
1041 otx2_mbox_msg_send(mbox, 0);
1042 }
1043
otx2_pfaf_mbox_intr_handler(int irq,void * pf_irq)1044 irqreturn_t otx2_pfaf_mbox_intr_handler(int irq, void *pf_irq)
1045 {
1046 struct otx2_nic *pf = (struct otx2_nic *)pf_irq;
1047 struct mbox *mw = &pf->mbox;
1048 struct otx2_mbox_dev *mdev;
1049 struct otx2_mbox *mbox;
1050 struct mbox_hdr *hdr;
1051 u64 mbox_data;
1052
1053 /* Clear the IRQ */
1054 otx2_write64(pf, RVU_PF_INT, BIT_ULL(0));
1055
1056
1057 mbox_data = otx2_read64(pf, RVU_PF_PFAF_MBOX0);
1058
1059 if (mbox_data & MBOX_UP_MSG) {
1060 mbox_data &= ~MBOX_UP_MSG;
1061 otx2_write64(pf, RVU_PF_PFAF_MBOX0, mbox_data);
1062
1063 mbox = &mw->mbox_up;
1064 mdev = &mbox->dev[0];
1065 otx2_sync_mbox_bbuf(mbox, 0);
1066
1067 hdr = (struct mbox_hdr *)(mdev->mbase + mbox->rx_start);
1068 if (hdr->num_msgs)
1069 queue_work(pf->mbox_wq, &mw->mbox_up_wrk);
1070
1071 trace_otx2_msg_interrupt(pf->pdev, "UP message from AF to PF",
1072 BIT_ULL(0));
1073
1074 trace_otx2_msg_status(pf->pdev, "PF-AF up work queued(interrupt)",
1075 hdr->num_msgs);
1076 }
1077
1078 if (mbox_data & MBOX_DOWN_MSG) {
1079 mbox_data &= ~MBOX_DOWN_MSG;
1080 otx2_write64(pf, RVU_PF_PFAF_MBOX0, mbox_data);
1081
1082 mbox = &mw->mbox;
1083 mdev = &mbox->dev[0];
1084 otx2_sync_mbox_bbuf(mbox, 0);
1085
1086 hdr = (struct mbox_hdr *)(mdev->mbase + mbox->rx_start);
1087 if (hdr->num_msgs)
1088 queue_work(pf->mbox_wq, &mw->mbox_wrk);
1089
1090 trace_otx2_msg_interrupt(pf->pdev, "DOWN reply from AF to PF",
1091 BIT_ULL(0));
1092
1093 trace_otx2_msg_status(pf->pdev, "PF-AF down work queued(interrupt)",
1094 hdr->num_msgs);
1095 }
1096
1097 return IRQ_HANDLED;
1098 }
1099
otx2_disable_mbox_intr(struct otx2_nic * pf)1100 void otx2_disable_mbox_intr(struct otx2_nic *pf)
1101 {
1102 int vector;
1103
1104 /* Disable AF => PF mailbox IRQ */
1105 if (!is_cn20k(pf->pdev)) {
1106 vector = pci_irq_vector(pf->pdev, RVU_PF_INT_VEC_AFPF_MBOX);
1107 otx2_write64(pf, RVU_PF_INT_ENA_W1C, BIT_ULL(0));
1108 } else {
1109 vector = pci_irq_vector(pf->pdev,
1110 RVU_MBOX_PF_INT_VEC_AFPF_MBOX);
1111 otx2_write64(pf, RVU_PF_INT_ENA_W1C,
1112 BIT_ULL(0) | BIT_ULL(1));
1113 }
1114 free_irq(vector, pf);
1115 }
1116 EXPORT_SYMBOL(otx2_disable_mbox_intr);
1117
otx2_register_mbox_intr(struct otx2_nic * pf,bool probe_af)1118 int otx2_register_mbox_intr(struct otx2_nic *pf, bool probe_af)
1119 {
1120 struct otx2_hw *hw = &pf->hw;
1121 struct msg_req *req;
1122 char *irq_name;
1123 int err;
1124
1125 /* Register mailbox interrupt handler */
1126 if (!is_cn20k(pf->pdev)) {
1127 irq_name = &hw->irq_name[RVU_PF_INT_VEC_AFPF_MBOX * NAME_SIZE];
1128 snprintf(irq_name, NAME_SIZE, "RVUPF%d AFPF Mbox",
1129 rvu_get_pf(pf->pdev, pf->pcifunc));
1130 err = request_irq(pci_irq_vector
1131 (pf->pdev, RVU_PF_INT_VEC_AFPF_MBOX),
1132 pf->hw_ops->pfaf_mbox_intr_handler,
1133 0, irq_name, pf);
1134 } else {
1135 irq_name = &hw->irq_name[RVU_MBOX_PF_INT_VEC_AFPF_MBOX *
1136 NAME_SIZE];
1137 snprintf(irq_name, NAME_SIZE, "RVUPF%d AFPF Mbox",
1138 rvu_get_pf(pf->pdev, pf->pcifunc));
1139 err = request_irq(pci_irq_vector
1140 (pf->pdev, RVU_MBOX_PF_INT_VEC_AFPF_MBOX),
1141 pf->hw_ops->pfaf_mbox_intr_handler,
1142 0, irq_name, pf);
1143 }
1144 if (err) {
1145 dev_err(pf->dev,
1146 "RVUPF: IRQ registration failed for PFAF mbox irq\n");
1147 return err;
1148 }
1149
1150 /* Enable mailbox interrupt for msgs coming from AF.
1151 * First clear to avoid spurious interrupts, if any.
1152 */
1153 if (!is_cn20k(pf->pdev)) {
1154 otx2_write64(pf, RVU_PF_INT, BIT_ULL(0));
1155 otx2_write64(pf, RVU_PF_INT_ENA_W1S, BIT_ULL(0));
1156 } else {
1157 otx2_write64(pf, RVU_PF_INT, BIT_ULL(0) | BIT_ULL(1));
1158 otx2_write64(pf, RVU_PF_INT_ENA_W1S, BIT_ULL(0) |
1159 BIT_ULL(1));
1160 }
1161
1162 if (!probe_af)
1163 return 0;
1164
1165 /* Check mailbox communication with AF */
1166 req = otx2_mbox_alloc_msg_ready(&pf->mbox);
1167 if (!req) {
1168 otx2_disable_mbox_intr(pf);
1169 return -ENOMEM;
1170 }
1171 err = otx2_sync_mbox_msg(&pf->mbox);
1172 if (err) {
1173 dev_warn(pf->dev,
1174 "AF not responding to mailbox, deferring probe\n");
1175 otx2_disable_mbox_intr(pf);
1176 return -EPROBE_DEFER;
1177 }
1178
1179 return 0;
1180 }
1181
otx2_pfaf_mbox_destroy(struct otx2_nic * pf)1182 void otx2_pfaf_mbox_destroy(struct otx2_nic *pf)
1183 {
1184 struct mbox *mbox = &pf->mbox;
1185
1186 if (pf->mbox_wq) {
1187 destroy_workqueue(pf->mbox_wq);
1188 pf->mbox_wq = NULL;
1189 }
1190
1191 if (mbox->mbox.hwbase && !is_cn20k(pf->pdev))
1192 iounmap((void __iomem *)mbox->mbox.hwbase);
1193
1194 otx2_mbox_destroy(&mbox->mbox);
1195 otx2_mbox_destroy(&mbox->mbox_up);
1196 }
1197 EXPORT_SYMBOL(otx2_pfaf_mbox_destroy);
1198
otx2_pfaf_mbox_init(struct otx2_nic * pf)1199 int otx2_pfaf_mbox_init(struct otx2_nic *pf)
1200 {
1201 struct mbox *mbox = &pf->mbox;
1202 void __iomem *hwbase;
1203 int err;
1204
1205 mbox->pfvf = pf;
1206 pf->mbox_wq = alloc_ordered_workqueue("otx2_pfaf_mailbox",
1207 WQ_HIGHPRI | WQ_MEM_RECLAIM);
1208 if (!pf->mbox_wq)
1209 return -ENOMEM;
1210
1211 /* For CN20K, AF allocates mbox memory in DRAM and writes PF
1212 * regions/offsets in RVU_MBOX_AF_PFX_ADDR, the RVU_PFX_FUNC_PFAF_MBOX
1213 * gives the aliased address to access AF/PF mailbox regions.
1214 */
1215 if (is_cn20k(pf->pdev))
1216 hwbase = pf->reg_base + RVU_PFX_FUNC_PFAF_MBOX +
1217 ((u64)BLKADDR_MBOX << RVU_FUNC_BLKADDR_SHIFT);
1218 else
1219 /* Mailbox is a reserved memory (in RAM) region shared between
1220 * admin function (i.e AF) and this PF, shouldn't be mapped as
1221 * device memory to allow unaligned accesses.
1222 */
1223 hwbase = ioremap_wc(pci_resource_start
1224 (pf->pdev, PCI_MBOX_BAR_NUM), MBOX_SIZE);
1225 if (!hwbase) {
1226 dev_err(pf->dev, "Unable to map PFAF mailbox region\n");
1227 err = -ENOMEM;
1228 goto exit;
1229 }
1230
1231 err = otx2_mbox_init(&mbox->mbox, hwbase, pf->pdev, pf->reg_base,
1232 MBOX_DIR_PFAF, 1);
1233 if (err)
1234 goto exit;
1235
1236 err = otx2_mbox_init(&mbox->mbox_up, hwbase, pf->pdev, pf->reg_base,
1237 MBOX_DIR_PFAF_UP, 1);
1238 if (err)
1239 goto exit;
1240
1241 err = otx2_mbox_bbuf_init(mbox, pf->pdev);
1242 if (err)
1243 goto exit;
1244
1245 INIT_WORK(&mbox->mbox_wrk, otx2_pfaf_mbox_handler);
1246 INIT_WORK(&mbox->mbox_up_wrk, otx2_pfaf_mbox_up_handler);
1247 mutex_init(&mbox->lock);
1248
1249 return 0;
1250 exit:
1251 otx2_pfaf_mbox_destroy(pf);
1252 return err;
1253 }
1254
otx2_cgx_config_linkevents(struct otx2_nic * pf,bool enable)1255 static int otx2_cgx_config_linkevents(struct otx2_nic *pf, bool enable)
1256 {
1257 struct msg_req *msg;
1258 int err;
1259
1260 mutex_lock(&pf->mbox.lock);
1261 if (enable)
1262 msg = otx2_mbox_alloc_msg_cgx_start_linkevents(&pf->mbox);
1263 else
1264 msg = otx2_mbox_alloc_msg_cgx_stop_linkevents(&pf->mbox);
1265
1266 if (!msg) {
1267 mutex_unlock(&pf->mbox.lock);
1268 return -ENOMEM;
1269 }
1270
1271 err = otx2_sync_mbox_msg(&pf->mbox);
1272 mutex_unlock(&pf->mbox.lock);
1273 return err;
1274 }
1275
otx2_reset_mac_stats(struct otx2_nic * pfvf)1276 int otx2_reset_mac_stats(struct otx2_nic *pfvf)
1277 {
1278 struct msg_req *req;
1279 int err;
1280
1281 mutex_lock(&pfvf->mbox.lock);
1282 req = otx2_mbox_alloc_msg_cgx_stats_rst(&pfvf->mbox);
1283 if (!req) {
1284 mutex_unlock(&pfvf->mbox.lock);
1285 return -ENOMEM;
1286 }
1287
1288 err = otx2_sync_mbox_msg(&pfvf->mbox);
1289 mutex_unlock(&pfvf->mbox.lock);
1290 return err;
1291 }
1292
otx2_cgx_config_loopback(struct otx2_nic * pf,bool enable)1293 static int otx2_cgx_config_loopback(struct otx2_nic *pf, bool enable)
1294 {
1295 struct msg_req *msg;
1296 int err;
1297
1298 if (enable && !bitmap_empty(pf->flow_cfg->dmacflt_bmap,
1299 pf->flow_cfg->dmacflt_max_flows))
1300 netdev_warn(pf->netdev,
1301 "CGX/RPM internal loopback might not work as DMAC filters are active\n");
1302
1303 mutex_lock(&pf->mbox.lock);
1304 if (enable)
1305 msg = otx2_mbox_alloc_msg_cgx_intlbk_enable(&pf->mbox);
1306 else
1307 msg = otx2_mbox_alloc_msg_cgx_intlbk_disable(&pf->mbox);
1308
1309 if (!msg) {
1310 mutex_unlock(&pf->mbox.lock);
1311 return -ENOMEM;
1312 }
1313
1314 err = otx2_sync_mbox_msg(&pf->mbox);
1315 mutex_unlock(&pf->mbox.lock);
1316 return err;
1317 }
1318
otx2_set_real_num_queues(struct net_device * netdev,int tx_queues,int rx_queues)1319 int otx2_set_real_num_queues(struct net_device *netdev,
1320 int tx_queues, int rx_queues)
1321 {
1322 int err;
1323
1324 err = netif_set_real_num_tx_queues(netdev, tx_queues);
1325 if (err) {
1326 netdev_err(netdev,
1327 "Failed to set no of Tx queues: %d\n", tx_queues);
1328 return err;
1329 }
1330
1331 err = netif_set_real_num_rx_queues(netdev, rx_queues);
1332 if (err)
1333 netdev_err(netdev,
1334 "Failed to set no of Rx queues: %d\n", rx_queues);
1335 return err;
1336 }
1337 EXPORT_SYMBOL(otx2_set_real_num_queues);
1338
1339 static char *nix_sqoperr_e_str[NIX_SQOPERR_MAX] = {
1340 "NIX_SQOPERR_OOR",
1341 "NIX_SQOPERR_CTX_FAULT",
1342 "NIX_SQOPERR_CTX_POISON",
1343 "NIX_SQOPERR_DISABLED",
1344 "NIX_SQOPERR_SIZE_ERR",
1345 "NIX_SQOPERR_OFLOW",
1346 "NIX_SQOPERR_SQB_NULL",
1347 "NIX_SQOPERR_SQB_FAULT",
1348 "NIX_SQOPERR_SQE_SZ_ZERO",
1349 };
1350
1351 static char *nix_mnqerr_e_str[NIX_MNQERR_MAX] = {
1352 "NIX_MNQERR_SQ_CTX_FAULT",
1353 "NIX_MNQERR_SQ_CTX_POISON",
1354 "NIX_MNQERR_SQB_FAULT",
1355 "NIX_MNQERR_SQB_POISON",
1356 "NIX_MNQERR_TOTAL_ERR",
1357 "NIX_MNQERR_LSO_ERR",
1358 "NIX_MNQERR_CQ_QUERY_ERR",
1359 "NIX_MNQERR_MAX_SQE_SIZE_ERR",
1360 "NIX_MNQERR_MAXLEN_ERR",
1361 "NIX_MNQERR_SQE_SIZEM1_ZERO",
1362 };
1363
1364 static char *nix_snd_status_e_str[NIX_SND_STATUS_MAX] = {
1365 [NIX_SND_STATUS_GOOD] = "NIX_SND_STATUS_GOOD",
1366 [NIX_SND_STATUS_SQ_CTX_FAULT] = "NIX_SND_STATUS_SQ_CTX_FAULT",
1367 [NIX_SND_STATUS_SQ_CTX_POISON] = "NIX_SND_STATUS_SQ_CTX_POISON",
1368 [NIX_SND_STATUS_SQB_FAULT] = "NIX_SND_STATUS_SQB_FAULT",
1369 [NIX_SND_STATUS_SQB_POISON] = "NIX_SND_STATUS_SQB_POISON",
1370 [NIX_SND_STATUS_HDR_ERR] = "NIX_SND_STATUS_HDR_ERR",
1371 [NIX_SND_STATUS_EXT_ERR] = "NIX_SND_STATUS_EXT_ERR",
1372 [NIX_SND_STATUS_JUMP_FAULT] = "NIX_SND_STATUS_JUMP_FAULT",
1373 [NIX_SND_STATUS_JUMP_POISON] = "NIX_SND_STATUS_JUMP_POISON",
1374 [NIX_SND_STATUS_CRC_ERR] = "NIX_SND_STATUS_CRC_ERR",
1375 [NIX_SND_STATUS_IMM_ERR] = "NIX_SND_STATUS_IMM_ERR",
1376 [NIX_SND_STATUS_SG_ERR] = "NIX_SND_STATUS_SG_ERR",
1377 [NIX_SND_STATUS_MEM_ERR] = "NIX_SND_STATUS_MEM_ERR",
1378 [NIX_SND_STATUS_INVALID_SUBDC] = "NIX_SND_STATUS_INVALID_SUBDC",
1379 [NIX_SND_STATUS_SUBDC_ORDER_ERR] = "NIX_SND_STATUS_SUBDC_ORDER_ERR",
1380 [NIX_SND_STATUS_DATA_FAULT] = "NIX_SND_STATUS_DATA_FAULT",
1381 [NIX_SND_STATUS_DATA_POISON] = "NIX_SND_STATUS_DATA_POISON",
1382 [NIX_SND_STATUS_NPC_DROP_ACTION] = "NIX_SND_STATUS_NPC_DROP_ACTION",
1383 [NIX_SND_STATUS_LOCK_VIOL] = "NIX_SND_STATUS_LOCK_VIOL",
1384 [NIX_SND_STATUS_NPC_UCAST_CHAN_ERR] = "NIX_SND_STAT_NPC_UCAST_CHAN_ERR",
1385 [NIX_SND_STATUS_NPC_MCAST_CHAN_ERR] = "NIX_SND_STAT_NPC_MCAST_CHAN_ERR",
1386 [NIX_SND_STATUS_NPC_MCAST_ABORT] = "NIX_SND_STATUS_NPC_MCAST_ABORT",
1387 [NIX_SND_STATUS_NPC_VTAG_PTR_ERR] = "NIX_SND_STATUS_NPC_VTAG_PTR_ERR",
1388 [NIX_SND_STATUS_NPC_VTAG_SIZE_ERR] = "NIX_SND_STATUS_NPC_VTAG_SIZE_ERR",
1389 [NIX_SND_STATUS_SEND_MEM_FAULT] = "NIX_SND_STATUS_SEND_MEM_FAULT",
1390 [NIX_SND_STATUS_SEND_STATS_ERR] = "NIX_SND_STATUS_SEND_STATS_ERR",
1391 };
1392
otx2_q_intr_handler(int irq,void * data)1393 static irqreturn_t otx2_q_intr_handler(int irq, void *data)
1394 {
1395 struct otx2_nic *pf = data;
1396 struct otx2_snd_queue *sq;
1397 void __iomem *ptr;
1398 u64 val, qidx = 0;
1399
1400 /* CQ */
1401 for (qidx = 0; qidx < pf->qset.cq_cnt; qidx++) {
1402 ptr = otx2_get_regaddr(pf, NIX_LF_CQ_OP_INT);
1403 val = otx2_atomic64_add((qidx << 44), ptr);
1404
1405 otx2_write64(pf, NIX_LF_CQ_OP_INT, (qidx << 44) |
1406 (val & NIX_CQERRINT_BITS));
1407 if (!(val & (NIX_CQERRINT_BITS | BIT_ULL(42))))
1408 continue;
1409
1410 if (val & BIT_ULL(42)) {
1411 netdev_err(pf->netdev,
1412 "CQ%lld: error reading NIX_LF_CQ_OP_INT, NIX_LF_ERR_INT 0x%llx\n",
1413 qidx, otx2_read64(pf, NIX_LF_ERR_INT));
1414 } else {
1415 if (val & BIT_ULL(NIX_CQERRINT_DOOR_ERR))
1416 netdev_err(pf->netdev, "CQ%lld: Doorbell error",
1417 qidx);
1418 if (val & BIT_ULL(NIX_CQERRINT_CQE_FAULT))
1419 netdev_err(pf->netdev,
1420 "CQ%lld: Memory fault on CQE write to LLC/DRAM",
1421 qidx);
1422 }
1423
1424 schedule_work(&pf->reset_task);
1425 }
1426
1427 /* SQ */
1428 for (qidx = 0; qidx < otx2_get_total_tx_queues(pf); qidx++) {
1429 u64 sq_op_err_dbg, mnq_err_dbg, snd_err_dbg;
1430 u8 sq_op_err_code, mnq_err_code, snd_err_code;
1431
1432 sq = &pf->qset.sq[qidx];
1433 if (!sq->sqb_ptrs)
1434 continue;
1435
1436 /* Below debug registers captures first errors corresponding to
1437 * those registers. We don't have to check against SQ qid as
1438 * these are fatal errors.
1439 */
1440
1441 ptr = otx2_get_regaddr(pf, NIX_LF_SQ_OP_INT);
1442 val = otx2_atomic64_add((qidx << 44), ptr);
1443 otx2_write64(pf, NIX_LF_SQ_OP_INT, (qidx << 44) |
1444 (val & NIX_SQINT_BITS));
1445
1446 if (val & BIT_ULL(42)) {
1447 netdev_err(pf->netdev,
1448 "SQ%lld: error reading NIX_LF_SQ_OP_INT, NIX_LF_ERR_INT 0x%llx\n",
1449 qidx, otx2_read64(pf, NIX_LF_ERR_INT));
1450 goto done;
1451 }
1452
1453 sq_op_err_dbg = otx2_read64(pf, NIX_LF_SQ_OP_ERR_DBG);
1454 if (!(sq_op_err_dbg & BIT(44)))
1455 goto chk_mnq_err_dbg;
1456
1457 sq_op_err_code = FIELD_GET(GENMASK(7, 0), sq_op_err_dbg);
1458 netdev_err(pf->netdev,
1459 "SQ%lld: NIX_LF_SQ_OP_ERR_DBG(0x%llx) err=%s(%#x)\n",
1460 qidx, sq_op_err_dbg,
1461 nix_sqoperr_e_str[sq_op_err_code],
1462 sq_op_err_code);
1463
1464 otx2_write64(pf, NIX_LF_SQ_OP_ERR_DBG, BIT_ULL(44));
1465
1466 if (sq_op_err_code == NIX_SQOPERR_SQB_NULL)
1467 goto chk_mnq_err_dbg;
1468
1469 /* Err is not NIX_SQOPERR_SQB_NULL, call aq function to read SQ structure.
1470 * TODO: But we are in irq context. How to call mbox functions which does sleep
1471 */
1472
1473 chk_mnq_err_dbg:
1474 mnq_err_dbg = otx2_read64(pf, NIX_LF_MNQ_ERR_DBG);
1475 if (!(mnq_err_dbg & BIT(44)))
1476 goto chk_snd_err_dbg;
1477
1478 mnq_err_code = FIELD_GET(GENMASK(7, 0), mnq_err_dbg);
1479 netdev_err(pf->netdev,
1480 "SQ%lld: NIX_LF_MNQ_ERR_DBG(0x%llx) err=%s(%#x)\n",
1481 qidx, mnq_err_dbg, nix_mnqerr_e_str[mnq_err_code],
1482 mnq_err_code);
1483 otx2_write64(pf, NIX_LF_MNQ_ERR_DBG, BIT_ULL(44));
1484
1485 chk_snd_err_dbg:
1486 snd_err_dbg = otx2_read64(pf, NIX_LF_SEND_ERR_DBG);
1487 if (snd_err_dbg & BIT(44)) {
1488 snd_err_code = FIELD_GET(GENMASK(7, 0), snd_err_dbg);
1489 netdev_err(pf->netdev,
1490 "SQ%lld: NIX_LF_SND_ERR_DBG:0x%llx err=%s(%#x)\n",
1491 qidx, snd_err_dbg,
1492 nix_snd_status_e_str[snd_err_code],
1493 snd_err_code);
1494 otx2_write64(pf, NIX_LF_SEND_ERR_DBG, BIT_ULL(44));
1495 }
1496
1497 done:
1498 /* Print values and reset */
1499 if (val & BIT_ULL(NIX_SQINT_SQB_ALLOC_FAIL))
1500 netdev_err(pf->netdev, "SQ%lld: SQB allocation failed",
1501 qidx);
1502
1503 schedule_work(&pf->reset_task);
1504 }
1505
1506 return IRQ_HANDLED;
1507 }
1508
otx2_cq_intr_handler(int irq,void * cq_irq)1509 irqreturn_t otx2_cq_intr_handler(int irq, void *cq_irq)
1510 {
1511 struct otx2_cq_poll *cq_poll = (struct otx2_cq_poll *)cq_irq;
1512 struct otx2_nic *pf = (struct otx2_nic *)cq_poll->dev;
1513 int qidx = cq_poll->cint_idx;
1514
1515 /* Disable interrupts.
1516 *
1517 * Completion interrupts behave in a level-triggered interrupt
1518 * fashion, and hence have to be cleared only after it is serviced.
1519 */
1520 otx2_write64(pf, NIX_LF_CINTX_ENA_W1C(qidx), BIT_ULL(0));
1521
1522 /* Schedule NAPI */
1523 pf->napi_events++;
1524 napi_schedule_irqoff(&cq_poll->napi);
1525
1526 return IRQ_HANDLED;
1527 }
1528 EXPORT_SYMBOL(otx2_cq_intr_handler);
1529
otx2_disable_napi(struct otx2_nic * pf)1530 void otx2_disable_napi(struct otx2_nic *pf)
1531 {
1532 struct otx2_qset *qset = &pf->qset;
1533 struct otx2_cq_poll *cq_poll;
1534 struct work_struct *work;
1535 int qidx;
1536
1537 for (qidx = 0; qidx < pf->hw.cint_cnt; qidx++) {
1538 cq_poll = &qset->napi[qidx];
1539 work = &cq_poll->dim.work;
1540 if (work->func)
1541 cancel_work_sync(work);
1542 napi_disable(&cq_poll->napi);
1543 netif_napi_del(&cq_poll->napi);
1544 }
1545 }
1546 EXPORT_SYMBOL(otx2_disable_napi);
1547
otx2_free_cq_res(struct otx2_nic * pf)1548 static void otx2_free_cq_res(struct otx2_nic *pf)
1549 {
1550 struct otx2_qset *qset = &pf->qset;
1551 struct otx2_cq_queue *cq;
1552 int qidx;
1553
1554 /* Disable CQs */
1555 otx2_ctx_disable(&pf->mbox, NIX_AQ_CTYPE_CQ, false);
1556 for (qidx = 0; qidx < qset->cq_cnt; qidx++) {
1557 cq = &qset->cq[qidx];
1558 qmem_free(pf->dev, cq->cqe);
1559 }
1560 }
1561
otx2_free_sq_res(struct otx2_nic * pf)1562 static void otx2_free_sq_res(struct otx2_nic *pf)
1563 {
1564 struct otx2_qset *qset = &pf->qset;
1565 struct otx2_snd_queue *sq;
1566 int qidx;
1567
1568 /* Disable SQs */
1569 otx2_ctx_disable(&pf->mbox, NIX_AQ_CTYPE_SQ, false);
1570 /* Free SQB pointers */
1571 otx2_sq_free_sqbs(pf);
1572 for (qidx = 0; qidx < otx2_get_total_tx_queues(pf); qidx++) {
1573 sq = &qset->sq[qidx];
1574 /* Skip freeing Qos queues if they are not initialized */
1575 if (!sq->sqe)
1576 continue;
1577 qmem_free(pf->dev, sq->sqe);
1578 qmem_free(pf->dev, sq->sqe_ring);
1579 qmem_free(pf->dev, sq->cpt_resp);
1580 qmem_free(pf->dev, sq->tso_hdrs);
1581 kfree(sq->sg);
1582 kfree(sq->sqb_ptrs);
1583 }
1584 }
1585
otx2_get_rbuf_size(struct otx2_nic * pf,int mtu)1586 static int otx2_get_rbuf_size(struct otx2_nic *pf, int mtu)
1587 {
1588 int frame_size;
1589 int total_size;
1590 int rbuf_size;
1591
1592 if (pf->hw.rbuf_len)
1593 return ALIGN(pf->hw.rbuf_len, OTX2_ALIGN) + OTX2_HEAD_ROOM;
1594
1595 /* The data transferred by NIX to memory consists of actual packet
1596 * plus additional data which has timestamp and/or EDSA/HIGIG2
1597 * headers if interface is configured in corresponding modes.
1598 * NIX transfers entire data using 6 segments/buffers and writes
1599 * a CQE_RX descriptor with those segment addresses. First segment
1600 * has additional data prepended to packet. Also software omits a
1601 * headroom of 128 bytes in each segment. Hence the total size of
1602 * memory needed to receive a packet with 'mtu' is:
1603 * frame size = mtu + additional data;
1604 * memory = frame_size + headroom * 6;
1605 * each receive buffer size = memory / 6;
1606 */
1607 frame_size = mtu + OTX2_ETH_HLEN + OTX2_HW_TIMESTAMP_LEN;
1608 total_size = frame_size + OTX2_HEAD_ROOM * 6;
1609 rbuf_size = total_size / 6;
1610
1611 return ALIGN(rbuf_size, 2048);
1612 }
1613
otx2_init_hw_resources(struct otx2_nic * pf)1614 int otx2_init_hw_resources(struct otx2_nic *pf)
1615 {
1616 struct nix_lf_free_req *free_req;
1617 struct mbox *mbox = &pf->mbox;
1618 struct otx2_hw *hw = &pf->hw;
1619 struct msg_req *req;
1620 int err = 0, lvl;
1621
1622 /* Set required NPA LF's pool counts
1623 * Auras and Pools are used in a 1:1 mapping,
1624 * so, aura count = pool count.
1625 */
1626 hw->rqpool_cnt = hw->rx_queues;
1627 hw->sqpool_cnt = otx2_get_total_tx_queues(pf);
1628 hw->pool_cnt = hw->rqpool_cnt + hw->sqpool_cnt;
1629
1630 if (!otx2_rep_dev(pf->pdev)) {
1631 /* Maximum hardware supported transmit length */
1632 pf->tx_max_pktlen = pf->netdev->max_mtu + OTX2_ETH_HLEN;
1633 pf->rbsize = otx2_get_rbuf_size(pf, pf->netdev->mtu);
1634 }
1635
1636 mutex_lock(&mbox->lock);
1637 /* NPA init */
1638 err = otx2_config_npa(pf);
1639 if (err)
1640 goto exit;
1641
1642 /* NIX init */
1643 err = otx2_config_nix(pf);
1644 if (err)
1645 goto err_free_npa_lf;
1646
1647 /* Default disable backpressure on NIX-CPT */
1648 otx2_nix_cpt_config_bp(pf, false);
1649
1650 /* Enable backpressure for CGX mapped PF/VFs */
1651 if (!is_otx2_lbkvf(pf->pdev))
1652 otx2_nix_config_bp(pf, true);
1653
1654 /* Init Auras and pools used by NIX RQ, for free buffer ptrs */
1655 err = otx2_rq_aura_pool_init(pf);
1656 if (err) {
1657 mutex_unlock(&mbox->lock);
1658 goto err_free_nix_lf;
1659 }
1660 /* Init Auras and pools used by NIX SQ, for queueing SQEs */
1661 err = otx2_sq_aura_pool_init(pf);
1662 if (err) {
1663 mutex_unlock(&mbox->lock);
1664 goto err_free_rq_ptrs;
1665 }
1666
1667 err = otx2_txsch_alloc(pf);
1668 if (err) {
1669 mutex_unlock(&mbox->lock);
1670 goto err_free_sq_ptrs;
1671 }
1672
1673 #ifdef CONFIG_DCB
1674 if (pf->pfc_en) {
1675 err = otx2_pfc_txschq_alloc(pf);
1676 if (err) {
1677 mutex_unlock(&mbox->lock);
1678 goto err_free_sq_ptrs;
1679 }
1680 }
1681 #endif
1682
1683 err = otx2_config_nix_queues(pf);
1684 if (err) {
1685 mutex_unlock(&mbox->lock);
1686 goto err_free_txsch;
1687 }
1688
1689 for (lvl = 0; lvl < NIX_TXSCH_LVL_CNT; lvl++) {
1690 int idx;
1691
1692 for (idx = 0; idx < pf->hw.txschq_cnt[lvl]; idx++) {
1693 err = otx2_txschq_config(pf, lvl, idx, false);
1694 if (err) {
1695 dev_err(pf->dev, "Failed to config TXSCH\n");
1696 mutex_unlock(&mbox->lock);
1697 goto err_free_nix_queues;
1698 }
1699 }
1700 }
1701
1702 #ifdef CONFIG_DCB
1703 if (pf->pfc_en) {
1704 err = otx2_pfc_txschq_config(pf);
1705 if (err) {
1706 mutex_unlock(&mbox->lock);
1707 goto err_free_nix_queues;
1708 }
1709 }
1710 #endif
1711
1712 mutex_unlock(&mbox->lock);
1713 return err;
1714
1715 err_free_nix_queues:
1716 otx2_free_sq_res(pf);
1717 otx2_free_cq_res(pf);
1718 otx2_ctx_disable(mbox, NIX_AQ_CTYPE_RQ, false);
1719 err_free_txsch:
1720 otx2_txschq_stop(pf);
1721 err_free_sq_ptrs:
1722 otx2_sq_free_sqbs(pf);
1723 err_free_rq_ptrs:
1724 otx2_free_aura_ptr(pf, AURA_NIX_RQ);
1725 otx2_ctx_disable(mbox, NPA_AQ_CTYPE_POOL, true);
1726 otx2_ctx_disable(mbox, NPA_AQ_CTYPE_AURA, true);
1727 otx2_aura_pool_free(pf);
1728 err_free_nix_lf:
1729 mutex_lock(&mbox->lock);
1730 free_req = otx2_mbox_alloc_msg_nix_lf_free(mbox);
1731 if (free_req) {
1732 free_req->flags = NIX_LF_DISABLE_FLOWS;
1733 if (otx2_sync_mbox_msg(mbox))
1734 dev_err(pf->dev, "%s failed to free nixlf\n", __func__);
1735 }
1736 err_free_npa_lf:
1737 /* Reset NPA LF */
1738 req = otx2_mbox_alloc_msg_npa_lf_free(mbox);
1739 if (req) {
1740 if (otx2_sync_mbox_msg(mbox))
1741 dev_err(pf->dev, "%s failed to free npalf\n", __func__);
1742 }
1743 exit:
1744 mutex_unlock(&mbox->lock);
1745 return err;
1746 }
1747 EXPORT_SYMBOL(otx2_init_hw_resources);
1748
otx2_free_hw_resources(struct otx2_nic * pf)1749 void otx2_free_hw_resources(struct otx2_nic *pf)
1750 {
1751 struct otx2_qset *qset = &pf->qset;
1752 struct nix_lf_free_req *free_req;
1753 struct mbox *mbox = &pf->mbox;
1754 struct otx2_cq_queue *cq;
1755 struct msg_req *req;
1756 int qidx;
1757
1758 /* Ensure all SQE are processed */
1759 otx2_sqb_flush(pf);
1760
1761 /* Stop transmission */
1762 otx2_txschq_stop(pf);
1763
1764 #ifdef CONFIG_DCB
1765 if (pf->pfc_en)
1766 otx2_pfc_txschq_stop(pf);
1767 #endif
1768
1769 if (!otx2_rep_dev(pf->pdev))
1770 otx2_clean_qos_queues(pf);
1771
1772 mutex_lock(&mbox->lock);
1773 /* Disable backpressure */
1774 if (!is_otx2_lbkvf(pf->pdev))
1775 otx2_nix_config_bp(pf, false);
1776 mutex_unlock(&mbox->lock);
1777
1778 /* Disable RQs */
1779 otx2_ctx_disable(mbox, NIX_AQ_CTYPE_RQ, false);
1780
1781 /*Dequeue all CQEs */
1782 for (qidx = 0; qidx < qset->cq_cnt; qidx++) {
1783 cq = &qset->cq[qidx];
1784 if (cq->cq_type == CQ_RX)
1785 otx2_cleanup_rx_cqes(pf, cq, qidx);
1786 else
1787 otx2_cleanup_tx_cqes(pf, cq);
1788 }
1789 otx2_free_pending_sqe(pf);
1790
1791 otx2_free_sq_res(pf);
1792
1793 /* Free RQ buffer pointers*/
1794 otx2_free_aura_ptr(pf, AURA_NIX_RQ);
1795
1796 otx2_free_cq_res(pf);
1797
1798 /* Free all ingress bandwidth profiles allocated */
1799 if (!otx2_rep_dev(pf->pdev))
1800 cn10k_free_all_ipolicers(pf);
1801
1802 mutex_lock(&mbox->lock);
1803 /* Reset NIX LF */
1804 free_req = otx2_mbox_alloc_msg_nix_lf_free(mbox);
1805 if (free_req) {
1806 free_req->flags = NIX_LF_DISABLE_FLOWS;
1807 if (!(pf->flags & OTX2_FLAG_PF_SHUTDOWN))
1808 free_req->flags |= NIX_LF_DONT_FREE_TX_VTAG;
1809 if (otx2_sync_mbox_msg(mbox))
1810 dev_err(pf->dev, "%s failed to free nixlf\n", __func__);
1811 }
1812 mutex_unlock(&mbox->lock);
1813
1814 /* Disable NPA Pool and Aura hw context */
1815 otx2_ctx_disable(mbox, NPA_AQ_CTYPE_POOL, true);
1816 otx2_ctx_disable(mbox, NPA_AQ_CTYPE_AURA, true);
1817 otx2_aura_pool_free(pf);
1818
1819 mutex_lock(&mbox->lock);
1820 /* Reset NPA LF */
1821 req = otx2_mbox_alloc_msg_npa_lf_free(mbox);
1822 if (req) {
1823 if (otx2_sync_mbox_msg(mbox))
1824 dev_err(pf->dev, "%s failed to free npalf\n", __func__);
1825 }
1826 mutex_unlock(&mbox->lock);
1827 }
1828 EXPORT_SYMBOL(otx2_free_hw_resources);
1829
otx2_promisc_use_mce_list(struct otx2_nic * pfvf)1830 static bool otx2_promisc_use_mce_list(struct otx2_nic *pfvf)
1831 {
1832 int vf;
1833
1834 /* The AF driver will determine whether to allow the VF netdev or not */
1835 if (is_otx2_vf(pfvf->pcifunc))
1836 return true;
1837
1838 /* check if there are any trusted VFs associated with the PF netdev */
1839 for (vf = 0; vf < pci_num_vf(pfvf->pdev); vf++)
1840 if (pfvf->vf_configs[vf].trusted)
1841 return true;
1842 return false;
1843 }
1844
otx2_do_set_rx_mode(struct otx2_nic * pf)1845 static void otx2_do_set_rx_mode(struct otx2_nic *pf)
1846 {
1847 struct net_device *netdev = pf->netdev;
1848 struct nix_rx_mode *req;
1849 bool promisc = false;
1850
1851 if (!(netdev->flags & IFF_UP))
1852 return;
1853
1854 if ((netdev->flags & IFF_PROMISC) ||
1855 (netdev_uc_count(netdev) > pf->flow_cfg->ucast_flt_cnt)) {
1856 promisc = true;
1857 }
1858
1859 /* Write unicast address to mcam entries or del from mcam */
1860 if (!promisc && netdev->priv_flags & IFF_UNICAST_FLT)
1861 __dev_uc_sync(netdev, otx2_add_macfilter, otx2_del_macfilter);
1862
1863 mutex_lock(&pf->mbox.lock);
1864 req = otx2_mbox_alloc_msg_nix_set_rx_mode(&pf->mbox);
1865 if (!req) {
1866 mutex_unlock(&pf->mbox.lock);
1867 return;
1868 }
1869
1870 req->mode = NIX_RX_MODE_UCAST;
1871
1872 if (promisc)
1873 req->mode |= NIX_RX_MODE_PROMISC;
1874 if (netdev->flags & (IFF_ALLMULTI | IFF_MULTICAST))
1875 req->mode |= NIX_RX_MODE_ALLMULTI;
1876
1877 if (otx2_promisc_use_mce_list(pf))
1878 req->mode |= NIX_RX_MODE_USE_MCE;
1879
1880 otx2_sync_mbox_msg(&pf->mbox);
1881 mutex_unlock(&pf->mbox.lock);
1882 }
1883
otx2_set_irq_coalesce(struct otx2_nic * pfvf)1884 static void otx2_set_irq_coalesce(struct otx2_nic *pfvf)
1885 {
1886 int cint;
1887
1888 for (cint = 0; cint < pfvf->hw.cint_cnt; cint++)
1889 otx2_config_irq_coalescing(pfvf, cint);
1890 }
1891
otx2_dim_work(struct work_struct * w)1892 static void otx2_dim_work(struct work_struct *w)
1893 {
1894 struct dim_cq_moder cur_moder;
1895 struct otx2_cq_poll *cq_poll;
1896 struct otx2_nic *pfvf;
1897 struct dim *dim;
1898
1899 dim = container_of(w, struct dim, work);
1900 cur_moder = net_dim_get_rx_moderation(dim->mode, dim->profile_ix);
1901 cq_poll = container_of(dim, struct otx2_cq_poll, dim);
1902 pfvf = (struct otx2_nic *)cq_poll->dev;
1903 pfvf->hw.cq_time_wait = (cur_moder.usec > CQ_TIMER_THRESH_MAX) ?
1904 CQ_TIMER_THRESH_MAX : cur_moder.usec;
1905 pfvf->hw.cq_ecount_wait = (cur_moder.pkts > NAPI_POLL_WEIGHT) ?
1906 NAPI_POLL_WEIGHT : cur_moder.pkts;
1907 otx2_set_irq_coalesce(pfvf);
1908 dim->state = DIM_START_MEASURE;
1909 }
1910
otx2_free_queue_mem(struct otx2_qset * qset)1911 void otx2_free_queue_mem(struct otx2_qset *qset)
1912 {
1913 kfree(qset->sq);
1914 qset->sq = NULL;
1915 kfree(qset->cq);
1916 qset->cq = NULL;
1917 kfree(qset->rq);
1918 qset->rq = NULL;
1919 kfree(qset->napi);
1920 qset->napi = NULL;
1921 }
1922 EXPORT_SYMBOL(otx2_free_queue_mem);
1923
otx2_alloc_queue_mem(struct otx2_nic * pf)1924 int otx2_alloc_queue_mem(struct otx2_nic *pf)
1925 {
1926 struct otx2_qset *qset = &pf->qset;
1927 struct otx2_cq_poll *cq_poll;
1928
1929
1930 /* RQ and SQs are mapped to different CQs,
1931 * so find out max CQ IRQs (i.e CINTs) needed.
1932 */
1933 pf->hw.non_qos_queues = pf->hw.tx_queues + pf->hw.xdp_queues;
1934 pf->hw.cint_cnt = max3(pf->hw.rx_queues, pf->hw.tx_queues,
1935 pf->hw.tc_tx_queues);
1936
1937 pf->qset.cq_cnt = pf->hw.rx_queues + otx2_get_total_tx_queues(pf);
1938
1939 qset->napi = kzalloc_objs(*cq_poll, pf->hw.cint_cnt);
1940 if (!qset->napi)
1941 return -ENOMEM;
1942
1943 /* CQ size of RQ */
1944 qset->rqe_cnt = qset->rqe_cnt ? qset->rqe_cnt : Q_COUNT(Q_SIZE_256);
1945 /* CQ size of SQ */
1946 qset->sqe_cnt = qset->sqe_cnt ? qset->sqe_cnt : Q_COUNT(Q_SIZE_4K);
1947
1948 qset->cq = kzalloc_objs(struct otx2_cq_queue, pf->qset.cq_cnt);
1949 if (!qset->cq)
1950 goto err_free_mem;
1951
1952 qset->sq = kzalloc_objs(struct otx2_snd_queue,
1953 otx2_get_total_tx_queues(pf));
1954 if (!qset->sq)
1955 goto err_free_mem;
1956
1957 qset->rq = kzalloc_objs(struct otx2_rcv_queue, pf->hw.rx_queues);
1958 if (!qset->rq)
1959 goto err_free_mem;
1960
1961 return 0;
1962
1963 err_free_mem:
1964 otx2_free_queue_mem(qset);
1965 return -ENOMEM;
1966 }
1967 EXPORT_SYMBOL(otx2_alloc_queue_mem);
1968
otx2_open(struct net_device * netdev)1969 int otx2_open(struct net_device *netdev)
1970 {
1971 struct otx2_nic *pf = netdev_priv(netdev);
1972 struct otx2_cq_poll *cq_poll = NULL;
1973 struct otx2_qset *qset = &pf->qset;
1974 int err = 0, qidx, vec;
1975 char *irq_name;
1976
1977 netif_carrier_off(netdev);
1978
1979 err = otx2_alloc_queue_mem(pf);
1980 if (err)
1981 return err;
1982
1983 err = otx2_init_hw_resources(pf);
1984 if (err)
1985 goto err_free_mem;
1986
1987 /* Register NAPI handler */
1988 for (qidx = 0; qidx < pf->hw.cint_cnt; qidx++) {
1989 cq_poll = &qset->napi[qidx];
1990 cq_poll->cint_idx = qidx;
1991 /* RQ0 & SQ0 are mapped to CINT0 and so on..
1992 * 'cq_ids[0]' points to RQ's CQ and
1993 * 'cq_ids[1]' points to SQ's CQ and
1994 * 'cq_ids[2]' points to XDP's CQ and
1995 */
1996 cq_poll->cq_ids[CQ_RX] =
1997 (qidx < pf->hw.rx_queues) ? qidx : CINT_INVALID_CQ;
1998 cq_poll->cq_ids[CQ_TX] = (qidx < pf->hw.tx_queues) ?
1999 qidx + pf->hw.rx_queues : CINT_INVALID_CQ;
2000 if (pf->xdp_prog)
2001 cq_poll->cq_ids[CQ_XDP] = (qidx < pf->hw.xdp_queues) ?
2002 (qidx + pf->hw.rx_queues +
2003 pf->hw.tx_queues) :
2004 CINT_INVALID_CQ;
2005 else
2006 cq_poll->cq_ids[CQ_XDP] = CINT_INVALID_CQ;
2007
2008 cq_poll->cq_ids[CQ_QOS] = (qidx < pf->hw.tc_tx_queues) ?
2009 (qidx + pf->hw.rx_queues +
2010 pf->hw.non_qos_queues) :
2011 CINT_INVALID_CQ;
2012
2013 cq_poll->dev = (void *)pf;
2014 cq_poll->dim.mode = DIM_CQ_PERIOD_MODE_START_FROM_CQE;
2015 INIT_WORK(&cq_poll->dim.work, otx2_dim_work);
2016 netif_napi_add(netdev, &cq_poll->napi, otx2_napi_handler);
2017 napi_enable(&cq_poll->napi);
2018 }
2019
2020 /* Set maximum frame size allowed in HW */
2021 err = otx2_hw_set_mtu(pf, netdev->mtu);
2022 if (err)
2023 goto err_disable_napi;
2024
2025 /* Setup segmentation algorithms, if failed, clear offload capability */
2026 otx2_setup_segmentation(pf);
2027
2028 /* Initialize RSS */
2029 err = otx2_rss_init(pf);
2030 if (err)
2031 goto err_disable_napi;
2032
2033 /* Register Queue IRQ handlers */
2034 vec = pf->hw.nix_msixoff + NIX_LF_QINT_VEC_START;
2035 irq_name = &pf->hw.irq_name[vec * NAME_SIZE];
2036
2037 snprintf(irq_name, NAME_SIZE, "%s-qerr", pf->netdev->name);
2038
2039 err = request_irq(pci_irq_vector(pf->pdev, vec),
2040 otx2_q_intr_handler, 0, irq_name, pf);
2041 if (err) {
2042 dev_err(pf->dev,
2043 "RVUPF%d: IRQ registration failed for QERR\n",
2044 rvu_get_pf(pf->pdev, pf->pcifunc));
2045 goto err_disable_napi;
2046 }
2047
2048 /* Enable QINT IRQ */
2049 otx2_write64(pf, NIX_LF_QINTX_ENA_W1S(0), BIT_ULL(0));
2050
2051 /* Register CQ IRQ handlers */
2052 vec = pf->hw.nix_msixoff + NIX_LF_CINT_VEC_START;
2053 for (qidx = 0; qidx < pf->hw.cint_cnt; qidx++) {
2054 irq_name = &pf->hw.irq_name[vec * NAME_SIZE];
2055 int name_len;
2056
2057 name_len = snprintf(irq_name, NAME_SIZE, "%s-rxtx-%d",
2058 pf->netdev->name, qidx);
2059 if (name_len >= NAME_SIZE) {
2060 dev_err(pf->dev,
2061 "RVUPF%d: IRQ registration failed for CQ%d, irq name is too long\n",
2062 rvu_get_pf(pf->pdev, pf->pcifunc), qidx);
2063 err = -EINVAL;
2064 goto err_free_cints;
2065 }
2066
2067 err = request_irq(pci_irq_vector(pf->pdev, vec),
2068 otx2_cq_intr_handler, 0, irq_name,
2069 &qset->napi[qidx]);
2070 if (err) {
2071 dev_err(pf->dev,
2072 "RVUPF%d: IRQ registration failed for CQ%d\n",
2073 rvu_get_pf(pf->pdev, pf->pcifunc), qidx);
2074 goto err_free_cints;
2075 }
2076 vec++;
2077
2078 otx2_config_irq_coalescing(pf, qidx);
2079
2080 /* Enable CQ IRQ */
2081 otx2_write64(pf, NIX_LF_CINTX_INT(qidx), BIT_ULL(0));
2082 otx2_write64(pf, NIX_LF_CINTX_ENA_W1S(qidx), BIT_ULL(0));
2083 }
2084
2085 otx2_set_cints_affinity(pf);
2086
2087 if (pf->flags & OTX2_FLAG_RX_VLAN_SUPPORT)
2088 otx2_enable_rxvlan(pf, true);
2089
2090 /* When reinitializing enable time stamping if it is enabled before */
2091 if (pf->flags & OTX2_FLAG_TX_TSTAMP_ENABLED) {
2092 pf->flags &= ~OTX2_FLAG_TX_TSTAMP_ENABLED;
2093 otx2_config_hw_tx_tstamp(pf, true);
2094 }
2095 if (pf->flags & OTX2_FLAG_RX_TSTAMP_ENABLED) {
2096 pf->flags &= ~OTX2_FLAG_RX_TSTAMP_ENABLED;
2097 otx2_config_hw_rx_tstamp(pf, true);
2098 }
2099
2100 pf->flags &= ~OTX2_FLAG_INTF_DOWN;
2101 pf->flags &= ~OTX2_FLAG_PORT_UP;
2102 /* 'intf_down' may be checked on any cpu */
2103 smp_wmb();
2104
2105 /* Enable QoS configuration before starting tx queues */
2106 otx2_qos_config_txschq(pf);
2107
2108 /* we have already received link status notification */
2109 if (pf->linfo.link_up && !(pf->pcifunc & RVU_PFVF_FUNC_MASK))
2110 otx2_handle_link_event(pf);
2111
2112 /* Install DMAC Filters */
2113 if (pf->flags & OTX2_FLAG_DMACFLTR_SUPPORT)
2114 otx2_dmacflt_reinstall_flows(pf);
2115
2116 otx2_tc_apply_ingress_police_rules(pf);
2117
2118 err = otx2_rxtx_enable(pf, true);
2119 /* If a mbox communication error happens at this point then interface
2120 * will end up in a state such that it is in down state but hardware
2121 * mcam entries are enabled to receive the packets. Hence disable the
2122 * packet I/O.
2123 */
2124 if (err == -EIO)
2125 goto err_disable_rxtx;
2126 else if (err)
2127 goto err_tx_stop_queues;
2128
2129 otx2_do_set_rx_mode(pf);
2130
2131 return 0;
2132
2133 err_disable_rxtx:
2134 otx2_rxtx_enable(pf, false);
2135 err_tx_stop_queues:
2136 netif_tx_stop_all_queues(netdev);
2137 netif_carrier_off(netdev);
2138 pf->flags |= OTX2_FLAG_INTF_DOWN;
2139 err_free_cints:
2140 otx2_free_cints(pf, qidx);
2141 vec = pci_irq_vector(pf->pdev,
2142 pf->hw.nix_msixoff + NIX_LF_QINT_VEC_START);
2143 otx2_write64(pf, NIX_LF_QINTX_ENA_W1C(0), BIT_ULL(0));
2144 free_irq(vec, pf);
2145 err_disable_napi:
2146 otx2_disable_napi(pf);
2147 otx2_free_hw_resources(pf);
2148 err_free_mem:
2149 otx2_free_queue_mem(qset);
2150 return err;
2151 }
2152 EXPORT_SYMBOL(otx2_open);
2153
otx2_stop(struct net_device * netdev)2154 int otx2_stop(struct net_device *netdev)
2155 {
2156 struct otx2_nic *pf = netdev_priv(netdev);
2157 struct otx2_cq_poll *cq_poll = NULL;
2158 struct otx2_qset *qset = &pf->qset;
2159 int qidx, vec, wrk;
2160
2161 /* If the DOWN flag is set resources are already freed */
2162 if (pf->flags & OTX2_FLAG_INTF_DOWN)
2163 return 0;
2164
2165 netif_carrier_off(netdev);
2166 netif_tx_stop_all_queues(netdev);
2167
2168 pf->flags |= OTX2_FLAG_INTF_DOWN;
2169 /* 'intf_down' may be checked on any cpu */
2170 smp_wmb();
2171
2172 /* First stop packet Rx/Tx */
2173 otx2_rxtx_enable(pf, false);
2174
2175 /* Clear RSS enable flag */
2176 pf->hw.rss_info.enable = false;
2177
2178 /* Cleanup Queue IRQ */
2179 vec = pci_irq_vector(pf->pdev,
2180 pf->hw.nix_msixoff + NIX_LF_QINT_VEC_START);
2181 otx2_write64(pf, NIX_LF_QINTX_ENA_W1C(0), BIT_ULL(0));
2182 free_irq(vec, pf);
2183
2184 /* Cleanup CQ NAPI and IRQ */
2185 vec = pf->hw.nix_msixoff + NIX_LF_CINT_VEC_START;
2186 for (qidx = 0; qidx < pf->hw.cint_cnt; qidx++) {
2187 /* Disable interrupt */
2188 otx2_write64(pf, NIX_LF_CINTX_ENA_W1C(qidx), BIT_ULL(0));
2189
2190 synchronize_irq(pci_irq_vector(pf->pdev, vec));
2191
2192 cq_poll = &qset->napi[qidx];
2193 napi_synchronize(&cq_poll->napi);
2194 vec++;
2195 }
2196
2197 netif_tx_disable(netdev);
2198
2199 for (wrk = 0; wrk < pf->qset.cq_cnt; wrk++)
2200 cancel_delayed_work_sync(&pf->refill_wrk[wrk].pool_refill_work);
2201 devm_kfree(pf->dev, pf->refill_wrk);
2202
2203 otx2_free_hw_resources(pf);
2204 otx2_free_cints(pf, pf->hw.cint_cnt);
2205 otx2_disable_napi(pf);
2206
2207 for (qidx = 0; qidx < netdev->num_tx_queues; qidx++)
2208 netdev_tx_reset_queue(netdev_get_tx_queue(netdev, qidx));
2209
2210 otx2_free_queue_mem(qset);
2211 /* Do not clear RQ/SQ ringsize settings */
2212 memset_startat(qset, 0, sqe_cnt);
2213 return 0;
2214 }
2215 EXPORT_SYMBOL(otx2_stop);
2216
otx2_xmit(struct sk_buff * skb,struct net_device * netdev)2217 static netdev_tx_t otx2_xmit(struct sk_buff *skb, struct net_device *netdev)
2218 {
2219 struct otx2_nic *pf = netdev_priv(netdev);
2220 int qidx = skb_get_queue_mapping(skb);
2221 struct otx2_dev_stats *dev_stats;
2222 struct otx2_snd_queue *sq;
2223 struct netdev_queue *txq;
2224 int sq_idx;
2225
2226 /* XDP SQs are not mapped with TXQs
2227 * advance qid to derive correct sq mapped with QOS
2228 */
2229 sq_idx = (qidx >= pf->hw.tx_queues) ? (qidx + pf->hw.xdp_queues) : qidx;
2230
2231 /* Check for minimum and maximum packet length */
2232 if (skb->len <= ETH_HLEN ||
2233 (!skb_shinfo(skb)->gso_size && skb->len > pf->tx_max_pktlen)) {
2234 dev_stats = &pf->hw.dev_stats;
2235 atomic_long_inc(&dev_stats->tx_discards);
2236 dev_kfree_skb(skb);
2237 return NETDEV_TX_OK;
2238 }
2239
2240 sq = &pf->qset.sq[sq_idx];
2241 txq = netdev_get_tx_queue(netdev, qidx);
2242
2243 if (!otx2_sq_append_skb(pf, txq, sq, skb, qidx)) {
2244 netif_tx_stop_queue(txq);
2245
2246 /* Check again, incase SQBs got freed up */
2247 smp_mb();
2248 if (((sq->num_sqbs - *sq->aura_fc_addr) * sq->sqe_per_sqb)
2249 > sq->sqe_thresh)
2250 netif_tx_wake_queue(txq);
2251
2252 return NETDEV_TX_BUSY;
2253 }
2254
2255 return NETDEV_TX_OK;
2256 }
2257
otx2_qos_select_htb_queue(struct otx2_nic * pf,struct sk_buff * skb,u16 htb_maj_id)2258 static int otx2_qos_select_htb_queue(struct otx2_nic *pf, struct sk_buff *skb,
2259 u16 htb_maj_id)
2260 {
2261 u16 classid;
2262
2263 if ((TC_H_MAJ(skb->priority) >> 16) == htb_maj_id)
2264 classid = TC_H_MIN(skb->priority);
2265 else
2266 classid = READ_ONCE(pf->qos.defcls);
2267
2268 if (!classid)
2269 return 0;
2270
2271 return otx2_get_txq_by_classid(pf, classid);
2272 }
2273
otx2_select_queue(struct net_device * netdev,struct sk_buff * skb,struct net_device * sb_dev)2274 u16 otx2_select_queue(struct net_device *netdev, struct sk_buff *skb,
2275 struct net_device *sb_dev)
2276 {
2277 struct otx2_nic *pf = netdev_priv(netdev);
2278 bool qos_enabled;
2279 #ifdef CONFIG_DCB
2280 u8 vlan_prio;
2281 #endif
2282 int txq;
2283
2284 qos_enabled = netdev->real_num_tx_queues > pf->hw.tx_queues;
2285 if (unlikely(qos_enabled)) {
2286 /* This smp_load_acquire() pairs with smp_store_release() in
2287 * otx2_qos_root_add() called from htb offload root creation
2288 */
2289 u16 htb_maj_id = smp_load_acquire(&pf->qos.maj_id);
2290
2291 if (unlikely(htb_maj_id)) {
2292 txq = otx2_qos_select_htb_queue(pf, skb, htb_maj_id);
2293 if (txq > 0)
2294 return txq;
2295 goto process_pfc;
2296 }
2297 }
2298
2299 process_pfc:
2300 #ifdef CONFIG_DCB
2301 if (!skb_vlan_tag_present(skb))
2302 goto pick_tx;
2303
2304 vlan_prio = skb->vlan_tci >> 13;
2305 if ((vlan_prio > pf->hw.tx_queues - 1) ||
2306 !pf->pfc_alloc_status[vlan_prio])
2307 goto pick_tx;
2308
2309 return vlan_prio;
2310
2311 pick_tx:
2312 #endif
2313 txq = netdev_pick_tx(netdev, skb, NULL);
2314 if (unlikely(qos_enabled))
2315 return txq % pf->hw.tx_queues;
2316
2317 return txq;
2318 }
2319 EXPORT_SYMBOL(otx2_select_queue);
2320
otx2_fix_features(struct net_device * dev,netdev_features_t features)2321 static netdev_features_t otx2_fix_features(struct net_device *dev,
2322 netdev_features_t features)
2323 {
2324 if (features & NETIF_F_HW_VLAN_CTAG_RX)
2325 features |= NETIF_F_HW_VLAN_STAG_RX;
2326 else
2327 features &= ~NETIF_F_HW_VLAN_STAG_RX;
2328
2329 return features;
2330 }
2331
otx2_set_rx_mode(struct net_device * netdev)2332 static void otx2_set_rx_mode(struct net_device *netdev)
2333 {
2334 struct otx2_nic *pf = netdev_priv(netdev);
2335
2336 queue_work(pf->otx2_wq, &pf->rx_mode_work);
2337 }
2338
otx2_rx_mode_wrk_handler(struct work_struct * work)2339 static void otx2_rx_mode_wrk_handler(struct work_struct *work)
2340 {
2341 struct otx2_nic *pf = container_of(work, struct otx2_nic, rx_mode_work);
2342
2343 otx2_do_set_rx_mode(pf);
2344 }
2345
otx2_set_features(struct net_device * netdev,netdev_features_t features)2346 static int otx2_set_features(struct net_device *netdev,
2347 netdev_features_t features)
2348 {
2349 netdev_features_t changed = features ^ netdev->features;
2350 struct otx2_nic *pf = netdev_priv(netdev);
2351
2352 if ((changed & NETIF_F_LOOPBACK) && netif_running(netdev))
2353 return otx2_cgx_config_loopback(pf,
2354 features & NETIF_F_LOOPBACK);
2355
2356 if ((changed & NETIF_F_HW_VLAN_CTAG_RX) && netif_running(netdev))
2357 return otx2_enable_rxvlan(pf,
2358 features & NETIF_F_HW_VLAN_CTAG_RX);
2359
2360 if (changed & NETIF_F_HW_ESP)
2361 return cn10k_ipsec_ethtool_init(netdev,
2362 features & NETIF_F_HW_ESP);
2363
2364 return otx2_handle_ntuple_tc_features(netdev, features);
2365 }
2366
otx2_reset_task(struct work_struct * work)2367 static void otx2_reset_task(struct work_struct *work)
2368 {
2369 struct otx2_nic *pf = container_of(work, struct otx2_nic, reset_task);
2370
2371 if (!netif_running(pf->netdev))
2372 return;
2373
2374 rtnl_lock();
2375 otx2_stop(pf->netdev);
2376 pf->reset_count++;
2377 otx2_open(pf->netdev);
2378 netif_trans_update(pf->netdev);
2379 rtnl_unlock();
2380 }
2381
otx2_config_hw_rx_tstamp(struct otx2_nic * pfvf,bool enable)2382 static int otx2_config_hw_rx_tstamp(struct otx2_nic *pfvf, bool enable)
2383 {
2384 struct msg_req *req;
2385 int err;
2386
2387 if (pfvf->flags & OTX2_FLAG_RX_TSTAMP_ENABLED && enable)
2388 return 0;
2389
2390 mutex_lock(&pfvf->mbox.lock);
2391 if (enable)
2392 req = otx2_mbox_alloc_msg_cgx_ptp_rx_enable(&pfvf->mbox);
2393 else
2394 req = otx2_mbox_alloc_msg_cgx_ptp_rx_disable(&pfvf->mbox);
2395 if (!req) {
2396 mutex_unlock(&pfvf->mbox.lock);
2397 return -ENOMEM;
2398 }
2399
2400 err = otx2_sync_mbox_msg(&pfvf->mbox);
2401 if (err) {
2402 mutex_unlock(&pfvf->mbox.lock);
2403 return err;
2404 }
2405
2406 mutex_unlock(&pfvf->mbox.lock);
2407 if (enable)
2408 pfvf->flags |= OTX2_FLAG_RX_TSTAMP_ENABLED;
2409 else
2410 pfvf->flags &= ~OTX2_FLAG_RX_TSTAMP_ENABLED;
2411 return 0;
2412 }
2413
otx2_config_hw_tx_tstamp(struct otx2_nic * pfvf,bool enable)2414 static int otx2_config_hw_tx_tstamp(struct otx2_nic *pfvf, bool enable)
2415 {
2416 struct msg_req *req;
2417 int err;
2418
2419 if (pfvf->flags & OTX2_FLAG_TX_TSTAMP_ENABLED && enable)
2420 return 0;
2421
2422 mutex_lock(&pfvf->mbox.lock);
2423 if (enable)
2424 req = otx2_mbox_alloc_msg_nix_lf_ptp_tx_enable(&pfvf->mbox);
2425 else
2426 req = otx2_mbox_alloc_msg_nix_lf_ptp_tx_disable(&pfvf->mbox);
2427 if (!req) {
2428 mutex_unlock(&pfvf->mbox.lock);
2429 return -ENOMEM;
2430 }
2431
2432 err = otx2_sync_mbox_msg(&pfvf->mbox);
2433 if (err) {
2434 mutex_unlock(&pfvf->mbox.lock);
2435 return err;
2436 }
2437
2438 mutex_unlock(&pfvf->mbox.lock);
2439 if (enable)
2440 pfvf->flags |= OTX2_FLAG_TX_TSTAMP_ENABLED;
2441 else
2442 pfvf->flags &= ~OTX2_FLAG_TX_TSTAMP_ENABLED;
2443 return 0;
2444 }
2445
otx2_config_hwtstamp_get(struct net_device * netdev,struct kernel_hwtstamp_config * config)2446 int otx2_config_hwtstamp_get(struct net_device *netdev,
2447 struct kernel_hwtstamp_config *config)
2448 {
2449 struct otx2_nic *pfvf = netdev_priv(netdev);
2450
2451 *config = pfvf->tstamp;
2452 return 0;
2453 }
2454 EXPORT_SYMBOL(otx2_config_hwtstamp_get);
2455
otx2_config_hwtstamp_set(struct net_device * netdev,struct kernel_hwtstamp_config * config,struct netlink_ext_ack * extack)2456 int otx2_config_hwtstamp_set(struct net_device *netdev,
2457 struct kernel_hwtstamp_config *config,
2458 struct netlink_ext_ack *extack)
2459 {
2460 struct otx2_nic *pfvf = netdev_priv(netdev);
2461
2462 if (!pfvf->ptp)
2463 return -ENODEV;
2464
2465 switch (config->tx_type) {
2466 case HWTSTAMP_TX_OFF:
2467 if (pfvf->flags & OTX2_FLAG_PTP_ONESTEP_SYNC)
2468 pfvf->flags &= ~OTX2_FLAG_PTP_ONESTEP_SYNC;
2469
2470 cancel_delayed_work(&pfvf->ptp->synctstamp_work);
2471 otx2_config_hw_tx_tstamp(pfvf, false);
2472 break;
2473 case HWTSTAMP_TX_ONESTEP_SYNC:
2474 if (!test_bit(CN10K_PTP_ONESTEP, &pfvf->hw.cap_flag)) {
2475 NL_SET_ERR_MSG_MOD(extack,
2476 "One-step time stamping is not supported");
2477 return -ERANGE;
2478 }
2479 pfvf->flags |= OTX2_FLAG_PTP_ONESTEP_SYNC;
2480 schedule_delayed_work(&pfvf->ptp->synctstamp_work,
2481 msecs_to_jiffies(500));
2482 fallthrough;
2483 case HWTSTAMP_TX_ON:
2484 otx2_config_hw_tx_tstamp(pfvf, true);
2485 break;
2486 default:
2487 return -ERANGE;
2488 }
2489
2490 switch (config->rx_filter) {
2491 case HWTSTAMP_FILTER_NONE:
2492 otx2_config_hw_rx_tstamp(pfvf, false);
2493 break;
2494 case HWTSTAMP_FILTER_ALL:
2495 case HWTSTAMP_FILTER_SOME:
2496 case HWTSTAMP_FILTER_PTP_V1_L4_EVENT:
2497 case HWTSTAMP_FILTER_PTP_V1_L4_SYNC:
2498 case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ:
2499 case HWTSTAMP_FILTER_PTP_V2_L4_EVENT:
2500 case HWTSTAMP_FILTER_PTP_V2_L4_SYNC:
2501 case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ:
2502 case HWTSTAMP_FILTER_PTP_V2_L2_EVENT:
2503 case HWTSTAMP_FILTER_PTP_V2_L2_SYNC:
2504 case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ:
2505 case HWTSTAMP_FILTER_PTP_V2_EVENT:
2506 case HWTSTAMP_FILTER_PTP_V2_SYNC:
2507 case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ:
2508 otx2_config_hw_rx_tstamp(pfvf, true);
2509 config->rx_filter = HWTSTAMP_FILTER_ALL;
2510 break;
2511 default:
2512 return -ERANGE;
2513 }
2514
2515 pfvf->tstamp = *config;
2516
2517 return 0;
2518 }
2519 EXPORT_SYMBOL(otx2_config_hwtstamp_set);
2520
otx2_do_set_vf_mac(struct otx2_nic * pf,int vf,const u8 * mac)2521 static int otx2_do_set_vf_mac(struct otx2_nic *pf, int vf, const u8 *mac)
2522 {
2523 struct npc_install_flow_req *req;
2524 int err;
2525
2526 mutex_lock(&pf->mbox.lock);
2527 req = otx2_mbox_alloc_msg_npc_install_flow(&pf->mbox);
2528 if (!req) {
2529 err = -ENOMEM;
2530 goto out;
2531 }
2532
2533 ether_addr_copy(req->packet.dmac, mac);
2534 eth_broadcast_addr((u8 *)&req->mask.dmac);
2535 req->features = BIT_ULL(NPC_DMAC);
2536 req->channel = pf->hw.rx_chan_base;
2537 req->intf = NIX_INTF_RX;
2538 req->default_rule = 1;
2539 req->append = 1;
2540 req->vf = vf + 1;
2541 req->op = NIX_RX_ACTION_DEFAULT;
2542
2543 err = otx2_sync_mbox_msg(&pf->mbox);
2544 out:
2545 mutex_unlock(&pf->mbox.lock);
2546 return err;
2547 }
2548
otx2_set_vf_mac(struct net_device * netdev,int vf,u8 * mac)2549 static int otx2_set_vf_mac(struct net_device *netdev, int vf, u8 *mac)
2550 {
2551 struct otx2_nic *pf = netdev_priv(netdev);
2552 struct pci_dev *pdev = pf->pdev;
2553 struct otx2_vf_config *config;
2554 int ret;
2555
2556 if (!netif_running(netdev))
2557 return -EAGAIN;
2558
2559 if (vf >= pf->total_vfs)
2560 return -EINVAL;
2561
2562 if (!is_valid_ether_addr(mac))
2563 return -EINVAL;
2564
2565 config = &pf->vf_configs[vf];
2566 ether_addr_copy(config->mac, mac);
2567
2568 ret = otx2_do_set_vf_mac(pf, vf, mac);
2569 if (ret == 0)
2570 dev_info(&pdev->dev,
2571 "Load/Reload VF driver\n");
2572
2573 return ret;
2574 }
2575
otx2_do_set_vf_vlan(struct otx2_nic * pf,int vf,u16 vlan,u8 qos,__be16 proto)2576 static int otx2_do_set_vf_vlan(struct otx2_nic *pf, int vf, u16 vlan, u8 qos,
2577 __be16 proto)
2578 {
2579 struct otx2_flow_config *flow_cfg = pf->flow_cfg;
2580 struct nix_vtag_config_rsp *vtag_rsp;
2581 struct npc_delete_flow_req *del_req;
2582 struct nix_vtag_config *vtag_req;
2583 struct npc_install_flow_req *req;
2584 struct otx2_vf_config *config;
2585 int err = 0;
2586 u32 idx;
2587
2588 config = &pf->vf_configs[vf];
2589
2590 if (!vlan && !config->vlan)
2591 goto out;
2592
2593 mutex_lock(&pf->mbox.lock);
2594
2595 /* free old tx vtag entry */
2596 if (config->vlan) {
2597 vtag_req = otx2_mbox_alloc_msg_nix_vtag_cfg(&pf->mbox);
2598 if (!vtag_req) {
2599 err = -ENOMEM;
2600 goto out;
2601 }
2602 vtag_req->cfg_type = 0;
2603 vtag_req->tx.free_vtag0 = 1;
2604 vtag_req->tx.vtag0_idx = config->tx_vtag_idx;
2605
2606 err = otx2_sync_mbox_msg(&pf->mbox);
2607 if (err)
2608 goto out;
2609 }
2610
2611 if (!vlan && config->vlan) {
2612 /* rx */
2613 del_req = otx2_mbox_alloc_msg_npc_delete_flow(&pf->mbox);
2614 if (!del_req) {
2615 err = -ENOMEM;
2616 goto out;
2617 }
2618 idx = ((vf * OTX2_PER_VF_VLAN_FLOWS) + OTX2_VF_VLAN_RX_INDEX);
2619 del_req->entry =
2620 flow_cfg->def_ent[flow_cfg->vf_vlan_offset + idx];
2621 err = otx2_sync_mbox_msg(&pf->mbox);
2622 if (err)
2623 goto out;
2624
2625 /* tx */
2626 del_req = otx2_mbox_alloc_msg_npc_delete_flow(&pf->mbox);
2627 if (!del_req) {
2628 err = -ENOMEM;
2629 goto out;
2630 }
2631 idx = ((vf * OTX2_PER_VF_VLAN_FLOWS) + OTX2_VF_VLAN_TX_INDEX);
2632 del_req->entry =
2633 flow_cfg->def_ent[flow_cfg->vf_vlan_offset + idx];
2634 err = otx2_sync_mbox_msg(&pf->mbox);
2635
2636 goto out;
2637 }
2638
2639 /* rx */
2640 req = otx2_mbox_alloc_msg_npc_install_flow(&pf->mbox);
2641 if (!req) {
2642 err = -ENOMEM;
2643 goto out;
2644 }
2645
2646 idx = ((vf * OTX2_PER_VF_VLAN_FLOWS) + OTX2_VF_VLAN_RX_INDEX);
2647 req->entry = flow_cfg->def_ent[flow_cfg->vf_vlan_offset + idx];
2648 req->packet.vlan_tci = htons(vlan);
2649 req->mask.vlan_tci = htons(VLAN_VID_MASK);
2650 /* af fills the destination mac addr */
2651 eth_broadcast_addr((u8 *)&req->mask.dmac);
2652 req->features = BIT_ULL(NPC_OUTER_VID) | BIT_ULL(NPC_DMAC);
2653 req->channel = pf->hw.rx_chan_base;
2654 req->intf = NIX_INTF_RX;
2655 req->vf = vf + 1;
2656 req->op = NIX_RX_ACTION_DEFAULT;
2657 req->vtag0_valid = true;
2658 req->vtag0_type = NIX_AF_LFX_RX_VTAG_TYPE7;
2659 req->set_cntr = 1;
2660
2661 err = otx2_sync_mbox_msg(&pf->mbox);
2662 if (err)
2663 goto out;
2664
2665 /* tx */
2666 vtag_req = otx2_mbox_alloc_msg_nix_vtag_cfg(&pf->mbox);
2667 if (!vtag_req) {
2668 err = -ENOMEM;
2669 goto out;
2670 }
2671
2672 /* configure tx vtag params */
2673 vtag_req->vtag_size = VTAGSIZE_T4;
2674 vtag_req->cfg_type = 0; /* tx vlan cfg */
2675 vtag_req->tx.cfg_vtag0 = 1;
2676 vtag_req->tx.vtag0 = ((u64)ntohs(proto) << 16) | vlan;
2677
2678 err = otx2_sync_mbox_msg(&pf->mbox);
2679 if (err)
2680 goto out;
2681
2682 vtag_rsp = (struct nix_vtag_config_rsp *)otx2_mbox_get_rsp
2683 (&pf->mbox.mbox, 0, &vtag_req->hdr);
2684 if (IS_ERR(vtag_rsp)) {
2685 err = PTR_ERR(vtag_rsp);
2686 goto out;
2687 }
2688 config->tx_vtag_idx = vtag_rsp->vtag0_idx;
2689
2690 req = otx2_mbox_alloc_msg_npc_install_flow(&pf->mbox);
2691 if (!req) {
2692 err = -ENOMEM;
2693 goto out;
2694 }
2695
2696 eth_zero_addr((u8 *)&req->mask.dmac);
2697 idx = ((vf * OTX2_PER_VF_VLAN_FLOWS) + OTX2_VF_VLAN_TX_INDEX);
2698 req->entry = flow_cfg->def_ent[flow_cfg->vf_vlan_offset + idx];
2699 req->features = BIT_ULL(NPC_DMAC);
2700 req->channel = pf->hw.tx_chan_base;
2701 req->intf = NIX_INTF_TX;
2702 req->vf = vf + 1;
2703 req->op = NIX_TX_ACTIONOP_UCAST_DEFAULT;
2704 req->vtag0_def = vtag_rsp->vtag0_idx;
2705 req->vtag0_op = VTAG_INSERT;
2706 req->set_cntr = 1;
2707
2708 err = otx2_sync_mbox_msg(&pf->mbox);
2709 out:
2710 config->vlan = vlan;
2711 mutex_unlock(&pf->mbox.lock);
2712 return err;
2713 }
2714
otx2_set_vf_vlan(struct net_device * netdev,int vf,u16 vlan,u8 qos,__be16 proto)2715 static int otx2_set_vf_vlan(struct net_device *netdev, int vf, u16 vlan, u8 qos,
2716 __be16 proto)
2717 {
2718 struct otx2_nic *pf = netdev_priv(netdev);
2719 struct pci_dev *pdev = pf->pdev;
2720
2721 if (!netif_running(netdev))
2722 return -EAGAIN;
2723
2724 if (vf >= pci_num_vf(pdev))
2725 return -EINVAL;
2726
2727 /* qos is currently unsupported */
2728 if (vlan >= VLAN_N_VID || qos)
2729 return -EINVAL;
2730
2731 if (proto != htons(ETH_P_8021Q))
2732 return -EPROTONOSUPPORT;
2733
2734 if (!(pf->flags & OTX2_FLAG_VF_VLAN_SUPPORT))
2735 return -EOPNOTSUPP;
2736
2737 return otx2_do_set_vf_vlan(pf, vf, vlan, qos, proto);
2738 }
2739
otx2_get_vf_config(struct net_device * netdev,int vf,struct ifla_vf_info * ivi)2740 static int otx2_get_vf_config(struct net_device *netdev, int vf,
2741 struct ifla_vf_info *ivi)
2742 {
2743 struct otx2_nic *pf = netdev_priv(netdev);
2744 struct pci_dev *pdev = pf->pdev;
2745 struct otx2_vf_config *config;
2746
2747 if (!netif_running(netdev))
2748 return -EAGAIN;
2749
2750 if (vf >= pci_num_vf(pdev))
2751 return -EINVAL;
2752
2753 config = &pf->vf_configs[vf];
2754 ivi->vf = vf;
2755 ether_addr_copy(ivi->mac, config->mac);
2756 ivi->vlan = config->vlan;
2757 ivi->trusted = config->trusted;
2758
2759 return 0;
2760 }
2761
otx2_xdp_xmit_tx(struct otx2_nic * pf,struct xdp_frame * xdpf,int qidx)2762 static int otx2_xdp_xmit_tx(struct otx2_nic *pf, struct xdp_frame *xdpf,
2763 int qidx)
2764 {
2765 u64 dma_addr;
2766 int err = 0;
2767
2768 dma_addr = otx2_dma_map_page(pf, virt_to_page(xdpf->data),
2769 offset_in_page(xdpf->data), xdpf->len,
2770 DMA_TO_DEVICE);
2771 if (dma_mapping_error(pf->dev, dma_addr))
2772 return -ENOMEM;
2773
2774 err = otx2_xdp_sq_append_pkt(pf, xdpf, dma_addr, xdpf->len,
2775 qidx, OTX2_XDP_REDIRECT);
2776 if (!err) {
2777 otx2_dma_unmap_page(pf, dma_addr, xdpf->len, DMA_TO_DEVICE);
2778 xdp_return_frame(xdpf);
2779 return -ENOMEM;
2780 }
2781 return 0;
2782 }
2783
otx2_xdp_xmit(struct net_device * netdev,int n,struct xdp_frame ** frames,u32 flags)2784 static int otx2_xdp_xmit(struct net_device *netdev, int n,
2785 struct xdp_frame **frames, u32 flags)
2786 {
2787 struct otx2_nic *pf = netdev_priv(netdev);
2788 int qidx = smp_processor_id();
2789 struct otx2_snd_queue *sq;
2790 int drops = 0, i;
2791
2792 if (!netif_running(netdev))
2793 return -ENETDOWN;
2794
2795 qidx += pf->hw.tx_queues;
2796 sq = pf->xdp_prog ? &pf->qset.sq[qidx] : NULL;
2797
2798 /* Abort xmit if xdp queue is not */
2799 if (unlikely(!sq))
2800 return -ENXIO;
2801
2802 if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK))
2803 return -EINVAL;
2804
2805 for (i = 0; i < n; i++) {
2806 struct xdp_frame *xdpf = frames[i];
2807 int err;
2808
2809 err = otx2_xdp_xmit_tx(pf, xdpf, qidx);
2810 if (err)
2811 drops++;
2812 }
2813 return n - drops;
2814 }
2815
otx2_xdp_setup(struct otx2_nic * pf,struct bpf_prog * prog)2816 static int otx2_xdp_setup(struct otx2_nic *pf, struct bpf_prog *prog)
2817 {
2818 struct net_device *dev = pf->netdev;
2819 bool if_up = netif_running(pf->netdev);
2820 struct bpf_prog *old_prog;
2821
2822 if (prog && dev->mtu > MAX_XDP_MTU) {
2823 netdev_warn(dev, "Jumbo frames not yet supported with XDP\n");
2824 return -EOPNOTSUPP;
2825 }
2826
2827 if (if_up)
2828 otx2_stop(pf->netdev);
2829
2830 old_prog = xchg(&pf->xdp_prog, prog);
2831
2832 if (old_prog)
2833 bpf_prog_put(old_prog);
2834
2835 if (pf->xdp_prog)
2836 bpf_prog_add(pf->xdp_prog, pf->hw.rx_queues - 1);
2837
2838 /* Network stack and XDP shared same rx queues.
2839 * Use separate tx queues for XDP and network stack.
2840 */
2841 if (pf->xdp_prog) {
2842 pf->hw.xdp_queues = pf->hw.rx_queues;
2843 xdp_features_set_redirect_target(dev, false);
2844 } else {
2845 pf->hw.xdp_queues = 0;
2846 xdp_features_clear_redirect_target(dev);
2847 }
2848
2849 if (if_up)
2850 otx2_open(pf->netdev);
2851
2852 return 0;
2853 }
2854
otx2_xdp(struct net_device * netdev,struct netdev_bpf * xdp)2855 static int otx2_xdp(struct net_device *netdev, struct netdev_bpf *xdp)
2856 {
2857 struct otx2_nic *pf = netdev_priv(netdev);
2858
2859 switch (xdp->command) {
2860 case XDP_SETUP_PROG:
2861 return otx2_xdp_setup(pf, xdp->prog);
2862 case XDP_SETUP_XSK_POOL:
2863 return otx2_xsk_pool_setup(pf, xdp->xsk.pool, xdp->xsk.queue_id);
2864 default:
2865 return -EINVAL;
2866 }
2867 }
2868
otx2_set_vf_permissions(struct otx2_nic * pf,int vf,int req_perm)2869 static int otx2_set_vf_permissions(struct otx2_nic *pf, int vf,
2870 int req_perm)
2871 {
2872 struct set_vf_perm *req;
2873 int rc;
2874
2875 mutex_lock(&pf->mbox.lock);
2876 req = otx2_mbox_alloc_msg_set_vf_perm(&pf->mbox);
2877 if (!req) {
2878 rc = -ENOMEM;
2879 goto out;
2880 }
2881
2882 /* Let AF reset VF permissions as sriov is disabled */
2883 if (req_perm == OTX2_RESET_VF_PERM) {
2884 req->flags |= RESET_VF_PERM;
2885 } else if (req_perm == OTX2_TRUSTED_VF) {
2886 if (pf->vf_configs[vf].trusted)
2887 req->flags |= VF_TRUSTED;
2888 }
2889
2890 req->vf = vf;
2891 rc = otx2_sync_mbox_msg(&pf->mbox);
2892 out:
2893 mutex_unlock(&pf->mbox.lock);
2894 return rc;
2895 }
2896
otx2_ndo_set_vf_trust(struct net_device * netdev,int vf,bool enable)2897 static int otx2_ndo_set_vf_trust(struct net_device *netdev, int vf,
2898 bool enable)
2899 {
2900 struct otx2_nic *pf = netdev_priv(netdev);
2901 struct pci_dev *pdev = pf->pdev;
2902 int rc;
2903
2904 if (vf >= pci_num_vf(pdev))
2905 return -EINVAL;
2906
2907 if (pf->vf_configs[vf].trusted == enable)
2908 return 0;
2909
2910 pf->vf_configs[vf].trusted = enable;
2911 rc = otx2_set_vf_permissions(pf, vf, OTX2_TRUSTED_VF);
2912
2913 if (rc) {
2914 pf->vf_configs[vf].trusted = !enable;
2915 } else {
2916 netdev_info(pf->netdev, "VF %d is %strusted\n",
2917 vf, enable ? "" : "not ");
2918 otx2_set_rx_mode(netdev);
2919 }
2920
2921 return rc;
2922 }
2923
2924 static const struct net_device_ops otx2_netdev_ops = {
2925 .ndo_open = otx2_open,
2926 .ndo_stop = otx2_stop,
2927 .ndo_start_xmit = otx2_xmit,
2928 .ndo_select_queue = otx2_select_queue,
2929 .ndo_fix_features = otx2_fix_features,
2930 .ndo_set_mac_address = otx2_set_mac_address,
2931 .ndo_change_mtu = otx2_change_mtu,
2932 .ndo_set_rx_mode = otx2_set_rx_mode,
2933 .ndo_set_features = otx2_set_features,
2934 .ndo_tx_timeout = otx2_tx_timeout,
2935 .ndo_get_stats64 = otx2_get_stats64,
2936 .ndo_set_vf_mac = otx2_set_vf_mac,
2937 .ndo_set_vf_vlan = otx2_set_vf_vlan,
2938 .ndo_get_vf_config = otx2_get_vf_config,
2939 .ndo_bpf = otx2_xdp,
2940 .ndo_xsk_wakeup = otx2_xsk_wakeup,
2941 .ndo_xdp_xmit = otx2_xdp_xmit,
2942 .ndo_setup_tc = otx2_setup_tc,
2943 .ndo_set_vf_trust = otx2_ndo_set_vf_trust,
2944 .ndo_hwtstamp_get = otx2_config_hwtstamp_get,
2945 .ndo_hwtstamp_set = otx2_config_hwtstamp_set,
2946 };
2947
otx2_wq_init(struct otx2_nic * pf)2948 int otx2_wq_init(struct otx2_nic *pf)
2949 {
2950 pf->otx2_wq = create_singlethread_workqueue("otx2_wq");
2951 if (!pf->otx2_wq)
2952 return -ENOMEM;
2953
2954 INIT_WORK(&pf->rx_mode_work, otx2_rx_mode_wrk_handler);
2955 INIT_WORK(&pf->reset_task, otx2_reset_task);
2956 return 0;
2957 }
2958
otx2_check_pf_usable(struct otx2_nic * nic)2959 int otx2_check_pf_usable(struct otx2_nic *nic)
2960 {
2961 u64 rev;
2962
2963 rev = otx2_read64(nic, RVU_PF_BLOCK_ADDRX_DISC(BLKADDR_RVUM));
2964 rev = (rev >> 12) & 0xFF;
2965 /* Check if AF has setup revision for RVUM block,
2966 * otherwise this driver probe should be deferred
2967 * until AF driver comes up.
2968 */
2969 if (!rev) {
2970 dev_warn(nic->dev,
2971 "AF is not initialized, deferring probe\n");
2972 return -EPROBE_DEFER;
2973 }
2974 return 0;
2975 }
2976
otx2_realloc_msix_vectors(struct otx2_nic * pf)2977 int otx2_realloc_msix_vectors(struct otx2_nic *pf)
2978 {
2979 struct otx2_hw *hw = &pf->hw;
2980 int num_vec, err;
2981
2982 /* NPA interrupts are inot registered, so alloc only
2983 * upto NIX vector offset.
2984 */
2985 num_vec = hw->nix_msixoff;
2986 num_vec += NIX_LF_CINT_VEC_START + hw->max_queues;
2987
2988 otx2_disable_mbox_intr(pf);
2989 pci_free_irq_vectors(hw->pdev);
2990 err = pci_alloc_irq_vectors(hw->pdev, num_vec, num_vec, PCI_IRQ_MSIX);
2991 if (err < 0) {
2992 dev_err(pf->dev, "%s: Failed to realloc %d IRQ vectors\n",
2993 __func__, num_vec);
2994 return err;
2995 }
2996
2997 return otx2_register_mbox_intr(pf, false);
2998 }
2999 EXPORT_SYMBOL(otx2_realloc_msix_vectors);
3000
otx2_sriov_vfcfg_init(struct otx2_nic * pf)3001 static int otx2_sriov_vfcfg_init(struct otx2_nic *pf)
3002 {
3003 int i;
3004
3005 pf->vf_configs = devm_kcalloc(pf->dev, pf->total_vfs,
3006 sizeof(struct otx2_vf_config),
3007 GFP_KERNEL);
3008 if (!pf->vf_configs)
3009 return -ENOMEM;
3010
3011 for (i = 0; i < pf->total_vfs; i++) {
3012 pf->vf_configs[i].pf = pf;
3013 pf->vf_configs[i].intf_down = true;
3014 pf->vf_configs[i].trusted = false;
3015 INIT_DELAYED_WORK(&pf->vf_configs[i].link_event_work,
3016 otx2_vf_link_event_task);
3017 }
3018
3019 return 0;
3020 }
3021
otx2_sriov_vfcfg_cleanup(struct otx2_nic * pf)3022 static void otx2_sriov_vfcfg_cleanup(struct otx2_nic *pf)
3023 {
3024 int i;
3025
3026 if (!pf->vf_configs)
3027 return;
3028
3029 for (i = 0; i < pf->total_vfs; i++) {
3030 cancel_delayed_work_sync(&pf->vf_configs[i].link_event_work);
3031 otx2_set_vf_permissions(pf, i, OTX2_RESET_VF_PERM);
3032 }
3033 }
3034
otx2_init_rsrc(struct pci_dev * pdev,struct otx2_nic * pf)3035 int otx2_init_rsrc(struct pci_dev *pdev, struct otx2_nic *pf)
3036 {
3037 struct device *dev = &pdev->dev;
3038 struct otx2_hw *hw = &pf->hw;
3039 int num_vec, err;
3040
3041 num_vec = pci_msix_vec_count(pdev);
3042 hw->irq_name = devm_kmalloc_array(&hw->pdev->dev, num_vec, NAME_SIZE,
3043 GFP_KERNEL);
3044 if (!hw->irq_name)
3045 return -ENOMEM;
3046
3047 hw->affinity_mask = devm_kcalloc(&hw->pdev->dev, num_vec,
3048 sizeof(cpumask_var_t), GFP_KERNEL);
3049 if (!hw->affinity_mask)
3050 return -ENOMEM;
3051
3052 /* Map CSRs */
3053 pf->reg_base = pcim_iomap(pdev, PCI_CFG_REG_BAR_NUM, 0);
3054 if (!pf->reg_base) {
3055 dev_err(dev, "Unable to map physical function CSRs, aborting\n");
3056 return -ENOMEM;
3057 }
3058
3059 err = otx2_check_pf_usable(pf);
3060 if (err)
3061 return err;
3062
3063 if (!is_cn20k(pf->pdev))
3064 err = pci_alloc_irq_vectors(hw->pdev, RVU_PF_INT_VEC_CNT,
3065 RVU_PF_INT_VEC_CNT, PCI_IRQ_MSIX);
3066 else
3067 err = pci_alloc_irq_vectors(hw->pdev, RVU_MBOX_PF_INT_VEC_CNT,
3068 RVU_MBOX_PF_INT_VEC_CNT,
3069 PCI_IRQ_MSIX);
3070 if (err < 0) {
3071 dev_err(dev, "%s: Failed to alloc %d IRQ vectors\n",
3072 __func__, num_vec);
3073 return err;
3074 }
3075
3076 otx2_setup_dev_hw_settings(pf);
3077
3078 if (is_cn20k(pf->pdev))
3079 cn20k_init(pf);
3080 else
3081 otx2_init_hw_ops(pf);
3082
3083 /* Init PF <=> AF mailbox stuff */
3084 err = otx2_pfaf_mbox_init(pf);
3085 if (err)
3086 goto err_free_irq_vectors;
3087
3088 /* Register mailbox interrupt */
3089 err = otx2_register_mbox_intr(pf, true);
3090 if (err)
3091 goto err_mbox_destroy;
3092
3093 /* Request AF to attach NPA and NIX LFs to this PF.
3094 * NIX and NPA LFs are needed for this PF to function as a NIC.
3095 */
3096 err = otx2_attach_npa_nix(pf);
3097 if (err)
3098 goto err_disable_mbox_intr;
3099
3100 err = otx2_realloc_msix_vectors(pf);
3101 if (err)
3102 goto err_detach_rsrc;
3103
3104 err = cn10k_lmtst_init(pf);
3105 if (err)
3106 goto err_detach_rsrc;
3107
3108 return 0;
3109
3110 err_detach_rsrc:
3111 if (pf->hw.lmt_info)
3112 free_percpu(pf->hw.lmt_info);
3113 if (test_bit(CN10K_LMTST, &pf->hw.cap_flag))
3114 qmem_free(pf->dev, pf->dync_lmt);
3115 otx2_detach_resources(&pf->mbox);
3116 err_disable_mbox_intr:
3117 otx2_disable_mbox_intr(pf);
3118 err_mbox_destroy:
3119 otx2_pfaf_mbox_destroy(pf);
3120 err_free_irq_vectors:
3121 pci_free_irq_vectors(hw->pdev);
3122
3123 return err;
3124 }
3125 EXPORT_SYMBOL(otx2_init_rsrc);
3126
otx2_probe(struct pci_dev * pdev,const struct pci_device_id * id)3127 static int otx2_probe(struct pci_dev *pdev, const struct pci_device_id *id)
3128 {
3129 struct device *dev = &pdev->dev;
3130 int err, qcount, qos_txqs;
3131 struct net_device *netdev;
3132 struct otx2_nic *pf;
3133 struct otx2_hw *hw;
3134
3135 err = pcim_enable_device(pdev);
3136 if (err) {
3137 dev_err(dev, "Failed to enable PCI device\n");
3138 return err;
3139 }
3140
3141 err = pcim_request_all_regions(pdev, DRV_NAME);
3142 if (err) {
3143 dev_err(dev, "PCI request regions failed 0x%x\n", err);
3144 return err;
3145 }
3146
3147 err = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(48));
3148 if (err) {
3149 dev_err(dev, "DMA mask config failed, abort\n");
3150 return err;
3151 }
3152
3153 pci_set_master(pdev);
3154
3155 /* Set number of queues */
3156 qcount = min_t(int, num_online_cpus(), OTX2_MAX_CQ_CNT);
3157 qos_txqs = min_t(int, qcount, OTX2_QOS_MAX_LEAF_NODES);
3158
3159 netdev = alloc_etherdev_mqs(sizeof(*pf), qcount + qos_txqs, qcount);
3160 if (!netdev)
3161 return -ENOMEM;
3162
3163 pci_set_drvdata(pdev, netdev);
3164 SET_NETDEV_DEV(netdev, &pdev->dev);
3165 pf = netdev_priv(netdev);
3166 pf->netdev = netdev;
3167 pf->pdev = pdev;
3168 pf->dev = dev;
3169 pf->total_vfs = pci_sriov_get_totalvfs(pdev);
3170 pf->flags |= OTX2_FLAG_INTF_DOWN;
3171
3172 hw = &pf->hw;
3173 hw->pdev = pdev;
3174 hw->rx_queues = qcount;
3175 hw->tx_queues = qcount;
3176 hw->non_qos_queues = qcount;
3177 hw->max_queues = qcount;
3178 hw->rbuf_len = OTX2_DEFAULT_RBUF_LEN;
3179 /* Use CQE of 128 byte descriptor size by default */
3180 hw->xqe_size = 128;
3181
3182 err = otx2_init_rsrc(pdev, pf);
3183 if (err)
3184 goto err_free_netdev;
3185
3186 err = otx2_set_real_num_queues(netdev, hw->tx_queues, hw->rx_queues);
3187 if (err)
3188 goto err_detach_rsrc;
3189
3190 /* Assign default mac address */
3191 otx2_get_mac_from_af(netdev);
3192
3193 /* Don't check for error. Proceed without ptp */
3194 otx2_ptp_init(pf);
3195
3196 /* NPA's pool is a stack to which SW frees buffer pointers via Aura.
3197 * HW allocates buffer pointer from stack and uses it for DMA'ing
3198 * ingress packet. In some scenarios HW can free back allocated buffer
3199 * pointers to pool. This makes it impossible for SW to maintain a
3200 * parallel list where physical addresses of buffer pointers (IOVAs)
3201 * given to HW can be saved for later reference.
3202 *
3203 * So the only way to convert Rx packet's buffer address is to use
3204 * IOMMU's iova_to_phys() handler which translates the address by
3205 * walking through the translation tables.
3206 */
3207 pf->iommu_domain = iommu_get_domain_for_dev(dev);
3208
3209 netdev->hw_features = (NETIF_F_RXCSUM | NETIF_F_IP_CSUM |
3210 NETIF_F_IPV6_CSUM | NETIF_F_RXHASH |
3211 NETIF_F_SG | NETIF_F_TSO | NETIF_F_TSO6 |
3212 NETIF_F_GSO_UDP_L4);
3213 netdev->features |= netdev->hw_features;
3214
3215 err = otx2_mcam_flow_init(pf);
3216 if (err)
3217 goto err_ptp_destroy;
3218
3219 otx2_set_hw_capabilities(pf);
3220
3221 err = cn10k_mcs_init(pf);
3222 if (err)
3223 goto err_del_mcam_entries;
3224
3225 if (pf->flags & OTX2_FLAG_NTUPLE_SUPPORT)
3226 netdev->hw_features |= NETIF_F_NTUPLE;
3227
3228 if (pf->flags & OTX2_FLAG_UCAST_FLTR_SUPPORT)
3229 netdev->priv_flags |= IFF_UNICAST_FLT;
3230
3231 /* Support TSO on tag interface */
3232 netdev->vlan_features |= netdev->features;
3233 netdev->hw_features |= NETIF_F_HW_VLAN_CTAG_TX |
3234 NETIF_F_HW_VLAN_STAG_TX;
3235 if (pf->flags & OTX2_FLAG_RX_VLAN_SUPPORT)
3236 netdev->hw_features |= NETIF_F_HW_VLAN_CTAG_RX |
3237 NETIF_F_HW_VLAN_STAG_RX;
3238 netdev->features |= netdev->hw_features;
3239
3240 /* HW supports tc offload but mutually exclusive with n-tuple filters */
3241 if (pf->flags & OTX2_FLAG_TC_FLOWER_SUPPORT)
3242 netdev->hw_features |= NETIF_F_HW_TC;
3243
3244 netdev->hw_features |= NETIF_F_LOOPBACK | NETIF_F_RXALL;
3245
3246 netif_set_tso_max_segs(netdev, OTX2_MAX_GSO_SEGS);
3247 netdev->watchdog_timeo = OTX2_TX_TIMEOUT;
3248
3249 netdev->netdev_ops = &otx2_netdev_ops;
3250 netdev->xdp_features = NETDEV_XDP_ACT_BASIC | NETDEV_XDP_ACT_REDIRECT |
3251 NETDEV_XDP_ACT_NDO_XMIT |
3252 NETDEV_XDP_ACT_XSK_ZEROCOPY;
3253
3254 netdev->min_mtu = OTX2_MIN_MTU;
3255 netdev->max_mtu = otx2_get_max_mtu(pf);
3256 hw->max_mtu = netdev->max_mtu;
3257
3258 /* reset CGX/RPM MAC stats */
3259 otx2_reset_mac_stats(pf);
3260
3261 err = cn10k_ipsec_init(netdev);
3262 if (err)
3263 goto err_mcs_free;
3264
3265 err = register_netdev(netdev);
3266 if (err) {
3267 dev_err(dev, "Failed to register netdevice\n");
3268 goto err_ipsec_clean;
3269 }
3270
3271 err = otx2_wq_init(pf);
3272 if (err)
3273 goto err_unreg_netdev;
3274
3275 otx2_set_ethtool_ops(netdev);
3276
3277 err = otx2_init_tc(pf);
3278 if (err)
3279 goto err_mcam_flow_del;
3280
3281 err = otx2_register_dl(pf);
3282 if (err)
3283 goto err_mcam_flow_del;
3284
3285 /* Initialize SR-IOV resources */
3286 err = otx2_sriov_vfcfg_init(pf);
3287 if (err)
3288 goto err_pf_sriov_init;
3289
3290 /* Enable link notifications */
3291 otx2_cgx_config_linkevents(pf, true);
3292
3293 pf->af_xdp_zc_qidx = bitmap_zalloc(qcount, GFP_KERNEL);
3294 if (!pf->af_xdp_zc_qidx) {
3295 err = -ENOMEM;
3296 goto err_sriov_cleannup;
3297 }
3298
3299 #ifdef CONFIG_DCB
3300 err = otx2_dcbnl_set_ops(netdev);
3301 if (err)
3302 goto err_free_zc_bmap;
3303 #endif
3304
3305 otx2_qos_init(pf, qos_txqs);
3306
3307 return 0;
3308
3309 #ifdef CONFIG_DCB
3310 err_free_zc_bmap:
3311 bitmap_free(pf->af_xdp_zc_qidx);
3312 #endif
3313 err_sriov_cleannup:
3314 otx2_sriov_vfcfg_cleanup(pf);
3315 err_pf_sriov_init:
3316 otx2_unregister_dl(pf);
3317 otx2_shutdown_tc(pf);
3318 err_mcam_flow_del:
3319 otx2_mcam_flow_del(pf);
3320 err_unreg_netdev:
3321 unregister_netdev(netdev);
3322 err_ipsec_clean:
3323 cn10k_ipsec_clean(pf);
3324 err_mcs_free:
3325 cn10k_mcs_free(pf);
3326 err_del_mcam_entries:
3327 otx2_mcam_flow_del(pf);
3328 err_ptp_destroy:
3329 otx2_ptp_destroy(pf);
3330 err_detach_rsrc:
3331 if (pf->hw.lmt_info)
3332 free_percpu(pf->hw.lmt_info);
3333 if (test_bit(CN10K_LMTST, &pf->hw.cap_flag))
3334 qmem_free(pf->dev, pf->dync_lmt);
3335 otx2_detach_resources(&pf->mbox);
3336 otx2_disable_mbox_intr(pf);
3337 otx2_pfaf_mbox_destroy(pf);
3338 pci_free_irq_vectors(hw->pdev);
3339 err_free_netdev:
3340 pci_set_drvdata(pdev, NULL);
3341 free_netdev(netdev);
3342 return err;
3343 }
3344
otx2_vf_link_event_task(struct work_struct * work)3345 static void otx2_vf_link_event_task(struct work_struct *work)
3346 {
3347 struct otx2_vf_config *config;
3348 struct cgx_link_info_msg *req;
3349 struct mbox_msghdr *msghdr;
3350 struct delayed_work *dwork;
3351 struct otx2_nic *pf;
3352 int vf_idx;
3353
3354 config = container_of(work, struct otx2_vf_config,
3355 link_event_work.work);
3356 vf_idx = config - config->pf->vf_configs;
3357 pf = config->pf;
3358
3359 if (config->intf_down)
3360 return;
3361
3362 mutex_lock(&pf->mbox.lock);
3363
3364 dwork = &config->link_event_work;
3365
3366 if (!otx2_mbox_wait_for_zero(&pf->mbox_pfvf[0].mbox_up, vf_idx)) {
3367 schedule_delayed_work(dwork, msecs_to_jiffies(100));
3368 mutex_unlock(&pf->mbox.lock);
3369 return;
3370 }
3371
3372 msghdr = otx2_mbox_alloc_msg_rsp(&pf->mbox_pfvf[0].mbox_up, vf_idx,
3373 sizeof(*req), sizeof(struct msg_rsp));
3374 if (!msghdr) {
3375 dev_err(pf->dev, "Failed to create VF%d link event\n", vf_idx);
3376 mutex_unlock(&pf->mbox.lock);
3377 return;
3378 }
3379
3380 req = (struct cgx_link_info_msg *)msghdr;
3381 req->hdr.id = MBOX_MSG_CGX_LINK_EVENT;
3382 req->hdr.sig = OTX2_MBOX_REQ_SIG;
3383 req->hdr.pcifunc = pf->pcifunc;
3384 memcpy(&req->link_info, &pf->linfo, sizeof(req->link_info));
3385
3386 otx2_mbox_wait_for_zero(&pf->mbox_pfvf[0].mbox_up, vf_idx);
3387
3388 otx2_sync_mbox_up_msg(&pf->mbox_pfvf[0], vf_idx);
3389
3390 mutex_unlock(&pf->mbox.lock);
3391 }
3392
otx2_sriov_enable(struct pci_dev * pdev,int numvfs)3393 static int otx2_sriov_enable(struct pci_dev *pdev, int numvfs)
3394 {
3395 struct net_device *netdev = pci_get_drvdata(pdev);
3396 struct otx2_nic *pf = netdev_priv(netdev);
3397 int ret;
3398
3399 /* Init PF <=> VF mailbox stuff */
3400 ret = otx2_pfvf_mbox_init(pf, numvfs);
3401 if (ret)
3402 return ret;
3403
3404 ret = otx2_register_pfvf_mbox_intr(pf, numvfs);
3405 if (ret)
3406 goto free_mbox;
3407
3408 ret = otx2_pf_flr_init(pf, numvfs);
3409 if (ret)
3410 goto free_intr;
3411
3412 ret = otx2_register_flr_me_intr(pf, numvfs);
3413 if (ret)
3414 goto free_flr;
3415
3416 ret = pci_enable_sriov(pdev, numvfs);
3417 if (ret)
3418 goto free_flr_intr;
3419
3420 return numvfs;
3421 free_flr_intr:
3422 otx2_disable_flr_me_intr(pf);
3423 free_flr:
3424 otx2_flr_wq_destroy(pf);
3425 free_intr:
3426 otx2_disable_pfvf_mbox_intr(pf, numvfs);
3427 free_mbox:
3428 otx2_pfvf_mbox_destroy(pf);
3429 return ret;
3430 }
3431
otx2_sriov_disable(struct pci_dev * pdev)3432 static int otx2_sriov_disable(struct pci_dev *pdev)
3433 {
3434 struct net_device *netdev = pci_get_drvdata(pdev);
3435 struct otx2_nic *pf = netdev_priv(netdev);
3436 int numvfs = pci_num_vf(pdev);
3437
3438 if (!numvfs)
3439 return 0;
3440
3441 pci_disable_sriov(pdev);
3442
3443 otx2_disable_flr_me_intr(pf);
3444 otx2_flr_wq_destroy(pf);
3445 otx2_disable_pfvf_mbox_intr(pf, numvfs);
3446 otx2_pfvf_mbox_destroy(pf);
3447
3448 return 0;
3449 }
3450
otx2_sriov_configure(struct pci_dev * pdev,int numvfs)3451 static int otx2_sriov_configure(struct pci_dev *pdev, int numvfs)
3452 {
3453 if (numvfs == 0)
3454 return otx2_sriov_disable(pdev);
3455 else
3456 return otx2_sriov_enable(pdev, numvfs);
3457 }
3458
otx2_ndc_sync(struct otx2_nic * pf)3459 static void otx2_ndc_sync(struct otx2_nic *pf)
3460 {
3461 struct mbox *mbox = &pf->mbox;
3462 struct ndc_sync_op *req;
3463
3464 mutex_lock(&mbox->lock);
3465
3466 req = otx2_mbox_alloc_msg_ndc_sync_op(mbox);
3467 if (!req) {
3468 mutex_unlock(&mbox->lock);
3469 return;
3470 }
3471
3472 req->nix_lf_tx_sync = 1;
3473 req->nix_lf_rx_sync = 1;
3474 req->npa_lf_sync = 1;
3475
3476 if (!otx2_sync_mbox_msg(mbox))
3477 dev_err(pf->dev, "NDC sync operation failed\n");
3478
3479 mutex_unlock(&mbox->lock);
3480 }
3481
otx2_remove(struct pci_dev * pdev)3482 static void otx2_remove(struct pci_dev *pdev)
3483 {
3484 struct net_device *netdev = pci_get_drvdata(pdev);
3485 struct otx2_nic *pf;
3486
3487 if (!netdev)
3488 return;
3489
3490 pf = netdev_priv(netdev);
3491
3492 pf->flags |= OTX2_FLAG_PF_SHUTDOWN;
3493
3494 if (pf->flags & OTX2_FLAG_TX_TSTAMP_ENABLED)
3495 otx2_config_hw_tx_tstamp(pf, false);
3496 if (pf->flags & OTX2_FLAG_RX_TSTAMP_ENABLED)
3497 otx2_config_hw_rx_tstamp(pf, false);
3498
3499 /* Disable 802.3x pause frames */
3500 if (pf->flags & OTX2_FLAG_RX_PAUSE_ENABLED ||
3501 (pf->flags & OTX2_FLAG_TX_PAUSE_ENABLED)) {
3502 pf->flags &= ~OTX2_FLAG_RX_PAUSE_ENABLED;
3503 pf->flags &= ~OTX2_FLAG_TX_PAUSE_ENABLED;
3504 otx2_config_pause_frm(pf);
3505 }
3506
3507 #ifdef CONFIG_DCB
3508 /* Disable PFC config */
3509 if (pf->pfc_en) {
3510 pf->pfc_en = 0;
3511 otx2_config_priority_flow_ctrl(pf);
3512 }
3513 #endif
3514 cancel_work_sync(&pf->reset_task);
3515 /* Disable link notifications */
3516 otx2_cgx_config_linkevents(pf, false);
3517
3518 otx2_unregister_dl(pf);
3519 unregister_netdev(netdev);
3520 cn10k_ipsec_clean(pf);
3521 cn10k_mcs_free(pf);
3522 otx2_sriov_disable(pf->pdev);
3523 otx2_sriov_vfcfg_cleanup(pf);
3524 if (pf->otx2_wq)
3525 destroy_workqueue(pf->otx2_wq);
3526
3527 otx2_ptp_destroy(pf);
3528 otx2_mcam_flow_del(pf);
3529 otx2_shutdown_tc(pf);
3530 otx2_shutdown_qos(pf);
3531 otx2_ndc_sync(pf);
3532 otx2_detach_resources(&pf->mbox);
3533 if (pf->hw.lmt_info)
3534 free_percpu(pf->hw.lmt_info);
3535 if (test_bit(CN10K_LMTST, &pf->hw.cap_flag))
3536 qmem_free(pf->dev, pf->dync_lmt);
3537 otx2_disable_mbox_intr(pf);
3538 otx2_pfaf_mbox_destroy(pf);
3539 pci_free_irq_vectors(pf->pdev);
3540 bitmap_free(pf->af_xdp_zc_qidx);
3541 pci_set_drvdata(pdev, NULL);
3542 free_netdev(netdev);
3543 }
3544
3545 static struct pci_driver otx2_pf_driver = {
3546 .name = DRV_NAME,
3547 .id_table = otx2_pf_id_table,
3548 .probe = otx2_probe,
3549 .shutdown = otx2_remove,
3550 .remove = otx2_remove,
3551 .sriov_configure = otx2_sriov_configure
3552 };
3553
otx2_rvupf_init_module(void)3554 static int __init otx2_rvupf_init_module(void)
3555 {
3556 pr_info("%s: %s\n", DRV_NAME, DRV_STRING);
3557
3558 return pci_register_driver(&otx2_pf_driver);
3559 }
3560
otx2_rvupf_cleanup_module(void)3561 static void __exit otx2_rvupf_cleanup_module(void)
3562 {
3563 pci_unregister_driver(&otx2_pf_driver);
3564 }
3565
3566 module_init(otx2_rvupf_init_module);
3567 module_exit(otx2_rvupf_cleanup_module);
3568