1 // SPDX-License-Identifier: GPL-2.0
2 /* Marvell RVU Physical Function ethernet driver
3 *
4 * Copyright (C) 2020 Marvell.
5 *
6 */
7
8 #include <linux/module.h>
9 #include <linux/interrupt.h>
10 #include <linux/pci.h>
11 #include <linux/etherdevice.h>
12 #include <linux/of.h>
13 #include <linux/if_vlan.h>
14 #include <linux/iommu.h>
15 #include <net/ip.h>
16 #include <linux/bpf.h>
17 #include <linux/bpf_trace.h>
18 #include <linux/bitfield.h>
19 #include <net/page_pool/types.h>
20
21 #include "otx2_reg.h"
22 #include "otx2_common.h"
23 #include "otx2_txrx.h"
24 #include "otx2_struct.h"
25 #include "otx2_ptp.h"
26 #include "cn10k.h"
27 #include "qos.h"
28 #include <rvu_trace.h>
29 #include "cn10k_ipsec.h"
30 #include "otx2_xsk.h"
31
32 #define DRV_NAME "rvu_nicpf"
33 #define DRV_STRING "Marvell RVU NIC Physical Function Driver"
34
35 /* Supported devices */
36 static const struct pci_device_id otx2_pf_id_table[] = {
37 { PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, PCI_DEVID_OCTEONTX2_RVU_PF) },
38 { 0, } /* end of table */
39 };
40
41 MODULE_AUTHOR("Sunil Goutham <sgoutham@marvell.com>");
42 MODULE_DESCRIPTION(DRV_STRING);
43 MODULE_LICENSE("GPL v2");
44 MODULE_DEVICE_TABLE(pci, otx2_pf_id_table);
45
46 static void otx2_vf_link_event_task(struct work_struct *work);
47
48 enum {
49 TYPE_PFAF,
50 TYPE_PFVF,
51 };
52
53 static int otx2_config_hw_tx_tstamp(struct otx2_nic *pfvf, bool enable);
54 static int otx2_config_hw_rx_tstamp(struct otx2_nic *pfvf, bool enable);
55
otx2_change_mtu(struct net_device * netdev,int new_mtu)56 static int otx2_change_mtu(struct net_device *netdev, int new_mtu)
57 {
58 struct otx2_nic *pf = netdev_priv(netdev);
59 bool if_up = netif_running(netdev);
60 int err = 0;
61
62 if (pf->xdp_prog && new_mtu > MAX_XDP_MTU) {
63 netdev_warn(netdev, "Jumbo frames not yet supported with XDP, current MTU %d.\n",
64 netdev->mtu);
65 return -EINVAL;
66 }
67 if (if_up)
68 otx2_stop(netdev);
69
70 netdev_info(netdev, "Changing MTU from %d to %d\n",
71 netdev->mtu, new_mtu);
72 WRITE_ONCE(netdev->mtu, new_mtu);
73
74 if (if_up)
75 err = otx2_open(netdev);
76
77 return err;
78 }
79
otx2_disable_flr_me_intr(struct otx2_nic * pf)80 static void otx2_disable_flr_me_intr(struct otx2_nic *pf)
81 {
82 int irq, vfs = pf->total_vfs;
83
84 /* Disable VFs ME interrupts */
85 otx2_write64(pf, RVU_PF_VFME_INT_ENA_W1CX(0), INTR_MASK(vfs));
86 irq = pci_irq_vector(pf->pdev, RVU_PF_INT_VEC_VFME0);
87 free_irq(irq, pf);
88
89 /* Disable VFs FLR interrupts */
90 otx2_write64(pf, RVU_PF_VFFLR_INT_ENA_W1CX(0), INTR_MASK(vfs));
91 irq = pci_irq_vector(pf->pdev, RVU_PF_INT_VEC_VFFLR0);
92 free_irq(irq, pf);
93
94 if (vfs <= 64)
95 return;
96
97 otx2_write64(pf, RVU_PF_VFME_INT_ENA_W1CX(1), INTR_MASK(vfs - 64));
98 irq = pci_irq_vector(pf->pdev, RVU_PF_INT_VEC_VFME1);
99 free_irq(irq, pf);
100
101 otx2_write64(pf, RVU_PF_VFFLR_INT_ENA_W1CX(1), INTR_MASK(vfs - 64));
102 irq = pci_irq_vector(pf->pdev, RVU_PF_INT_VEC_VFFLR1);
103 free_irq(irq, pf);
104 }
105
otx2_flr_wq_destroy(struct otx2_nic * pf)106 static void otx2_flr_wq_destroy(struct otx2_nic *pf)
107 {
108 if (!pf->flr_wq)
109 return;
110 destroy_workqueue(pf->flr_wq);
111 pf->flr_wq = NULL;
112 devm_kfree(pf->dev, pf->flr_wrk);
113 }
114
otx2_flr_handler(struct work_struct * work)115 static void otx2_flr_handler(struct work_struct *work)
116 {
117 struct flr_work *flrwork = container_of(work, struct flr_work, work);
118 struct otx2_nic *pf = flrwork->pf;
119 struct mbox *mbox = &pf->mbox;
120 struct msg_req *req;
121 int vf, reg = 0;
122
123 vf = flrwork - pf->flr_wrk;
124
125 mutex_lock(&mbox->lock);
126 req = otx2_mbox_alloc_msg_vf_flr(mbox);
127 if (!req) {
128 mutex_unlock(&mbox->lock);
129 return;
130 }
131 req->hdr.pcifunc &= RVU_PFVF_FUNC_MASK;
132 req->hdr.pcifunc |= (vf + 1) & RVU_PFVF_FUNC_MASK;
133
134 if (!otx2_sync_mbox_msg(&pf->mbox)) {
135 if (vf >= 64) {
136 reg = 1;
137 vf = vf - 64;
138 }
139 /* clear transcation pending bit */
140 otx2_write64(pf, RVU_PF_VFTRPENDX(reg), BIT_ULL(vf));
141 otx2_write64(pf, RVU_PF_VFFLR_INT_ENA_W1SX(reg), BIT_ULL(vf));
142 }
143
144 mutex_unlock(&mbox->lock);
145 }
146
otx2_pf_flr_intr_handler(int irq,void * pf_irq)147 static irqreturn_t otx2_pf_flr_intr_handler(int irq, void *pf_irq)
148 {
149 struct otx2_nic *pf = (struct otx2_nic *)pf_irq;
150 int reg, dev, vf, start_vf, num_reg = 1;
151 u64 intr;
152
153 if (pf->total_vfs > 64)
154 num_reg = 2;
155
156 for (reg = 0; reg < num_reg; reg++) {
157 intr = otx2_read64(pf, RVU_PF_VFFLR_INTX(reg));
158 if (!intr)
159 continue;
160 start_vf = 64 * reg;
161 for (vf = 0; vf < 64; vf++) {
162 if (!(intr & BIT_ULL(vf)))
163 continue;
164 dev = vf + start_vf;
165 queue_work(pf->flr_wq, &pf->flr_wrk[dev].work);
166 /* Clear interrupt */
167 otx2_write64(pf, RVU_PF_VFFLR_INTX(reg), BIT_ULL(vf));
168 /* Disable the interrupt */
169 otx2_write64(pf, RVU_PF_VFFLR_INT_ENA_W1CX(reg),
170 BIT_ULL(vf));
171 }
172 }
173 return IRQ_HANDLED;
174 }
175
otx2_pf_me_intr_handler(int irq,void * pf_irq)176 static irqreturn_t otx2_pf_me_intr_handler(int irq, void *pf_irq)
177 {
178 struct otx2_nic *pf = (struct otx2_nic *)pf_irq;
179 int vf, reg, num_reg = 1;
180 u64 intr;
181
182 if (pf->total_vfs > 64)
183 num_reg = 2;
184
185 for (reg = 0; reg < num_reg; reg++) {
186 intr = otx2_read64(pf, RVU_PF_VFME_INTX(reg));
187 if (!intr)
188 continue;
189 for (vf = 0; vf < 64; vf++) {
190 if (!(intr & BIT_ULL(vf)))
191 continue;
192 /* clear trpend bit */
193 otx2_write64(pf, RVU_PF_VFTRPENDX(reg), BIT_ULL(vf));
194 /* clear interrupt */
195 otx2_write64(pf, RVU_PF_VFME_INTX(reg), BIT_ULL(vf));
196 }
197 }
198 return IRQ_HANDLED;
199 }
200
otx2_register_flr_me_intr(struct otx2_nic * pf,int numvfs)201 static int otx2_register_flr_me_intr(struct otx2_nic *pf, int numvfs)
202 {
203 struct otx2_hw *hw = &pf->hw;
204 char *irq_name;
205 int ret;
206
207 /* Register ME interrupt handler*/
208 irq_name = &hw->irq_name[RVU_PF_INT_VEC_VFME0 * NAME_SIZE];
209 snprintf(irq_name, NAME_SIZE, "RVUPF%d_ME0",
210 rvu_get_pf(pf->pdev, pf->pcifunc));
211 ret = request_irq(pci_irq_vector(pf->pdev, RVU_PF_INT_VEC_VFME0),
212 otx2_pf_me_intr_handler, 0, irq_name, pf);
213 if (ret) {
214 dev_err(pf->dev,
215 "RVUPF: IRQ registration failed for ME0\n");
216 }
217
218 /* Register FLR interrupt handler */
219 irq_name = &hw->irq_name[RVU_PF_INT_VEC_VFFLR0 * NAME_SIZE];
220 snprintf(irq_name, NAME_SIZE, "RVUPF%d_FLR0",
221 rvu_get_pf(pf->pdev, pf->pcifunc));
222 ret = request_irq(pci_irq_vector(pf->pdev, RVU_PF_INT_VEC_VFFLR0),
223 otx2_pf_flr_intr_handler, 0, irq_name, pf);
224 if (ret) {
225 dev_err(pf->dev,
226 "RVUPF: IRQ registration failed for FLR0\n");
227 return ret;
228 }
229
230 if (numvfs > 64) {
231 irq_name = &hw->irq_name[RVU_PF_INT_VEC_VFME1 * NAME_SIZE];
232 snprintf(irq_name, NAME_SIZE, "RVUPF%d_ME1",
233 rvu_get_pf(pf->pdev, pf->pcifunc));
234 ret = request_irq(pci_irq_vector
235 (pf->pdev, RVU_PF_INT_VEC_VFME1),
236 otx2_pf_me_intr_handler, 0, irq_name, pf);
237 if (ret) {
238 dev_err(pf->dev,
239 "RVUPF: IRQ registration failed for ME1\n");
240 }
241 irq_name = &hw->irq_name[RVU_PF_INT_VEC_VFFLR1 * NAME_SIZE];
242 snprintf(irq_name, NAME_SIZE, "RVUPF%d_FLR1",
243 rvu_get_pf(pf->pdev, pf->pcifunc));
244 ret = request_irq(pci_irq_vector
245 (pf->pdev, RVU_PF_INT_VEC_VFFLR1),
246 otx2_pf_flr_intr_handler, 0, irq_name, pf);
247 if (ret) {
248 dev_err(pf->dev,
249 "RVUPF: IRQ registration failed for FLR1\n");
250 return ret;
251 }
252 }
253
254 /* Enable ME interrupt for all VFs*/
255 otx2_write64(pf, RVU_PF_VFME_INTX(0), INTR_MASK(numvfs));
256 otx2_write64(pf, RVU_PF_VFME_INT_ENA_W1SX(0), INTR_MASK(numvfs));
257
258 /* Enable FLR interrupt for all VFs*/
259 otx2_write64(pf, RVU_PF_VFFLR_INTX(0), INTR_MASK(numvfs));
260 otx2_write64(pf, RVU_PF_VFFLR_INT_ENA_W1SX(0), INTR_MASK(numvfs));
261
262 if (numvfs > 64) {
263 numvfs -= 64;
264
265 otx2_write64(pf, RVU_PF_VFME_INTX(1), INTR_MASK(numvfs));
266 otx2_write64(pf, RVU_PF_VFME_INT_ENA_W1SX(1),
267 INTR_MASK(numvfs));
268
269 otx2_write64(pf, RVU_PF_VFFLR_INTX(1), INTR_MASK(numvfs));
270 otx2_write64(pf, RVU_PF_VFFLR_INT_ENA_W1SX(1),
271 INTR_MASK(numvfs));
272 }
273 return 0;
274 }
275
otx2_pf_flr_init(struct otx2_nic * pf,int num_vfs)276 static int otx2_pf_flr_init(struct otx2_nic *pf, int num_vfs)
277 {
278 int vf;
279
280 pf->flr_wq = alloc_ordered_workqueue("otx2_pf_flr_wq", WQ_HIGHPRI);
281 if (!pf->flr_wq)
282 return -ENOMEM;
283
284 pf->flr_wrk = devm_kcalloc(pf->dev, num_vfs,
285 sizeof(struct flr_work), GFP_KERNEL);
286 if (!pf->flr_wrk) {
287 destroy_workqueue(pf->flr_wq);
288 return -ENOMEM;
289 }
290
291 for (vf = 0; vf < num_vfs; vf++) {
292 pf->flr_wrk[vf].pf = pf;
293 INIT_WORK(&pf->flr_wrk[vf].work, otx2_flr_handler);
294 }
295
296 return 0;
297 }
298
otx2_queue_vf_work(struct mbox * mw,struct workqueue_struct * mbox_wq,int first,int mdevs,u64 intr)299 void otx2_queue_vf_work(struct mbox *mw, struct workqueue_struct *mbox_wq,
300 int first, int mdevs, u64 intr)
301 {
302 struct otx2_mbox_dev *mdev;
303 struct otx2_mbox *mbox;
304 struct mbox_hdr *hdr;
305 int i;
306
307 for (i = first; i < mdevs; i++) {
308 /* start from 0 */
309 if (!(intr & BIT_ULL(i - first)))
310 continue;
311
312 mbox = &mw->mbox;
313 mdev = &mbox->dev[i];
314 hdr = mdev->mbase + mbox->rx_start;
315 /* The hdr->num_msgs is set to zero immediately in the interrupt
316 * handler to ensure that it holds a correct value next time
317 * when the interrupt handler is called. pf->mw[i].num_msgs
318 * holds the data for use in otx2_pfvf_mbox_handler and
319 * pf->mw[i].up_num_msgs holds the data for use in
320 * otx2_pfvf_mbox_up_handler.
321 */
322 if (hdr->num_msgs) {
323 mw[i].num_msgs = hdr->num_msgs;
324 hdr->num_msgs = 0;
325 queue_work(mbox_wq, &mw[i].mbox_wrk);
326 }
327
328 mbox = &mw->mbox_up;
329 mdev = &mbox->dev[i];
330 hdr = mdev->mbase + mbox->rx_start;
331 if (hdr->num_msgs) {
332 mw[i].up_num_msgs = hdr->num_msgs;
333 hdr->num_msgs = 0;
334 queue_work(mbox_wq, &mw[i].mbox_up_wrk);
335 }
336 }
337 }
338
otx2_forward_msg_pfvf(struct otx2_mbox_dev * mdev,struct otx2_mbox * pfvf_mbox,void * bbuf_base,int devid)339 static void otx2_forward_msg_pfvf(struct otx2_mbox_dev *mdev,
340 struct otx2_mbox *pfvf_mbox, void *bbuf_base,
341 int devid)
342 {
343 struct otx2_mbox_dev *src_mdev = mdev;
344 int offset;
345
346 /* Msgs are already copied, trigger VF's mbox irq */
347 smp_wmb();
348
349 otx2_mbox_wait_for_zero(pfvf_mbox, devid);
350
351 offset = pfvf_mbox->trigger | (devid << pfvf_mbox->tr_shift);
352 writeq(MBOX_DOWN_MSG, (void __iomem *)pfvf_mbox->reg_base + offset);
353
354 /* Restore VF's mbox bounce buffer region address */
355 src_mdev->mbase = bbuf_base;
356 }
357
otx2_forward_vf_mbox_msgs(struct otx2_nic * pf,struct otx2_mbox * src_mbox,int dir,int vf,int num_msgs)358 static int otx2_forward_vf_mbox_msgs(struct otx2_nic *pf,
359 struct otx2_mbox *src_mbox,
360 int dir, int vf, int num_msgs)
361 {
362 struct otx2_mbox_dev *src_mdev, *dst_mdev;
363 struct mbox_hdr *mbox_hdr;
364 struct mbox_hdr *req_hdr;
365 struct mbox *dst_mbox;
366 int dst_size, err;
367
368 if (dir == MBOX_DIR_PFAF) {
369 /* Set VF's mailbox memory as PF's bounce buffer memory, so
370 * that explicit copying of VF's msgs to PF=>AF mbox region
371 * and AF=>PF responses to VF's mbox region can be avoided.
372 */
373 src_mdev = &src_mbox->dev[vf];
374 mbox_hdr = src_mbox->hwbase +
375 src_mbox->rx_start + (vf * MBOX_SIZE);
376
377 dst_mbox = &pf->mbox;
378 dst_size = dst_mbox->mbox.tx_size -
379 ALIGN(sizeof(*mbox_hdr), MBOX_MSG_ALIGN);
380 /* Check if msgs fit into destination area and has valid size */
381 if (mbox_hdr->msg_size > dst_size || !mbox_hdr->msg_size)
382 return -EINVAL;
383
384 dst_mdev = &dst_mbox->mbox.dev[0];
385
386 mutex_lock(&pf->mbox.lock);
387 dst_mdev->mbase = src_mdev->mbase;
388 dst_mdev->msg_size = mbox_hdr->msg_size;
389 dst_mdev->num_msgs = num_msgs;
390 err = otx2_sync_mbox_msg(dst_mbox);
391 /* Error code -EIO indicate there is a communication failure
392 * to the AF. Rest of the error codes indicate that AF processed
393 * VF messages and set the error codes in response messages
394 * (if any) so simply forward responses to VF.
395 */
396 if (err == -EIO) {
397 dev_warn(pf->dev,
398 "AF not responding to VF%d messages\n", vf);
399 /* restore PF mbase and exit */
400 dst_mdev->mbase = pf->mbox.bbuf_base;
401 mutex_unlock(&pf->mbox.lock);
402 return err;
403 }
404 /* At this point, all the VF messages sent to AF are acked
405 * with proper responses and responses are copied to VF
406 * mailbox hence raise interrupt to VF.
407 */
408 req_hdr = (struct mbox_hdr *)(dst_mdev->mbase +
409 dst_mbox->mbox.rx_start);
410 req_hdr->num_msgs = num_msgs;
411
412 otx2_forward_msg_pfvf(dst_mdev, &pf->mbox_pfvf[0].mbox,
413 pf->mbox.bbuf_base, vf);
414 mutex_unlock(&pf->mbox.lock);
415 } else if (dir == MBOX_DIR_PFVF_UP) {
416 src_mdev = &src_mbox->dev[0];
417 mbox_hdr = src_mbox->hwbase + src_mbox->rx_start;
418 req_hdr = (struct mbox_hdr *)(src_mdev->mbase +
419 src_mbox->rx_start);
420 req_hdr->num_msgs = num_msgs;
421
422 dst_mbox = &pf->mbox_pfvf[0];
423 dst_size = dst_mbox->mbox_up.tx_size -
424 ALIGN(sizeof(*mbox_hdr), MBOX_MSG_ALIGN);
425 /* Check if msgs fit into destination area */
426 if (mbox_hdr->msg_size > dst_size)
427 return -EINVAL;
428
429 dst_mdev = &dst_mbox->mbox_up.dev[vf];
430 dst_mdev->mbase = src_mdev->mbase;
431 dst_mdev->msg_size = mbox_hdr->msg_size;
432 dst_mdev->num_msgs = mbox_hdr->num_msgs;
433 err = otx2_sync_mbox_up_msg(dst_mbox, vf);
434 if (err) {
435 dev_warn(pf->dev,
436 "VF%d is not responding to mailbox\n", vf);
437 return err;
438 }
439 } else if (dir == MBOX_DIR_VFPF_UP) {
440 req_hdr = (struct mbox_hdr *)(src_mbox->dev[0].mbase +
441 src_mbox->rx_start);
442 req_hdr->num_msgs = num_msgs;
443 otx2_forward_msg_pfvf(&pf->mbox_pfvf->mbox_up.dev[vf],
444 &pf->mbox.mbox_up,
445 pf->mbox_pfvf[vf].bbuf_base,
446 0);
447 }
448
449 return 0;
450 }
451
otx2_pfvf_mbox_handler(struct work_struct * work)452 static void otx2_pfvf_mbox_handler(struct work_struct *work)
453 {
454 struct mbox_msghdr *msg = NULL;
455 int offset, vf_idx, id, err;
456 struct otx2_mbox_dev *mdev;
457 struct otx2_mbox *mbox;
458 struct mbox *vf_mbox;
459 struct otx2_nic *pf;
460
461 vf_mbox = container_of(work, struct mbox, mbox_wrk);
462 pf = vf_mbox->pfvf;
463 vf_idx = vf_mbox - pf->mbox_pfvf;
464
465 mbox = &pf->mbox_pfvf[0].mbox;
466 mdev = &mbox->dev[vf_idx];
467
468 offset = ALIGN(sizeof(struct mbox_hdr), MBOX_MSG_ALIGN);
469
470 trace_otx2_msg_status(pf->pdev, "PF-VF down queue handler(forwarding)",
471 vf_mbox->num_msgs);
472
473 for (id = 0; id < vf_mbox->num_msgs; id++) {
474 msg = (struct mbox_msghdr *)(mdev->mbase + mbox->rx_start +
475 offset);
476
477 if (msg->sig != OTX2_MBOX_REQ_SIG)
478 goto inval_msg;
479
480 /* Set VF's number in each of the msg */
481 msg->pcifunc &= ~RVU_PFVF_FUNC_MASK;
482 msg->pcifunc |= (vf_idx + 1) & RVU_PFVF_FUNC_MASK;
483 offset = msg->next_msgoff;
484 }
485 err = otx2_forward_vf_mbox_msgs(pf, mbox, MBOX_DIR_PFAF, vf_idx,
486 vf_mbox->num_msgs);
487 if (err)
488 goto inval_msg;
489 return;
490
491 inval_msg:
492 otx2_reply_invalid_msg(mbox, vf_idx, 0, msg->id);
493 otx2_mbox_msg_send(mbox, vf_idx);
494 }
495
otx2_pfvf_mbox_up_handler(struct work_struct * work)496 static void otx2_pfvf_mbox_up_handler(struct work_struct *work)
497 {
498 struct mbox *vf_mbox = container_of(work, struct mbox, mbox_up_wrk);
499 struct otx2_nic *pf = vf_mbox->pfvf;
500 struct otx2_mbox_dev *mdev;
501 int offset, id, vf_idx = 0;
502 struct mbox_msghdr *msg;
503 struct otx2_mbox *mbox;
504
505 vf_idx = vf_mbox - pf->mbox_pfvf;
506 mbox = &pf->mbox_pfvf[0].mbox_up;
507 mdev = &mbox->dev[vf_idx];
508
509 offset = mbox->rx_start + ALIGN(sizeof(struct mbox_hdr), MBOX_MSG_ALIGN);
510
511 trace_otx2_msg_status(pf->pdev, "PF-VF up queue handler(response)",
512 vf_mbox->up_num_msgs);
513
514 for (id = 0; id < vf_mbox->up_num_msgs; id++) {
515 msg = mdev->mbase + offset;
516
517 if (msg->id >= MBOX_MSG_MAX) {
518 dev_err(pf->dev,
519 "Mbox msg with unknown ID 0x%x\n", msg->id);
520 goto end;
521 }
522
523 if (msg->sig != OTX2_MBOX_RSP_SIG) {
524 dev_err(pf->dev,
525 "Mbox msg with wrong signature %x, ID 0x%x\n",
526 msg->sig, msg->id);
527 goto end;
528 }
529
530 switch (msg->id) {
531 case MBOX_MSG_CGX_LINK_EVENT:
532 case MBOX_MSG_REP_EVENT_UP_NOTIFY:
533 break;
534 default:
535 if (msg->rc)
536 dev_err(pf->dev,
537 "Mbox msg response has err %d, ID 0x%x\n",
538 msg->rc, msg->id);
539 break;
540 }
541
542 end:
543 offset = mbox->rx_start + msg->next_msgoff;
544 if (mdev->msgs_acked == (vf_mbox->up_num_msgs - 1))
545 __otx2_mbox_reset(mbox, vf_idx);
546 mdev->msgs_acked++;
547 }
548 }
549
otx2_pfvf_mbox_intr_handler(int irq,void * pf_irq)550 irqreturn_t otx2_pfvf_mbox_intr_handler(int irq, void *pf_irq)
551 {
552 struct otx2_nic *pf = (struct otx2_nic *)(pf_irq);
553 int vfs = pf->total_vfs;
554 struct mbox *mbox;
555 u64 intr;
556
557 mbox = pf->mbox_pfvf;
558 /* Handle VF interrupts */
559 if (vfs > 64) {
560 intr = otx2_read64(pf, RVU_PF_VFPF_MBOX_INTX(1));
561 otx2_write64(pf, RVU_PF_VFPF_MBOX_INTX(1), intr);
562 otx2_queue_vf_work(mbox, pf->mbox_pfvf_wq, 64, vfs, intr);
563 if (intr)
564 trace_otx2_msg_interrupt(mbox->mbox.pdev, "VF(s) to PF", intr);
565 vfs = 64;
566 }
567
568 intr = otx2_read64(pf, RVU_PF_VFPF_MBOX_INTX(0));
569 otx2_write64(pf, RVU_PF_VFPF_MBOX_INTX(0), intr);
570
571 otx2_queue_vf_work(mbox, pf->mbox_pfvf_wq, 0, vfs, intr);
572
573 if (intr)
574 trace_otx2_msg_interrupt(mbox->mbox.pdev, "VF(s) to PF", intr);
575
576 return IRQ_HANDLED;
577 }
578
cn20k_pfvf_mbox_alloc(struct otx2_nic * pf,int numvfs)579 static void *cn20k_pfvf_mbox_alloc(struct otx2_nic *pf, int numvfs)
580 {
581 struct qmem *mbox_addr;
582 int err;
583
584 err = qmem_alloc(&pf->pdev->dev, &mbox_addr, numvfs, MBOX_SIZE);
585 if (err) {
586 dev_err(pf->dev, "qmem alloc fail\n");
587 return ERR_PTR(-ENOMEM);
588 }
589
590 otx2_write64(pf, RVU_PF_VF_MBOX_ADDR, (u64)mbox_addr->iova);
591 pf->pfvf_mbox_addr = mbox_addr;
592
593 return mbox_addr->base;
594 }
595
otx2_pfvf_mbox_init(struct otx2_nic * pf,int numvfs)596 static int otx2_pfvf_mbox_init(struct otx2_nic *pf, int numvfs)
597 {
598 void __iomem *hwbase;
599 struct mbox *mbox;
600 int err, vf;
601 u64 base;
602
603 if (!numvfs)
604 return -EINVAL;
605
606 pf->mbox_pfvf = devm_kcalloc(&pf->pdev->dev, numvfs,
607 sizeof(struct mbox), GFP_KERNEL);
608 if (!pf->mbox_pfvf)
609 return -ENOMEM;
610
611 pf->mbox_pfvf_wq = alloc_workqueue("otx2_pfvf_mailbox",
612 WQ_UNBOUND | WQ_HIGHPRI |
613 WQ_MEM_RECLAIM, 0);
614 if (!pf->mbox_pfvf_wq)
615 return -ENOMEM;
616
617 /* For CN20K, PF allocates mbox memory in DRAM and writes PF/VF
618 * regions/offsets in RVU_PF_VF_MBOX_ADDR, the RVU_PFX_FUNC_PFAF_MBOX
619 * gives the aliased address to access PF/VF mailbox regions.
620 */
621 if (is_cn20k(pf->pdev)) {
622 hwbase = (void __iomem *)cn20k_pfvf_mbox_alloc(pf, numvfs);
623 } else {
624 /* On CN10K platform, PF <-> VF mailbox region follows after
625 * PF <-> AF mailbox region.
626 */
627 if (test_bit(CN10K_MBOX, &pf->hw.cap_flag))
628 base = pci_resource_start(pf->pdev, PCI_MBOX_BAR_NUM) +
629 MBOX_SIZE;
630 else
631 base = readq(pf->reg_base + RVU_PF_VF_BAR4_ADDR);
632
633 hwbase = ioremap_wc(base, MBOX_SIZE * pf->total_vfs);
634 if (!hwbase) {
635 err = -ENOMEM;
636 goto free_wq;
637 }
638 }
639
640 mbox = &pf->mbox_pfvf[0];
641 err = otx2_mbox_init(&mbox->mbox, hwbase, pf->pdev, pf->reg_base,
642 MBOX_DIR_PFVF, numvfs);
643 if (err)
644 goto free_iomem;
645
646 err = otx2_mbox_init(&mbox->mbox_up, hwbase, pf->pdev, pf->reg_base,
647 MBOX_DIR_PFVF_UP, numvfs);
648 if (err)
649 goto free_iomem;
650
651 for (vf = 0; vf < numvfs; vf++) {
652 mbox->pfvf = pf;
653 INIT_WORK(&mbox->mbox_wrk, otx2_pfvf_mbox_handler);
654 INIT_WORK(&mbox->mbox_up_wrk, otx2_pfvf_mbox_up_handler);
655 mbox++;
656 }
657
658 return 0;
659
660 free_iomem:
661 if (hwbase && !(is_cn20k(pf->pdev)))
662 iounmap(hwbase);
663 free_wq:
664 destroy_workqueue(pf->mbox_pfvf_wq);
665 return err;
666 }
667
otx2_pfvf_mbox_destroy(struct otx2_nic * pf)668 static void otx2_pfvf_mbox_destroy(struct otx2_nic *pf)
669 {
670 struct mbox *mbox = &pf->mbox_pfvf[0];
671
672 if (!mbox)
673 return;
674
675 if (pf->mbox_pfvf_wq) {
676 destroy_workqueue(pf->mbox_pfvf_wq);
677 pf->mbox_pfvf_wq = NULL;
678 }
679
680 if (mbox->mbox.hwbase && !is_cn20k(pf->pdev))
681 iounmap(mbox->mbox.hwbase);
682 else
683 qmem_free(&pf->pdev->dev, pf->pfvf_mbox_addr);
684
685 otx2_mbox_destroy(&mbox->mbox);
686 }
687
otx2_enable_pfvf_mbox_intr(struct otx2_nic * pf,int numvfs)688 static void otx2_enable_pfvf_mbox_intr(struct otx2_nic *pf, int numvfs)
689 {
690 /* Clear PF <=> VF mailbox IRQ */
691 otx2_write64(pf, RVU_PF_VFPF_MBOX_INTX(0), ~0ull);
692 otx2_write64(pf, RVU_PF_VFPF_MBOX_INTX(1), ~0ull);
693
694 /* Enable PF <=> VF mailbox IRQ */
695 otx2_write64(pf, RVU_PF_VFPF_MBOX_INT_ENA_W1SX(0), INTR_MASK(numvfs));
696 if (numvfs > 64) {
697 numvfs -= 64;
698 otx2_write64(pf, RVU_PF_VFPF_MBOX_INT_ENA_W1SX(1),
699 INTR_MASK(numvfs));
700 }
701 }
702
otx2_disable_pfvf_mbox_intr(struct otx2_nic * pf,int numvfs)703 static void otx2_disable_pfvf_mbox_intr(struct otx2_nic *pf, int numvfs)
704 {
705 int vector;
706
707 if (is_cn20k(pf->pdev))
708 return cn20k_disable_pfvf_mbox_intr(pf, numvfs);
709
710 /* Disable PF <=> VF mailbox IRQ */
711 otx2_write64(pf, RVU_PF_VFPF_MBOX_INT_ENA_W1CX(0), ~0ull);
712 otx2_write64(pf, RVU_PF_VFPF_MBOX_INT_ENA_W1CX(1), ~0ull);
713
714 otx2_write64(pf, RVU_PF_VFPF_MBOX_INTX(0), ~0ull);
715 vector = pci_irq_vector(pf->pdev, RVU_PF_INT_VEC_VFPF_MBOX0);
716 free_irq(vector, pf);
717
718 if (numvfs > 64) {
719 otx2_write64(pf, RVU_PF_VFPF_MBOX_INTX(1), ~0ull);
720 vector = pci_irq_vector(pf->pdev, RVU_PF_INT_VEC_VFPF_MBOX1);
721 free_irq(vector, pf);
722 }
723 }
724
otx2_register_pfvf_mbox_intr(struct otx2_nic * pf,int numvfs)725 static int otx2_register_pfvf_mbox_intr(struct otx2_nic *pf, int numvfs)
726 {
727 struct otx2_hw *hw = &pf->hw;
728 char *irq_name;
729 int err;
730
731 if (is_cn20k(pf->pdev))
732 return cn20k_register_pfvf_mbox_intr(pf, numvfs);
733
734 /* Register MBOX0 interrupt handler */
735 irq_name = &hw->irq_name[RVU_PF_INT_VEC_VFPF_MBOX0 * NAME_SIZE];
736 if (pf->pcifunc)
737 snprintf(irq_name, NAME_SIZE,
738 "RVUPF%d_VF Mbox0", rvu_get_pf(pf->pdev, pf->pcifunc));
739 else
740 snprintf(irq_name, NAME_SIZE, "RVUPF_VF Mbox0");
741 err = request_irq(pci_irq_vector(pf->pdev, RVU_PF_INT_VEC_VFPF_MBOX0),
742 otx2_pfvf_mbox_intr_handler, 0, irq_name, pf);
743 if (err) {
744 dev_err(pf->dev,
745 "RVUPF: IRQ registration failed for PFVF mbox0 irq\n");
746 return err;
747 }
748
749 if (numvfs > 64) {
750 /* Register MBOX1 interrupt handler */
751 irq_name = &hw->irq_name[RVU_PF_INT_VEC_VFPF_MBOX1 * NAME_SIZE];
752 if (pf->pcifunc)
753 snprintf(irq_name, NAME_SIZE,
754 "RVUPF%d_VF Mbox1",
755 rvu_get_pf(pf->pdev, pf->pcifunc));
756 else
757 snprintf(irq_name, NAME_SIZE, "RVUPF_VF Mbox1");
758 err = request_irq(pci_irq_vector(pf->pdev,
759 RVU_PF_INT_VEC_VFPF_MBOX1),
760 otx2_pfvf_mbox_intr_handler,
761 0, irq_name, pf);
762 if (err) {
763 dev_err(pf->dev,
764 "RVUPF: IRQ registration failed for PFVF mbox1 irq\n");
765 return err;
766 }
767 }
768
769 otx2_enable_pfvf_mbox_intr(pf, numvfs);
770
771 return 0;
772 }
773
otx2_process_pfaf_mbox_msg(struct otx2_nic * pf,struct mbox_msghdr * msg)774 static void otx2_process_pfaf_mbox_msg(struct otx2_nic *pf,
775 struct mbox_msghdr *msg)
776 {
777 int devid;
778
779 if (msg->id >= MBOX_MSG_MAX) {
780 dev_err(pf->dev,
781 "Mbox msg with unknown ID 0x%x\n", msg->id);
782 return;
783 }
784
785 if (msg->sig != OTX2_MBOX_RSP_SIG) {
786 dev_err(pf->dev,
787 "Mbox msg with wrong signature %x, ID 0x%x\n",
788 msg->sig, msg->id);
789 return;
790 }
791
792 /* message response heading VF */
793 devid = msg->pcifunc & RVU_PFVF_FUNC_MASK;
794 if (devid) {
795 struct otx2_vf_config *config = &pf->vf_configs[devid - 1];
796 struct delayed_work *dwork;
797
798 switch (msg->id) {
799 case MBOX_MSG_NIX_LF_START_RX:
800 config->intf_down = false;
801 dwork = &config->link_event_work;
802 schedule_delayed_work(dwork, msecs_to_jiffies(100));
803 break;
804 case MBOX_MSG_NIX_LF_STOP_RX:
805 config->intf_down = true;
806 break;
807 }
808
809 return;
810 }
811
812 switch (msg->id) {
813 case MBOX_MSG_READY:
814 pf->pcifunc = msg->pcifunc;
815 break;
816 case MBOX_MSG_MSIX_OFFSET:
817 mbox_handler_msix_offset(pf, (struct msix_offset_rsp *)msg);
818 break;
819 case MBOX_MSG_NPA_LF_ALLOC:
820 mbox_handler_npa_lf_alloc(pf, (struct npa_lf_alloc_rsp *)msg);
821 break;
822 case MBOX_MSG_NIX_LF_ALLOC:
823 mbox_handler_nix_lf_alloc(pf, (struct nix_lf_alloc_rsp *)msg);
824 break;
825 case MBOX_MSG_NIX_BP_ENABLE:
826 mbox_handler_nix_bp_enable(pf, (struct nix_bp_cfg_rsp *)msg);
827 break;
828 case MBOX_MSG_CGX_STATS:
829 mbox_handler_cgx_stats(pf, (struct cgx_stats_rsp *)msg);
830 break;
831 case MBOX_MSG_CGX_FEC_STATS:
832 mbox_handler_cgx_fec_stats(pf, (struct cgx_fec_stats_rsp *)msg);
833 break;
834 default:
835 if (msg->rc)
836 dev_err(pf->dev,
837 "Mbox msg response has err %d, ID 0x%x\n",
838 msg->rc, msg->id);
839 break;
840 }
841 }
842
otx2_pfaf_mbox_handler(struct work_struct * work)843 static void otx2_pfaf_mbox_handler(struct work_struct *work)
844 {
845 struct otx2_mbox_dev *mdev;
846 struct mbox_hdr *rsp_hdr;
847 struct mbox_msghdr *msg;
848 struct otx2_mbox *mbox;
849 struct mbox *af_mbox;
850 struct otx2_nic *pf;
851 int offset, id;
852 u16 num_msgs;
853
854 af_mbox = container_of(work, struct mbox, mbox_wrk);
855 mbox = &af_mbox->mbox;
856 mdev = &mbox->dev[0];
857 rsp_hdr = (struct mbox_hdr *)(mdev->mbase + mbox->rx_start);
858 num_msgs = rsp_hdr->num_msgs;
859
860 offset = mbox->rx_start + ALIGN(sizeof(*rsp_hdr), MBOX_MSG_ALIGN);
861 pf = af_mbox->pfvf;
862
863 trace_otx2_msg_status(pf->pdev, "PF-AF down queue handler(response)",
864 num_msgs);
865
866 for (id = 0; id < num_msgs; id++) {
867 msg = (struct mbox_msghdr *)(mdev->mbase + offset);
868 otx2_process_pfaf_mbox_msg(pf, msg);
869 offset = mbox->rx_start + msg->next_msgoff;
870 if (mdev->msgs_acked == (num_msgs - 1))
871 __otx2_mbox_reset(mbox, 0);
872 mdev->msgs_acked++;
873 }
874
875 }
876
otx2_handle_link_event(struct otx2_nic * pf)877 static void otx2_handle_link_event(struct otx2_nic *pf)
878 {
879 struct cgx_link_user_info *linfo = &pf->linfo;
880 struct net_device *netdev = pf->netdev;
881
882 if (pf->flags & OTX2_FLAG_PORT_UP)
883 return;
884
885 pr_info("%s NIC Link is %s %d Mbps %s duplex\n", netdev->name,
886 linfo->link_up ? "UP" : "DOWN", linfo->speed,
887 linfo->full_duplex ? "Full" : "Half");
888 if (linfo->link_up) {
889 netif_carrier_on(netdev);
890 netif_tx_start_all_queues(netdev);
891 } else {
892 netif_tx_stop_all_queues(netdev);
893 netif_carrier_off(netdev);
894 }
895 }
896
otx2_mbox_up_handler_rep_event_up_notify(struct otx2_nic * pf,struct rep_event * info,struct msg_rsp * rsp)897 static int otx2_mbox_up_handler_rep_event_up_notify(struct otx2_nic *pf,
898 struct rep_event *info,
899 struct msg_rsp *rsp)
900 {
901 struct net_device *netdev = pf->netdev;
902
903 if (info->event == RVU_EVENT_MTU_CHANGE) {
904 netdev->mtu = info->evt_data.mtu;
905 return 0;
906 }
907
908 if (info->event == RVU_EVENT_PORT_STATE) {
909 if (info->evt_data.port_state) {
910 pf->flags |= OTX2_FLAG_PORT_UP;
911 netif_carrier_on(netdev);
912 netif_tx_start_all_queues(netdev);
913 } else {
914 pf->flags &= ~OTX2_FLAG_PORT_UP;
915 netif_tx_stop_all_queues(netdev);
916 netif_carrier_off(netdev);
917 }
918 return 0;
919 }
920 #ifdef CONFIG_RVU_ESWITCH
921 rvu_event_up_notify(pf, info);
922 #endif
923 return 0;
924 }
925
otx2_mbox_up_handler_mcs_intr_notify(struct otx2_nic * pf,struct mcs_intr_info * event,struct msg_rsp * rsp)926 int otx2_mbox_up_handler_mcs_intr_notify(struct otx2_nic *pf,
927 struct mcs_intr_info *event,
928 struct msg_rsp *rsp)
929 {
930 cn10k_handle_mcs_event(pf, event);
931
932 return 0;
933 }
934
otx2_mbox_up_handler_cgx_link_event(struct otx2_nic * pf,struct cgx_link_info_msg * msg,struct msg_rsp * rsp)935 int otx2_mbox_up_handler_cgx_link_event(struct otx2_nic *pf,
936 struct cgx_link_info_msg *msg,
937 struct msg_rsp *rsp)
938 {
939 int i;
940
941 /* Copy the link info sent by AF */
942 pf->linfo = msg->link_info;
943
944 /* notify VFs about link event */
945 for (i = 0; i < pci_num_vf(pf->pdev); i++) {
946 struct otx2_vf_config *config = &pf->vf_configs[i];
947 struct delayed_work *dwork = &config->link_event_work;
948
949 if (config->intf_down)
950 continue;
951
952 schedule_delayed_work(dwork, msecs_to_jiffies(100));
953 }
954
955 /* interface has not been fully configured yet */
956 if (pf->flags & OTX2_FLAG_INTF_DOWN)
957 return 0;
958
959 otx2_handle_link_event(pf);
960 return 0;
961 }
962
otx2_process_mbox_msg_up(struct otx2_nic * pf,struct mbox_msghdr * req)963 static int otx2_process_mbox_msg_up(struct otx2_nic *pf,
964 struct mbox_msghdr *req)
965 {
966 /* Check if valid, if not reply with a invalid msg */
967 if (req->sig != OTX2_MBOX_REQ_SIG) {
968 otx2_reply_invalid_msg(&pf->mbox.mbox_up, 0, 0, req->id);
969 return -ENODEV;
970 }
971
972 switch (req->id) {
973 #define M(_name, _id, _fn_name, _req_type, _rsp_type) \
974 case _id: { \
975 struct _rsp_type *rsp; \
976 int err; \
977 \
978 rsp = (struct _rsp_type *)otx2_mbox_alloc_msg( \
979 &pf->mbox.mbox_up, 0, \
980 sizeof(struct _rsp_type)); \
981 if (!rsp) \
982 return -ENOMEM; \
983 \
984 rsp->hdr.id = _id; \
985 rsp->hdr.sig = OTX2_MBOX_RSP_SIG; \
986 rsp->hdr.pcifunc = 0; \
987 rsp->hdr.rc = 0; \
988 \
989 err = otx2_mbox_up_handler_ ## _fn_name( \
990 pf, (struct _req_type *)req, rsp); \
991 return err; \
992 }
993 MBOX_UP_CGX_MESSAGES
994 MBOX_UP_MCS_MESSAGES
995 MBOX_UP_REP_MESSAGES
996 #undef M
997 break;
998 default:
999 otx2_reply_invalid_msg(&pf->mbox.mbox_up, 0, 0, req->id);
1000 return -ENODEV;
1001 }
1002 return 0;
1003 }
1004
otx2_pfaf_mbox_up_handler(struct work_struct * work)1005 static void otx2_pfaf_mbox_up_handler(struct work_struct *work)
1006 {
1007 struct mbox *af_mbox = container_of(work, struct mbox, mbox_up_wrk);
1008 struct otx2_mbox *mbox = &af_mbox->mbox_up;
1009 struct otx2_mbox_dev *mdev = &mbox->dev[0];
1010 struct otx2_nic *pf = af_mbox->pfvf;
1011 int offset, id, devid = 0;
1012 struct mbox_hdr *rsp_hdr;
1013 struct mbox_msghdr *msg;
1014 u16 num_msgs;
1015
1016 rsp_hdr = (struct mbox_hdr *)(mdev->mbase + mbox->rx_start);
1017 num_msgs = rsp_hdr->num_msgs;
1018
1019 offset = mbox->rx_start + ALIGN(sizeof(*rsp_hdr), MBOX_MSG_ALIGN);
1020
1021 trace_otx2_msg_status(pf->pdev, "PF-AF up queue handler(notification)",
1022 num_msgs);
1023
1024 for (id = 0; id < num_msgs; id++) {
1025 msg = (struct mbox_msghdr *)(mdev->mbase + offset);
1026
1027 devid = msg->pcifunc & RVU_PFVF_FUNC_MASK;
1028 /* Skip processing VF's messages */
1029 if (!devid)
1030 otx2_process_mbox_msg_up(pf, msg);
1031 offset = mbox->rx_start + msg->next_msgoff;
1032 }
1033 /* Forward to VF iff VFs are really present */
1034 if (devid && pci_num_vf(pf->pdev)) {
1035 otx2_forward_vf_mbox_msgs(pf, &pf->mbox.mbox_up,
1036 MBOX_DIR_PFVF_UP, devid - 1,
1037 num_msgs);
1038 return;
1039 }
1040
1041 otx2_mbox_msg_send(mbox, 0);
1042 }
1043
otx2_pfaf_mbox_intr_handler(int irq,void * pf_irq)1044 irqreturn_t otx2_pfaf_mbox_intr_handler(int irq, void *pf_irq)
1045 {
1046 struct otx2_nic *pf = (struct otx2_nic *)pf_irq;
1047 struct mbox *mw = &pf->mbox;
1048 struct otx2_mbox_dev *mdev;
1049 struct otx2_mbox *mbox;
1050 struct mbox_hdr *hdr;
1051 u64 mbox_data;
1052
1053 /* Clear the IRQ */
1054 otx2_write64(pf, RVU_PF_INT, BIT_ULL(0));
1055
1056
1057 mbox_data = otx2_read64(pf, RVU_PF_PFAF_MBOX0);
1058
1059 if (mbox_data & MBOX_UP_MSG) {
1060 mbox_data &= ~MBOX_UP_MSG;
1061 otx2_write64(pf, RVU_PF_PFAF_MBOX0, mbox_data);
1062
1063 mbox = &mw->mbox_up;
1064 mdev = &mbox->dev[0];
1065 otx2_sync_mbox_bbuf(mbox, 0);
1066
1067 hdr = (struct mbox_hdr *)(mdev->mbase + mbox->rx_start);
1068 if (hdr->num_msgs)
1069 queue_work(pf->mbox_wq, &mw->mbox_up_wrk);
1070
1071 trace_otx2_msg_interrupt(pf->pdev, "UP message from AF to PF",
1072 BIT_ULL(0));
1073
1074 trace_otx2_msg_status(pf->pdev, "PF-AF up work queued(interrupt)",
1075 hdr->num_msgs);
1076 }
1077
1078 if (mbox_data & MBOX_DOWN_MSG) {
1079 mbox_data &= ~MBOX_DOWN_MSG;
1080 otx2_write64(pf, RVU_PF_PFAF_MBOX0, mbox_data);
1081
1082 mbox = &mw->mbox;
1083 mdev = &mbox->dev[0];
1084 otx2_sync_mbox_bbuf(mbox, 0);
1085
1086 hdr = (struct mbox_hdr *)(mdev->mbase + mbox->rx_start);
1087 if (hdr->num_msgs)
1088 queue_work(pf->mbox_wq, &mw->mbox_wrk);
1089
1090 trace_otx2_msg_interrupt(pf->pdev, "DOWN reply from AF to PF",
1091 BIT_ULL(0));
1092
1093 trace_otx2_msg_status(pf->pdev, "PF-AF down work queued(interrupt)",
1094 hdr->num_msgs);
1095 }
1096
1097 return IRQ_HANDLED;
1098 }
1099
otx2_disable_mbox_intr(struct otx2_nic * pf)1100 void otx2_disable_mbox_intr(struct otx2_nic *pf)
1101 {
1102 int vector;
1103
1104 /* Disable AF => PF mailbox IRQ */
1105 if (!is_cn20k(pf->pdev)) {
1106 vector = pci_irq_vector(pf->pdev, RVU_PF_INT_VEC_AFPF_MBOX);
1107 otx2_write64(pf, RVU_PF_INT_ENA_W1C, BIT_ULL(0));
1108 } else {
1109 vector = pci_irq_vector(pf->pdev,
1110 RVU_MBOX_PF_INT_VEC_AFPF_MBOX);
1111 otx2_write64(pf, RVU_PF_INT_ENA_W1C,
1112 BIT_ULL(0) | BIT_ULL(1));
1113 }
1114 free_irq(vector, pf);
1115 }
1116 EXPORT_SYMBOL(otx2_disable_mbox_intr);
1117
otx2_register_mbox_intr(struct otx2_nic * pf,bool probe_af)1118 int otx2_register_mbox_intr(struct otx2_nic *pf, bool probe_af)
1119 {
1120 struct otx2_hw *hw = &pf->hw;
1121 struct msg_req *req;
1122 char *irq_name;
1123 int err;
1124
1125 /* Register mailbox interrupt handler */
1126 if (!is_cn20k(pf->pdev)) {
1127 irq_name = &hw->irq_name[RVU_PF_INT_VEC_AFPF_MBOX * NAME_SIZE];
1128 snprintf(irq_name, NAME_SIZE, "RVUPF%d AFPF Mbox",
1129 rvu_get_pf(pf->pdev, pf->pcifunc));
1130 err = request_irq(pci_irq_vector
1131 (pf->pdev, RVU_PF_INT_VEC_AFPF_MBOX),
1132 pf->hw_ops->pfaf_mbox_intr_handler,
1133 0, irq_name, pf);
1134 } else {
1135 irq_name = &hw->irq_name[RVU_MBOX_PF_INT_VEC_AFPF_MBOX *
1136 NAME_SIZE];
1137 snprintf(irq_name, NAME_SIZE, "RVUPF%d AFPF Mbox",
1138 rvu_get_pf(pf->pdev, pf->pcifunc));
1139 err = request_irq(pci_irq_vector
1140 (pf->pdev, RVU_MBOX_PF_INT_VEC_AFPF_MBOX),
1141 pf->hw_ops->pfaf_mbox_intr_handler,
1142 0, irq_name, pf);
1143 }
1144 if (err) {
1145 dev_err(pf->dev,
1146 "RVUPF: IRQ registration failed for PFAF mbox irq\n");
1147 return err;
1148 }
1149
1150 /* Enable mailbox interrupt for msgs coming from AF.
1151 * First clear to avoid spurious interrupts, if any.
1152 */
1153 if (!is_cn20k(pf->pdev)) {
1154 otx2_write64(pf, RVU_PF_INT, BIT_ULL(0));
1155 otx2_write64(pf, RVU_PF_INT_ENA_W1S, BIT_ULL(0));
1156 } else {
1157 otx2_write64(pf, RVU_PF_INT, BIT_ULL(0) | BIT_ULL(1));
1158 otx2_write64(pf, RVU_PF_INT_ENA_W1S, BIT_ULL(0) |
1159 BIT_ULL(1));
1160 }
1161
1162 if (!probe_af)
1163 return 0;
1164
1165 /* Check mailbox communication with AF */
1166 req = otx2_mbox_alloc_msg_ready(&pf->mbox);
1167 if (!req) {
1168 otx2_disable_mbox_intr(pf);
1169 return -ENOMEM;
1170 }
1171 err = otx2_sync_mbox_msg(&pf->mbox);
1172 if (err) {
1173 dev_warn(pf->dev,
1174 "AF not responding to mailbox, deferring probe\n");
1175 otx2_disable_mbox_intr(pf);
1176 return -EPROBE_DEFER;
1177 }
1178
1179 return 0;
1180 }
1181
otx2_pfaf_mbox_destroy(struct otx2_nic * pf)1182 void otx2_pfaf_mbox_destroy(struct otx2_nic *pf)
1183 {
1184 struct mbox *mbox = &pf->mbox;
1185
1186 if (pf->mbox_wq) {
1187 destroy_workqueue(pf->mbox_wq);
1188 pf->mbox_wq = NULL;
1189 }
1190
1191 if (mbox->mbox.hwbase && !is_cn20k(pf->pdev))
1192 iounmap((void __iomem *)mbox->mbox.hwbase);
1193
1194 otx2_mbox_destroy(&mbox->mbox);
1195 otx2_mbox_destroy(&mbox->mbox_up);
1196 }
1197 EXPORT_SYMBOL(otx2_pfaf_mbox_destroy);
1198
otx2_pfaf_mbox_init(struct otx2_nic * pf)1199 int otx2_pfaf_mbox_init(struct otx2_nic *pf)
1200 {
1201 struct mbox *mbox = &pf->mbox;
1202 void __iomem *hwbase;
1203 int err;
1204
1205 mbox->pfvf = pf;
1206 pf->mbox_wq = alloc_ordered_workqueue("otx2_pfaf_mailbox",
1207 WQ_HIGHPRI | WQ_MEM_RECLAIM);
1208 if (!pf->mbox_wq)
1209 return -ENOMEM;
1210
1211 /* For CN20K, AF allocates mbox memory in DRAM and writes PF
1212 * regions/offsets in RVU_MBOX_AF_PFX_ADDR, the RVU_PFX_FUNC_PFAF_MBOX
1213 * gives the aliased address to access AF/PF mailbox regions.
1214 */
1215 if (is_cn20k(pf->pdev))
1216 hwbase = pf->reg_base + RVU_PFX_FUNC_PFAF_MBOX +
1217 ((u64)BLKADDR_MBOX << RVU_FUNC_BLKADDR_SHIFT);
1218 else
1219 /* Mailbox is a reserved memory (in RAM) region shared between
1220 * admin function (i.e AF) and this PF, shouldn't be mapped as
1221 * device memory to allow unaligned accesses.
1222 */
1223 hwbase = ioremap_wc(pci_resource_start
1224 (pf->pdev, PCI_MBOX_BAR_NUM), MBOX_SIZE);
1225 if (!hwbase) {
1226 dev_err(pf->dev, "Unable to map PFAF mailbox region\n");
1227 err = -ENOMEM;
1228 goto exit;
1229 }
1230
1231 err = otx2_mbox_init(&mbox->mbox, hwbase, pf->pdev, pf->reg_base,
1232 MBOX_DIR_PFAF, 1);
1233 if (err)
1234 goto exit;
1235
1236 err = otx2_mbox_init(&mbox->mbox_up, hwbase, pf->pdev, pf->reg_base,
1237 MBOX_DIR_PFAF_UP, 1);
1238 if (err)
1239 goto exit;
1240
1241 err = otx2_mbox_bbuf_init(mbox, pf->pdev);
1242 if (err)
1243 goto exit;
1244
1245 INIT_WORK(&mbox->mbox_wrk, otx2_pfaf_mbox_handler);
1246 INIT_WORK(&mbox->mbox_up_wrk, otx2_pfaf_mbox_up_handler);
1247 mutex_init(&mbox->lock);
1248
1249 return 0;
1250 exit:
1251 otx2_pfaf_mbox_destroy(pf);
1252 return err;
1253 }
1254
otx2_cgx_config_linkevents(struct otx2_nic * pf,bool enable)1255 static int otx2_cgx_config_linkevents(struct otx2_nic *pf, bool enable)
1256 {
1257 struct msg_req *msg;
1258 int err;
1259
1260 mutex_lock(&pf->mbox.lock);
1261 if (enable)
1262 msg = otx2_mbox_alloc_msg_cgx_start_linkevents(&pf->mbox);
1263 else
1264 msg = otx2_mbox_alloc_msg_cgx_stop_linkevents(&pf->mbox);
1265
1266 if (!msg) {
1267 mutex_unlock(&pf->mbox.lock);
1268 return -ENOMEM;
1269 }
1270
1271 err = otx2_sync_mbox_msg(&pf->mbox);
1272 mutex_unlock(&pf->mbox.lock);
1273 return err;
1274 }
1275
otx2_reset_mac_stats(struct otx2_nic * pfvf)1276 int otx2_reset_mac_stats(struct otx2_nic *pfvf)
1277 {
1278 struct msg_req *req;
1279 int err;
1280
1281 mutex_lock(&pfvf->mbox.lock);
1282 req = otx2_mbox_alloc_msg_cgx_stats_rst(&pfvf->mbox);
1283 if (!req) {
1284 mutex_unlock(&pfvf->mbox.lock);
1285 return -ENOMEM;
1286 }
1287
1288 err = otx2_sync_mbox_msg(&pfvf->mbox);
1289 mutex_unlock(&pfvf->mbox.lock);
1290 return err;
1291 }
1292
otx2_cgx_config_loopback(struct otx2_nic * pf,bool enable)1293 static int otx2_cgx_config_loopback(struct otx2_nic *pf, bool enable)
1294 {
1295 struct msg_req *msg;
1296 int err;
1297
1298 if (enable && !bitmap_empty(pf->flow_cfg->dmacflt_bmap,
1299 pf->flow_cfg->dmacflt_max_flows))
1300 netdev_warn(pf->netdev,
1301 "CGX/RPM internal loopback might not work as DMAC filters are active\n");
1302
1303 mutex_lock(&pf->mbox.lock);
1304 if (enable)
1305 msg = otx2_mbox_alloc_msg_cgx_intlbk_enable(&pf->mbox);
1306 else
1307 msg = otx2_mbox_alloc_msg_cgx_intlbk_disable(&pf->mbox);
1308
1309 if (!msg) {
1310 mutex_unlock(&pf->mbox.lock);
1311 return -ENOMEM;
1312 }
1313
1314 err = otx2_sync_mbox_msg(&pf->mbox);
1315 mutex_unlock(&pf->mbox.lock);
1316 return err;
1317 }
1318
otx2_set_real_num_queues(struct net_device * netdev,int tx_queues,int rx_queues)1319 int otx2_set_real_num_queues(struct net_device *netdev,
1320 int tx_queues, int rx_queues)
1321 {
1322 int err;
1323
1324 err = netif_set_real_num_tx_queues(netdev, tx_queues);
1325 if (err) {
1326 netdev_err(netdev,
1327 "Failed to set no of Tx queues: %d\n", tx_queues);
1328 return err;
1329 }
1330
1331 err = netif_set_real_num_rx_queues(netdev, rx_queues);
1332 if (err)
1333 netdev_err(netdev,
1334 "Failed to set no of Rx queues: %d\n", rx_queues);
1335 return err;
1336 }
1337 EXPORT_SYMBOL(otx2_set_real_num_queues);
1338
1339 static char *nix_sqoperr_e_str[NIX_SQOPERR_MAX] = {
1340 "NIX_SQOPERR_OOR",
1341 "NIX_SQOPERR_CTX_FAULT",
1342 "NIX_SQOPERR_CTX_POISON",
1343 "NIX_SQOPERR_DISABLED",
1344 "NIX_SQOPERR_SIZE_ERR",
1345 "NIX_SQOPERR_OFLOW",
1346 "NIX_SQOPERR_SQB_NULL",
1347 "NIX_SQOPERR_SQB_FAULT",
1348 "NIX_SQOPERR_SQE_SZ_ZERO",
1349 };
1350
1351 static char *nix_mnqerr_e_str[NIX_MNQERR_MAX] = {
1352 "NIX_MNQERR_SQ_CTX_FAULT",
1353 "NIX_MNQERR_SQ_CTX_POISON",
1354 "NIX_MNQERR_SQB_FAULT",
1355 "NIX_MNQERR_SQB_POISON",
1356 "NIX_MNQERR_TOTAL_ERR",
1357 "NIX_MNQERR_LSO_ERR",
1358 "NIX_MNQERR_CQ_QUERY_ERR",
1359 "NIX_MNQERR_MAX_SQE_SIZE_ERR",
1360 "NIX_MNQERR_MAXLEN_ERR",
1361 "NIX_MNQERR_SQE_SIZEM1_ZERO",
1362 };
1363
1364 static char *nix_snd_status_e_str[NIX_SND_STATUS_MAX] = {
1365 [NIX_SND_STATUS_GOOD] = "NIX_SND_STATUS_GOOD",
1366 [NIX_SND_STATUS_SQ_CTX_FAULT] = "NIX_SND_STATUS_SQ_CTX_FAULT",
1367 [NIX_SND_STATUS_SQ_CTX_POISON] = "NIX_SND_STATUS_SQ_CTX_POISON",
1368 [NIX_SND_STATUS_SQB_FAULT] = "NIX_SND_STATUS_SQB_FAULT",
1369 [NIX_SND_STATUS_SQB_POISON] = "NIX_SND_STATUS_SQB_POISON",
1370 [NIX_SND_STATUS_HDR_ERR] = "NIX_SND_STATUS_HDR_ERR",
1371 [NIX_SND_STATUS_EXT_ERR] = "NIX_SND_STATUS_EXT_ERR",
1372 [NIX_SND_STATUS_JUMP_FAULT] = "NIX_SND_STATUS_JUMP_FAULT",
1373 [NIX_SND_STATUS_JUMP_POISON] = "NIX_SND_STATUS_JUMP_POISON",
1374 [NIX_SND_STATUS_CRC_ERR] = "NIX_SND_STATUS_CRC_ERR",
1375 [NIX_SND_STATUS_IMM_ERR] = "NIX_SND_STATUS_IMM_ERR",
1376 [NIX_SND_STATUS_SG_ERR] = "NIX_SND_STATUS_SG_ERR",
1377 [NIX_SND_STATUS_MEM_ERR] = "NIX_SND_STATUS_MEM_ERR",
1378 [NIX_SND_STATUS_INVALID_SUBDC] = "NIX_SND_STATUS_INVALID_SUBDC",
1379 [NIX_SND_STATUS_SUBDC_ORDER_ERR] = "NIX_SND_STATUS_SUBDC_ORDER_ERR",
1380 [NIX_SND_STATUS_DATA_FAULT] = "NIX_SND_STATUS_DATA_FAULT",
1381 [NIX_SND_STATUS_DATA_POISON] = "NIX_SND_STATUS_DATA_POISON",
1382 [NIX_SND_STATUS_NPC_DROP_ACTION] = "NIX_SND_STATUS_NPC_DROP_ACTION",
1383 [NIX_SND_STATUS_LOCK_VIOL] = "NIX_SND_STATUS_LOCK_VIOL",
1384 [NIX_SND_STATUS_NPC_UCAST_CHAN_ERR] = "NIX_SND_STAT_NPC_UCAST_CHAN_ERR",
1385 [NIX_SND_STATUS_NPC_MCAST_CHAN_ERR] = "NIX_SND_STAT_NPC_MCAST_CHAN_ERR",
1386 [NIX_SND_STATUS_NPC_MCAST_ABORT] = "NIX_SND_STATUS_NPC_MCAST_ABORT",
1387 [NIX_SND_STATUS_NPC_VTAG_PTR_ERR] = "NIX_SND_STATUS_NPC_VTAG_PTR_ERR",
1388 [NIX_SND_STATUS_NPC_VTAG_SIZE_ERR] = "NIX_SND_STATUS_NPC_VTAG_SIZE_ERR",
1389 [NIX_SND_STATUS_SEND_MEM_FAULT] = "NIX_SND_STATUS_SEND_MEM_FAULT",
1390 [NIX_SND_STATUS_SEND_STATS_ERR] = "NIX_SND_STATUS_SEND_STATS_ERR",
1391 };
1392
otx2_q_intr_handler(int irq,void * data)1393 static irqreturn_t otx2_q_intr_handler(int irq, void *data)
1394 {
1395 struct otx2_nic *pf = data;
1396 struct otx2_snd_queue *sq;
1397 void __iomem *ptr;
1398 u64 val, qidx = 0;
1399
1400 /* CQ */
1401 for (qidx = 0; qidx < pf->qset.cq_cnt; qidx++) {
1402 ptr = otx2_get_regaddr(pf, NIX_LF_CQ_OP_INT);
1403 val = otx2_atomic64_add((qidx << 44), ptr);
1404
1405 otx2_write64(pf, NIX_LF_CQ_OP_INT, (qidx << 44) |
1406 (val & NIX_CQERRINT_BITS));
1407 if (!(val & (NIX_CQERRINT_BITS | BIT_ULL(42))))
1408 continue;
1409
1410 if (val & BIT_ULL(42)) {
1411 netdev_err(pf->netdev,
1412 "CQ%lld: error reading NIX_LF_CQ_OP_INT, NIX_LF_ERR_INT 0x%llx\n",
1413 qidx, otx2_read64(pf, NIX_LF_ERR_INT));
1414 } else {
1415 if (val & BIT_ULL(NIX_CQERRINT_DOOR_ERR))
1416 netdev_err(pf->netdev, "CQ%lld: Doorbell error",
1417 qidx);
1418 if (val & BIT_ULL(NIX_CQERRINT_CQE_FAULT))
1419 netdev_err(pf->netdev,
1420 "CQ%lld: Memory fault on CQE write to LLC/DRAM",
1421 qidx);
1422 }
1423
1424 schedule_work(&pf->reset_task);
1425 }
1426
1427 /* SQ */
1428 for (qidx = 0; qidx < otx2_get_total_tx_queues(pf); qidx++) {
1429 u64 sq_op_err_dbg, mnq_err_dbg, snd_err_dbg;
1430 u8 sq_op_err_code, mnq_err_code, snd_err_code;
1431
1432 sq = &pf->qset.sq[qidx];
1433 if (!sq->sqb_ptrs)
1434 continue;
1435
1436 /* Below debug registers captures first errors corresponding to
1437 * those registers. We don't have to check against SQ qid as
1438 * these are fatal errors.
1439 */
1440
1441 ptr = otx2_get_regaddr(pf, NIX_LF_SQ_OP_INT);
1442 val = otx2_atomic64_add((qidx << 44), ptr);
1443 otx2_write64(pf, NIX_LF_SQ_OP_INT, (qidx << 44) |
1444 (val & NIX_SQINT_BITS));
1445
1446 if (val & BIT_ULL(42)) {
1447 netdev_err(pf->netdev,
1448 "SQ%lld: error reading NIX_LF_SQ_OP_INT, NIX_LF_ERR_INT 0x%llx\n",
1449 qidx, otx2_read64(pf, NIX_LF_ERR_INT));
1450 goto done;
1451 }
1452
1453 sq_op_err_dbg = otx2_read64(pf, NIX_LF_SQ_OP_ERR_DBG);
1454 if (!(sq_op_err_dbg & BIT(44)))
1455 goto chk_mnq_err_dbg;
1456
1457 sq_op_err_code = FIELD_GET(GENMASK(7, 0), sq_op_err_dbg);
1458 netdev_err(pf->netdev,
1459 "SQ%lld: NIX_LF_SQ_OP_ERR_DBG(0x%llx) err=%s(%#x)\n",
1460 qidx, sq_op_err_dbg,
1461 nix_sqoperr_e_str[sq_op_err_code],
1462 sq_op_err_code);
1463
1464 otx2_write64(pf, NIX_LF_SQ_OP_ERR_DBG, BIT_ULL(44));
1465
1466 if (sq_op_err_code == NIX_SQOPERR_SQB_NULL)
1467 goto chk_mnq_err_dbg;
1468
1469 /* Err is not NIX_SQOPERR_SQB_NULL, call aq function to read SQ structure.
1470 * TODO: But we are in irq context. How to call mbox functions which does sleep
1471 */
1472
1473 chk_mnq_err_dbg:
1474 mnq_err_dbg = otx2_read64(pf, NIX_LF_MNQ_ERR_DBG);
1475 if (!(mnq_err_dbg & BIT(44)))
1476 goto chk_snd_err_dbg;
1477
1478 mnq_err_code = FIELD_GET(GENMASK(7, 0), mnq_err_dbg);
1479 netdev_err(pf->netdev,
1480 "SQ%lld: NIX_LF_MNQ_ERR_DBG(0x%llx) err=%s(%#x)\n",
1481 qidx, mnq_err_dbg, nix_mnqerr_e_str[mnq_err_code],
1482 mnq_err_code);
1483 otx2_write64(pf, NIX_LF_MNQ_ERR_DBG, BIT_ULL(44));
1484
1485 chk_snd_err_dbg:
1486 snd_err_dbg = otx2_read64(pf, NIX_LF_SEND_ERR_DBG);
1487 if (snd_err_dbg & BIT(44)) {
1488 snd_err_code = FIELD_GET(GENMASK(7, 0), snd_err_dbg);
1489 netdev_err(pf->netdev,
1490 "SQ%lld: NIX_LF_SND_ERR_DBG:0x%llx err=%s(%#x)\n",
1491 qidx, snd_err_dbg,
1492 nix_snd_status_e_str[snd_err_code],
1493 snd_err_code);
1494 otx2_write64(pf, NIX_LF_SEND_ERR_DBG, BIT_ULL(44));
1495 }
1496
1497 done:
1498 /* Print values and reset */
1499 if (val & BIT_ULL(NIX_SQINT_SQB_ALLOC_FAIL))
1500 netdev_err(pf->netdev, "SQ%lld: SQB allocation failed",
1501 qidx);
1502
1503 schedule_work(&pf->reset_task);
1504 }
1505
1506 return IRQ_HANDLED;
1507 }
1508
otx2_cq_intr_handler(int irq,void * cq_irq)1509 irqreturn_t otx2_cq_intr_handler(int irq, void *cq_irq)
1510 {
1511 struct otx2_cq_poll *cq_poll = (struct otx2_cq_poll *)cq_irq;
1512 struct otx2_nic *pf = (struct otx2_nic *)cq_poll->dev;
1513 int qidx = cq_poll->cint_idx;
1514
1515 /* Disable interrupts.
1516 *
1517 * Completion interrupts behave in a level-triggered interrupt
1518 * fashion, and hence have to be cleared only after it is serviced.
1519 */
1520 otx2_write64(pf, NIX_LF_CINTX_ENA_W1C(qidx), BIT_ULL(0));
1521
1522 /* Schedule NAPI */
1523 pf->napi_events++;
1524 napi_schedule_irqoff(&cq_poll->napi);
1525
1526 return IRQ_HANDLED;
1527 }
1528 EXPORT_SYMBOL(otx2_cq_intr_handler);
1529
otx2_disable_napi(struct otx2_nic * pf)1530 void otx2_disable_napi(struct otx2_nic *pf)
1531 {
1532 struct otx2_qset *qset = &pf->qset;
1533 struct otx2_cq_poll *cq_poll;
1534 struct work_struct *work;
1535 int qidx;
1536
1537 for (qidx = 0; qidx < pf->hw.cint_cnt; qidx++) {
1538 cq_poll = &qset->napi[qidx];
1539 work = &cq_poll->dim.work;
1540 if (work->func)
1541 cancel_work_sync(work);
1542 napi_disable(&cq_poll->napi);
1543 netif_napi_del(&cq_poll->napi);
1544 }
1545 }
1546 EXPORT_SYMBOL(otx2_disable_napi);
1547
otx2_free_cq_res(struct otx2_nic * pf)1548 static void otx2_free_cq_res(struct otx2_nic *pf)
1549 {
1550 struct otx2_qset *qset = &pf->qset;
1551 struct otx2_cq_queue *cq;
1552 int qidx;
1553
1554 /* Disable CQs */
1555 otx2_ctx_disable(&pf->mbox, NIX_AQ_CTYPE_CQ, false);
1556 for (qidx = 0; qidx < qset->cq_cnt; qidx++) {
1557 cq = &qset->cq[qidx];
1558 qmem_free(pf->dev, cq->cqe);
1559 }
1560 }
1561
otx2_free_sq_res(struct otx2_nic * pf)1562 static void otx2_free_sq_res(struct otx2_nic *pf)
1563 {
1564 struct otx2_qset *qset = &pf->qset;
1565 struct otx2_snd_queue *sq;
1566 int qidx;
1567
1568 /* Disable SQs */
1569 otx2_ctx_disable(&pf->mbox, NIX_AQ_CTYPE_SQ, false);
1570 /* Free SQB pointers */
1571 otx2_sq_free_sqbs(pf);
1572 for (qidx = 0; qidx < otx2_get_total_tx_queues(pf); qidx++) {
1573 sq = &qset->sq[qidx];
1574 /* Skip freeing Qos queues if they are not initialized */
1575 if (!sq->sqe)
1576 continue;
1577 qmem_free(pf->dev, sq->sqe);
1578 qmem_free(pf->dev, sq->sqe_ring);
1579 qmem_free(pf->dev, sq->cpt_resp);
1580 qmem_free(pf->dev, sq->tso_hdrs);
1581 kfree(sq->sg);
1582 kfree(sq->sqb_ptrs);
1583 }
1584 }
1585
otx2_get_rbuf_size(struct otx2_nic * pf,int mtu)1586 static int otx2_get_rbuf_size(struct otx2_nic *pf, int mtu)
1587 {
1588 int frame_size;
1589 int total_size;
1590 int rbuf_size;
1591
1592 if (pf->hw.rbuf_len)
1593 return ALIGN(pf->hw.rbuf_len, OTX2_ALIGN) + OTX2_HEAD_ROOM;
1594
1595 /* The data transferred by NIX to memory consists of actual packet
1596 * plus additional data which has timestamp and/or EDSA/HIGIG2
1597 * headers if interface is configured in corresponding modes.
1598 * NIX transfers entire data using 6 segments/buffers and writes
1599 * a CQE_RX descriptor with those segment addresses. First segment
1600 * has additional data prepended to packet. Also software omits a
1601 * headroom of 128 bytes in each segment. Hence the total size of
1602 * memory needed to receive a packet with 'mtu' is:
1603 * frame size = mtu + additional data;
1604 * memory = frame_size + headroom * 6;
1605 * each receive buffer size = memory / 6;
1606 */
1607 frame_size = mtu + OTX2_ETH_HLEN + OTX2_HW_TIMESTAMP_LEN;
1608 total_size = frame_size + OTX2_HEAD_ROOM * 6;
1609 rbuf_size = total_size / 6;
1610
1611 return ALIGN(rbuf_size, 2048);
1612 }
1613
otx2_init_hw_resources(struct otx2_nic * pf)1614 int otx2_init_hw_resources(struct otx2_nic *pf)
1615 {
1616 struct nix_lf_free_req *free_req;
1617 struct mbox *mbox = &pf->mbox;
1618 struct otx2_hw *hw = &pf->hw;
1619 struct msg_req *req;
1620 int err = 0, lvl;
1621
1622 /* Set required NPA LF's pool counts
1623 * Auras and Pools are used in a 1:1 mapping,
1624 * so, aura count = pool count.
1625 */
1626 hw->rqpool_cnt = hw->rx_queues;
1627 hw->sqpool_cnt = otx2_get_total_tx_queues(pf);
1628 hw->pool_cnt = hw->rqpool_cnt + hw->sqpool_cnt;
1629
1630 if (!otx2_rep_dev(pf->pdev)) {
1631 /* Maximum hardware supported transmit length */
1632 pf->tx_max_pktlen = pf->netdev->max_mtu + OTX2_ETH_HLEN;
1633 pf->rbsize = otx2_get_rbuf_size(pf, pf->netdev->mtu);
1634 }
1635
1636 mutex_lock(&mbox->lock);
1637 /* NPA init */
1638 err = otx2_config_npa(pf);
1639 if (err)
1640 goto exit;
1641
1642 /* NIX init */
1643 err = otx2_config_nix(pf);
1644 if (err)
1645 goto err_free_npa_lf;
1646
1647 /* Default disable backpressure on NIX-CPT */
1648 otx2_nix_cpt_config_bp(pf, false);
1649
1650 /* Enable backpressure for CGX mapped PF/VFs */
1651 if (!is_otx2_lbkvf(pf->pdev))
1652 otx2_nix_config_bp(pf, true);
1653
1654 /* Init Auras and pools used by NIX RQ, for free buffer ptrs */
1655 err = otx2_rq_aura_pool_init(pf);
1656 if (err) {
1657 mutex_unlock(&mbox->lock);
1658 goto err_free_nix_lf;
1659 }
1660 /* Init Auras and pools used by NIX SQ, for queueing SQEs */
1661 err = otx2_sq_aura_pool_init(pf);
1662 if (err) {
1663 mutex_unlock(&mbox->lock);
1664 goto err_free_rq_ptrs;
1665 }
1666
1667 err = otx2_txsch_alloc(pf);
1668 if (err) {
1669 mutex_unlock(&mbox->lock);
1670 goto err_free_sq_ptrs;
1671 }
1672
1673 #ifdef CONFIG_DCB
1674 if (pf->pfc_en) {
1675 err = otx2_pfc_txschq_alloc(pf);
1676 if (err) {
1677 mutex_unlock(&mbox->lock);
1678 goto err_free_sq_ptrs;
1679 }
1680 }
1681 #endif
1682
1683 err = otx2_config_nix_queues(pf);
1684 if (err) {
1685 mutex_unlock(&mbox->lock);
1686 goto err_free_txsch;
1687 }
1688
1689 for (lvl = 0; lvl < NIX_TXSCH_LVL_CNT; lvl++) {
1690 int idx;
1691
1692 for (idx = 0; idx < pf->hw.txschq_cnt[lvl]; idx++) {
1693 err = otx2_txschq_config(pf, lvl, idx, false);
1694 if (err) {
1695 dev_err(pf->dev, "Failed to config TXSCH\n");
1696 mutex_unlock(&mbox->lock);
1697 goto err_free_nix_queues;
1698 }
1699 }
1700 }
1701
1702 #ifdef CONFIG_DCB
1703 if (pf->pfc_en) {
1704 err = otx2_pfc_txschq_config(pf);
1705 if (err) {
1706 mutex_unlock(&mbox->lock);
1707 goto err_free_nix_queues;
1708 }
1709 }
1710 #endif
1711
1712 mutex_unlock(&mbox->lock);
1713 return err;
1714
1715 err_free_nix_queues:
1716 otx2_free_sq_res(pf);
1717 otx2_free_cq_res(pf);
1718 otx2_ctx_disable(mbox, NIX_AQ_CTYPE_RQ, false);
1719 err_free_txsch:
1720 otx2_txschq_stop(pf);
1721 err_free_sq_ptrs:
1722 otx2_sq_free_sqbs(pf);
1723 err_free_rq_ptrs:
1724 otx2_free_aura_ptr(pf, AURA_NIX_RQ);
1725 otx2_ctx_disable(mbox, NPA_AQ_CTYPE_POOL, true);
1726 otx2_ctx_disable(mbox, NPA_AQ_CTYPE_AURA, true);
1727 otx2_aura_pool_free(pf);
1728 err_free_nix_lf:
1729 mutex_lock(&mbox->lock);
1730 free_req = otx2_mbox_alloc_msg_nix_lf_free(mbox);
1731 if (free_req) {
1732 free_req->flags = NIX_LF_DISABLE_FLOWS;
1733 if (otx2_sync_mbox_msg(mbox))
1734 dev_err(pf->dev, "%s failed to free nixlf\n", __func__);
1735 }
1736 err_free_npa_lf:
1737 /* Reset NPA LF */
1738 req = otx2_mbox_alloc_msg_npa_lf_free(mbox);
1739 if (req) {
1740 if (otx2_sync_mbox_msg(mbox))
1741 dev_err(pf->dev, "%s failed to free npalf\n", __func__);
1742 }
1743 exit:
1744 mutex_unlock(&mbox->lock);
1745 return err;
1746 }
1747 EXPORT_SYMBOL(otx2_init_hw_resources);
1748
otx2_free_hw_resources(struct otx2_nic * pf)1749 void otx2_free_hw_resources(struct otx2_nic *pf)
1750 {
1751 struct otx2_qset *qset = &pf->qset;
1752 struct nix_lf_free_req *free_req;
1753 struct mbox *mbox = &pf->mbox;
1754 struct otx2_cq_queue *cq;
1755 struct msg_req *req;
1756 int qidx;
1757
1758 /* Ensure all SQE are processed */
1759 otx2_sqb_flush(pf);
1760
1761 /* Stop transmission */
1762 otx2_txschq_stop(pf);
1763
1764 #ifdef CONFIG_DCB
1765 if (pf->pfc_en)
1766 otx2_pfc_txschq_stop(pf);
1767 #endif
1768
1769 if (!otx2_rep_dev(pf->pdev))
1770 otx2_clean_qos_queues(pf);
1771
1772 mutex_lock(&mbox->lock);
1773 /* Disable backpressure */
1774 if (!is_otx2_lbkvf(pf->pdev))
1775 otx2_nix_config_bp(pf, false);
1776 mutex_unlock(&mbox->lock);
1777
1778 /* Disable RQs */
1779 otx2_ctx_disable(mbox, NIX_AQ_CTYPE_RQ, false);
1780
1781 /*Dequeue all CQEs */
1782 for (qidx = 0; qidx < qset->cq_cnt; qidx++) {
1783 cq = &qset->cq[qidx];
1784 if (cq->cq_type == CQ_RX)
1785 otx2_cleanup_rx_cqes(pf, cq, qidx);
1786 else
1787 otx2_cleanup_tx_cqes(pf, cq);
1788 }
1789 otx2_free_pending_sqe(pf);
1790
1791 otx2_free_sq_res(pf);
1792
1793 /* Free RQ buffer pointers*/
1794 otx2_free_aura_ptr(pf, AURA_NIX_RQ);
1795
1796 otx2_free_cq_res(pf);
1797
1798 /* Free all ingress bandwidth profiles allocated */
1799 if (!otx2_rep_dev(pf->pdev))
1800 cn10k_free_all_ipolicers(pf);
1801
1802 mutex_lock(&mbox->lock);
1803 /* Reset NIX LF */
1804 free_req = otx2_mbox_alloc_msg_nix_lf_free(mbox);
1805 if (free_req) {
1806 free_req->flags = NIX_LF_DISABLE_FLOWS;
1807 if (!(pf->flags & OTX2_FLAG_PF_SHUTDOWN))
1808 free_req->flags |= NIX_LF_DONT_FREE_TX_VTAG;
1809 if (otx2_sync_mbox_msg(mbox))
1810 dev_err(pf->dev, "%s failed to free nixlf\n", __func__);
1811 }
1812 mutex_unlock(&mbox->lock);
1813
1814 /* Disable NPA Pool and Aura hw context */
1815 otx2_ctx_disable(mbox, NPA_AQ_CTYPE_POOL, true);
1816 otx2_ctx_disable(mbox, NPA_AQ_CTYPE_AURA, true);
1817 otx2_aura_pool_free(pf);
1818
1819 mutex_lock(&mbox->lock);
1820 /* Reset NPA LF */
1821 req = otx2_mbox_alloc_msg_npa_lf_free(mbox);
1822 if (req) {
1823 if (otx2_sync_mbox_msg(mbox))
1824 dev_err(pf->dev, "%s failed to free npalf\n", __func__);
1825 }
1826 mutex_unlock(&mbox->lock);
1827 }
1828 EXPORT_SYMBOL(otx2_free_hw_resources);
1829
otx2_promisc_use_mce_list(struct otx2_nic * pfvf)1830 static bool otx2_promisc_use_mce_list(struct otx2_nic *pfvf)
1831 {
1832 int vf;
1833
1834 /* The AF driver will determine whether to allow the VF netdev or not */
1835 if (is_otx2_vf(pfvf->pcifunc))
1836 return true;
1837
1838 /* check if there are any trusted VFs associated with the PF netdev */
1839 for (vf = 0; vf < pci_num_vf(pfvf->pdev); vf++)
1840 if (pfvf->vf_configs[vf].trusted)
1841 return true;
1842 return false;
1843 }
1844
otx2_do_set_rx_mode(struct otx2_nic * pf)1845 static void otx2_do_set_rx_mode(struct otx2_nic *pf)
1846 {
1847 struct net_device *netdev = pf->netdev;
1848 struct nix_rx_mode *req;
1849 bool promisc = false;
1850
1851 if (!(netdev->flags & IFF_UP))
1852 return;
1853
1854 if ((netdev->flags & IFF_PROMISC) ||
1855 (netdev_uc_count(netdev) > pf->flow_cfg->ucast_flt_cnt)) {
1856 promisc = true;
1857 }
1858
1859 /* Write unicast address to mcam entries or del from mcam */
1860 if (!promisc && netdev->priv_flags & IFF_UNICAST_FLT)
1861 __dev_uc_sync(netdev, otx2_add_macfilter, otx2_del_macfilter);
1862
1863 mutex_lock(&pf->mbox.lock);
1864 req = otx2_mbox_alloc_msg_nix_set_rx_mode(&pf->mbox);
1865 if (!req) {
1866 mutex_unlock(&pf->mbox.lock);
1867 return;
1868 }
1869
1870 req->mode = NIX_RX_MODE_UCAST;
1871
1872 if (promisc)
1873 req->mode |= NIX_RX_MODE_PROMISC;
1874 if (netdev->flags & (IFF_ALLMULTI | IFF_MULTICAST))
1875 req->mode |= NIX_RX_MODE_ALLMULTI;
1876
1877 if (otx2_promisc_use_mce_list(pf))
1878 req->mode |= NIX_RX_MODE_USE_MCE;
1879
1880 otx2_sync_mbox_msg(&pf->mbox);
1881 mutex_unlock(&pf->mbox.lock);
1882 }
1883
otx2_set_irq_coalesce(struct otx2_nic * pfvf)1884 static void otx2_set_irq_coalesce(struct otx2_nic *pfvf)
1885 {
1886 int cint;
1887
1888 for (cint = 0; cint < pfvf->hw.cint_cnt; cint++)
1889 otx2_config_irq_coalescing(pfvf, cint);
1890 }
1891
otx2_dim_work(struct work_struct * w)1892 static void otx2_dim_work(struct work_struct *w)
1893 {
1894 struct dim_cq_moder cur_moder;
1895 struct otx2_cq_poll *cq_poll;
1896 struct otx2_nic *pfvf;
1897 struct dim *dim;
1898
1899 dim = container_of(w, struct dim, work);
1900 cur_moder = net_dim_get_rx_moderation(dim->mode, dim->profile_ix);
1901 cq_poll = container_of(dim, struct otx2_cq_poll, dim);
1902 pfvf = (struct otx2_nic *)cq_poll->dev;
1903 pfvf->hw.cq_time_wait = (cur_moder.usec > CQ_TIMER_THRESH_MAX) ?
1904 CQ_TIMER_THRESH_MAX : cur_moder.usec;
1905 pfvf->hw.cq_ecount_wait = (cur_moder.pkts > NAPI_POLL_WEIGHT) ?
1906 NAPI_POLL_WEIGHT : cur_moder.pkts;
1907 otx2_set_irq_coalesce(pfvf);
1908 dim->state = DIM_START_MEASURE;
1909 }
1910
otx2_free_queue_mem(struct otx2_qset * qset)1911 void otx2_free_queue_mem(struct otx2_qset *qset)
1912 {
1913 kfree(qset->sq);
1914 qset->sq = NULL;
1915 kfree(qset->cq);
1916 qset->cq = NULL;
1917 kfree(qset->rq);
1918 qset->rq = NULL;
1919 kfree(qset->napi);
1920 qset->napi = NULL;
1921 }
1922 EXPORT_SYMBOL(otx2_free_queue_mem);
1923
otx2_alloc_queue_mem(struct otx2_nic * pf)1924 int otx2_alloc_queue_mem(struct otx2_nic *pf)
1925 {
1926 struct otx2_qset *qset = &pf->qset;
1927 struct otx2_cq_poll *cq_poll;
1928
1929
1930 /* RQ and SQs are mapped to different CQs,
1931 * so find out max CQ IRQs (i.e CINTs) needed.
1932 */
1933 pf->hw.non_qos_queues = pf->hw.tx_queues + pf->hw.xdp_queues;
1934 pf->hw.cint_cnt = max3(pf->hw.rx_queues, pf->hw.tx_queues,
1935 pf->hw.tc_tx_queues);
1936
1937 pf->qset.cq_cnt = pf->hw.rx_queues + otx2_get_total_tx_queues(pf);
1938
1939 qset->napi = kcalloc(pf->hw.cint_cnt, sizeof(*cq_poll), GFP_KERNEL);
1940 if (!qset->napi)
1941 return -ENOMEM;
1942
1943 /* CQ size of RQ */
1944 qset->rqe_cnt = qset->rqe_cnt ? qset->rqe_cnt : Q_COUNT(Q_SIZE_256);
1945 /* CQ size of SQ */
1946 qset->sqe_cnt = qset->sqe_cnt ? qset->sqe_cnt : Q_COUNT(Q_SIZE_4K);
1947
1948 qset->cq = kcalloc(pf->qset.cq_cnt,
1949 sizeof(struct otx2_cq_queue), GFP_KERNEL);
1950 if (!qset->cq)
1951 goto err_free_mem;
1952
1953 qset->sq = kcalloc(otx2_get_total_tx_queues(pf),
1954 sizeof(struct otx2_snd_queue), GFP_KERNEL);
1955 if (!qset->sq)
1956 goto err_free_mem;
1957
1958 qset->rq = kcalloc(pf->hw.rx_queues,
1959 sizeof(struct otx2_rcv_queue), GFP_KERNEL);
1960 if (!qset->rq)
1961 goto err_free_mem;
1962
1963 return 0;
1964
1965 err_free_mem:
1966 otx2_free_queue_mem(qset);
1967 return -ENOMEM;
1968 }
1969 EXPORT_SYMBOL(otx2_alloc_queue_mem);
1970
otx2_open(struct net_device * netdev)1971 int otx2_open(struct net_device *netdev)
1972 {
1973 struct otx2_nic *pf = netdev_priv(netdev);
1974 struct otx2_cq_poll *cq_poll = NULL;
1975 struct otx2_qset *qset = &pf->qset;
1976 int err = 0, qidx, vec;
1977 char *irq_name;
1978
1979 netif_carrier_off(netdev);
1980
1981 err = otx2_alloc_queue_mem(pf);
1982 if (err)
1983 return err;
1984
1985 err = otx2_init_hw_resources(pf);
1986 if (err)
1987 goto err_free_mem;
1988
1989 /* Register NAPI handler */
1990 for (qidx = 0; qidx < pf->hw.cint_cnt; qidx++) {
1991 cq_poll = &qset->napi[qidx];
1992 cq_poll->cint_idx = qidx;
1993 /* RQ0 & SQ0 are mapped to CINT0 and so on..
1994 * 'cq_ids[0]' points to RQ's CQ and
1995 * 'cq_ids[1]' points to SQ's CQ and
1996 * 'cq_ids[2]' points to XDP's CQ and
1997 */
1998 cq_poll->cq_ids[CQ_RX] =
1999 (qidx < pf->hw.rx_queues) ? qidx : CINT_INVALID_CQ;
2000 cq_poll->cq_ids[CQ_TX] = (qidx < pf->hw.tx_queues) ?
2001 qidx + pf->hw.rx_queues : CINT_INVALID_CQ;
2002 if (pf->xdp_prog)
2003 cq_poll->cq_ids[CQ_XDP] = (qidx < pf->hw.xdp_queues) ?
2004 (qidx + pf->hw.rx_queues +
2005 pf->hw.tx_queues) :
2006 CINT_INVALID_CQ;
2007 else
2008 cq_poll->cq_ids[CQ_XDP] = CINT_INVALID_CQ;
2009
2010 cq_poll->cq_ids[CQ_QOS] = (qidx < pf->hw.tc_tx_queues) ?
2011 (qidx + pf->hw.rx_queues +
2012 pf->hw.non_qos_queues) :
2013 CINT_INVALID_CQ;
2014
2015 cq_poll->dev = (void *)pf;
2016 cq_poll->dim.mode = DIM_CQ_PERIOD_MODE_START_FROM_CQE;
2017 INIT_WORK(&cq_poll->dim.work, otx2_dim_work);
2018 netif_napi_add(netdev, &cq_poll->napi, otx2_napi_handler);
2019 napi_enable(&cq_poll->napi);
2020 }
2021
2022 /* Set maximum frame size allowed in HW */
2023 err = otx2_hw_set_mtu(pf, netdev->mtu);
2024 if (err)
2025 goto err_disable_napi;
2026
2027 /* Setup segmentation algorithms, if failed, clear offload capability */
2028 otx2_setup_segmentation(pf);
2029
2030 /* Initialize RSS */
2031 err = otx2_rss_init(pf);
2032 if (err)
2033 goto err_disable_napi;
2034
2035 /* Register Queue IRQ handlers */
2036 vec = pf->hw.nix_msixoff + NIX_LF_QINT_VEC_START;
2037 irq_name = &pf->hw.irq_name[vec * NAME_SIZE];
2038
2039 snprintf(irq_name, NAME_SIZE, "%s-qerr", pf->netdev->name);
2040
2041 err = request_irq(pci_irq_vector(pf->pdev, vec),
2042 otx2_q_intr_handler, 0, irq_name, pf);
2043 if (err) {
2044 dev_err(pf->dev,
2045 "RVUPF%d: IRQ registration failed for QERR\n",
2046 rvu_get_pf(pf->pdev, pf->pcifunc));
2047 goto err_disable_napi;
2048 }
2049
2050 /* Enable QINT IRQ */
2051 otx2_write64(pf, NIX_LF_QINTX_ENA_W1S(0), BIT_ULL(0));
2052
2053 /* Register CQ IRQ handlers */
2054 vec = pf->hw.nix_msixoff + NIX_LF_CINT_VEC_START;
2055 for (qidx = 0; qidx < pf->hw.cint_cnt; qidx++) {
2056 irq_name = &pf->hw.irq_name[vec * NAME_SIZE];
2057 int name_len;
2058
2059 name_len = snprintf(irq_name, NAME_SIZE, "%s-rxtx-%d",
2060 pf->netdev->name, qidx);
2061 if (name_len >= NAME_SIZE) {
2062 dev_err(pf->dev,
2063 "RVUPF%d: IRQ registration failed for CQ%d, irq name is too long\n",
2064 rvu_get_pf(pf->pdev, pf->pcifunc), qidx);
2065 err = -EINVAL;
2066 goto err_free_cints;
2067 }
2068
2069 err = request_irq(pci_irq_vector(pf->pdev, vec),
2070 otx2_cq_intr_handler, 0, irq_name,
2071 &qset->napi[qidx]);
2072 if (err) {
2073 dev_err(pf->dev,
2074 "RVUPF%d: IRQ registration failed for CQ%d\n",
2075 rvu_get_pf(pf->pdev, pf->pcifunc), qidx);
2076 goto err_free_cints;
2077 }
2078 vec++;
2079
2080 otx2_config_irq_coalescing(pf, qidx);
2081
2082 /* Enable CQ IRQ */
2083 otx2_write64(pf, NIX_LF_CINTX_INT(qidx), BIT_ULL(0));
2084 otx2_write64(pf, NIX_LF_CINTX_ENA_W1S(qidx), BIT_ULL(0));
2085 }
2086
2087 otx2_set_cints_affinity(pf);
2088
2089 if (pf->flags & OTX2_FLAG_RX_VLAN_SUPPORT)
2090 otx2_enable_rxvlan(pf, true);
2091
2092 /* When reinitializing enable time stamping if it is enabled before */
2093 if (pf->flags & OTX2_FLAG_TX_TSTAMP_ENABLED) {
2094 pf->flags &= ~OTX2_FLAG_TX_TSTAMP_ENABLED;
2095 otx2_config_hw_tx_tstamp(pf, true);
2096 }
2097 if (pf->flags & OTX2_FLAG_RX_TSTAMP_ENABLED) {
2098 pf->flags &= ~OTX2_FLAG_RX_TSTAMP_ENABLED;
2099 otx2_config_hw_rx_tstamp(pf, true);
2100 }
2101
2102 pf->flags &= ~OTX2_FLAG_INTF_DOWN;
2103 pf->flags &= ~OTX2_FLAG_PORT_UP;
2104 /* 'intf_down' may be checked on any cpu */
2105 smp_wmb();
2106
2107 /* Enable QoS configuration before starting tx queues */
2108 otx2_qos_config_txschq(pf);
2109
2110 /* we have already received link status notification */
2111 if (pf->linfo.link_up && !(pf->pcifunc & RVU_PFVF_FUNC_MASK))
2112 otx2_handle_link_event(pf);
2113
2114 /* Install DMAC Filters */
2115 if (pf->flags & OTX2_FLAG_DMACFLTR_SUPPORT)
2116 otx2_dmacflt_reinstall_flows(pf);
2117
2118 otx2_tc_apply_ingress_police_rules(pf);
2119
2120 err = otx2_rxtx_enable(pf, true);
2121 /* If a mbox communication error happens at this point then interface
2122 * will end up in a state such that it is in down state but hardware
2123 * mcam entries are enabled to receive the packets. Hence disable the
2124 * packet I/O.
2125 */
2126 if (err == -EIO)
2127 goto err_disable_rxtx;
2128 else if (err)
2129 goto err_tx_stop_queues;
2130
2131 otx2_do_set_rx_mode(pf);
2132
2133 return 0;
2134
2135 err_disable_rxtx:
2136 otx2_rxtx_enable(pf, false);
2137 err_tx_stop_queues:
2138 netif_tx_stop_all_queues(netdev);
2139 netif_carrier_off(netdev);
2140 pf->flags |= OTX2_FLAG_INTF_DOWN;
2141 err_free_cints:
2142 otx2_free_cints(pf, qidx);
2143 vec = pci_irq_vector(pf->pdev,
2144 pf->hw.nix_msixoff + NIX_LF_QINT_VEC_START);
2145 otx2_write64(pf, NIX_LF_QINTX_ENA_W1C(0), BIT_ULL(0));
2146 free_irq(vec, pf);
2147 err_disable_napi:
2148 otx2_disable_napi(pf);
2149 otx2_free_hw_resources(pf);
2150 err_free_mem:
2151 otx2_free_queue_mem(qset);
2152 return err;
2153 }
2154 EXPORT_SYMBOL(otx2_open);
2155
otx2_stop(struct net_device * netdev)2156 int otx2_stop(struct net_device *netdev)
2157 {
2158 struct otx2_nic *pf = netdev_priv(netdev);
2159 struct otx2_cq_poll *cq_poll = NULL;
2160 struct otx2_qset *qset = &pf->qset;
2161 int qidx, vec, wrk;
2162
2163 /* If the DOWN flag is set resources are already freed */
2164 if (pf->flags & OTX2_FLAG_INTF_DOWN)
2165 return 0;
2166
2167 netif_carrier_off(netdev);
2168 netif_tx_stop_all_queues(netdev);
2169
2170 pf->flags |= OTX2_FLAG_INTF_DOWN;
2171 /* 'intf_down' may be checked on any cpu */
2172 smp_wmb();
2173
2174 /* First stop packet Rx/Tx */
2175 otx2_rxtx_enable(pf, false);
2176
2177 /* Clear RSS enable flag */
2178 pf->hw.rss_info.enable = false;
2179
2180 /* Cleanup Queue IRQ */
2181 vec = pci_irq_vector(pf->pdev,
2182 pf->hw.nix_msixoff + NIX_LF_QINT_VEC_START);
2183 otx2_write64(pf, NIX_LF_QINTX_ENA_W1C(0), BIT_ULL(0));
2184 free_irq(vec, pf);
2185
2186 /* Cleanup CQ NAPI and IRQ */
2187 vec = pf->hw.nix_msixoff + NIX_LF_CINT_VEC_START;
2188 for (qidx = 0; qidx < pf->hw.cint_cnt; qidx++) {
2189 /* Disable interrupt */
2190 otx2_write64(pf, NIX_LF_CINTX_ENA_W1C(qidx), BIT_ULL(0));
2191
2192 synchronize_irq(pci_irq_vector(pf->pdev, vec));
2193
2194 cq_poll = &qset->napi[qidx];
2195 napi_synchronize(&cq_poll->napi);
2196 vec++;
2197 }
2198
2199 netif_tx_disable(netdev);
2200
2201 for (wrk = 0; wrk < pf->qset.cq_cnt; wrk++)
2202 cancel_delayed_work_sync(&pf->refill_wrk[wrk].pool_refill_work);
2203 devm_kfree(pf->dev, pf->refill_wrk);
2204
2205 otx2_free_hw_resources(pf);
2206 otx2_free_cints(pf, pf->hw.cint_cnt);
2207 otx2_disable_napi(pf);
2208
2209 for (qidx = 0; qidx < netdev->num_tx_queues; qidx++)
2210 netdev_tx_reset_queue(netdev_get_tx_queue(netdev, qidx));
2211
2212 otx2_free_queue_mem(qset);
2213 /* Do not clear RQ/SQ ringsize settings */
2214 memset_startat(qset, 0, sqe_cnt);
2215 return 0;
2216 }
2217 EXPORT_SYMBOL(otx2_stop);
2218
otx2_xmit(struct sk_buff * skb,struct net_device * netdev)2219 static netdev_tx_t otx2_xmit(struct sk_buff *skb, struct net_device *netdev)
2220 {
2221 struct otx2_nic *pf = netdev_priv(netdev);
2222 int qidx = skb_get_queue_mapping(skb);
2223 struct otx2_snd_queue *sq;
2224 struct netdev_queue *txq;
2225 int sq_idx;
2226
2227 /* XDP SQs are not mapped with TXQs
2228 * advance qid to derive correct sq mapped with QOS
2229 */
2230 sq_idx = (qidx >= pf->hw.tx_queues) ? (qidx + pf->hw.xdp_queues) : qidx;
2231
2232 /* Check for minimum and maximum packet length */
2233 if (skb->len <= ETH_HLEN ||
2234 (!skb_shinfo(skb)->gso_size && skb->len > pf->tx_max_pktlen)) {
2235 dev_kfree_skb(skb);
2236 return NETDEV_TX_OK;
2237 }
2238
2239 sq = &pf->qset.sq[sq_idx];
2240 txq = netdev_get_tx_queue(netdev, qidx);
2241
2242 if (!otx2_sq_append_skb(pf, txq, sq, skb, qidx)) {
2243 netif_tx_stop_queue(txq);
2244
2245 /* Check again, incase SQBs got freed up */
2246 smp_mb();
2247 if (((sq->num_sqbs - *sq->aura_fc_addr) * sq->sqe_per_sqb)
2248 > sq->sqe_thresh)
2249 netif_tx_wake_queue(txq);
2250
2251 return NETDEV_TX_BUSY;
2252 }
2253
2254 return NETDEV_TX_OK;
2255 }
2256
otx2_qos_select_htb_queue(struct otx2_nic * pf,struct sk_buff * skb,u16 htb_maj_id)2257 static int otx2_qos_select_htb_queue(struct otx2_nic *pf, struct sk_buff *skb,
2258 u16 htb_maj_id)
2259 {
2260 u16 classid;
2261
2262 if ((TC_H_MAJ(skb->priority) >> 16) == htb_maj_id)
2263 classid = TC_H_MIN(skb->priority);
2264 else
2265 classid = READ_ONCE(pf->qos.defcls);
2266
2267 if (!classid)
2268 return 0;
2269
2270 return otx2_get_txq_by_classid(pf, classid);
2271 }
2272
otx2_select_queue(struct net_device * netdev,struct sk_buff * skb,struct net_device * sb_dev)2273 u16 otx2_select_queue(struct net_device *netdev, struct sk_buff *skb,
2274 struct net_device *sb_dev)
2275 {
2276 struct otx2_nic *pf = netdev_priv(netdev);
2277 bool qos_enabled;
2278 #ifdef CONFIG_DCB
2279 u8 vlan_prio;
2280 #endif
2281 int txq;
2282
2283 qos_enabled = netdev->real_num_tx_queues > pf->hw.tx_queues;
2284 if (unlikely(qos_enabled)) {
2285 /* This smp_load_acquire() pairs with smp_store_release() in
2286 * otx2_qos_root_add() called from htb offload root creation
2287 */
2288 u16 htb_maj_id = smp_load_acquire(&pf->qos.maj_id);
2289
2290 if (unlikely(htb_maj_id)) {
2291 txq = otx2_qos_select_htb_queue(pf, skb, htb_maj_id);
2292 if (txq > 0)
2293 return txq;
2294 goto process_pfc;
2295 }
2296 }
2297
2298 process_pfc:
2299 #ifdef CONFIG_DCB
2300 if (!skb_vlan_tag_present(skb))
2301 goto pick_tx;
2302
2303 vlan_prio = skb->vlan_tci >> 13;
2304 if ((vlan_prio > pf->hw.tx_queues - 1) ||
2305 !pf->pfc_alloc_status[vlan_prio])
2306 goto pick_tx;
2307
2308 return vlan_prio;
2309
2310 pick_tx:
2311 #endif
2312 txq = netdev_pick_tx(netdev, skb, NULL);
2313 if (unlikely(qos_enabled))
2314 return txq % pf->hw.tx_queues;
2315
2316 return txq;
2317 }
2318 EXPORT_SYMBOL(otx2_select_queue);
2319
otx2_fix_features(struct net_device * dev,netdev_features_t features)2320 static netdev_features_t otx2_fix_features(struct net_device *dev,
2321 netdev_features_t features)
2322 {
2323 if (features & NETIF_F_HW_VLAN_CTAG_RX)
2324 features |= NETIF_F_HW_VLAN_STAG_RX;
2325 else
2326 features &= ~NETIF_F_HW_VLAN_STAG_RX;
2327
2328 return features;
2329 }
2330
otx2_set_rx_mode(struct net_device * netdev)2331 static void otx2_set_rx_mode(struct net_device *netdev)
2332 {
2333 struct otx2_nic *pf = netdev_priv(netdev);
2334
2335 queue_work(pf->otx2_wq, &pf->rx_mode_work);
2336 }
2337
otx2_rx_mode_wrk_handler(struct work_struct * work)2338 static void otx2_rx_mode_wrk_handler(struct work_struct *work)
2339 {
2340 struct otx2_nic *pf = container_of(work, struct otx2_nic, rx_mode_work);
2341
2342 otx2_do_set_rx_mode(pf);
2343 }
2344
otx2_set_features(struct net_device * netdev,netdev_features_t features)2345 static int otx2_set_features(struct net_device *netdev,
2346 netdev_features_t features)
2347 {
2348 netdev_features_t changed = features ^ netdev->features;
2349 struct otx2_nic *pf = netdev_priv(netdev);
2350
2351 if ((changed & NETIF_F_LOOPBACK) && netif_running(netdev))
2352 return otx2_cgx_config_loopback(pf,
2353 features & NETIF_F_LOOPBACK);
2354
2355 if ((changed & NETIF_F_HW_VLAN_CTAG_RX) && netif_running(netdev))
2356 return otx2_enable_rxvlan(pf,
2357 features & NETIF_F_HW_VLAN_CTAG_RX);
2358
2359 if (changed & NETIF_F_HW_ESP)
2360 return cn10k_ipsec_ethtool_init(netdev,
2361 features & NETIF_F_HW_ESP);
2362
2363 return otx2_handle_ntuple_tc_features(netdev, features);
2364 }
2365
otx2_reset_task(struct work_struct * work)2366 static void otx2_reset_task(struct work_struct *work)
2367 {
2368 struct otx2_nic *pf = container_of(work, struct otx2_nic, reset_task);
2369
2370 if (!netif_running(pf->netdev))
2371 return;
2372
2373 rtnl_lock();
2374 otx2_stop(pf->netdev);
2375 pf->reset_count++;
2376 otx2_open(pf->netdev);
2377 netif_trans_update(pf->netdev);
2378 rtnl_unlock();
2379 }
2380
otx2_config_hw_rx_tstamp(struct otx2_nic * pfvf,bool enable)2381 static int otx2_config_hw_rx_tstamp(struct otx2_nic *pfvf, bool enable)
2382 {
2383 struct msg_req *req;
2384 int err;
2385
2386 if (pfvf->flags & OTX2_FLAG_RX_TSTAMP_ENABLED && enable)
2387 return 0;
2388
2389 mutex_lock(&pfvf->mbox.lock);
2390 if (enable)
2391 req = otx2_mbox_alloc_msg_cgx_ptp_rx_enable(&pfvf->mbox);
2392 else
2393 req = otx2_mbox_alloc_msg_cgx_ptp_rx_disable(&pfvf->mbox);
2394 if (!req) {
2395 mutex_unlock(&pfvf->mbox.lock);
2396 return -ENOMEM;
2397 }
2398
2399 err = otx2_sync_mbox_msg(&pfvf->mbox);
2400 if (err) {
2401 mutex_unlock(&pfvf->mbox.lock);
2402 return err;
2403 }
2404
2405 mutex_unlock(&pfvf->mbox.lock);
2406 if (enable)
2407 pfvf->flags |= OTX2_FLAG_RX_TSTAMP_ENABLED;
2408 else
2409 pfvf->flags &= ~OTX2_FLAG_RX_TSTAMP_ENABLED;
2410 return 0;
2411 }
2412
otx2_config_hw_tx_tstamp(struct otx2_nic * pfvf,bool enable)2413 static int otx2_config_hw_tx_tstamp(struct otx2_nic *pfvf, bool enable)
2414 {
2415 struct msg_req *req;
2416 int err;
2417
2418 if (pfvf->flags & OTX2_FLAG_TX_TSTAMP_ENABLED && enable)
2419 return 0;
2420
2421 mutex_lock(&pfvf->mbox.lock);
2422 if (enable)
2423 req = otx2_mbox_alloc_msg_nix_lf_ptp_tx_enable(&pfvf->mbox);
2424 else
2425 req = otx2_mbox_alloc_msg_nix_lf_ptp_tx_disable(&pfvf->mbox);
2426 if (!req) {
2427 mutex_unlock(&pfvf->mbox.lock);
2428 return -ENOMEM;
2429 }
2430
2431 err = otx2_sync_mbox_msg(&pfvf->mbox);
2432 if (err) {
2433 mutex_unlock(&pfvf->mbox.lock);
2434 return err;
2435 }
2436
2437 mutex_unlock(&pfvf->mbox.lock);
2438 if (enable)
2439 pfvf->flags |= OTX2_FLAG_TX_TSTAMP_ENABLED;
2440 else
2441 pfvf->flags &= ~OTX2_FLAG_TX_TSTAMP_ENABLED;
2442 return 0;
2443 }
2444
otx2_config_hwtstamp(struct net_device * netdev,struct ifreq * ifr)2445 int otx2_config_hwtstamp(struct net_device *netdev, struct ifreq *ifr)
2446 {
2447 struct otx2_nic *pfvf = netdev_priv(netdev);
2448 struct hwtstamp_config config;
2449
2450 if (!pfvf->ptp)
2451 return -ENODEV;
2452
2453 if (copy_from_user(&config, ifr->ifr_data, sizeof(config)))
2454 return -EFAULT;
2455
2456 switch (config.tx_type) {
2457 case HWTSTAMP_TX_OFF:
2458 if (pfvf->flags & OTX2_FLAG_PTP_ONESTEP_SYNC)
2459 pfvf->flags &= ~OTX2_FLAG_PTP_ONESTEP_SYNC;
2460
2461 cancel_delayed_work(&pfvf->ptp->synctstamp_work);
2462 otx2_config_hw_tx_tstamp(pfvf, false);
2463 break;
2464 case HWTSTAMP_TX_ONESTEP_SYNC:
2465 if (!test_bit(CN10K_PTP_ONESTEP, &pfvf->hw.cap_flag))
2466 return -ERANGE;
2467 pfvf->flags |= OTX2_FLAG_PTP_ONESTEP_SYNC;
2468 schedule_delayed_work(&pfvf->ptp->synctstamp_work,
2469 msecs_to_jiffies(500));
2470 fallthrough;
2471 case HWTSTAMP_TX_ON:
2472 otx2_config_hw_tx_tstamp(pfvf, true);
2473 break;
2474 default:
2475 return -ERANGE;
2476 }
2477
2478 switch (config.rx_filter) {
2479 case HWTSTAMP_FILTER_NONE:
2480 otx2_config_hw_rx_tstamp(pfvf, false);
2481 break;
2482 case HWTSTAMP_FILTER_ALL:
2483 case HWTSTAMP_FILTER_SOME:
2484 case HWTSTAMP_FILTER_PTP_V1_L4_EVENT:
2485 case HWTSTAMP_FILTER_PTP_V1_L4_SYNC:
2486 case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ:
2487 case HWTSTAMP_FILTER_PTP_V2_L4_EVENT:
2488 case HWTSTAMP_FILTER_PTP_V2_L4_SYNC:
2489 case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ:
2490 case HWTSTAMP_FILTER_PTP_V2_L2_EVENT:
2491 case HWTSTAMP_FILTER_PTP_V2_L2_SYNC:
2492 case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ:
2493 case HWTSTAMP_FILTER_PTP_V2_EVENT:
2494 case HWTSTAMP_FILTER_PTP_V2_SYNC:
2495 case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ:
2496 otx2_config_hw_rx_tstamp(pfvf, true);
2497 config.rx_filter = HWTSTAMP_FILTER_ALL;
2498 break;
2499 default:
2500 return -ERANGE;
2501 }
2502
2503 memcpy(&pfvf->tstamp, &config, sizeof(config));
2504
2505 return copy_to_user(ifr->ifr_data, &config,
2506 sizeof(config)) ? -EFAULT : 0;
2507 }
2508 EXPORT_SYMBOL(otx2_config_hwtstamp);
2509
otx2_ioctl(struct net_device * netdev,struct ifreq * req,int cmd)2510 int otx2_ioctl(struct net_device *netdev, struct ifreq *req, int cmd)
2511 {
2512 struct otx2_nic *pfvf = netdev_priv(netdev);
2513 struct hwtstamp_config *cfg = &pfvf->tstamp;
2514
2515 switch (cmd) {
2516 case SIOCSHWTSTAMP:
2517 return otx2_config_hwtstamp(netdev, req);
2518 case SIOCGHWTSTAMP:
2519 return copy_to_user(req->ifr_data, cfg,
2520 sizeof(*cfg)) ? -EFAULT : 0;
2521 default:
2522 return -EOPNOTSUPP;
2523 }
2524 }
2525 EXPORT_SYMBOL(otx2_ioctl);
2526
otx2_do_set_vf_mac(struct otx2_nic * pf,int vf,const u8 * mac)2527 static int otx2_do_set_vf_mac(struct otx2_nic *pf, int vf, const u8 *mac)
2528 {
2529 struct npc_install_flow_req *req;
2530 int err;
2531
2532 mutex_lock(&pf->mbox.lock);
2533 req = otx2_mbox_alloc_msg_npc_install_flow(&pf->mbox);
2534 if (!req) {
2535 err = -ENOMEM;
2536 goto out;
2537 }
2538
2539 ether_addr_copy(req->packet.dmac, mac);
2540 eth_broadcast_addr((u8 *)&req->mask.dmac);
2541 req->features = BIT_ULL(NPC_DMAC);
2542 req->channel = pf->hw.rx_chan_base;
2543 req->intf = NIX_INTF_RX;
2544 req->default_rule = 1;
2545 req->append = 1;
2546 req->vf = vf + 1;
2547 req->op = NIX_RX_ACTION_DEFAULT;
2548
2549 err = otx2_sync_mbox_msg(&pf->mbox);
2550 out:
2551 mutex_unlock(&pf->mbox.lock);
2552 return err;
2553 }
2554
otx2_set_vf_mac(struct net_device * netdev,int vf,u8 * mac)2555 static int otx2_set_vf_mac(struct net_device *netdev, int vf, u8 *mac)
2556 {
2557 struct otx2_nic *pf = netdev_priv(netdev);
2558 struct pci_dev *pdev = pf->pdev;
2559 struct otx2_vf_config *config;
2560 int ret;
2561
2562 if (!netif_running(netdev))
2563 return -EAGAIN;
2564
2565 if (vf >= pf->total_vfs)
2566 return -EINVAL;
2567
2568 if (!is_valid_ether_addr(mac))
2569 return -EINVAL;
2570
2571 config = &pf->vf_configs[vf];
2572 ether_addr_copy(config->mac, mac);
2573
2574 ret = otx2_do_set_vf_mac(pf, vf, mac);
2575 if (ret == 0)
2576 dev_info(&pdev->dev,
2577 "Load/Reload VF driver\n");
2578
2579 return ret;
2580 }
2581
otx2_do_set_vf_vlan(struct otx2_nic * pf,int vf,u16 vlan,u8 qos,__be16 proto)2582 static int otx2_do_set_vf_vlan(struct otx2_nic *pf, int vf, u16 vlan, u8 qos,
2583 __be16 proto)
2584 {
2585 struct otx2_flow_config *flow_cfg = pf->flow_cfg;
2586 struct nix_vtag_config_rsp *vtag_rsp;
2587 struct npc_delete_flow_req *del_req;
2588 struct nix_vtag_config *vtag_req;
2589 struct npc_install_flow_req *req;
2590 struct otx2_vf_config *config;
2591 int err = 0;
2592 u32 idx;
2593
2594 config = &pf->vf_configs[vf];
2595
2596 if (!vlan && !config->vlan)
2597 goto out;
2598
2599 mutex_lock(&pf->mbox.lock);
2600
2601 /* free old tx vtag entry */
2602 if (config->vlan) {
2603 vtag_req = otx2_mbox_alloc_msg_nix_vtag_cfg(&pf->mbox);
2604 if (!vtag_req) {
2605 err = -ENOMEM;
2606 goto out;
2607 }
2608 vtag_req->cfg_type = 0;
2609 vtag_req->tx.free_vtag0 = 1;
2610 vtag_req->tx.vtag0_idx = config->tx_vtag_idx;
2611
2612 err = otx2_sync_mbox_msg(&pf->mbox);
2613 if (err)
2614 goto out;
2615 }
2616
2617 if (!vlan && config->vlan) {
2618 /* rx */
2619 del_req = otx2_mbox_alloc_msg_npc_delete_flow(&pf->mbox);
2620 if (!del_req) {
2621 err = -ENOMEM;
2622 goto out;
2623 }
2624 idx = ((vf * OTX2_PER_VF_VLAN_FLOWS) + OTX2_VF_VLAN_RX_INDEX);
2625 del_req->entry =
2626 flow_cfg->def_ent[flow_cfg->vf_vlan_offset + idx];
2627 err = otx2_sync_mbox_msg(&pf->mbox);
2628 if (err)
2629 goto out;
2630
2631 /* tx */
2632 del_req = otx2_mbox_alloc_msg_npc_delete_flow(&pf->mbox);
2633 if (!del_req) {
2634 err = -ENOMEM;
2635 goto out;
2636 }
2637 idx = ((vf * OTX2_PER_VF_VLAN_FLOWS) + OTX2_VF_VLAN_TX_INDEX);
2638 del_req->entry =
2639 flow_cfg->def_ent[flow_cfg->vf_vlan_offset + idx];
2640 err = otx2_sync_mbox_msg(&pf->mbox);
2641
2642 goto out;
2643 }
2644
2645 /* rx */
2646 req = otx2_mbox_alloc_msg_npc_install_flow(&pf->mbox);
2647 if (!req) {
2648 err = -ENOMEM;
2649 goto out;
2650 }
2651
2652 idx = ((vf * OTX2_PER_VF_VLAN_FLOWS) + OTX2_VF_VLAN_RX_INDEX);
2653 req->entry = flow_cfg->def_ent[flow_cfg->vf_vlan_offset + idx];
2654 req->packet.vlan_tci = htons(vlan);
2655 req->mask.vlan_tci = htons(VLAN_VID_MASK);
2656 /* af fills the destination mac addr */
2657 eth_broadcast_addr((u8 *)&req->mask.dmac);
2658 req->features = BIT_ULL(NPC_OUTER_VID) | BIT_ULL(NPC_DMAC);
2659 req->channel = pf->hw.rx_chan_base;
2660 req->intf = NIX_INTF_RX;
2661 req->vf = vf + 1;
2662 req->op = NIX_RX_ACTION_DEFAULT;
2663 req->vtag0_valid = true;
2664 req->vtag0_type = NIX_AF_LFX_RX_VTAG_TYPE7;
2665 req->set_cntr = 1;
2666
2667 err = otx2_sync_mbox_msg(&pf->mbox);
2668 if (err)
2669 goto out;
2670
2671 /* tx */
2672 vtag_req = otx2_mbox_alloc_msg_nix_vtag_cfg(&pf->mbox);
2673 if (!vtag_req) {
2674 err = -ENOMEM;
2675 goto out;
2676 }
2677
2678 /* configure tx vtag params */
2679 vtag_req->vtag_size = VTAGSIZE_T4;
2680 vtag_req->cfg_type = 0; /* tx vlan cfg */
2681 vtag_req->tx.cfg_vtag0 = 1;
2682 vtag_req->tx.vtag0 = ((u64)ntohs(proto) << 16) | vlan;
2683
2684 err = otx2_sync_mbox_msg(&pf->mbox);
2685 if (err)
2686 goto out;
2687
2688 vtag_rsp = (struct nix_vtag_config_rsp *)otx2_mbox_get_rsp
2689 (&pf->mbox.mbox, 0, &vtag_req->hdr);
2690 if (IS_ERR(vtag_rsp)) {
2691 err = PTR_ERR(vtag_rsp);
2692 goto out;
2693 }
2694 config->tx_vtag_idx = vtag_rsp->vtag0_idx;
2695
2696 req = otx2_mbox_alloc_msg_npc_install_flow(&pf->mbox);
2697 if (!req) {
2698 err = -ENOMEM;
2699 goto out;
2700 }
2701
2702 eth_zero_addr((u8 *)&req->mask.dmac);
2703 idx = ((vf * OTX2_PER_VF_VLAN_FLOWS) + OTX2_VF_VLAN_TX_INDEX);
2704 req->entry = flow_cfg->def_ent[flow_cfg->vf_vlan_offset + idx];
2705 req->features = BIT_ULL(NPC_DMAC);
2706 req->channel = pf->hw.tx_chan_base;
2707 req->intf = NIX_INTF_TX;
2708 req->vf = vf + 1;
2709 req->op = NIX_TX_ACTIONOP_UCAST_DEFAULT;
2710 req->vtag0_def = vtag_rsp->vtag0_idx;
2711 req->vtag0_op = VTAG_INSERT;
2712 req->set_cntr = 1;
2713
2714 err = otx2_sync_mbox_msg(&pf->mbox);
2715 out:
2716 config->vlan = vlan;
2717 mutex_unlock(&pf->mbox.lock);
2718 return err;
2719 }
2720
otx2_set_vf_vlan(struct net_device * netdev,int vf,u16 vlan,u8 qos,__be16 proto)2721 static int otx2_set_vf_vlan(struct net_device *netdev, int vf, u16 vlan, u8 qos,
2722 __be16 proto)
2723 {
2724 struct otx2_nic *pf = netdev_priv(netdev);
2725 struct pci_dev *pdev = pf->pdev;
2726
2727 if (!netif_running(netdev))
2728 return -EAGAIN;
2729
2730 if (vf >= pci_num_vf(pdev))
2731 return -EINVAL;
2732
2733 /* qos is currently unsupported */
2734 if (vlan >= VLAN_N_VID || qos)
2735 return -EINVAL;
2736
2737 if (proto != htons(ETH_P_8021Q))
2738 return -EPROTONOSUPPORT;
2739
2740 if (!(pf->flags & OTX2_FLAG_VF_VLAN_SUPPORT))
2741 return -EOPNOTSUPP;
2742
2743 return otx2_do_set_vf_vlan(pf, vf, vlan, qos, proto);
2744 }
2745
otx2_get_vf_config(struct net_device * netdev,int vf,struct ifla_vf_info * ivi)2746 static int otx2_get_vf_config(struct net_device *netdev, int vf,
2747 struct ifla_vf_info *ivi)
2748 {
2749 struct otx2_nic *pf = netdev_priv(netdev);
2750 struct pci_dev *pdev = pf->pdev;
2751 struct otx2_vf_config *config;
2752
2753 if (!netif_running(netdev))
2754 return -EAGAIN;
2755
2756 if (vf >= pci_num_vf(pdev))
2757 return -EINVAL;
2758
2759 config = &pf->vf_configs[vf];
2760 ivi->vf = vf;
2761 ether_addr_copy(ivi->mac, config->mac);
2762 ivi->vlan = config->vlan;
2763 ivi->trusted = config->trusted;
2764
2765 return 0;
2766 }
2767
otx2_xdp_xmit_tx(struct otx2_nic * pf,struct xdp_frame * xdpf,int qidx)2768 static int otx2_xdp_xmit_tx(struct otx2_nic *pf, struct xdp_frame *xdpf,
2769 int qidx)
2770 {
2771 u64 dma_addr;
2772 int err = 0;
2773
2774 dma_addr = otx2_dma_map_page(pf, virt_to_page(xdpf->data),
2775 offset_in_page(xdpf->data), xdpf->len,
2776 DMA_TO_DEVICE);
2777 if (dma_mapping_error(pf->dev, dma_addr))
2778 return -ENOMEM;
2779
2780 err = otx2_xdp_sq_append_pkt(pf, xdpf, dma_addr, xdpf->len,
2781 qidx, OTX2_XDP_REDIRECT);
2782 if (!err) {
2783 otx2_dma_unmap_page(pf, dma_addr, xdpf->len, DMA_TO_DEVICE);
2784 xdp_return_frame(xdpf);
2785 return -ENOMEM;
2786 }
2787 return 0;
2788 }
2789
otx2_xdp_xmit(struct net_device * netdev,int n,struct xdp_frame ** frames,u32 flags)2790 static int otx2_xdp_xmit(struct net_device *netdev, int n,
2791 struct xdp_frame **frames, u32 flags)
2792 {
2793 struct otx2_nic *pf = netdev_priv(netdev);
2794 int qidx = smp_processor_id();
2795 struct otx2_snd_queue *sq;
2796 int drops = 0, i;
2797
2798 if (!netif_running(netdev))
2799 return -ENETDOWN;
2800
2801 qidx += pf->hw.tx_queues;
2802 sq = pf->xdp_prog ? &pf->qset.sq[qidx] : NULL;
2803
2804 /* Abort xmit if xdp queue is not */
2805 if (unlikely(!sq))
2806 return -ENXIO;
2807
2808 if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK))
2809 return -EINVAL;
2810
2811 for (i = 0; i < n; i++) {
2812 struct xdp_frame *xdpf = frames[i];
2813 int err;
2814
2815 err = otx2_xdp_xmit_tx(pf, xdpf, qidx);
2816 if (err)
2817 drops++;
2818 }
2819 return n - drops;
2820 }
2821
otx2_xdp_setup(struct otx2_nic * pf,struct bpf_prog * prog)2822 static int otx2_xdp_setup(struct otx2_nic *pf, struct bpf_prog *prog)
2823 {
2824 struct net_device *dev = pf->netdev;
2825 bool if_up = netif_running(pf->netdev);
2826 struct bpf_prog *old_prog;
2827
2828 if (prog && dev->mtu > MAX_XDP_MTU) {
2829 netdev_warn(dev, "Jumbo frames not yet supported with XDP\n");
2830 return -EOPNOTSUPP;
2831 }
2832
2833 if (if_up)
2834 otx2_stop(pf->netdev);
2835
2836 old_prog = xchg(&pf->xdp_prog, prog);
2837
2838 if (old_prog)
2839 bpf_prog_put(old_prog);
2840
2841 if (pf->xdp_prog)
2842 bpf_prog_add(pf->xdp_prog, pf->hw.rx_queues - 1);
2843
2844 /* Network stack and XDP shared same rx queues.
2845 * Use separate tx queues for XDP and network stack.
2846 */
2847 if (pf->xdp_prog) {
2848 pf->hw.xdp_queues = pf->hw.rx_queues;
2849 xdp_features_set_redirect_target(dev, false);
2850 } else {
2851 pf->hw.xdp_queues = 0;
2852 xdp_features_clear_redirect_target(dev);
2853 }
2854
2855 if (if_up)
2856 otx2_open(pf->netdev);
2857
2858 return 0;
2859 }
2860
otx2_xdp(struct net_device * netdev,struct netdev_bpf * xdp)2861 static int otx2_xdp(struct net_device *netdev, struct netdev_bpf *xdp)
2862 {
2863 struct otx2_nic *pf = netdev_priv(netdev);
2864
2865 switch (xdp->command) {
2866 case XDP_SETUP_PROG:
2867 return otx2_xdp_setup(pf, xdp->prog);
2868 case XDP_SETUP_XSK_POOL:
2869 return otx2_xsk_pool_setup(pf, xdp->xsk.pool, xdp->xsk.queue_id);
2870 default:
2871 return -EINVAL;
2872 }
2873 }
2874
otx2_set_vf_permissions(struct otx2_nic * pf,int vf,int req_perm)2875 static int otx2_set_vf_permissions(struct otx2_nic *pf, int vf,
2876 int req_perm)
2877 {
2878 struct set_vf_perm *req;
2879 int rc;
2880
2881 mutex_lock(&pf->mbox.lock);
2882 req = otx2_mbox_alloc_msg_set_vf_perm(&pf->mbox);
2883 if (!req) {
2884 rc = -ENOMEM;
2885 goto out;
2886 }
2887
2888 /* Let AF reset VF permissions as sriov is disabled */
2889 if (req_perm == OTX2_RESET_VF_PERM) {
2890 req->flags |= RESET_VF_PERM;
2891 } else if (req_perm == OTX2_TRUSTED_VF) {
2892 if (pf->vf_configs[vf].trusted)
2893 req->flags |= VF_TRUSTED;
2894 }
2895
2896 req->vf = vf;
2897 rc = otx2_sync_mbox_msg(&pf->mbox);
2898 out:
2899 mutex_unlock(&pf->mbox.lock);
2900 return rc;
2901 }
2902
otx2_ndo_set_vf_trust(struct net_device * netdev,int vf,bool enable)2903 static int otx2_ndo_set_vf_trust(struct net_device *netdev, int vf,
2904 bool enable)
2905 {
2906 struct otx2_nic *pf = netdev_priv(netdev);
2907 struct pci_dev *pdev = pf->pdev;
2908 int rc;
2909
2910 if (vf >= pci_num_vf(pdev))
2911 return -EINVAL;
2912
2913 if (pf->vf_configs[vf].trusted == enable)
2914 return 0;
2915
2916 pf->vf_configs[vf].trusted = enable;
2917 rc = otx2_set_vf_permissions(pf, vf, OTX2_TRUSTED_VF);
2918
2919 if (rc) {
2920 pf->vf_configs[vf].trusted = !enable;
2921 } else {
2922 netdev_info(pf->netdev, "VF %d is %strusted\n",
2923 vf, enable ? "" : "not ");
2924 otx2_set_rx_mode(netdev);
2925 }
2926
2927 return rc;
2928 }
2929
2930 static const struct net_device_ops otx2_netdev_ops = {
2931 .ndo_open = otx2_open,
2932 .ndo_stop = otx2_stop,
2933 .ndo_start_xmit = otx2_xmit,
2934 .ndo_select_queue = otx2_select_queue,
2935 .ndo_fix_features = otx2_fix_features,
2936 .ndo_set_mac_address = otx2_set_mac_address,
2937 .ndo_change_mtu = otx2_change_mtu,
2938 .ndo_set_rx_mode = otx2_set_rx_mode,
2939 .ndo_set_features = otx2_set_features,
2940 .ndo_tx_timeout = otx2_tx_timeout,
2941 .ndo_get_stats64 = otx2_get_stats64,
2942 .ndo_eth_ioctl = otx2_ioctl,
2943 .ndo_set_vf_mac = otx2_set_vf_mac,
2944 .ndo_set_vf_vlan = otx2_set_vf_vlan,
2945 .ndo_get_vf_config = otx2_get_vf_config,
2946 .ndo_bpf = otx2_xdp,
2947 .ndo_xsk_wakeup = otx2_xsk_wakeup,
2948 .ndo_xdp_xmit = otx2_xdp_xmit,
2949 .ndo_setup_tc = otx2_setup_tc,
2950 .ndo_set_vf_trust = otx2_ndo_set_vf_trust,
2951 };
2952
otx2_wq_init(struct otx2_nic * pf)2953 int otx2_wq_init(struct otx2_nic *pf)
2954 {
2955 pf->otx2_wq = create_singlethread_workqueue("otx2_wq");
2956 if (!pf->otx2_wq)
2957 return -ENOMEM;
2958
2959 INIT_WORK(&pf->rx_mode_work, otx2_rx_mode_wrk_handler);
2960 INIT_WORK(&pf->reset_task, otx2_reset_task);
2961 return 0;
2962 }
2963
otx2_check_pf_usable(struct otx2_nic * nic)2964 int otx2_check_pf_usable(struct otx2_nic *nic)
2965 {
2966 u64 rev;
2967
2968 rev = otx2_read64(nic, RVU_PF_BLOCK_ADDRX_DISC(BLKADDR_RVUM));
2969 rev = (rev >> 12) & 0xFF;
2970 /* Check if AF has setup revision for RVUM block,
2971 * otherwise this driver probe should be deferred
2972 * until AF driver comes up.
2973 */
2974 if (!rev) {
2975 dev_warn(nic->dev,
2976 "AF is not initialized, deferring probe\n");
2977 return -EPROBE_DEFER;
2978 }
2979 return 0;
2980 }
2981
otx2_realloc_msix_vectors(struct otx2_nic * pf)2982 int otx2_realloc_msix_vectors(struct otx2_nic *pf)
2983 {
2984 struct otx2_hw *hw = &pf->hw;
2985 int num_vec, err;
2986
2987 /* NPA interrupts are inot registered, so alloc only
2988 * upto NIX vector offset.
2989 */
2990 num_vec = hw->nix_msixoff;
2991 num_vec += NIX_LF_CINT_VEC_START + hw->max_queues;
2992
2993 otx2_disable_mbox_intr(pf);
2994 pci_free_irq_vectors(hw->pdev);
2995 err = pci_alloc_irq_vectors(hw->pdev, num_vec, num_vec, PCI_IRQ_MSIX);
2996 if (err < 0) {
2997 dev_err(pf->dev, "%s: Failed to realloc %d IRQ vectors\n",
2998 __func__, num_vec);
2999 return err;
3000 }
3001
3002 return otx2_register_mbox_intr(pf, false);
3003 }
3004 EXPORT_SYMBOL(otx2_realloc_msix_vectors);
3005
otx2_sriov_vfcfg_init(struct otx2_nic * pf)3006 static int otx2_sriov_vfcfg_init(struct otx2_nic *pf)
3007 {
3008 int i;
3009
3010 pf->vf_configs = devm_kcalloc(pf->dev, pf->total_vfs,
3011 sizeof(struct otx2_vf_config),
3012 GFP_KERNEL);
3013 if (!pf->vf_configs)
3014 return -ENOMEM;
3015
3016 for (i = 0; i < pf->total_vfs; i++) {
3017 pf->vf_configs[i].pf = pf;
3018 pf->vf_configs[i].intf_down = true;
3019 pf->vf_configs[i].trusted = false;
3020 INIT_DELAYED_WORK(&pf->vf_configs[i].link_event_work,
3021 otx2_vf_link_event_task);
3022 }
3023
3024 return 0;
3025 }
3026
otx2_sriov_vfcfg_cleanup(struct otx2_nic * pf)3027 static void otx2_sriov_vfcfg_cleanup(struct otx2_nic *pf)
3028 {
3029 int i;
3030
3031 if (!pf->vf_configs)
3032 return;
3033
3034 for (i = 0; i < pf->total_vfs; i++) {
3035 cancel_delayed_work_sync(&pf->vf_configs[i].link_event_work);
3036 otx2_set_vf_permissions(pf, i, OTX2_RESET_VF_PERM);
3037 }
3038 }
3039
otx2_init_rsrc(struct pci_dev * pdev,struct otx2_nic * pf)3040 int otx2_init_rsrc(struct pci_dev *pdev, struct otx2_nic *pf)
3041 {
3042 struct device *dev = &pdev->dev;
3043 struct otx2_hw *hw = &pf->hw;
3044 int num_vec, err;
3045
3046 num_vec = pci_msix_vec_count(pdev);
3047 hw->irq_name = devm_kmalloc_array(&hw->pdev->dev, num_vec, NAME_SIZE,
3048 GFP_KERNEL);
3049 if (!hw->irq_name)
3050 return -ENOMEM;
3051
3052 hw->affinity_mask = devm_kcalloc(&hw->pdev->dev, num_vec,
3053 sizeof(cpumask_var_t), GFP_KERNEL);
3054 if (!hw->affinity_mask)
3055 return -ENOMEM;
3056
3057 /* Map CSRs */
3058 pf->reg_base = pcim_iomap(pdev, PCI_CFG_REG_BAR_NUM, 0);
3059 if (!pf->reg_base) {
3060 dev_err(dev, "Unable to map physical function CSRs, aborting\n");
3061 return -ENOMEM;
3062 }
3063
3064 err = otx2_check_pf_usable(pf);
3065 if (err)
3066 return err;
3067
3068 if (!is_cn20k(pf->pdev))
3069 err = pci_alloc_irq_vectors(hw->pdev, RVU_PF_INT_VEC_CNT,
3070 RVU_PF_INT_VEC_CNT, PCI_IRQ_MSIX);
3071 else
3072 err = pci_alloc_irq_vectors(hw->pdev, RVU_MBOX_PF_INT_VEC_CNT,
3073 RVU_MBOX_PF_INT_VEC_CNT,
3074 PCI_IRQ_MSIX);
3075 if (err < 0) {
3076 dev_err(dev, "%s: Failed to alloc %d IRQ vectors\n",
3077 __func__, num_vec);
3078 return err;
3079 }
3080
3081 otx2_setup_dev_hw_settings(pf);
3082
3083 if (is_cn20k(pf->pdev))
3084 cn20k_init(pf);
3085 else
3086 otx2_init_hw_ops(pf);
3087
3088 /* Init PF <=> AF mailbox stuff */
3089 err = otx2_pfaf_mbox_init(pf);
3090 if (err)
3091 goto err_free_irq_vectors;
3092
3093 /* Register mailbox interrupt */
3094 err = otx2_register_mbox_intr(pf, true);
3095 if (err)
3096 goto err_mbox_destroy;
3097
3098 /* Request AF to attach NPA and NIX LFs to this PF.
3099 * NIX and NPA LFs are needed for this PF to function as a NIC.
3100 */
3101 err = otx2_attach_npa_nix(pf);
3102 if (err)
3103 goto err_disable_mbox_intr;
3104
3105 err = otx2_realloc_msix_vectors(pf);
3106 if (err)
3107 goto err_detach_rsrc;
3108
3109 err = cn10k_lmtst_init(pf);
3110 if (err)
3111 goto err_detach_rsrc;
3112
3113 return 0;
3114
3115 err_detach_rsrc:
3116 if (pf->hw.lmt_info)
3117 free_percpu(pf->hw.lmt_info);
3118 if (test_bit(CN10K_LMTST, &pf->hw.cap_flag))
3119 qmem_free(pf->dev, pf->dync_lmt);
3120 otx2_detach_resources(&pf->mbox);
3121 err_disable_mbox_intr:
3122 otx2_disable_mbox_intr(pf);
3123 err_mbox_destroy:
3124 otx2_pfaf_mbox_destroy(pf);
3125 err_free_irq_vectors:
3126 pci_free_irq_vectors(hw->pdev);
3127
3128 return err;
3129 }
3130 EXPORT_SYMBOL(otx2_init_rsrc);
3131
otx2_probe(struct pci_dev * pdev,const struct pci_device_id * id)3132 static int otx2_probe(struct pci_dev *pdev, const struct pci_device_id *id)
3133 {
3134 struct device *dev = &pdev->dev;
3135 int err, qcount, qos_txqs;
3136 struct net_device *netdev;
3137 struct otx2_nic *pf;
3138 struct otx2_hw *hw;
3139
3140 err = pcim_enable_device(pdev);
3141 if (err) {
3142 dev_err(dev, "Failed to enable PCI device\n");
3143 return err;
3144 }
3145
3146 err = pcim_request_all_regions(pdev, DRV_NAME);
3147 if (err) {
3148 dev_err(dev, "PCI request regions failed 0x%x\n", err);
3149 return err;
3150 }
3151
3152 err = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(48));
3153 if (err) {
3154 dev_err(dev, "DMA mask config failed, abort\n");
3155 return err;
3156 }
3157
3158 pci_set_master(pdev);
3159
3160 /* Set number of queues */
3161 qcount = min_t(int, num_online_cpus(), OTX2_MAX_CQ_CNT);
3162 qos_txqs = min_t(int, qcount, OTX2_QOS_MAX_LEAF_NODES);
3163
3164 netdev = alloc_etherdev_mqs(sizeof(*pf), qcount + qos_txqs, qcount);
3165 if (!netdev)
3166 return -ENOMEM;
3167
3168 pci_set_drvdata(pdev, netdev);
3169 SET_NETDEV_DEV(netdev, &pdev->dev);
3170 pf = netdev_priv(netdev);
3171 pf->netdev = netdev;
3172 pf->pdev = pdev;
3173 pf->dev = dev;
3174 pf->total_vfs = pci_sriov_get_totalvfs(pdev);
3175 pf->flags |= OTX2_FLAG_INTF_DOWN;
3176
3177 hw = &pf->hw;
3178 hw->pdev = pdev;
3179 hw->rx_queues = qcount;
3180 hw->tx_queues = qcount;
3181 hw->non_qos_queues = qcount;
3182 hw->max_queues = qcount;
3183 hw->rbuf_len = OTX2_DEFAULT_RBUF_LEN;
3184 /* Use CQE of 128 byte descriptor size by default */
3185 hw->xqe_size = 128;
3186
3187 err = otx2_init_rsrc(pdev, pf);
3188 if (err)
3189 goto err_free_netdev;
3190
3191 err = otx2_set_real_num_queues(netdev, hw->tx_queues, hw->rx_queues);
3192 if (err)
3193 goto err_detach_rsrc;
3194
3195 /* Assign default mac address */
3196 otx2_get_mac_from_af(netdev);
3197
3198 /* Don't check for error. Proceed without ptp */
3199 otx2_ptp_init(pf);
3200
3201 /* NPA's pool is a stack to which SW frees buffer pointers via Aura.
3202 * HW allocates buffer pointer from stack and uses it for DMA'ing
3203 * ingress packet. In some scenarios HW can free back allocated buffer
3204 * pointers to pool. This makes it impossible for SW to maintain a
3205 * parallel list where physical addresses of buffer pointers (IOVAs)
3206 * given to HW can be saved for later reference.
3207 *
3208 * So the only way to convert Rx packet's buffer address is to use
3209 * IOMMU's iova_to_phys() handler which translates the address by
3210 * walking through the translation tables.
3211 */
3212 pf->iommu_domain = iommu_get_domain_for_dev(dev);
3213
3214 netdev->hw_features = (NETIF_F_RXCSUM | NETIF_F_IP_CSUM |
3215 NETIF_F_IPV6_CSUM | NETIF_F_RXHASH |
3216 NETIF_F_SG | NETIF_F_TSO | NETIF_F_TSO6 |
3217 NETIF_F_GSO_UDP_L4);
3218 netdev->features |= netdev->hw_features;
3219
3220 err = otx2_mcam_flow_init(pf);
3221 if (err)
3222 goto err_ptp_destroy;
3223
3224 otx2_set_hw_capabilities(pf);
3225
3226 err = cn10k_mcs_init(pf);
3227 if (err)
3228 goto err_del_mcam_entries;
3229
3230 if (pf->flags & OTX2_FLAG_NTUPLE_SUPPORT)
3231 netdev->hw_features |= NETIF_F_NTUPLE;
3232
3233 if (pf->flags & OTX2_FLAG_UCAST_FLTR_SUPPORT)
3234 netdev->priv_flags |= IFF_UNICAST_FLT;
3235
3236 /* Support TSO on tag interface */
3237 netdev->vlan_features |= netdev->features;
3238 netdev->hw_features |= NETIF_F_HW_VLAN_CTAG_TX |
3239 NETIF_F_HW_VLAN_STAG_TX;
3240 if (pf->flags & OTX2_FLAG_RX_VLAN_SUPPORT)
3241 netdev->hw_features |= NETIF_F_HW_VLAN_CTAG_RX |
3242 NETIF_F_HW_VLAN_STAG_RX;
3243 netdev->features |= netdev->hw_features;
3244
3245 /* HW supports tc offload but mutually exclusive with n-tuple filters */
3246 if (pf->flags & OTX2_FLAG_TC_FLOWER_SUPPORT)
3247 netdev->hw_features |= NETIF_F_HW_TC;
3248
3249 netdev->hw_features |= NETIF_F_LOOPBACK | NETIF_F_RXALL;
3250
3251 netif_set_tso_max_segs(netdev, OTX2_MAX_GSO_SEGS);
3252 netdev->watchdog_timeo = OTX2_TX_TIMEOUT;
3253
3254 netdev->netdev_ops = &otx2_netdev_ops;
3255 netdev->xdp_features = NETDEV_XDP_ACT_BASIC | NETDEV_XDP_ACT_REDIRECT;
3256
3257 netdev->min_mtu = OTX2_MIN_MTU;
3258 netdev->max_mtu = otx2_get_max_mtu(pf);
3259 hw->max_mtu = netdev->max_mtu;
3260
3261 /* reset CGX/RPM MAC stats */
3262 otx2_reset_mac_stats(pf);
3263
3264 err = cn10k_ipsec_init(netdev);
3265 if (err)
3266 goto err_mcs_free;
3267
3268 err = register_netdev(netdev);
3269 if (err) {
3270 dev_err(dev, "Failed to register netdevice\n");
3271 goto err_ipsec_clean;
3272 }
3273
3274 err = otx2_wq_init(pf);
3275 if (err)
3276 goto err_unreg_netdev;
3277
3278 otx2_set_ethtool_ops(netdev);
3279
3280 err = otx2_init_tc(pf);
3281 if (err)
3282 goto err_mcam_flow_del;
3283
3284 err = otx2_register_dl(pf);
3285 if (err)
3286 goto err_mcam_flow_del;
3287
3288 /* Initialize SR-IOV resources */
3289 err = otx2_sriov_vfcfg_init(pf);
3290 if (err)
3291 goto err_pf_sriov_init;
3292
3293 /* Enable link notifications */
3294 otx2_cgx_config_linkevents(pf, true);
3295
3296 pf->af_xdp_zc_qidx = bitmap_zalloc(qcount, GFP_KERNEL);
3297 if (!pf->af_xdp_zc_qidx) {
3298 err = -ENOMEM;
3299 goto err_sriov_cleannup;
3300 }
3301
3302 #ifdef CONFIG_DCB
3303 err = otx2_dcbnl_set_ops(netdev);
3304 if (err)
3305 goto err_free_zc_bmap;
3306 #endif
3307
3308 otx2_qos_init(pf, qos_txqs);
3309
3310 return 0;
3311
3312 #ifdef CONFIG_DCB
3313 err_free_zc_bmap:
3314 bitmap_free(pf->af_xdp_zc_qidx);
3315 #endif
3316 err_sriov_cleannup:
3317 otx2_sriov_vfcfg_cleanup(pf);
3318 err_pf_sriov_init:
3319 otx2_shutdown_tc(pf);
3320 err_mcam_flow_del:
3321 otx2_mcam_flow_del(pf);
3322 err_unreg_netdev:
3323 unregister_netdev(netdev);
3324 err_ipsec_clean:
3325 cn10k_ipsec_clean(pf);
3326 err_mcs_free:
3327 cn10k_mcs_free(pf);
3328 err_del_mcam_entries:
3329 otx2_mcam_flow_del(pf);
3330 err_ptp_destroy:
3331 otx2_ptp_destroy(pf);
3332 err_detach_rsrc:
3333 if (pf->hw.lmt_info)
3334 free_percpu(pf->hw.lmt_info);
3335 if (test_bit(CN10K_LMTST, &pf->hw.cap_flag))
3336 qmem_free(pf->dev, pf->dync_lmt);
3337 otx2_detach_resources(&pf->mbox);
3338 otx2_disable_mbox_intr(pf);
3339 otx2_pfaf_mbox_destroy(pf);
3340 pci_free_irq_vectors(hw->pdev);
3341 err_free_netdev:
3342 pci_set_drvdata(pdev, NULL);
3343 free_netdev(netdev);
3344 return err;
3345 }
3346
otx2_vf_link_event_task(struct work_struct * work)3347 static void otx2_vf_link_event_task(struct work_struct *work)
3348 {
3349 struct otx2_vf_config *config;
3350 struct cgx_link_info_msg *req;
3351 struct mbox_msghdr *msghdr;
3352 struct delayed_work *dwork;
3353 struct otx2_nic *pf;
3354 int vf_idx;
3355
3356 config = container_of(work, struct otx2_vf_config,
3357 link_event_work.work);
3358 vf_idx = config - config->pf->vf_configs;
3359 pf = config->pf;
3360
3361 if (config->intf_down)
3362 return;
3363
3364 mutex_lock(&pf->mbox.lock);
3365
3366 dwork = &config->link_event_work;
3367
3368 if (!otx2_mbox_wait_for_zero(&pf->mbox_pfvf[0].mbox_up, vf_idx)) {
3369 schedule_delayed_work(dwork, msecs_to_jiffies(100));
3370 mutex_unlock(&pf->mbox.lock);
3371 return;
3372 }
3373
3374 msghdr = otx2_mbox_alloc_msg_rsp(&pf->mbox_pfvf[0].mbox_up, vf_idx,
3375 sizeof(*req), sizeof(struct msg_rsp));
3376 if (!msghdr) {
3377 dev_err(pf->dev, "Failed to create VF%d link event\n", vf_idx);
3378 mutex_unlock(&pf->mbox.lock);
3379 return;
3380 }
3381
3382 req = (struct cgx_link_info_msg *)msghdr;
3383 req->hdr.id = MBOX_MSG_CGX_LINK_EVENT;
3384 req->hdr.sig = OTX2_MBOX_REQ_SIG;
3385 req->hdr.pcifunc = pf->pcifunc;
3386 memcpy(&req->link_info, &pf->linfo, sizeof(req->link_info));
3387
3388 otx2_mbox_wait_for_zero(&pf->mbox_pfvf[0].mbox_up, vf_idx);
3389
3390 otx2_sync_mbox_up_msg(&pf->mbox_pfvf[0], vf_idx);
3391
3392 mutex_unlock(&pf->mbox.lock);
3393 }
3394
otx2_sriov_enable(struct pci_dev * pdev,int numvfs)3395 static int otx2_sriov_enable(struct pci_dev *pdev, int numvfs)
3396 {
3397 struct net_device *netdev = pci_get_drvdata(pdev);
3398 struct otx2_nic *pf = netdev_priv(netdev);
3399 int ret;
3400
3401 /* Init PF <=> VF mailbox stuff */
3402 ret = otx2_pfvf_mbox_init(pf, numvfs);
3403 if (ret)
3404 return ret;
3405
3406 ret = otx2_register_pfvf_mbox_intr(pf, numvfs);
3407 if (ret)
3408 goto free_mbox;
3409
3410 ret = otx2_pf_flr_init(pf, numvfs);
3411 if (ret)
3412 goto free_intr;
3413
3414 ret = otx2_register_flr_me_intr(pf, numvfs);
3415 if (ret)
3416 goto free_flr;
3417
3418 ret = pci_enable_sriov(pdev, numvfs);
3419 if (ret)
3420 goto free_flr_intr;
3421
3422 return numvfs;
3423 free_flr_intr:
3424 otx2_disable_flr_me_intr(pf);
3425 free_flr:
3426 otx2_flr_wq_destroy(pf);
3427 free_intr:
3428 otx2_disable_pfvf_mbox_intr(pf, numvfs);
3429 free_mbox:
3430 otx2_pfvf_mbox_destroy(pf);
3431 return ret;
3432 }
3433
otx2_sriov_disable(struct pci_dev * pdev)3434 static int otx2_sriov_disable(struct pci_dev *pdev)
3435 {
3436 struct net_device *netdev = pci_get_drvdata(pdev);
3437 struct otx2_nic *pf = netdev_priv(netdev);
3438 int numvfs = pci_num_vf(pdev);
3439
3440 if (!numvfs)
3441 return 0;
3442
3443 pci_disable_sriov(pdev);
3444
3445 otx2_disable_flr_me_intr(pf);
3446 otx2_flr_wq_destroy(pf);
3447 otx2_disable_pfvf_mbox_intr(pf, numvfs);
3448 otx2_pfvf_mbox_destroy(pf);
3449
3450 return 0;
3451 }
3452
otx2_sriov_configure(struct pci_dev * pdev,int numvfs)3453 static int otx2_sriov_configure(struct pci_dev *pdev, int numvfs)
3454 {
3455 if (numvfs == 0)
3456 return otx2_sriov_disable(pdev);
3457 else
3458 return otx2_sriov_enable(pdev, numvfs);
3459 }
3460
otx2_ndc_sync(struct otx2_nic * pf)3461 static void otx2_ndc_sync(struct otx2_nic *pf)
3462 {
3463 struct mbox *mbox = &pf->mbox;
3464 struct ndc_sync_op *req;
3465
3466 mutex_lock(&mbox->lock);
3467
3468 req = otx2_mbox_alloc_msg_ndc_sync_op(mbox);
3469 if (!req) {
3470 mutex_unlock(&mbox->lock);
3471 return;
3472 }
3473
3474 req->nix_lf_tx_sync = 1;
3475 req->nix_lf_rx_sync = 1;
3476 req->npa_lf_sync = 1;
3477
3478 if (!otx2_sync_mbox_msg(mbox))
3479 dev_err(pf->dev, "NDC sync operation failed\n");
3480
3481 mutex_unlock(&mbox->lock);
3482 }
3483
otx2_remove(struct pci_dev * pdev)3484 static void otx2_remove(struct pci_dev *pdev)
3485 {
3486 struct net_device *netdev = pci_get_drvdata(pdev);
3487 struct otx2_nic *pf;
3488
3489 if (!netdev)
3490 return;
3491
3492 pf = netdev_priv(netdev);
3493
3494 pf->flags |= OTX2_FLAG_PF_SHUTDOWN;
3495
3496 if (pf->flags & OTX2_FLAG_TX_TSTAMP_ENABLED)
3497 otx2_config_hw_tx_tstamp(pf, false);
3498 if (pf->flags & OTX2_FLAG_RX_TSTAMP_ENABLED)
3499 otx2_config_hw_rx_tstamp(pf, false);
3500
3501 /* Disable 802.3x pause frames */
3502 if (pf->flags & OTX2_FLAG_RX_PAUSE_ENABLED ||
3503 (pf->flags & OTX2_FLAG_TX_PAUSE_ENABLED)) {
3504 pf->flags &= ~OTX2_FLAG_RX_PAUSE_ENABLED;
3505 pf->flags &= ~OTX2_FLAG_TX_PAUSE_ENABLED;
3506 otx2_config_pause_frm(pf);
3507 }
3508
3509 #ifdef CONFIG_DCB
3510 /* Disable PFC config */
3511 if (pf->pfc_en) {
3512 pf->pfc_en = 0;
3513 otx2_config_priority_flow_ctrl(pf);
3514 }
3515 #endif
3516 cancel_work_sync(&pf->reset_task);
3517 /* Disable link notifications */
3518 otx2_cgx_config_linkevents(pf, false);
3519
3520 otx2_unregister_dl(pf);
3521 unregister_netdev(netdev);
3522 cn10k_ipsec_clean(pf);
3523 cn10k_mcs_free(pf);
3524 otx2_sriov_disable(pf->pdev);
3525 otx2_sriov_vfcfg_cleanup(pf);
3526 if (pf->otx2_wq)
3527 destroy_workqueue(pf->otx2_wq);
3528
3529 otx2_ptp_destroy(pf);
3530 otx2_mcam_flow_del(pf);
3531 otx2_shutdown_tc(pf);
3532 otx2_shutdown_qos(pf);
3533 otx2_ndc_sync(pf);
3534 otx2_detach_resources(&pf->mbox);
3535 if (pf->hw.lmt_info)
3536 free_percpu(pf->hw.lmt_info);
3537 if (test_bit(CN10K_LMTST, &pf->hw.cap_flag))
3538 qmem_free(pf->dev, pf->dync_lmt);
3539 otx2_disable_mbox_intr(pf);
3540 otx2_pfaf_mbox_destroy(pf);
3541 pci_free_irq_vectors(pf->pdev);
3542 pci_set_drvdata(pdev, NULL);
3543 free_netdev(netdev);
3544 }
3545
3546 static struct pci_driver otx2_pf_driver = {
3547 .name = DRV_NAME,
3548 .id_table = otx2_pf_id_table,
3549 .probe = otx2_probe,
3550 .shutdown = otx2_remove,
3551 .remove = otx2_remove,
3552 .sriov_configure = otx2_sriov_configure
3553 };
3554
otx2_rvupf_init_module(void)3555 static int __init otx2_rvupf_init_module(void)
3556 {
3557 pr_info("%s: %s\n", DRV_NAME, DRV_STRING);
3558
3559 return pci_register_driver(&otx2_pf_driver);
3560 }
3561
otx2_rvupf_cleanup_module(void)3562 static void __exit otx2_rvupf_cleanup_module(void)
3563 {
3564 pci_unregister_driver(&otx2_pf_driver);
3565 }
3566
3567 module_init(otx2_rvupf_init_module);
3568 module_exit(otx2_rvupf_cleanup_module);
3569