1 // SPDX-License-Identifier: GPL-2.0
2 /* Marvell RVU Physical Function ethernet driver
3 *
4 * Copyright (C) 2020 Marvell.
5 *
6 */
7
8 #include <linux/module.h>
9 #include <linux/interrupt.h>
10 #include <linux/pci.h>
11 #include <linux/etherdevice.h>
12 #include <linux/of.h>
13 #include <linux/if_vlan.h>
14 #include <linux/iommu.h>
15 #include <net/ip.h>
16 #include <linux/bpf.h>
17 #include <linux/bpf_trace.h>
18 #include <linux/bitfield.h>
19 #include <net/page_pool/types.h>
20
21 #include "otx2_reg.h"
22 #include "otx2_common.h"
23 #include "otx2_txrx.h"
24 #include "otx2_struct.h"
25 #include "otx2_ptp.h"
26 #include "cn10k.h"
27 #include "qos.h"
28 #include <rvu_trace.h>
29 #include "cn10k_ipsec.h"
30 #include "otx2_xsk.h"
31
32 #define DRV_NAME "rvu_nicpf"
33 #define DRV_STRING "Marvell RVU NIC Physical Function Driver"
34
35 /* Supported devices */
36 static const struct pci_device_id otx2_pf_id_table[] = {
37 { PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, PCI_DEVID_OCTEONTX2_RVU_PF) },
38 { 0, } /* end of table */
39 };
40
41 MODULE_AUTHOR("Sunil Goutham <sgoutham@marvell.com>");
42 MODULE_DESCRIPTION(DRV_STRING);
43 MODULE_LICENSE("GPL v2");
44 MODULE_DEVICE_TABLE(pci, otx2_pf_id_table);
45
46 static void otx2_vf_link_event_task(struct work_struct *work);
47
48 enum {
49 TYPE_PFAF,
50 TYPE_PFVF,
51 };
52
53 static int otx2_config_hw_tx_tstamp(struct otx2_nic *pfvf, bool enable);
54 static int otx2_config_hw_rx_tstamp(struct otx2_nic *pfvf, bool enable);
55
otx2_change_mtu(struct net_device * netdev,int new_mtu)56 static int otx2_change_mtu(struct net_device *netdev, int new_mtu)
57 {
58 struct otx2_nic *pf = netdev_priv(netdev);
59 bool if_up = netif_running(netdev);
60 int err = 0;
61
62 if (pf->xdp_prog && new_mtu > MAX_XDP_MTU) {
63 netdev_warn(netdev, "Jumbo frames not yet supported with XDP, current MTU %d.\n",
64 netdev->mtu);
65 return -EINVAL;
66 }
67 if (if_up)
68 otx2_stop(netdev);
69
70 netdev_info(netdev, "Changing MTU from %d to %d\n",
71 netdev->mtu, new_mtu);
72 WRITE_ONCE(netdev->mtu, new_mtu);
73
74 if (if_up)
75 err = otx2_open(netdev);
76
77 return err;
78 }
79
otx2_disable_flr_me_intr(struct otx2_nic * pf)80 static void otx2_disable_flr_me_intr(struct otx2_nic *pf)
81 {
82 int irq, vfs = pf->total_vfs;
83
84 /* Disable VFs ME interrupts */
85 otx2_write64(pf, RVU_PF_VFME_INT_ENA_W1CX(0), INTR_MASK(vfs));
86 irq = pci_irq_vector(pf->pdev, RVU_PF_INT_VEC_VFME0);
87 free_irq(irq, pf);
88
89 /* Disable VFs FLR interrupts */
90 otx2_write64(pf, RVU_PF_VFFLR_INT_ENA_W1CX(0), INTR_MASK(vfs));
91 irq = pci_irq_vector(pf->pdev, RVU_PF_INT_VEC_VFFLR0);
92 free_irq(irq, pf);
93
94 if (vfs <= 64)
95 return;
96
97 otx2_write64(pf, RVU_PF_VFME_INT_ENA_W1CX(1), INTR_MASK(vfs - 64));
98 irq = pci_irq_vector(pf->pdev, RVU_PF_INT_VEC_VFME1);
99 free_irq(irq, pf);
100
101 otx2_write64(pf, RVU_PF_VFFLR_INT_ENA_W1CX(1), INTR_MASK(vfs - 64));
102 irq = pci_irq_vector(pf->pdev, RVU_PF_INT_VEC_VFFLR1);
103 free_irq(irq, pf);
104 }
105
otx2_flr_wq_destroy(struct otx2_nic * pf)106 static void otx2_flr_wq_destroy(struct otx2_nic *pf)
107 {
108 if (!pf->flr_wq)
109 return;
110 destroy_workqueue(pf->flr_wq);
111 pf->flr_wq = NULL;
112 devm_kfree(pf->dev, pf->flr_wrk);
113 }
114
otx2_flr_handler(struct work_struct * work)115 static void otx2_flr_handler(struct work_struct *work)
116 {
117 struct flr_work *flrwork = container_of(work, struct flr_work, work);
118 struct otx2_nic *pf = flrwork->pf;
119 struct mbox *mbox = &pf->mbox;
120 struct msg_req *req;
121 int vf, reg = 0;
122
123 vf = flrwork - pf->flr_wrk;
124
125 mutex_lock(&mbox->lock);
126 req = otx2_mbox_alloc_msg_vf_flr(mbox);
127 if (!req) {
128 mutex_unlock(&mbox->lock);
129 return;
130 }
131 req->hdr.pcifunc &= RVU_PFVF_FUNC_MASK;
132 req->hdr.pcifunc |= (vf + 1) & RVU_PFVF_FUNC_MASK;
133
134 if (!otx2_sync_mbox_msg(&pf->mbox)) {
135 if (vf >= 64) {
136 reg = 1;
137 vf = vf - 64;
138 }
139 /* clear transcation pending bit */
140 otx2_write64(pf, RVU_PF_VFTRPENDX(reg), BIT_ULL(vf));
141 otx2_write64(pf, RVU_PF_VFFLR_INT_ENA_W1SX(reg), BIT_ULL(vf));
142 }
143
144 mutex_unlock(&mbox->lock);
145 }
146
otx2_pf_flr_intr_handler(int irq,void * pf_irq)147 static irqreturn_t otx2_pf_flr_intr_handler(int irq, void *pf_irq)
148 {
149 struct otx2_nic *pf = (struct otx2_nic *)pf_irq;
150 int reg, dev, vf, start_vf, num_reg = 1;
151 u64 intr;
152
153 if (pf->total_vfs > 64)
154 num_reg = 2;
155
156 for (reg = 0; reg < num_reg; reg++) {
157 intr = otx2_read64(pf, RVU_PF_VFFLR_INTX(reg));
158 if (!intr)
159 continue;
160 start_vf = 64 * reg;
161 for (vf = 0; vf < 64; vf++) {
162 if (!(intr & BIT_ULL(vf)))
163 continue;
164 dev = vf + start_vf;
165 queue_work(pf->flr_wq, &pf->flr_wrk[dev].work);
166 /* Clear interrupt */
167 otx2_write64(pf, RVU_PF_VFFLR_INTX(reg), BIT_ULL(vf));
168 /* Disable the interrupt */
169 otx2_write64(pf, RVU_PF_VFFLR_INT_ENA_W1CX(reg),
170 BIT_ULL(vf));
171 }
172 }
173 return IRQ_HANDLED;
174 }
175
otx2_pf_me_intr_handler(int irq,void * pf_irq)176 static irqreturn_t otx2_pf_me_intr_handler(int irq, void *pf_irq)
177 {
178 struct otx2_nic *pf = (struct otx2_nic *)pf_irq;
179 int vf, reg, num_reg = 1;
180 u64 intr;
181
182 if (pf->total_vfs > 64)
183 num_reg = 2;
184
185 for (reg = 0; reg < num_reg; reg++) {
186 intr = otx2_read64(pf, RVU_PF_VFME_INTX(reg));
187 if (!intr)
188 continue;
189 for (vf = 0; vf < 64; vf++) {
190 if (!(intr & BIT_ULL(vf)))
191 continue;
192 /* clear trpend bit */
193 otx2_write64(pf, RVU_PF_VFTRPENDX(reg), BIT_ULL(vf));
194 /* clear interrupt */
195 otx2_write64(pf, RVU_PF_VFME_INTX(reg), BIT_ULL(vf));
196 }
197 }
198 return IRQ_HANDLED;
199 }
200
otx2_register_flr_me_intr(struct otx2_nic * pf,int numvfs)201 static int otx2_register_flr_me_intr(struct otx2_nic *pf, int numvfs)
202 {
203 struct otx2_hw *hw = &pf->hw;
204 char *irq_name;
205 int ret;
206
207 /* Register ME interrupt handler*/
208 irq_name = &hw->irq_name[RVU_PF_INT_VEC_VFME0 * NAME_SIZE];
209 snprintf(irq_name, NAME_SIZE, "RVUPF%d_ME0",
210 rvu_get_pf(pf->pdev, pf->pcifunc));
211 ret = request_irq(pci_irq_vector(pf->pdev, RVU_PF_INT_VEC_VFME0),
212 otx2_pf_me_intr_handler, 0, irq_name, pf);
213 if (ret) {
214 dev_err(pf->dev,
215 "RVUPF: IRQ registration failed for ME0\n");
216 }
217
218 /* Register FLR interrupt handler */
219 irq_name = &hw->irq_name[RVU_PF_INT_VEC_VFFLR0 * NAME_SIZE];
220 snprintf(irq_name, NAME_SIZE, "RVUPF%d_FLR0",
221 rvu_get_pf(pf->pdev, pf->pcifunc));
222 ret = request_irq(pci_irq_vector(pf->pdev, RVU_PF_INT_VEC_VFFLR0),
223 otx2_pf_flr_intr_handler, 0, irq_name, pf);
224 if (ret) {
225 dev_err(pf->dev,
226 "RVUPF: IRQ registration failed for FLR0\n");
227 return ret;
228 }
229
230 if (numvfs > 64) {
231 irq_name = &hw->irq_name[RVU_PF_INT_VEC_VFME1 * NAME_SIZE];
232 snprintf(irq_name, NAME_SIZE, "RVUPF%d_ME1",
233 rvu_get_pf(pf->pdev, pf->pcifunc));
234 ret = request_irq(pci_irq_vector
235 (pf->pdev, RVU_PF_INT_VEC_VFME1),
236 otx2_pf_me_intr_handler, 0, irq_name, pf);
237 if (ret) {
238 dev_err(pf->dev,
239 "RVUPF: IRQ registration failed for ME1\n");
240 }
241 irq_name = &hw->irq_name[RVU_PF_INT_VEC_VFFLR1 * NAME_SIZE];
242 snprintf(irq_name, NAME_SIZE, "RVUPF%d_FLR1",
243 rvu_get_pf(pf->pdev, pf->pcifunc));
244 ret = request_irq(pci_irq_vector
245 (pf->pdev, RVU_PF_INT_VEC_VFFLR1),
246 otx2_pf_flr_intr_handler, 0, irq_name, pf);
247 if (ret) {
248 dev_err(pf->dev,
249 "RVUPF: IRQ registration failed for FLR1\n");
250 return ret;
251 }
252 }
253
254 /* Enable ME interrupt for all VFs*/
255 otx2_write64(pf, RVU_PF_VFME_INTX(0), INTR_MASK(numvfs));
256 otx2_write64(pf, RVU_PF_VFME_INT_ENA_W1SX(0), INTR_MASK(numvfs));
257
258 /* Enable FLR interrupt for all VFs*/
259 otx2_write64(pf, RVU_PF_VFFLR_INTX(0), INTR_MASK(numvfs));
260 otx2_write64(pf, RVU_PF_VFFLR_INT_ENA_W1SX(0), INTR_MASK(numvfs));
261
262 if (numvfs > 64) {
263 numvfs -= 64;
264
265 otx2_write64(pf, RVU_PF_VFME_INTX(1), INTR_MASK(numvfs));
266 otx2_write64(pf, RVU_PF_VFME_INT_ENA_W1SX(1),
267 INTR_MASK(numvfs));
268
269 otx2_write64(pf, RVU_PF_VFFLR_INTX(1), INTR_MASK(numvfs));
270 otx2_write64(pf, RVU_PF_VFFLR_INT_ENA_W1SX(1),
271 INTR_MASK(numvfs));
272 }
273 return 0;
274 }
275
otx2_pf_flr_init(struct otx2_nic * pf,int num_vfs)276 static int otx2_pf_flr_init(struct otx2_nic *pf, int num_vfs)
277 {
278 int vf;
279
280 pf->flr_wq = alloc_ordered_workqueue("otx2_pf_flr_wq", WQ_HIGHPRI);
281 if (!pf->flr_wq)
282 return -ENOMEM;
283
284 pf->flr_wrk = devm_kcalloc(pf->dev, num_vfs,
285 sizeof(struct flr_work), GFP_KERNEL);
286 if (!pf->flr_wrk) {
287 destroy_workqueue(pf->flr_wq);
288 return -ENOMEM;
289 }
290
291 for (vf = 0; vf < num_vfs; vf++) {
292 pf->flr_wrk[vf].pf = pf;
293 INIT_WORK(&pf->flr_wrk[vf].work, otx2_flr_handler);
294 }
295
296 return 0;
297 }
298
otx2_queue_vf_work(struct mbox * mw,struct workqueue_struct * mbox_wq,int first,int mdevs,u64 intr)299 void otx2_queue_vf_work(struct mbox *mw, struct workqueue_struct *mbox_wq,
300 int first, int mdevs, u64 intr)
301 {
302 struct otx2_mbox_dev *mdev;
303 struct otx2_mbox *mbox;
304 struct mbox_hdr *hdr;
305 int i;
306
307 for (i = first; i < mdevs; i++) {
308 /* start from 0 */
309 if (!(intr & BIT_ULL(i - first)))
310 continue;
311
312 mbox = &mw->mbox;
313 mdev = &mbox->dev[i];
314 hdr = mdev->mbase + mbox->rx_start;
315 /* The hdr->num_msgs is set to zero immediately in the interrupt
316 * handler to ensure that it holds a correct value next time
317 * when the interrupt handler is called. pf->mw[i].num_msgs
318 * holds the data for use in otx2_pfvf_mbox_handler and
319 * pf->mw[i].up_num_msgs holds the data for use in
320 * otx2_pfvf_mbox_up_handler.
321 */
322 if (hdr->num_msgs) {
323 mw[i].num_msgs = hdr->num_msgs;
324 hdr->num_msgs = 0;
325 queue_work(mbox_wq, &mw[i].mbox_wrk);
326 }
327
328 mbox = &mw->mbox_up;
329 mdev = &mbox->dev[i];
330 hdr = mdev->mbase + mbox->rx_start;
331 if (hdr->num_msgs) {
332 mw[i].up_num_msgs = hdr->num_msgs;
333 hdr->num_msgs = 0;
334 queue_work(mbox_wq, &mw[i].mbox_up_wrk);
335 }
336 }
337 }
338
otx2_forward_msg_pfvf(struct otx2_mbox_dev * mdev,struct otx2_mbox * pfvf_mbox,void * bbuf_base,int devid)339 static void otx2_forward_msg_pfvf(struct otx2_mbox_dev *mdev,
340 struct otx2_mbox *pfvf_mbox, void *bbuf_base,
341 int devid)
342 {
343 struct otx2_mbox_dev *src_mdev = mdev;
344 int offset;
345
346 /* Msgs are already copied, trigger VF's mbox irq */
347 smp_wmb();
348
349 otx2_mbox_wait_for_zero(pfvf_mbox, devid);
350
351 offset = pfvf_mbox->trigger | (devid << pfvf_mbox->tr_shift);
352 writeq(MBOX_DOWN_MSG, (void __iomem *)pfvf_mbox->reg_base + offset);
353
354 /* Restore VF's mbox bounce buffer region address */
355 src_mdev->mbase = bbuf_base;
356 }
357
otx2_forward_vf_mbox_msgs(struct otx2_nic * pf,struct otx2_mbox * src_mbox,int dir,int vf,int num_msgs)358 static int otx2_forward_vf_mbox_msgs(struct otx2_nic *pf,
359 struct otx2_mbox *src_mbox,
360 int dir, int vf, int num_msgs)
361 {
362 struct otx2_mbox_dev *src_mdev, *dst_mdev;
363 struct mbox_hdr *mbox_hdr;
364 struct mbox_hdr *req_hdr;
365 struct mbox *dst_mbox;
366 int dst_size, err;
367
368 if (dir == MBOX_DIR_PFAF) {
369 /* Set VF's mailbox memory as PF's bounce buffer memory, so
370 * that explicit copying of VF's msgs to PF=>AF mbox region
371 * and AF=>PF responses to VF's mbox region can be avoided.
372 */
373 src_mdev = &src_mbox->dev[vf];
374 mbox_hdr = src_mbox->hwbase +
375 src_mbox->rx_start + (vf * MBOX_SIZE);
376
377 dst_mbox = &pf->mbox;
378 dst_size = dst_mbox->mbox.tx_size -
379 ALIGN(sizeof(*mbox_hdr), MBOX_MSG_ALIGN);
380 /* Check if msgs fit into destination area and has valid size */
381 if (mbox_hdr->msg_size > dst_size || !mbox_hdr->msg_size)
382 return -EINVAL;
383
384 dst_mdev = &dst_mbox->mbox.dev[0];
385
386 mutex_lock(&pf->mbox.lock);
387 dst_mdev->mbase = src_mdev->mbase;
388 dst_mdev->msg_size = mbox_hdr->msg_size;
389 dst_mdev->num_msgs = num_msgs;
390 err = otx2_sync_mbox_msg(dst_mbox);
391 /* Error code -EIO indicate there is a communication failure
392 * to the AF. Rest of the error codes indicate that AF processed
393 * VF messages and set the error codes in response messages
394 * (if any) so simply forward responses to VF.
395 */
396 if (err == -EIO) {
397 dev_warn(pf->dev,
398 "AF not responding to VF%d messages\n", vf);
399 /* restore PF mbase and exit */
400 dst_mdev->mbase = pf->mbox.bbuf_base;
401 mutex_unlock(&pf->mbox.lock);
402 return err;
403 }
404 /* At this point, all the VF messages sent to AF are acked
405 * with proper responses and responses are copied to VF
406 * mailbox hence raise interrupt to VF.
407 */
408 req_hdr = (struct mbox_hdr *)(dst_mdev->mbase +
409 dst_mbox->mbox.rx_start);
410 req_hdr->num_msgs = num_msgs;
411
412 otx2_forward_msg_pfvf(dst_mdev, &pf->mbox_pfvf[0].mbox,
413 pf->mbox.bbuf_base, vf);
414 mutex_unlock(&pf->mbox.lock);
415 } else if (dir == MBOX_DIR_PFVF_UP) {
416 src_mdev = &src_mbox->dev[0];
417 mbox_hdr = src_mbox->hwbase + src_mbox->rx_start;
418 req_hdr = (struct mbox_hdr *)(src_mdev->mbase +
419 src_mbox->rx_start);
420 req_hdr->num_msgs = num_msgs;
421
422 dst_mbox = &pf->mbox_pfvf[0];
423 dst_size = dst_mbox->mbox_up.tx_size -
424 ALIGN(sizeof(*mbox_hdr), MBOX_MSG_ALIGN);
425 /* Check if msgs fit into destination area */
426 if (mbox_hdr->msg_size > dst_size)
427 return -EINVAL;
428
429 dst_mdev = &dst_mbox->mbox_up.dev[vf];
430 dst_mdev->mbase = src_mdev->mbase;
431 dst_mdev->msg_size = mbox_hdr->msg_size;
432 dst_mdev->num_msgs = mbox_hdr->num_msgs;
433 err = otx2_sync_mbox_up_msg(dst_mbox, vf);
434 if (err) {
435 dev_warn(pf->dev,
436 "VF%d is not responding to mailbox\n", vf);
437 return err;
438 }
439 } else if (dir == MBOX_DIR_VFPF_UP) {
440 req_hdr = (struct mbox_hdr *)(src_mbox->dev[0].mbase +
441 src_mbox->rx_start);
442 req_hdr->num_msgs = num_msgs;
443 otx2_forward_msg_pfvf(&pf->mbox_pfvf->mbox_up.dev[vf],
444 &pf->mbox.mbox_up,
445 pf->mbox_pfvf[vf].bbuf_base,
446 0);
447 }
448
449 return 0;
450 }
451
otx2_pfvf_mbox_handler(struct work_struct * work)452 static void otx2_pfvf_mbox_handler(struct work_struct *work)
453 {
454 struct mbox_msghdr *msg = NULL;
455 int offset, vf_idx, id, err;
456 struct otx2_mbox_dev *mdev;
457 struct otx2_mbox *mbox;
458 struct mbox *vf_mbox;
459 struct otx2_nic *pf;
460
461 vf_mbox = container_of(work, struct mbox, mbox_wrk);
462 pf = vf_mbox->pfvf;
463 vf_idx = vf_mbox - pf->mbox_pfvf;
464
465 mbox = &pf->mbox_pfvf[0].mbox;
466 mdev = &mbox->dev[vf_idx];
467
468 offset = ALIGN(sizeof(struct mbox_hdr), MBOX_MSG_ALIGN);
469
470 trace_otx2_msg_status(pf->pdev, "PF-VF down queue handler(forwarding)",
471 vf_mbox->num_msgs);
472
473 for (id = 0; id < vf_mbox->num_msgs; id++) {
474 msg = (struct mbox_msghdr *)(mdev->mbase + mbox->rx_start +
475 offset);
476
477 if (msg->sig != OTX2_MBOX_REQ_SIG)
478 goto inval_msg;
479
480 /* Set VF's number in each of the msg */
481 msg->pcifunc &= ~RVU_PFVF_FUNC_MASK;
482 msg->pcifunc |= (vf_idx + 1) & RVU_PFVF_FUNC_MASK;
483 offset = msg->next_msgoff;
484 }
485 err = otx2_forward_vf_mbox_msgs(pf, mbox, MBOX_DIR_PFAF, vf_idx,
486 vf_mbox->num_msgs);
487 if (err)
488 goto inval_msg;
489 return;
490
491 inval_msg:
492 otx2_reply_invalid_msg(mbox, vf_idx, 0, msg->id);
493 otx2_mbox_msg_send(mbox, vf_idx);
494 }
495
otx2_pfvf_mbox_up_handler(struct work_struct * work)496 static void otx2_pfvf_mbox_up_handler(struct work_struct *work)
497 {
498 struct mbox *vf_mbox = container_of(work, struct mbox, mbox_up_wrk);
499 struct otx2_nic *pf = vf_mbox->pfvf;
500 struct otx2_mbox_dev *mdev;
501 int offset, id, vf_idx = 0;
502 struct mbox_msghdr *msg;
503 struct otx2_mbox *mbox;
504
505 vf_idx = vf_mbox - pf->mbox_pfvf;
506 mbox = &pf->mbox_pfvf[0].mbox_up;
507 mdev = &mbox->dev[vf_idx];
508
509 offset = mbox->rx_start + ALIGN(sizeof(struct mbox_hdr), MBOX_MSG_ALIGN);
510
511 trace_otx2_msg_status(pf->pdev, "PF-VF up queue handler(response)",
512 vf_mbox->up_num_msgs);
513
514 for (id = 0; id < vf_mbox->up_num_msgs; id++) {
515 msg = mdev->mbase + offset;
516
517 if (msg->id >= MBOX_MSG_MAX) {
518 dev_err(pf->dev,
519 "Mbox msg with unknown ID 0x%x\n", msg->id);
520 goto end;
521 }
522
523 if (msg->sig != OTX2_MBOX_RSP_SIG) {
524 dev_err(pf->dev,
525 "Mbox msg with wrong signature %x, ID 0x%x\n",
526 msg->sig, msg->id);
527 goto end;
528 }
529
530 switch (msg->id) {
531 case MBOX_MSG_CGX_LINK_EVENT:
532 case MBOX_MSG_REP_EVENT_UP_NOTIFY:
533 break;
534 default:
535 if (msg->rc)
536 dev_err(pf->dev,
537 "Mbox msg response has err %d, ID 0x%x\n",
538 msg->rc, msg->id);
539 break;
540 }
541
542 end:
543 offset = mbox->rx_start + msg->next_msgoff;
544 if (mdev->msgs_acked == (vf_mbox->up_num_msgs - 1))
545 __otx2_mbox_reset(mbox, vf_idx);
546 mdev->msgs_acked++;
547 }
548 }
549
otx2_pfvf_mbox_intr_handler(int irq,void * pf_irq)550 irqreturn_t otx2_pfvf_mbox_intr_handler(int irq, void *pf_irq)
551 {
552 struct otx2_nic *pf = (struct otx2_nic *)(pf_irq);
553 int vfs = pf->total_vfs;
554 struct mbox *mbox;
555 u64 intr;
556
557 mbox = pf->mbox_pfvf;
558 /* Handle VF interrupts */
559 if (vfs > 64) {
560 intr = otx2_read64(pf, RVU_PF_VFPF_MBOX_INTX(1));
561 otx2_write64(pf, RVU_PF_VFPF_MBOX_INTX(1), intr);
562 otx2_queue_vf_work(mbox, pf->mbox_pfvf_wq, 64, vfs, intr);
563 if (intr)
564 trace_otx2_msg_interrupt(mbox->mbox.pdev, "VF(s) to PF", intr);
565 vfs = 64;
566 }
567
568 intr = otx2_read64(pf, RVU_PF_VFPF_MBOX_INTX(0));
569 otx2_write64(pf, RVU_PF_VFPF_MBOX_INTX(0), intr);
570
571 otx2_queue_vf_work(mbox, pf->mbox_pfvf_wq, 0, vfs, intr);
572
573 if (intr)
574 trace_otx2_msg_interrupt(mbox->mbox.pdev, "VF(s) to PF", intr);
575
576 return IRQ_HANDLED;
577 }
578
cn20k_pfvf_mbox_alloc(struct otx2_nic * pf,int numvfs)579 static void *cn20k_pfvf_mbox_alloc(struct otx2_nic *pf, int numvfs)
580 {
581 struct qmem *mbox_addr;
582 int err;
583
584 err = qmem_alloc(&pf->pdev->dev, &mbox_addr, numvfs, MBOX_SIZE);
585 if (err) {
586 dev_err(pf->dev, "qmem alloc fail\n");
587 return ERR_PTR(-ENOMEM);
588 }
589
590 otx2_write64(pf, RVU_PF_VF_MBOX_ADDR, (u64)mbox_addr->iova);
591 pf->pfvf_mbox_addr = mbox_addr;
592
593 return mbox_addr->base;
594 }
595
otx2_pfvf_mbox_init(struct otx2_nic * pf,int numvfs)596 static int otx2_pfvf_mbox_init(struct otx2_nic *pf, int numvfs)
597 {
598 void __iomem *hwbase;
599 struct mbox *mbox;
600 int err, vf;
601 u64 base;
602
603 if (!numvfs)
604 return -EINVAL;
605
606 pf->mbox_pfvf = devm_kcalloc(&pf->pdev->dev, numvfs,
607 sizeof(struct mbox), GFP_KERNEL);
608 if (!pf->mbox_pfvf)
609 return -ENOMEM;
610
611 pf->mbox_pfvf_wq = alloc_workqueue("otx2_pfvf_mailbox",
612 WQ_UNBOUND | WQ_HIGHPRI |
613 WQ_MEM_RECLAIM, 0);
614 if (!pf->mbox_pfvf_wq)
615 return -ENOMEM;
616
617 /* For CN20K, PF allocates mbox memory in DRAM and writes PF/VF
618 * regions/offsets in RVU_PF_VF_MBOX_ADDR, the RVU_PFX_FUNC_PFAF_MBOX
619 * gives the aliased address to access PF/VF mailbox regions.
620 */
621 if (is_cn20k(pf->pdev)) {
622 hwbase = (void __iomem *)cn20k_pfvf_mbox_alloc(pf, numvfs);
623 } else {
624 /* On CN10K platform, PF <-> VF mailbox region follows after
625 * PF <-> AF mailbox region.
626 */
627 if (test_bit(CN10K_MBOX, &pf->hw.cap_flag))
628 base = pci_resource_start(pf->pdev, PCI_MBOX_BAR_NUM) +
629 MBOX_SIZE;
630 else
631 base = readq(pf->reg_base + RVU_PF_VF_BAR4_ADDR);
632
633 hwbase = ioremap_wc(base, MBOX_SIZE * pf->total_vfs);
634 if (!hwbase) {
635 err = -ENOMEM;
636 goto free_wq;
637 }
638 }
639
640 mbox = &pf->mbox_pfvf[0];
641 err = otx2_mbox_init(&mbox->mbox, hwbase, pf->pdev, pf->reg_base,
642 MBOX_DIR_PFVF, numvfs);
643 if (err)
644 goto free_iomem;
645
646 err = otx2_mbox_init(&mbox->mbox_up, hwbase, pf->pdev, pf->reg_base,
647 MBOX_DIR_PFVF_UP, numvfs);
648 if (err)
649 goto free_iomem;
650
651 for (vf = 0; vf < numvfs; vf++) {
652 mbox->pfvf = pf;
653 INIT_WORK(&mbox->mbox_wrk, otx2_pfvf_mbox_handler);
654 INIT_WORK(&mbox->mbox_up_wrk, otx2_pfvf_mbox_up_handler);
655 mbox++;
656 }
657
658 return 0;
659
660 free_iomem:
661 if (hwbase && !(is_cn20k(pf->pdev)))
662 iounmap(hwbase);
663 free_wq:
664 destroy_workqueue(pf->mbox_pfvf_wq);
665 return err;
666 }
667
otx2_pfvf_mbox_destroy(struct otx2_nic * pf)668 static void otx2_pfvf_mbox_destroy(struct otx2_nic *pf)
669 {
670 struct mbox *mbox = &pf->mbox_pfvf[0];
671
672 if (!mbox)
673 return;
674
675 if (pf->mbox_pfvf_wq) {
676 destroy_workqueue(pf->mbox_pfvf_wq);
677 pf->mbox_pfvf_wq = NULL;
678 }
679
680 if (mbox->mbox.hwbase && !is_cn20k(pf->pdev))
681 iounmap(mbox->mbox.hwbase);
682 else
683 qmem_free(&pf->pdev->dev, pf->pfvf_mbox_addr);
684
685 otx2_mbox_destroy(&mbox->mbox);
686 }
687
otx2_enable_pfvf_mbox_intr(struct otx2_nic * pf,int numvfs)688 static void otx2_enable_pfvf_mbox_intr(struct otx2_nic *pf, int numvfs)
689 {
690 /* Clear PF <=> VF mailbox IRQ */
691 otx2_write64(pf, RVU_PF_VFPF_MBOX_INTX(0), ~0ull);
692 otx2_write64(pf, RVU_PF_VFPF_MBOX_INTX(1), ~0ull);
693
694 /* Enable PF <=> VF mailbox IRQ */
695 otx2_write64(pf, RVU_PF_VFPF_MBOX_INT_ENA_W1SX(0), INTR_MASK(numvfs));
696 if (numvfs > 64) {
697 numvfs -= 64;
698 otx2_write64(pf, RVU_PF_VFPF_MBOX_INT_ENA_W1SX(1),
699 INTR_MASK(numvfs));
700 }
701 }
702
otx2_disable_pfvf_mbox_intr(struct otx2_nic * pf,int numvfs)703 static void otx2_disable_pfvf_mbox_intr(struct otx2_nic *pf, int numvfs)
704 {
705 int vector;
706
707 if (is_cn20k(pf->pdev))
708 return cn20k_disable_pfvf_mbox_intr(pf, numvfs);
709
710 /* Disable PF <=> VF mailbox IRQ */
711 otx2_write64(pf, RVU_PF_VFPF_MBOX_INT_ENA_W1CX(0), ~0ull);
712 otx2_write64(pf, RVU_PF_VFPF_MBOX_INT_ENA_W1CX(1), ~0ull);
713
714 otx2_write64(pf, RVU_PF_VFPF_MBOX_INTX(0), ~0ull);
715 vector = pci_irq_vector(pf->pdev, RVU_PF_INT_VEC_VFPF_MBOX0);
716 free_irq(vector, pf);
717
718 if (numvfs > 64) {
719 otx2_write64(pf, RVU_PF_VFPF_MBOX_INTX(1), ~0ull);
720 vector = pci_irq_vector(pf->pdev, RVU_PF_INT_VEC_VFPF_MBOX1);
721 free_irq(vector, pf);
722 }
723 }
724
otx2_register_pfvf_mbox_intr(struct otx2_nic * pf,int numvfs)725 static int otx2_register_pfvf_mbox_intr(struct otx2_nic *pf, int numvfs)
726 {
727 struct otx2_hw *hw = &pf->hw;
728 char *irq_name;
729 int err;
730
731 if (is_cn20k(pf->pdev))
732 return cn20k_register_pfvf_mbox_intr(pf, numvfs);
733
734 /* Register MBOX0 interrupt handler */
735 irq_name = &hw->irq_name[RVU_PF_INT_VEC_VFPF_MBOX0 * NAME_SIZE];
736 if (pf->pcifunc)
737 snprintf(irq_name, NAME_SIZE,
738 "RVUPF%d_VF Mbox0", rvu_get_pf(pf->pdev, pf->pcifunc));
739 else
740 snprintf(irq_name, NAME_SIZE, "RVUPF_VF Mbox0");
741 err = request_irq(pci_irq_vector(pf->pdev, RVU_PF_INT_VEC_VFPF_MBOX0),
742 otx2_pfvf_mbox_intr_handler, 0, irq_name, pf);
743 if (err) {
744 dev_err(pf->dev,
745 "RVUPF: IRQ registration failed for PFVF mbox0 irq\n");
746 return err;
747 }
748
749 if (numvfs > 64) {
750 /* Register MBOX1 interrupt handler */
751 irq_name = &hw->irq_name[RVU_PF_INT_VEC_VFPF_MBOX1 * NAME_SIZE];
752 if (pf->pcifunc)
753 snprintf(irq_name, NAME_SIZE,
754 "RVUPF%d_VF Mbox1",
755 rvu_get_pf(pf->pdev, pf->pcifunc));
756 else
757 snprintf(irq_name, NAME_SIZE, "RVUPF_VF Mbox1");
758 err = request_irq(pci_irq_vector(pf->pdev,
759 RVU_PF_INT_VEC_VFPF_MBOX1),
760 otx2_pfvf_mbox_intr_handler,
761 0, irq_name, pf);
762 if (err) {
763 dev_err(pf->dev,
764 "RVUPF: IRQ registration failed for PFVF mbox1 irq\n");
765 return err;
766 }
767 }
768
769 otx2_enable_pfvf_mbox_intr(pf, numvfs);
770
771 return 0;
772 }
773
otx2_process_pfaf_mbox_msg(struct otx2_nic * pf,struct mbox_msghdr * msg)774 static void otx2_process_pfaf_mbox_msg(struct otx2_nic *pf,
775 struct mbox_msghdr *msg)
776 {
777 int devid;
778
779 if (msg->id >= MBOX_MSG_MAX) {
780 dev_err(pf->dev,
781 "Mbox msg with unknown ID 0x%x\n", msg->id);
782 return;
783 }
784
785 if (msg->sig != OTX2_MBOX_RSP_SIG) {
786 dev_err(pf->dev,
787 "Mbox msg with wrong signature %x, ID 0x%x\n",
788 msg->sig, msg->id);
789 return;
790 }
791
792 /* message response heading VF */
793 devid = msg->pcifunc & RVU_PFVF_FUNC_MASK;
794 if (devid) {
795 struct otx2_vf_config *config = &pf->vf_configs[devid - 1];
796 struct delayed_work *dwork;
797
798 switch (msg->id) {
799 case MBOX_MSG_NIX_LF_START_RX:
800 config->intf_down = false;
801 dwork = &config->link_event_work;
802 schedule_delayed_work(dwork, msecs_to_jiffies(100));
803 break;
804 case MBOX_MSG_NIX_LF_STOP_RX:
805 config->intf_down = true;
806 break;
807 }
808
809 return;
810 }
811
812 switch (msg->id) {
813 case MBOX_MSG_READY:
814 pf->pcifunc = msg->pcifunc;
815 break;
816 case MBOX_MSG_MSIX_OFFSET:
817 mbox_handler_msix_offset(pf, (struct msix_offset_rsp *)msg);
818 break;
819 case MBOX_MSG_NPA_LF_ALLOC:
820 mbox_handler_npa_lf_alloc(pf, (struct npa_lf_alloc_rsp *)msg);
821 break;
822 case MBOX_MSG_NIX_LF_ALLOC:
823 mbox_handler_nix_lf_alloc(pf, (struct nix_lf_alloc_rsp *)msg);
824 break;
825 case MBOX_MSG_NIX_BP_ENABLE:
826 mbox_handler_nix_bp_enable(pf, (struct nix_bp_cfg_rsp *)msg);
827 break;
828 case MBOX_MSG_CGX_STATS:
829 mbox_handler_cgx_stats(pf, (struct cgx_stats_rsp *)msg);
830 break;
831 case MBOX_MSG_CGX_FEC_STATS:
832 mbox_handler_cgx_fec_stats(pf, (struct cgx_fec_stats_rsp *)msg);
833 break;
834 default:
835 if (msg->rc)
836 dev_err(pf->dev,
837 "Mbox msg response has err %d, ID 0x%x\n",
838 msg->rc, msg->id);
839 break;
840 }
841 }
842
otx2_pfaf_mbox_handler(struct work_struct * work)843 static void otx2_pfaf_mbox_handler(struct work_struct *work)
844 {
845 struct otx2_mbox_dev *mdev;
846 struct mbox_hdr *rsp_hdr;
847 struct mbox_msghdr *msg;
848 struct otx2_mbox *mbox;
849 struct mbox *af_mbox;
850 struct otx2_nic *pf;
851 int offset, id;
852 u16 num_msgs;
853
854 af_mbox = container_of(work, struct mbox, mbox_wrk);
855 mbox = &af_mbox->mbox;
856 mdev = &mbox->dev[0];
857 rsp_hdr = (struct mbox_hdr *)(mdev->mbase + mbox->rx_start);
858 num_msgs = rsp_hdr->num_msgs;
859
860 offset = mbox->rx_start + ALIGN(sizeof(*rsp_hdr), MBOX_MSG_ALIGN);
861 pf = af_mbox->pfvf;
862
863 trace_otx2_msg_status(pf->pdev, "PF-AF down queue handler(response)",
864 num_msgs);
865
866 for (id = 0; id < num_msgs; id++) {
867 msg = (struct mbox_msghdr *)(mdev->mbase + offset);
868 otx2_process_pfaf_mbox_msg(pf, msg);
869 offset = mbox->rx_start + msg->next_msgoff;
870 if (mdev->msgs_acked == (num_msgs - 1))
871 __otx2_mbox_reset(mbox, 0);
872 mdev->msgs_acked++;
873 }
874
875 }
876
otx2_handle_link_event(struct otx2_nic * pf)877 static void otx2_handle_link_event(struct otx2_nic *pf)
878 {
879 struct cgx_link_user_info *linfo = &pf->linfo;
880 struct net_device *netdev = pf->netdev;
881
882 if (pf->flags & OTX2_FLAG_PORT_UP)
883 return;
884
885 pr_info("%s NIC Link is %s %d Mbps %s duplex\n", netdev->name,
886 linfo->link_up ? "UP" : "DOWN", linfo->speed,
887 linfo->full_duplex ? "Full" : "Half");
888 if (linfo->link_up) {
889 netif_carrier_on(netdev);
890 netif_tx_start_all_queues(netdev);
891 } else {
892 netif_carrier_off(netdev);
893 netif_tx_stop_all_queues(netdev);
894 }
895 }
896
otx2_mbox_up_handler_rep_event_up_notify(struct otx2_nic * pf,struct rep_event * info,struct msg_rsp * rsp)897 static int otx2_mbox_up_handler_rep_event_up_notify(struct otx2_nic *pf,
898 struct rep_event *info,
899 struct msg_rsp *rsp)
900 {
901 struct net_device *netdev = pf->netdev;
902
903 if (info->event == RVU_EVENT_MTU_CHANGE) {
904 netdev->mtu = info->evt_data.mtu;
905 return 0;
906 }
907
908 if (info->event == RVU_EVENT_PORT_STATE) {
909 if (info->evt_data.port_state) {
910 pf->flags |= OTX2_FLAG_PORT_UP;
911 netif_carrier_on(netdev);
912 netif_tx_start_all_queues(netdev);
913 } else {
914 pf->flags &= ~OTX2_FLAG_PORT_UP;
915 netif_tx_stop_all_queues(netdev);
916 netif_carrier_off(netdev);
917 }
918 return 0;
919 }
920 #ifdef CONFIG_RVU_ESWITCH
921 rvu_event_up_notify(pf, info);
922 #endif
923 return 0;
924 }
925
otx2_mbox_up_handler_mcs_intr_notify(struct otx2_nic * pf,struct mcs_intr_info * event,struct msg_rsp * rsp)926 int otx2_mbox_up_handler_mcs_intr_notify(struct otx2_nic *pf,
927 struct mcs_intr_info *event,
928 struct msg_rsp *rsp)
929 {
930 cn10k_handle_mcs_event(pf, event);
931
932 return 0;
933 }
934
otx2_mbox_up_handler_cgx_link_event(struct otx2_nic * pf,struct cgx_link_info_msg * msg,struct msg_rsp * rsp)935 int otx2_mbox_up_handler_cgx_link_event(struct otx2_nic *pf,
936 struct cgx_link_info_msg *msg,
937 struct msg_rsp *rsp)
938 {
939 int i;
940
941 /* Copy the link info sent by AF */
942 pf->linfo = msg->link_info;
943
944 /* notify VFs about link event */
945 for (i = 0; i < pci_num_vf(pf->pdev); i++) {
946 struct otx2_vf_config *config = &pf->vf_configs[i];
947 struct delayed_work *dwork = &config->link_event_work;
948
949 if (config->intf_down)
950 continue;
951
952 schedule_delayed_work(dwork, msecs_to_jiffies(100));
953 }
954
955 /* interface has not been fully configured yet */
956 if (pf->flags & OTX2_FLAG_INTF_DOWN)
957 return 0;
958
959 otx2_handle_link_event(pf);
960 return 0;
961 }
962
otx2_process_mbox_msg_up(struct otx2_nic * pf,struct mbox_msghdr * req)963 static int otx2_process_mbox_msg_up(struct otx2_nic *pf,
964 struct mbox_msghdr *req)
965 {
966 /* Check if valid, if not reply with a invalid msg */
967 if (req->sig != OTX2_MBOX_REQ_SIG) {
968 otx2_reply_invalid_msg(&pf->mbox.mbox_up, 0, 0, req->id);
969 return -ENODEV;
970 }
971
972 switch (req->id) {
973 #define M(_name, _id, _fn_name, _req_type, _rsp_type) \
974 case _id: { \
975 struct _rsp_type *rsp; \
976 int err; \
977 \
978 rsp = (struct _rsp_type *)otx2_mbox_alloc_msg( \
979 &pf->mbox.mbox_up, 0, \
980 sizeof(struct _rsp_type)); \
981 if (!rsp) \
982 return -ENOMEM; \
983 \
984 rsp->hdr.id = _id; \
985 rsp->hdr.sig = OTX2_MBOX_RSP_SIG; \
986 rsp->hdr.pcifunc = 0; \
987 rsp->hdr.rc = 0; \
988 \
989 err = otx2_mbox_up_handler_ ## _fn_name( \
990 pf, (struct _req_type *)req, rsp); \
991 return err; \
992 }
993 MBOX_UP_CGX_MESSAGES
994 MBOX_UP_MCS_MESSAGES
995 MBOX_UP_REP_MESSAGES
996 #undef M
997 break;
998 default:
999 otx2_reply_invalid_msg(&pf->mbox.mbox_up, 0, 0, req->id);
1000 return -ENODEV;
1001 }
1002 return 0;
1003 }
1004
otx2_pfaf_mbox_up_handler(struct work_struct * work)1005 static void otx2_pfaf_mbox_up_handler(struct work_struct *work)
1006 {
1007 struct mbox *af_mbox = container_of(work, struct mbox, mbox_up_wrk);
1008 struct otx2_mbox *mbox = &af_mbox->mbox_up;
1009 struct otx2_mbox_dev *mdev = &mbox->dev[0];
1010 struct otx2_nic *pf = af_mbox->pfvf;
1011 int offset, id, devid = 0;
1012 struct mbox_hdr *rsp_hdr;
1013 struct mbox_msghdr *msg;
1014 u16 num_msgs;
1015
1016 rsp_hdr = (struct mbox_hdr *)(mdev->mbase + mbox->rx_start);
1017 num_msgs = rsp_hdr->num_msgs;
1018
1019 offset = mbox->rx_start + ALIGN(sizeof(*rsp_hdr), MBOX_MSG_ALIGN);
1020
1021 trace_otx2_msg_status(pf->pdev, "PF-AF up queue handler(notification)",
1022 num_msgs);
1023
1024 for (id = 0; id < num_msgs; id++) {
1025 msg = (struct mbox_msghdr *)(mdev->mbase + offset);
1026
1027 devid = msg->pcifunc & RVU_PFVF_FUNC_MASK;
1028 /* Skip processing VF's messages */
1029 if (!devid)
1030 otx2_process_mbox_msg_up(pf, msg);
1031 offset = mbox->rx_start + msg->next_msgoff;
1032 }
1033 /* Forward to VF iff VFs are really present */
1034 if (devid && pci_num_vf(pf->pdev)) {
1035 otx2_forward_vf_mbox_msgs(pf, &pf->mbox.mbox_up,
1036 MBOX_DIR_PFVF_UP, devid - 1,
1037 num_msgs);
1038 return;
1039 }
1040
1041 otx2_mbox_msg_send(mbox, 0);
1042 }
1043
otx2_pfaf_mbox_intr_handler(int irq,void * pf_irq)1044 irqreturn_t otx2_pfaf_mbox_intr_handler(int irq, void *pf_irq)
1045 {
1046 struct otx2_nic *pf = (struct otx2_nic *)pf_irq;
1047 struct mbox *mw = &pf->mbox;
1048 struct otx2_mbox_dev *mdev;
1049 struct otx2_mbox *mbox;
1050 struct mbox_hdr *hdr;
1051 u64 mbox_data;
1052
1053 /* Clear the IRQ */
1054 otx2_write64(pf, RVU_PF_INT, BIT_ULL(0));
1055
1056 mbox_data = otx2_read64(pf, RVU_PF_PFAF_MBOX0);
1057
1058 if (mbox_data & MBOX_UP_MSG) {
1059 mbox_data &= ~MBOX_UP_MSG;
1060 otx2_write64(pf, RVU_PF_PFAF_MBOX0, mbox_data);
1061
1062 mbox = &mw->mbox_up;
1063 mdev = &mbox->dev[0];
1064 otx2_sync_mbox_bbuf(mbox, 0);
1065
1066 hdr = (struct mbox_hdr *)(mdev->mbase + mbox->rx_start);
1067 if (hdr->num_msgs)
1068 queue_work(pf->mbox_wq, &mw->mbox_up_wrk);
1069
1070 trace_otx2_msg_interrupt(pf->pdev, "UP message from AF to PF",
1071 BIT_ULL(0));
1072
1073 trace_otx2_msg_status(pf->pdev, "PF-AF up work queued(interrupt)",
1074 hdr->num_msgs);
1075 }
1076
1077 if (mbox_data & MBOX_DOWN_MSG) {
1078 mbox_data &= ~MBOX_DOWN_MSG;
1079 otx2_write64(pf, RVU_PF_PFAF_MBOX0, mbox_data);
1080
1081 mbox = &mw->mbox;
1082 mdev = &mbox->dev[0];
1083 otx2_sync_mbox_bbuf(mbox, 0);
1084
1085 hdr = (struct mbox_hdr *)(mdev->mbase + mbox->rx_start);
1086 if (hdr->num_msgs)
1087 queue_work(pf->mbox_wq, &mw->mbox_wrk);
1088
1089 trace_otx2_msg_interrupt(pf->pdev, "DOWN reply from AF to PF",
1090 BIT_ULL(0));
1091
1092 trace_otx2_msg_status(pf->pdev, "PF-AF down work queued(interrupt)",
1093 hdr->num_msgs);
1094 }
1095
1096 return IRQ_HANDLED;
1097 }
1098
otx2_disable_mbox_intr(struct otx2_nic * pf)1099 void otx2_disable_mbox_intr(struct otx2_nic *pf)
1100 {
1101 int vector;
1102
1103 /* Disable AF => PF mailbox IRQ */
1104 if (!is_cn20k(pf->pdev)) {
1105 vector = pci_irq_vector(pf->pdev, RVU_PF_INT_VEC_AFPF_MBOX);
1106 otx2_write64(pf, RVU_PF_INT_ENA_W1C, BIT_ULL(0));
1107 } else {
1108 vector = pci_irq_vector(pf->pdev,
1109 RVU_MBOX_PF_INT_VEC_AFPF_MBOX);
1110 otx2_write64(pf, RVU_PF_INT_ENA_W1C,
1111 BIT_ULL(0) | BIT_ULL(1));
1112 }
1113 free_irq(vector, pf);
1114 }
1115 EXPORT_SYMBOL(otx2_disable_mbox_intr);
1116
otx2_register_mbox_intr(struct otx2_nic * pf,bool probe_af)1117 int otx2_register_mbox_intr(struct otx2_nic *pf, bool probe_af)
1118 {
1119 struct otx2_hw *hw = &pf->hw;
1120 struct msg_req *req;
1121 u64 mbox_int_mask;
1122 char *irq_name;
1123 int err;
1124
1125 mbox_int_mask = !is_cn20k(pf->pdev) ? BIT_ULL(0) :
1126 BIT_ULL(0) | BIT_ULL(1);
1127
1128 /* Clear stale mailbox interrupt state before installing the handler. */
1129 otx2_write64(pf, RVU_PF_INT, mbox_int_mask);
1130
1131 /* Register mailbox interrupt handler */
1132 if (!is_cn20k(pf->pdev)) {
1133 irq_name = &hw->irq_name[RVU_PF_INT_VEC_AFPF_MBOX * NAME_SIZE];
1134 snprintf(irq_name, NAME_SIZE, "RVUPF%d AFPF Mbox",
1135 rvu_get_pf(pf->pdev, pf->pcifunc));
1136 err = request_irq(pci_irq_vector
1137 (pf->pdev, RVU_PF_INT_VEC_AFPF_MBOX),
1138 pf->hw_ops->pfaf_mbox_intr_handler,
1139 0, irq_name, pf);
1140 } else {
1141 irq_name = &hw->irq_name[RVU_MBOX_PF_INT_VEC_AFPF_MBOX *
1142 NAME_SIZE];
1143 snprintf(irq_name, NAME_SIZE, "RVUPF%d AFPF Mbox",
1144 rvu_get_pf(pf->pdev, pf->pcifunc));
1145 err = request_irq(pci_irq_vector
1146 (pf->pdev, RVU_MBOX_PF_INT_VEC_AFPF_MBOX),
1147 pf->hw_ops->pfaf_mbox_intr_handler,
1148 0, irq_name, pf);
1149 }
1150 if (err) {
1151 dev_err(pf->dev,
1152 "RVUPF: IRQ registration failed for PFAF mbox irq\n");
1153 return err;
1154 }
1155
1156 /* Enable mailbox interrupt for msgs coming from AF. */
1157 otx2_write64(pf, RVU_PF_INT_ENA_W1S, mbox_int_mask);
1158
1159 if (!probe_af)
1160 return 0;
1161
1162 /* Check mailbox communication with AF */
1163 req = otx2_mbox_alloc_msg_ready(&pf->mbox);
1164 if (!req) {
1165 otx2_disable_mbox_intr(pf);
1166 return -ENOMEM;
1167 }
1168 err = otx2_sync_mbox_msg(&pf->mbox);
1169 if (err) {
1170 dev_warn(pf->dev,
1171 "AF not responding to mailbox, deferring probe\n");
1172 otx2_disable_mbox_intr(pf);
1173 return -EPROBE_DEFER;
1174 }
1175
1176 return 0;
1177 }
1178
otx2_pfaf_mbox_destroy(struct otx2_nic * pf)1179 void otx2_pfaf_mbox_destroy(struct otx2_nic *pf)
1180 {
1181 struct mbox *mbox = &pf->mbox;
1182
1183 if (pf->mbox_wq) {
1184 destroy_workqueue(pf->mbox_wq);
1185 pf->mbox_wq = NULL;
1186 }
1187
1188 if (mbox->mbox.hwbase && !is_cn20k(pf->pdev))
1189 iounmap((void __iomem *)mbox->mbox.hwbase);
1190
1191 otx2_mbox_destroy(&mbox->mbox);
1192 otx2_mbox_destroy(&mbox->mbox_up);
1193 }
1194 EXPORT_SYMBOL(otx2_pfaf_mbox_destroy);
1195
otx2_pfaf_mbox_init(struct otx2_nic * pf)1196 int otx2_pfaf_mbox_init(struct otx2_nic *pf)
1197 {
1198 struct mbox *mbox = &pf->mbox;
1199 void __iomem *hwbase;
1200 int err;
1201
1202 mbox->pfvf = pf;
1203 pf->mbox_wq = alloc_ordered_workqueue("otx2_pfaf_mailbox",
1204 WQ_HIGHPRI | WQ_MEM_RECLAIM);
1205 if (!pf->mbox_wq)
1206 return -ENOMEM;
1207
1208 /* For CN20K, AF allocates mbox memory in DRAM and writes PF
1209 * regions/offsets in RVU_MBOX_AF_PFX_ADDR, the RVU_PFX_FUNC_PFAF_MBOX
1210 * gives the aliased address to access AF/PF mailbox regions.
1211 */
1212 if (is_cn20k(pf->pdev))
1213 hwbase = pf->reg_base + RVU_PFX_FUNC_PFAF_MBOX +
1214 ((u64)BLKADDR_MBOX << RVU_FUNC_BLKADDR_SHIFT);
1215 else
1216 /* Mailbox is a reserved memory (in RAM) region shared between
1217 * admin function (i.e AF) and this PF, shouldn't be mapped as
1218 * device memory to allow unaligned accesses.
1219 */
1220 hwbase = ioremap_wc(pci_resource_start
1221 (pf->pdev, PCI_MBOX_BAR_NUM), MBOX_SIZE);
1222 if (!hwbase) {
1223 dev_err(pf->dev, "Unable to map PFAF mailbox region\n");
1224 err = -ENOMEM;
1225 goto exit;
1226 }
1227
1228 err = otx2_mbox_init(&mbox->mbox, hwbase, pf->pdev, pf->reg_base,
1229 MBOX_DIR_PFAF, 1);
1230 if (err)
1231 goto exit;
1232
1233 err = otx2_mbox_init(&mbox->mbox_up, hwbase, pf->pdev, pf->reg_base,
1234 MBOX_DIR_PFAF_UP, 1);
1235 if (err)
1236 goto exit;
1237
1238 err = otx2_mbox_bbuf_init(mbox, pf->pdev);
1239 if (err)
1240 goto exit;
1241
1242 INIT_WORK(&mbox->mbox_wrk, otx2_pfaf_mbox_handler);
1243 INIT_WORK(&mbox->mbox_up_wrk, otx2_pfaf_mbox_up_handler);
1244 mutex_init(&mbox->lock);
1245
1246 return 0;
1247 exit:
1248 otx2_pfaf_mbox_destroy(pf);
1249 return err;
1250 }
1251
otx2_cgx_config_linkevents(struct otx2_nic * pf,bool enable)1252 static int otx2_cgx_config_linkevents(struct otx2_nic *pf, bool enable)
1253 {
1254 struct msg_req *msg;
1255 int err;
1256
1257 mutex_lock(&pf->mbox.lock);
1258 if (enable)
1259 msg = otx2_mbox_alloc_msg_cgx_start_linkevents(&pf->mbox);
1260 else
1261 msg = otx2_mbox_alloc_msg_cgx_stop_linkevents(&pf->mbox);
1262
1263 if (!msg) {
1264 mutex_unlock(&pf->mbox.lock);
1265 return -ENOMEM;
1266 }
1267
1268 err = otx2_sync_mbox_msg(&pf->mbox);
1269 mutex_unlock(&pf->mbox.lock);
1270 return err;
1271 }
1272
otx2_reset_mac_stats(struct otx2_nic * pfvf)1273 int otx2_reset_mac_stats(struct otx2_nic *pfvf)
1274 {
1275 struct msg_req *req;
1276 int err;
1277
1278 mutex_lock(&pfvf->mbox.lock);
1279 req = otx2_mbox_alloc_msg_cgx_stats_rst(&pfvf->mbox);
1280 if (!req) {
1281 mutex_unlock(&pfvf->mbox.lock);
1282 return -ENOMEM;
1283 }
1284
1285 err = otx2_sync_mbox_msg(&pfvf->mbox);
1286 mutex_unlock(&pfvf->mbox.lock);
1287 return err;
1288 }
1289
otx2_cgx_config_loopback(struct otx2_nic * pf,bool enable)1290 static int otx2_cgx_config_loopback(struct otx2_nic *pf, bool enable)
1291 {
1292 struct msg_req *msg;
1293 int err;
1294
1295 if (enable && !bitmap_empty(pf->flow_cfg->dmacflt_bmap,
1296 pf->flow_cfg->dmacflt_max_flows))
1297 netdev_warn(pf->netdev,
1298 "CGX/RPM internal loopback might not work as DMAC filters are active\n");
1299
1300 mutex_lock(&pf->mbox.lock);
1301 if (enable)
1302 msg = otx2_mbox_alloc_msg_cgx_intlbk_enable(&pf->mbox);
1303 else
1304 msg = otx2_mbox_alloc_msg_cgx_intlbk_disable(&pf->mbox);
1305
1306 if (!msg) {
1307 mutex_unlock(&pf->mbox.lock);
1308 return -ENOMEM;
1309 }
1310
1311 err = otx2_sync_mbox_msg(&pf->mbox);
1312 mutex_unlock(&pf->mbox.lock);
1313 return err;
1314 }
1315
otx2_set_real_num_queues(struct net_device * netdev,int tx_queues,int rx_queues)1316 int otx2_set_real_num_queues(struct net_device *netdev,
1317 int tx_queues, int rx_queues)
1318 {
1319 int err;
1320
1321 err = netif_set_real_num_tx_queues(netdev, tx_queues);
1322 if (err) {
1323 netdev_err(netdev,
1324 "Failed to set no of Tx queues: %d\n", tx_queues);
1325 return err;
1326 }
1327
1328 err = netif_set_real_num_rx_queues(netdev, rx_queues);
1329 if (err)
1330 netdev_err(netdev,
1331 "Failed to set no of Rx queues: %d\n", rx_queues);
1332 return err;
1333 }
1334 EXPORT_SYMBOL(otx2_set_real_num_queues);
1335
1336 static char *nix_sqoperr_e_str[NIX_SQOPERR_MAX] = {
1337 "NIX_SQOPERR_OOR",
1338 "NIX_SQOPERR_CTX_FAULT",
1339 "NIX_SQOPERR_CTX_POISON",
1340 "NIX_SQOPERR_DISABLED",
1341 "NIX_SQOPERR_SIZE_ERR",
1342 "NIX_SQOPERR_OFLOW",
1343 "NIX_SQOPERR_SQB_NULL",
1344 "NIX_SQOPERR_SQB_FAULT",
1345 "NIX_SQOPERR_SQE_SZ_ZERO",
1346 };
1347
1348 static char *nix_mnqerr_e_str[NIX_MNQERR_MAX] = {
1349 "NIX_MNQERR_SQ_CTX_FAULT",
1350 "NIX_MNQERR_SQ_CTX_POISON",
1351 "NIX_MNQERR_SQB_FAULT",
1352 "NIX_MNQERR_SQB_POISON",
1353 "NIX_MNQERR_TOTAL_ERR",
1354 "NIX_MNQERR_LSO_ERR",
1355 "NIX_MNQERR_CQ_QUERY_ERR",
1356 "NIX_MNQERR_MAX_SQE_SIZE_ERR",
1357 "NIX_MNQERR_MAXLEN_ERR",
1358 "NIX_MNQERR_SQE_SIZEM1_ZERO",
1359 };
1360
1361 static char *nix_snd_status_e_str[NIX_SND_STATUS_MAX] = {
1362 [NIX_SND_STATUS_GOOD] = "NIX_SND_STATUS_GOOD",
1363 [NIX_SND_STATUS_SQ_CTX_FAULT] = "NIX_SND_STATUS_SQ_CTX_FAULT",
1364 [NIX_SND_STATUS_SQ_CTX_POISON] = "NIX_SND_STATUS_SQ_CTX_POISON",
1365 [NIX_SND_STATUS_SQB_FAULT] = "NIX_SND_STATUS_SQB_FAULT",
1366 [NIX_SND_STATUS_SQB_POISON] = "NIX_SND_STATUS_SQB_POISON",
1367 [NIX_SND_STATUS_HDR_ERR] = "NIX_SND_STATUS_HDR_ERR",
1368 [NIX_SND_STATUS_EXT_ERR] = "NIX_SND_STATUS_EXT_ERR",
1369 [NIX_SND_STATUS_JUMP_FAULT] = "NIX_SND_STATUS_JUMP_FAULT",
1370 [NIX_SND_STATUS_JUMP_POISON] = "NIX_SND_STATUS_JUMP_POISON",
1371 [NIX_SND_STATUS_CRC_ERR] = "NIX_SND_STATUS_CRC_ERR",
1372 [NIX_SND_STATUS_IMM_ERR] = "NIX_SND_STATUS_IMM_ERR",
1373 [NIX_SND_STATUS_SG_ERR] = "NIX_SND_STATUS_SG_ERR",
1374 [NIX_SND_STATUS_MEM_ERR] = "NIX_SND_STATUS_MEM_ERR",
1375 [NIX_SND_STATUS_INVALID_SUBDC] = "NIX_SND_STATUS_INVALID_SUBDC",
1376 [NIX_SND_STATUS_SUBDC_ORDER_ERR] = "NIX_SND_STATUS_SUBDC_ORDER_ERR",
1377 [NIX_SND_STATUS_DATA_FAULT] = "NIX_SND_STATUS_DATA_FAULT",
1378 [NIX_SND_STATUS_DATA_POISON] = "NIX_SND_STATUS_DATA_POISON",
1379 [NIX_SND_STATUS_NPC_DROP_ACTION] = "NIX_SND_STATUS_NPC_DROP_ACTION",
1380 [NIX_SND_STATUS_LOCK_VIOL] = "NIX_SND_STATUS_LOCK_VIOL",
1381 [NIX_SND_STATUS_NPC_UCAST_CHAN_ERR] = "NIX_SND_STAT_NPC_UCAST_CHAN_ERR",
1382 [NIX_SND_STATUS_NPC_MCAST_CHAN_ERR] = "NIX_SND_STAT_NPC_MCAST_CHAN_ERR",
1383 [NIX_SND_STATUS_NPC_MCAST_ABORT] = "NIX_SND_STATUS_NPC_MCAST_ABORT",
1384 [NIX_SND_STATUS_NPC_VTAG_PTR_ERR] = "NIX_SND_STATUS_NPC_VTAG_PTR_ERR",
1385 [NIX_SND_STATUS_NPC_VTAG_SIZE_ERR] = "NIX_SND_STATUS_NPC_VTAG_SIZE_ERR",
1386 [NIX_SND_STATUS_SEND_MEM_FAULT] = "NIX_SND_STATUS_SEND_MEM_FAULT",
1387 [NIX_SND_STATUS_SEND_STATS_ERR] = "NIX_SND_STATUS_SEND_STATS_ERR",
1388 };
1389
otx2_q_intr_handler(int irq,void * data)1390 static irqreturn_t otx2_q_intr_handler(int irq, void *data)
1391 {
1392 struct otx2_nic *pf = data;
1393 struct otx2_snd_queue *sq;
1394 void __iomem *ptr;
1395 u64 val, qidx = 0;
1396
1397 /* CQ */
1398 for (qidx = 0; qidx < pf->qset.cq_cnt; qidx++) {
1399 ptr = otx2_get_regaddr(pf, NIX_LF_CQ_OP_INT);
1400 val = otx2_atomic64_add((qidx << 44), ptr);
1401
1402 otx2_write64(pf, NIX_LF_CQ_OP_INT, (qidx << 44) |
1403 (val & NIX_CQERRINT_BITS));
1404 if (!(val & (NIX_CQERRINT_BITS | BIT_ULL(42))))
1405 continue;
1406
1407 if (val & BIT_ULL(42)) {
1408 netdev_err(pf->netdev,
1409 "CQ%lld: error reading NIX_LF_CQ_OP_INT, NIX_LF_ERR_INT 0x%llx\n",
1410 qidx, otx2_read64(pf, NIX_LF_ERR_INT));
1411 } else {
1412 if (val & BIT_ULL(NIX_CQERRINT_DOOR_ERR))
1413 netdev_err(pf->netdev, "CQ%lld: Doorbell error",
1414 qidx);
1415 if (val & BIT_ULL(NIX_CQERRINT_CQE_FAULT))
1416 netdev_err(pf->netdev,
1417 "CQ%lld: Memory fault on CQE write to LLC/DRAM",
1418 qidx);
1419 }
1420
1421 schedule_work(&pf->reset_task);
1422 }
1423
1424 /* SQ */
1425 for (qidx = 0; qidx < otx2_get_total_tx_queues(pf); qidx++) {
1426 u64 sq_op_err_dbg, mnq_err_dbg, snd_err_dbg;
1427 u8 sq_op_err_code, mnq_err_code, snd_err_code;
1428
1429 sq = &pf->qset.sq[qidx];
1430 if (!sq->sqb_ptrs)
1431 continue;
1432
1433 /* Below debug registers captures first errors corresponding to
1434 * those registers. We don't have to check against SQ qid as
1435 * these are fatal errors.
1436 */
1437
1438 ptr = otx2_get_regaddr(pf, NIX_LF_SQ_OP_INT);
1439 val = otx2_atomic64_add((qidx << 44), ptr);
1440 otx2_write64(pf, NIX_LF_SQ_OP_INT, (qidx << 44) |
1441 (val & NIX_SQINT_BITS));
1442
1443 if (val & BIT_ULL(42)) {
1444 netdev_err(pf->netdev,
1445 "SQ%lld: error reading NIX_LF_SQ_OP_INT, NIX_LF_ERR_INT 0x%llx\n",
1446 qidx, otx2_read64(pf, NIX_LF_ERR_INT));
1447 goto done;
1448 }
1449
1450 sq_op_err_dbg = otx2_read64(pf, NIX_LF_SQ_OP_ERR_DBG);
1451 if (!(sq_op_err_dbg & BIT(44)))
1452 goto chk_mnq_err_dbg;
1453
1454 sq_op_err_code = FIELD_GET(GENMASK(7, 0), sq_op_err_dbg);
1455 netdev_err(pf->netdev,
1456 "SQ%lld: NIX_LF_SQ_OP_ERR_DBG(0x%llx) err=%s(%#x)\n",
1457 qidx, sq_op_err_dbg,
1458 nix_sqoperr_e_str[sq_op_err_code],
1459 sq_op_err_code);
1460
1461 otx2_write64(pf, NIX_LF_SQ_OP_ERR_DBG, BIT_ULL(44));
1462
1463 if (sq_op_err_code == NIX_SQOPERR_SQB_NULL)
1464 goto chk_mnq_err_dbg;
1465
1466 /* Err is not NIX_SQOPERR_SQB_NULL, call aq function to read SQ structure.
1467 * TODO: But we are in irq context. How to call mbox functions which does sleep
1468 */
1469
1470 chk_mnq_err_dbg:
1471 mnq_err_dbg = otx2_read64(pf, NIX_LF_MNQ_ERR_DBG);
1472 if (!(mnq_err_dbg & BIT(44)))
1473 goto chk_snd_err_dbg;
1474
1475 mnq_err_code = FIELD_GET(GENMASK(7, 0), mnq_err_dbg);
1476 netdev_err(pf->netdev,
1477 "SQ%lld: NIX_LF_MNQ_ERR_DBG(0x%llx) err=%s(%#x)\n",
1478 qidx, mnq_err_dbg, nix_mnqerr_e_str[mnq_err_code],
1479 mnq_err_code);
1480 otx2_write64(pf, NIX_LF_MNQ_ERR_DBG, BIT_ULL(44));
1481
1482 chk_snd_err_dbg:
1483 snd_err_dbg = otx2_read64(pf, NIX_LF_SEND_ERR_DBG);
1484 if (snd_err_dbg & BIT(44)) {
1485 snd_err_code = FIELD_GET(GENMASK(7, 0), snd_err_dbg);
1486 netdev_err(pf->netdev,
1487 "SQ%lld: NIX_LF_SND_ERR_DBG:0x%llx err=%s(%#x)\n",
1488 qidx, snd_err_dbg,
1489 nix_snd_status_e_str[snd_err_code],
1490 snd_err_code);
1491 otx2_write64(pf, NIX_LF_SEND_ERR_DBG, BIT_ULL(44));
1492 }
1493
1494 done:
1495 /* Print values and reset */
1496 if (val & BIT_ULL(NIX_SQINT_SQB_ALLOC_FAIL))
1497 netdev_err(pf->netdev, "SQ%lld: SQB allocation failed",
1498 qidx);
1499
1500 schedule_work(&pf->reset_task);
1501 }
1502
1503 return IRQ_HANDLED;
1504 }
1505
otx2_cq_intr_handler(int irq,void * cq_irq)1506 irqreturn_t otx2_cq_intr_handler(int irq, void *cq_irq)
1507 {
1508 struct otx2_cq_poll *cq_poll = (struct otx2_cq_poll *)cq_irq;
1509 struct otx2_nic *pf = (struct otx2_nic *)cq_poll->dev;
1510 int qidx = cq_poll->cint_idx;
1511
1512 /* Disable interrupts.
1513 *
1514 * Completion interrupts behave in a level-triggered interrupt
1515 * fashion, and hence have to be cleared only after it is serviced.
1516 */
1517 otx2_write64(pf, NIX_LF_CINTX_ENA_W1C(qidx), BIT_ULL(0));
1518
1519 /* Schedule NAPI */
1520 pf->napi_events++;
1521 napi_schedule_irqoff(&cq_poll->napi);
1522
1523 return IRQ_HANDLED;
1524 }
1525 EXPORT_SYMBOL(otx2_cq_intr_handler);
1526
otx2_disable_napi(struct otx2_nic * pf)1527 void otx2_disable_napi(struct otx2_nic *pf)
1528 {
1529 struct otx2_qset *qset = &pf->qset;
1530 struct otx2_cq_poll *cq_poll;
1531 struct work_struct *work;
1532 int qidx;
1533
1534 for (qidx = 0; qidx < pf->hw.cint_cnt; qidx++) {
1535 cq_poll = &qset->napi[qidx];
1536 work = &cq_poll->dim.work;
1537 if (work->func)
1538 cancel_work_sync(work);
1539 napi_disable(&cq_poll->napi);
1540 netif_napi_del(&cq_poll->napi);
1541 }
1542 }
1543 EXPORT_SYMBOL(otx2_disable_napi);
1544
otx2_free_cq_res(struct otx2_nic * pf)1545 static void otx2_free_cq_res(struct otx2_nic *pf)
1546 {
1547 struct otx2_qset *qset = &pf->qset;
1548 struct otx2_cq_queue *cq;
1549 int qidx;
1550
1551 /* Disable CQs */
1552 otx2_ctx_disable(&pf->mbox, NIX_AQ_CTYPE_CQ, false);
1553 for (qidx = 0; qidx < qset->cq_cnt; qidx++) {
1554 cq = &qset->cq[qidx];
1555 qmem_free(pf->dev, cq->cqe);
1556 }
1557 }
1558
otx2_free_sq_res(struct otx2_nic * pf)1559 static void otx2_free_sq_res(struct otx2_nic *pf)
1560 {
1561 struct otx2_qset *qset = &pf->qset;
1562 struct otx2_snd_queue *sq;
1563 int qidx;
1564
1565 /* Disable SQs */
1566 otx2_ctx_disable(&pf->mbox, NIX_AQ_CTYPE_SQ, false);
1567 /* Free SQB pointers */
1568 otx2_sq_free_sqbs(pf);
1569 for (qidx = 0; qidx < otx2_get_total_tx_queues(pf); qidx++) {
1570 sq = &qset->sq[qidx];
1571 /* sq->sqe is not initialized for unused QoS queues */
1572 if (sq->sqe) {
1573 qmem_free(pf->dev, sq->sqe);
1574 qmem_free(pf->dev, sq->sqe_ring);
1575 qmem_free(pf->dev, sq->cpt_resp);
1576 qmem_free(pf->dev, sq->tso_hdrs);
1577 qmem_free(pf->dev, sq->timestamps);
1578 kfree(sq->sg);
1579 }
1580 kfree(sq->sqb_ptrs);
1581 }
1582 }
1583
otx2_get_rbuf_size(struct otx2_nic * pf,int mtu)1584 static int otx2_get_rbuf_size(struct otx2_nic *pf, int mtu)
1585 {
1586 int frame_size;
1587 int total_size;
1588 int rbuf_size;
1589
1590 if (pf->hw.rbuf_len)
1591 return ALIGN(pf->hw.rbuf_len, OTX2_ALIGN) + OTX2_HEAD_ROOM;
1592
1593 /* The data transferred by NIX to memory consists of actual packet
1594 * plus additional data which has timestamp and/or EDSA/HIGIG2
1595 * headers if interface is configured in corresponding modes.
1596 * NIX transfers entire data using 6 segments/buffers and writes
1597 * a CQE_RX descriptor with those segment addresses. First segment
1598 * has additional data prepended to packet. Also software omits a
1599 * headroom of 128 bytes in each segment. Hence the total size of
1600 * memory needed to receive a packet with 'mtu' is:
1601 * frame size = mtu + additional data;
1602 * memory = frame_size + headroom * 6;
1603 * each receive buffer size = memory / 6;
1604 */
1605 frame_size = mtu + OTX2_ETH_HLEN + OTX2_HW_TIMESTAMP_LEN;
1606 total_size = frame_size + OTX2_HEAD_ROOM * 6;
1607 rbuf_size = total_size / 6;
1608
1609 return ALIGN(rbuf_size, 2048);
1610 }
1611
otx2_init_hw_resources(struct otx2_nic * pf)1612 int otx2_init_hw_resources(struct otx2_nic *pf)
1613 {
1614 struct nix_lf_free_req *free_req;
1615 struct mbox *mbox = &pf->mbox;
1616 struct otx2_hw *hw = &pf->hw;
1617 struct msg_req *req;
1618 int err = 0, lvl;
1619
1620 /* Set required NPA LF's pool counts
1621 * Auras and Pools are used in a 1:1 mapping,
1622 * so, aura count = pool count.
1623 */
1624 hw->rqpool_cnt = hw->rx_queues;
1625 hw->sqpool_cnt = otx2_get_total_tx_queues(pf);
1626 hw->pool_cnt = hw->rqpool_cnt + hw->sqpool_cnt;
1627
1628 if (!otx2_rep_dev(pf->pdev)) {
1629 /* Maximum hardware supported transmit length */
1630 pf->tx_max_pktlen = pf->netdev->max_mtu + OTX2_ETH_HLEN;
1631 pf->rbsize = otx2_get_rbuf_size(pf, pf->netdev->mtu);
1632 }
1633
1634 mutex_lock(&mbox->lock);
1635 /* NPA init */
1636 err = otx2_config_npa(pf);
1637 if (err)
1638 goto exit;
1639
1640 /* NIX init */
1641 err = otx2_config_nix(pf);
1642 if (err)
1643 goto err_free_npa_lf;
1644
1645 /* Default disable backpressure on NIX-CPT */
1646 otx2_nix_cpt_config_bp(pf, false);
1647
1648 /* Enable backpressure for CGX mapped PF/VFs */
1649 if (!is_otx2_lbkvf(pf->pdev))
1650 otx2_nix_config_bp(pf, true);
1651
1652 /* Init Auras and pools used by NIX RQ, for free buffer ptrs */
1653 err = otx2_rq_aura_pool_init(pf);
1654 if (err) {
1655 mutex_unlock(&mbox->lock);
1656 goto err_free_nix_lf;
1657 }
1658 /* Init Auras and pools used by NIX SQ, for queueing SQEs */
1659 err = otx2_sq_aura_pool_init(pf);
1660 if (err) {
1661 mutex_unlock(&mbox->lock);
1662 goto err_free_rq_ptrs;
1663 }
1664
1665 err = otx2_txsch_alloc(pf);
1666 if (err) {
1667 mutex_unlock(&mbox->lock);
1668 goto err_free_sq_ptrs;
1669 }
1670
1671 #ifdef CONFIG_DCB
1672 if (pf->pfc_en) {
1673 err = otx2_pfc_txschq_alloc(pf);
1674 if (err) {
1675 mutex_unlock(&mbox->lock);
1676 goto err_free_sq_ptrs;
1677 }
1678 }
1679 #endif
1680
1681 err = otx2_config_nix_queues(pf);
1682 if (err) {
1683 mutex_unlock(&mbox->lock);
1684 goto err_free_txsch;
1685 }
1686
1687 for (lvl = 0; lvl < NIX_TXSCH_LVL_CNT; lvl++) {
1688 int idx;
1689
1690 for (idx = 0; idx < pf->hw.txschq_cnt[lvl]; idx++) {
1691 err = otx2_txschq_config(pf, lvl, idx, false);
1692 if (err) {
1693 dev_err(pf->dev, "Failed to config TXSCH\n");
1694 mutex_unlock(&mbox->lock);
1695 goto err_free_nix_queues;
1696 }
1697 }
1698 }
1699
1700 #ifdef CONFIG_DCB
1701 if (pf->pfc_en) {
1702 err = otx2_pfc_txschq_config(pf);
1703 if (err) {
1704 mutex_unlock(&mbox->lock);
1705 goto err_free_nix_queues;
1706 }
1707 }
1708 #endif
1709
1710 mutex_unlock(&mbox->lock);
1711 return err;
1712
1713 err_free_nix_queues:
1714 otx2_free_cq_res(pf);
1715 otx2_ctx_disable(mbox, NIX_AQ_CTYPE_RQ, false);
1716 err_free_txsch:
1717 otx2_txschq_stop(pf);
1718 err_free_sq_ptrs:
1719 otx2_free_sq_res(pf);
1720 err_free_rq_ptrs:
1721 otx2_free_aura_ptr(pf, AURA_NIX_RQ);
1722 otx2_ctx_disable(mbox, NPA_AQ_CTYPE_POOL, true);
1723 otx2_ctx_disable(mbox, NPA_AQ_CTYPE_AURA, true);
1724 otx2_aura_pool_free(pf);
1725 err_free_nix_lf:
1726 mutex_lock(&mbox->lock);
1727 free_req = otx2_mbox_alloc_msg_nix_lf_free(mbox);
1728 if (free_req) {
1729 free_req->flags = NIX_LF_DISABLE_FLOWS | NIX_LF_DONT_FREE_DFT_IDXS;
1730 if (otx2_sync_mbox_msg(mbox))
1731 dev_err(pf->dev, "%s failed to free nixlf\n", __func__);
1732 }
1733 err_free_npa_lf:
1734 /* Reset NPA LF */
1735 req = otx2_mbox_alloc_msg_npa_lf_free(mbox);
1736 if (req) {
1737 if (otx2_sync_mbox_msg(mbox))
1738 dev_err(pf->dev, "%s failed to free npalf\n", __func__);
1739 }
1740 exit:
1741 mutex_unlock(&mbox->lock);
1742 return err;
1743 }
1744 EXPORT_SYMBOL(otx2_init_hw_resources);
1745
otx2_free_hw_resources(struct otx2_nic * pf)1746 void otx2_free_hw_resources(struct otx2_nic *pf)
1747 {
1748 struct otx2_qset *qset = &pf->qset;
1749 struct nix_lf_free_req *free_req;
1750 struct mbox *mbox = &pf->mbox;
1751 struct otx2_cq_queue *cq;
1752 struct msg_req *req;
1753 int qidx;
1754
1755 /* Ensure all SQE are processed */
1756 otx2_sqb_flush(pf);
1757
1758 /* Stop transmission */
1759 otx2_txschq_stop(pf);
1760
1761 #ifdef CONFIG_DCB
1762 if (pf->pfc_en)
1763 otx2_pfc_txschq_stop(pf);
1764 #endif
1765
1766 if (!otx2_rep_dev(pf->pdev))
1767 otx2_clean_qos_queues(pf);
1768
1769 mutex_lock(&mbox->lock);
1770 /* Disable backpressure */
1771 if (!is_otx2_lbkvf(pf->pdev))
1772 otx2_nix_config_bp(pf, false);
1773 mutex_unlock(&mbox->lock);
1774
1775 /* Disable RQs */
1776 otx2_ctx_disable(mbox, NIX_AQ_CTYPE_RQ, false);
1777
1778 /*Dequeue all CQEs */
1779 for (qidx = 0; qidx < qset->cq_cnt; qidx++) {
1780 cq = &qset->cq[qidx];
1781 if (cq->cq_type == CQ_RX)
1782 otx2_cleanup_rx_cqes(pf, cq, qidx);
1783 else
1784 otx2_cleanup_tx_cqes(pf, cq);
1785 }
1786 otx2_free_pending_sqe(pf);
1787
1788 otx2_free_sq_res(pf);
1789
1790 /* Free RQ buffer pointers*/
1791 otx2_free_aura_ptr(pf, AURA_NIX_RQ);
1792
1793 otx2_free_cq_res(pf);
1794
1795 /* Free all ingress bandwidth profiles allocated */
1796 if (!otx2_rep_dev(pf->pdev))
1797 cn10k_free_all_ipolicers(pf);
1798
1799 mutex_lock(&mbox->lock);
1800 /* Reset NIX LF */
1801 free_req = otx2_mbox_alloc_msg_nix_lf_free(mbox);
1802 if (free_req) {
1803 free_req->flags = NIX_LF_DISABLE_FLOWS | NIX_LF_DONT_FREE_DFT_IDXS;
1804 if (!(pf->flags & OTX2_FLAG_PF_SHUTDOWN))
1805 free_req->flags |= NIX_LF_DONT_FREE_TX_VTAG;
1806 if (otx2_sync_mbox_msg(mbox))
1807 dev_err(pf->dev, "%s failed to free nixlf\n", __func__);
1808 }
1809 mutex_unlock(&mbox->lock);
1810
1811 /* Disable NPA Pool and Aura hw context */
1812 otx2_ctx_disable(mbox, NPA_AQ_CTYPE_POOL, true);
1813 otx2_ctx_disable(mbox, NPA_AQ_CTYPE_AURA, true);
1814 otx2_aura_pool_free(pf);
1815
1816 mutex_lock(&mbox->lock);
1817 /* Reset NPA LF */
1818 req = otx2_mbox_alloc_msg_npa_lf_free(mbox);
1819 if (req) {
1820 if (otx2_sync_mbox_msg(mbox))
1821 dev_err(pf->dev, "%s failed to free npalf\n", __func__);
1822 }
1823 mutex_unlock(&mbox->lock);
1824 }
1825 EXPORT_SYMBOL(otx2_free_hw_resources);
1826
otx2_promisc_use_mce_list(struct otx2_nic * pfvf)1827 static bool otx2_promisc_use_mce_list(struct otx2_nic *pfvf)
1828 {
1829 int vf;
1830
1831 /* The AF driver will determine whether to allow the VF netdev or not */
1832 if (is_otx2_vf(pfvf->pcifunc))
1833 return true;
1834
1835 /* check if there are any trusted VFs associated with the PF netdev */
1836 for (vf = 0; vf < pci_num_vf(pfvf->pdev); vf++)
1837 if (pfvf->vf_configs[vf].trusted)
1838 return true;
1839 return false;
1840 }
1841
otx2_do_set_rx_mode(struct otx2_nic * pf)1842 static void otx2_do_set_rx_mode(struct otx2_nic *pf)
1843 {
1844 struct net_device *netdev = pf->netdev;
1845 struct nix_rx_mode *req;
1846 bool promisc = false;
1847
1848 if (!(netdev->flags & IFF_UP))
1849 return;
1850
1851 if ((netdev->flags & IFF_PROMISC) ||
1852 (netdev_uc_count(netdev) > pf->flow_cfg->ucast_flt_cnt)) {
1853 promisc = true;
1854 }
1855
1856 /* Write unicast address to mcam entries or del from mcam */
1857 if (!promisc && netdev->priv_flags & IFF_UNICAST_FLT)
1858 __dev_uc_sync(netdev, otx2_add_macfilter, otx2_del_macfilter);
1859
1860 mutex_lock(&pf->mbox.lock);
1861 req = otx2_mbox_alloc_msg_nix_set_rx_mode(&pf->mbox);
1862 if (!req) {
1863 mutex_unlock(&pf->mbox.lock);
1864 return;
1865 }
1866
1867 req->mode = NIX_RX_MODE_UCAST;
1868
1869 if (promisc)
1870 req->mode |= NIX_RX_MODE_PROMISC;
1871 if (netdev->flags & (IFF_ALLMULTI | IFF_MULTICAST))
1872 req->mode |= NIX_RX_MODE_ALLMULTI;
1873
1874 if (otx2_promisc_use_mce_list(pf))
1875 req->mode |= NIX_RX_MODE_USE_MCE;
1876
1877 otx2_sync_mbox_msg(&pf->mbox);
1878 mutex_unlock(&pf->mbox.lock);
1879 }
1880
otx2_set_irq_coalesce(struct otx2_nic * pfvf)1881 static void otx2_set_irq_coalesce(struct otx2_nic *pfvf)
1882 {
1883 int cint;
1884
1885 for (cint = 0; cint < pfvf->hw.cint_cnt; cint++)
1886 otx2_config_irq_coalescing(pfvf, cint);
1887 }
1888
otx2_dim_work(struct work_struct * w)1889 static void otx2_dim_work(struct work_struct *w)
1890 {
1891 struct dim_cq_moder cur_moder;
1892 struct otx2_cq_poll *cq_poll;
1893 struct otx2_nic *pfvf;
1894 struct dim *dim;
1895
1896 dim = container_of(w, struct dim, work);
1897 cur_moder = net_dim_get_rx_moderation(dim->mode, dim->profile_ix);
1898 cq_poll = container_of(dim, struct otx2_cq_poll, dim);
1899 pfvf = (struct otx2_nic *)cq_poll->dev;
1900 pfvf->hw.cq_time_wait = (cur_moder.usec > CQ_TIMER_THRESH_MAX) ?
1901 CQ_TIMER_THRESH_MAX : cur_moder.usec;
1902 pfvf->hw.cq_ecount_wait = (cur_moder.pkts > NAPI_POLL_WEIGHT) ?
1903 NAPI_POLL_WEIGHT : cur_moder.pkts;
1904 otx2_set_irq_coalesce(pfvf);
1905 dim->state = DIM_START_MEASURE;
1906 }
1907
otx2_free_queue_mem(struct otx2_qset * qset)1908 void otx2_free_queue_mem(struct otx2_qset *qset)
1909 {
1910 kfree(qset->sq);
1911 qset->sq = NULL;
1912 kfree(qset->cq);
1913 qset->cq = NULL;
1914 kfree(qset->rq);
1915 qset->rq = NULL;
1916 kfree(qset->napi);
1917 qset->napi = NULL;
1918 }
1919 EXPORT_SYMBOL(otx2_free_queue_mem);
1920
otx2_alloc_queue_mem(struct otx2_nic * pf)1921 int otx2_alloc_queue_mem(struct otx2_nic *pf)
1922 {
1923 struct otx2_qset *qset = &pf->qset;
1924 struct otx2_cq_poll *cq_poll;
1925
1926 /* RQ and SQs are mapped to different CQs,
1927 * so find out max CQ IRQs (i.e CINTs) needed.
1928 */
1929 pf->hw.non_qos_queues = pf->hw.tx_queues + pf->hw.xdp_queues;
1930 pf->hw.cint_cnt = max3(pf->hw.rx_queues, pf->hw.tx_queues,
1931 pf->hw.tc_tx_queues);
1932
1933 pf->qset.cq_cnt = pf->hw.rx_queues + otx2_get_total_tx_queues(pf);
1934
1935 qset->napi = kzalloc_objs(*cq_poll, pf->hw.cint_cnt);
1936 if (!qset->napi)
1937 return -ENOMEM;
1938
1939 /* CQ size of RQ */
1940 qset->rqe_cnt = qset->rqe_cnt ? qset->rqe_cnt : Q_COUNT(Q_SIZE_256);
1941 /* CQ size of SQ */
1942 qset->sqe_cnt = qset->sqe_cnt ? qset->sqe_cnt : Q_COUNT(Q_SIZE_4K);
1943
1944 qset->cq = kzalloc_objs(struct otx2_cq_queue, pf->qset.cq_cnt);
1945 if (!qset->cq)
1946 goto err_free_mem;
1947
1948 qset->sq = kzalloc_objs(struct otx2_snd_queue,
1949 otx2_get_total_tx_queues(pf));
1950 if (!qset->sq)
1951 goto err_free_mem;
1952
1953 qset->rq = kzalloc_objs(struct otx2_rcv_queue, pf->hw.rx_queues);
1954 if (!qset->rq)
1955 goto err_free_mem;
1956
1957 return 0;
1958
1959 err_free_mem:
1960 otx2_free_queue_mem(qset);
1961 return -ENOMEM;
1962 }
1963 EXPORT_SYMBOL(otx2_alloc_queue_mem);
1964
otx2_open(struct net_device * netdev)1965 int otx2_open(struct net_device *netdev)
1966 {
1967 struct otx2_nic *pf = netdev_priv(netdev);
1968 struct otx2_cq_poll *cq_poll = NULL;
1969 struct otx2_qset *qset = &pf->qset;
1970 int err = 0, qidx, vec;
1971 char *irq_name;
1972
1973 netif_carrier_off(netdev);
1974
1975 err = otx2_alloc_queue_mem(pf);
1976 if (err)
1977 return err;
1978
1979 err = otx2_init_hw_resources(pf);
1980 if (err)
1981 goto err_free_mem;
1982
1983 /* Register NAPI handler */
1984 for (qidx = 0; qidx < pf->hw.cint_cnt; qidx++) {
1985 cq_poll = &qset->napi[qidx];
1986 cq_poll->cint_idx = qidx;
1987 /* RQ0 & SQ0 are mapped to CINT0 and so on..
1988 * 'cq_ids[0]' points to RQ's CQ and
1989 * 'cq_ids[1]' points to SQ's CQ and
1990 * 'cq_ids[2]' points to XDP's CQ and
1991 */
1992 cq_poll->cq_ids[CQ_RX] =
1993 (qidx < pf->hw.rx_queues) ? qidx : CINT_INVALID_CQ;
1994 cq_poll->cq_ids[CQ_TX] = (qidx < pf->hw.tx_queues) ?
1995 qidx + pf->hw.rx_queues : CINT_INVALID_CQ;
1996 if (pf->xdp_prog)
1997 cq_poll->cq_ids[CQ_XDP] = (qidx < pf->hw.xdp_queues) ?
1998 (qidx + pf->hw.rx_queues +
1999 pf->hw.tx_queues) :
2000 CINT_INVALID_CQ;
2001 else
2002 cq_poll->cq_ids[CQ_XDP] = CINT_INVALID_CQ;
2003
2004 cq_poll->cq_ids[CQ_QOS] = (qidx < pf->hw.tc_tx_queues) ?
2005 (qidx + pf->hw.rx_queues +
2006 pf->hw.non_qos_queues) :
2007 CINT_INVALID_CQ;
2008
2009 cq_poll->dev = (void *)pf;
2010 cq_poll->dim.mode = DIM_CQ_PERIOD_MODE_START_FROM_CQE;
2011 INIT_WORK(&cq_poll->dim.work, otx2_dim_work);
2012 netif_napi_add(netdev, &cq_poll->napi, otx2_napi_handler);
2013 napi_enable(&cq_poll->napi);
2014 }
2015
2016 /* Set maximum frame size allowed in HW */
2017 err = otx2_hw_set_mtu(pf, netdev->mtu);
2018 if (err)
2019 goto err_disable_napi;
2020
2021 /* Setup segmentation algorithms, if failed, clear offload capability */
2022 otx2_setup_segmentation(pf);
2023
2024 /* Initialize RSS */
2025 err = otx2_rss_init(pf);
2026 if (err)
2027 goto err_disable_napi;
2028
2029 /* Register Queue IRQ handlers */
2030 vec = pf->hw.nix_msixoff + NIX_LF_QINT_VEC_START;
2031 irq_name = &pf->hw.irq_name[vec * NAME_SIZE];
2032
2033 snprintf(irq_name, NAME_SIZE, "%s-qerr", pf->netdev->name);
2034
2035 err = request_irq(pci_irq_vector(pf->pdev, vec),
2036 otx2_q_intr_handler, 0, irq_name, pf);
2037 if (err) {
2038 dev_err(pf->dev,
2039 "RVUPF%d: IRQ registration failed for QERR\n",
2040 rvu_get_pf(pf->pdev, pf->pcifunc));
2041 goto err_disable_napi;
2042 }
2043
2044 /* Enable QINT IRQ */
2045 otx2_write64(pf, NIX_LF_QINTX_ENA_W1S(0), BIT_ULL(0));
2046
2047 /* Register CQ IRQ handlers */
2048 vec = pf->hw.nix_msixoff + NIX_LF_CINT_VEC_START;
2049 for (qidx = 0; qidx < pf->hw.cint_cnt; qidx++) {
2050 irq_name = &pf->hw.irq_name[vec * NAME_SIZE];
2051 int name_len;
2052
2053 name_len = snprintf(irq_name, NAME_SIZE, "%s-rxtx-%d",
2054 pf->netdev->name, qidx);
2055 if (name_len >= NAME_SIZE) {
2056 dev_err(pf->dev,
2057 "RVUPF%d: IRQ registration failed for CQ%d, irq name is too long\n",
2058 rvu_get_pf(pf->pdev, pf->pcifunc), qidx);
2059 err = -EINVAL;
2060 goto err_free_cints;
2061 }
2062
2063 err = request_irq(pci_irq_vector(pf->pdev, vec),
2064 otx2_cq_intr_handler, 0, irq_name,
2065 &qset->napi[qidx]);
2066 if (err) {
2067 dev_err(pf->dev,
2068 "RVUPF%d: IRQ registration failed for CQ%d\n",
2069 rvu_get_pf(pf->pdev, pf->pcifunc), qidx);
2070 goto err_free_cints;
2071 }
2072 vec++;
2073
2074 otx2_config_irq_coalescing(pf, qidx);
2075
2076 /* Enable CQ IRQ */
2077 otx2_write64(pf, NIX_LF_CINTX_INT(qidx), BIT_ULL(0));
2078 otx2_write64(pf, NIX_LF_CINTX_ENA_W1S(qidx), BIT_ULL(0));
2079 }
2080
2081 otx2_set_cints_affinity(pf);
2082
2083 if (pf->flags & OTX2_FLAG_RX_VLAN_SUPPORT)
2084 otx2_enable_rxvlan(pf, true);
2085
2086 /* When reinitializing enable time stamping if it is enabled before */
2087 if (pf->flags & OTX2_FLAG_TX_TSTAMP_ENABLED) {
2088 pf->flags &= ~OTX2_FLAG_TX_TSTAMP_ENABLED;
2089 otx2_config_hw_tx_tstamp(pf, true);
2090 }
2091 if (pf->flags & OTX2_FLAG_RX_TSTAMP_ENABLED) {
2092 pf->flags &= ~OTX2_FLAG_RX_TSTAMP_ENABLED;
2093 otx2_config_hw_rx_tstamp(pf, true);
2094 }
2095
2096 pf->flags &= ~OTX2_FLAG_INTF_DOWN;
2097 pf->flags &= ~OTX2_FLAG_PORT_UP;
2098 /* 'intf_down' may be checked on any cpu */
2099 smp_wmb();
2100
2101 /* Enable QoS configuration before starting tx queues */
2102 otx2_qos_config_txschq(pf);
2103
2104 /* we have already received link status notification */
2105 if (pf->linfo.link_up && !(pf->pcifunc & RVU_PFVF_FUNC_MASK))
2106 otx2_handle_link_event(pf);
2107
2108 /* Install DMAC Filters */
2109 if (pf->flags & OTX2_FLAG_DMACFLTR_SUPPORT)
2110 otx2_dmacflt_reinstall_flows(pf);
2111
2112 otx2_tc_apply_ingress_police_rules(pf);
2113
2114 err = otx2_rxtx_enable(pf, true);
2115 /* If a mbox communication error happens at this point then interface
2116 * will end up in a state such that it is in down state but hardware
2117 * mcam entries are enabled to receive the packets. Hence disable the
2118 * packet I/O.
2119 */
2120 if (err == -EIO)
2121 goto err_disable_rxtx;
2122 else if (err)
2123 goto err_tx_stop_queues;
2124
2125 otx2_do_set_rx_mode(pf);
2126
2127 return 0;
2128
2129 err_disable_rxtx:
2130 otx2_rxtx_enable(pf, false);
2131 err_tx_stop_queues:
2132 netif_tx_stop_all_queues(netdev);
2133 netif_carrier_off(netdev);
2134 pf->flags |= OTX2_FLAG_INTF_DOWN;
2135 err_free_cints:
2136 otx2_free_cints(pf, qidx);
2137 vec = pci_irq_vector(pf->pdev,
2138 pf->hw.nix_msixoff + NIX_LF_QINT_VEC_START);
2139 otx2_write64(pf, NIX_LF_QINTX_ENA_W1C(0), BIT_ULL(0));
2140 free_irq(vec, pf);
2141 err_disable_napi:
2142 otx2_disable_napi(pf);
2143 otx2_free_hw_resources(pf);
2144 err_free_mem:
2145 otx2_free_queue_mem(qset);
2146 return err;
2147 }
2148 EXPORT_SYMBOL(otx2_open);
2149
otx2_stop(struct net_device * netdev)2150 int otx2_stop(struct net_device *netdev)
2151 {
2152 struct otx2_nic *pf = netdev_priv(netdev);
2153 struct otx2_cq_poll *cq_poll = NULL;
2154 struct otx2_qset *qset = &pf->qset;
2155 int qidx, vec, wrk;
2156
2157 /* If the DOWN flag is set resources are already freed */
2158 if (pf->flags & OTX2_FLAG_INTF_DOWN)
2159 return 0;
2160
2161 netif_carrier_off(netdev);
2162 netif_tx_stop_all_queues(netdev);
2163
2164 pf->flags |= OTX2_FLAG_INTF_DOWN;
2165 /* 'intf_down' may be checked on any cpu */
2166 smp_wmb();
2167
2168 /* First stop packet Rx/Tx */
2169 otx2_rxtx_enable(pf, false);
2170
2171 /* Clear RSS enable flag */
2172 pf->hw.rss_info.enable = false;
2173
2174 /* Cleanup Queue IRQ */
2175 vec = pci_irq_vector(pf->pdev,
2176 pf->hw.nix_msixoff + NIX_LF_QINT_VEC_START);
2177 otx2_write64(pf, NIX_LF_QINTX_ENA_W1C(0), BIT_ULL(0));
2178 free_irq(vec, pf);
2179
2180 /* Cleanup CQ NAPI and IRQ */
2181 vec = pf->hw.nix_msixoff + NIX_LF_CINT_VEC_START;
2182 for (qidx = 0; qidx < pf->hw.cint_cnt; qidx++) {
2183 /* Disable interrupt */
2184 otx2_write64(pf, NIX_LF_CINTX_ENA_W1C(qidx), BIT_ULL(0));
2185
2186 synchronize_irq(pci_irq_vector(pf->pdev, vec));
2187
2188 cq_poll = &qset->napi[qidx];
2189 napi_synchronize(&cq_poll->napi);
2190 vec++;
2191 }
2192
2193 netif_tx_disable(netdev);
2194
2195 for (wrk = 0; wrk < pf->qset.cq_cnt; wrk++)
2196 cancel_delayed_work_sync(&pf->refill_wrk[wrk].pool_refill_work);
2197 devm_kfree(pf->dev, pf->refill_wrk);
2198
2199 otx2_free_hw_resources(pf);
2200 otx2_free_cints(pf, pf->hw.cint_cnt);
2201 otx2_disable_napi(pf);
2202
2203 for (qidx = 0; qidx < netdev->num_tx_queues; qidx++)
2204 netdev_tx_reset_queue(netdev_get_tx_queue(netdev, qidx));
2205
2206 otx2_free_queue_mem(qset);
2207 /* Do not clear RQ/SQ ringsize settings */
2208 memset_startat(qset, 0, sqe_cnt);
2209 return 0;
2210 }
2211 EXPORT_SYMBOL(otx2_stop);
2212
otx2_xmit(struct sk_buff * skb,struct net_device * netdev)2213 static netdev_tx_t otx2_xmit(struct sk_buff *skb, struct net_device *netdev)
2214 {
2215 struct otx2_nic *pf = netdev_priv(netdev);
2216 int qidx = skb_get_queue_mapping(skb);
2217 struct otx2_dev_stats *dev_stats;
2218 struct otx2_snd_queue *sq;
2219 struct netdev_queue *txq;
2220 int sq_idx;
2221
2222 /* XDP SQs are not mapped with TXQs
2223 * advance qid to derive correct sq mapped with QOS
2224 */
2225 sq_idx = (qidx >= pf->hw.tx_queues) ? (qidx + pf->hw.xdp_queues) : qidx;
2226
2227 /* Check for minimum and maximum packet length */
2228 if (skb->len <= ETH_HLEN ||
2229 (!skb_shinfo(skb)->gso_size && skb->len > pf->tx_max_pktlen)) {
2230 dev_stats = &pf->hw.dev_stats;
2231 atomic_long_inc(&dev_stats->tx_discards);
2232 dev_kfree_skb(skb);
2233 return NETDEV_TX_OK;
2234 }
2235
2236 sq = &pf->qset.sq[sq_idx];
2237 txq = netdev_get_tx_queue(netdev, qidx);
2238
2239 if (!otx2_sq_append_skb(pf, txq, sq, skb, qidx)) {
2240 netif_tx_stop_queue(txq);
2241
2242 /* Check again, incase SQBs got freed up */
2243 smp_mb();
2244 if (((sq->num_sqbs - *sq->aura_fc_addr) * sq->sqe_per_sqb)
2245 > sq->sqe_thresh)
2246 netif_tx_wake_queue(txq);
2247
2248 return NETDEV_TX_BUSY;
2249 }
2250
2251 return NETDEV_TX_OK;
2252 }
2253
otx2_qos_select_htb_queue(struct otx2_nic * pf,struct sk_buff * skb,u16 htb_maj_id)2254 static int otx2_qos_select_htb_queue(struct otx2_nic *pf, struct sk_buff *skb,
2255 u16 htb_maj_id)
2256 {
2257 u16 classid;
2258
2259 if ((TC_H_MAJ(skb->priority) >> 16) == htb_maj_id)
2260 classid = TC_H_MIN(skb->priority);
2261 else
2262 classid = READ_ONCE(pf->qos.defcls);
2263
2264 if (!classid)
2265 return 0;
2266
2267 return otx2_get_txq_by_classid(pf, classid);
2268 }
2269
otx2_select_queue(struct net_device * netdev,struct sk_buff * skb,struct net_device * sb_dev)2270 u16 otx2_select_queue(struct net_device *netdev, struct sk_buff *skb,
2271 struct net_device *sb_dev)
2272 {
2273 struct otx2_nic *pf = netdev_priv(netdev);
2274 bool qos_enabled;
2275 #ifdef CONFIG_DCB
2276 u8 vlan_prio;
2277 #endif
2278 int txq;
2279
2280 qos_enabled = netdev->real_num_tx_queues > pf->hw.tx_queues;
2281 if (unlikely(qos_enabled)) {
2282 /* This smp_load_acquire() pairs with smp_store_release() in
2283 * otx2_qos_root_add() called from htb offload root creation
2284 */
2285 u16 htb_maj_id = smp_load_acquire(&pf->qos.maj_id);
2286
2287 if (unlikely(htb_maj_id)) {
2288 txq = otx2_qos_select_htb_queue(pf, skb, htb_maj_id);
2289 if (txq > 0)
2290 return txq;
2291 goto process_pfc;
2292 }
2293 }
2294
2295 process_pfc:
2296 #ifdef CONFIG_DCB
2297 if (!skb_vlan_tag_present(skb))
2298 goto pick_tx;
2299
2300 vlan_prio = skb->vlan_tci >> 13;
2301 if ((vlan_prio > pf->hw.tx_queues - 1) ||
2302 !pf->pfc_alloc_status[vlan_prio])
2303 goto pick_tx;
2304
2305 return vlan_prio;
2306
2307 pick_tx:
2308 #endif
2309 txq = netdev_pick_tx(netdev, skb, NULL);
2310 if (unlikely(qos_enabled))
2311 return txq % pf->hw.tx_queues;
2312
2313 return txq;
2314 }
2315 EXPORT_SYMBOL(otx2_select_queue);
2316
otx2_fix_features(struct net_device * dev,netdev_features_t features)2317 static netdev_features_t otx2_fix_features(struct net_device *dev,
2318 netdev_features_t features)
2319 {
2320 if (features & NETIF_F_HW_VLAN_CTAG_RX)
2321 features |= NETIF_F_HW_VLAN_STAG_RX;
2322 else
2323 features &= ~NETIF_F_HW_VLAN_STAG_RX;
2324
2325 return features;
2326 }
2327
otx2_set_rx_mode(struct net_device * netdev)2328 static void otx2_set_rx_mode(struct net_device *netdev)
2329 {
2330 struct otx2_nic *pf = netdev_priv(netdev);
2331
2332 queue_work(pf->otx2_wq, &pf->rx_mode_work);
2333 }
2334
otx2_rx_mode_wrk_handler(struct work_struct * work)2335 static void otx2_rx_mode_wrk_handler(struct work_struct *work)
2336 {
2337 struct otx2_nic *pf = container_of(work, struct otx2_nic, rx_mode_work);
2338
2339 otx2_do_set_rx_mode(pf);
2340 }
2341
otx2_set_features(struct net_device * netdev,netdev_features_t features)2342 static int otx2_set_features(struct net_device *netdev,
2343 netdev_features_t features)
2344 {
2345 netdev_features_t changed = features ^ netdev->features;
2346 struct otx2_nic *pf = netdev_priv(netdev);
2347
2348 if ((changed & NETIF_F_LOOPBACK) && netif_running(netdev))
2349 return otx2_cgx_config_loopback(pf,
2350 features & NETIF_F_LOOPBACK);
2351
2352 if ((changed & NETIF_F_HW_VLAN_CTAG_RX) && netif_running(netdev))
2353 return otx2_enable_rxvlan(pf,
2354 features & NETIF_F_HW_VLAN_CTAG_RX);
2355
2356 if (changed & NETIF_F_HW_ESP)
2357 return cn10k_ipsec_ethtool_init(netdev,
2358 features & NETIF_F_HW_ESP);
2359
2360 return otx2_handle_ntuple_tc_features(netdev, features);
2361 }
2362
otx2_reset_task(struct work_struct * work)2363 static void otx2_reset_task(struct work_struct *work)
2364 {
2365 struct otx2_nic *pf = container_of(work, struct otx2_nic, reset_task);
2366
2367 if (!netif_running(pf->netdev))
2368 return;
2369
2370 rtnl_lock();
2371 otx2_stop(pf->netdev);
2372 pf->reset_count++;
2373 otx2_open(pf->netdev);
2374 netif_trans_update(pf->netdev);
2375 rtnl_unlock();
2376 }
2377
otx2_config_hw_rx_tstamp(struct otx2_nic * pfvf,bool enable)2378 static int otx2_config_hw_rx_tstamp(struct otx2_nic *pfvf, bool enable)
2379 {
2380 struct msg_req *req;
2381 int err;
2382
2383 if (pfvf->flags & OTX2_FLAG_RX_TSTAMP_ENABLED && enable)
2384 return 0;
2385
2386 mutex_lock(&pfvf->mbox.lock);
2387 if (enable)
2388 req = otx2_mbox_alloc_msg_cgx_ptp_rx_enable(&pfvf->mbox);
2389 else
2390 req = otx2_mbox_alloc_msg_cgx_ptp_rx_disable(&pfvf->mbox);
2391 if (!req) {
2392 mutex_unlock(&pfvf->mbox.lock);
2393 return -ENOMEM;
2394 }
2395
2396 err = otx2_sync_mbox_msg(&pfvf->mbox);
2397 if (err) {
2398 mutex_unlock(&pfvf->mbox.lock);
2399 return err;
2400 }
2401
2402 mutex_unlock(&pfvf->mbox.lock);
2403 if (enable)
2404 pfvf->flags |= OTX2_FLAG_RX_TSTAMP_ENABLED;
2405 else
2406 pfvf->flags &= ~OTX2_FLAG_RX_TSTAMP_ENABLED;
2407 return 0;
2408 }
2409
otx2_config_hw_tx_tstamp(struct otx2_nic * pfvf,bool enable)2410 static int otx2_config_hw_tx_tstamp(struct otx2_nic *pfvf, bool enable)
2411 {
2412 struct msg_req *req;
2413 int err;
2414
2415 if (pfvf->flags & OTX2_FLAG_TX_TSTAMP_ENABLED && enable)
2416 return 0;
2417
2418 mutex_lock(&pfvf->mbox.lock);
2419 if (enable)
2420 req = otx2_mbox_alloc_msg_nix_lf_ptp_tx_enable(&pfvf->mbox);
2421 else
2422 req = otx2_mbox_alloc_msg_nix_lf_ptp_tx_disable(&pfvf->mbox);
2423 if (!req) {
2424 mutex_unlock(&pfvf->mbox.lock);
2425 return -ENOMEM;
2426 }
2427
2428 err = otx2_sync_mbox_msg(&pfvf->mbox);
2429 if (err) {
2430 mutex_unlock(&pfvf->mbox.lock);
2431 return err;
2432 }
2433
2434 mutex_unlock(&pfvf->mbox.lock);
2435 if (enable)
2436 pfvf->flags |= OTX2_FLAG_TX_TSTAMP_ENABLED;
2437 else
2438 pfvf->flags &= ~OTX2_FLAG_TX_TSTAMP_ENABLED;
2439 return 0;
2440 }
2441
otx2_config_hwtstamp_get(struct net_device * netdev,struct kernel_hwtstamp_config * config)2442 int otx2_config_hwtstamp_get(struct net_device *netdev,
2443 struct kernel_hwtstamp_config *config)
2444 {
2445 struct otx2_nic *pfvf = netdev_priv(netdev);
2446
2447 *config = pfvf->tstamp;
2448 return 0;
2449 }
2450 EXPORT_SYMBOL(otx2_config_hwtstamp_get);
2451
otx2_config_hwtstamp_set(struct net_device * netdev,struct kernel_hwtstamp_config * config,struct netlink_ext_ack * extack)2452 int otx2_config_hwtstamp_set(struct net_device *netdev,
2453 struct kernel_hwtstamp_config *config,
2454 struct netlink_ext_ack *extack)
2455 {
2456 struct otx2_nic *pfvf = netdev_priv(netdev);
2457
2458 if (!pfvf->ptp)
2459 return -ENODEV;
2460
2461 switch (config->tx_type) {
2462 case HWTSTAMP_TX_OFF:
2463 if (pfvf->flags & OTX2_FLAG_PTP_ONESTEP_SYNC)
2464 pfvf->flags &= ~OTX2_FLAG_PTP_ONESTEP_SYNC;
2465
2466 cancel_delayed_work(&pfvf->ptp->synctstamp_work);
2467 otx2_config_hw_tx_tstamp(pfvf, false);
2468 break;
2469 case HWTSTAMP_TX_ONESTEP_SYNC:
2470 if (!test_bit(CN10K_PTP_ONESTEP, &pfvf->hw.cap_flag)) {
2471 NL_SET_ERR_MSG_MOD(extack,
2472 "One-step time stamping is not supported");
2473 return -ERANGE;
2474 }
2475 pfvf->flags |= OTX2_FLAG_PTP_ONESTEP_SYNC;
2476 schedule_delayed_work(&pfvf->ptp->synctstamp_work,
2477 msecs_to_jiffies(500));
2478 fallthrough;
2479 case HWTSTAMP_TX_ON:
2480 otx2_config_hw_tx_tstamp(pfvf, true);
2481 break;
2482 default:
2483 return -ERANGE;
2484 }
2485
2486 switch (config->rx_filter) {
2487 case HWTSTAMP_FILTER_NONE:
2488 otx2_config_hw_rx_tstamp(pfvf, false);
2489 break;
2490 case HWTSTAMP_FILTER_ALL:
2491 case HWTSTAMP_FILTER_SOME:
2492 case HWTSTAMP_FILTER_PTP_V1_L4_EVENT:
2493 case HWTSTAMP_FILTER_PTP_V1_L4_SYNC:
2494 case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ:
2495 case HWTSTAMP_FILTER_PTP_V2_L4_EVENT:
2496 case HWTSTAMP_FILTER_PTP_V2_L4_SYNC:
2497 case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ:
2498 case HWTSTAMP_FILTER_PTP_V2_L2_EVENT:
2499 case HWTSTAMP_FILTER_PTP_V2_L2_SYNC:
2500 case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ:
2501 case HWTSTAMP_FILTER_PTP_V2_EVENT:
2502 case HWTSTAMP_FILTER_PTP_V2_SYNC:
2503 case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ:
2504 otx2_config_hw_rx_tstamp(pfvf, true);
2505 config->rx_filter = HWTSTAMP_FILTER_ALL;
2506 break;
2507 default:
2508 return -ERANGE;
2509 }
2510
2511 pfvf->tstamp = *config;
2512
2513 return 0;
2514 }
2515 EXPORT_SYMBOL(otx2_config_hwtstamp_set);
2516
otx2_do_set_vf_mac(struct otx2_nic * pf,int vf,const u8 * mac)2517 static int otx2_do_set_vf_mac(struct otx2_nic *pf, int vf, const u8 *mac)
2518 {
2519 struct npc_get_field_status_req *freq;
2520 struct npc_get_field_status_rsp *frsp;
2521 struct npc_install_flow_req *req;
2522 int err;
2523
2524 mutex_lock(&pf->mbox.lock);
2525
2526 /* Skip installing the DMAC filter if the hardware parser profile
2527 * does not support DMAC extraction.
2528 */
2529 freq = otx2_mbox_alloc_msg_npc_get_field_status(&pf->mbox);
2530 if (!freq) {
2531 err = -ENOMEM;
2532 goto out;
2533 }
2534
2535 freq->field = NPC_DMAC;
2536 err = otx2_sync_mbox_msg(&pf->mbox);
2537 if (err)
2538 goto out;
2539
2540 frsp = (struct npc_get_field_status_rsp *)otx2_mbox_get_rsp
2541 (&pf->mbox.mbox, 0, &freq->hdr);
2542 if (IS_ERR(frsp)) {
2543 err = PTR_ERR(frsp);
2544 goto out;
2545 }
2546
2547 if (!frsp->enable) {
2548 netdev_warn(pf->netdev,
2549 "VF %d MAC filter not installed: DMAC extraction not supported by parser profile\n",
2550 vf);
2551 err = -EOPNOTSUPP;
2552 goto out;
2553 }
2554
2555 req = otx2_mbox_alloc_msg_npc_install_flow(&pf->mbox);
2556 if (!req) {
2557 err = -ENOMEM;
2558 goto out;
2559 }
2560
2561 ether_addr_copy(req->packet.dmac, mac);
2562 eth_broadcast_addr((u8 *)&req->mask.dmac);
2563 req->features = BIT_ULL(NPC_DMAC);
2564 req->channel = pf->hw.rx_chan_base;
2565 req->intf = NIX_INTF_RX;
2566 req->default_rule = 1;
2567 req->append = 1;
2568 req->vf = vf + 1;
2569 req->op = NIX_RX_ACTION_DEFAULT;
2570
2571 err = otx2_sync_mbox_msg(&pf->mbox);
2572 out:
2573 mutex_unlock(&pf->mbox.lock);
2574 return err;
2575 }
2576
otx2_set_vf_mac(struct net_device * netdev,int vf,u8 * mac)2577 static int otx2_set_vf_mac(struct net_device *netdev, int vf, u8 *mac)
2578 {
2579 struct otx2_nic *pf = netdev_priv(netdev);
2580 struct pci_dev *pdev = pf->pdev;
2581 struct otx2_vf_config *config;
2582 int ret;
2583
2584 if (!netif_running(netdev))
2585 return -EAGAIN;
2586
2587 if (vf >= pf->total_vfs)
2588 return -EINVAL;
2589
2590 if (!is_valid_ether_addr(mac))
2591 return -EINVAL;
2592
2593 ret = otx2_do_set_vf_mac(pf, vf, mac);
2594 if (ret == 0) {
2595 config = &pf->vf_configs[vf];
2596 ether_addr_copy(config->mac, mac);
2597 dev_info(&pdev->dev, "Load/Reload VF driver\n");
2598 }
2599
2600 return ret;
2601 }
2602
otx2_do_set_vf_vlan(struct otx2_nic * pf,int vf,u16 vlan,u8 qos,__be16 proto)2603 static int otx2_do_set_vf_vlan(struct otx2_nic *pf, int vf, u16 vlan, u8 qos,
2604 __be16 proto)
2605 {
2606 struct otx2_flow_config *flow_cfg = pf->flow_cfg;
2607 struct nix_vtag_config_rsp *vtag_rsp;
2608 struct npc_delete_flow_req *del_req;
2609 struct nix_vtag_config *vtag_req;
2610 struct npc_install_flow_req *req;
2611 struct otx2_vf_config *config;
2612 int err = 0;
2613 u32 idx;
2614
2615 config = &pf->vf_configs[vf];
2616
2617 if (!vlan && !config->vlan)
2618 goto out;
2619
2620 mutex_lock(&pf->mbox.lock);
2621
2622 /* free old tx vtag entry */
2623 if (config->vlan) {
2624 vtag_req = otx2_mbox_alloc_msg_nix_vtag_cfg(&pf->mbox);
2625 if (!vtag_req) {
2626 err = -ENOMEM;
2627 goto out;
2628 }
2629 vtag_req->cfg_type = 0;
2630 vtag_req->tx.free_vtag0 = 1;
2631 vtag_req->tx.vtag0_idx = config->tx_vtag_idx;
2632
2633 err = otx2_sync_mbox_msg(&pf->mbox);
2634 if (err)
2635 goto out;
2636 }
2637
2638 if (!vlan && config->vlan) {
2639 /* rx */
2640 del_req = otx2_mbox_alloc_msg_npc_delete_flow(&pf->mbox);
2641 if (!del_req) {
2642 err = -ENOMEM;
2643 goto out;
2644 }
2645 idx = ((vf * OTX2_PER_VF_VLAN_FLOWS) + OTX2_VF_VLAN_RX_INDEX);
2646 del_req->entry =
2647 flow_cfg->def_ent[flow_cfg->vf_vlan_offset + idx];
2648 err = otx2_sync_mbox_msg(&pf->mbox);
2649 if (err)
2650 goto out;
2651
2652 /* tx */
2653 del_req = otx2_mbox_alloc_msg_npc_delete_flow(&pf->mbox);
2654 if (!del_req) {
2655 err = -ENOMEM;
2656 goto out;
2657 }
2658 idx = ((vf * OTX2_PER_VF_VLAN_FLOWS) + OTX2_VF_VLAN_TX_INDEX);
2659 del_req->entry =
2660 flow_cfg->def_ent[flow_cfg->vf_vlan_offset + idx];
2661 err = otx2_sync_mbox_msg(&pf->mbox);
2662
2663 goto out;
2664 }
2665
2666 /* rx */
2667 req = otx2_mbox_alloc_msg_npc_install_flow(&pf->mbox);
2668 if (!req) {
2669 err = -ENOMEM;
2670 goto out;
2671 }
2672
2673 idx = ((vf * OTX2_PER_VF_VLAN_FLOWS) + OTX2_VF_VLAN_RX_INDEX);
2674 req->entry = flow_cfg->def_ent[flow_cfg->vf_vlan_offset + idx];
2675 req->packet.vlan_tci = htons(vlan);
2676 req->mask.vlan_tci = htons(VLAN_VID_MASK);
2677 /* af fills the destination mac addr */
2678 eth_broadcast_addr((u8 *)&req->mask.dmac);
2679 req->features = BIT_ULL(NPC_OUTER_VID) | BIT_ULL(NPC_DMAC);
2680 req->channel = pf->hw.rx_chan_base;
2681 req->intf = NIX_INTF_RX;
2682 req->vf = vf + 1;
2683 req->op = NIX_RX_ACTION_DEFAULT;
2684 req->vtag0_valid = true;
2685 req->vtag0_type = NIX_AF_LFX_RX_VTAG_TYPE7;
2686 req->set_cntr = 1;
2687
2688 err = otx2_sync_mbox_msg(&pf->mbox);
2689 if (err)
2690 goto out;
2691
2692 /* tx */
2693 vtag_req = otx2_mbox_alloc_msg_nix_vtag_cfg(&pf->mbox);
2694 if (!vtag_req) {
2695 err = -ENOMEM;
2696 goto out;
2697 }
2698
2699 /* configure tx vtag params */
2700 vtag_req->vtag_size = VTAGSIZE_T4;
2701 vtag_req->cfg_type = 0; /* tx vlan cfg */
2702 vtag_req->tx.cfg_vtag0 = 1;
2703 vtag_req->tx.vtag0 = ((u64)ntohs(proto) << 16) | vlan;
2704
2705 err = otx2_sync_mbox_msg(&pf->mbox);
2706 if (err)
2707 goto out;
2708
2709 vtag_rsp = (struct nix_vtag_config_rsp *)otx2_mbox_get_rsp
2710 (&pf->mbox.mbox, 0, &vtag_req->hdr);
2711 if (IS_ERR(vtag_rsp)) {
2712 err = PTR_ERR(vtag_rsp);
2713 goto out;
2714 }
2715 config->tx_vtag_idx = vtag_rsp->vtag0_idx;
2716
2717 req = otx2_mbox_alloc_msg_npc_install_flow(&pf->mbox);
2718 if (!req) {
2719 err = -ENOMEM;
2720 goto out;
2721 }
2722
2723 eth_zero_addr((u8 *)&req->mask.dmac);
2724 idx = ((vf * OTX2_PER_VF_VLAN_FLOWS) + OTX2_VF_VLAN_TX_INDEX);
2725 req->entry = flow_cfg->def_ent[flow_cfg->vf_vlan_offset + idx];
2726 req->features = BIT_ULL(NPC_DMAC);
2727 req->channel = pf->hw.tx_chan_base;
2728 req->intf = NIX_INTF_TX;
2729 req->vf = vf + 1;
2730 req->op = NIX_TX_ACTIONOP_UCAST_DEFAULT;
2731 req->vtag0_def = vtag_rsp->vtag0_idx;
2732 req->vtag0_op = VTAG_INSERT;
2733 req->set_cntr = 1;
2734
2735 err = otx2_sync_mbox_msg(&pf->mbox);
2736 out:
2737 config->vlan = vlan;
2738 mutex_unlock(&pf->mbox.lock);
2739 return err;
2740 }
2741
otx2_set_vf_vlan(struct net_device * netdev,int vf,u16 vlan,u8 qos,__be16 proto)2742 static int otx2_set_vf_vlan(struct net_device *netdev, int vf, u16 vlan, u8 qos,
2743 __be16 proto)
2744 {
2745 struct otx2_nic *pf = netdev_priv(netdev);
2746 struct pci_dev *pdev = pf->pdev;
2747
2748 if (!netif_running(netdev))
2749 return -EAGAIN;
2750
2751 if (vf >= pci_num_vf(pdev))
2752 return -EINVAL;
2753
2754 /* qos is currently unsupported */
2755 if (vlan >= VLAN_N_VID || qos)
2756 return -EINVAL;
2757
2758 if (proto != htons(ETH_P_8021Q))
2759 return -EPROTONOSUPPORT;
2760
2761 if (!(pf->flags & OTX2_FLAG_VF_VLAN_SUPPORT))
2762 return -EOPNOTSUPP;
2763
2764 return otx2_do_set_vf_vlan(pf, vf, vlan, qos, proto);
2765 }
2766
otx2_get_vf_config(struct net_device * netdev,int vf,struct ifla_vf_info * ivi)2767 static int otx2_get_vf_config(struct net_device *netdev, int vf,
2768 struct ifla_vf_info *ivi)
2769 {
2770 struct otx2_nic *pf = netdev_priv(netdev);
2771 struct pci_dev *pdev = pf->pdev;
2772 struct otx2_vf_config *config;
2773
2774 if (!netif_running(netdev))
2775 return -EAGAIN;
2776
2777 if (vf >= pci_num_vf(pdev))
2778 return -EINVAL;
2779
2780 config = &pf->vf_configs[vf];
2781 ivi->vf = vf;
2782 ether_addr_copy(ivi->mac, config->mac);
2783 ivi->vlan = config->vlan;
2784 ivi->trusted = config->trusted;
2785
2786 return 0;
2787 }
2788
otx2_xdp_xmit_tx(struct otx2_nic * pf,struct xdp_frame * xdpf,int qidx)2789 static int otx2_xdp_xmit_tx(struct otx2_nic *pf, struct xdp_frame *xdpf,
2790 int qidx)
2791 {
2792 u64 dma_addr;
2793 int err = 0;
2794
2795 dma_addr = otx2_dma_map_page(pf, virt_to_page(xdpf->data),
2796 offset_in_page(xdpf->data), xdpf->len,
2797 DMA_TO_DEVICE);
2798 if (dma_mapping_error(pf->dev, dma_addr))
2799 return -ENOMEM;
2800
2801 err = otx2_xdp_sq_append_pkt(pf, xdpf, dma_addr, xdpf->len,
2802 qidx, OTX2_XDP_REDIRECT);
2803 if (!err) {
2804 otx2_dma_unmap_page(pf, dma_addr, xdpf->len, DMA_TO_DEVICE);
2805 xdp_return_frame(xdpf);
2806 return -ENOMEM;
2807 }
2808 return 0;
2809 }
2810
otx2_xdp_xmit(struct net_device * netdev,int n,struct xdp_frame ** frames,u32 flags)2811 static int otx2_xdp_xmit(struct net_device *netdev, int n,
2812 struct xdp_frame **frames, u32 flags)
2813 {
2814 struct otx2_nic *pf = netdev_priv(netdev);
2815 int qidx = smp_processor_id();
2816 struct otx2_snd_queue *sq;
2817 int drops = 0, i;
2818
2819 if (!netif_running(netdev))
2820 return -ENETDOWN;
2821
2822 qidx += pf->hw.tx_queues;
2823 sq = pf->xdp_prog ? &pf->qset.sq[qidx] : NULL;
2824
2825 /* Abort xmit if xdp queue is not */
2826 if (unlikely(!sq))
2827 return -ENXIO;
2828
2829 if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK))
2830 return -EINVAL;
2831
2832 for (i = 0; i < n; i++) {
2833 struct xdp_frame *xdpf = frames[i];
2834 int err;
2835
2836 err = otx2_xdp_xmit_tx(pf, xdpf, qidx);
2837 if (err)
2838 drops++;
2839 }
2840 return n - drops;
2841 }
2842
otx2_xdp_setup(struct otx2_nic * pf,struct bpf_prog * prog)2843 static int otx2_xdp_setup(struct otx2_nic *pf, struct bpf_prog *prog)
2844 {
2845 struct net_device *dev = pf->netdev;
2846 bool if_up = netif_running(pf->netdev);
2847 struct bpf_prog *old_prog;
2848
2849 if (prog && dev->mtu > MAX_XDP_MTU) {
2850 netdev_warn(dev, "Jumbo frames not yet supported with XDP\n");
2851 return -EOPNOTSUPP;
2852 }
2853
2854 if (if_up)
2855 otx2_stop(pf->netdev);
2856
2857 old_prog = xchg(&pf->xdp_prog, prog);
2858
2859 if (old_prog)
2860 bpf_prog_put(old_prog);
2861
2862 if (pf->xdp_prog)
2863 bpf_prog_add(pf->xdp_prog, pf->hw.rx_queues - 1);
2864
2865 /* Network stack and XDP shared same rx queues.
2866 * Use separate tx queues for XDP and network stack.
2867 */
2868 if (pf->xdp_prog) {
2869 pf->hw.xdp_queues = pf->hw.rx_queues;
2870 xdp_features_set_redirect_target(dev, false);
2871 } else {
2872 pf->hw.xdp_queues = 0;
2873 xdp_features_clear_redirect_target(dev);
2874 }
2875
2876 if (if_up)
2877 otx2_open(pf->netdev);
2878
2879 return 0;
2880 }
2881
otx2_xdp(struct net_device * netdev,struct netdev_bpf * xdp)2882 static int otx2_xdp(struct net_device *netdev, struct netdev_bpf *xdp)
2883 {
2884 struct otx2_nic *pf = netdev_priv(netdev);
2885
2886 switch (xdp->command) {
2887 case XDP_SETUP_PROG:
2888 return otx2_xdp_setup(pf, xdp->prog);
2889 case XDP_SETUP_XSK_POOL:
2890 return otx2_xsk_pool_setup(pf, xdp->xsk.pool, xdp->xsk.queue_id);
2891 default:
2892 return -EINVAL;
2893 }
2894 }
2895
otx2_set_vf_permissions(struct otx2_nic * pf,int vf,int req_perm)2896 static int otx2_set_vf_permissions(struct otx2_nic *pf, int vf,
2897 int req_perm)
2898 {
2899 struct set_vf_perm *req;
2900 int rc;
2901
2902 mutex_lock(&pf->mbox.lock);
2903 req = otx2_mbox_alloc_msg_set_vf_perm(&pf->mbox);
2904 if (!req) {
2905 rc = -ENOMEM;
2906 goto out;
2907 }
2908
2909 /* Let AF reset VF permissions as sriov is disabled */
2910 if (req_perm == OTX2_RESET_VF_PERM) {
2911 req->flags |= RESET_VF_PERM;
2912 } else if (req_perm == OTX2_TRUSTED_VF) {
2913 if (pf->vf_configs[vf].trusted)
2914 req->flags |= VF_TRUSTED;
2915 }
2916
2917 req->vf = vf;
2918 rc = otx2_sync_mbox_msg(&pf->mbox);
2919 out:
2920 mutex_unlock(&pf->mbox.lock);
2921 return rc;
2922 }
2923
otx2_ndo_set_vf_trust(struct net_device * netdev,int vf,bool enable)2924 static int otx2_ndo_set_vf_trust(struct net_device *netdev, int vf,
2925 bool enable)
2926 {
2927 struct otx2_nic *pf = netdev_priv(netdev);
2928 struct pci_dev *pdev = pf->pdev;
2929 int rc;
2930
2931 if (vf >= pci_num_vf(pdev))
2932 return -EINVAL;
2933
2934 if (pf->vf_configs[vf].trusted == enable)
2935 return 0;
2936
2937 pf->vf_configs[vf].trusted = enable;
2938 rc = otx2_set_vf_permissions(pf, vf, OTX2_TRUSTED_VF);
2939
2940 if (rc) {
2941 pf->vf_configs[vf].trusted = !enable;
2942 } else {
2943 netdev_info(pf->netdev, "VF %d is %strusted\n",
2944 vf, enable ? "" : "not ");
2945 otx2_set_rx_mode(netdev);
2946 }
2947
2948 return rc;
2949 }
2950
2951 static const struct net_device_ops otx2_netdev_ops = {
2952 .ndo_open = otx2_open,
2953 .ndo_stop = otx2_stop,
2954 .ndo_start_xmit = otx2_xmit,
2955 .ndo_select_queue = otx2_select_queue,
2956 .ndo_fix_features = otx2_fix_features,
2957 .ndo_set_mac_address = otx2_set_mac_address,
2958 .ndo_change_mtu = otx2_change_mtu,
2959 .ndo_set_rx_mode = otx2_set_rx_mode,
2960 .ndo_set_features = otx2_set_features,
2961 .ndo_tx_timeout = otx2_tx_timeout,
2962 .ndo_get_stats64 = otx2_get_stats64,
2963 .ndo_set_vf_mac = otx2_set_vf_mac,
2964 .ndo_set_vf_vlan = otx2_set_vf_vlan,
2965 .ndo_get_vf_config = otx2_get_vf_config,
2966 .ndo_bpf = otx2_xdp,
2967 .ndo_xsk_wakeup = otx2_xsk_wakeup,
2968 .ndo_xdp_xmit = otx2_xdp_xmit,
2969 .ndo_setup_tc = otx2_setup_tc,
2970 .ndo_set_vf_trust = otx2_ndo_set_vf_trust,
2971 .ndo_hwtstamp_get = otx2_config_hwtstamp_get,
2972 .ndo_hwtstamp_set = otx2_config_hwtstamp_set,
2973 };
2974
otx2_wq_init(struct otx2_nic * pf)2975 int otx2_wq_init(struct otx2_nic *pf)
2976 {
2977 pf->otx2_wq = create_singlethread_workqueue("otx2_wq");
2978 if (!pf->otx2_wq)
2979 return -ENOMEM;
2980
2981 INIT_WORK(&pf->rx_mode_work, otx2_rx_mode_wrk_handler);
2982 INIT_WORK(&pf->reset_task, otx2_reset_task);
2983 return 0;
2984 }
2985
otx2_check_pf_usable(struct otx2_nic * nic)2986 int otx2_check_pf_usable(struct otx2_nic *nic)
2987 {
2988 u64 rev;
2989
2990 rev = otx2_read64(nic, RVU_PF_BLOCK_ADDRX_DISC(BLKADDR_RVUM));
2991 rev = (rev >> 12) & 0xFF;
2992 /* Check if AF has setup revision for RVUM block,
2993 * otherwise this driver probe should be deferred
2994 * until AF driver comes up.
2995 */
2996 if (!rev) {
2997 dev_warn(nic->dev,
2998 "AF is not initialized, deferring probe\n");
2999 return -EPROBE_DEFER;
3000 }
3001 return 0;
3002 }
3003
otx2_realloc_msix_vectors(struct otx2_nic * pf)3004 int otx2_realloc_msix_vectors(struct otx2_nic *pf)
3005 {
3006 struct otx2_hw *hw = &pf->hw;
3007 int num_vec, err;
3008
3009 /* NPA interrupts are inot registered, so alloc only
3010 * upto NIX vector offset.
3011 */
3012 num_vec = hw->nix_msixoff;
3013 num_vec += NIX_LF_CINT_VEC_START + hw->max_queues;
3014
3015 otx2_disable_mbox_intr(pf);
3016 pci_free_irq_vectors(hw->pdev);
3017 err = pci_alloc_irq_vectors(hw->pdev, num_vec, num_vec, PCI_IRQ_MSIX);
3018 if (err < 0) {
3019 dev_err(pf->dev, "%s: Failed to realloc %d IRQ vectors\n",
3020 __func__, num_vec);
3021 return err;
3022 }
3023
3024 return otx2_register_mbox_intr(pf, false);
3025 }
3026 EXPORT_SYMBOL(otx2_realloc_msix_vectors);
3027
otx2_sriov_vfcfg_init(struct otx2_nic * pf)3028 static int otx2_sriov_vfcfg_init(struct otx2_nic *pf)
3029 {
3030 int i;
3031
3032 pf->vf_configs = devm_kcalloc(pf->dev, pf->total_vfs,
3033 sizeof(struct otx2_vf_config),
3034 GFP_KERNEL);
3035 if (!pf->vf_configs)
3036 return -ENOMEM;
3037
3038 for (i = 0; i < pf->total_vfs; i++) {
3039 pf->vf_configs[i].pf = pf;
3040 pf->vf_configs[i].intf_down = true;
3041 pf->vf_configs[i].trusted = false;
3042 INIT_DELAYED_WORK(&pf->vf_configs[i].link_event_work,
3043 otx2_vf_link_event_task);
3044 }
3045
3046 return 0;
3047 }
3048
otx2_sriov_vfcfg_cleanup(struct otx2_nic * pf)3049 static void otx2_sriov_vfcfg_cleanup(struct otx2_nic *pf)
3050 {
3051 int i;
3052
3053 if (!pf->vf_configs)
3054 return;
3055
3056 for (i = 0; i < pf->total_vfs; i++) {
3057 cancel_delayed_work_sync(&pf->vf_configs[i].link_event_work);
3058 otx2_set_vf_permissions(pf, i, OTX2_RESET_VF_PERM);
3059 }
3060 }
3061
otx2_init_rsrc(struct pci_dev * pdev,struct otx2_nic * pf)3062 int otx2_init_rsrc(struct pci_dev *pdev, struct otx2_nic *pf)
3063 {
3064 struct device *dev = &pdev->dev;
3065 struct otx2_hw *hw = &pf->hw;
3066 int num_vec, err;
3067
3068 num_vec = pci_msix_vec_count(pdev);
3069 hw->irq_name = devm_kmalloc_array(&hw->pdev->dev, num_vec, NAME_SIZE,
3070 GFP_KERNEL);
3071 if (!hw->irq_name)
3072 return -ENOMEM;
3073
3074 hw->affinity_mask = devm_kcalloc(&hw->pdev->dev, num_vec,
3075 sizeof(cpumask_var_t), GFP_KERNEL);
3076 if (!hw->affinity_mask)
3077 return -ENOMEM;
3078
3079 /* Map CSRs */
3080 pf->reg_base = pcim_iomap(pdev, PCI_CFG_REG_BAR_NUM, 0);
3081 if (!pf->reg_base) {
3082 dev_err(dev, "Unable to map physical function CSRs, aborting\n");
3083 return -ENOMEM;
3084 }
3085
3086 err = otx2_check_pf_usable(pf);
3087 if (err)
3088 return err;
3089
3090 if (!is_cn20k(pf->pdev))
3091 err = pci_alloc_irq_vectors(hw->pdev, RVU_PF_INT_VEC_CNT,
3092 RVU_PF_INT_VEC_CNT, PCI_IRQ_MSIX);
3093 else
3094 err = pci_alloc_irq_vectors(hw->pdev, RVU_MBOX_PF_INT_VEC_CNT,
3095 RVU_MBOX_PF_INT_VEC_CNT,
3096 PCI_IRQ_MSIX);
3097 if (err < 0) {
3098 dev_err(dev, "%s: Failed to alloc %d IRQ vectors\n",
3099 __func__, num_vec);
3100 return err;
3101 }
3102
3103 otx2_setup_dev_hw_settings(pf);
3104
3105 if (is_cn20k(pf->pdev))
3106 cn20k_init(pf);
3107 else
3108 otx2_init_hw_ops(pf);
3109
3110 /* Init PF <=> AF mailbox stuff */
3111 err = otx2_pfaf_mbox_init(pf);
3112 if (err)
3113 goto err_free_irq_vectors;
3114
3115 /* Register mailbox interrupt */
3116 err = otx2_register_mbox_intr(pf, true);
3117 if (err)
3118 goto err_mbox_destroy;
3119
3120 /* Request AF to attach NPA and NIX LFs to this PF.
3121 * NIX and NPA LFs are needed for this PF to function as a NIC.
3122 */
3123 err = otx2_attach_npa_nix(pf);
3124 if (err)
3125 goto err_disable_mbox_intr;
3126
3127 err = otx2_realloc_msix_vectors(pf);
3128 if (err)
3129 goto err_detach_rsrc;
3130
3131 err = cn10k_lmtst_init(pf);
3132 if (err)
3133 goto err_detach_rsrc;
3134
3135 return 0;
3136
3137 err_detach_rsrc:
3138 if (pf->hw.lmt_info)
3139 free_percpu(pf->hw.lmt_info);
3140 if (test_bit(CN10K_LMTST, &pf->hw.cap_flag))
3141 qmem_free(pf->dev, pf->dync_lmt);
3142 otx2_detach_resources(&pf->mbox);
3143 err_disable_mbox_intr:
3144 otx2_disable_mbox_intr(pf);
3145 err_mbox_destroy:
3146 otx2_pfaf_mbox_destroy(pf);
3147 err_free_irq_vectors:
3148 pci_free_irq_vectors(hw->pdev);
3149
3150 return err;
3151 }
3152 EXPORT_SYMBOL(otx2_init_rsrc);
3153
otx2_probe(struct pci_dev * pdev,const struct pci_device_id * id)3154 static int otx2_probe(struct pci_dev *pdev, const struct pci_device_id *id)
3155 {
3156 struct device *dev = &pdev->dev;
3157 int err, qcount, qos_txqs;
3158 struct net_device *netdev;
3159 struct otx2_nic *pf;
3160 struct otx2_hw *hw;
3161
3162 err = pcim_enable_device(pdev);
3163 if (err) {
3164 dev_err(dev, "Failed to enable PCI device\n");
3165 return err;
3166 }
3167
3168 err = pcim_request_all_regions(pdev, DRV_NAME);
3169 if (err) {
3170 dev_err(dev, "PCI request regions failed 0x%x\n", err);
3171 return err;
3172 }
3173
3174 err = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(48));
3175 if (err) {
3176 dev_err(dev, "DMA mask config failed, abort\n");
3177 return err;
3178 }
3179
3180 pci_set_master(pdev);
3181
3182 /* Set number of queues */
3183 qcount = min_t(int, num_online_cpus(), OTX2_MAX_CQ_CNT);
3184 qos_txqs = min_t(int, qcount, OTX2_QOS_MAX_LEAF_NODES);
3185
3186 netdev = alloc_etherdev_mqs(sizeof(*pf), qcount + qos_txqs, qcount);
3187 if (!netdev)
3188 return -ENOMEM;
3189
3190 pci_set_drvdata(pdev, netdev);
3191 SET_NETDEV_DEV(netdev, &pdev->dev);
3192 pf = netdev_priv(netdev);
3193 pf->netdev = netdev;
3194 pf->pdev = pdev;
3195 pf->dev = dev;
3196 pf->total_vfs = pci_sriov_get_totalvfs(pdev);
3197 pf->flags |= OTX2_FLAG_INTF_DOWN;
3198
3199 hw = &pf->hw;
3200 hw->pdev = pdev;
3201 hw->rx_queues = qcount;
3202 hw->tx_queues = qcount;
3203 hw->non_qos_queues = qcount;
3204 hw->max_queues = qcount;
3205 hw->rbuf_len = OTX2_DEFAULT_RBUF_LEN;
3206 /* Use CQE of 128 byte descriptor size by default */
3207 hw->xqe_size = 128;
3208
3209 err = otx2_init_rsrc(pdev, pf);
3210 if (err)
3211 goto err_free_netdev;
3212
3213 err = otx2_set_real_num_queues(netdev, hw->tx_queues, hw->rx_queues);
3214 if (err)
3215 goto err_detach_rsrc;
3216
3217 /* Assign default mac address */
3218 otx2_get_mac_from_af(netdev);
3219
3220 /* Don't check for error. Proceed without ptp */
3221 otx2_ptp_init(pf);
3222
3223 /* NPA's pool is a stack to which SW frees buffer pointers via Aura.
3224 * HW allocates buffer pointer from stack and uses it for DMA'ing
3225 * ingress packet. In some scenarios HW can free back allocated buffer
3226 * pointers to pool. This makes it impossible for SW to maintain a
3227 * parallel list where physical addresses of buffer pointers (IOVAs)
3228 * given to HW can be saved for later reference.
3229 *
3230 * So the only way to convert Rx packet's buffer address is to use
3231 * IOMMU's iova_to_phys() handler which translates the address by
3232 * walking through the translation tables.
3233 */
3234 pf->iommu_domain = iommu_get_domain_for_dev(dev);
3235
3236 netdev->hw_features = (NETIF_F_RXCSUM | NETIF_F_IP_CSUM |
3237 NETIF_F_IPV6_CSUM | NETIF_F_RXHASH |
3238 NETIF_F_SG | NETIF_F_TSO | NETIF_F_TSO6 |
3239 NETIF_F_GSO_UDP_L4);
3240 netdev->features |= netdev->hw_features;
3241
3242 err = otx2_mcam_flow_init(pf);
3243 if (err)
3244 goto err_ptp_destroy;
3245
3246 otx2_set_hw_capabilities(pf);
3247
3248 err = cn10k_mcs_init(pf);
3249 if (err)
3250 goto err_del_mcam_entries;
3251
3252 if (pf->flags & OTX2_FLAG_NTUPLE_SUPPORT)
3253 netdev->hw_features |= NETIF_F_NTUPLE;
3254
3255 if (pf->flags & OTX2_FLAG_UCAST_FLTR_SUPPORT)
3256 netdev->priv_flags |= IFF_UNICAST_FLT;
3257
3258 /* Support TSO on tag interface */
3259 netdev->vlan_features |= netdev->features;
3260 netdev->hw_features |= NETIF_F_HW_VLAN_CTAG_TX |
3261 NETIF_F_HW_VLAN_STAG_TX;
3262 if (pf->flags & OTX2_FLAG_RX_VLAN_SUPPORT)
3263 netdev->hw_features |= NETIF_F_HW_VLAN_CTAG_RX |
3264 NETIF_F_HW_VLAN_STAG_RX;
3265 netdev->features |= netdev->hw_features;
3266
3267 /* HW supports tc offload but mutually exclusive with n-tuple filters */
3268 if (pf->flags & OTX2_FLAG_TC_FLOWER_SUPPORT)
3269 netdev->hw_features |= NETIF_F_HW_TC;
3270
3271 netdev->hw_features |= NETIF_F_LOOPBACK | NETIF_F_RXALL;
3272
3273 netif_set_tso_max_segs(netdev, OTX2_MAX_GSO_SEGS);
3274 netdev->watchdog_timeo = OTX2_TX_TIMEOUT;
3275
3276 netdev->netdev_ops = &otx2_netdev_ops;
3277 netdev->xdp_features = NETDEV_XDP_ACT_BASIC | NETDEV_XDP_ACT_REDIRECT |
3278 NETDEV_XDP_ACT_NDO_XMIT |
3279 NETDEV_XDP_ACT_XSK_ZEROCOPY;
3280
3281 netdev->min_mtu = OTX2_MIN_MTU;
3282 netdev->max_mtu = otx2_get_max_mtu(pf);
3283 hw->max_mtu = netdev->max_mtu;
3284
3285 /* reset CGX/RPM MAC stats */
3286 otx2_reset_mac_stats(pf);
3287
3288 err = cn10k_ipsec_init(netdev);
3289 if (err)
3290 goto err_mcs_free;
3291
3292 err = register_netdev(netdev);
3293 if (err) {
3294 dev_err(dev, "Failed to register netdevice\n");
3295 goto err_ipsec_clean;
3296 }
3297
3298 err = otx2_wq_init(pf);
3299 if (err)
3300 goto err_unreg_netdev;
3301
3302 otx2_set_ethtool_ops(netdev);
3303
3304 err = otx2_init_tc(pf);
3305 if (err)
3306 goto err_mcam_flow_del;
3307
3308 err = otx2_register_dl(pf);
3309 if (err)
3310 goto err_mcam_flow_del;
3311
3312 /* Initialize SR-IOV resources */
3313 err = otx2_sriov_vfcfg_init(pf);
3314 if (err)
3315 goto err_pf_sriov_init;
3316
3317 /* Enable link notifications */
3318 otx2_cgx_config_linkevents(pf, true);
3319
3320 pf->af_xdp_zc_qidx = bitmap_zalloc(qcount, GFP_KERNEL);
3321 if (!pf->af_xdp_zc_qidx) {
3322 err = -ENOMEM;
3323 goto err_sriov_cleannup;
3324 }
3325
3326 #ifdef CONFIG_DCB
3327 err = otx2_dcbnl_set_ops(netdev);
3328 if (err)
3329 goto err_free_zc_bmap;
3330 #endif
3331
3332 otx2_qos_init(pf, qos_txqs);
3333
3334 return 0;
3335
3336 #ifdef CONFIG_DCB
3337 err_free_zc_bmap:
3338 bitmap_free(pf->af_xdp_zc_qidx);
3339 #endif
3340 err_sriov_cleannup:
3341 otx2_sriov_vfcfg_cleanup(pf);
3342 err_pf_sriov_init:
3343 otx2_unregister_dl(pf);
3344 otx2_shutdown_tc(pf);
3345 err_mcam_flow_del:
3346 otx2_mcam_flow_del(pf);
3347 err_unreg_netdev:
3348 unregister_netdev(netdev);
3349 err_ipsec_clean:
3350 cn10k_ipsec_clean(pf);
3351 err_mcs_free:
3352 cn10k_mcs_free(pf);
3353 err_del_mcam_entries:
3354 otx2_mcam_flow_del(pf);
3355 err_ptp_destroy:
3356 otx2_ptp_destroy(pf);
3357 err_detach_rsrc:
3358 if (pf->hw.lmt_info)
3359 free_percpu(pf->hw.lmt_info);
3360 if (test_bit(CN10K_LMTST, &pf->hw.cap_flag))
3361 qmem_free(pf->dev, pf->dync_lmt);
3362 otx2_detach_resources(&pf->mbox);
3363 otx2_disable_mbox_intr(pf);
3364 otx2_pfaf_mbox_destroy(pf);
3365 pci_free_irq_vectors(hw->pdev);
3366 err_free_netdev:
3367 pci_set_drvdata(pdev, NULL);
3368 free_netdev(netdev);
3369 return err;
3370 }
3371
otx2_vf_link_event_task(struct work_struct * work)3372 static void otx2_vf_link_event_task(struct work_struct *work)
3373 {
3374 struct otx2_vf_config *config;
3375 struct cgx_link_info_msg *req;
3376 struct mbox_msghdr *msghdr;
3377 struct delayed_work *dwork;
3378 struct otx2_nic *pf;
3379 int vf_idx;
3380
3381 config = container_of(work, struct otx2_vf_config,
3382 link_event_work.work);
3383 vf_idx = config - config->pf->vf_configs;
3384 pf = config->pf;
3385
3386 if (config->intf_down)
3387 return;
3388
3389 mutex_lock(&pf->mbox.lock);
3390
3391 dwork = &config->link_event_work;
3392
3393 if (!otx2_mbox_wait_for_zero(&pf->mbox_pfvf[0].mbox_up, vf_idx)) {
3394 schedule_delayed_work(dwork, msecs_to_jiffies(100));
3395 mutex_unlock(&pf->mbox.lock);
3396 return;
3397 }
3398
3399 msghdr = otx2_mbox_alloc_msg_rsp(&pf->mbox_pfvf[0].mbox_up, vf_idx,
3400 sizeof(*req), sizeof(struct msg_rsp));
3401 if (!msghdr) {
3402 dev_err(pf->dev, "Failed to create VF%d link event\n", vf_idx);
3403 mutex_unlock(&pf->mbox.lock);
3404 return;
3405 }
3406
3407 req = (struct cgx_link_info_msg *)msghdr;
3408 req->hdr.id = MBOX_MSG_CGX_LINK_EVENT;
3409 req->hdr.sig = OTX2_MBOX_REQ_SIG;
3410 req->hdr.pcifunc = pf->pcifunc;
3411 memcpy(&req->link_info, &pf->linfo, sizeof(req->link_info));
3412
3413 otx2_mbox_wait_for_zero(&pf->mbox_pfvf[0].mbox_up, vf_idx);
3414
3415 otx2_sync_mbox_up_msg(&pf->mbox_pfvf[0], vf_idx);
3416
3417 mutex_unlock(&pf->mbox.lock);
3418 }
3419
otx2_sriov_enable(struct pci_dev * pdev,int numvfs)3420 static int otx2_sriov_enable(struct pci_dev *pdev, int numvfs)
3421 {
3422 struct net_device *netdev = pci_get_drvdata(pdev);
3423 struct otx2_nic *pf = netdev_priv(netdev);
3424 int ret;
3425
3426 /* Init PF <=> VF mailbox stuff */
3427 ret = otx2_pfvf_mbox_init(pf, numvfs);
3428 if (ret)
3429 return ret;
3430
3431 ret = otx2_register_pfvf_mbox_intr(pf, numvfs);
3432 if (ret)
3433 goto free_mbox;
3434
3435 ret = otx2_pf_flr_init(pf, numvfs);
3436 if (ret)
3437 goto free_intr;
3438
3439 ret = otx2_register_flr_me_intr(pf, numvfs);
3440 if (ret)
3441 goto free_flr;
3442
3443 ret = pci_enable_sriov(pdev, numvfs);
3444 if (ret)
3445 goto free_flr_intr;
3446
3447 return numvfs;
3448 free_flr_intr:
3449 otx2_disable_flr_me_intr(pf);
3450 free_flr:
3451 otx2_flr_wq_destroy(pf);
3452 free_intr:
3453 otx2_disable_pfvf_mbox_intr(pf, numvfs);
3454 free_mbox:
3455 otx2_pfvf_mbox_destroy(pf);
3456 return ret;
3457 }
3458
otx2_sriov_disable(struct pci_dev * pdev)3459 static int otx2_sriov_disable(struct pci_dev *pdev)
3460 {
3461 struct net_device *netdev = pci_get_drvdata(pdev);
3462 struct otx2_nic *pf = netdev_priv(netdev);
3463 int numvfs = pci_num_vf(pdev);
3464
3465 if (!numvfs)
3466 return 0;
3467
3468 pci_disable_sriov(pdev);
3469
3470 otx2_disable_flr_me_intr(pf);
3471 otx2_flr_wq_destroy(pf);
3472 otx2_disable_pfvf_mbox_intr(pf, numvfs);
3473 otx2_pfvf_mbox_destroy(pf);
3474
3475 return 0;
3476 }
3477
otx2_sriov_configure(struct pci_dev * pdev,int numvfs)3478 static int otx2_sriov_configure(struct pci_dev *pdev, int numvfs)
3479 {
3480 if (numvfs == 0)
3481 return otx2_sriov_disable(pdev);
3482 else
3483 return otx2_sriov_enable(pdev, numvfs);
3484 }
3485
otx2_ndc_sync(struct otx2_nic * pf)3486 static void otx2_ndc_sync(struct otx2_nic *pf)
3487 {
3488 struct mbox *mbox = &pf->mbox;
3489 struct ndc_sync_op *req;
3490
3491 mutex_lock(&mbox->lock);
3492
3493 req = otx2_mbox_alloc_msg_ndc_sync_op(mbox);
3494 if (!req) {
3495 mutex_unlock(&mbox->lock);
3496 return;
3497 }
3498
3499 req->nix_lf_tx_sync = 1;
3500 req->nix_lf_rx_sync = 1;
3501 req->npa_lf_sync = 1;
3502
3503 if (otx2_sync_mbox_msg(mbox))
3504 dev_err(pf->dev, "NDC sync operation failed\n");
3505
3506 mutex_unlock(&mbox->lock);
3507 }
3508
otx2_remove(struct pci_dev * pdev)3509 static void otx2_remove(struct pci_dev *pdev)
3510 {
3511 struct net_device *netdev = pci_get_drvdata(pdev);
3512 struct otx2_nic *pf;
3513
3514 if (!netdev)
3515 return;
3516
3517 pf = netdev_priv(netdev);
3518
3519 pf->flags |= OTX2_FLAG_PF_SHUTDOWN;
3520
3521 if (pf->flags & OTX2_FLAG_TX_TSTAMP_ENABLED)
3522 otx2_config_hw_tx_tstamp(pf, false);
3523 if (pf->flags & OTX2_FLAG_RX_TSTAMP_ENABLED)
3524 otx2_config_hw_rx_tstamp(pf, false);
3525
3526 /* Disable 802.3x pause frames */
3527 if (pf->flags & OTX2_FLAG_RX_PAUSE_ENABLED ||
3528 (pf->flags & OTX2_FLAG_TX_PAUSE_ENABLED)) {
3529 pf->flags &= ~OTX2_FLAG_RX_PAUSE_ENABLED;
3530 pf->flags &= ~OTX2_FLAG_TX_PAUSE_ENABLED;
3531 otx2_config_pause_frm(pf);
3532 }
3533
3534 #ifdef CONFIG_DCB
3535 /* Disable PFC config */
3536 if (pf->pfc_en) {
3537 pf->pfc_en = 0;
3538 otx2_config_priority_flow_ctrl(pf);
3539 }
3540 #endif
3541 cancel_work_sync(&pf->reset_task);
3542 /* Disable link notifications */
3543 otx2_cgx_config_linkevents(pf, false);
3544
3545 otx2_unregister_dl(pf);
3546 unregister_netdev(netdev);
3547 cn10k_ipsec_clean(pf);
3548 cn10k_mcs_free(pf);
3549 otx2_sriov_disable(pf->pdev);
3550 otx2_sriov_vfcfg_cleanup(pf);
3551 if (pf->otx2_wq)
3552 destroy_workqueue(pf->otx2_wq);
3553
3554 otx2_ptp_destroy(pf);
3555 otx2_mcam_flow_del(pf);
3556 otx2_shutdown_tc(pf);
3557 otx2_shutdown_qos(pf);
3558 otx2_ndc_sync(pf);
3559 otx2_detach_resources(&pf->mbox);
3560 if (pf->hw.lmt_info)
3561 free_percpu(pf->hw.lmt_info);
3562 if (test_bit(CN10K_LMTST, &pf->hw.cap_flag))
3563 qmem_free(pf->dev, pf->dync_lmt);
3564 otx2_disable_mbox_intr(pf);
3565 otx2_pfaf_mbox_destroy(pf);
3566 pci_free_irq_vectors(pf->pdev);
3567 bitmap_free(pf->af_xdp_zc_qidx);
3568 pci_set_drvdata(pdev, NULL);
3569 free_netdev(netdev);
3570 }
3571
3572 static struct pci_driver otx2_pf_driver = {
3573 .name = DRV_NAME,
3574 .id_table = otx2_pf_id_table,
3575 .probe = otx2_probe,
3576 .shutdown = otx2_remove,
3577 .remove = otx2_remove,
3578 .sriov_configure = otx2_sriov_configure
3579 };
3580
otx2_rvupf_init_module(void)3581 static int __init otx2_rvupf_init_module(void)
3582 {
3583 pr_info("%s: %s\n", DRV_NAME, DRV_STRING);
3584
3585 return pci_register_driver(&otx2_pf_driver);
3586 }
3587
otx2_rvupf_cleanup_module(void)3588 static void __exit otx2_rvupf_cleanup_module(void)
3589 {
3590 pci_unregister_driver(&otx2_pf_driver);
3591 }
3592
3593 module_init(otx2_rvupf_init_module);
3594 module_exit(otx2_rvupf_cleanup_module);
3595