1 // SPDX-License-Identifier: GPL-2.0
2 /* Marvell Octeon EP (EndPoint) Ethernet Driver
3 *
4 * Copyright (C) 2020 Marvell.
5 *
6 */
7
8 #include <linux/types.h>
9 #include <linux/module.h>
10 #include <linux/pci.h>
11 #include <linux/netdevice.h>
12 #include <linux/etherdevice.h>
13 #include <linux/rtnetlink.h>
14 #include <linux/vmalloc.h>
15
16 #include "octep_config.h"
17 #include "octep_main.h"
18 #include "octep_ctrl_net.h"
19 #include "octep_pfvf_mbox.h"
20
21 #define OCTEP_INTR_POLL_TIME_MSECS 100
22 struct workqueue_struct *octep_wq;
23
24 /* Supported Devices */
25 static const struct pci_device_id octep_pci_id_tbl[] = {
26 {PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, OCTEP_PCI_DEVICE_ID_CN98_PF)},
27 {PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, OCTEP_PCI_DEVICE_ID_CN93_PF)},
28 {PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, OCTEP_PCI_DEVICE_ID_CNF95N_PF)},
29 {PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, OCTEP_PCI_DEVICE_ID_CN10KA_PF)},
30 {PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, OCTEP_PCI_DEVICE_ID_CNF10KA_PF)},
31 {PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, OCTEP_PCI_DEVICE_ID_CNF10KB_PF)},
32 {PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, OCTEP_PCI_DEVICE_ID_CN10KB_PF)},
33 {0, },
34 };
35 MODULE_DEVICE_TABLE(pci, octep_pci_id_tbl);
36
37 MODULE_AUTHOR("Veerasenareddy Burru <vburru@marvell.com>");
38 MODULE_DESCRIPTION(OCTEP_DRV_STRING);
39 MODULE_LICENSE("GPL");
40
41 /**
42 * octep_alloc_ioq_vectors() - Allocate Tx/Rx Queue interrupt info.
43 *
44 * @oct: Octeon device private data structure.
45 *
46 * Allocate resources to hold per Tx/Rx queue interrupt info.
47 * This is the information passed to interrupt handler, from which napi poll
48 * is scheduled and includes quick access to private data of Tx/Rx queue
49 * corresponding to the interrupt being handled.
50 *
51 * Return: 0, on successful allocation of resources for all queue interrupts.
52 * -1, if failed to allocate any resource.
53 */
octep_alloc_ioq_vectors(struct octep_device * oct)54 static int octep_alloc_ioq_vectors(struct octep_device *oct)
55 {
56 int i;
57 struct octep_ioq_vector *ioq_vector;
58
59 for (i = 0; i < oct->num_oqs; i++) {
60 oct->ioq_vector[i] = vzalloc(sizeof(*oct->ioq_vector[i]));
61 if (!oct->ioq_vector[i])
62 goto free_ioq_vector;
63
64 ioq_vector = oct->ioq_vector[i];
65 ioq_vector->iq = oct->iq[i];
66 ioq_vector->oq = oct->oq[i];
67 ioq_vector->octep_dev = oct;
68 }
69
70 dev_info(&oct->pdev->dev, "Allocated %d IOQ vectors\n", oct->num_oqs);
71 return 0;
72
73 free_ioq_vector:
74 while (i) {
75 i--;
76 vfree(oct->ioq_vector[i]);
77 oct->ioq_vector[i] = NULL;
78 }
79 return -1;
80 }
81
82 /**
83 * octep_free_ioq_vectors() - Free Tx/Rx Queue interrupt vector info.
84 *
85 * @oct: Octeon device private data structure.
86 */
octep_free_ioq_vectors(struct octep_device * oct)87 static void octep_free_ioq_vectors(struct octep_device *oct)
88 {
89 int i;
90
91 for (i = 0; i < oct->num_oqs; i++) {
92 if (oct->ioq_vector[i]) {
93 vfree(oct->ioq_vector[i]);
94 oct->ioq_vector[i] = NULL;
95 }
96 }
97 netdev_info(oct->netdev, "Freed IOQ Vectors\n");
98 }
99
100 /**
101 * octep_enable_msix_range() - enable MSI-x interrupts.
102 *
103 * @oct: Octeon device private data structure.
104 *
105 * Allocate and enable all MSI-x interrupts (queue and non-queue interrupts)
106 * for the Octeon device.
107 *
108 * Return: 0, on successfully enabling all MSI-x interrupts.
109 * -1, if failed to enable any MSI-x interrupt.
110 */
octep_enable_msix_range(struct octep_device * oct)111 static int octep_enable_msix_range(struct octep_device *oct)
112 {
113 int num_msix, msix_allocated;
114 int i;
115
116 /* Generic interrupts apart from input/output queues */
117 num_msix = oct->num_oqs + CFG_GET_NON_IOQ_MSIX(oct->conf);
118 oct->msix_entries = kzalloc_objs(struct msix_entry, num_msix);
119 if (!oct->msix_entries)
120 goto msix_alloc_err;
121
122 for (i = 0; i < num_msix; i++)
123 oct->msix_entries[i].entry = i;
124
125 msix_allocated = pci_enable_msix_range(oct->pdev, oct->msix_entries,
126 num_msix, num_msix);
127 if (msix_allocated != num_msix) {
128 dev_err(&oct->pdev->dev,
129 "Failed to enable %d msix irqs; got only %d\n",
130 num_msix, msix_allocated);
131 goto enable_msix_err;
132 }
133 oct->num_irqs = msix_allocated;
134 dev_info(&oct->pdev->dev, "MSI-X enabled successfully\n");
135
136 return 0;
137
138 enable_msix_err:
139 if (msix_allocated > 0)
140 pci_disable_msix(oct->pdev);
141 kfree(oct->msix_entries);
142 oct->msix_entries = NULL;
143 msix_alloc_err:
144 return -1;
145 }
146
147 /**
148 * octep_disable_msix() - disable MSI-x interrupts.
149 *
150 * @oct: Octeon device private data structure.
151 *
152 * Disable MSI-x on the Octeon device.
153 */
octep_disable_msix(struct octep_device * oct)154 static void octep_disable_msix(struct octep_device *oct)
155 {
156 pci_disable_msix(oct->pdev);
157 kfree(oct->msix_entries);
158 oct->msix_entries = NULL;
159 dev_info(&oct->pdev->dev, "Disabled MSI-X\n");
160 }
161
162 /**
163 * octep_mbox_intr_handler() - common handler for pfvf mbox interrupts.
164 *
165 * @irq: Interrupt number.
166 * @data: interrupt data.
167 *
168 * this is common handler for pfvf mbox interrupts.
169 */
octep_mbox_intr_handler(int irq,void * data)170 static irqreturn_t octep_mbox_intr_handler(int irq, void *data)
171 {
172 struct octep_device *oct = data;
173
174 return oct->hw_ops.mbox_intr_handler(oct);
175 }
176
177 /**
178 * octep_oei_intr_handler() - common handler for output endpoint interrupts.
179 *
180 * @irq: Interrupt number.
181 * @data: interrupt data.
182 *
183 * this is common handler for all output endpoint interrupts.
184 */
octep_oei_intr_handler(int irq,void * data)185 static irqreturn_t octep_oei_intr_handler(int irq, void *data)
186 {
187 struct octep_device *oct = data;
188
189 return oct->hw_ops.oei_intr_handler(oct);
190 }
191
192 /**
193 * octep_ire_intr_handler() - common handler for input ring error interrupts.
194 *
195 * @irq: Interrupt number.
196 * @data: interrupt data.
197 *
198 * this is common handler for input ring error interrupts.
199 */
octep_ire_intr_handler(int irq,void * data)200 static irqreturn_t octep_ire_intr_handler(int irq, void *data)
201 {
202 struct octep_device *oct = data;
203
204 return oct->hw_ops.ire_intr_handler(oct);
205 }
206
207 /**
208 * octep_ore_intr_handler() - common handler for output ring error interrupts.
209 *
210 * @irq: Interrupt number.
211 * @data: interrupt data.
212 *
213 * this is common handler for output ring error interrupts.
214 */
octep_ore_intr_handler(int irq,void * data)215 static irqreturn_t octep_ore_intr_handler(int irq, void *data)
216 {
217 struct octep_device *oct = data;
218
219 return oct->hw_ops.ore_intr_handler(oct);
220 }
221
222 /**
223 * octep_vfire_intr_handler() - common handler for vf input ring error interrupts.
224 *
225 * @irq: Interrupt number.
226 * @data: interrupt data.
227 *
228 * this is common handler for vf input ring error interrupts.
229 */
octep_vfire_intr_handler(int irq,void * data)230 static irqreturn_t octep_vfire_intr_handler(int irq, void *data)
231 {
232 struct octep_device *oct = data;
233
234 return oct->hw_ops.vfire_intr_handler(oct);
235 }
236
237 /**
238 * octep_vfore_intr_handler() - common handler for vf output ring error interrupts.
239 *
240 * @irq: Interrupt number.
241 * @data: interrupt data.
242 *
243 * this is common handler for vf output ring error interrupts.
244 */
octep_vfore_intr_handler(int irq,void * data)245 static irqreturn_t octep_vfore_intr_handler(int irq, void *data)
246 {
247 struct octep_device *oct = data;
248
249 return oct->hw_ops.vfore_intr_handler(oct);
250 }
251
252 /**
253 * octep_dma_intr_handler() - common handler for dpi dma related interrupts.
254 *
255 * @irq: Interrupt number.
256 * @data: interrupt data.
257 *
258 * this is common handler for dpi dma related interrupts.
259 */
octep_dma_intr_handler(int irq,void * data)260 static irqreturn_t octep_dma_intr_handler(int irq, void *data)
261 {
262 struct octep_device *oct = data;
263
264 return oct->hw_ops.dma_intr_handler(oct);
265 }
266
267 /**
268 * octep_dma_vf_intr_handler() - common handler for dpi dma transaction error interrupts for VFs.
269 *
270 * @irq: Interrupt number.
271 * @data: interrupt data.
272 *
273 * this is common handler for dpi dma transaction error interrupts for VFs.
274 */
octep_dma_vf_intr_handler(int irq,void * data)275 static irqreturn_t octep_dma_vf_intr_handler(int irq, void *data)
276 {
277 struct octep_device *oct = data;
278
279 return oct->hw_ops.dma_vf_intr_handler(oct);
280 }
281
282 /**
283 * octep_pp_vf_intr_handler() - common handler for pp transaction error interrupts for VFs.
284 *
285 * @irq: Interrupt number.
286 * @data: interrupt data.
287 *
288 * this is common handler for pp transaction error interrupts for VFs.
289 */
octep_pp_vf_intr_handler(int irq,void * data)290 static irqreturn_t octep_pp_vf_intr_handler(int irq, void *data)
291 {
292 struct octep_device *oct = data;
293
294 return oct->hw_ops.pp_vf_intr_handler(oct);
295 }
296
297 /**
298 * octep_misc_intr_handler() - common handler for mac related interrupts.
299 *
300 * @irq: Interrupt number.
301 * @data: interrupt data.
302 *
303 * this is common handler for mac related interrupts.
304 */
octep_misc_intr_handler(int irq,void * data)305 static irqreturn_t octep_misc_intr_handler(int irq, void *data)
306 {
307 struct octep_device *oct = data;
308
309 return oct->hw_ops.misc_intr_handler(oct);
310 }
311
312 /**
313 * octep_rsvd_intr_handler() - common handler for reserved interrupts (future use).
314 *
315 * @irq: Interrupt number.
316 * @data: interrupt data.
317 *
318 * this is common handler for all reserved interrupts.
319 */
octep_rsvd_intr_handler(int irq,void * data)320 static irqreturn_t octep_rsvd_intr_handler(int irq, void *data)
321 {
322 struct octep_device *oct = data;
323
324 return oct->hw_ops.rsvd_intr_handler(oct);
325 }
326
327 /**
328 * octep_ioq_intr_handler() - handler for all Tx/Rx queue interrupts.
329 *
330 * @irq: Interrupt number.
331 * @data: interrupt data contains pointers to Tx/Rx queue private data
332 * and correspong NAPI context.
333 *
334 * this is common handler for all non-queue (generic) interrupts.
335 */
octep_ioq_intr_handler(int irq,void * data)336 static irqreturn_t octep_ioq_intr_handler(int irq, void *data)
337 {
338 struct octep_ioq_vector *ioq_vector = data;
339 struct octep_device *oct = ioq_vector->octep_dev;
340
341 return oct->hw_ops.ioq_intr_handler(ioq_vector);
342 }
343
344 /**
345 * octep_request_irqs() - Register interrupt handlers.
346 *
347 * @oct: Octeon device private data structure.
348 *
349 * Register handlers for all queue and non-queue interrupts.
350 *
351 * Return: 0, on successful registration of all interrupt handlers.
352 * -1, on any error.
353 */
octep_request_irqs(struct octep_device * oct)354 static int octep_request_irqs(struct octep_device *oct)
355 {
356 struct net_device *netdev = oct->netdev;
357 struct octep_ioq_vector *ioq_vector;
358 struct msix_entry *msix_entry;
359 char **non_ioq_msix_names;
360 int num_non_ioq_msix;
361 int ret, i, j;
362
363 num_non_ioq_msix = CFG_GET_NON_IOQ_MSIX(oct->conf);
364 non_ioq_msix_names = CFG_GET_NON_IOQ_MSIX_NAMES(oct->conf);
365
366 oct->non_ioq_irq_names = kcalloc(num_non_ioq_msix,
367 OCTEP_MSIX_NAME_SIZE, GFP_KERNEL);
368 if (!oct->non_ioq_irq_names)
369 goto alloc_err;
370
371 /* First few MSI-X interrupts are non-queue interrupts */
372 for (i = 0; i < num_non_ioq_msix; i++) {
373 char *irq_name;
374
375 irq_name = &oct->non_ioq_irq_names[i * OCTEP_MSIX_NAME_SIZE];
376 msix_entry = &oct->msix_entries[i];
377
378 snprintf(irq_name, OCTEP_MSIX_NAME_SIZE,
379 "%s-%s", netdev->name, non_ioq_msix_names[i]);
380 if (!strncmp(non_ioq_msix_names[i], "epf_mbox_rint", strlen("epf_mbox_rint"))) {
381 ret = request_irq(msix_entry->vector,
382 octep_mbox_intr_handler, 0,
383 irq_name, oct);
384 } else if (!strncmp(non_ioq_msix_names[i], "epf_oei_rint",
385 strlen("epf_oei_rint"))) {
386 ret = request_irq(msix_entry->vector,
387 octep_oei_intr_handler, 0,
388 irq_name, oct);
389 } else if (!strncmp(non_ioq_msix_names[i], "epf_ire_rint",
390 strlen("epf_ire_rint"))) {
391 ret = request_irq(msix_entry->vector,
392 octep_ire_intr_handler, 0,
393 irq_name, oct);
394 } else if (!strncmp(non_ioq_msix_names[i], "epf_ore_rint",
395 strlen("epf_ore_rint"))) {
396 ret = request_irq(msix_entry->vector,
397 octep_ore_intr_handler, 0,
398 irq_name, oct);
399 } else if (!strncmp(non_ioq_msix_names[i], "epf_vfire_rint",
400 strlen("epf_vfire_rint"))) {
401 ret = request_irq(msix_entry->vector,
402 octep_vfire_intr_handler, 0,
403 irq_name, oct);
404 } else if (!strncmp(non_ioq_msix_names[i], "epf_vfore_rint",
405 strlen("epf_vfore_rint"))) {
406 ret = request_irq(msix_entry->vector,
407 octep_vfore_intr_handler, 0,
408 irq_name, oct);
409 } else if (!strncmp(non_ioq_msix_names[i], "epf_dma_rint",
410 strlen("epf_dma_rint"))) {
411 ret = request_irq(msix_entry->vector,
412 octep_dma_intr_handler, 0,
413 irq_name, oct);
414 } else if (!strncmp(non_ioq_msix_names[i], "epf_dma_vf_rint",
415 strlen("epf_dma_vf_rint"))) {
416 ret = request_irq(msix_entry->vector,
417 octep_dma_vf_intr_handler, 0,
418 irq_name, oct);
419 } else if (!strncmp(non_ioq_msix_names[i], "epf_pp_vf_rint",
420 strlen("epf_pp_vf_rint"))) {
421 ret = request_irq(msix_entry->vector,
422 octep_pp_vf_intr_handler, 0,
423 irq_name, oct);
424 } else if (!strncmp(non_ioq_msix_names[i], "epf_misc_rint",
425 strlen("epf_misc_rint"))) {
426 ret = request_irq(msix_entry->vector,
427 octep_misc_intr_handler, 0,
428 irq_name, oct);
429 } else {
430 ret = request_irq(msix_entry->vector,
431 octep_rsvd_intr_handler, 0,
432 irq_name, oct);
433 }
434
435 if (ret) {
436 netdev_err(netdev,
437 "request_irq failed for %s; err=%d",
438 irq_name, ret);
439 goto non_ioq_irq_err;
440 }
441 }
442
443 /* Request IRQs for Tx/Rx queues */
444 for (j = 0; j < oct->num_oqs; j++) {
445 ioq_vector = oct->ioq_vector[j];
446 msix_entry = &oct->msix_entries[j + num_non_ioq_msix];
447
448 snprintf(ioq_vector->name, sizeof(ioq_vector->name),
449 "%s-q%d", netdev->name, j);
450 ret = request_irq(msix_entry->vector,
451 octep_ioq_intr_handler, 0,
452 ioq_vector->name, ioq_vector);
453 if (ret) {
454 netdev_err(netdev,
455 "request_irq failed for Q-%d; err=%d",
456 j, ret);
457 goto ioq_irq_err;
458 }
459
460 cpumask_set_cpu(j % num_online_cpus(),
461 &ioq_vector->affinity_mask);
462 irq_set_affinity_hint(msix_entry->vector,
463 &ioq_vector->affinity_mask);
464 }
465
466 return 0;
467 ioq_irq_err:
468 while (j) {
469 --j;
470 ioq_vector = oct->ioq_vector[j];
471 msix_entry = &oct->msix_entries[j + num_non_ioq_msix];
472
473 irq_set_affinity_hint(msix_entry->vector, NULL);
474 free_irq(msix_entry->vector, ioq_vector);
475 }
476 non_ioq_irq_err:
477 while (i) {
478 --i;
479 free_irq(oct->msix_entries[i].vector, oct);
480 }
481 kfree(oct->non_ioq_irq_names);
482 oct->non_ioq_irq_names = NULL;
483 alloc_err:
484 return -1;
485 }
486
487 /**
488 * octep_free_irqs() - free all registered interrupts.
489 *
490 * @oct: Octeon device private data structure.
491 *
492 * Free all queue and non-queue interrupts of the Octeon device.
493 */
octep_free_irqs(struct octep_device * oct)494 static void octep_free_irqs(struct octep_device *oct)
495 {
496 int i;
497
498 /* First few MSI-X interrupts are non queue interrupts; free them */
499 for (i = 0; i < CFG_GET_NON_IOQ_MSIX(oct->conf); i++)
500 free_irq(oct->msix_entries[i].vector, oct);
501 kfree(oct->non_ioq_irq_names);
502
503 /* Free IRQs for Input/Output (Tx/Rx) queues */
504 for (i = CFG_GET_NON_IOQ_MSIX(oct->conf); i < oct->num_irqs; i++) {
505 irq_set_affinity_hint(oct->msix_entries[i].vector, NULL);
506 free_irq(oct->msix_entries[i].vector,
507 oct->ioq_vector[i - CFG_GET_NON_IOQ_MSIX(oct->conf)]);
508 }
509 netdev_info(oct->netdev, "IRQs freed\n");
510 }
511
512 /**
513 * octep_setup_irqs() - setup interrupts for the Octeon device.
514 *
515 * @oct: Octeon device private data structure.
516 *
517 * Allocate data structures to hold per interrupt information, allocate/enable
518 * MSI-x interrupt and register interrupt handlers.
519 *
520 * Return: 0, on successful allocation and registration of all interrupts.
521 * -1, on any error.
522 */
octep_setup_irqs(struct octep_device * oct)523 static int octep_setup_irqs(struct octep_device *oct)
524 {
525 if (octep_alloc_ioq_vectors(oct))
526 goto ioq_vector_err;
527
528 if (octep_enable_msix_range(oct))
529 goto enable_msix_err;
530
531 if (octep_request_irqs(oct))
532 goto request_irq_err;
533
534 return 0;
535
536 request_irq_err:
537 octep_disable_msix(oct);
538 enable_msix_err:
539 octep_free_ioq_vectors(oct);
540 ioq_vector_err:
541 return -1;
542 }
543
544 /**
545 * octep_clean_irqs() - free all interrupts and its resources.
546 *
547 * @oct: Octeon device private data structure.
548 */
octep_clean_irqs(struct octep_device * oct)549 static void octep_clean_irqs(struct octep_device *oct)
550 {
551 octep_free_irqs(oct);
552 octep_disable_msix(oct);
553 octep_free_ioq_vectors(oct);
554 }
555
556 /**
557 * octep_update_pkt() - Update IQ/OQ IN/OUT_CNT registers.
558 *
559 * @iq: Octeon Tx queue data structure.
560 * @oq: Octeon Rx queue data structure.
561 */
octep_update_pkt(struct octep_iq * iq,struct octep_oq * oq)562 static void octep_update_pkt(struct octep_iq *iq, struct octep_oq *oq)
563 {
564 u32 pkts_pend = READ_ONCE(oq->pkts_pending);
565 u32 last_pkt_count = READ_ONCE(oq->last_pkt_count);
566 u32 pkts_processed = READ_ONCE(iq->pkts_processed);
567 u32 pkt_in_done = READ_ONCE(iq->pkt_in_done);
568
569 netdev_dbg(iq->netdev, "enabling intr for Q-%u\n", iq->q_no);
570 if (pkts_processed) {
571 writel(pkts_processed, iq->inst_cnt_reg);
572 readl(iq->inst_cnt_reg);
573 WRITE_ONCE(iq->pkt_in_done, (pkt_in_done - pkts_processed));
574 WRITE_ONCE(iq->pkts_processed, 0);
575 }
576 if (last_pkt_count - pkts_pend) {
577 writel(last_pkt_count - pkts_pend, oq->pkts_sent_reg);
578 readl(oq->pkts_sent_reg);
579 WRITE_ONCE(oq->last_pkt_count, pkts_pend);
580 }
581
582 /* Flush the previous wrties before writing to RESEND bit */
583 smp_wmb();
584 }
585
586 /**
587 * octep_enable_ioq_irq() - Enable MSI-x interrupt of a Tx/Rx queue.
588 *
589 * @iq: Octeon Tx queue data structure.
590 * @oq: Octeon Rx queue data structure.
591 */
octep_enable_ioq_irq(struct octep_iq * iq,struct octep_oq * oq)592 static void octep_enable_ioq_irq(struct octep_iq *iq, struct octep_oq *oq)
593 {
594 writeq(1UL << OCTEP_OQ_INTR_RESEND_BIT, oq->pkts_sent_reg);
595 writeq(1UL << OCTEP_IQ_INTR_RESEND_BIT, iq->inst_cnt_reg);
596 }
597
598 /**
599 * octep_napi_poll() - NAPI poll function for Tx/Rx.
600 *
601 * @napi: pointer to napi context.
602 * @budget: max number of packets to be processed in single invocation.
603 */
octep_napi_poll(struct napi_struct * napi,int budget)604 static int octep_napi_poll(struct napi_struct *napi, int budget)
605 {
606 struct octep_ioq_vector *ioq_vector =
607 container_of(napi, struct octep_ioq_vector, napi);
608 u32 tx_pending, rx_done;
609
610 tx_pending = octep_iq_process_completions(ioq_vector->iq, budget);
611 rx_done = octep_oq_process_rx(ioq_vector->oq, budget);
612
613 /* need more polling if tx completion processing is still pending or
614 * processed at least 'budget' number of rx packets.
615 */
616 if (tx_pending || rx_done >= budget)
617 return budget;
618
619 octep_update_pkt(ioq_vector->iq, ioq_vector->oq);
620 napi_complete_done(napi, rx_done);
621 octep_enable_ioq_irq(ioq_vector->iq, ioq_vector->oq);
622 return rx_done;
623 }
624
625 /**
626 * octep_napi_add() - Add NAPI poll for all Tx/Rx queues.
627 *
628 * @oct: Octeon device private data structure.
629 */
octep_napi_add(struct octep_device * oct)630 static void octep_napi_add(struct octep_device *oct)
631 {
632 int i;
633
634 for (i = 0; i < oct->num_oqs; i++) {
635 netdev_dbg(oct->netdev, "Adding NAPI on Q-%d\n", i);
636 netif_napi_add(oct->netdev, &oct->ioq_vector[i]->napi,
637 octep_napi_poll);
638 oct->oq[i]->napi = &oct->ioq_vector[i]->napi;
639 }
640 }
641
642 /**
643 * octep_napi_delete() - delete NAPI poll callback for all Tx/Rx queues.
644 *
645 * @oct: Octeon device private data structure.
646 */
octep_napi_delete(struct octep_device * oct)647 static void octep_napi_delete(struct octep_device *oct)
648 {
649 int i;
650
651 for (i = 0; i < oct->num_oqs; i++) {
652 netdev_dbg(oct->netdev, "Deleting NAPI on Q-%d\n", i);
653 netif_napi_del(&oct->ioq_vector[i]->napi);
654 oct->oq[i]->napi = NULL;
655 }
656 }
657
658 /**
659 * octep_napi_enable() - enable NAPI for all Tx/Rx queues.
660 *
661 * @oct: Octeon device private data structure.
662 */
octep_napi_enable(struct octep_device * oct)663 static void octep_napi_enable(struct octep_device *oct)
664 {
665 int i;
666
667 for (i = 0; i < oct->num_oqs; i++) {
668 netdev_dbg(oct->netdev, "Enabling NAPI on Q-%d\n", i);
669 napi_enable(&oct->ioq_vector[i]->napi);
670 }
671 }
672
673 /**
674 * octep_napi_disable() - disable NAPI for all Tx/Rx queues.
675 *
676 * @oct: Octeon device private data structure.
677 */
octep_napi_disable(struct octep_device * oct)678 static void octep_napi_disable(struct octep_device *oct)
679 {
680 int i;
681
682 for (i = 0; i < oct->num_oqs; i++) {
683 netdev_dbg(oct->netdev, "Disabling NAPI on Q-%d\n", i);
684 napi_disable(&oct->ioq_vector[i]->napi);
685 }
686 }
687
octep_link_up(struct net_device * netdev)688 static void octep_link_up(struct net_device *netdev)
689 {
690 netif_carrier_on(netdev);
691 netif_tx_start_all_queues(netdev);
692 }
693
694 /**
695 * octep_open() - start the octeon network device.
696 *
697 * @netdev: pointer to kernel network device.
698 *
699 * setup Tx/Rx queues, interrupts and enable hardware operation of Tx/Rx queues
700 * and interrupts..
701 *
702 * Return: 0, on successfully setting up device and bring it up.
703 * -1, on any error.
704 */
octep_open(struct net_device * netdev)705 static int octep_open(struct net_device *netdev)
706 {
707 struct octep_device *oct = netdev_priv(netdev);
708 int err, ret;
709
710 netdev_info(netdev, "Starting netdev ...\n");
711 netif_carrier_off(netdev);
712
713 oct->hw_ops.reset_io_queues(oct);
714
715 if (octep_setup_iqs(oct))
716 goto setup_iq_err;
717 if (octep_setup_oqs(oct))
718 goto setup_oq_err;
719 if (octep_setup_irqs(oct))
720 goto setup_irq_err;
721
722 err = netif_set_real_num_tx_queues(netdev, oct->num_oqs);
723 if (err)
724 goto set_queues_err;
725 err = netif_set_real_num_rx_queues(netdev, oct->num_iqs);
726 if (err)
727 goto set_queues_err;
728
729 octep_napi_add(oct);
730 octep_napi_enable(oct);
731
732 oct->link_info.admin_up = 1;
733 octep_ctrl_net_set_rx_state(oct, OCTEP_CTRL_NET_INVALID_VFID, true,
734 false);
735 octep_ctrl_net_set_link_status(oct, OCTEP_CTRL_NET_INVALID_VFID, true,
736 false);
737 oct->poll_non_ioq_intr = false;
738
739 /* Enable the input and output queues for this Octeon device */
740 oct->hw_ops.enable_io_queues(oct);
741
742 /* Enable Octeon device interrupts */
743 oct->hw_ops.enable_interrupts(oct);
744
745 octep_oq_dbell_init(oct);
746
747 ret = octep_ctrl_net_get_link_status(oct, OCTEP_CTRL_NET_INVALID_VFID);
748 if (ret > 0)
749 octep_link_up(netdev);
750
751 return 0;
752
753 set_queues_err:
754 octep_clean_irqs(oct);
755 setup_irq_err:
756 octep_free_oqs(oct);
757 setup_oq_err:
758 octep_free_iqs(oct);
759 setup_iq_err:
760 return -1;
761 }
762
763 /**
764 * octep_stop() - stop the octeon network device.
765 *
766 * @netdev: pointer to kernel network device.
767 *
768 * stop the device Tx/Rx operations, bring down the link and
769 * free up all resources allocated for Tx/Rx queues and interrupts.
770 */
octep_stop(struct net_device * netdev)771 static int octep_stop(struct net_device *netdev)
772 {
773 struct octep_device *oct = netdev_priv(netdev);
774
775 netdev_info(netdev, "Stopping the device ...\n");
776
777 octep_ctrl_net_set_link_status(oct, OCTEP_CTRL_NET_INVALID_VFID, false,
778 false);
779 octep_ctrl_net_set_rx_state(oct, OCTEP_CTRL_NET_INVALID_VFID, false,
780 false);
781
782 /* Stop Tx from stack */
783 netif_tx_stop_all_queues(netdev);
784 netif_carrier_off(netdev);
785 netif_tx_disable(netdev);
786
787 oct->link_info.admin_up = 0;
788 oct->link_info.oper_up = 0;
789
790 oct->hw_ops.disable_interrupts(oct);
791 octep_napi_disable(oct);
792 octep_napi_delete(oct);
793
794 octep_clean_irqs(oct);
795 octep_clean_iqs(oct);
796
797 oct->hw_ops.disable_io_queues(oct);
798 oct->hw_ops.reset_io_queues(oct);
799 octep_free_oqs(oct);
800 octep_free_iqs(oct);
801
802 oct->poll_non_ioq_intr = true;
803 queue_delayed_work(octep_wq, &oct->intr_poll_task,
804 msecs_to_jiffies(OCTEP_INTR_POLL_TIME_MSECS));
805
806 netdev_info(netdev, "Device stopped !!\n");
807 return 0;
808 }
809
810 /**
811 * octep_iq_full_check() - check if a Tx queue is full.
812 *
813 * @iq: Octeon Tx queue data structure.
814 *
815 * Return: 0, if the Tx queue is not full.
816 * 1, if the Tx queue is full.
817 */
octep_iq_full_check(struct octep_iq * iq)818 static inline int octep_iq_full_check(struct octep_iq *iq)
819 {
820 if (likely((IQ_INSTR_SPACE(iq)) >
821 OCTEP_WAKE_QUEUE_THRESHOLD))
822 return 0;
823
824 /* Stop the queue if unable to send */
825 netif_stop_subqueue(iq->netdev, iq->q_no);
826
827 /* Allow for pending updates in write index
828 * from iq_process_completion in other cpus
829 * to reflect, in case queue gets free
830 * entries.
831 */
832 smp_mb();
833
834 /* check again and restart the queue, in case NAPI has just freed
835 * enough Tx ring entries.
836 */
837 if (unlikely(IQ_INSTR_SPACE(iq) >
838 OCTEP_WAKE_QUEUE_THRESHOLD)) {
839 netif_start_subqueue(iq->netdev, iq->q_no);
840 iq->stats->restart_cnt++;
841 return 0;
842 }
843
844 return 1;
845 }
846
847 /**
848 * octep_start_xmit() - Enqueue packet to Octoen hardware Tx Queue.
849 *
850 * @skb: packet skbuff pointer.
851 * @netdev: kernel network device.
852 *
853 * Return: NETDEV_TX_BUSY, if Tx Queue is full.
854 * NETDEV_TX_OK, if successfully enqueued to hardware Tx queue.
855 */
octep_start_xmit(struct sk_buff * skb,struct net_device * netdev)856 static netdev_tx_t octep_start_xmit(struct sk_buff *skb,
857 struct net_device *netdev)
858 {
859 struct octep_device *oct = netdev_priv(netdev);
860 netdev_features_t feat = netdev->features;
861 struct octep_tx_sglist_desc *sglist;
862 struct octep_tx_buffer *tx_buffer;
863 struct octep_tx_desc_hw *hw_desc;
864 struct skb_shared_info *shinfo;
865 struct octep_instr_hdr *ih;
866 struct octep_iq *iq;
867 skb_frag_t *frag;
868 u16 nr_frags, si;
869 int xmit_more;
870 u16 q_no, wi;
871
872 if (skb_put_padto(skb, ETH_ZLEN))
873 return NETDEV_TX_OK;
874
875 q_no = skb_get_queue_mapping(skb);
876 if (q_no >= oct->num_iqs) {
877 netdev_err(netdev, "Invalid Tx skb->queue_mapping=%d\n", q_no);
878 q_no = q_no % oct->num_iqs;
879 }
880
881 iq = oct->iq[q_no];
882
883 shinfo = skb_shinfo(skb);
884 nr_frags = shinfo->nr_frags;
885
886 wi = iq->host_write_index;
887 hw_desc = &iq->desc_ring[wi];
888 hw_desc->ih64 = 0;
889
890 tx_buffer = iq->buff_info + wi;
891 tx_buffer->skb = skb;
892
893 ih = &hw_desc->ih;
894 ih->pkind = oct->conf->fw_info.pkind;
895 ih->fsz = oct->conf->fw_info.fsz;
896 ih->tlen = skb->len + ih->fsz;
897
898 if (!nr_frags) {
899 tx_buffer->gather = 0;
900 tx_buffer->dma = dma_map_single(iq->dev, skb->data,
901 skb->len, DMA_TO_DEVICE);
902 if (dma_mapping_error(iq->dev, tx_buffer->dma))
903 goto dma_map_err;
904 hw_desc->dptr = tx_buffer->dma;
905 } else {
906 /* Scatter/Gather */
907 dma_addr_t dma;
908 u16 len;
909
910 sglist = tx_buffer->sglist;
911
912 ih->gsz = nr_frags + 1;
913 ih->gather = 1;
914 tx_buffer->gather = 1;
915
916 len = skb_headlen(skb);
917 dma = dma_map_single(iq->dev, skb->data, len, DMA_TO_DEVICE);
918 if (dma_mapping_error(iq->dev, dma))
919 goto dma_map_err;
920
921 memset(sglist, 0, OCTEP_SGLIST_SIZE_PER_PKT);
922 sglist[0].len[3] = len;
923 sglist[0].dma_ptr[0] = dma;
924
925 si = 1; /* entry 0 is main skb, mapped above */
926 frag = &shinfo->frags[0];
927 while (nr_frags--) {
928 len = skb_frag_size(frag);
929 dma = skb_frag_dma_map(iq->dev, frag, 0,
930 len, DMA_TO_DEVICE);
931 if (dma_mapping_error(iq->dev, dma))
932 goto dma_map_sg_err;
933
934 sglist[si >> 2].len[3 - (si & 3)] = len;
935 sglist[si >> 2].dma_ptr[si & 3] = dma;
936
937 frag++;
938 si++;
939 }
940 hw_desc->dptr = tx_buffer->sglist_dma;
941 }
942
943 if (oct->conf->fw_info.tx_ol_flags) {
944 if ((feat & (NETIF_F_TSO)) && (skb_is_gso(skb))) {
945 hw_desc->txm.ol_flags = OCTEP_TX_OFFLOAD_CKSUM;
946 hw_desc->txm.ol_flags |= OCTEP_TX_OFFLOAD_TSO;
947 hw_desc->txm.gso_size = skb_shinfo(skb)->gso_size;
948 hw_desc->txm.gso_segs = skb_shinfo(skb)->gso_segs;
949 } else if (feat & (NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM)) {
950 hw_desc->txm.ol_flags = OCTEP_TX_OFFLOAD_CKSUM;
951 }
952 /* due to ESR txm will be swapped by hw */
953 hw_desc->txm64[0] = (__force u64)cpu_to_be64(hw_desc->txm64[0]);
954 }
955
956 xmit_more = netdev_xmit_more();
957
958 __netdev_tx_sent_queue(iq->netdev_q, skb->len, xmit_more);
959
960 skb_tx_timestamp(skb);
961 iq->fill_cnt++;
962 wi++;
963 iq->host_write_index = wi & iq->ring_size_mask;
964
965 /* octep_iq_full_check stops the queue and returns
966 * true if so, in case the queue has become full
967 * by inserting current packet. If so, we can
968 * go ahead and ring doorbell.
969 */
970 if (!octep_iq_full_check(iq) && xmit_more &&
971 iq->fill_cnt < iq->fill_threshold)
972 return NETDEV_TX_OK;
973
974 /* Flush the hw descriptor before writing to doorbell */
975 wmb();
976 /* Ring Doorbell to notify the NIC of new packets */
977 writel(iq->fill_cnt, iq->doorbell_reg);
978 iq->stats->instr_posted += iq->fill_cnt;
979 iq->fill_cnt = 0;
980 return NETDEV_TX_OK;
981
982 dma_map_sg_err:
983 if (si > 0) {
984 dma_unmap_single(iq->dev, sglist[0].dma_ptr[0],
985 sglist[0].len[3], DMA_TO_DEVICE);
986 sglist[0].len[3] = 0;
987 }
988 while (si > 1) {
989 dma_unmap_page(iq->dev, sglist[si >> 2].dma_ptr[si & 3],
990 sglist[si >> 2].len[3 - (si & 3)], DMA_TO_DEVICE);
991 sglist[si >> 2].len[3 - (si & 3)] = 0;
992 si--;
993 }
994 tx_buffer->gather = 0;
995 dma_map_err:
996 dev_kfree_skb_any(skb);
997 return NETDEV_TX_OK;
998 }
999
1000 /**
1001 * octep_get_stats64() - Get Octeon network device statistics.
1002 *
1003 * @netdev: kernel network device.
1004 * @stats: pointer to stats structure to be filled in.
1005 */
octep_get_stats64(struct net_device * netdev,struct rtnl_link_stats64 * stats)1006 static void octep_get_stats64(struct net_device *netdev,
1007 struct rtnl_link_stats64 *stats)
1008 {
1009 struct octep_device *oct = netdev_priv(netdev);
1010 u64 tx_packets, tx_bytes, rx_packets, rx_bytes;
1011 int q;
1012
1013 tx_packets = 0;
1014 tx_bytes = 0;
1015 rx_packets = 0;
1016 rx_bytes = 0;
1017 for (q = 0; q < OCTEP_MAX_QUEUES; q++) {
1018 tx_packets += oct->stats_iq[q].instr_completed;
1019 tx_bytes += oct->stats_iq[q].bytes_sent;
1020 rx_packets += oct->stats_oq[q].packets;
1021 rx_bytes += oct->stats_oq[q].bytes;
1022 }
1023 stats->tx_packets = tx_packets;
1024 stats->tx_bytes = tx_bytes;
1025 stats->rx_packets = rx_packets;
1026 stats->rx_bytes = rx_bytes;
1027 }
1028
1029 /**
1030 * octep_tx_timeout_task - work queue task to Handle Tx queue timeout.
1031 *
1032 * @work: pointer to Tx queue timeout work_struct
1033 *
1034 * Stop and start the device so that it frees up all queue resources
1035 * and restarts the queues, that potentially clears a Tx queue timeout
1036 * condition.
1037 **/
octep_tx_timeout_task(struct work_struct * work)1038 static void octep_tx_timeout_task(struct work_struct *work)
1039 {
1040 struct octep_device *oct = container_of(work, struct octep_device,
1041 tx_timeout_task);
1042 struct net_device *netdev = oct->netdev;
1043
1044 rtnl_lock();
1045 if (netif_running(netdev)) {
1046 octep_stop(netdev);
1047 octep_open(netdev);
1048 }
1049 rtnl_unlock();
1050 }
1051
1052 /**
1053 * octep_tx_timeout() - Handle Tx Queue timeout.
1054 *
1055 * @netdev: pointer to kernel network device.
1056 * @txqueue: Timed out Tx queue number.
1057 *
1058 * Schedule a work to handle Tx queue timeout.
1059 */
octep_tx_timeout(struct net_device * netdev,unsigned int txqueue)1060 static void octep_tx_timeout(struct net_device *netdev, unsigned int txqueue)
1061 {
1062 struct octep_device *oct = netdev_priv(netdev);
1063
1064 queue_work(octep_wq, &oct->tx_timeout_task);
1065 }
1066
octep_set_mac(struct net_device * netdev,void * p)1067 static int octep_set_mac(struct net_device *netdev, void *p)
1068 {
1069 struct octep_device *oct = netdev_priv(netdev);
1070 struct sockaddr *addr = (struct sockaddr *)p;
1071 int err;
1072
1073 if (!is_valid_ether_addr(addr->sa_data))
1074 return -EADDRNOTAVAIL;
1075
1076 err = octep_ctrl_net_set_mac_addr(oct, OCTEP_CTRL_NET_INVALID_VFID,
1077 addr->sa_data, true);
1078 if (err)
1079 return err;
1080
1081 memcpy(oct->mac_addr, addr->sa_data, ETH_ALEN);
1082 eth_hw_addr_set(netdev, addr->sa_data);
1083
1084 return 0;
1085 }
1086
octep_change_mtu(struct net_device * netdev,int new_mtu)1087 static int octep_change_mtu(struct net_device *netdev, int new_mtu)
1088 {
1089 struct octep_device *oct = netdev_priv(netdev);
1090 struct octep_iface_link_info *link_info;
1091 int err = 0;
1092
1093 link_info = &oct->link_info;
1094 if (link_info->mtu == new_mtu)
1095 return 0;
1096
1097 err = octep_ctrl_net_set_mtu(oct, OCTEP_CTRL_NET_INVALID_VFID, new_mtu,
1098 true);
1099 if (!err) {
1100 oct->link_info.mtu = new_mtu;
1101 WRITE_ONCE(netdev->mtu, new_mtu);
1102 }
1103
1104 return err;
1105 }
1106
octep_set_features(struct net_device * dev,netdev_features_t features)1107 static int octep_set_features(struct net_device *dev, netdev_features_t features)
1108 {
1109 struct octep_ctrl_net_offloads offloads = { 0 };
1110 struct octep_device *oct = netdev_priv(dev);
1111 int err;
1112
1113 /* We only support features received from firmware */
1114 if ((features & dev->hw_features) != features)
1115 return -EINVAL;
1116
1117 if (features & NETIF_F_TSO)
1118 offloads.tx_offloads |= OCTEP_TX_OFFLOAD_TSO;
1119
1120 if (features & NETIF_F_TSO6)
1121 offloads.tx_offloads |= OCTEP_TX_OFFLOAD_TSO;
1122
1123 if (features & NETIF_F_IP_CSUM)
1124 offloads.tx_offloads |= OCTEP_TX_OFFLOAD_CKSUM;
1125
1126 if (features & NETIF_F_IPV6_CSUM)
1127 offloads.tx_offloads |= OCTEP_TX_OFFLOAD_CKSUM;
1128
1129 if (features & NETIF_F_RXCSUM)
1130 offloads.rx_offloads |= OCTEP_RX_OFFLOAD_CKSUM;
1131
1132 err = octep_ctrl_net_set_offloads(oct,
1133 OCTEP_CTRL_NET_INVALID_VFID,
1134 &offloads,
1135 true);
1136 if (!err)
1137 dev->features = features;
1138
1139 return err;
1140 }
1141
octep_is_vf_valid(struct octep_device * oct,int vf)1142 static bool octep_is_vf_valid(struct octep_device *oct, int vf)
1143 {
1144 if (vf >= CFG_GET_ACTIVE_VFS(oct->conf)) {
1145 netdev_err(oct->netdev, "Invalid VF ID %d\n", vf);
1146 return false;
1147 }
1148
1149 return true;
1150 }
1151
octep_get_vf_config(struct net_device * dev,int vf,struct ifla_vf_info * ivi)1152 static int octep_get_vf_config(struct net_device *dev, int vf,
1153 struct ifla_vf_info *ivi)
1154 {
1155 struct octep_device *oct = netdev_priv(dev);
1156
1157 if (!octep_is_vf_valid(oct, vf))
1158 return -EINVAL;
1159
1160 ivi->vf = vf;
1161 ether_addr_copy(ivi->mac, oct->vf_info[vf].mac_addr);
1162 ivi->spoofchk = true;
1163 ivi->linkstate = IFLA_VF_LINK_STATE_ENABLE;
1164 ivi->trusted = false;
1165
1166 return 0;
1167 }
1168
octep_set_vf_mac(struct net_device * dev,int vf,u8 * mac)1169 static int octep_set_vf_mac(struct net_device *dev, int vf, u8 *mac)
1170 {
1171 struct octep_device *oct = netdev_priv(dev);
1172 int err;
1173
1174 if (!octep_is_vf_valid(oct, vf))
1175 return -EINVAL;
1176
1177 if (!is_valid_ether_addr(mac)) {
1178 dev_err(&oct->pdev->dev, "Invalid MAC Address %pM\n", mac);
1179 return -EADDRNOTAVAIL;
1180 }
1181
1182 dev_dbg(&oct->pdev->dev, "set vf-%d mac to %pM\n", vf, mac);
1183 ether_addr_copy(oct->vf_info[vf].mac_addr, mac);
1184 oct->vf_info[vf].flags |= OCTEON_PFVF_FLAG_MAC_SET_BY_PF;
1185
1186 err = octep_ctrl_net_set_mac_addr(oct, vf, mac, true);
1187 if (err)
1188 dev_err(&oct->pdev->dev,
1189 "Set VF%d MAC address failed via host control Mbox\n",
1190 vf);
1191
1192 return err;
1193 }
1194
1195 static const struct net_device_ops octep_netdev_ops = {
1196 .ndo_open = octep_open,
1197 .ndo_stop = octep_stop,
1198 .ndo_start_xmit = octep_start_xmit,
1199 .ndo_get_stats64 = octep_get_stats64,
1200 .ndo_tx_timeout = octep_tx_timeout,
1201 .ndo_set_mac_address = octep_set_mac,
1202 .ndo_change_mtu = octep_change_mtu,
1203 .ndo_set_features = octep_set_features,
1204 .ndo_get_vf_config = octep_get_vf_config,
1205 .ndo_set_vf_mac = octep_set_vf_mac
1206 };
1207
1208 /**
1209 * octep_intr_poll_task - work queue task to process non-ioq interrupts.
1210 *
1211 * @work: pointer to mbox work_struct
1212 *
1213 * Process non-ioq interrupts to handle control mailbox, pfvf mailbox.
1214 **/
octep_intr_poll_task(struct work_struct * work)1215 static void octep_intr_poll_task(struct work_struct *work)
1216 {
1217 struct octep_device *oct = container_of(work, struct octep_device,
1218 intr_poll_task.work);
1219
1220 if (!oct->poll_non_ioq_intr) {
1221 dev_info(&oct->pdev->dev, "Interrupt poll task stopped.\n");
1222 return;
1223 }
1224
1225 oct->hw_ops.poll_non_ioq_interrupts(oct);
1226 queue_delayed_work(octep_wq, &oct->intr_poll_task,
1227 msecs_to_jiffies(OCTEP_INTR_POLL_TIME_MSECS));
1228 }
1229
1230 /**
1231 * octep_hb_timeout_task - work queue task to check firmware heartbeat.
1232 *
1233 * @work: pointer to hb work_struct
1234 *
1235 * Check for heartbeat miss count. Uninitialize oct device if miss count
1236 * exceeds configured max heartbeat miss count.
1237 *
1238 **/
octep_hb_timeout_task(struct work_struct * work)1239 static void octep_hb_timeout_task(struct work_struct *work)
1240 {
1241 struct octep_device *oct = container_of(work, struct octep_device,
1242 hb_task.work);
1243
1244 int miss_cnt;
1245
1246 miss_cnt = atomic_inc_return(&oct->hb_miss_cnt);
1247 if (miss_cnt < oct->conf->fw_info.hb_miss_count) {
1248 queue_delayed_work(octep_wq, &oct->hb_task,
1249 msecs_to_jiffies(oct->conf->fw_info.hb_interval));
1250 return;
1251 }
1252
1253 dev_err(&oct->pdev->dev, "Missed %u heartbeats. Uninitializing\n",
1254 miss_cnt);
1255 rtnl_lock();
1256 if (netif_running(oct->netdev))
1257 dev_close(oct->netdev);
1258 rtnl_unlock();
1259 }
1260
1261 /**
1262 * octep_ctrl_mbox_task - work queue task to handle ctrl mbox messages.
1263 *
1264 * @work: pointer to ctrl mbox work_struct
1265 *
1266 * Poll ctrl mbox message queue and handle control messages from firmware.
1267 **/
octep_ctrl_mbox_task(struct work_struct * work)1268 static void octep_ctrl_mbox_task(struct work_struct *work)
1269 {
1270 struct octep_device *oct = container_of(work, struct octep_device,
1271 ctrl_mbox_task);
1272
1273 octep_ctrl_net_recv_fw_messages(oct);
1274 }
1275
octep_devid_to_str(struct octep_device * oct)1276 static const char *octep_devid_to_str(struct octep_device *oct)
1277 {
1278 switch (oct->chip_id) {
1279 case OCTEP_PCI_DEVICE_ID_CN98_PF:
1280 return "CN98XX";
1281 case OCTEP_PCI_DEVICE_ID_CN93_PF:
1282 return "CN93XX";
1283 case OCTEP_PCI_DEVICE_ID_CNF95N_PF:
1284 return "CNF95N";
1285 case OCTEP_PCI_DEVICE_ID_CN10KA_PF:
1286 return "CN10KA";
1287 case OCTEP_PCI_DEVICE_ID_CNF10KA_PF:
1288 return "CNF10KA";
1289 case OCTEP_PCI_DEVICE_ID_CNF10KB_PF:
1290 return "CNF10KB";
1291 case OCTEP_PCI_DEVICE_ID_CN10KB_PF:
1292 return "CN10KB";
1293 default:
1294 return "Unsupported";
1295 }
1296 }
1297
1298 /**
1299 * octep_device_setup() - Setup Octeon Device.
1300 *
1301 * @oct: Octeon device private data structure.
1302 *
1303 * Setup Octeon device hardware operations, configuration, etc ...
1304 */
octep_device_setup(struct octep_device * oct)1305 int octep_device_setup(struct octep_device *oct)
1306 {
1307 struct pci_dev *pdev = oct->pdev;
1308 int i, ret;
1309
1310 /* allocate memory for oct->conf */
1311 oct->conf = kzalloc_obj(*oct->conf);
1312 if (!oct->conf)
1313 return -ENOMEM;
1314
1315 /* Map BAR regions */
1316 for (i = 0; i < OCTEP_MMIO_REGIONS; i++) {
1317 oct->mmio[i].hw_addr =
1318 ioremap(pci_resource_start(oct->pdev, i * 2),
1319 pci_resource_len(oct->pdev, i * 2));
1320 if (!oct->mmio[i].hw_addr)
1321 goto unmap_prev;
1322
1323 oct->mmio[i].mapped = 1;
1324 }
1325
1326 oct->chip_id = pdev->device;
1327 oct->rev_id = pdev->revision;
1328 dev_info(&pdev->dev, "chip_id = 0x%x\n", pdev->device);
1329
1330 switch (oct->chip_id) {
1331 case OCTEP_PCI_DEVICE_ID_CN98_PF:
1332 case OCTEP_PCI_DEVICE_ID_CN93_PF:
1333 case OCTEP_PCI_DEVICE_ID_CNF95N_PF:
1334 dev_info(&pdev->dev, "Setting up OCTEON %s PF PASS%d.%d\n",
1335 octep_devid_to_str(oct), OCTEP_MAJOR_REV(oct),
1336 OCTEP_MINOR_REV(oct));
1337 octep_device_setup_cn93_pf(oct);
1338 break;
1339 case OCTEP_PCI_DEVICE_ID_CNF10KA_PF:
1340 case OCTEP_PCI_DEVICE_ID_CN10KA_PF:
1341 case OCTEP_PCI_DEVICE_ID_CNF10KB_PF:
1342 case OCTEP_PCI_DEVICE_ID_CN10KB_PF:
1343 dev_info(&pdev->dev, "Setting up OCTEON %s PF PASS%d.%d\n",
1344 octep_devid_to_str(oct), OCTEP_MAJOR_REV(oct), OCTEP_MINOR_REV(oct));
1345 octep_device_setup_cnxk_pf(oct);
1346 break;
1347 default:
1348 dev_err(&pdev->dev,
1349 "%s: unsupported device\n", __func__);
1350 goto unsupported_dev;
1351 }
1352
1353
1354 ret = octep_ctrl_net_init(oct);
1355 if (ret)
1356 goto unsupported_dev;
1357
1358 INIT_WORK(&oct->tx_timeout_task, octep_tx_timeout_task);
1359 INIT_WORK(&oct->ctrl_mbox_task, octep_ctrl_mbox_task);
1360 INIT_DELAYED_WORK(&oct->intr_poll_task, octep_intr_poll_task);
1361 oct->poll_non_ioq_intr = true;
1362 queue_delayed_work(octep_wq, &oct->intr_poll_task,
1363 msecs_to_jiffies(OCTEP_INTR_POLL_TIME_MSECS));
1364
1365 atomic_set(&oct->hb_miss_cnt, 0);
1366 INIT_DELAYED_WORK(&oct->hb_task, octep_hb_timeout_task);
1367
1368 return 0;
1369
1370 unsupported_dev:
1371 i = OCTEP_MMIO_REGIONS;
1372 unmap_prev:
1373 while (i--)
1374 iounmap(oct->mmio[i].hw_addr);
1375
1376 kfree(oct->conf);
1377 return -1;
1378 }
1379
1380 /**
1381 * octep_device_cleanup() - Cleanup Octeon Device.
1382 *
1383 * @oct: Octeon device private data structure.
1384 *
1385 * Cleanup Octeon device allocated resources.
1386 */
octep_device_cleanup(struct octep_device * oct)1387 static void octep_device_cleanup(struct octep_device *oct)
1388 {
1389 int i;
1390
1391 oct->poll_non_ioq_intr = false;
1392 cancel_delayed_work_sync(&oct->intr_poll_task);
1393 cancel_work_sync(&oct->ctrl_mbox_task);
1394
1395 dev_info(&oct->pdev->dev, "Cleaning up Octeon Device ...\n");
1396
1397 for (i = 0; i < OCTEP_MAX_VF; i++) {
1398 vfree(oct->mbox[i]);
1399 oct->mbox[i] = NULL;
1400 }
1401
1402 octep_delete_pfvf_mbox(oct);
1403 octep_ctrl_net_uninit(oct);
1404 cancel_delayed_work_sync(&oct->hb_task);
1405
1406 oct->hw_ops.soft_reset(oct);
1407 for (i = 0; i < OCTEP_MMIO_REGIONS; i++) {
1408 if (oct->mmio[i].mapped)
1409 iounmap(oct->mmio[i].hw_addr);
1410 }
1411
1412 kfree(oct->conf);
1413 oct->conf = NULL;
1414 }
1415
get_fw_ready_status(struct pci_dev * pdev)1416 static bool get_fw_ready_status(struct pci_dev *pdev)
1417 {
1418 u32 pos = 0;
1419 u16 vsec_id;
1420 u8 status;
1421
1422 while ((pos = pci_find_next_ext_capability(pdev, pos,
1423 PCI_EXT_CAP_ID_VNDR))) {
1424 pci_read_config_word(pdev, pos + 4, &vsec_id);
1425 #define FW_STATUS_VSEC_ID 0xA3
1426 if (vsec_id != FW_STATUS_VSEC_ID)
1427 continue;
1428
1429 pci_read_config_byte(pdev, (pos + 8), &status);
1430 dev_info(&pdev->dev, "Firmware ready status = %u\n", status);
1431 #define FW_STATUS_READY 1ULL
1432 return status == FW_STATUS_READY;
1433 }
1434 return false;
1435 }
1436
1437 /**
1438 * octep_probe() - Octeon PCI device probe handler.
1439 *
1440 * @pdev: PCI device structure.
1441 * @ent: entry in Octeon PCI device ID table.
1442 *
1443 * Initializes and enables the Octeon PCI device for network operations.
1444 * Initializes Octeon private data structure and registers a network device.
1445 */
octep_probe(struct pci_dev * pdev,const struct pci_device_id * ent)1446 static int octep_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
1447 {
1448 struct octep_device *octep_dev = NULL;
1449 struct net_device *netdev;
1450 int max_rx_pktlen;
1451 int err;
1452
1453 err = pci_enable_device(pdev);
1454 if (err) {
1455 dev_err(&pdev->dev, "Failed to enable PCI device\n");
1456 return err;
1457 }
1458
1459 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
1460 if (err) {
1461 dev_err(&pdev->dev, "Failed to set DMA mask !!\n");
1462 goto err_dma_mask;
1463 }
1464
1465 err = pci_request_mem_regions(pdev, OCTEP_DRV_NAME);
1466 if (err) {
1467 dev_err(&pdev->dev, "Failed to map PCI memory regions\n");
1468 goto err_pci_regions;
1469 }
1470
1471 pci_set_master(pdev);
1472
1473 if (!get_fw_ready_status(pdev)) {
1474 dev_notice(&pdev->dev, "Firmware not ready; defer probe.\n");
1475 err = -EPROBE_DEFER;
1476 goto err_alloc_netdev;
1477 }
1478
1479 netdev = alloc_etherdev_mq(sizeof(struct octep_device),
1480 OCTEP_MAX_QUEUES);
1481 if (!netdev) {
1482 dev_err(&pdev->dev, "Failed to allocate netdev\n");
1483 err = -ENOMEM;
1484 goto err_alloc_netdev;
1485 }
1486 SET_NETDEV_DEV(netdev, &pdev->dev);
1487
1488 octep_dev = netdev_priv(netdev);
1489 octep_dev->netdev = netdev;
1490 octep_dev->pdev = pdev;
1491 octep_dev->dev = &pdev->dev;
1492 pci_set_drvdata(pdev, octep_dev);
1493
1494 err = octep_device_setup(octep_dev);
1495 if (err) {
1496 dev_err(&pdev->dev, "Device setup failed\n");
1497 goto err_octep_config;
1498 }
1499
1500 err = octep_setup_pfvf_mbox(octep_dev);
1501 if (err) {
1502 dev_err(&pdev->dev, "PF-VF mailbox setup failed\n");
1503 goto register_dev_err;
1504 }
1505
1506 err = octep_ctrl_net_get_info(octep_dev, OCTEP_CTRL_NET_INVALID_VFID,
1507 &octep_dev->conf->fw_info);
1508 if (err) {
1509 dev_err(&pdev->dev, "Failed to get firmware info\n");
1510 goto register_dev_err;
1511 }
1512 dev_info(&octep_dev->pdev->dev, "Heartbeat interval %u msecs Heartbeat miss count %u\n",
1513 octep_dev->conf->fw_info.hb_interval,
1514 octep_dev->conf->fw_info.hb_miss_count);
1515 queue_delayed_work(octep_wq, &octep_dev->hb_task,
1516 msecs_to_jiffies(octep_dev->conf->fw_info.hb_interval));
1517
1518 netdev->netdev_ops = &octep_netdev_ops;
1519 octep_set_ethtool_ops(netdev);
1520 netif_carrier_off(netdev);
1521
1522 netdev->hw_features = NETIF_F_SG;
1523 if (OCTEP_TX_IP_CSUM(octep_dev->conf->fw_info.tx_ol_flags))
1524 netdev->hw_features |= (NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM);
1525
1526 if (OCTEP_RX_IP_CSUM(octep_dev->conf->fw_info.rx_ol_flags))
1527 netdev->hw_features |= NETIF_F_RXCSUM;
1528
1529 max_rx_pktlen = octep_ctrl_net_get_mtu(octep_dev, OCTEP_CTRL_NET_INVALID_VFID);
1530 if (max_rx_pktlen < 0) {
1531 dev_err(&octep_dev->pdev->dev,
1532 "Failed to get max receive packet size; err = %d\n", max_rx_pktlen);
1533 err = max_rx_pktlen;
1534 goto register_dev_err;
1535 }
1536 netdev->min_mtu = OCTEP_MIN_MTU;
1537 netdev->max_mtu = max_rx_pktlen - (ETH_HLEN + ETH_FCS_LEN);
1538 netdev->mtu = OCTEP_DEFAULT_MTU;
1539
1540 if (OCTEP_TX_TSO(octep_dev->conf->fw_info.tx_ol_flags)) {
1541 netdev->hw_features |= NETIF_F_TSO;
1542 netif_set_tso_max_size(netdev, netdev->max_mtu);
1543 }
1544
1545 netdev->features |= netdev->hw_features;
1546 err = octep_ctrl_net_get_mac_addr(octep_dev, OCTEP_CTRL_NET_INVALID_VFID,
1547 octep_dev->mac_addr);
1548 if (err) {
1549 dev_err(&pdev->dev, "Failed to get mac address\n");
1550 goto register_dev_err;
1551 }
1552 eth_hw_addr_set(netdev, octep_dev->mac_addr);
1553
1554 err = register_netdev(netdev);
1555 if (err) {
1556 dev_err(&pdev->dev, "Failed to register netdev\n");
1557 goto register_dev_err;
1558 }
1559 dev_info(&pdev->dev, "Device probe successful\n");
1560 return 0;
1561
1562 register_dev_err:
1563 octep_device_cleanup(octep_dev);
1564 err_octep_config:
1565 free_netdev(netdev);
1566 err_alloc_netdev:
1567 pci_release_mem_regions(pdev);
1568 err_pci_regions:
1569 err_dma_mask:
1570 pci_disable_device(pdev);
1571 return err;
1572 }
1573
octep_sriov_disable(struct octep_device * oct)1574 static int octep_sriov_disable(struct octep_device *oct)
1575 {
1576 struct pci_dev *pdev = oct->pdev;
1577
1578 if (pci_vfs_assigned(oct->pdev)) {
1579 dev_warn(&pdev->dev, "Can't disable SRIOV while VFs are assigned\n");
1580 return -EPERM;
1581 }
1582
1583 pci_disable_sriov(pdev);
1584 CFG_GET_ACTIVE_VFS(oct->conf) = 0;
1585
1586 return 0;
1587 }
1588
1589 /**
1590 * octep_remove() - Remove Octeon PCI device from driver control.
1591 *
1592 * @pdev: PCI device structure of the Octeon device.
1593 *
1594 * Cleanup all resources allocated for the Octeon device.
1595 * Unregister from network device and disable the PCI device.
1596 */
octep_remove(struct pci_dev * pdev)1597 static void octep_remove(struct pci_dev *pdev)
1598 {
1599 struct octep_device *oct = pci_get_drvdata(pdev);
1600 struct net_device *netdev;
1601
1602 if (!oct)
1603 return;
1604
1605 netdev = oct->netdev;
1606 octep_sriov_disable(oct);
1607 if (netdev->reg_state == NETREG_REGISTERED)
1608 unregister_netdev(netdev);
1609
1610 cancel_work_sync(&oct->tx_timeout_task);
1611 octep_device_cleanup(oct);
1612 pci_release_mem_regions(pdev);
1613 free_netdev(netdev);
1614 pci_disable_device(pdev);
1615 }
1616
octep_sriov_enable(struct octep_device * oct,int num_vfs)1617 static int octep_sriov_enable(struct octep_device *oct, int num_vfs)
1618 {
1619 struct pci_dev *pdev = oct->pdev;
1620 int err;
1621
1622 CFG_GET_ACTIVE_VFS(oct->conf) = num_vfs;
1623 err = pci_enable_sriov(pdev, num_vfs);
1624 if (err) {
1625 dev_warn(&pdev->dev, "Failed to enable SRIOV err=%d\n", err);
1626 CFG_GET_ACTIVE_VFS(oct->conf) = 0;
1627 return err;
1628 }
1629
1630 return num_vfs;
1631 }
1632
octep_sriov_configure(struct pci_dev * pdev,int num_vfs)1633 static int octep_sriov_configure(struct pci_dev *pdev, int num_vfs)
1634 {
1635 struct octep_device *oct = pci_get_drvdata(pdev);
1636 int max_nvfs;
1637
1638 if (num_vfs == 0)
1639 return octep_sriov_disable(oct);
1640
1641 max_nvfs = CFG_GET_MAX_VFS(oct->conf);
1642
1643 if (num_vfs > max_nvfs) {
1644 dev_err(&pdev->dev, "Invalid VF count Max supported VFs = %d\n",
1645 max_nvfs);
1646 return -EINVAL;
1647 }
1648
1649 return octep_sriov_enable(oct, num_vfs);
1650 }
1651
1652 static struct pci_driver octep_driver = {
1653 .name = OCTEP_DRV_NAME,
1654 .id_table = octep_pci_id_tbl,
1655 .probe = octep_probe,
1656 .remove = octep_remove,
1657 .sriov_configure = octep_sriov_configure,
1658 };
1659
1660 /**
1661 * octep_init_module() - Module initialiation.
1662 *
1663 * create common resource for the driver and register PCI driver.
1664 */
octep_init_module(void)1665 static int __init octep_init_module(void)
1666 {
1667 int ret;
1668
1669 pr_info("%s: Loading %s ...\n", OCTEP_DRV_NAME, OCTEP_DRV_STRING);
1670
1671 /* work queue for all deferred tasks */
1672 octep_wq = create_singlethread_workqueue(OCTEP_DRV_NAME);
1673 if (!octep_wq) {
1674 pr_err("%s: Failed to create common workqueue\n",
1675 OCTEP_DRV_NAME);
1676 return -ENOMEM;
1677 }
1678
1679 ret = pci_register_driver(&octep_driver);
1680 if (ret < 0) {
1681 pr_err("%s: Failed to register PCI driver; err=%d\n",
1682 OCTEP_DRV_NAME, ret);
1683 destroy_workqueue(octep_wq);
1684 return ret;
1685 }
1686
1687 pr_info("%s: Loaded successfully !\n", OCTEP_DRV_NAME);
1688
1689 return ret;
1690 }
1691
1692 /**
1693 * octep_exit_module() - Module exit routine.
1694 *
1695 * unregister the driver with PCI subsystem and cleanup common resources.
1696 */
octep_exit_module(void)1697 static void __exit octep_exit_module(void)
1698 {
1699 pr_info("%s: Unloading ...\n", OCTEP_DRV_NAME);
1700
1701 pci_unregister_driver(&octep_driver);
1702 destroy_workqueue(octep_wq);
1703
1704 pr_info("%s: Unloading complete\n", OCTEP_DRV_NAME);
1705 }
1706
1707 module_init(octep_init_module);
1708 module_exit(octep_exit_module);
1709