1 // SPDX-License-Identifier: GPL-2.0
2 /* Marvell Octeon EP (EndPoint) VF Ethernet Driver
3 *
4 * Copyright (C) 2020 Marvell.
5 *
6 */
7
8 #include <linux/types.h>
9 #include <linux/module.h>
10 #include <linux/pci.h>
11 #include <linux/aer.h>
12 #include <linux/netdevice.h>
13 #include <linux/etherdevice.h>
14 #include <linux/rtnetlink.h>
15 #include <linux/vmalloc.h>
16 #include <net/netdev_queues.h>
17
18 #include "octep_vf_config.h"
19 #include "octep_vf_main.h"
20
21 struct workqueue_struct *octep_vf_wq;
22
23 /* Supported Devices */
24 static const struct pci_device_id octep_vf_pci_id_tbl[] = {
25 {PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, OCTEP_PCI_DEVICE_ID_CN93_VF)},
26 {PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, OCTEP_PCI_DEVICE_ID_CNF95N_VF)},
27 {PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, OCTEP_PCI_DEVICE_ID_CN98_VF)},
28 {PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, OCTEP_PCI_DEVICE_ID_CN10KA_VF)},
29 {PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, OCTEP_PCI_DEVICE_ID_CNF10KA_VF)},
30 {PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, OCTEP_PCI_DEVICE_ID_CNF10KB_VF)},
31 {PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, OCTEP_PCI_DEVICE_ID_CN10KB_VF)},
32 {0, },
33 };
34 MODULE_DEVICE_TABLE(pci, octep_vf_pci_id_tbl);
35
36 MODULE_AUTHOR("Veerasenareddy Burru <vburru@marvell.com>");
37 MODULE_DESCRIPTION(OCTEP_VF_DRV_STRING);
38 MODULE_LICENSE("GPL");
39
40 /**
41 * octep_vf_alloc_ioq_vectors() - Allocate Tx/Rx Queue interrupt info.
42 *
43 * @oct: Octeon device private data structure.
44 *
45 * Allocate resources to hold per Tx/Rx queue interrupt info.
46 * This is the information passed to interrupt handler, from which napi poll
47 * is scheduled and includes quick access to private data of Tx/Rx queue
48 * corresponding to the interrupt being handled.
49 *
50 * Return: 0, on successful allocation of resources for all queue interrupts.
51 * -1, if failed to allocate any resource.
52 */
octep_vf_alloc_ioq_vectors(struct octep_vf_device * oct)53 static int octep_vf_alloc_ioq_vectors(struct octep_vf_device *oct)
54 {
55 struct octep_vf_ioq_vector *ioq_vector;
56 int i;
57
58 for (i = 0; i < oct->num_oqs; i++) {
59 oct->ioq_vector[i] = vzalloc(sizeof(*oct->ioq_vector[i]));
60 if (!oct->ioq_vector[i])
61 goto free_ioq_vector;
62
63 ioq_vector = oct->ioq_vector[i];
64 ioq_vector->iq = oct->iq[i];
65 ioq_vector->oq = oct->oq[i];
66 ioq_vector->octep_vf_dev = oct;
67 }
68
69 dev_info(&oct->pdev->dev, "Allocated %d IOQ vectors\n", oct->num_oqs);
70 return 0;
71
72 free_ioq_vector:
73 while (i) {
74 i--;
75 vfree(oct->ioq_vector[i]);
76 oct->ioq_vector[i] = NULL;
77 }
78 return -1;
79 }
80
81 /**
82 * octep_vf_free_ioq_vectors() - Free Tx/Rx Queue interrupt vector info.
83 *
84 * @oct: Octeon device private data structure.
85 */
octep_vf_free_ioq_vectors(struct octep_vf_device * oct)86 static void octep_vf_free_ioq_vectors(struct octep_vf_device *oct)
87 {
88 int i;
89
90 for (i = 0; i < oct->num_oqs; i++) {
91 if (oct->ioq_vector[i]) {
92 vfree(oct->ioq_vector[i]);
93 oct->ioq_vector[i] = NULL;
94 }
95 }
96 netdev_info(oct->netdev, "Freed IOQ Vectors\n");
97 }
98
99 /**
100 * octep_vf_enable_msix_range() - enable MSI-x interrupts.
101 *
102 * @oct: Octeon device private data structure.
103 *
104 * Allocate and enable all MSI-x interrupts (queue and non-queue interrupts)
105 * for the Octeon device.
106 *
107 * Return: 0, on successfully enabling all MSI-x interrupts.
108 * -1, if failed to enable any MSI-x interrupt.
109 */
octep_vf_enable_msix_range(struct octep_vf_device * oct)110 static int octep_vf_enable_msix_range(struct octep_vf_device *oct)
111 {
112 int num_msix, msix_allocated;
113 int i;
114
115 /* Generic interrupts apart from input/output queues */
116 //num_msix = oct->num_oqs + CFG_GET_NON_IOQ_MSIX(oct->conf);
117 num_msix = oct->num_oqs;
118 oct->msix_entries = kcalloc(num_msix, sizeof(struct msix_entry), GFP_KERNEL);
119 if (!oct->msix_entries)
120 goto msix_alloc_err;
121
122 for (i = 0; i < num_msix; i++)
123 oct->msix_entries[i].entry = i;
124
125 msix_allocated = pci_enable_msix_range(oct->pdev, oct->msix_entries,
126 num_msix, num_msix);
127 if (msix_allocated != num_msix) {
128 dev_err(&oct->pdev->dev,
129 "Failed to enable %d msix irqs; got only %d\n",
130 num_msix, msix_allocated);
131 goto enable_msix_err;
132 }
133 oct->num_irqs = msix_allocated;
134 dev_info(&oct->pdev->dev, "MSI-X enabled successfully\n");
135
136 return 0;
137
138 enable_msix_err:
139 if (msix_allocated > 0)
140 pci_disable_msix(oct->pdev);
141 kfree(oct->msix_entries);
142 oct->msix_entries = NULL;
143 msix_alloc_err:
144 return -1;
145 }
146
147 /**
148 * octep_vf_disable_msix() - disable MSI-x interrupts.
149 *
150 * @oct: Octeon device private data structure.
151 *
152 * Disable MSI-x on the Octeon device.
153 */
octep_vf_disable_msix(struct octep_vf_device * oct)154 static void octep_vf_disable_msix(struct octep_vf_device *oct)
155 {
156 pci_disable_msix(oct->pdev);
157 kfree(oct->msix_entries);
158 oct->msix_entries = NULL;
159 dev_info(&oct->pdev->dev, "Disabled MSI-X\n");
160 }
161
162 /**
163 * octep_vf_ioq_intr_handler() - handler for all Tx/Rx queue interrupts.
164 *
165 * @irq: Interrupt number.
166 * @data: interrupt data contains pointers to Tx/Rx queue private data
167 * and correspong NAPI context.
168 *
169 * this is common handler for all non-queue (generic) interrupts.
170 */
octep_vf_ioq_intr_handler(int irq,void * data)171 static irqreturn_t octep_vf_ioq_intr_handler(int irq, void *data)
172 {
173 struct octep_vf_ioq_vector *ioq_vector = data;
174 struct octep_vf_device *oct = ioq_vector->octep_vf_dev;
175
176 return oct->hw_ops.ioq_intr_handler(ioq_vector);
177 }
178
179 /**
180 * octep_vf_request_irqs() - Register interrupt handlers.
181 *
182 * @oct: Octeon device private data structure.
183 *
184 * Register handlers for all queue and non-queue interrupts.
185 *
186 * Return: 0, on successful registration of all interrupt handlers.
187 * -1, on any error.
188 */
octep_vf_request_irqs(struct octep_vf_device * oct)189 static int octep_vf_request_irqs(struct octep_vf_device *oct)
190 {
191 struct net_device *netdev = oct->netdev;
192 struct octep_vf_ioq_vector *ioq_vector;
193 struct msix_entry *msix_entry;
194 int ret, i;
195
196 /* Request IRQs for Tx/Rx queues */
197 for (i = 0; i < oct->num_oqs; i++) {
198 ioq_vector = oct->ioq_vector[i];
199 msix_entry = &oct->msix_entries[i];
200
201 snprintf(ioq_vector->name, sizeof(ioq_vector->name),
202 "%s-q%d", netdev->name, i);
203 ret = request_irq(msix_entry->vector,
204 octep_vf_ioq_intr_handler, 0,
205 ioq_vector->name, ioq_vector);
206 if (ret) {
207 netdev_err(netdev,
208 "request_irq failed for Q-%d; err=%d",
209 i, ret);
210 goto ioq_irq_err;
211 }
212
213 cpumask_set_cpu(i % num_online_cpus(),
214 &ioq_vector->affinity_mask);
215 irq_set_affinity_hint(msix_entry->vector,
216 &ioq_vector->affinity_mask);
217 }
218
219 return 0;
220 ioq_irq_err:
221 while (i) {
222 --i;
223 free_irq(oct->msix_entries[i].vector, oct);
224 }
225 return -1;
226 }
227
228 /**
229 * octep_vf_free_irqs() - free all registered interrupts.
230 *
231 * @oct: Octeon device private data structure.
232 *
233 * Free all queue and non-queue interrupts of the Octeon device.
234 */
octep_vf_free_irqs(struct octep_vf_device * oct)235 static void octep_vf_free_irqs(struct octep_vf_device *oct)
236 {
237 int i;
238
239 for (i = 0; i < oct->num_irqs; i++) {
240 irq_set_affinity_hint(oct->msix_entries[i].vector, NULL);
241 free_irq(oct->msix_entries[i].vector, oct->ioq_vector[i]);
242 }
243 netdev_info(oct->netdev, "IRQs freed\n");
244 }
245
246 /**
247 * octep_vf_setup_irqs() - setup interrupts for the Octeon device.
248 *
249 * @oct: Octeon device private data structure.
250 *
251 * Allocate data structures to hold per interrupt information, allocate/enable
252 * MSI-x interrupt and register interrupt handlers.
253 *
254 * Return: 0, on successful allocation and registration of all interrupts.
255 * -1, on any error.
256 */
octep_vf_setup_irqs(struct octep_vf_device * oct)257 static int octep_vf_setup_irqs(struct octep_vf_device *oct)
258 {
259 if (octep_vf_alloc_ioq_vectors(oct))
260 goto ioq_vector_err;
261
262 if (octep_vf_enable_msix_range(oct))
263 goto enable_msix_err;
264
265 if (octep_vf_request_irqs(oct))
266 goto request_irq_err;
267
268 return 0;
269
270 request_irq_err:
271 octep_vf_disable_msix(oct);
272 enable_msix_err:
273 octep_vf_free_ioq_vectors(oct);
274 ioq_vector_err:
275 return -1;
276 }
277
278 /**
279 * octep_vf_clean_irqs() - free all interrupts and its resources.
280 *
281 * @oct: Octeon device private data structure.
282 */
octep_vf_clean_irqs(struct octep_vf_device * oct)283 static void octep_vf_clean_irqs(struct octep_vf_device *oct)
284 {
285 octep_vf_free_irqs(oct);
286 octep_vf_disable_msix(oct);
287 octep_vf_free_ioq_vectors(oct);
288 }
289
290 /**
291 * octep_vf_enable_ioq_irq() - Enable MSI-x interrupt of a Tx/Rx queue.
292 *
293 * @iq: Octeon Tx queue data structure.
294 * @oq: Octeon Rx queue data structure.
295 */
octep_vf_enable_ioq_irq(struct octep_vf_iq * iq,struct octep_vf_oq * oq)296 static void octep_vf_enable_ioq_irq(struct octep_vf_iq *iq, struct octep_vf_oq *oq)
297 {
298 u32 pkts_pend = oq->pkts_pending;
299
300 netdev_dbg(iq->netdev, "enabling intr for Q-%u\n", iq->q_no);
301 if (iq->pkts_processed) {
302 writel(iq->pkts_processed, iq->inst_cnt_reg);
303 iq->pkt_in_done -= iq->pkts_processed;
304 iq->pkts_processed = 0;
305 }
306 if (oq->last_pkt_count - pkts_pend) {
307 writel(oq->last_pkt_count - pkts_pend, oq->pkts_sent_reg);
308 oq->last_pkt_count = pkts_pend;
309 }
310
311 /* Flush the previous wrties before writing to RESEND bit */
312 smp_wmb();
313 writeq(1UL << OCTEP_VF_OQ_INTR_RESEND_BIT, oq->pkts_sent_reg);
314 writeq(1UL << OCTEP_VF_IQ_INTR_RESEND_BIT, iq->inst_cnt_reg);
315 }
316
317 /**
318 * octep_vf_napi_poll() - NAPI poll function for Tx/Rx.
319 *
320 * @napi: pointer to napi context.
321 * @budget: max number of packets to be processed in single invocation.
322 */
octep_vf_napi_poll(struct napi_struct * napi,int budget)323 static int octep_vf_napi_poll(struct napi_struct *napi, int budget)
324 {
325 struct octep_vf_ioq_vector *ioq_vector =
326 container_of(napi, struct octep_vf_ioq_vector, napi);
327 u32 tx_pending, rx_done;
328
329 tx_pending = octep_vf_iq_process_completions(ioq_vector->iq, 64);
330 rx_done = octep_vf_oq_process_rx(ioq_vector->oq, budget);
331
332 /* need more polling if tx completion processing is still pending or
333 * processed at least 'budget' number of rx packets.
334 */
335 if (tx_pending || rx_done >= budget)
336 return budget;
337
338 if (likely(napi_complete_done(napi, rx_done)))
339 octep_vf_enable_ioq_irq(ioq_vector->iq, ioq_vector->oq);
340
341 return rx_done;
342 }
343
344 /**
345 * octep_vf_napi_add() - Add NAPI poll for all Tx/Rx queues.
346 *
347 * @oct: Octeon device private data structure.
348 */
octep_vf_napi_add(struct octep_vf_device * oct)349 static void octep_vf_napi_add(struct octep_vf_device *oct)
350 {
351 int i;
352
353 for (i = 0; i < oct->num_oqs; i++) {
354 netdev_dbg(oct->netdev, "Adding NAPI on Q-%d\n", i);
355 netif_napi_add(oct->netdev, &oct->ioq_vector[i]->napi, octep_vf_napi_poll);
356 oct->oq[i]->napi = &oct->ioq_vector[i]->napi;
357 }
358 }
359
360 /**
361 * octep_vf_napi_delete() - delete NAPI poll callback for all Tx/Rx queues.
362 *
363 * @oct: Octeon device private data structure.
364 */
octep_vf_napi_delete(struct octep_vf_device * oct)365 static void octep_vf_napi_delete(struct octep_vf_device *oct)
366 {
367 int i;
368
369 for (i = 0; i < oct->num_oqs; i++) {
370 netdev_dbg(oct->netdev, "Deleting NAPI on Q-%d\n", i);
371 netif_napi_del(&oct->ioq_vector[i]->napi);
372 oct->oq[i]->napi = NULL;
373 }
374 }
375
376 /**
377 * octep_vf_napi_enable() - enable NAPI for all Tx/Rx queues.
378 *
379 * @oct: Octeon device private data structure.
380 */
octep_vf_napi_enable(struct octep_vf_device * oct)381 static void octep_vf_napi_enable(struct octep_vf_device *oct)
382 {
383 int i;
384
385 for (i = 0; i < oct->num_oqs; i++) {
386 netdev_dbg(oct->netdev, "Enabling NAPI on Q-%d\n", i);
387 napi_enable(&oct->ioq_vector[i]->napi);
388 }
389 }
390
391 /**
392 * octep_vf_napi_disable() - disable NAPI for all Tx/Rx queues.
393 *
394 * @oct: Octeon device private data structure.
395 */
octep_vf_napi_disable(struct octep_vf_device * oct)396 static void octep_vf_napi_disable(struct octep_vf_device *oct)
397 {
398 int i;
399
400 for (i = 0; i < oct->num_oqs; i++) {
401 netdev_dbg(oct->netdev, "Disabling NAPI on Q-%d\n", i);
402 napi_disable(&oct->ioq_vector[i]->napi);
403 }
404 }
405
octep_vf_link_up(struct net_device * netdev)406 static void octep_vf_link_up(struct net_device *netdev)
407 {
408 netif_carrier_on(netdev);
409 netif_tx_start_all_queues(netdev);
410 }
411
octep_vf_set_rx_state(struct octep_vf_device * oct,bool up)412 static void octep_vf_set_rx_state(struct octep_vf_device *oct, bool up)
413 {
414 int err;
415
416 err = octep_vf_mbox_set_rx_state(oct, up);
417 if (err)
418 netdev_err(oct->netdev, "Set Rx state to %d failed with err:%d\n", up, err);
419 }
420
octep_vf_get_link_status(struct octep_vf_device * oct)421 static int octep_vf_get_link_status(struct octep_vf_device *oct)
422 {
423 int err;
424
425 err = octep_vf_mbox_get_link_status(oct, &oct->link_info.oper_up);
426 if (err)
427 netdev_err(oct->netdev, "Get link status failed with err:%d\n", err);
428 return oct->link_info.oper_up;
429 }
430
octep_vf_set_link_status(struct octep_vf_device * oct,bool up)431 static void octep_vf_set_link_status(struct octep_vf_device *oct, bool up)
432 {
433 int err;
434
435 err = octep_vf_mbox_set_link_status(oct, up);
436 if (err) {
437 netdev_err(oct->netdev, "Set link status to %d failed with err:%d\n", up, err);
438 return;
439 }
440 oct->link_info.oper_up = up;
441 }
442
443 /**
444 * octep_vf_open() - start the octeon network device.
445 *
446 * @netdev: pointer to kernel network device.
447 *
448 * setup Tx/Rx queues, interrupts and enable hardware operation of Tx/Rx queues
449 * and interrupts..
450 *
451 * Return: 0, on successfully setting up device and bring it up.
452 * -1, on any error.
453 */
octep_vf_open(struct net_device * netdev)454 static int octep_vf_open(struct net_device *netdev)
455 {
456 struct octep_vf_device *oct = netdev_priv(netdev);
457 int err, ret;
458
459 netdev_info(netdev, "Starting netdev ...\n");
460 netif_carrier_off(netdev);
461
462 oct->hw_ops.reset_io_queues(oct);
463
464 if (octep_vf_setup_iqs(oct))
465 goto setup_iq_err;
466 if (octep_vf_setup_oqs(oct))
467 goto setup_oq_err;
468 if (octep_vf_setup_irqs(oct))
469 goto setup_irq_err;
470
471 err = netif_set_real_num_tx_queues(netdev, oct->num_oqs);
472 if (err)
473 goto set_queues_err;
474 err = netif_set_real_num_rx_queues(netdev, oct->num_iqs);
475 if (err)
476 goto set_queues_err;
477
478 octep_vf_napi_add(oct);
479 octep_vf_napi_enable(oct);
480
481 oct->link_info.admin_up = 1;
482 octep_vf_set_rx_state(oct, true);
483
484 ret = octep_vf_get_link_status(oct);
485 if (!ret)
486 octep_vf_set_link_status(oct, true);
487
488 /* Enable the input and output queues for this Octeon device */
489 oct->hw_ops.enable_io_queues(oct);
490
491 /* Enable Octeon device interrupts */
492 oct->hw_ops.enable_interrupts(oct);
493
494 octep_vf_oq_dbell_init(oct);
495
496 ret = octep_vf_get_link_status(oct);
497 if (ret)
498 octep_vf_link_up(netdev);
499
500 return 0;
501
502 set_queues_err:
503 octep_vf_napi_disable(oct);
504 octep_vf_napi_delete(oct);
505 octep_vf_clean_irqs(oct);
506 setup_irq_err:
507 octep_vf_free_oqs(oct);
508 setup_oq_err:
509 octep_vf_free_iqs(oct);
510 setup_iq_err:
511 return -1;
512 }
513
514 /**
515 * octep_vf_stop() - stop the octeon network device.
516 *
517 * @netdev: pointer to kernel network device.
518 *
519 * stop the device Tx/Rx operations, bring down the link and
520 * free up all resources allocated for Tx/Rx queues and interrupts.
521 */
octep_vf_stop(struct net_device * netdev)522 static int octep_vf_stop(struct net_device *netdev)
523 {
524 struct octep_vf_device *oct = netdev_priv(netdev);
525
526 netdev_info(netdev, "Stopping the device ...\n");
527
528 /* Stop Tx from stack */
529 netif_carrier_off(netdev);
530 netif_tx_disable(netdev);
531
532 octep_vf_set_link_status(oct, false);
533 octep_vf_set_rx_state(oct, false);
534
535 oct->link_info.admin_up = 0;
536 oct->link_info.oper_up = 0;
537
538 oct->hw_ops.disable_interrupts(oct);
539 octep_vf_napi_disable(oct);
540 octep_vf_napi_delete(oct);
541
542 octep_vf_clean_irqs(oct);
543 octep_vf_clean_iqs(oct);
544
545 oct->hw_ops.disable_io_queues(oct);
546 oct->hw_ops.reset_io_queues(oct);
547 octep_vf_free_oqs(oct);
548 octep_vf_free_iqs(oct);
549 netdev_info(netdev, "Device stopped !!\n");
550 return 0;
551 }
552
553 /**
554 * octep_vf_iq_full_check() - check if a Tx queue is full.
555 *
556 * @iq: Octeon Tx queue data structure.
557 *
558 * Return: 0, if the Tx queue is not full.
559 * 1, if the Tx queue is full.
560 */
octep_vf_iq_full_check(struct octep_vf_iq * iq)561 static int octep_vf_iq_full_check(struct octep_vf_iq *iq)
562 {
563 int ret;
564
565 ret = netif_subqueue_maybe_stop(iq->netdev, iq->q_no, IQ_INSTR_SPACE(iq),
566 OCTEP_VF_WAKE_QUEUE_THRESHOLD,
567 OCTEP_VF_WAKE_QUEUE_THRESHOLD);
568 switch (ret) {
569 case 0: /* Stopped the queue, since IQ is full */
570 return 1;
571 case -1: /*
572 * Pending updates in write index from
573 * iq_process_completion in other cpus
574 * caused queues to get re-enabled after
575 * being stopped
576 */
577 iq->stats.restart_cnt++;
578 fallthrough;
579 case 1: /* Queue left enabled, since IQ is not yet full*/
580 return 0;
581 }
582
583 return 1;
584 }
585
586 /**
587 * octep_vf_start_xmit() - Enqueue packet to Octoen hardware Tx Queue.
588 *
589 * @skb: packet skbuff pointer.
590 * @netdev: kernel network device.
591 *
592 * Return: NETDEV_TX_BUSY, if Tx Queue is full.
593 * NETDEV_TX_OK, if successfully enqueued to hardware Tx queue.
594 */
octep_vf_start_xmit(struct sk_buff * skb,struct net_device * netdev)595 static netdev_tx_t octep_vf_start_xmit(struct sk_buff *skb,
596 struct net_device *netdev)
597 {
598 struct octep_vf_device *oct = netdev_priv(netdev);
599 netdev_features_t feat = netdev->features;
600 struct octep_vf_tx_sglist_desc *sglist;
601 struct octep_vf_tx_buffer *tx_buffer;
602 struct octep_vf_tx_desc_hw *hw_desc;
603 struct skb_shared_info *shinfo;
604 struct octep_vf_instr_hdr *ih;
605 struct octep_vf_iq *iq;
606 skb_frag_t *frag;
607 u16 nr_frags, si;
608 int xmit_more;
609 u16 q_no, wi;
610
611 if (skb_put_padto(skb, ETH_ZLEN))
612 return NETDEV_TX_OK;
613
614 q_no = skb_get_queue_mapping(skb);
615 if (q_no >= oct->num_iqs) {
616 netdev_err(netdev, "Invalid Tx skb->queue_mapping=%d\n", q_no);
617 q_no = q_no % oct->num_iqs;
618 }
619
620 iq = oct->iq[q_no];
621
622 shinfo = skb_shinfo(skb);
623 nr_frags = shinfo->nr_frags;
624
625 wi = iq->host_write_index;
626 hw_desc = &iq->desc_ring[wi];
627 hw_desc->ih64 = 0;
628
629 tx_buffer = iq->buff_info + wi;
630 tx_buffer->skb = skb;
631
632 ih = &hw_desc->ih;
633 ih->tlen = skb->len;
634 ih->pkind = oct->fw_info.pkind;
635 ih->fsz = oct->fw_info.fsz;
636 ih->tlen = skb->len + ih->fsz;
637
638 if (!nr_frags) {
639 tx_buffer->gather = 0;
640 tx_buffer->dma = dma_map_single(iq->dev, skb->data,
641 skb->len, DMA_TO_DEVICE);
642 if (dma_mapping_error(iq->dev, tx_buffer->dma))
643 goto dma_map_err;
644 hw_desc->dptr = tx_buffer->dma;
645 } else {
646 /* Scatter/Gather */
647 dma_addr_t dma;
648 u16 len;
649
650 sglist = tx_buffer->sglist;
651
652 ih->gsz = nr_frags + 1;
653 ih->gather = 1;
654 tx_buffer->gather = 1;
655
656 len = skb_headlen(skb);
657 dma = dma_map_single(iq->dev, skb->data, len, DMA_TO_DEVICE);
658 if (dma_mapping_error(iq->dev, dma))
659 goto dma_map_err;
660
661 memset(sglist, 0, OCTEP_VF_SGLIST_SIZE_PER_PKT);
662 sglist[0].len[3] = len;
663 sglist[0].dma_ptr[0] = dma;
664
665 si = 1; /* entry 0 is main skb, mapped above */
666 frag = &shinfo->frags[0];
667 while (nr_frags--) {
668 len = skb_frag_size(frag);
669 dma = skb_frag_dma_map(iq->dev, frag, 0,
670 len, DMA_TO_DEVICE);
671 if (dma_mapping_error(iq->dev, dma))
672 goto dma_map_sg_err;
673
674 sglist[si >> 2].len[3 - (si & 3)] = len;
675 sglist[si >> 2].dma_ptr[si & 3] = dma;
676
677 frag++;
678 si++;
679 }
680 hw_desc->dptr = tx_buffer->sglist_dma;
681 }
682 if (oct->fw_info.tx_ol_flags) {
683 if ((feat & (NETIF_F_TSO)) && (skb_is_gso(skb))) {
684 hw_desc->txm.ol_flags = OCTEP_VF_TX_OFFLOAD_CKSUM;
685 hw_desc->txm.ol_flags |= OCTEP_VF_TX_OFFLOAD_TSO;
686 hw_desc->txm.gso_size = skb_shinfo(skb)->gso_size;
687 hw_desc->txm.gso_segs = skb_shinfo(skb)->gso_segs;
688 } else if (feat & (NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM)) {
689 hw_desc->txm.ol_flags = OCTEP_VF_TX_OFFLOAD_CKSUM;
690 }
691 /* due to ESR txm will be swapped by hw */
692 hw_desc->txm64[0] = (__force u64)cpu_to_be64(hw_desc->txm64[0]);
693 }
694
695 xmit_more = netdev_xmit_more();
696
697 netdev_tx_sent_queue(iq->netdev_q, skb->len);
698
699 skb_tx_timestamp(skb);
700 iq->fill_cnt++;
701 wi++;
702 iq->host_write_index = wi & iq->ring_size_mask;
703
704 /* octep_iq_full_check stops the queue and returns
705 * true if so, in case the queue has become full
706 * by inserting current packet. If so, we can
707 * go ahead and ring doorbell.
708 */
709 if (!octep_vf_iq_full_check(iq) && xmit_more &&
710 iq->fill_cnt < iq->fill_threshold)
711 return NETDEV_TX_OK;
712
713 goto ring_dbell;
714
715 dma_map_sg_err:
716 if (si > 0) {
717 dma_unmap_single(iq->dev, sglist[0].dma_ptr[0],
718 sglist[0].len[0], DMA_TO_DEVICE);
719 sglist[0].len[0] = 0;
720 }
721 while (si > 1) {
722 dma_unmap_page(iq->dev, sglist[si >> 2].dma_ptr[si & 3],
723 sglist[si >> 2].len[si & 3], DMA_TO_DEVICE);
724 sglist[si >> 2].len[si & 3] = 0;
725 si--;
726 }
727 tx_buffer->gather = 0;
728 dma_map_err:
729 dev_kfree_skb_any(skb);
730 ring_dbell:
731 /* Flush the hw descriptors before writing to doorbell */
732 smp_wmb();
733 writel(iq->fill_cnt, iq->doorbell_reg);
734 iq->stats.instr_posted += iq->fill_cnt;
735 iq->fill_cnt = 0;
736 return NETDEV_TX_OK;
737 }
738
octep_vf_get_if_stats(struct octep_vf_device * oct)739 int octep_vf_get_if_stats(struct octep_vf_device *oct)
740 {
741 struct octep_vf_iface_rxtx_stats vf_stats;
742 int ret, size;
743
744 memset(&vf_stats, 0, sizeof(struct octep_vf_iface_rxtx_stats));
745 ret = octep_vf_mbox_bulk_read(oct, OCTEP_PFVF_MBOX_CMD_GET_STATS,
746 (u8 *)&vf_stats, &size);
747
748 if (ret)
749 return ret;
750
751 memcpy(&oct->iface_rx_stats, &vf_stats.iface_rx_stats,
752 sizeof(struct octep_vf_iface_rx_stats));
753 memcpy(&oct->iface_tx_stats, &vf_stats.iface_tx_stats,
754 sizeof(struct octep_vf_iface_tx_stats));
755
756 return 0;
757 }
758
octep_vf_get_link_info(struct octep_vf_device * oct)759 int octep_vf_get_link_info(struct octep_vf_device *oct)
760 {
761 int ret, size;
762
763 ret = octep_vf_mbox_bulk_read(oct, OCTEP_PFVF_MBOX_CMD_GET_LINK_INFO,
764 (u8 *)&oct->link_info, &size);
765 if (ret) {
766 dev_err(&oct->pdev->dev, "Get VF link info failed via VF Mbox\n");
767 return ret;
768 }
769 return 0;
770 }
771
772 /**
773 * octep_vf_get_stats64() - Get Octeon network device statistics.
774 *
775 * @netdev: kernel network device.
776 * @stats: pointer to stats structure to be filled in.
777 */
octep_vf_get_stats64(struct net_device * netdev,struct rtnl_link_stats64 * stats)778 static void octep_vf_get_stats64(struct net_device *netdev,
779 struct rtnl_link_stats64 *stats)
780 {
781 struct octep_vf_device *oct = netdev_priv(netdev);
782 u64 tx_packets, tx_bytes, rx_packets, rx_bytes;
783 int q;
784
785 tx_packets = 0;
786 tx_bytes = 0;
787 rx_packets = 0;
788 rx_bytes = 0;
789 for (q = 0; q < oct->num_oqs; q++) {
790 struct octep_vf_iq *iq = oct->iq[q];
791 struct octep_vf_oq *oq = oct->oq[q];
792
793 tx_packets += iq->stats.instr_completed;
794 tx_bytes += iq->stats.bytes_sent;
795 rx_packets += oq->stats.packets;
796 rx_bytes += oq->stats.bytes;
797 }
798 stats->tx_packets = tx_packets;
799 stats->tx_bytes = tx_bytes;
800 stats->rx_packets = rx_packets;
801 stats->rx_bytes = rx_bytes;
802 if (!octep_vf_get_if_stats(oct)) {
803 stats->multicast = oct->iface_rx_stats.mcast_pkts;
804 stats->rx_errors = oct->iface_rx_stats.err_pkts;
805 stats->rx_dropped = oct->iface_rx_stats.dropped_pkts_fifo_full +
806 oct->iface_rx_stats.err_pkts;
807 stats->rx_missed_errors = oct->iface_rx_stats.dropped_pkts_fifo_full;
808 stats->tx_dropped = oct->iface_tx_stats.dropped;
809 }
810 }
811
812 /**
813 * octep_vf_tx_timeout_task - work queue task to Handle Tx queue timeout.
814 *
815 * @work: pointer to Tx queue timeout work_struct
816 *
817 * Stop and start the device so that it frees up all queue resources
818 * and restarts the queues, that potentially clears a Tx queue timeout
819 * condition.
820 **/
octep_vf_tx_timeout_task(struct work_struct * work)821 static void octep_vf_tx_timeout_task(struct work_struct *work)
822 {
823 struct octep_vf_device *oct = container_of(work, struct octep_vf_device,
824 tx_timeout_task);
825 struct net_device *netdev = oct->netdev;
826
827 rtnl_lock();
828 if (netif_running(netdev)) {
829 octep_vf_stop(netdev);
830 octep_vf_open(netdev);
831 }
832 rtnl_unlock();
833 netdev_put(netdev, NULL);
834 }
835
836 /**
837 * octep_vf_tx_timeout() - Handle Tx Queue timeout.
838 *
839 * @netdev: pointer to kernel network device.
840 * @txqueue: Timed out Tx queue number.
841 *
842 * Schedule a work to handle Tx queue timeout.
843 */
octep_vf_tx_timeout(struct net_device * netdev,unsigned int txqueue)844 static void octep_vf_tx_timeout(struct net_device *netdev, unsigned int txqueue)
845 {
846 struct octep_vf_device *oct = netdev_priv(netdev);
847
848 netdev_hold(netdev, NULL, GFP_ATOMIC);
849 schedule_work(&oct->tx_timeout_task);
850 }
851
octep_vf_set_mac(struct net_device * netdev,void * p)852 static int octep_vf_set_mac(struct net_device *netdev, void *p)
853 {
854 struct octep_vf_device *oct = netdev_priv(netdev);
855 struct sockaddr *addr = (struct sockaddr *)p;
856 int err;
857
858 if (!is_valid_ether_addr(addr->sa_data))
859 return -EADDRNOTAVAIL;
860
861 err = octep_vf_mbox_set_mac_addr(oct, addr->sa_data);
862 if (err)
863 return err;
864
865 memcpy(oct->mac_addr, addr->sa_data, ETH_ALEN);
866 eth_hw_addr_set(netdev, addr->sa_data);
867
868 return 0;
869 }
870
octep_vf_change_mtu(struct net_device * netdev,int new_mtu)871 static int octep_vf_change_mtu(struct net_device *netdev, int new_mtu)
872 {
873 struct octep_vf_device *oct = netdev_priv(netdev);
874 struct octep_vf_iface_link_info *link_info;
875 int err;
876
877 link_info = &oct->link_info;
878 if (link_info->mtu == new_mtu)
879 return 0;
880
881 err = octep_vf_mbox_set_mtu(oct, new_mtu);
882 if (!err) {
883 oct->link_info.mtu = new_mtu;
884 WRITE_ONCE(netdev->mtu, new_mtu);
885 }
886 return err;
887 }
888
octep_vf_set_features(struct net_device * netdev,netdev_features_t features)889 static int octep_vf_set_features(struct net_device *netdev,
890 netdev_features_t features)
891 {
892 struct octep_vf_device *oct = netdev_priv(netdev);
893 u16 rx_offloads = 0, tx_offloads = 0;
894 int err;
895
896 /* We only support features received from firmware */
897 if ((features & netdev->hw_features) != features)
898 return -EINVAL;
899
900 if (features & NETIF_F_TSO)
901 tx_offloads |= OCTEP_VF_TX_OFFLOAD_TSO;
902
903 if (features & NETIF_F_TSO6)
904 tx_offloads |= OCTEP_VF_TX_OFFLOAD_TSO;
905
906 if (features & NETIF_F_IP_CSUM)
907 tx_offloads |= OCTEP_VF_TX_OFFLOAD_CKSUM;
908
909 if (features & NETIF_F_IPV6_CSUM)
910 tx_offloads |= OCTEP_VF_TX_OFFLOAD_CKSUM;
911
912 if (features & NETIF_F_RXCSUM)
913 rx_offloads |= OCTEP_VF_RX_OFFLOAD_CKSUM;
914
915 err = octep_vf_mbox_set_offloads(oct, tx_offloads, rx_offloads);
916 if (!err)
917 netdev->features = features;
918
919 return err;
920 }
921
922 static const struct net_device_ops octep_vf_netdev_ops = {
923 .ndo_open = octep_vf_open,
924 .ndo_stop = octep_vf_stop,
925 .ndo_start_xmit = octep_vf_start_xmit,
926 .ndo_get_stats64 = octep_vf_get_stats64,
927 .ndo_tx_timeout = octep_vf_tx_timeout,
928 .ndo_set_mac_address = octep_vf_set_mac,
929 .ndo_change_mtu = octep_vf_change_mtu,
930 .ndo_set_features = octep_vf_set_features,
931 };
932
octep_vf_devid_to_str(struct octep_vf_device * oct)933 static const char *octep_vf_devid_to_str(struct octep_vf_device *oct)
934 {
935 switch (oct->chip_id) {
936 case OCTEP_PCI_DEVICE_ID_CN93_VF:
937 return "CN93XX";
938 case OCTEP_PCI_DEVICE_ID_CNF95N_VF:
939 return "CNF95N";
940 case OCTEP_PCI_DEVICE_ID_CN10KA_VF:
941 return "CN10KA";
942 case OCTEP_PCI_DEVICE_ID_CNF10KA_VF:
943 return "CNF10KA";
944 case OCTEP_PCI_DEVICE_ID_CNF10KB_VF:
945 return "CNF10KB";
946 case OCTEP_PCI_DEVICE_ID_CN10KB_VF:
947 return "CN10KB";
948 default:
949 return "Unsupported";
950 }
951 }
952
953 /**
954 * octep_vf_device_setup() - Setup Octeon Device.
955 *
956 * @oct: Octeon device private data structure.
957 *
958 * Setup Octeon device hardware operations, configuration, etc ...
959 */
octep_vf_device_setup(struct octep_vf_device * oct)960 int octep_vf_device_setup(struct octep_vf_device *oct)
961 {
962 struct pci_dev *pdev = oct->pdev;
963
964 /* allocate memory for oct->conf */
965 oct->conf = kzalloc(sizeof(*oct->conf), GFP_KERNEL);
966 if (!oct->conf)
967 return -ENOMEM;
968
969 /* Map BAR region 0 */
970 oct->mmio.hw_addr = ioremap(pci_resource_start(oct->pdev, 0),
971 pci_resource_len(oct->pdev, 0));
972 if (!oct->mmio.hw_addr) {
973 dev_err(&pdev->dev,
974 "Failed to remap BAR0; start=0x%llx len=0x%llx\n",
975 pci_resource_start(oct->pdev, 0),
976 pci_resource_len(oct->pdev, 0));
977 goto ioremap_err;
978 }
979 oct->mmio.mapped = 1;
980
981 oct->chip_id = pdev->device;
982 oct->rev_id = pdev->revision;
983 dev_info(&pdev->dev, "chip_id = 0x%x\n", pdev->device);
984
985 switch (oct->chip_id) {
986 case OCTEP_PCI_DEVICE_ID_CN93_VF:
987 case OCTEP_PCI_DEVICE_ID_CNF95N_VF:
988 case OCTEP_PCI_DEVICE_ID_CN98_VF:
989 dev_info(&pdev->dev, "Setting up OCTEON %s VF PASS%d.%d\n",
990 octep_vf_devid_to_str(oct), OCTEP_VF_MAJOR_REV(oct),
991 OCTEP_VF_MINOR_REV(oct));
992 octep_vf_device_setup_cn93(oct);
993 break;
994 case OCTEP_PCI_DEVICE_ID_CNF10KA_VF:
995 case OCTEP_PCI_DEVICE_ID_CN10KA_VF:
996 case OCTEP_PCI_DEVICE_ID_CNF10KB_VF:
997 case OCTEP_PCI_DEVICE_ID_CN10KB_VF:
998 dev_info(&pdev->dev, "Setting up OCTEON %s VF PASS%d.%d\n",
999 octep_vf_devid_to_str(oct), OCTEP_VF_MAJOR_REV(oct),
1000 OCTEP_VF_MINOR_REV(oct));
1001 octep_vf_device_setup_cnxk(oct);
1002 break;
1003 default:
1004 dev_err(&pdev->dev, "Unsupported device\n");
1005 goto unsupported_dev;
1006 }
1007
1008 return 0;
1009
1010 unsupported_dev:
1011 iounmap(oct->mmio.hw_addr);
1012 ioremap_err:
1013 kfree(oct->conf);
1014 return -EOPNOTSUPP;
1015 }
1016
1017 /**
1018 * octep_vf_device_cleanup() - Cleanup Octeon Device.
1019 *
1020 * @oct: Octeon device private data structure.
1021 *
1022 * Cleanup Octeon device allocated resources.
1023 */
octep_vf_device_cleanup(struct octep_vf_device * oct)1024 static void octep_vf_device_cleanup(struct octep_vf_device *oct)
1025 {
1026 dev_info(&oct->pdev->dev, "Cleaning up Octeon Device ...\n");
1027
1028 if (oct->mmio.mapped)
1029 iounmap(oct->mmio.hw_addr);
1030
1031 kfree(oct->conf);
1032 oct->conf = NULL;
1033 }
1034
octep_vf_get_mac_addr(struct octep_vf_device * oct,u8 * addr)1035 static int octep_vf_get_mac_addr(struct octep_vf_device *oct, u8 *addr)
1036 {
1037 return octep_vf_mbox_get_mac_addr(oct, addr);
1038 }
1039
1040 /**
1041 * octep_vf_probe() - Octeon PCI device probe handler.
1042 *
1043 * @pdev: PCI device structure.
1044 * @ent: entry in Octeon PCI device ID table.
1045 *
1046 * Initializes and enables the Octeon PCI device for network operations.
1047 * Initializes Octeon private data structure and registers a network device.
1048 */
octep_vf_probe(struct pci_dev * pdev,const struct pci_device_id * ent)1049 static int octep_vf_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
1050 {
1051 struct octep_vf_device *octep_vf_dev;
1052 struct net_device *netdev;
1053 int err;
1054
1055 err = pci_enable_device(pdev);
1056 if (err) {
1057 dev_err(&pdev->dev, "Failed to enable PCI device\n");
1058 return err;
1059 }
1060
1061 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
1062 if (err) {
1063 dev_err(&pdev->dev, "Failed to set DMA mask !!\n");
1064 goto disable_pci_device;
1065 }
1066
1067 err = pci_request_mem_regions(pdev, OCTEP_VF_DRV_NAME);
1068 if (err) {
1069 dev_err(&pdev->dev, "Failed to map PCI memory regions\n");
1070 goto disable_pci_device;
1071 }
1072
1073 pci_set_master(pdev);
1074
1075 netdev = alloc_etherdev_mq(sizeof(struct octep_vf_device),
1076 OCTEP_VF_MAX_QUEUES);
1077 if (!netdev) {
1078 dev_err(&pdev->dev, "Failed to allocate netdev\n");
1079 err = -ENOMEM;
1080 goto mem_regions_release;
1081 }
1082 SET_NETDEV_DEV(netdev, &pdev->dev);
1083
1084 octep_vf_dev = netdev_priv(netdev);
1085 octep_vf_dev->netdev = netdev;
1086 octep_vf_dev->pdev = pdev;
1087 octep_vf_dev->dev = &pdev->dev;
1088 pci_set_drvdata(pdev, octep_vf_dev);
1089
1090 err = octep_vf_device_setup(octep_vf_dev);
1091 if (err) {
1092 dev_err(&pdev->dev, "Device setup failed\n");
1093 goto netdevice_free;
1094 }
1095 INIT_WORK(&octep_vf_dev->tx_timeout_task, octep_vf_tx_timeout_task);
1096
1097 netdev->netdev_ops = &octep_vf_netdev_ops;
1098 octep_vf_set_ethtool_ops(netdev);
1099 netif_carrier_off(netdev);
1100
1101 if (octep_vf_setup_mbox(octep_vf_dev)) {
1102 dev_err(&pdev->dev, "VF Mailbox setup failed\n");
1103 err = -ENOMEM;
1104 goto device_cleanup;
1105 }
1106
1107 if (octep_vf_mbox_version_check(octep_vf_dev)) {
1108 dev_err(&pdev->dev, "PF VF Mailbox version mismatch\n");
1109 err = -EINVAL;
1110 goto delete_mbox;
1111 }
1112
1113 if (octep_vf_mbox_get_fw_info(octep_vf_dev)) {
1114 dev_err(&pdev->dev, "unable to get fw info\n");
1115 err = -EINVAL;
1116 goto delete_mbox;
1117 }
1118
1119 netdev->hw_features = NETIF_F_SG;
1120 if (OCTEP_VF_TX_IP_CSUM(octep_vf_dev->fw_info.tx_ol_flags))
1121 netdev->hw_features |= (NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM);
1122
1123 if (OCTEP_VF_RX_IP_CSUM(octep_vf_dev->fw_info.rx_ol_flags))
1124 netdev->hw_features |= NETIF_F_RXCSUM;
1125
1126 netdev->min_mtu = OCTEP_VF_MIN_MTU;
1127 netdev->max_mtu = OCTEP_VF_MAX_MTU;
1128 netdev->mtu = OCTEP_VF_DEFAULT_MTU;
1129
1130 if (OCTEP_VF_TX_TSO(octep_vf_dev->fw_info.tx_ol_flags)) {
1131 netdev->hw_features |= NETIF_F_TSO;
1132 netif_set_tso_max_size(netdev, netdev->max_mtu);
1133 }
1134
1135 netdev->features |= netdev->hw_features;
1136 octep_vf_get_mac_addr(octep_vf_dev, octep_vf_dev->mac_addr);
1137 eth_hw_addr_set(netdev, octep_vf_dev->mac_addr);
1138 err = register_netdev(netdev);
1139 if (err) {
1140 dev_err(&pdev->dev, "Failed to register netdev\n");
1141 goto delete_mbox;
1142 }
1143 dev_info(&pdev->dev, "Device probe successful\n");
1144 return 0;
1145
1146 delete_mbox:
1147 octep_vf_delete_mbox(octep_vf_dev);
1148 device_cleanup:
1149 octep_vf_device_cleanup(octep_vf_dev);
1150 netdevice_free:
1151 free_netdev(netdev);
1152 mem_regions_release:
1153 pci_release_mem_regions(pdev);
1154 disable_pci_device:
1155 pci_disable_device(pdev);
1156 dev_err(&pdev->dev, "Device probe failed\n");
1157 return err;
1158 }
1159
1160 /**
1161 * octep_vf_remove() - Remove Octeon PCI device from driver control.
1162 *
1163 * @pdev: PCI device structure of the Octeon device.
1164 *
1165 * Cleanup all resources allocated for the Octeon device.
1166 * Unregister from network device and disable the PCI device.
1167 */
octep_vf_remove(struct pci_dev * pdev)1168 static void octep_vf_remove(struct pci_dev *pdev)
1169 {
1170 struct octep_vf_device *oct = pci_get_drvdata(pdev);
1171 struct net_device *netdev;
1172
1173 if (!oct)
1174 return;
1175
1176 octep_vf_mbox_dev_remove(oct);
1177 cancel_work_sync(&oct->tx_timeout_task);
1178 netdev = oct->netdev;
1179 if (netdev->reg_state == NETREG_REGISTERED)
1180 unregister_netdev(netdev);
1181 octep_vf_delete_mbox(oct);
1182 octep_vf_device_cleanup(oct);
1183 pci_release_mem_regions(pdev);
1184 free_netdev(netdev);
1185 pci_disable_device(pdev);
1186 }
1187
1188 static struct pci_driver octep_vf_driver = {
1189 .name = OCTEP_VF_DRV_NAME,
1190 .id_table = octep_vf_pci_id_tbl,
1191 .probe = octep_vf_probe,
1192 .remove = octep_vf_remove,
1193 };
1194
1195 /**
1196 * octep_vf_init_module() - Module initialization.
1197 *
1198 * create common resource for the driver and register PCI driver.
1199 */
octep_vf_init_module(void)1200 static int __init octep_vf_init_module(void)
1201 {
1202 int ret;
1203
1204 pr_info("%s: Loading %s ...\n", OCTEP_VF_DRV_NAME, OCTEP_VF_DRV_STRING);
1205
1206 ret = pci_register_driver(&octep_vf_driver);
1207 if (ret < 0) {
1208 pr_err("%s: Failed to register PCI driver; err=%d\n",
1209 OCTEP_VF_DRV_NAME, ret);
1210 return ret;
1211 }
1212
1213 return ret;
1214 }
1215
1216 /**
1217 * octep_vf_exit_module() - Module exit routine.
1218 *
1219 * unregister the driver with PCI subsystem and cleanup common resources.
1220 */
octep_vf_exit_module(void)1221 static void __exit octep_vf_exit_module(void)
1222 {
1223 pr_info("%s: Unloading ...\n", OCTEP_VF_DRV_NAME);
1224
1225 pci_unregister_driver(&octep_vf_driver);
1226
1227 pr_info("%s: Unloading complete\n", OCTEP_VF_DRV_NAME);
1228 }
1229
1230 module_init(octep_vf_init_module);
1231 module_exit(octep_vf_exit_module);
1232