xref: /freebsd/sys/dev/qlnx/qlnxe/qlnx_os.c (revision 4f52dfbb8d6c4d446500c5b097e3806ec219fbd4)
1 /*
2  * Copyright (c) 2017-2018 Cavium, Inc.
3  * All rights reserved.
4  *
5  *  Redistribution and use in source and binary forms, with or without
6  *  modification, are permitted provided that the following conditions
7  *  are met:
8  *
9  *  1. Redistributions of source code must retain the above copyright
10  *     notice, this list of conditions and the following disclaimer.
11  *  2. Redistributions in binary form must reproduce the above copyright
12  *     notice, this list of conditions and the following disclaimer in the
13  *     documentation and/or other materials provided with the distribution.
14  *
15  *  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
16  *  AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
17  *  IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
18  *  ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
19  *  LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
20  *  CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
21  *  SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
22  *  INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
23  *  CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
24  *  ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
25  *  POSSIBILITY OF SUCH DAMAGE.
26  */
27 
28 
29 /*
30  * File: qlnx_os.c
31  * Author : David C Somayajulu, Cavium, Inc., San Jose, CA 95131.
32  */
33 
34 #include <sys/cdefs.h>
35 __FBSDID("$FreeBSD$");
36 
37 #include "qlnx_os.h"
38 #include "bcm_osal.h"
39 #include "reg_addr.h"
40 #include "ecore_gtt_reg_addr.h"
41 #include "ecore.h"
42 #include "ecore_chain.h"
43 #include "ecore_status.h"
44 #include "ecore_hw.h"
45 #include "ecore_rt_defs.h"
46 #include "ecore_init_ops.h"
47 #include "ecore_int.h"
48 #include "ecore_cxt.h"
49 #include "ecore_spq.h"
50 #include "ecore_init_fw_funcs.h"
51 #include "ecore_sp_commands.h"
52 #include "ecore_dev_api.h"
53 #include "ecore_l2_api.h"
54 #include "ecore_mcp.h"
55 #include "ecore_hw_defs.h"
56 #include "mcp_public.h"
57 #include "ecore_iro.h"
58 #include "nvm_cfg.h"
59 #include "ecore_dev_api.h"
60 #include "ecore_dbg_fw_funcs.h"
61 
62 #include "qlnx_ioctl.h"
63 #include "qlnx_def.h"
64 #include "qlnx_ver.h"
65 #include <sys/smp.h>
66 
67 
68 /*
69  * static functions
70  */
71 /*
72  * ioctl related functions
73  */
74 static void qlnx_add_sysctls(qlnx_host_t *ha);
75 
76 /*
77  * main driver
78  */
79 static void qlnx_release(qlnx_host_t *ha);
80 static void qlnx_fp_isr(void *arg);
81 static void qlnx_init_ifnet(device_t dev, qlnx_host_t *ha);
82 static void qlnx_init(void *arg);
83 static void qlnx_init_locked(qlnx_host_t *ha);
84 static int qlnx_set_multi(qlnx_host_t *ha, uint32_t add_multi);
85 static int qlnx_set_promisc(qlnx_host_t *ha);
86 static int qlnx_set_allmulti(qlnx_host_t *ha);
87 static int qlnx_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data);
88 static int qlnx_media_change(struct ifnet *ifp);
89 static void qlnx_media_status(struct ifnet *ifp, struct ifmediareq *ifmr);
90 static void qlnx_stop(qlnx_host_t *ha);
91 static int qlnx_send(qlnx_host_t *ha, struct qlnx_fastpath *fp,
92 		struct mbuf **m_headp);
93 static int qlnx_get_ifq_snd_maxlen(qlnx_host_t *ha);
94 static uint32_t qlnx_get_optics(qlnx_host_t *ha,
95 			struct qlnx_link_output *if_link);
96 static int qlnx_transmit(struct ifnet *ifp, struct mbuf  *mp);
97 static int qlnx_transmit_locked(struct ifnet *ifp, struct qlnx_fastpath *fp,
98 		struct mbuf *mp);
99 static void qlnx_qflush(struct ifnet *ifp);
100 
101 static int qlnx_alloc_parent_dma_tag(qlnx_host_t *ha);
102 static void qlnx_free_parent_dma_tag(qlnx_host_t *ha);
103 static int qlnx_alloc_tx_dma_tag(qlnx_host_t *ha);
104 static void qlnx_free_tx_dma_tag(qlnx_host_t *ha);
105 static int qlnx_alloc_rx_dma_tag(qlnx_host_t *ha);
106 static void qlnx_free_rx_dma_tag(qlnx_host_t *ha);
107 
108 static int qlnx_get_mfw_version(qlnx_host_t *ha, uint32_t *mfw_ver);
109 static int qlnx_get_flash_size(qlnx_host_t *ha, uint32_t *flash_size);
110 
111 static int qlnx_nic_setup(struct ecore_dev *cdev,
112 		struct ecore_pf_params *func_params);
113 static int qlnx_nic_start(struct ecore_dev *cdev);
114 static int qlnx_slowpath_start(qlnx_host_t *ha);
115 static int qlnx_slowpath_stop(qlnx_host_t *ha);
116 static int qlnx_init_hw(qlnx_host_t *ha);
117 static void qlnx_set_id(struct ecore_dev *cdev, char name[NAME_SIZE],
118 		char ver_str[VER_SIZE]);
119 static void qlnx_unload(qlnx_host_t *ha);
120 static int qlnx_load(qlnx_host_t *ha);
121 static void qlnx_hw_set_multi(qlnx_host_t *ha, uint8_t *mta, uint32_t mcnt,
122 		uint32_t add_mac);
123 static void qlnx_dump_buf8(qlnx_host_t *ha, const char *msg, void *dbuf,
124 		uint32_t len);
125 static int qlnx_alloc_rx_buffer(qlnx_host_t *ha, struct qlnx_rx_queue *rxq);
126 static void qlnx_reuse_rx_data(struct qlnx_rx_queue *rxq);
127 static void qlnx_update_rx_prod(struct ecore_hwfn *p_hwfn,
128 		struct qlnx_rx_queue *rxq);
129 static int qlnx_set_rx_accept_filter(qlnx_host_t *ha, uint8_t filter);
130 static int qlnx_grc_dumpsize(qlnx_host_t *ha, uint32_t *num_dwords,
131 		int hwfn_index);
132 static int qlnx_idle_chk_size(qlnx_host_t *ha, uint32_t *num_dwords,
133 		int hwfn_index);
134 static void qlnx_timer(void *arg);
135 static int qlnx_alloc_tx_br(qlnx_host_t *ha, struct qlnx_fastpath *fp);
136 static void qlnx_free_tx_br(qlnx_host_t *ha, struct qlnx_fastpath *fp);
137 static void qlnx_trigger_dump(qlnx_host_t *ha);
138 static uint16_t qlnx_num_tx_compl(qlnx_host_t *ha, struct qlnx_fastpath *fp,
139 			struct qlnx_tx_queue *txq);
140 static void qlnx_tx_int(qlnx_host_t *ha, struct qlnx_fastpath *fp,
141 		struct qlnx_tx_queue *txq);
142 static int qlnx_rx_int(qlnx_host_t *ha, struct qlnx_fastpath *fp, int budget,
143 		int lro_enable);
144 static void qlnx_fp_taskqueue(void *context, int pending);
145 static void qlnx_sample_storm_stats(qlnx_host_t *ha);
146 static int qlnx_alloc_tpa_mbuf(qlnx_host_t *ha, uint16_t rx_buf_size,
147 		struct qlnx_agg_info *tpa);
148 static void qlnx_free_tpa_mbuf(qlnx_host_t *ha, struct qlnx_agg_info *tpa);
149 
150 #if __FreeBSD_version >= 1100000
151 static uint64_t qlnx_get_counter(if_t ifp, ift_counter cnt);
152 #endif
153 
154 
155 /*
156  * Hooks to the Operating Systems
157  */
158 static int qlnx_pci_probe (device_t);
159 static int qlnx_pci_attach (device_t);
160 static int qlnx_pci_detach (device_t);
161 
162 static device_method_t qlnx_pci_methods[] = {
163 	/* Device interface */
164 	DEVMETHOD(device_probe, qlnx_pci_probe),
165 	DEVMETHOD(device_attach, qlnx_pci_attach),
166 	DEVMETHOD(device_detach, qlnx_pci_detach),
167 	{ 0, 0 }
168 };
169 
170 static driver_t qlnx_pci_driver = {
171 	"ql", qlnx_pci_methods, sizeof (qlnx_host_t),
172 };
173 
174 static devclass_t qlnx_devclass;
175 
176 MODULE_VERSION(if_qlnxe,1);
177 DRIVER_MODULE(if_qlnxe, pci, qlnx_pci_driver, qlnx_devclass, 0, 0);
178 
179 MODULE_DEPEND(if_qlnxe, pci, 1, 1, 1);
180 MODULE_DEPEND(if_qlnxe, ether, 1, 1, 1);
181 
182 MALLOC_DEFINE(M_QLNXBUF, "qlnxbuf", "Buffers for qlnx driver");
183 
184 
185 char qlnx_dev_str[64];
186 char qlnx_ver_str[VER_SIZE];
187 char qlnx_name_str[NAME_SIZE];
188 
189 /*
190  * Some PCI Configuration Space Related Defines
191  */
192 
193 #ifndef PCI_VENDOR_QLOGIC
194 #define PCI_VENDOR_QLOGIC		0x1077
195 #endif
196 
197 /* 40G Adapter QLE45xxx*/
198 #ifndef QLOGIC_PCI_DEVICE_ID_1634
199 #define QLOGIC_PCI_DEVICE_ID_1634	0x1634
200 #endif
201 
202 /* 100G Adapter QLE45xxx*/
203 #ifndef QLOGIC_PCI_DEVICE_ID_1644
204 #define QLOGIC_PCI_DEVICE_ID_1644	0x1644
205 #endif
206 
207 /* 25G Adapter QLE45xxx*/
208 #ifndef QLOGIC_PCI_DEVICE_ID_1656
209 #define QLOGIC_PCI_DEVICE_ID_1656	0x1656
210 #endif
211 
212 /* 50G Adapter QLE45xxx*/
213 #ifndef QLOGIC_PCI_DEVICE_ID_1654
214 #define QLOGIC_PCI_DEVICE_ID_1654	0x1654
215 #endif
216 
217 /* 10G/25G/40G Adapter QLE41xxx*/
218 #ifndef QLOGIC_PCI_DEVICE_ID_8070
219 #define QLOGIC_PCI_DEVICE_ID_8070	0x8070
220 #endif
221 
222 SYSCTL_NODE(_hw, OID_AUTO, qlnxe, CTLFLAG_RD, 0, "qlnxe driver parameters");
223 /* Number of Queues: 0 (Auto) or 1 to 32 (fixed queue number) */
224 static int qlnxe_queue_count = QLNX_DEFAULT_RSS;
225 SYSCTL_INT(_hw_qlnxe, OID_AUTO, queue_count, CTLFLAG_RDTUN,
226 		&qlnxe_queue_count, 0, "Multi-Queue queue count");
227 
228 static int
229 qlnx_valid_device(device_t dev)
230 {
231         uint16_t	device_id;
232 
233         device_id = pci_get_device(dev);
234 
235         if ((device_id == QLOGIC_PCI_DEVICE_ID_1634) ||
236                 (device_id == QLOGIC_PCI_DEVICE_ID_1644) ||
237                 (device_id == QLOGIC_PCI_DEVICE_ID_1656) ||
238                 (device_id == QLOGIC_PCI_DEVICE_ID_1654) ||
239                 (device_id == QLOGIC_PCI_DEVICE_ID_8070))
240                 return 0;
241 
242         return -1;
243 }
244 
245 /*
246  * Name:	qlnx_pci_probe
247  * Function:	Validate the PCI device to be a QLA80XX device
248  */
249 static int
250 qlnx_pci_probe(device_t dev)
251 {
252 	snprintf(qlnx_ver_str, sizeof(qlnx_ver_str), "v%d.%d.%d",
253 		QLNX_VERSION_MAJOR, QLNX_VERSION_MINOR, QLNX_VERSION_BUILD);
254 	snprintf(qlnx_name_str, sizeof(qlnx_name_str), "qlnx");
255 
256 	if (pci_get_vendor(dev) != PCI_VENDOR_QLOGIC) {
257                 return (ENXIO);
258 	}
259 
260         switch (pci_get_device(dev)) {
261 
262         case QLOGIC_PCI_DEVICE_ID_1644:
263 		snprintf(qlnx_dev_str, sizeof(qlnx_dev_str), "%s v%d.%d.%d",
264 			"Qlogic 100GbE PCI CNA Adapter-Ethernet Function",
265 			QLNX_VERSION_MAJOR, QLNX_VERSION_MINOR,
266 			QLNX_VERSION_BUILD);
267                 device_set_desc_copy(dev, qlnx_dev_str);
268 
269                 break;
270 
271         case QLOGIC_PCI_DEVICE_ID_1634:
272 		snprintf(qlnx_dev_str, sizeof(qlnx_dev_str), "%s v%d.%d.%d",
273 			"Qlogic 40GbE PCI CNA Adapter-Ethernet Function",
274 			QLNX_VERSION_MAJOR, QLNX_VERSION_MINOR,
275 			QLNX_VERSION_BUILD);
276                 device_set_desc_copy(dev, qlnx_dev_str);
277 
278                 break;
279 
280         case QLOGIC_PCI_DEVICE_ID_1656:
281 		snprintf(qlnx_dev_str, sizeof(qlnx_dev_str), "%s v%d.%d.%d",
282 			"Qlogic 25GbE PCI CNA Adapter-Ethernet Function",
283 			QLNX_VERSION_MAJOR, QLNX_VERSION_MINOR,
284 			QLNX_VERSION_BUILD);
285                 device_set_desc_copy(dev, qlnx_dev_str);
286 
287                 break;
288 
289         case QLOGIC_PCI_DEVICE_ID_1654:
290 		snprintf(qlnx_dev_str, sizeof(qlnx_dev_str), "%s v%d.%d.%d",
291 			"Qlogic 50GbE PCI CNA Adapter-Ethernet Function",
292 			QLNX_VERSION_MAJOR, QLNX_VERSION_MINOR,
293 			QLNX_VERSION_BUILD);
294                 device_set_desc_copy(dev, qlnx_dev_str);
295 
296                 break;
297 
298 	case QLOGIC_PCI_DEVICE_ID_8070:
299 		snprintf(qlnx_dev_str, sizeof(qlnx_dev_str), "%s v%d.%d.%d",
300 			"Qlogic 10GbE/25GbE/40GbE PCI CNA (AH) "
301 			"Adapter-Ethernet Function",
302 			QLNX_VERSION_MAJOR, QLNX_VERSION_MINOR,
303 			QLNX_VERSION_BUILD);
304 		device_set_desc_copy(dev, qlnx_dev_str);
305 
306 		break;
307 
308         default:
309                 return (ENXIO);
310         }
311 
312         return (BUS_PROBE_DEFAULT);
313 }
314 
315 static uint16_t
316 qlnx_num_tx_compl(qlnx_host_t *ha, struct qlnx_fastpath *fp,
317 	struct qlnx_tx_queue *txq)
318 {
319 	u16 hw_bd_cons;
320 	u16 ecore_cons_idx;
321 	uint16_t diff;
322 
323 	hw_bd_cons = le16toh(*txq->hw_cons_ptr);
324 
325 	ecore_cons_idx = ecore_chain_get_cons_idx(&txq->tx_pbl);
326 	if (hw_bd_cons < ecore_cons_idx) {
327 		diff = (1 << 16) - (ecore_cons_idx - hw_bd_cons);
328 	} else {
329 		diff = hw_bd_cons - ecore_cons_idx;
330 	}
331 	return diff;
332 }
333 
334 
335 static void
336 qlnx_sp_intr(void *arg)
337 {
338 	struct ecore_hwfn	*p_hwfn;
339 	qlnx_host_t		*ha;
340 	int			i;
341 
342 	p_hwfn = arg;
343 
344 	if (p_hwfn == NULL) {
345 		printf("%s: spurious slowpath intr\n", __func__);
346 		return;
347 	}
348 
349 	ha = (qlnx_host_t *)p_hwfn->p_dev;
350 
351 	QL_DPRINT2(ha, "enter\n");
352 
353 	for (i = 0; i < ha->cdev.num_hwfns; i++) {
354 		if (&ha->cdev.hwfns[i] == p_hwfn) {
355 			taskqueue_enqueue(ha->sp_taskqueue[i], &ha->sp_task[i]);
356 			break;
357 		}
358 	}
359 	QL_DPRINT2(ha, "exit\n");
360 
361 	return;
362 }
363 
364 static void
365 qlnx_sp_taskqueue(void *context, int pending)
366 {
367 	struct ecore_hwfn	*p_hwfn;
368 
369 	p_hwfn = context;
370 
371 	if (p_hwfn != NULL) {
372 		qlnx_sp_isr(p_hwfn);
373 	}
374 	return;
375 }
376 
377 static int
378 qlnx_create_sp_taskqueues(qlnx_host_t *ha)
379 {
380 	int	i;
381 	uint8_t	tq_name[32];
382 
383 	for (i = 0; i < ha->cdev.num_hwfns; i++) {
384 
385                 struct ecore_hwfn *p_hwfn = &ha->cdev.hwfns[i];
386 
387 		bzero(tq_name, sizeof (tq_name));
388 		snprintf(tq_name, sizeof (tq_name), "ql_sp_tq_%d", i);
389 
390 		TASK_INIT(&ha->sp_task[i], 0, qlnx_sp_taskqueue, p_hwfn);
391 
392 		ha->sp_taskqueue[i] = taskqueue_create_fast(tq_name, M_NOWAIT,
393 			 taskqueue_thread_enqueue, &ha->sp_taskqueue[i]);
394 
395 		if (ha->sp_taskqueue[i] == NULL)
396 			return (-1);
397 
398 		taskqueue_start_threads(&ha->sp_taskqueue[i], 1, PI_NET, "%s",
399 			tq_name);
400 
401 		QL_DPRINT1(ha, "%p\n", ha->sp_taskqueue[i]);
402 	}
403 
404 	return (0);
405 }
406 
407 static void
408 qlnx_destroy_sp_taskqueues(qlnx_host_t *ha)
409 {
410 	int	i;
411 
412 	for (i = 0; i < ha->cdev.num_hwfns; i++) {
413 		if (ha->sp_taskqueue[i] != NULL) {
414 			taskqueue_drain(ha->sp_taskqueue[i], &ha->sp_task[i]);
415 			taskqueue_free(ha->sp_taskqueue[i]);
416 		}
417 	}
418 	return;
419 }
420 
421 static void
422 qlnx_fp_taskqueue(void *context, int pending)
423 {
424         struct qlnx_fastpath	*fp;
425         qlnx_host_t		*ha;
426         struct ifnet		*ifp;
427 
428 #ifdef QLNX_RCV_IN_TASKQ
429 	int			lro_enable;
430 	int			rx_int = 0, total_rx_count = 0;
431 	struct thread		*cthread;
432 #endif /* #ifdef QLNX_RCV_IN_TASKQ */
433 
434         fp = context;
435 
436         if (fp == NULL)
437                 return;
438 
439         ha = (qlnx_host_t *)fp->edev;
440 
441         ifp = ha->ifp;
442 
443 #ifdef QLNX_RCV_IN_TASKQ
444 
445 	cthread = curthread;
446 
447 	thread_lock(cthread);
448 
449 	if (!sched_is_bound(cthread))
450 		sched_bind(cthread, fp->rss_id);
451 
452 	thread_unlock(cthread);
453 
454 	lro_enable = ifp->if_capenable & IFCAP_LRO;
455 
456 	rx_int = qlnx_rx_int(ha, fp, ha->rx_pkt_threshold, lro_enable);
457 
458 	if (rx_int) {
459 		fp->rx_pkts += rx_int;
460 		total_rx_count += rx_int;
461 	}
462 
463 #ifdef QLNX_SOFT_LRO
464 	{
465 		struct lro_ctrl *lro;
466 
467 		lro = &fp->rxq->lro;
468 
469 		if (lro_enable && total_rx_count) {
470 
471 #if (__FreeBSD_version >= 1100101) || (defined QLNX_QSORT_LRO)
472 
473 			if (ha->dbg_trace_lro_cnt) {
474 				if (lro->lro_mbuf_count & ~1023)
475 					fp->lro_cnt_1024++;
476 				else if (lro->lro_mbuf_count & ~511)
477 					fp->lro_cnt_512++;
478 				else if (lro->lro_mbuf_count & ~255)
479 					fp->lro_cnt_256++;
480 				else if (lro->lro_mbuf_count & ~127)
481 					fp->lro_cnt_128++;
482 				else if (lro->lro_mbuf_count & ~63)
483 					fp->lro_cnt_64++;
484 			}
485 			tcp_lro_flush_all(lro);
486 
487 #else
488 			struct lro_entry *queued;
489 
490 			while ((!SLIST_EMPTY(&lro->lro_active))) {
491 				queued = SLIST_FIRST(&lro->lro_active);
492 				SLIST_REMOVE_HEAD(&lro->lro_active, next);
493 				tcp_lro_flush(lro, queued);
494 			}
495 #endif /* #if (__FreeBSD_version >= 1100101) || (defined QLNX_QSORT_LRO) */
496 		}
497 	}
498 #endif /* #ifdef QLNX_SOFT_LRO */
499 
500 	ecore_sb_update_sb_idx(fp->sb_info);
501 	rmb();
502 
503 #endif /* #ifdef QLNX_RCV_IN_TASKQ */
504 
505         if(ifp->if_drv_flags & IFF_DRV_RUNNING) {
506 
507                 if (!drbr_empty(ifp, fp->tx_br)) {
508 
509                         if(mtx_trylock(&fp->tx_mtx)) {
510 
511 #ifdef QLNX_TRACE_PERF_DATA
512                                 tx_pkts = fp->tx_pkts_transmitted;
513                                 tx_compl = fp->tx_pkts_completed;
514 #endif
515 
516                                 qlnx_transmit_locked(ifp, fp, NULL);
517 
518 #ifdef QLNX_TRACE_PERF_DATA
519                                 fp->tx_pkts_trans_fp +=
520 					(fp->tx_pkts_transmitted - tx_pkts);
521                                 fp->tx_pkts_compl_fp +=
522 					(fp->tx_pkts_completed - tx_compl);
523 #endif
524                                 mtx_unlock(&fp->tx_mtx);
525                         }
526                 }
527         }
528 
529 #ifdef QLNX_RCV_IN_TASKQ
530 	if (rx_int) {
531 		if (fp->fp_taskqueue != NULL)
532 			taskqueue_enqueue(fp->fp_taskqueue, &fp->fp_task);
533 	} else {
534 		if (fp->tx_ring_full) {
535 			qlnx_mdelay(__func__, 100);
536 		}
537 		ecore_sb_ack(fp->sb_info, IGU_INT_ENABLE, 1);
538 	}
539 #endif /* #ifdef QLNX_RCV_IN_TASKQ */
540 
541         QL_DPRINT2(ha, "exit \n");
542         return;
543 }
544 
545 static int
546 qlnx_create_fp_taskqueues(qlnx_host_t *ha)
547 {
548 	int	i;
549 	uint8_t	tq_name[32];
550 	struct qlnx_fastpath *fp;
551 
552 	for (i = 0; i < ha->num_rss; i++) {
553 
554                 fp = &ha->fp_array[i];
555 
556 		bzero(tq_name, sizeof (tq_name));
557 		snprintf(tq_name, sizeof (tq_name), "ql_fp_tq_%d", i);
558 
559 		TASK_INIT(&fp->fp_task, 0, qlnx_fp_taskqueue, fp);
560 
561 		fp->fp_taskqueue = taskqueue_create_fast(tq_name, M_NOWAIT,
562 					taskqueue_thread_enqueue,
563 					&fp->fp_taskqueue);
564 
565 		if (fp->fp_taskqueue == NULL)
566 			return (-1);
567 
568 		taskqueue_start_threads(&fp->fp_taskqueue, 1, PI_NET, "%s",
569 			tq_name);
570 
571 		QL_DPRINT1(ha, "%p\n",fp->fp_taskqueue);
572 	}
573 
574 	return (0);
575 }
576 
577 static void
578 qlnx_destroy_fp_taskqueues(qlnx_host_t *ha)
579 {
580 	int			i;
581 	struct qlnx_fastpath	*fp;
582 
583 	for (i = 0; i < ha->num_rss; i++) {
584 
585                 fp = &ha->fp_array[i];
586 
587 		if (fp->fp_taskqueue != NULL) {
588 
589 			taskqueue_drain(fp->fp_taskqueue, &fp->fp_task);
590 			taskqueue_free(fp->fp_taskqueue);
591 			fp->fp_taskqueue = NULL;
592 		}
593 	}
594 	return;
595 }
596 
597 static void
598 qlnx_drain_fp_taskqueues(qlnx_host_t *ha)
599 {
600 	int			i;
601 	struct qlnx_fastpath	*fp;
602 
603 	for (i = 0; i < ha->num_rss; i++) {
604                 fp = &ha->fp_array[i];
605 
606 		if (fp->fp_taskqueue != NULL) {
607 			QLNX_UNLOCK(ha);
608 			taskqueue_drain(fp->fp_taskqueue, &fp->fp_task);
609 			QLNX_LOCK(ha);
610 		}
611 	}
612 	return;
613 }
614 
615 static void
616 qlnx_get_params(qlnx_host_t *ha)
617 {
618 	if ((qlnxe_queue_count < 0) || (qlnxe_queue_count > QLNX_MAX_RSS)) {
619 		device_printf(ha->pci_dev, "invalid queue_count value (%d)\n",
620 			qlnxe_queue_count);
621 		qlnxe_queue_count = 0;
622 	}
623 	return;
624 }
625 
626 /*
627  * Name:	qlnx_pci_attach
628  * Function:	attaches the device to the operating system
629  */
630 static int
631 qlnx_pci_attach(device_t dev)
632 {
633 	qlnx_host_t	*ha = NULL;
634 	uint32_t	rsrc_len_reg = 0;
635 	uint32_t	rsrc_len_dbells = 0;
636 	uint32_t	rsrc_len_msix = 0;
637 	int		i;
638 	uint32_t	mfw_ver;
639 
640         if ((ha = device_get_softc(dev)) == NULL) {
641                 device_printf(dev, "cannot get softc\n");
642                 return (ENOMEM);
643         }
644 
645         memset(ha, 0, sizeof (qlnx_host_t));
646 
647         if (qlnx_valid_device(dev) != 0) {
648                 device_printf(dev, "device is not valid device\n");
649                 return (ENXIO);
650 	}
651         ha->pci_func = pci_get_function(dev);
652 
653         ha->pci_dev = dev;
654 
655 	mtx_init(&ha->hw_lock, "qlnx_hw_lock", MTX_NETWORK_LOCK, MTX_DEF);
656 
657         ha->flags.lock_init = 1;
658 
659         pci_enable_busmaster(dev);
660 
661 	/*
662 	 * map the PCI BARs
663 	 */
664 
665         ha->reg_rid = PCIR_BAR(0);
666         ha->pci_reg = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &ha->reg_rid,
667                                 RF_ACTIVE);
668 
669         if (ha->pci_reg == NULL) {
670                 device_printf(dev, "unable to map BAR0\n");
671                 goto qlnx_pci_attach_err;
672         }
673 
674         rsrc_len_reg = (uint32_t) bus_get_resource_count(dev, SYS_RES_MEMORY,
675                                         ha->reg_rid);
676 
677         ha->dbells_rid = PCIR_BAR(2);
678         ha->pci_dbells = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
679                         &ha->dbells_rid, RF_ACTIVE);
680 
681         if (ha->pci_dbells == NULL) {
682                 device_printf(dev, "unable to map BAR1\n");
683                 goto qlnx_pci_attach_err;
684 	}
685 
686         rsrc_len_dbells = (uint32_t) bus_get_resource_count(dev, SYS_RES_MEMORY,
687                                         ha->dbells_rid);
688 
689 	ha->dbells_phys_addr = (uint64_t)
690 		bus_get_resource_start(dev, SYS_RES_MEMORY, ha->dbells_rid);;
691 	ha->dbells_size = rsrc_len_dbells;
692 
693         ha->msix_rid = PCIR_BAR(4);
694         ha->msix_bar = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
695                         &ha->msix_rid, RF_ACTIVE);
696 
697         if (ha->msix_bar == NULL) {
698                 device_printf(dev, "unable to map BAR2\n");
699                 goto qlnx_pci_attach_err;
700 	}
701 
702         rsrc_len_msix = (uint32_t) bus_get_resource_count(dev, SYS_RES_MEMORY,
703                                         ha->msix_rid);
704 	/*
705 	 * allocate dma tags
706 	 */
707 
708 	if (qlnx_alloc_parent_dma_tag(ha))
709                 goto qlnx_pci_attach_err;
710 
711 	if (qlnx_alloc_tx_dma_tag(ha))
712                 goto qlnx_pci_attach_err;
713 
714 	if (qlnx_alloc_rx_dma_tag(ha))
715                 goto qlnx_pci_attach_err;
716 
717 
718 	if (qlnx_init_hw(ha) != 0)
719 		goto qlnx_pci_attach_err;
720 
721 	qlnx_get_params(ha);
722 
723 	if((pci_get_device(dev) == QLOGIC_PCI_DEVICE_ID_1644) &&
724 		(qlnxe_queue_count == QLNX_DEFAULT_RSS)) {
725 		qlnxe_queue_count = QLNX_MAX_RSS;
726 	}
727 
728 	/*
729 	 * Allocate MSI-x vectors
730 	 */
731 	if(qlnxe_queue_count == 0)
732 		ha->num_rss = QLNX_DEFAULT_RSS;
733 	 else
734 		ha->num_rss = qlnxe_queue_count;
735 
736 	ha->num_tc = QLNX_MAX_TC;
737 
738         ha->msix_count = pci_msix_count(dev);
739 
740 	if (ha->msix_count > (mp_ncpus + ha->cdev.num_hwfns))
741 		ha->msix_count = mp_ncpus + ha->cdev.num_hwfns;
742 
743         if (!ha->msix_count ||
744 		(ha->msix_count < (ha->cdev.num_hwfns + 1 ))) {
745                 device_printf(dev, "%s: msix_count[%d] not enough\n", __func__,
746                         ha->msix_count);
747                 goto qlnx_pci_attach_err;
748         }
749 
750 	if (ha->msix_count > (ha->num_rss + ha->cdev.num_hwfns ))
751 		ha->msix_count = ha->num_rss + ha->cdev.num_hwfns;
752 	else
753 		ha->num_rss = ha->msix_count - ha->cdev.num_hwfns;
754 
755 	QL_DPRINT1(ha, "\n\t\t\tpci_reg [%p, 0x%08x 0x%08x]"
756 		"\n\t\t\tdbells [%p, 0x%08x 0x%08x]"
757 		"\n\t\t\tmsix [%p, 0x%08x 0x%08x 0x%x 0x%x]"
758 		"\n\t\t\t[ncpus = %d][num_rss = 0x%x] [num_tc = 0x%x]\n",
759 		 ha->pci_reg, rsrc_len_reg,
760 		ha->reg_rid, ha->pci_dbells, rsrc_len_dbells, ha->dbells_rid,
761 		ha->msix_bar, rsrc_len_msix, ha->msix_rid, pci_msix_count(dev),
762 		ha->msix_count, mp_ncpus, ha->num_rss, ha->num_tc);
763         if (pci_alloc_msix(dev, &ha->msix_count)) {
764                 device_printf(dev, "%s: pci_alloc_msix[%d] failed\n", __func__,
765                         ha->msix_count);
766                 ha->msix_count = 0;
767                 goto qlnx_pci_attach_err;
768         }
769 
770 	/*
771 	 * Initialize slow path interrupt and task queue
772 	 */
773 	if (qlnx_create_sp_taskqueues(ha) != 0)
774 		goto qlnx_pci_attach_err;
775 
776 	for (i = 0; i < ha->cdev.num_hwfns; i++) {
777 
778                 struct ecore_hwfn *p_hwfn = &ha->cdev.hwfns[i];
779 
780         	ha->sp_irq_rid[i] = i + 1;
781         	ha->sp_irq[i] = bus_alloc_resource_any(dev, SYS_RES_IRQ,
782                                 &ha->sp_irq_rid[i],
783                                 (RF_ACTIVE | RF_SHAREABLE));
784         	if (ha->sp_irq[i] == NULL) {
785                 	device_printf(dev,
786 				"could not allocate mbx interrupt\n");
787                 	goto qlnx_pci_attach_err;
788         	}
789 
790         	if (bus_setup_intr(dev, ha->sp_irq[i],
791 				(INTR_TYPE_NET | INTR_MPSAFE), NULL,
792 				qlnx_sp_intr, p_hwfn, &ha->sp_handle[i])) {
793                 	device_printf(dev,
794 				"could not setup slow path interrupt\n");
795 			goto qlnx_pci_attach_err;
796 		}
797 
798 		QL_DPRINT1(ha, "p_hwfn [%p] sp_irq_rid %d"
799 			" sp_irq %p sp_handle %p\n", p_hwfn,
800 			ha->sp_irq_rid[i], ha->sp_irq[i], ha->sp_handle[i]);
801 
802 	}
803 
804 	/*
805 	 * initialize fast path interrupt
806 	 */
807 	if (qlnx_create_fp_taskqueues(ha) != 0)
808 		goto qlnx_pci_attach_err;
809 
810         for (i = 0; i < ha->num_rss; i++) {
811                 ha->irq_vec[i].rss_idx = i;
812                 ha->irq_vec[i].ha = ha;
813                 ha->irq_vec[i].irq_rid = (1 + ha->cdev.num_hwfns) + i;
814 
815                 ha->irq_vec[i].irq = bus_alloc_resource_any(dev, SYS_RES_IRQ,
816                                 &ha->irq_vec[i].irq_rid,
817                                 (RF_ACTIVE | RF_SHAREABLE));
818 
819                 if (ha->irq_vec[i].irq == NULL) {
820                         device_printf(dev,
821 				"could not allocate interrupt[%d]\n", i);
822                         goto qlnx_pci_attach_err;
823                 }
824 
825 		if (qlnx_alloc_tx_br(ha, &ha->fp_array[i])) {
826                         device_printf(dev, "could not allocate tx_br[%d]\n", i);
827                         goto qlnx_pci_attach_err;
828 
829 		}
830 	}
831 
832 	callout_init(&ha->qlnx_callout, 1);
833 	ha->flags.callout_init = 1;
834 
835 	for (i = 0; i < ha->cdev.num_hwfns; i++) {
836 
837 		if (qlnx_grc_dumpsize(ha, &ha->grcdump_size[i], i) != 0)
838 			goto qlnx_pci_attach_err;
839 		if (ha->grcdump_size[i] == 0)
840 			goto qlnx_pci_attach_err;
841 
842 		ha->grcdump_size[i] = ha->grcdump_size[i] << 2;
843 		QL_DPRINT1(ha, "grcdump_size[%d] = 0x%08x\n",
844 			i, ha->grcdump_size[i]);
845 
846 		ha->grcdump[i] = qlnx_zalloc(ha->grcdump_size[i]);
847 		if (ha->grcdump[i] == NULL) {
848 			device_printf(dev, "grcdump alloc[%d] failed\n", i);
849 			goto qlnx_pci_attach_err;
850 		}
851 
852 		if (qlnx_idle_chk_size(ha, &ha->idle_chk_size[i], i) != 0)
853 			goto qlnx_pci_attach_err;
854 		if (ha->idle_chk_size[i] == 0)
855 			goto qlnx_pci_attach_err;
856 
857 		ha->idle_chk_size[i] = ha->idle_chk_size[i] << 2;
858 		QL_DPRINT1(ha, "idle_chk_size[%d] = 0x%08x\n",
859 			i, ha->idle_chk_size[i]);
860 
861 		ha->idle_chk[i] = qlnx_zalloc(ha->idle_chk_size[i]);
862 
863 		if (ha->idle_chk[i] == NULL) {
864 			device_printf(dev, "idle_chk alloc failed\n");
865 			goto qlnx_pci_attach_err;
866 		}
867 	}
868 
869 	if (qlnx_slowpath_start(ha) != 0) {
870 
871 		qlnx_mdelay(__func__, 1000);
872 		qlnx_trigger_dump(ha);
873 
874 		goto qlnx_pci_attach_err0;
875 	} else
876 		ha->flags.slowpath_start = 1;
877 
878 	if (qlnx_get_flash_size(ha, &ha->flash_size) != 0) {
879 		qlnx_mdelay(__func__, 1000);
880 		qlnx_trigger_dump(ha);
881 
882 		goto qlnx_pci_attach_err0;
883 	}
884 
885 	if (qlnx_get_mfw_version(ha, &mfw_ver) != 0) {
886 		qlnx_mdelay(__func__, 1000);
887 		qlnx_trigger_dump(ha);
888 
889 		goto qlnx_pci_attach_err0;
890 	}
891 	snprintf(ha->mfw_ver, sizeof(ha->mfw_ver), "%d.%d.%d.%d",
892 		((mfw_ver >> 24) & 0xFF), ((mfw_ver >> 16) & 0xFF),
893 		((mfw_ver >> 8) & 0xFF), (mfw_ver & 0xFF));
894 	snprintf(ha->stormfw_ver, sizeof(ha->stormfw_ver), "%d.%d.%d.%d",
895 		FW_MAJOR_VERSION, FW_MINOR_VERSION, FW_REVISION_VERSION,
896 		FW_ENGINEERING_VERSION);
897 
898 	QL_DPRINT1(ha, "STORM_FW version %s MFW version %s\n",
899 		 ha->stormfw_ver, ha->mfw_ver);
900 
901 	qlnx_init_ifnet(dev, ha);
902 
903 	/*
904 	 * add sysctls
905 	 */
906 	qlnx_add_sysctls(ha);
907 
908 qlnx_pci_attach_err0:
909         /*
910 	 * create ioctl device interface
911 	 */
912         if (qlnx_make_cdev(ha)) {
913                 device_printf(dev, "%s: ql_make_cdev failed\n", __func__);
914                 goto qlnx_pci_attach_err;
915         }
916 
917 	QL_DPRINT2(ha, "success\n");
918 
919         return (0);
920 
921 qlnx_pci_attach_err:
922 
923 	qlnx_release(ha);
924 
925 	return (ENXIO);
926 }
927 
928 /*
929  * Name:	qlnx_pci_detach
930  * Function:	Unhooks the device from the operating system
931  */
932 static int
933 qlnx_pci_detach(device_t dev)
934 {
935 	qlnx_host_t	*ha = NULL;
936 
937         if ((ha = device_get_softc(dev)) == NULL) {
938                 device_printf(dev, "cannot get softc\n");
939                 return (ENOMEM);
940         }
941 
942 	QLNX_LOCK(ha);
943 	qlnx_stop(ha);
944 	QLNX_UNLOCK(ha);
945 
946 	qlnx_release(ha);
947 
948         return (0);
949 }
950 
951 static int
952 qlnx_init_hw(qlnx_host_t *ha)
953 {
954 	int				rval = 0;
955 	struct ecore_hw_prepare_params	params;
956 
957 	ecore_init_struct(&ha->cdev);
958 
959 	/* ha->dp_module = ECORE_MSG_PROBE |
960 				ECORE_MSG_INTR |
961 				ECORE_MSG_SP |
962 				ECORE_MSG_LINK |
963 				ECORE_MSG_SPQ |
964 				ECORE_MSG_RDMA;
965 	ha->dp_level = ECORE_LEVEL_VERBOSE;*/
966 	ha->dp_level = ECORE_LEVEL_NOTICE;
967 
968 	ecore_init_dp(&ha->cdev, ha->dp_module, ha->dp_level, ha->pci_dev);
969 
970 	ha->cdev.regview = ha->pci_reg;
971 	ha->cdev.doorbells = ha->pci_dbells;
972 	ha->cdev.db_phys_addr = ha->dbells_phys_addr;
973 	ha->cdev.db_size = ha->dbells_size;
974 
975 	bzero(&params, sizeof (struct ecore_hw_prepare_params));
976 
977 	ha->personality = ECORE_PCI_DEFAULT;
978 
979 	params.personality = ha->personality;
980 
981 	params.drv_resc_alloc = false;
982 	params.chk_reg_fifo = false;
983 	params.initiate_pf_flr = true;
984 	params.epoch = 0;
985 
986 	ecore_hw_prepare(&ha->cdev, &params);
987 
988 	qlnx_set_id(&ha->cdev, qlnx_name_str, qlnx_ver_str);
989 
990 	return (rval);
991 }
992 
993 static void
994 qlnx_release(qlnx_host_t *ha)
995 {
996         device_t	dev;
997         int		i;
998 
999         dev = ha->pci_dev;
1000 
1001 	QL_DPRINT2(ha, "enter\n");
1002 
1003 	for (i = 0; i < QLNX_MAX_HW_FUNCS; i++) {
1004 		if (ha->idle_chk[i] != NULL) {
1005 			free(ha->idle_chk[i], M_QLNXBUF);
1006 			ha->idle_chk[i] = NULL;
1007 		}
1008 
1009 		if (ha->grcdump[i] != NULL) {
1010 			free(ha->grcdump[i], M_QLNXBUF);
1011 			ha->grcdump[i] = NULL;
1012 		}
1013 	}
1014 
1015         if (ha->flags.callout_init)
1016                 callout_drain(&ha->qlnx_callout);
1017 
1018 	if (ha->flags.slowpath_start) {
1019 		qlnx_slowpath_stop(ha);
1020 	}
1021 
1022 	ecore_hw_remove(&ha->cdev);
1023 
1024         qlnx_del_cdev(ha);
1025 
1026         if (ha->ifp != NULL)
1027                 ether_ifdetach(ha->ifp);
1028 
1029 	qlnx_free_tx_dma_tag(ha);
1030 
1031 	qlnx_free_rx_dma_tag(ha);
1032 
1033 	qlnx_free_parent_dma_tag(ha);
1034 
1035         for (i = 0; i < ha->num_rss; i++) {
1036 		struct qlnx_fastpath *fp = &ha->fp_array[i];
1037 
1038                 if (ha->irq_vec[i].handle) {
1039                         (void)bus_teardown_intr(dev, ha->irq_vec[i].irq,
1040                                         ha->irq_vec[i].handle);
1041                 }
1042 
1043                 if (ha->irq_vec[i].irq) {
1044                         (void)bus_release_resource(dev, SYS_RES_IRQ,
1045                                 ha->irq_vec[i].irq_rid,
1046                                 ha->irq_vec[i].irq);
1047                 }
1048 
1049 		qlnx_free_tx_br(ha, fp);
1050         }
1051 	qlnx_destroy_fp_taskqueues(ha);
1052 
1053  	for (i = 0; i < ha->cdev.num_hwfns; i++) {
1054         	if (ha->sp_handle[i])
1055                 	(void)bus_teardown_intr(dev, ha->sp_irq[i],
1056 				ha->sp_handle[i]);
1057 
1058         	if (ha->sp_irq[i])
1059 			(void) bus_release_resource(dev, SYS_RES_IRQ,
1060 				ha->sp_irq_rid[i], ha->sp_irq[i]);
1061 	}
1062 
1063 	qlnx_destroy_sp_taskqueues(ha);
1064 
1065         if (ha->msix_count)
1066                 pci_release_msi(dev);
1067 
1068         if (ha->flags.lock_init) {
1069                 mtx_destroy(&ha->hw_lock);
1070         }
1071 
1072         if (ha->pci_reg)
1073                 (void) bus_release_resource(dev, SYS_RES_MEMORY, ha->reg_rid,
1074                                 ha->pci_reg);
1075 
1076         if (ha->pci_dbells)
1077                 (void) bus_release_resource(dev, SYS_RES_MEMORY, ha->dbells_rid,
1078                                 ha->pci_dbells);
1079 
1080         if (ha->msix_bar)
1081                 (void) bus_release_resource(dev, SYS_RES_MEMORY, ha->msix_rid,
1082                                 ha->msix_bar);
1083 
1084 	QL_DPRINT2(ha, "exit\n");
1085 	return;
1086 }
1087 
1088 static void
1089 qlnx_trigger_dump(qlnx_host_t *ha)
1090 {
1091 	int	i;
1092 
1093 	if (ha->ifp != NULL)
1094 		ha->ifp->if_drv_flags &= ~(IFF_DRV_OACTIVE | IFF_DRV_RUNNING);
1095 
1096 	QL_DPRINT2(ha, "enter\n");
1097 
1098 	for (i = 0; i < ha->cdev.num_hwfns; i++) {
1099 		qlnx_grc_dump(ha, &ha->grcdump_dwords[i], i);
1100 		qlnx_idle_chk(ha, &ha->idle_chk_dwords[i], i);
1101 	}
1102 
1103 	QL_DPRINT2(ha, "exit\n");
1104 
1105 	return;
1106 }
1107 
1108 static int
1109 qlnx_trigger_dump_sysctl(SYSCTL_HANDLER_ARGS)
1110 {
1111         int		err, ret = 0;
1112         qlnx_host_t	*ha;
1113 
1114         err = sysctl_handle_int(oidp, &ret, 0, req);
1115 
1116         if (err || !req->newptr)
1117                 return (err);
1118 
1119         if (ret == 1) {
1120                 ha = (qlnx_host_t *)arg1;
1121                 qlnx_trigger_dump(ha);
1122         }
1123         return (err);
1124 }
1125 
1126 static int
1127 qlnx_set_tx_coalesce(SYSCTL_HANDLER_ARGS)
1128 {
1129         int			err, i, ret = 0, usecs = 0;
1130         qlnx_host_t		*ha;
1131 	struct ecore_hwfn	*p_hwfn;
1132 	struct qlnx_fastpath	*fp;
1133 
1134         err = sysctl_handle_int(oidp, &usecs, 0, req);
1135 
1136         if (err || !req->newptr || !usecs || (usecs > 255))
1137                 return (err);
1138 
1139         ha = (qlnx_host_t *)arg1;
1140 
1141 	for (i = 0; i < ha->num_rss; i++) {
1142 
1143 		p_hwfn = &ha->cdev.hwfns[(i % ha->cdev.num_hwfns)];
1144 
1145         	fp = &ha->fp_array[i];
1146 
1147 		if (fp->txq[0]->handle != NULL) {
1148 			ret = ecore_set_queue_coalesce(p_hwfn, 0,
1149 					(uint16_t)usecs, fp->txq[0]->handle);
1150 		}
1151         }
1152 
1153 	if (!ret)
1154 		ha->tx_coalesce_usecs = (uint8_t)usecs;
1155 
1156         return (err);
1157 }
1158 
1159 static int
1160 qlnx_set_rx_coalesce(SYSCTL_HANDLER_ARGS)
1161 {
1162         int			err, i, ret = 0, usecs = 0;
1163         qlnx_host_t		*ha;
1164 	struct ecore_hwfn	*p_hwfn;
1165 	struct qlnx_fastpath	*fp;
1166 
1167         err = sysctl_handle_int(oidp, &usecs, 0, req);
1168 
1169         if (err || !req->newptr || !usecs || (usecs > 255))
1170                 return (err);
1171 
1172         ha = (qlnx_host_t *)arg1;
1173 
1174 	for (i = 0; i < ha->num_rss; i++) {
1175 
1176 		p_hwfn = &ha->cdev.hwfns[(i % ha->cdev.num_hwfns)];
1177 
1178         	fp = &ha->fp_array[i];
1179 
1180 		if (fp->rxq->handle != NULL) {
1181 			ret = ecore_set_queue_coalesce(p_hwfn, (uint16_t)usecs,
1182 					 0, fp->rxq->handle);
1183 		}
1184 	}
1185 
1186 	if (!ret)
1187 		ha->rx_coalesce_usecs = (uint8_t)usecs;
1188 
1189         return (err);
1190 }
1191 
1192 static void
1193 qlnx_add_sp_stats_sysctls(qlnx_host_t *ha)
1194 {
1195         struct sysctl_ctx_list	*ctx;
1196         struct sysctl_oid_list	*children;
1197 	struct sysctl_oid	*ctx_oid;
1198 
1199         ctx = device_get_sysctl_ctx(ha->pci_dev);
1200 	children = SYSCTL_CHILDREN(device_get_sysctl_tree(ha->pci_dev));
1201 
1202 	ctx_oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "spstat",
1203 			CTLFLAG_RD, NULL, "spstat");
1204         children = SYSCTL_CHILDREN(ctx_oid);
1205 
1206 	SYSCTL_ADD_QUAD(ctx, children,
1207                 OID_AUTO, "sp_interrupts",
1208                 CTLFLAG_RD, &ha->sp_interrupts,
1209                 "No. of slowpath interrupts");
1210 
1211 	return;
1212 }
1213 
1214 static void
1215 qlnx_add_fp_stats_sysctls(qlnx_host_t *ha)
1216 {
1217         struct sysctl_ctx_list	*ctx;
1218         struct sysctl_oid_list	*children;
1219         struct sysctl_oid_list	*node_children;
1220 	struct sysctl_oid	*ctx_oid;
1221 	int			i, j;
1222 	uint8_t			name_str[16];
1223 
1224         ctx = device_get_sysctl_ctx(ha->pci_dev);
1225 	children = SYSCTL_CHILDREN(device_get_sysctl_tree(ha->pci_dev));
1226 
1227 	ctx_oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "fpstat",
1228 			CTLFLAG_RD, NULL, "fpstat");
1229 	children = SYSCTL_CHILDREN(ctx_oid);
1230 
1231 	for (i = 0; i < ha->num_rss; i++) {
1232 
1233 		bzero(name_str, (sizeof(uint8_t) * sizeof(name_str)));
1234 		snprintf(name_str, sizeof(name_str), "%d", i);
1235 
1236 		ctx_oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, name_str,
1237 			CTLFLAG_RD, NULL, name_str);
1238 		node_children = SYSCTL_CHILDREN(ctx_oid);
1239 
1240 		/* Tx Related */
1241 
1242 		SYSCTL_ADD_QUAD(ctx, node_children,
1243 			OID_AUTO, "tx_pkts_processed",
1244 			CTLFLAG_RD, &ha->fp_array[i].tx_pkts_processed,
1245 			"No. of packets processed for transmission");
1246 
1247 		SYSCTL_ADD_QUAD(ctx, node_children,
1248 			OID_AUTO, "tx_pkts_freed",
1249 			CTLFLAG_RD, &ha->fp_array[i].tx_pkts_freed,
1250 			"No. of freed packets");
1251 
1252 		SYSCTL_ADD_QUAD(ctx, node_children,
1253 			OID_AUTO, "tx_pkts_transmitted",
1254 			CTLFLAG_RD, &ha->fp_array[i].tx_pkts_transmitted,
1255 			"No. of transmitted packets");
1256 
1257 		SYSCTL_ADD_QUAD(ctx, node_children,
1258 			OID_AUTO, "tx_pkts_completed",
1259 			CTLFLAG_RD, &ha->fp_array[i].tx_pkts_completed,
1260 			"No. of transmit completions");
1261 
1262                 SYSCTL_ADD_QUAD(ctx, node_children,
1263                         OID_AUTO, "tx_non_tso_pkts",
1264                         CTLFLAG_RD, &ha->fp_array[i].tx_non_tso_pkts,
1265                         "No. of non LSO transmited packets");
1266 
1267 #ifdef QLNX_TRACE_PERF_DATA
1268 
1269                 SYSCTL_ADD_QUAD(ctx, node_children,
1270                         OID_AUTO, "tx_pkts_trans_ctx",
1271                         CTLFLAG_RD, &ha->fp_array[i].tx_pkts_trans_ctx,
1272                         "No. of transmitted packets in transmit context");
1273 
1274                 SYSCTL_ADD_QUAD(ctx, node_children,
1275                         OID_AUTO, "tx_pkts_compl_ctx",
1276                         CTLFLAG_RD, &ha->fp_array[i].tx_pkts_compl_ctx,
1277                         "No. of transmit completions in transmit context");
1278 
1279                 SYSCTL_ADD_QUAD(ctx, node_children,
1280                         OID_AUTO, "tx_pkts_trans_fp",
1281                         CTLFLAG_RD, &ha->fp_array[i].tx_pkts_trans_fp,
1282                         "No. of transmitted packets in taskqueue");
1283 
1284                 SYSCTL_ADD_QUAD(ctx, node_children,
1285                         OID_AUTO, "tx_pkts_compl_fp",
1286                         CTLFLAG_RD, &ha->fp_array[i].tx_pkts_compl_fp,
1287                         "No. of transmit completions in taskqueue");
1288 
1289                 SYSCTL_ADD_QUAD(ctx, node_children,
1290                         OID_AUTO, "tx_pkts_compl_intr",
1291                         CTLFLAG_RD, &ha->fp_array[i].tx_pkts_compl_intr,
1292                         "No. of transmit completions in interrupt ctx");
1293 #endif
1294 
1295                 SYSCTL_ADD_QUAD(ctx, node_children,
1296                         OID_AUTO, "tx_tso_pkts",
1297                         CTLFLAG_RD, &ha->fp_array[i].tx_tso_pkts,
1298                         "No. of LSO transmited packets");
1299 
1300 		SYSCTL_ADD_QUAD(ctx, node_children,
1301 			OID_AUTO, "tx_lso_wnd_min_len",
1302 			CTLFLAG_RD, &ha->fp_array[i].tx_lso_wnd_min_len,
1303 			"tx_lso_wnd_min_len");
1304 
1305 		SYSCTL_ADD_QUAD(ctx, node_children,
1306 			OID_AUTO, "tx_defrag",
1307 			CTLFLAG_RD, &ha->fp_array[i].tx_defrag,
1308 			"tx_defrag");
1309 
1310 		SYSCTL_ADD_QUAD(ctx, node_children,
1311 			OID_AUTO, "tx_nsegs_gt_elem_left",
1312 			CTLFLAG_RD, &ha->fp_array[i].tx_nsegs_gt_elem_left,
1313 			"tx_nsegs_gt_elem_left");
1314 
1315 		SYSCTL_ADD_UINT(ctx, node_children,
1316 			OID_AUTO, "tx_tso_max_nsegs",
1317 			CTLFLAG_RD, &ha->fp_array[i].tx_tso_max_nsegs,
1318 			ha->fp_array[i].tx_tso_max_nsegs, "tx_tso_max_nsegs");
1319 
1320 		SYSCTL_ADD_UINT(ctx, node_children,
1321 			OID_AUTO, "tx_tso_min_nsegs",
1322 			CTLFLAG_RD, &ha->fp_array[i].tx_tso_min_nsegs,
1323 			ha->fp_array[i].tx_tso_min_nsegs, "tx_tso_min_nsegs");
1324 
1325 		SYSCTL_ADD_UINT(ctx, node_children,
1326 			OID_AUTO, "tx_tso_max_pkt_len",
1327 			CTLFLAG_RD, &ha->fp_array[i].tx_tso_max_pkt_len,
1328 			ha->fp_array[i].tx_tso_max_pkt_len,
1329 			"tx_tso_max_pkt_len");
1330 
1331 		SYSCTL_ADD_UINT(ctx, node_children,
1332 			OID_AUTO, "tx_tso_min_pkt_len",
1333 			CTLFLAG_RD, &ha->fp_array[i].tx_tso_min_pkt_len,
1334 			ha->fp_array[i].tx_tso_min_pkt_len,
1335 			"tx_tso_min_pkt_len");
1336 
1337 		for (j = 0; j < QLNX_FP_MAX_SEGS; j++) {
1338 
1339 			bzero(name_str, (sizeof(uint8_t) * sizeof(name_str)));
1340 			snprintf(name_str, sizeof(name_str),
1341 				"tx_pkts_nseg_%02d", (j+1));
1342 
1343 			SYSCTL_ADD_QUAD(ctx, node_children,
1344 				OID_AUTO, name_str, CTLFLAG_RD,
1345 				&ha->fp_array[i].tx_pkts[j], name_str);
1346 		}
1347 
1348 #ifdef QLNX_TRACE_PERF_DATA
1349                 for (j = 0; j < 18; j++) {
1350 
1351                         bzero(name_str, (sizeof(uint8_t) * sizeof(name_str)));
1352                         snprintf(name_str, sizeof(name_str),
1353                                 "tx_pkts_hist_%02d", (j+1));
1354 
1355                         SYSCTL_ADD_QUAD(ctx, node_children,
1356                                 OID_AUTO, name_str, CTLFLAG_RD,
1357                                 &ha->fp_array[i].tx_pkts_hist[j], name_str);
1358                 }
1359                 for (j = 0; j < 5; j++) {
1360 
1361                         bzero(name_str, (sizeof(uint8_t) * sizeof(name_str)));
1362                         snprintf(name_str, sizeof(name_str),
1363                                 "tx_comInt_%02d", (j+1));
1364 
1365                         SYSCTL_ADD_QUAD(ctx, node_children,
1366                                 OID_AUTO, name_str, CTLFLAG_RD,
1367                                 &ha->fp_array[i].tx_comInt[j], name_str);
1368                 }
1369                 for (j = 0; j < 18; j++) {
1370 
1371                         bzero(name_str, (sizeof(uint8_t) * sizeof(name_str)));
1372                         snprintf(name_str, sizeof(name_str),
1373                                 "tx_pkts_q_%02d", (j+1));
1374 
1375                         SYSCTL_ADD_QUAD(ctx, node_children,
1376                                 OID_AUTO, name_str, CTLFLAG_RD,
1377                                 &ha->fp_array[i].tx_pkts_q[j], name_str);
1378                 }
1379 #endif
1380 
1381 		SYSCTL_ADD_QUAD(ctx, node_children,
1382 			OID_AUTO, "err_tx_nsegs_gt_elem_left",
1383 			CTLFLAG_RD, &ha->fp_array[i].err_tx_nsegs_gt_elem_left,
1384 			"err_tx_nsegs_gt_elem_left");
1385 
1386 		SYSCTL_ADD_QUAD(ctx, node_children,
1387 			OID_AUTO, "err_tx_dmamap_create",
1388 			CTLFLAG_RD, &ha->fp_array[i].err_tx_dmamap_create,
1389 			"err_tx_dmamap_create");
1390 
1391 		SYSCTL_ADD_QUAD(ctx, node_children,
1392 			OID_AUTO, "err_tx_defrag_dmamap_load",
1393 			CTLFLAG_RD, &ha->fp_array[i].err_tx_defrag_dmamap_load,
1394 			"err_tx_defrag_dmamap_load");
1395 
1396 		SYSCTL_ADD_QUAD(ctx, node_children,
1397 			OID_AUTO, "err_tx_non_tso_max_seg",
1398 			CTLFLAG_RD, &ha->fp_array[i].err_tx_non_tso_max_seg,
1399 			"err_tx_non_tso_max_seg");
1400 
1401 		SYSCTL_ADD_QUAD(ctx, node_children,
1402 			OID_AUTO, "err_tx_dmamap_load",
1403 			CTLFLAG_RD, &ha->fp_array[i].err_tx_dmamap_load,
1404 			"err_tx_dmamap_load");
1405 
1406 		SYSCTL_ADD_QUAD(ctx, node_children,
1407 			OID_AUTO, "err_tx_defrag",
1408 			CTLFLAG_RD, &ha->fp_array[i].err_tx_defrag,
1409 			"err_tx_defrag");
1410 
1411 		SYSCTL_ADD_QUAD(ctx, node_children,
1412 			OID_AUTO, "err_tx_free_pkt_null",
1413 			CTLFLAG_RD, &ha->fp_array[i].err_tx_free_pkt_null,
1414 			"err_tx_free_pkt_null");
1415 
1416 		SYSCTL_ADD_QUAD(ctx, node_children,
1417 			OID_AUTO, "err_tx_cons_idx_conflict",
1418 			CTLFLAG_RD, &ha->fp_array[i].err_tx_cons_idx_conflict,
1419 			"err_tx_cons_idx_conflict");
1420 
1421 		SYSCTL_ADD_QUAD(ctx, node_children,
1422 			OID_AUTO, "lro_cnt_64",
1423 			CTLFLAG_RD, &ha->fp_array[i].lro_cnt_64,
1424 			"lro_cnt_64");
1425 
1426 		SYSCTL_ADD_QUAD(ctx, node_children,
1427 			OID_AUTO, "lro_cnt_128",
1428 			CTLFLAG_RD, &ha->fp_array[i].lro_cnt_128,
1429 			"lro_cnt_128");
1430 
1431 		SYSCTL_ADD_QUAD(ctx, node_children,
1432 			OID_AUTO, "lro_cnt_256",
1433 			CTLFLAG_RD, &ha->fp_array[i].lro_cnt_256,
1434 			"lro_cnt_256");
1435 
1436 		SYSCTL_ADD_QUAD(ctx, node_children,
1437 			OID_AUTO, "lro_cnt_512",
1438 			CTLFLAG_RD, &ha->fp_array[i].lro_cnt_512,
1439 			"lro_cnt_512");
1440 
1441 		SYSCTL_ADD_QUAD(ctx, node_children,
1442 			OID_AUTO, "lro_cnt_1024",
1443 			CTLFLAG_RD, &ha->fp_array[i].lro_cnt_1024,
1444 			"lro_cnt_1024");
1445 
1446 		/* Rx Related */
1447 
1448 		SYSCTL_ADD_QUAD(ctx, node_children,
1449 			OID_AUTO, "rx_pkts",
1450 			CTLFLAG_RD, &ha->fp_array[i].rx_pkts,
1451 			"No. of received packets");
1452 
1453 		SYSCTL_ADD_QUAD(ctx, node_children,
1454 			OID_AUTO, "tpa_start",
1455 			CTLFLAG_RD, &ha->fp_array[i].tpa_start,
1456 			"No. of tpa_start packets");
1457 
1458 		SYSCTL_ADD_QUAD(ctx, node_children,
1459 			OID_AUTO, "tpa_cont",
1460 			CTLFLAG_RD, &ha->fp_array[i].tpa_cont,
1461 			"No. of tpa_cont packets");
1462 
1463 		SYSCTL_ADD_QUAD(ctx, node_children,
1464 			OID_AUTO, "tpa_end",
1465 			CTLFLAG_RD, &ha->fp_array[i].tpa_end,
1466 			"No. of tpa_end packets");
1467 
1468 		SYSCTL_ADD_QUAD(ctx, node_children,
1469 			OID_AUTO, "err_m_getcl",
1470 			CTLFLAG_RD, &ha->fp_array[i].err_m_getcl,
1471 			"err_m_getcl");
1472 
1473 		SYSCTL_ADD_QUAD(ctx, node_children,
1474 			OID_AUTO, "err_m_getjcl",
1475 			CTLFLAG_RD, &ha->fp_array[i].err_m_getjcl,
1476 			"err_m_getjcl");
1477 
1478 		SYSCTL_ADD_QUAD(ctx, node_children,
1479 			OID_AUTO, "err_rx_hw_errors",
1480 			CTLFLAG_RD, &ha->fp_array[i].err_rx_hw_errors,
1481 			"err_rx_hw_errors");
1482 
1483 		SYSCTL_ADD_QUAD(ctx, node_children,
1484 			OID_AUTO, "err_rx_alloc_errors",
1485 			CTLFLAG_RD, &ha->fp_array[i].err_rx_alloc_errors,
1486 			"err_rx_alloc_errors");
1487 	}
1488 
1489 	return;
1490 }
1491 
1492 static void
1493 qlnx_add_hw_stats_sysctls(qlnx_host_t *ha)
1494 {
1495         struct sysctl_ctx_list	*ctx;
1496         struct sysctl_oid_list	*children;
1497 	struct sysctl_oid	*ctx_oid;
1498 
1499         ctx = device_get_sysctl_ctx(ha->pci_dev);
1500 	children = SYSCTL_CHILDREN(device_get_sysctl_tree(ha->pci_dev));
1501 
1502 	ctx_oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "hwstat",
1503 			CTLFLAG_RD, NULL, "hwstat");
1504         children = SYSCTL_CHILDREN(ctx_oid);
1505 
1506 	SYSCTL_ADD_QUAD(ctx, children,
1507                 OID_AUTO, "no_buff_discards",
1508                 CTLFLAG_RD, &ha->hw_stats.common.no_buff_discards,
1509                 "No. of packets discarded due to lack of buffer");
1510 
1511 	SYSCTL_ADD_QUAD(ctx, children,
1512                 OID_AUTO, "packet_too_big_discard",
1513                 CTLFLAG_RD, &ha->hw_stats.common.packet_too_big_discard,
1514                 "No. of packets discarded because packet was too big");
1515 
1516 	SYSCTL_ADD_QUAD(ctx, children,
1517                 OID_AUTO, "ttl0_discard",
1518                 CTLFLAG_RD, &ha->hw_stats.common.ttl0_discard,
1519                 "ttl0_discard");
1520 
1521 	SYSCTL_ADD_QUAD(ctx, children,
1522                 OID_AUTO, "rx_ucast_bytes",
1523                 CTLFLAG_RD, &ha->hw_stats.common.rx_ucast_bytes,
1524                 "rx_ucast_bytes");
1525 
1526 	SYSCTL_ADD_QUAD(ctx, children,
1527                 OID_AUTO, "rx_mcast_bytes",
1528                 CTLFLAG_RD, &ha->hw_stats.common.rx_mcast_bytes,
1529                 "rx_mcast_bytes");
1530 
1531 	SYSCTL_ADD_QUAD(ctx, children,
1532                 OID_AUTO, "rx_bcast_bytes",
1533                 CTLFLAG_RD, &ha->hw_stats.common.rx_bcast_bytes,
1534                 "rx_bcast_bytes");
1535 
1536 	SYSCTL_ADD_QUAD(ctx, children,
1537                 OID_AUTO, "rx_ucast_pkts",
1538                 CTLFLAG_RD, &ha->hw_stats.common.rx_ucast_pkts,
1539                 "rx_ucast_pkts");
1540 
1541 	SYSCTL_ADD_QUAD(ctx, children,
1542                 OID_AUTO, "rx_mcast_pkts",
1543                 CTLFLAG_RD, &ha->hw_stats.common.rx_mcast_pkts,
1544                 "rx_mcast_pkts");
1545 
1546 	SYSCTL_ADD_QUAD(ctx, children,
1547                 OID_AUTO, "rx_bcast_pkts",
1548                 CTLFLAG_RD, &ha->hw_stats.common.rx_bcast_pkts,
1549                 "rx_bcast_pkts");
1550 
1551 	SYSCTL_ADD_QUAD(ctx, children,
1552                 OID_AUTO, "mftag_filter_discards",
1553                 CTLFLAG_RD, &ha->hw_stats.common.mftag_filter_discards,
1554                 "mftag_filter_discards");
1555 
1556 	SYSCTL_ADD_QUAD(ctx, children,
1557                 OID_AUTO, "mac_filter_discards",
1558                 CTLFLAG_RD, &ha->hw_stats.common.mac_filter_discards,
1559                 "mac_filter_discards");
1560 
1561 	SYSCTL_ADD_QUAD(ctx, children,
1562                 OID_AUTO, "tx_ucast_bytes",
1563                 CTLFLAG_RD, &ha->hw_stats.common.tx_ucast_bytes,
1564                 "tx_ucast_bytes");
1565 
1566 	SYSCTL_ADD_QUAD(ctx, children,
1567                 OID_AUTO, "tx_mcast_bytes",
1568                 CTLFLAG_RD, &ha->hw_stats.common.tx_mcast_bytes,
1569                 "tx_mcast_bytes");
1570 
1571 	SYSCTL_ADD_QUAD(ctx, children,
1572                 OID_AUTO, "tx_bcast_bytes",
1573                 CTLFLAG_RD, &ha->hw_stats.common.tx_bcast_bytes,
1574                 "tx_bcast_bytes");
1575 
1576 	SYSCTL_ADD_QUAD(ctx, children,
1577                 OID_AUTO, "tx_ucast_pkts",
1578                 CTLFLAG_RD, &ha->hw_stats.common.tx_ucast_pkts,
1579                 "tx_ucast_pkts");
1580 
1581 	SYSCTL_ADD_QUAD(ctx, children,
1582                 OID_AUTO, "tx_mcast_pkts",
1583                 CTLFLAG_RD, &ha->hw_stats.common.tx_mcast_pkts,
1584                 "tx_mcast_pkts");
1585 
1586 	SYSCTL_ADD_QUAD(ctx, children,
1587                 OID_AUTO, "tx_bcast_pkts",
1588                 CTLFLAG_RD, &ha->hw_stats.common.tx_bcast_pkts,
1589                 "tx_bcast_pkts");
1590 
1591 	SYSCTL_ADD_QUAD(ctx, children,
1592                 OID_AUTO, "tx_err_drop_pkts",
1593                 CTLFLAG_RD, &ha->hw_stats.common.tx_err_drop_pkts,
1594                 "tx_err_drop_pkts");
1595 
1596 	SYSCTL_ADD_QUAD(ctx, children,
1597                 OID_AUTO, "tpa_coalesced_pkts",
1598                 CTLFLAG_RD, &ha->hw_stats.common.tpa_coalesced_pkts,
1599                 "tpa_coalesced_pkts");
1600 
1601 	SYSCTL_ADD_QUAD(ctx, children,
1602                 OID_AUTO, "tpa_coalesced_events",
1603                 CTLFLAG_RD, &ha->hw_stats.common.tpa_coalesced_events,
1604                 "tpa_coalesced_events");
1605 
1606 	SYSCTL_ADD_QUAD(ctx, children,
1607                 OID_AUTO, "tpa_aborts_num",
1608                 CTLFLAG_RD, &ha->hw_stats.common.tpa_aborts_num,
1609                 "tpa_aborts_num");
1610 
1611 	SYSCTL_ADD_QUAD(ctx, children,
1612                 OID_AUTO, "tpa_not_coalesced_pkts",
1613                 CTLFLAG_RD, &ha->hw_stats.common.tpa_not_coalesced_pkts,
1614                 "tpa_not_coalesced_pkts");
1615 
1616 	SYSCTL_ADD_QUAD(ctx, children,
1617                 OID_AUTO, "tpa_coalesced_bytes",
1618                 CTLFLAG_RD, &ha->hw_stats.common.tpa_coalesced_bytes,
1619                 "tpa_coalesced_bytes");
1620 
1621 	SYSCTL_ADD_QUAD(ctx, children,
1622                 OID_AUTO, "rx_64_byte_packets",
1623                 CTLFLAG_RD, &ha->hw_stats.common.rx_64_byte_packets,
1624                 "rx_64_byte_packets");
1625 
1626 	SYSCTL_ADD_QUAD(ctx, children,
1627                 OID_AUTO, "rx_65_to_127_byte_packets",
1628                 CTLFLAG_RD, &ha->hw_stats.common.rx_65_to_127_byte_packets,
1629                 "rx_65_to_127_byte_packets");
1630 
1631 	SYSCTL_ADD_QUAD(ctx, children,
1632                 OID_AUTO, "rx_128_to_255_byte_packets",
1633                 CTLFLAG_RD, &ha->hw_stats.common.rx_128_to_255_byte_packets,
1634                 "rx_128_to_255_byte_packets");
1635 
1636 	SYSCTL_ADD_QUAD(ctx, children,
1637                 OID_AUTO, "rx_256_to_511_byte_packets",
1638                 CTLFLAG_RD, &ha->hw_stats.common.rx_256_to_511_byte_packets,
1639                 "rx_256_to_511_byte_packets");
1640 
1641 	SYSCTL_ADD_QUAD(ctx, children,
1642                 OID_AUTO, "rx_512_to_1023_byte_packets",
1643                 CTLFLAG_RD, &ha->hw_stats.common.rx_512_to_1023_byte_packets,
1644                 "rx_512_to_1023_byte_packets");
1645 
1646 	SYSCTL_ADD_QUAD(ctx, children,
1647                 OID_AUTO, "rx_1024_to_1518_byte_packets",
1648                 CTLFLAG_RD, &ha->hw_stats.common.rx_1024_to_1518_byte_packets,
1649                 "rx_1024_to_1518_byte_packets");
1650 
1651 	SYSCTL_ADD_QUAD(ctx, children,
1652                 OID_AUTO, "rx_1519_to_1522_byte_packets",
1653                 CTLFLAG_RD, &ha->hw_stats.bb.rx_1519_to_1522_byte_packets,
1654                 "rx_1519_to_1522_byte_packets");
1655 
1656 	SYSCTL_ADD_QUAD(ctx, children,
1657                 OID_AUTO, "rx_1523_to_2047_byte_packets",
1658                 CTLFLAG_RD, &ha->hw_stats.bb.rx_1519_to_2047_byte_packets,
1659                 "rx_1523_to_2047_byte_packets");
1660 
1661 	SYSCTL_ADD_QUAD(ctx, children,
1662                 OID_AUTO, "rx_2048_to_4095_byte_packets",
1663                 CTLFLAG_RD, &ha->hw_stats.bb.rx_2048_to_4095_byte_packets,
1664                 "rx_2048_to_4095_byte_packets");
1665 
1666 	SYSCTL_ADD_QUAD(ctx, children,
1667                 OID_AUTO, "rx_4096_to_9216_byte_packets",
1668                 CTLFLAG_RD, &ha->hw_stats.bb.rx_4096_to_9216_byte_packets,
1669                 "rx_4096_to_9216_byte_packets");
1670 
1671 	SYSCTL_ADD_QUAD(ctx, children,
1672                 OID_AUTO, "rx_9217_to_16383_byte_packets",
1673                 CTLFLAG_RD, &ha->hw_stats.bb.rx_9217_to_16383_byte_packets,
1674                 "rx_9217_to_16383_byte_packets");
1675 
1676 	SYSCTL_ADD_QUAD(ctx, children,
1677                 OID_AUTO, "rx_crc_errors",
1678                 CTLFLAG_RD, &ha->hw_stats.common.rx_crc_errors,
1679                 "rx_crc_errors");
1680 
1681 	SYSCTL_ADD_QUAD(ctx, children,
1682                 OID_AUTO, "rx_mac_crtl_frames",
1683                 CTLFLAG_RD, &ha->hw_stats.common.rx_mac_crtl_frames,
1684                 "rx_mac_crtl_frames");
1685 
1686 	SYSCTL_ADD_QUAD(ctx, children,
1687                 OID_AUTO, "rx_pause_frames",
1688                 CTLFLAG_RD, &ha->hw_stats.common.rx_pause_frames,
1689                 "rx_pause_frames");
1690 
1691 	SYSCTL_ADD_QUAD(ctx, children,
1692                 OID_AUTO, "rx_pfc_frames",
1693                 CTLFLAG_RD, &ha->hw_stats.common.rx_pfc_frames,
1694                 "rx_pfc_frames");
1695 
1696 	SYSCTL_ADD_QUAD(ctx, children,
1697                 OID_AUTO, "rx_align_errors",
1698                 CTLFLAG_RD, &ha->hw_stats.common.rx_align_errors,
1699                 "rx_align_errors");
1700 
1701 	SYSCTL_ADD_QUAD(ctx, children,
1702                 OID_AUTO, "rx_carrier_errors",
1703                 CTLFLAG_RD, &ha->hw_stats.common.rx_carrier_errors,
1704                 "rx_carrier_errors");
1705 
1706 	SYSCTL_ADD_QUAD(ctx, children,
1707                 OID_AUTO, "rx_oversize_packets",
1708                 CTLFLAG_RD, &ha->hw_stats.common.rx_oversize_packets,
1709                 "rx_oversize_packets");
1710 
1711 	SYSCTL_ADD_QUAD(ctx, children,
1712                 OID_AUTO, "rx_jabbers",
1713                 CTLFLAG_RD, &ha->hw_stats.common.rx_jabbers,
1714                 "rx_jabbers");
1715 
1716 	SYSCTL_ADD_QUAD(ctx, children,
1717                 OID_AUTO, "rx_undersize_packets",
1718                 CTLFLAG_RD, &ha->hw_stats.common.rx_undersize_packets,
1719                 "rx_undersize_packets");
1720 
1721 	SYSCTL_ADD_QUAD(ctx, children,
1722                 OID_AUTO, "rx_fragments",
1723                 CTLFLAG_RD, &ha->hw_stats.common.rx_fragments,
1724                 "rx_fragments");
1725 
1726 	SYSCTL_ADD_QUAD(ctx, children,
1727                 OID_AUTO, "tx_64_byte_packets",
1728                 CTLFLAG_RD, &ha->hw_stats.common.tx_64_byte_packets,
1729                 "tx_64_byte_packets");
1730 
1731 	SYSCTL_ADD_QUAD(ctx, children,
1732                 OID_AUTO, "tx_65_to_127_byte_packets",
1733                 CTLFLAG_RD, &ha->hw_stats.common.tx_65_to_127_byte_packets,
1734                 "tx_65_to_127_byte_packets");
1735 
1736 	SYSCTL_ADD_QUAD(ctx, children,
1737                 OID_AUTO, "tx_128_to_255_byte_packets",
1738                 CTLFLAG_RD, &ha->hw_stats.common.tx_128_to_255_byte_packets,
1739                 "tx_128_to_255_byte_packets");
1740 
1741 	SYSCTL_ADD_QUAD(ctx, children,
1742                 OID_AUTO, "tx_256_to_511_byte_packets",
1743                 CTLFLAG_RD, &ha->hw_stats.common.tx_256_to_511_byte_packets,
1744                 "tx_256_to_511_byte_packets");
1745 
1746 	SYSCTL_ADD_QUAD(ctx, children,
1747                 OID_AUTO, "tx_512_to_1023_byte_packets",
1748                 CTLFLAG_RD, &ha->hw_stats.common.tx_512_to_1023_byte_packets,
1749                 "tx_512_to_1023_byte_packets");
1750 
1751 	SYSCTL_ADD_QUAD(ctx, children,
1752                 OID_AUTO, "tx_1024_to_1518_byte_packets",
1753                 CTLFLAG_RD, &ha->hw_stats.common.tx_1024_to_1518_byte_packets,
1754                 "tx_1024_to_1518_byte_packets");
1755 
1756 	SYSCTL_ADD_QUAD(ctx, children,
1757                 OID_AUTO, "tx_1519_to_2047_byte_packets",
1758                 CTLFLAG_RD, &ha->hw_stats.bb.tx_1519_to_2047_byte_packets,
1759                 "tx_1519_to_2047_byte_packets");
1760 
1761 	SYSCTL_ADD_QUAD(ctx, children,
1762                 OID_AUTO, "tx_2048_to_4095_byte_packets",
1763                 CTLFLAG_RD, &ha->hw_stats.bb.tx_2048_to_4095_byte_packets,
1764                 "tx_2048_to_4095_byte_packets");
1765 
1766 	SYSCTL_ADD_QUAD(ctx, children,
1767                 OID_AUTO, "tx_4096_to_9216_byte_packets",
1768                 CTLFLAG_RD, &ha->hw_stats.bb.tx_4096_to_9216_byte_packets,
1769                 "tx_4096_to_9216_byte_packets");
1770 
1771 	SYSCTL_ADD_QUAD(ctx, children,
1772                 OID_AUTO, "tx_9217_to_16383_byte_packets",
1773                 CTLFLAG_RD, &ha->hw_stats.bb.tx_9217_to_16383_byte_packets,
1774                 "tx_9217_to_16383_byte_packets");
1775 
1776 	SYSCTL_ADD_QUAD(ctx, children,
1777                 OID_AUTO, "tx_pause_frames",
1778                 CTLFLAG_RD, &ha->hw_stats.common.tx_pause_frames,
1779                 "tx_pause_frames");
1780 
1781 	SYSCTL_ADD_QUAD(ctx, children,
1782                 OID_AUTO, "tx_pfc_frames",
1783                 CTLFLAG_RD, &ha->hw_stats.common.tx_pfc_frames,
1784                 "tx_pfc_frames");
1785 
1786 	SYSCTL_ADD_QUAD(ctx, children,
1787                 OID_AUTO, "tx_lpi_entry_count",
1788                 CTLFLAG_RD, &ha->hw_stats.bb.tx_lpi_entry_count,
1789                 "tx_lpi_entry_count");
1790 
1791 	SYSCTL_ADD_QUAD(ctx, children,
1792                 OID_AUTO, "tx_total_collisions",
1793                 CTLFLAG_RD, &ha->hw_stats.bb.tx_total_collisions,
1794                 "tx_total_collisions");
1795 
1796 	SYSCTL_ADD_QUAD(ctx, children,
1797                 OID_AUTO, "brb_truncates",
1798                 CTLFLAG_RD, &ha->hw_stats.common.brb_truncates,
1799                 "brb_truncates");
1800 
1801 	SYSCTL_ADD_QUAD(ctx, children,
1802                 OID_AUTO, "brb_discards",
1803                 CTLFLAG_RD, &ha->hw_stats.common.brb_discards,
1804                 "brb_discards");
1805 
1806 	SYSCTL_ADD_QUAD(ctx, children,
1807                 OID_AUTO, "rx_mac_bytes",
1808                 CTLFLAG_RD, &ha->hw_stats.common.rx_mac_bytes,
1809                 "rx_mac_bytes");
1810 
1811 	SYSCTL_ADD_QUAD(ctx, children,
1812                 OID_AUTO, "rx_mac_uc_packets",
1813                 CTLFLAG_RD, &ha->hw_stats.common.rx_mac_uc_packets,
1814                 "rx_mac_uc_packets");
1815 
1816 	SYSCTL_ADD_QUAD(ctx, children,
1817                 OID_AUTO, "rx_mac_mc_packets",
1818                 CTLFLAG_RD, &ha->hw_stats.common.rx_mac_mc_packets,
1819                 "rx_mac_mc_packets");
1820 
1821 	SYSCTL_ADD_QUAD(ctx, children,
1822                 OID_AUTO, "rx_mac_bc_packets",
1823                 CTLFLAG_RD, &ha->hw_stats.common.rx_mac_bc_packets,
1824                 "rx_mac_bc_packets");
1825 
1826 	SYSCTL_ADD_QUAD(ctx, children,
1827                 OID_AUTO, "rx_mac_frames_ok",
1828                 CTLFLAG_RD, &ha->hw_stats.common.rx_mac_frames_ok,
1829                 "rx_mac_frames_ok");
1830 
1831 	SYSCTL_ADD_QUAD(ctx, children,
1832                 OID_AUTO, "tx_mac_bytes",
1833                 CTLFLAG_RD, &ha->hw_stats.common.tx_mac_bytes,
1834                 "tx_mac_bytes");
1835 
1836 	SYSCTL_ADD_QUAD(ctx, children,
1837                 OID_AUTO, "tx_mac_uc_packets",
1838                 CTLFLAG_RD, &ha->hw_stats.common.tx_mac_uc_packets,
1839                 "tx_mac_uc_packets");
1840 
1841 	SYSCTL_ADD_QUAD(ctx, children,
1842                 OID_AUTO, "tx_mac_mc_packets",
1843                 CTLFLAG_RD, &ha->hw_stats.common.tx_mac_mc_packets,
1844                 "tx_mac_mc_packets");
1845 
1846 	SYSCTL_ADD_QUAD(ctx, children,
1847                 OID_AUTO, "tx_mac_bc_packets",
1848                 CTLFLAG_RD, &ha->hw_stats.common.tx_mac_bc_packets,
1849                 "tx_mac_bc_packets");
1850 
1851 	SYSCTL_ADD_QUAD(ctx, children,
1852                 OID_AUTO, "tx_mac_ctrl_frames",
1853                 CTLFLAG_RD, &ha->hw_stats.common.tx_mac_ctrl_frames,
1854                 "tx_mac_ctrl_frames");
1855 	return;
1856 }
1857 
1858 static void
1859 qlnx_add_sysctls(qlnx_host_t *ha)
1860 {
1861         device_t		dev = ha->pci_dev;
1862 	struct sysctl_ctx_list	*ctx;
1863 	struct sysctl_oid_list	*children;
1864 
1865 	ctx = device_get_sysctl_ctx(dev);
1866 	children = SYSCTL_CHILDREN(device_get_sysctl_tree(dev));
1867 
1868 	qlnx_add_fp_stats_sysctls(ha);
1869 	qlnx_add_sp_stats_sysctls(ha);
1870 	qlnx_add_hw_stats_sysctls(ha);
1871 
1872 	SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "Driver_Version",
1873 		CTLFLAG_RD, qlnx_ver_str, 0,
1874 		"Driver Version");
1875 
1876 	SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "STORMFW_Version",
1877 		CTLFLAG_RD, ha->stormfw_ver, 0,
1878 		"STORM Firmware Version");
1879 
1880 	SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "MFW_Version",
1881 		CTLFLAG_RD, ha->mfw_ver, 0,
1882 		"Management Firmware Version");
1883 
1884         SYSCTL_ADD_UINT(ctx, children,
1885                 OID_AUTO, "personality", CTLFLAG_RD,
1886                 &ha->personality, ha->personality,
1887 		"\tpersonality = 0 => Ethernet Only\n"
1888 		"\tpersonality = 3 => Ethernet and RoCE\n"
1889 		"\tpersonality = 4 => Ethernet and iWARP\n"
1890 		"\tpersonality = 6 => Default in Shared Memory\n");
1891 
1892         ha->dbg_level = 0;
1893         SYSCTL_ADD_UINT(ctx, children,
1894                 OID_AUTO, "debug", CTLFLAG_RW,
1895                 &ha->dbg_level, ha->dbg_level, "Debug Level");
1896 
1897         ha->dp_level = 0x01;
1898         SYSCTL_ADD_UINT(ctx, children,
1899                 OID_AUTO, "dp_level", CTLFLAG_RW,
1900                 &ha->dp_level, ha->dp_level, "DP Level");
1901 
1902         ha->dbg_trace_lro_cnt = 0;
1903         SYSCTL_ADD_UINT(ctx, children,
1904                 OID_AUTO, "dbg_trace_lro_cnt", CTLFLAG_RW,
1905                 &ha->dbg_trace_lro_cnt, ha->dbg_trace_lro_cnt,
1906 		"Trace LRO Counts");
1907 
1908         ha->dbg_trace_tso_pkt_len = 0;
1909         SYSCTL_ADD_UINT(ctx, children,
1910                 OID_AUTO, "dbg_trace_tso_pkt_len", CTLFLAG_RW,
1911                 &ha->dbg_trace_tso_pkt_len, ha->dbg_trace_tso_pkt_len,
1912 		"Trace TSO packet lengths");
1913 
1914         ha->dp_module = 0;
1915         SYSCTL_ADD_UINT(ctx, children,
1916                 OID_AUTO, "dp_module", CTLFLAG_RW,
1917                 &ha->dp_module, ha->dp_module, "DP Module");
1918 
1919         ha->err_inject = 0;
1920 
1921         SYSCTL_ADD_UINT(ctx, children,
1922                 OID_AUTO, "err_inject", CTLFLAG_RW,
1923                 &ha->err_inject, ha->err_inject, "Error Inject");
1924 
1925         ha->storm_stats_enable = 0;
1926 
1927         SYSCTL_ADD_UINT(ctx, children,
1928                 OID_AUTO, "storm_stats_enable", CTLFLAG_RW,
1929                 &ha->storm_stats_enable, ha->storm_stats_enable,
1930 		"Enable Storm Statistics Gathering");
1931 
1932         ha->storm_stats_index = 0;
1933 
1934         SYSCTL_ADD_UINT(ctx, children,
1935                 OID_AUTO, "storm_stats_index", CTLFLAG_RD,
1936                 &ha->storm_stats_index, ha->storm_stats_index,
1937 		"Enable Storm Statistics Gathering Current Index");
1938 
1939         ha->grcdump_taken = 0;
1940         SYSCTL_ADD_UINT(ctx, children,
1941                 OID_AUTO, "grcdump_taken", CTLFLAG_RD,
1942                 &ha->grcdump_taken, ha->grcdump_taken, "grcdump_taken");
1943 
1944         ha->idle_chk_taken = 0;
1945         SYSCTL_ADD_UINT(ctx, children,
1946                 OID_AUTO, "idle_chk_taken", CTLFLAG_RD,
1947                 &ha->idle_chk_taken, ha->idle_chk_taken, "idle_chk_taken");
1948 
1949         SYSCTL_ADD_UINT(ctx, children,
1950                 OID_AUTO, "rx_coalesce_usecs", CTLFLAG_RD,
1951                 &ha->rx_coalesce_usecs, ha->rx_coalesce_usecs,
1952 		"rx_coalesce_usecs");
1953 
1954         SYSCTL_ADD_UINT(ctx, children,
1955                 OID_AUTO, "tx_coalesce_usecs", CTLFLAG_RD,
1956                 &ha->tx_coalesce_usecs, ha->tx_coalesce_usecs,
1957 		"tx_coalesce_usecs");
1958 
1959 	ha->rx_pkt_threshold = 128;
1960         SYSCTL_ADD_UINT(ctx, children,
1961                 OID_AUTO, "rx_pkt_threshold", CTLFLAG_RW,
1962                 &ha->rx_pkt_threshold, ha->rx_pkt_threshold,
1963 		"No. of Rx Pkts to process at a time");
1964 
1965 	ha->rx_jumbo_buf_eq_mtu = 0;
1966         SYSCTL_ADD_UINT(ctx, children,
1967                 OID_AUTO, "rx_jumbo_buf_eq_mtu", CTLFLAG_RW,
1968                 &ha->rx_jumbo_buf_eq_mtu, ha->rx_jumbo_buf_eq_mtu,
1969 		"== 0 => Rx Jumbo buffers are capped to 4Kbytes\n"
1970 		"otherwise Rx Jumbo buffers are set to >= MTU size\n");
1971 
1972 	SYSCTL_ADD_PROC(ctx, children,
1973 		OID_AUTO, "trigger_dump", CTLTYPE_INT | CTLFLAG_RW,
1974 		(void *)ha, 0,
1975 		qlnx_trigger_dump_sysctl, "I", "trigger_dump");
1976 
1977 	SYSCTL_ADD_PROC(ctx, children,
1978 		OID_AUTO, "set_rx_coalesce_usecs", CTLTYPE_INT | CTLFLAG_RW,
1979 		(void *)ha, 0,
1980 		qlnx_set_rx_coalesce, "I",
1981 		"rx interrupt coalesce period microseconds");
1982 
1983 	SYSCTL_ADD_PROC(ctx, children,
1984 		OID_AUTO, "set_tx_coalesce_usecs", CTLTYPE_INT | CTLFLAG_RW,
1985 		(void *)ha, 0,
1986 		qlnx_set_tx_coalesce, "I",
1987 		"tx interrupt coalesce period microseconds");
1988 
1989 	SYSCTL_ADD_QUAD(ctx, children,
1990                 OID_AUTO, "err_illegal_intr", CTLFLAG_RD,
1991 		&ha->err_illegal_intr, "err_illegal_intr");
1992 
1993 	SYSCTL_ADD_QUAD(ctx, children,
1994                 OID_AUTO, "err_fp_null", CTLFLAG_RD,
1995 		&ha->err_fp_null, "err_fp_null");
1996 
1997 	SYSCTL_ADD_QUAD(ctx, children,
1998                 OID_AUTO, "err_get_proto_invalid_type", CTLFLAG_RD,
1999 		&ha->err_get_proto_invalid_type, "err_get_proto_invalid_type");
2000 	return;
2001 }
2002 
2003 
2004 
2005 /*****************************************************************************
2006  * Operating System Network Interface Functions
2007  *****************************************************************************/
2008 
2009 static void
2010 qlnx_init_ifnet(device_t dev, qlnx_host_t *ha)
2011 {
2012 	uint16_t	device_id;
2013         struct ifnet	*ifp;
2014 
2015         ifp = ha->ifp = if_alloc(IFT_ETHER);
2016 
2017         if (ifp == NULL)
2018                 panic("%s: cannot if_alloc()\n", device_get_nameunit(dev));
2019 
2020         if_initname(ifp, device_get_name(dev), device_get_unit(dev));
2021 
2022 	device_id = pci_get_device(ha->pci_dev);
2023 
2024 #if __FreeBSD_version >= 1000000
2025 
2026         if (device_id == QLOGIC_PCI_DEVICE_ID_1634)
2027 		ifp->if_baudrate = IF_Gbps(40);
2028         else if ((device_id == QLOGIC_PCI_DEVICE_ID_1656) ||
2029 			(device_id == QLOGIC_PCI_DEVICE_ID_8070))
2030 		ifp->if_baudrate = IF_Gbps(25);
2031         else if (device_id == QLOGIC_PCI_DEVICE_ID_1654)
2032 		ifp->if_baudrate = IF_Gbps(50);
2033         else if (device_id == QLOGIC_PCI_DEVICE_ID_1644)
2034 		ifp->if_baudrate = IF_Gbps(100);
2035 
2036         ifp->if_capabilities = IFCAP_LINKSTATE;
2037 #else
2038         ifp->if_mtu = ETHERMTU;
2039 	ifp->if_baudrate = (1 * 1000 * 1000 *1000);
2040 
2041 #endif /* #if __FreeBSD_version >= 1000000 */
2042 
2043         ifp->if_init = qlnx_init;
2044         ifp->if_softc = ha;
2045         ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
2046         ifp->if_ioctl = qlnx_ioctl;
2047         ifp->if_transmit = qlnx_transmit;
2048         ifp->if_qflush = qlnx_qflush;
2049 
2050         IFQ_SET_MAXLEN(&ifp->if_snd, qlnx_get_ifq_snd_maxlen(ha));
2051         ifp->if_snd.ifq_drv_maxlen = qlnx_get_ifq_snd_maxlen(ha);
2052         IFQ_SET_READY(&ifp->if_snd);
2053 
2054 #if __FreeBSD_version >= 1100036
2055 	if_setgetcounterfn(ifp, qlnx_get_counter);
2056 #endif
2057 
2058         ha->max_frame_size = ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN;
2059 
2060         memcpy(ha->primary_mac, qlnx_get_mac_addr(ha), ETH_ALEN);
2061         ether_ifattach(ifp, ha->primary_mac);
2062 	bcopy(IF_LLADDR(ha->ifp), ha->primary_mac, ETHER_ADDR_LEN);
2063 
2064 	ifp->if_capabilities = IFCAP_HWCSUM;
2065 	ifp->if_capabilities |= IFCAP_JUMBO_MTU;
2066 
2067 	ifp->if_capabilities |= IFCAP_VLAN_MTU;
2068 	ifp->if_capabilities |= IFCAP_VLAN_HWTAGGING;
2069 	ifp->if_capabilities |= IFCAP_VLAN_HWFILTER;
2070 	ifp->if_capabilities |= IFCAP_VLAN_HWCSUM;
2071 	ifp->if_capabilities |= IFCAP_VLAN_HWTSO;
2072 	ifp->if_capabilities |= IFCAP_TSO4;
2073 	ifp->if_capabilities |= IFCAP_TSO6;
2074 	ifp->if_capabilities |= IFCAP_LRO;
2075 
2076 	ifp->if_hw_tsomax =  QLNX_MAX_TSO_FRAME_SIZE -
2077 				(ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN);
2078 	ifp->if_hw_tsomaxsegcount = QLNX_MAX_SEGMENTS - 1 /* hdr */;
2079 	ifp->if_hw_tsomaxsegsize = QLNX_MAX_TX_MBUF_SIZE;
2080 
2081 
2082         ifp->if_capenable = ifp->if_capabilities;
2083 
2084 	ifp->if_hwassist = CSUM_IP;
2085 	ifp->if_hwassist |= CSUM_TCP | CSUM_UDP;
2086 	ifp->if_hwassist |= CSUM_TCP_IPV6 | CSUM_UDP_IPV6;
2087 	ifp->if_hwassist |= CSUM_TSO;
2088 
2089 	ifp->if_hdrlen = sizeof(struct ether_vlan_header);
2090 
2091         ifmedia_init(&ha->media, IFM_IMASK, qlnx_media_change,\
2092 		qlnx_media_status);
2093 
2094         if (device_id == QLOGIC_PCI_DEVICE_ID_1634) {
2095 		ifmedia_add(&ha->media, (IFM_ETHER | IFM_40G_LR4), 0, NULL);
2096 		ifmedia_add(&ha->media, (IFM_ETHER | IFM_40G_SR4), 0, NULL);
2097 		ifmedia_add(&ha->media, (IFM_ETHER | IFM_40G_CR4), 0, NULL);
2098         } else if ((device_id == QLOGIC_PCI_DEVICE_ID_1656) ||
2099 			(device_id == QLOGIC_PCI_DEVICE_ID_8070)) {
2100 		ifmedia_add(&ha->media, (IFM_ETHER | QLNX_IFM_25G_SR), 0, NULL);
2101 		ifmedia_add(&ha->media, (IFM_ETHER | QLNX_IFM_25G_CR), 0, NULL);
2102         } else if (device_id == QLOGIC_PCI_DEVICE_ID_1654) {
2103 		ifmedia_add(&ha->media, (IFM_ETHER | IFM_50G_KR2), 0, NULL);
2104 		ifmedia_add(&ha->media, (IFM_ETHER | IFM_50G_CR2), 0, NULL);
2105         } else if (device_id == QLOGIC_PCI_DEVICE_ID_1644) {
2106 		ifmedia_add(&ha->media,
2107 			(IFM_ETHER | QLNX_IFM_100G_LR4), 0, NULL);
2108 		ifmedia_add(&ha->media,
2109 			(IFM_ETHER | QLNX_IFM_100G_SR4), 0, NULL);
2110 		ifmedia_add(&ha->media,
2111 			(IFM_ETHER | QLNX_IFM_100G_CR4), 0, NULL);
2112 	}
2113 
2114         ifmedia_add(&ha->media, (IFM_ETHER | IFM_FDX), 0, NULL);
2115         ifmedia_add(&ha->media, (IFM_ETHER | IFM_AUTO), 0, NULL);
2116 
2117 
2118         ifmedia_set(&ha->media, (IFM_ETHER | IFM_AUTO));
2119 
2120         QL_DPRINT2(ha, "exit\n");
2121 
2122         return;
2123 }
2124 
2125 static void
2126 qlnx_init_locked(qlnx_host_t *ha)
2127 {
2128 	struct ifnet	*ifp = ha->ifp;
2129 
2130 	QL_DPRINT1(ha, "Driver Initialization start \n");
2131 
2132 	qlnx_stop(ha);
2133 
2134 	if (qlnx_load(ha) == 0) {
2135 		ifp->if_drv_flags |= IFF_DRV_RUNNING;
2136 		ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
2137 	}
2138 
2139 	return;
2140 }
2141 
2142 static void
2143 qlnx_init(void *arg)
2144 {
2145 	qlnx_host_t	*ha;
2146 
2147 	ha = (qlnx_host_t *)arg;
2148 
2149 	QL_DPRINT2(ha, "enter\n");
2150 
2151 	QLNX_LOCK(ha);
2152 	qlnx_init_locked(ha);
2153 	QLNX_UNLOCK(ha);
2154 
2155 	QL_DPRINT2(ha, "exit\n");
2156 
2157 	return;
2158 }
2159 
2160 static int
2161 qlnx_config_mcast_mac_addr(qlnx_host_t *ha, uint8_t *mac_addr, uint32_t add_mac)
2162 {
2163 	struct ecore_filter_mcast	*mcast;
2164 	struct ecore_dev		*cdev;
2165 	int				rc;
2166 
2167 	cdev = &ha->cdev;
2168 
2169 	mcast = &ha->ecore_mcast;
2170 	bzero(mcast, sizeof(struct ecore_filter_mcast));
2171 
2172 	if (add_mac)
2173 		mcast->opcode = ECORE_FILTER_ADD;
2174 	else
2175 		mcast->opcode = ECORE_FILTER_REMOVE;
2176 
2177 	mcast->num_mc_addrs = 1;
2178 	memcpy(mcast->mac, mac_addr, ETH_ALEN);
2179 
2180 	rc = ecore_filter_mcast_cmd(cdev, mcast, ECORE_SPQ_MODE_CB, NULL);
2181 
2182 	return (rc);
2183 }
2184 
2185 static int
2186 qlnx_hw_add_mcast(qlnx_host_t *ha, uint8_t *mta)
2187 {
2188         int	i;
2189 
2190         for (i = 0; i < QLNX_MAX_NUM_MULTICAST_ADDRS; i++) {
2191 
2192                 if (QL_MAC_CMP(ha->mcast[i].addr, mta) == 0)
2193                         return 0; /* its been already added */
2194         }
2195 
2196         for (i = 0; i < QLNX_MAX_NUM_MULTICAST_ADDRS; i++) {
2197 
2198                 if ((ha->mcast[i].addr[0] == 0) &&
2199                         (ha->mcast[i].addr[1] == 0) &&
2200                         (ha->mcast[i].addr[2] == 0) &&
2201                         (ha->mcast[i].addr[3] == 0) &&
2202                         (ha->mcast[i].addr[4] == 0) &&
2203                         (ha->mcast[i].addr[5] == 0)) {
2204 
2205                         if (qlnx_config_mcast_mac_addr(ha, mta, 1))
2206                                 return (-1);
2207 
2208                         bcopy(mta, ha->mcast[i].addr, ETH_ALEN);
2209                         ha->nmcast++;
2210 
2211                         return 0;
2212                 }
2213         }
2214         return 0;
2215 }
2216 
2217 static int
2218 qlnx_hw_del_mcast(qlnx_host_t *ha, uint8_t *mta)
2219 {
2220         int	i;
2221 
2222         for (i = 0; i < QLNX_MAX_NUM_MULTICAST_ADDRS; i++) {
2223                 if (QL_MAC_CMP(ha->mcast[i].addr, mta) == 0) {
2224 
2225                         if (qlnx_config_mcast_mac_addr(ha, mta, 0))
2226                                 return (-1);
2227 
2228                         ha->mcast[i].addr[0] = 0;
2229                         ha->mcast[i].addr[1] = 0;
2230                         ha->mcast[i].addr[2] = 0;
2231                         ha->mcast[i].addr[3] = 0;
2232                         ha->mcast[i].addr[4] = 0;
2233                         ha->mcast[i].addr[5] = 0;
2234 
2235                         ha->nmcast--;
2236 
2237                         return 0;
2238                 }
2239         }
2240         return 0;
2241 }
2242 
2243 /*
2244  * Name: qls_hw_set_multi
2245  * Function: Sets the Multicast Addresses provided the host O.S into the
2246  *      hardware (for the given interface)
2247  */
2248 static void
2249 qlnx_hw_set_multi(qlnx_host_t *ha, uint8_t *mta, uint32_t mcnt,
2250 	uint32_t add_mac)
2251 {
2252         int	i;
2253 
2254         for (i = 0; i < mcnt; i++) {
2255                 if (add_mac) {
2256                         if (qlnx_hw_add_mcast(ha, mta))
2257                                 break;
2258                 } else {
2259                         if (qlnx_hw_del_mcast(ha, mta))
2260                                 break;
2261                 }
2262 
2263                 mta += ETHER_HDR_LEN;
2264         }
2265         return;
2266 }
2267 
2268 
2269 #define QLNX_MCAST_ADDRS_SIZE (QLNX_MAX_NUM_MULTICAST_ADDRS * ETHER_HDR_LEN)
2270 static int
2271 qlnx_set_multi(qlnx_host_t *ha, uint32_t add_multi)
2272 {
2273 	uint8_t			mta[QLNX_MCAST_ADDRS_SIZE];
2274 	struct ifmultiaddr	*ifma;
2275 	int			mcnt = 0;
2276 	struct ifnet		*ifp = ha->ifp;
2277 	int			ret = 0;
2278 
2279 	if_maddr_rlock(ifp);
2280 
2281 	TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
2282 
2283 		if (ifma->ifma_addr->sa_family != AF_LINK)
2284 			continue;
2285 
2286 		if (mcnt == QLNX_MAX_NUM_MULTICAST_ADDRS)
2287 			break;
2288 
2289 		bcopy(LLADDR((struct sockaddr_dl *) ifma->ifma_addr),
2290 			&mta[mcnt * ETHER_HDR_LEN], ETHER_HDR_LEN);
2291 
2292 		mcnt++;
2293 	}
2294 
2295 	if_maddr_runlock(ifp);
2296 
2297 	QLNX_LOCK(ha);
2298 	qlnx_hw_set_multi(ha, mta, mcnt, add_multi);
2299 	QLNX_UNLOCK(ha);
2300 
2301 	return (ret);
2302 }
2303 
2304 static int
2305 qlnx_set_promisc(qlnx_host_t *ha)
2306 {
2307 	int	rc = 0;
2308 	uint8_t	filter;
2309 
2310 	filter = ha->filter;
2311 	filter |= ECORE_ACCEPT_MCAST_UNMATCHED;
2312 	filter |= ECORE_ACCEPT_UCAST_UNMATCHED;
2313 
2314 	rc = qlnx_set_rx_accept_filter(ha, filter);
2315 	return (rc);
2316 }
2317 
2318 static int
2319 qlnx_set_allmulti(qlnx_host_t *ha)
2320 {
2321 	int	rc = 0;
2322 	uint8_t	filter;
2323 
2324 	filter = ha->filter;
2325 	filter |= ECORE_ACCEPT_MCAST_UNMATCHED;
2326 	rc = qlnx_set_rx_accept_filter(ha, filter);
2327 
2328 	return (rc);
2329 }
2330 
2331 
2332 static int
2333 qlnx_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data)
2334 {
2335 	int		ret = 0, mask;
2336 	struct ifreq	*ifr = (struct ifreq *)data;
2337 	struct ifaddr	*ifa = (struct ifaddr *)data;
2338 	qlnx_host_t	*ha;
2339 
2340 	ha = (qlnx_host_t *)ifp->if_softc;
2341 
2342 	switch (cmd) {
2343 	case SIOCSIFADDR:
2344 		QL_DPRINT4(ha, "SIOCSIFADDR (0x%lx)\n", cmd);
2345 
2346 		if (ifa->ifa_addr->sa_family == AF_INET) {
2347 			ifp->if_flags |= IFF_UP;
2348 			if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) {
2349 				QLNX_LOCK(ha);
2350 				qlnx_init_locked(ha);
2351 				QLNX_UNLOCK(ha);
2352 			}
2353 			QL_DPRINT4(ha, "SIOCSIFADDR (0x%lx) ipv4 [0x%08x]\n",
2354 				   cmd, ntohl(IA_SIN(ifa)->sin_addr.s_addr));
2355 
2356 			arp_ifinit(ifp, ifa);
2357 		} else {
2358 			ether_ioctl(ifp, cmd, data);
2359 		}
2360 		break;
2361 
2362 	case SIOCSIFMTU:
2363 		QL_DPRINT4(ha, "SIOCSIFMTU (0x%lx)\n", cmd);
2364 
2365 		if (ifr->ifr_mtu > QLNX_MAX_MTU) {
2366 			ret = EINVAL;
2367 		} else {
2368 			QLNX_LOCK(ha);
2369 			ifp->if_mtu = ifr->ifr_mtu;
2370 			ha->max_frame_size =
2371 				ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN;
2372 			if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
2373 				qlnx_init_locked(ha);
2374 			}
2375 
2376 			QLNX_UNLOCK(ha);
2377 		}
2378 
2379 		break;
2380 
2381 	case SIOCSIFFLAGS:
2382 		QL_DPRINT4(ha, "SIOCSIFFLAGS (0x%lx)\n", cmd);
2383 
2384 		QLNX_LOCK(ha);
2385 
2386 		if (ifp->if_flags & IFF_UP) {
2387 			if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
2388 				if ((ifp->if_flags ^ ha->if_flags) &
2389 					IFF_PROMISC) {
2390 					ret = qlnx_set_promisc(ha);
2391 				} else if ((ifp->if_flags ^ ha->if_flags) &
2392 					IFF_ALLMULTI) {
2393 					ret = qlnx_set_allmulti(ha);
2394 				}
2395 			} else {
2396 				ha->max_frame_size = ifp->if_mtu +
2397 					ETHER_HDR_LEN + ETHER_CRC_LEN;
2398 				qlnx_init_locked(ha);
2399 			}
2400 		} else {
2401 			if (ifp->if_drv_flags & IFF_DRV_RUNNING)
2402 				qlnx_stop(ha);
2403 			ha->if_flags = ifp->if_flags;
2404 		}
2405 
2406 		QLNX_UNLOCK(ha);
2407 		break;
2408 
2409 	case SIOCADDMULTI:
2410 		QL_DPRINT4(ha, "%s (0x%lx)\n", "SIOCADDMULTI", cmd);
2411 
2412 		if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
2413 			if (qlnx_set_multi(ha, 1))
2414 				ret = EINVAL;
2415 		}
2416 		break;
2417 
2418 	case SIOCDELMULTI:
2419 		QL_DPRINT4(ha, "%s (0x%lx)\n", "SIOCDELMULTI", cmd);
2420 
2421 		if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
2422 			if (qlnx_set_multi(ha, 0))
2423 				ret = EINVAL;
2424 		}
2425 		break;
2426 
2427 	case SIOCSIFMEDIA:
2428 	case SIOCGIFMEDIA:
2429 		QL_DPRINT4(ha, "SIOCSIFMEDIA/SIOCGIFMEDIA (0x%lx)\n", cmd);
2430 
2431 		ret = ifmedia_ioctl(ifp, ifr, &ha->media, cmd);
2432 		break;
2433 
2434 	case SIOCSIFCAP:
2435 
2436 		mask = ifr->ifr_reqcap ^ ifp->if_capenable;
2437 
2438 		QL_DPRINT4(ha, "SIOCSIFCAP (0x%lx)\n", cmd);
2439 
2440 		if (mask & IFCAP_HWCSUM)
2441 			ifp->if_capenable ^= IFCAP_HWCSUM;
2442 		if (mask & IFCAP_TSO4)
2443 			ifp->if_capenable ^= IFCAP_TSO4;
2444 		if (mask & IFCAP_TSO6)
2445 			ifp->if_capenable ^= IFCAP_TSO6;
2446 		if (mask & IFCAP_VLAN_HWTAGGING)
2447 			ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
2448 		if (mask & IFCAP_VLAN_HWTSO)
2449 			ifp->if_capenable ^= IFCAP_VLAN_HWTSO;
2450 		if (mask & IFCAP_LRO)
2451 			ifp->if_capenable ^= IFCAP_LRO;
2452 
2453 		if (!(ifp->if_drv_flags & IFF_DRV_RUNNING))
2454 			qlnx_init(ha);
2455 
2456 		VLAN_CAPABILITIES(ifp);
2457 		break;
2458 
2459 #if (__FreeBSD_version >= 1100101)
2460 
2461 	case SIOCGI2C:
2462 	{
2463 		struct ifi2creq i2c;
2464 		struct ecore_hwfn *p_hwfn = &ha->cdev.hwfns[0];
2465 		struct ecore_ptt *p_ptt;
2466 
2467 		ret = copyin(ifr_data_get_ptr(ifr), &i2c, sizeof(i2c));
2468 
2469 		if (ret)
2470 			break;
2471 
2472 		if ((i2c.len > sizeof (i2c.data)) ||
2473 			(i2c.dev_addr != 0xA0 && i2c.dev_addr != 0xA2)) {
2474 			ret = EINVAL;
2475 			break;
2476 		}
2477 
2478 		p_ptt = ecore_ptt_acquire(p_hwfn);
2479 
2480 		if (!p_ptt) {
2481 			QL_DPRINT1(ha, "ecore_ptt_acquire failed\n");
2482 			ret = -1;
2483 			break;
2484 		}
2485 
2486 		ret = ecore_mcp_phy_sfp_read(p_hwfn, p_ptt,
2487 			(ha->pci_func & 0x1), i2c.dev_addr, i2c.offset,
2488 			i2c.len, &i2c.data[0]);
2489 
2490 		ecore_ptt_release(p_hwfn, p_ptt);
2491 
2492 		if (ret) {
2493 			ret = -1;
2494 			break;
2495 		}
2496 
2497 		ret = copyout(&i2c, ifr_data_get_ptr(ifr), sizeof(i2c));
2498 
2499 		QL_DPRINT8(ha, "SIOCGI2C copyout ret = %d \
2500 			 len = %d addr = 0x%02x offset = 0x%04x \
2501 			 data[0..7]=0x%02x 0x%02x 0x%02x 0x%02x 0x%02x \
2502 			 0x%02x 0x%02x 0x%02x\n",
2503 			ret, i2c.len, i2c.dev_addr, i2c.offset,
2504 			i2c.data[0], i2c.data[1], i2c.data[2], i2c.data[3],
2505 			i2c.data[4], i2c.data[5], i2c.data[6], i2c.data[7]);
2506 		break;
2507 	}
2508 #endif /* #if (__FreeBSD_version >= 1100101) */
2509 
2510 	default:
2511 		QL_DPRINT4(ha, "default (0x%lx)\n", cmd);
2512 		ret = ether_ioctl(ifp, cmd, data);
2513 		break;
2514 	}
2515 
2516 	return (ret);
2517 }
2518 
2519 static int
2520 qlnx_media_change(struct ifnet *ifp)
2521 {
2522 	qlnx_host_t	*ha;
2523 	struct ifmedia	*ifm;
2524 	int		ret = 0;
2525 
2526 	ha = (qlnx_host_t *)ifp->if_softc;
2527 
2528 	QL_DPRINT2(ha, "enter\n");
2529 
2530 	ifm = &ha->media;
2531 
2532 	if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER)
2533 		ret = EINVAL;
2534 
2535 	QL_DPRINT2(ha, "exit\n");
2536 
2537 	return (ret);
2538 }
2539 
2540 static void
2541 qlnx_media_status(struct ifnet *ifp, struct ifmediareq *ifmr)
2542 {
2543 	qlnx_host_t		*ha;
2544 
2545 	ha = (qlnx_host_t *)ifp->if_softc;
2546 
2547 	QL_DPRINT2(ha, "enter\n");
2548 
2549 	ifmr->ifm_status = IFM_AVALID;
2550 	ifmr->ifm_active = IFM_ETHER;
2551 
2552 	if (ha->link_up) {
2553 		ifmr->ifm_status |= IFM_ACTIVE;
2554 		ifmr->ifm_active |=
2555 			(IFM_FDX | qlnx_get_optics(ha, &ha->if_link));
2556 
2557 		if (ha->if_link.link_partner_caps &
2558 			(QLNX_LINK_CAP_Pause | QLNX_LINK_CAP_Asym_Pause))
2559 			ifmr->ifm_active |=
2560 				(IFM_ETH_RXPAUSE | IFM_ETH_TXPAUSE);
2561 	}
2562 
2563 	QL_DPRINT2(ha, "exit (%s)\n", (ha->link_up ? "link_up" : "link_down"));
2564 
2565 	return;
2566 }
2567 
2568 
2569 static void
2570 qlnx_free_tx_pkt(qlnx_host_t *ha, struct qlnx_fastpath *fp,
2571 	struct qlnx_tx_queue *txq)
2572 {
2573 	u16			idx;
2574 	struct mbuf		*mp;
2575 	bus_dmamap_t		map;
2576 	int			i;
2577 	struct eth_tx_bd	*tx_data_bd;
2578 	struct eth_tx_1st_bd	*first_bd;
2579 	int			nbds = 0;
2580 
2581 	idx = txq->sw_tx_cons;
2582 	mp = txq->sw_tx_ring[idx].mp;
2583 	map = txq->sw_tx_ring[idx].map;
2584 
2585 	if ((mp == NULL) || QL_ERR_INJECT(ha, QL_ERR_INJCT_TX_INT_MBUF_NULL)){
2586 
2587 		QL_RESET_ERR_INJECT(ha, QL_ERR_INJCT_TX_INT_MBUF_NULL);
2588 
2589 		QL_DPRINT1(ha, "(mp == NULL) "
2590 			" tx_idx = 0x%x"
2591 			" ecore_prod_idx = 0x%x"
2592 			" ecore_cons_idx = 0x%x"
2593 			" hw_bd_cons = 0x%x"
2594 			" txq_db_last = 0x%x"
2595 			" elem_left = 0x%x\n",
2596 			fp->rss_id,
2597 			ecore_chain_get_prod_idx(&txq->tx_pbl),
2598 			ecore_chain_get_cons_idx(&txq->tx_pbl),
2599 			le16toh(*txq->hw_cons_ptr),
2600 			txq->tx_db.raw,
2601 			ecore_chain_get_elem_left(&txq->tx_pbl));
2602 
2603 		fp->err_tx_free_pkt_null++;
2604 
2605 		//DEBUG
2606 		qlnx_trigger_dump(ha);
2607 
2608 		return;
2609 	} else {
2610 
2611 		QLNX_INC_OPACKETS((ha->ifp));
2612 		QLNX_INC_OBYTES((ha->ifp), (mp->m_pkthdr.len));
2613 
2614 		bus_dmamap_sync(ha->tx_tag, map, BUS_DMASYNC_POSTWRITE);
2615 		bus_dmamap_unload(ha->tx_tag, map);
2616 
2617 		fp->tx_pkts_freed++;
2618 		fp->tx_pkts_completed++;
2619 
2620 		m_freem(mp);
2621 	}
2622 
2623 	first_bd = (struct eth_tx_1st_bd *)ecore_chain_consume(&txq->tx_pbl);
2624 	nbds = first_bd->data.nbds;
2625 
2626 //	BD_SET_UNMAP_ADDR_LEN(first_bd, 0, 0);
2627 
2628 	for (i = 1; i < nbds; i++) {
2629 		tx_data_bd = ecore_chain_consume(&txq->tx_pbl);
2630 //		BD_SET_UNMAP_ADDR_LEN(tx_data_bd, 0, 0);
2631 	}
2632 	txq->sw_tx_ring[idx].flags = 0;
2633 	txq->sw_tx_ring[idx].mp = NULL;
2634 	txq->sw_tx_ring[idx].map = (bus_dmamap_t)0;
2635 
2636 	return;
2637 }
2638 
2639 static void
2640 qlnx_tx_int(qlnx_host_t *ha, struct qlnx_fastpath *fp,
2641 	struct qlnx_tx_queue *txq)
2642 {
2643 	u16 hw_bd_cons;
2644 	u16 ecore_cons_idx;
2645 	uint16_t diff;
2646 	uint16_t idx, idx2;
2647 
2648 	hw_bd_cons = le16toh(*txq->hw_cons_ptr);
2649 
2650 	while (hw_bd_cons !=
2651 		(ecore_cons_idx = ecore_chain_get_cons_idx(&txq->tx_pbl))) {
2652 
2653 		if (hw_bd_cons < ecore_cons_idx) {
2654 			diff = (1 << 16) - (ecore_cons_idx - hw_bd_cons);
2655 		} else {
2656 			diff = hw_bd_cons - ecore_cons_idx;
2657 		}
2658 		if ((diff > TX_RING_SIZE) ||
2659 			QL_ERR_INJECT(ha, QL_ERR_INJCT_TX_INT_DIFF)){
2660 
2661 			QL_RESET_ERR_INJECT(ha, QL_ERR_INJCT_TX_INT_DIFF);
2662 
2663 			QL_DPRINT1(ha, "(diff = 0x%x) "
2664 				" tx_idx = 0x%x"
2665 				" ecore_prod_idx = 0x%x"
2666 				" ecore_cons_idx = 0x%x"
2667 				" hw_bd_cons = 0x%x"
2668 				" txq_db_last = 0x%x"
2669 				" elem_left = 0x%x\n",
2670 				diff,
2671 				fp->rss_id,
2672 				ecore_chain_get_prod_idx(&txq->tx_pbl),
2673 				ecore_chain_get_cons_idx(&txq->tx_pbl),
2674 				le16toh(*txq->hw_cons_ptr),
2675 				txq->tx_db.raw,
2676 				ecore_chain_get_elem_left(&txq->tx_pbl));
2677 
2678 			fp->err_tx_cons_idx_conflict++;
2679 
2680 			//DEBUG
2681 			qlnx_trigger_dump(ha);
2682 		}
2683 
2684 		idx = (txq->sw_tx_cons + 1) & (TX_RING_SIZE - 1);
2685 		idx2 = (txq->sw_tx_cons + 2) & (TX_RING_SIZE - 1);
2686 		prefetch(txq->sw_tx_ring[idx].mp);
2687 		prefetch(txq->sw_tx_ring[idx2].mp);
2688 
2689 		qlnx_free_tx_pkt(ha, fp, txq);
2690 
2691 		txq->sw_tx_cons = (txq->sw_tx_cons + 1) & (TX_RING_SIZE - 1);
2692 	}
2693 	return;
2694 }
2695 
2696 static int
2697 qlnx_transmit_locked(struct ifnet *ifp,struct qlnx_fastpath  *fp, struct mbuf  *mp)
2698 {
2699         int                     ret = 0;
2700         struct qlnx_tx_queue    *txq;
2701         qlnx_host_t *           ha;
2702         uint16_t elem_left;
2703 
2704         txq = fp->txq[0];
2705         ha = (qlnx_host_t *)fp->edev;
2706 
2707 
2708         if ((!(ifp->if_drv_flags & IFF_DRV_RUNNING)) || (!ha->link_up)) {
2709                 if(mp != NULL)
2710                         ret = drbr_enqueue(ifp, fp->tx_br, mp);
2711                 return (ret);
2712         }
2713 
2714         if(mp != NULL)
2715                 ret  = drbr_enqueue(ifp, fp->tx_br, mp);
2716 
2717         mp = drbr_peek(ifp, fp->tx_br);
2718 
2719         while (mp != NULL) {
2720 
2721                 if (qlnx_send(ha, fp, &mp)) {
2722 
2723                         if (mp != NULL) {
2724                                 drbr_putback(ifp, fp->tx_br, mp);
2725                         } else {
2726                                 fp->tx_pkts_processed++;
2727                                 drbr_advance(ifp, fp->tx_br);
2728                         }
2729                         goto qlnx_transmit_locked_exit;
2730 
2731                 } else {
2732                         drbr_advance(ifp, fp->tx_br);
2733                         fp->tx_pkts_transmitted++;
2734                         fp->tx_pkts_processed++;
2735                 }
2736 
2737                 mp = drbr_peek(ifp, fp->tx_br);
2738         }
2739 
2740 qlnx_transmit_locked_exit:
2741         if((qlnx_num_tx_compl(ha,fp, fp->txq[0]) > QLNX_TX_COMPL_THRESH) ||
2742                 ((int)(elem_left = ecore_chain_get_elem_left(&txq->tx_pbl))
2743                                         < QLNX_TX_ELEM_MAX_THRESH))
2744                 (void)qlnx_tx_int(ha, fp, fp->txq[0]);
2745 
2746         QL_DPRINT2(ha, "%s: exit ret = %d\n", __func__, ret);
2747         return ret;
2748 }
2749 
2750 
2751 static int
2752 qlnx_transmit(struct ifnet *ifp, struct mbuf  *mp)
2753 {
2754         qlnx_host_t		*ha = (qlnx_host_t *)ifp->if_softc;
2755         struct qlnx_fastpath	*fp;
2756         int			rss_id = 0, ret = 0;
2757 
2758 #ifdef QLNX_TRACEPERF_DATA
2759         uint64_t tx_pkts = 0, tx_compl = 0;
2760 #endif
2761 
2762         QL_DPRINT2(ha, "enter\n");
2763 
2764 #if __FreeBSD_version >= 1100000
2765         if (M_HASHTYPE_GET(mp) != M_HASHTYPE_NONE)
2766 #else
2767         if (mp->m_flags & M_FLOWID)
2768 #endif
2769                 rss_id = (mp->m_pkthdr.flowid % ECORE_RSS_IND_TABLE_SIZE) %
2770 					ha->num_rss;
2771 
2772         fp = &ha->fp_array[rss_id];
2773 
2774         if (fp->tx_br == NULL) {
2775                 ret = EINVAL;
2776                 goto qlnx_transmit_exit;
2777         }
2778 
2779         if (mtx_trylock(&fp->tx_mtx)) {
2780 
2781 #ifdef QLNX_TRACEPERF_DATA
2782                         tx_pkts = fp->tx_pkts_transmitted;
2783                         tx_compl = fp->tx_pkts_completed;
2784 #endif
2785 
2786                         ret = qlnx_transmit_locked(ifp, fp, mp);
2787 
2788 #ifdef QLNX_TRACEPERF_DATA
2789                         fp->tx_pkts_trans_ctx += (fp->tx_pkts_transmitted - tx_pkts);
2790                         fp->tx_pkts_compl_ctx += (fp->tx_pkts_completed - tx_compl);
2791 #endif
2792                         mtx_unlock(&fp->tx_mtx);
2793         } else {
2794                 if (mp != NULL && (fp->fp_taskqueue != NULL)) {
2795                         ret = drbr_enqueue(ifp, fp->tx_br, mp);
2796                         taskqueue_enqueue(fp->fp_taskqueue, &fp->fp_task);
2797                 }
2798         }
2799 
2800 qlnx_transmit_exit:
2801 
2802         QL_DPRINT2(ha, "exit ret = %d\n", ret);
2803         return ret;
2804 }
2805 
2806 static void
2807 qlnx_qflush(struct ifnet *ifp)
2808 {
2809 	int			rss_id;
2810 	struct qlnx_fastpath	*fp;
2811 	struct mbuf		*mp;
2812 	qlnx_host_t		*ha;
2813 
2814 	ha = (qlnx_host_t *)ifp->if_softc;
2815 
2816 	QL_DPRINT2(ha, "enter\n");
2817 
2818 	for (rss_id = 0; rss_id < ha->num_rss; rss_id++) {
2819 
2820 		fp = &ha->fp_array[rss_id];
2821 
2822 		if (fp == NULL)
2823 			continue;
2824 
2825 		if (fp->tx_br) {
2826 			mtx_lock(&fp->tx_mtx);
2827 
2828 			while ((mp = drbr_dequeue(ifp, fp->tx_br)) != NULL) {
2829 				fp->tx_pkts_freed++;
2830 				m_freem(mp);
2831 			}
2832 			mtx_unlock(&fp->tx_mtx);
2833 		}
2834 	}
2835 	QL_DPRINT2(ha, "exit\n");
2836 
2837 	return;
2838 }
2839 
2840 static void
2841 qlnx_txq_doorbell_wr32(qlnx_host_t *ha, void *reg_addr, uint32_t value)
2842 {
2843 	struct ecore_dev	*cdev;
2844 	uint32_t		offset;
2845 
2846 	cdev = &ha->cdev;
2847 
2848 	offset = (uint32_t)((uint8_t *)reg_addr - (uint8_t *)cdev->doorbells);
2849 
2850 	bus_write_4(ha->pci_dbells, offset, value);
2851 	bus_barrier(ha->pci_reg,  0, 0, BUS_SPACE_BARRIER_READ);
2852 	bus_barrier(ha->pci_dbells,  0, 0, BUS_SPACE_BARRIER_READ);
2853 
2854 	return;
2855 }
2856 
2857 static uint32_t
2858 qlnx_tcp_offset(qlnx_host_t *ha, struct mbuf *mp)
2859 {
2860         struct ether_vlan_header	*eh = NULL;
2861         struct ip			*ip = NULL;
2862         struct ip6_hdr			*ip6 = NULL;
2863         struct tcphdr			*th = NULL;
2864         uint32_t			ehdrlen = 0, ip_hlen = 0, offset = 0;
2865         uint16_t			etype = 0;
2866         device_t			dev;
2867         uint8_t				buf[sizeof(struct ip6_hdr)];
2868 
2869         dev = ha->pci_dev;
2870 
2871         eh = mtod(mp, struct ether_vlan_header *);
2872 
2873         if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
2874                 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
2875                 etype = ntohs(eh->evl_proto);
2876         } else {
2877                 ehdrlen = ETHER_HDR_LEN;
2878                 etype = ntohs(eh->evl_encap_proto);
2879         }
2880 
2881         switch (etype) {
2882 
2883                 case ETHERTYPE_IP:
2884                         ip = (struct ip *)(mp->m_data + ehdrlen);
2885 
2886                         ip_hlen = sizeof (struct ip);
2887 
2888                         if (mp->m_len < (ehdrlen + ip_hlen)) {
2889                                 m_copydata(mp, ehdrlen, sizeof(struct ip), buf);
2890                                 ip = (struct ip *)buf;
2891                         }
2892 
2893                         th = (struct tcphdr *)(ip + 1);
2894 			offset = ip_hlen + ehdrlen + (th->th_off << 2);
2895                 break;
2896 
2897                 case ETHERTYPE_IPV6:
2898                         ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen);
2899 
2900                         ip_hlen = sizeof(struct ip6_hdr);
2901 
2902                         if (mp->m_len < (ehdrlen + ip_hlen)) {
2903                                 m_copydata(mp, ehdrlen, sizeof (struct ip6_hdr),
2904                                         buf);
2905                                 ip6 = (struct ip6_hdr *)buf;
2906                         }
2907                         th = (struct tcphdr *)(ip6 + 1);
2908 			offset = ip_hlen + ehdrlen + (th->th_off << 2);
2909                 break;
2910 
2911                 default:
2912                 break;
2913         }
2914 
2915         return (offset);
2916 }
2917 
2918 static __inline int
2919 qlnx_tso_check(struct qlnx_fastpath *fp, bus_dma_segment_t *segs, int nsegs,
2920 	uint32_t offset)
2921 {
2922 	int			i;
2923 	uint32_t		sum, nbds_in_hdr = 1;
2924 	uint32_t		window;
2925         bus_dma_segment_t	*s_seg;
2926 
2927 	/* If the header spans mulitple segments, skip those segments */
2928 
2929 	if (nsegs < ETH_TX_LSO_WINDOW_BDS_NUM)
2930 		return (0);
2931 
2932 	i = 0;
2933 
2934 	while ((i < nsegs) && (offset >= segs->ds_len)) {
2935 		offset = offset - segs->ds_len;
2936 		segs++;
2937 		i++;
2938 		nbds_in_hdr++;
2939 	}
2940 
2941 	window = ETH_TX_LSO_WINDOW_BDS_NUM - nbds_in_hdr;
2942 
2943 	nsegs = nsegs - i;
2944 
2945 	while (nsegs >= window) {
2946 
2947 		sum = 0;
2948 		s_seg = segs;
2949 
2950 		for (i = 0; i < window; i++){
2951 			sum += s_seg->ds_len;
2952 			s_seg++;
2953 		}
2954 
2955 		if (sum < ETH_TX_LSO_WINDOW_MIN_LEN) {
2956 			fp->tx_lso_wnd_min_len++;
2957 			return (-1);
2958 		}
2959 
2960 		nsegs = nsegs - 1;
2961 		segs++;
2962 	}
2963 
2964 	return (0);
2965 }
2966 
2967 static int
2968 qlnx_send(qlnx_host_t *ha, struct qlnx_fastpath *fp, struct mbuf **m_headp)
2969 {
2970 	bus_dma_segment_t	*segs;
2971 	bus_dmamap_t		map = 0;
2972 	uint32_t		nsegs = 0;
2973 	int			ret = -1;
2974 	struct mbuf		*m_head = *m_headp;
2975 	uint16_t		idx = 0;
2976 	uint16_t		elem_left;
2977 
2978 	uint8_t			nbd = 0;
2979 	struct qlnx_tx_queue    *txq;
2980 
2981 	struct eth_tx_1st_bd    *first_bd;
2982 	struct eth_tx_2nd_bd    *second_bd;
2983 	struct eth_tx_3rd_bd    *third_bd;
2984 	struct eth_tx_bd        *tx_data_bd;
2985 
2986 	int			seg_idx = 0;
2987 	uint32_t		nbds_in_hdr = 0;
2988 	uint32_t		offset = 0;
2989 
2990 #ifdef QLNX_TRACE_PERF_DATA
2991         uint16_t                bd_used;
2992 #endif
2993 
2994 	QL_DPRINT8(ha, "enter\n");
2995 
2996 	if (!ha->link_up)
2997 		return (-1);
2998 
2999 	first_bd	= NULL;
3000 	second_bd	= NULL;
3001 	third_bd	= NULL;
3002 	tx_data_bd	= NULL;
3003 
3004 	txq = fp->txq[0];
3005 
3006         if ((int)(elem_left = ecore_chain_get_elem_left(&txq->tx_pbl)) <
3007 		QLNX_TX_ELEM_MIN_THRESH) {
3008 
3009                 fp->tx_nsegs_gt_elem_left++;
3010                 fp->err_tx_nsegs_gt_elem_left++;
3011 
3012                 return (ENOBUFS);
3013         }
3014 
3015 	idx = txq->sw_tx_prod;
3016 
3017 	map = txq->sw_tx_ring[idx].map;
3018 	segs = txq->segs;
3019 
3020 	ret = bus_dmamap_load_mbuf_sg(ha->tx_tag, map, m_head, segs, &nsegs,
3021 			BUS_DMA_NOWAIT);
3022 
3023 	if (ha->dbg_trace_tso_pkt_len) {
3024 		if (m_head->m_pkthdr.csum_flags & CSUM_TSO) {
3025 			if (!fp->tx_tso_min_pkt_len) {
3026 				fp->tx_tso_min_pkt_len = m_head->m_pkthdr.len;
3027 				fp->tx_tso_min_pkt_len = m_head->m_pkthdr.len;
3028 			} else {
3029 				if (fp->tx_tso_min_pkt_len > m_head->m_pkthdr.len)
3030 					fp->tx_tso_min_pkt_len =
3031 						m_head->m_pkthdr.len;
3032 				if (fp->tx_tso_max_pkt_len < m_head->m_pkthdr.len)
3033 					fp->tx_tso_max_pkt_len =
3034 						m_head->m_pkthdr.len;
3035 			}
3036 		}
3037 	}
3038 
3039 	if (m_head->m_pkthdr.csum_flags & CSUM_TSO)
3040 		offset = qlnx_tcp_offset(ha, m_head);
3041 
3042 	if ((ret == EFBIG) ||
3043 		((nsegs > QLNX_MAX_SEGMENTS_NON_TSO) && (
3044 			(!(m_head->m_pkthdr.csum_flags & CSUM_TSO)) ||
3045 		((m_head->m_pkthdr.csum_flags & CSUM_TSO) &&
3046 			qlnx_tso_check(fp, segs, nsegs, offset))))) {
3047 
3048 		struct mbuf *m;
3049 
3050 		QL_DPRINT8(ha, "EFBIG [%d]\n", m_head->m_pkthdr.len);
3051 
3052 		fp->tx_defrag++;
3053 
3054 		m = m_defrag(m_head, M_NOWAIT);
3055 		if (m == NULL) {
3056 			fp->err_tx_defrag++;
3057 			fp->tx_pkts_freed++;
3058 			m_freem(m_head);
3059 			*m_headp = NULL;
3060 			QL_DPRINT1(ha, "m_defrag() = NULL [%d]\n", ret);
3061 			return (ENOBUFS);
3062 		}
3063 
3064 		m_head = m;
3065 		*m_headp = m_head;
3066 
3067 		if ((ret = bus_dmamap_load_mbuf_sg(ha->tx_tag, map, m_head,
3068 				segs, &nsegs, BUS_DMA_NOWAIT))) {
3069 
3070 			fp->err_tx_defrag_dmamap_load++;
3071 
3072 			QL_DPRINT1(ha,
3073 				"bus_dmamap_load_mbuf_sg failed0 [%d, %d]\n",
3074 				ret, m_head->m_pkthdr.len);
3075 
3076 			fp->tx_pkts_freed++;
3077 			m_freem(m_head);
3078 			*m_headp = NULL;
3079 
3080 			return (ret);
3081 		}
3082 
3083 		if ((nsegs > QLNX_MAX_SEGMENTS_NON_TSO) &&
3084 			!(m_head->m_pkthdr.csum_flags & CSUM_TSO)) {
3085 
3086 			fp->err_tx_non_tso_max_seg++;
3087 
3088 			QL_DPRINT1(ha,
3089 				"(%d) nsegs too many for non-TSO [%d, %d]\n",
3090 				ret, nsegs, m_head->m_pkthdr.len);
3091 
3092 			fp->tx_pkts_freed++;
3093 			m_freem(m_head);
3094 			*m_headp = NULL;
3095 
3096 			return (ret);
3097 		}
3098 		if (m_head->m_pkthdr.csum_flags & CSUM_TSO)
3099 			offset = qlnx_tcp_offset(ha, m_head);
3100 
3101 	} else if (ret) {
3102 
3103 		fp->err_tx_dmamap_load++;
3104 
3105 		QL_DPRINT1(ha, "bus_dmamap_load_mbuf_sg failed1 [%d, %d]\n",
3106 			   ret, m_head->m_pkthdr.len);
3107 		fp->tx_pkts_freed++;
3108 		m_freem(m_head);
3109 		*m_headp = NULL;
3110 		return (ret);
3111 	}
3112 
3113 	QL_ASSERT(ha, (nsegs != 0), ("qlnx_send: empty packet"));
3114 
3115 	if (ha->dbg_trace_tso_pkt_len) {
3116 		if (nsegs < QLNX_FP_MAX_SEGS)
3117 			fp->tx_pkts[(nsegs - 1)]++;
3118 		else
3119 			fp->tx_pkts[(QLNX_FP_MAX_SEGS - 1)]++;
3120 	}
3121 
3122 #ifdef QLNX_TRACE_PERF_DATA
3123         if (m_head->m_pkthdr.csum_flags & CSUM_TSO) {
3124                 if(m_head->m_pkthdr.len <= 2048)
3125                         fp->tx_pkts_hist[0]++;
3126                 else if((m_head->m_pkthdr.len > 2048) &&
3127 				(m_head->m_pkthdr.len <= 4096))
3128                         fp->tx_pkts_hist[1]++;
3129                 else if((m_head->m_pkthdr.len > 4096) &&
3130 				(m_head->m_pkthdr.len <= 8192))
3131                         fp->tx_pkts_hist[2]++;
3132                 else if((m_head->m_pkthdr.len > 8192) &&
3133 				(m_head->m_pkthdr.len <= 12288 ))
3134                         fp->tx_pkts_hist[3]++;
3135                 else if((m_head->m_pkthdr.len > 11288) &&
3136 				(m_head->m_pkthdr.len <= 16394))
3137                         fp->tx_pkts_hist[4]++;
3138                 else if((m_head->m_pkthdr.len > 16384) &&
3139 				(m_head->m_pkthdr.len <= 20480))
3140                         fp->tx_pkts_hist[5]++;
3141                 else if((m_head->m_pkthdr.len > 20480) &&
3142 				(m_head->m_pkthdr.len <= 24576))
3143                         fp->tx_pkts_hist[6]++;
3144                 else if((m_head->m_pkthdr.len > 24576) &&
3145 				(m_head->m_pkthdr.len <= 28672))
3146                         fp->tx_pkts_hist[7]++;
3147                 else if((m_head->m_pkthdr.len > 28762) &&
3148 				(m_head->m_pkthdr.len <= 32768))
3149                         fp->tx_pkts_hist[8]++;
3150                 else if((m_head->m_pkthdr.len > 32768) &&
3151 				(m_head->m_pkthdr.len <= 36864))
3152                         fp->tx_pkts_hist[9]++;
3153                 else if((m_head->m_pkthdr.len > 36864) &&
3154 				(m_head->m_pkthdr.len <= 40960))
3155                         fp->tx_pkts_hist[10]++;
3156                 else if((m_head->m_pkthdr.len > 40960) &&
3157 				(m_head->m_pkthdr.len <= 45056))
3158                         fp->tx_pkts_hist[11]++;
3159                 else if((m_head->m_pkthdr.len > 45056) &&
3160 				(m_head->m_pkthdr.len <= 49152))
3161                         fp->tx_pkts_hist[12]++;
3162                 else if((m_head->m_pkthdr.len > 49512) &&
3163 				m_head->m_pkthdr.len <= 53248))
3164                         fp->tx_pkts_hist[13]++;
3165                 else if((m_head->m_pkthdr.len > 53248) &&
3166 				(m_head->m_pkthdr.len <= 57344))
3167                         fp->tx_pkts_hist[14]++;
3168                 else if((m_head->m_pkthdr.len > 53248) &&
3169 				(m_head->m_pkthdr.len <= 57344))
3170                         fp->tx_pkts_hist[15]++;
3171                 else if((m_head->m_pkthdr.len > 57344) &&
3172 				(m_head->m_pkthdr.len <= 61440))
3173                         fp->tx_pkts_hist[16]++;
3174                 else
3175                         fp->tx_pkts_hist[17]++;
3176         }
3177 
3178         if (m_head->m_pkthdr.csum_flags & CSUM_TSO) {
3179 
3180                 elem_left =  ecore_chain_get_elem_left(&txq->tx_pbl);
3181                 bd_used = TX_RING_SIZE - elem_left;
3182 
3183                 if(bd_used <= 100)
3184                         fp->tx_pkts_q[0]++;
3185                 else if((bd_used > 100) && (bd_used <= 500))
3186                         fp->tx_pkts_q[1]++;
3187                 else if((bd_used > 500) && (bd_used <= 1000))
3188                         fp->tx_pkts_q[2]++;
3189                 else if((bd_used > 1000) && (bd_used <= 2000))
3190                         fp->tx_pkts_q[3]++;
3191                 else if((bd_used > 3000) && (bd_used <= 4000))
3192                         fp->tx_pkts_q[4]++;
3193                 else if((bd_used > 4000) && (bd_used <= 5000))
3194                         fp->tx_pkts_q[5]++;
3195                 else if((bd_used > 6000) && (bd_used <= 7000))
3196                         fp->tx_pkts_q[6]++;
3197                 else if((bd_used > 7000) && (bd_used <= 8000))
3198                         fp->tx_pkts_q[7]++;
3199                 else if((bd_used > 8000) && (bd_used <= 9000))
3200                         fp->tx_pkts_q[8]++;
3201                 else if((bd_used > 9000) && (bd_used <= 10000))
3202                         fp->tx_pkts_q[9]++;
3203                 else if((bd_used > 10000) && (bd_used <= 11000))
3204                         fp->tx_pkts_q[10]++;
3205                 else if((bd_used > 11000) && (bd_used <= 12000))
3206                         fp->tx_pkts_q[11]++;
3207                 else if((bd_used > 12000) && (bd_used <= 13000))
3208                         fp->tx_pkts_q[12]++;
3209                 else if((bd_used > 13000) && (bd_used <= 14000))
3210                         fp->tx_pkts_q[13]++;
3211                 else if((bd_used > 14000) && (bd_used <= 15000))
3212                         fp->tx_pkts_q[14]++;
3213                else if((bd_used > 15000) && (bd_used <= 16000))
3214                         fp->tx_pkts_q[15]++;
3215                 else
3216                         fp->tx_pkts_q[16]++;
3217         }
3218 
3219 #endif /* end of QLNX_TRACE_PERF_DATA */
3220 
3221 	if ((nsegs + QLNX_TX_ELEM_RESERVE) >
3222 		(int)(elem_left = ecore_chain_get_elem_left(&txq->tx_pbl))) {
3223 
3224 		QL_DPRINT1(ha, "(%d, 0x%x) insuffient BDs"
3225 			" in chain[%d] trying to free packets\n",
3226 			nsegs, elem_left, fp->rss_id);
3227 
3228 		fp->tx_nsegs_gt_elem_left++;
3229 
3230 		(void)qlnx_tx_int(ha, fp, txq);
3231 
3232 		if ((nsegs + QLNX_TX_ELEM_RESERVE) > (int)(elem_left =
3233 			ecore_chain_get_elem_left(&txq->tx_pbl))) {
3234 
3235 			QL_DPRINT1(ha,
3236 				"(%d, 0x%x) insuffient BDs in chain[%d]\n",
3237 				nsegs, elem_left, fp->rss_id);
3238 
3239 			fp->err_tx_nsegs_gt_elem_left++;
3240 			fp->tx_ring_full = 1;
3241 			if (ha->storm_stats_enable)
3242 				ha->storm_stats_gather = 1;
3243 			return (ENOBUFS);
3244 		}
3245 	}
3246 
3247 	bus_dmamap_sync(ha->tx_tag, map, BUS_DMASYNC_PREWRITE);
3248 
3249 	txq->sw_tx_ring[idx].mp = m_head;
3250 
3251 	first_bd = (struct eth_tx_1st_bd *)ecore_chain_produce(&txq->tx_pbl);
3252 
3253 	memset(first_bd, 0, sizeof(*first_bd));
3254 
3255 	first_bd->data.bd_flags.bitfields =
3256 		1 << ETH_TX_1ST_BD_FLAGS_START_BD_SHIFT;
3257 
3258 	BD_SET_UNMAP_ADDR_LEN(first_bd, segs->ds_addr, segs->ds_len);
3259 
3260 	nbd++;
3261 
3262 	if (m_head->m_pkthdr.csum_flags & CSUM_IP) {
3263 		first_bd->data.bd_flags.bitfields |=
3264 			(1 << ETH_TX_1ST_BD_FLAGS_IP_CSUM_SHIFT);
3265 	}
3266 
3267 	if (m_head->m_pkthdr.csum_flags &
3268 		(CSUM_UDP | CSUM_TCP | CSUM_TCP_IPV6 | CSUM_UDP_IPV6)) {
3269 		first_bd->data.bd_flags.bitfields |=
3270 			(1 << ETH_TX_1ST_BD_FLAGS_L4_CSUM_SHIFT);
3271 	}
3272 
3273         if (m_head->m_flags & M_VLANTAG) {
3274                 first_bd->data.vlan = m_head->m_pkthdr.ether_vtag;
3275 		first_bd->data.bd_flags.bitfields |=
3276 			(1 << ETH_TX_1ST_BD_FLAGS_VLAN_INSERTION_SHIFT);
3277         }
3278 
3279 	if (m_head->m_pkthdr.csum_flags & CSUM_TSO) {
3280 
3281                 first_bd->data.bd_flags.bitfields |=
3282 			(1 << ETH_TX_1ST_BD_FLAGS_LSO_SHIFT);
3283 		first_bd->data.bd_flags.bitfields |=
3284 			(1 << ETH_TX_1ST_BD_FLAGS_IP_CSUM_SHIFT);
3285 
3286 		nbds_in_hdr = 1;
3287 
3288 		if (offset == segs->ds_len) {
3289 			BD_SET_UNMAP_ADDR_LEN(first_bd, segs->ds_addr, offset);
3290 			segs++;
3291 			seg_idx++;
3292 
3293 			second_bd = (struct eth_tx_2nd_bd *)
3294 					ecore_chain_produce(&txq->tx_pbl);
3295 			memset(second_bd, 0, sizeof(*second_bd));
3296 			nbd++;
3297 
3298 			if (seg_idx < nsegs) {
3299 				BD_SET_UNMAP_ADDR_LEN(second_bd, \
3300 					(segs->ds_addr), (segs->ds_len));
3301 				segs++;
3302 				seg_idx++;
3303 			}
3304 
3305 			third_bd = (struct eth_tx_3rd_bd *)
3306 					ecore_chain_produce(&txq->tx_pbl);
3307 			memset(third_bd, 0, sizeof(*third_bd));
3308 			third_bd->data.lso_mss = m_head->m_pkthdr.tso_segsz;
3309 			third_bd->data.bitfields |=
3310 				(nbds_in_hdr<<ETH_TX_DATA_3RD_BD_HDR_NBD_SHIFT);
3311 			nbd++;
3312 
3313 			if (seg_idx < nsegs) {
3314 				BD_SET_UNMAP_ADDR_LEN(third_bd, \
3315 					(segs->ds_addr), (segs->ds_len));
3316 				segs++;
3317 				seg_idx++;
3318 			}
3319 
3320 			for (; seg_idx < nsegs; seg_idx++) {
3321 				tx_data_bd = (struct eth_tx_bd *)
3322 					ecore_chain_produce(&txq->tx_pbl);
3323 				memset(tx_data_bd, 0, sizeof(*tx_data_bd));
3324 				BD_SET_UNMAP_ADDR_LEN(tx_data_bd, \
3325 					segs->ds_addr,\
3326 					segs->ds_len);
3327 				segs++;
3328 				nbd++;
3329 			}
3330 
3331 		} else if (offset < segs->ds_len) {
3332 			BD_SET_UNMAP_ADDR_LEN(first_bd, segs->ds_addr, offset);
3333 
3334 			second_bd = (struct eth_tx_2nd_bd *)
3335 					ecore_chain_produce(&txq->tx_pbl);
3336 			memset(second_bd, 0, sizeof(*second_bd));
3337 			BD_SET_UNMAP_ADDR_LEN(second_bd, \
3338 				(segs->ds_addr + offset),\
3339 				(segs->ds_len - offset));
3340 			nbd++;
3341 			segs++;
3342 
3343 			third_bd = (struct eth_tx_3rd_bd *)
3344 					ecore_chain_produce(&txq->tx_pbl);
3345 			memset(third_bd, 0, sizeof(*third_bd));
3346 
3347 			BD_SET_UNMAP_ADDR_LEN(third_bd, \
3348 					segs->ds_addr,\
3349 					segs->ds_len);
3350 			third_bd->data.lso_mss = m_head->m_pkthdr.tso_segsz;
3351 			third_bd->data.bitfields |=
3352 				(nbds_in_hdr<<ETH_TX_DATA_3RD_BD_HDR_NBD_SHIFT);
3353 			segs++;
3354 			nbd++;
3355 
3356 			for (seg_idx = 2; seg_idx < nsegs; seg_idx++) {
3357 				tx_data_bd = (struct eth_tx_bd *)
3358 					ecore_chain_produce(&txq->tx_pbl);
3359 				memset(tx_data_bd, 0, sizeof(*tx_data_bd));
3360 				BD_SET_UNMAP_ADDR_LEN(tx_data_bd, \
3361 					segs->ds_addr,\
3362 					segs->ds_len);
3363 				segs++;
3364 				nbd++;
3365 			}
3366 
3367 		} else {
3368 			offset = offset - segs->ds_len;
3369 			segs++;
3370 
3371 			for (seg_idx = 1; seg_idx < nsegs; seg_idx++) {
3372 
3373 				if (offset)
3374 					nbds_in_hdr++;
3375 
3376 				tx_data_bd = (struct eth_tx_bd *)
3377 					ecore_chain_produce(&txq->tx_pbl);
3378 				memset(tx_data_bd, 0, sizeof(*tx_data_bd));
3379 
3380 				if (second_bd == NULL) {
3381 					second_bd = (struct eth_tx_2nd_bd *)
3382 								tx_data_bd;
3383 				} else if (third_bd == NULL) {
3384 					third_bd = (struct eth_tx_3rd_bd *)
3385 								tx_data_bd;
3386 				}
3387 
3388 				if (offset && (offset < segs->ds_len)) {
3389 					BD_SET_UNMAP_ADDR_LEN(tx_data_bd,\
3390 						segs->ds_addr, offset);
3391 
3392 					tx_data_bd = (struct eth_tx_bd *)
3393 					ecore_chain_produce(&txq->tx_pbl);
3394 
3395 					memset(tx_data_bd, 0,
3396 						sizeof(*tx_data_bd));
3397 
3398 					if (second_bd == NULL) {
3399 						second_bd =
3400 					(struct eth_tx_2nd_bd *)tx_data_bd;
3401 					} else if (third_bd == NULL) {
3402 						third_bd =
3403 					(struct eth_tx_3rd_bd *)tx_data_bd;
3404 					}
3405 					BD_SET_UNMAP_ADDR_LEN(tx_data_bd,\
3406 						(segs->ds_addr + offset), \
3407 						(segs->ds_len - offset));
3408 					nbd++;
3409 					offset = 0;
3410 				} else {
3411 					if (offset)
3412 						offset = offset - segs->ds_len;
3413 					BD_SET_UNMAP_ADDR_LEN(tx_data_bd,\
3414 						segs->ds_addr, segs->ds_len);
3415 				}
3416 				segs++;
3417 				nbd++;
3418 			}
3419 
3420 			if (third_bd == NULL) {
3421 				third_bd = (struct eth_tx_3rd_bd *)
3422 					ecore_chain_produce(&txq->tx_pbl);
3423 				memset(third_bd, 0, sizeof(*third_bd));
3424 			}
3425 
3426 			third_bd->data.lso_mss = m_head->m_pkthdr.tso_segsz;
3427 			third_bd->data.bitfields |=
3428 				(nbds_in_hdr<<ETH_TX_DATA_3RD_BD_HDR_NBD_SHIFT);
3429 		}
3430 		fp->tx_tso_pkts++;
3431 	} else {
3432 		segs++;
3433 		for (seg_idx = 1; seg_idx < nsegs; seg_idx++) {
3434 			tx_data_bd = (struct eth_tx_bd *)
3435 					ecore_chain_produce(&txq->tx_pbl);
3436 			memset(tx_data_bd, 0, sizeof(*tx_data_bd));
3437 			BD_SET_UNMAP_ADDR_LEN(tx_data_bd, segs->ds_addr,\
3438 				segs->ds_len);
3439 			segs++;
3440 			nbd++;
3441 		}
3442 		first_bd->data.bitfields =
3443 			(m_head->m_pkthdr.len & ETH_TX_DATA_1ST_BD_PKT_LEN_MASK)
3444 				 << ETH_TX_DATA_1ST_BD_PKT_LEN_SHIFT;
3445 		first_bd->data.bitfields =
3446 			htole16(first_bd->data.bitfields);
3447 		fp->tx_non_tso_pkts++;
3448 	}
3449 
3450 
3451 	first_bd->data.nbds = nbd;
3452 
3453 	if (ha->dbg_trace_tso_pkt_len) {
3454 		if (fp->tx_tso_max_nsegs < nsegs)
3455 			fp->tx_tso_max_nsegs = nsegs;
3456 
3457 		if ((nsegs < fp->tx_tso_min_nsegs) || (!fp->tx_tso_min_nsegs))
3458 			fp->tx_tso_min_nsegs = nsegs;
3459 	}
3460 
3461 	txq->sw_tx_ring[idx].nsegs = nsegs;
3462 	txq->sw_tx_prod = (txq->sw_tx_prod + 1) & (TX_RING_SIZE - 1);
3463 
3464 	txq->tx_db.data.bd_prod =
3465 		htole16(ecore_chain_get_prod_idx(&txq->tx_pbl));
3466 
3467 	qlnx_txq_doorbell_wr32(ha, txq->doorbell_addr, txq->tx_db.raw);
3468 
3469 	QL_DPRINT8(ha, "exit\n");
3470 	return (0);
3471 }
3472 
3473 static void
3474 qlnx_stop(qlnx_host_t *ha)
3475 {
3476 	struct ifnet	*ifp = ha->ifp;
3477 	device_t	dev;
3478 	int		i;
3479 
3480 	dev = ha->pci_dev;
3481 
3482 	ifp->if_drv_flags &= ~(IFF_DRV_OACTIVE | IFF_DRV_RUNNING);
3483 
3484 	/*
3485 	 * We simply lock and unlock each fp->tx_mtx to
3486 	 * propagate the if_drv_flags
3487 	 * state to each tx thread
3488 	 */
3489         QL_DPRINT1(ha, "QLNX STATE = %d\n",ha->state);
3490 
3491 	if (ha->state == QLNX_STATE_OPEN) {
3492         	for (i = 0; i < ha->num_rss; i++) {
3493 			struct qlnx_fastpath *fp = &ha->fp_array[i];
3494 
3495 			mtx_lock(&fp->tx_mtx);
3496 			mtx_unlock(&fp->tx_mtx);
3497 
3498 			if (fp->fp_taskqueue != NULL)
3499 				taskqueue_enqueue(fp->fp_taskqueue,
3500 					&fp->fp_task);
3501 		}
3502 	}
3503 
3504 	qlnx_unload(ha);
3505 
3506 	return;
3507 }
3508 
3509 static int
3510 qlnx_get_ifq_snd_maxlen(qlnx_host_t *ha)
3511 {
3512         return(TX_RING_SIZE - 1);
3513 }
3514 
3515 uint8_t *
3516 qlnx_get_mac_addr(qlnx_host_t *ha)
3517 {
3518 	struct ecore_hwfn	*p_hwfn;
3519 
3520 	p_hwfn = &ha->cdev.hwfns[0];
3521         return (p_hwfn->hw_info.hw_mac_addr);
3522 }
3523 
3524 static uint32_t
3525 qlnx_get_optics(qlnx_host_t *ha, struct qlnx_link_output *if_link)
3526 {
3527 	uint32_t	ifm_type = 0;
3528 
3529 	switch (if_link->media_type) {
3530 
3531 	case MEDIA_MODULE_FIBER:
3532 	case MEDIA_UNSPECIFIED:
3533 		if (if_link->speed == (100 * 1000))
3534 			ifm_type = QLNX_IFM_100G_SR4;
3535 		else if (if_link->speed == (40 * 1000))
3536 			ifm_type = IFM_40G_SR4;
3537 		else if (if_link->speed == (25 * 1000))
3538 			ifm_type = QLNX_IFM_25G_SR;
3539 		else if (if_link->speed == (10 * 1000))
3540 			ifm_type = (IFM_10G_LR | IFM_10G_SR);
3541 		else if (if_link->speed == (1 * 1000))
3542 			ifm_type = (IFM_1000_SX | IFM_1000_LX);
3543 
3544 		break;
3545 
3546 	case MEDIA_DA_TWINAX:
3547 		if (if_link->speed == (100 * 1000))
3548 			ifm_type = QLNX_IFM_100G_CR4;
3549 		else if (if_link->speed == (40 * 1000))
3550 			ifm_type = IFM_40G_CR4;
3551 		else if (if_link->speed == (25 * 1000))
3552 			ifm_type = QLNX_IFM_25G_CR;
3553 		else if (if_link->speed == (10 * 1000))
3554 			ifm_type = IFM_10G_TWINAX;
3555 
3556 		break;
3557 
3558 	default :
3559 		ifm_type = IFM_UNKNOWN;
3560 		break;
3561 	}
3562 	return (ifm_type);
3563 }
3564 
3565 
3566 
3567 /*****************************************************************************
3568  * Interrupt Service Functions
3569  *****************************************************************************/
3570 
3571 static int
3572 qlnx_rx_jumbo_chain(qlnx_host_t *ha, struct qlnx_fastpath *fp,
3573 	struct mbuf *mp_head, uint16_t len)
3574 {
3575 	struct mbuf		*mp, *mpf, *mpl;
3576 	struct sw_rx_data	*sw_rx_data;
3577 	struct qlnx_rx_queue	*rxq;
3578 	uint16_t 		len_in_buffer;
3579 
3580 	rxq = fp->rxq;
3581 	mpf = mpl = mp = NULL;
3582 
3583 	while (len) {
3584 
3585         	rxq->sw_rx_cons  = (rxq->sw_rx_cons + 1) & (RX_RING_SIZE - 1);
3586 
3587                 sw_rx_data = &rxq->sw_rx_ring[rxq->sw_rx_cons];
3588                 mp = sw_rx_data->data;
3589 
3590 		if (mp == NULL) {
3591                 	QL_DPRINT1(ha, "mp = NULL\n");
3592 			fp->err_rx_mp_null++;
3593         		rxq->sw_rx_cons  =
3594 				(rxq->sw_rx_cons + 1) & (RX_RING_SIZE - 1);
3595 
3596 			if (mpf != NULL)
3597 				m_freem(mpf);
3598 
3599 			return (-1);
3600 		}
3601 		bus_dmamap_sync(ha->rx_tag, sw_rx_data->map,
3602 			BUS_DMASYNC_POSTREAD);
3603 
3604                 if (qlnx_alloc_rx_buffer(ha, rxq) != 0) {
3605 
3606                         QL_DPRINT1(ha, "New buffer allocation failed, dropping"
3607 				" incoming packet and reusing its buffer\n");
3608 
3609                         qlnx_reuse_rx_data(rxq);
3610                         fp->err_rx_alloc_errors++;
3611 
3612 			if (mpf != NULL)
3613 				m_freem(mpf);
3614 
3615 			return (-1);
3616 		}
3617                 ecore_chain_consume(&rxq->rx_bd_ring);
3618 
3619 		if (len > rxq->rx_buf_size)
3620 			len_in_buffer = rxq->rx_buf_size;
3621 		else
3622 			len_in_buffer = len;
3623 
3624 		len = len - len_in_buffer;
3625 
3626 		mp->m_flags &= ~M_PKTHDR;
3627 		mp->m_next = NULL;
3628 		mp->m_len = len_in_buffer;
3629 
3630 		if (mpf == NULL)
3631 			mpf = mpl = mp;
3632 		else {
3633 			mpl->m_next = mp;
3634 			mpl = mp;
3635 		}
3636 	}
3637 
3638 	if (mpf != NULL)
3639 		mp_head->m_next = mpf;
3640 
3641 	return (0);
3642 }
3643 
3644 static void
3645 qlnx_tpa_start(qlnx_host_t *ha,
3646 	struct qlnx_fastpath *fp,
3647 	struct qlnx_rx_queue *rxq,
3648 	struct eth_fast_path_rx_tpa_start_cqe *cqe)
3649 {
3650 	uint32_t		agg_index;
3651         struct ifnet		*ifp = ha->ifp;
3652 	struct mbuf		*mp;
3653 	struct mbuf		*mpf = NULL, *mpl = NULL, *mpc = NULL;
3654 	struct sw_rx_data	*sw_rx_data;
3655 	dma_addr_t		addr;
3656 	bus_dmamap_t		map;
3657 	struct eth_rx_bd	*rx_bd;
3658 	int			i;
3659 	device_t		dev;
3660 #if __FreeBSD_version >= 1100000
3661 	uint8_t			hash_type;
3662 #endif /* #if __FreeBSD_version >= 1100000 */
3663 
3664 	dev = ha->pci_dev;
3665 	agg_index = cqe->tpa_agg_index;
3666 
3667         QL_DPRINT7(ha, "[rss_id = %d]: enter\n \
3668                 \t type = 0x%x\n \
3669                 \t bitfields = 0x%x\n \
3670                 \t seg_len = 0x%x\n \
3671                 \t pars_flags = 0x%x\n \
3672                 \t vlan_tag = 0x%x\n \
3673                 \t rss_hash = 0x%x\n \
3674                 \t len_on_first_bd = 0x%x\n \
3675                 \t placement_offset = 0x%x\n \
3676                 \t tpa_agg_index = 0x%x\n \
3677                 \t header_len = 0x%x\n \
3678                 \t ext_bd_len_list[0] = 0x%x\n \
3679                 \t ext_bd_len_list[1] = 0x%x\n \
3680                 \t ext_bd_len_list[2] = 0x%x\n \
3681                 \t ext_bd_len_list[3] = 0x%x\n \
3682                 \t ext_bd_len_list[4] = 0x%x\n",
3683                 fp->rss_id, cqe->type, cqe->bitfields, cqe->seg_len,
3684                 cqe->pars_flags.flags, cqe->vlan_tag,
3685                 cqe->rss_hash, cqe->len_on_first_bd, cqe->placement_offset,
3686                 cqe->tpa_agg_index, cqe->header_len,
3687                 cqe->ext_bd_len_list[0], cqe->ext_bd_len_list[1],
3688                 cqe->ext_bd_len_list[2], cqe->ext_bd_len_list[3],
3689                 cqe->ext_bd_len_list[4]);
3690 
3691 	if (agg_index >= ETH_TPA_MAX_AGGS_NUM) {
3692 		fp->err_rx_tpa_invalid_agg_num++;
3693 		return;
3694 	}
3695 
3696 	sw_rx_data = &rxq->sw_rx_ring[rxq->sw_rx_cons];
3697 	bus_dmamap_sync(ha->rx_tag, sw_rx_data->map, BUS_DMASYNC_POSTREAD);
3698 	mp = sw_rx_data->data;
3699 
3700 	QL_DPRINT7(ha, "[rss_id = %d]: mp = %p \n ", fp->rss_id, mp);
3701 
3702 	if (mp == NULL) {
3703                	QL_DPRINT7(ha, "[%d]: mp = NULL\n", fp->rss_id);
3704 		fp->err_rx_mp_null++;
3705        		rxq->sw_rx_cons = (rxq->sw_rx_cons + 1) & (RX_RING_SIZE - 1);
3706 
3707 		return;
3708 	}
3709 
3710 	if ((le16toh(cqe->pars_flags.flags)) & CQE_FLAGS_ERR) {
3711 
3712 		QL_DPRINT7(ha, "[%d]: CQE in CONS = %u has error,"
3713 			" flags = %x, dropping incoming packet\n", fp->rss_id,
3714 			rxq->sw_rx_cons, le16toh(cqe->pars_flags.flags));
3715 
3716 		fp->err_rx_hw_errors++;
3717 
3718 		qlnx_reuse_rx_data(rxq);
3719 
3720 		QLNX_INC_IERRORS(ifp);
3721 
3722 		return;
3723 	}
3724 
3725 	if (qlnx_alloc_rx_buffer(ha, rxq) != 0) {
3726 
3727 		QL_DPRINT7(ha, "[%d]: New buffer allocation failed,"
3728 			" dropping incoming packet and reusing its buffer\n",
3729 			fp->rss_id);
3730 
3731 		fp->err_rx_alloc_errors++;
3732 		QLNX_INC_IQDROPS(ifp);
3733 
3734 		/*
3735 		 * Load the tpa mbuf into the rx ring and save the
3736 		 * posted mbuf
3737 		 */
3738 
3739 		map = sw_rx_data->map;
3740 		addr = sw_rx_data->dma_addr;
3741 
3742 		sw_rx_data = &rxq->sw_rx_ring[rxq->sw_rx_prod];
3743 
3744 		sw_rx_data->data = rxq->tpa_info[agg_index].rx_buf.data;
3745 		sw_rx_data->dma_addr = rxq->tpa_info[agg_index].rx_buf.dma_addr;
3746 		sw_rx_data->map = rxq->tpa_info[agg_index].rx_buf.map;
3747 
3748 		rxq->tpa_info[agg_index].rx_buf.data = mp;
3749 		rxq->tpa_info[agg_index].rx_buf.dma_addr = addr;
3750 		rxq->tpa_info[agg_index].rx_buf.map = map;
3751 
3752 		rx_bd = (struct eth_rx_bd *)
3753 				ecore_chain_produce(&rxq->rx_bd_ring);
3754 
3755 		rx_bd->addr.hi = htole32(U64_HI(sw_rx_data->dma_addr));
3756 		rx_bd->addr.lo = htole32(U64_LO(sw_rx_data->dma_addr));
3757 
3758 		bus_dmamap_sync(ha->rx_tag, sw_rx_data->map,
3759 			BUS_DMASYNC_PREREAD);
3760 
3761 		rxq->sw_rx_prod = (rxq->sw_rx_prod + 1) & (RX_RING_SIZE - 1);
3762 		rxq->sw_rx_cons = (rxq->sw_rx_cons + 1) & (RX_RING_SIZE - 1);
3763 
3764 		ecore_chain_consume(&rxq->rx_bd_ring);
3765 
3766 		/* Now reuse any buffers posted in ext_bd_len_list */
3767 		for (i = 0; i < ETH_TPA_CQE_START_LEN_LIST_SIZE; i++) {
3768 
3769 			if (cqe->ext_bd_len_list[i] == 0)
3770 				break;
3771 
3772 			qlnx_reuse_rx_data(rxq);
3773 		}
3774 
3775 		rxq->tpa_info[agg_index].agg_state = QLNX_AGG_STATE_ERROR;
3776 		return;
3777 	}
3778 
3779 	if (rxq->tpa_info[agg_index].agg_state != QLNX_AGG_STATE_NONE) {
3780 
3781 		QL_DPRINT7(ha, "[%d]: invalid aggregation state,"
3782 			" dropping incoming packet and reusing its buffer\n",
3783 			fp->rss_id);
3784 
3785 		QLNX_INC_IQDROPS(ifp);
3786 
3787 		/* if we already have mbuf head in aggregation free it */
3788 		if (rxq->tpa_info[agg_index].mpf) {
3789 			m_freem(rxq->tpa_info[agg_index].mpf);
3790 			rxq->tpa_info[agg_index].mpl = NULL;
3791 		}
3792 		rxq->tpa_info[agg_index].mpf = mp;
3793 		rxq->tpa_info[agg_index].mpl = NULL;
3794 
3795 		rxq->sw_rx_cons = (rxq->sw_rx_cons + 1) & (RX_RING_SIZE - 1);
3796 		ecore_chain_consume(&rxq->rx_bd_ring);
3797 
3798 		/* Now reuse any buffers posted in ext_bd_len_list */
3799 		for (i = 0; i < ETH_TPA_CQE_START_LEN_LIST_SIZE; i++) {
3800 
3801 			if (cqe->ext_bd_len_list[i] == 0)
3802 				break;
3803 
3804 			qlnx_reuse_rx_data(rxq);
3805 		}
3806 		rxq->tpa_info[agg_index].agg_state = QLNX_AGG_STATE_ERROR;
3807 
3808 		return;
3809 	}
3810 
3811 	/*
3812 	 * first process the ext_bd_len_list
3813 	 * if this fails then we simply drop the packet
3814 	 */
3815 	ecore_chain_consume(&rxq->rx_bd_ring);
3816 	rxq->sw_rx_cons  = (rxq->sw_rx_cons + 1) & (RX_RING_SIZE - 1);
3817 
3818 	for (i = 0; i < ETH_TPA_CQE_START_LEN_LIST_SIZE; i++) {
3819 
3820 		QL_DPRINT7(ha, "[%d]: 4\n ", fp->rss_id);
3821 
3822 		if (cqe->ext_bd_len_list[i] == 0)
3823 			break;
3824 
3825 		sw_rx_data = &rxq->sw_rx_ring[rxq->sw_rx_cons];
3826 		bus_dmamap_sync(ha->rx_tag, sw_rx_data->map,
3827 			BUS_DMASYNC_POSTREAD);
3828 
3829 		mpc = sw_rx_data->data;
3830 
3831 		if (mpc == NULL) {
3832 			QL_DPRINT7(ha, "[%d]: mpc = NULL\n", fp->rss_id);
3833 			fp->err_rx_mp_null++;
3834 			if (mpf != NULL)
3835 				m_freem(mpf);
3836 			mpf = mpl = NULL;
3837 			rxq->tpa_info[agg_index].agg_state =
3838 						QLNX_AGG_STATE_ERROR;
3839 			ecore_chain_consume(&rxq->rx_bd_ring);
3840 			rxq->sw_rx_cons =
3841 				(rxq->sw_rx_cons + 1) & (RX_RING_SIZE - 1);
3842 			continue;
3843 		}
3844 
3845 		if (qlnx_alloc_rx_buffer(ha, rxq) != 0) {
3846 			QL_DPRINT7(ha, "[%d]: New buffer allocation failed,"
3847 				" dropping incoming packet and reusing its"
3848 				" buffer\n", fp->rss_id);
3849 
3850 			qlnx_reuse_rx_data(rxq);
3851 
3852 			if (mpf != NULL)
3853 				m_freem(mpf);
3854 			mpf = mpl = NULL;
3855 
3856 			rxq->tpa_info[agg_index].agg_state =
3857 						QLNX_AGG_STATE_ERROR;
3858 
3859 			ecore_chain_consume(&rxq->rx_bd_ring);
3860 			rxq->sw_rx_cons =
3861 				(rxq->sw_rx_cons + 1) & (RX_RING_SIZE - 1);
3862 
3863 			continue;
3864 		}
3865 
3866 		mpc->m_flags &= ~M_PKTHDR;
3867 		mpc->m_next = NULL;
3868 		mpc->m_len = cqe->ext_bd_len_list[i];
3869 
3870 
3871 		if (mpf == NULL) {
3872 			mpf = mpl = mpc;
3873 		} else {
3874 			mpl->m_len = ha->rx_buf_size;
3875 			mpl->m_next = mpc;
3876 			mpl = mpc;
3877 		}
3878 
3879 		ecore_chain_consume(&rxq->rx_bd_ring);
3880 		rxq->sw_rx_cons =
3881 			(rxq->sw_rx_cons + 1) & (RX_RING_SIZE - 1);
3882 	}
3883 
3884 	if (rxq->tpa_info[agg_index].agg_state != QLNX_AGG_STATE_NONE) {
3885 
3886 		QL_DPRINT7(ha, "[%d]: invalid aggregation state, dropping"
3887 			" incoming packet and reusing its buffer\n",
3888 			fp->rss_id);
3889 
3890 		QLNX_INC_IQDROPS(ifp);
3891 
3892 		rxq->tpa_info[agg_index].mpf = mp;
3893 		rxq->tpa_info[agg_index].mpl = NULL;
3894 
3895 		return;
3896 	}
3897 
3898         rxq->tpa_info[agg_index].placement_offset = cqe->placement_offset;
3899 
3900         if (mpf != NULL) {
3901                 mp->m_len = ha->rx_buf_size;
3902                 mp->m_next = mpf;
3903                 rxq->tpa_info[agg_index].mpf = mp;
3904                 rxq->tpa_info[agg_index].mpl = mpl;
3905         } else {
3906                 mp->m_len = cqe->len_on_first_bd + cqe->placement_offset;
3907                 rxq->tpa_info[agg_index].mpf = mp;
3908                 rxq->tpa_info[agg_index].mpl = mp;
3909                 mp->m_next = NULL;
3910         }
3911 
3912 	mp->m_flags |= M_PKTHDR;
3913 
3914 	/* assign packet to this interface interface */
3915 	mp->m_pkthdr.rcvif = ifp;
3916 
3917 	/* assume no hardware checksum has complated */
3918 	mp->m_pkthdr.csum_flags = 0;
3919 
3920 	//mp->m_pkthdr.flowid = fp->rss_id;
3921 	mp->m_pkthdr.flowid = cqe->rss_hash;
3922 
3923 #if __FreeBSD_version >= 1100000
3924 
3925 	hash_type = cqe->bitfields &
3926 			(ETH_FAST_PATH_RX_REG_CQE_RSS_HASH_TYPE_MASK <<
3927 			ETH_FAST_PATH_RX_REG_CQE_RSS_HASH_TYPE_SHIFT);
3928 
3929 	switch (hash_type) {
3930 
3931 	case RSS_HASH_TYPE_IPV4:
3932 		M_HASHTYPE_SET(mp, M_HASHTYPE_RSS_IPV4);
3933 		break;
3934 
3935 	case RSS_HASH_TYPE_TCP_IPV4:
3936 		M_HASHTYPE_SET(mp, M_HASHTYPE_RSS_TCP_IPV4);
3937 		break;
3938 
3939 	case RSS_HASH_TYPE_IPV6:
3940 		M_HASHTYPE_SET(mp, M_HASHTYPE_RSS_IPV6);
3941 		break;
3942 
3943 	case RSS_HASH_TYPE_TCP_IPV6:
3944 		M_HASHTYPE_SET(mp, M_HASHTYPE_RSS_TCP_IPV6);
3945 		break;
3946 
3947 	default:
3948 		M_HASHTYPE_SET(mp, M_HASHTYPE_OPAQUE);
3949 		break;
3950 	}
3951 
3952 #else
3953 	mp->m_flags |= M_FLOWID;
3954 #endif
3955 
3956 	mp->m_pkthdr.csum_flags |= (CSUM_IP_CHECKED | CSUM_IP_VALID |
3957 					CSUM_DATA_VALID | CSUM_PSEUDO_HDR);
3958 
3959 	mp->m_pkthdr.csum_data = 0xFFFF;
3960 
3961 	if (CQE_HAS_VLAN(cqe->pars_flags.flags)) {
3962 		mp->m_pkthdr.ether_vtag = le16toh(cqe->vlan_tag);
3963 		mp->m_flags |= M_VLANTAG;
3964 	}
3965 
3966 	rxq->tpa_info[agg_index].agg_state = QLNX_AGG_STATE_START;
3967 
3968         QL_DPRINT7(ha, "[%d]: 5\n\tagg_state = %d\n\t mpf = %p mpl = %p\n",
3969 		fp->rss_id, rxq->tpa_info[agg_index].agg_state,
3970                 rxq->tpa_info[agg_index].mpf, rxq->tpa_info[agg_index].mpl);
3971 
3972 	return;
3973 }
3974 
3975 static void
3976 qlnx_tpa_cont(qlnx_host_t *ha, struct qlnx_fastpath *fp,
3977 	struct qlnx_rx_queue *rxq,
3978 	struct eth_fast_path_rx_tpa_cont_cqe *cqe)
3979 {
3980 	struct sw_rx_data	*sw_rx_data;
3981 	int			i;
3982 	struct mbuf		*mpf = NULL, *mpl = NULL, *mpc = NULL;
3983 	struct mbuf		*mp;
3984 	uint32_t		agg_index;
3985 	device_t		dev;
3986 
3987 	dev = ha->pci_dev;
3988 
3989         QL_DPRINT7(ha, "[%d]: enter\n \
3990                 \t type = 0x%x\n \
3991                 \t tpa_agg_index = 0x%x\n \
3992                 \t len_list[0] = 0x%x\n \
3993                 \t len_list[1] = 0x%x\n \
3994                 \t len_list[2] = 0x%x\n \
3995                 \t len_list[3] = 0x%x\n \
3996                 \t len_list[4] = 0x%x\n \
3997                 \t len_list[5] = 0x%x\n",
3998                 fp->rss_id, cqe->type, cqe->tpa_agg_index,
3999                 cqe->len_list[0], cqe->len_list[1], cqe->len_list[2],
4000                 cqe->len_list[3], cqe->len_list[4], cqe->len_list[5]);
4001 
4002 	agg_index = cqe->tpa_agg_index;
4003 
4004 	if (agg_index >= ETH_TPA_MAX_AGGS_NUM) {
4005 		QL_DPRINT7(ha, "[%d]: 0\n ", fp->rss_id);
4006 		fp->err_rx_tpa_invalid_agg_num++;
4007 		return;
4008 	}
4009 
4010 
4011 	for (i = 0; i < ETH_TPA_CQE_CONT_LEN_LIST_SIZE; i++) {
4012 
4013 		QL_DPRINT7(ha, "[%d]: 1\n ", fp->rss_id);
4014 
4015 		if (cqe->len_list[i] == 0)
4016 			break;
4017 
4018 		if (rxq->tpa_info[agg_index].agg_state !=
4019 			QLNX_AGG_STATE_START) {
4020 			qlnx_reuse_rx_data(rxq);
4021 			continue;
4022 		}
4023 
4024 		sw_rx_data = &rxq->sw_rx_ring[rxq->sw_rx_cons];
4025 		bus_dmamap_sync(ha->rx_tag, sw_rx_data->map,
4026 			BUS_DMASYNC_POSTREAD);
4027 
4028 		mpc = sw_rx_data->data;
4029 
4030 		if (mpc == NULL) {
4031 
4032 			QL_DPRINT7(ha, "[%d]: mpc = NULL\n", fp->rss_id);
4033 
4034 			fp->err_rx_mp_null++;
4035 			if (mpf != NULL)
4036 				m_freem(mpf);
4037 			mpf = mpl = NULL;
4038 			rxq->tpa_info[agg_index].agg_state =
4039 						QLNX_AGG_STATE_ERROR;
4040 			ecore_chain_consume(&rxq->rx_bd_ring);
4041 			rxq->sw_rx_cons =
4042 				(rxq->sw_rx_cons + 1) & (RX_RING_SIZE - 1);
4043 			continue;
4044 		}
4045 
4046 		if (qlnx_alloc_rx_buffer(ha, rxq) != 0) {
4047 
4048 			QL_DPRINT7(ha, "[%d]: New buffer allocation failed,"
4049 				" dropping incoming packet and reusing its"
4050 				" buffer\n", fp->rss_id);
4051 
4052 			qlnx_reuse_rx_data(rxq);
4053 
4054 			if (mpf != NULL)
4055 				m_freem(mpf);
4056 			mpf = mpl = NULL;
4057 
4058 			rxq->tpa_info[agg_index].agg_state =
4059 						QLNX_AGG_STATE_ERROR;
4060 
4061 			ecore_chain_consume(&rxq->rx_bd_ring);
4062 			rxq->sw_rx_cons =
4063 				(rxq->sw_rx_cons + 1) & (RX_RING_SIZE - 1);
4064 
4065 			continue;
4066 		}
4067 
4068 		mpc->m_flags &= ~M_PKTHDR;
4069 		mpc->m_next = NULL;
4070 		mpc->m_len = cqe->len_list[i];
4071 
4072 
4073 		if (mpf == NULL) {
4074 			mpf = mpl = mpc;
4075 		} else {
4076 			mpl->m_len = ha->rx_buf_size;
4077 			mpl->m_next = mpc;
4078 			mpl = mpc;
4079 		}
4080 
4081 		ecore_chain_consume(&rxq->rx_bd_ring);
4082 		rxq->sw_rx_cons =
4083 			(rxq->sw_rx_cons + 1) & (RX_RING_SIZE - 1);
4084 	}
4085 
4086         QL_DPRINT7(ha, "[%d]: 2\n" "\tmpf = %p mpl = %p\n",
4087                   fp->rss_id, mpf, mpl);
4088 
4089 	if (mpf != NULL) {
4090 		mp = rxq->tpa_info[agg_index].mpl;
4091 		mp->m_len = ha->rx_buf_size;
4092 		mp->m_next = mpf;
4093 		rxq->tpa_info[agg_index].mpl = mpl;
4094 	}
4095 
4096 	return;
4097 }
4098 
4099 static int
4100 qlnx_tpa_end(qlnx_host_t *ha, struct qlnx_fastpath *fp,
4101 	struct qlnx_rx_queue *rxq,
4102 	struct eth_fast_path_rx_tpa_end_cqe *cqe)
4103 {
4104 	struct sw_rx_data	*sw_rx_data;
4105 	int			i;
4106 	struct mbuf		*mpf = NULL, *mpl = NULL, *mpc = NULL;
4107 	struct mbuf		*mp;
4108 	uint32_t		agg_index;
4109 	uint32_t		len = 0;
4110         struct ifnet		*ifp = ha->ifp;
4111 	device_t		dev;
4112 
4113 	dev = ha->pci_dev;
4114 
4115         QL_DPRINT7(ha, "[%d]: enter\n \
4116                 \t type = 0x%x\n \
4117                 \t tpa_agg_index = 0x%x\n \
4118                 \t total_packet_len = 0x%x\n \
4119                 \t num_of_bds = 0x%x\n \
4120                 \t end_reason = 0x%x\n \
4121                 \t num_of_coalesced_segs = 0x%x\n \
4122                 \t ts_delta = 0x%x\n \
4123                 \t len_list[0] = 0x%x\n \
4124                 \t len_list[1] = 0x%x\n \
4125                 \t len_list[2] = 0x%x\n \
4126                 \t len_list[3] = 0x%x\n",
4127                  fp->rss_id, cqe->type, cqe->tpa_agg_index,
4128                 cqe->total_packet_len, cqe->num_of_bds,
4129                 cqe->end_reason, cqe->num_of_coalesced_segs, cqe->ts_delta,
4130                 cqe->len_list[0], cqe->len_list[1], cqe->len_list[2],
4131                 cqe->len_list[3]);
4132 
4133 	agg_index = cqe->tpa_agg_index;
4134 
4135 	if (agg_index >= ETH_TPA_MAX_AGGS_NUM) {
4136 
4137 		QL_DPRINT7(ha, "[%d]: 0\n ", fp->rss_id);
4138 
4139 		fp->err_rx_tpa_invalid_agg_num++;
4140 		return (0);
4141 	}
4142 
4143 
4144 	for (i = 0; i < ETH_TPA_CQE_END_LEN_LIST_SIZE; i++) {
4145 
4146 		QL_DPRINT7(ha, "[%d]: 1\n ", fp->rss_id);
4147 
4148 		if (cqe->len_list[i] == 0)
4149 			break;
4150 
4151 		if (rxq->tpa_info[agg_index].agg_state !=
4152 			QLNX_AGG_STATE_START) {
4153 
4154 			QL_DPRINT7(ha, "[%d]: 2\n ", fp->rss_id);
4155 
4156 			qlnx_reuse_rx_data(rxq);
4157 			continue;
4158 		}
4159 
4160 		sw_rx_data = &rxq->sw_rx_ring[rxq->sw_rx_cons];
4161 		bus_dmamap_sync(ha->rx_tag, sw_rx_data->map,
4162 			BUS_DMASYNC_POSTREAD);
4163 
4164 		mpc = sw_rx_data->data;
4165 
4166 		if (mpc == NULL) {
4167 
4168 			QL_DPRINT7(ha, "[%d]: mpc = NULL\n", fp->rss_id);
4169 
4170 			fp->err_rx_mp_null++;
4171 			if (mpf != NULL)
4172 				m_freem(mpf);
4173 			mpf = mpl = NULL;
4174 			rxq->tpa_info[agg_index].agg_state =
4175 						QLNX_AGG_STATE_ERROR;
4176 			ecore_chain_consume(&rxq->rx_bd_ring);
4177 			rxq->sw_rx_cons =
4178 				(rxq->sw_rx_cons + 1) & (RX_RING_SIZE - 1);
4179 			continue;
4180 		}
4181 
4182 		if (qlnx_alloc_rx_buffer(ha, rxq) != 0) {
4183 			QL_DPRINT7(ha, "[%d]: New buffer allocation failed,"
4184 				" dropping incoming packet and reusing its"
4185 				" buffer\n", fp->rss_id);
4186 
4187 			qlnx_reuse_rx_data(rxq);
4188 
4189 			if (mpf != NULL)
4190 				m_freem(mpf);
4191 			mpf = mpl = NULL;
4192 
4193 			rxq->tpa_info[agg_index].agg_state =
4194 						QLNX_AGG_STATE_ERROR;
4195 
4196 			ecore_chain_consume(&rxq->rx_bd_ring);
4197 			rxq->sw_rx_cons =
4198 				(rxq->sw_rx_cons + 1) & (RX_RING_SIZE - 1);
4199 
4200 			continue;
4201 		}
4202 
4203 		mpc->m_flags &= ~M_PKTHDR;
4204 		mpc->m_next = NULL;
4205 		mpc->m_len = cqe->len_list[i];
4206 
4207 
4208 		if (mpf == NULL) {
4209 			mpf = mpl = mpc;
4210 		} else {
4211 			mpl->m_len = ha->rx_buf_size;
4212 			mpl->m_next = mpc;
4213 			mpl = mpc;
4214 		}
4215 
4216 		ecore_chain_consume(&rxq->rx_bd_ring);
4217 		rxq->sw_rx_cons =
4218 			(rxq->sw_rx_cons + 1) & (RX_RING_SIZE - 1);
4219 	}
4220 
4221 	QL_DPRINT7(ha, "[%d]: 5\n ", fp->rss_id);
4222 
4223 	if (mpf != NULL) {
4224 
4225 		QL_DPRINT7(ha, "[%d]: 6\n ", fp->rss_id);
4226 
4227 		mp = rxq->tpa_info[agg_index].mpl;
4228 		mp->m_len = ha->rx_buf_size;
4229 		mp->m_next = mpf;
4230 	}
4231 
4232 	if (rxq->tpa_info[agg_index].agg_state != QLNX_AGG_STATE_START) {
4233 
4234 		QL_DPRINT7(ha, "[%d]: 7\n ", fp->rss_id);
4235 
4236 		if (rxq->tpa_info[agg_index].mpf != NULL)
4237 			m_freem(rxq->tpa_info[agg_index].mpf);
4238 		rxq->tpa_info[agg_index].mpf = NULL;
4239 		rxq->tpa_info[agg_index].mpl = NULL;
4240 		rxq->tpa_info[agg_index].agg_state = QLNX_AGG_STATE_NONE;
4241 		return (0);
4242 	}
4243 
4244 	mp = rxq->tpa_info[agg_index].mpf;
4245 	m_adj(mp, rxq->tpa_info[agg_index].placement_offset);
4246 	mp->m_pkthdr.len = cqe->total_packet_len;
4247 
4248 	if (mp->m_next  == NULL)
4249 		mp->m_len = mp->m_pkthdr.len;
4250 	else {
4251 		/* compute the total packet length */
4252 		mpf = mp;
4253 		while (mpf != NULL) {
4254 			len += mpf->m_len;
4255 			mpf = mpf->m_next;
4256 		}
4257 
4258 		if (cqe->total_packet_len > len) {
4259 			mpl = rxq->tpa_info[agg_index].mpl;
4260 			mpl->m_len += (cqe->total_packet_len - len);
4261 		}
4262 	}
4263 
4264 	QLNX_INC_IPACKETS(ifp);
4265 	QLNX_INC_IBYTES(ifp, (cqe->total_packet_len));
4266 
4267         QL_DPRINT7(ha, "[%d]: 8 csum_data = 0x%x csum_flags = 0x%lx\n \
4268 		m_len = 0x%x m_pkthdr_len = 0x%x\n",
4269                 fp->rss_id, mp->m_pkthdr.csum_data,
4270                 mp->m_pkthdr.csum_flags, mp->m_len, mp->m_pkthdr.len);
4271 
4272 	(*ifp->if_input)(ifp, mp);
4273 
4274 	rxq->tpa_info[agg_index].mpf = NULL;
4275 	rxq->tpa_info[agg_index].mpl = NULL;
4276 	rxq->tpa_info[agg_index].agg_state = QLNX_AGG_STATE_NONE;
4277 
4278 	return (cqe->num_of_coalesced_segs);
4279 }
4280 
4281 static int
4282 qlnx_rx_int(qlnx_host_t *ha, struct qlnx_fastpath *fp, int budget,
4283 	int lro_enable)
4284 {
4285         uint16_t		hw_comp_cons, sw_comp_cons;
4286         int			rx_pkt = 0;
4287         struct qlnx_rx_queue	*rxq = fp->rxq;
4288         struct ifnet		*ifp = ha->ifp;
4289 	struct ecore_dev	*cdev = &ha->cdev;
4290 	struct ecore_hwfn       *p_hwfn;
4291 
4292 #ifdef QLNX_SOFT_LRO
4293 	struct lro_ctrl		*lro;
4294 
4295 	lro = &rxq->lro;
4296 #endif /* #ifdef QLNX_SOFT_LRO */
4297 
4298         hw_comp_cons = le16toh(*rxq->hw_cons_ptr);
4299         sw_comp_cons = ecore_chain_get_cons_idx(&rxq->rx_comp_ring);
4300 
4301 	p_hwfn = &ha->cdev.hwfns[(fp->rss_id % cdev->num_hwfns)];
4302 
4303         /* Memory barrier to prevent the CPU from doing speculative reads of CQE
4304          * / BD in the while-loop before reading hw_comp_cons. If the CQE is
4305          * read before it is written by FW, then FW writes CQE and SB, and then
4306          * the CPU reads the hw_comp_cons, it will use an old CQE.
4307          */
4308 
4309         /* Loop to complete all indicated BDs */
4310         while (sw_comp_cons != hw_comp_cons) {
4311                 union eth_rx_cqe		*cqe;
4312                 struct eth_fast_path_rx_reg_cqe	*fp_cqe;
4313                 struct sw_rx_data		*sw_rx_data;
4314 		register struct mbuf		*mp;
4315                 enum eth_rx_cqe_type		cqe_type;
4316                 uint16_t			len, pad, len_on_first_bd;
4317                 uint8_t				*data;
4318 #if __FreeBSD_version >= 1100000
4319 		uint8_t				hash_type;
4320 #endif /* #if __FreeBSD_version >= 1100000 */
4321 
4322                 /* Get the CQE from the completion ring */
4323                 cqe = (union eth_rx_cqe *)
4324                         ecore_chain_consume(&rxq->rx_comp_ring);
4325                 cqe_type = cqe->fast_path_regular.type;
4326 
4327                 if (cqe_type == ETH_RX_CQE_TYPE_SLOW_PATH) {
4328                         QL_DPRINT3(ha, "Got a slowath CQE\n");
4329 
4330                         ecore_eth_cqe_completion(p_hwfn,
4331                                         (struct eth_slow_path_rx_cqe *)cqe);
4332                         goto next_cqe;
4333                 }
4334 
4335 		if (cqe_type != ETH_RX_CQE_TYPE_REGULAR) {
4336 
4337 			switch (cqe_type) {
4338 
4339 			case ETH_RX_CQE_TYPE_TPA_START:
4340 				qlnx_tpa_start(ha, fp, rxq,
4341 					&cqe->fast_path_tpa_start);
4342 				fp->tpa_start++;
4343 				break;
4344 
4345 			case ETH_RX_CQE_TYPE_TPA_CONT:
4346 				qlnx_tpa_cont(ha, fp, rxq,
4347 					&cqe->fast_path_tpa_cont);
4348 				fp->tpa_cont++;
4349 				break;
4350 
4351 			case ETH_RX_CQE_TYPE_TPA_END:
4352 				rx_pkt += qlnx_tpa_end(ha, fp, rxq,
4353 						&cqe->fast_path_tpa_end);
4354 				fp->tpa_end++;
4355 				break;
4356 
4357 			default:
4358 				break;
4359 			}
4360 
4361                         goto next_cqe;
4362 		}
4363 
4364                 /* Get the data from the SW ring */
4365                 sw_rx_data = &rxq->sw_rx_ring[rxq->sw_rx_cons];
4366                 mp = sw_rx_data->data;
4367 
4368 		if (mp == NULL) {
4369                 	QL_DPRINT1(ha, "mp = NULL\n");
4370 			fp->err_rx_mp_null++;
4371         		rxq->sw_rx_cons  =
4372 				(rxq->sw_rx_cons + 1) & (RX_RING_SIZE - 1);
4373 			goto next_cqe;
4374 		}
4375 		bus_dmamap_sync(ha->rx_tag, sw_rx_data->map,
4376 			BUS_DMASYNC_POSTREAD);
4377 
4378                 /* non GRO */
4379                 fp_cqe = &cqe->fast_path_regular;/* MK CR TPA check assembly */
4380                 len =  le16toh(fp_cqe->pkt_len);
4381                 pad = fp_cqe->placement_offset;
4382 
4383 		QL_DPRINT3(ha, "CQE type = %x, flags = %x, vlan = %x,"
4384 			" len %u, parsing flags = %d pad  = %d\n",
4385 			cqe_type, fp_cqe->bitfields,
4386 			le16toh(fp_cqe->vlan_tag),
4387 			len, le16toh(fp_cqe->pars_flags.flags), pad);
4388 
4389 		data = mtod(mp, uint8_t *);
4390 		data = data + pad;
4391 
4392 		if (0)
4393 			qlnx_dump_buf8(ha, __func__, data, len);
4394 
4395                 /* For every Rx BD consumed, we allocate a new BD so the BD ring
4396                  * is always with a fixed size. If allocation fails, we take the
4397                  * consumed BD and return it to the ring in the PROD position.
4398                  * The packet that was received on that BD will be dropped (and
4399                  * not passed to the upper stack).
4400                  */
4401 		/* If this is an error packet then drop it */
4402 		if ((le16toh(cqe->fast_path_regular.pars_flags.flags)) &
4403 			CQE_FLAGS_ERR) {
4404 
4405 			QL_DPRINT1(ha, "CQE in CONS = %u has error, flags = %x,"
4406 				" dropping incoming packet\n", sw_comp_cons,
4407 			le16toh(cqe->fast_path_regular.pars_flags.flags));
4408 			fp->err_rx_hw_errors++;
4409 
4410                         qlnx_reuse_rx_data(rxq);
4411 
4412 			QLNX_INC_IERRORS(ifp);
4413 
4414 			goto next_cqe;
4415 		}
4416 
4417                 if (qlnx_alloc_rx_buffer(ha, rxq) != 0) {
4418 
4419                         QL_DPRINT1(ha, "New buffer allocation failed, dropping"
4420 				" incoming packet and reusing its buffer\n");
4421                         qlnx_reuse_rx_data(rxq);
4422 
4423                         fp->err_rx_alloc_errors++;
4424 
4425 			QLNX_INC_IQDROPS(ifp);
4426 
4427                         goto next_cqe;
4428                 }
4429 
4430                 ecore_chain_consume(&rxq->rx_bd_ring);
4431 
4432 		len_on_first_bd = fp_cqe->len_on_first_bd;
4433 		m_adj(mp, pad);
4434 		mp->m_pkthdr.len = len;
4435 
4436 		QL_DPRINT1(ha, "len = %d len_on_first_bd = %d\n",
4437 			   len, len_on_first_bd);
4438 		if ((len > 60 ) && (len > len_on_first_bd)) {
4439 
4440 			mp->m_len = len_on_first_bd;
4441 
4442 			if (qlnx_rx_jumbo_chain(ha, fp, mp,
4443 				(len - len_on_first_bd)) != 0) {
4444 
4445 				m_freem(mp);
4446 
4447 				QLNX_INC_IQDROPS(ifp);
4448 
4449                         	goto next_cqe;
4450 			}
4451 
4452 		} else if (len_on_first_bd < len) {
4453 			fp->err_rx_jumbo_chain_pkts++;
4454 		} else {
4455 			mp->m_len = len;
4456 		}
4457 
4458 		mp->m_flags |= M_PKTHDR;
4459 
4460 		/* assign packet to this interface interface */
4461 		mp->m_pkthdr.rcvif = ifp;
4462 
4463 		/* assume no hardware checksum has complated */
4464 		mp->m_pkthdr.csum_flags = 0;
4465 
4466 		mp->m_pkthdr.flowid = fp_cqe->rss_hash;
4467 
4468 #if __FreeBSD_version >= 1100000
4469 
4470 		hash_type = fp_cqe->bitfields &
4471 				(ETH_FAST_PATH_RX_REG_CQE_RSS_HASH_TYPE_MASK <<
4472 				ETH_FAST_PATH_RX_REG_CQE_RSS_HASH_TYPE_SHIFT);
4473 
4474 		switch (hash_type) {
4475 
4476 		case RSS_HASH_TYPE_IPV4:
4477 			M_HASHTYPE_SET(mp, M_HASHTYPE_RSS_IPV4);
4478 			break;
4479 
4480 		case RSS_HASH_TYPE_TCP_IPV4:
4481 			M_HASHTYPE_SET(mp, M_HASHTYPE_RSS_TCP_IPV4);
4482 			break;
4483 
4484 		case RSS_HASH_TYPE_IPV6:
4485 			M_HASHTYPE_SET(mp, M_HASHTYPE_RSS_IPV6);
4486 			break;
4487 
4488 		case RSS_HASH_TYPE_TCP_IPV6:
4489 			M_HASHTYPE_SET(mp, M_HASHTYPE_RSS_TCP_IPV6);
4490 			break;
4491 
4492 		default:
4493 			M_HASHTYPE_SET(mp, M_HASHTYPE_OPAQUE);
4494 			break;
4495 		}
4496 
4497 #else
4498 		mp->m_flags |= M_FLOWID;
4499 #endif
4500 
4501 		if (CQE_L3_PACKET(fp_cqe->pars_flags.flags)) {
4502 			mp->m_pkthdr.csum_flags |= CSUM_IP_CHECKED;
4503 		}
4504 
4505 		if (!(CQE_IP_HDR_ERR(fp_cqe->pars_flags.flags))) {
4506 			mp->m_pkthdr.csum_flags |= CSUM_IP_VALID;
4507 		}
4508 
4509 		if (CQE_L4_HAS_CSUM(fp_cqe->pars_flags.flags)) {
4510 			mp->m_pkthdr.csum_data = 0xFFFF;
4511 			mp->m_pkthdr.csum_flags |=
4512 				(CSUM_DATA_VALID | CSUM_PSEUDO_HDR);
4513 		}
4514 
4515 		if (CQE_HAS_VLAN(fp_cqe->pars_flags.flags)) {
4516 			mp->m_pkthdr.ether_vtag = le16toh(fp_cqe->vlan_tag);
4517 			mp->m_flags |= M_VLANTAG;
4518 		}
4519 
4520 		QLNX_INC_IPACKETS(ifp);
4521 		QLNX_INC_IBYTES(ifp, len);
4522 
4523 #ifdef QLNX_SOFT_LRO
4524 
4525 		if (lro_enable) {
4526 
4527 #if (__FreeBSD_version >= 1100101) || (defined QLNX_QSORT_LRO)
4528 
4529 			tcp_lro_queue_mbuf(lro, mp);
4530 
4531 #else
4532 
4533 			if (tcp_lro_rx(lro, mp, 0))
4534 				(*ifp->if_input)(ifp, mp);
4535 
4536 #endif /* #if (__FreeBSD_version >= 1100101) || (defined QLNX_QSORT_LRO) */
4537 
4538 		} else {
4539 			(*ifp->if_input)(ifp, mp);
4540 		}
4541 #else
4542 
4543 		(*ifp->if_input)(ifp, mp);
4544 
4545 #endif /* #ifdef QLNX_SOFT_LRO */
4546 
4547                 rx_pkt++;
4548 
4549         	rxq->sw_rx_cons  = (rxq->sw_rx_cons + 1) & (RX_RING_SIZE - 1);
4550 
4551 next_cqe:	/* don't consume bd rx buffer */
4552                 ecore_chain_recycle_consumed(&rxq->rx_comp_ring);
4553                 sw_comp_cons = ecore_chain_get_cons_idx(&rxq->rx_comp_ring);
4554 
4555 		/* CR TPA - revisit how to handle budget in TPA perhaps
4556 		   increase on "end" */
4557                 if (rx_pkt == budget)
4558                         break;
4559         } /* repeat while sw_comp_cons != hw_comp_cons... */
4560 
4561         /* Update producers */
4562         qlnx_update_rx_prod(p_hwfn, rxq);
4563 
4564         return rx_pkt;
4565 }
4566 
4567 
4568 /*
4569  * fast path interrupt
4570  */
4571 
4572 static void
4573 qlnx_fp_isr(void *arg)
4574 {
4575         qlnx_ivec_t		*ivec = arg;
4576         qlnx_host_t		*ha;
4577         struct qlnx_fastpath	*fp = NULL;
4578         int			idx;
4579 
4580         ha = ivec->ha;
4581 
4582         if (ha->state != QLNX_STATE_OPEN) {
4583                 return;
4584         }
4585 
4586         idx = ivec->rss_idx;
4587 
4588         if ((idx = ivec->rss_idx) >= ha->num_rss) {
4589                 QL_DPRINT1(ha, "illegal interrupt[%d]\n", idx);
4590                 ha->err_illegal_intr++;
4591                 return;
4592         }
4593         fp = &ha->fp_array[idx];
4594 
4595         if (fp == NULL) {
4596                 ha->err_fp_null++;
4597         } else {
4598 
4599 #ifdef QLNX_RCV_IN_TASKQ
4600                 ecore_sb_ack(fp->sb_info, IGU_INT_DISABLE, 0);
4601 		if (fp->fp_taskqueue != NULL)
4602 			taskqueue_enqueue(fp->fp_taskqueue, &fp->fp_task);
4603 #else
4604 		int			rx_int = 0, total_rx_count = 0;
4605 		int 			lro_enable, tc;
4606 		struct qlnx_tx_queue	*txq;
4607 		uint16_t		elem_left;
4608 
4609 		lro_enable = ha->ifp->if_capenable & IFCAP_LRO;
4610 
4611                 ecore_sb_ack(fp->sb_info, IGU_INT_DISABLE, 0);
4612 
4613                 do {
4614                         for (tc = 0; tc < ha->num_tc; tc++) {
4615 
4616 				txq = fp->txq[tc];
4617 
4618 				if((int)(elem_left =
4619 					ecore_chain_get_elem_left(&txq->tx_pbl)) <
4620 						QLNX_TX_ELEM_THRESH)  {
4621 
4622                                 	if (mtx_trylock(&fp->tx_mtx)) {
4623 #ifdef QLNX_TRACE_PERF_DATA
4624 						tx_compl = fp->tx_pkts_completed;
4625 #endif
4626 
4627 						qlnx_tx_int(ha, fp, fp->txq[tc]);
4628 #ifdef QLNX_TRACE_PERF_DATA
4629 						fp->tx_pkts_compl_intr +=
4630 							(fp->tx_pkts_completed - tx_compl);
4631 						if ((fp->tx_pkts_completed - tx_compl) <= 32)
4632 							fp->tx_comInt[0]++;
4633 						else if (((fp->tx_pkts_completed - tx_compl) > 32) &&
4634 							((fp->tx_pkts_completed - tx_compl) <= 64))
4635 							fp->tx_comInt[1]++;
4636 						else if(((fp->tx_pkts_completed - tx_compl) > 64) &&
4637 							((fp->tx_pkts_completed - tx_compl) <= 128))
4638 							fp->tx_comInt[2]++;
4639 						else if(((fp->tx_pkts_completed - tx_compl) > 128))
4640 							fp->tx_comInt[3]++;
4641 #endif
4642 						mtx_unlock(&fp->tx_mtx);
4643 					}
4644 				}
4645                         }
4646 
4647                         rx_int = qlnx_rx_int(ha, fp, ha->rx_pkt_threshold,
4648                                         lro_enable);
4649 
4650                         if (rx_int) {
4651                                 fp->rx_pkts += rx_int;
4652                                 total_rx_count += rx_int;
4653                         }
4654 
4655                 } while (rx_int);
4656 
4657 #ifdef QLNX_SOFT_LRO
4658                 {
4659                         struct lro_ctrl *lro;
4660 
4661                         lro = &fp->rxq->lro;
4662 
4663                         if (lro_enable && total_rx_count) {
4664 
4665 #if (__FreeBSD_version >= 1100101) || (defined QLNX_QSORT_LRO)
4666 
4667 #ifdef QLNX_TRACE_LRO_CNT
4668                                 if (lro->lro_mbuf_count & ~1023)
4669                                         fp->lro_cnt_1024++;
4670                                 else if (lro->lro_mbuf_count & ~511)
4671                                         fp->lro_cnt_512++;
4672                                 else if (lro->lro_mbuf_count & ~255)
4673                                         fp->lro_cnt_256++;
4674                                 else if (lro->lro_mbuf_count & ~127)
4675                                         fp->lro_cnt_128++;
4676                                 else if (lro->lro_mbuf_count & ~63)
4677                                         fp->lro_cnt_64++;
4678 #endif /* #ifdef QLNX_TRACE_LRO_CNT */
4679 
4680                                 tcp_lro_flush_all(lro);
4681 
4682 #else
4683                                 struct lro_entry *queued;
4684 
4685                                 while ((!SLIST_EMPTY(&lro->lro_active))) {
4686                                         queued = SLIST_FIRST(&lro->lro_active);
4687                                         SLIST_REMOVE_HEAD(&lro->lro_active, \
4688                                                 next);
4689                                         tcp_lro_flush(lro, queued);
4690                                 }
4691 #endif /* #if (__FreeBSD_version >= 1100101) || (defined QLNX_QSORT_LRO) */
4692                         }
4693                 }
4694 #endif /* #ifdef QLNX_SOFT_LRO */
4695 
4696                 ecore_sb_update_sb_idx(fp->sb_info);
4697                 rmb();
4698                 ecore_sb_ack(fp->sb_info, IGU_INT_ENABLE, 1);
4699 
4700 #endif /* #ifdef QLNX_RCV_IN_TASKQ */
4701         }
4702 
4703         return;
4704 }
4705 
4706 
4707 /*
4708  * slow path interrupt processing function
4709  * can be invoked in polled mode or in interrupt mode via taskqueue.
4710  */
4711 void
4712 qlnx_sp_isr(void *arg)
4713 {
4714 	struct ecore_hwfn	*p_hwfn;
4715 	qlnx_host_t		*ha;
4716 
4717 	p_hwfn = arg;
4718 
4719 	ha = (qlnx_host_t *)p_hwfn->p_dev;
4720 
4721 	ha->sp_interrupts++;
4722 
4723 	QL_DPRINT2(ha, "enter\n");
4724 
4725 	ecore_int_sp_dpc(p_hwfn);
4726 
4727 	QL_DPRINT2(ha, "exit\n");
4728 
4729 	return;
4730 }
4731 
4732 /*****************************************************************************
4733  * Support Functions for DMA'able Memory
4734  *****************************************************************************/
4735 
4736 static void
4737 qlnx_dmamap_callback(void *arg, bus_dma_segment_t *segs, int nsegs, int error)
4738 {
4739         *((bus_addr_t *)arg) = 0;
4740 
4741         if (error) {
4742                 printf("%s: bus_dmamap_load failed (%d)\n", __func__, error);
4743                 return;
4744         }
4745 
4746         *((bus_addr_t *)arg) = segs[0].ds_addr;
4747 
4748         return;
4749 }
4750 
4751 static int
4752 qlnx_alloc_dmabuf(qlnx_host_t *ha, qlnx_dma_t *dma_buf)
4753 {
4754         int             ret = 0;
4755         device_t        dev;
4756         bus_addr_t      b_addr;
4757 
4758         dev = ha->pci_dev;
4759 
4760         ret = bus_dma_tag_create(
4761                         ha->parent_tag,/* parent */
4762                         dma_buf->alignment,
4763                         ((bus_size_t)(1ULL << 32)),/* boundary */
4764                         BUS_SPACE_MAXADDR,      /* lowaddr */
4765                         BUS_SPACE_MAXADDR,      /* highaddr */
4766                         NULL, NULL,             /* filter, filterarg */
4767                         dma_buf->size,          /* maxsize */
4768                         1,                      /* nsegments */
4769                         dma_buf->size,          /* maxsegsize */
4770                         0,                      /* flags */
4771                         NULL, NULL,             /* lockfunc, lockarg */
4772                         &dma_buf->dma_tag);
4773 
4774         if (ret) {
4775                 QL_DPRINT1(ha, "could not create dma tag\n");
4776                 goto qlnx_alloc_dmabuf_exit;
4777         }
4778         ret = bus_dmamem_alloc(dma_buf->dma_tag,
4779                         (void **)&dma_buf->dma_b,
4780                         (BUS_DMA_ZERO | BUS_DMA_COHERENT | BUS_DMA_NOWAIT),
4781                         &dma_buf->dma_map);
4782         if (ret) {
4783                 bus_dma_tag_destroy(dma_buf->dma_tag);
4784                 QL_DPRINT1(ha, "bus_dmamem_alloc failed\n");
4785                 goto qlnx_alloc_dmabuf_exit;
4786         }
4787 
4788         ret = bus_dmamap_load(dma_buf->dma_tag,
4789                         dma_buf->dma_map,
4790                         dma_buf->dma_b,
4791                         dma_buf->size,
4792                         qlnx_dmamap_callback,
4793                         &b_addr, BUS_DMA_NOWAIT);
4794 
4795         if (ret || !b_addr) {
4796                 bus_dma_tag_destroy(dma_buf->dma_tag);
4797                 bus_dmamem_free(dma_buf->dma_tag, dma_buf->dma_b,
4798                         dma_buf->dma_map);
4799                 ret = -1;
4800                 goto qlnx_alloc_dmabuf_exit;
4801         }
4802 
4803         dma_buf->dma_addr = b_addr;
4804 
4805 qlnx_alloc_dmabuf_exit:
4806 
4807         return ret;
4808 }
4809 
4810 static void
4811 qlnx_free_dmabuf(qlnx_host_t *ha, qlnx_dma_t *dma_buf)
4812 {
4813 	bus_dmamap_unload(dma_buf->dma_tag, dma_buf->dma_map);
4814         bus_dmamem_free(dma_buf->dma_tag, dma_buf->dma_b, dma_buf->dma_map);
4815         bus_dma_tag_destroy(dma_buf->dma_tag);
4816 	return;
4817 }
4818 
4819 void *
4820 qlnx_dma_alloc_coherent(void *ecore_dev, bus_addr_t *phys, uint32_t size)
4821 {
4822 	qlnx_dma_t	dma_buf;
4823 	qlnx_dma_t	*dma_p;
4824 	qlnx_host_t	*ha;
4825 	device_t        dev;
4826 
4827 	ha = (qlnx_host_t *)ecore_dev;
4828 	dev = ha->pci_dev;
4829 
4830 	size = (size + (PAGE_SIZE - 1)) & ~(PAGE_SIZE - 1);
4831 
4832 	memset(&dma_buf, 0, sizeof (qlnx_dma_t));
4833 
4834 	dma_buf.size = size + PAGE_SIZE;
4835 	dma_buf.alignment = 8;
4836 
4837 	if (qlnx_alloc_dmabuf((qlnx_host_t *)ecore_dev, &dma_buf) != 0)
4838 		return (NULL);
4839 	bzero((uint8_t *)dma_buf.dma_b, dma_buf.size);
4840 
4841 	*phys = dma_buf.dma_addr;
4842 
4843 	dma_p = (qlnx_dma_t *)((uint8_t *)dma_buf.dma_b + size);
4844 
4845 	memcpy(dma_p, &dma_buf, sizeof(qlnx_dma_t));
4846 /*
4847 	QL_DPRINT5(ha, "[%p %p %p %p 0x%08x ]\n",
4848 		(void *)dma_buf.dma_map, (void *)dma_buf.dma_tag,
4849 		dma_buf.dma_b, (void *)dma_buf.dma_addr, size);
4850 */
4851 	return (dma_buf.dma_b);
4852 }
4853 
4854 void
4855 qlnx_dma_free_coherent(void *ecore_dev, void *v_addr, bus_addr_t phys,
4856 	uint32_t size)
4857 {
4858 	qlnx_dma_t dma_buf, *dma_p;
4859 	qlnx_host_t	*ha;
4860 	device_t        dev;
4861 
4862 	ha = (qlnx_host_t *)ecore_dev;
4863 	dev = ha->pci_dev;
4864 
4865 	if (v_addr == NULL)
4866 		return;
4867 
4868 	size = (size + (PAGE_SIZE - 1)) & ~(PAGE_SIZE - 1);
4869 
4870 	dma_p = (qlnx_dma_t *)((uint8_t *)v_addr + size);
4871 /*
4872 	QL_DPRINT5(ha, "[%p %p %p %p 0x%08x ]\n",
4873 		(void *)dma_p->dma_map, (void *)dma_p->dma_tag,
4874 		dma_p->dma_b, (void *)dma_p->dma_addr, size);
4875 */
4876 	dma_buf = *dma_p;
4877 
4878 	qlnx_free_dmabuf((qlnx_host_t *)ecore_dev, &dma_buf);
4879 	return;
4880 }
4881 
4882 static int
4883 qlnx_alloc_parent_dma_tag(qlnx_host_t *ha)
4884 {
4885         int             ret;
4886         device_t        dev;
4887 
4888         dev = ha->pci_dev;
4889 
4890         /*
4891          * Allocate parent DMA Tag
4892          */
4893         ret = bus_dma_tag_create(
4894                         bus_get_dma_tag(dev),   /* parent */
4895                         1,((bus_size_t)(1ULL << 32)),/* alignment, boundary */
4896                         BUS_SPACE_MAXADDR,      /* lowaddr */
4897                         BUS_SPACE_MAXADDR,      /* highaddr */
4898                         NULL, NULL,             /* filter, filterarg */
4899                         BUS_SPACE_MAXSIZE_32BIT,/* maxsize */
4900                         0,                      /* nsegments */
4901                         BUS_SPACE_MAXSIZE_32BIT,/* maxsegsize */
4902                         0,                      /* flags */
4903                         NULL, NULL,             /* lockfunc, lockarg */
4904                         &ha->parent_tag);
4905 
4906         if (ret) {
4907                 QL_DPRINT1(ha, "could not create parent dma tag\n");
4908                 return (-1);
4909         }
4910 
4911         ha->flags.parent_tag = 1;
4912 
4913         return (0);
4914 }
4915 
4916 static void
4917 qlnx_free_parent_dma_tag(qlnx_host_t *ha)
4918 {
4919         if (ha->parent_tag != NULL) {
4920                 bus_dma_tag_destroy(ha->parent_tag);
4921 		ha->parent_tag = NULL;
4922         }
4923 	return;
4924 }
4925 
4926 static int
4927 qlnx_alloc_tx_dma_tag(qlnx_host_t *ha)
4928 {
4929         if (bus_dma_tag_create(NULL,    /* parent */
4930                 1, 0,    /* alignment, bounds */
4931                 BUS_SPACE_MAXADDR,       /* lowaddr */
4932                 BUS_SPACE_MAXADDR,       /* highaddr */
4933                 NULL, NULL,      /* filter, filterarg */
4934                 QLNX_MAX_TSO_FRAME_SIZE,     /* maxsize */
4935                 QLNX_MAX_SEGMENTS,        /* nsegments */
4936                 QLNX_MAX_TX_MBUF_SIZE,	  /* maxsegsize */
4937                 0,        /* flags */
4938                 NULL,    /* lockfunc */
4939                 NULL,    /* lockfuncarg */
4940                 &ha->tx_tag)) {
4941 
4942                 QL_DPRINT1(ha, "tx_tag alloc failed\n");
4943                 return (-1);
4944         }
4945 
4946 	return (0);
4947 }
4948 
4949 static void
4950 qlnx_free_tx_dma_tag(qlnx_host_t *ha)
4951 {
4952         if (ha->tx_tag != NULL) {
4953                 bus_dma_tag_destroy(ha->tx_tag);
4954 		ha->tx_tag = NULL;
4955         }
4956 	return;
4957 }
4958 
4959 static int
4960 qlnx_alloc_rx_dma_tag(qlnx_host_t *ha)
4961 {
4962         if (bus_dma_tag_create(NULL,    /* parent */
4963                         1, 0,    /* alignment, bounds */
4964                         BUS_SPACE_MAXADDR,       /* lowaddr */
4965                         BUS_SPACE_MAXADDR,       /* highaddr */
4966                         NULL, NULL,      /* filter, filterarg */
4967                         MJUM9BYTES,     /* maxsize */
4968                         1,        /* nsegments */
4969                         MJUM9BYTES,        /* maxsegsize */
4970                         0,        /* flags */
4971                         NULL,    /* lockfunc */
4972                         NULL,    /* lockfuncarg */
4973                         &ha->rx_tag)) {
4974 
4975                 QL_DPRINT1(ha, " rx_tag alloc failed\n");
4976 
4977                 return (-1);
4978         }
4979 	return (0);
4980 }
4981 
4982 static void
4983 qlnx_free_rx_dma_tag(qlnx_host_t *ha)
4984 {
4985         if (ha->rx_tag != NULL) {
4986                 bus_dma_tag_destroy(ha->rx_tag);
4987 		ha->rx_tag = NULL;
4988         }
4989 	return;
4990 }
4991 
4992 /*********************************
4993  * Exported functions
4994  *********************************/
4995 uint32_t
4996 qlnx_pci_bus_get_bar_size(void *ecore_dev, uint8_t bar_id)
4997 {
4998 	uint32_t bar_size;
4999 
5000 	bar_id = bar_id * 2;
5001 
5002 	bar_size = bus_get_resource_count(((qlnx_host_t *)ecore_dev)->pci_dev,
5003 				SYS_RES_MEMORY,
5004 				PCIR_BAR(bar_id));
5005 
5006 	return (bar_size);
5007 }
5008 
5009 uint32_t
5010 qlnx_pci_read_config_byte(void *ecore_dev, uint32_t pci_reg, uint8_t *reg_value)
5011 {
5012 	*reg_value = pci_read_config(((qlnx_host_t *)ecore_dev)->pci_dev,
5013 				pci_reg, 1);
5014 	return 0;
5015 }
5016 
5017 uint32_t
5018 qlnx_pci_read_config_word(void *ecore_dev, uint32_t pci_reg,
5019 	uint16_t *reg_value)
5020 {
5021 	*reg_value = pci_read_config(((qlnx_host_t *)ecore_dev)->pci_dev,
5022 				pci_reg, 2);
5023 	return 0;
5024 }
5025 
5026 uint32_t
5027 qlnx_pci_read_config_dword(void *ecore_dev, uint32_t pci_reg,
5028 	uint32_t *reg_value)
5029 {
5030 	*reg_value = pci_read_config(((qlnx_host_t *)ecore_dev)->pci_dev,
5031 				pci_reg, 4);
5032 	return 0;
5033 }
5034 
5035 void
5036 qlnx_pci_write_config_byte(void *ecore_dev, uint32_t pci_reg, uint8_t reg_value)
5037 {
5038 	pci_write_config(((qlnx_host_t *)ecore_dev)->pci_dev,
5039 		pci_reg, reg_value, 1);
5040 	return;
5041 }
5042 
5043 void
5044 qlnx_pci_write_config_word(void *ecore_dev, uint32_t pci_reg,
5045 	uint16_t reg_value)
5046 {
5047 	pci_write_config(((qlnx_host_t *)ecore_dev)->pci_dev,
5048 		pci_reg, reg_value, 2);
5049 	return;
5050 }
5051 
5052 void
5053 qlnx_pci_write_config_dword(void *ecore_dev, uint32_t pci_reg,
5054 	uint32_t reg_value)
5055 {
5056 	pci_write_config(((qlnx_host_t *)ecore_dev)->pci_dev,
5057 		pci_reg, reg_value, 4);
5058 	return;
5059 }
5060 
5061 
5062 int
5063 qlnx_pci_find_capability(void *ecore_dev, int cap)
5064 {
5065 	int		reg;
5066 	qlnx_host_t	*ha;
5067 
5068 	ha = ecore_dev;
5069 
5070 	if (pci_find_cap(ha->pci_dev, PCIY_EXPRESS, &reg) == 0)
5071 		return reg;
5072 	else {
5073 		QL_DPRINT1(ha, "failed\n");
5074 		return 0;
5075 	}
5076 }
5077 
5078 uint32_t
5079 qlnx_reg_rd32(void *hwfn, uint32_t reg_addr)
5080 {
5081 	uint32_t		data32;
5082 	struct ecore_dev	*cdev;
5083 	struct ecore_hwfn	*p_hwfn;
5084 
5085 	p_hwfn = hwfn;
5086 
5087 	cdev = p_hwfn->p_dev;
5088 
5089 	reg_addr = (uint32_t)((uint8_t *)(p_hwfn->regview) -
5090 			(uint8_t *)(cdev->regview)) + reg_addr;
5091 
5092 	data32 = bus_read_4(((qlnx_host_t *)cdev)->pci_reg, reg_addr);
5093 
5094 	return (data32);
5095 }
5096 
5097 void
5098 qlnx_reg_wr32(void *hwfn, uint32_t reg_addr, uint32_t value)
5099 {
5100 	struct ecore_dev	*cdev;
5101 	struct ecore_hwfn	*p_hwfn;
5102 
5103 	p_hwfn = hwfn;
5104 
5105 	cdev = p_hwfn->p_dev;
5106 
5107 	reg_addr = (uint32_t)((uint8_t *)(p_hwfn->regview) -
5108 			(uint8_t *)(cdev->regview)) + reg_addr;
5109 
5110 	bus_write_4(((qlnx_host_t *)cdev)->pci_reg, reg_addr, value);
5111 
5112 	return;
5113 }
5114 
5115 void
5116 qlnx_reg_wr16(void *hwfn, uint32_t reg_addr, uint16_t value)
5117 {
5118 	struct ecore_dev	*cdev;
5119 	struct ecore_hwfn	*p_hwfn;
5120 
5121 	p_hwfn = hwfn;
5122 
5123 	cdev = p_hwfn->p_dev;
5124 
5125 	reg_addr = (uint32_t)((uint8_t *)(p_hwfn->regview) -
5126 			(uint8_t *)(cdev->regview)) + reg_addr;
5127 
5128 	bus_write_2(((qlnx_host_t *)cdev)->pci_reg, reg_addr, value);
5129 
5130 	return;
5131 }
5132 
5133 void
5134 qlnx_dbell_wr32(void *hwfn, uint32_t reg_addr, uint32_t value)
5135 {
5136 	struct ecore_dev	*cdev;
5137 	struct ecore_hwfn	*p_hwfn;
5138 
5139 	p_hwfn = hwfn;
5140 
5141 	cdev = p_hwfn->p_dev;
5142 
5143 	reg_addr = (uint32_t)((uint8_t *)(p_hwfn->doorbells) -
5144 			(uint8_t *)(cdev->doorbells)) + reg_addr;
5145 
5146 	bus_write_4(((qlnx_host_t *)cdev)->pci_dbells, reg_addr, value);
5147 
5148 	return;
5149 }
5150 
5151 uint32_t
5152 qlnx_direct_reg_rd32(void *p_hwfn, uint32_t *reg_addr)
5153 {
5154 	uint32_t		data32;
5155 	uint32_t		offset;
5156 	struct ecore_dev	*cdev;
5157 
5158 	cdev = ((struct ecore_hwfn *)p_hwfn)->p_dev;
5159 	offset = (uint32_t)((uint8_t *)reg_addr - (uint8_t *)(cdev->regview));
5160 
5161 	data32 = bus_read_4(((qlnx_host_t *)cdev)->pci_reg, offset);
5162 
5163 	return (data32);
5164 }
5165 
5166 void
5167 qlnx_direct_reg_wr32(void *p_hwfn, void *reg_addr, uint32_t value)
5168 {
5169 	uint32_t		offset;
5170 	struct ecore_dev	*cdev;
5171 
5172 	cdev = ((struct ecore_hwfn *)p_hwfn)->p_dev;
5173 	offset = (uint32_t)((uint8_t *)reg_addr - (uint8_t *)(cdev->regview));
5174 
5175 	bus_write_4(((qlnx_host_t *)cdev)->pci_reg, offset, value);
5176 
5177 	return;
5178 }
5179 
5180 void
5181 qlnx_direct_reg_wr64(void *p_hwfn, void *reg_addr, uint64_t value)
5182 {
5183 	uint32_t		offset;
5184 	struct ecore_dev	*cdev;
5185 
5186 	cdev = ((struct ecore_hwfn *)p_hwfn)->p_dev;
5187 	offset = (uint32_t)((uint8_t *)reg_addr - (uint8_t *)(cdev->regview));
5188 
5189 	bus_write_8(((qlnx_host_t *)cdev)->pci_reg, offset, value);
5190 	return;
5191 }
5192 
5193 void *
5194 qlnx_zalloc(uint32_t size)
5195 {
5196 	caddr_t	va;
5197 
5198 	va = malloc((unsigned long)size, M_QLNXBUF, M_NOWAIT);
5199 	bzero(va, size);
5200 	return ((void *)va);
5201 }
5202 
5203 void
5204 qlnx_barrier(void *p_hwfn)
5205 {
5206 	qlnx_host_t	*ha;
5207 
5208 	ha = (qlnx_host_t *)((struct ecore_hwfn *)p_hwfn)->p_dev;
5209 	bus_barrier(ha->pci_reg,  0, 0, BUS_SPACE_BARRIER_WRITE);
5210 }
5211 
5212 void
5213 qlnx_link_update(void *p_hwfn)
5214 {
5215 	qlnx_host_t	*ha;
5216 	int		prev_link_state;
5217 
5218 	ha = (qlnx_host_t *)((struct ecore_hwfn *)p_hwfn)->p_dev;
5219 
5220 	qlnx_fill_link(p_hwfn, &ha->if_link);
5221 
5222 	prev_link_state = ha->link_up;
5223 	ha->link_up = ha->if_link.link_up;
5224 
5225         if (prev_link_state !=  ha->link_up) {
5226                 if (ha->link_up) {
5227                         if_link_state_change(ha->ifp, LINK_STATE_UP);
5228                 } else {
5229                         if_link_state_change(ha->ifp, LINK_STATE_DOWN);
5230                 }
5231         }
5232         return;
5233 }
5234 
5235 void
5236 qlnx_fill_link(struct ecore_hwfn *hwfn, struct qlnx_link_output *if_link)
5237 {
5238 	struct ecore_mcp_link_params    link_params;
5239 	struct ecore_mcp_link_state     link_state;
5240 
5241 	memset(if_link, 0, sizeof(*if_link));
5242 	memset(&link_params, 0, sizeof(struct ecore_mcp_link_params));
5243 	memset(&link_state, 0, sizeof(struct ecore_mcp_link_state));
5244 
5245 	/* Prepare source inputs */
5246 	/* we only deal with physical functions */
5247 	memcpy(&link_params, ecore_mcp_get_link_params(hwfn),
5248 		sizeof(link_params));
5249 	memcpy(&link_state, ecore_mcp_get_link_state(hwfn),
5250 		sizeof(link_state));
5251 
5252 	ecore_mcp_get_media_type(hwfn->p_dev, &if_link->media_type);
5253 
5254 	/* Set the link parameters to pass to protocol driver */
5255 	if (link_state.link_up) {
5256 		if_link->link_up = true;
5257 		if_link->speed = link_state.speed;
5258 	}
5259 
5260 	if_link->supported_caps = QLNX_LINK_CAP_FIBRE;
5261 
5262 	if (link_params.speed.autoneg)
5263 		if_link->supported_caps |= QLNX_LINK_CAP_Autoneg;
5264 
5265 	if (link_params.pause.autoneg ||
5266 		(link_params.pause.forced_rx && link_params.pause.forced_tx))
5267 		if_link->supported_caps |= QLNX_LINK_CAP_Asym_Pause;
5268 
5269 	if (link_params.pause.autoneg || link_params.pause.forced_rx ||
5270 		link_params.pause.forced_tx)
5271 		if_link->supported_caps |= QLNX_LINK_CAP_Pause;
5272 
5273 	if (link_params.speed.advertised_speeds &
5274 		NVM_CFG1_PORT_DRV_SPEED_CAPABILITY_MASK_1G)
5275 		if_link->supported_caps |= QLNX_LINK_CAP_1000baseT_Half |
5276                                            QLNX_LINK_CAP_1000baseT_Full;
5277 
5278 	if (link_params.speed.advertised_speeds &
5279 		NVM_CFG1_PORT_DRV_SPEED_CAPABILITY_MASK_10G)
5280 		if_link->supported_caps |= QLNX_LINK_CAP_10000baseKR_Full;
5281 
5282 	if (link_params.speed.advertised_speeds &
5283 		NVM_CFG1_PORT_DRV_SPEED_CAPABILITY_MASK_25G)
5284 		if_link->supported_caps |= QLNX_LINK_CAP_25000baseKR_Full;
5285 
5286 	if (link_params.speed.advertised_speeds &
5287 		NVM_CFG1_PORT_DRV_LINK_SPEED_40G)
5288 		if_link->supported_caps |= QLNX_LINK_CAP_40000baseLR4_Full;
5289 
5290 	if (link_params.speed.advertised_speeds &
5291 		NVM_CFG1_PORT_DRV_SPEED_CAPABILITY_MASK_50G)
5292 		if_link->supported_caps |= QLNX_LINK_CAP_50000baseKR2_Full;
5293 
5294 	if (link_params.speed.advertised_speeds &
5295 		NVM_CFG1_PORT_DRV_SPEED_CAPABILITY_MASK_BB_100G)
5296 		if_link->supported_caps |= QLNX_LINK_CAP_100000baseKR4_Full;
5297 
5298 	if_link->advertised_caps = if_link->supported_caps;
5299 
5300 	if_link->autoneg = link_params.speed.autoneg;
5301 	if_link->duplex = QLNX_LINK_DUPLEX;
5302 
5303 	/* Link partner capabilities */
5304 
5305 	if (link_state.partner_adv_speed & ECORE_LINK_PARTNER_SPEED_1G_HD)
5306 		if_link->link_partner_caps |= QLNX_LINK_CAP_1000baseT_Half;
5307 
5308 	if (link_state.partner_adv_speed & ECORE_LINK_PARTNER_SPEED_1G_FD)
5309 		if_link->link_partner_caps |= QLNX_LINK_CAP_1000baseT_Full;
5310 
5311 	if (link_state.partner_adv_speed & ECORE_LINK_PARTNER_SPEED_10G)
5312 		if_link->link_partner_caps |= QLNX_LINK_CAP_10000baseKR_Full;
5313 
5314 	if (link_state.partner_adv_speed & ECORE_LINK_PARTNER_SPEED_25G)
5315 		if_link->link_partner_caps |= QLNX_LINK_CAP_25000baseKR_Full;
5316 
5317 	if (link_state.partner_adv_speed & ECORE_LINK_PARTNER_SPEED_40G)
5318 		if_link->link_partner_caps |= QLNX_LINK_CAP_40000baseLR4_Full;
5319 
5320 	if (link_state.partner_adv_speed & ECORE_LINK_PARTNER_SPEED_50G)
5321 		if_link->link_partner_caps |= QLNX_LINK_CAP_50000baseKR2_Full;
5322 
5323 	if (link_state.partner_adv_speed & ECORE_LINK_PARTNER_SPEED_100G)
5324 		if_link->link_partner_caps |= QLNX_LINK_CAP_100000baseKR4_Full;
5325 
5326 	if (link_state.an_complete)
5327 		if_link->link_partner_caps |= QLNX_LINK_CAP_Autoneg;
5328 
5329 	if (link_state.partner_adv_pause)
5330 		if_link->link_partner_caps |= QLNX_LINK_CAP_Pause;
5331 
5332 	if ((link_state.partner_adv_pause ==
5333 		ECORE_LINK_PARTNER_ASYMMETRIC_PAUSE) ||
5334 		(link_state.partner_adv_pause ==
5335 			ECORE_LINK_PARTNER_BOTH_PAUSE))
5336 		if_link->link_partner_caps |= QLNX_LINK_CAP_Asym_Pause;
5337 
5338 	return;
5339 }
5340 
5341 static int
5342 qlnx_nic_setup(struct ecore_dev *cdev, struct ecore_pf_params *func_params)
5343 {
5344         int	rc, i;
5345 
5346         for (i = 0; i < cdev->num_hwfns; i++) {
5347                 struct ecore_hwfn *p_hwfn = &cdev->hwfns[i];
5348                 p_hwfn->pf_params = *func_params;
5349         }
5350 
5351         rc = ecore_resc_alloc(cdev);
5352         if (rc)
5353                 goto qlnx_nic_setup_exit;
5354 
5355         ecore_resc_setup(cdev);
5356 
5357 qlnx_nic_setup_exit:
5358 
5359         return rc;
5360 }
5361 
5362 static int
5363 qlnx_nic_start(struct ecore_dev *cdev)
5364 {
5365         int				rc;
5366 	struct ecore_hw_init_params	params;
5367 
5368 	bzero(&params, sizeof (struct ecore_hw_init_params));
5369 
5370 	params.p_tunn = NULL;
5371 	params.b_hw_start = true;
5372 	params.int_mode = cdev->int_mode;
5373 	params.allow_npar_tx_switch = true;
5374 	params.bin_fw_data = NULL;
5375 
5376         rc = ecore_hw_init(cdev, &params);
5377         if (rc) {
5378                 ecore_resc_free(cdev);
5379                 return rc;
5380         }
5381 
5382         return 0;
5383 }
5384 
5385 static int
5386 qlnx_slowpath_start(qlnx_host_t *ha)
5387 {
5388 	struct ecore_dev	*cdev;
5389 	struct ecore_pf_params	pf_params;
5390 	int			rc;
5391 
5392 	memset(&pf_params, 0, sizeof(struct ecore_pf_params));
5393 	pf_params.eth_pf_params.num_cons  =
5394 		(ha->num_rss) * (ha->num_tc + 1);
5395 
5396 	cdev = &ha->cdev;
5397 
5398 	rc = qlnx_nic_setup(cdev, &pf_params);
5399         if (rc)
5400                 goto qlnx_slowpath_start_exit;
5401 
5402         cdev->int_mode = ECORE_INT_MODE_MSIX;
5403         cdev->int_coalescing_mode = ECORE_COAL_MODE_ENABLE;
5404 
5405 #ifdef QLNX_MAX_COALESCE
5406 	cdev->rx_coalesce_usecs = 255;
5407 	cdev->tx_coalesce_usecs = 255;
5408 #endif
5409 
5410 	rc = qlnx_nic_start(cdev);
5411 
5412 	ha->rx_coalesce_usecs = cdev->rx_coalesce_usecs;
5413 	ha->tx_coalesce_usecs = cdev->tx_coalesce_usecs;
5414 
5415 qlnx_slowpath_start_exit:
5416 
5417 	return (rc);
5418 }
5419 
5420 static int
5421 qlnx_slowpath_stop(qlnx_host_t *ha)
5422 {
5423 	struct ecore_dev	*cdev;
5424 	device_t		dev = ha->pci_dev;
5425 	int			i;
5426 
5427 	cdev = &ha->cdev;
5428 
5429 	ecore_hw_stop(cdev);
5430 
5431  	for (i = 0; i < ha->cdev.num_hwfns; i++) {
5432 
5433         	if (ha->sp_handle[i])
5434                 	(void)bus_teardown_intr(dev, ha->sp_irq[i],
5435 				ha->sp_handle[i]);
5436 
5437 		ha->sp_handle[i] = NULL;
5438 
5439         	if (ha->sp_irq[i])
5440 			(void) bus_release_resource(dev, SYS_RES_IRQ,
5441 				ha->sp_irq_rid[i], ha->sp_irq[i]);
5442 		ha->sp_irq[i] = NULL;
5443 	}
5444 
5445         ecore_resc_free(cdev);
5446 
5447         return 0;
5448 }
5449 
5450 static void
5451 qlnx_set_id(struct ecore_dev *cdev, char name[NAME_SIZE],
5452 	char ver_str[VER_SIZE])
5453 {
5454         int	i;
5455 
5456         memcpy(cdev->name, name, NAME_SIZE);
5457 
5458         for_each_hwfn(cdev, i) {
5459                 snprintf(cdev->hwfns[i].name, NAME_SIZE, "%s-%d", name, i);
5460         }
5461 
5462         cdev->drv_type = DRV_ID_DRV_TYPE_FREEBSD;
5463 
5464 	return ;
5465 }
5466 
5467 void
5468 qlnx_get_protocol_stats(void *cdev, int proto_type, void *proto_stats)
5469 {
5470 	enum ecore_mcp_protocol_type	type;
5471 	union ecore_mcp_protocol_stats	*stats;
5472 	struct ecore_eth_stats		eth_stats;
5473 	qlnx_host_t			*ha;
5474 
5475 	ha = cdev;
5476 	stats = proto_stats;
5477 	type = proto_type;
5478 
5479         switch (type) {
5480 
5481         case ECORE_MCP_LAN_STATS:
5482                 ecore_get_vport_stats((struct ecore_dev *)cdev, &eth_stats);
5483                 stats->lan_stats.ucast_rx_pkts = eth_stats.common.rx_ucast_pkts;
5484                 stats->lan_stats.ucast_tx_pkts = eth_stats.common.tx_ucast_pkts;
5485                 stats->lan_stats.fcs_err = -1;
5486                 break;
5487 
5488 	default:
5489 		ha->err_get_proto_invalid_type++;
5490 
5491 		QL_DPRINT1(ha, "invalid protocol type 0x%x\n", type);
5492 		break;
5493 	}
5494 	return;
5495 }
5496 
5497 static int
5498 qlnx_get_mfw_version(qlnx_host_t *ha, uint32_t *mfw_ver)
5499 {
5500 	struct ecore_hwfn	*p_hwfn;
5501 	struct ecore_ptt	*p_ptt;
5502 
5503 	p_hwfn = &ha->cdev.hwfns[0];
5504 	p_ptt = ecore_ptt_acquire(p_hwfn);
5505 
5506 	if (p_ptt ==  NULL) {
5507                 QL_DPRINT1(ha, "ecore_ptt_acquire failed\n");
5508                 return (-1);
5509 	}
5510 	ecore_mcp_get_mfw_ver(p_hwfn, p_ptt, mfw_ver, NULL);
5511 
5512 	ecore_ptt_release(p_hwfn, p_ptt);
5513 
5514 	return (0);
5515 }
5516 
5517 static int
5518 qlnx_get_flash_size(qlnx_host_t *ha, uint32_t *flash_size)
5519 {
5520 	struct ecore_hwfn	*p_hwfn;
5521 	struct ecore_ptt	*p_ptt;
5522 
5523 	p_hwfn = &ha->cdev.hwfns[0];
5524 	p_ptt = ecore_ptt_acquire(p_hwfn);
5525 
5526 	if (p_ptt ==  NULL) {
5527                 QL_DPRINT1(ha,"ecore_ptt_acquire failed\n");
5528                 return (-1);
5529 	}
5530 	ecore_mcp_get_flash_size(p_hwfn, p_ptt, flash_size);
5531 
5532 	ecore_ptt_release(p_hwfn, p_ptt);
5533 
5534 	return (0);
5535 }
5536 
5537 static int
5538 qlnx_alloc_mem_arrays(qlnx_host_t *ha)
5539 {
5540 	struct ecore_dev	*cdev;
5541 
5542 	cdev = &ha->cdev;
5543 
5544 	bzero(&ha->txq_array[0], (sizeof(struct qlnx_tx_queue) * QLNX_MAX_RSS));
5545 	bzero(&ha->rxq_array[0], (sizeof(struct qlnx_rx_queue) * QLNX_MAX_RSS));
5546 	bzero(&ha->sb_array[0], (sizeof(struct ecore_sb_info) * QLNX_MAX_RSS));
5547 
5548         return 0;
5549 }
5550 
5551 static void
5552 qlnx_init_fp(qlnx_host_t *ha)
5553 {
5554 	int rss_id, txq_array_index, tc;
5555 
5556 	for (rss_id = 0; rss_id < ha->num_rss; rss_id++) {
5557 
5558 		struct qlnx_fastpath *fp = &ha->fp_array[rss_id];
5559 
5560 		fp->rss_id = rss_id;
5561 		fp->edev = ha;
5562 		fp->sb_info = &ha->sb_array[rss_id];
5563 		fp->rxq = &ha->rxq_array[rss_id];
5564 		fp->rxq->rxq_id = rss_id;
5565 
5566 		for (tc = 0; tc < ha->num_tc; tc++) {
5567                         txq_array_index = tc * ha->num_rss + rss_id;
5568                         fp->txq[tc] = &ha->txq_array[txq_array_index];
5569                         fp->txq[tc]->index = txq_array_index;
5570 		}
5571 
5572 		snprintf(fp->name, sizeof(fp->name), "%s-fp-%d", qlnx_name_str,
5573 			rss_id);
5574 
5575 		fp->tx_ring_full = 0;
5576 
5577 		/* reset all the statistics counters */
5578 
5579 		fp->tx_pkts_processed = 0;
5580 		fp->tx_pkts_freed = 0;
5581 		fp->tx_pkts_transmitted = 0;
5582 		fp->tx_pkts_completed = 0;
5583 
5584 #ifdef QLNX_TRACE_PERF_DATA
5585 		fp->tx_pkts_trans_ctx = 0;
5586 		fp->tx_pkts_compl_ctx = 0;
5587 		fp->tx_pkts_trans_fp = 0;
5588 		fp->tx_pkts_compl_fp = 0;
5589 		fp->tx_pkts_compl_intr = 0;
5590 #endif
5591 		fp->tx_lso_wnd_min_len = 0;
5592 		fp->tx_defrag = 0;
5593 		fp->tx_nsegs_gt_elem_left = 0;
5594 		fp->tx_tso_max_nsegs = 0;
5595 		fp->tx_tso_min_nsegs = 0;
5596 		fp->err_tx_nsegs_gt_elem_left = 0;
5597 		fp->err_tx_dmamap_create = 0;
5598 		fp->err_tx_defrag_dmamap_load = 0;
5599 		fp->err_tx_non_tso_max_seg = 0;
5600 		fp->err_tx_dmamap_load = 0;
5601 		fp->err_tx_defrag = 0;
5602 		fp->err_tx_free_pkt_null = 0;
5603 		fp->err_tx_cons_idx_conflict = 0;
5604 
5605 		fp->rx_pkts = 0;
5606 		fp->err_m_getcl = 0;
5607 		fp->err_m_getjcl = 0;
5608         }
5609 	return;
5610 }
5611 
5612 static void
5613 qlnx_free_mem_sb(qlnx_host_t *ha, struct ecore_sb_info *sb_info)
5614 {
5615 	struct ecore_dev	*cdev;
5616 
5617 	cdev = &ha->cdev;
5618 
5619         if (sb_info->sb_virt) {
5620                 OSAL_DMA_FREE_COHERENT(cdev, ((void *)sb_info->sb_virt),
5621 			(sb_info->sb_phys), (sizeof(*sb_info->sb_virt)));
5622 		sb_info->sb_virt = NULL;
5623 	}
5624 }
5625 
5626 static int
5627 qlnx_sb_init(struct ecore_dev *cdev, struct ecore_sb_info *sb_info,
5628 	void *sb_virt_addr, bus_addr_t sb_phy_addr, u16 sb_id)
5629 {
5630         struct ecore_hwfn	*p_hwfn;
5631         int			hwfn_index, rc;
5632         u16			rel_sb_id;
5633 
5634         hwfn_index = sb_id % cdev->num_hwfns;
5635         p_hwfn = &cdev->hwfns[hwfn_index];
5636         rel_sb_id = sb_id / cdev->num_hwfns;
5637 
5638         QL_DPRINT2(((qlnx_host_t *)cdev),
5639                 "hwfn_index = %d p_hwfn = %p sb_id = 0x%x rel_sb_id = 0x%x \
5640                 sb_info = %p sb_virt_addr = %p sb_phy_addr = %p\n",
5641                 hwfn_index, p_hwfn, sb_id, rel_sb_id, sb_info,
5642                 sb_virt_addr, (void *)sb_phy_addr);
5643 
5644         rc = ecore_int_sb_init(p_hwfn, p_hwfn->p_main_ptt, sb_info,
5645                              sb_virt_addr, sb_phy_addr, rel_sb_id);
5646 
5647         return rc;
5648 }
5649 
5650 /* This function allocates fast-path status block memory */
5651 static int
5652 qlnx_alloc_mem_sb(qlnx_host_t *ha, struct ecore_sb_info *sb_info, u16 sb_id)
5653 {
5654         struct status_block_e4	*sb_virt;
5655         bus_addr_t		sb_phys;
5656         int			rc;
5657 	uint32_t		size;
5658 	struct ecore_dev	*cdev;
5659 
5660 	cdev = &ha->cdev;
5661 
5662 	size = sizeof(*sb_virt);
5663 	sb_virt = OSAL_DMA_ALLOC_COHERENT(cdev, (&sb_phys), size);
5664 
5665         if (!sb_virt) {
5666                 QL_DPRINT1(ha, "Status block allocation failed\n");
5667                 return -ENOMEM;
5668         }
5669 
5670         rc = qlnx_sb_init(cdev, sb_info, sb_virt, sb_phys, sb_id);
5671         if (rc) {
5672                 OSAL_DMA_FREE_COHERENT(cdev, sb_virt, sb_phys, size);
5673         }
5674 
5675 	return rc;
5676 }
5677 
5678 static void
5679 qlnx_free_rx_buffers(qlnx_host_t *ha, struct qlnx_rx_queue *rxq)
5680 {
5681         int			i;
5682 	struct sw_rx_data	*rx_buf;
5683 
5684         for (i = 0; i < rxq->num_rx_buffers; i++) {
5685 
5686                 rx_buf = &rxq->sw_rx_ring[i];
5687 
5688 		if (rx_buf->data != NULL) {
5689 			if (rx_buf->map != NULL) {
5690 				bus_dmamap_unload(ha->rx_tag, rx_buf->map);
5691 				bus_dmamap_destroy(ha->rx_tag, rx_buf->map);
5692 				rx_buf->map = NULL;
5693 			}
5694 			m_freem(rx_buf->data);
5695 			rx_buf->data = NULL;
5696 		}
5697         }
5698 	return;
5699 }
5700 
5701 static void
5702 qlnx_free_mem_rxq(qlnx_host_t *ha, struct qlnx_rx_queue *rxq)
5703 {
5704 	struct ecore_dev	*cdev;
5705 	int			i;
5706 
5707 	cdev = &ha->cdev;
5708 
5709 	qlnx_free_rx_buffers(ha, rxq);
5710 
5711 	for (i = 0; i < ETH_TPA_MAX_AGGS_NUM; i++) {
5712 		qlnx_free_tpa_mbuf(ha, &rxq->tpa_info[i]);
5713 		if (rxq->tpa_info[i].mpf != NULL)
5714 			m_freem(rxq->tpa_info[i].mpf);
5715 	}
5716 
5717 	bzero((void *)&rxq->sw_rx_ring[0],
5718 		(sizeof (struct sw_rx_data) * RX_RING_SIZE));
5719 
5720         /* Free the real RQ ring used by FW */
5721 	if (rxq->rx_bd_ring.p_virt_addr) {
5722                 ecore_chain_free(cdev, &rxq->rx_bd_ring);
5723                 rxq->rx_bd_ring.p_virt_addr = NULL;
5724         }
5725 
5726         /* Free the real completion ring used by FW */
5727         if (rxq->rx_comp_ring.p_virt_addr &&
5728                         rxq->rx_comp_ring.pbl_sp.p_virt_table) {
5729                 ecore_chain_free(cdev, &rxq->rx_comp_ring);
5730                 rxq->rx_comp_ring.p_virt_addr = NULL;
5731                 rxq->rx_comp_ring.pbl_sp.p_virt_table = NULL;
5732         }
5733 
5734 #ifdef QLNX_SOFT_LRO
5735 	{
5736 		struct lro_ctrl *lro;
5737 
5738 		lro = &rxq->lro;
5739 		tcp_lro_free(lro);
5740 	}
5741 #endif /* #ifdef QLNX_SOFT_LRO */
5742 
5743 	return;
5744 }
5745 
5746 static int
5747 qlnx_alloc_rx_buffer(qlnx_host_t *ha, struct qlnx_rx_queue *rxq)
5748 {
5749         register struct mbuf	*mp;
5750         uint16_t		rx_buf_size;
5751         struct sw_rx_data	*sw_rx_data;
5752         struct eth_rx_bd	*rx_bd;
5753         dma_addr_t		dma_addr;
5754 	bus_dmamap_t		map;
5755 	bus_dma_segment_t       segs[1];
5756 	int			nsegs;
5757 	int			ret;
5758 	struct ecore_dev	*cdev;
5759 
5760 	cdev = &ha->cdev;
5761 
5762         rx_buf_size = rxq->rx_buf_size;
5763 
5764 	mp = m_getjcl(M_NOWAIT, MT_DATA, M_PKTHDR, rx_buf_size);
5765 
5766         if (mp == NULL) {
5767                 QL_DPRINT1(ha, "Failed to allocate Rx data\n");
5768                 return -ENOMEM;
5769         }
5770 
5771 	mp->m_len = mp->m_pkthdr.len = rx_buf_size;
5772 
5773 	map = (bus_dmamap_t)0;
5774 
5775 	ret = bus_dmamap_load_mbuf_sg(ha->rx_tag, map, mp, segs, &nsegs,
5776 			BUS_DMA_NOWAIT);
5777 	dma_addr = segs[0].ds_addr;
5778 
5779 	if (ret || !dma_addr || (nsegs != 1)) {
5780 		m_freem(mp);
5781 		QL_DPRINT1(ha, "bus_dmamap_load failed[%d, 0x%016llx, %d]\n",
5782                            ret, (long long unsigned int)dma_addr, nsegs);
5783 		return -ENOMEM;
5784 	}
5785 
5786         sw_rx_data = &rxq->sw_rx_ring[rxq->sw_rx_prod];
5787         sw_rx_data->data = mp;
5788         sw_rx_data->dma_addr = dma_addr;
5789         sw_rx_data->map = map;
5790 
5791         /* Advance PROD and get BD pointer */
5792         rx_bd = (struct eth_rx_bd *)ecore_chain_produce(&rxq->rx_bd_ring);
5793         rx_bd->addr.hi = htole32(U64_HI(dma_addr));
5794         rx_bd->addr.lo = htole32(U64_LO(dma_addr));
5795 	bus_dmamap_sync(ha->rx_tag, map, BUS_DMASYNC_PREREAD);
5796 
5797         rxq->sw_rx_prod = (rxq->sw_rx_prod + 1) & (RX_RING_SIZE - 1);
5798 
5799         return 0;
5800 }
5801 
5802 static int
5803 qlnx_alloc_tpa_mbuf(qlnx_host_t *ha, uint16_t rx_buf_size,
5804 	struct qlnx_agg_info *tpa)
5805 {
5806 	struct mbuf		*mp;
5807         dma_addr_t		dma_addr;
5808 	bus_dmamap_t		map;
5809 	bus_dma_segment_t       segs[1];
5810 	int			nsegs;
5811 	int			ret;
5812         struct sw_rx_data	*rx_buf;
5813 
5814 	mp = m_getjcl(M_NOWAIT, MT_DATA, M_PKTHDR, rx_buf_size);
5815 
5816         if (mp == NULL) {
5817                 QL_DPRINT1(ha, "Failed to allocate Rx data\n");
5818                 return -ENOMEM;
5819         }
5820 
5821 	mp->m_len = mp->m_pkthdr.len = rx_buf_size;
5822 
5823 	map = (bus_dmamap_t)0;
5824 
5825 	ret = bus_dmamap_load_mbuf_sg(ha->rx_tag, map, mp, segs, &nsegs,
5826 			BUS_DMA_NOWAIT);
5827 	dma_addr = segs[0].ds_addr;
5828 
5829 	if (ret || !dma_addr || (nsegs != 1)) {
5830 		m_freem(mp);
5831 		QL_DPRINT1(ha, "bus_dmamap_load failed[%d, 0x%016llx, %d]\n",
5832 			ret, (long long unsigned int)dma_addr, nsegs);
5833 		return -ENOMEM;
5834 	}
5835 
5836         rx_buf = &tpa->rx_buf;
5837 
5838 	memset(rx_buf, 0, sizeof (struct sw_rx_data));
5839 
5840         rx_buf->data = mp;
5841         rx_buf->dma_addr = dma_addr;
5842         rx_buf->map = map;
5843 
5844 	bus_dmamap_sync(ha->rx_tag, map, BUS_DMASYNC_PREREAD);
5845 
5846 	return (0);
5847 }
5848 
5849 static void
5850 qlnx_free_tpa_mbuf(qlnx_host_t *ha, struct qlnx_agg_info *tpa)
5851 {
5852         struct sw_rx_data	*rx_buf;
5853 
5854 	rx_buf = &tpa->rx_buf;
5855 
5856 	if (rx_buf->data != NULL) {
5857 		if (rx_buf->map != NULL) {
5858 			bus_dmamap_unload(ha->rx_tag, rx_buf->map);
5859 			bus_dmamap_destroy(ha->rx_tag, rx_buf->map);
5860 			rx_buf->map = NULL;
5861 		}
5862 		m_freem(rx_buf->data);
5863 		rx_buf->data = NULL;
5864 	}
5865 	return;
5866 }
5867 
5868 /* This function allocates all memory needed per Rx queue */
5869 static int
5870 qlnx_alloc_mem_rxq(qlnx_host_t *ha, struct qlnx_rx_queue *rxq)
5871 {
5872         int			i, rc, num_allocated;
5873 	struct ifnet		*ifp;
5874 	struct ecore_dev	 *cdev;
5875 
5876 	cdev = &ha->cdev;
5877 	ifp = ha->ifp;
5878 
5879         rxq->num_rx_buffers = RX_RING_SIZE;
5880 
5881 	rxq->rx_buf_size = ha->rx_buf_size;
5882 
5883         /* Allocate the parallel driver ring for Rx buffers */
5884 	bzero((void *)&rxq->sw_rx_ring[0],
5885 		(sizeof (struct sw_rx_data) * RX_RING_SIZE));
5886 
5887         /* Allocate FW Rx ring  */
5888 
5889         rc = ecore_chain_alloc(cdev,
5890 			ECORE_CHAIN_USE_TO_CONSUME_PRODUCE,
5891 			ECORE_CHAIN_MODE_NEXT_PTR,
5892 			ECORE_CHAIN_CNT_TYPE_U16,
5893 			RX_RING_SIZE,
5894 			sizeof(struct eth_rx_bd),
5895 			&rxq->rx_bd_ring, NULL);
5896 
5897         if (rc)
5898                 goto err;
5899 
5900         /* Allocate FW completion ring */
5901         rc = ecore_chain_alloc(cdev,
5902                         ECORE_CHAIN_USE_TO_CONSUME,
5903                         ECORE_CHAIN_MODE_PBL,
5904 			ECORE_CHAIN_CNT_TYPE_U16,
5905                         RX_RING_SIZE,
5906                         sizeof(union eth_rx_cqe),
5907                         &rxq->rx_comp_ring, NULL);
5908 
5909         if (rc)
5910                 goto err;
5911 
5912         /* Allocate buffers for the Rx ring */
5913 
5914 	for (i = 0; i < ETH_TPA_MAX_AGGS_NUM; i++) {
5915 		rc = qlnx_alloc_tpa_mbuf(ha, rxq->rx_buf_size,
5916 			&rxq->tpa_info[i]);
5917                 if (rc)
5918                         break;
5919 
5920 	}
5921 
5922         for (i = 0; i < rxq->num_rx_buffers; i++) {
5923                 rc = qlnx_alloc_rx_buffer(ha, rxq);
5924                 if (rc)
5925                         break;
5926         }
5927         num_allocated = i;
5928         if (!num_allocated) {
5929 		QL_DPRINT1(ha, "Rx buffers allocation failed\n");
5930                 goto err;
5931         } else if (num_allocated < rxq->num_rx_buffers) {
5932 		QL_DPRINT1(ha, "Allocated less buffers than"
5933 			" desired (%d allocated)\n", num_allocated);
5934         }
5935 
5936 #ifdef QLNX_SOFT_LRO
5937 
5938 	{
5939 		struct lro_ctrl *lro;
5940 
5941 		lro = &rxq->lro;
5942 
5943 #if (__FreeBSD_version >= 1100101) || (defined QLNX_QSORT_LRO)
5944 		if (tcp_lro_init_args(lro, ifp, 0, rxq->num_rx_buffers)) {
5945 			QL_DPRINT1(ha, "tcp_lro_init[%d] failed\n",
5946 				   rxq->rxq_id);
5947 			goto err;
5948 		}
5949 #else
5950 		if (tcp_lro_init(lro)) {
5951 			QL_DPRINT1(ha, "tcp_lro_init[%d] failed\n",
5952 				   rxq->rxq_id);
5953 			goto err;
5954 		}
5955 #endif /* #if (__FreeBSD_version >= 1100101) || (defined QLNX_QSORT_LRO) */
5956 
5957 		lro->ifp = ha->ifp;
5958 	}
5959 #endif /* #ifdef QLNX_SOFT_LRO */
5960         return 0;
5961 
5962 err:
5963         qlnx_free_mem_rxq(ha, rxq);
5964         return -ENOMEM;
5965 }
5966 
5967 
5968 static void
5969 qlnx_free_mem_txq(qlnx_host_t *ha, struct qlnx_fastpath *fp,
5970 	struct qlnx_tx_queue *txq)
5971 {
5972 	struct ecore_dev	*cdev;
5973 
5974 	cdev = &ha->cdev;
5975 
5976 	bzero((void *)&txq->sw_tx_ring[0],
5977 		(sizeof (struct sw_tx_bd) * TX_RING_SIZE));
5978 
5979         /* Free the real RQ ring used by FW */
5980         if (txq->tx_pbl.p_virt_addr) {
5981                 ecore_chain_free(cdev, &txq->tx_pbl);
5982                 txq->tx_pbl.p_virt_addr = NULL;
5983         }
5984 	return;
5985 }
5986 
5987 /* This function allocates all memory needed per Tx queue */
5988 static int
5989 qlnx_alloc_mem_txq(qlnx_host_t *ha, struct qlnx_fastpath *fp,
5990 	struct qlnx_tx_queue *txq)
5991 {
5992         int			ret = ECORE_SUCCESS;
5993         union eth_tx_bd_types	*p_virt;
5994 	struct ecore_dev	*cdev;
5995 
5996 	cdev = &ha->cdev;
5997 
5998 	bzero((void *)&txq->sw_tx_ring[0],
5999 		(sizeof (struct sw_tx_bd) * TX_RING_SIZE));
6000 
6001         /* Allocate the real Tx ring to be used by FW */
6002         ret = ecore_chain_alloc(cdev,
6003                         ECORE_CHAIN_USE_TO_CONSUME_PRODUCE,
6004                         ECORE_CHAIN_MODE_PBL,
6005 			ECORE_CHAIN_CNT_TYPE_U16,
6006                         TX_RING_SIZE,
6007                         sizeof(*p_virt),
6008                         &txq->tx_pbl, NULL);
6009 
6010         if (ret != ECORE_SUCCESS) {
6011                 goto err;
6012         }
6013 
6014 	txq->num_tx_buffers = TX_RING_SIZE;
6015 
6016         return 0;
6017 
6018 err:
6019         qlnx_free_mem_txq(ha, fp, txq);
6020         return -ENOMEM;
6021 }
6022 
6023 static void
6024 qlnx_free_tx_br(qlnx_host_t *ha, struct qlnx_fastpath *fp)
6025 {
6026 	struct mbuf	*mp;
6027 	struct ifnet	*ifp = ha->ifp;
6028 
6029 	if (mtx_initialized(&fp->tx_mtx)) {
6030 
6031 		if (fp->tx_br != NULL) {
6032 
6033 			mtx_lock(&fp->tx_mtx);
6034 
6035 			while ((mp = drbr_dequeue(ifp, fp->tx_br)) != NULL) {
6036 				fp->tx_pkts_freed++;
6037 				m_freem(mp);
6038 			}
6039 
6040 			mtx_unlock(&fp->tx_mtx);
6041 
6042 			buf_ring_free(fp->tx_br, M_DEVBUF);
6043 			fp->tx_br = NULL;
6044 		}
6045 		mtx_destroy(&fp->tx_mtx);
6046 	}
6047 	return;
6048 }
6049 
6050 static void
6051 qlnx_free_mem_fp(qlnx_host_t *ha, struct qlnx_fastpath *fp)
6052 {
6053         int	tc;
6054 
6055         qlnx_free_mem_sb(ha, fp->sb_info);
6056 
6057         qlnx_free_mem_rxq(ha, fp->rxq);
6058 
6059         for (tc = 0; tc < ha->num_tc; tc++)
6060                 qlnx_free_mem_txq(ha, fp, fp->txq[tc]);
6061 
6062 	return;
6063 }
6064 
6065 static int
6066 qlnx_alloc_tx_br(qlnx_host_t *ha, struct qlnx_fastpath *fp)
6067 {
6068 	snprintf(fp->tx_mtx_name, sizeof(fp->tx_mtx_name),
6069 		"qlnx%d_fp%d_tx_mq_lock", ha->dev_unit, fp->rss_id);
6070 
6071 	mtx_init(&fp->tx_mtx, fp->tx_mtx_name, NULL, MTX_DEF);
6072 
6073         fp->tx_br = buf_ring_alloc(TX_RING_SIZE, M_DEVBUF,
6074                                    M_NOWAIT, &fp->tx_mtx);
6075         if (fp->tx_br == NULL) {
6076 		QL_DPRINT1(ha, "buf_ring_alloc failed for fp[%d, %d]\n",
6077 			ha->dev_unit, fp->rss_id);
6078 		return -ENOMEM;
6079         }
6080 	return 0;
6081 }
6082 
6083 static int
6084 qlnx_alloc_mem_fp(qlnx_host_t *ha, struct qlnx_fastpath *fp)
6085 {
6086         int	rc, tc;
6087 
6088         rc = qlnx_alloc_mem_sb(ha, fp->sb_info, fp->rss_id);
6089         if (rc)
6090                 goto err;
6091 
6092 	if (ha->rx_jumbo_buf_eq_mtu) {
6093 		if (ha->max_frame_size <= MCLBYTES)
6094 			ha->rx_buf_size = MCLBYTES;
6095 		else if (ha->max_frame_size <= MJUMPAGESIZE)
6096 			ha->rx_buf_size = MJUMPAGESIZE;
6097 		else if (ha->max_frame_size <= MJUM9BYTES)
6098 			ha->rx_buf_size = MJUM9BYTES;
6099 		else if (ha->max_frame_size <= MJUM16BYTES)
6100 			ha->rx_buf_size = MJUM16BYTES;
6101 	} else {
6102 		if (ha->max_frame_size <= MCLBYTES)
6103 			ha->rx_buf_size = MCLBYTES;
6104 		else
6105 			ha->rx_buf_size = MJUMPAGESIZE;
6106 	}
6107 
6108         rc = qlnx_alloc_mem_rxq(ha, fp->rxq);
6109         if (rc)
6110                 goto err;
6111 
6112         for (tc = 0; tc < ha->num_tc; tc++) {
6113                 rc = qlnx_alloc_mem_txq(ha, fp, fp->txq[tc]);
6114                 if (rc)
6115                         goto err;
6116         }
6117 
6118         return 0;
6119 
6120 err:
6121         qlnx_free_mem_fp(ha, fp);
6122         return -ENOMEM;
6123 }
6124 
6125 static void
6126 qlnx_free_mem_load(qlnx_host_t *ha)
6127 {
6128         int			i;
6129 	struct ecore_dev	*cdev;
6130 
6131 	cdev = &ha->cdev;
6132 
6133         for (i = 0; i < ha->num_rss; i++) {
6134                 struct qlnx_fastpath *fp = &ha->fp_array[i];
6135 
6136                 qlnx_free_mem_fp(ha, fp);
6137         }
6138 	return;
6139 }
6140 
6141 static int
6142 qlnx_alloc_mem_load(qlnx_host_t *ha)
6143 {
6144         int	rc = 0, rss_id;
6145 
6146         for (rss_id = 0; rss_id < ha->num_rss; rss_id++) {
6147                 struct qlnx_fastpath *fp = &ha->fp_array[rss_id];
6148 
6149                 rc = qlnx_alloc_mem_fp(ha, fp);
6150                 if (rc)
6151                         break;
6152         }
6153 	return (rc);
6154 }
6155 
6156 static int
6157 qlnx_start_vport(struct ecore_dev *cdev,
6158                 u8 vport_id,
6159                 u16 mtu,
6160                 u8 drop_ttl0_flg,
6161                 u8 inner_vlan_removal_en_flg,
6162 		u8 tx_switching,
6163 		u8 hw_lro_enable)
6164 {
6165         int					rc, i;
6166 	struct ecore_sp_vport_start_params	vport_start_params = { 0 };
6167 	qlnx_host_t				*ha;
6168 
6169 	ha = (qlnx_host_t *)cdev;
6170 
6171 	vport_start_params.remove_inner_vlan = inner_vlan_removal_en_flg;
6172 	vport_start_params.tx_switching = 0;
6173 	vport_start_params.handle_ptp_pkts = 0;
6174 	vport_start_params.only_untagged = 0;
6175 	vport_start_params.drop_ttl0 = drop_ttl0_flg;
6176 
6177 	vport_start_params.tpa_mode =
6178 		(hw_lro_enable ? ECORE_TPA_MODE_RSC : ECORE_TPA_MODE_NONE);
6179 	vport_start_params.max_buffers_per_cqe = QLNX_TPA_MAX_AGG_BUFFERS;
6180 
6181 	vport_start_params.vport_id = vport_id;
6182 	vport_start_params.mtu = mtu;
6183 
6184 
6185 	QL_DPRINT2(ha, "Setting mtu to %d and VPORT ID = %d\n", mtu, vport_id);
6186 
6187         for_each_hwfn(cdev, i) {
6188                 struct ecore_hwfn *p_hwfn = &cdev->hwfns[i];
6189 
6190 		vport_start_params.concrete_fid = p_hwfn->hw_info.concrete_fid;
6191 		vport_start_params.opaque_fid = p_hwfn->hw_info.opaque_fid;
6192 
6193                 rc = ecore_sp_vport_start(p_hwfn, &vport_start_params);
6194 
6195                 if (rc) {
6196 			QL_DPRINT1(ha, "Failed to start VPORT V-PORT %d"
6197 				" with MTU %d\n" , vport_id, mtu);
6198                         return -ENOMEM;
6199                 }
6200 
6201                 ecore_hw_start_fastpath(p_hwfn);
6202 
6203 		QL_DPRINT2(ha, "Started V-PORT %d with MTU %d\n",
6204 			vport_id, mtu);
6205         }
6206         return 0;
6207 }
6208 
6209 
6210 static int
6211 qlnx_update_vport(struct ecore_dev *cdev,
6212 	struct qlnx_update_vport_params *params)
6213 {
6214         struct ecore_sp_vport_update_params	sp_params;
6215         int					rc, i, j, fp_index;
6216 	struct ecore_hwfn			*p_hwfn;
6217         struct ecore_rss_params			*rss;
6218 	qlnx_host_t				*ha = (qlnx_host_t *)cdev;
6219         struct qlnx_fastpath			*fp;
6220 
6221         memset(&sp_params, 0, sizeof(sp_params));
6222         /* Translate protocol params into sp params */
6223         sp_params.vport_id = params->vport_id;
6224 
6225         sp_params.update_vport_active_rx_flg =
6226 		params->update_vport_active_rx_flg;
6227         sp_params.vport_active_rx_flg = params->vport_active_rx_flg;
6228 
6229         sp_params.update_vport_active_tx_flg =
6230 		params->update_vport_active_tx_flg;
6231         sp_params.vport_active_tx_flg = params->vport_active_tx_flg;
6232 
6233         sp_params.update_inner_vlan_removal_flg =
6234                 params->update_inner_vlan_removal_flg;
6235         sp_params.inner_vlan_removal_flg = params->inner_vlan_removal_flg;
6236 
6237 	sp_params.sge_tpa_params = params->sge_tpa_params;
6238 
6239         /* RSS - is a bit tricky, since upper-layer isn't familiar with hwfns.
6240          * We need to re-fix the rss values per engine for CMT.
6241          */
6242 	if (params->rss_params->update_rss_config)
6243         sp_params.rss_params = params->rss_params;
6244 	else
6245 		sp_params.rss_params =  NULL;
6246 
6247         for_each_hwfn(cdev, i) {
6248 
6249 		p_hwfn = &cdev->hwfns[i];
6250 
6251 		if ((cdev->num_hwfns > 1) &&
6252 			params->rss_params->update_rss_config &&
6253 			params->rss_params->rss_enable) {
6254 
6255 			rss = params->rss_params;
6256 
6257 			for (j = 0; j < ECORE_RSS_IND_TABLE_SIZE; j++) {
6258 
6259 				fp_index = ((cdev->num_hwfns * j) + i) %
6260 						ha->num_rss;
6261 
6262                 		fp = &ha->fp_array[fp_index];
6263                         	rss->rss_ind_table[j] = fp->rxq->handle;
6264 			}
6265 
6266 			for (j = 0; j < ECORE_RSS_IND_TABLE_SIZE;) {
6267 				QL_DPRINT3(ha, "%p %p %p %p %p %p %p %p \n",
6268 					rss->rss_ind_table[j],
6269 					rss->rss_ind_table[j+1],
6270 					rss->rss_ind_table[j+2],
6271 					rss->rss_ind_table[j+3],
6272 					rss->rss_ind_table[j+4],
6273 					rss->rss_ind_table[j+5],
6274 					rss->rss_ind_table[j+6],
6275 					rss->rss_ind_table[j+7]);
6276 					j += 8;
6277 			}
6278 		}
6279 
6280                 sp_params.opaque_fid = p_hwfn->hw_info.opaque_fid;
6281 
6282 		QL_DPRINT1(ha, "Update sp vport ID=%d\n", params->vport_id);
6283 
6284                 rc = ecore_sp_vport_update(p_hwfn, &sp_params,
6285                                            ECORE_SPQ_MODE_EBLOCK, NULL);
6286                 if (rc) {
6287 			QL_DPRINT1(ha, "Failed to update VPORT\n");
6288                         return rc;
6289                 }
6290 
6291                 QL_DPRINT2(ha, "Updated V-PORT %d: tx_active_flag %d, \
6292 			rx_active_flag %d [tx_update %d], [rx_update %d]\n",
6293 			params->vport_id, params->vport_active_tx_flg,
6294 			params->vport_active_rx_flg,
6295 			params->update_vport_active_tx_flg,
6296 			params->update_vport_active_rx_flg);
6297         }
6298 
6299         return 0;
6300 }
6301 
6302 static void
6303 qlnx_reuse_rx_data(struct qlnx_rx_queue *rxq)
6304 {
6305         struct eth_rx_bd	*rx_bd_cons =
6306 					ecore_chain_consume(&rxq->rx_bd_ring);
6307         struct eth_rx_bd	*rx_bd_prod =
6308 					ecore_chain_produce(&rxq->rx_bd_ring);
6309         struct sw_rx_data	*sw_rx_data_cons =
6310 					&rxq->sw_rx_ring[rxq->sw_rx_cons];
6311         struct sw_rx_data	*sw_rx_data_prod =
6312 					&rxq->sw_rx_ring[rxq->sw_rx_prod];
6313 
6314         sw_rx_data_prod->data = sw_rx_data_cons->data;
6315         memcpy(rx_bd_prod, rx_bd_cons, sizeof(struct eth_rx_bd));
6316 
6317         rxq->sw_rx_cons  = (rxq->sw_rx_cons + 1) & (RX_RING_SIZE - 1);
6318         rxq->sw_rx_prod  = (rxq->sw_rx_prod + 1) & (RX_RING_SIZE - 1);
6319 
6320 	return;
6321 }
6322 
6323 static void
6324 qlnx_update_rx_prod(struct ecore_hwfn *p_hwfn, struct qlnx_rx_queue *rxq)
6325 {
6326 
6327         uint16_t	 	bd_prod;
6328         uint16_t		cqe_prod;
6329 	union {
6330 		struct eth_rx_prod_data rx_prod_data;
6331 		uint32_t		data32;
6332 	} rx_prods;
6333 
6334         bd_prod = ecore_chain_get_prod_idx(&rxq->rx_bd_ring);
6335         cqe_prod = ecore_chain_get_prod_idx(&rxq->rx_comp_ring);
6336 
6337         /* Update producers */
6338         rx_prods.rx_prod_data.bd_prod = htole16(bd_prod);
6339         rx_prods.rx_prod_data.cqe_prod = htole16(cqe_prod);
6340 
6341         /* Make sure that the BD and SGE data is updated before updating the
6342          * producers since FW might read the BD/SGE right after the producer
6343          * is updated.
6344          */
6345 	wmb();
6346 
6347         internal_ram_wr(p_hwfn, rxq->hw_rxq_prod_addr,
6348 		sizeof(rx_prods), &rx_prods.data32);
6349 
6350         /* mmiowb is needed to synchronize doorbell writes from more than one
6351          * processor. It guarantees that the write arrives to the device before
6352          * the napi lock is released and another qlnx_poll is called (possibly
6353          * on another CPU). Without this barrier, the next doorbell can bypass
6354          * this doorbell. This is applicable to IA64/Altix systems.
6355          */
6356         wmb();
6357 
6358 	return;
6359 }
6360 
6361 static uint32_t qlnx_hash_key[] = {
6362                 ((0x6d << 24)|(0x5a << 16)|(0x56 << 8)|0xda),
6363                 ((0x25 << 24)|(0x5b << 16)|(0x0e << 8)|0xc2),
6364                 ((0x41 << 24)|(0x67 << 16)|(0x25 << 8)|0x3d),
6365                 ((0x43 << 24)|(0xa3 << 16)|(0x8f << 8)|0xb0),
6366                 ((0xd0 << 24)|(0xca << 16)|(0x2b << 8)|0xcb),
6367                 ((0xae << 24)|(0x7b << 16)|(0x30 << 8)|0xb4),
6368                 ((0x77 << 24)|(0xcb << 16)|(0x2d << 8)|0xa3),
6369                 ((0x80 << 24)|(0x30 << 16)|(0xf2 << 8)|0x0c),
6370                 ((0x6a << 24)|(0x42 << 16)|(0xb7 << 8)|0x3b),
6371                 ((0xbe << 24)|(0xac << 16)|(0x01 << 8)|0xfa)};
6372 
6373 static int
6374 qlnx_start_queues(qlnx_host_t *ha)
6375 {
6376         int				rc, tc, i, vport_id = 0,
6377 					drop_ttl0_flg = 1, vlan_removal_en = 1,
6378 					tx_switching = 0, hw_lro_enable = 0;
6379         struct ecore_dev		*cdev = &ha->cdev;
6380         struct ecore_rss_params		*rss_params = &ha->rss_params;
6381         struct qlnx_update_vport_params	vport_update_params;
6382         struct ifnet			*ifp;
6383         struct ecore_hwfn		*p_hwfn;
6384 	struct ecore_sge_tpa_params	tpa_params;
6385 	struct ecore_queue_start_common_params qparams;
6386         struct qlnx_fastpath		*fp;
6387 
6388 	ifp = ha->ifp;
6389 
6390 	QL_DPRINT1(ha, "Num RSS = %d\n", ha->num_rss);
6391 
6392         if (!ha->num_rss) {
6393 		QL_DPRINT1(ha, "Cannot update V-VPORT as active as there"
6394 			" are no Rx queues\n");
6395                 return -EINVAL;
6396         }
6397 
6398 #ifndef QLNX_SOFT_LRO
6399         hw_lro_enable = ifp->if_capenable & IFCAP_LRO;
6400 #endif /* #ifndef QLNX_SOFT_LRO */
6401 
6402         rc = qlnx_start_vport(cdev, vport_id, ifp->if_mtu, drop_ttl0_flg,
6403 			vlan_removal_en, tx_switching, hw_lro_enable);
6404 
6405         if (rc) {
6406                 QL_DPRINT1(ha, "Start V-PORT failed %d\n", rc);
6407                 return rc;
6408         }
6409 
6410 	QL_DPRINT2(ha, "Start vport ramrod passed, "
6411 		"vport_id = %d, MTU = %d, vlan_removal_en = %d\n",
6412 		vport_id, (int)(ifp->if_mtu + 0xe), vlan_removal_en);
6413 
6414         for_each_rss(i) {
6415 		struct ecore_rxq_start_ret_params rx_ret_params;
6416 		struct ecore_txq_start_ret_params tx_ret_params;
6417 
6418                 fp = &ha->fp_array[i];
6419         	p_hwfn = &cdev->hwfns[(fp->rss_id % cdev->num_hwfns)];
6420 
6421 		bzero(&qparams, sizeof(struct ecore_queue_start_common_params));
6422 		bzero(&rx_ret_params,
6423 			sizeof (struct ecore_rxq_start_ret_params));
6424 
6425 		qparams.queue_id = i ;
6426 		qparams.vport_id = vport_id;
6427 		qparams.stats_id = vport_id;
6428 		qparams.p_sb = fp->sb_info;
6429 		qparams.sb_idx = RX_PI;
6430 
6431 
6432 		rc = ecore_eth_rx_queue_start(p_hwfn,
6433 			p_hwfn->hw_info.opaque_fid,
6434 			&qparams,
6435 			fp->rxq->rx_buf_size,	/* bd_max_bytes */
6436 			/* bd_chain_phys_addr */
6437 			fp->rxq->rx_bd_ring.p_phys_addr,
6438 			/* cqe_pbl_addr */
6439 			ecore_chain_get_pbl_phys(&fp->rxq->rx_comp_ring),
6440 			/* cqe_pbl_size */
6441 			ecore_chain_get_page_cnt(&fp->rxq->rx_comp_ring),
6442 			&rx_ret_params);
6443 
6444                 if (rc) {
6445                 	QL_DPRINT1(ha, "Start RXQ #%d failed %d\n", i, rc);
6446                         return rc;
6447                 }
6448 
6449 		fp->rxq->hw_rxq_prod_addr	= rx_ret_params.p_prod;
6450 		fp->rxq->handle			= rx_ret_params.p_handle;
6451                 fp->rxq->hw_cons_ptr		=
6452 				&fp->sb_info->sb_virt->pi_array[RX_PI];
6453 
6454                 qlnx_update_rx_prod(p_hwfn, fp->rxq);
6455 
6456                 for (tc = 0; tc < ha->num_tc; tc++) {
6457                         struct qlnx_tx_queue *txq = fp->txq[tc];
6458 
6459 			bzero(&qparams,
6460 				sizeof(struct ecore_queue_start_common_params));
6461 			bzero(&tx_ret_params,
6462 				sizeof (struct ecore_txq_start_ret_params));
6463 
6464 			qparams.queue_id = txq->index / cdev->num_hwfns ;
6465 			qparams.vport_id = vport_id;
6466 			qparams.stats_id = vport_id;
6467 			qparams.p_sb = fp->sb_info;
6468 			qparams.sb_idx = TX_PI(tc);
6469 
6470 			rc = ecore_eth_tx_queue_start(p_hwfn,
6471 				p_hwfn->hw_info.opaque_fid,
6472 				&qparams, tc,
6473 				/* bd_chain_phys_addr */
6474 				ecore_chain_get_pbl_phys(&txq->tx_pbl),
6475 				ecore_chain_get_page_cnt(&txq->tx_pbl),
6476 				&tx_ret_params);
6477 
6478                         if (rc) {
6479                 		QL_DPRINT1(ha, "Start TXQ #%d failed %d\n",
6480 					   txq->index, rc);
6481                                 return rc;
6482                         }
6483 
6484 			txq->doorbell_addr = tx_ret_params.p_doorbell;
6485 			txq->handle = tx_ret_params.p_handle;
6486 
6487                         txq->hw_cons_ptr =
6488                                 &fp->sb_info->sb_virt->pi_array[TX_PI(tc)];
6489                         SET_FIELD(txq->tx_db.data.params,
6490                                   ETH_DB_DATA_DEST, DB_DEST_XCM);
6491                         SET_FIELD(txq->tx_db.data.params, ETH_DB_DATA_AGG_CMD,
6492                                   DB_AGG_CMD_SET);
6493                         SET_FIELD(txq->tx_db.data.params,
6494                                   ETH_DB_DATA_AGG_VAL_SEL,
6495                                   DQ_XCM_ETH_TX_BD_PROD_CMD);
6496 
6497                         txq->tx_db.data.agg_flags = DQ_XCM_ETH_DQ_CF_CMD;
6498                 }
6499         }
6500 
6501         /* Fill struct with RSS params */
6502         if (ha->num_rss > 1) {
6503 
6504                 rss_params->update_rss_config = 1;
6505                 rss_params->rss_enable = 1;
6506                 rss_params->update_rss_capabilities = 1;
6507                 rss_params->update_rss_ind_table = 1;
6508                 rss_params->update_rss_key = 1;
6509                 rss_params->rss_caps = ECORE_RSS_IPV4 | ECORE_RSS_IPV6 |
6510                                        ECORE_RSS_IPV4_TCP | ECORE_RSS_IPV6_TCP;
6511                 rss_params->rss_table_size_log = 7; /* 2^7 = 128 */
6512 
6513                 for (i = 0; i < ECORE_RSS_IND_TABLE_SIZE; i++) {
6514                 	fp = &ha->fp_array[(i % ha->num_rss)];
6515                         rss_params->rss_ind_table[i] = fp->rxq->handle;
6516 		}
6517 
6518                 for (i = 0; i < ECORE_RSS_KEY_SIZE; i++)
6519 			rss_params->rss_key[i] = (__le32)qlnx_hash_key[i];
6520 
6521         } else {
6522                 memset(rss_params, 0, sizeof(*rss_params));
6523         }
6524 
6525 
6526         /* Prepare and send the vport enable */
6527         memset(&vport_update_params, 0, sizeof(vport_update_params));
6528         vport_update_params.vport_id = vport_id;
6529         vport_update_params.update_vport_active_tx_flg = 1;
6530         vport_update_params.vport_active_tx_flg = 1;
6531         vport_update_params.update_vport_active_rx_flg = 1;
6532         vport_update_params.vport_active_rx_flg = 1;
6533         vport_update_params.rss_params = rss_params;
6534         vport_update_params.update_inner_vlan_removal_flg = 1;
6535         vport_update_params.inner_vlan_removal_flg = 1;
6536 
6537 	if (hw_lro_enable) {
6538 		memset(&tpa_params, 0, sizeof (struct ecore_sge_tpa_params));
6539 
6540 		tpa_params.max_buffers_per_cqe = QLNX_TPA_MAX_AGG_BUFFERS;
6541 
6542 		tpa_params.update_tpa_en_flg = 1;
6543 		tpa_params.tpa_ipv4_en_flg = 1;
6544 		tpa_params.tpa_ipv6_en_flg = 1;
6545 
6546 		tpa_params.update_tpa_param_flg = 1;
6547 		tpa_params.tpa_pkt_split_flg = 0;
6548 		tpa_params.tpa_hdr_data_split_flg = 0;
6549 		tpa_params.tpa_gro_consistent_flg = 0;
6550 		tpa_params.tpa_max_aggs_num = ETH_TPA_MAX_AGGS_NUM;
6551 		tpa_params.tpa_max_size = (uint16_t)(-1);
6552 		tpa_params.tpa_min_size_to_start = ifp->if_mtu/2;
6553 		tpa_params.tpa_min_size_to_cont = ifp->if_mtu/2;
6554 
6555 		vport_update_params.sge_tpa_params = &tpa_params;
6556 	}
6557 
6558         rc = qlnx_update_vport(cdev, &vport_update_params);
6559         if (rc) {
6560 		QL_DPRINT1(ha, "Update V-PORT failed %d\n", rc);
6561                 return rc;
6562         }
6563 
6564         return 0;
6565 }
6566 
6567 static int
6568 qlnx_drain_txq(qlnx_host_t *ha, struct qlnx_fastpath *fp,
6569 	struct qlnx_tx_queue *txq)
6570 {
6571 	uint16_t	hw_bd_cons;
6572 	uint16_t	ecore_cons_idx;
6573 
6574 	QL_DPRINT2(ha, "enter\n");
6575 
6576 	hw_bd_cons = le16toh(*txq->hw_cons_ptr);
6577 
6578 	while (hw_bd_cons !=
6579 		(ecore_cons_idx = ecore_chain_get_cons_idx(&txq->tx_pbl))) {
6580 
6581 		mtx_lock(&fp->tx_mtx);
6582 
6583 		(void)qlnx_tx_int(ha, fp, txq);
6584 
6585 		mtx_unlock(&fp->tx_mtx);
6586 
6587 		qlnx_mdelay(__func__, 2);
6588 
6589 		hw_bd_cons = le16toh(*txq->hw_cons_ptr);
6590 	}
6591 
6592 	QL_DPRINT2(ha, "[%d, %d]: done\n", fp->rss_id, txq->index);
6593 
6594         return 0;
6595 }
6596 
6597 static int
6598 qlnx_stop_queues(qlnx_host_t *ha)
6599 {
6600         struct qlnx_update_vport_params	vport_update_params;
6601         struct ecore_dev		*cdev;
6602         struct qlnx_fastpath		*fp;
6603         int				rc, tc, i;
6604 
6605         cdev = &ha->cdev;
6606 
6607         /* Disable the vport */
6608 
6609         memset(&vport_update_params, 0, sizeof(vport_update_params));
6610 
6611         vport_update_params.vport_id = 0;
6612         vport_update_params.update_vport_active_tx_flg = 1;
6613         vport_update_params.vport_active_tx_flg = 0;
6614         vport_update_params.update_vport_active_rx_flg = 1;
6615         vport_update_params.vport_active_rx_flg = 0;
6616         vport_update_params.rss_params = &ha->rss_params;
6617         vport_update_params.rss_params->update_rss_config = 0;
6618         vport_update_params.rss_params->rss_enable = 0;
6619         vport_update_params.update_inner_vlan_removal_flg = 0;
6620         vport_update_params.inner_vlan_removal_flg = 0;
6621 
6622 	QL_DPRINT1(ha, "Update vport ID= %d\n", vport_update_params.vport_id);
6623 
6624         rc = qlnx_update_vport(cdev, &vport_update_params);
6625         if (rc) {
6626 		QL_DPRINT1(ha, "Failed to update vport\n");
6627                 return rc;
6628         }
6629 
6630         /* Flush Tx queues. If needed, request drain from MCP */
6631         for_each_rss(i) {
6632                 fp = &ha->fp_array[i];
6633 
6634                 for (tc = 0; tc < ha->num_tc; tc++) {
6635                         struct qlnx_tx_queue *txq = fp->txq[tc];
6636 
6637                         rc = qlnx_drain_txq(ha, fp, txq);
6638                         if (rc)
6639                                 return rc;
6640                 }
6641         }
6642 
6643         /* Stop all Queues in reverse order*/
6644         for (i = ha->num_rss - 1; i >= 0; i--) {
6645 
6646 		struct ecore_hwfn *p_hwfn = &cdev->hwfns[(i % cdev->num_hwfns)];
6647 
6648                 fp = &ha->fp_array[i];
6649 
6650                 /* Stop the Tx Queue(s)*/
6651                 for (tc = 0; tc < ha->num_tc; tc++) {
6652 			int tx_queue_id;
6653 
6654 			tx_queue_id = tc * ha->num_rss + i;
6655 			rc = ecore_eth_tx_queue_stop(p_hwfn,
6656 					fp->txq[tc]->handle);
6657 
6658                         if (rc) {
6659 				QL_DPRINT1(ha, "Failed to stop TXQ #%d\n",
6660 					   tx_queue_id);
6661                                 return rc;
6662                         }
6663                 }
6664 
6665                 /* Stop the Rx Queue*/
6666 		rc = ecore_eth_rx_queue_stop(p_hwfn, fp->rxq->handle, false,
6667 				false);
6668                 if (rc) {
6669                         QL_DPRINT1(ha, "Failed to stop RXQ #%d\n", i);
6670                         return rc;
6671                 }
6672         }
6673 
6674         /* Stop the vport */
6675 	for_each_hwfn(cdev, i) {
6676 
6677 		struct ecore_hwfn *p_hwfn = &cdev->hwfns[i];
6678 
6679 		rc = ecore_sp_vport_stop(p_hwfn, p_hwfn->hw_info.opaque_fid, 0);
6680 
6681 		if (rc) {
6682                         QL_DPRINT1(ha, "Failed to stop VPORT\n");
6683 			return rc;
6684 		}
6685 	}
6686 
6687         return rc;
6688 }
6689 
6690 static int
6691 qlnx_set_ucast_rx_mac(qlnx_host_t *ha,
6692 	enum ecore_filter_opcode opcode,
6693 	unsigned char mac[ETH_ALEN])
6694 {
6695 	struct ecore_filter_ucast	ucast;
6696 	struct ecore_dev		*cdev;
6697 	int				rc;
6698 
6699 	cdev = &ha->cdev;
6700 
6701 	bzero(&ucast, sizeof(struct ecore_filter_ucast));
6702 
6703         ucast.opcode = opcode;
6704         ucast.type = ECORE_FILTER_MAC;
6705         ucast.is_rx_filter = 1;
6706         ucast.vport_to_add_to = 0;
6707         memcpy(&ucast.mac[0], mac, ETH_ALEN);
6708 
6709 	rc = ecore_filter_ucast_cmd(cdev, &ucast, ECORE_SPQ_MODE_CB, NULL);
6710 
6711         return (rc);
6712 }
6713 
6714 static int
6715 qlnx_remove_all_ucast_mac(qlnx_host_t *ha)
6716 {
6717 	struct ecore_filter_ucast	ucast;
6718 	struct ecore_dev		*cdev;
6719 	int				rc;
6720 
6721 	bzero(&ucast, sizeof(struct ecore_filter_ucast));
6722 
6723 	ucast.opcode = ECORE_FILTER_REPLACE;
6724 	ucast.type = ECORE_FILTER_MAC;
6725 	ucast.is_rx_filter = 1;
6726 
6727 	cdev = &ha->cdev;
6728 
6729 	rc = ecore_filter_ucast_cmd(cdev, &ucast, ECORE_SPQ_MODE_CB, NULL);
6730 
6731 	return (rc);
6732 }
6733 
6734 static int
6735 qlnx_remove_all_mcast_mac(qlnx_host_t *ha)
6736 {
6737 	struct ecore_filter_mcast	*mcast;
6738 	struct ecore_dev		*cdev;
6739 	int				rc, i;
6740 
6741 	cdev = &ha->cdev;
6742 
6743 	mcast = &ha->ecore_mcast;
6744 	bzero(mcast, sizeof(struct ecore_filter_mcast));
6745 
6746 	mcast->opcode = ECORE_FILTER_REMOVE;
6747 
6748 	for (i = 0; i < QLNX_MAX_NUM_MULTICAST_ADDRS; i++) {
6749 
6750 		if (ha->mcast[i].addr[0] || ha->mcast[i].addr[1] ||
6751 			ha->mcast[i].addr[2] || ha->mcast[i].addr[3] ||
6752 			ha->mcast[i].addr[4] || ha->mcast[i].addr[5]) {
6753 
6754 			memcpy(&mcast->mac[i], &ha->mcast[i].addr[0], ETH_ALEN);
6755 			mcast->num_mc_addrs++;
6756 		}
6757 	}
6758 	mcast = &ha->ecore_mcast;
6759 
6760 	rc = ecore_filter_mcast_cmd(cdev, mcast, ECORE_SPQ_MODE_CB, NULL);
6761 
6762 	bzero(ha->mcast, (sizeof(qlnx_mcast_t) * QLNX_MAX_NUM_MULTICAST_ADDRS));
6763 	ha->nmcast = 0;
6764 
6765 	return (rc);
6766 }
6767 
6768 static int
6769 qlnx_clean_filters(qlnx_host_t *ha)
6770 {
6771         int	rc = 0;
6772 
6773 	/* Remove all unicast macs */
6774 	rc = qlnx_remove_all_ucast_mac(ha);
6775 	if (rc)
6776 		return rc;
6777 
6778 	/* Remove all multicast macs */
6779 	rc = qlnx_remove_all_mcast_mac(ha);
6780 	if (rc)
6781 		return rc;
6782 
6783         rc = qlnx_set_ucast_rx_mac(ha, ECORE_FILTER_FLUSH, ha->primary_mac);
6784 
6785         return (rc);
6786 }
6787 
6788 static int
6789 qlnx_set_rx_accept_filter(qlnx_host_t *ha, uint8_t filter)
6790 {
6791 	struct ecore_filter_accept_flags	accept;
6792 	int					rc = 0;
6793 	struct ecore_dev			*cdev;
6794 
6795 	cdev = &ha->cdev;
6796 
6797 	bzero(&accept, sizeof(struct ecore_filter_accept_flags));
6798 
6799 	accept.update_rx_mode_config = 1;
6800 	accept.rx_accept_filter = filter;
6801 
6802 	accept.update_tx_mode_config = 1;
6803 	accept.tx_accept_filter = ECORE_ACCEPT_UCAST_MATCHED |
6804 		ECORE_ACCEPT_MCAST_MATCHED | ECORE_ACCEPT_BCAST;
6805 
6806 	rc = ecore_filter_accept_cmd(cdev, 0, accept, false, false,
6807 			ECORE_SPQ_MODE_CB, NULL);
6808 
6809 	return (rc);
6810 }
6811 
6812 static int
6813 qlnx_set_rx_mode(qlnx_host_t *ha)
6814 {
6815 	int	rc = 0;
6816 	uint8_t	filter;
6817 
6818 	rc = qlnx_set_ucast_rx_mac(ha, ECORE_FILTER_REPLACE, ha->primary_mac);
6819         if (rc)
6820                 return rc;
6821 
6822 	rc = qlnx_remove_all_mcast_mac(ha);
6823         if (rc)
6824                 return rc;
6825 
6826 	filter = ECORE_ACCEPT_UCAST_MATCHED |
6827 			ECORE_ACCEPT_MCAST_MATCHED |
6828 			ECORE_ACCEPT_BCAST;
6829 	ha->filter = filter;
6830 
6831 	rc = qlnx_set_rx_accept_filter(ha, filter);
6832 
6833 	return (rc);
6834 }
6835 
6836 static int
6837 qlnx_set_link(qlnx_host_t *ha, bool link_up)
6838 {
6839         int			i, rc = 0;
6840 	struct ecore_dev	*cdev;
6841 	struct ecore_hwfn	*hwfn;
6842 	struct ecore_ptt	*ptt;
6843 
6844 	cdev = &ha->cdev;
6845 
6846         for_each_hwfn(cdev, i) {
6847 
6848                 hwfn = &cdev->hwfns[i];
6849 
6850                 ptt = ecore_ptt_acquire(hwfn);
6851        	        if (!ptt)
6852                         return -EBUSY;
6853 
6854                 rc = ecore_mcp_set_link(hwfn, ptt, link_up);
6855 
6856                 ecore_ptt_release(hwfn, ptt);
6857 
6858                 if (rc)
6859                         return rc;
6860         }
6861         return (rc);
6862 }
6863 
6864 #if __FreeBSD_version >= 1100000
6865 static uint64_t
6866 qlnx_get_counter(if_t ifp, ift_counter cnt)
6867 {
6868 	qlnx_host_t *ha;
6869 	uint64_t count;
6870 
6871         ha = (qlnx_host_t *)if_getsoftc(ifp);
6872 
6873         switch (cnt) {
6874 
6875         case IFCOUNTER_IPACKETS:
6876 		count = ha->hw_stats.common.rx_ucast_pkts +
6877 			ha->hw_stats.common.rx_mcast_pkts +
6878 			ha->hw_stats.common.rx_bcast_pkts;
6879 		break;
6880 
6881         case IFCOUNTER_IERRORS:
6882 		count = ha->hw_stats.common.rx_crc_errors +
6883 			ha->hw_stats.common.rx_align_errors +
6884 			ha->hw_stats.common.rx_oversize_packets +
6885 			ha->hw_stats.common.rx_undersize_packets;
6886 		break;
6887 
6888         case IFCOUNTER_OPACKETS:
6889 		count = ha->hw_stats.common.tx_ucast_pkts +
6890 			ha->hw_stats.common.tx_mcast_pkts +
6891 			ha->hw_stats.common.tx_bcast_pkts;
6892 		break;
6893 
6894         case IFCOUNTER_OERRORS:
6895                 count = ha->hw_stats.common.tx_err_drop_pkts;
6896 		break;
6897 
6898         case IFCOUNTER_COLLISIONS:
6899                 return (0);
6900 
6901         case IFCOUNTER_IBYTES:
6902 		count = ha->hw_stats.common.rx_ucast_bytes +
6903 			ha->hw_stats.common.rx_mcast_bytes +
6904 			ha->hw_stats.common.rx_bcast_bytes;
6905 		break;
6906 
6907         case IFCOUNTER_OBYTES:
6908 		count = ha->hw_stats.common.tx_ucast_bytes +
6909 			ha->hw_stats.common.tx_mcast_bytes +
6910 			ha->hw_stats.common.tx_bcast_bytes;
6911 		break;
6912 
6913         case IFCOUNTER_IMCASTS:
6914 		count = ha->hw_stats.common.rx_mcast_bytes;
6915 		break;
6916 
6917         case IFCOUNTER_OMCASTS:
6918 		count = ha->hw_stats.common.tx_mcast_bytes;
6919 		break;
6920 
6921         case IFCOUNTER_IQDROPS:
6922         case IFCOUNTER_OQDROPS:
6923         case IFCOUNTER_NOPROTO:
6924 
6925         default:
6926                 return (if_get_counter_default(ifp, cnt));
6927         }
6928 	return (count);
6929 }
6930 #endif
6931 
6932 
6933 static void
6934 qlnx_timer(void *arg)
6935 {
6936 	qlnx_host_t	*ha;
6937 
6938 	ha = (qlnx_host_t *)arg;
6939 
6940        	ecore_get_vport_stats(&ha->cdev, &ha->hw_stats);
6941 
6942 	if (ha->storm_stats_gather)
6943 		qlnx_sample_storm_stats(ha);
6944 
6945 	callout_reset(&ha->qlnx_callout, hz, qlnx_timer, ha);
6946 
6947 	return;
6948 }
6949 
6950 static int
6951 qlnx_load(qlnx_host_t *ha)
6952 {
6953 	int			i;
6954 	int			rc = 0;
6955 	struct ecore_dev	*cdev;
6956         device_t		dev;
6957 
6958 	cdev = &ha->cdev;
6959         dev = ha->pci_dev;
6960 
6961 	QL_DPRINT2(ha, "enter\n");
6962 
6963         rc = qlnx_alloc_mem_arrays(ha);
6964         if (rc)
6965                 goto qlnx_load_exit0;
6966 
6967         qlnx_init_fp(ha);
6968 
6969         rc = qlnx_alloc_mem_load(ha);
6970         if (rc)
6971                 goto qlnx_load_exit1;
6972 
6973         QL_DPRINT2(ha, "Allocated %d RSS queues on %d TC/s\n",
6974 		   ha->num_rss, ha->num_tc);
6975 
6976 	for (i = 0; i < ha->num_rss; i++) {
6977 
6978 		if ((rc = bus_setup_intr(dev, ha->irq_vec[i].irq,
6979                         (INTR_TYPE_NET | INTR_MPSAFE),
6980                         NULL, qlnx_fp_isr, &ha->irq_vec[i],
6981                         &ha->irq_vec[i].handle))) {
6982 
6983                         QL_DPRINT1(ha, "could not setup interrupt\n");
6984                         goto qlnx_load_exit2;
6985 		}
6986 
6987 		QL_DPRINT2(ha, "rss_id = %d irq_rid %d \
6988 			 irq %p handle %p\n", i,
6989 			ha->irq_vec[i].irq_rid,
6990 			ha->irq_vec[i].irq, ha->irq_vec[i].handle);
6991 
6992 		bus_bind_intr(dev, ha->irq_vec[i].irq, (i % mp_ncpus));
6993 	}
6994 
6995         rc = qlnx_start_queues(ha);
6996         if (rc)
6997                 goto qlnx_load_exit2;
6998 
6999         QL_DPRINT2(ha, "Start VPORT, RXQ and TXQ succeeded\n");
7000 
7001         /* Add primary mac and set Rx filters */
7002         rc = qlnx_set_rx_mode(ha);
7003         if (rc)
7004                 goto qlnx_load_exit2;
7005 
7006         /* Ask for link-up using current configuration */
7007 	qlnx_set_link(ha, true);
7008 
7009         ha->state = QLNX_STATE_OPEN;
7010 
7011 	bzero(&ha->hw_stats, sizeof(struct ecore_eth_stats));
7012 
7013 	if (ha->flags.callout_init)
7014         	callout_reset(&ha->qlnx_callout, hz, qlnx_timer, ha);
7015 
7016         goto qlnx_load_exit0;
7017 
7018 qlnx_load_exit2:
7019         qlnx_free_mem_load(ha);
7020 
7021 qlnx_load_exit1:
7022         ha->num_rss = 0;
7023 
7024 qlnx_load_exit0:
7025 	QL_DPRINT2(ha, "exit [%d]\n", rc);
7026         return rc;
7027 }
7028 
7029 static void
7030 qlnx_drain_soft_lro(qlnx_host_t *ha)
7031 {
7032 #ifdef QLNX_SOFT_LRO
7033 
7034 	struct ifnet	*ifp;
7035 	int		i;
7036 
7037 	ifp = ha->ifp;
7038 
7039 
7040 	if (ifp->if_capenable & IFCAP_LRO) {
7041 
7042 	        for (i = 0; i < ha->num_rss; i++) {
7043 
7044 			struct qlnx_fastpath *fp = &ha->fp_array[i];
7045 			struct lro_ctrl *lro;
7046 
7047 			lro = &fp->rxq->lro;
7048 
7049 #if (__FreeBSD_version >= 1100101) || (defined QLNX_QSORT_LRO)
7050 
7051 			tcp_lro_flush_all(lro);
7052 
7053 #else
7054 			struct lro_entry *queued;
7055 
7056 			while ((!SLIST_EMPTY(&lro->lro_active))){
7057 				queued = SLIST_FIRST(&lro->lro_active);
7058 				SLIST_REMOVE_HEAD(&lro->lro_active, next);
7059 				tcp_lro_flush(lro, queued);
7060 			}
7061 
7062 #endif /* #if (__FreeBSD_version >= 1100101) || (defined QLNX_QSORT_LRO) */
7063 
7064                 }
7065 	}
7066 
7067 #endif /* #ifdef QLNX_SOFT_LRO */
7068 
7069 	return;
7070 }
7071 
7072 static void
7073 qlnx_unload(qlnx_host_t *ha)
7074 {
7075 	struct ecore_dev	*cdev;
7076         device_t		dev;
7077 	int			i;
7078 
7079 	cdev = &ha->cdev;
7080         dev = ha->pci_dev;
7081 
7082 	QL_DPRINT2(ha, "enter\n");
7083         QL_DPRINT1(ha, " QLNX STATE = %d\n",ha->state);
7084 
7085 	if (ha->state == QLNX_STATE_OPEN) {
7086 
7087 		qlnx_set_link(ha, false);
7088 		qlnx_clean_filters(ha);
7089 		qlnx_stop_queues(ha);
7090 		ecore_hw_stop_fastpath(cdev);
7091 
7092 		for (i = 0; i < ha->num_rss; i++) {
7093 			if (ha->irq_vec[i].handle) {
7094 				(void)bus_teardown_intr(dev,
7095 					ha->irq_vec[i].irq,
7096 					ha->irq_vec[i].handle);
7097 				ha->irq_vec[i].handle = NULL;
7098 			}
7099 		}
7100 
7101 		qlnx_drain_fp_taskqueues(ha);
7102 		qlnx_drain_soft_lro(ha);
7103         	qlnx_free_mem_load(ha);
7104 	}
7105 
7106 	if (ha->flags.callout_init)
7107 		callout_drain(&ha->qlnx_callout);
7108 
7109 	qlnx_mdelay(__func__, 1000);
7110 
7111         ha->state = QLNX_STATE_CLOSED;
7112 
7113 	QL_DPRINT2(ha, "exit\n");
7114 	return;
7115 }
7116 
7117 static int
7118 qlnx_grc_dumpsize(qlnx_host_t *ha, uint32_t *num_dwords, int hwfn_index)
7119 {
7120 	int			rval = -1;
7121 	struct ecore_hwfn	*p_hwfn;
7122 	struct ecore_ptt	*p_ptt;
7123 
7124 	ecore_dbg_set_app_ver(ecore_dbg_get_fw_func_ver());
7125 
7126 	p_hwfn = &ha->cdev.hwfns[hwfn_index];
7127 	p_ptt = ecore_ptt_acquire(p_hwfn);
7128 
7129         if (!p_ptt) {
7130 		QL_DPRINT1(ha, "ecore_ptt_acquire failed\n");
7131                 return (rval);
7132         }
7133 
7134         rval = ecore_dbg_grc_get_dump_buf_size(p_hwfn, p_ptt, num_dwords);
7135 
7136 	if (rval == DBG_STATUS_OK)
7137                 rval = 0;
7138         else {
7139 		QL_DPRINT1(ha, "ecore_dbg_grc_get_dump_buf_size failed"
7140 			"[0x%x]\n", rval);
7141 	}
7142 
7143         ecore_ptt_release(p_hwfn, p_ptt);
7144 
7145         return (rval);
7146 }
7147 
7148 static int
7149 qlnx_idle_chk_size(qlnx_host_t *ha, uint32_t *num_dwords, int hwfn_index)
7150 {
7151 	int			rval = -1;
7152 	struct ecore_hwfn	*p_hwfn;
7153 	struct ecore_ptt	*p_ptt;
7154 
7155 	ecore_dbg_set_app_ver(ecore_dbg_get_fw_func_ver());
7156 
7157 	p_hwfn = &ha->cdev.hwfns[hwfn_index];
7158 	p_ptt = ecore_ptt_acquire(p_hwfn);
7159 
7160         if (!p_ptt) {
7161 		QL_DPRINT1(ha, "ecore_ptt_acquire failed\n");
7162                 return (rval);
7163         }
7164 
7165         rval = ecore_dbg_idle_chk_get_dump_buf_size(p_hwfn, p_ptt, num_dwords);
7166 
7167 	if (rval == DBG_STATUS_OK)
7168                 rval = 0;
7169         else {
7170 		QL_DPRINT1(ha, "ecore_dbg_idle_chk_get_dump_buf_size failed"
7171 			" [0x%x]\n", rval);
7172 	}
7173 
7174         ecore_ptt_release(p_hwfn, p_ptt);
7175 
7176         return (rval);
7177 }
7178 
7179 
7180 static void
7181 qlnx_sample_storm_stats(qlnx_host_t *ha)
7182 {
7183         int			i, index;
7184         struct ecore_dev	*cdev;
7185 	qlnx_storm_stats_t	*s_stats;
7186 	uint32_t		reg;
7187         struct ecore_ptt	*p_ptt;
7188         struct ecore_hwfn	*hwfn;
7189 
7190 	if (ha->storm_stats_index >= QLNX_STORM_STATS_SAMPLES_PER_HWFN) {
7191 		ha->storm_stats_gather = 0;
7192 		return;
7193 	}
7194 
7195         cdev = &ha->cdev;
7196 
7197         for_each_hwfn(cdev, i) {
7198 
7199                 hwfn = &cdev->hwfns[i];
7200 
7201                 p_ptt = ecore_ptt_acquire(hwfn);
7202                 if (!p_ptt)
7203                         return;
7204 
7205 		index = ha->storm_stats_index +
7206 				(i * QLNX_STORM_STATS_SAMPLES_PER_HWFN);
7207 
7208 		s_stats = &ha->storm_stats[index];
7209 
7210 		/* XSTORM */
7211 		reg = XSEM_REG_FAST_MEMORY +
7212 				SEM_FAST_REG_STORM_ACTIVE_CYCLES_BB_K2;
7213 		s_stats->xstorm_active_cycles = ecore_rd(hwfn, p_ptt, reg);
7214 
7215 		reg = XSEM_REG_FAST_MEMORY +
7216 				SEM_FAST_REG_STORM_STALL_CYCLES_BB_K2;
7217 		s_stats->xstorm_stall_cycles = ecore_rd(hwfn, p_ptt, reg);
7218 
7219 		reg = XSEM_REG_FAST_MEMORY +
7220 				SEM_FAST_REG_IDLE_SLEEPING_CYCLES_BB_K2;
7221 		s_stats->xstorm_sleeping_cycles = ecore_rd(hwfn, p_ptt, reg);
7222 
7223 		reg = XSEM_REG_FAST_MEMORY +
7224 				SEM_FAST_REG_IDLE_INACTIVE_CYCLES_BB_K2;
7225 		s_stats->xstorm_inactive_cycles = ecore_rd(hwfn, p_ptt, reg);
7226 
7227 		/* YSTORM */
7228 		reg = YSEM_REG_FAST_MEMORY +
7229 				SEM_FAST_REG_STORM_ACTIVE_CYCLES_BB_K2;
7230 		s_stats->ystorm_active_cycles = ecore_rd(hwfn, p_ptt, reg);
7231 
7232 		reg = YSEM_REG_FAST_MEMORY +
7233 				SEM_FAST_REG_STORM_STALL_CYCLES_BB_K2;
7234 		s_stats->ystorm_stall_cycles = ecore_rd(hwfn, p_ptt, reg);
7235 
7236 		reg = YSEM_REG_FAST_MEMORY +
7237 				SEM_FAST_REG_IDLE_SLEEPING_CYCLES_BB_K2;
7238 		s_stats->ystorm_sleeping_cycles = ecore_rd(hwfn, p_ptt, reg);
7239 
7240 		reg = YSEM_REG_FAST_MEMORY +
7241 				SEM_FAST_REG_IDLE_INACTIVE_CYCLES_BB_K2;
7242 		s_stats->ystorm_inactive_cycles = ecore_rd(hwfn, p_ptt, reg);
7243 
7244 		/* PSTORM */
7245 		reg = PSEM_REG_FAST_MEMORY +
7246 				SEM_FAST_REG_STORM_ACTIVE_CYCLES_BB_K2;
7247 		s_stats->pstorm_active_cycles = ecore_rd(hwfn, p_ptt, reg);
7248 
7249 		reg = PSEM_REG_FAST_MEMORY +
7250 				SEM_FAST_REG_STORM_STALL_CYCLES_BB_K2;
7251 		s_stats->pstorm_stall_cycles = ecore_rd(hwfn, p_ptt, reg);
7252 
7253 		reg = PSEM_REG_FAST_MEMORY +
7254 				SEM_FAST_REG_IDLE_SLEEPING_CYCLES_BB_K2;
7255 		s_stats->pstorm_sleeping_cycles = ecore_rd(hwfn, p_ptt, reg);
7256 
7257 		reg = PSEM_REG_FAST_MEMORY +
7258 				SEM_FAST_REG_IDLE_INACTIVE_CYCLES_BB_K2;
7259 		s_stats->pstorm_inactive_cycles = ecore_rd(hwfn, p_ptt, reg);
7260 
7261 		/* TSTORM */
7262 		reg = TSEM_REG_FAST_MEMORY +
7263 				SEM_FAST_REG_STORM_ACTIVE_CYCLES_BB_K2;
7264 		s_stats->tstorm_active_cycles = ecore_rd(hwfn, p_ptt, reg);
7265 
7266 		reg = TSEM_REG_FAST_MEMORY +
7267 				SEM_FAST_REG_STORM_STALL_CYCLES_BB_K2;
7268 		s_stats->tstorm_stall_cycles = ecore_rd(hwfn, p_ptt, reg);
7269 
7270 		reg = TSEM_REG_FAST_MEMORY +
7271 				SEM_FAST_REG_IDLE_SLEEPING_CYCLES_BB_K2;
7272 		s_stats->tstorm_sleeping_cycles = ecore_rd(hwfn, p_ptt, reg);
7273 
7274 		reg = TSEM_REG_FAST_MEMORY +
7275 				SEM_FAST_REG_IDLE_INACTIVE_CYCLES_BB_K2;
7276 		s_stats->tstorm_inactive_cycles = ecore_rd(hwfn, p_ptt, reg);
7277 
7278 		/* MSTORM */
7279 		reg = MSEM_REG_FAST_MEMORY +
7280 				SEM_FAST_REG_STORM_ACTIVE_CYCLES_BB_K2;
7281 		s_stats->mstorm_active_cycles = ecore_rd(hwfn, p_ptt, reg);
7282 
7283 		reg = MSEM_REG_FAST_MEMORY +
7284 				SEM_FAST_REG_STORM_STALL_CYCLES_BB_K2;
7285 		s_stats->mstorm_stall_cycles = ecore_rd(hwfn, p_ptt, reg);
7286 
7287 		reg = MSEM_REG_FAST_MEMORY +
7288 				SEM_FAST_REG_IDLE_SLEEPING_CYCLES_BB_K2;
7289 		s_stats->mstorm_sleeping_cycles = ecore_rd(hwfn, p_ptt, reg);
7290 
7291 		reg = MSEM_REG_FAST_MEMORY +
7292 				SEM_FAST_REG_IDLE_INACTIVE_CYCLES_BB_K2;
7293 		s_stats->mstorm_inactive_cycles = ecore_rd(hwfn, p_ptt, reg);
7294 
7295 		/* USTORM */
7296 		reg = USEM_REG_FAST_MEMORY +
7297 				SEM_FAST_REG_STORM_ACTIVE_CYCLES_BB_K2;
7298 		s_stats->ustorm_active_cycles = ecore_rd(hwfn, p_ptt, reg);
7299 
7300 		reg = USEM_REG_FAST_MEMORY +
7301 				SEM_FAST_REG_STORM_STALL_CYCLES_BB_K2;
7302 		s_stats->ustorm_stall_cycles = ecore_rd(hwfn, p_ptt, reg);
7303 
7304 		reg = USEM_REG_FAST_MEMORY +
7305 				SEM_FAST_REG_IDLE_SLEEPING_CYCLES_BB_K2;
7306 		s_stats->ustorm_sleeping_cycles = ecore_rd(hwfn, p_ptt, reg);
7307 
7308 		reg = USEM_REG_FAST_MEMORY +
7309 				SEM_FAST_REG_IDLE_INACTIVE_CYCLES_BB_K2;
7310 		s_stats->ustorm_inactive_cycles = ecore_rd(hwfn, p_ptt, reg);
7311 
7312                 ecore_ptt_release(hwfn, p_ptt);
7313         }
7314 
7315 	ha->storm_stats_index++;
7316 
7317         return;
7318 }
7319 
7320 /*
7321  * Name: qlnx_dump_buf8
7322  * Function: dumps a buffer as bytes
7323  */
7324 static void
7325 qlnx_dump_buf8(qlnx_host_t *ha, const char *msg, void *dbuf, uint32_t len)
7326 {
7327         device_t	dev;
7328         uint32_t	i = 0;
7329         uint8_t		*buf;
7330 
7331         dev = ha->pci_dev;
7332         buf = dbuf;
7333 
7334         device_printf(dev, "%s: %s 0x%x dump start\n", __func__, msg, len);
7335 
7336         while (len >= 16) {
7337                 device_printf(dev,"0x%08x:"
7338                         " %02x %02x %02x %02x %02x %02x %02x %02x"
7339                         " %02x %02x %02x %02x %02x %02x %02x %02x\n", i,
7340                         buf[0], buf[1], buf[2], buf[3],
7341                         buf[4], buf[5], buf[6], buf[7],
7342                         buf[8], buf[9], buf[10], buf[11],
7343                         buf[12], buf[13], buf[14], buf[15]);
7344                 i += 16;
7345                 len -= 16;
7346                 buf += 16;
7347         }
7348         switch (len) {
7349         case 1:
7350                 device_printf(dev,"0x%08x: %02x\n", i, buf[0]);
7351                 break;
7352         case 2:
7353                 device_printf(dev,"0x%08x: %02x %02x\n", i, buf[0], buf[1]);
7354                 break;
7355         case 3:
7356                 device_printf(dev,"0x%08x: %02x %02x %02x\n",
7357                         i, buf[0], buf[1], buf[2]);
7358                 break;
7359         case 4:
7360                 device_printf(dev,"0x%08x: %02x %02x %02x %02x\n", i,
7361                         buf[0], buf[1], buf[2], buf[3]);
7362                 break;
7363         case 5:
7364                 device_printf(dev,"0x%08x:"
7365                         " %02x %02x %02x %02x %02x\n", i,
7366                         buf[0], buf[1], buf[2], buf[3], buf[4]);
7367                 break;
7368         case 6:
7369                 device_printf(dev,"0x%08x:"
7370                         " %02x %02x %02x %02x %02x %02x\n", i,
7371                         buf[0], buf[1], buf[2], buf[3], buf[4], buf[5]);
7372                 break;
7373         case 7:
7374                 device_printf(dev,"0x%08x:"
7375                         " %02x %02x %02x %02x %02x %02x %02x\n", i,
7376                         buf[0], buf[1], buf[2], buf[3], buf[4], buf[5], buf[6]);
7377                 break;
7378         case 8:
7379                 device_printf(dev,"0x%08x:"
7380                         " %02x %02x %02x %02x %02x %02x %02x %02x\n", i,
7381                         buf[0], buf[1], buf[2], buf[3], buf[4], buf[5], buf[6],
7382                         buf[7]);
7383                 break;
7384         case 9:
7385                 device_printf(dev,"0x%08x:"
7386                         " %02x %02x %02x %02x %02x %02x %02x %02x"
7387                         " %02x\n", i,
7388                         buf[0], buf[1], buf[2], buf[3], buf[4], buf[5], buf[6],
7389                         buf[7], buf[8]);
7390                 break;
7391         case 10:
7392                 device_printf(dev,"0x%08x:"
7393                         " %02x %02x %02x %02x %02x %02x %02x %02x"
7394                         " %02x %02x\n", i,
7395                         buf[0], buf[1], buf[2], buf[3], buf[4], buf[5], buf[6],
7396                         buf[7], buf[8], buf[9]);
7397                 break;
7398         case 11:
7399                 device_printf(dev,"0x%08x:"
7400                         " %02x %02x %02x %02x %02x %02x %02x %02x"
7401                         " %02x %02x %02x\n", i,
7402                         buf[0], buf[1], buf[2], buf[3], buf[4], buf[5], buf[6],
7403                         buf[7], buf[8], buf[9], buf[10]);
7404                 break;
7405         case 12:
7406                 device_printf(dev,"0x%08x:"
7407                         " %02x %02x %02x %02x %02x %02x %02x %02x"
7408                         " %02x %02x %02x %02x\n", i,
7409                         buf[0], buf[1], buf[2], buf[3], buf[4], buf[5], buf[6],
7410                         buf[7], buf[8], buf[9], buf[10], buf[11]);
7411                 break;
7412         case 13:
7413                 device_printf(dev,"0x%08x:"
7414                         " %02x %02x %02x %02x %02x %02x %02x %02x"
7415                         " %02x %02x %02x %02x %02x\n", i,
7416                         buf[0], buf[1], buf[2], buf[3], buf[4], buf[5], buf[6],
7417                         buf[7], buf[8], buf[9], buf[10], buf[11], buf[12]);
7418                 break;
7419         case 14:
7420                 device_printf(dev,"0x%08x:"
7421                         " %02x %02x %02x %02x %02x %02x %02x %02x"
7422                         " %02x %02x %02x %02x %02x %02x\n", i,
7423                         buf[0], buf[1], buf[2], buf[3], buf[4], buf[5], buf[6],
7424                         buf[7], buf[8], buf[9], buf[10], buf[11], buf[12],
7425                         buf[13]);
7426                 break;
7427         case 15:
7428                 device_printf(dev,"0x%08x:"
7429                         " %02x %02x %02x %02x %02x %02x %02x %02x"
7430                         " %02x %02x %02x %02x %02x %02x %02x\n", i,
7431                         buf[0], buf[1], buf[2], buf[3], buf[4], buf[5], buf[6],
7432                         buf[7], buf[8], buf[9], buf[10], buf[11], buf[12],
7433                         buf[13], buf[14]);
7434                 break;
7435         default:
7436                 break;
7437         }
7438 
7439         device_printf(dev, "%s: %s dump end\n", __func__, msg);
7440 
7441         return;
7442 }
7443 
7444