xref: /freebsd/sys/dev/ena/ena.c (revision 939d033cab9c78eba4e50cbe601f42e55ed02891)
1 /*-
2  * BSD LICENSE
3  *
4  * Copyright (c) 2015-2017 Amazon.com, Inc. or its affiliates.
5  * All rights reserved.
6  *
7  * Redistribution and use in source and binary forms, with or without
8  * modification, are permitted provided that the following conditions
9  * are met:
10  *
11  * 1. Redistributions of source code must retain the above copyright
12  *    notice, this list of conditions and the following disclaimer.
13  *
14  * 2. Redistributions in binary form must reproduce the above copyright
15  *    notice, this list of conditions and the following disclaimer in the
16  *    documentation and/or other materials provided with the distribution.
17  *
18  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21  * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22  * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24  * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29  */
30 #include <sys/cdefs.h>
31 __FBSDID("$FreeBSD$");
32 
33 #include <sys/param.h>
34 #include <sys/systm.h>
35 #include <sys/bus.h>
36 #include <sys/endian.h>
37 #include <sys/kernel.h>
38 #include <sys/kthread.h>
39 #include <sys/malloc.h>
40 #include <sys/mbuf.h>
41 #include <sys/module.h>
42 #include <sys/rman.h>
43 #include <sys/smp.h>
44 #include <sys/socket.h>
45 #include <sys/sockio.h>
46 #include <sys/sysctl.h>
47 #include <sys/taskqueue.h>
48 #include <sys/time.h>
49 #include <sys/eventhandler.h>
50 
51 #include <machine/bus.h>
52 #include <machine/resource.h>
53 #include <machine/in_cksum.h>
54 
55 #include <net/bpf.h>
56 #include <net/ethernet.h>
57 #include <net/if.h>
58 #include <net/if_var.h>
59 #include <net/if_arp.h>
60 #include <net/if_dl.h>
61 #include <net/if_media.h>
62 #include <net/rss_config.h>
63 #include <net/if_types.h>
64 #include <net/if_vlan_var.h>
65 
66 #include <netinet/in_rss.h>
67 #include <netinet/in_systm.h>
68 #include <netinet/in.h>
69 #include <netinet/if_ether.h>
70 #include <netinet/ip.h>
71 #include <netinet/ip6.h>
72 #include <netinet/tcp.h>
73 #include <netinet/udp.h>
74 
75 #include <dev/pci/pcivar.h>
76 #include <dev/pci/pcireg.h>
77 
78 #include "ena.h"
79 #include "ena_sysctl.h"
80 
81 /*********************************************************
82  *  Function prototypes
83  *********************************************************/
84 static int	ena_probe(device_t);
85 static void	ena_intr_msix_mgmnt(void *);
86 static int	ena_allocate_pci_resources(struct ena_adapter*);
87 static void	ena_free_pci_resources(struct ena_adapter *);
88 static int	ena_change_mtu(if_t, int);
89 static inline void ena_alloc_counters(counter_u64_t *, int);
90 static inline void ena_free_counters(counter_u64_t *, int);
91 static inline void ena_reset_counters(counter_u64_t *, int);
92 static void	ena_init_io_rings_common(struct ena_adapter *,
93     struct ena_ring *, uint16_t);
94 static int	ena_init_io_rings(struct ena_adapter *);
95 static void	ena_free_io_ring_resources(struct ena_adapter *, unsigned int);
96 static void	ena_free_all_io_rings_resources(struct ena_adapter *);
97 static int	ena_setup_tx_dma_tag(struct ena_adapter *);
98 static int	ena_free_tx_dma_tag(struct ena_adapter *);
99 static int	ena_setup_rx_dma_tag(struct ena_adapter *);
100 static int	ena_free_rx_dma_tag(struct ena_adapter *);
101 static int	ena_setup_tx_resources(struct ena_adapter *, int);
102 static void	ena_free_tx_resources(struct ena_adapter *, int);
103 static int	ena_setup_all_tx_resources(struct ena_adapter *);
104 static void	ena_free_all_tx_resources(struct ena_adapter *);
105 static int	ena_setup_rx_resources(struct ena_adapter *, unsigned int);
106 static void	ena_free_rx_resources(struct ena_adapter *, unsigned int);
107 static int	ena_setup_all_rx_resources(struct ena_adapter *);
108 static void	ena_free_all_rx_resources(struct ena_adapter *);
109 static inline int ena_alloc_rx_mbuf(struct ena_adapter *, struct ena_ring *,
110     struct ena_rx_buffer *);
111 static void	ena_free_rx_mbuf(struct ena_adapter *, struct ena_ring *,
112     struct ena_rx_buffer *);
113 static int	ena_refill_rx_bufs(struct ena_ring *, uint32_t);
114 static void	ena_free_rx_bufs(struct ena_adapter *, unsigned int);
115 static void	ena_refill_all_rx_bufs(struct ena_adapter *);
116 static void	ena_free_all_rx_bufs(struct ena_adapter *);
117 static void	ena_free_tx_bufs(struct ena_adapter *, unsigned int);
118 static void	ena_free_all_tx_bufs(struct ena_adapter *);
119 static void	ena_destroy_all_tx_queues(struct ena_adapter *);
120 static void	ena_destroy_all_rx_queues(struct ena_adapter *);
121 static void	ena_destroy_all_io_queues(struct ena_adapter *);
122 static int	ena_create_io_queues(struct ena_adapter *);
123 static int	ena_tx_cleanup(struct ena_ring *);
124 static int	ena_rx_cleanup(struct ena_ring *);
125 static int	validate_tx_req_id(struct ena_ring *, uint16_t);
126 static void	ena_rx_hash_mbuf(struct ena_ring *, struct ena_com_rx_ctx *,
127     struct mbuf *);
128 static struct mbuf* ena_rx_mbuf(struct ena_ring *, struct ena_com_rx_buf_info *,
129     struct ena_com_rx_ctx *, uint16_t *);
130 static inline void ena_rx_checksum(struct ena_ring *, struct ena_com_rx_ctx *,
131     struct mbuf *);
132 static void	ena_handle_msix(void *);
133 static int	ena_enable_msix(struct ena_adapter *);
134 static void	ena_setup_mgmnt_intr(struct ena_adapter *);
135 static void	ena_setup_io_intr(struct ena_adapter *);
136 static int	ena_request_mgmnt_irq(struct ena_adapter *);
137 static int	ena_request_io_irq(struct ena_adapter *);
138 static void	ena_free_mgmnt_irq(struct ena_adapter *);
139 static void	ena_free_io_irq(struct ena_adapter *);
140 static void	ena_free_irqs(struct ena_adapter*);
141 static void	ena_disable_msix(struct ena_adapter *);
142 static void	ena_unmask_all_io_irqs(struct ena_adapter *);
143 static int	ena_rss_configure(struct ena_adapter *);
144 static int	ena_up_complete(struct ena_adapter *);
145 static int	ena_up(struct ena_adapter *);
146 static void	ena_down(struct ena_adapter *);
147 static uint64_t	ena_get_counter(if_t, ift_counter);
148 static int	ena_media_change(if_t);
149 static void	ena_media_status(if_t, struct ifmediareq *);
150 static void	ena_init(void *);
151 static int	ena_ioctl(if_t, u_long, caddr_t);
152 static int	ena_get_dev_offloads(struct ena_com_dev_get_features_ctx *);
153 static void	ena_update_host_info(struct ena_admin_host_info *, if_t);
154 static void	ena_update_hwassist(struct ena_adapter *);
155 static int	ena_setup_ifnet(device_t, struct ena_adapter *,
156     struct ena_com_dev_get_features_ctx *);
157 static void	ena_tx_csum(struct ena_com_tx_ctx *, struct mbuf *);
158 static int	ena_check_and_collapse_mbuf(struct ena_ring *tx_ring,
159     struct mbuf **mbuf);
160 static int	ena_xmit_mbuf(struct ena_ring *, struct mbuf **);
161 static void	ena_start_xmit(struct ena_ring *);
162 static int	ena_mq_start(if_t, struct mbuf *);
163 static void	ena_deferred_mq_start(void *, int);
164 static void	ena_qflush(if_t);
165 static int	ena_calc_io_queue_num(struct ena_adapter *,
166     struct ena_com_dev_get_features_ctx *);
167 static int	ena_calc_queue_size(struct ena_adapter *, uint16_t *,
168     uint16_t *, struct ena_com_dev_get_features_ctx *);
169 static int	ena_rss_init_default(struct ena_adapter *);
170 static void	ena_rss_init_default_deferred(void *);
171 static void	ena_config_host_info(struct ena_com_dev *);
172 static int	ena_attach(device_t);
173 static int	ena_detach(device_t);
174 static int	ena_device_init(struct ena_adapter *, device_t,
175     struct ena_com_dev_get_features_ctx *, int *);
176 static int	ena_enable_msix_and_set_admin_interrupts(struct ena_adapter *,
177     int);
178 static void ena_update_on_link_change(void *, struct ena_admin_aenq_entry *);
179 static void	unimplemented_aenq_handler(void *,
180     struct ena_admin_aenq_entry *);
181 static void	ena_timer_service(void *);
182 
183 static char ena_version[] = DEVICE_NAME DRV_MODULE_NAME " v" DRV_MODULE_VERSION;
184 
185 static SYSCTL_NODE(_hw, OID_AUTO, ena, CTLFLAG_RD, 0, "ENA driver parameters");
186 
187 /*
188  * Tuneable number of buffers in the buf-ring (drbr)
189  */
190 static int ena_buf_ring_size = 4096;
191 SYSCTL_INT(_hw_ena, OID_AUTO, buf_ring_size, CTLFLAG_RWTUN,
192     &ena_buf_ring_size, 0, "Size of the bufring");
193 
194 
195 static ena_vendor_info_t ena_vendor_info_array[] = {
196     { PCI_VENDOR_ID_AMAZON, PCI_DEV_ID_ENA_PF, 0},
197     { PCI_VENDOR_ID_AMAZON, PCI_DEV_ID_ENA_LLQ_PF, 0},
198     { PCI_VENDOR_ID_AMAZON, PCI_DEV_ID_ENA_VF, 0},
199     { PCI_VENDOR_ID_AMAZON, PCI_DEV_ID_ENA_LLQ_VF, 0},
200     /* Last entry */
201     { 0, 0, 0 }
202 };
203 
204 /*
205  * Contains pointers to event handlers, e.g. link state chage.
206  */
207 static struct ena_aenq_handlers aenq_handlers;
208 
209 void
210 ena_dmamap_callback(void *arg, bus_dma_segment_t *segs, int nseg, int error)
211 {
212 	if (error)
213 		return;
214 	*(bus_addr_t *) arg = segs[0].ds_addr;
215 	return;
216 }
217 
218 int
219 ena_dma_alloc(device_t dmadev, bus_size_t size,
220     ena_mem_handle_t *dma , int mapflags)
221 {
222 	struct ena_adapter* adapter = device_get_softc(dmadev);
223 	uint32_t maxsize = ((size - 1)/PAGE_SIZE + 1) * PAGE_SIZE;
224 	uint64_t dma_space_addr = ENA_DMA_BIT_MASK(adapter->dma_width);
225 	int error;
226 
227 	if (dma_space_addr == 0)
228 		dma_space_addr = BUS_SPACE_MAXADDR;
229 	error = bus_dma_tag_create(bus_get_dma_tag(dmadev), /* parent */
230 	    8, 0,	      /* alignment, bounds 		*/
231 	    dma_space_addr,   /* lowaddr of exclusion window	*/
232 	    BUS_SPACE_MAXADDR,/* highaddr of exclusion window	*/
233 	    NULL, NULL,	      /* filter, filterarg 		*/
234 	    maxsize,	      /* maxsize 			*/
235 	    1,		      /* nsegments 			*/
236 	    maxsize,	      /* maxsegsize 			*/
237 	    BUS_DMA_ALLOCNOW, /* flags 				*/
238 	    NULL,	      /* lockfunc 			*/
239 	    NULL,	      /* lockarg 			*/
240 	    &dma->tag);
241 	if (error) {
242 		device_printf(dmadev,
243 		"%s: bus_dma_tag_create failed: %d\n",
244 		__func__, error);
245 		goto fail_tag;
246 	}
247 
248 	error = bus_dmamem_alloc(dma->tag, (void**) &dma->vaddr,
249 	    BUS_DMA_COHERENT | BUS_DMA_ZERO, &dma->map);
250 	if (error) {
251 		device_printf(dmadev,
252 		"%s: bus_dmamem_alloc(%ju) failed: %d\n",
253 		__func__, (uintmax_t)size, error);
254 		goto fail_map_create;
255 	}
256 
257 	dma->paddr = 0;
258 	error = bus_dmamap_load(dma->tag, dma->map, dma->vaddr,
259 	    size, ena_dmamap_callback, &dma->paddr, mapflags);
260 	if (error || dma->paddr == 0) {
261 		device_printf(dmadev,
262 		"%s: bus_dmamap_load failed: %d\n",
263 		__func__, error);
264 		goto fail_map_load;
265 	}
266 
267 	return (0);
268 
269 fail_map_load:
270 	bus_dmamap_unload(dma->tag, dma->map);
271 fail_map_create:
272 	bus_dmamem_free(dma->tag, dma->vaddr, dma->map);
273 	bus_dma_tag_destroy(dma->tag);
274 fail_tag:
275 	dma->tag = NULL;
276 
277 	return (error);
278 }
279 
280 static int
281 ena_allocate_pci_resources(struct ena_adapter* adapter)
282 {
283 	device_t pdev = adapter->pdev;
284 	int rid;
285 
286 	rid = PCIR_BAR(ENA_REG_BAR);
287 	adapter->memory = NULL;
288 	adapter->registers = bus_alloc_resource_any(pdev, SYS_RES_MEMORY,
289 	    &rid, RF_ACTIVE);
290 	if (adapter->registers == NULL) {
291 		device_printf(pdev, "Unable to allocate bus resource: "
292 		    "registers\n");
293 		return (ENXIO);
294 	}
295 
296 	return (0);
297 }
298 
299 static void
300 ena_free_pci_resources(struct ena_adapter *adapter)
301 {
302 	device_t pdev = adapter->pdev;
303 
304 	if (adapter->memory != NULL) {
305 		bus_release_resource(pdev, SYS_RES_MEMORY,
306 		    PCIR_BAR(ENA_MEM_BAR), adapter->memory);
307 	}
308 
309 	if (adapter->registers != NULL) {
310 		bus_release_resource(pdev, SYS_RES_MEMORY,
311 		    PCIR_BAR(ENA_REG_BAR), adapter->registers);
312 	}
313 
314 	return;
315 }
316 
317 static int
318 ena_probe(device_t dev)
319 {
320 	ena_vendor_info_t *ent;
321 	char		adapter_name[60];
322 	uint16_t	pci_vendor_id = 0;
323 	uint16_t	pci_device_id = 0;
324 
325 	pci_vendor_id = pci_get_vendor(dev);
326 	pci_device_id = pci_get_device(dev);
327 
328 	ent = ena_vendor_info_array;
329 	while (ent->vendor_id != 0) {
330 		if ((pci_vendor_id == ent->vendor_id) &&
331 		    (pci_device_id == ent->device_id)) {
332 			ena_trace(ENA_DBG, "vendor=%x device=%x ",
333 			    pci_vendor_id, pci_device_id);
334 
335 			sprintf(adapter_name, DEVICE_DESC);
336 			device_set_desc_copy(dev, adapter_name);
337 			return (BUS_PROBE_DEFAULT);
338 		}
339 
340 		ent++;
341 
342 	}
343 
344 	return (ENXIO);
345 }
346 
347 static int
348 ena_change_mtu(if_t ifp, int new_mtu)
349 {
350 	struct ena_adapter *adapter = if_getsoftc(ifp);
351 	struct ena_com_dev_get_features_ctx get_feat_ctx;
352 	int rc, old_mtu, max_frame;
353 
354 	rc = ena_com_get_dev_attr_feat(adapter->ena_dev, &get_feat_ctx);
355 	if (rc) {
356 		device_printf(adapter->pdev,
357 		    "Cannot get attribute for ena device\n");
358 		return (ENXIO);
359 	}
360 
361 	/* Save old MTU in case of fail */
362 	old_mtu = if_getmtu(ifp);
363 
364 	/* Change MTU and calculate max frame */
365 	if_setmtu(ifp, new_mtu);
366 	max_frame = ETHER_MAX_FRAME(ifp, ETHERTYPE_VLAN, 1);
367 
368 	if ((new_mtu < ENA_MIN_FRAME_LEN) ||
369 	    (new_mtu > get_feat_ctx.dev_attr.max_mtu) ||
370 	    (max_frame > ENA_MAX_FRAME_LEN)) {
371 		device_printf(adapter->pdev, "Invalid MTU setting. "
372 		    "new_mtu: %d\n", new_mtu);
373 		goto error;
374 	}
375 
376 	rc = ena_com_set_dev_mtu(adapter->ena_dev, new_mtu);
377 	if (rc != 0)
378 		goto error;
379 
380 	return (0);
381 error:
382 	if_setmtu(ifp, old_mtu);
383 	return (EINVAL);
384 }
385 
386 static inline void
387 ena_alloc_counters(counter_u64_t *begin, int size)
388 {
389 	counter_u64_t *end = (counter_u64_t *)((char *)begin + size);
390 
391 	for (; begin < end; ++begin)
392 		*begin = counter_u64_alloc(M_WAITOK);
393 }
394 
395 static inline void
396 ena_free_counters(counter_u64_t *begin, int size)
397 {
398 	counter_u64_t *end = (counter_u64_t *)((char *)begin + size);
399 
400 	for (; begin < end; ++begin)
401 		counter_u64_free(*begin);
402 }
403 
404 static inline void
405 ena_reset_counters(counter_u64_t *begin, int size)
406 {
407 	counter_u64_t *end = (counter_u64_t *)((char *)begin + size);
408 
409 	for (; begin < end; ++begin)
410 		counter_u64_zero(*begin);
411 }
412 
413 static void
414 ena_init_io_rings_common(struct ena_adapter *adapter, struct ena_ring *ring,
415     uint16_t qid)
416 {
417 
418 	ring->qid = qid;
419 	ring->adapter = adapter;
420 	ring->ena_dev = adapter->ena_dev;
421 }
422 
423 static int
424 ena_init_io_rings(struct ena_adapter *adapter)
425 {
426 	struct ena_com_dev *ena_dev;
427 	struct ena_ring *txr, *rxr;
428 	struct ena_que *que;
429 	int i;
430 	int rc;
431 
432 	ena_dev = adapter->ena_dev;
433 
434 	for (i = 0; i < adapter->num_queues; i++) {
435 		txr = &adapter->tx_ring[i];
436 		rxr = &adapter->rx_ring[i];
437 
438 		/* TX/RX common ring state */
439 		ena_init_io_rings_common(adapter, txr, i);
440 		ena_init_io_rings_common(adapter, rxr, i);
441 
442 		/* TX specific ring state */
443 		txr->ring_size = adapter->tx_ring_size;
444 		txr->tx_max_header_size = ena_dev->tx_max_header_size;
445 		txr->tx_mem_queue_type = ena_dev->tx_mem_queue_type;
446 		txr->smoothed_interval =
447 		    ena_com_get_nonadaptive_moderation_interval_tx(ena_dev);
448 
449 		/* Allocate a buf ring */
450 		txr->br = buf_ring_alloc(ena_buf_ring_size, M_DEVBUF,
451 		    M_WAITOK, &txr->ring_mtx);
452 		if (txr->br == NULL) {
453 			device_printf(adapter->pdev,
454 			    "Error while setting up bufring\n");
455 			rc = ENOMEM;
456 			goto err_bufr_free;
457 		}
458 
459 		/* Alloc TX statistics. */
460 		ena_alloc_counters((counter_u64_t *)&txr->tx_stats,
461 		    sizeof(txr->tx_stats));
462 
463 		/* RX specific ring state */
464 		rxr->ring_size = adapter->rx_ring_size;
465 		rxr->rx_small_copy_len = adapter->small_copy_len;
466 		rxr->smoothed_interval =
467 		    ena_com_get_nonadaptive_moderation_interval_rx(ena_dev);
468 
469 		/* Alloc RX statistics. */
470 		ena_alloc_counters((counter_u64_t *)&rxr->rx_stats,
471 		    sizeof(rxr->rx_stats));
472 
473 		/* Initialize locks */
474 		snprintf(txr->mtx_name, nitems(txr->mtx_name), "%s:tx(%d)",
475 		    device_get_nameunit(adapter->pdev), i);
476 		snprintf(rxr->mtx_name, nitems(rxr->mtx_name), "%s:rx(%d)",
477 		    device_get_nameunit(adapter->pdev), i);
478 
479 		mtx_init(&txr->ring_mtx, txr->mtx_name, NULL, MTX_DEF);
480 
481 		que = &adapter->que[i];
482 		que->adapter = adapter;
483 		que->id = i;
484 		que->tx_ring = txr;
485 		que->rx_ring = rxr;
486 
487 		txr->que = que;
488 		rxr->que = que;
489 	}
490 
491 	return 0;
492 
493 err_bufr_free:
494 	while (i--)
495 		ena_free_io_ring_resources(adapter, i);
496 
497 	return (rc);
498 }
499 
500 static void
501 ena_free_io_ring_resources(struct ena_adapter *adapter, unsigned int qid)
502 {
503 	struct ena_ring *txr = &adapter->tx_ring[qid];
504 	struct ena_ring *rxr = &adapter->rx_ring[qid];
505 
506 	ena_free_counters((counter_u64_t *)&txr->tx_stats,
507 	    sizeof(txr->tx_stats));
508 	ena_free_counters((counter_u64_t *)&rxr->rx_stats,
509 	    sizeof(rxr->rx_stats));
510 
511 	mtx_destroy(&txr->ring_mtx);
512 
513 	drbr_free(txr->br, M_DEVBUF);
514 
515 }
516 
517 static void
518 ena_free_all_io_rings_resources(struct ena_adapter *adapter)
519 {
520 	int i;
521 
522 	for (i = 0; i < adapter->num_queues; i++)
523 		ena_free_io_ring_resources(adapter, i);
524 
525 }
526 
527 static int
528 ena_setup_tx_dma_tag(struct ena_adapter *adapter)
529 {
530 	int ret;
531 
532 	/* Create DMA tag for Tx buffers */
533 	ret = bus_dma_tag_create(bus_get_dma_tag(adapter->pdev),
534 	    1, 0,				  /* alignment, bounds 	     */
535 	    ENA_DMA_BIT_MASK(adapter->dma_width), /* lowaddr of excl window  */
536 	    BUS_SPACE_MAXADDR, 			  /* highaddr of excl window */
537 	    NULL, NULL,				  /* filter, filterarg 	     */
538 	    ENA_TSO_MAXSIZE,			  /* maxsize 		     */
539 	    adapter->max_tx_sgl_size - 1,	  /* nsegments 		     */
540 	    ENA_TSO_MAXSIZE,			  /* maxsegsize 	     */
541 	    0,					  /* flags 		     */
542 	    NULL,				  /* lockfunc 		     */
543 	    NULL,				  /* lockfuncarg 	     */
544 	    &adapter->tx_buf_tag);
545 
546 	if (ret != 0)
547 		device_printf(adapter->pdev, "Unable to create Tx DMA tag\n");
548 
549 	return (ret);
550 }
551 
552 static int
553 ena_free_tx_dma_tag(struct ena_adapter *adapter)
554 {
555 	int ret;
556 
557 	ret = bus_dma_tag_destroy(adapter->tx_buf_tag);
558 
559 	if (ret == 0)
560 		adapter->tx_buf_tag = NULL;
561 
562 	return (ret);
563 }
564 
565 static int
566 ena_setup_rx_dma_tag(struct ena_adapter *adapter)
567 {
568 	int ret;
569 
570 	/* Create DMA tag for Rx buffers*/
571 	ret = bus_dma_tag_create(bus_get_dma_tag(adapter->pdev), /* parent   */
572 	    1, 0,				  /* alignment, bounds 	     */
573 	    ENA_DMA_BIT_MASK(adapter->dma_width), /* lowaddr of excl window  */
574 	    BUS_SPACE_MAXADDR, 			  /* highaddr of excl window */
575 	    NULL, NULL,				  /* filter, filterarg 	     */
576 	    MJUM16BYTES,			  /* maxsize 		     */
577 	    1,					  /* nsegments 		     */
578 	    MJUM16BYTES,			  /* maxsegsize 	     */
579 	    0,					  /* flags 		     */
580 	    NULL,				  /* lockfunc 		     */
581 	    NULL,				  /* lockarg 		     */
582 	    &adapter->rx_buf_tag);
583 
584 	if (ret != 0)
585 		device_printf(adapter->pdev, "Unable to create Rx DMA tag\n");
586 
587 	return (ret);
588 }
589 
590 static int
591 ena_free_rx_dma_tag(struct ena_adapter *adapter)
592 {
593 	int ret;
594 
595 	ret = bus_dma_tag_destroy(adapter->rx_buf_tag);
596 
597 	if (ret == 0)
598 		adapter->rx_buf_tag = NULL;
599 
600 	return (ret);
601 }
602 
603 
604 /**
605  * ena_setup_tx_resources - allocate Tx resources (Descriptors)
606  * @adapter: network interface device structure
607  * @qid: queue index
608  *
609  * Returns 0 on success, otherwise on failure.
610  **/
611 static int
612 ena_setup_tx_resources(struct ena_adapter *adapter, int qid)
613 {
614 	struct ena_que *que = &adapter->que[qid];
615 	struct ena_ring *tx_ring = que->tx_ring;
616 	int size, i, err;
617 #ifdef	RSS
618 	cpuset_t cpu_mask;
619 #endif
620 
621 	size = sizeof(struct ena_tx_buffer) * tx_ring->ring_size;
622 
623 	tx_ring->tx_buffer_info = malloc(size, M_DEVBUF, M_NOWAIT | M_ZERO);
624 	if (!tx_ring->tx_buffer_info)
625 		goto err_tx_buffer_info;
626 
627 	size = sizeof(uint16_t) * tx_ring->ring_size;
628 	tx_ring->free_tx_ids = malloc(size, M_DEVBUF, M_NOWAIT | M_ZERO);
629 	if (!tx_ring->free_tx_ids)
630 		goto err_tx_reqs;
631 
632 	/* Req id stack for TX OOO completions */
633 	for (i = 0; i < tx_ring->ring_size; i++)
634 		tx_ring->free_tx_ids[i] = i;
635 
636 	/* Reset TX statistics. */
637 	ena_reset_counters((counter_u64_t *)&tx_ring->tx_stats,
638 	    sizeof(tx_ring->tx_stats));
639 
640 	tx_ring->next_to_use = 0;
641 	tx_ring->next_to_clean = 0;
642 
643 	/* Make sure that drbr is empty */
644 	ENA_RING_MTX_LOCK(tx_ring);
645 	drbr_flush(adapter->ifp, tx_ring->br);
646 	ENA_RING_MTX_UNLOCK(tx_ring);
647 
648 	/* ... and create the buffer DMA maps */
649 	for (i = 0; i < tx_ring->ring_size; i++) {
650 		err = bus_dmamap_create(adapter->tx_buf_tag, 0,
651 		    &tx_ring->tx_buffer_info[i].map);
652 		if (err != 0) {
653 			device_printf(adapter->pdev,
654 			    "Unable to create Tx DMA map for buffer %d\n", i);
655 			goto err_tx_map;
656 		}
657 	}
658 
659 	/* Allocate taskqueues */
660 	TASK_INIT(&tx_ring->enqueue_task, 0, ena_deferred_mq_start, tx_ring);
661 	tx_ring->enqueue_tq = taskqueue_create_fast("ena_tx_enque", M_NOWAIT,
662 	    taskqueue_thread_enqueue, &tx_ring->enqueue_tq);
663 	if (tx_ring->enqueue_tq == NULL) {
664 		device_printf(adapter->pdev,
665 		    "Unable to create taskqueue for enqueue task\n");
666 		i = tx_ring->ring_size;
667 		goto err_tx_map;
668 	}
669 
670 	/* RSS set cpu for thread */
671 #ifdef RSS
672 	CPU_SETOF(que->cpu, &cpu_mask);
673 	taskqueue_start_threads_cpuset(&tx_ring->enqueue_tq, 1, PI_NET,
674 	    &cpu_mask, "%s tx_ring enq (bucket %d)",
675 	    device_get_nameunit(adapter->pdev), que->cpu);
676 #else /* RSS */
677 	taskqueue_start_threads(&tx_ring->enqueue_tq, 1, PI_NET,
678 	    "%s txeq %d", device_get_nameunit(adapter->pdev), que->cpu);
679 #endif /* RSS */
680 
681 	return (0);
682 
683 err_tx_map:
684 	while (i--) {
685 		bus_dmamap_destroy(adapter->tx_buf_tag,
686 		    tx_ring->tx_buffer_info[i].map);
687 	}
688 	ENA_MEM_FREE(adapter->ena_dev->dmadev, tx_ring->free_tx_ids);
689 err_tx_reqs:
690 	ENA_MEM_FREE(adapter->ena_dev->dmadev, tx_ring->tx_buffer_info);
691 err_tx_buffer_info:
692 	return (ENOMEM);
693 }
694 
695 /**
696  * ena_free_tx_resources - Free Tx Resources per Queue
697  * @adapter: network interface device structure
698  * @qid: queue index
699  *
700  * Free all transmit software resources
701  **/
702 static void
703 ena_free_tx_resources(struct ena_adapter *adapter, int qid)
704 {
705 	struct ena_ring *tx_ring = &adapter->tx_ring[qid];
706 
707 	while (taskqueue_cancel(tx_ring->enqueue_tq, &tx_ring->enqueue_task,
708 	    NULL))
709 		taskqueue_drain(tx_ring->enqueue_tq, &tx_ring->enqueue_task);
710 
711 	taskqueue_free(tx_ring->enqueue_tq);
712 
713 	ENA_RING_MTX_LOCK(tx_ring);
714 	/* Flush buffer ring, */
715 	drbr_flush(adapter->ifp, tx_ring->br);
716 
717 	/* Free buffer DMA maps, */
718 	for (int i = 0; i < tx_ring->ring_size; i++) {
719 		m_freem(tx_ring->tx_buffer_info[i].mbuf);
720 		tx_ring->tx_buffer_info[i].mbuf = NULL;
721 		bus_dmamap_unload(adapter->tx_buf_tag,
722 		    tx_ring->tx_buffer_info[i].map);
723 		bus_dmamap_destroy(adapter->tx_buf_tag,
724 		    tx_ring->tx_buffer_info[i].map);
725 	}
726 	ENA_RING_MTX_UNLOCK(tx_ring);
727 
728 	/* And free allocated memory. */
729 	ENA_MEM_FREE(adapter->ena_dev->dmadev, tx_ring->tx_buffer_info);
730 	tx_ring->tx_buffer_info = NULL;
731 
732 	ENA_MEM_FREE(adapter->ena_dev->dmadev, tx_ring->free_tx_ids);
733 	tx_ring->free_tx_ids = NULL;
734 }
735 
736 /**
737  * ena_setup_all_tx_resources - allocate all queues Tx resources
738  * @adapter: network interface device structure
739  *
740  * Returns 0 on success, otherwise on failure.
741  **/
742 static int
743 ena_setup_all_tx_resources(struct ena_adapter *adapter)
744 {
745 	int i, rc;
746 
747 	for (i = 0; i < adapter->num_queues; i++) {
748 		rc = ena_setup_tx_resources(adapter, i);
749 		if (!rc)
750 			continue;
751 
752 		device_printf(adapter->pdev,
753 		    "Allocation for Tx Queue %u failed\n", i);
754 		goto err_setup_tx;
755 	}
756 
757 	return (0);
758 
759 err_setup_tx:
760 	/* Rewind the index freeing the rings as we go */
761 	while (i--)
762 		ena_free_tx_resources(adapter, i);
763 	return (rc);
764 }
765 
766 /**
767  * ena_free_all_tx_resources - Free Tx Resources for All Queues
768  * @adapter: network interface device structure
769  *
770  * Free all transmit software resources
771  **/
772 static void
773 ena_free_all_tx_resources(struct ena_adapter *adapter)
774 {
775 	int i;
776 
777 	for (i = 0; i < adapter->num_queues; i++)
778 		ena_free_tx_resources(adapter, i);
779 
780 	return;
781 }
782 
783 /**
784  * ena_setup_rx_resources - allocate Rx resources (Descriptors)
785  * @adapter: network interface device structure
786  * @qid: queue index
787  *
788  * Returns 0 on success, otherwise on failure.
789  **/
790 static int
791 ena_setup_rx_resources(struct ena_adapter *adapter, unsigned int qid)
792 {
793 	struct ena_que *que = &adapter->que[qid];
794 	struct ena_ring *rx_ring = que->rx_ring;
795 	int size, err, i;
796 #ifdef	RSS
797 	cpuset_t cpu_mask;
798 #endif
799 
800 	size = sizeof(struct ena_rx_buffer) * rx_ring->ring_size;
801 
802 	/*
803 	 * Alloc extra element so in rx path
804 	 * we can always prefetch rx_info + 1
805 	 */
806 	size += sizeof(struct ena_rx_buffer);
807 
808 	rx_ring->rx_buffer_info = ENA_MEM_ALLOC(adapter->ena_dev->dmadev, size);
809 	if (!rx_ring->rx_buffer_info)
810 		return (ENOMEM);
811 
812 	/* Reset RX statistics. */
813 	ena_reset_counters((counter_u64_t *)&rx_ring->rx_stats,
814 	    sizeof(rx_ring->rx_stats));
815 
816 	rx_ring->next_to_clean = 0;
817 	rx_ring->next_to_use = 0;
818 
819 	/* ... and create the buffer DMA maps */
820 	for (i = 0; i < rx_ring->ring_size; i++) {
821 		err = bus_dmamap_create(adapter->rx_buf_tag, 0,
822 		    &(rx_ring->rx_buffer_info[i].map));
823 		if (err != 0) {
824 			device_printf(adapter->pdev,
825 			    "Unable to create Rx DMA map for buffer %d\n", i);
826 			goto err_rx_dma;
827 		}
828 	}
829 
830 	/* Create LRO for the ring */
831 	if (adapter->ifp->if_capenable & IFCAP_LRO) {
832 		int err = tcp_lro_init(&rx_ring->lro);
833 		if (err) {
834 			device_printf(adapter->pdev,
835 			    "LRO[%d] Initialization failed!\n", qid);
836 		} else {
837 			ena_trace(ENA_INFO,
838 			    "RX Soft LRO[%d] Initialized\n", qid);
839 			rx_ring->lro.ifp = adapter->ifp;
840 		}
841 	}
842 
843 	return (0);
844 
845 err_rx_dma:
846 	while (i--) {
847 		bus_dmamap_destroy(adapter->rx_buf_tag,
848 		    rx_ring->rx_buffer_info[i].map);
849 	}
850 
851 	ENA_MEM_FREE(adapter->ena_dev->dmadev, rx_ring->rx_buffer_info);
852 	rx_ring->rx_buffer_info = NULL;
853 	ena_trace(ENA_ALERT, "RX resource allocation fail");
854 	return (ENOMEM);
855 }
856 
857 /**
858  * ena_free_rx_resources - Free Rx Resources
859  * @adapter: network interface device structure
860  * @qid: queue index
861  *
862  * Free all receive software resources
863  **/
864 static void
865 ena_free_rx_resources(struct ena_adapter *adapter, unsigned int qid)
866 {
867 	struct ena_ring *rx_ring = &adapter->rx_ring[qid];
868 
869 	ena_trace(ENA_INFO, "%s qid %d\n", __func__, qid);
870 
871 	/* Free buffer DMA maps, */
872 	for (int i = 0; i < rx_ring->ring_size; i++) {
873 		m_freem(rx_ring->rx_buffer_info[i].mbuf);
874 		rx_ring->rx_buffer_info[i].mbuf = NULL;
875 		bus_dmamap_unload(adapter->rx_buf_tag,
876 		    rx_ring->rx_buffer_info[i].map);
877 		bus_dmamap_destroy(adapter->rx_buf_tag,
878 		    rx_ring->rx_buffer_info[i].map);
879 	}
880 
881 	/* free LRO resources, */
882 	tcp_lro_free(&rx_ring->lro);
883 
884 	/* free allocated memory */
885 	ENA_MEM_FREE(adapter->ena_dev->dmadev, rx_ring->rx_buffer_info);
886 	rx_ring->rx_buffer_info = NULL;
887 
888 	return;
889 }
890 
891 /**
892  * ena_setup_all_rx_resources - allocate all queues Rx resources
893  * @adapter: network interface device structure
894  *
895  * Returns 0 on success, otherwise on failure.
896  **/
897 static int
898 ena_setup_all_rx_resources(struct ena_adapter *adapter)
899 {
900 	int i, rc = 0;
901 
902 	for (i = 0; i < adapter->num_queues; i++) {
903 		rc = ena_setup_rx_resources(adapter, i);
904 		if (!rc)
905 			continue;
906 
907 		device_printf(adapter->pdev,
908 		    "Allocation for Rx Queue %u failed\n", i);
909 		goto err_setup_rx;
910 	}
911 	return (0);
912 
913 err_setup_rx:
914 	/* rewind the index freeing the rings as we go */
915 	while (i--)
916 		ena_free_rx_resources(adapter, i);
917 	return (rc);
918 }
919 
920 /**
921  * ena_free_all_rx_resources - Free Rx resources for all queues
922  * @adapter: network interface device structure
923  *
924  * Free all receive software resources
925  **/
926 static void
927 ena_free_all_rx_resources(struct ena_adapter *adapter)
928 {
929 	int i;
930 
931 	for (i = 0; i < adapter->num_queues; i++)
932 		ena_free_rx_resources(adapter, i);
933 
934 	return;
935 }
936 
937 static inline int
938 ena_alloc_rx_mbuf(struct ena_adapter *adapter,
939     struct ena_ring *rx_ring, struct ena_rx_buffer *rx_info)
940 {
941 	struct ena_com_buf *ena_buf;
942 	bus_dma_segment_t segs[1];
943 	int nsegs, error;
944 
945 	/* if previous allocated frag is not used */
946 	if (rx_info->mbuf != NULL)
947 		return (0);
948 
949 	/* Get mbuf using UMA allocator */
950 	rx_info->mbuf = m_getjcl(M_NOWAIT, MT_DATA, M_PKTHDR, MJUM16BYTES);
951 
952 	if (!rx_info->mbuf) {
953 		counter_u64_add(rx_ring->rx_stats.mbuf_alloc_fail, 1);
954 		return (ENOMEM);
955 	}
956 	/* Set mbuf length*/
957 	rx_info->mbuf->m_pkthdr.len = rx_info->mbuf->m_len = MJUM16BYTES;
958 
959 	/* Map packets for DMA */
960 	ena_trace(ENA_DBG | ENA_RSC | ENA_RXPTH,
961 	    "Using tag %p for buffers' DMA mapping, mbuf %p len: %d",
962 	    adapter->rx_buf_tag,rx_info->mbuf, rx_info->mbuf->m_len);
963 	error = bus_dmamap_load_mbuf_sg(adapter->rx_buf_tag, rx_info->map,
964 	    rx_info->mbuf, segs, &nsegs, BUS_DMA_NOWAIT);
965 	if (error || (nsegs != 1)) {
966 		device_printf(adapter->pdev, "failed to map mbuf, error: %d, "
967 		    "nsegs: %d\n", error, nsegs);
968 		counter_u64_add(rx_ring->rx_stats.dma_mapping_err, 1);
969 		goto exit;
970 
971 	}
972 
973 	bus_dmamap_sync(adapter->rx_buf_tag, rx_info->map, BUS_DMASYNC_PREREAD);
974 
975 	ena_buf = &rx_info->ena_buf;
976 	ena_buf->paddr = segs[0].ds_addr;
977 	ena_buf->len = MJUM16BYTES;
978 
979 	ena_trace(ENA_DBG | ENA_RSC | ENA_RXPTH,
980 	    "ALLOC RX BUF: mbuf %p, rx_info %p, len %d, paddr %#jx\n",
981 	    rx_info->mbuf, rx_info,ena_buf->len, (uintmax_t)ena_buf->paddr);
982 
983 	return (0);
984 
985 exit:
986 	m_freem(rx_info->mbuf);
987 	rx_info->mbuf = NULL;
988 	return (EFAULT);
989 }
990 
991 static void
992 ena_free_rx_mbuf(struct ena_adapter *adapter, struct ena_ring *rx_ring,
993     struct ena_rx_buffer *rx_info)
994 {
995 
996 	if (!rx_info->mbuf)
997 		return;
998 
999 	bus_dmamap_unload(adapter->rx_buf_tag, rx_info->map);
1000 	m_freem(rx_info->mbuf);
1001 	rx_info->mbuf = NULL;
1002 
1003 	return;
1004 }
1005 
1006 
1007 /**
1008  * ena_refill_rx_bufs - Refills ring with descriptors
1009  * @rx_ring: the ring which we want to feed with free descriptors
1010  * @num: number of descriptors to refill
1011  * Refills the ring with newly allocated DMA-mapped mbufs for receiving
1012  **/
1013 static int
1014 ena_refill_rx_bufs(struct ena_ring *rx_ring, uint32_t num)
1015 {
1016 	struct ena_adapter *adapter = rx_ring->adapter;
1017 	uint16_t next_to_use;
1018 	uint32_t i;
1019 	int rc;
1020 
1021 	ena_trace(ENA_DBG | ENA_RXPTH | ENA_RSC, "refill qid: %d",
1022 	    rx_ring->qid);
1023 
1024 	next_to_use = rx_ring->next_to_use;
1025 
1026 	for (i = 0; i < num; i++) {
1027 		ena_trace(ENA_DBG | ENA_RXPTH | ENA_RSC,
1028 		    "RX buffer - next to use: %d", next_to_use);
1029 
1030 		struct ena_rx_buffer *rx_info =
1031 		    &rx_ring->rx_buffer_info[next_to_use];
1032 
1033 		rc = ena_alloc_rx_mbuf(adapter, rx_ring, rx_info);
1034 		if (rc != 0) {
1035 			device_printf(adapter->pdev,
1036 			    "failed to alloc buffer for rx queue\n");
1037 			break;
1038 		}
1039 		rc = ena_com_add_single_rx_desc(rx_ring->ena_com_io_sq,
1040 		    &rx_info->ena_buf, next_to_use);
1041 		if (unlikely(rc)) {
1042 			device_printf(adapter->pdev,
1043 			    "failed to add buffer for rx queue %d\n",
1044 			    rx_ring->qid);
1045 			break;
1046 		}
1047 		next_to_use = ENA_RX_RING_IDX_NEXT(next_to_use,
1048 		    rx_ring->ring_size);
1049 	}
1050 
1051 	if (i < num) {
1052 		counter_u64_add(rx_ring->rx_stats.refil_partial, 1);
1053 		device_printf(adapter->pdev,
1054 		    "refilled rx queue %d with %d pages only\n",
1055 		    rx_ring->qid, i);
1056 	}
1057 
1058 	if (i != 0) {
1059 		wmb();
1060 		ena_com_write_sq_doorbell(rx_ring->ena_com_io_sq);
1061 	}
1062 	rx_ring->next_to_use = next_to_use;
1063 	return (i);
1064 }
1065 
1066 static void
1067 ena_free_rx_bufs(struct ena_adapter *adapter, unsigned int qid)
1068 {
1069 	struct ena_ring *rx_ring = &adapter->rx_ring[qid];
1070 	unsigned int i;
1071 
1072 	for (i = 0; i < rx_ring->ring_size; i++) {
1073 		struct ena_rx_buffer *rx_info = &rx_ring->rx_buffer_info[i];
1074 
1075 		if (rx_info->mbuf)
1076 			ena_free_rx_mbuf(adapter, rx_ring, rx_info);
1077 	}
1078 
1079 	return;
1080 }
1081 
1082 /**
1083  * ena_refill_all_rx_bufs - allocate all queues Rx buffers
1084  * @adapter: network interface device structure
1085  *
1086  */
1087 static void
1088 ena_refill_all_rx_bufs(struct ena_adapter *adapter)
1089 {
1090 	struct ena_ring *rx_ring;
1091 	int i, rc, bufs_num;
1092 
1093 	for (i = 0; i < adapter->num_queues; i++) {
1094 		rx_ring = &adapter->rx_ring[i];
1095 		bufs_num = rx_ring->ring_size - 1;
1096 		rc = ena_refill_rx_bufs(rx_ring, bufs_num);
1097 
1098 		if (unlikely(rc != bufs_num))
1099 			device_printf(adapter->pdev,
1100 			    "refilling Queue %d failed. allocated %d buffers"
1101 			    " from: %d\n", i, rc, bufs_num);
1102 	}
1103 }
1104 
1105 static void
1106 ena_free_all_rx_bufs(struct ena_adapter *adapter)
1107 {
1108 	int i;
1109 
1110 	for (i = 0; i < adapter->num_queues; i++)
1111 		ena_free_rx_bufs(adapter, i);
1112 	return;
1113 }
1114 
1115 /**
1116  * ena_free_tx_bufs - Free Tx Buffers per Queue
1117  * @adapter: network interface device structure
1118  * @qid: queue index
1119  **/
1120 static void
1121 ena_free_tx_bufs(struct ena_adapter *adapter, unsigned int qid)
1122 {
1123 	struct ena_ring *tx_ring = &adapter->tx_ring[qid];
1124 
1125 	ENA_RING_MTX_LOCK(tx_ring);
1126 	for (int i = 0; i < tx_ring->ring_size; i++) {
1127 		struct ena_tx_buffer *tx_info = &tx_ring->tx_buffer_info[i];
1128 
1129 		if (tx_info->mbuf == NULL)
1130 			continue;
1131 
1132 		ena_trace(ENA_DBG | ENA_TXPTH | ENA_RSC,
1133 		    "free uncompleted Tx mbufs qid[%d] idx: 0x%x", qid, i);
1134 
1135 		bus_dmamap_unload(adapter->tx_buf_tag, tx_info->map);
1136 		m_free(tx_info->mbuf);
1137 		tx_info->mbuf = NULL;
1138 	}
1139 	ENA_RING_MTX_UNLOCK(tx_ring);
1140 
1141 	return;
1142 }
1143 
1144 static void
1145 ena_free_all_tx_bufs(struct ena_adapter *adapter)
1146 {
1147 
1148 	for (int i = 0; i < adapter->num_queues; i++)
1149 		ena_free_tx_bufs(adapter, i);
1150 
1151 	return;
1152 }
1153 
1154 static void
1155 ena_destroy_all_tx_queues(struct ena_adapter *adapter)
1156 {
1157 	uint16_t ena_qid;
1158 	int i;
1159 
1160 	for (i = 0; i < adapter->num_queues; i++) {
1161 		ena_qid = ENA_IO_TXQ_IDX(i);
1162 		ena_com_destroy_io_queue(adapter->ena_dev, ena_qid);
1163 	}
1164 }
1165 
1166 static void
1167 ena_destroy_all_rx_queues(struct ena_adapter *adapter)
1168 {
1169 	uint16_t ena_qid;
1170 	int i;
1171 
1172 	for (i = 0; i < adapter->num_queues; i++) {
1173 		ena_qid = ENA_IO_RXQ_IDX(i);
1174 		ena_com_destroy_io_queue(adapter->ena_dev, ena_qid);
1175 	}
1176 }
1177 
1178 static void
1179 ena_destroy_all_io_queues(struct ena_adapter *adapter)
1180 {
1181 	ena_destroy_all_tx_queues(adapter);
1182 	ena_destroy_all_rx_queues(adapter);
1183 }
1184 
1185 static int
1186 validate_tx_req_id(struct ena_ring *tx_ring, uint16_t req_id)
1187 {
1188 	struct ena_tx_buffer *tx_info = NULL;
1189 
1190 	if (likely(req_id < tx_ring->ring_size)) {
1191 		tx_info = &tx_ring->tx_buffer_info[req_id];
1192 		if (tx_info->mbuf)
1193 			return 0;
1194 	}
1195 
1196 	counter_u64_add(tx_ring->tx_stats.bad_req_id, 1);
1197 
1198 	return (EFAULT);
1199 }
1200 
1201 static int
1202 ena_create_io_queues(struct ena_adapter *adapter)
1203 {
1204 	struct ena_com_dev *ena_dev = adapter->ena_dev;
1205 	struct ena_com_create_io_ctx ctx;
1206 	struct ena_ring *ring;
1207 	uint16_t ena_qid;
1208 	uint32_t msix_vector;
1209 	int rc, i;
1210 
1211 	/* Create TX queues */
1212 	for (i = 0; i < adapter->num_queues; i++) {
1213 		msix_vector = ENA_IO_IRQ_IDX(i);
1214 		ena_qid = ENA_IO_TXQ_IDX(i);
1215 		ctx.mem_queue_type = ena_dev->tx_mem_queue_type;
1216 		ctx.direction = ENA_COM_IO_QUEUE_DIRECTION_TX;
1217 		ctx.queue_size = adapter->tx_ring_size;
1218 		ctx.msix_vector = msix_vector;
1219 		ctx.qid = ena_qid;
1220 		rc = ena_com_create_io_queue(ena_dev, &ctx);
1221 		if (rc) {
1222 			device_printf(adapter->pdev,
1223 			    "Failed to create io TX queue #%d rc: %d\n", i, rc);
1224 			goto err_tx;
1225 		}
1226 		ring = &adapter->tx_ring[i];
1227 		rc = ena_com_get_io_handlers(ena_dev, ena_qid,
1228 		    &ring->ena_com_io_sq,
1229 		    &ring->ena_com_io_cq);
1230 		if (rc) {
1231 			device_printf(adapter->pdev,
1232 			    "Failed to get TX queue handlers. TX queue num"
1233 			    " %d rc: %d\n", i, rc);
1234 			ena_com_destroy_io_queue(ena_dev, ena_qid);
1235 			goto err_tx;
1236 		}
1237 	}
1238 
1239 	/* Create RX queues */
1240 	for (i = 0; i < adapter->num_queues; i++) {
1241 		msix_vector = ENA_IO_IRQ_IDX(i);
1242 		ena_qid = ENA_IO_RXQ_IDX(i);
1243 		ctx.mem_queue_type = ENA_ADMIN_PLACEMENT_POLICY_HOST;
1244 		ctx.direction = ENA_COM_IO_QUEUE_DIRECTION_RX;
1245 		ctx.queue_size = adapter->rx_ring_size;
1246 		ctx.msix_vector = msix_vector;
1247 		ctx.qid = ena_qid;
1248 		rc = ena_com_create_io_queue(ena_dev, &ctx);
1249 		if (rc) {
1250 			device_printf(adapter->pdev,
1251 			    "Failed to create io RX queue[%d] rc: %d\n", i, rc);
1252 			goto err_rx;
1253 		}
1254 
1255 		ring = &adapter->rx_ring[i];
1256 		rc = ena_com_get_io_handlers(ena_dev, ena_qid,
1257 		    &ring->ena_com_io_sq,
1258 		    &ring->ena_com_io_cq);
1259 		if (rc) {
1260 			device_printf(adapter->pdev,
1261 			    "Failed to get RX queue handlers. RX queue num"
1262 			    " %d rc: %d\n", i, rc);
1263 			ena_com_destroy_io_queue(ena_dev, ena_qid);
1264 			goto err_rx;
1265 		}
1266 	}
1267 
1268 	return (0);
1269 
1270 err_rx:
1271 	while (i--)
1272 		ena_com_destroy_io_queue(ena_dev, ENA_IO_RXQ_IDX(i));
1273 	i = adapter->num_queues;
1274 err_tx:
1275 	while (i--)
1276 		ena_com_destroy_io_queue(ena_dev, ENA_IO_TXQ_IDX(i));
1277 
1278 	return (ENXIO);
1279 }
1280 
1281 /**
1282  * ena_tx_cleanup - clear sent packets and corresponding descriptors
1283  * @tx_ring: ring for which we want to clean packets
1284  *
1285  * Once packets are sent, we ask the device in a loop for no longer used
1286  * descriptors. We find the related mbuf chain in a map (index in an array)
1287  * and free it, then update ring state.
1288  * This is performed in "endless" loop, updating ring pointers every
1289  * TX_COMMIT. The first check of free descriptor is performed before the actual
1290  * loop, then repeated at the loop end.
1291  **/
1292 static int
1293 ena_tx_cleanup(struct ena_ring *tx_ring)
1294 {
1295 	struct ena_adapter *adapter;
1296 	struct ena_com_io_cq* io_cq;
1297 	uint16_t next_to_clean;
1298 	uint16_t req_id;
1299 	uint16_t ena_qid;
1300 	unsigned int total_done = 0;
1301 	int rc;
1302 	int commit = TX_COMMIT;
1303 	int budget = TX_BUDGET;
1304 	int work_done;
1305 
1306 	adapter = tx_ring->que->adapter;
1307 	ena_qid = ENA_IO_TXQ_IDX(tx_ring->que->id);
1308 	io_cq = &adapter->ena_dev->io_cq_queues[ena_qid];
1309 	next_to_clean = tx_ring->next_to_clean;
1310 
1311 	do {
1312 		struct ena_tx_buffer *tx_info;
1313 		struct mbuf *mbuf;
1314 
1315 		rc = ena_com_tx_comp_req_id_get(io_cq, &req_id);
1316 		if (rc != 0)
1317 			break;
1318 
1319 		rc = validate_tx_req_id(tx_ring, req_id);
1320 		if (rc)
1321 			break;
1322 
1323 		tx_info = &tx_ring->tx_buffer_info[req_id];
1324 
1325 		mbuf = tx_info->mbuf;
1326 
1327 		tx_info->mbuf = NULL;
1328 		bintime_clear(&tx_info->timestamp);
1329 
1330 		if (tx_info->num_of_bufs != 0) {
1331 			/* Map is no longer required */
1332 			bus_dmamap_unload(adapter->tx_buf_tag, tx_info->map);
1333 		}
1334 
1335 		m_freem(mbuf);
1336 
1337 		total_done += tx_info->tx_descs;
1338 
1339 		tx_ring->free_tx_ids[next_to_clean] = req_id;
1340 		next_to_clean = ENA_TX_RING_IDX_NEXT(next_to_clean,
1341 		    tx_ring->ring_size);
1342 
1343 		if (--commit == 0) {
1344 			commit = TX_COMMIT;
1345 			/* update ring state every TX_COMMIT descriptor */
1346 			tx_ring->next_to_clean = next_to_clean;
1347 			ena_com_comp_ack(&adapter->ena_dev->io_sq_queues[ena_qid], total_done);
1348 			ena_com_update_dev_comp_head(io_cq);
1349 			total_done = 0;
1350 		}
1351 	} while (--budget);
1352 
1353 	work_done = TX_BUDGET - budget;
1354 
1355 	/* If there is still something to commit update ring state */
1356 	if (commit != TX_COMMIT) {
1357 		tx_ring->next_to_clean = next_to_clean;
1358 		ena_com_comp_ack(&adapter->ena_dev->io_sq_queues[ena_qid], total_done);
1359 		ena_com_update_dev_comp_head(io_cq);
1360 	}
1361 
1362 	taskqueue_enqueue(tx_ring->enqueue_tq, &tx_ring->enqueue_task);
1363 
1364 	return (work_done);
1365 }
1366 
1367 static void
1368 ena_rx_hash_mbuf(struct ena_ring *rx_ring, struct ena_com_rx_ctx *ena_rx_ctx,
1369     struct mbuf *mbuf)
1370 {
1371 	struct ena_adapter *adapter = rx_ring->adapter;
1372 
1373 	if (adapter->rss_support) {
1374 		mbuf->m_pkthdr.flowid = ena_rx_ctx->hash;
1375 
1376 		if (ena_rx_ctx->frag &&
1377 		    ena_rx_ctx->l3_proto != ENA_ETH_IO_L4_PROTO_UNKNOWN) {
1378 			M_HASHTYPE_SET(mbuf, M_HASHTYPE_OPAQUE_HASH);
1379 			return;
1380 		}
1381 
1382 		switch (ena_rx_ctx->l3_proto) {
1383 		case ENA_ETH_IO_L3_PROTO_IPV4:
1384 			switch (ena_rx_ctx->l4_proto) {
1385 			case ENA_ETH_IO_L4_PROTO_TCP:
1386 				M_HASHTYPE_SET(mbuf, M_HASHTYPE_RSS_TCP_IPV4);
1387 				break;
1388 			case ENA_ETH_IO_L4_PROTO_UDP:
1389 				M_HASHTYPE_SET(mbuf, M_HASHTYPE_RSS_UDP_IPV4);
1390 				break;
1391 			default:
1392 				M_HASHTYPE_SET(mbuf, M_HASHTYPE_RSS_IPV4);
1393 			}
1394 			break;
1395 		case ENA_ETH_IO_L3_PROTO_IPV6:
1396 			switch (ena_rx_ctx->l4_proto) {
1397 			case ENA_ETH_IO_L4_PROTO_TCP:
1398 				M_HASHTYPE_SET(mbuf, M_HASHTYPE_RSS_TCP_IPV6);
1399 				break;
1400 			case ENA_ETH_IO_L4_PROTO_UDP:
1401 				M_HASHTYPE_SET(mbuf, M_HASHTYPE_RSS_UDP_IPV6);
1402 				break;
1403 			default:
1404 				M_HASHTYPE_SET(mbuf, M_HASHTYPE_RSS_IPV6);
1405 			}
1406 			break;
1407 		case ENA_ETH_IO_L3_PROTO_UNKNOWN:
1408 			M_HASHTYPE_SET(mbuf, M_HASHTYPE_NONE);
1409 			break;
1410 		default:
1411 			M_HASHTYPE_SET(mbuf, M_HASHTYPE_OPAQUE_HASH);
1412 		}
1413 	} else {
1414 		mbuf->m_pkthdr.flowid = rx_ring->qid;
1415 		M_HASHTYPE_SET(mbuf, M_HASHTYPE_NONE);
1416 	}
1417 }
1418 
1419 /**
1420  * ena_rx_mbuf - assemble mbuf from descriptors
1421  * @rx_ring: ring for which we want to clean packets
1422  * @ena_bufs: buffer info
1423  * @ena_rx_ctx: metadata for this packet(s)
1424  * @next_to_clean: ring pointer
1425  *
1426  **/
1427 static struct mbuf*
1428 ena_rx_mbuf(struct ena_ring *rx_ring, struct ena_com_rx_buf_info *ena_bufs,
1429     struct ena_com_rx_ctx *ena_rx_ctx, uint16_t *next_to_clean)
1430 {
1431 	struct mbuf *mbuf;
1432 	struct ena_rx_buffer *rx_info;
1433 	struct ena_adapter *adapter;
1434 	unsigned int len, buf = 0;
1435 	unsigned int descs = ena_rx_ctx->descs;
1436 
1437 	adapter = rx_ring->adapter;
1438 	rx_info = &rx_ring->rx_buffer_info[*next_to_clean];
1439 
1440 	ENA_ASSERT(rx_info->mbuf, "Invalid alloc frag buffer\n");
1441 
1442 	len = ena_bufs[0].len;
1443 	ena_trace(ENA_DBG | ENA_RXPTH, "rx_info %p, mbuf %p, paddr %jx",
1444 	    rx_info, rx_info->mbuf, (uintmax_t)rx_info->ena_buf.paddr);
1445 
1446 	mbuf = rx_info->mbuf;
1447 	mbuf->m_flags |= M_PKTHDR;
1448 	mbuf->m_pkthdr.len = len;
1449 	mbuf->m_len = len;
1450 	mbuf->m_pkthdr.rcvif = rx_ring->que->adapter->ifp;
1451 
1452 	/* Fill mbuf with hash key and it's interpretation for optimization */
1453 	ena_rx_hash_mbuf(rx_ring, ena_rx_ctx, mbuf);
1454 
1455 	ena_trace(ENA_DBG | ENA_RXPTH, "rx mbuf 0x%p, flags=0x%x, len: %d",
1456 	    mbuf, mbuf->m_flags, mbuf->m_pkthdr.len);
1457 
1458 	/* DMA address is not needed anymore, unmap it */
1459 	bus_dmamap_unload(rx_ring->adapter->rx_buf_tag, rx_info->map);
1460 
1461 	rx_info->mbuf = NULL;
1462 	*next_to_clean = ENA_RX_RING_IDX_NEXT(*next_to_clean,
1463 	    rx_ring->ring_size);
1464 
1465 	/*
1466 	 * While we have more than 1 descriptors for one rcvd packet, append
1467 	 * other mbufs to the main one
1468 	 */
1469 	while (--descs) {
1470 		rx_info = &rx_ring->rx_buffer_info[*next_to_clean];
1471 		len = ena_bufs[++buf].len;
1472 
1473 		if (!m_append(mbuf, len, rx_info->mbuf->m_data)) {
1474 			counter_u64_add(rx_ring->rx_stats.mbuf_alloc_fail, 1);
1475 			ena_trace(ENA_WARNING, "Failed to append Rx mbuf %p",
1476 			    mbuf);
1477 		}
1478 		/* Free already appended mbuf, it won't be useful anymore */
1479 		bus_dmamap_unload(rx_ring->adapter->rx_buf_tag, rx_info->map);
1480 		m_freem(rx_info->mbuf);
1481 		rx_info->mbuf = NULL;
1482 
1483 		*next_to_clean = ENA_RX_RING_IDX_NEXT(*next_to_clean,
1484 		    rx_ring->ring_size);
1485 	}
1486 
1487 	return (mbuf);
1488 }
1489 
1490 /**
1491  * ena_rx_checksum - indicate in mbuf if hw indicated a good cksum
1492  **/
1493 static inline void
1494 ena_rx_checksum(struct ena_ring *rx_ring, struct ena_com_rx_ctx *ena_rx_ctx,
1495     struct mbuf *mbuf)
1496 {
1497 
1498 	/* if IP and error */
1499 	if ((ena_rx_ctx->l3_proto == ENA_ETH_IO_L3_PROTO_IPV4) &&
1500 	    (ena_rx_ctx->l3_csum_err)) {
1501 		/* ipv4 checksum error */
1502 		mbuf->m_pkthdr.csum_flags = 0;
1503 		counter_u64_add(rx_ring->rx_stats.bad_csum, 1);
1504 		return;
1505 	}
1506 
1507 	/* if TCP/UDP */
1508 	if ((ena_rx_ctx->l4_proto == ENA_ETH_IO_L4_PROTO_TCP) ||
1509 	    (ena_rx_ctx->l4_proto == ENA_ETH_IO_L4_PROTO_UDP)) {
1510 		if (ena_rx_ctx->l4_csum_err) {
1511 			/* TCP/UDP checksum error */
1512 			mbuf->m_pkthdr.csum_flags = 0;
1513 			counter_u64_add(rx_ring->rx_stats.bad_csum, 1);
1514 		} else {
1515 			mbuf->m_pkthdr.csum_flags = CSUM_IP_CHECKED;
1516 			mbuf->m_pkthdr.csum_flags |= CSUM_IP_VALID;
1517 		}
1518 	}
1519 
1520 	return;
1521 }
1522 
1523 /**
1524  * ena_rx_cleanup - handle rx irq
1525  * @arg: ring for which irq is being handled
1526  **/
1527 static int
1528 ena_rx_cleanup(struct ena_ring *rx_ring)
1529 {
1530 	struct ena_adapter *adapter;
1531 	struct mbuf *mbuf;
1532 	struct ena_com_rx_ctx ena_rx_ctx;
1533 	struct ena_com_io_cq* io_cq;
1534 	struct ena_com_io_sq* io_sq;
1535 	/* struct ena_eth_io_intr_reg intr_reg; */
1536 	if_t ifp;
1537 	uint16_t ena_qid;
1538 	uint16_t next_to_clean;
1539 	uint32_t refill_required;
1540 	uint32_t refill_threshold;
1541 	uint32_t do_if_input = 0;
1542 	unsigned int qid;
1543 	int rc;
1544 	int budget = RX_BUDGET;
1545 
1546 	adapter = rx_ring->que->adapter;
1547 	ifp = adapter->ifp;
1548 	qid = rx_ring->que->id;
1549 	ena_qid = ENA_IO_RXQ_IDX(qid);
1550 	io_cq = &adapter->ena_dev->io_cq_queues[ena_qid];
1551 	io_sq = &adapter->ena_dev->io_sq_queues[ena_qid];
1552 	next_to_clean = rx_ring->next_to_clean;
1553 
1554 	do {
1555 		ena_rx_ctx.ena_bufs = rx_ring->ena_bufs;
1556 		ena_rx_ctx.max_bufs = adapter->max_rx_sgl_size;
1557 		ena_rx_ctx.descs = 0;
1558 		rc = ena_com_rx_pkt(io_cq, io_sq, &ena_rx_ctx);
1559 
1560 		if (unlikely(rc))
1561 			goto error;
1562 
1563 		if (unlikely(ena_rx_ctx.descs == 0))
1564 			break;
1565 
1566 		/* Receive mbuf from the ring */
1567 		mbuf = ena_rx_mbuf(rx_ring, rx_ring->ena_bufs,
1568 		    &ena_rx_ctx, &next_to_clean);
1569 
1570 		/* Exit if we failed to retrieve a buffer */
1571 		if (unlikely(!mbuf)) {
1572 			next_to_clean = ENA_RX_RING_IDX_ADD(next_to_clean,
1573 			    ena_rx_ctx.descs, rx_ring->ring_size);
1574 			break;
1575 		}
1576 		ena_trace(ENA_DBG | ENA_RXPTH, "Rx: %d bytes",
1577 		    mbuf->m_pkthdr.len);
1578 
1579 		if ((ifp->if_capenable & IFCAP_RXCSUM) ||
1580 		    (ifp->if_capenable & IFCAP_RXCSUM_IPV6)) {
1581 			ena_rx_checksum(rx_ring, &ena_rx_ctx, mbuf);
1582 		}
1583 
1584 		counter_enter();
1585 		counter_u64_add_protected(rx_ring->rx_stats.bytes,
1586 		    mbuf->m_pkthdr.len);
1587 		counter_u64_add_protected(adapter->hw_stats.rx_bytes,
1588 		    mbuf->m_pkthdr.len);
1589 		counter_exit();
1590 		/*
1591 		 * LRO is only for IP/TCP packets and TCP checksum of the packet
1592 		 * should be computed by hardware.
1593 		 */
1594 		do_if_input = 1;
1595 		if ((ifp->if_capenable & IFCAP_LRO) &&
1596 		    (mbuf->m_pkthdr.csum_flags & CSUM_IP_VALID) &&
1597 		    ena_rx_ctx.l4_proto == ENA_ETH_IO_L4_PROTO_TCP) {
1598 			/*
1599 			 * Send to the stack if:
1600 			 *  - LRO not enabled, or
1601 			 *  - no LRO resources, or
1602 			 *  - lro enqueue fails
1603 			 */
1604 			if (rx_ring->lro.lro_cnt != 0 &&
1605 			    tcp_lro_rx(&rx_ring->lro, mbuf, 0) == 0)
1606 					do_if_input = 0;
1607 		}
1608 		if (do_if_input) {
1609 			ena_trace(ENA_DBG | ENA_RXPTH, "calling if_input() with mbuf %p",
1610 			    mbuf);
1611 			(*ifp->if_input)(ifp, mbuf);
1612 		}
1613 
1614 		counter_enter();
1615 		counter_u64_add_protected(rx_ring->rx_stats.cnt, 1);
1616 		counter_u64_add_protected(adapter->hw_stats.rx_packets, 1);
1617 		counter_exit();
1618 	} while (--budget);
1619 
1620 	rx_ring->next_to_clean = next_to_clean;
1621 
1622 	refill_required = ena_com_free_desc(io_sq);
1623 	refill_threshold = rx_ring->ring_size / ENA_RX_REFILL_THRESH_DEVIDER;
1624 
1625 	if (refill_required > refill_threshold) {
1626 		ena_com_update_dev_comp_head(rx_ring->ena_com_io_cq);
1627 		ena_refill_rx_bufs(rx_ring, refill_required);
1628 	}
1629 
1630 	tcp_lro_flush_all(&rx_ring->lro);
1631 
1632 	return (RX_BUDGET - budget);
1633 
1634 error:
1635 	counter_u64_add(rx_ring->rx_stats.bad_desc_num, 1);
1636 	return (RX_BUDGET - budget);
1637 }
1638 
1639 /*********************************************************************
1640  *
1641  *  MSIX & Interrupt Service routine
1642  *
1643  **********************************************************************/
1644 
1645 /**
1646  * ena_handle_msix - MSIX Interrupt Handler for admin/async queue
1647  * @arg: interrupt number
1648  **/
1649 static void
1650 ena_intr_msix_mgmnt(void *arg)
1651 {
1652 	struct ena_adapter *adapter = (struct ena_adapter *)arg;
1653 
1654 	ena_com_admin_q_comp_intr_handler(adapter->ena_dev);
1655 	if (likely(adapter->running))
1656 		ena_com_aenq_intr_handler(adapter->ena_dev, arg);
1657 }
1658 
1659 /**
1660  * ena_handle_msix - MSIX Interrupt Handler for Tx/Rx
1661  * @arg: interrupt number
1662  **/
1663 static void
1664 ena_handle_msix(void *arg)
1665 {
1666 	struct ena_que	*que = arg;
1667 	struct ena_adapter *adapter = que->adapter;
1668 	if_t ifp = adapter->ifp;
1669 	struct ena_ring *tx_ring;
1670 	struct ena_ring *rx_ring;
1671 	struct ena_com_io_cq* io_cq;
1672 	struct ena_eth_io_intr_reg intr_reg;
1673 	int qid, ena_qid;
1674 	int txc, rxc, i;
1675 
1676 	if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
1677 		return;
1678 
1679 	ena_trace(ENA_DBG, "MSI-X TX/RX routine");
1680 
1681 	tx_ring = que->tx_ring;
1682 	rx_ring = que->rx_ring;
1683 	qid = que->id;
1684 	ena_qid = ENA_IO_TXQ_IDX(qid);
1685 	io_cq = &adapter->ena_dev->io_cq_queues[ena_qid];
1686 
1687 	for (i = 0; i < CLEAN_BUDGET; ++i) {
1688 		rxc = ena_rx_cleanup(rx_ring);
1689 
1690 		/* Protection from calling ena_tx_cleanup from ena_start_xmit */
1691 		ENA_RING_MTX_LOCK(tx_ring);
1692 		txc = ena_tx_cleanup(tx_ring);
1693 		ENA_RING_MTX_UNLOCK(tx_ring);
1694 
1695 		if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
1696 			return;
1697 
1698 		if (txc != TX_BUDGET && rxc != RX_BUDGET)
1699 		       break;
1700 	}
1701 
1702 	/* Signal that work is done and unmask interrupt */
1703 	ena_com_update_intr_reg(&intr_reg,
1704 	    RX_IRQ_INTERVAL,
1705 	    TX_IRQ_INTERVAL,
1706 	    true);
1707 	ena_com_unmask_intr(io_cq, &intr_reg);
1708 }
1709 
1710 static int
1711 ena_enable_msix(struct ena_adapter *adapter)
1712 {
1713 	device_t dev = adapter->pdev;
1714 	int i, msix_vecs, rc = 0;
1715 
1716 	/* Reserved the max msix vectors we might need */
1717 	msix_vecs = ENA_MAX_MSIX_VEC(adapter->num_queues);
1718 
1719 	adapter->msix_entries = ENA_MEM_ALLOC(adapter->ena_dev->dmadev,
1720 	    msix_vecs * sizeof(struct msix_entry));
1721 	if (!adapter->msix_entries) {
1722 		device_printf(dev,
1723 		    "Failed to allocate msix_entries, vectors %d\n", msix_vecs);
1724 		rc = ENOMEM;
1725 		goto error;
1726 	}
1727 	device_printf(dev, "Allocated msix_entries, vectors (cnt: %d)\n",
1728 	    msix_vecs);
1729 
1730 	for (i = 0; i < msix_vecs; i++) {
1731 		adapter->msix_entries[i].entry = i;
1732 		/* Vectors must start from 1 */
1733 		adapter->msix_entries[i].vector = i + 1;
1734 	}
1735 
1736 	rc = pci_alloc_msix(dev, &msix_vecs);
1737 	if (rc != 0) {
1738 		device_printf(dev,
1739 		    "Failed to enable MSIX, vectors %d rc %d\n", msix_vecs, rc);
1740 		ENA_MEM_FREE(adapter->ena_dev->dmadev, adapter->msix_entries);
1741 		adapter->msix_entries = NULL;
1742 		rc = ENOSPC;
1743 		goto error;
1744 	}
1745 
1746 	adapter->msix_vecs = msix_vecs;
1747 	adapter->msix_enabled = true;
1748 
1749 error:
1750 	return (rc);
1751 }
1752 
1753 static void
1754 ena_setup_mgmnt_intr(struct ena_adapter *adapter)
1755 {
1756 
1757 	snprintf(adapter->irq_tbl[ENA_MGMNT_IRQ_IDX].name,
1758 	    ENA_IRQNAME_SIZE, "ena-mgmnt@pci:%s",
1759 	    device_get_nameunit(adapter->pdev));
1760 	/*
1761 	 * Handler is NULL on purpose, it will be set
1762 	 * when mgmnt interrupt is acquired
1763 	 */
1764 	adapter->irq_tbl[ENA_MGMNT_IRQ_IDX].handler = NULL;
1765 	adapter->irq_tbl[ENA_MGMNT_IRQ_IDX].data = adapter;
1766 	adapter->irq_tbl[ENA_MGMNT_IRQ_IDX].vector =
1767 	    adapter->msix_entries[ENA_MGMNT_IRQ_IDX].vector;
1768 
1769 	return;
1770 }
1771 
1772 static void
1773 ena_setup_io_intr(struct ena_adapter *adapter)
1774 {
1775 	static int last_bind_cpu = -1;
1776 	int irq_idx;
1777 	ena_trace(ENA_DBG, "enter");
1778 
1779 	for (int i = 0; i < adapter->num_queues; i++) {
1780 		irq_idx = ENA_IO_IRQ_IDX(i);
1781 
1782 		snprintf(adapter->irq_tbl[irq_idx].name, ENA_IRQNAME_SIZE,
1783 		    "%s-TxRx-%d", device_get_nameunit(adapter->pdev), i);
1784 		adapter->irq_tbl[irq_idx].handler = ena_handle_msix;
1785 		adapter->irq_tbl[irq_idx].data = &adapter->que[i];
1786 		adapter->irq_tbl[irq_idx].vector =
1787 		    adapter->msix_entries[irq_idx].vector;
1788 		ena_trace(ENA_INFO | ENA_IOQ, "ena_setup_io_intr vector: %d\n",
1789 		    adapter->msix_entries[irq_idx].vector);
1790 #ifdef	RSS
1791 		adapter->que[i].cpu = adapter->irq_tbl[irq_idx].cpu =
1792 		    rss_getcpu(i % rss_getnumbuckets());
1793 #else
1794 		/*
1795 		 * We still want to bind rings to the corresponding cpu
1796 		 * using something similar to the RSS round-robin technique.
1797 		 */
1798 		if (last_bind_cpu < 0)
1799 			last_bind_cpu = CPU_FIRST();
1800 		adapter->que[i].cpu = adapter->irq_tbl[irq_idx].cpu =
1801 		    last_bind_cpu;
1802 		last_bind_cpu = CPU_NEXT(last_bind_cpu);
1803 #endif
1804 	}
1805 
1806 	return;
1807 }
1808 
1809 static int
1810 ena_request_mgmnt_irq(struct ena_adapter *adapter)
1811 {
1812 	struct ena_irq *irq;
1813 	unsigned long flags;
1814 	int rc, rcc;
1815 
1816 	flags = RF_ACTIVE | RF_SHAREABLE;
1817 
1818 	irq = &adapter->irq_tbl[ENA_MGMNT_IRQ_IDX];
1819 	irq->res = bus_alloc_resource_any(adapter->pdev, SYS_RES_IRQ,
1820 	    &irq->vector, flags);
1821 
1822 	if (irq->res == NULL) {
1823 		device_printf(adapter->pdev, "could not allocate "
1824 		    "irq vector: %d\n", irq->vector);
1825 		rc = ENXIO;
1826 		goto exit_res;
1827 	}
1828 
1829 	if ((rc = bus_activate_resource(adapter->pdev, SYS_RES_IRQ, irq->vector,
1830 	    irq->res)) != 0) {
1831 		device_printf(adapter->pdev, "could not activate "
1832 		    "irq vector: %d\n", irq->vector);
1833 		goto exit_intr;
1834 	}
1835 
1836 	if ((rc = bus_setup_intr(adapter->pdev, irq->res,
1837 	    INTR_TYPE_NET | INTR_MPSAFE, NULL,
1838 	    ena_intr_msix_mgmnt, irq->data, &irq->cookie)) != 0) {
1839 		device_printf(adapter->pdev, "failed to register "
1840 		    "interrupt handler for irq %ju: %d\n",
1841 		    rman_get_start(irq->res), rc);
1842 		goto exit_intr;
1843 	}
1844 	irq->requested = true;
1845 
1846 	return (rc);
1847 
1848 exit_intr:
1849 	device_printf(adapter->pdev, "exit_intr: releasing resource"
1850 	    " for irq %d\n", irq->vector);
1851 	rcc = bus_release_resource(adapter->pdev, SYS_RES_IRQ,
1852 	    irq->vector, irq->res);
1853 	if (rcc)
1854 		device_printf(adapter->pdev, "dev has no parent while "
1855 		    "releasing res for irq: %d\n", irq->vector);
1856 	irq->res = NULL;
1857 
1858 exit_res:
1859 	return (rc);
1860 }
1861 
1862 static int
1863 ena_request_io_irq(struct ena_adapter *adapter)
1864 {
1865 	struct ena_irq *irq;
1866 	unsigned long flags = 0;
1867 	int rc = 0, i, rcc;
1868 
1869 	if (!adapter->msix_enabled) {
1870 		device_printf(adapter->pdev, "failed to request irq\n");
1871 		return (EINVAL);
1872 	} else {
1873 		flags = RF_ACTIVE | RF_SHAREABLE;
1874 	}
1875 
1876 	for (i = ENA_IO_IRQ_FIRST_IDX; i < adapter->msix_vecs; i++) {
1877 		irq = &adapter->irq_tbl[i];
1878 
1879 		if (irq->requested)
1880 			continue;
1881 
1882 		irq->res = bus_alloc_resource_any(adapter->pdev, SYS_RES_IRQ,
1883 		    &irq->vector, flags);
1884 		if (irq->res == NULL) {
1885 			device_printf(adapter->pdev, "could not allocate "
1886 			    "irq vector: %d\n", irq->vector);
1887 			goto err;
1888 		}
1889 
1890 		if ((rc = bus_setup_intr(adapter->pdev, irq->res,
1891 			    INTR_TYPE_NET | INTR_MPSAFE, NULL, irq->handler,
1892 			    irq->data, &irq->cookie)) != 0) {
1893 			device_printf(adapter->pdev, "failed to register "
1894 			    "interrupt handler for irq %ju: %d\n",
1895 			    rman_get_start(irq->res), rc);
1896 			goto err;
1897 		}
1898 		irq->requested = true;
1899 
1900 #ifdef	RSS
1901 		device_printf(adapter->pdev, "queue %d - RSS bucket %d\n",
1902 		    i - ENA_IO_IRQ_FIRST_IDX, irq->cpu);
1903 #else
1904 		device_printf(adapter->pdev, "queue %d - cpu %d\n",
1905 		    i - ENA_IO_IRQ_FIRST_IDX, irq->cpu);
1906 #endif
1907 	}
1908 
1909 	return (rc);
1910 
1911 err:
1912 
1913 	for (; i >= ENA_IO_IRQ_FIRST_IDX; i--) {
1914 		irq = &adapter->irq_tbl[i];
1915 		rcc = 0;
1916 
1917 		/* Once we entered err: section and irq->requested is true we
1918 		   free both intr and resources */
1919 		if (irq->requested == true)
1920 			rcc = bus_teardown_intr(adapter->pdev, irq->res, irq->cookie);
1921 		if (rcc)
1922 			device_printf(adapter->pdev, "could not release"
1923 			    " irq: %d, error: %d\n", irq->vector, rcc);
1924 
1925 		/* If we entred err: section without irq->requested set we know
1926 		   it was bus_alloc_resource_any() that needs cleanup, provided
1927 		   res is not NULL. In case res is NULL no work in needed in
1928 		   this iteration */
1929 		rcc = 0;
1930 		if (irq->res != NULL) {
1931 			rcc = bus_release_resource(adapter->pdev, SYS_RES_IRQ,
1932 			    irq->vector, irq->res);
1933 		}
1934 		if (rcc)
1935 			device_printf(adapter->pdev, "dev has no parent while "
1936 			    "releasing res for irq: %d\n", irq->vector);
1937 		irq->requested = false;
1938 		irq->res = NULL;
1939 	}
1940 
1941 	return (rc);
1942 }
1943 
1944 static void
1945 ena_free_mgmnt_irq(struct ena_adapter *adapter)
1946 {
1947 	struct ena_irq *irq;
1948 	int rc;
1949 
1950 	irq = &adapter->irq_tbl[ENA_MGMNT_IRQ_IDX];
1951 	if (irq->requested) {
1952 		ena_trace(ENA_INFO | ENA_ADMQ, "tear down irq: %d\n",
1953 		    irq->vector);
1954 		rc = bus_teardown_intr(adapter->pdev, irq->res, irq->cookie);
1955 		if (rc)
1956 			device_printf(adapter->pdev, "failed to tear "
1957 			    "down irq: %d\n", irq->vector);
1958 		irq->requested = 0;
1959 	}
1960 
1961 	if (irq->res != NULL) {
1962 		ena_trace(ENA_INFO | ENA_ADMQ, "release resource irq: %d\n",
1963 		    irq->vector);
1964 		rc = bus_release_resource(adapter->pdev, SYS_RES_IRQ,
1965 		    irq->vector, irq->res);
1966 		irq->res = NULL;
1967 		if (rc)
1968 			device_printf(adapter->pdev, "dev has no parent while "
1969 			    "releasing res for irq: %d\n", irq->vector);
1970 	}
1971 
1972 	return;
1973 }
1974 
1975 static void
1976 ena_free_io_irq(struct ena_adapter *adapter)
1977 {
1978 	struct ena_irq *irq;
1979 	int rc;
1980 
1981 	for (int i = ENA_IO_IRQ_FIRST_IDX; i < adapter->msix_vecs; i++) {
1982 		irq = &adapter->irq_tbl[i];
1983 		if (irq->requested) {
1984 			ena_trace(ENA_INFO | ENA_IOQ, "tear down irq: %d\n",
1985 			    irq->vector);
1986 			rc = bus_teardown_intr(adapter->pdev, irq->res,
1987 			    irq->cookie);
1988 			if (rc) {
1989 				device_printf(adapter->pdev, "failed to tear "
1990 				    "down irq: %d\n", irq->vector);
1991 			}
1992 			irq->requested = 0;
1993 		}
1994 
1995 		if (irq->res != NULL) {
1996 			ena_trace(ENA_INFO | ENA_IOQ, "release resource irq: %d\n",
1997 			    irq->vector);
1998 			rc = bus_release_resource(adapter->pdev, SYS_RES_IRQ,
1999 			    irq->vector, irq->res);
2000 			irq->res = NULL;
2001 			if (rc) {
2002 				device_printf(adapter->pdev, "dev has no parent"
2003 				    " while releasing res for irq: %d\n",
2004 				    irq->vector);
2005 			}
2006 		}
2007 	}
2008 
2009 	return;
2010 }
2011 
2012 static void
2013 ena_free_irqs(struct ena_adapter* adapter)
2014 {
2015 
2016 	ena_free_io_irq(adapter);
2017 	ena_free_mgmnt_irq(adapter);
2018 	ena_disable_msix(adapter);
2019 }
2020 
2021 static void
2022 ena_disable_msix(struct ena_adapter *adapter)
2023 {
2024 
2025 	pci_release_msi(adapter->pdev);
2026 
2027 	adapter->msix_vecs = 0;
2028 	ENA_MEM_FREE(adapter->ena_dev->dmadev, adapter->msix_entries);
2029 	adapter->msix_entries = NULL;
2030 }
2031 
2032 static void
2033 ena_unmask_all_io_irqs(struct ena_adapter *adapter)
2034 {
2035 	struct ena_com_io_cq* io_cq;
2036 	struct ena_eth_io_intr_reg intr_reg;
2037 	uint16_t ena_qid;
2038 	int i;
2039 
2040 	/* Unmask interrupts for all queues */
2041 	for (i = 0; i < adapter->num_queues; i++) {
2042 		ena_qid = ENA_IO_TXQ_IDX(i);
2043 		io_cq = &adapter->ena_dev->io_cq_queues[ena_qid];
2044 		ena_com_update_intr_reg(&intr_reg, 0, 0, true);
2045 		ena_com_unmask_intr(io_cq, &intr_reg);
2046 	}
2047 }
2048 
2049 /* Configure the Rx forwarding */
2050 static int ena_rss_configure(struct ena_adapter *adapter)
2051 {
2052 	struct ena_com_dev *ena_dev = adapter->ena_dev;
2053 	int rc;
2054 
2055 	/* Set indirect table */
2056 	rc = ena_com_indirect_table_set(ena_dev);
2057 	if (unlikely(rc && rc != EOPNOTSUPP))
2058 		return rc;
2059 
2060 	/* Configure hash function (if supported) */
2061 	rc = ena_com_set_hash_function(ena_dev);
2062 	if (unlikely(rc && (rc != EOPNOTSUPP)))
2063 		return rc;
2064 
2065 	/* Configure hash inputs (if supported) */
2066 	rc = ena_com_set_hash_ctrl(ena_dev);
2067 	if (unlikely(rc && (rc != EOPNOTSUPP)))
2068 		return rc;
2069 
2070 	return 0;
2071 }
2072 
2073 static int
2074 ena_up_complete(struct ena_adapter *adapter)
2075 {
2076 	int rc;
2077 
2078 	if (adapter->rss_support) {
2079 		rc = ena_rss_configure(adapter);
2080 		if (rc)
2081 			return (rc);
2082 	}
2083 
2084 	ena_change_mtu(adapter->ifp, adapter->ifp->if_mtu);
2085 	ena_refill_all_rx_bufs(adapter);
2086 	ena_reset_counters((counter_u64_t *)&adapter->hw_stats,
2087 	    sizeof(adapter->hw_stats));
2088 
2089 	return (0);
2090 }
2091 
2092 static int
2093 ena_up(struct ena_adapter *adapter)
2094 {
2095 	int rc = 0;
2096 
2097 	if (!device_is_attached(adapter->pdev)) {
2098 		device_printf(adapter->pdev, "device is not attached!\n");
2099 		return (ENXIO);
2100 	}
2101 
2102 	if (!adapter->running) {
2103 		device_printf(adapter->pdev, "device is not running!\n");
2104 		return (ENXIO);
2105 	}
2106 
2107 	if (!adapter->up) {
2108 		device_printf(adapter->pdev, "device is going UP\n");
2109 
2110 		/* setup interrupts for IO queues */
2111 		ena_setup_io_intr(adapter);
2112 		rc = ena_request_io_irq(adapter);
2113 		if (rc) {
2114 			ena_trace(ENA_ALERT, "err_req_irq");
2115 			goto err_req_irq;
2116 		}
2117 
2118 		/* allocate transmit descriptors */
2119 		rc = ena_setup_all_tx_resources(adapter);
2120 		if (rc) {
2121 			ena_trace(ENA_ALERT, "err_setup_tx");
2122 			goto err_setup_tx;
2123 		}
2124 
2125 		/* allocate receive descriptors */
2126 		rc = ena_setup_all_rx_resources(adapter);
2127 		if (rc) {
2128 			ena_trace(ENA_ALERT, "err_setup_rx");
2129 			goto err_setup_rx;
2130 		}
2131 
2132 		/* create IO queues for Rx & Tx */
2133 		rc = ena_create_io_queues(adapter);
2134 		if (rc) {
2135 			ena_trace(ENA_ALERT,
2136 			    "create IO queues failed");
2137 			goto err_io_que;
2138 		}
2139 
2140 		if (adapter->link_status)
2141 			if_link_state_change(adapter->ifp, LINK_STATE_UP);
2142 
2143 		rc = ena_up_complete(adapter);
2144 		if (rc)
2145 			goto err_up_complete;
2146 
2147 		counter_u64_add(adapter->dev_stats.interface_up, 1);
2148 
2149 		ena_update_hwassist(adapter);
2150 
2151 		if_setdrvflagbits(adapter->ifp, IFF_DRV_RUNNING,
2152 		    IFF_DRV_OACTIVE);
2153 
2154 		callout_reset_sbt(&adapter->timer_service, SBT_1S, SBT_1S,
2155 		    ena_timer_service, (void *)adapter, 0);
2156 
2157 		adapter->up = true;
2158 
2159 		ena_unmask_all_io_irqs(adapter);
2160 	}
2161 
2162 	return (0);
2163 
2164 err_up_complete:
2165 	ena_destroy_all_io_queues(adapter);
2166 err_io_que:
2167 	ena_free_all_rx_resources(adapter);
2168 err_setup_rx:
2169 	ena_free_all_tx_resources(adapter);
2170 err_setup_tx:
2171 	ena_free_io_irq(adapter);
2172 err_req_irq:
2173 	return (rc);
2174 }
2175 
2176 static uint64_t
2177 ena_get_counter(if_t ifp, ift_counter cnt)
2178 {
2179 	struct ena_adapter *adapter;
2180 	struct ena_hw_stats *stats;
2181 
2182 	adapter = if_getsoftc(ifp);
2183 	stats = &adapter->hw_stats;
2184 
2185 	switch (cnt) {
2186 	case IFCOUNTER_IPACKETS:
2187 		return (counter_u64_fetch(stats->rx_packets));
2188 	case IFCOUNTER_OPACKETS:
2189 		return (counter_u64_fetch(stats->tx_packets));
2190 	case IFCOUNTER_IBYTES:
2191 		return (counter_u64_fetch(stats->rx_bytes));
2192 	case IFCOUNTER_OBYTES:
2193 		return (counter_u64_fetch(stats->tx_bytes));
2194 	case IFCOUNTER_IQDROPS:
2195 		return (counter_u64_fetch(stats->rx_drops));
2196 	default:
2197 		return (if_get_counter_default(ifp, cnt));
2198 	}
2199 }
2200 
2201 static int
2202 ena_media_change(if_t ifp)
2203 {
2204 	/* Media Change is not supported by firmware */
2205 	return (0);
2206 }
2207 
2208 static void
2209 ena_media_status(if_t ifp, struct ifmediareq *ifmr)
2210 {
2211 	struct ena_adapter *adapter = if_getsoftc(ifp);
2212 	ena_trace(ENA_DBG, "enter");
2213 
2214 	ENA_DEV_LOCK;
2215 
2216 	ifmr->ifm_status = IFM_AVALID;
2217 	ifmr->ifm_active = IFM_ETHER;
2218 
2219 	if (!adapter->link_status) {
2220 		ENA_DEV_UNLOCK;
2221 		ena_trace(ENA_WARNING, "link_status = false");
2222 		return;
2223 	}
2224 
2225 	ifmr->ifm_status |= IFM_ACTIVE;
2226 	ifmr->ifm_active |= IFM_10G_T | IFM_FDX;
2227 
2228 	ENA_DEV_UNLOCK;
2229 
2230 	return;
2231 }
2232 
2233 static void
2234 ena_init(void *arg)
2235 {
2236 	struct ena_adapter *adapter = (struct ena_adapter *)arg;
2237 
2238 	if (adapter->up == false) {
2239 		sx_xlock(&adapter->ioctl_sx);
2240 		ena_up(adapter);
2241 		sx_unlock(&adapter->ioctl_sx);
2242 	}
2243 
2244 	return;
2245 }
2246 
2247 static int
2248 ena_ioctl(if_t ifp, u_long command, caddr_t data)
2249 {
2250 	struct ena_adapter *adapter;
2251 	struct ifreq *ifr;
2252 	int rc;
2253 
2254 	adapter = ifp->if_softc;
2255 	ifr = (struct ifreq *)data;
2256 
2257 	/*
2258 	 * Acquiring lock to prevent from running up and down routines parallel.
2259 	 */
2260 	rc = 0;
2261 	switch (command) {
2262 	case SIOCSIFMTU:
2263 		sx_xlock(&adapter->ioctl_sx);
2264 		ena_down(adapter);
2265 
2266 		ena_change_mtu(ifp, ifr->ifr_mtu);
2267 
2268 		rc = ena_up(adapter);
2269 		sx_unlock(&adapter->ioctl_sx);
2270 		break;
2271 
2272 	case SIOCSIFFLAGS:
2273 		if (ifp->if_flags & IFF_UP) {
2274 			if ((ifp->if_drv_flags & IFF_DRV_RUNNING)) {
2275 				if (ifp->if_flags & (IFF_PROMISC |
2276 				    IFF_ALLMULTI)) {
2277 					device_printf(adapter->pdev,
2278 					    "ioctl promisc/allmulti\n");
2279 				}
2280 			} else {
2281 				sx_xlock(&adapter->ioctl_sx);
2282 				rc = ena_up(adapter);
2283 				sx_unlock(&adapter->ioctl_sx);
2284 			}
2285 		} else {
2286 			if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
2287 				sx_xlock(&adapter->ioctl_sx);
2288 				ena_down(adapter);
2289 				sx_unlock(&adapter->ioctl_sx);
2290 			}
2291 		}
2292 		break;
2293 
2294 	case SIOCADDMULTI:
2295 	case SIOCDELMULTI:
2296 		break;
2297 
2298 	case SIOCSIFMEDIA:
2299 	case SIOCGIFMEDIA:
2300 		rc = ifmedia_ioctl(ifp, ifr, &adapter->media, command);
2301 		break;
2302 
2303 	case SIOCSIFCAP:
2304 		{
2305 			int reinit = 0;
2306 
2307 			if (ifr->ifr_reqcap != ifp->if_capenable) {
2308 				ifp->if_capenable = ifr->ifr_reqcap;
2309 				reinit = 1;
2310 			}
2311 
2312 			if (reinit && (ifp->if_drv_flags & IFF_DRV_RUNNING)) {
2313 				sx_xlock(&adapter->ioctl_sx);
2314 				ena_down(adapter);
2315 				rc = ena_up(adapter);
2316 				sx_unlock(&adapter->ioctl_sx);
2317 			}
2318 		}
2319 
2320 		break;
2321 	default:
2322 		rc = ether_ioctl(ifp, command, data);
2323 		break;
2324 	}
2325 
2326 	return (rc);
2327 }
2328 
2329 static int
2330 ena_get_dev_offloads(struct ena_com_dev_get_features_ctx *feat)
2331 {
2332 	int caps = 0;
2333 
2334 	if (feat->offload.tx &
2335 	    (ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV4_CSUM_FULL_MASK |
2336 	    ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV4_CSUM_PART_MASK |
2337 	    ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L3_CSUM_IPV4_MASK))
2338 		caps |= IFCAP_TXCSUM;
2339 
2340 	if (feat->offload.tx &
2341 	    (ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV6_CSUM_FULL_MASK |
2342 	    ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV6_CSUM_PART_MASK))
2343 		caps |= IFCAP_TXCSUM_IPV6;
2344 
2345 	if (feat->offload.tx & ENA_ADMIN_FEATURE_OFFLOAD_DESC_TSO_IPV4_MASK)
2346 		caps |= IFCAP_TSO4;
2347 
2348 	if (feat->offload.tx & ENA_ADMIN_FEATURE_OFFLOAD_DESC_TSO_IPV6_MASK)
2349 		caps |= IFCAP_TSO6;
2350 
2351 	if (feat->offload.rx_supported &
2352 	    (ENA_ADMIN_FEATURE_OFFLOAD_DESC_RX_L4_IPV4_CSUM_MASK |
2353 	    ENA_ADMIN_FEATURE_OFFLOAD_DESC_RX_L3_CSUM_IPV4_MASK))
2354 		caps |= IFCAP_RXCSUM;
2355 
2356 	if (feat->offload.rx_supported &
2357 	    ENA_ADMIN_FEATURE_OFFLOAD_DESC_RX_L4_IPV6_CSUM_MASK)
2358 		caps |= IFCAP_RXCSUM_IPV6;
2359 
2360 	caps |= IFCAP_LRO | IFCAP_JUMBO_MTU;
2361 
2362 	return (caps);
2363 }
2364 
2365 static void
2366 ena_update_host_info(struct ena_admin_host_info *host_info, if_t ifp)
2367 {
2368 
2369 	host_info->supported_network_features[0] =
2370 	    (uint32_t)if_getcapabilities(ifp);
2371 }
2372 
2373 static void
2374 ena_update_hwassist(struct ena_adapter *adapter)
2375 {
2376 	if_t ifp = adapter->ifp;
2377 	uint32_t feat = adapter->tx_offload_cap;
2378 	int cap = if_getcapenable(ifp);
2379 	int flags = 0;
2380 
2381 	if_clearhwassist(ifp);
2382 
2383 	if (cap & IFCAP_TXCSUM) {
2384 		if (feat & ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L3_CSUM_IPV4_MASK)
2385 			flags |= CSUM_IP;
2386 		if (feat &
2387 		    (ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV4_CSUM_FULL_MASK |
2388 		    ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV4_CSUM_PART_MASK))
2389 			flags |= CSUM_IP_UDP | CSUM_IP_TCP;
2390 	}
2391 
2392 	if (cap & IFCAP_TXCSUM_IPV6)
2393 		flags |= CSUM_IP6_UDP | CSUM_IP6_TCP;
2394 
2395 	if (cap & IFCAP_TSO4)
2396 		flags |= CSUM_IP_TSO;
2397 
2398 	if (cap & IFCAP_TSO6)
2399 		flags |= CSUM_IP6_TSO;
2400 
2401 	if_sethwassistbits(ifp, flags, 0);
2402 }
2403 
2404 static int
2405 ena_setup_ifnet(device_t pdev, struct ena_adapter *adapter,
2406     struct ena_com_dev_get_features_ctx *feat)
2407 {
2408 	if_t ifp;
2409 	int caps = 0;
2410 
2411 	ena_trace(ENA_DBG, "enter");
2412 
2413 	ifp = adapter->ifp = if_gethandle(IFT_ETHER);
2414 	if (ifp == 0) {
2415 		device_printf(pdev, "can not allocate ifnet structure\n");
2416 		return (ENXIO);
2417 	}
2418 	if_initname(ifp, device_get_name(pdev), device_get_unit(pdev));
2419 	if_setdev(ifp, pdev);
2420 	if_setsoftc(ifp, adapter);
2421 
2422 	if_setflags(ifp, IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST);
2423 	if_setinitfn(ifp, ena_init);
2424 	if_settransmitfn(ifp, ena_mq_start);
2425 	if_setqflushfn(ifp, ena_qflush);
2426 	if_setioctlfn(ifp, ena_ioctl);
2427 	if_setgetcounterfn(ifp, ena_get_counter);
2428 
2429 	if_setsendqlen(ifp, adapter->tx_ring_size);
2430 	if_setsendqready(ifp);
2431 	if_setmtu(ifp, ETHERMTU);
2432 	if_setbaudrate(ifp, 0);
2433 	/* Zeroize capabilities... */
2434 	if_setcapabilities(ifp, 0);
2435 	if_setcapenable(ifp, 0);
2436 	/* check hardware support */
2437 	caps = ena_get_dev_offloads(feat);
2438 	/* ... and set them */
2439 	if_setcapabilitiesbit(ifp, caps, 0);
2440 
2441 	/* TSO parameters */
2442 	ifp->if_hw_tsomax = ENA_TSO_MAXSIZE -
2443 	    (ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN);
2444 	ifp->if_hw_tsomaxsegcount = adapter->max_tx_sgl_size - 1;
2445 	ifp->if_hw_tsomaxsegsize = ENA_TSO_MAXSIZE;
2446 
2447 	if_setifheaderlen(ifp, sizeof(struct ether_vlan_header));
2448 	if_setcapenable(ifp, if_getcapabilities(ifp));
2449 
2450 	/*
2451 	 * Specify the media types supported by this adapter and register
2452 	 * callbacks to update media and link information
2453 	 */
2454 	ifmedia_init(&adapter->media, IFM_IMASK,
2455 	    ena_media_change, ena_media_status);
2456 	ifmedia_add(&adapter->media, IFM_ETHER | IFM_AUTO, 0, NULL);
2457 	ifmedia_set(&adapter->media, IFM_ETHER | IFM_AUTO);
2458 
2459 	ether_ifattach(ifp, adapter->mac_addr);
2460 
2461 	return (0);
2462 }
2463 
2464 static void
2465 ena_down(struct ena_adapter *adapter)
2466 {
2467 	int rc;
2468 
2469 	if (adapter->up) {
2470 		device_printf(adapter->pdev, "device is going DOWN\n");
2471 
2472 		callout_drain(&adapter->timer_service);
2473 
2474 		adapter->up = false;
2475 		if_setdrvflagbits(adapter->ifp, IFF_DRV_OACTIVE,
2476 		    IFF_DRV_RUNNING);
2477 
2478 		ena_free_io_irq(adapter);
2479 
2480 		if (adapter->trigger_reset) {
2481 			rc = ena_com_dev_reset(adapter->ena_dev,
2482 			    adapter->reset_reason);
2483 			if (rc)
2484 				device_printf(adapter->pdev,
2485 				    "Device reset failed\n");
2486 		}
2487 
2488 		ena_destroy_all_io_queues(adapter);
2489 
2490 		ena_free_all_tx_bufs(adapter);
2491 		ena_free_all_rx_bufs(adapter);
2492 		ena_free_all_tx_resources(adapter);
2493 		ena_free_all_rx_resources(adapter);
2494 
2495 		counter_u64_add(adapter->dev_stats.interface_down, 1);
2496 	}
2497 
2498 	return;
2499 }
2500 
2501 static void
2502 ena_tx_csum(struct ena_com_tx_ctx *ena_tx_ctx, struct mbuf *mbuf)
2503 {
2504 	struct ena_com_tx_meta *ena_meta;
2505 	struct ether_vlan_header *eh;
2506 	u32 mss;
2507 	bool offload;
2508 	uint16_t etype;
2509 	int ehdrlen;
2510 	struct ip *ip;
2511 	int iphlen;
2512 	struct tcphdr *th;
2513 
2514 	offload = false;
2515 	ena_meta = &ena_tx_ctx->ena_meta;
2516 	mss = mbuf->m_pkthdr.tso_segsz;
2517 
2518 	if (mss != 0)
2519 		offload = true;
2520 
2521 	if ((mbuf->m_pkthdr.csum_flags & CSUM_TSO) != 0)
2522 		offload = true;
2523 
2524 	if ((mbuf->m_pkthdr.csum_flags & CSUM_OFFLOAD) != 0)
2525 		offload = true;
2526 
2527 	if (offload == false) {
2528 		ena_tx_ctx->meta_valid = 0;
2529 		return;
2530 	}
2531 
2532 	/* Determine where frame payload starts. */
2533 	eh = mtod(mbuf, struct ether_vlan_header *);
2534 	if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
2535 		etype = ntohs(eh->evl_proto);
2536 		ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
2537 	} else {
2538 		etype = ntohs(eh->evl_encap_proto);
2539 		ehdrlen = ETHER_HDR_LEN;
2540 	}
2541 
2542 	ip = (struct ip *)(mbuf->m_data + ehdrlen);
2543 	iphlen = ip->ip_hl << 2;
2544 	th = (struct tcphdr *)((caddr_t)ip + iphlen);
2545 
2546 	if ((mbuf->m_pkthdr.csum_flags & CSUM_IP) != 0) {
2547 		ena_tx_ctx->l3_csum_enable = 1;
2548 	}
2549 	if ((mbuf->m_pkthdr.csum_flags & CSUM_TSO) != 0) {
2550 		ena_tx_ctx->tso_enable = 1;
2551 		ena_meta->l4_hdr_len = (th->th_off);
2552 	}
2553 
2554 	switch (etype) {
2555 	case ETHERTYPE_IP:
2556 		ena_tx_ctx->l3_proto = ENA_ETH_IO_L3_PROTO_IPV4;
2557 		if (ip->ip_off == 0)
2558 			ena_tx_ctx->df = 1;
2559 		break;
2560 	case ETHERTYPE_IPV6:
2561 		ena_tx_ctx->l3_proto = ENA_ETH_IO_L3_PROTO_IPV6;
2562 
2563 	default:
2564 		break;
2565 	}
2566 
2567 	if (ip->ip_p == IPPROTO_TCP) {
2568 		ena_tx_ctx->l4_proto = ENA_ETH_IO_L4_PROTO_TCP;
2569 		if (mbuf->m_pkthdr.csum_flags & (CSUM_IP_TCP | CSUM_IP6_TCP))
2570 		    ena_tx_ctx->l4_csum_enable = 1;
2571 		else
2572 		    ena_tx_ctx->l4_csum_enable = 0;
2573 	} else if (ip->ip_p == IPPROTO_UDP) {
2574 		ena_tx_ctx->l4_proto = ENA_ETH_IO_L4_PROTO_UDP;
2575 		if (mbuf->m_pkthdr.csum_flags & (CSUM_IP_UDP | CSUM_IP6_UDP))
2576 		    ena_tx_ctx->l4_csum_enable = 1;
2577 		else
2578 		    ena_tx_ctx->l4_csum_enable = 0;
2579 	} else {
2580 		ena_tx_ctx->l4_proto = ENA_ETH_IO_L4_PROTO_UNKNOWN;
2581 		ena_tx_ctx->l4_csum_enable = 0;
2582 	}
2583 
2584 	ena_meta->mss = mss;
2585 	ena_meta->l3_hdr_len = iphlen;
2586 	ena_meta->l3_hdr_offset = ehdrlen;
2587 	ena_tx_ctx->meta_valid = 1;
2588 }
2589 
2590 static int
2591 ena_check_and_collapse_mbuf(struct ena_ring *tx_ring, struct mbuf **mbuf)
2592 {
2593 	struct ena_adapter *adapter;
2594 	struct mbuf *collapsed_mbuf;
2595 	int num_frags;
2596 
2597 	adapter = tx_ring->adapter;
2598 	num_frags = ena_mbuf_count(*mbuf);
2599 
2600 	/* One segment must be reserved for configuration descriptor. */
2601 	if (num_frags < adapter->max_tx_sgl_size)
2602 		return (0);
2603 	counter_u64_add(tx_ring->tx_stats.collapse, 1);
2604 
2605 	collapsed_mbuf = m_collapse(*mbuf, M_NOWAIT,
2606 	    adapter->max_tx_sgl_size - 1);
2607 	if (collapsed_mbuf == NULL) {
2608 		counter_u64_add(tx_ring->tx_stats.collapse_err, 1);
2609 		return (ENOMEM);
2610 	}
2611 
2612 	/* If mbuf was collapsed succesfully, original mbuf is released. */
2613 	*mbuf = collapsed_mbuf;
2614 
2615 	return (0);
2616 }
2617 
2618 static int
2619 ena_xmit_mbuf(struct ena_ring *tx_ring, struct mbuf **mbuf)
2620 {
2621 	struct ena_adapter *adapter;
2622 	struct ena_tx_buffer *tx_info;
2623 	struct ena_com_tx_ctx ena_tx_ctx;
2624 	struct ena_com_dev *ena_dev;
2625 	struct ena_com_buf *ena_buf;
2626 	struct ena_com_io_sq* io_sq;
2627 	bus_dma_segment_t segs[ENA_BUS_DMA_SEGS];
2628 	void *push_hdr;
2629 	uint16_t next_to_use;
2630 	uint16_t req_id;
2631 	uint16_t push_len;
2632 	uint16_t ena_qid;
2633 	uint32_t len, nsegs, header_len;
2634 	int i, rc;
2635 	int nb_hw_desc;
2636 
2637 	ena_qid = ENA_IO_TXQ_IDX(tx_ring->que->id);
2638 	adapter = tx_ring->que->adapter;
2639 	ena_dev = adapter->ena_dev;
2640 	io_sq = &adapter->ena_dev->io_sq_queues[ena_qid];
2641 
2642 	ENA_ASSERT(*mbuf, "mbuf is NULL\n");
2643 
2644 	rc = ena_check_and_collapse_mbuf(tx_ring, mbuf);
2645 	if (rc) {
2646 		ena_trace(ENA_WARNING,
2647 		    "Failed to collapse mbuf! err: %d", rc);
2648 		return (rc);
2649 	}
2650 
2651 	next_to_use = tx_ring->next_to_use;
2652 	req_id = tx_ring->free_tx_ids[next_to_use];
2653 	tx_info = &tx_ring->tx_buffer_info[req_id];
2654 
2655 	tx_info->mbuf = *mbuf;
2656 	tx_info->num_of_bufs = 0;
2657 
2658 	ena_buf = tx_info->bufs;
2659 	len = (*mbuf)->m_len;
2660 
2661 	ena_trace(ENA_DBG | ENA_TXPTH, "Tx: %d bytes", (*mbuf)->m_pkthdr.len);
2662 
2663 	push_len = 0;
2664 	header_len = min_t(uint32_t, len, tx_ring->tx_max_header_size);
2665 	push_hdr = NULL;
2666 
2667 	rc = bus_dmamap_load_mbuf_sg(adapter->tx_buf_tag, tx_info->map,
2668 	    *mbuf, segs, &nsegs, BUS_DMA_NOWAIT);
2669 
2670 	if (rc || (nsegs == 0)) {
2671 		ena_trace(ENA_WARNING,
2672 		    "dmamap load failed! err: %d nsegs: %d", rc, nsegs);
2673 		counter_u64_add(tx_ring->tx_stats.dma_mapping_err, 1);
2674 		tx_info->mbuf = NULL;
2675 		if (rc == ENOMEM)
2676 			return (ENA_COM_NO_MEM);
2677 		else
2678 			return (ENA_COM_INVAL);
2679 	}
2680 
2681 	for (i = 0; i < nsegs; i++) {
2682 		ena_buf->len = segs[i].ds_len;
2683 		ena_buf->paddr = segs[i].ds_addr;
2684 		ena_buf++;
2685 	}
2686 	tx_info->num_of_bufs = nsegs;
2687 
2688 	memset(&ena_tx_ctx, 0x0, sizeof(struct ena_com_tx_ctx));
2689 	ena_tx_ctx.ena_bufs = tx_info->bufs;
2690 	ena_tx_ctx.push_header = push_hdr;
2691 	ena_tx_ctx.num_bufs = tx_info->num_of_bufs;
2692 	ena_tx_ctx.req_id = req_id;
2693 	ena_tx_ctx.header_len = header_len;
2694 
2695 	/* Set flags and meta data */
2696 	ena_tx_csum(&ena_tx_ctx, *mbuf);
2697 	/* Prepare the packet's descriptors and send them to device */
2698 	rc = ena_com_prepare_tx(io_sq, &ena_tx_ctx, &nb_hw_desc);
2699 	if (rc != 0) {
2700 		ena_trace(ENA_WARNING, "failed to prepare tx bufs\n");
2701 		counter_enter();
2702 		counter_u64_add_protected(tx_ring->tx_stats.queue_stop, 1);
2703 		counter_u64_add_protected(tx_ring->tx_stats.prepare_ctx_err, 1);
2704 		counter_exit();
2705 		goto dma_error;
2706 	}
2707 
2708 	counter_enter();
2709 	counter_u64_add_protected(tx_ring->tx_stats.cnt, 1);
2710 	counter_u64_add_protected(tx_ring->tx_stats.bytes,  (*mbuf)->m_pkthdr.len);
2711 
2712 	counter_u64_add_protected(adapter->hw_stats.tx_packets, 1);
2713 	counter_u64_add_protected(adapter->hw_stats.tx_bytes,
2714 	    (*mbuf)->m_pkthdr.len);
2715 	counter_exit();
2716 
2717 	tx_info->tx_descs = nb_hw_desc;
2718 	getbinuptime(&tx_info->timestamp);
2719 	tx_info->print_once = true;
2720 
2721 	tx_ring->next_to_use = ENA_TX_RING_IDX_NEXT(next_to_use,
2722 	    tx_ring->ring_size);
2723 
2724 	bus_dmamap_sync(adapter->tx_buf_tag, tx_info->map, BUS_DMASYNC_PREWRITE);
2725 
2726 	return (0);
2727 
2728 dma_error:
2729 	tx_info->mbuf = NULL;
2730 	bus_dmamap_unload(adapter->tx_buf_tag, tx_info->map);
2731 
2732 	return (rc);
2733 }
2734 
2735 static void
2736 ena_start_xmit(struct ena_ring *tx_ring)
2737 {
2738 	struct mbuf *mbuf;
2739 	struct ena_adapter *adapter = tx_ring->adapter;
2740 	struct ena_com_io_sq* io_sq;
2741 	int ena_qid;
2742 	int acum_pkts = 0;
2743 	int ret = 0;
2744 
2745 	if ((adapter->ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
2746 		return;
2747 
2748 	if (!adapter->link_status)
2749 		return;
2750 
2751 	ena_qid = ENA_IO_TXQ_IDX(tx_ring->que->id);
2752 	io_sq = &adapter->ena_dev->io_sq_queues[ena_qid];
2753 
2754 	while ((mbuf = drbr_peek(adapter->ifp, tx_ring->br)) != NULL) {
2755 		ena_trace(ENA_DBG | ENA_TXPTH, "\ndequeued mbuf %p with flags %#x and"
2756 		    " header csum flags %#jx",
2757 		    mbuf, mbuf->m_flags, mbuf->m_pkthdr.csum_flags);
2758 
2759 		if (!ena_com_sq_have_enough_space(io_sq,
2760 		    ENA_TX_CLEANUP_THRESHOLD))
2761 			ena_tx_cleanup(tx_ring);
2762 
2763 		if ((ret = ena_xmit_mbuf(tx_ring, &mbuf)) != 0) {
2764 			if (ret == ENA_COM_NO_MEM) {
2765 				drbr_putback(adapter->ifp, tx_ring->br, mbuf);
2766 			} else if (ret == ENA_COM_NO_SPACE) {
2767 				drbr_putback(adapter->ifp, tx_ring->br, mbuf);
2768 			} else {
2769 				m_freem(mbuf);
2770 				drbr_advance(adapter->ifp, tx_ring->br);
2771 			}
2772 
2773 			break;
2774 		}
2775 
2776 		drbr_advance(adapter->ifp, tx_ring->br);
2777 
2778 		if ((adapter->ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
2779 			return;
2780 
2781 		acum_pkts++;
2782 
2783 		BPF_MTAP(adapter->ifp, mbuf);
2784 
2785 		if (acum_pkts == DB_THRESHOLD) {
2786 			acum_pkts = 0;
2787 			wmb();
2788 			/* Trigger the dma engine */
2789 			ena_com_write_sq_doorbell(io_sq);
2790 			counter_u64_add(tx_ring->tx_stats.doorbells, 1);
2791 		}
2792 
2793 	}
2794 
2795 	if (acum_pkts) {
2796 		wmb();
2797 		/* Trigger the dma engine */
2798 		ena_com_write_sq_doorbell(io_sq);
2799 		counter_u64_add(tx_ring->tx_stats.doorbells, 1);
2800 	}
2801 
2802 	if (!ena_com_sq_have_enough_space(io_sq,
2803 	    ENA_TX_CLEANUP_THRESHOLD))
2804 		ena_tx_cleanup(tx_ring);
2805 }
2806 
2807 static void
2808 ena_deferred_mq_start(void *arg, int pending)
2809 {
2810 	struct ena_ring *tx_ring = (struct ena_ring *)arg;
2811 	struct ifnet *ifp = tx_ring->adapter->ifp;
2812 
2813 	while (drbr_empty(ifp, tx_ring->br) == FALSE &&
2814 	    (ifp->if_drv_flags & IFF_DRV_RUNNING) != 0) {
2815 		ENA_RING_MTX_LOCK(tx_ring);
2816 		ena_start_xmit(tx_ring);
2817 		ENA_RING_MTX_UNLOCK(tx_ring);
2818 	}
2819 }
2820 
2821 static int
2822 ena_mq_start(if_t ifp, struct mbuf *m)
2823 {
2824 	struct ena_adapter *adapter = ifp->if_softc;
2825 	struct ena_ring *tx_ring;
2826 	int ret, is_drbr_empty;
2827 	uint32_t i;
2828 
2829 	if ((adapter->ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
2830 		return (ENODEV);
2831 
2832 	/* Which queue to use */
2833 	/*
2834 	 * If everything is setup correctly, it should be the
2835 	 * same bucket that the current CPU we're on is.
2836 	 * It should improve performance.
2837 	 */
2838 	if (M_HASHTYPE_GET(m) != M_HASHTYPE_NONE) {
2839 #ifdef	RSS
2840 		if (rss_hash2bucket(m->m_pkthdr.flowid,
2841 		    M_HASHTYPE_GET(m), &i) == 0) {
2842 			i = i % adapter->num_queues;
2843 
2844 		} else
2845 #endif
2846 		{
2847 			i = m->m_pkthdr.flowid % adapter->num_queues;
2848 		}
2849 	} else {
2850 		i = curcpu % adapter->num_queues;
2851 	}
2852 	tx_ring = &adapter->tx_ring[i];
2853 
2854 	/* Check if drbr is empty before putting packet */
2855 	is_drbr_empty = drbr_empty(ifp, tx_ring->br);
2856 	ret = drbr_enqueue(ifp, tx_ring->br, m);
2857 	if (ret) {
2858 		taskqueue_enqueue(tx_ring->enqueue_tq, &tx_ring->enqueue_task);
2859 		return (ret);
2860 	}
2861 
2862 	if (is_drbr_empty && ENA_RING_MTX_TRYLOCK(tx_ring)) {
2863 		ena_start_xmit(tx_ring);
2864 		ENA_RING_MTX_UNLOCK(tx_ring);
2865 	} else {
2866 		taskqueue_enqueue(tx_ring->enqueue_tq, &tx_ring->enqueue_task);
2867 	}
2868 
2869 	return (0);
2870 }
2871 
2872 static void
2873 ena_qflush(if_t ifp)
2874 {
2875 	struct ena_adapter *adapter = ifp->if_softc;
2876 	struct ena_ring *tx_ring = adapter->tx_ring;
2877 	int i;
2878 
2879 	for(i = 0; i < adapter->num_queues; ++i, ++tx_ring)
2880 		if (drbr_empty(ifp, tx_ring->br) == FALSE) {
2881 			ENA_RING_MTX_LOCK(tx_ring);
2882 			drbr_flush(ifp, tx_ring->br);
2883 			ENA_RING_MTX_UNLOCK(tx_ring);
2884 		}
2885 
2886 	if_qflush(ifp);
2887 
2888 	return;
2889 }
2890 
2891 static int ena_calc_io_queue_num(struct ena_adapter *adapter,
2892     struct ena_com_dev_get_features_ctx *get_feat_ctx)
2893 {
2894 	int io_sq_num, io_cq_num, io_queue_num;
2895 
2896 	io_sq_num = get_feat_ctx->max_queues.max_sq_num;
2897 	io_cq_num = get_feat_ctx->max_queues.max_sq_num;
2898 
2899 	io_queue_num = min_t(int, mp_ncpus, ENA_MAX_NUM_IO_QUEUES);
2900 	io_queue_num = min_t(int, io_queue_num, io_sq_num);
2901 	io_queue_num = min_t(int, io_queue_num, io_cq_num);
2902 	/* 1 IRQ for for mgmnt and 1 IRQ for each TX/RX pair */
2903 	io_queue_num = min_t(int, io_queue_num,
2904 	    pci_msix_count(adapter->pdev) - 1);
2905 #ifdef	RSS
2906 	io_queue_num = min_t(int, io_queue_num, rss_getnumbuckets());
2907 #endif
2908 
2909 	return io_queue_num;
2910 }
2911 
2912 static int ena_calc_queue_size(struct ena_adapter *adapter,
2913     uint16_t *max_tx_sgl_size,  uint16_t *max_rx_sgl_size,
2914     struct ena_com_dev_get_features_ctx *feat)
2915 {
2916 	uint32_t queue_size = ENA_DEFAULT_RING_SIZE;
2917 	uint32_t v;
2918 	uint32_t q;
2919 
2920 	queue_size = min_t(uint32_t, queue_size,
2921 	    feat->max_queues.max_cq_depth);
2922 	queue_size = min_t(uint32_t, queue_size,
2923 	    feat->max_queues.max_sq_depth);
2924 
2925 	/* round down to the nearest power of 2 */
2926 	v = queue_size;
2927 	while (v != 0) {
2928 		if (powerof2(queue_size))
2929 			break;
2930 		v /= 2;
2931 		q = rounddown2(queue_size, v);
2932 		if (q != 0) {
2933 			queue_size = q;
2934 			break;
2935 		}
2936 	}
2937 
2938 	if (unlikely(!queue_size)) {
2939 		device_printf(adapter->pdev, "Invalid queue size\n");
2940 		return ENA_COM_FAULT;
2941 	}
2942 
2943 	*max_tx_sgl_size = min_t(uint16_t, ENA_PKT_MAX_BUFS,
2944 	    feat->max_queues.max_packet_tx_descs);
2945 	*max_rx_sgl_size = min_t(uint16_t, ENA_PKT_MAX_BUFS,
2946 	    feat->max_queues.max_packet_rx_descs);
2947 
2948 	return queue_size;
2949 }
2950 
2951 static int ena_rss_init_default(struct ena_adapter *adapter)
2952 {
2953 	struct ena_com_dev *ena_dev = adapter->ena_dev;
2954 	device_t dev = adapter->pdev;
2955 	int qid, rc, i;
2956 
2957 	rc = ena_com_rss_init(ena_dev, ENA_RX_RSS_TABLE_LOG_SIZE);
2958 	if (unlikely(rc)) {
2959 		device_printf(dev, "Cannot init RSS\n");
2960 		goto err_rss_init;
2961 	}
2962 
2963 	for (i = 0; i < ENA_RX_RSS_TABLE_SIZE; i++) {
2964 #ifdef	RSS
2965 		qid = rss_get_indirection_to_bucket(i);
2966 		qid = qid % adapter->num_queues;
2967 #else
2968 		qid = i % adapter->num_queues;
2969 #endif
2970 		rc = ena_com_indirect_table_fill_entry(ena_dev, i,
2971 						       ENA_IO_RXQ_IDX(qid));
2972 		if (unlikely(rc && (rc != EOPNOTSUPP))) {
2973 			device_printf(dev, "Cannot fill indirect table\n");
2974 			goto err_fill_indir;
2975 		}
2976 	}
2977 
2978 	rc = ena_com_fill_hash_function(ena_dev, ENA_ADMIN_CRC32, NULL,
2979 					ENA_HASH_KEY_SIZE, 0xFFFFFFFF);
2980 	if (unlikely(rc && (rc != EOPNOTSUPP))) {
2981 		device_printf(dev, "Cannot fill hash function\n");
2982 		goto err_fill_indir;
2983 	}
2984 
2985 	rc = ena_com_set_default_hash_ctrl(ena_dev);
2986 	if (unlikely(rc && (rc != EOPNOTSUPP))) {
2987 		device_printf(dev, "Cannot fill hash control\n");
2988 		goto err_fill_indir;
2989 	}
2990 
2991 	return (0);
2992 
2993 err_fill_indir:
2994 	ena_com_rss_destroy(ena_dev);
2995 err_rss_init:
2996 	return (rc);
2997 }
2998 
2999 static void
3000 ena_rss_init_default_deferred(void *arg)
3001 {
3002 	struct ena_adapter *adapter;
3003 	devclass_t dc;
3004 	int max;
3005 	int rc;
3006 
3007 	dc = devclass_find("ena");
3008 	if (dc == NULL) {
3009 		ena_trace(ENA_DBG, "No devclass ena\n");
3010 		return;
3011 	}
3012 
3013 	max = devclass_get_maxunit(dc);
3014 	while (max-- >= 0) {
3015 		adapter = devclass_get_softc(dc, max);
3016 		if (adapter != NULL) {
3017 			rc = ena_rss_init_default(adapter);
3018 			adapter->rss_support = true;
3019 			if (rc) {
3020 				device_printf(adapter->pdev,
3021 				    "WARNING: RSS was not properly initialized,"
3022 				    " it will affect bandwith\n");
3023 				adapter->rss_support = false;
3024 			}
3025 		}
3026 	}
3027 }
3028 SYSINIT(ena_rss_init, SI_SUB_KICK_SCHEDULER, SI_ORDER_SECOND, ena_rss_init_default_deferred, NULL);
3029 
3030 static void ena_config_host_info(struct ena_com_dev *ena_dev)
3031 {
3032 	struct ena_admin_host_info *host_info;
3033 	int rc;
3034 
3035 	/* Allocate only the host info */
3036 	rc = ena_com_allocate_host_info(ena_dev);
3037 	if (rc) {
3038 		ena_trace(ENA_ALERT, "Cannot allocate host info\n");
3039 		return;
3040 	}
3041 
3042 	host_info = ena_dev->host_attr.host_info;
3043 
3044 	host_info->os_type = ENA_ADMIN_OS_FREEBSD;
3045 	host_info->kernel_ver = osreldate;
3046 
3047 	sprintf(host_info->kernel_ver_str, "%d", osreldate);
3048 	host_info->os_dist = 0;
3049 	strncpy(host_info->os_dist_str, osrelease,
3050 	    sizeof(host_info->os_dist_str) - 1);
3051 
3052 	host_info->driver_version =
3053 		(DRV_MODULE_VER_MAJOR) |
3054 		(DRV_MODULE_VER_MINOR << ENA_ADMIN_HOST_INFO_MINOR_SHIFT) |
3055 		(DRV_MODULE_VER_SUBMINOR << ENA_ADMIN_HOST_INFO_SUB_MINOR_SHIFT);
3056 
3057 	rc = ena_com_set_host_attributes(ena_dev);
3058 	if (rc) {
3059 		if (rc == EOPNOTSUPP)
3060 			ena_trace(ENA_WARNING, "Cannot set host attributes\n");
3061 		else
3062 			ena_trace(ENA_ALERT, "Cannot set host attributes\n");
3063 
3064 		goto err;
3065 	}
3066 
3067 	return;
3068 
3069 err:
3070 	ena_com_delete_host_info(ena_dev);
3071 }
3072 
3073 static int
3074 ena_device_init(struct ena_adapter *adapter, device_t pdev,
3075 	struct ena_com_dev_get_features_ctx *get_feat_ctx, int *wd_active)
3076 {
3077 	struct ena_com_dev* ena_dev = adapter->ena_dev;
3078 	bool readless_supported;
3079 	uint32_t aenq_groups;
3080 	int dma_width;
3081 	int rc;
3082 
3083 	rc = ena_com_mmio_reg_read_request_init(ena_dev);
3084 	if (rc) {
3085 		device_printf(pdev, "failed to init mmio read less\n");
3086 		return rc;
3087 	}
3088 
3089 	/*
3090 	 * The PCIe configuration space revision id indicate if mmio reg
3091 	 * read is disabled
3092 	 */
3093 	readless_supported = !(pci_get_revid(pdev) & ENA_MMIO_DISABLE_REG_READ);
3094 	ena_com_set_mmio_read_mode(ena_dev, readless_supported);
3095 
3096 	rc = ena_com_dev_reset(ena_dev, ENA_REGS_RESET_NORMAL);
3097 	if (rc) {
3098 		device_printf(pdev, "Can not reset device\n");
3099 		goto err_mmio_read_less;
3100 	}
3101 
3102 	rc = ena_com_validate_version(ena_dev);
3103 	if (rc) {
3104 		device_printf(pdev, "device version is too low\n");
3105 		goto err_mmio_read_less;
3106 	}
3107 
3108 	dma_width = ena_com_get_dma_width(ena_dev);
3109 	if (dma_width < 0) {
3110 		device_printf(pdev, "Invalid dma width value %d", dma_width);
3111 		rc = dma_width;
3112 		goto err_mmio_read_less;
3113 	}
3114 	adapter->dma_width = dma_width;
3115 
3116 	/* ENA admin level init */
3117 	rc = ena_com_admin_init(ena_dev, &aenq_handlers, true);
3118 	if (rc) {
3119 		device_printf(pdev,
3120 		    "Can not initialize ena admin queue with device\n");
3121 		goto err_mmio_read_less;
3122 	}
3123 
3124 	/*
3125 	 * To enable the msix interrupts the driver needs to know the number
3126 	 * of queues. So the driver uses polling mode to retrieve this
3127 	 * information
3128 	 */
3129 	ena_com_set_admin_polling_mode(ena_dev, true);
3130 
3131 	ena_config_host_info(ena_dev);
3132 
3133 	/* Get Device Attributes */
3134 	rc = ena_com_get_dev_attr_feat(ena_dev, get_feat_ctx);
3135 	if (rc) {
3136 		device_printf(pdev,
3137 		    "Cannot get attribute for ena device rc: %d\n", rc);
3138 		goto err_admin_init;
3139 	}
3140 
3141 	aenq_groups = BIT(ENA_ADMIN_LINK_CHANGE) |
3142 	    BIT(ENA_ADMIN_FATAL_ERROR) |
3143 	    BIT(ENA_ADMIN_WARNING) |
3144 	    BIT(ENA_ADMIN_NOTIFICATION) |
3145 	    BIT(ENA_ADMIN_KEEP_ALIVE);
3146 
3147 	aenq_groups &= get_feat_ctx->aenq.supported_groups;
3148 	rc = ena_com_set_aenq_config(ena_dev, aenq_groups);
3149 	if (rc) {
3150 		device_printf(pdev, "Cannot configure aenq groups rc: %d\n", rc);
3151 		goto err_admin_init;
3152 	}
3153 
3154 	*wd_active = !!(aenq_groups & BIT(ENA_ADMIN_KEEP_ALIVE));
3155 
3156 	return 0;
3157 
3158 err_admin_init:
3159 	ena_com_delete_host_info(ena_dev);
3160 	ena_com_admin_destroy(ena_dev);
3161 err_mmio_read_less:
3162 	ena_com_mmio_reg_read_request_destroy(ena_dev);
3163 
3164 	return rc;
3165 }
3166 
3167 static int ena_enable_msix_and_set_admin_interrupts(struct ena_adapter *adapter,
3168     int io_vectors)
3169 {
3170 	struct ena_com_dev *ena_dev = adapter->ena_dev;
3171 	int rc;
3172 
3173 	rc = ena_enable_msix(adapter);
3174 	if (rc) {
3175 		device_printf(adapter->pdev, "Error with MSI-X enablement\n");
3176 		return rc;
3177 	}
3178 
3179 	ena_setup_mgmnt_intr(adapter);
3180 
3181 	rc = ena_request_mgmnt_irq(adapter);
3182 	if (rc) {
3183 		device_printf(adapter->pdev, "Cannot setup mgmnt queue intr\n");
3184 		goto err_disable_msix;
3185 	}
3186 
3187 	ena_com_set_admin_polling_mode(ena_dev, false);
3188 
3189 	ena_com_admin_aenq_enable(ena_dev);
3190 
3191 	return 0;
3192 
3193 err_disable_msix:
3194 	ena_disable_msix(adapter);
3195 
3196 	return rc;
3197 }
3198 
3199 /* Function called on ENA_ADMIN_KEEP_ALIVE event */
3200 static void ena_keep_alive_wd(void *adapter_data,
3201     struct ena_admin_aenq_entry *aenq_e)
3202 {
3203 	struct ena_adapter *adapter = (struct ena_adapter *)adapter_data;
3204 	struct ena_admin_aenq_keep_alive_desc *desc;
3205 	sbintime_t stime;
3206 	uint64_t rx_drops;
3207 
3208 	desc = (struct ena_admin_aenq_keep_alive_desc *)aenq_e;
3209 
3210 	rx_drops = ((uint64_t)desc->rx_drops_high << 32) | desc->rx_drops_low;
3211 	counter_u64_zero(adapter->hw_stats.rx_drops);
3212 	counter_u64_add(adapter->hw_stats.rx_drops, rx_drops);
3213 
3214 	stime = getsbinuptime();
3215 	atomic_store_rel_64(&adapter->keep_alive_timestamp, stime);
3216 }
3217 
3218 /* Check for keep alive expiration */
3219 static void check_for_missing_keep_alive(struct ena_adapter *adapter)
3220 {
3221 	sbintime_t timestamp, time;
3222 
3223 	if (adapter->wd_active == 0)
3224 		return;
3225 
3226 	if (adapter->keep_alive_timeout == 0)
3227 		return;
3228 
3229 	timestamp = atomic_load_acq_64(&adapter->keep_alive_timestamp);
3230 	time = getsbinuptime() - timestamp;
3231 	if (unlikely(time > adapter->keep_alive_timeout)) {
3232 		device_printf(adapter->pdev,
3233 		    "Keep alive watchdog timeout.\n");
3234 		counter_u64_add(adapter->dev_stats.wd_expired, 1);
3235 		adapter->reset_reason = ENA_REGS_RESET_KEEP_ALIVE_TO;
3236 		adapter->trigger_reset = true;
3237 	}
3238 }
3239 
3240 /* Check if admin queue is enabled */
3241 static void check_for_admin_com_state(struct ena_adapter *adapter)
3242 {
3243 	if (unlikely(!ena_com_get_admin_running_state(adapter->ena_dev))) {
3244 		device_printf(adapter->pdev,
3245 		    "ENA admin queue is not in running state!\n");
3246 		counter_u64_add(adapter->dev_stats.admin_q_pause, 1);
3247 		adapter->reset_reason = ENA_REGS_RESET_ADMIN_TO;
3248 		adapter->trigger_reset = true;
3249 	}
3250 }
3251 
3252 /*
3253  * Check for TX which were not completed on time.
3254  * Timeout is defined by "missing_tx_timeout".
3255  * Reset will be performed if number of incompleted
3256  * transactions exceeds "missing_tx_threshold".
3257  */
3258 static void check_for_missing_tx_completions(struct ena_adapter *adapter)
3259 {
3260 	struct ena_ring *tx_ring;
3261 	struct ena_tx_buffer *tx_info;
3262 	struct bintime curtime, time;
3263 	int i, j, budget, missed_tx;
3264 
3265 	/* Make sure the driver doesn't turn the device in other process */
3266 	rmb();
3267 
3268 	if (!adapter->up)
3269 		return;
3270 
3271 	if (adapter->trigger_reset)
3272 		return;
3273 
3274 	if (adapter->missing_tx_timeout == 0)
3275 		return;
3276 
3277 	budget = adapter->missing_tx_max_queues;
3278 	getbinuptime(&curtime);
3279 
3280 	for (i = adapter->next_monitored_tx_qid; i < adapter->num_queues; i++) {
3281 		tx_ring = &adapter->tx_ring[i];
3282 
3283 		missed_tx = 0;
3284 
3285 		for (j = 0; j < tx_ring->ring_size; j++) {
3286 			tx_info = &tx_ring->tx_buffer_info[j];
3287 
3288 			if (!bintime_isset(&tx_info->timestamp))
3289 				continue;
3290 
3291 			time = curtime;
3292 			bintime_sub(&time, &tx_info->timestamp);
3293 
3294 			/* Check again if packet is still waiting */
3295 			if (bintime_isset(&tx_info->timestamp) && unlikely(
3296 			    bttosbt(time) > adapter->missing_tx_timeout)) {
3297 				if (tx_info->print_once)
3298 					device_printf(adapter->pdev,
3299 					    "Found a Tx that wasn't completed "
3300 					    "on time, qid %d, index %d.\n",
3301 					    tx_ring->qid, j);
3302 
3303 				tx_info->print_once = false;
3304 				missed_tx++;
3305 
3306 				if (unlikely(missed_tx >
3307 				    adapter->missing_tx_threshold)) {
3308 					device_printf(adapter->pdev,
3309 					    "The number of lost tx completion "
3310 					    "is above the threshold (%d > %d). "
3311 					    "Reset the device\n", missed_tx,
3312 					    adapter->missing_tx_threshold);
3313 					adapter->reset_reason =
3314 					    ENA_REGS_RESET_MISS_TX_CMPL;
3315 					adapter->trigger_reset = true;
3316 					return;
3317 				}
3318 			}
3319 		}
3320 
3321 		budget--;
3322 		if (!budget) {
3323 			i++;
3324 			break;
3325 		}
3326 	}
3327 
3328 	adapter->next_monitored_tx_qid = i % adapter->num_queues;
3329 }
3330 
3331 
3332 static void
3333 ena_timer_service(void *data)
3334 {
3335 	struct ena_adapter *adapter = (struct ena_adapter *)data;
3336 	struct ena_admin_host_info *host_info =
3337 	    adapter->ena_dev->host_attr.host_info;
3338 
3339 	check_for_missing_keep_alive(adapter);
3340 
3341 	check_for_admin_com_state(adapter);
3342 
3343 	check_for_missing_tx_completions(adapter);
3344 
3345 	if (host_info)
3346 		ena_update_host_info(host_info, adapter->ifp);
3347 
3348 	if (unlikely(adapter->trigger_reset)) {
3349 		device_printf(adapter->pdev, "Trigger reset is on\n");
3350 		taskqueue_enqueue(adapter->reset_tq, &adapter->reset_task);
3351 		return;
3352 	}
3353 
3354 	/*
3355 	 * Schedule another timeout one second from now.
3356 	 */
3357 	callout_schedule_sbt(&adapter->timer_service, SBT_1S, SBT_1S, 0);
3358 }
3359 
3360 static void
3361 ena_reset_task(void *arg, int pending)
3362 {
3363 	struct ena_com_dev_get_features_ctx get_feat_ctx;
3364 	struct ena_adapter *adapter = (struct ena_adapter *)arg;
3365 	struct ena_com_dev *ena_dev = adapter->ena_dev;
3366 	bool dev_up;
3367 	int rc;
3368 
3369 	if (unlikely(!adapter->trigger_reset)) {
3370 		device_printf(adapter->pdev,
3371 		    "device reset scheduled but trigger_reset is off\n");
3372 		return;
3373 	}
3374 
3375 	sx_xlock(&adapter->ioctl_sx);
3376 
3377 	callout_drain(&adapter->timer_service);
3378 
3379 	dev_up = adapter->up;
3380 
3381 	ena_com_set_admin_running_state(ena_dev, false);
3382 	ena_down(adapter);
3383 	ena_free_mgmnt_irq(adapter);
3384 	ena_disable_msix(adapter);
3385 	ena_com_abort_admin_commands(ena_dev);
3386 	ena_com_wait_for_abort_completion(ena_dev);
3387 	ena_com_admin_destroy(ena_dev);
3388 	ena_com_mmio_reg_read_request_destroy(ena_dev);
3389 
3390 	adapter->reset_reason = ENA_REGS_RESET_NORMAL;
3391 	adapter->trigger_reset = false;
3392 
3393 	/* Finished destroy part. Restart the device */
3394 	rc = ena_device_init(adapter, adapter->pdev, &get_feat_ctx,
3395 	    &adapter->wd_active);
3396 	if (rc) {
3397 		device_printf(adapter->pdev,
3398 		    "ENA device init failed! (err: %d)\n", rc);
3399 		goto err_dev_free;
3400 	}
3401 
3402 	rc = ena_enable_msix_and_set_admin_interrupts(adapter,
3403 	    adapter->num_queues);
3404 	if (rc) {
3405 		device_printf(adapter->pdev, "Enable MSI-X failed\n");
3406 		goto err_com_free;
3407 	}
3408 
3409 	/* If the interface was up before the reset bring it up */
3410 	if (dev_up) {
3411 		rc = ena_up(adapter);
3412 		if (rc) {
3413 			device_printf(adapter->pdev,
3414 			    "Failed to create I/O queues\n");
3415 			goto err_msix_free;
3416 		}
3417 	}
3418 
3419 	callout_reset_sbt(&adapter->timer_service, SBT_1S, SBT_1S,
3420 	    ena_timer_service, (void *)adapter, 0);
3421 
3422 	sx_unlock(&adapter->ioctl_sx);
3423 
3424 	return;
3425 
3426 err_msix_free:
3427 	ena_free_mgmnt_irq(adapter);
3428 	ena_disable_msix(adapter);
3429 err_com_free:
3430 	ena_com_admin_destroy(ena_dev);
3431 err_dev_free:
3432 	device_printf(adapter->pdev, "ENA reset failed!\n");
3433 	adapter->running = false;
3434 	sx_unlock(&adapter->ioctl_sx);
3435 }
3436 
3437 /**
3438  * ena_attach - Device Initialization Routine
3439  * @pdev: device information struct
3440  *
3441  * Returns 0 on success, otherwise on failure.
3442  *
3443  * ena_attach initializes an adapter identified by a device structure.
3444  * The OS initialization, configuring of the adapter private structure,
3445  * and a hardware reset occur.
3446  **/
3447 static int
3448 ena_attach(device_t pdev)
3449 {
3450 	struct ena_com_dev_get_features_ctx get_feat_ctx;
3451 	static int version_printed;
3452 	struct ena_adapter *adapter;
3453 	struct ena_com_dev *ena_dev = NULL;
3454 	uint16_t tx_sgl_size = 0;
3455 	uint16_t rx_sgl_size = 0;
3456 	int io_queue_num;
3457 	int queue_size;
3458 	int rc;
3459 	struct sysctl_ctx_list *ctx;
3460 	struct sysctl_oid_list *children;
3461 
3462 	adapter = device_get_softc(pdev);
3463 	adapter->pdev = pdev;
3464 	ctx = device_get_sysctl_ctx(pdev);
3465 	children = SYSCTL_CHILDREN(device_get_sysctl_tree(pdev));
3466 
3467 	mtx_init(&adapter->global_mtx, "ENA global mtx", NULL, MTX_DEF);
3468 	sx_init(&adapter->ioctl_sx, "ENA ioctl sx");
3469 
3470 	/* Sysctl calls for Watchdog service */
3471 	SYSCTL_ADD_INT(ctx, children, OID_AUTO, "wd_active",
3472 	    CTLFLAG_RWTUN, &adapter->wd_active, 0,
3473 	    "Watchdog is active");
3474 
3475 	SYSCTL_ADD_QUAD(ctx, children, OID_AUTO, "keep_alive_timeout",
3476 	    CTLFLAG_RWTUN, &adapter->keep_alive_timeout,
3477 	    "Timeout for Keep Alive messages");
3478 
3479 	SYSCTL_ADD_QUAD(ctx, children, OID_AUTO, "missing_tx_timeout",
3480 	    CTLFLAG_RWTUN, &adapter->missing_tx_timeout,
3481 	    "Timeout for TX completion");
3482 
3483 	SYSCTL_ADD_U32(ctx, children, OID_AUTO, "missing_tx_max_queues",
3484 	    CTLFLAG_RWTUN, &adapter->missing_tx_max_queues, 0,
3485 	    "Number of TX queues to check per run");
3486 
3487 	SYSCTL_ADD_U32(ctx, children, OID_AUTO, "missing_tx_threshold",
3488 	    CTLFLAG_RWTUN, &adapter->missing_tx_threshold, 0,
3489 	    "Max number of timeouted packets");
3490 
3491 	/* Set up the timer service */
3492 	callout_init_mtx(&adapter->timer_service, &adapter->global_mtx, 0);
3493 	adapter->keep_alive_timeout = DEFAULT_KEEP_ALIVE_TO;
3494 	adapter->missing_tx_timeout = DEFAULT_TX_CMP_TO;
3495 	adapter->missing_tx_max_queues = DEFAULT_TX_MONITORED_QUEUES;
3496 	adapter->missing_tx_threshold = DEFAULT_TX_CMP_THRESHOLD;
3497 
3498 	if (version_printed++ == 0)
3499 		device_printf(pdev, "%s\n", ena_version);
3500 
3501 	rc = ena_allocate_pci_resources(adapter);
3502 	if (rc) {
3503 		device_printf(pdev, "PCI resource allocation failed!\n");
3504 		ena_free_pci_resources(adapter);
3505 		goto err_pci_res;
3506 	}
3507 
3508 	/* Allocate memory for ena_dev structure */
3509 	ena_dev = ENA_MEM_ALLOC(pdev, sizeof(struct ena_com_dev));
3510 	if (!ena_dev) {
3511 		device_printf(pdev, "allocating ena_dev failed\n");
3512 		rc = ENOMEM;
3513 		goto err_select_region;
3514 	}
3515 
3516 	adapter->ena_dev = ena_dev;
3517 	ena_dev->dmadev = pdev;
3518 	ena_dev->bus = malloc(sizeof(struct ena_bus), M_DEVBUF,
3519 	    M_WAITOK | M_ZERO);
3520 
3521 	/* Store register resources */
3522 	((struct ena_bus*)(ena_dev->bus))->reg_bar_t =
3523 	    rman_get_bustag(adapter->registers);
3524 	((struct ena_bus*)(ena_dev->bus))->reg_bar_h =
3525 	    rman_get_bushandle(adapter->registers);
3526 
3527 	if (((struct ena_bus*)(ena_dev->bus))->reg_bar_h == 0) {
3528 		device_printf(pdev, "failed to pmap registers bar\n");
3529 		rc = ENXIO;
3530 		goto err_dev_free;
3531 	}
3532 
3533 	ena_dev->tx_mem_queue_type = ENA_ADMIN_PLACEMENT_POLICY_HOST;
3534 
3535 	/* Device initialization */
3536 	rc = ena_device_init(adapter, pdev, &get_feat_ctx, &adapter->wd_active);
3537 	if (rc) {
3538 		device_printf(pdev, "ENA device init failed! (err: %d)\n", rc);
3539 		rc = ENXIO;
3540 		goto err_bus_free;
3541 	}
3542 
3543 	adapter->keep_alive_timestamp = getsbinuptime();
3544 
3545 	adapter->tx_offload_cap = get_feat_ctx.offload.tx;
3546 
3547 	/* Set for sure that interface is not up */
3548 	adapter->up = false;
3549 
3550 	memcpy(adapter->mac_addr, get_feat_ctx.dev_attr.mac_addr,
3551 	    ETHER_ADDR_LEN);
3552 
3553 	adapter->small_copy_len =
3554 	    ENA_DEFAULT_SMALL_PACKET_LEN;
3555 
3556 	/* calculate IO queue number to create */
3557 	io_queue_num = ena_calc_io_queue_num(adapter, &get_feat_ctx);
3558 
3559 	ENA_ASSERT(io_queue_num > 0, "Invalid queue number: %d\n",
3560 	    io_queue_num);
3561 	adapter->num_queues = io_queue_num;
3562 
3563 	/* calculatre ring sizes */
3564 	queue_size = ena_calc_queue_size(adapter,&tx_sgl_size,
3565 	    &rx_sgl_size, &get_feat_ctx);
3566 	if ((queue_size <= 0) || (io_queue_num <= 0)) {
3567 		rc = ENA_COM_FAULT;
3568 		goto err_com_free;
3569 	}
3570 
3571 	adapter->reset_reason = ENA_REGS_RESET_NORMAL;
3572 
3573 	adapter->tx_ring_size = queue_size;
3574 	adapter->rx_ring_size = queue_size;
3575 
3576 	adapter->max_tx_sgl_size = tx_sgl_size;
3577 	adapter->max_rx_sgl_size = rx_sgl_size;
3578 
3579 	/* set up dma tags for rx and tx buffers */
3580 	rc = ena_setup_tx_dma_tag(adapter);
3581 	if (rc)
3582 		goto dma_tx_err;
3583 
3584 	rc = ena_setup_rx_dma_tag(adapter);
3585 	if (rc)
3586 		goto dma_rx_err;
3587 
3588 	/* initialize rings basic information */
3589 	device_printf(pdev, "initalize %d io queues\n", io_queue_num);
3590 	rc = ena_init_io_rings(adapter);
3591 	if (rc) {
3592 		device_printf(pdev,"Error with initialization of IO rings\n");
3593 		goto err_io_init;
3594 	}
3595 
3596 	/* setup network interface */
3597 	rc = ena_setup_ifnet(pdev, adapter, &get_feat_ctx);
3598 	if (rc) {
3599 		device_printf(pdev,"Error with network interface setup\n");
3600 		goto err_com_free;
3601 	}
3602 
3603 	rc = ena_enable_msix_and_set_admin_interrupts(adapter, io_queue_num);
3604 	if (rc) {
3605 		device_printf(pdev,
3606 		    "Failed to enable and set the admin interrupts\n");
3607 		goto err_ifp_free;
3608 	}
3609 
3610 	/* Initialize reset task queue */
3611 	TASK_INIT(&adapter->reset_task, 0, ena_reset_task, adapter);
3612 	adapter->reset_tq = taskqueue_create("ena_reset_enqueue",
3613 	    M_WAITOK | M_ZERO, taskqueue_thread_enqueue, &adapter->reset_tq);
3614 	if (adapter->reset_tq == NULL) {
3615 		device_printf(adapter->pdev,
3616 		    "Unable to create reset task queue\n");
3617 		goto err_reset_tq;
3618 	}
3619 	taskqueue_start_threads(&adapter->reset_tq, 1, PI_NET,
3620 	    "%s rstq", device_get_nameunit(adapter->pdev));
3621 
3622 	/* Initialize statistics */
3623 	ena_alloc_counters((counter_u64_t *)&adapter->dev_stats,
3624 	    sizeof(struct ena_stats_dev));
3625 	ena_alloc_counters((counter_u64_t *)&adapter->hw_stats,
3626 	    sizeof(struct ena_hw_stats));
3627 	ena_sysctl_add_nodes(adapter);
3628 
3629 	/* Tell the stack that the interface is not active */
3630 	if_setdrvflagbits(adapter->ifp, IFF_DRV_OACTIVE, IFF_DRV_RUNNING);
3631 
3632 	adapter->running = true;
3633 	return (0);
3634 
3635 err_reset_tq:
3636 	ena_com_dev_reset(ena_dev, ENA_REGS_RESET_INIT_ERR);
3637 	ena_free_mgmnt_irq(adapter);
3638 	ena_disable_msix(adapter);
3639 err_ifp_free:
3640 	if_detach(adapter->ifp);
3641 	if_free(adapter->ifp);
3642 err_com_free:
3643 	ena_free_all_io_rings_resources(adapter);
3644 err_io_init:
3645 	ena_free_rx_dma_tag(adapter);
3646 dma_rx_err:
3647 	ena_free_tx_dma_tag(adapter);
3648 dma_tx_err:
3649 	ena_com_admin_destroy(ena_dev);
3650 	ena_com_delete_host_info(ena_dev);
3651 err_bus_free:
3652 	free(ena_dev->bus, M_DEVBUF);
3653 err_dev_free:
3654 	free(ena_dev, M_DEVBUF);
3655 err_select_region:
3656 	ena_free_pci_resources(adapter);
3657 err_pci_res:
3658 	return (rc);
3659 }
3660 
3661 /**
3662  * ena_detach - Device Removal Routine
3663  * @pdev: device information struct
3664  *
3665  * ena_detach is called by the device subsystem to alert the driver
3666  * that it should release a PCI device.
3667  **/
3668 static int
3669 ena_detach(device_t pdev)
3670 {
3671 	struct ena_adapter *adapter = device_get_softc(pdev);
3672 	struct ena_com_dev *ena_dev = adapter->ena_dev;
3673 	int rc;
3674 
3675 	/* Make sure VLANS are not using driver */
3676 	if (adapter->ifp->if_vlantrunk != NULL) {
3677 		device_printf(adapter->pdev ,"VLAN is in use, detach first\n");
3678 		return (EBUSY);
3679 	}
3680 
3681 	/* Free reset task and callout */
3682 	callout_drain(&adapter->timer_service);
3683 	while (taskqueue_cancel(adapter->reset_tq, &adapter->reset_task, NULL))
3684 		taskqueue_drain(adapter->reset_tq, &adapter->reset_task);
3685 	taskqueue_free(adapter->reset_tq);
3686 
3687 	sx_xlock(&adapter->ioctl_sx);
3688 	ena_down(adapter);
3689 	sx_unlock(&adapter->ioctl_sx);
3690 
3691 	if (adapter->ifp != NULL) {
3692 		ether_ifdetach(adapter->ifp);
3693 		if_free(adapter->ifp);
3694 	}
3695 
3696 	ena_free_all_io_rings_resources(adapter);
3697 
3698 	ena_free_counters((counter_u64_t *)&adapter->hw_stats,
3699 	    sizeof(struct ena_hw_stats));
3700 	ena_free_counters((counter_u64_t *)&adapter->dev_stats,
3701 	    sizeof(struct ena_stats_dev));
3702 
3703 	if (adapter->rss_support)
3704 		ena_com_rss_destroy(ena_dev);
3705 
3706 	rc = ena_free_rx_dma_tag(adapter);
3707 	if (rc)
3708 		device_printf(adapter->pdev,
3709 		    "Unmapped RX DMA tag associations\n");
3710 
3711 	rc = ena_free_tx_dma_tag(adapter);
3712 	if (rc)
3713 		device_printf(adapter->pdev,
3714 		    "Unmapped TX DMA tag associations\n");
3715 
3716 	/* Reset the device only if the device is running. */
3717 	if (adapter->running)
3718 		ena_com_dev_reset(ena_dev, adapter->reset_reason);
3719 
3720 	ena_com_delete_host_info(ena_dev);
3721 
3722 	ena_com_admin_destroy(ena_dev);
3723 
3724 	ena_free_irqs(adapter);
3725 
3726 	ena_com_mmio_reg_read_request_destroy(ena_dev);
3727 
3728 	ena_free_pci_resources(adapter);
3729 
3730 	mtx_destroy(&adapter->global_mtx);
3731 	sx_destroy(&adapter->ioctl_sx);
3732 
3733 	if (ena_dev->bus != NULL)
3734 		free(ena_dev->bus, M_DEVBUF);
3735 
3736 	if (ena_dev != NULL)
3737 		free(ena_dev, M_DEVBUF);
3738 
3739 	return (bus_generic_detach(pdev));
3740 }
3741 
3742 /******************************************************************************
3743  ******************************** AENQ Handlers *******************************
3744  *****************************************************************************/
3745 /**
3746  * ena_update_on_link_change:
3747  * Notify the network interface about the change in link status
3748  **/
3749 static void
3750 ena_update_on_link_change(void *adapter_data,
3751     struct ena_admin_aenq_entry *aenq_e)
3752 {
3753 	struct ena_adapter *adapter = (struct ena_adapter *)adapter_data;
3754 	struct ena_admin_aenq_link_change_desc *aenq_desc;
3755 	int status;
3756 	if_t ifp;
3757 
3758 	aenq_desc = (struct ena_admin_aenq_link_change_desc *)aenq_e;
3759 	ifp = adapter->ifp;
3760 	status = aenq_desc->flags &
3761 	    ENA_ADMIN_AENQ_LINK_CHANGE_DESC_LINK_STATUS_MASK;
3762 
3763 	if (status != 0) {
3764 		device_printf(adapter->pdev, "link is UP\n");
3765 		if_link_state_change(ifp, LINK_STATE_UP);
3766 	} else if (status == 0) {
3767 		device_printf(adapter->pdev, "link is DOWN\n");
3768 		if_link_state_change(ifp, LINK_STATE_DOWN);
3769 	} else {
3770 		device_printf(adapter->pdev, "invalid value recvd\n");
3771 		BUG();
3772 	}
3773 
3774 	adapter->link_status = status;
3775 
3776 	return;
3777 }
3778 
3779 /**
3780  * This handler will called for unknown event group or unimplemented handlers
3781  **/
3782 static void
3783 unimplemented_aenq_handler(void *data,
3784     struct ena_admin_aenq_entry *aenq_e)
3785 {
3786 	return;
3787 }
3788 
3789 static struct ena_aenq_handlers aenq_handlers = {
3790     .handlers = {
3791 	    [ENA_ADMIN_LINK_CHANGE] = ena_update_on_link_change,
3792 	    [ENA_ADMIN_KEEP_ALIVE] = ena_keep_alive_wd,
3793     },
3794     .unimplemented_handler = unimplemented_aenq_handler
3795 };
3796 
3797 /*********************************************************************
3798  *  FreeBSD Device Interface Entry Points
3799  *********************************************************************/
3800 
3801 static device_method_t ena_methods[] = {
3802     /* Device interface */
3803     DEVMETHOD(device_probe, ena_probe),
3804     DEVMETHOD(device_attach, ena_attach),
3805     DEVMETHOD(device_detach, ena_detach),
3806     DEVMETHOD_END
3807 };
3808 
3809 static driver_t ena_driver = {
3810     "ena", ena_methods, sizeof(struct ena_adapter),
3811 };
3812 
3813 devclass_t ena_devclass;
3814 DRIVER_MODULE(ena, pci, ena_driver, ena_devclass, 0, 0);
3815 MODULE_DEPEND(ena, pci, 1, 1, 1);
3816 MODULE_DEPEND(ena, ether, 1, 1, 1);
3817 
3818 /*********************************************************************/
3819