1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
3 *
4 * Copyright (c) 2015-2024 Amazon.com, Inc. or its affiliates.
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 *
11 * 1. Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 *
14 * 2. Redistributions in binary form must reproduce the above copyright
15 * notice, this list of conditions and the following disclaimer in the
16 * documentation and/or other materials provided with the distribution.
17 *
18 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29 */
30 #include <sys/cdefs.h>
31 #include "opt_rss.h"
32
33 #include <sys/param.h>
34 #include <sys/systm.h>
35 #include <sys/bus.h>
36 #include <sys/endian.h>
37 #include <sys/eventhandler.h>
38 #include <sys/kernel.h>
39 #include <sys/kthread.h>
40 #include <sys/malloc.h>
41 #include <sys/mbuf.h>
42 #include <sys/module.h>
43 #include <sys/rman.h>
44 #include <sys/smp.h>
45 #include <sys/socket.h>
46 #include <sys/sockio.h>
47 #include <sys/sysctl.h>
48 #include <sys/taskqueue.h>
49 #include <sys/time.h>
50
51 #include <vm/vm.h>
52 #include <vm/pmap.h>
53
54 #include <machine/atomic.h>
55 #include <machine/bus.h>
56 #include <machine/in_cksum.h>
57 #include <machine/resource.h>
58
59 #include <dev/pci/pcireg.h>
60 #include <dev/pci/pcivar.h>
61
62 #include <net/bpf.h>
63 #include <net/ethernet.h>
64 #include <net/if.h>
65 #include <net/if_arp.h>
66 #include <net/if_dl.h>
67 #include <net/if_media.h>
68 #include <net/if_types.h>
69 #include <net/if_var.h>
70 #include <net/if_vlan_var.h>
71 #include <netinet/in.h>
72 #include <netinet/in_systm.h>
73 #include <netinet/if_ether.h>
74 #include <netinet/ip.h>
75 #include <netinet/ip6.h>
76 #include <netinet/tcp.h>
77 #include <netinet/udp.h>
78
79 #include "ena.h"
80 #include "ena_datapath.h"
81 #include "ena_rss.h"
82 #include "ena_sysctl.h"
83
84 #ifdef DEV_NETMAP
85 #include "ena_netmap.h"
86 #endif /* DEV_NETMAP */
87
88 /*********************************************************
89 * Function prototypes
90 *********************************************************/
91 static int ena_probe(device_t);
92 static void ena_intr_msix_mgmnt(void *);
93 static void ena_free_pci_resources(struct ena_adapter *);
94 static int ena_change_mtu(if_t, int);
95 static inline void ena_alloc_counters(counter_u64_t *, int);
96 static inline void ena_free_counters(counter_u64_t *, int);
97 static inline void ena_reset_counters(counter_u64_t *, int);
98 static void ena_init_io_rings_common(struct ena_adapter *, struct ena_ring *,
99 uint16_t);
100 static void ena_init_io_rings_basic(struct ena_adapter *);
101 static void ena_init_io_rings_advanced(struct ena_adapter *);
102 static void ena_init_io_rings(struct ena_adapter *);
103 static void ena_free_io_ring_resources(struct ena_adapter *, unsigned int);
104 static void ena_free_all_io_rings_resources(struct ena_adapter *);
105 static int ena_setup_tx_dma_tag(struct ena_adapter *);
106 static int ena_free_tx_dma_tag(struct ena_adapter *);
107 static int ena_setup_rx_dma_tag(struct ena_adapter *);
108 static int ena_free_rx_dma_tag(struct ena_adapter *);
109 static void ena_release_all_tx_dmamap(struct ena_ring *);
110 static int ena_setup_tx_resources(struct ena_adapter *, int);
111 static void ena_free_tx_resources(struct ena_adapter *, int);
112 static int ena_setup_all_tx_resources(struct ena_adapter *);
113 static void ena_free_all_tx_resources(struct ena_adapter *);
114 static int ena_setup_rx_resources(struct ena_adapter *, unsigned int);
115 static void ena_free_rx_resources(struct ena_adapter *, unsigned int);
116 static int ena_setup_all_rx_resources(struct ena_adapter *);
117 static void ena_free_all_rx_resources(struct ena_adapter *);
118 static inline int ena_alloc_rx_mbuf(struct ena_adapter *, struct ena_ring *,
119 struct ena_rx_buffer *);
120 static void ena_free_rx_mbuf(struct ena_adapter *, struct ena_ring *,
121 struct ena_rx_buffer *);
122 static void ena_free_rx_bufs(struct ena_adapter *, unsigned int);
123 static void ena_refill_all_rx_bufs(struct ena_adapter *);
124 static void ena_free_all_rx_bufs(struct ena_adapter *);
125 static void ena_free_tx_bufs(struct ena_adapter *, unsigned int);
126 static void ena_free_all_tx_bufs(struct ena_adapter *);
127 static void ena_destroy_all_tx_queues(struct ena_adapter *);
128 static void ena_destroy_all_rx_queues(struct ena_adapter *);
129 static void ena_destroy_all_io_queues(struct ena_adapter *);
130 static int ena_create_io_queues(struct ena_adapter *);
131 static int ena_handle_msix(void *);
132 static int ena_enable_msix(struct ena_adapter *);
133 static void ena_setup_mgmnt_intr(struct ena_adapter *);
134 static int ena_setup_io_intr(struct ena_adapter *);
135 static int ena_request_mgmnt_irq(struct ena_adapter *);
136 static int ena_request_io_irq(struct ena_adapter *);
137 static void ena_free_mgmnt_irq(struct ena_adapter *);
138 static void ena_free_io_irq(struct ena_adapter *);
139 static void ena_free_irqs(struct ena_adapter *);
140 static void ena_disable_msix(struct ena_adapter *);
141 static void ena_unmask_all_io_irqs(struct ena_adapter *);
142 static int ena_up_complete(struct ena_adapter *);
143 static uint64_t ena_get_counter(if_t, ift_counter);
144 static int ena_media_change(if_t);
145 static void ena_media_status(if_t, struct ifmediareq *);
146 static void ena_init(void *);
147 static int ena_ioctl(if_t, u_long, caddr_t);
148 static int ena_get_dev_offloads(struct ena_com_dev_get_features_ctx *);
149 static void ena_update_host_info(struct ena_admin_host_info *, if_t);
150 static void ena_update_hwassist(struct ena_adapter *);
151 static void ena_setup_ifnet(device_t, struct ena_adapter *,
152 struct ena_com_dev_get_features_ctx *);
153 static int ena_enable_wc(device_t, struct resource *);
154 static int ena_set_queues_placement_policy(device_t, struct ena_com_dev *,
155 struct ena_admin_feature_llq_desc *, struct ena_llq_configurations *);
156 static int ena_map_llq_mem_bar(device_t, struct ena_com_dev *);
157 static uint32_t ena_calc_max_io_queue_num(device_t, struct ena_com_dev *,
158 struct ena_com_dev_get_features_ctx *);
159 static int ena_calc_io_queue_size(struct ena_calc_queue_size_ctx *, struct ena_adapter *);
160 static void ena_config_host_info(struct ena_com_dev *, device_t);
161 static int ena_attach(device_t);
162 static int ena_detach(device_t);
163 static int ena_device_init(struct ena_adapter *, device_t,
164 struct ena_com_dev_get_features_ctx *, int *);
165 static int ena_enable_msix_and_set_admin_interrupts(struct ena_adapter *);
166 static void ena_update_on_link_change(void *, struct ena_admin_aenq_entry *);
167 static void unimplemented_aenq_handler(void *, struct ena_admin_aenq_entry *);
168 static int ena_copy_eni_metrics(struct ena_adapter *);
169 static int ena_copy_srd_metrics(struct ena_adapter *);
170 static int ena_copy_customer_metrics(struct ena_adapter *);
171 static void ena_timer_service(void *);
172 static enum ena_regs_reset_reason_types check_cdesc_in_tx_cq(struct ena_adapter *,
173 struct ena_ring *);
174 #ifdef DEV_NETMAP
175 static int ena_reinit_netmap(struct ena_adapter *adapter);
176 #endif
177
178 static char ena_version[] = ENA_DEVICE_NAME ENA_DRV_MODULE_NAME
179 " v" ENA_DRV_MODULE_VERSION;
180
181 static ena_vendor_info_t ena_vendor_info_array[] = {
182 { PCI_VENDOR_ID_AMAZON, PCI_DEV_ID_ENA_PF, 0 },
183 { PCI_VENDOR_ID_AMAZON, PCI_DEV_ID_ENA_PF_RSERV0, 0 },
184 { PCI_VENDOR_ID_AMAZON, PCI_DEV_ID_ENA_VF, 0 },
185 { PCI_VENDOR_ID_AMAZON, PCI_DEV_ID_ENA_VF_RSERV0, 0 },
186 /* Last entry */
187 { 0, 0, 0 }
188 };
189
190 struct sx ena_global_lock;
191
192 /*
193 * Contains pointers to event handlers, e.g. link state chage.
194 */
195 static struct ena_aenq_handlers aenq_handlers;
196
197 void
ena_dmamap_callback(void * arg,bus_dma_segment_t * segs,int nseg,int error)198 ena_dmamap_callback(void *arg, bus_dma_segment_t *segs, int nseg, int error)
199 {
200 if (error != 0)
201 return;
202 *(bus_addr_t *)arg = segs[0].ds_addr;
203 }
204
205 int
ena_dma_alloc(device_t dmadev,bus_size_t size,ena_mem_handle_t * dma,int mapflags,bus_size_t alignment,int domain)206 ena_dma_alloc(device_t dmadev, bus_size_t size, ena_mem_handle_t *dma,
207 int mapflags, bus_size_t alignment, int domain)
208 {
209 struct ena_adapter *adapter = device_get_softc(dmadev);
210 device_t pdev = adapter->pdev;
211 uint32_t maxsize;
212 uint64_t dma_space_addr;
213 int error;
214
215 maxsize = ((size - 1) / PAGE_SIZE + 1) * PAGE_SIZE;
216
217 dma_space_addr = ENA_DMA_BIT_MASK(adapter->dma_width);
218 if (unlikely(dma_space_addr == 0))
219 dma_space_addr = BUS_SPACE_MAXADDR;
220
221 error = bus_dma_tag_create(bus_get_dma_tag(dmadev), /* parent */
222 alignment, 0, /* alignment, bounds */
223 dma_space_addr, /* lowaddr of exclusion window */
224 BUS_SPACE_MAXADDR, /* highaddr of exclusion window */
225 NULL, NULL, /* filter, filterarg */
226 maxsize, /* maxsize */
227 1, /* nsegments */
228 maxsize, /* maxsegsize */
229 BUS_DMA_ALLOCNOW, /* flags */
230 NULL, /* lockfunc */
231 NULL, /* lockarg */
232 &dma->tag);
233 if (unlikely(error != 0)) {
234 ena_log(pdev, ERR, "bus_dma_tag_create failed: %d\n", error);
235 goto fail_tag;
236 }
237
238 error = bus_dma_tag_set_domain(dma->tag, domain);
239 if (unlikely(error != 0)) {
240 ena_log(pdev, ERR, "bus_dma_tag_set_domain failed: %d\n",
241 error);
242 goto fail_map_create;
243 }
244
245 error = bus_dmamem_alloc(dma->tag, (void **)&dma->vaddr,
246 BUS_DMA_COHERENT | BUS_DMA_ZERO, &dma->map);
247 if (unlikely(error != 0)) {
248 ena_log(pdev, ERR, "bus_dmamem_alloc(%ju) failed: %d\n",
249 (uintmax_t)size, error);
250 goto fail_map_create;
251 }
252
253 dma->paddr = 0;
254 error = bus_dmamap_load(dma->tag, dma->map, dma->vaddr, size,
255 ena_dmamap_callback, &dma->paddr, mapflags);
256 if (unlikely((error != 0) || (dma->paddr == 0))) {
257 ena_log(pdev, ERR, "bus_dmamap_load failed: %d\n", error);
258 goto fail_map_load;
259 }
260
261 bus_dmamap_sync(dma->tag, dma->map,
262 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
263
264 return (0);
265
266 fail_map_load:
267 bus_dmamem_free(dma->tag, dma->vaddr, dma->map);
268 fail_map_create:
269 bus_dma_tag_destroy(dma->tag);
270 fail_tag:
271 dma->tag = NULL;
272 dma->vaddr = NULL;
273 dma->paddr = 0;
274
275 return (error);
276 }
277
278 static void
ena_free_pci_resources(struct ena_adapter * adapter)279 ena_free_pci_resources(struct ena_adapter *adapter)
280 {
281 device_t pdev = adapter->pdev;
282
283 if (adapter->memory != NULL) {
284 bus_release_resource(pdev, SYS_RES_MEMORY,
285 PCIR_BAR(ENA_MEM_BAR), adapter->memory);
286 }
287
288 if (adapter->registers != NULL) {
289 bus_release_resource(pdev, SYS_RES_MEMORY,
290 PCIR_BAR(ENA_REG_BAR), adapter->registers);
291 }
292
293 if (adapter->msix != NULL) {
294 bus_release_resource(pdev, SYS_RES_MEMORY, adapter->msix_rid,
295 adapter->msix);
296 }
297 }
298
299 static int
ena_probe(device_t dev)300 ena_probe(device_t dev)
301 {
302 ena_vendor_info_t *ent;
303 uint16_t pci_vendor_id = 0;
304 uint16_t pci_device_id = 0;
305
306 pci_vendor_id = pci_get_vendor(dev);
307 pci_device_id = pci_get_device(dev);
308
309 ent = ena_vendor_info_array;
310 while (ent->vendor_id != 0) {
311 if ((pci_vendor_id == ent->vendor_id) &&
312 (pci_device_id == ent->device_id)) {
313 ena_log_raw(DBG, "vendor=%x device=%x\n", pci_vendor_id,
314 pci_device_id);
315
316 device_set_desc(dev, ENA_DEVICE_DESC);
317 return (BUS_PROBE_DEFAULT);
318 }
319
320 ent++;
321 }
322
323 return (ENXIO);
324 }
325
326 static int
ena_change_mtu(if_t ifp,int new_mtu)327 ena_change_mtu(if_t ifp, int new_mtu)
328 {
329 struct ena_adapter *adapter = if_getsoftc(ifp);
330 device_t pdev = adapter->pdev;
331 int rc;
332
333 if ((new_mtu > adapter->max_mtu) || (new_mtu < ENA_MIN_MTU)) {
334 ena_log(pdev, ERR, "Invalid MTU setting. new_mtu: %d max mtu: %d min mtu: %d\n",
335 new_mtu, adapter->max_mtu, ENA_MIN_MTU);
336 return (EINVAL);
337 }
338
339 rc = ena_com_set_dev_mtu(adapter->ena_dev, new_mtu);
340 if (likely(rc == 0)) {
341 ena_log(pdev, DBG, "set MTU to %d\n", new_mtu);
342 if_setmtu(ifp, new_mtu);
343 } else {
344 ena_log(pdev, ERR, "Failed to set MTU to %d\n", new_mtu);
345 }
346
347 return (rc);
348 }
349
350 static inline void
ena_alloc_counters(counter_u64_t * begin,int size)351 ena_alloc_counters(counter_u64_t *begin, int size)
352 {
353 counter_u64_t *end = (counter_u64_t *)((char *)begin + size);
354
355 for (; begin < end; ++begin)
356 *begin = counter_u64_alloc(M_WAITOK);
357 }
358
359 static inline void
ena_free_counters(counter_u64_t * begin,int size)360 ena_free_counters(counter_u64_t *begin, int size)
361 {
362 counter_u64_t *end = (counter_u64_t *)((char *)begin + size);
363
364 for (; begin < end; ++begin)
365 counter_u64_free(*begin);
366 }
367
368 static inline void
ena_reset_counters(counter_u64_t * begin,int size)369 ena_reset_counters(counter_u64_t *begin, int size)
370 {
371 counter_u64_t *end = (counter_u64_t *)((char *)begin + size);
372
373 for (; begin < end; ++begin)
374 counter_u64_zero(*begin);
375 }
376
377 static void
ena_init_io_rings_common(struct ena_adapter * adapter,struct ena_ring * ring,uint16_t qid)378 ena_init_io_rings_common(struct ena_adapter *adapter, struct ena_ring *ring,
379 uint16_t qid)
380 {
381 ring->qid = qid;
382 ring->adapter = adapter;
383 ring->ena_dev = adapter->ena_dev;
384 atomic_store_8(&ring->first_interrupt, 0);
385 ring->no_interrupt_event_cnt = 0;
386 }
387
388 static void
ena_init_io_rings_basic(struct ena_adapter * adapter)389 ena_init_io_rings_basic(struct ena_adapter *adapter)
390 {
391 struct ena_com_dev *ena_dev;
392 struct ena_ring *txr, *rxr;
393 struct ena_que *que;
394 int i;
395
396 ena_dev = adapter->ena_dev;
397
398 for (i = 0; i < adapter->num_io_queues; i++) {
399 txr = &adapter->tx_ring[i];
400 rxr = &adapter->rx_ring[i];
401
402 /* TX/RX common ring state */
403 ena_init_io_rings_common(adapter, txr, i);
404 ena_init_io_rings_common(adapter, rxr, i);
405
406 /* TX specific ring state */
407 txr->tx_max_header_size = ena_dev->tx_max_header_size;
408 txr->tx_mem_queue_type = ena_dev->tx_mem_queue_type;
409
410 que = &adapter->que[i];
411 que->adapter = adapter;
412 que->id = i;
413 que->tx_ring = txr;
414 que->rx_ring = rxr;
415
416 txr->que = que;
417 rxr->que = que;
418
419 rxr->empty_rx_queue = 0;
420 rxr->rx_mbuf_sz = ena_mbuf_sz;
421 }
422 }
423
424 static void
ena_init_io_rings_advanced(struct ena_adapter * adapter)425 ena_init_io_rings_advanced(struct ena_adapter *adapter)
426 {
427 struct ena_ring *txr, *rxr;
428 int i;
429
430 for (i = 0; i < adapter->num_io_queues; i++) {
431 txr = &adapter->tx_ring[i];
432 rxr = &adapter->rx_ring[i];
433
434 /* Allocate a buf ring */
435 txr->buf_ring_size = adapter->buf_ring_size;
436 txr->br = buf_ring_alloc(txr->buf_ring_size, M_DEVBUF, M_WAITOK,
437 &txr->ring_mtx);
438
439 /* Allocate Tx statistics. */
440 ena_alloc_counters((counter_u64_t *)&txr->tx_stats,
441 sizeof(txr->tx_stats));
442 txr->tx_last_cleanup_ticks = ticks;
443
444 /* Allocate Rx statistics. */
445 ena_alloc_counters((counter_u64_t *)&rxr->rx_stats,
446 sizeof(rxr->rx_stats));
447
448 /* Initialize locks */
449 snprintf(txr->mtx_name, nitems(txr->mtx_name), "%s:tx(%d)",
450 device_get_nameunit(adapter->pdev), i);
451 snprintf(rxr->mtx_name, nitems(rxr->mtx_name), "%s:rx(%d)",
452 device_get_nameunit(adapter->pdev), i);
453
454 mtx_init(&txr->ring_mtx, txr->mtx_name, NULL, MTX_DEF);
455 }
456 }
457
458 static void
ena_init_io_rings(struct ena_adapter * adapter)459 ena_init_io_rings(struct ena_adapter *adapter)
460 {
461 /*
462 * IO rings initialization can be divided into the 2 steps:
463 * 1. Initialize variables and fields with initial values and copy
464 * them from adapter/ena_dev (basic)
465 * 2. Allocate mutex, counters and buf_ring (advanced)
466 */
467 ena_init_io_rings_basic(adapter);
468 ena_init_io_rings_advanced(adapter);
469 }
470
471 static void
ena_free_io_ring_resources(struct ena_adapter * adapter,unsigned int qid)472 ena_free_io_ring_resources(struct ena_adapter *adapter, unsigned int qid)
473 {
474 struct ena_ring *txr = &adapter->tx_ring[qid];
475 struct ena_ring *rxr = &adapter->rx_ring[qid];
476
477 ena_free_counters((counter_u64_t *)&txr->tx_stats,
478 sizeof(txr->tx_stats));
479 ena_free_counters((counter_u64_t *)&rxr->rx_stats,
480 sizeof(rxr->rx_stats));
481
482 ENA_RING_MTX_LOCK(txr);
483 drbr_free(txr->br, M_DEVBUF);
484 ENA_RING_MTX_UNLOCK(txr);
485
486 mtx_destroy(&txr->ring_mtx);
487 }
488
489 static void
ena_free_all_io_rings_resources(struct ena_adapter * adapter)490 ena_free_all_io_rings_resources(struct ena_adapter *adapter)
491 {
492 int i;
493
494 for (i = 0; i < adapter->num_io_queues; i++)
495 ena_free_io_ring_resources(adapter, i);
496 }
497
498 static int
ena_setup_tx_dma_tag(struct ena_adapter * adapter)499 ena_setup_tx_dma_tag(struct ena_adapter *adapter)
500 {
501 int ret;
502
503 /* Create DMA tag for Tx buffers */
504 ret = bus_dma_tag_create(bus_get_dma_tag(adapter->pdev),
505 1, 0, /* alignment, bounds */
506 ENA_DMA_BIT_MASK(adapter->dma_width), /* lowaddr of excl window */
507 BUS_SPACE_MAXADDR, /* highaddr of excl window */
508 NULL, NULL, /* filter, filterarg */
509 ENA_TSO_MAXSIZE, /* maxsize */
510 adapter->max_tx_sgl_size - 1, /* nsegments */
511 ENA_TSO_MAXSIZE, /* maxsegsize */
512 0, /* flags */
513 NULL, /* lockfunc */
514 NULL, /* lockfuncarg */
515 &adapter->tx_buf_tag);
516
517 return (ret);
518 }
519
520 static int
ena_free_tx_dma_tag(struct ena_adapter * adapter)521 ena_free_tx_dma_tag(struct ena_adapter *adapter)
522 {
523 int ret;
524
525 ret = bus_dma_tag_destroy(adapter->tx_buf_tag);
526
527 if (likely(ret == 0))
528 adapter->tx_buf_tag = NULL;
529
530 return (ret);
531 }
532
533 static int
ena_setup_rx_dma_tag(struct ena_adapter * adapter)534 ena_setup_rx_dma_tag(struct ena_adapter *adapter)
535 {
536 int ret;
537
538 /* Create DMA tag for Rx buffers*/
539 ret = bus_dma_tag_create(bus_get_dma_tag(adapter->pdev), /* parent */
540 1, 0, /* alignment, bounds */
541 ENA_DMA_BIT_MASK(adapter->dma_width), /* lowaddr of excl window */
542 BUS_SPACE_MAXADDR, /* highaddr of excl window */
543 NULL, NULL, /* filter, filterarg */
544 ena_mbuf_sz, /* maxsize */
545 adapter->max_rx_sgl_size, /* nsegments */
546 ena_mbuf_sz, /* maxsegsize */
547 0, /* flags */
548 NULL, /* lockfunc */
549 NULL, /* lockarg */
550 &adapter->rx_buf_tag);
551
552 return (ret);
553 }
554
555 static int
ena_free_rx_dma_tag(struct ena_adapter * adapter)556 ena_free_rx_dma_tag(struct ena_adapter *adapter)
557 {
558 int ret;
559
560 ret = bus_dma_tag_destroy(adapter->rx_buf_tag);
561
562 if (likely(ret == 0))
563 adapter->rx_buf_tag = NULL;
564
565 return (ret);
566 }
567
568 int
validate_tx_req_id(struct ena_ring * tx_ring,uint16_t req_id,int tx_req_id_rc)569 validate_tx_req_id(struct ena_ring *tx_ring, uint16_t req_id, int tx_req_id_rc)
570 {
571 struct ena_adapter *adapter = tx_ring->adapter;
572 enum ena_regs_reset_reason_types reset_reason = ENA_REGS_RESET_INV_TX_REQ_ID;
573
574 if (unlikely(tx_req_id_rc != 0)) {
575 if (tx_req_id_rc == ENA_COM_FAULT) {
576 reset_reason = ENA_REGS_RESET_TX_DESCRIPTOR_MALFORMED;
577 ena_log(adapter->pdev, ERR,
578 "TX descriptor malformed. req_id %hu qid %hu\n",
579 req_id, tx_ring->qid);
580 } else if (tx_req_id_rc == ENA_COM_INVAL) {
581 ena_log_nm(adapter->pdev, WARN,
582 "Invalid req_id %hu in qid %hu\n",
583 req_id, tx_ring->qid);
584 counter_u64_add(tx_ring->tx_stats.bad_req_id, 1);
585 }
586
587 ena_trigger_reset(adapter, reset_reason);
588 return (EFAULT);
589 }
590
591 return (0);
592 }
593
594 static void
ena_release_all_tx_dmamap(struct ena_ring * tx_ring)595 ena_release_all_tx_dmamap(struct ena_ring *tx_ring)
596 {
597 struct ena_adapter *adapter = tx_ring->adapter;
598 struct ena_tx_buffer *tx_info;
599 bus_dma_tag_t tx_tag = adapter->tx_buf_tag;
600 int i;
601 #ifdef DEV_NETMAP
602 struct ena_netmap_tx_info *nm_info;
603 int j;
604 #endif /* DEV_NETMAP */
605
606 for (i = 0; i < tx_ring->ring_size; ++i) {
607 tx_info = &tx_ring->tx_buffer_info[i];
608 #ifdef DEV_NETMAP
609 if (if_getcapenable(adapter->ifp) & IFCAP_NETMAP) {
610 nm_info = &tx_info->nm_info;
611 for (j = 0; j < ENA_PKT_MAX_BUFS; ++j) {
612 if (nm_info->map_seg[j] != NULL) {
613 bus_dmamap_destroy(tx_tag,
614 nm_info->map_seg[j]);
615 nm_info->map_seg[j] = NULL;
616 }
617 }
618 }
619 #endif /* DEV_NETMAP */
620 if (tx_info->dmamap != NULL) {
621 bus_dmamap_destroy(tx_tag, tx_info->dmamap);
622 tx_info->dmamap = NULL;
623 }
624 }
625 }
626
627 /**
628 * ena_setup_tx_resources - allocate Tx resources (Descriptors)
629 * @adapter: network interface device structure
630 * @qid: queue index
631 *
632 * Returns 0 on success, otherwise on failure.
633 **/
634 static int
ena_setup_tx_resources(struct ena_adapter * adapter,int qid)635 ena_setup_tx_resources(struct ena_adapter *adapter, int qid)
636 {
637 device_t pdev = adapter->pdev;
638 char thread_name[MAXCOMLEN + 1];
639 struct ena_que *que = &adapter->que[qid];
640 struct ena_ring *tx_ring = que->tx_ring;
641 cpuset_t *cpu_mask = NULL;
642 int size, i, err;
643 #ifdef DEV_NETMAP
644 bus_dmamap_t *map;
645 int j;
646
647 ena_netmap_reset_tx_ring(adapter, qid);
648 #endif /* DEV_NETMAP */
649
650 size = sizeof(struct ena_tx_buffer) * tx_ring->ring_size;
651
652 tx_ring->tx_buffer_info = malloc(size, M_DEVBUF, M_NOWAIT | M_ZERO);
653 if (unlikely(tx_ring->tx_buffer_info == NULL))
654 return (ENOMEM);
655
656 size = sizeof(uint16_t) * tx_ring->ring_size;
657 tx_ring->free_tx_ids = malloc(size, M_DEVBUF, M_NOWAIT | M_ZERO);
658 if (unlikely(tx_ring->free_tx_ids == NULL))
659 goto err_buf_info_free;
660
661 size = tx_ring->tx_max_header_size;
662 tx_ring->push_buf_intermediate_buf = malloc(size, M_DEVBUF,
663 M_NOWAIT | M_ZERO);
664 if (unlikely(tx_ring->push_buf_intermediate_buf == NULL))
665 goto err_tx_ids_free;
666
667 /* Req id stack for TX OOO completions */
668 for (i = 0; i < tx_ring->ring_size; i++)
669 tx_ring->free_tx_ids[i] = i;
670
671 /* Reset TX statistics. */
672 ena_reset_counters((counter_u64_t *)&tx_ring->tx_stats,
673 sizeof(tx_ring->tx_stats));
674
675 tx_ring->next_to_use = 0;
676 tx_ring->next_to_clean = 0;
677 tx_ring->acum_pkts = 0;
678
679 /* Make sure that drbr is empty */
680 ENA_RING_MTX_LOCK(tx_ring);
681 drbr_flush(adapter->ifp, tx_ring->br);
682 ENA_RING_MTX_UNLOCK(tx_ring);
683
684 /* ... and create the buffer DMA maps */
685 for (i = 0; i < tx_ring->ring_size; i++) {
686 err = bus_dmamap_create(adapter->tx_buf_tag, 0,
687 &tx_ring->tx_buffer_info[i].dmamap);
688 if (unlikely(err != 0)) {
689 ena_log(pdev, ERR,
690 "Unable to create Tx DMA map for buffer %d\n", i);
691 goto err_map_release;
692 }
693
694 #ifdef DEV_NETMAP
695 if (if_getcapenable(adapter->ifp) & IFCAP_NETMAP) {
696 map = tx_ring->tx_buffer_info[i].nm_info.map_seg;
697 for (j = 0; j < ENA_PKT_MAX_BUFS; j++) {
698 err = bus_dmamap_create(adapter->tx_buf_tag, 0,
699 &map[j]);
700 if (unlikely(err != 0)) {
701 ena_log(pdev, ERR,
702 "Unable to create Tx DMA for buffer %d %d\n",
703 i, j);
704 goto err_map_release;
705 }
706 }
707 }
708 #endif /* DEV_NETMAP */
709 }
710
711 /* Allocate taskqueues */
712 TASK_INIT(&tx_ring->enqueue_task, 0, ena_deferred_mq_start, tx_ring);
713 tx_ring->enqueue_tq = taskqueue_create_fast("ena_tx_enque", M_NOWAIT,
714 taskqueue_thread_enqueue, &tx_ring->enqueue_tq);
715 if (unlikely(tx_ring->enqueue_tq == NULL)) {
716 ena_log(pdev, ERR,
717 "Unable to create taskqueue for enqueue task\n");
718 i = tx_ring->ring_size;
719 goto err_map_release;
720 }
721
722 tx_ring->running = true;
723
724 #ifdef RSS
725 cpu_mask = &que->cpu_mask;
726 snprintf(thread_name, sizeof(thread_name), "%s txeq %d",
727 device_get_nameunit(adapter->pdev), que->cpu);
728 #else
729 if (que->domain >= 0)
730 cpu_mask = &cpuset_domain[que->domain];
731 snprintf(thread_name, sizeof(thread_name), "%s txeq %d",
732 device_get_nameunit(adapter->pdev), que->id);
733 #endif
734 taskqueue_start_threads_cpuset(&tx_ring->enqueue_tq, 1, PI_NET,
735 cpu_mask, "%s", thread_name);
736
737 return (0);
738
739 err_map_release:
740 ena_release_all_tx_dmamap(tx_ring);
741 err_tx_ids_free:
742 free(tx_ring->free_tx_ids, M_DEVBUF);
743 tx_ring->free_tx_ids = NULL;
744 err_buf_info_free:
745 free(tx_ring->tx_buffer_info, M_DEVBUF);
746 tx_ring->tx_buffer_info = NULL;
747
748 return (ENOMEM);
749 }
750
751 /**
752 * ena_free_tx_resources - Free Tx Resources per Queue
753 * @adapter: network interface device structure
754 * @qid: queue index
755 *
756 * Free all transmit software resources
757 **/
758 static void
ena_free_tx_resources(struct ena_adapter * adapter,int qid)759 ena_free_tx_resources(struct ena_adapter *adapter, int qid)
760 {
761 struct ena_ring *tx_ring = &adapter->tx_ring[qid];
762 #ifdef DEV_NETMAP
763 struct ena_netmap_tx_info *nm_info;
764 int j;
765 #endif /* DEV_NETMAP */
766
767 while (taskqueue_cancel(tx_ring->enqueue_tq, &tx_ring->enqueue_task, NULL))
768 taskqueue_drain(tx_ring->enqueue_tq, &tx_ring->enqueue_task);
769
770 taskqueue_free(tx_ring->enqueue_tq);
771
772 ENA_RING_MTX_LOCK(tx_ring);
773 /* Flush buffer ring, */
774 drbr_flush(adapter->ifp, tx_ring->br);
775
776 /* Free buffer DMA maps, */
777 for (int i = 0; i < tx_ring->ring_size; i++) {
778 bus_dmamap_sync(adapter->tx_buf_tag,
779 tx_ring->tx_buffer_info[i].dmamap, BUS_DMASYNC_POSTWRITE);
780 bus_dmamap_unload(adapter->tx_buf_tag,
781 tx_ring->tx_buffer_info[i].dmamap);
782 bus_dmamap_destroy(adapter->tx_buf_tag,
783 tx_ring->tx_buffer_info[i].dmamap);
784
785 #ifdef DEV_NETMAP
786 if (if_getcapenable(adapter->ifp) & IFCAP_NETMAP) {
787 nm_info = &tx_ring->tx_buffer_info[i].nm_info;
788 for (j = 0; j < ENA_PKT_MAX_BUFS; j++) {
789 if (nm_info->socket_buf_idx[j] != 0) {
790 bus_dmamap_sync(adapter->tx_buf_tag,
791 nm_info->map_seg[j],
792 BUS_DMASYNC_POSTWRITE);
793 ena_netmap_unload(adapter,
794 nm_info->map_seg[j]);
795 }
796 bus_dmamap_destroy(adapter->tx_buf_tag,
797 nm_info->map_seg[j]);
798 nm_info->socket_buf_idx[j] = 0;
799 }
800 }
801 #endif /* DEV_NETMAP */
802
803 m_freem(tx_ring->tx_buffer_info[i].mbuf);
804 tx_ring->tx_buffer_info[i].mbuf = NULL;
805 }
806 ENA_RING_MTX_UNLOCK(tx_ring);
807
808 /* And free allocated memory. */
809 free(tx_ring->tx_buffer_info, M_DEVBUF);
810 tx_ring->tx_buffer_info = NULL;
811
812 free(tx_ring->free_tx_ids, M_DEVBUF);
813 tx_ring->free_tx_ids = NULL;
814
815 free(tx_ring->push_buf_intermediate_buf, M_DEVBUF);
816 tx_ring->push_buf_intermediate_buf = NULL;
817 }
818
819 /**
820 * ena_setup_all_tx_resources - allocate all queues Tx resources
821 * @adapter: network interface device structure
822 *
823 * Returns 0 on success, otherwise on failure.
824 **/
825 static int
ena_setup_all_tx_resources(struct ena_adapter * adapter)826 ena_setup_all_tx_resources(struct ena_adapter *adapter)
827 {
828 int i, rc;
829
830 for (i = 0; i < adapter->num_io_queues; i++) {
831 rc = ena_setup_tx_resources(adapter, i);
832 if (rc != 0) {
833 ena_log(adapter->pdev, ERR,
834 "Allocation for Tx Queue %u failed\n", i);
835 goto err_setup_tx;
836 }
837 }
838
839 return (0);
840
841 err_setup_tx:
842 /* Rewind the index freeing the rings as we go */
843 while (i--)
844 ena_free_tx_resources(adapter, i);
845 return (rc);
846 }
847
848 /**
849 * ena_free_all_tx_resources - Free Tx Resources for All Queues
850 * @adapter: network interface device structure
851 *
852 * Free all transmit software resources
853 **/
854 static void
ena_free_all_tx_resources(struct ena_adapter * adapter)855 ena_free_all_tx_resources(struct ena_adapter *adapter)
856 {
857 int i;
858
859 for (i = 0; i < adapter->num_io_queues; i++)
860 ena_free_tx_resources(adapter, i);
861 }
862
863 /**
864 * ena_setup_rx_resources - allocate Rx resources (Descriptors)
865 * @adapter: network interface device structure
866 * @qid: queue index
867 *
868 * Returns 0 on success, otherwise on failure.
869 **/
870 static int
ena_setup_rx_resources(struct ena_adapter * adapter,unsigned int qid)871 ena_setup_rx_resources(struct ena_adapter *adapter, unsigned int qid)
872 {
873 device_t pdev = adapter->pdev;
874 struct ena_que *que = &adapter->que[qid];
875 struct ena_ring *rx_ring = que->rx_ring;
876 int size, err, i;
877
878 size = sizeof(struct ena_rx_buffer) * rx_ring->ring_size;
879
880 #ifdef DEV_NETMAP
881 ena_netmap_reset_rx_ring(adapter, qid);
882 rx_ring->initialized = false;
883 #endif /* DEV_NETMAP */
884
885 /*
886 * Alloc extra element so in rx path
887 * we can always prefetch rx_info + 1
888 */
889 size += sizeof(struct ena_rx_buffer);
890
891 rx_ring->rx_buffer_info = malloc(size, M_DEVBUF, M_WAITOK | M_ZERO);
892
893 size = sizeof(uint16_t) * rx_ring->ring_size;
894 rx_ring->free_rx_ids = malloc(size, M_DEVBUF, M_WAITOK);
895
896 for (i = 0; i < rx_ring->ring_size; i++)
897 rx_ring->free_rx_ids[i] = i;
898
899 /* Reset RX statistics. */
900 ena_reset_counters((counter_u64_t *)&rx_ring->rx_stats,
901 sizeof(rx_ring->rx_stats));
902
903 rx_ring->next_to_clean = 0;
904 rx_ring->next_to_use = 0;
905
906 /* ... and create the buffer DMA maps */
907 for (i = 0; i < rx_ring->ring_size; i++) {
908 err = bus_dmamap_create(adapter->rx_buf_tag, 0,
909 &(rx_ring->rx_buffer_info[i].map));
910 if (err != 0) {
911 ena_log(pdev, ERR,
912 "Unable to create Rx DMA map for buffer %d\n", i);
913 goto err_buf_info_unmap;
914 }
915 }
916
917 /* Create LRO for the ring */
918 if ((if_getcapenable(adapter->ifp) & IFCAP_LRO) != 0) {
919 int err = tcp_lro_init(&rx_ring->lro);
920 if (err != 0) {
921 ena_log(pdev, ERR, "LRO[%d] Initialization failed!\n",
922 qid);
923 } else {
924 ena_log(pdev, DBG, "RX Soft LRO[%d] Initialized\n",
925 qid);
926 rx_ring->lro.ifp = adapter->ifp;
927 }
928 }
929
930 return (0);
931
932 err_buf_info_unmap:
933 while (i--) {
934 bus_dmamap_destroy(adapter->rx_buf_tag,
935 rx_ring->rx_buffer_info[i].map);
936 }
937
938 free(rx_ring->free_rx_ids, M_DEVBUF);
939 rx_ring->free_rx_ids = NULL;
940 free(rx_ring->rx_buffer_info, M_DEVBUF);
941 rx_ring->rx_buffer_info = NULL;
942 return (ENOMEM);
943 }
944
945 /**
946 * ena_free_rx_resources - Free Rx Resources
947 * @adapter: network interface device structure
948 * @qid: queue index
949 *
950 * Free all receive software resources
951 **/
952 static void
ena_free_rx_resources(struct ena_adapter * adapter,unsigned int qid)953 ena_free_rx_resources(struct ena_adapter *adapter, unsigned int qid)
954 {
955 struct ena_ring *rx_ring = &adapter->rx_ring[qid];
956
957 /* Free buffer DMA maps, */
958 for (int i = 0; i < rx_ring->ring_size; i++) {
959 bus_dmamap_sync(adapter->rx_buf_tag,
960 rx_ring->rx_buffer_info[i].map, BUS_DMASYNC_POSTREAD);
961 m_freem(rx_ring->rx_buffer_info[i].mbuf);
962 rx_ring->rx_buffer_info[i].mbuf = NULL;
963 bus_dmamap_unload(adapter->rx_buf_tag,
964 rx_ring->rx_buffer_info[i].map);
965 bus_dmamap_destroy(adapter->rx_buf_tag,
966 rx_ring->rx_buffer_info[i].map);
967 }
968
969 /* free LRO resources, */
970 tcp_lro_free(&rx_ring->lro);
971
972 /* free allocated memory */
973 free(rx_ring->rx_buffer_info, M_DEVBUF);
974 rx_ring->rx_buffer_info = NULL;
975
976 free(rx_ring->free_rx_ids, M_DEVBUF);
977 rx_ring->free_rx_ids = NULL;
978 }
979
980 /**
981 * ena_setup_all_rx_resources - allocate all queues Rx resources
982 * @adapter: network interface device structure
983 *
984 * Returns 0 on success, otherwise on failure.
985 **/
986 static int
ena_setup_all_rx_resources(struct ena_adapter * adapter)987 ena_setup_all_rx_resources(struct ena_adapter *adapter)
988 {
989 int i, rc = 0;
990
991 for (i = 0; i < adapter->num_io_queues; i++) {
992 rc = ena_setup_rx_resources(adapter, i);
993 if (rc != 0) {
994 ena_log(adapter->pdev, ERR,
995 "Allocation for Rx Queue %u failed\n", i);
996 goto err_setup_rx;
997 }
998 }
999 return (0);
1000
1001 err_setup_rx:
1002 /* rewind the index freeing the rings as we go */
1003 while (i--)
1004 ena_free_rx_resources(adapter, i);
1005 return (rc);
1006 }
1007
1008 /**
1009 * ena_free_all_rx_resources - Free Rx resources for all queues
1010 * @adapter: network interface device structure
1011 *
1012 * Free all receive software resources
1013 **/
1014 static void
ena_free_all_rx_resources(struct ena_adapter * adapter)1015 ena_free_all_rx_resources(struct ena_adapter *adapter)
1016 {
1017 int i;
1018
1019 for (i = 0; i < adapter->num_io_queues; i++)
1020 ena_free_rx_resources(adapter, i);
1021 }
1022
1023 static inline int
ena_alloc_rx_mbuf(struct ena_adapter * adapter,struct ena_ring * rx_ring,struct ena_rx_buffer * rx_info)1024 ena_alloc_rx_mbuf(struct ena_adapter *adapter, struct ena_ring *rx_ring,
1025 struct ena_rx_buffer *rx_info)
1026 {
1027 device_t pdev = adapter->pdev;
1028 struct ena_com_buf *ena_buf;
1029 bus_dma_segment_t segs[1];
1030 int nsegs, error;
1031 int mlen;
1032
1033 /* if previous allocated frag is not used */
1034 if (unlikely(rx_info->mbuf != NULL))
1035 return (0);
1036
1037 /* Get mbuf using UMA allocator */
1038 rx_info->mbuf = m_getjcl(M_NOWAIT, MT_DATA, M_PKTHDR,
1039 rx_ring->rx_mbuf_sz);
1040
1041 if (unlikely(rx_info->mbuf == NULL)) {
1042 counter_u64_add(rx_ring->rx_stats.mjum_alloc_fail, 1);
1043 rx_info->mbuf = m_getcl(M_NOWAIT, MT_DATA, M_PKTHDR);
1044 if (unlikely(rx_info->mbuf == NULL)) {
1045 counter_u64_add(rx_ring->rx_stats.mbuf_alloc_fail, 1);
1046 return (ENOMEM);
1047 }
1048 mlen = MCLBYTES;
1049 } else {
1050 mlen = rx_ring->rx_mbuf_sz;
1051 }
1052 /* Set mbuf length*/
1053 rx_info->mbuf->m_pkthdr.len = rx_info->mbuf->m_len = mlen;
1054
1055 /* Map packets for DMA */
1056 ena_log(pdev, DBG,
1057 "Using tag %p for buffers' DMA mapping, mbuf %p len: %d\n",
1058 adapter->rx_buf_tag, rx_info->mbuf, rx_info->mbuf->m_len);
1059 error = bus_dmamap_load_mbuf_sg(adapter->rx_buf_tag, rx_info->map,
1060 rx_info->mbuf, segs, &nsegs, BUS_DMA_NOWAIT);
1061 if (unlikely((error != 0) || (nsegs != 1))) {
1062 ena_log(pdev, WARN,
1063 "failed to map mbuf, error: %d, nsegs: %d\n", error, nsegs);
1064 counter_u64_add(rx_ring->rx_stats.dma_mapping_err, 1);
1065 goto exit;
1066 }
1067
1068 bus_dmamap_sync(adapter->rx_buf_tag, rx_info->map, BUS_DMASYNC_PREREAD);
1069
1070 ena_buf = &rx_info->ena_buf;
1071 ena_buf->paddr = segs[0].ds_addr;
1072 ena_buf->len = mlen;
1073
1074 ena_log(pdev, DBG,
1075 "ALLOC RX BUF: mbuf %p, rx_info %p, len %d, paddr %#jx\n",
1076 rx_info->mbuf, rx_info, ena_buf->len, (uintmax_t)ena_buf->paddr);
1077
1078 return (0);
1079
1080 exit:
1081 m_freem(rx_info->mbuf);
1082 rx_info->mbuf = NULL;
1083 return (EFAULT);
1084 }
1085
1086 static void
ena_free_rx_mbuf(struct ena_adapter * adapter,struct ena_ring * rx_ring,struct ena_rx_buffer * rx_info)1087 ena_free_rx_mbuf(struct ena_adapter *adapter, struct ena_ring *rx_ring,
1088 struct ena_rx_buffer *rx_info)
1089 {
1090 if (rx_info->mbuf == NULL) {
1091 ena_log(adapter->pdev, WARN,
1092 "Trying to free unallocated buffer\n");
1093 return;
1094 }
1095
1096 bus_dmamap_sync(adapter->rx_buf_tag, rx_info->map,
1097 BUS_DMASYNC_POSTREAD);
1098 bus_dmamap_unload(adapter->rx_buf_tag, rx_info->map);
1099 m_freem(rx_info->mbuf);
1100 rx_info->mbuf = NULL;
1101 }
1102
1103 /**
1104 * ena_refill_rx_bufs - Refills ring with descriptors
1105 * @rx_ring: the ring which we want to feed with free descriptors
1106 * @num: number of descriptors to refill
1107 * Refills the ring with newly allocated DMA-mapped mbufs for receiving
1108 **/
1109 int
ena_refill_rx_bufs(struct ena_ring * rx_ring,uint32_t num)1110 ena_refill_rx_bufs(struct ena_ring *rx_ring, uint32_t num)
1111 {
1112 struct ena_adapter *adapter = rx_ring->adapter;
1113 device_t pdev = adapter->pdev;
1114 uint16_t next_to_use, req_id;
1115 uint32_t i;
1116 int rc;
1117
1118 ena_log_io(adapter->pdev, DBG, "refill qid: %d\n", rx_ring->qid);
1119
1120 next_to_use = rx_ring->next_to_use;
1121
1122 for (i = 0; i < num; i++) {
1123 struct ena_rx_buffer *rx_info;
1124
1125 ena_log_io(pdev, DBG, "RX buffer - next to use: %d\n",
1126 next_to_use);
1127
1128 req_id = rx_ring->free_rx_ids[next_to_use];
1129 rx_info = &rx_ring->rx_buffer_info[req_id];
1130 #ifdef DEV_NETMAP
1131 if (ena_rx_ring_in_netmap(adapter, rx_ring->qid))
1132 rc = ena_netmap_alloc_rx_slot(adapter, rx_ring,
1133 rx_info);
1134 else
1135 #endif /* DEV_NETMAP */
1136 rc = ena_alloc_rx_mbuf(adapter, rx_ring, rx_info);
1137 if (unlikely(rc != 0)) {
1138 ena_log_io(pdev, WARN,
1139 "failed to alloc buffer for rx queue %d\n",
1140 rx_ring->qid);
1141 break;
1142 }
1143 rc = ena_com_add_single_rx_desc(rx_ring->ena_com_io_sq,
1144 &rx_info->ena_buf, req_id);
1145 if (unlikely(rc != 0)) {
1146 ena_log_io(pdev, WARN,
1147 "failed to add buffer for rx queue %d\n",
1148 rx_ring->qid);
1149 break;
1150 }
1151 next_to_use = ENA_RX_RING_IDX_NEXT(next_to_use,
1152 rx_ring->ring_size);
1153 }
1154
1155 if (unlikely(i < num)) {
1156 counter_u64_add(rx_ring->rx_stats.refil_partial, 1);
1157 ena_log_io(pdev, WARN,
1158 "refilled rx qid %d with only %d mbufs (from %d)\n",
1159 rx_ring->qid, i, num);
1160 }
1161
1162 if (likely(i != 0))
1163 ena_com_write_sq_doorbell(rx_ring->ena_com_io_sq);
1164
1165 rx_ring->next_to_use = next_to_use;
1166 return (i);
1167 }
1168
1169 #ifdef DEV_NETMAP
1170 static int
ena_reinit_netmap(struct ena_adapter * adapter)1171 ena_reinit_netmap(struct ena_adapter *adapter)
1172 {
1173 int rc;
1174
1175 netmap_detach(adapter->ifp);
1176 rc = ena_netmap_attach(adapter);
1177 if (rc != 0)
1178 ena_log(adapter->pdev, ERR, "netmap attach failed: %d\n", rc);
1179
1180 return rc;
1181 }
1182
1183 #endif /* DEV_NETMAP */
1184 int
ena_update_buf_ring_size(struct ena_adapter * adapter,uint32_t new_buf_ring_size)1185 ena_update_buf_ring_size(struct ena_adapter *adapter,
1186 uint32_t new_buf_ring_size)
1187 {
1188 uint32_t old_buf_ring_size;
1189 int rc = 0;
1190 bool dev_was_up;
1191
1192 old_buf_ring_size = adapter->buf_ring_size;
1193 adapter->buf_ring_size = new_buf_ring_size;
1194
1195 dev_was_up = ENA_FLAG_ISSET(ENA_FLAG_DEV_UP, adapter);
1196 ena_down(adapter);
1197
1198 /* Reconfigure buf ring for all Tx rings. */
1199 ena_free_all_io_rings_resources(adapter);
1200 ena_init_io_rings_advanced(adapter);
1201 #ifdef DEV_NETMAP
1202 rc = ena_reinit_netmap(adapter);
1203 if (rc != 0)
1204 return rc;
1205
1206 #endif /* DEV_NETMAP */
1207 if (dev_was_up) {
1208 /*
1209 * If ena_up() fails, it's not because of recent buf_ring size
1210 * changes. Because of that, we just want to revert old drbr
1211 * value and trigger the reset because something else had to
1212 * go wrong.
1213 */
1214 rc = ena_up(adapter);
1215 if (unlikely(rc != 0)) {
1216 ena_log(adapter->pdev, ERR,
1217 "Failed to configure device after setting new drbr size: %u. Reverting old value: %u and triggering the reset\n",
1218 new_buf_ring_size, old_buf_ring_size);
1219
1220 /* Revert old size and trigger the reset */
1221 adapter->buf_ring_size = old_buf_ring_size;
1222 ena_free_all_io_rings_resources(adapter);
1223 ena_init_io_rings_advanced(adapter);
1224 #ifdef DEV_NETMAP
1225 rc = ena_reinit_netmap(adapter);
1226 if (rc != 0)
1227 return rc;
1228
1229 #endif /* DEV_NETMAP */
1230 ENA_FLAG_SET_ATOMIC(ENA_FLAG_DEV_UP_BEFORE_RESET,
1231 adapter);
1232 ena_trigger_reset(adapter, ENA_REGS_RESET_OS_TRIGGER);
1233 }
1234 }
1235
1236 return (rc);
1237 }
1238
1239 int
ena_update_queue_size(struct ena_adapter * adapter,uint32_t new_tx_size,uint32_t new_rx_size)1240 ena_update_queue_size(struct ena_adapter *adapter, uint32_t new_tx_size,
1241 uint32_t new_rx_size)
1242 {
1243 uint32_t old_tx_size, old_rx_size;
1244 int rc = 0;
1245 bool dev_was_up;
1246
1247 old_tx_size = adapter->requested_tx_ring_size;
1248 old_rx_size = adapter->requested_rx_ring_size;
1249 adapter->requested_tx_ring_size = new_tx_size;
1250 adapter->requested_rx_ring_size = new_rx_size;
1251
1252 dev_was_up = ENA_FLAG_ISSET(ENA_FLAG_DEV_UP, adapter);
1253 ena_down(adapter);
1254
1255 /* Configure queues with new size. */
1256 ena_init_io_rings_basic(adapter);
1257 #ifdef DEV_NETMAP
1258 rc = ena_reinit_netmap(adapter);
1259 if (rc != 0)
1260 return rc;
1261
1262 #endif /* DEV_NETMAP */
1263 if (dev_was_up) {
1264 rc = ena_up(adapter);
1265 if (unlikely(rc != 0)) {
1266 ena_log(adapter->pdev, ERR,
1267 "Failed to configure device with the new sizes - Tx: %u Rx: %u. Reverting old values - Tx: %u Rx: %u\n",
1268 new_tx_size, new_rx_size, old_tx_size, old_rx_size);
1269
1270 /* Revert old size. */
1271 adapter->requested_tx_ring_size = old_tx_size;
1272 adapter->requested_rx_ring_size = old_rx_size;
1273 ena_init_io_rings_basic(adapter);
1274 #ifdef DEV_NETMAP
1275 rc = ena_reinit_netmap(adapter);
1276 if (rc != 0)
1277 return rc;
1278
1279 #endif /* DEV_NETMAP */
1280 /* And try again. */
1281 rc = ena_up(adapter);
1282 if (unlikely(rc != 0)) {
1283 ena_log(adapter->pdev, ERR,
1284 "Failed to revert old queue sizes. Triggering device reset.\n");
1285 /*
1286 * If we've failed again, something had to go
1287 * wrong. After reset, the device should try to
1288 * go up
1289 */
1290 ENA_FLAG_SET_ATOMIC(
1291 ENA_FLAG_DEV_UP_BEFORE_RESET, adapter);
1292 ena_trigger_reset(adapter,
1293 ENA_REGS_RESET_OS_TRIGGER);
1294 }
1295 }
1296 }
1297
1298 return (rc);
1299 }
1300
1301 static void
ena_update_io_rings(struct ena_adapter * adapter,uint32_t num)1302 ena_update_io_rings(struct ena_adapter *adapter, uint32_t num)
1303 {
1304 ena_free_all_io_rings_resources(adapter);
1305 /* Force indirection table to be reinitialized */
1306 ena_com_rss_destroy(adapter->ena_dev);
1307
1308 adapter->num_io_queues = num;
1309 ena_init_io_rings(adapter);
1310 }
1311
1312 int
ena_update_base_cpu(struct ena_adapter * adapter,int new_num)1313 ena_update_base_cpu(struct ena_adapter *adapter, int new_num)
1314 {
1315 int old_num;
1316 int rc = 0;
1317 bool dev_was_up;
1318
1319 dev_was_up = ENA_FLAG_ISSET(ENA_FLAG_DEV_UP, adapter);
1320 old_num = adapter->irq_cpu_base;
1321
1322 ena_down(adapter);
1323
1324 adapter->irq_cpu_base = new_num;
1325
1326 if (dev_was_up) {
1327 rc = ena_up(adapter);
1328 if (unlikely(rc != 0)) {
1329 ena_log(adapter->pdev, ERR,
1330 "Failed to configure device %d IRQ base CPU. "
1331 "Reverting to previous value: %d\n",
1332 new_num, old_num);
1333
1334 adapter->irq_cpu_base = old_num;
1335
1336 rc = ena_up(adapter);
1337 if (unlikely(rc != 0)) {
1338 ena_log(adapter->pdev, ERR,
1339 "Failed to revert to previous setup."
1340 "Triggering device reset.\n");
1341 ENA_FLAG_SET_ATOMIC(
1342 ENA_FLAG_DEV_UP_BEFORE_RESET, adapter);
1343 ena_trigger_reset(adapter,
1344 ENA_REGS_RESET_OS_TRIGGER);
1345 }
1346 }
1347 }
1348 return (rc);
1349 }
1350
1351 int
ena_update_cpu_stride(struct ena_adapter * adapter,uint32_t new_num)1352 ena_update_cpu_stride(struct ena_adapter *adapter, uint32_t new_num)
1353 {
1354 uint32_t old_num;
1355 int rc = 0;
1356 bool dev_was_up;
1357
1358 dev_was_up = ENA_FLAG_ISSET(ENA_FLAG_DEV_UP, adapter);
1359 old_num = adapter->irq_cpu_stride;
1360
1361 ena_down(adapter);
1362
1363 adapter->irq_cpu_stride = new_num;
1364
1365 if (dev_was_up) {
1366 rc = ena_up(adapter);
1367 if (unlikely(rc != 0)) {
1368 ena_log(adapter->pdev, ERR,
1369 "Failed to configure device %d IRQ CPU stride. "
1370 "Reverting to previous value: %d\n",
1371 new_num, old_num);
1372
1373 adapter->irq_cpu_stride = old_num;
1374
1375 rc = ena_up(adapter);
1376 if (unlikely(rc != 0)) {
1377 ena_log(adapter->pdev, ERR,
1378 "Failed to revert to previous setup."
1379 "Triggering device reset.\n");
1380 ENA_FLAG_SET_ATOMIC(
1381 ENA_FLAG_DEV_UP_BEFORE_RESET, adapter);
1382 ena_trigger_reset(adapter,
1383 ENA_REGS_RESET_OS_TRIGGER);
1384 }
1385 }
1386 }
1387 return (rc);
1388 }
1389
1390 /* Caller should sanitize new_num */
1391 int
ena_update_io_queue_nb(struct ena_adapter * adapter,uint32_t new_num)1392 ena_update_io_queue_nb(struct ena_adapter *adapter, uint32_t new_num)
1393 {
1394 uint32_t old_num;
1395 int rc = 0;
1396 bool dev_was_up;
1397
1398 dev_was_up = ENA_FLAG_ISSET(ENA_FLAG_DEV_UP, adapter);
1399 old_num = adapter->num_io_queues;
1400 ena_down(adapter);
1401
1402 ena_update_io_rings(adapter, new_num);
1403 #ifdef DEV_NETMAP
1404 rc = ena_reinit_netmap(adapter);
1405 if (rc != 0)
1406 return rc;
1407
1408 #endif /* DEV_NETMAP */
1409 if (dev_was_up) {
1410 rc = ena_up(adapter);
1411 if (unlikely(rc != 0)) {
1412 ena_log(adapter->pdev, ERR,
1413 "Failed to configure device with %u IO queues. "
1414 "Reverting to previous value: %u\n",
1415 new_num, old_num);
1416
1417 ena_update_io_rings(adapter, old_num);
1418 #ifdef DEV_NETMAP
1419 rc = ena_reinit_netmap(adapter);
1420 if (rc != 0)
1421 return rc;
1422
1423 #endif /* DEV_NETMAP */
1424 rc = ena_up(adapter);
1425 if (unlikely(rc != 0)) {
1426 ena_log(adapter->pdev, ERR,
1427 "Failed to revert to previous setup IO "
1428 "queues. Triggering device reset.\n");
1429 ENA_FLAG_SET_ATOMIC(
1430 ENA_FLAG_DEV_UP_BEFORE_RESET, adapter);
1431 ena_trigger_reset(adapter,
1432 ENA_REGS_RESET_OS_TRIGGER);
1433 }
1434 }
1435 }
1436
1437 return (rc);
1438 }
1439
1440 static void
ena_free_rx_bufs(struct ena_adapter * adapter,unsigned int qid)1441 ena_free_rx_bufs(struct ena_adapter *adapter, unsigned int qid)
1442 {
1443 struct ena_ring *rx_ring = &adapter->rx_ring[qid];
1444 unsigned int i;
1445
1446 for (i = 0; i < rx_ring->ring_size; i++) {
1447 struct ena_rx_buffer *rx_info = &rx_ring->rx_buffer_info[i];
1448
1449 if (rx_info->mbuf != NULL)
1450 ena_free_rx_mbuf(adapter, rx_ring, rx_info);
1451 #ifdef DEV_NETMAP
1452 if (((if_getflags(adapter->ifp) & IFF_DYING) == 0) &&
1453 (if_getcapenable(adapter->ifp) & IFCAP_NETMAP)) {
1454 if (rx_info->netmap_buf_idx != 0)
1455 ena_netmap_free_rx_slot(adapter, rx_ring,
1456 rx_info);
1457 }
1458 #endif /* DEV_NETMAP */
1459 }
1460 }
1461
1462 /**
1463 * ena_refill_all_rx_bufs - allocate all queues Rx buffers
1464 * @adapter: network interface device structure
1465 *
1466 */
1467 static void
ena_refill_all_rx_bufs(struct ena_adapter * adapter)1468 ena_refill_all_rx_bufs(struct ena_adapter *adapter)
1469 {
1470 struct ena_ring *rx_ring;
1471 int i, rc, bufs_num;
1472
1473 for (i = 0; i < adapter->num_io_queues; i++) {
1474 rx_ring = &adapter->rx_ring[i];
1475 bufs_num = rx_ring->ring_size - 1;
1476 rc = ena_refill_rx_bufs(rx_ring, bufs_num);
1477 if (unlikely(rc != bufs_num))
1478 ena_log_io(adapter->pdev, WARN,
1479 "refilling Queue %d failed. "
1480 "Allocated %d buffers from: %d\n",
1481 i, rc, bufs_num);
1482 #ifdef DEV_NETMAP
1483 rx_ring->initialized = true;
1484 #endif /* DEV_NETMAP */
1485 }
1486 }
1487
1488 static void
ena_free_all_rx_bufs(struct ena_adapter * adapter)1489 ena_free_all_rx_bufs(struct ena_adapter *adapter)
1490 {
1491 int i;
1492
1493 for (i = 0; i < adapter->num_io_queues; i++)
1494 ena_free_rx_bufs(adapter, i);
1495 }
1496
1497 /**
1498 * ena_free_tx_bufs - Free Tx Buffers per Queue
1499 * @adapter: network interface device structure
1500 * @qid: queue index
1501 **/
1502 static void
ena_free_tx_bufs(struct ena_adapter * adapter,unsigned int qid)1503 ena_free_tx_bufs(struct ena_adapter *adapter, unsigned int qid)
1504 {
1505 bool print_once = true;
1506 struct ena_ring *tx_ring = &adapter->tx_ring[qid];
1507
1508 ENA_RING_MTX_LOCK(tx_ring);
1509 for (int i = 0; i < tx_ring->ring_size; i++) {
1510 struct ena_tx_buffer *tx_info = &tx_ring->tx_buffer_info[i];
1511
1512 if (tx_info->mbuf == NULL)
1513 continue;
1514
1515 if (print_once) {
1516 ena_log(adapter->pdev, WARN,
1517 "free uncompleted tx mbuf qid %d idx 0x%x\n", qid,
1518 i);
1519 print_once = false;
1520 } else {
1521 ena_log(adapter->pdev, DBG,
1522 "free uncompleted tx mbuf qid %d idx 0x%x\n", qid,
1523 i);
1524 }
1525
1526 bus_dmamap_sync(adapter->tx_buf_tag, tx_info->dmamap,
1527 BUS_DMASYNC_POSTWRITE);
1528 bus_dmamap_unload(adapter->tx_buf_tag, tx_info->dmamap);
1529
1530 m_free(tx_info->mbuf);
1531 tx_info->mbuf = NULL;
1532 }
1533 ENA_RING_MTX_UNLOCK(tx_ring);
1534 }
1535
1536 static void
ena_free_all_tx_bufs(struct ena_adapter * adapter)1537 ena_free_all_tx_bufs(struct ena_adapter *adapter)
1538 {
1539 for (int i = 0; i < adapter->num_io_queues; i++)
1540 ena_free_tx_bufs(adapter, i);
1541 }
1542
1543 static void
ena_destroy_all_tx_queues(struct ena_adapter * adapter)1544 ena_destroy_all_tx_queues(struct ena_adapter *adapter)
1545 {
1546 uint16_t ena_qid;
1547 int i;
1548
1549 for (i = 0; i < adapter->num_io_queues; i++) {
1550 ena_qid = ENA_IO_TXQ_IDX(i);
1551 ena_com_destroy_io_queue(adapter->ena_dev, ena_qid);
1552 }
1553 }
1554
1555 static void
ena_destroy_all_rx_queues(struct ena_adapter * adapter)1556 ena_destroy_all_rx_queues(struct ena_adapter *adapter)
1557 {
1558 uint16_t ena_qid;
1559 int i;
1560
1561 for (i = 0; i < adapter->num_io_queues; i++) {
1562 ena_qid = ENA_IO_RXQ_IDX(i);
1563 ena_com_destroy_io_queue(adapter->ena_dev, ena_qid);
1564 }
1565 }
1566
1567 static void
ena_destroy_all_io_queues(struct ena_adapter * adapter)1568 ena_destroy_all_io_queues(struct ena_adapter *adapter)
1569 {
1570 struct ena_que *queue;
1571 int i;
1572
1573 for (i = 0; i < adapter->num_io_queues; i++) {
1574 queue = &adapter->que[i];
1575 while (taskqueue_cancel(queue->cleanup_tq, &queue->cleanup_task, NULL))
1576 taskqueue_drain(queue->cleanup_tq, &queue->cleanup_task);
1577 taskqueue_free(queue->cleanup_tq);
1578 }
1579
1580 ena_destroy_all_tx_queues(adapter);
1581 ena_destroy_all_rx_queues(adapter);
1582 }
1583
1584 static int
ena_create_io_queues(struct ena_adapter * adapter)1585 ena_create_io_queues(struct ena_adapter *adapter)
1586 {
1587 struct ena_com_dev *ena_dev = adapter->ena_dev;
1588 struct ena_com_create_io_ctx ctx;
1589 struct ena_ring *ring;
1590 struct ena_que *queue;
1591 uint16_t ena_qid;
1592 uint32_t msix_vector;
1593 cpuset_t *cpu_mask = NULL;
1594 int rc, i;
1595
1596 /* Create TX queues */
1597 for (i = 0; i < adapter->num_io_queues; i++) {
1598 msix_vector = ENA_IO_IRQ_IDX(i);
1599 ena_qid = ENA_IO_TXQ_IDX(i);
1600 ctx.mem_queue_type = ena_dev->tx_mem_queue_type;
1601 ctx.direction = ENA_COM_IO_QUEUE_DIRECTION_TX;
1602 ctx.queue_size = adapter->requested_tx_ring_size;
1603 ctx.msix_vector = msix_vector;
1604 ctx.qid = ena_qid;
1605 ctx.numa_node = adapter->que[i].domain;
1606
1607 rc = ena_com_create_io_queue(ena_dev, &ctx);
1608 if (rc != 0) {
1609 ena_log(adapter->pdev, ERR,
1610 "Failed to create io TX queue #%d rc: %d\n", i, rc);
1611 goto err_tx;
1612 }
1613 ring = &adapter->tx_ring[i];
1614 rc = ena_com_get_io_handlers(ena_dev, ena_qid,
1615 &ring->ena_com_io_sq, &ring->ena_com_io_cq);
1616 if (rc != 0) {
1617 ena_log(adapter->pdev, ERR,
1618 "Failed to get TX queue handlers. TX queue num"
1619 " %d rc: %d\n",
1620 i, rc);
1621 ena_com_destroy_io_queue(ena_dev, ena_qid);
1622 goto err_tx;
1623 }
1624
1625 if (ctx.numa_node >= 0) {
1626 ena_com_update_numa_node(ring->ena_com_io_cq,
1627 ctx.numa_node);
1628 }
1629 }
1630
1631 /* Create RX queues */
1632 for (i = 0; i < adapter->num_io_queues; i++) {
1633 msix_vector = ENA_IO_IRQ_IDX(i);
1634 ena_qid = ENA_IO_RXQ_IDX(i);
1635 ctx.mem_queue_type = ENA_ADMIN_PLACEMENT_POLICY_HOST;
1636 ctx.direction = ENA_COM_IO_QUEUE_DIRECTION_RX;
1637 ctx.queue_size = adapter->requested_rx_ring_size;
1638 ctx.msix_vector = msix_vector;
1639 ctx.qid = ena_qid;
1640 ctx.numa_node = adapter->que[i].domain;
1641
1642 rc = ena_com_create_io_queue(ena_dev, &ctx);
1643 if (unlikely(rc != 0)) {
1644 ena_log(adapter->pdev, ERR,
1645 "Failed to create io RX queue[%d] rc: %d\n", i, rc);
1646 goto err_rx;
1647 }
1648
1649 ring = &adapter->rx_ring[i];
1650 rc = ena_com_get_io_handlers(ena_dev, ena_qid,
1651 &ring->ena_com_io_sq, &ring->ena_com_io_cq);
1652 if (unlikely(rc != 0)) {
1653 ena_log(adapter->pdev, ERR,
1654 "Failed to get RX queue handlers. RX queue num"
1655 " %d rc: %d\n",
1656 i, rc);
1657 ena_com_destroy_io_queue(ena_dev, ena_qid);
1658 goto err_rx;
1659 }
1660
1661 if (ctx.numa_node >= 0) {
1662 ena_com_update_numa_node(ring->ena_com_io_cq,
1663 ctx.numa_node);
1664 }
1665 }
1666
1667 for (i = 0; i < adapter->num_io_queues; i++) {
1668 queue = &adapter->que[i];
1669
1670 NET_TASK_INIT(&queue->cleanup_task, 0, ena_cleanup, queue);
1671 queue->cleanup_tq = taskqueue_create_fast("ena cleanup",
1672 M_WAITOK, taskqueue_thread_enqueue, &queue->cleanup_tq);
1673
1674 #ifdef RSS
1675 cpu_mask = &queue->cpu_mask;
1676 #else
1677 if (queue->domain >= 0)
1678 cpu_mask = &cpuset_domain[queue->domain];
1679 #endif
1680 taskqueue_start_threads_cpuset(&queue->cleanup_tq, 1, PI_NET,
1681 cpu_mask, "%s queue %d cleanup",
1682 device_get_nameunit(adapter->pdev), i);
1683 }
1684
1685 return (0);
1686
1687 err_rx:
1688 while (i--)
1689 ena_com_destroy_io_queue(ena_dev, ENA_IO_RXQ_IDX(i));
1690 i = adapter->num_io_queues;
1691 err_tx:
1692 while (i--)
1693 ena_com_destroy_io_queue(ena_dev, ENA_IO_TXQ_IDX(i));
1694
1695 return (ENXIO);
1696 }
1697
1698 /*********************************************************************
1699 *
1700 * MSIX & Interrupt Service routine
1701 *
1702 **********************************************************************/
1703
1704 /**
1705 * ena_handle_msix - MSIX Interrupt Handler for admin/async queue
1706 * @arg: interrupt number
1707 **/
1708 static void
ena_intr_msix_mgmnt(void * arg)1709 ena_intr_msix_mgmnt(void *arg)
1710 {
1711 struct ena_adapter *adapter = (struct ena_adapter *)arg;
1712
1713 ena_com_admin_q_comp_intr_handler(adapter->ena_dev);
1714 if (likely(ENA_FLAG_ISSET(ENA_FLAG_DEVICE_RUNNING, adapter)))
1715 ena_com_aenq_intr_handler(adapter->ena_dev, arg);
1716 }
1717
1718 /**
1719 * ena_handle_msix - MSIX Interrupt Handler for Tx/Rx
1720 * @arg: queue
1721 **/
1722 static int
ena_handle_msix(void * arg)1723 ena_handle_msix(void *arg)
1724 {
1725 struct ena_que *queue = arg;
1726 struct ena_adapter *adapter = queue->adapter;
1727 if_t ifp = adapter->ifp;
1728
1729 if (unlikely((if_getdrvflags(ifp) & IFF_DRV_RUNNING) == 0))
1730 return (FILTER_STRAY);
1731
1732 taskqueue_enqueue(queue->cleanup_tq, &queue->cleanup_task);
1733
1734 return (FILTER_HANDLED);
1735 }
1736
1737 static int
ena_enable_msix(struct ena_adapter * adapter)1738 ena_enable_msix(struct ena_adapter *adapter)
1739 {
1740 device_t dev = adapter->pdev;
1741 int msix_vecs, msix_req;
1742 int i, rc = 0;
1743
1744 if (ENA_FLAG_ISSET(ENA_FLAG_MSIX_ENABLED, adapter)) {
1745 ena_log(dev, ERR, "Error, MSI-X is already enabled\n");
1746 return (EINVAL);
1747 }
1748
1749 /* Reserved the max msix vectors we might need */
1750 msix_vecs = ENA_MAX_MSIX_VEC(adapter->max_num_io_queues);
1751
1752 adapter->msix_entries = malloc(msix_vecs * sizeof(struct msix_entry),
1753 M_DEVBUF, M_WAITOK | M_ZERO);
1754
1755 ena_log(dev, DBG, "trying to enable MSI-X, vectors: %d\n", msix_vecs);
1756
1757 for (i = 0; i < msix_vecs; i++) {
1758 adapter->msix_entries[i].entry = i;
1759 /* Vectors must start from 1 */
1760 adapter->msix_entries[i].vector = i + 1;
1761 }
1762
1763 msix_req = msix_vecs;
1764 rc = pci_alloc_msix(dev, &msix_vecs);
1765 if (unlikely(rc != 0)) {
1766 ena_log(dev, ERR, "Failed to enable MSIX, vectors %d rc %d\n",
1767 msix_vecs, rc);
1768
1769 rc = ENOSPC;
1770 goto err_msix_free;
1771 }
1772
1773 if (msix_vecs != msix_req) {
1774 if (msix_vecs == ENA_ADMIN_MSIX_VEC) {
1775 ena_log(dev, ERR,
1776 "Not enough number of MSI-x allocated: %d\n",
1777 msix_vecs);
1778 pci_release_msi(dev);
1779 rc = ENOSPC;
1780 goto err_msix_free;
1781 }
1782 ena_log(dev, ERR,
1783 "Enable only %d MSI-x (out of %d), reduce "
1784 "the number of queues\n",
1785 msix_vecs, msix_req);
1786 }
1787
1788 adapter->msix_vecs = msix_vecs;
1789 ENA_FLAG_SET_ATOMIC(ENA_FLAG_MSIX_ENABLED, adapter);
1790
1791 return (0);
1792
1793 err_msix_free:
1794 free(adapter->msix_entries, M_DEVBUF);
1795 adapter->msix_entries = NULL;
1796
1797 return (rc);
1798 }
1799
1800 static void
ena_setup_mgmnt_intr(struct ena_adapter * adapter)1801 ena_setup_mgmnt_intr(struct ena_adapter *adapter)
1802 {
1803 snprintf(adapter->irq_tbl[ENA_MGMNT_IRQ_IDX].name, ENA_IRQNAME_SIZE,
1804 "ena-mgmnt@pci:%s", device_get_nameunit(adapter->pdev));
1805 /*
1806 * Handler is NULL on purpose, it will be set
1807 * when mgmnt interrupt is acquired
1808 */
1809 adapter->irq_tbl[ENA_MGMNT_IRQ_IDX].handler = NULL;
1810 adapter->irq_tbl[ENA_MGMNT_IRQ_IDX].data = adapter;
1811 adapter->irq_tbl[ENA_MGMNT_IRQ_IDX].vector =
1812 adapter->msix_entries[ENA_MGMNT_IRQ_IDX].vector;
1813 }
1814
1815 static int
ena_setup_io_intr(struct ena_adapter * adapter)1816 ena_setup_io_intr(struct ena_adapter *adapter)
1817 {
1818 #ifdef RSS
1819 int num_buckets = rss_getnumbuckets();
1820 static int last_bind = 0;
1821 int cur_bind;
1822 int idx;
1823 #else
1824 int domain;
1825 #endif
1826 int irq_idx;
1827
1828 if (adapter->msix_entries == NULL)
1829 return (EINVAL);
1830
1831 #ifdef RSS
1832 if (adapter->first_bind < 0) {
1833 adapter->first_bind = last_bind;
1834 last_bind = (last_bind + adapter->num_io_queues) % num_buckets;
1835 }
1836 cur_bind = adapter->first_bind;
1837 #else
1838 if (bus_get_domain(adapter->pdev, &domain))
1839 domain = -1;
1840 #endif
1841
1842 for (int i = 0; i < adapter->num_io_queues; i++) {
1843 irq_idx = ENA_IO_IRQ_IDX(i);
1844
1845 snprintf(adapter->irq_tbl[irq_idx].name, ENA_IRQNAME_SIZE,
1846 "%s-TxRx-%d", device_get_nameunit(adapter->pdev), i);
1847 adapter->irq_tbl[irq_idx].handler = ena_handle_msix;
1848 adapter->irq_tbl[irq_idx].data = &adapter->que[i];
1849 adapter->irq_tbl[irq_idx].vector =
1850 adapter->msix_entries[irq_idx].vector;
1851 ena_log(adapter->pdev, DBG, "ena_setup_io_intr vector: %d\n",
1852 adapter->msix_entries[irq_idx].vector);
1853
1854 if (adapter->irq_cpu_base > ENA_BASE_CPU_UNSPECIFIED) {
1855 adapter->que[i].cpu = adapter->irq_tbl[irq_idx].cpu =
1856 (unsigned)(adapter->irq_cpu_base +
1857 i * adapter->irq_cpu_stride) % (unsigned)mp_ncpus;
1858 CPU_SETOF(adapter->que[i].cpu, &adapter->que[i].cpu_mask);
1859 }
1860
1861 #ifdef RSS
1862 adapter->que[i].cpu = adapter->irq_tbl[irq_idx].cpu =
1863 rss_getcpu(cur_bind);
1864 cur_bind = (cur_bind + 1) % num_buckets;
1865 CPU_SETOF(adapter->que[i].cpu, &adapter->que[i].cpu_mask);
1866
1867 for (idx = 0; idx < MAXMEMDOM; ++idx) {
1868 if (CPU_ISSET(adapter->que[i].cpu, &cpuset_domain[idx]))
1869 break;
1870 }
1871 adapter->que[i].domain = idx;
1872 #else
1873 adapter->que[i].domain = domain;
1874 #endif /* RSS */
1875 }
1876
1877 return (0);
1878 }
1879
1880 static int
ena_request_mgmnt_irq(struct ena_adapter * adapter)1881 ena_request_mgmnt_irq(struct ena_adapter *adapter)
1882 {
1883 device_t pdev = adapter->pdev;
1884 struct ena_irq *irq;
1885 unsigned long flags;
1886 int rc, rcc;
1887
1888 flags = RF_ACTIVE | RF_SHAREABLE;
1889
1890 irq = &adapter->irq_tbl[ENA_MGMNT_IRQ_IDX];
1891 irq->res = bus_alloc_resource_any(adapter->pdev, SYS_RES_IRQ,
1892 &irq->vector, flags);
1893
1894 if (unlikely(irq->res == NULL)) {
1895 ena_log(pdev, ERR, "could not allocate irq vector: %d\n",
1896 irq->vector);
1897 return (ENXIO);
1898 }
1899
1900 rc = bus_setup_intr(adapter->pdev, irq->res,
1901 INTR_TYPE_NET | INTR_MPSAFE, NULL, ena_intr_msix_mgmnt, irq->data,
1902 &irq->cookie);
1903 if (unlikely(rc != 0)) {
1904 ena_log(pdev, ERR,
1905 "failed to register interrupt handler for irq %ju: %d\n",
1906 rman_get_start(irq->res), rc);
1907 goto err_res_free;
1908 }
1909 irq->requested = true;
1910
1911 return (rc);
1912
1913 err_res_free:
1914 ena_log(pdev, INFO, "releasing resource for irq %d\n", irq->vector);
1915 rcc = bus_release_resource(adapter->pdev, SYS_RES_IRQ, irq->vector,
1916 irq->res);
1917 if (unlikely(rcc != 0))
1918 ena_log(pdev, ERR,
1919 "dev has no parent while releasing res for irq: %d\n",
1920 irq->vector);
1921 irq->res = NULL;
1922
1923 return (rc);
1924 }
1925
1926 static int
ena_request_io_irq(struct ena_adapter * adapter)1927 ena_request_io_irq(struct ena_adapter *adapter)
1928 {
1929 device_t pdev = adapter->pdev;
1930 struct ena_irq *irq;
1931 unsigned long flags = 0;
1932 int rc = 0, i, rcc;
1933
1934 if (unlikely(!ENA_FLAG_ISSET(ENA_FLAG_MSIX_ENABLED, adapter))) {
1935 ena_log(pdev, ERR,
1936 "failed to request I/O IRQ: MSI-X is not enabled\n");
1937 return (EINVAL);
1938 } else {
1939 flags = RF_ACTIVE | RF_SHAREABLE;
1940 }
1941
1942 for (i = ENA_IO_IRQ_FIRST_IDX; i < adapter->msix_vecs; i++) {
1943 irq = &adapter->irq_tbl[i];
1944
1945 if (unlikely(irq->requested))
1946 continue;
1947
1948 irq->res = bus_alloc_resource_any(adapter->pdev, SYS_RES_IRQ,
1949 &irq->vector, flags);
1950 if (unlikely(irq->res == NULL)) {
1951 rc = ENOMEM;
1952 ena_log(pdev, ERR,
1953 "could not allocate irq vector: %d\n", irq->vector);
1954 goto err;
1955 }
1956
1957 rc = bus_setup_intr(adapter->pdev, irq->res,
1958 INTR_TYPE_NET | INTR_MPSAFE, irq->handler, NULL, irq->data,
1959 &irq->cookie);
1960 if (unlikely(rc != 0)) {
1961 ena_log(pdev, ERR,
1962 "failed to register interrupt handler for irq %ju: %d\n",
1963 rman_get_start(irq->res), rc);
1964 goto err;
1965 }
1966 irq->requested = true;
1967
1968 if (adapter->rss_enabled || adapter->irq_cpu_base > ENA_BASE_CPU_UNSPECIFIED) {
1969 rc = bus_bind_intr(adapter->pdev, irq->res, irq->cpu);
1970 if (unlikely(rc != 0)) {
1971 ena_log(pdev, ERR,
1972 "failed to bind interrupt handler for irq %ju to cpu %d: %d\n",
1973 rman_get_start(irq->res), irq->cpu, rc);
1974 goto err;
1975 }
1976
1977 ena_log(pdev, INFO, "queue %d - cpu %d\n",
1978 i - ENA_IO_IRQ_FIRST_IDX, irq->cpu);
1979 }
1980 }
1981 return (rc);
1982
1983 err:
1984
1985 for (; i >= ENA_IO_IRQ_FIRST_IDX; i--) {
1986 irq = &adapter->irq_tbl[i];
1987 rcc = 0;
1988
1989 /* Once we entered err: section and irq->requested is true we
1990 free both intr and resources */
1991 if (irq->requested) {
1992 rcc = bus_teardown_intr(adapter->pdev, irq->res,
1993 irq->cookie);
1994 if (unlikely(rcc != 0))
1995 ena_log(pdev, ERR,
1996 "could not release irq: %d, error: %d\n",
1997 irq->vector, rcc);
1998 }
1999
2000 /* If we entered err: section without irq->requested set we know
2001 it was bus_alloc_resource_any() that needs cleanup, provided
2002 res is not NULL. In case res is NULL no work in needed in
2003 this iteration */
2004 rcc = 0;
2005 if (irq->res != NULL) {
2006 rcc = bus_release_resource(adapter->pdev, SYS_RES_IRQ,
2007 irq->vector, irq->res);
2008 }
2009 if (unlikely(rcc != 0))
2010 ena_log(pdev, ERR,
2011 "dev has no parent while releasing res for irq: %d\n",
2012 irq->vector);
2013 irq->requested = false;
2014 irq->res = NULL;
2015 }
2016
2017 return (rc);
2018 }
2019
2020 static void
ena_free_mgmnt_irq(struct ena_adapter * adapter)2021 ena_free_mgmnt_irq(struct ena_adapter *adapter)
2022 {
2023 device_t pdev = adapter->pdev;
2024 struct ena_irq *irq;
2025 int rc;
2026
2027 irq = &adapter->irq_tbl[ENA_MGMNT_IRQ_IDX];
2028 if (irq->requested) {
2029 ena_log(pdev, DBG, "tear down irq: %d\n", irq->vector);
2030 rc = bus_teardown_intr(adapter->pdev, irq->res, irq->cookie);
2031 if (unlikely(rc != 0))
2032 ena_log(pdev, ERR, "failed to tear down irq: %d\n",
2033 irq->vector);
2034 irq->requested = 0;
2035 }
2036
2037 if (irq->res != NULL) {
2038 ena_log(pdev, DBG, "release resource irq: %d\n", irq->vector);
2039 rc = bus_release_resource(adapter->pdev, SYS_RES_IRQ,
2040 irq->vector, irq->res);
2041 irq->res = NULL;
2042 if (unlikely(rc != 0))
2043 ena_log(pdev, ERR,
2044 "dev has no parent while releasing res for irq: %d\n",
2045 irq->vector);
2046 }
2047 }
2048
2049 static void
ena_free_io_irq(struct ena_adapter * adapter)2050 ena_free_io_irq(struct ena_adapter *adapter)
2051 {
2052 device_t pdev = adapter->pdev;
2053 struct ena_irq *irq;
2054 int rc;
2055
2056 for (int i = ENA_IO_IRQ_FIRST_IDX; i < adapter->msix_vecs; i++) {
2057 irq = &adapter->irq_tbl[i];
2058 if (irq->requested) {
2059 ena_log(pdev, DBG, "tear down irq: %d\n", irq->vector);
2060 rc = bus_teardown_intr(adapter->pdev, irq->res,
2061 irq->cookie);
2062 if (unlikely(rc != 0)) {
2063 ena_log(pdev, ERR,
2064 "failed to tear down irq: %d\n",
2065 irq->vector);
2066 }
2067 irq->requested = 0;
2068 }
2069
2070 if (irq->res != NULL) {
2071 ena_log(pdev, DBG, "release resource irq: %d\n",
2072 irq->vector);
2073 rc = bus_release_resource(adapter->pdev, SYS_RES_IRQ,
2074 irq->vector, irq->res);
2075 irq->res = NULL;
2076 if (unlikely(rc != 0)) {
2077 ena_log(pdev, ERR,
2078 "dev has no parent while releasing res for irq: %d\n",
2079 irq->vector);
2080 }
2081 }
2082 }
2083 }
2084
2085 static void
ena_free_irqs(struct ena_adapter * adapter)2086 ena_free_irqs(struct ena_adapter *adapter)
2087 {
2088 ena_free_io_irq(adapter);
2089 ena_free_mgmnt_irq(adapter);
2090 ena_disable_msix(adapter);
2091 }
2092
2093 static void
ena_disable_msix(struct ena_adapter * adapter)2094 ena_disable_msix(struct ena_adapter *adapter)
2095 {
2096 if (ENA_FLAG_ISSET(ENA_FLAG_MSIX_ENABLED, adapter)) {
2097 ENA_FLAG_CLEAR_ATOMIC(ENA_FLAG_MSIX_ENABLED, adapter);
2098 pci_release_msi(adapter->pdev);
2099 }
2100
2101 adapter->msix_vecs = 0;
2102 free(adapter->msix_entries, M_DEVBUF);
2103 adapter->msix_entries = NULL;
2104 }
2105
2106 static void
ena_unmask_all_io_irqs(struct ena_adapter * adapter)2107 ena_unmask_all_io_irqs(struct ena_adapter *adapter)
2108 {
2109 struct ena_com_io_cq *io_cq;
2110 struct ena_eth_io_intr_reg intr_reg;
2111 struct ena_ring *tx_ring;
2112 uint16_t ena_qid;
2113 int i;
2114
2115 /* Unmask interrupts for all queues */
2116 for (i = 0; i < adapter->num_io_queues; i++) {
2117 ena_qid = ENA_IO_TXQ_IDX(i);
2118 io_cq = &adapter->ena_dev->io_cq_queues[ena_qid];
2119 ena_com_update_intr_reg(&intr_reg, 0, 0, true, false);
2120 tx_ring = &adapter->tx_ring[i];
2121 counter_u64_add(tx_ring->tx_stats.unmask_interrupt_num, 1);
2122 ena_com_unmask_intr(io_cq, &intr_reg);
2123 }
2124 }
2125
2126 static int
ena_up_complete(struct ena_adapter * adapter)2127 ena_up_complete(struct ena_adapter *adapter)
2128 {
2129 int rc;
2130
2131 if (likely(ENA_FLAG_ISSET(ENA_FLAG_RSS_ACTIVE, adapter))) {
2132 rc = ena_rss_configure(adapter);
2133 if (rc != 0) {
2134 ena_log(adapter->pdev, ERR,
2135 "Failed to configure RSS\n");
2136 return (rc);
2137 }
2138 }
2139
2140 rc = ena_change_mtu(adapter->ifp, if_getmtu(adapter->ifp));
2141 if (unlikely(rc != 0))
2142 return (rc);
2143
2144 ena_refill_all_rx_bufs(adapter);
2145 ena_reset_counters((counter_u64_t *)&adapter->hw_stats,
2146 sizeof(adapter->hw_stats));
2147
2148 return (0);
2149 }
2150
2151 static void
set_io_rings_size(struct ena_adapter * adapter,int new_tx_size,int new_rx_size)2152 set_io_rings_size(struct ena_adapter *adapter, int new_tx_size, int new_rx_size)
2153 {
2154 int i;
2155
2156 for (i = 0; i < adapter->num_io_queues; i++) {
2157 adapter->tx_ring[i].ring_size = new_tx_size;
2158 adapter->rx_ring[i].ring_size = new_rx_size;
2159 }
2160 }
2161
2162 static int
create_queues_with_size_backoff(struct ena_adapter * adapter)2163 create_queues_with_size_backoff(struct ena_adapter *adapter)
2164 {
2165 device_t pdev = adapter->pdev;
2166 int rc;
2167 uint32_t cur_rx_ring_size, cur_tx_ring_size;
2168 uint32_t new_rx_ring_size, new_tx_ring_size;
2169
2170 /*
2171 * Current queue sizes might be set to smaller than the requested
2172 * ones due to past queue allocation failures.
2173 */
2174 set_io_rings_size(adapter, adapter->requested_tx_ring_size,
2175 adapter->requested_rx_ring_size);
2176
2177 while (1) {
2178 /* Allocate transmit descriptors */
2179 rc = ena_setup_all_tx_resources(adapter);
2180 if (unlikely(rc != 0)) {
2181 ena_log(pdev, ERR, "err_setup_tx\n");
2182 goto err_setup_tx;
2183 }
2184
2185 /* Allocate receive descriptors */
2186 rc = ena_setup_all_rx_resources(adapter);
2187 if (unlikely(rc != 0)) {
2188 ena_log(pdev, ERR, "err_setup_rx\n");
2189 goto err_setup_rx;
2190 }
2191
2192 /* Create IO queues for Rx & Tx */
2193 rc = ena_create_io_queues(adapter);
2194 if (unlikely(rc != 0)) {
2195 ena_log(pdev, ERR, "create IO queues failed\n");
2196 goto err_io_que;
2197 }
2198
2199 return (0);
2200
2201 err_io_que:
2202 ena_free_all_rx_resources(adapter);
2203 err_setup_rx:
2204 ena_free_all_tx_resources(adapter);
2205 err_setup_tx:
2206 /*
2207 * Lower the ring size if ENOMEM. Otherwise, return the
2208 * error straightaway.
2209 */
2210 if (unlikely(rc != ENOMEM)) {
2211 ena_log(pdev, ERR,
2212 "Queue creation failed with error code: %d\n", rc);
2213 return (rc);
2214 }
2215
2216 cur_tx_ring_size = adapter->tx_ring[0].ring_size;
2217 cur_rx_ring_size = adapter->rx_ring[0].ring_size;
2218
2219 ena_log(pdev, ERR,
2220 "Not enough memory to create queues with sizes TX=%d, RX=%d\n",
2221 cur_tx_ring_size, cur_rx_ring_size);
2222
2223 new_tx_ring_size = cur_tx_ring_size;
2224 new_rx_ring_size = cur_rx_ring_size;
2225
2226 /*
2227 * Decrease the size of a larger queue, or decrease both if they
2228 * are the same size.
2229 */
2230 if (cur_rx_ring_size <= cur_tx_ring_size)
2231 new_tx_ring_size = cur_tx_ring_size / 2;
2232 if (cur_rx_ring_size >= cur_tx_ring_size)
2233 new_rx_ring_size = cur_rx_ring_size / 2;
2234
2235 if (new_tx_ring_size < ENA_MIN_RING_SIZE ||
2236 new_rx_ring_size < ENA_MIN_RING_SIZE) {
2237 ena_log(pdev, ERR,
2238 "Queue creation failed with the smallest possible queue size"
2239 "of %d for both queues. Not retrying with smaller queues\n",
2240 ENA_MIN_RING_SIZE);
2241 return (rc);
2242 }
2243
2244 ena_log(pdev, INFO,
2245 "Retrying queue creation with sizes TX=%d, RX=%d\n",
2246 new_tx_ring_size, new_rx_ring_size);
2247
2248 set_io_rings_size(adapter, new_tx_ring_size, new_rx_ring_size);
2249 }
2250 }
2251
2252 int
ena_up(struct ena_adapter * adapter)2253 ena_up(struct ena_adapter *adapter)
2254 {
2255 int rc = 0;
2256
2257 ENA_LOCK_ASSERT();
2258
2259 if (unlikely(device_is_attached(adapter->pdev) == 0)) {
2260 ena_log(adapter->pdev, ERR, "device is not attached!\n");
2261 return (ENXIO);
2262 }
2263
2264 if (ENA_FLAG_ISSET(ENA_FLAG_DEV_UP, adapter))
2265 return (0);
2266
2267 ena_log(adapter->pdev, INFO, "device is going UP\n");
2268
2269 /* setup interrupts for IO queues */
2270 rc = ena_setup_io_intr(adapter);
2271 if (unlikely(rc != 0)) {
2272 ena_log(adapter->pdev, ERR, "error setting up IO interrupt\n");
2273 goto error;
2274 }
2275 rc = ena_request_io_irq(adapter);
2276 if (unlikely(rc != 0)) {
2277 ena_log(adapter->pdev, ERR, "err_req_irq\n");
2278 goto error;
2279 }
2280
2281 ena_log(adapter->pdev, INFO,
2282 "Creating %u IO queues. Rx queue size: %d, Tx queue size: %d, LLQ is %s\n",
2283 adapter->num_io_queues,
2284 adapter->requested_rx_ring_size,
2285 adapter->requested_tx_ring_size,
2286 (adapter->ena_dev->tx_mem_queue_type ==
2287 ENA_ADMIN_PLACEMENT_POLICY_DEV) ? "ENABLED" : "DISABLED");
2288
2289 rc = create_queues_with_size_backoff(adapter);
2290 if (unlikely(rc != 0)) {
2291 ena_log(adapter->pdev, ERR,
2292 "error creating queues with size backoff\n");
2293 goto err_create_queues_with_backoff;
2294 }
2295
2296 if (ENA_FLAG_ISSET(ENA_FLAG_LINK_UP, adapter))
2297 if_link_state_change(adapter->ifp, LINK_STATE_UP);
2298
2299 rc = ena_up_complete(adapter);
2300 if (unlikely(rc != 0))
2301 goto err_up_complete;
2302
2303 counter_u64_add(adapter->dev_stats.interface_up, 1);
2304
2305 ena_update_hwassist(adapter);
2306
2307 if_setdrvflagbits(adapter->ifp, IFF_DRV_RUNNING, IFF_DRV_OACTIVE);
2308
2309 ENA_FLAG_SET_ATOMIC(ENA_FLAG_DEV_UP, adapter);
2310
2311 ena_unmask_all_io_irqs(adapter);
2312
2313 return (0);
2314
2315 err_up_complete:
2316 ena_destroy_all_io_queues(adapter);
2317 ena_free_all_rx_resources(adapter);
2318 ena_free_all_tx_resources(adapter);
2319 err_create_queues_with_backoff:
2320 ena_free_io_irq(adapter);
2321 error:
2322 return (rc);
2323 }
2324
2325 static uint64_t
ena_get_counter(if_t ifp,ift_counter cnt)2326 ena_get_counter(if_t ifp, ift_counter cnt)
2327 {
2328 struct ena_adapter *adapter;
2329 struct ena_hw_stats *stats;
2330
2331 adapter = if_getsoftc(ifp);
2332 stats = &adapter->hw_stats;
2333
2334 switch (cnt) {
2335 case IFCOUNTER_IPACKETS:
2336 return (counter_u64_fetch(stats->rx_packets));
2337 case IFCOUNTER_OPACKETS:
2338 return (counter_u64_fetch(stats->tx_packets));
2339 case IFCOUNTER_IBYTES:
2340 return (counter_u64_fetch(stats->rx_bytes));
2341 case IFCOUNTER_OBYTES:
2342 return (counter_u64_fetch(stats->tx_bytes));
2343 case IFCOUNTER_IQDROPS:
2344 return (counter_u64_fetch(stats->rx_drops));
2345 case IFCOUNTER_OQDROPS:
2346 return (counter_u64_fetch(stats->tx_drops));
2347 default:
2348 return (if_get_counter_default(ifp, cnt));
2349 }
2350 }
2351
2352 static int
ena_media_change(if_t ifp)2353 ena_media_change(if_t ifp)
2354 {
2355 /* Media Change is not supported by firmware */
2356 return (0);
2357 }
2358
2359 static void
ena_media_status(if_t ifp,struct ifmediareq * ifmr)2360 ena_media_status(if_t ifp, struct ifmediareq *ifmr)
2361 {
2362 struct ena_adapter *adapter = if_getsoftc(ifp);
2363 ena_log(adapter->pdev, DBG, "Media status update\n");
2364
2365 ENA_LOCK_LOCK();
2366
2367 ifmr->ifm_status = IFM_AVALID;
2368 ifmr->ifm_active = IFM_ETHER;
2369
2370 if (!ENA_FLAG_ISSET(ENA_FLAG_LINK_UP, adapter)) {
2371 ENA_LOCK_UNLOCK();
2372 ena_log(adapter->pdev, INFO, "Link is down\n");
2373 return;
2374 }
2375
2376 ifmr->ifm_status |= IFM_ACTIVE;
2377 ifmr->ifm_active |= IFM_UNKNOWN | IFM_FDX;
2378
2379 ENA_LOCK_UNLOCK();
2380 }
2381
2382 static void
ena_init(void * arg)2383 ena_init(void *arg)
2384 {
2385 struct ena_adapter *adapter = (struct ena_adapter *)arg;
2386
2387 if (!ENA_FLAG_ISSET(ENA_FLAG_DEV_UP, adapter)) {
2388 ENA_LOCK_LOCK();
2389 ena_up(adapter);
2390 ENA_LOCK_UNLOCK();
2391 }
2392 }
2393
2394 static int
ena_ioctl(if_t ifp,u_long command,caddr_t data)2395 ena_ioctl(if_t ifp, u_long command, caddr_t data)
2396 {
2397 struct ena_adapter *adapter;
2398 struct ifreq *ifr;
2399 int rc;
2400
2401 adapter = if_getsoftc(ifp);
2402 ifr = (struct ifreq *)data;
2403
2404 /*
2405 * Acquiring lock to prevent from running up and down routines parallel.
2406 */
2407 rc = 0;
2408 switch (command) {
2409 case SIOCSIFMTU:
2410 if (if_getmtu(ifp) == ifr->ifr_mtu)
2411 break;
2412 ENA_LOCK_LOCK();
2413 ena_down(adapter);
2414
2415 ena_change_mtu(ifp, ifr->ifr_mtu);
2416
2417 rc = ena_up(adapter);
2418 ENA_LOCK_UNLOCK();
2419 break;
2420
2421 case SIOCSIFFLAGS:
2422 if ((if_getflags(ifp) & IFF_UP) != 0) {
2423 if ((if_getdrvflags(ifp) & IFF_DRV_RUNNING) != 0) {
2424 if ((if_getflags(ifp) & (IFF_PROMISC |
2425 IFF_ALLMULTI)) != 0) {
2426 ena_log(adapter->pdev, INFO,
2427 "ioctl promisc/allmulti\n");
2428 }
2429 } else {
2430 ENA_LOCK_LOCK();
2431 rc = ena_up(adapter);
2432 ENA_LOCK_UNLOCK();
2433 }
2434 } else {
2435 if ((if_getdrvflags(ifp) & IFF_DRV_RUNNING) != 0) {
2436 ENA_LOCK_LOCK();
2437 ena_down(adapter);
2438 ENA_LOCK_UNLOCK();
2439 }
2440 }
2441 break;
2442
2443 case SIOCADDMULTI:
2444 case SIOCDELMULTI:
2445 break;
2446
2447 case SIOCSIFMEDIA:
2448 case SIOCGIFMEDIA:
2449 rc = ifmedia_ioctl(ifp, ifr, &adapter->media, command);
2450 break;
2451
2452 case SIOCSIFCAP:
2453 {
2454 int reinit = 0;
2455
2456 if (ifr->ifr_reqcap != if_getcapenable(ifp)) {
2457 if_setcapenable(ifp, ifr->ifr_reqcap);
2458 reinit = 1;
2459 }
2460
2461 if ((reinit != 0) &&
2462 ((if_getdrvflags(ifp) & IFF_DRV_RUNNING) != 0)) {
2463 ENA_LOCK_LOCK();
2464 ena_down(adapter);
2465 rc = ena_up(adapter);
2466 ENA_LOCK_UNLOCK();
2467 }
2468 }
2469
2470 break;
2471 default:
2472 rc = ether_ioctl(ifp, command, data);
2473 break;
2474 }
2475
2476 return (rc);
2477 }
2478
2479 static int
ena_get_dev_offloads(struct ena_com_dev_get_features_ctx * feat)2480 ena_get_dev_offloads(struct ena_com_dev_get_features_ctx *feat)
2481 {
2482 int caps = 0;
2483
2484 if ((feat->offload.tx &
2485 (ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV4_CSUM_FULL_MASK |
2486 ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV4_CSUM_PART_MASK |
2487 ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L3_CSUM_IPV4_MASK)) != 0)
2488 caps |= IFCAP_TXCSUM;
2489
2490 if ((feat->offload.tx &
2491 (ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV6_CSUM_FULL_MASK |
2492 ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV6_CSUM_PART_MASK)) != 0)
2493 caps |= IFCAP_TXCSUM_IPV6;
2494
2495 if ((feat->offload.tx & ENA_ADMIN_FEATURE_OFFLOAD_DESC_TSO_IPV4_MASK) != 0)
2496 caps |= IFCAP_TSO4;
2497
2498 if ((feat->offload.tx & ENA_ADMIN_FEATURE_OFFLOAD_DESC_TSO_IPV6_MASK) != 0)
2499 caps |= IFCAP_TSO6;
2500
2501 if ((feat->offload.rx_supported &
2502 (ENA_ADMIN_FEATURE_OFFLOAD_DESC_RX_L4_IPV4_CSUM_MASK |
2503 ENA_ADMIN_FEATURE_OFFLOAD_DESC_RX_L3_CSUM_IPV4_MASK)) != 0)
2504 caps |= IFCAP_RXCSUM;
2505
2506 if ((feat->offload.rx_supported &
2507 ENA_ADMIN_FEATURE_OFFLOAD_DESC_RX_L4_IPV6_CSUM_MASK) != 0)
2508 caps |= IFCAP_RXCSUM_IPV6;
2509
2510 caps |= IFCAP_LRO | IFCAP_JUMBO_MTU;
2511
2512 return (caps);
2513 }
2514
2515 static void
ena_update_host_info(struct ena_admin_host_info * host_info,if_t ifp)2516 ena_update_host_info(struct ena_admin_host_info *host_info, if_t ifp)
2517 {
2518 host_info->supported_network_features[0] = (uint32_t)if_getcapabilities(ifp);
2519 }
2520
2521 static void
ena_update_hwassist(struct ena_adapter * adapter)2522 ena_update_hwassist(struct ena_adapter *adapter)
2523 {
2524 if_t ifp = adapter->ifp;
2525 uint32_t feat = adapter->tx_offload_cap;
2526 int cap = if_getcapenable(ifp);
2527 int flags = 0;
2528
2529 if_clearhwassist(ifp);
2530
2531 if ((cap & IFCAP_TXCSUM) != 0) {
2532 if ((feat &
2533 ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L3_CSUM_IPV4_MASK) != 0)
2534 flags |= CSUM_IP;
2535 if ((feat &
2536 (ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV4_CSUM_FULL_MASK |
2537 ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV4_CSUM_PART_MASK)) != 0)
2538 flags |= CSUM_IP_UDP | CSUM_IP_TCP;
2539 }
2540
2541 if ((cap & IFCAP_TXCSUM_IPV6) != 0)
2542 flags |= CSUM_IP6_UDP | CSUM_IP6_TCP;
2543
2544 if ((cap & IFCAP_TSO4) != 0)
2545 flags |= CSUM_IP_TSO;
2546
2547 if ((cap & IFCAP_TSO6) != 0)
2548 flags |= CSUM_IP6_TSO;
2549
2550 if_sethwassistbits(ifp, flags, 0);
2551 }
2552
2553 static void
ena_setup_ifnet(device_t pdev,struct ena_adapter * adapter,struct ena_com_dev_get_features_ctx * feat)2554 ena_setup_ifnet(device_t pdev, struct ena_adapter *adapter,
2555 struct ena_com_dev_get_features_ctx *feat)
2556 {
2557 if_t ifp;
2558 int caps = 0;
2559
2560 ifp = adapter->ifp = if_gethandle(IFT_ETHER);
2561 if_initname(ifp, device_get_name(pdev), device_get_unit(pdev));
2562 if_setdev(ifp, pdev);
2563 if_setsoftc(ifp, adapter);
2564
2565 if_setflags(ifp, IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST);
2566 if_setinitfn(ifp, ena_init);
2567 if_settransmitfn(ifp, ena_mq_start);
2568 if_setqflushfn(ifp, ena_qflush);
2569 if_setioctlfn(ifp, ena_ioctl);
2570 if_setgetcounterfn(ifp, ena_get_counter);
2571
2572 if_setsendqlen(ifp, adapter->requested_tx_ring_size);
2573 if_setsendqready(ifp);
2574 if_setmtu(ifp, ETHERMTU);
2575 if_setbaudrate(ifp, 0);
2576 /* Zeroize capabilities... */
2577 if_setcapabilities(ifp, 0);
2578 if_setcapenable(ifp, 0);
2579 /* check hardware support */
2580 caps = ena_get_dev_offloads(feat);
2581 /* ... and set them */
2582 if_setcapabilitiesbit(ifp, caps, 0);
2583
2584 /* TSO parameters */
2585 if_sethwtsomax(ifp, ENA_TSO_MAXSIZE -
2586 (ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN));
2587 if_sethwtsomaxsegcount(ifp, adapter->max_tx_sgl_size - 1);
2588 if_sethwtsomaxsegsize(ifp, ENA_TSO_MAXSIZE);
2589
2590 if_setifheaderlen(ifp, sizeof(struct ether_vlan_header));
2591 if_setcapenable(ifp, if_getcapabilities(ifp));
2592
2593 /*
2594 * Specify the media types supported by this adapter and register
2595 * callbacks to update media and link information
2596 */
2597 ifmedia_init(&adapter->media, IFM_IMASK, ena_media_change,
2598 ena_media_status);
2599 ifmedia_add(&adapter->media, IFM_ETHER | IFM_AUTO, 0, NULL);
2600 ifmedia_set(&adapter->media, IFM_ETHER | IFM_AUTO);
2601
2602 ether_ifattach(ifp, adapter->mac_addr);
2603 }
2604
2605 void
ena_down(struct ena_adapter * adapter)2606 ena_down(struct ena_adapter *adapter)
2607 {
2608 int rc;
2609
2610 ENA_LOCK_ASSERT();
2611
2612 if (!ENA_FLAG_ISSET(ENA_FLAG_DEV_UP, adapter))
2613 return;
2614
2615 ena_log(adapter->pdev, INFO, "device is going DOWN\n");
2616
2617 ENA_FLAG_CLEAR_ATOMIC(ENA_FLAG_DEV_UP, adapter);
2618 if_setdrvflagbits(adapter->ifp, IFF_DRV_OACTIVE, IFF_DRV_RUNNING);
2619
2620 ena_free_io_irq(adapter);
2621
2622 if (ENA_FLAG_ISSET(ENA_FLAG_TRIGGER_RESET, adapter)) {
2623 rc = ena_com_dev_reset(adapter->ena_dev, adapter->reset_reason);
2624 if (unlikely(rc != 0))
2625 ena_log(adapter->pdev, ERR, "Device reset failed\n");
2626 }
2627
2628 ena_destroy_all_io_queues(adapter);
2629
2630 ena_free_all_tx_bufs(adapter);
2631 ena_free_all_rx_bufs(adapter);
2632 ena_free_all_tx_resources(adapter);
2633 ena_free_all_rx_resources(adapter);
2634
2635 counter_u64_add(adapter->dev_stats.interface_down, 1);
2636 }
2637
2638 static uint32_t
ena_calc_max_io_queue_num(device_t pdev,struct ena_com_dev * ena_dev,struct ena_com_dev_get_features_ctx * get_feat_ctx)2639 ena_calc_max_io_queue_num(device_t pdev, struct ena_com_dev *ena_dev,
2640 struct ena_com_dev_get_features_ctx *get_feat_ctx)
2641 {
2642 uint32_t io_tx_sq_num, io_tx_cq_num, io_rx_num, max_num_io_queues;
2643
2644 /* Regular queues capabilities */
2645 if (ena_dev->supported_features & BIT(ENA_ADMIN_MAX_QUEUES_EXT)) {
2646 struct ena_admin_queue_ext_feature_fields *max_queue_ext =
2647 &get_feat_ctx->max_queue_ext.max_queue_ext;
2648 io_rx_num = min_t(int, max_queue_ext->max_rx_sq_num,
2649 max_queue_ext->max_rx_cq_num);
2650
2651 io_tx_sq_num = max_queue_ext->max_tx_sq_num;
2652 io_tx_cq_num = max_queue_ext->max_tx_cq_num;
2653 } else {
2654 struct ena_admin_queue_feature_desc *max_queues =
2655 &get_feat_ctx->max_queues;
2656 io_tx_sq_num = max_queues->max_sq_num;
2657 io_tx_cq_num = max_queues->max_cq_num;
2658 io_rx_num = min_t(int, io_tx_sq_num, io_tx_cq_num);
2659 }
2660
2661 /* In case of LLQ use the llq fields for the tx SQ/CQ */
2662 if (ena_dev->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV)
2663 io_tx_sq_num = get_feat_ctx->llq.max_llq_num;
2664
2665 max_num_io_queues = min_t(uint32_t, mp_ncpus, ENA_MAX_NUM_IO_QUEUES);
2666 max_num_io_queues = min_t(uint32_t, max_num_io_queues, io_rx_num);
2667 max_num_io_queues = min_t(uint32_t, max_num_io_queues, io_tx_sq_num);
2668 max_num_io_queues = min_t(uint32_t, max_num_io_queues, io_tx_cq_num);
2669 /* 1 IRQ for mgmnt and 1 IRQ for each TX/RX pair */
2670 max_num_io_queues = min_t(uint32_t, max_num_io_queues,
2671 pci_msix_count(pdev) - 1);
2672 #ifdef RSS
2673 max_num_io_queues = min_t(uint32_t, max_num_io_queues,
2674 rss_getnumbuckets());
2675 #endif
2676
2677 return (max_num_io_queues);
2678 }
2679
2680 static int
ena_enable_wc(device_t pdev,struct resource * res)2681 ena_enable_wc(device_t pdev, struct resource *res)
2682 {
2683 #if defined(__i386) || defined(__amd64) || defined(__aarch64__)
2684 void *va;
2685 vm_size_t len;
2686 int rc;
2687
2688 va = rman_get_virtual(res);
2689 len = rman_get_size(res);
2690 /* Enable write combining */
2691 rc = pmap_change_attr(va, len, VM_MEMATTR_WRITE_COMBINING);
2692 if (unlikely(rc != 0)) {
2693 ena_log(pdev, ERR, "pmap_change_attr failed, %d\n", rc);
2694 return (rc);
2695 }
2696
2697 return (0);
2698 #endif
2699 return (EOPNOTSUPP);
2700 }
2701
2702 static int
ena_set_queues_placement_policy(device_t pdev,struct ena_com_dev * ena_dev,struct ena_admin_feature_llq_desc * llq,struct ena_llq_configurations * llq_default_configurations)2703 ena_set_queues_placement_policy(device_t pdev, struct ena_com_dev *ena_dev,
2704 struct ena_admin_feature_llq_desc *llq,
2705 struct ena_llq_configurations *llq_default_configurations)
2706 {
2707 int rc;
2708 uint32_t llq_feature_mask;
2709
2710 llq_feature_mask = 1 << ENA_ADMIN_LLQ;
2711 if (!(ena_dev->supported_features & llq_feature_mask)) {
2712 ena_log(pdev, WARN,
2713 "LLQ is not supported. Fallback to host mode policy.\n");
2714 ena_dev->tx_mem_queue_type = ENA_ADMIN_PLACEMENT_POLICY_HOST;
2715 return (0);
2716 }
2717
2718 if (ena_dev->mem_bar == NULL) {
2719 ena_log(pdev, WARN,
2720 "LLQ is advertised as supported but device doesn't expose mem bar.\n");
2721 ena_dev->tx_mem_queue_type = ENA_ADMIN_PLACEMENT_POLICY_HOST;
2722 return (0);
2723 }
2724
2725 rc = ena_com_config_dev_mode(ena_dev, llq, llq_default_configurations);
2726 if (unlikely(rc != 0)) {
2727 ena_log(pdev, WARN,
2728 "Failed to configure the device mode. "
2729 "Fallback to host mode policy.\n");
2730 ena_dev->tx_mem_queue_type = ENA_ADMIN_PLACEMENT_POLICY_HOST;
2731 }
2732
2733 return (0);
2734 }
2735
2736 static int
ena_map_llq_mem_bar(device_t pdev,struct ena_com_dev * ena_dev)2737 ena_map_llq_mem_bar(device_t pdev, struct ena_com_dev *ena_dev)
2738 {
2739 struct ena_adapter *adapter = device_get_softc(pdev);
2740 int rc, rid;
2741
2742 /* Try to allocate resources for LLQ bar */
2743 rid = PCIR_BAR(ENA_MEM_BAR);
2744 adapter->memory = bus_alloc_resource_any(pdev, SYS_RES_MEMORY, &rid,
2745 RF_ACTIVE);
2746 if (unlikely(adapter->memory == NULL)) {
2747 ena_log(pdev, WARN,
2748 "Unable to allocate LLQ bar resource. LLQ mode won't be used.\n");
2749 return (0);
2750 }
2751
2752 /* Enable write combining for better LLQ performance */
2753 rc = ena_enable_wc(adapter->pdev, adapter->memory);
2754 if (unlikely(rc != 0)) {
2755 ena_log(pdev, ERR, "failed to enable write combining.\n");
2756 return (rc);
2757 }
2758
2759 /*
2760 * Save virtual address of the device's memory region
2761 * for the ena_com layer.
2762 */
2763 ena_dev->mem_bar = rman_get_virtual(adapter->memory);
2764
2765 return (0);
2766 }
2767
2768 static inline void
ena_set_llq_configurations(struct ena_llq_configurations * llq_config,struct ena_admin_feature_llq_desc * llq,struct ena_adapter * adapter)2769 ena_set_llq_configurations(struct ena_llq_configurations *llq_config,
2770 struct ena_admin_feature_llq_desc *llq, struct ena_adapter *adapter)
2771 {
2772 bool use_large_llq;
2773
2774 llq_config->llq_header_location = ENA_ADMIN_INLINE_HEADER;
2775 llq_config->llq_stride_ctrl = ENA_ADMIN_MULTIPLE_DESCS_PER_ENTRY;
2776 llq_config->llq_num_decs_before_header =
2777 ENA_ADMIN_LLQ_NUM_DESCS_BEFORE_HEADER_2;
2778
2779 switch (ena_force_large_llq_header) {
2780 case ENA_LLQ_HEADER_SIZE_POLICY_REGULAR:
2781 use_large_llq = false;
2782 break;
2783 case ENA_LLQ_HEADER_SIZE_POLICY_LARGE:
2784 use_large_llq = true;
2785 break;
2786 case ENA_LLQ_HEADER_SIZE_POLICY_DEFAULT:
2787 use_large_llq =
2788 (llq->entry_size_recommended == ENA_ADMIN_LIST_ENTRY_SIZE_256B);
2789 break;
2790 default:
2791 use_large_llq = false;
2792 ena_log(adapter->pdev, WARN,
2793 "force_large_llq_header should have values [0-2]\n");
2794 break;
2795 }
2796
2797 if (!(llq->entry_size_ctrl_supported & ENA_ADMIN_LIST_ENTRY_SIZE_256B))
2798 use_large_llq = false;
2799
2800 if (use_large_llq) {
2801 llq_config->llq_ring_entry_size = ENA_ADMIN_LIST_ENTRY_SIZE_256B;
2802 llq_config->llq_ring_entry_size_value = 256;
2803 adapter->llq_policy = ENA_ADMIN_LIST_ENTRY_SIZE_256B;
2804 } else {
2805 llq_config->llq_ring_entry_size = ENA_ADMIN_LIST_ENTRY_SIZE_128B;
2806 llq_config->llq_ring_entry_size_value = 128;
2807 adapter->llq_policy = ENA_ADMIN_LIST_ENTRY_SIZE_128B;
2808 }
2809 }
2810
2811 static int
ena_calc_io_queue_size(struct ena_calc_queue_size_ctx * ctx,struct ena_adapter * adapter)2812 ena_calc_io_queue_size(struct ena_calc_queue_size_ctx *ctx, struct ena_adapter *adapter)
2813 {
2814 struct ena_admin_feature_llq_desc *llq = &ctx->get_feat_ctx->llq;
2815 struct ena_com_dev *ena_dev = ctx->ena_dev;
2816 uint32_t tx_queue_size = ENA_DEFAULT_RING_SIZE;
2817 uint32_t rx_queue_size = ENA_DEFAULT_RING_SIZE;
2818 uint32_t max_tx_queue_size;
2819 uint32_t max_rx_queue_size;
2820
2821 if (ena_dev->supported_features & BIT(ENA_ADMIN_MAX_QUEUES_EXT)) {
2822 struct ena_admin_queue_ext_feature_fields *max_queue_ext =
2823 &ctx->get_feat_ctx->max_queue_ext.max_queue_ext;
2824 max_rx_queue_size = min_t(uint32_t,
2825 max_queue_ext->max_rx_cq_depth,
2826 max_queue_ext->max_rx_sq_depth);
2827 max_tx_queue_size = max_queue_ext->max_tx_cq_depth;
2828
2829 if (ena_dev->tx_mem_queue_type ==
2830 ENA_ADMIN_PLACEMENT_POLICY_DEV)
2831 max_tx_queue_size = min_t(uint32_t, max_tx_queue_size,
2832 llq->max_llq_depth);
2833 else
2834 max_tx_queue_size = min_t(uint32_t, max_tx_queue_size,
2835 max_queue_ext->max_tx_sq_depth);
2836
2837 ctx->max_tx_sgl_size = min_t(uint16_t, ENA_PKT_MAX_BUFS,
2838 max_queue_ext->max_per_packet_tx_descs);
2839 ctx->max_rx_sgl_size = min_t(uint16_t, ENA_PKT_MAX_BUFS,
2840 max_queue_ext->max_per_packet_rx_descs);
2841 } else {
2842 struct ena_admin_queue_feature_desc *max_queues =
2843 &ctx->get_feat_ctx->max_queues;
2844 max_rx_queue_size = min_t(uint32_t, max_queues->max_cq_depth,
2845 max_queues->max_sq_depth);
2846 max_tx_queue_size = max_queues->max_cq_depth;
2847
2848 if (ena_dev->tx_mem_queue_type ==
2849 ENA_ADMIN_PLACEMENT_POLICY_DEV)
2850 max_tx_queue_size = min_t(uint32_t, max_tx_queue_size,
2851 llq->max_llq_depth);
2852 else
2853 max_tx_queue_size = min_t(uint32_t, max_tx_queue_size,
2854 max_queues->max_sq_depth);
2855
2856 ctx->max_tx_sgl_size = min_t(uint16_t, ENA_PKT_MAX_BUFS,
2857 max_queues->max_packet_tx_descs);
2858 ctx->max_rx_sgl_size = min_t(uint16_t, ENA_PKT_MAX_BUFS,
2859 max_queues->max_packet_rx_descs);
2860 }
2861
2862 if (adapter->llq_policy == ENA_ADMIN_LIST_ENTRY_SIZE_256B) {
2863 if (ena_dev->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV) {
2864 if (llq->max_wide_llq_depth != max_tx_queue_size) {
2865 if (llq->max_wide_llq_depth == 0) {
2866 /* if there is no large llq max depth from device, we divide
2867 * the queue size by 2, leaving the amount of memory
2868 * used by the queues unchanged.
2869 */
2870 max_tx_queue_size /= 2;
2871 } else {
2872 max_tx_queue_size = llq->max_wide_llq_depth;
2873 }
2874 ena_log(ctx->pdev, INFO,
2875 "Using large LLQ headers and decreasing maximum Tx queue size to %d\n",
2876 max_tx_queue_size);
2877 } else {
2878 ena_log(ctx->pdev, INFO, "Using large LLQ headers\n");
2879 }
2880 } else {
2881 ena_log(ctx->pdev, WARN,
2882 "Using large headers failed: LLQ is disabled or device does not support large headers\n");
2883 }
2884 }
2885
2886 /* round down to the nearest power of 2 */
2887 max_tx_queue_size = 1 << (flsl(max_tx_queue_size) - 1);
2888 max_rx_queue_size = 1 << (flsl(max_rx_queue_size) - 1);
2889
2890 tx_queue_size = clamp_val(tx_queue_size, ENA_MIN_RING_SIZE,
2891 max_tx_queue_size);
2892 rx_queue_size = clamp_val(rx_queue_size, ENA_MIN_RING_SIZE,
2893 max_rx_queue_size);
2894
2895 tx_queue_size = 1 << (flsl(tx_queue_size) - 1);
2896 rx_queue_size = 1 << (flsl(rx_queue_size) - 1);
2897
2898 ctx->max_tx_queue_size = max_tx_queue_size;
2899 ctx->max_rx_queue_size = max_rx_queue_size;
2900 ctx->tx_queue_size = tx_queue_size;
2901 ctx->rx_queue_size = rx_queue_size;
2902
2903 return (0);
2904 }
2905
2906 static void
ena_config_host_info(struct ena_com_dev * ena_dev,device_t dev)2907 ena_config_host_info(struct ena_com_dev *ena_dev, device_t dev)
2908 {
2909 struct ena_admin_host_info *host_info;
2910 uintptr_t rid;
2911 int rc;
2912
2913 /* Allocate only the host info */
2914 rc = ena_com_allocate_host_info(ena_dev);
2915 if (unlikely(rc != 0)) {
2916 ena_log(dev, ERR, "Cannot allocate host info\n");
2917 return;
2918 }
2919
2920 host_info = ena_dev->host_attr.host_info;
2921
2922 if (pci_get_id(dev, PCI_ID_RID, &rid) == 0)
2923 host_info->bdf = rid;
2924 host_info->os_type = ENA_ADMIN_OS_FREEBSD;
2925 host_info->kernel_ver = osreldate;
2926
2927 sprintf(host_info->kernel_ver_str, "%d", osreldate);
2928 host_info->os_dist = 0;
2929 strncpy(host_info->os_dist_str, osrelease,
2930 sizeof(host_info->os_dist_str) - 1);
2931
2932 host_info->driver_version = (ENA_DRV_MODULE_VER_MAJOR) |
2933 (ENA_DRV_MODULE_VER_MINOR << ENA_ADMIN_HOST_INFO_MINOR_SHIFT) |
2934 (ENA_DRV_MODULE_VER_SUBMINOR << ENA_ADMIN_HOST_INFO_SUB_MINOR_SHIFT);
2935 host_info->num_cpus = mp_ncpus;
2936 host_info->driver_supported_features =
2937 ENA_ADMIN_HOST_INFO_RX_OFFSET_MASK |
2938 ENA_ADMIN_HOST_INFO_RSS_CONFIGURABLE_FUNCTION_KEY_MASK;
2939
2940 rc = ena_com_set_host_attributes(ena_dev);
2941 if (unlikely(rc != 0)) {
2942 if (rc == EOPNOTSUPP)
2943 ena_log(dev, WARN, "Cannot set host attributes\n");
2944 else
2945 ena_log(dev, ERR, "Cannot set host attributes\n");
2946
2947 goto err;
2948 }
2949
2950 return;
2951
2952 err:
2953 ena_com_delete_host_info(ena_dev);
2954 }
2955
2956 static int
ena_device_init(struct ena_adapter * adapter,device_t pdev,struct ena_com_dev_get_features_ctx * get_feat_ctx,int * wd_active)2957 ena_device_init(struct ena_adapter *adapter, device_t pdev,
2958 struct ena_com_dev_get_features_ctx *get_feat_ctx, int *wd_active)
2959 {
2960 struct ena_llq_configurations llq_config;
2961 struct ena_com_dev *ena_dev = adapter->ena_dev;
2962 bool readless_supported;
2963 uint32_t aenq_groups;
2964 int dma_width;
2965 int rc;
2966
2967 rc = ena_com_mmio_reg_read_request_init(ena_dev);
2968 if (unlikely(rc != 0)) {
2969 ena_log(pdev, ERR, "failed to init mmio read less\n");
2970 return (rc);
2971 }
2972
2973 /*
2974 * The PCIe configuration space revision id indicate if mmio reg
2975 * read is disabled
2976 */
2977 readless_supported = !(pci_get_revid(pdev) & ENA_MMIO_DISABLE_REG_READ);
2978 ena_com_set_mmio_read_mode(ena_dev, readless_supported);
2979
2980 rc = ena_com_dev_reset(ena_dev, ENA_REGS_RESET_NORMAL);
2981 if (unlikely(rc != 0)) {
2982 ena_log(pdev, ERR, "Can not reset device\n");
2983 goto err_mmio_read_less;
2984 }
2985
2986 rc = ena_com_validate_version(ena_dev);
2987 if (unlikely(rc != 0)) {
2988 ena_log(pdev, ERR, "device version is too low\n");
2989 goto err_mmio_read_less;
2990 }
2991
2992 dma_width = ena_com_get_dma_width(ena_dev);
2993 if (unlikely(dma_width < 0)) {
2994 ena_log(pdev, ERR, "Invalid dma width value %d", dma_width);
2995 rc = dma_width;
2996 goto err_mmio_read_less;
2997 }
2998 adapter->dma_width = dma_width;
2999
3000 /* ENA admin level init */
3001 rc = ena_com_admin_init(ena_dev, &aenq_handlers);
3002 if (unlikely(rc != 0)) {
3003 ena_log(pdev, ERR,
3004 "Can not initialize ena admin queue with device\n");
3005 goto err_mmio_read_less;
3006 }
3007
3008 /*
3009 * To enable the msix interrupts the driver needs to know the number
3010 * of queues. So the driver uses polling mode to retrieve this
3011 * information
3012 */
3013 ena_com_set_admin_polling_mode(ena_dev, true);
3014
3015 ena_config_host_info(ena_dev, pdev);
3016
3017 /* Get Device Attributes */
3018 rc = ena_com_get_dev_attr_feat(ena_dev, get_feat_ctx);
3019 if (unlikely(rc != 0)) {
3020 ena_log(pdev, ERR,
3021 "Cannot get attribute for ena device rc: %d\n", rc);
3022 goto err_admin_init;
3023 }
3024
3025 aenq_groups = BIT(ENA_ADMIN_LINK_CHANGE) |
3026 BIT(ENA_ADMIN_FATAL_ERROR) |
3027 BIT(ENA_ADMIN_WARNING) |
3028 BIT(ENA_ADMIN_NOTIFICATION) |
3029 BIT(ENA_ADMIN_KEEP_ALIVE) |
3030 BIT(ENA_ADMIN_CONF_NOTIFICATIONS) |
3031 BIT(ENA_ADMIN_DEVICE_REQUEST_RESET);
3032
3033 aenq_groups &= get_feat_ctx->aenq.supported_groups;
3034 rc = ena_com_set_aenq_config(ena_dev, aenq_groups);
3035 if (unlikely(rc != 0)) {
3036 ena_log(pdev, ERR, "Cannot configure aenq groups rc: %d\n", rc);
3037 goto err_admin_init;
3038 }
3039
3040 *wd_active = !!(aenq_groups & BIT(ENA_ADMIN_KEEP_ALIVE));
3041
3042 ena_set_llq_configurations(&llq_config, &get_feat_ctx->llq, adapter);
3043
3044 rc = ena_set_queues_placement_policy(pdev, ena_dev, &get_feat_ctx->llq,
3045 &llq_config);
3046 if (unlikely(rc != 0)) {
3047 ena_log(pdev, ERR, "Failed to set placement policy\n");
3048 goto err_admin_init;
3049 }
3050
3051 return (0);
3052
3053 err_admin_init:
3054 ena_com_delete_host_info(ena_dev);
3055 ena_com_admin_destroy(ena_dev);
3056 err_mmio_read_less:
3057 ena_com_mmio_reg_read_request_destroy(ena_dev);
3058
3059 return (rc);
3060 }
3061
3062 static int
ena_enable_msix_and_set_admin_interrupts(struct ena_adapter * adapter)3063 ena_enable_msix_and_set_admin_interrupts(struct ena_adapter *adapter)
3064 {
3065 struct ena_com_dev *ena_dev = adapter->ena_dev;
3066 int rc;
3067
3068 rc = ena_enable_msix(adapter);
3069 if (unlikely(rc != 0)) {
3070 ena_log(adapter->pdev, ERR, "Error with MSI-X enablement\n");
3071 return (rc);
3072 }
3073
3074 ena_setup_mgmnt_intr(adapter);
3075
3076 rc = ena_request_mgmnt_irq(adapter);
3077 if (unlikely(rc != 0)) {
3078 ena_log(adapter->pdev, ERR, "Cannot setup mgmnt queue intr\n");
3079 goto err_disable_msix;
3080 }
3081
3082 ena_com_set_admin_polling_mode(ena_dev, false);
3083
3084 ena_com_admin_aenq_enable(ena_dev);
3085
3086 return (0);
3087
3088 err_disable_msix:
3089 ena_disable_msix(adapter);
3090
3091 return (rc);
3092 }
3093
3094 /* Function called on ENA_ADMIN_KEEP_ALIVE event */
3095 static void
ena_keep_alive_wd(void * adapter_data,struct ena_admin_aenq_entry * aenq_e)3096 ena_keep_alive_wd(void *adapter_data, struct ena_admin_aenq_entry *aenq_e)
3097 {
3098 struct ena_adapter *adapter = (struct ena_adapter *)adapter_data;
3099 struct ena_admin_aenq_keep_alive_desc *desc;
3100 sbintime_t stime;
3101 uint64_t rx_drops;
3102 uint64_t tx_drops;
3103 uint64_t rx_overruns;
3104
3105 desc = (struct ena_admin_aenq_keep_alive_desc *)aenq_e;
3106
3107 rx_drops = ((uint64_t)desc->rx_drops_high << 32) | desc->rx_drops_low;
3108 tx_drops = ((uint64_t)desc->tx_drops_high << 32) | desc->tx_drops_low;
3109 rx_overruns = ((uint64_t)desc->rx_overruns_high << 32) | desc->rx_overruns_low;
3110 counter_u64_zero(adapter->hw_stats.rx_drops);
3111 counter_u64_add(adapter->hw_stats.rx_drops, rx_drops);
3112 counter_u64_zero(adapter->hw_stats.tx_drops);
3113 counter_u64_add(adapter->hw_stats.tx_drops, tx_drops);
3114 counter_u64_zero(adapter->hw_stats.rx_overruns);
3115 counter_u64_add(adapter->hw_stats.rx_overruns, rx_overruns);
3116
3117 stime = getsbinuptime();
3118 atomic_store_rel_64(&adapter->keep_alive_timestamp, stime);
3119 }
3120
3121 /* Check for keep alive expiration */
3122 static void
check_for_missing_keep_alive(struct ena_adapter * adapter)3123 check_for_missing_keep_alive(struct ena_adapter *adapter)
3124 {
3125 sbintime_t timestamp, time;
3126 enum ena_regs_reset_reason_types reset_reason = ENA_REGS_RESET_KEEP_ALIVE_TO;
3127
3128 if (adapter->wd_active == 0)
3129 return;
3130
3131 if (adapter->keep_alive_timeout == ENA_HW_HINTS_NO_TIMEOUT)
3132 return;
3133
3134 timestamp = atomic_load_acq_64(&adapter->keep_alive_timestamp);
3135 time = getsbinuptime() - timestamp;
3136 if (unlikely(time > adapter->keep_alive_timeout)) {
3137 ena_log(adapter->pdev, ERR, "Keep alive watchdog timeout.\n");
3138 if (ena_com_aenq_has_keep_alive(adapter->ena_dev))
3139 reset_reason = ENA_REGS_RESET_MISSING_ADMIN_INTERRUPT;
3140
3141 ena_trigger_reset(adapter, reset_reason);
3142 }
3143 }
3144
3145 /* Check if admin queue is enabled */
3146 static void
check_for_admin_com_state(struct ena_adapter * adapter)3147 check_for_admin_com_state(struct ena_adapter *adapter)
3148 {
3149 enum ena_regs_reset_reason_types reset_reason = ENA_REGS_RESET_ADMIN_TO;
3150 if (unlikely(ena_com_get_admin_running_state(adapter->ena_dev) == false)) {
3151 ena_log(adapter->pdev, ERR,
3152 "ENA admin queue is not in running state!\n");
3153 counter_u64_add(adapter->dev_stats.admin_q_pause, 1);
3154 if (ena_com_get_missing_admin_interrupt(adapter->ena_dev))
3155 reset_reason = ENA_REGS_RESET_MISSING_ADMIN_INTERRUPT;
3156
3157 ena_trigger_reset(adapter, reset_reason);
3158 }
3159 }
3160
3161 static int
check_for_rx_interrupt_queue(struct ena_adapter * adapter,struct ena_ring * rx_ring)3162 check_for_rx_interrupt_queue(struct ena_adapter *adapter,
3163 struct ena_ring *rx_ring)
3164 {
3165 if (likely(atomic_load_8(&rx_ring->first_interrupt)))
3166 return (0);
3167
3168 if (ena_com_cq_empty(rx_ring->ena_com_io_cq))
3169 return (0);
3170
3171 rx_ring->no_interrupt_event_cnt++;
3172
3173 if (rx_ring->no_interrupt_event_cnt ==
3174 ENA_MAX_NO_INTERRUPT_ITERATIONS) {
3175 ena_log(adapter->pdev, ERR,
3176 "Potential MSIX issue on Rx side Queue = %d. Reset the device\n",
3177 rx_ring->qid);
3178 ena_trigger_reset(adapter, ENA_REGS_RESET_MISS_INTERRUPT);
3179 return (EIO);
3180 }
3181
3182 return (0);
3183 }
3184
3185 static enum ena_regs_reset_reason_types
check_cdesc_in_tx_cq(struct ena_adapter * adapter,struct ena_ring * tx_ring)3186 check_cdesc_in_tx_cq(struct ena_adapter *adapter,
3187 struct ena_ring *tx_ring)
3188 {
3189 device_t pdev = adapter->pdev;
3190 int rc;
3191 u16 req_id;
3192
3193 rc = ena_com_tx_comp_req_id_get(tx_ring->ena_com_io_cq, &req_id);
3194 /* TX CQ is empty */
3195 if (rc == ENA_COM_TRY_AGAIN) {
3196 ena_log(pdev, ERR,
3197 "No completion descriptors found in CQ %d\n",
3198 tx_ring->qid);
3199 return ENA_REGS_RESET_MISS_TX_CMPL;
3200 }
3201
3202 /* TX CQ has cdescs */
3203 ena_log(pdev, ERR,
3204 "Completion descriptors found in CQ %d",
3205 tx_ring->qid);
3206
3207 return ENA_REGS_RESET_MISS_INTERRUPT;
3208 }
3209
3210 static int
check_missing_comp_in_tx_queue(struct ena_adapter * adapter,struct ena_ring * tx_ring)3211 check_missing_comp_in_tx_queue(struct ena_adapter *adapter,
3212 struct ena_ring *tx_ring)
3213 {
3214 uint32_t missed_tx = 0, new_missed_tx = 0;
3215 device_t pdev = adapter->pdev;
3216 sbintime_t curtime;
3217 struct ena_tx_buffer *tx_buf;
3218 int time_since_last_cleanup;
3219 int missing_tx_comp_to;
3220 sbintime_t time_offset;
3221 int i, rc = 0;
3222 enum ena_regs_reset_reason_types reset_reason = ENA_REGS_RESET_MISS_TX_CMPL;
3223 bool cleanup_scheduled, cleanup_running;
3224
3225 curtime = getsbinuptime();
3226
3227 for (i = 0; i < tx_ring->ring_size; i++) {
3228 sbintime_t ts;
3229
3230 tx_buf = &tx_ring->tx_buffer_info[i];
3231
3232 ts = atomic_load_64(&tx_buf->timestamp);
3233 if (ts == 0)
3234 continue;
3235
3236 time_offset = curtime - ts;
3237
3238 if (unlikely(!atomic_load_8(&tx_ring->first_interrupt) &&
3239 time_offset > 2 * adapter->missing_tx_timeout)) {
3240 /*
3241 * If after graceful period interrupt is still not
3242 * received, we schedule a reset.
3243 */
3244 ena_log(pdev, ERR,
3245 "Potential MSIX issue on Tx side Queue = %d. "
3246 "Reset the device\n",
3247 tx_ring->qid);
3248 ena_trigger_reset(adapter,
3249 ENA_REGS_RESET_MISS_INTERRUPT);
3250 return (EIO);
3251 }
3252
3253 /* Check again if packet is still waiting */
3254 if (unlikely(time_offset > adapter->missing_tx_timeout)) {
3255
3256 if (tx_buf->print_once) {
3257 time_since_last_cleanup = TICKS_2_MSEC(ticks -
3258 tx_ring->tx_last_cleanup_ticks);
3259 missing_tx_comp_to = sbttoms(
3260 adapter->missing_tx_timeout);
3261 ena_log(pdev, WARN,
3262 "Found a Tx that wasn't completed on time, qid %d, index %d. "
3263 "%d msecs have passed since last cleanup. Missing Tx timeout value %d msecs.\n",
3264 tx_ring->qid, i, time_since_last_cleanup,
3265 missing_tx_comp_to);
3266 /* Add new TX completions which are missed */
3267 new_missed_tx++;
3268 }
3269
3270 tx_buf->print_once = false;
3271 missed_tx++;
3272 }
3273 }
3274 /* Checking if this TX ring missing TX completions have passed the threshold */
3275 if (unlikely(missed_tx > adapter->missing_tx_threshold)) {
3276 ena_log(pdev, ERR,
3277 "The number of lost tx completion is above the threshold "
3278 "(%d > %d). Reset the device\n",
3279 missed_tx, adapter->missing_tx_threshold);
3280 /* Set the reset flag to prevent ena_cleanup() from running */
3281 ENA_FLAG_SET_ATOMIC(ENA_FLAG_TRIGGER_RESET, adapter);
3282 /* Need to make sure that ENA_FLAG_TRIGGER_RESET is visible to ena_cleanup() and
3283 * that cleanup_running is visible to check_missing_comp_in_tx_queue() to
3284 * prevent the case of accessing CQ concurrently with check_cdesc_in_tx_cq()
3285 */
3286 mb();
3287 cleanup_scheduled = !!(atomic_load_16(&tx_ring->que->cleanup_task.ta_pending));
3288 cleanup_running = !!(atomic_load_8((&tx_ring->cleanup_running)));
3289 if (!(cleanup_scheduled || cleanup_running))
3290 reset_reason = check_cdesc_in_tx_cq(adapter, tx_ring);
3291
3292 adapter->reset_reason = reset_reason;
3293 rc = EIO;
3294 }
3295 /* Add the newly discovered missing TX completions */
3296 counter_u64_add(tx_ring->tx_stats.missing_tx_comp, new_missed_tx);
3297
3298 return (rc);
3299 }
3300
3301 /*
3302 * Check for TX which were not completed on time.
3303 * Timeout is defined by "missing_tx_timeout".
3304 * Reset will be performed if number of incompleted
3305 * transactions exceeds "missing_tx_threshold".
3306 */
3307 static void
check_for_missing_completions(struct ena_adapter * adapter)3308 check_for_missing_completions(struct ena_adapter *adapter)
3309 {
3310 struct ena_ring *tx_ring;
3311 struct ena_ring *rx_ring;
3312 int i, budget, rc;
3313
3314 /* Make sure the driver doesn't turn the device in other process */
3315 rmb();
3316
3317 if (!ENA_FLAG_ISSET(ENA_FLAG_DEV_UP, adapter))
3318 return;
3319
3320 if (ENA_FLAG_ISSET(ENA_FLAG_TRIGGER_RESET, adapter))
3321 return;
3322
3323 if (adapter->missing_tx_timeout == ENA_HW_HINTS_NO_TIMEOUT)
3324 return;
3325
3326 budget = adapter->missing_tx_max_queues;
3327
3328 for (i = adapter->next_monitored_tx_qid; i < adapter->num_io_queues; i++) {
3329 tx_ring = &adapter->tx_ring[i];
3330 rx_ring = &adapter->rx_ring[i];
3331
3332 rc = check_missing_comp_in_tx_queue(adapter, tx_ring);
3333 if (unlikely(rc != 0))
3334 return;
3335
3336 rc = check_for_rx_interrupt_queue(adapter, rx_ring);
3337 if (unlikely(rc != 0))
3338 return;
3339
3340 budget--;
3341 if (budget == 0) {
3342 i++;
3343 break;
3344 }
3345 }
3346
3347 adapter->next_monitored_tx_qid = i % adapter->num_io_queues;
3348 }
3349
3350 /* trigger rx cleanup after 2 consecutive detections */
3351 #define EMPTY_RX_REFILL 2
3352 /* For the rare case where the device runs out of Rx descriptors and the
3353 * msix handler failed to refill new Rx descriptors (due to a lack of memory
3354 * for example).
3355 * This case will lead to a deadlock:
3356 * The device won't send interrupts since all the new Rx packets will be dropped
3357 * The msix handler won't allocate new Rx descriptors so the device won't be
3358 * able to send new packets.
3359 *
3360 * When such a situation is detected - execute rx cleanup task in another thread
3361 */
3362 static void
check_for_empty_rx_ring(struct ena_adapter * adapter)3363 check_for_empty_rx_ring(struct ena_adapter *adapter)
3364 {
3365 struct ena_ring *rx_ring;
3366 int i, refill_required;
3367
3368 if (!ENA_FLAG_ISSET(ENA_FLAG_DEV_UP, adapter))
3369 return;
3370
3371 if (ENA_FLAG_ISSET(ENA_FLAG_TRIGGER_RESET, adapter))
3372 return;
3373
3374 for (i = 0; i < adapter->num_io_queues; i++) {
3375 rx_ring = &adapter->rx_ring[i];
3376
3377 refill_required = ena_com_free_q_entries(
3378 rx_ring->ena_com_io_sq);
3379 if (unlikely(refill_required == (rx_ring->ring_size - 1))) {
3380 rx_ring->empty_rx_queue++;
3381
3382 if (rx_ring->empty_rx_queue >= EMPTY_RX_REFILL) {
3383 counter_u64_add(rx_ring->rx_stats.empty_rx_ring,
3384 1);
3385
3386 ena_log(adapter->pdev, WARN,
3387 "Rx ring %d is stalled. Triggering the refill function\n",
3388 i);
3389
3390 taskqueue_enqueue(rx_ring->que->cleanup_tq,
3391 &rx_ring->que->cleanup_task);
3392 rx_ring->empty_rx_queue = 0;
3393 }
3394 } else {
3395 rx_ring->empty_rx_queue = 0;
3396 }
3397 }
3398 }
3399
3400 static void
ena_update_hints(struct ena_adapter * adapter,struct ena_admin_ena_hw_hints * hints)3401 ena_update_hints(struct ena_adapter *adapter,
3402 struct ena_admin_ena_hw_hints *hints)
3403 {
3404 struct ena_com_dev *ena_dev = adapter->ena_dev;
3405
3406 if (hints->admin_completion_tx_timeout)
3407 ena_dev->admin_queue.completion_timeout =
3408 hints->admin_completion_tx_timeout * 1000;
3409
3410 if (hints->mmio_read_timeout)
3411 /* convert to usec */
3412 ena_dev->mmio_read.reg_read_to = hints->mmio_read_timeout * 1000;
3413
3414 if (hints->missed_tx_completion_count_threshold_to_reset)
3415 adapter->missing_tx_threshold =
3416 hints->missed_tx_completion_count_threshold_to_reset;
3417
3418 if (hints->missing_tx_completion_timeout) {
3419 if (hints->missing_tx_completion_timeout ==
3420 ENA_HW_HINTS_NO_TIMEOUT)
3421 adapter->missing_tx_timeout = ENA_HW_HINTS_NO_TIMEOUT;
3422 else
3423 adapter->missing_tx_timeout = SBT_1MS *
3424 hints->missing_tx_completion_timeout;
3425 }
3426
3427 if (hints->driver_watchdog_timeout) {
3428 if (hints->driver_watchdog_timeout == ENA_HW_HINTS_NO_TIMEOUT)
3429 adapter->keep_alive_timeout = ENA_HW_HINTS_NO_TIMEOUT;
3430 else
3431 adapter->keep_alive_timeout = SBT_1MS *
3432 hints->driver_watchdog_timeout;
3433 }
3434 }
3435
3436 /**
3437 * ena_copy_eni_metrics - Get and copy ENI metrics from the HW.
3438 * @adapter: ENA device adapter
3439 *
3440 * Returns 0 on success, EOPNOTSUPP if current HW doesn't support those metrics
3441 * and other error codes on failure.
3442 *
3443 * This function can possibly cause a race with other calls to the admin queue.
3444 * Because of that, the caller should either lock this function or make sure
3445 * that there is no race in the current context.
3446 */
3447 static int
ena_copy_eni_metrics(struct ena_adapter * adapter)3448 ena_copy_eni_metrics(struct ena_adapter *adapter)
3449 {
3450 static bool print_once = true;
3451 int rc;
3452
3453 rc = ena_com_get_eni_stats(adapter->ena_dev, &adapter->eni_metrics);
3454
3455 if (rc != 0) {
3456 if (rc == ENA_COM_UNSUPPORTED) {
3457 if (print_once) {
3458 ena_log(adapter->pdev, WARN,
3459 "Retrieving ENI metrics is not supported.\n");
3460 print_once = false;
3461 } else {
3462 ena_log(adapter->pdev, DBG,
3463 "Retrieving ENI metrics is not supported.\n");
3464 }
3465 } else {
3466 ena_log(adapter->pdev, ERR,
3467 "Failed to get ENI metrics: %d\n", rc);
3468 }
3469 }
3470
3471 return (rc);
3472 }
3473
3474 static int
ena_copy_srd_metrics(struct ena_adapter * adapter)3475 ena_copy_srd_metrics(struct ena_adapter *adapter)
3476 {
3477 return ena_com_get_ena_srd_info(adapter->ena_dev, &adapter->ena_srd_info);
3478 }
3479
3480 static int
ena_copy_customer_metrics(struct ena_adapter * adapter)3481 ena_copy_customer_metrics(struct ena_adapter *adapter)
3482 {
3483 struct ena_com_dev *dev;
3484 u32 supported_metrics_count;
3485 int rc, len;
3486
3487 dev = adapter->ena_dev;
3488
3489 supported_metrics_count = ena_com_get_customer_metric_count(dev);
3490 len = supported_metrics_count * sizeof(u64);
3491
3492 /* Fill the data buffer */
3493 rc = ena_com_get_customer_metrics(adapter->ena_dev,
3494 (char *)(adapter->customer_metrics_array), len);
3495
3496 return (rc);
3497 }
3498
3499 static void
ena_timer_service(void * data)3500 ena_timer_service(void *data)
3501 {
3502 struct ena_adapter *adapter = (struct ena_adapter *)data;
3503 struct ena_admin_host_info *host_info =
3504 adapter->ena_dev->host_attr.host_info;
3505
3506 check_for_missing_keep_alive(adapter);
3507
3508 check_for_admin_com_state(adapter);
3509
3510 check_for_missing_completions(adapter);
3511
3512 check_for_empty_rx_ring(adapter);
3513
3514 /*
3515 * User controller update of the ENA metrics.
3516 * If the delay was set to 0, then the stats shouldn't be updated at
3517 * all.
3518 * Otherwise, wait 'metrics_sample_interval' seconds, before
3519 * updating stats.
3520 * As timer service is executed every second, it's enough to increment
3521 * appropriate counter each time the timer service is executed.
3522 */
3523 if ((adapter->metrics_sample_interval != 0) &&
3524 (++adapter->metrics_sample_interval_cnt >=
3525 adapter->metrics_sample_interval)) {
3526 taskqueue_enqueue(adapter->metrics_tq, &adapter->metrics_task);
3527 adapter->metrics_sample_interval_cnt = 0;
3528 }
3529
3530
3531 if (host_info != NULL)
3532 ena_update_host_info(host_info, adapter->ifp);
3533
3534 if (unlikely(ENA_FLAG_ISSET(ENA_FLAG_TRIGGER_RESET, adapter))) {
3535 /*
3536 * Timeout when validating version indicates that the device
3537 * became unresponsive. If that happens skip the reset and
3538 * reschedule timer service, so the reset can be retried later.
3539 */
3540 if (ena_com_validate_version(adapter->ena_dev) ==
3541 ENA_COM_TIMER_EXPIRED) {
3542 ena_log(adapter->pdev, WARN,
3543 "FW unresponsive, skipping reset\n");
3544 ENA_TIMER_RESET(adapter);
3545 return;
3546 }
3547 ena_log(adapter->pdev, WARN, "Trigger reset is on\n");
3548 taskqueue_enqueue(adapter->reset_tq, &adapter->reset_task);
3549 return;
3550 }
3551
3552 /*
3553 * Schedule another timeout one second from now.
3554 */
3555 ENA_TIMER_RESET(adapter);
3556 }
3557
3558 void
ena_destroy_device(struct ena_adapter * adapter,bool graceful)3559 ena_destroy_device(struct ena_adapter *adapter, bool graceful)
3560 {
3561 if_t ifp = adapter->ifp;
3562 struct ena_com_dev *ena_dev = adapter->ena_dev;
3563 bool dev_up;
3564
3565 if (!ENA_FLAG_ISSET(ENA_FLAG_DEVICE_RUNNING, adapter))
3566 return;
3567
3568 if (!graceful)
3569 if_link_state_change(ifp, LINK_STATE_DOWN);
3570
3571 ENA_TIMER_DRAIN(adapter);
3572
3573 dev_up = ENA_FLAG_ISSET(ENA_FLAG_DEV_UP, adapter);
3574 if (dev_up)
3575 ENA_FLAG_SET_ATOMIC(ENA_FLAG_DEV_UP_BEFORE_RESET, adapter);
3576
3577 if (!graceful)
3578 ena_com_set_admin_running_state(ena_dev, false);
3579
3580 if (ENA_FLAG_ISSET(ENA_FLAG_DEV_UP, adapter))
3581 ena_down(adapter);
3582
3583 /*
3584 * Stop the device from sending AENQ events (if the device was up, and
3585 * the trigger reset was on, ena_down already performs device reset)
3586 */
3587 if (!(ENA_FLAG_ISSET(ENA_FLAG_TRIGGER_RESET, adapter) && dev_up))
3588 ena_com_dev_reset(adapter->ena_dev, adapter->reset_reason);
3589
3590 ena_free_mgmnt_irq(adapter);
3591
3592 ena_disable_msix(adapter);
3593
3594 /*
3595 * IO rings resources should be freed because `ena_restore_device()`
3596 * calls (not directly) `ena_enable_msix()`, which re-allocates MSIX
3597 * vectors. The amount of MSIX vectors after destroy-restore may be
3598 * different than before. Therefore, IO rings resources should be
3599 * established from scratch each time.
3600 */
3601 ena_free_all_io_rings_resources(adapter);
3602
3603 ena_com_abort_admin_commands(ena_dev);
3604
3605 ena_com_wait_for_abort_completion(ena_dev);
3606
3607 ena_com_admin_destroy(ena_dev);
3608
3609 ena_com_mmio_reg_read_request_destroy(ena_dev);
3610
3611 adapter->reset_reason = ENA_REGS_RESET_NORMAL;
3612
3613 ENA_FLAG_CLEAR_ATOMIC(ENA_FLAG_TRIGGER_RESET, adapter);
3614 ENA_FLAG_CLEAR_ATOMIC(ENA_FLAG_DEVICE_RUNNING, adapter);
3615 }
3616
3617 static int
ena_device_validate_params(struct ena_adapter * adapter,struct ena_com_dev_get_features_ctx * get_feat_ctx)3618 ena_device_validate_params(struct ena_adapter *adapter,
3619 struct ena_com_dev_get_features_ctx *get_feat_ctx)
3620 {
3621 if (memcmp(get_feat_ctx->dev_attr.mac_addr, adapter->mac_addr,
3622 ETHER_ADDR_LEN) != 0) {
3623 ena_log(adapter->pdev, ERR, "Error, mac addresses differ\n");
3624 return (EINVAL);
3625 }
3626
3627 if (get_feat_ctx->dev_attr.max_mtu < if_getmtu(adapter->ifp)) {
3628 ena_log(adapter->pdev, ERR,
3629 "Error, device max mtu is smaller than ifp MTU\n");
3630 return (EINVAL);
3631 }
3632
3633 return 0;
3634 }
3635
3636 int
ena_restore_device(struct ena_adapter * adapter)3637 ena_restore_device(struct ena_adapter *adapter)
3638 {
3639 struct ena_com_dev_get_features_ctx get_feat_ctx;
3640 struct ena_com_dev *ena_dev = adapter->ena_dev;
3641 if_t ifp = adapter->ifp;
3642 device_t dev = adapter->pdev;
3643 int wd_active;
3644 int rc;
3645
3646 ENA_FLAG_SET_ATOMIC(ENA_FLAG_ONGOING_RESET, adapter);
3647
3648 rc = ena_device_init(adapter, dev, &get_feat_ctx, &wd_active);
3649 if (rc != 0) {
3650 ena_log(dev, ERR, "Cannot initialize device\n");
3651 goto err;
3652 }
3653 /*
3654 * Only enable WD if it was enabled before reset, so it won't override
3655 * value set by the user by the sysctl.
3656 */
3657 if (adapter->wd_active != 0)
3658 adapter->wd_active = wd_active;
3659
3660 rc = ena_device_validate_params(adapter, &get_feat_ctx);
3661 if (rc != 0) {
3662 ena_log(dev, ERR, "Validation of device parameters failed\n");
3663 goto err_device_destroy;
3664 }
3665
3666 ENA_FLAG_CLEAR_ATOMIC(ENA_FLAG_ONGOING_RESET, adapter);
3667 /* Make sure we don't have a race with AENQ Links state handler */
3668 if (ENA_FLAG_ISSET(ENA_FLAG_LINK_UP, adapter))
3669 if_link_state_change(ifp, LINK_STATE_UP);
3670
3671 rc = ena_enable_msix_and_set_admin_interrupts(adapter);
3672 if (rc != 0) {
3673 ena_log(dev, ERR, "Enable MSI-X failed\n");
3674 goto err_device_destroy;
3675 }
3676
3677 /*
3678 * Effective value of used MSIX vectors should be the same as before
3679 * `ena_destroy_device()`, if possible, or closest to it if less vectors
3680 * are available.
3681 */
3682 if ((adapter->msix_vecs - ENA_ADMIN_MSIX_VEC) < adapter->num_io_queues)
3683 adapter->num_io_queues = adapter->msix_vecs - ENA_ADMIN_MSIX_VEC;
3684
3685 /* Re-initialize rings basic information */
3686 ena_init_io_rings(adapter);
3687
3688 /* If the interface was up before the reset bring it up */
3689 if (ENA_FLAG_ISSET(ENA_FLAG_DEV_UP_BEFORE_RESET, adapter)) {
3690 rc = ena_up(adapter);
3691 if (rc != 0) {
3692 ena_log(dev, ERR, "Failed to create I/O queues\n");
3693 goto err_disable_msix;
3694 }
3695 }
3696
3697 /* Indicate that device is running again and ready to work */
3698 ENA_FLAG_SET_ATOMIC(ENA_FLAG_DEVICE_RUNNING, adapter);
3699
3700 /*
3701 * As the AENQ handlers weren't executed during reset because
3702 * the flag ENA_FLAG_DEVICE_RUNNING was turned off, the
3703 * timestamp must be updated again That will prevent next reset
3704 * caused by missing keep alive.
3705 */
3706 adapter->keep_alive_timestamp = getsbinuptime();
3707 ENA_TIMER_RESET(adapter);
3708
3709 ENA_FLAG_CLEAR_ATOMIC(ENA_FLAG_DEV_UP_BEFORE_RESET, adapter);
3710
3711 return (rc);
3712
3713 err_disable_msix:
3714 ena_free_mgmnt_irq(adapter);
3715 ena_disable_msix(adapter);
3716 err_device_destroy:
3717 ena_com_abort_admin_commands(ena_dev);
3718 ena_com_wait_for_abort_completion(ena_dev);
3719 ena_com_admin_destroy(ena_dev);
3720 ena_com_dev_reset(ena_dev, ENA_REGS_RESET_DRIVER_INVALID_STATE);
3721 ena_com_mmio_reg_read_request_destroy(ena_dev);
3722 err:
3723 ENA_FLAG_CLEAR_ATOMIC(ENA_FLAG_DEVICE_RUNNING, adapter);
3724 ENA_FLAG_CLEAR_ATOMIC(ENA_FLAG_ONGOING_RESET, adapter);
3725 ena_log(dev, ERR, "Reset attempt failed. Can not reset the device\n");
3726
3727 return (rc);
3728 }
3729
3730 static void
ena_metrics_task(void * arg,int pending)3731 ena_metrics_task(void *arg, int pending)
3732 {
3733 struct ena_adapter *adapter = (struct ena_adapter *)arg;
3734
3735 ENA_LOCK_LOCK();
3736
3737 if (ena_com_get_cap(adapter->ena_dev, ENA_ADMIN_CUSTOMER_METRICS))
3738 (void)ena_copy_customer_metrics(adapter);
3739 else if (ena_com_get_cap(adapter->ena_dev, ENA_ADMIN_ENI_STATS))
3740 (void)ena_copy_eni_metrics(adapter);
3741
3742 if (ena_com_get_cap(adapter->ena_dev, ENA_ADMIN_ENA_SRD_INFO))
3743 (void)ena_copy_srd_metrics(adapter);
3744
3745 ENA_LOCK_UNLOCK();
3746 }
3747
3748 static void
ena_reset_task(void * arg,int pending)3749 ena_reset_task(void *arg, int pending)
3750 {
3751 struct ena_adapter *adapter = (struct ena_adapter *)arg;
3752
3753 ENA_LOCK_LOCK();
3754 if (likely(ENA_FLAG_ISSET(ENA_FLAG_TRIGGER_RESET, adapter))) {
3755 ena_increment_reset_counter(adapter);
3756 ena_destroy_device(adapter, false);
3757 ena_restore_device(adapter);
3758
3759 ena_log(adapter->pdev, INFO,
3760 "Device reset completed successfully, Driver info: %s\n",
3761 ena_version);
3762 }
3763 ENA_LOCK_UNLOCK();
3764 }
3765
3766 static void
ena_free_stats(struct ena_adapter * adapter)3767 ena_free_stats(struct ena_adapter *adapter)
3768 {
3769 ena_free_counters((counter_u64_t *)&adapter->hw_stats,
3770 sizeof(struct ena_hw_stats));
3771 ena_free_counters((counter_u64_t *)&adapter->dev_stats,
3772 sizeof(struct ena_stats_dev));
3773
3774 }
3775 /**
3776 * ena_attach - Device Initialization Routine
3777 * @pdev: device information struct
3778 *
3779 * Returns 0 on success, otherwise on failure.
3780 *
3781 * ena_attach initializes an adapter identified by a device structure.
3782 * The OS initialization, configuring of the adapter private structure,
3783 * and a hardware reset occur.
3784 **/
3785 static int
ena_attach(device_t pdev)3786 ena_attach(device_t pdev)
3787 {
3788 struct ena_com_dev_get_features_ctx get_feat_ctx;
3789 struct ena_calc_queue_size_ctx calc_queue_ctx = { 0 };
3790 static int version_printed;
3791 struct ena_adapter *adapter;
3792 struct ena_com_dev *ena_dev = NULL;
3793 uint32_t max_num_io_queues;
3794 int msix_rid;
3795 int rid, rc;
3796
3797 adapter = device_get_softc(pdev);
3798 adapter->pdev = pdev;
3799 adapter->first_bind = -1;
3800
3801 /*
3802 * Set up the timer service - driver is responsible for avoiding
3803 * concurrency, as the callout won't be using any locking inside.
3804 */
3805 ENA_TIMER_INIT(adapter);
3806 adapter->keep_alive_timeout = ENA_DEFAULT_KEEP_ALIVE_TO;
3807 adapter->missing_tx_timeout = ENA_DEFAULT_TX_CMP_TO;
3808 adapter->missing_tx_max_queues = ENA_DEFAULT_TX_MONITORED_QUEUES;
3809 adapter->missing_tx_threshold = ENA_DEFAULT_TX_CMP_THRESHOLD;
3810
3811 adapter->irq_cpu_base = ENA_BASE_CPU_UNSPECIFIED;
3812 adapter->irq_cpu_stride = 0;
3813
3814 #ifdef RSS
3815 adapter->rss_enabled = 1;
3816 #endif
3817
3818 if (version_printed++ == 0)
3819 ena_log(pdev, INFO, "%s\n", ena_version);
3820
3821 /* Allocate memory for ena_dev structure */
3822 ena_dev = malloc(sizeof(struct ena_com_dev), M_DEVBUF,
3823 M_WAITOK | M_ZERO);
3824
3825 adapter->ena_dev = ena_dev;
3826 ena_dev->dmadev = pdev;
3827
3828 rid = PCIR_BAR(ENA_REG_BAR);
3829 adapter->memory = NULL;
3830 adapter->registers = bus_alloc_resource_any(pdev, SYS_RES_MEMORY, &rid,
3831 RF_ACTIVE);
3832 if (unlikely(adapter->registers == NULL)) {
3833 ena_log(pdev, ERR,
3834 "unable to allocate bus resource: registers!\n");
3835 rc = ENOMEM;
3836 goto err_dev_free;
3837 }
3838
3839 /* MSIx vector table may reside on BAR0 with registers or on BAR1. */
3840 msix_rid = pci_msix_table_bar(pdev);
3841 if (msix_rid != rid) {
3842 adapter->msix = bus_alloc_resource_any(pdev, SYS_RES_MEMORY,
3843 &msix_rid, RF_ACTIVE);
3844 if (unlikely(adapter->msix == NULL)) {
3845 ena_log(pdev, ERR,
3846 "unable to allocate bus resource: msix!\n");
3847 rc = ENOMEM;
3848 goto err_pci_free;
3849 }
3850 adapter->msix_rid = msix_rid;
3851 }
3852
3853 ena_dev->bus = malloc(sizeof(struct ena_bus), M_DEVBUF,
3854 M_WAITOK | M_ZERO);
3855
3856 /* Store register resources */
3857 ((struct ena_bus *)(ena_dev->bus))->reg_bar_t = rman_get_bustag(
3858 adapter->registers);
3859 ((struct ena_bus *)(ena_dev->bus))->reg_bar_h = rman_get_bushandle(
3860 adapter->registers);
3861
3862 if (unlikely(((struct ena_bus *)(ena_dev->bus))->reg_bar_h == 0)) {
3863 ena_log(pdev, ERR, "failed to pmap registers bar\n");
3864 rc = ENXIO;
3865 goto err_bus_free;
3866 }
3867
3868 rc = ena_map_llq_mem_bar(pdev, ena_dev);
3869 if (unlikely(rc != 0)) {
3870 ena_log(pdev, ERR, "Failed to map ENA mem bar");
3871 goto err_bus_free;
3872 }
3873
3874 ena_dev->ena_min_poll_delay_us = ENA_ADMIN_POLL_DELAY_US;
3875
3876 /* Initially clear all the flags */
3877 ENA_FLAG_ZERO(adapter);
3878
3879 /* Device initialization */
3880 rc = ena_device_init(adapter, pdev, &get_feat_ctx, &adapter->wd_active);
3881 if (unlikely(rc != 0)) {
3882 ena_log(pdev, ERR, "ENA device init failed! (err: %d)\n", rc);
3883 rc = ENXIO;
3884 goto err_bus_free;
3885 }
3886
3887 if (ena_dev->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV)
3888 adapter->disable_meta_caching = !!(
3889 get_feat_ctx.llq.accel_mode.u.get.supported_flags &
3890 BIT(ENA_ADMIN_DISABLE_META_CACHING));
3891
3892 adapter->keep_alive_timestamp = getsbinuptime();
3893
3894 adapter->tx_offload_cap = get_feat_ctx.offload.tx;
3895
3896 memcpy(adapter->mac_addr, get_feat_ctx.dev_attr.mac_addr,
3897 ETHER_ADDR_LEN);
3898
3899 calc_queue_ctx.pdev = pdev;
3900 calc_queue_ctx.ena_dev = ena_dev;
3901 calc_queue_ctx.get_feat_ctx = &get_feat_ctx;
3902
3903 /* Calculate initial and maximum IO queue number and size */
3904 max_num_io_queues = ena_calc_max_io_queue_num(pdev, ena_dev,
3905 &get_feat_ctx);
3906 rc = ena_calc_io_queue_size(&calc_queue_ctx, adapter);
3907 if (unlikely((rc != 0) || (max_num_io_queues <= 0))) {
3908 rc = EFAULT;
3909 goto err_com_free;
3910 }
3911
3912 adapter->requested_tx_ring_size = calc_queue_ctx.tx_queue_size;
3913 adapter->requested_rx_ring_size = calc_queue_ctx.rx_queue_size;
3914 adapter->max_tx_ring_size = calc_queue_ctx.max_tx_queue_size;
3915 adapter->max_rx_ring_size = calc_queue_ctx.max_rx_queue_size;
3916 adapter->max_tx_sgl_size = calc_queue_ctx.max_tx_sgl_size;
3917 adapter->max_rx_sgl_size = calc_queue_ctx.max_rx_sgl_size;
3918
3919 adapter->max_num_io_queues = max_num_io_queues;
3920
3921 adapter->buf_ring_size = ENA_DEFAULT_BUF_RING_SIZE;
3922
3923 adapter->max_mtu = get_feat_ctx.dev_attr.max_mtu;
3924
3925 adapter->reset_reason = ENA_REGS_RESET_NORMAL;
3926
3927 /* set up dma tags for rx and tx buffers */
3928 rc = ena_setup_tx_dma_tag(adapter);
3929 if (unlikely(rc != 0)) {
3930 ena_log(pdev, ERR, "Failed to create TX DMA tag\n");
3931 goto err_com_free;
3932 }
3933
3934 rc = ena_setup_rx_dma_tag(adapter);
3935 if (unlikely(rc != 0)) {
3936 ena_log(pdev, ERR, "Failed to create RX DMA tag\n");
3937 goto err_tx_tag_free;
3938 }
3939
3940 /*
3941 * The amount of requested MSIX vectors is equal to
3942 * adapter::max_num_io_queues (see `ena_enable_msix()`), plus a constant
3943 * number of admin queue interrupts. The former is initially determined
3944 * by HW capabilities (see `ena_calc_max_io_queue_num())` but may not be
3945 * achieved if there are not enough system resources. By default, the
3946 * number of effectively used IO queues is the same but later on it can
3947 * be limited by the user using sysctl interface.
3948 */
3949 rc = ena_enable_msix_and_set_admin_interrupts(adapter);
3950 if (unlikely(rc != 0)) {
3951 ena_log(pdev, ERR,
3952 "Failed to enable and set the admin interrupts\n");
3953 goto err_io_free;
3954 }
3955 /* By default all of allocated MSIX vectors are actively used */
3956 adapter->num_io_queues = adapter->msix_vecs - ENA_ADMIN_MSIX_VEC;
3957
3958 /* initialize rings basic information */
3959 ena_init_io_rings(adapter);
3960
3961 rc = ena_com_allocate_customer_metrics_buffer(ena_dev);
3962 if (rc) {
3963 ena_log(pdev, ERR, "Failed to allocate customer metrics buffer.\n");
3964 goto err_msix_free;
3965 }
3966
3967 rc = ena_sysctl_allocate_customer_metrics_buffer(adapter);
3968 if (unlikely(rc)){
3969 ena_log(pdev, ERR, "Failed to allocate sysctl customer metrics buffer.\n");
3970 goto err_metrics_buffer_destroy;
3971 }
3972
3973 /* Initialize statistics */
3974 ena_alloc_counters((counter_u64_t *)&adapter->dev_stats,
3975 sizeof(struct ena_stats_dev));
3976 ena_alloc_counters((counter_u64_t *)&adapter->hw_stats,
3977 sizeof(struct ena_hw_stats));
3978 ena_sysctl_add_nodes(adapter);
3979
3980 /* setup network interface */
3981 ena_setup_ifnet(pdev, adapter, &get_feat_ctx);
3982
3983 /* Initialize reset task queue */
3984 TASK_INIT(&adapter->reset_task, 0, ena_reset_task, adapter);
3985 adapter->reset_tq = taskqueue_create("ena_reset_enqueue",
3986 M_WAITOK | M_ZERO, taskqueue_thread_enqueue, &adapter->reset_tq);
3987 taskqueue_start_threads(&adapter->reset_tq, 1, PI_NET, "%s rstq",
3988 device_get_nameunit(adapter->pdev));
3989
3990 /* Initialize metrics task queue */
3991 TASK_INIT(&adapter->metrics_task, 0, ena_metrics_task, adapter);
3992 adapter->metrics_tq = taskqueue_create("ena_metrics_enqueue",
3993 M_WAITOK | M_ZERO, taskqueue_thread_enqueue, &adapter->metrics_tq);
3994 taskqueue_start_threads(&adapter->metrics_tq, 1, PI_NET, "%s metricsq",
3995 device_get_nameunit(adapter->pdev));
3996
3997 #ifdef DEV_NETMAP
3998 rc = ena_netmap_attach(adapter);
3999 if (rc != 0) {
4000 ena_log(pdev, ERR, "netmap attach failed: %d\n", rc);
4001 goto err_detach;
4002 }
4003 #endif /* DEV_NETMAP */
4004
4005 /* Tell the stack that the interface is not active */
4006 if_setdrvflagbits(adapter->ifp, IFF_DRV_OACTIVE, IFF_DRV_RUNNING);
4007 ENA_FLAG_SET_ATOMIC(ENA_FLAG_DEVICE_RUNNING, adapter);
4008
4009 /* Run the timer service */
4010 ENA_TIMER_RESET(adapter);
4011
4012 return (0);
4013
4014 #ifdef DEV_NETMAP
4015 err_detach:
4016 ether_ifdetach(adapter->ifp);
4017 ifmedia_removeall(&adapter->media);
4018 free(adapter->customer_metrics_array, M_DEVBUF);
4019 #endif /* DEV_NETMAP */
4020 err_metrics_buffer_destroy:
4021 ena_com_delete_customer_metrics_buffer(ena_dev);
4022 err_msix_free:
4023 ena_free_stats(adapter);
4024 ena_com_dev_reset(adapter->ena_dev, ENA_REGS_RESET_INIT_ERR);
4025 ena_free_mgmnt_irq(adapter);
4026 ena_disable_msix(adapter);
4027 err_io_free:
4028 ena_free_all_io_rings_resources(adapter);
4029 ena_free_rx_dma_tag(adapter);
4030 err_tx_tag_free:
4031 ena_free_tx_dma_tag(adapter);
4032 err_com_free:
4033 ena_com_admin_destroy(ena_dev);
4034 ena_com_delete_host_info(ena_dev);
4035 ena_com_mmio_reg_read_request_destroy(ena_dev);
4036 err_bus_free:
4037 free(ena_dev->bus, M_DEVBUF);
4038 err_pci_free:
4039 ena_free_pci_resources(adapter);
4040 err_dev_free:
4041 free(ena_dev, M_DEVBUF);
4042
4043 return (rc);
4044 }
4045
4046 /**
4047 * ena_detach - Device Removal Routine
4048 * @pdev: device information struct
4049 *
4050 * ena_detach is called by the device subsystem to alert the driver
4051 * that it should release a PCI device.
4052 **/
4053 static int
ena_detach(device_t pdev)4054 ena_detach(device_t pdev)
4055 {
4056 struct ena_adapter *adapter = device_get_softc(pdev);
4057 struct ena_com_dev *ena_dev = adapter->ena_dev;
4058 int rc;
4059
4060 /* Make sure VLANS are not using driver */
4061 if (if_vlantrunkinuse(adapter->ifp)) {
4062 ena_log(adapter->pdev, ERR, "VLAN is in use, detach first\n");
4063 return (EBUSY);
4064 }
4065
4066 rc = bus_generic_detach(pdev);
4067 if (rc != 0)
4068 return (rc);
4069
4070 ether_ifdetach(adapter->ifp);
4071
4072 ifmedia_removeall(&adapter->media);
4073
4074 /* Stop timer service */
4075 ENA_LOCK_LOCK();
4076 ENA_TIMER_DRAIN(adapter);
4077 ENA_LOCK_UNLOCK();
4078
4079 /* Release metrics task */
4080 while (taskqueue_cancel(adapter->metrics_tq, &adapter->metrics_task, NULL))
4081 taskqueue_drain(adapter->metrics_tq, &adapter->metrics_task);
4082 taskqueue_free(adapter->metrics_tq);
4083
4084 /* Release reset task */
4085 while (taskqueue_cancel(adapter->reset_tq, &adapter->reset_task, NULL))
4086 taskqueue_drain(adapter->reset_tq, &adapter->reset_task);
4087 taskqueue_free(adapter->reset_tq);
4088
4089 ENA_LOCK_LOCK();
4090 ena_down(adapter);
4091 ena_destroy_device(adapter, true);
4092 ENA_LOCK_UNLOCK();
4093
4094 /* Restore unregistered sysctl queue nodes. */
4095 ena_sysctl_update_queue_node_nb(adapter, adapter->num_io_queues,
4096 adapter->max_num_io_queues);
4097
4098 #ifdef DEV_NETMAP
4099 netmap_detach(adapter->ifp);
4100 #endif /* DEV_NETMAP */
4101
4102 ena_free_stats(adapter);
4103
4104 rc = ena_free_rx_dma_tag(adapter);
4105 if (unlikely(rc != 0))
4106 ena_log(adapter->pdev, WARN,
4107 "Unmapped RX DMA tag associations\n");
4108
4109 rc = ena_free_tx_dma_tag(adapter);
4110 if (unlikely(rc != 0))
4111 ena_log(adapter->pdev, WARN,
4112 "Unmapped TX DMA tag associations\n");
4113
4114 ena_free_irqs(adapter);
4115
4116 ena_free_pci_resources(adapter);
4117
4118 if (adapter->rss_indir != NULL)
4119 free(adapter->rss_indir, M_DEVBUF);
4120
4121 if (likely(ENA_FLAG_ISSET(ENA_FLAG_RSS_ACTIVE, adapter)))
4122 ena_com_rss_destroy(ena_dev);
4123
4124 ena_com_delete_host_info(ena_dev);
4125
4126 free(adapter->customer_metrics_array, M_DEVBUF);
4127
4128 ena_com_delete_customer_metrics_buffer(ena_dev);
4129
4130 if_free(adapter->ifp);
4131
4132 free(ena_dev->bus, M_DEVBUF);
4133
4134 free(ena_dev, M_DEVBUF);
4135
4136 return (0);
4137 }
4138
4139 /******************************************************************************
4140 ******************************** AENQ Handlers *******************************
4141 *****************************************************************************/
4142 /**
4143 * ena_update_on_link_change:
4144 * Notify the network interface about the change in link status
4145 **/
4146 static void
ena_update_on_link_change(void * adapter_data,struct ena_admin_aenq_entry * aenq_e)4147 ena_update_on_link_change(void *adapter_data,
4148 struct ena_admin_aenq_entry *aenq_e)
4149 {
4150 struct ena_adapter *adapter = (struct ena_adapter *)adapter_data;
4151 struct ena_admin_aenq_link_change_desc *aenq_desc;
4152 int status;
4153 if_t ifp;
4154
4155 aenq_desc = (struct ena_admin_aenq_link_change_desc *)aenq_e;
4156 ifp = adapter->ifp;
4157 status = aenq_desc->flags &
4158 ENA_ADMIN_AENQ_LINK_CHANGE_DESC_LINK_STATUS_MASK;
4159
4160 if (status != 0) {
4161 ena_log(adapter->pdev, INFO, "link is UP\n");
4162 ENA_FLAG_SET_ATOMIC(ENA_FLAG_LINK_UP, adapter);
4163 if (!ENA_FLAG_ISSET(ENA_FLAG_ONGOING_RESET, adapter))
4164 if_link_state_change(ifp, LINK_STATE_UP);
4165 } else {
4166 ena_log(adapter->pdev, INFO, "link is DOWN\n");
4167 if_link_state_change(ifp, LINK_STATE_DOWN);
4168 ENA_FLAG_CLEAR_ATOMIC(ENA_FLAG_LINK_UP, adapter);
4169 }
4170 }
4171
4172 static void
ena_notification(void * adapter_data,struct ena_admin_aenq_entry * aenq_e)4173 ena_notification(void *adapter_data, struct ena_admin_aenq_entry *aenq_e)
4174 {
4175 struct ena_adapter *adapter = (struct ena_adapter *)adapter_data;
4176 struct ena_admin_ena_hw_hints *hints;
4177
4178 ENA_WARN(aenq_e->aenq_common_desc.group != ENA_ADMIN_NOTIFICATION,
4179 adapter->ena_dev, "Invalid group(%x) expected %x\n",
4180 aenq_e->aenq_common_desc.group, ENA_ADMIN_NOTIFICATION);
4181
4182 switch (aenq_e->aenq_common_desc.syndrome) {
4183 case ENA_ADMIN_UPDATE_HINTS:
4184 hints =
4185 (struct ena_admin_ena_hw_hints *)(&aenq_e->inline_data_w4);
4186 ena_update_hints(adapter, hints);
4187 break;
4188 default:
4189 ena_log(adapter->pdev, ERR,
4190 "Invalid aenq notification link state %d\n",
4191 aenq_e->aenq_common_desc.syndrome);
4192 }
4193 }
4194
4195 static void
ena_lock_init(void * arg)4196 ena_lock_init(void *arg)
4197 {
4198 ENA_LOCK_INIT();
4199 }
4200 SYSINIT(ena_lock_init, SI_SUB_LOCK, SI_ORDER_FIRST, ena_lock_init, NULL);
4201
4202 static void
ena_lock_uninit(void * arg)4203 ena_lock_uninit(void *arg)
4204 {
4205 ENA_LOCK_DESTROY();
4206 }
4207 SYSUNINIT(ena_lock_uninit, SI_SUB_LOCK, SI_ORDER_FIRST, ena_lock_uninit, NULL);
4208
4209 /**
4210 * This handler will called for unknown event group or unimplemented handlers
4211 **/
4212 static void
unimplemented_aenq_handler(void * adapter_data,struct ena_admin_aenq_entry * aenq_e)4213 unimplemented_aenq_handler(void *adapter_data,
4214 struct ena_admin_aenq_entry *aenq_e)
4215 {
4216 struct ena_adapter *adapter = (struct ena_adapter *)adapter_data;
4217
4218 ena_log(adapter->pdev, ERR,
4219 "Unknown event was received or event with unimplemented handler\n");
4220 }
4221
ena_conf_notification(void * adapter_data,struct ena_admin_aenq_entry * aenq_e)4222 static void ena_conf_notification(void *adapter_data,
4223 struct ena_admin_aenq_entry *aenq_e)
4224 {
4225 struct ena_adapter *adapter = (struct ena_adapter *)adapter_data;
4226 struct ena_admin_aenq_conf_notifications_desc *desc;
4227 u64 bitmap, bit;
4228
4229 desc = (struct ena_admin_aenq_conf_notifications_desc *)aenq_e;
4230 bitmap = desc->notifications_bitmap;
4231
4232 if (bitmap == 0) {
4233 ena_log(adapter->pdev, INFO,
4234 "Empty configuration notification bitmap\n");
4235 return;
4236 }
4237
4238 for (bit = ffsll(bitmap); bit != 0; bit = ffsll(bitmap)) {
4239 bit--;
4240 ena_log(adapter->pdev, INFO,
4241 "Sub-optimal configuration notification code: %" PRIu64 " Refer to AWS ENA documentation for additional details and mitigation options.\n",
4242 bit + 1);
4243 // Clear the processed bit
4244 bitmap &= ~(1UL << bit);
4245 }
4246 }
4247
ena_admin_device_request_reset(void * adapter_data,struct ena_admin_aenq_entry * aenq_e)4248 static void ena_admin_device_request_reset(void *adapter_data,
4249 struct ena_admin_aenq_entry *aenq_e)
4250 {
4251 struct ena_adapter *adapter = (struct ena_adapter *)adapter_data;
4252 ena_log(adapter->pdev, WARN,
4253 "The device has detected an unhealthy state, reset is requested\n");
4254 ena_trigger_reset(adapter, ENA_REGS_RESET_DEVICE_REQUEST);
4255 }
4256
4257 static struct ena_aenq_handlers aenq_handlers = {
4258 .handlers = {
4259 [ENA_ADMIN_LINK_CHANGE] = ena_update_on_link_change,
4260 [ENA_ADMIN_NOTIFICATION] = ena_notification,
4261 [ENA_ADMIN_KEEP_ALIVE] = ena_keep_alive_wd,
4262 [ENA_ADMIN_CONF_NOTIFICATIONS] = ena_conf_notification,
4263 [ENA_ADMIN_DEVICE_REQUEST_RESET] = ena_admin_device_request_reset,
4264 },
4265 .unimplemented_handler = unimplemented_aenq_handler
4266 };
4267
4268 /*********************************************************************
4269 * FreeBSD Device Interface Entry Points
4270 *********************************************************************/
4271
4272 static device_method_t ena_methods[] = { /* Device interface */
4273 DEVMETHOD(device_probe, ena_probe),
4274 DEVMETHOD(device_attach, ena_attach),
4275 DEVMETHOD(device_detach, ena_detach), DEVMETHOD_END
4276 };
4277
4278 static driver_t ena_driver = {
4279 "ena",
4280 ena_methods,
4281 sizeof(struct ena_adapter),
4282 };
4283
4284 DRIVER_MODULE(ena, pci, ena_driver, 0, 0);
4285 MODULE_PNP_INFO("U16:vendor;U16:device", pci, ena, ena_vendor_info_array,
4286 nitems(ena_vendor_info_array) - 1);
4287 MODULE_DEPEND(ena, pci, 1, 1, 1);
4288 MODULE_DEPEND(ena, ether, 1, 1, 1);
4289 #ifdef DEV_NETMAP
4290 MODULE_DEPEND(ena, netmap, 1, 1, 1);
4291 #endif /* DEV_NETMAP */
4292
4293 /*********************************************************************/
4294