xref: /linux/drivers/net/ethernet/amazon/ena/ena_netdev.c (revision f3c790c34d228d46c4a05fe5d001d1bbea444e2b)
1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
2 /*
3  * Copyright 2015-2020 Amazon.com, Inc. or its affiliates. All rights reserved.
4  */
5 
6 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
7 
8 #include <linux/ethtool.h>
9 #include <linux/kernel.h>
10 #include <linux/module.h>
11 #include <linux/numa.h>
12 #include <linux/pci.h>
13 #include <linux/utsname.h>
14 #include <linux/version.h>
15 #include <linux/vmalloc.h>
16 #include <net/ip.h>
17 
18 #include "ena_netdev.h"
19 #include "ena_pci_id_tbl.h"
20 #include "ena_xdp.h"
21 
22 #include "ena_phc.h"
23 
24 #include "ena_devlink.h"
25 
26 #include "ena_debugfs.h"
27 
28 MODULE_AUTHOR("Amazon.com, Inc. or its affiliates");
29 MODULE_DESCRIPTION(DEVICE_NAME);
30 MODULE_LICENSE("GPL");
31 
32 /* Time in jiffies before concluding the transmitter is hung. */
33 #define TX_TIMEOUT  (5 * HZ)
34 
35 #define ENA_MAX_RINGS min_t(unsigned int, ENA_MAX_NUM_IO_QUEUES, num_possible_cpus())
36 
37 #define DEFAULT_MSG_ENABLE (NETIF_MSG_DRV | NETIF_MSG_PROBE | NETIF_MSG_IFUP | \
38 		NETIF_MSG_IFDOWN | NETIF_MSG_TX_ERR | NETIF_MSG_RX_ERR)
39 
40 static struct ena_aenq_handlers aenq_handlers;
41 
42 static struct workqueue_struct *ena_wq;
43 
44 MODULE_DEVICE_TABLE(pci, ena_pci_tbl);
45 
46 static int ena_rss_init_default(struct ena_adapter *adapter);
47 static void check_for_admin_com_state(struct ena_adapter *adapter);
48 
49 static void ena_tx_timeout(struct net_device *dev, unsigned int txqueue)
50 {
51 	enum ena_regs_reset_reason_types reset_reason = ENA_REGS_RESET_OS_NETDEV_WD;
52 	struct ena_adapter *adapter = netdev_priv(dev);
53 	unsigned int time_since_last_napi, threshold;
54 	struct ena_ring *tx_ring;
55 	int napi_scheduled;
56 
57 	if (txqueue >= adapter->num_io_queues) {
58 		netdev_err(dev, "TX timeout on invalid queue %u\n", txqueue);
59 		goto schedule_reset;
60 	}
61 
62 	threshold = jiffies_to_usecs(dev->watchdog_timeo);
63 	tx_ring = &adapter->tx_ring[txqueue];
64 
65 	time_since_last_napi = jiffies_to_usecs(jiffies - tx_ring->tx_stats.last_napi_jiffies);
66 	napi_scheduled = !!(tx_ring->napi->state & NAPIF_STATE_SCHED);
67 
68 	netdev_err(dev,
69 		   "TX q %d is paused for too long (threshold %u). Time since last napi %u usec. napi scheduled: %d\n",
70 		   txqueue,
71 		   threshold,
72 		   time_since_last_napi,
73 		   napi_scheduled);
74 
75 	if (threshold < time_since_last_napi && napi_scheduled) {
76 		netdev_err(dev,
77 			   "napi handler hasn't been called for a long time but is scheduled\n");
78 		reset_reason = ENA_REGS_RESET_SUSPECTED_POLL_STARVATION;
79 	}
80 schedule_reset:
81 	/* Change the state of the device to trigger reset
82 	 * Check that we are not in the middle or a trigger already
83 	 */
84 	if (test_and_set_bit(ENA_FLAG_TRIGGER_RESET, &adapter->flags))
85 		return;
86 
87 	ena_reset_device(adapter, reset_reason);
88 	ena_increase_stat(&adapter->dev_stats.tx_timeout, 1, &adapter->syncp);
89 }
90 
91 static void update_rx_ring_mtu(struct ena_adapter *adapter, int mtu)
92 {
93 	int i;
94 
95 	for (i = 0; i < adapter->num_io_queues; i++)
96 		adapter->rx_ring[i].mtu = mtu;
97 }
98 
99 static int ena_change_mtu(struct net_device *dev, int new_mtu)
100 {
101 	struct ena_adapter *adapter = netdev_priv(dev);
102 	int ret;
103 
104 	ret = ena_com_set_dev_mtu(adapter->ena_dev, new_mtu);
105 	if (!ret) {
106 		netif_dbg(adapter, drv, dev, "Set MTU to %d\n", new_mtu);
107 		update_rx_ring_mtu(adapter, new_mtu);
108 		WRITE_ONCE(dev->mtu, new_mtu);
109 	} else {
110 		netif_err(adapter, drv, dev, "Failed to set MTU to %d\n",
111 			  new_mtu);
112 	}
113 
114 	return ret;
115 }
116 
117 int ena_xmit_common(struct ena_adapter *adapter,
118 		    struct ena_ring *ring,
119 		    struct ena_tx_buffer *tx_info,
120 		    struct ena_com_tx_ctx *ena_tx_ctx,
121 		    u16 next_to_use,
122 		    u32 bytes)
123 {
124 	int rc, nb_hw_desc;
125 
126 	if (unlikely(ena_com_is_doorbell_needed(ring->ena_com_io_sq,
127 						ena_tx_ctx))) {
128 		netif_dbg(adapter, tx_queued, adapter->netdev,
129 			  "llq tx max burst size of queue %d achieved, writing doorbell to send burst\n",
130 			  ring->qid);
131 		ena_ring_tx_doorbell(ring);
132 	}
133 
134 	/* prepare the packet's descriptors to dma engine */
135 	rc = ena_com_prepare_tx(ring->ena_com_io_sq, ena_tx_ctx,
136 				&nb_hw_desc);
137 
138 	/* In case there isn't enough space in the queue for the packet,
139 	 * we simply drop it. All other failure reasons of
140 	 * ena_com_prepare_tx() are fatal and therefore require a device reset.
141 	 */
142 	if (unlikely(rc)) {
143 		netif_err(adapter, tx_queued, adapter->netdev,
144 			  "Failed to prepare tx bufs\n");
145 		ena_increase_stat(&ring->tx_stats.prepare_ctx_err, 1, &ring->syncp);
146 		if (rc != -ENOMEM)
147 			ena_reset_device(adapter, ENA_REGS_RESET_DRIVER_INVALID_STATE);
148 		return rc;
149 	}
150 
151 	u64_stats_update_begin(&ring->syncp);
152 	ring->tx_stats.cnt++;
153 	ring->tx_stats.bytes += bytes;
154 	u64_stats_update_end(&ring->syncp);
155 
156 	tx_info->tx_descs = nb_hw_desc;
157 	tx_info->total_tx_size = bytes;
158 	tx_info->last_jiffies = jiffies;
159 	tx_info->print_once = 0;
160 
161 	ring->next_to_use = ENA_TX_RING_IDX_NEXT(next_to_use,
162 						 ring->ring_size);
163 	return 0;
164 }
165 
166 static void ena_init_io_rings_common(struct ena_adapter *adapter,
167 				     struct ena_ring *ring, u16 qid)
168 {
169 	ring->qid = qid;
170 	ring->pdev = adapter->pdev;
171 	ring->dev = &adapter->pdev->dev;
172 	ring->netdev = adapter->netdev;
173 	ring->napi = &adapter->ena_napi[qid].napi;
174 	ring->adapter = adapter;
175 	ring->ena_dev = adapter->ena_dev;
176 	ring->per_napi_packets = 0;
177 	ring->cpu = 0;
178 	ring->numa_node = 0;
179 	ring->no_interrupt_event_cnt = 0;
180 	u64_stats_init(&ring->syncp);
181 }
182 
183 void ena_init_io_rings(struct ena_adapter *adapter,
184 		       int first_index, int count)
185 {
186 	struct ena_com_dev *ena_dev;
187 	struct ena_ring *txr, *rxr;
188 	int i;
189 
190 	ena_dev = adapter->ena_dev;
191 
192 	for (i = first_index; i < first_index + count; i++) {
193 		txr = &adapter->tx_ring[i];
194 		rxr = &adapter->rx_ring[i];
195 
196 		/* TX common ring state */
197 		ena_init_io_rings_common(adapter, txr, i);
198 
199 		/* TX specific ring state */
200 		txr->ring_size = adapter->requested_tx_ring_size;
201 		txr->tx_max_header_size = ena_dev->tx_max_header_size;
202 		txr->tx_mem_queue_type = ena_dev->tx_mem_queue_type;
203 		txr->sgl_size = adapter->max_tx_sgl_size;
204 		txr->smoothed_interval =
205 			ena_com_get_nonadaptive_moderation_interval_tx(ena_dev);
206 		txr->disable_meta_caching = adapter->disable_meta_caching;
207 		spin_lock_init(&txr->xdp_tx_lock);
208 
209 		/* Don't init RX queues for xdp queues */
210 		if (!ENA_IS_XDP_INDEX(adapter, i)) {
211 			/* RX common ring state */
212 			ena_init_io_rings_common(adapter, rxr, i);
213 
214 			/* RX specific ring state */
215 			rxr->ring_size = adapter->requested_rx_ring_size;
216 			rxr->rx_copybreak = adapter->rx_copybreak;
217 			rxr->sgl_size = adapter->max_rx_sgl_size;
218 			rxr->smoothed_interval =
219 				ena_com_get_nonadaptive_moderation_interval_rx(ena_dev);
220 			rxr->empty_rx_queue = 0;
221 			rxr->rx_headroom = NET_SKB_PAD;
222 			adapter->ena_napi[i].dim.mode = DIM_CQ_PERIOD_MODE_START_FROM_EQE;
223 			rxr->xdp_ring = &adapter->tx_ring[i + adapter->num_io_queues];
224 		}
225 	}
226 }
227 
228 /* ena_setup_tx_resources - allocate I/O Tx resources (Descriptors)
229  * @adapter: network interface device structure
230  * @qid: queue index
231  *
232  * Return 0 on success, negative on failure
233  */
234 static int ena_setup_tx_resources(struct ena_adapter *adapter, int qid)
235 {
236 	struct ena_ring *tx_ring = &adapter->tx_ring[qid];
237 	struct ena_irq *ena_irq = &adapter->irq_tbl[ENA_IO_IRQ_IDX(qid)];
238 	int size, i, node;
239 
240 	if (tx_ring->tx_buffer_info) {
241 		netif_err(adapter, ifup,
242 			  adapter->netdev, "tx_buffer_info info is not NULL");
243 		return -EEXIST;
244 	}
245 
246 	size = sizeof(struct ena_tx_buffer) * tx_ring->ring_size;
247 	node = cpu_to_node(ena_irq->cpu);
248 
249 	tx_ring->tx_buffer_info = vzalloc_node(size, node);
250 	if (!tx_ring->tx_buffer_info) {
251 		tx_ring->tx_buffer_info = vzalloc(size);
252 		if (!tx_ring->tx_buffer_info)
253 			goto err_tx_buffer_info;
254 	}
255 
256 	size = sizeof(u16) * tx_ring->ring_size;
257 	tx_ring->free_ids = vzalloc_node(size, node);
258 	if (!tx_ring->free_ids) {
259 		tx_ring->free_ids = vzalloc(size);
260 		if (!tx_ring->free_ids)
261 			goto err_tx_free_ids;
262 	}
263 
264 	size = tx_ring->tx_max_header_size;
265 	tx_ring->push_buf_intermediate_buf = vzalloc_node(size, node);
266 	if (!tx_ring->push_buf_intermediate_buf) {
267 		tx_ring->push_buf_intermediate_buf = vzalloc(size);
268 		if (!tx_ring->push_buf_intermediate_buf)
269 			goto err_push_buf_intermediate_buf;
270 	}
271 
272 	/* Req id ring for TX out of order completions */
273 	for (i = 0; i < tx_ring->ring_size; i++)
274 		tx_ring->free_ids[i] = i;
275 
276 	/* Reset tx statistics */
277 	memset(&tx_ring->tx_stats, 0x0, sizeof(tx_ring->tx_stats));
278 
279 	tx_ring->next_to_use = 0;
280 	tx_ring->next_to_clean = 0;
281 	tx_ring->cpu = ena_irq->cpu;
282 	tx_ring->numa_node = node;
283 	return 0;
284 
285 err_push_buf_intermediate_buf:
286 	vfree(tx_ring->free_ids);
287 	tx_ring->free_ids = NULL;
288 err_tx_free_ids:
289 	vfree(tx_ring->tx_buffer_info);
290 	tx_ring->tx_buffer_info = NULL;
291 err_tx_buffer_info:
292 	return -ENOMEM;
293 }
294 
295 /* ena_free_tx_resources - Free I/O Tx Resources per Queue
296  * @adapter: network interface device structure
297  * @qid: queue index
298  *
299  * Free all transmit software resources
300  */
301 static void ena_free_tx_resources(struct ena_adapter *adapter, int qid)
302 {
303 	struct ena_ring *tx_ring = &adapter->tx_ring[qid];
304 
305 	vfree(tx_ring->tx_buffer_info);
306 	tx_ring->tx_buffer_info = NULL;
307 
308 	vfree(tx_ring->free_ids);
309 	tx_ring->free_ids = NULL;
310 
311 	vfree(tx_ring->push_buf_intermediate_buf);
312 	tx_ring->push_buf_intermediate_buf = NULL;
313 }
314 
315 int ena_setup_tx_resources_in_range(struct ena_adapter *adapter,
316 				    int first_index, int count)
317 {
318 	int i, rc = 0;
319 
320 	for (i = first_index; i < first_index + count; i++) {
321 		rc = ena_setup_tx_resources(adapter, i);
322 		if (rc)
323 			goto err_setup_tx;
324 	}
325 
326 	return 0;
327 
328 err_setup_tx:
329 
330 	netif_err(adapter, ifup, adapter->netdev,
331 		  "Tx queue %d: allocation failed\n", i);
332 
333 	/* rewind the index freeing the rings as we go */
334 	while (first_index < i--)
335 		ena_free_tx_resources(adapter, i);
336 	return rc;
337 }
338 
339 void ena_free_all_io_tx_resources_in_range(struct ena_adapter *adapter,
340 					   int first_index, int count)
341 {
342 	int i;
343 
344 	for (i = first_index; i < first_index + count; i++)
345 		ena_free_tx_resources(adapter, i);
346 }
347 
348 /* ena_free_all_io_tx_resources - Free I/O Tx Resources for All Queues
349  * @adapter: board private structure
350  *
351  * Free all transmit software resources
352  */
353 void ena_free_all_io_tx_resources(struct ena_adapter *adapter)
354 {
355 	ena_free_all_io_tx_resources_in_range(adapter,
356 					      0,
357 					      adapter->xdp_num_queues +
358 					      adapter->num_io_queues);
359 }
360 
361 /* ena_setup_rx_resources - allocate I/O Rx resources (Descriptors)
362  * @adapter: network interface device structure
363  * @qid: queue index
364  *
365  * Returns 0 on success, negative on failure
366  */
367 static int ena_setup_rx_resources(struct ena_adapter *adapter,
368 				  u32 qid)
369 {
370 	struct ena_ring *rx_ring = &adapter->rx_ring[qid];
371 	struct ena_irq *ena_irq = &adapter->irq_tbl[ENA_IO_IRQ_IDX(qid)];
372 	int size, node, i;
373 
374 	if (rx_ring->rx_buffer_info) {
375 		netif_err(adapter, ifup, adapter->netdev,
376 			  "rx_buffer_info is not NULL");
377 		return -EEXIST;
378 	}
379 
380 	/* alloc extra element so in rx path
381 	 * we can always prefetch rx_info + 1
382 	 */
383 	size = sizeof(struct ena_rx_buffer) * (rx_ring->ring_size + 1);
384 	node = cpu_to_node(ena_irq->cpu);
385 
386 	rx_ring->rx_buffer_info = vzalloc_node(size, node);
387 	if (!rx_ring->rx_buffer_info) {
388 		rx_ring->rx_buffer_info = vzalloc(size);
389 		if (!rx_ring->rx_buffer_info)
390 			return -ENOMEM;
391 	}
392 
393 	size = sizeof(u16) * rx_ring->ring_size;
394 	rx_ring->free_ids = vzalloc_node(size, node);
395 	if (!rx_ring->free_ids) {
396 		rx_ring->free_ids = vzalloc(size);
397 		if (!rx_ring->free_ids) {
398 			vfree(rx_ring->rx_buffer_info);
399 			rx_ring->rx_buffer_info = NULL;
400 			return -ENOMEM;
401 		}
402 	}
403 
404 	/* Req id ring for receiving RX pkts out of order */
405 	for (i = 0; i < rx_ring->ring_size; i++)
406 		rx_ring->free_ids[i] = i;
407 
408 	/* Reset rx statistics */
409 	memset(&rx_ring->rx_stats, 0x0, sizeof(rx_ring->rx_stats));
410 
411 	rx_ring->next_to_clean = 0;
412 	rx_ring->next_to_use = 0;
413 	rx_ring->cpu = ena_irq->cpu;
414 	rx_ring->numa_node = node;
415 
416 	return 0;
417 }
418 
419 /* ena_free_rx_resources - Free I/O Rx Resources
420  * @adapter: network interface device structure
421  * @qid: queue index
422  *
423  * Free all receive software resources
424  */
425 static void ena_free_rx_resources(struct ena_adapter *adapter,
426 				  u32 qid)
427 {
428 	struct ena_ring *rx_ring = &adapter->rx_ring[qid];
429 
430 	vfree(rx_ring->rx_buffer_info);
431 	rx_ring->rx_buffer_info = NULL;
432 
433 	vfree(rx_ring->free_ids);
434 	rx_ring->free_ids = NULL;
435 }
436 
437 /* ena_setup_all_rx_resources - allocate I/O Rx queues resources for all queues
438  * @adapter: board private structure
439  *
440  * Return 0 on success, negative on failure
441  */
442 static int ena_setup_all_rx_resources(struct ena_adapter *adapter)
443 {
444 	int i, rc = 0;
445 
446 	for (i = 0; i < adapter->num_io_queues; i++) {
447 		rc = ena_setup_rx_resources(adapter, i);
448 		if (rc)
449 			goto err_setup_rx;
450 	}
451 
452 	return 0;
453 
454 err_setup_rx:
455 
456 	netif_err(adapter, ifup, adapter->netdev,
457 		  "Rx queue %d: allocation failed\n", i);
458 
459 	/* rewind the index freeing the rings as we go */
460 	while (i--)
461 		ena_free_rx_resources(adapter, i);
462 	return rc;
463 }
464 
465 /* ena_free_all_io_rx_resources - Free I/O Rx Resources for All Queues
466  * @adapter: board private structure
467  *
468  * Free all receive software resources
469  */
470 static void ena_free_all_io_rx_resources(struct ena_adapter *adapter)
471 {
472 	int i;
473 
474 	for (i = 0; i < adapter->num_io_queues; i++)
475 		ena_free_rx_resources(adapter, i);
476 }
477 
478 static struct page *ena_alloc_map_page(struct ena_ring *rx_ring,
479 				       dma_addr_t *dma)
480 {
481 	struct page *page;
482 
483 	/* This would allocate the page on the same NUMA node the executing code
484 	 * is running on.
485 	 */
486 	page = dev_alloc_page();
487 	if (!page) {
488 		ena_increase_stat(&rx_ring->rx_stats.page_alloc_fail, 1, &rx_ring->syncp);
489 		return ERR_PTR(-ENOSPC);
490 	}
491 
492 	/* To enable NIC-side port-mirroring, AKA SPAN port,
493 	 * we make the buffer readable from the nic as well
494 	 */
495 	*dma = dma_map_page(rx_ring->dev, page, 0, ENA_PAGE_SIZE,
496 			    DMA_BIDIRECTIONAL);
497 	if (unlikely(dma_mapping_error(rx_ring->dev, *dma))) {
498 		ena_increase_stat(&rx_ring->rx_stats.dma_mapping_err, 1,
499 				  &rx_ring->syncp);
500 		__free_page(page);
501 		return ERR_PTR(-EIO);
502 	}
503 
504 	return page;
505 }
506 
507 static int ena_alloc_rx_buffer(struct ena_ring *rx_ring,
508 			       struct ena_rx_buffer *rx_info)
509 {
510 	int headroom = rx_ring->rx_headroom;
511 	struct ena_com_buf *ena_buf;
512 	struct page *page;
513 	dma_addr_t dma;
514 	int tailroom;
515 
516 	/* restore page offset value in case it has been changed by device */
517 	rx_info->buf_offset = headroom;
518 
519 	/* if previous allocated page is not used */
520 	if (unlikely(rx_info->page))
521 		return 0;
522 
523 	/* We handle DMA here */
524 	page = ena_alloc_map_page(rx_ring, &dma);
525 	if (IS_ERR(page))
526 		return PTR_ERR(page);
527 
528 	netif_dbg(rx_ring->adapter, rx_status, rx_ring->netdev,
529 		  "Allocate page %p, rx_info %p\n", page, rx_info);
530 
531 	tailroom = SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
532 
533 	rx_info->page = page;
534 	rx_info->dma_addr = dma;
535 	rx_info->page_offset = 0;
536 	ena_buf = &rx_info->ena_buf;
537 	ena_buf->paddr = dma + headroom;
538 	ena_buf->len = ENA_PAGE_SIZE - headroom - tailroom;
539 
540 	return 0;
541 }
542 
543 static void ena_unmap_rx_buff_attrs(struct ena_ring *rx_ring,
544 				    struct ena_rx_buffer *rx_info,
545 				    unsigned long attrs)
546 {
547 	dma_unmap_page_attrs(rx_ring->dev, rx_info->dma_addr, ENA_PAGE_SIZE, DMA_BIDIRECTIONAL,
548 			     attrs);
549 }
550 
551 static void ena_free_rx_page(struct ena_ring *rx_ring,
552 			     struct ena_rx_buffer *rx_info)
553 {
554 	struct page *page = rx_info->page;
555 
556 	if (unlikely(!page)) {
557 		netif_warn(rx_ring->adapter, rx_err, rx_ring->netdev,
558 			   "Trying to free unallocated buffer\n");
559 		return;
560 	}
561 
562 	ena_unmap_rx_buff_attrs(rx_ring, rx_info, 0);
563 
564 	__free_page(page);
565 	rx_info->page = NULL;
566 }
567 
568 static int ena_refill_rx_bufs(struct ena_ring *rx_ring, u32 num)
569 {
570 	u16 next_to_use, req_id;
571 	u32 i;
572 	int rc;
573 
574 	next_to_use = rx_ring->next_to_use;
575 
576 	for (i = 0; i < num; i++) {
577 		struct ena_rx_buffer *rx_info;
578 
579 		req_id = rx_ring->free_ids[next_to_use];
580 
581 		rx_info = &rx_ring->rx_buffer_info[req_id];
582 
583 		rc = ena_alloc_rx_buffer(rx_ring, rx_info);
584 		if (unlikely(rc < 0)) {
585 			netif_warn(rx_ring->adapter, rx_err, rx_ring->netdev,
586 				   "Failed to allocate buffer for rx queue %d\n",
587 				   rx_ring->qid);
588 			break;
589 		}
590 		rc = ena_com_add_single_rx_desc(rx_ring->ena_com_io_sq,
591 						&rx_info->ena_buf,
592 						req_id);
593 		if (unlikely(rc)) {
594 			netif_warn(rx_ring->adapter, rx_status, rx_ring->netdev,
595 				   "Failed to add buffer for rx queue %d\n",
596 				   rx_ring->qid);
597 			break;
598 		}
599 		next_to_use = ENA_RX_RING_IDX_NEXT(next_to_use,
600 						   rx_ring->ring_size);
601 	}
602 
603 	if (unlikely(i < num)) {
604 		ena_increase_stat(&rx_ring->rx_stats.refil_partial, 1,
605 				  &rx_ring->syncp);
606 		netif_warn(rx_ring->adapter, rx_err, rx_ring->netdev,
607 			   "Refilled rx qid %d with only %d buffers (from %d)\n",
608 			   rx_ring->qid, i, num);
609 	}
610 
611 	/* ena_com_write_sq_doorbell issues a wmb() */
612 	if (likely(i))
613 		ena_com_write_sq_doorbell(rx_ring->ena_com_io_sq);
614 
615 	rx_ring->next_to_use = next_to_use;
616 
617 	return i;
618 }
619 
620 static void ena_free_rx_bufs(struct ena_adapter *adapter,
621 			     u32 qid)
622 {
623 	struct ena_ring *rx_ring = &adapter->rx_ring[qid];
624 	u32 i;
625 
626 	for (i = 0; i < rx_ring->ring_size; i++) {
627 		struct ena_rx_buffer *rx_info = &rx_ring->rx_buffer_info[i];
628 
629 		if (rx_info->page)
630 			ena_free_rx_page(rx_ring, rx_info);
631 	}
632 }
633 
634 /* ena_refill_all_rx_bufs - allocate all queues Rx buffers
635  * @adapter: board private structure
636  */
637 static void ena_refill_all_rx_bufs(struct ena_adapter *adapter)
638 {
639 	struct ena_ring *rx_ring;
640 	int i, rc, bufs_num;
641 
642 	for (i = 0; i < adapter->num_io_queues; i++) {
643 		rx_ring = &adapter->rx_ring[i];
644 		bufs_num = rx_ring->ring_size - 1;
645 		rc = ena_refill_rx_bufs(rx_ring, bufs_num);
646 
647 		if (unlikely(rc != bufs_num))
648 			netif_warn(rx_ring->adapter, rx_status, rx_ring->netdev,
649 				   "Refilling Queue %d failed. allocated %d buffers from: %d\n",
650 				   i, rc, bufs_num);
651 	}
652 }
653 
654 static void ena_free_all_rx_bufs(struct ena_adapter *adapter)
655 {
656 	int i;
657 
658 	for (i = 0; i < adapter->num_io_queues; i++)
659 		ena_free_rx_bufs(adapter, i);
660 }
661 
662 void ena_unmap_tx_buff(struct ena_ring *tx_ring,
663 		       struct ena_tx_buffer *tx_info)
664 {
665 	struct ena_com_buf *ena_buf;
666 	u32 cnt;
667 	int i;
668 
669 	ena_buf = tx_info->bufs;
670 	cnt = tx_info->num_of_bufs;
671 
672 	if (unlikely(!cnt))
673 		return;
674 
675 	if (tx_info->map_linear_data) {
676 		dma_unmap_single(tx_ring->dev,
677 				 dma_unmap_addr(ena_buf, paddr),
678 				 dma_unmap_len(ena_buf, len),
679 				 DMA_TO_DEVICE);
680 		ena_buf++;
681 		cnt--;
682 	}
683 
684 	/* unmap remaining mapped pages */
685 	for (i = 0; i < cnt; i++) {
686 		dma_unmap_page(tx_ring->dev, dma_unmap_addr(ena_buf, paddr),
687 			       dma_unmap_len(ena_buf, len), DMA_TO_DEVICE);
688 		ena_buf++;
689 	}
690 }
691 
692 /* ena_free_tx_bufs - Free Tx Buffers per Queue
693  * @tx_ring: TX ring for which buffers be freed
694  */
695 static void ena_free_tx_bufs(struct ena_ring *tx_ring)
696 {
697 	bool print_once = true;
698 	bool is_xdp_ring;
699 	u32 i;
700 
701 	is_xdp_ring = ENA_IS_XDP_INDEX(tx_ring->adapter, tx_ring->qid);
702 
703 	for (i = 0; i < tx_ring->ring_size; i++) {
704 		struct ena_tx_buffer *tx_info = &tx_ring->tx_buffer_info[i];
705 
706 		if (!tx_info->skb)
707 			continue;
708 
709 		if (print_once) {
710 			netif_notice(tx_ring->adapter, ifdown, tx_ring->netdev,
711 				     "Free uncompleted tx skb qid %d idx 0x%x\n",
712 				     tx_ring->qid, i);
713 			print_once = false;
714 		} else {
715 			netif_dbg(tx_ring->adapter, ifdown, tx_ring->netdev,
716 				  "Free uncompleted tx skb qid %d idx 0x%x\n",
717 				  tx_ring->qid, i);
718 		}
719 
720 		ena_unmap_tx_buff(tx_ring, tx_info);
721 
722 		if (is_xdp_ring)
723 			xdp_return_frame(tx_info->xdpf);
724 		else
725 			dev_kfree_skb_any(tx_info->skb);
726 	}
727 
728 	if (!is_xdp_ring)
729 		netdev_tx_reset_queue(netdev_get_tx_queue(tx_ring->netdev,
730 							  tx_ring->qid));
731 }
732 
733 static void ena_free_all_tx_bufs(struct ena_adapter *adapter)
734 {
735 	struct ena_ring *tx_ring;
736 	int i;
737 
738 	for (i = 0; i < adapter->num_io_queues + adapter->xdp_num_queues; i++) {
739 		tx_ring = &adapter->tx_ring[i];
740 		ena_free_tx_bufs(tx_ring);
741 	}
742 }
743 
744 static void ena_destroy_all_tx_queues(struct ena_adapter *adapter)
745 {
746 	u16 ena_qid;
747 	int i;
748 
749 	for (i = 0; i < adapter->num_io_queues + adapter->xdp_num_queues; i++) {
750 		ena_qid = ENA_IO_TXQ_IDX(i);
751 		ena_com_destroy_io_queue(adapter->ena_dev, ena_qid);
752 	}
753 }
754 
755 static void ena_destroy_xdp_tx_queues(struct ena_adapter *adapter)
756 {
757 	u16 ena_qid;
758 	int i;
759 
760 	for (i = adapter->xdp_first_ring;
761 	     i < adapter->xdp_first_ring + adapter->xdp_num_queues; i++) {
762 		ena_qid = ENA_IO_TXQ_IDX(i);
763 		ena_com_destroy_io_queue(adapter->ena_dev, ena_qid);
764 	}
765 }
766 
767 static void ena_destroy_all_rx_queues(struct ena_adapter *adapter)
768 {
769 	u16 ena_qid;
770 	int i;
771 
772 	for (i = 0; i < adapter->num_io_queues; i++) {
773 		ena_qid = ENA_IO_RXQ_IDX(i);
774 		cancel_work_sync(&adapter->ena_napi[i].dim.work);
775 		ena_xdp_unregister_rxq_info(&adapter->rx_ring[i]);
776 		ena_com_destroy_io_queue(adapter->ena_dev, ena_qid);
777 	}
778 }
779 
780 static void ena_destroy_all_io_queues(struct ena_adapter *adapter)
781 {
782 	ena_destroy_all_tx_queues(adapter);
783 	ena_destroy_all_rx_queues(adapter);
784 }
785 
786 int handle_invalid_req_id(struct ena_ring *ring, u16 req_id,
787 			  struct ena_tx_buffer *tx_info, bool is_xdp)
788 {
789 	if (tx_info)
790 		netif_err(ring->adapter,
791 			  tx_done,
792 			  ring->netdev,
793 			  "tx_info doesn't have valid %s. qid %u req_id %u",
794 			   is_xdp ? "xdp frame" : "skb", ring->qid, req_id);
795 	else
796 		netif_err(ring->adapter,
797 			  tx_done,
798 			  ring->netdev,
799 			  "Invalid req_id %u in qid %u\n",
800 			  req_id, ring->qid);
801 
802 	ena_increase_stat(&ring->tx_stats.bad_req_id, 1, &ring->syncp);
803 	ena_reset_device(ring->adapter, ENA_REGS_RESET_INV_TX_REQ_ID);
804 
805 	return -EFAULT;
806 }
807 
808 static int validate_tx_req_id(struct ena_ring *tx_ring, u16 req_id)
809 {
810 	struct ena_tx_buffer *tx_info;
811 
812 	tx_info = &tx_ring->tx_buffer_info[req_id];
813 	if (likely(tx_info->skb))
814 		return 0;
815 
816 	return handle_invalid_req_id(tx_ring, req_id, tx_info, false);
817 }
818 
819 static int ena_clean_tx_irq(struct ena_ring *tx_ring, u32 budget)
820 {
821 	struct netdev_queue *txq;
822 	bool above_thresh;
823 	u32 tx_bytes = 0;
824 	u32 total_done = 0;
825 	u16 next_to_clean;
826 	u16 req_id;
827 	int tx_pkts = 0;
828 	int rc;
829 
830 	next_to_clean = tx_ring->next_to_clean;
831 	txq = netdev_get_tx_queue(tx_ring->netdev, tx_ring->qid);
832 
833 	while (tx_pkts < budget) {
834 		struct ena_tx_buffer *tx_info;
835 		struct sk_buff *skb;
836 
837 		rc = ena_com_tx_comp_req_id_get(tx_ring->ena_com_io_cq,
838 						&req_id);
839 		if (rc) {
840 			if (unlikely(rc == -EINVAL))
841 				handle_invalid_req_id(tx_ring, req_id, NULL, false);
842 			break;
843 		}
844 
845 		/* validate that the request id points to a valid skb */
846 		rc = validate_tx_req_id(tx_ring, req_id);
847 		if (rc)
848 			break;
849 
850 		tx_info = &tx_ring->tx_buffer_info[req_id];
851 		skb = tx_info->skb;
852 
853 		/* prefetch skb_end_pointer() to speedup skb_shinfo(skb) */
854 		prefetch(&skb->end);
855 
856 		tx_info->skb = NULL;
857 		tx_info->last_jiffies = 0;
858 
859 		ena_unmap_tx_buff(tx_ring, tx_info);
860 
861 		netif_dbg(tx_ring->adapter, tx_done, tx_ring->netdev,
862 			  "tx_poll: q %d skb %p completed\n", tx_ring->qid,
863 			  skb);
864 
865 		tx_bytes += tx_info->total_tx_size;
866 		dev_kfree_skb(skb);
867 		tx_pkts++;
868 		total_done += tx_info->tx_descs;
869 
870 		tx_ring->free_ids[next_to_clean] = req_id;
871 		next_to_clean = ENA_TX_RING_IDX_NEXT(next_to_clean,
872 						     tx_ring->ring_size);
873 	}
874 
875 	tx_ring->next_to_clean = next_to_clean;
876 	ena_com_comp_ack(tx_ring->ena_com_io_sq, total_done);
877 
878 	netdev_tx_completed_queue(txq, tx_pkts, tx_bytes);
879 
880 	netif_dbg(tx_ring->adapter, tx_done, tx_ring->netdev,
881 		  "tx_poll: q %d done. total pkts: %d\n",
882 		  tx_ring->qid, tx_pkts);
883 
884 	/* need to make the rings circular update visible to
885 	 * ena_start_xmit() before checking for netif_queue_stopped().
886 	 */
887 	smp_mb();
888 
889 	above_thresh = ena_com_sq_have_enough_space(tx_ring->ena_com_io_sq,
890 						    ENA_TX_WAKEUP_THRESH);
891 	if (unlikely(netif_tx_queue_stopped(txq) && above_thresh)) {
892 		__netif_tx_lock(txq, smp_processor_id());
893 		above_thresh =
894 			ena_com_sq_have_enough_space(tx_ring->ena_com_io_sq,
895 						     ENA_TX_WAKEUP_THRESH);
896 		if (netif_tx_queue_stopped(txq) && above_thresh &&
897 		    test_bit(ENA_FLAG_DEV_UP, &tx_ring->adapter->flags)) {
898 			netif_tx_wake_queue(txq);
899 			ena_increase_stat(&tx_ring->tx_stats.queue_wakeup, 1,
900 					  &tx_ring->syncp);
901 		}
902 		__netif_tx_unlock(txq);
903 	}
904 
905 	return tx_pkts;
906 }
907 
908 static struct sk_buff *ena_alloc_skb(struct ena_ring *rx_ring, void *first_frag, u16 len)
909 {
910 	struct sk_buff *skb;
911 
912 	if (!first_frag)
913 		skb = napi_alloc_skb(rx_ring->napi, len);
914 	else
915 		skb = napi_build_skb(first_frag, len);
916 
917 	if (unlikely(!skb)) {
918 		ena_increase_stat(&rx_ring->rx_stats.skb_alloc_fail, 1,
919 				  &rx_ring->syncp);
920 
921 		netif_dbg(rx_ring->adapter, rx_err, rx_ring->netdev,
922 			  "Failed to allocate skb. first_frag %s\n",
923 			  first_frag ? "provided" : "not provided");
924 	}
925 
926 	return skb;
927 }
928 
929 static bool ena_try_rx_buf_page_reuse(struct ena_rx_buffer *rx_info, u16 buf_len,
930 				      u16 len, int pkt_offset)
931 {
932 	struct ena_com_buf *ena_buf = &rx_info->ena_buf;
933 
934 	/* More than ENA_MIN_RX_BUF_SIZE left in the reused buffer
935 	 * for data + headroom + tailroom.
936 	 */
937 	if (SKB_DATA_ALIGN(len + pkt_offset) + ENA_MIN_RX_BUF_SIZE <= ena_buf->len) {
938 		page_ref_inc(rx_info->page);
939 		rx_info->page_offset += buf_len;
940 		ena_buf->paddr += buf_len;
941 		ena_buf->len -= buf_len;
942 		return true;
943 	}
944 
945 	return false;
946 }
947 
948 static struct sk_buff *ena_rx_skb(struct ena_ring *rx_ring,
949 				  struct ena_com_rx_buf_info *ena_bufs,
950 				  u32 descs,
951 				  u16 *next_to_clean)
952 {
953 	int tailroom = SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
954 	bool is_xdp_loaded = ena_xdp_present_ring(rx_ring);
955 	struct ena_rx_buffer *rx_info;
956 	struct ena_adapter *adapter;
957 	int page_offset, pkt_offset;
958 	dma_addr_t pre_reuse_paddr;
959 	u16 len, req_id, buf = 0;
960 	bool reuse_rx_buf_page;
961 	struct sk_buff *skb;
962 	void *buf_addr;
963 	int buf_offset;
964 	u16 buf_len;
965 
966 	len = ena_bufs[buf].len;
967 	req_id = ena_bufs[buf].req_id;
968 
969 	rx_info = &rx_ring->rx_buffer_info[req_id];
970 
971 	if (unlikely(!rx_info->page)) {
972 		adapter = rx_ring->adapter;
973 		netif_err(adapter, rx_err, rx_ring->netdev,
974 			  "Page is NULL. qid %u req_id %u\n", rx_ring->qid, req_id);
975 		ena_increase_stat(&rx_ring->rx_stats.bad_req_id, 1, &rx_ring->syncp);
976 		ena_reset_device(adapter, ENA_REGS_RESET_INV_RX_REQ_ID);
977 		return NULL;
978 	}
979 
980 	netif_dbg(rx_ring->adapter, rx_status, rx_ring->netdev,
981 		  "rx_info %p page %p\n",
982 		  rx_info, rx_info->page);
983 
984 	buf_offset = rx_info->buf_offset;
985 	pkt_offset = buf_offset - rx_ring->rx_headroom;
986 	page_offset = rx_info->page_offset;
987 	buf_addr = page_address(rx_info->page) + page_offset;
988 
989 	if (len <= rx_ring->rx_copybreak && likely(descs == 1)) {
990 		skb = ena_alloc_skb(rx_ring, NULL, len);
991 		if (unlikely(!skb))
992 			return NULL;
993 
994 		skb_copy_to_linear_data(skb, buf_addr + buf_offset, len);
995 		dma_sync_single_for_device(rx_ring->dev,
996 					   dma_unmap_addr(&rx_info->ena_buf, paddr) + pkt_offset,
997 					   len,
998 					   DMA_FROM_DEVICE);
999 
1000 		skb_put(skb, len);
1001 		netif_dbg(rx_ring->adapter, rx_status, rx_ring->netdev,
1002 			  "RX allocated small packet. len %d.\n", skb->len);
1003 		skb->protocol = eth_type_trans(skb, rx_ring->netdev);
1004 		rx_ring->free_ids[*next_to_clean] = req_id;
1005 		*next_to_clean = ENA_RX_RING_IDX_ADD(*next_to_clean, descs,
1006 						     rx_ring->ring_size);
1007 		return skb;
1008 	}
1009 
1010 	buf_len = SKB_DATA_ALIGN(len + buf_offset + tailroom);
1011 
1012 	skb = ena_alloc_skb(rx_ring, buf_addr, buf_len);
1013 	if (unlikely(!skb))
1014 		return NULL;
1015 
1016 	/* If XDP isn't loaded try to reuse part of the RX buffer */
1017 	reuse_rx_buf_page = !is_xdp_loaded &&
1018 			    ena_try_rx_buf_page_reuse(rx_info, buf_len, len, pkt_offset);
1019 
1020 	if (!reuse_rx_buf_page)
1021 		ena_unmap_rx_buff_attrs(rx_ring, rx_info, DMA_ATTR_SKIP_CPU_SYNC);
1022 
1023 	/* Populate skb's linear part */
1024 	skb_reserve(skb, buf_offset);
1025 	skb_put(skb, len);
1026 	skb->protocol = eth_type_trans(skb, rx_ring->netdev);
1027 
1028 	do {
1029 		netif_dbg(rx_ring->adapter, rx_status, rx_ring->netdev,
1030 			  "RX skb updated. len %d. data_len %d\n",
1031 			  skb->len, skb->data_len);
1032 
1033 		if (!reuse_rx_buf_page)
1034 			rx_info->page = NULL;
1035 
1036 		rx_ring->free_ids[*next_to_clean] = req_id;
1037 		*next_to_clean =
1038 			ENA_RX_RING_IDX_NEXT(*next_to_clean,
1039 					     rx_ring->ring_size);
1040 		if (likely(--descs == 0))
1041 			break;
1042 
1043 		buf++;
1044 		len = ena_bufs[buf].len;
1045 		req_id = ena_bufs[buf].req_id;
1046 
1047 		rx_info = &rx_ring->rx_buffer_info[req_id];
1048 
1049 		/* rx_info->buf_offset includes rx_ring->rx_headroom */
1050 		buf_offset = rx_info->buf_offset;
1051 		pkt_offset = buf_offset - rx_ring->rx_headroom;
1052 		buf_len = SKB_DATA_ALIGN(len + buf_offset + tailroom);
1053 		page_offset = rx_info->page_offset;
1054 
1055 		pre_reuse_paddr = dma_unmap_addr(&rx_info->ena_buf, paddr);
1056 
1057 		reuse_rx_buf_page = !is_xdp_loaded &&
1058 				    ena_try_rx_buf_page_reuse(rx_info, buf_len, len, pkt_offset);
1059 
1060 		dma_sync_single_for_cpu(rx_ring->dev,
1061 					pre_reuse_paddr + pkt_offset,
1062 					len,
1063 					DMA_FROM_DEVICE);
1064 
1065 		if (!reuse_rx_buf_page)
1066 			ena_unmap_rx_buff_attrs(rx_ring, rx_info, DMA_ATTR_SKIP_CPU_SYNC);
1067 
1068 		skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, rx_info->page,
1069 				page_offset + buf_offset, len, buf_len);
1070 
1071 	} while (1);
1072 
1073 	return skb;
1074 }
1075 
1076 /* ena_rx_checksum - indicate in skb if hw indicated a good cksum
1077  * @adapter: structure containing adapter specific data
1078  * @ena_rx_ctx: received packet context/metadata
1079  * @skb: skb currently being received and modified
1080  */
1081 static void ena_rx_checksum(struct ena_ring *rx_ring,
1082 				   struct ena_com_rx_ctx *ena_rx_ctx,
1083 				   struct sk_buff *skb)
1084 {
1085 	/* Rx csum disabled */
1086 	if (unlikely(!(rx_ring->netdev->features & NETIF_F_RXCSUM))) {
1087 		skb->ip_summed = CHECKSUM_NONE;
1088 		return;
1089 	}
1090 
1091 	/* For fragmented packets the checksum isn't valid */
1092 	if (ena_rx_ctx->frag) {
1093 		skb->ip_summed = CHECKSUM_NONE;
1094 		return;
1095 	}
1096 
1097 	/* if IP and error */
1098 	if (unlikely((ena_rx_ctx->l3_proto == ENA_ETH_IO_L3_PROTO_IPV4) &&
1099 		     (ena_rx_ctx->l3_csum_err))) {
1100 		/* ipv4 checksum error */
1101 		skb->ip_summed = CHECKSUM_NONE;
1102 		ena_increase_stat(&rx_ring->rx_stats.csum_bad, 1,
1103 				  &rx_ring->syncp);
1104 		netif_dbg(rx_ring->adapter, rx_err, rx_ring->netdev,
1105 			  "RX IPv4 header checksum error\n");
1106 		return;
1107 	}
1108 
1109 	/* if TCP/UDP */
1110 	if (likely((ena_rx_ctx->l4_proto == ENA_ETH_IO_L4_PROTO_TCP) ||
1111 		   (ena_rx_ctx->l4_proto == ENA_ETH_IO_L4_PROTO_UDP))) {
1112 		if (unlikely(ena_rx_ctx->l4_csum_err)) {
1113 			/* TCP/UDP checksum error */
1114 			ena_increase_stat(&rx_ring->rx_stats.csum_bad, 1,
1115 					  &rx_ring->syncp);
1116 			netif_dbg(rx_ring->adapter, rx_err, rx_ring->netdev,
1117 				  "RX L4 checksum error\n");
1118 			skb->ip_summed = CHECKSUM_NONE;
1119 			return;
1120 		}
1121 
1122 		if (likely(ena_rx_ctx->l4_csum_checked)) {
1123 			skb->ip_summed = CHECKSUM_UNNECESSARY;
1124 			ena_increase_stat(&rx_ring->rx_stats.csum_good, 1,
1125 					  &rx_ring->syncp);
1126 		} else {
1127 			ena_increase_stat(&rx_ring->rx_stats.csum_unchecked, 1,
1128 					  &rx_ring->syncp);
1129 			skb->ip_summed = CHECKSUM_NONE;
1130 		}
1131 	} else {
1132 		skb->ip_summed = CHECKSUM_NONE;
1133 		return;
1134 	}
1135 
1136 }
1137 
1138 static void ena_set_rx_hash(struct ena_ring *rx_ring,
1139 			    struct ena_com_rx_ctx *ena_rx_ctx,
1140 			    struct sk_buff *skb)
1141 {
1142 	enum pkt_hash_types hash_type;
1143 
1144 	if (likely(rx_ring->netdev->features & NETIF_F_RXHASH)) {
1145 		if (likely((ena_rx_ctx->l4_proto == ENA_ETH_IO_L4_PROTO_TCP) ||
1146 			   (ena_rx_ctx->l4_proto == ENA_ETH_IO_L4_PROTO_UDP)))
1147 
1148 			hash_type = PKT_HASH_TYPE_L4;
1149 		else
1150 			hash_type = PKT_HASH_TYPE_NONE;
1151 
1152 		/* Override hash type if the packet is fragmented */
1153 		if (ena_rx_ctx->frag)
1154 			hash_type = PKT_HASH_TYPE_NONE;
1155 
1156 		skb_set_hash(skb, ena_rx_ctx->hash, hash_type);
1157 	}
1158 }
1159 
1160 static int ena_xdp_handle_buff(struct ena_ring *rx_ring, struct xdp_buff *xdp, u16 num_descs)
1161 {
1162 	struct ena_rx_buffer *rx_info;
1163 	int ret;
1164 
1165 	/* XDP multi-buffer packets not supported */
1166 	if (unlikely(num_descs > 1)) {
1167 		netdev_err_once(rx_ring->adapter->netdev,
1168 				"xdp: dropped unsupported multi-buffer packets\n");
1169 		ena_increase_stat(&rx_ring->rx_stats.xdp_drop, 1, &rx_ring->syncp);
1170 		return ENA_XDP_DROP;
1171 	}
1172 
1173 	rx_info = &rx_ring->rx_buffer_info[rx_ring->ena_bufs[0].req_id];
1174 	xdp_prepare_buff(xdp, page_address(rx_info->page),
1175 			 rx_info->buf_offset,
1176 			 rx_ring->ena_bufs[0].len, false);
1177 
1178 	ret = ena_xdp_execute(rx_ring, xdp);
1179 
1180 	/* The xdp program might expand the headers */
1181 	if (ret == ENA_XDP_PASS) {
1182 		rx_info->buf_offset = xdp->data - xdp->data_hard_start;
1183 		rx_ring->ena_bufs[0].len = xdp->data_end - xdp->data;
1184 	}
1185 
1186 	return ret;
1187 }
1188 
1189 /* ena_clean_rx_irq - Cleanup RX irq
1190  * @rx_ring: RX ring to clean
1191  * @napi: napi handler
1192  * @budget: how many packets driver is allowed to clean
1193  *
1194  * Returns the number of cleaned buffers.
1195  */
1196 static int ena_clean_rx_irq(struct ena_ring *rx_ring, struct napi_struct *napi,
1197 			    u32 budget)
1198 {
1199 	u16 next_to_clean = rx_ring->next_to_clean;
1200 	struct ena_com_rx_ctx ena_rx_ctx;
1201 	struct ena_rx_buffer *rx_info;
1202 	struct ena_adapter *adapter;
1203 	u32 res_budget, work_done;
1204 	int rx_copybreak_pkt = 0;
1205 	int refill_threshold;
1206 	struct sk_buff *skb;
1207 	int refill_required;
1208 	struct xdp_buff xdp;
1209 	int xdp_flags = 0;
1210 	int total_len = 0;
1211 	int xdp_verdict;
1212 	u8 pkt_offset;
1213 	int rc = 0;
1214 	int i;
1215 
1216 	netif_dbg(rx_ring->adapter, rx_status, rx_ring->netdev,
1217 		  "%s qid %d\n", __func__, rx_ring->qid);
1218 	res_budget = budget;
1219 	xdp_init_buff(&xdp, ENA_PAGE_SIZE, &rx_ring->xdp_rxq);
1220 
1221 	do {
1222 		xdp_verdict = ENA_XDP_PASS;
1223 		skb = NULL;
1224 		ena_rx_ctx.ena_bufs = rx_ring->ena_bufs;
1225 		ena_rx_ctx.max_bufs = rx_ring->sgl_size;
1226 		ena_rx_ctx.descs = 0;
1227 		ena_rx_ctx.pkt_offset = 0;
1228 		rc = ena_com_rx_pkt(rx_ring->ena_com_io_cq,
1229 				    rx_ring->ena_com_io_sq,
1230 				    &ena_rx_ctx);
1231 		if (unlikely(rc))
1232 			goto error;
1233 
1234 		if (unlikely(ena_rx_ctx.descs == 0))
1235 			break;
1236 
1237 		/* First descriptor might have an offset set by the device */
1238 		rx_info = &rx_ring->rx_buffer_info[rx_ring->ena_bufs[0].req_id];
1239 		pkt_offset = ena_rx_ctx.pkt_offset;
1240 		rx_info->buf_offset += pkt_offset;
1241 
1242 		netif_dbg(rx_ring->adapter, rx_status, rx_ring->netdev,
1243 			  "rx_poll: q %d got packet from ena. descs #: %d l3 proto %d l4 proto %d hash: %x\n",
1244 			  rx_ring->qid, ena_rx_ctx.descs, ena_rx_ctx.l3_proto,
1245 			  ena_rx_ctx.l4_proto, ena_rx_ctx.hash);
1246 
1247 		dma_sync_single_for_cpu(rx_ring->dev,
1248 					dma_unmap_addr(&rx_info->ena_buf, paddr) + pkt_offset,
1249 					rx_ring->ena_bufs[0].len,
1250 					DMA_FROM_DEVICE);
1251 
1252 		if (ena_xdp_present_ring(rx_ring))
1253 			xdp_verdict = ena_xdp_handle_buff(rx_ring, &xdp, ena_rx_ctx.descs);
1254 
1255 		/* allocate skb and fill it */
1256 		if (xdp_verdict == ENA_XDP_PASS)
1257 			skb = ena_rx_skb(rx_ring,
1258 					 rx_ring->ena_bufs,
1259 					 ena_rx_ctx.descs,
1260 					 &next_to_clean);
1261 
1262 		if (unlikely(!skb)) {
1263 			for (i = 0; i < ena_rx_ctx.descs; i++) {
1264 				int req_id = rx_ring->ena_bufs[i].req_id;
1265 
1266 				rx_ring->free_ids[next_to_clean] = req_id;
1267 				next_to_clean =
1268 					ENA_RX_RING_IDX_NEXT(next_to_clean,
1269 							     rx_ring->ring_size);
1270 
1271 				/* Packets was passed for transmission, unmap it
1272 				 * from RX side.
1273 				 */
1274 				if (xdp_verdict & ENA_XDP_FORWARDED) {
1275 					ena_unmap_rx_buff_attrs(rx_ring,
1276 								&rx_ring->rx_buffer_info[req_id],
1277 								DMA_ATTR_SKIP_CPU_SYNC);
1278 					rx_ring->rx_buffer_info[req_id].page = NULL;
1279 				}
1280 			}
1281 			if (xdp_verdict != ENA_XDP_PASS) {
1282 				xdp_flags |= xdp_verdict;
1283 				total_len += ena_rx_ctx.ena_bufs[0].len;
1284 				res_budget--;
1285 				continue;
1286 			}
1287 			break;
1288 		}
1289 
1290 		ena_rx_checksum(rx_ring, &ena_rx_ctx, skb);
1291 
1292 		ena_set_rx_hash(rx_ring, &ena_rx_ctx, skb);
1293 
1294 		skb_record_rx_queue(skb, rx_ring->qid);
1295 
1296 		if (rx_ring->ena_bufs[0].len <= rx_ring->rx_copybreak &&
1297 		    ena_rx_ctx.descs == 1)
1298 			rx_copybreak_pkt++;
1299 
1300 		total_len += skb->len;
1301 
1302 		napi_gro_receive(napi, skb);
1303 
1304 		res_budget--;
1305 	} while (likely(res_budget));
1306 
1307 	work_done = budget - res_budget;
1308 	rx_ring->per_napi_packets += work_done;
1309 	u64_stats_update_begin(&rx_ring->syncp);
1310 	rx_ring->rx_stats.bytes += total_len;
1311 	rx_ring->rx_stats.cnt += work_done;
1312 	rx_ring->rx_stats.rx_copybreak_pkt += rx_copybreak_pkt;
1313 	u64_stats_update_end(&rx_ring->syncp);
1314 
1315 	rx_ring->next_to_clean = next_to_clean;
1316 
1317 	refill_required = ena_com_free_q_entries(rx_ring->ena_com_io_sq);
1318 	refill_threshold =
1319 		min_t(int, rx_ring->ring_size / ENA_RX_REFILL_THRESH_DIVIDER,
1320 		      ENA_RX_REFILL_THRESH_PACKET);
1321 
1322 	/* Optimization, try to batch new rx buffers */
1323 	if (refill_required > refill_threshold)
1324 		ena_refill_rx_bufs(rx_ring, refill_required);
1325 
1326 	if (xdp_flags & ENA_XDP_REDIRECT)
1327 		xdp_do_flush();
1328 
1329 	return work_done;
1330 
1331 error:
1332 	if (xdp_flags & ENA_XDP_REDIRECT)
1333 		xdp_do_flush();
1334 
1335 	adapter = netdev_priv(rx_ring->netdev);
1336 
1337 	if (rc == -ENOSPC) {
1338 		ena_increase_stat(&rx_ring->rx_stats.bad_desc_num, 1, &rx_ring->syncp);
1339 		ena_reset_device(adapter, ENA_REGS_RESET_TOO_MANY_RX_DESCS);
1340 	} else if (rc == -EFAULT) {
1341 		ena_reset_device(adapter, ENA_REGS_RESET_RX_DESCRIPTOR_MALFORMED);
1342 	} else {
1343 		ena_increase_stat(&rx_ring->rx_stats.bad_req_id, 1,
1344 				  &rx_ring->syncp);
1345 		ena_reset_device(adapter, ENA_REGS_RESET_INV_RX_REQ_ID);
1346 	}
1347 	return 0;
1348 }
1349 
1350 static void ena_dim_work(struct work_struct *w)
1351 {
1352 	struct dim *dim = container_of(w, struct dim, work);
1353 	struct dim_cq_moder cur_moder =
1354 		net_dim_get_rx_moderation(dim->mode, dim->profile_ix);
1355 	struct ena_napi *ena_napi = container_of(dim, struct ena_napi, dim);
1356 
1357 	ena_napi->rx_ring->smoothed_interval = cur_moder.usec;
1358 	dim->state = DIM_START_MEASURE;
1359 }
1360 
1361 static void ena_adjust_adaptive_rx_intr_moderation(struct ena_napi *ena_napi)
1362 {
1363 	struct dim_sample dim_sample;
1364 	struct ena_ring *rx_ring = ena_napi->rx_ring;
1365 
1366 	if (!rx_ring->per_napi_packets)
1367 		return;
1368 
1369 	rx_ring->non_empty_napi_events++;
1370 
1371 	dim_update_sample(rx_ring->non_empty_napi_events,
1372 			  rx_ring->rx_stats.cnt,
1373 			  rx_ring->rx_stats.bytes,
1374 			  &dim_sample);
1375 
1376 	net_dim(&ena_napi->dim, &dim_sample);
1377 
1378 	rx_ring->per_napi_packets = 0;
1379 }
1380 
1381 void ena_unmask_interrupt(struct ena_ring *tx_ring,
1382 			  struct ena_ring *rx_ring)
1383 {
1384 	u32 rx_interval = tx_ring->smoothed_interval;
1385 	struct ena_eth_io_intr_reg intr_reg;
1386 
1387 	/* Rx ring can be NULL when for XDP tx queues which don't have an
1388 	 * accompanying rx_ring pair.
1389 	 */
1390 	if (rx_ring)
1391 		rx_interval = ena_com_get_adaptive_moderation_enabled(rx_ring->ena_dev) ?
1392 			rx_ring->smoothed_interval :
1393 			ena_com_get_nonadaptive_moderation_interval_rx(rx_ring->ena_dev);
1394 
1395 	/* Update intr register: rx intr delay,
1396 	 * tx intr delay and interrupt unmask
1397 	 */
1398 	ena_com_update_intr_reg(&intr_reg,
1399 				rx_interval,
1400 				tx_ring->smoothed_interval,
1401 				true);
1402 
1403 	ena_increase_stat(&tx_ring->tx_stats.unmask_interrupt, 1,
1404 			  &tx_ring->syncp);
1405 
1406 	/* It is a shared MSI-X.
1407 	 * Tx and Rx CQ have pointer to it.
1408 	 * So we use one of them to reach the intr reg
1409 	 * The Tx ring is used because the rx_ring is NULL for XDP queues
1410 	 */
1411 	ena_com_unmask_intr(tx_ring->ena_com_io_cq, &intr_reg);
1412 }
1413 
1414 void ena_update_ring_numa_node(struct ena_ring *tx_ring,
1415 			       struct ena_ring *rx_ring)
1416 {
1417 	int cpu = get_cpu();
1418 	int numa_node;
1419 
1420 	/* Check only one ring since the 2 rings are running on the same cpu */
1421 	if (likely(tx_ring->cpu == cpu))
1422 		goto out;
1423 
1424 	tx_ring->cpu = cpu;
1425 	if (rx_ring)
1426 		rx_ring->cpu = cpu;
1427 
1428 	numa_node = cpu_to_node(cpu);
1429 
1430 	if (likely(tx_ring->numa_node == numa_node))
1431 		goto out;
1432 
1433 	put_cpu();
1434 
1435 	if (numa_node != NUMA_NO_NODE) {
1436 		ena_com_update_numa_node(tx_ring->ena_com_io_cq, numa_node);
1437 		tx_ring->numa_node = numa_node;
1438 		if (rx_ring) {
1439 			rx_ring->numa_node = numa_node;
1440 			ena_com_update_numa_node(rx_ring->ena_com_io_cq,
1441 						 numa_node);
1442 		}
1443 	}
1444 
1445 	return;
1446 out:
1447 	put_cpu();
1448 }
1449 
1450 static int ena_io_poll(struct napi_struct *napi, int budget)
1451 {
1452 	struct ena_napi *ena_napi = container_of(napi, struct ena_napi, napi);
1453 	struct ena_ring *tx_ring, *rx_ring;
1454 	int tx_work_done;
1455 	int rx_work_done = 0;
1456 	int tx_budget;
1457 	int napi_comp_call = 0;
1458 	int ret;
1459 
1460 	tx_ring = ena_napi->tx_ring;
1461 	rx_ring = ena_napi->rx_ring;
1462 
1463 	tx_budget = tx_ring->ring_size / ENA_TX_POLL_BUDGET_DIVIDER;
1464 
1465 	if (!test_bit(ENA_FLAG_DEV_UP, &tx_ring->adapter->flags) ||
1466 	    test_bit(ENA_FLAG_TRIGGER_RESET, &tx_ring->adapter->flags)) {
1467 		napi_complete_done(napi, 0);
1468 		return 0;
1469 	}
1470 
1471 	tx_work_done = ena_clean_tx_irq(tx_ring, tx_budget);
1472 	/* On netpoll the budget is zero and the handler should only clean the
1473 	 * tx completions.
1474 	 */
1475 	if (likely(budget))
1476 		rx_work_done = ena_clean_rx_irq(rx_ring, napi, budget);
1477 
1478 	/* If the device is about to reset or down, avoid unmask
1479 	 * the interrupt and return 0 so NAPI won't reschedule
1480 	 */
1481 	if (unlikely(!test_bit(ENA_FLAG_DEV_UP, &tx_ring->adapter->flags) ||
1482 		     test_bit(ENA_FLAG_TRIGGER_RESET, &tx_ring->adapter->flags))) {
1483 		napi_complete_done(napi, 0);
1484 		ret = 0;
1485 
1486 	} else if ((budget > rx_work_done) && (tx_budget > tx_work_done)) {
1487 		napi_comp_call = 1;
1488 
1489 		/* Update numa and unmask the interrupt only when schedule
1490 		 * from the interrupt context (vs from sk_busy_loop)
1491 		 */
1492 		if (napi_complete_done(napi, rx_work_done) &&
1493 		    READ_ONCE(ena_napi->interrupts_masked)) {
1494 			smp_rmb(); /* make sure interrupts_masked is read */
1495 			WRITE_ONCE(ena_napi->interrupts_masked, false);
1496 			/* We apply adaptive moderation on Rx path only.
1497 			 * Tx uses static interrupt moderation.
1498 			 */
1499 			if (ena_com_get_adaptive_moderation_enabled(rx_ring->ena_dev))
1500 				ena_adjust_adaptive_rx_intr_moderation(ena_napi);
1501 
1502 			ena_update_ring_numa_node(tx_ring, rx_ring);
1503 			ena_unmask_interrupt(tx_ring, rx_ring);
1504 		}
1505 
1506 		ret = rx_work_done;
1507 	} else {
1508 		ret = budget;
1509 	}
1510 
1511 	u64_stats_update_begin(&tx_ring->syncp);
1512 	tx_ring->tx_stats.napi_comp += napi_comp_call;
1513 	tx_ring->tx_stats.tx_poll++;
1514 	u64_stats_update_end(&tx_ring->syncp);
1515 
1516 	tx_ring->tx_stats.last_napi_jiffies = jiffies;
1517 
1518 	return ret;
1519 }
1520 
1521 static irqreturn_t ena_intr_msix_mgmnt(int irq, void *data)
1522 {
1523 	struct ena_adapter *adapter = (struct ena_adapter *)data;
1524 
1525 	ena_com_admin_q_comp_intr_handler(adapter->ena_dev);
1526 
1527 	/* Don't call the aenq handler before probe is done */
1528 	if (likely(test_bit(ENA_FLAG_DEVICE_RUNNING, &adapter->flags)))
1529 		ena_com_aenq_intr_handler(adapter->ena_dev, data);
1530 
1531 	return IRQ_HANDLED;
1532 }
1533 
1534 /* ena_intr_msix_io - MSI-X Interrupt Handler for Tx/Rx
1535  * @irq: interrupt number
1536  * @data: pointer to a network interface private napi device structure
1537  */
1538 static irqreturn_t ena_intr_msix_io(int irq, void *data)
1539 {
1540 	struct ena_napi *ena_napi = data;
1541 
1542 	/* Used to check HW health */
1543 	WRITE_ONCE(ena_napi->first_interrupt, true);
1544 
1545 	WRITE_ONCE(ena_napi->interrupts_masked, true);
1546 	smp_wmb(); /* write interrupts_masked before calling napi */
1547 
1548 	napi_schedule_irqoff(&ena_napi->napi);
1549 
1550 	return IRQ_HANDLED;
1551 }
1552 
1553 /* Reserve a single MSI-X vector for management (admin + aenq).
1554  * plus reserve one vector for each potential io queue.
1555  * the number of potential io queues is the minimum of what the device
1556  * supports and the number of vCPUs.
1557  */
1558 static int ena_enable_msix(struct ena_adapter *adapter)
1559 {
1560 	int msix_vecs, irq_cnt;
1561 
1562 	if (test_bit(ENA_FLAG_MSIX_ENABLED, &adapter->flags)) {
1563 		netif_err(adapter, probe, adapter->netdev,
1564 			  "Error, MSI-X is already enabled\n");
1565 		return -EPERM;
1566 	}
1567 
1568 	/* Reserved the max msix vectors we might need */
1569 	msix_vecs = ENA_MAX_MSIX_VEC(adapter->max_num_io_queues);
1570 	netif_dbg(adapter, probe, adapter->netdev,
1571 		  "Trying to enable MSI-X, vectors %d\n", msix_vecs);
1572 
1573 	irq_cnt = pci_alloc_irq_vectors(adapter->pdev, ENA_MIN_MSIX_VEC,
1574 					msix_vecs, PCI_IRQ_MSIX);
1575 
1576 	if (irq_cnt < 0) {
1577 		netif_err(adapter, probe, adapter->netdev,
1578 			  "Failed to enable MSI-X. irq_cnt %d\n", irq_cnt);
1579 		return -ENOSPC;
1580 	}
1581 
1582 	if (irq_cnt != msix_vecs) {
1583 		netif_notice(adapter, probe, adapter->netdev,
1584 			     "Enable only %d MSI-X (out of %d), reduce the number of queues\n",
1585 			     irq_cnt, msix_vecs);
1586 		adapter->num_io_queues = irq_cnt - ENA_ADMIN_MSIX_VEC;
1587 	}
1588 
1589 	if (netif_enable_cpu_rmap(adapter->netdev, adapter->num_io_queues))
1590 		netif_warn(adapter, probe, adapter->netdev,
1591 			   "Failed to map IRQs to CPUs\n");
1592 
1593 	adapter->msix_vecs = irq_cnt;
1594 	set_bit(ENA_FLAG_MSIX_ENABLED, &adapter->flags);
1595 
1596 	return 0;
1597 }
1598 
1599 static void ena_setup_mgmnt_intr(struct ena_adapter *adapter)
1600 {
1601 	u32 cpu;
1602 
1603 	snprintf(adapter->irq_tbl[ENA_MGMNT_IRQ_IDX].name,
1604 		 ENA_IRQNAME_SIZE, "ena-mgmnt@pci:%s",
1605 		 pci_name(adapter->pdev));
1606 	adapter->irq_tbl[ENA_MGMNT_IRQ_IDX].handler =
1607 		ena_intr_msix_mgmnt;
1608 	adapter->irq_tbl[ENA_MGMNT_IRQ_IDX].data = adapter;
1609 	adapter->irq_tbl[ENA_MGMNT_IRQ_IDX].vector =
1610 		pci_irq_vector(adapter->pdev, ENA_MGMNT_IRQ_IDX);
1611 	cpu = cpumask_first(cpu_online_mask);
1612 	adapter->irq_tbl[ENA_MGMNT_IRQ_IDX].cpu = cpu;
1613 	cpumask_set_cpu(cpu,
1614 			&adapter->irq_tbl[ENA_MGMNT_IRQ_IDX].affinity_hint_mask);
1615 }
1616 
1617 static void ena_setup_io_intr(struct ena_adapter *adapter)
1618 {
1619 	struct net_device *netdev;
1620 	int irq_idx, i, cpu;
1621 	int io_queue_count;
1622 
1623 	netdev = adapter->netdev;
1624 	io_queue_count = adapter->num_io_queues + adapter->xdp_num_queues;
1625 
1626 	for (i = 0; i < io_queue_count; i++) {
1627 		irq_idx = ENA_IO_IRQ_IDX(i);
1628 		cpu = i % num_online_cpus();
1629 
1630 		snprintf(adapter->irq_tbl[irq_idx].name, ENA_IRQNAME_SIZE,
1631 			 "%s-Tx-Rx-%d", netdev->name, i);
1632 		adapter->irq_tbl[irq_idx].handler = ena_intr_msix_io;
1633 		adapter->irq_tbl[irq_idx].data = &adapter->ena_napi[i];
1634 		adapter->irq_tbl[irq_idx].vector =
1635 			pci_irq_vector(adapter->pdev, irq_idx);
1636 		adapter->irq_tbl[irq_idx].cpu = cpu;
1637 
1638 		cpumask_set_cpu(cpu,
1639 				&adapter->irq_tbl[irq_idx].affinity_hint_mask);
1640 	}
1641 }
1642 
1643 static int ena_request_mgmnt_irq(struct ena_adapter *adapter)
1644 {
1645 	unsigned long flags = 0;
1646 	struct ena_irq *irq;
1647 	int rc;
1648 
1649 	irq = &adapter->irq_tbl[ENA_MGMNT_IRQ_IDX];
1650 	rc = request_irq(irq->vector, irq->handler, flags, irq->name,
1651 			 irq->data);
1652 	if (rc) {
1653 		netif_err(adapter, probe, adapter->netdev,
1654 			  "Failed to request admin irq\n");
1655 		return rc;
1656 	}
1657 
1658 	netif_dbg(adapter, probe, adapter->netdev,
1659 		  "Set affinity hint of mgmnt irq.to 0x%lx (irq vector: %d)\n",
1660 		  irq->affinity_hint_mask.bits[0], irq->vector);
1661 
1662 	irq_set_affinity_hint(irq->vector, &irq->affinity_hint_mask);
1663 
1664 	return rc;
1665 }
1666 
1667 static int ena_request_io_irq(struct ena_adapter *adapter)
1668 {
1669 	u32 io_queue_count = adapter->num_io_queues + adapter->xdp_num_queues;
1670 	int rc = 0, i, k, irq_idx;
1671 	unsigned long flags = 0;
1672 	struct ena_irq *irq;
1673 
1674 	if (!test_bit(ENA_FLAG_MSIX_ENABLED, &adapter->flags)) {
1675 		netif_err(adapter, ifup, adapter->netdev,
1676 			  "Failed to request I/O IRQ: MSI-X is not enabled\n");
1677 		return -EINVAL;
1678 	}
1679 
1680 	for (i = ENA_IO_IRQ_FIRST_IDX; i < ENA_MAX_MSIX_VEC(io_queue_count); i++) {
1681 		irq = &adapter->irq_tbl[i];
1682 		rc = request_irq(irq->vector, irq->handler, flags, irq->name,
1683 				 irq->data);
1684 		if (rc) {
1685 			netif_err(adapter, ifup, adapter->netdev,
1686 				  "Failed to request I/O IRQ. index %d rc %d\n",
1687 				   i, rc);
1688 			goto err;
1689 		}
1690 
1691 		netif_dbg(adapter, ifup, adapter->netdev,
1692 			  "Set affinity hint of irq. index %d to 0x%lx (irq vector: %d)\n",
1693 			  i, irq->affinity_hint_mask.bits[0], irq->vector);
1694 
1695 		irq_set_affinity_hint(irq->vector, &irq->affinity_hint_mask);
1696 	}
1697 
1698 	/* Now that IO IRQs have been successfully allocated map them to the
1699 	 * corresponding IO NAPI instance. Note that the mgmnt IRQ does not
1700 	 * have a NAPI, so care must be taken to correctly map IRQs to NAPIs.
1701 	 */
1702 	for (i = 0; i < io_queue_count; i++) {
1703 		irq_idx = ENA_IO_IRQ_IDX(i);
1704 		irq = &adapter->irq_tbl[irq_idx];
1705 		netif_napi_set_irq(&adapter->ena_napi[i].napi, irq->vector);
1706 	}
1707 
1708 	return rc;
1709 
1710 err:
1711 	for (k = ENA_IO_IRQ_FIRST_IDX; k < i; k++) {
1712 		irq = &adapter->irq_tbl[k];
1713 		free_irq(irq->vector, irq->data);
1714 	}
1715 
1716 	return rc;
1717 }
1718 
1719 static void ena_free_mgmnt_irq(struct ena_adapter *adapter)
1720 {
1721 	struct ena_irq *irq;
1722 
1723 	irq = &adapter->irq_tbl[ENA_MGMNT_IRQ_IDX];
1724 	synchronize_irq(irq->vector);
1725 	irq_set_affinity_hint(irq->vector, NULL);
1726 	free_irq(irq->vector, irq->data);
1727 }
1728 
1729 static void ena_free_io_irq(struct ena_adapter *adapter)
1730 {
1731 	u32 io_queue_count = adapter->num_io_queues + adapter->xdp_num_queues;
1732 	struct ena_irq *irq;
1733 	int i;
1734 
1735 	for (i = ENA_IO_IRQ_FIRST_IDX; i < ENA_MAX_MSIX_VEC(io_queue_count); i++) {
1736 		struct ena_napi *ena_napi;
1737 
1738 		irq = &adapter->irq_tbl[i];
1739 		irq_set_affinity_hint(irq->vector, NULL);
1740 		ena_napi = irq->data;
1741 		netif_napi_set_irq(&ena_napi->napi, -1);
1742 		free_irq(irq->vector, irq->data);
1743 	}
1744 }
1745 
1746 static void ena_disable_msix(struct ena_adapter *adapter)
1747 {
1748 	if (test_and_clear_bit(ENA_FLAG_MSIX_ENABLED, &adapter->flags))
1749 		pci_free_irq_vectors(adapter->pdev);
1750 }
1751 
1752 static void ena_disable_io_intr_sync(struct ena_adapter *adapter)
1753 {
1754 	u32 io_queue_count = adapter->num_io_queues + adapter->xdp_num_queues;
1755 	int i;
1756 
1757 	if (!netif_running(adapter->netdev))
1758 		return;
1759 
1760 	for (i = ENA_IO_IRQ_FIRST_IDX; i < ENA_MAX_MSIX_VEC(io_queue_count); i++)
1761 		synchronize_irq(adapter->irq_tbl[i].vector);
1762 }
1763 
1764 static void ena_del_napi_in_range(struct ena_adapter *adapter,
1765 				  int first_index,
1766 				  int count)
1767 {
1768 	int i;
1769 
1770 	for (i = first_index; i < first_index + count; i++) {
1771 		netif_napi_del(&adapter->ena_napi[i].napi);
1772 
1773 		WARN_ON(ENA_IS_XDP_INDEX(adapter, i) &&
1774 			adapter->ena_napi[i].rx_ring);
1775 	}
1776 }
1777 
1778 static void ena_init_napi_in_range(struct ena_adapter *adapter,
1779 				   int first_index, int count)
1780 {
1781 	int (*napi_handler)(struct napi_struct *napi, int budget);
1782 	int i;
1783 
1784 	for (i = first_index; i < first_index + count; i++) {
1785 		struct ena_napi *napi = &adapter->ena_napi[i];
1786 		struct ena_ring *rx_ring, *tx_ring;
1787 
1788 		memset(napi, 0, sizeof(*napi));
1789 
1790 		rx_ring = &adapter->rx_ring[i];
1791 		tx_ring = &adapter->tx_ring[i];
1792 
1793 		napi_handler = ena_io_poll;
1794 		if (ENA_IS_XDP_INDEX(adapter, i))
1795 			napi_handler = ena_xdp_io_poll;
1796 
1797 		netif_napi_add_config(adapter->netdev, &napi->napi, napi_handler, i);
1798 
1799 		if (!ENA_IS_XDP_INDEX(adapter, i))
1800 			napi->rx_ring = rx_ring;
1801 
1802 		napi->tx_ring = tx_ring;
1803 		napi->qid = i;
1804 	}
1805 }
1806 
1807 static void ena_napi_disable_in_range(struct ena_adapter *adapter,
1808 				      int first_index,
1809 				      int count)
1810 {
1811 	struct napi_struct *napi;
1812 	int i;
1813 
1814 	for (i = first_index; i < first_index + count; i++) {
1815 		napi = &adapter->ena_napi[i].napi;
1816 		if (!ENA_IS_XDP_INDEX(adapter, i)) {
1817 			/* This API is supported for non-XDP queues only */
1818 			netif_queue_set_napi(adapter->netdev, i,
1819 					     NETDEV_QUEUE_TYPE_TX, NULL);
1820 			netif_queue_set_napi(adapter->netdev, i,
1821 					     NETDEV_QUEUE_TYPE_RX, NULL);
1822 		}
1823 		napi_disable(napi);
1824 	}
1825 }
1826 
1827 static void ena_napi_enable_in_range(struct ena_adapter *adapter,
1828 				     int first_index,
1829 				     int count)
1830 {
1831 	struct napi_struct *napi;
1832 	int i;
1833 
1834 	for (i = first_index; i < first_index + count; i++) {
1835 		napi = &adapter->ena_napi[i].napi;
1836 		napi_enable(napi);
1837 		if (!ENA_IS_XDP_INDEX(adapter, i)) {
1838 			/* This API is supported for non-XDP queues only */
1839 			netif_queue_set_napi(adapter->netdev, i,
1840 					     NETDEV_QUEUE_TYPE_RX, napi);
1841 			netif_queue_set_napi(adapter->netdev, i,
1842 					     NETDEV_QUEUE_TYPE_TX, napi);
1843 		}
1844 	}
1845 }
1846 
1847 /* Configure the Rx forwarding */
1848 static int ena_rss_configure(struct ena_adapter *adapter)
1849 {
1850 	struct ena_com_dev *ena_dev = adapter->ena_dev;
1851 	int rc;
1852 
1853 	/* In case the RSS table wasn't initialized by probe */
1854 	if (!ena_dev->rss.tbl_log_size) {
1855 		rc = ena_rss_init_default(adapter);
1856 		if (rc && (rc != -EOPNOTSUPP)) {
1857 			netif_err(adapter, ifup, adapter->netdev, "Failed to init RSS rc: %d\n", rc);
1858 			return rc;
1859 		}
1860 	}
1861 
1862 	/* Set indirect table */
1863 	rc = ena_com_indirect_table_set(ena_dev);
1864 	if (unlikely(rc && rc != -EOPNOTSUPP))
1865 		return rc;
1866 
1867 	/* Configure hash function (if supported) */
1868 	rc = ena_com_set_hash_function(ena_dev);
1869 	if (unlikely(rc && (rc != -EOPNOTSUPP)))
1870 		return rc;
1871 
1872 	/* Configure hash inputs (if supported) */
1873 	rc = ena_com_set_hash_ctrl(ena_dev);
1874 	if (unlikely(rc && (rc != -EOPNOTSUPP)))
1875 		return rc;
1876 
1877 	return 0;
1878 }
1879 
1880 static int ena_up_complete(struct ena_adapter *adapter)
1881 {
1882 	int rc;
1883 
1884 	rc = ena_rss_configure(adapter);
1885 	if (rc)
1886 		return rc;
1887 
1888 	ena_change_mtu(adapter->netdev, adapter->netdev->mtu);
1889 
1890 	ena_refill_all_rx_bufs(adapter);
1891 
1892 	/* enable transmits */
1893 	netif_tx_start_all_queues(adapter->netdev);
1894 
1895 	ena_napi_enable_in_range(adapter,
1896 				 0,
1897 				 adapter->xdp_num_queues + adapter->num_io_queues);
1898 
1899 	return 0;
1900 }
1901 
1902 static int ena_create_io_tx_queue(struct ena_adapter *adapter, int qid)
1903 {
1904 	struct ena_com_create_io_ctx ctx;
1905 	struct ena_com_dev *ena_dev;
1906 	struct ena_ring *tx_ring;
1907 	u32 msix_vector;
1908 	u16 ena_qid;
1909 	int rc;
1910 
1911 	ena_dev = adapter->ena_dev;
1912 
1913 	tx_ring = &adapter->tx_ring[qid];
1914 	msix_vector = ENA_IO_IRQ_IDX(qid);
1915 	ena_qid = ENA_IO_TXQ_IDX(qid);
1916 
1917 	memset(&ctx, 0x0, sizeof(ctx));
1918 
1919 	ctx.direction = ENA_COM_IO_QUEUE_DIRECTION_TX;
1920 	ctx.qid = ena_qid;
1921 	ctx.mem_queue_type = ena_dev->tx_mem_queue_type;
1922 	ctx.msix_vector = msix_vector;
1923 	ctx.queue_size = tx_ring->ring_size;
1924 	ctx.numa_node = tx_ring->numa_node;
1925 
1926 	rc = ena_com_create_io_queue(ena_dev, &ctx);
1927 	if (rc) {
1928 		netif_err(adapter, ifup, adapter->netdev,
1929 			  "Failed to create I/O TX queue num %d rc: %d\n",
1930 			  qid, rc);
1931 		return rc;
1932 	}
1933 
1934 	rc = ena_com_get_io_handlers(ena_dev, ena_qid,
1935 				     &tx_ring->ena_com_io_sq,
1936 				     &tx_ring->ena_com_io_cq);
1937 	if (rc) {
1938 		netif_err(adapter, ifup, adapter->netdev,
1939 			  "Failed to get TX queue handlers. TX queue num %d rc: %d\n",
1940 			  qid, rc);
1941 		ena_com_destroy_io_queue(ena_dev, ena_qid);
1942 		return rc;
1943 	}
1944 
1945 	ena_com_update_numa_node(tx_ring->ena_com_io_cq, ctx.numa_node);
1946 	return rc;
1947 }
1948 
1949 int ena_create_io_tx_queues_in_range(struct ena_adapter *adapter,
1950 				     int first_index, int count)
1951 {
1952 	struct ena_com_dev *ena_dev = adapter->ena_dev;
1953 	int rc, i;
1954 
1955 	for (i = first_index; i < first_index + count; i++) {
1956 		rc = ena_create_io_tx_queue(adapter, i);
1957 		if (rc)
1958 			goto create_err;
1959 	}
1960 
1961 	return 0;
1962 
1963 create_err:
1964 	while (i-- > first_index)
1965 		ena_com_destroy_io_queue(ena_dev, ENA_IO_TXQ_IDX(i));
1966 
1967 	return rc;
1968 }
1969 
1970 static int ena_create_io_rx_queue(struct ena_adapter *adapter, int qid)
1971 {
1972 	struct ena_com_dev *ena_dev;
1973 	struct ena_com_create_io_ctx ctx;
1974 	struct ena_ring *rx_ring;
1975 	u32 msix_vector;
1976 	u16 ena_qid;
1977 	int rc;
1978 
1979 	ena_dev = adapter->ena_dev;
1980 
1981 	rx_ring = &adapter->rx_ring[qid];
1982 	msix_vector = ENA_IO_IRQ_IDX(qid);
1983 	ena_qid = ENA_IO_RXQ_IDX(qid);
1984 
1985 	memset(&ctx, 0x0, sizeof(ctx));
1986 
1987 	ctx.qid = ena_qid;
1988 	ctx.direction = ENA_COM_IO_QUEUE_DIRECTION_RX;
1989 	ctx.mem_queue_type = ENA_ADMIN_PLACEMENT_POLICY_HOST;
1990 	ctx.msix_vector = msix_vector;
1991 	ctx.queue_size = rx_ring->ring_size;
1992 	ctx.numa_node = rx_ring->numa_node;
1993 
1994 	rc = ena_com_create_io_queue(ena_dev, &ctx);
1995 	if (rc) {
1996 		netif_err(adapter, ifup, adapter->netdev,
1997 			  "Failed to create I/O RX queue num %d rc: %d\n",
1998 			  qid, rc);
1999 		return rc;
2000 	}
2001 
2002 	rc = ena_com_get_io_handlers(ena_dev, ena_qid,
2003 				     &rx_ring->ena_com_io_sq,
2004 				     &rx_ring->ena_com_io_cq);
2005 	if (rc) {
2006 		netif_err(adapter, ifup, adapter->netdev,
2007 			  "Failed to get RX queue handlers. RX queue num %d rc: %d\n",
2008 			  qid, rc);
2009 		goto err;
2010 	}
2011 
2012 	ena_com_update_numa_node(rx_ring->ena_com_io_cq, ctx.numa_node);
2013 
2014 	return rc;
2015 err:
2016 	ena_com_destroy_io_queue(ena_dev, ena_qid);
2017 	return rc;
2018 }
2019 
2020 static int ena_create_all_io_rx_queues(struct ena_adapter *adapter)
2021 {
2022 	struct ena_com_dev *ena_dev = adapter->ena_dev;
2023 	int rc, i;
2024 
2025 	for (i = 0; i < adapter->num_io_queues; i++) {
2026 		rc = ena_create_io_rx_queue(adapter, i);
2027 		if (rc)
2028 			goto create_err;
2029 		INIT_WORK(&adapter->ena_napi[i].dim.work, ena_dim_work);
2030 
2031 		ena_xdp_register_rxq_info(&adapter->rx_ring[i]);
2032 	}
2033 
2034 	return 0;
2035 
2036 create_err:
2037 	while (i--) {
2038 		ena_xdp_unregister_rxq_info(&adapter->rx_ring[i]);
2039 		cancel_work_sync(&adapter->ena_napi[i].dim.work);
2040 		ena_com_destroy_io_queue(ena_dev, ENA_IO_RXQ_IDX(i));
2041 	}
2042 
2043 	return rc;
2044 }
2045 
2046 static void set_io_rings_size(struct ena_adapter *adapter,
2047 			      int new_tx_size,
2048 			      int new_rx_size)
2049 {
2050 	int i;
2051 
2052 	for (i = 0; i < adapter->num_io_queues; i++) {
2053 		adapter->tx_ring[i].ring_size = new_tx_size;
2054 		adapter->rx_ring[i].ring_size = new_rx_size;
2055 	}
2056 }
2057 
2058 /* This function allows queue allocation to backoff when the system is
2059  * low on memory. If there is not enough memory to allocate io queues
2060  * the driver will try to allocate smaller queues.
2061  *
2062  * The backoff algorithm is as follows:
2063  *  1. Try to allocate TX and RX and if successful.
2064  *  1.1. return success
2065  *
2066  *  2. Divide by 2 the size of the larger of RX and TX queues (or both if their size is the same).
2067  *
2068  *  3. If TX or RX is smaller than 256
2069  *  3.1. return failure.
2070  *  4. else
2071  *  4.1. go back to 1.
2072  */
2073 static int create_queues_with_size_backoff(struct ena_adapter *adapter)
2074 {
2075 	int rc, cur_rx_ring_size, cur_tx_ring_size;
2076 	int new_rx_ring_size, new_tx_ring_size;
2077 
2078 	/* current queue sizes might be set to smaller than the requested
2079 	 * ones due to past queue allocation failures.
2080 	 */
2081 	set_io_rings_size(adapter, adapter->requested_tx_ring_size,
2082 			  adapter->requested_rx_ring_size);
2083 
2084 	while (1) {
2085 		if (ena_xdp_present(adapter)) {
2086 			rc = ena_setup_and_create_all_xdp_queues(adapter);
2087 
2088 			if (rc)
2089 				goto err_setup_tx;
2090 		}
2091 		rc = ena_setup_tx_resources_in_range(adapter,
2092 						     0,
2093 						     adapter->num_io_queues);
2094 		if (rc) {
2095 			ena_destroy_xdp_tx_queues(adapter);
2096 			ena_free_all_io_tx_resources_in_range(adapter,
2097 							      adapter->xdp_first_ring,
2098 							      adapter->xdp_num_queues);
2099 			goto err_setup_tx;
2100 		}
2101 
2102 		rc = ena_create_io_tx_queues_in_range(adapter,
2103 						      0,
2104 						      adapter->num_io_queues);
2105 		if (rc) {
2106 			ena_destroy_xdp_tx_queues(adapter);
2107 			goto err_create_tx_queues;
2108 		}
2109 
2110 		rc = ena_setup_all_rx_resources(adapter);
2111 		if (rc)
2112 			goto err_setup_rx;
2113 
2114 		rc = ena_create_all_io_rx_queues(adapter);
2115 		if (rc)
2116 			goto err_create_rx_queues;
2117 
2118 		return 0;
2119 
2120 err_create_rx_queues:
2121 		ena_free_all_io_rx_resources(adapter);
2122 err_setup_rx:
2123 		ena_destroy_all_tx_queues(adapter);
2124 err_create_tx_queues:
2125 		ena_free_all_io_tx_resources(adapter);
2126 err_setup_tx:
2127 		if (rc != -ENOMEM) {
2128 			netif_err(adapter, ifup, adapter->netdev,
2129 				  "Queue creation failed with error code %d\n",
2130 				  rc);
2131 			return rc;
2132 		}
2133 
2134 		cur_tx_ring_size = adapter->tx_ring[0].ring_size;
2135 		cur_rx_ring_size = adapter->rx_ring[0].ring_size;
2136 
2137 		netif_err(adapter, ifup, adapter->netdev,
2138 			  "Not enough memory to create queues with sizes TX=%d, RX=%d\n",
2139 			  cur_tx_ring_size, cur_rx_ring_size);
2140 
2141 		new_tx_ring_size = cur_tx_ring_size;
2142 		new_rx_ring_size = cur_rx_ring_size;
2143 
2144 		/* Decrease the size of the larger queue, or
2145 		 * decrease both if they are the same size.
2146 		 */
2147 		if (cur_rx_ring_size <= cur_tx_ring_size)
2148 			new_tx_ring_size = cur_tx_ring_size / 2;
2149 		if (cur_rx_ring_size >= cur_tx_ring_size)
2150 			new_rx_ring_size = cur_rx_ring_size / 2;
2151 
2152 		if (new_tx_ring_size < ENA_MIN_RING_SIZE ||
2153 		    new_rx_ring_size < ENA_MIN_RING_SIZE) {
2154 			netif_err(adapter, ifup, adapter->netdev,
2155 				  "Queue creation failed with the smallest possible queue size of %d for both queues. Not retrying with smaller queues\n",
2156 				  ENA_MIN_RING_SIZE);
2157 			return rc;
2158 		}
2159 
2160 		netif_err(adapter, ifup, adapter->netdev,
2161 			  "Retrying queue creation with sizes TX=%d, RX=%d\n",
2162 			  new_tx_ring_size,
2163 			  new_rx_ring_size);
2164 
2165 		set_io_rings_size(adapter, new_tx_ring_size,
2166 				  new_rx_ring_size);
2167 	}
2168 }
2169 
2170 int ena_up(struct ena_adapter *adapter)
2171 {
2172 	int io_queue_count, rc, i;
2173 
2174 	netif_dbg(adapter, ifup, adapter->netdev, "%s\n", __func__);
2175 
2176 	io_queue_count = adapter->num_io_queues + adapter->xdp_num_queues;
2177 	ena_setup_io_intr(adapter);
2178 
2179 	/* napi poll functions should be initialized before running
2180 	 * request_irq(), to handle a rare condition where there is a pending
2181 	 * interrupt, causing the ISR to fire immediately while the poll
2182 	 * function wasn't set yet, causing a null dereference
2183 	 */
2184 	ena_init_napi_in_range(adapter, 0, io_queue_count);
2185 
2186 	/* Enabling DIM needs to happen before enabling IRQs since DIM
2187 	 * is run from napi routine
2188 	 */
2189 	if (ena_com_interrupt_moderation_supported(adapter->ena_dev))
2190 		ena_com_enable_adaptive_moderation(adapter->ena_dev);
2191 
2192 	rc = ena_request_io_irq(adapter);
2193 	if (rc)
2194 		goto err_req_irq;
2195 
2196 	rc = create_queues_with_size_backoff(adapter);
2197 	if (rc)
2198 		goto err_create_queues_with_backoff;
2199 
2200 	rc = ena_up_complete(adapter);
2201 	if (rc)
2202 		goto err_up;
2203 
2204 	if (test_bit(ENA_FLAG_LINK_UP, &adapter->flags))
2205 		netif_carrier_on(adapter->netdev);
2206 
2207 	ena_increase_stat(&adapter->dev_stats.interface_up, 1,
2208 			  &adapter->syncp);
2209 
2210 	set_bit(ENA_FLAG_DEV_UP, &adapter->flags);
2211 
2212 	/* Enable completion queues interrupt */
2213 	for (i = 0; i < adapter->num_io_queues; i++)
2214 		ena_unmask_interrupt(&adapter->tx_ring[i],
2215 				     &adapter->rx_ring[i]);
2216 
2217 	/* schedule napi in case we had pending packets
2218 	 * from the last time we disable napi
2219 	 */
2220 	for (i = 0; i < io_queue_count; i++)
2221 		napi_schedule(&adapter->ena_napi[i].napi);
2222 
2223 	return rc;
2224 
2225 err_up:
2226 	ena_destroy_all_tx_queues(adapter);
2227 	ena_free_all_io_tx_resources(adapter);
2228 	ena_destroy_all_rx_queues(adapter);
2229 	ena_free_all_io_rx_resources(adapter);
2230 err_create_queues_with_backoff:
2231 	ena_free_io_irq(adapter);
2232 err_req_irq:
2233 	ena_del_napi_in_range(adapter, 0, io_queue_count);
2234 
2235 	return rc;
2236 }
2237 
2238 void ena_down(struct ena_adapter *adapter)
2239 {
2240 	int io_queue_count = adapter->num_io_queues + adapter->xdp_num_queues;
2241 
2242 	netif_dbg(adapter, ifdown, adapter->netdev, "%s\n", __func__);
2243 
2244 	clear_bit(ENA_FLAG_DEV_UP, &adapter->flags);
2245 
2246 	ena_increase_stat(&adapter->dev_stats.interface_down, 1,
2247 			  &adapter->syncp);
2248 
2249 	netif_carrier_off(adapter->netdev);
2250 	netif_tx_disable(adapter->netdev);
2251 
2252 	/* After this point the napi handler won't enable the tx queue */
2253 	ena_napi_disable_in_range(adapter, 0, io_queue_count);
2254 
2255 	if (test_bit(ENA_FLAG_TRIGGER_RESET, &adapter->flags)) {
2256 		int rc;
2257 
2258 		rc = ena_com_dev_reset(adapter->ena_dev, adapter->reset_reason);
2259 		if (rc)
2260 			netif_err(adapter, ifdown, adapter->netdev,
2261 				  "Device reset failed\n");
2262 		/* stop submitting admin commands on a device that was reset */
2263 		ena_com_set_admin_running_state(adapter->ena_dev, false);
2264 	}
2265 
2266 	ena_destroy_all_io_queues(adapter);
2267 
2268 	ena_disable_io_intr_sync(adapter);
2269 	ena_free_io_irq(adapter);
2270 	ena_del_napi_in_range(adapter, 0, io_queue_count);
2271 
2272 	ena_free_all_tx_bufs(adapter);
2273 	ena_free_all_rx_bufs(adapter);
2274 	ena_free_all_io_tx_resources(adapter);
2275 	ena_free_all_io_rx_resources(adapter);
2276 }
2277 
2278 /* ena_open - Called when a network interface is made active
2279  * @netdev: network interface device structure
2280  *
2281  * Returns 0 on success, negative value on failure
2282  *
2283  * The open entry point is called when a network interface is made
2284  * active by the system (IFF_UP).  At this point all resources needed
2285  * for transmit and receive operations are allocated, the interrupt
2286  * handler is registered with the OS, the watchdog timer is started,
2287  * and the stack is notified that the interface is ready.
2288  */
2289 static int ena_open(struct net_device *netdev)
2290 {
2291 	struct ena_adapter *adapter = netdev_priv(netdev);
2292 	int rc;
2293 
2294 	/* Notify the stack of the actual queue counts. */
2295 	rc = netif_set_real_num_tx_queues(netdev, adapter->num_io_queues);
2296 	if (rc) {
2297 		netif_err(adapter, ifup, netdev, "Can't set num tx queues\n");
2298 		return rc;
2299 	}
2300 
2301 	rc = netif_set_real_num_rx_queues(netdev, adapter->num_io_queues);
2302 	if (rc) {
2303 		netif_err(adapter, ifup, netdev, "Can't set num rx queues\n");
2304 		return rc;
2305 	}
2306 
2307 	return ena_up(adapter);
2308 }
2309 
2310 /* ena_close - Disables a network interface
2311  * @netdev: network interface device structure
2312  *
2313  * Returns 0, this is not allowed to fail
2314  *
2315  * The close entry point is called when an interface is de-activated
2316  * by the OS.  The hardware is still under the drivers control, but
2317  * needs to be disabled.  A global MAC reset is issued to stop the
2318  * hardware, and all transmit and receive resources are freed.
2319  */
2320 static int ena_close(struct net_device *netdev)
2321 {
2322 	struct ena_adapter *adapter = netdev_priv(netdev);
2323 
2324 	netif_dbg(adapter, ifdown, netdev, "%s\n", __func__);
2325 
2326 	if (!test_bit(ENA_FLAG_DEVICE_RUNNING, &adapter->flags))
2327 		return 0;
2328 
2329 	if (test_bit(ENA_FLAG_DEV_UP, &adapter->flags))
2330 		ena_down(adapter);
2331 
2332 	/* Check for device status and issue reset if needed*/
2333 	check_for_admin_com_state(adapter);
2334 	if (unlikely(test_bit(ENA_FLAG_TRIGGER_RESET, &adapter->flags))) {
2335 		netif_err(adapter, ifdown, adapter->netdev,
2336 			  "Destroy failure, restarting device\n");
2337 		ena_dump_stats_to_dmesg(adapter);
2338 		/* rtnl lock already obtained in dev_ioctl() layer */
2339 		ena_destroy_device(adapter, false);
2340 		ena_restore_device(adapter);
2341 	}
2342 
2343 	return 0;
2344 }
2345 
2346 int ena_update_queue_params(struct ena_adapter *adapter,
2347 			    u32 new_tx_size,
2348 			    u32 new_rx_size,
2349 			    u32 new_llq_header_len)
2350 {
2351 	bool dev_was_up, large_llq_changed = false;
2352 	int rc = 0;
2353 
2354 	dev_was_up = test_bit(ENA_FLAG_DEV_UP, &adapter->flags);
2355 	ena_close(adapter->netdev);
2356 	adapter->requested_tx_ring_size = new_tx_size;
2357 	adapter->requested_rx_ring_size = new_rx_size;
2358 	ena_init_io_rings(adapter,
2359 			  0,
2360 			  adapter->xdp_num_queues +
2361 			  adapter->num_io_queues);
2362 
2363 	large_llq_changed = adapter->ena_dev->tx_mem_queue_type ==
2364 			    ENA_ADMIN_PLACEMENT_POLICY_DEV;
2365 	large_llq_changed &=
2366 		new_llq_header_len != adapter->ena_dev->tx_max_header_size;
2367 
2368 	/* a check that the configuration is valid is done by caller */
2369 	if (large_llq_changed) {
2370 		adapter->large_llq_header_enabled = !adapter->large_llq_header_enabled;
2371 
2372 		ena_destroy_device(adapter, false);
2373 		rc = ena_restore_device(adapter);
2374 	}
2375 
2376 	return dev_was_up && !rc ? ena_up(adapter) : rc;
2377 }
2378 
2379 int ena_set_rx_copybreak(struct ena_adapter *adapter, u32 rx_copybreak)
2380 {
2381 	struct ena_ring *rx_ring;
2382 	int i;
2383 
2384 	if (rx_copybreak > min_t(u16, adapter->netdev->mtu, ENA_PAGE_SIZE))
2385 		return -EINVAL;
2386 
2387 	adapter->rx_copybreak = rx_copybreak;
2388 
2389 	for (i = 0; i < adapter->num_io_queues; i++) {
2390 		rx_ring = &adapter->rx_ring[i];
2391 		rx_ring->rx_copybreak = rx_copybreak;
2392 	}
2393 
2394 	return 0;
2395 }
2396 
2397 int ena_update_queue_count(struct ena_adapter *adapter, u32 new_channel_count)
2398 {
2399 	struct ena_com_dev *ena_dev = adapter->ena_dev;
2400 	int prev_channel_count;
2401 	bool dev_was_up;
2402 
2403 	dev_was_up = test_bit(ENA_FLAG_DEV_UP, &adapter->flags);
2404 	ena_close(adapter->netdev);
2405 	prev_channel_count = adapter->num_io_queues;
2406 	adapter->num_io_queues = new_channel_count;
2407 	if (ena_xdp_present(adapter) &&
2408 	    ena_xdp_allowed(adapter) == ENA_XDP_ALLOWED) {
2409 		adapter->xdp_first_ring = new_channel_count;
2410 		adapter->xdp_num_queues = new_channel_count;
2411 		if (prev_channel_count > new_channel_count)
2412 			ena_xdp_exchange_program_rx_in_range(adapter,
2413 							     NULL,
2414 							     new_channel_count,
2415 							     prev_channel_count);
2416 		else
2417 			ena_xdp_exchange_program_rx_in_range(adapter,
2418 							     adapter->xdp_bpf_prog,
2419 							     prev_channel_count,
2420 							     new_channel_count);
2421 	}
2422 
2423 	/* We need to destroy the rss table so that the indirection
2424 	 * table will be reinitialized by ena_up()
2425 	 */
2426 	ena_com_rss_destroy(ena_dev);
2427 	ena_init_io_rings(adapter,
2428 			  0,
2429 			  adapter->xdp_num_queues +
2430 			  adapter->num_io_queues);
2431 	return dev_was_up ? ena_open(adapter->netdev) : 0;
2432 }
2433 
2434 static void ena_tx_csum(struct ena_com_tx_ctx *ena_tx_ctx,
2435 			struct sk_buff *skb,
2436 			bool disable_meta_caching)
2437 {
2438 	u32 mss = skb_shinfo(skb)->gso_size;
2439 	struct ena_com_tx_meta *ena_meta = &ena_tx_ctx->ena_meta;
2440 	u8 l4_protocol = 0;
2441 
2442 	if ((skb->ip_summed == CHECKSUM_PARTIAL) || mss) {
2443 		ena_tx_ctx->l4_csum_enable = 1;
2444 		if (mss) {
2445 			ena_tx_ctx->tso_enable = 1;
2446 			ena_meta->l4_hdr_len = tcp_hdr(skb)->doff;
2447 			ena_tx_ctx->l4_csum_partial = 0;
2448 		} else {
2449 			ena_tx_ctx->tso_enable = 0;
2450 			ena_meta->l4_hdr_len = 0;
2451 			ena_tx_ctx->l4_csum_partial = 1;
2452 		}
2453 
2454 		switch (ip_hdr(skb)->version) {
2455 		case IPVERSION:
2456 			ena_tx_ctx->l3_proto = ENA_ETH_IO_L3_PROTO_IPV4;
2457 			if (ip_hdr(skb)->frag_off & htons(IP_DF))
2458 				ena_tx_ctx->df = 1;
2459 			if (mss)
2460 				ena_tx_ctx->l3_csum_enable = 1;
2461 			l4_protocol = ip_hdr(skb)->protocol;
2462 			break;
2463 		case 6:
2464 			ena_tx_ctx->l3_proto = ENA_ETH_IO_L3_PROTO_IPV6;
2465 			l4_protocol = ipv6_hdr(skb)->nexthdr;
2466 			break;
2467 		default:
2468 			break;
2469 		}
2470 
2471 		if (l4_protocol == IPPROTO_TCP)
2472 			ena_tx_ctx->l4_proto = ENA_ETH_IO_L4_PROTO_TCP;
2473 		else
2474 			ena_tx_ctx->l4_proto = ENA_ETH_IO_L4_PROTO_UDP;
2475 
2476 		ena_meta->mss = mss;
2477 		ena_meta->l3_hdr_len = skb_network_header_len(skb);
2478 		ena_meta->l3_hdr_offset = skb_network_offset(skb);
2479 		ena_tx_ctx->meta_valid = 1;
2480 	} else if (disable_meta_caching) {
2481 		memset(ena_meta, 0, sizeof(*ena_meta));
2482 		ena_tx_ctx->meta_valid = 1;
2483 	} else {
2484 		ena_tx_ctx->meta_valid = 0;
2485 	}
2486 }
2487 
2488 static int ena_check_and_linearize_skb(struct ena_ring *tx_ring,
2489 				       struct sk_buff *skb)
2490 {
2491 	int num_frags, header_len, rc;
2492 
2493 	num_frags = skb_shinfo(skb)->nr_frags;
2494 	header_len = skb_headlen(skb);
2495 
2496 	if (num_frags < tx_ring->sgl_size)
2497 		return 0;
2498 
2499 	if ((num_frags == tx_ring->sgl_size) &&
2500 	    (header_len < tx_ring->tx_max_header_size))
2501 		return 0;
2502 
2503 	ena_increase_stat(&tx_ring->tx_stats.linearize, 1, &tx_ring->syncp);
2504 
2505 	rc = skb_linearize(skb);
2506 	if (unlikely(rc)) {
2507 		ena_increase_stat(&tx_ring->tx_stats.linearize_failed, 1,
2508 				  &tx_ring->syncp);
2509 	}
2510 
2511 	return rc;
2512 }
2513 
2514 static int ena_tx_map_skb(struct ena_ring *tx_ring,
2515 			  struct ena_tx_buffer *tx_info,
2516 			  struct sk_buff *skb,
2517 			  void **push_hdr,
2518 			  u16 *header_len)
2519 {
2520 	struct ena_adapter *adapter = tx_ring->adapter;
2521 	struct ena_com_buf *ena_buf;
2522 	dma_addr_t dma;
2523 	u32 skb_head_len, frag_len, last_frag;
2524 	u16 push_len = 0;
2525 	u16 delta = 0;
2526 	int i = 0;
2527 
2528 	skb_head_len = skb_headlen(skb);
2529 	tx_info->skb = skb;
2530 	ena_buf = tx_info->bufs;
2531 
2532 	if (tx_ring->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV) {
2533 		/* When the device is LLQ mode, the driver will copy
2534 		 * the header into the device memory space.
2535 		 * the ena_com layer assume the header is in a linear
2536 		 * memory space.
2537 		 * This assumption might be wrong since part of the header
2538 		 * can be in the fragmented buffers.
2539 		 * Use skb_header_pointer to make sure the header is in a
2540 		 * linear memory space.
2541 		 */
2542 
2543 		push_len = min_t(u32, skb->len, tx_ring->tx_max_header_size);
2544 		*push_hdr = skb_header_pointer(skb, 0, push_len,
2545 					       tx_ring->push_buf_intermediate_buf);
2546 		*header_len = push_len;
2547 		if (unlikely(skb->data != *push_hdr)) {
2548 			ena_increase_stat(&tx_ring->tx_stats.llq_buffer_copy, 1,
2549 					  &tx_ring->syncp);
2550 
2551 			delta = push_len - skb_head_len;
2552 		}
2553 	} else {
2554 		*push_hdr = NULL;
2555 		*header_len = min_t(u32, skb_head_len,
2556 				    tx_ring->tx_max_header_size);
2557 	}
2558 
2559 	netif_dbg(adapter, tx_queued, adapter->netdev,
2560 		  "skb: %p header_buf->vaddr: %p push_len: %d\n", skb,
2561 		  *push_hdr, push_len);
2562 
2563 	if (skb_head_len > push_len) {
2564 		dma = dma_map_single(tx_ring->dev, skb->data + push_len,
2565 				     skb_head_len - push_len, DMA_TO_DEVICE);
2566 		if (unlikely(dma_mapping_error(tx_ring->dev, dma)))
2567 			goto error_report_dma_error;
2568 
2569 		ena_buf->paddr = dma;
2570 		ena_buf->len = skb_head_len - push_len;
2571 
2572 		ena_buf++;
2573 		tx_info->num_of_bufs++;
2574 		tx_info->map_linear_data = 1;
2575 	} else {
2576 		tx_info->map_linear_data = 0;
2577 	}
2578 
2579 	last_frag = skb_shinfo(skb)->nr_frags;
2580 
2581 	for (i = 0; i < last_frag; i++) {
2582 		const skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
2583 
2584 		frag_len = skb_frag_size(frag);
2585 
2586 		if (unlikely(delta >= frag_len)) {
2587 			delta -= frag_len;
2588 			continue;
2589 		}
2590 
2591 		dma = skb_frag_dma_map(tx_ring->dev, frag, delta,
2592 				       frag_len - delta, DMA_TO_DEVICE);
2593 		if (unlikely(dma_mapping_error(tx_ring->dev, dma)))
2594 			goto error_report_dma_error;
2595 
2596 		ena_buf->paddr = dma;
2597 		ena_buf->len = frag_len - delta;
2598 		ena_buf++;
2599 		tx_info->num_of_bufs++;
2600 		delta = 0;
2601 	}
2602 
2603 	return 0;
2604 
2605 error_report_dma_error:
2606 	ena_increase_stat(&tx_ring->tx_stats.dma_mapping_err, 1,
2607 			  &tx_ring->syncp);
2608 	netif_warn(adapter, tx_queued, adapter->netdev, "Failed to map skb\n");
2609 
2610 	tx_info->skb = NULL;
2611 
2612 	tx_info->num_of_bufs += i;
2613 	ena_unmap_tx_buff(tx_ring, tx_info);
2614 
2615 	return -EINVAL;
2616 }
2617 
2618 /* Called with netif_tx_lock. */
2619 static netdev_tx_t ena_start_xmit(struct sk_buff *skb, struct net_device *dev)
2620 {
2621 	struct ena_adapter *adapter = netdev_priv(dev);
2622 	struct ena_tx_buffer *tx_info;
2623 	struct ena_com_tx_ctx ena_tx_ctx;
2624 	struct ena_ring *tx_ring;
2625 	struct netdev_queue *txq;
2626 	void *push_hdr;
2627 	u16 next_to_use, req_id, header_len;
2628 	int qid, rc;
2629 
2630 	netif_dbg(adapter, tx_queued, dev, "%s skb %p\n", __func__, skb);
2631 	/*  Determine which tx ring we will be placed on */
2632 	qid = skb_get_queue_mapping(skb);
2633 	tx_ring = &adapter->tx_ring[qid];
2634 	txq = netdev_get_tx_queue(dev, qid);
2635 
2636 	rc = ena_check_and_linearize_skb(tx_ring, skb);
2637 	if (unlikely(rc))
2638 		goto error_drop_packet;
2639 
2640 	next_to_use = tx_ring->next_to_use;
2641 	req_id = tx_ring->free_ids[next_to_use];
2642 	tx_info = &tx_ring->tx_buffer_info[req_id];
2643 	tx_info->num_of_bufs = 0;
2644 
2645 	WARN(tx_info->skb, "SKB isn't NULL req_id %d\n", req_id);
2646 
2647 	rc = ena_tx_map_skb(tx_ring, tx_info, skb, &push_hdr, &header_len);
2648 	if (unlikely(rc))
2649 		goto error_drop_packet;
2650 
2651 	memset(&ena_tx_ctx, 0x0, sizeof(struct ena_com_tx_ctx));
2652 	ena_tx_ctx.ena_bufs = tx_info->bufs;
2653 	ena_tx_ctx.push_header = push_hdr;
2654 	ena_tx_ctx.num_bufs = tx_info->num_of_bufs;
2655 	ena_tx_ctx.req_id = req_id;
2656 	ena_tx_ctx.header_len = header_len;
2657 
2658 	/* set flags and meta data */
2659 	ena_tx_csum(&ena_tx_ctx, skb, tx_ring->disable_meta_caching);
2660 
2661 	rc = ena_xmit_common(adapter,
2662 			     tx_ring,
2663 			     tx_info,
2664 			     &ena_tx_ctx,
2665 			     next_to_use,
2666 			     skb->len);
2667 	if (rc)
2668 		goto error_unmap_dma;
2669 
2670 	netdev_tx_sent_queue(txq, skb->len);
2671 
2672 	/* stop the queue when no more space available, the packet can have up
2673 	 * to sgl_size + 2. one for the meta descriptor and one for header
2674 	 * (if the header is larger than tx_max_header_size).
2675 	 */
2676 	if (unlikely(!ena_com_sq_have_enough_space(tx_ring->ena_com_io_sq,
2677 						   tx_ring->sgl_size + 2))) {
2678 		netif_dbg(adapter, tx_queued, dev, "%s stop queue %d\n",
2679 			  __func__, qid);
2680 
2681 		netif_tx_stop_queue(txq);
2682 		ena_increase_stat(&tx_ring->tx_stats.queue_stop, 1,
2683 				  &tx_ring->syncp);
2684 
2685 		/* There is a rare condition where this function decide to
2686 		 * stop the queue but meanwhile clean_tx_irq updates
2687 		 * next_to_completion and terminates.
2688 		 * The queue will remain stopped forever.
2689 		 * To solve this issue add a mb() to make sure that
2690 		 * netif_tx_stop_queue() write is vissible before checking if
2691 		 * there is additional space in the queue.
2692 		 */
2693 		smp_mb();
2694 
2695 		if (ena_com_sq_have_enough_space(tx_ring->ena_com_io_sq,
2696 						 ENA_TX_WAKEUP_THRESH)) {
2697 			netif_tx_wake_queue(txq);
2698 			ena_increase_stat(&tx_ring->tx_stats.queue_wakeup, 1,
2699 					  &tx_ring->syncp);
2700 		}
2701 	}
2702 
2703 	skb_tx_timestamp(skb);
2704 
2705 	if (netif_xmit_stopped(txq) || !netdev_xmit_more())
2706 		/* trigger the dma engine. ena_ring_tx_doorbell()
2707 		 * calls a memory barrier inside it.
2708 		 */
2709 		ena_ring_tx_doorbell(tx_ring);
2710 
2711 	return NETDEV_TX_OK;
2712 
2713 error_unmap_dma:
2714 	ena_unmap_tx_buff(tx_ring, tx_info);
2715 	tx_info->skb = NULL;
2716 
2717 error_drop_packet:
2718 	dev_kfree_skb(skb);
2719 	return NETDEV_TX_OK;
2720 }
2721 
2722 static void ena_config_host_info(struct ena_com_dev *ena_dev, struct pci_dev *pdev)
2723 {
2724 	struct device *dev = &pdev->dev;
2725 	struct ena_admin_host_info *host_info;
2726 	ssize_t ret;
2727 	int rc;
2728 
2729 	/* Allocate only the host info */
2730 	rc = ena_com_allocate_host_info(ena_dev);
2731 	if (rc) {
2732 		dev_err(dev, "Cannot allocate host info\n");
2733 		return;
2734 	}
2735 
2736 	host_info = ena_dev->host_attr.host_info;
2737 
2738 	host_info->bdf = pci_dev_id(pdev);
2739 	host_info->os_type = ENA_ADMIN_OS_LINUX;
2740 	host_info->kernel_ver = LINUX_VERSION_CODE;
2741 	ret = strscpy(host_info->kernel_ver_str, utsname()->version,
2742 		      sizeof(host_info->kernel_ver_str));
2743 	if (ret < 0)
2744 		dev_dbg(dev,
2745 			"kernel version string will be truncated, status = %zd\n", ret);
2746 
2747 	host_info->os_dist = 0;
2748 	ret = strscpy(host_info->os_dist_str, utsname()->release,
2749 		      sizeof(host_info->os_dist_str));
2750 	if (ret < 0)
2751 		dev_dbg(dev,
2752 			"OS distribution string will be truncated, status = %zd\n", ret);
2753 
2754 	host_info->driver_version =
2755 		(DRV_MODULE_GEN_MAJOR) |
2756 		(DRV_MODULE_GEN_MINOR << ENA_ADMIN_HOST_INFO_MINOR_SHIFT) |
2757 		(DRV_MODULE_GEN_SUBMINOR << ENA_ADMIN_HOST_INFO_SUB_MINOR_SHIFT) |
2758 		("K"[0] << ENA_ADMIN_HOST_INFO_MODULE_TYPE_SHIFT);
2759 	host_info->num_cpus = num_online_cpus();
2760 
2761 	host_info->driver_supported_features =
2762 		ENA_ADMIN_HOST_INFO_RX_OFFSET_MASK |
2763 		ENA_ADMIN_HOST_INFO_INTERRUPT_MODERATION_MASK |
2764 		ENA_ADMIN_HOST_INFO_RX_BUF_MIRRORING_MASK |
2765 		ENA_ADMIN_HOST_INFO_RSS_CONFIGURABLE_FUNCTION_KEY_MASK |
2766 		ENA_ADMIN_HOST_INFO_RX_PAGE_REUSE_MASK |
2767 		ENA_ADMIN_HOST_INFO_PHC_MASK;
2768 
2769 	rc = ena_com_set_host_attributes(ena_dev);
2770 	if (rc) {
2771 		if (rc == -EOPNOTSUPP)
2772 			dev_warn(dev, "Cannot set host attributes\n");
2773 		else
2774 			dev_err(dev, "Cannot set host attributes\n");
2775 
2776 		goto err;
2777 	}
2778 
2779 	return;
2780 
2781 err:
2782 	ena_com_delete_host_info(ena_dev);
2783 }
2784 
2785 static void ena_config_debug_area(struct ena_adapter *adapter)
2786 {
2787 	u32 debug_area_size;
2788 	int rc, ss_count;
2789 
2790 	ss_count = ena_get_sset_count(adapter->netdev, ETH_SS_STATS);
2791 	if (ss_count <= 0) {
2792 		netif_err(adapter, drv, adapter->netdev,
2793 			  "SS count is negative\n");
2794 		return;
2795 	}
2796 
2797 	/* allocate 32 bytes for each string and 64bit for the value */
2798 	debug_area_size = ss_count * ETH_GSTRING_LEN + sizeof(u64) * ss_count;
2799 
2800 	rc = ena_com_allocate_debug_area(adapter->ena_dev, debug_area_size);
2801 	if (rc) {
2802 		netif_err(adapter, drv, adapter->netdev,
2803 			  "Cannot allocate debug area\n");
2804 		return;
2805 	}
2806 
2807 	rc = ena_com_set_host_attributes(adapter->ena_dev);
2808 	if (rc) {
2809 		if (rc == -EOPNOTSUPP)
2810 			netif_warn(adapter, drv, adapter->netdev, "Cannot set host attributes\n");
2811 		else
2812 			netif_err(adapter, drv, adapter->netdev,
2813 				  "Cannot set host attributes\n");
2814 		goto err;
2815 	}
2816 
2817 	return;
2818 err:
2819 	ena_com_delete_debug_area(adapter->ena_dev);
2820 }
2821 
2822 static void ena_get_stats64(struct net_device *netdev,
2823 			    struct rtnl_link_stats64 *stats)
2824 {
2825 	struct ena_adapter *adapter = netdev_priv(netdev);
2826 	struct ena_ring *rx_ring, *tx_ring;
2827 	u64 total_xdp_rx_drops = 0;
2828 	unsigned int start;
2829 	u64 rx_drops;
2830 	u64 tx_drops;
2831 	int i;
2832 
2833 	if (!test_bit(ENA_FLAG_DEV_UP, &adapter->flags))
2834 		return;
2835 
2836 	for (i = 0; i < adapter->num_io_queues + adapter->xdp_num_queues; i++) {
2837 		u64 bytes, packets, xdp_rx_drops;
2838 
2839 		tx_ring = &adapter->tx_ring[i];
2840 
2841 		do {
2842 			start = u64_stats_fetch_begin(&tx_ring->syncp);
2843 			packets = tx_ring->tx_stats.cnt;
2844 			bytes = tx_ring->tx_stats.bytes;
2845 		} while (u64_stats_fetch_retry(&tx_ring->syncp, start));
2846 
2847 		stats->tx_packets += packets;
2848 		stats->tx_bytes += bytes;
2849 
2850 		/* In XDP there isn't an RX queue counterpart */
2851 		if (ENA_IS_XDP_INDEX(adapter, i))
2852 			continue;
2853 
2854 		rx_ring = &adapter->rx_ring[i];
2855 
2856 		do {
2857 			start = u64_stats_fetch_begin(&rx_ring->syncp);
2858 			packets = rx_ring->rx_stats.cnt;
2859 			bytes = rx_ring->rx_stats.bytes;
2860 			xdp_rx_drops = rx_ring->rx_stats.xdp_drop;
2861 		} while (u64_stats_fetch_retry(&rx_ring->syncp, start));
2862 
2863 		stats->rx_packets += packets;
2864 		stats->rx_bytes += bytes;
2865 		total_xdp_rx_drops += xdp_rx_drops;
2866 	}
2867 
2868 	do {
2869 		start = u64_stats_fetch_begin(&adapter->syncp);
2870 		rx_drops = adapter->dev_stats.rx_drops;
2871 		tx_drops = adapter->dev_stats.tx_drops;
2872 	} while (u64_stats_fetch_retry(&adapter->syncp, start));
2873 
2874 	stats->rx_dropped = rx_drops + total_xdp_rx_drops;
2875 	stats->tx_dropped = tx_drops;
2876 
2877 	stats->multicast = 0;
2878 	stats->collisions = 0;
2879 
2880 	stats->rx_length_errors = 0;
2881 	stats->rx_crc_errors = 0;
2882 	stats->rx_frame_errors = 0;
2883 	stats->rx_fifo_errors = 0;
2884 	stats->rx_missed_errors = 0;
2885 	stats->tx_window_errors = 0;
2886 
2887 	stats->rx_errors = 0;
2888 	stats->tx_errors = 0;
2889 }
2890 
2891 static const struct net_device_ops ena_netdev_ops = {
2892 	.ndo_open		= ena_open,
2893 	.ndo_stop		= ena_close,
2894 	.ndo_start_xmit		= ena_start_xmit,
2895 	.ndo_get_stats64	= ena_get_stats64,
2896 	.ndo_tx_timeout		= ena_tx_timeout,
2897 	.ndo_change_mtu		= ena_change_mtu,
2898 	.ndo_validate_addr	= eth_validate_addr,
2899 	.ndo_bpf		= ena_xdp,
2900 	.ndo_xdp_xmit		= ena_xdp_xmit,
2901 };
2902 
2903 static int ena_calc_io_queue_size(struct ena_adapter *adapter,
2904 				  struct ena_com_dev_get_features_ctx *get_feat_ctx)
2905 {
2906 	struct ena_admin_feature_llq_desc *llq = &get_feat_ctx->llq;
2907 	struct ena_com_dev *ena_dev = adapter->ena_dev;
2908 	u32 tx_queue_size = ENA_DEFAULT_RING_SIZE;
2909 	u32 rx_queue_size = ENA_DEFAULT_RING_SIZE;
2910 	u32 max_tx_queue_size;
2911 	u32 max_rx_queue_size;
2912 
2913 	/* If this function is called after driver load, the ring sizes have already
2914 	 * been configured. Take it into account when recalculating ring size.
2915 	 */
2916 	if (adapter->tx_ring->ring_size)
2917 		tx_queue_size = adapter->tx_ring->ring_size;
2918 
2919 	if (adapter->rx_ring->ring_size)
2920 		rx_queue_size = adapter->rx_ring->ring_size;
2921 
2922 	if (ena_dev->supported_features & BIT(ENA_ADMIN_MAX_QUEUES_EXT)) {
2923 		struct ena_admin_queue_ext_feature_fields *max_queue_ext =
2924 			&get_feat_ctx->max_queue_ext.max_queue_ext;
2925 		max_rx_queue_size = min_t(u32, max_queue_ext->max_rx_cq_depth,
2926 					  max_queue_ext->max_rx_sq_depth);
2927 		max_tx_queue_size = max_queue_ext->max_tx_cq_depth;
2928 
2929 		if (ena_dev->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV)
2930 			max_tx_queue_size = min_t(u32, max_tx_queue_size,
2931 						  llq->max_llq_depth);
2932 		else
2933 			max_tx_queue_size = min_t(u32, max_tx_queue_size,
2934 						  max_queue_ext->max_tx_sq_depth);
2935 
2936 		adapter->max_tx_sgl_size = min_t(u16, ENA_PKT_MAX_BUFS,
2937 						 max_queue_ext->max_per_packet_tx_descs);
2938 		adapter->max_rx_sgl_size = min_t(u16, ENA_PKT_MAX_BUFS,
2939 						 max_queue_ext->max_per_packet_rx_descs);
2940 	} else {
2941 		struct ena_admin_queue_feature_desc *max_queues =
2942 			&get_feat_ctx->max_queues;
2943 		max_rx_queue_size = min_t(u32, max_queues->max_cq_depth,
2944 					  max_queues->max_sq_depth);
2945 		max_tx_queue_size = max_queues->max_cq_depth;
2946 
2947 		if (ena_dev->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV)
2948 			max_tx_queue_size = min_t(u32, max_tx_queue_size,
2949 						  llq->max_llq_depth);
2950 		else
2951 			max_tx_queue_size = min_t(u32, max_tx_queue_size,
2952 						  max_queues->max_sq_depth);
2953 
2954 		adapter->max_tx_sgl_size = min_t(u16, ENA_PKT_MAX_BUFS,
2955 						 max_queues->max_packet_tx_descs);
2956 		adapter->max_rx_sgl_size = min_t(u16, ENA_PKT_MAX_BUFS,
2957 						 max_queues->max_packet_rx_descs);
2958 	}
2959 
2960 	max_tx_queue_size = rounddown_pow_of_two(max_tx_queue_size);
2961 	max_rx_queue_size = rounddown_pow_of_two(max_rx_queue_size);
2962 
2963 	if (max_tx_queue_size < ENA_MIN_RING_SIZE) {
2964 		netdev_err(adapter->netdev, "Device max TX queue size: %d < minimum: %d\n",
2965 			   max_tx_queue_size, ENA_MIN_RING_SIZE);
2966 		return -EINVAL;
2967 	}
2968 
2969 	if (max_rx_queue_size < ENA_MIN_RING_SIZE) {
2970 		netdev_err(adapter->netdev, "Device max RX queue size: %d < minimum: %d\n",
2971 			   max_rx_queue_size, ENA_MIN_RING_SIZE);
2972 		return -EINVAL;
2973 	}
2974 
2975 	/* When forcing large headers, we multiply the entry size by 2, and therefore divide
2976 	 * the queue size by 2, leaving the amount of memory used by the queues unchanged.
2977 	 */
2978 	if (adapter->large_llq_header_enabled) {
2979 		if ((llq->entry_size_ctrl_supported & ENA_ADMIN_LIST_ENTRY_SIZE_256B) &&
2980 		    ena_dev->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV) {
2981 			max_tx_queue_size /= 2;
2982 			dev_info(&adapter->pdev->dev,
2983 				 "Forcing large headers and decreasing maximum TX queue size to %d\n",
2984 				 max_tx_queue_size);
2985 		} else {
2986 			dev_err(&adapter->pdev->dev,
2987 				"Forcing large headers failed: LLQ is disabled or device does not support large headers\n");
2988 
2989 			adapter->large_llq_header_enabled = false;
2990 		}
2991 	}
2992 
2993 	tx_queue_size = clamp_val(tx_queue_size, ENA_MIN_RING_SIZE,
2994 				  max_tx_queue_size);
2995 	rx_queue_size = clamp_val(rx_queue_size, ENA_MIN_RING_SIZE,
2996 				  max_rx_queue_size);
2997 
2998 	tx_queue_size = rounddown_pow_of_two(tx_queue_size);
2999 	rx_queue_size = rounddown_pow_of_two(rx_queue_size);
3000 
3001 	adapter->max_tx_ring_size  = max_tx_queue_size;
3002 	adapter->max_rx_ring_size = max_rx_queue_size;
3003 	adapter->requested_tx_ring_size = tx_queue_size;
3004 	adapter->requested_rx_ring_size = rx_queue_size;
3005 
3006 	return 0;
3007 }
3008 
3009 static int ena_device_validate_params(struct ena_adapter *adapter,
3010 				      struct ena_com_dev_get_features_ctx *get_feat_ctx)
3011 {
3012 	struct net_device *netdev = adapter->netdev;
3013 	int rc;
3014 
3015 	rc = ether_addr_equal(get_feat_ctx->dev_attr.mac_addr,
3016 			      adapter->mac_addr);
3017 	if (!rc) {
3018 		netif_err(adapter, drv, netdev,
3019 			  "Error, mac address are different\n");
3020 		return -EINVAL;
3021 	}
3022 
3023 	if (get_feat_ctx->dev_attr.max_mtu < netdev->mtu) {
3024 		netif_err(adapter, drv, netdev,
3025 			  "Error, device max mtu is smaller than netdev MTU\n");
3026 		return -EINVAL;
3027 	}
3028 
3029 	return 0;
3030 }
3031 
3032 static void set_default_llq_configurations(struct ena_adapter *adapter,
3033 					   struct ena_llq_configurations *llq_config,
3034 					   struct ena_admin_feature_llq_desc *llq)
3035 {
3036 	struct ena_com_dev *ena_dev = adapter->ena_dev;
3037 
3038 	llq_config->llq_header_location = ENA_ADMIN_INLINE_HEADER;
3039 	llq_config->llq_stride_ctrl = ENA_ADMIN_MULTIPLE_DESCS_PER_ENTRY;
3040 	llq_config->llq_num_decs_before_header = ENA_ADMIN_LLQ_NUM_DESCS_BEFORE_HEADER_2;
3041 
3042 	adapter->large_llq_header_supported =
3043 		!!(ena_dev->supported_features & BIT(ENA_ADMIN_LLQ));
3044 	adapter->large_llq_header_supported &=
3045 		!!(llq->entry_size_ctrl_supported &
3046 			ENA_ADMIN_LIST_ENTRY_SIZE_256B);
3047 
3048 	if ((llq->entry_size_ctrl_supported & ENA_ADMIN_LIST_ENTRY_SIZE_256B) &&
3049 	    adapter->large_llq_header_enabled) {
3050 		llq_config->llq_ring_entry_size = ENA_ADMIN_LIST_ENTRY_SIZE_256B;
3051 		llq_config->llq_ring_entry_size_value = 256;
3052 	} else {
3053 		llq_config->llq_ring_entry_size = ENA_ADMIN_LIST_ENTRY_SIZE_128B;
3054 		llq_config->llq_ring_entry_size_value = 128;
3055 	}
3056 }
3057 
3058 static int ena_set_queues_placement_policy(struct pci_dev *pdev,
3059 					   struct ena_com_dev *ena_dev,
3060 					   struct ena_admin_feature_llq_desc *llq,
3061 					   struct ena_llq_configurations *llq_default_configurations)
3062 {
3063 	int rc;
3064 	u32 llq_feature_mask;
3065 
3066 	llq_feature_mask = 1 << ENA_ADMIN_LLQ;
3067 	if (!(ena_dev->supported_features & llq_feature_mask)) {
3068 		dev_warn(&pdev->dev,
3069 			"LLQ is not supported Fallback to host mode policy.\n");
3070 		ena_dev->tx_mem_queue_type = ENA_ADMIN_PLACEMENT_POLICY_HOST;
3071 		return 0;
3072 	}
3073 
3074 	if (!ena_dev->mem_bar) {
3075 		netdev_err(ena_dev->net_device,
3076 			   "LLQ is advertised as supported but device doesn't expose mem bar\n");
3077 		ena_dev->tx_mem_queue_type = ENA_ADMIN_PLACEMENT_POLICY_HOST;
3078 		return 0;
3079 	}
3080 
3081 	rc = ena_com_config_dev_mode(ena_dev, llq, llq_default_configurations);
3082 	if (unlikely(rc)) {
3083 		dev_err(&pdev->dev,
3084 			"Failed to configure the device mode.  Fallback to host mode policy.\n");
3085 		ena_dev->tx_mem_queue_type = ENA_ADMIN_PLACEMENT_POLICY_HOST;
3086 	}
3087 
3088 	return 0;
3089 }
3090 
3091 static int ena_map_llq_mem_bar(struct pci_dev *pdev, struct ena_com_dev *ena_dev,
3092 			       int bars)
3093 {
3094 	bool has_mem_bar = !!(bars & BIT(ENA_MEM_BAR));
3095 
3096 	if (!has_mem_bar)
3097 		return 0;
3098 
3099 	ena_dev->mem_bar = devm_ioremap_wc(&pdev->dev,
3100 					   pci_resource_start(pdev, ENA_MEM_BAR),
3101 					   pci_resource_len(pdev, ENA_MEM_BAR));
3102 
3103 	if (!ena_dev->mem_bar)
3104 		return -EFAULT;
3105 
3106 	return 0;
3107 }
3108 
3109 static int ena_device_init(struct ena_adapter *adapter, struct pci_dev *pdev,
3110 			   struct ena_com_dev_get_features_ctx *get_feat_ctx,
3111 			   bool *wd_state)
3112 {
3113 	struct ena_com_dev *ena_dev = adapter->ena_dev;
3114 	struct net_device *netdev = adapter->netdev;
3115 	struct ena_llq_configurations llq_config;
3116 	struct device *dev = &pdev->dev;
3117 	bool readless_supported;
3118 	u32 aenq_groups;
3119 	int dma_width;
3120 	int rc;
3121 
3122 	rc = ena_com_mmio_reg_read_request_init(ena_dev);
3123 	if (rc) {
3124 		dev_err(dev, "Failed to init mmio read less\n");
3125 		return rc;
3126 	}
3127 
3128 	/* The PCIe configuration space revision id indicate if mmio reg
3129 	 * read is disabled
3130 	 */
3131 	readless_supported = !(pdev->revision & ENA_MMIO_DISABLE_REG_READ);
3132 	ena_com_set_mmio_read_mode(ena_dev, readless_supported);
3133 
3134 	rc = ena_com_dev_reset(ena_dev, ENA_REGS_RESET_NORMAL);
3135 	if (rc) {
3136 		dev_err(dev, "Can not reset device\n");
3137 		goto err_mmio_read_less;
3138 	}
3139 
3140 	rc = ena_com_validate_version(ena_dev);
3141 	if (rc) {
3142 		dev_err(dev, "Device version is too low\n");
3143 		goto err_mmio_read_less;
3144 	}
3145 
3146 	dma_width = ena_com_get_dma_width(ena_dev);
3147 	if (dma_width < 0) {
3148 		dev_err(dev, "Invalid dma width value %d", dma_width);
3149 		rc = dma_width;
3150 		goto err_mmio_read_less;
3151 	}
3152 
3153 	rc = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(dma_width));
3154 	if (rc) {
3155 		dev_err(dev, "dma_set_mask_and_coherent failed %d\n", rc);
3156 		goto err_mmio_read_less;
3157 	}
3158 
3159 	ena_devlink_params_get(adapter->devlink);
3160 
3161 	/* ENA admin level init */
3162 	rc = ena_com_admin_init(ena_dev, &aenq_handlers);
3163 	if (rc) {
3164 		dev_err(dev,
3165 			"Can not initialize ena admin queue with device\n");
3166 		goto err_mmio_read_less;
3167 	}
3168 
3169 	/* To enable the msix interrupts the driver needs to know the number
3170 	 * of queues. So the driver uses polling mode to retrieve this
3171 	 * information
3172 	 */
3173 	ena_com_set_admin_polling_mode(ena_dev, true);
3174 
3175 	ena_config_host_info(ena_dev, pdev);
3176 
3177 	/* Get Device Attributes*/
3178 	rc = ena_com_get_dev_attr_feat(ena_dev, get_feat_ctx);
3179 	if (rc) {
3180 		dev_err(dev, "Cannot get attribute for ena device rc=%d\n", rc);
3181 		goto err_admin_init;
3182 	}
3183 
3184 	/* Try to turn all the available aenq groups */
3185 	aenq_groups = BIT(ENA_ADMIN_LINK_CHANGE) |
3186 		BIT(ENA_ADMIN_FATAL_ERROR) |
3187 		BIT(ENA_ADMIN_WARNING) |
3188 		BIT(ENA_ADMIN_NOTIFICATION) |
3189 		BIT(ENA_ADMIN_KEEP_ALIVE);
3190 
3191 	aenq_groups &= get_feat_ctx->aenq.supported_groups;
3192 
3193 	rc = ena_com_set_aenq_config(ena_dev, aenq_groups);
3194 	if (rc) {
3195 		dev_err(dev, "Cannot configure aenq groups rc= %d\n", rc);
3196 		goto err_admin_init;
3197 	}
3198 
3199 	*wd_state = !!(aenq_groups & BIT(ENA_ADMIN_KEEP_ALIVE));
3200 
3201 	set_default_llq_configurations(adapter, &llq_config, &get_feat_ctx->llq);
3202 
3203 	rc = ena_set_queues_placement_policy(pdev, ena_dev, &get_feat_ctx->llq,
3204 					     &llq_config);
3205 	if (rc) {
3206 		netdev_err(netdev, "Cannot set queues placement policy rc= %d\n", rc);
3207 		goto err_admin_init;
3208 	}
3209 
3210 	rc = ena_calc_io_queue_size(adapter, get_feat_ctx);
3211 	if (unlikely(rc))
3212 		goto err_admin_init;
3213 
3214 	rc = ena_phc_init(adapter);
3215 	if (unlikely(rc && (rc != -EOPNOTSUPP)))
3216 		netdev_err(netdev, "Failed initializing PHC, error: %d\n", rc);
3217 
3218 	return 0;
3219 
3220 err_admin_init:
3221 	ena_com_abort_admin_commands(ena_dev);
3222 	ena_com_wait_for_abort_completion(ena_dev);
3223 	ena_com_delete_host_info(ena_dev);
3224 	ena_com_admin_destroy(ena_dev);
3225 err_mmio_read_less:
3226 	ena_com_mmio_reg_read_request_destroy(ena_dev);
3227 
3228 	return rc;
3229 }
3230 
3231 static int ena_enable_msix_and_set_admin_interrupts(struct ena_adapter *adapter)
3232 {
3233 	struct ena_com_dev *ena_dev = adapter->ena_dev;
3234 	struct device *dev = &adapter->pdev->dev;
3235 	int rc;
3236 
3237 	rc = ena_enable_msix(adapter);
3238 	if (rc) {
3239 		dev_err(dev, "Can not reserve msix vectors\n");
3240 		return rc;
3241 	}
3242 
3243 	ena_setup_mgmnt_intr(adapter);
3244 
3245 	rc = ena_request_mgmnt_irq(adapter);
3246 	if (rc) {
3247 		dev_err(dev, "Can not setup management interrupts\n");
3248 		goto err_disable_msix;
3249 	}
3250 
3251 	ena_com_set_admin_polling_mode(ena_dev, false);
3252 
3253 	ena_com_admin_aenq_enable(ena_dev);
3254 
3255 	return 0;
3256 
3257 err_disable_msix:
3258 	ena_disable_msix(adapter);
3259 
3260 	return rc;
3261 }
3262 
3263 int ena_destroy_device(struct ena_adapter *adapter, bool graceful)
3264 {
3265 	struct net_device *netdev = adapter->netdev;
3266 	struct ena_com_dev *ena_dev = adapter->ena_dev;
3267 	bool dev_up;
3268 	int rc = 0;
3269 
3270 	if (!test_bit(ENA_FLAG_DEVICE_RUNNING, &adapter->flags))
3271 		return 0;
3272 
3273 	netif_carrier_off(netdev);
3274 
3275 	timer_delete_sync(&adapter->timer_service);
3276 
3277 	dev_up = test_bit(ENA_FLAG_DEV_UP, &adapter->flags);
3278 	adapter->dev_up_before_reset = dev_up;
3279 	if (!graceful)
3280 		ena_com_set_admin_running_state(ena_dev, false);
3281 
3282 	if (dev_up)
3283 		ena_down(adapter);
3284 
3285 	/* Stop the device from sending AENQ events (in case reset flag is set
3286 	 *  and device is up, ena_down() already reset the device.
3287 	 */
3288 	if (!(test_bit(ENA_FLAG_TRIGGER_RESET, &adapter->flags) && dev_up))
3289 		rc = ena_com_dev_reset(adapter->ena_dev, adapter->reset_reason);
3290 
3291 	ena_free_mgmnt_irq(adapter);
3292 
3293 	ena_disable_msix(adapter);
3294 
3295 	ena_com_abort_admin_commands(ena_dev);
3296 
3297 	ena_com_wait_for_abort_completion(ena_dev);
3298 
3299 	ena_com_admin_destroy(ena_dev);
3300 
3301 	ena_phc_destroy(adapter);
3302 
3303 	ena_com_mmio_reg_read_request_destroy(ena_dev);
3304 
3305 	/* return reset reason to default value */
3306 	adapter->reset_reason = ENA_REGS_RESET_NORMAL;
3307 
3308 	clear_bit(ENA_FLAG_TRIGGER_RESET, &adapter->flags);
3309 	clear_bit(ENA_FLAG_DEVICE_RUNNING, &adapter->flags);
3310 
3311 	return rc;
3312 }
3313 
3314 int ena_restore_device(struct ena_adapter *adapter)
3315 {
3316 	struct ena_com_dev_get_features_ctx get_feat_ctx;
3317 	struct ena_com_dev *ena_dev = adapter->ena_dev;
3318 	struct pci_dev *pdev = adapter->pdev;
3319 	struct ena_ring *txr;
3320 	int rc, count, i;
3321 	bool wd_state;
3322 
3323 	set_bit(ENA_FLAG_ONGOING_RESET, &adapter->flags);
3324 	rc = ena_device_init(adapter, adapter->pdev, &get_feat_ctx, &wd_state);
3325 	if (rc) {
3326 		dev_err(&pdev->dev, "Can not initialize device\n");
3327 		goto err;
3328 	}
3329 	adapter->wd_state = wd_state;
3330 
3331 	count =  adapter->xdp_num_queues + adapter->num_io_queues;
3332 	for (i = 0 ; i < count; i++) {
3333 		txr = &adapter->tx_ring[i];
3334 		txr->tx_mem_queue_type = ena_dev->tx_mem_queue_type;
3335 		txr->tx_max_header_size = ena_dev->tx_max_header_size;
3336 	}
3337 
3338 	rc = ena_device_validate_params(adapter, &get_feat_ctx);
3339 	if (rc) {
3340 		dev_err(&pdev->dev, "Validation of device parameters failed\n");
3341 		goto err_device_destroy;
3342 	}
3343 
3344 	rc = ena_enable_msix_and_set_admin_interrupts(adapter);
3345 	if (rc) {
3346 		dev_err(&pdev->dev, "Enable MSI-X failed\n");
3347 		goto err_device_destroy;
3348 	}
3349 	/* If the interface was up before the reset bring it up */
3350 	if (adapter->dev_up_before_reset) {
3351 		rc = ena_up(adapter);
3352 		if (rc) {
3353 			dev_err(&pdev->dev, "Failed to create I/O queues\n");
3354 			goto err_disable_msix;
3355 		}
3356 	}
3357 
3358 	set_bit(ENA_FLAG_DEVICE_RUNNING, &adapter->flags);
3359 
3360 	clear_bit(ENA_FLAG_ONGOING_RESET, &adapter->flags);
3361 	if (test_bit(ENA_FLAG_LINK_UP, &adapter->flags))
3362 		netif_carrier_on(adapter->netdev);
3363 
3364 	mod_timer(&adapter->timer_service, round_jiffies(jiffies + HZ));
3365 	adapter->last_keep_alive_jiffies = jiffies;
3366 
3367 	return rc;
3368 err_disable_msix:
3369 	ena_free_mgmnt_irq(adapter);
3370 	ena_disable_msix(adapter);
3371 err_device_destroy:
3372 	ena_com_abort_admin_commands(ena_dev);
3373 	ena_com_wait_for_abort_completion(ena_dev);
3374 	ena_com_admin_destroy(ena_dev);
3375 	ena_com_dev_reset(ena_dev, ENA_REGS_RESET_DRIVER_INVALID_STATE);
3376 	ena_phc_destroy(adapter);
3377 	ena_com_mmio_reg_read_request_destroy(ena_dev);
3378 err:
3379 	clear_bit(ENA_FLAG_DEVICE_RUNNING, &adapter->flags);
3380 	clear_bit(ENA_FLAG_ONGOING_RESET, &adapter->flags);
3381 	dev_err(&pdev->dev,
3382 		"Reset attempt failed. Can not reset the device\n");
3383 
3384 	return rc;
3385 }
3386 
3387 static void ena_fw_reset_device(struct work_struct *work)
3388 {
3389 	int rc = 0;
3390 
3391 	struct ena_adapter *adapter =
3392 		container_of(work, struct ena_adapter, reset_task);
3393 
3394 	rtnl_lock();
3395 
3396 	if (likely(test_bit(ENA_FLAG_TRIGGER_RESET, &adapter->flags))) {
3397 		rc |= ena_destroy_device(adapter, false);
3398 		rc |= ena_restore_device(adapter);
3399 		adapter->dev_stats.reset_fail += !!rc;
3400 
3401 		dev_err(&adapter->pdev->dev, "Device reset completed successfully\n");
3402 	}
3403 
3404 	rtnl_unlock();
3405 }
3406 
3407 static int check_for_rx_interrupt_queue(struct ena_adapter *adapter,
3408 					struct ena_ring *rx_ring)
3409 {
3410 	struct ena_napi *ena_napi = container_of(rx_ring->napi, struct ena_napi, napi);
3411 
3412 	if (likely(READ_ONCE(ena_napi->first_interrupt)))
3413 		return 0;
3414 
3415 	if (ena_com_cq_empty(rx_ring->ena_com_io_cq))
3416 		return 0;
3417 
3418 	rx_ring->no_interrupt_event_cnt++;
3419 
3420 	if (rx_ring->no_interrupt_event_cnt == ENA_MAX_NO_INTERRUPT_ITERATIONS) {
3421 		netif_err(adapter, rx_err, adapter->netdev,
3422 			  "Potential MSIX issue on Rx side Queue = %d. Reset the device\n",
3423 			  rx_ring->qid);
3424 
3425 		ena_reset_device(adapter, ENA_REGS_RESET_MISS_INTERRUPT);
3426 		return -EIO;
3427 	}
3428 
3429 	return 0;
3430 }
3431 
3432 static int check_missing_comp_in_tx_queue(struct ena_adapter *adapter,
3433 					  struct ena_ring *tx_ring)
3434 {
3435 	struct ena_napi *ena_napi = container_of(tx_ring->napi, struct ena_napi, napi);
3436 	enum ena_regs_reset_reason_types reset_reason = ENA_REGS_RESET_MISS_TX_CMPL;
3437 	unsigned int time_since_last_napi;
3438 	unsigned int missing_tx_comp_to;
3439 	bool is_tx_comp_time_expired;
3440 	struct ena_tx_buffer *tx_buf;
3441 	unsigned long last_jiffies;
3442 	int napi_scheduled;
3443 	u32 missed_tx = 0;
3444 	int i, rc = 0;
3445 
3446 	missing_tx_comp_to = jiffies_to_msecs(adapter->missing_tx_completion_to);
3447 
3448 	for (i = 0; i < tx_ring->ring_size; i++) {
3449 		tx_buf = &tx_ring->tx_buffer_info[i];
3450 		last_jiffies = tx_buf->last_jiffies;
3451 
3452 		if (last_jiffies == 0)
3453 			/* no pending Tx at this location */
3454 			continue;
3455 
3456 		is_tx_comp_time_expired = time_is_before_jiffies(last_jiffies +
3457 			 2 * adapter->missing_tx_completion_to);
3458 
3459 		if (unlikely(!READ_ONCE(ena_napi->first_interrupt) && is_tx_comp_time_expired)) {
3460 			/* If after graceful period interrupt is still not
3461 			 * received, we schedule a reset
3462 			 */
3463 			netif_err(adapter, tx_err, adapter->netdev,
3464 				  "Potential MSIX issue on Tx side Queue = %d. Reset the device\n",
3465 				  tx_ring->qid);
3466 			ena_reset_device(adapter, ENA_REGS_RESET_MISS_INTERRUPT);
3467 			return -EIO;
3468 		}
3469 
3470 		is_tx_comp_time_expired = time_is_before_jiffies(last_jiffies +
3471 			adapter->missing_tx_completion_to);
3472 
3473 		if (unlikely(is_tx_comp_time_expired)) {
3474 			time_since_last_napi =
3475 				jiffies_to_usecs(jiffies - tx_ring->tx_stats.last_napi_jiffies);
3476 			napi_scheduled = !!(ena_napi->napi.state & NAPIF_STATE_SCHED);
3477 
3478 			if (missing_tx_comp_to < time_since_last_napi && napi_scheduled) {
3479 				/* We suspect napi isn't called because the
3480 				 * bottom half is not run. Require a bigger
3481 				 * timeout for these cases
3482 				 */
3483 				if (!time_is_before_jiffies(last_jiffies +
3484 					2 * adapter->missing_tx_completion_to))
3485 					continue;
3486 
3487 				reset_reason = ENA_REGS_RESET_SUSPECTED_POLL_STARVATION;
3488 			}
3489 
3490 			missed_tx++;
3491 
3492 			if (tx_buf->print_once)
3493 				continue;
3494 
3495 			netif_notice(adapter, tx_err, adapter->netdev,
3496 				     "TX hasn't completed, qid %d, index %d. %u usecs from last napi execution, napi scheduled: %d\n",
3497 				     tx_ring->qid, i, time_since_last_napi, napi_scheduled);
3498 
3499 			tx_buf->print_once = 1;
3500 		}
3501 	}
3502 
3503 	if (unlikely(missed_tx > adapter->missing_tx_completion_threshold)) {
3504 		netif_err(adapter, tx_err, adapter->netdev,
3505 			  "Lost TX completions are above the threshold (%d > %d). Completion transmission timeout: %u.\n",
3506 			  missed_tx,
3507 			  adapter->missing_tx_completion_threshold,
3508 			  missing_tx_comp_to);
3509 		netif_err(adapter, tx_err, adapter->netdev,
3510 			  "Resetting the device\n");
3511 
3512 		ena_reset_device(adapter, reset_reason);
3513 		rc = -EIO;
3514 	}
3515 
3516 	ena_increase_stat(&tx_ring->tx_stats.missed_tx, missed_tx,
3517 			  &tx_ring->syncp);
3518 
3519 	return rc;
3520 }
3521 
3522 static void check_for_missing_completions(struct ena_adapter *adapter)
3523 {
3524 	struct ena_ring *tx_ring;
3525 	struct ena_ring *rx_ring;
3526 	int qid, budget, rc;
3527 	int io_queue_count;
3528 
3529 	io_queue_count = adapter->xdp_num_queues + adapter->num_io_queues;
3530 
3531 	/* Make sure the driver doesn't turn the device in other process */
3532 	smp_rmb();
3533 
3534 	if (!test_bit(ENA_FLAG_DEV_UP, &adapter->flags))
3535 		return;
3536 
3537 	if (test_bit(ENA_FLAG_TRIGGER_RESET, &adapter->flags))
3538 		return;
3539 
3540 	if (adapter->missing_tx_completion_to == ENA_HW_HINTS_NO_TIMEOUT)
3541 		return;
3542 
3543 	budget = min_t(u32, io_queue_count, ENA_MONITORED_TX_QUEUES);
3544 
3545 	qid = adapter->last_monitored_tx_qid;
3546 
3547 	while (budget) {
3548 		qid = (qid + 1) % io_queue_count;
3549 
3550 		tx_ring = &adapter->tx_ring[qid];
3551 		rx_ring = &adapter->rx_ring[qid];
3552 
3553 		rc = check_missing_comp_in_tx_queue(adapter, tx_ring);
3554 		if (unlikely(rc))
3555 			return;
3556 
3557 		rc =  !ENA_IS_XDP_INDEX(adapter, qid) ?
3558 			check_for_rx_interrupt_queue(adapter, rx_ring) : 0;
3559 		if (unlikely(rc))
3560 			return;
3561 
3562 		budget--;
3563 	}
3564 
3565 	adapter->last_monitored_tx_qid = qid;
3566 }
3567 
3568 /* trigger napi schedule after 2 consecutive detections */
3569 #define EMPTY_RX_REFILL 2
3570 /* For the rare case where the device runs out of Rx descriptors and the
3571  * napi handler failed to refill new Rx descriptors (due to a lack of memory
3572  * for example).
3573  * This case will lead to a deadlock:
3574  * The device won't send interrupts since all the new Rx packets will be dropped
3575  * The napi handler won't allocate new Rx descriptors so the device will be
3576  * able to send new packets.
3577  *
3578  * This scenario can happen when the kernel's vm.min_free_kbytes is too small.
3579  * It is recommended to have at least 512MB, with a minimum of 128MB for
3580  * constrained environment).
3581  *
3582  * When such a situation is detected - Reschedule napi
3583  */
3584 static void check_for_empty_rx_ring(struct ena_adapter *adapter)
3585 {
3586 	struct ena_ring *rx_ring;
3587 	int i, refill_required;
3588 
3589 	if (!test_bit(ENA_FLAG_DEV_UP, &adapter->flags))
3590 		return;
3591 
3592 	if (test_bit(ENA_FLAG_TRIGGER_RESET, &adapter->flags))
3593 		return;
3594 
3595 	for (i = 0; i < adapter->num_io_queues; i++) {
3596 		rx_ring = &adapter->rx_ring[i];
3597 
3598 		refill_required = ena_com_free_q_entries(rx_ring->ena_com_io_sq);
3599 		if (unlikely(refill_required == (rx_ring->ring_size - 1))) {
3600 			rx_ring->empty_rx_queue++;
3601 
3602 			if (rx_ring->empty_rx_queue >= EMPTY_RX_REFILL) {
3603 				ena_increase_stat(&rx_ring->rx_stats.empty_rx_ring, 1,
3604 						  &rx_ring->syncp);
3605 
3606 				netif_err(adapter, drv, adapter->netdev,
3607 					  "Trigger refill for ring %d\n", i);
3608 
3609 				napi_schedule(rx_ring->napi);
3610 				rx_ring->empty_rx_queue = 0;
3611 			}
3612 		} else {
3613 			rx_ring->empty_rx_queue = 0;
3614 		}
3615 	}
3616 }
3617 
3618 /* Check for keep alive expiration */
3619 static void check_for_missing_keep_alive(struct ena_adapter *adapter)
3620 {
3621 	unsigned long keep_alive_expired;
3622 
3623 	if (!adapter->wd_state)
3624 		return;
3625 
3626 	if (adapter->keep_alive_timeout == ENA_HW_HINTS_NO_TIMEOUT)
3627 		return;
3628 
3629 	keep_alive_expired = adapter->last_keep_alive_jiffies +
3630 			     adapter->keep_alive_timeout;
3631 	if (unlikely(time_is_before_jiffies(keep_alive_expired))) {
3632 		netif_err(adapter, drv, adapter->netdev,
3633 			  "Keep alive watchdog timeout.\n");
3634 		ena_increase_stat(&adapter->dev_stats.wd_expired, 1,
3635 				  &adapter->syncp);
3636 		ena_reset_device(adapter, ENA_REGS_RESET_KEEP_ALIVE_TO);
3637 	}
3638 }
3639 
3640 static void check_for_admin_com_state(struct ena_adapter *adapter)
3641 {
3642 	if (unlikely(!ena_com_get_admin_running_state(adapter->ena_dev))) {
3643 		netif_err(adapter, drv, adapter->netdev,
3644 			  "ENA admin queue is not in running state!\n");
3645 		ena_increase_stat(&adapter->dev_stats.admin_q_pause, 1,
3646 				  &adapter->syncp);
3647 		ena_reset_device(adapter, ENA_REGS_RESET_ADMIN_TO);
3648 	}
3649 }
3650 
3651 static void ena_update_hints(struct ena_adapter *adapter,
3652 			     struct ena_admin_ena_hw_hints *hints)
3653 {
3654 	struct net_device *netdev = adapter->netdev;
3655 
3656 	if (hints->admin_completion_tx_timeout)
3657 		adapter->ena_dev->admin_queue.completion_timeout =
3658 			hints->admin_completion_tx_timeout * 1000;
3659 
3660 	if (hints->mmio_read_timeout)
3661 		/* convert to usec */
3662 		adapter->ena_dev->mmio_read.reg_read_to =
3663 			hints->mmio_read_timeout * 1000;
3664 
3665 	if (hints->missed_tx_completion_count_threshold_to_reset)
3666 		adapter->missing_tx_completion_threshold =
3667 			hints->missed_tx_completion_count_threshold_to_reset;
3668 
3669 	if (hints->missing_tx_completion_timeout) {
3670 		if (hints->missing_tx_completion_timeout == ENA_HW_HINTS_NO_TIMEOUT)
3671 			adapter->missing_tx_completion_to = ENA_HW_HINTS_NO_TIMEOUT;
3672 		else
3673 			adapter->missing_tx_completion_to =
3674 				msecs_to_jiffies(hints->missing_tx_completion_timeout);
3675 	}
3676 
3677 	if (hints->netdev_wd_timeout)
3678 		netdev->watchdog_timeo = msecs_to_jiffies(hints->netdev_wd_timeout);
3679 
3680 	if (hints->driver_watchdog_timeout) {
3681 		if (hints->driver_watchdog_timeout == ENA_HW_HINTS_NO_TIMEOUT)
3682 			adapter->keep_alive_timeout = ENA_HW_HINTS_NO_TIMEOUT;
3683 		else
3684 			adapter->keep_alive_timeout =
3685 				msecs_to_jiffies(hints->driver_watchdog_timeout);
3686 	}
3687 }
3688 
3689 static void ena_update_host_info(struct ena_admin_host_info *host_info,
3690 				 struct net_device *netdev)
3691 {
3692 	host_info->supported_network_features[0] =
3693 		netdev->features & GENMASK_ULL(31, 0);
3694 	host_info->supported_network_features[1] =
3695 		(netdev->features & GENMASK_ULL(63, 32)) >> 32;
3696 }
3697 
3698 static void ena_timer_service(struct timer_list *t)
3699 {
3700 	struct ena_adapter *adapter = timer_container_of(adapter, t,
3701 							 timer_service);
3702 	u8 *debug_area = adapter->ena_dev->host_attr.debug_area_virt_addr;
3703 	struct ena_admin_host_info *host_info =
3704 		adapter->ena_dev->host_attr.host_info;
3705 
3706 	check_for_missing_keep_alive(adapter);
3707 
3708 	check_for_admin_com_state(adapter);
3709 
3710 	check_for_missing_completions(adapter);
3711 
3712 	check_for_empty_rx_ring(adapter);
3713 
3714 	if (debug_area)
3715 		ena_dump_stats_to_buf(adapter, debug_area);
3716 
3717 	if (host_info)
3718 		ena_update_host_info(host_info, adapter->netdev);
3719 
3720 	if (unlikely(test_bit(ENA_FLAG_TRIGGER_RESET, &adapter->flags))) {
3721 		netif_err(adapter, drv, adapter->netdev,
3722 			  "Trigger reset is on\n");
3723 		ena_dump_stats_to_dmesg(adapter);
3724 		queue_work(ena_wq, &adapter->reset_task);
3725 		return;
3726 	}
3727 
3728 	/* Reset the timer */
3729 	mod_timer(&adapter->timer_service, round_jiffies(jiffies + HZ));
3730 }
3731 
3732 static u32 ena_calc_max_io_queue_num(struct pci_dev *pdev,
3733 				     struct ena_com_dev *ena_dev,
3734 				     struct ena_com_dev_get_features_ctx *get_feat_ctx)
3735 {
3736 	u32 io_tx_sq_num, io_tx_cq_num, io_rx_num, max_num_io_queues;
3737 
3738 	if (ena_dev->supported_features & BIT(ENA_ADMIN_MAX_QUEUES_EXT)) {
3739 		struct ena_admin_queue_ext_feature_fields *max_queue_ext =
3740 			&get_feat_ctx->max_queue_ext.max_queue_ext;
3741 		io_rx_num = min_t(u32, max_queue_ext->max_rx_sq_num,
3742 				  max_queue_ext->max_rx_cq_num);
3743 
3744 		io_tx_sq_num = max_queue_ext->max_tx_sq_num;
3745 		io_tx_cq_num = max_queue_ext->max_tx_cq_num;
3746 	} else {
3747 		struct ena_admin_queue_feature_desc *max_queues =
3748 			&get_feat_ctx->max_queues;
3749 		io_tx_sq_num = max_queues->max_sq_num;
3750 		io_tx_cq_num = max_queues->max_cq_num;
3751 		io_rx_num = min_t(u32, io_tx_sq_num, io_tx_cq_num);
3752 	}
3753 
3754 	/* In case of LLQ use the llq fields for the tx SQ/CQ */
3755 	if (ena_dev->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV)
3756 		io_tx_sq_num = get_feat_ctx->llq.max_llq_num;
3757 
3758 	max_num_io_queues = min_t(u32, num_online_cpus(), ENA_MAX_NUM_IO_QUEUES);
3759 	max_num_io_queues = min_t(u32, max_num_io_queues, io_rx_num);
3760 	max_num_io_queues = min_t(u32, max_num_io_queues, io_tx_sq_num);
3761 	max_num_io_queues = min_t(u32, max_num_io_queues, io_tx_cq_num);
3762 	/* 1 IRQ for mgmnt and 1 IRQs for each IO direction */
3763 	max_num_io_queues = min_t(u32, max_num_io_queues, pci_msix_vec_count(pdev) - 1);
3764 
3765 	return max_num_io_queues;
3766 }
3767 
3768 static void ena_set_dev_offloads(struct ena_com_dev_get_features_ctx *feat,
3769 				 struct net_device *netdev)
3770 {
3771 	netdev_features_t dev_features = 0;
3772 
3773 	/* Set offload features */
3774 	if (feat->offload.tx &
3775 		ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV4_CSUM_PART_MASK)
3776 		dev_features |= NETIF_F_IP_CSUM;
3777 
3778 	if (feat->offload.tx &
3779 		ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV6_CSUM_PART_MASK)
3780 		dev_features |= NETIF_F_IPV6_CSUM;
3781 
3782 	if (feat->offload.tx & ENA_ADMIN_FEATURE_OFFLOAD_DESC_TSO_IPV4_MASK)
3783 		dev_features |= NETIF_F_TSO;
3784 
3785 	if (feat->offload.tx & ENA_ADMIN_FEATURE_OFFLOAD_DESC_TSO_IPV6_MASK)
3786 		dev_features |= NETIF_F_TSO6;
3787 
3788 	if (feat->offload.tx & ENA_ADMIN_FEATURE_OFFLOAD_DESC_TSO_ECN_MASK)
3789 		dev_features |= NETIF_F_TSO_ECN;
3790 
3791 	if (feat->offload.rx_supported &
3792 		ENA_ADMIN_FEATURE_OFFLOAD_DESC_RX_L4_IPV4_CSUM_MASK)
3793 		dev_features |= NETIF_F_RXCSUM;
3794 
3795 	if (feat->offload.rx_supported &
3796 		ENA_ADMIN_FEATURE_OFFLOAD_DESC_RX_L4_IPV6_CSUM_MASK)
3797 		dev_features |= NETIF_F_RXCSUM;
3798 
3799 	netdev->features =
3800 		dev_features |
3801 		NETIF_F_SG |
3802 		NETIF_F_RXHASH |
3803 		NETIF_F_HIGHDMA;
3804 
3805 	netdev->hw_features |= netdev->features;
3806 	netdev->vlan_features |= netdev->features;
3807 }
3808 
3809 static void ena_set_conf_feat_params(struct ena_adapter *adapter,
3810 				     struct ena_com_dev_get_features_ctx *feat)
3811 {
3812 	struct net_device *netdev = adapter->netdev;
3813 
3814 	/* Copy mac address */
3815 	if (!is_valid_ether_addr(feat->dev_attr.mac_addr)) {
3816 		eth_hw_addr_random(netdev);
3817 		ether_addr_copy(adapter->mac_addr, netdev->dev_addr);
3818 	} else {
3819 		ether_addr_copy(adapter->mac_addr, feat->dev_attr.mac_addr);
3820 		eth_hw_addr_set(netdev, adapter->mac_addr);
3821 	}
3822 
3823 	/* Set offload features */
3824 	ena_set_dev_offloads(feat, netdev);
3825 
3826 	adapter->max_mtu = feat->dev_attr.max_mtu;
3827 	netdev->max_mtu = adapter->max_mtu;
3828 	netdev->min_mtu = ENA_MIN_MTU;
3829 }
3830 
3831 static int ena_rss_init_default(struct ena_adapter *adapter)
3832 {
3833 	struct ena_com_dev *ena_dev = adapter->ena_dev;
3834 	struct device *dev = &adapter->pdev->dev;
3835 	int rc, i;
3836 	u32 val;
3837 
3838 	rc = ena_com_rss_init(ena_dev, ENA_RX_RSS_TABLE_LOG_SIZE);
3839 	if (unlikely(rc)) {
3840 		dev_err(dev, "Cannot init indirect table\n");
3841 		goto err_rss_init;
3842 	}
3843 
3844 	for (i = 0; i < ENA_RX_RSS_TABLE_SIZE; i++) {
3845 		val = ethtool_rxfh_indir_default(i, adapter->num_io_queues);
3846 		rc = ena_com_indirect_table_fill_entry(ena_dev, i,
3847 						       ENA_IO_RXQ_IDX(val));
3848 		if (unlikely(rc)) {
3849 			dev_err(dev, "Cannot fill indirect table\n");
3850 			goto err_fill_indir;
3851 		}
3852 	}
3853 
3854 	rc = ena_com_fill_hash_function(ena_dev, ENA_ADMIN_TOEPLITZ, NULL, ENA_HASH_KEY_SIZE,
3855 					0xFFFFFFFF);
3856 	if (unlikely(rc && (rc != -EOPNOTSUPP))) {
3857 		dev_err(dev, "Cannot fill hash function\n");
3858 		goto err_fill_indir;
3859 	}
3860 
3861 	rc = ena_com_set_default_hash_ctrl(ena_dev);
3862 	if (unlikely(rc && (rc != -EOPNOTSUPP))) {
3863 		dev_err(dev, "Cannot fill hash control\n");
3864 		goto err_fill_indir;
3865 	}
3866 
3867 	return 0;
3868 
3869 err_fill_indir:
3870 	ena_com_rss_destroy(ena_dev);
3871 err_rss_init:
3872 
3873 	return rc;
3874 }
3875 
3876 static void ena_release_bars(struct ena_com_dev *ena_dev, struct pci_dev *pdev)
3877 {
3878 	int release_bars = pci_select_bars(pdev, IORESOURCE_MEM) & ENA_BAR_MASK;
3879 
3880 	pci_release_selected_regions(pdev, release_bars);
3881 }
3882 
3883 /* ena_probe - Device Initialization Routine
3884  * @pdev: PCI device information struct
3885  * @ent: entry in ena_pci_tbl
3886  *
3887  * Returns 0 on success, negative on failure
3888  *
3889  * ena_probe initializes an adapter identified by a pci_dev structure.
3890  * The OS initialization, configuring of the adapter private structure,
3891  * and a hardware reset occur.
3892  */
3893 static int ena_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
3894 {
3895 	struct ena_com_dev_get_features_ctx get_feat_ctx;
3896 	struct ena_com_dev *ena_dev = NULL;
3897 	struct ena_adapter *adapter;
3898 	struct net_device *netdev;
3899 	static int adapters_found;
3900 	struct devlink *devlink;
3901 	u32 max_num_io_queues;
3902 	bool wd_state;
3903 	int bars, rc;
3904 
3905 	dev_dbg(&pdev->dev, "%s\n", __func__);
3906 
3907 	rc = pci_enable_device_mem(pdev);
3908 	if (rc) {
3909 		dev_err(&pdev->dev, "pci_enable_device_mem() failed!\n");
3910 		return rc;
3911 	}
3912 
3913 	rc = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(ENA_MAX_PHYS_ADDR_SIZE_BITS));
3914 	if (rc) {
3915 		dev_err(&pdev->dev, "dma_set_mask_and_coherent failed %d\n", rc);
3916 		goto err_disable_device;
3917 	}
3918 
3919 	pci_set_master(pdev);
3920 
3921 	ena_dev = vzalloc(sizeof(*ena_dev));
3922 	if (!ena_dev) {
3923 		rc = -ENOMEM;
3924 		goto err_disable_device;
3925 	}
3926 
3927 	bars = pci_select_bars(pdev, IORESOURCE_MEM) & ENA_BAR_MASK;
3928 	rc = pci_request_selected_regions(pdev, bars, DRV_MODULE_NAME);
3929 	if (rc) {
3930 		dev_err(&pdev->dev, "pci_request_selected_regions failed %d\n",
3931 			rc);
3932 		goto err_free_ena_dev;
3933 	}
3934 
3935 	ena_dev->reg_bar = devm_ioremap(&pdev->dev,
3936 					pci_resource_start(pdev, ENA_REG_BAR),
3937 					pci_resource_len(pdev, ENA_REG_BAR));
3938 	if (!ena_dev->reg_bar) {
3939 		dev_err(&pdev->dev, "Failed to remap regs bar\n");
3940 		rc = -EFAULT;
3941 		goto err_free_region;
3942 	}
3943 
3944 	ena_dev->ena_min_poll_delay_us = ENA_ADMIN_POLL_DELAY_US;
3945 
3946 	ena_dev->dmadev = &pdev->dev;
3947 
3948 	netdev = alloc_etherdev_mq(sizeof(struct ena_adapter), ENA_MAX_RINGS);
3949 	if (!netdev) {
3950 		dev_err(&pdev->dev, "alloc_etherdev_mq failed\n");
3951 		rc = -ENOMEM;
3952 		goto err_free_region;
3953 	}
3954 
3955 	SET_NETDEV_DEV(netdev, &pdev->dev);
3956 	adapter = netdev_priv(netdev);
3957 	adapter->ena_dev = ena_dev;
3958 	adapter->netdev = netdev;
3959 	adapter->pdev = pdev;
3960 	adapter->msg_enable = DEFAULT_MSG_ENABLE;
3961 
3962 	ena_dev->net_device = netdev;
3963 
3964 	pci_set_drvdata(pdev, adapter);
3965 
3966 	rc = ena_phc_alloc(adapter);
3967 	if (rc) {
3968 		netdev_err(netdev, "ena_phc_alloc failed\n");
3969 		goto err_netdev_destroy;
3970 	}
3971 
3972 	rc = ena_com_allocate_customer_metrics_buffer(ena_dev);
3973 	if (rc) {
3974 		netdev_err(netdev, "ena_com_allocate_customer_metrics_buffer failed\n");
3975 		goto err_free_phc;
3976 	}
3977 
3978 	rc = ena_map_llq_mem_bar(pdev, ena_dev, bars);
3979 	if (rc) {
3980 		dev_err(&pdev->dev, "ENA LLQ bar mapping failed\n");
3981 		goto err_metrics_destroy;
3982 	}
3983 
3984 	/* Need to do this before ena_device_init */
3985 	devlink = ena_devlink_alloc(adapter);
3986 	if (!devlink) {
3987 		netdev_err(netdev, "ena_devlink_alloc failed\n");
3988 		rc = -ENOMEM;
3989 		goto err_metrics_destroy;
3990 	}
3991 
3992 	rc = ena_device_init(adapter, pdev, &get_feat_ctx, &wd_state);
3993 	if (rc) {
3994 		dev_err(&pdev->dev, "ENA device init failed\n");
3995 		if (rc == -ETIME)
3996 			rc = -EPROBE_DEFER;
3997 		goto ena_devlink_destroy;
3998 	}
3999 
4000 	/* Initial TX and RX interrupt delay. Assumes 1 usec granularity.
4001 	 * Updated during device initialization with the real granularity
4002 	 */
4003 	ena_dev->intr_moder_tx_interval = ENA_INTR_INITIAL_TX_INTERVAL_USECS;
4004 	ena_dev->intr_moder_rx_interval = ENA_INTR_INITIAL_RX_INTERVAL_USECS;
4005 	ena_dev->intr_delay_resolution = ENA_DEFAULT_INTR_DELAY_RESOLUTION;
4006 	max_num_io_queues = ena_calc_max_io_queue_num(pdev, ena_dev, &get_feat_ctx);
4007 	if (unlikely(!max_num_io_queues)) {
4008 		rc = -EFAULT;
4009 		goto err_device_destroy;
4010 	}
4011 
4012 	ena_set_conf_feat_params(adapter, &get_feat_ctx);
4013 
4014 	adapter->reset_reason = ENA_REGS_RESET_NORMAL;
4015 
4016 	adapter->num_io_queues = max_num_io_queues;
4017 	adapter->max_num_io_queues = max_num_io_queues;
4018 	adapter->last_monitored_tx_qid = 0;
4019 
4020 	adapter->xdp_first_ring = 0;
4021 	adapter->xdp_num_queues = 0;
4022 
4023 	adapter->rx_copybreak = ENA_DEFAULT_RX_COPYBREAK;
4024 	if (ena_dev->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV)
4025 		adapter->disable_meta_caching =
4026 			!!(get_feat_ctx.llq.accel_mode.u.get.supported_flags &
4027 			   BIT(ENA_ADMIN_DISABLE_META_CACHING));
4028 
4029 	adapter->wd_state = wd_state;
4030 
4031 	snprintf(adapter->name, ENA_NAME_MAX_LEN, "ena_%d", adapters_found);
4032 
4033 	rc = ena_com_init_interrupt_moderation(adapter->ena_dev);
4034 	if (rc) {
4035 		dev_err(&pdev->dev,
4036 			"Failed to query interrupt moderation feature\n");
4037 		goto err_device_destroy;
4038 	}
4039 
4040 	ena_init_io_rings(adapter,
4041 			  0,
4042 			  adapter->xdp_num_queues +
4043 			  adapter->num_io_queues);
4044 
4045 	netdev->netdev_ops = &ena_netdev_ops;
4046 	netdev->watchdog_timeo = TX_TIMEOUT;
4047 	ena_set_ethtool_ops(netdev);
4048 
4049 	netdev->priv_flags |= IFF_UNICAST_FLT;
4050 
4051 	u64_stats_init(&adapter->syncp);
4052 
4053 	rc = ena_enable_msix_and_set_admin_interrupts(adapter);
4054 	if (rc) {
4055 		dev_err(&pdev->dev,
4056 			"Failed to enable and set the admin interrupts\n");
4057 		goto err_worker_destroy;
4058 	}
4059 	rc = ena_rss_init_default(adapter);
4060 	if (rc && (rc != -EOPNOTSUPP)) {
4061 		dev_err(&pdev->dev, "Cannot init RSS rc: %d\n", rc);
4062 		goto err_free_msix;
4063 	}
4064 
4065 	ena_config_debug_area(adapter);
4066 
4067 	if (ena_xdp_legal_queue_count(adapter, adapter->num_io_queues))
4068 		netdev->xdp_features = NETDEV_XDP_ACT_BASIC |
4069 				       NETDEV_XDP_ACT_REDIRECT;
4070 
4071 	memcpy(adapter->netdev->perm_addr, adapter->mac_addr, netdev->addr_len);
4072 
4073 	netif_carrier_off(netdev);
4074 
4075 	rc = register_netdev(netdev);
4076 	if (rc) {
4077 		dev_err(&pdev->dev, "Cannot register net device\n");
4078 		goto err_rss;
4079 	}
4080 
4081 	ena_debugfs_init(netdev);
4082 
4083 	INIT_WORK(&adapter->reset_task, ena_fw_reset_device);
4084 
4085 	adapter->last_keep_alive_jiffies = jiffies;
4086 	adapter->keep_alive_timeout = ENA_DEVICE_KALIVE_TIMEOUT;
4087 	adapter->missing_tx_completion_to = TX_TIMEOUT;
4088 	adapter->missing_tx_completion_threshold = MAX_NUM_OF_TIMEOUTED_PACKETS;
4089 
4090 	ena_update_hints(adapter, &get_feat_ctx.hw_hints);
4091 
4092 	timer_setup(&adapter->timer_service, ena_timer_service, 0);
4093 	mod_timer(&adapter->timer_service, round_jiffies(jiffies + HZ));
4094 
4095 	dev_info(&pdev->dev,
4096 		 "%s found at mem %lx, mac addr %pM\n",
4097 		 DEVICE_NAME, (long)pci_resource_start(pdev, 0),
4098 		 netdev->dev_addr);
4099 
4100 	set_bit(ENA_FLAG_DEVICE_RUNNING, &adapter->flags);
4101 
4102 	adapters_found++;
4103 
4104 	/* From this point, the devlink device is visible to users.
4105 	 * Perform the registration last to ensure that all the resources
4106 	 * are available and that the netdevice is registered.
4107 	 */
4108 	ena_devlink_register(devlink, &pdev->dev);
4109 
4110 	return 0;
4111 
4112 err_rss:
4113 	ena_com_delete_debug_area(ena_dev);
4114 	ena_com_rss_destroy(ena_dev);
4115 err_free_msix:
4116 	ena_com_dev_reset(ena_dev, ENA_REGS_RESET_INIT_ERR);
4117 	/* stop submitting admin commands on a device that was reset */
4118 	ena_com_set_admin_running_state(ena_dev, false);
4119 	ena_free_mgmnt_irq(adapter);
4120 	ena_disable_msix(adapter);
4121 err_worker_destroy:
4122 	timer_delete(&adapter->timer_service);
4123 err_device_destroy:
4124 	ena_com_delete_host_info(ena_dev);
4125 	ena_com_admin_destroy(ena_dev);
4126 	ena_phc_destroy(adapter);
4127 	ena_com_mmio_reg_read_request_destroy(ena_dev);
4128 ena_devlink_destroy:
4129 	ena_devlink_free(devlink);
4130 err_metrics_destroy:
4131 	ena_com_delete_customer_metrics_buffer(ena_dev);
4132 err_free_phc:
4133 	ena_phc_free(adapter);
4134 err_netdev_destroy:
4135 	free_netdev(netdev);
4136 err_free_region:
4137 	ena_release_bars(ena_dev, pdev);
4138 err_free_ena_dev:
4139 	vfree(ena_dev);
4140 err_disable_device:
4141 	pci_disable_device(pdev);
4142 	return rc;
4143 }
4144 
4145 /*****************************************************************************/
4146 
4147 /* __ena_shutoff - Helper used in both PCI remove/shutdown routines
4148  * @pdev: PCI device information struct
4149  * @shutdown: Is it a shutdown operation? If false, means it is a removal
4150  *
4151  * __ena_shutoff is a helper routine that does the real work on shutdown and
4152  * removal paths; the difference between those paths is with regards to whether
4153  * dettach or unregister the netdevice.
4154  */
4155 static void __ena_shutoff(struct pci_dev *pdev, bool shutdown)
4156 {
4157 	struct ena_adapter *adapter = pci_get_drvdata(pdev);
4158 	struct ena_com_dev *ena_dev;
4159 	struct net_device *netdev;
4160 
4161 	ena_dev = adapter->ena_dev;
4162 	netdev = adapter->netdev;
4163 
4164 	ena_debugfs_terminate(netdev);
4165 
4166 	/* Make sure timer and reset routine won't be called after
4167 	 * freeing device resources.
4168 	 */
4169 	timer_delete_sync(&adapter->timer_service);
4170 	cancel_work_sync(&adapter->reset_task);
4171 
4172 	rtnl_lock(); /* lock released inside the below if-else block */
4173 	adapter->reset_reason = ENA_REGS_RESET_SHUTDOWN;
4174 	ena_destroy_device(adapter, true);
4175 
4176 	ena_phc_free(adapter);
4177 
4178 	ena_devlink_unregister(adapter->devlink);
4179 	ena_devlink_free(adapter->devlink);
4180 
4181 	if (shutdown) {
4182 		netif_device_detach(netdev);
4183 		dev_close(netdev);
4184 		rtnl_unlock();
4185 	} else {
4186 		rtnl_unlock();
4187 		unregister_netdev(netdev);
4188 		free_netdev(netdev);
4189 	}
4190 
4191 	ena_com_rss_destroy(ena_dev);
4192 
4193 	ena_com_delete_debug_area(ena_dev);
4194 
4195 	ena_com_delete_host_info(ena_dev);
4196 
4197 	ena_com_delete_customer_metrics_buffer(ena_dev);
4198 
4199 	ena_release_bars(ena_dev, pdev);
4200 
4201 	pci_disable_device(pdev);
4202 
4203 	vfree(ena_dev);
4204 }
4205 
4206 /* ena_remove - Device Removal Routine
4207  * @pdev: PCI device information struct
4208  *
4209  * ena_remove is called by the PCI subsystem to alert the driver
4210  * that it should release a PCI device.
4211  */
4212 
4213 static void ena_remove(struct pci_dev *pdev)
4214 {
4215 	__ena_shutoff(pdev, false);
4216 }
4217 
4218 /* ena_shutdown - Device Shutdown Routine
4219  * @pdev: PCI device information struct
4220  *
4221  * ena_shutdown is called by the PCI subsystem to alert the driver that
4222  * a shutdown/reboot (or kexec) is happening and device must be disabled.
4223  */
4224 
4225 static void ena_shutdown(struct pci_dev *pdev)
4226 {
4227 	__ena_shutoff(pdev, true);
4228 }
4229 
4230 /* ena_suspend - PM suspend callback
4231  * @dev_d: Device information struct
4232  */
4233 static int __maybe_unused ena_suspend(struct device *dev_d)
4234 {
4235 	struct pci_dev *pdev = to_pci_dev(dev_d);
4236 	struct ena_adapter *adapter = pci_get_drvdata(pdev);
4237 
4238 	ena_increase_stat(&adapter->dev_stats.suspend, 1, &adapter->syncp);
4239 
4240 	rtnl_lock();
4241 	if (unlikely(test_bit(ENA_FLAG_TRIGGER_RESET, &adapter->flags))) {
4242 		dev_err(&pdev->dev,
4243 			"Ignoring device reset request as the device is being suspended\n");
4244 		clear_bit(ENA_FLAG_TRIGGER_RESET, &adapter->flags);
4245 	}
4246 	ena_destroy_device(adapter, true);
4247 	rtnl_unlock();
4248 	return 0;
4249 }
4250 
4251 /* ena_resume - PM resume callback
4252  * @dev_d: Device information struct
4253  */
4254 static int __maybe_unused ena_resume(struct device *dev_d)
4255 {
4256 	struct ena_adapter *adapter = dev_get_drvdata(dev_d);
4257 	int rc;
4258 
4259 	ena_increase_stat(&adapter->dev_stats.resume, 1, &adapter->syncp);
4260 
4261 	rtnl_lock();
4262 	rc = ena_restore_device(adapter);
4263 	rtnl_unlock();
4264 	return rc;
4265 }
4266 
4267 static SIMPLE_DEV_PM_OPS(ena_pm_ops, ena_suspend, ena_resume);
4268 
4269 static struct pci_driver ena_pci_driver = {
4270 	.name		= DRV_MODULE_NAME,
4271 	.id_table	= ena_pci_tbl,
4272 	.probe		= ena_probe,
4273 	.remove		= ena_remove,
4274 	.shutdown	= ena_shutdown,
4275 	.driver.pm	= &ena_pm_ops,
4276 	.sriov_configure = pci_sriov_configure_simple,
4277 };
4278 
4279 static int __init ena_init(void)
4280 {
4281 	int ret;
4282 
4283 	ena_wq = create_singlethread_workqueue(DRV_MODULE_NAME);
4284 	if (!ena_wq) {
4285 		pr_err("Failed to create workqueue\n");
4286 		return -ENOMEM;
4287 	}
4288 
4289 	ret = pci_register_driver(&ena_pci_driver);
4290 	if (ret)
4291 		destroy_workqueue(ena_wq);
4292 
4293 	return ret;
4294 }
4295 
4296 static void __exit ena_cleanup(void)
4297 {
4298 	pci_unregister_driver(&ena_pci_driver);
4299 
4300 	if (ena_wq) {
4301 		destroy_workqueue(ena_wq);
4302 		ena_wq = NULL;
4303 	}
4304 }
4305 
4306 /******************************************************************************
4307  ******************************** AENQ Handlers *******************************
4308  *****************************************************************************/
4309 /* ena_update_on_link_change:
4310  * Notify the network interface about the change in link status
4311  */
4312 static void ena_update_on_link_change(void *adapter_data,
4313 				      struct ena_admin_aenq_entry *aenq_e)
4314 {
4315 	struct ena_adapter *adapter = (struct ena_adapter *)adapter_data;
4316 	struct ena_admin_aenq_link_change_desc *aenq_desc =
4317 		(struct ena_admin_aenq_link_change_desc *)aenq_e;
4318 	int status = aenq_desc->flags &
4319 		ENA_ADMIN_AENQ_LINK_CHANGE_DESC_LINK_STATUS_MASK;
4320 
4321 	if (status) {
4322 		netif_dbg(adapter, ifup, adapter->netdev, "%s\n", __func__);
4323 		set_bit(ENA_FLAG_LINK_UP, &adapter->flags);
4324 		if (!test_bit(ENA_FLAG_ONGOING_RESET, &adapter->flags))
4325 			netif_carrier_on(adapter->netdev);
4326 	} else {
4327 		clear_bit(ENA_FLAG_LINK_UP, &adapter->flags);
4328 		netif_carrier_off(adapter->netdev);
4329 	}
4330 }
4331 
4332 static void ena_keep_alive_wd(void *adapter_data,
4333 			      struct ena_admin_aenq_entry *aenq_e)
4334 {
4335 	struct ena_adapter *adapter = (struct ena_adapter *)adapter_data;
4336 	struct ena_admin_aenq_keep_alive_desc *desc;
4337 	u64 rx_drops;
4338 	u64 tx_drops;
4339 
4340 	desc = (struct ena_admin_aenq_keep_alive_desc *)aenq_e;
4341 	adapter->last_keep_alive_jiffies = jiffies;
4342 
4343 	rx_drops = ((u64)desc->rx_drops_high << 32) | desc->rx_drops_low;
4344 	tx_drops = ((u64)desc->tx_drops_high << 32) | desc->tx_drops_low;
4345 
4346 	u64_stats_update_begin(&adapter->syncp);
4347 	/* These stats are accumulated by the device, so the counters indicate
4348 	 * all drops since last reset.
4349 	 */
4350 	adapter->dev_stats.rx_drops = rx_drops;
4351 	adapter->dev_stats.tx_drops = tx_drops;
4352 	u64_stats_update_end(&adapter->syncp);
4353 }
4354 
4355 static void ena_notification(void *adapter_data,
4356 			     struct ena_admin_aenq_entry *aenq_e)
4357 {
4358 	struct ena_adapter *adapter = (struct ena_adapter *)adapter_data;
4359 	struct ena_admin_ena_hw_hints *hints;
4360 
4361 	WARN(aenq_e->aenq_common_desc.group != ENA_ADMIN_NOTIFICATION,
4362 	     "Invalid group(%x) expected %x\n",
4363 	     aenq_e->aenq_common_desc.group,
4364 	     ENA_ADMIN_NOTIFICATION);
4365 
4366 	switch (aenq_e->aenq_common_desc.syndrome) {
4367 	case ENA_ADMIN_UPDATE_HINTS:
4368 		hints = (struct ena_admin_ena_hw_hints *)
4369 			(&aenq_e->inline_data_w4);
4370 		ena_update_hints(adapter, hints);
4371 		break;
4372 	default:
4373 		netif_err(adapter, drv, adapter->netdev,
4374 			  "Invalid aenq notification link state %d\n",
4375 			  aenq_e->aenq_common_desc.syndrome);
4376 	}
4377 }
4378 
4379 /* This handler will called for unknown event group or unimplemented handlers*/
4380 static void unimplemented_aenq_handler(void *data,
4381 				       struct ena_admin_aenq_entry *aenq_e)
4382 {
4383 	struct ena_adapter *adapter = (struct ena_adapter *)data;
4384 
4385 	netif_err(adapter, drv, adapter->netdev,
4386 		  "Unknown event was received or event with unimplemented handler\n");
4387 }
4388 
4389 static struct ena_aenq_handlers aenq_handlers = {
4390 	.handlers = {
4391 		[ENA_ADMIN_LINK_CHANGE] = ena_update_on_link_change,
4392 		[ENA_ADMIN_NOTIFICATION] = ena_notification,
4393 		[ENA_ADMIN_KEEP_ALIVE] = ena_keep_alive_wd,
4394 	},
4395 	.unimplemented_handler = unimplemented_aenq_handler
4396 };
4397 
4398 module_init(ena_init);
4399 module_exit(ena_cleanup);
4400