xref: /freebsd/sys/dev/ena/ena.h (revision ef36b3f75658d201edb495068db5e1be49593de5)
1 /*-
2  * BSD LICENSE
3  *
4  * Copyright (c) 2015-2017 Amazon.com, Inc. or its affiliates.
5  * All rights reserved.
6  *
7  * Redistribution and use in source and binary forms, with or without
8  * modification, are permitted provided that the following conditions
9  * are met:
10  *
11  * 1. Redistributions of source code must retain the above copyright
12  *    notice, this list of conditions and the following disclaimer.
13  *
14  * 2. Redistributions in binary form must reproduce the above copyright
15  *    notice, this list of conditions and the following disclaimer in the
16  *    documentation and/or other materials provided with the distribution.
17  *
18  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21  * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22  * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24  * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29  *
30  * $FreeBSD$
31  *
32  */
33 
34 #ifndef ENA_H
35 #define ENA_H
36 
37 #include <sys/types.h>
38 
39 #include "ena-com/ena_com.h"
40 #include "ena-com/ena_eth_com.h"
41 
42 #define DRV_MODULE_VER_MAJOR	0
43 #define DRV_MODULE_VER_MINOR	7
44 #define DRV_MODULE_VER_SUBMINOR 0
45 
46 #define DRV_MODULE_NAME		"ena"
47 
48 #ifndef DRV_MODULE_VERSION
49 #define DRV_MODULE_VERSION				\
50 	__XSTRING(DRV_MODULE_VER_MAJOR) "."		\
51 	__XSTRING(DRV_MODULE_VER_MINOR) "."		\
52 	__XSTRING(DRV_MODULE_VER_SUBMINOR)
53 #endif
54 #define DEVICE_NAME	"Elastic Network Adapter (ENA)"
55 #define DEVICE_DESC	"ENA adapter"
56 
57 /* Calculate DMA mask - width for ena cannot exceed 48, so it is safe */
58 #define ENA_DMA_BIT_MASK(x)		((1ULL << (x)) - 1ULL)
59 
60 /* 1 for AENQ + ADMIN */
61 #define	ENA_MAX_MSIX_VEC(io_queues)	(1 + (io_queues))
62 
63 #define	ENA_REG_BAR			0
64 #define	ENA_MEM_BAR			2
65 
66 #define	ENA_BUS_DMA_SEGS		32
67 
68 #define	ENA_DEFAULT_RING_SIZE		1024
69 #define	ENA_DEFAULT_SMALL_PACKET_LEN	128
70 #define	ENA_DEFAULT_MAX_RX_BUFF_ALLOC_SIZE	1536
71 
72 #define	ENA_RX_REFILL_THRESH_DEVIDER	8
73 
74 #define	ENA_MAX_PUSH_PKT_SIZE		128
75 
76 #define	ENA_NAME_MAX_LEN		20
77 #define	ENA_IRQNAME_SIZE		40
78 
79 #define	ENA_PKT_MAX_BUFS		19
80 #define	ENA_STALL_TIMEOUT		100
81 
82 #define	ENA_RX_RSS_TABLE_LOG_SIZE	7
83 #define	ENA_RX_RSS_TABLE_SIZE		(1 << ENA_RX_RSS_TABLE_LOG_SIZE)
84 
85 #define	ENA_HASH_KEY_SIZE		40
86 
87 #define	ENA_DMA_BITS_MASK		40
88 #define	ENA_MAX_FRAME_LEN		10000
89 #define	ENA_MIN_FRAME_LEN		60
90 #define	ENA_RX_HASH_KEY_NUM		10
91 #define	ENA_RX_THASH_TABLE_SIZE		(1 << 8)
92 
93 #define ENA_TX_CLEANUP_TRESHOLD		128
94 
95 #define DB_THRESHOLD	64
96 
97 #define TX_COMMIT	32
98  /*
99  * TX budget for cleaning. It should be half of the RX budget to reduce amount
100  *  of TCP retransmissions.
101  */
102 #define TX_BUDGET	128
103 /* RX cleanup budget. -1 stands for infinity. */
104 #define RX_BUDGET	256
105 /*
106  * How many times we can repeat cleanup in the io irq handling routine if the
107  * RX or TX budget was depleted.
108  */
109 #define CLEAN_BUDGET	8
110 
111 #define RX_IRQ_INTERVAL 20
112 #define TX_IRQ_INTERVAL 50
113 
114 #define	ENA_MAX_MTU		9216
115 #define	ENA_TSO_MAXSIZE		65536
116 #define	ENA_TSO_NSEGS		ENA_PKT_MAX_BUFS
117 #define	ENA_RX_OFFSET		NET_SKB_PAD + NET_IP_ALIGN
118 
119 #define	ENA_MMIO_DISABLE_REG_READ	BIT(0)
120 
121 #define	ENA_TX_RING_IDX_NEXT(idx, ring_size) (((idx) + 1) & ((ring_size) - 1))
122 
123 #define	ENA_RX_RING_IDX_NEXT(idx, ring_size) (((idx) + 1) & ((ring_size) - 1))
124 #define	ENA_RX_RING_IDX_ADD(idx, n, ring_size)	\
125 	(((idx) + (n)) & ((ring_size) - 1))
126 
127 #define	ENA_IO_TXQ_IDX(q)		(2 * (q))
128 #define	ENA_IO_RXQ_IDX(q)		(2 * (q) + 1)
129 
130 #define	ENA_MGMNT_IRQ_IDX		0
131 #define	ENA_IO_IRQ_FIRST_IDX		1
132 #define	ENA_IO_IRQ_IDX(q)		(ENA_IO_IRQ_FIRST_IDX + (q))
133 
134 /*
135  * ENA device should send keep alive msg every 1 sec.
136  * We wait for 6 sec just to be on the safe side.
137  */
138 #define DEFAULT_KEEP_ALIVE_TO		(SBT_1S * 6)
139 
140 /* Time in jiffies before concluding the transmitter is hung. */
141 #define DEFAULT_TX_CMP_TO		(SBT_1S * 5)
142 
143 /* Number of queues to check for missing queues per timer tick */
144 #define DEFAULT_TX_MONITORED_QUEUES	(4)
145 
146 /* Max number of timeouted packets before device reset */
147 #define DEFAULT_TX_CMP_THRESHOLD	(128)
148 
149 /*
150  * Supported PCI vendor and devices IDs
151  */
152 #define	PCI_VENDOR_ID_AMAZON	0x1d0f
153 
154 #define	PCI_DEV_ID_ENA_PF	0x0ec2
155 #define	PCI_DEV_ID_ENA_LLQ_PF	0x1ec2
156 #define	PCI_DEV_ID_ENA_VF	0xec20
157 #define	PCI_DEV_ID_ENA_LLQ_VF	0xec21
158 
159 struct msix_entry {
160 	int entry;
161 	int vector;
162 };
163 
164 typedef struct _ena_vendor_info_t {
165 	unsigned int vendor_id;
166 	unsigned int device_id;
167 	unsigned int index;
168 } ena_vendor_info_t;
169 
170 struct ena_irq {
171 	/* Interrupt resources */
172 	struct resource *res;
173 	driver_intr_t *handler;
174 	void *data;
175 	void *cookie;
176 	unsigned int vector;
177 	bool requested;
178 	int cpu;
179 	char name[ENA_IRQNAME_SIZE];
180 };
181 
182 struct ena_que {
183 	struct ena_adapter *adapter;
184 	struct ena_ring *tx_ring;
185 	struct ena_ring *rx_ring;
186 	uint32_t id;
187 	int cpu;
188 };
189 
190 struct ena_tx_buffer {
191 	struct mbuf *mbuf;
192 	/* # of ena desc for this specific mbuf
193 	 * (includes data desc and metadata desc) */
194 	unsigned int tx_descs;
195 	/* # of buffers used by this mbuf */
196 	unsigned int num_of_bufs;
197 	bus_dmamap_t map;
198 
199 	/* Used to detect missing tx packets */
200 	struct bintime timestamp;
201 	bool print_once;
202 
203 	struct ena_com_buf bufs[ENA_PKT_MAX_BUFS];
204 } __aligned(CACHE_LINE_SIZE);
205 
206 struct ena_rx_buffer {
207 	struct mbuf *mbuf;
208 	bus_dmamap_t map;
209 	struct ena_com_buf ena_buf;
210 } __aligned(CACHE_LINE_SIZE);
211 
212 
213 struct ena_stats_tx {
214 	counter_u64_t cnt;
215 	counter_u64_t bytes;
216 	counter_u64_t queue_stop;
217 	counter_u64_t prepare_ctx_err;
218 	counter_u64_t queue_wakeup;
219 	counter_u64_t dma_mapping_err;
220 	/* Not counted */
221 	counter_u64_t unsupported_desc_num;
222 	/* Not counted */
223 	counter_u64_t napi_comp;
224 	/* Not counted */
225 	counter_u64_t tx_poll;
226 	counter_u64_t doorbells;
227 	counter_u64_t missing_tx_comp;
228 	counter_u64_t bad_req_id;
229 	counter_u64_t collapse;
230 	counter_u64_t collapse_err;
231 };
232 
233 struct ena_stats_rx {
234 	counter_u64_t cnt;
235 	counter_u64_t bytes;
236 	counter_u64_t refil_partial;
237 	counter_u64_t bad_csum;
238 	/* Not counted */
239 	counter_u64_t page_alloc_fail;
240 	counter_u64_t mbuf_alloc_fail;
241 	counter_u64_t dma_mapping_err;
242 	counter_u64_t bad_desc_num;
243 	/* Not counted */
244 	counter_u64_t small_copy_len_pkt;
245 };
246 
247 
248 struct ena_ring {
249 	/* Holds the empty requests for TX out of order completions */
250 	uint16_t *free_tx_ids;
251 	struct ena_com_dev *ena_dev;
252 	struct ena_adapter *adapter;
253 	struct ena_com_io_cq *ena_com_io_cq;
254 	struct ena_com_io_sq *ena_com_io_sq;
255 
256 	/* The maximum length the driver can push to the device (For LLQ) */
257 	enum ena_admin_placement_policy_type tx_mem_queue_type;
258 	uint16_t rx_small_copy_len;
259 	uint16_t qid;
260 	uint16_t mtu;
261 	uint8_t tx_max_header_size;
262 
263 	struct ena_com_rx_buf_info ena_bufs[ENA_PKT_MAX_BUFS];
264 	uint32_t  smoothed_interval;
265 	enum ena_intr_moder_level moder_tbl_idx;
266 
267 	struct ena_que *que;
268 	struct lro_ctrl lro;
269 
270 	uint16_t next_to_use;
271 	uint16_t next_to_clean;
272 
273 	union {
274 		struct ena_tx_buffer *tx_buffer_info; /* contex of tx packet */
275 		struct ena_rx_buffer *rx_buffer_info; /* contex of rx packet */
276 	};
277 	int ring_size; /* number of tx/rx_buffer_info's entries */
278 
279 	struct buf_ring *br; /* only for TX */
280 	struct mtx ring_mtx;
281 	char mtx_name[16];
282 	struct task enqueue_task;
283 	struct taskqueue *enqueue_tq;
284 	struct task cmpl_task;
285 	struct taskqueue *cmpl_tq;
286 
287 	union {
288 		struct ena_stats_tx tx_stats;
289 		struct ena_stats_rx rx_stats;
290 	};
291 
292 } __aligned(CACHE_LINE_SIZE);
293 
294 struct ena_stats_dev {
295 	/* Not counted */
296 	counter_u64_t tx_timeout;
297 	/* Not counted */
298 	counter_u64_t io_suspend;
299 	/* Not counted */
300 	counter_u64_t io_resume;
301 	/* Not counted */
302 	counter_u64_t wd_expired;
303 	counter_u64_t interface_up;
304 	counter_u64_t interface_down;
305 	/* Not counted */
306 	counter_u64_t admin_q_pause;
307 };
308 
309 struct ena_hw_stats {
310 	uint64_t rx_packets;
311 	uint64_t tx_packets;
312 
313 	uint64_t rx_bytes;
314 	uint64_t tx_bytes;
315 
316 	uint64_t rx_drops;
317 };
318 
319 /* Board specific private data structure */
320 struct ena_adapter {
321 	struct ena_com_dev *ena_dev;
322 
323 	/* OS defined structs */
324 	if_t ifp;
325 	device_t pdev;
326 	struct ifmedia	media;
327 
328 	/* OS resources */
329 	struct resource * memory;
330 	struct resource * registers;
331 
332 	struct mtx global_mtx;
333 	struct sx ioctl_sx;
334 
335 	/* MSI-X */
336 	uint32_t msix_enabled;
337 	struct msix_entry *msix_entries;
338 	int msix_vecs;
339 
340 	/* DMA tags used throughout the driver adapter for Tx and Rx */
341 	bus_dma_tag_t tx_buf_tag;
342 	bus_dma_tag_t rx_buf_tag;
343 	int dma_width;
344 	/*
345 	 * RX packets that shorter that this len will be copied to the skb
346 	 * header
347 	 */
348 	unsigned int small_copy_len;
349 
350 	uint16_t max_tx_sgl_size;
351 	uint16_t max_rx_sgl_size;
352 
353 	uint32_t tx_offload_cap;
354 
355 	/* Tx fast path data */
356 	int num_queues;
357 
358 	unsigned int tx_usecs, rx_usecs; /* Interrupt coalescing */
359 
360 	unsigned int tx_ring_size;
361 	unsigned int rx_ring_size;
362 
363 	/* RSS*/
364 	uint8_t	 rss_ind_tbl[ENA_RX_RSS_TABLE_SIZE];
365 	bool rss_support;
366 
367 	uint32_t msg_enable;
368 
369 	uint8_t mac_addr[ETHER_ADDR_LEN];
370 	/* mdio and phy*/
371 
372 	char name[ENA_NAME_MAX_LEN];
373 	bool link_status;
374 	bool trigger_reset;
375 	bool up;
376 	bool running;
377 
378 	uint32_t wol;
379 
380 	/* Queue will represent one TX and one RX ring */
381 	struct ena_que que[ENA_MAX_NUM_IO_QUEUES]
382 	    __aligned(CACHE_LINE_SIZE);
383 
384 	/* TX */
385 	struct ena_ring tx_ring[ENA_MAX_NUM_IO_QUEUES]
386 	    __aligned(CACHE_LINE_SIZE);
387 
388 	/* RX */
389 	struct ena_ring rx_ring[ENA_MAX_NUM_IO_QUEUES]
390 	    __aligned(CACHE_LINE_SIZE);
391 
392 	struct ena_irq irq_tbl[ENA_MAX_MSIX_VEC(ENA_MAX_NUM_IO_QUEUES)];
393 
394 	/* Timer service */
395 	struct callout timer_service;
396 	sbintime_t keep_alive_timestamp;
397 	uint32_t next_monitored_tx_qid;
398 	struct task reset_task;
399 	struct taskqueue *reset_tq;
400 	int wd_active;
401 	sbintime_t keep_alive_timeout;
402 	sbintime_t missing_tx_timeout;
403 	uint32_t missing_tx_max_queues;
404 	uint32_t missing_tx_threshold;
405 
406 	/* Task updating hw stats */
407 	struct task stats_task;
408 	struct taskqueue *stats_tq;
409 
410 	/* Statistics */
411 	struct ena_stats_dev dev_stats;
412 	struct ena_hw_stats hw_stats;
413 };
414 
415 
416 #define	ENA_DEV_LOCK			mtx_lock(&adapter->global_mtx)
417 #define	ENA_DEV_UNLOCK			mtx_unlock(&adapter->global_mtx)
418 
419 #define	ENA_RING_MTX_LOCK(_ring)		mtx_lock(&(_ring)->ring_mtx)
420 #define	ENA_RING_MTX_TRYLOCK(_ring)		mtx_trylock(&(_ring)->ring_mtx)
421 #define	ENA_RING_MTX_UNLOCK(_ring)		mtx_unlock(&(_ring)->ring_mtx)
422 
423 struct ena_dev *ena_efa_enadev_get(device_t pdev);
424 
425 int ena_register_adapter(struct ena_adapter *adapter);
426 void ena_unregister_adapter(struct ena_adapter *adapter);
427 
428 int ena_update_stats_counters(struct ena_adapter *adapter);
429 
430 static inline int ena_mbuf_count(struct mbuf *mbuf)
431 {
432 	int count = 1;
433 
434 	while ((mbuf = mbuf->m_next) != NULL)
435 		++count;
436 
437 	return count;
438 }
439 
440 #endif /* !(ENA_H) */
441