xref: /freebsd/sys/contrib/ena-com/ena_plat.h (revision c66ec88fed842fbaad62c30d510644ceb7bd2d71)
1 /*-
2  * SPDX-License-Identifier: BSD-3-Clause
3  *
4  * Copyright (c) 2015-2020 Amazon.com, Inc. or its affiliates.
5  * All rights reserved.
6  *
7  * Redistribution and use in source and binary forms, with or without
8  * modification, are permitted provided that the following conditions
9  * are met:
10  *
11  * * Redistributions of source code must retain the above copyright
12  * notice, this list of conditions and the following disclaimer.
13  * * Redistributions in binary form must reproduce the above copyright
14  * notice, this list of conditions and the following disclaimer in
15  * the documentation and/or other materials provided with the
16  * distribution.
17  * * Neither the name of copyright holder nor the names of its
18  * contributors may be used to endorse or promote products derived
19  * from this software without specific prior written permission.
20  *
21  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
24  * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
25  * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
27  * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
28  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
29  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
30  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
31  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
32  */
33 
34 #ifndef ENA_PLAT_H_
35 #define ENA_PLAT_H_
36 
37 #include <sys/cdefs.h>
38 __FBSDID("$FreeBSD$");
39 
40 #include <sys/param.h>
41 #include <sys/systm.h>
42 
43 #include <sys/bus.h>
44 #include <sys/condvar.h>
45 #include <sys/endian.h>
46 #include <sys/kernel.h>
47 #include <sys/kthread.h>
48 #include <sys/malloc.h>
49 #include <sys/mbuf.h>
50 #include <sys/module.h>
51 #include <sys/rman.h>
52 #include <sys/proc.h>
53 #include <sys/smp.h>
54 #include <sys/socket.h>
55 #include <sys/sockio.h>
56 #include <sys/sysctl.h>
57 #include <sys/taskqueue.h>
58 #include <sys/eventhandler.h>
59 #include <sys/types.h>
60 #include <sys/timetc.h>
61 #include <sys/cdefs.h>
62 
63 #include <machine/atomic.h>
64 #include <machine/bus.h>
65 #include <machine/in_cksum.h>
66 #include <machine/pcpu.h>
67 #include <machine/resource.h>
68 #include <machine/_inttypes.h>
69 
70 #include <net/bpf.h>
71 #include <net/ethernet.h>
72 #include <net/if.h>
73 #include <net/if_var.h>
74 #include <net/if_arp.h>
75 #include <net/if_dl.h>
76 #include <net/if_media.h>
77 
78 #include <net/if_types.h>
79 #include <net/if_vlan_var.h>
80 
81 #include <netinet/in_systm.h>
82 #include <netinet/in.h>
83 #include <netinet/if_ether.h>
84 #include <netinet/ip.h>
85 #include <netinet/ip6.h>
86 #include <netinet/tcp.h>
87 #include <netinet/tcp_lro.h>
88 #include <netinet/udp.h>
89 
90 #include <dev/led/led.h>
91 #include <dev/pci/pcivar.h>
92 #include <dev/pci/pcireg.h>
93 
94 extern struct ena_bus_space ebs;
95 
96 /* Levels */
97 #define ENA_ALERT 	(1 << 0) /* Alerts are providing more error info.     */
98 #define ENA_WARNING 	(1 << 1) /* Driver output is more error sensitive.    */
99 #define ENA_INFO 	(1 << 2) /* Provides additional driver info. 	      */
100 #define ENA_DBG 	(1 << 3) /* Driver output for debugging.	      */
101 /* Detailed info that will be printed with ENA_INFO or ENA_DEBUG flag. 	      */
102 #define ENA_TXPTH 	(1 << 4) /* Allows TX path tracing. 		      */
103 #define ENA_RXPTH 	(1 << 5) /* Allows RX path tracing.		      */
104 #define ENA_RSC 	(1 << 6) /* Goes with TXPTH or RXPTH, free/alloc res. */
105 #define ENA_IOQ 	(1 << 7) /* Detailed info about IO queues. 	      */
106 #define ENA_ADMQ	(1 << 8) /* Detailed info about admin queue. 	      */
107 #define ENA_NETMAP	(1 << 9) /* Detailed info about netmap. 	      */
108 
109 #define DEFAULT_ALLOC_ALIGNMENT	8
110 
111 extern int ena_log_level;
112 
113 #define container_of(ptr, type, member)					\
114 	({								\
115 		const __typeof(((type *)0)->member) *__p = (ptr);	\
116 		(type *)((uintptr_t)__p - offsetof(type, member));	\
117 	})
118 
119 #define ena_trace_raw(ctx, level, fmt, args...)			\
120 	do {							\
121 		((void)(ctx));					\
122 		if (((level) & ena_log_level) != (level))	\
123 			break;					\
124 		printf(fmt, ##args);				\
125 	} while (0)
126 
127 #define ena_trace(ctx, level, fmt, args...)			\
128 	ena_trace_raw(ctx, level, "%s() [TID:%d]: "		\
129 	    fmt, __func__, curthread->td_tid, ##args)
130 
131 
132 #define ena_trc_dbg(ctx, format, arg...)	\
133 	ena_trace(ctx, ENA_DBG, format, ##arg)
134 #define ena_trc_info(ctx, format, arg...)	\
135 	ena_trace(ctx, ENA_INFO, format, ##arg)
136 #define ena_trc_warn(ctx, format, arg...)	\
137 	ena_trace(ctx, ENA_WARNING, format, ##arg)
138 #define ena_trc_err(ctx, format, arg...)	\
139 	ena_trace(ctx, ENA_ALERT, format, ##arg)
140 
141 #define unlikely(x)	__predict_false(!!(x))
142 #define likely(x)  	__predict_true(!!(x))
143 
144 #define __iomem
145 #define ____cacheline_aligned __aligned(CACHE_LINE_SIZE)
146 
147 #define MAX_ERRNO 4095
148 #define IS_ERR_VALUE(x) unlikely((x) <= (unsigned long)MAX_ERRNO)
149 
150 #define ENA_WARN(cond, ctx, format, arg...)				\
151 	do {								\
152 		if (unlikely((cond))) {					\
153 			ena_trc_warn(ctx, format, ##arg);		\
154 		}							\
155 	} while (0)
156 
157 static inline long IS_ERR(const void *ptr)
158 {
159 	return IS_ERR_VALUE((unsigned long)ptr);
160 }
161 
162 static inline void *ERR_PTR(long error)
163 {
164 	return (void *)error;
165 }
166 
167 static inline long PTR_ERR(const void *ptr)
168 {
169 	return (long) ptr;
170 }
171 
172 #define GENMASK(h, l)	(((~0U) - (1U << (l)) + 1) & (~0U >> (32 - 1 - (h))))
173 #define GENMASK_ULL(h, l)	(((~0ULL) << (l)) & (~0ULL >> (64 - 1 - (h))))
174 #define BIT(x)			(1UL << (x))
175 
176 #define ENA_ABORT() 		BUG()
177 #define BUG() 			panic("ENA BUG")
178 
179 #define SZ_256			(256)
180 #define SZ_4K			(4096)
181 
182 #define	ENA_COM_OK		0
183 #define ENA_COM_FAULT		EFAULT
184 #define	ENA_COM_INVAL		EINVAL
185 #define ENA_COM_NO_MEM		ENOMEM
186 #define	ENA_COM_NO_SPACE	ENOSPC
187 #define ENA_COM_TRY_AGAIN	-1
188 #define	ENA_COM_UNSUPPORTED	EOPNOTSUPP
189 #define	ENA_COM_NO_DEVICE	ENODEV
190 #define	ENA_COM_PERMISSION	EPERM
191 #define ENA_COM_TIMER_EXPIRED	ETIMEDOUT
192 #define ENA_COM_EIO		EIO
193 
194 #define ENA_MSLEEP(x) 		pause_sbt("ena", SBT_1MS * (x), SBT_1MS, 0)
195 #define ENA_USLEEP(x) 		pause_sbt("ena", SBT_1US * (x), SBT_1US, 0)
196 #define ENA_UDELAY(x) 		DELAY(x)
197 #define ENA_GET_SYSTEM_TIMEOUT(timeout_us) \
198     ((long)cputick2usec(cpu_ticks()) + (timeout_us))
199 #define ENA_TIME_EXPIRE(timeout)  ((timeout) < cputick2usec(cpu_ticks()))
200 #define ENA_MIGHT_SLEEP()
201 
202 #define min_t(type, _x, _y) ((type)(_x) < (type)(_y) ? (type)(_x) : (type)(_y))
203 #define max_t(type, _x, _y) ((type)(_x) > (type)(_y) ? (type)(_x) : (type)(_y))
204 
205 #define ENA_MIN32(x,y) 	MIN(x, y)
206 #define ENA_MIN16(x,y)	MIN(x, y)
207 #define ENA_MIN8(x,y)	MIN(x, y)
208 
209 #define ENA_MAX32(x,y) 	MAX(x, y)
210 #define ENA_MAX16(x,y) 	MAX(x, y)
211 #define ENA_MAX8(x,y) 	MAX(x, y)
212 
213 /* Spinlock related methods */
214 #define ena_spinlock_t 	struct mtx
215 #define ENA_SPINLOCK_INIT(spinlock)				\
216 	mtx_init(&(spinlock), "ena_spin", NULL, MTX_SPIN)
217 #define ENA_SPINLOCK_DESTROY(spinlock)				\
218 	do {							\
219 		if (mtx_initialized(&(spinlock)))		\
220 		    mtx_destroy(&(spinlock));			\
221 	} while (0)
222 #define ENA_SPINLOCK_LOCK(spinlock, flags)			\
223 	do {							\
224 		(void)(flags);					\
225 		mtx_lock_spin(&(spinlock));			\
226 	} while (0)
227 #define ENA_SPINLOCK_UNLOCK(spinlock, flags)			\
228 	do {							\
229 		(void)(flags);					\
230 		mtx_unlock_spin(&(spinlock));			\
231 	} while (0)
232 
233 
234 /* Wait queue related methods */
235 #define ena_wait_event_t struct { struct cv wq; struct mtx mtx; }
236 #define ENA_WAIT_EVENT_INIT(waitqueue)					\
237 	do {								\
238 		cv_init(&((waitqueue).wq), "cv");			\
239 		mtx_init(&((waitqueue).mtx), "wq", NULL, MTX_DEF);	\
240 	} while (0)
241 #define ENA_WAIT_EVENTS_DESTROY(admin_queue)				\
242 	do {								\
243 		struct ena_comp_ctx *comp_ctx;				\
244 		int i;							\
245 		for (i = 0; i < admin_queue->q_depth; i++) {		\
246 			comp_ctx = get_comp_ctxt(admin_queue, i, false); \
247 			if (comp_ctx != NULL) {				\
248 				cv_destroy(&((comp_ctx->wait_event).wq)); \
249 				mtx_destroy(&((comp_ctx->wait_event).mtx)); \
250 			}						\
251 		}							\
252 	} while (0)
253 #define ENA_WAIT_EVENT_CLEAR(waitqueue)					\
254 	cv_init(&((waitqueue).wq), (waitqueue).wq.cv_description)
255 #define ENA_WAIT_EVENT_WAIT(waitqueue, timeout_us)			\
256 	do {								\
257 		mtx_lock(&((waitqueue).mtx));				\
258 		cv_timedwait(&((waitqueue).wq), &((waitqueue).mtx),	\
259 		    timeout_us * hz / 1000 / 1000 );			\
260 		mtx_unlock(&((waitqueue).mtx));				\
261 	} while (0)
262 #define ENA_WAIT_EVENT_SIGNAL(waitqueue)		\
263 	do {						\
264 		mtx_lock(&((waitqueue).mtx));		\
265 		cv_broadcast(&((waitqueue).wq));	\
266 		mtx_unlock(&((waitqueue).mtx));		\
267 	} while (0)
268 
269 #define dma_addr_t 	bus_addr_t
270 #define u8 		uint8_t
271 #define u16 		uint16_t
272 #define u32 		uint32_t
273 #define u64 		uint64_t
274 
275 typedef struct {
276 	bus_addr_t              paddr;
277 	caddr_t                 vaddr;
278         bus_dma_tag_t           tag;
279 	bus_dmamap_t            map;
280         bus_dma_segment_t       seg;
281 	int                     nseg;
282 } ena_mem_handle_t;
283 
284 struct ena_bus {
285 	bus_space_handle_t 	reg_bar_h;
286 	bus_space_tag_t 	reg_bar_t;
287 	bus_space_handle_t	mem_bar_h;
288 	bus_space_tag_t 	mem_bar_t;
289 };
290 
291 typedef uint32_t ena_atomic32_t;
292 
293 #define ENA_PRIu64 PRIu64
294 
295 typedef uint64_t ena_time_t;
296 typedef struct ifnet ena_netdev;
297 
298 void	ena_dmamap_callback(void *arg, bus_dma_segment_t *segs, int nseg,
299     int error);
300 int	ena_dma_alloc(device_t dmadev, bus_size_t size, ena_mem_handle_t *dma,
301     int mapflags, bus_size_t alignment);
302 
303 static inline uint32_t
304 ena_reg_read32(struct ena_bus *bus, bus_size_t offset)
305 {
306 	uint32_t v = bus_space_read_4(bus->reg_bar_t, bus->reg_bar_h, offset);
307 	rmb();
308 	return v;
309 }
310 
311 #define ENA_MEMCPY_TO_DEVICE_64(dst, src, size)				\
312 	do {								\
313 		int count, i;						\
314 		volatile uint64_t *to = (volatile uint64_t *)(dst);	\
315 		const uint64_t *from = (const uint64_t *)(src);		\
316 		count = (size) / 8;					\
317 									\
318 		for (i = 0; i < count; i++, from++, to++)		\
319 			*to = *from;					\
320 	} while (0)
321 
322 #define ENA_MEM_ALLOC(dmadev, size) malloc(size, M_DEVBUF, M_NOWAIT | M_ZERO)
323 #define ENA_MEM_ALLOC_NODE(dmadev, size, virt, node, dev_node) (virt = NULL)
324 #define ENA_MEM_FREE(dmadev, ptr, size)					\
325 	do { 								\
326 		(void)(size);						\
327 		free(ptr, M_DEVBUF);					\
328 	} while (0)
329 #define ENA_MEM_ALLOC_COHERENT_NODE_ALIGNED(dmadev, size, virt, phys,	\
330     handle, node, dev_node, alignment) 					\
331 	do {								\
332 		((virt) = NULL);					\
333 		(void)(dev_node);					\
334 	} while (0)
335 
336 #define ENA_MEM_ALLOC_COHERENT_NODE(dmadev, size, virt, phys, handle,	\
337     node, dev_node)							\
338 	ENA_MEM_ALLOC_COHERENT_NODE_ALIGNED(dmadev, size, virt,		\
339 	    phys, handle, node, dev_node, DEFAULT_ALLOC_ALIGNMENT)
340 
341 #define ENA_MEM_ALLOC_COHERENT_ALIGNED(dmadev, size, virt, phys, dma,	\
342     alignment)								\
343 	do {								\
344 		ena_dma_alloc((dmadev), (size), &(dma), 0, alignment);	\
345 		(virt) = (void *)(dma).vaddr;				\
346 		(phys) = (dma).paddr;					\
347 	} while (0)
348 
349 #define ENA_MEM_ALLOC_COHERENT(dmadev, size, virt, phys, dma)		\
350 	ENA_MEM_ALLOC_COHERENT_ALIGNED(dmadev, size, virt,		\
351 	    phys, dma, DEFAULT_ALLOC_ALIGNMENT)
352 
353 #define ENA_MEM_FREE_COHERENT(dmadev, size, virt, phys, dma)		\
354 	do {								\
355 		(void)size;						\
356 		bus_dmamap_unload((dma).tag, (dma).map);		\
357 		bus_dmamem_free((dma).tag, (virt), (dma).map);		\
358 		bus_dma_tag_destroy((dma).tag);				\
359 		(dma).tag = NULL;					\
360 		(virt) = NULL;						\
361 	} while (0)
362 
363 /* Register R/W methods */
364 #define ENA_REG_WRITE32(bus, value, offset)				\
365 	do {								\
366 		wmb();							\
367 		ENA_REG_WRITE32_RELAXED(bus, value, offset);		\
368 	} while (0)
369 
370 #define ENA_REG_WRITE32_RELAXED(bus, value, offset)			\
371 	bus_space_write_4(						\
372 			  ((struct ena_bus*)bus)->reg_bar_t,		\
373 			  ((struct ena_bus*)bus)->reg_bar_h,		\
374 			  (bus_size_t)(offset), (value))
375 
376 #define ENA_REG_READ32(bus, offset)					\
377 	ena_reg_read32((struct ena_bus*)(bus), (bus_size_t)(offset))
378 
379 #define ENA_DB_SYNC_WRITE(mem_handle) bus_dmamap_sync(			\
380 	(mem_handle)->tag, (mem_handle)->map, BUS_DMASYNC_PREWRITE)
381 #define ENA_DB_SYNC_PREREAD(mem_handle) bus_dmamap_sync(		\
382 	(mem_handle)->tag, (mem_handle)->map, BUS_DMASYNC_PREREAD)
383 #define ENA_DB_SYNC_POSTREAD(mem_handle) bus_dmamap_sync(		\
384 	(mem_handle)->tag, (mem_handle)->map, BUS_DMASYNC_POSTREAD)
385 #define ENA_DB_SYNC(mem_handle) ENA_DB_SYNC_WRITE(mem_handle)
386 
387 #define time_after(a,b)	((long)((unsigned long)(b) - (unsigned long)(a)) < 0)
388 
389 #define VLAN_HLEN 	sizeof(struct ether_vlan_header)
390 #define CSUM_OFFLOAD 	(CSUM_IP|CSUM_TCP|CSUM_UDP)
391 
392 #define prefetch(x)	(void)(x)
393 #define prefetchw(x)	(void)(x)
394 
395 /* DMA buffers access */
396 #define	dma_unmap_addr(p, name)			((p)->dma->name)
397 #define	dma_unmap_addr_set(p, name, v)		(((p)->dma->name) = (v))
398 #define	dma_unmap_len(p, name)			((p)->name)
399 #define	dma_unmap_len_set(p, name, v)		(((p)->name) = (v))
400 
401 #define memcpy_toio memcpy
402 
403 #define ATOMIC32_INC(I32_PTR)		atomic_add_int(I32_PTR, 1)
404 #define ATOMIC32_DEC(I32_PTR) 		atomic_add_int(I32_PTR, -1)
405 #define ATOMIC32_READ(I32_PTR) 		atomic_load_acq_int(I32_PTR)
406 #define ATOMIC32_SET(I32_PTR, VAL) 	atomic_store_rel_int(I32_PTR, VAL)
407 
408 #define	barrier() __asm__ __volatile__("": : :"memory")
409 #define dma_rmb() barrier()
410 #define mmiowb() barrier()
411 
412 #define	ACCESS_ONCE(x) (*(volatile __typeof(x) *)&(x))
413 #define READ_ONCE(x)  ({			\
414 			__typeof(x) __var;	\
415 			barrier();		\
416 			__var = ACCESS_ONCE(x);	\
417 			barrier();		\
418 			__var;			\
419 		})
420 #define READ_ONCE8(x) READ_ONCE(x)
421 #define READ_ONCE16(x) READ_ONCE(x)
422 #define READ_ONCE32(x) READ_ONCE(x)
423 
424 #define upper_32_bits(n) ((uint32_t)(((n) >> 16) >> 16))
425 #define lower_32_bits(n) ((uint32_t)(n))
426 
427 #define DIV_ROUND_UP(n, d) (((n) + (d) - 1) / (d))
428 
429 #define ENA_FFS(x) ffs(x)
430 
431 void	ena_rss_key_fill(void *key, size_t size);
432 
433 #define ENA_RSS_FILL_KEY(key, size) ena_rss_key_fill(key, size)
434 
435 #include "ena_defs/ena_includes.h"
436 
437 #endif /* ENA_PLAT_H_ */
438