1 /*- 2 * SPDX-License-Identifier: BSD-3-Clause 3 * 4 * Copyright (c) 2015-2021 Amazon.com, Inc. or its affiliates. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * * Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * * Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in 15 * the documentation and/or other materials provided with the 16 * distribution. 17 * * Neither the name of copyright holder nor the names of its 18 * contributors may be used to endorse or promote products derived 19 * from this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 */ 33 34 #ifndef ENA_PLAT_H_ 35 #define ENA_PLAT_H_ 36 37 #include <sys/cdefs.h> 38 __FBSDID("$FreeBSD$"); 39 40 #include <sys/param.h> 41 #include <sys/systm.h> 42 43 #include <sys/bus.h> 44 #include <sys/condvar.h> 45 #include <sys/domainset.h> 46 #include <sys/endian.h> 47 #include <sys/kernel.h> 48 #include <sys/kthread.h> 49 #include <sys/malloc.h> 50 #include <sys/mbuf.h> 51 #include <sys/module.h> 52 #include <sys/rman.h> 53 #include <sys/proc.h> 54 #include <sys/smp.h> 55 #include <sys/socket.h> 56 #include <sys/sockio.h> 57 #include <sys/sysctl.h> 58 #include <sys/taskqueue.h> 59 #include <sys/eventhandler.h> 60 #include <sys/types.h> 61 #include <sys/timetc.h> 62 #include <sys/cdefs.h> 63 64 #include <machine/atomic.h> 65 #include <machine/bus.h> 66 #include <machine/in_cksum.h> 67 #include <machine/pcpu.h> 68 #include <machine/resource.h> 69 #include <machine/_inttypes.h> 70 71 #include <net/bpf.h> 72 #include <net/ethernet.h> 73 #include <net/if.h> 74 #include <net/if_var.h> 75 #include <net/if_arp.h> 76 #include <net/if_dl.h> 77 #include <net/if_media.h> 78 79 #include <net/if_types.h> 80 #include <net/if_vlan_var.h> 81 82 #include <netinet/in_systm.h> 83 #include <netinet/in.h> 84 #include <netinet/if_ether.h> 85 #include <netinet/ip.h> 86 #include <netinet/ip6.h> 87 #include <netinet/tcp.h> 88 #include <netinet/tcp_lro.h> 89 #include <netinet/udp.h> 90 91 #include <dev/led/led.h> 92 #include <dev/pci/pcivar.h> 93 #include <dev/pci/pcireg.h> 94 95 #include "ena_fbsd_log.h" 96 97 extern struct ena_bus_space ebs; 98 99 #define DEFAULT_ALLOC_ALIGNMENT 8 100 #define ENA_CDESC_RING_SIZE_ALIGNMENT (1 << 12) /* 4K */ 101 102 #define container_of(ptr, type, member) \ 103 ({ \ 104 const __typeof(((type *)0)->member) *__p = (ptr); \ 105 (type *)((uintptr_t)__p - offsetof(type, member)); \ 106 }) 107 108 #define ena_trace(ctx, level, fmt, args...) \ 109 ena_log((ctx)->dmadev, level, "%s() [TID:%d]: " \ 110 fmt, __func__, curthread->td_tid, ##args) 111 112 #define ena_trc_dbg(ctx, format, arg...) \ 113 ena_trace(ctx, DBG, format, ##arg) 114 #define ena_trc_info(ctx, format, arg...) \ 115 ena_trace(ctx, INFO, format, ##arg) 116 #define ena_trc_warn(ctx, format, arg...) \ 117 ena_trace(ctx, WARN, format, ##arg) 118 #define ena_trc_err(ctx, format, arg...) \ 119 ena_trace(ctx, ERR, format, ##arg) 120 121 #define unlikely(x) __predict_false(!!(x)) 122 #define likely(x) __predict_true(!!(x)) 123 124 #define __iomem 125 #define ____cacheline_aligned __aligned(CACHE_LINE_SIZE) 126 127 #define MAX_ERRNO 4095 128 #define IS_ERR_VALUE(x) unlikely((x) <= (unsigned long)MAX_ERRNO) 129 130 #define ENA_WARN(cond, ctx, format, arg...) \ 131 do { \ 132 if (unlikely((cond))) { \ 133 ena_trc_warn(ctx, format, ##arg); \ 134 } \ 135 } while (0) 136 137 static inline long IS_ERR(const void *ptr) 138 { 139 return IS_ERR_VALUE((unsigned long)ptr); 140 } 141 142 static inline void *ERR_PTR(long error) 143 { 144 return (void *)error; 145 } 146 147 static inline long PTR_ERR(const void *ptr) 148 { 149 return (long) ptr; 150 } 151 152 #define GENMASK(h, l) (((~0U) - (1U << (l)) + 1) & (~0U >> (32 - 1 - (h)))) 153 #define GENMASK_ULL(h, l) (((~0ULL) << (l)) & (~0ULL >> (64 - 1 - (h)))) 154 #define BIT(x) (1UL << (x)) 155 156 #define ENA_ABORT() BUG() 157 #define BUG() panic("ENA BUG") 158 159 #define SZ_256 (256) 160 #define SZ_4K (4096) 161 162 #define ENA_COM_OK 0 163 #define ENA_COM_FAULT EFAULT 164 #define ENA_COM_INVAL EINVAL 165 #define ENA_COM_NO_MEM ENOMEM 166 #define ENA_COM_NO_SPACE ENOSPC 167 #define ENA_COM_TRY_AGAIN -1 168 #define ENA_COM_UNSUPPORTED EOPNOTSUPP 169 #define ENA_COM_NO_DEVICE ENODEV 170 #define ENA_COM_PERMISSION EPERM 171 #define ENA_COM_TIMER_EXPIRED ETIMEDOUT 172 #define ENA_COM_EIO EIO 173 174 #define ENA_NODE_ANY (-1) 175 176 #define ENA_MSLEEP(x) pause_sbt("ena", SBT_1MS * (x), SBT_1MS, 0) 177 #define ENA_USLEEP(x) pause_sbt("ena", SBT_1US * (x), SBT_1US, 0) 178 #define ENA_UDELAY(x) DELAY(x) 179 #define ENA_GET_SYSTEM_TIMEOUT(timeout_us) \ 180 ((long)cputick2usec(cpu_ticks()) + (timeout_us)) 181 #define ENA_TIME_EXPIRE(timeout) ((timeout) < cputick2usec(cpu_ticks())) 182 #define ENA_MIGHT_SLEEP() 183 184 #define min_t(type, _x, _y) ((type)(_x) < (type)(_y) ? (type)(_x) : (type)(_y)) 185 #define max_t(type, _x, _y) ((type)(_x) > (type)(_y) ? (type)(_x) : (type)(_y)) 186 187 #define ENA_MIN32(x,y) MIN(x, y) 188 #define ENA_MIN16(x,y) MIN(x, y) 189 #define ENA_MIN8(x,y) MIN(x, y) 190 191 #define ENA_MAX32(x,y) MAX(x, y) 192 #define ENA_MAX16(x,y) MAX(x, y) 193 #define ENA_MAX8(x,y) MAX(x, y) 194 195 /* Spinlock related methods */ 196 #define ena_spinlock_t struct mtx 197 #define ENA_SPINLOCK_INIT(spinlock) \ 198 mtx_init(&(spinlock), "ena_spin", NULL, MTX_SPIN) 199 #define ENA_SPINLOCK_DESTROY(spinlock) \ 200 do { \ 201 if (mtx_initialized(&(spinlock))) \ 202 mtx_destroy(&(spinlock)); \ 203 } while (0) 204 #define ENA_SPINLOCK_LOCK(spinlock, flags) \ 205 do { \ 206 (void)(flags); \ 207 mtx_lock_spin(&(spinlock)); \ 208 } while (0) 209 #define ENA_SPINLOCK_UNLOCK(spinlock, flags) \ 210 do { \ 211 (void)(flags); \ 212 mtx_unlock_spin(&(spinlock)); \ 213 } while (0) 214 215 216 /* Wait queue related methods */ 217 #define ena_wait_event_t struct { struct cv wq; struct mtx mtx; } 218 #define ENA_WAIT_EVENT_INIT(waitqueue) \ 219 do { \ 220 cv_init(&((waitqueue).wq), "cv"); \ 221 mtx_init(&((waitqueue).mtx), "wq", NULL, MTX_DEF); \ 222 } while (0) 223 #define ENA_WAIT_EVENTS_DESTROY(admin_queue) \ 224 do { \ 225 struct ena_comp_ctx *comp_ctx; \ 226 int i; \ 227 for (i = 0; i < admin_queue->q_depth; i++) { \ 228 comp_ctx = get_comp_ctxt(admin_queue, i, false); \ 229 if (comp_ctx != NULL) { \ 230 cv_destroy(&((comp_ctx->wait_event).wq)); \ 231 mtx_destroy(&((comp_ctx->wait_event).mtx)); \ 232 } \ 233 } \ 234 } while (0) 235 #define ENA_WAIT_EVENT_CLEAR(waitqueue) \ 236 cv_init(&((waitqueue).wq), (waitqueue).wq.cv_description) 237 #define ENA_WAIT_EVENT_WAIT(waitqueue, timeout_us) \ 238 do { \ 239 mtx_lock(&((waitqueue).mtx)); \ 240 cv_timedwait(&((waitqueue).wq), &((waitqueue).mtx), \ 241 timeout_us * hz / 1000 / 1000 ); \ 242 mtx_unlock(&((waitqueue).mtx)); \ 243 } while (0) 244 #define ENA_WAIT_EVENT_SIGNAL(waitqueue) \ 245 do { \ 246 mtx_lock(&((waitqueue).mtx)); \ 247 cv_broadcast(&((waitqueue).wq)); \ 248 mtx_unlock(&((waitqueue).mtx)); \ 249 } while (0) 250 251 #define dma_addr_t bus_addr_t 252 #define u8 uint8_t 253 #define u16 uint16_t 254 #define u32 uint32_t 255 #define u64 uint64_t 256 257 typedef struct { 258 bus_addr_t paddr; 259 caddr_t vaddr; 260 bus_dma_tag_t tag; 261 bus_dmamap_t map; 262 bus_dma_segment_t seg; 263 int nseg; 264 } ena_mem_handle_t; 265 266 struct ena_bus { 267 bus_space_handle_t reg_bar_h; 268 bus_space_tag_t reg_bar_t; 269 bus_space_handle_t mem_bar_h; 270 bus_space_tag_t mem_bar_t; 271 }; 272 273 typedef uint32_t ena_atomic32_t; 274 275 #define ENA_PRIu64 PRIu64 276 277 typedef uint64_t ena_time_t; 278 typedef struct ifnet ena_netdev; 279 280 void ena_dmamap_callback(void *arg, bus_dma_segment_t *segs, int nseg, 281 int error); 282 int ena_dma_alloc(device_t dmadev, bus_size_t size, ena_mem_handle_t *dma, 283 int mapflags, bus_size_t alignment, int domain); 284 285 static inline uint32_t 286 ena_reg_read32(struct ena_bus *bus, bus_size_t offset) 287 { 288 uint32_t v = bus_space_read_4(bus->reg_bar_t, bus->reg_bar_h, offset); 289 rmb(); 290 return v; 291 } 292 293 #define ENA_MEMCPY_TO_DEVICE_64(dst, src, size) \ 294 do { \ 295 int count, i; \ 296 volatile uint64_t *to = (volatile uint64_t *)(dst); \ 297 const uint64_t *from = (const uint64_t *)(src); \ 298 count = (size) / 8; \ 299 \ 300 for (i = 0; i < count; i++, from++, to++) \ 301 *to = *from; \ 302 } while (0) 303 304 #define ENA_MEM_ALLOC(dmadev, size) malloc(size, M_DEVBUF, M_NOWAIT | M_ZERO) 305 306 #define ENA_MEM_ALLOC_NODE(dmadev, size, virt, node, dev_node) \ 307 do { \ 308 (virt) = malloc_domainset((size), M_DEVBUF, \ 309 (node) < 0 ? DOMAINSET_RR() : DOMAINSET_PREF(node), \ 310 M_NOWAIT | M_ZERO); \ 311 (void)(dev_node); \ 312 } while (0) 313 314 #define ENA_MEM_FREE(dmadev, ptr, size) \ 315 do { \ 316 (void)(size); \ 317 free(ptr, M_DEVBUF); \ 318 } while (0) 319 #define ENA_MEM_ALLOC_COHERENT_NODE_ALIGNED(dmadev, size, virt, phys, \ 320 dma, node, dev_node, alignment) \ 321 do { \ 322 ena_dma_alloc((dmadev), (size), &(dma), 0, (alignment), \ 323 (node)); \ 324 (virt) = (void *)(dma).vaddr; \ 325 (phys) = (dma).paddr; \ 326 (void)(dev_node); \ 327 } while (0) 328 329 #define ENA_MEM_ALLOC_COHERENT_NODE(dmadev, size, virt, phys, handle, \ 330 node, dev_node) \ 331 ENA_MEM_ALLOC_COHERENT_NODE_ALIGNED(dmadev, size, virt, \ 332 phys, handle, node, dev_node, DEFAULT_ALLOC_ALIGNMENT) 333 334 #define ENA_MEM_ALLOC_COHERENT_ALIGNED(dmadev, size, virt, phys, dma, \ 335 alignment) \ 336 do { \ 337 ena_dma_alloc((dmadev), (size), &(dma), 0, (alignment), \ 338 ENA_NODE_ANY); \ 339 (virt) = (void *)(dma).vaddr; \ 340 (phys) = (dma).paddr; \ 341 } while (0) 342 343 #define ENA_MEM_ALLOC_COHERENT(dmadev, size, virt, phys, dma) \ 344 ENA_MEM_ALLOC_COHERENT_ALIGNED(dmadev, size, virt, \ 345 phys, dma, DEFAULT_ALLOC_ALIGNMENT) 346 347 #define ENA_MEM_FREE_COHERENT(dmadev, size, virt, phys, dma) \ 348 do { \ 349 (void)size; \ 350 bus_dmamap_unload((dma).tag, (dma).map); \ 351 bus_dmamem_free((dma).tag, (virt), (dma).map); \ 352 bus_dma_tag_destroy((dma).tag); \ 353 (dma).tag = NULL; \ 354 (virt) = NULL; \ 355 } while (0) 356 357 /* Register R/W methods */ 358 #define ENA_REG_WRITE32(bus, value, offset) \ 359 do { \ 360 wmb(); \ 361 ENA_REG_WRITE32_RELAXED(bus, value, offset); \ 362 } while (0) 363 364 #define ENA_REG_WRITE32_RELAXED(bus, value, offset) \ 365 bus_space_write_4( \ 366 ((struct ena_bus*)bus)->reg_bar_t, \ 367 ((struct ena_bus*)bus)->reg_bar_h, \ 368 (bus_size_t)(offset), (value)) 369 370 #define ENA_REG_READ32(bus, offset) \ 371 ena_reg_read32((struct ena_bus*)(bus), (bus_size_t)(offset)) 372 373 #define ENA_DB_SYNC_WRITE(mem_handle) bus_dmamap_sync( \ 374 (mem_handle)->tag, (mem_handle)->map, BUS_DMASYNC_PREWRITE) 375 #define ENA_DB_SYNC_PREREAD(mem_handle) bus_dmamap_sync( \ 376 (mem_handle)->tag, (mem_handle)->map, BUS_DMASYNC_PREREAD) 377 #define ENA_DB_SYNC_POSTREAD(mem_handle) bus_dmamap_sync( \ 378 (mem_handle)->tag, (mem_handle)->map, BUS_DMASYNC_POSTREAD) 379 #define ENA_DB_SYNC(mem_handle) ENA_DB_SYNC_WRITE(mem_handle) 380 381 #define time_after(a,b) ((long)((unsigned long)(b) - (unsigned long)(a)) < 0) 382 383 #define VLAN_HLEN sizeof(struct ether_vlan_header) 384 385 #define prefetch(x) (void)(x) 386 #define prefetchw(x) (void)(x) 387 388 /* DMA buffers access */ 389 #define dma_unmap_addr(p, name) ((p)->dma->name) 390 #define dma_unmap_addr_set(p, name, v) (((p)->dma->name) = (v)) 391 #define dma_unmap_len(p, name) ((p)->name) 392 #define dma_unmap_len_set(p, name, v) (((p)->name) = (v)) 393 394 #define memcpy_toio memcpy 395 396 #define ATOMIC32_INC(I32_PTR) atomic_add_int(I32_PTR, 1) 397 #define ATOMIC32_DEC(I32_PTR) atomic_add_int(I32_PTR, -1) 398 #define ATOMIC32_READ(I32_PTR) atomic_load_acq_int(I32_PTR) 399 #define ATOMIC32_SET(I32_PTR, VAL) atomic_store_rel_int(I32_PTR, VAL) 400 401 #define barrier() __asm__ __volatile__("": : :"memory") 402 #define dma_rmb() barrier() 403 #define mmiowb() barrier() 404 405 #define ACCESS_ONCE(x) (*(volatile __typeof(x) *)&(x)) 406 #define READ_ONCE(x) ({ \ 407 __typeof(x) __var; \ 408 barrier(); \ 409 __var = ACCESS_ONCE(x); \ 410 barrier(); \ 411 __var; \ 412 }) 413 #define READ_ONCE8(x) READ_ONCE(x) 414 #define READ_ONCE16(x) READ_ONCE(x) 415 #define READ_ONCE32(x) READ_ONCE(x) 416 417 #define upper_32_bits(n) ((uint32_t)(((n) >> 16) >> 16)) 418 #define lower_32_bits(n) ((uint32_t)(n)) 419 420 #define DIV_ROUND_UP(n, d) (((n) + (d) - 1) / (d)) 421 422 #define ENA_FFS(x) ffs(x) 423 424 void ena_rss_key_fill(void *key, size_t size); 425 426 #define ENA_RSS_FILL_KEY(key, size) ena_rss_key_fill(key, size) 427 428 #include "ena_defs/ena_includes.h" 429 430 #endif /* ENA_PLAT_H_ */ 431