1 /*- 2 * BSD LICENSE 3 * 4 * Copyright (c) 2015-2017 Amazon.com, Inc. or its affiliates. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * * Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * * Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in 15 * the documentation and/or other materials provided with the 16 * distribution. 17 * * Neither the name of copyright holder nor the names of its 18 * contributors may be used to endorse or promote products derived 19 * from this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 */ 33 34 #ifndef ENA_PLAT_H_ 35 #define ENA_PLAT_H_ 36 37 #include <sys/cdefs.h> 38 __FBSDID("$FreeBSD$"); 39 40 #include <sys/param.h> 41 #include <sys/systm.h> 42 43 #include <sys/bus.h> 44 #include <sys/condvar.h> 45 #include <sys/endian.h> 46 #include <sys/kernel.h> 47 #include <sys/kthread.h> 48 #include <sys/malloc.h> 49 #include <sys/mbuf.h> 50 #include <sys/module.h> 51 #include <sys/rman.h> 52 #include <sys/proc.h> 53 #include <sys/smp.h> 54 #include <sys/socket.h> 55 #include <sys/sockio.h> 56 #include <sys/sysctl.h> 57 #include <sys/taskqueue.h> 58 #include <sys/eventhandler.h> 59 #include <sys/types.h> 60 #include <sys/timetc.h> 61 #include <sys/cdefs.h> 62 63 #include <machine/atomic.h> 64 #include <machine/bus.h> 65 #include <machine/in_cksum.h> 66 #include <machine/pcpu.h> 67 #include <machine/resource.h> 68 69 #include <net/bpf.h> 70 #include <net/ethernet.h> 71 #include <net/if.h> 72 #include <net/if_var.h> 73 #include <net/if_arp.h> 74 #include <net/if_dl.h> 75 #include <net/if_media.h> 76 77 #include <net/if_types.h> 78 #include <net/if_vlan_var.h> 79 80 #include <netinet/in_systm.h> 81 #include <netinet/in.h> 82 #include <netinet/if_ether.h> 83 #include <netinet/ip.h> 84 #include <netinet/ip6.h> 85 #include <netinet/tcp.h> 86 #include <netinet/tcp_lro.h> 87 #include <netinet/udp.h> 88 89 #include <dev/led/led.h> 90 #include <dev/pci/pcivar.h> 91 #include <dev/pci/pcireg.h> 92 93 extern struct ena_bus_space ebs; 94 95 /* Levels */ 96 #define ENA_ALERT (1 << 0) /* Alerts are providing more error info. */ 97 #define ENA_WARNING (1 << 1) /* Driver output is more error sensitive. */ 98 #define ENA_INFO (1 << 2) /* Provides additional driver info. */ 99 #define ENA_DBG (1 << 3) /* Driver output for debugging. */ 100 /* Detailed info that will be printed with ENA_INFO or ENA_DEBUG flag. */ 101 #define ENA_TXPTH (1 << 4) /* Allows TX path tracing. */ 102 #define ENA_RXPTH (1 << 5) /* Allows RX path tracing. */ 103 #define ENA_RSC (1 << 6) /* Goes with TXPTH or RXPTH, free/alloc res. */ 104 #define ENA_IOQ (1 << 7) /* Detailed info about IO queues. */ 105 #define ENA_ADMQ (1 << 8) /* Detailed info about admin queue. */ 106 107 #ifndef ENA_DEBUG_LEVEL 108 #define ENA_DEBUG_LEVEL (ENA_ALERT | ENA_WARNING) 109 #endif 110 111 #ifdef ENA_TRACE 112 #define ena_trace_raw(level, fmt, args...) \ 113 do { \ 114 if (((level) & ENA_DEBUG_LEVEL) != (level)) \ 115 break; \ 116 printf(fmt, ##args); \ 117 } while (0) 118 119 #define ena_trace(level, fmt, args...) \ 120 ena_trace_raw(level, "%s() [TID:%d]: " \ 121 fmt " \n", __func__, curthread->td_tid, ##args) 122 123 #else /* ENA_TRACE */ 124 #define ena_trace_raw(...) 125 #define ena_trace(...) 126 #endif /* ENA_TRACE */ 127 128 #define ena_trc_dbg(format, arg...) ena_trace(ENA_DBG, format, ##arg) 129 #define ena_trc_info(format, arg...) ena_trace(ENA_INFO, format, ##arg) 130 #define ena_trc_warn(format, arg...) ena_trace(ENA_WARNING, format, ##arg) 131 #define ena_trc_err(format, arg...) ena_trace(ENA_ALERT, format, ##arg) 132 133 #define unlikely(x) __predict_false(x) 134 #define likely(x) __predict_true(x) 135 136 #define __iomem 137 #define ____cacheline_aligned __aligned(CACHE_LINE_SIZE) 138 139 #define MAX_ERRNO 4095 140 #define IS_ERR_VALUE(x) unlikely((x) <= (unsigned long)MAX_ERRNO) 141 142 #define WARN_ON(condition) \ 143 do { \ 144 int __ret_warn_on = !!(condition); \ 145 if (unlikely(__ret_warn_on)) \ 146 printf("%s %s", __FUNCTION__, __FILE__); \ 147 unlikely(__ret_warn_on); \ 148 } while (0) 149 150 #define ENA_ASSERT(cond, format, arg...) \ 151 do { \ 152 if (unlikely(!(cond))) { \ 153 ena_trc_err( \ 154 "Assert failed on %s:%s:%d:" format, \ 155 __FILE__, __func__, __LINE__, ##arg); \ 156 WARN_ON(cond); \ 157 } \ 158 } while (0) 159 160 #define ENA_WARN(cond, format, arg...) \ 161 do { \ 162 if (unlikely((cond))) { \ 163 ena_trc_warn(format, ##arg); \ 164 } \ 165 } while (0) 166 167 static inline long IS_ERR(const void *ptr) 168 { 169 return IS_ERR_VALUE((unsigned long)ptr); 170 } 171 172 static inline void *ERR_PTR(long error) 173 { 174 return (void *)error; 175 } 176 177 static inline long PTR_ERR(const void *ptr) 178 { 179 return (long) ptr; 180 } 181 182 #define GENMASK(h, l) (((1U << ((h) - (l) + 1)) - 1) << (l)) 183 #define GENMASK_ULL(h, l) (((~0ULL) << (l)) & (~0ULL >> (64 - 1 - (h)))) 184 #define BIT(x) (1 << (x)) 185 186 #define ENA_ABORT() BUG() 187 #define BUG() panic("ENA BUG") 188 189 #define SZ_256 (256) 190 #define SZ_4K (4096) 191 192 #define ENA_COM_OK 0 193 #define ENA_COM_FAULT EFAULT 194 #define ENA_COM_INVAL EINVAL 195 #define ENA_COM_NO_MEM ENOMEM 196 #define ENA_COM_NO_SPACE ENOSPC 197 #define ENA_COM_TRY_AGAIN -1 198 #define ENA_COM_NO_DEVICE ENODEV 199 #define ENA_COM_PERMISSION EPERM 200 #define ENA_COM_TIMER_EXPIRED ETIMEDOUT 201 202 #define ENA_MSLEEP(x) pause_sbt("ena", SBT_1MS * (x), SBT_1MS, 0) 203 #define ENA_UDELAY(x) DELAY(x) 204 #define ENA_GET_SYSTEM_TIMEOUT(timeout_us) \ 205 ((long)cputick2usec(cpu_ticks()) + (timeout_us)) 206 #define ENA_TIME_EXPIRE(timeout) ((timeout) < (long)cputick2usec(cpu_ticks())) 207 #define ENA_MIGHT_SLEEP() 208 209 #define min_t(type, _x, _y) ((type)(_x) < (type)(_y) ? (type)(_x) : (type)(_y)) 210 #define max_t(type, _x, _y) ((type)(_x) > (type)(_y) ? (type)(_x) : (type)(_y)) 211 212 #define ENA_MIN32(x,y) MIN(x, y) 213 #define ENA_MIN16(x,y) MIN(x, y) 214 #define ENA_MIN8(x,y) MIN(x, y) 215 216 #define ENA_MAX32(x,y) MAX(x, y) 217 #define ENA_MAX16(x,y) MAX(x, y) 218 #define ENA_MAX8(x,y) MAX(x, y) 219 220 /* Spinlock related methods */ 221 #define ena_spinlock_t struct mtx 222 #define ENA_SPINLOCK_INIT(spinlock) \ 223 mtx_init(&(spinlock), "ena_spin", NULL, MTX_SPIN) 224 #define ENA_SPINLOCK_DESTROY(spinlock) \ 225 do { \ 226 if (mtx_initialized(&(spinlock))) \ 227 mtx_destroy(&(spinlock)); \ 228 } while (0) 229 #define ENA_SPINLOCK_LOCK(spinlock, flags) \ 230 do { \ 231 (void)(flags); \ 232 mtx_lock_spin(&(spinlock)); \ 233 } while (0) 234 #define ENA_SPINLOCK_UNLOCK(spinlock, flags) \ 235 do { \ 236 (void)(flags); \ 237 mtx_unlock_spin(&(spinlock)); \ 238 } while (0) 239 240 241 /* Wait queue related methods */ 242 #define ena_wait_event_t struct { struct cv wq; struct mtx mtx; } 243 #define ENA_WAIT_EVENT_INIT(waitqueue) \ 244 do { \ 245 cv_init(&((waitqueue).wq), "cv"); \ 246 mtx_init(&((waitqueue).mtx), "wq", NULL, MTX_DEF); \ 247 } while (0) 248 #define ENA_WAIT_EVENT_DESTROY(waitqueue) \ 249 do { \ 250 cv_destroy(&((waitqueue).wq)); \ 251 mtx_destroy(&((waitqueue).mtx)); \ 252 } while (0) 253 #define ENA_WAIT_EVENT_CLEAR(waitqueue) \ 254 cv_init(&((waitqueue).wq), (waitqueue).wq.cv_description) 255 #define ENA_WAIT_EVENT_WAIT(waitqueue, timeout_us) \ 256 do { \ 257 mtx_lock(&((waitqueue).mtx)); \ 258 cv_timedwait(&((waitqueue).wq), &((waitqueue).mtx), \ 259 timeout_us * hz / 1000 / 1000 ); \ 260 mtx_unlock(&((waitqueue).mtx)); \ 261 } while (0) 262 #define ENA_WAIT_EVENT_SIGNAL(waitqueue) cv_broadcast(&((waitqueue).wq)) 263 264 #define dma_addr_t bus_addr_t 265 #define u8 uint8_t 266 #define u16 uint16_t 267 #define u32 uint32_t 268 #define u64 uint64_t 269 270 typedef struct { 271 bus_addr_t paddr; 272 caddr_t vaddr; 273 bus_dma_tag_t tag; 274 bus_dmamap_t map; 275 bus_dma_segment_t seg; 276 int nseg; 277 } ena_mem_handle_t; 278 279 struct ena_bus { 280 bus_space_handle_t reg_bar_h; 281 bus_space_tag_t reg_bar_t; 282 bus_space_handle_t mem_bar_h; 283 bus_space_tag_t mem_bar_t; 284 }; 285 286 typedef uint32_t ena_atomic32_t; 287 288 void ena_dmamap_callback(void *arg, bus_dma_segment_t *segs, int nseg, 289 int error); 290 int ena_dma_alloc(device_t dmadev, bus_size_t size, ena_mem_handle_t *dma, 291 int mapflags); 292 293 #define ENA_MEM_ALLOC(dmadev, size) malloc(size, M_DEVBUF, M_NOWAIT | M_ZERO) 294 #define ENA_MEM_ALLOC_NODE(dmadev, size, virt, node, dev_node) (virt = NULL) 295 #define ENA_MEM_FREE(dmadev, ptr) free(ptr, M_DEVBUF) 296 #define ENA_MEM_ALLOC_COHERENT_NODE(dmadev, size, virt, phys, handle, node, \ 297 dev_node) \ 298 do { \ 299 ((virt) = NULL); \ 300 (void)(dev_node); \ 301 } while (0) 302 303 #define ENA_MEM_ALLOC_COHERENT(dmadev, size, virt, phys, dma) \ 304 do { \ 305 ena_dma_alloc((dmadev), (size), &(dma), 0); \ 306 (virt) = (void *)(dma).vaddr; \ 307 (phys) = (dma).paddr; \ 308 } while (0) 309 310 #define ENA_MEM_FREE_COHERENT(dmadev, size, virt, phys, dma) \ 311 do { \ 312 (void)size; \ 313 bus_dmamap_unload((dma).tag, (dma).map); \ 314 bus_dmamem_free((dma).tag, (virt), (dma).map); \ 315 bus_dma_tag_destroy((dma).tag); \ 316 (dma).tag = NULL; \ 317 (virt) = NULL; \ 318 } while (0) 319 320 /* Register R/W methods */ 321 #define ENA_REG_WRITE32(bus, value, offset) \ 322 bus_space_write_4( \ 323 ((struct ena_bus*)bus)->reg_bar_t, \ 324 ((struct ena_bus*)bus)->reg_bar_h, \ 325 (bus_size_t)(offset), (value)) 326 327 #define ENA_REG_READ32(bus, offset) \ 328 bus_space_read_4( \ 329 ((struct ena_bus*)bus)->reg_bar_t, \ 330 ((struct ena_bus*)bus)->reg_bar_h, \ 331 (bus_size_t)(offset)) 332 333 #define time_after(a,b) ((long)((unsigned long)(b) - (unsigned long)(a)) < 0) 334 335 #define VLAN_HLEN sizeof(struct ether_vlan_header) 336 #define CSUM_OFFLOAD (CSUM_IP|CSUM_TCP|CSUM_UDP) 337 338 #if defined(__i386__) || defined(__amd64__) 339 static __inline 340 void prefetch(void *x) 341 { 342 __asm volatile("prefetcht0 %0" :: "m" (*(unsigned long *)x)); 343 } 344 #else 345 #define prefetch(x) 346 #endif 347 348 /* DMA buffers access */ 349 #define dma_unmap_addr(p, name) ((p)->dma->name) 350 #define dma_unmap_addr_set(p, name, v) (((p)->dma->name) = (v)) 351 #define dma_unmap_len(p, name) ((p)->name) 352 #define dma_unmap_len_set(p, name, v) (((p)->name) = (v)) 353 354 #define memcpy_toio memcpy 355 356 #define ATOMIC32_INC(I32_PTR) atomic_add_int(I32_PTR, 1) 357 #define ATOMIC32_DEC(I32_PTR) atomic_add_int(I32_PTR, -1) 358 #define ATOMIC32_READ(I32_PTR) atomic_load_acq_int(I32_PTR) 359 #define ATOMIC32_SET(I32_PTR, VAL) atomic_store_rel_int(I32_PTR, VAL) 360 361 #define barrier() __asm__ __volatile__("": : :"memory") 362 #define ACCESS_ONCE(x) (*(volatile __typeof(x) *)&(x)) 363 #define READ_ONCE(x) ({ \ 364 __typeof(x) __var; \ 365 barrier(); \ 366 __var = ACCESS_ONCE(x); \ 367 barrier(); \ 368 __var; \ 369 }) 370 371 #include "ena_common_defs.h" 372 #include "ena_admin_defs.h" 373 #include "ena_eth_io_defs.h" 374 #include "ena_regs_defs.h" 375 376 #endif /* ENA_PLAT_H_ */ 377