1 /* 2 * Copyright (C) 2011-2012 Matteo Landi, Luigi Rizzo. All rights reserved. 3 * 4 * Redistribution and use in source and binary forms, with or without 5 * modification, are permitted provided that the following conditions 6 * are met: 7 * 1. Redistributions of source code must retain the above copyright 8 * notice, this list of conditions and the following disclaimer. 9 * 2. Redistributions in binary form must reproduce the above copyright 10 * notice, this list of conditions and the following disclaimer in the 11 * documentation and/or other materials provided with the distribution. 12 * 13 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 14 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 15 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 16 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 17 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 18 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 19 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 20 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 21 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 22 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 23 * SUCH DAMAGE. 24 */ 25 26 /* 27 * $FreeBSD$ 28 * $Id: netmap_kern.h 10602 2012-02-21 16:47:55Z luigi $ 29 * 30 * The header contains the definitions of constants and function 31 * prototypes used only in kernelspace. 32 */ 33 34 #ifndef _NET_NETMAP_KERN_H_ 35 #define _NET_NETMAP_KERN_H_ 36 37 #define NETMAP_MEM2 // use the new memory allocator 38 39 #if defined(__FreeBSD__) 40 #define NM_LOCK_T struct mtx 41 #define NM_SELINFO_T struct selinfo 42 #define MBUF_LEN(m) ((m)->m_pkthdr.len) 43 #define NM_SEND_UP(ifp, m) ((ifp)->if_input)(ifp, m) 44 #elif defined (linux) 45 #define NM_LOCK_T spinlock_t 46 #define NM_SELINFO_T wait_queue_head_t 47 #define MBUF_LEN(m) ((m)->len) 48 #define NM_SEND_UP(ifp, m) netif_rx(m) 49 #else 50 #error unsupported platform 51 #endif 52 53 #ifdef MALLOC_DECLARE 54 MALLOC_DECLARE(M_NETMAP); 55 #endif 56 57 #define ND(format, ...) 58 #define D(format, ...) \ 59 do { \ 60 struct timeval __xxts; \ 61 microtime(&__xxts); \ 62 printf("%03d.%06d %s [%d] " format "\n", \ 63 (int)__xxts.tv_sec % 1000, (int)__xxts.tv_usec, \ 64 __FUNCTION__, __LINE__, ##__VA_ARGS__); \ 65 } while (0) 66 67 struct netmap_adapter; 68 69 /* 70 * private, kernel view of a ring. Keeps track of the status of 71 * a ring across system calls. 72 * 73 * nr_hwcur index of the next buffer to refill. 74 * It corresponds to ring->cur - ring->reserved 75 * 76 * nr_hwavail the number of slots "owned" by userspace. 77 * nr_hwavail =:= ring->avail + ring->reserved 78 * 79 * The indexes in the NIC and netmap rings are offset by nkr_hwofs slots. 80 * This is so that, on a reset, buffers owned by userspace are not 81 * modified by the kernel. In particular: 82 * RX rings: the next empty buffer (hwcur + hwavail + hwofs) coincides with 83 * the next empty buffer as known by the hardware (next_to_check or so). 84 * TX rings: hwcur + hwofs coincides with next_to_send 85 */ 86 struct netmap_kring { 87 struct netmap_ring *ring; 88 u_int nr_hwcur; 89 int nr_hwavail; 90 u_int nr_kflags; /* private driver flags */ 91 #define NKR_PENDINTR 0x1 // Pending interrupt. 92 u_int nkr_num_slots; 93 94 int nkr_hwofs; /* offset between NIC and netmap ring */ 95 struct netmap_adapter *na; 96 NM_SELINFO_T si; /* poll/select wait queue */ 97 NM_LOCK_T q_lock; /* used if no device lock available */ 98 } __attribute__((__aligned__(64))); 99 100 /* 101 * This struct extends the 'struct adapter' (or 102 * equivalent) device descriptor. It contains all fields needed to 103 * support netmap operation. 104 */ 105 struct netmap_adapter { 106 int refcount; /* number of user-space descriptors using this 107 interface, which is equal to the number of 108 struct netmap_if objs in the mapped region. */ 109 /* 110 * The selwakeup in the interrupt thread can use per-ring 111 * and/or global wait queues. We track how many clients 112 * of each type we have so we can optimize the drivers, 113 * and especially avoid huge contention on the locks. 114 */ 115 int na_single; /* threads attached to a single hw queue */ 116 int na_multi; /* threads attached to multiple hw queues */ 117 118 int separate_locks; /* set if the interface suports different 119 locks for rx, tx and core. */ 120 121 u_int num_rx_rings; /* number of tx/rx ring pairs */ 122 u_int num_tx_rings; // if nonzero, overrides num_rx_rings 123 124 u_int num_tx_desc; /* number of descriptor in each queue */ 125 u_int num_rx_desc; 126 //u_int buff_size; // XXX deprecate, use NETMAP_BUF_SIZE 127 128 /* tx_rings and rx_rings are private but allocated 129 * as a contiguous chunk of memory. Each array has 130 * N+1 entries, for the adapter queues and for the host queue. 131 */ 132 struct netmap_kring *tx_rings; /* array of TX rings. */ 133 struct netmap_kring *rx_rings; /* array of RX rings. */ 134 135 NM_SELINFO_T tx_si, rx_si; /* global wait queues */ 136 137 /* copy of if_qflush and if_transmit pointers, to intercept 138 * packets from the network stack when netmap is active. 139 */ 140 int (*if_transmit)(struct ifnet *, struct mbuf *); 141 142 /* references to the ifnet and device routines, used by 143 * the generic netmap functions. 144 */ 145 struct ifnet *ifp; /* adapter is ifp->if_softc */ 146 147 NM_LOCK_T core_lock; /* used if no device lock available */ 148 149 int (*nm_register)(struct ifnet *, int onoff); 150 void (*nm_lock)(struct ifnet *, int what, u_int ringid); 151 int (*nm_txsync)(struct ifnet *, u_int ring, int lock); 152 int (*nm_rxsync)(struct ifnet *, u_int ring, int lock); 153 #ifdef linux 154 struct net_device_ops nm_ndo; 155 #endif /* linux */ 156 }; 157 158 /* 159 * The combination of "enable" (ifp->if_capabilities &IFCAP_NETMAP) 160 * and refcount gives the status of the interface, namely: 161 * 162 * enable refcount Status 163 * 164 * FALSE 0 normal operation 165 * FALSE != 0 -- (impossible) 166 * TRUE 1 netmap mode 167 * TRUE 0 being deleted. 168 */ 169 170 #define NETMAP_DELETING(_na) ( ((_na)->refcount == 0) && \ 171 ( (_na)->ifp->if_capenable & IFCAP_NETMAP) ) 172 173 /* 174 * parameters for (*nm_lock)(adapter, what, index) 175 */ 176 enum { 177 NETMAP_NO_LOCK = 0, 178 NETMAP_CORE_LOCK, NETMAP_CORE_UNLOCK, 179 NETMAP_TX_LOCK, NETMAP_TX_UNLOCK, 180 NETMAP_RX_LOCK, NETMAP_RX_UNLOCK, 181 #ifdef __FreeBSD__ 182 #define NETMAP_REG_LOCK NETMAP_CORE_LOCK 183 #define NETMAP_REG_UNLOCK NETMAP_CORE_UNLOCK 184 #else 185 NETMAP_REG_LOCK, NETMAP_REG_UNLOCK 186 #endif 187 }; 188 189 /* 190 * The following are support routines used by individual drivers to 191 * support netmap operation. 192 * 193 * netmap_attach() initializes a struct netmap_adapter, allocating the 194 * struct netmap_ring's and the struct selinfo. 195 * 196 * netmap_detach() frees the memory allocated by netmap_attach(). 197 * 198 * netmap_start() replaces the if_transmit routine of the interface, 199 * and is used to intercept packets coming from the stack. 200 * 201 * netmap_load_map/netmap_reload_map are helper routines to set/reset 202 * the dmamap for a packet buffer 203 * 204 * netmap_reset() is a helper routine to be called in the driver 205 * when reinitializing a ring. 206 */ 207 int netmap_attach(struct netmap_adapter *, int); 208 void netmap_detach(struct ifnet *); 209 int netmap_start(struct ifnet *, struct mbuf *); 210 enum txrx { NR_RX = 0, NR_TX = 1 }; 211 struct netmap_slot *netmap_reset(struct netmap_adapter *na, 212 enum txrx tx, int n, u_int new_cur); 213 int netmap_ring_reinit(struct netmap_kring *); 214 215 extern int netmap_buf_size; 216 #define NETMAP_BUF_SIZE netmap_buf_size 217 extern int netmap_mitigate; 218 extern int netmap_no_pendintr; 219 extern u_int netmap_total_buffers; 220 extern char *netmap_buffer_base; 221 extern int netmap_verbose; // XXX debugging 222 enum { /* verbose flags */ 223 NM_VERB_ON = 1, /* generic verbose */ 224 NM_VERB_HOST = 0x2, /* verbose host stack */ 225 NM_VERB_RXSYNC = 0x10, /* verbose on rxsync/txsync */ 226 NM_VERB_TXSYNC = 0x20, 227 NM_VERB_RXINTR = 0x100, /* verbose on rx/tx intr (driver) */ 228 NM_VERB_TXINTR = 0x200, 229 NM_VERB_NIC_RXSYNC = 0x1000, /* verbose on rx/tx intr (driver) */ 230 NM_VERB_NIC_TXSYNC = 0x2000, 231 }; 232 233 /* 234 * NA returns a pointer to the struct netmap adapter from the ifp, 235 * WNA is used to write it. 236 */ 237 #ifndef WNA 238 #define WNA(_ifp) (_ifp)->if_pspare[0] 239 #endif 240 #define NA(_ifp) ((struct netmap_adapter *)WNA(_ifp)) 241 242 243 /* Callback invoked by the dma machinery after a successfull dmamap_load */ 244 static void netmap_dmamap_cb(__unused void *arg, 245 __unused bus_dma_segment_t * segs, __unused int nseg, __unused int error) 246 { 247 } 248 249 /* bus_dmamap_load wrapper: call aforementioned function if map != NULL. 250 * XXX can we do it without a callback ? 251 */ 252 static inline void 253 netmap_load_map(bus_dma_tag_t tag, bus_dmamap_t map, void *buf) 254 { 255 if (map) 256 bus_dmamap_load(tag, map, buf, NETMAP_BUF_SIZE, 257 netmap_dmamap_cb, NULL, BUS_DMA_NOWAIT); 258 } 259 260 /* update the map when a buffer changes. */ 261 static inline void 262 netmap_reload_map(bus_dma_tag_t tag, bus_dmamap_t map, void *buf) 263 { 264 if (map) { 265 bus_dmamap_unload(tag, map); 266 bus_dmamap_load(tag, map, buf, NETMAP_BUF_SIZE, 267 netmap_dmamap_cb, NULL, BUS_DMA_NOWAIT); 268 } 269 } 270 271 /* 272 * functions to map NIC to KRING indexes (n2k) and vice versa (k2n) 273 */ 274 static inline int 275 netmap_idx_n2k(struct netmap_kring *kr, int idx) 276 { 277 int n = kr->nkr_num_slots; 278 idx += kr->nkr_hwofs; 279 if (idx < 0) 280 return idx + n; 281 else if (idx < n) 282 return idx; 283 else 284 return idx - n; 285 } 286 287 288 static inline int 289 netmap_idx_k2n(struct netmap_kring *kr, int idx) 290 { 291 int n = kr->nkr_num_slots; 292 idx -= kr->nkr_hwofs; 293 if (idx < 0) 294 return idx + n; 295 else if (idx < n) 296 return idx; 297 else 298 return idx - n; 299 } 300 301 302 #ifdef NETMAP_MEM2 303 /* Entries of the look-up table. */ 304 struct lut_entry { 305 void *vaddr; /* virtual address. */ 306 vm_paddr_t paddr; /* phisical address. */ 307 }; 308 309 struct netmap_obj_pool; 310 extern struct lut_entry *netmap_buffer_lut; 311 #define NMB_VA(i) (netmap_buffer_lut[i].vaddr) 312 #define NMB_PA(i) (netmap_buffer_lut[i].paddr) 313 #else /* NETMAP_MEM1 */ 314 #define NMB_VA(i) (netmap_buffer_base + (i * NETMAP_BUF_SIZE) ) 315 #endif /* NETMAP_MEM2 */ 316 317 /* 318 * NMB return the virtual address of a buffer (buffer 0 on bad index) 319 * PNMB also fills the physical address 320 */ 321 static inline void * 322 NMB(struct netmap_slot *slot) 323 { 324 uint32_t i = slot->buf_idx; 325 return (i >= netmap_total_buffers) ? NMB_VA(0) : NMB_VA(i); 326 } 327 328 static inline void * 329 PNMB(struct netmap_slot *slot, uint64_t *pp) 330 { 331 uint32_t i = slot->buf_idx; 332 void *ret = (i >= netmap_total_buffers) ? NMB_VA(0) : NMB_VA(i); 333 #ifdef NETMAP_MEM2 334 *pp = (i >= netmap_total_buffers) ? NMB_PA(0) : NMB_PA(i); 335 #else 336 *pp = vtophys(ret); 337 #endif 338 return ret; 339 } 340 341 /* default functions to handle rx/tx interrupts */ 342 int netmap_rx_irq(struct ifnet *, int, int *); 343 #define netmap_tx_irq(_n, _q) netmap_rx_irq(_n, _q, NULL) 344 #endif /* _NET_NETMAP_KERN_H_ */ 345