1 /* 2 * Copyright (C) 2011 Luigi Rizzo. All rights reserved. 3 * 4 * Redistribution and use in source and binary forms, with or without 5 * modification, are permitted provided that the following conditions 6 * are met: 7 * 1. Redistributions of source code must retain the above copyright 8 * notice, this list of conditions and the following disclaimer. 9 * 2. Redistributions in binary form must reproduce the above copyright 10 * notice, this list of conditions and the following disclaimer in the 11 * documentation and/or other materials provided with the distribution. 12 * 13 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 14 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 15 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 16 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 17 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 18 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 19 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 20 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 21 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 22 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 23 * SUCH DAMAGE. 24 */ 25 26 /* 27 * $FreeBSD$ 28 * $Id: if_re_netmap.h 10609 2012-02-22 19:44:58Z luigi $ 29 * 30 * netmap support for "re" 31 * For details on netmap support please see ixgbe_netmap.h 32 */ 33 34 35 #include <net/netmap.h> 36 #include <sys/selinfo.h> 37 #include <vm/vm.h> 38 #include <vm/pmap.h> /* vtophys ? */ 39 #include <dev/netmap/netmap_kern.h> 40 41 42 /* 43 * wrapper to export locks to the generic code 44 * We should not use the tx/rx locks 45 */ 46 static void 47 re_netmap_lock_wrapper(struct ifnet *ifp, int what, u_int queueid) 48 { 49 struct rl_softc *adapter = ifp->if_softc; 50 51 switch (what) { 52 case NETMAP_CORE_LOCK: 53 RL_LOCK(adapter); 54 break; 55 case NETMAP_CORE_UNLOCK: 56 RL_UNLOCK(adapter); 57 break; 58 59 case NETMAP_TX_LOCK: 60 case NETMAP_RX_LOCK: 61 case NETMAP_TX_UNLOCK: 62 case NETMAP_RX_UNLOCK: 63 D("invalid lock call %d, no tx/rx locks here", what); 64 break; 65 } 66 } 67 68 69 /* 70 * support for netmap register/unregisted. We are already under core lock. 71 * only called on the first register or the last unregister. 72 */ 73 static int 74 re_netmap_reg(struct ifnet *ifp, int onoff) 75 { 76 struct rl_softc *adapter = ifp->if_softc; 77 struct netmap_adapter *na = NA(ifp); 78 int error = 0; 79 80 if (na == NULL) 81 return EINVAL; 82 /* Tell the stack that the interface is no longer active */ 83 ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE); 84 85 re_stop(adapter); 86 87 if (onoff) { 88 ifp->if_capenable |= IFCAP_NETMAP; 89 90 /* save if_transmit to restore it later */ 91 na->if_transmit = ifp->if_transmit; 92 ifp->if_transmit = netmap_start; 93 94 re_init_locked(adapter); 95 96 if ((ifp->if_drv_flags & (IFF_DRV_RUNNING | IFF_DRV_OACTIVE)) == 0) { 97 error = ENOMEM; 98 goto fail; 99 } 100 } else { 101 fail: 102 /* restore if_transmit */ 103 ifp->if_transmit = na->if_transmit; 104 ifp->if_capenable &= ~IFCAP_NETMAP; 105 re_init_locked(adapter); /* also enables intr */ 106 } 107 return (error); 108 } 109 110 111 /* 112 * Reconcile kernel and user view of the transmit ring. 113 */ 114 static int 115 re_netmap_txsync(struct ifnet *ifp, u_int ring_nr, int do_lock) 116 { 117 struct rl_softc *sc = ifp->if_softc; 118 struct rl_txdesc *txd = sc->rl_ldata.rl_tx_desc; 119 struct netmap_adapter *na = NA(sc->rl_ifp); 120 struct netmap_kring *kring = &na->tx_rings[ring_nr]; 121 struct netmap_ring *ring = kring->ring; 122 int j, k, l, n, lim = kring->nkr_num_slots - 1; 123 124 k = ring->cur; 125 if (k > lim) 126 return netmap_ring_reinit(kring); 127 128 if (do_lock) 129 RL_LOCK(sc); 130 131 /* Sync the TX descriptor list */ 132 bus_dmamap_sync(sc->rl_ldata.rl_tx_list_tag, 133 sc->rl_ldata.rl_tx_list_map, 134 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 135 136 /* XXX move after the transmissions */ 137 /* record completed transmissions */ 138 for (n = 0, l = sc->rl_ldata.rl_tx_considx; 139 l != sc->rl_ldata.rl_tx_prodidx; 140 n++, l = RL_TX_DESC_NXT(sc, l)) { 141 uint32_t cmdstat = 142 le32toh(sc->rl_ldata.rl_tx_list[l].rl_cmdstat); 143 if (cmdstat & RL_TDESC_STAT_OWN) 144 break; 145 } 146 if (n > 0) { 147 sc->rl_ldata.rl_tx_considx = l; 148 sc->rl_ldata.rl_tx_free += n; 149 kring->nr_hwavail += n; 150 } 151 152 /* update avail to what the kernel knows */ 153 ring->avail = kring->nr_hwavail; 154 155 j = kring->nr_hwcur; 156 if (j != k) { /* we have new packets to send */ 157 l = sc->rl_ldata.rl_tx_prodidx; 158 for (n = 0; j != k; n++) { 159 struct netmap_slot *slot = &ring->slot[j]; 160 struct rl_desc *desc = &sc->rl_ldata.rl_tx_list[l]; 161 int cmd = slot->len | RL_TDESC_CMD_EOF | 162 RL_TDESC_CMD_OWN | RL_TDESC_CMD_SOF ; 163 uint64_t paddr; 164 void *addr = PNMB(slot, &paddr); 165 int len = slot->len; 166 167 if (addr == netmap_buffer_base || len > NETMAP_BUF_SIZE) { 168 if (do_lock) 169 RL_UNLOCK(sc); 170 // XXX what about prodidx ? 171 return netmap_ring_reinit(kring); 172 } 173 174 if (l == lim) /* mark end of ring */ 175 cmd |= RL_TDESC_CMD_EOR; 176 177 if (slot->flags & NS_BUF_CHANGED) { 178 desc->rl_bufaddr_lo = htole32(RL_ADDR_LO(paddr)); 179 desc->rl_bufaddr_hi = htole32(RL_ADDR_HI(paddr)); 180 /* buffer has changed, unload and reload map */ 181 netmap_reload_map(sc->rl_ldata.rl_tx_mtag, 182 txd[l].tx_dmamap, addr); 183 slot->flags &= ~NS_BUF_CHANGED; 184 } 185 slot->flags &= ~NS_REPORT; 186 desc->rl_cmdstat = htole32(cmd); 187 bus_dmamap_sync(sc->rl_ldata.rl_tx_mtag, 188 txd[l].tx_dmamap, BUS_DMASYNC_PREWRITE); 189 j = (j == lim) ? 0 : j + 1; 190 l = (l == lim) ? 0 : l + 1; 191 } 192 sc->rl_ldata.rl_tx_prodidx = l; 193 kring->nr_hwcur = k; /* the saved ring->cur */ 194 ring->avail -= n; // XXX see others 195 kring->nr_hwavail = ring->avail; 196 197 bus_dmamap_sync(sc->rl_ldata.rl_tx_list_tag, 198 sc->rl_ldata.rl_tx_list_map, 199 BUS_DMASYNC_PREWRITE|BUS_DMASYNC_PREREAD); 200 201 /* start ? */ 202 CSR_WRITE_1(sc, sc->rl_txstart, RL_TXSTART_START); 203 } 204 if (do_lock) 205 RL_UNLOCK(sc); 206 return 0; 207 } 208 209 210 /* 211 * Reconcile kernel and user view of the receive ring. 212 */ 213 static int 214 re_netmap_rxsync(struct ifnet *ifp, u_int ring_nr, int do_lock) 215 { 216 struct rl_softc *sc = ifp->if_softc; 217 struct rl_rxdesc *rxd = sc->rl_ldata.rl_rx_desc; 218 struct netmap_adapter *na = NA(sc->rl_ifp); 219 struct netmap_kring *kring = &na->rx_rings[ring_nr]; 220 struct netmap_ring *ring = kring->ring; 221 int j, l, n, lim = kring->nkr_num_slots - 1; 222 int force_update = do_lock || kring->nr_kflags & NKR_PENDINTR; 223 u_int k = ring->cur, resvd = ring->reserved; 224 225 k = ring->cur; 226 if (k > lim) 227 return netmap_ring_reinit(kring); 228 229 if (do_lock) 230 RL_LOCK(sc); 231 /* XXX check sync modes */ 232 bus_dmamap_sync(sc->rl_ldata.rl_rx_list_tag, 233 sc->rl_ldata.rl_rx_list_map, 234 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 235 236 /* 237 * Import newly received packets into the netmap ring. 238 * j is an index in the netmap ring, l in the NIC ring. 239 * 240 * The device uses all the buffers in the ring, so we need 241 * another termination condition in addition to RL_RDESC_STAT_OWN 242 * cleared (all buffers could have it cleared. The easiest one 243 * is to limit the amount of data reported up to 'lim' 244 */ 245 l = sc->rl_ldata.rl_rx_prodidx; /* next pkt to check */ 246 j = netmap_idx_n2k(kring, l); /* the kring index */ 247 if (netmap_no_pendintr || force_update) { 248 for (n = kring->nr_hwavail; n < lim ; n++) { 249 struct rl_desc *cur_rx = &sc->rl_ldata.rl_rx_list[l]; 250 uint32_t rxstat = le32toh(cur_rx->rl_cmdstat); 251 uint32_t total_len; 252 253 if ((rxstat & RL_RDESC_STAT_OWN) != 0) 254 break; 255 total_len = rxstat & sc->rl_rxlenmask; 256 /* XXX subtract crc */ 257 total_len = (total_len < 4) ? 0 : total_len - 4; 258 kring->ring->slot[j].len = total_len; 259 /* sync was in re_newbuf() */ 260 bus_dmamap_sync(sc->rl_ldata.rl_rx_mtag, 261 rxd[l].rx_dmamap, BUS_DMASYNC_POSTREAD); 262 j = (j == lim) ? 0 : j + 1; 263 l = (l == lim) ? 0 : l + 1; 264 } 265 if (n != kring->nr_hwavail) { 266 sc->rl_ldata.rl_rx_prodidx = l; 267 sc->rl_ifp->if_ipackets += n - kring->nr_hwavail; 268 kring->nr_hwavail = n; 269 } 270 kring->nr_kflags &= ~NKR_PENDINTR; 271 } 272 273 /* skip past packets that userspace has released */ 274 j = kring->nr_hwcur; 275 if (resvd > 0) { 276 if (resvd + ring->avail >= lim + 1) { 277 D("XXX invalid reserve/avail %d %d", resvd, ring->avail); 278 ring->reserved = resvd = 0; // XXX panic... 279 } 280 k = (k >= resvd) ? k - resvd : k + lim + 1 - resvd; 281 } 282 if (j != k) { /* userspace has released some packets. */ 283 l = netmap_idx_k2n(kring, j); /* the NIC index */ 284 for (n = 0; j != k; n++) { 285 struct netmap_slot *slot = ring->slot + j; 286 struct rl_desc *desc = &sc->rl_ldata.rl_rx_list[l]; 287 int cmd = NETMAP_BUF_SIZE | RL_RDESC_CMD_OWN; 288 uint64_t paddr; 289 void *addr = PNMB(slot, &paddr); 290 291 if (addr == netmap_buffer_base) { /* bad buf */ 292 if (do_lock) 293 RL_UNLOCK(sc); 294 return netmap_ring_reinit(kring); 295 } 296 297 if (l == lim) /* mark end of ring */ 298 cmd |= RL_RDESC_CMD_EOR; 299 300 slot->flags &= ~NS_REPORT; 301 if (slot->flags & NS_BUF_CHANGED) { 302 netmap_reload_map(sc->rl_ldata.rl_rx_mtag, 303 rxd[l].rx_dmamap, addr); 304 desc->rl_bufaddr_lo = htole32(RL_ADDR_LO(paddr)); 305 desc->rl_bufaddr_hi = htole32(RL_ADDR_HI(paddr)); 306 slot->flags &= ~NS_BUF_CHANGED; 307 } 308 desc->rl_cmdstat = htole32(cmd); 309 bus_dmamap_sync(sc->rl_ldata.rl_rx_mtag, 310 rxd[l].rx_dmamap, BUS_DMASYNC_PREREAD); 311 j = (j == lim) ? 0 : j + 1; 312 l = (l == lim) ? 0 : l + 1; 313 } 314 kring->nr_hwavail -= n; 315 kring->nr_hwcur = k; 316 /* Flush the RX DMA ring */ 317 318 bus_dmamap_sync(sc->rl_ldata.rl_rx_list_tag, 319 sc->rl_ldata.rl_rx_list_map, 320 BUS_DMASYNC_PREWRITE|BUS_DMASYNC_PREREAD); 321 } 322 /* tell userspace that there are new packets */ 323 ring->avail = kring->nr_hwavail - resvd; 324 if (do_lock) 325 RL_UNLOCK(sc); 326 return 0; 327 } 328 329 /* 330 * Additional routines to init the tx and rx rings. 331 * In other drivers we do that inline in the main code. 332 */ 333 static void 334 re_netmap_tx_init(struct rl_softc *sc) 335 { 336 struct rl_txdesc *txd; 337 struct rl_desc *desc; 338 int i, n; 339 struct netmap_adapter *na = NA(sc->rl_ifp); 340 struct netmap_slot *slot = netmap_reset(na, NR_TX, 0, 0); 341 342 /* slot is NULL if we are not in netmap mode */ 343 if (!slot) 344 return; 345 /* in netmap mode, overwrite addresses and maps */ 346 txd = sc->rl_ldata.rl_tx_desc; 347 desc = sc->rl_ldata.rl_tx_list; 348 n = sc->rl_ldata.rl_tx_desc_cnt; 349 350 /* l points in the netmap ring, i points in the NIC ring */ 351 for (i = 0; i < n; i++) { 352 uint64_t paddr; 353 int l = netmap_idx_n2k(&na->tx_rings[0], i); 354 void *addr = PNMB(slot + l, &paddr); 355 356 desc[i].rl_bufaddr_lo = htole32(RL_ADDR_LO(paddr)); 357 desc[i].rl_bufaddr_hi = htole32(RL_ADDR_HI(paddr)); 358 netmap_load_map(sc->rl_ldata.rl_tx_mtag, 359 txd[i].tx_dmamap, addr); 360 } 361 } 362 363 static void 364 re_netmap_rx_init(struct rl_softc *sc) 365 { 366 struct netmap_adapter *na = NA(sc->rl_ifp); 367 struct netmap_slot *slot = netmap_reset(na, NR_RX, 0, 0); 368 struct rl_desc *desc = sc->rl_ldata.rl_rx_list; 369 uint32_t cmdstat; 370 int i, n, max_avail; 371 372 if (!slot) 373 return; 374 n = sc->rl_ldata.rl_rx_desc_cnt; 375 /* 376 * Userspace owned hwavail packets before the reset, 377 * so the NIC that last hwavail descriptors of the ring 378 * are still owned by the driver (and keep one empty). 379 */ 380 max_avail = n - 1 - na->rx_rings[0].nr_hwavail; 381 for (i = 0; i < n; i++) { 382 void *addr; 383 uint64_t paddr; 384 int l = netmap_idx_n2k(&na->rx_rings[0], i); 385 386 addr = PNMB(slot + l, &paddr); 387 388 netmap_reload_map(sc->rl_ldata.rl_rx_mtag, 389 sc->rl_ldata.rl_rx_desc[i].rx_dmamap, addr); 390 bus_dmamap_sync(sc->rl_ldata.rl_rx_mtag, 391 sc->rl_ldata.rl_rx_desc[i].rx_dmamap, BUS_DMASYNC_PREREAD); 392 desc[i].rl_bufaddr_lo = htole32(RL_ADDR_LO(paddr)); 393 desc[i].rl_bufaddr_hi = htole32(RL_ADDR_HI(paddr)); 394 cmdstat = NETMAP_BUF_SIZE; 395 if (i == n - 1) /* mark the end of ring */ 396 cmdstat |= RL_RDESC_CMD_EOR; 397 if (i < max_avail) 398 cmdstat |= RL_RDESC_CMD_OWN; 399 desc[i].rl_cmdstat = htole32(cmdstat); 400 } 401 } 402 403 404 static void 405 re_netmap_attach(struct rl_softc *sc) 406 { 407 struct netmap_adapter na; 408 409 bzero(&na, sizeof(na)); 410 411 na.ifp = sc->rl_ifp; 412 na.separate_locks = 0; 413 na.num_tx_desc = sc->rl_ldata.rl_tx_desc_cnt; 414 na.num_rx_desc = sc->rl_ldata.rl_rx_desc_cnt; 415 na.nm_txsync = re_netmap_txsync; 416 na.nm_rxsync = re_netmap_rxsync; 417 na.nm_lock = re_netmap_lock_wrapper; 418 na.nm_register = re_netmap_reg; 419 netmap_attach(&na, 1); 420 } 421 /* end of file */ 422