1 /* 2 * Copyright (C) 2011 Luigi Rizzo. All rights reserved. 3 * 4 * Redistribution and use in source and binary forms, with or without 5 * modification, are permitted provided that the following conditions 6 * are met: 7 * 1. Redistributions of source code must retain the above copyright 8 * notice, this list of conditions and the following disclaimer. 9 * 2. Redistributions in binary form must reproduce the above copyright 10 * notice, this list of conditions and the following disclaimer in the 11 * documentation and/or other materials provided with the distribution. 12 * 13 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 14 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 15 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 16 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 17 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 18 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 19 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 20 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 21 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 22 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 23 * SUCH DAMAGE. 24 */ 25 26 /* 27 * $FreeBSD$ 28 * $Id: if_re_netmap.h 10609 2012-02-22 19:44:58Z luigi $ 29 * 30 * netmap support for "re" 31 * For details on netmap support please see ixgbe_netmap.h 32 */ 33 34 35 #include <net/netmap.h> 36 #include <sys/selinfo.h> 37 #include <vm/vm.h> 38 #include <vm/pmap.h> /* vtophys ? */ 39 #include <dev/netmap/netmap_kern.h> 40 41 42 /* 43 * wrapper to export locks to the generic code 44 * We should not use the tx/rx locks 45 */ 46 static void 47 re_netmap_lock_wrapper(struct ifnet *ifp, int what, u_int queueid) 48 { 49 struct rl_softc *adapter = ifp->if_softc; 50 51 switch (what) { 52 case NETMAP_CORE_LOCK: 53 RL_LOCK(adapter); 54 break; 55 case NETMAP_CORE_UNLOCK: 56 RL_UNLOCK(adapter); 57 break; 58 59 case NETMAP_TX_LOCK: 60 case NETMAP_RX_LOCK: 61 case NETMAP_TX_UNLOCK: 62 case NETMAP_RX_UNLOCK: 63 D("invalid lock call %d, no tx/rx locks here", what); 64 break; 65 } 66 } 67 68 69 /* 70 * support for netmap register/unregisted. We are already under core lock. 71 * only called on the first register or the last unregister. 72 */ 73 static int 74 re_netmap_reg(struct ifnet *ifp, int onoff) 75 { 76 struct rl_softc *adapter = ifp->if_softc; 77 struct netmap_adapter *na = NA(ifp); 78 int error = 0; 79 80 if (na == NULL) 81 return EINVAL; 82 /* Tell the stack that the interface is no longer active */ 83 ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE); 84 85 re_stop(adapter); 86 87 if (onoff) { 88 ifp->if_capenable |= IFCAP_NETMAP; 89 90 /* save if_transmit to restore it later */ 91 na->if_transmit = ifp->if_transmit; 92 ifp->if_transmit = netmap_start; 93 94 re_init_locked(adapter); 95 96 if ((ifp->if_drv_flags & (IFF_DRV_RUNNING | IFF_DRV_OACTIVE)) == 0) { 97 error = ENOMEM; 98 goto fail; 99 } 100 } else { 101 fail: 102 /* restore if_transmit */ 103 ifp->if_transmit = na->if_transmit; 104 ifp->if_capenable &= ~IFCAP_NETMAP; 105 re_init_locked(adapter); /* also enables intr */ 106 } 107 return (error); 108 } 109 110 111 /* 112 * Reconcile kernel and user view of the transmit ring. 113 */ 114 static int 115 re_netmap_txsync(struct ifnet *ifp, u_int ring_nr, int do_lock) 116 { 117 struct rl_softc *sc = ifp->if_softc; 118 struct rl_txdesc *txd = sc->rl_ldata.rl_tx_desc; 119 struct netmap_adapter *na = NA(sc->rl_ifp); 120 struct netmap_kring *kring = &na->tx_rings[ring_nr]; 121 struct netmap_ring *ring = kring->ring; 122 int j, k, l, n, lim = kring->nkr_num_slots - 1; 123 124 k = ring->cur; 125 if (k > lim) 126 return netmap_ring_reinit(kring); 127 128 if (do_lock) 129 RL_LOCK(sc); 130 131 /* Sync the TX descriptor list */ 132 bus_dmamap_sync(sc->rl_ldata.rl_tx_list_tag, 133 sc->rl_ldata.rl_tx_list_map, 134 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 135 136 /* XXX move after the transmissions */ 137 /* record completed transmissions */ 138 for (n = 0, l = sc->rl_ldata.rl_tx_considx; 139 l != sc->rl_ldata.rl_tx_prodidx; 140 n++, l = RL_TX_DESC_NXT(sc, l)) { 141 uint32_t cmdstat = 142 le32toh(sc->rl_ldata.rl_tx_list[l].rl_cmdstat); 143 if (cmdstat & RL_TDESC_STAT_OWN) 144 break; 145 } 146 if (n > 0) { 147 sc->rl_ldata.rl_tx_considx = l; 148 sc->rl_ldata.rl_tx_free += n; 149 kring->nr_hwavail += n; 150 } 151 152 /* update avail to what the kernel knows */ 153 ring->avail = kring->nr_hwavail; 154 155 j = kring->nr_hwcur; 156 if (j != k) { /* we have new packets to send */ 157 l = sc->rl_ldata.rl_tx_prodidx; 158 for (n = 0; j != k; n++) { 159 struct netmap_slot *slot = &ring->slot[j]; 160 struct rl_desc *desc = &sc->rl_ldata.rl_tx_list[l]; 161 int cmd = slot->len | RL_TDESC_CMD_EOF | 162 RL_TDESC_CMD_OWN | RL_TDESC_CMD_SOF ; 163 uint64_t paddr; 164 void *addr = PNMB(slot, &paddr); 165 int len = slot->len; 166 167 if (addr == netmap_buffer_base || len > NETMAP_BUF_SIZE) { 168 if (do_lock) 169 RL_UNLOCK(sc); 170 // XXX what about prodidx ? 171 return netmap_ring_reinit(kring); 172 } 173 174 if (l == lim) /* mark end of ring */ 175 cmd |= RL_TDESC_CMD_EOR; 176 177 if (slot->flags & NS_BUF_CHANGED) { 178 desc->rl_bufaddr_lo = htole32(RL_ADDR_LO(paddr)); 179 desc->rl_bufaddr_hi = htole32(RL_ADDR_HI(paddr)); 180 /* buffer has changed, unload and reload map */ 181 netmap_reload_map(sc->rl_ldata.rl_tx_mtag, 182 txd[l].tx_dmamap, addr); 183 slot->flags &= ~NS_BUF_CHANGED; 184 } 185 slot->flags &= ~NS_REPORT; 186 desc->rl_cmdstat = htole32(cmd); 187 bus_dmamap_sync(sc->rl_ldata.rl_tx_mtag, 188 txd[l].tx_dmamap, BUS_DMASYNC_PREWRITE); 189 j = (j == lim) ? 0 : j + 1; 190 l = (l == lim) ? 0 : l + 1; 191 } 192 sc->rl_ldata.rl_tx_prodidx = l; 193 kring->nr_hwcur = k; /* the saved ring->cur */ 194 ring->avail -= n; // XXX see others 195 kring->nr_hwavail = ring->avail; 196 197 bus_dmamap_sync(sc->rl_ldata.rl_tx_list_tag, 198 sc->rl_ldata.rl_tx_list_map, 199 BUS_DMASYNC_PREWRITE|BUS_DMASYNC_PREREAD); 200 201 /* start ? */ 202 CSR_WRITE_1(sc, sc->rl_txstart, RL_TXSTART_START); 203 } 204 if (do_lock) 205 RL_UNLOCK(sc); 206 return 0; 207 } 208 209 210 /* 211 * Reconcile kernel and user view of the receive ring. 212 */ 213 static int 214 re_netmap_rxsync(struct ifnet *ifp, u_int ring_nr, int do_lock) 215 { 216 struct rl_softc *sc = ifp->if_softc; 217 struct rl_rxdesc *rxd = sc->rl_ldata.rl_rx_desc; 218 struct netmap_adapter *na = NA(sc->rl_ifp); 219 struct netmap_kring *kring = &na->rx_rings[ring_nr]; 220 struct netmap_ring *ring = kring->ring; 221 int j, l, n, lim = kring->nkr_num_slots - 1; 222 int force_update = do_lock || kring->nr_kflags & NKR_PENDINTR; 223 u_int k = ring->cur, resvd = ring->reserved; 224 225 k = ring->cur; 226 if (k > lim) 227 return netmap_ring_reinit(kring); 228 229 if (do_lock) 230 RL_LOCK(sc); 231 /* XXX check sync modes */ 232 bus_dmamap_sync(sc->rl_ldata.rl_rx_list_tag, 233 sc->rl_ldata.rl_rx_list_map, 234 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 235 236 /* 237 * Import newly received packets into the netmap ring. 238 * j is an index in the netmap ring, l in the NIC ring. 239 * 240 * The device uses all the buffers in the ring, so we need 241 * another termination condition in addition to RL_RDESC_STAT_OWN 242 * cleared (all buffers could have it cleared. The easiest one 243 * is to limit the amount of data reported up to 'lim' 244 */ 245 l = sc->rl_ldata.rl_rx_prodidx; /* next pkt to check */ 246 j = netmap_idx_n2k(kring, l); /* the kring index */ 247 if (netmap_no_pendintr || force_update) { 248 uint16_t slot_flags = kring->nkr_slot_flags; 249 250 for (n = kring->nr_hwavail; n < lim ; n++) { 251 struct rl_desc *cur_rx = &sc->rl_ldata.rl_rx_list[l]; 252 uint32_t rxstat = le32toh(cur_rx->rl_cmdstat); 253 uint32_t total_len; 254 255 if ((rxstat & RL_RDESC_STAT_OWN) != 0) 256 break; 257 total_len = rxstat & sc->rl_rxlenmask; 258 /* XXX subtract crc */ 259 total_len = (total_len < 4) ? 0 : total_len - 4; 260 kring->ring->slot[j].len = total_len; 261 kring->ring->slot[j].flags = slot_flags; 262 /* sync was in re_newbuf() */ 263 bus_dmamap_sync(sc->rl_ldata.rl_rx_mtag, 264 rxd[l].rx_dmamap, BUS_DMASYNC_POSTREAD); 265 j = (j == lim) ? 0 : j + 1; 266 l = (l == lim) ? 0 : l + 1; 267 } 268 if (n != kring->nr_hwavail) { 269 sc->rl_ldata.rl_rx_prodidx = l; 270 sc->rl_ifp->if_ipackets += n - kring->nr_hwavail; 271 kring->nr_hwavail = n; 272 } 273 kring->nr_kflags &= ~NKR_PENDINTR; 274 } 275 276 /* skip past packets that userspace has released */ 277 j = kring->nr_hwcur; 278 if (resvd > 0) { 279 if (resvd + ring->avail >= lim + 1) { 280 D("XXX invalid reserve/avail %d %d", resvd, ring->avail); 281 ring->reserved = resvd = 0; // XXX panic... 282 } 283 k = (k >= resvd) ? k - resvd : k + lim + 1 - resvd; 284 } 285 if (j != k) { /* userspace has released some packets. */ 286 l = netmap_idx_k2n(kring, j); /* the NIC index */ 287 for (n = 0; j != k; n++) { 288 struct netmap_slot *slot = ring->slot + j; 289 struct rl_desc *desc = &sc->rl_ldata.rl_rx_list[l]; 290 int cmd = NETMAP_BUF_SIZE | RL_RDESC_CMD_OWN; 291 uint64_t paddr; 292 void *addr = PNMB(slot, &paddr); 293 294 if (addr == netmap_buffer_base) { /* bad buf */ 295 if (do_lock) 296 RL_UNLOCK(sc); 297 return netmap_ring_reinit(kring); 298 } 299 300 if (l == lim) /* mark end of ring */ 301 cmd |= RL_RDESC_CMD_EOR; 302 303 slot->flags &= ~NS_REPORT; 304 if (slot->flags & NS_BUF_CHANGED) { 305 netmap_reload_map(sc->rl_ldata.rl_rx_mtag, 306 rxd[l].rx_dmamap, addr); 307 desc->rl_bufaddr_lo = htole32(RL_ADDR_LO(paddr)); 308 desc->rl_bufaddr_hi = htole32(RL_ADDR_HI(paddr)); 309 slot->flags &= ~NS_BUF_CHANGED; 310 } 311 desc->rl_cmdstat = htole32(cmd); 312 bus_dmamap_sync(sc->rl_ldata.rl_rx_mtag, 313 rxd[l].rx_dmamap, BUS_DMASYNC_PREREAD); 314 j = (j == lim) ? 0 : j + 1; 315 l = (l == lim) ? 0 : l + 1; 316 } 317 kring->nr_hwavail -= n; 318 kring->nr_hwcur = k; 319 /* Flush the RX DMA ring */ 320 321 bus_dmamap_sync(sc->rl_ldata.rl_rx_list_tag, 322 sc->rl_ldata.rl_rx_list_map, 323 BUS_DMASYNC_PREWRITE|BUS_DMASYNC_PREREAD); 324 } 325 /* tell userspace that there are new packets */ 326 ring->avail = kring->nr_hwavail - resvd; 327 if (do_lock) 328 RL_UNLOCK(sc); 329 return 0; 330 } 331 332 /* 333 * Additional routines to init the tx and rx rings. 334 * In other drivers we do that inline in the main code. 335 */ 336 static void 337 re_netmap_tx_init(struct rl_softc *sc) 338 { 339 struct rl_txdesc *txd; 340 struct rl_desc *desc; 341 int i, n; 342 struct netmap_adapter *na = NA(sc->rl_ifp); 343 struct netmap_slot *slot = netmap_reset(na, NR_TX, 0, 0); 344 345 /* slot is NULL if we are not in netmap mode */ 346 if (!slot) 347 return; 348 /* in netmap mode, overwrite addresses and maps */ 349 txd = sc->rl_ldata.rl_tx_desc; 350 desc = sc->rl_ldata.rl_tx_list; 351 n = sc->rl_ldata.rl_tx_desc_cnt; 352 353 /* l points in the netmap ring, i points in the NIC ring */ 354 for (i = 0; i < n; i++) { 355 uint64_t paddr; 356 int l = netmap_idx_n2k(&na->tx_rings[0], i); 357 void *addr = PNMB(slot + l, &paddr); 358 359 desc[i].rl_bufaddr_lo = htole32(RL_ADDR_LO(paddr)); 360 desc[i].rl_bufaddr_hi = htole32(RL_ADDR_HI(paddr)); 361 netmap_load_map(sc->rl_ldata.rl_tx_mtag, 362 txd[i].tx_dmamap, addr); 363 } 364 } 365 366 static void 367 re_netmap_rx_init(struct rl_softc *sc) 368 { 369 struct netmap_adapter *na = NA(sc->rl_ifp); 370 struct netmap_slot *slot = netmap_reset(na, NR_RX, 0, 0); 371 struct rl_desc *desc = sc->rl_ldata.rl_rx_list; 372 uint32_t cmdstat; 373 int i, n, max_avail; 374 375 if (!slot) 376 return; 377 n = sc->rl_ldata.rl_rx_desc_cnt; 378 /* 379 * Userspace owned hwavail packets before the reset, 380 * so the NIC that last hwavail descriptors of the ring 381 * are still owned by the driver (and keep one empty). 382 */ 383 max_avail = n - 1 - na->rx_rings[0].nr_hwavail; 384 for (i = 0; i < n; i++) { 385 void *addr; 386 uint64_t paddr; 387 int l = netmap_idx_n2k(&na->rx_rings[0], i); 388 389 addr = PNMB(slot + l, &paddr); 390 391 netmap_reload_map(sc->rl_ldata.rl_rx_mtag, 392 sc->rl_ldata.rl_rx_desc[i].rx_dmamap, addr); 393 bus_dmamap_sync(sc->rl_ldata.rl_rx_mtag, 394 sc->rl_ldata.rl_rx_desc[i].rx_dmamap, BUS_DMASYNC_PREREAD); 395 desc[i].rl_bufaddr_lo = htole32(RL_ADDR_LO(paddr)); 396 desc[i].rl_bufaddr_hi = htole32(RL_ADDR_HI(paddr)); 397 cmdstat = NETMAP_BUF_SIZE; 398 if (i == n - 1) /* mark the end of ring */ 399 cmdstat |= RL_RDESC_CMD_EOR; 400 if (i < max_avail) 401 cmdstat |= RL_RDESC_CMD_OWN; 402 desc[i].rl_cmdstat = htole32(cmdstat); 403 } 404 } 405 406 407 static void 408 re_netmap_attach(struct rl_softc *sc) 409 { 410 struct netmap_adapter na; 411 412 bzero(&na, sizeof(na)); 413 414 na.ifp = sc->rl_ifp; 415 na.separate_locks = 0; 416 na.num_tx_desc = sc->rl_ldata.rl_tx_desc_cnt; 417 na.num_rx_desc = sc->rl_ldata.rl_rx_desc_cnt; 418 na.nm_txsync = re_netmap_txsync; 419 na.nm_rxsync = re_netmap_rxsync; 420 na.nm_lock = re_netmap_lock_wrapper; 421 na.nm_register = re_netmap_reg; 422 netmap_attach(&na, 1); 423 } 424 /* end of file */ 425