1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3 * 4 * Copyright (c) 2015 Bjoern A. Zeeb 5 * Copyright (c) 2020 Denis Salopek 6 * 7 * This software was developed by SRI International and the University of 8 * Cambridge Computer Laboratory under DARPA/AFRL contract FA8750-11-C-0249 9 * ("MRC2"), as part of the DARPA MRC research programme. 10 * 11 * Redistribution and use in source and binary forms, with or without 12 * modification, are permitted provided that the following conditions 13 * are met: 14 * 1. Redistributions of source code must retain the above copyright 15 * notice, this list of conditions and the following disclaimer. 16 * 2. Redistributions in binary form must reproduce the above copyright 17 * notice, this list of conditions and the following disclaimer in the 18 * documentation and/or other materials provided with the distribution. 19 * 20 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 23 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 24 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 25 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 26 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 27 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 28 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 29 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 30 * POSSIBILITY OF SUCH DAMAGE. 31 */ 32 33 #include <sys/cdefs.h> 34 __FBSDID("$FreeBSD$"); 35 36 #include <sys/param.h> 37 #include <sys/bus.h> 38 #include <sys/endian.h> 39 #include <sys/kernel.h> 40 #include <sys/limits.h> 41 #include <sys/module.h> 42 #include <sys/rman.h> 43 #include <sys/socket.h> 44 #include <sys/sockio.h> 45 #include <sys/sysctl.h> 46 #include <sys/taskqueue.h> 47 48 #include <net/if.h> 49 #include <net/if_media.h> 50 #include <net/if_types.h> 51 #include <net/if_var.h> 52 53 #include <netinet/in.h> 54 #include <netinet/if_ether.h> 55 56 #include <dev/pci/pcivar.h> 57 #include <dev/pci/pcireg.h> 58 59 #include <machine/bus.h> 60 61 #include "adapter.h" 62 63 #define PCI_VENDOR_ID_XILINX 0x10ee 64 #define PCI_DEVICE_ID_SUME 0x7028 65 66 /* SUME bus driver interface */ 67 static int sume_probe(device_t); 68 static int sume_attach(device_t); 69 static int sume_detach(device_t); 70 71 static device_method_t sume_methods[] = { 72 DEVMETHOD(device_probe, sume_probe), 73 DEVMETHOD(device_attach, sume_attach), 74 DEVMETHOD(device_detach, sume_detach), 75 DEVMETHOD_END 76 }; 77 78 static driver_t sume_driver = { 79 "sume", 80 sume_methods, 81 sizeof(struct sume_adapter) 82 }; 83 84 /* 85 * The DMA engine for SUME generates interrupts for each RX/TX transaction. 86 * Depending on the channel (0 if packet transaction, 1 if register transaction) 87 * the used bits of the interrupt vector will be the lowest or the second lowest 88 * 5 bits. 89 * 90 * When receiving packets from SUME (RX): 91 * (1) SUME received a packet on one of the interfaces. 92 * (2) SUME generates an interrupt vector, bit 00001 is set (channel 0 - new RX 93 * transaction). 94 * (3) We read the length of the incoming packet and the offset along with the 95 * 'last' flag from the SUME registers. 96 * (4) We prepare for the DMA transaction by setting the bouncebuffer on the 97 * address buf_addr. For now, this is how it's done: 98 * - First 3*sizeof(uint32_t) bytes are: lower and upper 32 bits of physical 99 * address where we want the data to arrive (buf_addr[0] and buf_addr[1]), 100 * and length of incoming data (buf_addr[2]). 101 * - Data will start right after, at buf_addr+3*sizeof(uint32_t). The 102 * physical address buf_hw_addr is a block of contiguous memory mapped to 103 * buf_addr, so we can set the incoming data's physical address (buf_addr[0] 104 * and buf_addr[1]) to buf_hw_addr+3*sizeof(uint32_t). 105 * (5) We notify SUME that the bouncebuffer is ready for the transaction by 106 * writing the lower/upper physical address buf_hw_addr to the SUME 107 * registers RIFFA_TX_SG_ADDR_LO_REG_OFF and RIFFA_TX_SG_ADDR_HI_REG_OFF as 108 * well as the number of segments to the register RIFFA_TX_SG_LEN_REG_OFF. 109 * (6) SUME generates an interrupt vector, bit 00010 is set (channel 0 - 110 * bouncebuffer received). 111 * (7) SUME generates an interrupt vector, bit 00100 is set (channel 0 - 112 * transaction is done). 113 * (8) SUME can do both steps (6) and (7) using the same interrupt. 114 * (8) We read the first 16 bytes (metadata) of the received data and note the 115 * incoming interface so we can later forward it to the right one in the OS 116 * (sume0, sume1, sume2 or sume3). 117 * (10) We create an mbuf and copy the data from the bouncebuffer to the mbuf 118 * and set the mbuf rcvif to the incoming interface. 119 * (11) We forward the mbuf to the appropriate interface via ifp->if_input. 120 * 121 * When sending packets to SUME (TX): 122 * (1) The OS calls sume_if_start() function on TX. 123 * (2) We get the mbuf packet data and copy it to the 124 * buf_addr+3*sizeof(uint32_t) + metadata 16 bytes. 125 * (3) We create the metadata based on the output interface and copy it to the 126 * buf_addr+3*sizeof(uint32_t). 127 * (4) We write the offset/last and length of the packet to the SUME registers 128 * RIFFA_RX_OFFLAST_REG_OFF and RIFFA_RX_LEN_REG_OFF. 129 * (5) We fill the bouncebuffer by filling the first 3*sizeof(uint32_t) bytes 130 * with the physical address and length just as in RX step (4). 131 * (6) We notify SUME that the bouncebuffer is ready by writing to SUME 132 * registers RIFFA_RX_SG_ADDR_LO_REG_OFF, RIFFA_RX_SG_ADDR_HI_REG_OFF and 133 * RIFFA_RX_SG_LEN_REG_OFF just as in RX step (5). 134 * (7) SUME generates an interrupt vector, bit 01000 is set (channel 0 - 135 * bouncebuffer is read). 136 * (8) SUME generates an interrupt vector, bit 10000 is set (channel 0 - 137 * transaction is done). 138 * (9) SUME can do both steps (7) and (8) using the same interrupt. 139 * 140 * Internal registers 141 * Every module in the SUME hardware has its own set of internal registers 142 * (IDs, for debugging and statistic purposes, etc.). Their base addresses are 143 * defined in 'projects/reference_nic/hw/tcl/reference_nic_defines.tcl' and the 144 * offsets to different memory locations of every module are defined in their 145 * corresponding folder inside the library. These registers can be RO/RW and 146 * there is a special method to fetch/change this data over 1 or 2 DMA 147 * transactions. For writing, by calling the sume_module_reg_write(). For 148 * reading, by calling the sume_module_reg_write() and then 149 * sume_module_reg_read(). Check those functions for more information. 150 */ 151 152 MALLOC_DECLARE(M_SUME); 153 MALLOC_DEFINE(M_SUME, "sume", "NetFPGA SUME device driver"); 154 155 static void check_tx_queues(struct sume_adapter *); 156 static void sume_fill_bb_desc(struct sume_adapter *, struct riffa_chnl_dir *, 157 uint64_t); 158 159 static struct unrhdr *unr; 160 161 static struct { 162 uint16_t device; 163 char *desc; 164 } sume_pciids[] = { 165 {PCI_DEVICE_ID_SUME, "NetFPGA SUME reference NIC"}, 166 }; 167 168 static inline uint32_t 169 read_reg(struct sume_adapter *adapter, int offset) 170 { 171 172 return (bus_space_read_4(adapter->bt, adapter->bh, offset << 2)); 173 } 174 175 static inline void 176 write_reg(struct sume_adapter *adapter, int offset, uint32_t val) 177 { 178 179 bus_space_write_4(adapter->bt, adapter->bh, offset << 2, val); 180 } 181 182 static int 183 sume_probe(device_t dev) 184 { 185 int i; 186 uint16_t v = pci_get_vendor(dev); 187 uint16_t d = pci_get_device(dev); 188 189 if (v != PCI_VENDOR_ID_XILINX) 190 return (ENXIO); 191 192 for (i = 0; i < nitems(sume_pciids); i++) { 193 if (d == sume_pciids[i].device) { 194 device_set_desc(dev, sume_pciids[i].desc); 195 return (BUS_PROBE_DEFAULT); 196 } 197 } 198 199 return (ENXIO); 200 } 201 202 /* 203 * Building mbuf for packet received from SUME. We expect to receive 'len' 204 * bytes of data (including metadata) written from the bouncebuffer address 205 * buf_addr+3*sizeof(uint32_t). Metadata will tell us which SUME interface 206 * received the packet (sport will be 1, 2, 4 or 8), the packet length (plen), 207 * and the magic word needs to be 0xcafe. When we have the packet data, we 208 * create an mbuf and copy the data to it using m_copyback() function, set the 209 * correct interface to rcvif and return the mbuf to be later sent to the OS 210 * with if_input. 211 */ 212 static struct mbuf * 213 sume_rx_build_mbuf(struct sume_adapter *adapter, uint32_t len) 214 { 215 struct nf_priv *nf_priv; 216 struct mbuf *m; 217 struct ifnet *ifp = NULL; 218 int np; 219 uint16_t dport, plen, magic; 220 device_t dev = adapter->dev; 221 uint8_t *indata = (uint8_t *) 222 adapter->recv[SUME_RIFFA_CHANNEL_DATA]->buf_addr + 223 sizeof(struct nf_bb_desc); 224 struct nf_metadata *mdata = (struct nf_metadata *) indata; 225 226 /* The metadata header is 16 bytes. */ 227 if (len < sizeof(struct nf_metadata)) { 228 device_printf(dev, "short frame (%d)\n", len); 229 adapter->packets_err++; 230 adapter->bytes_err += len; 231 return (NULL); 232 } 233 234 dport = le16toh(mdata->dport); 235 plen = le16toh(mdata->plen); 236 magic = le16toh(mdata->magic); 237 238 if (sizeof(struct nf_metadata) + plen > len || 239 magic != SUME_RIFFA_MAGIC) { 240 device_printf(dev, "corrupted packet (%zd + %d > %d || magic " 241 "0x%04x != 0x%04x)\n", sizeof(struct nf_metadata), plen, 242 len, magic, SUME_RIFFA_MAGIC); 243 return (NULL); 244 } 245 246 /* We got the packet from one of the even bits */ 247 np = (ffs(dport & SUME_DPORT_MASK) >> 1) - 1; 248 if (np > SUME_NPORTS) { 249 device_printf(dev, "invalid destination port 0x%04x (%d)\n", 250 dport, np); 251 adapter->packets_err++; 252 adapter->bytes_err += plen; 253 return (NULL); 254 } 255 ifp = adapter->ifp[np]; 256 nf_priv = ifp->if_softc; 257 nf_priv->stats.rx_packets++; 258 nf_priv->stats.rx_bytes += plen; 259 260 /* If the interface is down, well, we are done. */ 261 if (!(ifp->if_flags & IFF_UP)) { 262 nf_priv->stats.ifc_down_packets++; 263 nf_priv->stats.ifc_down_bytes += plen; 264 return (NULL); 265 } 266 267 if (adapter->sume_debug) 268 printf("Building mbuf with length: %d\n", plen); 269 270 m = m_getm(NULL, plen, M_NOWAIT, MT_DATA); 271 if (m == NULL) { 272 adapter->packets_err++; 273 adapter->bytes_err += plen; 274 return (NULL); 275 } 276 277 /* Copy the data in at the right offset. */ 278 m_copyback(m, 0, plen, (void *) (indata + sizeof(struct nf_metadata))); 279 m->m_pkthdr.rcvif = ifp; 280 281 return (m); 282 } 283 284 /* 285 * SUME interrupt handler for when we get a valid interrupt from the board. 286 * Theoretically, we can receive interrupt for any of the available channels, 287 * but RIFFA DMA uses only 2: 0 and 1, so we use only vect0. The vector is a 32 288 * bit number, using 5 bits for every channel, the least significant bits 289 * correspond to channel 0 and the next 5 bits correspond to channel 1. Vector 290 * bits for RX/TX are: 291 * RX 292 * bit 0 - new transaction from SUME 293 * bit 1 - SUME received our bouncebuffer address 294 * bit 2 - SUME copied the received data to our bouncebuffer, transaction done 295 * TX 296 * bit 3 - SUME received our bouncebuffer address 297 * bit 4 - SUME copied the data from our bouncebuffer, transaction done 298 * 299 * There are two finite state machines (one for TX, one for RX). We loop 300 * through channels 0 and 1 to check and our current state and which interrupt 301 * bit is set. 302 * TX 303 * SUME_RIFFA_CHAN_STATE_IDLE: waiting for the first TX transaction. 304 * SUME_RIFFA_CHAN_STATE_READY: we prepared (filled with data) the bouncebuffer 305 * and triggered the SUME for the TX transaction. Waiting for interrupt bit 3 306 * to go to the next state. 307 * SUME_RIFFA_CHAN_STATE_READ: waiting for interrupt bit 4 (for SUME to send 308 * our packet). Then we get the length of the sent data and go back to the 309 * IDLE state. 310 * RX 311 * SUME_RIFFA_CHAN_STATE_IDLE: waiting for the interrupt bit 0 (new RX 312 * transaction). When we get it, we prepare our bouncebuffer for reading and 313 * trigger the SUME to start the transaction. Go to the next state. 314 * SUME_RIFFA_CHAN_STATE_READY: waiting for the interrupt bit 1 (SUME got our 315 * bouncebuffer). Go to the next state. 316 * SUME_RIFFA_CHAN_STATE_READ: SUME copied data and our bouncebuffer is ready, 317 * we can build the mbuf and go back to the IDLE state. 318 */ 319 static void 320 sume_intr_handler(void *arg) 321 { 322 struct sume_adapter *adapter = arg; 323 uint32_t vect, vect0, len; 324 int ch, loops; 325 device_t dev = adapter->dev; 326 struct mbuf *m = NULL; 327 struct ifnet *ifp = NULL; 328 struct riffa_chnl_dir *send, *recv; 329 330 SUME_LOCK(adapter); 331 332 vect0 = read_reg(adapter, RIFFA_IRQ_REG0_OFF); 333 if ((vect0 & SUME_INVALID_VECT) != 0) { 334 SUME_UNLOCK(adapter); 335 return; 336 } 337 338 /* 339 * We only have one interrupt for all channels and no way 340 * to quickly lookup for which channel(s) we got an interrupt? 341 */ 342 for (ch = 0; ch < SUME_RIFFA_CHANNELS; ch++) { 343 vect = vect0 >> (5 * ch); 344 send = adapter->send[ch]; 345 recv = adapter->recv[ch]; 346 347 loops = 0; 348 while ((vect & (SUME_MSI_TXBUF | SUME_MSI_TXDONE)) && 349 loops <= 5) { 350 if (adapter->sume_debug) 351 device_printf(dev, "TX ch %d state %u vect = " 352 "0x%08x\n", ch, send->state, vect); 353 switch (send->state) { 354 case SUME_RIFFA_CHAN_STATE_IDLE: 355 break; 356 case SUME_RIFFA_CHAN_STATE_READY: 357 if (!(vect & SUME_MSI_TXBUF)) { 358 device_printf(dev, "ch %d unexpected " 359 "interrupt in send+3 state %u: " 360 "vect = 0x%08x\n", ch, send->state, 361 vect); 362 send->recovery = 1; 363 break; 364 } 365 send->state = SUME_RIFFA_CHAN_STATE_READ; 366 vect &= ~SUME_MSI_TXBUF; 367 break; 368 case SUME_RIFFA_CHAN_STATE_READ: 369 if (!(vect & SUME_MSI_TXDONE)) { 370 device_printf(dev, "ch %d unexpected " 371 "interrupt in send+4 state %u: " 372 "vect = 0x%08x\n", ch, send->state, 373 vect); 374 send->recovery = 1; 375 break; 376 } 377 send->state = SUME_RIFFA_CHAN_STATE_LEN; 378 379 len = read_reg(adapter, RIFFA_CHNL_REG(ch, 380 RIFFA_RX_TNFR_LEN_REG_OFF)); 381 if (ch == SUME_RIFFA_CHANNEL_DATA) { 382 send->state = 383 SUME_RIFFA_CHAN_STATE_IDLE; 384 check_tx_queues(adapter); 385 } else if (ch == SUME_RIFFA_CHANNEL_REG) 386 wakeup(&send->event); 387 else { 388 device_printf(dev, "ch %d unexpected " 389 "interrupt in send+4 state %u: " 390 "vect = 0x%08x\n", ch, send->state, 391 vect); 392 send->recovery = 1; 393 } 394 vect &= ~SUME_MSI_TXDONE; 395 break; 396 case SUME_RIFFA_CHAN_STATE_LEN: 397 break; 398 default: 399 device_printf(dev, "unknown TX state!\n"); 400 } 401 loops++; 402 } 403 404 if ((vect & (SUME_MSI_TXBUF | SUME_MSI_TXDONE)) && 405 send->recovery) 406 device_printf(dev, "ch %d ignoring vect = 0x%08x " 407 "during TX; not in recovery; state = %d loops = " 408 "%d\n", ch, vect, send->state, loops); 409 410 loops = 0; 411 while ((vect & (SUME_MSI_RXQUE | SUME_MSI_RXBUF | 412 SUME_MSI_RXDONE)) && loops < 5) { 413 if (adapter->sume_debug) 414 device_printf(dev, "RX ch %d state %u vect = " 415 "0x%08x\n", ch, recv->state, vect); 416 switch (recv->state) { 417 case SUME_RIFFA_CHAN_STATE_IDLE: 418 if (!(vect & SUME_MSI_RXQUE)) { 419 device_printf(dev, "ch %d unexpected " 420 "interrupt in recv+0 state %u: " 421 "vect = 0x%08x\n", ch, recv->state, 422 vect); 423 recv->recovery = 1; 424 break; 425 } 426 uint32_t max_ptr; 427 428 /* Clear recovery state. */ 429 recv->recovery = 0; 430 431 /* Get offset and length. */ 432 recv->offlast = read_reg(adapter, 433 RIFFA_CHNL_REG(ch, 434 RIFFA_TX_OFFLAST_REG_OFF)); 435 recv->len = read_reg(adapter, RIFFA_CHNL_REG(ch, 436 RIFFA_TX_LEN_REG_OFF)); 437 438 /* Boundary checks. */ 439 max_ptr = (uint32_t)((uintptr_t)recv->buf_addr 440 + SUME_RIFFA_OFFSET(recv->offlast) 441 + SUME_RIFFA_LEN(recv->len) - 1); 442 if (max_ptr < 443 (uint32_t)((uintptr_t)recv->buf_addr)) 444 device_printf(dev, "receive buffer " 445 "wrap-around overflow.\n"); 446 if (SUME_RIFFA_OFFSET(recv->offlast) + 447 SUME_RIFFA_LEN(recv->len) > 448 adapter->sg_buf_size) 449 device_printf(dev, "receive buffer too" 450 " small.\n"); 451 452 /* Fill the bouncebuf "descriptor". */ 453 sume_fill_bb_desc(adapter, recv, 454 SUME_RIFFA_LEN(recv->len)); 455 456 bus_dmamap_sync(recv->ch_tag, recv->ch_map, 457 BUS_DMASYNC_PREREAD | 458 BUS_DMASYNC_PREWRITE); 459 write_reg(adapter, RIFFA_CHNL_REG(ch, 460 RIFFA_TX_SG_ADDR_LO_REG_OFF), 461 SUME_RIFFA_LO_ADDR(recv->buf_hw_addr)); 462 write_reg(adapter, RIFFA_CHNL_REG(ch, 463 RIFFA_TX_SG_ADDR_HI_REG_OFF), 464 SUME_RIFFA_HI_ADDR(recv->buf_hw_addr)); 465 write_reg(adapter, RIFFA_CHNL_REG(ch, 466 RIFFA_TX_SG_LEN_REG_OFF), 467 4 * recv->num_sg); 468 bus_dmamap_sync(recv->ch_tag, recv->ch_map, 469 BUS_DMASYNC_POSTREAD | 470 BUS_DMASYNC_POSTWRITE); 471 472 recv->state = SUME_RIFFA_CHAN_STATE_READY; 473 vect &= ~SUME_MSI_RXQUE; 474 break; 475 case SUME_RIFFA_CHAN_STATE_READY: 476 if (!(vect & SUME_MSI_RXBUF)) { 477 device_printf(dev, "ch %d unexpected " 478 "interrupt in recv+1 state %u: " 479 "vect = 0x%08x\n", ch, recv->state, 480 vect); 481 recv->recovery = 1; 482 break; 483 } 484 recv->state = SUME_RIFFA_CHAN_STATE_READ; 485 vect &= ~SUME_MSI_RXBUF; 486 break; 487 case SUME_RIFFA_CHAN_STATE_READ: 488 if (!(vect & SUME_MSI_RXDONE)) { 489 device_printf(dev, "ch %d unexpected " 490 "interrupt in recv+2 state %u: " 491 "vect = 0x%08x\n", ch, recv->state, 492 vect); 493 recv->recovery = 1; 494 break; 495 } 496 len = read_reg(adapter, RIFFA_CHNL_REG(ch, 497 RIFFA_TX_TNFR_LEN_REG_OFF)); 498 499 /* Remember, len and recv->len are words. */ 500 if (ch == SUME_RIFFA_CHANNEL_DATA) { 501 m = sume_rx_build_mbuf(adapter, 502 len << 2); 503 recv->state = 504 SUME_RIFFA_CHAN_STATE_IDLE; 505 } else if (ch == SUME_RIFFA_CHANNEL_REG) 506 wakeup(&recv->event); 507 else { 508 device_printf(dev, "ch %d unexpected " 509 "interrupt in recv+2 state %u: " 510 "vect = 0x%08x\n", ch, recv->state, 511 vect); 512 recv->recovery = 1; 513 } 514 vect &= ~SUME_MSI_RXDONE; 515 break; 516 case SUME_RIFFA_CHAN_STATE_LEN: 517 break; 518 default: 519 device_printf(dev, "unknown RX state!\n"); 520 } 521 loops++; 522 } 523 524 if ((vect & (SUME_MSI_RXQUE | SUME_MSI_RXBUF | 525 SUME_MSI_RXDONE)) && recv->recovery) { 526 device_printf(dev, "ch %d ignoring vect = 0x%08x " 527 "during RX; not in recovery; state = %d, loops = " 528 "%d\n", ch, vect, recv->state, loops); 529 530 /* Clean the unfinished transaction. */ 531 if (ch == SUME_RIFFA_CHANNEL_REG && 532 vect & SUME_MSI_RXDONE) { 533 read_reg(adapter, RIFFA_CHNL_REG(ch, 534 RIFFA_TX_TNFR_LEN_REG_OFF)); 535 recv->recovery = 0; 536 } 537 } 538 } 539 SUME_UNLOCK(adapter); 540 541 if (m != NULL) { 542 ifp = m->m_pkthdr.rcvif; 543 (*ifp->if_input)(ifp, m); 544 } 545 } 546 547 /* 548 * As we cannot disable interrupt generation, ignore early interrupts by waiting 549 * for the adapter to go into the 'running' state. 550 */ 551 static int 552 sume_intr_filter(void *arg) 553 { 554 struct sume_adapter *adapter = arg; 555 556 if (adapter->running == 0) 557 return (FILTER_STRAY); 558 559 return (FILTER_SCHEDULE_THREAD); 560 } 561 562 static int 563 sume_probe_riffa_pci(struct sume_adapter *adapter) 564 { 565 device_t dev = adapter->dev; 566 int error, count, capmem; 567 uint32_t reg, devctl, linkctl; 568 569 pci_enable_busmaster(dev); 570 571 adapter->rid = PCIR_BAR(0); 572 adapter->bar0_addr = bus_alloc_resource_any(dev, SYS_RES_MEMORY, 573 &adapter->rid, RF_ACTIVE); 574 if (adapter->bar0_addr == NULL) { 575 device_printf(dev, "unable to allocate bus resource: " 576 "BAR0 address\n"); 577 return (ENXIO); 578 } 579 adapter->bt = rman_get_bustag(adapter->bar0_addr); 580 adapter->bh = rman_get_bushandle(adapter->bar0_addr); 581 adapter->bar0_len = rman_get_size(adapter->bar0_addr); 582 if (adapter->bar0_len != 1024) { 583 device_printf(dev, "BAR0 resource length %lu != 1024\n", 584 adapter->bar0_len); 585 return (ENXIO); 586 } 587 588 count = pci_msi_count(dev); 589 error = pci_alloc_msi(dev, &count); 590 if (error) { 591 device_printf(dev, "unable to allocate bus resource: PCI " 592 "MSI\n"); 593 return (error); 594 } 595 596 adapter->irq.rid = 1; /* Should be 1, thus says pci_alloc_msi() */ 597 adapter->irq.res = bus_alloc_resource_any(dev, SYS_RES_IRQ, 598 &adapter->irq.rid, RF_SHAREABLE | RF_ACTIVE); 599 if (adapter->irq.res == NULL) { 600 device_printf(dev, "unable to allocate bus resource: IRQ " 601 "memory\n"); 602 return (ENXIO); 603 } 604 605 error = bus_setup_intr(dev, adapter->irq.res, INTR_MPSAFE | 606 INTR_TYPE_NET, sume_intr_filter, sume_intr_handler, adapter, 607 &adapter->irq.tag); 608 if (error) { 609 device_printf(dev, "failed to setup interrupt for rid %d, name" 610 " %s: %d\n", adapter->irq.rid, "SUME_INTR", error); 611 return (ENXIO); 612 } 613 614 if (pci_find_cap(dev, PCIY_EXPRESS, &capmem) != 0) { 615 device_printf(dev, "PCI not PCIe capable\n"); 616 return (ENXIO); 617 } 618 619 devctl = pci_read_config(dev, capmem + PCIER_DEVICE_CTL, 2); 620 pci_write_config(dev, capmem + PCIER_DEVICE_CTL, (devctl | 621 PCIEM_CTL_EXT_TAG_FIELD), 2); 622 623 devctl = pci_read_config(dev, capmem + PCIER_DEVICE_CTL2, 2); 624 pci_write_config(dev, capmem + PCIER_DEVICE_CTL2, (devctl | 625 PCIEM_CTL2_ID_ORDERED_REQ_EN), 2); 626 627 linkctl = pci_read_config(dev, capmem + PCIER_LINK_CTL, 2); 628 pci_write_config(dev, capmem + PCIER_LINK_CTL, (linkctl | 629 PCIEM_LINK_CTL_RCB), 2); 630 631 reg = read_reg(adapter, RIFFA_INFO_REG_OFF); 632 adapter->num_sg = RIFFA_SG_ELEMS * ((reg >> 19) & 0xf); 633 adapter->sg_buf_size = RIFFA_SG_BUF_SIZE * ((reg >> 19) & 0xf); 634 635 error = ENODEV; 636 /* Check bus master is enabled. */ 637 if (((reg >> 4) & 0x1) != 1) { 638 device_printf(dev, "bus master not enabled: %d\n", 639 (reg >> 4) & 0x1); 640 return (error); 641 } 642 /* Check link parameters are valid. */ 643 if (((reg >> 5) & 0x3f) == 0 || ((reg >> 11) & 0x3) == 0) { 644 device_printf(dev, "link parameters not valid: %d %d\n", 645 (reg >> 5) & 0x3f, (reg >> 11) & 0x3); 646 return (error); 647 } 648 /* Check # of channels are within valid range. */ 649 if ((reg & 0xf) == 0 || (reg & 0xf) > RIFFA_MAX_CHNLS) { 650 device_printf(dev, "number of channels out of range: %d\n", 651 reg & 0xf); 652 return (error); 653 } 654 /* Check bus width. */ 655 if (((reg >> 19) & 0xf) == 0 || 656 ((reg >> 19) & 0xf) > RIFFA_MAX_BUS_WIDTH_PARAM) { 657 device_printf(dev, "bus width out of range: %d\n", 658 (reg >> 19) & 0xf); 659 return (error); 660 } 661 662 device_printf(dev, "[riffa] # of channels: %d\n", 663 reg & 0xf); 664 device_printf(dev, "[riffa] bus interface width: %d\n", 665 ((reg >> 19) & 0xf) << 5); 666 device_printf(dev, "[riffa] bus master enabled: %d\n", 667 (reg >> 4) & 0x1); 668 device_printf(dev, "[riffa] negotiated link width: %d\n", 669 (reg >> 5) & 0x3f); 670 device_printf(dev, "[riffa] negotiated rate width: %d MTs\n", 671 ((reg >> 11) & 0x3) * 2500); 672 device_printf(dev, "[riffa] max downstream payload: %d B\n", 673 128 << ((reg >> 13) & 0x7)); 674 device_printf(dev, "[riffa] max upstream payload: %d B\n", 675 128 << ((reg >> 16) & 0x7)); 676 677 return (0); 678 } 679 680 /* If there is no sume_if_init, the ether_ioctl panics. */ 681 static void 682 sume_if_init(void *sc) 683 { 684 } 685 686 /* Write the address and length for our incoming / outgoing transaction. */ 687 static void 688 sume_fill_bb_desc(struct sume_adapter *adapter, struct riffa_chnl_dir *p, 689 uint64_t len) 690 { 691 struct nf_bb_desc *bouncebuf = (struct nf_bb_desc *) p->buf_addr; 692 693 bouncebuf->lower = (p->buf_hw_addr + sizeof(struct nf_bb_desc)); 694 bouncebuf->upper = (p->buf_hw_addr + sizeof(struct nf_bb_desc)) >> 32; 695 bouncebuf->len = len >> 2; 696 } 697 698 /* Module register locked write. */ 699 static int 700 sume_modreg_write_locked(struct sume_adapter *adapter) 701 { 702 struct riffa_chnl_dir *send = adapter->send[SUME_RIFFA_CHANNEL_REG]; 703 704 /* Let the FPGA know about the transfer. */ 705 write_reg(adapter, RIFFA_CHNL_REG(SUME_RIFFA_CHANNEL_REG, 706 RIFFA_RX_OFFLAST_REG_OFF), SUME_OFFLAST); 707 write_reg(adapter, RIFFA_CHNL_REG(SUME_RIFFA_CHANNEL_REG, 708 RIFFA_RX_LEN_REG_OFF), send->len); /* words */ 709 710 /* Fill the bouncebuf "descriptor". */ 711 sume_fill_bb_desc(adapter, send, SUME_RIFFA_LEN(send->len)); 712 713 /* Update the state before intiating the DMA to avoid races. */ 714 send->state = SUME_RIFFA_CHAN_STATE_READY; 715 716 bus_dmamap_sync(send->ch_tag, send->ch_map, 717 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 718 /* DMA. */ 719 write_reg(adapter, RIFFA_CHNL_REG(SUME_RIFFA_CHANNEL_REG, 720 RIFFA_RX_SG_ADDR_LO_REG_OFF), 721 SUME_RIFFA_LO_ADDR(send->buf_hw_addr)); 722 write_reg(adapter, RIFFA_CHNL_REG(SUME_RIFFA_CHANNEL_REG, 723 RIFFA_RX_SG_ADDR_HI_REG_OFF), 724 SUME_RIFFA_HI_ADDR(send->buf_hw_addr)); 725 write_reg(adapter, RIFFA_CHNL_REG(SUME_RIFFA_CHANNEL_REG, 726 RIFFA_RX_SG_LEN_REG_OFF), 4 * send->num_sg); 727 bus_dmamap_sync(send->ch_tag, send->ch_map, 728 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 729 730 return (0); 731 } 732 733 /* 734 * Request a register read or write (depending on optype). 735 * If optype is set (0x1f) this will result in a register write, 736 * otherwise this will result in a register read request at the given 737 * address and the result will need to be DMAed back. 738 */ 739 static int 740 sume_module_reg_write(struct nf_priv *nf_priv, struct sume_ifreq *sifr, 741 uint32_t optype) 742 { 743 struct sume_adapter *adapter = nf_priv->adapter; 744 struct riffa_chnl_dir *send = adapter->send[SUME_RIFFA_CHANNEL_REG]; 745 struct nf_regop_data *data; 746 int error; 747 748 /* 749 * 1. Make sure the channel is free; otherwise return EBUSY. 750 * 2. Prepare the memory in the bounce buffer (which we always 751 * use for regs). 752 * 3. Start the DMA process. 753 * 4. Sleep and wait for result and return success or error. 754 */ 755 SUME_LOCK(adapter); 756 757 if (send->state != SUME_RIFFA_CHAN_STATE_IDLE) { 758 SUME_UNLOCK(adapter); 759 return (EBUSY); 760 } 761 762 data = (struct nf_regop_data *) (send->buf_addr + 763 sizeof(struct nf_bb_desc)); 764 data->addr = htole32(sifr->addr); 765 data->val = htole32(sifr->val); 766 /* Tag to indentify request. */ 767 data->rtag = htole32(++send->rtag); 768 data->optype = htole32(optype); 769 send->len = sizeof(struct nf_regop_data) / 4; /* words */ 770 771 error = sume_modreg_write_locked(adapter); 772 if (error) { 773 SUME_UNLOCK(adapter); 774 return (EFAULT); 775 } 776 777 /* Timeout after 1s. */ 778 if (send->state != SUME_RIFFA_CHAN_STATE_LEN) 779 error = msleep(&send->event, &adapter->lock, 0, 780 "Waiting recv finish", 1 * hz); 781 782 /* This was a write so we are done; were interrupted, or timed out. */ 783 if (optype != SUME_MR_READ || error != 0 || error == EWOULDBLOCK) { 784 send->state = SUME_RIFFA_CHAN_STATE_IDLE; 785 if (optype == SUME_MR_READ) 786 error = EWOULDBLOCK; 787 else 788 error = 0; 789 } else 790 error = 0; 791 792 /* 793 * For read requests we will update state once we are done 794 * having read the result to avoid any two outstanding 795 * transactions, or we need a queue and validate tags, 796 * which is a lot of work for a low priority, infrequent 797 * event. 798 */ 799 800 SUME_UNLOCK(adapter); 801 802 return (error); 803 } 804 805 /* Module register read. */ 806 static int 807 sume_module_reg_read(struct nf_priv *nf_priv, struct sume_ifreq *sifr) 808 { 809 struct sume_adapter *adapter = nf_priv->adapter; 810 struct riffa_chnl_dir *recv = adapter->recv[SUME_RIFFA_CHANNEL_REG]; 811 struct riffa_chnl_dir *send = adapter->send[SUME_RIFFA_CHANNEL_REG]; 812 struct nf_regop_data *data; 813 int error = 0; 814 815 /* 816 * 0. Sleep waiting for result if needed (unless condition is 817 * true already). 818 * 1. Read DMA results. 819 * 2. Update state on *TX* to IDLE to allow next read to start. 820 */ 821 SUME_LOCK(adapter); 822 823 bus_dmamap_sync(recv->ch_tag, recv->ch_map, 824 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 825 /* 826 * We only need to be woken up at the end of the transaction. 827 * Timeout after 1s. 828 */ 829 if (recv->state != SUME_RIFFA_CHAN_STATE_READ) 830 error = msleep(&recv->event, &adapter->lock, 0, 831 "Waiting transaction finish", 1 * hz); 832 833 if (recv->state != SUME_RIFFA_CHAN_STATE_READ || error == EWOULDBLOCK) { 834 SUME_UNLOCK(adapter); 835 device_printf(adapter->dev, "wait error: %d\n", error); 836 return (EWOULDBLOCK); 837 } 838 839 bus_dmamap_sync(recv->ch_tag, recv->ch_map, 840 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 841 842 /* 843 * Read reply data and validate address and tag. 844 * Note: we do access the send side without lock but the state 845 * machine does prevent the data from changing. 846 */ 847 data = (struct nf_regop_data *) (recv->buf_addr + 848 sizeof(struct nf_bb_desc)); 849 850 if (le32toh(data->rtag) != send->rtag) 851 device_printf(adapter->dev, "rtag error: 0x%08x 0x%08x\n", 852 le32toh(data->rtag), send->rtag); 853 854 sifr->val = le32toh(data->val); 855 recv->state = SUME_RIFFA_CHAN_STATE_IDLE; 856 857 /* We are done. */ 858 send->state = SUME_RIFFA_CHAN_STATE_IDLE; 859 860 SUME_UNLOCK(adapter); 861 862 return (0); 863 } 864 865 /* Read value from a module register and return it to a sume_ifreq. */ 866 static int 867 get_modreg_value(struct nf_priv *nf_priv, struct sume_ifreq *sifr) 868 { 869 int error; 870 871 error = sume_module_reg_write(nf_priv, sifr, SUME_MR_READ); 872 if (!error) 873 error = sume_module_reg_read(nf_priv, sifr); 874 875 return (error); 876 } 877 878 static int 879 sume_if_ioctl(struct ifnet *ifp, unsigned long cmd, caddr_t data) 880 { 881 struct ifreq *ifr = (struct ifreq *) data; 882 struct nf_priv *nf_priv = ifp->if_softc; 883 struct sume_ifreq sifr; 884 int error = 0; 885 886 switch (cmd) { 887 case SIOCGIFMEDIA: 888 case SIOCGIFXMEDIA: 889 error = ifmedia_ioctl(ifp, ifr, &nf_priv->media, cmd); 890 break; 891 892 case SUME_IOCTL_CMD_WRITE_REG: 893 error = copyin(ifr_data_get_ptr(ifr), &sifr, sizeof(sifr)); 894 if (error) { 895 error = EINVAL; 896 break; 897 } 898 error = sume_module_reg_write(nf_priv, &sifr, SUME_MR_WRITE); 899 break; 900 901 case SUME_IOCTL_CMD_READ_REG: 902 error = copyin(ifr_data_get_ptr(ifr), &sifr, sizeof(sifr)); 903 if (error) { 904 error = EINVAL; 905 break; 906 } 907 908 error = get_modreg_value(nf_priv, &sifr); 909 if (error) 910 break; 911 912 error = copyout(&sifr, ifr_data_get_ptr(ifr), sizeof(sifr)); 913 if (error) 914 error = EINVAL; 915 916 break; 917 918 case SIOCSIFFLAGS: 919 /* Silence tcpdump 'promisc mode not supported' warning. */ 920 if (ifp->if_flags & IFF_PROMISC) 921 break; 922 923 default: 924 error = ether_ioctl(ifp, cmd, data); 925 break; 926 } 927 928 return (error); 929 } 930 931 static int 932 sume_media_change(struct ifnet *ifp) 933 { 934 struct nf_priv *nf_priv = ifp->if_softc; 935 struct ifmedia *ifm = &nf_priv->media; 936 937 if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER) 938 return (EINVAL); 939 940 if (IFM_SUBTYPE(ifm->ifm_media) == IFM_10G_SR) 941 ifp->if_baudrate = ifmedia_baudrate(IFM_ETHER | IFM_10G_SR); 942 else 943 ifp->if_baudrate = ifmedia_baudrate(ifm->ifm_media); 944 945 return (0); 946 } 947 948 static void 949 sume_update_link_status(struct ifnet *ifp) 950 { 951 struct nf_priv *nf_priv = ifp->if_softc; 952 struct sume_adapter *adapter = nf_priv->adapter; 953 struct sume_ifreq sifr; 954 int link_status; 955 956 sifr.addr = SUME_STATUS_ADDR(nf_priv->port); 957 sifr.val = 0; 958 959 if (get_modreg_value(nf_priv, &sifr)) 960 return; 961 962 link_status = SUME_LINK_STATUS(sifr.val); 963 964 if (!link_status && nf_priv->link_up) { 965 if_link_state_change(ifp, LINK_STATE_DOWN); 966 nf_priv->link_up = 0; 967 if (adapter->sume_debug) 968 device_printf(adapter->dev, "port %d link state " 969 "changed to DOWN\n", nf_priv->unit); 970 } else if (link_status && !nf_priv->link_up) { 971 nf_priv->link_up = 1; 972 if_link_state_change(ifp, LINK_STATE_UP); 973 if (adapter->sume_debug) 974 device_printf(adapter->dev, "port %d link state " 975 "changed to UP\n", nf_priv->unit); 976 } 977 } 978 979 static void 980 sume_media_status(struct ifnet *ifp, struct ifmediareq *ifmr) 981 { 982 struct nf_priv *nf_priv = ifp->if_softc; 983 struct ifmedia *ifm = &nf_priv->media; 984 985 if (ifm->ifm_cur->ifm_media == (IFM_ETHER | IFM_10G_SR) && 986 (ifp->if_flags & IFF_UP)) 987 ifmr->ifm_active = IFM_ETHER | IFM_10G_SR; 988 else 989 ifmr->ifm_active = ifm->ifm_cur->ifm_media; 990 991 ifmr->ifm_status |= IFM_AVALID; 992 993 sume_update_link_status(ifp); 994 995 if (nf_priv->link_up) 996 ifmr->ifm_status |= IFM_ACTIVE; 997 } 998 999 /* 1000 * Packet to transmit. We take the packet data from the mbuf and copy it to the 1001 * bouncebuffer address buf_addr+3*sizeof(uint32_t)+16. The 16 bytes before the 1002 * packet data are for metadata: sport/dport (depending on our source 1003 * interface), packet length and magic 0xcafe. We tell the SUME about the 1004 * transfer, fill the first 3*sizeof(uint32_t) bytes of the bouncebuffer with 1005 * the information about the start and length of the packet and trigger the 1006 * transaction. 1007 */ 1008 static int 1009 sume_if_start_locked(struct ifnet *ifp) 1010 { 1011 struct mbuf *m; 1012 struct nf_priv *nf_priv = ifp->if_softc; 1013 struct sume_adapter *adapter = nf_priv->adapter; 1014 struct riffa_chnl_dir *send = adapter->send[SUME_RIFFA_CHANNEL_DATA]; 1015 uint8_t *outbuf; 1016 struct nf_metadata *mdata; 1017 int plen = SUME_MIN_PKT_SIZE; 1018 1019 KASSERT(mtx_owned(&adapter->lock), ("SUME lock not owned")); 1020 KASSERT(send->state == SUME_RIFFA_CHAN_STATE_IDLE, 1021 ("SUME not in IDLE state")); 1022 1023 IFQ_DEQUEUE(&ifp->if_snd, m); 1024 if (m == NULL) 1025 return (EINVAL); 1026 1027 /* Packets large enough do not need to be padded */ 1028 if (m->m_pkthdr.len > SUME_MIN_PKT_SIZE) 1029 plen = m->m_pkthdr.len; 1030 1031 if (adapter->sume_debug) 1032 device_printf(adapter->dev, "sending %d bytes to %s%d\n", plen, 1033 SUME_ETH_DEVICE_NAME, nf_priv->unit); 1034 1035 outbuf = (uint8_t *) send->buf_addr + sizeof(struct nf_bb_desc); 1036 mdata = (struct nf_metadata *) outbuf; 1037 1038 /* Clear the recovery flag. */ 1039 send->recovery = 0; 1040 1041 /* Make sure we fit with the 16 bytes nf_metadata. */ 1042 if (m->m_pkthdr.len + sizeof(struct nf_metadata) > 1043 adapter->sg_buf_size) { 1044 device_printf(adapter->dev, "packet too big for bounce buffer " 1045 "(%d)\n", m->m_pkthdr.len); 1046 m_freem(m); 1047 nf_priv->stats.tx_dropped++; 1048 return (ENOMEM); 1049 } 1050 1051 bus_dmamap_sync(send->ch_tag, send->ch_map, 1052 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 1053 1054 /* Zero out the padded data */ 1055 if (m->m_pkthdr.len < SUME_MIN_PKT_SIZE) 1056 bzero(outbuf + sizeof(struct nf_metadata), SUME_MIN_PKT_SIZE); 1057 /* Skip the first 16 bytes for the metadata. */ 1058 m_copydata(m, 0, m->m_pkthdr.len, outbuf + sizeof(struct nf_metadata)); 1059 send->len = (sizeof(struct nf_metadata) + plen + 3) / 4; 1060 1061 /* Fill in the metadata: CPU(DMA) ports are odd, MAC ports are even. */ 1062 mdata->sport = htole16(1 << (nf_priv->port * 2 + 1)); 1063 mdata->dport = htole16(1 << (nf_priv->port * 2)); 1064 mdata->plen = htole16(plen); 1065 mdata->magic = htole16(SUME_RIFFA_MAGIC); 1066 mdata->t1 = htole32(0); 1067 mdata->t2 = htole32(0); 1068 1069 /* Let the FPGA know about the transfer. */ 1070 write_reg(adapter, RIFFA_CHNL_REG(SUME_RIFFA_CHANNEL_DATA, 1071 RIFFA_RX_OFFLAST_REG_OFF), SUME_OFFLAST); 1072 write_reg(adapter, RIFFA_CHNL_REG(SUME_RIFFA_CHANNEL_DATA, 1073 RIFFA_RX_LEN_REG_OFF), send->len); 1074 1075 /* Fill the bouncebuf "descriptor". */ 1076 sume_fill_bb_desc(adapter, send, SUME_RIFFA_LEN(send->len)); 1077 1078 /* Update the state before intiating the DMA to avoid races. */ 1079 send->state = SUME_RIFFA_CHAN_STATE_READY; 1080 1081 /* DMA. */ 1082 write_reg(adapter, RIFFA_CHNL_REG(SUME_RIFFA_CHANNEL_DATA, 1083 RIFFA_RX_SG_ADDR_LO_REG_OFF), 1084 SUME_RIFFA_LO_ADDR(send->buf_hw_addr)); 1085 write_reg(adapter, RIFFA_CHNL_REG(SUME_RIFFA_CHANNEL_DATA, 1086 RIFFA_RX_SG_ADDR_HI_REG_OFF), 1087 SUME_RIFFA_HI_ADDR(send->buf_hw_addr)); 1088 write_reg(adapter, RIFFA_CHNL_REG(SUME_RIFFA_CHANNEL_DATA, 1089 RIFFA_RX_SG_LEN_REG_OFF), 4 * send->num_sg); 1090 1091 bus_dmamap_sync(send->ch_tag, send->ch_map, 1092 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 1093 1094 nf_priv->stats.tx_packets++; 1095 nf_priv->stats.tx_bytes += plen; 1096 1097 /* We can free as long as we use the bounce buffer. */ 1098 m_freem(m); 1099 1100 adapter->last_ifc = nf_priv->port; 1101 1102 /* Reset watchdog counter. */ 1103 adapter->wd_counter = 0; 1104 1105 return (0); 1106 } 1107 1108 static void 1109 sume_if_start(struct ifnet *ifp) 1110 { 1111 struct nf_priv *nf_priv = ifp->if_softc; 1112 struct sume_adapter *adapter = nf_priv->adapter; 1113 1114 if (!adapter->running || !(ifp->if_flags & IFF_UP)) 1115 return; 1116 1117 SUME_LOCK(adapter); 1118 if (adapter->send[SUME_RIFFA_CHANNEL_DATA]->state == 1119 SUME_RIFFA_CHAN_STATE_IDLE) 1120 sume_if_start_locked(ifp); 1121 SUME_UNLOCK(adapter); 1122 } 1123 1124 /* 1125 * We call this function at the end of every TX transaction to check for 1126 * remaining packets in the TX queues for every UP interface. 1127 */ 1128 static void 1129 check_tx_queues(struct sume_adapter *adapter) 1130 { 1131 int i, last_ifc; 1132 1133 KASSERT(mtx_owned(&adapter->lock), ("SUME lock not owned")); 1134 1135 last_ifc = adapter->last_ifc; 1136 1137 /* Check all interfaces */ 1138 for (i = last_ifc + 1; i < last_ifc + SUME_NPORTS + 1; i++) { 1139 struct ifnet *ifp = adapter->ifp[i % SUME_NPORTS]; 1140 1141 if (!(ifp->if_flags & IFF_UP)) 1142 continue; 1143 1144 if (!sume_if_start_locked(ifp)) 1145 break; 1146 } 1147 } 1148 1149 static int 1150 sume_ifp_alloc(struct sume_adapter *adapter, uint32_t port) 1151 { 1152 struct ifnet *ifp; 1153 struct nf_priv *nf_priv = malloc(sizeof(struct nf_priv), M_SUME, 1154 M_ZERO | M_WAITOK); 1155 1156 ifp = if_alloc(IFT_ETHER); 1157 if (ifp == NULL) { 1158 device_printf(adapter->dev, "cannot allocate ifnet\n"); 1159 return (ENOMEM); 1160 } 1161 1162 adapter->ifp[port] = ifp; 1163 ifp->if_softc = nf_priv; 1164 1165 nf_priv->adapter = adapter; 1166 nf_priv->unit = alloc_unr(unr); 1167 nf_priv->port = port; 1168 nf_priv->link_up = 0; 1169 1170 if_initname(ifp, SUME_ETH_DEVICE_NAME, nf_priv->unit); 1171 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 1172 1173 ifp->if_init = sume_if_init; 1174 ifp->if_start = sume_if_start; 1175 ifp->if_ioctl = sume_if_ioctl; 1176 1177 uint8_t hw_addr[ETHER_ADDR_LEN] = DEFAULT_ETHER_ADDRESS; 1178 hw_addr[ETHER_ADDR_LEN-1] = nf_priv->unit; 1179 ether_ifattach(ifp, hw_addr); 1180 1181 ifmedia_init(&nf_priv->media, IFM_IMASK, sume_media_change, 1182 sume_media_status); 1183 ifmedia_add(&nf_priv->media, IFM_ETHER | IFM_10G_SR, 0, NULL); 1184 ifmedia_set(&nf_priv->media, IFM_ETHER | IFM_10G_SR); 1185 1186 ifp->if_drv_flags |= IFF_DRV_RUNNING; 1187 1188 return (0); 1189 } 1190 1191 static void 1192 callback_dma(void *arg, bus_dma_segment_t *segs, int nseg, int err) 1193 { 1194 if (err) 1195 return; 1196 1197 KASSERT(nseg == 1, ("%d segments returned!", nseg)); 1198 1199 *(bus_addr_t *) arg = segs[0].ds_addr; 1200 } 1201 1202 static int 1203 sume_probe_riffa_buffer(const struct sume_adapter *adapter, 1204 struct riffa_chnl_dir ***p, const char *dir) 1205 { 1206 struct riffa_chnl_dir **rp; 1207 bus_addr_t hw_addr; 1208 int error, ch; 1209 device_t dev = adapter->dev; 1210 1211 error = ENOMEM; 1212 *p = malloc(SUME_RIFFA_CHANNELS * sizeof(struct riffa_chnl_dir *), 1213 M_SUME, M_ZERO | M_WAITOK); 1214 if (*p == NULL) { 1215 device_printf(dev, "malloc(%s) failed.\n", dir); 1216 return (error); 1217 } 1218 1219 rp = *p; 1220 /* Allocate the chnl_dir structs themselves. */ 1221 for (ch = 0; ch < SUME_RIFFA_CHANNELS; ch++) { 1222 /* One direction. */ 1223 rp[ch] = malloc(sizeof(struct riffa_chnl_dir), M_SUME, 1224 M_ZERO | M_WAITOK); 1225 if (rp[ch] == NULL) { 1226 device_printf(dev, "malloc(%s[%d]) riffa_chnl_dir " 1227 "failed.\n", dir, ch); 1228 return (error); 1229 } 1230 1231 int err = bus_dma_tag_create(bus_get_dma_tag(dev), 1232 4, 0, 1233 BUS_SPACE_MAXADDR, 1234 BUS_SPACE_MAXADDR, 1235 NULL, NULL, 1236 adapter->sg_buf_size, 1237 1, 1238 adapter->sg_buf_size, 1239 0, 1240 NULL, 1241 NULL, 1242 &rp[ch]->ch_tag); 1243 1244 if (err) { 1245 device_printf(dev, "bus_dma_tag_create(%s[%d]) " 1246 "failed.\n", dir, ch); 1247 return (err); 1248 } 1249 1250 err = bus_dmamem_alloc(rp[ch]->ch_tag, (void **) 1251 &rp[ch]->buf_addr, BUS_DMA_WAITOK | BUS_DMA_COHERENT | 1252 BUS_DMA_ZERO, &rp[ch]->ch_map); 1253 if (err) { 1254 device_printf(dev, "bus_dmamem_alloc(%s[%d]) failed.\n", 1255 dir, ch); 1256 return (err); 1257 } 1258 1259 bzero(rp[ch]->buf_addr, adapter->sg_buf_size); 1260 1261 err = bus_dmamap_load(rp[ch]->ch_tag, rp[ch]->ch_map, 1262 rp[ch]->buf_addr, adapter->sg_buf_size, callback_dma, 1263 &hw_addr, BUS_DMA_NOWAIT); 1264 if (err) { 1265 device_printf(dev, "bus_dmamap_load(%s[%d]) failed.\n", 1266 dir, ch); 1267 return (err); 1268 } 1269 rp[ch]->buf_hw_addr = hw_addr; 1270 rp[ch]->num_sg = 1; 1271 rp[ch]->state = SUME_RIFFA_CHAN_STATE_IDLE; 1272 1273 rp[ch]->rtag = SUME_INIT_RTAG; 1274 } 1275 1276 return (0); 1277 } 1278 1279 static int 1280 sume_probe_riffa_buffers(struct sume_adapter *adapter) 1281 { 1282 int error; 1283 1284 error = sume_probe_riffa_buffer(adapter, &adapter->recv, "recv"); 1285 if (error) 1286 return (error); 1287 1288 error = sume_probe_riffa_buffer(adapter, &adapter->send, "send"); 1289 1290 return (error); 1291 } 1292 1293 static void 1294 sume_sysctl_init(struct sume_adapter *adapter) 1295 { 1296 device_t dev = adapter->dev; 1297 struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(dev); 1298 struct sysctl_oid *tree = device_get_sysctl_tree(dev); 1299 struct sysctl_oid_list *child = SYSCTL_CHILDREN(tree); 1300 struct sysctl_oid *tmp_tree; 1301 char namebuf[MAX_IFC_NAME_LEN]; 1302 int i; 1303 1304 tree = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, "sume", CTLFLAG_RW, 1305 0, "SUME top-level tree"); 1306 if (tree == NULL) { 1307 device_printf(dev, "SYSCTL_ADD_NODE failed.\n"); 1308 return; 1309 } 1310 SYSCTL_ADD_INT(ctx, child, OID_AUTO, "debug", CTLFLAG_RW, 1311 &adapter->sume_debug, 0, "debug int leaf"); 1312 1313 /* total RX error stats */ 1314 SYSCTL_ADD_U64(ctx, child, OID_AUTO, "rx_epkts", 1315 CTLFLAG_RD, &adapter->packets_err, 0, "rx errors"); 1316 SYSCTL_ADD_U64(ctx, child, OID_AUTO, "rx_ebytes", 1317 CTLFLAG_RD, &adapter->bytes_err, 0, "rx error bytes"); 1318 1319 for (i = SUME_NPORTS - 1; i >= 0; i--) { 1320 struct ifnet *ifp = adapter->ifp[i]; 1321 if (ifp == NULL) 1322 continue; 1323 1324 struct nf_priv *nf_priv = ifp->if_softc; 1325 1326 snprintf(namebuf, MAX_IFC_NAME_LEN, "%s%d", 1327 SUME_ETH_DEVICE_NAME, nf_priv->unit); 1328 tmp_tree = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf, 1329 CTLFLAG_RW, 0, "SUME ifc tree"); 1330 if (tmp_tree == NULL) { 1331 device_printf(dev, "SYSCTL_ADD_NODE failed.\n"); 1332 return; 1333 } 1334 1335 /* Packets dropped by down interface. */ 1336 SYSCTL_ADD_U64(ctx, SYSCTL_CHILDREN(tmp_tree), OID_AUTO, 1337 "ifc_down_bytes", CTLFLAG_RD, 1338 &nf_priv->stats.ifc_down_bytes, 0, "ifc_down bytes"); 1339 SYSCTL_ADD_U64(ctx, SYSCTL_CHILDREN(tmp_tree), OID_AUTO, 1340 "ifc_down_packets", CTLFLAG_RD, 1341 &nf_priv->stats.ifc_down_packets, 0, "ifc_down packets"); 1342 1343 /* HW RX stats */ 1344 SYSCTL_ADD_U64(ctx, SYSCTL_CHILDREN(tmp_tree), OID_AUTO, 1345 "hw_rx_packets", CTLFLAG_RD, &nf_priv->stats.hw_rx_packets, 1346 0, "hw_rx packets"); 1347 1348 /* HW TX stats */ 1349 SYSCTL_ADD_U64(ctx, SYSCTL_CHILDREN(tmp_tree), OID_AUTO, 1350 "hw_tx_packets", CTLFLAG_RD, &nf_priv->stats.hw_tx_packets, 1351 0, "hw_tx packets"); 1352 1353 /* RX stats */ 1354 SYSCTL_ADD_U64(ctx, SYSCTL_CHILDREN(tmp_tree), OID_AUTO, 1355 "rx_bytes", CTLFLAG_RD, &nf_priv->stats.rx_bytes, 0, 1356 "rx bytes"); 1357 SYSCTL_ADD_U64(ctx, SYSCTL_CHILDREN(tmp_tree), OID_AUTO, 1358 "rx_dropped", CTLFLAG_RD, &nf_priv->stats.rx_dropped, 0, 1359 "rx dropped"); 1360 SYSCTL_ADD_U64(ctx, SYSCTL_CHILDREN(tmp_tree), OID_AUTO, 1361 "rx_packets", CTLFLAG_RD, &nf_priv->stats.rx_packets, 0, 1362 "rx packets"); 1363 1364 /* TX stats */ 1365 SYSCTL_ADD_U64(ctx, SYSCTL_CHILDREN(tmp_tree), OID_AUTO, 1366 "tx_bytes", CTLFLAG_RD, &nf_priv->stats.tx_bytes, 0, 1367 "tx bytes"); 1368 SYSCTL_ADD_U64(ctx, SYSCTL_CHILDREN(tmp_tree), OID_AUTO, 1369 "tx_dropped", CTLFLAG_RD, &nf_priv->stats.tx_dropped, 0, 1370 "tx dropped"); 1371 SYSCTL_ADD_U64(ctx, SYSCTL_CHILDREN(tmp_tree), OID_AUTO, 1372 "tx_packets", CTLFLAG_RD, &nf_priv->stats.tx_packets, 0, 1373 "tx packets"); 1374 } 1375 } 1376 1377 static void 1378 sume_local_timer(void *arg) 1379 { 1380 struct sume_adapter *adapter = arg; 1381 1382 if (!adapter->running) 1383 return; 1384 1385 taskqueue_enqueue(adapter->tq, &adapter->stat_task); 1386 1387 SUME_LOCK(adapter); 1388 if (adapter->send[SUME_RIFFA_CHANNEL_DATA]->state != 1389 SUME_RIFFA_CHAN_STATE_IDLE && ++adapter->wd_counter >= 3) { 1390 /* Resetting interfaces if stuck for 3 seconds. */ 1391 device_printf(adapter->dev, "TX stuck, resetting adapter.\n"); 1392 read_reg(adapter, RIFFA_INFO_REG_OFF); 1393 1394 adapter->send[SUME_RIFFA_CHANNEL_DATA]->state = 1395 SUME_RIFFA_CHAN_STATE_IDLE; 1396 adapter->wd_counter = 0; 1397 1398 check_tx_queues(adapter); 1399 } 1400 SUME_UNLOCK(adapter); 1401 1402 callout_reset(&adapter->timer, 1 * hz, sume_local_timer, adapter); 1403 } 1404 1405 static void 1406 sume_get_stats(void *context, int pending) 1407 { 1408 struct sume_adapter *adapter = context; 1409 int i; 1410 1411 for (i = 0; i < SUME_NPORTS; i++) { 1412 struct ifnet *ifp = adapter->ifp[i]; 1413 1414 if (ifp->if_flags & IFF_UP) { 1415 struct nf_priv *nf_priv = ifp->if_softc; 1416 struct sume_ifreq sifr; 1417 1418 sume_update_link_status(ifp); 1419 1420 /* Get RX counter. */ 1421 sifr.addr = SUME_STAT_RX_ADDR(nf_priv->port); 1422 sifr.val = 0; 1423 1424 if (!get_modreg_value(nf_priv, &sifr)) 1425 nf_priv->stats.hw_rx_packets += sifr.val; 1426 1427 /* Get TX counter. */ 1428 sifr.addr = SUME_STAT_TX_ADDR(nf_priv->port); 1429 sifr.val = 0; 1430 1431 if (!get_modreg_value(nf_priv, &sifr)) 1432 nf_priv->stats.hw_tx_packets += sifr.val; 1433 } 1434 } 1435 } 1436 1437 static int 1438 sume_attach(device_t dev) 1439 { 1440 struct sume_adapter *adapter = device_get_softc(dev); 1441 adapter->dev = dev; 1442 int error, i; 1443 1444 mtx_init(&adapter->lock, "Global lock", NULL, MTX_DEF); 1445 1446 adapter->running = 0; 1447 1448 /* OK finish up RIFFA. */ 1449 error = sume_probe_riffa_pci(adapter); 1450 if (error != 0) 1451 goto error; 1452 1453 error = sume_probe_riffa_buffers(adapter); 1454 if (error != 0) 1455 goto error; 1456 1457 /* Now do the network interfaces. */ 1458 for (i = 0; i < SUME_NPORTS; i++) { 1459 error = sume_ifp_alloc(adapter, i); 1460 if (error != 0) 1461 goto error; 1462 } 1463 1464 /* Register stats and register sysctls. */ 1465 sume_sysctl_init(adapter); 1466 1467 /* Reset the HW. */ 1468 read_reg(adapter, RIFFA_INFO_REG_OFF); 1469 1470 /* Ready to go, "enable" IRQ. */ 1471 adapter->running = 1; 1472 1473 callout_init(&adapter->timer, 1); 1474 TASK_INIT(&adapter->stat_task, 0, sume_get_stats, adapter); 1475 1476 adapter->tq = taskqueue_create("sume_stats", M_NOWAIT, 1477 taskqueue_thread_enqueue, &adapter->tq); 1478 taskqueue_start_threads(&adapter->tq, 1, PI_NET, "%s stattaskq", 1479 device_get_nameunit(adapter->dev)); 1480 1481 callout_reset(&adapter->timer, 1 * hz, sume_local_timer, adapter); 1482 1483 return (0); 1484 1485 error: 1486 sume_detach(dev); 1487 1488 return (error); 1489 } 1490 1491 static void 1492 sume_remove_riffa_buffer(const struct sume_adapter *adapter, 1493 struct riffa_chnl_dir **pp) 1494 { 1495 int ch; 1496 1497 for (ch = 0; ch < SUME_RIFFA_CHANNELS; ch++) { 1498 if (pp[ch] == NULL) 1499 continue; 1500 1501 if (pp[ch]->buf_hw_addr != 0) { 1502 bus_dmamem_free(pp[ch]->ch_tag, pp[ch]->buf_addr, 1503 pp[ch]->ch_map); 1504 pp[ch]->buf_hw_addr = 0; 1505 } 1506 1507 free(pp[ch], M_SUME); 1508 } 1509 } 1510 1511 static void 1512 sume_remove_riffa_buffers(struct sume_adapter *adapter) 1513 { 1514 if (adapter->send != NULL) { 1515 sume_remove_riffa_buffer(adapter, adapter->send); 1516 free(adapter->send, M_SUME); 1517 adapter->send = NULL; 1518 } 1519 if (adapter->recv != NULL) { 1520 sume_remove_riffa_buffer(adapter, adapter->recv); 1521 free(adapter->recv, M_SUME); 1522 adapter->recv = NULL; 1523 } 1524 } 1525 1526 static int 1527 sume_detach(device_t dev) 1528 { 1529 struct sume_adapter *adapter = device_get_softc(dev); 1530 int i; 1531 struct nf_priv *nf_priv; 1532 1533 KASSERT(mtx_initialized(&adapter->lock), ("SUME mutex not " 1534 "initialized")); 1535 adapter->running = 0; 1536 1537 /* Drain the stats callout and task queue. */ 1538 callout_drain(&adapter->timer); 1539 1540 if (adapter->tq) { 1541 taskqueue_drain(adapter->tq, &adapter->stat_task); 1542 taskqueue_free(adapter->tq); 1543 } 1544 1545 for (i = 0; i < SUME_NPORTS; i++) { 1546 struct ifnet *ifp = adapter->ifp[i]; 1547 if (ifp == NULL) 1548 continue; 1549 1550 ifp->if_drv_flags &= ~IFF_DRV_RUNNING; 1551 nf_priv = ifp->if_softc; 1552 1553 if (ifp->if_flags & IFF_UP) 1554 if_down(ifp); 1555 ifmedia_removeall(&nf_priv->media); 1556 free_unr(unr, nf_priv->unit); 1557 1558 ifp->if_flags &= ~IFF_UP; 1559 ether_ifdetach(ifp); 1560 if_free(ifp); 1561 1562 free(nf_priv, M_SUME); 1563 } 1564 1565 sume_remove_riffa_buffers(adapter); 1566 1567 if (adapter->irq.tag) 1568 bus_teardown_intr(dev, adapter->irq.res, adapter->irq.tag); 1569 if (adapter->irq.res) 1570 bus_release_resource(dev, SYS_RES_IRQ, adapter->irq.rid, 1571 adapter->irq.res); 1572 1573 pci_release_msi(dev); 1574 1575 if (adapter->bar0_addr) 1576 bus_release_resource(dev, SYS_RES_MEMORY, adapter->rid, 1577 adapter->bar0_addr); 1578 1579 mtx_destroy(&adapter->lock); 1580 1581 return (0); 1582 } 1583 1584 static int 1585 mod_event(module_t mod, int cmd, void *arg) 1586 { 1587 switch (cmd) { 1588 case MOD_LOAD: 1589 unr = new_unrhdr(0, INT_MAX, NULL); 1590 break; 1591 1592 case MOD_UNLOAD: 1593 delete_unrhdr(unr); 1594 break; 1595 } 1596 1597 return (0); 1598 } 1599 1600 DRIVER_MODULE(sume, pci, sume_driver, mod_event, NULL); 1601 MODULE_VERSION(sume, 1); 1602