1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause 3 * 4 * Copyright (c) 2021 Microsoft Corp. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 14 * 2. Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in the 16 * documentation and/or other materials provided with the distribution. 17 * 18 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 19 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 20 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 21 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 22 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 23 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 24 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 25 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 26 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 27 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 28 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29 */ 30 31 #include <sys/param.h> 32 #include <sys/systm.h> 33 #include <sys/bus.h> 34 #include <sys/kernel.h> 35 #include <sys/kthread.h> 36 #include <sys/malloc.h> 37 #include <sys/mbuf.h> 38 #include <sys/smp.h> 39 #include <sys/socket.h> 40 #include <sys/sockio.h> 41 #include <sys/time.h> 42 #include <sys/eventhandler.h> 43 44 #include <machine/bus.h> 45 #include <machine/resource.h> 46 #include <machine/in_cksum.h> 47 48 #include <net/if.h> 49 #include <net/if_var.h> 50 #include <net/if_types.h> 51 #include <net/if_vlan_var.h> 52 #include <net/rss_config.h> 53 54 #include <netinet/in_systm.h> 55 #include <netinet/in.h> 56 #include <netinet/if_ether.h> 57 #include <netinet/ip.h> 58 #include <netinet/ip6.h> 59 #include <netinet/tcp.h> 60 #include <netinet/udp.h> 61 62 #include "mana.h" 63 #include "mana_sysctl.h" 64 65 static int mana_up(struct mana_port_context *apc); 66 static int mana_down(struct mana_port_context *apc); 67 68 extern unsigned int mana_tx_req_size; 69 extern unsigned int mana_rx_req_size; 70 extern unsigned int mana_rx_refill_threshold; 71 72 static int 73 mana_ifmedia_change(if_t ifp __unused) 74 { 75 return EOPNOTSUPP; 76 } 77 78 static void 79 mana_ifmedia_status(if_t ifp, struct ifmediareq *ifmr) 80 { 81 struct mana_port_context *apc = if_getsoftc(ifp); 82 83 if (!apc) { 84 if_printf(ifp, "Port not available\n"); 85 return; 86 } 87 88 MANA_APC_LOCK_LOCK(apc); 89 90 ifmr->ifm_status = IFM_AVALID; 91 ifmr->ifm_active = IFM_ETHER; 92 93 if (!apc->port_is_up) { 94 MANA_APC_LOCK_UNLOCK(apc); 95 mana_dbg(NULL, "Port %u link is down\n", apc->port_idx); 96 return; 97 } 98 99 ifmr->ifm_status |= IFM_ACTIVE; 100 ifmr->ifm_active |= IFM_100G_DR | IFM_FDX; 101 102 MANA_APC_LOCK_UNLOCK(apc); 103 } 104 105 static uint64_t 106 mana_get_counter(if_t ifp, ift_counter cnt) 107 { 108 struct mana_port_context *apc = if_getsoftc(ifp); 109 struct mana_port_stats *stats = &apc->port_stats; 110 111 switch (cnt) { 112 case IFCOUNTER_IPACKETS: 113 return (counter_u64_fetch(stats->rx_packets)); 114 case IFCOUNTER_OPACKETS: 115 return (counter_u64_fetch(stats->tx_packets)); 116 case IFCOUNTER_IBYTES: 117 return (counter_u64_fetch(stats->rx_bytes)); 118 case IFCOUNTER_OBYTES: 119 return (counter_u64_fetch(stats->tx_bytes)); 120 case IFCOUNTER_IQDROPS: 121 return (counter_u64_fetch(stats->rx_drops)); 122 case IFCOUNTER_OQDROPS: 123 return (counter_u64_fetch(stats->tx_drops)); 124 default: 125 return (if_get_counter_default(ifp, cnt)); 126 } 127 } 128 129 static void 130 mana_qflush(if_t ifp) 131 { 132 if_qflush(ifp); 133 } 134 135 int 136 mana_restart(struct mana_port_context *apc) 137 { 138 int rc = 0; 139 140 MANA_APC_LOCK_LOCK(apc); 141 if (apc->port_is_up) 142 mana_down(apc); 143 144 rc = mana_up(apc); 145 MANA_APC_LOCK_UNLOCK(apc); 146 147 return (rc); 148 } 149 150 static int 151 mana_ioctl(if_t ifp, u_long command, caddr_t data) 152 { 153 struct mana_port_context *apc = if_getsoftc(ifp); 154 struct ifrsskey *ifrk; 155 struct ifrsshash *ifrh; 156 struct ifreq *ifr; 157 uint16_t new_mtu; 158 int rc = 0, mask; 159 160 switch (command) { 161 case SIOCSIFMTU: 162 ifr = (struct ifreq *)data; 163 new_mtu = ifr->ifr_mtu; 164 if (if_getmtu(ifp) == new_mtu) 165 break; 166 if ((new_mtu > apc->max_mtu) || 167 (new_mtu < apc->min_mtu)) { 168 if_printf(ifp, "Invalid MTU. new_mtu: %d, " 169 "max allowed: %d, min allowed: %d\n", 170 new_mtu, apc->max_mtu, apc->min_mtu); 171 return EINVAL; 172 } 173 MANA_APC_LOCK_LOCK(apc); 174 if (apc->port_is_up) 175 mana_down(apc); 176 177 apc->frame_size = new_mtu + ETHER_HDR_LEN; 178 apc->mtu = new_mtu; 179 if_setmtu(ifp, new_mtu); 180 mana_dbg(NULL, "Set MTU to %d\n", new_mtu); 181 182 rc = mana_up(apc); 183 MANA_APC_LOCK_UNLOCK(apc); 184 break; 185 186 case SIOCSIFFLAGS: 187 if (if_getflags(ifp) & IFF_UP) { 188 if ((if_getdrvflags(ifp) & IFF_DRV_RUNNING) == 0) { 189 MANA_APC_LOCK_LOCK(apc); 190 if (!apc->port_is_up) 191 rc = mana_up(apc); 192 MANA_APC_LOCK_UNLOCK(apc); 193 } 194 } else { 195 if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) { 196 MANA_APC_LOCK_LOCK(apc); 197 if (apc->port_is_up) 198 mana_down(apc); 199 MANA_APC_LOCK_UNLOCK(apc); 200 } 201 } 202 break; 203 204 case SIOCSIFCAP: 205 MANA_APC_LOCK_LOCK(apc); 206 ifr = (struct ifreq *)data; 207 /* 208 * Fix up requested capabilities w/ supported capabilities, 209 * since the supported capabilities could have been changed. 210 */ 211 mask = (ifr->ifr_reqcap & if_getcapabilities(ifp)) ^ 212 if_getcapenable(ifp); 213 214 if (mask & IFCAP_TXCSUM) { 215 if_togglecapenable(ifp, IFCAP_TXCSUM); 216 if_togglehwassist(ifp, (CSUM_TCP | CSUM_UDP | CSUM_IP)); 217 218 if ((IFCAP_TSO4 & if_getcapenable(ifp)) && 219 !(IFCAP_TXCSUM & if_getcapenable(ifp))) { 220 mask &= ~IFCAP_TSO4; 221 if_setcapenablebit(ifp, 0, IFCAP_TSO4); 222 if_sethwassistbits(ifp, 0, CSUM_IP_TSO); 223 mana_warn(NULL, 224 "Also disabled tso4 due to -txcsum.\n"); 225 } 226 } 227 228 if (mask & IFCAP_TXCSUM_IPV6) { 229 if_togglecapenable(ifp, IFCAP_TXCSUM_IPV6); 230 if_togglehwassist(ifp, (CSUM_UDP_IPV6 | CSUM_TCP_IPV6)); 231 232 if ((IFCAP_TSO6 & if_getcapenable(ifp)) && 233 !(IFCAP_TXCSUM_IPV6 & if_getcapenable(ifp))) { 234 mask &= ~IFCAP_TSO6; 235 if_setcapenablebit(ifp, 0, IFCAP_TSO6); 236 if_sethwassistbits(ifp, 0, CSUM_IP6_TSO); 237 mana_warn(ifp, 238 "Also disabled tso6 due to -txcsum6.\n"); 239 } 240 } 241 242 if (mask & IFCAP_RXCSUM) 243 if_togglecapenable(ifp, IFCAP_RXCSUM); 244 /* We can't diff IPv6 packets from IPv4 packets on RX path. */ 245 if (mask & IFCAP_RXCSUM_IPV6) 246 if_togglecapenable(ifp, IFCAP_RXCSUM_IPV6); 247 248 if (mask & IFCAP_LRO) 249 if_togglecapenable(ifp, IFCAP_LRO); 250 251 if (mask & IFCAP_TSO4) { 252 if (!(IFCAP_TSO4 & if_getcapenable(ifp)) && 253 !(IFCAP_TXCSUM & if_getcapenable(ifp))) { 254 MANA_APC_LOCK_UNLOCK(apc); 255 if_printf(ifp, "Enable txcsum first.\n"); 256 rc = EAGAIN; 257 goto out; 258 } 259 if_togglecapenable(ifp, IFCAP_TSO4); 260 if_togglehwassist(ifp, CSUM_IP_TSO); 261 } 262 263 if (mask & IFCAP_TSO6) { 264 if (!(IFCAP_TSO6 & if_getcapenable(ifp)) && 265 !(IFCAP_TXCSUM_IPV6 & if_getcapenable(ifp))) { 266 MANA_APC_LOCK_UNLOCK(apc); 267 if_printf(ifp, "Enable txcsum6 first.\n"); 268 rc = EAGAIN; 269 goto out; 270 } 271 if_togglecapenable(ifp, IFCAP_TSO6); 272 if_togglehwassist(ifp, CSUM_IP6_TSO); 273 } 274 275 MANA_APC_LOCK_UNLOCK(apc); 276 out: 277 break; 278 279 case SIOCSIFMEDIA: 280 case SIOCGIFMEDIA: 281 case SIOCGIFXMEDIA: 282 ifr = (struct ifreq *)data; 283 rc = ifmedia_ioctl(ifp, ifr, &apc->media, command); 284 break; 285 286 case SIOCGIFRSSKEY: 287 ifrk = (struct ifrsskey *)data; 288 ifrk->ifrk_func = RSS_FUNC_TOEPLITZ; 289 ifrk->ifrk_keylen = MANA_HASH_KEY_SIZE; 290 memcpy(ifrk->ifrk_key, apc->hashkey, MANA_HASH_KEY_SIZE); 291 break; 292 293 case SIOCGIFRSSHASH: 294 ifrh = (struct ifrsshash *)data; 295 ifrh->ifrh_func = RSS_FUNC_TOEPLITZ; 296 ifrh->ifrh_types = 297 RSS_TYPE_TCP_IPV4 | 298 RSS_TYPE_UDP_IPV4 | 299 RSS_TYPE_TCP_IPV6 | 300 RSS_TYPE_UDP_IPV6; 301 break; 302 303 default: 304 rc = ether_ioctl(ifp, command, data); 305 break; 306 } 307 308 return (rc); 309 } 310 311 static inline void 312 mana_alloc_counters(counter_u64_t *begin, int size) 313 { 314 counter_u64_t *end = (counter_u64_t *)((char *)begin + size); 315 316 for (; begin < end; ++begin) 317 *begin = counter_u64_alloc(M_WAITOK); 318 } 319 320 static inline void 321 mana_free_counters(counter_u64_t *begin, int size) 322 { 323 counter_u64_t *end = (counter_u64_t *)((char *)begin + size); 324 325 for (; begin < end; ++begin) 326 counter_u64_free(*begin); 327 } 328 329 static bool 330 mana_can_tx(struct gdma_queue *wq) 331 { 332 return mana_gd_wq_avail_space(wq) >= MAX_TX_WQE_SIZE; 333 } 334 335 static inline int 336 mana_tx_map_mbuf(struct mana_port_context *apc, 337 struct mana_send_buf_info *tx_info, 338 struct mbuf **m_head, struct mana_tx_package *tp, 339 struct mana_stats *tx_stats) 340 { 341 struct gdma_dev *gd = apc->ac->gdma_dev; 342 bus_dma_segment_t segs[MAX_MBUF_FRAGS]; 343 struct mbuf *m = *m_head; 344 int err, nsegs, i; 345 346 err = bus_dmamap_load_mbuf_sg(apc->tx_buf_tag, tx_info->dma_map, 347 m, segs, &nsegs, BUS_DMA_NOWAIT); 348 if (err == EFBIG) { 349 struct mbuf *m_new; 350 351 counter_u64_add(tx_stats->collapse, 1); 352 m_new = m_collapse(m, M_NOWAIT, MAX_MBUF_FRAGS); 353 if (unlikely(m_new == NULL)) { 354 counter_u64_add(tx_stats->collapse_err, 1); 355 return ENOBUFS; 356 } else { 357 *m_head = m = m_new; 358 } 359 360 mana_warn(NULL, 361 "Too many segs in orig mbuf, m_collapse called\n"); 362 363 err = bus_dmamap_load_mbuf_sg(apc->tx_buf_tag, 364 tx_info->dma_map, m, segs, &nsegs, BUS_DMA_NOWAIT); 365 } 366 if (!err) { 367 for (i = 0; i < nsegs; i++) { 368 tp->wqe_req.sgl[i].address = segs[i].ds_addr; 369 tp->wqe_req.sgl[i].mem_key = gd->gpa_mkey; 370 tp->wqe_req.sgl[i].size = segs[i].ds_len; 371 } 372 tp->wqe_req.num_sge = nsegs; 373 374 tx_info->mbuf = *m_head; 375 376 bus_dmamap_sync(apc->tx_buf_tag, tx_info->dma_map, 377 BUS_DMASYNC_PREWRITE); 378 } 379 380 return err; 381 } 382 383 static inline void 384 mana_tx_unmap_mbuf(struct mana_port_context *apc, 385 struct mana_send_buf_info *tx_info) 386 { 387 bus_dmamap_sync(apc->tx_buf_tag, tx_info->dma_map, 388 BUS_DMASYNC_POSTWRITE); 389 bus_dmamap_unload(apc->tx_buf_tag, tx_info->dma_map); 390 if (tx_info->mbuf) { 391 m_freem(tx_info->mbuf); 392 tx_info->mbuf = NULL; 393 } 394 } 395 396 static inline int 397 mana_load_rx_mbuf(struct mana_port_context *apc, struct mana_rxq *rxq, 398 struct mana_recv_buf_oob *rx_oob, bool alloc_mbuf) 399 { 400 bus_dma_segment_t segs[1]; 401 struct mbuf *mbuf; 402 int nsegs, err; 403 uint32_t mlen; 404 405 if (alloc_mbuf) { 406 mbuf = m_getjcl(M_NOWAIT, MT_DATA, M_PKTHDR, rxq->datasize); 407 if (unlikely(mbuf == NULL)) 408 return ENOMEM; 409 410 mbuf->m_pkthdr.len = mbuf->m_len = rxq->datasize; 411 mlen = rxq->datasize; 412 } else { 413 if (rx_oob->mbuf) { 414 mbuf = rx_oob->mbuf; 415 mlen = rx_oob->mbuf->m_pkthdr.len; 416 } else { 417 return ENOMEM; 418 } 419 } 420 421 err = bus_dmamap_load_mbuf_sg(apc->rx_buf_tag, rx_oob->dma_map, 422 mbuf, segs, &nsegs, BUS_DMA_NOWAIT); 423 424 if (unlikely((err != 0) || (nsegs != 1))) { 425 mana_warn(NULL, "Failed to map mbuf, error: %d, " 426 "nsegs: %d\n", err, nsegs); 427 counter_u64_add(rxq->stats.dma_mapping_err, 1); 428 goto error; 429 } 430 431 bus_dmamap_sync(apc->rx_buf_tag, rx_oob->dma_map, 432 BUS_DMASYNC_PREREAD); 433 434 rx_oob->mbuf = mbuf; 435 rx_oob->num_sge = 1; 436 rx_oob->sgl[0].address = segs[0].ds_addr; 437 rx_oob->sgl[0].size = mlen; 438 rx_oob->sgl[0].mem_key = apc->ac->gdma_dev->gpa_mkey; 439 440 return 0; 441 442 error: 443 m_freem(mbuf); 444 return EFAULT; 445 } 446 447 static inline void 448 mana_unload_rx_mbuf(struct mana_port_context *apc, struct mana_rxq *rxq, 449 struct mana_recv_buf_oob *rx_oob, bool free_mbuf) 450 { 451 bus_dmamap_sync(apc->rx_buf_tag, rx_oob->dma_map, 452 BUS_DMASYNC_POSTREAD); 453 bus_dmamap_unload(apc->rx_buf_tag, rx_oob->dma_map); 454 455 if (free_mbuf && rx_oob->mbuf) { 456 m_freem(rx_oob->mbuf); 457 rx_oob->mbuf = NULL; 458 } 459 } 460 461 462 /* Use couple mbuf PH_loc spaces for l3 and l4 protocal type */ 463 #define MANA_L3_PROTO(_mbuf) ((_mbuf)->m_pkthdr.PH_loc.sixteen[0]) 464 #define MANA_L4_PROTO(_mbuf) ((_mbuf)->m_pkthdr.PH_loc.sixteen[1]) 465 466 #define MANA_TXQ_FULL (IFF_DRV_RUNNING | IFF_DRV_OACTIVE) 467 468 static void 469 mana_xmit(struct mana_txq *txq) 470 { 471 enum mana_tx_pkt_format pkt_fmt = MANA_SHORT_PKT_FMT; 472 struct mana_send_buf_info *tx_info; 473 if_t ndev = txq->ndev; 474 struct mbuf *mbuf; 475 struct mana_port_context *apc = if_getsoftc(ndev); 476 unsigned int tx_queue_size = apc->tx_queue_size; 477 struct mana_port_stats *port_stats = &apc->port_stats; 478 struct gdma_dev *gd = apc->ac->gdma_dev; 479 uint64_t packets, bytes; 480 uint16_t next_to_use; 481 struct mana_tx_package pkg = {}; 482 struct mana_stats *tx_stats; 483 struct gdma_queue *gdma_sq; 484 struct mana_cq *cq; 485 int err, len; 486 bool is_tso; 487 488 gdma_sq = txq->gdma_sq; 489 cq = &apc->tx_qp[txq->idx].tx_cq; 490 tx_stats = &txq->stats; 491 492 packets = 0; 493 bytes = 0; 494 next_to_use = txq->next_to_use; 495 496 while ((mbuf = drbr_peek(ndev, txq->txq_br)) != NULL) { 497 if (!apc->port_is_up || 498 (if_getdrvflags(ndev) & MANA_TXQ_FULL) != IFF_DRV_RUNNING) { 499 drbr_putback(ndev, txq->txq_br, mbuf); 500 break; 501 } 502 503 if (!mana_can_tx(gdma_sq)) { 504 /* SQ is full. Set the IFF_DRV_OACTIVE flag */ 505 if_setdrvflagbits(apc->ndev, IFF_DRV_OACTIVE, 0); 506 counter_u64_add(tx_stats->stop, 1); 507 uint64_t stops = counter_u64_fetch(tx_stats->stop); 508 uint64_t wakeups = counter_u64_fetch(tx_stats->wakeup); 509 #define MANA_TXQ_STOP_THRESHOLD 50 510 if (stops > MANA_TXQ_STOP_THRESHOLD && wakeups > 0 && 511 stops > wakeups && txq->alt_txq_idx == txq->idx) { 512 txq->alt_txq_idx = 513 (txq->idx + (stops / wakeups)) 514 % apc->num_queues; 515 counter_u64_add(tx_stats->alt_chg, 1); 516 } 517 518 drbr_putback(ndev, txq->txq_br, mbuf); 519 520 taskqueue_enqueue(cq->cleanup_tq, &cq->cleanup_task); 521 break; 522 } 523 524 tx_info = &txq->tx_buf_info[next_to_use]; 525 526 memset(&pkg, 0, sizeof(struct mana_tx_package)); 527 pkg.wqe_req.sgl = pkg.sgl_array; 528 529 err = mana_tx_map_mbuf(apc, tx_info, &mbuf, &pkg, tx_stats); 530 if (unlikely(err)) { 531 mana_dbg(NULL, 532 "Failed to map tx mbuf, err %d\n", err); 533 534 counter_u64_add(tx_stats->dma_mapping_err, 1); 535 536 /* The mbuf is still there. Free it */ 537 m_freem(mbuf); 538 /* Advance the drbr queue */ 539 drbr_advance(ndev, txq->txq_br); 540 continue; 541 } 542 543 pkg.tx_oob.s_oob.vcq_num = cq->gdma_id; 544 pkg.tx_oob.s_oob.vsq_frame = txq->vsq_frame; 545 546 if (txq->vp_offset > MANA_SHORT_VPORT_OFFSET_MAX) { 547 pkg.tx_oob.l_oob.long_vp_offset = txq->vp_offset; 548 pkt_fmt = MANA_LONG_PKT_FMT; 549 } else { 550 pkg.tx_oob.s_oob.short_vp_offset = txq->vp_offset; 551 } 552 553 pkg.tx_oob.s_oob.pkt_fmt = pkt_fmt; 554 555 if (pkt_fmt == MANA_SHORT_PKT_FMT) 556 pkg.wqe_req.inline_oob_size = sizeof(struct mana_tx_short_oob); 557 else 558 pkg.wqe_req.inline_oob_size = sizeof(struct mana_tx_oob); 559 560 pkg.wqe_req.inline_oob_data = &pkg.tx_oob; 561 pkg.wqe_req.flags = 0; 562 pkg.wqe_req.client_data_unit = 0; 563 564 is_tso = false; 565 if (mbuf->m_pkthdr.csum_flags & CSUM_TSO) { 566 is_tso = true; 567 568 if (MANA_L3_PROTO(mbuf) == ETHERTYPE_IP) 569 pkg.tx_oob.s_oob.is_outer_ipv4 = 1; 570 else 571 pkg.tx_oob.s_oob.is_outer_ipv6 = 1; 572 573 pkg.tx_oob.s_oob.comp_iphdr_csum = 1; 574 pkg.tx_oob.s_oob.comp_tcp_csum = 1; 575 pkg.tx_oob.s_oob.trans_off = mbuf->m_pkthdr.l3hlen; 576 577 pkg.wqe_req.client_data_unit = mbuf->m_pkthdr.tso_segsz; 578 pkg.wqe_req.flags = GDMA_WR_OOB_IN_SGL | GDMA_WR_PAD_BY_SGE0; 579 } else if (mbuf->m_pkthdr.csum_flags & 580 (CSUM_IP_UDP | CSUM_IP_TCP | CSUM_IP6_UDP | CSUM_IP6_TCP)) { 581 if (MANA_L3_PROTO(mbuf) == ETHERTYPE_IP) { 582 pkg.tx_oob.s_oob.is_outer_ipv4 = 1; 583 pkg.tx_oob.s_oob.comp_iphdr_csum = 1; 584 } else { 585 pkg.tx_oob.s_oob.is_outer_ipv6 = 1; 586 } 587 588 if (MANA_L4_PROTO(mbuf) == IPPROTO_TCP) { 589 pkg.tx_oob.s_oob.comp_tcp_csum = 1; 590 pkg.tx_oob.s_oob.trans_off = 591 mbuf->m_pkthdr.l3hlen; 592 } else { 593 pkg.tx_oob.s_oob.comp_udp_csum = 1; 594 } 595 } else if (mbuf->m_pkthdr.csum_flags & CSUM_IP) { 596 pkg.tx_oob.s_oob.is_outer_ipv4 = 1; 597 pkg.tx_oob.s_oob.comp_iphdr_csum = 1; 598 } else { 599 if (MANA_L3_PROTO(mbuf) == ETHERTYPE_IP) 600 pkg.tx_oob.s_oob.is_outer_ipv4 = 1; 601 else if (MANA_L3_PROTO(mbuf) == ETHERTYPE_IPV6) 602 pkg.tx_oob.s_oob.is_outer_ipv6 = 1; 603 } 604 605 len = mbuf->m_pkthdr.len; 606 607 err = mana_gd_post_work_request(gdma_sq, &pkg.wqe_req, 608 (struct gdma_posted_wqe_info *)&tx_info->wqe_inf); 609 if (unlikely(err)) { 610 /* Should not happen */ 611 if_printf(ndev, "Failed to post TX OOB: %d\n", err); 612 613 mana_tx_unmap_mbuf(apc, tx_info); 614 615 drbr_advance(ndev, txq->txq_br); 616 continue; 617 } 618 619 next_to_use = MANA_IDX_NEXT(next_to_use, tx_queue_size); 620 621 (void)atomic_inc_return(&txq->pending_sends); 622 623 drbr_advance(ndev, txq->txq_br); 624 625 mana_gd_wq_ring_doorbell(gd->gdma_context, gdma_sq); 626 627 packets++; 628 bytes += len; 629 630 if (is_tso) { 631 txq->tso_pkts++; 632 txq->tso_bytes += len; 633 } 634 } 635 636 counter_enter(); 637 counter_u64_add_protected(tx_stats->packets, packets); 638 counter_u64_add_protected(port_stats->tx_packets, packets); 639 counter_u64_add_protected(tx_stats->bytes, bytes); 640 counter_u64_add_protected(port_stats->tx_bytes, bytes); 641 counter_exit(); 642 643 txq->next_to_use = next_to_use; 644 } 645 646 static void 647 mana_xmit_taskfunc(void *arg, int pending) 648 { 649 struct mana_txq *txq = (struct mana_txq *)arg; 650 if_t ndev = txq->ndev; 651 struct mana_port_context *apc = if_getsoftc(ndev); 652 653 while (!drbr_empty(ndev, txq->txq_br) && apc->port_is_up && 654 (if_getdrvflags(ndev) & MANA_TXQ_FULL) == IFF_DRV_RUNNING) { 655 mtx_lock(&txq->txq_mtx); 656 mana_xmit(txq); 657 mtx_unlock(&txq->txq_mtx); 658 } 659 } 660 661 #define PULLUP_HDR(m, len) \ 662 do { \ 663 if (unlikely((m)->m_len < (len))) { \ 664 (m) = m_pullup((m), (len)); \ 665 if ((m) == NULL) \ 666 return (NULL); \ 667 } \ 668 } while (0) 669 670 /* 671 * If this function failed, the mbuf would be freed. 672 */ 673 static inline struct mbuf * 674 mana_tso_fixup(struct mbuf *mbuf) 675 { 676 struct ether_vlan_header *eh = mtod(mbuf, struct ether_vlan_header *); 677 struct tcphdr *th; 678 uint16_t etype; 679 int ehlen; 680 681 if (eh->evl_encap_proto == ntohs(ETHERTYPE_VLAN)) { 682 etype = ntohs(eh->evl_proto); 683 ehlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; 684 } else { 685 etype = ntohs(eh->evl_encap_proto); 686 ehlen = ETHER_HDR_LEN; 687 } 688 689 if (etype == ETHERTYPE_IP) { 690 struct ip *ip; 691 int iphlen; 692 693 PULLUP_HDR(mbuf, ehlen + sizeof(*ip)); 694 ip = mtodo(mbuf, ehlen); 695 iphlen = ip->ip_hl << 2; 696 mbuf->m_pkthdr.l3hlen = ehlen + iphlen; 697 698 PULLUP_HDR(mbuf, ehlen + iphlen + sizeof(*th)); 699 th = mtodo(mbuf, ehlen + iphlen); 700 701 ip->ip_len = 0; 702 ip->ip_sum = 0; 703 th->th_sum = in_pseudo(ip->ip_src.s_addr, 704 ip->ip_dst.s_addr, htons(IPPROTO_TCP)); 705 } else if (etype == ETHERTYPE_IPV6) { 706 struct ip6_hdr *ip6; 707 708 PULLUP_HDR(mbuf, ehlen + sizeof(*ip6) + sizeof(*th)); 709 ip6 = mtodo(mbuf, ehlen); 710 if (ip6->ip6_nxt != IPPROTO_TCP) { 711 /* Realy something wrong, just return */ 712 mana_dbg(NULL, "TSO mbuf not TCP, freed.\n"); 713 m_freem(mbuf); 714 return NULL; 715 } 716 mbuf->m_pkthdr.l3hlen = ehlen + sizeof(*ip6); 717 718 th = mtodo(mbuf, ehlen + sizeof(*ip6)); 719 720 ip6->ip6_plen = 0; 721 th->th_sum = in6_cksum_pseudo(ip6, 0, IPPROTO_TCP, 0); 722 } else { 723 /* CSUM_TSO is set but not IP protocol. */ 724 mana_warn(NULL, "TSO mbuf not right, freed.\n"); 725 m_freem(mbuf); 726 return NULL; 727 } 728 729 MANA_L3_PROTO(mbuf) = etype; 730 731 return (mbuf); 732 } 733 734 /* 735 * If this function failed, the mbuf would be freed. 736 */ 737 static inline struct mbuf * 738 mana_mbuf_csum_check(struct mbuf *mbuf) 739 { 740 struct ether_vlan_header *eh = mtod(mbuf, struct ether_vlan_header *); 741 struct mbuf *mbuf_next; 742 uint16_t etype; 743 int offset; 744 int ehlen; 745 746 if (eh->evl_encap_proto == ntohs(ETHERTYPE_VLAN)) { 747 etype = ntohs(eh->evl_proto); 748 ehlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; 749 } else { 750 etype = ntohs(eh->evl_encap_proto); 751 ehlen = ETHER_HDR_LEN; 752 } 753 754 mbuf_next = m_getptr(mbuf, ehlen, &offset); 755 756 MANA_L4_PROTO(mbuf) = 0; 757 if (etype == ETHERTYPE_IP) { 758 const struct ip *ip; 759 int iphlen; 760 761 ip = (struct ip *)(mtodo(mbuf_next, offset)); 762 iphlen = ip->ip_hl << 2; 763 mbuf->m_pkthdr.l3hlen = ehlen + iphlen; 764 765 MANA_L4_PROTO(mbuf) = ip->ip_p; 766 } else if (etype == ETHERTYPE_IPV6) { 767 const struct ip6_hdr *ip6; 768 769 ip6 = (struct ip6_hdr *)(mtodo(mbuf_next, offset)); 770 mbuf->m_pkthdr.l3hlen = ehlen + sizeof(*ip6); 771 772 MANA_L4_PROTO(mbuf) = ip6->ip6_nxt; 773 } else { 774 MANA_L4_PROTO(mbuf) = 0; 775 } 776 777 MANA_L3_PROTO(mbuf) = etype; 778 779 return (mbuf); 780 } 781 782 static int 783 mana_start_xmit(if_t ifp, struct mbuf *m) 784 { 785 struct mana_port_context *apc = if_getsoftc(ifp); 786 struct mana_txq *txq; 787 int is_drbr_empty; 788 uint16_t txq_id; 789 int err; 790 791 if (unlikely((!apc->port_is_up) || 792 (if_getdrvflags(ifp) & IFF_DRV_RUNNING) == 0)) 793 return ENODEV; 794 795 if (m->m_pkthdr.csum_flags & CSUM_TSO) { 796 m = mana_tso_fixup(m); 797 if (unlikely(m == NULL)) { 798 counter_enter(); 799 counter_u64_add_protected(apc->port_stats.tx_drops, 1); 800 counter_exit(); 801 return EIO; 802 } 803 } else { 804 m = mana_mbuf_csum_check(m); 805 if (unlikely(m == NULL)) { 806 counter_enter(); 807 counter_u64_add_protected(apc->port_stats.tx_drops, 1); 808 counter_exit(); 809 return EIO; 810 } 811 } 812 813 if (M_HASHTYPE_GET(m) != M_HASHTYPE_NONE) { 814 uint32_t hash = m->m_pkthdr.flowid; 815 txq_id = apc->indir_table[(hash) & MANA_INDIRECT_TABLE_MASK] % 816 apc->num_queues; 817 } else { 818 txq_id = m->m_pkthdr.flowid % apc->num_queues; 819 } 820 821 if (apc->enable_tx_altq) 822 txq_id = apc->tx_qp[txq_id].txq.alt_txq_idx; 823 824 txq = &apc->tx_qp[txq_id].txq; 825 826 is_drbr_empty = drbr_empty(ifp, txq->txq_br); 827 err = drbr_enqueue(ifp, txq->txq_br, m); 828 if (unlikely(err)) { 829 mana_warn(NULL, "txq %u failed to enqueue: %d\n", 830 txq_id, err); 831 taskqueue_enqueue(txq->enqueue_tq, &txq->enqueue_task); 832 return err; 833 } 834 835 if (is_drbr_empty && mtx_trylock(&txq->txq_mtx)) { 836 mana_xmit(txq); 837 mtx_unlock(&txq->txq_mtx); 838 } else { 839 taskqueue_enqueue(txq->enqueue_tq, &txq->enqueue_task); 840 } 841 842 return 0; 843 } 844 845 static void 846 mana_cleanup_port_context(struct mana_port_context *apc) 847 { 848 bus_dma_tag_destroy(apc->tx_buf_tag); 849 bus_dma_tag_destroy(apc->rx_buf_tag); 850 apc->rx_buf_tag = NULL; 851 852 free(apc->rxqs, M_DEVBUF); 853 apc->rxqs = NULL; 854 855 mana_free_counters((counter_u64_t *)&apc->port_stats, 856 sizeof(struct mana_port_stats)); 857 } 858 859 static int 860 mana_init_port_context(struct mana_port_context *apc) 861 { 862 device_t dev = apc->ac->gdma_dev->gdma_context->dev; 863 uint32_t tso_maxsize; 864 int err; 865 866 tso_maxsize = MANA_TSO_MAX_SZ; 867 868 /* Create DMA tag for tx bufs */ 869 err = bus_dma_tag_create(bus_get_dma_tag(dev), /* parent */ 870 1, 0, /* alignment, boundary */ 871 BUS_SPACE_MAXADDR, /* lowaddr */ 872 BUS_SPACE_MAXADDR, /* highaddr */ 873 NULL, NULL, /* filter, filterarg */ 874 tso_maxsize, /* maxsize */ 875 MAX_MBUF_FRAGS, /* nsegments */ 876 tso_maxsize, /* maxsegsize */ 877 0, /* flags */ 878 NULL, NULL, /* lockfunc, lockfuncarg*/ 879 &apc->tx_buf_tag); 880 if (unlikely(err)) { 881 device_printf(dev, "Feiled to create TX DMA tag\n"); 882 return err; 883 } 884 885 /* Create DMA tag for rx bufs */ 886 err = bus_dma_tag_create(bus_get_dma_tag(dev), /* parent */ 887 64, 0, /* alignment, boundary */ 888 BUS_SPACE_MAXADDR, /* lowaddr */ 889 BUS_SPACE_MAXADDR, /* highaddr */ 890 NULL, NULL, /* filter, filterarg */ 891 MJUM16BYTES, /* maxsize */ 892 1, /* nsegments */ 893 MJUM16BYTES, /* maxsegsize */ 894 0, /* flags */ 895 NULL, NULL, /* lockfunc, lockfuncarg*/ 896 &apc->rx_buf_tag); 897 if (unlikely(err)) { 898 device_printf(dev, "Feiled to create RX DMA tag\n"); 899 return err; 900 } 901 902 apc->rxqs = mallocarray(apc->num_queues, sizeof(struct mana_rxq *), 903 M_DEVBUF, M_WAITOK | M_ZERO); 904 905 return 0; 906 } 907 908 static int 909 mana_send_request(struct mana_context *ac, void *in_buf, 910 uint32_t in_len, void *out_buf, uint32_t out_len) 911 { 912 struct gdma_context *gc = ac->gdma_dev->gdma_context; 913 struct gdma_resp_hdr *resp = out_buf; 914 struct gdma_req_hdr *req = in_buf; 915 device_t dev = gc->dev; 916 static atomic_t activity_id; 917 int err; 918 919 req->dev_id = gc->mana.dev_id; 920 req->activity_id = atomic_inc_return(&activity_id); 921 922 mana_dbg(NULL, "activity_id = %u\n", activity_id); 923 924 err = mana_gd_send_request(gc, in_len, in_buf, out_len, 925 out_buf); 926 if (err || resp->status) { 927 device_printf(dev, "Failed to send mana message: %d, 0x%x\n", 928 err, resp->status); 929 return err ? err : EPROTO; 930 } 931 932 if (req->dev_id.as_uint32 != resp->dev_id.as_uint32 || 933 req->activity_id != resp->activity_id) { 934 device_printf(dev, 935 "Unexpected mana message response: %x,%x,%x,%x\n", 936 req->dev_id.as_uint32, resp->dev_id.as_uint32, 937 req->activity_id, resp->activity_id); 938 return EPROTO; 939 } 940 941 return 0; 942 } 943 944 static int 945 mana_verify_resp_hdr(const struct gdma_resp_hdr *resp_hdr, 946 const enum mana_command_code expected_code, 947 const uint32_t min_size) 948 { 949 if (resp_hdr->response.msg_type != expected_code) 950 return EPROTO; 951 952 if (resp_hdr->response.msg_version < GDMA_MESSAGE_V1) 953 return EPROTO; 954 955 if (resp_hdr->response.msg_size < min_size) 956 return EPROTO; 957 958 return 0; 959 } 960 961 static int 962 mana_query_device_cfg(struct mana_context *ac, uint32_t proto_major_ver, 963 uint32_t proto_minor_ver, uint32_t proto_micro_ver, 964 uint16_t *max_num_vports) 965 { 966 struct gdma_context *gc = ac->gdma_dev->gdma_context; 967 struct mana_query_device_cfg_resp resp = {}; 968 struct mana_query_device_cfg_req req = {}; 969 device_t dev = gc->dev; 970 int err = 0; 971 972 mana_gd_init_req_hdr(&req.hdr, MANA_QUERY_DEV_CONFIG, 973 sizeof(req), sizeof(resp)); 974 975 req.hdr.resp.msg_version = GDMA_MESSAGE_V2; 976 977 req.proto_major_ver = proto_major_ver; 978 req.proto_minor_ver = proto_minor_ver; 979 req.proto_micro_ver = proto_micro_ver; 980 981 err = mana_send_request(ac, &req, sizeof(req), &resp, sizeof(resp)); 982 if (err) { 983 device_printf(dev, "Failed to query config: %d", err); 984 return err; 985 } 986 987 err = mana_verify_resp_hdr(&resp.hdr, MANA_QUERY_DEV_CONFIG, 988 sizeof(resp)); 989 if (err || resp.hdr.status) { 990 device_printf(dev, "Invalid query result: %d, 0x%x\n", err, 991 resp.hdr.status); 992 if (!err) 993 err = EPROTO; 994 return err; 995 } 996 997 *max_num_vports = resp.max_num_vports; 998 999 if (resp.hdr.response.msg_version >= GDMA_MESSAGE_V2) 1000 gc->adapter_mtu = resp.adapter_mtu; 1001 else 1002 gc->adapter_mtu = ETHERMTU + ETHER_HDR_LEN; 1003 1004 mana_dbg(NULL, "mana max_num_vports from device = %d, " 1005 "adapter_mtu = %u\n", 1006 *max_num_vports, gc->adapter_mtu); 1007 1008 return 0; 1009 } 1010 1011 static int 1012 mana_query_vport_cfg(struct mana_port_context *apc, uint32_t vport_index, 1013 uint32_t *max_sq, uint32_t *max_rq, uint32_t *num_indir_entry) 1014 { 1015 struct mana_query_vport_cfg_resp resp = {}; 1016 struct mana_query_vport_cfg_req req = {}; 1017 int err; 1018 1019 mana_gd_init_req_hdr(&req.hdr, MANA_QUERY_VPORT_CONFIG, 1020 sizeof(req), sizeof(resp)); 1021 1022 req.vport_index = vport_index; 1023 1024 err = mana_send_request(apc->ac, &req, sizeof(req), &resp, 1025 sizeof(resp)); 1026 if (err) 1027 return err; 1028 1029 err = mana_verify_resp_hdr(&resp.hdr, MANA_QUERY_VPORT_CONFIG, 1030 sizeof(resp)); 1031 if (err) 1032 return err; 1033 1034 if (resp.hdr.status) 1035 return EPROTO; 1036 1037 *max_sq = resp.max_num_sq; 1038 *max_rq = resp.max_num_rq; 1039 *num_indir_entry = resp.num_indirection_ent; 1040 1041 apc->port_handle = resp.vport; 1042 memcpy(apc->mac_addr, resp.mac_addr, ETHER_ADDR_LEN); 1043 1044 return 0; 1045 } 1046 1047 void 1048 mana_uncfg_vport(struct mana_port_context *apc) 1049 { 1050 apc->vport_use_count--; 1051 if (apc->vport_use_count < 0) { 1052 mana_err(NULL, 1053 "WARNING: vport_use_count less than 0: %u\n", 1054 apc->vport_use_count); 1055 } 1056 } 1057 1058 int 1059 mana_cfg_vport(struct mana_port_context *apc, uint32_t protection_dom_id, 1060 uint32_t doorbell_pg_id) 1061 { 1062 struct mana_config_vport_resp resp = {}; 1063 struct mana_config_vport_req req = {}; 1064 int err; 1065 1066 /* This function is used to program the Ethernet port in the hardware 1067 * table. It can be called from the Ethernet driver or the RDMA driver. 1068 * 1069 * For Ethernet usage, the hardware supports only one active user on a 1070 * physical port. The driver checks on the port usage before programming 1071 * the hardware when creating the RAW QP (RDMA driver) or exposing the 1072 * device to kernel NET layer (Ethernet driver). 1073 * 1074 * Because the RDMA driver doesn't know in advance which QP type the 1075 * user will create, it exposes the device with all its ports. The user 1076 * may not be able to create RAW QP on a port if this port is already 1077 * in used by the Ethernet driver from the kernel. 1078 * 1079 * This physical port limitation only applies to the RAW QP. For RC QP, 1080 * the hardware doesn't have this limitation. The user can create RC 1081 * QPs on a physical port up to the hardware limits independent of the 1082 * Ethernet usage on the same port. 1083 */ 1084 if (apc->vport_use_count > 0) { 1085 return EBUSY; 1086 } 1087 apc->vport_use_count++; 1088 1089 mana_gd_init_req_hdr(&req.hdr, MANA_CONFIG_VPORT_TX, 1090 sizeof(req), sizeof(resp)); 1091 req.vport = apc->port_handle; 1092 req.pdid = protection_dom_id; 1093 req.doorbell_pageid = doorbell_pg_id; 1094 1095 err = mana_send_request(apc->ac, &req, sizeof(req), &resp, 1096 sizeof(resp)); 1097 if (err) { 1098 if_printf(apc->ndev, "Failed to configure vPort: %d\n", err); 1099 goto out; 1100 } 1101 1102 err = mana_verify_resp_hdr(&resp.hdr, MANA_CONFIG_VPORT_TX, 1103 sizeof(resp)); 1104 if (err || resp.hdr.status) { 1105 if_printf(apc->ndev, "Failed to configure vPort: %d, 0x%x\n", 1106 err, resp.hdr.status); 1107 if (!err) 1108 err = EPROTO; 1109 1110 goto out; 1111 } 1112 1113 apc->tx_shortform_allowed = resp.short_form_allowed; 1114 apc->tx_vp_offset = resp.tx_vport_offset; 1115 1116 if_printf(apc->ndev, "Configured vPort %ju PD %u DB %u\n", 1117 apc->port_handle, protection_dom_id, doorbell_pg_id); 1118 1119 out: 1120 if (err) 1121 mana_uncfg_vport(apc); 1122 1123 return err; 1124 } 1125 1126 static int 1127 mana_cfg_vport_steering(struct mana_port_context *apc, 1128 enum TRI_STATE rx, 1129 bool update_default_rxobj, bool update_key, 1130 bool update_tab) 1131 { 1132 uint16_t num_entries = MANA_INDIRECT_TABLE_SIZE; 1133 struct mana_cfg_rx_steer_req *req = NULL; 1134 struct mana_cfg_rx_steer_resp resp = {}; 1135 if_t ndev = apc->ndev; 1136 mana_handle_t *req_indir_tab; 1137 uint32_t req_buf_size; 1138 int err; 1139 1140 req_buf_size = sizeof(*req) + sizeof(mana_handle_t) * num_entries; 1141 req = malloc(req_buf_size, M_DEVBUF, M_WAITOK | M_ZERO); 1142 1143 mana_gd_init_req_hdr(&req->hdr, MANA_CONFIG_VPORT_RX, req_buf_size, 1144 sizeof(resp)); 1145 1146 req->vport = apc->port_handle; 1147 req->num_indir_entries = num_entries; 1148 req->indir_tab_offset = sizeof(*req); 1149 req->rx_enable = rx; 1150 req->rss_enable = apc->rss_state; 1151 req->update_default_rxobj = update_default_rxobj; 1152 req->update_hashkey = update_key; 1153 req->update_indir_tab = update_tab; 1154 req->default_rxobj = apc->default_rxobj; 1155 1156 if (update_key) 1157 memcpy(&req->hashkey, apc->hashkey, MANA_HASH_KEY_SIZE); 1158 1159 if (update_tab) { 1160 req_indir_tab = (mana_handle_t *)(req + 1); 1161 memcpy(req_indir_tab, apc->rxobj_table, 1162 req->num_indir_entries * sizeof(mana_handle_t)); 1163 } 1164 1165 err = mana_send_request(apc->ac, req, req_buf_size, &resp, 1166 sizeof(resp)); 1167 if (err) { 1168 if_printf(ndev, "Failed to configure vPort RX: %d\n", err); 1169 goto out; 1170 } 1171 1172 err = mana_verify_resp_hdr(&resp.hdr, MANA_CONFIG_VPORT_RX, 1173 sizeof(resp)); 1174 if (err) { 1175 if_printf(ndev, "vPort RX configuration failed: %d\n", err); 1176 goto out; 1177 } 1178 1179 if (resp.hdr.status) { 1180 if_printf(ndev, "vPort RX configuration failed: 0x%x\n", 1181 resp.hdr.status); 1182 err = EPROTO; 1183 } 1184 1185 if_printf(ndev, "Configured steering vPort %ju entries %u\n", 1186 apc->port_handle, num_entries); 1187 1188 out: 1189 free(req, M_DEVBUF); 1190 return err; 1191 } 1192 1193 int 1194 mana_create_wq_obj(struct mana_port_context *apc, 1195 mana_handle_t vport, 1196 uint32_t wq_type, struct mana_obj_spec *wq_spec, 1197 struct mana_obj_spec *cq_spec, 1198 mana_handle_t *wq_obj) 1199 { 1200 struct mana_create_wqobj_resp resp = {}; 1201 struct mana_create_wqobj_req req = {}; 1202 if_t ndev = apc->ndev; 1203 int err; 1204 1205 mana_gd_init_req_hdr(&req.hdr, MANA_CREATE_WQ_OBJ, 1206 sizeof(req), sizeof(resp)); 1207 req.vport = vport; 1208 req.wq_type = wq_type; 1209 req.wq_gdma_region = wq_spec->gdma_region; 1210 req.cq_gdma_region = cq_spec->gdma_region; 1211 req.wq_size = wq_spec->queue_size; 1212 req.cq_size = cq_spec->queue_size; 1213 req.cq_moderation_ctx_id = cq_spec->modr_ctx_id; 1214 req.cq_parent_qid = cq_spec->attached_eq; 1215 1216 err = mana_send_request(apc->ac, &req, sizeof(req), &resp, 1217 sizeof(resp)); 1218 if (err) { 1219 if_printf(ndev, "Failed to create WQ object: %d\n", err); 1220 goto out; 1221 } 1222 1223 err = mana_verify_resp_hdr(&resp.hdr, MANA_CREATE_WQ_OBJ, 1224 sizeof(resp)); 1225 if (err || resp.hdr.status) { 1226 if_printf(ndev, "Failed to create WQ object: %d, 0x%x\n", err, 1227 resp.hdr.status); 1228 if (!err) 1229 err = EPROTO; 1230 goto out; 1231 } 1232 1233 if (resp.wq_obj == INVALID_MANA_HANDLE) { 1234 if_printf(ndev, "Got an invalid WQ object handle\n"); 1235 err = EPROTO; 1236 goto out; 1237 } 1238 1239 *wq_obj = resp.wq_obj; 1240 wq_spec->queue_index = resp.wq_id; 1241 cq_spec->queue_index = resp.cq_id; 1242 1243 return 0; 1244 out: 1245 return err; 1246 } 1247 1248 void 1249 mana_destroy_wq_obj(struct mana_port_context *apc, uint32_t wq_type, 1250 mana_handle_t wq_obj) 1251 { 1252 struct mana_destroy_wqobj_resp resp = {}; 1253 struct mana_destroy_wqobj_req req = {}; 1254 if_t ndev = apc->ndev; 1255 int err; 1256 1257 mana_gd_init_req_hdr(&req.hdr, MANA_DESTROY_WQ_OBJ, 1258 sizeof(req), sizeof(resp)); 1259 req.wq_type = wq_type; 1260 req.wq_obj_handle = wq_obj; 1261 1262 err = mana_send_request(apc->ac, &req, sizeof(req), &resp, 1263 sizeof(resp)); 1264 if (err) { 1265 if_printf(ndev, "Failed to destroy WQ object: %d\n", err); 1266 return; 1267 } 1268 1269 err = mana_verify_resp_hdr(&resp.hdr, MANA_DESTROY_WQ_OBJ, 1270 sizeof(resp)); 1271 if (err || resp.hdr.status) 1272 if_printf(ndev, "Failed to destroy WQ object: %d, 0x%x\n", 1273 err, resp.hdr.status); 1274 } 1275 1276 static void 1277 mana_destroy_eq(struct mana_context *ac) 1278 { 1279 struct gdma_context *gc = ac->gdma_dev->gdma_context; 1280 struct gdma_queue *eq; 1281 int i; 1282 1283 if (!ac->eqs) 1284 return; 1285 1286 for (i = 0; i < gc->max_num_queues; i++) { 1287 eq = ac->eqs[i].eq; 1288 if (!eq) 1289 continue; 1290 1291 mana_gd_destroy_queue(gc, eq); 1292 } 1293 1294 free(ac->eqs, M_DEVBUF); 1295 ac->eqs = NULL; 1296 } 1297 1298 static int 1299 mana_create_eq(struct mana_context *ac) 1300 { 1301 struct gdma_dev *gd = ac->gdma_dev; 1302 struct gdma_context *gc = gd->gdma_context; 1303 struct gdma_queue_spec spec = {}; 1304 int err; 1305 int i; 1306 1307 ac->eqs = mallocarray(gc->max_num_queues, sizeof(struct mana_eq), 1308 M_DEVBUF, M_WAITOK | M_ZERO); 1309 1310 spec.type = GDMA_EQ; 1311 spec.monitor_avl_buf = false; 1312 spec.queue_size = EQ_SIZE; 1313 spec.eq.callback = NULL; 1314 spec.eq.context = ac->eqs; 1315 spec.eq.log2_throttle_limit = LOG2_EQ_THROTTLE; 1316 1317 for (i = 0; i < gc->max_num_queues; i++) { 1318 err = mana_gd_create_mana_eq(gd, &spec, &ac->eqs[i].eq); 1319 if (err) 1320 goto out; 1321 } 1322 1323 return 0; 1324 out: 1325 mana_destroy_eq(ac); 1326 return err; 1327 } 1328 1329 static int 1330 mana_fence_rq(struct mana_port_context *apc, struct mana_rxq *rxq) 1331 { 1332 struct mana_fence_rq_resp resp = {}; 1333 struct mana_fence_rq_req req = {}; 1334 int err; 1335 1336 init_completion(&rxq->fence_event); 1337 1338 mana_gd_init_req_hdr(&req.hdr, MANA_FENCE_RQ, 1339 sizeof(req), sizeof(resp)); 1340 req.wq_obj_handle = rxq->rxobj; 1341 1342 err = mana_send_request(apc->ac, &req, sizeof(req), &resp, 1343 sizeof(resp)); 1344 if (err) { 1345 if_printf(apc->ndev, "Failed to fence RQ %u: %d\n", 1346 rxq->rxq_idx, err); 1347 return err; 1348 } 1349 1350 err = mana_verify_resp_hdr(&resp.hdr, MANA_FENCE_RQ, sizeof(resp)); 1351 if (err || resp.hdr.status) { 1352 if_printf(apc->ndev, "Failed to fence RQ %u: %d, 0x%x\n", 1353 rxq->rxq_idx, err, resp.hdr.status); 1354 if (!err) 1355 err = EPROTO; 1356 1357 return err; 1358 } 1359 1360 if (wait_for_completion_timeout(&rxq->fence_event, 10 * hz)) { 1361 if_printf(apc->ndev, "Failed to fence RQ %u: timed out\n", 1362 rxq->rxq_idx); 1363 return ETIMEDOUT; 1364 } 1365 1366 return 0; 1367 } 1368 1369 static void 1370 mana_fence_rqs(struct mana_port_context *apc) 1371 { 1372 unsigned int rxq_idx; 1373 struct mana_rxq *rxq; 1374 int err; 1375 1376 for (rxq_idx = 0; rxq_idx < apc->num_queues; rxq_idx++) { 1377 rxq = apc->rxqs[rxq_idx]; 1378 err = mana_fence_rq(apc, rxq); 1379 1380 /* In case of any error, use sleep instead. */ 1381 if (err) 1382 gdma_msleep(100); 1383 } 1384 } 1385 1386 static int 1387 mana_move_wq_tail(struct gdma_queue *wq, uint32_t num_units) 1388 { 1389 uint32_t used_space_old; 1390 uint32_t used_space_new; 1391 1392 used_space_old = wq->head - wq->tail; 1393 used_space_new = wq->head - (wq->tail + num_units); 1394 1395 if (used_space_new > used_space_old) { 1396 mana_err(NULL, 1397 "WARNING: new used space %u greater than old one %u\n", 1398 used_space_new, used_space_old); 1399 return ERANGE; 1400 } 1401 1402 wq->tail += num_units; 1403 return 0; 1404 } 1405 1406 static void 1407 mana_poll_tx_cq(struct mana_cq *cq) 1408 { 1409 struct gdma_comp *completions = cq->gdma_comp_buf; 1410 struct gdma_posted_wqe_info *wqe_info; 1411 struct mana_send_buf_info *tx_info; 1412 unsigned int pkt_transmitted = 0; 1413 unsigned int wqe_unit_cnt = 0; 1414 struct mana_txq *txq = cq->txq; 1415 struct mana_port_context *apc; 1416 unsigned int tx_queue_size; 1417 uint16_t next_to_complete; 1418 if_t ndev; 1419 int comp_read; 1420 int txq_idx = txq->idx; 1421 int i; 1422 int sa_drop = 0; 1423 1424 struct gdma_queue *gdma_wq; 1425 unsigned int avail_space; 1426 bool txq_full = false; 1427 1428 ndev = txq->ndev; 1429 apc = if_getsoftc(ndev); 1430 tx_queue_size = apc->tx_queue_size; 1431 1432 comp_read = mana_gd_poll_cq(cq->gdma_cq, completions, 1433 CQE_POLLING_BUFFER); 1434 1435 if (comp_read < 1) 1436 return; 1437 1438 next_to_complete = txq->next_to_complete; 1439 1440 for (i = 0; i < comp_read; i++) { 1441 struct mana_tx_comp_oob *cqe_oob; 1442 1443 if (!completions[i].is_sq) { 1444 mana_err(NULL, "WARNING: Not for SQ\n"); 1445 return; 1446 } 1447 1448 cqe_oob = (struct mana_tx_comp_oob *)completions[i].cqe_data; 1449 if (cqe_oob->cqe_hdr.client_type != 1450 MANA_CQE_COMPLETION) { 1451 mana_err(NULL, 1452 "WARNING: Invalid CQE client type %u\n", 1453 cqe_oob->cqe_hdr.client_type); 1454 return; 1455 } 1456 1457 switch (cqe_oob->cqe_hdr.cqe_type) { 1458 case CQE_TX_OKAY: 1459 break; 1460 1461 case CQE_TX_SA_DROP: 1462 case CQE_TX_MTU_DROP: 1463 case CQE_TX_INVALID_OOB: 1464 case CQE_TX_INVALID_ETH_TYPE: 1465 case CQE_TX_HDR_PROCESSING_ERROR: 1466 case CQE_TX_VF_DISABLED: 1467 case CQE_TX_VPORT_IDX_OUT_OF_RANGE: 1468 case CQE_TX_VPORT_DISABLED: 1469 case CQE_TX_VLAN_TAGGING_VIOLATION: 1470 sa_drop ++; 1471 mana_dbg(NULL, 1472 "TX: txq %d CQE error %d, ntc = %d, " 1473 "pending sends = %d: err ignored.\n", 1474 txq_idx, cqe_oob->cqe_hdr.cqe_type, 1475 next_to_complete, txq->pending_sends); 1476 counter_u64_add(txq->stats.cqe_err, 1); 1477 break; 1478 1479 default: 1480 /* If the CQE type is unknown, log a debug msg, 1481 * and still free the mbuf, etc. 1482 */ 1483 mana_dbg(NULL, 1484 "ERROR: TX: Unknown CQE type %d\n", 1485 cqe_oob->cqe_hdr.cqe_type); 1486 counter_u64_add(txq->stats.cqe_unknown_type, 1); 1487 break; 1488 } 1489 if (txq->gdma_txq_id != completions[i].wq_num) { 1490 mana_dbg(NULL, 1491 "txq gdma id not match completion wq num: " 1492 "%d != %d\n", 1493 txq->gdma_txq_id, completions[i].wq_num); 1494 break; 1495 } 1496 1497 tx_info = &txq->tx_buf_info[next_to_complete]; 1498 if (!tx_info->mbuf) { 1499 mana_err(NULL, 1500 "WARNING: txq %d Empty mbuf on tx_info: %u, " 1501 "ntu = %u, pending_sends = %d, " 1502 "transmitted = %d, sa_drop = %d, i = %d, comp_read = %d\n", 1503 txq_idx, next_to_complete, txq->next_to_use, 1504 txq->pending_sends, pkt_transmitted, sa_drop, 1505 i, comp_read); 1506 break; 1507 } 1508 1509 wqe_info = &tx_info->wqe_inf; 1510 wqe_unit_cnt += wqe_info->wqe_size_in_bu; 1511 1512 mana_tx_unmap_mbuf(apc, tx_info); 1513 mb(); 1514 1515 next_to_complete = 1516 MANA_IDX_NEXT(next_to_complete, tx_queue_size); 1517 1518 pkt_transmitted++; 1519 } 1520 1521 txq->next_to_complete = next_to_complete; 1522 1523 if (wqe_unit_cnt == 0) { 1524 mana_err(NULL, 1525 "WARNING: TX ring not proceeding!\n"); 1526 return; 1527 } 1528 1529 mana_move_wq_tail(txq->gdma_sq, wqe_unit_cnt); 1530 1531 /* Ensure tail updated before checking q stop */ 1532 wmb(); 1533 1534 gdma_wq = txq->gdma_sq; 1535 avail_space = mana_gd_wq_avail_space(gdma_wq); 1536 1537 1538 if ((if_getdrvflags(ndev) & MANA_TXQ_FULL) == MANA_TXQ_FULL) { 1539 txq_full = true; 1540 } 1541 1542 /* Ensure checking txq_full before apc->port_is_up. */ 1543 rmb(); 1544 1545 if (txq_full && apc->port_is_up && avail_space >= MAX_TX_WQE_SIZE) { 1546 /* Grab the txq lock and re-test */ 1547 mtx_lock(&txq->txq_mtx); 1548 avail_space = mana_gd_wq_avail_space(gdma_wq); 1549 1550 if ((if_getdrvflags(ndev) & MANA_TXQ_FULL) == MANA_TXQ_FULL && 1551 apc->port_is_up && avail_space >= MAX_TX_WQE_SIZE) { 1552 /* Clear the Q full flag */ 1553 if_setdrvflagbits(apc->ndev, IFF_DRV_RUNNING, 1554 IFF_DRV_OACTIVE); 1555 counter_u64_add(txq->stats.wakeup, 1); 1556 if (txq->alt_txq_idx != txq->idx) { 1557 uint64_t stops = counter_u64_fetch(txq->stats.stop); 1558 uint64_t wakeups = counter_u64_fetch(txq->stats.wakeup); 1559 /* Reset alt_txq_idx back if it is not overloaded */ 1560 if (stops < wakeups) { 1561 txq->alt_txq_idx = txq->idx; 1562 counter_u64_add(txq->stats.alt_reset, 1); 1563 } 1564 } 1565 rmb(); 1566 /* Schedule a tx enqueue task */ 1567 taskqueue_enqueue(txq->enqueue_tq, &txq->enqueue_task); 1568 } 1569 mtx_unlock(&txq->txq_mtx); 1570 } 1571 1572 if (atomic_sub_return(pkt_transmitted, &txq->pending_sends) < 0) 1573 mana_err(NULL, 1574 "WARNING: TX %d pending_sends error: %d\n", 1575 txq->idx, txq->pending_sends); 1576 1577 cq->work_done = pkt_transmitted; 1578 } 1579 1580 static void 1581 mana_post_pkt_rxq(struct mana_rxq *rxq, 1582 struct mana_recv_buf_oob *recv_buf_oob) 1583 { 1584 int err; 1585 1586 err = mana_gd_post_work_request(rxq->gdma_rq, &recv_buf_oob->wqe_req, 1587 &recv_buf_oob->wqe_inf); 1588 if (err) { 1589 mana_err(NULL, "WARNING: rxq %u post pkt err %d\n", 1590 rxq->rxq_idx, err); 1591 return; 1592 } 1593 1594 if (recv_buf_oob->wqe_inf.wqe_size_in_bu != 1) { 1595 mana_err(NULL, "WARNING: rxq %u wqe_size_in_bu %u\n", 1596 rxq->rxq_idx, recv_buf_oob->wqe_inf.wqe_size_in_bu); 1597 } 1598 } 1599 1600 static void 1601 mana_rx_mbuf(struct mbuf *mbuf, struct mana_rxcomp_oob *cqe, 1602 struct mana_rxq *rxq) 1603 { 1604 struct mana_stats *rx_stats = &rxq->stats; 1605 if_t ndev = rxq->ndev; 1606 uint32_t pkt_len = cqe->ppi[0].pkt_len; 1607 uint16_t rxq_idx = rxq->rxq_idx; 1608 struct mana_port_context *apc; 1609 bool do_lro = false; 1610 bool do_if_input; 1611 1612 apc = if_getsoftc(ndev); 1613 rxq->rx_cq.work_done++; 1614 1615 if (!mbuf) { 1616 return; 1617 } 1618 1619 mbuf->m_flags |= M_PKTHDR; 1620 mbuf->m_pkthdr.len = pkt_len; 1621 mbuf->m_len = pkt_len; 1622 mbuf->m_pkthdr.rcvif = ndev; 1623 1624 if ((if_getcapenable(ndev) & IFCAP_RXCSUM || 1625 if_getcapenable(ndev) & IFCAP_RXCSUM_IPV6) && 1626 (cqe->rx_iphdr_csum_succeed)) { 1627 mbuf->m_pkthdr.csum_flags = CSUM_IP_CHECKED; 1628 mbuf->m_pkthdr.csum_flags |= CSUM_IP_VALID; 1629 if (cqe->rx_tcp_csum_succeed || cqe->rx_udp_csum_succeed) { 1630 mbuf->m_pkthdr.csum_flags |= 1631 (CSUM_DATA_VALID | CSUM_PSEUDO_HDR); 1632 mbuf->m_pkthdr.csum_data = 0xffff; 1633 1634 if (cqe->rx_tcp_csum_succeed) 1635 do_lro = true; 1636 } 1637 } 1638 1639 if (cqe->rx_hashtype != 0) { 1640 mbuf->m_pkthdr.flowid = cqe->ppi[0].pkt_hash; 1641 1642 uint16_t hashtype = cqe->rx_hashtype; 1643 if (hashtype & NDIS_HASH_IPV4_MASK) { 1644 hashtype &= NDIS_HASH_IPV4_MASK; 1645 switch (hashtype) { 1646 case NDIS_HASH_TCP_IPV4: 1647 M_HASHTYPE_SET(mbuf, M_HASHTYPE_RSS_TCP_IPV4); 1648 break; 1649 case NDIS_HASH_UDP_IPV4: 1650 M_HASHTYPE_SET(mbuf, M_HASHTYPE_RSS_UDP_IPV4); 1651 break; 1652 default: 1653 M_HASHTYPE_SET(mbuf, M_HASHTYPE_RSS_IPV4); 1654 } 1655 } else if (hashtype & NDIS_HASH_IPV6_MASK) { 1656 hashtype &= NDIS_HASH_IPV6_MASK; 1657 switch (hashtype) { 1658 case NDIS_HASH_TCP_IPV6: 1659 M_HASHTYPE_SET(mbuf, M_HASHTYPE_RSS_TCP_IPV6); 1660 break; 1661 case NDIS_HASH_TCP_IPV6_EX: 1662 M_HASHTYPE_SET(mbuf, 1663 M_HASHTYPE_RSS_TCP_IPV6_EX); 1664 break; 1665 case NDIS_HASH_UDP_IPV6: 1666 M_HASHTYPE_SET(mbuf, M_HASHTYPE_RSS_UDP_IPV6); 1667 break; 1668 case NDIS_HASH_UDP_IPV6_EX: 1669 M_HASHTYPE_SET(mbuf, 1670 M_HASHTYPE_RSS_UDP_IPV6_EX); 1671 break; 1672 default: 1673 M_HASHTYPE_SET(mbuf, M_HASHTYPE_RSS_IPV6); 1674 } 1675 } else { 1676 M_HASHTYPE_SET(mbuf, M_HASHTYPE_OPAQUE_HASH); 1677 } 1678 } else { 1679 mbuf->m_pkthdr.flowid = rxq_idx; 1680 M_HASHTYPE_SET(mbuf, M_HASHTYPE_NONE); 1681 } 1682 1683 do_if_input = true; 1684 if ((if_getcapenable(ndev) & IFCAP_LRO) && do_lro) { 1685 rxq->lro_tried++; 1686 if (rxq->lro.lro_cnt != 0 && 1687 tcp_lro_rx(&rxq->lro, mbuf, 0) == 0) 1688 do_if_input = false; 1689 else 1690 rxq->lro_failed++; 1691 } 1692 if (do_if_input) { 1693 if_input(ndev, mbuf); 1694 } 1695 1696 counter_enter(); 1697 counter_u64_add_protected(rx_stats->packets, 1); 1698 counter_u64_add_protected(apc->port_stats.rx_packets, 1); 1699 counter_u64_add_protected(rx_stats->bytes, pkt_len); 1700 counter_u64_add_protected(apc->port_stats.rx_bytes, pkt_len); 1701 counter_exit(); 1702 } 1703 1704 static int 1705 mana_refill_rx_mbufs(struct mana_port_context *apc, 1706 struct mana_rxq *rxq, uint32_t num) 1707 { 1708 struct mana_recv_buf_oob *rxbuf_oob; 1709 uint32_t next_to_refill; 1710 uint32_t i; 1711 int err; 1712 1713 next_to_refill = rxq->next_to_refill; 1714 1715 for (i = 0; i < num; i++) { 1716 if (next_to_refill == rxq->buf_index) { 1717 mana_warn(NULL, "refilling index reached current, " 1718 "aborted! rxq %u, oob idx %u\n", 1719 rxq->rxq_idx, next_to_refill); 1720 break; 1721 } 1722 1723 rxbuf_oob = &rxq->rx_oobs[next_to_refill]; 1724 1725 if (likely(rxbuf_oob->mbuf == NULL)) { 1726 err = mana_load_rx_mbuf(apc, rxq, rxbuf_oob, true); 1727 } else { 1728 mana_warn(NULL, "mbuf not null when refilling, " 1729 "rxq %u, oob idx %u, reusing\n", 1730 rxq->rxq_idx, next_to_refill); 1731 err = mana_load_rx_mbuf(apc, rxq, rxbuf_oob, false); 1732 } 1733 1734 if (unlikely(err != 0)) { 1735 mana_dbg(NULL, 1736 "failed to load rx mbuf, err = %d, rxq = %u\n", 1737 err, rxq->rxq_idx); 1738 counter_u64_add(rxq->stats.mbuf_alloc_fail, 1); 1739 break; 1740 } 1741 1742 mana_post_pkt_rxq(rxq, rxbuf_oob); 1743 1744 next_to_refill = MANA_IDX_NEXT(next_to_refill, 1745 rxq->num_rx_buf); 1746 } 1747 1748 if (likely(i != 0)) { 1749 struct gdma_context *gc = 1750 rxq->gdma_rq->gdma_dev->gdma_context; 1751 1752 mana_gd_wq_ring_doorbell(gc, rxq->gdma_rq); 1753 } 1754 1755 if (unlikely(i < num)) { 1756 counter_u64_add(rxq->stats.partial_refill, 1); 1757 mana_dbg(NULL, 1758 "refilled rxq %u with only %u mbufs (%u requested)\n", 1759 rxq->rxq_idx, i, num); 1760 } 1761 1762 rxq->next_to_refill = next_to_refill; 1763 return (i); 1764 } 1765 1766 static void 1767 mana_process_rx_cqe(struct mana_rxq *rxq, struct mana_cq *cq, 1768 struct gdma_comp *cqe) 1769 { 1770 struct mana_rxcomp_oob *oob = (struct mana_rxcomp_oob *)cqe->cqe_data; 1771 struct mana_recv_buf_oob *rxbuf_oob; 1772 if_t ndev = rxq->ndev; 1773 struct mana_port_context *apc; 1774 struct mbuf *old_mbuf; 1775 uint32_t refill_required; 1776 uint32_t curr, pktlen; 1777 1778 switch (oob->cqe_hdr.cqe_type) { 1779 case CQE_RX_OKAY: 1780 break; 1781 1782 case CQE_RX_TRUNCATED: 1783 apc = if_getsoftc(ndev); 1784 counter_u64_add(apc->port_stats.rx_drops, 1); 1785 rxbuf_oob = &rxq->rx_oobs[rxq->buf_index]; 1786 if_printf(ndev, "Dropped a truncated packet\n"); 1787 goto drop; 1788 1789 case CQE_RX_COALESCED_4: 1790 if_printf(ndev, "RX coalescing is unsupported\n"); 1791 return; 1792 1793 case CQE_RX_OBJECT_FENCE: 1794 complete(&rxq->fence_event); 1795 return; 1796 1797 default: 1798 if_printf(ndev, "Unknown RX CQE type = %d\n", 1799 oob->cqe_hdr.cqe_type); 1800 return; 1801 } 1802 1803 if (oob->cqe_hdr.cqe_type != CQE_RX_OKAY) 1804 return; 1805 1806 pktlen = oob->ppi[0].pkt_len; 1807 1808 if (pktlen == 0) { 1809 /* data packets should never have packetlength of zero */ 1810 if_printf(ndev, "RX pkt len=0, rq=%u, cq=%u, rxobj=0x%jx\n", 1811 rxq->gdma_id, cq->gdma_id, rxq->rxobj); 1812 return; 1813 } 1814 1815 curr = rxq->buf_index; 1816 rxbuf_oob = &rxq->rx_oobs[curr]; 1817 if (rxbuf_oob->wqe_inf.wqe_size_in_bu != 1) { 1818 mana_err(NULL, "WARNING: Rx Incorrect complete " 1819 "WQE size %u\n", 1820 rxbuf_oob->wqe_inf.wqe_size_in_bu); 1821 } 1822 1823 apc = if_getsoftc(ndev); 1824 1825 old_mbuf = rxbuf_oob->mbuf; 1826 1827 /* Unload DMA map for the old mbuf */ 1828 mana_unload_rx_mbuf(apc, rxq, rxbuf_oob, false); 1829 /* Clear the mbuf pointer to avoid reuse */ 1830 rxbuf_oob->mbuf = NULL; 1831 1832 mana_rx_mbuf(old_mbuf, oob, rxq); 1833 1834 drop: 1835 mana_move_wq_tail(rxq->gdma_rq, rxbuf_oob->wqe_inf.wqe_size_in_bu); 1836 1837 rxq->buf_index = MANA_IDX_NEXT(rxq->buf_index, rxq->num_rx_buf); 1838 1839 /* Check if refill is needed */ 1840 refill_required = MANA_GET_SPACE(rxq->next_to_refill, 1841 rxq->buf_index, rxq->num_rx_buf); 1842 1843 if (refill_required >= rxq->refill_thresh) { 1844 /* Refill empty rx_oobs with new mbufs */ 1845 mana_refill_rx_mbufs(apc, rxq, refill_required); 1846 } 1847 } 1848 1849 static void 1850 mana_poll_rx_cq(struct mana_cq *cq) 1851 { 1852 struct gdma_comp *comp = cq->gdma_comp_buf; 1853 int comp_read, i; 1854 1855 comp_read = mana_gd_poll_cq(cq->gdma_cq, comp, CQE_POLLING_BUFFER); 1856 KASSERT(comp_read <= CQE_POLLING_BUFFER, 1857 ("comp_read %d great than buf size %d", 1858 comp_read, CQE_POLLING_BUFFER)); 1859 1860 for (i = 0; i < comp_read; i++) { 1861 if (comp[i].is_sq == true) { 1862 mana_err(NULL, 1863 "WARNING: CQE not for receive queue\n"); 1864 return; 1865 } 1866 1867 /* verify recv cqe references the right rxq */ 1868 if (comp[i].wq_num != cq->rxq->gdma_id) { 1869 mana_err(NULL, 1870 "WARNING: Received CQE %d not for " 1871 "this receive queue %d\n", 1872 comp[i].wq_num, cq->rxq->gdma_id); 1873 return; 1874 } 1875 1876 mana_process_rx_cqe(cq->rxq, cq, &comp[i]); 1877 } 1878 1879 tcp_lro_flush_all(&cq->rxq->lro); 1880 } 1881 1882 static void 1883 mana_cq_handler(void *context, struct gdma_queue *gdma_queue) 1884 { 1885 struct mana_cq *cq = context; 1886 uint8_t arm_bit; 1887 1888 KASSERT(cq->gdma_cq == gdma_queue, 1889 ("cq do not match %p, %p", cq->gdma_cq, gdma_queue)); 1890 1891 if (cq->type == MANA_CQ_TYPE_RX) { 1892 mana_poll_rx_cq(cq); 1893 } else { 1894 mana_poll_tx_cq(cq); 1895 } 1896 1897 if (cq->work_done < cq->budget && cq->do_not_ring_db == false) 1898 arm_bit = SET_ARM_BIT; 1899 else 1900 arm_bit = 0; 1901 1902 mana_gd_ring_cq(gdma_queue, arm_bit); 1903 } 1904 1905 #define MANA_POLL_BUDGET 256 1906 #define MANA_RX_BUDGET 8 1907 #define MANA_TX_BUDGET 8 1908 1909 static void 1910 mana_poll(void *arg, int pending) 1911 { 1912 struct mana_cq *cq = arg; 1913 int i; 1914 1915 cq->work_done = 0; 1916 if (cq->type == MANA_CQ_TYPE_RX) { 1917 cq->budget = MANA_RX_BUDGET; 1918 } else { 1919 cq->budget = MANA_TX_BUDGET; 1920 } 1921 1922 for (i = 0; i < MANA_POLL_BUDGET; i++) { 1923 /* 1924 * If this is the last loop, set the budget big enough 1925 * so it will arm the CQ any way. 1926 */ 1927 if (i == (MANA_POLL_BUDGET - 1)) 1928 cq->budget = CQE_POLLING_BUFFER + 1; 1929 1930 mana_cq_handler(cq, cq->gdma_cq); 1931 1932 if (cq->work_done < cq->budget) 1933 break; 1934 1935 cq->work_done = 0; 1936 } 1937 } 1938 1939 static void 1940 mana_schedule_task(void *arg, struct gdma_queue *gdma_queue) 1941 { 1942 struct mana_cq *cq = arg; 1943 1944 taskqueue_enqueue(cq->cleanup_tq, &cq->cleanup_task); 1945 } 1946 1947 static void 1948 mana_deinit_cq(struct mana_port_context *apc, struct mana_cq *cq) 1949 { 1950 struct gdma_dev *gd = apc->ac->gdma_dev; 1951 1952 if (!cq->gdma_cq) 1953 return; 1954 1955 /* Drain cleanup taskqueue */ 1956 if (cq->cleanup_tq) { 1957 while (taskqueue_cancel(cq->cleanup_tq, 1958 &cq->cleanup_task, NULL)) { 1959 taskqueue_drain(cq->cleanup_tq, 1960 &cq->cleanup_task); 1961 } 1962 1963 taskqueue_free(cq->cleanup_tq); 1964 } 1965 1966 mana_gd_destroy_queue(gd->gdma_context, cq->gdma_cq); 1967 } 1968 1969 static void 1970 mana_deinit_txq(struct mana_port_context *apc, struct mana_txq *txq) 1971 { 1972 struct gdma_dev *gd = apc->ac->gdma_dev; 1973 struct mana_send_buf_info *txbuf_info; 1974 uint32_t pending_sends; 1975 int i; 1976 1977 if (!txq->gdma_sq) 1978 return; 1979 1980 if ((pending_sends = atomic_read(&txq->pending_sends)) > 0) { 1981 mana_err(NULL, 1982 "WARNING: txq pending sends not zero: %u\n", 1983 pending_sends); 1984 } 1985 1986 if (txq->next_to_use != txq->next_to_complete) { 1987 mana_err(NULL, 1988 "WARNING: txq buf not completed, " 1989 "next use %u, next complete %u\n", 1990 txq->next_to_use, txq->next_to_complete); 1991 } 1992 1993 /* Flush buf ring. Grab txq mtx lock */ 1994 if (txq->txq_br) { 1995 mtx_lock(&txq->txq_mtx); 1996 drbr_flush(apc->ndev, txq->txq_br); 1997 mtx_unlock(&txq->txq_mtx); 1998 buf_ring_free(txq->txq_br, M_DEVBUF); 1999 } 2000 2001 /* Drain taskqueue */ 2002 if (txq->enqueue_tq) { 2003 while (taskqueue_cancel(txq->enqueue_tq, 2004 &txq->enqueue_task, NULL)) { 2005 taskqueue_drain(txq->enqueue_tq, 2006 &txq->enqueue_task); 2007 } 2008 2009 taskqueue_free(txq->enqueue_tq); 2010 } 2011 2012 if (txq->tx_buf_info) { 2013 /* Free all mbufs which are still in-flight */ 2014 for (i = 0; i < apc->tx_queue_size; i++) { 2015 txbuf_info = &txq->tx_buf_info[i]; 2016 if (txbuf_info->mbuf) { 2017 mana_tx_unmap_mbuf(apc, txbuf_info); 2018 } 2019 } 2020 2021 free(txq->tx_buf_info, M_DEVBUF); 2022 } 2023 2024 mana_free_counters((counter_u64_t *)&txq->stats, 2025 sizeof(txq->stats)); 2026 2027 mana_gd_destroy_queue(gd->gdma_context, txq->gdma_sq); 2028 2029 mtx_destroy(&txq->txq_mtx); 2030 } 2031 2032 static void 2033 mana_destroy_txq(struct mana_port_context *apc) 2034 { 2035 int i; 2036 2037 if (!apc->tx_qp) 2038 return; 2039 2040 for (i = 0; i < apc->num_queues; i++) { 2041 mana_destroy_wq_obj(apc, GDMA_SQ, apc->tx_qp[i].tx_object); 2042 2043 mana_deinit_cq(apc, &apc->tx_qp[i].tx_cq); 2044 2045 mana_deinit_txq(apc, &apc->tx_qp[i].txq); 2046 } 2047 2048 free(apc->tx_qp, M_DEVBUF); 2049 apc->tx_qp = NULL; 2050 } 2051 2052 static int 2053 mana_create_txq(struct mana_port_context *apc, if_t net) 2054 { 2055 struct mana_context *ac = apc->ac; 2056 struct gdma_dev *gd = ac->gdma_dev; 2057 struct mana_obj_spec wq_spec; 2058 struct mana_obj_spec cq_spec; 2059 struct gdma_queue_spec spec; 2060 struct gdma_context *gc; 2061 struct mana_txq *txq; 2062 struct mana_cq *cq; 2063 uint32_t txq_size; 2064 uint32_t cq_size; 2065 int err; 2066 int i; 2067 2068 apc->tx_qp = mallocarray(apc->num_queues, sizeof(struct mana_tx_qp), 2069 M_DEVBUF, M_WAITOK | M_ZERO); 2070 2071 /* The minimum size of the WQE is 32 bytes, hence 2072 * apc->tx_queue_size represents the maximum number of WQEs 2073 * the SQ can store. This value is then used to size other queues 2074 * to prevent overflow. 2075 * Also note that the txq_size is always going to be page aligned, 2076 * as min val of apc->tx_queue_size is 128 and that would make 2077 * txq_size 128 * 32 = 4096 and the other higher values of 2078 * apc->tx_queue_size are always power of two. 2079 */ 2080 txq_size = apc->tx_queue_size * 32; 2081 KASSERT(IS_ALIGNED(txq_size, PAGE_SIZE), 2082 ("txq size not page aligned")); 2083 2084 cq_size = apc->tx_queue_size * COMP_ENTRY_SIZE; 2085 cq_size = ALIGN(cq_size, PAGE_SIZE); 2086 2087 gc = gd->gdma_context; 2088 2089 for (i = 0; i < apc->num_queues; i++) { 2090 apc->tx_qp[i].tx_object = INVALID_MANA_HANDLE; 2091 2092 /* Create SQ */ 2093 txq = &apc->tx_qp[i].txq; 2094 2095 txq->ndev = net; 2096 txq->vp_offset = apc->tx_vp_offset; 2097 txq->idx = i; 2098 txq->alt_txq_idx = i; 2099 2100 memset(&spec, 0, sizeof(spec)); 2101 spec.type = GDMA_SQ; 2102 spec.monitor_avl_buf = true; 2103 spec.queue_size = txq_size; 2104 err = mana_gd_create_mana_wq_cq(gd, &spec, &txq->gdma_sq); 2105 if (err) 2106 goto out; 2107 2108 /* Create SQ's CQ */ 2109 cq = &apc->tx_qp[i].tx_cq; 2110 cq->type = MANA_CQ_TYPE_TX; 2111 2112 cq->txq = txq; 2113 2114 memset(&spec, 0, sizeof(spec)); 2115 spec.type = GDMA_CQ; 2116 spec.monitor_avl_buf = false; 2117 spec.queue_size = cq_size; 2118 spec.cq.callback = mana_schedule_task; 2119 spec.cq.parent_eq = ac->eqs[i].eq; 2120 spec.cq.context = cq; 2121 err = mana_gd_create_mana_wq_cq(gd, &spec, &cq->gdma_cq); 2122 if (err) 2123 goto out; 2124 2125 memset(&wq_spec, 0, sizeof(wq_spec)); 2126 memset(&cq_spec, 0, sizeof(cq_spec)); 2127 2128 wq_spec.gdma_region = txq->gdma_sq->mem_info.dma_region_handle; 2129 wq_spec.queue_size = txq->gdma_sq->queue_size; 2130 2131 cq_spec.gdma_region = cq->gdma_cq->mem_info.dma_region_handle; 2132 cq_spec.queue_size = cq->gdma_cq->queue_size; 2133 cq_spec.modr_ctx_id = 0; 2134 cq_spec.attached_eq = cq->gdma_cq->cq.parent->id; 2135 2136 err = mana_create_wq_obj(apc, apc->port_handle, GDMA_SQ, 2137 &wq_spec, &cq_spec, &apc->tx_qp[i].tx_object); 2138 2139 if (err) 2140 goto out; 2141 2142 txq->gdma_sq->id = wq_spec.queue_index; 2143 cq->gdma_cq->id = cq_spec.queue_index; 2144 2145 txq->gdma_sq->mem_info.dma_region_handle = 2146 GDMA_INVALID_DMA_REGION; 2147 cq->gdma_cq->mem_info.dma_region_handle = 2148 GDMA_INVALID_DMA_REGION; 2149 2150 txq->gdma_txq_id = txq->gdma_sq->id; 2151 2152 cq->gdma_id = cq->gdma_cq->id; 2153 2154 mana_dbg(NULL, 2155 "txq %d, txq gdma id %d, txq cq gdma id %d\n", 2156 i, txq->gdma_txq_id, cq->gdma_id); 2157 2158 if (cq->gdma_id >= gc->max_num_cqs) { 2159 if_printf(net, "CQ id %u too large.\n", cq->gdma_id); 2160 err = EINVAL; 2161 goto out; 2162 } 2163 2164 gc->cq_table[cq->gdma_id] = cq->gdma_cq; 2165 2166 /* Initialize tx specific data */ 2167 txq->tx_buf_info = malloc(apc->tx_queue_size * 2168 sizeof(struct mana_send_buf_info), 2169 M_DEVBUF, M_WAITOK | M_ZERO); 2170 2171 snprintf(txq->txq_mtx_name, nitems(txq->txq_mtx_name), 2172 "mana:tx(%d)", i); 2173 mtx_init(&txq->txq_mtx, txq->txq_mtx_name, NULL, MTX_DEF); 2174 2175 txq->txq_br = buf_ring_alloc(4 * apc->tx_queue_size, 2176 M_DEVBUF, M_WAITOK, &txq->txq_mtx); 2177 2178 /* Allocate taskqueue for deferred send */ 2179 TASK_INIT(&txq->enqueue_task, 0, mana_xmit_taskfunc, txq); 2180 txq->enqueue_tq = taskqueue_create_fast("mana_tx_enque", 2181 M_NOWAIT, taskqueue_thread_enqueue, &txq->enqueue_tq); 2182 if (unlikely(txq->enqueue_tq == NULL)) { 2183 if_printf(net, 2184 "Unable to create tx %d enqueue task queue\n", i); 2185 err = ENOMEM; 2186 goto out; 2187 } 2188 taskqueue_start_threads(&txq->enqueue_tq, 1, PI_NET, 2189 "mana txq p%u-tx%d", apc->port_idx, i); 2190 2191 mana_alloc_counters((counter_u64_t *)&txq->stats, 2192 sizeof(txq->stats)); 2193 2194 /* Allocate and start the cleanup task on CQ */ 2195 cq->do_not_ring_db = false; 2196 2197 NET_TASK_INIT(&cq->cleanup_task, 0, mana_poll, cq); 2198 cq->cleanup_tq = 2199 taskqueue_create_fast("mana tx cq cleanup", 2200 M_WAITOK, taskqueue_thread_enqueue, 2201 &cq->cleanup_tq); 2202 2203 if (apc->last_tx_cq_bind_cpu < 0) 2204 apc->last_tx_cq_bind_cpu = CPU_FIRST(); 2205 cq->cpu = apc->last_tx_cq_bind_cpu; 2206 apc->last_tx_cq_bind_cpu = CPU_NEXT(apc->last_tx_cq_bind_cpu); 2207 2208 if (apc->bind_cleanup_thread_cpu) { 2209 cpuset_t cpu_mask; 2210 CPU_SETOF(cq->cpu, &cpu_mask); 2211 taskqueue_start_threads_cpuset(&cq->cleanup_tq, 2212 1, PI_NET, &cpu_mask, 2213 "mana cq p%u-tx%u-cpu%d", 2214 apc->port_idx, txq->idx, cq->cpu); 2215 } else { 2216 taskqueue_start_threads(&cq->cleanup_tq, 1, 2217 PI_NET, "mana cq p%u-tx%u", 2218 apc->port_idx, txq->idx); 2219 } 2220 2221 mana_gd_ring_cq(cq->gdma_cq, SET_ARM_BIT); 2222 } 2223 2224 return 0; 2225 out: 2226 mana_destroy_txq(apc); 2227 return err; 2228 } 2229 2230 static void 2231 mana_destroy_rxq(struct mana_port_context *apc, struct mana_rxq *rxq, 2232 bool validate_state) 2233 { 2234 struct gdma_context *gc = apc->ac->gdma_dev->gdma_context; 2235 struct mana_recv_buf_oob *rx_oob; 2236 int i; 2237 2238 if (!rxq) 2239 return; 2240 2241 if (validate_state) { 2242 /* 2243 * XXX Cancel and drain cleanup task queue here. 2244 */ 2245 ; 2246 } 2247 2248 mana_destroy_wq_obj(apc, GDMA_RQ, rxq->rxobj); 2249 2250 mana_deinit_cq(apc, &rxq->rx_cq); 2251 2252 mana_free_counters((counter_u64_t *)&rxq->stats, 2253 sizeof(rxq->stats)); 2254 2255 /* Free LRO resources */ 2256 tcp_lro_free(&rxq->lro); 2257 2258 for (i = 0; i < rxq->num_rx_buf; i++) { 2259 rx_oob = &rxq->rx_oobs[i]; 2260 2261 if (rx_oob->mbuf) 2262 mana_unload_rx_mbuf(apc, rxq, rx_oob, true); 2263 2264 bus_dmamap_destroy(apc->rx_buf_tag, rx_oob->dma_map); 2265 } 2266 2267 if (rxq->gdma_rq) 2268 mana_gd_destroy_queue(gc, rxq->gdma_rq); 2269 2270 free(rxq, M_DEVBUF); 2271 } 2272 2273 #define MANA_WQE_HEADER_SIZE 16 2274 #define MANA_WQE_SGE_SIZE 16 2275 2276 static int 2277 mana_alloc_rx_wqe(struct mana_port_context *apc, 2278 struct mana_rxq *rxq, uint32_t *rxq_size, uint32_t *cq_size) 2279 { 2280 struct mana_recv_buf_oob *rx_oob; 2281 uint32_t buf_idx; 2282 int err; 2283 2284 if (rxq->datasize == 0) { 2285 mana_err(NULL, 2286 "WARNING: Invalid rxq datasize %u\n", rxq->datasize); 2287 } 2288 2289 *rxq_size = 0; 2290 *cq_size = 0; 2291 2292 for (buf_idx = 0; buf_idx < rxq->num_rx_buf; buf_idx++) { 2293 rx_oob = &rxq->rx_oobs[buf_idx]; 2294 memset(rx_oob, 0, sizeof(*rx_oob)); 2295 2296 err = bus_dmamap_create(apc->rx_buf_tag, 0, 2297 &rx_oob->dma_map); 2298 if (err) { 2299 mana_err(NULL, 2300 "Failed to create rx DMA map for buf %d\n", 2301 buf_idx); 2302 return err; 2303 } 2304 2305 err = mana_load_rx_mbuf(apc, rxq, rx_oob, true); 2306 if (err) { 2307 mana_err(NULL, 2308 "Failed to create rx DMA map for buf %d\n", 2309 buf_idx); 2310 bus_dmamap_destroy(apc->rx_buf_tag, rx_oob->dma_map); 2311 return err; 2312 } 2313 2314 rx_oob->wqe_req.sgl = rx_oob->sgl; 2315 rx_oob->wqe_req.num_sge = rx_oob->num_sge; 2316 rx_oob->wqe_req.inline_oob_size = 0; 2317 rx_oob->wqe_req.inline_oob_data = NULL; 2318 rx_oob->wqe_req.flags = 0; 2319 rx_oob->wqe_req.client_data_unit = 0; 2320 2321 *rxq_size += ALIGN(MANA_WQE_HEADER_SIZE + 2322 MANA_WQE_SGE_SIZE * rx_oob->num_sge, 32); 2323 *cq_size += COMP_ENTRY_SIZE; 2324 } 2325 2326 return 0; 2327 } 2328 2329 static int 2330 mana_push_wqe(struct mana_rxq *rxq) 2331 { 2332 struct mana_recv_buf_oob *rx_oob; 2333 uint32_t buf_idx; 2334 int err; 2335 2336 for (buf_idx = 0; buf_idx < rxq->num_rx_buf; buf_idx++) { 2337 rx_oob = &rxq->rx_oobs[buf_idx]; 2338 2339 err = mana_gd_post_and_ring(rxq->gdma_rq, &rx_oob->wqe_req, 2340 &rx_oob->wqe_inf); 2341 if (err) 2342 return ENOSPC; 2343 } 2344 2345 return 0; 2346 } 2347 2348 static uint32_t 2349 mana_calc_rx_datasize(struct mana_port_context *apc) 2350 { 2351 uint32_t effective_mtu = 0; 2352 2353 if (apc->frame_size > MJUM16BYTES) { 2354 mana_err(NULL, "mana frame_size %u is too big\n", 2355 apc->frame_size); 2356 effective_mtu = MJUM16BYTES; 2357 } else if (apc->frame_size > MJUM9BYTES) { 2358 effective_mtu = MJUM16BYTES; 2359 } else if (apc->frame_size > MJUMPAGESIZE) { 2360 effective_mtu = MJUM9BYTES; 2361 } else if (apc->frame_size > MCLBYTES) { 2362 effective_mtu = MJUMPAGESIZE; 2363 } else { 2364 effective_mtu = MCLBYTES; 2365 } 2366 2367 return effective_mtu; 2368 } 2369 2370 static struct mana_rxq * 2371 mana_create_rxq(struct mana_port_context *apc, uint32_t rxq_idx, 2372 struct mana_eq *eq, if_t ndev) 2373 { 2374 struct gdma_dev *gd = apc->ac->gdma_dev; 2375 struct mana_obj_spec wq_spec; 2376 struct mana_obj_spec cq_spec; 2377 struct gdma_queue_spec spec; 2378 struct mana_cq *cq = NULL; 2379 uint32_t cq_size, rq_size; 2380 struct gdma_context *gc; 2381 struct mana_rxq *rxq; 2382 int err; 2383 2384 gc = gd->gdma_context; 2385 2386 rxq = malloc(sizeof(*rxq) + 2387 apc->rx_queue_size * sizeof(struct mana_recv_buf_oob), 2388 M_DEVBUF, M_WAITOK | M_ZERO); 2389 rxq->ndev = ndev; 2390 rxq->num_rx_buf = apc->rx_queue_size; 2391 rxq->rxq_idx = rxq_idx; 2392 2393 rxq->datasize = mana_calc_rx_datasize(apc); 2394 mana_dbg(NULL, "Setting rxq %d datasize %d\n", 2395 rxq_idx, rxq->datasize); 2396 2397 /* 2398 * Two steps to set the mbuf refill_thresh. 2399 * 1) If mana_rx_refill_threshold is set, honor it. 2400 * Set to default value otherwise. 2401 * 2) Select the smaller of 1) above and 1/4 of the 2402 * rx buffer size. 2403 */ 2404 if (mana_rx_refill_threshold != 0) 2405 rxq->refill_thresh = mana_rx_refill_threshold; 2406 else 2407 rxq->refill_thresh = MANA_RX_REFILL_THRESH; 2408 rxq->refill_thresh = min_t(uint32_t, 2409 rxq->num_rx_buf / 4, rxq->refill_thresh); 2410 2411 mana_dbg(NULL, "Setting rxq %d refill thresh %u\n", 2412 rxq_idx, rxq->refill_thresh); 2413 2414 rxq->rxobj = INVALID_MANA_HANDLE; 2415 2416 err = mana_alloc_rx_wqe(apc, rxq, &rq_size, &cq_size); 2417 if (err) 2418 goto out; 2419 2420 /* Create LRO for the RQ */ 2421 if (if_getcapenable(ndev) & IFCAP_LRO) { 2422 err = tcp_lro_init(&rxq->lro); 2423 if (err) { 2424 if_printf(ndev, "Failed to create LRO for rxq %d\n", 2425 rxq_idx); 2426 } else { 2427 rxq->lro.ifp = ndev; 2428 } 2429 } 2430 2431 mana_alloc_counters((counter_u64_t *)&rxq->stats, 2432 sizeof(rxq->stats)); 2433 2434 rq_size = ALIGN(rq_size, PAGE_SIZE); 2435 cq_size = ALIGN(cq_size, PAGE_SIZE); 2436 2437 /* Create RQ */ 2438 memset(&spec, 0, sizeof(spec)); 2439 spec.type = GDMA_RQ; 2440 spec.monitor_avl_buf = true; 2441 spec.queue_size = rq_size; 2442 err = mana_gd_create_mana_wq_cq(gd, &spec, &rxq->gdma_rq); 2443 if (err) 2444 goto out; 2445 2446 /* Create RQ's CQ */ 2447 cq = &rxq->rx_cq; 2448 cq->type = MANA_CQ_TYPE_RX; 2449 cq->rxq = rxq; 2450 2451 memset(&spec, 0, sizeof(spec)); 2452 spec.type = GDMA_CQ; 2453 spec.monitor_avl_buf = false; 2454 spec.queue_size = cq_size; 2455 spec.cq.callback = mana_schedule_task; 2456 spec.cq.parent_eq = eq->eq; 2457 spec.cq.context = cq; 2458 err = mana_gd_create_mana_wq_cq(gd, &spec, &cq->gdma_cq); 2459 if (err) 2460 goto out; 2461 2462 memset(&wq_spec, 0, sizeof(wq_spec)); 2463 memset(&cq_spec, 0, sizeof(cq_spec)); 2464 wq_spec.gdma_region = rxq->gdma_rq->mem_info.dma_region_handle; 2465 wq_spec.queue_size = rxq->gdma_rq->queue_size; 2466 2467 cq_spec.gdma_region = cq->gdma_cq->mem_info.dma_region_handle; 2468 cq_spec.queue_size = cq->gdma_cq->queue_size; 2469 cq_spec.modr_ctx_id = 0; 2470 cq_spec.attached_eq = cq->gdma_cq->cq.parent->id; 2471 2472 err = mana_create_wq_obj(apc, apc->port_handle, GDMA_RQ, 2473 &wq_spec, &cq_spec, &rxq->rxobj); 2474 if (err) 2475 goto out; 2476 2477 rxq->gdma_rq->id = wq_spec.queue_index; 2478 cq->gdma_cq->id = cq_spec.queue_index; 2479 2480 rxq->gdma_rq->mem_info.dma_region_handle = GDMA_INVALID_DMA_REGION; 2481 cq->gdma_cq->mem_info.dma_region_handle = GDMA_INVALID_DMA_REGION; 2482 2483 rxq->gdma_id = rxq->gdma_rq->id; 2484 cq->gdma_id = cq->gdma_cq->id; 2485 2486 err = mana_push_wqe(rxq); 2487 if (err) 2488 goto out; 2489 2490 if (cq->gdma_id >= gc->max_num_cqs) { 2491 err = EINVAL; 2492 goto out; 2493 } 2494 2495 gc->cq_table[cq->gdma_id] = cq->gdma_cq; 2496 2497 /* Allocate and start the cleanup task on CQ */ 2498 cq->do_not_ring_db = false; 2499 2500 NET_TASK_INIT(&cq->cleanup_task, 0, mana_poll, cq); 2501 cq->cleanup_tq = 2502 taskqueue_create_fast("mana rx cq cleanup", 2503 M_WAITOK, taskqueue_thread_enqueue, 2504 &cq->cleanup_tq); 2505 2506 if (apc->last_rx_cq_bind_cpu < 0) 2507 apc->last_rx_cq_bind_cpu = CPU_FIRST(); 2508 cq->cpu = apc->last_rx_cq_bind_cpu; 2509 apc->last_rx_cq_bind_cpu = CPU_NEXT(apc->last_rx_cq_bind_cpu); 2510 2511 if (apc->bind_cleanup_thread_cpu) { 2512 cpuset_t cpu_mask; 2513 CPU_SETOF(cq->cpu, &cpu_mask); 2514 taskqueue_start_threads_cpuset(&cq->cleanup_tq, 2515 1, PI_NET, &cpu_mask, 2516 "mana cq p%u-rx%u-cpu%d", 2517 apc->port_idx, rxq->rxq_idx, cq->cpu); 2518 } else { 2519 taskqueue_start_threads(&cq->cleanup_tq, 1, 2520 PI_NET, "mana cq p%u-rx%u", 2521 apc->port_idx, rxq->rxq_idx); 2522 } 2523 2524 mana_gd_ring_cq(cq->gdma_cq, SET_ARM_BIT); 2525 out: 2526 if (!err) 2527 return rxq; 2528 2529 if_printf(ndev, "Failed to create RXQ: err = %d\n", err); 2530 2531 mana_destroy_rxq(apc, rxq, false); 2532 2533 if (cq) 2534 mana_deinit_cq(apc, cq); 2535 2536 return NULL; 2537 } 2538 2539 static int 2540 mana_add_rx_queues(struct mana_port_context *apc, if_t ndev) 2541 { 2542 struct mana_context *ac = apc->ac; 2543 struct mana_rxq *rxq; 2544 int err = 0; 2545 int i; 2546 2547 for (i = 0; i < apc->num_queues; i++) { 2548 rxq = mana_create_rxq(apc, i, &ac->eqs[i], ndev); 2549 if (!rxq) { 2550 err = ENOMEM; 2551 goto out; 2552 } 2553 2554 apc->rxqs[i] = rxq; 2555 } 2556 2557 apc->default_rxobj = apc->rxqs[0]->rxobj; 2558 out: 2559 return err; 2560 } 2561 2562 static void 2563 mana_destroy_vport(struct mana_port_context *apc) 2564 { 2565 struct mana_rxq *rxq; 2566 uint32_t rxq_idx; 2567 2568 for (rxq_idx = 0; rxq_idx < apc->num_queues; rxq_idx++) { 2569 rxq = apc->rxqs[rxq_idx]; 2570 if (!rxq) 2571 continue; 2572 2573 mana_destroy_rxq(apc, rxq, true); 2574 apc->rxqs[rxq_idx] = NULL; 2575 } 2576 2577 mana_destroy_txq(apc); 2578 2579 mana_uncfg_vport(apc); 2580 } 2581 2582 static int 2583 mana_create_vport(struct mana_port_context *apc, if_t net) 2584 { 2585 struct gdma_dev *gd = apc->ac->gdma_dev; 2586 int err; 2587 2588 apc->default_rxobj = INVALID_MANA_HANDLE; 2589 2590 err = mana_cfg_vport(apc, gd->pdid, gd->doorbell); 2591 if (err) 2592 return err; 2593 2594 return mana_create_txq(apc, net); 2595 } 2596 2597 2598 static void mana_rss_table_init(struct mana_port_context *apc) 2599 { 2600 int i; 2601 2602 for (i = 0; i < MANA_INDIRECT_TABLE_SIZE; i++) 2603 apc->indir_table[i] = i % apc->num_queues; 2604 } 2605 2606 int mana_config_rss(struct mana_port_context *apc, enum TRI_STATE rx, 2607 bool update_hash, bool update_tab) 2608 { 2609 uint32_t queue_idx; 2610 int err; 2611 int i; 2612 2613 if (update_tab) { 2614 for (i = 0; i < MANA_INDIRECT_TABLE_SIZE; i++) { 2615 queue_idx = apc->indir_table[i]; 2616 apc->rxobj_table[i] = apc->rxqs[queue_idx]->rxobj; 2617 } 2618 } 2619 2620 err = mana_cfg_vport_steering(apc, rx, true, update_hash, update_tab); 2621 if (err) 2622 return err; 2623 2624 mana_fence_rqs(apc); 2625 2626 return 0; 2627 } 2628 2629 static int 2630 mana_init_port(if_t ndev) 2631 { 2632 struct mana_port_context *apc = if_getsoftc(ndev); 2633 uint32_t max_txq, max_rxq, max_queues; 2634 int port_idx = apc->port_idx; 2635 uint32_t num_indirect_entries; 2636 int err; 2637 2638 err = mana_init_port_context(apc); 2639 if (err) 2640 return err; 2641 2642 err = mana_query_vport_cfg(apc, port_idx, &max_txq, &max_rxq, 2643 &num_indirect_entries); 2644 if (err) { 2645 if_printf(ndev, "Failed to query info for vPort %d\n", 2646 port_idx); 2647 goto reset_apc; 2648 } 2649 2650 max_queues = min_t(uint32_t, max_txq, max_rxq); 2651 if (apc->max_queues > max_queues) 2652 apc->max_queues = max_queues; 2653 2654 if (apc->num_queues > apc->max_queues) 2655 apc->num_queues = apc->max_queues; 2656 2657 return 0; 2658 2659 reset_apc: 2660 bus_dma_tag_destroy(apc->rx_buf_tag); 2661 apc->rx_buf_tag = NULL; 2662 free(apc->rxqs, M_DEVBUF); 2663 apc->rxqs = NULL; 2664 return err; 2665 } 2666 2667 int 2668 mana_alloc_queues(if_t ndev) 2669 { 2670 struct mana_port_context *apc = if_getsoftc(ndev); 2671 int err; 2672 2673 err = mana_create_vport(apc, ndev); 2674 if (err) 2675 return err; 2676 2677 err = mana_add_rx_queues(apc, ndev); 2678 if (err) 2679 goto destroy_vport; 2680 2681 apc->rss_state = apc->num_queues > 1 ? TRI_STATE_TRUE : TRI_STATE_FALSE; 2682 2683 mana_rss_table_init(apc); 2684 2685 err = mana_config_rss(apc, TRI_STATE_TRUE, true, true); 2686 if (err) 2687 goto destroy_vport; 2688 2689 return 0; 2690 2691 destroy_vport: 2692 mana_destroy_vport(apc); 2693 return err; 2694 } 2695 2696 static int 2697 mana_up(struct mana_port_context *apc) 2698 { 2699 int err; 2700 2701 mana_dbg(NULL, "mana_up called\n"); 2702 2703 err = mana_alloc_queues(apc->ndev); 2704 if (err) { 2705 mana_err(NULL, "Faile alloc mana queues: %d\n", err); 2706 return err; 2707 } 2708 2709 /* Add queue specific sysctl */ 2710 mana_sysctl_add_queues(apc); 2711 2712 apc->port_is_up = true; 2713 2714 /* Ensure port state updated before txq state */ 2715 wmb(); 2716 2717 if_link_state_change(apc->ndev, LINK_STATE_UP); 2718 if_setdrvflagbits(apc->ndev, IFF_DRV_RUNNING, IFF_DRV_OACTIVE); 2719 2720 return 0; 2721 } 2722 2723 2724 static void 2725 mana_init(void *arg) 2726 { 2727 struct mana_port_context *apc = (struct mana_port_context *)arg; 2728 2729 MANA_APC_LOCK_LOCK(apc); 2730 if (!apc->port_is_up) { 2731 mana_up(apc); 2732 } 2733 MANA_APC_LOCK_UNLOCK(apc); 2734 } 2735 2736 static int 2737 mana_dealloc_queues(if_t ndev) 2738 { 2739 struct mana_port_context *apc = if_getsoftc(ndev); 2740 struct mana_txq *txq; 2741 int i, err; 2742 2743 if (apc->port_is_up) 2744 return EINVAL; 2745 2746 /* No packet can be transmitted now since apc->port_is_up is false. 2747 * There is still a tiny chance that mana_poll_tx_cq() can re-enable 2748 * a txq because it may not timely see apc->port_is_up being cleared 2749 * to false, but it doesn't matter since mana_start_xmit() drops any 2750 * new packets due to apc->port_is_up being false. 2751 * 2752 * Drain all the in-flight TX packets 2753 */ 2754 for (i = 0; i < apc->num_queues; i++) { 2755 txq = &apc->tx_qp[i].txq; 2756 2757 struct mana_cq *tx_cq = &apc->tx_qp[i].tx_cq; 2758 struct mana_cq *rx_cq = &(apc->rxqs[i]->rx_cq); 2759 2760 tx_cq->do_not_ring_db = true; 2761 rx_cq->do_not_ring_db = true; 2762 2763 /* Schedule a cleanup task */ 2764 taskqueue_enqueue(tx_cq->cleanup_tq, &tx_cq->cleanup_task); 2765 2766 while (atomic_read(&txq->pending_sends) > 0) 2767 usleep_range(1000, 2000); 2768 } 2769 2770 /* We're 100% sure the queues can no longer be woken up, because 2771 * we're sure now mana_poll_tx_cq() can't be running. 2772 */ 2773 2774 apc->rss_state = TRI_STATE_FALSE; 2775 err = mana_config_rss(apc, TRI_STATE_FALSE, false, false); 2776 if (err) { 2777 if_printf(ndev, "Failed to disable vPort: %d\n", err); 2778 return err; 2779 } 2780 2781 mana_destroy_vport(apc); 2782 2783 return 0; 2784 } 2785 2786 static int 2787 mana_down(struct mana_port_context *apc) 2788 { 2789 int err = 0; 2790 2791 apc->port_st_save = apc->port_is_up; 2792 apc->port_is_up = false; 2793 2794 /* Ensure port state updated before txq state */ 2795 wmb(); 2796 2797 if (apc->port_st_save) { 2798 if_setdrvflagbits(apc->ndev, IFF_DRV_OACTIVE, 2799 IFF_DRV_RUNNING); 2800 if_link_state_change(apc->ndev, LINK_STATE_DOWN); 2801 2802 mana_sysctl_free_queues(apc); 2803 2804 err = mana_dealloc_queues(apc->ndev); 2805 if (err) { 2806 if_printf(apc->ndev, 2807 "Failed to bring down mana interface: %d\n", err); 2808 } 2809 } 2810 2811 return err; 2812 } 2813 2814 int 2815 mana_detach(if_t ndev) 2816 { 2817 struct mana_port_context *apc = if_getsoftc(ndev); 2818 int err; 2819 2820 ether_ifdetach(ndev); 2821 2822 if (!apc) 2823 return 0; 2824 2825 MANA_APC_LOCK_LOCK(apc); 2826 err = mana_down(apc); 2827 MANA_APC_LOCK_UNLOCK(apc); 2828 2829 mana_cleanup_port_context(apc); 2830 2831 MANA_APC_LOCK_DESTROY(apc); 2832 2833 free(apc, M_DEVBUF); 2834 2835 return err; 2836 } 2837 2838 static unsigned int 2839 mana_get_tx_queue_size(int port_idx, unsigned int request_size) 2840 { 2841 unsigned int new_size; 2842 2843 if (request_size == 0) 2844 /* Uninitialized */ 2845 new_size = DEF_SEND_BUFFERS_PER_QUEUE; 2846 else 2847 new_size = roundup_pow_of_two(request_size); 2848 2849 if (new_size < MIN_SEND_BUFFERS_PER_QUEUE || 2850 new_size > MAX_SEND_BUFFERS_PER_QUEUE) { 2851 mana_info(NULL, "mana port %d: requested tx buffer " 2852 "size %u out of allowable range (%u - %u), " 2853 "setting to default\n", 2854 port_idx, request_size, 2855 MIN_SEND_BUFFERS_PER_QUEUE, 2856 MAX_SEND_BUFFERS_PER_QUEUE); 2857 new_size = DEF_SEND_BUFFERS_PER_QUEUE; 2858 } 2859 mana_info(NULL, "mana port %d: tx buffer size %u " 2860 "(%u requested)\n", 2861 port_idx, new_size, request_size); 2862 2863 return (new_size); 2864 } 2865 2866 static unsigned int 2867 mana_get_rx_queue_size(int port_idx, unsigned int request_size) 2868 { 2869 unsigned int new_size; 2870 2871 if (request_size == 0) 2872 /* Uninitialized */ 2873 new_size = DEF_RX_BUFFERS_PER_QUEUE; 2874 else 2875 new_size = roundup_pow_of_two(request_size); 2876 2877 if (new_size < MIN_RX_BUFFERS_PER_QUEUE || 2878 new_size > MAX_RX_BUFFERS_PER_QUEUE) { 2879 mana_info(NULL, "mana port %d: requested rx buffer " 2880 "size %u out of allowable range (%u - %u), " 2881 "setting to default\n", 2882 port_idx, request_size, 2883 MIN_RX_BUFFERS_PER_QUEUE, 2884 MAX_RX_BUFFERS_PER_QUEUE); 2885 new_size = DEF_RX_BUFFERS_PER_QUEUE; 2886 } 2887 mana_info(NULL, "mana port %d: rx buffer size %u " 2888 "(%u requested)\n", 2889 port_idx, new_size, request_size); 2890 2891 return (new_size); 2892 } 2893 2894 static int 2895 mana_probe_port(struct mana_context *ac, int port_idx, 2896 if_t *ndev_storage) 2897 { 2898 struct gdma_context *gc = ac->gdma_dev->gdma_context; 2899 struct mana_port_context *apc; 2900 uint32_t hwassist; 2901 if_t ndev; 2902 int err; 2903 2904 ndev = if_alloc_dev(IFT_ETHER, gc->dev); 2905 *ndev_storage = ndev; 2906 2907 apc = malloc(sizeof(*apc), M_DEVBUF, M_WAITOK | M_ZERO); 2908 apc->ac = ac; 2909 apc->ndev = ndev; 2910 apc->max_queues = gc->max_num_queues; 2911 apc->num_queues = min_t(unsigned int, 2912 gc->max_num_queues, MANA_MAX_NUM_QUEUES); 2913 apc->tx_queue_size = mana_get_tx_queue_size(port_idx, 2914 mana_tx_req_size); 2915 apc->rx_queue_size = mana_get_rx_queue_size(port_idx, 2916 mana_rx_req_size); 2917 apc->port_handle = INVALID_MANA_HANDLE; 2918 apc->port_idx = port_idx; 2919 apc->last_tx_cq_bind_cpu = -1; 2920 apc->last_rx_cq_bind_cpu = -1; 2921 apc->vport_use_count = 0; 2922 apc->max_mtu = gc->adapter_mtu - ETHER_HDR_LEN; 2923 apc->min_mtu = MIN_FRAME_SIZE; 2924 apc->mtu = ETHERMTU; 2925 apc->frame_size = apc->mtu + ETHER_HDR_LEN; 2926 2927 MANA_APC_LOCK_INIT(apc); 2928 2929 if_initname(ndev, device_get_name(gc->dev), port_idx); 2930 if_setdev(ndev,gc->dev); 2931 if_setsoftc(ndev, apc); 2932 2933 if_setflags(ndev, IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST); 2934 if_setinitfn(ndev, mana_init); 2935 if_settransmitfn(ndev, mana_start_xmit); 2936 if_setqflushfn(ndev, mana_qflush); 2937 if_setioctlfn(ndev, mana_ioctl); 2938 if_setgetcounterfn(ndev, mana_get_counter); 2939 2940 if_setmtu(ndev, apc->mtu); 2941 if_setbaudrate(ndev, IF_Gbps(100)); 2942 2943 _Static_assert(sizeof(apc->hashkey) == RSS_KEYSIZE, 2944 "RSS key size mismatch"); 2945 rss_getkey(apc->hashkey); 2946 2947 err = mana_init_port(ndev); 2948 if (err) 2949 goto reset_apc; 2950 2951 if_setcapabilitiesbit(ndev, 2952 IFCAP_TXCSUM | IFCAP_TXCSUM_IPV6 | 2953 IFCAP_RXCSUM | IFCAP_RXCSUM_IPV6 | 2954 IFCAP_TSO4 | IFCAP_TSO6 | 2955 IFCAP_LRO | IFCAP_LINKSTATE, 0); 2956 2957 /* Enable all available capabilities by default. */ 2958 if_setcapenable(ndev, if_getcapabilities(ndev)); 2959 2960 /* TSO parameters */ 2961 if_sethwtsomax(ndev, MANA_TSO_MAX_SZ - 2962 (ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN)); 2963 if_sethwtsomaxsegcount(ndev, MAX_MBUF_FRAGS); 2964 if_sethwtsomaxsegsize(ndev, PAGE_SIZE); 2965 2966 hwassist = 0; 2967 if (if_getcapenable(ndev) & (IFCAP_TSO4 | IFCAP_TSO6)) 2968 hwassist |= CSUM_TSO; 2969 if (if_getcapenable(ndev) & IFCAP_TXCSUM) 2970 hwassist |= (CSUM_TCP | CSUM_UDP | CSUM_IP); 2971 if (if_getcapenable(ndev) & IFCAP_TXCSUM_IPV6) 2972 hwassist |= (CSUM_UDP_IPV6 | CSUM_TCP_IPV6); 2973 mana_dbg(NULL, "set hwassist 0x%x\n", hwassist); 2974 if_sethwassist(ndev, hwassist); 2975 2976 ifmedia_init(&apc->media, IFM_IMASK, 2977 mana_ifmedia_change, mana_ifmedia_status); 2978 ifmedia_add(&apc->media, IFM_ETHER | IFM_AUTO, 0, NULL); 2979 ifmedia_set(&apc->media, IFM_ETHER | IFM_AUTO); 2980 2981 ether_ifattach(ndev, apc->mac_addr); 2982 2983 /* Initialize statistics */ 2984 mana_alloc_counters((counter_u64_t *)&apc->port_stats, 2985 sizeof(struct mana_port_stats)); 2986 mana_sysctl_add_port(apc); 2987 2988 /* Tell the stack that the interface is not active */ 2989 if_setdrvflagbits(ndev, IFF_DRV_OACTIVE, IFF_DRV_RUNNING); 2990 2991 return 0; 2992 2993 reset_apc: 2994 free(apc, M_DEVBUF); 2995 *ndev_storage = NULL; 2996 if_printf(ndev, "Failed to probe vPort %d: %d\n", port_idx, err); 2997 if_free(ndev); 2998 return err; 2999 } 3000 3001 int mana_probe(struct gdma_dev *gd) 3002 { 3003 struct gdma_context *gc = gd->gdma_context; 3004 device_t dev = gc->dev; 3005 struct mana_context *ac; 3006 int err; 3007 int i; 3008 3009 device_printf(dev, "%s protocol version: %d.%d.%d\n", DEVICE_NAME, 3010 MANA_MAJOR_VERSION, MANA_MINOR_VERSION, MANA_MICRO_VERSION); 3011 3012 err = mana_gd_register_device(gd); 3013 if (err) 3014 return err; 3015 3016 ac = malloc(sizeof(*ac), M_DEVBUF, M_WAITOK | M_ZERO); 3017 ac->gdma_dev = gd; 3018 ac->num_ports = 1; 3019 gd->driver_data = ac; 3020 3021 err = mana_create_eq(ac); 3022 if (err) 3023 goto out; 3024 3025 err = mana_query_device_cfg(ac, MANA_MAJOR_VERSION, MANA_MINOR_VERSION, 3026 MANA_MICRO_VERSION, &ac->num_ports); 3027 if (err) 3028 goto out; 3029 3030 if (ac->num_ports > MAX_PORTS_IN_MANA_DEV) 3031 ac->num_ports = MAX_PORTS_IN_MANA_DEV; 3032 3033 for (i = 0; i < ac->num_ports; i++) { 3034 err = mana_probe_port(ac, i, &ac->ports[i]); 3035 if (err) { 3036 device_printf(dev, 3037 "Failed to probe mana port %d\n", i); 3038 break; 3039 } 3040 } 3041 3042 out: 3043 if (err) 3044 mana_remove(gd); 3045 3046 return err; 3047 } 3048 3049 void 3050 mana_remove(struct gdma_dev *gd) 3051 { 3052 struct gdma_context *gc = gd->gdma_context; 3053 struct mana_context *ac = gd->driver_data; 3054 device_t dev = gc->dev; 3055 if_t ndev; 3056 int i; 3057 3058 for (i = 0; i < ac->num_ports; i++) { 3059 ndev = ac->ports[i]; 3060 if (!ndev) { 3061 if (i == 0) 3062 device_printf(dev, "No net device to remove\n"); 3063 goto out; 3064 } 3065 3066 mana_detach(ndev); 3067 3068 if_free(ndev); 3069 } 3070 3071 mana_destroy_eq(ac); 3072 3073 out: 3074 mana_gd_deregister_device(gd); 3075 gd->driver_data = NULL; 3076 gd->gdma_context = NULL; 3077 free(ac, M_DEVBUF); 3078 } 3079