1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3 * 4 * Copyright (c) 2011 Chelsio Communications, Inc. 5 * All rights reserved. 6 * Written by: Navdeep Parhar <np@FreeBSD.org> 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27 * SUCH DAMAGE. 28 */ 29 30 #include <sys/cdefs.h> 31 __FBSDID("$FreeBSD$"); 32 33 #include "opt_inet.h" 34 #include "opt_inet6.h" 35 #include "opt_kern_tls.h" 36 #include "opt_ratelimit.h" 37 38 #include <sys/types.h> 39 #include <sys/eventhandler.h> 40 #include <sys/mbuf.h> 41 #include <sys/socket.h> 42 #include <sys/kernel.h> 43 #include <sys/ktls.h> 44 #include <sys/malloc.h> 45 #include <sys/queue.h> 46 #include <sys/sbuf.h> 47 #include <sys/taskqueue.h> 48 #include <sys/time.h> 49 #include <sys/sglist.h> 50 #include <sys/sysctl.h> 51 #include <sys/smp.h> 52 #include <sys/socketvar.h> 53 #include <sys/counter.h> 54 #include <net/bpf.h> 55 #include <net/ethernet.h> 56 #include <net/if.h> 57 #include <net/if_vlan_var.h> 58 #include <net/if_vxlan.h> 59 #include <netinet/in.h> 60 #include <netinet/ip.h> 61 #include <netinet/ip6.h> 62 #include <netinet/tcp.h> 63 #include <netinet/udp.h> 64 #include <machine/in_cksum.h> 65 #include <machine/md_var.h> 66 #include <vm/vm.h> 67 #include <vm/pmap.h> 68 #ifdef DEV_NETMAP 69 #include <machine/bus.h> 70 #include <sys/selinfo.h> 71 #include <net/if_var.h> 72 #include <net/netmap.h> 73 #include <dev/netmap/netmap_kern.h> 74 #endif 75 76 #include "common/common.h" 77 #include "common/t4_regs.h" 78 #include "common/t4_regs_values.h" 79 #include "common/t4_msg.h" 80 #include "t4_l2t.h" 81 #include "t4_mp_ring.h" 82 83 #ifdef T4_PKT_TIMESTAMP 84 #define RX_COPY_THRESHOLD (MINCLSIZE - 8) 85 #else 86 #define RX_COPY_THRESHOLD MINCLSIZE 87 #endif 88 89 /* Internal mbuf flags stored in PH_loc.eight[1]. */ 90 #define MC_NOMAP 0x01 91 #define MC_RAW_WR 0x02 92 #define MC_TLS 0x04 93 94 /* 95 * Ethernet frames are DMA'd at this byte offset into the freelist buffer. 96 * 0-7 are valid values. 97 */ 98 static int fl_pktshift = 0; 99 SYSCTL_INT(_hw_cxgbe, OID_AUTO, fl_pktshift, CTLFLAG_RDTUN, &fl_pktshift, 0, 100 "payload DMA offset in rx buffer (bytes)"); 101 102 /* 103 * Pad ethernet payload up to this boundary. 104 * -1: driver should figure out a good value. 105 * 0: disable padding. 106 * Any power of 2 from 32 to 4096 (both inclusive) is also a valid value. 107 */ 108 int fl_pad = -1; 109 SYSCTL_INT(_hw_cxgbe, OID_AUTO, fl_pad, CTLFLAG_RDTUN, &fl_pad, 0, 110 "payload pad boundary (bytes)"); 111 112 /* 113 * Status page length. 114 * -1: driver should figure out a good value. 115 * 64 or 128 are the only other valid values. 116 */ 117 static int spg_len = -1; 118 SYSCTL_INT(_hw_cxgbe, OID_AUTO, spg_len, CTLFLAG_RDTUN, &spg_len, 0, 119 "status page size (bytes)"); 120 121 /* 122 * Congestion drops. 123 * -1: no congestion feedback (not recommended). 124 * 0: backpressure the channel instead of dropping packets right away. 125 * 1: no backpressure, drop packets for the congested queue immediately. 126 */ 127 static int cong_drop = 0; 128 SYSCTL_INT(_hw_cxgbe, OID_AUTO, cong_drop, CTLFLAG_RDTUN, &cong_drop, 0, 129 "Congestion control for RX queues (0 = backpressure, 1 = drop"); 130 131 /* 132 * Deliver multiple frames in the same free list buffer if they fit. 133 * -1: let the driver decide whether to enable buffer packing or not. 134 * 0: disable buffer packing. 135 * 1: enable buffer packing. 136 */ 137 static int buffer_packing = -1; 138 SYSCTL_INT(_hw_cxgbe, OID_AUTO, buffer_packing, CTLFLAG_RDTUN, &buffer_packing, 139 0, "Enable buffer packing"); 140 141 /* 142 * Start next frame in a packed buffer at this boundary. 143 * -1: driver should figure out a good value. 144 * T4: driver will ignore this and use the same value as fl_pad above. 145 * T5: 16, or a power of 2 from 64 to 4096 (both inclusive) is a valid value. 146 */ 147 static int fl_pack = -1; 148 SYSCTL_INT(_hw_cxgbe, OID_AUTO, fl_pack, CTLFLAG_RDTUN, &fl_pack, 0, 149 "payload pack boundary (bytes)"); 150 151 /* 152 * Largest rx cluster size that the driver is allowed to allocate. 153 */ 154 static int largest_rx_cluster = MJUM16BYTES; 155 SYSCTL_INT(_hw_cxgbe, OID_AUTO, largest_rx_cluster, CTLFLAG_RDTUN, 156 &largest_rx_cluster, 0, "Largest rx cluster (bytes)"); 157 158 /* 159 * Size of cluster allocation that's most likely to succeed. The driver will 160 * fall back to this size if it fails to allocate clusters larger than this. 161 */ 162 static int safest_rx_cluster = PAGE_SIZE; 163 SYSCTL_INT(_hw_cxgbe, OID_AUTO, safest_rx_cluster, CTLFLAG_RDTUN, 164 &safest_rx_cluster, 0, "Safe rx cluster (bytes)"); 165 166 #ifdef RATELIMIT 167 /* 168 * Knob to control TCP timestamp rewriting, and the granularity of the tick used 169 * for rewriting. -1 and 0-3 are all valid values. 170 * -1: hardware should leave the TCP timestamps alone. 171 * 0: 1ms 172 * 1: 100us 173 * 2: 10us 174 * 3: 1us 175 */ 176 static int tsclk = -1; 177 SYSCTL_INT(_hw_cxgbe, OID_AUTO, tsclk, CTLFLAG_RDTUN, &tsclk, 0, 178 "Control TCP timestamp rewriting when using pacing"); 179 180 static int eo_max_backlog = 1024 * 1024; 181 SYSCTL_INT(_hw_cxgbe, OID_AUTO, eo_max_backlog, CTLFLAG_RDTUN, &eo_max_backlog, 182 0, "Maximum backlog of ratelimited data per flow"); 183 #endif 184 185 /* 186 * The interrupt holdoff timers are multiplied by this value on T6+. 187 * 1 and 3-17 (both inclusive) are legal values. 188 */ 189 static int tscale = 1; 190 SYSCTL_INT(_hw_cxgbe, OID_AUTO, tscale, CTLFLAG_RDTUN, &tscale, 0, 191 "Interrupt holdoff timer scale on T6+"); 192 193 /* 194 * Number of LRO entries in the lro_ctrl structure per rx queue. 195 */ 196 static int lro_entries = TCP_LRO_ENTRIES; 197 SYSCTL_INT(_hw_cxgbe, OID_AUTO, lro_entries, CTLFLAG_RDTUN, &lro_entries, 0, 198 "Number of LRO entries per RX queue"); 199 200 /* 201 * This enables presorting of frames before they're fed into tcp_lro_rx. 202 */ 203 static int lro_mbufs = 0; 204 SYSCTL_INT(_hw_cxgbe, OID_AUTO, lro_mbufs, CTLFLAG_RDTUN, &lro_mbufs, 0, 205 "Enable presorting of LRO frames"); 206 207 static counter_u64_t pullups; 208 SYSCTL_COUNTER_U64(_hw_cxgbe, OID_AUTO, pullups, CTLFLAG_RD, &pullups, 209 "Number of mbuf pullups performed"); 210 211 static counter_u64_t defrags; 212 SYSCTL_COUNTER_U64(_hw_cxgbe, OID_AUTO, defrags, CTLFLAG_RD, &defrags, 213 "Number of mbuf defrags performed"); 214 215 216 static int service_iq(struct sge_iq *, int); 217 static int service_iq_fl(struct sge_iq *, int); 218 static struct mbuf *get_fl_payload(struct adapter *, struct sge_fl *, uint32_t); 219 static int eth_rx(struct adapter *, struct sge_rxq *, const struct iq_desc *, 220 u_int); 221 static inline void init_iq(struct sge_iq *, struct adapter *, int, int, int); 222 static inline void init_fl(struct adapter *, struct sge_fl *, int, int, char *); 223 static inline void init_eq(struct adapter *, struct sge_eq *, int, int, uint8_t, 224 uint16_t, char *); 225 static int alloc_ring(struct adapter *, size_t, bus_dma_tag_t *, bus_dmamap_t *, 226 bus_addr_t *, void **); 227 static int free_ring(struct adapter *, bus_dma_tag_t, bus_dmamap_t, bus_addr_t, 228 void *); 229 static int alloc_iq_fl(struct vi_info *, struct sge_iq *, struct sge_fl *, 230 int, int); 231 static int free_iq_fl(struct vi_info *, struct sge_iq *, struct sge_fl *); 232 static void add_iq_sysctls(struct sysctl_ctx_list *, struct sysctl_oid *, 233 struct sge_iq *); 234 static void add_fl_sysctls(struct adapter *, struct sysctl_ctx_list *, 235 struct sysctl_oid *, struct sge_fl *); 236 static int alloc_fwq(struct adapter *); 237 static int free_fwq(struct adapter *); 238 static int alloc_ctrlq(struct adapter *, struct sge_wrq *, int, 239 struct sysctl_oid *); 240 static int alloc_rxq(struct vi_info *, struct sge_rxq *, int, int, 241 struct sysctl_oid *); 242 static int free_rxq(struct vi_info *, struct sge_rxq *); 243 #ifdef TCP_OFFLOAD 244 static int alloc_ofld_rxq(struct vi_info *, struct sge_ofld_rxq *, int, int, 245 struct sysctl_oid *); 246 static int free_ofld_rxq(struct vi_info *, struct sge_ofld_rxq *); 247 #endif 248 #ifdef DEV_NETMAP 249 static int alloc_nm_rxq(struct vi_info *, struct sge_nm_rxq *, int, int, 250 struct sysctl_oid *); 251 static int free_nm_rxq(struct vi_info *, struct sge_nm_rxq *); 252 static int alloc_nm_txq(struct vi_info *, struct sge_nm_txq *, int, int, 253 struct sysctl_oid *); 254 static int free_nm_txq(struct vi_info *, struct sge_nm_txq *); 255 #endif 256 static int ctrl_eq_alloc(struct adapter *, struct sge_eq *); 257 static int eth_eq_alloc(struct adapter *, struct vi_info *, struct sge_eq *); 258 #if defined(TCP_OFFLOAD) || defined(RATELIMIT) 259 static int ofld_eq_alloc(struct adapter *, struct vi_info *, struct sge_eq *); 260 #endif 261 static int alloc_eq(struct adapter *, struct vi_info *, struct sge_eq *); 262 static int free_eq(struct adapter *, struct sge_eq *); 263 static int alloc_wrq(struct adapter *, struct vi_info *, struct sge_wrq *, 264 struct sysctl_oid *); 265 static int free_wrq(struct adapter *, struct sge_wrq *); 266 static int alloc_txq(struct vi_info *, struct sge_txq *, int, 267 struct sysctl_oid *); 268 static int free_txq(struct vi_info *, struct sge_txq *); 269 static void oneseg_dma_callback(void *, bus_dma_segment_t *, int, int); 270 static inline void ring_fl_db(struct adapter *, struct sge_fl *); 271 static int refill_fl(struct adapter *, struct sge_fl *, int); 272 static void refill_sfl(void *); 273 static int alloc_fl_sdesc(struct sge_fl *); 274 static void free_fl_sdesc(struct adapter *, struct sge_fl *); 275 static int find_refill_source(struct adapter *, int, bool); 276 static void add_fl_to_sfl(struct adapter *, struct sge_fl *); 277 278 static inline void get_pkt_gl(struct mbuf *, struct sglist *); 279 static inline u_int txpkt_len16(u_int, const u_int); 280 static inline u_int txpkt_vm_len16(u_int, const u_int); 281 static inline void calculate_mbuf_len16(struct mbuf *, bool); 282 static inline u_int txpkts0_len16(u_int); 283 static inline u_int txpkts1_len16(void); 284 static u_int write_raw_wr(struct sge_txq *, void *, struct mbuf *, u_int); 285 static u_int write_txpkt_wr(struct adapter *, struct sge_txq *, struct mbuf *, 286 u_int); 287 static u_int write_txpkt_vm_wr(struct adapter *, struct sge_txq *, 288 struct mbuf *); 289 static int add_to_txpkts_vf(struct adapter *, struct sge_txq *, struct mbuf *, 290 int, bool *); 291 static int add_to_txpkts_pf(struct adapter *, struct sge_txq *, struct mbuf *, 292 int, bool *); 293 static u_int write_txpkts_wr(struct adapter *, struct sge_txq *); 294 static u_int write_txpkts_vm_wr(struct adapter *, struct sge_txq *); 295 static void write_gl_to_txd(struct sge_txq *, struct mbuf *, caddr_t *, int); 296 static inline void copy_to_txd(struct sge_eq *, caddr_t, caddr_t *, int); 297 static inline void ring_eq_db(struct adapter *, struct sge_eq *, u_int); 298 static inline uint16_t read_hw_cidx(struct sge_eq *); 299 static inline u_int reclaimable_tx_desc(struct sge_eq *); 300 static inline u_int total_available_tx_desc(struct sge_eq *); 301 static u_int reclaim_tx_descs(struct sge_txq *, u_int); 302 static void tx_reclaim(void *, int); 303 static __be64 get_flit(struct sglist_seg *, int, int); 304 static int handle_sge_egr_update(struct sge_iq *, const struct rss_header *, 305 struct mbuf *); 306 static int handle_fw_msg(struct sge_iq *, const struct rss_header *, 307 struct mbuf *); 308 static int t4_handle_wrerr_rpl(struct adapter *, const __be64 *); 309 static void wrq_tx_drain(void *, int); 310 static void drain_wrq_wr_list(struct adapter *, struct sge_wrq *); 311 312 static int sysctl_uint16(SYSCTL_HANDLER_ARGS); 313 static int sysctl_bufsizes(SYSCTL_HANDLER_ARGS); 314 #ifdef RATELIMIT 315 static inline u_int txpkt_eo_len16(u_int, u_int, u_int); 316 static int ethofld_fw4_ack(struct sge_iq *, const struct rss_header *, 317 struct mbuf *); 318 #endif 319 320 static counter_u64_t extfree_refs; 321 static counter_u64_t extfree_rels; 322 323 an_handler_t t4_an_handler; 324 fw_msg_handler_t t4_fw_msg_handler[NUM_FW6_TYPES]; 325 cpl_handler_t t4_cpl_handler[NUM_CPL_CMDS]; 326 cpl_handler_t set_tcb_rpl_handlers[NUM_CPL_COOKIES]; 327 cpl_handler_t l2t_write_rpl_handlers[NUM_CPL_COOKIES]; 328 cpl_handler_t act_open_rpl_handlers[NUM_CPL_COOKIES]; 329 cpl_handler_t abort_rpl_rss_handlers[NUM_CPL_COOKIES]; 330 cpl_handler_t fw4_ack_handlers[NUM_CPL_COOKIES]; 331 332 void 333 t4_register_an_handler(an_handler_t h) 334 { 335 uintptr_t *loc; 336 337 MPASS(h == NULL || t4_an_handler == NULL); 338 339 loc = (uintptr_t *)&t4_an_handler; 340 atomic_store_rel_ptr(loc, (uintptr_t)h); 341 } 342 343 void 344 t4_register_fw_msg_handler(int type, fw_msg_handler_t h) 345 { 346 uintptr_t *loc; 347 348 MPASS(type < nitems(t4_fw_msg_handler)); 349 MPASS(h == NULL || t4_fw_msg_handler[type] == NULL); 350 /* 351 * These are dispatched by the handler for FW{4|6}_CPL_MSG using the CPL 352 * handler dispatch table. Reject any attempt to install a handler for 353 * this subtype. 354 */ 355 MPASS(type != FW_TYPE_RSSCPL); 356 MPASS(type != FW6_TYPE_RSSCPL); 357 358 loc = (uintptr_t *)&t4_fw_msg_handler[type]; 359 atomic_store_rel_ptr(loc, (uintptr_t)h); 360 } 361 362 void 363 t4_register_cpl_handler(int opcode, cpl_handler_t h) 364 { 365 uintptr_t *loc; 366 367 MPASS(opcode < nitems(t4_cpl_handler)); 368 MPASS(h == NULL || t4_cpl_handler[opcode] == NULL); 369 370 loc = (uintptr_t *)&t4_cpl_handler[opcode]; 371 atomic_store_rel_ptr(loc, (uintptr_t)h); 372 } 373 374 static int 375 set_tcb_rpl_handler(struct sge_iq *iq, const struct rss_header *rss, 376 struct mbuf *m) 377 { 378 const struct cpl_set_tcb_rpl *cpl = (const void *)(rss + 1); 379 u_int tid; 380 int cookie; 381 382 MPASS(m == NULL); 383 384 tid = GET_TID(cpl); 385 if (is_hpftid(iq->adapter, tid) || is_ftid(iq->adapter, tid)) { 386 /* 387 * The return code for filter-write is put in the CPL cookie so 388 * we have to rely on the hardware tid (is_ftid) to determine 389 * that this is a response to a filter. 390 */ 391 cookie = CPL_COOKIE_FILTER; 392 } else { 393 cookie = G_COOKIE(cpl->cookie); 394 } 395 MPASS(cookie > CPL_COOKIE_RESERVED); 396 MPASS(cookie < nitems(set_tcb_rpl_handlers)); 397 398 return (set_tcb_rpl_handlers[cookie](iq, rss, m)); 399 } 400 401 static int 402 l2t_write_rpl_handler(struct sge_iq *iq, const struct rss_header *rss, 403 struct mbuf *m) 404 { 405 const struct cpl_l2t_write_rpl *rpl = (const void *)(rss + 1); 406 unsigned int cookie; 407 408 MPASS(m == NULL); 409 410 cookie = GET_TID(rpl) & F_SYNC_WR ? CPL_COOKIE_TOM : CPL_COOKIE_FILTER; 411 return (l2t_write_rpl_handlers[cookie](iq, rss, m)); 412 } 413 414 static int 415 act_open_rpl_handler(struct sge_iq *iq, const struct rss_header *rss, 416 struct mbuf *m) 417 { 418 const struct cpl_act_open_rpl *cpl = (const void *)(rss + 1); 419 u_int cookie = G_TID_COOKIE(G_AOPEN_ATID(be32toh(cpl->atid_status))); 420 421 MPASS(m == NULL); 422 MPASS(cookie != CPL_COOKIE_RESERVED); 423 424 return (act_open_rpl_handlers[cookie](iq, rss, m)); 425 } 426 427 static int 428 abort_rpl_rss_handler(struct sge_iq *iq, const struct rss_header *rss, 429 struct mbuf *m) 430 { 431 struct adapter *sc = iq->adapter; 432 u_int cookie; 433 434 MPASS(m == NULL); 435 if (is_hashfilter(sc)) 436 cookie = CPL_COOKIE_HASHFILTER; 437 else 438 cookie = CPL_COOKIE_TOM; 439 440 return (abort_rpl_rss_handlers[cookie](iq, rss, m)); 441 } 442 443 static int 444 fw4_ack_handler(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) 445 { 446 struct adapter *sc = iq->adapter; 447 const struct cpl_fw4_ack *cpl = (const void *)(rss + 1); 448 unsigned int tid = G_CPL_FW4_ACK_FLOWID(be32toh(OPCODE_TID(cpl))); 449 u_int cookie; 450 451 MPASS(m == NULL); 452 if (is_etid(sc, tid)) 453 cookie = CPL_COOKIE_ETHOFLD; 454 else 455 cookie = CPL_COOKIE_TOM; 456 457 return (fw4_ack_handlers[cookie](iq, rss, m)); 458 } 459 460 static void 461 t4_init_shared_cpl_handlers(void) 462 { 463 464 t4_register_cpl_handler(CPL_SET_TCB_RPL, set_tcb_rpl_handler); 465 t4_register_cpl_handler(CPL_L2T_WRITE_RPL, l2t_write_rpl_handler); 466 t4_register_cpl_handler(CPL_ACT_OPEN_RPL, act_open_rpl_handler); 467 t4_register_cpl_handler(CPL_ABORT_RPL_RSS, abort_rpl_rss_handler); 468 t4_register_cpl_handler(CPL_FW4_ACK, fw4_ack_handler); 469 } 470 471 void 472 t4_register_shared_cpl_handler(int opcode, cpl_handler_t h, int cookie) 473 { 474 uintptr_t *loc; 475 476 MPASS(opcode < nitems(t4_cpl_handler)); 477 MPASS(cookie > CPL_COOKIE_RESERVED); 478 MPASS(cookie < NUM_CPL_COOKIES); 479 MPASS(t4_cpl_handler[opcode] != NULL); 480 481 switch (opcode) { 482 case CPL_SET_TCB_RPL: 483 loc = (uintptr_t *)&set_tcb_rpl_handlers[cookie]; 484 break; 485 case CPL_L2T_WRITE_RPL: 486 loc = (uintptr_t *)&l2t_write_rpl_handlers[cookie]; 487 break; 488 case CPL_ACT_OPEN_RPL: 489 loc = (uintptr_t *)&act_open_rpl_handlers[cookie]; 490 break; 491 case CPL_ABORT_RPL_RSS: 492 loc = (uintptr_t *)&abort_rpl_rss_handlers[cookie]; 493 break; 494 case CPL_FW4_ACK: 495 loc = (uintptr_t *)&fw4_ack_handlers[cookie]; 496 break; 497 default: 498 MPASS(0); 499 return; 500 } 501 MPASS(h == NULL || *loc == (uintptr_t)NULL); 502 atomic_store_rel_ptr(loc, (uintptr_t)h); 503 } 504 505 /* 506 * Called on MOD_LOAD. Validates and calculates the SGE tunables. 507 */ 508 void 509 t4_sge_modload(void) 510 { 511 512 if (fl_pktshift < 0 || fl_pktshift > 7) { 513 printf("Invalid hw.cxgbe.fl_pktshift value (%d)," 514 " using 0 instead.\n", fl_pktshift); 515 fl_pktshift = 0; 516 } 517 518 if (spg_len != 64 && spg_len != 128) { 519 int len; 520 521 #if defined(__i386__) || defined(__amd64__) 522 len = cpu_clflush_line_size > 64 ? 128 : 64; 523 #else 524 len = 64; 525 #endif 526 if (spg_len != -1) { 527 printf("Invalid hw.cxgbe.spg_len value (%d)," 528 " using %d instead.\n", spg_len, len); 529 } 530 spg_len = len; 531 } 532 533 if (cong_drop < -1 || cong_drop > 1) { 534 printf("Invalid hw.cxgbe.cong_drop value (%d)," 535 " using 0 instead.\n", cong_drop); 536 cong_drop = 0; 537 } 538 539 if (tscale != 1 && (tscale < 3 || tscale > 17)) { 540 printf("Invalid hw.cxgbe.tscale value (%d)," 541 " using 1 instead.\n", tscale); 542 tscale = 1; 543 } 544 545 extfree_refs = counter_u64_alloc(M_WAITOK); 546 extfree_rels = counter_u64_alloc(M_WAITOK); 547 pullups = counter_u64_alloc(M_WAITOK); 548 defrags = counter_u64_alloc(M_WAITOK); 549 counter_u64_zero(extfree_refs); 550 counter_u64_zero(extfree_rels); 551 counter_u64_zero(pullups); 552 counter_u64_zero(defrags); 553 554 t4_init_shared_cpl_handlers(); 555 t4_register_cpl_handler(CPL_FW4_MSG, handle_fw_msg); 556 t4_register_cpl_handler(CPL_FW6_MSG, handle_fw_msg); 557 t4_register_cpl_handler(CPL_SGE_EGR_UPDATE, handle_sge_egr_update); 558 #ifdef RATELIMIT 559 t4_register_shared_cpl_handler(CPL_FW4_ACK, ethofld_fw4_ack, 560 CPL_COOKIE_ETHOFLD); 561 #endif 562 t4_register_fw_msg_handler(FW6_TYPE_CMD_RPL, t4_handle_fw_rpl); 563 t4_register_fw_msg_handler(FW6_TYPE_WRERR_RPL, t4_handle_wrerr_rpl); 564 } 565 566 void 567 t4_sge_modunload(void) 568 { 569 570 counter_u64_free(extfree_refs); 571 counter_u64_free(extfree_rels); 572 counter_u64_free(pullups); 573 counter_u64_free(defrags); 574 } 575 576 uint64_t 577 t4_sge_extfree_refs(void) 578 { 579 uint64_t refs, rels; 580 581 rels = counter_u64_fetch(extfree_rels); 582 refs = counter_u64_fetch(extfree_refs); 583 584 return (refs - rels); 585 } 586 587 /* max 4096 */ 588 #define MAX_PACK_BOUNDARY 512 589 590 static inline void 591 setup_pad_and_pack_boundaries(struct adapter *sc) 592 { 593 uint32_t v, m; 594 int pad, pack, pad_shift; 595 596 pad_shift = chip_id(sc) > CHELSIO_T5 ? X_T6_INGPADBOUNDARY_SHIFT : 597 X_INGPADBOUNDARY_SHIFT; 598 pad = fl_pad; 599 if (fl_pad < (1 << pad_shift) || 600 fl_pad > (1 << (pad_shift + M_INGPADBOUNDARY)) || 601 !powerof2(fl_pad)) { 602 /* 603 * If there is any chance that we might use buffer packing and 604 * the chip is a T4, then pick 64 as the pad/pack boundary. Set 605 * it to the minimum allowed in all other cases. 606 */ 607 pad = is_t4(sc) && buffer_packing ? 64 : 1 << pad_shift; 608 609 /* 610 * For fl_pad = 0 we'll still write a reasonable value to the 611 * register but all the freelists will opt out of padding. 612 * We'll complain here only if the user tried to set it to a 613 * value greater than 0 that was invalid. 614 */ 615 if (fl_pad > 0) { 616 device_printf(sc->dev, "Invalid hw.cxgbe.fl_pad value" 617 " (%d), using %d instead.\n", fl_pad, pad); 618 } 619 } 620 m = V_INGPADBOUNDARY(M_INGPADBOUNDARY); 621 v = V_INGPADBOUNDARY(ilog2(pad) - pad_shift); 622 t4_set_reg_field(sc, A_SGE_CONTROL, m, v); 623 624 if (is_t4(sc)) { 625 if (fl_pack != -1 && fl_pack != pad) { 626 /* Complain but carry on. */ 627 device_printf(sc->dev, "hw.cxgbe.fl_pack (%d) ignored," 628 " using %d instead.\n", fl_pack, pad); 629 } 630 return; 631 } 632 633 pack = fl_pack; 634 if (fl_pack < 16 || fl_pack == 32 || fl_pack > 4096 || 635 !powerof2(fl_pack)) { 636 if (sc->params.pci.mps > MAX_PACK_BOUNDARY) 637 pack = MAX_PACK_BOUNDARY; 638 else 639 pack = max(sc->params.pci.mps, CACHE_LINE_SIZE); 640 MPASS(powerof2(pack)); 641 if (pack < 16) 642 pack = 16; 643 if (pack == 32) 644 pack = 64; 645 if (pack > 4096) 646 pack = 4096; 647 if (fl_pack != -1) { 648 device_printf(sc->dev, "Invalid hw.cxgbe.fl_pack value" 649 " (%d), using %d instead.\n", fl_pack, pack); 650 } 651 } 652 m = V_INGPACKBOUNDARY(M_INGPACKBOUNDARY); 653 if (pack == 16) 654 v = V_INGPACKBOUNDARY(0); 655 else 656 v = V_INGPACKBOUNDARY(ilog2(pack) - 5); 657 658 MPASS(!is_t4(sc)); /* T4 doesn't have SGE_CONTROL2 */ 659 t4_set_reg_field(sc, A_SGE_CONTROL2, m, v); 660 } 661 662 /* 663 * adap->params.vpd.cclk must be set up before this is called. 664 */ 665 void 666 t4_tweak_chip_settings(struct adapter *sc) 667 { 668 int i, reg; 669 uint32_t v, m; 670 int intr_timer[SGE_NTIMERS] = {1, 5, 10, 50, 100, 200}; 671 int timer_max = M_TIMERVALUE0 * 1000 / sc->params.vpd.cclk; 672 int intr_pktcount[SGE_NCOUNTERS] = {1, 8, 16, 32}; /* 63 max */ 673 uint16_t indsz = min(RX_COPY_THRESHOLD - 1, M_INDICATESIZE); 674 static int sw_buf_sizes[] = { 675 MCLBYTES, 676 #if MJUMPAGESIZE != MCLBYTES 677 MJUMPAGESIZE, 678 #endif 679 MJUM9BYTES, 680 MJUM16BYTES 681 }; 682 683 KASSERT(sc->flags & MASTER_PF, 684 ("%s: trying to change chip settings when not master.", __func__)); 685 686 m = V_PKTSHIFT(M_PKTSHIFT) | F_RXPKTCPLMODE | F_EGRSTATUSPAGESIZE; 687 v = V_PKTSHIFT(fl_pktshift) | F_RXPKTCPLMODE | 688 V_EGRSTATUSPAGESIZE(spg_len == 128); 689 t4_set_reg_field(sc, A_SGE_CONTROL, m, v); 690 691 setup_pad_and_pack_boundaries(sc); 692 693 v = V_HOSTPAGESIZEPF0(PAGE_SHIFT - 10) | 694 V_HOSTPAGESIZEPF1(PAGE_SHIFT - 10) | 695 V_HOSTPAGESIZEPF2(PAGE_SHIFT - 10) | 696 V_HOSTPAGESIZEPF3(PAGE_SHIFT - 10) | 697 V_HOSTPAGESIZEPF4(PAGE_SHIFT - 10) | 698 V_HOSTPAGESIZEPF5(PAGE_SHIFT - 10) | 699 V_HOSTPAGESIZEPF6(PAGE_SHIFT - 10) | 700 V_HOSTPAGESIZEPF7(PAGE_SHIFT - 10); 701 t4_write_reg(sc, A_SGE_HOST_PAGE_SIZE, v); 702 703 t4_write_reg(sc, A_SGE_FL_BUFFER_SIZE0, 4096); 704 t4_write_reg(sc, A_SGE_FL_BUFFER_SIZE1, 65536); 705 reg = A_SGE_FL_BUFFER_SIZE2; 706 for (i = 0; i < nitems(sw_buf_sizes); i++) { 707 MPASS(reg <= A_SGE_FL_BUFFER_SIZE15); 708 t4_write_reg(sc, reg, sw_buf_sizes[i]); 709 reg += 4; 710 MPASS(reg <= A_SGE_FL_BUFFER_SIZE15); 711 t4_write_reg(sc, reg, sw_buf_sizes[i] - CL_METADATA_SIZE); 712 reg += 4; 713 } 714 715 v = V_THRESHOLD_0(intr_pktcount[0]) | V_THRESHOLD_1(intr_pktcount[1]) | 716 V_THRESHOLD_2(intr_pktcount[2]) | V_THRESHOLD_3(intr_pktcount[3]); 717 t4_write_reg(sc, A_SGE_INGRESS_RX_THRESHOLD, v); 718 719 KASSERT(intr_timer[0] <= timer_max, 720 ("%s: not a single usable timer (%d, %d)", __func__, intr_timer[0], 721 timer_max)); 722 for (i = 1; i < nitems(intr_timer); i++) { 723 KASSERT(intr_timer[i] >= intr_timer[i - 1], 724 ("%s: timers not listed in increasing order (%d)", 725 __func__, i)); 726 727 while (intr_timer[i] > timer_max) { 728 if (i == nitems(intr_timer) - 1) { 729 intr_timer[i] = timer_max; 730 break; 731 } 732 intr_timer[i] += intr_timer[i - 1]; 733 intr_timer[i] /= 2; 734 } 735 } 736 737 v = V_TIMERVALUE0(us_to_core_ticks(sc, intr_timer[0])) | 738 V_TIMERVALUE1(us_to_core_ticks(sc, intr_timer[1])); 739 t4_write_reg(sc, A_SGE_TIMER_VALUE_0_AND_1, v); 740 v = V_TIMERVALUE2(us_to_core_ticks(sc, intr_timer[2])) | 741 V_TIMERVALUE3(us_to_core_ticks(sc, intr_timer[3])); 742 t4_write_reg(sc, A_SGE_TIMER_VALUE_2_AND_3, v); 743 v = V_TIMERVALUE4(us_to_core_ticks(sc, intr_timer[4])) | 744 V_TIMERVALUE5(us_to_core_ticks(sc, intr_timer[5])); 745 t4_write_reg(sc, A_SGE_TIMER_VALUE_4_AND_5, v); 746 747 if (chip_id(sc) >= CHELSIO_T6) { 748 m = V_TSCALE(M_TSCALE); 749 if (tscale == 1) 750 v = 0; 751 else 752 v = V_TSCALE(tscale - 2); 753 t4_set_reg_field(sc, A_SGE_ITP_CONTROL, m, v); 754 755 if (sc->debug_flags & DF_DISABLE_TCB_CACHE) { 756 m = V_RDTHRESHOLD(M_RDTHRESHOLD) | F_WRTHRTHRESHEN | 757 V_WRTHRTHRESH(M_WRTHRTHRESH); 758 t4_tp_pio_read(sc, &v, 1, A_TP_CMM_CONFIG, 1); 759 v &= ~m; 760 v |= V_RDTHRESHOLD(1) | F_WRTHRTHRESHEN | 761 V_WRTHRTHRESH(16); 762 t4_tp_pio_write(sc, &v, 1, A_TP_CMM_CONFIG, 1); 763 } 764 } 765 766 /* 4K, 16K, 64K, 256K DDP "page sizes" for TDDP */ 767 v = V_HPZ0(0) | V_HPZ1(2) | V_HPZ2(4) | V_HPZ3(6); 768 t4_write_reg(sc, A_ULP_RX_TDDP_PSZ, v); 769 770 /* 771 * 4K, 8K, 16K, 64K DDP "page sizes" for iSCSI DDP. These have been 772 * chosen with MAXPHYS = 128K in mind. The largest DDP buffer that we 773 * may have to deal with is MAXPHYS + 1 page. 774 */ 775 v = V_HPZ0(0) | V_HPZ1(1) | V_HPZ2(2) | V_HPZ3(4); 776 t4_write_reg(sc, A_ULP_RX_ISCSI_PSZ, v); 777 778 /* We use multiple DDP page sizes both in plain-TOE and ISCSI modes. */ 779 m = v = F_TDDPTAGTCB | F_ISCSITAGTCB; 780 t4_set_reg_field(sc, A_ULP_RX_CTL, m, v); 781 782 m = V_INDICATESIZE(M_INDICATESIZE) | F_REARMDDPOFFSET | 783 F_RESETDDPOFFSET; 784 v = V_INDICATESIZE(indsz) | F_REARMDDPOFFSET | F_RESETDDPOFFSET; 785 t4_set_reg_field(sc, A_TP_PARA_REG5, m, v); 786 } 787 788 /* 789 * SGE wants the buffer to be at least 64B and then a multiple of 16. Its 790 * address mut be 16B aligned. If padding is in use the buffer's start and end 791 * need to be aligned to the pad boundary as well. We'll just make sure that 792 * the size is a multiple of the pad boundary here, it is up to the buffer 793 * allocation code to make sure the start of the buffer is aligned. 794 */ 795 static inline int 796 hwsz_ok(struct adapter *sc, int hwsz) 797 { 798 int mask = fl_pad ? sc->params.sge.pad_boundary - 1 : 16 - 1; 799 800 return (hwsz >= 64 && (hwsz & mask) == 0); 801 } 802 803 /* 804 * XXX: driver really should be able to deal with unexpected settings. 805 */ 806 int 807 t4_read_chip_settings(struct adapter *sc) 808 { 809 struct sge *s = &sc->sge; 810 struct sge_params *sp = &sc->params.sge; 811 int i, j, n, rc = 0; 812 uint32_t m, v, r; 813 uint16_t indsz = min(RX_COPY_THRESHOLD - 1, M_INDICATESIZE); 814 static int sw_buf_sizes[] = { /* Sorted by size */ 815 MCLBYTES, 816 #if MJUMPAGESIZE != MCLBYTES 817 MJUMPAGESIZE, 818 #endif 819 MJUM9BYTES, 820 MJUM16BYTES 821 }; 822 struct rx_buf_info *rxb; 823 824 m = F_RXPKTCPLMODE; 825 v = F_RXPKTCPLMODE; 826 r = sc->params.sge.sge_control; 827 if ((r & m) != v) { 828 device_printf(sc->dev, "invalid SGE_CONTROL(0x%x)\n", r); 829 rc = EINVAL; 830 } 831 832 /* 833 * If this changes then every single use of PAGE_SHIFT in the driver 834 * needs to be carefully reviewed for PAGE_SHIFT vs sp->page_shift. 835 */ 836 if (sp->page_shift != PAGE_SHIFT) { 837 device_printf(sc->dev, "invalid SGE_HOST_PAGE_SIZE(0x%x)\n", r); 838 rc = EINVAL; 839 } 840 841 s->safe_zidx = -1; 842 rxb = &s->rx_buf_info[0]; 843 for (i = 0; i < SW_ZONE_SIZES; i++, rxb++) { 844 rxb->size1 = sw_buf_sizes[i]; 845 rxb->zone = m_getzone(rxb->size1); 846 rxb->type = m_gettype(rxb->size1); 847 rxb->size2 = 0; 848 rxb->hwidx1 = -1; 849 rxb->hwidx2 = -1; 850 for (j = 0; j < SGE_FLBUF_SIZES; j++) { 851 int hwsize = sp->sge_fl_buffer_size[j]; 852 853 if (!hwsz_ok(sc, hwsize)) 854 continue; 855 856 /* hwidx for size1 */ 857 if (rxb->hwidx1 == -1 && rxb->size1 == hwsize) 858 rxb->hwidx1 = j; 859 860 /* hwidx for size2 (buffer packing) */ 861 if (rxb->size1 - CL_METADATA_SIZE < hwsize) 862 continue; 863 n = rxb->size1 - hwsize - CL_METADATA_SIZE; 864 if (n == 0) { 865 rxb->hwidx2 = j; 866 rxb->size2 = hwsize; 867 break; /* stop looking */ 868 } 869 if (rxb->hwidx2 != -1) { 870 if (n < sp->sge_fl_buffer_size[rxb->hwidx2] - 871 hwsize - CL_METADATA_SIZE) { 872 rxb->hwidx2 = j; 873 rxb->size2 = hwsize; 874 } 875 } else if (n <= 2 * CL_METADATA_SIZE) { 876 rxb->hwidx2 = j; 877 rxb->size2 = hwsize; 878 } 879 } 880 if (rxb->hwidx2 != -1) 881 sc->flags |= BUF_PACKING_OK; 882 if (s->safe_zidx == -1 && rxb->size1 == safest_rx_cluster) 883 s->safe_zidx = i; 884 } 885 886 if (sc->flags & IS_VF) 887 return (0); 888 889 v = V_HPZ0(0) | V_HPZ1(2) | V_HPZ2(4) | V_HPZ3(6); 890 r = t4_read_reg(sc, A_ULP_RX_TDDP_PSZ); 891 if (r != v) { 892 device_printf(sc->dev, "invalid ULP_RX_TDDP_PSZ(0x%x)\n", r); 893 rc = EINVAL; 894 } 895 896 m = v = F_TDDPTAGTCB; 897 r = t4_read_reg(sc, A_ULP_RX_CTL); 898 if ((r & m) != v) { 899 device_printf(sc->dev, "invalid ULP_RX_CTL(0x%x)\n", r); 900 rc = EINVAL; 901 } 902 903 m = V_INDICATESIZE(M_INDICATESIZE) | F_REARMDDPOFFSET | 904 F_RESETDDPOFFSET; 905 v = V_INDICATESIZE(indsz) | F_REARMDDPOFFSET | F_RESETDDPOFFSET; 906 r = t4_read_reg(sc, A_TP_PARA_REG5); 907 if ((r & m) != v) { 908 device_printf(sc->dev, "invalid TP_PARA_REG5(0x%x)\n", r); 909 rc = EINVAL; 910 } 911 912 t4_init_tp_params(sc, 1); 913 914 t4_read_mtu_tbl(sc, sc->params.mtus, NULL); 915 t4_load_mtus(sc, sc->params.mtus, sc->params.a_wnd, sc->params.b_wnd); 916 917 return (rc); 918 } 919 920 int 921 t4_create_dma_tag(struct adapter *sc) 922 { 923 int rc; 924 925 rc = bus_dma_tag_create(bus_get_dma_tag(sc->dev), 1, 0, 926 BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, NULL, NULL, BUS_SPACE_MAXSIZE, 927 BUS_SPACE_UNRESTRICTED, BUS_SPACE_MAXSIZE, BUS_DMA_ALLOCNOW, NULL, 928 NULL, &sc->dmat); 929 if (rc != 0) { 930 device_printf(sc->dev, 931 "failed to create main DMA tag: %d\n", rc); 932 } 933 934 return (rc); 935 } 936 937 void 938 t4_sge_sysctls(struct adapter *sc, struct sysctl_ctx_list *ctx, 939 struct sysctl_oid_list *children) 940 { 941 struct sge_params *sp = &sc->params.sge; 942 943 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "buffer_sizes", 944 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_NEEDGIANT, sc, 0, 945 sysctl_bufsizes, "A", "freelist buffer sizes"); 946 947 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "fl_pktshift", CTLFLAG_RD, 948 NULL, sp->fl_pktshift, "payload DMA offset in rx buffer (bytes)"); 949 950 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "fl_pad", CTLFLAG_RD, 951 NULL, sp->pad_boundary, "payload pad boundary (bytes)"); 952 953 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "spg_len", CTLFLAG_RD, 954 NULL, sp->spg_len, "status page size (bytes)"); 955 956 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "cong_drop", CTLFLAG_RD, 957 NULL, cong_drop, "congestion drop setting"); 958 959 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "fl_pack", CTLFLAG_RD, 960 NULL, sp->pack_boundary, "payload pack boundary (bytes)"); 961 } 962 963 int 964 t4_destroy_dma_tag(struct adapter *sc) 965 { 966 if (sc->dmat) 967 bus_dma_tag_destroy(sc->dmat); 968 969 return (0); 970 } 971 972 /* 973 * Allocate and initialize the firmware event queue, control queues, and special 974 * purpose rx queues owned by the adapter. 975 * 976 * Returns errno on failure. Resources allocated up to that point may still be 977 * allocated. Caller is responsible for cleanup in case this function fails. 978 */ 979 int 980 t4_setup_adapter_queues(struct adapter *sc) 981 { 982 struct sysctl_oid *oid; 983 struct sysctl_oid_list *children; 984 int rc, i; 985 986 ADAPTER_LOCK_ASSERT_NOTOWNED(sc); 987 988 sysctl_ctx_init(&sc->ctx); 989 sc->flags |= ADAP_SYSCTL_CTX; 990 991 /* 992 * Firmware event queue 993 */ 994 rc = alloc_fwq(sc); 995 if (rc != 0) 996 return (rc); 997 998 /* 999 * That's all for the VF driver. 1000 */ 1001 if (sc->flags & IS_VF) 1002 return (rc); 1003 1004 oid = device_get_sysctl_tree(sc->dev); 1005 children = SYSCTL_CHILDREN(oid); 1006 1007 /* 1008 * XXX: General purpose rx queues, one per port. 1009 */ 1010 1011 /* 1012 * Control queues, one per port. 1013 */ 1014 oid = SYSCTL_ADD_NODE(&sc->ctx, children, OID_AUTO, "ctrlq", 1015 CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "control queues"); 1016 for_each_port(sc, i) { 1017 struct sge_wrq *ctrlq = &sc->sge.ctrlq[i]; 1018 1019 rc = alloc_ctrlq(sc, ctrlq, i, oid); 1020 if (rc != 0) 1021 return (rc); 1022 } 1023 1024 return (rc); 1025 } 1026 1027 /* 1028 * Idempotent 1029 */ 1030 int 1031 t4_teardown_adapter_queues(struct adapter *sc) 1032 { 1033 int i; 1034 1035 ADAPTER_LOCK_ASSERT_NOTOWNED(sc); 1036 1037 /* Do this before freeing the queue */ 1038 if (sc->flags & ADAP_SYSCTL_CTX) { 1039 sysctl_ctx_free(&sc->ctx); 1040 sc->flags &= ~ADAP_SYSCTL_CTX; 1041 } 1042 1043 if (!(sc->flags & IS_VF)) { 1044 for_each_port(sc, i) 1045 free_wrq(sc, &sc->sge.ctrlq[i]); 1046 } 1047 free_fwq(sc); 1048 1049 return (0); 1050 } 1051 1052 /* Maximum payload that could arrive with a single iq descriptor. */ 1053 static inline int 1054 max_rx_payload(struct adapter *sc, struct ifnet *ifp, const bool ofld) 1055 { 1056 int maxp; 1057 1058 /* large enough even when hw VLAN extraction is disabled */ 1059 maxp = sc->params.sge.fl_pktshift + ETHER_HDR_LEN + 1060 ETHER_VLAN_ENCAP_LEN + ifp->if_mtu; 1061 if (ofld && sc->tt.tls && sc->cryptocaps & FW_CAPS_CONFIG_TLSKEYS && 1062 maxp < sc->params.tp.max_rx_pdu) 1063 maxp = sc->params.tp.max_rx_pdu; 1064 return (maxp); 1065 } 1066 1067 int 1068 t4_setup_vi_queues(struct vi_info *vi) 1069 { 1070 int rc = 0, i, intr_idx, iqidx; 1071 struct sge_rxq *rxq; 1072 struct sge_txq *txq; 1073 #ifdef TCP_OFFLOAD 1074 struct sge_ofld_rxq *ofld_rxq; 1075 #endif 1076 #if defined(TCP_OFFLOAD) || defined(RATELIMIT) 1077 struct sge_wrq *ofld_txq; 1078 #endif 1079 #ifdef DEV_NETMAP 1080 int saved_idx; 1081 struct sge_nm_rxq *nm_rxq; 1082 struct sge_nm_txq *nm_txq; 1083 #endif 1084 char name[16]; 1085 struct port_info *pi = vi->pi; 1086 struct adapter *sc = pi->adapter; 1087 struct ifnet *ifp = vi->ifp; 1088 struct sysctl_oid *oid = device_get_sysctl_tree(vi->dev); 1089 struct sysctl_oid_list *children = SYSCTL_CHILDREN(oid); 1090 int maxp; 1091 1092 /* Interrupt vector to start from (when using multiple vectors) */ 1093 intr_idx = vi->first_intr; 1094 1095 #ifdef DEV_NETMAP 1096 saved_idx = intr_idx; 1097 if (ifp->if_capabilities & IFCAP_NETMAP) { 1098 1099 /* netmap is supported with direct interrupts only. */ 1100 MPASS(!forwarding_intr_to_fwq(sc)); 1101 1102 /* 1103 * We don't have buffers to back the netmap rx queues 1104 * right now so we create the queues in a way that 1105 * doesn't set off any congestion signal in the chip. 1106 */ 1107 oid = SYSCTL_ADD_NODE(&vi->ctx, children, OID_AUTO, "nm_rxq", 1108 CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "rx queues"); 1109 for_each_nm_rxq(vi, i, nm_rxq) { 1110 rc = alloc_nm_rxq(vi, nm_rxq, intr_idx, i, oid); 1111 if (rc != 0) 1112 goto done; 1113 intr_idx++; 1114 } 1115 1116 oid = SYSCTL_ADD_NODE(&vi->ctx, children, OID_AUTO, "nm_txq", 1117 CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "tx queues"); 1118 for_each_nm_txq(vi, i, nm_txq) { 1119 iqidx = vi->first_nm_rxq + (i % vi->nnmrxq); 1120 rc = alloc_nm_txq(vi, nm_txq, iqidx, i, oid); 1121 if (rc != 0) 1122 goto done; 1123 } 1124 } 1125 1126 /* Normal rx queues and netmap rx queues share the same interrupts. */ 1127 intr_idx = saved_idx; 1128 #endif 1129 1130 /* 1131 * Allocate rx queues first because a default iqid is required when 1132 * creating a tx queue. 1133 */ 1134 maxp = max_rx_payload(sc, ifp, false); 1135 oid = SYSCTL_ADD_NODE(&vi->ctx, children, OID_AUTO, "rxq", 1136 CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "rx queues"); 1137 for_each_rxq(vi, i, rxq) { 1138 1139 init_iq(&rxq->iq, sc, vi->tmr_idx, vi->pktc_idx, vi->qsize_rxq); 1140 1141 snprintf(name, sizeof(name), "%s rxq%d-fl", 1142 device_get_nameunit(vi->dev), i); 1143 init_fl(sc, &rxq->fl, vi->qsize_rxq / 8, maxp, name); 1144 1145 rc = alloc_rxq(vi, rxq, 1146 forwarding_intr_to_fwq(sc) ? -1 : intr_idx, i, oid); 1147 if (rc != 0) 1148 goto done; 1149 intr_idx++; 1150 } 1151 #ifdef DEV_NETMAP 1152 if (ifp->if_capabilities & IFCAP_NETMAP) 1153 intr_idx = saved_idx + max(vi->nrxq, vi->nnmrxq); 1154 #endif 1155 #ifdef TCP_OFFLOAD 1156 maxp = max_rx_payload(sc, ifp, true); 1157 oid = SYSCTL_ADD_NODE(&vi->ctx, children, OID_AUTO, "ofld_rxq", 1158 CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "rx queues for offloaded TCP connections"); 1159 for_each_ofld_rxq(vi, i, ofld_rxq) { 1160 1161 init_iq(&ofld_rxq->iq, sc, vi->ofld_tmr_idx, vi->ofld_pktc_idx, 1162 vi->qsize_rxq); 1163 1164 snprintf(name, sizeof(name), "%s ofld_rxq%d-fl", 1165 device_get_nameunit(vi->dev), i); 1166 init_fl(sc, &ofld_rxq->fl, vi->qsize_rxq / 8, maxp, name); 1167 1168 rc = alloc_ofld_rxq(vi, ofld_rxq, 1169 forwarding_intr_to_fwq(sc) ? -1 : intr_idx, i, oid); 1170 if (rc != 0) 1171 goto done; 1172 intr_idx++; 1173 } 1174 #endif 1175 1176 /* 1177 * Now the tx queues. 1178 */ 1179 oid = SYSCTL_ADD_NODE(&vi->ctx, children, OID_AUTO, "txq", 1180 CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "tx queues"); 1181 for_each_txq(vi, i, txq) { 1182 iqidx = vi->first_rxq + (i % vi->nrxq); 1183 snprintf(name, sizeof(name), "%s txq%d", 1184 device_get_nameunit(vi->dev), i); 1185 init_eq(sc, &txq->eq, EQ_ETH, vi->qsize_txq, pi->tx_chan, 1186 sc->sge.rxq[iqidx].iq.cntxt_id, name); 1187 1188 rc = alloc_txq(vi, txq, i, oid); 1189 if (rc != 0) 1190 goto done; 1191 } 1192 #if defined(TCP_OFFLOAD) || defined(RATELIMIT) 1193 oid = SYSCTL_ADD_NODE(&vi->ctx, children, OID_AUTO, "ofld_txq", 1194 CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "tx queues for TOE/ETHOFLD"); 1195 for_each_ofld_txq(vi, i, ofld_txq) { 1196 struct sysctl_oid *oid2; 1197 1198 snprintf(name, sizeof(name), "%s ofld_txq%d", 1199 device_get_nameunit(vi->dev), i); 1200 if (vi->nofldrxq > 0) { 1201 iqidx = vi->first_ofld_rxq + (i % vi->nofldrxq); 1202 init_eq(sc, &ofld_txq->eq, EQ_OFLD, vi->qsize_txq, 1203 pi->tx_chan, sc->sge.ofld_rxq[iqidx].iq.cntxt_id, 1204 name); 1205 } else { 1206 iqidx = vi->first_rxq + (i % vi->nrxq); 1207 init_eq(sc, &ofld_txq->eq, EQ_OFLD, vi->qsize_txq, 1208 pi->tx_chan, sc->sge.rxq[iqidx].iq.cntxt_id, name); 1209 } 1210 1211 snprintf(name, sizeof(name), "%d", i); 1212 oid2 = SYSCTL_ADD_NODE(&vi->ctx, SYSCTL_CHILDREN(oid), OID_AUTO, 1213 name, CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "offload tx queue"); 1214 1215 rc = alloc_wrq(sc, vi, ofld_txq, oid2); 1216 if (rc != 0) 1217 goto done; 1218 } 1219 #endif 1220 done: 1221 if (rc) 1222 t4_teardown_vi_queues(vi); 1223 1224 return (rc); 1225 } 1226 1227 /* 1228 * Idempotent 1229 */ 1230 int 1231 t4_teardown_vi_queues(struct vi_info *vi) 1232 { 1233 int i; 1234 struct sge_rxq *rxq; 1235 struct sge_txq *txq; 1236 #if defined(TCP_OFFLOAD) || defined(RATELIMIT) 1237 struct port_info *pi = vi->pi; 1238 struct adapter *sc = pi->adapter; 1239 struct sge_wrq *ofld_txq; 1240 #endif 1241 #ifdef TCP_OFFLOAD 1242 struct sge_ofld_rxq *ofld_rxq; 1243 #endif 1244 #ifdef DEV_NETMAP 1245 struct sge_nm_rxq *nm_rxq; 1246 struct sge_nm_txq *nm_txq; 1247 #endif 1248 1249 /* Do this before freeing the queues */ 1250 if (vi->flags & VI_SYSCTL_CTX) { 1251 sysctl_ctx_free(&vi->ctx); 1252 vi->flags &= ~VI_SYSCTL_CTX; 1253 } 1254 1255 #ifdef DEV_NETMAP 1256 if (vi->ifp->if_capabilities & IFCAP_NETMAP) { 1257 for_each_nm_txq(vi, i, nm_txq) { 1258 free_nm_txq(vi, nm_txq); 1259 } 1260 1261 for_each_nm_rxq(vi, i, nm_rxq) { 1262 free_nm_rxq(vi, nm_rxq); 1263 } 1264 } 1265 #endif 1266 1267 /* 1268 * Take down all the tx queues first, as they reference the rx queues 1269 * (for egress updates, etc.). 1270 */ 1271 1272 for_each_txq(vi, i, txq) { 1273 free_txq(vi, txq); 1274 } 1275 #if defined(TCP_OFFLOAD) || defined(RATELIMIT) 1276 for_each_ofld_txq(vi, i, ofld_txq) { 1277 free_wrq(sc, ofld_txq); 1278 } 1279 #endif 1280 1281 /* 1282 * Then take down the rx queues. 1283 */ 1284 1285 for_each_rxq(vi, i, rxq) { 1286 free_rxq(vi, rxq); 1287 } 1288 #ifdef TCP_OFFLOAD 1289 for_each_ofld_rxq(vi, i, ofld_rxq) { 1290 free_ofld_rxq(vi, ofld_rxq); 1291 } 1292 #endif 1293 1294 return (0); 1295 } 1296 1297 /* 1298 * Interrupt handler when the driver is using only 1 interrupt. This is a very 1299 * unusual scenario. 1300 * 1301 * a) Deals with errors, if any. 1302 * b) Services firmware event queue, which is taking interrupts for all other 1303 * queues. 1304 */ 1305 void 1306 t4_intr_all(void *arg) 1307 { 1308 struct adapter *sc = arg; 1309 struct sge_iq *fwq = &sc->sge.fwq; 1310 1311 MPASS(sc->intr_count == 1); 1312 1313 if (sc->intr_type == INTR_INTX) 1314 t4_write_reg(sc, MYPF_REG(A_PCIE_PF_CLI), 0); 1315 1316 t4_intr_err(arg); 1317 t4_intr_evt(fwq); 1318 } 1319 1320 /* 1321 * Interrupt handler for errors (installed directly when multiple interrupts are 1322 * being used, or called by t4_intr_all). 1323 */ 1324 void 1325 t4_intr_err(void *arg) 1326 { 1327 struct adapter *sc = arg; 1328 uint32_t v; 1329 const bool verbose = (sc->debug_flags & DF_VERBOSE_SLOWINTR) != 0; 1330 1331 if (sc->flags & ADAP_ERR) 1332 return; 1333 1334 v = t4_read_reg(sc, MYPF_REG(A_PL_PF_INT_CAUSE)); 1335 if (v & F_PFSW) { 1336 sc->swintr++; 1337 t4_write_reg(sc, MYPF_REG(A_PL_PF_INT_CAUSE), v); 1338 } 1339 1340 t4_slow_intr_handler(sc, verbose); 1341 } 1342 1343 /* 1344 * Interrupt handler for iq-only queues. The firmware event queue is the only 1345 * such queue right now. 1346 */ 1347 void 1348 t4_intr_evt(void *arg) 1349 { 1350 struct sge_iq *iq = arg; 1351 1352 if (atomic_cmpset_int(&iq->state, IQS_IDLE, IQS_BUSY)) { 1353 service_iq(iq, 0); 1354 (void) atomic_cmpset_int(&iq->state, IQS_BUSY, IQS_IDLE); 1355 } 1356 } 1357 1358 /* 1359 * Interrupt handler for iq+fl queues. 1360 */ 1361 void 1362 t4_intr(void *arg) 1363 { 1364 struct sge_iq *iq = arg; 1365 1366 if (atomic_cmpset_int(&iq->state, IQS_IDLE, IQS_BUSY)) { 1367 service_iq_fl(iq, 0); 1368 (void) atomic_cmpset_int(&iq->state, IQS_BUSY, IQS_IDLE); 1369 } 1370 } 1371 1372 #ifdef DEV_NETMAP 1373 /* 1374 * Interrupt handler for netmap rx queues. 1375 */ 1376 void 1377 t4_nm_intr(void *arg) 1378 { 1379 struct sge_nm_rxq *nm_rxq = arg; 1380 1381 if (atomic_cmpset_int(&nm_rxq->nm_state, NM_ON, NM_BUSY)) { 1382 service_nm_rxq(nm_rxq); 1383 (void) atomic_cmpset_int(&nm_rxq->nm_state, NM_BUSY, NM_ON); 1384 } 1385 } 1386 1387 /* 1388 * Interrupt handler for vectors shared between NIC and netmap rx queues. 1389 */ 1390 void 1391 t4_vi_intr(void *arg) 1392 { 1393 struct irq *irq = arg; 1394 1395 MPASS(irq->nm_rxq != NULL); 1396 t4_nm_intr(irq->nm_rxq); 1397 1398 MPASS(irq->rxq != NULL); 1399 t4_intr(irq->rxq); 1400 } 1401 #endif 1402 1403 /* 1404 * Deals with interrupts on an iq-only (no freelist) queue. 1405 */ 1406 static int 1407 service_iq(struct sge_iq *iq, int budget) 1408 { 1409 struct sge_iq *q; 1410 struct adapter *sc = iq->adapter; 1411 struct iq_desc *d = &iq->desc[iq->cidx]; 1412 int ndescs = 0, limit; 1413 int rsp_type; 1414 uint32_t lq; 1415 STAILQ_HEAD(, sge_iq) iql = STAILQ_HEAD_INITIALIZER(iql); 1416 1417 KASSERT(iq->state == IQS_BUSY, ("%s: iq %p not BUSY", __func__, iq)); 1418 KASSERT((iq->flags & IQ_HAS_FL) == 0, 1419 ("%s: called for iq %p with fl (iq->flags 0x%x)", __func__, iq, 1420 iq->flags)); 1421 MPASS((iq->flags & IQ_ADJ_CREDIT) == 0); 1422 MPASS((iq->flags & IQ_LRO_ENABLED) == 0); 1423 1424 limit = budget ? budget : iq->qsize / 16; 1425 1426 /* 1427 * We always come back and check the descriptor ring for new indirect 1428 * interrupts and other responses after running a single handler. 1429 */ 1430 for (;;) { 1431 while ((d->rsp.u.type_gen & F_RSPD_GEN) == iq->gen) { 1432 1433 rmb(); 1434 1435 rsp_type = G_RSPD_TYPE(d->rsp.u.type_gen); 1436 lq = be32toh(d->rsp.pldbuflen_qid); 1437 1438 switch (rsp_type) { 1439 case X_RSPD_TYPE_FLBUF: 1440 panic("%s: data for an iq (%p) with no freelist", 1441 __func__, iq); 1442 1443 /* NOTREACHED */ 1444 1445 case X_RSPD_TYPE_CPL: 1446 KASSERT(d->rss.opcode < NUM_CPL_CMDS, 1447 ("%s: bad opcode %02x.", __func__, 1448 d->rss.opcode)); 1449 t4_cpl_handler[d->rss.opcode](iq, &d->rss, NULL); 1450 break; 1451 1452 case X_RSPD_TYPE_INTR: 1453 /* 1454 * There are 1K interrupt-capable queues (qids 0 1455 * through 1023). A response type indicating a 1456 * forwarded interrupt with a qid >= 1K is an 1457 * iWARP async notification. 1458 */ 1459 if (__predict_true(lq >= 1024)) { 1460 t4_an_handler(iq, &d->rsp); 1461 break; 1462 } 1463 1464 q = sc->sge.iqmap[lq - sc->sge.iq_start - 1465 sc->sge.iq_base]; 1466 if (atomic_cmpset_int(&q->state, IQS_IDLE, 1467 IQS_BUSY)) { 1468 if (service_iq_fl(q, q->qsize / 16) == 0) { 1469 (void) atomic_cmpset_int(&q->state, 1470 IQS_BUSY, IQS_IDLE); 1471 } else { 1472 STAILQ_INSERT_TAIL(&iql, q, 1473 link); 1474 } 1475 } 1476 break; 1477 1478 default: 1479 KASSERT(0, 1480 ("%s: illegal response type %d on iq %p", 1481 __func__, rsp_type, iq)); 1482 log(LOG_ERR, 1483 "%s: illegal response type %d on iq %p", 1484 device_get_nameunit(sc->dev), rsp_type, iq); 1485 break; 1486 } 1487 1488 d++; 1489 if (__predict_false(++iq->cidx == iq->sidx)) { 1490 iq->cidx = 0; 1491 iq->gen ^= F_RSPD_GEN; 1492 d = &iq->desc[0]; 1493 } 1494 if (__predict_false(++ndescs == limit)) { 1495 t4_write_reg(sc, sc->sge_gts_reg, 1496 V_CIDXINC(ndescs) | 1497 V_INGRESSQID(iq->cntxt_id) | 1498 V_SEINTARM(V_QINTR_TIMER_IDX(X_TIMERREG_UPDATE_CIDX))); 1499 ndescs = 0; 1500 1501 if (budget) { 1502 return (EINPROGRESS); 1503 } 1504 } 1505 } 1506 1507 if (STAILQ_EMPTY(&iql)) 1508 break; 1509 1510 /* 1511 * Process the head only, and send it to the back of the list if 1512 * it's still not done. 1513 */ 1514 q = STAILQ_FIRST(&iql); 1515 STAILQ_REMOVE_HEAD(&iql, link); 1516 if (service_iq_fl(q, q->qsize / 8) == 0) 1517 (void) atomic_cmpset_int(&q->state, IQS_BUSY, IQS_IDLE); 1518 else 1519 STAILQ_INSERT_TAIL(&iql, q, link); 1520 } 1521 1522 t4_write_reg(sc, sc->sge_gts_reg, V_CIDXINC(ndescs) | 1523 V_INGRESSQID((u32)iq->cntxt_id) | V_SEINTARM(iq->intr_params)); 1524 1525 return (0); 1526 } 1527 1528 static inline int 1529 sort_before_lro(struct lro_ctrl *lro) 1530 { 1531 1532 return (lro->lro_mbuf_max != 0); 1533 } 1534 1535 static inline uint64_t 1536 last_flit_to_ns(struct adapter *sc, uint64_t lf) 1537 { 1538 uint64_t n = be64toh(lf) & 0xfffffffffffffff; /* 60b, not 64b. */ 1539 1540 if (n > UINT64_MAX / 1000000) 1541 return (n / sc->params.vpd.cclk * 1000000); 1542 else 1543 return (n * 1000000 / sc->params.vpd.cclk); 1544 } 1545 1546 static inline void 1547 move_to_next_rxbuf(struct sge_fl *fl) 1548 { 1549 1550 fl->rx_offset = 0; 1551 if (__predict_false((++fl->cidx & 7) == 0)) { 1552 uint16_t cidx = fl->cidx >> 3; 1553 1554 if (__predict_false(cidx == fl->sidx)) 1555 fl->cidx = cidx = 0; 1556 fl->hw_cidx = cidx; 1557 } 1558 } 1559 1560 /* 1561 * Deals with interrupts on an iq+fl queue. 1562 */ 1563 static int 1564 service_iq_fl(struct sge_iq *iq, int budget) 1565 { 1566 struct sge_rxq *rxq = iq_to_rxq(iq); 1567 struct sge_fl *fl; 1568 struct adapter *sc = iq->adapter; 1569 struct iq_desc *d = &iq->desc[iq->cidx]; 1570 int ndescs, limit; 1571 int rsp_type, starved; 1572 uint32_t lq; 1573 uint16_t fl_hw_cidx; 1574 struct mbuf *m0; 1575 #if defined(INET) || defined(INET6) 1576 const struct timeval lro_timeout = {0, sc->lro_timeout}; 1577 struct lro_ctrl *lro = &rxq->lro; 1578 #endif 1579 1580 KASSERT(iq->state == IQS_BUSY, ("%s: iq %p not BUSY", __func__, iq)); 1581 MPASS(iq->flags & IQ_HAS_FL); 1582 1583 ndescs = 0; 1584 #if defined(INET) || defined(INET6) 1585 if (iq->flags & IQ_ADJ_CREDIT) { 1586 MPASS(sort_before_lro(lro)); 1587 iq->flags &= ~IQ_ADJ_CREDIT; 1588 if ((d->rsp.u.type_gen & F_RSPD_GEN) != iq->gen) { 1589 tcp_lro_flush_all(lro); 1590 t4_write_reg(sc, sc->sge_gts_reg, V_CIDXINC(1) | 1591 V_INGRESSQID((u32)iq->cntxt_id) | 1592 V_SEINTARM(iq->intr_params)); 1593 return (0); 1594 } 1595 ndescs = 1; 1596 } 1597 #else 1598 MPASS((iq->flags & IQ_ADJ_CREDIT) == 0); 1599 #endif 1600 1601 limit = budget ? budget : iq->qsize / 16; 1602 fl = &rxq->fl; 1603 fl_hw_cidx = fl->hw_cidx; /* stable snapshot */ 1604 while ((d->rsp.u.type_gen & F_RSPD_GEN) == iq->gen) { 1605 1606 rmb(); 1607 1608 m0 = NULL; 1609 rsp_type = G_RSPD_TYPE(d->rsp.u.type_gen); 1610 lq = be32toh(d->rsp.pldbuflen_qid); 1611 1612 switch (rsp_type) { 1613 case X_RSPD_TYPE_FLBUF: 1614 if (lq & F_RSPD_NEWBUF) { 1615 if (fl->rx_offset > 0) 1616 move_to_next_rxbuf(fl); 1617 lq = G_RSPD_LEN(lq); 1618 } 1619 if (IDXDIFF(fl->hw_cidx, fl_hw_cidx, fl->sidx) > 4) { 1620 FL_LOCK(fl); 1621 refill_fl(sc, fl, 64); 1622 FL_UNLOCK(fl); 1623 fl_hw_cidx = fl->hw_cidx; 1624 } 1625 1626 if (d->rss.opcode == CPL_RX_PKT) { 1627 if (__predict_true(eth_rx(sc, rxq, d, lq) == 0)) 1628 break; 1629 goto out; 1630 } 1631 m0 = get_fl_payload(sc, fl, lq); 1632 if (__predict_false(m0 == NULL)) 1633 goto out; 1634 1635 /* fall through */ 1636 1637 case X_RSPD_TYPE_CPL: 1638 KASSERT(d->rss.opcode < NUM_CPL_CMDS, 1639 ("%s: bad opcode %02x.", __func__, d->rss.opcode)); 1640 t4_cpl_handler[d->rss.opcode](iq, &d->rss, m0); 1641 break; 1642 1643 case X_RSPD_TYPE_INTR: 1644 1645 /* 1646 * There are 1K interrupt-capable queues (qids 0 1647 * through 1023). A response type indicating a 1648 * forwarded interrupt with a qid >= 1K is an 1649 * iWARP async notification. That is the only 1650 * acceptable indirect interrupt on this queue. 1651 */ 1652 if (__predict_false(lq < 1024)) { 1653 panic("%s: indirect interrupt on iq_fl %p " 1654 "with qid %u", __func__, iq, lq); 1655 } 1656 1657 t4_an_handler(iq, &d->rsp); 1658 break; 1659 1660 default: 1661 KASSERT(0, ("%s: illegal response type %d on iq %p", 1662 __func__, rsp_type, iq)); 1663 log(LOG_ERR, "%s: illegal response type %d on iq %p", 1664 device_get_nameunit(sc->dev), rsp_type, iq); 1665 break; 1666 } 1667 1668 d++; 1669 if (__predict_false(++iq->cidx == iq->sidx)) { 1670 iq->cidx = 0; 1671 iq->gen ^= F_RSPD_GEN; 1672 d = &iq->desc[0]; 1673 } 1674 if (__predict_false(++ndescs == limit)) { 1675 t4_write_reg(sc, sc->sge_gts_reg, V_CIDXINC(ndescs) | 1676 V_INGRESSQID(iq->cntxt_id) | 1677 V_SEINTARM(V_QINTR_TIMER_IDX(X_TIMERREG_UPDATE_CIDX))); 1678 1679 #if defined(INET) || defined(INET6) 1680 if (iq->flags & IQ_LRO_ENABLED && 1681 !sort_before_lro(lro) && 1682 sc->lro_timeout != 0) { 1683 tcp_lro_flush_inactive(lro, &lro_timeout); 1684 } 1685 #endif 1686 if (budget) 1687 return (EINPROGRESS); 1688 ndescs = 0; 1689 } 1690 } 1691 out: 1692 #if defined(INET) || defined(INET6) 1693 if (iq->flags & IQ_LRO_ENABLED) { 1694 if (ndescs > 0 && lro->lro_mbuf_count > 8) { 1695 MPASS(sort_before_lro(lro)); 1696 /* hold back one credit and don't flush LRO state */ 1697 iq->flags |= IQ_ADJ_CREDIT; 1698 ndescs--; 1699 } else { 1700 tcp_lro_flush_all(lro); 1701 } 1702 } 1703 #endif 1704 1705 t4_write_reg(sc, sc->sge_gts_reg, V_CIDXINC(ndescs) | 1706 V_INGRESSQID((u32)iq->cntxt_id) | V_SEINTARM(iq->intr_params)); 1707 1708 FL_LOCK(fl); 1709 starved = refill_fl(sc, fl, 64); 1710 FL_UNLOCK(fl); 1711 if (__predict_false(starved != 0)) 1712 add_fl_to_sfl(sc, fl); 1713 1714 return (0); 1715 } 1716 1717 static inline struct cluster_metadata * 1718 cl_metadata(struct fl_sdesc *sd) 1719 { 1720 1721 return ((void *)(sd->cl + sd->moff)); 1722 } 1723 1724 static void 1725 rxb_free(struct mbuf *m) 1726 { 1727 struct cluster_metadata *clm = m->m_ext.ext_arg1; 1728 1729 uma_zfree(clm->zone, clm->cl); 1730 counter_u64_add(extfree_rels, 1); 1731 } 1732 1733 /* 1734 * The mbuf returned comes from zone_muf and carries the payload in one of these 1735 * ways 1736 * a) complete frame inside the mbuf 1737 * b) m_cljset (for clusters without metadata) 1738 * d) m_extaddref (cluster with metadata) 1739 */ 1740 static struct mbuf * 1741 get_scatter_segment(struct adapter *sc, struct sge_fl *fl, int fr_offset, 1742 int remaining) 1743 { 1744 struct mbuf *m; 1745 struct fl_sdesc *sd = &fl->sdesc[fl->cidx]; 1746 struct rx_buf_info *rxb = &sc->sge.rx_buf_info[sd->zidx]; 1747 struct cluster_metadata *clm; 1748 int len, blen; 1749 caddr_t payload; 1750 1751 if (fl->flags & FL_BUF_PACKING) { 1752 u_int l, pad; 1753 1754 blen = rxb->size2 - fl->rx_offset; /* max possible in this buf */ 1755 len = min(remaining, blen); 1756 payload = sd->cl + fl->rx_offset; 1757 1758 l = fr_offset + len; 1759 pad = roundup2(l, fl->buf_boundary) - l; 1760 if (fl->rx_offset + len + pad < rxb->size2) 1761 blen = len + pad; 1762 MPASS(fl->rx_offset + blen <= rxb->size2); 1763 } else { 1764 MPASS(fl->rx_offset == 0); /* not packing */ 1765 blen = rxb->size1; 1766 len = min(remaining, blen); 1767 payload = sd->cl; 1768 } 1769 1770 if (fr_offset == 0) { 1771 m = m_gethdr(M_NOWAIT, MT_DATA); 1772 if (__predict_false(m == NULL)) 1773 return (NULL); 1774 m->m_pkthdr.len = remaining; 1775 } else { 1776 m = m_get(M_NOWAIT, MT_DATA); 1777 if (__predict_false(m == NULL)) 1778 return (NULL); 1779 } 1780 m->m_len = len; 1781 1782 if (sc->sc_do_rxcopy && len < RX_COPY_THRESHOLD) { 1783 /* copy data to mbuf */ 1784 bcopy(payload, mtod(m, caddr_t), len); 1785 if (fl->flags & FL_BUF_PACKING) { 1786 fl->rx_offset += blen; 1787 MPASS(fl->rx_offset <= rxb->size2); 1788 if (fl->rx_offset < rxb->size2) 1789 return (m); /* without advancing the cidx */ 1790 } 1791 } else if (fl->flags & FL_BUF_PACKING) { 1792 clm = cl_metadata(sd); 1793 if (sd->nmbuf++ == 0) { 1794 clm->refcount = 1; 1795 clm->zone = rxb->zone; 1796 clm->cl = sd->cl; 1797 counter_u64_add(extfree_refs, 1); 1798 } 1799 m_extaddref(m, payload, blen, &clm->refcount, rxb_free, clm, 1800 NULL); 1801 1802 fl->rx_offset += blen; 1803 MPASS(fl->rx_offset <= rxb->size2); 1804 if (fl->rx_offset < rxb->size2) 1805 return (m); /* without advancing the cidx */ 1806 } else { 1807 m_cljset(m, sd->cl, rxb->type); 1808 sd->cl = NULL; /* consumed, not a recycle candidate */ 1809 } 1810 1811 move_to_next_rxbuf(fl); 1812 1813 return (m); 1814 } 1815 1816 static struct mbuf * 1817 get_fl_payload(struct adapter *sc, struct sge_fl *fl, const u_int plen) 1818 { 1819 struct mbuf *m0, *m, **pnext; 1820 u_int remaining; 1821 1822 if (__predict_false(fl->flags & FL_BUF_RESUME)) { 1823 M_ASSERTPKTHDR(fl->m0); 1824 MPASS(fl->m0->m_pkthdr.len == plen); 1825 MPASS(fl->remaining < plen); 1826 1827 m0 = fl->m0; 1828 pnext = fl->pnext; 1829 remaining = fl->remaining; 1830 fl->flags &= ~FL_BUF_RESUME; 1831 goto get_segment; 1832 } 1833 1834 /* 1835 * Payload starts at rx_offset in the current hw buffer. Its length is 1836 * 'len' and it may span multiple hw buffers. 1837 */ 1838 1839 m0 = get_scatter_segment(sc, fl, 0, plen); 1840 if (m0 == NULL) 1841 return (NULL); 1842 remaining = plen - m0->m_len; 1843 pnext = &m0->m_next; 1844 while (remaining > 0) { 1845 get_segment: 1846 MPASS(fl->rx_offset == 0); 1847 m = get_scatter_segment(sc, fl, plen - remaining, remaining); 1848 if (__predict_false(m == NULL)) { 1849 fl->m0 = m0; 1850 fl->pnext = pnext; 1851 fl->remaining = remaining; 1852 fl->flags |= FL_BUF_RESUME; 1853 return (NULL); 1854 } 1855 *pnext = m; 1856 pnext = &m->m_next; 1857 remaining -= m->m_len; 1858 } 1859 *pnext = NULL; 1860 1861 M_ASSERTPKTHDR(m0); 1862 return (m0); 1863 } 1864 1865 static int 1866 skip_scatter_segment(struct adapter *sc, struct sge_fl *fl, int fr_offset, 1867 int remaining) 1868 { 1869 struct fl_sdesc *sd = &fl->sdesc[fl->cidx]; 1870 struct rx_buf_info *rxb = &sc->sge.rx_buf_info[sd->zidx]; 1871 int len, blen; 1872 1873 if (fl->flags & FL_BUF_PACKING) { 1874 u_int l, pad; 1875 1876 blen = rxb->size2 - fl->rx_offset; /* max possible in this buf */ 1877 len = min(remaining, blen); 1878 1879 l = fr_offset + len; 1880 pad = roundup2(l, fl->buf_boundary) - l; 1881 if (fl->rx_offset + len + pad < rxb->size2) 1882 blen = len + pad; 1883 fl->rx_offset += blen; 1884 MPASS(fl->rx_offset <= rxb->size2); 1885 if (fl->rx_offset < rxb->size2) 1886 return (len); /* without advancing the cidx */ 1887 } else { 1888 MPASS(fl->rx_offset == 0); /* not packing */ 1889 blen = rxb->size1; 1890 len = min(remaining, blen); 1891 } 1892 move_to_next_rxbuf(fl); 1893 return (len); 1894 } 1895 1896 static inline void 1897 skip_fl_payload(struct adapter *sc, struct sge_fl *fl, int plen) 1898 { 1899 int remaining, fr_offset, len; 1900 1901 fr_offset = 0; 1902 remaining = plen; 1903 while (remaining > 0) { 1904 len = skip_scatter_segment(sc, fl, fr_offset, remaining); 1905 fr_offset += len; 1906 remaining -= len; 1907 } 1908 } 1909 1910 static inline int 1911 get_segment_len(struct adapter *sc, struct sge_fl *fl, int plen) 1912 { 1913 int len; 1914 struct fl_sdesc *sd = &fl->sdesc[fl->cidx]; 1915 struct rx_buf_info *rxb = &sc->sge.rx_buf_info[sd->zidx]; 1916 1917 if (fl->flags & FL_BUF_PACKING) 1918 len = rxb->size2 - fl->rx_offset; 1919 else 1920 len = rxb->size1; 1921 1922 return (min(plen, len)); 1923 } 1924 1925 static int 1926 eth_rx(struct adapter *sc, struct sge_rxq *rxq, const struct iq_desc *d, 1927 u_int plen) 1928 { 1929 struct mbuf *m0; 1930 struct ifnet *ifp = rxq->ifp; 1931 struct sge_fl *fl = &rxq->fl; 1932 struct vi_info *vi = ifp->if_softc; 1933 const struct cpl_rx_pkt *cpl; 1934 #if defined(INET) || defined(INET6) 1935 struct lro_ctrl *lro = &rxq->lro; 1936 #endif 1937 uint16_t err_vec, tnl_type, tnlhdr_len; 1938 static const int sw_hashtype[4][2] = { 1939 {M_HASHTYPE_NONE, M_HASHTYPE_NONE}, 1940 {M_HASHTYPE_RSS_IPV4, M_HASHTYPE_RSS_IPV6}, 1941 {M_HASHTYPE_RSS_TCP_IPV4, M_HASHTYPE_RSS_TCP_IPV6}, 1942 {M_HASHTYPE_RSS_UDP_IPV4, M_HASHTYPE_RSS_UDP_IPV6}, 1943 }; 1944 static const int sw_csum_flags[2][2] = { 1945 { 1946 /* IP, inner IP */ 1947 CSUM_ENCAP_VXLAN | 1948 CSUM_L3_CALC | CSUM_L3_VALID | 1949 CSUM_L4_CALC | CSUM_L4_VALID | 1950 CSUM_INNER_L3_CALC | CSUM_INNER_L3_VALID | 1951 CSUM_INNER_L4_CALC | CSUM_INNER_L4_VALID, 1952 1953 /* IP, inner IP6 */ 1954 CSUM_ENCAP_VXLAN | 1955 CSUM_L3_CALC | CSUM_L3_VALID | 1956 CSUM_L4_CALC | CSUM_L4_VALID | 1957 CSUM_INNER_L4_CALC | CSUM_INNER_L4_VALID, 1958 }, 1959 { 1960 /* IP6, inner IP */ 1961 CSUM_ENCAP_VXLAN | 1962 CSUM_L4_CALC | CSUM_L4_VALID | 1963 CSUM_INNER_L3_CALC | CSUM_INNER_L3_VALID | 1964 CSUM_INNER_L4_CALC | CSUM_INNER_L4_VALID, 1965 1966 /* IP6, inner IP6 */ 1967 CSUM_ENCAP_VXLAN | 1968 CSUM_L4_CALC | CSUM_L4_VALID | 1969 CSUM_INNER_L4_CALC | CSUM_INNER_L4_VALID, 1970 }, 1971 }; 1972 1973 MPASS(plen > sc->params.sge.fl_pktshift); 1974 if (vi->pfil != NULL && PFIL_HOOKED_IN(vi->pfil) && 1975 __predict_true((fl->flags & FL_BUF_RESUME) == 0)) { 1976 struct fl_sdesc *sd = &fl->sdesc[fl->cidx]; 1977 caddr_t frame; 1978 int rc, slen; 1979 1980 slen = get_segment_len(sc, fl, plen) - 1981 sc->params.sge.fl_pktshift; 1982 frame = sd->cl + fl->rx_offset + sc->params.sge.fl_pktshift; 1983 CURVNET_SET_QUIET(ifp->if_vnet); 1984 rc = pfil_run_hooks(vi->pfil, frame, ifp, 1985 slen | PFIL_MEMPTR | PFIL_IN, NULL); 1986 CURVNET_RESTORE(); 1987 if (rc == PFIL_DROPPED || rc == PFIL_CONSUMED) { 1988 skip_fl_payload(sc, fl, plen); 1989 return (0); 1990 } 1991 if (rc == PFIL_REALLOCED) { 1992 skip_fl_payload(sc, fl, plen); 1993 m0 = pfil_mem2mbuf(frame); 1994 goto have_mbuf; 1995 } 1996 } 1997 1998 m0 = get_fl_payload(sc, fl, plen); 1999 if (__predict_false(m0 == NULL)) 2000 return (ENOMEM); 2001 2002 m0->m_pkthdr.len -= sc->params.sge.fl_pktshift; 2003 m0->m_len -= sc->params.sge.fl_pktshift; 2004 m0->m_data += sc->params.sge.fl_pktshift; 2005 2006 have_mbuf: 2007 m0->m_pkthdr.rcvif = ifp; 2008 M_HASHTYPE_SET(m0, sw_hashtype[d->rss.hash_type][d->rss.ipv6]); 2009 m0->m_pkthdr.flowid = be32toh(d->rss.hash_val); 2010 2011 cpl = (const void *)(&d->rss + 1); 2012 if (sc->params.tp.rx_pkt_encap) { 2013 const uint16_t ev = be16toh(cpl->err_vec); 2014 2015 err_vec = G_T6_COMPR_RXERR_VEC(ev); 2016 tnl_type = G_T6_RX_TNL_TYPE(ev); 2017 tnlhdr_len = G_T6_RX_TNLHDR_LEN(ev); 2018 } else { 2019 err_vec = be16toh(cpl->err_vec); 2020 tnl_type = 0; 2021 tnlhdr_len = 0; 2022 } 2023 if (cpl->csum_calc && err_vec == 0) { 2024 int ipv6 = !!(cpl->l2info & htobe32(F_RXF_IP6)); 2025 2026 /* checksum(s) calculated and found to be correct. */ 2027 2028 MPASS((cpl->l2info & htobe32(F_RXF_IP)) ^ 2029 (cpl->l2info & htobe32(F_RXF_IP6))); 2030 m0->m_pkthdr.csum_data = be16toh(cpl->csum); 2031 if (tnl_type == 0) { 2032 if (!ipv6 && ifp->if_capenable & IFCAP_RXCSUM) { 2033 m0->m_pkthdr.csum_flags = CSUM_L3_CALC | 2034 CSUM_L3_VALID | CSUM_L4_CALC | 2035 CSUM_L4_VALID; 2036 } else if (ipv6 && ifp->if_capenable & IFCAP_RXCSUM_IPV6) { 2037 m0->m_pkthdr.csum_flags = CSUM_L4_CALC | 2038 CSUM_L4_VALID; 2039 } 2040 rxq->rxcsum++; 2041 } else { 2042 MPASS(tnl_type == RX_PKT_TNL_TYPE_VXLAN); 2043 if (__predict_false(cpl->ip_frag)) { 2044 /* 2045 * csum_data is for the inner frame (which is an 2046 * IP fragment) and is not 0xffff. There is no 2047 * way to pass the inner csum_data to the stack. 2048 * We don't want the stack to use the inner 2049 * csum_data to validate the outer frame or it 2050 * will get rejected. So we fix csum_data here 2051 * and let sw do the checksum of inner IP 2052 * fragments. 2053 * 2054 * XXX: Need 32b for csum_data2 in an rx mbuf. 2055 * Maybe stuff it into rcv_tstmp? 2056 */ 2057 m0->m_pkthdr.csum_data = 0xffff; 2058 if (ipv6) { 2059 m0->m_pkthdr.csum_flags = CSUM_L4_CALC | 2060 CSUM_L4_VALID; 2061 } else { 2062 m0->m_pkthdr.csum_flags = CSUM_L3_CALC | 2063 CSUM_L3_VALID | CSUM_L4_CALC | 2064 CSUM_L4_VALID; 2065 } 2066 } else { 2067 int outer_ipv6; 2068 2069 MPASS(m0->m_pkthdr.csum_data == 0xffff); 2070 2071 outer_ipv6 = tnlhdr_len >= 2072 sizeof(struct ether_header) + 2073 sizeof(struct ip6_hdr); 2074 m0->m_pkthdr.csum_flags = 2075 sw_csum_flags[outer_ipv6][ipv6]; 2076 } 2077 rxq->vxlan_rxcsum++; 2078 } 2079 } 2080 2081 if (cpl->vlan_ex) { 2082 m0->m_pkthdr.ether_vtag = be16toh(cpl->vlan); 2083 m0->m_flags |= M_VLANTAG; 2084 rxq->vlan_extraction++; 2085 } 2086 2087 if (rxq->iq.flags & IQ_RX_TIMESTAMP) { 2088 /* 2089 * Fill up rcv_tstmp but do not set M_TSTMP. 2090 * rcv_tstmp is not in the format that the 2091 * kernel expects and we don't want to mislead 2092 * it. For now this is only for custom code 2093 * that knows how to interpret cxgbe's stamp. 2094 */ 2095 m0->m_pkthdr.rcv_tstmp = 2096 last_flit_to_ns(sc, d->rsp.u.last_flit); 2097 #ifdef notyet 2098 m0->m_flags |= M_TSTMP; 2099 #endif 2100 } 2101 2102 #ifdef NUMA 2103 m0->m_pkthdr.numa_domain = ifp->if_numa_domain; 2104 #endif 2105 #if defined(INET) || defined(INET6) 2106 if (rxq->iq.flags & IQ_LRO_ENABLED && tnl_type == 0 && 2107 (M_HASHTYPE_GET(m0) == M_HASHTYPE_RSS_TCP_IPV4 || 2108 M_HASHTYPE_GET(m0) == M_HASHTYPE_RSS_TCP_IPV6)) { 2109 if (sort_before_lro(lro)) { 2110 tcp_lro_queue_mbuf(lro, m0); 2111 return (0); /* queued for sort, then LRO */ 2112 } 2113 if (tcp_lro_rx(lro, m0, 0) == 0) 2114 return (0); /* queued for LRO */ 2115 } 2116 #endif 2117 ifp->if_input(ifp, m0); 2118 2119 return (0); 2120 } 2121 2122 /* 2123 * Must drain the wrq or make sure that someone else will. 2124 */ 2125 static void 2126 wrq_tx_drain(void *arg, int n) 2127 { 2128 struct sge_wrq *wrq = arg; 2129 struct sge_eq *eq = &wrq->eq; 2130 2131 EQ_LOCK(eq); 2132 if (TAILQ_EMPTY(&wrq->incomplete_wrs) && !STAILQ_EMPTY(&wrq->wr_list)) 2133 drain_wrq_wr_list(wrq->adapter, wrq); 2134 EQ_UNLOCK(eq); 2135 } 2136 2137 static void 2138 drain_wrq_wr_list(struct adapter *sc, struct sge_wrq *wrq) 2139 { 2140 struct sge_eq *eq = &wrq->eq; 2141 u_int available, dbdiff; /* # of hardware descriptors */ 2142 u_int n; 2143 struct wrqe *wr; 2144 struct fw_eth_tx_pkt_wr *dst; /* any fw WR struct will do */ 2145 2146 EQ_LOCK_ASSERT_OWNED(eq); 2147 MPASS(TAILQ_EMPTY(&wrq->incomplete_wrs)); 2148 wr = STAILQ_FIRST(&wrq->wr_list); 2149 MPASS(wr != NULL); /* Must be called with something useful to do */ 2150 MPASS(eq->pidx == eq->dbidx); 2151 dbdiff = 0; 2152 2153 do { 2154 eq->cidx = read_hw_cidx(eq); 2155 if (eq->pidx == eq->cidx) 2156 available = eq->sidx - 1; 2157 else 2158 available = IDXDIFF(eq->cidx, eq->pidx, eq->sidx) - 1; 2159 2160 MPASS(wr->wrq == wrq); 2161 n = howmany(wr->wr_len, EQ_ESIZE); 2162 if (available < n) 2163 break; 2164 2165 dst = (void *)&eq->desc[eq->pidx]; 2166 if (__predict_true(eq->sidx - eq->pidx > n)) { 2167 /* Won't wrap, won't end exactly at the status page. */ 2168 bcopy(&wr->wr[0], dst, wr->wr_len); 2169 eq->pidx += n; 2170 } else { 2171 int first_portion = (eq->sidx - eq->pidx) * EQ_ESIZE; 2172 2173 bcopy(&wr->wr[0], dst, first_portion); 2174 if (wr->wr_len > first_portion) { 2175 bcopy(&wr->wr[first_portion], &eq->desc[0], 2176 wr->wr_len - first_portion); 2177 } 2178 eq->pidx = n - (eq->sidx - eq->pidx); 2179 } 2180 wrq->tx_wrs_copied++; 2181 2182 if (available < eq->sidx / 4 && 2183 atomic_cmpset_int(&eq->equiq, 0, 1)) { 2184 /* 2185 * XXX: This is not 100% reliable with some 2186 * types of WRs. But this is a very unusual 2187 * situation for an ofld/ctrl queue anyway. 2188 */ 2189 dst->equiq_to_len16 |= htobe32(F_FW_WR_EQUIQ | 2190 F_FW_WR_EQUEQ); 2191 } 2192 2193 dbdiff += n; 2194 if (dbdiff >= 16) { 2195 ring_eq_db(sc, eq, dbdiff); 2196 dbdiff = 0; 2197 } 2198 2199 STAILQ_REMOVE_HEAD(&wrq->wr_list, link); 2200 free_wrqe(wr); 2201 MPASS(wrq->nwr_pending > 0); 2202 wrq->nwr_pending--; 2203 MPASS(wrq->ndesc_needed >= n); 2204 wrq->ndesc_needed -= n; 2205 } while ((wr = STAILQ_FIRST(&wrq->wr_list)) != NULL); 2206 2207 if (dbdiff) 2208 ring_eq_db(sc, eq, dbdiff); 2209 } 2210 2211 /* 2212 * Doesn't fail. Holds on to work requests it can't send right away. 2213 */ 2214 void 2215 t4_wrq_tx_locked(struct adapter *sc, struct sge_wrq *wrq, struct wrqe *wr) 2216 { 2217 #ifdef INVARIANTS 2218 struct sge_eq *eq = &wrq->eq; 2219 #endif 2220 2221 EQ_LOCK_ASSERT_OWNED(eq); 2222 MPASS(wr != NULL); 2223 MPASS(wr->wr_len > 0 && wr->wr_len <= SGE_MAX_WR_LEN); 2224 MPASS((wr->wr_len & 0x7) == 0); 2225 2226 STAILQ_INSERT_TAIL(&wrq->wr_list, wr, link); 2227 wrq->nwr_pending++; 2228 wrq->ndesc_needed += howmany(wr->wr_len, EQ_ESIZE); 2229 2230 if (!TAILQ_EMPTY(&wrq->incomplete_wrs)) 2231 return; /* commit_wrq_wr will drain wr_list as well. */ 2232 2233 drain_wrq_wr_list(sc, wrq); 2234 2235 /* Doorbell must have caught up to the pidx. */ 2236 MPASS(eq->pidx == eq->dbidx); 2237 } 2238 2239 void 2240 t4_update_fl_bufsize(struct ifnet *ifp) 2241 { 2242 struct vi_info *vi = ifp->if_softc; 2243 struct adapter *sc = vi->adapter; 2244 struct sge_rxq *rxq; 2245 #ifdef TCP_OFFLOAD 2246 struct sge_ofld_rxq *ofld_rxq; 2247 #endif 2248 struct sge_fl *fl; 2249 int i, maxp; 2250 2251 maxp = max_rx_payload(sc, ifp, false); 2252 for_each_rxq(vi, i, rxq) { 2253 fl = &rxq->fl; 2254 2255 FL_LOCK(fl); 2256 fl->zidx = find_refill_source(sc, maxp, 2257 fl->flags & FL_BUF_PACKING); 2258 FL_UNLOCK(fl); 2259 } 2260 #ifdef TCP_OFFLOAD 2261 maxp = max_rx_payload(sc, ifp, true); 2262 for_each_ofld_rxq(vi, i, ofld_rxq) { 2263 fl = &ofld_rxq->fl; 2264 2265 FL_LOCK(fl); 2266 fl->zidx = find_refill_source(sc, maxp, 2267 fl->flags & FL_BUF_PACKING); 2268 FL_UNLOCK(fl); 2269 } 2270 #endif 2271 } 2272 2273 static inline int 2274 mbuf_nsegs(struct mbuf *m) 2275 { 2276 2277 M_ASSERTPKTHDR(m); 2278 KASSERT(m->m_pkthdr.inner_l5hlen > 0, 2279 ("%s: mbuf %p missing information on # of segments.", __func__, m)); 2280 2281 return (m->m_pkthdr.inner_l5hlen); 2282 } 2283 2284 static inline void 2285 set_mbuf_nsegs(struct mbuf *m, uint8_t nsegs) 2286 { 2287 2288 M_ASSERTPKTHDR(m); 2289 m->m_pkthdr.inner_l5hlen = nsegs; 2290 } 2291 2292 static inline int 2293 mbuf_cflags(struct mbuf *m) 2294 { 2295 2296 M_ASSERTPKTHDR(m); 2297 return (m->m_pkthdr.PH_loc.eight[4]); 2298 } 2299 2300 static inline void 2301 set_mbuf_cflags(struct mbuf *m, uint8_t flags) 2302 { 2303 2304 M_ASSERTPKTHDR(m); 2305 m->m_pkthdr.PH_loc.eight[4] = flags; 2306 } 2307 2308 static inline int 2309 mbuf_len16(struct mbuf *m) 2310 { 2311 int n; 2312 2313 M_ASSERTPKTHDR(m); 2314 n = m->m_pkthdr.PH_loc.eight[0]; 2315 if (!(mbuf_cflags(m) & MC_TLS)) 2316 MPASS(n > 0 && n <= SGE_MAX_WR_LEN / 16); 2317 2318 return (n); 2319 } 2320 2321 static inline void 2322 set_mbuf_len16(struct mbuf *m, uint8_t len16) 2323 { 2324 2325 M_ASSERTPKTHDR(m); 2326 if (!(mbuf_cflags(m) & MC_TLS)) 2327 MPASS(len16 > 0 && len16 <= SGE_MAX_WR_LEN / 16); 2328 m->m_pkthdr.PH_loc.eight[0] = len16; 2329 } 2330 2331 #ifdef RATELIMIT 2332 static inline int 2333 mbuf_eo_nsegs(struct mbuf *m) 2334 { 2335 2336 M_ASSERTPKTHDR(m); 2337 return (m->m_pkthdr.PH_loc.eight[1]); 2338 } 2339 2340 static inline void 2341 set_mbuf_eo_nsegs(struct mbuf *m, uint8_t nsegs) 2342 { 2343 2344 M_ASSERTPKTHDR(m); 2345 m->m_pkthdr.PH_loc.eight[1] = nsegs; 2346 } 2347 2348 static inline int 2349 mbuf_eo_len16(struct mbuf *m) 2350 { 2351 int n; 2352 2353 M_ASSERTPKTHDR(m); 2354 n = m->m_pkthdr.PH_loc.eight[2]; 2355 MPASS(n > 0 && n <= SGE_MAX_WR_LEN / 16); 2356 2357 return (n); 2358 } 2359 2360 static inline void 2361 set_mbuf_eo_len16(struct mbuf *m, uint8_t len16) 2362 { 2363 2364 M_ASSERTPKTHDR(m); 2365 m->m_pkthdr.PH_loc.eight[2] = len16; 2366 } 2367 2368 static inline int 2369 mbuf_eo_tsclk_tsoff(struct mbuf *m) 2370 { 2371 2372 M_ASSERTPKTHDR(m); 2373 return (m->m_pkthdr.PH_loc.eight[3]); 2374 } 2375 2376 static inline void 2377 set_mbuf_eo_tsclk_tsoff(struct mbuf *m, uint8_t tsclk_tsoff) 2378 { 2379 2380 M_ASSERTPKTHDR(m); 2381 m->m_pkthdr.PH_loc.eight[3] = tsclk_tsoff; 2382 } 2383 2384 static inline int 2385 needs_eo(struct cxgbe_snd_tag *cst) 2386 { 2387 2388 return (cst != NULL && cst->type == IF_SND_TAG_TYPE_RATE_LIMIT); 2389 } 2390 #endif 2391 2392 /* 2393 * Try to allocate an mbuf to contain a raw work request. To make it 2394 * easy to construct the work request, don't allocate a chain but a 2395 * single mbuf. 2396 */ 2397 struct mbuf * 2398 alloc_wr_mbuf(int len, int how) 2399 { 2400 struct mbuf *m; 2401 2402 if (len <= MHLEN) 2403 m = m_gethdr(how, MT_DATA); 2404 else if (len <= MCLBYTES) 2405 m = m_getcl(how, MT_DATA, M_PKTHDR); 2406 else 2407 m = NULL; 2408 if (m == NULL) 2409 return (NULL); 2410 m->m_pkthdr.len = len; 2411 m->m_len = len; 2412 set_mbuf_cflags(m, MC_RAW_WR); 2413 set_mbuf_len16(m, howmany(len, 16)); 2414 return (m); 2415 } 2416 2417 static inline bool 2418 needs_hwcsum(struct mbuf *m) 2419 { 2420 const uint32_t csum_flags = CSUM_IP | CSUM_IP_UDP | CSUM_IP_TCP | 2421 CSUM_IP_TSO | CSUM_INNER_IP | CSUM_INNER_IP_UDP | 2422 CSUM_INNER_IP_TCP | CSUM_INNER_IP_TSO | CSUM_IP6_UDP | 2423 CSUM_IP6_TCP | CSUM_IP6_TSO | CSUM_INNER_IP6_UDP | 2424 CSUM_INNER_IP6_TCP | CSUM_INNER_IP6_TSO; 2425 2426 M_ASSERTPKTHDR(m); 2427 2428 return (m->m_pkthdr.csum_flags & csum_flags); 2429 } 2430 2431 static inline bool 2432 needs_tso(struct mbuf *m) 2433 { 2434 const uint32_t csum_flags = CSUM_IP_TSO | CSUM_IP6_TSO | 2435 CSUM_INNER_IP_TSO | CSUM_INNER_IP6_TSO; 2436 2437 M_ASSERTPKTHDR(m); 2438 2439 return (m->m_pkthdr.csum_flags & csum_flags); 2440 } 2441 2442 static inline bool 2443 needs_vxlan_csum(struct mbuf *m) 2444 { 2445 2446 M_ASSERTPKTHDR(m); 2447 2448 return (m->m_pkthdr.csum_flags & CSUM_ENCAP_VXLAN); 2449 } 2450 2451 static inline bool 2452 needs_vxlan_tso(struct mbuf *m) 2453 { 2454 const uint32_t csum_flags = CSUM_ENCAP_VXLAN | CSUM_INNER_IP_TSO | 2455 CSUM_INNER_IP6_TSO; 2456 2457 M_ASSERTPKTHDR(m); 2458 2459 return ((m->m_pkthdr.csum_flags & csum_flags) != 0 && 2460 (m->m_pkthdr.csum_flags & csum_flags) != CSUM_ENCAP_VXLAN); 2461 } 2462 2463 static inline bool 2464 needs_inner_tcp_csum(struct mbuf *m) 2465 { 2466 const uint32_t csum_flags = CSUM_INNER_IP_TSO | CSUM_INNER_IP6_TSO; 2467 2468 M_ASSERTPKTHDR(m); 2469 2470 return (m->m_pkthdr.csum_flags & csum_flags); 2471 } 2472 2473 static inline bool 2474 needs_l3_csum(struct mbuf *m) 2475 { 2476 const uint32_t csum_flags = CSUM_IP | CSUM_IP_TSO | CSUM_INNER_IP | 2477 CSUM_INNER_IP_TSO; 2478 2479 M_ASSERTPKTHDR(m); 2480 2481 return (m->m_pkthdr.csum_flags & csum_flags); 2482 } 2483 2484 static inline bool 2485 needs_outer_tcp_csum(struct mbuf *m) 2486 { 2487 const uint32_t csum_flags = CSUM_IP_TCP | CSUM_IP_TSO | CSUM_IP6_TCP | 2488 CSUM_IP6_TSO; 2489 2490 M_ASSERTPKTHDR(m); 2491 2492 return (m->m_pkthdr.csum_flags & csum_flags); 2493 } 2494 2495 #ifdef RATELIMIT 2496 static inline bool 2497 needs_outer_l4_csum(struct mbuf *m) 2498 { 2499 const uint32_t csum_flags = CSUM_IP_UDP | CSUM_IP_TCP | CSUM_IP_TSO | 2500 CSUM_IP6_UDP | CSUM_IP6_TCP | CSUM_IP6_TSO; 2501 2502 M_ASSERTPKTHDR(m); 2503 2504 return (m->m_pkthdr.csum_flags & csum_flags); 2505 } 2506 2507 static inline bool 2508 needs_outer_udp_csum(struct mbuf *m) 2509 { 2510 const uint32_t csum_flags = CSUM_IP_UDP | CSUM_IP6_UDP; 2511 2512 M_ASSERTPKTHDR(m); 2513 2514 return (m->m_pkthdr.csum_flags & csum_flags); 2515 } 2516 #endif 2517 2518 static inline bool 2519 needs_vlan_insertion(struct mbuf *m) 2520 { 2521 2522 M_ASSERTPKTHDR(m); 2523 2524 return (m->m_flags & M_VLANTAG); 2525 } 2526 2527 static void * 2528 m_advance(struct mbuf **pm, int *poffset, int len) 2529 { 2530 struct mbuf *m = *pm; 2531 int offset = *poffset; 2532 uintptr_t p = 0; 2533 2534 MPASS(len > 0); 2535 2536 for (;;) { 2537 if (offset + len < m->m_len) { 2538 offset += len; 2539 p = mtod(m, uintptr_t) + offset; 2540 break; 2541 } 2542 len -= m->m_len - offset; 2543 m = m->m_next; 2544 offset = 0; 2545 MPASS(m != NULL); 2546 } 2547 *poffset = offset; 2548 *pm = m; 2549 return ((void *)p); 2550 } 2551 2552 static inline int 2553 count_mbuf_ext_pgs(struct mbuf *m, int skip, vm_paddr_t *nextaddr) 2554 { 2555 vm_paddr_t paddr; 2556 int i, len, off, pglen, pgoff, seglen, segoff; 2557 int nsegs = 0; 2558 2559 M_ASSERTEXTPG(m); 2560 off = mtod(m, vm_offset_t); 2561 len = m->m_len; 2562 off += skip; 2563 len -= skip; 2564 2565 if (m->m_epg_hdrlen != 0) { 2566 if (off >= m->m_epg_hdrlen) { 2567 off -= m->m_epg_hdrlen; 2568 } else { 2569 seglen = m->m_epg_hdrlen - off; 2570 segoff = off; 2571 seglen = min(seglen, len); 2572 off = 0; 2573 len -= seglen; 2574 paddr = pmap_kextract( 2575 (vm_offset_t)&m->m_epg_hdr[segoff]); 2576 if (*nextaddr != paddr) 2577 nsegs++; 2578 *nextaddr = paddr + seglen; 2579 } 2580 } 2581 pgoff = m->m_epg_1st_off; 2582 for (i = 0; i < m->m_epg_npgs && len > 0; i++) { 2583 pglen = m_epg_pagelen(m, i, pgoff); 2584 if (off >= pglen) { 2585 off -= pglen; 2586 pgoff = 0; 2587 continue; 2588 } 2589 seglen = pglen - off; 2590 segoff = pgoff + off; 2591 off = 0; 2592 seglen = min(seglen, len); 2593 len -= seglen; 2594 paddr = m->m_epg_pa[i] + segoff; 2595 if (*nextaddr != paddr) 2596 nsegs++; 2597 *nextaddr = paddr + seglen; 2598 pgoff = 0; 2599 }; 2600 if (len != 0) { 2601 seglen = min(len, m->m_epg_trllen - off); 2602 len -= seglen; 2603 paddr = pmap_kextract((vm_offset_t)&m->m_epg_trail[off]); 2604 if (*nextaddr != paddr) 2605 nsegs++; 2606 *nextaddr = paddr + seglen; 2607 } 2608 2609 return (nsegs); 2610 } 2611 2612 2613 /* 2614 * Can deal with empty mbufs in the chain that have m_len = 0, but the chain 2615 * must have at least one mbuf that's not empty. It is possible for this 2616 * routine to return 0 if skip accounts for all the contents of the mbuf chain. 2617 */ 2618 static inline int 2619 count_mbuf_nsegs(struct mbuf *m, int skip, uint8_t *cflags) 2620 { 2621 vm_paddr_t nextaddr, paddr; 2622 vm_offset_t va; 2623 int len, nsegs; 2624 2625 M_ASSERTPKTHDR(m); 2626 MPASS(m->m_pkthdr.len > 0); 2627 MPASS(m->m_pkthdr.len >= skip); 2628 2629 nsegs = 0; 2630 nextaddr = 0; 2631 for (; m; m = m->m_next) { 2632 len = m->m_len; 2633 if (__predict_false(len == 0)) 2634 continue; 2635 if (skip >= len) { 2636 skip -= len; 2637 continue; 2638 } 2639 if ((m->m_flags & M_EXTPG) != 0) { 2640 *cflags |= MC_NOMAP; 2641 nsegs += count_mbuf_ext_pgs(m, skip, &nextaddr); 2642 skip = 0; 2643 continue; 2644 } 2645 va = mtod(m, vm_offset_t) + skip; 2646 len -= skip; 2647 skip = 0; 2648 paddr = pmap_kextract(va); 2649 nsegs += sglist_count((void *)(uintptr_t)va, len); 2650 if (paddr == nextaddr) 2651 nsegs--; 2652 nextaddr = pmap_kextract(va + len - 1) + 1; 2653 } 2654 2655 return (nsegs); 2656 } 2657 2658 /* 2659 * The maximum number of segments that can fit in a WR. 2660 */ 2661 static int 2662 max_nsegs_allowed(struct mbuf *m, bool vm_wr) 2663 { 2664 2665 if (vm_wr) { 2666 if (needs_tso(m)) 2667 return (TX_SGL_SEGS_VM_TSO); 2668 return (TX_SGL_SEGS_VM); 2669 } 2670 2671 if (needs_tso(m)) { 2672 if (needs_vxlan_tso(m)) 2673 return (TX_SGL_SEGS_VXLAN_TSO); 2674 else 2675 return (TX_SGL_SEGS_TSO); 2676 } 2677 2678 return (TX_SGL_SEGS); 2679 } 2680 2681 /* 2682 * Analyze the mbuf to determine its tx needs. The mbuf passed in may change: 2683 * a) caller can assume it's been freed if this function returns with an error. 2684 * b) it may get defragged up if the gather list is too long for the hardware. 2685 */ 2686 int 2687 parse_pkt(struct mbuf **mp, bool vm_wr) 2688 { 2689 struct mbuf *m0 = *mp, *m; 2690 int rc, nsegs, defragged = 0, offset; 2691 struct ether_header *eh; 2692 void *l3hdr; 2693 #if defined(INET) || defined(INET6) 2694 struct tcphdr *tcp; 2695 #endif 2696 #if defined(KERN_TLS) || defined(RATELIMIT) 2697 struct cxgbe_snd_tag *cst; 2698 #endif 2699 uint16_t eh_type; 2700 uint8_t cflags; 2701 2702 cflags = 0; 2703 M_ASSERTPKTHDR(m0); 2704 if (__predict_false(m0->m_pkthdr.len < ETHER_HDR_LEN)) { 2705 rc = EINVAL; 2706 fail: 2707 m_freem(m0); 2708 *mp = NULL; 2709 return (rc); 2710 } 2711 restart: 2712 /* 2713 * First count the number of gather list segments in the payload. 2714 * Defrag the mbuf if nsegs exceeds the hardware limit. 2715 */ 2716 M_ASSERTPKTHDR(m0); 2717 MPASS(m0->m_pkthdr.len > 0); 2718 nsegs = count_mbuf_nsegs(m0, 0, &cflags); 2719 #if defined(KERN_TLS) || defined(RATELIMIT) 2720 if (m0->m_pkthdr.csum_flags & CSUM_SND_TAG) 2721 cst = mst_to_cst(m0->m_pkthdr.snd_tag); 2722 else 2723 cst = NULL; 2724 #endif 2725 #ifdef KERN_TLS 2726 if (cst != NULL && cst->type == IF_SND_TAG_TYPE_TLS) { 2727 int len16; 2728 2729 cflags |= MC_TLS; 2730 set_mbuf_cflags(m0, cflags); 2731 rc = t6_ktls_parse_pkt(m0, &nsegs, &len16); 2732 if (rc != 0) 2733 goto fail; 2734 set_mbuf_nsegs(m0, nsegs); 2735 set_mbuf_len16(m0, len16); 2736 return (0); 2737 } 2738 #endif 2739 if (nsegs > max_nsegs_allowed(m0, vm_wr)) { 2740 if (defragged++ > 0) { 2741 rc = EFBIG; 2742 goto fail; 2743 } 2744 counter_u64_add(defrags, 1); 2745 if ((m = m_defrag(m0, M_NOWAIT)) == NULL) { 2746 rc = ENOMEM; 2747 goto fail; 2748 } 2749 *mp = m0 = m; /* update caller's copy after defrag */ 2750 goto restart; 2751 } 2752 2753 if (__predict_false(nsegs > 2 && m0->m_pkthdr.len <= MHLEN && 2754 !(cflags & MC_NOMAP))) { 2755 counter_u64_add(pullups, 1); 2756 m0 = m_pullup(m0, m0->m_pkthdr.len); 2757 if (m0 == NULL) { 2758 /* Should have left well enough alone. */ 2759 rc = EFBIG; 2760 goto fail; 2761 } 2762 *mp = m0; /* update caller's copy after pullup */ 2763 goto restart; 2764 } 2765 set_mbuf_nsegs(m0, nsegs); 2766 set_mbuf_cflags(m0, cflags); 2767 calculate_mbuf_len16(m0, vm_wr); 2768 2769 #ifdef RATELIMIT 2770 /* 2771 * Ethofld is limited to TCP and UDP for now, and only when L4 hw 2772 * checksumming is enabled. needs_outer_l4_csum happens to check for 2773 * all the right things. 2774 */ 2775 if (__predict_false(needs_eo(cst) && !needs_outer_l4_csum(m0))) { 2776 m_snd_tag_rele(m0->m_pkthdr.snd_tag); 2777 m0->m_pkthdr.snd_tag = NULL; 2778 m0->m_pkthdr.csum_flags &= ~CSUM_SND_TAG; 2779 cst = NULL; 2780 } 2781 #endif 2782 2783 if (!needs_hwcsum(m0) 2784 #ifdef RATELIMIT 2785 && !needs_eo(cst) 2786 #endif 2787 ) 2788 return (0); 2789 2790 m = m0; 2791 eh = mtod(m, struct ether_header *); 2792 eh_type = ntohs(eh->ether_type); 2793 if (eh_type == ETHERTYPE_VLAN) { 2794 struct ether_vlan_header *evh = (void *)eh; 2795 2796 eh_type = ntohs(evh->evl_proto); 2797 m0->m_pkthdr.l2hlen = sizeof(*evh); 2798 } else 2799 m0->m_pkthdr.l2hlen = sizeof(*eh); 2800 2801 offset = 0; 2802 l3hdr = m_advance(&m, &offset, m0->m_pkthdr.l2hlen); 2803 2804 switch (eh_type) { 2805 #ifdef INET6 2806 case ETHERTYPE_IPV6: 2807 m0->m_pkthdr.l3hlen = sizeof(struct ip6_hdr); 2808 break; 2809 #endif 2810 #ifdef INET 2811 case ETHERTYPE_IP: 2812 { 2813 struct ip *ip = l3hdr; 2814 2815 if (needs_vxlan_csum(m0)) { 2816 /* Driver will do the outer IP hdr checksum. */ 2817 ip->ip_sum = 0; 2818 if (needs_vxlan_tso(m0)) { 2819 const uint16_t ipl = ip->ip_len; 2820 2821 ip->ip_len = 0; 2822 ip->ip_sum = ~in_cksum_hdr(ip); 2823 ip->ip_len = ipl; 2824 } else 2825 ip->ip_sum = in_cksum_hdr(ip); 2826 } 2827 m0->m_pkthdr.l3hlen = ip->ip_hl << 2; 2828 break; 2829 } 2830 #endif 2831 default: 2832 panic("%s: ethertype 0x%04x unknown. if_cxgbe must be compiled" 2833 " with the same INET/INET6 options as the kernel.", 2834 __func__, eh_type); 2835 } 2836 2837 if (needs_vxlan_csum(m0)) { 2838 m0->m_pkthdr.l4hlen = sizeof(struct udphdr); 2839 m0->m_pkthdr.l5hlen = sizeof(struct vxlan_header); 2840 2841 /* Inner headers. */ 2842 eh = m_advance(&m, &offset, m0->m_pkthdr.l3hlen + 2843 sizeof(struct udphdr) + sizeof(struct vxlan_header)); 2844 eh_type = ntohs(eh->ether_type); 2845 if (eh_type == ETHERTYPE_VLAN) { 2846 struct ether_vlan_header *evh = (void *)eh; 2847 2848 eh_type = ntohs(evh->evl_proto); 2849 m0->m_pkthdr.inner_l2hlen = sizeof(*evh); 2850 } else 2851 m0->m_pkthdr.inner_l2hlen = sizeof(*eh); 2852 l3hdr = m_advance(&m, &offset, m0->m_pkthdr.inner_l2hlen); 2853 2854 switch (eh_type) { 2855 #ifdef INET6 2856 case ETHERTYPE_IPV6: 2857 m0->m_pkthdr.inner_l3hlen = sizeof(struct ip6_hdr); 2858 break; 2859 #endif 2860 #ifdef INET 2861 case ETHERTYPE_IP: 2862 { 2863 struct ip *ip = l3hdr; 2864 2865 m0->m_pkthdr.inner_l3hlen = ip->ip_hl << 2; 2866 break; 2867 } 2868 #endif 2869 default: 2870 panic("%s: VXLAN hw offload requested with unknown " 2871 "ethertype 0x%04x. if_cxgbe must be compiled" 2872 " with the same INET/INET6 options as the kernel.", 2873 __func__, eh_type); 2874 } 2875 #if defined(INET) || defined(INET6) 2876 if (needs_inner_tcp_csum(m0)) { 2877 tcp = m_advance(&m, &offset, m0->m_pkthdr.inner_l3hlen); 2878 m0->m_pkthdr.inner_l4hlen = tcp->th_off * 4; 2879 } 2880 #endif 2881 MPASS((m0->m_pkthdr.csum_flags & CSUM_SND_TAG) == 0); 2882 m0->m_pkthdr.csum_flags &= CSUM_INNER_IP6_UDP | 2883 CSUM_INNER_IP6_TCP | CSUM_INNER_IP6_TSO | CSUM_INNER_IP | 2884 CSUM_INNER_IP_UDP | CSUM_INNER_IP_TCP | CSUM_INNER_IP_TSO | 2885 CSUM_ENCAP_VXLAN; 2886 } 2887 2888 #if defined(INET) || defined(INET6) 2889 if (needs_outer_tcp_csum(m0)) { 2890 tcp = m_advance(&m, &offset, m0->m_pkthdr.l3hlen); 2891 m0->m_pkthdr.l4hlen = tcp->th_off * 4; 2892 #ifdef RATELIMIT 2893 if (tsclk >= 0 && *(uint32_t *)(tcp + 1) == ntohl(0x0101080a)) { 2894 set_mbuf_eo_tsclk_tsoff(m0, 2895 V_FW_ETH_TX_EO_WR_TSCLK(tsclk) | 2896 V_FW_ETH_TX_EO_WR_TSOFF(sizeof(*tcp) / 2 + 1)); 2897 } else 2898 set_mbuf_eo_tsclk_tsoff(m0, 0); 2899 } else if (needs_outer_udp_csum(m0)) { 2900 m0->m_pkthdr.l4hlen = sizeof(struct udphdr); 2901 #endif 2902 } 2903 #ifdef RATELIMIT 2904 if (needs_eo(cst)) { 2905 u_int immhdrs; 2906 2907 /* EO WRs have the headers in the WR and not the GL. */ 2908 immhdrs = m0->m_pkthdr.l2hlen + m0->m_pkthdr.l3hlen + 2909 m0->m_pkthdr.l4hlen; 2910 cflags = 0; 2911 nsegs = count_mbuf_nsegs(m0, immhdrs, &cflags); 2912 MPASS(cflags == mbuf_cflags(m0)); 2913 set_mbuf_eo_nsegs(m0, nsegs); 2914 set_mbuf_eo_len16(m0, 2915 txpkt_eo_len16(nsegs, immhdrs, needs_tso(m0))); 2916 } 2917 #endif 2918 #endif 2919 MPASS(m0 == *mp); 2920 return (0); 2921 } 2922 2923 void * 2924 start_wrq_wr(struct sge_wrq *wrq, int len16, struct wrq_cookie *cookie) 2925 { 2926 struct sge_eq *eq = &wrq->eq; 2927 struct adapter *sc = wrq->adapter; 2928 int ndesc, available; 2929 struct wrqe *wr; 2930 void *w; 2931 2932 MPASS(len16 > 0); 2933 ndesc = tx_len16_to_desc(len16); 2934 MPASS(ndesc > 0 && ndesc <= SGE_MAX_WR_NDESC); 2935 2936 EQ_LOCK(eq); 2937 2938 if (TAILQ_EMPTY(&wrq->incomplete_wrs) && !STAILQ_EMPTY(&wrq->wr_list)) 2939 drain_wrq_wr_list(sc, wrq); 2940 2941 if (!STAILQ_EMPTY(&wrq->wr_list)) { 2942 slowpath: 2943 EQ_UNLOCK(eq); 2944 wr = alloc_wrqe(len16 * 16, wrq); 2945 if (__predict_false(wr == NULL)) 2946 return (NULL); 2947 cookie->pidx = -1; 2948 cookie->ndesc = ndesc; 2949 return (&wr->wr); 2950 } 2951 2952 eq->cidx = read_hw_cidx(eq); 2953 if (eq->pidx == eq->cidx) 2954 available = eq->sidx - 1; 2955 else 2956 available = IDXDIFF(eq->cidx, eq->pidx, eq->sidx) - 1; 2957 if (available < ndesc) 2958 goto slowpath; 2959 2960 cookie->pidx = eq->pidx; 2961 cookie->ndesc = ndesc; 2962 TAILQ_INSERT_TAIL(&wrq->incomplete_wrs, cookie, link); 2963 2964 w = &eq->desc[eq->pidx]; 2965 IDXINCR(eq->pidx, ndesc, eq->sidx); 2966 if (__predict_false(cookie->pidx + ndesc > eq->sidx)) { 2967 w = &wrq->ss[0]; 2968 wrq->ss_pidx = cookie->pidx; 2969 wrq->ss_len = len16 * 16; 2970 } 2971 2972 EQ_UNLOCK(eq); 2973 2974 return (w); 2975 } 2976 2977 void 2978 commit_wrq_wr(struct sge_wrq *wrq, void *w, struct wrq_cookie *cookie) 2979 { 2980 struct sge_eq *eq = &wrq->eq; 2981 struct adapter *sc = wrq->adapter; 2982 int ndesc, pidx; 2983 struct wrq_cookie *prev, *next; 2984 2985 if (cookie->pidx == -1) { 2986 struct wrqe *wr = __containerof(w, struct wrqe, wr); 2987 2988 t4_wrq_tx(sc, wr); 2989 return; 2990 } 2991 2992 if (__predict_false(w == &wrq->ss[0])) { 2993 int n = (eq->sidx - wrq->ss_pidx) * EQ_ESIZE; 2994 2995 MPASS(wrq->ss_len > n); /* WR had better wrap around. */ 2996 bcopy(&wrq->ss[0], &eq->desc[wrq->ss_pidx], n); 2997 bcopy(&wrq->ss[n], &eq->desc[0], wrq->ss_len - n); 2998 wrq->tx_wrs_ss++; 2999 } else 3000 wrq->tx_wrs_direct++; 3001 3002 EQ_LOCK(eq); 3003 ndesc = cookie->ndesc; /* Can be more than SGE_MAX_WR_NDESC here. */ 3004 pidx = cookie->pidx; 3005 MPASS(pidx >= 0 && pidx < eq->sidx); 3006 prev = TAILQ_PREV(cookie, wrq_incomplete_wrs, link); 3007 next = TAILQ_NEXT(cookie, link); 3008 if (prev == NULL) { 3009 MPASS(pidx == eq->dbidx); 3010 if (next == NULL || ndesc >= 16) { 3011 int available; 3012 struct fw_eth_tx_pkt_wr *dst; /* any fw WR struct will do */ 3013 3014 /* 3015 * Note that the WR via which we'll request tx updates 3016 * is at pidx and not eq->pidx, which has moved on 3017 * already. 3018 */ 3019 dst = (void *)&eq->desc[pidx]; 3020 available = IDXDIFF(eq->cidx, eq->pidx, eq->sidx) - 1; 3021 if (available < eq->sidx / 4 && 3022 atomic_cmpset_int(&eq->equiq, 0, 1)) { 3023 /* 3024 * XXX: This is not 100% reliable with some 3025 * types of WRs. But this is a very unusual 3026 * situation for an ofld/ctrl queue anyway. 3027 */ 3028 dst->equiq_to_len16 |= htobe32(F_FW_WR_EQUIQ | 3029 F_FW_WR_EQUEQ); 3030 } 3031 3032 ring_eq_db(wrq->adapter, eq, ndesc); 3033 } else { 3034 MPASS(IDXDIFF(next->pidx, pidx, eq->sidx) == ndesc); 3035 next->pidx = pidx; 3036 next->ndesc += ndesc; 3037 } 3038 } else { 3039 MPASS(IDXDIFF(pidx, prev->pidx, eq->sidx) == prev->ndesc); 3040 prev->ndesc += ndesc; 3041 } 3042 TAILQ_REMOVE(&wrq->incomplete_wrs, cookie, link); 3043 3044 if (TAILQ_EMPTY(&wrq->incomplete_wrs) && !STAILQ_EMPTY(&wrq->wr_list)) 3045 drain_wrq_wr_list(sc, wrq); 3046 3047 #ifdef INVARIANTS 3048 if (TAILQ_EMPTY(&wrq->incomplete_wrs)) { 3049 /* Doorbell must have caught up to the pidx. */ 3050 MPASS(wrq->eq.pidx == wrq->eq.dbidx); 3051 } 3052 #endif 3053 EQ_UNLOCK(eq); 3054 } 3055 3056 static u_int 3057 can_resume_eth_tx(struct mp_ring *r) 3058 { 3059 struct sge_eq *eq = r->cookie; 3060 3061 return (total_available_tx_desc(eq) > eq->sidx / 8); 3062 } 3063 3064 static inline bool 3065 cannot_use_txpkts(struct mbuf *m) 3066 { 3067 /* maybe put a GL limit too, to avoid silliness? */ 3068 3069 return (needs_tso(m) || (mbuf_cflags(m) & (MC_RAW_WR | MC_TLS)) != 0); 3070 } 3071 3072 static inline int 3073 discard_tx(struct sge_eq *eq) 3074 { 3075 3076 return ((eq->flags & (EQ_ENABLED | EQ_QFLUSH)) != EQ_ENABLED); 3077 } 3078 3079 static inline int 3080 wr_can_update_eq(void *p) 3081 { 3082 struct fw_eth_tx_pkts_wr *wr = p; 3083 3084 switch (G_FW_WR_OP(be32toh(wr->op_pkd))) { 3085 case FW_ULPTX_WR: 3086 case FW_ETH_TX_PKT_WR: 3087 case FW_ETH_TX_PKTS_WR: 3088 case FW_ETH_TX_PKTS2_WR: 3089 case FW_ETH_TX_PKT_VM_WR: 3090 case FW_ETH_TX_PKTS_VM_WR: 3091 return (1); 3092 default: 3093 return (0); 3094 } 3095 } 3096 3097 static inline void 3098 set_txupdate_flags(struct sge_txq *txq, u_int avail, 3099 struct fw_eth_tx_pkt_wr *wr) 3100 { 3101 struct sge_eq *eq = &txq->eq; 3102 struct txpkts *txp = &txq->txp; 3103 3104 if ((txp->npkt > 0 || avail < eq->sidx / 2) && 3105 atomic_cmpset_int(&eq->equiq, 0, 1)) { 3106 wr->equiq_to_len16 |= htobe32(F_FW_WR_EQUEQ | F_FW_WR_EQUIQ); 3107 eq->equeqidx = eq->pidx; 3108 } else if (IDXDIFF(eq->pidx, eq->equeqidx, eq->sidx) >= 32) { 3109 wr->equiq_to_len16 |= htobe32(F_FW_WR_EQUEQ); 3110 eq->equeqidx = eq->pidx; 3111 } 3112 } 3113 3114 /* 3115 * r->items[cidx] to r->items[pidx], with a wraparound at r->size, are ready to 3116 * be consumed. Return the actual number consumed. 0 indicates a stall. 3117 */ 3118 static u_int 3119 eth_tx(struct mp_ring *r, u_int cidx, u_int pidx, bool *coalescing) 3120 { 3121 struct sge_txq *txq = r->cookie; 3122 struct ifnet *ifp = txq->ifp; 3123 struct sge_eq *eq = &txq->eq; 3124 struct txpkts *txp = &txq->txp; 3125 struct vi_info *vi = ifp->if_softc; 3126 struct adapter *sc = vi->adapter; 3127 u_int total, remaining; /* # of packets */ 3128 u_int n, avail, dbdiff; /* # of hardware descriptors */ 3129 int i, rc; 3130 struct mbuf *m0; 3131 bool snd; 3132 void *wr; /* start of the last WR written to the ring */ 3133 3134 TXQ_LOCK_ASSERT_OWNED(txq); 3135 3136 remaining = IDXDIFF(pidx, cidx, r->size); 3137 if (__predict_false(discard_tx(eq))) { 3138 for (i = 0; i < txp->npkt; i++) 3139 m_freem(txp->mb[i]); 3140 txp->npkt = 0; 3141 while (cidx != pidx) { 3142 m0 = r->items[cidx]; 3143 m_freem(m0); 3144 if (++cidx == r->size) 3145 cidx = 0; 3146 } 3147 reclaim_tx_descs(txq, eq->sidx); 3148 *coalescing = false; 3149 return (remaining); /* emptied */ 3150 } 3151 3152 /* How many hardware descriptors do we have readily available. */ 3153 if (eq->pidx == eq->cidx) { 3154 avail = eq->sidx - 1; 3155 if (txp->score++ >= 5) 3156 txp->score = 5; /* tx is completely idle, reset. */ 3157 } else 3158 avail = IDXDIFF(eq->cidx, eq->pidx, eq->sidx) - 1; 3159 3160 total = 0; 3161 if (remaining == 0) { 3162 if (txp->score-- == 1) /* egr_update had to drain txpkts */ 3163 txp->score = 1; 3164 goto send_txpkts; 3165 } 3166 3167 dbdiff = 0; 3168 MPASS(remaining > 0); 3169 while (remaining > 0) { 3170 m0 = r->items[cidx]; 3171 M_ASSERTPKTHDR(m0); 3172 MPASS(m0->m_nextpkt == NULL); 3173 3174 if (avail < 2 * SGE_MAX_WR_NDESC) 3175 avail += reclaim_tx_descs(txq, 64); 3176 3177 if (txp->npkt > 0 || remaining > 1 || txp->score > 3 || 3178 atomic_load_int(&txq->eq.equiq) != 0) { 3179 if (vi->flags & TX_USES_VM_WR) 3180 rc = add_to_txpkts_vf(sc, txq, m0, avail, &snd); 3181 else 3182 rc = add_to_txpkts_pf(sc, txq, m0, avail, &snd); 3183 } else { 3184 snd = false; 3185 rc = EINVAL; 3186 } 3187 if (snd) { 3188 MPASS(txp->npkt > 0); 3189 for (i = 0; i < txp->npkt; i++) 3190 ETHER_BPF_MTAP(ifp, txp->mb[i]); 3191 if (txp->npkt > 1) { 3192 if (txp->score++ >= 10) 3193 txp->score = 10; 3194 MPASS(avail >= tx_len16_to_desc(txp->len16)); 3195 if (vi->flags & TX_USES_VM_WR) 3196 n = write_txpkts_vm_wr(sc, txq); 3197 else 3198 n = write_txpkts_wr(sc, txq); 3199 } else { 3200 MPASS(avail >= 3201 tx_len16_to_desc(mbuf_len16(txp->mb[0]))); 3202 if (vi->flags & TX_USES_VM_WR) 3203 n = write_txpkt_vm_wr(sc, txq, 3204 txp->mb[0]); 3205 else 3206 n = write_txpkt_wr(sc, txq, txp->mb[0], 3207 avail); 3208 } 3209 MPASS(n <= SGE_MAX_WR_NDESC); 3210 avail -= n; 3211 dbdiff += n; 3212 wr = &eq->desc[eq->pidx]; 3213 IDXINCR(eq->pidx, n, eq->sidx); 3214 txp->npkt = 0; /* emptied */ 3215 } 3216 if (rc == 0) { 3217 /* m0 was coalesced into txq->txpkts. */ 3218 goto next_mbuf; 3219 } 3220 if (rc == EAGAIN) { 3221 /* 3222 * m0 is suitable for tx coalescing but could not be 3223 * combined with the existing txq->txpkts, which has now 3224 * been transmitted. Start a new txpkts with m0. 3225 */ 3226 MPASS(snd); 3227 MPASS(txp->npkt == 0); 3228 continue; 3229 } 3230 3231 MPASS(rc != 0 && rc != EAGAIN); 3232 MPASS(txp->npkt == 0); 3233 3234 n = tx_len16_to_desc(mbuf_len16(m0)); 3235 if (__predict_false(avail < n)) { 3236 avail += reclaim_tx_descs(txq, min(n, 32)); 3237 if (avail < n) 3238 break; /* out of descriptors */ 3239 } 3240 3241 wr = &eq->desc[eq->pidx]; 3242 if (mbuf_cflags(m0) & MC_RAW_WR) { 3243 n = write_raw_wr(txq, wr, m0, avail); 3244 #ifdef KERN_TLS 3245 } else if (mbuf_cflags(m0) & MC_TLS) { 3246 ETHER_BPF_MTAP(ifp, m0); 3247 n = t6_ktls_write_wr(txq, wr, m0, mbuf_nsegs(m0), 3248 avail); 3249 #endif 3250 } else { 3251 ETHER_BPF_MTAP(ifp, m0); 3252 if (vi->flags & TX_USES_VM_WR) 3253 n = write_txpkt_vm_wr(sc, txq, m0); 3254 else 3255 n = write_txpkt_wr(sc, txq, m0, avail); 3256 } 3257 MPASS(n >= 1 && n <= avail); 3258 if (!(mbuf_cflags(m0) & MC_TLS)) 3259 MPASS(n <= SGE_MAX_WR_NDESC); 3260 3261 avail -= n; 3262 dbdiff += n; 3263 IDXINCR(eq->pidx, n, eq->sidx); 3264 3265 if (dbdiff >= 512 / EQ_ESIZE) { /* X_FETCHBURSTMAX_512B */ 3266 if (wr_can_update_eq(wr)) 3267 set_txupdate_flags(txq, avail, wr); 3268 ring_eq_db(sc, eq, dbdiff); 3269 avail += reclaim_tx_descs(txq, 32); 3270 dbdiff = 0; 3271 } 3272 next_mbuf: 3273 total++; 3274 remaining--; 3275 if (__predict_false(++cidx == r->size)) 3276 cidx = 0; 3277 } 3278 if (dbdiff != 0) { 3279 if (wr_can_update_eq(wr)) 3280 set_txupdate_flags(txq, avail, wr); 3281 ring_eq_db(sc, eq, dbdiff); 3282 reclaim_tx_descs(txq, 32); 3283 } else if (eq->pidx == eq->cidx && txp->npkt > 0 && 3284 atomic_load_int(&txq->eq.equiq) == 0) { 3285 /* 3286 * If nothing was submitted to the chip for tx (it was coalesced 3287 * into txpkts instead) and there is no tx update outstanding 3288 * then we need to send txpkts now. 3289 */ 3290 send_txpkts: 3291 MPASS(txp->npkt > 0); 3292 for (i = 0; i < txp->npkt; i++) 3293 ETHER_BPF_MTAP(ifp, txp->mb[i]); 3294 if (txp->npkt > 1) { 3295 MPASS(avail >= tx_len16_to_desc(txp->len16)); 3296 if (vi->flags & TX_USES_VM_WR) 3297 n = write_txpkts_vm_wr(sc, txq); 3298 else 3299 n = write_txpkts_wr(sc, txq); 3300 } else { 3301 MPASS(avail >= 3302 tx_len16_to_desc(mbuf_len16(txp->mb[0]))); 3303 if (vi->flags & TX_USES_VM_WR) 3304 n = write_txpkt_vm_wr(sc, txq, txp->mb[0]); 3305 else 3306 n = write_txpkt_wr(sc, txq, txp->mb[0], avail); 3307 } 3308 MPASS(n <= SGE_MAX_WR_NDESC); 3309 wr = &eq->desc[eq->pidx]; 3310 IDXINCR(eq->pidx, n, eq->sidx); 3311 txp->npkt = 0; /* emptied */ 3312 3313 MPASS(wr_can_update_eq(wr)); 3314 set_txupdate_flags(txq, avail - n, wr); 3315 ring_eq_db(sc, eq, n); 3316 reclaim_tx_descs(txq, 32); 3317 } 3318 *coalescing = txp->npkt > 0; 3319 3320 return (total); 3321 } 3322 3323 static inline void 3324 init_iq(struct sge_iq *iq, struct adapter *sc, int tmr_idx, int pktc_idx, 3325 int qsize) 3326 { 3327 3328 KASSERT(tmr_idx >= 0 && tmr_idx < SGE_NTIMERS, 3329 ("%s: bad tmr_idx %d", __func__, tmr_idx)); 3330 KASSERT(pktc_idx < SGE_NCOUNTERS, /* -ve is ok, means don't use */ 3331 ("%s: bad pktc_idx %d", __func__, pktc_idx)); 3332 3333 iq->flags = 0; 3334 iq->adapter = sc; 3335 iq->intr_params = V_QINTR_TIMER_IDX(tmr_idx); 3336 iq->intr_pktc_idx = SGE_NCOUNTERS - 1; 3337 if (pktc_idx >= 0) { 3338 iq->intr_params |= F_QINTR_CNT_EN; 3339 iq->intr_pktc_idx = pktc_idx; 3340 } 3341 iq->qsize = roundup2(qsize, 16); /* See FW_IQ_CMD/iqsize */ 3342 iq->sidx = iq->qsize - sc->params.sge.spg_len / IQ_ESIZE; 3343 } 3344 3345 static inline void 3346 init_fl(struct adapter *sc, struct sge_fl *fl, int qsize, int maxp, char *name) 3347 { 3348 3349 fl->qsize = qsize; 3350 fl->sidx = qsize - sc->params.sge.spg_len / EQ_ESIZE; 3351 strlcpy(fl->lockname, name, sizeof(fl->lockname)); 3352 if (sc->flags & BUF_PACKING_OK && 3353 ((!is_t4(sc) && buffer_packing) || /* T5+: enabled unless 0 */ 3354 (is_t4(sc) && buffer_packing == 1)))/* T4: disabled unless 1 */ 3355 fl->flags |= FL_BUF_PACKING; 3356 fl->zidx = find_refill_source(sc, maxp, fl->flags & FL_BUF_PACKING); 3357 fl->safe_zidx = sc->sge.safe_zidx; 3358 } 3359 3360 static inline void 3361 init_eq(struct adapter *sc, struct sge_eq *eq, int eqtype, int qsize, 3362 uint8_t tx_chan, uint16_t iqid, char *name) 3363 { 3364 KASSERT(eqtype <= EQ_TYPEMASK, ("%s: bad qtype %d", __func__, eqtype)); 3365 3366 eq->flags = eqtype & EQ_TYPEMASK; 3367 eq->tx_chan = tx_chan; 3368 eq->iqid = iqid; 3369 eq->sidx = qsize - sc->params.sge.spg_len / EQ_ESIZE; 3370 strlcpy(eq->lockname, name, sizeof(eq->lockname)); 3371 } 3372 3373 static int 3374 alloc_ring(struct adapter *sc, size_t len, bus_dma_tag_t *tag, 3375 bus_dmamap_t *map, bus_addr_t *pa, void **va) 3376 { 3377 int rc; 3378 3379 rc = bus_dma_tag_create(sc->dmat, 512, 0, BUS_SPACE_MAXADDR, 3380 BUS_SPACE_MAXADDR, NULL, NULL, len, 1, len, 0, NULL, NULL, tag); 3381 if (rc != 0) { 3382 device_printf(sc->dev, "cannot allocate DMA tag: %d\n", rc); 3383 goto done; 3384 } 3385 3386 rc = bus_dmamem_alloc(*tag, va, 3387 BUS_DMA_WAITOK | BUS_DMA_COHERENT | BUS_DMA_ZERO, map); 3388 if (rc != 0) { 3389 device_printf(sc->dev, "cannot allocate DMA memory: %d\n", rc); 3390 goto done; 3391 } 3392 3393 rc = bus_dmamap_load(*tag, *map, *va, len, oneseg_dma_callback, pa, 0); 3394 if (rc != 0) { 3395 device_printf(sc->dev, "cannot load DMA map: %d\n", rc); 3396 goto done; 3397 } 3398 done: 3399 if (rc) 3400 free_ring(sc, *tag, *map, *pa, *va); 3401 3402 return (rc); 3403 } 3404 3405 static int 3406 free_ring(struct adapter *sc, bus_dma_tag_t tag, bus_dmamap_t map, 3407 bus_addr_t pa, void *va) 3408 { 3409 if (pa) 3410 bus_dmamap_unload(tag, map); 3411 if (va) 3412 bus_dmamem_free(tag, va, map); 3413 if (tag) 3414 bus_dma_tag_destroy(tag); 3415 3416 return (0); 3417 } 3418 3419 /* 3420 * Allocates the ring for an ingress queue and an optional freelist. If the 3421 * freelist is specified it will be allocated and then associated with the 3422 * ingress queue. 3423 * 3424 * Returns errno on failure. Resources allocated up to that point may still be 3425 * allocated. Caller is responsible for cleanup in case this function fails. 3426 * 3427 * If the ingress queue will take interrupts directly then the intr_idx 3428 * specifies the vector, starting from 0. -1 means the interrupts for this 3429 * queue should be forwarded to the fwq. 3430 */ 3431 static int 3432 alloc_iq_fl(struct vi_info *vi, struct sge_iq *iq, struct sge_fl *fl, 3433 int intr_idx, int cong) 3434 { 3435 int rc, i, cntxt_id; 3436 size_t len; 3437 struct fw_iq_cmd c; 3438 struct port_info *pi = vi->pi; 3439 struct adapter *sc = iq->adapter; 3440 struct sge_params *sp = &sc->params.sge; 3441 __be32 v = 0; 3442 3443 len = iq->qsize * IQ_ESIZE; 3444 rc = alloc_ring(sc, len, &iq->desc_tag, &iq->desc_map, &iq->ba, 3445 (void **)&iq->desc); 3446 if (rc != 0) 3447 return (rc); 3448 3449 bzero(&c, sizeof(c)); 3450 c.op_to_vfn = htobe32(V_FW_CMD_OP(FW_IQ_CMD) | F_FW_CMD_REQUEST | 3451 F_FW_CMD_WRITE | F_FW_CMD_EXEC | V_FW_IQ_CMD_PFN(sc->pf) | 3452 V_FW_IQ_CMD_VFN(0)); 3453 3454 c.alloc_to_len16 = htobe32(F_FW_IQ_CMD_ALLOC | F_FW_IQ_CMD_IQSTART | 3455 FW_LEN16(c)); 3456 3457 /* Special handling for firmware event queue */ 3458 if (iq == &sc->sge.fwq) 3459 v |= F_FW_IQ_CMD_IQASYNCH; 3460 3461 if (intr_idx < 0) { 3462 /* Forwarded interrupts, all headed to fwq */ 3463 v |= F_FW_IQ_CMD_IQANDST; 3464 v |= V_FW_IQ_CMD_IQANDSTINDEX(sc->sge.fwq.cntxt_id); 3465 } else { 3466 KASSERT(intr_idx < sc->intr_count, 3467 ("%s: invalid direct intr_idx %d", __func__, intr_idx)); 3468 v |= V_FW_IQ_CMD_IQANDSTINDEX(intr_idx); 3469 } 3470 3471 c.type_to_iqandstindex = htobe32(v | 3472 V_FW_IQ_CMD_TYPE(FW_IQ_TYPE_FL_INT_CAP) | 3473 V_FW_IQ_CMD_VIID(vi->viid) | 3474 V_FW_IQ_CMD_IQANUD(X_UPDATEDELIVERY_INTERRUPT)); 3475 c.iqdroprss_to_iqesize = htobe16(V_FW_IQ_CMD_IQPCIECH(pi->tx_chan) | 3476 F_FW_IQ_CMD_IQGTSMODE | 3477 V_FW_IQ_CMD_IQINTCNTTHRESH(iq->intr_pktc_idx) | 3478 V_FW_IQ_CMD_IQESIZE(ilog2(IQ_ESIZE) - 4)); 3479 c.iqsize = htobe16(iq->qsize); 3480 c.iqaddr = htobe64(iq->ba); 3481 if (cong >= 0) 3482 c.iqns_to_fl0congen = htobe32(F_FW_IQ_CMD_IQFLINTCONGEN); 3483 3484 if (fl) { 3485 mtx_init(&fl->fl_lock, fl->lockname, NULL, MTX_DEF); 3486 3487 len = fl->qsize * EQ_ESIZE; 3488 rc = alloc_ring(sc, len, &fl->desc_tag, &fl->desc_map, 3489 &fl->ba, (void **)&fl->desc); 3490 if (rc) 3491 return (rc); 3492 3493 /* Allocate space for one software descriptor per buffer. */ 3494 rc = alloc_fl_sdesc(fl); 3495 if (rc != 0) { 3496 device_printf(sc->dev, 3497 "failed to setup fl software descriptors: %d\n", 3498 rc); 3499 return (rc); 3500 } 3501 3502 if (fl->flags & FL_BUF_PACKING) { 3503 fl->lowat = roundup2(sp->fl_starve_threshold2, 8); 3504 fl->buf_boundary = sp->pack_boundary; 3505 } else { 3506 fl->lowat = roundup2(sp->fl_starve_threshold, 8); 3507 fl->buf_boundary = 16; 3508 } 3509 if (fl_pad && fl->buf_boundary < sp->pad_boundary) 3510 fl->buf_boundary = sp->pad_boundary; 3511 3512 c.iqns_to_fl0congen |= 3513 htobe32(V_FW_IQ_CMD_FL0HOSTFCMODE(X_HOSTFCMODE_NONE) | 3514 F_FW_IQ_CMD_FL0FETCHRO | F_FW_IQ_CMD_FL0DATARO | 3515 (fl_pad ? F_FW_IQ_CMD_FL0PADEN : 0) | 3516 (fl->flags & FL_BUF_PACKING ? F_FW_IQ_CMD_FL0PACKEN : 3517 0)); 3518 if (cong >= 0) { 3519 c.iqns_to_fl0congen |= 3520 htobe32(V_FW_IQ_CMD_FL0CNGCHMAP(cong) | 3521 F_FW_IQ_CMD_FL0CONGCIF | 3522 F_FW_IQ_CMD_FL0CONGEN); 3523 } 3524 c.fl0dcaen_to_fl0cidxfthresh = 3525 htobe16(V_FW_IQ_CMD_FL0FBMIN(chip_id(sc) <= CHELSIO_T5 ? 3526 X_FETCHBURSTMIN_128B : X_FETCHBURSTMIN_64B_T6) | 3527 V_FW_IQ_CMD_FL0FBMAX(chip_id(sc) <= CHELSIO_T5 ? 3528 X_FETCHBURSTMAX_512B : X_FETCHBURSTMAX_256B)); 3529 c.fl0size = htobe16(fl->qsize); 3530 c.fl0addr = htobe64(fl->ba); 3531 } 3532 3533 rc = -t4_wr_mbox(sc, sc->mbox, &c, sizeof(c), &c); 3534 if (rc != 0) { 3535 device_printf(sc->dev, 3536 "failed to create ingress queue: %d\n", rc); 3537 return (rc); 3538 } 3539 3540 iq->cidx = 0; 3541 iq->gen = F_RSPD_GEN; 3542 iq->intr_next = iq->intr_params; 3543 iq->cntxt_id = be16toh(c.iqid); 3544 iq->abs_id = be16toh(c.physiqid); 3545 iq->flags |= IQ_ALLOCATED; 3546 3547 cntxt_id = iq->cntxt_id - sc->sge.iq_start; 3548 if (cntxt_id >= sc->sge.niq) { 3549 panic ("%s: iq->cntxt_id (%d) more than the max (%d)", __func__, 3550 cntxt_id, sc->sge.niq - 1); 3551 } 3552 sc->sge.iqmap[cntxt_id] = iq; 3553 3554 if (fl) { 3555 u_int qid; 3556 3557 iq->flags |= IQ_HAS_FL; 3558 fl->cntxt_id = be16toh(c.fl0id); 3559 fl->pidx = fl->cidx = 0; 3560 3561 cntxt_id = fl->cntxt_id - sc->sge.eq_start; 3562 if (cntxt_id >= sc->sge.neq) { 3563 panic("%s: fl->cntxt_id (%d) more than the max (%d)", 3564 __func__, cntxt_id, sc->sge.neq - 1); 3565 } 3566 sc->sge.eqmap[cntxt_id] = (void *)fl; 3567 3568 qid = fl->cntxt_id; 3569 if (isset(&sc->doorbells, DOORBELL_UDB)) { 3570 uint32_t s_qpp = sc->params.sge.eq_s_qpp; 3571 uint32_t mask = (1 << s_qpp) - 1; 3572 volatile uint8_t *udb; 3573 3574 udb = sc->udbs_base + UDBS_DB_OFFSET; 3575 udb += (qid >> s_qpp) << PAGE_SHIFT; 3576 qid &= mask; 3577 if (qid < PAGE_SIZE / UDBS_SEG_SIZE) { 3578 udb += qid << UDBS_SEG_SHIFT; 3579 qid = 0; 3580 } 3581 fl->udb = (volatile void *)udb; 3582 } 3583 fl->dbval = V_QID(qid) | sc->chip_params->sge_fl_db; 3584 3585 FL_LOCK(fl); 3586 /* Enough to make sure the SGE doesn't think it's starved */ 3587 refill_fl(sc, fl, fl->lowat); 3588 FL_UNLOCK(fl); 3589 } 3590 3591 if (chip_id(sc) >= CHELSIO_T5 && !(sc->flags & IS_VF) && cong >= 0) { 3592 uint32_t param, val; 3593 3594 param = V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DMAQ) | 3595 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DMAQ_CONM_CTXT) | 3596 V_FW_PARAMS_PARAM_YZ(iq->cntxt_id); 3597 if (cong == 0) 3598 val = 1 << 19; 3599 else { 3600 val = 2 << 19; 3601 for (i = 0; i < 4; i++) { 3602 if (cong & (1 << i)) 3603 val |= 1 << (i << 2); 3604 } 3605 } 3606 3607 rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val); 3608 if (rc != 0) { 3609 /* report error but carry on */ 3610 device_printf(sc->dev, 3611 "failed to set congestion manager context for " 3612 "ingress queue %d: %d\n", iq->cntxt_id, rc); 3613 } 3614 } 3615 3616 /* Enable IQ interrupts */ 3617 atomic_store_rel_int(&iq->state, IQS_IDLE); 3618 t4_write_reg(sc, sc->sge_gts_reg, V_SEINTARM(iq->intr_params) | 3619 V_INGRESSQID(iq->cntxt_id)); 3620 3621 return (0); 3622 } 3623 3624 static int 3625 free_iq_fl(struct vi_info *vi, struct sge_iq *iq, struct sge_fl *fl) 3626 { 3627 int rc; 3628 struct adapter *sc = iq->adapter; 3629 device_t dev; 3630 3631 if (sc == NULL) 3632 return (0); /* nothing to do */ 3633 3634 dev = vi ? vi->dev : sc->dev; 3635 3636 if (iq->flags & IQ_ALLOCATED) { 3637 rc = -t4_iq_free(sc, sc->mbox, sc->pf, 0, 3638 FW_IQ_TYPE_FL_INT_CAP, iq->cntxt_id, 3639 fl ? fl->cntxt_id : 0xffff, 0xffff); 3640 if (rc != 0) { 3641 device_printf(dev, 3642 "failed to free queue %p: %d\n", iq, rc); 3643 return (rc); 3644 } 3645 iq->flags &= ~IQ_ALLOCATED; 3646 } 3647 3648 free_ring(sc, iq->desc_tag, iq->desc_map, iq->ba, iq->desc); 3649 3650 bzero(iq, sizeof(*iq)); 3651 3652 if (fl) { 3653 free_ring(sc, fl->desc_tag, fl->desc_map, fl->ba, 3654 fl->desc); 3655 3656 if (fl->sdesc) 3657 free_fl_sdesc(sc, fl); 3658 3659 if (mtx_initialized(&fl->fl_lock)) 3660 mtx_destroy(&fl->fl_lock); 3661 3662 bzero(fl, sizeof(*fl)); 3663 } 3664 3665 return (0); 3666 } 3667 3668 static void 3669 add_iq_sysctls(struct sysctl_ctx_list *ctx, struct sysctl_oid *oid, 3670 struct sge_iq *iq) 3671 { 3672 struct sysctl_oid_list *children = SYSCTL_CHILDREN(oid); 3673 3674 SYSCTL_ADD_UAUTO(ctx, children, OID_AUTO, "ba", CTLFLAG_RD, &iq->ba, 3675 "bus address of descriptor ring"); 3676 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "dmalen", CTLFLAG_RD, NULL, 3677 iq->qsize * IQ_ESIZE, "descriptor ring size in bytes"); 3678 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "abs_id", 3679 CTLTYPE_INT | CTLFLAG_RD | CTLFLAG_NEEDGIANT, &iq->abs_id, 0, 3680 sysctl_uint16, "I", "absolute id of the queue"); 3681 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cntxt_id", 3682 CTLTYPE_INT | CTLFLAG_RD | CTLFLAG_NEEDGIANT, &iq->cntxt_id, 0, 3683 sysctl_uint16, "I", "SGE context id of the queue"); 3684 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cidx", 3685 CTLTYPE_INT | CTLFLAG_RD | CTLFLAG_NEEDGIANT, &iq->cidx, 0, 3686 sysctl_uint16, "I", "consumer index"); 3687 } 3688 3689 static void 3690 add_fl_sysctls(struct adapter *sc, struct sysctl_ctx_list *ctx, 3691 struct sysctl_oid *oid, struct sge_fl *fl) 3692 { 3693 struct sysctl_oid_list *children = SYSCTL_CHILDREN(oid); 3694 3695 oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "fl", 3696 CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "freelist"); 3697 children = SYSCTL_CHILDREN(oid); 3698 3699 SYSCTL_ADD_UAUTO(ctx, children, OID_AUTO, "ba", CTLFLAG_RD, 3700 &fl->ba, "bus address of descriptor ring"); 3701 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "dmalen", CTLFLAG_RD, NULL, 3702 fl->sidx * EQ_ESIZE + sc->params.sge.spg_len, 3703 "desc ring size in bytes"); 3704 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cntxt_id", 3705 CTLTYPE_INT | CTLFLAG_RD | CTLFLAG_NEEDGIANT, &fl->cntxt_id, 0, 3706 sysctl_uint16, "I", "SGE context id of the freelist"); 3707 SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "padding", CTLFLAG_RD, NULL, 3708 fl_pad ? 1 : 0, "padding enabled"); 3709 SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "packing", CTLFLAG_RD, NULL, 3710 fl->flags & FL_BUF_PACKING ? 1 : 0, "packing enabled"); 3711 SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "cidx", CTLFLAG_RD, &fl->cidx, 3712 0, "consumer index"); 3713 if (fl->flags & FL_BUF_PACKING) { 3714 SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "rx_offset", 3715 CTLFLAG_RD, &fl->rx_offset, 0, "packing rx offset"); 3716 } 3717 SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "pidx", CTLFLAG_RD, &fl->pidx, 3718 0, "producer index"); 3719 SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "cluster_allocated", 3720 CTLFLAG_RD, &fl->cl_allocated, "# of clusters allocated"); 3721 SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "cluster_recycled", 3722 CTLFLAG_RD, &fl->cl_recycled, "# of clusters recycled"); 3723 SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "cluster_fast_recycled", 3724 CTLFLAG_RD, &fl->cl_fast_recycled, "# of clusters recycled (fast)"); 3725 } 3726 3727 static int 3728 alloc_fwq(struct adapter *sc) 3729 { 3730 int rc, intr_idx; 3731 struct sge_iq *fwq = &sc->sge.fwq; 3732 struct sysctl_oid *oid = device_get_sysctl_tree(sc->dev); 3733 struct sysctl_oid_list *children = SYSCTL_CHILDREN(oid); 3734 3735 init_iq(fwq, sc, 0, 0, FW_IQ_QSIZE); 3736 if (sc->flags & IS_VF) 3737 intr_idx = 0; 3738 else 3739 intr_idx = sc->intr_count > 1 ? 1 : 0; 3740 rc = alloc_iq_fl(&sc->port[0]->vi[0], fwq, NULL, intr_idx, -1); 3741 if (rc != 0) { 3742 device_printf(sc->dev, 3743 "failed to create firmware event queue: %d\n", rc); 3744 return (rc); 3745 } 3746 3747 oid = SYSCTL_ADD_NODE(&sc->ctx, children, OID_AUTO, "fwq", 3748 CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "firmware event queue"); 3749 add_iq_sysctls(&sc->ctx, oid, fwq); 3750 3751 return (0); 3752 } 3753 3754 static int 3755 free_fwq(struct adapter *sc) 3756 { 3757 return free_iq_fl(NULL, &sc->sge.fwq, NULL); 3758 } 3759 3760 static int 3761 alloc_ctrlq(struct adapter *sc, struct sge_wrq *ctrlq, int idx, 3762 struct sysctl_oid *oid) 3763 { 3764 int rc; 3765 char name[16]; 3766 struct sysctl_oid_list *children; 3767 3768 snprintf(name, sizeof(name), "%s ctrlq%d", device_get_nameunit(sc->dev), 3769 idx); 3770 init_eq(sc, &ctrlq->eq, EQ_CTRL, CTRL_EQ_QSIZE, sc->port[idx]->tx_chan, 3771 sc->sge.fwq.cntxt_id, name); 3772 3773 children = SYSCTL_CHILDREN(oid); 3774 snprintf(name, sizeof(name), "%d", idx); 3775 oid = SYSCTL_ADD_NODE(&sc->ctx, children, OID_AUTO, name, 3776 CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "ctrl queue"); 3777 rc = alloc_wrq(sc, NULL, ctrlq, oid); 3778 3779 return (rc); 3780 } 3781 3782 int 3783 tnl_cong(struct port_info *pi, int drop) 3784 { 3785 3786 if (drop == -1) 3787 return (-1); 3788 else if (drop == 1) 3789 return (0); 3790 else 3791 return (pi->rx_e_chan_map); 3792 } 3793 3794 static int 3795 alloc_rxq(struct vi_info *vi, struct sge_rxq *rxq, int intr_idx, int idx, 3796 struct sysctl_oid *oid) 3797 { 3798 int rc; 3799 struct adapter *sc = vi->adapter; 3800 struct sysctl_oid_list *children; 3801 char name[16]; 3802 3803 rc = alloc_iq_fl(vi, &rxq->iq, &rxq->fl, intr_idx, 3804 tnl_cong(vi->pi, cong_drop)); 3805 if (rc != 0) 3806 return (rc); 3807 3808 if (idx == 0) 3809 sc->sge.iq_base = rxq->iq.abs_id - rxq->iq.cntxt_id; 3810 else 3811 KASSERT(rxq->iq.cntxt_id + sc->sge.iq_base == rxq->iq.abs_id, 3812 ("iq_base mismatch")); 3813 KASSERT(sc->sge.iq_base == 0 || sc->flags & IS_VF, 3814 ("PF with non-zero iq_base")); 3815 3816 /* 3817 * The freelist is just barely above the starvation threshold right now, 3818 * fill it up a bit more. 3819 */ 3820 FL_LOCK(&rxq->fl); 3821 refill_fl(sc, &rxq->fl, 128); 3822 FL_UNLOCK(&rxq->fl); 3823 3824 #if defined(INET) || defined(INET6) 3825 rc = tcp_lro_init_args(&rxq->lro, vi->ifp, lro_entries, lro_mbufs); 3826 if (rc != 0) 3827 return (rc); 3828 MPASS(rxq->lro.ifp == vi->ifp); /* also indicates LRO init'ed */ 3829 3830 if (vi->ifp->if_capenable & IFCAP_LRO) 3831 rxq->iq.flags |= IQ_LRO_ENABLED; 3832 #endif 3833 if (vi->ifp->if_capenable & IFCAP_HWRXTSTMP) 3834 rxq->iq.flags |= IQ_RX_TIMESTAMP; 3835 rxq->ifp = vi->ifp; 3836 3837 children = SYSCTL_CHILDREN(oid); 3838 3839 snprintf(name, sizeof(name), "%d", idx); 3840 oid = SYSCTL_ADD_NODE(&vi->ctx, children, OID_AUTO, name, 3841 CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "rx queue"); 3842 children = SYSCTL_CHILDREN(oid); 3843 3844 add_iq_sysctls(&vi->ctx, oid, &rxq->iq); 3845 #if defined(INET) || defined(INET6) 3846 SYSCTL_ADD_U64(&vi->ctx, children, OID_AUTO, "lro_queued", CTLFLAG_RD, 3847 &rxq->lro.lro_queued, 0, NULL); 3848 SYSCTL_ADD_U64(&vi->ctx, children, OID_AUTO, "lro_flushed", CTLFLAG_RD, 3849 &rxq->lro.lro_flushed, 0, NULL); 3850 #endif 3851 SYSCTL_ADD_UQUAD(&vi->ctx, children, OID_AUTO, "rxcsum", CTLFLAG_RD, 3852 &rxq->rxcsum, "# of times hardware assisted with checksum"); 3853 SYSCTL_ADD_UQUAD(&vi->ctx, children, OID_AUTO, "vlan_extraction", 3854 CTLFLAG_RD, &rxq->vlan_extraction, 3855 "# of times hardware extracted 802.1Q tag"); 3856 SYSCTL_ADD_UQUAD(&vi->ctx, children, OID_AUTO, "vxlan_rxcsum", 3857 CTLFLAG_RD, &rxq->vxlan_rxcsum, 3858 "# of times hardware assisted with inner checksum (VXLAN) "); 3859 3860 add_fl_sysctls(sc, &vi->ctx, oid, &rxq->fl); 3861 3862 return (rc); 3863 } 3864 3865 static int 3866 free_rxq(struct vi_info *vi, struct sge_rxq *rxq) 3867 { 3868 int rc; 3869 3870 #if defined(INET) || defined(INET6) 3871 if (rxq->lro.ifp) { 3872 tcp_lro_free(&rxq->lro); 3873 rxq->lro.ifp = NULL; 3874 } 3875 #endif 3876 3877 rc = free_iq_fl(vi, &rxq->iq, &rxq->fl); 3878 if (rc == 0) 3879 bzero(rxq, sizeof(*rxq)); 3880 3881 return (rc); 3882 } 3883 3884 #ifdef TCP_OFFLOAD 3885 static int 3886 alloc_ofld_rxq(struct vi_info *vi, struct sge_ofld_rxq *ofld_rxq, 3887 int intr_idx, int idx, struct sysctl_oid *oid) 3888 { 3889 struct port_info *pi = vi->pi; 3890 int rc; 3891 struct sysctl_oid_list *children; 3892 char name[16]; 3893 3894 rc = alloc_iq_fl(vi, &ofld_rxq->iq, &ofld_rxq->fl, intr_idx, 0); 3895 if (rc != 0) 3896 return (rc); 3897 3898 children = SYSCTL_CHILDREN(oid); 3899 3900 snprintf(name, sizeof(name), "%d", idx); 3901 oid = SYSCTL_ADD_NODE(&vi->ctx, children, OID_AUTO, name, 3902 CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "rx queue"); 3903 add_iq_sysctls(&vi->ctx, oid, &ofld_rxq->iq); 3904 add_fl_sysctls(pi->adapter, &vi->ctx, oid, &ofld_rxq->fl); 3905 3906 return (rc); 3907 } 3908 3909 static int 3910 free_ofld_rxq(struct vi_info *vi, struct sge_ofld_rxq *ofld_rxq) 3911 { 3912 int rc; 3913 3914 rc = free_iq_fl(vi, &ofld_rxq->iq, &ofld_rxq->fl); 3915 if (rc == 0) 3916 bzero(ofld_rxq, sizeof(*ofld_rxq)); 3917 3918 return (rc); 3919 } 3920 #endif 3921 3922 #ifdef DEV_NETMAP 3923 static int 3924 alloc_nm_rxq(struct vi_info *vi, struct sge_nm_rxq *nm_rxq, int intr_idx, 3925 int idx, struct sysctl_oid *oid) 3926 { 3927 int rc; 3928 struct sysctl_oid_list *children; 3929 struct sysctl_ctx_list *ctx; 3930 char name[16]; 3931 size_t len; 3932 struct adapter *sc = vi->adapter; 3933 struct netmap_adapter *na = NA(vi->ifp); 3934 3935 MPASS(na != NULL); 3936 3937 len = vi->qsize_rxq * IQ_ESIZE; 3938 rc = alloc_ring(sc, len, &nm_rxq->iq_desc_tag, &nm_rxq->iq_desc_map, 3939 &nm_rxq->iq_ba, (void **)&nm_rxq->iq_desc); 3940 if (rc != 0) 3941 return (rc); 3942 3943 len = na->num_rx_desc * EQ_ESIZE + sc->params.sge.spg_len; 3944 rc = alloc_ring(sc, len, &nm_rxq->fl_desc_tag, &nm_rxq->fl_desc_map, 3945 &nm_rxq->fl_ba, (void **)&nm_rxq->fl_desc); 3946 if (rc != 0) 3947 return (rc); 3948 3949 nm_rxq->vi = vi; 3950 nm_rxq->nid = idx; 3951 nm_rxq->iq_cidx = 0; 3952 nm_rxq->iq_sidx = vi->qsize_rxq - sc->params.sge.spg_len / IQ_ESIZE; 3953 nm_rxq->iq_gen = F_RSPD_GEN; 3954 nm_rxq->fl_pidx = nm_rxq->fl_cidx = 0; 3955 nm_rxq->fl_sidx = na->num_rx_desc; 3956 nm_rxq->fl_sidx2 = nm_rxq->fl_sidx; /* copy for rxsync cacheline */ 3957 nm_rxq->intr_idx = intr_idx; 3958 nm_rxq->iq_cntxt_id = INVALID_NM_RXQ_CNTXT_ID; 3959 3960 ctx = &vi->ctx; 3961 children = SYSCTL_CHILDREN(oid); 3962 3963 snprintf(name, sizeof(name), "%d", idx); 3964 oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, name, 3965 CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "rx queue"); 3966 children = SYSCTL_CHILDREN(oid); 3967 3968 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "abs_id", 3969 CTLTYPE_INT | CTLFLAG_RD | CTLFLAG_NEEDGIANT, &nm_rxq->iq_abs_id, 3970 0, sysctl_uint16, "I", "absolute id of the queue"); 3971 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cntxt_id", 3972 CTLTYPE_INT | CTLFLAG_RD | CTLFLAG_NEEDGIANT, &nm_rxq->iq_cntxt_id, 3973 0, sysctl_uint16, "I", "SGE context id of the queue"); 3974 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cidx", 3975 CTLTYPE_INT | CTLFLAG_RD | CTLFLAG_NEEDGIANT, &nm_rxq->iq_cidx, 0, 3976 sysctl_uint16, "I", "consumer index"); 3977 3978 children = SYSCTL_CHILDREN(oid); 3979 oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "fl", 3980 CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "freelist"); 3981 children = SYSCTL_CHILDREN(oid); 3982 3983 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cntxt_id", 3984 CTLTYPE_INT | CTLFLAG_RD | CTLFLAG_NEEDGIANT, &nm_rxq->fl_cntxt_id, 3985 0, sysctl_uint16, "I", "SGE context id of the freelist"); 3986 SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "cidx", CTLFLAG_RD, 3987 &nm_rxq->fl_cidx, 0, "consumer index"); 3988 SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "pidx", CTLFLAG_RD, 3989 &nm_rxq->fl_pidx, 0, "producer index"); 3990 3991 return (rc); 3992 } 3993 3994 3995 static int 3996 free_nm_rxq(struct vi_info *vi, struct sge_nm_rxq *nm_rxq) 3997 { 3998 struct adapter *sc = vi->adapter; 3999 4000 if (vi->flags & VI_INIT_DONE) 4001 MPASS(nm_rxq->iq_cntxt_id == INVALID_NM_RXQ_CNTXT_ID); 4002 else 4003 MPASS(nm_rxq->iq_cntxt_id == 0); 4004 4005 free_ring(sc, nm_rxq->iq_desc_tag, nm_rxq->iq_desc_map, nm_rxq->iq_ba, 4006 nm_rxq->iq_desc); 4007 free_ring(sc, nm_rxq->fl_desc_tag, nm_rxq->fl_desc_map, nm_rxq->fl_ba, 4008 nm_rxq->fl_desc); 4009 4010 return (0); 4011 } 4012 4013 static int 4014 alloc_nm_txq(struct vi_info *vi, struct sge_nm_txq *nm_txq, int iqidx, int idx, 4015 struct sysctl_oid *oid) 4016 { 4017 int rc; 4018 size_t len; 4019 struct port_info *pi = vi->pi; 4020 struct adapter *sc = pi->adapter; 4021 struct netmap_adapter *na = NA(vi->ifp); 4022 char name[16]; 4023 struct sysctl_oid_list *children = SYSCTL_CHILDREN(oid); 4024 4025 len = na->num_tx_desc * EQ_ESIZE + sc->params.sge.spg_len; 4026 rc = alloc_ring(sc, len, &nm_txq->desc_tag, &nm_txq->desc_map, 4027 &nm_txq->ba, (void **)&nm_txq->desc); 4028 if (rc) 4029 return (rc); 4030 4031 nm_txq->pidx = nm_txq->cidx = 0; 4032 nm_txq->sidx = na->num_tx_desc; 4033 nm_txq->nid = idx; 4034 nm_txq->iqidx = iqidx; 4035 nm_txq->cpl_ctrl0 = htobe32(V_TXPKT_OPCODE(CPL_TX_PKT) | 4036 V_TXPKT_INTF(pi->tx_chan) | V_TXPKT_PF(sc->pf) | 4037 V_TXPKT_VF(vi->vin) | V_TXPKT_VF_VLD(vi->vfvld)); 4038 if (sc->params.fw_vers >= FW_VERSION32(1, 24, 11, 0)) 4039 nm_txq->op_pkd = htobe32(V_FW_WR_OP(FW_ETH_TX_PKTS2_WR)); 4040 else 4041 nm_txq->op_pkd = htobe32(V_FW_WR_OP(FW_ETH_TX_PKTS_WR)); 4042 nm_txq->cntxt_id = INVALID_NM_TXQ_CNTXT_ID; 4043 4044 snprintf(name, sizeof(name), "%d", idx); 4045 oid = SYSCTL_ADD_NODE(&vi->ctx, children, OID_AUTO, name, 4046 CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "netmap tx queue"); 4047 children = SYSCTL_CHILDREN(oid); 4048 4049 SYSCTL_ADD_UINT(&vi->ctx, children, OID_AUTO, "cntxt_id", CTLFLAG_RD, 4050 &nm_txq->cntxt_id, 0, "SGE context id of the queue"); 4051 SYSCTL_ADD_PROC(&vi->ctx, children, OID_AUTO, "cidx", 4052 CTLTYPE_INT | CTLFLAG_RD | CTLFLAG_NEEDGIANT, &nm_txq->cidx, 0, 4053 sysctl_uint16, "I", "consumer index"); 4054 SYSCTL_ADD_PROC(&vi->ctx, children, OID_AUTO, "pidx", 4055 CTLTYPE_INT | CTLFLAG_RD | CTLFLAG_NEEDGIANT, &nm_txq->pidx, 0, 4056 sysctl_uint16, "I", "producer index"); 4057 4058 return (rc); 4059 } 4060 4061 static int 4062 free_nm_txq(struct vi_info *vi, struct sge_nm_txq *nm_txq) 4063 { 4064 struct adapter *sc = vi->adapter; 4065 4066 if (vi->flags & VI_INIT_DONE) 4067 MPASS(nm_txq->cntxt_id == INVALID_NM_TXQ_CNTXT_ID); 4068 else 4069 MPASS(nm_txq->cntxt_id == 0); 4070 4071 free_ring(sc, nm_txq->desc_tag, nm_txq->desc_map, nm_txq->ba, 4072 nm_txq->desc); 4073 4074 return (0); 4075 } 4076 #endif 4077 4078 /* 4079 * Returns a reasonable automatic cidx flush threshold for a given queue size. 4080 */ 4081 static u_int 4082 qsize_to_fthresh(int qsize) 4083 { 4084 u_int fthresh; 4085 4086 while (!powerof2(qsize)) 4087 qsize++; 4088 fthresh = ilog2(qsize); 4089 if (fthresh > X_CIDXFLUSHTHRESH_128) 4090 fthresh = X_CIDXFLUSHTHRESH_128; 4091 4092 return (fthresh); 4093 } 4094 4095 static int 4096 ctrl_eq_alloc(struct adapter *sc, struct sge_eq *eq) 4097 { 4098 int rc, cntxt_id; 4099 struct fw_eq_ctrl_cmd c; 4100 int qsize = eq->sidx + sc->params.sge.spg_len / EQ_ESIZE; 4101 4102 bzero(&c, sizeof(c)); 4103 4104 c.op_to_vfn = htobe32(V_FW_CMD_OP(FW_EQ_CTRL_CMD) | F_FW_CMD_REQUEST | 4105 F_FW_CMD_WRITE | F_FW_CMD_EXEC | V_FW_EQ_CTRL_CMD_PFN(sc->pf) | 4106 V_FW_EQ_CTRL_CMD_VFN(0)); 4107 c.alloc_to_len16 = htobe32(F_FW_EQ_CTRL_CMD_ALLOC | 4108 F_FW_EQ_CTRL_CMD_EQSTART | FW_LEN16(c)); 4109 c.cmpliqid_eqid = htonl(V_FW_EQ_CTRL_CMD_CMPLIQID(eq->iqid)); 4110 c.physeqid_pkd = htobe32(0); 4111 c.fetchszm_to_iqid = 4112 htobe32(V_FW_EQ_CTRL_CMD_HOSTFCMODE(X_HOSTFCMODE_STATUS_PAGE) | 4113 V_FW_EQ_CTRL_CMD_PCIECHN(eq->tx_chan) | 4114 F_FW_EQ_CTRL_CMD_FETCHRO | V_FW_EQ_CTRL_CMD_IQID(eq->iqid)); 4115 c.dcaen_to_eqsize = 4116 htobe32(V_FW_EQ_CTRL_CMD_FBMIN(chip_id(sc) <= CHELSIO_T5 ? 4117 X_FETCHBURSTMIN_64B : X_FETCHBURSTMIN_64B_T6) | 4118 V_FW_EQ_CTRL_CMD_FBMAX(X_FETCHBURSTMAX_512B) | 4119 V_FW_EQ_CTRL_CMD_CIDXFTHRESH(qsize_to_fthresh(qsize)) | 4120 V_FW_EQ_CTRL_CMD_EQSIZE(qsize)); 4121 c.eqaddr = htobe64(eq->ba); 4122 4123 rc = -t4_wr_mbox(sc, sc->mbox, &c, sizeof(c), &c); 4124 if (rc != 0) { 4125 device_printf(sc->dev, 4126 "failed to create control queue %d: %d\n", eq->tx_chan, rc); 4127 return (rc); 4128 } 4129 eq->flags |= EQ_ALLOCATED; 4130 4131 eq->cntxt_id = G_FW_EQ_CTRL_CMD_EQID(be32toh(c.cmpliqid_eqid)); 4132 cntxt_id = eq->cntxt_id - sc->sge.eq_start; 4133 if (cntxt_id >= sc->sge.neq) 4134 panic("%s: eq->cntxt_id (%d) more than the max (%d)", __func__, 4135 cntxt_id, sc->sge.neq - 1); 4136 sc->sge.eqmap[cntxt_id] = eq; 4137 4138 return (rc); 4139 } 4140 4141 static int 4142 eth_eq_alloc(struct adapter *sc, struct vi_info *vi, struct sge_eq *eq) 4143 { 4144 int rc, cntxt_id; 4145 struct fw_eq_eth_cmd c; 4146 int qsize = eq->sidx + sc->params.sge.spg_len / EQ_ESIZE; 4147 4148 bzero(&c, sizeof(c)); 4149 4150 c.op_to_vfn = htobe32(V_FW_CMD_OP(FW_EQ_ETH_CMD) | F_FW_CMD_REQUEST | 4151 F_FW_CMD_WRITE | F_FW_CMD_EXEC | V_FW_EQ_ETH_CMD_PFN(sc->pf) | 4152 V_FW_EQ_ETH_CMD_VFN(0)); 4153 c.alloc_to_len16 = htobe32(F_FW_EQ_ETH_CMD_ALLOC | 4154 F_FW_EQ_ETH_CMD_EQSTART | FW_LEN16(c)); 4155 c.autoequiqe_to_viid = htobe32(F_FW_EQ_ETH_CMD_AUTOEQUIQE | 4156 F_FW_EQ_ETH_CMD_AUTOEQUEQE | V_FW_EQ_ETH_CMD_VIID(vi->viid)); 4157 c.fetchszm_to_iqid = 4158 htobe32(V_FW_EQ_ETH_CMD_HOSTFCMODE(X_HOSTFCMODE_NONE) | 4159 V_FW_EQ_ETH_CMD_PCIECHN(eq->tx_chan) | F_FW_EQ_ETH_CMD_FETCHRO | 4160 V_FW_EQ_ETH_CMD_IQID(eq->iqid)); 4161 c.dcaen_to_eqsize = 4162 htobe32(V_FW_EQ_ETH_CMD_FBMIN(chip_id(sc) <= CHELSIO_T5 ? 4163 X_FETCHBURSTMIN_64B : X_FETCHBURSTMIN_64B_T6) | 4164 V_FW_EQ_ETH_CMD_FBMAX(X_FETCHBURSTMAX_512B) | 4165 V_FW_EQ_ETH_CMD_EQSIZE(qsize)); 4166 c.eqaddr = htobe64(eq->ba); 4167 4168 rc = -t4_wr_mbox(sc, sc->mbox, &c, sizeof(c), &c); 4169 if (rc != 0) { 4170 device_printf(vi->dev, 4171 "failed to create Ethernet egress queue: %d\n", rc); 4172 return (rc); 4173 } 4174 eq->flags |= EQ_ALLOCATED; 4175 4176 eq->cntxt_id = G_FW_EQ_ETH_CMD_EQID(be32toh(c.eqid_pkd)); 4177 eq->abs_id = G_FW_EQ_ETH_CMD_PHYSEQID(be32toh(c.physeqid_pkd)); 4178 cntxt_id = eq->cntxt_id - sc->sge.eq_start; 4179 if (cntxt_id >= sc->sge.neq) 4180 panic("%s: eq->cntxt_id (%d) more than the max (%d)", __func__, 4181 cntxt_id, sc->sge.neq - 1); 4182 sc->sge.eqmap[cntxt_id] = eq; 4183 4184 return (rc); 4185 } 4186 4187 #if defined(TCP_OFFLOAD) || defined(RATELIMIT) 4188 static int 4189 ofld_eq_alloc(struct adapter *sc, struct vi_info *vi, struct sge_eq *eq) 4190 { 4191 int rc, cntxt_id; 4192 struct fw_eq_ofld_cmd c; 4193 int qsize = eq->sidx + sc->params.sge.spg_len / EQ_ESIZE; 4194 4195 bzero(&c, sizeof(c)); 4196 4197 c.op_to_vfn = htonl(V_FW_CMD_OP(FW_EQ_OFLD_CMD) | F_FW_CMD_REQUEST | 4198 F_FW_CMD_WRITE | F_FW_CMD_EXEC | V_FW_EQ_OFLD_CMD_PFN(sc->pf) | 4199 V_FW_EQ_OFLD_CMD_VFN(0)); 4200 c.alloc_to_len16 = htonl(F_FW_EQ_OFLD_CMD_ALLOC | 4201 F_FW_EQ_OFLD_CMD_EQSTART | FW_LEN16(c)); 4202 c.fetchszm_to_iqid = 4203 htonl(V_FW_EQ_OFLD_CMD_HOSTFCMODE(X_HOSTFCMODE_STATUS_PAGE) | 4204 V_FW_EQ_OFLD_CMD_PCIECHN(eq->tx_chan) | 4205 F_FW_EQ_OFLD_CMD_FETCHRO | V_FW_EQ_OFLD_CMD_IQID(eq->iqid)); 4206 c.dcaen_to_eqsize = 4207 htobe32(V_FW_EQ_OFLD_CMD_FBMIN(chip_id(sc) <= CHELSIO_T5 ? 4208 X_FETCHBURSTMIN_64B : X_FETCHBURSTMIN_64B_T6) | 4209 V_FW_EQ_OFLD_CMD_FBMAX(X_FETCHBURSTMAX_512B) | 4210 V_FW_EQ_OFLD_CMD_CIDXFTHRESH(qsize_to_fthresh(qsize)) | 4211 V_FW_EQ_OFLD_CMD_EQSIZE(qsize)); 4212 c.eqaddr = htobe64(eq->ba); 4213 4214 rc = -t4_wr_mbox(sc, sc->mbox, &c, sizeof(c), &c); 4215 if (rc != 0) { 4216 device_printf(vi->dev, 4217 "failed to create egress queue for TCP offload: %d\n", rc); 4218 return (rc); 4219 } 4220 eq->flags |= EQ_ALLOCATED; 4221 4222 eq->cntxt_id = G_FW_EQ_OFLD_CMD_EQID(be32toh(c.eqid_pkd)); 4223 cntxt_id = eq->cntxt_id - sc->sge.eq_start; 4224 if (cntxt_id >= sc->sge.neq) 4225 panic("%s: eq->cntxt_id (%d) more than the max (%d)", __func__, 4226 cntxt_id, sc->sge.neq - 1); 4227 sc->sge.eqmap[cntxt_id] = eq; 4228 4229 return (rc); 4230 } 4231 #endif 4232 4233 static int 4234 alloc_eq(struct adapter *sc, struct vi_info *vi, struct sge_eq *eq) 4235 { 4236 int rc, qsize; 4237 size_t len; 4238 4239 mtx_init(&eq->eq_lock, eq->lockname, NULL, MTX_DEF); 4240 4241 qsize = eq->sidx + sc->params.sge.spg_len / EQ_ESIZE; 4242 len = qsize * EQ_ESIZE; 4243 rc = alloc_ring(sc, len, &eq->desc_tag, &eq->desc_map, 4244 &eq->ba, (void **)&eq->desc); 4245 if (rc) 4246 return (rc); 4247 4248 eq->pidx = eq->cidx = eq->dbidx = 0; 4249 /* Note that equeqidx is not used with sge_wrq (OFLD/CTRL) queues. */ 4250 eq->equeqidx = 0; 4251 eq->doorbells = sc->doorbells; 4252 4253 switch (eq->flags & EQ_TYPEMASK) { 4254 case EQ_CTRL: 4255 rc = ctrl_eq_alloc(sc, eq); 4256 break; 4257 4258 case EQ_ETH: 4259 rc = eth_eq_alloc(sc, vi, eq); 4260 break; 4261 4262 #if defined(TCP_OFFLOAD) || defined(RATELIMIT) 4263 case EQ_OFLD: 4264 rc = ofld_eq_alloc(sc, vi, eq); 4265 break; 4266 #endif 4267 4268 default: 4269 panic("%s: invalid eq type %d.", __func__, 4270 eq->flags & EQ_TYPEMASK); 4271 } 4272 if (rc != 0) { 4273 device_printf(sc->dev, 4274 "failed to allocate egress queue(%d): %d\n", 4275 eq->flags & EQ_TYPEMASK, rc); 4276 } 4277 4278 if (isset(&eq->doorbells, DOORBELL_UDB) || 4279 isset(&eq->doorbells, DOORBELL_UDBWC) || 4280 isset(&eq->doorbells, DOORBELL_WCWR)) { 4281 uint32_t s_qpp = sc->params.sge.eq_s_qpp; 4282 uint32_t mask = (1 << s_qpp) - 1; 4283 volatile uint8_t *udb; 4284 4285 udb = sc->udbs_base + UDBS_DB_OFFSET; 4286 udb += (eq->cntxt_id >> s_qpp) << PAGE_SHIFT; /* pg offset */ 4287 eq->udb_qid = eq->cntxt_id & mask; /* id in page */ 4288 if (eq->udb_qid >= PAGE_SIZE / UDBS_SEG_SIZE) 4289 clrbit(&eq->doorbells, DOORBELL_WCWR); 4290 else { 4291 udb += eq->udb_qid << UDBS_SEG_SHIFT; /* seg offset */ 4292 eq->udb_qid = 0; 4293 } 4294 eq->udb = (volatile void *)udb; 4295 } 4296 4297 return (rc); 4298 } 4299 4300 static int 4301 free_eq(struct adapter *sc, struct sge_eq *eq) 4302 { 4303 int rc; 4304 4305 if (eq->flags & EQ_ALLOCATED) { 4306 switch (eq->flags & EQ_TYPEMASK) { 4307 case EQ_CTRL: 4308 rc = -t4_ctrl_eq_free(sc, sc->mbox, sc->pf, 0, 4309 eq->cntxt_id); 4310 break; 4311 4312 case EQ_ETH: 4313 rc = -t4_eth_eq_free(sc, sc->mbox, sc->pf, 0, 4314 eq->cntxt_id); 4315 break; 4316 4317 #if defined(TCP_OFFLOAD) || defined(RATELIMIT) 4318 case EQ_OFLD: 4319 rc = -t4_ofld_eq_free(sc, sc->mbox, sc->pf, 0, 4320 eq->cntxt_id); 4321 break; 4322 #endif 4323 4324 default: 4325 panic("%s: invalid eq type %d.", __func__, 4326 eq->flags & EQ_TYPEMASK); 4327 } 4328 if (rc != 0) { 4329 device_printf(sc->dev, 4330 "failed to free egress queue (%d): %d\n", 4331 eq->flags & EQ_TYPEMASK, rc); 4332 return (rc); 4333 } 4334 eq->flags &= ~EQ_ALLOCATED; 4335 } 4336 4337 free_ring(sc, eq->desc_tag, eq->desc_map, eq->ba, eq->desc); 4338 4339 if (mtx_initialized(&eq->eq_lock)) 4340 mtx_destroy(&eq->eq_lock); 4341 4342 bzero(eq, sizeof(*eq)); 4343 return (0); 4344 } 4345 4346 static int 4347 alloc_wrq(struct adapter *sc, struct vi_info *vi, struct sge_wrq *wrq, 4348 struct sysctl_oid *oid) 4349 { 4350 int rc; 4351 struct sysctl_ctx_list *ctx = vi ? &vi->ctx : &sc->ctx; 4352 struct sysctl_oid_list *children = SYSCTL_CHILDREN(oid); 4353 4354 rc = alloc_eq(sc, vi, &wrq->eq); 4355 if (rc) 4356 return (rc); 4357 4358 wrq->adapter = sc; 4359 TASK_INIT(&wrq->wrq_tx_task, 0, wrq_tx_drain, wrq); 4360 TAILQ_INIT(&wrq->incomplete_wrs); 4361 STAILQ_INIT(&wrq->wr_list); 4362 wrq->nwr_pending = 0; 4363 wrq->ndesc_needed = 0; 4364 4365 SYSCTL_ADD_UAUTO(ctx, children, OID_AUTO, "ba", CTLFLAG_RD, 4366 &wrq->eq.ba, "bus address of descriptor ring"); 4367 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "dmalen", CTLFLAG_RD, NULL, 4368 wrq->eq.sidx * EQ_ESIZE + sc->params.sge.spg_len, 4369 "desc ring size in bytes"); 4370 SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "cntxt_id", CTLFLAG_RD, 4371 &wrq->eq.cntxt_id, 0, "SGE context id of the queue"); 4372 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cidx", 4373 CTLTYPE_INT | CTLFLAG_RD | CTLFLAG_NEEDGIANT, &wrq->eq.cidx, 0, 4374 sysctl_uint16, "I", "consumer index"); 4375 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "pidx", 4376 CTLTYPE_INT | CTLFLAG_RD | CTLFLAG_NEEDGIANT, &wrq->eq.pidx, 0, 4377 sysctl_uint16, "I", "producer index"); 4378 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "sidx", CTLFLAG_RD, NULL, 4379 wrq->eq.sidx, "status page index"); 4380 SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "tx_wrs_direct", CTLFLAG_RD, 4381 &wrq->tx_wrs_direct, "# of work requests (direct)"); 4382 SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "tx_wrs_copied", CTLFLAG_RD, 4383 &wrq->tx_wrs_copied, "# of work requests (copied)"); 4384 SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "tx_wrs_sspace", CTLFLAG_RD, 4385 &wrq->tx_wrs_ss, "# of work requests (copied from scratch space)"); 4386 4387 return (rc); 4388 } 4389 4390 static int 4391 free_wrq(struct adapter *sc, struct sge_wrq *wrq) 4392 { 4393 int rc; 4394 4395 rc = free_eq(sc, &wrq->eq); 4396 if (rc) 4397 return (rc); 4398 4399 bzero(wrq, sizeof(*wrq)); 4400 return (0); 4401 } 4402 4403 static int 4404 alloc_txq(struct vi_info *vi, struct sge_txq *txq, int idx, 4405 struct sysctl_oid *oid) 4406 { 4407 int rc; 4408 struct port_info *pi = vi->pi; 4409 struct adapter *sc = pi->adapter; 4410 struct sge_eq *eq = &txq->eq; 4411 struct txpkts *txp; 4412 char name[16]; 4413 struct sysctl_oid_list *children = SYSCTL_CHILDREN(oid); 4414 4415 rc = mp_ring_alloc(&txq->r, eq->sidx, txq, eth_tx, can_resume_eth_tx, 4416 M_CXGBE, &eq->eq_lock, M_WAITOK); 4417 if (rc != 0) { 4418 device_printf(sc->dev, "failed to allocate mp_ring: %d\n", rc); 4419 return (rc); 4420 } 4421 4422 rc = alloc_eq(sc, vi, eq); 4423 if (rc != 0) { 4424 mp_ring_free(txq->r); 4425 txq->r = NULL; 4426 return (rc); 4427 } 4428 4429 /* Can't fail after this point. */ 4430 4431 if (idx == 0) 4432 sc->sge.eq_base = eq->abs_id - eq->cntxt_id; 4433 else 4434 KASSERT(eq->cntxt_id + sc->sge.eq_base == eq->abs_id, 4435 ("eq_base mismatch")); 4436 KASSERT(sc->sge.eq_base == 0 || sc->flags & IS_VF, 4437 ("PF with non-zero eq_base")); 4438 4439 TASK_INIT(&txq->tx_reclaim_task, 0, tx_reclaim, eq); 4440 txq->ifp = vi->ifp; 4441 txq->gl = sglist_alloc(TX_SGL_SEGS, M_WAITOK); 4442 if (vi->flags & TX_USES_VM_WR) 4443 txq->cpl_ctrl0 = htobe32(V_TXPKT_OPCODE(CPL_TX_PKT_XT) | 4444 V_TXPKT_INTF(pi->tx_chan)); 4445 else 4446 txq->cpl_ctrl0 = htobe32(V_TXPKT_OPCODE(CPL_TX_PKT_XT) | 4447 V_TXPKT_INTF(pi->tx_chan) | V_TXPKT_PF(sc->pf) | 4448 V_TXPKT_VF(vi->vin) | V_TXPKT_VF_VLD(vi->vfvld)); 4449 txq->tc_idx = -1; 4450 txq->sdesc = malloc(eq->sidx * sizeof(struct tx_sdesc), M_CXGBE, 4451 M_ZERO | M_WAITOK); 4452 4453 txp = &txq->txp; 4454 txp->score = 5; 4455 MPASS(nitems(txp->mb) >= sc->params.max_pkts_per_eth_tx_pkts_wr); 4456 txq->txp.max_npkt = min(nitems(txp->mb), 4457 sc->params.max_pkts_per_eth_tx_pkts_wr); 4458 if (vi->flags & TX_USES_VM_WR && !(sc->flags & IS_VF)) 4459 txq->txp.max_npkt--; 4460 4461 snprintf(name, sizeof(name), "%d", idx); 4462 oid = SYSCTL_ADD_NODE(&vi->ctx, children, OID_AUTO, name, 4463 CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "tx queue"); 4464 children = SYSCTL_CHILDREN(oid); 4465 4466 SYSCTL_ADD_UAUTO(&vi->ctx, children, OID_AUTO, "ba", CTLFLAG_RD, 4467 &eq->ba, "bus address of descriptor ring"); 4468 SYSCTL_ADD_INT(&vi->ctx, children, OID_AUTO, "dmalen", CTLFLAG_RD, NULL, 4469 eq->sidx * EQ_ESIZE + sc->params.sge.spg_len, 4470 "desc ring size in bytes"); 4471 SYSCTL_ADD_UINT(&vi->ctx, children, OID_AUTO, "abs_id", CTLFLAG_RD, 4472 &eq->abs_id, 0, "absolute id of the queue"); 4473 SYSCTL_ADD_UINT(&vi->ctx, children, OID_AUTO, "cntxt_id", CTLFLAG_RD, 4474 &eq->cntxt_id, 0, "SGE context id of the queue"); 4475 SYSCTL_ADD_PROC(&vi->ctx, children, OID_AUTO, "cidx", 4476 CTLTYPE_INT | CTLFLAG_RD | CTLFLAG_NEEDGIANT, &eq->cidx, 0, 4477 sysctl_uint16, "I", "consumer index"); 4478 SYSCTL_ADD_PROC(&vi->ctx, children, OID_AUTO, "pidx", 4479 CTLTYPE_INT | CTLFLAG_RD | CTLFLAG_NEEDGIANT, &eq->pidx, 0, 4480 sysctl_uint16, "I", "producer index"); 4481 SYSCTL_ADD_INT(&vi->ctx, children, OID_AUTO, "sidx", CTLFLAG_RD, NULL, 4482 eq->sidx, "status page index"); 4483 4484 SYSCTL_ADD_PROC(&vi->ctx, children, OID_AUTO, "tc", 4485 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_NEEDGIANT, vi, idx, sysctl_tc, 4486 "I", "traffic class (-1 means none)"); 4487 4488 SYSCTL_ADD_UQUAD(&vi->ctx, children, OID_AUTO, "txcsum", CTLFLAG_RD, 4489 &txq->txcsum, "# of times hardware assisted with checksum"); 4490 SYSCTL_ADD_UQUAD(&vi->ctx, children, OID_AUTO, "vlan_insertion", 4491 CTLFLAG_RD, &txq->vlan_insertion, 4492 "# of times hardware inserted 802.1Q tag"); 4493 SYSCTL_ADD_UQUAD(&vi->ctx, children, OID_AUTO, "tso_wrs", CTLFLAG_RD, 4494 &txq->tso_wrs, "# of TSO work requests"); 4495 SYSCTL_ADD_UQUAD(&vi->ctx, children, OID_AUTO, "imm_wrs", CTLFLAG_RD, 4496 &txq->imm_wrs, "# of work requests with immediate data"); 4497 SYSCTL_ADD_UQUAD(&vi->ctx, children, OID_AUTO, "sgl_wrs", CTLFLAG_RD, 4498 &txq->sgl_wrs, "# of work requests with direct SGL"); 4499 SYSCTL_ADD_UQUAD(&vi->ctx, children, OID_AUTO, "txpkt_wrs", CTLFLAG_RD, 4500 &txq->txpkt_wrs, "# of txpkt work requests (one pkt/WR)"); 4501 SYSCTL_ADD_UQUAD(&vi->ctx, children, OID_AUTO, "txpkts0_wrs", 4502 CTLFLAG_RD, &txq->txpkts0_wrs, 4503 "# of txpkts (type 0) work requests"); 4504 SYSCTL_ADD_UQUAD(&vi->ctx, children, OID_AUTO, "txpkts1_wrs", 4505 CTLFLAG_RD, &txq->txpkts1_wrs, 4506 "# of txpkts (type 1) work requests"); 4507 SYSCTL_ADD_UQUAD(&vi->ctx, children, OID_AUTO, "txpkts0_pkts", 4508 CTLFLAG_RD, &txq->txpkts0_pkts, 4509 "# of frames tx'd using type0 txpkts work requests"); 4510 SYSCTL_ADD_UQUAD(&vi->ctx, children, OID_AUTO, "txpkts1_pkts", 4511 CTLFLAG_RD, &txq->txpkts1_pkts, 4512 "# of frames tx'd using type1 txpkts work requests"); 4513 SYSCTL_ADD_UQUAD(&vi->ctx, children, OID_AUTO, "raw_wrs", CTLFLAG_RD, 4514 &txq->raw_wrs, "# of raw work requests (non-packets)"); 4515 SYSCTL_ADD_UQUAD(&vi->ctx, children, OID_AUTO, "vxlan_tso_wrs", 4516 CTLFLAG_RD, &txq->vxlan_tso_wrs, "# of VXLAN TSO work requests"); 4517 SYSCTL_ADD_UQUAD(&vi->ctx, children, OID_AUTO, "vxlan_txcsum", 4518 CTLFLAG_RD, &txq->vxlan_txcsum, 4519 "# of times hardware assisted with inner checksums (VXLAN)"); 4520 4521 #ifdef KERN_TLS 4522 if (sc->flags & KERN_TLS_OK) { 4523 SYSCTL_ADD_UQUAD(&vi->ctx, children, OID_AUTO, 4524 "kern_tls_records", CTLFLAG_RD, &txq->kern_tls_records, 4525 "# of NIC TLS records transmitted"); 4526 SYSCTL_ADD_UQUAD(&vi->ctx, children, OID_AUTO, 4527 "kern_tls_short", CTLFLAG_RD, &txq->kern_tls_short, 4528 "# of short NIC TLS records transmitted"); 4529 SYSCTL_ADD_UQUAD(&vi->ctx, children, OID_AUTO, 4530 "kern_tls_partial", CTLFLAG_RD, &txq->kern_tls_partial, 4531 "# of partial NIC TLS records transmitted"); 4532 SYSCTL_ADD_UQUAD(&vi->ctx, children, OID_AUTO, 4533 "kern_tls_full", CTLFLAG_RD, &txq->kern_tls_full, 4534 "# of full NIC TLS records transmitted"); 4535 SYSCTL_ADD_UQUAD(&vi->ctx, children, OID_AUTO, 4536 "kern_tls_octets", CTLFLAG_RD, &txq->kern_tls_octets, 4537 "# of payload octets in transmitted NIC TLS records"); 4538 SYSCTL_ADD_UQUAD(&vi->ctx, children, OID_AUTO, 4539 "kern_tls_waste", CTLFLAG_RD, &txq->kern_tls_waste, 4540 "# of octets DMAd but not transmitted in NIC TLS records"); 4541 SYSCTL_ADD_UQUAD(&vi->ctx, children, OID_AUTO, 4542 "kern_tls_options", CTLFLAG_RD, &txq->kern_tls_options, 4543 "# of NIC TLS options-only packets transmitted"); 4544 SYSCTL_ADD_UQUAD(&vi->ctx, children, OID_AUTO, 4545 "kern_tls_header", CTLFLAG_RD, &txq->kern_tls_header, 4546 "# of NIC TLS header-only packets transmitted"); 4547 SYSCTL_ADD_UQUAD(&vi->ctx, children, OID_AUTO, 4548 "kern_tls_fin", CTLFLAG_RD, &txq->kern_tls_fin, 4549 "# of NIC TLS FIN-only packets transmitted"); 4550 SYSCTL_ADD_UQUAD(&vi->ctx, children, OID_AUTO, 4551 "kern_tls_fin_short", CTLFLAG_RD, &txq->kern_tls_fin_short, 4552 "# of NIC TLS padded FIN packets on short TLS records"); 4553 SYSCTL_ADD_UQUAD(&vi->ctx, children, OID_AUTO, 4554 "kern_tls_cbc", CTLFLAG_RD, &txq->kern_tls_cbc, 4555 "# of NIC TLS sessions using AES-CBC"); 4556 SYSCTL_ADD_UQUAD(&vi->ctx, children, OID_AUTO, 4557 "kern_tls_gcm", CTLFLAG_RD, &txq->kern_tls_gcm, 4558 "# of NIC TLS sessions using AES-GCM"); 4559 } 4560 #endif 4561 mp_ring_sysctls(txq->r, &vi->ctx, children); 4562 4563 return (0); 4564 } 4565 4566 static int 4567 free_txq(struct vi_info *vi, struct sge_txq *txq) 4568 { 4569 int rc; 4570 struct adapter *sc = vi->adapter; 4571 struct sge_eq *eq = &txq->eq; 4572 4573 rc = free_eq(sc, eq); 4574 if (rc) 4575 return (rc); 4576 4577 sglist_free(txq->gl); 4578 free(txq->sdesc, M_CXGBE); 4579 mp_ring_free(txq->r); 4580 4581 bzero(txq, sizeof(*txq)); 4582 return (0); 4583 } 4584 4585 static void 4586 oneseg_dma_callback(void *arg, bus_dma_segment_t *segs, int nseg, int error) 4587 { 4588 bus_addr_t *ba = arg; 4589 4590 KASSERT(nseg == 1, 4591 ("%s meant for single segment mappings only.", __func__)); 4592 4593 *ba = error ? 0 : segs->ds_addr; 4594 } 4595 4596 static inline void 4597 ring_fl_db(struct adapter *sc, struct sge_fl *fl) 4598 { 4599 uint32_t n, v; 4600 4601 n = IDXDIFF(fl->pidx >> 3, fl->dbidx, fl->sidx); 4602 MPASS(n > 0); 4603 4604 wmb(); 4605 v = fl->dbval | V_PIDX(n); 4606 if (fl->udb) 4607 *fl->udb = htole32(v); 4608 else 4609 t4_write_reg(sc, sc->sge_kdoorbell_reg, v); 4610 IDXINCR(fl->dbidx, n, fl->sidx); 4611 } 4612 4613 /* 4614 * Fills up the freelist by allocating up to 'n' buffers. Buffers that are 4615 * recycled do not count towards this allocation budget. 4616 * 4617 * Returns non-zero to indicate that this freelist should be added to the list 4618 * of starving freelists. 4619 */ 4620 static int 4621 refill_fl(struct adapter *sc, struct sge_fl *fl, int n) 4622 { 4623 __be64 *d; 4624 struct fl_sdesc *sd; 4625 uintptr_t pa; 4626 caddr_t cl; 4627 struct rx_buf_info *rxb; 4628 struct cluster_metadata *clm; 4629 uint16_t max_pidx; 4630 uint16_t hw_cidx = fl->hw_cidx; /* stable snapshot */ 4631 4632 FL_LOCK_ASSERT_OWNED(fl); 4633 4634 /* 4635 * We always stop at the beginning of the hardware descriptor that's just 4636 * before the one with the hw cidx. This is to avoid hw pidx = hw cidx, 4637 * which would mean an empty freelist to the chip. 4638 */ 4639 max_pidx = __predict_false(hw_cidx == 0) ? fl->sidx - 1 : hw_cidx - 1; 4640 if (fl->pidx == max_pidx * 8) 4641 return (0); 4642 4643 d = &fl->desc[fl->pidx]; 4644 sd = &fl->sdesc[fl->pidx]; 4645 4646 while (n > 0) { 4647 4648 if (sd->cl != NULL) { 4649 4650 if (sd->nmbuf == 0) { 4651 /* 4652 * Fast recycle without involving any atomics on 4653 * the cluster's metadata (if the cluster has 4654 * metadata). This happens when all frames 4655 * received in the cluster were small enough to 4656 * fit within a single mbuf each. 4657 */ 4658 fl->cl_fast_recycled++; 4659 goto recycled; 4660 } 4661 4662 /* 4663 * Cluster is guaranteed to have metadata. Clusters 4664 * without metadata always take the fast recycle path 4665 * when they're recycled. 4666 */ 4667 clm = cl_metadata(sd); 4668 MPASS(clm != NULL); 4669 4670 if (atomic_fetchadd_int(&clm->refcount, -1) == 1) { 4671 fl->cl_recycled++; 4672 counter_u64_add(extfree_rels, 1); 4673 goto recycled; 4674 } 4675 sd->cl = NULL; /* gave up my reference */ 4676 } 4677 MPASS(sd->cl == NULL); 4678 rxb = &sc->sge.rx_buf_info[fl->zidx]; 4679 cl = uma_zalloc(rxb->zone, M_NOWAIT); 4680 if (__predict_false(cl == NULL)) { 4681 if (fl->zidx != fl->safe_zidx) { 4682 rxb = &sc->sge.rx_buf_info[fl->safe_zidx]; 4683 cl = uma_zalloc(rxb->zone, M_NOWAIT); 4684 } 4685 if (cl == NULL) 4686 break; 4687 } 4688 fl->cl_allocated++; 4689 n--; 4690 4691 pa = pmap_kextract((vm_offset_t)cl); 4692 sd->cl = cl; 4693 sd->zidx = fl->zidx; 4694 4695 if (fl->flags & FL_BUF_PACKING) { 4696 *d = htobe64(pa | rxb->hwidx2); 4697 sd->moff = rxb->size2; 4698 } else { 4699 *d = htobe64(pa | rxb->hwidx1); 4700 sd->moff = 0; 4701 } 4702 recycled: 4703 sd->nmbuf = 0; 4704 d++; 4705 sd++; 4706 if (__predict_false((++fl->pidx & 7) == 0)) { 4707 uint16_t pidx = fl->pidx >> 3; 4708 4709 if (__predict_false(pidx == fl->sidx)) { 4710 fl->pidx = 0; 4711 pidx = 0; 4712 sd = fl->sdesc; 4713 d = fl->desc; 4714 } 4715 if (n < 8 || pidx == max_pidx) 4716 break; 4717 4718 if (IDXDIFF(pidx, fl->dbidx, fl->sidx) >= 4) 4719 ring_fl_db(sc, fl); 4720 } 4721 } 4722 4723 if ((fl->pidx >> 3) != fl->dbidx) 4724 ring_fl_db(sc, fl); 4725 4726 return (FL_RUNNING_LOW(fl) && !(fl->flags & FL_STARVING)); 4727 } 4728 4729 /* 4730 * Attempt to refill all starving freelists. 4731 */ 4732 static void 4733 refill_sfl(void *arg) 4734 { 4735 struct adapter *sc = arg; 4736 struct sge_fl *fl, *fl_temp; 4737 4738 mtx_assert(&sc->sfl_lock, MA_OWNED); 4739 TAILQ_FOREACH_SAFE(fl, &sc->sfl, link, fl_temp) { 4740 FL_LOCK(fl); 4741 refill_fl(sc, fl, 64); 4742 if (FL_NOT_RUNNING_LOW(fl) || fl->flags & FL_DOOMED) { 4743 TAILQ_REMOVE(&sc->sfl, fl, link); 4744 fl->flags &= ~FL_STARVING; 4745 } 4746 FL_UNLOCK(fl); 4747 } 4748 4749 if (!TAILQ_EMPTY(&sc->sfl)) 4750 callout_schedule(&sc->sfl_callout, hz / 5); 4751 } 4752 4753 static int 4754 alloc_fl_sdesc(struct sge_fl *fl) 4755 { 4756 4757 fl->sdesc = malloc(fl->sidx * 8 * sizeof(struct fl_sdesc), M_CXGBE, 4758 M_ZERO | M_WAITOK); 4759 4760 return (0); 4761 } 4762 4763 static void 4764 free_fl_sdesc(struct adapter *sc, struct sge_fl *fl) 4765 { 4766 struct fl_sdesc *sd; 4767 struct cluster_metadata *clm; 4768 int i; 4769 4770 sd = fl->sdesc; 4771 for (i = 0; i < fl->sidx * 8; i++, sd++) { 4772 if (sd->cl == NULL) 4773 continue; 4774 4775 if (sd->nmbuf == 0) 4776 uma_zfree(sc->sge.rx_buf_info[sd->zidx].zone, sd->cl); 4777 else if (fl->flags & FL_BUF_PACKING) { 4778 clm = cl_metadata(sd); 4779 if (atomic_fetchadd_int(&clm->refcount, -1) == 1) { 4780 uma_zfree(sc->sge.rx_buf_info[sd->zidx].zone, 4781 sd->cl); 4782 counter_u64_add(extfree_rels, 1); 4783 } 4784 } 4785 sd->cl = NULL; 4786 } 4787 4788 free(fl->sdesc, M_CXGBE); 4789 fl->sdesc = NULL; 4790 } 4791 4792 static inline void 4793 get_pkt_gl(struct mbuf *m, struct sglist *gl) 4794 { 4795 int rc; 4796 4797 M_ASSERTPKTHDR(m); 4798 4799 sglist_reset(gl); 4800 rc = sglist_append_mbuf(gl, m); 4801 if (__predict_false(rc != 0)) { 4802 panic("%s: mbuf %p (%d segs) was vetted earlier but now fails " 4803 "with %d.", __func__, m, mbuf_nsegs(m), rc); 4804 } 4805 4806 KASSERT(gl->sg_nseg == mbuf_nsegs(m), 4807 ("%s: nsegs changed for mbuf %p from %d to %d", __func__, m, 4808 mbuf_nsegs(m), gl->sg_nseg)); 4809 #if 0 /* vm_wr not readily available here. */ 4810 KASSERT(gl->sg_nseg > 0 && gl->sg_nseg <= max_nsegs_allowed(m, vm_wr), 4811 ("%s: %d segments, should have been 1 <= nsegs <= %d", __func__, 4812 gl->sg_nseg, max_nsegs_allowed(m, vm_wr))); 4813 #endif 4814 } 4815 4816 /* 4817 * len16 for a txpkt WR with a GL. Includes the firmware work request header. 4818 */ 4819 static inline u_int 4820 txpkt_len16(u_int nsegs, const u_int extra) 4821 { 4822 u_int n; 4823 4824 MPASS(nsegs > 0); 4825 4826 nsegs--; /* first segment is part of ulptx_sgl */ 4827 n = extra + sizeof(struct fw_eth_tx_pkt_wr) + 4828 sizeof(struct cpl_tx_pkt_core) + 4829 sizeof(struct ulptx_sgl) + 8 * ((3 * nsegs) / 2 + (nsegs & 1)); 4830 4831 return (howmany(n, 16)); 4832 } 4833 4834 /* 4835 * len16 for a txpkt_vm WR with a GL. Includes the firmware work 4836 * request header. 4837 */ 4838 static inline u_int 4839 txpkt_vm_len16(u_int nsegs, const u_int extra) 4840 { 4841 u_int n; 4842 4843 MPASS(nsegs > 0); 4844 4845 nsegs--; /* first segment is part of ulptx_sgl */ 4846 n = extra + sizeof(struct fw_eth_tx_pkt_vm_wr) + 4847 sizeof(struct cpl_tx_pkt_core) + 4848 sizeof(struct ulptx_sgl) + 8 * ((3 * nsegs) / 2 + (nsegs & 1)); 4849 4850 return (howmany(n, 16)); 4851 } 4852 4853 static inline void 4854 calculate_mbuf_len16(struct mbuf *m, bool vm_wr) 4855 { 4856 const int lso = sizeof(struct cpl_tx_pkt_lso_core); 4857 const int tnl_lso = sizeof(struct cpl_tx_tnl_lso); 4858 4859 if (vm_wr) { 4860 if (needs_tso(m)) 4861 set_mbuf_len16(m, txpkt_vm_len16(mbuf_nsegs(m), lso)); 4862 else 4863 set_mbuf_len16(m, txpkt_vm_len16(mbuf_nsegs(m), 0)); 4864 return; 4865 } 4866 4867 if (needs_tso(m)) { 4868 if (needs_vxlan_tso(m)) 4869 set_mbuf_len16(m, txpkt_len16(mbuf_nsegs(m), tnl_lso)); 4870 else 4871 set_mbuf_len16(m, txpkt_len16(mbuf_nsegs(m), lso)); 4872 } else 4873 set_mbuf_len16(m, txpkt_len16(mbuf_nsegs(m), 0)); 4874 } 4875 4876 /* 4877 * len16 for a txpkts type 0 WR with a GL. Does not include the firmware work 4878 * request header. 4879 */ 4880 static inline u_int 4881 txpkts0_len16(u_int nsegs) 4882 { 4883 u_int n; 4884 4885 MPASS(nsegs > 0); 4886 4887 nsegs--; /* first segment is part of ulptx_sgl */ 4888 n = sizeof(struct ulp_txpkt) + sizeof(struct ulptx_idata) + 4889 sizeof(struct cpl_tx_pkt_core) + sizeof(struct ulptx_sgl) + 4890 8 * ((3 * nsegs) / 2 + (nsegs & 1)); 4891 4892 return (howmany(n, 16)); 4893 } 4894 4895 /* 4896 * len16 for a txpkts type 1 WR with a GL. Does not include the firmware work 4897 * request header. 4898 */ 4899 static inline u_int 4900 txpkts1_len16(void) 4901 { 4902 u_int n; 4903 4904 n = sizeof(struct cpl_tx_pkt_core) + sizeof(struct ulptx_sgl); 4905 4906 return (howmany(n, 16)); 4907 } 4908 4909 static inline u_int 4910 imm_payload(u_int ndesc) 4911 { 4912 u_int n; 4913 4914 n = ndesc * EQ_ESIZE - sizeof(struct fw_eth_tx_pkt_wr) - 4915 sizeof(struct cpl_tx_pkt_core); 4916 4917 return (n); 4918 } 4919 4920 static inline uint64_t 4921 csum_to_ctrl(struct adapter *sc, struct mbuf *m) 4922 { 4923 uint64_t ctrl; 4924 int csum_type, l2hlen, l3hlen; 4925 int x, y; 4926 static const int csum_types[3][2] = { 4927 {TX_CSUM_TCPIP, TX_CSUM_TCPIP6}, 4928 {TX_CSUM_UDPIP, TX_CSUM_UDPIP6}, 4929 {TX_CSUM_IP, 0} 4930 }; 4931 4932 M_ASSERTPKTHDR(m); 4933 4934 if (!needs_hwcsum(m)) 4935 return (F_TXPKT_IPCSUM_DIS | F_TXPKT_L4CSUM_DIS); 4936 4937 MPASS(m->m_pkthdr.l2hlen >= ETHER_HDR_LEN); 4938 MPASS(m->m_pkthdr.l3hlen >= sizeof(struct ip)); 4939 4940 if (needs_vxlan_csum(m)) { 4941 MPASS(m->m_pkthdr.l4hlen > 0); 4942 MPASS(m->m_pkthdr.l5hlen > 0); 4943 MPASS(m->m_pkthdr.inner_l2hlen >= ETHER_HDR_LEN); 4944 MPASS(m->m_pkthdr.inner_l3hlen >= sizeof(struct ip)); 4945 4946 l2hlen = m->m_pkthdr.l2hlen + m->m_pkthdr.l3hlen + 4947 m->m_pkthdr.l4hlen + m->m_pkthdr.l5hlen + 4948 m->m_pkthdr.inner_l2hlen - ETHER_HDR_LEN; 4949 l3hlen = m->m_pkthdr.inner_l3hlen; 4950 } else { 4951 l2hlen = m->m_pkthdr.l2hlen - ETHER_HDR_LEN; 4952 l3hlen = m->m_pkthdr.l3hlen; 4953 } 4954 4955 ctrl = 0; 4956 if (!needs_l3_csum(m)) 4957 ctrl |= F_TXPKT_IPCSUM_DIS; 4958 4959 if (m->m_pkthdr.csum_flags & (CSUM_IP_TCP | CSUM_INNER_IP_TCP | 4960 CSUM_IP6_TCP | CSUM_INNER_IP6_TCP)) 4961 x = 0; /* TCP */ 4962 else if (m->m_pkthdr.csum_flags & (CSUM_IP_UDP | CSUM_INNER_IP_UDP | 4963 CSUM_IP6_UDP | CSUM_INNER_IP6_UDP)) 4964 x = 1; /* UDP */ 4965 else 4966 x = 2; 4967 4968 if (m->m_pkthdr.csum_flags & (CSUM_IP | CSUM_IP_TCP | CSUM_IP_UDP | 4969 CSUM_INNER_IP | CSUM_INNER_IP_TCP | CSUM_INNER_IP_UDP)) 4970 y = 0; /* IPv4 */ 4971 else { 4972 MPASS(m->m_pkthdr.csum_flags & (CSUM_IP6_TCP | CSUM_IP6_UDP | 4973 CSUM_INNER_IP6_TCP | CSUM_INNER_IP6_UDP)); 4974 y = 1; /* IPv6 */ 4975 } 4976 /* 4977 * needs_hwcsum returned true earlier so there must be some kind of 4978 * checksum to calculate. 4979 */ 4980 csum_type = csum_types[x][y]; 4981 MPASS(csum_type != 0); 4982 if (csum_type == TX_CSUM_IP) 4983 ctrl |= F_TXPKT_L4CSUM_DIS; 4984 ctrl |= V_TXPKT_CSUM_TYPE(csum_type) | V_TXPKT_IPHDR_LEN(l3hlen); 4985 if (chip_id(sc) <= CHELSIO_T5) 4986 ctrl |= V_TXPKT_ETHHDR_LEN(l2hlen); 4987 else 4988 ctrl |= V_T6_TXPKT_ETHHDR_LEN(l2hlen); 4989 4990 return (ctrl); 4991 } 4992 4993 static inline void * 4994 write_lso_cpl(void *cpl, struct mbuf *m0) 4995 { 4996 struct cpl_tx_pkt_lso_core *lso; 4997 uint32_t ctrl; 4998 4999 KASSERT(m0->m_pkthdr.l2hlen > 0 && m0->m_pkthdr.l3hlen > 0 && 5000 m0->m_pkthdr.l4hlen > 0, 5001 ("%s: mbuf %p needs TSO but missing header lengths", 5002 __func__, m0)); 5003 5004 ctrl = V_LSO_OPCODE(CPL_TX_PKT_LSO) | 5005 F_LSO_FIRST_SLICE | F_LSO_LAST_SLICE | 5006 V_LSO_ETHHDR_LEN((m0->m_pkthdr.l2hlen - ETHER_HDR_LEN) >> 2) | 5007 V_LSO_IPHDR_LEN(m0->m_pkthdr.l3hlen >> 2) | 5008 V_LSO_TCPHDR_LEN(m0->m_pkthdr.l4hlen >> 2); 5009 if (m0->m_pkthdr.l3hlen == sizeof(struct ip6_hdr)) 5010 ctrl |= F_LSO_IPV6; 5011 5012 lso = cpl; 5013 lso->lso_ctrl = htobe32(ctrl); 5014 lso->ipid_ofst = htobe16(0); 5015 lso->mss = htobe16(m0->m_pkthdr.tso_segsz); 5016 lso->seqno_offset = htobe32(0); 5017 lso->len = htobe32(m0->m_pkthdr.len); 5018 5019 return (lso + 1); 5020 } 5021 5022 static void * 5023 write_tnl_lso_cpl(void *cpl, struct mbuf *m0) 5024 { 5025 struct cpl_tx_tnl_lso *tnl_lso = cpl; 5026 uint32_t ctrl; 5027 5028 KASSERT(m0->m_pkthdr.inner_l2hlen > 0 && 5029 m0->m_pkthdr.inner_l3hlen > 0 && m0->m_pkthdr.inner_l4hlen > 0 && 5030 m0->m_pkthdr.inner_l5hlen > 0, 5031 ("%s: mbuf %p needs VXLAN_TSO but missing inner header lengths", 5032 __func__, m0)); 5033 KASSERT(m0->m_pkthdr.l2hlen > 0 && m0->m_pkthdr.l3hlen > 0 && 5034 m0->m_pkthdr.l4hlen > 0 && m0->m_pkthdr.l5hlen > 0, 5035 ("%s: mbuf %p needs VXLAN_TSO but missing outer header lengths", 5036 __func__, m0)); 5037 5038 /* Outer headers. */ 5039 ctrl = V_CPL_TX_TNL_LSO_OPCODE(CPL_TX_TNL_LSO) | 5040 F_CPL_TX_TNL_LSO_FIRST | F_CPL_TX_TNL_LSO_LAST | 5041 V_CPL_TX_TNL_LSO_ETHHDRLENOUT( 5042 (m0->m_pkthdr.l2hlen - ETHER_HDR_LEN) >> 2) | 5043 V_CPL_TX_TNL_LSO_IPHDRLENOUT(m0->m_pkthdr.l3hlen >> 2) | 5044 F_CPL_TX_TNL_LSO_IPLENSETOUT; 5045 if (m0->m_pkthdr.l3hlen == sizeof(struct ip6_hdr)) 5046 ctrl |= F_CPL_TX_TNL_LSO_IPV6OUT; 5047 else { 5048 ctrl |= F_CPL_TX_TNL_LSO_IPHDRCHKOUT | 5049 F_CPL_TX_TNL_LSO_IPIDINCOUT; 5050 } 5051 tnl_lso->op_to_IpIdSplitOut = htobe32(ctrl); 5052 tnl_lso->IpIdOffsetOut = 0; 5053 tnl_lso->UdpLenSetOut_to_TnlHdrLen = 5054 htobe16(F_CPL_TX_TNL_LSO_UDPCHKCLROUT | 5055 F_CPL_TX_TNL_LSO_UDPLENSETOUT | 5056 V_CPL_TX_TNL_LSO_TNLHDRLEN(m0->m_pkthdr.l2hlen + 5057 m0->m_pkthdr.l3hlen + m0->m_pkthdr.l4hlen + 5058 m0->m_pkthdr.l5hlen) | 5059 V_CPL_TX_TNL_LSO_TNLTYPE(TX_TNL_TYPE_VXLAN)); 5060 tnl_lso->r1 = 0; 5061 5062 /* Inner headers. */ 5063 ctrl = V_CPL_TX_TNL_LSO_ETHHDRLEN( 5064 (m0->m_pkthdr.inner_l2hlen - ETHER_HDR_LEN) >> 2) | 5065 V_CPL_TX_TNL_LSO_IPHDRLEN(m0->m_pkthdr.inner_l3hlen >> 2) | 5066 V_CPL_TX_TNL_LSO_TCPHDRLEN(m0->m_pkthdr.inner_l4hlen >> 2); 5067 if (m0->m_pkthdr.inner_l3hlen == sizeof(struct ip6_hdr)) 5068 ctrl |= F_CPL_TX_TNL_LSO_IPV6; 5069 tnl_lso->Flow_to_TcpHdrLen = htobe32(ctrl); 5070 tnl_lso->IpIdOffset = 0; 5071 tnl_lso->IpIdSplit_to_Mss = 5072 htobe16(V_CPL_TX_TNL_LSO_MSS(m0->m_pkthdr.tso_segsz)); 5073 tnl_lso->TCPSeqOffset = 0; 5074 tnl_lso->EthLenOffset_Size = 5075 htobe32(V_CPL_TX_TNL_LSO_SIZE(m0->m_pkthdr.len)); 5076 5077 return (tnl_lso + 1); 5078 } 5079 5080 #define VM_TX_L2HDR_LEN 16 /* ethmacdst to vlantci */ 5081 5082 /* 5083 * Write a VM txpkt WR for this packet to the hardware descriptors, update the 5084 * software descriptor, and advance the pidx. It is guaranteed that enough 5085 * descriptors are available. 5086 * 5087 * The return value is the # of hardware descriptors used. 5088 */ 5089 static u_int 5090 write_txpkt_vm_wr(struct adapter *sc, struct sge_txq *txq, struct mbuf *m0) 5091 { 5092 struct sge_eq *eq; 5093 struct fw_eth_tx_pkt_vm_wr *wr; 5094 struct tx_sdesc *txsd; 5095 struct cpl_tx_pkt_core *cpl; 5096 uint32_t ctrl; /* used in many unrelated places */ 5097 uint64_t ctrl1; 5098 int len16, ndesc, pktlen, nsegs; 5099 caddr_t dst; 5100 5101 TXQ_LOCK_ASSERT_OWNED(txq); 5102 M_ASSERTPKTHDR(m0); 5103 5104 len16 = mbuf_len16(m0); 5105 nsegs = mbuf_nsegs(m0); 5106 pktlen = m0->m_pkthdr.len; 5107 ctrl = sizeof(struct cpl_tx_pkt_core); 5108 if (needs_tso(m0)) 5109 ctrl += sizeof(struct cpl_tx_pkt_lso_core); 5110 ndesc = tx_len16_to_desc(len16); 5111 5112 /* Firmware work request header */ 5113 eq = &txq->eq; 5114 wr = (void *)&eq->desc[eq->pidx]; 5115 wr->op_immdlen = htobe32(V_FW_WR_OP(FW_ETH_TX_PKT_VM_WR) | 5116 V_FW_ETH_TX_PKT_WR_IMMDLEN(ctrl)); 5117 5118 ctrl = V_FW_WR_LEN16(len16); 5119 wr->equiq_to_len16 = htobe32(ctrl); 5120 wr->r3[0] = 0; 5121 wr->r3[1] = 0; 5122 5123 /* 5124 * Copy over ethmacdst, ethmacsrc, ethtype, and vlantci. 5125 * vlantci is ignored unless the ethtype is 0x8100, so it's 5126 * simpler to always copy it rather than making it 5127 * conditional. Also, it seems that we do not have to set 5128 * vlantci or fake the ethtype when doing VLAN tag insertion. 5129 */ 5130 m_copydata(m0, 0, VM_TX_L2HDR_LEN, wr->ethmacdst); 5131 5132 if (needs_tso(m0)) { 5133 cpl = write_lso_cpl(wr + 1, m0); 5134 txq->tso_wrs++; 5135 } else 5136 cpl = (void *)(wr + 1); 5137 5138 /* Checksum offload */ 5139 ctrl1 = csum_to_ctrl(sc, m0); 5140 if (ctrl1 != (F_TXPKT_IPCSUM_DIS | F_TXPKT_L4CSUM_DIS)) 5141 txq->txcsum++; /* some hardware assistance provided */ 5142 5143 /* VLAN tag insertion */ 5144 if (needs_vlan_insertion(m0)) { 5145 ctrl1 |= F_TXPKT_VLAN_VLD | 5146 V_TXPKT_VLAN(m0->m_pkthdr.ether_vtag); 5147 txq->vlan_insertion++; 5148 } 5149 5150 /* CPL header */ 5151 cpl->ctrl0 = txq->cpl_ctrl0; 5152 cpl->pack = 0; 5153 cpl->len = htobe16(pktlen); 5154 cpl->ctrl1 = htobe64(ctrl1); 5155 5156 /* SGL */ 5157 dst = (void *)(cpl + 1); 5158 5159 /* 5160 * A packet using TSO will use up an entire descriptor for the 5161 * firmware work request header, LSO CPL, and TX_PKT_XT CPL. 5162 * If this descriptor is the last descriptor in the ring, wrap 5163 * around to the front of the ring explicitly for the start of 5164 * the sgl. 5165 */ 5166 if (dst == (void *)&eq->desc[eq->sidx]) { 5167 dst = (void *)&eq->desc[0]; 5168 write_gl_to_txd(txq, m0, &dst, 0); 5169 } else 5170 write_gl_to_txd(txq, m0, &dst, eq->sidx - ndesc < eq->pidx); 5171 txq->sgl_wrs++; 5172 txq->txpkt_wrs++; 5173 5174 txsd = &txq->sdesc[eq->pidx]; 5175 txsd->m = m0; 5176 txsd->desc_used = ndesc; 5177 5178 return (ndesc); 5179 } 5180 5181 /* 5182 * Write a raw WR to the hardware descriptors, update the software 5183 * descriptor, and advance the pidx. It is guaranteed that enough 5184 * descriptors are available. 5185 * 5186 * The return value is the # of hardware descriptors used. 5187 */ 5188 static u_int 5189 write_raw_wr(struct sge_txq *txq, void *wr, struct mbuf *m0, u_int available) 5190 { 5191 struct sge_eq *eq = &txq->eq; 5192 struct tx_sdesc *txsd; 5193 struct mbuf *m; 5194 caddr_t dst; 5195 int len16, ndesc; 5196 5197 len16 = mbuf_len16(m0); 5198 ndesc = tx_len16_to_desc(len16); 5199 MPASS(ndesc <= available); 5200 5201 dst = wr; 5202 for (m = m0; m != NULL; m = m->m_next) 5203 copy_to_txd(eq, mtod(m, caddr_t), &dst, m->m_len); 5204 5205 txq->raw_wrs++; 5206 5207 txsd = &txq->sdesc[eq->pidx]; 5208 txsd->m = m0; 5209 txsd->desc_used = ndesc; 5210 5211 return (ndesc); 5212 } 5213 5214 /* 5215 * Write a txpkt WR for this packet to the hardware descriptors, update the 5216 * software descriptor, and advance the pidx. It is guaranteed that enough 5217 * descriptors are available. 5218 * 5219 * The return value is the # of hardware descriptors used. 5220 */ 5221 static u_int 5222 write_txpkt_wr(struct adapter *sc, struct sge_txq *txq, struct mbuf *m0, 5223 u_int available) 5224 { 5225 struct sge_eq *eq; 5226 struct fw_eth_tx_pkt_wr *wr; 5227 struct tx_sdesc *txsd; 5228 struct cpl_tx_pkt_core *cpl; 5229 uint32_t ctrl; /* used in many unrelated places */ 5230 uint64_t ctrl1; 5231 int len16, ndesc, pktlen, nsegs; 5232 caddr_t dst; 5233 5234 TXQ_LOCK_ASSERT_OWNED(txq); 5235 M_ASSERTPKTHDR(m0); 5236 5237 len16 = mbuf_len16(m0); 5238 nsegs = mbuf_nsegs(m0); 5239 pktlen = m0->m_pkthdr.len; 5240 ctrl = sizeof(struct cpl_tx_pkt_core); 5241 if (needs_tso(m0)) { 5242 if (needs_vxlan_tso(m0)) 5243 ctrl += sizeof(struct cpl_tx_tnl_lso); 5244 else 5245 ctrl += sizeof(struct cpl_tx_pkt_lso_core); 5246 } else if (!(mbuf_cflags(m0) & MC_NOMAP) && pktlen <= imm_payload(2) && 5247 available >= 2) { 5248 /* Immediate data. Recalculate len16 and set nsegs to 0. */ 5249 ctrl += pktlen; 5250 len16 = howmany(sizeof(struct fw_eth_tx_pkt_wr) + 5251 sizeof(struct cpl_tx_pkt_core) + pktlen, 16); 5252 nsegs = 0; 5253 } 5254 ndesc = tx_len16_to_desc(len16); 5255 MPASS(ndesc <= available); 5256 5257 /* Firmware work request header */ 5258 eq = &txq->eq; 5259 wr = (void *)&eq->desc[eq->pidx]; 5260 wr->op_immdlen = htobe32(V_FW_WR_OP(FW_ETH_TX_PKT_WR) | 5261 V_FW_ETH_TX_PKT_WR_IMMDLEN(ctrl)); 5262 5263 ctrl = V_FW_WR_LEN16(len16); 5264 wr->equiq_to_len16 = htobe32(ctrl); 5265 wr->r3 = 0; 5266 5267 if (needs_tso(m0)) { 5268 if (needs_vxlan_tso(m0)) { 5269 cpl = write_tnl_lso_cpl(wr + 1, m0); 5270 txq->vxlan_tso_wrs++; 5271 } else { 5272 cpl = write_lso_cpl(wr + 1, m0); 5273 txq->tso_wrs++; 5274 } 5275 } else 5276 cpl = (void *)(wr + 1); 5277 5278 /* Checksum offload */ 5279 ctrl1 = csum_to_ctrl(sc, m0); 5280 if (ctrl1 != (F_TXPKT_IPCSUM_DIS | F_TXPKT_L4CSUM_DIS)) { 5281 /* some hardware assistance provided */ 5282 if (needs_vxlan_csum(m0)) 5283 txq->vxlan_txcsum++; 5284 else 5285 txq->txcsum++; 5286 } 5287 5288 /* VLAN tag insertion */ 5289 if (needs_vlan_insertion(m0)) { 5290 ctrl1 |= F_TXPKT_VLAN_VLD | 5291 V_TXPKT_VLAN(m0->m_pkthdr.ether_vtag); 5292 txq->vlan_insertion++; 5293 } 5294 5295 /* CPL header */ 5296 cpl->ctrl0 = txq->cpl_ctrl0; 5297 cpl->pack = 0; 5298 cpl->len = htobe16(pktlen); 5299 cpl->ctrl1 = htobe64(ctrl1); 5300 5301 /* SGL */ 5302 dst = (void *)(cpl + 1); 5303 if (__predict_false((uintptr_t)dst == (uintptr_t)&eq->desc[eq->sidx])) 5304 dst = (caddr_t)&eq->desc[0]; 5305 if (nsegs > 0) { 5306 5307 write_gl_to_txd(txq, m0, &dst, eq->sidx - ndesc < eq->pidx); 5308 txq->sgl_wrs++; 5309 } else { 5310 struct mbuf *m; 5311 5312 for (m = m0; m != NULL; m = m->m_next) { 5313 copy_to_txd(eq, mtod(m, caddr_t), &dst, m->m_len); 5314 #ifdef INVARIANTS 5315 pktlen -= m->m_len; 5316 #endif 5317 } 5318 #ifdef INVARIANTS 5319 KASSERT(pktlen == 0, ("%s: %d bytes left.", __func__, pktlen)); 5320 #endif 5321 txq->imm_wrs++; 5322 } 5323 5324 txq->txpkt_wrs++; 5325 5326 txsd = &txq->sdesc[eq->pidx]; 5327 txsd->m = m0; 5328 txsd->desc_used = ndesc; 5329 5330 return (ndesc); 5331 } 5332 5333 static inline bool 5334 cmp_l2hdr(struct txpkts *txp, struct mbuf *m) 5335 { 5336 int len; 5337 5338 MPASS(txp->npkt > 0); 5339 MPASS(m->m_len >= VM_TX_L2HDR_LEN); 5340 5341 if (txp->ethtype == be16toh(ETHERTYPE_VLAN)) 5342 len = VM_TX_L2HDR_LEN; 5343 else 5344 len = sizeof(struct ether_header); 5345 5346 return (memcmp(m->m_data, &txp->ethmacdst[0], len) != 0); 5347 } 5348 5349 static inline void 5350 save_l2hdr(struct txpkts *txp, struct mbuf *m) 5351 { 5352 MPASS(m->m_len >= VM_TX_L2HDR_LEN); 5353 5354 memcpy(&txp->ethmacdst[0], mtod(m, const void *), VM_TX_L2HDR_LEN); 5355 } 5356 5357 static int 5358 add_to_txpkts_vf(struct adapter *sc, struct sge_txq *txq, struct mbuf *m, 5359 int avail, bool *send) 5360 { 5361 struct txpkts *txp = &txq->txp; 5362 5363 /* Cannot have TSO and coalesce at the same time. */ 5364 if (cannot_use_txpkts(m)) { 5365 cannot_coalesce: 5366 *send = txp->npkt > 0; 5367 return (EINVAL); 5368 } 5369 5370 /* VF allows coalescing of type 1 (1 GL) only */ 5371 if (mbuf_nsegs(m) > 1) 5372 goto cannot_coalesce; 5373 5374 *send = false; 5375 if (txp->npkt > 0) { 5376 MPASS(tx_len16_to_desc(txp->len16) <= avail); 5377 MPASS(txp->npkt < txp->max_npkt); 5378 MPASS(txp->wr_type == 1); /* VF supports type 1 only */ 5379 5380 if (tx_len16_to_desc(txp->len16 + txpkts1_len16()) > avail) { 5381 retry_after_send: 5382 *send = true; 5383 return (EAGAIN); 5384 } 5385 if (m->m_pkthdr.len + txp->plen > 65535) 5386 goto retry_after_send; 5387 if (cmp_l2hdr(txp, m)) 5388 goto retry_after_send; 5389 5390 txp->len16 += txpkts1_len16(); 5391 txp->plen += m->m_pkthdr.len; 5392 txp->mb[txp->npkt++] = m; 5393 if (txp->npkt == txp->max_npkt) 5394 *send = true; 5395 } else { 5396 txp->len16 = howmany(sizeof(struct fw_eth_tx_pkts_vm_wr), 16) + 5397 txpkts1_len16(); 5398 if (tx_len16_to_desc(txp->len16) > avail) 5399 goto cannot_coalesce; 5400 txp->npkt = 1; 5401 txp->wr_type = 1; 5402 txp->plen = m->m_pkthdr.len; 5403 txp->mb[0] = m; 5404 save_l2hdr(txp, m); 5405 } 5406 return (0); 5407 } 5408 5409 static int 5410 add_to_txpkts_pf(struct adapter *sc, struct sge_txq *txq, struct mbuf *m, 5411 int avail, bool *send) 5412 { 5413 struct txpkts *txp = &txq->txp; 5414 int nsegs; 5415 5416 MPASS(!(sc->flags & IS_VF)); 5417 5418 /* Cannot have TSO and coalesce at the same time. */ 5419 if (cannot_use_txpkts(m)) { 5420 cannot_coalesce: 5421 *send = txp->npkt > 0; 5422 return (EINVAL); 5423 } 5424 5425 *send = false; 5426 nsegs = mbuf_nsegs(m); 5427 if (txp->npkt == 0) { 5428 if (m->m_pkthdr.len > 65535) 5429 goto cannot_coalesce; 5430 if (nsegs > 1) { 5431 txp->wr_type = 0; 5432 txp->len16 = 5433 howmany(sizeof(struct fw_eth_tx_pkts_wr), 16) + 5434 txpkts0_len16(nsegs); 5435 } else { 5436 txp->wr_type = 1; 5437 txp->len16 = 5438 howmany(sizeof(struct fw_eth_tx_pkts_wr), 16) + 5439 txpkts1_len16(); 5440 } 5441 if (tx_len16_to_desc(txp->len16) > avail) 5442 goto cannot_coalesce; 5443 txp->npkt = 1; 5444 txp->plen = m->m_pkthdr.len; 5445 txp->mb[0] = m; 5446 } else { 5447 MPASS(tx_len16_to_desc(txp->len16) <= avail); 5448 MPASS(txp->npkt < txp->max_npkt); 5449 5450 if (m->m_pkthdr.len + txp->plen > 65535) { 5451 retry_after_send: 5452 *send = true; 5453 return (EAGAIN); 5454 } 5455 5456 MPASS(txp->wr_type == 0 || txp->wr_type == 1); 5457 if (txp->wr_type == 0) { 5458 if (tx_len16_to_desc(txp->len16 + 5459 txpkts0_len16(nsegs)) > min(avail, SGE_MAX_WR_NDESC)) 5460 goto retry_after_send; 5461 txp->len16 += txpkts0_len16(nsegs); 5462 } else { 5463 if (nsegs != 1) 5464 goto retry_after_send; 5465 if (tx_len16_to_desc(txp->len16 + txpkts1_len16()) > 5466 avail) 5467 goto retry_after_send; 5468 txp->len16 += txpkts1_len16(); 5469 } 5470 5471 txp->plen += m->m_pkthdr.len; 5472 txp->mb[txp->npkt++] = m; 5473 if (txp->npkt == txp->max_npkt) 5474 *send = true; 5475 } 5476 return (0); 5477 } 5478 5479 /* 5480 * Write a txpkts WR for the packets in txp to the hardware descriptors, update 5481 * the software descriptor, and advance the pidx. It is guaranteed that enough 5482 * descriptors are available. 5483 * 5484 * The return value is the # of hardware descriptors used. 5485 */ 5486 static u_int 5487 write_txpkts_wr(struct adapter *sc, struct sge_txq *txq) 5488 { 5489 const struct txpkts *txp = &txq->txp; 5490 struct sge_eq *eq = &txq->eq; 5491 struct fw_eth_tx_pkts_wr *wr; 5492 struct tx_sdesc *txsd; 5493 struct cpl_tx_pkt_core *cpl; 5494 uint64_t ctrl1; 5495 int ndesc, i, checkwrap; 5496 struct mbuf *m, *last; 5497 void *flitp; 5498 5499 TXQ_LOCK_ASSERT_OWNED(txq); 5500 MPASS(txp->npkt > 0); 5501 MPASS(txp->len16 <= howmany(SGE_MAX_WR_LEN, 16)); 5502 5503 wr = (void *)&eq->desc[eq->pidx]; 5504 wr->op_pkd = htobe32(V_FW_WR_OP(FW_ETH_TX_PKTS_WR)); 5505 wr->equiq_to_len16 = htobe32(V_FW_WR_LEN16(txp->len16)); 5506 wr->plen = htobe16(txp->plen); 5507 wr->npkt = txp->npkt; 5508 wr->r3 = 0; 5509 wr->type = txp->wr_type; 5510 flitp = wr + 1; 5511 5512 /* 5513 * At this point we are 16B into a hardware descriptor. If checkwrap is 5514 * set then we know the WR is going to wrap around somewhere. We'll 5515 * check for that at appropriate points. 5516 */ 5517 ndesc = tx_len16_to_desc(txp->len16); 5518 last = NULL; 5519 checkwrap = eq->sidx - ndesc < eq->pidx; 5520 for (i = 0; i < txp->npkt; i++) { 5521 m = txp->mb[i]; 5522 if (txp->wr_type == 0) { 5523 struct ulp_txpkt *ulpmc; 5524 struct ulptx_idata *ulpsc; 5525 5526 /* ULP master command */ 5527 ulpmc = flitp; 5528 ulpmc->cmd_dest = htobe32(V_ULPTX_CMD(ULP_TX_PKT) | 5529 V_ULP_TXPKT_DEST(0) | V_ULP_TXPKT_FID(eq->iqid)); 5530 ulpmc->len = htobe32(txpkts0_len16(mbuf_nsegs(m))); 5531 5532 /* ULP subcommand */ 5533 ulpsc = (void *)(ulpmc + 1); 5534 ulpsc->cmd_more = htobe32(V_ULPTX_CMD(ULP_TX_SC_IMM) | 5535 F_ULP_TX_SC_MORE); 5536 ulpsc->len = htobe32(sizeof(struct cpl_tx_pkt_core)); 5537 5538 cpl = (void *)(ulpsc + 1); 5539 if (checkwrap && 5540 (uintptr_t)cpl == (uintptr_t)&eq->desc[eq->sidx]) 5541 cpl = (void *)&eq->desc[0]; 5542 } else { 5543 cpl = flitp; 5544 } 5545 5546 /* Checksum offload */ 5547 ctrl1 = csum_to_ctrl(sc, m); 5548 if (ctrl1 != (F_TXPKT_IPCSUM_DIS | F_TXPKT_L4CSUM_DIS)) { 5549 /* some hardware assistance provided */ 5550 if (needs_vxlan_csum(m)) 5551 txq->vxlan_txcsum++; 5552 else 5553 txq->txcsum++; 5554 } 5555 5556 /* VLAN tag insertion */ 5557 if (needs_vlan_insertion(m)) { 5558 ctrl1 |= F_TXPKT_VLAN_VLD | 5559 V_TXPKT_VLAN(m->m_pkthdr.ether_vtag); 5560 txq->vlan_insertion++; 5561 } 5562 5563 /* CPL header */ 5564 cpl->ctrl0 = txq->cpl_ctrl0; 5565 cpl->pack = 0; 5566 cpl->len = htobe16(m->m_pkthdr.len); 5567 cpl->ctrl1 = htobe64(ctrl1); 5568 5569 flitp = cpl + 1; 5570 if (checkwrap && 5571 (uintptr_t)flitp == (uintptr_t)&eq->desc[eq->sidx]) 5572 flitp = (void *)&eq->desc[0]; 5573 5574 write_gl_to_txd(txq, m, (caddr_t *)(&flitp), checkwrap); 5575 5576 if (last != NULL) 5577 last->m_nextpkt = m; 5578 last = m; 5579 } 5580 5581 txq->sgl_wrs++; 5582 if (txp->wr_type == 0) { 5583 txq->txpkts0_pkts += txp->npkt; 5584 txq->txpkts0_wrs++; 5585 } else { 5586 txq->txpkts1_pkts += txp->npkt; 5587 txq->txpkts1_wrs++; 5588 } 5589 5590 txsd = &txq->sdesc[eq->pidx]; 5591 txsd->m = txp->mb[0]; 5592 txsd->desc_used = ndesc; 5593 5594 return (ndesc); 5595 } 5596 5597 static u_int 5598 write_txpkts_vm_wr(struct adapter *sc, struct sge_txq *txq) 5599 { 5600 const struct txpkts *txp = &txq->txp; 5601 struct sge_eq *eq = &txq->eq; 5602 struct fw_eth_tx_pkts_vm_wr *wr; 5603 struct tx_sdesc *txsd; 5604 struct cpl_tx_pkt_core *cpl; 5605 uint64_t ctrl1; 5606 int ndesc, i; 5607 struct mbuf *m, *last; 5608 void *flitp; 5609 5610 TXQ_LOCK_ASSERT_OWNED(txq); 5611 MPASS(txp->npkt > 0); 5612 MPASS(txp->wr_type == 1); /* VF supports type 1 only */ 5613 MPASS(txp->mb[0] != NULL); 5614 MPASS(txp->len16 <= howmany(SGE_MAX_WR_LEN, 16)); 5615 5616 wr = (void *)&eq->desc[eq->pidx]; 5617 wr->op_pkd = htobe32(V_FW_WR_OP(FW_ETH_TX_PKTS_VM_WR)); 5618 wr->equiq_to_len16 = htobe32(V_FW_WR_LEN16(txp->len16)); 5619 wr->r3 = 0; 5620 wr->plen = htobe16(txp->plen); 5621 wr->npkt = txp->npkt; 5622 wr->r4 = 0; 5623 memcpy(&wr->ethmacdst[0], &txp->ethmacdst[0], 16); 5624 flitp = wr + 1; 5625 5626 /* 5627 * At this point we are 32B into a hardware descriptor. Each mbuf in 5628 * the WR will take 32B so we check for the end of the descriptor ring 5629 * before writing odd mbufs (mb[1], 3, 5, ..) 5630 */ 5631 ndesc = tx_len16_to_desc(txp->len16); 5632 last = NULL; 5633 for (i = 0; i < txp->npkt; i++) { 5634 m = txp->mb[i]; 5635 if (i & 1 && (uintptr_t)flitp == (uintptr_t)&eq->desc[eq->sidx]) 5636 flitp = &eq->desc[0]; 5637 cpl = flitp; 5638 5639 /* Checksum offload */ 5640 ctrl1 = csum_to_ctrl(sc, m); 5641 if (ctrl1 != (F_TXPKT_IPCSUM_DIS | F_TXPKT_L4CSUM_DIS)) 5642 txq->txcsum++; /* some hardware assistance provided */ 5643 5644 /* VLAN tag insertion */ 5645 if (needs_vlan_insertion(m)) { 5646 ctrl1 |= F_TXPKT_VLAN_VLD | 5647 V_TXPKT_VLAN(m->m_pkthdr.ether_vtag); 5648 txq->vlan_insertion++; 5649 } 5650 5651 /* CPL header */ 5652 cpl->ctrl0 = txq->cpl_ctrl0; 5653 cpl->pack = 0; 5654 cpl->len = htobe16(m->m_pkthdr.len); 5655 cpl->ctrl1 = htobe64(ctrl1); 5656 5657 flitp = cpl + 1; 5658 MPASS(mbuf_nsegs(m) == 1); 5659 write_gl_to_txd(txq, m, (caddr_t *)(&flitp), 0); 5660 5661 if (last != NULL) 5662 last->m_nextpkt = m; 5663 last = m; 5664 } 5665 5666 txq->sgl_wrs++; 5667 txq->txpkts1_pkts += txp->npkt; 5668 txq->txpkts1_wrs++; 5669 5670 txsd = &txq->sdesc[eq->pidx]; 5671 txsd->m = txp->mb[0]; 5672 txsd->desc_used = ndesc; 5673 5674 return (ndesc); 5675 } 5676 5677 /* 5678 * If the SGL ends on an address that is not 16 byte aligned, this function will 5679 * add a 0 filled flit at the end. 5680 */ 5681 static void 5682 write_gl_to_txd(struct sge_txq *txq, struct mbuf *m, caddr_t *to, int checkwrap) 5683 { 5684 struct sge_eq *eq = &txq->eq; 5685 struct sglist *gl = txq->gl; 5686 struct sglist_seg *seg; 5687 __be64 *flitp, *wrap; 5688 struct ulptx_sgl *usgl; 5689 int i, nflits, nsegs; 5690 5691 KASSERT(((uintptr_t)(*to) & 0xf) == 0, 5692 ("%s: SGL must start at a 16 byte boundary: %p", __func__, *to)); 5693 MPASS((uintptr_t)(*to) >= (uintptr_t)&eq->desc[0]); 5694 MPASS((uintptr_t)(*to) < (uintptr_t)&eq->desc[eq->sidx]); 5695 5696 get_pkt_gl(m, gl); 5697 nsegs = gl->sg_nseg; 5698 MPASS(nsegs > 0); 5699 5700 nflits = (3 * (nsegs - 1)) / 2 + ((nsegs - 1) & 1) + 2; 5701 flitp = (__be64 *)(*to); 5702 wrap = (__be64 *)(&eq->desc[eq->sidx]); 5703 seg = &gl->sg_segs[0]; 5704 usgl = (void *)flitp; 5705 5706 /* 5707 * We start at a 16 byte boundary somewhere inside the tx descriptor 5708 * ring, so we're at least 16 bytes away from the status page. There is 5709 * no chance of a wrap around in the middle of usgl (which is 16 bytes). 5710 */ 5711 5712 usgl->cmd_nsge = htobe32(V_ULPTX_CMD(ULP_TX_SC_DSGL) | 5713 V_ULPTX_NSGE(nsegs)); 5714 usgl->len0 = htobe32(seg->ss_len); 5715 usgl->addr0 = htobe64(seg->ss_paddr); 5716 seg++; 5717 5718 if (checkwrap == 0 || (uintptr_t)(flitp + nflits) <= (uintptr_t)wrap) { 5719 5720 /* Won't wrap around at all */ 5721 5722 for (i = 0; i < nsegs - 1; i++, seg++) { 5723 usgl->sge[i / 2].len[i & 1] = htobe32(seg->ss_len); 5724 usgl->sge[i / 2].addr[i & 1] = htobe64(seg->ss_paddr); 5725 } 5726 if (i & 1) 5727 usgl->sge[i / 2].len[1] = htobe32(0); 5728 flitp += nflits; 5729 } else { 5730 5731 /* Will wrap somewhere in the rest of the SGL */ 5732 5733 /* 2 flits already written, write the rest flit by flit */ 5734 flitp = (void *)(usgl + 1); 5735 for (i = 0; i < nflits - 2; i++) { 5736 if (flitp == wrap) 5737 flitp = (void *)eq->desc; 5738 *flitp++ = get_flit(seg, nsegs - 1, i); 5739 } 5740 } 5741 5742 if (nflits & 1) { 5743 MPASS(((uintptr_t)flitp) & 0xf); 5744 *flitp++ = 0; 5745 } 5746 5747 MPASS((((uintptr_t)flitp) & 0xf) == 0); 5748 if (__predict_false(flitp == wrap)) 5749 *to = (void *)eq->desc; 5750 else 5751 *to = (void *)flitp; 5752 } 5753 5754 static inline void 5755 copy_to_txd(struct sge_eq *eq, caddr_t from, caddr_t *to, int len) 5756 { 5757 5758 MPASS((uintptr_t)(*to) >= (uintptr_t)&eq->desc[0]); 5759 MPASS((uintptr_t)(*to) < (uintptr_t)&eq->desc[eq->sidx]); 5760 5761 if (__predict_true((uintptr_t)(*to) + len <= 5762 (uintptr_t)&eq->desc[eq->sidx])) { 5763 bcopy(from, *to, len); 5764 (*to) += len; 5765 } else { 5766 int portion = (uintptr_t)&eq->desc[eq->sidx] - (uintptr_t)(*to); 5767 5768 bcopy(from, *to, portion); 5769 from += portion; 5770 portion = len - portion; /* remaining */ 5771 bcopy(from, (void *)eq->desc, portion); 5772 (*to) = (caddr_t)eq->desc + portion; 5773 } 5774 } 5775 5776 static inline void 5777 ring_eq_db(struct adapter *sc, struct sge_eq *eq, u_int n) 5778 { 5779 u_int db; 5780 5781 MPASS(n > 0); 5782 5783 db = eq->doorbells; 5784 if (n > 1) 5785 clrbit(&db, DOORBELL_WCWR); 5786 wmb(); 5787 5788 switch (ffs(db) - 1) { 5789 case DOORBELL_UDB: 5790 *eq->udb = htole32(V_QID(eq->udb_qid) | V_PIDX(n)); 5791 break; 5792 5793 case DOORBELL_WCWR: { 5794 volatile uint64_t *dst, *src; 5795 int i; 5796 5797 /* 5798 * Queues whose 128B doorbell segment fits in the page do not 5799 * use relative qid (udb_qid is always 0). Only queues with 5800 * doorbell segments can do WCWR. 5801 */ 5802 KASSERT(eq->udb_qid == 0 && n == 1, 5803 ("%s: inappropriate doorbell (0x%x, %d, %d) for eq %p", 5804 __func__, eq->doorbells, n, eq->dbidx, eq)); 5805 5806 dst = (volatile void *)((uintptr_t)eq->udb + UDBS_WR_OFFSET - 5807 UDBS_DB_OFFSET); 5808 i = eq->dbidx; 5809 src = (void *)&eq->desc[i]; 5810 while (src != (void *)&eq->desc[i + 1]) 5811 *dst++ = *src++; 5812 wmb(); 5813 break; 5814 } 5815 5816 case DOORBELL_UDBWC: 5817 *eq->udb = htole32(V_QID(eq->udb_qid) | V_PIDX(n)); 5818 wmb(); 5819 break; 5820 5821 case DOORBELL_KDB: 5822 t4_write_reg(sc, sc->sge_kdoorbell_reg, 5823 V_QID(eq->cntxt_id) | V_PIDX(n)); 5824 break; 5825 } 5826 5827 IDXINCR(eq->dbidx, n, eq->sidx); 5828 } 5829 5830 static inline u_int 5831 reclaimable_tx_desc(struct sge_eq *eq) 5832 { 5833 uint16_t hw_cidx; 5834 5835 hw_cidx = read_hw_cidx(eq); 5836 return (IDXDIFF(hw_cidx, eq->cidx, eq->sidx)); 5837 } 5838 5839 static inline u_int 5840 total_available_tx_desc(struct sge_eq *eq) 5841 { 5842 uint16_t hw_cidx, pidx; 5843 5844 hw_cidx = read_hw_cidx(eq); 5845 pidx = eq->pidx; 5846 5847 if (pidx == hw_cidx) 5848 return (eq->sidx - 1); 5849 else 5850 return (IDXDIFF(hw_cidx, pidx, eq->sidx) - 1); 5851 } 5852 5853 static inline uint16_t 5854 read_hw_cidx(struct sge_eq *eq) 5855 { 5856 struct sge_qstat *spg = (void *)&eq->desc[eq->sidx]; 5857 uint16_t cidx = spg->cidx; /* stable snapshot */ 5858 5859 return (be16toh(cidx)); 5860 } 5861 5862 /* 5863 * Reclaim 'n' descriptors approximately. 5864 */ 5865 static u_int 5866 reclaim_tx_descs(struct sge_txq *txq, u_int n) 5867 { 5868 struct tx_sdesc *txsd; 5869 struct sge_eq *eq = &txq->eq; 5870 u_int can_reclaim, reclaimed; 5871 5872 TXQ_LOCK_ASSERT_OWNED(txq); 5873 MPASS(n > 0); 5874 5875 reclaimed = 0; 5876 can_reclaim = reclaimable_tx_desc(eq); 5877 while (can_reclaim && reclaimed < n) { 5878 int ndesc; 5879 struct mbuf *m, *nextpkt; 5880 5881 txsd = &txq->sdesc[eq->cidx]; 5882 ndesc = txsd->desc_used; 5883 5884 /* Firmware doesn't return "partial" credits. */ 5885 KASSERT(can_reclaim >= ndesc, 5886 ("%s: unexpected number of credits: %d, %d", 5887 __func__, can_reclaim, ndesc)); 5888 KASSERT(ndesc != 0, 5889 ("%s: descriptor with no credits: cidx %d", 5890 __func__, eq->cidx)); 5891 5892 for (m = txsd->m; m != NULL; m = nextpkt) { 5893 nextpkt = m->m_nextpkt; 5894 m->m_nextpkt = NULL; 5895 m_freem(m); 5896 } 5897 reclaimed += ndesc; 5898 can_reclaim -= ndesc; 5899 IDXINCR(eq->cidx, ndesc, eq->sidx); 5900 } 5901 5902 return (reclaimed); 5903 } 5904 5905 static void 5906 tx_reclaim(void *arg, int n) 5907 { 5908 struct sge_txq *txq = arg; 5909 struct sge_eq *eq = &txq->eq; 5910 5911 do { 5912 if (TXQ_TRYLOCK(txq) == 0) 5913 break; 5914 n = reclaim_tx_descs(txq, 32); 5915 if (eq->cidx == eq->pidx) 5916 eq->equeqidx = eq->pidx; 5917 TXQ_UNLOCK(txq); 5918 } while (n > 0); 5919 } 5920 5921 static __be64 5922 get_flit(struct sglist_seg *segs, int nsegs, int idx) 5923 { 5924 int i = (idx / 3) * 2; 5925 5926 switch (idx % 3) { 5927 case 0: { 5928 uint64_t rc; 5929 5930 rc = (uint64_t)segs[i].ss_len << 32; 5931 if (i + 1 < nsegs) 5932 rc |= (uint64_t)(segs[i + 1].ss_len); 5933 5934 return (htobe64(rc)); 5935 } 5936 case 1: 5937 return (htobe64(segs[i].ss_paddr)); 5938 case 2: 5939 return (htobe64(segs[i + 1].ss_paddr)); 5940 } 5941 5942 return (0); 5943 } 5944 5945 static int 5946 find_refill_source(struct adapter *sc, int maxp, bool packing) 5947 { 5948 int i, zidx = -1; 5949 struct rx_buf_info *rxb = &sc->sge.rx_buf_info[0]; 5950 5951 if (packing) { 5952 for (i = 0; i < SW_ZONE_SIZES; i++, rxb++) { 5953 if (rxb->hwidx2 == -1) 5954 continue; 5955 if (rxb->size1 < PAGE_SIZE && 5956 rxb->size1 < largest_rx_cluster) 5957 continue; 5958 if (rxb->size1 > largest_rx_cluster) 5959 break; 5960 MPASS(rxb->size1 - rxb->size2 >= CL_METADATA_SIZE); 5961 if (rxb->size2 >= maxp) 5962 return (i); 5963 zidx = i; 5964 } 5965 } else { 5966 for (i = 0; i < SW_ZONE_SIZES; i++, rxb++) { 5967 if (rxb->hwidx1 == -1) 5968 continue; 5969 if (rxb->size1 > largest_rx_cluster) 5970 break; 5971 if (rxb->size1 >= maxp) 5972 return (i); 5973 zidx = i; 5974 } 5975 } 5976 5977 return (zidx); 5978 } 5979 5980 static void 5981 add_fl_to_sfl(struct adapter *sc, struct sge_fl *fl) 5982 { 5983 mtx_lock(&sc->sfl_lock); 5984 FL_LOCK(fl); 5985 if ((fl->flags & FL_DOOMED) == 0) { 5986 fl->flags |= FL_STARVING; 5987 TAILQ_INSERT_TAIL(&sc->sfl, fl, link); 5988 callout_reset(&sc->sfl_callout, hz / 5, refill_sfl, sc); 5989 } 5990 FL_UNLOCK(fl); 5991 mtx_unlock(&sc->sfl_lock); 5992 } 5993 5994 static void 5995 handle_wrq_egr_update(struct adapter *sc, struct sge_eq *eq) 5996 { 5997 struct sge_wrq *wrq = (void *)eq; 5998 5999 atomic_readandclear_int(&eq->equiq); 6000 taskqueue_enqueue(sc->tq[eq->tx_chan], &wrq->wrq_tx_task); 6001 } 6002 6003 static void 6004 handle_eth_egr_update(struct adapter *sc, struct sge_eq *eq) 6005 { 6006 struct sge_txq *txq = (void *)eq; 6007 6008 MPASS((eq->flags & EQ_TYPEMASK) == EQ_ETH); 6009 6010 atomic_readandclear_int(&eq->equiq); 6011 if (mp_ring_is_idle(txq->r)) 6012 taskqueue_enqueue(sc->tq[eq->tx_chan], &txq->tx_reclaim_task); 6013 else 6014 mp_ring_check_drainage(txq->r, 64); 6015 } 6016 6017 static int 6018 handle_sge_egr_update(struct sge_iq *iq, const struct rss_header *rss, 6019 struct mbuf *m) 6020 { 6021 const struct cpl_sge_egr_update *cpl = (const void *)(rss + 1); 6022 unsigned int qid = G_EGR_QID(ntohl(cpl->opcode_qid)); 6023 struct adapter *sc = iq->adapter; 6024 struct sge *s = &sc->sge; 6025 struct sge_eq *eq; 6026 static void (*h[])(struct adapter *, struct sge_eq *) = {NULL, 6027 &handle_wrq_egr_update, &handle_eth_egr_update, 6028 &handle_wrq_egr_update}; 6029 6030 KASSERT(m == NULL, ("%s: payload with opcode %02x", __func__, 6031 rss->opcode)); 6032 6033 eq = s->eqmap[qid - s->eq_start - s->eq_base]; 6034 (*h[eq->flags & EQ_TYPEMASK])(sc, eq); 6035 6036 return (0); 6037 } 6038 6039 /* handle_fw_msg works for both fw4_msg and fw6_msg because this is valid */ 6040 CTASSERT(offsetof(struct cpl_fw4_msg, data) == \ 6041 offsetof(struct cpl_fw6_msg, data)); 6042 6043 static int 6044 handle_fw_msg(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) 6045 { 6046 struct adapter *sc = iq->adapter; 6047 const struct cpl_fw6_msg *cpl = (const void *)(rss + 1); 6048 6049 KASSERT(m == NULL, ("%s: payload with opcode %02x", __func__, 6050 rss->opcode)); 6051 6052 if (cpl->type == FW_TYPE_RSSCPL || cpl->type == FW6_TYPE_RSSCPL) { 6053 const struct rss_header *rss2; 6054 6055 rss2 = (const struct rss_header *)&cpl->data[0]; 6056 return (t4_cpl_handler[rss2->opcode](iq, rss2, m)); 6057 } 6058 6059 return (t4_fw_msg_handler[cpl->type](sc, &cpl->data[0])); 6060 } 6061 6062 /** 6063 * t4_handle_wrerr_rpl - process a FW work request error message 6064 * @adap: the adapter 6065 * @rpl: start of the FW message 6066 */ 6067 static int 6068 t4_handle_wrerr_rpl(struct adapter *adap, const __be64 *rpl) 6069 { 6070 u8 opcode = *(const u8 *)rpl; 6071 const struct fw_error_cmd *e = (const void *)rpl; 6072 unsigned int i; 6073 6074 if (opcode != FW_ERROR_CMD) { 6075 log(LOG_ERR, 6076 "%s: Received WRERR_RPL message with opcode %#x\n", 6077 device_get_nameunit(adap->dev), opcode); 6078 return (EINVAL); 6079 } 6080 log(LOG_ERR, "%s: FW_ERROR (%s) ", device_get_nameunit(adap->dev), 6081 G_FW_ERROR_CMD_FATAL(be32toh(e->op_to_type)) ? "fatal" : 6082 "non-fatal"); 6083 switch (G_FW_ERROR_CMD_TYPE(be32toh(e->op_to_type))) { 6084 case FW_ERROR_TYPE_EXCEPTION: 6085 log(LOG_ERR, "exception info:\n"); 6086 for (i = 0; i < nitems(e->u.exception.info); i++) 6087 log(LOG_ERR, "%s%08x", i == 0 ? "\t" : " ", 6088 be32toh(e->u.exception.info[i])); 6089 log(LOG_ERR, "\n"); 6090 break; 6091 case FW_ERROR_TYPE_HWMODULE: 6092 log(LOG_ERR, "HW module regaddr %08x regval %08x\n", 6093 be32toh(e->u.hwmodule.regaddr), 6094 be32toh(e->u.hwmodule.regval)); 6095 break; 6096 case FW_ERROR_TYPE_WR: 6097 log(LOG_ERR, "WR cidx %d PF %d VF %d eqid %d hdr:\n", 6098 be16toh(e->u.wr.cidx), 6099 G_FW_ERROR_CMD_PFN(be16toh(e->u.wr.pfn_vfn)), 6100 G_FW_ERROR_CMD_VFN(be16toh(e->u.wr.pfn_vfn)), 6101 be32toh(e->u.wr.eqid)); 6102 for (i = 0; i < nitems(e->u.wr.wrhdr); i++) 6103 log(LOG_ERR, "%s%02x", i == 0 ? "\t" : " ", 6104 e->u.wr.wrhdr[i]); 6105 log(LOG_ERR, "\n"); 6106 break; 6107 case FW_ERROR_TYPE_ACL: 6108 log(LOG_ERR, "ACL cidx %d PF %d VF %d eqid %d %s", 6109 be16toh(e->u.acl.cidx), 6110 G_FW_ERROR_CMD_PFN(be16toh(e->u.acl.pfn_vfn)), 6111 G_FW_ERROR_CMD_VFN(be16toh(e->u.acl.pfn_vfn)), 6112 be32toh(e->u.acl.eqid), 6113 G_FW_ERROR_CMD_MV(be16toh(e->u.acl.mv_pkd)) ? "vlanid" : 6114 "MAC"); 6115 for (i = 0; i < nitems(e->u.acl.val); i++) 6116 log(LOG_ERR, " %02x", e->u.acl.val[i]); 6117 log(LOG_ERR, "\n"); 6118 break; 6119 default: 6120 log(LOG_ERR, "type %#x\n", 6121 G_FW_ERROR_CMD_TYPE(be32toh(e->op_to_type))); 6122 return (EINVAL); 6123 } 6124 return (0); 6125 } 6126 6127 static int 6128 sysctl_uint16(SYSCTL_HANDLER_ARGS) 6129 { 6130 uint16_t *id = arg1; 6131 int i = *id; 6132 6133 return sysctl_handle_int(oidp, &i, 0, req); 6134 } 6135 6136 static inline bool 6137 bufidx_used(struct adapter *sc, int idx) 6138 { 6139 struct rx_buf_info *rxb = &sc->sge.rx_buf_info[0]; 6140 int i; 6141 6142 for (i = 0; i < SW_ZONE_SIZES; i++, rxb++) { 6143 if (rxb->size1 > largest_rx_cluster) 6144 continue; 6145 if (rxb->hwidx1 == idx || rxb->hwidx2 == idx) 6146 return (true); 6147 } 6148 6149 return (false); 6150 } 6151 6152 static int 6153 sysctl_bufsizes(SYSCTL_HANDLER_ARGS) 6154 { 6155 struct adapter *sc = arg1; 6156 struct sge_params *sp = &sc->params.sge; 6157 int i, rc; 6158 struct sbuf sb; 6159 char c; 6160 6161 sbuf_new(&sb, NULL, 128, SBUF_AUTOEXTEND); 6162 for (i = 0; i < SGE_FLBUF_SIZES; i++) { 6163 if (bufidx_used(sc, i)) 6164 c = '*'; 6165 else 6166 c = '\0'; 6167 6168 sbuf_printf(&sb, "%u%c ", sp->sge_fl_buffer_size[i], c); 6169 } 6170 sbuf_trim(&sb); 6171 sbuf_finish(&sb); 6172 rc = sysctl_handle_string(oidp, sbuf_data(&sb), sbuf_len(&sb), req); 6173 sbuf_delete(&sb); 6174 return (rc); 6175 } 6176 6177 #ifdef RATELIMIT 6178 /* 6179 * len16 for a txpkt WR with a GL. Includes the firmware work request header. 6180 */ 6181 static inline u_int 6182 txpkt_eo_len16(u_int nsegs, u_int immhdrs, u_int tso) 6183 { 6184 u_int n; 6185 6186 MPASS(immhdrs > 0); 6187 6188 n = roundup2(sizeof(struct fw_eth_tx_eo_wr) + 6189 sizeof(struct cpl_tx_pkt_core) + immhdrs, 16); 6190 if (__predict_false(nsegs == 0)) 6191 goto done; 6192 6193 nsegs--; /* first segment is part of ulptx_sgl */ 6194 n += sizeof(struct ulptx_sgl) + 8 * ((3 * nsegs) / 2 + (nsegs & 1)); 6195 if (tso) 6196 n += sizeof(struct cpl_tx_pkt_lso_core); 6197 6198 done: 6199 return (howmany(n, 16)); 6200 } 6201 6202 #define ETID_FLOWC_NPARAMS 6 6203 #define ETID_FLOWC_LEN (roundup2((sizeof(struct fw_flowc_wr) + \ 6204 ETID_FLOWC_NPARAMS * sizeof(struct fw_flowc_mnemval)), 16)) 6205 #define ETID_FLOWC_LEN16 (howmany(ETID_FLOWC_LEN, 16)) 6206 6207 static int 6208 send_etid_flowc_wr(struct cxgbe_rate_tag *cst, struct port_info *pi, 6209 struct vi_info *vi) 6210 { 6211 struct wrq_cookie cookie; 6212 u_int pfvf = pi->adapter->pf << S_FW_VIID_PFN; 6213 struct fw_flowc_wr *flowc; 6214 6215 mtx_assert(&cst->lock, MA_OWNED); 6216 MPASS((cst->flags & (EO_FLOWC_PENDING | EO_FLOWC_RPL_PENDING)) == 6217 EO_FLOWC_PENDING); 6218 6219 flowc = start_wrq_wr(cst->eo_txq, ETID_FLOWC_LEN16, &cookie); 6220 if (__predict_false(flowc == NULL)) 6221 return (ENOMEM); 6222 6223 bzero(flowc, ETID_FLOWC_LEN); 6224 flowc->op_to_nparams = htobe32(V_FW_WR_OP(FW_FLOWC_WR) | 6225 V_FW_FLOWC_WR_NPARAMS(ETID_FLOWC_NPARAMS) | V_FW_WR_COMPL(0)); 6226 flowc->flowid_len16 = htonl(V_FW_WR_LEN16(ETID_FLOWC_LEN16) | 6227 V_FW_WR_FLOWID(cst->etid)); 6228 flowc->mnemval[0].mnemonic = FW_FLOWC_MNEM_PFNVFN; 6229 flowc->mnemval[0].val = htobe32(pfvf); 6230 flowc->mnemval[1].mnemonic = FW_FLOWC_MNEM_CH; 6231 flowc->mnemval[1].val = htobe32(pi->tx_chan); 6232 flowc->mnemval[2].mnemonic = FW_FLOWC_MNEM_PORT; 6233 flowc->mnemval[2].val = htobe32(pi->tx_chan); 6234 flowc->mnemval[3].mnemonic = FW_FLOWC_MNEM_IQID; 6235 flowc->mnemval[3].val = htobe32(cst->iqid); 6236 flowc->mnemval[4].mnemonic = FW_FLOWC_MNEM_EOSTATE; 6237 flowc->mnemval[4].val = htobe32(FW_FLOWC_MNEM_EOSTATE_ESTABLISHED); 6238 flowc->mnemval[5].mnemonic = FW_FLOWC_MNEM_SCHEDCLASS; 6239 flowc->mnemval[5].val = htobe32(cst->schedcl); 6240 6241 commit_wrq_wr(cst->eo_txq, flowc, &cookie); 6242 6243 cst->flags &= ~EO_FLOWC_PENDING; 6244 cst->flags |= EO_FLOWC_RPL_PENDING; 6245 MPASS(cst->tx_credits >= ETID_FLOWC_LEN16); /* flowc is first WR. */ 6246 cst->tx_credits -= ETID_FLOWC_LEN16; 6247 6248 return (0); 6249 } 6250 6251 #define ETID_FLUSH_LEN16 (howmany(sizeof (struct fw_flowc_wr), 16)) 6252 6253 void 6254 send_etid_flush_wr(struct cxgbe_rate_tag *cst) 6255 { 6256 struct fw_flowc_wr *flowc; 6257 struct wrq_cookie cookie; 6258 6259 mtx_assert(&cst->lock, MA_OWNED); 6260 6261 flowc = start_wrq_wr(cst->eo_txq, ETID_FLUSH_LEN16, &cookie); 6262 if (__predict_false(flowc == NULL)) 6263 CXGBE_UNIMPLEMENTED(__func__); 6264 6265 bzero(flowc, ETID_FLUSH_LEN16 * 16); 6266 flowc->op_to_nparams = htobe32(V_FW_WR_OP(FW_FLOWC_WR) | 6267 V_FW_FLOWC_WR_NPARAMS(0) | F_FW_WR_COMPL); 6268 flowc->flowid_len16 = htobe32(V_FW_WR_LEN16(ETID_FLUSH_LEN16) | 6269 V_FW_WR_FLOWID(cst->etid)); 6270 6271 commit_wrq_wr(cst->eo_txq, flowc, &cookie); 6272 6273 cst->flags |= EO_FLUSH_RPL_PENDING; 6274 MPASS(cst->tx_credits >= ETID_FLUSH_LEN16); 6275 cst->tx_credits -= ETID_FLUSH_LEN16; 6276 cst->ncompl++; 6277 } 6278 6279 static void 6280 write_ethofld_wr(struct cxgbe_rate_tag *cst, struct fw_eth_tx_eo_wr *wr, 6281 struct mbuf *m0, int compl) 6282 { 6283 struct cpl_tx_pkt_core *cpl; 6284 uint64_t ctrl1; 6285 uint32_t ctrl; /* used in many unrelated places */ 6286 int len16, pktlen, nsegs, immhdrs; 6287 caddr_t dst; 6288 uintptr_t p; 6289 struct ulptx_sgl *usgl; 6290 struct sglist sg; 6291 struct sglist_seg segs[38]; /* XXX: find real limit. XXX: get off the stack */ 6292 6293 mtx_assert(&cst->lock, MA_OWNED); 6294 M_ASSERTPKTHDR(m0); 6295 KASSERT(m0->m_pkthdr.l2hlen > 0 && m0->m_pkthdr.l3hlen > 0 && 6296 m0->m_pkthdr.l4hlen > 0, 6297 ("%s: ethofld mbuf %p is missing header lengths", __func__, m0)); 6298 6299 len16 = mbuf_eo_len16(m0); 6300 nsegs = mbuf_eo_nsegs(m0); 6301 pktlen = m0->m_pkthdr.len; 6302 ctrl = sizeof(struct cpl_tx_pkt_core); 6303 if (needs_tso(m0)) 6304 ctrl += sizeof(struct cpl_tx_pkt_lso_core); 6305 immhdrs = m0->m_pkthdr.l2hlen + m0->m_pkthdr.l3hlen + m0->m_pkthdr.l4hlen; 6306 ctrl += immhdrs; 6307 6308 wr->op_immdlen = htobe32(V_FW_WR_OP(FW_ETH_TX_EO_WR) | 6309 V_FW_ETH_TX_EO_WR_IMMDLEN(ctrl) | V_FW_WR_COMPL(!!compl)); 6310 wr->equiq_to_len16 = htobe32(V_FW_WR_LEN16(len16) | 6311 V_FW_WR_FLOWID(cst->etid)); 6312 wr->r3 = 0; 6313 if (needs_outer_udp_csum(m0)) { 6314 wr->u.udpseg.type = FW_ETH_TX_EO_TYPE_UDPSEG; 6315 wr->u.udpseg.ethlen = m0->m_pkthdr.l2hlen; 6316 wr->u.udpseg.iplen = htobe16(m0->m_pkthdr.l3hlen); 6317 wr->u.udpseg.udplen = m0->m_pkthdr.l4hlen; 6318 wr->u.udpseg.rtplen = 0; 6319 wr->u.udpseg.r4 = 0; 6320 wr->u.udpseg.mss = htobe16(pktlen - immhdrs); 6321 wr->u.udpseg.schedpktsize = wr->u.udpseg.mss; 6322 wr->u.udpseg.plen = htobe32(pktlen - immhdrs); 6323 cpl = (void *)(wr + 1); 6324 } else { 6325 MPASS(needs_outer_tcp_csum(m0)); 6326 wr->u.tcpseg.type = FW_ETH_TX_EO_TYPE_TCPSEG; 6327 wr->u.tcpseg.ethlen = m0->m_pkthdr.l2hlen; 6328 wr->u.tcpseg.iplen = htobe16(m0->m_pkthdr.l3hlen); 6329 wr->u.tcpseg.tcplen = m0->m_pkthdr.l4hlen; 6330 wr->u.tcpseg.tsclk_tsoff = mbuf_eo_tsclk_tsoff(m0); 6331 wr->u.tcpseg.r4 = 0; 6332 wr->u.tcpseg.r5 = 0; 6333 wr->u.tcpseg.plen = htobe32(pktlen - immhdrs); 6334 6335 if (needs_tso(m0)) { 6336 struct cpl_tx_pkt_lso_core *lso = (void *)(wr + 1); 6337 6338 wr->u.tcpseg.mss = htobe16(m0->m_pkthdr.tso_segsz); 6339 6340 ctrl = V_LSO_OPCODE(CPL_TX_PKT_LSO) | 6341 F_LSO_FIRST_SLICE | F_LSO_LAST_SLICE | 6342 V_LSO_ETHHDR_LEN((m0->m_pkthdr.l2hlen - 6343 ETHER_HDR_LEN) >> 2) | 6344 V_LSO_IPHDR_LEN(m0->m_pkthdr.l3hlen >> 2) | 6345 V_LSO_TCPHDR_LEN(m0->m_pkthdr.l4hlen >> 2); 6346 if (m0->m_pkthdr.l3hlen == sizeof(struct ip6_hdr)) 6347 ctrl |= F_LSO_IPV6; 6348 lso->lso_ctrl = htobe32(ctrl); 6349 lso->ipid_ofst = htobe16(0); 6350 lso->mss = htobe16(m0->m_pkthdr.tso_segsz); 6351 lso->seqno_offset = htobe32(0); 6352 lso->len = htobe32(pktlen); 6353 6354 cpl = (void *)(lso + 1); 6355 } else { 6356 wr->u.tcpseg.mss = htobe16(0xffff); 6357 cpl = (void *)(wr + 1); 6358 } 6359 } 6360 6361 /* Checksum offload must be requested for ethofld. */ 6362 MPASS(needs_outer_l4_csum(m0)); 6363 ctrl1 = csum_to_ctrl(cst->adapter, m0); 6364 6365 /* VLAN tag insertion */ 6366 if (needs_vlan_insertion(m0)) { 6367 ctrl1 |= F_TXPKT_VLAN_VLD | 6368 V_TXPKT_VLAN(m0->m_pkthdr.ether_vtag); 6369 } 6370 6371 /* CPL header */ 6372 cpl->ctrl0 = cst->ctrl0; 6373 cpl->pack = 0; 6374 cpl->len = htobe16(pktlen); 6375 cpl->ctrl1 = htobe64(ctrl1); 6376 6377 /* Copy Ethernet, IP & TCP/UDP hdrs as immediate data */ 6378 p = (uintptr_t)(cpl + 1); 6379 m_copydata(m0, 0, immhdrs, (void *)p); 6380 6381 /* SGL */ 6382 dst = (void *)(cpl + 1); 6383 if (nsegs > 0) { 6384 int i, pad; 6385 6386 /* zero-pad upto next 16Byte boundary, if not 16Byte aligned */ 6387 p += immhdrs; 6388 pad = 16 - (immhdrs & 0xf); 6389 bzero((void *)p, pad); 6390 6391 usgl = (void *)(p + pad); 6392 usgl->cmd_nsge = htobe32(V_ULPTX_CMD(ULP_TX_SC_DSGL) | 6393 V_ULPTX_NSGE(nsegs)); 6394 6395 sglist_init(&sg, nitems(segs), segs); 6396 for (; m0 != NULL; m0 = m0->m_next) { 6397 if (__predict_false(m0->m_len == 0)) 6398 continue; 6399 if (immhdrs >= m0->m_len) { 6400 immhdrs -= m0->m_len; 6401 continue; 6402 } 6403 if (m0->m_flags & M_EXTPG) 6404 sglist_append_mbuf_epg(&sg, m0, 6405 mtod(m0, vm_offset_t), m0->m_len); 6406 else 6407 sglist_append(&sg, mtod(m0, char *) + immhdrs, 6408 m0->m_len - immhdrs); 6409 immhdrs = 0; 6410 } 6411 MPASS(sg.sg_nseg == nsegs); 6412 6413 /* 6414 * Zero pad last 8B in case the WR doesn't end on a 16B 6415 * boundary. 6416 */ 6417 *(uint64_t *)((char *)wr + len16 * 16 - 8) = 0; 6418 6419 usgl->len0 = htobe32(segs[0].ss_len); 6420 usgl->addr0 = htobe64(segs[0].ss_paddr); 6421 for (i = 0; i < nsegs - 1; i++) { 6422 usgl->sge[i / 2].len[i & 1] = htobe32(segs[i + 1].ss_len); 6423 usgl->sge[i / 2].addr[i & 1] = htobe64(segs[i + 1].ss_paddr); 6424 } 6425 if (i & 1) 6426 usgl->sge[i / 2].len[1] = htobe32(0); 6427 } 6428 6429 } 6430 6431 static void 6432 ethofld_tx(struct cxgbe_rate_tag *cst) 6433 { 6434 struct mbuf *m; 6435 struct wrq_cookie cookie; 6436 int next_credits, compl; 6437 struct fw_eth_tx_eo_wr *wr; 6438 6439 mtx_assert(&cst->lock, MA_OWNED); 6440 6441 while ((m = mbufq_first(&cst->pending_tx)) != NULL) { 6442 M_ASSERTPKTHDR(m); 6443 6444 /* How many len16 credits do we need to send this mbuf. */ 6445 next_credits = mbuf_eo_len16(m); 6446 MPASS(next_credits > 0); 6447 if (next_credits > cst->tx_credits) { 6448 /* 6449 * Tx will make progress eventually because there is at 6450 * least one outstanding fw4_ack that will return 6451 * credits and kick the tx. 6452 */ 6453 MPASS(cst->ncompl > 0); 6454 return; 6455 } 6456 wr = start_wrq_wr(cst->eo_txq, next_credits, &cookie); 6457 if (__predict_false(wr == NULL)) { 6458 /* XXX: wishful thinking, not a real assertion. */ 6459 MPASS(cst->ncompl > 0); 6460 return; 6461 } 6462 cst->tx_credits -= next_credits; 6463 cst->tx_nocompl += next_credits; 6464 compl = cst->ncompl == 0 || cst->tx_nocompl >= cst->tx_total / 2; 6465 ETHER_BPF_MTAP(cst->com.com.ifp, m); 6466 write_ethofld_wr(cst, wr, m, compl); 6467 commit_wrq_wr(cst->eo_txq, wr, &cookie); 6468 if (compl) { 6469 cst->ncompl++; 6470 cst->tx_nocompl = 0; 6471 } 6472 (void) mbufq_dequeue(&cst->pending_tx); 6473 6474 /* 6475 * Drop the mbuf's reference on the tag now rather 6476 * than waiting until m_freem(). This ensures that 6477 * cxgbe_rate_tag_free gets called when the inp drops 6478 * its reference on the tag and there are no more 6479 * mbufs in the pending_tx queue and can flush any 6480 * pending requests. Otherwise if the last mbuf 6481 * doesn't request a completion the etid will never be 6482 * released. 6483 */ 6484 m->m_pkthdr.snd_tag = NULL; 6485 m->m_pkthdr.csum_flags &= ~CSUM_SND_TAG; 6486 m_snd_tag_rele(&cst->com.com); 6487 6488 mbufq_enqueue(&cst->pending_fwack, m); 6489 } 6490 } 6491 6492 int 6493 ethofld_transmit(struct ifnet *ifp, struct mbuf *m0) 6494 { 6495 struct cxgbe_rate_tag *cst; 6496 int rc; 6497 6498 MPASS(m0->m_nextpkt == NULL); 6499 MPASS(m0->m_pkthdr.csum_flags & CSUM_SND_TAG); 6500 MPASS(m0->m_pkthdr.snd_tag != NULL); 6501 cst = mst_to_crt(m0->m_pkthdr.snd_tag); 6502 6503 mtx_lock(&cst->lock); 6504 MPASS(cst->flags & EO_SND_TAG_REF); 6505 6506 if (__predict_false(cst->flags & EO_FLOWC_PENDING)) { 6507 struct vi_info *vi = ifp->if_softc; 6508 struct port_info *pi = vi->pi; 6509 struct adapter *sc = pi->adapter; 6510 const uint32_t rss_mask = vi->rss_size - 1; 6511 uint32_t rss_hash; 6512 6513 cst->eo_txq = &sc->sge.ofld_txq[vi->first_ofld_txq]; 6514 if (M_HASHTYPE_ISHASH(m0)) 6515 rss_hash = m0->m_pkthdr.flowid; 6516 else 6517 rss_hash = arc4random(); 6518 /* We assume RSS hashing */ 6519 cst->iqid = vi->rss[rss_hash & rss_mask]; 6520 cst->eo_txq += rss_hash % vi->nofldtxq; 6521 rc = send_etid_flowc_wr(cst, pi, vi); 6522 if (rc != 0) 6523 goto done; 6524 } 6525 6526 if (__predict_false(cst->plen + m0->m_pkthdr.len > eo_max_backlog)) { 6527 rc = ENOBUFS; 6528 goto done; 6529 } 6530 6531 mbufq_enqueue(&cst->pending_tx, m0); 6532 cst->plen += m0->m_pkthdr.len; 6533 6534 /* 6535 * Hold an extra reference on the tag while generating work 6536 * requests to ensure that we don't try to free the tag during 6537 * ethofld_tx() in case we are sending the final mbuf after 6538 * the inp was freed. 6539 */ 6540 m_snd_tag_ref(&cst->com.com); 6541 ethofld_tx(cst); 6542 mtx_unlock(&cst->lock); 6543 m_snd_tag_rele(&cst->com.com); 6544 return (0); 6545 6546 done: 6547 mtx_unlock(&cst->lock); 6548 if (__predict_false(rc != 0)) 6549 m_freem(m0); 6550 return (rc); 6551 } 6552 6553 static int 6554 ethofld_fw4_ack(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m0) 6555 { 6556 struct adapter *sc = iq->adapter; 6557 const struct cpl_fw4_ack *cpl = (const void *)(rss + 1); 6558 struct mbuf *m; 6559 u_int etid = G_CPL_FW4_ACK_FLOWID(be32toh(OPCODE_TID(cpl))); 6560 struct cxgbe_rate_tag *cst; 6561 uint8_t credits = cpl->credits; 6562 6563 cst = lookup_etid(sc, etid); 6564 mtx_lock(&cst->lock); 6565 if (__predict_false(cst->flags & EO_FLOWC_RPL_PENDING)) { 6566 MPASS(credits >= ETID_FLOWC_LEN16); 6567 credits -= ETID_FLOWC_LEN16; 6568 cst->flags &= ~EO_FLOWC_RPL_PENDING; 6569 } 6570 6571 KASSERT(cst->ncompl > 0, 6572 ("%s: etid %u (%p) wasn't expecting completion.", 6573 __func__, etid, cst)); 6574 cst->ncompl--; 6575 6576 while (credits > 0) { 6577 m = mbufq_dequeue(&cst->pending_fwack); 6578 if (__predict_false(m == NULL)) { 6579 /* 6580 * The remaining credits are for the final flush that 6581 * was issued when the tag was freed by the kernel. 6582 */ 6583 MPASS((cst->flags & 6584 (EO_FLUSH_RPL_PENDING | EO_SND_TAG_REF)) == 6585 EO_FLUSH_RPL_PENDING); 6586 MPASS(credits == ETID_FLUSH_LEN16); 6587 MPASS(cst->tx_credits + cpl->credits == cst->tx_total); 6588 MPASS(cst->ncompl == 0); 6589 6590 cst->flags &= ~EO_FLUSH_RPL_PENDING; 6591 cst->tx_credits += cpl->credits; 6592 cxgbe_rate_tag_free_locked(cst); 6593 return (0); /* cst is gone. */ 6594 } 6595 KASSERT(m != NULL, 6596 ("%s: too many credits (%u, %u)", __func__, cpl->credits, 6597 credits)); 6598 KASSERT(credits >= mbuf_eo_len16(m), 6599 ("%s: too few credits (%u, %u, %u)", __func__, 6600 cpl->credits, credits, mbuf_eo_len16(m))); 6601 credits -= mbuf_eo_len16(m); 6602 cst->plen -= m->m_pkthdr.len; 6603 m_freem(m); 6604 } 6605 6606 cst->tx_credits += cpl->credits; 6607 MPASS(cst->tx_credits <= cst->tx_total); 6608 6609 if (cst->flags & EO_SND_TAG_REF) { 6610 /* 6611 * As with ethofld_transmit(), hold an extra reference 6612 * so that the tag is stable across ethold_tx(). 6613 */ 6614 m_snd_tag_ref(&cst->com.com); 6615 m = mbufq_first(&cst->pending_tx); 6616 if (m != NULL && cst->tx_credits >= mbuf_eo_len16(m)) 6617 ethofld_tx(cst); 6618 mtx_unlock(&cst->lock); 6619 m_snd_tag_rele(&cst->com.com); 6620 } else { 6621 /* 6622 * There shouldn't be any pending packets if the tag 6623 * was freed by the kernel since any pending packet 6624 * should hold a reference to the tag. 6625 */ 6626 MPASS(mbufq_first(&cst->pending_tx) == NULL); 6627 mtx_unlock(&cst->lock); 6628 } 6629 6630 return (0); 6631 } 6632 #endif 6633