1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause 3 * 4 * Copyright (c) 2021-2026 Dmitry Salychev 5 * Copyright (c) 2022 Mathew McBride 6 * Copyright (c) 2026 Bjoern A. Zeeb 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27 * SUCH DAMAGE. 28 */ 29 30 #include <sys/cdefs.h> 31 /* 32 * The DPAA2 Network Interface (DPNI) driver. 33 * 34 * The DPNI object is a network interface that is configurable to support a wide 35 * range of features from a very basic Ethernet interface up to a 36 * high-functioning network interface. The DPNI supports features that are 37 * expected by standard network stacks, from basic features to offloads. 38 * 39 * DPNIs work with Ethernet traffic, starting with the L2 header. Additional 40 * functions are provided for standard network protocols (L2, L3, L4, etc.). 41 */ 42 43 #include <sys/param.h> 44 #include <sys/systm.h> 45 #include <sys/kernel.h> 46 #include <sys/bus.h> 47 #include <sys/rman.h> 48 #include <sys/module.h> 49 #include <sys/malloc.h> 50 #include <sys/mutex.h> 51 #include <sys/socket.h> 52 #include <sys/sockio.h> 53 #include <sys/sysctl.h> 54 #include <sys/mbuf.h> 55 #include <sys/taskqueue.h> 56 #include <sys/sysctl.h> 57 #include <sys/buf_ring.h> 58 #include <sys/smp.h> 59 #include <sys/proc.h> 60 #include <sys/sbuf.h> 61 62 #include <vm/vm.h> 63 #include <vm/pmap.h> 64 65 #include <machine/bus.h> 66 #include <machine/resource.h> 67 #include <machine/atomic.h> 68 #include <machine/vmparam.h> 69 70 #include <net/ethernet.h> 71 #include <net/bpf.h> 72 #include <net/if.h> 73 #include <net/if_dl.h> 74 #include <net/if_media.h> 75 #include <net/if_types.h> 76 #include <net/if_var.h> 77 78 #include <dev/pci/pcivar.h> 79 #include <dev/mii/mii.h> 80 #include <dev/mii/miivar.h> 81 #include <dev/mdio/mdio.h> 82 83 #include "opt_acpi.h" 84 #include "opt_platform.h" 85 86 #include "pcib_if.h" 87 #include "pci_if.h" 88 #include "miibus_if.h" 89 #include "memac_mdio_if.h" 90 91 #include "dpaa2_types.h" 92 #include "dpaa2_mc.h" 93 #include "dpaa2_mc_if.h" 94 #include "dpaa2_mcp.h" 95 #include "dpaa2_swp.h" 96 #include "dpaa2_swp_if.h" 97 #include "dpaa2_cmd_if.h" 98 #include "dpaa2_ni.h" 99 #include "dpaa2_channel.h" 100 #include "dpaa2_buf.h" 101 #include "dpaa2_frame.h" 102 103 #define BIT(x) (1ul << (x)) 104 #define WRIOP_VERSION(x, y, z) ((x) << 10 | (y) << 5 | (z) << 0) 105 #define ARRAY_SIZE(a) (sizeof(a) / sizeof((a)[0])) 106 107 /* Frame Dequeue Response status bits. */ 108 #define IS_NULL_RESPONSE(stat) ((((stat) >> 4) & 1) == 0) 109 110 #define ALIGN_UP(x, y) roundup2((x), (y)) 111 #define ALIGN_DOWN(x, y) rounddown2((x), (y)) 112 #define CACHE_LINE_ALIGN(x) ALIGN_UP((x), CACHE_LINE_SIZE) 113 114 #define DPNI_LOCK(__sc) do { \ 115 mtx_assert(&(__sc)->lock, MA_NOTOWNED); \ 116 mtx_lock(&(__sc)->lock); \ 117 } while (0) 118 #define DPNI_UNLOCK(__sc) do { \ 119 mtx_assert(&(__sc)->lock, MA_OWNED); \ 120 mtx_unlock(&(__sc)->lock); \ 121 } while (0) 122 #define DPNI_LOCK_ASSERT(__sc) do { \ 123 mtx_assert(&(__sc)->lock, MA_OWNED); \ 124 } while (0) 125 126 #define DPAA2_TX_RING(sc, chan, tc) \ 127 (&(sc)->channels[(chan)]->txc_queue.tx_rings[(tc)]) 128 129 MALLOC_DEFINE(M_DPAA2_TXB, "dpaa2_txb", "DPAA2 DMA-mapped buffer (Tx)"); 130 131 /* 132 * Minimum and maximum valid values for the cleanup sysctls. 133 */ 134 #define DPAA2_CLEAN_BUDGET_MIN 8 135 #define DPAA2_CLEAN_BUDGET_MAX 2048 136 137 #define DPNI_IRQ_INDEX 0 /* Index of the only DPNI IRQ. */ 138 #define DPNI_IRQ_LINK_CHANGED 1 /* Link state changed */ 139 #define DPNI_IRQ_EP_CHANGED 2 /* DPAA2 endpoint dis/connected */ 140 141 /* Default maximum RX frame length w/o CRC. */ 142 #define DPAA2_ETH_MFL (ETHER_MAX_LEN_JUMBO + ETHER_VLAN_ENCAP_LEN - \ 143 ETHER_CRC_LEN) 144 145 /* Minimally supported version of the DPNI API. */ 146 #define DPNI_VER_MAJOR 7 147 #define DPNI_VER_MINOR 0 148 149 /* Rx/Tx buffers configuration. */ 150 #define BUF_ALIGN_V1 256 /* WRIOP v1.0.0 limitation */ 151 #define BUF_ALIGN 64 152 #define BUF_SWA_SIZE 64 /* SW annotation size */ 153 #define BUF_RX_HWA_SIZE 64 /* HW annotation size */ 154 #define BUF_TX_HWA_SIZE 128 /* HW annotation size */ 155 156 #define DPAA2_RX_BUFRING_SZ (4096u) 157 #define DPAA2_RXE_BUFRING_SZ (1024u) 158 #define DPAA2_TXC_BUFRING_SZ (4096u) 159 160 /* Size of a buffer to keep a QoS table key configuration. */ 161 #define ETH_QOS_KCFG_BUF_SIZE (PAGE_SIZE) 162 163 /* Required by struct dpni_rx_tc_dist_cfg::key_cfg_iova */ 164 #define DPAA2_CLASSIFIER_DMA_SIZE (PAGE_SIZE) 165 166 /* Buffers layout options. */ 167 #define BUF_LOPT_TIMESTAMP 0x1 168 #define BUF_LOPT_PARSER_RESULT 0x2 169 #define BUF_LOPT_FRAME_STATUS 0x4 170 #define BUF_LOPT_PRIV_DATA_SZ 0x8 171 #define BUF_LOPT_DATA_ALIGN 0x10 172 #define BUF_LOPT_DATA_HEAD_ROOM 0x20 173 #define BUF_LOPT_DATA_TAIL_ROOM 0x40 174 175 #define DPAA2_NI_BUF_ADDR_MASK (0x1FFFFFFFFFFFFul) /* 49-bit addresses max. */ 176 #define DPAA2_NI_BUF_CHAN_MASK (0xFu) 177 #define DPAA2_NI_BUF_CHAN_SHIFT (60) 178 #define DPAA2_NI_BUF_IDX_MASK (0x7FFFu) 179 #define DPAA2_NI_BUF_IDX_SHIFT (49) 180 #define DPAA2_NI_TX_IDX_MASK (0x7u) 181 #define DPAA2_NI_TX_IDX_SHIFT (57) 182 #define DPAA2_NI_TXBUF_IDX_MASK (0xFFu) 183 #define DPAA2_NI_TXBUF_IDX_SHIFT (49) 184 185 /* Enables TCAM for Flow Steering and QoS look-ups. */ 186 #define DPNI_OPT_HAS_KEY_MASKING 0x10 187 188 /* Unique IDs for the supported Rx classification header fields. */ 189 #define DPAA2_ETH_DIST_ETHDST BIT(0) 190 #define DPAA2_ETH_DIST_ETHSRC BIT(1) 191 #define DPAA2_ETH_DIST_ETHTYPE BIT(2) 192 #define DPAA2_ETH_DIST_VLAN BIT(3) 193 #define DPAA2_ETH_DIST_IPSRC BIT(4) 194 #define DPAA2_ETH_DIST_IPDST BIT(5) 195 #define DPAA2_ETH_DIST_IPPROTO BIT(6) 196 #define DPAA2_ETH_DIST_L4SRC BIT(7) 197 #define DPAA2_ETH_DIST_L4DST BIT(8) 198 #define DPAA2_ETH_DIST_ALL (~0ULL) 199 200 /* L3-L4 network traffic flow hash options. */ 201 #define RXH_L2DA (1 << 1) 202 #define RXH_VLAN (1 << 2) 203 #define RXH_L3_PROTO (1 << 3) 204 #define RXH_IP_SRC (1 << 4) 205 #define RXH_IP_DST (1 << 5) 206 #define RXH_L4_B_0_1 (1 << 6) /* src port in case of TCP/UDP/SCTP */ 207 #define RXH_L4_B_2_3 (1 << 7) /* dst port in case of TCP/UDP/SCTP */ 208 #define RXH_DISCARD (1 << 31) 209 210 /* Transmit checksum offload */ 211 #define DPAA2_CSUM_TX_OFFLOAD (CSUM_IP | CSUM_DELAY_DATA | CSUM_DELAY_DATA_IPV6) 212 213 /* Default Rx hash options, set during attaching. */ 214 #define DPAA2_RXH_DEFAULT (RXH_IP_SRC | RXH_IP_DST | RXH_L4_B_0_1 | RXH_L4_B_2_3) 215 216 MALLOC_DEFINE(M_DPAA2_NI, "dpaa2_ni", "DPAA2 Network Interface"); 217 218 /* 219 * DPAA2 Network Interface resource specification. 220 * 221 * NOTE: Don't forget to update macros in dpaa2_ni.h in case of any changes in 222 * the specification! 223 */ 224 struct resource_spec dpaa2_ni_spec[] = { 225 /* 226 * DPMCP resources. 227 * 228 * NOTE: MC command portals (MCPs) are used to send commands to, and 229 * receive responses from, the MC firmware. One portal per DPNI. 230 */ 231 { DPAA2_DEV_MCP, DPAA2_NI_MCP_RID(0), RF_ACTIVE | RF_SHAREABLE | RF_OPTIONAL }, 232 /* 233 * DPIO resources (software portals). 234 * 235 * NOTE: One per running core. While DPIOs are the source of data 236 * availability interrupts, the DPCONs are used to identify the 237 * network interface that has produced ingress data to that core. 238 */ 239 { DPAA2_DEV_IO, DPAA2_NI_IO_RID(0), RF_ACTIVE | RF_SHAREABLE }, 240 { DPAA2_DEV_IO, DPAA2_NI_IO_RID(1), RF_ACTIVE | RF_SHAREABLE | RF_OPTIONAL }, 241 { DPAA2_DEV_IO, DPAA2_NI_IO_RID(2), RF_ACTIVE | RF_SHAREABLE | RF_OPTIONAL }, 242 { DPAA2_DEV_IO, DPAA2_NI_IO_RID(3), RF_ACTIVE | RF_SHAREABLE | RF_OPTIONAL }, 243 { DPAA2_DEV_IO, DPAA2_NI_IO_RID(4), RF_ACTIVE | RF_SHAREABLE | RF_OPTIONAL }, 244 { DPAA2_DEV_IO, DPAA2_NI_IO_RID(5), RF_ACTIVE | RF_SHAREABLE | RF_OPTIONAL }, 245 { DPAA2_DEV_IO, DPAA2_NI_IO_RID(6), RF_ACTIVE | RF_SHAREABLE | RF_OPTIONAL }, 246 { DPAA2_DEV_IO, DPAA2_NI_IO_RID(7), RF_ACTIVE | RF_SHAREABLE | RF_OPTIONAL }, 247 { DPAA2_DEV_IO, DPAA2_NI_IO_RID(8), RF_ACTIVE | RF_SHAREABLE | RF_OPTIONAL }, 248 { DPAA2_DEV_IO, DPAA2_NI_IO_RID(9), RF_ACTIVE | RF_SHAREABLE | RF_OPTIONAL }, 249 { DPAA2_DEV_IO, DPAA2_NI_IO_RID(10), RF_ACTIVE | RF_SHAREABLE | RF_OPTIONAL }, 250 { DPAA2_DEV_IO, DPAA2_NI_IO_RID(11), RF_ACTIVE | RF_SHAREABLE | RF_OPTIONAL }, 251 { DPAA2_DEV_IO, DPAA2_NI_IO_RID(12), RF_ACTIVE | RF_SHAREABLE | RF_OPTIONAL }, 252 { DPAA2_DEV_IO, DPAA2_NI_IO_RID(13), RF_ACTIVE | RF_SHAREABLE | RF_OPTIONAL }, 253 { DPAA2_DEV_IO, DPAA2_NI_IO_RID(14), RF_ACTIVE | RF_SHAREABLE | RF_OPTIONAL }, 254 { DPAA2_DEV_IO, DPAA2_NI_IO_RID(15), RF_ACTIVE | RF_SHAREABLE | RF_OPTIONAL }, 255 /* 256 * DPBP resources (buffer pools). 257 * 258 * NOTE: One per network interface. 259 */ 260 { DPAA2_DEV_BP, DPAA2_NI_BP_RID(0), RF_ACTIVE }, 261 /* 262 * DPCON resources (channels). 263 * 264 * NOTE: One DPCON per core where Rx or Tx confirmation traffic to be 265 * distributed to. 266 * NOTE: Since it is necessary to distinguish between traffic from 267 * different network interfaces arriving on the same core, the 268 * DPCONs must be private to the DPNIs. 269 */ 270 { DPAA2_DEV_CON, DPAA2_NI_CON_RID(0), RF_ACTIVE }, 271 { DPAA2_DEV_CON, DPAA2_NI_CON_RID(1), RF_ACTIVE | RF_OPTIONAL }, 272 { DPAA2_DEV_CON, DPAA2_NI_CON_RID(2), RF_ACTIVE | RF_OPTIONAL }, 273 { DPAA2_DEV_CON, DPAA2_NI_CON_RID(3), RF_ACTIVE | RF_OPTIONAL }, 274 { DPAA2_DEV_CON, DPAA2_NI_CON_RID(4), RF_ACTIVE | RF_OPTIONAL }, 275 { DPAA2_DEV_CON, DPAA2_NI_CON_RID(5), RF_ACTIVE | RF_OPTIONAL }, 276 { DPAA2_DEV_CON, DPAA2_NI_CON_RID(6), RF_ACTIVE | RF_OPTIONAL }, 277 { DPAA2_DEV_CON, DPAA2_NI_CON_RID(7), RF_ACTIVE | RF_OPTIONAL }, 278 { DPAA2_DEV_CON, DPAA2_NI_CON_RID(8), RF_ACTIVE | RF_OPTIONAL }, 279 { DPAA2_DEV_CON, DPAA2_NI_CON_RID(9), RF_ACTIVE | RF_OPTIONAL }, 280 { DPAA2_DEV_CON, DPAA2_NI_CON_RID(10), RF_ACTIVE | RF_OPTIONAL }, 281 { DPAA2_DEV_CON, DPAA2_NI_CON_RID(11), RF_ACTIVE | RF_OPTIONAL }, 282 { DPAA2_DEV_CON, DPAA2_NI_CON_RID(12), RF_ACTIVE | RF_OPTIONAL }, 283 { DPAA2_DEV_CON, DPAA2_NI_CON_RID(13), RF_ACTIVE | RF_OPTIONAL }, 284 { DPAA2_DEV_CON, DPAA2_NI_CON_RID(14), RF_ACTIVE | RF_OPTIONAL }, 285 { DPAA2_DEV_CON, DPAA2_NI_CON_RID(15), RF_ACTIVE | RF_OPTIONAL }, 286 287 RESOURCE_SPEC_END 288 }; 289 290 /* Supported header fields for Rx hash distribution key */ 291 static const struct dpaa2_eth_dist_fields dist_fields[] = { 292 { 293 /* L2 header */ 294 .rxnfc_field = RXH_L2DA, 295 .cls_prot = NET_PROT_ETH, 296 .cls_field = NH_FLD_ETH_DA, 297 .id = DPAA2_ETH_DIST_ETHDST, 298 .size = 6, 299 }, { 300 .cls_prot = NET_PROT_ETH, 301 .cls_field = NH_FLD_ETH_SA, 302 .id = DPAA2_ETH_DIST_ETHSRC, 303 .size = 6, 304 }, { 305 /* This is the last ethertype field parsed: 306 * depending on frame format, it can be the MAC ethertype 307 * or the VLAN etype. 308 */ 309 .cls_prot = NET_PROT_ETH, 310 .cls_field = NH_FLD_ETH_TYPE, 311 .id = DPAA2_ETH_DIST_ETHTYPE, 312 .size = 2, 313 }, { 314 /* VLAN header */ 315 .rxnfc_field = RXH_VLAN, 316 .cls_prot = NET_PROT_VLAN, 317 .cls_field = NH_FLD_VLAN_TCI, 318 .id = DPAA2_ETH_DIST_VLAN, 319 .size = 2, 320 }, { 321 /* IP header */ 322 .rxnfc_field = RXH_IP_SRC, 323 .cls_prot = NET_PROT_IP, 324 .cls_field = NH_FLD_IP_SRC, 325 .id = DPAA2_ETH_DIST_IPSRC, 326 .size = 4, 327 }, { 328 .rxnfc_field = RXH_IP_DST, 329 .cls_prot = NET_PROT_IP, 330 .cls_field = NH_FLD_IP_DST, 331 .id = DPAA2_ETH_DIST_IPDST, 332 .size = 4, 333 }, { 334 .rxnfc_field = RXH_L3_PROTO, 335 .cls_prot = NET_PROT_IP, 336 .cls_field = NH_FLD_IP_PROTO, 337 .id = DPAA2_ETH_DIST_IPPROTO, 338 .size = 1, 339 }, { 340 /* Using UDP ports, this is functionally equivalent to raw 341 * byte pairs from L4 header. 342 */ 343 .rxnfc_field = RXH_L4_B_0_1, 344 .cls_prot = NET_PROT_UDP, 345 .cls_field = NH_FLD_UDP_PORT_SRC, 346 .id = DPAA2_ETH_DIST_L4SRC, 347 .size = 2, 348 }, { 349 .rxnfc_field = RXH_L4_B_2_3, 350 .cls_prot = NET_PROT_UDP, 351 .cls_field = NH_FLD_UDP_PORT_DST, 352 .id = DPAA2_ETH_DIST_L4DST, 353 .size = 2, 354 }, 355 }; 356 357 static struct dpni_stat { 358 int page; 359 int cnt; 360 char *name; 361 char *desc; 362 } dpni_stat_sysctls[] = { 363 /* PAGE, COUNTER, NAME, DESCRIPTION */ 364 { 0, 0, "in_all_frames", "All accepted ingress frames" }, 365 { 0, 1, "in_all_bytes", "Bytes in all accepted ingress frames" }, 366 { 0, 2, "in_mc_frames", "Multicast accepted ingress frames" }, 367 { 0, 3, "in_mc_bytes", "Bytes in received multicast frames" }, 368 { 0, 4, "in_bc_frames", "Broadcast accepted ingress frames" }, 369 { 0, 5, "in_bc_bytes", "Bytes in broadcast multicast frames" }, 370 371 { 1, 0, "eg_all_frames", "All egress frames transmitted" }, 372 { 1, 1, "eg_all_bytes", "Bytes in all frames transmitted" }, 373 { 1, 2, "eg_mc_frames", "Multicast egress frames transmitted" }, 374 { 1, 3, "eg_mc_bytes", "Bytes in transmitted multicast frame" }, 375 { 1, 4, "eg_bc_frames", "Broadcast egress frames transmitted" }, 376 { 1, 5, "eg_bc_bytes", "Bytes in broadcast multicast frames" }, 377 378 { 2, 0, "in_filtered_frames", "All ingress frames discarded due to filtering" }, 379 { 2, 1, "in_discarded_frames", "All frames discarded due to errors" }, 380 { 2, 2, "in_nobuf_discards", "Discards on ingress side due to buffer depletion in DPNI buffer pools" }, 381 { 2, 3, "eg_frames_disc", "Frames discarded on transmit due to DPNI configuration and/or frame state" }, 382 { 2, 4, "eg_frames_tx", "Frames that have been confirmed after transmission" }, 383 384 /* XXX FIXME Page 3/4 can take a param as well not encoded here. */ 385 /* XXX 3/0 and 3/1 have the same description in the manual? Where's the difference? */ 386 { 3, 0, "bytes_dequeued", "Cumulative count of the number of bytes dequeued" }, 387 { 3, 1, "frames_dequeued", "Cumulative count of the number of frames dequeued" }, 388 { 3, 2, "bytes_enqueued_rej", "Cumulative count of the number of bytes in all frames whose enqueue was rejected." }, 389 { 3, 3, "frames_enqueued_rej", "Cumulative count of all frame enqueues rejected." }, 390 391 { 4, 0, "fames_rej_tc", "Rejected frames in associated congestion point (valid if this TC has an associated congestion point)" }, 392 { 4, 1, "bytes_rej_tc", "Rejected bytes in associated congestion point (valid if this TC has an associated congestion point)" }, 393 394 { 5, 0, "pol_red", "Policer RED packet counter. 32bit value valid only when policer is enabled." }, 395 { 5, 1, "pol_yel", "Policer YELLOW packet counter. 32bit value valid only when policer is enabled." }, 396 { 5, 2, "pol_gre", "Policer GREEN packet counter. 32bit value valid only when policer is enabled." }, 397 { 5, 3, "pol_re_red", "Policer recolored RED packet counter. 32bit value valid only when policer is enabled." }, 398 { 5, 4, "pol_re_yel", "Policer recolored YELLOW packet counter. 32bit value valid only when policer is enabled." }, 399 }; 400 401 struct dpaa2_ni_rx_ctx { 402 struct mbuf *head; 403 struct mbuf *tail; 404 int cnt; 405 bool last; 406 }; 407 408 /* Device interface */ 409 static int dpaa2_ni_probe(device_t); 410 static int dpaa2_ni_attach(device_t); 411 static int dpaa2_ni_detach(device_t); 412 413 /* DPAA2 network interface setup and configuration */ 414 static int dpaa2_ni_setup(device_t); 415 static int dpaa2_ni_setup_channels(device_t); 416 static int dpaa2_ni_bind(device_t); 417 static int dpaa2_ni_setup_rx_dist(device_t); 418 static int dpaa2_ni_setup_irqs(device_t); 419 static int dpaa2_ni_setup_msi(struct dpaa2_ni_softc *); 420 static int dpaa2_ni_setup_if_caps(struct dpaa2_ni_softc *); 421 static int dpaa2_ni_setup_if_flags(struct dpaa2_ni_softc *); 422 static int dpaa2_ni_setup_sysctls(struct dpaa2_ni_softc *); 423 static int dpaa2_ni_setup_dma(struct dpaa2_ni_softc *); 424 425 /* Tx/Rx flow configuration */ 426 static int dpaa2_ni_setup_rx_flow(device_t, struct dpaa2_ni_fq *); 427 static int dpaa2_ni_setup_tx_flow(device_t, struct dpaa2_ni_fq *); 428 static int dpaa2_ni_setup_rx_err_flow(device_t, struct dpaa2_ni_fq *); 429 430 /* Configuration subroutines */ 431 static int dpaa2_ni_set_buf_layout(device_t); 432 static int dpaa2_ni_set_pause_frame(device_t); 433 static int dpaa2_ni_set_qos_table(device_t); 434 static int dpaa2_ni_set_mac_addr(device_t); 435 static int dpaa2_ni_set_hash(device_t, uint64_t); 436 static int dpaa2_ni_set_dist_key(device_t, enum dpaa2_ni_dist_mode, uint64_t); 437 438 /* Various subroutines */ 439 static int dpaa2_ni_cmp_api_version(struct dpaa2_ni_softc *, uint16_t, uint16_t); 440 static int dpaa2_ni_prepare_key_cfg(struct dpkg_profile_cfg *, uint8_t *); 441 static int dpaa2_ni_update_csum_flags(struct dpaa2_fd *, struct mbuf *); 442 443 /* Network interface routines */ 444 static void dpaa2_ni_init(void *); 445 static int dpaa2_ni_transmit(if_t , struct mbuf *); 446 static void dpaa2_ni_qflush(if_t ); 447 static int dpaa2_ni_ioctl(if_t , u_long, caddr_t); 448 static int dpaa2_ni_update_mac_filters(if_t ); 449 static u_int dpaa2_ni_add_maddr(void *, struct sockaddr_dl *, u_int); 450 451 /* Interrupt handlers */ 452 static void dpaa2_ni_intr(void *); 453 454 /* MII handlers */ 455 static void dpaa2_ni_miibus_statchg(device_t); 456 static int dpaa2_ni_media_change(if_t ); 457 static void dpaa2_ni_media_status(if_t , struct ifmediareq *); 458 static void dpaa2_ni_media_tick(void *); 459 460 /* Tx/Rx routines. */ 461 static int dpaa2_ni_rx_cleanup(struct dpaa2_channel *, const int budget); 462 static int dpaa2_ni_tx_cleanup(struct dpaa2_channel *, const int budget); 463 static void dpaa2_ni_tx(struct dpaa2_ni_softc *, struct dpaa2_channel *, 464 struct dpaa2_ni_tx_ring *, struct mbuf *); 465 static void dpaa2_ni_cleanup_task(void *, int); 466 467 /* Tx/Rx subroutines */ 468 static int dpaa2_ni_consume_frames(struct dpaa2_channel *, struct dpaa2_ni_fq **, 469 uint32_t *); 470 static int dpaa2_ni_rx(struct dpaa2_channel *, struct dpaa2_ni_fq *, 471 struct dpaa2_fd *, struct dpaa2_ni_rx_ctx *); 472 static int dpaa2_ni_rx_err(struct dpaa2_channel *, struct dpaa2_ni_fq *, 473 struct dpaa2_fd *); 474 static int dpaa2_ni_tx_conf(struct dpaa2_channel *, struct dpaa2_ni_fq *, 475 struct dpaa2_fd *); 476 477 /* sysctl(9) */ 478 static int dpaa2_ni_collect_stats(SYSCTL_HANDLER_ARGS); 479 static int dpaa2_ni_collect_buf_num(SYSCTL_HANDLER_ARGS); 480 static int dpaa2_ni_collect_buf_free(SYSCTL_HANDLER_ARGS); 481 static int dpaa2_ni_sysctl_link_state(SYSCTL_HANDLER_ARGS); 482 static int dpaa2_ni_sysctl_handle_int(struct sysctl_req *req, 483 struct dpaa2_atomic *value); 484 static int dpaa2_ni_sysctl_handle_clean_budget(SYSCTL_HANDLER_ARGS); 485 static int dpaa2_ni_sysctl_handle_tx_budget(SYSCTL_HANDLER_ARGS); 486 static int dpaa2_ni_sysctl_handle_rx_budget(SYSCTL_HANDLER_ARGS); 487 488 static int 489 dpaa2_ni_probe(device_t dev) 490 { 491 /* DPNI device will be added by a parent resource container itself. */ 492 device_set_desc(dev, "DPAA2 Network Interface"); 493 return (BUS_PROBE_DEFAULT); 494 } 495 496 static int 497 dpaa2_ni_attach(device_t dev) 498 { 499 device_t pdev = device_get_parent(dev); 500 device_t child = dev; 501 device_t mcp_dev; 502 struct dpaa2_ni_softc *sc = device_get_softc(dev); 503 struct dpaa2_devinfo *rcinfo = device_get_ivars(pdev); 504 struct dpaa2_devinfo *dinfo = device_get_ivars(dev); 505 struct dpaa2_devinfo *mcp_dinfo; 506 struct dpaa2_cmd cmd; 507 uint16_t rc_token, ni_token; 508 if_t ifp; 509 char tq_name[32]; 510 int error; 511 512 sc->dev = dev; 513 sc->ifp = NULL; 514 sc->miibus = NULL; 515 sc->mii = NULL; 516 sc->media_status = 0; 517 sc->if_flags = 0; 518 sc->link_state = LINK_STATE_UNKNOWN; 519 sc->buf_align = 0; 520 521 /* For debug purposes only! */ 522 sc->rx_anomaly_frames = 0; 523 sc->rx_single_buf_frames = 0; 524 sc->rx_sg_buf_frames = 0; 525 sc->rx_enq_rej_frames = 0; 526 sc->rx_ieoi_err_frames = 0; 527 sc->rx_other_err_frames = 0; 528 sc->tx_single_buf_frames = 0; 529 sc->tx_sg_frames = 0; 530 531 DPAA2_ATOMIC_XCHG(&sc->buf_num, 0); 532 DPAA2_ATOMIC_XCHG(&sc->buf_free, 0); 533 DPAA2_ATOMIC_XCHG(&sc->clean_budget, 128); 534 DPAA2_ATOMIC_XCHG(&sc->tx_budget, 256); 535 DPAA2_ATOMIC_XCHG(&sc->rx_budget, 512); 536 537 sc->rxd_dmat = NULL; 538 sc->qos_dmat = NULL; 539 540 sc->qos_kcfg.dmap = NULL; 541 sc->qos_kcfg.paddr = 0; 542 sc->qos_kcfg.vaddr = NULL; 543 544 sc->rxd_kcfg.dmap = NULL; 545 sc->rxd_kcfg.paddr = 0; 546 sc->rxd_kcfg.vaddr = NULL; 547 548 sc->mac.dpmac_id = 0; 549 sc->mac.phy_dev = NULL; 550 memset(sc->mac.addr, 0, ETHER_ADDR_LEN); 551 552 error = bus_alloc_resources(sc->dev, dpaa2_ni_spec, sc->res); 553 if (error) { 554 device_printf(dev, "%s: failed to allocate resources: " 555 "error=%d\n", __func__, error); 556 goto err_exit; 557 } 558 559 /* Obtain MC portal. */ 560 mcp_dev = (device_t) rman_get_start(sc->res[DPAA2_NI_MCP_RID(0)]); 561 mcp_dinfo = device_get_ivars(mcp_dev); 562 dinfo->portal = mcp_dinfo->portal; 563 564 mtx_init(&sc->lock, device_get_nameunit(dev), "dpaa2_ni", MTX_DEF); 565 566 /* Allocate network interface */ 567 ifp = if_alloc(IFT_ETHER); 568 sc->ifp = ifp; 569 if_initname(ifp, DPAA2_NI_IFNAME, device_get_unit(sc->dev)); 570 571 if_setsoftc(ifp, sc); 572 if_setflags(ifp, IFF_SIMPLEX | IFF_MULTICAST | IFF_BROADCAST); 573 if_setinitfn(ifp, dpaa2_ni_init); 574 if_setioctlfn(ifp, dpaa2_ni_ioctl); 575 if_settransmitfn(ifp, dpaa2_ni_transmit); 576 if_setqflushfn(ifp, dpaa2_ni_qflush); 577 578 if_sethwassist(sc->ifp, DPAA2_CSUM_TX_OFFLOAD); 579 if_setcapabilities(ifp, IFCAP_VLAN_MTU | IFCAP_HWCSUM | 580 IFCAP_HWCSUM_IPV6 | IFCAP_JUMBO_MTU); 581 if_setcapenable(ifp, if_getcapabilities(ifp)); 582 583 DPAA2_CMD_INIT(&cmd); 584 585 /* Open resource container and network interface object. */ 586 error = DPAA2_CMD_RC_OPEN(dev, child, &cmd, rcinfo->id, &rc_token); 587 if (error) { 588 device_printf(dev, "%s: failed to open resource container: " 589 "id=%d, error=%d\n", __func__, rcinfo->id, error); 590 goto err_exit; 591 } 592 error = DPAA2_CMD_NI_OPEN(dev, child, &cmd, dinfo->id, &ni_token); 593 if (error) { 594 device_printf(dev, "%s: failed to open network interface: " 595 "id=%d, error=%d\n", __func__, dinfo->id, error); 596 goto close_rc; 597 } 598 599 bzero(tq_name, sizeof(tq_name)); 600 snprintf(tq_name, sizeof(tq_name), "%s_tqbp", device_get_nameunit(dev)); 601 602 /* 603 * XXX-DSL: Release new buffers on Buffer Pool State Change Notification 604 * (BPSCN) returned as a result to the VDQ command instead. 605 * It is similar to CDAN processed in dpaa2_io_intr(). 606 */ 607 /* Create a taskqueue thread to release new buffers to the pool. */ 608 sc->bp_taskq = taskqueue_create(tq_name, M_WAITOK, 609 taskqueue_thread_enqueue, &sc->bp_taskq); 610 taskqueue_start_threads(&sc->bp_taskq, 1, PI_NET, "%s", tq_name); 611 612 /* sc->cleanup_taskq = taskqueue_create("dpaa2_ch cleanup", M_WAITOK, */ 613 /* taskqueue_thread_enqueue, &sc->cleanup_taskq); */ 614 /* taskqueue_start_threads(&sc->cleanup_taskq, 1, PI_NET, */ 615 /* "dpaa2_ch cleanup"); */ 616 617 error = dpaa2_ni_setup(dev); 618 if (error) { 619 device_printf(dev, "%s: failed to setup DPNI: error=%d\n", 620 __func__, error); 621 goto close_ni; 622 } 623 error = dpaa2_ni_setup_channels(dev); 624 if (error) { 625 device_printf(dev, "%s: failed to setup QBMan channels: " 626 "error=%d\n", __func__, error); 627 goto close_ni; 628 } 629 630 error = dpaa2_ni_bind(dev); 631 if (error) { 632 device_printf(dev, "%s: failed to bind DPNI: error=%d\n", 633 __func__, error); 634 goto close_ni; 635 } 636 error = dpaa2_ni_setup_irqs(dev); 637 if (error) { 638 device_printf(dev, "%s: failed to setup IRQs: error=%d\n", 639 __func__, error); 640 goto close_ni; 641 } 642 error = dpaa2_ni_setup_sysctls(sc); 643 if (error) { 644 device_printf(dev, "%s: failed to setup sysctls: error=%d\n", 645 __func__, error); 646 goto close_ni; 647 } 648 error = dpaa2_ni_setup_if_caps(sc); 649 if (error) { 650 device_printf(dev, "%s: failed to setup interface capabilities: " 651 "error=%d\n", __func__, error); 652 goto close_ni; 653 } 654 655 ether_ifattach(sc->ifp, sc->mac.addr); 656 callout_init(&sc->mii_callout, 0); 657 658 return (0); 659 660 close_ni: 661 DPAA2_CMD_NI_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, ni_token)); 662 close_rc: 663 DPAA2_CMD_RC_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, rc_token)); 664 err_exit: 665 return (ENXIO); 666 } 667 668 static void 669 dpaa2_ni_fixed_media_status(if_t ifp, struct ifmediareq* ifmr) 670 { 671 struct dpaa2_ni_softc *sc = if_getsoftc(ifp); 672 673 DPNI_LOCK(sc); 674 ifmr->ifm_count = 0; 675 ifmr->ifm_mask = 0; 676 ifmr->ifm_status = IFM_AVALID | IFM_ACTIVE; 677 ifmr->ifm_current = ifmr->ifm_active = 678 sc->fixed_ifmedia.ifm_cur->ifm_media; 679 680 /* 681 * In non-PHY usecases, we need to signal link state up, otherwise 682 * certain things requiring a link event (e.g async DHCP client) from 683 * devd do not happen. 684 */ 685 if (if_getlinkstate(ifp) == LINK_STATE_UNKNOWN) { 686 if_link_state_change(ifp, LINK_STATE_UP); 687 } 688 689 /* 690 * TODO: Check the status of the link partner (DPMAC, DPNI or other) and 691 * reset if down. This is different to the DPAA2_MAC_LINK_TYPE_PHY as 692 * the MC firmware sets the status, instead of us telling the MC what 693 * it is. 694 */ 695 DPNI_UNLOCK(sc); 696 697 return; 698 } 699 700 static void 701 dpaa2_ni_setup_fixed_link(struct dpaa2_ni_softc *sc) 702 { 703 /* 704 * FIXME: When the DPNI is connected to a DPMAC, we can get the 705 * 'apparent' speed from it. 706 */ 707 sc->fixed_link = true; 708 709 ifmedia_init(&sc->fixed_ifmedia, 0, dpaa2_ni_media_change, 710 dpaa2_ni_fixed_media_status); 711 ifmedia_add(&sc->fixed_ifmedia, IFM_ETHER | IFM_1000_T, 0, NULL); 712 ifmedia_set(&sc->fixed_ifmedia, IFM_ETHER | IFM_1000_T); 713 } 714 715 static int 716 dpaa2_ni_detach(device_t dev) 717 { 718 /* TBD */ 719 return (0); 720 } 721 722 /** 723 * @brief Configure DPAA2 network interface object. 724 */ 725 static int 726 dpaa2_ni_setup(device_t dev) 727 { 728 device_t pdev = device_get_parent(dev); 729 device_t child = dev; 730 struct dpaa2_ni_softc *sc = device_get_softc(dev); 731 struct dpaa2_devinfo *rcinfo = device_get_ivars(pdev); 732 struct dpaa2_devinfo *dinfo = device_get_ivars(dev); 733 struct dpaa2_ep_desc ep1_desc, ep2_desc; /* endpoint descriptors */ 734 struct dpaa2_cmd cmd; 735 uint8_t eth_bca[ETHER_ADDR_LEN]; /* broadcast physical address */ 736 uint16_t rc_token, ni_token, mac_token; 737 struct dpaa2_mac_attr attr; 738 enum dpaa2_mac_link_type link_type; 739 uint32_t link; 740 int error; 741 742 DPAA2_CMD_INIT(&cmd); 743 744 error = DPAA2_CMD_RC_OPEN(dev, child, &cmd, rcinfo->id, &rc_token); 745 if (error) { 746 device_printf(dev, "%s: failed to open resource container: " 747 "id=%d, error=%d\n", __func__, rcinfo->id, error); 748 goto err_exit; 749 } 750 error = DPAA2_CMD_NI_OPEN(dev, child, &cmd, dinfo->id, &ni_token); 751 if (error) { 752 device_printf(dev, "%s: failed to open network interface: " 753 "id=%d, error=%d\n", __func__, dinfo->id, error); 754 goto close_rc; 755 } 756 757 /* Check if we can work with this DPNI object. */ 758 error = DPAA2_CMD_NI_GET_API_VERSION(dev, child, &cmd, &sc->api_major, 759 &sc->api_minor); 760 if (error) { 761 device_printf(dev, "%s: failed to get DPNI API version\n", 762 __func__); 763 goto close_ni; 764 } 765 if (dpaa2_ni_cmp_api_version(sc, DPNI_VER_MAJOR, DPNI_VER_MINOR) < 0) { 766 device_printf(dev, "%s: DPNI API version %u.%u not supported, " 767 "need >= %u.%u\n", __func__, sc->api_major, sc->api_minor, 768 DPNI_VER_MAJOR, DPNI_VER_MINOR); 769 error = ENODEV; 770 goto close_ni; 771 } 772 773 /* Reset the DPNI object. */ 774 error = DPAA2_CMD_NI_RESET(dev, child, &cmd); 775 if (error) { 776 device_printf(dev, "%s: failed to reset DPNI: id=%d\n", 777 __func__, dinfo->id); 778 goto close_ni; 779 } 780 781 /* Obtain attributes of the DPNI object. */ 782 error = DPAA2_CMD_NI_GET_ATTRIBUTES(dev, child, &cmd, &sc->attr); 783 if (error) { 784 device_printf(dev, "%s: failed to obtain DPNI attributes: " 785 "id=%d\n", __func__, dinfo->id); 786 goto close_ni; 787 } 788 if (bootverbose) { 789 device_printf(dev, "\toptions=0x%#x queues=%d tx_channels=%d " 790 "wriop_version=%#x\n", sc->attr.options, sc->attr.num.queues, 791 sc->attr.num.channels, sc->attr.wriop_ver); 792 device_printf(dev, "\ttraffic classes: rx=%d tx=%d " 793 "cgs_groups=%d\n", sc->attr.num.rx_tcs, sc->attr.num.tx_tcs, 794 sc->attr.num.cgs); 795 device_printf(dev, "\ttable entries: mac=%d vlan=%d qos=%d " 796 "fs=%d\n", sc->attr.entries.mac, sc->attr.entries.vlan, 797 sc->attr.entries.qos, sc->attr.entries.fs); 798 device_printf(dev, "\tkey sizes: qos=%d fs=%d\n", 799 sc->attr.key_size.qos, sc->attr.key_size.fs); 800 } 801 802 /* Configure buffer layouts of the DPNI queues. */ 803 error = dpaa2_ni_set_buf_layout(dev); 804 if (error) { 805 device_printf(dev, "%s: failed to configure buffer layout\n", 806 __func__); 807 goto close_ni; 808 } 809 810 /* Configure DMA resources. */ 811 error = dpaa2_ni_setup_dma(sc); 812 if (error) { 813 device_printf(dev, "%s: failed to setup DMA\n", __func__); 814 goto close_ni; 815 } 816 817 /* Setup link between DPNI and an object it's connected to. */ 818 ep1_desc.obj_id = dinfo->id; 819 ep1_desc.if_id = 0; /* DPNI has the only endpoint */ 820 ep1_desc.type = dinfo->dtype; 821 822 error = DPAA2_CMD_RC_GET_CONN(dev, child, DPAA2_CMD_TK(&cmd, rc_token), 823 &ep1_desc, &ep2_desc, &link); 824 if (error) { 825 device_printf(dev, "%s: failed to obtain an object DPNI is " 826 "connected to: error=%d\n", __func__, error); 827 } else { 828 device_printf(dev, "connected to %s (id=%d)\n", 829 dpaa2_ttos(ep2_desc.type), ep2_desc.obj_id); 830 831 error = dpaa2_ni_set_mac_addr(dev); 832 if (error) { 833 device_printf(dev, "%s: failed to set MAC address: " 834 "error=%d\n", __func__, error); 835 } 836 837 if (ep2_desc.type == DPAA2_DEV_MAC) { 838 /* 839 * This is the simplest case when DPNI is connected to 840 * DPMAC directly. 841 */ 842 sc->mac.dpmac_id = ep2_desc.obj_id; 843 844 link_type = DPAA2_MAC_LINK_TYPE_NONE; 845 846 /* 847 * Need to determine if DPMAC type is PHY (attached to 848 * conventional MII PHY) or FIXED (usually SFP/SerDes, 849 * link state managed by MC firmware). 850 */ 851 error = DPAA2_CMD_MAC_OPEN(sc->dev, child, 852 DPAA2_CMD_TK(&cmd, rc_token), sc->mac.dpmac_id, 853 &mac_token); 854 /* 855 * Under VFIO, the DPMAC might be sitting in another 856 * container (DPRC) we don't have access to. 857 * Assume DPAA2_MAC_LINK_TYPE_FIXED if this is 858 * the case. 859 */ 860 if (error) { 861 device_printf(dev, "%s: failed to open " 862 "connected DPMAC: %d (assuming in other DPRC)\n", __func__, 863 sc->mac.dpmac_id); 864 link_type = DPAA2_MAC_LINK_TYPE_FIXED; 865 } else { 866 error = DPAA2_CMD_MAC_GET_ATTRIBUTES(dev, child, 867 &cmd, &attr); 868 if (error) { 869 device_printf(dev, "%s: failed to get " 870 "DPMAC attributes: id=%d, " 871 "error=%d\n", __func__, dinfo->id, 872 error); 873 } else { 874 link_type = attr.link_type; 875 } 876 } 877 DPAA2_CMD_MAC_CLOSE(dev, child, &cmd); 878 879 if (link_type == DPAA2_MAC_LINK_TYPE_FIXED) { 880 device_printf(dev, "connected DPMAC is in FIXED " 881 "mode\n"); 882 dpaa2_ni_setup_fixed_link(sc); 883 } else if (link_type == DPAA2_MAC_LINK_TYPE_PHY) { 884 device_printf(dev, "connected DPMAC is in PHY " 885 "mode\n"); 886 error = DPAA2_MC_GET_PHY_DEV(dev, 887 &sc->mac.phy_dev, sc->mac.dpmac_id); 888 if (error == 0) { 889 error = MEMAC_MDIO_SET_NI_DEV( 890 sc->mac.phy_dev, dev); 891 if (error != 0) { 892 device_printf(dev, "%s: failed " 893 "to set dpni dev on memac " 894 "mdio dev %s: error=%d\n", 895 __func__, 896 device_get_nameunit( 897 sc->mac.phy_dev), error); 898 } 899 } 900 if (error == 0) { 901 error = MEMAC_MDIO_GET_PHY_LOC( 902 sc->mac.phy_dev, &sc->mac.phy_loc); 903 if (error == ENODEV) { 904 error = 0; 905 } 906 if (error != 0) { 907 device_printf(dev, "%s: failed " 908 "to get phy location from " 909 "memac mdio dev %s: error=%d\n", 910 __func__, device_get_nameunit( 911 sc->mac.phy_dev), error); 912 } 913 } 914 if (error == 0) { 915 error = mii_attach(sc->mac.phy_dev, 916 &sc->miibus, sc->ifp, 917 dpaa2_ni_media_change, 918 dpaa2_ni_media_status, 919 BMSR_DEFCAPMASK, sc->mac.phy_loc, 920 MII_OFFSET_ANY, 0); 921 if (error != 0) { 922 device_printf(dev, "%s: failed " 923 "to attach to miibus: " 924 "error=%d\n", 925 __func__, error); 926 } 927 } 928 if (error == 0) { 929 sc->mii = device_get_softc(sc->miibus); 930 } 931 } else { 932 device_printf(dev, "%s: DPMAC link type is not " 933 "supported\n", __func__); 934 } 935 } else if (ep2_desc.type == DPAA2_DEV_NI || 936 ep2_desc.type == DPAA2_DEV_MUX || 937 ep2_desc.type == DPAA2_DEV_SW) { 938 dpaa2_ni_setup_fixed_link(sc); 939 } 940 } 941 942 /* Select mode to enqueue frames. */ 943 /* ... TBD ... */ 944 945 /* 946 * Update link configuration to enable Rx/Tx pause frames support. 947 * 948 * NOTE: MC may generate an interrupt to the DPMAC and request changes 949 * in link configuration. It might be necessary to attach miibus 950 * and PHY before this point. 951 */ 952 error = dpaa2_ni_set_pause_frame(dev); 953 if (error) { 954 device_printf(dev, "%s: failed to configure Rx/Tx pause " 955 "frames\n", __func__); 956 goto close_ni; 957 } 958 959 /* Configure ingress traffic classification. */ 960 error = dpaa2_ni_set_qos_table(dev); 961 if (error) { 962 device_printf(dev, "%s: failed to configure QoS table: " 963 "error=%d\n", __func__, error); 964 goto close_ni; 965 } 966 967 /* Add broadcast physical address to the MAC filtering table. */ 968 memset(eth_bca, 0xff, ETHER_ADDR_LEN); 969 error = DPAA2_CMD_NI_ADD_MAC_ADDR(dev, child, DPAA2_CMD_TK(&cmd, 970 ni_token), eth_bca); 971 if (error) { 972 device_printf(dev, "%s: failed to add broadcast physical " 973 "address to the MAC filtering table\n", __func__); 974 goto close_ni; 975 } 976 977 /* Set the maximum allowed length for received frames. */ 978 error = DPAA2_CMD_NI_SET_MFL(dev, child, &cmd, DPAA2_ETH_MFL); 979 if (error) { 980 device_printf(dev, "%s: failed to set maximum length for " 981 "received frames\n", __func__); 982 goto close_ni; 983 } 984 985 (void)DPAA2_CMD_NI_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, ni_token)); 986 (void)DPAA2_CMD_RC_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, rc_token)); 987 return (0); 988 989 close_ni: 990 (void)DPAA2_CMD_NI_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, ni_token)); 991 close_rc: 992 (void)DPAA2_CMD_RC_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, rc_token)); 993 err_exit: 994 return (error); 995 } 996 997 /** 998 * @brief Сonfigure QBMan channels and register data availability notifications. 999 */ 1000 static int 1001 dpaa2_ni_setup_channels(device_t dev) 1002 { 1003 device_t iodev, condev, bpdev; 1004 struct dpaa2_ni_softc *sc = device_get_softc(dev); 1005 uint32_t i, num_chan; 1006 int error; 1007 1008 /* Calculate number of the channels based on the allocated resources */ 1009 for (i = 0; i < DPAA2_NI_IO_RES_NUM; i++) { 1010 if (!sc->res[DPAA2_NI_IO_RID(i)]) { 1011 break; 1012 } 1013 } 1014 num_chan = i; 1015 for (i = 0; i < DPAA2_NI_CON_RES_NUM; i++) { 1016 if (!sc->res[DPAA2_NI_CON_RID(i)]) { 1017 break; 1018 } 1019 } 1020 num_chan = i < num_chan ? i : num_chan; 1021 sc->chan_n = num_chan > DPAA2_MAX_CHANNELS 1022 ? DPAA2_MAX_CHANNELS : num_chan; 1023 sc->chan_n = sc->chan_n > sc->attr.num.queues 1024 ? sc->attr.num.queues : sc->chan_n; 1025 1026 KASSERT(sc->chan_n > 0u, ("%s: positive number of channels expected: " 1027 "chan_n=%d", __func__, sc->chan_n)); 1028 1029 device_printf(dev, "channels=%d\n", sc->chan_n); 1030 1031 for (i = 0; i < sc->chan_n; i++) { 1032 iodev = (device_t)rman_get_start(sc->res[DPAA2_NI_IO_RID(i)]); 1033 condev = (device_t)rman_get_start(sc->res[DPAA2_NI_CON_RID(i)]); 1034 /* Only one buffer pool available at the moment */ 1035 bpdev = (device_t)rman_get_start(sc->res[DPAA2_NI_BP_RID(0)]); 1036 1037 error = dpaa2_chan_setup(dev, iodev, condev, bpdev, 1038 &sc->channels[i], i, dpaa2_ni_cleanup_task); 1039 if (error != 0) { 1040 device_printf(dev, "%s: dpaa2_chan_setup() failed: " 1041 "error=%d, chan_id=%d\n", __func__, error, i); 1042 return (error); 1043 } 1044 } 1045 1046 /* There is exactly one Rx error queue per network interface */ 1047 error = dpaa2_chan_setup_fq(dev, sc->channels[0], DPAA2_NI_QUEUE_RX_ERR); 1048 if (error != 0) { 1049 device_printf(dev, "%s: failed to prepare RxError queue: " 1050 "error=%d\n", __func__, error); 1051 return (error); 1052 } 1053 1054 return (0); 1055 } 1056 1057 /** 1058 * @brief Bind DPNI to DPBPs, DPIOs, frame queues and channels. 1059 */ 1060 static int 1061 dpaa2_ni_bind(device_t dev) 1062 { 1063 device_t pdev = device_get_parent(dev); 1064 device_t child = dev; 1065 device_t bp_dev; 1066 struct dpaa2_ni_softc *sc = device_get_softc(dev); 1067 struct dpaa2_devinfo *rcinfo = device_get_ivars(pdev); 1068 struct dpaa2_devinfo *dinfo = device_get_ivars(dev); 1069 struct dpaa2_devinfo *bp_info; 1070 struct dpaa2_cmd cmd; 1071 struct dpaa2_ni_pools_cfg pools_cfg; 1072 struct dpaa2_ni_err_cfg err_cfg; 1073 struct dpaa2_channel *chan; 1074 uint16_t rc_token, ni_token; 1075 int error; 1076 1077 DPAA2_CMD_INIT(&cmd); 1078 1079 error = DPAA2_CMD_RC_OPEN(dev, child, &cmd, rcinfo->id, &rc_token); 1080 if (error) { 1081 device_printf(dev, "%s: failed to open resource container: " 1082 "id=%d, error=%d\n", __func__, rcinfo->id, error); 1083 goto err_exit; 1084 } 1085 error = DPAA2_CMD_NI_OPEN(dev, child, &cmd, dinfo->id, &ni_token); 1086 if (error) { 1087 device_printf(dev, "%s: failed to open network interface: " 1088 "id=%d, error=%d\n", __func__, dinfo->id, error); 1089 goto close_rc; 1090 } 1091 1092 /* Select buffer pool (only one available at the moment). */ 1093 bp_dev = (device_t) rman_get_start(sc->res[DPAA2_NI_BP_RID(0)]); 1094 bp_info = device_get_ivars(bp_dev); 1095 1096 /* Configure buffers pool. */ 1097 pools_cfg.pools_num = 1; 1098 pools_cfg.pools[0].bp_obj_id = bp_info->id; 1099 pools_cfg.pools[0].backup_flag = 0; 1100 pools_cfg.pools[0].buf_sz = sc->buf_sz; 1101 error = DPAA2_CMD_NI_SET_POOLS(dev, child, &cmd, &pools_cfg); 1102 if (error) { 1103 device_printf(dev, "%s: failed to set buffer pools\n", __func__); 1104 goto close_ni; 1105 } 1106 1107 /* Setup ingress traffic distribution. */ 1108 error = dpaa2_ni_setup_rx_dist(dev); 1109 if (error && error != EOPNOTSUPP) { 1110 device_printf(dev, "%s: failed to setup ingress traffic " 1111 "distribution\n", __func__); 1112 goto close_ni; 1113 } 1114 if (bootverbose && error == EOPNOTSUPP) { 1115 device_printf(dev, "Ingress traffic distribution not " 1116 "supported\n"); 1117 } 1118 1119 /* Configure handling of error frames. */ 1120 err_cfg.err_mask = DPAA2_NI_FAS_RX_ERR_MASK; 1121 err_cfg.set_err_fas = false; 1122 err_cfg.action = DPAA2_NI_ERR_DISCARD; 1123 error = DPAA2_CMD_NI_SET_ERR_BEHAVIOR(dev, child, &cmd, &err_cfg); 1124 if (error) { 1125 device_printf(dev, "%s: failed to set errors behavior\n", 1126 __func__); 1127 goto close_ni; 1128 } 1129 1130 /* Configure channel queues to generate CDANs. */ 1131 for (uint32_t i = 0; i < sc->chan_n; i++) { 1132 chan = sc->channels[i]; 1133 1134 /* Setup Rx flows. */ 1135 for (uint32_t j = 0; j < chan->rxq_n; j++) { 1136 error = dpaa2_ni_setup_rx_flow(dev, &chan->rx_queues[j]); 1137 if (error) { 1138 device_printf(dev, "%s: failed to setup Rx " 1139 "flow: error=%d\n", __func__, error); 1140 goto close_ni; 1141 } 1142 } 1143 1144 /* Setup Tx flow. */ 1145 error = dpaa2_ni_setup_tx_flow(dev, &chan->txc_queue); 1146 if (error) { 1147 device_printf(dev, "%s: failed to setup Tx " 1148 "flow: error=%d\n", __func__, error); 1149 goto close_ni; 1150 } 1151 } 1152 1153 /* Configure RxError queue to generate CDAN. */ 1154 error = dpaa2_ni_setup_rx_err_flow(dev, &sc->rxe_queue); 1155 if (error) { 1156 device_printf(dev, "%s: failed to setup RxError flow: " 1157 "error=%d\n", __func__, error); 1158 goto close_ni; 1159 } 1160 1161 /* 1162 * Get the Queuing Destination ID (QDID) that should be used for frame 1163 * enqueue operations. 1164 */ 1165 error = DPAA2_CMD_NI_GET_QDID(dev, child, &cmd, DPAA2_NI_QUEUE_TX, 1166 &sc->tx_qdid); 1167 if (error) { 1168 device_printf(dev, "%s: failed to get Tx queuing destination " 1169 "ID\n", __func__); 1170 goto close_ni; 1171 } 1172 1173 (void)DPAA2_CMD_NI_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, ni_token)); 1174 (void)DPAA2_CMD_RC_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, rc_token)); 1175 return (0); 1176 1177 close_ni: 1178 (void)DPAA2_CMD_NI_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, ni_token)); 1179 close_rc: 1180 (void)DPAA2_CMD_RC_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, rc_token)); 1181 err_exit: 1182 return (error); 1183 } 1184 1185 /** 1186 * @brief Setup ingress traffic distribution. 1187 * 1188 * NOTE: Ingress traffic distribution is valid only when DPNI_OPT_NO_FS option 1189 * hasn't been set for DPNI and a number of DPNI queues > 1. 1190 */ 1191 static int 1192 dpaa2_ni_setup_rx_dist(device_t dev) 1193 { 1194 /* 1195 * Have the interface implicitly distribute traffic based on the default 1196 * hash key. 1197 */ 1198 return (dpaa2_ni_set_hash(dev, DPAA2_RXH_DEFAULT)); 1199 } 1200 1201 static int 1202 dpaa2_ni_setup_rx_flow(device_t dev, struct dpaa2_ni_fq *fq) 1203 { 1204 device_t pdev = device_get_parent(dev); 1205 device_t child = dev; 1206 struct dpaa2_devinfo *rcinfo = device_get_ivars(pdev); 1207 struct dpaa2_devinfo *dinfo = device_get_ivars(dev); 1208 struct dpaa2_devinfo *con_info; 1209 struct dpaa2_cmd cmd; 1210 struct dpaa2_ni_queue_cfg queue_cfg = {0}; 1211 uint16_t rc_token, ni_token; 1212 int error; 1213 1214 DPAA2_CMD_INIT(&cmd); 1215 1216 error = DPAA2_CMD_RC_OPEN(dev, child, &cmd, rcinfo->id, &rc_token); 1217 if (error) { 1218 device_printf(dev, "%s: failed to open resource container: " 1219 "id=%d, error=%d\n", __func__, rcinfo->id, error); 1220 goto err_exit; 1221 } 1222 error = DPAA2_CMD_NI_OPEN(dev, child, &cmd, dinfo->id, &ni_token); 1223 if (error) { 1224 device_printf(dev, "%s: failed to open network interface: " 1225 "id=%d, error=%d\n", __func__, dinfo->id, error); 1226 goto close_rc; 1227 } 1228 1229 /* Obtain DPCON associated with the FQ's channel. */ 1230 con_info = device_get_ivars(fq->chan->con_dev); 1231 1232 queue_cfg.type = DPAA2_NI_QUEUE_RX; 1233 queue_cfg.tc = fq->tc; 1234 queue_cfg.idx = fq->flowid; 1235 error = DPAA2_CMD_NI_GET_QUEUE(dev, child, &cmd, &queue_cfg); 1236 if (error) { 1237 device_printf(dev, "%s: failed to obtain Rx queue " 1238 "configuration: tc=%d, flowid=%d\n", __func__, queue_cfg.tc, 1239 queue_cfg.idx); 1240 goto close_ni; 1241 } 1242 1243 fq->fqid = queue_cfg.fqid; 1244 1245 queue_cfg.dest_id = con_info->id; 1246 queue_cfg.dest_type = DPAA2_NI_DEST_DPCON; 1247 queue_cfg.priority = 1; 1248 queue_cfg.user_ctx = (uint64_t)(uintmax_t) fq; 1249 queue_cfg.options = 1250 DPAA2_NI_QUEUE_OPT_USER_CTX | 1251 DPAA2_NI_QUEUE_OPT_DEST; 1252 error = DPAA2_CMD_NI_SET_QUEUE(dev, child, &cmd, &queue_cfg); 1253 if (error) { 1254 device_printf(dev, "%s: failed to update Rx queue " 1255 "configuration: tc=%d, flowid=%d\n", __func__, queue_cfg.tc, 1256 queue_cfg.idx); 1257 goto close_ni; 1258 } 1259 1260 if (bootverbose) { 1261 device_printf(dev, "RX queue idx=%d, tc=%d, chan=%d, fqid=%d, " 1262 "user_ctx=%#jx\n", fq->flowid, fq->tc, fq->chan->id, 1263 fq->fqid, (uint64_t) fq); 1264 } 1265 1266 (void)DPAA2_CMD_NI_CLOSE(dev, child, &cmd); 1267 (void)DPAA2_CMD_RC_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, rc_token)); 1268 return (0); 1269 1270 close_ni: 1271 (void)DPAA2_CMD_NI_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, ni_token)); 1272 close_rc: 1273 (void)DPAA2_CMD_RC_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, rc_token)); 1274 err_exit: 1275 return (error); 1276 } 1277 1278 static int 1279 dpaa2_ni_setup_tx_flow(device_t dev, struct dpaa2_ni_fq *fq) 1280 { 1281 device_t pdev = device_get_parent(dev); 1282 device_t child = dev; 1283 struct dpaa2_ni_softc *sc = device_get_softc(dev); 1284 struct dpaa2_channel *ch = fq->chan; 1285 struct dpaa2_devinfo *rcinfo = device_get_ivars(pdev); 1286 struct dpaa2_devinfo *dinfo = device_get_ivars(dev); 1287 struct dpaa2_devinfo *con_info; 1288 struct dpaa2_ni_queue_cfg queue_cfg = {0}; 1289 struct dpaa2_ni_tx_ring *tx; 1290 struct dpaa2_buf *buf; 1291 struct dpaa2_cmd cmd; 1292 uint32_t tx_rings_n = 0; 1293 uint16_t rc_token, ni_token; 1294 int error; 1295 1296 DPAA2_CMD_INIT(&cmd); 1297 1298 error = DPAA2_CMD_RC_OPEN(dev, child, &cmd, rcinfo->id, &rc_token); 1299 if (error) { 1300 device_printf(dev, "%s: failed to open resource container: " 1301 "id=%d, error=%d\n", __func__, rcinfo->id, error); 1302 goto err_exit; 1303 } 1304 error = DPAA2_CMD_NI_OPEN(dev, child, &cmd, dinfo->id, &ni_token); 1305 if (error) { 1306 device_printf(dev, "%s: failed to open network interface: " 1307 "id=%d, error=%d\n", __func__, dinfo->id, error); 1308 goto close_rc; 1309 } 1310 1311 /* Obtain DPCON associated with the FQ's channel. */ 1312 con_info = device_get_ivars(fq->chan->con_dev); 1313 1314 KASSERT(sc->attr.num.tx_tcs <= DPAA2_MAX_TCS, 1315 ("%s: too many Tx traffic classes: tx_tcs=%d\n", __func__, 1316 sc->attr.num.tx_tcs)); 1317 KASSERT(DPAA2_NI_BUFS_PER_TX <= DPAA2_NI_MAX_BPTX, 1318 ("%s: too many Tx buffers (%d): max=%d\n", __func__, 1319 DPAA2_NI_BUFS_PER_TX, DPAA2_NI_MAX_BPTX)); 1320 1321 /* Setup Tx rings. */ 1322 for (int i = 0; i < sc->attr.num.tx_tcs; i++) { 1323 queue_cfg.type = DPAA2_NI_QUEUE_TX; 1324 queue_cfg.tc = i; 1325 queue_cfg.idx = fq->flowid; 1326 queue_cfg.chan_id = fq->chan->id; 1327 1328 error = DPAA2_CMD_NI_GET_QUEUE(dev, child, &cmd, &queue_cfg); 1329 if (error) { 1330 device_printf(dev, "%s: failed to obtain Tx queue " 1331 "configuration: tc=%d, flowid=%d\n", __func__, 1332 queue_cfg.tc, queue_cfg.idx); 1333 goto close_ni; 1334 } 1335 1336 tx = &fq->tx_rings[i]; 1337 tx->fq = fq; 1338 tx->fqid = queue_cfg.fqid; 1339 tx->txid = tx_rings_n; 1340 1341 if (bootverbose) { 1342 device_printf(dev, "TX queue idx=%d, tc=%d, chan=%d, " 1343 "fqid=%d\n", fq->flowid, i, fq->chan->id, 1344 queue_cfg.fqid); 1345 } 1346 1347 mtx_init(&tx->lock, "dpaa2_tx_ring", NULL, MTX_DEF); 1348 1349 /* Allocate Tx ring buffer. */ 1350 tx->br = buf_ring_alloc(DPAA2_TX_BUFRING_SZ, M_DEVBUF, M_NOWAIT, 1351 &tx->lock); 1352 if (tx->br == NULL) { 1353 device_printf(dev, "%s: failed to setup Tx ring buffer" 1354 " (2) fqid=%d\n", __func__, tx->fqid); 1355 goto close_ni; 1356 } 1357 1358 /* Configure Tx buffers */ 1359 for (uint64_t j = 0; j < DPAA2_NI_BUFS_PER_TX; j++) { 1360 buf = malloc(sizeof(struct dpaa2_buf), M_DPAA2_TXB, 1361 M_WAITOK); 1362 /* Keep DMA tag and Tx ring linked to the buffer */ 1363 DPAA2_BUF_INIT_TAGOPT(buf, ch->tx_dmat, tx); 1364 1365 buf->sgt = malloc(sizeof(struct dpaa2_buf), M_DPAA2_TXB, 1366 M_WAITOK); 1367 /* Link SGT to DMA tag and back to its Tx buffer */ 1368 DPAA2_BUF_INIT_TAGOPT(buf->sgt, ch->sgt_dmat, buf); 1369 1370 error = dpaa2_buf_seed_txb(dev, buf); 1371 1372 /* Add Tx buffer to the ring */ 1373 buf_ring_enqueue(tx->br, buf); 1374 } 1375 1376 tx_rings_n++; 1377 } 1378 1379 /* All Tx queues which belong to the same flowid have the same qdbin. */ 1380 fq->tx_qdbin = queue_cfg.qdbin; 1381 1382 queue_cfg.type = DPAA2_NI_QUEUE_TX_CONF; 1383 queue_cfg.tc = 0; /* ignored for TxConf queue */ 1384 queue_cfg.idx = fq->flowid; 1385 error = DPAA2_CMD_NI_GET_QUEUE(dev, child, &cmd, &queue_cfg); 1386 if (error) { 1387 device_printf(dev, "%s: failed to obtain TxConf queue " 1388 "configuration: tc=%d, flowid=%d\n", __func__, queue_cfg.tc, 1389 queue_cfg.idx); 1390 goto close_ni; 1391 } 1392 1393 fq->fqid = queue_cfg.fqid; 1394 1395 queue_cfg.dest_id = con_info->id; 1396 queue_cfg.dest_type = DPAA2_NI_DEST_DPCON; 1397 queue_cfg.priority = 0; 1398 queue_cfg.user_ctx = (uint64_t)(uintmax_t) fq; 1399 queue_cfg.options = 1400 DPAA2_NI_QUEUE_OPT_USER_CTX | 1401 DPAA2_NI_QUEUE_OPT_DEST; 1402 error = DPAA2_CMD_NI_SET_QUEUE(dev, child, &cmd, &queue_cfg); 1403 if (error) { 1404 device_printf(dev, "%s: failed to update TxConf queue " 1405 "configuration: tc=%d, flowid=%d\n", __func__, queue_cfg.tc, 1406 queue_cfg.idx); 1407 goto close_ni; 1408 } 1409 1410 (void)DPAA2_CMD_NI_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, ni_token)); 1411 (void)DPAA2_CMD_RC_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, rc_token)); 1412 return (0); 1413 1414 close_ni: 1415 (void)DPAA2_CMD_NI_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, ni_token)); 1416 close_rc: 1417 (void)DPAA2_CMD_RC_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, rc_token)); 1418 err_exit: 1419 return (error); 1420 } 1421 1422 static int 1423 dpaa2_ni_setup_rx_err_flow(device_t dev, struct dpaa2_ni_fq *fq) 1424 { 1425 device_t pdev = device_get_parent(dev); 1426 device_t child = dev; 1427 struct dpaa2_devinfo *rcinfo = device_get_ivars(pdev); 1428 struct dpaa2_devinfo *dinfo = device_get_ivars(dev); 1429 struct dpaa2_devinfo *con_info; 1430 struct dpaa2_ni_queue_cfg queue_cfg = {0}; 1431 struct dpaa2_cmd cmd; 1432 uint16_t rc_token, ni_token; 1433 int error; 1434 1435 DPAA2_CMD_INIT(&cmd); 1436 1437 error = DPAA2_CMD_RC_OPEN(dev, child, &cmd, rcinfo->id, &rc_token); 1438 if (error) { 1439 device_printf(dev, "%s: failed to open resource container: " 1440 "id=%d, error=%d\n", __func__, rcinfo->id, error); 1441 goto err_exit; 1442 } 1443 error = DPAA2_CMD_NI_OPEN(dev, child, &cmd, dinfo->id, &ni_token); 1444 if (error) { 1445 device_printf(dev, "%s: failed to open network interface: " 1446 "id=%d, error=%d\n", __func__, dinfo->id, error); 1447 goto close_rc; 1448 } 1449 1450 /* Obtain DPCON associated with the FQ's channel. */ 1451 con_info = device_get_ivars(fq->chan->con_dev); 1452 1453 queue_cfg.type = DPAA2_NI_QUEUE_RX_ERR; 1454 queue_cfg.tc = fq->tc; /* ignored */ 1455 queue_cfg.idx = fq->flowid; /* ignored */ 1456 error = DPAA2_CMD_NI_GET_QUEUE(dev, child, &cmd, &queue_cfg); 1457 if (error) { 1458 device_printf(dev, "%s: failed to obtain RxErr queue " 1459 "configuration\n", __func__); 1460 goto close_ni; 1461 } 1462 1463 fq->fqid = queue_cfg.fqid; 1464 1465 queue_cfg.dest_id = con_info->id; 1466 queue_cfg.dest_type = DPAA2_NI_DEST_DPCON; 1467 queue_cfg.priority = 1; 1468 queue_cfg.user_ctx = (uint64_t)(uintmax_t) fq; 1469 queue_cfg.options = 1470 DPAA2_NI_QUEUE_OPT_USER_CTX | 1471 DPAA2_NI_QUEUE_OPT_DEST; 1472 error = DPAA2_CMD_NI_SET_QUEUE(dev, child, &cmd, &queue_cfg); 1473 if (error) { 1474 device_printf(dev, "%s: failed to update RxErr queue " 1475 "configuration\n", __func__); 1476 goto close_ni; 1477 } 1478 1479 (void)DPAA2_CMD_NI_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, ni_token)); 1480 (void)DPAA2_CMD_RC_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, rc_token)); 1481 return (0); 1482 1483 close_ni: 1484 (void)DPAA2_CMD_NI_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, ni_token)); 1485 close_rc: 1486 (void)DPAA2_CMD_RC_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, rc_token)); 1487 err_exit: 1488 return (error); 1489 } 1490 1491 /** 1492 * @brief Configure DPNI object to generate interrupts. 1493 */ 1494 static int 1495 dpaa2_ni_setup_irqs(device_t dev) 1496 { 1497 device_t pdev = device_get_parent(dev); 1498 device_t child = dev; 1499 struct dpaa2_ni_softc *sc = device_get_softc(dev); 1500 struct dpaa2_devinfo *rcinfo = device_get_ivars(pdev); 1501 struct dpaa2_devinfo *dinfo = device_get_ivars(dev); 1502 struct dpaa2_cmd cmd; 1503 uint16_t rc_token, ni_token; 1504 int error; 1505 1506 DPAA2_CMD_INIT(&cmd); 1507 1508 error = DPAA2_CMD_RC_OPEN(dev, child, &cmd, rcinfo->id, &rc_token); 1509 if (error) { 1510 device_printf(dev, "%s: failed to open resource container: " 1511 "id=%d, error=%d\n", __func__, rcinfo->id, error); 1512 goto err_exit; 1513 } 1514 error = DPAA2_CMD_NI_OPEN(dev, child, &cmd, dinfo->id, &ni_token); 1515 if (error) { 1516 device_printf(dev, "%s: failed to open network interface: " 1517 "id=%d, error=%d\n", __func__, dinfo->id, error); 1518 goto close_rc; 1519 } 1520 1521 /* Configure IRQs. */ 1522 error = dpaa2_ni_setup_msi(sc); 1523 if (error) { 1524 device_printf(dev, "%s: failed to allocate MSI\n", __func__); 1525 goto close_ni; 1526 } 1527 if ((sc->irq_res = bus_alloc_resource_any(dev, SYS_RES_IRQ, 1528 &sc->irq_rid[0], RF_ACTIVE | RF_SHAREABLE)) == NULL) { 1529 device_printf(dev, "%s: failed to allocate IRQ resource\n", 1530 __func__); 1531 goto close_ni; 1532 } 1533 if (bus_setup_intr(dev, sc->irq_res, INTR_TYPE_NET | INTR_MPSAFE, 1534 NULL, dpaa2_ni_intr, sc, &sc->intr)) { 1535 device_printf(dev, "%s: failed to setup IRQ resource\n", 1536 __func__); 1537 goto close_ni; 1538 } 1539 1540 error = DPAA2_CMD_NI_SET_IRQ_MASK(dev, child, &cmd, DPNI_IRQ_INDEX, 1541 DPNI_IRQ_LINK_CHANGED | DPNI_IRQ_EP_CHANGED); 1542 if (error) { 1543 device_printf(dev, "%s: failed to set DPNI IRQ mask\n", 1544 __func__); 1545 goto close_ni; 1546 } 1547 1548 error = DPAA2_CMD_NI_SET_IRQ_ENABLE(dev, child, &cmd, DPNI_IRQ_INDEX, 1549 true); 1550 if (error) { 1551 device_printf(dev, "%s: failed to enable DPNI IRQ\n", __func__); 1552 goto close_ni; 1553 } 1554 1555 (void)DPAA2_CMD_NI_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, ni_token)); 1556 (void)DPAA2_CMD_RC_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, rc_token)); 1557 return (0); 1558 1559 close_ni: 1560 (void)DPAA2_CMD_NI_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, ni_token)); 1561 close_rc: 1562 (void)DPAA2_CMD_RC_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, rc_token)); 1563 err_exit: 1564 return (error); 1565 } 1566 1567 /** 1568 * @brief Allocate MSI interrupts for DPNI. 1569 */ 1570 static int 1571 dpaa2_ni_setup_msi(struct dpaa2_ni_softc *sc) 1572 { 1573 int val; 1574 1575 val = pci_msi_count(sc->dev); 1576 if (val < DPAA2_NI_MSI_COUNT) 1577 device_printf(sc->dev, "MSI: actual=%d, expected=%d\n", val, 1578 DPAA2_IO_MSI_COUNT); 1579 val = MIN(val, DPAA2_NI_MSI_COUNT); 1580 1581 if (pci_alloc_msi(sc->dev, &val) != 0) 1582 return (EINVAL); 1583 1584 for (int i = 0; i < val; i++) 1585 sc->irq_rid[i] = i + 1; 1586 1587 return (0); 1588 } 1589 1590 /** 1591 * @brief Update DPNI according to the updated interface capabilities. 1592 */ 1593 static int 1594 dpaa2_ni_setup_if_caps(struct dpaa2_ni_softc *sc) 1595 { 1596 bool en_rxcsum, en_txcsum; 1597 device_t pdev = device_get_parent(sc->dev); 1598 device_t dev = sc->dev; 1599 device_t child = dev; 1600 struct dpaa2_devinfo *rcinfo = device_get_ivars(pdev); 1601 struct dpaa2_devinfo *dinfo = device_get_ivars(dev); 1602 struct dpaa2_cmd cmd; 1603 uint16_t rc_token, ni_token; 1604 int error; 1605 1606 DPAA2_CMD_INIT(&cmd); 1607 1608 /* 1609 * XXX-DSL: DPAA2 allows to validate L3/L4 checksums on reception and/or 1610 * generate L3/L4 checksums on transmission without 1611 * differentiating between IPv4/v6, i.e. enable for both 1612 * protocols if requested. 1613 */ 1614 en_rxcsum = if_getcapenable(sc->ifp) & 1615 (IFCAP_RXCSUM | IFCAP_RXCSUM_IPV6); 1616 en_txcsum = if_getcapenable(sc->ifp) & 1617 (IFCAP_TXCSUM | IFCAP_TXCSUM_IPV6); 1618 1619 error = DPAA2_CMD_RC_OPEN(dev, child, &cmd, rcinfo->id, &rc_token); 1620 if (error) { 1621 device_printf(dev, "%s: failed to open resource container: " 1622 "id=%d, error=%d\n", __func__, rcinfo->id, error); 1623 goto err_exit; 1624 } 1625 error = DPAA2_CMD_NI_OPEN(dev, child, &cmd, dinfo->id, &ni_token); 1626 if (error) { 1627 device_printf(dev, "%s: failed to open network interface: " 1628 "id=%d, error=%d\n", __func__, dinfo->id, error); 1629 goto close_rc; 1630 } 1631 1632 /* Setup checksums validation. */ 1633 error = DPAA2_CMD_NI_SET_OFFLOAD(dev, child, &cmd, 1634 DPAA2_NI_OFL_RX_L3_CSUM, en_rxcsum); 1635 if (error) { 1636 device_printf(dev, "%s: failed to %s L3 checksum validation\n", 1637 __func__, en_rxcsum ? "enable" : "disable"); 1638 goto close_ni; 1639 } 1640 error = DPAA2_CMD_NI_SET_OFFLOAD(dev, child, &cmd, 1641 DPAA2_NI_OFL_RX_L4_CSUM, en_rxcsum); 1642 if (error) { 1643 device_printf(dev, "%s: failed to %s L4 checksum validation\n", 1644 __func__, en_rxcsum ? "enable" : "disable"); 1645 goto close_ni; 1646 } 1647 1648 /* Setup checksums generation. */ 1649 error = DPAA2_CMD_NI_SET_OFFLOAD(dev, child, &cmd, 1650 DPAA2_NI_OFL_TX_L3_CSUM, en_txcsum); 1651 if (error) { 1652 device_printf(dev, "%s: failed to %s L3 checksum generation\n", 1653 __func__, en_txcsum ? "enable" : "disable"); 1654 goto close_ni; 1655 } 1656 error = DPAA2_CMD_NI_SET_OFFLOAD(dev, child, &cmd, 1657 DPAA2_NI_OFL_TX_L4_CSUM, en_txcsum); 1658 if (error) { 1659 device_printf(dev, "%s: failed to %s L4 checksum generation\n", 1660 __func__, en_txcsum ? "enable" : "disable"); 1661 goto close_ni; 1662 } 1663 1664 if (bootverbose) { 1665 device_printf(dev, "%s: L3/L4 checksum validation %s\n", 1666 __func__, en_rxcsum ? "enabled" : "disabled"); 1667 device_printf(dev, "%s: L3/L4 checksum generation %s\n", 1668 __func__, en_txcsum ? "enabled" : "disabled"); 1669 } 1670 1671 (void)DPAA2_CMD_NI_CLOSE(dev, child, &cmd); 1672 (void)DPAA2_CMD_RC_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, rc_token)); 1673 return (0); 1674 1675 close_ni: 1676 (void)DPAA2_CMD_NI_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, ni_token)); 1677 close_rc: 1678 (void)DPAA2_CMD_RC_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, rc_token)); 1679 err_exit: 1680 return (error); 1681 } 1682 1683 /** 1684 * @brief Update DPNI according to the updated interface flags. 1685 */ 1686 static int 1687 dpaa2_ni_setup_if_flags(struct dpaa2_ni_softc *sc) 1688 { 1689 const bool en_promisc = if_getflags(sc->ifp) & IFF_PROMISC; 1690 const bool en_allmulti = if_getflags(sc->ifp) & IFF_ALLMULTI; 1691 device_t pdev = device_get_parent(sc->dev); 1692 device_t dev = sc->dev; 1693 device_t child = dev; 1694 struct dpaa2_devinfo *rcinfo = device_get_ivars(pdev); 1695 struct dpaa2_devinfo *dinfo = device_get_ivars(dev); 1696 struct dpaa2_cmd cmd; 1697 uint16_t rc_token, ni_token; 1698 int error; 1699 1700 DPAA2_CMD_INIT(&cmd); 1701 1702 error = DPAA2_CMD_RC_OPEN(dev, child, &cmd, rcinfo->id, &rc_token); 1703 if (error) { 1704 device_printf(dev, "%s: failed to open resource container: " 1705 "id=%d, error=%d\n", __func__, rcinfo->id, error); 1706 goto err_exit; 1707 } 1708 error = DPAA2_CMD_NI_OPEN(dev, child, &cmd, dinfo->id, &ni_token); 1709 if (error) { 1710 device_printf(dev, "%s: failed to open network interface: " 1711 "id=%d, error=%d\n", __func__, dinfo->id, error); 1712 goto close_rc; 1713 } 1714 1715 error = DPAA2_CMD_NI_SET_MULTI_PROMISC(dev, child, &cmd, 1716 en_promisc ? true : en_allmulti); 1717 if (error) { 1718 device_printf(dev, "%s: failed to %s multicast promiscuous " 1719 "mode\n", __func__, en_allmulti ? "enable" : "disable"); 1720 goto close_ni; 1721 } 1722 1723 error = DPAA2_CMD_NI_SET_UNI_PROMISC(dev, child, &cmd, en_promisc); 1724 if (error) { 1725 device_printf(dev, "%s: failed to %s unicast promiscuous mode\n", 1726 __func__, en_promisc ? "enable" : "disable"); 1727 goto close_ni; 1728 } 1729 1730 (void)DPAA2_CMD_NI_CLOSE(dev, child, &cmd); 1731 (void)DPAA2_CMD_RC_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, rc_token)); 1732 return (0); 1733 1734 close_ni: 1735 (void)DPAA2_CMD_NI_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, ni_token)); 1736 close_rc: 1737 (void)DPAA2_CMD_RC_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, rc_token)); 1738 err_exit: 1739 return (error); 1740 } 1741 1742 static int 1743 dpaa2_ni_setup_sysctls(struct dpaa2_ni_softc *sc) 1744 { 1745 struct sysctl_ctx_list *ctx; 1746 struct sysctl_oid *node, *node2; 1747 struct sysctl_oid_list *parent, *parent2; 1748 char cbuf[128]; 1749 int i; 1750 1751 ctx = device_get_sysctl_ctx(sc->dev); 1752 parent = SYSCTL_CHILDREN(device_get_sysctl_tree(sc->dev)); 1753 1754 /* Add DPNI statistics. */ 1755 node = SYSCTL_ADD_NODE(ctx, parent, OID_AUTO, "stats", 1756 CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "DPNI Statistics"); 1757 parent = SYSCTL_CHILDREN(node); 1758 for (i = 0; i < nitems(dpni_stat_sysctls); ++i) { 1759 SYSCTL_ADD_PROC(ctx, parent, i, dpni_stat_sysctls[i].name, 1760 CTLTYPE_U64 | CTLFLAG_RD, sc, 0, dpaa2_ni_collect_stats, 1761 "IU", dpni_stat_sysctls[i].desc); 1762 } 1763 SYSCTL_ADD_UQUAD(ctx, parent, OID_AUTO, "rx_anomaly_frames", 1764 CTLFLAG_RD, &sc->rx_anomaly_frames, 1765 "Rx frames in the buffers outside of the buffer pools"); 1766 SYSCTL_ADD_UQUAD(ctx, parent, OID_AUTO, "rx_single_buf_frames", 1767 CTLFLAG_RD, &sc->rx_single_buf_frames, 1768 "Rx frames in single buffers"); 1769 SYSCTL_ADD_UQUAD(ctx, parent, OID_AUTO, "rx_sg_buf_frames", 1770 CTLFLAG_RD, &sc->rx_sg_buf_frames, 1771 "Rx frames in scatter/gather list"); 1772 SYSCTL_ADD_UQUAD(ctx, parent, OID_AUTO, "rx_enq_rej_frames", 1773 CTLFLAG_RD, &sc->rx_enq_rej_frames, 1774 "Enqueue rejected by QMan"); 1775 SYSCTL_ADD_UQUAD(ctx, parent, OID_AUTO, "rx_ieoi_err_frames", 1776 CTLFLAG_RD, &sc->rx_ieoi_err_frames, 1777 "QMan IEOI error"); 1778 SYSCTL_ADD_UQUAD(ctx, parent, OID_AUTO, "rx_other_err_frames", 1779 CTLFLAG_RD, &sc->rx_other_err_frames, 1780 "Other Rx frames with errors"); 1781 SYSCTL_ADD_UQUAD(ctx, parent, OID_AUTO, "tx_single_buf_frames", 1782 CTLFLAG_RD, &sc->tx_single_buf_frames, 1783 "Tx single buffer frames"); 1784 SYSCTL_ADD_UQUAD(ctx, parent, OID_AUTO, "tx_sg_frames", 1785 CTLFLAG_RD, &sc->tx_sg_frames, 1786 "Tx S/G frames"); 1787 1788 SYSCTL_ADD_PROC(ctx, parent, OID_AUTO, "buf_num", 1789 CTLTYPE_U32 | CTLFLAG_RD, sc, 0, dpaa2_ni_collect_buf_num, 1790 "IU", "number of Rx buffers in the buffer pool"); 1791 SYSCTL_ADD_PROC(ctx, parent, OID_AUTO, "buf_free", 1792 CTLTYPE_U32 | CTLFLAG_RD, sc, 0, dpaa2_ni_collect_buf_free, 1793 "IU", "number of free Rx buffers in the buffer pool"); 1794 1795 /* Add channels statistics. */ 1796 parent = SYSCTL_CHILDREN(device_get_sysctl_tree(sc->dev)); 1797 node = SYSCTL_ADD_NODE(ctx, parent, OID_AUTO, "channels", 1798 CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "DPNI Channels"); 1799 parent = SYSCTL_CHILDREN(node); 1800 for (int i = 0; i < sc->chan_n; i++) { 1801 snprintf(cbuf, sizeof(cbuf), "%d", i); 1802 1803 node2 = SYSCTL_ADD_NODE(ctx, parent, OID_AUTO, cbuf, 1804 CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "DPNI Channel"); 1805 parent2 = SYSCTL_CHILDREN(node2); 1806 1807 SYSCTL_ADD_UQUAD(ctx, parent2, OID_AUTO, "tx_frames", 1808 CTLFLAG_RD, &sc->channels[i]->tx_frames, 1809 "Tx frames counter"); 1810 SYSCTL_ADD_UQUAD(ctx, parent2, OID_AUTO, "tx_dropped", 1811 CTLFLAG_RD, &sc->channels[i]->tx_dropped, 1812 "Tx dropped counter"); 1813 } 1814 1815 /* Add Link debugging options. */ 1816 parent = SYSCTL_CHILDREN(device_get_sysctl_tree(sc->dev)); 1817 node = SYSCTL_ADD_PROC(ctx, parent, OID_AUTO, "link", 1818 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, 1819 sc, 0, dpaa2_ni_sysctl_link_state, 1820 "A", "DPNI link state information"); 1821 1822 /* Add configuration tunables */ 1823 parent = SYSCTL_CHILDREN(device_get_sysctl_tree(sc->dev)); 1824 node = SYSCTL_ADD_NODE(ctx, parent, OID_AUTO, "config", 1825 CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "configuration tunables"); 1826 parent = SYSCTL_CHILDREN(node); 1827 1828 /* Add cleanup budget tunables. */ 1829 SYSCTL_ADD_PROC(ctx, parent, OID_AUTO, "clean_budget", 1830 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE, sc, 0, 1831 dpaa2_ni_sysctl_handle_clean_budget, "d", "clean budget"); 1832 SYSCTL_ADD_PROC(ctx, parent, OID_AUTO, "tx_budget", 1833 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE, sc, 0, 1834 dpaa2_ni_sysctl_handle_tx_budget, "d", "tx budget"); 1835 SYSCTL_ADD_PROC(ctx, parent, OID_AUTO, "rx_budget", 1836 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE, sc, 0, 1837 dpaa2_ni_sysctl_handle_rx_budget, "d", "rx budget"); 1838 1839 return (0); 1840 } 1841 1842 static int 1843 dpaa2_ni_setup_dma(struct dpaa2_ni_softc *sc) 1844 { 1845 device_t dev = sc->dev; 1846 int error; 1847 1848 KASSERT((sc->buf_align == BUF_ALIGN) || (sc->buf_align == BUF_ALIGN_V1), 1849 ("unexpected buffer alignment: %d\n", sc->buf_align)); 1850 1851 /* DMA tag for Rx distribution key. */ 1852 error = bus_dma_tag_create( 1853 bus_get_dma_tag(dev), 1854 PAGE_SIZE, 0, /* alignment, boundary */ 1855 BUS_SPACE_MAXADDR, /* low restricted addr */ 1856 BUS_SPACE_MAXADDR, /* high restricted addr */ 1857 NULL, NULL, /* filter, filterarg */ 1858 DPAA2_CLASSIFIER_DMA_SIZE, 1, /* maxsize, nsegments */ 1859 DPAA2_CLASSIFIER_DMA_SIZE, 0, /* maxsegsize, flags */ 1860 NULL, NULL, /* lockfunc, lockarg */ 1861 &sc->rxd_dmat); 1862 if (error) { 1863 device_printf(dev, "%s: failed to create DMA tag for Rx " 1864 "distribution key\n", __func__); 1865 return (error); 1866 } 1867 1868 error = bus_dma_tag_create( 1869 bus_get_dma_tag(dev), 1870 PAGE_SIZE, 0, /* alignment, boundary */ 1871 BUS_SPACE_MAXADDR, /* low restricted addr */ 1872 BUS_SPACE_MAXADDR, /* high restricted addr */ 1873 NULL, NULL, /* filter, filterarg */ 1874 ETH_QOS_KCFG_BUF_SIZE, 1, /* maxsize, nsegments */ 1875 ETH_QOS_KCFG_BUF_SIZE, 0, /* maxsegsize, flags */ 1876 NULL, NULL, /* lockfunc, lockarg */ 1877 &sc->qos_dmat); 1878 if (error) { 1879 device_printf(dev, "%s: failed to create DMA tag for QoS key\n", 1880 __func__); 1881 return (error); 1882 } 1883 1884 return (0); 1885 } 1886 1887 /** 1888 * @brief Configure buffer layouts of the different DPNI queues. 1889 */ 1890 static int 1891 dpaa2_ni_set_buf_layout(device_t dev) 1892 { 1893 device_t pdev = device_get_parent(dev); 1894 device_t child = dev; 1895 struct dpaa2_devinfo *rcinfo = device_get_ivars(pdev); 1896 struct dpaa2_devinfo *dinfo = device_get_ivars(dev); 1897 struct dpaa2_ni_softc *sc = device_get_softc(dev); 1898 struct dpaa2_ni_buf_layout buf_layout = {0}; 1899 struct dpaa2_cmd cmd; 1900 uint16_t rc_token, ni_token; 1901 int error; 1902 1903 DPAA2_CMD_INIT(&cmd); 1904 1905 error = DPAA2_CMD_RC_OPEN(dev, child, &cmd, rcinfo->id, &rc_token); 1906 if (error) { 1907 device_printf(dev, "%s: failed to open resource container: " 1908 "id=%d, error=%d\n", __func__, rcinfo->id, error); 1909 goto err_exit; 1910 } 1911 error = DPAA2_CMD_NI_OPEN(dev, child, &cmd, dinfo->id, &ni_token); 1912 if (error) { 1913 device_printf(sc->dev, "%s: failed to open DPMAC: id=%d, " 1914 "error=%d\n", __func__, dinfo->id, error); 1915 goto close_rc; 1916 } 1917 1918 /* 1919 * Select Rx/Tx buffer alignment. It's necessary to ensure that the 1920 * buffer size seen by WRIOP is a multiple of 64 or 256 bytes depending 1921 * on the WRIOP version. 1922 */ 1923 sc->buf_align = (sc->attr.wriop_ver == WRIOP_VERSION(0, 0, 0) || 1924 sc->attr.wriop_ver == WRIOP_VERSION(1, 0, 0)) 1925 ? BUF_ALIGN_V1 : BUF_ALIGN; 1926 1927 /* 1928 * We need to ensure that the buffer size seen by WRIOP is a multiple 1929 * of 64 or 256 bytes depending on the WRIOP version. 1930 */ 1931 sc->buf_sz = ALIGN_DOWN(DPAA2_RX_BUF_SIZE, sc->buf_align); 1932 1933 if (bootverbose) { 1934 device_printf(dev, "Rx/Tx buffers: size=%d, alignment=%d\n", 1935 sc->buf_sz, sc->buf_align); 1936 } 1937 1938 /* 1939 * Frame Descriptor Tx buffer layout 1940 * 1941 * ADDR -> |---------------------| 1942 * | SW FRAME ANNOTATION | BUF_SWA_SIZE bytes 1943 * |---------------------| 1944 * | HW FRAME ANNOTATION | BUF_TX_HWA_SIZE bytes 1945 * |---------------------| 1946 * | DATA HEADROOM | 1947 * ADDR + OFFSET -> |---------------------| 1948 * | | 1949 * | | 1950 * | FRAME DATA | 1951 * | | 1952 * | | 1953 * |---------------------| 1954 * | DATA TAILROOM | 1955 * |---------------------| 1956 * 1957 * NOTE: It's for a single buffer frame only. 1958 */ 1959 buf_layout.queue_type = DPAA2_NI_QUEUE_TX; 1960 buf_layout.pd_size = BUF_SWA_SIZE; 1961 buf_layout.pass_timestamp = true; 1962 buf_layout.pass_frame_status = true; 1963 buf_layout.options = 1964 BUF_LOPT_PRIV_DATA_SZ | 1965 BUF_LOPT_TIMESTAMP | /* requires 128 bytes in HWA */ 1966 BUF_LOPT_FRAME_STATUS; 1967 error = DPAA2_CMD_NI_SET_BUF_LAYOUT(dev, child, &cmd, &buf_layout); 1968 if (error) { 1969 device_printf(dev, "%s: failed to set Tx buffer layout\n", 1970 __func__); 1971 goto close_ni; 1972 } 1973 1974 /* Tx-confirmation buffer layout */ 1975 buf_layout.queue_type = DPAA2_NI_QUEUE_TX_CONF; 1976 buf_layout.options = 1977 BUF_LOPT_TIMESTAMP | 1978 BUF_LOPT_FRAME_STATUS; 1979 error = DPAA2_CMD_NI_SET_BUF_LAYOUT(dev, child, &cmd, &buf_layout); 1980 if (error) { 1981 device_printf(dev, "%s: failed to set TxConf buffer layout\n", 1982 __func__); 1983 goto close_ni; 1984 } 1985 1986 /* 1987 * Driver should reserve the amount of space indicated by this command 1988 * as headroom in all Tx frames. 1989 */ 1990 error = DPAA2_CMD_NI_GET_TX_DATA_OFF(dev, child, &cmd, &sc->tx_data_off); 1991 if (error) { 1992 device_printf(dev, "%s: failed to obtain Tx data offset\n", 1993 __func__); 1994 goto close_ni; 1995 } 1996 1997 if (bootverbose) { 1998 device_printf(dev, "Tx data offset=%d\n", sc->tx_data_off); 1999 } 2000 if ((sc->tx_data_off % 64) != 0) { 2001 device_printf(dev, "Tx data offset (%d) is not a multiplication " 2002 "of 64 bytes\n", sc->tx_data_off); 2003 } 2004 2005 /* 2006 * Frame Descriptor Rx buffer layout 2007 * 2008 * ADDR -> |---------------------| 2009 * | SW FRAME ANNOTATION | BUF_SWA_SIZE bytes 2010 * |---------------------| 2011 * | HW FRAME ANNOTATION | BUF_RX_HWA_SIZE bytes 2012 * |---------------------| 2013 * | DATA HEADROOM | OFFSET-BUF_RX_HWA_SIZE 2014 * ADDR + OFFSET -> |---------------------| 2015 * | | 2016 * | | 2017 * | FRAME DATA | 2018 * | | 2019 * | | 2020 * |---------------------| 2021 * | DATA TAILROOM | 0 bytes 2022 * |---------------------| 2023 * 2024 * NOTE: It's for a single buffer frame only. 2025 */ 2026 buf_layout.queue_type = DPAA2_NI_QUEUE_RX; 2027 buf_layout.pd_size = BUF_SWA_SIZE; 2028 buf_layout.fd_align = sc->buf_align; 2029 buf_layout.head_size = sc->tx_data_off - BUF_RX_HWA_SIZE - BUF_SWA_SIZE; 2030 buf_layout.tail_size = 0; 2031 buf_layout.pass_frame_status = true; 2032 buf_layout.pass_parser_result = true; 2033 buf_layout.pass_timestamp = true; 2034 buf_layout.options = 2035 BUF_LOPT_PRIV_DATA_SZ | 2036 BUF_LOPT_DATA_ALIGN | 2037 BUF_LOPT_DATA_HEAD_ROOM | 2038 BUF_LOPT_DATA_TAIL_ROOM | 2039 BUF_LOPT_FRAME_STATUS | 2040 BUF_LOPT_PARSER_RESULT | 2041 BUF_LOPT_TIMESTAMP; 2042 error = DPAA2_CMD_NI_SET_BUF_LAYOUT(dev, child, &cmd, &buf_layout); 2043 if (error) { 2044 device_printf(dev, "%s: failed to set Rx buffer layout\n", 2045 __func__); 2046 goto close_ni; 2047 } 2048 2049 error = 0; 2050 close_ni: 2051 (void)DPAA2_CMD_NI_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, ni_token)); 2052 close_rc: 2053 (void)DPAA2_CMD_RC_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, rc_token)); 2054 err_exit: 2055 return (error); 2056 } 2057 2058 /** 2059 * @brief Enable Rx/Tx pause frames. 2060 * 2061 * NOTE: DPNI stops sending when a pause frame is received (Rx frame) or DPNI 2062 * itself generates pause frames (Tx frame). 2063 */ 2064 static int 2065 dpaa2_ni_set_pause_frame(device_t dev) 2066 { 2067 device_t pdev = device_get_parent(dev); 2068 device_t child = dev; 2069 struct dpaa2_devinfo *rcinfo = device_get_ivars(pdev); 2070 struct dpaa2_devinfo *dinfo = device_get_ivars(dev); 2071 struct dpaa2_ni_softc *sc = device_get_softc(dev); 2072 struct dpaa2_ni_link_cfg link_cfg = {0}; 2073 struct dpaa2_cmd cmd; 2074 uint16_t rc_token, ni_token; 2075 int error; 2076 2077 DPAA2_CMD_INIT(&cmd); 2078 2079 error = DPAA2_CMD_RC_OPEN(dev, child, &cmd, rcinfo->id, &rc_token); 2080 if (error) { 2081 device_printf(dev, "%s: failed to open resource container: " 2082 "id=%d, error=%d\n", __func__, rcinfo->id, error); 2083 goto err_exit; 2084 } 2085 error = DPAA2_CMD_NI_OPEN(dev, child, &cmd, dinfo->id, &ni_token); 2086 if (error) { 2087 device_printf(sc->dev, "%s: failed to open DPMAC: id=%d, " 2088 "error=%d\n", __func__, dinfo->id, error); 2089 goto close_rc; 2090 } 2091 2092 error = DPAA2_CMD_NI_GET_LINK_CFG(dev, child, &cmd, &link_cfg); 2093 if (error) { 2094 device_printf(dev, "%s: failed to obtain link configuration: " 2095 "error=%d\n", __func__, error); 2096 goto close_ni; 2097 } 2098 2099 /* Enable both Rx and Tx pause frames by default. */ 2100 link_cfg.options |= DPAA2_NI_LINK_OPT_PAUSE; 2101 link_cfg.options &= ~DPAA2_NI_LINK_OPT_ASYM_PAUSE; 2102 2103 error = DPAA2_CMD_NI_SET_LINK_CFG(dev, child, &cmd, &link_cfg); 2104 if (error) { 2105 device_printf(dev, "%s: failed to set link configuration: " 2106 "error=%d\n", __func__, error); 2107 goto close_ni; 2108 } 2109 2110 sc->link_options = link_cfg.options; 2111 error = 0; 2112 close_ni: 2113 (void)DPAA2_CMD_NI_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, ni_token)); 2114 close_rc: 2115 (void)DPAA2_CMD_RC_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, rc_token)); 2116 err_exit: 2117 return (error); 2118 } 2119 2120 /** 2121 * @brief Configure QoS table to determine the traffic class for the received 2122 * frame. 2123 */ 2124 static int 2125 dpaa2_ni_set_qos_table(device_t dev) 2126 { 2127 device_t pdev = device_get_parent(dev); 2128 device_t child = dev; 2129 struct dpaa2_devinfo *rcinfo = device_get_ivars(pdev); 2130 struct dpaa2_devinfo *dinfo = device_get_ivars(dev); 2131 struct dpaa2_ni_softc *sc = device_get_softc(dev); 2132 struct dpaa2_ni_qos_table tbl; 2133 struct dpaa2_buf *buf = &sc->qos_kcfg; 2134 struct dpaa2_cmd cmd; 2135 uint16_t rc_token, ni_token; 2136 int error; 2137 2138 if (sc->attr.num.rx_tcs == 1 || 2139 !(sc->attr.options & DPNI_OPT_HAS_KEY_MASKING)) { 2140 if (bootverbose) { 2141 device_printf(dev, "Ingress traffic classification is " 2142 "not supported\n"); 2143 } 2144 return (0); 2145 } 2146 2147 /* 2148 * Allocate a buffer visible to the device to hold the QoS table key 2149 * configuration. 2150 */ 2151 2152 if (__predict_true(buf->dmat == NULL)) { 2153 buf->dmat = sc->qos_dmat; 2154 } 2155 2156 error = bus_dmamem_alloc(buf->dmat, (void **)&buf->vaddr, 2157 BUS_DMA_ZERO | BUS_DMA_COHERENT, &buf->dmap); 2158 if (error) { 2159 device_printf(dev, "%s: failed to allocate a buffer for QoS key " 2160 "configuration\n", __func__); 2161 goto err_exit; 2162 } 2163 2164 error = bus_dmamap_load(buf->dmat, buf->dmap, buf->vaddr, 2165 ETH_QOS_KCFG_BUF_SIZE, dpaa2_dmamap_oneseg_cb, &buf->paddr, 2166 BUS_DMA_NOWAIT); 2167 if (error) { 2168 device_printf(dev, "%s: failed to map QoS key configuration " 2169 "buffer into bus space\n", __func__); 2170 goto err_exit; 2171 } 2172 2173 DPAA2_CMD_INIT(&cmd); 2174 2175 error = DPAA2_CMD_RC_OPEN(dev, child, &cmd, rcinfo->id, &rc_token); 2176 if (error) { 2177 device_printf(dev, "%s: failed to open resource container: " 2178 "id=%d, error=%d\n", __func__, rcinfo->id, error); 2179 goto err_exit; 2180 } 2181 error = DPAA2_CMD_NI_OPEN(dev, child, &cmd, dinfo->id, &ni_token); 2182 if (error) { 2183 device_printf(sc->dev, "%s: failed to open DPMAC: id=%d, " 2184 "error=%d\n", __func__, dinfo->id, error); 2185 goto close_rc; 2186 } 2187 2188 tbl.default_tc = 0; 2189 tbl.discard_on_miss = false; 2190 tbl.keep_entries = false; 2191 tbl.kcfg_busaddr = buf->paddr; 2192 error = DPAA2_CMD_NI_SET_QOS_TABLE(dev, child, &cmd, &tbl); 2193 if (error) { 2194 device_printf(dev, "%s: failed to set QoS table\n", __func__); 2195 goto close_ni; 2196 } 2197 2198 error = DPAA2_CMD_NI_CLEAR_QOS_TABLE(dev, child, &cmd); 2199 if (error) { 2200 device_printf(dev, "%s: failed to clear QoS table\n", __func__); 2201 goto close_ni; 2202 } 2203 2204 error = 0; 2205 close_ni: 2206 (void)DPAA2_CMD_NI_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, ni_token)); 2207 close_rc: 2208 (void)DPAA2_CMD_RC_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, rc_token)); 2209 err_exit: 2210 return (error); 2211 } 2212 2213 static int 2214 dpaa2_ni_set_mac_addr(device_t dev) 2215 { 2216 device_t pdev = device_get_parent(dev); 2217 device_t child = dev; 2218 struct dpaa2_ni_softc *sc = device_get_softc(dev); 2219 if_t ifp = sc->ifp; 2220 struct dpaa2_devinfo *rcinfo = device_get_ivars(pdev); 2221 struct dpaa2_devinfo *dinfo = device_get_ivars(dev); 2222 struct dpaa2_cmd cmd; 2223 struct ether_addr rnd_mac_addr; 2224 uint16_t rc_token, ni_token; 2225 uint8_t mac_addr[ETHER_ADDR_LEN]; 2226 uint8_t dpni_mac_addr[ETHER_ADDR_LEN]; 2227 int error; 2228 2229 DPAA2_CMD_INIT(&cmd); 2230 2231 error = DPAA2_CMD_RC_OPEN(dev, child, &cmd, rcinfo->id, &rc_token); 2232 if (error) { 2233 device_printf(dev, "%s: failed to open resource container: " 2234 "id=%d, error=%d\n", __func__, rcinfo->id, error); 2235 goto err_exit; 2236 } 2237 error = DPAA2_CMD_NI_OPEN(dev, child, &cmd, dinfo->id, &ni_token); 2238 if (error) { 2239 device_printf(sc->dev, "%s: failed to open DPMAC: id=%d, " 2240 "error=%d\n", __func__, dinfo->id, error); 2241 goto close_rc; 2242 } 2243 2244 /* 2245 * Get the MAC address associated with the physical port, if the DPNI is 2246 * connected to a DPMAC directly associated with one of the physical 2247 * ports. 2248 */ 2249 error = DPAA2_CMD_NI_GET_PORT_MAC_ADDR(dev, child, &cmd, mac_addr); 2250 if (error) { 2251 device_printf(dev, "%s: failed to obtain the MAC address " 2252 "associated with the physical port\n", __func__); 2253 goto close_ni; 2254 } 2255 2256 /* Get primary MAC address from the DPNI attributes. */ 2257 error = DPAA2_CMD_NI_GET_PRIM_MAC_ADDR(dev, child, &cmd, dpni_mac_addr); 2258 if (error) { 2259 device_printf(dev, "%s: failed to obtain primary MAC address\n", 2260 __func__); 2261 goto close_ni; 2262 } 2263 2264 if (!ETHER_IS_ZERO(mac_addr)) { 2265 /* Set MAC address of the physical port as DPNI's primary one. */ 2266 error = DPAA2_CMD_NI_SET_PRIM_MAC_ADDR(dev, child, &cmd, 2267 mac_addr); 2268 if (error) { 2269 device_printf(dev, "%s: failed to set primary MAC " 2270 "address\n", __func__); 2271 goto close_ni; 2272 } 2273 for (int i = 0; i < ETHER_ADDR_LEN; i++) { 2274 sc->mac.addr[i] = mac_addr[i]; 2275 } 2276 } else if (ETHER_IS_ZERO(dpni_mac_addr)) { 2277 /* Generate random MAC address as DPNI's primary one. */ 2278 ether_gen_addr(ifp, &rnd_mac_addr); 2279 for (int i = 0; i < ETHER_ADDR_LEN; i++) { 2280 mac_addr[i] = rnd_mac_addr.octet[i]; 2281 } 2282 2283 error = DPAA2_CMD_NI_SET_PRIM_MAC_ADDR(dev, child, &cmd, 2284 mac_addr); 2285 if (error) { 2286 device_printf(dev, "%s: failed to set random primary " 2287 "MAC address\n", __func__); 2288 goto close_ni; 2289 } 2290 for (int i = 0; i < ETHER_ADDR_LEN; i++) { 2291 sc->mac.addr[i] = mac_addr[i]; 2292 } 2293 } else { 2294 for (int i = 0; i < ETHER_ADDR_LEN; i++) { 2295 sc->mac.addr[i] = dpni_mac_addr[i]; 2296 } 2297 } 2298 2299 error = 0; 2300 close_ni: 2301 (void)DPAA2_CMD_NI_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, ni_token)); 2302 close_rc: 2303 (void)DPAA2_CMD_RC_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, rc_token)); 2304 err_exit: 2305 return (error); 2306 } 2307 2308 static void 2309 dpaa2_ni_miibus_statchg(device_t dev) 2310 { 2311 device_t pdev = device_get_parent(dev); 2312 device_t child = dev; 2313 struct dpaa2_ni_softc *sc = device_get_softc(dev); 2314 struct dpaa2_mac_link_state mac_link = { 0 }; 2315 struct dpaa2_devinfo *rcinfo = device_get_ivars(pdev); 2316 struct dpaa2_cmd cmd; 2317 uint16_t rc_token, mac_token; 2318 int error, link_state; 2319 2320 if (sc->fixed_link || sc->mii == NULL) { 2321 return; 2322 } 2323 if ((if_getdrvflags(sc->ifp) & IFF_DRV_RUNNING) == 0) { 2324 /* 2325 * We will receive calls and adjust the changes but 2326 * not have setup everything (called before dpaa2_ni_init() 2327 * really). This will then setup the link and internal 2328 * sc->link_state and not trigger the update once needed, 2329 * so basically dpmac never knows about it. 2330 */ 2331 return; 2332 } 2333 2334 /* 2335 * Note: ifp link state will only be changed AFTER we are called so we 2336 * cannot rely on ifp->if_linkstate here. 2337 */ 2338 if (sc->mii->mii_media_status & IFM_AVALID) { 2339 if (sc->mii->mii_media_status & IFM_ACTIVE) { 2340 link_state = LINK_STATE_UP; 2341 } else { 2342 link_state = LINK_STATE_DOWN; 2343 } 2344 } else { 2345 link_state = LINK_STATE_UNKNOWN; 2346 } 2347 2348 if (link_state != sc->link_state) { 2349 sc->link_state = link_state; 2350 2351 DPAA2_CMD_INIT(&cmd); 2352 2353 error = DPAA2_CMD_RC_OPEN(dev, child, &cmd, rcinfo->id, 2354 &rc_token); 2355 if (error) { 2356 device_printf(dev, "%s: failed to open resource " 2357 "container: id=%d, error=%d\n", __func__, rcinfo->id, 2358 error); 2359 goto err_exit; 2360 } 2361 error = DPAA2_CMD_MAC_OPEN(dev, child, &cmd, sc->mac.dpmac_id, 2362 &mac_token); 2363 if (error) { 2364 device_printf(sc->dev, "%s: failed to open DPMAC: " 2365 "id=%d, error=%d\n", __func__, sc->mac.dpmac_id, 2366 error); 2367 goto close_rc; 2368 } 2369 2370 if (link_state == LINK_STATE_UP || 2371 link_state == LINK_STATE_DOWN) { 2372 /* Update DPMAC link state. */ 2373 mac_link.supported = sc->mii->mii_media.ifm_media; 2374 mac_link.advert = sc->mii->mii_media.ifm_media; 2375 mac_link.rate = 1000; /* TODO: Where to get from? */ /* ifmedia_baudrate? */ 2376 mac_link.options = 2377 DPAA2_MAC_LINK_OPT_AUTONEG | 2378 DPAA2_MAC_LINK_OPT_PAUSE; 2379 mac_link.up = (link_state == LINK_STATE_UP) ? true : false; 2380 mac_link.state_valid = true; 2381 2382 /* Inform DPMAC about link state. */ 2383 error = DPAA2_CMD_MAC_SET_LINK_STATE(dev, child, &cmd, 2384 &mac_link); 2385 if (error) { 2386 device_printf(sc->dev, "%s: failed to set DPMAC " 2387 "link state: id=%d, error=%d\n", __func__, 2388 sc->mac.dpmac_id, error); 2389 } 2390 } 2391 (void)DPAA2_CMD_MAC_CLOSE(dev, child, &cmd); 2392 (void)DPAA2_CMD_RC_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, 2393 rc_token)); 2394 } 2395 2396 return; 2397 2398 close_rc: 2399 (void)DPAA2_CMD_RC_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, rc_token)); 2400 err_exit: 2401 return; 2402 } 2403 2404 /** 2405 * @brief Callback function to process media change request. 2406 */ 2407 static int 2408 dpaa2_ni_media_change_locked(struct dpaa2_ni_softc *sc) 2409 { 2410 2411 DPNI_LOCK_ASSERT(sc); 2412 if (sc->mii) { 2413 mii_mediachg(sc->mii); 2414 sc->media_status = sc->mii->mii_media.ifm_media; 2415 } else if (sc->fixed_link) { 2416 if_printf(sc->ifp, "%s: can't change media in fixed mode\n", 2417 __func__); 2418 } 2419 2420 return (0); 2421 } 2422 2423 static int 2424 dpaa2_ni_media_change(if_t ifp) 2425 { 2426 struct dpaa2_ni_softc *sc = if_getsoftc(ifp); 2427 int error; 2428 2429 DPNI_LOCK(sc); 2430 error = dpaa2_ni_media_change_locked(sc); 2431 DPNI_UNLOCK(sc); 2432 return (error); 2433 } 2434 2435 /** 2436 * @brief Callback function to process media status request. 2437 */ 2438 static void 2439 dpaa2_ni_media_status(if_t ifp, struct ifmediareq *ifmr) 2440 { 2441 struct dpaa2_ni_softc *sc = if_getsoftc(ifp); 2442 2443 DPNI_LOCK(sc); 2444 if (sc->mii) { 2445 mii_pollstat(sc->mii); 2446 ifmr->ifm_active = sc->mii->mii_media_active; 2447 ifmr->ifm_status = sc->mii->mii_media_status; 2448 } 2449 DPNI_UNLOCK(sc); 2450 } 2451 2452 /** 2453 * @brief Callout function to check and update media status. 2454 */ 2455 static void 2456 dpaa2_ni_media_tick(void *arg) 2457 { 2458 struct dpaa2_ni_softc *sc = (struct dpaa2_ni_softc *) arg; 2459 2460 /* Check for media type change */ 2461 if (sc->mii) { 2462 mii_tick(sc->mii); 2463 if (sc->media_status != sc->mii->mii_media.ifm_media) { 2464 printf("%s: media type changed (ifm_media=%x)\n", 2465 __func__, sc->mii->mii_media.ifm_media); 2466 dpaa2_ni_media_change(sc->ifp); 2467 } 2468 } 2469 2470 /* Schedule another timeout one second from now */ 2471 callout_reset(&sc->mii_callout, hz, dpaa2_ni_media_tick, sc); 2472 } 2473 2474 static void 2475 dpaa2_ni_init(void *arg) 2476 { 2477 struct dpaa2_ni_softc *sc = (struct dpaa2_ni_softc *) arg; 2478 if_t ifp = sc->ifp; 2479 device_t pdev = device_get_parent(sc->dev); 2480 device_t dev = sc->dev; 2481 device_t child = dev; 2482 struct dpaa2_devinfo *rcinfo = device_get_ivars(pdev); 2483 struct dpaa2_devinfo *dinfo = device_get_ivars(dev); 2484 struct dpaa2_cmd cmd; 2485 uint16_t rc_token, ni_token; 2486 int error; 2487 2488 DPNI_LOCK(sc); 2489 if ((if_getdrvflags(ifp) & IFF_DRV_RUNNING) != 0) { 2490 DPNI_UNLOCK(sc); 2491 return; 2492 } 2493 DPNI_UNLOCK(sc); 2494 2495 DPAA2_CMD_INIT(&cmd); 2496 2497 error = DPAA2_CMD_RC_OPEN(dev, child, &cmd, rcinfo->id, &rc_token); 2498 if (error) { 2499 device_printf(dev, "%s: failed to open resource container: " 2500 "id=%d, error=%d\n", __func__, rcinfo->id, error); 2501 goto err_exit; 2502 } 2503 error = DPAA2_CMD_NI_OPEN(dev, child, &cmd, dinfo->id, &ni_token); 2504 if (error) { 2505 device_printf(dev, "%s: failed to open network interface: " 2506 "id=%d, error=%d\n", __func__, dinfo->id, error); 2507 goto close_rc; 2508 } 2509 2510 error = DPAA2_CMD_NI_ENABLE(dev, child, &cmd); 2511 if (error) { 2512 device_printf(dev, "%s: failed to enable DPNI: error=%d\n", 2513 __func__, error); 2514 } 2515 2516 error = dpaa2_ni_setup_if_flags(sc); 2517 if (error) { 2518 device_printf(dev, "%s: failed to update interface flags: " 2519 "error=%d\n", __func__, error); 2520 } 2521 error = dpaa2_ni_update_mac_filters(ifp); 2522 if (error) { 2523 device_printf(dev, "%s: failed to update MAC filters: " 2524 "error=%d\n", __func__, error); 2525 } 2526 2527 DPNI_LOCK(sc); 2528 /* Announce we are up and running and can queue packets. */ 2529 if_setdrvflagbits(ifp, IFF_DRV_RUNNING, IFF_DRV_OACTIVE); 2530 2531 if (sc->mii) { 2532 /* 2533 * mii_mediachg() will trigger a call into 2534 * dpaa2_ni_miibus_statchg() to setup link state. 2535 */ 2536 dpaa2_ni_media_change_locked(sc); 2537 } 2538 callout_reset(&sc->mii_callout, hz, dpaa2_ni_media_tick, sc); 2539 2540 DPNI_UNLOCK(sc); 2541 2542 (void)DPAA2_CMD_NI_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, ni_token)); 2543 (void)DPAA2_CMD_RC_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, rc_token)); 2544 return; 2545 2546 close_rc: 2547 (void)DPAA2_CMD_RC_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, rc_token)); 2548 err_exit: 2549 return; 2550 } 2551 2552 static int 2553 dpaa2_ni_transmit(if_t ifp, struct mbuf *m) 2554 { 2555 struct dpaa2_ni_softc *sc = if_getsoftc(ifp); 2556 struct dpaa2_channel *ch; 2557 uint32_t fqid; 2558 bool found = false; 2559 int chidx = 0, error; 2560 2561 if (__predict_false(!(if_getdrvflags(ifp) & IFF_DRV_RUNNING))) { 2562 return (0); 2563 } 2564 2565 if (M_HASHTYPE_GET(m) != M_HASHTYPE_NONE) { 2566 fqid = m->m_pkthdr.flowid; 2567 for (int i = 0; i < sc->chan_n; i++) { 2568 ch = sc->channels[i]; 2569 for (int j = 0; j < ch->rxq_n; j++) { 2570 if (fqid == ch->rx_queues[j].fqid) { 2571 chidx = ch->flowid; 2572 found = true; 2573 break; 2574 } 2575 } 2576 if (found) { 2577 break; 2578 } 2579 } 2580 } 2581 2582 ch = sc->channels[chidx]; 2583 error = buf_ring_enqueue(ch->xmit_br, m); 2584 if (__predict_false(error != 0)) { 2585 if_inc_counter(ifp, IFCOUNTER_OQDROPS, 1); 2586 m_freem(m); 2587 } else { 2588 taskqueue_enqueue(ch->cleanup_tq, &ch->cleanup_task); 2589 } 2590 2591 return (error); 2592 } 2593 2594 static void 2595 dpaa2_ni_qflush(if_t ifp) 2596 { 2597 /* TODO: Find a way to drain Tx queues in QBMan. */ 2598 if_qflush(ifp); 2599 } 2600 2601 static int 2602 dpaa2_ni_ioctl(if_t ifp, u_long c, caddr_t data) 2603 { 2604 struct dpaa2_ni_softc *sc = if_getsoftc(ifp); 2605 struct ifreq *ifr = (struct ifreq *) data; 2606 device_t pdev = device_get_parent(sc->dev); 2607 device_t dev = sc->dev; 2608 device_t child = dev; 2609 struct dpaa2_devinfo *rcinfo = device_get_ivars(pdev); 2610 struct dpaa2_devinfo *dinfo = device_get_ivars(dev); 2611 struct dpaa2_cmd cmd; 2612 uint32_t changed = 0; 2613 uint16_t rc_token, ni_token; 2614 int mtu, error, rc = 0; 2615 2616 DPAA2_CMD_INIT(&cmd); 2617 2618 error = DPAA2_CMD_RC_OPEN(dev, child, &cmd, rcinfo->id, &rc_token); 2619 if (error) { 2620 device_printf(dev, "%s: failed to open resource container: " 2621 "id=%d, error=%d\n", __func__, rcinfo->id, error); 2622 goto err_exit; 2623 } 2624 error = DPAA2_CMD_NI_OPEN(dev, child, &cmd, dinfo->id, &ni_token); 2625 if (error) { 2626 device_printf(dev, "%s: failed to open network interface: " 2627 "id=%d, error=%d\n", __func__, dinfo->id, error); 2628 goto close_rc; 2629 } 2630 2631 switch (c) { 2632 case SIOCSIFMTU: 2633 DPNI_LOCK(sc); 2634 mtu = ifr->ifr_mtu; 2635 if (mtu < ETHERMIN || mtu > ETHERMTU_JUMBO) { 2636 DPNI_UNLOCK(sc); 2637 error = EINVAL; 2638 goto close_ni; 2639 } 2640 if_setmtu(ifp, mtu); 2641 DPNI_UNLOCK(sc); 2642 2643 /* Update maximum frame length. */ 2644 mtu += ETHER_HDR_LEN; 2645 if (if_getcapenable(ifp) & IFCAP_VLAN_MTU) 2646 mtu += ETHER_VLAN_ENCAP_LEN; 2647 error = DPAA2_CMD_NI_SET_MFL(dev, child, &cmd, mtu); 2648 if (error) { 2649 device_printf(dev, "%s: failed to update maximum frame " 2650 "length: error=%d\n", __func__, error); 2651 goto close_ni; 2652 } 2653 break; 2654 case SIOCSIFCAP: 2655 changed = if_getcapenable(ifp) ^ ifr->ifr_reqcap; 2656 if ((changed & (IFCAP_RXCSUM | IFCAP_RXCSUM_IPV6)) != 0) 2657 if_togglecapenable(ifp, IFCAP_RXCSUM | IFCAP_RXCSUM_IPV6); 2658 if ((changed & (IFCAP_TXCSUM | IFCAP_TXCSUM_IPV6)) != 0) { 2659 if_togglecapenable(ifp, IFCAP_TXCSUM | IFCAP_TXCSUM_IPV6); 2660 if_togglehwassist(ifp, DPAA2_CSUM_TX_OFFLOAD); 2661 } 2662 2663 rc = dpaa2_ni_setup_if_caps(sc); 2664 if (rc) { 2665 printf("%s: failed to update iface capabilities: " 2666 "error=%d\n", __func__, rc); 2667 rc = ENXIO; 2668 } 2669 break; 2670 case SIOCSIFFLAGS: 2671 DPNI_LOCK(sc); 2672 if (if_getflags(ifp) & IFF_UP) { 2673 if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) { 2674 changed = if_getflags(ifp) ^ sc->if_flags; 2675 if (changed & IFF_PROMISC || 2676 changed & IFF_ALLMULTI) { 2677 rc = dpaa2_ni_setup_if_flags(sc); 2678 } 2679 } else { 2680 DPNI_UNLOCK(sc); 2681 dpaa2_ni_init(sc); 2682 DPNI_LOCK(sc); 2683 } 2684 } else if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) { 2685 /* FIXME: Disable DPNI. See dpaa2_ni_init(). */ 2686 } 2687 2688 sc->if_flags = if_getflags(ifp); 2689 DPNI_UNLOCK(sc); 2690 break; 2691 case SIOCADDMULTI: 2692 case SIOCDELMULTI: 2693 DPNI_LOCK(sc); 2694 if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) { 2695 DPNI_UNLOCK(sc); 2696 rc = dpaa2_ni_update_mac_filters(ifp); 2697 if (rc) { 2698 device_printf(dev, "%s: failed to update MAC " 2699 "filters: error=%d\n", __func__, rc); 2700 } 2701 DPNI_LOCK(sc); 2702 } 2703 DPNI_UNLOCK(sc); 2704 break; 2705 case SIOCGIFMEDIA: 2706 case SIOCSIFMEDIA: 2707 if (sc->mii) 2708 rc = ifmedia_ioctl(ifp, ifr, &sc->mii->mii_media, c); 2709 else if(sc->fixed_link) { 2710 rc = ifmedia_ioctl(ifp, ifr, &sc->fixed_ifmedia, c); 2711 } 2712 break; 2713 default: 2714 rc = ether_ioctl(ifp, c, data); 2715 break; 2716 } 2717 2718 (void)DPAA2_CMD_NI_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, ni_token)); 2719 (void)DPAA2_CMD_RC_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, rc_token)); 2720 return (rc); 2721 2722 close_ni: 2723 (void)DPAA2_CMD_NI_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, ni_token)); 2724 close_rc: 2725 (void)DPAA2_CMD_RC_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, rc_token)); 2726 err_exit: 2727 return (error); 2728 } 2729 2730 static int 2731 dpaa2_ni_update_mac_filters(if_t ifp) 2732 { 2733 struct dpaa2_ni_softc *sc = if_getsoftc(ifp); 2734 struct dpaa2_ni_mcaddr_ctx ctx; 2735 device_t pdev = device_get_parent(sc->dev); 2736 device_t dev = sc->dev; 2737 device_t child = dev; 2738 struct dpaa2_devinfo *rcinfo = device_get_ivars(pdev); 2739 struct dpaa2_devinfo *dinfo = device_get_ivars(dev); 2740 struct dpaa2_cmd cmd; 2741 uint16_t rc_token, ni_token; 2742 int error; 2743 2744 DPAA2_CMD_INIT(&cmd); 2745 2746 error = DPAA2_CMD_RC_OPEN(dev, child, &cmd, rcinfo->id, &rc_token); 2747 if (error) { 2748 device_printf(dev, "%s: failed to open resource container: " 2749 "id=%d, error=%d\n", __func__, rcinfo->id, error); 2750 goto err_exit; 2751 } 2752 error = DPAA2_CMD_NI_OPEN(dev, child, &cmd, dinfo->id, &ni_token); 2753 if (error) { 2754 device_printf(dev, "%s: failed to open network interface: " 2755 "id=%d, error=%d\n", __func__, dinfo->id, error); 2756 goto close_rc; 2757 } 2758 2759 /* Remove all multicast MAC filters. */ 2760 error = DPAA2_CMD_NI_CLEAR_MAC_FILTERS(dev, child, &cmd, false, true); 2761 if (error) { 2762 device_printf(dev, "%s: failed to clear multicast MAC filters: " 2763 "error=%d\n", __func__, error); 2764 goto close_ni; 2765 } 2766 2767 ctx.ifp = ifp; 2768 ctx.error = 0; 2769 ctx.nent = 0; 2770 2771 if_foreach_llmaddr(ifp, dpaa2_ni_add_maddr, &ctx); 2772 2773 error = ctx.error; 2774 close_ni: 2775 (void)DPAA2_CMD_NI_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, ni_token)); 2776 close_rc: 2777 (void)DPAA2_CMD_RC_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, rc_token)); 2778 err_exit: 2779 return (error); 2780 } 2781 2782 static u_int 2783 dpaa2_ni_add_maddr(void *arg, struct sockaddr_dl *sdl, u_int cnt) 2784 { 2785 struct dpaa2_ni_mcaddr_ctx *ctx = arg; 2786 struct dpaa2_ni_softc *sc = if_getsoftc(ctx->ifp); 2787 device_t pdev = device_get_parent(sc->dev); 2788 device_t dev = sc->dev; 2789 device_t child = dev; 2790 struct dpaa2_devinfo *rcinfo = device_get_ivars(pdev); 2791 struct dpaa2_devinfo *dinfo = device_get_ivars(dev); 2792 struct dpaa2_cmd cmd; 2793 uint16_t rc_token, ni_token; 2794 int error; 2795 2796 if (ctx->error != 0) { 2797 return (0); 2798 } 2799 2800 if (ETHER_IS_MULTICAST(LLADDR(sdl))) { 2801 DPAA2_CMD_INIT(&cmd); 2802 2803 error = DPAA2_CMD_RC_OPEN(dev, child, &cmd, rcinfo->id, 2804 &rc_token); 2805 if (error) { 2806 device_printf(dev, "%s: failed to open resource " 2807 "container: id=%d, error=%d\n", __func__, rcinfo->id, 2808 error); 2809 return (0); 2810 } 2811 error = DPAA2_CMD_NI_OPEN(dev, child, &cmd, dinfo->id, 2812 &ni_token); 2813 if (error) { 2814 device_printf(dev, "%s: failed to open network interface: " 2815 "id=%d, error=%d\n", __func__, dinfo->id, error); 2816 (void)DPAA2_CMD_RC_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, 2817 rc_token)); 2818 return (0); 2819 } 2820 2821 ctx->error = DPAA2_CMD_NI_ADD_MAC_ADDR(dev, child, &cmd, 2822 LLADDR(sdl)); 2823 2824 (void)DPAA2_CMD_NI_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, 2825 ni_token)); 2826 (void)DPAA2_CMD_RC_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, 2827 rc_token)); 2828 2829 if (ctx->error != 0) { 2830 device_printf(dev, "%s: can't add more then %d MAC " 2831 "addresses, switching to the multicast promiscuous " 2832 "mode\n", __func__, ctx->nent); 2833 2834 /* Enable multicast promiscuous mode. */ 2835 DPNI_LOCK(sc); 2836 if_setflagbits(ctx->ifp, IFF_ALLMULTI, 0); 2837 sc->if_flags |= IFF_ALLMULTI; 2838 ctx->error = dpaa2_ni_setup_if_flags(sc); 2839 DPNI_UNLOCK(sc); 2840 2841 return (0); 2842 } 2843 ctx->nent++; 2844 } 2845 2846 return (1); 2847 } 2848 2849 static void 2850 dpaa2_ni_intr(void *arg) 2851 { 2852 struct dpaa2_ni_softc *sc = (struct dpaa2_ni_softc *) arg; 2853 device_t pdev = device_get_parent(sc->dev); 2854 device_t dev = sc->dev; 2855 device_t child = dev; 2856 struct dpaa2_devinfo *rcinfo = device_get_ivars(pdev); 2857 struct dpaa2_devinfo *dinfo = device_get_ivars(dev); 2858 struct dpaa2_cmd cmd; 2859 uint32_t status = ~0u; /* clear all IRQ status bits */ 2860 uint16_t rc_token, ni_token; 2861 int error; 2862 2863 DPAA2_CMD_INIT(&cmd); 2864 2865 error = DPAA2_CMD_RC_OPEN(dev, child, &cmd, rcinfo->id, &rc_token); 2866 if (error) { 2867 device_printf(dev, "%s: failed to open resource container: " 2868 "id=%d, error=%d\n", __func__, rcinfo->id, error); 2869 goto err_exit; 2870 } 2871 error = DPAA2_CMD_NI_OPEN(dev, child, &cmd, dinfo->id, &ni_token); 2872 if (error) { 2873 device_printf(dev, "%s: failed to open network interface: " 2874 "id=%d, error=%d\n", __func__, dinfo->id, error); 2875 goto close_rc; 2876 } 2877 2878 error = DPAA2_CMD_NI_GET_IRQ_STATUS(dev, child, &cmd, DPNI_IRQ_INDEX, 2879 &status); 2880 if (error) { 2881 device_printf(sc->dev, "%s: failed to obtain IRQ status: " 2882 "error=%d\n", __func__, error); 2883 } 2884 2885 (void)DPAA2_CMD_NI_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, ni_token)); 2886 close_rc: 2887 (void)DPAA2_CMD_RC_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, rc_token)); 2888 err_exit: 2889 return; 2890 } 2891 2892 /** 2893 * @brief Execute channel's Rx/Tx routines. 2894 * 2895 * NOTE: Should not be re-entrant for the same channel. It is achieved by 2896 * enqueuing the cleanup routine on a single-threaded taskqueue. 2897 */ 2898 static void 2899 dpaa2_ni_cleanup_task(void *arg, int count) 2900 { 2901 struct dpaa2_channel *ch = (struct dpaa2_channel *)arg; 2902 struct dpaa2_ni_softc *sc = device_get_softc(ch->ni_dev); 2903 const int clean_budget = DPAA2_ATOMIC_READ(&sc->clean_budget); 2904 const int tx_budget = DPAA2_ATOMIC_READ(&sc->tx_budget); 2905 const int rx_budget = DPAA2_ATOMIC_READ(&sc->rx_budget); 2906 int error, rxc, txc; 2907 2908 for (int i = 0; i < clean_budget; i++) { 2909 rxc = dpaa2_ni_rx_cleanup(ch, rx_budget); 2910 txc = dpaa2_ni_tx_cleanup(ch, tx_budget); 2911 2912 if (__predict_false((if_getdrvflags(sc->ifp) & 2913 IFF_DRV_RUNNING) == 0)) { 2914 return; 2915 } 2916 2917 if ((txc != tx_budget) && (rxc != rx_budget)) { 2918 break; 2919 } 2920 } 2921 2922 /* Re-arm channel to generate CDAN */ 2923 error = DPAA2_SWP_CONF_WQ_CHANNEL(ch->io_dev, &ch->ctx); 2924 if (error != 0) { 2925 panic("%s: failed to rearm channel: chan_id=%d, error=%d\n", 2926 __func__, ch->id, error); 2927 } 2928 } 2929 2930 /** 2931 * @brief Poll frames from a specific channel when CDAN is received. 2932 */ 2933 static int 2934 dpaa2_ni_rx_cleanup(struct dpaa2_channel *ch, const int budget) 2935 { 2936 struct dpaa2_io_softc *iosc = device_get_softc(ch->io_dev); 2937 struct dpaa2_swp *swp = iosc->swp; 2938 struct dpaa2_ni_fq *fq; 2939 struct dpaa2_buf *buf = &ch->store; 2940 int budget_remaining = budget; 2941 int error, consumed = 0; 2942 2943 do { 2944 error = dpaa2_swp_pull(swp, ch->id, buf, DPAA2_ETH_STORE_FRAMES); 2945 if (error) { 2946 device_printf(ch->ni_dev, "%s: failed to pull frames: " 2947 "chan_id=%d, error=%d\n", __func__, ch->id, error); 2948 break; 2949 } 2950 error = dpaa2_ni_consume_frames(ch, &fq, &consumed); 2951 if (error == ENOENT || error == EALREADY) { 2952 break; 2953 } 2954 if (error == ETIMEDOUT) { 2955 device_printf(ch->ni_dev, "%s: timeout to consume " 2956 "frames: chan_id=%d\n", __func__, ch->id); 2957 } 2958 } while (--budget_remaining ); 2959 2960 return (budget - budget_remaining); 2961 } 2962 2963 static int 2964 dpaa2_ni_tx_cleanup(struct dpaa2_channel *ch, const int budget) 2965 { 2966 struct dpaa2_ni_softc *sc = device_get_softc(ch->ni_dev); 2967 struct dpaa2_ni_tx_ring *tx = &ch->txc_queue.tx_rings[0]; 2968 struct mbuf *m = NULL; 2969 int budget_remaining = budget; 2970 2971 do { 2972 mtx_assert(&ch->xmit_mtx, MA_NOTOWNED); 2973 mtx_lock(&ch->xmit_mtx); 2974 m = buf_ring_dequeue_sc(ch->xmit_br); 2975 mtx_unlock(&ch->xmit_mtx); 2976 2977 if (__predict_false(m == NULL)) { 2978 /* TODO: Do not give up easily */ 2979 break; 2980 } else { 2981 dpaa2_ni_tx(sc, ch, tx, m); 2982 } 2983 } while (--budget_remaining); 2984 2985 return (budget - budget_remaining); 2986 } 2987 2988 static void 2989 dpaa2_ni_tx(struct dpaa2_ni_softc *sc, struct dpaa2_channel *ch, 2990 struct dpaa2_ni_tx_ring *tx, struct mbuf *m) 2991 { 2992 device_t dev = sc->dev; 2993 struct dpaa2_ni_fq *fq = tx->fq; 2994 struct dpaa2_buf *buf, *sgt; 2995 struct dpaa2_fd fd; 2996 struct mbuf *md; 2997 bus_dma_segment_t segs[DPAA2_TX_SEGLIMIT]; 2998 int rc, nsegs; 2999 int error; 3000 int len; 3001 bool mcast; 3002 3003 mtx_assert(&tx->lock, MA_NOTOWNED); 3004 mtx_lock(&tx->lock); 3005 buf = buf_ring_dequeue_sc(tx->br); 3006 mtx_unlock(&tx->lock); 3007 if (__predict_false(buf == NULL)) { 3008 /* TODO: Do not give up easily */ 3009 m_freem(m); 3010 return; 3011 } else { 3012 DPAA2_BUF_ASSERT_TXREADY(buf); 3013 buf->m = m; 3014 sgt = buf->sgt; 3015 } 3016 len = m->m_pkthdr.len; 3017 mcast = (m->m_flags & M_MCAST) != 0; 3018 3019 #if defined(INVARIANTS) 3020 struct dpaa2_ni_tx_ring *btx = (struct dpaa2_ni_tx_ring *)buf->opt; 3021 KASSERT(buf->opt == tx, ("%s: unexpected Tx ring", __func__)); 3022 KASSERT(btx->fq->chan == ch, ("%s: unexpected channel", __func__)); 3023 #endif /* INVARIANTS */ 3024 3025 BPF_MTAP(sc->ifp, m); 3026 3027 error = bus_dmamap_load_mbuf_sg(buf->dmat, buf->dmap, m, segs, &nsegs, 3028 BUS_DMA_NOWAIT); 3029 if (__predict_false(error != 0)) { 3030 /* Too many fragments, trying to defragment... */ 3031 md = m_collapse(m, M_NOWAIT, DPAA2_TX_SEGLIMIT); 3032 if (md == NULL) { 3033 device_printf(dev, "%s: m_collapse() failed\n", __func__); 3034 fq->chan->tx_dropped++; 3035 if_inc_counter(sc->ifp, IFCOUNTER_OERRORS, 1); 3036 goto err; 3037 } 3038 3039 buf->m = m = md; 3040 error = bus_dmamap_load_mbuf_sg(buf->dmat, buf->dmap, m, segs, 3041 &nsegs, BUS_DMA_NOWAIT); 3042 if (__predict_false(error != 0)) { 3043 device_printf(dev, "%s: bus_dmamap_load_mbuf_sg() " 3044 "failed: error=%d\n", __func__, error); 3045 fq->chan->tx_dropped++; 3046 if_inc_counter(sc->ifp, IFCOUNTER_OERRORS, 1); 3047 goto err; 3048 } 3049 } 3050 3051 error = dpaa2_fd_build(dev, sc->tx_data_off, buf, segs, nsegs, &fd); 3052 if (__predict_false(error != 0)) { 3053 device_printf(dev, "%s: failed to build frame descriptor: " 3054 "error=%d\n", __func__, error); 3055 fq->chan->tx_dropped++; 3056 if_inc_counter(sc->ifp, IFCOUNTER_OERRORS, 1); 3057 goto err_unload; 3058 } else 3059 sc->tx_sg_frames++; /* for sysctl(9) */ 3060 3061 bus_dmamap_sync(buf->dmat, buf->dmap, BUS_DMASYNC_PREWRITE); 3062 bus_dmamap_sync(sgt->dmat, sgt->dmap, BUS_DMASYNC_PREWRITE); 3063 3064 /* TODO: Enqueue several frames in a single command */ 3065 for (int i = 0; i < DPAA2_NI_ENQUEUE_RETRIES; i++) { 3066 /* TODO: Return error codes instead of # of frames */ 3067 rc = DPAA2_SWP_ENQ_MULTIPLE_FQ(fq->chan->io_dev, tx->fqid, &fd, 1); 3068 if (rc == 1) { 3069 break; 3070 } 3071 } 3072 3073 if (rc != 1) { 3074 fq->chan->tx_dropped++; 3075 if_inc_counter(sc->ifp, IFCOUNTER_OERRORS, 1); 3076 goto err_unload; 3077 } else { 3078 if (mcast) 3079 if_inc_counter(sc->ifp, IFCOUNTER_OMCASTS, 1); 3080 if_inc_counter(sc->ifp, IFCOUNTER_OPACKETS, 1); 3081 if_inc_counter(sc->ifp, IFCOUNTER_OBYTES, len); 3082 fq->chan->tx_frames++; 3083 } 3084 return; 3085 3086 err_unload: 3087 bus_dmamap_unload(buf->dmat, buf->dmap); 3088 if (sgt->paddr != 0) { 3089 bus_dmamap_unload(sgt->dmat, sgt->dmap); 3090 } 3091 err: 3092 m_freem(buf->m); 3093 buf_ring_enqueue(tx->br, buf); 3094 } 3095 3096 static int 3097 dpaa2_ni_consume_frames(struct dpaa2_channel *chan, struct dpaa2_ni_fq **src, 3098 uint32_t *consumed) 3099 { 3100 struct dpaa2_ni_fq *fq = NULL; 3101 struct dpaa2_dq *dq; 3102 struct dpaa2_fd *fd; 3103 struct dpaa2_ni_rx_ctx ctx = { 3104 .head = NULL, 3105 .tail = NULL, 3106 .cnt = 0, 3107 .last = false 3108 }; 3109 int rc, frames = 0; 3110 3111 do { 3112 rc = dpaa2_chan_next_frame(chan, &dq); 3113 if (rc == EINPROGRESS) { 3114 if (dq != NULL && !IS_NULL_RESPONSE(dq->fdr.desc.stat)) { 3115 fd = &dq->fdr.fd; 3116 fq = (struct dpaa2_ni_fq *) dq->fdr.desc.fqd_ctx; 3117 3118 switch (fq->type) { 3119 case DPAA2_NI_QUEUE_RX: 3120 (void)dpaa2_ni_rx(chan, fq, fd, &ctx); 3121 break; 3122 case DPAA2_NI_QUEUE_RX_ERR: 3123 (void)dpaa2_ni_rx_err(chan, fq, fd); 3124 break; 3125 case DPAA2_NI_QUEUE_TX_CONF: 3126 (void)dpaa2_ni_tx_conf(chan, fq, fd); 3127 break; 3128 default: 3129 panic("%s: unknown queue type (1)", 3130 __func__); 3131 } 3132 frames++; 3133 } 3134 } else if (rc == EALREADY || rc == ENOENT) { 3135 if (dq != NULL && !IS_NULL_RESPONSE(dq->fdr.desc.stat)) { 3136 fd = &dq->fdr.fd; 3137 fq = (struct dpaa2_ni_fq *) dq->fdr.desc.fqd_ctx; 3138 3139 switch (fq->type) { 3140 case DPAA2_NI_QUEUE_RX: 3141 /* 3142 * Last VDQ response (mbuf) in a chain 3143 * obtained from the Rx queue. 3144 */ 3145 ctx.last = true; 3146 (void)dpaa2_ni_rx(chan, fq, fd, &ctx); 3147 break; 3148 case DPAA2_NI_QUEUE_RX_ERR: 3149 (void)dpaa2_ni_rx_err(chan, fq, fd); 3150 break; 3151 case DPAA2_NI_QUEUE_TX_CONF: 3152 (void)dpaa2_ni_tx_conf(chan, fq, fd); 3153 break; 3154 default: 3155 panic("%s: unknown queue type (2)", 3156 __func__); 3157 } 3158 frames++; 3159 } 3160 break; 3161 } else { 3162 panic("%s: should not reach here: rc=%d", __func__, rc); 3163 } 3164 } while (true); 3165 3166 KASSERT(chan->store_idx < chan->store_sz, ("%s: store_idx(%d) >= " 3167 "store_sz(%d)", __func__, chan->store_idx, chan->store_sz)); 3168 3169 /* 3170 * VDQ operation pulls frames from a single queue into the store. 3171 * Return the frame queue and a number of consumed frames as an output. 3172 */ 3173 if (src != NULL) { 3174 *src = fq; 3175 } 3176 if (consumed != NULL) { 3177 *consumed = frames; 3178 } 3179 3180 return (rc); 3181 } 3182 3183 /** 3184 * @brief Receive frames. 3185 */ 3186 static int 3187 dpaa2_ni_rx(struct dpaa2_channel *ch, struct dpaa2_ni_fq *fq, 3188 struct dpaa2_fd *fd, struct dpaa2_ni_rx_ctx *ctx) 3189 { 3190 bus_addr_t paddr; 3191 struct dpaa2_swa *swa; 3192 struct dpaa2_buf *buf; 3193 struct dpaa2_bufext_rx *bext; 3194 struct dpaa2_channel *bch; 3195 struct dpaa2_ni_softc *sc; 3196 struct dpaa2_bp_softc *bpsc; 3197 struct mbuf *m; 3198 device_t bpdev; 3199 bus_addr_t released[DPAA2_SWP_BUFS_PER_CMD]; 3200 void *buf_data; 3201 int buf_len, error, released_n = 0; 3202 bool update_csum_flags; 3203 3204 error = dpaa2_fa_get_swa(fd, &swa); 3205 if (__predict_false(error != 0)) 3206 panic("%s: frame has no software annotation: error=%d", 3207 __func__, error); 3208 3209 paddr = (bus_addr_t)fd->addr; 3210 buf = swa->buf; 3211 bext = (struct dpaa2_bufext_rx *)buf->opt; 3212 bch = bext->ch; 3213 sc = device_get_softc(bch->ni_dev); 3214 update_csum_flags = true; 3215 3216 KASSERT(swa->magic == DPAA2_MAGIC, ("%s: wrong magic", __func__)); 3217 /* 3218 * NOTE: Current channel might not be the same as the "buffer" channel 3219 * and it's fine. It must not be NULL though. 3220 */ 3221 KASSERT(bch != NULL, ("%s: buffer channel is NULL", __func__)); 3222 3223 if (__predict_false(paddr != buf->paddr)) { 3224 panic("%s: unexpected physical address: fd(%#jx) != buf(%#jx)", 3225 __func__, paddr, buf->paddr); 3226 } 3227 3228 switch (dpaa2_fd_err(fd)) { 3229 case 0: 3230 /* 3231 * FD[ERR] = 0 value is reserved to indicate that there is no 3232 * error encoded in this field. See 3.4.5 Error handling, 3233 * LX2160A DPAA2 Low-Level Hardware Reference Manual, Rev. 0, 3234 * 06/2020. 3235 */ 3236 break; 3237 case 1: /* Enqueue rejected by QMan */ 3238 sc->rx_enq_rej_frames++; 3239 break; 3240 case 2: /* QMan IEOI error */ 3241 sc->rx_ieoi_err_frames++; 3242 break; 3243 default: 3244 sc->rx_other_err_frames++; 3245 break; 3246 } 3247 3248 switch (dpaa2_fd_format(fd)) { 3249 case DPAA2_FD_SINGLE: 3250 sc->rx_single_buf_frames++; 3251 break; 3252 case DPAA2_FD_SG: 3253 sc->rx_sg_buf_frames++; 3254 break; 3255 default: 3256 update_csum_flags = false; 3257 break; 3258 } 3259 3260 mtx_assert(&bext->dma_mtx, MA_NOTOWNED); 3261 mtx_lock(&bext->dma_mtx); 3262 3263 bus_dmamap_sync(buf->dmat, buf->dmap, BUS_DMASYNC_POSTREAD); 3264 bus_dmamap_unload(buf->dmat, buf->dmap); 3265 3266 m = buf->m; 3267 buf_len = dpaa2_fd_data_len(fd); 3268 buf_data = (uint8_t *)buf->vaddr + dpaa2_fd_offset(fd); 3269 3270 /* Prepare buffer to be re-cycled */ 3271 buf->m = NULL; 3272 buf->paddr = 0; 3273 buf->vaddr = NULL; 3274 buf->seg.ds_addr = 0; 3275 buf->seg.ds_len = 0; 3276 buf->nseg = 0; 3277 3278 mtx_unlock(&bext->dma_mtx); 3279 3280 m->m_flags |= M_PKTHDR; 3281 m->m_data = buf_data; 3282 m->m_len = buf_len; 3283 m->m_pkthdr.len = buf_len; 3284 m->m_pkthdr.rcvif = sc->ifp; 3285 m->m_pkthdr.flowid = fq->fqid; 3286 M_HASHTYPE_SET(m, M_HASHTYPE_OPAQUE); 3287 if_inc_counter(sc->ifp, IFCOUNTER_IPACKETS, 1); 3288 3289 if (update_csum_flags && ((if_getcapenable(sc->ifp) & (IFCAP_RXCSUM | 3290 IFCAP_RXCSUM_IPV6)) != 0)) { 3291 error = dpaa2_ni_update_csum_flags(fd, m); 3292 if (error != 0) 3293 device_printf(sc->dev, "%s: failed to update checksum " 3294 "flags: error=%d\n", __func__, error); 3295 } 3296 3297 if (ctx->head == NULL) { 3298 KASSERT(ctx->tail == NULL, ("%s: tail already given?", __func__)); 3299 ctx->head = m; 3300 ctx->tail = m; 3301 } else { 3302 KASSERT(ctx->head != NULL, ("%s: head is NULL", __func__)); 3303 ctx->tail->m_nextpkt = m; 3304 ctx->tail = m; 3305 } 3306 ctx->cnt++; 3307 3308 if (ctx->last) { 3309 ctx->tail->m_nextpkt = NULL; 3310 if_input(sc->ifp, ctx->head); 3311 } 3312 3313 /* Keep the buffer to be recycled */ 3314 ch->recycled[ch->recycled_n++] = buf; 3315 3316 /* Re-seed and release recycled buffers back to the pool */ 3317 if (ch->recycled_n == DPAA2_SWP_BUFS_PER_CMD) { 3318 /* Release new buffers to the pool if needed */ 3319 taskqueue_enqueue(sc->bp_taskq, &ch->bp_task); 3320 3321 for (int i = 0; i < ch->recycled_n; i++) { 3322 buf = ch->recycled[i]; 3323 bext = (struct dpaa2_bufext_rx *)buf->opt; 3324 bch = bext->ch; 3325 3326 mtx_assert(&bext->dma_mtx, MA_NOTOWNED); 3327 mtx_lock(&bext->dma_mtx); 3328 error = dpaa2_buf_seed_rxb(sc->dev, buf, 3329 DPAA2_RX_BUF_SIZE); 3330 mtx_unlock(&bext->dma_mtx); 3331 3332 if (__predict_false(error != 0)) { 3333 /* TODO: What else to do with the buffer? */ 3334 panic("%s: failed to recycle buffer: error=%d", 3335 __func__, error); 3336 } 3337 3338 /* Prepare buffer to be released in a single command */ 3339 released[released_n++] = buf->paddr; 3340 } 3341 3342 /* There's only one buffer pool for now */ 3343 bpdev = (device_t)rman_get_start(sc->res[DPAA2_NI_BP_RID(0)]); 3344 bpsc = device_get_softc(bpdev); 3345 3346 error = DPAA2_SWP_RELEASE_BUFS(ch->io_dev, bpsc->attr.bpid, 3347 released, released_n); 3348 if (__predict_false(error != 0)) { 3349 device_printf(sc->dev, "%s: failed to release buffers " 3350 "to the pool: error=%d\n", __func__, error); 3351 return (error); 3352 } 3353 ch->recycled_n = 0; 3354 } 3355 3356 return (0); 3357 } 3358 3359 /** 3360 * @brief Receive Rx error frames. 3361 */ 3362 static int 3363 dpaa2_ni_rx_err(struct dpaa2_channel *ch, struct dpaa2_ni_fq *fq, 3364 struct dpaa2_fd *fd) 3365 { 3366 bus_addr_t paddr; 3367 struct dpaa2_swa *swa; 3368 struct dpaa2_buf *buf; 3369 struct dpaa2_bufext_rx *bext; 3370 struct dpaa2_channel *bch; 3371 struct dpaa2_ni_softc *sc; 3372 device_t bpdev; 3373 struct dpaa2_bp_softc *bpsc; 3374 int error; 3375 3376 error = dpaa2_fa_get_swa(fd, &swa); 3377 if (__predict_false(error != 0)) 3378 panic("%s: frame has no software annotation: error=%d", 3379 __func__, error); 3380 3381 paddr = (bus_addr_t)fd->addr; 3382 buf = swa->buf; 3383 bext = (struct dpaa2_bufext_rx *)buf->opt; 3384 bch = bext->ch; 3385 sc = device_get_softc(bch->ni_dev); 3386 3387 KASSERT(swa->magic == DPAA2_MAGIC, ("%s: wrong magic", __func__)); 3388 /* 3389 * NOTE: Current channel might not be the same as the "buffer" channel 3390 * and it's fine. It must not be NULL though. 3391 */ 3392 KASSERT(bch != NULL, ("%s: buffer channel is NULL", __func__)); 3393 3394 if (__predict_false(paddr != buf->paddr)) { 3395 panic("%s: unexpected physical address: fd(%#jx) != buf(%#jx)", 3396 __func__, paddr, buf->paddr); 3397 } 3398 3399 /* There's only one buffer pool for now */ 3400 bpdev = (device_t)rman_get_start(sc->res[DPAA2_NI_BP_RID(0)]); 3401 bpsc = device_get_softc(bpdev); 3402 3403 /* Release buffer to QBMan buffer pool */ 3404 error = DPAA2_SWP_RELEASE_BUFS(ch->io_dev, bpsc->attr.bpid, &paddr, 1); 3405 if (error != 0) { 3406 device_printf(sc->dev, "%s: failed to release frame buffer to " 3407 "the pool: error=%d\n", __func__, error); 3408 return (error); 3409 } 3410 3411 return (0); 3412 } 3413 3414 /** 3415 * @brief Receive Tx confirmation frames. 3416 */ 3417 static int 3418 dpaa2_ni_tx_conf(struct dpaa2_channel *ch, struct dpaa2_ni_fq *fq, 3419 struct dpaa2_fd *fd) 3420 { 3421 bus_addr_t paddr; 3422 struct dpaa2_swa *swa; 3423 struct dpaa2_buf *buf; 3424 struct dpaa2_buf *sgt; 3425 struct dpaa2_ni_tx_ring *tx; 3426 struct dpaa2_channel *bch; 3427 int error; 3428 3429 error = dpaa2_fa_get_swa(fd, &swa); 3430 if (__predict_false(error != 0)) 3431 panic("%s: frame has no software annotation: error=%d", 3432 __func__, error); 3433 3434 paddr = (bus_addr_t)fd->addr; 3435 buf = swa->buf; 3436 sgt = buf->sgt; 3437 tx = (struct dpaa2_ni_tx_ring *)buf->opt; 3438 bch = tx->fq->chan; 3439 3440 KASSERT(swa->magic == DPAA2_MAGIC, ("%s: wrong magic", __func__)); 3441 KASSERT(tx != NULL, ("%s: Tx ring is NULL", __func__)); 3442 KASSERT(sgt != NULL, ("%s: S/G table is NULL", __func__)); 3443 /* 3444 * NOTE: Current channel might not be the same as the "buffer" channel 3445 * and it's fine. It must not be NULL though. 3446 */ 3447 KASSERT(bch != NULL, ("%s: buffer channel is NULL", __func__)); 3448 3449 if (paddr != buf->paddr) { 3450 panic("%s: unexpected physical address: fd(%#jx) != buf(%#jx)", 3451 __func__, paddr, buf->paddr); 3452 } 3453 3454 mtx_assert(&bch->dma_mtx, MA_NOTOWNED); 3455 mtx_lock(&bch->dma_mtx); 3456 3457 bus_dmamap_sync(buf->dmat, buf->dmap, BUS_DMASYNC_POSTWRITE); 3458 bus_dmamap_sync(sgt->dmat, sgt->dmap, BUS_DMASYNC_POSTWRITE); 3459 bus_dmamap_unload(buf->dmat, buf->dmap); 3460 bus_dmamap_unload(sgt->dmat, sgt->dmap); 3461 m_freem(buf->m); 3462 buf->m = NULL; 3463 buf->paddr = 0; 3464 buf->vaddr = NULL; 3465 sgt->paddr = 0; 3466 3467 mtx_unlock(&bch->dma_mtx); 3468 3469 /* Return Tx buffer back to the ring */ 3470 buf_ring_enqueue(tx->br, buf); 3471 3472 return (0); 3473 } 3474 3475 /** 3476 * @brief Compare versions of the DPAA2 network interface API. 3477 */ 3478 static int 3479 dpaa2_ni_cmp_api_version(struct dpaa2_ni_softc *sc, uint16_t major, 3480 uint16_t minor) 3481 { 3482 if (sc->api_major == major) { 3483 return sc->api_minor - minor; 3484 } 3485 return sc->api_major - major; 3486 } 3487 3488 /** 3489 * @brief Collect statistics of the network interface. 3490 */ 3491 static int 3492 dpaa2_ni_collect_stats(SYSCTL_HANDLER_ARGS) 3493 { 3494 struct dpaa2_ni_softc *sc = (struct dpaa2_ni_softc *) arg1; 3495 struct dpni_stat *stat = &dpni_stat_sysctls[oidp->oid_number]; 3496 device_t pdev = device_get_parent(sc->dev); 3497 device_t dev = sc->dev; 3498 device_t child = dev; 3499 struct dpaa2_devinfo *rcinfo = device_get_ivars(pdev); 3500 struct dpaa2_devinfo *dinfo = device_get_ivars(dev); 3501 struct dpaa2_cmd cmd; 3502 uint64_t cnt[DPAA2_NI_STAT_COUNTERS_PER_PAGE]; 3503 uint64_t result = 0; 3504 uint16_t rc_token, ni_token; 3505 int error; 3506 3507 DPAA2_CMD_INIT(&cmd); 3508 3509 error = DPAA2_CMD_RC_OPEN(dev, child, &cmd, rcinfo->id, &rc_token); 3510 if (error) { 3511 device_printf(dev, "%s: failed to open resource container: " 3512 "id=%d, error=%d\n", __func__, rcinfo->id, error); 3513 goto exit; 3514 } 3515 error = DPAA2_CMD_NI_OPEN(dev, child, &cmd, dinfo->id, &ni_token); 3516 if (error) { 3517 device_printf(dev, "%s: failed to open network interface: " 3518 "id=%d, error=%d\n", __func__, dinfo->id, error); 3519 goto close_rc; 3520 } 3521 3522 error = DPAA2_CMD_NI_GET_STATISTICS(dev, child, &cmd, stat->page, 0, cnt); 3523 if (!error) { 3524 result = cnt[stat->cnt]; 3525 } 3526 3527 (void)DPAA2_CMD_NI_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, ni_token)); 3528 close_rc: 3529 (void)DPAA2_CMD_RC_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, rc_token)); 3530 exit: 3531 return (sysctl_handle_64(oidp, &result, 0, req)); 3532 } 3533 3534 static int 3535 dpaa2_ni_sysctl_link_state(SYSCTL_HANDLER_ARGS) 3536 { 3537 struct dpaa2_ni_softc *sc; 3538 struct dpaa2_devinfo *rcinfo; 3539 struct dpaa2_devinfo *dinfo; 3540 struct dpaa2_cmd cmd; 3541 struct dpaa2_ni_link_state ls; 3542 struct sbuf s; 3543 int error; 3544 uint16_t rc_token, ni_token; 3545 3546 if (req->newptr) 3547 return (EPERM); 3548 3549 sc = (struct dpaa2_ni_softc *)arg1; 3550 3551 rcinfo = device_get_ivars(device_get_parent(sc->dev)); 3552 dinfo = device_get_ivars(sc->dev); 3553 3554 DPAA2_CMD_INIT(&cmd); 3555 3556 error = DPAA2_CMD_RC_OPEN(sc->dev, sc->dev, &cmd, rcinfo->id, &rc_token); 3557 if (error != 0) { 3558 device_printf(sc->dev, "%s: failed to open resource container: " 3559 "id=%d, error=%d\n", __func__, rcinfo->id, error); 3560 goto exit; 3561 } 3562 error = DPAA2_CMD_NI_OPEN(sc->dev, sc->dev, &cmd, dinfo->id, &ni_token); 3563 if (error != 0) { 3564 device_printf(sc->dev, "%s: failed to open network interface: " 3565 "id=%d, error=%d\n", __func__, dinfo->id, error); 3566 goto close_rc; 3567 } 3568 3569 error = DPAA2_CMD_NI_GET_LINK_STATE(sc->dev, sc->dev, &cmd, &ls); 3570 3571 (void)DPAA2_CMD_NI_CLOSE(sc->dev, sc->dev, DPAA2_CMD_TK(&cmd, ni_token)); 3572 close_rc: 3573 (void)DPAA2_CMD_RC_CLOSE(sc->dev, sc->dev, DPAA2_CMD_TK(&cmd, rc_token)); 3574 3575 if (error != 0) 3576 goto exit; 3577 3578 #define NI_LINK_STATE_OPTIONS_BITS \ 3579 "\20\1AUTONEG\2HDX\3PAUSE\4ASYM_PAUSE" 3580 3581 sbuf_new_for_sysctl(&s, NULL, 1024, req); 3582 sbuf_putc(&s, '\n'); 3583 sbuf_printf(&s, "Link State: %s (%s)\n", ls.link_up ? "UP" : "DOWN", 3584 ls.state_valid ? "VALID" : "IGNORE"); 3585 sbuf_printf(&s, "Link Rate: %ju\n", (uintmax_t)ls.rate); 3586 sbuf_printf(&s, "Link Options: %b\n", 3587 (int)ls.options, NI_LINK_STATE_OPTIONS_BITS); 3588 sbuf_printf(&s, "Link Speed Capabilities: %#018jx\n", 3589 (uintmax_t)ls.sup_speeds); 3590 sbuf_printf(&s, "Link Speed Advertised for autoneg: %#018jx\n", 3591 (uintmax_t)ls.adv_speeds); 3592 sbuf_finish(&s); 3593 sbuf_delete(&s); 3594 3595 exit: 3596 return (error); 3597 } 3598 3599 static int 3600 dpaa2_ni_collect_buf_num(SYSCTL_HANDLER_ARGS) 3601 { 3602 struct dpaa2_ni_softc *sc = (struct dpaa2_ni_softc *) arg1; 3603 uint32_t buf_num = DPAA2_ATOMIC_READ(&sc->buf_num); 3604 3605 return (sysctl_handle_32(oidp, &buf_num, 0, req)); 3606 } 3607 3608 static int 3609 dpaa2_ni_collect_buf_free(SYSCTL_HANDLER_ARGS) 3610 { 3611 struct dpaa2_ni_softc *sc = (struct dpaa2_ni_softc *) arg1; 3612 uint32_t buf_free = DPAA2_ATOMIC_READ(&sc->buf_free); 3613 3614 return (sysctl_handle_32(oidp, &buf_free, 0, req)); 3615 } 3616 3617 /* 3618 * Common sysctl handler function for integer values stored as dpaa2_atomic. 3619 * Reads the current value from the atomic object and writes the new one to it. 3620 */ 3621 static inline int 3622 dpaa2_ni_sysctl_handle_int(struct sysctl_req *req, struct dpaa2_atomic *value) 3623 { 3624 int error, tmp; 3625 3626 tmp = DPAA2_ATOMIC_READ(value); 3627 error = SYSCTL_OUT(req, &tmp, sizeof(tmp)); 3628 if (error || req->newptr == NULL) 3629 return error; 3630 error = SYSCTL_IN(req, &tmp, sizeof(tmp)); 3631 if (error) 3632 return error; 3633 if ((tmp < DPAA2_CLEAN_BUDGET_MIN) || (tmp > DPAA2_CLEAN_BUDGET_MAX)) 3634 return EINVAL; 3635 DPAA2_ATOMIC_XCHG(value, tmp); 3636 3637 return 0; 3638 } 3639 3640 static int 3641 dpaa2_ni_sysctl_handle_clean_budget(SYSCTL_HANDLER_ARGS) 3642 { 3643 struct dpaa2_ni_softc *sc = (struct dpaa2_ni_softc *)arg1; 3644 3645 return dpaa2_ni_sysctl_handle_int(req, &sc->clean_budget); 3646 } 3647 3648 static int 3649 dpaa2_ni_sysctl_handle_tx_budget(SYSCTL_HANDLER_ARGS) 3650 { 3651 struct dpaa2_ni_softc *sc = (struct dpaa2_ni_softc *)arg1; 3652 3653 return dpaa2_ni_sysctl_handle_int(req, &sc->tx_budget); 3654 } 3655 3656 static int 3657 dpaa2_ni_sysctl_handle_rx_budget(SYSCTL_HANDLER_ARGS) 3658 { 3659 struct dpaa2_ni_softc *sc = (struct dpaa2_ni_softc *)arg1; 3660 3661 return dpaa2_ni_sysctl_handle_int(req, &sc->rx_budget); 3662 } 3663 3664 static int 3665 dpaa2_ni_set_hash(device_t dev, uint64_t flags) 3666 { 3667 struct dpaa2_ni_softc *sc = device_get_softc(dev); 3668 uint64_t key = 0; 3669 int i; 3670 3671 if (!(sc->attr.num.queues > 1)) { 3672 return (EOPNOTSUPP); 3673 } 3674 3675 for (i = 0; i < ARRAY_SIZE(dist_fields); i++) { 3676 if (dist_fields[i].rxnfc_field & flags) { 3677 key |= dist_fields[i].id; 3678 } 3679 } 3680 3681 return (dpaa2_ni_set_dist_key(dev, DPAA2_NI_DIST_MODE_HASH, key)); 3682 } 3683 3684 /** 3685 * @brief Set Rx distribution (hash or flow classification) key flags is a 3686 * combination of RXH_ bits. 3687 */ 3688 static int 3689 dpaa2_ni_set_dist_key(device_t dev, enum dpaa2_ni_dist_mode type, uint64_t flags) 3690 { 3691 device_t pdev = device_get_parent(dev); 3692 device_t child = dev; 3693 struct dpaa2_ni_softc *sc = device_get_softc(dev); 3694 struct dpaa2_devinfo *rcinfo = device_get_ivars(pdev); 3695 struct dpaa2_devinfo *dinfo = device_get_ivars(dev); 3696 struct dpkg_profile_cfg cls_cfg; 3697 struct dpkg_extract *key; 3698 struct dpaa2_buf *buf = &sc->rxd_kcfg; 3699 struct dpaa2_cmd cmd; 3700 uint16_t rc_token, ni_token; 3701 int i, error = 0; 3702 3703 if (__predict_true(buf->dmat == NULL)) { 3704 buf->dmat = sc->rxd_dmat; 3705 } 3706 3707 memset(&cls_cfg, 0, sizeof(cls_cfg)); 3708 3709 /* Configure extracts according to the given flags. */ 3710 for (i = 0; i < ARRAY_SIZE(dist_fields); i++) { 3711 key = &cls_cfg.extracts[cls_cfg.num_extracts]; 3712 3713 if (!(flags & dist_fields[i].id)) { 3714 continue; 3715 } 3716 3717 if (cls_cfg.num_extracts >= DPKG_MAX_NUM_OF_EXTRACTS) { 3718 device_printf(dev, "%s: failed to add key extraction " 3719 "rule\n", __func__); 3720 return (E2BIG); 3721 } 3722 3723 key->type = DPKG_EXTRACT_FROM_HDR; 3724 key->extract.from_hdr.prot = dist_fields[i].cls_prot; 3725 key->extract.from_hdr.type = DPKG_FULL_FIELD; 3726 key->extract.from_hdr.field = dist_fields[i].cls_field; 3727 cls_cfg.num_extracts++; 3728 } 3729 3730 error = bus_dmamem_alloc(buf->dmat, (void **)&buf->vaddr, 3731 BUS_DMA_ZERO | BUS_DMA_COHERENT, &buf->dmap); 3732 if (error != 0) { 3733 device_printf(dev, "%s: failed to allocate a buffer for Rx " 3734 "traffic distribution key configuration\n", __func__); 3735 return (error); 3736 } 3737 3738 error = dpaa2_ni_prepare_key_cfg(&cls_cfg, (uint8_t *)buf->vaddr); 3739 if (error != 0) { 3740 device_printf(dev, "%s: failed to prepare key configuration: " 3741 "error=%d\n", __func__, error); 3742 return (error); 3743 } 3744 3745 /* Prepare for setting the Rx dist. */ 3746 error = bus_dmamap_load(buf->dmat, buf->dmap, buf->vaddr, 3747 DPAA2_CLASSIFIER_DMA_SIZE, dpaa2_dmamap_oneseg_cb, &buf->paddr, 3748 BUS_DMA_NOWAIT); 3749 if (error != 0) { 3750 device_printf(sc->dev, "%s: failed to map a buffer for Rx " 3751 "traffic distribution key configuration\n", __func__); 3752 return (error); 3753 } 3754 3755 if (type == DPAA2_NI_DIST_MODE_HASH) { 3756 DPAA2_CMD_INIT(&cmd); 3757 3758 error = DPAA2_CMD_RC_OPEN(dev, child, &cmd, rcinfo->id, 3759 &rc_token); 3760 if (error) { 3761 device_printf(dev, "%s: failed to open resource " 3762 "container: id=%d, error=%d\n", __func__, rcinfo->id, 3763 error); 3764 goto err_exit; 3765 } 3766 error = DPAA2_CMD_NI_OPEN(dev, child, &cmd, dinfo->id, 3767 &ni_token); 3768 if (error) { 3769 device_printf(dev, "%s: failed to open network " 3770 "interface: id=%d, error=%d\n", __func__, dinfo->id, 3771 error); 3772 goto close_rc; 3773 } 3774 3775 error = DPAA2_CMD_NI_SET_RX_TC_DIST(dev, child, &cmd, 3776 sc->attr.num.queues, 0, DPAA2_NI_DIST_MODE_HASH, buf->paddr); 3777 if (error != 0) { 3778 device_printf(dev, "%s: failed to set distribution mode " 3779 "and size for the traffic class\n", __func__); 3780 } 3781 3782 (void)DPAA2_CMD_NI_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, 3783 ni_token)); 3784 close_rc: 3785 (void)DPAA2_CMD_RC_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, 3786 rc_token)); 3787 } 3788 3789 err_exit: 3790 return (error); 3791 } 3792 3793 /** 3794 * @brief Prepares extract parameters. 3795 * 3796 * cfg: Defining a full Key Generation profile. 3797 * key_cfg_buf: Zeroed 256 bytes of memory before mapping it to DMA. 3798 */ 3799 static int 3800 dpaa2_ni_prepare_key_cfg(struct dpkg_profile_cfg *cfg, uint8_t *key_cfg_buf) 3801 { 3802 struct dpni_ext_set_rx_tc_dist *dpni_ext; 3803 struct dpni_dist_extract *extr; 3804 int i, j; 3805 3806 if (cfg->num_extracts > DPKG_MAX_NUM_OF_EXTRACTS) 3807 return (EINVAL); 3808 3809 dpni_ext = (struct dpni_ext_set_rx_tc_dist *) key_cfg_buf; 3810 dpni_ext->num_extracts = cfg->num_extracts; 3811 3812 for (i = 0; i < cfg->num_extracts; i++) { 3813 extr = &dpni_ext->extracts[i]; 3814 3815 switch (cfg->extracts[i].type) { 3816 case DPKG_EXTRACT_FROM_HDR: 3817 extr->prot = cfg->extracts[i].extract.from_hdr.prot; 3818 extr->efh_type = 3819 cfg->extracts[i].extract.from_hdr.type & 0x0Fu; 3820 extr->size = cfg->extracts[i].extract.from_hdr.size; 3821 extr->offset = cfg->extracts[i].extract.from_hdr.offset; 3822 extr->field = cfg->extracts[i].extract.from_hdr.field; 3823 extr->hdr_index = 3824 cfg->extracts[i].extract.from_hdr.hdr_index; 3825 break; 3826 case DPKG_EXTRACT_FROM_DATA: 3827 extr->size = cfg->extracts[i].extract.from_data.size; 3828 extr->offset = 3829 cfg->extracts[i].extract.from_data.offset; 3830 break; 3831 case DPKG_EXTRACT_FROM_PARSE: 3832 extr->size = cfg->extracts[i].extract.from_parse.size; 3833 extr->offset = 3834 cfg->extracts[i].extract.from_parse.offset; 3835 break; 3836 default: 3837 return (EINVAL); 3838 } 3839 3840 extr->num_of_byte_masks = cfg->extracts[i].num_of_byte_masks; 3841 extr->extract_type = cfg->extracts[i].type & 0x0Fu; 3842 3843 for (j = 0; j < DPKG_NUM_OF_MASKS; j++) { 3844 extr->masks[j].mask = cfg->extracts[i].masks[j].mask; 3845 extr->masks[j].offset = 3846 cfg->extracts[i].masks[j].offset; 3847 } 3848 } 3849 3850 return (0); 3851 } 3852 3853 static int 3854 dpaa2_ni_update_csum_flags(struct dpaa2_fd *fd, struct mbuf *m) 3855 { 3856 struct dpaa2_hwa_fas fas; 3857 uint32_t status; 3858 int rc; 3859 3860 if (__predict_false((dpaa2_fd_get_frc(fd) & DPAA2_FD_FRC_FASV)) == 0u) 3861 return (EINVAL); 3862 3863 /* 3864 * XXX-DSL: Frame context of the frame descriptor (FD[FRC]) contains 3865 * an Accelerator ID in the MSbits on some SoCs (e.g. LS1088A), 3866 * but a frame ParseSummary on the others (e.g. LX2160A). 3867 * However, frame annotation valid bits seem to be at the 3868 * same offsets. This is the reason why different accelerators 3869 * are treated the same here. It isn't clear whether this is 3870 * a hardware limitation of the SoCs, version of the firmware 3871 * or DPL configuration. 3872 */ 3873 3874 rc = dpaa2_fa_get_fas(fd, &fas); 3875 if (rc != 0) 3876 return (rc); 3877 3878 status = le32toh(fas.status); 3879 rc = 0; 3880 3881 /* L3 */ 3882 if ((status & DPAA2_FAS_L3CV) != 0) { 3883 m->m_pkthdr.csum_flags |= CSUM_L3_CALC; 3884 if ((status & DPAA2_FAS_L3CE) == 0) 3885 m->m_pkthdr.csum_flags |= CSUM_L3_VALID; 3886 } 3887 /* L4 */ 3888 if ((status & DPAA2_FAS_L4CV) != 0) { 3889 m->m_pkthdr.csum_flags |= CSUM_L4_CALC; 3890 m->m_pkthdr.csum_data = 0xffff; 3891 if ((status & DPAA2_FAS_L4CE) == 0) 3892 m->m_pkthdr.csum_flags |= CSUM_L4_VALID; 3893 } 3894 3895 return (rc); 3896 } 3897 3898 static device_method_t dpaa2_ni_methods[] = { 3899 /* Device interface */ 3900 DEVMETHOD(device_probe, dpaa2_ni_probe), 3901 DEVMETHOD(device_attach, dpaa2_ni_attach), 3902 DEVMETHOD(device_detach, dpaa2_ni_detach), 3903 3904 /* mii via memac_mdio */ 3905 DEVMETHOD(miibus_statchg, dpaa2_ni_miibus_statchg), 3906 3907 DEVMETHOD_END 3908 }; 3909 3910 static driver_t dpaa2_ni_driver = { 3911 "dpaa2_ni", 3912 dpaa2_ni_methods, 3913 sizeof(struct dpaa2_ni_softc), 3914 }; 3915 3916 DRIVER_MODULE(miibus, dpaa2_ni, miibus_driver, 0, 0); 3917 DRIVER_MODULE(dpaa2_ni, dpaa2_rc, dpaa2_ni_driver, 0, 0); 3918 3919 MODULE_DEPEND(dpaa2_ni, miibus, 1, 1, 1); 3920 #ifdef DEV_ACPI 3921 MODULE_DEPEND(dpaa2_ni, memac_mdio_acpi, 1, 1, 1); 3922 #endif 3923 #ifdef FDT 3924 MODULE_DEPEND(dpaa2_ni, memac_mdio_fdt, 1, 1, 1); 3925 #endif 3926