1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause 3 * 4 * Copyright (c) 2021-2026 Dmitry Salychev 5 * Copyright (c) 2022 Mathew McBride 6 * Copyright (c) 2026 Bjoern A. Zeeb 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27 * SUCH DAMAGE. 28 */ 29 30 #include <sys/cdefs.h> 31 /* 32 * The DPAA2 Network Interface (DPNI) driver. 33 * 34 * The DPNI object is a network interface that is configurable to support a wide 35 * range of features from a very basic Ethernet interface up to a 36 * high-functioning network interface. The DPNI supports features that are 37 * expected by standard network stacks, from basic features to offloads. 38 * 39 * DPNIs work with Ethernet traffic, starting with the L2 header. Additional 40 * functions are provided for standard network protocols (L2, L3, L4, etc.). 41 */ 42 43 #include <sys/param.h> 44 #include <sys/systm.h> 45 #include <sys/kernel.h> 46 #include <sys/bus.h> 47 #include <sys/rman.h> 48 #include <sys/module.h> 49 #include <sys/malloc.h> 50 #include <sys/mutex.h> 51 #include <sys/socket.h> 52 #include <sys/sockio.h> 53 #include <sys/sysctl.h> 54 #include <sys/mbuf.h> 55 #include <sys/taskqueue.h> 56 #include <sys/sysctl.h> 57 #include <sys/buf_ring.h> 58 #include <sys/smp.h> 59 #include <sys/proc.h> 60 #include <sys/sbuf.h> 61 62 #include <vm/vm.h> 63 #include <vm/pmap.h> 64 65 #include <machine/bus.h> 66 #include <machine/resource.h> 67 #include <machine/atomic.h> 68 #include <machine/vmparam.h> 69 70 #include <net/ethernet.h> 71 #include <net/bpf.h> 72 #include <net/if.h> 73 #include <net/if_dl.h> 74 #include <net/if_media.h> 75 #include <net/if_types.h> 76 #include <net/if_var.h> 77 78 #include <dev/pci/pcivar.h> 79 #include <dev/mii/mii.h> 80 #include <dev/mii/miivar.h> 81 #include <dev/mdio/mdio.h> 82 83 #include "opt_acpi.h" 84 #include "opt_platform.h" 85 86 #include "pcib_if.h" 87 #include "pci_if.h" 88 #include "miibus_if.h" 89 #include "memac_mdio_if.h" 90 91 #include "dpaa2_types.h" 92 #include "dpaa2_mc.h" 93 #include "dpaa2_mc_if.h" 94 #include "dpaa2_mcp.h" 95 #include "dpaa2_swp.h" 96 #include "dpaa2_swp_if.h" 97 #include "dpaa2_cmd_if.h" 98 #include "dpaa2_ni.h" 99 #include "dpaa2_channel.h" 100 #include "dpaa2_buf.h" 101 #include "dpaa2_frame.h" 102 103 #define BIT(x) (1ul << (x)) 104 #define WRIOP_VERSION(x, y, z) ((x) << 10 | (y) << 5 | (z) << 0) 105 #define ARRAY_SIZE(a) (sizeof(a) / sizeof((a)[0])) 106 107 /* Frame Dequeue Response status bits. */ 108 #define IS_NULL_RESPONSE(stat) ((((stat) >> 4) & 1) == 0) 109 110 #define ALIGN_UP(x, y) roundup2((x), (y)) 111 #define ALIGN_DOWN(x, y) rounddown2((x), (y)) 112 #define CACHE_LINE_ALIGN(x) ALIGN_UP((x), CACHE_LINE_SIZE) 113 114 #define DPNI_LOCK(__sc) do { \ 115 mtx_assert(&(__sc)->lock, MA_NOTOWNED); \ 116 mtx_lock(&(__sc)->lock); \ 117 } while (0) 118 #define DPNI_UNLOCK(__sc) do { \ 119 mtx_assert(&(__sc)->lock, MA_OWNED); \ 120 mtx_unlock(&(__sc)->lock); \ 121 } while (0) 122 #define DPNI_LOCK_ASSERT(__sc) do { \ 123 mtx_assert(&(__sc)->lock, MA_OWNED); \ 124 } while (0) 125 126 #define DPAA2_TX_RING(sc, chan, tc) \ 127 (&(sc)->channels[(chan)]->txc_queue.tx_rings[(tc)]) 128 129 MALLOC_DEFINE(M_DPAA2_TXB, "dpaa2_txb", "DPAA2 DMA-mapped buffer (Tx)"); 130 131 /* 132 * How many times channel cleanup routine will be repeated if the RX or TX 133 * budget was depleted. 134 */ 135 #define DPAA2_CLEAN_BUDGET 64 /* sysctl(9)? */ 136 /* TX/RX budget for the channel cleanup task */ 137 #define DPAA2_TX_BUDGET 128 /* sysctl(9)? */ 138 #define DPAA2_RX_BUDGET 256 /* sysctl(9)? */ 139 140 #define DPNI_IRQ_INDEX 0 /* Index of the only DPNI IRQ. */ 141 #define DPNI_IRQ_LINK_CHANGED 1 /* Link state changed */ 142 #define DPNI_IRQ_EP_CHANGED 2 /* DPAA2 endpoint dis/connected */ 143 144 /* Default maximum RX frame length w/o CRC. */ 145 #define DPAA2_ETH_MFL (ETHER_MAX_LEN_JUMBO + ETHER_VLAN_ENCAP_LEN - \ 146 ETHER_CRC_LEN) 147 148 /* Minimally supported version of the DPNI API. */ 149 #define DPNI_VER_MAJOR 7 150 #define DPNI_VER_MINOR 0 151 152 /* Rx/Tx buffers configuration. */ 153 #define BUF_ALIGN_V1 256 /* WRIOP v1.0.0 limitation */ 154 #define BUF_ALIGN 64 155 #define BUF_SWA_SIZE 64 /* SW annotation size */ 156 #define BUF_RX_HWA_SIZE 64 /* HW annotation size */ 157 #define BUF_TX_HWA_SIZE 128 /* HW annotation size */ 158 159 #define DPAA2_RX_BUFRING_SZ (4096u) 160 #define DPAA2_RXE_BUFRING_SZ (1024u) 161 #define DPAA2_TXC_BUFRING_SZ (4096u) 162 163 /* Size of a buffer to keep a QoS table key configuration. */ 164 #define ETH_QOS_KCFG_BUF_SIZE (PAGE_SIZE) 165 166 /* Required by struct dpni_rx_tc_dist_cfg::key_cfg_iova */ 167 #define DPAA2_CLASSIFIER_DMA_SIZE (PAGE_SIZE) 168 169 /* Buffers layout options. */ 170 #define BUF_LOPT_TIMESTAMP 0x1 171 #define BUF_LOPT_PARSER_RESULT 0x2 172 #define BUF_LOPT_FRAME_STATUS 0x4 173 #define BUF_LOPT_PRIV_DATA_SZ 0x8 174 #define BUF_LOPT_DATA_ALIGN 0x10 175 #define BUF_LOPT_DATA_HEAD_ROOM 0x20 176 #define BUF_LOPT_DATA_TAIL_ROOM 0x40 177 178 #define DPAA2_NI_BUF_ADDR_MASK (0x1FFFFFFFFFFFFul) /* 49-bit addresses max. */ 179 #define DPAA2_NI_BUF_CHAN_MASK (0xFu) 180 #define DPAA2_NI_BUF_CHAN_SHIFT (60) 181 #define DPAA2_NI_BUF_IDX_MASK (0x7FFFu) 182 #define DPAA2_NI_BUF_IDX_SHIFT (49) 183 #define DPAA2_NI_TX_IDX_MASK (0x7u) 184 #define DPAA2_NI_TX_IDX_SHIFT (57) 185 #define DPAA2_NI_TXBUF_IDX_MASK (0xFFu) 186 #define DPAA2_NI_TXBUF_IDX_SHIFT (49) 187 188 /* Enables TCAM for Flow Steering and QoS look-ups. */ 189 #define DPNI_OPT_HAS_KEY_MASKING 0x10 190 191 /* Unique IDs for the supported Rx classification header fields. */ 192 #define DPAA2_ETH_DIST_ETHDST BIT(0) 193 #define DPAA2_ETH_DIST_ETHSRC BIT(1) 194 #define DPAA2_ETH_DIST_ETHTYPE BIT(2) 195 #define DPAA2_ETH_DIST_VLAN BIT(3) 196 #define DPAA2_ETH_DIST_IPSRC BIT(4) 197 #define DPAA2_ETH_DIST_IPDST BIT(5) 198 #define DPAA2_ETH_DIST_IPPROTO BIT(6) 199 #define DPAA2_ETH_DIST_L4SRC BIT(7) 200 #define DPAA2_ETH_DIST_L4DST BIT(8) 201 #define DPAA2_ETH_DIST_ALL (~0ULL) 202 203 /* L3-L4 network traffic flow hash options. */ 204 #define RXH_L2DA (1 << 1) 205 #define RXH_VLAN (1 << 2) 206 #define RXH_L3_PROTO (1 << 3) 207 #define RXH_IP_SRC (1 << 4) 208 #define RXH_IP_DST (1 << 5) 209 #define RXH_L4_B_0_1 (1 << 6) /* src port in case of TCP/UDP/SCTP */ 210 #define RXH_L4_B_2_3 (1 << 7) /* dst port in case of TCP/UDP/SCTP */ 211 #define RXH_DISCARD (1 << 31) 212 213 /* Transmit checksum offload */ 214 #define DPAA2_CSUM_TX_OFFLOAD (CSUM_IP | CSUM_DELAY_DATA | CSUM_DELAY_DATA_IPV6) 215 216 /* Default Rx hash options, set during attaching. */ 217 #define DPAA2_RXH_DEFAULT (RXH_IP_SRC | RXH_IP_DST | RXH_L4_B_0_1 | RXH_L4_B_2_3) 218 219 MALLOC_DEFINE(M_DPAA2_NI, "dpaa2_ni", "DPAA2 Network Interface"); 220 221 /* 222 * DPAA2 Network Interface resource specification. 223 * 224 * NOTE: Don't forget to update macros in dpaa2_ni.h in case of any changes in 225 * the specification! 226 */ 227 struct resource_spec dpaa2_ni_spec[] = { 228 /* 229 * DPMCP resources. 230 * 231 * NOTE: MC command portals (MCPs) are used to send commands to, and 232 * receive responses from, the MC firmware. One portal per DPNI. 233 */ 234 { DPAA2_DEV_MCP, DPAA2_NI_MCP_RID(0), RF_ACTIVE | RF_SHAREABLE | RF_OPTIONAL }, 235 /* 236 * DPIO resources (software portals). 237 * 238 * NOTE: One per running core. While DPIOs are the source of data 239 * availability interrupts, the DPCONs are used to identify the 240 * network interface that has produced ingress data to that core. 241 */ 242 { DPAA2_DEV_IO, DPAA2_NI_IO_RID(0), RF_ACTIVE | RF_SHAREABLE }, 243 { DPAA2_DEV_IO, DPAA2_NI_IO_RID(1), RF_ACTIVE | RF_SHAREABLE | RF_OPTIONAL }, 244 { DPAA2_DEV_IO, DPAA2_NI_IO_RID(2), RF_ACTIVE | RF_SHAREABLE | RF_OPTIONAL }, 245 { DPAA2_DEV_IO, DPAA2_NI_IO_RID(3), RF_ACTIVE | RF_SHAREABLE | RF_OPTIONAL }, 246 { DPAA2_DEV_IO, DPAA2_NI_IO_RID(4), RF_ACTIVE | RF_SHAREABLE | RF_OPTIONAL }, 247 { DPAA2_DEV_IO, DPAA2_NI_IO_RID(5), RF_ACTIVE | RF_SHAREABLE | RF_OPTIONAL }, 248 { DPAA2_DEV_IO, DPAA2_NI_IO_RID(6), RF_ACTIVE | RF_SHAREABLE | RF_OPTIONAL }, 249 { DPAA2_DEV_IO, DPAA2_NI_IO_RID(7), RF_ACTIVE | RF_SHAREABLE | RF_OPTIONAL }, 250 { DPAA2_DEV_IO, DPAA2_NI_IO_RID(8), RF_ACTIVE | RF_SHAREABLE | RF_OPTIONAL }, 251 { DPAA2_DEV_IO, DPAA2_NI_IO_RID(9), RF_ACTIVE | RF_SHAREABLE | RF_OPTIONAL }, 252 { DPAA2_DEV_IO, DPAA2_NI_IO_RID(10), RF_ACTIVE | RF_SHAREABLE | RF_OPTIONAL }, 253 { DPAA2_DEV_IO, DPAA2_NI_IO_RID(11), RF_ACTIVE | RF_SHAREABLE | RF_OPTIONAL }, 254 { DPAA2_DEV_IO, DPAA2_NI_IO_RID(12), RF_ACTIVE | RF_SHAREABLE | RF_OPTIONAL }, 255 { DPAA2_DEV_IO, DPAA2_NI_IO_RID(13), RF_ACTIVE | RF_SHAREABLE | RF_OPTIONAL }, 256 { DPAA2_DEV_IO, DPAA2_NI_IO_RID(14), RF_ACTIVE | RF_SHAREABLE | RF_OPTIONAL }, 257 { DPAA2_DEV_IO, DPAA2_NI_IO_RID(15), RF_ACTIVE | RF_SHAREABLE | RF_OPTIONAL }, 258 /* 259 * DPBP resources (buffer pools). 260 * 261 * NOTE: One per network interface. 262 */ 263 { DPAA2_DEV_BP, DPAA2_NI_BP_RID(0), RF_ACTIVE }, 264 /* 265 * DPCON resources (channels). 266 * 267 * NOTE: One DPCON per core where Rx or Tx confirmation traffic to be 268 * distributed to. 269 * NOTE: Since it is necessary to distinguish between traffic from 270 * different network interfaces arriving on the same core, the 271 * DPCONs must be private to the DPNIs. 272 */ 273 { DPAA2_DEV_CON, DPAA2_NI_CON_RID(0), RF_ACTIVE }, 274 { DPAA2_DEV_CON, DPAA2_NI_CON_RID(1), RF_ACTIVE | RF_OPTIONAL }, 275 { DPAA2_DEV_CON, DPAA2_NI_CON_RID(2), RF_ACTIVE | RF_OPTIONAL }, 276 { DPAA2_DEV_CON, DPAA2_NI_CON_RID(3), RF_ACTIVE | RF_OPTIONAL }, 277 { DPAA2_DEV_CON, DPAA2_NI_CON_RID(4), RF_ACTIVE | RF_OPTIONAL }, 278 { DPAA2_DEV_CON, DPAA2_NI_CON_RID(5), RF_ACTIVE | RF_OPTIONAL }, 279 { DPAA2_DEV_CON, DPAA2_NI_CON_RID(6), RF_ACTIVE | RF_OPTIONAL }, 280 { DPAA2_DEV_CON, DPAA2_NI_CON_RID(7), RF_ACTIVE | RF_OPTIONAL }, 281 { DPAA2_DEV_CON, DPAA2_NI_CON_RID(8), RF_ACTIVE | RF_OPTIONAL }, 282 { DPAA2_DEV_CON, DPAA2_NI_CON_RID(9), RF_ACTIVE | RF_OPTIONAL }, 283 { DPAA2_DEV_CON, DPAA2_NI_CON_RID(10), RF_ACTIVE | RF_OPTIONAL }, 284 { DPAA2_DEV_CON, DPAA2_NI_CON_RID(11), RF_ACTIVE | RF_OPTIONAL }, 285 { DPAA2_DEV_CON, DPAA2_NI_CON_RID(12), RF_ACTIVE | RF_OPTIONAL }, 286 { DPAA2_DEV_CON, DPAA2_NI_CON_RID(13), RF_ACTIVE | RF_OPTIONAL }, 287 { DPAA2_DEV_CON, DPAA2_NI_CON_RID(14), RF_ACTIVE | RF_OPTIONAL }, 288 { DPAA2_DEV_CON, DPAA2_NI_CON_RID(15), RF_ACTIVE | RF_OPTIONAL }, 289 290 RESOURCE_SPEC_END 291 }; 292 293 /* Supported header fields for Rx hash distribution key */ 294 static const struct dpaa2_eth_dist_fields dist_fields[] = { 295 { 296 /* L2 header */ 297 .rxnfc_field = RXH_L2DA, 298 .cls_prot = NET_PROT_ETH, 299 .cls_field = NH_FLD_ETH_DA, 300 .id = DPAA2_ETH_DIST_ETHDST, 301 .size = 6, 302 }, { 303 .cls_prot = NET_PROT_ETH, 304 .cls_field = NH_FLD_ETH_SA, 305 .id = DPAA2_ETH_DIST_ETHSRC, 306 .size = 6, 307 }, { 308 /* This is the last ethertype field parsed: 309 * depending on frame format, it can be the MAC ethertype 310 * or the VLAN etype. 311 */ 312 .cls_prot = NET_PROT_ETH, 313 .cls_field = NH_FLD_ETH_TYPE, 314 .id = DPAA2_ETH_DIST_ETHTYPE, 315 .size = 2, 316 }, { 317 /* VLAN header */ 318 .rxnfc_field = RXH_VLAN, 319 .cls_prot = NET_PROT_VLAN, 320 .cls_field = NH_FLD_VLAN_TCI, 321 .id = DPAA2_ETH_DIST_VLAN, 322 .size = 2, 323 }, { 324 /* IP header */ 325 .rxnfc_field = RXH_IP_SRC, 326 .cls_prot = NET_PROT_IP, 327 .cls_field = NH_FLD_IP_SRC, 328 .id = DPAA2_ETH_DIST_IPSRC, 329 .size = 4, 330 }, { 331 .rxnfc_field = RXH_IP_DST, 332 .cls_prot = NET_PROT_IP, 333 .cls_field = NH_FLD_IP_DST, 334 .id = DPAA2_ETH_DIST_IPDST, 335 .size = 4, 336 }, { 337 .rxnfc_field = RXH_L3_PROTO, 338 .cls_prot = NET_PROT_IP, 339 .cls_field = NH_FLD_IP_PROTO, 340 .id = DPAA2_ETH_DIST_IPPROTO, 341 .size = 1, 342 }, { 343 /* Using UDP ports, this is functionally equivalent to raw 344 * byte pairs from L4 header. 345 */ 346 .rxnfc_field = RXH_L4_B_0_1, 347 .cls_prot = NET_PROT_UDP, 348 .cls_field = NH_FLD_UDP_PORT_SRC, 349 .id = DPAA2_ETH_DIST_L4SRC, 350 .size = 2, 351 }, { 352 .rxnfc_field = RXH_L4_B_2_3, 353 .cls_prot = NET_PROT_UDP, 354 .cls_field = NH_FLD_UDP_PORT_DST, 355 .id = DPAA2_ETH_DIST_L4DST, 356 .size = 2, 357 }, 358 }; 359 360 static struct dpni_stat { 361 int page; 362 int cnt; 363 char *name; 364 char *desc; 365 } dpni_stat_sysctls[] = { 366 /* PAGE, COUNTER, NAME, DESCRIPTION */ 367 { 0, 0, "in_all_frames", "All accepted ingress frames" }, 368 { 0, 1, "in_all_bytes", "Bytes in all accepted ingress frames" }, 369 { 0, 2, "in_mc_frames", "Multicast accepted ingress frames" }, 370 { 0, 3, "in_mc_bytes", "Bytes in received multicast frames" }, 371 { 0, 4, "in_bc_frames", "Broadcast accepted ingress frames" }, 372 { 0, 5, "in_bc_bytes", "Bytes in broadcast multicast frames" }, 373 374 { 1, 0, "eg_all_frames", "All egress frames transmitted" }, 375 { 1, 1, "eg_all_bytes", "Bytes in all frames transmitted" }, 376 { 1, 2, "eg_mc_frames", "Multicast egress frames transmitted" }, 377 { 1, 3, "eg_mc_bytes", "Bytes in transmitted multicast frame" }, 378 { 1, 4, "eg_bc_frames", "Broadcast egress frames transmitted" }, 379 { 1, 5, "eg_bc_bytes", "Bytes in broadcast multicast frames" }, 380 381 { 2, 0, "in_filtered_frames", "All ingress frames discarded due to filtering" }, 382 { 2, 1, "in_discarded_frames", "All frames discarded due to errors" }, 383 { 2, 2, "in_nobuf_discards", "Discards on ingress side due to buffer depletion in DPNI buffer pools" }, 384 { 2, 3, "eg_frames_disc", "Frames discarded on transmit due to DPNI configuration and/or frame state" }, 385 { 2, 4, "eg_frames_tx", "Frames that have been confirmed after transmission" }, 386 387 /* XXX FIXME Page 3/4 can take a param as well not encoded here. */ 388 /* XXX 3/0 and 3/1 have the same description in the manual? Where's the difference? */ 389 { 3, 0, "bytes_dequeued", "Cumulative count of the number of bytes dequeued" }, 390 { 3, 1, "frames_dequeued", "Cumulative count of the number of frames dequeued" }, 391 { 3, 2, "bytes_enqueued_rej", "Cumulative count of the number of bytes in all frames whose enqueue was rejected." }, 392 { 3, 3, "frames_enqueued_rej", "Cumulative count of all frame enqueues rejected." }, 393 394 { 4, 0, "fames_rej_tc", "Rejected frames in associated congestion point (valid if this TC has an associated congestion point)" }, 395 { 4, 1, "bytes_rej_tc", "Rejected bytes in associated congestion point (valid if this TC has an associated congestion point)" }, 396 397 { 5, 0, "pol_red", "Policer RED packet counter. 32bit value valid only when policer is enabled." }, 398 { 5, 1, "pol_yel", "Policer YELLOW packet counter. 32bit value valid only when policer is enabled." }, 399 { 5, 2, "pol_gre", "Policer GREEN packet counter. 32bit value valid only when policer is enabled." }, 400 { 5, 3, "pol_re_red", "Policer recolored RED packet counter. 32bit value valid only when policer is enabled." }, 401 { 5, 4, "pol_re_yel", "Policer recolored YELLOW packet counter. 32bit value valid only when policer is enabled." }, 402 }; 403 404 struct dpaa2_ni_rx_ctx { 405 struct mbuf *head; 406 struct mbuf *tail; 407 int cnt; 408 bool last; 409 }; 410 411 /* Device interface */ 412 static int dpaa2_ni_probe(device_t); 413 static int dpaa2_ni_attach(device_t); 414 static int dpaa2_ni_detach(device_t); 415 416 /* DPAA2 network interface setup and configuration */ 417 static int dpaa2_ni_setup(device_t); 418 static int dpaa2_ni_setup_channels(device_t); 419 static int dpaa2_ni_bind(device_t); 420 static int dpaa2_ni_setup_rx_dist(device_t); 421 static int dpaa2_ni_setup_irqs(device_t); 422 static int dpaa2_ni_setup_msi(struct dpaa2_ni_softc *); 423 static int dpaa2_ni_setup_if_caps(struct dpaa2_ni_softc *); 424 static int dpaa2_ni_setup_if_flags(struct dpaa2_ni_softc *); 425 static int dpaa2_ni_setup_sysctls(struct dpaa2_ni_softc *); 426 static int dpaa2_ni_setup_dma(struct dpaa2_ni_softc *); 427 428 /* Tx/Rx flow configuration */ 429 static int dpaa2_ni_setup_rx_flow(device_t, struct dpaa2_ni_fq *); 430 static int dpaa2_ni_setup_tx_flow(device_t, struct dpaa2_ni_fq *); 431 static int dpaa2_ni_setup_rx_err_flow(device_t, struct dpaa2_ni_fq *); 432 433 /* Configuration subroutines */ 434 static int dpaa2_ni_set_buf_layout(device_t); 435 static int dpaa2_ni_set_pause_frame(device_t); 436 static int dpaa2_ni_set_qos_table(device_t); 437 static int dpaa2_ni_set_mac_addr(device_t); 438 static int dpaa2_ni_set_hash(device_t, uint64_t); 439 static int dpaa2_ni_set_dist_key(device_t, enum dpaa2_ni_dist_mode, uint64_t); 440 441 /* Various subroutines */ 442 static int dpaa2_ni_cmp_api_version(struct dpaa2_ni_softc *, uint16_t, uint16_t); 443 static int dpaa2_ni_prepare_key_cfg(struct dpkg_profile_cfg *, uint8_t *); 444 static int dpaa2_ni_update_csum_flags(struct dpaa2_fd *, struct mbuf *); 445 446 /* Network interface routines */ 447 static void dpaa2_ni_init(void *); 448 static int dpaa2_ni_transmit(if_t , struct mbuf *); 449 static void dpaa2_ni_qflush(if_t ); 450 static int dpaa2_ni_ioctl(if_t , u_long, caddr_t); 451 static int dpaa2_ni_update_mac_filters(if_t ); 452 static u_int dpaa2_ni_add_maddr(void *, struct sockaddr_dl *, u_int); 453 454 /* Interrupt handlers */ 455 static void dpaa2_ni_intr(void *); 456 457 /* MII handlers */ 458 static void dpaa2_ni_miibus_statchg(device_t); 459 static int dpaa2_ni_media_change(if_t ); 460 static void dpaa2_ni_media_status(if_t , struct ifmediareq *); 461 static void dpaa2_ni_media_tick(void *); 462 463 /* Tx/Rx routines. */ 464 static int dpaa2_ni_rx_cleanup(struct dpaa2_channel *); 465 static int dpaa2_ni_tx_cleanup(struct dpaa2_channel *); 466 static void dpaa2_ni_tx(struct dpaa2_ni_softc *, struct dpaa2_channel *, 467 struct dpaa2_ni_tx_ring *, struct mbuf *); 468 static void dpaa2_ni_cleanup_task(void *, int); 469 470 /* Tx/Rx subroutines */ 471 static int dpaa2_ni_consume_frames(struct dpaa2_channel *, struct dpaa2_ni_fq **, 472 uint32_t *); 473 static int dpaa2_ni_rx(struct dpaa2_channel *, struct dpaa2_ni_fq *, 474 struct dpaa2_fd *, struct dpaa2_ni_rx_ctx *); 475 static int dpaa2_ni_rx_err(struct dpaa2_channel *, struct dpaa2_ni_fq *, 476 struct dpaa2_fd *); 477 static int dpaa2_ni_tx_conf(struct dpaa2_channel *, struct dpaa2_ni_fq *, 478 struct dpaa2_fd *); 479 480 /* sysctl(9) */ 481 static int dpaa2_ni_collect_stats(SYSCTL_HANDLER_ARGS); 482 static int dpaa2_ni_collect_buf_num(SYSCTL_HANDLER_ARGS); 483 static int dpaa2_ni_collect_buf_free(SYSCTL_HANDLER_ARGS); 484 static int dpaa2_ni_sysctl_link_state(SYSCTL_HANDLER_ARGS); 485 486 static int 487 dpaa2_ni_probe(device_t dev) 488 { 489 /* DPNI device will be added by a parent resource container itself. */ 490 device_set_desc(dev, "DPAA2 Network Interface"); 491 return (BUS_PROBE_DEFAULT); 492 } 493 494 static int 495 dpaa2_ni_attach(device_t dev) 496 { 497 device_t pdev = device_get_parent(dev); 498 device_t child = dev; 499 device_t mcp_dev; 500 struct dpaa2_ni_softc *sc = device_get_softc(dev); 501 struct dpaa2_devinfo *rcinfo = device_get_ivars(pdev); 502 struct dpaa2_devinfo *dinfo = device_get_ivars(dev); 503 struct dpaa2_devinfo *mcp_dinfo; 504 struct dpaa2_cmd cmd; 505 uint16_t rc_token, ni_token; 506 if_t ifp; 507 char tq_name[32]; 508 int error; 509 510 sc->dev = dev; 511 sc->ifp = NULL; 512 sc->miibus = NULL; 513 sc->mii = NULL; 514 sc->media_status = 0; 515 sc->if_flags = 0; 516 sc->link_state = LINK_STATE_UNKNOWN; 517 sc->buf_align = 0; 518 519 /* For debug purposes only! */ 520 sc->rx_anomaly_frames = 0; 521 sc->rx_single_buf_frames = 0; 522 sc->rx_sg_buf_frames = 0; 523 sc->rx_enq_rej_frames = 0; 524 sc->rx_ieoi_err_frames = 0; 525 sc->rx_other_err_frames = 0; 526 sc->tx_single_buf_frames = 0; 527 sc->tx_sg_frames = 0; 528 529 DPAA2_ATOMIC_XCHG(&sc->buf_num, 0); 530 DPAA2_ATOMIC_XCHG(&sc->buf_free, 0); 531 532 sc->rxd_dmat = NULL; 533 sc->qos_dmat = NULL; 534 535 sc->qos_kcfg.dmap = NULL; 536 sc->qos_kcfg.paddr = 0; 537 sc->qos_kcfg.vaddr = NULL; 538 539 sc->rxd_kcfg.dmap = NULL; 540 sc->rxd_kcfg.paddr = 0; 541 sc->rxd_kcfg.vaddr = NULL; 542 543 sc->mac.dpmac_id = 0; 544 sc->mac.phy_dev = NULL; 545 memset(sc->mac.addr, 0, ETHER_ADDR_LEN); 546 547 error = bus_alloc_resources(sc->dev, dpaa2_ni_spec, sc->res); 548 if (error) { 549 device_printf(dev, "%s: failed to allocate resources: " 550 "error=%d\n", __func__, error); 551 goto err_exit; 552 } 553 554 /* Obtain MC portal. */ 555 mcp_dev = (device_t) rman_get_start(sc->res[DPAA2_NI_MCP_RID(0)]); 556 mcp_dinfo = device_get_ivars(mcp_dev); 557 dinfo->portal = mcp_dinfo->portal; 558 559 mtx_init(&sc->lock, device_get_nameunit(dev), "dpaa2_ni", MTX_DEF); 560 561 /* Allocate network interface */ 562 ifp = if_alloc(IFT_ETHER); 563 sc->ifp = ifp; 564 if_initname(ifp, DPAA2_NI_IFNAME, device_get_unit(sc->dev)); 565 566 if_setsoftc(ifp, sc); 567 if_setflags(ifp, IFF_SIMPLEX | IFF_MULTICAST | IFF_BROADCAST); 568 if_setinitfn(ifp, dpaa2_ni_init); 569 if_setioctlfn(ifp, dpaa2_ni_ioctl); 570 if_settransmitfn(ifp, dpaa2_ni_transmit); 571 if_setqflushfn(ifp, dpaa2_ni_qflush); 572 573 if_sethwassist(sc->ifp, DPAA2_CSUM_TX_OFFLOAD); 574 if_setcapabilities(ifp, IFCAP_VLAN_MTU | IFCAP_HWCSUM | 575 IFCAP_HWCSUM_IPV6 | IFCAP_JUMBO_MTU); 576 if_setcapenable(ifp, if_getcapabilities(ifp)); 577 578 DPAA2_CMD_INIT(&cmd); 579 580 /* Open resource container and network interface object. */ 581 error = DPAA2_CMD_RC_OPEN(dev, child, &cmd, rcinfo->id, &rc_token); 582 if (error) { 583 device_printf(dev, "%s: failed to open resource container: " 584 "id=%d, error=%d\n", __func__, rcinfo->id, error); 585 goto err_exit; 586 } 587 error = DPAA2_CMD_NI_OPEN(dev, child, &cmd, dinfo->id, &ni_token); 588 if (error) { 589 device_printf(dev, "%s: failed to open network interface: " 590 "id=%d, error=%d\n", __func__, dinfo->id, error); 591 goto close_rc; 592 } 593 594 bzero(tq_name, sizeof(tq_name)); 595 snprintf(tq_name, sizeof(tq_name), "%s_tqbp", device_get_nameunit(dev)); 596 597 /* 598 * XXX-DSL: Release new buffers on Buffer Pool State Change Notification 599 * (BPSCN) returned as a result to the VDQ command instead. 600 * It is similar to CDAN processed in dpaa2_io_intr(). 601 */ 602 /* Create a taskqueue thread to release new buffers to the pool. */ 603 sc->bp_taskq = taskqueue_create(tq_name, M_WAITOK, 604 taskqueue_thread_enqueue, &sc->bp_taskq); 605 taskqueue_start_threads(&sc->bp_taskq, 1, PI_NET, "%s", tq_name); 606 607 /* sc->cleanup_taskq = taskqueue_create("dpaa2_ch cleanup", M_WAITOK, */ 608 /* taskqueue_thread_enqueue, &sc->cleanup_taskq); */ 609 /* taskqueue_start_threads(&sc->cleanup_taskq, 1, PI_NET, */ 610 /* "dpaa2_ch cleanup"); */ 611 612 error = dpaa2_ni_setup(dev); 613 if (error) { 614 device_printf(dev, "%s: failed to setup DPNI: error=%d\n", 615 __func__, error); 616 goto close_ni; 617 } 618 error = dpaa2_ni_setup_channels(dev); 619 if (error) { 620 device_printf(dev, "%s: failed to setup QBMan channels: " 621 "error=%d\n", __func__, error); 622 goto close_ni; 623 } 624 625 error = dpaa2_ni_bind(dev); 626 if (error) { 627 device_printf(dev, "%s: failed to bind DPNI: error=%d\n", 628 __func__, error); 629 goto close_ni; 630 } 631 error = dpaa2_ni_setup_irqs(dev); 632 if (error) { 633 device_printf(dev, "%s: failed to setup IRQs: error=%d\n", 634 __func__, error); 635 goto close_ni; 636 } 637 error = dpaa2_ni_setup_sysctls(sc); 638 if (error) { 639 device_printf(dev, "%s: failed to setup sysctls: error=%d\n", 640 __func__, error); 641 goto close_ni; 642 } 643 error = dpaa2_ni_setup_if_caps(sc); 644 if (error) { 645 device_printf(dev, "%s: failed to setup interface capabilities: " 646 "error=%d\n", __func__, error); 647 goto close_ni; 648 } 649 650 ether_ifattach(sc->ifp, sc->mac.addr); 651 callout_init(&sc->mii_callout, 0); 652 653 return (0); 654 655 close_ni: 656 DPAA2_CMD_NI_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, ni_token)); 657 close_rc: 658 DPAA2_CMD_RC_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, rc_token)); 659 err_exit: 660 return (ENXIO); 661 } 662 663 static void 664 dpaa2_ni_fixed_media_status(if_t ifp, struct ifmediareq* ifmr) 665 { 666 struct dpaa2_ni_softc *sc = if_getsoftc(ifp); 667 668 DPNI_LOCK(sc); 669 ifmr->ifm_count = 0; 670 ifmr->ifm_mask = 0; 671 ifmr->ifm_status = IFM_AVALID | IFM_ACTIVE; 672 ifmr->ifm_current = ifmr->ifm_active = 673 sc->fixed_ifmedia.ifm_cur->ifm_media; 674 675 /* 676 * In non-PHY usecases, we need to signal link state up, otherwise 677 * certain things requiring a link event (e.g async DHCP client) from 678 * devd do not happen. 679 */ 680 if (if_getlinkstate(ifp) == LINK_STATE_UNKNOWN) { 681 if_link_state_change(ifp, LINK_STATE_UP); 682 } 683 684 /* 685 * TODO: Check the status of the link partner (DPMAC, DPNI or other) and 686 * reset if down. This is different to the DPAA2_MAC_LINK_TYPE_PHY as 687 * the MC firmware sets the status, instead of us telling the MC what 688 * it is. 689 */ 690 DPNI_UNLOCK(sc); 691 692 return; 693 } 694 695 static void 696 dpaa2_ni_setup_fixed_link(struct dpaa2_ni_softc *sc) 697 { 698 /* 699 * FIXME: When the DPNI is connected to a DPMAC, we can get the 700 * 'apparent' speed from it. 701 */ 702 sc->fixed_link = true; 703 704 ifmedia_init(&sc->fixed_ifmedia, 0, dpaa2_ni_media_change, 705 dpaa2_ni_fixed_media_status); 706 ifmedia_add(&sc->fixed_ifmedia, IFM_ETHER | IFM_1000_T, 0, NULL); 707 ifmedia_set(&sc->fixed_ifmedia, IFM_ETHER | IFM_1000_T); 708 } 709 710 static int 711 dpaa2_ni_detach(device_t dev) 712 { 713 /* TBD */ 714 return (0); 715 } 716 717 /** 718 * @brief Configure DPAA2 network interface object. 719 */ 720 static int 721 dpaa2_ni_setup(device_t dev) 722 { 723 device_t pdev = device_get_parent(dev); 724 device_t child = dev; 725 struct dpaa2_ni_softc *sc = device_get_softc(dev); 726 struct dpaa2_devinfo *rcinfo = device_get_ivars(pdev); 727 struct dpaa2_devinfo *dinfo = device_get_ivars(dev); 728 struct dpaa2_ep_desc ep1_desc, ep2_desc; /* endpoint descriptors */ 729 struct dpaa2_cmd cmd; 730 uint8_t eth_bca[ETHER_ADDR_LEN]; /* broadcast physical address */ 731 uint16_t rc_token, ni_token, mac_token; 732 struct dpaa2_mac_attr attr; 733 enum dpaa2_mac_link_type link_type; 734 uint32_t link; 735 int error; 736 737 DPAA2_CMD_INIT(&cmd); 738 739 error = DPAA2_CMD_RC_OPEN(dev, child, &cmd, rcinfo->id, &rc_token); 740 if (error) { 741 device_printf(dev, "%s: failed to open resource container: " 742 "id=%d, error=%d\n", __func__, rcinfo->id, error); 743 goto err_exit; 744 } 745 error = DPAA2_CMD_NI_OPEN(dev, child, &cmd, dinfo->id, &ni_token); 746 if (error) { 747 device_printf(dev, "%s: failed to open network interface: " 748 "id=%d, error=%d\n", __func__, dinfo->id, error); 749 goto close_rc; 750 } 751 752 /* Check if we can work with this DPNI object. */ 753 error = DPAA2_CMD_NI_GET_API_VERSION(dev, child, &cmd, &sc->api_major, 754 &sc->api_minor); 755 if (error) { 756 device_printf(dev, "%s: failed to get DPNI API version\n", 757 __func__); 758 goto close_ni; 759 } 760 if (dpaa2_ni_cmp_api_version(sc, DPNI_VER_MAJOR, DPNI_VER_MINOR) < 0) { 761 device_printf(dev, "%s: DPNI API version %u.%u not supported, " 762 "need >= %u.%u\n", __func__, sc->api_major, sc->api_minor, 763 DPNI_VER_MAJOR, DPNI_VER_MINOR); 764 error = ENODEV; 765 goto close_ni; 766 } 767 768 /* Reset the DPNI object. */ 769 error = DPAA2_CMD_NI_RESET(dev, child, &cmd); 770 if (error) { 771 device_printf(dev, "%s: failed to reset DPNI: id=%d\n", 772 __func__, dinfo->id); 773 goto close_ni; 774 } 775 776 /* Obtain attributes of the DPNI object. */ 777 error = DPAA2_CMD_NI_GET_ATTRIBUTES(dev, child, &cmd, &sc->attr); 778 if (error) { 779 device_printf(dev, "%s: failed to obtain DPNI attributes: " 780 "id=%d\n", __func__, dinfo->id); 781 goto close_ni; 782 } 783 if (bootverbose) { 784 device_printf(dev, "\toptions=0x%#x queues=%d tx_channels=%d " 785 "wriop_version=%#x\n", sc->attr.options, sc->attr.num.queues, 786 sc->attr.num.channels, sc->attr.wriop_ver); 787 device_printf(dev, "\ttraffic classes: rx=%d tx=%d " 788 "cgs_groups=%d\n", sc->attr.num.rx_tcs, sc->attr.num.tx_tcs, 789 sc->attr.num.cgs); 790 device_printf(dev, "\ttable entries: mac=%d vlan=%d qos=%d " 791 "fs=%d\n", sc->attr.entries.mac, sc->attr.entries.vlan, 792 sc->attr.entries.qos, sc->attr.entries.fs); 793 device_printf(dev, "\tkey sizes: qos=%d fs=%d\n", 794 sc->attr.key_size.qos, sc->attr.key_size.fs); 795 } 796 797 /* Configure buffer layouts of the DPNI queues. */ 798 error = dpaa2_ni_set_buf_layout(dev); 799 if (error) { 800 device_printf(dev, "%s: failed to configure buffer layout\n", 801 __func__); 802 goto close_ni; 803 } 804 805 /* Configure DMA resources. */ 806 error = dpaa2_ni_setup_dma(sc); 807 if (error) { 808 device_printf(dev, "%s: failed to setup DMA\n", __func__); 809 goto close_ni; 810 } 811 812 /* Setup link between DPNI and an object it's connected to. */ 813 ep1_desc.obj_id = dinfo->id; 814 ep1_desc.if_id = 0; /* DPNI has the only endpoint */ 815 ep1_desc.type = dinfo->dtype; 816 817 error = DPAA2_CMD_RC_GET_CONN(dev, child, DPAA2_CMD_TK(&cmd, rc_token), 818 &ep1_desc, &ep2_desc, &link); 819 if (error) { 820 device_printf(dev, "%s: failed to obtain an object DPNI is " 821 "connected to: error=%d\n", __func__, error); 822 } else { 823 device_printf(dev, "connected to %s (id=%d)\n", 824 dpaa2_ttos(ep2_desc.type), ep2_desc.obj_id); 825 826 error = dpaa2_ni_set_mac_addr(dev); 827 if (error) { 828 device_printf(dev, "%s: failed to set MAC address: " 829 "error=%d\n", __func__, error); 830 } 831 832 if (ep2_desc.type == DPAA2_DEV_MAC) { 833 /* 834 * This is the simplest case when DPNI is connected to 835 * DPMAC directly. 836 */ 837 sc->mac.dpmac_id = ep2_desc.obj_id; 838 839 link_type = DPAA2_MAC_LINK_TYPE_NONE; 840 841 /* 842 * Need to determine if DPMAC type is PHY (attached to 843 * conventional MII PHY) or FIXED (usually SFP/SerDes, 844 * link state managed by MC firmware). 845 */ 846 error = DPAA2_CMD_MAC_OPEN(sc->dev, child, 847 DPAA2_CMD_TK(&cmd, rc_token), sc->mac.dpmac_id, 848 &mac_token); 849 /* 850 * Under VFIO, the DPMAC might be sitting in another 851 * container (DPRC) we don't have access to. 852 * Assume DPAA2_MAC_LINK_TYPE_FIXED if this is 853 * the case. 854 */ 855 if (error) { 856 device_printf(dev, "%s: failed to open " 857 "connected DPMAC: %d (assuming in other DPRC)\n", __func__, 858 sc->mac.dpmac_id); 859 link_type = DPAA2_MAC_LINK_TYPE_FIXED; 860 } else { 861 error = DPAA2_CMD_MAC_GET_ATTRIBUTES(dev, child, 862 &cmd, &attr); 863 if (error) { 864 device_printf(dev, "%s: failed to get " 865 "DPMAC attributes: id=%d, " 866 "error=%d\n", __func__, dinfo->id, 867 error); 868 } else { 869 link_type = attr.link_type; 870 } 871 } 872 DPAA2_CMD_MAC_CLOSE(dev, child, &cmd); 873 874 if (link_type == DPAA2_MAC_LINK_TYPE_FIXED) { 875 device_printf(dev, "connected DPMAC is in FIXED " 876 "mode\n"); 877 dpaa2_ni_setup_fixed_link(sc); 878 } else if (link_type == DPAA2_MAC_LINK_TYPE_PHY) { 879 device_printf(dev, "connected DPMAC is in PHY " 880 "mode\n"); 881 error = DPAA2_MC_GET_PHY_DEV(dev, 882 &sc->mac.phy_dev, sc->mac.dpmac_id); 883 if (error == 0) { 884 error = MEMAC_MDIO_SET_NI_DEV( 885 sc->mac.phy_dev, dev); 886 if (error != 0) { 887 device_printf(dev, "%s: failed " 888 "to set dpni dev on memac " 889 "mdio dev %s: error=%d\n", 890 __func__, 891 device_get_nameunit( 892 sc->mac.phy_dev), error); 893 } 894 } 895 if (error == 0) { 896 error = MEMAC_MDIO_GET_PHY_LOC( 897 sc->mac.phy_dev, &sc->mac.phy_loc); 898 if (error == ENODEV) { 899 error = 0; 900 } 901 if (error != 0) { 902 device_printf(dev, "%s: failed " 903 "to get phy location from " 904 "memac mdio dev %s: error=%d\n", 905 __func__, device_get_nameunit( 906 sc->mac.phy_dev), error); 907 } 908 } 909 if (error == 0) { 910 error = mii_attach(sc->mac.phy_dev, 911 &sc->miibus, sc->ifp, 912 dpaa2_ni_media_change, 913 dpaa2_ni_media_status, 914 BMSR_DEFCAPMASK, sc->mac.phy_loc, 915 MII_OFFSET_ANY, 0); 916 if (error != 0) { 917 device_printf(dev, "%s: failed " 918 "to attach to miibus: " 919 "error=%d\n", 920 __func__, error); 921 } 922 } 923 if (error == 0) { 924 sc->mii = device_get_softc(sc->miibus); 925 } 926 } else { 927 device_printf(dev, "%s: DPMAC link type is not " 928 "supported\n", __func__); 929 } 930 } else if (ep2_desc.type == DPAA2_DEV_NI || 931 ep2_desc.type == DPAA2_DEV_MUX || 932 ep2_desc.type == DPAA2_DEV_SW) { 933 dpaa2_ni_setup_fixed_link(sc); 934 } 935 } 936 937 /* Select mode to enqueue frames. */ 938 /* ... TBD ... */ 939 940 /* 941 * Update link configuration to enable Rx/Tx pause frames support. 942 * 943 * NOTE: MC may generate an interrupt to the DPMAC and request changes 944 * in link configuration. It might be necessary to attach miibus 945 * and PHY before this point. 946 */ 947 error = dpaa2_ni_set_pause_frame(dev); 948 if (error) { 949 device_printf(dev, "%s: failed to configure Rx/Tx pause " 950 "frames\n", __func__); 951 goto close_ni; 952 } 953 954 /* Configure ingress traffic classification. */ 955 error = dpaa2_ni_set_qos_table(dev); 956 if (error) { 957 device_printf(dev, "%s: failed to configure QoS table: " 958 "error=%d\n", __func__, error); 959 goto close_ni; 960 } 961 962 /* Add broadcast physical address to the MAC filtering table. */ 963 memset(eth_bca, 0xff, ETHER_ADDR_LEN); 964 error = DPAA2_CMD_NI_ADD_MAC_ADDR(dev, child, DPAA2_CMD_TK(&cmd, 965 ni_token), eth_bca); 966 if (error) { 967 device_printf(dev, "%s: failed to add broadcast physical " 968 "address to the MAC filtering table\n", __func__); 969 goto close_ni; 970 } 971 972 /* Set the maximum allowed length for received frames. */ 973 error = DPAA2_CMD_NI_SET_MFL(dev, child, &cmd, DPAA2_ETH_MFL); 974 if (error) { 975 device_printf(dev, "%s: failed to set maximum length for " 976 "received frames\n", __func__); 977 goto close_ni; 978 } 979 980 (void)DPAA2_CMD_NI_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, ni_token)); 981 (void)DPAA2_CMD_RC_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, rc_token)); 982 return (0); 983 984 close_ni: 985 (void)DPAA2_CMD_NI_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, ni_token)); 986 close_rc: 987 (void)DPAA2_CMD_RC_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, rc_token)); 988 err_exit: 989 return (error); 990 } 991 992 /** 993 * @brief Сonfigure QBMan channels and register data availability notifications. 994 */ 995 static int 996 dpaa2_ni_setup_channels(device_t dev) 997 { 998 device_t iodev, condev, bpdev; 999 struct dpaa2_ni_softc *sc = device_get_softc(dev); 1000 uint32_t i, num_chan; 1001 int error; 1002 1003 /* Calculate number of the channels based on the allocated resources */ 1004 for (i = 0; i < DPAA2_NI_IO_RES_NUM; i++) { 1005 if (!sc->res[DPAA2_NI_IO_RID(i)]) { 1006 break; 1007 } 1008 } 1009 num_chan = i; 1010 for (i = 0; i < DPAA2_NI_CON_RES_NUM; i++) { 1011 if (!sc->res[DPAA2_NI_CON_RID(i)]) { 1012 break; 1013 } 1014 } 1015 num_chan = i < num_chan ? i : num_chan; 1016 sc->chan_n = num_chan > DPAA2_MAX_CHANNELS 1017 ? DPAA2_MAX_CHANNELS : num_chan; 1018 sc->chan_n = sc->chan_n > sc->attr.num.queues 1019 ? sc->attr.num.queues : sc->chan_n; 1020 1021 KASSERT(sc->chan_n > 0u, ("%s: positive number of channels expected: " 1022 "chan_n=%d", __func__, sc->chan_n)); 1023 1024 device_printf(dev, "channels=%d\n", sc->chan_n); 1025 1026 for (i = 0; i < sc->chan_n; i++) { 1027 iodev = (device_t)rman_get_start(sc->res[DPAA2_NI_IO_RID(i)]); 1028 condev = (device_t)rman_get_start(sc->res[DPAA2_NI_CON_RID(i)]); 1029 /* Only one buffer pool available at the moment */ 1030 bpdev = (device_t)rman_get_start(sc->res[DPAA2_NI_BP_RID(0)]); 1031 1032 error = dpaa2_chan_setup(dev, iodev, condev, bpdev, 1033 &sc->channels[i], i, dpaa2_ni_cleanup_task); 1034 if (error != 0) { 1035 device_printf(dev, "%s: dpaa2_chan_setup() failed: " 1036 "error=%d, chan_id=%d\n", __func__, error, i); 1037 return (error); 1038 } 1039 } 1040 1041 /* There is exactly one Rx error queue per network interface */ 1042 error = dpaa2_chan_setup_fq(dev, sc->channels[0], DPAA2_NI_QUEUE_RX_ERR); 1043 if (error != 0) { 1044 device_printf(dev, "%s: failed to prepare RxError queue: " 1045 "error=%d\n", __func__, error); 1046 return (error); 1047 } 1048 1049 return (0); 1050 } 1051 1052 /** 1053 * @brief Bind DPNI to DPBPs, DPIOs, frame queues and channels. 1054 */ 1055 static int 1056 dpaa2_ni_bind(device_t dev) 1057 { 1058 device_t pdev = device_get_parent(dev); 1059 device_t child = dev; 1060 device_t bp_dev; 1061 struct dpaa2_ni_softc *sc = device_get_softc(dev); 1062 struct dpaa2_devinfo *rcinfo = device_get_ivars(pdev); 1063 struct dpaa2_devinfo *dinfo = device_get_ivars(dev); 1064 struct dpaa2_devinfo *bp_info; 1065 struct dpaa2_cmd cmd; 1066 struct dpaa2_ni_pools_cfg pools_cfg; 1067 struct dpaa2_ni_err_cfg err_cfg; 1068 struct dpaa2_channel *chan; 1069 uint16_t rc_token, ni_token; 1070 int error; 1071 1072 DPAA2_CMD_INIT(&cmd); 1073 1074 error = DPAA2_CMD_RC_OPEN(dev, child, &cmd, rcinfo->id, &rc_token); 1075 if (error) { 1076 device_printf(dev, "%s: failed to open resource container: " 1077 "id=%d, error=%d\n", __func__, rcinfo->id, error); 1078 goto err_exit; 1079 } 1080 error = DPAA2_CMD_NI_OPEN(dev, child, &cmd, dinfo->id, &ni_token); 1081 if (error) { 1082 device_printf(dev, "%s: failed to open network interface: " 1083 "id=%d, error=%d\n", __func__, dinfo->id, error); 1084 goto close_rc; 1085 } 1086 1087 /* Select buffer pool (only one available at the moment). */ 1088 bp_dev = (device_t) rman_get_start(sc->res[DPAA2_NI_BP_RID(0)]); 1089 bp_info = device_get_ivars(bp_dev); 1090 1091 /* Configure buffers pool. */ 1092 pools_cfg.pools_num = 1; 1093 pools_cfg.pools[0].bp_obj_id = bp_info->id; 1094 pools_cfg.pools[0].backup_flag = 0; 1095 pools_cfg.pools[0].buf_sz = sc->buf_sz; 1096 error = DPAA2_CMD_NI_SET_POOLS(dev, child, &cmd, &pools_cfg); 1097 if (error) { 1098 device_printf(dev, "%s: failed to set buffer pools\n", __func__); 1099 goto close_ni; 1100 } 1101 1102 /* Setup ingress traffic distribution. */ 1103 error = dpaa2_ni_setup_rx_dist(dev); 1104 if (error && error != EOPNOTSUPP) { 1105 device_printf(dev, "%s: failed to setup ingress traffic " 1106 "distribution\n", __func__); 1107 goto close_ni; 1108 } 1109 if (bootverbose && error == EOPNOTSUPP) { 1110 device_printf(dev, "Ingress traffic distribution not " 1111 "supported\n"); 1112 } 1113 1114 /* Configure handling of error frames. */ 1115 err_cfg.err_mask = DPAA2_NI_FAS_RX_ERR_MASK; 1116 err_cfg.set_err_fas = false; 1117 err_cfg.action = DPAA2_NI_ERR_DISCARD; 1118 error = DPAA2_CMD_NI_SET_ERR_BEHAVIOR(dev, child, &cmd, &err_cfg); 1119 if (error) { 1120 device_printf(dev, "%s: failed to set errors behavior\n", 1121 __func__); 1122 goto close_ni; 1123 } 1124 1125 /* Configure channel queues to generate CDANs. */ 1126 for (uint32_t i = 0; i < sc->chan_n; i++) { 1127 chan = sc->channels[i]; 1128 1129 /* Setup Rx flows. */ 1130 for (uint32_t j = 0; j < chan->rxq_n; j++) { 1131 error = dpaa2_ni_setup_rx_flow(dev, &chan->rx_queues[j]); 1132 if (error) { 1133 device_printf(dev, "%s: failed to setup Rx " 1134 "flow: error=%d\n", __func__, error); 1135 goto close_ni; 1136 } 1137 } 1138 1139 /* Setup Tx flow. */ 1140 error = dpaa2_ni_setup_tx_flow(dev, &chan->txc_queue); 1141 if (error) { 1142 device_printf(dev, "%s: failed to setup Tx " 1143 "flow: error=%d\n", __func__, error); 1144 goto close_ni; 1145 } 1146 } 1147 1148 /* Configure RxError queue to generate CDAN. */ 1149 error = dpaa2_ni_setup_rx_err_flow(dev, &sc->rxe_queue); 1150 if (error) { 1151 device_printf(dev, "%s: failed to setup RxError flow: " 1152 "error=%d\n", __func__, error); 1153 goto close_ni; 1154 } 1155 1156 /* 1157 * Get the Queuing Destination ID (QDID) that should be used for frame 1158 * enqueue operations. 1159 */ 1160 error = DPAA2_CMD_NI_GET_QDID(dev, child, &cmd, DPAA2_NI_QUEUE_TX, 1161 &sc->tx_qdid); 1162 if (error) { 1163 device_printf(dev, "%s: failed to get Tx queuing destination " 1164 "ID\n", __func__); 1165 goto close_ni; 1166 } 1167 1168 (void)DPAA2_CMD_NI_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, ni_token)); 1169 (void)DPAA2_CMD_RC_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, rc_token)); 1170 return (0); 1171 1172 close_ni: 1173 (void)DPAA2_CMD_NI_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, ni_token)); 1174 close_rc: 1175 (void)DPAA2_CMD_RC_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, rc_token)); 1176 err_exit: 1177 return (error); 1178 } 1179 1180 /** 1181 * @brief Setup ingress traffic distribution. 1182 * 1183 * NOTE: Ingress traffic distribution is valid only when DPNI_OPT_NO_FS option 1184 * hasn't been set for DPNI and a number of DPNI queues > 1. 1185 */ 1186 static int 1187 dpaa2_ni_setup_rx_dist(device_t dev) 1188 { 1189 /* 1190 * Have the interface implicitly distribute traffic based on the default 1191 * hash key. 1192 */ 1193 return (dpaa2_ni_set_hash(dev, DPAA2_RXH_DEFAULT)); 1194 } 1195 1196 static int 1197 dpaa2_ni_setup_rx_flow(device_t dev, struct dpaa2_ni_fq *fq) 1198 { 1199 device_t pdev = device_get_parent(dev); 1200 device_t child = dev; 1201 struct dpaa2_devinfo *rcinfo = device_get_ivars(pdev); 1202 struct dpaa2_devinfo *dinfo = device_get_ivars(dev); 1203 struct dpaa2_devinfo *con_info; 1204 struct dpaa2_cmd cmd; 1205 struct dpaa2_ni_queue_cfg queue_cfg = {0}; 1206 uint16_t rc_token, ni_token; 1207 int error; 1208 1209 DPAA2_CMD_INIT(&cmd); 1210 1211 error = DPAA2_CMD_RC_OPEN(dev, child, &cmd, rcinfo->id, &rc_token); 1212 if (error) { 1213 device_printf(dev, "%s: failed to open resource container: " 1214 "id=%d, error=%d\n", __func__, rcinfo->id, error); 1215 goto err_exit; 1216 } 1217 error = DPAA2_CMD_NI_OPEN(dev, child, &cmd, dinfo->id, &ni_token); 1218 if (error) { 1219 device_printf(dev, "%s: failed to open network interface: " 1220 "id=%d, error=%d\n", __func__, dinfo->id, error); 1221 goto close_rc; 1222 } 1223 1224 /* Obtain DPCON associated with the FQ's channel. */ 1225 con_info = device_get_ivars(fq->chan->con_dev); 1226 1227 queue_cfg.type = DPAA2_NI_QUEUE_RX; 1228 queue_cfg.tc = fq->tc; 1229 queue_cfg.idx = fq->flowid; 1230 error = DPAA2_CMD_NI_GET_QUEUE(dev, child, &cmd, &queue_cfg); 1231 if (error) { 1232 device_printf(dev, "%s: failed to obtain Rx queue " 1233 "configuration: tc=%d, flowid=%d\n", __func__, queue_cfg.tc, 1234 queue_cfg.idx); 1235 goto close_ni; 1236 } 1237 1238 fq->fqid = queue_cfg.fqid; 1239 1240 queue_cfg.dest_id = con_info->id; 1241 queue_cfg.dest_type = DPAA2_NI_DEST_DPCON; 1242 queue_cfg.priority = 1; 1243 queue_cfg.user_ctx = (uint64_t)(uintmax_t) fq; 1244 queue_cfg.options = 1245 DPAA2_NI_QUEUE_OPT_USER_CTX | 1246 DPAA2_NI_QUEUE_OPT_DEST; 1247 error = DPAA2_CMD_NI_SET_QUEUE(dev, child, &cmd, &queue_cfg); 1248 if (error) { 1249 device_printf(dev, "%s: failed to update Rx queue " 1250 "configuration: tc=%d, flowid=%d\n", __func__, queue_cfg.tc, 1251 queue_cfg.idx); 1252 goto close_ni; 1253 } 1254 1255 if (bootverbose) { 1256 device_printf(dev, "RX queue idx=%d, tc=%d, chan=%d, fqid=%d, " 1257 "user_ctx=%#jx\n", fq->flowid, fq->tc, fq->chan->id, 1258 fq->fqid, (uint64_t) fq); 1259 } 1260 1261 (void)DPAA2_CMD_NI_CLOSE(dev, child, &cmd); 1262 (void)DPAA2_CMD_RC_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, rc_token)); 1263 return (0); 1264 1265 close_ni: 1266 (void)DPAA2_CMD_NI_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, ni_token)); 1267 close_rc: 1268 (void)DPAA2_CMD_RC_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, rc_token)); 1269 err_exit: 1270 return (error); 1271 } 1272 1273 static int 1274 dpaa2_ni_setup_tx_flow(device_t dev, struct dpaa2_ni_fq *fq) 1275 { 1276 device_t pdev = device_get_parent(dev); 1277 device_t child = dev; 1278 struct dpaa2_ni_softc *sc = device_get_softc(dev); 1279 struct dpaa2_channel *ch = fq->chan; 1280 struct dpaa2_devinfo *rcinfo = device_get_ivars(pdev); 1281 struct dpaa2_devinfo *dinfo = device_get_ivars(dev); 1282 struct dpaa2_devinfo *con_info; 1283 struct dpaa2_ni_queue_cfg queue_cfg = {0}; 1284 struct dpaa2_ni_tx_ring *tx; 1285 struct dpaa2_buf *buf; 1286 struct dpaa2_cmd cmd; 1287 uint32_t tx_rings_n = 0; 1288 uint16_t rc_token, ni_token; 1289 int error; 1290 1291 DPAA2_CMD_INIT(&cmd); 1292 1293 error = DPAA2_CMD_RC_OPEN(dev, child, &cmd, rcinfo->id, &rc_token); 1294 if (error) { 1295 device_printf(dev, "%s: failed to open resource container: " 1296 "id=%d, error=%d\n", __func__, rcinfo->id, error); 1297 goto err_exit; 1298 } 1299 error = DPAA2_CMD_NI_OPEN(dev, child, &cmd, dinfo->id, &ni_token); 1300 if (error) { 1301 device_printf(dev, "%s: failed to open network interface: " 1302 "id=%d, error=%d\n", __func__, dinfo->id, error); 1303 goto close_rc; 1304 } 1305 1306 /* Obtain DPCON associated with the FQ's channel. */ 1307 con_info = device_get_ivars(fq->chan->con_dev); 1308 1309 KASSERT(sc->attr.num.tx_tcs <= DPAA2_MAX_TCS, 1310 ("%s: too many Tx traffic classes: tx_tcs=%d\n", __func__, 1311 sc->attr.num.tx_tcs)); 1312 KASSERT(DPAA2_NI_BUFS_PER_TX <= DPAA2_NI_MAX_BPTX, 1313 ("%s: too many Tx buffers (%d): max=%d\n", __func__, 1314 DPAA2_NI_BUFS_PER_TX, DPAA2_NI_MAX_BPTX)); 1315 1316 /* Setup Tx rings. */ 1317 for (int i = 0; i < sc->attr.num.tx_tcs; i++) { 1318 queue_cfg.type = DPAA2_NI_QUEUE_TX; 1319 queue_cfg.tc = i; 1320 queue_cfg.idx = fq->flowid; 1321 queue_cfg.chan_id = fq->chan->id; 1322 1323 error = DPAA2_CMD_NI_GET_QUEUE(dev, child, &cmd, &queue_cfg); 1324 if (error) { 1325 device_printf(dev, "%s: failed to obtain Tx queue " 1326 "configuration: tc=%d, flowid=%d\n", __func__, 1327 queue_cfg.tc, queue_cfg.idx); 1328 goto close_ni; 1329 } 1330 1331 tx = &fq->tx_rings[i]; 1332 tx->fq = fq; 1333 tx->fqid = queue_cfg.fqid; 1334 tx->txid = tx_rings_n; 1335 1336 if (bootverbose) { 1337 device_printf(dev, "TX queue idx=%d, tc=%d, chan=%d, " 1338 "fqid=%d\n", fq->flowid, i, fq->chan->id, 1339 queue_cfg.fqid); 1340 } 1341 1342 mtx_init(&tx->lock, "dpaa2_tx_ring", NULL, MTX_DEF); 1343 1344 /* Allocate Tx ring buffer. */ 1345 tx->br = buf_ring_alloc(DPAA2_TX_BUFRING_SZ, M_DEVBUF, M_NOWAIT, 1346 &tx->lock); 1347 if (tx->br == NULL) { 1348 device_printf(dev, "%s: failed to setup Tx ring buffer" 1349 " (2) fqid=%d\n", __func__, tx->fqid); 1350 goto close_ni; 1351 } 1352 1353 /* Configure Tx buffers */ 1354 for (uint64_t j = 0; j < DPAA2_NI_BUFS_PER_TX; j++) { 1355 buf = malloc(sizeof(struct dpaa2_buf), M_DPAA2_TXB, 1356 M_WAITOK); 1357 /* Keep DMA tag and Tx ring linked to the buffer */ 1358 DPAA2_BUF_INIT_TAGOPT(buf, ch->tx_dmat, tx); 1359 1360 buf->sgt = malloc(sizeof(struct dpaa2_buf), M_DPAA2_TXB, 1361 M_WAITOK); 1362 /* Link SGT to DMA tag and back to its Tx buffer */ 1363 DPAA2_BUF_INIT_TAGOPT(buf->sgt, ch->sgt_dmat, buf); 1364 1365 error = dpaa2_buf_seed_txb(dev, buf); 1366 1367 /* Add Tx buffer to the ring */ 1368 buf_ring_enqueue(tx->br, buf); 1369 } 1370 1371 tx_rings_n++; 1372 } 1373 1374 /* All Tx queues which belong to the same flowid have the same qdbin. */ 1375 fq->tx_qdbin = queue_cfg.qdbin; 1376 1377 queue_cfg.type = DPAA2_NI_QUEUE_TX_CONF; 1378 queue_cfg.tc = 0; /* ignored for TxConf queue */ 1379 queue_cfg.idx = fq->flowid; 1380 error = DPAA2_CMD_NI_GET_QUEUE(dev, child, &cmd, &queue_cfg); 1381 if (error) { 1382 device_printf(dev, "%s: failed to obtain TxConf queue " 1383 "configuration: tc=%d, flowid=%d\n", __func__, queue_cfg.tc, 1384 queue_cfg.idx); 1385 goto close_ni; 1386 } 1387 1388 fq->fqid = queue_cfg.fqid; 1389 1390 queue_cfg.dest_id = con_info->id; 1391 queue_cfg.dest_type = DPAA2_NI_DEST_DPCON; 1392 queue_cfg.priority = 0; 1393 queue_cfg.user_ctx = (uint64_t)(uintmax_t) fq; 1394 queue_cfg.options = 1395 DPAA2_NI_QUEUE_OPT_USER_CTX | 1396 DPAA2_NI_QUEUE_OPT_DEST; 1397 error = DPAA2_CMD_NI_SET_QUEUE(dev, child, &cmd, &queue_cfg); 1398 if (error) { 1399 device_printf(dev, "%s: failed to update TxConf queue " 1400 "configuration: tc=%d, flowid=%d\n", __func__, queue_cfg.tc, 1401 queue_cfg.idx); 1402 goto close_ni; 1403 } 1404 1405 (void)DPAA2_CMD_NI_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, ni_token)); 1406 (void)DPAA2_CMD_RC_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, rc_token)); 1407 return (0); 1408 1409 close_ni: 1410 (void)DPAA2_CMD_NI_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, ni_token)); 1411 close_rc: 1412 (void)DPAA2_CMD_RC_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, rc_token)); 1413 err_exit: 1414 return (error); 1415 } 1416 1417 static int 1418 dpaa2_ni_setup_rx_err_flow(device_t dev, struct dpaa2_ni_fq *fq) 1419 { 1420 device_t pdev = device_get_parent(dev); 1421 device_t child = dev; 1422 struct dpaa2_devinfo *rcinfo = device_get_ivars(pdev); 1423 struct dpaa2_devinfo *dinfo = device_get_ivars(dev); 1424 struct dpaa2_devinfo *con_info; 1425 struct dpaa2_ni_queue_cfg queue_cfg = {0}; 1426 struct dpaa2_cmd cmd; 1427 uint16_t rc_token, ni_token; 1428 int error; 1429 1430 DPAA2_CMD_INIT(&cmd); 1431 1432 error = DPAA2_CMD_RC_OPEN(dev, child, &cmd, rcinfo->id, &rc_token); 1433 if (error) { 1434 device_printf(dev, "%s: failed to open resource container: " 1435 "id=%d, error=%d\n", __func__, rcinfo->id, error); 1436 goto err_exit; 1437 } 1438 error = DPAA2_CMD_NI_OPEN(dev, child, &cmd, dinfo->id, &ni_token); 1439 if (error) { 1440 device_printf(dev, "%s: failed to open network interface: " 1441 "id=%d, error=%d\n", __func__, dinfo->id, error); 1442 goto close_rc; 1443 } 1444 1445 /* Obtain DPCON associated with the FQ's channel. */ 1446 con_info = device_get_ivars(fq->chan->con_dev); 1447 1448 queue_cfg.type = DPAA2_NI_QUEUE_RX_ERR; 1449 queue_cfg.tc = fq->tc; /* ignored */ 1450 queue_cfg.idx = fq->flowid; /* ignored */ 1451 error = DPAA2_CMD_NI_GET_QUEUE(dev, child, &cmd, &queue_cfg); 1452 if (error) { 1453 device_printf(dev, "%s: failed to obtain RxErr queue " 1454 "configuration\n", __func__); 1455 goto close_ni; 1456 } 1457 1458 fq->fqid = queue_cfg.fqid; 1459 1460 queue_cfg.dest_id = con_info->id; 1461 queue_cfg.dest_type = DPAA2_NI_DEST_DPCON; 1462 queue_cfg.priority = 1; 1463 queue_cfg.user_ctx = (uint64_t)(uintmax_t) fq; 1464 queue_cfg.options = 1465 DPAA2_NI_QUEUE_OPT_USER_CTX | 1466 DPAA2_NI_QUEUE_OPT_DEST; 1467 error = DPAA2_CMD_NI_SET_QUEUE(dev, child, &cmd, &queue_cfg); 1468 if (error) { 1469 device_printf(dev, "%s: failed to update RxErr queue " 1470 "configuration\n", __func__); 1471 goto close_ni; 1472 } 1473 1474 (void)DPAA2_CMD_NI_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, ni_token)); 1475 (void)DPAA2_CMD_RC_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, rc_token)); 1476 return (0); 1477 1478 close_ni: 1479 (void)DPAA2_CMD_NI_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, ni_token)); 1480 close_rc: 1481 (void)DPAA2_CMD_RC_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, rc_token)); 1482 err_exit: 1483 return (error); 1484 } 1485 1486 /** 1487 * @brief Configure DPNI object to generate interrupts. 1488 */ 1489 static int 1490 dpaa2_ni_setup_irqs(device_t dev) 1491 { 1492 device_t pdev = device_get_parent(dev); 1493 device_t child = dev; 1494 struct dpaa2_ni_softc *sc = device_get_softc(dev); 1495 struct dpaa2_devinfo *rcinfo = device_get_ivars(pdev); 1496 struct dpaa2_devinfo *dinfo = device_get_ivars(dev); 1497 struct dpaa2_cmd cmd; 1498 uint16_t rc_token, ni_token; 1499 int error; 1500 1501 DPAA2_CMD_INIT(&cmd); 1502 1503 error = DPAA2_CMD_RC_OPEN(dev, child, &cmd, rcinfo->id, &rc_token); 1504 if (error) { 1505 device_printf(dev, "%s: failed to open resource container: " 1506 "id=%d, error=%d\n", __func__, rcinfo->id, error); 1507 goto err_exit; 1508 } 1509 error = DPAA2_CMD_NI_OPEN(dev, child, &cmd, dinfo->id, &ni_token); 1510 if (error) { 1511 device_printf(dev, "%s: failed to open network interface: " 1512 "id=%d, error=%d\n", __func__, dinfo->id, error); 1513 goto close_rc; 1514 } 1515 1516 /* Configure IRQs. */ 1517 error = dpaa2_ni_setup_msi(sc); 1518 if (error) { 1519 device_printf(dev, "%s: failed to allocate MSI\n", __func__); 1520 goto close_ni; 1521 } 1522 if ((sc->irq_res = bus_alloc_resource_any(dev, SYS_RES_IRQ, 1523 &sc->irq_rid[0], RF_ACTIVE | RF_SHAREABLE)) == NULL) { 1524 device_printf(dev, "%s: failed to allocate IRQ resource\n", 1525 __func__); 1526 goto close_ni; 1527 } 1528 if (bus_setup_intr(dev, sc->irq_res, INTR_TYPE_NET | INTR_MPSAFE, 1529 NULL, dpaa2_ni_intr, sc, &sc->intr)) { 1530 device_printf(dev, "%s: failed to setup IRQ resource\n", 1531 __func__); 1532 goto close_ni; 1533 } 1534 1535 error = DPAA2_CMD_NI_SET_IRQ_MASK(dev, child, &cmd, DPNI_IRQ_INDEX, 1536 DPNI_IRQ_LINK_CHANGED | DPNI_IRQ_EP_CHANGED); 1537 if (error) { 1538 device_printf(dev, "%s: failed to set DPNI IRQ mask\n", 1539 __func__); 1540 goto close_ni; 1541 } 1542 1543 error = DPAA2_CMD_NI_SET_IRQ_ENABLE(dev, child, &cmd, DPNI_IRQ_INDEX, 1544 true); 1545 if (error) { 1546 device_printf(dev, "%s: failed to enable DPNI IRQ\n", __func__); 1547 goto close_ni; 1548 } 1549 1550 (void)DPAA2_CMD_NI_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, ni_token)); 1551 (void)DPAA2_CMD_RC_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, rc_token)); 1552 return (0); 1553 1554 close_ni: 1555 (void)DPAA2_CMD_NI_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, ni_token)); 1556 close_rc: 1557 (void)DPAA2_CMD_RC_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, rc_token)); 1558 err_exit: 1559 return (error); 1560 } 1561 1562 /** 1563 * @brief Allocate MSI interrupts for DPNI. 1564 */ 1565 static int 1566 dpaa2_ni_setup_msi(struct dpaa2_ni_softc *sc) 1567 { 1568 int val; 1569 1570 val = pci_msi_count(sc->dev); 1571 if (val < DPAA2_NI_MSI_COUNT) 1572 device_printf(sc->dev, "MSI: actual=%d, expected=%d\n", val, 1573 DPAA2_IO_MSI_COUNT); 1574 val = MIN(val, DPAA2_NI_MSI_COUNT); 1575 1576 if (pci_alloc_msi(sc->dev, &val) != 0) 1577 return (EINVAL); 1578 1579 for (int i = 0; i < val; i++) 1580 sc->irq_rid[i] = i + 1; 1581 1582 return (0); 1583 } 1584 1585 /** 1586 * @brief Update DPNI according to the updated interface capabilities. 1587 */ 1588 static int 1589 dpaa2_ni_setup_if_caps(struct dpaa2_ni_softc *sc) 1590 { 1591 bool en_rxcsum, en_txcsum; 1592 device_t pdev = device_get_parent(sc->dev); 1593 device_t dev = sc->dev; 1594 device_t child = dev; 1595 struct dpaa2_devinfo *rcinfo = device_get_ivars(pdev); 1596 struct dpaa2_devinfo *dinfo = device_get_ivars(dev); 1597 struct dpaa2_cmd cmd; 1598 uint16_t rc_token, ni_token; 1599 int error; 1600 1601 DPAA2_CMD_INIT(&cmd); 1602 1603 /* 1604 * XXX-DSL: DPAA2 allows to validate L3/L4 checksums on reception and/or 1605 * generate L3/L4 checksums on transmission without 1606 * differentiating between IPv4/v6, i.e. enable for both 1607 * protocols if requested. 1608 */ 1609 en_rxcsum = if_getcapenable(sc->ifp) & 1610 (IFCAP_RXCSUM | IFCAP_RXCSUM_IPV6); 1611 en_txcsum = if_getcapenable(sc->ifp) & 1612 (IFCAP_TXCSUM | IFCAP_TXCSUM_IPV6); 1613 1614 error = DPAA2_CMD_RC_OPEN(dev, child, &cmd, rcinfo->id, &rc_token); 1615 if (error) { 1616 device_printf(dev, "%s: failed to open resource container: " 1617 "id=%d, error=%d\n", __func__, rcinfo->id, error); 1618 goto err_exit; 1619 } 1620 error = DPAA2_CMD_NI_OPEN(dev, child, &cmd, dinfo->id, &ni_token); 1621 if (error) { 1622 device_printf(dev, "%s: failed to open network interface: " 1623 "id=%d, error=%d\n", __func__, dinfo->id, error); 1624 goto close_rc; 1625 } 1626 1627 /* Setup checksums validation. */ 1628 error = DPAA2_CMD_NI_SET_OFFLOAD(dev, child, &cmd, 1629 DPAA2_NI_OFL_RX_L3_CSUM, en_rxcsum); 1630 if (error) { 1631 device_printf(dev, "%s: failed to %s L3 checksum validation\n", 1632 __func__, en_rxcsum ? "enable" : "disable"); 1633 goto close_ni; 1634 } 1635 error = DPAA2_CMD_NI_SET_OFFLOAD(dev, child, &cmd, 1636 DPAA2_NI_OFL_RX_L4_CSUM, en_rxcsum); 1637 if (error) { 1638 device_printf(dev, "%s: failed to %s L4 checksum validation\n", 1639 __func__, en_rxcsum ? "enable" : "disable"); 1640 goto close_ni; 1641 } 1642 1643 /* Setup checksums generation. */ 1644 error = DPAA2_CMD_NI_SET_OFFLOAD(dev, child, &cmd, 1645 DPAA2_NI_OFL_TX_L3_CSUM, en_txcsum); 1646 if (error) { 1647 device_printf(dev, "%s: failed to %s L3 checksum generation\n", 1648 __func__, en_txcsum ? "enable" : "disable"); 1649 goto close_ni; 1650 } 1651 error = DPAA2_CMD_NI_SET_OFFLOAD(dev, child, &cmd, 1652 DPAA2_NI_OFL_TX_L4_CSUM, en_txcsum); 1653 if (error) { 1654 device_printf(dev, "%s: failed to %s L4 checksum generation\n", 1655 __func__, en_txcsum ? "enable" : "disable"); 1656 goto close_ni; 1657 } 1658 1659 if (bootverbose) { 1660 device_printf(dev, "%s: L3/L4 checksum validation %s\n", 1661 __func__, en_rxcsum ? "enabled" : "disabled"); 1662 device_printf(dev, "%s: L3/L4 checksum generation %s\n", 1663 __func__, en_txcsum ? "enabled" : "disabled"); 1664 } 1665 1666 (void)DPAA2_CMD_NI_CLOSE(dev, child, &cmd); 1667 (void)DPAA2_CMD_RC_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, rc_token)); 1668 return (0); 1669 1670 close_ni: 1671 (void)DPAA2_CMD_NI_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, ni_token)); 1672 close_rc: 1673 (void)DPAA2_CMD_RC_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, rc_token)); 1674 err_exit: 1675 return (error); 1676 } 1677 1678 /** 1679 * @brief Update DPNI according to the updated interface flags. 1680 */ 1681 static int 1682 dpaa2_ni_setup_if_flags(struct dpaa2_ni_softc *sc) 1683 { 1684 const bool en_promisc = if_getflags(sc->ifp) & IFF_PROMISC; 1685 const bool en_allmulti = if_getflags(sc->ifp) & IFF_ALLMULTI; 1686 device_t pdev = device_get_parent(sc->dev); 1687 device_t dev = sc->dev; 1688 device_t child = dev; 1689 struct dpaa2_devinfo *rcinfo = device_get_ivars(pdev); 1690 struct dpaa2_devinfo *dinfo = device_get_ivars(dev); 1691 struct dpaa2_cmd cmd; 1692 uint16_t rc_token, ni_token; 1693 int error; 1694 1695 DPAA2_CMD_INIT(&cmd); 1696 1697 error = DPAA2_CMD_RC_OPEN(dev, child, &cmd, rcinfo->id, &rc_token); 1698 if (error) { 1699 device_printf(dev, "%s: failed to open resource container: " 1700 "id=%d, error=%d\n", __func__, rcinfo->id, error); 1701 goto err_exit; 1702 } 1703 error = DPAA2_CMD_NI_OPEN(dev, child, &cmd, dinfo->id, &ni_token); 1704 if (error) { 1705 device_printf(dev, "%s: failed to open network interface: " 1706 "id=%d, error=%d\n", __func__, dinfo->id, error); 1707 goto close_rc; 1708 } 1709 1710 error = DPAA2_CMD_NI_SET_MULTI_PROMISC(dev, child, &cmd, 1711 en_promisc ? true : en_allmulti); 1712 if (error) { 1713 device_printf(dev, "%s: failed to %s multicast promiscuous " 1714 "mode\n", __func__, en_allmulti ? "enable" : "disable"); 1715 goto close_ni; 1716 } 1717 1718 error = DPAA2_CMD_NI_SET_UNI_PROMISC(dev, child, &cmd, en_promisc); 1719 if (error) { 1720 device_printf(dev, "%s: failed to %s unicast promiscuous mode\n", 1721 __func__, en_promisc ? "enable" : "disable"); 1722 goto close_ni; 1723 } 1724 1725 (void)DPAA2_CMD_NI_CLOSE(dev, child, &cmd); 1726 (void)DPAA2_CMD_RC_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, rc_token)); 1727 return (0); 1728 1729 close_ni: 1730 (void)DPAA2_CMD_NI_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, ni_token)); 1731 close_rc: 1732 (void)DPAA2_CMD_RC_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, rc_token)); 1733 err_exit: 1734 return (error); 1735 } 1736 1737 static int 1738 dpaa2_ni_setup_sysctls(struct dpaa2_ni_softc *sc) 1739 { 1740 struct sysctl_ctx_list *ctx; 1741 struct sysctl_oid *node, *node2; 1742 struct sysctl_oid_list *parent, *parent2; 1743 char cbuf[128]; 1744 int i; 1745 1746 ctx = device_get_sysctl_ctx(sc->dev); 1747 parent = SYSCTL_CHILDREN(device_get_sysctl_tree(sc->dev)); 1748 1749 /* Add DPNI statistics. */ 1750 node = SYSCTL_ADD_NODE(ctx, parent, OID_AUTO, "stats", 1751 CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "DPNI Statistics"); 1752 parent = SYSCTL_CHILDREN(node); 1753 for (i = 0; i < nitems(dpni_stat_sysctls); ++i) { 1754 SYSCTL_ADD_PROC(ctx, parent, i, dpni_stat_sysctls[i].name, 1755 CTLTYPE_U64 | CTLFLAG_RD, sc, 0, dpaa2_ni_collect_stats, 1756 "IU", dpni_stat_sysctls[i].desc); 1757 } 1758 SYSCTL_ADD_UQUAD(ctx, parent, OID_AUTO, "rx_anomaly_frames", 1759 CTLFLAG_RD, &sc->rx_anomaly_frames, 1760 "Rx frames in the buffers outside of the buffer pools"); 1761 SYSCTL_ADD_UQUAD(ctx, parent, OID_AUTO, "rx_single_buf_frames", 1762 CTLFLAG_RD, &sc->rx_single_buf_frames, 1763 "Rx frames in single buffers"); 1764 SYSCTL_ADD_UQUAD(ctx, parent, OID_AUTO, "rx_sg_buf_frames", 1765 CTLFLAG_RD, &sc->rx_sg_buf_frames, 1766 "Rx frames in scatter/gather list"); 1767 SYSCTL_ADD_UQUAD(ctx, parent, OID_AUTO, "rx_enq_rej_frames", 1768 CTLFLAG_RD, &sc->rx_enq_rej_frames, 1769 "Enqueue rejected by QMan"); 1770 SYSCTL_ADD_UQUAD(ctx, parent, OID_AUTO, "rx_ieoi_err_frames", 1771 CTLFLAG_RD, &sc->rx_ieoi_err_frames, 1772 "QMan IEOI error"); 1773 SYSCTL_ADD_UQUAD(ctx, parent, OID_AUTO, "rx_other_err_frames", 1774 CTLFLAG_RD, &sc->rx_other_err_frames, 1775 "Other Rx frames with errors"); 1776 SYSCTL_ADD_UQUAD(ctx, parent, OID_AUTO, "tx_single_buf_frames", 1777 CTLFLAG_RD, &sc->tx_single_buf_frames, 1778 "Tx single buffer frames"); 1779 SYSCTL_ADD_UQUAD(ctx, parent, OID_AUTO, "tx_sg_frames", 1780 CTLFLAG_RD, &sc->tx_sg_frames, 1781 "Tx S/G frames"); 1782 1783 SYSCTL_ADD_PROC(ctx, parent, OID_AUTO, "buf_num", 1784 CTLTYPE_U32 | CTLFLAG_RD, sc, 0, dpaa2_ni_collect_buf_num, 1785 "IU", "number of Rx buffers in the buffer pool"); 1786 SYSCTL_ADD_PROC(ctx, parent, OID_AUTO, "buf_free", 1787 CTLTYPE_U32 | CTLFLAG_RD, sc, 0, dpaa2_ni_collect_buf_free, 1788 "IU", "number of free Rx buffers in the buffer pool"); 1789 1790 /* Add channels statistics. */ 1791 parent = SYSCTL_CHILDREN(device_get_sysctl_tree(sc->dev)); 1792 node = SYSCTL_ADD_NODE(ctx, parent, OID_AUTO, "channels", 1793 CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "DPNI Channels"); 1794 parent = SYSCTL_CHILDREN(node); 1795 for (int i = 0; i < sc->chan_n; i++) { 1796 snprintf(cbuf, sizeof(cbuf), "%d", i); 1797 1798 node2 = SYSCTL_ADD_NODE(ctx, parent, OID_AUTO, cbuf, 1799 CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "DPNI Channel"); 1800 parent2 = SYSCTL_CHILDREN(node2); 1801 1802 SYSCTL_ADD_UQUAD(ctx, parent2, OID_AUTO, "tx_frames", 1803 CTLFLAG_RD, &sc->channels[i]->tx_frames, 1804 "Tx frames counter"); 1805 SYSCTL_ADD_UQUAD(ctx, parent2, OID_AUTO, "tx_dropped", 1806 CTLFLAG_RD, &sc->channels[i]->tx_dropped, 1807 "Tx dropped counter"); 1808 } 1809 1810 /* Add Link debugging options. */ 1811 parent = SYSCTL_CHILDREN(device_get_sysctl_tree(sc->dev)); 1812 node = SYSCTL_ADD_PROC(ctx, parent, OID_AUTO, "link", 1813 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, 1814 sc, 0, dpaa2_ni_sysctl_link_state, 1815 "A", "DPNI link state information"); 1816 1817 return (0); 1818 } 1819 1820 static int 1821 dpaa2_ni_setup_dma(struct dpaa2_ni_softc *sc) 1822 { 1823 device_t dev = sc->dev; 1824 int error; 1825 1826 KASSERT((sc->buf_align == BUF_ALIGN) || (sc->buf_align == BUF_ALIGN_V1), 1827 ("unexpected buffer alignment: %d\n", sc->buf_align)); 1828 1829 /* DMA tag for Rx distribution key. */ 1830 error = bus_dma_tag_create( 1831 bus_get_dma_tag(dev), 1832 PAGE_SIZE, 0, /* alignment, boundary */ 1833 BUS_SPACE_MAXADDR, /* low restricted addr */ 1834 BUS_SPACE_MAXADDR, /* high restricted addr */ 1835 NULL, NULL, /* filter, filterarg */ 1836 DPAA2_CLASSIFIER_DMA_SIZE, 1, /* maxsize, nsegments */ 1837 DPAA2_CLASSIFIER_DMA_SIZE, 0, /* maxsegsize, flags */ 1838 NULL, NULL, /* lockfunc, lockarg */ 1839 &sc->rxd_dmat); 1840 if (error) { 1841 device_printf(dev, "%s: failed to create DMA tag for Rx " 1842 "distribution key\n", __func__); 1843 return (error); 1844 } 1845 1846 error = bus_dma_tag_create( 1847 bus_get_dma_tag(dev), 1848 PAGE_SIZE, 0, /* alignment, boundary */ 1849 BUS_SPACE_MAXADDR, /* low restricted addr */ 1850 BUS_SPACE_MAXADDR, /* high restricted addr */ 1851 NULL, NULL, /* filter, filterarg */ 1852 ETH_QOS_KCFG_BUF_SIZE, 1, /* maxsize, nsegments */ 1853 ETH_QOS_KCFG_BUF_SIZE, 0, /* maxsegsize, flags */ 1854 NULL, NULL, /* lockfunc, lockarg */ 1855 &sc->qos_dmat); 1856 if (error) { 1857 device_printf(dev, "%s: failed to create DMA tag for QoS key\n", 1858 __func__); 1859 return (error); 1860 } 1861 1862 return (0); 1863 } 1864 1865 /** 1866 * @brief Configure buffer layouts of the different DPNI queues. 1867 */ 1868 static int 1869 dpaa2_ni_set_buf_layout(device_t dev) 1870 { 1871 device_t pdev = device_get_parent(dev); 1872 device_t child = dev; 1873 struct dpaa2_devinfo *rcinfo = device_get_ivars(pdev); 1874 struct dpaa2_devinfo *dinfo = device_get_ivars(dev); 1875 struct dpaa2_ni_softc *sc = device_get_softc(dev); 1876 struct dpaa2_ni_buf_layout buf_layout = {0}; 1877 struct dpaa2_cmd cmd; 1878 uint16_t rc_token, ni_token; 1879 int error; 1880 1881 DPAA2_CMD_INIT(&cmd); 1882 1883 error = DPAA2_CMD_RC_OPEN(dev, child, &cmd, rcinfo->id, &rc_token); 1884 if (error) { 1885 device_printf(dev, "%s: failed to open resource container: " 1886 "id=%d, error=%d\n", __func__, rcinfo->id, error); 1887 goto err_exit; 1888 } 1889 error = DPAA2_CMD_NI_OPEN(dev, child, &cmd, dinfo->id, &ni_token); 1890 if (error) { 1891 device_printf(sc->dev, "%s: failed to open DPMAC: id=%d, " 1892 "error=%d\n", __func__, dinfo->id, error); 1893 goto close_rc; 1894 } 1895 1896 /* 1897 * Select Rx/Tx buffer alignment. It's necessary to ensure that the 1898 * buffer size seen by WRIOP is a multiple of 64 or 256 bytes depending 1899 * on the WRIOP version. 1900 */ 1901 sc->buf_align = (sc->attr.wriop_ver == WRIOP_VERSION(0, 0, 0) || 1902 sc->attr.wriop_ver == WRIOP_VERSION(1, 0, 0)) 1903 ? BUF_ALIGN_V1 : BUF_ALIGN; 1904 1905 /* 1906 * We need to ensure that the buffer size seen by WRIOP is a multiple 1907 * of 64 or 256 bytes depending on the WRIOP version. 1908 */ 1909 sc->buf_sz = ALIGN_DOWN(DPAA2_RX_BUF_SIZE, sc->buf_align); 1910 1911 if (bootverbose) { 1912 device_printf(dev, "Rx/Tx buffers: size=%d, alignment=%d\n", 1913 sc->buf_sz, sc->buf_align); 1914 } 1915 1916 /* 1917 * Frame Descriptor Tx buffer layout 1918 * 1919 * ADDR -> |---------------------| 1920 * | SW FRAME ANNOTATION | BUF_SWA_SIZE bytes 1921 * |---------------------| 1922 * | HW FRAME ANNOTATION | BUF_TX_HWA_SIZE bytes 1923 * |---------------------| 1924 * | DATA HEADROOM | 1925 * ADDR + OFFSET -> |---------------------| 1926 * | | 1927 * | | 1928 * | FRAME DATA | 1929 * | | 1930 * | | 1931 * |---------------------| 1932 * | DATA TAILROOM | 1933 * |---------------------| 1934 * 1935 * NOTE: It's for a single buffer frame only. 1936 */ 1937 buf_layout.queue_type = DPAA2_NI_QUEUE_TX; 1938 buf_layout.pd_size = BUF_SWA_SIZE; 1939 buf_layout.pass_timestamp = true; 1940 buf_layout.pass_frame_status = true; 1941 buf_layout.options = 1942 BUF_LOPT_PRIV_DATA_SZ | 1943 BUF_LOPT_TIMESTAMP | /* requires 128 bytes in HWA */ 1944 BUF_LOPT_FRAME_STATUS; 1945 error = DPAA2_CMD_NI_SET_BUF_LAYOUT(dev, child, &cmd, &buf_layout); 1946 if (error) { 1947 device_printf(dev, "%s: failed to set Tx buffer layout\n", 1948 __func__); 1949 goto close_ni; 1950 } 1951 1952 /* Tx-confirmation buffer layout */ 1953 buf_layout.queue_type = DPAA2_NI_QUEUE_TX_CONF; 1954 buf_layout.options = 1955 BUF_LOPT_TIMESTAMP | 1956 BUF_LOPT_FRAME_STATUS; 1957 error = DPAA2_CMD_NI_SET_BUF_LAYOUT(dev, child, &cmd, &buf_layout); 1958 if (error) { 1959 device_printf(dev, "%s: failed to set TxConf buffer layout\n", 1960 __func__); 1961 goto close_ni; 1962 } 1963 1964 /* 1965 * Driver should reserve the amount of space indicated by this command 1966 * as headroom in all Tx frames. 1967 */ 1968 error = DPAA2_CMD_NI_GET_TX_DATA_OFF(dev, child, &cmd, &sc->tx_data_off); 1969 if (error) { 1970 device_printf(dev, "%s: failed to obtain Tx data offset\n", 1971 __func__); 1972 goto close_ni; 1973 } 1974 1975 if (bootverbose) { 1976 device_printf(dev, "Tx data offset=%d\n", sc->tx_data_off); 1977 } 1978 if ((sc->tx_data_off % 64) != 0) { 1979 device_printf(dev, "Tx data offset (%d) is not a multiplication " 1980 "of 64 bytes\n", sc->tx_data_off); 1981 } 1982 1983 /* 1984 * Frame Descriptor Rx buffer layout 1985 * 1986 * ADDR -> |---------------------| 1987 * | SW FRAME ANNOTATION | BUF_SWA_SIZE bytes 1988 * |---------------------| 1989 * | HW FRAME ANNOTATION | BUF_RX_HWA_SIZE bytes 1990 * |---------------------| 1991 * | DATA HEADROOM | OFFSET-BUF_RX_HWA_SIZE 1992 * ADDR + OFFSET -> |---------------------| 1993 * | | 1994 * | | 1995 * | FRAME DATA | 1996 * | | 1997 * | | 1998 * |---------------------| 1999 * | DATA TAILROOM | 0 bytes 2000 * |---------------------| 2001 * 2002 * NOTE: It's for a single buffer frame only. 2003 */ 2004 buf_layout.queue_type = DPAA2_NI_QUEUE_RX; 2005 buf_layout.pd_size = BUF_SWA_SIZE; 2006 buf_layout.fd_align = sc->buf_align; 2007 buf_layout.head_size = sc->tx_data_off - BUF_RX_HWA_SIZE - BUF_SWA_SIZE; 2008 buf_layout.tail_size = 0; 2009 buf_layout.pass_frame_status = true; 2010 buf_layout.pass_parser_result = true; 2011 buf_layout.pass_timestamp = true; 2012 buf_layout.options = 2013 BUF_LOPT_PRIV_DATA_SZ | 2014 BUF_LOPT_DATA_ALIGN | 2015 BUF_LOPT_DATA_HEAD_ROOM | 2016 BUF_LOPT_DATA_TAIL_ROOM | 2017 BUF_LOPT_FRAME_STATUS | 2018 BUF_LOPT_PARSER_RESULT | 2019 BUF_LOPT_TIMESTAMP; 2020 error = DPAA2_CMD_NI_SET_BUF_LAYOUT(dev, child, &cmd, &buf_layout); 2021 if (error) { 2022 device_printf(dev, "%s: failed to set Rx buffer layout\n", 2023 __func__); 2024 goto close_ni; 2025 } 2026 2027 error = 0; 2028 close_ni: 2029 (void)DPAA2_CMD_NI_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, ni_token)); 2030 close_rc: 2031 (void)DPAA2_CMD_RC_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, rc_token)); 2032 err_exit: 2033 return (error); 2034 } 2035 2036 /** 2037 * @brief Enable Rx/Tx pause frames. 2038 * 2039 * NOTE: DPNI stops sending when a pause frame is received (Rx frame) or DPNI 2040 * itself generates pause frames (Tx frame). 2041 */ 2042 static int 2043 dpaa2_ni_set_pause_frame(device_t dev) 2044 { 2045 device_t pdev = device_get_parent(dev); 2046 device_t child = dev; 2047 struct dpaa2_devinfo *rcinfo = device_get_ivars(pdev); 2048 struct dpaa2_devinfo *dinfo = device_get_ivars(dev); 2049 struct dpaa2_ni_softc *sc = device_get_softc(dev); 2050 struct dpaa2_ni_link_cfg link_cfg = {0}; 2051 struct dpaa2_cmd cmd; 2052 uint16_t rc_token, ni_token; 2053 int error; 2054 2055 DPAA2_CMD_INIT(&cmd); 2056 2057 error = DPAA2_CMD_RC_OPEN(dev, child, &cmd, rcinfo->id, &rc_token); 2058 if (error) { 2059 device_printf(dev, "%s: failed to open resource container: " 2060 "id=%d, error=%d\n", __func__, rcinfo->id, error); 2061 goto err_exit; 2062 } 2063 error = DPAA2_CMD_NI_OPEN(dev, child, &cmd, dinfo->id, &ni_token); 2064 if (error) { 2065 device_printf(sc->dev, "%s: failed to open DPMAC: id=%d, " 2066 "error=%d\n", __func__, dinfo->id, error); 2067 goto close_rc; 2068 } 2069 2070 error = DPAA2_CMD_NI_GET_LINK_CFG(dev, child, &cmd, &link_cfg); 2071 if (error) { 2072 device_printf(dev, "%s: failed to obtain link configuration: " 2073 "error=%d\n", __func__, error); 2074 goto close_ni; 2075 } 2076 2077 /* Enable both Rx and Tx pause frames by default. */ 2078 link_cfg.options |= DPAA2_NI_LINK_OPT_PAUSE; 2079 link_cfg.options &= ~DPAA2_NI_LINK_OPT_ASYM_PAUSE; 2080 2081 error = DPAA2_CMD_NI_SET_LINK_CFG(dev, child, &cmd, &link_cfg); 2082 if (error) { 2083 device_printf(dev, "%s: failed to set link configuration: " 2084 "error=%d\n", __func__, error); 2085 goto close_ni; 2086 } 2087 2088 sc->link_options = link_cfg.options; 2089 error = 0; 2090 close_ni: 2091 (void)DPAA2_CMD_NI_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, ni_token)); 2092 close_rc: 2093 (void)DPAA2_CMD_RC_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, rc_token)); 2094 err_exit: 2095 return (error); 2096 } 2097 2098 /** 2099 * @brief Configure QoS table to determine the traffic class for the received 2100 * frame. 2101 */ 2102 static int 2103 dpaa2_ni_set_qos_table(device_t dev) 2104 { 2105 device_t pdev = device_get_parent(dev); 2106 device_t child = dev; 2107 struct dpaa2_devinfo *rcinfo = device_get_ivars(pdev); 2108 struct dpaa2_devinfo *dinfo = device_get_ivars(dev); 2109 struct dpaa2_ni_softc *sc = device_get_softc(dev); 2110 struct dpaa2_ni_qos_table tbl; 2111 struct dpaa2_buf *buf = &sc->qos_kcfg; 2112 struct dpaa2_cmd cmd; 2113 uint16_t rc_token, ni_token; 2114 int error; 2115 2116 if (sc->attr.num.rx_tcs == 1 || 2117 !(sc->attr.options & DPNI_OPT_HAS_KEY_MASKING)) { 2118 if (bootverbose) { 2119 device_printf(dev, "Ingress traffic classification is " 2120 "not supported\n"); 2121 } 2122 return (0); 2123 } 2124 2125 /* 2126 * Allocate a buffer visible to the device to hold the QoS table key 2127 * configuration. 2128 */ 2129 2130 if (__predict_true(buf->dmat == NULL)) { 2131 buf->dmat = sc->qos_dmat; 2132 } 2133 2134 error = bus_dmamem_alloc(buf->dmat, (void **)&buf->vaddr, 2135 BUS_DMA_ZERO | BUS_DMA_COHERENT, &buf->dmap); 2136 if (error) { 2137 device_printf(dev, "%s: failed to allocate a buffer for QoS key " 2138 "configuration\n", __func__); 2139 goto err_exit; 2140 } 2141 2142 error = bus_dmamap_load(buf->dmat, buf->dmap, buf->vaddr, 2143 ETH_QOS_KCFG_BUF_SIZE, dpaa2_dmamap_oneseg_cb, &buf->paddr, 2144 BUS_DMA_NOWAIT); 2145 if (error) { 2146 device_printf(dev, "%s: failed to map QoS key configuration " 2147 "buffer into bus space\n", __func__); 2148 goto err_exit; 2149 } 2150 2151 DPAA2_CMD_INIT(&cmd); 2152 2153 error = DPAA2_CMD_RC_OPEN(dev, child, &cmd, rcinfo->id, &rc_token); 2154 if (error) { 2155 device_printf(dev, "%s: failed to open resource container: " 2156 "id=%d, error=%d\n", __func__, rcinfo->id, error); 2157 goto err_exit; 2158 } 2159 error = DPAA2_CMD_NI_OPEN(dev, child, &cmd, dinfo->id, &ni_token); 2160 if (error) { 2161 device_printf(sc->dev, "%s: failed to open DPMAC: id=%d, " 2162 "error=%d\n", __func__, dinfo->id, error); 2163 goto close_rc; 2164 } 2165 2166 tbl.default_tc = 0; 2167 tbl.discard_on_miss = false; 2168 tbl.keep_entries = false; 2169 tbl.kcfg_busaddr = buf->paddr; 2170 error = DPAA2_CMD_NI_SET_QOS_TABLE(dev, child, &cmd, &tbl); 2171 if (error) { 2172 device_printf(dev, "%s: failed to set QoS table\n", __func__); 2173 goto close_ni; 2174 } 2175 2176 error = DPAA2_CMD_NI_CLEAR_QOS_TABLE(dev, child, &cmd); 2177 if (error) { 2178 device_printf(dev, "%s: failed to clear QoS table\n", __func__); 2179 goto close_ni; 2180 } 2181 2182 error = 0; 2183 close_ni: 2184 (void)DPAA2_CMD_NI_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, ni_token)); 2185 close_rc: 2186 (void)DPAA2_CMD_RC_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, rc_token)); 2187 err_exit: 2188 return (error); 2189 } 2190 2191 static int 2192 dpaa2_ni_set_mac_addr(device_t dev) 2193 { 2194 device_t pdev = device_get_parent(dev); 2195 device_t child = dev; 2196 struct dpaa2_ni_softc *sc = device_get_softc(dev); 2197 if_t ifp = sc->ifp; 2198 struct dpaa2_devinfo *rcinfo = device_get_ivars(pdev); 2199 struct dpaa2_devinfo *dinfo = device_get_ivars(dev); 2200 struct dpaa2_cmd cmd; 2201 struct ether_addr rnd_mac_addr; 2202 uint16_t rc_token, ni_token; 2203 uint8_t mac_addr[ETHER_ADDR_LEN]; 2204 uint8_t dpni_mac_addr[ETHER_ADDR_LEN]; 2205 int error; 2206 2207 DPAA2_CMD_INIT(&cmd); 2208 2209 error = DPAA2_CMD_RC_OPEN(dev, child, &cmd, rcinfo->id, &rc_token); 2210 if (error) { 2211 device_printf(dev, "%s: failed to open resource container: " 2212 "id=%d, error=%d\n", __func__, rcinfo->id, error); 2213 goto err_exit; 2214 } 2215 error = DPAA2_CMD_NI_OPEN(dev, child, &cmd, dinfo->id, &ni_token); 2216 if (error) { 2217 device_printf(sc->dev, "%s: failed to open DPMAC: id=%d, " 2218 "error=%d\n", __func__, dinfo->id, error); 2219 goto close_rc; 2220 } 2221 2222 /* 2223 * Get the MAC address associated with the physical port, if the DPNI is 2224 * connected to a DPMAC directly associated with one of the physical 2225 * ports. 2226 */ 2227 error = DPAA2_CMD_NI_GET_PORT_MAC_ADDR(dev, child, &cmd, mac_addr); 2228 if (error) { 2229 device_printf(dev, "%s: failed to obtain the MAC address " 2230 "associated with the physical port\n", __func__); 2231 goto close_ni; 2232 } 2233 2234 /* Get primary MAC address from the DPNI attributes. */ 2235 error = DPAA2_CMD_NI_GET_PRIM_MAC_ADDR(dev, child, &cmd, dpni_mac_addr); 2236 if (error) { 2237 device_printf(dev, "%s: failed to obtain primary MAC address\n", 2238 __func__); 2239 goto close_ni; 2240 } 2241 2242 if (!ETHER_IS_ZERO(mac_addr)) { 2243 /* Set MAC address of the physical port as DPNI's primary one. */ 2244 error = DPAA2_CMD_NI_SET_PRIM_MAC_ADDR(dev, child, &cmd, 2245 mac_addr); 2246 if (error) { 2247 device_printf(dev, "%s: failed to set primary MAC " 2248 "address\n", __func__); 2249 goto close_ni; 2250 } 2251 for (int i = 0; i < ETHER_ADDR_LEN; i++) { 2252 sc->mac.addr[i] = mac_addr[i]; 2253 } 2254 } else if (ETHER_IS_ZERO(dpni_mac_addr)) { 2255 /* Generate random MAC address as DPNI's primary one. */ 2256 ether_gen_addr(ifp, &rnd_mac_addr); 2257 for (int i = 0; i < ETHER_ADDR_LEN; i++) { 2258 mac_addr[i] = rnd_mac_addr.octet[i]; 2259 } 2260 2261 error = DPAA2_CMD_NI_SET_PRIM_MAC_ADDR(dev, child, &cmd, 2262 mac_addr); 2263 if (error) { 2264 device_printf(dev, "%s: failed to set random primary " 2265 "MAC address\n", __func__); 2266 goto close_ni; 2267 } 2268 for (int i = 0; i < ETHER_ADDR_LEN; i++) { 2269 sc->mac.addr[i] = mac_addr[i]; 2270 } 2271 } else { 2272 for (int i = 0; i < ETHER_ADDR_LEN; i++) { 2273 sc->mac.addr[i] = dpni_mac_addr[i]; 2274 } 2275 } 2276 2277 error = 0; 2278 close_ni: 2279 (void)DPAA2_CMD_NI_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, ni_token)); 2280 close_rc: 2281 (void)DPAA2_CMD_RC_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, rc_token)); 2282 err_exit: 2283 return (error); 2284 } 2285 2286 static void 2287 dpaa2_ni_miibus_statchg(device_t dev) 2288 { 2289 device_t pdev = device_get_parent(dev); 2290 device_t child = dev; 2291 struct dpaa2_ni_softc *sc = device_get_softc(dev); 2292 struct dpaa2_mac_link_state mac_link = { 0 }; 2293 struct dpaa2_devinfo *rcinfo = device_get_ivars(pdev); 2294 struct dpaa2_cmd cmd; 2295 uint16_t rc_token, mac_token; 2296 int error, link_state; 2297 2298 if (sc->fixed_link || sc->mii == NULL) { 2299 return; 2300 } 2301 if ((if_getdrvflags(sc->ifp) & IFF_DRV_RUNNING) == 0) { 2302 /* 2303 * We will receive calls and adjust the changes but 2304 * not have setup everything (called before dpaa2_ni_init() 2305 * really). This will then setup the link and internal 2306 * sc->link_state and not trigger the update once needed, 2307 * so basically dpmac never knows about it. 2308 */ 2309 return; 2310 } 2311 2312 /* 2313 * Note: ifp link state will only be changed AFTER we are called so we 2314 * cannot rely on ifp->if_linkstate here. 2315 */ 2316 if (sc->mii->mii_media_status & IFM_AVALID) { 2317 if (sc->mii->mii_media_status & IFM_ACTIVE) { 2318 link_state = LINK_STATE_UP; 2319 } else { 2320 link_state = LINK_STATE_DOWN; 2321 } 2322 } else { 2323 link_state = LINK_STATE_UNKNOWN; 2324 } 2325 2326 if (link_state != sc->link_state) { 2327 sc->link_state = link_state; 2328 2329 DPAA2_CMD_INIT(&cmd); 2330 2331 error = DPAA2_CMD_RC_OPEN(dev, child, &cmd, rcinfo->id, 2332 &rc_token); 2333 if (error) { 2334 device_printf(dev, "%s: failed to open resource " 2335 "container: id=%d, error=%d\n", __func__, rcinfo->id, 2336 error); 2337 goto err_exit; 2338 } 2339 error = DPAA2_CMD_MAC_OPEN(dev, child, &cmd, sc->mac.dpmac_id, 2340 &mac_token); 2341 if (error) { 2342 device_printf(sc->dev, "%s: failed to open DPMAC: " 2343 "id=%d, error=%d\n", __func__, sc->mac.dpmac_id, 2344 error); 2345 goto close_rc; 2346 } 2347 2348 if (link_state == LINK_STATE_UP || 2349 link_state == LINK_STATE_DOWN) { 2350 /* Update DPMAC link state. */ 2351 mac_link.supported = sc->mii->mii_media.ifm_media; 2352 mac_link.advert = sc->mii->mii_media.ifm_media; 2353 mac_link.rate = 1000; /* TODO: Where to get from? */ /* ifmedia_baudrate? */ 2354 mac_link.options = 2355 DPAA2_MAC_LINK_OPT_AUTONEG | 2356 DPAA2_MAC_LINK_OPT_PAUSE; 2357 mac_link.up = (link_state == LINK_STATE_UP) ? true : false; 2358 mac_link.state_valid = true; 2359 2360 /* Inform DPMAC about link state. */ 2361 error = DPAA2_CMD_MAC_SET_LINK_STATE(dev, child, &cmd, 2362 &mac_link); 2363 if (error) { 2364 device_printf(sc->dev, "%s: failed to set DPMAC " 2365 "link state: id=%d, error=%d\n", __func__, 2366 sc->mac.dpmac_id, error); 2367 } 2368 } 2369 (void)DPAA2_CMD_MAC_CLOSE(dev, child, &cmd); 2370 (void)DPAA2_CMD_RC_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, 2371 rc_token)); 2372 } 2373 2374 return; 2375 2376 close_rc: 2377 (void)DPAA2_CMD_RC_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, rc_token)); 2378 err_exit: 2379 return; 2380 } 2381 2382 /** 2383 * @brief Callback function to process media change request. 2384 */ 2385 static int 2386 dpaa2_ni_media_change_locked(struct dpaa2_ni_softc *sc) 2387 { 2388 2389 DPNI_LOCK_ASSERT(sc); 2390 if (sc->mii) { 2391 mii_mediachg(sc->mii); 2392 sc->media_status = sc->mii->mii_media.ifm_media; 2393 } else if (sc->fixed_link) { 2394 if_printf(sc->ifp, "%s: can't change media in fixed mode\n", 2395 __func__); 2396 } 2397 2398 return (0); 2399 } 2400 2401 static int 2402 dpaa2_ni_media_change(if_t ifp) 2403 { 2404 struct dpaa2_ni_softc *sc = if_getsoftc(ifp); 2405 int error; 2406 2407 DPNI_LOCK(sc); 2408 error = dpaa2_ni_media_change_locked(sc); 2409 DPNI_UNLOCK(sc); 2410 return (error); 2411 } 2412 2413 /** 2414 * @brief Callback function to process media status request. 2415 */ 2416 static void 2417 dpaa2_ni_media_status(if_t ifp, struct ifmediareq *ifmr) 2418 { 2419 struct dpaa2_ni_softc *sc = if_getsoftc(ifp); 2420 2421 DPNI_LOCK(sc); 2422 if (sc->mii) { 2423 mii_pollstat(sc->mii); 2424 ifmr->ifm_active = sc->mii->mii_media_active; 2425 ifmr->ifm_status = sc->mii->mii_media_status; 2426 } 2427 DPNI_UNLOCK(sc); 2428 } 2429 2430 /** 2431 * @brief Callout function to check and update media status. 2432 */ 2433 static void 2434 dpaa2_ni_media_tick(void *arg) 2435 { 2436 struct dpaa2_ni_softc *sc = (struct dpaa2_ni_softc *) arg; 2437 2438 /* Check for media type change */ 2439 if (sc->mii) { 2440 mii_tick(sc->mii); 2441 if (sc->media_status != sc->mii->mii_media.ifm_media) { 2442 printf("%s: media type changed (ifm_media=%x)\n", 2443 __func__, sc->mii->mii_media.ifm_media); 2444 dpaa2_ni_media_change(sc->ifp); 2445 } 2446 } 2447 2448 /* Schedule another timeout one second from now */ 2449 callout_reset(&sc->mii_callout, hz, dpaa2_ni_media_tick, sc); 2450 } 2451 2452 static void 2453 dpaa2_ni_init(void *arg) 2454 { 2455 struct dpaa2_ni_softc *sc = (struct dpaa2_ni_softc *) arg; 2456 if_t ifp = sc->ifp; 2457 device_t pdev = device_get_parent(sc->dev); 2458 device_t dev = sc->dev; 2459 device_t child = dev; 2460 struct dpaa2_devinfo *rcinfo = device_get_ivars(pdev); 2461 struct dpaa2_devinfo *dinfo = device_get_ivars(dev); 2462 struct dpaa2_cmd cmd; 2463 uint16_t rc_token, ni_token; 2464 int error; 2465 2466 DPNI_LOCK(sc); 2467 if ((if_getdrvflags(ifp) & IFF_DRV_RUNNING) != 0) { 2468 DPNI_UNLOCK(sc); 2469 return; 2470 } 2471 DPNI_UNLOCK(sc); 2472 2473 DPAA2_CMD_INIT(&cmd); 2474 2475 error = DPAA2_CMD_RC_OPEN(dev, child, &cmd, rcinfo->id, &rc_token); 2476 if (error) { 2477 device_printf(dev, "%s: failed to open resource container: " 2478 "id=%d, error=%d\n", __func__, rcinfo->id, error); 2479 goto err_exit; 2480 } 2481 error = DPAA2_CMD_NI_OPEN(dev, child, &cmd, dinfo->id, &ni_token); 2482 if (error) { 2483 device_printf(dev, "%s: failed to open network interface: " 2484 "id=%d, error=%d\n", __func__, dinfo->id, error); 2485 goto close_rc; 2486 } 2487 2488 error = DPAA2_CMD_NI_ENABLE(dev, child, &cmd); 2489 if (error) { 2490 device_printf(dev, "%s: failed to enable DPNI: error=%d\n", 2491 __func__, error); 2492 } 2493 2494 error = dpaa2_ni_setup_if_flags(sc); 2495 if (error) { 2496 device_printf(dev, "%s: failed to update interface flags: " 2497 "error=%d\n", __func__, error); 2498 } 2499 error = dpaa2_ni_update_mac_filters(ifp); 2500 if (error) { 2501 device_printf(dev, "%s: failed to update MAC filters: " 2502 "error=%d\n", __func__, error); 2503 } 2504 2505 DPNI_LOCK(sc); 2506 /* Announce we are up and running and can queue packets. */ 2507 if_setdrvflagbits(ifp, IFF_DRV_RUNNING, IFF_DRV_OACTIVE); 2508 2509 if (sc->mii) { 2510 /* 2511 * mii_mediachg() will trigger a call into 2512 * dpaa2_ni_miibus_statchg() to setup link state. 2513 */ 2514 dpaa2_ni_media_change_locked(sc); 2515 } 2516 callout_reset(&sc->mii_callout, hz, dpaa2_ni_media_tick, sc); 2517 2518 DPNI_UNLOCK(sc); 2519 2520 (void)DPAA2_CMD_NI_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, ni_token)); 2521 (void)DPAA2_CMD_RC_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, rc_token)); 2522 return; 2523 2524 close_rc: 2525 (void)DPAA2_CMD_RC_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, rc_token)); 2526 err_exit: 2527 return; 2528 } 2529 2530 static int 2531 dpaa2_ni_transmit(if_t ifp, struct mbuf *m) 2532 { 2533 struct dpaa2_ni_softc *sc = if_getsoftc(ifp); 2534 struct dpaa2_channel *ch; 2535 uint32_t fqid; 2536 bool found = false; 2537 int chidx = 0, error; 2538 2539 if (__predict_false(!(if_getdrvflags(ifp) & IFF_DRV_RUNNING))) { 2540 return (0); 2541 } 2542 2543 if (M_HASHTYPE_GET(m) != M_HASHTYPE_NONE) { 2544 fqid = m->m_pkthdr.flowid; 2545 for (int i = 0; i < sc->chan_n; i++) { 2546 ch = sc->channels[i]; 2547 for (int j = 0; j < ch->rxq_n; j++) { 2548 if (fqid == ch->rx_queues[j].fqid) { 2549 chidx = ch->flowid; 2550 found = true; 2551 break; 2552 } 2553 } 2554 if (found) { 2555 break; 2556 } 2557 } 2558 } 2559 2560 ch = sc->channels[chidx]; 2561 error = buf_ring_enqueue(ch->xmit_br, m); 2562 if (__predict_false(error != 0)) { 2563 if_inc_counter(ifp, IFCOUNTER_OQDROPS, 1); 2564 m_freem(m); 2565 } else { 2566 taskqueue_enqueue(ch->cleanup_tq, &ch->cleanup_task); 2567 } 2568 2569 return (error); 2570 } 2571 2572 static void 2573 dpaa2_ni_qflush(if_t ifp) 2574 { 2575 /* TODO: Find a way to drain Tx queues in QBMan. */ 2576 if_qflush(ifp); 2577 } 2578 2579 static int 2580 dpaa2_ni_ioctl(if_t ifp, u_long c, caddr_t data) 2581 { 2582 struct dpaa2_ni_softc *sc = if_getsoftc(ifp); 2583 struct ifreq *ifr = (struct ifreq *) data; 2584 device_t pdev = device_get_parent(sc->dev); 2585 device_t dev = sc->dev; 2586 device_t child = dev; 2587 struct dpaa2_devinfo *rcinfo = device_get_ivars(pdev); 2588 struct dpaa2_devinfo *dinfo = device_get_ivars(dev); 2589 struct dpaa2_cmd cmd; 2590 uint32_t changed = 0; 2591 uint16_t rc_token, ni_token; 2592 int mtu, error, rc = 0; 2593 2594 DPAA2_CMD_INIT(&cmd); 2595 2596 error = DPAA2_CMD_RC_OPEN(dev, child, &cmd, rcinfo->id, &rc_token); 2597 if (error) { 2598 device_printf(dev, "%s: failed to open resource container: " 2599 "id=%d, error=%d\n", __func__, rcinfo->id, error); 2600 goto err_exit; 2601 } 2602 error = DPAA2_CMD_NI_OPEN(dev, child, &cmd, dinfo->id, &ni_token); 2603 if (error) { 2604 device_printf(dev, "%s: failed to open network interface: " 2605 "id=%d, error=%d\n", __func__, dinfo->id, error); 2606 goto close_rc; 2607 } 2608 2609 switch (c) { 2610 case SIOCSIFMTU: 2611 DPNI_LOCK(sc); 2612 mtu = ifr->ifr_mtu; 2613 if (mtu < ETHERMIN || mtu > ETHERMTU_JUMBO) { 2614 DPNI_UNLOCK(sc); 2615 error = EINVAL; 2616 goto close_ni; 2617 } 2618 if_setmtu(ifp, mtu); 2619 DPNI_UNLOCK(sc); 2620 2621 /* Update maximum frame length. */ 2622 mtu += ETHER_HDR_LEN; 2623 if (if_getcapenable(ifp) & IFCAP_VLAN_MTU) 2624 mtu += ETHER_VLAN_ENCAP_LEN; 2625 error = DPAA2_CMD_NI_SET_MFL(dev, child, &cmd, mtu); 2626 if (error) { 2627 device_printf(dev, "%s: failed to update maximum frame " 2628 "length: error=%d\n", __func__, error); 2629 goto close_ni; 2630 } 2631 break; 2632 case SIOCSIFCAP: 2633 changed = if_getcapenable(ifp) ^ ifr->ifr_reqcap; 2634 if ((changed & (IFCAP_RXCSUM | IFCAP_RXCSUM_IPV6)) != 0) 2635 if_togglecapenable(ifp, IFCAP_RXCSUM | IFCAP_RXCSUM_IPV6); 2636 if ((changed & (IFCAP_TXCSUM | IFCAP_TXCSUM_IPV6)) != 0) { 2637 if_togglecapenable(ifp, IFCAP_TXCSUM | IFCAP_TXCSUM_IPV6); 2638 if_togglehwassist(ifp, DPAA2_CSUM_TX_OFFLOAD); 2639 } 2640 2641 rc = dpaa2_ni_setup_if_caps(sc); 2642 if (rc) { 2643 printf("%s: failed to update iface capabilities: " 2644 "error=%d\n", __func__, rc); 2645 rc = ENXIO; 2646 } 2647 break; 2648 case SIOCSIFFLAGS: 2649 DPNI_LOCK(sc); 2650 if (if_getflags(ifp) & IFF_UP) { 2651 if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) { 2652 changed = if_getflags(ifp) ^ sc->if_flags; 2653 if (changed & IFF_PROMISC || 2654 changed & IFF_ALLMULTI) { 2655 rc = dpaa2_ni_setup_if_flags(sc); 2656 } 2657 } else { 2658 DPNI_UNLOCK(sc); 2659 dpaa2_ni_init(sc); 2660 DPNI_LOCK(sc); 2661 } 2662 } else if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) { 2663 /* FIXME: Disable DPNI. See dpaa2_ni_init(). */ 2664 } 2665 2666 sc->if_flags = if_getflags(ifp); 2667 DPNI_UNLOCK(sc); 2668 break; 2669 case SIOCADDMULTI: 2670 case SIOCDELMULTI: 2671 DPNI_LOCK(sc); 2672 if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) { 2673 DPNI_UNLOCK(sc); 2674 rc = dpaa2_ni_update_mac_filters(ifp); 2675 if (rc) { 2676 device_printf(dev, "%s: failed to update MAC " 2677 "filters: error=%d\n", __func__, rc); 2678 } 2679 DPNI_LOCK(sc); 2680 } 2681 DPNI_UNLOCK(sc); 2682 break; 2683 case SIOCGIFMEDIA: 2684 case SIOCSIFMEDIA: 2685 if (sc->mii) 2686 rc = ifmedia_ioctl(ifp, ifr, &sc->mii->mii_media, c); 2687 else if(sc->fixed_link) { 2688 rc = ifmedia_ioctl(ifp, ifr, &sc->fixed_ifmedia, c); 2689 } 2690 break; 2691 default: 2692 rc = ether_ioctl(ifp, c, data); 2693 break; 2694 } 2695 2696 (void)DPAA2_CMD_NI_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, ni_token)); 2697 (void)DPAA2_CMD_RC_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, rc_token)); 2698 return (rc); 2699 2700 close_ni: 2701 (void)DPAA2_CMD_NI_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, ni_token)); 2702 close_rc: 2703 (void)DPAA2_CMD_RC_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, rc_token)); 2704 err_exit: 2705 return (error); 2706 } 2707 2708 static int 2709 dpaa2_ni_update_mac_filters(if_t ifp) 2710 { 2711 struct dpaa2_ni_softc *sc = if_getsoftc(ifp); 2712 struct dpaa2_ni_mcaddr_ctx ctx; 2713 device_t pdev = device_get_parent(sc->dev); 2714 device_t dev = sc->dev; 2715 device_t child = dev; 2716 struct dpaa2_devinfo *rcinfo = device_get_ivars(pdev); 2717 struct dpaa2_devinfo *dinfo = device_get_ivars(dev); 2718 struct dpaa2_cmd cmd; 2719 uint16_t rc_token, ni_token; 2720 int error; 2721 2722 DPAA2_CMD_INIT(&cmd); 2723 2724 error = DPAA2_CMD_RC_OPEN(dev, child, &cmd, rcinfo->id, &rc_token); 2725 if (error) { 2726 device_printf(dev, "%s: failed to open resource container: " 2727 "id=%d, error=%d\n", __func__, rcinfo->id, error); 2728 goto err_exit; 2729 } 2730 error = DPAA2_CMD_NI_OPEN(dev, child, &cmd, dinfo->id, &ni_token); 2731 if (error) { 2732 device_printf(dev, "%s: failed to open network interface: " 2733 "id=%d, error=%d\n", __func__, dinfo->id, error); 2734 goto close_rc; 2735 } 2736 2737 /* Remove all multicast MAC filters. */ 2738 error = DPAA2_CMD_NI_CLEAR_MAC_FILTERS(dev, child, &cmd, false, true); 2739 if (error) { 2740 device_printf(dev, "%s: failed to clear multicast MAC filters: " 2741 "error=%d\n", __func__, error); 2742 goto close_ni; 2743 } 2744 2745 ctx.ifp = ifp; 2746 ctx.error = 0; 2747 ctx.nent = 0; 2748 2749 if_foreach_llmaddr(ifp, dpaa2_ni_add_maddr, &ctx); 2750 2751 error = ctx.error; 2752 close_ni: 2753 (void)DPAA2_CMD_NI_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, ni_token)); 2754 close_rc: 2755 (void)DPAA2_CMD_RC_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, rc_token)); 2756 err_exit: 2757 return (error); 2758 } 2759 2760 static u_int 2761 dpaa2_ni_add_maddr(void *arg, struct sockaddr_dl *sdl, u_int cnt) 2762 { 2763 struct dpaa2_ni_mcaddr_ctx *ctx = arg; 2764 struct dpaa2_ni_softc *sc = if_getsoftc(ctx->ifp); 2765 device_t pdev = device_get_parent(sc->dev); 2766 device_t dev = sc->dev; 2767 device_t child = dev; 2768 struct dpaa2_devinfo *rcinfo = device_get_ivars(pdev); 2769 struct dpaa2_devinfo *dinfo = device_get_ivars(dev); 2770 struct dpaa2_cmd cmd; 2771 uint16_t rc_token, ni_token; 2772 int error; 2773 2774 if (ctx->error != 0) { 2775 return (0); 2776 } 2777 2778 if (ETHER_IS_MULTICAST(LLADDR(sdl))) { 2779 DPAA2_CMD_INIT(&cmd); 2780 2781 error = DPAA2_CMD_RC_OPEN(dev, child, &cmd, rcinfo->id, 2782 &rc_token); 2783 if (error) { 2784 device_printf(dev, "%s: failed to open resource " 2785 "container: id=%d, error=%d\n", __func__, rcinfo->id, 2786 error); 2787 return (0); 2788 } 2789 error = DPAA2_CMD_NI_OPEN(dev, child, &cmd, dinfo->id, 2790 &ni_token); 2791 if (error) { 2792 device_printf(dev, "%s: failed to open network interface: " 2793 "id=%d, error=%d\n", __func__, dinfo->id, error); 2794 (void)DPAA2_CMD_RC_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, 2795 rc_token)); 2796 return (0); 2797 } 2798 2799 ctx->error = DPAA2_CMD_NI_ADD_MAC_ADDR(dev, child, &cmd, 2800 LLADDR(sdl)); 2801 2802 (void)DPAA2_CMD_NI_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, 2803 ni_token)); 2804 (void)DPAA2_CMD_RC_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, 2805 rc_token)); 2806 2807 if (ctx->error != 0) { 2808 device_printf(dev, "%s: can't add more then %d MAC " 2809 "addresses, switching to the multicast promiscuous " 2810 "mode\n", __func__, ctx->nent); 2811 2812 /* Enable multicast promiscuous mode. */ 2813 DPNI_LOCK(sc); 2814 if_setflagbits(ctx->ifp, IFF_ALLMULTI, 0); 2815 sc->if_flags |= IFF_ALLMULTI; 2816 ctx->error = dpaa2_ni_setup_if_flags(sc); 2817 DPNI_UNLOCK(sc); 2818 2819 return (0); 2820 } 2821 ctx->nent++; 2822 } 2823 2824 return (1); 2825 } 2826 2827 static void 2828 dpaa2_ni_intr(void *arg) 2829 { 2830 struct dpaa2_ni_softc *sc = (struct dpaa2_ni_softc *) arg; 2831 device_t pdev = device_get_parent(sc->dev); 2832 device_t dev = sc->dev; 2833 device_t child = dev; 2834 struct dpaa2_devinfo *rcinfo = device_get_ivars(pdev); 2835 struct dpaa2_devinfo *dinfo = device_get_ivars(dev); 2836 struct dpaa2_cmd cmd; 2837 uint32_t status = ~0u; /* clear all IRQ status bits */ 2838 uint16_t rc_token, ni_token; 2839 int error; 2840 2841 DPAA2_CMD_INIT(&cmd); 2842 2843 error = DPAA2_CMD_RC_OPEN(dev, child, &cmd, rcinfo->id, &rc_token); 2844 if (error) { 2845 device_printf(dev, "%s: failed to open resource container: " 2846 "id=%d, error=%d\n", __func__, rcinfo->id, error); 2847 goto err_exit; 2848 } 2849 error = DPAA2_CMD_NI_OPEN(dev, child, &cmd, dinfo->id, &ni_token); 2850 if (error) { 2851 device_printf(dev, "%s: failed to open network interface: " 2852 "id=%d, error=%d\n", __func__, dinfo->id, error); 2853 goto close_rc; 2854 } 2855 2856 error = DPAA2_CMD_NI_GET_IRQ_STATUS(dev, child, &cmd, DPNI_IRQ_INDEX, 2857 &status); 2858 if (error) { 2859 device_printf(sc->dev, "%s: failed to obtain IRQ status: " 2860 "error=%d\n", __func__, error); 2861 } 2862 2863 (void)DPAA2_CMD_NI_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, ni_token)); 2864 close_rc: 2865 (void)DPAA2_CMD_RC_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, rc_token)); 2866 err_exit: 2867 return; 2868 } 2869 2870 /** 2871 * @brief Execute channel's Rx/Tx routines. 2872 * 2873 * NOTE: Should not be re-entrant for the same channel. It is achieved by 2874 * enqueuing the cleanup routine on a single-threaded taskqueue. 2875 */ 2876 static void 2877 dpaa2_ni_cleanup_task(void *arg, int count) 2878 { 2879 struct dpaa2_channel *ch = (struct dpaa2_channel *)arg; 2880 struct dpaa2_ni_softc *sc = device_get_softc(ch->ni_dev); 2881 int error, rxc, txc; 2882 2883 for (int i = 0; i < DPAA2_CLEAN_BUDGET; i++) { 2884 rxc = dpaa2_ni_rx_cleanup(ch); 2885 txc = dpaa2_ni_tx_cleanup(ch); 2886 2887 if (__predict_false((if_getdrvflags(sc->ifp) & 2888 IFF_DRV_RUNNING) == 0)) { 2889 return; 2890 } 2891 2892 if ((txc != DPAA2_TX_BUDGET) && (rxc != DPAA2_RX_BUDGET)) { 2893 break; 2894 } 2895 } 2896 2897 /* Re-arm channel to generate CDAN */ 2898 error = DPAA2_SWP_CONF_WQ_CHANNEL(ch->io_dev, &ch->ctx); 2899 if (error != 0) { 2900 panic("%s: failed to rearm channel: chan_id=%d, error=%d\n", 2901 __func__, ch->id, error); 2902 } 2903 } 2904 2905 /** 2906 * @brief Poll frames from a specific channel when CDAN is received. 2907 */ 2908 static int 2909 dpaa2_ni_rx_cleanup(struct dpaa2_channel *ch) 2910 { 2911 struct dpaa2_io_softc *iosc = device_get_softc(ch->io_dev); 2912 struct dpaa2_swp *swp = iosc->swp; 2913 struct dpaa2_ni_fq *fq; 2914 struct dpaa2_buf *buf = &ch->store; 2915 int budget = DPAA2_RX_BUDGET; 2916 int error, consumed = 0; 2917 2918 do { 2919 error = dpaa2_swp_pull(swp, ch->id, buf, DPAA2_ETH_STORE_FRAMES); 2920 if (error) { 2921 device_printf(ch->ni_dev, "%s: failed to pull frames: " 2922 "chan_id=%d, error=%d\n", __func__, ch->id, error); 2923 break; 2924 } 2925 error = dpaa2_ni_consume_frames(ch, &fq, &consumed); 2926 if (error == ENOENT || error == EALREADY) { 2927 break; 2928 } 2929 if (error == ETIMEDOUT) { 2930 device_printf(ch->ni_dev, "%s: timeout to consume " 2931 "frames: chan_id=%d\n", __func__, ch->id); 2932 } 2933 } while (--budget); 2934 2935 return (DPAA2_RX_BUDGET - budget); 2936 } 2937 2938 static int 2939 dpaa2_ni_tx_cleanup(struct dpaa2_channel *ch) 2940 { 2941 struct dpaa2_ni_softc *sc = device_get_softc(ch->ni_dev); 2942 struct dpaa2_ni_tx_ring *tx = &ch->txc_queue.tx_rings[0]; 2943 struct mbuf *m = NULL; 2944 int budget = DPAA2_TX_BUDGET; 2945 2946 do { 2947 mtx_assert(&ch->xmit_mtx, MA_NOTOWNED); 2948 mtx_lock(&ch->xmit_mtx); 2949 m = buf_ring_dequeue_sc(ch->xmit_br); 2950 mtx_unlock(&ch->xmit_mtx); 2951 2952 if (__predict_false(m == NULL)) { 2953 /* TODO: Do not give up easily */ 2954 break; 2955 } else { 2956 dpaa2_ni_tx(sc, ch, tx, m); 2957 } 2958 } while (--budget); 2959 2960 return (DPAA2_TX_BUDGET - budget); 2961 } 2962 2963 static void 2964 dpaa2_ni_tx(struct dpaa2_ni_softc *sc, struct dpaa2_channel *ch, 2965 struct dpaa2_ni_tx_ring *tx, struct mbuf *m) 2966 { 2967 device_t dev = sc->dev; 2968 struct dpaa2_ni_fq *fq = tx->fq; 2969 struct dpaa2_buf *buf, *sgt; 2970 struct dpaa2_fd fd; 2971 struct mbuf *md; 2972 bus_dma_segment_t segs[DPAA2_TX_SEGLIMIT]; 2973 int rc, nsegs; 2974 int error; 2975 int len; 2976 bool mcast; 2977 2978 mtx_assert(&tx->lock, MA_NOTOWNED); 2979 mtx_lock(&tx->lock); 2980 buf = buf_ring_dequeue_sc(tx->br); 2981 mtx_unlock(&tx->lock); 2982 if (__predict_false(buf == NULL)) { 2983 /* TODO: Do not give up easily */ 2984 m_freem(m); 2985 return; 2986 } else { 2987 DPAA2_BUF_ASSERT_TXREADY(buf); 2988 buf->m = m; 2989 sgt = buf->sgt; 2990 } 2991 len = m->m_pkthdr.len; 2992 mcast = (m->m_flags & M_MCAST) != 0; 2993 2994 #if defined(INVARIANTS) 2995 struct dpaa2_ni_tx_ring *btx = (struct dpaa2_ni_tx_ring *)buf->opt; 2996 KASSERT(buf->opt == tx, ("%s: unexpected Tx ring", __func__)); 2997 KASSERT(btx->fq->chan == ch, ("%s: unexpected channel", __func__)); 2998 #endif /* INVARIANTS */ 2999 3000 BPF_MTAP(sc->ifp, m); 3001 3002 error = bus_dmamap_load_mbuf_sg(buf->dmat, buf->dmap, m, segs, &nsegs, 3003 BUS_DMA_NOWAIT); 3004 if (__predict_false(error != 0)) { 3005 /* Too many fragments, trying to defragment... */ 3006 md = m_collapse(m, M_NOWAIT, DPAA2_TX_SEGLIMIT); 3007 if (md == NULL) { 3008 device_printf(dev, "%s: m_collapse() failed\n", __func__); 3009 fq->chan->tx_dropped++; 3010 if_inc_counter(sc->ifp, IFCOUNTER_OERRORS, 1); 3011 goto err; 3012 } 3013 3014 buf->m = m = md; 3015 error = bus_dmamap_load_mbuf_sg(buf->dmat, buf->dmap, m, segs, 3016 &nsegs, BUS_DMA_NOWAIT); 3017 if (__predict_false(error != 0)) { 3018 device_printf(dev, "%s: bus_dmamap_load_mbuf_sg() " 3019 "failed: error=%d\n", __func__, error); 3020 fq->chan->tx_dropped++; 3021 if_inc_counter(sc->ifp, IFCOUNTER_OERRORS, 1); 3022 goto err; 3023 } 3024 } 3025 3026 error = dpaa2_fd_build(dev, sc->tx_data_off, buf, segs, nsegs, &fd); 3027 if (__predict_false(error != 0)) { 3028 device_printf(dev, "%s: failed to build frame descriptor: " 3029 "error=%d\n", __func__, error); 3030 fq->chan->tx_dropped++; 3031 if_inc_counter(sc->ifp, IFCOUNTER_OERRORS, 1); 3032 goto err_unload; 3033 } else 3034 sc->tx_sg_frames++; /* for sysctl(9) */ 3035 3036 bus_dmamap_sync(buf->dmat, buf->dmap, BUS_DMASYNC_PREWRITE); 3037 bus_dmamap_sync(sgt->dmat, sgt->dmap, BUS_DMASYNC_PREWRITE); 3038 3039 /* TODO: Enqueue several frames in a single command */ 3040 for (int i = 0; i < DPAA2_NI_ENQUEUE_RETRIES; i++) { 3041 /* TODO: Return error codes instead of # of frames */ 3042 rc = DPAA2_SWP_ENQ_MULTIPLE_FQ(fq->chan->io_dev, tx->fqid, &fd, 1); 3043 if (rc == 1) { 3044 break; 3045 } 3046 } 3047 3048 if (rc != 1) { 3049 fq->chan->tx_dropped++; 3050 if_inc_counter(sc->ifp, IFCOUNTER_OERRORS, 1); 3051 goto err_unload; 3052 } else { 3053 if (mcast) 3054 if_inc_counter(sc->ifp, IFCOUNTER_OMCASTS, 1); 3055 if_inc_counter(sc->ifp, IFCOUNTER_OPACKETS, 1); 3056 if_inc_counter(sc->ifp, IFCOUNTER_OBYTES, len); 3057 fq->chan->tx_frames++; 3058 } 3059 return; 3060 3061 err_unload: 3062 bus_dmamap_unload(buf->dmat, buf->dmap); 3063 if (sgt->paddr != 0) { 3064 bus_dmamap_unload(sgt->dmat, sgt->dmap); 3065 } 3066 err: 3067 m_freem(buf->m); 3068 buf_ring_enqueue(tx->br, buf); 3069 } 3070 3071 static int 3072 dpaa2_ni_consume_frames(struct dpaa2_channel *chan, struct dpaa2_ni_fq **src, 3073 uint32_t *consumed) 3074 { 3075 struct dpaa2_ni_fq *fq = NULL; 3076 struct dpaa2_dq *dq; 3077 struct dpaa2_fd *fd; 3078 struct dpaa2_ni_rx_ctx ctx = { 3079 .head = NULL, 3080 .tail = NULL, 3081 .cnt = 0, 3082 .last = false 3083 }; 3084 int rc, frames = 0; 3085 3086 do { 3087 rc = dpaa2_chan_next_frame(chan, &dq); 3088 if (rc == EINPROGRESS) { 3089 if (dq != NULL && !IS_NULL_RESPONSE(dq->fdr.desc.stat)) { 3090 fd = &dq->fdr.fd; 3091 fq = (struct dpaa2_ni_fq *) dq->fdr.desc.fqd_ctx; 3092 3093 switch (fq->type) { 3094 case DPAA2_NI_QUEUE_RX: 3095 (void)dpaa2_ni_rx(chan, fq, fd, &ctx); 3096 break; 3097 case DPAA2_NI_QUEUE_RX_ERR: 3098 (void)dpaa2_ni_rx_err(chan, fq, fd); 3099 break; 3100 case DPAA2_NI_QUEUE_TX_CONF: 3101 (void)dpaa2_ni_tx_conf(chan, fq, fd); 3102 break; 3103 default: 3104 panic("%s: unknown queue type (1)", 3105 __func__); 3106 } 3107 frames++; 3108 } 3109 } else if (rc == EALREADY || rc == ENOENT) { 3110 if (dq != NULL && !IS_NULL_RESPONSE(dq->fdr.desc.stat)) { 3111 fd = &dq->fdr.fd; 3112 fq = (struct dpaa2_ni_fq *) dq->fdr.desc.fqd_ctx; 3113 3114 switch (fq->type) { 3115 case DPAA2_NI_QUEUE_RX: 3116 /* 3117 * Last VDQ response (mbuf) in a chain 3118 * obtained from the Rx queue. 3119 */ 3120 ctx.last = true; 3121 (void)dpaa2_ni_rx(chan, fq, fd, &ctx); 3122 break; 3123 case DPAA2_NI_QUEUE_RX_ERR: 3124 (void)dpaa2_ni_rx_err(chan, fq, fd); 3125 break; 3126 case DPAA2_NI_QUEUE_TX_CONF: 3127 (void)dpaa2_ni_tx_conf(chan, fq, fd); 3128 break; 3129 default: 3130 panic("%s: unknown queue type (2)", 3131 __func__); 3132 } 3133 frames++; 3134 } 3135 break; 3136 } else { 3137 panic("%s: should not reach here: rc=%d", __func__, rc); 3138 } 3139 } while (true); 3140 3141 KASSERT(chan->store_idx < chan->store_sz, ("%s: store_idx(%d) >= " 3142 "store_sz(%d)", __func__, chan->store_idx, chan->store_sz)); 3143 3144 /* 3145 * VDQ operation pulls frames from a single queue into the store. 3146 * Return the frame queue and a number of consumed frames as an output. 3147 */ 3148 if (src != NULL) { 3149 *src = fq; 3150 } 3151 if (consumed != NULL) { 3152 *consumed = frames; 3153 } 3154 3155 return (rc); 3156 } 3157 3158 /** 3159 * @brief Receive frames. 3160 */ 3161 static int 3162 dpaa2_ni_rx(struct dpaa2_channel *ch, struct dpaa2_ni_fq *fq, 3163 struct dpaa2_fd *fd, struct dpaa2_ni_rx_ctx *ctx) 3164 { 3165 bus_addr_t paddr; 3166 struct dpaa2_swa *swa; 3167 struct dpaa2_buf *buf; 3168 struct dpaa2_channel *bch; 3169 struct dpaa2_ni_softc *sc; 3170 struct dpaa2_bp_softc *bpsc; 3171 struct mbuf *m; 3172 device_t bpdev; 3173 bus_addr_t released[DPAA2_SWP_BUFS_PER_CMD]; 3174 void *buf_data; 3175 int buf_len, error, released_n = 0; 3176 bool update_csum_flags; 3177 3178 error = dpaa2_fa_get_swa(fd, &swa); 3179 if (__predict_false(error != 0)) 3180 panic("%s: frame has no software annotation: error=%d", 3181 __func__, error); 3182 3183 paddr = (bus_addr_t)fd->addr; 3184 buf = swa->buf; 3185 bch = (struct dpaa2_channel *)buf->opt; 3186 sc = device_get_softc(bch->ni_dev); 3187 update_csum_flags = true; 3188 3189 KASSERT(swa->magic == DPAA2_MAGIC, ("%s: wrong magic", __func__)); 3190 /* 3191 * NOTE: Current channel might not be the same as the "buffer" channel 3192 * and it's fine. It must not be NULL though. 3193 */ 3194 KASSERT(bch != NULL, ("%s: buffer channel is NULL", __func__)); 3195 3196 if (__predict_false(paddr != buf->paddr)) { 3197 panic("%s: unexpected physical address: fd(%#jx) != buf(%#jx)", 3198 __func__, paddr, buf->paddr); 3199 } 3200 3201 switch (dpaa2_fd_err(fd)) { 3202 case 0: 3203 /* 3204 * FD[ERR] = 0 value is reserved to indicate that there is no 3205 * error encoded in this field. See 3.4.5 Error handling, 3206 * LX2160A DPAA2 Low-Level Hardware Reference Manual, Rev. 0, 3207 * 06/2020. 3208 */ 3209 break; 3210 case 1: /* Enqueue rejected by QMan */ 3211 sc->rx_enq_rej_frames++; 3212 break; 3213 case 2: /* QMan IEOI error */ 3214 sc->rx_ieoi_err_frames++; 3215 break; 3216 default: 3217 sc->rx_other_err_frames++; 3218 break; 3219 } 3220 3221 switch (dpaa2_fd_format(fd)) { 3222 case DPAA2_FD_SINGLE: 3223 sc->rx_single_buf_frames++; 3224 break; 3225 case DPAA2_FD_SG: 3226 sc->rx_sg_buf_frames++; 3227 break; 3228 default: 3229 update_csum_flags = false; 3230 break; 3231 } 3232 3233 mtx_assert(&bch->dma_mtx, MA_NOTOWNED); 3234 mtx_lock(&bch->dma_mtx); 3235 3236 bus_dmamap_sync(buf->dmat, buf->dmap, BUS_DMASYNC_POSTREAD); 3237 bus_dmamap_unload(buf->dmat, buf->dmap); 3238 3239 m = buf->m; 3240 buf_len = dpaa2_fd_data_len(fd); 3241 buf_data = (uint8_t *)buf->vaddr + dpaa2_fd_offset(fd); 3242 3243 /* Prepare buffer to be re-cycled */ 3244 buf->m = NULL; 3245 buf->paddr = 0; 3246 buf->vaddr = NULL; 3247 buf->seg.ds_addr = 0; 3248 buf->seg.ds_len = 0; 3249 buf->nseg = 0; 3250 3251 mtx_unlock(&bch->dma_mtx); 3252 3253 m->m_flags |= M_PKTHDR; 3254 m->m_data = buf_data; 3255 m->m_len = buf_len; 3256 m->m_pkthdr.len = buf_len; 3257 m->m_pkthdr.rcvif = sc->ifp; 3258 m->m_pkthdr.flowid = fq->fqid; 3259 M_HASHTYPE_SET(m, M_HASHTYPE_OPAQUE); 3260 if_inc_counter(sc->ifp, IFCOUNTER_IPACKETS, 1); 3261 3262 if (update_csum_flags && ((if_getcapenable(sc->ifp) & (IFCAP_RXCSUM | 3263 IFCAP_RXCSUM_IPV6)) != 0)) { 3264 error = dpaa2_ni_update_csum_flags(fd, m); 3265 if (error != 0) 3266 device_printf(sc->dev, "%s: failed to update checksum " 3267 "flags: error=%d\n", __func__, error); 3268 } 3269 3270 if (ctx->head == NULL) { 3271 KASSERT(ctx->tail == NULL, ("%s: tail already given?", __func__)); 3272 ctx->head = m; 3273 ctx->tail = m; 3274 } else { 3275 KASSERT(ctx->head != NULL, ("%s: head is NULL", __func__)); 3276 ctx->tail->m_nextpkt = m; 3277 ctx->tail = m; 3278 } 3279 ctx->cnt++; 3280 3281 if (ctx->last) { 3282 ctx->tail->m_nextpkt = NULL; 3283 if_input(sc->ifp, ctx->head); 3284 } 3285 3286 /* Keep the buffer to be recycled */ 3287 ch->recycled[ch->recycled_n++] = buf; 3288 3289 /* Re-seed and release recycled buffers back to the pool */ 3290 if (ch->recycled_n == DPAA2_SWP_BUFS_PER_CMD) { 3291 /* Release new buffers to the pool if needed */ 3292 taskqueue_enqueue(sc->bp_taskq, &ch->bp_task); 3293 3294 for (int i = 0; i < ch->recycled_n; i++) { 3295 buf = ch->recycled[i]; 3296 bch = (struct dpaa2_channel *)buf->opt; 3297 3298 mtx_assert(&bch->dma_mtx, MA_NOTOWNED); 3299 mtx_lock(&bch->dma_mtx); 3300 error = dpaa2_buf_seed_rxb(sc->dev, buf, 3301 DPAA2_RX_BUF_SIZE, &bch->dma_mtx); 3302 mtx_unlock(&bch->dma_mtx); 3303 3304 if (__predict_false(error != 0)) { 3305 /* TODO: What else to do with the buffer? */ 3306 panic("%s: failed to recycle buffer: error=%d", 3307 __func__, error); 3308 } 3309 3310 /* Prepare buffer to be released in a single command */ 3311 released[released_n++] = buf->paddr; 3312 } 3313 3314 /* There's only one buffer pool for now */ 3315 bpdev = (device_t)rman_get_start(sc->res[DPAA2_NI_BP_RID(0)]); 3316 bpsc = device_get_softc(bpdev); 3317 3318 error = DPAA2_SWP_RELEASE_BUFS(ch->io_dev, bpsc->attr.bpid, 3319 released, released_n); 3320 if (__predict_false(error != 0)) { 3321 device_printf(sc->dev, "%s: failed to release buffers " 3322 "to the pool: error=%d\n", __func__, error); 3323 return (error); 3324 } 3325 ch->recycled_n = 0; 3326 } 3327 3328 return (0); 3329 } 3330 3331 /** 3332 * @brief Receive Rx error frames. 3333 */ 3334 static int 3335 dpaa2_ni_rx_err(struct dpaa2_channel *ch, struct dpaa2_ni_fq *fq, 3336 struct dpaa2_fd *fd) 3337 { 3338 bus_addr_t paddr; 3339 struct dpaa2_swa *swa; 3340 struct dpaa2_buf *buf; 3341 struct dpaa2_channel *bch; 3342 struct dpaa2_ni_softc *sc; 3343 device_t bpdev; 3344 struct dpaa2_bp_softc *bpsc; 3345 int error; 3346 3347 error = dpaa2_fa_get_swa(fd, &swa); 3348 if (__predict_false(error != 0)) 3349 panic("%s: frame has no software annotation: error=%d", 3350 __func__, error); 3351 3352 paddr = (bus_addr_t)fd->addr; 3353 buf = swa->buf; 3354 bch = (struct dpaa2_channel *)buf->opt; 3355 sc = device_get_softc(bch->ni_dev); 3356 3357 KASSERT(swa->magic == DPAA2_MAGIC, ("%s: wrong magic", __func__)); 3358 /* 3359 * NOTE: Current channel might not be the same as the "buffer" channel 3360 * and it's fine. It must not be NULL though. 3361 */ 3362 KASSERT(bch != NULL, ("%s: buffer channel is NULL", __func__)); 3363 3364 if (__predict_false(paddr != buf->paddr)) { 3365 panic("%s: unexpected physical address: fd(%#jx) != buf(%#jx)", 3366 __func__, paddr, buf->paddr); 3367 } 3368 3369 /* There's only one buffer pool for now */ 3370 bpdev = (device_t)rman_get_start(sc->res[DPAA2_NI_BP_RID(0)]); 3371 bpsc = device_get_softc(bpdev); 3372 3373 /* Release buffer to QBMan buffer pool */ 3374 error = DPAA2_SWP_RELEASE_BUFS(ch->io_dev, bpsc->attr.bpid, &paddr, 1); 3375 if (error != 0) { 3376 device_printf(sc->dev, "%s: failed to release frame buffer to " 3377 "the pool: error=%d\n", __func__, error); 3378 return (error); 3379 } 3380 3381 return (0); 3382 } 3383 3384 /** 3385 * @brief Receive Tx confirmation frames. 3386 */ 3387 static int 3388 dpaa2_ni_tx_conf(struct dpaa2_channel *ch, struct dpaa2_ni_fq *fq, 3389 struct dpaa2_fd *fd) 3390 { 3391 bus_addr_t paddr; 3392 struct dpaa2_swa *swa; 3393 struct dpaa2_buf *buf; 3394 struct dpaa2_buf *sgt; 3395 struct dpaa2_ni_tx_ring *tx; 3396 struct dpaa2_channel *bch; 3397 int error; 3398 3399 error = dpaa2_fa_get_swa(fd, &swa); 3400 if (__predict_false(error != 0)) 3401 panic("%s: frame has no software annotation: error=%d", 3402 __func__, error); 3403 3404 paddr = (bus_addr_t)fd->addr; 3405 buf = swa->buf; 3406 sgt = buf->sgt; 3407 tx = (struct dpaa2_ni_tx_ring *)buf->opt; 3408 bch = tx->fq->chan; 3409 3410 KASSERT(swa->magic == DPAA2_MAGIC, ("%s: wrong magic", __func__)); 3411 KASSERT(tx != NULL, ("%s: Tx ring is NULL", __func__)); 3412 KASSERT(sgt != NULL, ("%s: S/G table is NULL", __func__)); 3413 /* 3414 * NOTE: Current channel might not be the same as the "buffer" channel 3415 * and it's fine. It must not be NULL though. 3416 */ 3417 KASSERT(bch != NULL, ("%s: buffer channel is NULL", __func__)); 3418 3419 if (paddr != buf->paddr) { 3420 panic("%s: unexpected physical address: fd(%#jx) != buf(%#jx)", 3421 __func__, paddr, buf->paddr); 3422 } 3423 3424 mtx_assert(&bch->dma_mtx, MA_NOTOWNED); 3425 mtx_lock(&bch->dma_mtx); 3426 3427 bus_dmamap_sync(buf->dmat, buf->dmap, BUS_DMASYNC_POSTWRITE); 3428 bus_dmamap_sync(sgt->dmat, sgt->dmap, BUS_DMASYNC_POSTWRITE); 3429 bus_dmamap_unload(buf->dmat, buf->dmap); 3430 bus_dmamap_unload(sgt->dmat, sgt->dmap); 3431 m_freem(buf->m); 3432 buf->m = NULL; 3433 buf->paddr = 0; 3434 buf->vaddr = NULL; 3435 sgt->paddr = 0; 3436 3437 mtx_unlock(&bch->dma_mtx); 3438 3439 /* Return Tx buffer back to the ring */ 3440 buf_ring_enqueue(tx->br, buf); 3441 3442 return (0); 3443 } 3444 3445 /** 3446 * @brief Compare versions of the DPAA2 network interface API. 3447 */ 3448 static int 3449 dpaa2_ni_cmp_api_version(struct dpaa2_ni_softc *sc, uint16_t major, 3450 uint16_t minor) 3451 { 3452 if (sc->api_major == major) { 3453 return sc->api_minor - minor; 3454 } 3455 return sc->api_major - major; 3456 } 3457 3458 /** 3459 * @brief Collect statistics of the network interface. 3460 */ 3461 static int 3462 dpaa2_ni_collect_stats(SYSCTL_HANDLER_ARGS) 3463 { 3464 struct dpaa2_ni_softc *sc = (struct dpaa2_ni_softc *) arg1; 3465 struct dpni_stat *stat = &dpni_stat_sysctls[oidp->oid_number]; 3466 device_t pdev = device_get_parent(sc->dev); 3467 device_t dev = sc->dev; 3468 device_t child = dev; 3469 struct dpaa2_devinfo *rcinfo = device_get_ivars(pdev); 3470 struct dpaa2_devinfo *dinfo = device_get_ivars(dev); 3471 struct dpaa2_cmd cmd; 3472 uint64_t cnt[DPAA2_NI_STAT_COUNTERS_PER_PAGE]; 3473 uint64_t result = 0; 3474 uint16_t rc_token, ni_token; 3475 int error; 3476 3477 DPAA2_CMD_INIT(&cmd); 3478 3479 error = DPAA2_CMD_RC_OPEN(dev, child, &cmd, rcinfo->id, &rc_token); 3480 if (error) { 3481 device_printf(dev, "%s: failed to open resource container: " 3482 "id=%d, error=%d\n", __func__, rcinfo->id, error); 3483 goto exit; 3484 } 3485 error = DPAA2_CMD_NI_OPEN(dev, child, &cmd, dinfo->id, &ni_token); 3486 if (error) { 3487 device_printf(dev, "%s: failed to open network interface: " 3488 "id=%d, error=%d\n", __func__, dinfo->id, error); 3489 goto close_rc; 3490 } 3491 3492 error = DPAA2_CMD_NI_GET_STATISTICS(dev, child, &cmd, stat->page, 0, cnt); 3493 if (!error) { 3494 result = cnt[stat->cnt]; 3495 } 3496 3497 (void)DPAA2_CMD_NI_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, ni_token)); 3498 close_rc: 3499 (void)DPAA2_CMD_RC_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, rc_token)); 3500 exit: 3501 return (sysctl_handle_64(oidp, &result, 0, req)); 3502 } 3503 3504 static int 3505 dpaa2_ni_sysctl_link_state(SYSCTL_HANDLER_ARGS) 3506 { 3507 struct dpaa2_ni_softc *sc; 3508 struct dpaa2_devinfo *rcinfo; 3509 struct dpaa2_devinfo *dinfo; 3510 struct dpaa2_cmd cmd; 3511 struct dpaa2_ni_link_state ls; 3512 struct sbuf s; 3513 int error; 3514 uint16_t rc_token, ni_token; 3515 3516 if (req->newptr) 3517 return (EPERM); 3518 3519 sc = (struct dpaa2_ni_softc *)arg1; 3520 3521 rcinfo = device_get_ivars(device_get_parent(sc->dev)); 3522 dinfo = device_get_ivars(sc->dev); 3523 3524 DPAA2_CMD_INIT(&cmd); 3525 3526 error = DPAA2_CMD_RC_OPEN(sc->dev, sc->dev, &cmd, rcinfo->id, &rc_token); 3527 if (error != 0) { 3528 device_printf(sc->dev, "%s: failed to open resource container: " 3529 "id=%d, error=%d\n", __func__, rcinfo->id, error); 3530 goto exit; 3531 } 3532 error = DPAA2_CMD_NI_OPEN(sc->dev, sc->dev, &cmd, dinfo->id, &ni_token); 3533 if (error != 0) { 3534 device_printf(sc->dev, "%s: failed to open network interface: " 3535 "id=%d, error=%d\n", __func__, dinfo->id, error); 3536 goto close_rc; 3537 } 3538 3539 error = DPAA2_CMD_NI_GET_LINK_STATE(sc->dev, sc->dev, &cmd, &ls); 3540 3541 (void)DPAA2_CMD_NI_CLOSE(sc->dev, sc->dev, DPAA2_CMD_TK(&cmd, ni_token)); 3542 close_rc: 3543 (void)DPAA2_CMD_RC_CLOSE(sc->dev, sc->dev, DPAA2_CMD_TK(&cmd, rc_token)); 3544 3545 if (error != 0) 3546 goto exit; 3547 3548 #define NI_LINK_STATE_OPTIONS_BITS \ 3549 "\20\1AUTONEG\2HDX\3PAUSE\4ASYM_PAUSE" 3550 3551 sbuf_new_for_sysctl(&s, NULL, 1024, req); 3552 sbuf_putc(&s, '\n'); 3553 sbuf_printf(&s, "Link State: %s (%s)\n", ls.link_up ? "UP" : "DOWN", 3554 ls.state_valid ? "VALID" : "IGNORE"); 3555 sbuf_printf(&s, "Link Rate: %ju\n", (uintmax_t)ls.rate); 3556 sbuf_printf(&s, "Link Options: %b\n", 3557 (int)ls.options, NI_LINK_STATE_OPTIONS_BITS); 3558 sbuf_printf(&s, "Link Speed Capabilities: %#018jx\n", 3559 (uintmax_t)ls.sup_speeds); 3560 sbuf_printf(&s, "Link Speed Advertised for autoneg: %#018jx\n", 3561 (uintmax_t)ls.adv_speeds); 3562 sbuf_finish(&s); 3563 sbuf_delete(&s); 3564 3565 exit: 3566 return (error); 3567 } 3568 3569 static int 3570 dpaa2_ni_collect_buf_num(SYSCTL_HANDLER_ARGS) 3571 { 3572 struct dpaa2_ni_softc *sc = (struct dpaa2_ni_softc *) arg1; 3573 uint32_t buf_num = DPAA2_ATOMIC_READ(&sc->buf_num); 3574 3575 return (sysctl_handle_32(oidp, &buf_num, 0, req)); 3576 } 3577 3578 static int 3579 dpaa2_ni_collect_buf_free(SYSCTL_HANDLER_ARGS) 3580 { 3581 struct dpaa2_ni_softc *sc = (struct dpaa2_ni_softc *) arg1; 3582 uint32_t buf_free = DPAA2_ATOMIC_READ(&sc->buf_free); 3583 3584 return (sysctl_handle_32(oidp, &buf_free, 0, req)); 3585 } 3586 3587 static int 3588 dpaa2_ni_set_hash(device_t dev, uint64_t flags) 3589 { 3590 struct dpaa2_ni_softc *sc = device_get_softc(dev); 3591 uint64_t key = 0; 3592 int i; 3593 3594 if (!(sc->attr.num.queues > 1)) { 3595 return (EOPNOTSUPP); 3596 } 3597 3598 for (i = 0; i < ARRAY_SIZE(dist_fields); i++) { 3599 if (dist_fields[i].rxnfc_field & flags) { 3600 key |= dist_fields[i].id; 3601 } 3602 } 3603 3604 return (dpaa2_ni_set_dist_key(dev, DPAA2_NI_DIST_MODE_HASH, key)); 3605 } 3606 3607 /** 3608 * @brief Set Rx distribution (hash or flow classification) key flags is a 3609 * combination of RXH_ bits. 3610 */ 3611 static int 3612 dpaa2_ni_set_dist_key(device_t dev, enum dpaa2_ni_dist_mode type, uint64_t flags) 3613 { 3614 device_t pdev = device_get_parent(dev); 3615 device_t child = dev; 3616 struct dpaa2_ni_softc *sc = device_get_softc(dev); 3617 struct dpaa2_devinfo *rcinfo = device_get_ivars(pdev); 3618 struct dpaa2_devinfo *dinfo = device_get_ivars(dev); 3619 struct dpkg_profile_cfg cls_cfg; 3620 struct dpkg_extract *key; 3621 struct dpaa2_buf *buf = &sc->rxd_kcfg; 3622 struct dpaa2_cmd cmd; 3623 uint16_t rc_token, ni_token; 3624 int i, error = 0; 3625 3626 if (__predict_true(buf->dmat == NULL)) { 3627 buf->dmat = sc->rxd_dmat; 3628 } 3629 3630 memset(&cls_cfg, 0, sizeof(cls_cfg)); 3631 3632 /* Configure extracts according to the given flags. */ 3633 for (i = 0; i < ARRAY_SIZE(dist_fields); i++) { 3634 key = &cls_cfg.extracts[cls_cfg.num_extracts]; 3635 3636 if (!(flags & dist_fields[i].id)) { 3637 continue; 3638 } 3639 3640 if (cls_cfg.num_extracts >= DPKG_MAX_NUM_OF_EXTRACTS) { 3641 device_printf(dev, "%s: failed to add key extraction " 3642 "rule\n", __func__); 3643 return (E2BIG); 3644 } 3645 3646 key->type = DPKG_EXTRACT_FROM_HDR; 3647 key->extract.from_hdr.prot = dist_fields[i].cls_prot; 3648 key->extract.from_hdr.type = DPKG_FULL_FIELD; 3649 key->extract.from_hdr.field = dist_fields[i].cls_field; 3650 cls_cfg.num_extracts++; 3651 } 3652 3653 error = bus_dmamem_alloc(buf->dmat, (void **)&buf->vaddr, 3654 BUS_DMA_ZERO | BUS_DMA_COHERENT, &buf->dmap); 3655 if (error != 0) { 3656 device_printf(dev, "%s: failed to allocate a buffer for Rx " 3657 "traffic distribution key configuration\n", __func__); 3658 return (error); 3659 } 3660 3661 error = dpaa2_ni_prepare_key_cfg(&cls_cfg, (uint8_t *)buf->vaddr); 3662 if (error != 0) { 3663 device_printf(dev, "%s: failed to prepare key configuration: " 3664 "error=%d\n", __func__, error); 3665 return (error); 3666 } 3667 3668 /* Prepare for setting the Rx dist. */ 3669 error = bus_dmamap_load(buf->dmat, buf->dmap, buf->vaddr, 3670 DPAA2_CLASSIFIER_DMA_SIZE, dpaa2_dmamap_oneseg_cb, &buf->paddr, 3671 BUS_DMA_NOWAIT); 3672 if (error != 0) { 3673 device_printf(sc->dev, "%s: failed to map a buffer for Rx " 3674 "traffic distribution key configuration\n", __func__); 3675 return (error); 3676 } 3677 3678 if (type == DPAA2_NI_DIST_MODE_HASH) { 3679 DPAA2_CMD_INIT(&cmd); 3680 3681 error = DPAA2_CMD_RC_OPEN(dev, child, &cmd, rcinfo->id, 3682 &rc_token); 3683 if (error) { 3684 device_printf(dev, "%s: failed to open resource " 3685 "container: id=%d, error=%d\n", __func__, rcinfo->id, 3686 error); 3687 goto err_exit; 3688 } 3689 error = DPAA2_CMD_NI_OPEN(dev, child, &cmd, dinfo->id, 3690 &ni_token); 3691 if (error) { 3692 device_printf(dev, "%s: failed to open network " 3693 "interface: id=%d, error=%d\n", __func__, dinfo->id, 3694 error); 3695 goto close_rc; 3696 } 3697 3698 error = DPAA2_CMD_NI_SET_RX_TC_DIST(dev, child, &cmd, 3699 sc->attr.num.queues, 0, DPAA2_NI_DIST_MODE_HASH, buf->paddr); 3700 if (error != 0) { 3701 device_printf(dev, "%s: failed to set distribution mode " 3702 "and size for the traffic class\n", __func__); 3703 } 3704 3705 (void)DPAA2_CMD_NI_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, 3706 ni_token)); 3707 close_rc: 3708 (void)DPAA2_CMD_RC_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, 3709 rc_token)); 3710 } 3711 3712 err_exit: 3713 return (error); 3714 } 3715 3716 /** 3717 * @brief Prepares extract parameters. 3718 * 3719 * cfg: Defining a full Key Generation profile. 3720 * key_cfg_buf: Zeroed 256 bytes of memory before mapping it to DMA. 3721 */ 3722 static int 3723 dpaa2_ni_prepare_key_cfg(struct dpkg_profile_cfg *cfg, uint8_t *key_cfg_buf) 3724 { 3725 struct dpni_ext_set_rx_tc_dist *dpni_ext; 3726 struct dpni_dist_extract *extr; 3727 int i, j; 3728 3729 if (cfg->num_extracts > DPKG_MAX_NUM_OF_EXTRACTS) 3730 return (EINVAL); 3731 3732 dpni_ext = (struct dpni_ext_set_rx_tc_dist *) key_cfg_buf; 3733 dpni_ext->num_extracts = cfg->num_extracts; 3734 3735 for (i = 0; i < cfg->num_extracts; i++) { 3736 extr = &dpni_ext->extracts[i]; 3737 3738 switch (cfg->extracts[i].type) { 3739 case DPKG_EXTRACT_FROM_HDR: 3740 extr->prot = cfg->extracts[i].extract.from_hdr.prot; 3741 extr->efh_type = 3742 cfg->extracts[i].extract.from_hdr.type & 0x0Fu; 3743 extr->size = cfg->extracts[i].extract.from_hdr.size; 3744 extr->offset = cfg->extracts[i].extract.from_hdr.offset; 3745 extr->field = cfg->extracts[i].extract.from_hdr.field; 3746 extr->hdr_index = 3747 cfg->extracts[i].extract.from_hdr.hdr_index; 3748 break; 3749 case DPKG_EXTRACT_FROM_DATA: 3750 extr->size = cfg->extracts[i].extract.from_data.size; 3751 extr->offset = 3752 cfg->extracts[i].extract.from_data.offset; 3753 break; 3754 case DPKG_EXTRACT_FROM_PARSE: 3755 extr->size = cfg->extracts[i].extract.from_parse.size; 3756 extr->offset = 3757 cfg->extracts[i].extract.from_parse.offset; 3758 break; 3759 default: 3760 return (EINVAL); 3761 } 3762 3763 extr->num_of_byte_masks = cfg->extracts[i].num_of_byte_masks; 3764 extr->extract_type = cfg->extracts[i].type & 0x0Fu; 3765 3766 for (j = 0; j < DPKG_NUM_OF_MASKS; j++) { 3767 extr->masks[j].mask = cfg->extracts[i].masks[j].mask; 3768 extr->masks[j].offset = 3769 cfg->extracts[i].masks[j].offset; 3770 } 3771 } 3772 3773 return (0); 3774 } 3775 3776 static int 3777 dpaa2_ni_update_csum_flags(struct dpaa2_fd *fd, struct mbuf *m) 3778 { 3779 struct dpaa2_hwa_fas fas; 3780 uint32_t status; 3781 int rc; 3782 3783 if (__predict_false((dpaa2_fd_get_frc(fd) & DPAA2_FD_FRC_FASV)) == 0u) 3784 return (EINVAL); 3785 3786 /* 3787 * XXX-DSL: Frame context of the frame descriptor (FD[FRC]) contains 3788 * an Accelerator ID in the MSbits on some SoCs (e.g. LS1088A), 3789 * but a frame ParseSummary on the others (e.g. LX2160A). 3790 * However, frame annotation valid bits seem to be at the 3791 * same offsets. This is the reason why different accelerators 3792 * are treated the same here. It isn't clear whether this is 3793 * a hardware limitation of the SoCs, version of the firmware 3794 * or DPL configuration. 3795 */ 3796 3797 rc = dpaa2_fa_get_fas(fd, &fas); 3798 if (rc != 0) 3799 return (rc); 3800 3801 status = le32toh(fas.status); 3802 rc = 0; 3803 3804 /* L3 */ 3805 if ((status & DPAA2_FAS_L3CV) != 0) { 3806 m->m_pkthdr.csum_flags |= CSUM_L3_CALC; 3807 if ((status & DPAA2_FAS_L3CE) == 0) 3808 m->m_pkthdr.csum_flags |= CSUM_L3_VALID; 3809 } 3810 /* L4 */ 3811 if ((status & DPAA2_FAS_L4CV) != 0) { 3812 m->m_pkthdr.csum_flags |= CSUM_L4_CALC; 3813 m->m_pkthdr.csum_data = 0xffff; 3814 if ((status & DPAA2_FAS_L4CE) == 0) 3815 m->m_pkthdr.csum_flags |= CSUM_L4_VALID; 3816 } 3817 3818 return (rc); 3819 } 3820 3821 static device_method_t dpaa2_ni_methods[] = { 3822 /* Device interface */ 3823 DEVMETHOD(device_probe, dpaa2_ni_probe), 3824 DEVMETHOD(device_attach, dpaa2_ni_attach), 3825 DEVMETHOD(device_detach, dpaa2_ni_detach), 3826 3827 /* mii via memac_mdio */ 3828 DEVMETHOD(miibus_statchg, dpaa2_ni_miibus_statchg), 3829 3830 DEVMETHOD_END 3831 }; 3832 3833 static driver_t dpaa2_ni_driver = { 3834 "dpaa2_ni", 3835 dpaa2_ni_methods, 3836 sizeof(struct dpaa2_ni_softc), 3837 }; 3838 3839 DRIVER_MODULE(miibus, dpaa2_ni, miibus_driver, 0, 0); 3840 DRIVER_MODULE(dpaa2_ni, dpaa2_rc, dpaa2_ni_driver, 0, 0); 3841 3842 MODULE_DEPEND(dpaa2_ni, miibus, 1, 1, 1); 3843 #ifdef DEV_ACPI 3844 MODULE_DEPEND(dpaa2_ni, memac_mdio_acpi, 1, 1, 1); 3845 #endif 3846 #ifdef FDT 3847 MODULE_DEPEND(dpaa2_ni, memac_mdio_fdt, 1, 1, 1); 3848 #endif 3849