1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause 3 * 4 * Copyright © 2021-2023 Dmitry Salychev 5 * Copyright © 2022 Mathew McBride 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26 * SUCH DAMAGE. 27 */ 28 29 #include <sys/cdefs.h> 30 /* 31 * The DPAA2 Network Interface (DPNI) driver. 32 * 33 * The DPNI object is a network interface that is configurable to support a wide 34 * range of features from a very basic Ethernet interface up to a 35 * high-functioning network interface. The DPNI supports features that are 36 * expected by standard network stacks, from basic features to offloads. 37 * 38 * DPNIs work with Ethernet traffic, starting with the L2 header. Additional 39 * functions are provided for standard network protocols (L2, L3, L4, etc.). 40 */ 41 42 #include <sys/param.h> 43 #include <sys/systm.h> 44 #include <sys/kernel.h> 45 #include <sys/bus.h> 46 #include <sys/rman.h> 47 #include <sys/module.h> 48 #include <sys/malloc.h> 49 #include <sys/mutex.h> 50 #include <sys/socket.h> 51 #include <sys/sockio.h> 52 #include <sys/sysctl.h> 53 #include <sys/mbuf.h> 54 #include <sys/taskqueue.h> 55 #include <sys/sysctl.h> 56 #include <sys/buf_ring.h> 57 #include <sys/smp.h> 58 #include <sys/proc.h> 59 60 #include <vm/vm.h> 61 #include <vm/pmap.h> 62 63 #include <machine/bus.h> 64 #include <machine/resource.h> 65 #include <machine/atomic.h> 66 #include <machine/vmparam.h> 67 68 #include <net/ethernet.h> 69 #include <net/bpf.h> 70 #include <net/if.h> 71 #include <net/if_dl.h> 72 #include <net/if_media.h> 73 #include <net/if_types.h> 74 #include <net/if_var.h> 75 76 #include <dev/pci/pcivar.h> 77 #include <dev/mii/mii.h> 78 #include <dev/mii/miivar.h> 79 #include <dev/mdio/mdio.h> 80 81 #include "opt_acpi.h" 82 #include "opt_platform.h" 83 84 #include "pcib_if.h" 85 #include "pci_if.h" 86 #include "miibus_if.h" 87 #include "memac_mdio_if.h" 88 89 #include "dpaa2_types.h" 90 #include "dpaa2_mc.h" 91 #include "dpaa2_mc_if.h" 92 #include "dpaa2_mcp.h" 93 #include "dpaa2_swp.h" 94 #include "dpaa2_swp_if.h" 95 #include "dpaa2_cmd_if.h" 96 #include "dpaa2_ni.h" 97 #include "dpaa2_channel.h" 98 #include "dpaa2_buf.h" 99 #include "dpaa2_frame.h" 100 101 #define BIT(x) (1ul << (x)) 102 #define WRIOP_VERSION(x, y, z) ((x) << 10 | (y) << 5 | (z) << 0) 103 #define ARRAY_SIZE(a) (sizeof(a) / sizeof((a)[0])) 104 105 /* Frame Dequeue Response status bits. */ 106 #define IS_NULL_RESPONSE(stat) ((((stat) >> 4) & 1) == 0) 107 108 #define ALIGN_UP(x, y) roundup2((x), (y)) 109 #define ALIGN_DOWN(x, y) rounddown2((x), (y)) 110 #define CACHE_LINE_ALIGN(x) ALIGN_UP((x), CACHE_LINE_SIZE) 111 112 #define DPNI_LOCK(__sc) do { \ 113 mtx_assert(&(__sc)->lock, MA_NOTOWNED); \ 114 mtx_lock(&(__sc)->lock); \ 115 } while (0) 116 #define DPNI_UNLOCK(__sc) do { \ 117 mtx_assert(&(__sc)->lock, MA_OWNED); \ 118 mtx_unlock(&(__sc)->lock); \ 119 } while (0) 120 #define DPNI_LOCK_ASSERT(__sc) do { \ 121 mtx_assert(&(__sc)->lock, MA_OWNED); \ 122 } while (0) 123 124 #define DPAA2_TX_RING(sc, chan, tc) \ 125 (&(sc)->channels[(chan)]->txc_queue.tx_rings[(tc)]) 126 127 MALLOC_DEFINE(M_DPAA2_TXB, "dpaa2_txb", "DPAA2 DMA-mapped buffer (Tx)"); 128 129 /* 130 * How many times channel cleanup routine will be repeated if the RX or TX 131 * budget was depleted. 132 */ 133 #define DPAA2_CLEAN_BUDGET 64 /* sysctl(9)? */ 134 /* TX/RX budget for the channel cleanup task */ 135 #define DPAA2_TX_BUDGET 128 /* sysctl(9)? */ 136 #define DPAA2_RX_BUDGET 256 /* sysctl(9)? */ 137 138 #define DPNI_IRQ_INDEX 0 /* Index of the only DPNI IRQ. */ 139 #define DPNI_IRQ_LINK_CHANGED 1 /* Link state changed */ 140 #define DPNI_IRQ_EP_CHANGED 2 /* DPAA2 endpoint dis/connected */ 141 142 /* Default maximum RX frame length w/o CRC. */ 143 #define DPAA2_ETH_MFL (ETHER_MAX_LEN_JUMBO + ETHER_VLAN_ENCAP_LEN - \ 144 ETHER_CRC_LEN) 145 146 /* Minimally supported version of the DPNI API. */ 147 #define DPNI_VER_MAJOR 7 148 #define DPNI_VER_MINOR 0 149 150 /* Rx/Tx buffers configuration. */ 151 #define BUF_ALIGN_V1 256 /* WRIOP v1.0.0 limitation */ 152 #define BUF_ALIGN 64 153 #define BUF_SWA_SIZE 64 /* SW annotation size */ 154 #define BUF_RX_HWA_SIZE 64 /* HW annotation size */ 155 #define BUF_TX_HWA_SIZE 128 /* HW annotation size */ 156 157 #define DPAA2_RX_BUFRING_SZ (4096u) 158 #define DPAA2_RXE_BUFRING_SZ (1024u) 159 #define DPAA2_TXC_BUFRING_SZ (4096u) 160 161 /* Size of a buffer to keep a QoS table key configuration. */ 162 #define ETH_QOS_KCFG_BUF_SIZE (PAGE_SIZE) 163 164 /* Required by struct dpni_rx_tc_dist_cfg::key_cfg_iova */ 165 #define DPAA2_CLASSIFIER_DMA_SIZE (PAGE_SIZE) 166 167 /* Buffers layout options. */ 168 #define BUF_LOPT_TIMESTAMP 0x1 169 #define BUF_LOPT_PARSER_RESULT 0x2 170 #define BUF_LOPT_FRAME_STATUS 0x4 171 #define BUF_LOPT_PRIV_DATA_SZ 0x8 172 #define BUF_LOPT_DATA_ALIGN 0x10 173 #define BUF_LOPT_DATA_HEAD_ROOM 0x20 174 #define BUF_LOPT_DATA_TAIL_ROOM 0x40 175 176 #define DPAA2_NI_BUF_ADDR_MASK (0x1FFFFFFFFFFFFul) /* 49-bit addresses max. */ 177 #define DPAA2_NI_BUF_CHAN_MASK (0xFu) 178 #define DPAA2_NI_BUF_CHAN_SHIFT (60) 179 #define DPAA2_NI_BUF_IDX_MASK (0x7FFFu) 180 #define DPAA2_NI_BUF_IDX_SHIFT (49) 181 #define DPAA2_NI_TX_IDX_MASK (0x7u) 182 #define DPAA2_NI_TX_IDX_SHIFT (57) 183 #define DPAA2_NI_TXBUF_IDX_MASK (0xFFu) 184 #define DPAA2_NI_TXBUF_IDX_SHIFT (49) 185 186 /* Enables TCAM for Flow Steering and QoS look-ups. */ 187 #define DPNI_OPT_HAS_KEY_MASKING 0x10 188 189 /* Unique IDs for the supported Rx classification header fields. */ 190 #define DPAA2_ETH_DIST_ETHDST BIT(0) 191 #define DPAA2_ETH_DIST_ETHSRC BIT(1) 192 #define DPAA2_ETH_DIST_ETHTYPE BIT(2) 193 #define DPAA2_ETH_DIST_VLAN BIT(3) 194 #define DPAA2_ETH_DIST_IPSRC BIT(4) 195 #define DPAA2_ETH_DIST_IPDST BIT(5) 196 #define DPAA2_ETH_DIST_IPPROTO BIT(6) 197 #define DPAA2_ETH_DIST_L4SRC BIT(7) 198 #define DPAA2_ETH_DIST_L4DST BIT(8) 199 #define DPAA2_ETH_DIST_ALL (~0ULL) 200 201 /* L3-L4 network traffic flow hash options. */ 202 #define RXH_L2DA (1 << 1) 203 #define RXH_VLAN (1 << 2) 204 #define RXH_L3_PROTO (1 << 3) 205 #define RXH_IP_SRC (1 << 4) 206 #define RXH_IP_DST (1 << 5) 207 #define RXH_L4_B_0_1 (1 << 6) /* src port in case of TCP/UDP/SCTP */ 208 #define RXH_L4_B_2_3 (1 << 7) /* dst port in case of TCP/UDP/SCTP */ 209 #define RXH_DISCARD (1 << 31) 210 211 /* Transmit checksum offload */ 212 #define DPAA2_CSUM_TX_OFFLOAD (CSUM_IP | CSUM_DELAY_DATA | CSUM_DELAY_DATA_IPV6) 213 214 /* Default Rx hash options, set during attaching. */ 215 #define DPAA2_RXH_DEFAULT (RXH_IP_SRC | RXH_IP_DST | RXH_L4_B_0_1 | RXH_L4_B_2_3) 216 217 MALLOC_DEFINE(M_DPAA2_NI, "dpaa2_ni", "DPAA2 Network Interface"); 218 219 /* 220 * DPAA2 Network Interface resource specification. 221 * 222 * NOTE: Don't forget to update macros in dpaa2_ni.h in case of any changes in 223 * the specification! 224 */ 225 struct resource_spec dpaa2_ni_spec[] = { 226 /* 227 * DPMCP resources. 228 * 229 * NOTE: MC command portals (MCPs) are used to send commands to, and 230 * receive responses from, the MC firmware. One portal per DPNI. 231 */ 232 { DPAA2_DEV_MCP, DPAA2_NI_MCP_RID(0), RF_ACTIVE | RF_SHAREABLE | RF_OPTIONAL }, 233 /* 234 * DPIO resources (software portals). 235 * 236 * NOTE: One per running core. While DPIOs are the source of data 237 * availability interrupts, the DPCONs are used to identify the 238 * network interface that has produced ingress data to that core. 239 */ 240 { DPAA2_DEV_IO, DPAA2_NI_IO_RID(0), RF_ACTIVE | RF_SHAREABLE }, 241 { DPAA2_DEV_IO, DPAA2_NI_IO_RID(1), RF_ACTIVE | RF_SHAREABLE | RF_OPTIONAL }, 242 { DPAA2_DEV_IO, DPAA2_NI_IO_RID(2), RF_ACTIVE | RF_SHAREABLE | RF_OPTIONAL }, 243 { DPAA2_DEV_IO, DPAA2_NI_IO_RID(3), RF_ACTIVE | RF_SHAREABLE | RF_OPTIONAL }, 244 { DPAA2_DEV_IO, DPAA2_NI_IO_RID(4), RF_ACTIVE | RF_SHAREABLE | RF_OPTIONAL }, 245 { DPAA2_DEV_IO, DPAA2_NI_IO_RID(5), RF_ACTIVE | RF_SHAREABLE | RF_OPTIONAL }, 246 { DPAA2_DEV_IO, DPAA2_NI_IO_RID(6), RF_ACTIVE | RF_SHAREABLE | RF_OPTIONAL }, 247 { DPAA2_DEV_IO, DPAA2_NI_IO_RID(7), RF_ACTIVE | RF_SHAREABLE | RF_OPTIONAL }, 248 { DPAA2_DEV_IO, DPAA2_NI_IO_RID(8), RF_ACTIVE | RF_SHAREABLE | RF_OPTIONAL }, 249 { DPAA2_DEV_IO, DPAA2_NI_IO_RID(9), RF_ACTIVE | RF_SHAREABLE | RF_OPTIONAL }, 250 { DPAA2_DEV_IO, DPAA2_NI_IO_RID(10), RF_ACTIVE | RF_SHAREABLE | RF_OPTIONAL }, 251 { DPAA2_DEV_IO, DPAA2_NI_IO_RID(11), RF_ACTIVE | RF_SHAREABLE | RF_OPTIONAL }, 252 { DPAA2_DEV_IO, DPAA2_NI_IO_RID(12), RF_ACTIVE | RF_SHAREABLE | RF_OPTIONAL }, 253 { DPAA2_DEV_IO, DPAA2_NI_IO_RID(13), RF_ACTIVE | RF_SHAREABLE | RF_OPTIONAL }, 254 { DPAA2_DEV_IO, DPAA2_NI_IO_RID(14), RF_ACTIVE | RF_SHAREABLE | RF_OPTIONAL }, 255 { DPAA2_DEV_IO, DPAA2_NI_IO_RID(15), RF_ACTIVE | RF_SHAREABLE | RF_OPTIONAL }, 256 /* 257 * DPBP resources (buffer pools). 258 * 259 * NOTE: One per network interface. 260 */ 261 { DPAA2_DEV_BP, DPAA2_NI_BP_RID(0), RF_ACTIVE }, 262 /* 263 * DPCON resources (channels). 264 * 265 * NOTE: One DPCON per core where Rx or Tx confirmation traffic to be 266 * distributed to. 267 * NOTE: Since it is necessary to distinguish between traffic from 268 * different network interfaces arriving on the same core, the 269 * DPCONs must be private to the DPNIs. 270 */ 271 { DPAA2_DEV_CON, DPAA2_NI_CON_RID(0), RF_ACTIVE }, 272 { DPAA2_DEV_CON, DPAA2_NI_CON_RID(1), RF_ACTIVE | RF_OPTIONAL }, 273 { DPAA2_DEV_CON, DPAA2_NI_CON_RID(2), RF_ACTIVE | RF_OPTIONAL }, 274 { DPAA2_DEV_CON, DPAA2_NI_CON_RID(3), RF_ACTIVE | RF_OPTIONAL }, 275 { DPAA2_DEV_CON, DPAA2_NI_CON_RID(4), RF_ACTIVE | RF_OPTIONAL }, 276 { DPAA2_DEV_CON, DPAA2_NI_CON_RID(5), RF_ACTIVE | RF_OPTIONAL }, 277 { DPAA2_DEV_CON, DPAA2_NI_CON_RID(6), RF_ACTIVE | RF_OPTIONAL }, 278 { DPAA2_DEV_CON, DPAA2_NI_CON_RID(7), RF_ACTIVE | RF_OPTIONAL }, 279 { DPAA2_DEV_CON, DPAA2_NI_CON_RID(8), RF_ACTIVE | RF_OPTIONAL }, 280 { DPAA2_DEV_CON, DPAA2_NI_CON_RID(9), RF_ACTIVE | RF_OPTIONAL }, 281 { DPAA2_DEV_CON, DPAA2_NI_CON_RID(10), RF_ACTIVE | RF_OPTIONAL }, 282 { DPAA2_DEV_CON, DPAA2_NI_CON_RID(11), RF_ACTIVE | RF_OPTIONAL }, 283 { DPAA2_DEV_CON, DPAA2_NI_CON_RID(12), RF_ACTIVE | RF_OPTIONAL }, 284 { DPAA2_DEV_CON, DPAA2_NI_CON_RID(13), RF_ACTIVE | RF_OPTIONAL }, 285 { DPAA2_DEV_CON, DPAA2_NI_CON_RID(14), RF_ACTIVE | RF_OPTIONAL }, 286 { DPAA2_DEV_CON, DPAA2_NI_CON_RID(15), RF_ACTIVE | RF_OPTIONAL }, 287 288 RESOURCE_SPEC_END 289 }; 290 291 /* Supported header fields for Rx hash distribution key */ 292 static const struct dpaa2_eth_dist_fields dist_fields[] = { 293 { 294 /* L2 header */ 295 .rxnfc_field = RXH_L2DA, 296 .cls_prot = NET_PROT_ETH, 297 .cls_field = NH_FLD_ETH_DA, 298 .id = DPAA2_ETH_DIST_ETHDST, 299 .size = 6, 300 }, { 301 .cls_prot = NET_PROT_ETH, 302 .cls_field = NH_FLD_ETH_SA, 303 .id = DPAA2_ETH_DIST_ETHSRC, 304 .size = 6, 305 }, { 306 /* This is the last ethertype field parsed: 307 * depending on frame format, it can be the MAC ethertype 308 * or the VLAN etype. 309 */ 310 .cls_prot = NET_PROT_ETH, 311 .cls_field = NH_FLD_ETH_TYPE, 312 .id = DPAA2_ETH_DIST_ETHTYPE, 313 .size = 2, 314 }, { 315 /* VLAN header */ 316 .rxnfc_field = RXH_VLAN, 317 .cls_prot = NET_PROT_VLAN, 318 .cls_field = NH_FLD_VLAN_TCI, 319 .id = DPAA2_ETH_DIST_VLAN, 320 .size = 2, 321 }, { 322 /* IP header */ 323 .rxnfc_field = RXH_IP_SRC, 324 .cls_prot = NET_PROT_IP, 325 .cls_field = NH_FLD_IP_SRC, 326 .id = DPAA2_ETH_DIST_IPSRC, 327 .size = 4, 328 }, { 329 .rxnfc_field = RXH_IP_DST, 330 .cls_prot = NET_PROT_IP, 331 .cls_field = NH_FLD_IP_DST, 332 .id = DPAA2_ETH_DIST_IPDST, 333 .size = 4, 334 }, { 335 .rxnfc_field = RXH_L3_PROTO, 336 .cls_prot = NET_PROT_IP, 337 .cls_field = NH_FLD_IP_PROTO, 338 .id = DPAA2_ETH_DIST_IPPROTO, 339 .size = 1, 340 }, { 341 /* Using UDP ports, this is functionally equivalent to raw 342 * byte pairs from L4 header. 343 */ 344 .rxnfc_field = RXH_L4_B_0_1, 345 .cls_prot = NET_PROT_UDP, 346 .cls_field = NH_FLD_UDP_PORT_SRC, 347 .id = DPAA2_ETH_DIST_L4SRC, 348 .size = 2, 349 }, { 350 .rxnfc_field = RXH_L4_B_2_3, 351 .cls_prot = NET_PROT_UDP, 352 .cls_field = NH_FLD_UDP_PORT_DST, 353 .id = DPAA2_ETH_DIST_L4DST, 354 .size = 2, 355 }, 356 }; 357 358 static struct dpni_stat { 359 int page; 360 int cnt; 361 char *name; 362 char *desc; 363 } dpni_stat_sysctls[DPAA2_NI_STAT_SYSCTLS] = { 364 /* PAGE, COUNTER, NAME, DESCRIPTION */ 365 { 0, 0, "in_all_frames", "All accepted ingress frames" }, 366 { 0, 1, "in_all_bytes", "Bytes in all accepted ingress frames" }, 367 { 0, 2, "in_multi_frames", "Multicast accepted ingress frames" }, 368 { 1, 0, "eg_all_frames", "All egress frames transmitted" }, 369 { 1, 1, "eg_all_bytes", "Bytes in all frames transmitted" }, 370 { 1, 2, "eg_multi_frames", "Multicast egress frames transmitted" }, 371 { 2, 0, "in_filtered_frames", "All ingress frames discarded due to " 372 "filtering" }, 373 { 2, 1, "in_discarded_frames", "All frames discarded due to errors" }, 374 { 2, 2, "in_nobuf_discards", "Discards on ingress side due to buffer " 375 "depletion in DPNI buffer pools" }, 376 }; 377 378 struct dpaa2_ni_rx_ctx { 379 struct mbuf *head; 380 struct mbuf *tail; 381 int cnt; 382 bool last; 383 }; 384 385 /* Device interface */ 386 static int dpaa2_ni_probe(device_t); 387 static int dpaa2_ni_attach(device_t); 388 static int dpaa2_ni_detach(device_t); 389 390 /* DPAA2 network interface setup and configuration */ 391 static int dpaa2_ni_setup(device_t); 392 static int dpaa2_ni_setup_channels(device_t); 393 static int dpaa2_ni_bind(device_t); 394 static int dpaa2_ni_setup_rx_dist(device_t); 395 static int dpaa2_ni_setup_irqs(device_t); 396 static int dpaa2_ni_setup_msi(struct dpaa2_ni_softc *); 397 static int dpaa2_ni_setup_if_caps(struct dpaa2_ni_softc *); 398 static int dpaa2_ni_setup_if_flags(struct dpaa2_ni_softc *); 399 static int dpaa2_ni_setup_sysctls(struct dpaa2_ni_softc *); 400 static int dpaa2_ni_setup_dma(struct dpaa2_ni_softc *); 401 402 /* Tx/Rx flow configuration */ 403 static int dpaa2_ni_setup_rx_flow(device_t, struct dpaa2_ni_fq *); 404 static int dpaa2_ni_setup_tx_flow(device_t, struct dpaa2_ni_fq *); 405 static int dpaa2_ni_setup_rx_err_flow(device_t, struct dpaa2_ni_fq *); 406 407 /* Configuration subroutines */ 408 static int dpaa2_ni_set_buf_layout(device_t); 409 static int dpaa2_ni_set_pause_frame(device_t); 410 static int dpaa2_ni_set_qos_table(device_t); 411 static int dpaa2_ni_set_mac_addr(device_t); 412 static int dpaa2_ni_set_hash(device_t, uint64_t); 413 static int dpaa2_ni_set_dist_key(device_t, enum dpaa2_ni_dist_mode, uint64_t); 414 415 /* Various subroutines */ 416 static int dpaa2_ni_cmp_api_version(struct dpaa2_ni_softc *, uint16_t, uint16_t); 417 static int dpaa2_ni_prepare_key_cfg(struct dpkg_profile_cfg *, uint8_t *); 418 419 /* Network interface routines */ 420 static void dpaa2_ni_init(void *); 421 static int dpaa2_ni_transmit(if_t , struct mbuf *); 422 static void dpaa2_ni_qflush(if_t ); 423 static int dpaa2_ni_ioctl(if_t , u_long, caddr_t); 424 static int dpaa2_ni_update_mac_filters(if_t ); 425 static u_int dpaa2_ni_add_maddr(void *, struct sockaddr_dl *, u_int); 426 427 /* Interrupt handlers */ 428 static void dpaa2_ni_intr(void *); 429 430 /* MII handlers */ 431 static void dpaa2_ni_miibus_statchg(device_t); 432 static int dpaa2_ni_media_change(if_t ); 433 static void dpaa2_ni_media_status(if_t , struct ifmediareq *); 434 static void dpaa2_ni_media_tick(void *); 435 436 /* Tx/Rx routines. */ 437 static int dpaa2_ni_rx_cleanup(struct dpaa2_channel *); 438 static int dpaa2_ni_tx_cleanup(struct dpaa2_channel *); 439 static void dpaa2_ni_tx(struct dpaa2_ni_softc *, struct dpaa2_channel *, 440 struct dpaa2_ni_tx_ring *, struct mbuf *); 441 static void dpaa2_ni_cleanup_task(void *, int); 442 443 /* Tx/Rx subroutines */ 444 static int dpaa2_ni_consume_frames(struct dpaa2_channel *, struct dpaa2_ni_fq **, 445 uint32_t *); 446 static int dpaa2_ni_rx(struct dpaa2_channel *, struct dpaa2_ni_fq *, 447 struct dpaa2_fd *, struct dpaa2_ni_rx_ctx *); 448 static int dpaa2_ni_rx_err(struct dpaa2_channel *, struct dpaa2_ni_fq *, 449 struct dpaa2_fd *); 450 static int dpaa2_ni_tx_conf(struct dpaa2_channel *, struct dpaa2_ni_fq *, 451 struct dpaa2_fd *); 452 453 /* sysctl(9) */ 454 static int dpaa2_ni_collect_stats(SYSCTL_HANDLER_ARGS); 455 static int dpaa2_ni_collect_buf_num(SYSCTL_HANDLER_ARGS); 456 static int dpaa2_ni_collect_buf_free(SYSCTL_HANDLER_ARGS); 457 458 static int 459 dpaa2_ni_probe(device_t dev) 460 { 461 /* DPNI device will be added by a parent resource container itself. */ 462 device_set_desc(dev, "DPAA2 Network Interface"); 463 return (BUS_PROBE_DEFAULT); 464 } 465 466 static int 467 dpaa2_ni_attach(device_t dev) 468 { 469 device_t pdev = device_get_parent(dev); 470 device_t child = dev; 471 device_t mcp_dev; 472 struct dpaa2_ni_softc *sc = device_get_softc(dev); 473 struct dpaa2_devinfo *rcinfo = device_get_ivars(pdev); 474 struct dpaa2_devinfo *dinfo = device_get_ivars(dev); 475 struct dpaa2_devinfo *mcp_dinfo; 476 struct dpaa2_cmd cmd; 477 uint16_t rc_token, ni_token; 478 if_t ifp; 479 char tq_name[32]; 480 int error; 481 482 sc->dev = dev; 483 sc->ifp = NULL; 484 sc->miibus = NULL; 485 sc->mii = NULL; 486 sc->media_status = 0; 487 sc->if_flags = 0; 488 sc->link_state = LINK_STATE_UNKNOWN; 489 sc->buf_align = 0; 490 491 /* For debug purposes only! */ 492 sc->rx_anomaly_frames = 0; 493 sc->rx_single_buf_frames = 0; 494 sc->rx_sg_buf_frames = 0; 495 sc->rx_enq_rej_frames = 0; 496 sc->rx_ieoi_err_frames = 0; 497 sc->tx_single_buf_frames = 0; 498 sc->tx_sg_frames = 0; 499 500 DPAA2_ATOMIC_XCHG(&sc->buf_num, 0); 501 DPAA2_ATOMIC_XCHG(&sc->buf_free, 0); 502 503 sc->rxd_dmat = NULL; 504 sc->qos_dmat = NULL; 505 506 sc->qos_kcfg.dmap = NULL; 507 sc->qos_kcfg.paddr = 0; 508 sc->qos_kcfg.vaddr = NULL; 509 510 sc->rxd_kcfg.dmap = NULL; 511 sc->rxd_kcfg.paddr = 0; 512 sc->rxd_kcfg.vaddr = NULL; 513 514 sc->mac.dpmac_id = 0; 515 sc->mac.phy_dev = NULL; 516 memset(sc->mac.addr, 0, ETHER_ADDR_LEN); 517 518 error = bus_alloc_resources(sc->dev, dpaa2_ni_spec, sc->res); 519 if (error) { 520 device_printf(dev, "%s: failed to allocate resources: " 521 "error=%d\n", __func__, error); 522 goto err_exit; 523 } 524 525 /* Obtain MC portal. */ 526 mcp_dev = (device_t) rman_get_start(sc->res[DPAA2_NI_MCP_RID(0)]); 527 mcp_dinfo = device_get_ivars(mcp_dev); 528 dinfo->portal = mcp_dinfo->portal; 529 530 mtx_init(&sc->lock, device_get_nameunit(dev), "dpaa2_ni", MTX_DEF); 531 532 /* Allocate network interface */ 533 ifp = if_alloc(IFT_ETHER); 534 sc->ifp = ifp; 535 if_initname(ifp, DPAA2_NI_IFNAME, device_get_unit(sc->dev)); 536 537 if_setsoftc(ifp, sc); 538 if_setflags(ifp, IFF_SIMPLEX | IFF_MULTICAST | IFF_BROADCAST); 539 if_setinitfn(ifp, dpaa2_ni_init); 540 if_setioctlfn(ifp, dpaa2_ni_ioctl); 541 if_settransmitfn(ifp, dpaa2_ni_transmit); 542 if_setqflushfn(ifp, dpaa2_ni_qflush); 543 544 if_sethwassist(sc->ifp, DPAA2_CSUM_TX_OFFLOAD); 545 if_setcapabilities(ifp, IFCAP_VLAN_MTU | IFCAP_HWCSUM | 546 IFCAP_HWCSUM_IPV6 | IFCAP_JUMBO_MTU); 547 if_setcapenable(ifp, if_getcapabilities(ifp)); 548 549 DPAA2_CMD_INIT(&cmd); 550 551 /* Open resource container and network interface object. */ 552 error = DPAA2_CMD_RC_OPEN(dev, child, &cmd, rcinfo->id, &rc_token); 553 if (error) { 554 device_printf(dev, "%s: failed to open resource container: " 555 "id=%d, error=%d\n", __func__, rcinfo->id, error); 556 goto err_exit; 557 } 558 error = DPAA2_CMD_NI_OPEN(dev, child, &cmd, dinfo->id, &ni_token); 559 if (error) { 560 device_printf(dev, "%s: failed to open network interface: " 561 "id=%d, error=%d\n", __func__, dinfo->id, error); 562 goto close_rc; 563 } 564 565 bzero(tq_name, sizeof(tq_name)); 566 snprintf(tq_name, sizeof(tq_name), "%s_tqbp", device_get_nameunit(dev)); 567 568 /* 569 * XXX-DSL: Release new buffers on Buffer Pool State Change Notification 570 * (BPSCN) returned as a result to the VDQ command instead. 571 * It is similar to CDAN processed in dpaa2_io_intr(). 572 */ 573 /* Create a taskqueue thread to release new buffers to the pool. */ 574 sc->bp_taskq = taskqueue_create(tq_name, M_WAITOK, 575 taskqueue_thread_enqueue, &sc->bp_taskq); 576 taskqueue_start_threads(&sc->bp_taskq, 1, PI_NET, "%s", tq_name); 577 578 /* sc->cleanup_taskq = taskqueue_create("dpaa2_ch cleanup", M_WAITOK, */ 579 /* taskqueue_thread_enqueue, &sc->cleanup_taskq); */ 580 /* taskqueue_start_threads(&sc->cleanup_taskq, 1, PI_NET, */ 581 /* "dpaa2_ch cleanup"); */ 582 583 error = dpaa2_ni_setup(dev); 584 if (error) { 585 device_printf(dev, "%s: failed to setup DPNI: error=%d\n", 586 __func__, error); 587 goto close_ni; 588 } 589 error = dpaa2_ni_setup_channels(dev); 590 if (error) { 591 device_printf(dev, "%s: failed to setup QBMan channels: " 592 "error=%d\n", __func__, error); 593 goto close_ni; 594 } 595 596 error = dpaa2_ni_bind(dev); 597 if (error) { 598 device_printf(dev, "%s: failed to bind DPNI: error=%d\n", 599 __func__, error); 600 goto close_ni; 601 } 602 error = dpaa2_ni_setup_irqs(dev); 603 if (error) { 604 device_printf(dev, "%s: failed to setup IRQs: error=%d\n", 605 __func__, error); 606 goto close_ni; 607 } 608 error = dpaa2_ni_setup_sysctls(sc); 609 if (error) { 610 device_printf(dev, "%s: failed to setup sysctls: error=%d\n", 611 __func__, error); 612 goto close_ni; 613 } 614 error = dpaa2_ni_setup_if_caps(sc); 615 if (error) { 616 device_printf(dev, "%s: failed to setup interface capabilities: " 617 "error=%d\n", __func__, error); 618 goto close_ni; 619 } 620 621 ether_ifattach(sc->ifp, sc->mac.addr); 622 callout_init(&sc->mii_callout, 0); 623 624 return (0); 625 626 close_ni: 627 DPAA2_CMD_NI_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, ni_token)); 628 close_rc: 629 DPAA2_CMD_RC_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, rc_token)); 630 err_exit: 631 return (ENXIO); 632 } 633 634 static void 635 dpaa2_ni_fixed_media_status(if_t ifp, struct ifmediareq* ifmr) 636 { 637 struct dpaa2_ni_softc *sc = if_getsoftc(ifp); 638 639 DPNI_LOCK(sc); 640 ifmr->ifm_count = 0; 641 ifmr->ifm_mask = 0; 642 ifmr->ifm_status = IFM_AVALID | IFM_ACTIVE; 643 ifmr->ifm_current = ifmr->ifm_active = 644 sc->fixed_ifmedia.ifm_cur->ifm_media; 645 646 /* 647 * In non-PHY usecases, we need to signal link state up, otherwise 648 * certain things requiring a link event (e.g async DHCP client) from 649 * devd do not happen. 650 */ 651 if (if_getlinkstate(ifp) == LINK_STATE_UNKNOWN) { 652 if_link_state_change(ifp, LINK_STATE_UP); 653 } 654 655 /* 656 * TODO: Check the status of the link partner (DPMAC, DPNI or other) and 657 * reset if down. This is different to the DPAA2_MAC_LINK_TYPE_PHY as 658 * the MC firmware sets the status, instead of us telling the MC what 659 * it is. 660 */ 661 DPNI_UNLOCK(sc); 662 663 return; 664 } 665 666 static void 667 dpaa2_ni_setup_fixed_link(struct dpaa2_ni_softc *sc) 668 { 669 /* 670 * FIXME: When the DPNI is connected to a DPMAC, we can get the 671 * 'apparent' speed from it. 672 */ 673 sc->fixed_link = true; 674 675 ifmedia_init(&sc->fixed_ifmedia, 0, dpaa2_ni_media_change, 676 dpaa2_ni_fixed_media_status); 677 ifmedia_add(&sc->fixed_ifmedia, IFM_ETHER | IFM_1000_T, 0, NULL); 678 ifmedia_set(&sc->fixed_ifmedia, IFM_ETHER | IFM_1000_T); 679 } 680 681 static int 682 dpaa2_ni_detach(device_t dev) 683 { 684 /* TBD */ 685 return (0); 686 } 687 688 /** 689 * @brief Configure DPAA2 network interface object. 690 */ 691 static int 692 dpaa2_ni_setup(device_t dev) 693 { 694 device_t pdev = device_get_parent(dev); 695 device_t child = dev; 696 struct dpaa2_ni_softc *sc = device_get_softc(dev); 697 struct dpaa2_devinfo *rcinfo = device_get_ivars(pdev); 698 struct dpaa2_devinfo *dinfo = device_get_ivars(dev); 699 struct dpaa2_ep_desc ep1_desc, ep2_desc; /* endpoint descriptors */ 700 struct dpaa2_cmd cmd; 701 uint8_t eth_bca[ETHER_ADDR_LEN]; /* broadcast physical address */ 702 uint16_t rc_token, ni_token, mac_token; 703 struct dpaa2_mac_attr attr; 704 enum dpaa2_mac_link_type link_type; 705 uint32_t link; 706 int error; 707 708 DPAA2_CMD_INIT(&cmd); 709 710 error = DPAA2_CMD_RC_OPEN(dev, child, &cmd, rcinfo->id, &rc_token); 711 if (error) { 712 device_printf(dev, "%s: failed to open resource container: " 713 "id=%d, error=%d\n", __func__, rcinfo->id, error); 714 goto err_exit; 715 } 716 error = DPAA2_CMD_NI_OPEN(dev, child, &cmd, dinfo->id, &ni_token); 717 if (error) { 718 device_printf(dev, "%s: failed to open network interface: " 719 "id=%d, error=%d\n", __func__, dinfo->id, error); 720 goto close_rc; 721 } 722 723 /* Check if we can work with this DPNI object. */ 724 error = DPAA2_CMD_NI_GET_API_VERSION(dev, child, &cmd, &sc->api_major, 725 &sc->api_minor); 726 if (error) { 727 device_printf(dev, "%s: failed to get DPNI API version\n", 728 __func__); 729 goto close_ni; 730 } 731 if (dpaa2_ni_cmp_api_version(sc, DPNI_VER_MAJOR, DPNI_VER_MINOR) < 0) { 732 device_printf(dev, "%s: DPNI API version %u.%u not supported, " 733 "need >= %u.%u\n", __func__, sc->api_major, sc->api_minor, 734 DPNI_VER_MAJOR, DPNI_VER_MINOR); 735 error = ENODEV; 736 goto close_ni; 737 } 738 739 /* Reset the DPNI object. */ 740 error = DPAA2_CMD_NI_RESET(dev, child, &cmd); 741 if (error) { 742 device_printf(dev, "%s: failed to reset DPNI: id=%d\n", 743 __func__, dinfo->id); 744 goto close_ni; 745 } 746 747 /* Obtain attributes of the DPNI object. */ 748 error = DPAA2_CMD_NI_GET_ATTRIBUTES(dev, child, &cmd, &sc->attr); 749 if (error) { 750 device_printf(dev, "%s: failed to obtain DPNI attributes: " 751 "id=%d\n", __func__, dinfo->id); 752 goto close_ni; 753 } 754 if (bootverbose) { 755 device_printf(dev, "\toptions=0x%#x queues=%d tx_channels=%d " 756 "wriop_version=%#x\n", sc->attr.options, sc->attr.num.queues, 757 sc->attr.num.channels, sc->attr.wriop_ver); 758 device_printf(dev, "\ttraffic classes: rx=%d tx=%d " 759 "cgs_groups=%d\n", sc->attr.num.rx_tcs, sc->attr.num.tx_tcs, 760 sc->attr.num.cgs); 761 device_printf(dev, "\ttable entries: mac=%d vlan=%d qos=%d " 762 "fs=%d\n", sc->attr.entries.mac, sc->attr.entries.vlan, 763 sc->attr.entries.qos, sc->attr.entries.fs); 764 device_printf(dev, "\tkey sizes: qos=%d fs=%d\n", 765 sc->attr.key_size.qos, sc->attr.key_size.fs); 766 } 767 768 /* Configure buffer layouts of the DPNI queues. */ 769 error = dpaa2_ni_set_buf_layout(dev); 770 if (error) { 771 device_printf(dev, "%s: failed to configure buffer layout\n", 772 __func__); 773 goto close_ni; 774 } 775 776 /* Configure DMA resources. */ 777 error = dpaa2_ni_setup_dma(sc); 778 if (error) { 779 device_printf(dev, "%s: failed to setup DMA\n", __func__); 780 goto close_ni; 781 } 782 783 /* Setup link between DPNI and an object it's connected to. */ 784 ep1_desc.obj_id = dinfo->id; 785 ep1_desc.if_id = 0; /* DPNI has the only endpoint */ 786 ep1_desc.type = dinfo->dtype; 787 788 error = DPAA2_CMD_RC_GET_CONN(dev, child, DPAA2_CMD_TK(&cmd, rc_token), 789 &ep1_desc, &ep2_desc, &link); 790 if (error) { 791 device_printf(dev, "%s: failed to obtain an object DPNI is " 792 "connected to: error=%d\n", __func__, error); 793 } else { 794 device_printf(dev, "connected to %s (id=%d)\n", 795 dpaa2_ttos(ep2_desc.type), ep2_desc.obj_id); 796 797 error = dpaa2_ni_set_mac_addr(dev); 798 if (error) { 799 device_printf(dev, "%s: failed to set MAC address: " 800 "error=%d\n", __func__, error); 801 } 802 803 if (ep2_desc.type == DPAA2_DEV_MAC) { 804 /* 805 * This is the simplest case when DPNI is connected to 806 * DPMAC directly. 807 */ 808 sc->mac.dpmac_id = ep2_desc.obj_id; 809 810 link_type = DPAA2_MAC_LINK_TYPE_NONE; 811 812 /* 813 * Need to determine if DPMAC type is PHY (attached to 814 * conventional MII PHY) or FIXED (usually SFP/SerDes, 815 * link state managed by MC firmware). 816 */ 817 error = DPAA2_CMD_MAC_OPEN(sc->dev, child, 818 DPAA2_CMD_TK(&cmd, rc_token), sc->mac.dpmac_id, 819 &mac_token); 820 /* 821 * Under VFIO, the DPMAC might be sitting in another 822 * container (DPRC) we don't have access to. 823 * Assume DPAA2_MAC_LINK_TYPE_FIXED if this is 824 * the case. 825 */ 826 if (error) { 827 device_printf(dev, "%s: failed to open " 828 "connected DPMAC: %d (assuming in other DPRC)\n", __func__, 829 sc->mac.dpmac_id); 830 link_type = DPAA2_MAC_LINK_TYPE_FIXED; 831 } else { 832 error = DPAA2_CMD_MAC_GET_ATTRIBUTES(dev, child, 833 &cmd, &attr); 834 if (error) { 835 device_printf(dev, "%s: failed to get " 836 "DPMAC attributes: id=%d, " 837 "error=%d\n", __func__, dinfo->id, 838 error); 839 } else { 840 link_type = attr.link_type; 841 } 842 } 843 DPAA2_CMD_MAC_CLOSE(dev, child, &cmd); 844 845 if (link_type == DPAA2_MAC_LINK_TYPE_FIXED) { 846 device_printf(dev, "connected DPMAC is in FIXED " 847 "mode\n"); 848 dpaa2_ni_setup_fixed_link(sc); 849 } else if (link_type == DPAA2_MAC_LINK_TYPE_PHY) { 850 device_printf(dev, "connected DPMAC is in PHY " 851 "mode\n"); 852 error = DPAA2_MC_GET_PHY_DEV(dev, 853 &sc->mac.phy_dev, sc->mac.dpmac_id); 854 if (error == 0) { 855 error = MEMAC_MDIO_SET_NI_DEV( 856 sc->mac.phy_dev, dev); 857 if (error != 0) { 858 device_printf(dev, "%s: failed " 859 "to set dpni dev on memac " 860 "mdio dev %s: error=%d\n", 861 __func__, 862 device_get_nameunit( 863 sc->mac.phy_dev), error); 864 } 865 } 866 if (error == 0) { 867 error = MEMAC_MDIO_GET_PHY_LOC( 868 sc->mac.phy_dev, &sc->mac.phy_loc); 869 if (error == ENODEV) { 870 error = 0; 871 } 872 if (error != 0) { 873 device_printf(dev, "%s: failed " 874 "to get phy location from " 875 "memac mdio dev %s: error=%d\n", 876 __func__, device_get_nameunit( 877 sc->mac.phy_dev), error); 878 } 879 } 880 if (error == 0) { 881 error = mii_attach(sc->mac.phy_dev, 882 &sc->miibus, sc->ifp, 883 dpaa2_ni_media_change, 884 dpaa2_ni_media_status, 885 BMSR_DEFCAPMASK, sc->mac.phy_loc, 886 MII_OFFSET_ANY, 0); 887 if (error != 0) { 888 device_printf(dev, "%s: failed " 889 "to attach to miibus: " 890 "error=%d\n", 891 __func__, error); 892 } 893 } 894 if (error == 0) { 895 sc->mii = device_get_softc(sc->miibus); 896 } 897 } else { 898 device_printf(dev, "%s: DPMAC link type is not " 899 "supported\n", __func__); 900 } 901 } else if (ep2_desc.type == DPAA2_DEV_NI || 902 ep2_desc.type == DPAA2_DEV_MUX || 903 ep2_desc.type == DPAA2_DEV_SW) { 904 dpaa2_ni_setup_fixed_link(sc); 905 } 906 } 907 908 /* Select mode to enqueue frames. */ 909 /* ... TBD ... */ 910 911 /* 912 * Update link configuration to enable Rx/Tx pause frames support. 913 * 914 * NOTE: MC may generate an interrupt to the DPMAC and request changes 915 * in link configuration. It might be necessary to attach miibus 916 * and PHY before this point. 917 */ 918 error = dpaa2_ni_set_pause_frame(dev); 919 if (error) { 920 device_printf(dev, "%s: failed to configure Rx/Tx pause " 921 "frames\n", __func__); 922 goto close_ni; 923 } 924 925 /* Configure ingress traffic classification. */ 926 error = dpaa2_ni_set_qos_table(dev); 927 if (error) { 928 device_printf(dev, "%s: failed to configure QoS table: " 929 "error=%d\n", __func__, error); 930 goto close_ni; 931 } 932 933 /* Add broadcast physical address to the MAC filtering table. */ 934 memset(eth_bca, 0xff, ETHER_ADDR_LEN); 935 error = DPAA2_CMD_NI_ADD_MAC_ADDR(dev, child, DPAA2_CMD_TK(&cmd, 936 ni_token), eth_bca); 937 if (error) { 938 device_printf(dev, "%s: failed to add broadcast physical " 939 "address to the MAC filtering table\n", __func__); 940 goto close_ni; 941 } 942 943 /* Set the maximum allowed length for received frames. */ 944 error = DPAA2_CMD_NI_SET_MFL(dev, child, &cmd, DPAA2_ETH_MFL); 945 if (error) { 946 device_printf(dev, "%s: failed to set maximum length for " 947 "received frames\n", __func__); 948 goto close_ni; 949 } 950 951 (void)DPAA2_CMD_NI_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, ni_token)); 952 (void)DPAA2_CMD_RC_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, rc_token)); 953 return (0); 954 955 close_ni: 956 (void)DPAA2_CMD_NI_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, ni_token)); 957 close_rc: 958 (void)DPAA2_CMD_RC_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, rc_token)); 959 err_exit: 960 return (error); 961 } 962 963 /** 964 * @brief Сonfigure QBMan channels and register data availability notifications. 965 */ 966 static int 967 dpaa2_ni_setup_channels(device_t dev) 968 { 969 device_t iodev, condev, bpdev; 970 struct dpaa2_ni_softc *sc = device_get_softc(dev); 971 uint32_t i, num_chan; 972 int error; 973 974 /* Calculate number of the channels based on the allocated resources */ 975 for (i = 0; i < DPAA2_NI_IO_RES_NUM; i++) { 976 if (!sc->res[DPAA2_NI_IO_RID(i)]) { 977 break; 978 } 979 } 980 num_chan = i; 981 for (i = 0; i < DPAA2_NI_CON_RES_NUM; i++) { 982 if (!sc->res[DPAA2_NI_CON_RID(i)]) { 983 break; 984 } 985 } 986 num_chan = i < num_chan ? i : num_chan; 987 sc->chan_n = num_chan > DPAA2_MAX_CHANNELS 988 ? DPAA2_MAX_CHANNELS : num_chan; 989 sc->chan_n = sc->chan_n > sc->attr.num.queues 990 ? sc->attr.num.queues : sc->chan_n; 991 992 KASSERT(sc->chan_n > 0u, ("%s: positive number of channels expected: " 993 "chan_n=%d", __func__, sc->chan_n)); 994 995 device_printf(dev, "channels=%d\n", sc->chan_n); 996 997 for (i = 0; i < sc->chan_n; i++) { 998 iodev = (device_t)rman_get_start(sc->res[DPAA2_NI_IO_RID(i)]); 999 condev = (device_t)rman_get_start(sc->res[DPAA2_NI_CON_RID(i)]); 1000 /* Only one buffer pool available at the moment */ 1001 bpdev = (device_t)rman_get_start(sc->res[DPAA2_NI_BP_RID(0)]); 1002 1003 error = dpaa2_chan_setup(dev, iodev, condev, bpdev, 1004 &sc->channels[i], i, dpaa2_ni_cleanup_task); 1005 if (error != 0) { 1006 device_printf(dev, "%s: dpaa2_chan_setup() failed: " 1007 "error=%d, chan_id=%d\n", __func__, error, i); 1008 return (error); 1009 } 1010 } 1011 1012 /* There is exactly one Rx error queue per network interface */ 1013 error = dpaa2_chan_setup_fq(dev, sc->channels[0], DPAA2_NI_QUEUE_RX_ERR); 1014 if (error != 0) { 1015 device_printf(dev, "%s: failed to prepare RxError queue: " 1016 "error=%d\n", __func__, error); 1017 return (error); 1018 } 1019 1020 return (0); 1021 } 1022 1023 /** 1024 * @brief Bind DPNI to DPBPs, DPIOs, frame queues and channels. 1025 */ 1026 static int 1027 dpaa2_ni_bind(device_t dev) 1028 { 1029 device_t pdev = device_get_parent(dev); 1030 device_t child = dev; 1031 device_t bp_dev; 1032 struct dpaa2_ni_softc *sc = device_get_softc(dev); 1033 struct dpaa2_devinfo *rcinfo = device_get_ivars(pdev); 1034 struct dpaa2_devinfo *dinfo = device_get_ivars(dev); 1035 struct dpaa2_devinfo *bp_info; 1036 struct dpaa2_cmd cmd; 1037 struct dpaa2_ni_pools_cfg pools_cfg; 1038 struct dpaa2_ni_err_cfg err_cfg; 1039 struct dpaa2_channel *chan; 1040 uint16_t rc_token, ni_token; 1041 int error; 1042 1043 DPAA2_CMD_INIT(&cmd); 1044 1045 error = DPAA2_CMD_RC_OPEN(dev, child, &cmd, rcinfo->id, &rc_token); 1046 if (error) { 1047 device_printf(dev, "%s: failed to open resource container: " 1048 "id=%d, error=%d\n", __func__, rcinfo->id, error); 1049 goto err_exit; 1050 } 1051 error = DPAA2_CMD_NI_OPEN(dev, child, &cmd, dinfo->id, &ni_token); 1052 if (error) { 1053 device_printf(dev, "%s: failed to open network interface: " 1054 "id=%d, error=%d\n", __func__, dinfo->id, error); 1055 goto close_rc; 1056 } 1057 1058 /* Select buffer pool (only one available at the moment). */ 1059 bp_dev = (device_t) rman_get_start(sc->res[DPAA2_NI_BP_RID(0)]); 1060 bp_info = device_get_ivars(bp_dev); 1061 1062 /* Configure buffers pool. */ 1063 pools_cfg.pools_num = 1; 1064 pools_cfg.pools[0].bp_obj_id = bp_info->id; 1065 pools_cfg.pools[0].backup_flag = 0; 1066 pools_cfg.pools[0].buf_sz = sc->buf_sz; 1067 error = DPAA2_CMD_NI_SET_POOLS(dev, child, &cmd, &pools_cfg); 1068 if (error) { 1069 device_printf(dev, "%s: failed to set buffer pools\n", __func__); 1070 goto close_ni; 1071 } 1072 1073 /* Setup ingress traffic distribution. */ 1074 error = dpaa2_ni_setup_rx_dist(dev); 1075 if (error && error != EOPNOTSUPP) { 1076 device_printf(dev, "%s: failed to setup ingress traffic " 1077 "distribution\n", __func__); 1078 goto close_ni; 1079 } 1080 if (bootverbose && error == EOPNOTSUPP) { 1081 device_printf(dev, "Ingress traffic distribution not " 1082 "supported\n"); 1083 } 1084 1085 /* Configure handling of error frames. */ 1086 err_cfg.err_mask = DPAA2_NI_FAS_RX_ERR_MASK; 1087 err_cfg.set_err_fas = false; 1088 err_cfg.action = DPAA2_NI_ERR_DISCARD; 1089 error = DPAA2_CMD_NI_SET_ERR_BEHAVIOR(dev, child, &cmd, &err_cfg); 1090 if (error) { 1091 device_printf(dev, "%s: failed to set errors behavior\n", 1092 __func__); 1093 goto close_ni; 1094 } 1095 1096 /* Configure channel queues to generate CDANs. */ 1097 for (uint32_t i = 0; i < sc->chan_n; i++) { 1098 chan = sc->channels[i]; 1099 1100 /* Setup Rx flows. */ 1101 for (uint32_t j = 0; j < chan->rxq_n; j++) { 1102 error = dpaa2_ni_setup_rx_flow(dev, &chan->rx_queues[j]); 1103 if (error) { 1104 device_printf(dev, "%s: failed to setup Rx " 1105 "flow: error=%d\n", __func__, error); 1106 goto close_ni; 1107 } 1108 } 1109 1110 /* Setup Tx flow. */ 1111 error = dpaa2_ni_setup_tx_flow(dev, &chan->txc_queue); 1112 if (error) { 1113 device_printf(dev, "%s: failed to setup Tx " 1114 "flow: error=%d\n", __func__, error); 1115 goto close_ni; 1116 } 1117 } 1118 1119 /* Configure RxError queue to generate CDAN. */ 1120 error = dpaa2_ni_setup_rx_err_flow(dev, &sc->rxe_queue); 1121 if (error) { 1122 device_printf(dev, "%s: failed to setup RxError flow: " 1123 "error=%d\n", __func__, error); 1124 goto close_ni; 1125 } 1126 1127 /* 1128 * Get the Queuing Destination ID (QDID) that should be used for frame 1129 * enqueue operations. 1130 */ 1131 error = DPAA2_CMD_NI_GET_QDID(dev, child, &cmd, DPAA2_NI_QUEUE_TX, 1132 &sc->tx_qdid); 1133 if (error) { 1134 device_printf(dev, "%s: failed to get Tx queuing destination " 1135 "ID\n", __func__); 1136 goto close_ni; 1137 } 1138 1139 (void)DPAA2_CMD_NI_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, ni_token)); 1140 (void)DPAA2_CMD_RC_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, rc_token)); 1141 return (0); 1142 1143 close_ni: 1144 (void)DPAA2_CMD_NI_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, ni_token)); 1145 close_rc: 1146 (void)DPAA2_CMD_RC_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, rc_token)); 1147 err_exit: 1148 return (error); 1149 } 1150 1151 /** 1152 * @brief Setup ingress traffic distribution. 1153 * 1154 * NOTE: Ingress traffic distribution is valid only when DPNI_OPT_NO_FS option 1155 * hasn't been set for DPNI and a number of DPNI queues > 1. 1156 */ 1157 static int 1158 dpaa2_ni_setup_rx_dist(device_t dev) 1159 { 1160 /* 1161 * Have the interface implicitly distribute traffic based on the default 1162 * hash key. 1163 */ 1164 return (dpaa2_ni_set_hash(dev, DPAA2_RXH_DEFAULT)); 1165 } 1166 1167 static int 1168 dpaa2_ni_setup_rx_flow(device_t dev, struct dpaa2_ni_fq *fq) 1169 { 1170 device_t pdev = device_get_parent(dev); 1171 device_t child = dev; 1172 struct dpaa2_devinfo *rcinfo = device_get_ivars(pdev); 1173 struct dpaa2_devinfo *dinfo = device_get_ivars(dev); 1174 struct dpaa2_devinfo *con_info; 1175 struct dpaa2_cmd cmd; 1176 struct dpaa2_ni_queue_cfg queue_cfg = {0}; 1177 uint16_t rc_token, ni_token; 1178 int error; 1179 1180 DPAA2_CMD_INIT(&cmd); 1181 1182 error = DPAA2_CMD_RC_OPEN(dev, child, &cmd, rcinfo->id, &rc_token); 1183 if (error) { 1184 device_printf(dev, "%s: failed to open resource container: " 1185 "id=%d, error=%d\n", __func__, rcinfo->id, error); 1186 goto err_exit; 1187 } 1188 error = DPAA2_CMD_NI_OPEN(dev, child, &cmd, dinfo->id, &ni_token); 1189 if (error) { 1190 device_printf(dev, "%s: failed to open network interface: " 1191 "id=%d, error=%d\n", __func__, dinfo->id, error); 1192 goto close_rc; 1193 } 1194 1195 /* Obtain DPCON associated with the FQ's channel. */ 1196 con_info = device_get_ivars(fq->chan->con_dev); 1197 1198 queue_cfg.type = DPAA2_NI_QUEUE_RX; 1199 queue_cfg.tc = fq->tc; 1200 queue_cfg.idx = fq->flowid; 1201 error = DPAA2_CMD_NI_GET_QUEUE(dev, child, &cmd, &queue_cfg); 1202 if (error) { 1203 device_printf(dev, "%s: failed to obtain Rx queue " 1204 "configuration: tc=%d, flowid=%d\n", __func__, queue_cfg.tc, 1205 queue_cfg.idx); 1206 goto close_ni; 1207 } 1208 1209 fq->fqid = queue_cfg.fqid; 1210 1211 queue_cfg.dest_id = con_info->id; 1212 queue_cfg.dest_type = DPAA2_NI_DEST_DPCON; 1213 queue_cfg.priority = 1; 1214 queue_cfg.user_ctx = (uint64_t)(uintmax_t) fq; 1215 queue_cfg.options = 1216 DPAA2_NI_QUEUE_OPT_USER_CTX | 1217 DPAA2_NI_QUEUE_OPT_DEST; 1218 error = DPAA2_CMD_NI_SET_QUEUE(dev, child, &cmd, &queue_cfg); 1219 if (error) { 1220 device_printf(dev, "%s: failed to update Rx queue " 1221 "configuration: tc=%d, flowid=%d\n", __func__, queue_cfg.tc, 1222 queue_cfg.idx); 1223 goto close_ni; 1224 } 1225 1226 if (bootverbose) { 1227 device_printf(dev, "RX queue idx=%d, tc=%d, chan=%d, fqid=%d, " 1228 "user_ctx=%#jx\n", fq->flowid, fq->tc, fq->chan->id, 1229 fq->fqid, (uint64_t) fq); 1230 } 1231 1232 (void)DPAA2_CMD_NI_CLOSE(dev, child, &cmd); 1233 (void)DPAA2_CMD_RC_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, rc_token)); 1234 return (0); 1235 1236 close_ni: 1237 (void)DPAA2_CMD_NI_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, ni_token)); 1238 close_rc: 1239 (void)DPAA2_CMD_RC_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, rc_token)); 1240 err_exit: 1241 return (error); 1242 } 1243 1244 static int 1245 dpaa2_ni_setup_tx_flow(device_t dev, struct dpaa2_ni_fq *fq) 1246 { 1247 device_t pdev = device_get_parent(dev); 1248 device_t child = dev; 1249 struct dpaa2_ni_softc *sc = device_get_softc(dev); 1250 struct dpaa2_channel *ch = fq->chan; 1251 struct dpaa2_devinfo *rcinfo = device_get_ivars(pdev); 1252 struct dpaa2_devinfo *dinfo = device_get_ivars(dev); 1253 struct dpaa2_devinfo *con_info; 1254 struct dpaa2_ni_queue_cfg queue_cfg = {0}; 1255 struct dpaa2_ni_tx_ring *tx; 1256 struct dpaa2_buf *buf; 1257 struct dpaa2_cmd cmd; 1258 uint32_t tx_rings_n = 0; 1259 uint16_t rc_token, ni_token; 1260 int error; 1261 1262 DPAA2_CMD_INIT(&cmd); 1263 1264 error = DPAA2_CMD_RC_OPEN(dev, child, &cmd, rcinfo->id, &rc_token); 1265 if (error) { 1266 device_printf(dev, "%s: failed to open resource container: " 1267 "id=%d, error=%d\n", __func__, rcinfo->id, error); 1268 goto err_exit; 1269 } 1270 error = DPAA2_CMD_NI_OPEN(dev, child, &cmd, dinfo->id, &ni_token); 1271 if (error) { 1272 device_printf(dev, "%s: failed to open network interface: " 1273 "id=%d, error=%d\n", __func__, dinfo->id, error); 1274 goto close_rc; 1275 } 1276 1277 /* Obtain DPCON associated with the FQ's channel. */ 1278 con_info = device_get_ivars(fq->chan->con_dev); 1279 1280 KASSERT(sc->attr.num.tx_tcs <= DPAA2_MAX_TCS, 1281 ("%s: too many Tx traffic classes: tx_tcs=%d\n", __func__, 1282 sc->attr.num.tx_tcs)); 1283 KASSERT(DPAA2_NI_BUFS_PER_TX <= DPAA2_NI_MAX_BPTX, 1284 ("%s: too many Tx buffers (%d): max=%d\n", __func__, 1285 DPAA2_NI_BUFS_PER_TX, DPAA2_NI_MAX_BPTX)); 1286 1287 /* Setup Tx rings. */ 1288 for (int i = 0; i < sc->attr.num.tx_tcs; i++) { 1289 queue_cfg.type = DPAA2_NI_QUEUE_TX; 1290 queue_cfg.tc = i; 1291 queue_cfg.idx = fq->flowid; 1292 queue_cfg.chan_id = fq->chan->id; 1293 1294 error = DPAA2_CMD_NI_GET_QUEUE(dev, child, &cmd, &queue_cfg); 1295 if (error) { 1296 device_printf(dev, "%s: failed to obtain Tx queue " 1297 "configuration: tc=%d, flowid=%d\n", __func__, 1298 queue_cfg.tc, queue_cfg.idx); 1299 goto close_ni; 1300 } 1301 1302 tx = &fq->tx_rings[i]; 1303 tx->fq = fq; 1304 tx->fqid = queue_cfg.fqid; 1305 tx->txid = tx_rings_n; 1306 1307 if (bootverbose) { 1308 device_printf(dev, "TX queue idx=%d, tc=%d, chan=%d, " 1309 "fqid=%d\n", fq->flowid, i, fq->chan->id, 1310 queue_cfg.fqid); 1311 } 1312 1313 mtx_init(&tx->lock, "dpaa2_tx_ring", NULL, MTX_DEF); 1314 1315 /* Allocate Tx ring buffer. */ 1316 tx->br = buf_ring_alloc(DPAA2_TX_BUFRING_SZ, M_DEVBUF, M_NOWAIT, 1317 &tx->lock); 1318 if (tx->br == NULL) { 1319 device_printf(dev, "%s: failed to setup Tx ring buffer" 1320 " (2) fqid=%d\n", __func__, tx->fqid); 1321 goto close_ni; 1322 } 1323 1324 /* Configure Tx buffers */ 1325 for (uint64_t j = 0; j < DPAA2_NI_BUFS_PER_TX; j++) { 1326 buf = malloc(sizeof(struct dpaa2_buf), M_DPAA2_TXB, 1327 M_WAITOK); 1328 /* Keep DMA tag and Tx ring linked to the buffer */ 1329 DPAA2_BUF_INIT_TAGOPT(buf, ch->tx_dmat, tx); 1330 1331 buf->sgt = malloc(sizeof(struct dpaa2_buf), M_DPAA2_TXB, 1332 M_WAITOK); 1333 /* Link SGT to DMA tag and back to its Tx buffer */ 1334 DPAA2_BUF_INIT_TAGOPT(buf->sgt, ch->sgt_dmat, buf); 1335 1336 error = dpaa2_buf_seed_txb(dev, buf); 1337 1338 /* Add Tx buffer to the ring */ 1339 buf_ring_enqueue(tx->br, buf); 1340 } 1341 1342 tx_rings_n++; 1343 } 1344 1345 /* All Tx queues which belong to the same flowid have the same qdbin. */ 1346 fq->tx_qdbin = queue_cfg.qdbin; 1347 1348 queue_cfg.type = DPAA2_NI_QUEUE_TX_CONF; 1349 queue_cfg.tc = 0; /* ignored for TxConf queue */ 1350 queue_cfg.idx = fq->flowid; 1351 error = DPAA2_CMD_NI_GET_QUEUE(dev, child, &cmd, &queue_cfg); 1352 if (error) { 1353 device_printf(dev, "%s: failed to obtain TxConf queue " 1354 "configuration: tc=%d, flowid=%d\n", __func__, queue_cfg.tc, 1355 queue_cfg.idx); 1356 goto close_ni; 1357 } 1358 1359 fq->fqid = queue_cfg.fqid; 1360 1361 queue_cfg.dest_id = con_info->id; 1362 queue_cfg.dest_type = DPAA2_NI_DEST_DPCON; 1363 queue_cfg.priority = 0; 1364 queue_cfg.user_ctx = (uint64_t)(uintmax_t) fq; 1365 queue_cfg.options = 1366 DPAA2_NI_QUEUE_OPT_USER_CTX | 1367 DPAA2_NI_QUEUE_OPT_DEST; 1368 error = DPAA2_CMD_NI_SET_QUEUE(dev, child, &cmd, &queue_cfg); 1369 if (error) { 1370 device_printf(dev, "%s: failed to update TxConf queue " 1371 "configuration: tc=%d, flowid=%d\n", __func__, queue_cfg.tc, 1372 queue_cfg.idx); 1373 goto close_ni; 1374 } 1375 1376 (void)DPAA2_CMD_NI_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, ni_token)); 1377 (void)DPAA2_CMD_RC_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, rc_token)); 1378 return (0); 1379 1380 close_ni: 1381 (void)DPAA2_CMD_NI_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, ni_token)); 1382 close_rc: 1383 (void)DPAA2_CMD_RC_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, rc_token)); 1384 err_exit: 1385 return (error); 1386 } 1387 1388 static int 1389 dpaa2_ni_setup_rx_err_flow(device_t dev, struct dpaa2_ni_fq *fq) 1390 { 1391 device_t pdev = device_get_parent(dev); 1392 device_t child = dev; 1393 struct dpaa2_devinfo *rcinfo = device_get_ivars(pdev); 1394 struct dpaa2_devinfo *dinfo = device_get_ivars(dev); 1395 struct dpaa2_devinfo *con_info; 1396 struct dpaa2_ni_queue_cfg queue_cfg = {0}; 1397 struct dpaa2_cmd cmd; 1398 uint16_t rc_token, ni_token; 1399 int error; 1400 1401 DPAA2_CMD_INIT(&cmd); 1402 1403 error = DPAA2_CMD_RC_OPEN(dev, child, &cmd, rcinfo->id, &rc_token); 1404 if (error) { 1405 device_printf(dev, "%s: failed to open resource container: " 1406 "id=%d, error=%d\n", __func__, rcinfo->id, error); 1407 goto err_exit; 1408 } 1409 error = DPAA2_CMD_NI_OPEN(dev, child, &cmd, dinfo->id, &ni_token); 1410 if (error) { 1411 device_printf(dev, "%s: failed to open network interface: " 1412 "id=%d, error=%d\n", __func__, dinfo->id, error); 1413 goto close_rc; 1414 } 1415 1416 /* Obtain DPCON associated with the FQ's channel. */ 1417 con_info = device_get_ivars(fq->chan->con_dev); 1418 1419 queue_cfg.type = DPAA2_NI_QUEUE_RX_ERR; 1420 queue_cfg.tc = fq->tc; /* ignored */ 1421 queue_cfg.idx = fq->flowid; /* ignored */ 1422 error = DPAA2_CMD_NI_GET_QUEUE(dev, child, &cmd, &queue_cfg); 1423 if (error) { 1424 device_printf(dev, "%s: failed to obtain RxErr queue " 1425 "configuration\n", __func__); 1426 goto close_ni; 1427 } 1428 1429 fq->fqid = queue_cfg.fqid; 1430 1431 queue_cfg.dest_id = con_info->id; 1432 queue_cfg.dest_type = DPAA2_NI_DEST_DPCON; 1433 queue_cfg.priority = 1; 1434 queue_cfg.user_ctx = (uint64_t)(uintmax_t) fq; 1435 queue_cfg.options = 1436 DPAA2_NI_QUEUE_OPT_USER_CTX | 1437 DPAA2_NI_QUEUE_OPT_DEST; 1438 error = DPAA2_CMD_NI_SET_QUEUE(dev, child, &cmd, &queue_cfg); 1439 if (error) { 1440 device_printf(dev, "%s: failed to update RxErr queue " 1441 "configuration\n", __func__); 1442 goto close_ni; 1443 } 1444 1445 (void)DPAA2_CMD_NI_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, ni_token)); 1446 (void)DPAA2_CMD_RC_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, rc_token)); 1447 return (0); 1448 1449 close_ni: 1450 (void)DPAA2_CMD_NI_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, ni_token)); 1451 close_rc: 1452 (void)DPAA2_CMD_RC_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, rc_token)); 1453 err_exit: 1454 return (error); 1455 } 1456 1457 /** 1458 * @brief Configure DPNI object to generate interrupts. 1459 */ 1460 static int 1461 dpaa2_ni_setup_irqs(device_t dev) 1462 { 1463 device_t pdev = device_get_parent(dev); 1464 device_t child = dev; 1465 struct dpaa2_ni_softc *sc = device_get_softc(dev); 1466 struct dpaa2_devinfo *rcinfo = device_get_ivars(pdev); 1467 struct dpaa2_devinfo *dinfo = device_get_ivars(dev); 1468 struct dpaa2_cmd cmd; 1469 uint16_t rc_token, ni_token; 1470 int error; 1471 1472 DPAA2_CMD_INIT(&cmd); 1473 1474 error = DPAA2_CMD_RC_OPEN(dev, child, &cmd, rcinfo->id, &rc_token); 1475 if (error) { 1476 device_printf(dev, "%s: failed to open resource container: " 1477 "id=%d, error=%d\n", __func__, rcinfo->id, error); 1478 goto err_exit; 1479 } 1480 error = DPAA2_CMD_NI_OPEN(dev, child, &cmd, dinfo->id, &ni_token); 1481 if (error) { 1482 device_printf(dev, "%s: failed to open network interface: " 1483 "id=%d, error=%d\n", __func__, dinfo->id, error); 1484 goto close_rc; 1485 } 1486 1487 /* Configure IRQs. */ 1488 error = dpaa2_ni_setup_msi(sc); 1489 if (error) { 1490 device_printf(dev, "%s: failed to allocate MSI\n", __func__); 1491 goto close_ni; 1492 } 1493 if ((sc->irq_res = bus_alloc_resource_any(dev, SYS_RES_IRQ, 1494 &sc->irq_rid[0], RF_ACTIVE | RF_SHAREABLE)) == NULL) { 1495 device_printf(dev, "%s: failed to allocate IRQ resource\n", 1496 __func__); 1497 goto close_ni; 1498 } 1499 if (bus_setup_intr(dev, sc->irq_res, INTR_TYPE_NET | INTR_MPSAFE, 1500 NULL, dpaa2_ni_intr, sc, &sc->intr)) { 1501 device_printf(dev, "%s: failed to setup IRQ resource\n", 1502 __func__); 1503 goto close_ni; 1504 } 1505 1506 error = DPAA2_CMD_NI_SET_IRQ_MASK(dev, child, &cmd, DPNI_IRQ_INDEX, 1507 DPNI_IRQ_LINK_CHANGED | DPNI_IRQ_EP_CHANGED); 1508 if (error) { 1509 device_printf(dev, "%s: failed to set DPNI IRQ mask\n", 1510 __func__); 1511 goto close_ni; 1512 } 1513 1514 error = DPAA2_CMD_NI_SET_IRQ_ENABLE(dev, child, &cmd, DPNI_IRQ_INDEX, 1515 true); 1516 if (error) { 1517 device_printf(dev, "%s: failed to enable DPNI IRQ\n", __func__); 1518 goto close_ni; 1519 } 1520 1521 (void)DPAA2_CMD_NI_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, ni_token)); 1522 (void)DPAA2_CMD_RC_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, rc_token)); 1523 return (0); 1524 1525 close_ni: 1526 (void)DPAA2_CMD_NI_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, ni_token)); 1527 close_rc: 1528 (void)DPAA2_CMD_RC_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, rc_token)); 1529 err_exit: 1530 return (error); 1531 } 1532 1533 /** 1534 * @brief Allocate MSI interrupts for DPNI. 1535 */ 1536 static int 1537 dpaa2_ni_setup_msi(struct dpaa2_ni_softc *sc) 1538 { 1539 int val; 1540 1541 val = pci_msi_count(sc->dev); 1542 if (val < DPAA2_NI_MSI_COUNT) 1543 device_printf(sc->dev, "MSI: actual=%d, expected=%d\n", val, 1544 DPAA2_IO_MSI_COUNT); 1545 val = MIN(val, DPAA2_NI_MSI_COUNT); 1546 1547 if (pci_alloc_msi(sc->dev, &val) != 0) 1548 return (EINVAL); 1549 1550 for (int i = 0; i < val; i++) 1551 sc->irq_rid[i] = i + 1; 1552 1553 return (0); 1554 } 1555 1556 /** 1557 * @brief Update DPNI according to the updated interface capabilities. 1558 */ 1559 static int 1560 dpaa2_ni_setup_if_caps(struct dpaa2_ni_softc *sc) 1561 { 1562 bool en_rxcsum, en_txcsum; 1563 device_t pdev = device_get_parent(sc->dev); 1564 device_t dev = sc->dev; 1565 device_t child = dev; 1566 struct dpaa2_devinfo *rcinfo = device_get_ivars(pdev); 1567 struct dpaa2_devinfo *dinfo = device_get_ivars(dev); 1568 struct dpaa2_cmd cmd; 1569 uint16_t rc_token, ni_token; 1570 int error; 1571 1572 DPAA2_CMD_INIT(&cmd); 1573 1574 /* 1575 * XXX-DSL: DPAA2 allows to validate L3/L4 checksums on reception and/or 1576 * generate L3/L4 checksums on transmission without 1577 * differentiating between IPv4/v6, i.e. enable for both 1578 * protocols if requested. 1579 */ 1580 en_rxcsum = if_getcapenable(sc->ifp) & 1581 (IFCAP_RXCSUM | IFCAP_RXCSUM_IPV6); 1582 en_txcsum = if_getcapenable(sc->ifp) & 1583 (IFCAP_TXCSUM | IFCAP_TXCSUM_IPV6); 1584 1585 error = DPAA2_CMD_RC_OPEN(dev, child, &cmd, rcinfo->id, &rc_token); 1586 if (error) { 1587 device_printf(dev, "%s: failed to open resource container: " 1588 "id=%d, error=%d\n", __func__, rcinfo->id, error); 1589 goto err_exit; 1590 } 1591 error = DPAA2_CMD_NI_OPEN(dev, child, &cmd, dinfo->id, &ni_token); 1592 if (error) { 1593 device_printf(dev, "%s: failed to open network interface: " 1594 "id=%d, error=%d\n", __func__, dinfo->id, error); 1595 goto close_rc; 1596 } 1597 1598 /* Setup checksums validation. */ 1599 error = DPAA2_CMD_NI_SET_OFFLOAD(dev, child, &cmd, 1600 DPAA2_NI_OFL_RX_L3_CSUM, en_rxcsum); 1601 if (error) { 1602 device_printf(dev, "%s: failed to %s L3 checksum validation\n", 1603 __func__, en_rxcsum ? "enable" : "disable"); 1604 goto close_ni; 1605 } 1606 error = DPAA2_CMD_NI_SET_OFFLOAD(dev, child, &cmd, 1607 DPAA2_NI_OFL_RX_L4_CSUM, en_rxcsum); 1608 if (error) { 1609 device_printf(dev, "%s: failed to %s L4 checksum validation\n", 1610 __func__, en_rxcsum ? "enable" : "disable"); 1611 goto close_ni; 1612 } 1613 1614 /* Setup checksums generation. */ 1615 error = DPAA2_CMD_NI_SET_OFFLOAD(dev, child, &cmd, 1616 DPAA2_NI_OFL_TX_L3_CSUM, en_txcsum); 1617 if (error) { 1618 device_printf(dev, "%s: failed to %s L3 checksum generation\n", 1619 __func__, en_txcsum ? "enable" : "disable"); 1620 goto close_ni; 1621 } 1622 error = DPAA2_CMD_NI_SET_OFFLOAD(dev, child, &cmd, 1623 DPAA2_NI_OFL_TX_L4_CSUM, en_txcsum); 1624 if (error) { 1625 device_printf(dev, "%s: failed to %s L4 checksum generation\n", 1626 __func__, en_txcsum ? "enable" : "disable"); 1627 goto close_ni; 1628 } 1629 1630 if (bootverbose) { 1631 device_printf(dev, "%s: L3/L4 checksum validation %s\n", 1632 __func__, en_rxcsum ? "enabled" : "disabled"); 1633 device_printf(dev, "%s: L3/L4 checksum generation %s\n", 1634 __func__, en_txcsum ? "enabled" : "disabled"); 1635 } 1636 1637 (void)DPAA2_CMD_NI_CLOSE(dev, child, &cmd); 1638 (void)DPAA2_CMD_RC_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, rc_token)); 1639 return (0); 1640 1641 close_ni: 1642 (void)DPAA2_CMD_NI_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, ni_token)); 1643 close_rc: 1644 (void)DPAA2_CMD_RC_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, rc_token)); 1645 err_exit: 1646 return (error); 1647 } 1648 1649 /** 1650 * @brief Update DPNI according to the updated interface flags. 1651 */ 1652 static int 1653 dpaa2_ni_setup_if_flags(struct dpaa2_ni_softc *sc) 1654 { 1655 const bool en_promisc = if_getflags(sc->ifp) & IFF_PROMISC; 1656 const bool en_allmulti = if_getflags(sc->ifp) & IFF_ALLMULTI; 1657 device_t pdev = device_get_parent(sc->dev); 1658 device_t dev = sc->dev; 1659 device_t child = dev; 1660 struct dpaa2_devinfo *rcinfo = device_get_ivars(pdev); 1661 struct dpaa2_devinfo *dinfo = device_get_ivars(dev); 1662 struct dpaa2_cmd cmd; 1663 uint16_t rc_token, ni_token; 1664 int error; 1665 1666 DPAA2_CMD_INIT(&cmd); 1667 1668 error = DPAA2_CMD_RC_OPEN(dev, child, &cmd, rcinfo->id, &rc_token); 1669 if (error) { 1670 device_printf(dev, "%s: failed to open resource container: " 1671 "id=%d, error=%d\n", __func__, rcinfo->id, error); 1672 goto err_exit; 1673 } 1674 error = DPAA2_CMD_NI_OPEN(dev, child, &cmd, dinfo->id, &ni_token); 1675 if (error) { 1676 device_printf(dev, "%s: failed to open network interface: " 1677 "id=%d, error=%d\n", __func__, dinfo->id, error); 1678 goto close_rc; 1679 } 1680 1681 error = DPAA2_CMD_NI_SET_MULTI_PROMISC(dev, child, &cmd, 1682 en_promisc ? true : en_allmulti); 1683 if (error) { 1684 device_printf(dev, "%s: failed to %s multicast promiscuous " 1685 "mode\n", __func__, en_allmulti ? "enable" : "disable"); 1686 goto close_ni; 1687 } 1688 1689 error = DPAA2_CMD_NI_SET_UNI_PROMISC(dev, child, &cmd, en_promisc); 1690 if (error) { 1691 device_printf(dev, "%s: failed to %s unicast promiscuous mode\n", 1692 __func__, en_promisc ? "enable" : "disable"); 1693 goto close_ni; 1694 } 1695 1696 (void)DPAA2_CMD_NI_CLOSE(dev, child, &cmd); 1697 (void)DPAA2_CMD_RC_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, rc_token)); 1698 return (0); 1699 1700 close_ni: 1701 (void)DPAA2_CMD_NI_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, ni_token)); 1702 close_rc: 1703 (void)DPAA2_CMD_RC_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, rc_token)); 1704 err_exit: 1705 return (error); 1706 } 1707 1708 static int 1709 dpaa2_ni_setup_sysctls(struct dpaa2_ni_softc *sc) 1710 { 1711 struct sysctl_ctx_list *ctx; 1712 struct sysctl_oid *node, *node2; 1713 struct sysctl_oid_list *parent, *parent2; 1714 char cbuf[128]; 1715 int i; 1716 1717 ctx = device_get_sysctl_ctx(sc->dev); 1718 parent = SYSCTL_CHILDREN(device_get_sysctl_tree(sc->dev)); 1719 1720 /* Add DPNI statistics. */ 1721 node = SYSCTL_ADD_NODE(ctx, parent, OID_AUTO, "stats", 1722 CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "DPNI Statistics"); 1723 parent = SYSCTL_CHILDREN(node); 1724 for (i = 0; i < DPAA2_NI_STAT_SYSCTLS; ++i) { 1725 SYSCTL_ADD_PROC(ctx, parent, i, dpni_stat_sysctls[i].name, 1726 CTLTYPE_U64 | CTLFLAG_RD, sc, 0, dpaa2_ni_collect_stats, 1727 "IU", dpni_stat_sysctls[i].desc); 1728 } 1729 SYSCTL_ADD_UQUAD(ctx, parent, OID_AUTO, "rx_anomaly_frames", 1730 CTLFLAG_RD, &sc->rx_anomaly_frames, 1731 "Rx frames in the buffers outside of the buffer pools"); 1732 SYSCTL_ADD_UQUAD(ctx, parent, OID_AUTO, "rx_single_buf_frames", 1733 CTLFLAG_RD, &sc->rx_single_buf_frames, 1734 "Rx frames in single buffers"); 1735 SYSCTL_ADD_UQUAD(ctx, parent, OID_AUTO, "rx_sg_buf_frames", 1736 CTLFLAG_RD, &sc->rx_sg_buf_frames, 1737 "Rx frames in scatter/gather list"); 1738 SYSCTL_ADD_UQUAD(ctx, parent, OID_AUTO, "rx_enq_rej_frames", 1739 CTLFLAG_RD, &sc->rx_enq_rej_frames, 1740 "Enqueue rejected by QMan"); 1741 SYSCTL_ADD_UQUAD(ctx, parent, OID_AUTO, "rx_ieoi_err_frames", 1742 CTLFLAG_RD, &sc->rx_ieoi_err_frames, 1743 "QMan IEOI error"); 1744 SYSCTL_ADD_UQUAD(ctx, parent, OID_AUTO, "tx_single_buf_frames", 1745 CTLFLAG_RD, &sc->tx_single_buf_frames, 1746 "Tx single buffer frames"); 1747 SYSCTL_ADD_UQUAD(ctx, parent, OID_AUTO, "tx_sg_frames", 1748 CTLFLAG_RD, &sc->tx_sg_frames, 1749 "Tx S/G frames"); 1750 1751 SYSCTL_ADD_PROC(ctx, parent, OID_AUTO, "buf_num", 1752 CTLTYPE_U32 | CTLFLAG_RD, sc, 0, dpaa2_ni_collect_buf_num, 1753 "IU", "number of Rx buffers in the buffer pool"); 1754 SYSCTL_ADD_PROC(ctx, parent, OID_AUTO, "buf_free", 1755 CTLTYPE_U32 | CTLFLAG_RD, sc, 0, dpaa2_ni_collect_buf_free, 1756 "IU", "number of free Rx buffers in the buffer pool"); 1757 1758 parent = SYSCTL_CHILDREN(device_get_sysctl_tree(sc->dev)); 1759 1760 /* Add channels statistics. */ 1761 node = SYSCTL_ADD_NODE(ctx, parent, OID_AUTO, "channels", 1762 CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "DPNI Channels"); 1763 parent = SYSCTL_CHILDREN(node); 1764 for (int i = 0; i < sc->chan_n; i++) { 1765 snprintf(cbuf, sizeof(cbuf), "%d", i); 1766 1767 node2 = SYSCTL_ADD_NODE(ctx, parent, OID_AUTO, cbuf, 1768 CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "DPNI Channel"); 1769 parent2 = SYSCTL_CHILDREN(node2); 1770 1771 SYSCTL_ADD_UQUAD(ctx, parent2, OID_AUTO, "tx_frames", 1772 CTLFLAG_RD, &sc->channels[i]->tx_frames, 1773 "Tx frames counter"); 1774 SYSCTL_ADD_UQUAD(ctx, parent2, OID_AUTO, "tx_dropped", 1775 CTLFLAG_RD, &sc->channels[i]->tx_dropped, 1776 "Tx dropped counter"); 1777 } 1778 1779 return (0); 1780 } 1781 1782 static int 1783 dpaa2_ni_setup_dma(struct dpaa2_ni_softc *sc) 1784 { 1785 device_t dev = sc->dev; 1786 int error; 1787 1788 KASSERT((sc->buf_align == BUF_ALIGN) || (sc->buf_align == BUF_ALIGN_V1), 1789 ("unexpected buffer alignment: %d\n", sc->buf_align)); 1790 1791 /* DMA tag for Rx distribution key. */ 1792 error = bus_dma_tag_create( 1793 bus_get_dma_tag(dev), 1794 PAGE_SIZE, 0, /* alignment, boundary */ 1795 BUS_SPACE_MAXADDR, /* low restricted addr */ 1796 BUS_SPACE_MAXADDR, /* high restricted addr */ 1797 NULL, NULL, /* filter, filterarg */ 1798 DPAA2_CLASSIFIER_DMA_SIZE, 1, /* maxsize, nsegments */ 1799 DPAA2_CLASSIFIER_DMA_SIZE, 0, /* maxsegsize, flags */ 1800 NULL, NULL, /* lockfunc, lockarg */ 1801 &sc->rxd_dmat); 1802 if (error) { 1803 device_printf(dev, "%s: failed to create DMA tag for Rx " 1804 "distribution key\n", __func__); 1805 return (error); 1806 } 1807 1808 error = bus_dma_tag_create( 1809 bus_get_dma_tag(dev), 1810 PAGE_SIZE, 0, /* alignment, boundary */ 1811 BUS_SPACE_MAXADDR, /* low restricted addr */ 1812 BUS_SPACE_MAXADDR, /* high restricted addr */ 1813 NULL, NULL, /* filter, filterarg */ 1814 ETH_QOS_KCFG_BUF_SIZE, 1, /* maxsize, nsegments */ 1815 ETH_QOS_KCFG_BUF_SIZE, 0, /* maxsegsize, flags */ 1816 NULL, NULL, /* lockfunc, lockarg */ 1817 &sc->qos_dmat); 1818 if (error) { 1819 device_printf(dev, "%s: failed to create DMA tag for QoS key\n", 1820 __func__); 1821 return (error); 1822 } 1823 1824 return (0); 1825 } 1826 1827 /** 1828 * @brief Configure buffer layouts of the different DPNI queues. 1829 */ 1830 static int 1831 dpaa2_ni_set_buf_layout(device_t dev) 1832 { 1833 device_t pdev = device_get_parent(dev); 1834 device_t child = dev; 1835 struct dpaa2_devinfo *rcinfo = device_get_ivars(pdev); 1836 struct dpaa2_devinfo *dinfo = device_get_ivars(dev); 1837 struct dpaa2_ni_softc *sc = device_get_softc(dev); 1838 struct dpaa2_ni_buf_layout buf_layout = {0}; 1839 struct dpaa2_cmd cmd; 1840 uint16_t rc_token, ni_token; 1841 int error; 1842 1843 DPAA2_CMD_INIT(&cmd); 1844 1845 error = DPAA2_CMD_RC_OPEN(dev, child, &cmd, rcinfo->id, &rc_token); 1846 if (error) { 1847 device_printf(dev, "%s: failed to open resource container: " 1848 "id=%d, error=%d\n", __func__, rcinfo->id, error); 1849 goto err_exit; 1850 } 1851 error = DPAA2_CMD_NI_OPEN(dev, child, &cmd, dinfo->id, &ni_token); 1852 if (error) { 1853 device_printf(sc->dev, "%s: failed to open DPMAC: id=%d, " 1854 "error=%d\n", __func__, dinfo->id, error); 1855 goto close_rc; 1856 } 1857 1858 /* 1859 * Select Rx/Tx buffer alignment. It's necessary to ensure that the 1860 * buffer size seen by WRIOP is a multiple of 64 or 256 bytes depending 1861 * on the WRIOP version. 1862 */ 1863 sc->buf_align = (sc->attr.wriop_ver == WRIOP_VERSION(0, 0, 0) || 1864 sc->attr.wriop_ver == WRIOP_VERSION(1, 0, 0)) 1865 ? BUF_ALIGN_V1 : BUF_ALIGN; 1866 1867 /* 1868 * We need to ensure that the buffer size seen by WRIOP is a multiple 1869 * of 64 or 256 bytes depending on the WRIOP version. 1870 */ 1871 sc->buf_sz = ALIGN_DOWN(DPAA2_RX_BUF_SIZE, sc->buf_align); 1872 1873 if (bootverbose) { 1874 device_printf(dev, "Rx/Tx buffers: size=%d, alignment=%d\n", 1875 sc->buf_sz, sc->buf_align); 1876 } 1877 1878 /* 1879 * Frame Descriptor Tx buffer layout 1880 * 1881 * ADDR -> |---------------------| 1882 * | SW FRAME ANNOTATION | BUF_SWA_SIZE bytes 1883 * |---------------------| 1884 * | HW FRAME ANNOTATION | BUF_TX_HWA_SIZE bytes 1885 * |---------------------| 1886 * | DATA HEADROOM | 1887 * ADDR + OFFSET -> |---------------------| 1888 * | | 1889 * | | 1890 * | FRAME DATA | 1891 * | | 1892 * | | 1893 * |---------------------| 1894 * | DATA TAILROOM | 1895 * |---------------------| 1896 * 1897 * NOTE: It's for a single buffer frame only. 1898 */ 1899 buf_layout.queue_type = DPAA2_NI_QUEUE_TX; 1900 buf_layout.pd_size = BUF_SWA_SIZE; 1901 buf_layout.pass_timestamp = true; 1902 buf_layout.pass_frame_status = true; 1903 buf_layout.options = 1904 BUF_LOPT_PRIV_DATA_SZ | 1905 BUF_LOPT_TIMESTAMP | /* requires 128 bytes in HWA */ 1906 BUF_LOPT_FRAME_STATUS; 1907 error = DPAA2_CMD_NI_SET_BUF_LAYOUT(dev, child, &cmd, &buf_layout); 1908 if (error) { 1909 device_printf(dev, "%s: failed to set Tx buffer layout\n", 1910 __func__); 1911 goto close_ni; 1912 } 1913 1914 /* Tx-confirmation buffer layout */ 1915 buf_layout.queue_type = DPAA2_NI_QUEUE_TX_CONF; 1916 buf_layout.options = 1917 BUF_LOPT_TIMESTAMP | 1918 BUF_LOPT_FRAME_STATUS; 1919 error = DPAA2_CMD_NI_SET_BUF_LAYOUT(dev, child, &cmd, &buf_layout); 1920 if (error) { 1921 device_printf(dev, "%s: failed to set TxConf buffer layout\n", 1922 __func__); 1923 goto close_ni; 1924 } 1925 1926 /* 1927 * Driver should reserve the amount of space indicated by this command 1928 * as headroom in all Tx frames. 1929 */ 1930 error = DPAA2_CMD_NI_GET_TX_DATA_OFF(dev, child, &cmd, &sc->tx_data_off); 1931 if (error) { 1932 device_printf(dev, "%s: failed to obtain Tx data offset\n", 1933 __func__); 1934 goto close_ni; 1935 } 1936 1937 if (bootverbose) { 1938 device_printf(dev, "Tx data offset=%d\n", sc->tx_data_off); 1939 } 1940 if ((sc->tx_data_off % 64) != 0) { 1941 device_printf(dev, "Tx data offset (%d) is not a multiplication " 1942 "of 64 bytes\n", sc->tx_data_off); 1943 } 1944 1945 /* 1946 * Frame Descriptor Rx buffer layout 1947 * 1948 * ADDR -> |---------------------| 1949 * | SW FRAME ANNOTATION | BUF_SWA_SIZE bytes 1950 * |---------------------| 1951 * | HW FRAME ANNOTATION | BUF_RX_HWA_SIZE bytes 1952 * |---------------------| 1953 * | DATA HEADROOM | OFFSET-BUF_RX_HWA_SIZE 1954 * ADDR + OFFSET -> |---------------------| 1955 * | | 1956 * | | 1957 * | FRAME DATA | 1958 * | | 1959 * | | 1960 * |---------------------| 1961 * | DATA TAILROOM | 0 bytes 1962 * |---------------------| 1963 * 1964 * NOTE: It's for a single buffer frame only. 1965 */ 1966 buf_layout.queue_type = DPAA2_NI_QUEUE_RX; 1967 buf_layout.pd_size = BUF_SWA_SIZE; 1968 buf_layout.fd_align = sc->buf_align; 1969 buf_layout.head_size = sc->tx_data_off - BUF_RX_HWA_SIZE - BUF_SWA_SIZE; 1970 buf_layout.tail_size = 0; 1971 buf_layout.pass_frame_status = true; 1972 buf_layout.pass_parser_result = true; 1973 buf_layout.pass_timestamp = true; 1974 buf_layout.options = 1975 BUF_LOPT_PRIV_DATA_SZ | 1976 BUF_LOPT_DATA_ALIGN | 1977 BUF_LOPT_DATA_HEAD_ROOM | 1978 BUF_LOPT_DATA_TAIL_ROOM | 1979 BUF_LOPT_FRAME_STATUS | 1980 BUF_LOPT_PARSER_RESULT | 1981 BUF_LOPT_TIMESTAMP; 1982 error = DPAA2_CMD_NI_SET_BUF_LAYOUT(dev, child, &cmd, &buf_layout); 1983 if (error) { 1984 device_printf(dev, "%s: failed to set Rx buffer layout\n", 1985 __func__); 1986 goto close_ni; 1987 } 1988 1989 error = 0; 1990 close_ni: 1991 (void)DPAA2_CMD_NI_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, ni_token)); 1992 close_rc: 1993 (void)DPAA2_CMD_RC_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, rc_token)); 1994 err_exit: 1995 return (error); 1996 } 1997 1998 /** 1999 * @brief Enable Rx/Tx pause frames. 2000 * 2001 * NOTE: DPNI stops sending when a pause frame is received (Rx frame) or DPNI 2002 * itself generates pause frames (Tx frame). 2003 */ 2004 static int 2005 dpaa2_ni_set_pause_frame(device_t dev) 2006 { 2007 device_t pdev = device_get_parent(dev); 2008 device_t child = dev; 2009 struct dpaa2_devinfo *rcinfo = device_get_ivars(pdev); 2010 struct dpaa2_devinfo *dinfo = device_get_ivars(dev); 2011 struct dpaa2_ni_softc *sc = device_get_softc(dev); 2012 struct dpaa2_ni_link_cfg link_cfg = {0}; 2013 struct dpaa2_cmd cmd; 2014 uint16_t rc_token, ni_token; 2015 int error; 2016 2017 DPAA2_CMD_INIT(&cmd); 2018 2019 error = DPAA2_CMD_RC_OPEN(dev, child, &cmd, rcinfo->id, &rc_token); 2020 if (error) { 2021 device_printf(dev, "%s: failed to open resource container: " 2022 "id=%d, error=%d\n", __func__, rcinfo->id, error); 2023 goto err_exit; 2024 } 2025 error = DPAA2_CMD_NI_OPEN(dev, child, &cmd, dinfo->id, &ni_token); 2026 if (error) { 2027 device_printf(sc->dev, "%s: failed to open DPMAC: id=%d, " 2028 "error=%d\n", __func__, dinfo->id, error); 2029 goto close_rc; 2030 } 2031 2032 error = DPAA2_CMD_NI_GET_LINK_CFG(dev, child, &cmd, &link_cfg); 2033 if (error) { 2034 device_printf(dev, "%s: failed to obtain link configuration: " 2035 "error=%d\n", __func__, error); 2036 goto close_ni; 2037 } 2038 2039 /* Enable both Rx and Tx pause frames by default. */ 2040 link_cfg.options |= DPAA2_NI_LINK_OPT_PAUSE; 2041 link_cfg.options &= ~DPAA2_NI_LINK_OPT_ASYM_PAUSE; 2042 2043 error = DPAA2_CMD_NI_SET_LINK_CFG(dev, child, &cmd, &link_cfg); 2044 if (error) { 2045 device_printf(dev, "%s: failed to set link configuration: " 2046 "error=%d\n", __func__, error); 2047 goto close_ni; 2048 } 2049 2050 sc->link_options = link_cfg.options; 2051 error = 0; 2052 close_ni: 2053 (void)DPAA2_CMD_NI_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, ni_token)); 2054 close_rc: 2055 (void)DPAA2_CMD_RC_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, rc_token)); 2056 err_exit: 2057 return (error); 2058 } 2059 2060 /** 2061 * @brief Configure QoS table to determine the traffic class for the received 2062 * frame. 2063 */ 2064 static int 2065 dpaa2_ni_set_qos_table(device_t dev) 2066 { 2067 device_t pdev = device_get_parent(dev); 2068 device_t child = dev; 2069 struct dpaa2_devinfo *rcinfo = device_get_ivars(pdev); 2070 struct dpaa2_devinfo *dinfo = device_get_ivars(dev); 2071 struct dpaa2_ni_softc *sc = device_get_softc(dev); 2072 struct dpaa2_ni_qos_table tbl; 2073 struct dpaa2_buf *buf = &sc->qos_kcfg; 2074 struct dpaa2_cmd cmd; 2075 uint16_t rc_token, ni_token; 2076 int error; 2077 2078 if (sc->attr.num.rx_tcs == 1 || 2079 !(sc->attr.options & DPNI_OPT_HAS_KEY_MASKING)) { 2080 if (bootverbose) { 2081 device_printf(dev, "Ingress traffic classification is " 2082 "not supported\n"); 2083 } 2084 return (0); 2085 } 2086 2087 /* 2088 * Allocate a buffer visible to the device to hold the QoS table key 2089 * configuration. 2090 */ 2091 2092 if (__predict_true(buf->dmat == NULL)) { 2093 buf->dmat = sc->qos_dmat; 2094 } 2095 2096 error = bus_dmamem_alloc(buf->dmat, (void **)&buf->vaddr, 2097 BUS_DMA_ZERO | BUS_DMA_COHERENT, &buf->dmap); 2098 if (error) { 2099 device_printf(dev, "%s: failed to allocate a buffer for QoS key " 2100 "configuration\n", __func__); 2101 goto err_exit; 2102 } 2103 2104 error = bus_dmamap_load(buf->dmat, buf->dmap, buf->vaddr, 2105 ETH_QOS_KCFG_BUF_SIZE, dpaa2_dmamap_oneseg_cb, &buf->paddr, 2106 BUS_DMA_NOWAIT); 2107 if (error) { 2108 device_printf(dev, "%s: failed to map QoS key configuration " 2109 "buffer into bus space\n", __func__); 2110 goto err_exit; 2111 } 2112 2113 DPAA2_CMD_INIT(&cmd); 2114 2115 error = DPAA2_CMD_RC_OPEN(dev, child, &cmd, rcinfo->id, &rc_token); 2116 if (error) { 2117 device_printf(dev, "%s: failed to open resource container: " 2118 "id=%d, error=%d\n", __func__, rcinfo->id, error); 2119 goto err_exit; 2120 } 2121 error = DPAA2_CMD_NI_OPEN(dev, child, &cmd, dinfo->id, &ni_token); 2122 if (error) { 2123 device_printf(sc->dev, "%s: failed to open DPMAC: id=%d, " 2124 "error=%d\n", __func__, dinfo->id, error); 2125 goto close_rc; 2126 } 2127 2128 tbl.default_tc = 0; 2129 tbl.discard_on_miss = false; 2130 tbl.keep_entries = false; 2131 tbl.kcfg_busaddr = buf->paddr; 2132 error = DPAA2_CMD_NI_SET_QOS_TABLE(dev, child, &cmd, &tbl); 2133 if (error) { 2134 device_printf(dev, "%s: failed to set QoS table\n", __func__); 2135 goto close_ni; 2136 } 2137 2138 error = DPAA2_CMD_NI_CLEAR_QOS_TABLE(dev, child, &cmd); 2139 if (error) { 2140 device_printf(dev, "%s: failed to clear QoS table\n", __func__); 2141 goto close_ni; 2142 } 2143 2144 error = 0; 2145 close_ni: 2146 (void)DPAA2_CMD_NI_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, ni_token)); 2147 close_rc: 2148 (void)DPAA2_CMD_RC_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, rc_token)); 2149 err_exit: 2150 return (error); 2151 } 2152 2153 static int 2154 dpaa2_ni_set_mac_addr(device_t dev) 2155 { 2156 device_t pdev = device_get_parent(dev); 2157 device_t child = dev; 2158 struct dpaa2_ni_softc *sc = device_get_softc(dev); 2159 if_t ifp = sc->ifp; 2160 struct dpaa2_devinfo *rcinfo = device_get_ivars(pdev); 2161 struct dpaa2_devinfo *dinfo = device_get_ivars(dev); 2162 struct dpaa2_cmd cmd; 2163 struct ether_addr rnd_mac_addr; 2164 uint16_t rc_token, ni_token; 2165 uint8_t mac_addr[ETHER_ADDR_LEN]; 2166 uint8_t dpni_mac_addr[ETHER_ADDR_LEN]; 2167 int error; 2168 2169 DPAA2_CMD_INIT(&cmd); 2170 2171 error = DPAA2_CMD_RC_OPEN(dev, child, &cmd, rcinfo->id, &rc_token); 2172 if (error) { 2173 device_printf(dev, "%s: failed to open resource container: " 2174 "id=%d, error=%d\n", __func__, rcinfo->id, error); 2175 goto err_exit; 2176 } 2177 error = DPAA2_CMD_NI_OPEN(dev, child, &cmd, dinfo->id, &ni_token); 2178 if (error) { 2179 device_printf(sc->dev, "%s: failed to open DPMAC: id=%d, " 2180 "error=%d\n", __func__, dinfo->id, error); 2181 goto close_rc; 2182 } 2183 2184 /* 2185 * Get the MAC address associated with the physical port, if the DPNI is 2186 * connected to a DPMAC directly associated with one of the physical 2187 * ports. 2188 */ 2189 error = DPAA2_CMD_NI_GET_PORT_MAC_ADDR(dev, child, &cmd, mac_addr); 2190 if (error) { 2191 device_printf(dev, "%s: failed to obtain the MAC address " 2192 "associated with the physical port\n", __func__); 2193 goto close_ni; 2194 } 2195 2196 /* Get primary MAC address from the DPNI attributes. */ 2197 error = DPAA2_CMD_NI_GET_PRIM_MAC_ADDR(dev, child, &cmd, dpni_mac_addr); 2198 if (error) { 2199 device_printf(dev, "%s: failed to obtain primary MAC address\n", 2200 __func__); 2201 goto close_ni; 2202 } 2203 2204 if (!ETHER_IS_ZERO(mac_addr)) { 2205 /* Set MAC address of the physical port as DPNI's primary one. */ 2206 error = DPAA2_CMD_NI_SET_PRIM_MAC_ADDR(dev, child, &cmd, 2207 mac_addr); 2208 if (error) { 2209 device_printf(dev, "%s: failed to set primary MAC " 2210 "address\n", __func__); 2211 goto close_ni; 2212 } 2213 for (int i = 0; i < ETHER_ADDR_LEN; i++) { 2214 sc->mac.addr[i] = mac_addr[i]; 2215 } 2216 } else if (ETHER_IS_ZERO(dpni_mac_addr)) { 2217 /* Generate random MAC address as DPNI's primary one. */ 2218 ether_gen_addr(ifp, &rnd_mac_addr); 2219 for (int i = 0; i < ETHER_ADDR_LEN; i++) { 2220 mac_addr[i] = rnd_mac_addr.octet[i]; 2221 } 2222 2223 error = DPAA2_CMD_NI_SET_PRIM_MAC_ADDR(dev, child, &cmd, 2224 mac_addr); 2225 if (error) { 2226 device_printf(dev, "%s: failed to set random primary " 2227 "MAC address\n", __func__); 2228 goto close_ni; 2229 } 2230 for (int i = 0; i < ETHER_ADDR_LEN; i++) { 2231 sc->mac.addr[i] = mac_addr[i]; 2232 } 2233 } else { 2234 for (int i = 0; i < ETHER_ADDR_LEN; i++) { 2235 sc->mac.addr[i] = dpni_mac_addr[i]; 2236 } 2237 } 2238 2239 error = 0; 2240 close_ni: 2241 (void)DPAA2_CMD_NI_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, ni_token)); 2242 close_rc: 2243 (void)DPAA2_CMD_RC_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, rc_token)); 2244 err_exit: 2245 return (error); 2246 } 2247 2248 static void 2249 dpaa2_ni_miibus_statchg(device_t dev) 2250 { 2251 device_t pdev = device_get_parent(dev); 2252 device_t child = dev; 2253 struct dpaa2_ni_softc *sc = device_get_softc(dev); 2254 struct dpaa2_mac_link_state mac_link = { 0 }; 2255 struct dpaa2_devinfo *rcinfo = device_get_ivars(pdev); 2256 struct dpaa2_cmd cmd; 2257 uint16_t rc_token, mac_token; 2258 int error, link_state; 2259 2260 if (sc->fixed_link || sc->mii == NULL) { 2261 return; 2262 } 2263 if ((if_getdrvflags(sc->ifp) & IFF_DRV_RUNNING) == 0) { 2264 /* 2265 * We will receive calls and adjust the changes but 2266 * not have setup everything (called before dpaa2_ni_init() 2267 * really). This will then setup the link and internal 2268 * sc->link_state and not trigger the update once needed, 2269 * so basically dpmac never knows about it. 2270 */ 2271 return; 2272 } 2273 2274 /* 2275 * Note: ifp link state will only be changed AFTER we are called so we 2276 * cannot rely on ifp->if_linkstate here. 2277 */ 2278 if (sc->mii->mii_media_status & IFM_AVALID) { 2279 if (sc->mii->mii_media_status & IFM_ACTIVE) { 2280 link_state = LINK_STATE_UP; 2281 } else { 2282 link_state = LINK_STATE_DOWN; 2283 } 2284 } else { 2285 link_state = LINK_STATE_UNKNOWN; 2286 } 2287 2288 if (link_state != sc->link_state) { 2289 sc->link_state = link_state; 2290 2291 DPAA2_CMD_INIT(&cmd); 2292 2293 error = DPAA2_CMD_RC_OPEN(dev, child, &cmd, rcinfo->id, 2294 &rc_token); 2295 if (error) { 2296 device_printf(dev, "%s: failed to open resource " 2297 "container: id=%d, error=%d\n", __func__, rcinfo->id, 2298 error); 2299 goto err_exit; 2300 } 2301 error = DPAA2_CMD_MAC_OPEN(dev, child, &cmd, sc->mac.dpmac_id, 2302 &mac_token); 2303 if (error) { 2304 device_printf(sc->dev, "%s: failed to open DPMAC: " 2305 "id=%d, error=%d\n", __func__, sc->mac.dpmac_id, 2306 error); 2307 goto close_rc; 2308 } 2309 2310 if (link_state == LINK_STATE_UP || 2311 link_state == LINK_STATE_DOWN) { 2312 /* Update DPMAC link state. */ 2313 mac_link.supported = sc->mii->mii_media.ifm_media; 2314 mac_link.advert = sc->mii->mii_media.ifm_media; 2315 mac_link.rate = 1000; /* TODO: Where to get from? */ /* ifmedia_baudrate? */ 2316 mac_link.options = 2317 DPAA2_MAC_LINK_OPT_AUTONEG | 2318 DPAA2_MAC_LINK_OPT_PAUSE; 2319 mac_link.up = (link_state == LINK_STATE_UP) ? true : false; 2320 mac_link.state_valid = true; 2321 2322 /* Inform DPMAC about link state. */ 2323 error = DPAA2_CMD_MAC_SET_LINK_STATE(dev, child, &cmd, 2324 &mac_link); 2325 if (error) { 2326 device_printf(sc->dev, "%s: failed to set DPMAC " 2327 "link state: id=%d, error=%d\n", __func__, 2328 sc->mac.dpmac_id, error); 2329 } 2330 } 2331 (void)DPAA2_CMD_MAC_CLOSE(dev, child, &cmd); 2332 (void)DPAA2_CMD_RC_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, 2333 rc_token)); 2334 } 2335 2336 return; 2337 2338 close_rc: 2339 (void)DPAA2_CMD_RC_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, rc_token)); 2340 err_exit: 2341 return; 2342 } 2343 2344 /** 2345 * @brief Callback function to process media change request. 2346 */ 2347 static int 2348 dpaa2_ni_media_change_locked(struct dpaa2_ni_softc *sc) 2349 { 2350 2351 DPNI_LOCK_ASSERT(sc); 2352 if (sc->mii) { 2353 mii_mediachg(sc->mii); 2354 sc->media_status = sc->mii->mii_media.ifm_media; 2355 } else if (sc->fixed_link) { 2356 if_printf(sc->ifp, "%s: can't change media in fixed mode\n", 2357 __func__); 2358 } 2359 2360 return (0); 2361 } 2362 2363 static int 2364 dpaa2_ni_media_change(if_t ifp) 2365 { 2366 struct dpaa2_ni_softc *sc = if_getsoftc(ifp); 2367 int error; 2368 2369 DPNI_LOCK(sc); 2370 error = dpaa2_ni_media_change_locked(sc); 2371 DPNI_UNLOCK(sc); 2372 return (error); 2373 } 2374 2375 /** 2376 * @brief Callback function to process media status request. 2377 */ 2378 static void 2379 dpaa2_ni_media_status(if_t ifp, struct ifmediareq *ifmr) 2380 { 2381 struct dpaa2_ni_softc *sc = if_getsoftc(ifp); 2382 2383 DPNI_LOCK(sc); 2384 if (sc->mii) { 2385 mii_pollstat(sc->mii); 2386 ifmr->ifm_active = sc->mii->mii_media_active; 2387 ifmr->ifm_status = sc->mii->mii_media_status; 2388 } 2389 DPNI_UNLOCK(sc); 2390 } 2391 2392 /** 2393 * @brief Callout function to check and update media status. 2394 */ 2395 static void 2396 dpaa2_ni_media_tick(void *arg) 2397 { 2398 struct dpaa2_ni_softc *sc = (struct dpaa2_ni_softc *) arg; 2399 2400 /* Check for media type change */ 2401 if (sc->mii) { 2402 mii_tick(sc->mii); 2403 if (sc->media_status != sc->mii->mii_media.ifm_media) { 2404 printf("%s: media type changed (ifm_media=%x)\n", 2405 __func__, sc->mii->mii_media.ifm_media); 2406 dpaa2_ni_media_change(sc->ifp); 2407 } 2408 } 2409 2410 /* Schedule another timeout one second from now */ 2411 callout_reset(&sc->mii_callout, hz, dpaa2_ni_media_tick, sc); 2412 } 2413 2414 static void 2415 dpaa2_ni_init(void *arg) 2416 { 2417 struct dpaa2_ni_softc *sc = (struct dpaa2_ni_softc *) arg; 2418 if_t ifp = sc->ifp; 2419 device_t pdev = device_get_parent(sc->dev); 2420 device_t dev = sc->dev; 2421 device_t child = dev; 2422 struct dpaa2_devinfo *rcinfo = device_get_ivars(pdev); 2423 struct dpaa2_devinfo *dinfo = device_get_ivars(dev); 2424 struct dpaa2_cmd cmd; 2425 uint16_t rc_token, ni_token; 2426 int error; 2427 2428 DPNI_LOCK(sc); 2429 if ((if_getdrvflags(ifp) & IFF_DRV_RUNNING) != 0) { 2430 DPNI_UNLOCK(sc); 2431 return; 2432 } 2433 DPNI_UNLOCK(sc); 2434 2435 DPAA2_CMD_INIT(&cmd); 2436 2437 error = DPAA2_CMD_RC_OPEN(dev, child, &cmd, rcinfo->id, &rc_token); 2438 if (error) { 2439 device_printf(dev, "%s: failed to open resource container: " 2440 "id=%d, error=%d\n", __func__, rcinfo->id, error); 2441 goto err_exit; 2442 } 2443 error = DPAA2_CMD_NI_OPEN(dev, child, &cmd, dinfo->id, &ni_token); 2444 if (error) { 2445 device_printf(dev, "%s: failed to open network interface: " 2446 "id=%d, error=%d\n", __func__, dinfo->id, error); 2447 goto close_rc; 2448 } 2449 2450 error = DPAA2_CMD_NI_ENABLE(dev, child, &cmd); 2451 if (error) { 2452 device_printf(dev, "%s: failed to enable DPNI: error=%d\n", 2453 __func__, error); 2454 } 2455 2456 DPNI_LOCK(sc); 2457 /* Announce we are up and running and can queue packets. */ 2458 if_setdrvflagbits(ifp, IFF_DRV_RUNNING, IFF_DRV_OACTIVE); 2459 2460 if (sc->mii) { 2461 /* 2462 * mii_mediachg() will trigger a call into 2463 * dpaa2_ni_miibus_statchg() to setup link state. 2464 */ 2465 dpaa2_ni_media_change_locked(sc); 2466 } 2467 callout_reset(&sc->mii_callout, hz, dpaa2_ni_media_tick, sc); 2468 2469 DPNI_UNLOCK(sc); 2470 2471 (void)DPAA2_CMD_NI_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, ni_token)); 2472 (void)DPAA2_CMD_RC_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, rc_token)); 2473 return; 2474 2475 close_rc: 2476 (void)DPAA2_CMD_RC_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, rc_token)); 2477 err_exit: 2478 return; 2479 } 2480 2481 static int 2482 dpaa2_ni_transmit(if_t ifp, struct mbuf *m) 2483 { 2484 struct dpaa2_ni_softc *sc = if_getsoftc(ifp); 2485 struct dpaa2_channel *ch; 2486 uint32_t fqid; 2487 bool found = false; 2488 int chidx = 0, error; 2489 2490 if (__predict_false(!(if_getdrvflags(ifp) & IFF_DRV_RUNNING))) { 2491 return (0); 2492 } 2493 2494 if (M_HASHTYPE_GET(m) != M_HASHTYPE_NONE) { 2495 fqid = m->m_pkthdr.flowid; 2496 for (int i = 0; i < sc->chan_n; i++) { 2497 ch = sc->channels[i]; 2498 for (int j = 0; j < ch->rxq_n; j++) { 2499 if (fqid == ch->rx_queues[j].fqid) { 2500 chidx = ch->flowid; 2501 found = true; 2502 break; 2503 } 2504 } 2505 if (found) { 2506 break; 2507 } 2508 } 2509 } 2510 2511 ch = sc->channels[chidx]; 2512 error = buf_ring_enqueue(ch->xmit_br, m); 2513 if (__predict_false(error != 0)) { 2514 if_inc_counter(ifp, IFCOUNTER_OQDROPS, 1); 2515 m_freem(m); 2516 } else { 2517 taskqueue_enqueue(ch->cleanup_tq, &ch->cleanup_task); 2518 } 2519 2520 return (error); 2521 } 2522 2523 static void 2524 dpaa2_ni_qflush(if_t ifp) 2525 { 2526 /* TODO: Find a way to drain Tx queues in QBMan. */ 2527 if_qflush(ifp); 2528 } 2529 2530 static int 2531 dpaa2_ni_ioctl(if_t ifp, u_long c, caddr_t data) 2532 { 2533 struct dpaa2_ni_softc *sc = if_getsoftc(ifp); 2534 struct ifreq *ifr = (struct ifreq *) data; 2535 device_t pdev = device_get_parent(sc->dev); 2536 device_t dev = sc->dev; 2537 device_t child = dev; 2538 struct dpaa2_devinfo *rcinfo = device_get_ivars(pdev); 2539 struct dpaa2_devinfo *dinfo = device_get_ivars(dev); 2540 struct dpaa2_cmd cmd; 2541 uint32_t changed = 0; 2542 uint16_t rc_token, ni_token; 2543 int mtu, error, rc = 0; 2544 2545 DPAA2_CMD_INIT(&cmd); 2546 2547 error = DPAA2_CMD_RC_OPEN(dev, child, &cmd, rcinfo->id, &rc_token); 2548 if (error) { 2549 device_printf(dev, "%s: failed to open resource container: " 2550 "id=%d, error=%d\n", __func__, rcinfo->id, error); 2551 goto err_exit; 2552 } 2553 error = DPAA2_CMD_NI_OPEN(dev, child, &cmd, dinfo->id, &ni_token); 2554 if (error) { 2555 device_printf(dev, "%s: failed to open network interface: " 2556 "id=%d, error=%d\n", __func__, dinfo->id, error); 2557 goto close_rc; 2558 } 2559 2560 switch (c) { 2561 case SIOCSIFMTU: 2562 DPNI_LOCK(sc); 2563 mtu = ifr->ifr_mtu; 2564 if (mtu < ETHERMIN || mtu > ETHERMTU_JUMBO) { 2565 DPNI_UNLOCK(sc); 2566 error = EINVAL; 2567 goto close_ni; 2568 } 2569 if_setmtu(ifp, mtu); 2570 DPNI_UNLOCK(sc); 2571 2572 /* Update maximum frame length. */ 2573 mtu += ETHER_HDR_LEN; 2574 if (if_getcapenable(ifp) & IFCAP_VLAN_MTU) 2575 mtu += ETHER_VLAN_ENCAP_LEN; 2576 error = DPAA2_CMD_NI_SET_MFL(dev, child, &cmd, mtu); 2577 if (error) { 2578 device_printf(dev, "%s: failed to update maximum frame " 2579 "length: error=%d\n", __func__, error); 2580 goto close_ni; 2581 } 2582 break; 2583 case SIOCSIFCAP: 2584 changed = if_getcapenable(ifp) ^ ifr->ifr_reqcap; 2585 if ((changed & (IFCAP_RXCSUM | IFCAP_RXCSUM_IPV6)) != 0) 2586 if_togglecapenable(ifp, IFCAP_RXCSUM | IFCAP_RXCSUM_IPV6); 2587 if ((changed & (IFCAP_TXCSUM | IFCAP_TXCSUM_IPV6)) != 0) { 2588 if_togglecapenable(ifp, IFCAP_TXCSUM | IFCAP_TXCSUM_IPV6); 2589 if_togglehwassist(ifp, DPAA2_CSUM_TX_OFFLOAD); 2590 } 2591 2592 rc = dpaa2_ni_setup_if_caps(sc); 2593 if (rc) { 2594 printf("%s: failed to update iface capabilities: " 2595 "error=%d\n", __func__, rc); 2596 rc = ENXIO; 2597 } 2598 break; 2599 case SIOCSIFFLAGS: 2600 DPNI_LOCK(sc); 2601 if (if_getflags(ifp) & IFF_UP) { 2602 if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) { 2603 changed = if_getflags(ifp) ^ sc->if_flags; 2604 if (changed & IFF_PROMISC || 2605 changed & IFF_ALLMULTI) { 2606 rc = dpaa2_ni_setup_if_flags(sc); 2607 } 2608 } else { 2609 DPNI_UNLOCK(sc); 2610 dpaa2_ni_init(sc); 2611 DPNI_LOCK(sc); 2612 } 2613 } else if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) { 2614 /* FIXME: Disable DPNI. See dpaa2_ni_init(). */ 2615 } 2616 2617 sc->if_flags = if_getflags(ifp); 2618 DPNI_UNLOCK(sc); 2619 break; 2620 case SIOCADDMULTI: 2621 case SIOCDELMULTI: 2622 DPNI_LOCK(sc); 2623 if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) { 2624 DPNI_UNLOCK(sc); 2625 rc = dpaa2_ni_update_mac_filters(ifp); 2626 if (rc) { 2627 device_printf(dev, "%s: failed to update MAC " 2628 "filters: error=%d\n", __func__, rc); 2629 } 2630 DPNI_LOCK(sc); 2631 } 2632 DPNI_UNLOCK(sc); 2633 break; 2634 case SIOCGIFMEDIA: 2635 case SIOCSIFMEDIA: 2636 if (sc->mii) 2637 rc = ifmedia_ioctl(ifp, ifr, &sc->mii->mii_media, c); 2638 else if(sc->fixed_link) { 2639 rc = ifmedia_ioctl(ifp, ifr, &sc->fixed_ifmedia, c); 2640 } 2641 break; 2642 default: 2643 rc = ether_ioctl(ifp, c, data); 2644 break; 2645 } 2646 2647 (void)DPAA2_CMD_NI_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, ni_token)); 2648 (void)DPAA2_CMD_RC_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, rc_token)); 2649 return (rc); 2650 2651 close_ni: 2652 (void)DPAA2_CMD_NI_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, ni_token)); 2653 close_rc: 2654 (void)DPAA2_CMD_RC_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, rc_token)); 2655 err_exit: 2656 return (error); 2657 } 2658 2659 static int 2660 dpaa2_ni_update_mac_filters(if_t ifp) 2661 { 2662 struct dpaa2_ni_softc *sc = if_getsoftc(ifp); 2663 struct dpaa2_ni_mcaddr_ctx ctx; 2664 device_t pdev = device_get_parent(sc->dev); 2665 device_t dev = sc->dev; 2666 device_t child = dev; 2667 struct dpaa2_devinfo *rcinfo = device_get_ivars(pdev); 2668 struct dpaa2_devinfo *dinfo = device_get_ivars(dev); 2669 struct dpaa2_cmd cmd; 2670 uint16_t rc_token, ni_token; 2671 int error; 2672 2673 DPAA2_CMD_INIT(&cmd); 2674 2675 error = DPAA2_CMD_RC_OPEN(dev, child, &cmd, rcinfo->id, &rc_token); 2676 if (error) { 2677 device_printf(dev, "%s: failed to open resource container: " 2678 "id=%d, error=%d\n", __func__, rcinfo->id, error); 2679 goto err_exit; 2680 } 2681 error = DPAA2_CMD_NI_OPEN(dev, child, &cmd, dinfo->id, &ni_token); 2682 if (error) { 2683 device_printf(dev, "%s: failed to open network interface: " 2684 "id=%d, error=%d\n", __func__, dinfo->id, error); 2685 goto close_rc; 2686 } 2687 2688 /* Remove all multicast MAC filters. */ 2689 error = DPAA2_CMD_NI_CLEAR_MAC_FILTERS(dev, child, &cmd, false, true); 2690 if (error) { 2691 device_printf(dev, "%s: failed to clear multicast MAC filters: " 2692 "error=%d\n", __func__, error); 2693 goto close_ni; 2694 } 2695 2696 ctx.ifp = ifp; 2697 ctx.error = 0; 2698 ctx.nent = 0; 2699 2700 if_foreach_llmaddr(ifp, dpaa2_ni_add_maddr, &ctx); 2701 2702 error = ctx.error; 2703 close_ni: 2704 (void)DPAA2_CMD_NI_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, ni_token)); 2705 close_rc: 2706 (void)DPAA2_CMD_RC_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, rc_token)); 2707 err_exit: 2708 return (error); 2709 } 2710 2711 static u_int 2712 dpaa2_ni_add_maddr(void *arg, struct sockaddr_dl *sdl, u_int cnt) 2713 { 2714 struct dpaa2_ni_mcaddr_ctx *ctx = arg; 2715 struct dpaa2_ni_softc *sc = if_getsoftc(ctx->ifp); 2716 device_t pdev = device_get_parent(sc->dev); 2717 device_t dev = sc->dev; 2718 device_t child = dev; 2719 struct dpaa2_devinfo *rcinfo = device_get_ivars(pdev); 2720 struct dpaa2_devinfo *dinfo = device_get_ivars(dev); 2721 struct dpaa2_cmd cmd; 2722 uint16_t rc_token, ni_token; 2723 int error; 2724 2725 if (ctx->error != 0) { 2726 return (0); 2727 } 2728 2729 if (ETHER_IS_MULTICAST(LLADDR(sdl))) { 2730 DPAA2_CMD_INIT(&cmd); 2731 2732 error = DPAA2_CMD_RC_OPEN(dev, child, &cmd, rcinfo->id, 2733 &rc_token); 2734 if (error) { 2735 device_printf(dev, "%s: failed to open resource " 2736 "container: id=%d, error=%d\n", __func__, rcinfo->id, 2737 error); 2738 return (0); 2739 } 2740 error = DPAA2_CMD_NI_OPEN(dev, child, &cmd, dinfo->id, 2741 &ni_token); 2742 if (error) { 2743 device_printf(dev, "%s: failed to open network interface: " 2744 "id=%d, error=%d\n", __func__, dinfo->id, error); 2745 (void)DPAA2_CMD_RC_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, 2746 rc_token)); 2747 return (0); 2748 } 2749 2750 ctx->error = DPAA2_CMD_NI_ADD_MAC_ADDR(dev, child, &cmd, 2751 LLADDR(sdl)); 2752 2753 (void)DPAA2_CMD_NI_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, 2754 ni_token)); 2755 (void)DPAA2_CMD_RC_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, 2756 rc_token)); 2757 2758 if (ctx->error != 0) { 2759 device_printf(dev, "%s: can't add more then %d MAC " 2760 "addresses, switching to the multicast promiscuous " 2761 "mode\n", __func__, ctx->nent); 2762 2763 /* Enable multicast promiscuous mode. */ 2764 DPNI_LOCK(sc); 2765 if_setflagbits(ctx->ifp, IFF_ALLMULTI, 0); 2766 sc->if_flags |= IFF_ALLMULTI; 2767 ctx->error = dpaa2_ni_setup_if_flags(sc); 2768 DPNI_UNLOCK(sc); 2769 2770 return (0); 2771 } 2772 ctx->nent++; 2773 } 2774 2775 return (1); 2776 } 2777 2778 static void 2779 dpaa2_ni_intr(void *arg) 2780 { 2781 struct dpaa2_ni_softc *sc = (struct dpaa2_ni_softc *) arg; 2782 device_t pdev = device_get_parent(sc->dev); 2783 device_t dev = sc->dev; 2784 device_t child = dev; 2785 struct dpaa2_devinfo *rcinfo = device_get_ivars(pdev); 2786 struct dpaa2_devinfo *dinfo = device_get_ivars(dev); 2787 struct dpaa2_cmd cmd; 2788 uint32_t status = ~0u; /* clear all IRQ status bits */ 2789 uint16_t rc_token, ni_token; 2790 int error; 2791 2792 DPAA2_CMD_INIT(&cmd); 2793 2794 error = DPAA2_CMD_RC_OPEN(dev, child, &cmd, rcinfo->id, &rc_token); 2795 if (error) { 2796 device_printf(dev, "%s: failed to open resource container: " 2797 "id=%d, error=%d\n", __func__, rcinfo->id, error); 2798 goto err_exit; 2799 } 2800 error = DPAA2_CMD_NI_OPEN(dev, child, &cmd, dinfo->id, &ni_token); 2801 if (error) { 2802 device_printf(dev, "%s: failed to open network interface: " 2803 "id=%d, error=%d\n", __func__, dinfo->id, error); 2804 goto close_rc; 2805 } 2806 2807 error = DPAA2_CMD_NI_GET_IRQ_STATUS(dev, child, &cmd, DPNI_IRQ_INDEX, 2808 &status); 2809 if (error) { 2810 device_printf(sc->dev, "%s: failed to obtain IRQ status: " 2811 "error=%d\n", __func__, error); 2812 } 2813 2814 (void)DPAA2_CMD_NI_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, ni_token)); 2815 close_rc: 2816 (void)DPAA2_CMD_RC_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, rc_token)); 2817 err_exit: 2818 return; 2819 } 2820 2821 /** 2822 * @brief Execute channel's Rx/Tx routines. 2823 * 2824 * NOTE: Should not be re-entrant for the same channel. It is achieved by 2825 * enqueuing the cleanup routine on a single-threaded taskqueue. 2826 */ 2827 static void 2828 dpaa2_ni_cleanup_task(void *arg, int count) 2829 { 2830 struct dpaa2_channel *ch = (struct dpaa2_channel *)arg; 2831 struct dpaa2_ni_softc *sc = device_get_softc(ch->ni_dev); 2832 int error, rxc, txc; 2833 2834 for (int i = 0; i < DPAA2_CLEAN_BUDGET; i++) { 2835 rxc = dpaa2_ni_rx_cleanup(ch); 2836 txc = dpaa2_ni_tx_cleanup(ch); 2837 2838 if (__predict_false((if_getdrvflags(sc->ifp) & 2839 IFF_DRV_RUNNING) == 0)) { 2840 return; 2841 } 2842 2843 if ((txc != DPAA2_TX_BUDGET) && (rxc != DPAA2_RX_BUDGET)) { 2844 break; 2845 } 2846 } 2847 2848 /* Re-arm channel to generate CDAN */ 2849 error = DPAA2_SWP_CONF_WQ_CHANNEL(ch->io_dev, &ch->ctx); 2850 if (error != 0) { 2851 panic("%s: failed to rearm channel: chan_id=%d, error=%d\n", 2852 __func__, ch->id, error); 2853 } 2854 } 2855 2856 /** 2857 * @brief Poll frames from a specific channel when CDAN is received. 2858 */ 2859 static int 2860 dpaa2_ni_rx_cleanup(struct dpaa2_channel *ch) 2861 { 2862 struct dpaa2_io_softc *iosc = device_get_softc(ch->io_dev); 2863 struct dpaa2_swp *swp = iosc->swp; 2864 struct dpaa2_ni_fq *fq; 2865 struct dpaa2_buf *buf = &ch->store; 2866 int budget = DPAA2_RX_BUDGET; 2867 int error, consumed = 0; 2868 2869 do { 2870 error = dpaa2_swp_pull(swp, ch->id, buf, DPAA2_ETH_STORE_FRAMES); 2871 if (error) { 2872 device_printf(ch->ni_dev, "%s: failed to pull frames: " 2873 "chan_id=%d, error=%d\n", __func__, ch->id, error); 2874 break; 2875 } 2876 error = dpaa2_ni_consume_frames(ch, &fq, &consumed); 2877 if (error == ENOENT || error == EALREADY) { 2878 break; 2879 } 2880 if (error == ETIMEDOUT) { 2881 device_printf(ch->ni_dev, "%s: timeout to consume " 2882 "frames: chan_id=%d\n", __func__, ch->id); 2883 } 2884 } while (--budget); 2885 2886 return (DPAA2_RX_BUDGET - budget); 2887 } 2888 2889 static int 2890 dpaa2_ni_tx_cleanup(struct dpaa2_channel *ch) 2891 { 2892 struct dpaa2_ni_softc *sc = device_get_softc(ch->ni_dev); 2893 struct dpaa2_ni_tx_ring *tx = &ch->txc_queue.tx_rings[0]; 2894 struct mbuf *m = NULL; 2895 int budget = DPAA2_TX_BUDGET; 2896 2897 do { 2898 mtx_assert(&ch->xmit_mtx, MA_NOTOWNED); 2899 mtx_lock(&ch->xmit_mtx); 2900 m = buf_ring_dequeue_sc(ch->xmit_br); 2901 mtx_unlock(&ch->xmit_mtx); 2902 2903 if (__predict_false(m == NULL)) { 2904 /* TODO: Do not give up easily */ 2905 break; 2906 } else { 2907 dpaa2_ni_tx(sc, ch, tx, m); 2908 } 2909 } while (--budget); 2910 2911 return (DPAA2_TX_BUDGET - budget); 2912 } 2913 2914 static void 2915 dpaa2_ni_tx(struct dpaa2_ni_softc *sc, struct dpaa2_channel *ch, 2916 struct dpaa2_ni_tx_ring *tx, struct mbuf *m) 2917 { 2918 device_t dev = sc->dev; 2919 struct dpaa2_ni_fq *fq = tx->fq; 2920 struct dpaa2_buf *buf, *sgt; 2921 struct dpaa2_fd fd; 2922 struct mbuf *md; 2923 bus_dma_segment_t segs[DPAA2_TX_SEGLIMIT]; 2924 int rc, nsegs; 2925 int error; 2926 int len; 2927 bool mcast; 2928 2929 mtx_assert(&tx->lock, MA_NOTOWNED); 2930 mtx_lock(&tx->lock); 2931 buf = buf_ring_dequeue_sc(tx->br); 2932 mtx_unlock(&tx->lock); 2933 if (__predict_false(buf == NULL)) { 2934 /* TODO: Do not give up easily */ 2935 m_freem(m); 2936 return; 2937 } else { 2938 DPAA2_BUF_ASSERT_TXREADY(buf); 2939 buf->m = m; 2940 sgt = buf->sgt; 2941 } 2942 len = m->m_pkthdr.len; 2943 mcast = (m->m_flags & M_MCAST) != 0; 2944 2945 #if defined(INVARIANTS) 2946 struct dpaa2_ni_tx_ring *btx = (struct dpaa2_ni_tx_ring *)buf->opt; 2947 KASSERT(buf->opt == tx, ("%s: unexpected Tx ring", __func__)); 2948 KASSERT(btx->fq->chan == ch, ("%s: unexpected channel", __func__)); 2949 #endif /* INVARIANTS */ 2950 2951 BPF_MTAP(sc->ifp, m); 2952 2953 error = bus_dmamap_load_mbuf_sg(buf->dmat, buf->dmap, m, segs, &nsegs, 2954 BUS_DMA_NOWAIT); 2955 if (__predict_false(error != 0)) { 2956 /* Too many fragments, trying to defragment... */ 2957 md = m_collapse(m, M_NOWAIT, DPAA2_TX_SEGLIMIT); 2958 if (md == NULL) { 2959 device_printf(dev, "%s: m_collapse() failed\n", __func__); 2960 fq->chan->tx_dropped++; 2961 if_inc_counter(sc->ifp, IFCOUNTER_OERRORS, 1); 2962 goto err; 2963 } 2964 2965 buf->m = m = md; 2966 error = bus_dmamap_load_mbuf_sg(buf->dmat, buf->dmap, m, segs, 2967 &nsegs, BUS_DMA_NOWAIT); 2968 if (__predict_false(error != 0)) { 2969 device_printf(dev, "%s: bus_dmamap_load_mbuf_sg() " 2970 "failed: error=%d\n", __func__, error); 2971 fq->chan->tx_dropped++; 2972 if_inc_counter(sc->ifp, IFCOUNTER_OERRORS, 1); 2973 goto err; 2974 } 2975 } 2976 2977 error = dpaa2_fd_build(dev, sc->tx_data_off, buf, segs, nsegs, &fd); 2978 if (__predict_false(error != 0)) { 2979 device_printf(dev, "%s: failed to build frame descriptor: " 2980 "error=%d\n", __func__, error); 2981 fq->chan->tx_dropped++; 2982 if_inc_counter(sc->ifp, IFCOUNTER_OERRORS, 1); 2983 goto err_unload; 2984 } else 2985 sc->tx_sg_frames++; /* for sysctl(9) */ 2986 2987 bus_dmamap_sync(buf->dmat, buf->dmap, BUS_DMASYNC_PREWRITE); 2988 bus_dmamap_sync(sgt->dmat, sgt->dmap, BUS_DMASYNC_PREWRITE); 2989 2990 /* TODO: Enqueue several frames in a single command */ 2991 for (int i = 0; i < DPAA2_NI_ENQUEUE_RETRIES; i++) { 2992 /* TODO: Return error codes instead of # of frames */ 2993 rc = DPAA2_SWP_ENQ_MULTIPLE_FQ(fq->chan->io_dev, tx->fqid, &fd, 1); 2994 if (rc == 1) { 2995 break; 2996 } 2997 } 2998 2999 if (rc != 1) { 3000 fq->chan->tx_dropped++; 3001 if_inc_counter(sc->ifp, IFCOUNTER_OERRORS, 1); 3002 goto err_unload; 3003 } else { 3004 if (mcast) 3005 if_inc_counter(sc->ifp, IFCOUNTER_OMCASTS, 1); 3006 if_inc_counter(sc->ifp, IFCOUNTER_OPACKETS, 1); 3007 if_inc_counter(sc->ifp, IFCOUNTER_OBYTES, len); 3008 fq->chan->tx_frames++; 3009 } 3010 return; 3011 3012 err_unload: 3013 bus_dmamap_unload(buf->dmat, buf->dmap); 3014 if (sgt->paddr != 0) { 3015 bus_dmamap_unload(sgt->dmat, sgt->dmap); 3016 } 3017 err: 3018 m_freem(buf->m); 3019 buf_ring_enqueue(tx->br, buf); 3020 } 3021 3022 static int 3023 dpaa2_ni_consume_frames(struct dpaa2_channel *chan, struct dpaa2_ni_fq **src, 3024 uint32_t *consumed) 3025 { 3026 struct dpaa2_ni_fq *fq = NULL; 3027 struct dpaa2_dq *dq; 3028 struct dpaa2_fd *fd; 3029 struct dpaa2_ni_rx_ctx ctx = { 3030 .head = NULL, 3031 .tail = NULL, 3032 .cnt = 0, 3033 .last = false 3034 }; 3035 int rc, frames = 0; 3036 3037 do { 3038 rc = dpaa2_chan_next_frame(chan, &dq); 3039 if (rc == EINPROGRESS) { 3040 if (dq != NULL && !IS_NULL_RESPONSE(dq->fdr.desc.stat)) { 3041 fd = &dq->fdr.fd; 3042 fq = (struct dpaa2_ni_fq *) dq->fdr.desc.fqd_ctx; 3043 3044 switch (fq->type) { 3045 case DPAA2_NI_QUEUE_RX: 3046 (void)dpaa2_ni_rx(chan, fq, fd, &ctx); 3047 break; 3048 case DPAA2_NI_QUEUE_RX_ERR: 3049 (void)dpaa2_ni_rx_err(chan, fq, fd); 3050 break; 3051 case DPAA2_NI_QUEUE_TX_CONF: 3052 (void)dpaa2_ni_tx_conf(chan, fq, fd); 3053 break; 3054 default: 3055 panic("%s: unknown queue type (1)", 3056 __func__); 3057 } 3058 frames++; 3059 } 3060 } else if (rc == EALREADY || rc == ENOENT) { 3061 if (dq != NULL && !IS_NULL_RESPONSE(dq->fdr.desc.stat)) { 3062 fd = &dq->fdr.fd; 3063 fq = (struct dpaa2_ni_fq *) dq->fdr.desc.fqd_ctx; 3064 3065 switch (fq->type) { 3066 case DPAA2_NI_QUEUE_RX: 3067 /* 3068 * Last VDQ response (mbuf) in a chain 3069 * obtained from the Rx queue. 3070 */ 3071 ctx.last = true; 3072 (void)dpaa2_ni_rx(chan, fq, fd, &ctx); 3073 break; 3074 case DPAA2_NI_QUEUE_RX_ERR: 3075 (void)dpaa2_ni_rx_err(chan, fq, fd); 3076 break; 3077 case DPAA2_NI_QUEUE_TX_CONF: 3078 (void)dpaa2_ni_tx_conf(chan, fq, fd); 3079 break; 3080 default: 3081 panic("%s: unknown queue type (2)", 3082 __func__); 3083 } 3084 frames++; 3085 } 3086 break; 3087 } else { 3088 panic("%s: should not reach here: rc=%d", __func__, rc); 3089 } 3090 } while (true); 3091 3092 KASSERT(chan->store_idx < chan->store_sz, ("%s: store_idx(%d) >= " 3093 "store_sz(%d)", __func__, chan->store_idx, chan->store_sz)); 3094 3095 /* 3096 * VDQ operation pulls frames from a single queue into the store. 3097 * Return the frame queue and a number of consumed frames as an output. 3098 */ 3099 if (src != NULL) { 3100 *src = fq; 3101 } 3102 if (consumed != NULL) { 3103 *consumed = frames; 3104 } 3105 3106 return (rc); 3107 } 3108 3109 /** 3110 * @brief Receive frames. 3111 */ 3112 static int 3113 dpaa2_ni_rx(struct dpaa2_channel *ch, struct dpaa2_ni_fq *fq, 3114 struct dpaa2_fd *fd, struct dpaa2_ni_rx_ctx *ctx) 3115 { 3116 bus_addr_t paddr; 3117 struct dpaa2_swa *swa; 3118 struct dpaa2_buf *buf; 3119 struct dpaa2_channel *bch; 3120 struct dpaa2_ni_softc *sc; 3121 struct dpaa2_bp_softc *bpsc; 3122 struct mbuf *m; 3123 device_t bpdev; 3124 bus_addr_t released[DPAA2_SWP_BUFS_PER_CMD]; 3125 void *buf_data; 3126 int buf_len, error, released_n = 0; 3127 3128 error = dpaa2_fa_get_swa(fd, &swa); 3129 if (__predict_false(error != 0)) 3130 panic("%s: frame has no software annotation: error=%d", 3131 __func__, error); 3132 3133 paddr = (bus_addr_t)fd->addr; 3134 buf = swa->buf; 3135 bch = (struct dpaa2_channel *)buf->opt; 3136 sc = device_get_softc(bch->ni_dev); 3137 3138 KASSERT(swa->magic == DPAA2_MAGIC, ("%s: wrong magic", __func__)); 3139 /* 3140 * NOTE: Current channel might not be the same as the "buffer" channel 3141 * and it's fine. It must not be NULL though. 3142 */ 3143 KASSERT(bch != NULL, ("%s: buffer channel is NULL", __func__)); 3144 3145 if (__predict_false(paddr != buf->paddr)) { 3146 panic("%s: unexpected physical address: fd(%#jx) != buf(%#jx)", 3147 __func__, paddr, buf->paddr); 3148 } 3149 3150 switch (dpaa2_fd_err(fd)) { 3151 case 1: /* Enqueue rejected by QMan */ 3152 sc->rx_enq_rej_frames++; 3153 break; 3154 case 2: /* QMan IEOI error */ 3155 sc->rx_ieoi_err_frames++; 3156 break; 3157 default: 3158 break; 3159 } 3160 switch (dpaa2_fd_format(fd)) { 3161 case DPAA2_FD_SINGLE: 3162 sc->rx_single_buf_frames++; 3163 break; 3164 case DPAA2_FD_SG: 3165 sc->rx_sg_buf_frames++; 3166 break; 3167 default: 3168 break; 3169 } 3170 3171 mtx_assert(&bch->dma_mtx, MA_NOTOWNED); 3172 mtx_lock(&bch->dma_mtx); 3173 3174 bus_dmamap_sync(buf->dmat, buf->dmap, BUS_DMASYNC_POSTREAD); 3175 bus_dmamap_unload(buf->dmat, buf->dmap); 3176 3177 m = buf->m; 3178 buf_len = dpaa2_fd_data_len(fd); 3179 buf_data = (uint8_t *)buf->vaddr + dpaa2_fd_offset(fd); 3180 3181 /* Prepare buffer to be re-cycled */ 3182 buf->m = NULL; 3183 buf->paddr = 0; 3184 buf->vaddr = NULL; 3185 buf->seg.ds_addr = 0; 3186 buf->seg.ds_len = 0; 3187 buf->nseg = 0; 3188 3189 mtx_unlock(&bch->dma_mtx); 3190 3191 m->m_flags |= M_PKTHDR; 3192 m->m_data = buf_data; 3193 m->m_len = buf_len; 3194 m->m_pkthdr.len = buf_len; 3195 m->m_pkthdr.rcvif = sc->ifp; 3196 m->m_pkthdr.flowid = fq->fqid; 3197 M_HASHTYPE_SET(m, M_HASHTYPE_OPAQUE); 3198 if_inc_counter(sc->ifp, IFCOUNTER_IPACKETS, 1); 3199 3200 if (ctx->head == NULL) { 3201 KASSERT(ctx->tail == NULL, ("%s: tail already given?", __func__)); 3202 ctx->head = m; 3203 ctx->tail = m; 3204 } else { 3205 KASSERT(ctx->head != NULL, ("%s: head is NULL", __func__)); 3206 ctx->tail->m_nextpkt = m; 3207 ctx->tail = m; 3208 } 3209 ctx->cnt++; 3210 3211 if (ctx->last) { 3212 ctx->tail->m_nextpkt = NULL; 3213 if_input(sc->ifp, ctx->head); 3214 } 3215 3216 /* Keep the buffer to be recycled */ 3217 ch->recycled[ch->recycled_n++] = buf; 3218 3219 /* Re-seed and release recycled buffers back to the pool */ 3220 if (ch->recycled_n == DPAA2_SWP_BUFS_PER_CMD) { 3221 /* Release new buffers to the pool if needed */ 3222 taskqueue_enqueue(sc->bp_taskq, &ch->bp_task); 3223 3224 for (int i = 0; i < ch->recycled_n; i++) { 3225 buf = ch->recycled[i]; 3226 bch = (struct dpaa2_channel *)buf->opt; 3227 3228 mtx_assert(&bch->dma_mtx, MA_NOTOWNED); 3229 mtx_lock(&bch->dma_mtx); 3230 error = dpaa2_buf_seed_rxb(sc->dev, buf, 3231 DPAA2_RX_BUF_SIZE, &bch->dma_mtx); 3232 mtx_unlock(&bch->dma_mtx); 3233 3234 if (__predict_false(error != 0)) { 3235 /* TODO: What else to do with the buffer? */ 3236 panic("%s: failed to recycle buffer: error=%d", 3237 __func__, error); 3238 } 3239 3240 /* Prepare buffer to be released in a single command */ 3241 released[released_n++] = buf->paddr; 3242 } 3243 3244 /* There's only one buffer pool for now */ 3245 bpdev = (device_t)rman_get_start(sc->res[DPAA2_NI_BP_RID(0)]); 3246 bpsc = device_get_softc(bpdev); 3247 3248 error = DPAA2_SWP_RELEASE_BUFS(ch->io_dev, bpsc->attr.bpid, 3249 released, released_n); 3250 if (__predict_false(error != 0)) { 3251 device_printf(sc->dev, "%s: failed to release buffers " 3252 "to the pool: error=%d\n", __func__, error); 3253 return (error); 3254 } 3255 ch->recycled_n = 0; 3256 } 3257 3258 return (0); 3259 } 3260 3261 /** 3262 * @brief Receive Rx error frames. 3263 */ 3264 static int 3265 dpaa2_ni_rx_err(struct dpaa2_channel *ch, struct dpaa2_ni_fq *fq, 3266 struct dpaa2_fd *fd) 3267 { 3268 bus_addr_t paddr; 3269 struct dpaa2_swa *swa; 3270 struct dpaa2_buf *buf; 3271 struct dpaa2_channel *bch; 3272 struct dpaa2_ni_softc *sc; 3273 device_t bpdev; 3274 struct dpaa2_bp_softc *bpsc; 3275 int error; 3276 3277 error = dpaa2_fa_get_swa(fd, &swa); 3278 if (__predict_false(error != 0)) 3279 panic("%s: frame has no software annotation: error=%d", 3280 __func__, error); 3281 3282 paddr = (bus_addr_t)fd->addr; 3283 buf = swa->buf; 3284 bch = (struct dpaa2_channel *)buf->opt; 3285 sc = device_get_softc(bch->ni_dev); 3286 3287 KASSERT(swa->magic == DPAA2_MAGIC, ("%s: wrong magic", __func__)); 3288 /* 3289 * NOTE: Current channel might not be the same as the "buffer" channel 3290 * and it's fine. It must not be NULL though. 3291 */ 3292 KASSERT(bch != NULL, ("%s: buffer channel is NULL", __func__)); 3293 3294 if (__predict_false(paddr != buf->paddr)) { 3295 panic("%s: unexpected physical address: fd(%#jx) != buf(%#jx)", 3296 __func__, paddr, buf->paddr); 3297 } 3298 3299 /* There's only one buffer pool for now */ 3300 bpdev = (device_t)rman_get_start(sc->res[DPAA2_NI_BP_RID(0)]); 3301 bpsc = device_get_softc(bpdev); 3302 3303 /* Release buffer to QBMan buffer pool */ 3304 error = DPAA2_SWP_RELEASE_BUFS(ch->io_dev, bpsc->attr.bpid, &paddr, 1); 3305 if (error != 0) { 3306 device_printf(sc->dev, "%s: failed to release frame buffer to " 3307 "the pool: error=%d\n", __func__, error); 3308 return (error); 3309 } 3310 3311 return (0); 3312 } 3313 3314 /** 3315 * @brief Receive Tx confirmation frames. 3316 */ 3317 static int 3318 dpaa2_ni_tx_conf(struct dpaa2_channel *ch, struct dpaa2_ni_fq *fq, 3319 struct dpaa2_fd *fd) 3320 { 3321 bus_addr_t paddr; 3322 struct dpaa2_swa *swa; 3323 struct dpaa2_buf *buf; 3324 struct dpaa2_buf *sgt; 3325 struct dpaa2_ni_tx_ring *tx; 3326 struct dpaa2_channel *bch; 3327 int error; 3328 3329 error = dpaa2_fa_get_swa(fd, &swa); 3330 if (__predict_false(error != 0)) 3331 panic("%s: frame has no software annotation: error=%d", 3332 __func__, error); 3333 3334 paddr = (bus_addr_t)fd->addr; 3335 buf = swa->buf; 3336 sgt = buf->sgt; 3337 tx = (struct dpaa2_ni_tx_ring *)buf->opt; 3338 bch = tx->fq->chan; 3339 3340 KASSERT(swa->magic == DPAA2_MAGIC, ("%s: wrong magic", __func__)); 3341 KASSERT(tx != NULL, ("%s: Tx ring is NULL", __func__)); 3342 KASSERT(sgt != NULL, ("%s: S/G table is NULL", __func__)); 3343 /* 3344 * NOTE: Current channel might not be the same as the "buffer" channel 3345 * and it's fine. It must not be NULL though. 3346 */ 3347 KASSERT(bch != NULL, ("%s: buffer channel is NULL", __func__)); 3348 3349 if (paddr != buf->paddr) { 3350 panic("%s: unexpected physical address: fd(%#jx) != buf(%#jx)", 3351 __func__, paddr, buf->paddr); 3352 } 3353 3354 mtx_assert(&bch->dma_mtx, MA_NOTOWNED); 3355 mtx_lock(&bch->dma_mtx); 3356 3357 bus_dmamap_sync(buf->dmat, buf->dmap, BUS_DMASYNC_POSTWRITE); 3358 bus_dmamap_sync(sgt->dmat, sgt->dmap, BUS_DMASYNC_POSTWRITE); 3359 bus_dmamap_unload(buf->dmat, buf->dmap); 3360 bus_dmamap_unload(sgt->dmat, sgt->dmap); 3361 m_freem(buf->m); 3362 buf->m = NULL; 3363 buf->paddr = 0; 3364 buf->vaddr = NULL; 3365 sgt->paddr = 0; 3366 3367 mtx_unlock(&bch->dma_mtx); 3368 3369 /* Return Tx buffer back to the ring */ 3370 buf_ring_enqueue(tx->br, buf); 3371 3372 return (0); 3373 } 3374 3375 /** 3376 * @brief Compare versions of the DPAA2 network interface API. 3377 */ 3378 static int 3379 dpaa2_ni_cmp_api_version(struct dpaa2_ni_softc *sc, uint16_t major, 3380 uint16_t minor) 3381 { 3382 if (sc->api_major == major) { 3383 return sc->api_minor - minor; 3384 } 3385 return sc->api_major - major; 3386 } 3387 3388 /** 3389 * @brief Collect statistics of the network interface. 3390 */ 3391 static int 3392 dpaa2_ni_collect_stats(SYSCTL_HANDLER_ARGS) 3393 { 3394 struct dpaa2_ni_softc *sc = (struct dpaa2_ni_softc *) arg1; 3395 struct dpni_stat *stat = &dpni_stat_sysctls[oidp->oid_number]; 3396 device_t pdev = device_get_parent(sc->dev); 3397 device_t dev = sc->dev; 3398 device_t child = dev; 3399 struct dpaa2_devinfo *rcinfo = device_get_ivars(pdev); 3400 struct dpaa2_devinfo *dinfo = device_get_ivars(dev); 3401 struct dpaa2_cmd cmd; 3402 uint64_t cnt[DPAA2_NI_STAT_COUNTERS]; 3403 uint64_t result = 0; 3404 uint16_t rc_token, ni_token; 3405 int error; 3406 3407 DPAA2_CMD_INIT(&cmd); 3408 3409 error = DPAA2_CMD_RC_OPEN(dev, child, &cmd, rcinfo->id, &rc_token); 3410 if (error) { 3411 device_printf(dev, "%s: failed to open resource container: " 3412 "id=%d, error=%d\n", __func__, rcinfo->id, error); 3413 goto exit; 3414 } 3415 error = DPAA2_CMD_NI_OPEN(dev, child, &cmd, dinfo->id, &ni_token); 3416 if (error) { 3417 device_printf(dev, "%s: failed to open network interface: " 3418 "id=%d, error=%d\n", __func__, dinfo->id, error); 3419 goto close_rc; 3420 } 3421 3422 error = DPAA2_CMD_NI_GET_STATISTICS(dev, child, &cmd, stat->page, 0, cnt); 3423 if (!error) { 3424 result = cnt[stat->cnt]; 3425 } 3426 3427 (void)DPAA2_CMD_NI_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, ni_token)); 3428 close_rc: 3429 (void)DPAA2_CMD_RC_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, rc_token)); 3430 exit: 3431 return (sysctl_handle_64(oidp, &result, 0, req)); 3432 } 3433 3434 static int 3435 dpaa2_ni_collect_buf_num(SYSCTL_HANDLER_ARGS) 3436 { 3437 struct dpaa2_ni_softc *sc = (struct dpaa2_ni_softc *) arg1; 3438 uint32_t buf_num = DPAA2_ATOMIC_READ(&sc->buf_num); 3439 3440 return (sysctl_handle_32(oidp, &buf_num, 0, req)); 3441 } 3442 3443 static int 3444 dpaa2_ni_collect_buf_free(SYSCTL_HANDLER_ARGS) 3445 { 3446 struct dpaa2_ni_softc *sc = (struct dpaa2_ni_softc *) arg1; 3447 uint32_t buf_free = DPAA2_ATOMIC_READ(&sc->buf_free); 3448 3449 return (sysctl_handle_32(oidp, &buf_free, 0, req)); 3450 } 3451 3452 static int 3453 dpaa2_ni_set_hash(device_t dev, uint64_t flags) 3454 { 3455 struct dpaa2_ni_softc *sc = device_get_softc(dev); 3456 uint64_t key = 0; 3457 int i; 3458 3459 if (!(sc->attr.num.queues > 1)) { 3460 return (EOPNOTSUPP); 3461 } 3462 3463 for (i = 0; i < ARRAY_SIZE(dist_fields); i++) { 3464 if (dist_fields[i].rxnfc_field & flags) { 3465 key |= dist_fields[i].id; 3466 } 3467 } 3468 3469 return (dpaa2_ni_set_dist_key(dev, DPAA2_NI_DIST_MODE_HASH, key)); 3470 } 3471 3472 /** 3473 * @brief Set Rx distribution (hash or flow classification) key flags is a 3474 * combination of RXH_ bits. 3475 */ 3476 static int 3477 dpaa2_ni_set_dist_key(device_t dev, enum dpaa2_ni_dist_mode type, uint64_t flags) 3478 { 3479 device_t pdev = device_get_parent(dev); 3480 device_t child = dev; 3481 struct dpaa2_ni_softc *sc = device_get_softc(dev); 3482 struct dpaa2_devinfo *rcinfo = device_get_ivars(pdev); 3483 struct dpaa2_devinfo *dinfo = device_get_ivars(dev); 3484 struct dpkg_profile_cfg cls_cfg; 3485 struct dpkg_extract *key; 3486 struct dpaa2_buf *buf = &sc->rxd_kcfg; 3487 struct dpaa2_cmd cmd; 3488 uint16_t rc_token, ni_token; 3489 int i, error = 0; 3490 3491 if (__predict_true(buf->dmat == NULL)) { 3492 buf->dmat = sc->rxd_dmat; 3493 } 3494 3495 memset(&cls_cfg, 0, sizeof(cls_cfg)); 3496 3497 /* Configure extracts according to the given flags. */ 3498 for (i = 0; i < ARRAY_SIZE(dist_fields); i++) { 3499 key = &cls_cfg.extracts[cls_cfg.num_extracts]; 3500 3501 if (!(flags & dist_fields[i].id)) { 3502 continue; 3503 } 3504 3505 if (cls_cfg.num_extracts >= DPKG_MAX_NUM_OF_EXTRACTS) { 3506 device_printf(dev, "%s: failed to add key extraction " 3507 "rule\n", __func__); 3508 return (E2BIG); 3509 } 3510 3511 key->type = DPKG_EXTRACT_FROM_HDR; 3512 key->extract.from_hdr.prot = dist_fields[i].cls_prot; 3513 key->extract.from_hdr.type = DPKG_FULL_FIELD; 3514 key->extract.from_hdr.field = dist_fields[i].cls_field; 3515 cls_cfg.num_extracts++; 3516 } 3517 3518 error = bus_dmamem_alloc(buf->dmat, (void **)&buf->vaddr, 3519 BUS_DMA_ZERO | BUS_DMA_COHERENT, &buf->dmap); 3520 if (error != 0) { 3521 device_printf(dev, "%s: failed to allocate a buffer for Rx " 3522 "traffic distribution key configuration\n", __func__); 3523 return (error); 3524 } 3525 3526 error = dpaa2_ni_prepare_key_cfg(&cls_cfg, (uint8_t *)buf->vaddr); 3527 if (error != 0) { 3528 device_printf(dev, "%s: failed to prepare key configuration: " 3529 "error=%d\n", __func__, error); 3530 return (error); 3531 } 3532 3533 /* Prepare for setting the Rx dist. */ 3534 error = bus_dmamap_load(buf->dmat, buf->dmap, buf->vaddr, 3535 DPAA2_CLASSIFIER_DMA_SIZE, dpaa2_dmamap_oneseg_cb, &buf->paddr, 3536 BUS_DMA_NOWAIT); 3537 if (error != 0) { 3538 device_printf(sc->dev, "%s: failed to map a buffer for Rx " 3539 "traffic distribution key configuration\n", __func__); 3540 return (error); 3541 } 3542 3543 if (type == DPAA2_NI_DIST_MODE_HASH) { 3544 DPAA2_CMD_INIT(&cmd); 3545 3546 error = DPAA2_CMD_RC_OPEN(dev, child, &cmd, rcinfo->id, 3547 &rc_token); 3548 if (error) { 3549 device_printf(dev, "%s: failed to open resource " 3550 "container: id=%d, error=%d\n", __func__, rcinfo->id, 3551 error); 3552 goto err_exit; 3553 } 3554 error = DPAA2_CMD_NI_OPEN(dev, child, &cmd, dinfo->id, 3555 &ni_token); 3556 if (error) { 3557 device_printf(dev, "%s: failed to open network " 3558 "interface: id=%d, error=%d\n", __func__, dinfo->id, 3559 error); 3560 goto close_rc; 3561 } 3562 3563 error = DPAA2_CMD_NI_SET_RX_TC_DIST(dev, child, &cmd, 3564 sc->attr.num.queues, 0, DPAA2_NI_DIST_MODE_HASH, buf->paddr); 3565 if (error != 0) { 3566 device_printf(dev, "%s: failed to set distribution mode " 3567 "and size for the traffic class\n", __func__); 3568 } 3569 3570 (void)DPAA2_CMD_NI_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, 3571 ni_token)); 3572 close_rc: 3573 (void)DPAA2_CMD_RC_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, 3574 rc_token)); 3575 } 3576 3577 err_exit: 3578 return (error); 3579 } 3580 3581 /** 3582 * @brief Prepares extract parameters. 3583 * 3584 * cfg: Defining a full Key Generation profile. 3585 * key_cfg_buf: Zeroed 256 bytes of memory before mapping it to DMA. 3586 */ 3587 static int 3588 dpaa2_ni_prepare_key_cfg(struct dpkg_profile_cfg *cfg, uint8_t *key_cfg_buf) 3589 { 3590 struct dpni_ext_set_rx_tc_dist *dpni_ext; 3591 struct dpni_dist_extract *extr; 3592 int i, j; 3593 3594 if (cfg->num_extracts > DPKG_MAX_NUM_OF_EXTRACTS) 3595 return (EINVAL); 3596 3597 dpni_ext = (struct dpni_ext_set_rx_tc_dist *) key_cfg_buf; 3598 dpni_ext->num_extracts = cfg->num_extracts; 3599 3600 for (i = 0; i < cfg->num_extracts; i++) { 3601 extr = &dpni_ext->extracts[i]; 3602 3603 switch (cfg->extracts[i].type) { 3604 case DPKG_EXTRACT_FROM_HDR: 3605 extr->prot = cfg->extracts[i].extract.from_hdr.prot; 3606 extr->efh_type = 3607 cfg->extracts[i].extract.from_hdr.type & 0x0Fu; 3608 extr->size = cfg->extracts[i].extract.from_hdr.size; 3609 extr->offset = cfg->extracts[i].extract.from_hdr.offset; 3610 extr->field = cfg->extracts[i].extract.from_hdr.field; 3611 extr->hdr_index = 3612 cfg->extracts[i].extract.from_hdr.hdr_index; 3613 break; 3614 case DPKG_EXTRACT_FROM_DATA: 3615 extr->size = cfg->extracts[i].extract.from_data.size; 3616 extr->offset = 3617 cfg->extracts[i].extract.from_data.offset; 3618 break; 3619 case DPKG_EXTRACT_FROM_PARSE: 3620 extr->size = cfg->extracts[i].extract.from_parse.size; 3621 extr->offset = 3622 cfg->extracts[i].extract.from_parse.offset; 3623 break; 3624 default: 3625 return (EINVAL); 3626 } 3627 3628 extr->num_of_byte_masks = cfg->extracts[i].num_of_byte_masks; 3629 extr->extract_type = cfg->extracts[i].type & 0x0Fu; 3630 3631 for (j = 0; j < DPKG_NUM_OF_MASKS; j++) { 3632 extr->masks[j].mask = cfg->extracts[i].masks[j].mask; 3633 extr->masks[j].offset = 3634 cfg->extracts[i].masks[j].offset; 3635 } 3636 } 3637 3638 return (0); 3639 } 3640 3641 static device_method_t dpaa2_ni_methods[] = { 3642 /* Device interface */ 3643 DEVMETHOD(device_probe, dpaa2_ni_probe), 3644 DEVMETHOD(device_attach, dpaa2_ni_attach), 3645 DEVMETHOD(device_detach, dpaa2_ni_detach), 3646 3647 /* mii via memac_mdio */ 3648 DEVMETHOD(miibus_statchg, dpaa2_ni_miibus_statchg), 3649 3650 DEVMETHOD_END 3651 }; 3652 3653 static driver_t dpaa2_ni_driver = { 3654 "dpaa2_ni", 3655 dpaa2_ni_methods, 3656 sizeof(struct dpaa2_ni_softc), 3657 }; 3658 3659 DRIVER_MODULE(miibus, dpaa2_ni, miibus_driver, 0, 0); 3660 DRIVER_MODULE(dpaa2_ni, dpaa2_rc, dpaa2_ni_driver, 0, 0); 3661 3662 MODULE_DEPEND(dpaa2_ni, miibus, 1, 1, 1); 3663 #ifdef DEV_ACPI 3664 MODULE_DEPEND(dpaa2_ni, memac_mdio_acpi, 1, 1, 1); 3665 #endif 3666 #ifdef FDT 3667 MODULE_DEPEND(dpaa2_ni, memac_mdio_fdt, 1, 1, 1); 3668 #endif 3669