1 // SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause) 2 /* Copyright 2014-2016 Freescale Semiconductor Inc. 3 * Copyright 2016-2020 NXP 4 */ 5 #include <linux/init.h> 6 #include <linux/module.h> 7 #include <linux/platform_device.h> 8 #include <linux/etherdevice.h> 9 #include <linux/of_net.h> 10 #include <linux/interrupt.h> 11 #include <linux/msi.h> 12 #include <linux/kthread.h> 13 #include <linux/iommu.h> 14 #include <linux/net_tstamp.h> 15 #include <linux/fsl/mc.h> 16 #include <linux/bpf.h> 17 #include <linux/bpf_trace.h> 18 #include <net/sock.h> 19 20 #include "dpaa2-eth.h" 21 22 /* CREATE_TRACE_POINTS only needs to be defined once. Other dpa files 23 * using trace events only need to #include <trace/events/sched.h> 24 */ 25 #define CREATE_TRACE_POINTS 26 #include "dpaa2-eth-trace.h" 27 28 MODULE_LICENSE("Dual BSD/GPL"); 29 MODULE_AUTHOR("Freescale Semiconductor, Inc"); 30 MODULE_DESCRIPTION("Freescale DPAA2 Ethernet Driver"); 31 32 static void *dpaa2_iova_to_virt(struct iommu_domain *domain, 33 dma_addr_t iova_addr) 34 { 35 phys_addr_t phys_addr; 36 37 phys_addr = domain ? iommu_iova_to_phys(domain, iova_addr) : iova_addr; 38 39 return phys_to_virt(phys_addr); 40 } 41 42 static void validate_rx_csum(struct dpaa2_eth_priv *priv, 43 u32 fd_status, 44 struct sk_buff *skb) 45 { 46 skb_checksum_none_assert(skb); 47 48 /* HW checksum validation is disabled, nothing to do here */ 49 if (!(priv->net_dev->features & NETIF_F_RXCSUM)) 50 return; 51 52 /* Read checksum validation bits */ 53 if (!((fd_status & DPAA2_FAS_L3CV) && 54 (fd_status & DPAA2_FAS_L4CV))) 55 return; 56 57 /* Inform the stack there's no need to compute L3/L4 csum anymore */ 58 skb->ip_summed = CHECKSUM_UNNECESSARY; 59 } 60 61 /* Free a received FD. 62 * Not to be used for Tx conf FDs or on any other paths. 63 */ 64 static void free_rx_fd(struct dpaa2_eth_priv *priv, 65 const struct dpaa2_fd *fd, 66 void *vaddr) 67 { 68 struct device *dev = priv->net_dev->dev.parent; 69 dma_addr_t addr = dpaa2_fd_get_addr(fd); 70 u8 fd_format = dpaa2_fd_get_format(fd); 71 struct dpaa2_sg_entry *sgt; 72 void *sg_vaddr; 73 int i; 74 75 /* If single buffer frame, just free the data buffer */ 76 if (fd_format == dpaa2_fd_single) 77 goto free_buf; 78 else if (fd_format != dpaa2_fd_sg) 79 /* We don't support any other format */ 80 return; 81 82 /* For S/G frames, we first need to free all SG entries 83 * except the first one, which was taken care of already 84 */ 85 sgt = vaddr + dpaa2_fd_get_offset(fd); 86 for (i = 1; i < DPAA2_ETH_MAX_SG_ENTRIES; i++) { 87 addr = dpaa2_sg_get_addr(&sgt[i]); 88 sg_vaddr = dpaa2_iova_to_virt(priv->iommu_domain, addr); 89 dma_unmap_page(dev, addr, priv->rx_buf_size, 90 DMA_BIDIRECTIONAL); 91 92 free_pages((unsigned long)sg_vaddr, 0); 93 if (dpaa2_sg_is_final(&sgt[i])) 94 break; 95 } 96 97 free_buf: 98 free_pages((unsigned long)vaddr, 0); 99 } 100 101 /* Build a linear skb based on a single-buffer frame descriptor */ 102 static struct sk_buff *build_linear_skb(struct dpaa2_eth_channel *ch, 103 const struct dpaa2_fd *fd, 104 void *fd_vaddr) 105 { 106 struct sk_buff *skb = NULL; 107 u16 fd_offset = dpaa2_fd_get_offset(fd); 108 u32 fd_length = dpaa2_fd_get_len(fd); 109 110 ch->buf_count--; 111 112 skb = build_skb(fd_vaddr, DPAA2_ETH_RX_BUF_RAW_SIZE); 113 if (unlikely(!skb)) 114 return NULL; 115 116 skb_reserve(skb, fd_offset); 117 skb_put(skb, fd_length); 118 119 return skb; 120 } 121 122 /* Build a non linear (fragmented) skb based on a S/G table */ 123 static struct sk_buff *build_frag_skb(struct dpaa2_eth_priv *priv, 124 struct dpaa2_eth_channel *ch, 125 struct dpaa2_sg_entry *sgt) 126 { 127 struct sk_buff *skb = NULL; 128 struct device *dev = priv->net_dev->dev.parent; 129 void *sg_vaddr; 130 dma_addr_t sg_addr; 131 u16 sg_offset; 132 u32 sg_length; 133 struct page *page, *head_page; 134 int page_offset; 135 int i; 136 137 for (i = 0; i < DPAA2_ETH_MAX_SG_ENTRIES; i++) { 138 struct dpaa2_sg_entry *sge = &sgt[i]; 139 140 /* NOTE: We only support SG entries in dpaa2_sg_single format, 141 * but this is the only format we may receive from HW anyway 142 */ 143 144 /* Get the address and length from the S/G entry */ 145 sg_addr = dpaa2_sg_get_addr(sge); 146 sg_vaddr = dpaa2_iova_to_virt(priv->iommu_domain, sg_addr); 147 dma_unmap_page(dev, sg_addr, priv->rx_buf_size, 148 DMA_BIDIRECTIONAL); 149 150 sg_length = dpaa2_sg_get_len(sge); 151 152 if (i == 0) { 153 /* We build the skb around the first data buffer */ 154 skb = build_skb(sg_vaddr, DPAA2_ETH_RX_BUF_RAW_SIZE); 155 if (unlikely(!skb)) { 156 /* Free the first SG entry now, since we already 157 * unmapped it and obtained the virtual address 158 */ 159 free_pages((unsigned long)sg_vaddr, 0); 160 161 /* We still need to subtract the buffers used 162 * by this FD from our software counter 163 */ 164 while (!dpaa2_sg_is_final(&sgt[i]) && 165 i < DPAA2_ETH_MAX_SG_ENTRIES) 166 i++; 167 break; 168 } 169 170 sg_offset = dpaa2_sg_get_offset(sge); 171 skb_reserve(skb, sg_offset); 172 skb_put(skb, sg_length); 173 } else { 174 /* Rest of the data buffers are stored as skb frags */ 175 page = virt_to_page(sg_vaddr); 176 head_page = virt_to_head_page(sg_vaddr); 177 178 /* Offset in page (which may be compound). 179 * Data in subsequent SG entries is stored from the 180 * beginning of the buffer, so we don't need to add the 181 * sg_offset. 182 */ 183 page_offset = ((unsigned long)sg_vaddr & 184 (PAGE_SIZE - 1)) + 185 (page_address(page) - page_address(head_page)); 186 187 skb_add_rx_frag(skb, i - 1, head_page, page_offset, 188 sg_length, priv->rx_buf_size); 189 } 190 191 if (dpaa2_sg_is_final(sge)) 192 break; 193 } 194 195 WARN_ONCE(i == DPAA2_ETH_MAX_SG_ENTRIES, "Final bit not set in SGT"); 196 197 /* Count all data buffers + SG table buffer */ 198 ch->buf_count -= i + 2; 199 200 return skb; 201 } 202 203 /* Free buffers acquired from the buffer pool or which were meant to 204 * be released in the pool 205 */ 206 static void free_bufs(struct dpaa2_eth_priv *priv, u64 *buf_array, int count) 207 { 208 struct device *dev = priv->net_dev->dev.parent; 209 void *vaddr; 210 int i; 211 212 for (i = 0; i < count; i++) { 213 vaddr = dpaa2_iova_to_virt(priv->iommu_domain, buf_array[i]); 214 dma_unmap_page(dev, buf_array[i], priv->rx_buf_size, 215 DMA_BIDIRECTIONAL); 216 free_pages((unsigned long)vaddr, 0); 217 } 218 } 219 220 static void xdp_release_buf(struct dpaa2_eth_priv *priv, 221 struct dpaa2_eth_channel *ch, 222 dma_addr_t addr) 223 { 224 int retries = 0; 225 int err; 226 227 ch->xdp.drop_bufs[ch->xdp.drop_cnt++] = addr; 228 if (ch->xdp.drop_cnt < DPAA2_ETH_BUFS_PER_CMD) 229 return; 230 231 while ((err = dpaa2_io_service_release(ch->dpio, priv->bpid, 232 ch->xdp.drop_bufs, 233 ch->xdp.drop_cnt)) == -EBUSY) { 234 if (retries++ >= DPAA2_ETH_SWP_BUSY_RETRIES) 235 break; 236 cpu_relax(); 237 } 238 239 if (err) { 240 free_bufs(priv, ch->xdp.drop_bufs, ch->xdp.drop_cnt); 241 ch->buf_count -= ch->xdp.drop_cnt; 242 } 243 244 ch->xdp.drop_cnt = 0; 245 } 246 247 static int dpaa2_eth_xdp_flush(struct dpaa2_eth_priv *priv, 248 struct dpaa2_eth_fq *fq, 249 struct dpaa2_eth_xdp_fds *xdp_fds) 250 { 251 int total_enqueued = 0, retries = 0, enqueued; 252 struct dpaa2_eth_drv_stats *percpu_extras; 253 int num_fds, err, max_retries; 254 struct dpaa2_fd *fds; 255 256 percpu_extras = this_cpu_ptr(priv->percpu_extras); 257 258 /* try to enqueue all the FDs until the max number of retries is hit */ 259 fds = xdp_fds->fds; 260 num_fds = xdp_fds->num; 261 max_retries = num_fds * DPAA2_ETH_ENQUEUE_RETRIES; 262 while (total_enqueued < num_fds && retries < max_retries) { 263 err = priv->enqueue(priv, fq, &fds[total_enqueued], 264 0, num_fds - total_enqueued, &enqueued); 265 if (err == -EBUSY) { 266 percpu_extras->tx_portal_busy += ++retries; 267 continue; 268 } 269 total_enqueued += enqueued; 270 } 271 xdp_fds->num = 0; 272 273 return total_enqueued; 274 } 275 276 static void xdp_tx_flush(struct dpaa2_eth_priv *priv, 277 struct dpaa2_eth_channel *ch, 278 struct dpaa2_eth_fq *fq) 279 { 280 struct rtnl_link_stats64 *percpu_stats; 281 struct dpaa2_fd *fds; 282 int enqueued, i; 283 284 percpu_stats = this_cpu_ptr(priv->percpu_stats); 285 286 // enqueue the array of XDP_TX frames 287 enqueued = dpaa2_eth_xdp_flush(priv, fq, &fq->xdp_tx_fds); 288 289 /* update statistics */ 290 percpu_stats->tx_packets += enqueued; 291 fds = fq->xdp_tx_fds.fds; 292 for (i = 0; i < enqueued; i++) { 293 percpu_stats->tx_bytes += dpaa2_fd_get_len(&fds[i]); 294 ch->stats.xdp_tx++; 295 } 296 for (i = enqueued; i < fq->xdp_tx_fds.num; i++) { 297 xdp_release_buf(priv, ch, dpaa2_fd_get_addr(&fds[i])); 298 percpu_stats->tx_errors++; 299 ch->stats.xdp_tx_err++; 300 } 301 fq->xdp_tx_fds.num = 0; 302 } 303 304 static void xdp_enqueue(struct dpaa2_eth_priv *priv, 305 struct dpaa2_eth_channel *ch, 306 struct dpaa2_fd *fd, 307 void *buf_start, u16 queue_id) 308 { 309 struct dpaa2_faead *faead; 310 struct dpaa2_fd *dest_fd; 311 struct dpaa2_eth_fq *fq; 312 u32 ctrl, frc; 313 314 /* Mark the egress frame hardware annotation area as valid */ 315 frc = dpaa2_fd_get_frc(fd); 316 dpaa2_fd_set_frc(fd, frc | DPAA2_FD_FRC_FAEADV); 317 dpaa2_fd_set_ctrl(fd, DPAA2_FD_CTRL_ASAL); 318 319 /* Instruct hardware to release the FD buffer directly into 320 * the buffer pool once transmission is completed, instead of 321 * sending a Tx confirmation frame to us 322 */ 323 ctrl = DPAA2_FAEAD_A4V | DPAA2_FAEAD_A2V | DPAA2_FAEAD_EBDDV; 324 faead = dpaa2_get_faead(buf_start, false); 325 faead->ctrl = cpu_to_le32(ctrl); 326 faead->conf_fqid = 0; 327 328 fq = &priv->fq[queue_id]; 329 dest_fd = &fq->xdp_tx_fds.fds[fq->xdp_tx_fds.num++]; 330 memcpy(dest_fd, fd, sizeof(*dest_fd)); 331 332 if (fq->xdp_tx_fds.num < DEV_MAP_BULK_SIZE) 333 return; 334 335 xdp_tx_flush(priv, ch, fq); 336 } 337 338 static u32 run_xdp(struct dpaa2_eth_priv *priv, 339 struct dpaa2_eth_channel *ch, 340 struct dpaa2_eth_fq *rx_fq, 341 struct dpaa2_fd *fd, void *vaddr) 342 { 343 dma_addr_t addr = dpaa2_fd_get_addr(fd); 344 struct bpf_prog *xdp_prog; 345 struct xdp_buff xdp; 346 u32 xdp_act = XDP_PASS; 347 int err; 348 349 rcu_read_lock(); 350 351 xdp_prog = READ_ONCE(ch->xdp.prog); 352 if (!xdp_prog) 353 goto out; 354 355 xdp.data = vaddr + dpaa2_fd_get_offset(fd); 356 xdp.data_end = xdp.data + dpaa2_fd_get_len(fd); 357 xdp.data_hard_start = xdp.data - XDP_PACKET_HEADROOM; 358 xdp_set_data_meta_invalid(&xdp); 359 xdp.rxq = &ch->xdp_rxq; 360 361 xdp.frame_sz = DPAA2_ETH_RX_BUF_RAW_SIZE - 362 (dpaa2_fd_get_offset(fd) - XDP_PACKET_HEADROOM); 363 364 xdp_act = bpf_prog_run_xdp(xdp_prog, &xdp); 365 366 /* xdp.data pointer may have changed */ 367 dpaa2_fd_set_offset(fd, xdp.data - vaddr); 368 dpaa2_fd_set_len(fd, xdp.data_end - xdp.data); 369 370 switch (xdp_act) { 371 case XDP_PASS: 372 break; 373 case XDP_TX: 374 xdp_enqueue(priv, ch, fd, vaddr, rx_fq->flowid); 375 break; 376 default: 377 bpf_warn_invalid_xdp_action(xdp_act); 378 /* fall through */ 379 case XDP_ABORTED: 380 trace_xdp_exception(priv->net_dev, xdp_prog, xdp_act); 381 /* fall through */ 382 case XDP_DROP: 383 xdp_release_buf(priv, ch, addr); 384 ch->stats.xdp_drop++; 385 break; 386 case XDP_REDIRECT: 387 dma_unmap_page(priv->net_dev->dev.parent, addr, 388 priv->rx_buf_size, DMA_BIDIRECTIONAL); 389 ch->buf_count--; 390 391 /* Allow redirect use of full headroom */ 392 xdp.data_hard_start = vaddr; 393 xdp.frame_sz = DPAA2_ETH_RX_BUF_RAW_SIZE; 394 395 err = xdp_do_redirect(priv->net_dev, &xdp, xdp_prog); 396 if (unlikely(err)) 397 ch->stats.xdp_drop++; 398 else 399 ch->stats.xdp_redirect++; 400 break; 401 } 402 403 ch->xdp.res |= xdp_act; 404 out: 405 rcu_read_unlock(); 406 return xdp_act; 407 } 408 409 /* Main Rx frame processing routine */ 410 static void dpaa2_eth_rx(struct dpaa2_eth_priv *priv, 411 struct dpaa2_eth_channel *ch, 412 const struct dpaa2_fd *fd, 413 struct dpaa2_eth_fq *fq) 414 { 415 dma_addr_t addr = dpaa2_fd_get_addr(fd); 416 u8 fd_format = dpaa2_fd_get_format(fd); 417 void *vaddr; 418 struct sk_buff *skb; 419 struct rtnl_link_stats64 *percpu_stats; 420 struct dpaa2_eth_drv_stats *percpu_extras; 421 struct device *dev = priv->net_dev->dev.parent; 422 struct dpaa2_fas *fas; 423 void *buf_data; 424 u32 status = 0; 425 u32 xdp_act; 426 427 /* Tracing point */ 428 trace_dpaa2_rx_fd(priv->net_dev, fd); 429 430 vaddr = dpaa2_iova_to_virt(priv->iommu_domain, addr); 431 dma_sync_single_for_cpu(dev, addr, priv->rx_buf_size, 432 DMA_BIDIRECTIONAL); 433 434 fas = dpaa2_get_fas(vaddr, false); 435 prefetch(fas); 436 buf_data = vaddr + dpaa2_fd_get_offset(fd); 437 prefetch(buf_data); 438 439 percpu_stats = this_cpu_ptr(priv->percpu_stats); 440 percpu_extras = this_cpu_ptr(priv->percpu_extras); 441 442 if (fd_format == dpaa2_fd_single) { 443 xdp_act = run_xdp(priv, ch, fq, (struct dpaa2_fd *)fd, vaddr); 444 if (xdp_act != XDP_PASS) { 445 percpu_stats->rx_packets++; 446 percpu_stats->rx_bytes += dpaa2_fd_get_len(fd); 447 return; 448 } 449 450 dma_unmap_page(dev, addr, priv->rx_buf_size, 451 DMA_BIDIRECTIONAL); 452 skb = build_linear_skb(ch, fd, vaddr); 453 } else if (fd_format == dpaa2_fd_sg) { 454 WARN_ON(priv->xdp_prog); 455 456 dma_unmap_page(dev, addr, priv->rx_buf_size, 457 DMA_BIDIRECTIONAL); 458 skb = build_frag_skb(priv, ch, buf_data); 459 free_pages((unsigned long)vaddr, 0); 460 percpu_extras->rx_sg_frames++; 461 percpu_extras->rx_sg_bytes += dpaa2_fd_get_len(fd); 462 } else { 463 /* We don't support any other format */ 464 goto err_frame_format; 465 } 466 467 if (unlikely(!skb)) 468 goto err_build_skb; 469 470 prefetch(skb->data); 471 472 /* Get the timestamp value */ 473 if (priv->rx_tstamp) { 474 struct skb_shared_hwtstamps *shhwtstamps = skb_hwtstamps(skb); 475 __le64 *ts = dpaa2_get_ts(vaddr, false); 476 u64 ns; 477 478 memset(shhwtstamps, 0, sizeof(*shhwtstamps)); 479 480 ns = DPAA2_PTP_CLK_PERIOD_NS * le64_to_cpup(ts); 481 shhwtstamps->hwtstamp = ns_to_ktime(ns); 482 } 483 484 /* Check if we need to validate the L4 csum */ 485 if (likely(dpaa2_fd_get_frc(fd) & DPAA2_FD_FRC_FASV)) { 486 status = le32_to_cpu(fas->status); 487 validate_rx_csum(priv, status, skb); 488 } 489 490 skb->protocol = eth_type_trans(skb, priv->net_dev); 491 skb_record_rx_queue(skb, fq->flowid); 492 493 percpu_stats->rx_packets++; 494 percpu_stats->rx_bytes += dpaa2_fd_get_len(fd); 495 496 list_add_tail(&skb->list, ch->rx_list); 497 498 return; 499 500 err_build_skb: 501 free_rx_fd(priv, fd, vaddr); 502 err_frame_format: 503 percpu_stats->rx_dropped++; 504 } 505 506 /* Consume all frames pull-dequeued into the store. This is the simplest way to 507 * make sure we don't accidentally issue another volatile dequeue which would 508 * overwrite (leak) frames already in the store. 509 * 510 * Observance of NAPI budget is not our concern, leaving that to the caller. 511 */ 512 static int consume_frames(struct dpaa2_eth_channel *ch, 513 struct dpaa2_eth_fq **src) 514 { 515 struct dpaa2_eth_priv *priv = ch->priv; 516 struct dpaa2_eth_fq *fq = NULL; 517 struct dpaa2_dq *dq; 518 const struct dpaa2_fd *fd; 519 int cleaned = 0, retries = 0; 520 int is_last; 521 522 do { 523 dq = dpaa2_io_store_next(ch->store, &is_last); 524 if (unlikely(!dq)) { 525 /* If we're here, we *must* have placed a 526 * volatile dequeue comnmand, so keep reading through 527 * the store until we get some sort of valid response 528 * token (either a valid frame or an "empty dequeue") 529 */ 530 if (retries++ >= DPAA2_ETH_SWP_BUSY_RETRIES) { 531 netdev_err_once(priv->net_dev, 532 "Unable to read a valid dequeue response\n"); 533 return -ETIMEDOUT; 534 } 535 continue; 536 } 537 538 fd = dpaa2_dq_fd(dq); 539 fq = (struct dpaa2_eth_fq *)(uintptr_t)dpaa2_dq_fqd_ctx(dq); 540 541 fq->consume(priv, ch, fd, fq); 542 cleaned++; 543 retries = 0; 544 } while (!is_last); 545 546 if (!cleaned) 547 return 0; 548 549 fq->stats.frames += cleaned; 550 ch->stats.frames += cleaned; 551 552 /* A dequeue operation only pulls frames from a single queue 553 * into the store. Return the frame queue as an out param. 554 */ 555 if (src) 556 *src = fq; 557 558 return cleaned; 559 } 560 561 /* Configure the egress frame annotation for timestamp update */ 562 static void enable_tx_tstamp(struct dpaa2_fd *fd, void *buf_start) 563 { 564 struct dpaa2_faead *faead; 565 u32 ctrl, frc; 566 567 /* Mark the egress frame annotation area as valid */ 568 frc = dpaa2_fd_get_frc(fd); 569 dpaa2_fd_set_frc(fd, frc | DPAA2_FD_FRC_FAEADV); 570 571 /* Set hardware annotation size */ 572 ctrl = dpaa2_fd_get_ctrl(fd); 573 dpaa2_fd_set_ctrl(fd, ctrl | DPAA2_FD_CTRL_ASAL); 574 575 /* enable UPD (update prepanded data) bit in FAEAD field of 576 * hardware frame annotation area 577 */ 578 ctrl = DPAA2_FAEAD_A2V | DPAA2_FAEAD_UPDV | DPAA2_FAEAD_UPD; 579 faead = dpaa2_get_faead(buf_start, true); 580 faead->ctrl = cpu_to_le32(ctrl); 581 } 582 583 /* Create a frame descriptor based on a fragmented skb */ 584 static int build_sg_fd(struct dpaa2_eth_priv *priv, 585 struct sk_buff *skb, 586 struct dpaa2_fd *fd) 587 { 588 struct device *dev = priv->net_dev->dev.parent; 589 void *sgt_buf = NULL; 590 dma_addr_t addr; 591 int nr_frags = skb_shinfo(skb)->nr_frags; 592 struct dpaa2_sg_entry *sgt; 593 int i, err; 594 int sgt_buf_size; 595 struct scatterlist *scl, *crt_scl; 596 int num_sg; 597 int num_dma_bufs; 598 struct dpaa2_eth_swa *swa; 599 600 /* Create and map scatterlist. 601 * We don't advertise NETIF_F_FRAGLIST, so skb_to_sgvec() will not have 602 * to go beyond nr_frags+1. 603 * Note: We don't support chained scatterlists 604 */ 605 if (unlikely(PAGE_SIZE / sizeof(struct scatterlist) < nr_frags + 1)) 606 return -EINVAL; 607 608 scl = kcalloc(nr_frags + 1, sizeof(struct scatterlist), GFP_ATOMIC); 609 if (unlikely(!scl)) 610 return -ENOMEM; 611 612 sg_init_table(scl, nr_frags + 1); 613 num_sg = skb_to_sgvec(skb, scl, 0, skb->len); 614 if (unlikely(num_sg < 0)) { 615 err = -ENOMEM; 616 goto dma_map_sg_failed; 617 } 618 num_dma_bufs = dma_map_sg(dev, scl, num_sg, DMA_BIDIRECTIONAL); 619 if (unlikely(!num_dma_bufs)) { 620 err = -ENOMEM; 621 goto dma_map_sg_failed; 622 } 623 624 /* Prepare the HW SGT structure */ 625 sgt_buf_size = priv->tx_data_offset + 626 sizeof(struct dpaa2_sg_entry) * num_dma_bufs; 627 sgt_buf = napi_alloc_frag(sgt_buf_size + DPAA2_ETH_TX_BUF_ALIGN); 628 if (unlikely(!sgt_buf)) { 629 err = -ENOMEM; 630 goto sgt_buf_alloc_failed; 631 } 632 sgt_buf = PTR_ALIGN(sgt_buf, DPAA2_ETH_TX_BUF_ALIGN); 633 memset(sgt_buf, 0, sgt_buf_size); 634 635 sgt = (struct dpaa2_sg_entry *)(sgt_buf + priv->tx_data_offset); 636 637 /* Fill in the HW SGT structure. 638 * 639 * sgt_buf is zeroed out, so the following fields are implicit 640 * in all sgt entries: 641 * - offset is 0 642 * - format is 'dpaa2_sg_single' 643 */ 644 for_each_sg(scl, crt_scl, num_dma_bufs, i) { 645 dpaa2_sg_set_addr(&sgt[i], sg_dma_address(crt_scl)); 646 dpaa2_sg_set_len(&sgt[i], sg_dma_len(crt_scl)); 647 } 648 dpaa2_sg_set_final(&sgt[i - 1], true); 649 650 /* Store the skb backpointer in the SGT buffer. 651 * Fit the scatterlist and the number of buffers alongside the 652 * skb backpointer in the software annotation area. We'll need 653 * all of them on Tx Conf. 654 */ 655 swa = (struct dpaa2_eth_swa *)sgt_buf; 656 swa->type = DPAA2_ETH_SWA_SG; 657 swa->sg.skb = skb; 658 swa->sg.scl = scl; 659 swa->sg.num_sg = num_sg; 660 swa->sg.sgt_size = sgt_buf_size; 661 662 /* Separately map the SGT buffer */ 663 addr = dma_map_single(dev, sgt_buf, sgt_buf_size, DMA_BIDIRECTIONAL); 664 if (unlikely(dma_mapping_error(dev, addr))) { 665 err = -ENOMEM; 666 goto dma_map_single_failed; 667 } 668 dpaa2_fd_set_offset(fd, priv->tx_data_offset); 669 dpaa2_fd_set_format(fd, dpaa2_fd_sg); 670 dpaa2_fd_set_addr(fd, addr); 671 dpaa2_fd_set_len(fd, skb->len); 672 dpaa2_fd_set_ctrl(fd, FD_CTRL_PTA); 673 674 if (priv->tx_tstamp && skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) 675 enable_tx_tstamp(fd, sgt_buf); 676 677 return 0; 678 679 dma_map_single_failed: 680 skb_free_frag(sgt_buf); 681 sgt_buf_alloc_failed: 682 dma_unmap_sg(dev, scl, num_sg, DMA_BIDIRECTIONAL); 683 dma_map_sg_failed: 684 kfree(scl); 685 return err; 686 } 687 688 /* Create a SG frame descriptor based on a linear skb. 689 * 690 * This function is used on the Tx path when the skb headroom is not large 691 * enough for the HW requirements, thus instead of realloc-ing the skb we 692 * create a SG frame descriptor with only one entry. 693 */ 694 static int build_sg_fd_single_buf(struct dpaa2_eth_priv *priv, 695 struct sk_buff *skb, 696 struct dpaa2_fd *fd) 697 { 698 struct device *dev = priv->net_dev->dev.parent; 699 struct dpaa2_eth_sgt_cache *sgt_cache; 700 struct dpaa2_sg_entry *sgt; 701 struct dpaa2_eth_swa *swa; 702 dma_addr_t addr, sgt_addr; 703 void *sgt_buf = NULL; 704 int sgt_buf_size; 705 int err; 706 707 /* Prepare the HW SGT structure */ 708 sgt_cache = this_cpu_ptr(priv->sgt_cache); 709 sgt_buf_size = priv->tx_data_offset + sizeof(struct dpaa2_sg_entry); 710 711 if (sgt_cache->count == 0) 712 sgt_buf = kzalloc(sgt_buf_size + DPAA2_ETH_TX_BUF_ALIGN, 713 GFP_ATOMIC); 714 else 715 sgt_buf = sgt_cache->buf[--sgt_cache->count]; 716 if (unlikely(!sgt_buf)) 717 return -ENOMEM; 718 719 sgt_buf = PTR_ALIGN(sgt_buf, DPAA2_ETH_TX_BUF_ALIGN); 720 sgt = (struct dpaa2_sg_entry *)(sgt_buf + priv->tx_data_offset); 721 722 addr = dma_map_single(dev, skb->data, skb->len, DMA_BIDIRECTIONAL); 723 if (unlikely(dma_mapping_error(dev, addr))) { 724 err = -ENOMEM; 725 goto data_map_failed; 726 } 727 728 /* Fill in the HW SGT structure */ 729 dpaa2_sg_set_addr(sgt, addr); 730 dpaa2_sg_set_len(sgt, skb->len); 731 dpaa2_sg_set_final(sgt, true); 732 733 /* Store the skb backpointer in the SGT buffer */ 734 swa = (struct dpaa2_eth_swa *)sgt_buf; 735 swa->type = DPAA2_ETH_SWA_SINGLE; 736 swa->single.skb = skb; 737 swa->sg.sgt_size = sgt_buf_size; 738 739 /* Separately map the SGT buffer */ 740 sgt_addr = dma_map_single(dev, sgt_buf, sgt_buf_size, DMA_BIDIRECTIONAL); 741 if (unlikely(dma_mapping_error(dev, sgt_addr))) { 742 err = -ENOMEM; 743 goto sgt_map_failed; 744 } 745 746 dpaa2_fd_set_offset(fd, priv->tx_data_offset); 747 dpaa2_fd_set_format(fd, dpaa2_fd_sg); 748 dpaa2_fd_set_addr(fd, sgt_addr); 749 dpaa2_fd_set_len(fd, skb->len); 750 dpaa2_fd_set_ctrl(fd, FD_CTRL_PTA); 751 752 if (priv->tx_tstamp && skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) 753 enable_tx_tstamp(fd, sgt_buf); 754 755 return 0; 756 757 sgt_map_failed: 758 dma_unmap_single(dev, addr, skb->len, DMA_BIDIRECTIONAL); 759 data_map_failed: 760 if (sgt_cache->count >= DPAA2_ETH_SGT_CACHE_SIZE) 761 kfree(sgt_buf); 762 else 763 sgt_cache->buf[sgt_cache->count++] = sgt_buf; 764 765 return err; 766 } 767 768 /* Create a frame descriptor based on a linear skb */ 769 static int build_single_fd(struct dpaa2_eth_priv *priv, 770 struct sk_buff *skb, 771 struct dpaa2_fd *fd) 772 { 773 struct device *dev = priv->net_dev->dev.parent; 774 u8 *buffer_start, *aligned_start; 775 struct dpaa2_eth_swa *swa; 776 dma_addr_t addr; 777 778 buffer_start = skb->data - dpaa2_eth_needed_headroom(priv, skb); 779 780 /* If there's enough room to align the FD address, do it. 781 * It will help hardware optimize accesses. 782 */ 783 aligned_start = PTR_ALIGN(buffer_start - DPAA2_ETH_TX_BUF_ALIGN, 784 DPAA2_ETH_TX_BUF_ALIGN); 785 if (aligned_start >= skb->head) 786 buffer_start = aligned_start; 787 788 /* Store a backpointer to the skb at the beginning of the buffer 789 * (in the private data area) such that we can release it 790 * on Tx confirm 791 */ 792 swa = (struct dpaa2_eth_swa *)buffer_start; 793 swa->type = DPAA2_ETH_SWA_SINGLE; 794 swa->single.skb = skb; 795 796 addr = dma_map_single(dev, buffer_start, 797 skb_tail_pointer(skb) - buffer_start, 798 DMA_BIDIRECTIONAL); 799 if (unlikely(dma_mapping_error(dev, addr))) 800 return -ENOMEM; 801 802 dpaa2_fd_set_addr(fd, addr); 803 dpaa2_fd_set_offset(fd, (u16)(skb->data - buffer_start)); 804 dpaa2_fd_set_len(fd, skb->len); 805 dpaa2_fd_set_format(fd, dpaa2_fd_single); 806 dpaa2_fd_set_ctrl(fd, FD_CTRL_PTA); 807 808 if (priv->tx_tstamp && skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) 809 enable_tx_tstamp(fd, buffer_start); 810 811 return 0; 812 } 813 814 /* FD freeing routine on the Tx path 815 * 816 * DMA-unmap and free FD and possibly SGT buffer allocated on Tx. The skb 817 * back-pointed to is also freed. 818 * This can be called either from dpaa2_eth_tx_conf() or on the error path of 819 * dpaa2_eth_tx(). 820 */ 821 static void free_tx_fd(const struct dpaa2_eth_priv *priv, 822 struct dpaa2_eth_fq *fq, 823 const struct dpaa2_fd *fd, bool in_napi) 824 { 825 struct device *dev = priv->net_dev->dev.parent; 826 dma_addr_t fd_addr, sg_addr; 827 struct sk_buff *skb = NULL; 828 unsigned char *buffer_start; 829 struct dpaa2_eth_swa *swa; 830 u8 fd_format = dpaa2_fd_get_format(fd); 831 u32 fd_len = dpaa2_fd_get_len(fd); 832 833 struct dpaa2_eth_sgt_cache *sgt_cache; 834 struct dpaa2_sg_entry *sgt; 835 836 fd_addr = dpaa2_fd_get_addr(fd); 837 buffer_start = dpaa2_iova_to_virt(priv->iommu_domain, fd_addr); 838 swa = (struct dpaa2_eth_swa *)buffer_start; 839 840 if (fd_format == dpaa2_fd_single) { 841 if (swa->type == DPAA2_ETH_SWA_SINGLE) { 842 skb = swa->single.skb; 843 /* Accessing the skb buffer is safe before dma unmap, 844 * because we didn't map the actual skb shell. 845 */ 846 dma_unmap_single(dev, fd_addr, 847 skb_tail_pointer(skb) - buffer_start, 848 DMA_BIDIRECTIONAL); 849 } else { 850 WARN_ONCE(swa->type != DPAA2_ETH_SWA_XDP, "Wrong SWA type"); 851 dma_unmap_single(dev, fd_addr, swa->xdp.dma_size, 852 DMA_BIDIRECTIONAL); 853 } 854 } else if (fd_format == dpaa2_fd_sg) { 855 if (swa->type == DPAA2_ETH_SWA_SG) { 856 skb = swa->sg.skb; 857 858 /* Unmap the scatterlist */ 859 dma_unmap_sg(dev, swa->sg.scl, swa->sg.num_sg, 860 DMA_BIDIRECTIONAL); 861 kfree(swa->sg.scl); 862 863 /* Unmap the SGT buffer */ 864 dma_unmap_single(dev, fd_addr, swa->sg.sgt_size, 865 DMA_BIDIRECTIONAL); 866 } else { 867 skb = swa->single.skb; 868 869 /* Unmap the SGT Buffer */ 870 dma_unmap_single(dev, fd_addr, swa->single.sgt_size, 871 DMA_BIDIRECTIONAL); 872 873 sgt = (struct dpaa2_sg_entry *)(buffer_start + 874 priv->tx_data_offset); 875 sg_addr = dpaa2_sg_get_addr(sgt); 876 dma_unmap_single(dev, sg_addr, skb->len, DMA_BIDIRECTIONAL); 877 } 878 } else { 879 netdev_dbg(priv->net_dev, "Invalid FD format\n"); 880 return; 881 } 882 883 if (swa->type != DPAA2_ETH_SWA_XDP && in_napi) { 884 fq->dq_frames++; 885 fq->dq_bytes += fd_len; 886 } 887 888 if (swa->type == DPAA2_ETH_SWA_XDP) { 889 xdp_return_frame(swa->xdp.xdpf); 890 return; 891 } 892 893 /* Get the timestamp value */ 894 if (priv->tx_tstamp && skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) { 895 struct skb_shared_hwtstamps shhwtstamps; 896 __le64 *ts = dpaa2_get_ts(buffer_start, true); 897 u64 ns; 898 899 memset(&shhwtstamps, 0, sizeof(shhwtstamps)); 900 901 ns = DPAA2_PTP_CLK_PERIOD_NS * le64_to_cpup(ts); 902 shhwtstamps.hwtstamp = ns_to_ktime(ns); 903 skb_tstamp_tx(skb, &shhwtstamps); 904 } 905 906 /* Free SGT buffer allocated on tx */ 907 if (fd_format != dpaa2_fd_single) { 908 sgt_cache = this_cpu_ptr(priv->sgt_cache); 909 if (swa->type == DPAA2_ETH_SWA_SG) { 910 skb_free_frag(buffer_start); 911 } else { 912 if (sgt_cache->count >= DPAA2_ETH_SGT_CACHE_SIZE) 913 kfree(buffer_start); 914 else 915 sgt_cache->buf[sgt_cache->count++] = buffer_start; 916 } 917 } 918 919 /* Move on with skb release */ 920 napi_consume_skb(skb, in_napi); 921 } 922 923 static netdev_tx_t dpaa2_eth_tx(struct sk_buff *skb, struct net_device *net_dev) 924 { 925 struct dpaa2_eth_priv *priv = netdev_priv(net_dev); 926 struct dpaa2_fd fd; 927 struct rtnl_link_stats64 *percpu_stats; 928 struct dpaa2_eth_drv_stats *percpu_extras; 929 struct dpaa2_eth_fq *fq; 930 struct netdev_queue *nq; 931 u16 queue_mapping; 932 unsigned int needed_headroom; 933 u32 fd_len; 934 u8 prio = 0; 935 int err, i; 936 937 percpu_stats = this_cpu_ptr(priv->percpu_stats); 938 percpu_extras = this_cpu_ptr(priv->percpu_extras); 939 940 needed_headroom = dpaa2_eth_needed_headroom(priv, skb); 941 942 /* We'll be holding a back-reference to the skb until Tx Confirmation; 943 * we don't want that overwritten by a concurrent Tx with a cloned skb. 944 */ 945 skb = skb_unshare(skb, GFP_ATOMIC); 946 if (unlikely(!skb)) { 947 /* skb_unshare() has already freed the skb */ 948 percpu_stats->tx_dropped++; 949 return NETDEV_TX_OK; 950 } 951 952 /* Setup the FD fields */ 953 memset(&fd, 0, sizeof(fd)); 954 955 if (skb_is_nonlinear(skb)) { 956 err = build_sg_fd(priv, skb, &fd); 957 percpu_extras->tx_sg_frames++; 958 percpu_extras->tx_sg_bytes += skb->len; 959 } else if (skb_headroom(skb) < needed_headroom) { 960 err = build_sg_fd_single_buf(priv, skb, &fd); 961 percpu_extras->tx_sg_frames++; 962 percpu_extras->tx_sg_bytes += skb->len; 963 percpu_extras->tx_converted_sg_frames++; 964 percpu_extras->tx_converted_sg_bytes += skb->len; 965 } else { 966 err = build_single_fd(priv, skb, &fd); 967 } 968 969 if (unlikely(err)) { 970 percpu_stats->tx_dropped++; 971 goto err_build_fd; 972 } 973 974 /* Tracing point */ 975 trace_dpaa2_tx_fd(net_dev, &fd); 976 977 /* TxConf FQ selection relies on queue id from the stack. 978 * In case of a forwarded frame from another DPNI interface, we choose 979 * a queue affined to the same core that processed the Rx frame 980 */ 981 queue_mapping = skb_get_queue_mapping(skb); 982 983 if (net_dev->num_tc) { 984 prio = netdev_txq_to_tc(net_dev, queue_mapping); 985 /* Hardware interprets priority level 0 as being the highest, 986 * so we need to do a reverse mapping to the netdev tc index 987 */ 988 prio = net_dev->num_tc - prio - 1; 989 /* We have only one FQ array entry for all Tx hardware queues 990 * with the same flow id (but different priority levels) 991 */ 992 queue_mapping %= dpaa2_eth_queue_count(priv); 993 } 994 fq = &priv->fq[queue_mapping]; 995 996 fd_len = dpaa2_fd_get_len(&fd); 997 nq = netdev_get_tx_queue(net_dev, queue_mapping); 998 netdev_tx_sent_queue(nq, fd_len); 999 1000 /* Everything that happens after this enqueues might race with 1001 * the Tx confirmation callback for this frame 1002 */ 1003 for (i = 0; i < DPAA2_ETH_ENQUEUE_RETRIES; i++) { 1004 err = priv->enqueue(priv, fq, &fd, prio, 1, NULL); 1005 if (err != -EBUSY) 1006 break; 1007 } 1008 percpu_extras->tx_portal_busy += i; 1009 if (unlikely(err < 0)) { 1010 percpu_stats->tx_errors++; 1011 /* Clean up everything, including freeing the skb */ 1012 free_tx_fd(priv, fq, &fd, false); 1013 netdev_tx_completed_queue(nq, 1, fd_len); 1014 } else { 1015 percpu_stats->tx_packets++; 1016 percpu_stats->tx_bytes += fd_len; 1017 } 1018 1019 return NETDEV_TX_OK; 1020 1021 err_build_fd: 1022 dev_kfree_skb(skb); 1023 1024 return NETDEV_TX_OK; 1025 } 1026 1027 /* Tx confirmation frame processing routine */ 1028 static void dpaa2_eth_tx_conf(struct dpaa2_eth_priv *priv, 1029 struct dpaa2_eth_channel *ch __always_unused, 1030 const struct dpaa2_fd *fd, 1031 struct dpaa2_eth_fq *fq) 1032 { 1033 struct rtnl_link_stats64 *percpu_stats; 1034 struct dpaa2_eth_drv_stats *percpu_extras; 1035 u32 fd_len = dpaa2_fd_get_len(fd); 1036 u32 fd_errors; 1037 1038 /* Tracing point */ 1039 trace_dpaa2_tx_conf_fd(priv->net_dev, fd); 1040 1041 percpu_extras = this_cpu_ptr(priv->percpu_extras); 1042 percpu_extras->tx_conf_frames++; 1043 percpu_extras->tx_conf_bytes += fd_len; 1044 1045 /* Check frame errors in the FD field */ 1046 fd_errors = dpaa2_fd_get_ctrl(fd) & DPAA2_FD_TX_ERR_MASK; 1047 free_tx_fd(priv, fq, fd, true); 1048 1049 if (likely(!fd_errors)) 1050 return; 1051 1052 if (net_ratelimit()) 1053 netdev_dbg(priv->net_dev, "TX frame FD error: 0x%08x\n", 1054 fd_errors); 1055 1056 percpu_stats = this_cpu_ptr(priv->percpu_stats); 1057 /* Tx-conf logically pertains to the egress path. */ 1058 percpu_stats->tx_errors++; 1059 } 1060 1061 static int set_rx_csum(struct dpaa2_eth_priv *priv, bool enable) 1062 { 1063 int err; 1064 1065 err = dpni_set_offload(priv->mc_io, 0, priv->mc_token, 1066 DPNI_OFF_RX_L3_CSUM, enable); 1067 if (err) { 1068 netdev_err(priv->net_dev, 1069 "dpni_set_offload(RX_L3_CSUM) failed\n"); 1070 return err; 1071 } 1072 1073 err = dpni_set_offload(priv->mc_io, 0, priv->mc_token, 1074 DPNI_OFF_RX_L4_CSUM, enable); 1075 if (err) { 1076 netdev_err(priv->net_dev, 1077 "dpni_set_offload(RX_L4_CSUM) failed\n"); 1078 return err; 1079 } 1080 1081 return 0; 1082 } 1083 1084 static int set_tx_csum(struct dpaa2_eth_priv *priv, bool enable) 1085 { 1086 int err; 1087 1088 err = dpni_set_offload(priv->mc_io, 0, priv->mc_token, 1089 DPNI_OFF_TX_L3_CSUM, enable); 1090 if (err) { 1091 netdev_err(priv->net_dev, "dpni_set_offload(TX_L3_CSUM) failed\n"); 1092 return err; 1093 } 1094 1095 err = dpni_set_offload(priv->mc_io, 0, priv->mc_token, 1096 DPNI_OFF_TX_L4_CSUM, enable); 1097 if (err) { 1098 netdev_err(priv->net_dev, "dpni_set_offload(TX_L4_CSUM) failed\n"); 1099 return err; 1100 } 1101 1102 return 0; 1103 } 1104 1105 /* Perform a single release command to add buffers 1106 * to the specified buffer pool 1107 */ 1108 static int add_bufs(struct dpaa2_eth_priv *priv, 1109 struct dpaa2_eth_channel *ch, u16 bpid) 1110 { 1111 struct device *dev = priv->net_dev->dev.parent; 1112 u64 buf_array[DPAA2_ETH_BUFS_PER_CMD]; 1113 struct page *page; 1114 dma_addr_t addr; 1115 int retries = 0; 1116 int i, err; 1117 1118 for (i = 0; i < DPAA2_ETH_BUFS_PER_CMD; i++) { 1119 /* Allocate buffer visible to WRIOP + skb shared info + 1120 * alignment padding 1121 */ 1122 /* allocate one page for each Rx buffer. WRIOP sees 1123 * the entire page except for a tailroom reserved for 1124 * skb shared info 1125 */ 1126 page = dev_alloc_pages(0); 1127 if (!page) 1128 goto err_alloc; 1129 1130 addr = dma_map_page(dev, page, 0, priv->rx_buf_size, 1131 DMA_BIDIRECTIONAL); 1132 if (unlikely(dma_mapping_error(dev, addr))) 1133 goto err_map; 1134 1135 buf_array[i] = addr; 1136 1137 /* tracing point */ 1138 trace_dpaa2_eth_buf_seed(priv->net_dev, 1139 page, DPAA2_ETH_RX_BUF_RAW_SIZE, 1140 addr, priv->rx_buf_size, 1141 bpid); 1142 } 1143 1144 release_bufs: 1145 /* In case the portal is busy, retry until successful */ 1146 while ((err = dpaa2_io_service_release(ch->dpio, bpid, 1147 buf_array, i)) == -EBUSY) { 1148 if (retries++ >= DPAA2_ETH_SWP_BUSY_RETRIES) 1149 break; 1150 cpu_relax(); 1151 } 1152 1153 /* If release command failed, clean up and bail out; 1154 * not much else we can do about it 1155 */ 1156 if (err) { 1157 free_bufs(priv, buf_array, i); 1158 return 0; 1159 } 1160 1161 return i; 1162 1163 err_map: 1164 __free_pages(page, 0); 1165 err_alloc: 1166 /* If we managed to allocate at least some buffers, 1167 * release them to hardware 1168 */ 1169 if (i) 1170 goto release_bufs; 1171 1172 return 0; 1173 } 1174 1175 static int seed_pool(struct dpaa2_eth_priv *priv, u16 bpid) 1176 { 1177 int i, j; 1178 int new_count; 1179 1180 for (j = 0; j < priv->num_channels; j++) { 1181 for (i = 0; i < DPAA2_ETH_NUM_BUFS; 1182 i += DPAA2_ETH_BUFS_PER_CMD) { 1183 new_count = add_bufs(priv, priv->channel[j], bpid); 1184 priv->channel[j]->buf_count += new_count; 1185 1186 if (new_count < DPAA2_ETH_BUFS_PER_CMD) { 1187 return -ENOMEM; 1188 } 1189 } 1190 } 1191 1192 return 0; 1193 } 1194 1195 /** 1196 * Drain the specified number of buffers from the DPNI's private buffer pool. 1197 * @count must not exceeed DPAA2_ETH_BUFS_PER_CMD 1198 */ 1199 static void drain_bufs(struct dpaa2_eth_priv *priv, int count) 1200 { 1201 u64 buf_array[DPAA2_ETH_BUFS_PER_CMD]; 1202 int retries = 0; 1203 int ret; 1204 1205 do { 1206 ret = dpaa2_io_service_acquire(NULL, priv->bpid, 1207 buf_array, count); 1208 if (ret < 0) { 1209 if (ret == -EBUSY && 1210 retries++ < DPAA2_ETH_SWP_BUSY_RETRIES) 1211 continue; 1212 netdev_err(priv->net_dev, "dpaa2_io_service_acquire() failed\n"); 1213 return; 1214 } 1215 free_bufs(priv, buf_array, ret); 1216 retries = 0; 1217 } while (ret); 1218 } 1219 1220 static void drain_pool(struct dpaa2_eth_priv *priv) 1221 { 1222 int i; 1223 1224 drain_bufs(priv, DPAA2_ETH_BUFS_PER_CMD); 1225 drain_bufs(priv, 1); 1226 1227 for (i = 0; i < priv->num_channels; i++) 1228 priv->channel[i]->buf_count = 0; 1229 } 1230 1231 /* Function is called from softirq context only, so we don't need to guard 1232 * the access to percpu count 1233 */ 1234 static int refill_pool(struct dpaa2_eth_priv *priv, 1235 struct dpaa2_eth_channel *ch, 1236 u16 bpid) 1237 { 1238 int new_count; 1239 1240 if (likely(ch->buf_count >= DPAA2_ETH_REFILL_THRESH)) 1241 return 0; 1242 1243 do { 1244 new_count = add_bufs(priv, ch, bpid); 1245 if (unlikely(!new_count)) { 1246 /* Out of memory; abort for now, we'll try later on */ 1247 break; 1248 } 1249 ch->buf_count += new_count; 1250 } while (ch->buf_count < DPAA2_ETH_NUM_BUFS); 1251 1252 if (unlikely(ch->buf_count < DPAA2_ETH_NUM_BUFS)) 1253 return -ENOMEM; 1254 1255 return 0; 1256 } 1257 1258 static void dpaa2_eth_sgt_cache_drain(struct dpaa2_eth_priv *priv) 1259 { 1260 struct dpaa2_eth_sgt_cache *sgt_cache; 1261 u16 count; 1262 int k, i; 1263 1264 for_each_possible_cpu(k) { 1265 sgt_cache = per_cpu_ptr(priv->sgt_cache, k); 1266 count = sgt_cache->count; 1267 1268 for (i = 0; i < count; i++) 1269 kfree(sgt_cache->buf[i]); 1270 sgt_cache->count = 0; 1271 } 1272 } 1273 1274 static int pull_channel(struct dpaa2_eth_channel *ch) 1275 { 1276 int err; 1277 int dequeues = -1; 1278 1279 /* Retry while portal is busy */ 1280 do { 1281 err = dpaa2_io_service_pull_channel(ch->dpio, ch->ch_id, 1282 ch->store); 1283 dequeues++; 1284 cpu_relax(); 1285 } while (err == -EBUSY && dequeues < DPAA2_ETH_SWP_BUSY_RETRIES); 1286 1287 ch->stats.dequeue_portal_busy += dequeues; 1288 if (unlikely(err)) 1289 ch->stats.pull_err++; 1290 1291 return err; 1292 } 1293 1294 /* NAPI poll routine 1295 * 1296 * Frames are dequeued from the QMan channel associated with this NAPI context. 1297 * Rx, Tx confirmation and (if configured) Rx error frames all count 1298 * towards the NAPI budget. 1299 */ 1300 static int dpaa2_eth_poll(struct napi_struct *napi, int budget) 1301 { 1302 struct dpaa2_eth_channel *ch; 1303 struct dpaa2_eth_priv *priv; 1304 int rx_cleaned = 0, txconf_cleaned = 0; 1305 struct dpaa2_eth_fq *fq, *txc_fq = NULL; 1306 struct netdev_queue *nq; 1307 int store_cleaned, work_done; 1308 struct list_head rx_list; 1309 int retries = 0; 1310 u16 flowid; 1311 int err; 1312 1313 ch = container_of(napi, struct dpaa2_eth_channel, napi); 1314 ch->xdp.res = 0; 1315 priv = ch->priv; 1316 1317 INIT_LIST_HEAD(&rx_list); 1318 ch->rx_list = &rx_list; 1319 1320 do { 1321 err = pull_channel(ch); 1322 if (unlikely(err)) 1323 break; 1324 1325 /* Refill pool if appropriate */ 1326 refill_pool(priv, ch, priv->bpid); 1327 1328 store_cleaned = consume_frames(ch, &fq); 1329 if (store_cleaned <= 0) 1330 break; 1331 if (fq->type == DPAA2_RX_FQ) { 1332 rx_cleaned += store_cleaned; 1333 flowid = fq->flowid; 1334 } else { 1335 txconf_cleaned += store_cleaned; 1336 /* We have a single Tx conf FQ on this channel */ 1337 txc_fq = fq; 1338 } 1339 1340 /* If we either consumed the whole NAPI budget with Rx frames 1341 * or we reached the Tx confirmations threshold, we're done. 1342 */ 1343 if (rx_cleaned >= budget || 1344 txconf_cleaned >= DPAA2_ETH_TXCONF_PER_NAPI) { 1345 work_done = budget; 1346 goto out; 1347 } 1348 } while (store_cleaned); 1349 1350 /* We didn't consume the entire budget, so finish napi and 1351 * re-enable data availability notifications 1352 */ 1353 napi_complete_done(napi, rx_cleaned); 1354 do { 1355 err = dpaa2_io_service_rearm(ch->dpio, &ch->nctx); 1356 cpu_relax(); 1357 } while (err == -EBUSY && retries++ < DPAA2_ETH_SWP_BUSY_RETRIES); 1358 WARN_ONCE(err, "CDAN notifications rearm failed on core %d", 1359 ch->nctx.desired_cpu); 1360 1361 work_done = max(rx_cleaned, 1); 1362 1363 out: 1364 netif_receive_skb_list(ch->rx_list); 1365 1366 if (txc_fq && txc_fq->dq_frames) { 1367 nq = netdev_get_tx_queue(priv->net_dev, txc_fq->flowid); 1368 netdev_tx_completed_queue(nq, txc_fq->dq_frames, 1369 txc_fq->dq_bytes); 1370 txc_fq->dq_frames = 0; 1371 txc_fq->dq_bytes = 0; 1372 } 1373 1374 if (ch->xdp.res & XDP_REDIRECT) 1375 xdp_do_flush_map(); 1376 else if (rx_cleaned && ch->xdp.res & XDP_TX) 1377 xdp_tx_flush(priv, ch, &priv->fq[flowid]); 1378 1379 return work_done; 1380 } 1381 1382 static void enable_ch_napi(struct dpaa2_eth_priv *priv) 1383 { 1384 struct dpaa2_eth_channel *ch; 1385 int i; 1386 1387 for (i = 0; i < priv->num_channels; i++) { 1388 ch = priv->channel[i]; 1389 napi_enable(&ch->napi); 1390 } 1391 } 1392 1393 static void disable_ch_napi(struct dpaa2_eth_priv *priv) 1394 { 1395 struct dpaa2_eth_channel *ch; 1396 int i; 1397 1398 for (i = 0; i < priv->num_channels; i++) { 1399 ch = priv->channel[i]; 1400 napi_disable(&ch->napi); 1401 } 1402 } 1403 1404 void dpaa2_eth_set_rx_taildrop(struct dpaa2_eth_priv *priv, 1405 bool tx_pause, bool pfc) 1406 { 1407 struct dpni_taildrop td = {0}; 1408 struct dpaa2_eth_fq *fq; 1409 int i, err; 1410 1411 /* FQ taildrop: threshold is in bytes, per frame queue. Enabled if 1412 * flow control is disabled (as it might interfere with either the 1413 * buffer pool depletion trigger for pause frames or with the group 1414 * congestion trigger for PFC frames) 1415 */ 1416 td.enable = !tx_pause; 1417 if (priv->rx_fqtd_enabled == td.enable) 1418 goto set_cgtd; 1419 1420 td.threshold = DPAA2_ETH_FQ_TAILDROP_THRESH; 1421 td.units = DPNI_CONGESTION_UNIT_BYTES; 1422 1423 for (i = 0; i < priv->num_fqs; i++) { 1424 fq = &priv->fq[i]; 1425 if (fq->type != DPAA2_RX_FQ) 1426 continue; 1427 err = dpni_set_taildrop(priv->mc_io, 0, priv->mc_token, 1428 DPNI_CP_QUEUE, DPNI_QUEUE_RX, 1429 fq->tc, fq->flowid, &td); 1430 if (err) { 1431 netdev_err(priv->net_dev, 1432 "dpni_set_taildrop(FQ) failed\n"); 1433 return; 1434 } 1435 } 1436 1437 priv->rx_fqtd_enabled = td.enable; 1438 1439 set_cgtd: 1440 /* Congestion group taildrop: threshold is in frames, per group 1441 * of FQs belonging to the same traffic class 1442 * Enabled if general Tx pause disabled or if PFCs are enabled 1443 * (congestion group threhsold for PFC generation is lower than the 1444 * CG taildrop threshold, so it won't interfere with it; we also 1445 * want frames in non-PFC enabled traffic classes to be kept in check) 1446 */ 1447 td.enable = !tx_pause || (tx_pause && pfc); 1448 if (priv->rx_cgtd_enabled == td.enable) 1449 return; 1450 1451 td.threshold = DPAA2_ETH_CG_TAILDROP_THRESH(priv); 1452 td.units = DPNI_CONGESTION_UNIT_FRAMES; 1453 for (i = 0; i < dpaa2_eth_tc_count(priv); i++) { 1454 err = dpni_set_taildrop(priv->mc_io, 0, priv->mc_token, 1455 DPNI_CP_GROUP, DPNI_QUEUE_RX, 1456 i, 0, &td); 1457 if (err) { 1458 netdev_err(priv->net_dev, 1459 "dpni_set_taildrop(CG) failed\n"); 1460 return; 1461 } 1462 } 1463 1464 priv->rx_cgtd_enabled = td.enable; 1465 } 1466 1467 static int link_state_update(struct dpaa2_eth_priv *priv) 1468 { 1469 struct dpni_link_state state = {0}; 1470 bool tx_pause; 1471 int err; 1472 1473 err = dpni_get_link_state(priv->mc_io, 0, priv->mc_token, &state); 1474 if (unlikely(err)) { 1475 netdev_err(priv->net_dev, 1476 "dpni_get_link_state() failed\n"); 1477 return err; 1478 } 1479 1480 /* If Tx pause frame settings have changed, we need to update 1481 * Rx FQ taildrop configuration as well. We configure taildrop 1482 * only when pause frame generation is disabled. 1483 */ 1484 tx_pause = dpaa2_eth_tx_pause_enabled(state.options); 1485 dpaa2_eth_set_rx_taildrop(priv, tx_pause, priv->pfc_enabled); 1486 1487 /* When we manage the MAC/PHY using phylink there is no need 1488 * to manually update the netif_carrier. 1489 */ 1490 if (priv->mac) 1491 goto out; 1492 1493 /* Chech link state; speed / duplex changes are not treated yet */ 1494 if (priv->link_state.up == state.up) 1495 goto out; 1496 1497 if (state.up) { 1498 netif_carrier_on(priv->net_dev); 1499 netif_tx_start_all_queues(priv->net_dev); 1500 } else { 1501 netif_tx_stop_all_queues(priv->net_dev); 1502 netif_carrier_off(priv->net_dev); 1503 } 1504 1505 netdev_info(priv->net_dev, "Link Event: state %s\n", 1506 state.up ? "up" : "down"); 1507 1508 out: 1509 priv->link_state = state; 1510 1511 return 0; 1512 } 1513 1514 static int dpaa2_eth_open(struct net_device *net_dev) 1515 { 1516 struct dpaa2_eth_priv *priv = netdev_priv(net_dev); 1517 int err; 1518 1519 err = seed_pool(priv, priv->bpid); 1520 if (err) { 1521 /* Not much to do; the buffer pool, though not filled up, 1522 * may still contain some buffers which would enable us 1523 * to limp on. 1524 */ 1525 netdev_err(net_dev, "Buffer seeding failed for DPBP %d (bpid=%d)\n", 1526 priv->dpbp_dev->obj_desc.id, priv->bpid); 1527 } 1528 1529 if (!priv->mac) { 1530 /* We'll only start the txqs when the link is actually ready; 1531 * make sure we don't race against the link up notification, 1532 * which may come immediately after dpni_enable(); 1533 */ 1534 netif_tx_stop_all_queues(net_dev); 1535 1536 /* Also, explicitly set carrier off, otherwise 1537 * netif_carrier_ok() will return true and cause 'ip link show' 1538 * to report the LOWER_UP flag, even though the link 1539 * notification wasn't even received. 1540 */ 1541 netif_carrier_off(net_dev); 1542 } 1543 enable_ch_napi(priv); 1544 1545 err = dpni_enable(priv->mc_io, 0, priv->mc_token); 1546 if (err < 0) { 1547 netdev_err(net_dev, "dpni_enable() failed\n"); 1548 goto enable_err; 1549 } 1550 1551 if (!priv->mac) { 1552 /* If the DPMAC object has already processed the link up 1553 * interrupt, we have to learn the link state ourselves. 1554 */ 1555 err = link_state_update(priv); 1556 if (err < 0) { 1557 netdev_err(net_dev, "Can't update link state\n"); 1558 goto link_state_err; 1559 } 1560 } else { 1561 phylink_start(priv->mac->phylink); 1562 } 1563 1564 return 0; 1565 1566 link_state_err: 1567 enable_err: 1568 disable_ch_napi(priv); 1569 drain_pool(priv); 1570 return err; 1571 } 1572 1573 /* Total number of in-flight frames on ingress queues */ 1574 static u32 ingress_fq_count(struct dpaa2_eth_priv *priv) 1575 { 1576 struct dpaa2_eth_fq *fq; 1577 u32 fcnt = 0, bcnt = 0, total = 0; 1578 int i, err; 1579 1580 for (i = 0; i < priv->num_fqs; i++) { 1581 fq = &priv->fq[i]; 1582 err = dpaa2_io_query_fq_count(NULL, fq->fqid, &fcnt, &bcnt); 1583 if (err) { 1584 netdev_warn(priv->net_dev, "query_fq_count failed"); 1585 break; 1586 } 1587 total += fcnt; 1588 } 1589 1590 return total; 1591 } 1592 1593 static void wait_for_ingress_fq_empty(struct dpaa2_eth_priv *priv) 1594 { 1595 int retries = 10; 1596 u32 pending; 1597 1598 do { 1599 pending = ingress_fq_count(priv); 1600 if (pending) 1601 msleep(100); 1602 } while (pending && --retries); 1603 } 1604 1605 #define DPNI_TX_PENDING_VER_MAJOR 7 1606 #define DPNI_TX_PENDING_VER_MINOR 13 1607 static void wait_for_egress_fq_empty(struct dpaa2_eth_priv *priv) 1608 { 1609 union dpni_statistics stats; 1610 int retries = 10; 1611 int err; 1612 1613 if (dpaa2_eth_cmp_dpni_ver(priv, DPNI_TX_PENDING_VER_MAJOR, 1614 DPNI_TX_PENDING_VER_MINOR) < 0) 1615 goto out; 1616 1617 do { 1618 err = dpni_get_statistics(priv->mc_io, 0, priv->mc_token, 6, 1619 &stats); 1620 if (err) 1621 goto out; 1622 if (stats.page_6.tx_pending_frames == 0) 1623 return; 1624 } while (--retries); 1625 1626 out: 1627 msleep(500); 1628 } 1629 1630 static int dpaa2_eth_stop(struct net_device *net_dev) 1631 { 1632 struct dpaa2_eth_priv *priv = netdev_priv(net_dev); 1633 int dpni_enabled = 0; 1634 int retries = 10; 1635 1636 if (!priv->mac) { 1637 netif_tx_stop_all_queues(net_dev); 1638 netif_carrier_off(net_dev); 1639 } else { 1640 phylink_stop(priv->mac->phylink); 1641 } 1642 1643 /* On dpni_disable(), the MC firmware will: 1644 * - stop MAC Rx and wait for all Rx frames to be enqueued to software 1645 * - cut off WRIOP dequeues from egress FQs and wait until transmission 1646 * of all in flight Tx frames is finished (and corresponding Tx conf 1647 * frames are enqueued back to software) 1648 * 1649 * Before calling dpni_disable(), we wait for all Tx frames to arrive 1650 * on WRIOP. After it finishes, wait until all remaining frames on Rx 1651 * and Tx conf queues are consumed on NAPI poll. 1652 */ 1653 wait_for_egress_fq_empty(priv); 1654 1655 do { 1656 dpni_disable(priv->mc_io, 0, priv->mc_token); 1657 dpni_is_enabled(priv->mc_io, 0, priv->mc_token, &dpni_enabled); 1658 if (dpni_enabled) 1659 /* Allow the hardware some slack */ 1660 msleep(100); 1661 } while (dpni_enabled && --retries); 1662 if (!retries) { 1663 netdev_warn(net_dev, "Retry count exceeded disabling DPNI\n"); 1664 /* Must go on and disable NAPI nonetheless, so we don't crash at 1665 * the next "ifconfig up" 1666 */ 1667 } 1668 1669 wait_for_ingress_fq_empty(priv); 1670 disable_ch_napi(priv); 1671 1672 /* Empty the buffer pool */ 1673 drain_pool(priv); 1674 1675 /* Empty the Scatter-Gather Buffer cache */ 1676 dpaa2_eth_sgt_cache_drain(priv); 1677 1678 return 0; 1679 } 1680 1681 static int dpaa2_eth_set_addr(struct net_device *net_dev, void *addr) 1682 { 1683 struct dpaa2_eth_priv *priv = netdev_priv(net_dev); 1684 struct device *dev = net_dev->dev.parent; 1685 int err; 1686 1687 err = eth_mac_addr(net_dev, addr); 1688 if (err < 0) { 1689 dev_err(dev, "eth_mac_addr() failed (%d)\n", err); 1690 return err; 1691 } 1692 1693 err = dpni_set_primary_mac_addr(priv->mc_io, 0, priv->mc_token, 1694 net_dev->dev_addr); 1695 if (err) { 1696 dev_err(dev, "dpni_set_primary_mac_addr() failed (%d)\n", err); 1697 return err; 1698 } 1699 1700 return 0; 1701 } 1702 1703 /** Fill in counters maintained by the GPP driver. These may be different from 1704 * the hardware counters obtained by ethtool. 1705 */ 1706 static void dpaa2_eth_get_stats(struct net_device *net_dev, 1707 struct rtnl_link_stats64 *stats) 1708 { 1709 struct dpaa2_eth_priv *priv = netdev_priv(net_dev); 1710 struct rtnl_link_stats64 *percpu_stats; 1711 u64 *cpustats; 1712 u64 *netstats = (u64 *)stats; 1713 int i, j; 1714 int num = sizeof(struct rtnl_link_stats64) / sizeof(u64); 1715 1716 for_each_possible_cpu(i) { 1717 percpu_stats = per_cpu_ptr(priv->percpu_stats, i); 1718 cpustats = (u64 *)percpu_stats; 1719 for (j = 0; j < num; j++) 1720 netstats[j] += cpustats[j]; 1721 } 1722 } 1723 1724 /* Copy mac unicast addresses from @net_dev to @priv. 1725 * Its sole purpose is to make dpaa2_eth_set_rx_mode() more readable. 1726 */ 1727 static void add_uc_hw_addr(const struct net_device *net_dev, 1728 struct dpaa2_eth_priv *priv) 1729 { 1730 struct netdev_hw_addr *ha; 1731 int err; 1732 1733 netdev_for_each_uc_addr(ha, net_dev) { 1734 err = dpni_add_mac_addr(priv->mc_io, 0, priv->mc_token, 1735 ha->addr); 1736 if (err) 1737 netdev_warn(priv->net_dev, 1738 "Could not add ucast MAC %pM to the filtering table (err %d)\n", 1739 ha->addr, err); 1740 } 1741 } 1742 1743 /* Copy mac multicast addresses from @net_dev to @priv 1744 * Its sole purpose is to make dpaa2_eth_set_rx_mode() more readable. 1745 */ 1746 static void add_mc_hw_addr(const struct net_device *net_dev, 1747 struct dpaa2_eth_priv *priv) 1748 { 1749 struct netdev_hw_addr *ha; 1750 int err; 1751 1752 netdev_for_each_mc_addr(ha, net_dev) { 1753 err = dpni_add_mac_addr(priv->mc_io, 0, priv->mc_token, 1754 ha->addr); 1755 if (err) 1756 netdev_warn(priv->net_dev, 1757 "Could not add mcast MAC %pM to the filtering table (err %d)\n", 1758 ha->addr, err); 1759 } 1760 } 1761 1762 static void dpaa2_eth_set_rx_mode(struct net_device *net_dev) 1763 { 1764 struct dpaa2_eth_priv *priv = netdev_priv(net_dev); 1765 int uc_count = netdev_uc_count(net_dev); 1766 int mc_count = netdev_mc_count(net_dev); 1767 u8 max_mac = priv->dpni_attrs.mac_filter_entries; 1768 u32 options = priv->dpni_attrs.options; 1769 u16 mc_token = priv->mc_token; 1770 struct fsl_mc_io *mc_io = priv->mc_io; 1771 int err; 1772 1773 /* Basic sanity checks; these probably indicate a misconfiguration */ 1774 if (options & DPNI_OPT_NO_MAC_FILTER && max_mac != 0) 1775 netdev_info(net_dev, 1776 "mac_filter_entries=%d, DPNI_OPT_NO_MAC_FILTER option must be disabled\n", 1777 max_mac); 1778 1779 /* Force promiscuous if the uc or mc counts exceed our capabilities. */ 1780 if (uc_count > max_mac) { 1781 netdev_info(net_dev, 1782 "Unicast addr count reached %d, max allowed is %d; forcing promisc\n", 1783 uc_count, max_mac); 1784 goto force_promisc; 1785 } 1786 if (mc_count + uc_count > max_mac) { 1787 netdev_info(net_dev, 1788 "Unicast + multicast addr count reached %d, max allowed is %d; forcing promisc\n", 1789 uc_count + mc_count, max_mac); 1790 goto force_mc_promisc; 1791 } 1792 1793 /* Adjust promisc settings due to flag combinations */ 1794 if (net_dev->flags & IFF_PROMISC) 1795 goto force_promisc; 1796 if (net_dev->flags & IFF_ALLMULTI) { 1797 /* First, rebuild unicast filtering table. This should be done 1798 * in promisc mode, in order to avoid frame loss while we 1799 * progressively add entries to the table. 1800 * We don't know whether we had been in promisc already, and 1801 * making an MC call to find out is expensive; so set uc promisc 1802 * nonetheless. 1803 */ 1804 err = dpni_set_unicast_promisc(mc_io, 0, mc_token, 1); 1805 if (err) 1806 netdev_warn(net_dev, "Can't set uc promisc\n"); 1807 1808 /* Actual uc table reconstruction. */ 1809 err = dpni_clear_mac_filters(mc_io, 0, mc_token, 1, 0); 1810 if (err) 1811 netdev_warn(net_dev, "Can't clear uc filters\n"); 1812 add_uc_hw_addr(net_dev, priv); 1813 1814 /* Finally, clear uc promisc and set mc promisc as requested. */ 1815 err = dpni_set_unicast_promisc(mc_io, 0, mc_token, 0); 1816 if (err) 1817 netdev_warn(net_dev, "Can't clear uc promisc\n"); 1818 goto force_mc_promisc; 1819 } 1820 1821 /* Neither unicast, nor multicast promisc will be on... eventually. 1822 * For now, rebuild mac filtering tables while forcing both of them on. 1823 */ 1824 err = dpni_set_unicast_promisc(mc_io, 0, mc_token, 1); 1825 if (err) 1826 netdev_warn(net_dev, "Can't set uc promisc (%d)\n", err); 1827 err = dpni_set_multicast_promisc(mc_io, 0, mc_token, 1); 1828 if (err) 1829 netdev_warn(net_dev, "Can't set mc promisc (%d)\n", err); 1830 1831 /* Actual mac filtering tables reconstruction */ 1832 err = dpni_clear_mac_filters(mc_io, 0, mc_token, 1, 1); 1833 if (err) 1834 netdev_warn(net_dev, "Can't clear mac filters\n"); 1835 add_mc_hw_addr(net_dev, priv); 1836 add_uc_hw_addr(net_dev, priv); 1837 1838 /* Now we can clear both ucast and mcast promisc, without risking 1839 * to drop legitimate frames anymore. 1840 */ 1841 err = dpni_set_unicast_promisc(mc_io, 0, mc_token, 0); 1842 if (err) 1843 netdev_warn(net_dev, "Can't clear ucast promisc\n"); 1844 err = dpni_set_multicast_promisc(mc_io, 0, mc_token, 0); 1845 if (err) 1846 netdev_warn(net_dev, "Can't clear mcast promisc\n"); 1847 1848 return; 1849 1850 force_promisc: 1851 err = dpni_set_unicast_promisc(mc_io, 0, mc_token, 1); 1852 if (err) 1853 netdev_warn(net_dev, "Can't set ucast promisc\n"); 1854 force_mc_promisc: 1855 err = dpni_set_multicast_promisc(mc_io, 0, mc_token, 1); 1856 if (err) 1857 netdev_warn(net_dev, "Can't set mcast promisc\n"); 1858 } 1859 1860 static int dpaa2_eth_set_features(struct net_device *net_dev, 1861 netdev_features_t features) 1862 { 1863 struct dpaa2_eth_priv *priv = netdev_priv(net_dev); 1864 netdev_features_t changed = features ^ net_dev->features; 1865 bool enable; 1866 int err; 1867 1868 if (changed & NETIF_F_RXCSUM) { 1869 enable = !!(features & NETIF_F_RXCSUM); 1870 err = set_rx_csum(priv, enable); 1871 if (err) 1872 return err; 1873 } 1874 1875 if (changed & (NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM)) { 1876 enable = !!(features & (NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM)); 1877 err = set_tx_csum(priv, enable); 1878 if (err) 1879 return err; 1880 } 1881 1882 return 0; 1883 } 1884 1885 static int dpaa2_eth_ts_ioctl(struct net_device *dev, struct ifreq *rq, int cmd) 1886 { 1887 struct dpaa2_eth_priv *priv = netdev_priv(dev); 1888 struct hwtstamp_config config; 1889 1890 if (copy_from_user(&config, rq->ifr_data, sizeof(config))) 1891 return -EFAULT; 1892 1893 switch (config.tx_type) { 1894 case HWTSTAMP_TX_OFF: 1895 priv->tx_tstamp = false; 1896 break; 1897 case HWTSTAMP_TX_ON: 1898 priv->tx_tstamp = true; 1899 break; 1900 default: 1901 return -ERANGE; 1902 } 1903 1904 if (config.rx_filter == HWTSTAMP_FILTER_NONE) { 1905 priv->rx_tstamp = false; 1906 } else { 1907 priv->rx_tstamp = true; 1908 /* TS is set for all frame types, not only those requested */ 1909 config.rx_filter = HWTSTAMP_FILTER_ALL; 1910 } 1911 1912 return copy_to_user(rq->ifr_data, &config, sizeof(config)) ? 1913 -EFAULT : 0; 1914 } 1915 1916 static int dpaa2_eth_ioctl(struct net_device *dev, struct ifreq *rq, int cmd) 1917 { 1918 struct dpaa2_eth_priv *priv = netdev_priv(dev); 1919 1920 if (cmd == SIOCSHWTSTAMP) 1921 return dpaa2_eth_ts_ioctl(dev, rq, cmd); 1922 1923 if (priv->mac) 1924 return phylink_mii_ioctl(priv->mac->phylink, rq, cmd); 1925 1926 return -EOPNOTSUPP; 1927 } 1928 1929 static bool xdp_mtu_valid(struct dpaa2_eth_priv *priv, int mtu) 1930 { 1931 int mfl, linear_mfl; 1932 1933 mfl = DPAA2_ETH_L2_MAX_FRM(mtu); 1934 linear_mfl = priv->rx_buf_size - DPAA2_ETH_RX_HWA_SIZE - 1935 dpaa2_eth_rx_head_room(priv) - XDP_PACKET_HEADROOM; 1936 1937 if (mfl > linear_mfl) { 1938 netdev_warn(priv->net_dev, "Maximum MTU for XDP is %d\n", 1939 linear_mfl - VLAN_ETH_HLEN); 1940 return false; 1941 } 1942 1943 return true; 1944 } 1945 1946 static int set_rx_mfl(struct dpaa2_eth_priv *priv, int mtu, bool has_xdp) 1947 { 1948 int mfl, err; 1949 1950 /* We enforce a maximum Rx frame length based on MTU only if we have 1951 * an XDP program attached (in order to avoid Rx S/G frames). 1952 * Otherwise, we accept all incoming frames as long as they are not 1953 * larger than maximum size supported in hardware 1954 */ 1955 if (has_xdp) 1956 mfl = DPAA2_ETH_L2_MAX_FRM(mtu); 1957 else 1958 mfl = DPAA2_ETH_MFL; 1959 1960 err = dpni_set_max_frame_length(priv->mc_io, 0, priv->mc_token, mfl); 1961 if (err) { 1962 netdev_err(priv->net_dev, "dpni_set_max_frame_length failed\n"); 1963 return err; 1964 } 1965 1966 return 0; 1967 } 1968 1969 static int dpaa2_eth_change_mtu(struct net_device *dev, int new_mtu) 1970 { 1971 struct dpaa2_eth_priv *priv = netdev_priv(dev); 1972 int err; 1973 1974 if (!priv->xdp_prog) 1975 goto out; 1976 1977 if (!xdp_mtu_valid(priv, new_mtu)) 1978 return -EINVAL; 1979 1980 err = set_rx_mfl(priv, new_mtu, true); 1981 if (err) 1982 return err; 1983 1984 out: 1985 dev->mtu = new_mtu; 1986 return 0; 1987 } 1988 1989 static int update_rx_buffer_headroom(struct dpaa2_eth_priv *priv, bool has_xdp) 1990 { 1991 struct dpni_buffer_layout buf_layout = {0}; 1992 int err; 1993 1994 err = dpni_get_buffer_layout(priv->mc_io, 0, priv->mc_token, 1995 DPNI_QUEUE_RX, &buf_layout); 1996 if (err) { 1997 netdev_err(priv->net_dev, "dpni_get_buffer_layout failed\n"); 1998 return err; 1999 } 2000 2001 /* Reserve extra headroom for XDP header size changes */ 2002 buf_layout.data_head_room = dpaa2_eth_rx_head_room(priv) + 2003 (has_xdp ? XDP_PACKET_HEADROOM : 0); 2004 buf_layout.options = DPNI_BUF_LAYOUT_OPT_DATA_HEAD_ROOM; 2005 err = dpni_set_buffer_layout(priv->mc_io, 0, priv->mc_token, 2006 DPNI_QUEUE_RX, &buf_layout); 2007 if (err) { 2008 netdev_err(priv->net_dev, "dpni_set_buffer_layout failed\n"); 2009 return err; 2010 } 2011 2012 return 0; 2013 } 2014 2015 static int setup_xdp(struct net_device *dev, struct bpf_prog *prog) 2016 { 2017 struct dpaa2_eth_priv *priv = netdev_priv(dev); 2018 struct dpaa2_eth_channel *ch; 2019 struct bpf_prog *old; 2020 bool up, need_update; 2021 int i, err; 2022 2023 if (prog && !xdp_mtu_valid(priv, dev->mtu)) 2024 return -EINVAL; 2025 2026 if (prog) 2027 bpf_prog_add(prog, priv->num_channels); 2028 2029 up = netif_running(dev); 2030 need_update = (!!priv->xdp_prog != !!prog); 2031 2032 if (up) 2033 dpaa2_eth_stop(dev); 2034 2035 /* While in xdp mode, enforce a maximum Rx frame size based on MTU. 2036 * Also, when switching between xdp/non-xdp modes we need to reconfigure 2037 * our Rx buffer layout. Buffer pool was drained on dpaa2_eth_stop, 2038 * so we are sure no old format buffers will be used from now on. 2039 */ 2040 if (need_update) { 2041 err = set_rx_mfl(priv, dev->mtu, !!prog); 2042 if (err) 2043 goto out_err; 2044 err = update_rx_buffer_headroom(priv, !!prog); 2045 if (err) 2046 goto out_err; 2047 } 2048 2049 old = xchg(&priv->xdp_prog, prog); 2050 if (old) 2051 bpf_prog_put(old); 2052 2053 for (i = 0; i < priv->num_channels; i++) { 2054 ch = priv->channel[i]; 2055 old = xchg(&ch->xdp.prog, prog); 2056 if (old) 2057 bpf_prog_put(old); 2058 } 2059 2060 if (up) { 2061 err = dpaa2_eth_open(dev); 2062 if (err) 2063 return err; 2064 } 2065 2066 return 0; 2067 2068 out_err: 2069 if (prog) 2070 bpf_prog_sub(prog, priv->num_channels); 2071 if (up) 2072 dpaa2_eth_open(dev); 2073 2074 return err; 2075 } 2076 2077 static int dpaa2_eth_xdp(struct net_device *dev, struct netdev_bpf *xdp) 2078 { 2079 struct dpaa2_eth_priv *priv = netdev_priv(dev); 2080 2081 switch (xdp->command) { 2082 case XDP_SETUP_PROG: 2083 return setup_xdp(dev, xdp->prog); 2084 case XDP_QUERY_PROG: 2085 xdp->prog_id = priv->xdp_prog ? priv->xdp_prog->aux->id : 0; 2086 break; 2087 default: 2088 return -EINVAL; 2089 } 2090 2091 return 0; 2092 } 2093 2094 static int dpaa2_eth_xdp_create_fd(struct net_device *net_dev, 2095 struct xdp_frame *xdpf, 2096 struct dpaa2_fd *fd) 2097 { 2098 struct dpaa2_eth_priv *priv = netdev_priv(net_dev); 2099 struct device *dev = net_dev->dev.parent; 2100 unsigned int needed_headroom; 2101 struct dpaa2_eth_swa *swa; 2102 void *buffer_start, *aligned_start; 2103 dma_addr_t addr; 2104 2105 /* We require a minimum headroom to be able to transmit the frame. 2106 * Otherwise return an error and let the original net_device handle it 2107 */ 2108 needed_headroom = dpaa2_eth_needed_headroom(priv, NULL); 2109 if (xdpf->headroom < needed_headroom) 2110 return -EINVAL; 2111 2112 /* Setup the FD fields */ 2113 memset(fd, 0, sizeof(*fd)); 2114 2115 /* Align FD address, if possible */ 2116 buffer_start = xdpf->data - needed_headroom; 2117 aligned_start = PTR_ALIGN(buffer_start - DPAA2_ETH_TX_BUF_ALIGN, 2118 DPAA2_ETH_TX_BUF_ALIGN); 2119 if (aligned_start >= xdpf->data - xdpf->headroom) 2120 buffer_start = aligned_start; 2121 2122 swa = (struct dpaa2_eth_swa *)buffer_start; 2123 /* fill in necessary fields here */ 2124 swa->type = DPAA2_ETH_SWA_XDP; 2125 swa->xdp.dma_size = xdpf->data + xdpf->len - buffer_start; 2126 swa->xdp.xdpf = xdpf; 2127 2128 addr = dma_map_single(dev, buffer_start, 2129 swa->xdp.dma_size, 2130 DMA_BIDIRECTIONAL); 2131 if (unlikely(dma_mapping_error(dev, addr))) 2132 return -ENOMEM; 2133 2134 dpaa2_fd_set_addr(fd, addr); 2135 dpaa2_fd_set_offset(fd, xdpf->data - buffer_start); 2136 dpaa2_fd_set_len(fd, xdpf->len); 2137 dpaa2_fd_set_format(fd, dpaa2_fd_single); 2138 dpaa2_fd_set_ctrl(fd, FD_CTRL_PTA); 2139 2140 return 0; 2141 } 2142 2143 static int dpaa2_eth_xdp_xmit(struct net_device *net_dev, int n, 2144 struct xdp_frame **frames, u32 flags) 2145 { 2146 struct dpaa2_eth_priv *priv = netdev_priv(net_dev); 2147 struct dpaa2_eth_xdp_fds *xdp_redirect_fds; 2148 struct rtnl_link_stats64 *percpu_stats; 2149 struct dpaa2_eth_fq *fq; 2150 struct dpaa2_fd *fds; 2151 int enqueued, i, err; 2152 2153 if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK)) 2154 return -EINVAL; 2155 2156 if (!netif_running(net_dev)) 2157 return -ENETDOWN; 2158 2159 fq = &priv->fq[smp_processor_id()]; 2160 xdp_redirect_fds = &fq->xdp_redirect_fds; 2161 fds = xdp_redirect_fds->fds; 2162 2163 percpu_stats = this_cpu_ptr(priv->percpu_stats); 2164 2165 /* create a FD for each xdp_frame in the list received */ 2166 for (i = 0; i < n; i++) { 2167 err = dpaa2_eth_xdp_create_fd(net_dev, frames[i], &fds[i]); 2168 if (err) 2169 break; 2170 } 2171 xdp_redirect_fds->num = i; 2172 2173 /* enqueue all the frame descriptors */ 2174 enqueued = dpaa2_eth_xdp_flush(priv, fq, xdp_redirect_fds); 2175 2176 /* update statistics */ 2177 percpu_stats->tx_packets += enqueued; 2178 for (i = 0; i < enqueued; i++) 2179 percpu_stats->tx_bytes += dpaa2_fd_get_len(&fds[i]); 2180 for (i = enqueued; i < n; i++) 2181 xdp_return_frame_rx_napi(frames[i]); 2182 2183 return enqueued; 2184 } 2185 2186 static int update_xps(struct dpaa2_eth_priv *priv) 2187 { 2188 struct net_device *net_dev = priv->net_dev; 2189 struct cpumask xps_mask; 2190 struct dpaa2_eth_fq *fq; 2191 int i, num_queues, netdev_queues; 2192 int err = 0; 2193 2194 num_queues = dpaa2_eth_queue_count(priv); 2195 netdev_queues = (net_dev->num_tc ? : 1) * num_queues; 2196 2197 /* The first <num_queues> entries in priv->fq array are Tx/Tx conf 2198 * queues, so only process those 2199 */ 2200 for (i = 0; i < netdev_queues; i++) { 2201 fq = &priv->fq[i % num_queues]; 2202 2203 cpumask_clear(&xps_mask); 2204 cpumask_set_cpu(fq->target_cpu, &xps_mask); 2205 2206 err = netif_set_xps_queue(net_dev, &xps_mask, i); 2207 if (err) { 2208 netdev_warn_once(net_dev, "Error setting XPS queue\n"); 2209 break; 2210 } 2211 } 2212 2213 return err; 2214 } 2215 2216 static int dpaa2_eth_setup_tc(struct net_device *net_dev, 2217 enum tc_setup_type type, void *type_data) 2218 { 2219 struct dpaa2_eth_priv *priv = netdev_priv(net_dev); 2220 struct tc_mqprio_qopt *mqprio = type_data; 2221 u8 num_tc, num_queues; 2222 int i; 2223 2224 if (type != TC_SETUP_QDISC_MQPRIO) 2225 return -EOPNOTSUPP; 2226 2227 mqprio->hw = TC_MQPRIO_HW_OFFLOAD_TCS; 2228 num_queues = dpaa2_eth_queue_count(priv); 2229 num_tc = mqprio->num_tc; 2230 2231 if (num_tc == net_dev->num_tc) 2232 return 0; 2233 2234 if (num_tc > dpaa2_eth_tc_count(priv)) { 2235 netdev_err(net_dev, "Max %d traffic classes supported\n", 2236 dpaa2_eth_tc_count(priv)); 2237 return -EOPNOTSUPP; 2238 } 2239 2240 if (!num_tc) { 2241 netdev_reset_tc(net_dev); 2242 netif_set_real_num_tx_queues(net_dev, num_queues); 2243 goto out; 2244 } 2245 2246 netdev_set_num_tc(net_dev, num_tc); 2247 netif_set_real_num_tx_queues(net_dev, num_tc * num_queues); 2248 2249 for (i = 0; i < num_tc; i++) 2250 netdev_set_tc_queue(net_dev, i, num_queues, i * num_queues); 2251 2252 out: 2253 update_xps(priv); 2254 2255 return 0; 2256 } 2257 2258 static const struct net_device_ops dpaa2_eth_ops = { 2259 .ndo_open = dpaa2_eth_open, 2260 .ndo_start_xmit = dpaa2_eth_tx, 2261 .ndo_stop = dpaa2_eth_stop, 2262 .ndo_set_mac_address = dpaa2_eth_set_addr, 2263 .ndo_get_stats64 = dpaa2_eth_get_stats, 2264 .ndo_set_rx_mode = dpaa2_eth_set_rx_mode, 2265 .ndo_set_features = dpaa2_eth_set_features, 2266 .ndo_do_ioctl = dpaa2_eth_ioctl, 2267 .ndo_change_mtu = dpaa2_eth_change_mtu, 2268 .ndo_bpf = dpaa2_eth_xdp, 2269 .ndo_xdp_xmit = dpaa2_eth_xdp_xmit, 2270 .ndo_setup_tc = dpaa2_eth_setup_tc, 2271 }; 2272 2273 static void cdan_cb(struct dpaa2_io_notification_ctx *ctx) 2274 { 2275 struct dpaa2_eth_channel *ch; 2276 2277 ch = container_of(ctx, struct dpaa2_eth_channel, nctx); 2278 2279 /* Update NAPI statistics */ 2280 ch->stats.cdan++; 2281 2282 napi_schedule_irqoff(&ch->napi); 2283 } 2284 2285 /* Allocate and configure a DPCON object */ 2286 static struct fsl_mc_device *setup_dpcon(struct dpaa2_eth_priv *priv) 2287 { 2288 struct fsl_mc_device *dpcon; 2289 struct device *dev = priv->net_dev->dev.parent; 2290 int err; 2291 2292 err = fsl_mc_object_allocate(to_fsl_mc_device(dev), 2293 FSL_MC_POOL_DPCON, &dpcon); 2294 if (err) { 2295 if (err == -ENXIO) 2296 err = -EPROBE_DEFER; 2297 else 2298 dev_info(dev, "Not enough DPCONs, will go on as-is\n"); 2299 return ERR_PTR(err); 2300 } 2301 2302 err = dpcon_open(priv->mc_io, 0, dpcon->obj_desc.id, &dpcon->mc_handle); 2303 if (err) { 2304 dev_err(dev, "dpcon_open() failed\n"); 2305 goto free; 2306 } 2307 2308 err = dpcon_reset(priv->mc_io, 0, dpcon->mc_handle); 2309 if (err) { 2310 dev_err(dev, "dpcon_reset() failed\n"); 2311 goto close; 2312 } 2313 2314 err = dpcon_enable(priv->mc_io, 0, dpcon->mc_handle); 2315 if (err) { 2316 dev_err(dev, "dpcon_enable() failed\n"); 2317 goto close; 2318 } 2319 2320 return dpcon; 2321 2322 close: 2323 dpcon_close(priv->mc_io, 0, dpcon->mc_handle); 2324 free: 2325 fsl_mc_object_free(dpcon); 2326 2327 return NULL; 2328 } 2329 2330 static void free_dpcon(struct dpaa2_eth_priv *priv, 2331 struct fsl_mc_device *dpcon) 2332 { 2333 dpcon_disable(priv->mc_io, 0, dpcon->mc_handle); 2334 dpcon_close(priv->mc_io, 0, dpcon->mc_handle); 2335 fsl_mc_object_free(dpcon); 2336 } 2337 2338 static struct dpaa2_eth_channel * 2339 alloc_channel(struct dpaa2_eth_priv *priv) 2340 { 2341 struct dpaa2_eth_channel *channel; 2342 struct dpcon_attr attr; 2343 struct device *dev = priv->net_dev->dev.parent; 2344 int err; 2345 2346 channel = kzalloc(sizeof(*channel), GFP_KERNEL); 2347 if (!channel) 2348 return NULL; 2349 2350 channel->dpcon = setup_dpcon(priv); 2351 if (IS_ERR_OR_NULL(channel->dpcon)) { 2352 err = PTR_ERR_OR_ZERO(channel->dpcon); 2353 goto err_setup; 2354 } 2355 2356 err = dpcon_get_attributes(priv->mc_io, 0, channel->dpcon->mc_handle, 2357 &attr); 2358 if (err) { 2359 dev_err(dev, "dpcon_get_attributes() failed\n"); 2360 goto err_get_attr; 2361 } 2362 2363 channel->dpcon_id = attr.id; 2364 channel->ch_id = attr.qbman_ch_id; 2365 channel->priv = priv; 2366 2367 return channel; 2368 2369 err_get_attr: 2370 free_dpcon(priv, channel->dpcon); 2371 err_setup: 2372 kfree(channel); 2373 return ERR_PTR(err); 2374 } 2375 2376 static void free_channel(struct dpaa2_eth_priv *priv, 2377 struct dpaa2_eth_channel *channel) 2378 { 2379 free_dpcon(priv, channel->dpcon); 2380 kfree(channel); 2381 } 2382 2383 /* DPIO setup: allocate and configure QBMan channels, setup core affinity 2384 * and register data availability notifications 2385 */ 2386 static int setup_dpio(struct dpaa2_eth_priv *priv) 2387 { 2388 struct dpaa2_io_notification_ctx *nctx; 2389 struct dpaa2_eth_channel *channel; 2390 struct dpcon_notification_cfg dpcon_notif_cfg; 2391 struct device *dev = priv->net_dev->dev.parent; 2392 int i, err; 2393 2394 /* We want the ability to spread ingress traffic (RX, TX conf) to as 2395 * many cores as possible, so we need one channel for each core 2396 * (unless there's fewer queues than cores, in which case the extra 2397 * channels would be wasted). 2398 * Allocate one channel per core and register it to the core's 2399 * affine DPIO. If not enough channels are available for all cores 2400 * or if some cores don't have an affine DPIO, there will be no 2401 * ingress frame processing on those cores. 2402 */ 2403 cpumask_clear(&priv->dpio_cpumask); 2404 for_each_online_cpu(i) { 2405 /* Try to allocate a channel */ 2406 channel = alloc_channel(priv); 2407 if (IS_ERR_OR_NULL(channel)) { 2408 err = PTR_ERR_OR_ZERO(channel); 2409 if (err != -EPROBE_DEFER) 2410 dev_info(dev, 2411 "No affine channel for cpu %d and above\n", i); 2412 goto err_alloc_ch; 2413 } 2414 2415 priv->channel[priv->num_channels] = channel; 2416 2417 nctx = &channel->nctx; 2418 nctx->is_cdan = 1; 2419 nctx->cb = cdan_cb; 2420 nctx->id = channel->ch_id; 2421 nctx->desired_cpu = i; 2422 2423 /* Register the new context */ 2424 channel->dpio = dpaa2_io_service_select(i); 2425 err = dpaa2_io_service_register(channel->dpio, nctx, dev); 2426 if (err) { 2427 dev_dbg(dev, "No affine DPIO for cpu %d\n", i); 2428 /* If no affine DPIO for this core, there's probably 2429 * none available for next cores either. Signal we want 2430 * to retry later, in case the DPIO devices weren't 2431 * probed yet. 2432 */ 2433 err = -EPROBE_DEFER; 2434 goto err_service_reg; 2435 } 2436 2437 /* Register DPCON notification with MC */ 2438 dpcon_notif_cfg.dpio_id = nctx->dpio_id; 2439 dpcon_notif_cfg.priority = 0; 2440 dpcon_notif_cfg.user_ctx = nctx->qman64; 2441 err = dpcon_set_notification(priv->mc_io, 0, 2442 channel->dpcon->mc_handle, 2443 &dpcon_notif_cfg); 2444 if (err) { 2445 dev_err(dev, "dpcon_set_notification failed()\n"); 2446 goto err_set_cdan; 2447 } 2448 2449 /* If we managed to allocate a channel and also found an affine 2450 * DPIO for this core, add it to the final mask 2451 */ 2452 cpumask_set_cpu(i, &priv->dpio_cpumask); 2453 priv->num_channels++; 2454 2455 /* Stop if we already have enough channels to accommodate all 2456 * RX and TX conf queues 2457 */ 2458 if (priv->num_channels == priv->dpni_attrs.num_queues) 2459 break; 2460 } 2461 2462 return 0; 2463 2464 err_set_cdan: 2465 dpaa2_io_service_deregister(channel->dpio, nctx, dev); 2466 err_service_reg: 2467 free_channel(priv, channel); 2468 err_alloc_ch: 2469 if (err == -EPROBE_DEFER) { 2470 for (i = 0; i < priv->num_channels; i++) { 2471 channel = priv->channel[i]; 2472 nctx = &channel->nctx; 2473 dpaa2_io_service_deregister(channel->dpio, nctx, dev); 2474 free_channel(priv, channel); 2475 } 2476 priv->num_channels = 0; 2477 return err; 2478 } 2479 2480 if (cpumask_empty(&priv->dpio_cpumask)) { 2481 dev_err(dev, "No cpu with an affine DPIO/DPCON\n"); 2482 return -ENODEV; 2483 } 2484 2485 dev_info(dev, "Cores %*pbl available for processing ingress traffic\n", 2486 cpumask_pr_args(&priv->dpio_cpumask)); 2487 2488 return 0; 2489 } 2490 2491 static void free_dpio(struct dpaa2_eth_priv *priv) 2492 { 2493 struct device *dev = priv->net_dev->dev.parent; 2494 struct dpaa2_eth_channel *ch; 2495 int i; 2496 2497 /* deregister CDAN notifications and free channels */ 2498 for (i = 0; i < priv->num_channels; i++) { 2499 ch = priv->channel[i]; 2500 dpaa2_io_service_deregister(ch->dpio, &ch->nctx, dev); 2501 free_channel(priv, ch); 2502 } 2503 } 2504 2505 static struct dpaa2_eth_channel *get_affine_channel(struct dpaa2_eth_priv *priv, 2506 int cpu) 2507 { 2508 struct device *dev = priv->net_dev->dev.parent; 2509 int i; 2510 2511 for (i = 0; i < priv->num_channels; i++) 2512 if (priv->channel[i]->nctx.desired_cpu == cpu) 2513 return priv->channel[i]; 2514 2515 /* We should never get here. Issue a warning and return 2516 * the first channel, because it's still better than nothing 2517 */ 2518 dev_warn(dev, "No affine channel found for cpu %d\n", cpu); 2519 2520 return priv->channel[0]; 2521 } 2522 2523 static void set_fq_affinity(struct dpaa2_eth_priv *priv) 2524 { 2525 struct device *dev = priv->net_dev->dev.parent; 2526 struct dpaa2_eth_fq *fq; 2527 int rx_cpu, txc_cpu; 2528 int i; 2529 2530 /* For each FQ, pick one channel/CPU to deliver frames to. 2531 * This may well change at runtime, either through irqbalance or 2532 * through direct user intervention. 2533 */ 2534 rx_cpu = txc_cpu = cpumask_first(&priv->dpio_cpumask); 2535 2536 for (i = 0; i < priv->num_fqs; i++) { 2537 fq = &priv->fq[i]; 2538 switch (fq->type) { 2539 case DPAA2_RX_FQ: 2540 fq->target_cpu = rx_cpu; 2541 rx_cpu = cpumask_next(rx_cpu, &priv->dpio_cpumask); 2542 if (rx_cpu >= nr_cpu_ids) 2543 rx_cpu = cpumask_first(&priv->dpio_cpumask); 2544 break; 2545 case DPAA2_TX_CONF_FQ: 2546 fq->target_cpu = txc_cpu; 2547 txc_cpu = cpumask_next(txc_cpu, &priv->dpio_cpumask); 2548 if (txc_cpu >= nr_cpu_ids) 2549 txc_cpu = cpumask_first(&priv->dpio_cpumask); 2550 break; 2551 default: 2552 dev_err(dev, "Unknown FQ type: %d\n", fq->type); 2553 } 2554 fq->channel = get_affine_channel(priv, fq->target_cpu); 2555 } 2556 2557 update_xps(priv); 2558 } 2559 2560 static void setup_fqs(struct dpaa2_eth_priv *priv) 2561 { 2562 int i, j; 2563 2564 /* We have one TxConf FQ per Tx flow. 2565 * The number of Tx and Rx queues is the same. 2566 * Tx queues come first in the fq array. 2567 */ 2568 for (i = 0; i < dpaa2_eth_queue_count(priv); i++) { 2569 priv->fq[priv->num_fqs].type = DPAA2_TX_CONF_FQ; 2570 priv->fq[priv->num_fqs].consume = dpaa2_eth_tx_conf; 2571 priv->fq[priv->num_fqs++].flowid = (u16)i; 2572 } 2573 2574 for (j = 0; j < dpaa2_eth_tc_count(priv); j++) { 2575 for (i = 0; i < dpaa2_eth_queue_count(priv); i++) { 2576 priv->fq[priv->num_fqs].type = DPAA2_RX_FQ; 2577 priv->fq[priv->num_fqs].consume = dpaa2_eth_rx; 2578 priv->fq[priv->num_fqs].tc = (u8)j; 2579 priv->fq[priv->num_fqs++].flowid = (u16)i; 2580 } 2581 } 2582 2583 /* For each FQ, decide on which core to process incoming frames */ 2584 set_fq_affinity(priv); 2585 } 2586 2587 /* Allocate and configure one buffer pool for each interface */ 2588 static int setup_dpbp(struct dpaa2_eth_priv *priv) 2589 { 2590 int err; 2591 struct fsl_mc_device *dpbp_dev; 2592 struct device *dev = priv->net_dev->dev.parent; 2593 struct dpbp_attr dpbp_attrs; 2594 2595 err = fsl_mc_object_allocate(to_fsl_mc_device(dev), FSL_MC_POOL_DPBP, 2596 &dpbp_dev); 2597 if (err) { 2598 if (err == -ENXIO) 2599 err = -EPROBE_DEFER; 2600 else 2601 dev_err(dev, "DPBP device allocation failed\n"); 2602 return err; 2603 } 2604 2605 priv->dpbp_dev = dpbp_dev; 2606 2607 err = dpbp_open(priv->mc_io, 0, priv->dpbp_dev->obj_desc.id, 2608 &dpbp_dev->mc_handle); 2609 if (err) { 2610 dev_err(dev, "dpbp_open() failed\n"); 2611 goto err_open; 2612 } 2613 2614 err = dpbp_reset(priv->mc_io, 0, dpbp_dev->mc_handle); 2615 if (err) { 2616 dev_err(dev, "dpbp_reset() failed\n"); 2617 goto err_reset; 2618 } 2619 2620 err = dpbp_enable(priv->mc_io, 0, dpbp_dev->mc_handle); 2621 if (err) { 2622 dev_err(dev, "dpbp_enable() failed\n"); 2623 goto err_enable; 2624 } 2625 2626 err = dpbp_get_attributes(priv->mc_io, 0, dpbp_dev->mc_handle, 2627 &dpbp_attrs); 2628 if (err) { 2629 dev_err(dev, "dpbp_get_attributes() failed\n"); 2630 goto err_get_attr; 2631 } 2632 priv->bpid = dpbp_attrs.bpid; 2633 2634 return 0; 2635 2636 err_get_attr: 2637 dpbp_disable(priv->mc_io, 0, dpbp_dev->mc_handle); 2638 err_enable: 2639 err_reset: 2640 dpbp_close(priv->mc_io, 0, dpbp_dev->mc_handle); 2641 err_open: 2642 fsl_mc_object_free(dpbp_dev); 2643 2644 return err; 2645 } 2646 2647 static void free_dpbp(struct dpaa2_eth_priv *priv) 2648 { 2649 drain_pool(priv); 2650 dpbp_disable(priv->mc_io, 0, priv->dpbp_dev->mc_handle); 2651 dpbp_close(priv->mc_io, 0, priv->dpbp_dev->mc_handle); 2652 fsl_mc_object_free(priv->dpbp_dev); 2653 } 2654 2655 static int set_buffer_layout(struct dpaa2_eth_priv *priv) 2656 { 2657 struct device *dev = priv->net_dev->dev.parent; 2658 struct dpni_buffer_layout buf_layout = {0}; 2659 u16 rx_buf_align; 2660 int err; 2661 2662 /* We need to check for WRIOP version 1.0.0, but depending on the MC 2663 * version, this number is not always provided correctly on rev1. 2664 * We need to check for both alternatives in this situation. 2665 */ 2666 if (priv->dpni_attrs.wriop_version == DPAA2_WRIOP_VERSION(0, 0, 0) || 2667 priv->dpni_attrs.wriop_version == DPAA2_WRIOP_VERSION(1, 0, 0)) 2668 rx_buf_align = DPAA2_ETH_RX_BUF_ALIGN_REV1; 2669 else 2670 rx_buf_align = DPAA2_ETH_RX_BUF_ALIGN; 2671 2672 /* We need to ensure that the buffer size seen by WRIOP is a multiple 2673 * of 64 or 256 bytes depending on the WRIOP version. 2674 */ 2675 priv->rx_buf_size = ALIGN_DOWN(DPAA2_ETH_RX_BUF_SIZE, rx_buf_align); 2676 2677 /* tx buffer */ 2678 buf_layout.private_data_size = DPAA2_ETH_SWA_SIZE; 2679 buf_layout.pass_timestamp = true; 2680 buf_layout.options = DPNI_BUF_LAYOUT_OPT_PRIVATE_DATA_SIZE | 2681 DPNI_BUF_LAYOUT_OPT_TIMESTAMP; 2682 err = dpni_set_buffer_layout(priv->mc_io, 0, priv->mc_token, 2683 DPNI_QUEUE_TX, &buf_layout); 2684 if (err) { 2685 dev_err(dev, "dpni_set_buffer_layout(TX) failed\n"); 2686 return err; 2687 } 2688 2689 /* tx-confirm buffer */ 2690 buf_layout.options = DPNI_BUF_LAYOUT_OPT_TIMESTAMP; 2691 err = dpni_set_buffer_layout(priv->mc_io, 0, priv->mc_token, 2692 DPNI_QUEUE_TX_CONFIRM, &buf_layout); 2693 if (err) { 2694 dev_err(dev, "dpni_set_buffer_layout(TX_CONF) failed\n"); 2695 return err; 2696 } 2697 2698 /* Now that we've set our tx buffer layout, retrieve the minimum 2699 * required tx data offset. 2700 */ 2701 err = dpni_get_tx_data_offset(priv->mc_io, 0, priv->mc_token, 2702 &priv->tx_data_offset); 2703 if (err) { 2704 dev_err(dev, "dpni_get_tx_data_offset() failed\n"); 2705 return err; 2706 } 2707 2708 if ((priv->tx_data_offset % 64) != 0) 2709 dev_warn(dev, "Tx data offset (%d) not a multiple of 64B\n", 2710 priv->tx_data_offset); 2711 2712 /* rx buffer */ 2713 buf_layout.pass_frame_status = true; 2714 buf_layout.pass_parser_result = true; 2715 buf_layout.data_align = rx_buf_align; 2716 buf_layout.data_head_room = dpaa2_eth_rx_head_room(priv); 2717 buf_layout.private_data_size = 0; 2718 buf_layout.options = DPNI_BUF_LAYOUT_OPT_PARSER_RESULT | 2719 DPNI_BUF_LAYOUT_OPT_FRAME_STATUS | 2720 DPNI_BUF_LAYOUT_OPT_DATA_ALIGN | 2721 DPNI_BUF_LAYOUT_OPT_DATA_HEAD_ROOM | 2722 DPNI_BUF_LAYOUT_OPT_TIMESTAMP; 2723 err = dpni_set_buffer_layout(priv->mc_io, 0, priv->mc_token, 2724 DPNI_QUEUE_RX, &buf_layout); 2725 if (err) { 2726 dev_err(dev, "dpni_set_buffer_layout(RX) failed\n"); 2727 return err; 2728 } 2729 2730 return 0; 2731 } 2732 2733 #define DPNI_ENQUEUE_FQID_VER_MAJOR 7 2734 #define DPNI_ENQUEUE_FQID_VER_MINOR 9 2735 2736 static inline int dpaa2_eth_enqueue_qd(struct dpaa2_eth_priv *priv, 2737 struct dpaa2_eth_fq *fq, 2738 struct dpaa2_fd *fd, u8 prio, 2739 u32 num_frames __always_unused, 2740 int *frames_enqueued) 2741 { 2742 int err; 2743 2744 err = dpaa2_io_service_enqueue_qd(fq->channel->dpio, 2745 priv->tx_qdid, prio, 2746 fq->tx_qdbin, fd); 2747 if (!err && frames_enqueued) 2748 *frames_enqueued = 1; 2749 return err; 2750 } 2751 2752 static inline int dpaa2_eth_enqueue_fq_multiple(struct dpaa2_eth_priv *priv, 2753 struct dpaa2_eth_fq *fq, 2754 struct dpaa2_fd *fd, 2755 u8 prio, u32 num_frames, 2756 int *frames_enqueued) 2757 { 2758 int err; 2759 2760 err = dpaa2_io_service_enqueue_multiple_fq(fq->channel->dpio, 2761 fq->tx_fqid[prio], 2762 fd, num_frames); 2763 2764 if (err == 0) 2765 return -EBUSY; 2766 2767 if (frames_enqueued) 2768 *frames_enqueued = err; 2769 return 0; 2770 } 2771 2772 static void set_enqueue_mode(struct dpaa2_eth_priv *priv) 2773 { 2774 if (dpaa2_eth_cmp_dpni_ver(priv, DPNI_ENQUEUE_FQID_VER_MAJOR, 2775 DPNI_ENQUEUE_FQID_VER_MINOR) < 0) 2776 priv->enqueue = dpaa2_eth_enqueue_qd; 2777 else 2778 priv->enqueue = dpaa2_eth_enqueue_fq_multiple; 2779 } 2780 2781 static int set_pause(struct dpaa2_eth_priv *priv) 2782 { 2783 struct device *dev = priv->net_dev->dev.parent; 2784 struct dpni_link_cfg link_cfg = {0}; 2785 int err; 2786 2787 /* Get the default link options so we don't override other flags */ 2788 err = dpni_get_link_cfg(priv->mc_io, 0, priv->mc_token, &link_cfg); 2789 if (err) { 2790 dev_err(dev, "dpni_get_link_cfg() failed\n"); 2791 return err; 2792 } 2793 2794 /* By default, enable both Rx and Tx pause frames */ 2795 link_cfg.options |= DPNI_LINK_OPT_PAUSE; 2796 link_cfg.options &= ~DPNI_LINK_OPT_ASYM_PAUSE; 2797 err = dpni_set_link_cfg(priv->mc_io, 0, priv->mc_token, &link_cfg); 2798 if (err) { 2799 dev_err(dev, "dpni_set_link_cfg() failed\n"); 2800 return err; 2801 } 2802 2803 priv->link_state.options = link_cfg.options; 2804 2805 return 0; 2806 } 2807 2808 static void update_tx_fqids(struct dpaa2_eth_priv *priv) 2809 { 2810 struct dpni_queue_id qid = {0}; 2811 struct dpaa2_eth_fq *fq; 2812 struct dpni_queue queue; 2813 int i, j, err; 2814 2815 /* We only use Tx FQIDs for FQID-based enqueue, so check 2816 * if DPNI version supports it before updating FQIDs 2817 */ 2818 if (dpaa2_eth_cmp_dpni_ver(priv, DPNI_ENQUEUE_FQID_VER_MAJOR, 2819 DPNI_ENQUEUE_FQID_VER_MINOR) < 0) 2820 return; 2821 2822 for (i = 0; i < priv->num_fqs; i++) { 2823 fq = &priv->fq[i]; 2824 if (fq->type != DPAA2_TX_CONF_FQ) 2825 continue; 2826 for (j = 0; j < dpaa2_eth_tc_count(priv); j++) { 2827 err = dpni_get_queue(priv->mc_io, 0, priv->mc_token, 2828 DPNI_QUEUE_TX, j, fq->flowid, 2829 &queue, &qid); 2830 if (err) 2831 goto out_err; 2832 2833 fq->tx_fqid[j] = qid.fqid; 2834 if (fq->tx_fqid[j] == 0) 2835 goto out_err; 2836 } 2837 } 2838 2839 priv->enqueue = dpaa2_eth_enqueue_fq_multiple; 2840 2841 return; 2842 2843 out_err: 2844 netdev_info(priv->net_dev, 2845 "Error reading Tx FQID, fallback to QDID-based enqueue\n"); 2846 priv->enqueue = dpaa2_eth_enqueue_qd; 2847 } 2848 2849 /* Configure ingress classification based on VLAN PCP */ 2850 static int set_vlan_qos(struct dpaa2_eth_priv *priv) 2851 { 2852 struct device *dev = priv->net_dev->dev.parent; 2853 struct dpkg_profile_cfg kg_cfg = {0}; 2854 struct dpni_qos_tbl_cfg qos_cfg = {0}; 2855 struct dpni_rule_cfg key_params; 2856 void *dma_mem, *key, *mask; 2857 u8 key_size = 2; /* VLAN TCI field */ 2858 int i, pcp, err; 2859 2860 /* VLAN-based classification only makes sense if we have multiple 2861 * traffic classes. 2862 * Also, we need to extract just the 3-bit PCP field from the VLAN 2863 * header and we can only do that by using a mask 2864 */ 2865 if (dpaa2_eth_tc_count(priv) == 1 || !dpaa2_eth_fs_mask_enabled(priv)) { 2866 dev_dbg(dev, "VLAN-based QoS classification not supported\n"); 2867 return -EOPNOTSUPP; 2868 } 2869 2870 dma_mem = kzalloc(DPAA2_CLASSIFIER_DMA_SIZE, GFP_KERNEL); 2871 if (!dma_mem) 2872 return -ENOMEM; 2873 2874 kg_cfg.num_extracts = 1; 2875 kg_cfg.extracts[0].type = DPKG_EXTRACT_FROM_HDR; 2876 kg_cfg.extracts[0].extract.from_hdr.prot = NET_PROT_VLAN; 2877 kg_cfg.extracts[0].extract.from_hdr.type = DPKG_FULL_FIELD; 2878 kg_cfg.extracts[0].extract.from_hdr.field = NH_FLD_VLAN_TCI; 2879 2880 err = dpni_prepare_key_cfg(&kg_cfg, dma_mem); 2881 if (err) { 2882 dev_err(dev, "dpni_prepare_key_cfg failed\n"); 2883 goto out_free_tbl; 2884 } 2885 2886 /* set QoS table */ 2887 qos_cfg.default_tc = 0; 2888 qos_cfg.discard_on_miss = 0; 2889 qos_cfg.key_cfg_iova = dma_map_single(dev, dma_mem, 2890 DPAA2_CLASSIFIER_DMA_SIZE, 2891 DMA_TO_DEVICE); 2892 if (dma_mapping_error(dev, qos_cfg.key_cfg_iova)) { 2893 dev_err(dev, "QoS table DMA mapping failed\n"); 2894 err = -ENOMEM; 2895 goto out_free_tbl; 2896 } 2897 2898 err = dpni_set_qos_table(priv->mc_io, 0, priv->mc_token, &qos_cfg); 2899 if (err) { 2900 dev_err(dev, "dpni_set_qos_table failed\n"); 2901 goto out_unmap_tbl; 2902 } 2903 2904 /* Add QoS table entries */ 2905 key = kzalloc(key_size * 2, GFP_KERNEL); 2906 if (!key) { 2907 err = -ENOMEM; 2908 goto out_unmap_tbl; 2909 } 2910 mask = key + key_size; 2911 *(__be16 *)mask = cpu_to_be16(VLAN_PRIO_MASK); 2912 2913 key_params.key_iova = dma_map_single(dev, key, key_size * 2, 2914 DMA_TO_DEVICE); 2915 if (dma_mapping_error(dev, key_params.key_iova)) { 2916 dev_err(dev, "Qos table entry DMA mapping failed\n"); 2917 err = -ENOMEM; 2918 goto out_free_key; 2919 } 2920 2921 key_params.mask_iova = key_params.key_iova + key_size; 2922 key_params.key_size = key_size; 2923 2924 /* We add rules for PCP-based distribution starting with highest 2925 * priority (VLAN PCP = 7). If this DPNI doesn't have enough traffic 2926 * classes to accommodate all priority levels, the lowest ones end up 2927 * on TC 0 which was configured as default 2928 */ 2929 for (i = dpaa2_eth_tc_count(priv) - 1, pcp = 7; i >= 0; i--, pcp--) { 2930 *(__be16 *)key = cpu_to_be16(pcp << VLAN_PRIO_SHIFT); 2931 dma_sync_single_for_device(dev, key_params.key_iova, 2932 key_size * 2, DMA_TO_DEVICE); 2933 2934 err = dpni_add_qos_entry(priv->mc_io, 0, priv->mc_token, 2935 &key_params, i, i); 2936 if (err) { 2937 dev_err(dev, "dpni_add_qos_entry failed\n"); 2938 dpni_clear_qos_table(priv->mc_io, 0, priv->mc_token); 2939 goto out_unmap_key; 2940 } 2941 } 2942 2943 priv->vlan_cls_enabled = true; 2944 2945 /* Table and key memory is not persistent, clean everything up after 2946 * configuration is finished 2947 */ 2948 out_unmap_key: 2949 dma_unmap_single(dev, key_params.key_iova, key_size * 2, DMA_TO_DEVICE); 2950 out_free_key: 2951 kfree(key); 2952 out_unmap_tbl: 2953 dma_unmap_single(dev, qos_cfg.key_cfg_iova, DPAA2_CLASSIFIER_DMA_SIZE, 2954 DMA_TO_DEVICE); 2955 out_free_tbl: 2956 kfree(dma_mem); 2957 2958 return err; 2959 } 2960 2961 /* Configure the DPNI object this interface is associated with */ 2962 static int setup_dpni(struct fsl_mc_device *ls_dev) 2963 { 2964 struct device *dev = &ls_dev->dev; 2965 struct dpaa2_eth_priv *priv; 2966 struct net_device *net_dev; 2967 int err; 2968 2969 net_dev = dev_get_drvdata(dev); 2970 priv = netdev_priv(net_dev); 2971 2972 /* get a handle for the DPNI object */ 2973 err = dpni_open(priv->mc_io, 0, ls_dev->obj_desc.id, &priv->mc_token); 2974 if (err) { 2975 dev_err(dev, "dpni_open() failed\n"); 2976 return err; 2977 } 2978 2979 /* Check if we can work with this DPNI object */ 2980 err = dpni_get_api_version(priv->mc_io, 0, &priv->dpni_ver_major, 2981 &priv->dpni_ver_minor); 2982 if (err) { 2983 dev_err(dev, "dpni_get_api_version() failed\n"); 2984 goto close; 2985 } 2986 if (dpaa2_eth_cmp_dpni_ver(priv, DPNI_VER_MAJOR, DPNI_VER_MINOR) < 0) { 2987 dev_err(dev, "DPNI version %u.%u not supported, need >= %u.%u\n", 2988 priv->dpni_ver_major, priv->dpni_ver_minor, 2989 DPNI_VER_MAJOR, DPNI_VER_MINOR); 2990 err = -ENOTSUPP; 2991 goto close; 2992 } 2993 2994 ls_dev->mc_io = priv->mc_io; 2995 ls_dev->mc_handle = priv->mc_token; 2996 2997 err = dpni_reset(priv->mc_io, 0, priv->mc_token); 2998 if (err) { 2999 dev_err(dev, "dpni_reset() failed\n"); 3000 goto close; 3001 } 3002 3003 err = dpni_get_attributes(priv->mc_io, 0, priv->mc_token, 3004 &priv->dpni_attrs); 3005 if (err) { 3006 dev_err(dev, "dpni_get_attributes() failed (err=%d)\n", err); 3007 goto close; 3008 } 3009 3010 err = set_buffer_layout(priv); 3011 if (err) 3012 goto close; 3013 3014 set_enqueue_mode(priv); 3015 3016 /* Enable pause frame support */ 3017 if (dpaa2_eth_has_pause_support(priv)) { 3018 err = set_pause(priv); 3019 if (err) 3020 goto close; 3021 } 3022 3023 err = set_vlan_qos(priv); 3024 if (err && err != -EOPNOTSUPP) 3025 goto close; 3026 3027 priv->cls_rules = devm_kcalloc(dev, dpaa2_eth_fs_count(priv), 3028 sizeof(struct dpaa2_eth_cls_rule), 3029 GFP_KERNEL); 3030 if (!priv->cls_rules) { 3031 err = -ENOMEM; 3032 goto close; 3033 } 3034 3035 return 0; 3036 3037 close: 3038 dpni_close(priv->mc_io, 0, priv->mc_token); 3039 3040 return err; 3041 } 3042 3043 static void free_dpni(struct dpaa2_eth_priv *priv) 3044 { 3045 int err; 3046 3047 err = dpni_reset(priv->mc_io, 0, priv->mc_token); 3048 if (err) 3049 netdev_warn(priv->net_dev, "dpni_reset() failed (err %d)\n", 3050 err); 3051 3052 dpni_close(priv->mc_io, 0, priv->mc_token); 3053 } 3054 3055 static int setup_rx_flow(struct dpaa2_eth_priv *priv, 3056 struct dpaa2_eth_fq *fq) 3057 { 3058 struct device *dev = priv->net_dev->dev.parent; 3059 struct dpni_queue queue; 3060 struct dpni_queue_id qid; 3061 int err; 3062 3063 err = dpni_get_queue(priv->mc_io, 0, priv->mc_token, 3064 DPNI_QUEUE_RX, fq->tc, fq->flowid, &queue, &qid); 3065 if (err) { 3066 dev_err(dev, "dpni_get_queue(RX) failed\n"); 3067 return err; 3068 } 3069 3070 fq->fqid = qid.fqid; 3071 3072 queue.destination.id = fq->channel->dpcon_id; 3073 queue.destination.type = DPNI_DEST_DPCON; 3074 queue.destination.priority = 1; 3075 queue.user_context = (u64)(uintptr_t)fq; 3076 err = dpni_set_queue(priv->mc_io, 0, priv->mc_token, 3077 DPNI_QUEUE_RX, fq->tc, fq->flowid, 3078 DPNI_QUEUE_OPT_USER_CTX | DPNI_QUEUE_OPT_DEST, 3079 &queue); 3080 if (err) { 3081 dev_err(dev, "dpni_set_queue(RX) failed\n"); 3082 return err; 3083 } 3084 3085 /* xdp_rxq setup */ 3086 /* only once for each channel */ 3087 if (fq->tc > 0) 3088 return 0; 3089 3090 err = xdp_rxq_info_reg(&fq->channel->xdp_rxq, priv->net_dev, 3091 fq->flowid); 3092 if (err) { 3093 dev_err(dev, "xdp_rxq_info_reg failed\n"); 3094 return err; 3095 } 3096 3097 err = xdp_rxq_info_reg_mem_model(&fq->channel->xdp_rxq, 3098 MEM_TYPE_PAGE_ORDER0, NULL); 3099 if (err) { 3100 dev_err(dev, "xdp_rxq_info_reg_mem_model failed\n"); 3101 return err; 3102 } 3103 3104 return 0; 3105 } 3106 3107 static int setup_tx_flow(struct dpaa2_eth_priv *priv, 3108 struct dpaa2_eth_fq *fq) 3109 { 3110 struct device *dev = priv->net_dev->dev.parent; 3111 struct dpni_queue queue; 3112 struct dpni_queue_id qid; 3113 int i, err; 3114 3115 for (i = 0; i < dpaa2_eth_tc_count(priv); i++) { 3116 err = dpni_get_queue(priv->mc_io, 0, priv->mc_token, 3117 DPNI_QUEUE_TX, i, fq->flowid, 3118 &queue, &qid); 3119 if (err) { 3120 dev_err(dev, "dpni_get_queue(TX) failed\n"); 3121 return err; 3122 } 3123 fq->tx_fqid[i] = qid.fqid; 3124 } 3125 3126 /* All Tx queues belonging to the same flowid have the same qdbin */ 3127 fq->tx_qdbin = qid.qdbin; 3128 3129 err = dpni_get_queue(priv->mc_io, 0, priv->mc_token, 3130 DPNI_QUEUE_TX_CONFIRM, 0, fq->flowid, 3131 &queue, &qid); 3132 if (err) { 3133 dev_err(dev, "dpni_get_queue(TX_CONF) failed\n"); 3134 return err; 3135 } 3136 3137 fq->fqid = qid.fqid; 3138 3139 queue.destination.id = fq->channel->dpcon_id; 3140 queue.destination.type = DPNI_DEST_DPCON; 3141 queue.destination.priority = 0; 3142 queue.user_context = (u64)(uintptr_t)fq; 3143 err = dpni_set_queue(priv->mc_io, 0, priv->mc_token, 3144 DPNI_QUEUE_TX_CONFIRM, 0, fq->flowid, 3145 DPNI_QUEUE_OPT_USER_CTX | DPNI_QUEUE_OPT_DEST, 3146 &queue); 3147 if (err) { 3148 dev_err(dev, "dpni_set_queue(TX_CONF) failed\n"); 3149 return err; 3150 } 3151 3152 return 0; 3153 } 3154 3155 /* Supported header fields for Rx hash distribution key */ 3156 static const struct dpaa2_eth_dist_fields dist_fields[] = { 3157 { 3158 /* L2 header */ 3159 .rxnfc_field = RXH_L2DA, 3160 .cls_prot = NET_PROT_ETH, 3161 .cls_field = NH_FLD_ETH_DA, 3162 .id = DPAA2_ETH_DIST_ETHDST, 3163 .size = 6, 3164 }, { 3165 .cls_prot = NET_PROT_ETH, 3166 .cls_field = NH_FLD_ETH_SA, 3167 .id = DPAA2_ETH_DIST_ETHSRC, 3168 .size = 6, 3169 }, { 3170 /* This is the last ethertype field parsed: 3171 * depending on frame format, it can be the MAC ethertype 3172 * or the VLAN etype. 3173 */ 3174 .cls_prot = NET_PROT_ETH, 3175 .cls_field = NH_FLD_ETH_TYPE, 3176 .id = DPAA2_ETH_DIST_ETHTYPE, 3177 .size = 2, 3178 }, { 3179 /* VLAN header */ 3180 .rxnfc_field = RXH_VLAN, 3181 .cls_prot = NET_PROT_VLAN, 3182 .cls_field = NH_FLD_VLAN_TCI, 3183 .id = DPAA2_ETH_DIST_VLAN, 3184 .size = 2, 3185 }, { 3186 /* IP header */ 3187 .rxnfc_field = RXH_IP_SRC, 3188 .cls_prot = NET_PROT_IP, 3189 .cls_field = NH_FLD_IP_SRC, 3190 .id = DPAA2_ETH_DIST_IPSRC, 3191 .size = 4, 3192 }, { 3193 .rxnfc_field = RXH_IP_DST, 3194 .cls_prot = NET_PROT_IP, 3195 .cls_field = NH_FLD_IP_DST, 3196 .id = DPAA2_ETH_DIST_IPDST, 3197 .size = 4, 3198 }, { 3199 .rxnfc_field = RXH_L3_PROTO, 3200 .cls_prot = NET_PROT_IP, 3201 .cls_field = NH_FLD_IP_PROTO, 3202 .id = DPAA2_ETH_DIST_IPPROTO, 3203 .size = 1, 3204 }, { 3205 /* Using UDP ports, this is functionally equivalent to raw 3206 * byte pairs from L4 header. 3207 */ 3208 .rxnfc_field = RXH_L4_B_0_1, 3209 .cls_prot = NET_PROT_UDP, 3210 .cls_field = NH_FLD_UDP_PORT_SRC, 3211 .id = DPAA2_ETH_DIST_L4SRC, 3212 .size = 2, 3213 }, { 3214 .rxnfc_field = RXH_L4_B_2_3, 3215 .cls_prot = NET_PROT_UDP, 3216 .cls_field = NH_FLD_UDP_PORT_DST, 3217 .id = DPAA2_ETH_DIST_L4DST, 3218 .size = 2, 3219 }, 3220 }; 3221 3222 /* Configure the Rx hash key using the legacy API */ 3223 static int config_legacy_hash_key(struct dpaa2_eth_priv *priv, dma_addr_t key) 3224 { 3225 struct device *dev = priv->net_dev->dev.parent; 3226 struct dpni_rx_tc_dist_cfg dist_cfg; 3227 int i, err = 0; 3228 3229 memset(&dist_cfg, 0, sizeof(dist_cfg)); 3230 3231 dist_cfg.key_cfg_iova = key; 3232 dist_cfg.dist_size = dpaa2_eth_queue_count(priv); 3233 dist_cfg.dist_mode = DPNI_DIST_MODE_HASH; 3234 3235 for (i = 0; i < dpaa2_eth_tc_count(priv); i++) { 3236 err = dpni_set_rx_tc_dist(priv->mc_io, 0, priv->mc_token, 3237 i, &dist_cfg); 3238 if (err) { 3239 dev_err(dev, "dpni_set_rx_tc_dist failed\n"); 3240 break; 3241 } 3242 } 3243 3244 return err; 3245 } 3246 3247 /* Configure the Rx hash key using the new API */ 3248 static int config_hash_key(struct dpaa2_eth_priv *priv, dma_addr_t key) 3249 { 3250 struct device *dev = priv->net_dev->dev.parent; 3251 struct dpni_rx_dist_cfg dist_cfg; 3252 int i, err = 0; 3253 3254 memset(&dist_cfg, 0, sizeof(dist_cfg)); 3255 3256 dist_cfg.key_cfg_iova = key; 3257 dist_cfg.dist_size = dpaa2_eth_queue_count(priv); 3258 dist_cfg.enable = 1; 3259 3260 for (i = 0; i < dpaa2_eth_tc_count(priv); i++) { 3261 dist_cfg.tc = i; 3262 err = dpni_set_rx_hash_dist(priv->mc_io, 0, priv->mc_token, 3263 &dist_cfg); 3264 if (err) { 3265 dev_err(dev, "dpni_set_rx_hash_dist failed\n"); 3266 break; 3267 } 3268 } 3269 3270 return err; 3271 } 3272 3273 /* Configure the Rx flow classification key */ 3274 static int config_cls_key(struct dpaa2_eth_priv *priv, dma_addr_t key) 3275 { 3276 struct device *dev = priv->net_dev->dev.parent; 3277 struct dpni_rx_dist_cfg dist_cfg; 3278 int i, err = 0; 3279 3280 memset(&dist_cfg, 0, sizeof(dist_cfg)); 3281 3282 dist_cfg.key_cfg_iova = key; 3283 dist_cfg.dist_size = dpaa2_eth_queue_count(priv); 3284 dist_cfg.enable = 1; 3285 3286 for (i = 0; i < dpaa2_eth_tc_count(priv); i++) { 3287 dist_cfg.tc = i; 3288 err = dpni_set_rx_fs_dist(priv->mc_io, 0, priv->mc_token, 3289 &dist_cfg); 3290 if (err) { 3291 dev_err(dev, "dpni_set_rx_fs_dist failed\n"); 3292 break; 3293 } 3294 } 3295 3296 return err; 3297 } 3298 3299 /* Size of the Rx flow classification key */ 3300 int dpaa2_eth_cls_key_size(u64 fields) 3301 { 3302 int i, size = 0; 3303 3304 for (i = 0; i < ARRAY_SIZE(dist_fields); i++) { 3305 if (!(fields & dist_fields[i].id)) 3306 continue; 3307 size += dist_fields[i].size; 3308 } 3309 3310 return size; 3311 } 3312 3313 /* Offset of header field in Rx classification key */ 3314 int dpaa2_eth_cls_fld_off(int prot, int field) 3315 { 3316 int i, off = 0; 3317 3318 for (i = 0; i < ARRAY_SIZE(dist_fields); i++) { 3319 if (dist_fields[i].cls_prot == prot && 3320 dist_fields[i].cls_field == field) 3321 return off; 3322 off += dist_fields[i].size; 3323 } 3324 3325 WARN_ONCE(1, "Unsupported header field used for Rx flow cls\n"); 3326 return 0; 3327 } 3328 3329 /* Prune unused fields from the classification rule. 3330 * Used when masking is not supported 3331 */ 3332 void dpaa2_eth_cls_trim_rule(void *key_mem, u64 fields) 3333 { 3334 int off = 0, new_off = 0; 3335 int i, size; 3336 3337 for (i = 0; i < ARRAY_SIZE(dist_fields); i++) { 3338 size = dist_fields[i].size; 3339 if (dist_fields[i].id & fields) { 3340 memcpy(key_mem + new_off, key_mem + off, size); 3341 new_off += size; 3342 } 3343 off += size; 3344 } 3345 } 3346 3347 /* Set Rx distribution (hash or flow classification) key 3348 * flags is a combination of RXH_ bits 3349 */ 3350 static int dpaa2_eth_set_dist_key(struct net_device *net_dev, 3351 enum dpaa2_eth_rx_dist type, u64 flags) 3352 { 3353 struct device *dev = net_dev->dev.parent; 3354 struct dpaa2_eth_priv *priv = netdev_priv(net_dev); 3355 struct dpkg_profile_cfg cls_cfg; 3356 u32 rx_hash_fields = 0; 3357 dma_addr_t key_iova; 3358 u8 *dma_mem; 3359 int i; 3360 int err = 0; 3361 3362 memset(&cls_cfg, 0, sizeof(cls_cfg)); 3363 3364 for (i = 0; i < ARRAY_SIZE(dist_fields); i++) { 3365 struct dpkg_extract *key = 3366 &cls_cfg.extracts[cls_cfg.num_extracts]; 3367 3368 /* For both Rx hashing and classification keys 3369 * we set only the selected fields. 3370 */ 3371 if (!(flags & dist_fields[i].id)) 3372 continue; 3373 if (type == DPAA2_ETH_RX_DIST_HASH) 3374 rx_hash_fields |= dist_fields[i].rxnfc_field; 3375 3376 if (cls_cfg.num_extracts >= DPKG_MAX_NUM_OF_EXTRACTS) { 3377 dev_err(dev, "error adding key extraction rule, too many rules?\n"); 3378 return -E2BIG; 3379 } 3380 3381 key->type = DPKG_EXTRACT_FROM_HDR; 3382 key->extract.from_hdr.prot = dist_fields[i].cls_prot; 3383 key->extract.from_hdr.type = DPKG_FULL_FIELD; 3384 key->extract.from_hdr.field = dist_fields[i].cls_field; 3385 cls_cfg.num_extracts++; 3386 } 3387 3388 dma_mem = kzalloc(DPAA2_CLASSIFIER_DMA_SIZE, GFP_KERNEL); 3389 if (!dma_mem) 3390 return -ENOMEM; 3391 3392 err = dpni_prepare_key_cfg(&cls_cfg, dma_mem); 3393 if (err) { 3394 dev_err(dev, "dpni_prepare_key_cfg error %d\n", err); 3395 goto free_key; 3396 } 3397 3398 /* Prepare for setting the rx dist */ 3399 key_iova = dma_map_single(dev, dma_mem, DPAA2_CLASSIFIER_DMA_SIZE, 3400 DMA_TO_DEVICE); 3401 if (dma_mapping_error(dev, key_iova)) { 3402 dev_err(dev, "DMA mapping failed\n"); 3403 err = -ENOMEM; 3404 goto free_key; 3405 } 3406 3407 if (type == DPAA2_ETH_RX_DIST_HASH) { 3408 if (dpaa2_eth_has_legacy_dist(priv)) 3409 err = config_legacy_hash_key(priv, key_iova); 3410 else 3411 err = config_hash_key(priv, key_iova); 3412 } else { 3413 err = config_cls_key(priv, key_iova); 3414 } 3415 3416 dma_unmap_single(dev, key_iova, DPAA2_CLASSIFIER_DMA_SIZE, 3417 DMA_TO_DEVICE); 3418 if (!err && type == DPAA2_ETH_RX_DIST_HASH) 3419 priv->rx_hash_fields = rx_hash_fields; 3420 3421 free_key: 3422 kfree(dma_mem); 3423 return err; 3424 } 3425 3426 int dpaa2_eth_set_hash(struct net_device *net_dev, u64 flags) 3427 { 3428 struct dpaa2_eth_priv *priv = netdev_priv(net_dev); 3429 u64 key = 0; 3430 int i; 3431 3432 if (!dpaa2_eth_hash_enabled(priv)) 3433 return -EOPNOTSUPP; 3434 3435 for (i = 0; i < ARRAY_SIZE(dist_fields); i++) 3436 if (dist_fields[i].rxnfc_field & flags) 3437 key |= dist_fields[i].id; 3438 3439 return dpaa2_eth_set_dist_key(net_dev, DPAA2_ETH_RX_DIST_HASH, key); 3440 } 3441 3442 int dpaa2_eth_set_cls(struct net_device *net_dev, u64 flags) 3443 { 3444 return dpaa2_eth_set_dist_key(net_dev, DPAA2_ETH_RX_DIST_CLS, flags); 3445 } 3446 3447 static int dpaa2_eth_set_default_cls(struct dpaa2_eth_priv *priv) 3448 { 3449 struct device *dev = priv->net_dev->dev.parent; 3450 int err; 3451 3452 /* Check if we actually support Rx flow classification */ 3453 if (dpaa2_eth_has_legacy_dist(priv)) { 3454 dev_dbg(dev, "Rx cls not supported by current MC version\n"); 3455 return -EOPNOTSUPP; 3456 } 3457 3458 if (!dpaa2_eth_fs_enabled(priv)) { 3459 dev_dbg(dev, "Rx cls disabled in DPNI options\n"); 3460 return -EOPNOTSUPP; 3461 } 3462 3463 if (!dpaa2_eth_hash_enabled(priv)) { 3464 dev_dbg(dev, "Rx cls disabled for single queue DPNIs\n"); 3465 return -EOPNOTSUPP; 3466 } 3467 3468 /* If there is no support for masking in the classification table, 3469 * we don't set a default key, as it will depend on the rules 3470 * added by the user at runtime. 3471 */ 3472 if (!dpaa2_eth_fs_mask_enabled(priv)) 3473 goto out; 3474 3475 err = dpaa2_eth_set_cls(priv->net_dev, DPAA2_ETH_DIST_ALL); 3476 if (err) 3477 return err; 3478 3479 out: 3480 priv->rx_cls_enabled = 1; 3481 3482 return 0; 3483 } 3484 3485 /* Bind the DPNI to its needed objects and resources: buffer pool, DPIOs, 3486 * frame queues and channels 3487 */ 3488 static int bind_dpni(struct dpaa2_eth_priv *priv) 3489 { 3490 struct net_device *net_dev = priv->net_dev; 3491 struct device *dev = net_dev->dev.parent; 3492 struct dpni_pools_cfg pools_params; 3493 struct dpni_error_cfg err_cfg; 3494 int err = 0; 3495 int i; 3496 3497 pools_params.num_dpbp = 1; 3498 pools_params.pools[0].dpbp_id = priv->dpbp_dev->obj_desc.id; 3499 pools_params.pools[0].backup_pool = 0; 3500 pools_params.pools[0].buffer_size = priv->rx_buf_size; 3501 err = dpni_set_pools(priv->mc_io, 0, priv->mc_token, &pools_params); 3502 if (err) { 3503 dev_err(dev, "dpni_set_pools() failed\n"); 3504 return err; 3505 } 3506 3507 /* have the interface implicitly distribute traffic based on 3508 * the default hash key 3509 */ 3510 err = dpaa2_eth_set_hash(net_dev, DPAA2_RXH_DEFAULT); 3511 if (err && err != -EOPNOTSUPP) 3512 dev_err(dev, "Failed to configure hashing\n"); 3513 3514 /* Configure the flow classification key; it includes all 3515 * supported header fields and cannot be modified at runtime 3516 */ 3517 err = dpaa2_eth_set_default_cls(priv); 3518 if (err && err != -EOPNOTSUPP) 3519 dev_err(dev, "Failed to configure Rx classification key\n"); 3520 3521 /* Configure handling of error frames */ 3522 err_cfg.errors = DPAA2_FAS_RX_ERR_MASK; 3523 err_cfg.set_frame_annotation = 1; 3524 err_cfg.error_action = DPNI_ERROR_ACTION_DISCARD; 3525 err = dpni_set_errors_behavior(priv->mc_io, 0, priv->mc_token, 3526 &err_cfg); 3527 if (err) { 3528 dev_err(dev, "dpni_set_errors_behavior failed\n"); 3529 return err; 3530 } 3531 3532 /* Configure Rx and Tx conf queues to generate CDANs */ 3533 for (i = 0; i < priv->num_fqs; i++) { 3534 switch (priv->fq[i].type) { 3535 case DPAA2_RX_FQ: 3536 err = setup_rx_flow(priv, &priv->fq[i]); 3537 break; 3538 case DPAA2_TX_CONF_FQ: 3539 err = setup_tx_flow(priv, &priv->fq[i]); 3540 break; 3541 default: 3542 dev_err(dev, "Invalid FQ type %d\n", priv->fq[i].type); 3543 return -EINVAL; 3544 } 3545 if (err) 3546 return err; 3547 } 3548 3549 err = dpni_get_qdid(priv->mc_io, 0, priv->mc_token, 3550 DPNI_QUEUE_TX, &priv->tx_qdid); 3551 if (err) { 3552 dev_err(dev, "dpni_get_qdid() failed\n"); 3553 return err; 3554 } 3555 3556 return 0; 3557 } 3558 3559 /* Allocate rings for storing incoming frame descriptors */ 3560 static int alloc_rings(struct dpaa2_eth_priv *priv) 3561 { 3562 struct net_device *net_dev = priv->net_dev; 3563 struct device *dev = net_dev->dev.parent; 3564 int i; 3565 3566 for (i = 0; i < priv->num_channels; i++) { 3567 priv->channel[i]->store = 3568 dpaa2_io_store_create(DPAA2_ETH_STORE_SIZE, dev); 3569 if (!priv->channel[i]->store) { 3570 netdev_err(net_dev, "dpaa2_io_store_create() failed\n"); 3571 goto err_ring; 3572 } 3573 } 3574 3575 return 0; 3576 3577 err_ring: 3578 for (i = 0; i < priv->num_channels; i++) { 3579 if (!priv->channel[i]->store) 3580 break; 3581 dpaa2_io_store_destroy(priv->channel[i]->store); 3582 } 3583 3584 return -ENOMEM; 3585 } 3586 3587 static void free_rings(struct dpaa2_eth_priv *priv) 3588 { 3589 int i; 3590 3591 for (i = 0; i < priv->num_channels; i++) 3592 dpaa2_io_store_destroy(priv->channel[i]->store); 3593 } 3594 3595 static int set_mac_addr(struct dpaa2_eth_priv *priv) 3596 { 3597 struct net_device *net_dev = priv->net_dev; 3598 struct device *dev = net_dev->dev.parent; 3599 u8 mac_addr[ETH_ALEN], dpni_mac_addr[ETH_ALEN]; 3600 int err; 3601 3602 /* Get firmware address, if any */ 3603 err = dpni_get_port_mac_addr(priv->mc_io, 0, priv->mc_token, mac_addr); 3604 if (err) { 3605 dev_err(dev, "dpni_get_port_mac_addr() failed\n"); 3606 return err; 3607 } 3608 3609 /* Get DPNI attributes address, if any */ 3610 err = dpni_get_primary_mac_addr(priv->mc_io, 0, priv->mc_token, 3611 dpni_mac_addr); 3612 if (err) { 3613 dev_err(dev, "dpni_get_primary_mac_addr() failed\n"); 3614 return err; 3615 } 3616 3617 /* First check if firmware has any address configured by bootloader */ 3618 if (!is_zero_ether_addr(mac_addr)) { 3619 /* If the DPMAC addr != DPNI addr, update it */ 3620 if (!ether_addr_equal(mac_addr, dpni_mac_addr)) { 3621 err = dpni_set_primary_mac_addr(priv->mc_io, 0, 3622 priv->mc_token, 3623 mac_addr); 3624 if (err) { 3625 dev_err(dev, "dpni_set_primary_mac_addr() failed\n"); 3626 return err; 3627 } 3628 } 3629 memcpy(net_dev->dev_addr, mac_addr, net_dev->addr_len); 3630 } else if (is_zero_ether_addr(dpni_mac_addr)) { 3631 /* No MAC address configured, fill in net_dev->dev_addr 3632 * with a random one 3633 */ 3634 eth_hw_addr_random(net_dev); 3635 dev_dbg_once(dev, "device(s) have all-zero hwaddr, replaced with random\n"); 3636 3637 err = dpni_set_primary_mac_addr(priv->mc_io, 0, priv->mc_token, 3638 net_dev->dev_addr); 3639 if (err) { 3640 dev_err(dev, "dpni_set_primary_mac_addr() failed\n"); 3641 return err; 3642 } 3643 3644 /* Override NET_ADDR_RANDOM set by eth_hw_addr_random(); for all 3645 * practical purposes, this will be our "permanent" mac address, 3646 * at least until the next reboot. This move will also permit 3647 * register_netdevice() to properly fill up net_dev->perm_addr. 3648 */ 3649 net_dev->addr_assign_type = NET_ADDR_PERM; 3650 } else { 3651 /* NET_ADDR_PERM is default, all we have to do is 3652 * fill in the device addr. 3653 */ 3654 memcpy(net_dev->dev_addr, dpni_mac_addr, net_dev->addr_len); 3655 } 3656 3657 return 0; 3658 } 3659 3660 static int netdev_init(struct net_device *net_dev) 3661 { 3662 struct device *dev = net_dev->dev.parent; 3663 struct dpaa2_eth_priv *priv = netdev_priv(net_dev); 3664 u32 options = priv->dpni_attrs.options; 3665 u64 supported = 0, not_supported = 0; 3666 u8 bcast_addr[ETH_ALEN]; 3667 u8 num_queues; 3668 int err; 3669 3670 net_dev->netdev_ops = &dpaa2_eth_ops; 3671 net_dev->ethtool_ops = &dpaa2_ethtool_ops; 3672 3673 err = set_mac_addr(priv); 3674 if (err) 3675 return err; 3676 3677 /* Explicitly add the broadcast address to the MAC filtering table */ 3678 eth_broadcast_addr(bcast_addr); 3679 err = dpni_add_mac_addr(priv->mc_io, 0, priv->mc_token, bcast_addr); 3680 if (err) { 3681 dev_err(dev, "dpni_add_mac_addr() failed\n"); 3682 return err; 3683 } 3684 3685 /* Set MTU upper limit; lower limit is 68B (default value) */ 3686 net_dev->max_mtu = DPAA2_ETH_MAX_MTU; 3687 err = dpni_set_max_frame_length(priv->mc_io, 0, priv->mc_token, 3688 DPAA2_ETH_MFL); 3689 if (err) { 3690 dev_err(dev, "dpni_set_max_frame_length() failed\n"); 3691 return err; 3692 } 3693 3694 /* Set actual number of queues in the net device */ 3695 num_queues = dpaa2_eth_queue_count(priv); 3696 err = netif_set_real_num_tx_queues(net_dev, num_queues); 3697 if (err) { 3698 dev_err(dev, "netif_set_real_num_tx_queues() failed\n"); 3699 return err; 3700 } 3701 err = netif_set_real_num_rx_queues(net_dev, num_queues); 3702 if (err) { 3703 dev_err(dev, "netif_set_real_num_rx_queues() failed\n"); 3704 return err; 3705 } 3706 3707 /* Capabilities listing */ 3708 supported |= IFF_LIVE_ADDR_CHANGE; 3709 3710 if (options & DPNI_OPT_NO_MAC_FILTER) 3711 not_supported |= IFF_UNICAST_FLT; 3712 else 3713 supported |= IFF_UNICAST_FLT; 3714 3715 net_dev->priv_flags |= supported; 3716 net_dev->priv_flags &= ~not_supported; 3717 3718 /* Features */ 3719 net_dev->features = NETIF_F_RXCSUM | 3720 NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | 3721 NETIF_F_SG | NETIF_F_HIGHDMA | 3722 NETIF_F_LLTX; 3723 net_dev->hw_features = net_dev->features; 3724 3725 return 0; 3726 } 3727 3728 static int poll_link_state(void *arg) 3729 { 3730 struct dpaa2_eth_priv *priv = (struct dpaa2_eth_priv *)arg; 3731 int err; 3732 3733 while (!kthread_should_stop()) { 3734 err = link_state_update(priv); 3735 if (unlikely(err)) 3736 return err; 3737 3738 msleep(DPAA2_ETH_LINK_STATE_REFRESH); 3739 } 3740 3741 return 0; 3742 } 3743 3744 static int dpaa2_eth_connect_mac(struct dpaa2_eth_priv *priv) 3745 { 3746 struct fsl_mc_device *dpni_dev, *dpmac_dev; 3747 struct dpaa2_mac *mac; 3748 int err; 3749 3750 dpni_dev = to_fsl_mc_device(priv->net_dev->dev.parent); 3751 dpmac_dev = fsl_mc_get_endpoint(dpni_dev); 3752 if (IS_ERR(dpmac_dev) || dpmac_dev->dev.type != &fsl_mc_bus_dpmac_type) 3753 return 0; 3754 3755 if (dpaa2_mac_is_type_fixed(dpmac_dev, priv->mc_io)) 3756 return 0; 3757 3758 mac = kzalloc(sizeof(struct dpaa2_mac), GFP_KERNEL); 3759 if (!mac) 3760 return -ENOMEM; 3761 3762 mac->mc_dev = dpmac_dev; 3763 mac->mc_io = priv->mc_io; 3764 mac->net_dev = priv->net_dev; 3765 3766 err = dpaa2_mac_connect(mac); 3767 if (err) { 3768 netdev_err(priv->net_dev, "Error connecting to the MAC endpoint\n"); 3769 kfree(mac); 3770 return err; 3771 } 3772 priv->mac = mac; 3773 3774 return 0; 3775 } 3776 3777 static void dpaa2_eth_disconnect_mac(struct dpaa2_eth_priv *priv) 3778 { 3779 if (!priv->mac) 3780 return; 3781 3782 dpaa2_mac_disconnect(priv->mac); 3783 kfree(priv->mac); 3784 priv->mac = NULL; 3785 } 3786 3787 static irqreturn_t dpni_irq0_handler_thread(int irq_num, void *arg) 3788 { 3789 u32 status = ~0; 3790 struct device *dev = (struct device *)arg; 3791 struct fsl_mc_device *dpni_dev = to_fsl_mc_device(dev); 3792 struct net_device *net_dev = dev_get_drvdata(dev); 3793 struct dpaa2_eth_priv *priv = netdev_priv(net_dev); 3794 int err; 3795 3796 err = dpni_get_irq_status(dpni_dev->mc_io, 0, dpni_dev->mc_handle, 3797 DPNI_IRQ_INDEX, &status); 3798 if (unlikely(err)) { 3799 netdev_err(net_dev, "Can't get irq status (err %d)\n", err); 3800 return IRQ_HANDLED; 3801 } 3802 3803 if (status & DPNI_IRQ_EVENT_LINK_CHANGED) 3804 link_state_update(netdev_priv(net_dev)); 3805 3806 if (status & DPNI_IRQ_EVENT_ENDPOINT_CHANGED) { 3807 set_mac_addr(netdev_priv(net_dev)); 3808 update_tx_fqids(priv); 3809 3810 rtnl_lock(); 3811 if (priv->mac) 3812 dpaa2_eth_disconnect_mac(priv); 3813 else 3814 dpaa2_eth_connect_mac(priv); 3815 rtnl_unlock(); 3816 } 3817 3818 return IRQ_HANDLED; 3819 } 3820 3821 static int setup_irqs(struct fsl_mc_device *ls_dev) 3822 { 3823 int err = 0; 3824 struct fsl_mc_device_irq *irq; 3825 3826 err = fsl_mc_allocate_irqs(ls_dev); 3827 if (err) { 3828 dev_err(&ls_dev->dev, "MC irqs allocation failed\n"); 3829 return err; 3830 } 3831 3832 irq = ls_dev->irqs[0]; 3833 err = devm_request_threaded_irq(&ls_dev->dev, irq->msi_desc->irq, 3834 NULL, dpni_irq0_handler_thread, 3835 IRQF_NO_SUSPEND | IRQF_ONESHOT, 3836 dev_name(&ls_dev->dev), &ls_dev->dev); 3837 if (err < 0) { 3838 dev_err(&ls_dev->dev, "devm_request_threaded_irq(): %d\n", err); 3839 goto free_mc_irq; 3840 } 3841 3842 err = dpni_set_irq_mask(ls_dev->mc_io, 0, ls_dev->mc_handle, 3843 DPNI_IRQ_INDEX, DPNI_IRQ_EVENT_LINK_CHANGED | 3844 DPNI_IRQ_EVENT_ENDPOINT_CHANGED); 3845 if (err < 0) { 3846 dev_err(&ls_dev->dev, "dpni_set_irq_mask(): %d\n", err); 3847 goto free_irq; 3848 } 3849 3850 err = dpni_set_irq_enable(ls_dev->mc_io, 0, ls_dev->mc_handle, 3851 DPNI_IRQ_INDEX, 1); 3852 if (err < 0) { 3853 dev_err(&ls_dev->dev, "dpni_set_irq_enable(): %d\n", err); 3854 goto free_irq; 3855 } 3856 3857 return 0; 3858 3859 free_irq: 3860 devm_free_irq(&ls_dev->dev, irq->msi_desc->irq, &ls_dev->dev); 3861 free_mc_irq: 3862 fsl_mc_free_irqs(ls_dev); 3863 3864 return err; 3865 } 3866 3867 static void add_ch_napi(struct dpaa2_eth_priv *priv) 3868 { 3869 int i; 3870 struct dpaa2_eth_channel *ch; 3871 3872 for (i = 0; i < priv->num_channels; i++) { 3873 ch = priv->channel[i]; 3874 /* NAPI weight *MUST* be a multiple of DPAA2_ETH_STORE_SIZE */ 3875 netif_napi_add(priv->net_dev, &ch->napi, dpaa2_eth_poll, 3876 NAPI_POLL_WEIGHT); 3877 } 3878 } 3879 3880 static void del_ch_napi(struct dpaa2_eth_priv *priv) 3881 { 3882 int i; 3883 struct dpaa2_eth_channel *ch; 3884 3885 for (i = 0; i < priv->num_channels; i++) { 3886 ch = priv->channel[i]; 3887 netif_napi_del(&ch->napi); 3888 } 3889 } 3890 3891 static int dpaa2_eth_probe(struct fsl_mc_device *dpni_dev) 3892 { 3893 struct device *dev; 3894 struct net_device *net_dev = NULL; 3895 struct dpaa2_eth_priv *priv = NULL; 3896 int err = 0; 3897 3898 dev = &dpni_dev->dev; 3899 3900 /* Net device */ 3901 net_dev = alloc_etherdev_mq(sizeof(*priv), DPAA2_ETH_MAX_NETDEV_QUEUES); 3902 if (!net_dev) { 3903 dev_err(dev, "alloc_etherdev_mq() failed\n"); 3904 return -ENOMEM; 3905 } 3906 3907 SET_NETDEV_DEV(net_dev, dev); 3908 dev_set_drvdata(dev, net_dev); 3909 3910 priv = netdev_priv(net_dev); 3911 priv->net_dev = net_dev; 3912 3913 priv->iommu_domain = iommu_get_domain_for_dev(dev); 3914 3915 /* Obtain a MC portal */ 3916 err = fsl_mc_portal_allocate(dpni_dev, FSL_MC_IO_ATOMIC_CONTEXT_PORTAL, 3917 &priv->mc_io); 3918 if (err) { 3919 if (err == -ENXIO) 3920 err = -EPROBE_DEFER; 3921 else 3922 dev_err(dev, "MC portal allocation failed\n"); 3923 goto err_portal_alloc; 3924 } 3925 3926 /* MC objects initialization and configuration */ 3927 err = setup_dpni(dpni_dev); 3928 if (err) 3929 goto err_dpni_setup; 3930 3931 err = setup_dpio(priv); 3932 if (err) 3933 goto err_dpio_setup; 3934 3935 setup_fqs(priv); 3936 3937 err = setup_dpbp(priv); 3938 if (err) 3939 goto err_dpbp_setup; 3940 3941 err = bind_dpni(priv); 3942 if (err) 3943 goto err_bind; 3944 3945 /* Add a NAPI context for each channel */ 3946 add_ch_napi(priv); 3947 3948 /* Percpu statistics */ 3949 priv->percpu_stats = alloc_percpu(*priv->percpu_stats); 3950 if (!priv->percpu_stats) { 3951 dev_err(dev, "alloc_percpu(percpu_stats) failed\n"); 3952 err = -ENOMEM; 3953 goto err_alloc_percpu_stats; 3954 } 3955 priv->percpu_extras = alloc_percpu(*priv->percpu_extras); 3956 if (!priv->percpu_extras) { 3957 dev_err(dev, "alloc_percpu(percpu_extras) failed\n"); 3958 err = -ENOMEM; 3959 goto err_alloc_percpu_extras; 3960 } 3961 3962 priv->sgt_cache = alloc_percpu(*priv->sgt_cache); 3963 if (!priv->sgt_cache) { 3964 dev_err(dev, "alloc_percpu(sgt_cache) failed\n"); 3965 err = -ENOMEM; 3966 goto err_alloc_sgt_cache; 3967 } 3968 3969 err = netdev_init(net_dev); 3970 if (err) 3971 goto err_netdev_init; 3972 3973 /* Configure checksum offload based on current interface flags */ 3974 err = set_rx_csum(priv, !!(net_dev->features & NETIF_F_RXCSUM)); 3975 if (err) 3976 goto err_csum; 3977 3978 err = set_tx_csum(priv, !!(net_dev->features & 3979 (NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM))); 3980 if (err) 3981 goto err_csum; 3982 3983 err = alloc_rings(priv); 3984 if (err) 3985 goto err_alloc_rings; 3986 3987 #ifdef CONFIG_FSL_DPAA2_ETH_DCB 3988 if (dpaa2_eth_has_pause_support(priv) && priv->vlan_cls_enabled) { 3989 priv->dcbx_mode = DCB_CAP_DCBX_HOST | DCB_CAP_DCBX_VER_IEEE; 3990 net_dev->dcbnl_ops = &dpaa2_eth_dcbnl_ops; 3991 } else { 3992 dev_dbg(dev, "PFC not supported\n"); 3993 } 3994 #endif 3995 3996 err = setup_irqs(dpni_dev); 3997 if (err) { 3998 netdev_warn(net_dev, "Failed to set link interrupt, fall back to polling\n"); 3999 priv->poll_thread = kthread_run(poll_link_state, priv, 4000 "%s_poll_link", net_dev->name); 4001 if (IS_ERR(priv->poll_thread)) { 4002 dev_err(dev, "Error starting polling thread\n"); 4003 goto err_poll_thread; 4004 } 4005 priv->do_link_poll = true; 4006 } 4007 4008 err = dpaa2_eth_connect_mac(priv); 4009 if (err) 4010 goto err_connect_mac; 4011 4012 err = register_netdev(net_dev); 4013 if (err < 0) { 4014 dev_err(dev, "register_netdev() failed\n"); 4015 goto err_netdev_reg; 4016 } 4017 4018 #ifdef CONFIG_DEBUG_FS 4019 dpaa2_dbg_add(priv); 4020 #endif 4021 4022 dev_info(dev, "Probed interface %s\n", net_dev->name); 4023 return 0; 4024 4025 err_netdev_reg: 4026 dpaa2_eth_disconnect_mac(priv); 4027 err_connect_mac: 4028 if (priv->do_link_poll) 4029 kthread_stop(priv->poll_thread); 4030 else 4031 fsl_mc_free_irqs(dpni_dev); 4032 err_poll_thread: 4033 free_rings(priv); 4034 err_alloc_rings: 4035 err_csum: 4036 err_netdev_init: 4037 free_percpu(priv->sgt_cache); 4038 err_alloc_sgt_cache: 4039 free_percpu(priv->percpu_extras); 4040 err_alloc_percpu_extras: 4041 free_percpu(priv->percpu_stats); 4042 err_alloc_percpu_stats: 4043 del_ch_napi(priv); 4044 err_bind: 4045 free_dpbp(priv); 4046 err_dpbp_setup: 4047 free_dpio(priv); 4048 err_dpio_setup: 4049 free_dpni(priv); 4050 err_dpni_setup: 4051 fsl_mc_portal_free(priv->mc_io); 4052 err_portal_alloc: 4053 dev_set_drvdata(dev, NULL); 4054 free_netdev(net_dev); 4055 4056 return err; 4057 } 4058 4059 static int dpaa2_eth_remove(struct fsl_mc_device *ls_dev) 4060 { 4061 struct device *dev; 4062 struct net_device *net_dev; 4063 struct dpaa2_eth_priv *priv; 4064 4065 dev = &ls_dev->dev; 4066 net_dev = dev_get_drvdata(dev); 4067 priv = netdev_priv(net_dev); 4068 4069 #ifdef CONFIG_DEBUG_FS 4070 dpaa2_dbg_remove(priv); 4071 #endif 4072 rtnl_lock(); 4073 dpaa2_eth_disconnect_mac(priv); 4074 rtnl_unlock(); 4075 4076 unregister_netdev(net_dev); 4077 4078 if (priv->do_link_poll) 4079 kthread_stop(priv->poll_thread); 4080 else 4081 fsl_mc_free_irqs(ls_dev); 4082 4083 free_rings(priv); 4084 free_percpu(priv->sgt_cache); 4085 free_percpu(priv->percpu_stats); 4086 free_percpu(priv->percpu_extras); 4087 4088 del_ch_napi(priv); 4089 free_dpbp(priv); 4090 free_dpio(priv); 4091 free_dpni(priv); 4092 4093 fsl_mc_portal_free(priv->mc_io); 4094 4095 free_netdev(net_dev); 4096 4097 dev_dbg(net_dev->dev.parent, "Removed interface %s\n", net_dev->name); 4098 4099 return 0; 4100 } 4101 4102 static const struct fsl_mc_device_id dpaa2_eth_match_id_table[] = { 4103 { 4104 .vendor = FSL_MC_VENDOR_FREESCALE, 4105 .obj_type = "dpni", 4106 }, 4107 { .vendor = 0x0 } 4108 }; 4109 MODULE_DEVICE_TABLE(fslmc, dpaa2_eth_match_id_table); 4110 4111 static struct fsl_mc_driver dpaa2_eth_driver = { 4112 .driver = { 4113 .name = KBUILD_MODNAME, 4114 .owner = THIS_MODULE, 4115 }, 4116 .probe = dpaa2_eth_probe, 4117 .remove = dpaa2_eth_remove, 4118 .match_id_table = dpaa2_eth_match_id_table 4119 }; 4120 4121 static int __init dpaa2_eth_driver_init(void) 4122 { 4123 int err; 4124 4125 dpaa2_eth_dbg_init(); 4126 err = fsl_mc_driver_register(&dpaa2_eth_driver); 4127 if (err) { 4128 dpaa2_eth_dbg_exit(); 4129 return err; 4130 } 4131 4132 return 0; 4133 } 4134 4135 static void __exit dpaa2_eth_driver_exit(void) 4136 { 4137 dpaa2_eth_dbg_exit(); 4138 fsl_mc_driver_unregister(&dpaa2_eth_driver); 4139 } 4140 4141 module_init(dpaa2_eth_driver_init); 4142 module_exit(dpaa2_eth_driver_exit); 4143