1 // SPDX-License-Identifier: GPL-2.0+ 2 3 #include <linux/bpf.h> 4 #include <linux/filter.h> 5 #include <net/page_pool/helpers.h> 6 7 #include "lan966x_main.h" 8 9 static int lan966x_fdma_rx_dataptr_cb(struct fdma *fdma, int dcb, int db, 10 u64 *dataptr) 11 { 12 struct lan966x *lan966x = (struct lan966x *)fdma->priv; 13 struct lan966x_rx *rx = &lan966x->rx; 14 struct page *page; 15 16 page = page_pool_dev_alloc_pages(rx->page_pool); 17 if (unlikely(!page)) 18 return -ENOMEM; 19 20 rx->page[dcb][db] = page; 21 *dataptr = page_pool_get_dma_addr(page) + XDP_PACKET_HEADROOM; 22 23 return 0; 24 } 25 26 static int lan966x_fdma_tx_dataptr_cb(struct fdma *fdma, int dcb, int db, 27 u64 *dataptr) 28 { 29 struct lan966x *lan966x = (struct lan966x *)fdma->priv; 30 31 *dataptr = lan966x->tx.dcbs_buf[dcb].dma_addr; 32 33 return 0; 34 } 35 36 static int lan966x_fdma_xdp_tx_dataptr_cb(struct fdma *fdma, int dcb, int db, 37 u64 *dataptr) 38 { 39 struct lan966x *lan966x = (struct lan966x *)fdma->priv; 40 41 *dataptr = lan966x->tx.dcbs_buf[dcb].dma_addr + XDP_PACKET_HEADROOM; 42 43 return 0; 44 } 45 46 static int lan966x_fdma_channel_active(struct lan966x *lan966x) 47 { 48 return lan_rd(lan966x, FDMA_CH_ACTIVE); 49 } 50 51 static void lan966x_fdma_rx_free_pages(struct lan966x_rx *rx) 52 { 53 struct fdma *fdma = &rx->fdma; 54 int i, j; 55 56 for (i = 0; i < fdma->n_dcbs; ++i) { 57 for (j = 0; j < fdma->n_dbs; ++j) 58 page_pool_put_full_page(rx->page_pool, 59 rx->page[i][j], false); 60 } 61 } 62 63 static void lan966x_fdma_rx_free_page(struct lan966x_rx *rx) 64 { 65 struct fdma *fdma = &rx->fdma; 66 struct page *page; 67 68 page = rx->page[fdma->dcb_index][fdma->db_index]; 69 if (unlikely(!page)) 70 return; 71 72 page_pool_recycle_direct(rx->page_pool, page); 73 } 74 75 static int lan966x_fdma_rx_alloc_page_pool(struct lan966x_rx *rx) 76 { 77 struct lan966x *lan966x = rx->lan966x; 78 struct page_pool_params pp_params = { 79 .order = rx->page_order, 80 .flags = PP_FLAG_DMA_MAP | PP_FLAG_DMA_SYNC_DEV, 81 .pool_size = rx->fdma.n_dcbs, 82 .nid = NUMA_NO_NODE, 83 .dev = lan966x->dev, 84 .dma_dir = DMA_FROM_DEVICE, 85 .offset = XDP_PACKET_HEADROOM, 86 .max_len = rx->max_mtu - 87 SKB_DATA_ALIGN(sizeof(struct skb_shared_info)), 88 }; 89 90 if (lan966x_xdp_present(lan966x)) 91 pp_params.dma_dir = DMA_BIDIRECTIONAL; 92 93 rx->page_pool = page_pool_create(&pp_params); 94 95 for (int i = 0; i < lan966x->num_phys_ports; ++i) { 96 struct lan966x_port *port; 97 98 if (!lan966x->ports[i]) 99 continue; 100 101 port = lan966x->ports[i]; 102 xdp_rxq_info_unreg_mem_model(&port->xdp_rxq); 103 xdp_rxq_info_reg_mem_model(&port->xdp_rxq, MEM_TYPE_PAGE_POOL, 104 rx->page_pool); 105 } 106 107 return PTR_ERR_OR_ZERO(rx->page_pool); 108 } 109 110 static int lan966x_fdma_rx_alloc(struct lan966x_rx *rx) 111 { 112 struct lan966x *lan966x = rx->lan966x; 113 struct fdma *fdma = &rx->fdma; 114 int err; 115 116 if (lan966x_fdma_rx_alloc_page_pool(rx)) 117 return PTR_ERR(rx->page_pool); 118 119 err = fdma_alloc_coherent(lan966x->dev, fdma); 120 if (err) 121 return err; 122 123 fdma_dcbs_init(fdma, FDMA_DCB_INFO_DATAL(fdma->db_size), 124 FDMA_DCB_STATUS_INTR); 125 126 return 0; 127 } 128 129 static void lan966x_fdma_rx_start(struct lan966x_rx *rx) 130 { 131 struct lan966x *lan966x = rx->lan966x; 132 struct fdma *fdma = &rx->fdma; 133 u32 mask; 134 135 /* When activating a channel, first is required to write the first DCB 136 * address and then to activate it 137 */ 138 lan_wr(lower_32_bits((u64)fdma->dma), lan966x, 139 FDMA_DCB_LLP(fdma->channel_id)); 140 lan_wr(upper_32_bits((u64)fdma->dma), lan966x, 141 FDMA_DCB_LLP1(fdma->channel_id)); 142 143 lan_wr(FDMA_CH_CFG_CH_DCB_DB_CNT_SET(fdma->n_dbs) | 144 FDMA_CH_CFG_CH_INTR_DB_EOF_ONLY_SET(1) | 145 FDMA_CH_CFG_CH_INJ_PORT_SET(0) | 146 FDMA_CH_CFG_CH_MEM_SET(1), 147 lan966x, FDMA_CH_CFG(fdma->channel_id)); 148 149 /* Start fdma */ 150 lan_rmw(FDMA_PORT_CTRL_XTR_STOP_SET(0), 151 FDMA_PORT_CTRL_XTR_STOP, 152 lan966x, FDMA_PORT_CTRL(0)); 153 154 /* Enable interrupts */ 155 mask = lan_rd(lan966x, FDMA_INTR_DB_ENA); 156 mask = FDMA_INTR_DB_ENA_INTR_DB_ENA_GET(mask); 157 mask |= BIT(fdma->channel_id); 158 lan_rmw(FDMA_INTR_DB_ENA_INTR_DB_ENA_SET(mask), 159 FDMA_INTR_DB_ENA_INTR_DB_ENA, 160 lan966x, FDMA_INTR_DB_ENA); 161 162 /* Activate the channel */ 163 lan_rmw(FDMA_CH_ACTIVATE_CH_ACTIVATE_SET(BIT(fdma->channel_id)), 164 FDMA_CH_ACTIVATE_CH_ACTIVATE, 165 lan966x, FDMA_CH_ACTIVATE); 166 } 167 168 static void lan966x_fdma_rx_disable(struct lan966x_rx *rx) 169 { 170 struct lan966x *lan966x = rx->lan966x; 171 struct fdma *fdma = &rx->fdma; 172 u32 val; 173 174 /* Disable the channel */ 175 lan_rmw(FDMA_CH_DISABLE_CH_DISABLE_SET(BIT(fdma->channel_id)), 176 FDMA_CH_DISABLE_CH_DISABLE, 177 lan966x, FDMA_CH_DISABLE); 178 179 readx_poll_timeout_atomic(lan966x_fdma_channel_active, lan966x, 180 val, !(val & BIT(fdma->channel_id)), 181 READL_SLEEP_US, READL_TIMEOUT_US); 182 183 lan_rmw(FDMA_CH_DB_DISCARD_DB_DISCARD_SET(BIT(fdma->channel_id)), 184 FDMA_CH_DB_DISCARD_DB_DISCARD, 185 lan966x, FDMA_CH_DB_DISCARD); 186 } 187 188 static void lan966x_fdma_rx_reload(struct lan966x_rx *rx) 189 { 190 struct lan966x *lan966x = rx->lan966x; 191 192 lan_rmw(FDMA_CH_RELOAD_CH_RELOAD_SET(BIT(rx->fdma.channel_id)), 193 FDMA_CH_RELOAD_CH_RELOAD, 194 lan966x, FDMA_CH_RELOAD); 195 } 196 197 static int lan966x_fdma_tx_alloc(struct lan966x_tx *tx) 198 { 199 struct lan966x *lan966x = tx->lan966x; 200 struct fdma *fdma = &tx->fdma; 201 int err; 202 203 tx->dcbs_buf = kcalloc(fdma->n_dcbs, sizeof(struct lan966x_tx_dcb_buf), 204 GFP_KERNEL); 205 if (!tx->dcbs_buf) 206 return -ENOMEM; 207 208 err = fdma_alloc_coherent(lan966x->dev, fdma); 209 if (err) 210 goto out; 211 212 fdma_dcbs_init(fdma, 0, 0); 213 214 return 0; 215 216 out: 217 kfree(tx->dcbs_buf); 218 return -ENOMEM; 219 } 220 221 static void lan966x_fdma_tx_free(struct lan966x_tx *tx) 222 { 223 struct lan966x *lan966x = tx->lan966x; 224 225 kfree(tx->dcbs_buf); 226 fdma_free_coherent(lan966x->dev, &tx->fdma); 227 } 228 229 static void lan966x_fdma_tx_activate(struct lan966x_tx *tx) 230 { 231 struct lan966x *lan966x = tx->lan966x; 232 struct fdma *fdma = &tx->fdma; 233 u32 mask; 234 235 /* When activating a channel, first is required to write the first DCB 236 * address and then to activate it 237 */ 238 lan_wr(lower_32_bits((u64)fdma->dma), lan966x, 239 FDMA_DCB_LLP(fdma->channel_id)); 240 lan_wr(upper_32_bits((u64)fdma->dma), lan966x, 241 FDMA_DCB_LLP1(fdma->channel_id)); 242 243 lan_wr(FDMA_CH_CFG_CH_DCB_DB_CNT_SET(fdma->n_dbs) | 244 FDMA_CH_CFG_CH_INTR_DB_EOF_ONLY_SET(1) | 245 FDMA_CH_CFG_CH_INJ_PORT_SET(0) | 246 FDMA_CH_CFG_CH_MEM_SET(1), 247 lan966x, FDMA_CH_CFG(fdma->channel_id)); 248 249 /* Start fdma */ 250 lan_rmw(FDMA_PORT_CTRL_INJ_STOP_SET(0), 251 FDMA_PORT_CTRL_INJ_STOP, 252 lan966x, FDMA_PORT_CTRL(0)); 253 254 /* Enable interrupts */ 255 mask = lan_rd(lan966x, FDMA_INTR_DB_ENA); 256 mask = FDMA_INTR_DB_ENA_INTR_DB_ENA_GET(mask); 257 mask |= BIT(fdma->channel_id); 258 lan_rmw(FDMA_INTR_DB_ENA_INTR_DB_ENA_SET(mask), 259 FDMA_INTR_DB_ENA_INTR_DB_ENA, 260 lan966x, FDMA_INTR_DB_ENA); 261 262 /* Activate the channel */ 263 lan_rmw(FDMA_CH_ACTIVATE_CH_ACTIVATE_SET(BIT(fdma->channel_id)), 264 FDMA_CH_ACTIVATE_CH_ACTIVATE, 265 lan966x, FDMA_CH_ACTIVATE); 266 } 267 268 static void lan966x_fdma_tx_disable(struct lan966x_tx *tx) 269 { 270 struct lan966x *lan966x = tx->lan966x; 271 struct fdma *fdma = &tx->fdma; 272 u32 val; 273 274 /* Disable the channel */ 275 lan_rmw(FDMA_CH_DISABLE_CH_DISABLE_SET(BIT(fdma->channel_id)), 276 FDMA_CH_DISABLE_CH_DISABLE, 277 lan966x, FDMA_CH_DISABLE); 278 279 readx_poll_timeout_atomic(lan966x_fdma_channel_active, lan966x, 280 val, !(val & BIT(fdma->channel_id)), 281 READL_SLEEP_US, READL_TIMEOUT_US); 282 283 lan_rmw(FDMA_CH_DB_DISCARD_DB_DISCARD_SET(BIT(fdma->channel_id)), 284 FDMA_CH_DB_DISCARD_DB_DISCARD, 285 lan966x, FDMA_CH_DB_DISCARD); 286 287 tx->activated = false; 288 } 289 290 static void lan966x_fdma_tx_reload(struct lan966x_tx *tx) 291 { 292 struct lan966x *lan966x = tx->lan966x; 293 294 /* Write the registers to reload the channel */ 295 lan_rmw(FDMA_CH_RELOAD_CH_RELOAD_SET(BIT(tx->fdma.channel_id)), 296 FDMA_CH_RELOAD_CH_RELOAD, 297 lan966x, FDMA_CH_RELOAD); 298 } 299 300 static void lan966x_fdma_wakeup_netdev(struct lan966x *lan966x) 301 { 302 struct lan966x_port *port; 303 int i; 304 305 for (i = 0; i < lan966x->num_phys_ports; ++i) { 306 port = lan966x->ports[i]; 307 if (!port) 308 continue; 309 310 if (netif_queue_stopped(port->dev)) 311 netif_wake_queue(port->dev); 312 } 313 } 314 315 static void lan966x_fdma_stop_netdev(struct lan966x *lan966x) 316 { 317 struct lan966x_port *port; 318 int i; 319 320 for (i = 0; i < lan966x->num_phys_ports; ++i) { 321 port = lan966x->ports[i]; 322 if (!port) 323 continue; 324 325 netif_stop_queue(port->dev); 326 } 327 } 328 329 static void lan966x_fdma_tx_clear_buf(struct lan966x *lan966x, int weight) 330 { 331 struct lan966x_tx *tx = &lan966x->tx; 332 struct lan966x_rx *rx = &lan966x->rx; 333 struct lan966x_tx_dcb_buf *dcb_buf; 334 struct fdma *fdma = &tx->fdma; 335 struct xdp_frame_bulk bq; 336 unsigned long flags; 337 bool clear = false; 338 struct fdma_db *db; 339 int i; 340 341 xdp_frame_bulk_init(&bq); 342 343 spin_lock_irqsave(&lan966x->tx_lock, flags); 344 for (i = 0; i < fdma->n_dcbs; ++i) { 345 dcb_buf = &tx->dcbs_buf[i]; 346 347 if (!dcb_buf->used) 348 continue; 349 350 db = fdma_db_get(fdma, i, 0); 351 if (!fdma_db_is_done(db)) 352 continue; 353 354 dcb_buf->dev->stats.tx_packets++; 355 dcb_buf->dev->stats.tx_bytes += dcb_buf->len; 356 357 dcb_buf->used = false; 358 if (dcb_buf->use_skb) { 359 dma_unmap_single(lan966x->dev, 360 dcb_buf->dma_addr, 361 dcb_buf->len, 362 DMA_TO_DEVICE); 363 364 if (!dcb_buf->ptp) 365 napi_consume_skb(dcb_buf->data.skb, weight); 366 } else { 367 if (dcb_buf->xdp_ndo) 368 dma_unmap_single(lan966x->dev, 369 dcb_buf->dma_addr, 370 dcb_buf->len, 371 DMA_TO_DEVICE); 372 373 if (dcb_buf->xdp_ndo) 374 xdp_return_frame_bulk(dcb_buf->data.xdpf, &bq); 375 else 376 page_pool_recycle_direct(rx->page_pool, 377 dcb_buf->data.page); 378 } 379 380 clear = true; 381 } 382 383 xdp_flush_frame_bulk(&bq); 384 385 if (clear) 386 lan966x_fdma_wakeup_netdev(lan966x); 387 388 spin_unlock_irqrestore(&lan966x->tx_lock, flags); 389 } 390 391 static int lan966x_fdma_rx_check_frame(struct lan966x_rx *rx, u64 *src_port) 392 { 393 struct lan966x *lan966x = rx->lan966x; 394 struct fdma *fdma = &rx->fdma; 395 struct lan966x_port *port; 396 struct fdma_db *db; 397 struct page *page; 398 399 db = fdma_db_next_get(fdma); 400 page = rx->page[fdma->dcb_index][fdma->db_index]; 401 if (unlikely(!page)) 402 return FDMA_ERROR; 403 404 dma_sync_single_for_cpu(lan966x->dev, 405 (dma_addr_t)db->dataptr + XDP_PACKET_HEADROOM, 406 FDMA_DCB_STATUS_BLOCKL(db->status), 407 DMA_FROM_DEVICE); 408 409 lan966x_ifh_get_src_port(page_address(page) + XDP_PACKET_HEADROOM, 410 src_port); 411 if (WARN_ON(*src_port >= lan966x->num_phys_ports)) 412 return FDMA_ERROR; 413 414 port = lan966x->ports[*src_port]; 415 if (!lan966x_xdp_port_present(port)) 416 return FDMA_PASS; 417 418 return lan966x_xdp_run(port, page, FDMA_DCB_STATUS_BLOCKL(db->status)); 419 } 420 421 static struct sk_buff *lan966x_fdma_rx_get_frame(struct lan966x_rx *rx, 422 u64 src_port) 423 { 424 struct lan966x *lan966x = rx->lan966x; 425 struct fdma *fdma = &rx->fdma; 426 struct sk_buff *skb; 427 struct fdma_db *db; 428 struct page *page; 429 u64 timestamp; 430 431 /* Get the received frame and unmap it */ 432 db = fdma_db_next_get(fdma); 433 page = rx->page[fdma->dcb_index][fdma->db_index]; 434 435 skb = build_skb(page_address(page), fdma->db_size); 436 if (unlikely(!skb)) 437 goto free_page; 438 439 skb_mark_for_recycle(skb); 440 441 skb_reserve(skb, XDP_PACKET_HEADROOM); 442 skb_put(skb, FDMA_DCB_STATUS_BLOCKL(db->status)); 443 444 lan966x_ifh_get_timestamp(skb->data, ×tamp); 445 446 skb->dev = lan966x->ports[src_port]->dev; 447 skb_pull(skb, IFH_LEN_BYTES); 448 449 if (likely(!(skb->dev->features & NETIF_F_RXFCS))) 450 skb_trim(skb, skb->len - ETH_FCS_LEN); 451 452 lan966x_ptp_rxtstamp(lan966x, skb, src_port, timestamp); 453 skb->protocol = eth_type_trans(skb, skb->dev); 454 455 if (lan966x->bridge_mask & BIT(src_port)) { 456 skb->offload_fwd_mark = 1; 457 458 skb_reset_network_header(skb); 459 if (!lan966x_hw_offload(lan966x, src_port, skb)) 460 skb->offload_fwd_mark = 0; 461 } 462 463 skb->dev->stats.rx_bytes += skb->len; 464 skb->dev->stats.rx_packets++; 465 466 return skb; 467 468 free_page: 469 page_pool_recycle_direct(rx->page_pool, page); 470 471 return NULL; 472 } 473 474 static int lan966x_fdma_napi_poll(struct napi_struct *napi, int weight) 475 { 476 struct lan966x *lan966x = container_of(napi, struct lan966x, napi); 477 struct lan966x_rx *rx = &lan966x->rx; 478 int old_dcb, dcb_reload, counter = 0; 479 struct fdma *fdma = &rx->fdma; 480 bool redirect = false; 481 struct sk_buff *skb; 482 u64 src_port; 483 484 dcb_reload = fdma->dcb_index; 485 486 lan966x_fdma_tx_clear_buf(lan966x, weight); 487 488 /* Get all received skb */ 489 while (counter < weight) { 490 if (!fdma_has_frames(fdma)) 491 break; 492 493 counter++; 494 495 switch (lan966x_fdma_rx_check_frame(rx, &src_port)) { 496 case FDMA_PASS: 497 break; 498 case FDMA_ERROR: 499 lan966x_fdma_rx_free_page(rx); 500 fdma_dcb_advance(fdma); 501 goto allocate_new; 502 case FDMA_REDIRECT: 503 redirect = true; 504 fallthrough; 505 case FDMA_TX: 506 fdma_dcb_advance(fdma); 507 continue; 508 case FDMA_DROP: 509 lan966x_fdma_rx_free_page(rx); 510 fdma_dcb_advance(fdma); 511 continue; 512 } 513 514 skb = lan966x_fdma_rx_get_frame(rx, src_port); 515 fdma_dcb_advance(fdma); 516 if (!skb) 517 goto allocate_new; 518 519 napi_gro_receive(&lan966x->napi, skb); 520 } 521 522 allocate_new: 523 /* Allocate new pages and map them */ 524 while (dcb_reload != fdma->dcb_index) { 525 old_dcb = dcb_reload; 526 dcb_reload++; 527 dcb_reload &= fdma->n_dcbs - 1; 528 529 fdma_dcb_add(fdma, old_dcb, FDMA_DCB_INFO_DATAL(fdma->db_size), 530 FDMA_DCB_STATUS_INTR); 531 532 lan966x_fdma_rx_reload(rx); 533 } 534 535 if (redirect) 536 xdp_do_flush(); 537 538 if (counter < weight && napi_complete_done(napi, counter)) 539 lan_wr(0xff, lan966x, FDMA_INTR_DB_ENA); 540 541 return counter; 542 } 543 544 irqreturn_t lan966x_fdma_irq_handler(int irq, void *args) 545 { 546 struct lan966x *lan966x = args; 547 u32 db, err, err_type; 548 549 db = lan_rd(lan966x, FDMA_INTR_DB); 550 err = lan_rd(lan966x, FDMA_INTR_ERR); 551 552 if (db) { 553 lan_wr(0, lan966x, FDMA_INTR_DB_ENA); 554 lan_wr(db, lan966x, FDMA_INTR_DB); 555 556 napi_schedule(&lan966x->napi); 557 } 558 559 if (err) { 560 err_type = lan_rd(lan966x, FDMA_ERRORS); 561 562 WARN(1, "Unexpected error: %d, error_type: %d\n", err, err_type); 563 564 lan_wr(err, lan966x, FDMA_INTR_ERR); 565 lan_wr(err_type, lan966x, FDMA_ERRORS); 566 } 567 568 return IRQ_HANDLED; 569 } 570 571 static int lan966x_fdma_get_next_dcb(struct lan966x_tx *tx) 572 { 573 struct lan966x_tx_dcb_buf *dcb_buf; 574 struct fdma *fdma = &tx->fdma; 575 int i; 576 577 for (i = 0; i < fdma->n_dcbs; ++i) { 578 dcb_buf = &tx->dcbs_buf[i]; 579 if (!dcb_buf->used && 580 !fdma_is_last(&tx->fdma, &tx->fdma.dcbs[i])) 581 return i; 582 } 583 584 return -1; 585 } 586 587 static void lan966x_fdma_tx_start(struct lan966x_tx *tx) 588 { 589 struct lan966x *lan966x = tx->lan966x; 590 591 if (likely(lan966x->tx.activated)) { 592 lan966x_fdma_tx_reload(tx); 593 } else { 594 /* Because it is first time, then just activate */ 595 lan966x->tx.activated = true; 596 lan966x_fdma_tx_activate(tx); 597 } 598 } 599 600 int lan966x_fdma_xmit_xdpf(struct lan966x_port *port, void *ptr, u32 len) 601 { 602 struct lan966x *lan966x = port->lan966x; 603 struct lan966x_tx_dcb_buf *next_dcb_buf; 604 struct lan966x_tx *tx = &lan966x->tx; 605 struct xdp_frame *xdpf; 606 dma_addr_t dma_addr; 607 struct page *page; 608 int next_to_use; 609 __be32 *ifh; 610 int ret = 0; 611 612 spin_lock(&lan966x->tx_lock); 613 614 /* Get next index */ 615 next_to_use = lan966x_fdma_get_next_dcb(tx); 616 if (next_to_use < 0) { 617 netif_stop_queue(port->dev); 618 ret = NETDEV_TX_BUSY; 619 goto out; 620 } 621 622 /* Get the next buffer */ 623 next_dcb_buf = &tx->dcbs_buf[next_to_use]; 624 625 /* Generate new IFH */ 626 if (!len) { 627 xdpf = ptr; 628 629 if (xdpf->headroom < IFH_LEN_BYTES) { 630 ret = NETDEV_TX_OK; 631 goto out; 632 } 633 634 ifh = xdpf->data - IFH_LEN_BYTES; 635 memset(ifh, 0x0, sizeof(__be32) * IFH_LEN); 636 lan966x_ifh_set_bypass(ifh, 1); 637 lan966x_ifh_set_port(ifh, BIT_ULL(port->chip_port)); 638 639 dma_addr = dma_map_single(lan966x->dev, 640 xdpf->data - IFH_LEN_BYTES, 641 xdpf->len + IFH_LEN_BYTES, 642 DMA_TO_DEVICE); 643 if (dma_mapping_error(lan966x->dev, dma_addr)) { 644 ret = NETDEV_TX_OK; 645 goto out; 646 } 647 648 next_dcb_buf->data.xdpf = xdpf; 649 next_dcb_buf->len = xdpf->len + IFH_LEN_BYTES; 650 } else { 651 page = ptr; 652 653 ifh = page_address(page) + XDP_PACKET_HEADROOM; 654 memset(ifh, 0x0, sizeof(__be32) * IFH_LEN); 655 lan966x_ifh_set_bypass(ifh, 1); 656 lan966x_ifh_set_port(ifh, BIT_ULL(port->chip_port)); 657 658 dma_addr = page_pool_get_dma_addr(page); 659 dma_sync_single_for_device(lan966x->dev, 660 dma_addr + XDP_PACKET_HEADROOM, 661 len + IFH_LEN_BYTES, 662 DMA_TO_DEVICE); 663 664 next_dcb_buf->data.page = page; 665 next_dcb_buf->len = len + IFH_LEN_BYTES; 666 } 667 668 /* Fill up the buffer */ 669 next_dcb_buf->use_skb = false; 670 next_dcb_buf->xdp_ndo = !len; 671 next_dcb_buf->dma_addr = dma_addr; 672 next_dcb_buf->used = true; 673 next_dcb_buf->ptp = false; 674 next_dcb_buf->dev = port->dev; 675 676 __fdma_dcb_add(&tx->fdma, 677 next_to_use, 678 0, 679 FDMA_DCB_STATUS_INTR | 680 FDMA_DCB_STATUS_SOF | 681 FDMA_DCB_STATUS_EOF | 682 FDMA_DCB_STATUS_BLOCKO(0) | 683 FDMA_DCB_STATUS_BLOCKL(next_dcb_buf->len), 684 &fdma_nextptr_cb, 685 &lan966x_fdma_xdp_tx_dataptr_cb); 686 687 /* Start the transmission */ 688 lan966x_fdma_tx_start(tx); 689 690 out: 691 spin_unlock(&lan966x->tx_lock); 692 693 return ret; 694 } 695 696 int lan966x_fdma_xmit(struct sk_buff *skb, __be32 *ifh, struct net_device *dev) 697 { 698 struct lan966x_port *port = netdev_priv(dev); 699 struct lan966x *lan966x = port->lan966x; 700 struct lan966x_tx_dcb_buf *next_dcb_buf; 701 struct lan966x_tx *tx = &lan966x->tx; 702 int needed_headroom; 703 int needed_tailroom; 704 dma_addr_t dma_addr; 705 int next_to_use; 706 int err; 707 708 /* Get next index */ 709 next_to_use = lan966x_fdma_get_next_dcb(tx); 710 if (next_to_use < 0) { 711 netif_stop_queue(dev); 712 return NETDEV_TX_BUSY; 713 } 714 715 if (skb_put_padto(skb, ETH_ZLEN)) { 716 dev->stats.tx_dropped++; 717 return NETDEV_TX_OK; 718 } 719 720 /* skb processing */ 721 needed_headroom = max_t(int, IFH_LEN_BYTES - skb_headroom(skb), 0); 722 needed_tailroom = max_t(int, ETH_FCS_LEN - skb_tailroom(skb), 0); 723 if (needed_headroom || needed_tailroom || skb_header_cloned(skb)) { 724 err = pskb_expand_head(skb, needed_headroom, needed_tailroom, 725 GFP_ATOMIC); 726 if (unlikely(err)) { 727 dev->stats.tx_dropped++; 728 err = NETDEV_TX_OK; 729 goto release; 730 } 731 } 732 733 skb_tx_timestamp(skb); 734 skb_push(skb, IFH_LEN_BYTES); 735 memcpy(skb->data, ifh, IFH_LEN_BYTES); 736 skb_put(skb, 4); 737 738 dma_addr = dma_map_single(lan966x->dev, skb->data, skb->len, 739 DMA_TO_DEVICE); 740 if (dma_mapping_error(lan966x->dev, dma_addr)) { 741 dev->stats.tx_dropped++; 742 err = NETDEV_TX_OK; 743 goto release; 744 } 745 746 /* Fill up the buffer */ 747 next_dcb_buf = &tx->dcbs_buf[next_to_use]; 748 next_dcb_buf->use_skb = true; 749 next_dcb_buf->data.skb = skb; 750 next_dcb_buf->xdp_ndo = false; 751 next_dcb_buf->len = skb->len; 752 next_dcb_buf->dma_addr = dma_addr; 753 next_dcb_buf->used = true; 754 next_dcb_buf->ptp = false; 755 next_dcb_buf->dev = dev; 756 757 fdma_dcb_add(&tx->fdma, 758 next_to_use, 759 0, 760 FDMA_DCB_STATUS_INTR | 761 FDMA_DCB_STATUS_SOF | 762 FDMA_DCB_STATUS_EOF | 763 FDMA_DCB_STATUS_BLOCKO(0) | 764 FDMA_DCB_STATUS_BLOCKL(skb->len)); 765 766 if (skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP && 767 LAN966X_SKB_CB(skb)->rew_op == IFH_REW_OP_TWO_STEP_PTP) 768 next_dcb_buf->ptp = true; 769 770 /* Start the transmission */ 771 lan966x_fdma_tx_start(tx); 772 773 return NETDEV_TX_OK; 774 775 release: 776 if (skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP && 777 LAN966X_SKB_CB(skb)->rew_op == IFH_REW_OP_TWO_STEP_PTP) 778 lan966x_ptp_txtstamp_release(port, skb); 779 780 dev_kfree_skb_any(skb); 781 return err; 782 } 783 784 static int lan966x_fdma_get_max_mtu(struct lan966x *lan966x) 785 { 786 int max_mtu = 0; 787 int i; 788 789 for (i = 0; i < lan966x->num_phys_ports; ++i) { 790 struct lan966x_port *port; 791 int mtu; 792 793 port = lan966x->ports[i]; 794 if (!port) 795 continue; 796 797 mtu = lan_rd(lan966x, DEV_MAC_MAXLEN_CFG(port->chip_port)); 798 if (mtu > max_mtu) 799 max_mtu = mtu; 800 } 801 802 return max_mtu; 803 } 804 805 static int lan966x_qsys_sw_status(struct lan966x *lan966x) 806 { 807 return lan_rd(lan966x, QSYS_SW_STATUS(CPU_PORT)); 808 } 809 810 static int lan966x_fdma_reload(struct lan966x *lan966x, int new_mtu) 811 { 812 struct page_pool *page_pool; 813 dma_addr_t rx_dma; 814 void *rx_dcbs; 815 u32 size; 816 int err; 817 818 /* Store these for later to free them */ 819 rx_dma = lan966x->rx.fdma.dma; 820 rx_dcbs = lan966x->rx.fdma.dcbs; 821 page_pool = lan966x->rx.page_pool; 822 823 napi_synchronize(&lan966x->napi); 824 napi_disable(&lan966x->napi); 825 lan966x_fdma_stop_netdev(lan966x); 826 827 lan966x_fdma_rx_disable(&lan966x->rx); 828 lan966x_fdma_rx_free_pages(&lan966x->rx); 829 lan966x->rx.page_order = round_up(new_mtu, PAGE_SIZE) / PAGE_SIZE - 1; 830 lan966x->rx.max_mtu = new_mtu; 831 err = lan966x_fdma_rx_alloc(&lan966x->rx); 832 if (err) 833 goto restore; 834 lan966x_fdma_rx_start(&lan966x->rx); 835 836 size = sizeof(struct fdma_dcb) * lan966x->rx.fdma.n_dcbs; 837 size = ALIGN(size, PAGE_SIZE); 838 dma_free_coherent(lan966x->dev, size, rx_dcbs, rx_dma); 839 840 page_pool_destroy(page_pool); 841 842 lan966x_fdma_wakeup_netdev(lan966x); 843 napi_enable(&lan966x->napi); 844 845 return err; 846 restore: 847 lan966x->rx.page_pool = page_pool; 848 lan966x->rx.fdma.dma = rx_dma; 849 lan966x->rx.fdma.dcbs = rx_dcbs; 850 lan966x_fdma_rx_start(&lan966x->rx); 851 852 return err; 853 } 854 855 static int lan966x_fdma_get_max_frame(struct lan966x *lan966x) 856 { 857 return lan966x_fdma_get_max_mtu(lan966x) + 858 IFH_LEN_BYTES + 859 SKB_DATA_ALIGN(sizeof(struct skb_shared_info)) + 860 VLAN_HLEN * 2 + 861 XDP_PACKET_HEADROOM; 862 } 863 864 static int __lan966x_fdma_reload(struct lan966x *lan966x, int max_mtu) 865 { 866 int err; 867 u32 val; 868 869 /* Disable the CPU port */ 870 lan_rmw(QSYS_SW_PORT_MODE_PORT_ENA_SET(0), 871 QSYS_SW_PORT_MODE_PORT_ENA, 872 lan966x, QSYS_SW_PORT_MODE(CPU_PORT)); 873 874 /* Flush the CPU queues */ 875 readx_poll_timeout(lan966x_qsys_sw_status, lan966x, 876 val, !(QSYS_SW_STATUS_EQ_AVAIL_GET(val)), 877 READL_SLEEP_US, READL_TIMEOUT_US); 878 879 /* Add a sleep in case there are frames between the queues and the CPU 880 * port 881 */ 882 usleep_range(1000, 2000); 883 884 err = lan966x_fdma_reload(lan966x, max_mtu); 885 886 /* Enable back the CPU port */ 887 lan_rmw(QSYS_SW_PORT_MODE_PORT_ENA_SET(1), 888 QSYS_SW_PORT_MODE_PORT_ENA, 889 lan966x, QSYS_SW_PORT_MODE(CPU_PORT)); 890 891 return err; 892 } 893 894 int lan966x_fdma_change_mtu(struct lan966x *lan966x) 895 { 896 int max_mtu; 897 898 max_mtu = lan966x_fdma_get_max_frame(lan966x); 899 if (max_mtu == lan966x->rx.max_mtu) 900 return 0; 901 902 return __lan966x_fdma_reload(lan966x, max_mtu); 903 } 904 905 int lan966x_fdma_reload_page_pool(struct lan966x *lan966x) 906 { 907 int max_mtu; 908 909 max_mtu = lan966x_fdma_get_max_frame(lan966x); 910 return __lan966x_fdma_reload(lan966x, max_mtu); 911 } 912 913 void lan966x_fdma_netdev_init(struct lan966x *lan966x, struct net_device *dev) 914 { 915 if (lan966x->fdma_ndev) 916 return; 917 918 lan966x->fdma_ndev = dev; 919 netif_napi_add(dev, &lan966x->napi, lan966x_fdma_napi_poll); 920 napi_enable(&lan966x->napi); 921 } 922 923 void lan966x_fdma_netdev_deinit(struct lan966x *lan966x, struct net_device *dev) 924 { 925 if (lan966x->fdma_ndev == dev) { 926 netif_napi_del(&lan966x->napi); 927 lan966x->fdma_ndev = NULL; 928 } 929 } 930 931 int lan966x_fdma_init(struct lan966x *lan966x) 932 { 933 int err; 934 935 if (!lan966x->fdma) 936 return 0; 937 938 lan966x->rx.lan966x = lan966x; 939 lan966x->rx.fdma.channel_id = FDMA_XTR_CHANNEL; 940 lan966x->rx.fdma.n_dcbs = FDMA_DCB_MAX; 941 lan966x->rx.fdma.n_dbs = FDMA_RX_DCB_MAX_DBS; 942 lan966x->rx.fdma.priv = lan966x; 943 lan966x->rx.fdma.size = fdma_get_size(&lan966x->rx.fdma); 944 lan966x->rx.fdma.db_size = PAGE_SIZE << lan966x->rx.page_order; 945 lan966x->rx.fdma.ops.nextptr_cb = &fdma_nextptr_cb; 946 lan966x->rx.fdma.ops.dataptr_cb = &lan966x_fdma_rx_dataptr_cb; 947 lan966x->rx.max_mtu = lan966x_fdma_get_max_frame(lan966x); 948 lan966x->tx.lan966x = lan966x; 949 lan966x->tx.fdma.channel_id = FDMA_INJ_CHANNEL; 950 lan966x->tx.fdma.n_dcbs = FDMA_DCB_MAX; 951 lan966x->tx.fdma.n_dbs = FDMA_TX_DCB_MAX_DBS; 952 lan966x->tx.fdma.priv = lan966x; 953 lan966x->tx.fdma.size = fdma_get_size(&lan966x->tx.fdma); 954 lan966x->tx.fdma.db_size = PAGE_SIZE << lan966x->rx.page_order; 955 lan966x->tx.fdma.ops.nextptr_cb = &fdma_nextptr_cb; 956 lan966x->tx.fdma.ops.dataptr_cb = &lan966x_fdma_tx_dataptr_cb; 957 958 err = lan966x_fdma_rx_alloc(&lan966x->rx); 959 if (err) 960 return err; 961 962 err = lan966x_fdma_tx_alloc(&lan966x->tx); 963 if (err) { 964 fdma_free_coherent(lan966x->dev, &lan966x->rx.fdma); 965 return err; 966 } 967 968 lan966x_fdma_rx_start(&lan966x->rx); 969 970 return 0; 971 } 972 973 void lan966x_fdma_deinit(struct lan966x *lan966x) 974 { 975 if (!lan966x->fdma) 976 return; 977 978 lan966x_fdma_rx_disable(&lan966x->rx); 979 lan966x_fdma_tx_disable(&lan966x->tx); 980 981 napi_synchronize(&lan966x->napi); 982 napi_disable(&lan966x->napi); 983 984 lan966x_fdma_rx_free_pages(&lan966x->rx); 985 fdma_free_coherent(lan966x->dev, &lan966x->rx.fdma); 986 page_pool_destroy(lan966x->rx.page_pool); 987 lan966x_fdma_tx_free(&lan966x->tx); 988 } 989