1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright (C) 2021 Gerhard Engleder <gerhard@engleder-embedded.com> */ 3 4 /* TSN endpoint Ethernet MAC driver 5 * 6 * The TSN endpoint Ethernet MAC is a FPGA based network device for real-time 7 * communication. It is designed for endpoints within TSN (Time Sensitive 8 * Networking) networks; e.g., for PLCs in the industrial automation case. 9 * 10 * It supports multiple TX/RX queue pairs. The first TX/RX queue pair is used 11 * by the driver. 12 * 13 * More information can be found here: 14 * - www.embedded-experts.at/tsn 15 * - www.engleder-embedded.com 16 */ 17 18 #include "tsnep.h" 19 #include "tsnep_hw.h" 20 21 #include <linux/module.h> 22 #include <linux/of.h> 23 #include <linux/of_net.h> 24 #include <linux/of_mdio.h> 25 #include <linux/interrupt.h> 26 #include <linux/etherdevice.h> 27 #include <linux/phy.h> 28 #include <linux/iopoll.h> 29 #include <linux/bpf.h> 30 #include <linux/bpf_trace.h> 31 #include <net/page_pool/helpers.h> 32 #include <net/xdp_sock_drv.h> 33 34 #define TSNEP_RX_OFFSET (max(NET_SKB_PAD, XDP_PACKET_HEADROOM) + NET_IP_ALIGN) 35 #define TSNEP_HEADROOM ALIGN(TSNEP_RX_OFFSET, 4) 36 #define TSNEP_MAX_RX_BUF_SIZE (PAGE_SIZE - TSNEP_HEADROOM - \ 37 SKB_DATA_ALIGN(sizeof(struct skb_shared_info))) 38 /* XSK buffer shall store at least Q-in-Q frame */ 39 #define TSNEP_XSK_RX_BUF_SIZE (ALIGN(TSNEP_RX_INLINE_METADATA_SIZE + \ 40 ETH_FRAME_LEN + ETH_FCS_LEN + \ 41 VLAN_HLEN * 2, 4)) 42 43 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 44 #define DMA_ADDR_HIGH(dma_addr) ((u32)(((dma_addr) >> 32) & 0xFFFFFFFF)) 45 #else 46 #define DMA_ADDR_HIGH(dma_addr) ((u32)(0)) 47 #endif 48 #define DMA_ADDR_LOW(dma_addr) ((u32)((dma_addr) & 0xFFFFFFFF)) 49 50 #define TSNEP_COALESCE_USECS_DEFAULT 64 51 #define TSNEP_COALESCE_USECS_MAX ((ECM_INT_DELAY_MASK >> ECM_INT_DELAY_SHIFT) * \ 52 ECM_INT_DELAY_BASE_US + ECM_INT_DELAY_BASE_US - 1) 53 54 /* mapping type */ 55 #define TSNEP_TX_TYPE_MAP BIT(0) 56 #define TSNEP_TX_TYPE_MAP_PAGE BIT(1) 57 #define TSNEP_TX_TYPE_INLINE BIT(2) 58 /* buffer type */ 59 #define TSNEP_TX_TYPE_SKB BIT(8) 60 #define TSNEP_TX_TYPE_SKB_MAP (TSNEP_TX_TYPE_SKB | TSNEP_TX_TYPE_MAP) 61 #define TSNEP_TX_TYPE_SKB_INLINE (TSNEP_TX_TYPE_SKB | TSNEP_TX_TYPE_INLINE) 62 #define TSNEP_TX_TYPE_SKB_FRAG BIT(9) 63 #define TSNEP_TX_TYPE_SKB_FRAG_MAP_PAGE (TSNEP_TX_TYPE_SKB_FRAG | TSNEP_TX_TYPE_MAP_PAGE) 64 #define TSNEP_TX_TYPE_SKB_FRAG_INLINE (TSNEP_TX_TYPE_SKB_FRAG | TSNEP_TX_TYPE_INLINE) 65 #define TSNEP_TX_TYPE_XDP_TX BIT(10) 66 #define TSNEP_TX_TYPE_XDP_NDO BIT(11) 67 #define TSNEP_TX_TYPE_XDP_NDO_MAP_PAGE (TSNEP_TX_TYPE_XDP_NDO | TSNEP_TX_TYPE_MAP_PAGE) 68 #define TSNEP_TX_TYPE_XDP (TSNEP_TX_TYPE_XDP_TX | TSNEP_TX_TYPE_XDP_NDO) 69 #define TSNEP_TX_TYPE_XSK BIT(12) 70 71 #define TSNEP_XDP_TX BIT(0) 72 #define TSNEP_XDP_REDIRECT BIT(1) 73 74 static void tsnep_enable_irq(struct tsnep_adapter *adapter, u32 mask) 75 { 76 iowrite32(mask, adapter->addr + ECM_INT_ENABLE); 77 } 78 79 static void tsnep_disable_irq(struct tsnep_adapter *adapter, u32 mask) 80 { 81 mask |= ECM_INT_DISABLE; 82 iowrite32(mask, adapter->addr + ECM_INT_ENABLE); 83 } 84 85 static irqreturn_t tsnep_irq(int irq, void *arg) 86 { 87 struct tsnep_adapter *adapter = arg; 88 u32 active = ioread32(adapter->addr + ECM_INT_ACTIVE); 89 90 /* acknowledge interrupt */ 91 if (active != 0) 92 iowrite32(active, adapter->addr + ECM_INT_ACKNOWLEDGE); 93 94 /* handle link interrupt */ 95 if ((active & ECM_INT_LINK) != 0) 96 phy_mac_interrupt(adapter->netdev->phydev); 97 98 /* handle TX/RX queue 0 interrupt */ 99 if ((active & adapter->queue[0].irq_mask) != 0) { 100 if (napi_schedule_prep(&adapter->queue[0].napi)) { 101 tsnep_disable_irq(adapter, adapter->queue[0].irq_mask); 102 /* schedule after masking to avoid races */ 103 __napi_schedule(&adapter->queue[0].napi); 104 } 105 } 106 107 return IRQ_HANDLED; 108 } 109 110 static irqreturn_t tsnep_irq_txrx(int irq, void *arg) 111 { 112 struct tsnep_queue *queue = arg; 113 114 /* handle TX/RX queue interrupt */ 115 if (napi_schedule_prep(&queue->napi)) { 116 tsnep_disable_irq(queue->adapter, queue->irq_mask); 117 /* schedule after masking to avoid races */ 118 __napi_schedule(&queue->napi); 119 } 120 121 return IRQ_HANDLED; 122 } 123 124 int tsnep_set_irq_coalesce(struct tsnep_queue *queue, u32 usecs) 125 { 126 if (usecs > TSNEP_COALESCE_USECS_MAX) 127 return -ERANGE; 128 129 usecs /= ECM_INT_DELAY_BASE_US; 130 usecs <<= ECM_INT_DELAY_SHIFT; 131 usecs &= ECM_INT_DELAY_MASK; 132 133 queue->irq_delay &= ~ECM_INT_DELAY_MASK; 134 queue->irq_delay |= usecs; 135 iowrite8(queue->irq_delay, queue->irq_delay_addr); 136 137 return 0; 138 } 139 140 u32 tsnep_get_irq_coalesce(struct tsnep_queue *queue) 141 { 142 u32 usecs; 143 144 usecs = (queue->irq_delay & ECM_INT_DELAY_MASK); 145 usecs >>= ECM_INT_DELAY_SHIFT; 146 usecs *= ECM_INT_DELAY_BASE_US; 147 148 return usecs; 149 } 150 151 static int tsnep_mdiobus_read(struct mii_bus *bus, int addr, int regnum) 152 { 153 struct tsnep_adapter *adapter = bus->priv; 154 u32 md; 155 int retval; 156 157 md = ECM_MD_READ; 158 if (!adapter->suppress_preamble) 159 md |= ECM_MD_PREAMBLE; 160 md |= (regnum << ECM_MD_ADDR_SHIFT) & ECM_MD_ADDR_MASK; 161 md |= (addr << ECM_MD_PHY_ADDR_SHIFT) & ECM_MD_PHY_ADDR_MASK; 162 iowrite32(md, adapter->addr + ECM_MD_CONTROL); 163 retval = readl_poll_timeout_atomic(adapter->addr + ECM_MD_STATUS, md, 164 !(md & ECM_MD_BUSY), 16, 1000); 165 if (retval != 0) 166 return retval; 167 168 return (md & ECM_MD_DATA_MASK) >> ECM_MD_DATA_SHIFT; 169 } 170 171 static int tsnep_mdiobus_write(struct mii_bus *bus, int addr, int regnum, 172 u16 val) 173 { 174 struct tsnep_adapter *adapter = bus->priv; 175 u32 md; 176 int retval; 177 178 md = ECM_MD_WRITE; 179 if (!adapter->suppress_preamble) 180 md |= ECM_MD_PREAMBLE; 181 md |= (regnum << ECM_MD_ADDR_SHIFT) & ECM_MD_ADDR_MASK; 182 md |= (addr << ECM_MD_PHY_ADDR_SHIFT) & ECM_MD_PHY_ADDR_MASK; 183 md |= ((u32)val << ECM_MD_DATA_SHIFT) & ECM_MD_DATA_MASK; 184 iowrite32(md, adapter->addr + ECM_MD_CONTROL); 185 retval = readl_poll_timeout_atomic(adapter->addr + ECM_MD_STATUS, md, 186 !(md & ECM_MD_BUSY), 16, 1000); 187 if (retval != 0) 188 return retval; 189 190 return 0; 191 } 192 193 static void tsnep_set_link_mode(struct tsnep_adapter *adapter) 194 { 195 u32 mode; 196 197 switch (adapter->phydev->speed) { 198 case SPEED_100: 199 mode = ECM_LINK_MODE_100; 200 break; 201 case SPEED_1000: 202 mode = ECM_LINK_MODE_1000; 203 break; 204 default: 205 mode = ECM_LINK_MODE_OFF; 206 break; 207 } 208 iowrite32(mode, adapter->addr + ECM_STATUS); 209 } 210 211 static void tsnep_phy_link_status_change(struct net_device *netdev) 212 { 213 struct tsnep_adapter *adapter = netdev_priv(netdev); 214 struct phy_device *phydev = netdev->phydev; 215 216 if (phydev->link) 217 tsnep_set_link_mode(adapter); 218 219 phy_print_status(netdev->phydev); 220 } 221 222 static int tsnep_phy_loopback(struct tsnep_adapter *adapter, bool enable) 223 { 224 int speed; 225 226 if (enable) { 227 if (adapter->phydev->autoneg == AUTONEG_DISABLE && 228 adapter->phydev->speed == SPEED_100) 229 speed = SPEED_100; 230 else 231 speed = SPEED_1000; 232 } else { 233 speed = 0; 234 } 235 236 return phy_loopback(adapter->phydev, enable, speed); 237 } 238 239 static int tsnep_phy_open(struct tsnep_adapter *adapter) 240 { 241 struct phy_device *phydev; 242 struct ethtool_keee ethtool_keee; 243 int retval; 244 245 retval = phy_connect_direct(adapter->netdev, adapter->phydev, 246 tsnep_phy_link_status_change, 247 adapter->phy_mode); 248 if (retval) 249 return retval; 250 phydev = adapter->netdev->phydev; 251 252 /* MAC supports only 100Mbps|1000Mbps full duplex 253 * SPE (Single Pair Ethernet) is also an option but not implemented yet 254 */ 255 phy_remove_link_mode(phydev, ETHTOOL_LINK_MODE_10baseT_Half_BIT); 256 phy_remove_link_mode(phydev, ETHTOOL_LINK_MODE_10baseT_Full_BIT); 257 phy_remove_link_mode(phydev, ETHTOOL_LINK_MODE_100baseT_Half_BIT); 258 phy_remove_link_mode(phydev, ETHTOOL_LINK_MODE_1000baseT_Half_BIT); 259 260 /* disable EEE autoneg, EEE not supported by TSNEP */ 261 memset(ðtool_keee, 0, sizeof(ethtool_keee)); 262 phy_ethtool_set_eee(adapter->phydev, ðtool_keee); 263 264 adapter->phydev->irq = PHY_MAC_INTERRUPT; 265 phy_start(adapter->phydev); 266 267 return 0; 268 } 269 270 static void tsnep_phy_close(struct tsnep_adapter *adapter) 271 { 272 phy_stop(adapter->netdev->phydev); 273 phy_disconnect(adapter->netdev->phydev); 274 } 275 276 static void tsnep_tx_ring_cleanup(struct tsnep_tx *tx) 277 { 278 struct device *dmadev = tx->adapter->dmadev; 279 int i; 280 281 memset(tx->entry, 0, sizeof(tx->entry)); 282 283 for (i = 0; i < TSNEP_RING_PAGE_COUNT; i++) { 284 if (tx->page[i]) { 285 dma_free_coherent(dmadev, PAGE_SIZE, tx->page[i], 286 tx->page_dma[i]); 287 tx->page[i] = NULL; 288 tx->page_dma[i] = 0; 289 } 290 } 291 } 292 293 static int tsnep_tx_ring_create(struct tsnep_tx *tx) 294 { 295 struct device *dmadev = tx->adapter->dmadev; 296 struct tsnep_tx_entry *entry; 297 struct tsnep_tx_entry *next_entry; 298 int i, j; 299 int retval; 300 301 for (i = 0; i < TSNEP_RING_PAGE_COUNT; i++) { 302 tx->page[i] = 303 dma_alloc_coherent(dmadev, PAGE_SIZE, &tx->page_dma[i], 304 GFP_KERNEL); 305 if (!tx->page[i]) { 306 retval = -ENOMEM; 307 goto alloc_failed; 308 } 309 for (j = 0; j < TSNEP_RING_ENTRIES_PER_PAGE; j++) { 310 entry = &tx->entry[TSNEP_RING_ENTRIES_PER_PAGE * i + j]; 311 entry->desc_wb = (struct tsnep_tx_desc_wb *) 312 (((u8 *)tx->page[i]) + TSNEP_DESC_SIZE * j); 313 entry->desc = (struct tsnep_tx_desc *) 314 (((u8 *)entry->desc_wb) + TSNEP_DESC_OFFSET); 315 entry->desc_dma = tx->page_dma[i] + TSNEP_DESC_SIZE * j; 316 entry->owner_user_flag = false; 317 } 318 } 319 for (i = 0; i < TSNEP_RING_SIZE; i++) { 320 entry = &tx->entry[i]; 321 next_entry = &tx->entry[(i + 1) & TSNEP_RING_MASK]; 322 entry->desc->next = __cpu_to_le64(next_entry->desc_dma); 323 } 324 325 return 0; 326 327 alloc_failed: 328 tsnep_tx_ring_cleanup(tx); 329 return retval; 330 } 331 332 static void tsnep_tx_init(struct tsnep_tx *tx) 333 { 334 dma_addr_t dma; 335 336 dma = tx->entry[0].desc_dma | TSNEP_RESET_OWNER_COUNTER; 337 iowrite32(DMA_ADDR_LOW(dma), tx->addr + TSNEP_TX_DESC_ADDR_LOW); 338 iowrite32(DMA_ADDR_HIGH(dma), tx->addr + TSNEP_TX_DESC_ADDR_HIGH); 339 tx->write = 0; 340 tx->read = 0; 341 tx->owner_counter = 1; 342 tx->increment_owner_counter = TSNEP_RING_SIZE - 1; 343 } 344 345 static void tsnep_tx_enable(struct tsnep_tx *tx) 346 { 347 struct netdev_queue *nq; 348 349 nq = netdev_get_tx_queue(tx->adapter->netdev, tx->queue_index); 350 351 __netif_tx_lock_bh(nq); 352 netif_tx_wake_queue(nq); 353 __netif_tx_unlock_bh(nq); 354 } 355 356 static void tsnep_tx_disable(struct tsnep_tx *tx, struct napi_struct *napi) 357 { 358 struct netdev_queue *nq; 359 u32 val; 360 361 nq = netdev_get_tx_queue(tx->adapter->netdev, tx->queue_index); 362 363 __netif_tx_lock_bh(nq); 364 netif_tx_stop_queue(nq); 365 __netif_tx_unlock_bh(nq); 366 367 /* wait until TX is done in hardware */ 368 readx_poll_timeout(ioread32, tx->addr + TSNEP_CONTROL, val, 369 ((val & TSNEP_CONTROL_TX_ENABLE) == 0), 10000, 370 1000000); 371 372 /* wait until TX is also done in software */ 373 while (READ_ONCE(tx->read) != tx->write) { 374 napi_schedule(napi); 375 napi_synchronize(napi); 376 } 377 } 378 379 static void tsnep_tx_activate(struct tsnep_tx *tx, int index, int length, 380 bool last) 381 { 382 struct tsnep_tx_entry *entry = &tx->entry[index]; 383 384 entry->properties = 0; 385 /* xdpf and zc are union with skb */ 386 if (entry->skb) { 387 entry->properties = length & TSNEP_DESC_LENGTH_MASK; 388 entry->properties |= TSNEP_DESC_INTERRUPT_FLAG; 389 if ((entry->type & TSNEP_TX_TYPE_SKB) && 390 (skb_shinfo(entry->skb)->tx_flags & SKBTX_IN_PROGRESS)) 391 entry->properties |= TSNEP_DESC_EXTENDED_WRITEBACK_FLAG; 392 393 /* toggle user flag to prevent false acknowledge 394 * 395 * Only the first fragment is acknowledged. For all other 396 * fragments no acknowledge is done and the last written owner 397 * counter stays in the writeback descriptor. Therefore, it is 398 * possible that the last written owner counter is identical to 399 * the new incremented owner counter and a false acknowledge is 400 * detected before the real acknowledge has been done by 401 * hardware. 402 * 403 * The user flag is used to prevent this situation. The user 404 * flag is copied to the writeback descriptor by the hardware 405 * and is used as additional acknowledge data. By toggeling the 406 * user flag only for the first fragment (which is 407 * acknowledged), it is guaranteed that the last acknowledge 408 * done for this descriptor has used a different user flag and 409 * cannot be detected as false acknowledge. 410 */ 411 entry->owner_user_flag = !entry->owner_user_flag; 412 } 413 if (last) 414 entry->properties |= TSNEP_TX_DESC_LAST_FRAGMENT_FLAG; 415 if (index == tx->increment_owner_counter) { 416 tx->owner_counter++; 417 if (tx->owner_counter == 4) 418 tx->owner_counter = 1; 419 tx->increment_owner_counter--; 420 if (tx->increment_owner_counter < 0) 421 tx->increment_owner_counter = TSNEP_RING_SIZE - 1; 422 } 423 entry->properties |= 424 (tx->owner_counter << TSNEP_DESC_OWNER_COUNTER_SHIFT) & 425 TSNEP_DESC_OWNER_COUNTER_MASK; 426 if (entry->owner_user_flag) 427 entry->properties |= TSNEP_TX_DESC_OWNER_USER_FLAG; 428 entry->desc->more_properties = 429 __cpu_to_le32(entry->len & TSNEP_DESC_LENGTH_MASK); 430 if (entry->type & TSNEP_TX_TYPE_INLINE) 431 entry->properties |= TSNEP_TX_DESC_DATA_AFTER_DESC_FLAG; 432 433 /* descriptor properties shall be written last, because valid data is 434 * signaled there 435 */ 436 dma_wmb(); 437 438 entry->desc->properties = __cpu_to_le32(entry->properties); 439 } 440 441 static int tsnep_tx_desc_available(struct tsnep_tx *tx) 442 { 443 if (tx->read <= tx->write) 444 return TSNEP_RING_SIZE - tx->write + tx->read - 1; 445 else 446 return tx->read - tx->write - 1; 447 } 448 449 static int tsnep_tx_map_frag(skb_frag_t *frag, struct tsnep_tx_entry *entry, 450 struct device *dmadev, dma_addr_t *dma) 451 { 452 unsigned int len; 453 int mapped; 454 455 len = skb_frag_size(frag); 456 if (likely(len > TSNEP_DESC_SIZE_DATA_AFTER_INLINE)) { 457 *dma = skb_frag_dma_map(dmadev, frag, 0, len, DMA_TO_DEVICE); 458 if (dma_mapping_error(dmadev, *dma)) 459 return -ENOMEM; 460 entry->type = TSNEP_TX_TYPE_SKB_FRAG_MAP_PAGE; 461 mapped = 1; 462 } else { 463 void *fragdata = skb_frag_address_safe(frag); 464 465 if (likely(fragdata)) { 466 memcpy(&entry->desc->tx, fragdata, len); 467 } else { 468 struct page *page = skb_frag_page(frag); 469 470 fragdata = kmap_local_page(page); 471 memcpy(&entry->desc->tx, fragdata + skb_frag_off(frag), 472 len); 473 kunmap_local(fragdata); 474 } 475 entry->type = TSNEP_TX_TYPE_SKB_FRAG_INLINE; 476 mapped = 0; 477 } 478 479 return mapped; 480 } 481 482 static int tsnep_tx_map(struct sk_buff *skb, struct tsnep_tx *tx, int count) 483 { 484 struct device *dmadev = tx->adapter->dmadev; 485 struct tsnep_tx_entry *entry; 486 unsigned int len; 487 int map_len = 0; 488 dma_addr_t dma; 489 int i, mapped; 490 491 for (i = 0; i < count; i++) { 492 entry = &tx->entry[(tx->write + i) & TSNEP_RING_MASK]; 493 494 if (!i) { 495 len = skb_headlen(skb); 496 if (likely(len > TSNEP_DESC_SIZE_DATA_AFTER_INLINE)) { 497 dma = dma_map_single(dmadev, skb->data, len, 498 DMA_TO_DEVICE); 499 if (dma_mapping_error(dmadev, dma)) 500 return -ENOMEM; 501 entry->type = TSNEP_TX_TYPE_SKB_MAP; 502 mapped = 1; 503 } else { 504 memcpy(&entry->desc->tx, skb->data, len); 505 entry->type = TSNEP_TX_TYPE_SKB_INLINE; 506 mapped = 0; 507 } 508 } else { 509 skb_frag_t *frag = &skb_shinfo(skb)->frags[i - 1]; 510 511 len = skb_frag_size(frag); 512 mapped = tsnep_tx_map_frag(frag, entry, dmadev, &dma); 513 if (mapped < 0) 514 return mapped; 515 } 516 517 entry->len = len; 518 if (likely(mapped)) { 519 dma_unmap_addr_set(entry, dma, dma); 520 entry->desc->tx = __cpu_to_le64(dma); 521 } 522 523 map_len += len; 524 } 525 526 return map_len; 527 } 528 529 static int tsnep_tx_unmap(struct tsnep_tx *tx, int index, int count) 530 { 531 struct device *dmadev = tx->adapter->dmadev; 532 struct tsnep_tx_entry *entry; 533 int map_len = 0; 534 int i; 535 536 for (i = 0; i < count; i++) { 537 entry = &tx->entry[(index + i) & TSNEP_RING_MASK]; 538 539 if (entry->len) { 540 if (entry->type & TSNEP_TX_TYPE_MAP) 541 dma_unmap_single(dmadev, 542 dma_unmap_addr(entry, dma), 543 dma_unmap_len(entry, len), 544 DMA_TO_DEVICE); 545 else if (entry->type & TSNEP_TX_TYPE_MAP_PAGE) 546 dma_unmap_page(dmadev, 547 dma_unmap_addr(entry, dma), 548 dma_unmap_len(entry, len), 549 DMA_TO_DEVICE); 550 map_len += entry->len; 551 entry->len = 0; 552 } 553 } 554 555 return map_len; 556 } 557 558 static netdev_tx_t tsnep_xmit_frame_ring(struct sk_buff *skb, 559 struct tsnep_tx *tx) 560 { 561 int count = 1; 562 struct tsnep_tx_entry *entry; 563 int length; 564 int i; 565 int retval; 566 567 if (skb_shinfo(skb)->nr_frags > 0) 568 count += skb_shinfo(skb)->nr_frags; 569 570 if (tsnep_tx_desc_available(tx) < count) { 571 /* ring full, shall not happen because queue is stopped if full 572 * below 573 */ 574 netif_stop_subqueue(tx->adapter->netdev, tx->queue_index); 575 576 return NETDEV_TX_BUSY; 577 } 578 579 entry = &tx->entry[tx->write]; 580 entry->skb = skb; 581 582 retval = tsnep_tx_map(skb, tx, count); 583 if (retval < 0) { 584 tsnep_tx_unmap(tx, tx->write, count); 585 dev_kfree_skb_any(entry->skb); 586 entry->skb = NULL; 587 588 tx->dropped++; 589 590 return NETDEV_TX_OK; 591 } 592 length = retval; 593 594 if (skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) 595 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS; 596 597 for (i = 0; i < count; i++) 598 tsnep_tx_activate(tx, (tx->write + i) & TSNEP_RING_MASK, length, 599 i == count - 1); 600 tx->write = (tx->write + count) & TSNEP_RING_MASK; 601 602 skb_tx_timestamp(skb); 603 604 /* descriptor properties shall be valid before hardware is notified */ 605 dma_wmb(); 606 607 iowrite32(TSNEP_CONTROL_TX_ENABLE, tx->addr + TSNEP_CONTROL); 608 609 if (tsnep_tx_desc_available(tx) < (MAX_SKB_FRAGS + 1)) { 610 /* ring can get full with next frame */ 611 netif_stop_subqueue(tx->adapter->netdev, tx->queue_index); 612 } 613 614 return NETDEV_TX_OK; 615 } 616 617 static int tsnep_xdp_tx_map(struct xdp_frame *xdpf, struct tsnep_tx *tx, 618 struct skb_shared_info *shinfo, int count, u32 type) 619 { 620 struct device *dmadev = tx->adapter->dmadev; 621 struct tsnep_tx_entry *entry; 622 struct page *page; 623 skb_frag_t *frag; 624 unsigned int len; 625 int map_len = 0; 626 dma_addr_t dma; 627 void *data; 628 int i; 629 630 frag = NULL; 631 len = xdpf->len; 632 for (i = 0; i < count; i++) { 633 entry = &tx->entry[(tx->write + i) & TSNEP_RING_MASK]; 634 if (type & TSNEP_TX_TYPE_XDP_NDO) { 635 data = unlikely(frag) ? skb_frag_address(frag) : 636 xdpf->data; 637 dma = dma_map_single(dmadev, data, len, DMA_TO_DEVICE); 638 if (dma_mapping_error(dmadev, dma)) 639 return -ENOMEM; 640 641 entry->type = TSNEP_TX_TYPE_XDP_NDO_MAP_PAGE; 642 } else { 643 page = unlikely(frag) ? skb_frag_page(frag) : 644 virt_to_page(xdpf->data); 645 dma = page_pool_get_dma_addr(page); 646 if (unlikely(frag)) 647 dma += skb_frag_off(frag); 648 else 649 dma += sizeof(*xdpf) + xdpf->headroom; 650 dma_sync_single_for_device(dmadev, dma, len, 651 DMA_BIDIRECTIONAL); 652 653 entry->type = TSNEP_TX_TYPE_XDP_TX; 654 } 655 656 entry->len = len; 657 dma_unmap_addr_set(entry, dma, dma); 658 659 entry->desc->tx = __cpu_to_le64(dma); 660 661 map_len += len; 662 663 if (i + 1 < count) { 664 frag = &shinfo->frags[i]; 665 len = skb_frag_size(frag); 666 } 667 } 668 669 return map_len; 670 } 671 672 /* This function requires __netif_tx_lock is held by the caller. */ 673 static bool tsnep_xdp_xmit_frame_ring(struct xdp_frame *xdpf, 674 struct tsnep_tx *tx, u32 type) 675 { 676 struct skb_shared_info *shinfo = xdp_get_shared_info_from_frame(xdpf); 677 struct tsnep_tx_entry *entry; 678 int count, length, retval, i; 679 680 count = 1; 681 if (unlikely(xdp_frame_has_frags(xdpf))) 682 count += shinfo->nr_frags; 683 684 /* ensure that TX ring is not filled up by XDP, always MAX_SKB_FRAGS 685 * will be available for normal TX path and queue is stopped there if 686 * necessary 687 */ 688 if (tsnep_tx_desc_available(tx) < (MAX_SKB_FRAGS + 1 + count)) 689 return false; 690 691 entry = &tx->entry[tx->write]; 692 entry->xdpf = xdpf; 693 694 retval = tsnep_xdp_tx_map(xdpf, tx, shinfo, count, type); 695 if (retval < 0) { 696 tsnep_tx_unmap(tx, tx->write, count); 697 entry->xdpf = NULL; 698 699 tx->dropped++; 700 701 return false; 702 } 703 length = retval; 704 705 for (i = 0; i < count; i++) 706 tsnep_tx_activate(tx, (tx->write + i) & TSNEP_RING_MASK, length, 707 i == count - 1); 708 tx->write = (tx->write + count) & TSNEP_RING_MASK; 709 710 /* descriptor properties shall be valid before hardware is notified */ 711 dma_wmb(); 712 713 return true; 714 } 715 716 static void tsnep_xdp_xmit_flush(struct tsnep_tx *tx) 717 { 718 iowrite32(TSNEP_CONTROL_TX_ENABLE, tx->addr + TSNEP_CONTROL); 719 } 720 721 static bool tsnep_xdp_xmit_back(struct tsnep_adapter *adapter, 722 struct xdp_buff *xdp, 723 struct netdev_queue *tx_nq, struct tsnep_tx *tx, 724 bool zc) 725 { 726 struct xdp_frame *xdpf = xdp_convert_buff_to_frame(xdp); 727 bool xmit; 728 u32 type; 729 730 if (unlikely(!xdpf)) 731 return false; 732 733 /* no page pool for zero copy */ 734 if (zc) 735 type = TSNEP_TX_TYPE_XDP_NDO; 736 else 737 type = TSNEP_TX_TYPE_XDP_TX; 738 739 __netif_tx_lock(tx_nq, smp_processor_id()); 740 741 xmit = tsnep_xdp_xmit_frame_ring(xdpf, tx, type); 742 743 /* Avoid transmit queue timeout since we share it with the slow path */ 744 if (xmit) 745 txq_trans_cond_update(tx_nq); 746 747 __netif_tx_unlock(tx_nq); 748 749 return xmit; 750 } 751 752 static int tsnep_xdp_tx_map_zc(struct xdp_desc *xdpd, struct tsnep_tx *tx) 753 { 754 struct tsnep_tx_entry *entry; 755 dma_addr_t dma; 756 757 entry = &tx->entry[tx->write]; 758 entry->zc = true; 759 760 dma = xsk_buff_raw_get_dma(tx->xsk_pool, xdpd->addr); 761 xsk_buff_raw_dma_sync_for_device(tx->xsk_pool, dma, xdpd->len); 762 763 entry->type = TSNEP_TX_TYPE_XSK; 764 entry->len = xdpd->len; 765 766 entry->desc->tx = __cpu_to_le64(dma); 767 768 return xdpd->len; 769 } 770 771 static void tsnep_xdp_xmit_frame_ring_zc(struct xdp_desc *xdpd, 772 struct tsnep_tx *tx) 773 { 774 int length; 775 776 length = tsnep_xdp_tx_map_zc(xdpd, tx); 777 778 tsnep_tx_activate(tx, tx->write, length, true); 779 tx->write = (tx->write + 1) & TSNEP_RING_MASK; 780 } 781 782 static void tsnep_xdp_xmit_zc(struct tsnep_tx *tx) 783 { 784 int desc_available = tsnep_tx_desc_available(tx); 785 struct xdp_desc *descs = tx->xsk_pool->tx_descs; 786 int batch, i; 787 788 /* ensure that TX ring is not filled up by XDP, always MAX_SKB_FRAGS 789 * will be available for normal TX path and queue is stopped there if 790 * necessary 791 */ 792 if (desc_available <= (MAX_SKB_FRAGS + 1)) 793 return; 794 desc_available -= MAX_SKB_FRAGS + 1; 795 796 batch = xsk_tx_peek_release_desc_batch(tx->xsk_pool, desc_available); 797 for (i = 0; i < batch; i++) 798 tsnep_xdp_xmit_frame_ring_zc(&descs[i], tx); 799 800 if (batch) { 801 /* descriptor properties shall be valid before hardware is 802 * notified 803 */ 804 dma_wmb(); 805 806 tsnep_xdp_xmit_flush(tx); 807 } 808 } 809 810 static bool tsnep_tx_poll(struct tsnep_tx *tx, int napi_budget) 811 { 812 struct tsnep_tx_entry *entry; 813 struct netdev_queue *nq; 814 int xsk_frames = 0; 815 int budget = 128; 816 int length; 817 int count; 818 819 nq = netdev_get_tx_queue(tx->adapter->netdev, tx->queue_index); 820 __netif_tx_lock(nq, smp_processor_id()); 821 822 do { 823 if (tx->read == tx->write) 824 break; 825 826 entry = &tx->entry[tx->read]; 827 if ((__le32_to_cpu(entry->desc_wb->properties) & 828 TSNEP_TX_DESC_OWNER_MASK) != 829 (entry->properties & TSNEP_TX_DESC_OWNER_MASK)) 830 break; 831 832 /* descriptor properties shall be read first, because valid data 833 * is signaled there 834 */ 835 dma_rmb(); 836 837 count = 1; 838 if ((entry->type & TSNEP_TX_TYPE_SKB) && 839 skb_shinfo(entry->skb)->nr_frags > 0) 840 count += skb_shinfo(entry->skb)->nr_frags; 841 else if ((entry->type & TSNEP_TX_TYPE_XDP) && 842 xdp_frame_has_frags(entry->xdpf)) 843 count += xdp_get_shared_info_from_frame(entry->xdpf)->nr_frags; 844 845 length = tsnep_tx_unmap(tx, tx->read, count); 846 847 if ((entry->type & TSNEP_TX_TYPE_SKB) && 848 (skb_shinfo(entry->skb)->tx_flags & SKBTX_IN_PROGRESS) && 849 (__le32_to_cpu(entry->desc_wb->properties) & 850 TSNEP_DESC_EXTENDED_WRITEBACK_FLAG)) { 851 struct skb_shared_hwtstamps hwtstamps; 852 u64 timestamp; 853 854 if (entry->skb->sk && 855 READ_ONCE(entry->skb->sk->sk_tsflags) & SOF_TIMESTAMPING_BIND_PHC) 856 timestamp = 857 __le64_to_cpu(entry->desc_wb->counter); 858 else 859 timestamp = 860 __le64_to_cpu(entry->desc_wb->timestamp); 861 862 memset(&hwtstamps, 0, sizeof(hwtstamps)); 863 hwtstamps.hwtstamp = ns_to_ktime(timestamp); 864 865 skb_tstamp_tx(entry->skb, &hwtstamps); 866 } 867 868 if (entry->type & TSNEP_TX_TYPE_SKB) 869 napi_consume_skb(entry->skb, napi_budget); 870 else if (entry->type & TSNEP_TX_TYPE_XDP) 871 xdp_return_frame_rx_napi(entry->xdpf); 872 else 873 xsk_frames++; 874 /* xdpf and zc are union with skb */ 875 entry->skb = NULL; 876 877 tx->read = (tx->read + count) & TSNEP_RING_MASK; 878 879 tx->packets++; 880 tx->bytes += length + ETH_FCS_LEN; 881 882 budget--; 883 } while (likely(budget)); 884 885 if (tx->xsk_pool) { 886 if (xsk_frames) 887 xsk_tx_completed(tx->xsk_pool, xsk_frames); 888 if (xsk_uses_need_wakeup(tx->xsk_pool)) 889 xsk_set_tx_need_wakeup(tx->xsk_pool); 890 tsnep_xdp_xmit_zc(tx); 891 } 892 893 if ((tsnep_tx_desc_available(tx) >= ((MAX_SKB_FRAGS + 1) * 2)) && 894 netif_tx_queue_stopped(nq)) { 895 netif_tx_wake_queue(nq); 896 } 897 898 __netif_tx_unlock(nq); 899 900 return budget != 0; 901 } 902 903 static bool tsnep_tx_pending(struct tsnep_tx *tx) 904 { 905 struct tsnep_tx_entry *entry; 906 struct netdev_queue *nq; 907 bool pending = false; 908 909 nq = netdev_get_tx_queue(tx->adapter->netdev, tx->queue_index); 910 __netif_tx_lock(nq, smp_processor_id()); 911 912 if (tx->read != tx->write) { 913 entry = &tx->entry[tx->read]; 914 if ((__le32_to_cpu(entry->desc_wb->properties) & 915 TSNEP_TX_DESC_OWNER_MASK) == 916 (entry->properties & TSNEP_TX_DESC_OWNER_MASK)) 917 pending = true; 918 } 919 920 __netif_tx_unlock(nq); 921 922 return pending; 923 } 924 925 static int tsnep_tx_open(struct tsnep_tx *tx) 926 { 927 int retval; 928 929 retval = tsnep_tx_ring_create(tx); 930 if (retval) 931 return retval; 932 933 tsnep_tx_init(tx); 934 935 return 0; 936 } 937 938 static void tsnep_tx_close(struct tsnep_tx *tx) 939 { 940 tsnep_tx_ring_cleanup(tx); 941 } 942 943 static void tsnep_rx_ring_cleanup(struct tsnep_rx *rx) 944 { 945 struct device *dmadev = rx->adapter->dmadev; 946 struct tsnep_rx_entry *entry; 947 int i; 948 949 for (i = 0; i < TSNEP_RING_SIZE; i++) { 950 entry = &rx->entry[i]; 951 if (!rx->xsk_pool && entry->page) 952 page_pool_put_full_page(rx->page_pool, entry->page, 953 false); 954 if (rx->xsk_pool && entry->xdp) 955 xsk_buff_free(entry->xdp); 956 /* xdp is union with page */ 957 entry->page = NULL; 958 } 959 960 if (rx->page_pool) 961 page_pool_destroy(rx->page_pool); 962 963 memset(rx->entry, 0, sizeof(rx->entry)); 964 965 for (i = 0; i < TSNEP_RING_PAGE_COUNT; i++) { 966 if (rx->page[i]) { 967 dma_free_coherent(dmadev, PAGE_SIZE, rx->page[i], 968 rx->page_dma[i]); 969 rx->page[i] = NULL; 970 rx->page_dma[i] = 0; 971 } 972 } 973 } 974 975 static int tsnep_rx_ring_create(struct tsnep_rx *rx) 976 { 977 struct device *dmadev = rx->adapter->dmadev; 978 struct tsnep_rx_entry *entry; 979 struct page_pool_params pp_params = { 0 }; 980 struct tsnep_rx_entry *next_entry; 981 int i, j; 982 int retval; 983 984 for (i = 0; i < TSNEP_RING_PAGE_COUNT; i++) { 985 rx->page[i] = 986 dma_alloc_coherent(dmadev, PAGE_SIZE, &rx->page_dma[i], 987 GFP_KERNEL); 988 if (!rx->page[i]) { 989 retval = -ENOMEM; 990 goto failed; 991 } 992 for (j = 0; j < TSNEP_RING_ENTRIES_PER_PAGE; j++) { 993 entry = &rx->entry[TSNEP_RING_ENTRIES_PER_PAGE * i + j]; 994 entry->desc_wb = (struct tsnep_rx_desc_wb *) 995 (((u8 *)rx->page[i]) + TSNEP_DESC_SIZE * j); 996 entry->desc = (struct tsnep_rx_desc *) 997 (((u8 *)entry->desc_wb) + TSNEP_DESC_OFFSET); 998 entry->desc_dma = rx->page_dma[i] + TSNEP_DESC_SIZE * j; 999 } 1000 } 1001 1002 pp_params.flags = PP_FLAG_DMA_MAP | PP_FLAG_DMA_SYNC_DEV; 1003 pp_params.order = 0; 1004 pp_params.pool_size = TSNEP_RING_SIZE; 1005 pp_params.nid = dev_to_node(dmadev); 1006 pp_params.dev = dmadev; 1007 pp_params.dma_dir = DMA_BIDIRECTIONAL; 1008 pp_params.max_len = TSNEP_MAX_RX_BUF_SIZE; 1009 pp_params.offset = TSNEP_RX_OFFSET; 1010 rx->page_pool = page_pool_create(&pp_params); 1011 if (IS_ERR(rx->page_pool)) { 1012 retval = PTR_ERR(rx->page_pool); 1013 rx->page_pool = NULL; 1014 goto failed; 1015 } 1016 1017 for (i = 0; i < TSNEP_RING_SIZE; i++) { 1018 entry = &rx->entry[i]; 1019 next_entry = &rx->entry[(i + 1) & TSNEP_RING_MASK]; 1020 entry->desc->next = __cpu_to_le64(next_entry->desc_dma); 1021 } 1022 1023 return 0; 1024 1025 failed: 1026 tsnep_rx_ring_cleanup(rx); 1027 return retval; 1028 } 1029 1030 static void tsnep_rx_init(struct tsnep_rx *rx) 1031 { 1032 dma_addr_t dma; 1033 1034 dma = rx->entry[0].desc_dma | TSNEP_RESET_OWNER_COUNTER; 1035 iowrite32(DMA_ADDR_LOW(dma), rx->addr + TSNEP_RX_DESC_ADDR_LOW); 1036 iowrite32(DMA_ADDR_HIGH(dma), rx->addr + TSNEP_RX_DESC_ADDR_HIGH); 1037 rx->write = 0; 1038 rx->read = 0; 1039 rx->owner_counter = 1; 1040 rx->increment_owner_counter = TSNEP_RING_SIZE - 1; 1041 } 1042 1043 static void tsnep_rx_enable(struct tsnep_rx *rx) 1044 { 1045 /* descriptor properties shall be valid before hardware is notified */ 1046 dma_wmb(); 1047 1048 iowrite32(TSNEP_CONTROL_RX_ENABLE, rx->addr + TSNEP_CONTROL); 1049 } 1050 1051 static void tsnep_rx_disable(struct tsnep_rx *rx) 1052 { 1053 u32 val; 1054 1055 iowrite32(TSNEP_CONTROL_RX_DISABLE, rx->addr + TSNEP_CONTROL); 1056 readx_poll_timeout(ioread32, rx->addr + TSNEP_CONTROL, val, 1057 ((val & TSNEP_CONTROL_RX_ENABLE) == 0), 10000, 1058 1000000); 1059 } 1060 1061 static int tsnep_rx_desc_available(struct tsnep_rx *rx) 1062 { 1063 if (rx->read <= rx->write) 1064 return TSNEP_RING_SIZE - rx->write + rx->read - 1; 1065 else 1066 return rx->read - rx->write - 1; 1067 } 1068 1069 static void tsnep_rx_free_page_buffer(struct tsnep_rx *rx) 1070 { 1071 struct page **page; 1072 1073 /* last entry of page_buffer is always zero, because ring cannot be 1074 * filled completely 1075 */ 1076 page = rx->page_buffer; 1077 while (*page) { 1078 page_pool_put_full_page(rx->page_pool, *page, false); 1079 *page = NULL; 1080 page++; 1081 } 1082 } 1083 1084 static int tsnep_rx_alloc_page_buffer(struct tsnep_rx *rx) 1085 { 1086 int i; 1087 1088 /* alloc for all ring entries except the last one, because ring cannot 1089 * be filled completely 1090 */ 1091 for (i = 0; i < TSNEP_RING_SIZE - 1; i++) { 1092 rx->page_buffer[i] = page_pool_dev_alloc_pages(rx->page_pool); 1093 if (!rx->page_buffer[i]) { 1094 tsnep_rx_free_page_buffer(rx); 1095 1096 return -ENOMEM; 1097 } 1098 } 1099 1100 return 0; 1101 } 1102 1103 static void tsnep_rx_set_page(struct tsnep_rx *rx, struct tsnep_rx_entry *entry, 1104 struct page *page) 1105 { 1106 entry->page = page; 1107 entry->len = TSNEP_MAX_RX_BUF_SIZE; 1108 entry->dma = page_pool_get_dma_addr(entry->page); 1109 entry->desc->rx = __cpu_to_le64(entry->dma + TSNEP_RX_OFFSET); 1110 } 1111 1112 static int tsnep_rx_alloc_buffer(struct tsnep_rx *rx, int index) 1113 { 1114 struct tsnep_rx_entry *entry = &rx->entry[index]; 1115 struct page *page; 1116 1117 page = page_pool_dev_alloc_pages(rx->page_pool); 1118 if (unlikely(!page)) 1119 return -ENOMEM; 1120 tsnep_rx_set_page(rx, entry, page); 1121 1122 return 0; 1123 } 1124 1125 static void tsnep_rx_reuse_buffer(struct tsnep_rx *rx, int index) 1126 { 1127 struct tsnep_rx_entry *entry = &rx->entry[index]; 1128 struct tsnep_rx_entry *read = &rx->entry[rx->read]; 1129 1130 tsnep_rx_set_page(rx, entry, read->page); 1131 read->page = NULL; 1132 } 1133 1134 static void tsnep_rx_activate(struct tsnep_rx *rx, int index) 1135 { 1136 struct tsnep_rx_entry *entry = &rx->entry[index]; 1137 1138 /* TSNEP_MAX_RX_BUF_SIZE and TSNEP_XSK_RX_BUF_SIZE are multiple of 4 */ 1139 entry->properties = entry->len & TSNEP_DESC_LENGTH_MASK; 1140 entry->properties |= TSNEP_DESC_INTERRUPT_FLAG; 1141 if (index == rx->increment_owner_counter) { 1142 rx->owner_counter++; 1143 if (rx->owner_counter == 4) 1144 rx->owner_counter = 1; 1145 rx->increment_owner_counter--; 1146 if (rx->increment_owner_counter < 0) 1147 rx->increment_owner_counter = TSNEP_RING_SIZE - 1; 1148 } 1149 entry->properties |= 1150 (rx->owner_counter << TSNEP_DESC_OWNER_COUNTER_SHIFT) & 1151 TSNEP_DESC_OWNER_COUNTER_MASK; 1152 1153 /* descriptor properties shall be written last, because valid data is 1154 * signaled there 1155 */ 1156 dma_wmb(); 1157 1158 entry->desc->properties = __cpu_to_le32(entry->properties); 1159 } 1160 1161 static int tsnep_rx_alloc(struct tsnep_rx *rx, int count, bool reuse) 1162 { 1163 bool alloc_failed = false; 1164 int i, index; 1165 1166 for (i = 0; i < count && !alloc_failed; i++) { 1167 index = (rx->write + i) & TSNEP_RING_MASK; 1168 1169 if (unlikely(tsnep_rx_alloc_buffer(rx, index))) { 1170 rx->alloc_failed++; 1171 alloc_failed = true; 1172 1173 /* reuse only if no other allocation was successful */ 1174 if (i == 0 && reuse) 1175 tsnep_rx_reuse_buffer(rx, index); 1176 else 1177 break; 1178 } 1179 1180 tsnep_rx_activate(rx, index); 1181 } 1182 1183 if (i) 1184 rx->write = (rx->write + i) & TSNEP_RING_MASK; 1185 1186 return i; 1187 } 1188 1189 static int tsnep_rx_refill(struct tsnep_rx *rx, int count, bool reuse) 1190 { 1191 int desc_refilled; 1192 1193 desc_refilled = tsnep_rx_alloc(rx, count, reuse); 1194 if (desc_refilled) 1195 tsnep_rx_enable(rx); 1196 1197 return desc_refilled; 1198 } 1199 1200 static void tsnep_rx_set_xdp(struct tsnep_rx *rx, struct tsnep_rx_entry *entry, 1201 struct xdp_buff *xdp) 1202 { 1203 entry->xdp = xdp; 1204 entry->len = TSNEP_XSK_RX_BUF_SIZE; 1205 entry->dma = xsk_buff_xdp_get_dma(entry->xdp); 1206 entry->desc->rx = __cpu_to_le64(entry->dma); 1207 } 1208 1209 static void tsnep_rx_reuse_buffer_zc(struct tsnep_rx *rx, int index) 1210 { 1211 struct tsnep_rx_entry *entry = &rx->entry[index]; 1212 struct tsnep_rx_entry *read = &rx->entry[rx->read]; 1213 1214 tsnep_rx_set_xdp(rx, entry, read->xdp); 1215 read->xdp = NULL; 1216 } 1217 1218 static int tsnep_rx_alloc_zc(struct tsnep_rx *rx, int count, bool reuse) 1219 { 1220 u32 allocated; 1221 int i; 1222 1223 allocated = xsk_buff_alloc_batch(rx->xsk_pool, rx->xdp_batch, count); 1224 for (i = 0; i < allocated; i++) { 1225 int index = (rx->write + i) & TSNEP_RING_MASK; 1226 struct tsnep_rx_entry *entry = &rx->entry[index]; 1227 1228 tsnep_rx_set_xdp(rx, entry, rx->xdp_batch[i]); 1229 tsnep_rx_activate(rx, index); 1230 } 1231 if (i == 0) { 1232 rx->alloc_failed++; 1233 1234 if (reuse) { 1235 tsnep_rx_reuse_buffer_zc(rx, rx->write); 1236 tsnep_rx_activate(rx, rx->write); 1237 } 1238 } 1239 1240 if (i) 1241 rx->write = (rx->write + i) & TSNEP_RING_MASK; 1242 1243 return i; 1244 } 1245 1246 static void tsnep_rx_free_zc(struct tsnep_rx *rx) 1247 { 1248 int i; 1249 1250 for (i = 0; i < TSNEP_RING_SIZE; i++) { 1251 struct tsnep_rx_entry *entry = &rx->entry[i]; 1252 1253 if (entry->xdp) 1254 xsk_buff_free(entry->xdp); 1255 entry->xdp = NULL; 1256 } 1257 } 1258 1259 static int tsnep_rx_refill_zc(struct tsnep_rx *rx, int count, bool reuse) 1260 { 1261 int desc_refilled; 1262 1263 desc_refilled = tsnep_rx_alloc_zc(rx, count, reuse); 1264 if (desc_refilled) 1265 tsnep_rx_enable(rx); 1266 1267 return desc_refilled; 1268 } 1269 1270 static void tsnep_xsk_rx_need_wakeup(struct tsnep_rx *rx, int desc_available) 1271 { 1272 if (desc_available) 1273 xsk_set_rx_need_wakeup(rx->xsk_pool); 1274 else 1275 xsk_clear_rx_need_wakeup(rx->xsk_pool); 1276 } 1277 1278 static bool tsnep_xdp_run_prog(struct tsnep_rx *rx, struct bpf_prog *prog, 1279 struct xdp_buff *xdp, int *status, 1280 struct netdev_queue *tx_nq, struct tsnep_tx *tx) 1281 { 1282 unsigned int length; 1283 unsigned int sync; 1284 u32 act; 1285 1286 length = xdp->data_end - xdp->data_hard_start - XDP_PACKET_HEADROOM; 1287 1288 act = bpf_prog_run_xdp(prog, xdp); 1289 switch (act) { 1290 case XDP_PASS: 1291 return false; 1292 case XDP_TX: 1293 if (!tsnep_xdp_xmit_back(rx->adapter, xdp, tx_nq, tx, false)) 1294 goto out_failure; 1295 *status |= TSNEP_XDP_TX; 1296 return true; 1297 case XDP_REDIRECT: 1298 if (xdp_do_redirect(rx->adapter->netdev, xdp, prog) < 0) 1299 goto out_failure; 1300 *status |= TSNEP_XDP_REDIRECT; 1301 return true; 1302 default: 1303 bpf_warn_invalid_xdp_action(rx->adapter->netdev, prog, act); 1304 fallthrough; 1305 case XDP_ABORTED: 1306 out_failure: 1307 trace_xdp_exception(rx->adapter->netdev, prog, act); 1308 fallthrough; 1309 case XDP_DROP: 1310 /* Due xdp_adjust_tail: DMA sync for_device cover max len CPU 1311 * touch 1312 */ 1313 sync = xdp->data_end - xdp->data_hard_start - 1314 XDP_PACKET_HEADROOM; 1315 sync = max(sync, length); 1316 page_pool_put_page(rx->page_pool, virt_to_head_page(xdp->data), 1317 sync, true); 1318 return true; 1319 } 1320 } 1321 1322 static bool tsnep_xdp_run_prog_zc(struct tsnep_rx *rx, struct bpf_prog *prog, 1323 struct xdp_buff *xdp, int *status, 1324 struct netdev_queue *tx_nq, 1325 struct tsnep_tx *tx) 1326 { 1327 u32 act; 1328 1329 act = bpf_prog_run_xdp(prog, xdp); 1330 1331 /* XDP_REDIRECT is the main action for zero-copy */ 1332 if (likely(act == XDP_REDIRECT)) { 1333 if (xdp_do_redirect(rx->adapter->netdev, xdp, prog) < 0) 1334 goto out_failure; 1335 *status |= TSNEP_XDP_REDIRECT; 1336 return true; 1337 } 1338 1339 switch (act) { 1340 case XDP_PASS: 1341 return false; 1342 case XDP_TX: 1343 if (!tsnep_xdp_xmit_back(rx->adapter, xdp, tx_nq, tx, true)) 1344 goto out_failure; 1345 *status |= TSNEP_XDP_TX; 1346 return true; 1347 default: 1348 bpf_warn_invalid_xdp_action(rx->adapter->netdev, prog, act); 1349 fallthrough; 1350 case XDP_ABORTED: 1351 out_failure: 1352 trace_xdp_exception(rx->adapter->netdev, prog, act); 1353 fallthrough; 1354 case XDP_DROP: 1355 xsk_buff_free(xdp); 1356 return true; 1357 } 1358 } 1359 1360 static void tsnep_finalize_xdp(struct tsnep_adapter *adapter, int status, 1361 struct netdev_queue *tx_nq, struct tsnep_tx *tx) 1362 { 1363 if (status & TSNEP_XDP_TX) { 1364 __netif_tx_lock(tx_nq, smp_processor_id()); 1365 tsnep_xdp_xmit_flush(tx); 1366 __netif_tx_unlock(tx_nq); 1367 } 1368 1369 if (status & TSNEP_XDP_REDIRECT) 1370 xdp_do_flush(); 1371 } 1372 1373 static struct sk_buff *tsnep_build_skb(struct tsnep_rx *rx, struct page *page, 1374 int length) 1375 { 1376 struct sk_buff *skb; 1377 1378 skb = napi_build_skb(page_address(page), PAGE_SIZE); 1379 if (unlikely(!skb)) 1380 return NULL; 1381 1382 /* update pointers within the skb to store the data */ 1383 skb_reserve(skb, TSNEP_RX_OFFSET + TSNEP_RX_INLINE_METADATA_SIZE); 1384 __skb_put(skb, length - ETH_FCS_LEN); 1385 1386 if (rx->adapter->hwtstamp_config.rx_filter == HWTSTAMP_FILTER_ALL) { 1387 struct skb_shared_hwtstamps *hwtstamps = skb_hwtstamps(skb); 1388 struct tsnep_rx_inline *rx_inline = 1389 (struct tsnep_rx_inline *)(page_address(page) + 1390 TSNEP_RX_OFFSET); 1391 1392 skb_shinfo(skb)->tx_flags |= 1393 SKBTX_HW_TSTAMP_NETDEV; 1394 memset(hwtstamps, 0, sizeof(*hwtstamps)); 1395 hwtstamps->netdev_data = rx_inline; 1396 } 1397 1398 skb_record_rx_queue(skb, rx->queue_index); 1399 skb->protocol = eth_type_trans(skb, rx->adapter->netdev); 1400 1401 return skb; 1402 } 1403 1404 static void tsnep_rx_page(struct tsnep_rx *rx, struct napi_struct *napi, 1405 struct page *page, int length) 1406 { 1407 struct sk_buff *skb; 1408 1409 skb = tsnep_build_skb(rx, page, length); 1410 if (skb) { 1411 skb_mark_for_recycle(skb); 1412 1413 rx->packets++; 1414 rx->bytes += length; 1415 if (skb->pkt_type == PACKET_MULTICAST) 1416 rx->multicast++; 1417 1418 napi_gro_receive(napi, skb); 1419 } else { 1420 page_pool_recycle_direct(rx->page_pool, page); 1421 1422 rx->dropped++; 1423 } 1424 } 1425 1426 static int tsnep_rx_poll(struct tsnep_rx *rx, struct napi_struct *napi, 1427 int budget) 1428 { 1429 struct device *dmadev = rx->adapter->dmadev; 1430 enum dma_data_direction dma_dir; 1431 struct tsnep_rx_entry *entry; 1432 struct netdev_queue *tx_nq; 1433 struct bpf_prog *prog; 1434 struct xdp_buff xdp; 1435 struct tsnep_tx *tx; 1436 int desc_available; 1437 int xdp_status = 0; 1438 int done = 0; 1439 int length; 1440 1441 desc_available = tsnep_rx_desc_available(rx); 1442 dma_dir = page_pool_get_dma_dir(rx->page_pool); 1443 prog = READ_ONCE(rx->adapter->xdp_prog); 1444 if (prog) { 1445 tx_nq = netdev_get_tx_queue(rx->adapter->netdev, 1446 rx->tx_queue_index); 1447 tx = &rx->adapter->tx[rx->tx_queue_index]; 1448 1449 xdp_init_buff(&xdp, PAGE_SIZE, &rx->xdp_rxq); 1450 } 1451 1452 while (likely(done < budget) && (rx->read != rx->write)) { 1453 entry = &rx->entry[rx->read]; 1454 if ((__le32_to_cpu(entry->desc_wb->properties) & 1455 TSNEP_DESC_OWNER_COUNTER_MASK) != 1456 (entry->properties & TSNEP_DESC_OWNER_COUNTER_MASK)) 1457 break; 1458 done++; 1459 1460 if (desc_available >= TSNEP_RING_RX_REFILL) { 1461 bool reuse = desc_available >= TSNEP_RING_RX_REUSE; 1462 1463 desc_available -= tsnep_rx_refill(rx, desc_available, 1464 reuse); 1465 if (!entry->page) { 1466 /* buffer has been reused for refill to prevent 1467 * empty RX ring, thus buffer cannot be used for 1468 * RX processing 1469 */ 1470 rx->read = (rx->read + 1) & TSNEP_RING_MASK; 1471 desc_available++; 1472 1473 rx->dropped++; 1474 1475 continue; 1476 } 1477 } 1478 1479 /* descriptor properties shall be read first, because valid data 1480 * is signaled there 1481 */ 1482 dma_rmb(); 1483 1484 prefetch(page_address(entry->page) + TSNEP_RX_OFFSET); 1485 length = __le32_to_cpu(entry->desc_wb->properties) & 1486 TSNEP_DESC_LENGTH_MASK; 1487 dma_sync_single_range_for_cpu(dmadev, entry->dma, 1488 TSNEP_RX_OFFSET, length, dma_dir); 1489 1490 /* RX metadata with timestamps is in front of actual data, 1491 * subtract metadata size to get length of actual data and 1492 * consider metadata size as offset of actual data during RX 1493 * processing 1494 */ 1495 length -= TSNEP_RX_INLINE_METADATA_SIZE; 1496 1497 rx->read = (rx->read + 1) & TSNEP_RING_MASK; 1498 desc_available++; 1499 1500 if (prog) { 1501 bool consume; 1502 1503 xdp_prepare_buff(&xdp, page_address(entry->page), 1504 XDP_PACKET_HEADROOM + TSNEP_RX_INLINE_METADATA_SIZE, 1505 length - ETH_FCS_LEN, false); 1506 1507 consume = tsnep_xdp_run_prog(rx, prog, &xdp, 1508 &xdp_status, tx_nq, tx); 1509 if (consume) { 1510 rx->packets++; 1511 rx->bytes += length; 1512 1513 entry->page = NULL; 1514 1515 continue; 1516 } 1517 } 1518 1519 tsnep_rx_page(rx, napi, entry->page, length); 1520 entry->page = NULL; 1521 } 1522 1523 if (xdp_status) 1524 tsnep_finalize_xdp(rx->adapter, xdp_status, tx_nq, tx); 1525 1526 if (desc_available) 1527 tsnep_rx_refill(rx, desc_available, false); 1528 1529 return done; 1530 } 1531 1532 static int tsnep_rx_poll_zc(struct tsnep_rx *rx, struct napi_struct *napi, 1533 int budget) 1534 { 1535 struct tsnep_rx_entry *entry; 1536 struct netdev_queue *tx_nq; 1537 struct bpf_prog *prog; 1538 struct tsnep_tx *tx; 1539 int desc_available; 1540 int xdp_status = 0; 1541 struct page *page; 1542 int done = 0; 1543 int length; 1544 1545 desc_available = tsnep_rx_desc_available(rx); 1546 prog = READ_ONCE(rx->adapter->xdp_prog); 1547 if (prog) { 1548 tx_nq = netdev_get_tx_queue(rx->adapter->netdev, 1549 rx->tx_queue_index); 1550 tx = &rx->adapter->tx[rx->tx_queue_index]; 1551 } 1552 1553 while (likely(done < budget) && (rx->read != rx->write)) { 1554 entry = &rx->entry[rx->read]; 1555 if ((__le32_to_cpu(entry->desc_wb->properties) & 1556 TSNEP_DESC_OWNER_COUNTER_MASK) != 1557 (entry->properties & TSNEP_DESC_OWNER_COUNTER_MASK)) 1558 break; 1559 done++; 1560 1561 if (desc_available >= TSNEP_RING_RX_REFILL) { 1562 bool reuse = desc_available >= TSNEP_RING_RX_REUSE; 1563 1564 desc_available -= tsnep_rx_refill_zc(rx, desc_available, 1565 reuse); 1566 if (!entry->xdp) { 1567 /* buffer has been reused for refill to prevent 1568 * empty RX ring, thus buffer cannot be used for 1569 * RX processing 1570 */ 1571 rx->read = (rx->read + 1) & TSNEP_RING_MASK; 1572 desc_available++; 1573 1574 rx->dropped++; 1575 1576 continue; 1577 } 1578 } 1579 1580 /* descriptor properties shall be read first, because valid data 1581 * is signaled there 1582 */ 1583 dma_rmb(); 1584 1585 prefetch(entry->xdp->data); 1586 length = __le32_to_cpu(entry->desc_wb->properties) & 1587 TSNEP_DESC_LENGTH_MASK; 1588 xsk_buff_set_size(entry->xdp, length - ETH_FCS_LEN); 1589 xsk_buff_dma_sync_for_cpu(entry->xdp); 1590 1591 /* RX metadata with timestamps is in front of actual data, 1592 * subtract metadata size to get length of actual data and 1593 * consider metadata size as offset of actual data during RX 1594 * processing 1595 */ 1596 length -= TSNEP_RX_INLINE_METADATA_SIZE; 1597 1598 rx->read = (rx->read + 1) & TSNEP_RING_MASK; 1599 desc_available++; 1600 1601 if (prog) { 1602 bool consume; 1603 1604 entry->xdp->data += TSNEP_RX_INLINE_METADATA_SIZE; 1605 entry->xdp->data_meta += TSNEP_RX_INLINE_METADATA_SIZE; 1606 1607 consume = tsnep_xdp_run_prog_zc(rx, prog, entry->xdp, 1608 &xdp_status, tx_nq, tx); 1609 if (consume) { 1610 rx->packets++; 1611 rx->bytes += length; 1612 1613 entry->xdp = NULL; 1614 1615 continue; 1616 } 1617 } 1618 1619 page = page_pool_dev_alloc_pages(rx->page_pool); 1620 if (page) { 1621 memcpy(page_address(page) + TSNEP_RX_OFFSET, 1622 entry->xdp->data - TSNEP_RX_INLINE_METADATA_SIZE, 1623 length + TSNEP_RX_INLINE_METADATA_SIZE); 1624 tsnep_rx_page(rx, napi, page, length); 1625 } else { 1626 rx->dropped++; 1627 } 1628 xsk_buff_free(entry->xdp); 1629 entry->xdp = NULL; 1630 } 1631 1632 if (xdp_status) 1633 tsnep_finalize_xdp(rx->adapter, xdp_status, tx_nq, tx); 1634 1635 if (desc_available) 1636 desc_available -= tsnep_rx_refill_zc(rx, desc_available, false); 1637 1638 if (xsk_uses_need_wakeup(rx->xsk_pool)) { 1639 tsnep_xsk_rx_need_wakeup(rx, desc_available); 1640 1641 return done; 1642 } 1643 1644 return desc_available ? budget : done; 1645 } 1646 1647 static bool tsnep_rx_pending(struct tsnep_rx *rx) 1648 { 1649 struct tsnep_rx_entry *entry; 1650 1651 if (rx->read != rx->write) { 1652 entry = &rx->entry[rx->read]; 1653 if ((__le32_to_cpu(entry->desc_wb->properties) & 1654 TSNEP_DESC_OWNER_COUNTER_MASK) == 1655 (entry->properties & TSNEP_DESC_OWNER_COUNTER_MASK)) 1656 return true; 1657 } 1658 1659 return false; 1660 } 1661 1662 static int tsnep_rx_open(struct tsnep_rx *rx) 1663 { 1664 int desc_available; 1665 int retval; 1666 1667 retval = tsnep_rx_ring_create(rx); 1668 if (retval) 1669 return retval; 1670 1671 tsnep_rx_init(rx); 1672 1673 desc_available = tsnep_rx_desc_available(rx); 1674 if (rx->xsk_pool) 1675 retval = tsnep_rx_alloc_zc(rx, desc_available, false); 1676 else 1677 retval = tsnep_rx_alloc(rx, desc_available, false); 1678 if (retval != desc_available) { 1679 retval = -ENOMEM; 1680 1681 goto alloc_failed; 1682 } 1683 1684 /* prealloc pages to prevent allocation failures when XSK pool is 1685 * disabled at runtime 1686 */ 1687 if (rx->xsk_pool) { 1688 retval = tsnep_rx_alloc_page_buffer(rx); 1689 if (retval) 1690 goto alloc_failed; 1691 } 1692 1693 return 0; 1694 1695 alloc_failed: 1696 tsnep_rx_ring_cleanup(rx); 1697 return retval; 1698 } 1699 1700 static void tsnep_rx_close(struct tsnep_rx *rx) 1701 { 1702 if (rx->xsk_pool) 1703 tsnep_rx_free_page_buffer(rx); 1704 1705 tsnep_rx_ring_cleanup(rx); 1706 } 1707 1708 static void tsnep_rx_reopen(struct tsnep_rx *rx) 1709 { 1710 struct page **page = rx->page_buffer; 1711 int i; 1712 1713 tsnep_rx_init(rx); 1714 1715 for (i = 0; i < TSNEP_RING_SIZE; i++) { 1716 struct tsnep_rx_entry *entry = &rx->entry[i]; 1717 1718 /* defined initial values for properties are required for 1719 * correct owner counter checking 1720 */ 1721 entry->desc->properties = 0; 1722 entry->desc_wb->properties = 0; 1723 1724 /* prevent allocation failures by reusing kept pages */ 1725 if (*page) { 1726 tsnep_rx_set_page(rx, entry, *page); 1727 tsnep_rx_activate(rx, rx->write); 1728 rx->write++; 1729 1730 *page = NULL; 1731 page++; 1732 } 1733 } 1734 } 1735 1736 static void tsnep_rx_reopen_xsk(struct tsnep_rx *rx) 1737 { 1738 struct page **page = rx->page_buffer; 1739 u32 allocated; 1740 int i; 1741 1742 tsnep_rx_init(rx); 1743 1744 /* alloc all ring entries except the last one, because ring cannot be 1745 * filled completely, as many buffers as possible is enough as wakeup is 1746 * done if new buffers are available 1747 */ 1748 allocated = xsk_buff_alloc_batch(rx->xsk_pool, rx->xdp_batch, 1749 TSNEP_RING_SIZE - 1); 1750 1751 for (i = 0; i < TSNEP_RING_SIZE; i++) { 1752 struct tsnep_rx_entry *entry = &rx->entry[i]; 1753 1754 /* keep pages to prevent allocation failures when xsk is 1755 * disabled 1756 */ 1757 if (entry->page) { 1758 *page = entry->page; 1759 entry->page = NULL; 1760 1761 page++; 1762 } 1763 1764 /* defined initial values for properties are required for 1765 * correct owner counter checking 1766 */ 1767 entry->desc->properties = 0; 1768 entry->desc_wb->properties = 0; 1769 1770 if (allocated) { 1771 tsnep_rx_set_xdp(rx, entry, 1772 rx->xdp_batch[allocated - 1]); 1773 tsnep_rx_activate(rx, rx->write); 1774 rx->write++; 1775 1776 allocated--; 1777 } 1778 } 1779 1780 /* set need wakeup flag immediately if ring is not filled completely, 1781 * first polling would be too late as need wakeup signalisation would 1782 * be delayed for an indefinite time 1783 */ 1784 if (xsk_uses_need_wakeup(rx->xsk_pool)) 1785 tsnep_xsk_rx_need_wakeup(rx, tsnep_rx_desc_available(rx)); 1786 } 1787 1788 static bool tsnep_pending(struct tsnep_queue *queue) 1789 { 1790 if (queue->tx && tsnep_tx_pending(queue->tx)) 1791 return true; 1792 1793 if (queue->rx && tsnep_rx_pending(queue->rx)) 1794 return true; 1795 1796 return false; 1797 } 1798 1799 static int tsnep_poll(struct napi_struct *napi, int budget) 1800 { 1801 struct tsnep_queue *queue = container_of(napi, struct tsnep_queue, 1802 napi); 1803 bool complete = true; 1804 int done = 0; 1805 1806 if (queue->tx) 1807 complete = tsnep_tx_poll(queue->tx, budget); 1808 1809 /* handle case where we are called by netpoll with a budget of 0 */ 1810 if (unlikely(budget <= 0)) 1811 return budget; 1812 1813 if (queue->rx) { 1814 done = queue->rx->xsk_pool ? 1815 tsnep_rx_poll_zc(queue->rx, napi, budget) : 1816 tsnep_rx_poll(queue->rx, napi, budget); 1817 if (done >= budget) 1818 complete = false; 1819 } 1820 1821 /* if all work not completed, return budget and keep polling */ 1822 if (!complete) 1823 return budget; 1824 1825 if (likely(napi_complete_done(napi, done))) { 1826 tsnep_enable_irq(queue->adapter, queue->irq_mask); 1827 1828 /* reschedule if work is already pending, prevent rotten packets 1829 * which are transmitted or received after polling but before 1830 * interrupt enable 1831 */ 1832 if (tsnep_pending(queue)) { 1833 tsnep_disable_irq(queue->adapter, queue->irq_mask); 1834 napi_schedule(napi); 1835 } 1836 } 1837 1838 return min(done, budget - 1); 1839 } 1840 1841 static int tsnep_request_irq(struct tsnep_queue *queue, bool first) 1842 { 1843 const char *name = netdev_name(queue->adapter->netdev); 1844 irq_handler_t handler; 1845 void *dev; 1846 int retval; 1847 1848 if (first) { 1849 sprintf(queue->name, "%s-mac", name); 1850 handler = tsnep_irq; 1851 dev = queue->adapter; 1852 } else { 1853 if (queue->tx && queue->rx) 1854 snprintf(queue->name, sizeof(queue->name), "%s-txrx-%d", 1855 name, queue->rx->queue_index); 1856 else if (queue->tx) 1857 snprintf(queue->name, sizeof(queue->name), "%s-tx-%d", 1858 name, queue->tx->queue_index); 1859 else 1860 snprintf(queue->name, sizeof(queue->name), "%s-rx-%d", 1861 name, queue->rx->queue_index); 1862 handler = tsnep_irq_txrx; 1863 dev = queue; 1864 } 1865 1866 retval = request_irq(queue->irq, handler, 0, queue->name, dev); 1867 if (retval) { 1868 /* if name is empty, then interrupt won't be freed */ 1869 memset(queue->name, 0, sizeof(queue->name)); 1870 } 1871 1872 return retval; 1873 } 1874 1875 static void tsnep_free_irq(struct tsnep_queue *queue, bool first) 1876 { 1877 void *dev; 1878 1879 if (!strlen(queue->name)) 1880 return; 1881 1882 if (first) 1883 dev = queue->adapter; 1884 else 1885 dev = queue; 1886 1887 free_irq(queue->irq, dev); 1888 memset(queue->name, 0, sizeof(queue->name)); 1889 } 1890 1891 static void tsnep_queue_close(struct tsnep_queue *queue, bool first) 1892 { 1893 struct tsnep_rx *rx = queue->rx; 1894 1895 tsnep_free_irq(queue, first); 1896 1897 if (rx) { 1898 if (xdp_rxq_info_is_reg(&rx->xdp_rxq)) 1899 xdp_rxq_info_unreg(&rx->xdp_rxq); 1900 if (xdp_rxq_info_is_reg(&rx->xdp_rxq_zc)) 1901 xdp_rxq_info_unreg(&rx->xdp_rxq_zc); 1902 } 1903 1904 netif_napi_del(&queue->napi); 1905 } 1906 1907 static int tsnep_queue_open(struct tsnep_adapter *adapter, 1908 struct tsnep_queue *queue, bool first) 1909 { 1910 struct tsnep_rx *rx = queue->rx; 1911 struct tsnep_tx *tx = queue->tx; 1912 int retval; 1913 1914 netif_napi_add(adapter->netdev, &queue->napi, tsnep_poll); 1915 1916 if (rx) { 1917 /* choose TX queue for XDP_TX */ 1918 if (tx) 1919 rx->tx_queue_index = tx->queue_index; 1920 else if (rx->queue_index < adapter->num_tx_queues) 1921 rx->tx_queue_index = rx->queue_index; 1922 else 1923 rx->tx_queue_index = 0; 1924 1925 /* prepare both memory models to eliminate possible registration 1926 * errors when memory model is switched between page pool and 1927 * XSK pool during runtime 1928 */ 1929 retval = xdp_rxq_info_reg(&rx->xdp_rxq, adapter->netdev, 1930 rx->queue_index, queue->napi.napi_id); 1931 if (retval) 1932 goto failed; 1933 retval = xdp_rxq_info_reg_mem_model(&rx->xdp_rxq, 1934 MEM_TYPE_PAGE_POOL, 1935 rx->page_pool); 1936 if (retval) 1937 goto failed; 1938 retval = xdp_rxq_info_reg(&rx->xdp_rxq_zc, adapter->netdev, 1939 rx->queue_index, queue->napi.napi_id); 1940 if (retval) 1941 goto failed; 1942 retval = xdp_rxq_info_reg_mem_model(&rx->xdp_rxq_zc, 1943 MEM_TYPE_XSK_BUFF_POOL, 1944 NULL); 1945 if (retval) 1946 goto failed; 1947 if (rx->xsk_pool) 1948 xsk_pool_set_rxq_info(rx->xsk_pool, &rx->xdp_rxq_zc); 1949 } 1950 1951 retval = tsnep_request_irq(queue, first); 1952 if (retval) { 1953 netif_err(adapter, drv, adapter->netdev, 1954 "can't get assigned irq %d.\n", queue->irq); 1955 goto failed; 1956 } 1957 1958 return 0; 1959 1960 failed: 1961 tsnep_queue_close(queue, first); 1962 1963 return retval; 1964 } 1965 1966 static void tsnep_queue_enable(struct tsnep_queue *queue) 1967 { 1968 struct tsnep_adapter *adapter = queue->adapter; 1969 1970 netif_napi_set_irq(&queue->napi, queue->irq); 1971 napi_enable(&queue->napi); 1972 tsnep_enable_irq(adapter, queue->irq_mask); 1973 1974 if (queue->tx) { 1975 netif_queue_set_napi(adapter->netdev, queue->tx->queue_index, 1976 NETDEV_QUEUE_TYPE_TX, &queue->napi); 1977 tsnep_tx_enable(queue->tx); 1978 } 1979 1980 if (queue->rx) { 1981 netif_queue_set_napi(adapter->netdev, queue->rx->queue_index, 1982 NETDEV_QUEUE_TYPE_RX, &queue->napi); 1983 tsnep_rx_enable(queue->rx); 1984 } 1985 } 1986 1987 static void tsnep_queue_disable(struct tsnep_queue *queue) 1988 { 1989 struct tsnep_adapter *adapter = queue->adapter; 1990 1991 if (queue->rx) 1992 netif_queue_set_napi(adapter->netdev, queue->rx->queue_index, 1993 NETDEV_QUEUE_TYPE_RX, NULL); 1994 1995 if (queue->tx) { 1996 tsnep_tx_disable(queue->tx, &queue->napi); 1997 netif_queue_set_napi(adapter->netdev, queue->tx->queue_index, 1998 NETDEV_QUEUE_TYPE_TX, NULL); 1999 } 2000 2001 napi_disable(&queue->napi); 2002 tsnep_disable_irq(adapter, queue->irq_mask); 2003 2004 /* disable RX after NAPI polling has been disabled, because RX can be 2005 * enabled during NAPI polling 2006 */ 2007 if (queue->rx) 2008 tsnep_rx_disable(queue->rx); 2009 } 2010 2011 static int tsnep_netdev_open(struct net_device *netdev) 2012 { 2013 struct tsnep_adapter *adapter = netdev_priv(netdev); 2014 int i, retval; 2015 2016 for (i = 0; i < adapter->num_queues; i++) { 2017 if (adapter->queue[i].tx) { 2018 retval = tsnep_tx_open(adapter->queue[i].tx); 2019 if (retval) 2020 goto failed; 2021 } 2022 if (adapter->queue[i].rx) { 2023 retval = tsnep_rx_open(adapter->queue[i].rx); 2024 if (retval) 2025 goto failed; 2026 } 2027 2028 retval = tsnep_queue_open(adapter, &adapter->queue[i], i == 0); 2029 if (retval) 2030 goto failed; 2031 } 2032 2033 retval = netif_set_real_num_tx_queues(adapter->netdev, 2034 adapter->num_tx_queues); 2035 if (retval) 2036 goto failed; 2037 retval = netif_set_real_num_rx_queues(adapter->netdev, 2038 adapter->num_rx_queues); 2039 if (retval) 2040 goto failed; 2041 2042 tsnep_enable_irq(adapter, ECM_INT_LINK); 2043 retval = tsnep_phy_open(adapter); 2044 if (retval) 2045 goto phy_failed; 2046 2047 for (i = 0; i < adapter->num_queues; i++) 2048 tsnep_queue_enable(&adapter->queue[i]); 2049 2050 return 0; 2051 2052 phy_failed: 2053 tsnep_disable_irq(adapter, ECM_INT_LINK); 2054 failed: 2055 for (i = 0; i < adapter->num_queues; i++) { 2056 tsnep_queue_close(&adapter->queue[i], i == 0); 2057 2058 if (adapter->queue[i].rx) 2059 tsnep_rx_close(adapter->queue[i].rx); 2060 if (adapter->queue[i].tx) 2061 tsnep_tx_close(adapter->queue[i].tx); 2062 } 2063 return retval; 2064 } 2065 2066 static int tsnep_netdev_close(struct net_device *netdev) 2067 { 2068 struct tsnep_adapter *adapter = netdev_priv(netdev); 2069 int i; 2070 2071 tsnep_disable_irq(adapter, ECM_INT_LINK); 2072 tsnep_phy_close(adapter); 2073 2074 for (i = 0; i < adapter->num_queues; i++) { 2075 tsnep_queue_disable(&adapter->queue[i]); 2076 2077 tsnep_queue_close(&adapter->queue[i], i == 0); 2078 2079 if (adapter->queue[i].rx) 2080 tsnep_rx_close(adapter->queue[i].rx); 2081 if (adapter->queue[i].tx) 2082 tsnep_tx_close(adapter->queue[i].tx); 2083 } 2084 2085 return 0; 2086 } 2087 2088 int tsnep_enable_xsk(struct tsnep_queue *queue, struct xsk_buff_pool *pool) 2089 { 2090 bool running = netif_running(queue->adapter->netdev); 2091 u32 frame_size; 2092 2093 frame_size = xsk_pool_get_rx_frame_size(pool); 2094 if (frame_size < TSNEP_XSK_RX_BUF_SIZE) 2095 return -EOPNOTSUPP; 2096 2097 queue->rx->page_buffer = kcalloc(TSNEP_RING_SIZE, 2098 sizeof(*queue->rx->page_buffer), 2099 GFP_KERNEL); 2100 if (!queue->rx->page_buffer) 2101 return -ENOMEM; 2102 queue->rx->xdp_batch = kcalloc(TSNEP_RING_SIZE, 2103 sizeof(*queue->rx->xdp_batch), 2104 GFP_KERNEL); 2105 if (!queue->rx->xdp_batch) { 2106 kfree(queue->rx->page_buffer); 2107 queue->rx->page_buffer = NULL; 2108 2109 return -ENOMEM; 2110 } 2111 2112 xsk_pool_set_rxq_info(pool, &queue->rx->xdp_rxq_zc); 2113 2114 if (running) 2115 tsnep_queue_disable(queue); 2116 2117 queue->tx->xsk_pool = pool; 2118 queue->rx->xsk_pool = pool; 2119 2120 if (running) { 2121 tsnep_rx_reopen_xsk(queue->rx); 2122 tsnep_queue_enable(queue); 2123 } 2124 2125 return 0; 2126 } 2127 2128 void tsnep_disable_xsk(struct tsnep_queue *queue) 2129 { 2130 bool running = netif_running(queue->adapter->netdev); 2131 2132 if (running) 2133 tsnep_queue_disable(queue); 2134 2135 tsnep_rx_free_zc(queue->rx); 2136 2137 queue->rx->xsk_pool = NULL; 2138 queue->tx->xsk_pool = NULL; 2139 2140 if (running) { 2141 tsnep_rx_reopen(queue->rx); 2142 tsnep_queue_enable(queue); 2143 } 2144 2145 kfree(queue->rx->xdp_batch); 2146 queue->rx->xdp_batch = NULL; 2147 kfree(queue->rx->page_buffer); 2148 queue->rx->page_buffer = NULL; 2149 } 2150 2151 static netdev_tx_t tsnep_netdev_xmit_frame(struct sk_buff *skb, 2152 struct net_device *netdev) 2153 { 2154 struct tsnep_adapter *adapter = netdev_priv(netdev); 2155 u16 queue_mapping = skb_get_queue_mapping(skb); 2156 2157 if (queue_mapping >= adapter->num_tx_queues) 2158 queue_mapping = 0; 2159 2160 return tsnep_xmit_frame_ring(skb, &adapter->tx[queue_mapping]); 2161 } 2162 2163 static int tsnep_netdev_ioctl(struct net_device *netdev, struct ifreq *ifr, 2164 int cmd) 2165 { 2166 if (!netif_running(netdev)) 2167 return -EINVAL; 2168 if (cmd == SIOCSHWTSTAMP || cmd == SIOCGHWTSTAMP) 2169 return tsnep_ptp_ioctl(netdev, ifr, cmd); 2170 return phy_mii_ioctl(netdev->phydev, ifr, cmd); 2171 } 2172 2173 static void tsnep_netdev_set_multicast(struct net_device *netdev) 2174 { 2175 struct tsnep_adapter *adapter = netdev_priv(netdev); 2176 2177 u16 rx_filter = 0; 2178 2179 /* configured MAC address and broadcasts are never filtered */ 2180 if (netdev->flags & IFF_PROMISC) { 2181 rx_filter |= TSNEP_RX_FILTER_ACCEPT_ALL_MULTICASTS; 2182 rx_filter |= TSNEP_RX_FILTER_ACCEPT_ALL_UNICASTS; 2183 } else if (!netdev_mc_empty(netdev) || (netdev->flags & IFF_ALLMULTI)) { 2184 rx_filter |= TSNEP_RX_FILTER_ACCEPT_ALL_MULTICASTS; 2185 } 2186 iowrite16(rx_filter, adapter->addr + TSNEP_RX_FILTER); 2187 } 2188 2189 static void tsnep_netdev_get_stats64(struct net_device *netdev, 2190 struct rtnl_link_stats64 *stats) 2191 { 2192 struct tsnep_adapter *adapter = netdev_priv(netdev); 2193 u32 reg; 2194 u32 val; 2195 int i; 2196 2197 for (i = 0; i < adapter->num_tx_queues; i++) { 2198 stats->tx_packets += adapter->tx[i].packets; 2199 stats->tx_bytes += adapter->tx[i].bytes; 2200 stats->tx_dropped += adapter->tx[i].dropped; 2201 } 2202 for (i = 0; i < adapter->num_rx_queues; i++) { 2203 stats->rx_packets += adapter->rx[i].packets; 2204 stats->rx_bytes += adapter->rx[i].bytes; 2205 stats->rx_dropped += adapter->rx[i].dropped; 2206 stats->multicast += adapter->rx[i].multicast; 2207 2208 reg = ioread32(adapter->addr + TSNEP_QUEUE(i) + 2209 TSNEP_RX_STATISTIC); 2210 val = (reg & TSNEP_RX_STATISTIC_NO_DESC_MASK) >> 2211 TSNEP_RX_STATISTIC_NO_DESC_SHIFT; 2212 stats->rx_dropped += val; 2213 val = (reg & TSNEP_RX_STATISTIC_BUFFER_TOO_SMALL_MASK) >> 2214 TSNEP_RX_STATISTIC_BUFFER_TOO_SMALL_SHIFT; 2215 stats->rx_dropped += val; 2216 val = (reg & TSNEP_RX_STATISTIC_FIFO_OVERFLOW_MASK) >> 2217 TSNEP_RX_STATISTIC_FIFO_OVERFLOW_SHIFT; 2218 stats->rx_errors += val; 2219 stats->rx_fifo_errors += val; 2220 val = (reg & TSNEP_RX_STATISTIC_INVALID_FRAME_MASK) >> 2221 TSNEP_RX_STATISTIC_INVALID_FRAME_SHIFT; 2222 stats->rx_errors += val; 2223 stats->rx_frame_errors += val; 2224 } 2225 2226 reg = ioread32(adapter->addr + ECM_STAT); 2227 val = (reg & ECM_STAT_RX_ERR_MASK) >> ECM_STAT_RX_ERR_SHIFT; 2228 stats->rx_errors += val; 2229 val = (reg & ECM_STAT_INV_FRM_MASK) >> ECM_STAT_INV_FRM_SHIFT; 2230 stats->rx_errors += val; 2231 stats->rx_crc_errors += val; 2232 val = (reg & ECM_STAT_FWD_RX_ERR_MASK) >> ECM_STAT_FWD_RX_ERR_SHIFT; 2233 stats->rx_errors += val; 2234 } 2235 2236 static void tsnep_mac_set_address(struct tsnep_adapter *adapter, u8 *addr) 2237 { 2238 iowrite32(*(u32 *)addr, adapter->addr + TSNEP_MAC_ADDRESS_LOW); 2239 iowrite16(*(u16 *)(addr + sizeof(u32)), 2240 adapter->addr + TSNEP_MAC_ADDRESS_HIGH); 2241 2242 ether_addr_copy(adapter->mac_address, addr); 2243 netif_info(adapter, drv, adapter->netdev, "MAC address set to %pM\n", 2244 addr); 2245 } 2246 2247 static int tsnep_netdev_set_mac_address(struct net_device *netdev, void *addr) 2248 { 2249 struct tsnep_adapter *adapter = netdev_priv(netdev); 2250 struct sockaddr *sock_addr = addr; 2251 int retval; 2252 2253 retval = eth_prepare_mac_addr_change(netdev, sock_addr); 2254 if (retval) 2255 return retval; 2256 eth_hw_addr_set(netdev, sock_addr->sa_data); 2257 tsnep_mac_set_address(adapter, sock_addr->sa_data); 2258 2259 return 0; 2260 } 2261 2262 static int tsnep_netdev_set_features(struct net_device *netdev, 2263 netdev_features_t features) 2264 { 2265 struct tsnep_adapter *adapter = netdev_priv(netdev); 2266 netdev_features_t changed = netdev->features ^ features; 2267 bool enable; 2268 int retval = 0; 2269 2270 if (changed & NETIF_F_LOOPBACK) { 2271 enable = !!(features & NETIF_F_LOOPBACK); 2272 retval = tsnep_phy_loopback(adapter, enable); 2273 } 2274 2275 return retval; 2276 } 2277 2278 static ktime_t tsnep_netdev_get_tstamp(struct net_device *netdev, 2279 const struct skb_shared_hwtstamps *hwtstamps, 2280 bool cycles) 2281 { 2282 struct tsnep_rx_inline *rx_inline = hwtstamps->netdev_data; 2283 u64 timestamp; 2284 2285 if (cycles) 2286 timestamp = __le64_to_cpu(rx_inline->counter); 2287 else 2288 timestamp = __le64_to_cpu(rx_inline->timestamp); 2289 2290 return ns_to_ktime(timestamp); 2291 } 2292 2293 static int tsnep_netdev_bpf(struct net_device *dev, struct netdev_bpf *bpf) 2294 { 2295 struct tsnep_adapter *adapter = netdev_priv(dev); 2296 2297 switch (bpf->command) { 2298 case XDP_SETUP_PROG: 2299 return tsnep_xdp_setup_prog(adapter, bpf->prog, bpf->extack); 2300 case XDP_SETUP_XSK_POOL: 2301 return tsnep_xdp_setup_pool(adapter, bpf->xsk.pool, 2302 bpf->xsk.queue_id); 2303 default: 2304 return -EOPNOTSUPP; 2305 } 2306 } 2307 2308 static struct tsnep_tx *tsnep_xdp_get_tx(struct tsnep_adapter *adapter, u32 cpu) 2309 { 2310 if (cpu >= TSNEP_MAX_QUEUES) 2311 cpu &= TSNEP_MAX_QUEUES - 1; 2312 2313 while (cpu >= adapter->num_tx_queues) 2314 cpu -= adapter->num_tx_queues; 2315 2316 return &adapter->tx[cpu]; 2317 } 2318 2319 static int tsnep_netdev_xdp_xmit(struct net_device *dev, int n, 2320 struct xdp_frame **xdp, u32 flags) 2321 { 2322 struct tsnep_adapter *adapter = netdev_priv(dev); 2323 u32 cpu = smp_processor_id(); 2324 struct netdev_queue *nq; 2325 struct tsnep_tx *tx; 2326 int nxmit; 2327 bool xmit; 2328 2329 if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK)) 2330 return -EINVAL; 2331 2332 tx = tsnep_xdp_get_tx(adapter, cpu); 2333 nq = netdev_get_tx_queue(adapter->netdev, tx->queue_index); 2334 2335 __netif_tx_lock(nq, cpu); 2336 2337 for (nxmit = 0; nxmit < n; nxmit++) { 2338 xmit = tsnep_xdp_xmit_frame_ring(xdp[nxmit], tx, 2339 TSNEP_TX_TYPE_XDP_NDO); 2340 if (!xmit) 2341 break; 2342 2343 /* avoid transmit queue timeout since we share it with the slow 2344 * path 2345 */ 2346 txq_trans_cond_update(nq); 2347 } 2348 2349 if (flags & XDP_XMIT_FLUSH) 2350 tsnep_xdp_xmit_flush(tx); 2351 2352 __netif_tx_unlock(nq); 2353 2354 return nxmit; 2355 } 2356 2357 static int tsnep_netdev_xsk_wakeup(struct net_device *dev, u32 queue_id, 2358 u32 flags) 2359 { 2360 struct tsnep_adapter *adapter = netdev_priv(dev); 2361 struct tsnep_queue *queue; 2362 2363 if (queue_id >= adapter->num_rx_queues || 2364 queue_id >= adapter->num_tx_queues) 2365 return -EINVAL; 2366 2367 queue = &adapter->queue[queue_id]; 2368 2369 if (!napi_if_scheduled_mark_missed(&queue->napi)) 2370 napi_schedule(&queue->napi); 2371 2372 return 0; 2373 } 2374 2375 static const struct net_device_ops tsnep_netdev_ops = { 2376 .ndo_open = tsnep_netdev_open, 2377 .ndo_stop = tsnep_netdev_close, 2378 .ndo_start_xmit = tsnep_netdev_xmit_frame, 2379 .ndo_eth_ioctl = tsnep_netdev_ioctl, 2380 .ndo_set_rx_mode = tsnep_netdev_set_multicast, 2381 .ndo_get_stats64 = tsnep_netdev_get_stats64, 2382 .ndo_set_mac_address = tsnep_netdev_set_mac_address, 2383 .ndo_set_features = tsnep_netdev_set_features, 2384 .ndo_get_tstamp = tsnep_netdev_get_tstamp, 2385 .ndo_setup_tc = tsnep_tc_setup, 2386 .ndo_bpf = tsnep_netdev_bpf, 2387 .ndo_xdp_xmit = tsnep_netdev_xdp_xmit, 2388 .ndo_xsk_wakeup = tsnep_netdev_xsk_wakeup, 2389 }; 2390 2391 static int tsnep_mac_init(struct tsnep_adapter *adapter) 2392 { 2393 int retval; 2394 2395 /* initialize RX filtering, at least configured MAC address and 2396 * broadcast are not filtered 2397 */ 2398 iowrite16(0, adapter->addr + TSNEP_RX_FILTER); 2399 2400 /* try to get MAC address in the following order: 2401 * - device tree 2402 * - valid MAC address already set 2403 * - MAC address register if valid 2404 * - random MAC address 2405 */ 2406 retval = of_get_mac_address(adapter->pdev->dev.of_node, 2407 adapter->mac_address); 2408 if (retval == -EPROBE_DEFER) 2409 return retval; 2410 if (retval && !is_valid_ether_addr(adapter->mac_address)) { 2411 *(u32 *)adapter->mac_address = 2412 ioread32(adapter->addr + TSNEP_MAC_ADDRESS_LOW); 2413 *(u16 *)(adapter->mac_address + sizeof(u32)) = 2414 ioread16(adapter->addr + TSNEP_MAC_ADDRESS_HIGH); 2415 if (!is_valid_ether_addr(adapter->mac_address)) 2416 eth_random_addr(adapter->mac_address); 2417 } 2418 2419 tsnep_mac_set_address(adapter, adapter->mac_address); 2420 eth_hw_addr_set(adapter->netdev, adapter->mac_address); 2421 2422 return 0; 2423 } 2424 2425 static int tsnep_mdio_init(struct tsnep_adapter *adapter) 2426 { 2427 struct device_node *np = adapter->pdev->dev.of_node; 2428 int retval; 2429 2430 if (np) { 2431 np = of_get_child_by_name(np, "mdio"); 2432 if (!np) 2433 return 0; 2434 2435 adapter->suppress_preamble = 2436 of_property_read_bool(np, "suppress-preamble"); 2437 } 2438 2439 adapter->mdiobus = devm_mdiobus_alloc(&adapter->pdev->dev); 2440 if (!adapter->mdiobus) { 2441 retval = -ENOMEM; 2442 2443 goto out; 2444 } 2445 2446 adapter->mdiobus->priv = (void *)adapter; 2447 adapter->mdiobus->parent = &adapter->pdev->dev; 2448 adapter->mdiobus->read = tsnep_mdiobus_read; 2449 adapter->mdiobus->write = tsnep_mdiobus_write; 2450 adapter->mdiobus->name = TSNEP "-mdiobus"; 2451 snprintf(adapter->mdiobus->id, MII_BUS_ID_SIZE, "%s", 2452 adapter->pdev->name); 2453 2454 /* do not scan broadcast address */ 2455 adapter->mdiobus->phy_mask = 0x0000001; 2456 2457 retval = of_mdiobus_register(adapter->mdiobus, np); 2458 2459 out: 2460 of_node_put(np); 2461 2462 return retval; 2463 } 2464 2465 static int tsnep_phy_init(struct tsnep_adapter *adapter) 2466 { 2467 struct device_node *phy_node; 2468 int retval; 2469 2470 retval = of_get_phy_mode(adapter->pdev->dev.of_node, 2471 &adapter->phy_mode); 2472 if (retval) 2473 adapter->phy_mode = PHY_INTERFACE_MODE_GMII; 2474 2475 phy_node = of_parse_phandle(adapter->pdev->dev.of_node, "phy-handle", 2476 0); 2477 adapter->phydev = of_phy_find_device(phy_node); 2478 of_node_put(phy_node); 2479 if (!adapter->phydev && adapter->mdiobus) 2480 adapter->phydev = phy_find_first(adapter->mdiobus); 2481 if (!adapter->phydev) 2482 return -EIO; 2483 2484 return 0; 2485 } 2486 2487 static int tsnep_queue_init(struct tsnep_adapter *adapter, int queue_count) 2488 { 2489 u32 irq_mask = ECM_INT_TX_0 | ECM_INT_RX_0; 2490 char name[8]; 2491 int i; 2492 int retval; 2493 2494 /* one TX/RX queue pair for netdev is mandatory */ 2495 if (platform_irq_count(adapter->pdev) == 1) 2496 retval = platform_get_irq(adapter->pdev, 0); 2497 else 2498 retval = platform_get_irq_byname(adapter->pdev, "mac"); 2499 if (retval < 0) 2500 return retval; 2501 adapter->num_tx_queues = 1; 2502 adapter->num_rx_queues = 1; 2503 adapter->num_queues = 1; 2504 adapter->queue[0].adapter = adapter; 2505 adapter->queue[0].irq = retval; 2506 adapter->queue[0].tx = &adapter->tx[0]; 2507 adapter->queue[0].tx->adapter = adapter; 2508 adapter->queue[0].tx->addr = adapter->addr + TSNEP_QUEUE(0); 2509 adapter->queue[0].tx->queue_index = 0; 2510 adapter->queue[0].rx = &adapter->rx[0]; 2511 adapter->queue[0].rx->adapter = adapter; 2512 adapter->queue[0].rx->addr = adapter->addr + TSNEP_QUEUE(0); 2513 adapter->queue[0].rx->queue_index = 0; 2514 adapter->queue[0].irq_mask = irq_mask; 2515 adapter->queue[0].irq_delay_addr = adapter->addr + ECM_INT_DELAY; 2516 retval = tsnep_set_irq_coalesce(&adapter->queue[0], 2517 TSNEP_COALESCE_USECS_DEFAULT); 2518 if (retval < 0) 2519 return retval; 2520 2521 adapter->netdev->irq = adapter->queue[0].irq; 2522 2523 /* add additional TX/RX queue pairs only if dedicated interrupt is 2524 * available 2525 */ 2526 for (i = 1; i < queue_count; i++) { 2527 sprintf(name, "txrx-%d", i); 2528 retval = platform_get_irq_byname_optional(adapter->pdev, name); 2529 if (retval < 0) 2530 break; 2531 2532 adapter->num_tx_queues++; 2533 adapter->num_rx_queues++; 2534 adapter->num_queues++; 2535 adapter->queue[i].adapter = adapter; 2536 adapter->queue[i].irq = retval; 2537 adapter->queue[i].tx = &adapter->tx[i]; 2538 adapter->queue[i].tx->adapter = adapter; 2539 adapter->queue[i].tx->addr = adapter->addr + TSNEP_QUEUE(i); 2540 adapter->queue[i].tx->queue_index = i; 2541 adapter->queue[i].rx = &adapter->rx[i]; 2542 adapter->queue[i].rx->adapter = adapter; 2543 adapter->queue[i].rx->addr = adapter->addr + TSNEP_QUEUE(i); 2544 adapter->queue[i].rx->queue_index = i; 2545 adapter->queue[i].irq_mask = 2546 irq_mask << (ECM_INT_TXRX_SHIFT * i); 2547 adapter->queue[i].irq_delay_addr = 2548 adapter->addr + ECM_INT_DELAY + ECM_INT_DELAY_OFFSET * i; 2549 retval = tsnep_set_irq_coalesce(&adapter->queue[i], 2550 TSNEP_COALESCE_USECS_DEFAULT); 2551 if (retval < 0) 2552 return retval; 2553 } 2554 2555 return 0; 2556 } 2557 2558 static int tsnep_probe(struct platform_device *pdev) 2559 { 2560 struct tsnep_adapter *adapter; 2561 struct net_device *netdev; 2562 struct resource *io; 2563 u32 type; 2564 int revision; 2565 int version; 2566 int queue_count; 2567 int retval; 2568 2569 netdev = devm_alloc_etherdev_mqs(&pdev->dev, 2570 sizeof(struct tsnep_adapter), 2571 TSNEP_MAX_QUEUES, TSNEP_MAX_QUEUES); 2572 if (!netdev) 2573 return -ENODEV; 2574 SET_NETDEV_DEV(netdev, &pdev->dev); 2575 adapter = netdev_priv(netdev); 2576 platform_set_drvdata(pdev, adapter); 2577 adapter->pdev = pdev; 2578 adapter->dmadev = &pdev->dev; 2579 adapter->netdev = netdev; 2580 adapter->msg_enable = NETIF_MSG_DRV | NETIF_MSG_PROBE | 2581 NETIF_MSG_LINK | NETIF_MSG_IFUP | 2582 NETIF_MSG_IFDOWN | NETIF_MSG_TX_QUEUED; 2583 2584 netdev->min_mtu = ETH_MIN_MTU; 2585 netdev->max_mtu = TSNEP_MAX_FRAME_SIZE; 2586 2587 mutex_init(&adapter->gate_control_lock); 2588 mutex_init(&adapter->rxnfc_lock); 2589 INIT_LIST_HEAD(&adapter->rxnfc_rules); 2590 2591 adapter->addr = devm_platform_get_and_ioremap_resource(pdev, 0, &io); 2592 if (IS_ERR(adapter->addr)) 2593 return PTR_ERR(adapter->addr); 2594 netdev->mem_start = io->start; 2595 netdev->mem_end = io->end; 2596 2597 type = ioread32(adapter->addr + ECM_TYPE); 2598 revision = (type & ECM_REVISION_MASK) >> ECM_REVISION_SHIFT; 2599 version = (type & ECM_VERSION_MASK) >> ECM_VERSION_SHIFT; 2600 queue_count = (type & ECM_QUEUE_COUNT_MASK) >> ECM_QUEUE_COUNT_SHIFT; 2601 adapter->gate_control = type & ECM_GATE_CONTROL; 2602 adapter->rxnfc_max = TSNEP_RX_ASSIGN_ETHER_TYPE_COUNT; 2603 2604 tsnep_disable_irq(adapter, ECM_INT_ALL); 2605 2606 retval = tsnep_queue_init(adapter, queue_count); 2607 if (retval) 2608 return retval; 2609 2610 retval = dma_set_mask_and_coherent(&adapter->pdev->dev, 2611 DMA_BIT_MASK(64)); 2612 if (retval) { 2613 dev_err(&adapter->pdev->dev, "no usable DMA configuration.\n"); 2614 return retval; 2615 } 2616 2617 retval = tsnep_mac_init(adapter); 2618 if (retval) 2619 return retval; 2620 2621 retval = tsnep_mdio_init(adapter); 2622 if (retval) 2623 goto mdio_init_failed; 2624 2625 retval = tsnep_phy_init(adapter); 2626 if (retval) 2627 goto phy_init_failed; 2628 2629 retval = tsnep_ptp_init(adapter); 2630 if (retval) 2631 goto ptp_init_failed; 2632 2633 retval = tsnep_tc_init(adapter); 2634 if (retval) 2635 goto tc_init_failed; 2636 2637 retval = tsnep_rxnfc_init(adapter); 2638 if (retval) 2639 goto rxnfc_init_failed; 2640 2641 netdev->netdev_ops = &tsnep_netdev_ops; 2642 netdev->ethtool_ops = &tsnep_ethtool_ops; 2643 netdev->features = NETIF_F_SG; 2644 netdev->hw_features = netdev->features | NETIF_F_LOOPBACK; 2645 2646 netdev->xdp_features = NETDEV_XDP_ACT_BASIC | NETDEV_XDP_ACT_REDIRECT | 2647 NETDEV_XDP_ACT_NDO_XMIT | 2648 NETDEV_XDP_ACT_NDO_XMIT_SG | 2649 NETDEV_XDP_ACT_XSK_ZEROCOPY; 2650 2651 /* carrier off reporting is important to ethtool even BEFORE open */ 2652 netif_carrier_off(netdev); 2653 2654 retval = register_netdev(netdev); 2655 if (retval) 2656 goto register_failed; 2657 2658 dev_info(&adapter->pdev->dev, "device version %d.%02d\n", version, 2659 revision); 2660 if (adapter->gate_control) 2661 dev_info(&adapter->pdev->dev, "gate control detected\n"); 2662 2663 return 0; 2664 2665 register_failed: 2666 tsnep_rxnfc_cleanup(adapter); 2667 rxnfc_init_failed: 2668 tsnep_tc_cleanup(adapter); 2669 tc_init_failed: 2670 tsnep_ptp_cleanup(adapter); 2671 ptp_init_failed: 2672 phy_init_failed: 2673 if (adapter->mdiobus) 2674 mdiobus_unregister(adapter->mdiobus); 2675 mdio_init_failed: 2676 return retval; 2677 } 2678 2679 static void tsnep_remove(struct platform_device *pdev) 2680 { 2681 struct tsnep_adapter *adapter = platform_get_drvdata(pdev); 2682 2683 unregister_netdev(adapter->netdev); 2684 2685 tsnep_rxnfc_cleanup(adapter); 2686 2687 tsnep_tc_cleanup(adapter); 2688 2689 tsnep_ptp_cleanup(adapter); 2690 2691 if (adapter->mdiobus) 2692 mdiobus_unregister(adapter->mdiobus); 2693 2694 tsnep_disable_irq(adapter, ECM_INT_ALL); 2695 } 2696 2697 static const struct of_device_id tsnep_of_match[] = { 2698 { .compatible = "engleder,tsnep", }, 2699 { }, 2700 }; 2701 MODULE_DEVICE_TABLE(of, tsnep_of_match); 2702 2703 static struct platform_driver tsnep_driver = { 2704 .driver = { 2705 .name = TSNEP, 2706 .of_match_table = tsnep_of_match, 2707 }, 2708 .probe = tsnep_probe, 2709 .remove = tsnep_remove, 2710 }; 2711 module_platform_driver(tsnep_driver); 2712 2713 MODULE_AUTHOR("Gerhard Engleder <gerhard@engleder-embedded.com>"); 2714 MODULE_DESCRIPTION("TSN endpoint Ethernet MAC driver"); 2715 MODULE_LICENSE("GPL"); 2716