1 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause 2 /* 3 * rtase is the Linux device driver released for Realtek Automotive Switch 4 * controllers with PCI-Express interface. 5 * 6 * Copyright(c) 2024 Realtek Semiconductor Corp. 7 * 8 * Below is a simplified block diagram of the chip and its relevant interfaces. 9 * 10 * ************************* 11 * * * 12 * * CPU network device * 13 * * * 14 * * +-------------+ * 15 * * | PCIE Host | * 16 * ***********++************ 17 * || 18 * PCIE 19 * || 20 * ********************++********************** 21 * * | PCIE Endpoint | * 22 * * +---------------+ * 23 * * | GMAC | * 24 * * +--++--+ Realtek * 25 * * || RTL90xx Series * 26 * * || * 27 * * +-------------++----------------+ * 28 * * | | MAC | | * 29 * * | +-----+ | * 30 * * | | * 31 * * | Ethernet Switch Core | * 32 * * | | * 33 * * | +-----+ +-----+ | * 34 * * | | MAC |...........| MAC | | * 35 * * +---+-----+-----------+-----+---+ * 36 * * | PHY |...........| PHY | * 37 * * +--++-+ +--++-+ * 38 * *************||****************||*********** 39 * 40 * The block of the Realtek RTL90xx series is our entire chip architecture, 41 * the GMAC is connected to the switch core, and there is no PHY in between. 42 * In addition, this driver is mainly used to control GMAC, but does not 43 * control the switch core, so it is not the same as DSA. Linux only plays 44 * the role of a normal leaf node in this model. 45 */ 46 47 #include <linux/crc32.h> 48 #include <linux/dma-mapping.h> 49 #include <linux/etherdevice.h> 50 #include <linux/if_vlan.h> 51 #include <linux/in.h> 52 #include <linux/init.h> 53 #include <linux/interrupt.h> 54 #include <linux/io.h> 55 #include <linux/iopoll.h> 56 #include <linux/ip.h> 57 #include <linux/ipv6.h> 58 #include <linux/mdio.h> 59 #include <linux/module.h> 60 #include <linux/netdevice.h> 61 #include <linux/pci.h> 62 #include <linux/pm_runtime.h> 63 #include <linux/prefetch.h> 64 #include <linux/rtnetlink.h> 65 #include <linux/tcp.h> 66 #include <asm/irq.h> 67 #include <net/ip6_checksum.h> 68 #include <net/netdev_queues.h> 69 #include <net/page_pool/helpers.h> 70 #include <net/pkt_cls.h> 71 72 #include "rtase.h" 73 74 #define RTK_OPTS1_DEBUG_VALUE 0x0BADBEEF 75 #define RTK_MAGIC_NUMBER 0x0BADBADBADBADBAD 76 77 static const struct pci_device_id rtase_pci_tbl[] = { 78 {PCI_VDEVICE(REALTEK, 0x906A)}, 79 {} 80 }; 81 82 MODULE_DEVICE_TABLE(pci, rtase_pci_tbl); 83 84 MODULE_AUTHOR("Realtek ARD Software Team"); 85 MODULE_DESCRIPTION("Network Driver for the PCIe interface of Realtek Automotive Ethernet Switch"); 86 MODULE_LICENSE("Dual BSD/GPL"); 87 88 struct rtase_counters { 89 __le64 tx_packets; 90 __le64 rx_packets; 91 __le64 tx_errors; 92 __le32 rx_errors; 93 __le16 rx_missed; 94 __le16 align_errors; 95 __le32 tx_one_collision; 96 __le32 tx_multi_collision; 97 __le64 rx_unicast; 98 __le64 rx_broadcast; 99 __le32 rx_multicast; 100 __le16 tx_aborted; 101 __le16 tx_underrun; 102 } __packed; 103 104 static void rtase_w8(const struct rtase_private *tp, u16 reg, u8 val8) 105 { 106 writeb(val8, tp->mmio_addr + reg); 107 } 108 109 static void rtase_w16(const struct rtase_private *tp, u16 reg, u16 val16) 110 { 111 writew(val16, tp->mmio_addr + reg); 112 } 113 114 static void rtase_w32(const struct rtase_private *tp, u16 reg, u32 val32) 115 { 116 writel(val32, tp->mmio_addr + reg); 117 } 118 119 static u8 rtase_r8(const struct rtase_private *tp, u16 reg) 120 { 121 return readb(tp->mmio_addr + reg); 122 } 123 124 static u16 rtase_r16(const struct rtase_private *tp, u16 reg) 125 { 126 return readw(tp->mmio_addr + reg); 127 } 128 129 static u32 rtase_r32(const struct rtase_private *tp, u16 reg) 130 { 131 return readl(tp->mmio_addr + reg); 132 } 133 134 static void rtase_free_desc(struct rtase_private *tp) 135 { 136 struct pci_dev *pdev = tp->pdev; 137 u32 i; 138 139 for (i = 0; i < tp->func_tx_queue_num; i++) { 140 if (!tp->tx_ring[i].desc) 141 continue; 142 143 dma_free_coherent(&pdev->dev, RTASE_TX_RING_DESC_SIZE, 144 tp->tx_ring[i].desc, 145 tp->tx_ring[i].phy_addr); 146 tp->tx_ring[i].desc = NULL; 147 } 148 149 for (i = 0; i < tp->func_rx_queue_num; i++) { 150 if (!tp->rx_ring[i].desc) 151 continue; 152 153 dma_free_coherent(&pdev->dev, RTASE_RX_RING_DESC_SIZE, 154 tp->rx_ring[i].desc, 155 tp->rx_ring[i].phy_addr); 156 tp->rx_ring[i].desc = NULL; 157 } 158 } 159 160 static int rtase_alloc_desc(struct rtase_private *tp) 161 { 162 struct pci_dev *pdev = tp->pdev; 163 u32 i; 164 165 /* rx and tx descriptors needs 256 bytes alignment. 166 * dma_alloc_coherent provides more. 167 */ 168 for (i = 0; i < tp->func_tx_queue_num; i++) { 169 tp->tx_ring[i].desc = 170 dma_alloc_coherent(&pdev->dev, 171 RTASE_TX_RING_DESC_SIZE, 172 &tp->tx_ring[i].phy_addr, 173 GFP_KERNEL); 174 if (!tp->tx_ring[i].desc) 175 goto err_out; 176 } 177 178 for (i = 0; i < tp->func_rx_queue_num; i++) { 179 tp->rx_ring[i].desc = 180 dma_alloc_coherent(&pdev->dev, 181 RTASE_RX_RING_DESC_SIZE, 182 &tp->rx_ring[i].phy_addr, 183 GFP_KERNEL); 184 if (!tp->rx_ring[i].desc) 185 goto err_out; 186 } 187 188 return 0; 189 190 err_out: 191 rtase_free_desc(tp); 192 return -ENOMEM; 193 } 194 195 static void rtase_unmap_tx_skb(struct pci_dev *pdev, u32 len, 196 struct rtase_tx_desc *desc) 197 { 198 dma_unmap_single(&pdev->dev, le64_to_cpu(desc->addr), len, 199 DMA_TO_DEVICE); 200 desc->opts1 = cpu_to_le32(RTK_OPTS1_DEBUG_VALUE); 201 desc->opts2 = 0x00; 202 desc->addr = cpu_to_le64(RTK_MAGIC_NUMBER); 203 } 204 205 static void rtase_tx_clear_range(struct rtase_ring *ring, u32 start, u32 n) 206 { 207 struct rtase_tx_desc *desc_base = ring->desc; 208 struct rtase_private *tp = ring->ivec->tp; 209 u32 i; 210 211 for (i = 0; i < n; i++) { 212 u32 entry = (start + i) % RTASE_NUM_DESC; 213 struct rtase_tx_desc *desc = desc_base + entry; 214 u32 len = ring->mis.len[entry]; 215 struct sk_buff *skb; 216 217 if (len == 0) 218 continue; 219 220 rtase_unmap_tx_skb(tp->pdev, len, desc); 221 ring->mis.len[entry] = 0; 222 skb = ring->skbuff[entry]; 223 if (!skb) 224 continue; 225 226 tp->stats.tx_dropped++; 227 dev_kfree_skb_any(skb); 228 ring->skbuff[entry] = NULL; 229 } 230 } 231 232 static void rtase_tx_clear(struct rtase_private *tp) 233 { 234 struct rtase_ring *ring; 235 u16 i; 236 237 for (i = 0; i < tp->func_tx_queue_num; i++) { 238 ring = &tp->tx_ring[i]; 239 rtase_tx_clear_range(ring, ring->dirty_idx, RTASE_NUM_DESC); 240 ring->cur_idx = 0; 241 ring->dirty_idx = 0; 242 243 netdev_tx_reset_subqueue(tp->dev, i); 244 } 245 } 246 247 static void rtase_mark_to_asic(union rtase_rx_desc *desc, u32 rx_buf_sz) 248 { 249 u32 eor = le32_to_cpu(desc->desc_cmd.opts1) & RTASE_RING_END; 250 251 desc->desc_status.opts2 = 0; 252 /* force memory writes to complete before releasing descriptor */ 253 dma_wmb(); 254 WRITE_ONCE(desc->desc_cmd.opts1, 255 cpu_to_le32(RTASE_DESC_OWN | eor | rx_buf_sz)); 256 } 257 258 static u32 rtase_tx_avail(struct rtase_ring *ring) 259 { 260 return READ_ONCE(ring->dirty_idx) + RTASE_NUM_DESC - 261 READ_ONCE(ring->cur_idx); 262 } 263 264 static int tx_handler(struct rtase_ring *ring, int budget) 265 { 266 const struct rtase_private *tp = ring->ivec->tp; 267 struct net_device *dev = tp->dev; 268 u32 dirty_tx, tx_left; 269 u32 bytes_compl = 0; 270 u32 pkts_compl = 0; 271 int workdone = 0; 272 273 dirty_tx = ring->dirty_idx; 274 tx_left = READ_ONCE(ring->cur_idx) - dirty_tx; 275 276 while (tx_left > 0) { 277 u32 entry = dirty_tx % RTASE_NUM_DESC; 278 struct rtase_tx_desc *desc = ring->desc + 279 sizeof(struct rtase_tx_desc) * entry; 280 u32 status; 281 282 status = le32_to_cpu(desc->opts1); 283 284 if (status & RTASE_DESC_OWN) 285 break; 286 287 rtase_unmap_tx_skb(tp->pdev, ring->mis.len[entry], desc); 288 ring->mis.len[entry] = 0; 289 if (ring->skbuff[entry]) { 290 pkts_compl++; 291 bytes_compl += ring->skbuff[entry]->len; 292 napi_consume_skb(ring->skbuff[entry], budget); 293 ring->skbuff[entry] = NULL; 294 } 295 296 dirty_tx++; 297 tx_left--; 298 workdone++; 299 300 if (workdone == RTASE_TX_BUDGET_DEFAULT) 301 break; 302 } 303 304 if (ring->dirty_idx != dirty_tx) { 305 dev_sw_netstats_tx_add(dev, pkts_compl, bytes_compl); 306 WRITE_ONCE(ring->dirty_idx, dirty_tx); 307 308 netif_subqueue_completed_wake(dev, ring->index, pkts_compl, 309 bytes_compl, 310 rtase_tx_avail(ring), 311 RTASE_TX_START_THRS); 312 313 if (ring->cur_idx != dirty_tx) 314 rtase_w8(tp, RTASE_TPPOLL, BIT(ring->index)); 315 } 316 317 return 0; 318 } 319 320 static void rtase_tx_desc_init(struct rtase_private *tp, u16 idx) 321 { 322 struct rtase_ring *ring = &tp->tx_ring[idx]; 323 struct rtase_tx_desc *desc; 324 u32 i; 325 326 memset(ring->desc, 0x0, RTASE_TX_RING_DESC_SIZE); 327 memset(ring->skbuff, 0x0, sizeof(ring->skbuff)); 328 ring->cur_idx = 0; 329 ring->dirty_idx = 0; 330 ring->index = idx; 331 ring->type = NETDEV_QUEUE_TYPE_TX; 332 ring->alloc_fail = 0; 333 334 for (i = 0; i < RTASE_NUM_DESC; i++) { 335 ring->mis.len[i] = 0; 336 if ((RTASE_NUM_DESC - 1) == i) { 337 desc = ring->desc + sizeof(struct rtase_tx_desc) * i; 338 desc->opts1 = cpu_to_le32(RTASE_RING_END); 339 } 340 } 341 342 ring->ring_handler = tx_handler; 343 if (idx < 4) { 344 ring->ivec = &tp->int_vector[idx]; 345 list_add_tail(&ring->ring_entry, 346 &tp->int_vector[idx].ring_list); 347 } else { 348 ring->ivec = &tp->int_vector[0]; 349 list_add_tail(&ring->ring_entry, &tp->int_vector[0].ring_list); 350 } 351 352 netif_queue_set_napi(tp->dev, ring->index, 353 ring->type, &ring->ivec->napi); 354 } 355 356 static void rtase_map_to_asic(union rtase_rx_desc *desc, dma_addr_t mapping, 357 u32 rx_buf_sz) 358 { 359 desc->desc_cmd.addr = cpu_to_le64(mapping); 360 361 rtase_mark_to_asic(desc, rx_buf_sz); 362 } 363 364 static void rtase_make_unusable_by_asic(union rtase_rx_desc *desc) 365 { 366 desc->desc_cmd.addr = cpu_to_le64(RTK_MAGIC_NUMBER); 367 desc->desc_cmd.opts1 &= ~cpu_to_le32(RTASE_DESC_OWN | RSVD_MASK); 368 } 369 370 static int rtase_alloc_rx_data_buf(struct rtase_ring *ring, 371 void **p_data_buf, 372 union rtase_rx_desc *desc, 373 dma_addr_t *rx_phy_addr) 374 { 375 struct rtase_int_vector *ivec = ring->ivec; 376 const struct rtase_private *tp = ivec->tp; 377 dma_addr_t mapping; 378 struct page *page; 379 380 page = page_pool_dev_alloc_pages(tp->page_pool); 381 if (!page) { 382 ring->alloc_fail++; 383 goto err_out; 384 } 385 386 *p_data_buf = page_address(page); 387 mapping = page_pool_get_dma_addr(page); 388 *rx_phy_addr = mapping; 389 rtase_map_to_asic(desc, mapping, tp->rx_buf_sz); 390 391 return 0; 392 393 err_out: 394 rtase_make_unusable_by_asic(desc); 395 396 return -ENOMEM; 397 } 398 399 static u32 rtase_rx_ring_fill(struct rtase_ring *ring, u32 ring_start, 400 u32 ring_end) 401 { 402 union rtase_rx_desc *desc_base = ring->desc; 403 u32 cur; 404 405 for (cur = ring_start; ring_end - cur > 0; cur++) { 406 u32 i = cur % RTASE_NUM_DESC; 407 union rtase_rx_desc *desc = desc_base + i; 408 int ret; 409 410 if (ring->data_buf[i]) 411 continue; 412 413 ret = rtase_alloc_rx_data_buf(ring, &ring->data_buf[i], desc, 414 &ring->mis.data_phy_addr[i]); 415 if (ret) 416 break; 417 } 418 419 return cur - ring_start; 420 } 421 422 static void rtase_mark_as_last_descriptor(union rtase_rx_desc *desc) 423 { 424 desc->desc_cmd.opts1 |= cpu_to_le32(RTASE_RING_END); 425 } 426 427 static void rtase_rx_ring_clear(struct page_pool *page_pool, 428 struct rtase_ring *ring) 429 { 430 union rtase_rx_desc *desc; 431 struct page *page; 432 u32 i; 433 434 for (i = 0; i < RTASE_NUM_DESC; i++) { 435 desc = ring->desc + sizeof(union rtase_rx_desc) * i; 436 page = virt_to_head_page(ring->data_buf[i]); 437 438 if (ring->data_buf[i]) 439 page_pool_put_full_page(page_pool, page, true); 440 441 rtase_make_unusable_by_asic(desc); 442 } 443 } 444 445 static int rtase_fragmented_frame(u32 status) 446 { 447 return (status & (RTASE_RX_FIRST_FRAG | RTASE_RX_LAST_FRAG)) != 448 (RTASE_RX_FIRST_FRAG | RTASE_RX_LAST_FRAG); 449 } 450 451 static void rtase_rx_csum(const struct rtase_private *tp, struct sk_buff *skb, 452 const union rtase_rx_desc *desc) 453 { 454 u32 opts2 = le32_to_cpu(desc->desc_status.opts2); 455 456 /* rx csum offload */ 457 if (((opts2 & RTASE_RX_V4F) && !(opts2 & RTASE_RX_IPF)) || 458 (opts2 & RTASE_RX_V6F)) { 459 if (((opts2 & RTASE_RX_TCPT) && !(opts2 & RTASE_RX_TCPF)) || 460 ((opts2 & RTASE_RX_UDPT) && !(opts2 & RTASE_RX_UDPF))) 461 skb->ip_summed = CHECKSUM_UNNECESSARY; 462 else 463 skb->ip_summed = CHECKSUM_NONE; 464 } else { 465 skb->ip_summed = CHECKSUM_NONE; 466 } 467 } 468 469 static void rtase_rx_vlan_skb(union rtase_rx_desc *desc, struct sk_buff *skb) 470 { 471 u32 opts2 = le32_to_cpu(desc->desc_status.opts2); 472 473 if (!(opts2 & RTASE_RX_VLAN_TAG)) 474 return; 475 476 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), 477 swab16(opts2 & RTASE_VLAN_TAG_MASK)); 478 } 479 480 static void rtase_rx_skb(const struct rtase_ring *ring, struct sk_buff *skb) 481 { 482 struct rtase_int_vector *ivec = ring->ivec; 483 484 napi_gro_receive(&ivec->napi, skb); 485 } 486 487 static int rx_handler(struct rtase_ring *ring, int budget) 488 { 489 union rtase_rx_desc *desc_base = ring->desc; 490 u32 pkt_size, cur_rx, delta, entry, status; 491 struct rtase_private *tp = ring->ivec->tp; 492 struct net_device *dev = tp->dev; 493 union rtase_rx_desc *desc; 494 struct sk_buff *skb; 495 int workdone = 0; 496 497 cur_rx = ring->cur_idx; 498 entry = cur_rx % RTASE_NUM_DESC; 499 desc = &desc_base[entry]; 500 501 while (workdone < budget) { 502 status = le32_to_cpu(desc->desc_status.opts1); 503 504 if (status & RTASE_DESC_OWN) 505 break; 506 507 /* This barrier is needed to keep us from reading 508 * any other fields out of the rx descriptor until 509 * we know the status of RTASE_DESC_OWN 510 */ 511 dma_rmb(); 512 513 if (unlikely(status & RTASE_RX_RES)) { 514 if (net_ratelimit()) 515 netdev_warn(dev, "Rx ERROR. status = %08x\n", 516 status); 517 518 tp->stats.rx_errors++; 519 520 if (status & (RTASE_RX_RWT | RTASE_RX_RUNT)) 521 tp->stats.rx_length_errors++; 522 523 if (status & RTASE_RX_CRC) 524 tp->stats.rx_crc_errors++; 525 526 if (dev->features & NETIF_F_RXALL) 527 goto process_pkt; 528 529 rtase_mark_to_asic(desc, tp->rx_buf_sz); 530 goto skip_process_pkt; 531 } 532 533 process_pkt: 534 pkt_size = status & RTASE_RX_PKT_SIZE_MASK; 535 if (likely(!(dev->features & NETIF_F_RXFCS))) 536 pkt_size -= ETH_FCS_LEN; 537 538 /* The driver does not support incoming fragmented frames. 539 * They are seen as a symptom of over-mtu sized frames. 540 */ 541 if (unlikely(rtase_fragmented_frame(status))) { 542 tp->stats.rx_dropped++; 543 tp->stats.rx_length_errors++; 544 rtase_mark_to_asic(desc, tp->rx_buf_sz); 545 goto skip_process_pkt; 546 } 547 548 dma_sync_single_for_cpu(&tp->pdev->dev, 549 ring->mis.data_phy_addr[entry], 550 tp->rx_buf_sz, DMA_FROM_DEVICE); 551 552 skb = build_skb(ring->data_buf[entry], PAGE_SIZE); 553 if (!skb) { 554 tp->stats.rx_dropped++; 555 rtase_mark_to_asic(desc, tp->rx_buf_sz); 556 goto skip_process_pkt; 557 } 558 ring->data_buf[entry] = NULL; 559 560 if (dev->features & NETIF_F_RXCSUM) 561 rtase_rx_csum(tp, skb, desc); 562 563 skb_put(skb, pkt_size); 564 skb_mark_for_recycle(skb); 565 skb->protocol = eth_type_trans(skb, dev); 566 567 if (skb->pkt_type == PACKET_MULTICAST) 568 tp->stats.multicast++; 569 570 rtase_rx_vlan_skb(desc, skb); 571 rtase_rx_skb(ring, skb); 572 573 dev_sw_netstats_rx_add(dev, pkt_size); 574 575 skip_process_pkt: 576 workdone++; 577 cur_rx++; 578 entry = cur_rx % RTASE_NUM_DESC; 579 desc = ring->desc + sizeof(union rtase_rx_desc) * entry; 580 } 581 582 ring->cur_idx = cur_rx; 583 delta = rtase_rx_ring_fill(ring, ring->dirty_idx, ring->cur_idx); 584 ring->dirty_idx += delta; 585 586 return workdone; 587 } 588 589 static void rtase_rx_desc_init(struct rtase_private *tp, u16 idx) 590 { 591 struct rtase_ring *ring = &tp->rx_ring[idx]; 592 u16 i; 593 594 memset(ring->desc, 0x0, RTASE_RX_RING_DESC_SIZE); 595 memset(ring->data_buf, 0x0, sizeof(ring->data_buf)); 596 ring->cur_idx = 0; 597 ring->dirty_idx = 0; 598 ring->index = idx; 599 ring->type = NETDEV_QUEUE_TYPE_RX; 600 ring->alloc_fail = 0; 601 602 for (i = 0; i < RTASE_NUM_DESC; i++) 603 ring->mis.data_phy_addr[i] = 0; 604 605 ring->ring_handler = rx_handler; 606 ring->ivec = &tp->int_vector[idx]; 607 netif_queue_set_napi(tp->dev, ring->index, 608 ring->type, &ring->ivec->napi); 609 list_add_tail(&ring->ring_entry, &tp->int_vector[idx].ring_list); 610 } 611 612 static void rtase_rx_clear(struct rtase_private *tp) 613 { 614 u32 i; 615 616 for (i = 0; i < tp->func_rx_queue_num; i++) 617 rtase_rx_ring_clear(tp->page_pool, &tp->rx_ring[i]); 618 619 page_pool_destroy(tp->page_pool); 620 tp->page_pool = NULL; 621 } 622 623 static int rtase_init_ring(const struct net_device *dev) 624 { 625 struct rtase_private *tp = netdev_priv(dev); 626 struct page_pool_params pp_params = { 0 }; 627 struct page_pool *page_pool; 628 u32 num; 629 u16 i; 630 631 pp_params.flags = PP_FLAG_DMA_MAP | PP_FLAG_DMA_SYNC_DEV; 632 pp_params.order = 0; 633 pp_params.pool_size = RTASE_NUM_DESC * tp->func_rx_queue_num; 634 pp_params.nid = dev_to_node(&tp->pdev->dev); 635 pp_params.dev = &tp->pdev->dev; 636 pp_params.dma_dir = DMA_FROM_DEVICE; 637 pp_params.max_len = PAGE_SIZE; 638 pp_params.offset = 0; 639 640 page_pool = page_pool_create(&pp_params); 641 if (IS_ERR(page_pool)) { 642 netdev_err(tp->dev, "failed to create page pool\n"); 643 return -ENOMEM; 644 } 645 646 tp->page_pool = page_pool; 647 648 for (i = 0; i < tp->func_tx_queue_num; i++) 649 rtase_tx_desc_init(tp, i); 650 651 for (i = 0; i < tp->func_rx_queue_num; i++) { 652 rtase_rx_desc_init(tp, i); 653 654 num = rtase_rx_ring_fill(&tp->rx_ring[i], 0, RTASE_NUM_DESC); 655 if (num != RTASE_NUM_DESC) 656 goto err_out; 657 658 rtase_mark_as_last_descriptor(tp->rx_ring[i].desc + 659 sizeof(union rtase_rx_desc) * 660 (RTASE_NUM_DESC - 1)); 661 } 662 663 return 0; 664 665 err_out: 666 rtase_rx_clear(tp); 667 return -ENOMEM; 668 } 669 670 static void rtase_interrupt_mitigation(const struct rtase_private *tp) 671 { 672 u32 i; 673 674 for (i = 0; i < tp->func_tx_queue_num; i++) 675 rtase_w16(tp, RTASE_INT_MITI_TX + i * 2, tp->tx_int_mit); 676 677 for (i = 0; i < tp->func_rx_queue_num; i++) 678 rtase_w16(tp, RTASE_INT_MITI_RX + i * 2, tp->rx_int_mit); 679 } 680 681 static void rtase_tally_counter_addr_fill(const struct rtase_private *tp) 682 { 683 rtase_w32(tp, RTASE_DTCCR4, upper_32_bits(tp->tally_paddr)); 684 rtase_w32(tp, RTASE_DTCCR0, lower_32_bits(tp->tally_paddr)); 685 } 686 687 static void rtase_tally_counter_clear(const struct rtase_private *tp) 688 { 689 u32 cmd = lower_32_bits(tp->tally_paddr); 690 691 rtase_w32(tp, RTASE_DTCCR4, upper_32_bits(tp->tally_paddr)); 692 rtase_w32(tp, RTASE_DTCCR0, cmd | RTASE_COUNTER_RESET); 693 } 694 695 static void rtase_desc_addr_fill(const struct rtase_private *tp) 696 { 697 const struct rtase_ring *ring; 698 u16 i, cmd, val; 699 int err; 700 701 for (i = 0; i < tp->func_tx_queue_num; i++) { 702 ring = &tp->tx_ring[i]; 703 704 rtase_w32(tp, RTASE_TX_DESC_ADDR0, 705 lower_32_bits(ring->phy_addr)); 706 rtase_w32(tp, RTASE_TX_DESC_ADDR4, 707 upper_32_bits(ring->phy_addr)); 708 709 cmd = i | RTASE_TX_DESC_CMD_WE | RTASE_TX_DESC_CMD_CS; 710 rtase_w16(tp, RTASE_TX_DESC_COMMAND, cmd); 711 712 err = read_poll_timeout(rtase_r16, val, 713 !(val & RTASE_TX_DESC_CMD_CS), 10, 714 1000, false, tp, 715 RTASE_TX_DESC_COMMAND); 716 717 if (err == -ETIMEDOUT) 718 netdev_err(tp->dev, 719 "error occurred in fill tx descriptor\n"); 720 } 721 722 for (i = 0; i < tp->func_rx_queue_num; i++) { 723 ring = &tp->rx_ring[i]; 724 725 if (i == 0) { 726 rtase_w32(tp, RTASE_Q0_RX_DESC_ADDR0, 727 lower_32_bits(ring->phy_addr)); 728 rtase_w32(tp, RTASE_Q0_RX_DESC_ADDR4, 729 upper_32_bits(ring->phy_addr)); 730 } else { 731 rtase_w32(tp, (RTASE_Q1_RX_DESC_ADDR0 + ((i - 1) * 8)), 732 lower_32_bits(ring->phy_addr)); 733 rtase_w32(tp, (RTASE_Q1_RX_DESC_ADDR4 + ((i - 1) * 8)), 734 upper_32_bits(ring->phy_addr)); 735 } 736 } 737 } 738 739 static void rtase_hw_set_features(const struct net_device *dev, 740 netdev_features_t features) 741 { 742 const struct rtase_private *tp = netdev_priv(dev); 743 u16 rx_config, val; 744 745 rx_config = rtase_r16(tp, RTASE_RX_CONFIG_0); 746 if (features & NETIF_F_RXALL) 747 rx_config |= (RTASE_ACCEPT_ERR | RTASE_ACCEPT_RUNT); 748 else 749 rx_config &= ~(RTASE_ACCEPT_ERR | RTASE_ACCEPT_RUNT); 750 751 rtase_w16(tp, RTASE_RX_CONFIG_0, rx_config); 752 753 val = rtase_r16(tp, RTASE_CPLUS_CMD); 754 if (features & NETIF_F_RXCSUM) 755 rtase_w16(tp, RTASE_CPLUS_CMD, val | RTASE_RX_CHKSUM); 756 else 757 rtase_w16(tp, RTASE_CPLUS_CMD, val & ~RTASE_RX_CHKSUM); 758 759 rx_config = rtase_r16(tp, RTASE_RX_CONFIG_1); 760 if (dev->features & NETIF_F_HW_VLAN_CTAG_RX) 761 rx_config |= (RTASE_INNER_VLAN_DETAG_EN | 762 RTASE_OUTER_VLAN_DETAG_EN); 763 else 764 rx_config &= ~(RTASE_INNER_VLAN_DETAG_EN | 765 RTASE_OUTER_VLAN_DETAG_EN); 766 767 rtase_w16(tp, RTASE_RX_CONFIG_1, rx_config); 768 } 769 770 static void rtase_hw_set_rx_packet_filter(struct net_device *dev) 771 { 772 u32 mc_filter[2] = { 0xFFFFFFFF, 0xFFFFFFFF }; 773 struct rtase_private *tp = netdev_priv(dev); 774 u16 rx_mode; 775 776 rx_mode = rtase_r16(tp, RTASE_RX_CONFIG_0) & ~RTASE_ACCEPT_MASK; 777 rx_mode |= RTASE_ACCEPT_BROADCAST | RTASE_ACCEPT_MYPHYS; 778 779 if (dev->flags & IFF_PROMISC) { 780 rx_mode |= RTASE_ACCEPT_MULTICAST | RTASE_ACCEPT_ALLPHYS; 781 } else if (dev->flags & IFF_ALLMULTI) { 782 rx_mode |= RTASE_ACCEPT_MULTICAST; 783 } else { 784 struct netdev_hw_addr *hw_addr; 785 786 mc_filter[0] = 0; 787 mc_filter[1] = 0; 788 789 netdev_for_each_mc_addr(hw_addr, dev) { 790 u32 bit_nr = eth_hw_addr_crc(hw_addr); 791 u32 idx = u32_get_bits(bit_nr, BIT(31)); 792 u32 bit = u32_get_bits(bit_nr, 793 RTASE_MULTICAST_FILTER_MASK); 794 795 mc_filter[idx] |= BIT(bit); 796 rx_mode |= RTASE_ACCEPT_MULTICAST; 797 } 798 } 799 800 if (dev->features & NETIF_F_RXALL) 801 rx_mode |= RTASE_ACCEPT_ERR | RTASE_ACCEPT_RUNT; 802 803 rtase_w32(tp, RTASE_MAR0, swab32(mc_filter[1])); 804 rtase_w32(tp, RTASE_MAR1, swab32(mc_filter[0])); 805 rtase_w16(tp, RTASE_RX_CONFIG_0, rx_mode); 806 } 807 808 static void rtase_irq_dis_and_clear(const struct rtase_private *tp) 809 { 810 const struct rtase_int_vector *ivec = &tp->int_vector[0]; 811 u32 val1; 812 u16 val2; 813 u8 i; 814 815 rtase_w32(tp, ivec->imr_addr, 0); 816 val1 = rtase_r32(tp, ivec->isr_addr); 817 rtase_w32(tp, ivec->isr_addr, val1); 818 819 for (i = 1; i < tp->int_nums; i++) { 820 ivec = &tp->int_vector[i]; 821 rtase_w16(tp, ivec->imr_addr, 0); 822 val2 = rtase_r16(tp, ivec->isr_addr); 823 rtase_w16(tp, ivec->isr_addr, val2); 824 } 825 } 826 827 static void rtase_poll_timeout(const struct rtase_private *tp, u32 cond, 828 u32 sleep_us, u64 timeout_us, u16 reg) 829 { 830 int err; 831 u8 val; 832 833 err = read_poll_timeout(rtase_r8, val, val & cond, sleep_us, 834 timeout_us, false, tp, reg); 835 836 if (err == -ETIMEDOUT) 837 netdev_err(tp->dev, "poll reg 0x00%x timeout\n", reg); 838 } 839 840 static void rtase_nic_reset(const struct net_device *dev) 841 { 842 const struct rtase_private *tp = netdev_priv(dev); 843 u16 rx_config; 844 u8 val; 845 846 rx_config = rtase_r16(tp, RTASE_RX_CONFIG_0); 847 rtase_w16(tp, RTASE_RX_CONFIG_0, rx_config & ~RTASE_ACCEPT_MASK); 848 849 val = rtase_r8(tp, RTASE_MISC); 850 rtase_w8(tp, RTASE_MISC, val | RTASE_RX_DV_GATE_EN); 851 852 val = rtase_r8(tp, RTASE_CHIP_CMD); 853 rtase_w8(tp, RTASE_CHIP_CMD, val | RTASE_STOP_REQ); 854 mdelay(2); 855 856 rtase_poll_timeout(tp, RTASE_STOP_REQ_DONE, 100, 150000, 857 RTASE_CHIP_CMD); 858 859 rtase_poll_timeout(tp, RTASE_TX_FIFO_EMPTY, 100, 100000, 860 RTASE_FIFOR); 861 862 rtase_poll_timeout(tp, RTASE_RX_FIFO_EMPTY, 100, 100000, 863 RTASE_FIFOR); 864 865 val = rtase_r8(tp, RTASE_CHIP_CMD); 866 rtase_w8(tp, RTASE_CHIP_CMD, val & ~(RTASE_TE | RTASE_RE)); 867 val = rtase_r8(tp, RTASE_CHIP_CMD); 868 rtase_w8(tp, RTASE_CHIP_CMD, val & ~RTASE_STOP_REQ); 869 870 rtase_w16(tp, RTASE_RX_CONFIG_0, rx_config); 871 } 872 873 static void rtase_hw_reset(const struct net_device *dev) 874 { 875 const struct rtase_private *tp = netdev_priv(dev); 876 877 rtase_irq_dis_and_clear(tp); 878 879 rtase_nic_reset(dev); 880 } 881 882 static void rtase_set_rx_queue(const struct rtase_private *tp) 883 { 884 u16 reg_data; 885 886 reg_data = rtase_r16(tp, RTASE_FCR); 887 switch (tp->func_rx_queue_num) { 888 case 1: 889 u16p_replace_bits(®_data, 0x1, RTASE_FCR_RXQ_MASK); 890 break; 891 case 2: 892 u16p_replace_bits(®_data, 0x2, RTASE_FCR_RXQ_MASK); 893 break; 894 case 4: 895 u16p_replace_bits(®_data, 0x3, RTASE_FCR_RXQ_MASK); 896 break; 897 } 898 rtase_w16(tp, RTASE_FCR, reg_data); 899 } 900 901 static void rtase_set_tx_queue(const struct rtase_private *tp) 902 { 903 u16 reg_data; 904 905 reg_data = rtase_r16(tp, RTASE_TX_CONFIG_1); 906 switch (tp->tx_queue_ctrl) { 907 case 1: 908 u16p_replace_bits(®_data, 0x0, RTASE_TC_MODE_MASK); 909 break; 910 case 2: 911 u16p_replace_bits(®_data, 0x1, RTASE_TC_MODE_MASK); 912 break; 913 case 3: 914 case 4: 915 u16p_replace_bits(®_data, 0x2, RTASE_TC_MODE_MASK); 916 break; 917 default: 918 u16p_replace_bits(®_data, 0x3, RTASE_TC_MODE_MASK); 919 break; 920 } 921 rtase_w16(tp, RTASE_TX_CONFIG_1, reg_data); 922 } 923 924 static void rtase_hw_config(struct net_device *dev) 925 { 926 const struct rtase_private *tp = netdev_priv(dev); 927 u32 reg_data32; 928 u16 reg_data16; 929 930 rtase_hw_reset(dev); 931 932 /* set rx dma burst */ 933 reg_data16 = rtase_r16(tp, RTASE_RX_CONFIG_0); 934 reg_data16 &= ~(RTASE_RX_SINGLE_TAG | RTASE_RX_SINGLE_FETCH); 935 u16p_replace_bits(®_data16, RTASE_RX_DMA_BURST_256, 936 RTASE_RX_MX_DMA_MASK); 937 rtase_w16(tp, RTASE_RX_CONFIG_0, reg_data16); 938 939 /* new rx descritpor */ 940 reg_data16 = rtase_r16(tp, RTASE_RX_CONFIG_1); 941 reg_data16 |= RTASE_RX_NEW_DESC_FORMAT_EN | RTASE_PCIE_NEW_FLOW; 942 u16p_replace_bits(®_data16, 0xF, RTASE_RX_MAX_FETCH_DESC_MASK); 943 rtase_w16(tp, RTASE_RX_CONFIG_1, reg_data16); 944 945 rtase_set_rx_queue(tp); 946 947 rtase_interrupt_mitigation(tp); 948 949 /* set tx dma burst size and interframe gap time */ 950 reg_data32 = rtase_r32(tp, RTASE_TX_CONFIG_0); 951 u32p_replace_bits(®_data32, RTASE_TX_DMA_BURST_UNLIMITED, 952 RTASE_TX_DMA_MASK); 953 u32p_replace_bits(®_data32, RTASE_INTERFRAMEGAP, 954 RTASE_TX_INTER_FRAME_GAP_MASK); 955 rtase_w32(tp, RTASE_TX_CONFIG_0, reg_data32); 956 957 /* new tx descriptor */ 958 reg_data16 = rtase_r16(tp, RTASE_TFUN_CTRL); 959 rtase_w16(tp, RTASE_TFUN_CTRL, reg_data16 | 960 RTASE_TX_NEW_DESC_FORMAT_EN); 961 962 /* tx fetch desc number */ 963 rtase_w8(tp, RTASE_TDFNR, 0x10); 964 965 /* tag num select */ 966 reg_data16 = rtase_r16(tp, RTASE_MTPS); 967 u16p_replace_bits(®_data16, 0x4, RTASE_TAG_NUM_SEL_MASK); 968 rtase_w16(tp, RTASE_MTPS, reg_data16); 969 970 rtase_set_tx_queue(tp); 971 972 rtase_w16(tp, RTASE_TOKSEL, 0x5555); 973 974 rtase_tally_counter_addr_fill(tp); 975 rtase_desc_addr_fill(tp); 976 rtase_hw_set_features(dev, dev->features); 977 978 /* enable flow control */ 979 reg_data16 = rtase_r16(tp, RTASE_GPHY_STD_00); 980 reg_data16 &= ~(RTASE_TXFLOW_EN | RTASE_RXFLOW_EN); 981 rtase_w16(tp, RTASE_GPHY_STD_00, reg_data16); 982 reg_data16 = rtase_r16(tp, RTASE_CPLUS_CMD); 983 reg_data16 |= (RTASE_FORCE_TXFLOW_EN | RTASE_FORCE_RXFLOW_EN); 984 rtase_w16(tp, RTASE_CPLUS_CMD, reg_data16); 985 /* set near fifo threshold - rx missed issue. */ 986 rtase_w16(tp, RTASE_RFIFONFULL, 0x190); 987 988 rtase_w16(tp, RTASE_RMS, tp->rx_buf_sz); 989 990 rtase_hw_set_rx_packet_filter(dev); 991 } 992 993 static void rtase_nic_enable(const struct net_device *dev) 994 { 995 const struct rtase_private *tp = netdev_priv(dev); 996 u16 rcr = rtase_r16(tp, RTASE_RX_CONFIG_1); 997 u8 val; 998 999 rtase_w16(tp, RTASE_RX_CONFIG_1, rcr & ~RTASE_PCIE_RELOAD_EN); 1000 rtase_w16(tp, RTASE_RX_CONFIG_1, rcr | RTASE_PCIE_RELOAD_EN); 1001 1002 val = rtase_r8(tp, RTASE_CHIP_CMD); 1003 rtase_w8(tp, RTASE_CHIP_CMD, val | RTASE_TE | RTASE_RE); 1004 1005 val = rtase_r8(tp, RTASE_MISC); 1006 rtase_w8(tp, RTASE_MISC, val & ~RTASE_RX_DV_GATE_EN); 1007 } 1008 1009 static void rtase_enable_hw_interrupt(const struct rtase_private *tp) 1010 { 1011 const struct rtase_int_vector *ivec = &tp->int_vector[0]; 1012 u32 i; 1013 1014 rtase_w32(tp, ivec->imr_addr, ivec->imr); 1015 1016 for (i = 1; i < tp->int_nums; i++) { 1017 ivec = &tp->int_vector[i]; 1018 rtase_w16(tp, ivec->imr_addr, ivec->imr); 1019 } 1020 } 1021 1022 static void rtase_hw_start(const struct net_device *dev) 1023 { 1024 const struct rtase_private *tp = netdev_priv(dev); 1025 1026 rtase_nic_enable(dev); 1027 rtase_enable_hw_interrupt(tp); 1028 } 1029 1030 /* the interrupt handler does RXQ0 and TXQ0, TXQ4~7 interrutp status 1031 */ 1032 static irqreturn_t rtase_interrupt(int irq, void *dev_instance) 1033 { 1034 const struct rtase_private *tp; 1035 struct rtase_int_vector *ivec; 1036 u32 status; 1037 1038 ivec = dev_instance; 1039 tp = ivec->tp; 1040 status = rtase_r32(tp, ivec->isr_addr); 1041 1042 rtase_w32(tp, ivec->imr_addr, 0x0); 1043 rtase_w32(tp, ivec->isr_addr, status & ~RTASE_FOVW); 1044 1045 if (napi_schedule_prep(&ivec->napi)) 1046 __napi_schedule(&ivec->napi); 1047 1048 return IRQ_HANDLED; 1049 } 1050 1051 /* the interrupt handler does RXQ1&TXQ1 or RXQ2&TXQ2 or RXQ3&TXQ3 interrupt 1052 * status according to interrupt vector 1053 */ 1054 static irqreturn_t rtase_q_interrupt(int irq, void *dev_instance) 1055 { 1056 const struct rtase_private *tp; 1057 struct rtase_int_vector *ivec; 1058 u16 status; 1059 1060 ivec = dev_instance; 1061 tp = ivec->tp; 1062 status = rtase_r16(tp, ivec->isr_addr); 1063 1064 rtase_w16(tp, ivec->imr_addr, 0x0); 1065 rtase_w16(tp, ivec->isr_addr, status); 1066 1067 if (napi_schedule_prep(&ivec->napi)) 1068 __napi_schedule(&ivec->napi); 1069 1070 return IRQ_HANDLED; 1071 } 1072 1073 static int rtase_poll(struct napi_struct *napi, int budget) 1074 { 1075 const struct rtase_int_vector *ivec; 1076 const struct rtase_private *tp; 1077 struct rtase_ring *ring; 1078 int total_workdone = 0; 1079 1080 ivec = container_of(napi, struct rtase_int_vector, napi); 1081 tp = ivec->tp; 1082 1083 list_for_each_entry(ring, &ivec->ring_list, ring_entry) 1084 total_workdone += ring->ring_handler(ring, budget); 1085 1086 if (total_workdone >= budget) 1087 return budget; 1088 1089 if (napi_complete_done(napi, total_workdone)) { 1090 if (!ivec->index) 1091 rtase_w32(tp, ivec->imr_addr, ivec->imr); 1092 else 1093 rtase_w16(tp, ivec->imr_addr, ivec->imr); 1094 } 1095 1096 return total_workdone; 1097 } 1098 1099 static int rtase_open(struct net_device *dev) 1100 { 1101 struct rtase_private *tp = netdev_priv(dev); 1102 const struct pci_dev *pdev = tp->pdev; 1103 struct rtase_int_vector *ivec; 1104 u16 i = 0, j; 1105 int ret; 1106 1107 ivec = &tp->int_vector[0]; 1108 tp->rx_buf_sz = RTASE_RX_BUF_SIZE; 1109 1110 ret = rtase_alloc_desc(tp); 1111 if (ret) 1112 return ret; 1113 1114 ret = rtase_init_ring(dev); 1115 if (ret) 1116 goto err_free_all_allocated_mem; 1117 1118 rtase_hw_config(dev); 1119 1120 if (tp->sw_flag & RTASE_SWF_MSIX_ENABLED) { 1121 ret = request_irq(ivec->irq, rtase_interrupt, 0, 1122 dev->name, ivec); 1123 if (ret) 1124 goto err_free_all_allocated_irq; 1125 1126 /* request other interrupts to handle multiqueue */ 1127 for (i = 1; i < tp->int_nums; i++) { 1128 ivec = &tp->int_vector[i]; 1129 snprintf(ivec->name, sizeof(ivec->name), "%s_int%u", 1130 tp->dev->name, i); 1131 ret = request_irq(ivec->irq, rtase_q_interrupt, 0, 1132 ivec->name, ivec); 1133 if (ret) 1134 goto err_free_all_allocated_irq; 1135 } 1136 } else { 1137 ret = request_irq(pdev->irq, rtase_interrupt, 0, dev->name, 1138 ivec); 1139 if (ret) 1140 goto err_free_all_allocated_mem; 1141 } 1142 1143 rtase_hw_start(dev); 1144 1145 for (i = 0; i < tp->int_nums; i++) { 1146 ivec = &tp->int_vector[i]; 1147 napi_enable(&ivec->napi); 1148 } 1149 1150 netif_carrier_on(dev); 1151 netif_wake_queue(dev); 1152 1153 return 0; 1154 1155 err_free_all_allocated_irq: 1156 for (j = 0; j < i; j++) 1157 free_irq(tp->int_vector[j].irq, &tp->int_vector[j]); 1158 1159 err_free_all_allocated_mem: 1160 rtase_free_desc(tp); 1161 1162 return ret; 1163 } 1164 1165 static void rtase_down(struct net_device *dev) 1166 { 1167 struct rtase_private *tp = netdev_priv(dev); 1168 struct rtase_int_vector *ivec; 1169 struct rtase_ring *ring, *tmp; 1170 u32 i; 1171 1172 for (i = 0; i < tp->int_nums; i++) { 1173 ivec = &tp->int_vector[i]; 1174 napi_disable(&ivec->napi); 1175 list_for_each_entry_safe(ring, tmp, &ivec->ring_list, 1176 ring_entry) { 1177 netif_queue_set_napi(tp->dev, ring->index, 1178 ring->type, NULL); 1179 1180 list_del(&ring->ring_entry); 1181 } 1182 } 1183 1184 netif_tx_disable(dev); 1185 1186 netif_carrier_off(dev); 1187 1188 rtase_hw_reset(dev); 1189 1190 rtase_tx_clear(tp); 1191 1192 rtase_rx_clear(tp); 1193 } 1194 1195 static int rtase_close(struct net_device *dev) 1196 { 1197 struct rtase_private *tp = netdev_priv(dev); 1198 const struct pci_dev *pdev = tp->pdev; 1199 u32 i; 1200 1201 rtase_down(dev); 1202 1203 if (tp->sw_flag & RTASE_SWF_MSIX_ENABLED) { 1204 for (i = 0; i < tp->int_nums; i++) 1205 free_irq(tp->int_vector[i].irq, &tp->int_vector[i]); 1206 1207 } else { 1208 free_irq(pdev->irq, &tp->int_vector[0]); 1209 } 1210 1211 rtase_free_desc(tp); 1212 1213 return 0; 1214 } 1215 1216 static u32 rtase_tx_vlan_tag(const struct rtase_private *tp, 1217 const struct sk_buff *skb) 1218 { 1219 return (skb_vlan_tag_present(skb)) ? 1220 (RTASE_TX_VLAN_TAG | swab16(skb_vlan_tag_get(skb))) : 0x00; 1221 } 1222 1223 static u32 rtase_tx_csum(struct sk_buff *skb, const struct net_device *dev) 1224 { 1225 u32 csum_cmd = 0; 1226 u8 ip_protocol; 1227 1228 switch (vlan_get_protocol(skb)) { 1229 case htons(ETH_P_IP): 1230 csum_cmd = RTASE_TX_IPCS_C; 1231 ip_protocol = ip_hdr(skb)->protocol; 1232 break; 1233 1234 case htons(ETH_P_IPV6): 1235 csum_cmd = RTASE_TX_IPV6F_C; 1236 ip_protocol = ipv6_hdr(skb)->nexthdr; 1237 break; 1238 1239 default: 1240 ip_protocol = IPPROTO_RAW; 1241 break; 1242 } 1243 1244 if (ip_protocol == IPPROTO_TCP) 1245 csum_cmd |= RTASE_TX_TCPCS_C; 1246 else if (ip_protocol == IPPROTO_UDP) 1247 csum_cmd |= RTASE_TX_UDPCS_C; 1248 1249 csum_cmd |= u32_encode_bits(skb_transport_offset(skb), 1250 RTASE_TCPHO_MASK); 1251 1252 return csum_cmd; 1253 } 1254 1255 static int rtase_xmit_frags(struct rtase_ring *ring, struct sk_buff *skb, 1256 u32 opts1, u32 opts2) 1257 { 1258 const struct skb_shared_info *info = skb_shinfo(skb); 1259 const struct rtase_private *tp = ring->ivec->tp; 1260 const u8 nr_frags = info->nr_frags; 1261 struct rtase_tx_desc *txd = NULL; 1262 u32 cur_frag, entry; 1263 1264 entry = ring->cur_idx; 1265 for (cur_frag = 0; cur_frag < nr_frags; cur_frag++) { 1266 const skb_frag_t *frag = &info->frags[cur_frag]; 1267 dma_addr_t mapping; 1268 u32 status, len; 1269 void *addr; 1270 1271 entry = (entry + 1) % RTASE_NUM_DESC; 1272 1273 txd = ring->desc + sizeof(struct rtase_tx_desc) * entry; 1274 len = skb_frag_size(frag); 1275 addr = skb_frag_address(frag); 1276 mapping = dma_map_single(&tp->pdev->dev, addr, len, 1277 DMA_TO_DEVICE); 1278 1279 if (unlikely(dma_mapping_error(&tp->pdev->dev, mapping))) { 1280 if (unlikely(net_ratelimit())) 1281 netdev_err(tp->dev, 1282 "Failed to map TX fragments DMA!\n"); 1283 1284 goto err_out; 1285 } 1286 1287 if (((entry + 1) % RTASE_NUM_DESC) == 0) 1288 status = (opts1 | len | RTASE_RING_END); 1289 else 1290 status = opts1 | len; 1291 1292 if (cur_frag == (nr_frags - 1)) { 1293 ring->skbuff[entry] = skb; 1294 status |= RTASE_TX_LAST_FRAG; 1295 } 1296 1297 ring->mis.len[entry] = len; 1298 txd->addr = cpu_to_le64(mapping); 1299 txd->opts2 = cpu_to_le32(opts2); 1300 1301 /* make sure the operating fields have been updated */ 1302 dma_wmb(); 1303 txd->opts1 = cpu_to_le32(status); 1304 } 1305 1306 return cur_frag; 1307 1308 err_out: 1309 rtase_tx_clear_range(ring, ring->cur_idx + 1, cur_frag); 1310 return -EIO; 1311 } 1312 1313 static netdev_tx_t rtase_start_xmit(struct sk_buff *skb, 1314 struct net_device *dev) 1315 { 1316 struct skb_shared_info *shinfo = skb_shinfo(skb); 1317 struct rtase_private *tp = netdev_priv(dev); 1318 u32 q_idx, entry, len, opts1, opts2; 1319 struct netdev_queue *tx_queue; 1320 bool stop_queue, door_bell; 1321 u32 mss = shinfo->gso_size; 1322 struct rtase_tx_desc *txd; 1323 struct rtase_ring *ring; 1324 dma_addr_t mapping; 1325 int frags; 1326 1327 /* multiqueues */ 1328 q_idx = skb_get_queue_mapping(skb); 1329 ring = &tp->tx_ring[q_idx]; 1330 tx_queue = netdev_get_tx_queue(dev, q_idx); 1331 1332 if (unlikely(!rtase_tx_avail(ring))) { 1333 if (net_ratelimit()) 1334 netdev_err(dev, 1335 "BUG! Tx Ring full when queue awake!\n"); 1336 1337 netif_stop_queue(dev); 1338 return NETDEV_TX_BUSY; 1339 } 1340 1341 entry = ring->cur_idx % RTASE_NUM_DESC; 1342 txd = ring->desc + sizeof(struct rtase_tx_desc) * entry; 1343 1344 opts1 = RTASE_DESC_OWN; 1345 opts2 = rtase_tx_vlan_tag(tp, skb); 1346 1347 /* tcp segmentation offload (or tcp large send) */ 1348 if (mss) { 1349 if (shinfo->gso_type & SKB_GSO_TCPV4) { 1350 opts1 |= RTASE_GIANT_SEND_V4; 1351 } else if (shinfo->gso_type & SKB_GSO_TCPV6) { 1352 if (skb_cow_head(skb, 0)) 1353 goto err_dma_0; 1354 1355 tcp_v6_gso_csum_prep(skb); 1356 opts1 |= RTASE_GIANT_SEND_V6; 1357 } else { 1358 WARN_ON_ONCE(1); 1359 } 1360 1361 opts1 |= u32_encode_bits(skb_transport_offset(skb), 1362 RTASE_TCPHO_MASK); 1363 opts2 |= u32_encode_bits(mss, RTASE_MSS_MASK); 1364 } else if (skb->ip_summed == CHECKSUM_PARTIAL) { 1365 opts2 |= rtase_tx_csum(skb, dev); 1366 } 1367 1368 frags = rtase_xmit_frags(ring, skb, opts1, opts2); 1369 if (unlikely(frags < 0)) 1370 goto err_dma_0; 1371 1372 if (frags) { 1373 len = skb_headlen(skb); 1374 opts1 |= RTASE_TX_FIRST_FRAG; 1375 } else { 1376 len = skb->len; 1377 ring->skbuff[entry] = skb; 1378 opts1 |= RTASE_TX_FIRST_FRAG | RTASE_TX_LAST_FRAG; 1379 } 1380 1381 if (((entry + 1) % RTASE_NUM_DESC) == 0) 1382 opts1 |= (len | RTASE_RING_END); 1383 else 1384 opts1 |= len; 1385 1386 mapping = dma_map_single(&tp->pdev->dev, skb->data, len, 1387 DMA_TO_DEVICE); 1388 1389 if (unlikely(dma_mapping_error(&tp->pdev->dev, mapping))) { 1390 if (unlikely(net_ratelimit())) 1391 netdev_err(dev, "Failed to map TX DMA!\n"); 1392 1393 goto err_dma_1; 1394 } 1395 1396 ring->mis.len[entry] = len; 1397 txd->addr = cpu_to_le64(mapping); 1398 txd->opts2 = cpu_to_le32(opts2); 1399 txd->opts1 = cpu_to_le32(opts1 & ~RTASE_DESC_OWN); 1400 1401 /* make sure the operating fields have been updated */ 1402 dma_wmb(); 1403 1404 door_bell = __netdev_tx_sent_queue(tx_queue, skb->len, 1405 netdev_xmit_more()); 1406 1407 txd->opts1 = cpu_to_le32(opts1); 1408 1409 skb_tx_timestamp(skb); 1410 1411 /* tx needs to see descriptor changes before updated cur_idx */ 1412 smp_wmb(); 1413 1414 WRITE_ONCE(ring->cur_idx, ring->cur_idx + frags + 1); 1415 1416 stop_queue = !netif_subqueue_maybe_stop(dev, ring->index, 1417 rtase_tx_avail(ring), 1418 RTASE_TX_STOP_THRS, 1419 RTASE_TX_START_THRS); 1420 1421 if (door_bell || stop_queue) 1422 rtase_w8(tp, RTASE_TPPOLL, BIT(ring->index)); 1423 1424 return NETDEV_TX_OK; 1425 1426 err_dma_1: 1427 ring->skbuff[entry] = NULL; 1428 rtase_tx_clear_range(ring, ring->cur_idx + 1, frags); 1429 1430 err_dma_0: 1431 tp->stats.tx_dropped++; 1432 dev_kfree_skb_any(skb); 1433 return NETDEV_TX_OK; 1434 } 1435 1436 static void rtase_set_rx_mode(struct net_device *dev) 1437 { 1438 rtase_hw_set_rx_packet_filter(dev); 1439 } 1440 1441 static void rtase_enable_eem_write(const struct rtase_private *tp) 1442 { 1443 u8 val; 1444 1445 val = rtase_r8(tp, RTASE_EEM); 1446 rtase_w8(tp, RTASE_EEM, val | RTASE_EEM_UNLOCK); 1447 } 1448 1449 static void rtase_disable_eem_write(const struct rtase_private *tp) 1450 { 1451 u8 val; 1452 1453 val = rtase_r8(tp, RTASE_EEM); 1454 rtase_w8(tp, RTASE_EEM, val & ~RTASE_EEM_UNLOCK); 1455 } 1456 1457 static void rtase_rar_set(const struct rtase_private *tp, const u8 *addr) 1458 { 1459 u32 rar_low, rar_high; 1460 1461 rar_low = (u32)addr[0] | ((u32)addr[1] << 8) | 1462 ((u32)addr[2] << 16) | ((u32)addr[3] << 24); 1463 1464 rar_high = (u32)addr[4] | ((u32)addr[5] << 8); 1465 1466 rtase_enable_eem_write(tp); 1467 rtase_w32(tp, RTASE_MAC0, rar_low); 1468 rtase_w32(tp, RTASE_MAC4, rar_high); 1469 rtase_disable_eem_write(tp); 1470 rtase_w16(tp, RTASE_LBK_CTRL, RTASE_LBK_ATLD | RTASE_LBK_CLR); 1471 } 1472 1473 static int rtase_set_mac_address(struct net_device *dev, void *p) 1474 { 1475 struct rtase_private *tp = netdev_priv(dev); 1476 int ret; 1477 1478 ret = eth_mac_addr(dev, p); 1479 if (ret) 1480 return ret; 1481 1482 rtase_rar_set(tp, dev->dev_addr); 1483 1484 return 0; 1485 } 1486 1487 static int rtase_change_mtu(struct net_device *dev, int new_mtu) 1488 { 1489 dev->mtu = new_mtu; 1490 1491 netdev_update_features(dev); 1492 1493 return 0; 1494 } 1495 1496 static void rtase_wait_for_quiescence(const struct net_device *dev) 1497 { 1498 struct rtase_private *tp = netdev_priv(dev); 1499 struct rtase_int_vector *ivec; 1500 u32 i; 1501 1502 for (i = 0; i < tp->int_nums; i++) { 1503 ivec = &tp->int_vector[i]; 1504 synchronize_irq(ivec->irq); 1505 /* wait for any pending NAPI task to complete */ 1506 napi_disable(&ivec->napi); 1507 } 1508 1509 rtase_irq_dis_and_clear(tp); 1510 1511 for (i = 0; i < tp->int_nums; i++) { 1512 ivec = &tp->int_vector[i]; 1513 napi_enable(&ivec->napi); 1514 } 1515 } 1516 1517 static void rtase_sw_reset(struct net_device *dev) 1518 { 1519 struct rtase_private *tp = netdev_priv(dev); 1520 struct rtase_ring *ring, *tmp; 1521 struct rtase_int_vector *ivec; 1522 int ret; 1523 u32 i; 1524 1525 netif_stop_queue(dev); 1526 netif_carrier_off(dev); 1527 rtase_hw_reset(dev); 1528 1529 /* let's wait a bit while any (async) irq lands on */ 1530 rtase_wait_for_quiescence(dev); 1531 rtase_tx_clear(tp); 1532 rtase_rx_clear(tp); 1533 1534 for (i = 0; i < tp->int_nums; i++) { 1535 ivec = &tp->int_vector[i]; 1536 list_for_each_entry_safe(ring, tmp, &ivec->ring_list, 1537 ring_entry) { 1538 netif_queue_set_napi(tp->dev, ring->index, 1539 ring->type, NULL); 1540 1541 list_del(&ring->ring_entry); 1542 } 1543 } 1544 1545 ret = rtase_init_ring(dev); 1546 if (ret) { 1547 netdev_err(dev, "unable to init ring\n"); 1548 rtase_free_desc(tp); 1549 return; 1550 } 1551 1552 rtase_hw_config(dev); 1553 /* always link, so start to transmit & receive */ 1554 rtase_hw_start(dev); 1555 1556 netif_carrier_on(dev); 1557 netif_wake_queue(dev); 1558 } 1559 1560 static void rtase_dump_tally_counter(const struct rtase_private *tp) 1561 { 1562 dma_addr_t paddr = tp->tally_paddr; 1563 u32 cmd = lower_32_bits(paddr); 1564 u32 val; 1565 int err; 1566 1567 rtase_w32(tp, RTASE_DTCCR4, upper_32_bits(paddr)); 1568 rtase_w32(tp, RTASE_DTCCR0, cmd); 1569 rtase_w32(tp, RTASE_DTCCR0, cmd | RTASE_COUNTER_DUMP); 1570 1571 err = read_poll_timeout_atomic(rtase_r32, val, 1572 !(val & RTASE_COUNTER_DUMP), 1573 10, 250, false, tp, RTASE_DTCCR0); 1574 1575 if (err == -ETIMEDOUT) 1576 netdev_err(tp->dev, "error occurred in dump tally counter\n"); 1577 } 1578 1579 static void rtase_dump_state(const struct net_device *dev) 1580 { 1581 const struct rtase_private *tp = netdev_priv(dev); 1582 int max_reg_size = RTASE_PCI_REGS_SIZE; 1583 const struct rtase_counters *counters; 1584 const struct rtase_ring *ring; 1585 u32 dword_rd; 1586 int n = 0; 1587 1588 ring = &tp->tx_ring[0]; 1589 netdev_err(dev, "Tx descriptor info:\n"); 1590 netdev_err(dev, "Tx curIdx = 0x%x\n", ring->cur_idx); 1591 netdev_err(dev, "Tx dirtyIdx = 0x%x\n", ring->dirty_idx); 1592 netdev_err(dev, "Tx phyAddr = %pad\n", &ring->phy_addr); 1593 1594 ring = &tp->rx_ring[0]; 1595 netdev_err(dev, "Rx descriptor info:\n"); 1596 netdev_err(dev, "Rx curIdx = 0x%x\n", ring->cur_idx); 1597 netdev_err(dev, "Rx dirtyIdx = 0x%x\n", ring->dirty_idx); 1598 netdev_err(dev, "Rx phyAddr = %pad\n", &ring->phy_addr); 1599 1600 netdev_err(dev, "Device Registers:\n"); 1601 netdev_err(dev, "Chip Command = 0x%02x\n", 1602 rtase_r8(tp, RTASE_CHIP_CMD)); 1603 netdev_err(dev, "IMR = %08x\n", rtase_r32(tp, RTASE_IMR0)); 1604 netdev_err(dev, "ISR = %08x\n", rtase_r32(tp, RTASE_ISR0)); 1605 netdev_err(dev, "Boot Ctrl Reg(0xE004) = %04x\n", 1606 rtase_r16(tp, RTASE_BOOT_CTL)); 1607 netdev_err(dev, "EPHY ISR(0xE014) = %04x\n", 1608 rtase_r16(tp, RTASE_EPHY_ISR)); 1609 netdev_err(dev, "EPHY IMR(0xE016) = %04x\n", 1610 rtase_r16(tp, RTASE_EPHY_IMR)); 1611 netdev_err(dev, "CLKSW SET REG(0xE018) = %04x\n", 1612 rtase_r16(tp, RTASE_CLKSW_SET)); 1613 1614 netdev_err(dev, "Dump PCI Registers:\n"); 1615 1616 while (n < max_reg_size) { 1617 if ((n % RTASE_DWORD_MOD) == 0) 1618 netdev_err(tp->dev, "0x%03x:\n", n); 1619 1620 pci_read_config_dword(tp->pdev, n, &dword_rd); 1621 netdev_err(tp->dev, "%08x\n", dword_rd); 1622 n += 4; 1623 } 1624 1625 netdev_err(dev, "Dump tally counter:\n"); 1626 counters = tp->tally_vaddr; 1627 rtase_dump_tally_counter(tp); 1628 1629 netdev_err(dev, "tx_packets %lld\n", 1630 le64_to_cpu(counters->tx_packets)); 1631 netdev_err(dev, "rx_packets %lld\n", 1632 le64_to_cpu(counters->rx_packets)); 1633 netdev_err(dev, "tx_errors %lld\n", 1634 le64_to_cpu(counters->tx_errors)); 1635 netdev_err(dev, "rx_errors %d\n", 1636 le32_to_cpu(counters->rx_errors)); 1637 netdev_err(dev, "rx_missed %d\n", 1638 le16_to_cpu(counters->rx_missed)); 1639 netdev_err(dev, "align_errors %d\n", 1640 le16_to_cpu(counters->align_errors)); 1641 netdev_err(dev, "tx_one_collision %d\n", 1642 le32_to_cpu(counters->tx_one_collision)); 1643 netdev_err(dev, "tx_multi_collision %d\n", 1644 le32_to_cpu(counters->tx_multi_collision)); 1645 netdev_err(dev, "rx_unicast %lld\n", 1646 le64_to_cpu(counters->rx_unicast)); 1647 netdev_err(dev, "rx_broadcast %lld\n", 1648 le64_to_cpu(counters->rx_broadcast)); 1649 netdev_err(dev, "rx_multicast %d\n", 1650 le32_to_cpu(counters->rx_multicast)); 1651 netdev_err(dev, "tx_aborted %d\n", 1652 le16_to_cpu(counters->tx_aborted)); 1653 netdev_err(dev, "tx_underrun %d\n", 1654 le16_to_cpu(counters->tx_underrun)); 1655 } 1656 1657 static void rtase_tx_timeout(struct net_device *dev, unsigned int txqueue) 1658 { 1659 rtase_dump_state(dev); 1660 rtase_sw_reset(dev); 1661 } 1662 1663 static void rtase_get_stats64(struct net_device *dev, 1664 struct rtnl_link_stats64 *stats) 1665 { 1666 const struct rtase_private *tp = netdev_priv(dev); 1667 const struct rtase_counters *counters; 1668 1669 counters = tp->tally_vaddr; 1670 1671 dev_fetch_sw_netstats(stats, dev->tstats); 1672 1673 /* fetch additional counter values missing in stats collected by driver 1674 * from tally counter 1675 */ 1676 rtase_dump_tally_counter(tp); 1677 stats->rx_errors = tp->stats.rx_errors; 1678 stats->tx_errors = le64_to_cpu(counters->tx_errors); 1679 stats->rx_dropped = tp->stats.rx_dropped; 1680 stats->tx_dropped = tp->stats.tx_dropped; 1681 stats->multicast = tp->stats.multicast; 1682 stats->rx_length_errors = tp->stats.rx_length_errors; 1683 } 1684 1685 static void rtase_set_hw_cbs(const struct rtase_private *tp, u32 queue) 1686 { 1687 u32 idle = tp->tx_qos[queue].idleslope * RTASE_1T_CLOCK; 1688 u32 val, i; 1689 1690 val = u32_encode_bits(idle / RTASE_1T_POWER, RTASE_IDLESLOPE_INT_MASK); 1691 idle %= RTASE_1T_POWER; 1692 1693 for (i = 1; i <= RTASE_IDLESLOPE_INT_SHIFT; i++) { 1694 idle *= 2; 1695 if ((idle / RTASE_1T_POWER) == 1) 1696 val |= BIT(RTASE_IDLESLOPE_INT_SHIFT - i); 1697 1698 idle %= RTASE_1T_POWER; 1699 } 1700 1701 rtase_w32(tp, RTASE_TXQCRDT_0 + queue * 4, val); 1702 } 1703 1704 static int rtase_setup_tc_cbs(struct rtase_private *tp, 1705 const struct tc_cbs_qopt_offload *qopt) 1706 { 1707 int queue = qopt->queue; 1708 1709 if (queue < 0 || queue >= tp->func_tx_queue_num) 1710 return -EINVAL; 1711 1712 if (!qopt->enable) { 1713 tp->tx_qos[queue].hicredit = 0; 1714 tp->tx_qos[queue].locredit = 0; 1715 tp->tx_qos[queue].idleslope = 0; 1716 tp->tx_qos[queue].sendslope = 0; 1717 1718 rtase_w32(tp, RTASE_TXQCRDT_0 + queue * 4, 0); 1719 } else { 1720 tp->tx_qos[queue].hicredit = qopt->hicredit; 1721 tp->tx_qos[queue].locredit = qopt->locredit; 1722 tp->tx_qos[queue].idleslope = qopt->idleslope; 1723 tp->tx_qos[queue].sendslope = qopt->sendslope; 1724 1725 rtase_set_hw_cbs(tp, queue); 1726 } 1727 1728 return 0; 1729 } 1730 1731 static int rtase_setup_tc(struct net_device *dev, enum tc_setup_type type, 1732 void *type_data) 1733 { 1734 struct rtase_private *tp = netdev_priv(dev); 1735 1736 switch (type) { 1737 case TC_SETUP_QDISC_CBS: 1738 return rtase_setup_tc_cbs(tp, type_data); 1739 default: 1740 return -EOPNOTSUPP; 1741 } 1742 } 1743 1744 static netdev_features_t rtase_fix_features(struct net_device *dev, 1745 netdev_features_t features) 1746 { 1747 netdev_features_t features_fix = features; 1748 1749 /* not support TSO for jumbo frames */ 1750 if (dev->mtu > ETH_DATA_LEN) 1751 features_fix &= ~NETIF_F_ALL_TSO; 1752 1753 return features_fix; 1754 } 1755 1756 static int rtase_set_features(struct net_device *dev, 1757 netdev_features_t features) 1758 { 1759 netdev_features_t features_set = features; 1760 1761 features_set &= NETIF_F_RXALL | NETIF_F_RXCSUM | 1762 NETIF_F_HW_VLAN_CTAG_RX; 1763 1764 if (features_set ^ dev->features) 1765 rtase_hw_set_features(dev, features_set); 1766 1767 return 0; 1768 } 1769 1770 static const struct net_device_ops rtase_netdev_ops = { 1771 .ndo_open = rtase_open, 1772 .ndo_stop = rtase_close, 1773 .ndo_start_xmit = rtase_start_xmit, 1774 .ndo_set_rx_mode = rtase_set_rx_mode, 1775 .ndo_set_mac_address = rtase_set_mac_address, 1776 .ndo_change_mtu = rtase_change_mtu, 1777 .ndo_tx_timeout = rtase_tx_timeout, 1778 .ndo_get_stats64 = rtase_get_stats64, 1779 .ndo_setup_tc = rtase_setup_tc, 1780 .ndo_fix_features = rtase_fix_features, 1781 .ndo_set_features = rtase_set_features, 1782 }; 1783 1784 static void rtase_get_mac_address(struct net_device *dev) 1785 { 1786 struct rtase_private *tp = netdev_priv(dev); 1787 u8 mac_addr[ETH_ALEN] __aligned(2) = {}; 1788 u32 i; 1789 1790 for (i = 0; i < ETH_ALEN; i++) 1791 mac_addr[i] = rtase_r8(tp, RTASE_MAC0 + i); 1792 1793 if (!is_valid_ether_addr(mac_addr)) { 1794 eth_hw_addr_random(dev); 1795 netdev_warn(dev, "Random ether addr %pM\n", dev->dev_addr); 1796 } else { 1797 eth_hw_addr_set(dev, mac_addr); 1798 ether_addr_copy(dev->perm_addr, dev->dev_addr); 1799 } 1800 1801 rtase_rar_set(tp, dev->dev_addr); 1802 } 1803 1804 static int rtase_get_settings(struct net_device *dev, 1805 struct ethtool_link_ksettings *cmd) 1806 { 1807 u32 supported = SUPPORTED_MII | SUPPORTED_Pause | SUPPORTED_Asym_Pause; 1808 const struct rtase_private *tp = netdev_priv(dev); 1809 1810 ethtool_convert_legacy_u32_to_link_mode(cmd->link_modes.supported, 1811 supported); 1812 1813 switch (tp->hw_ver) { 1814 case RTASE_HW_VER_906X_7XA: 1815 case RTASE_HW_VER_906X_7XC: 1816 cmd->base.speed = SPEED_5000; 1817 break; 1818 case RTASE_HW_VER_907XD_V1: 1819 case RTASE_HW_VER_907XD_VA: 1820 cmd->base.speed = SPEED_10000; 1821 break; 1822 } 1823 1824 cmd->base.duplex = DUPLEX_FULL; 1825 cmd->base.port = PORT_MII; 1826 cmd->base.autoneg = AUTONEG_DISABLE; 1827 1828 return 0; 1829 } 1830 1831 static void rtase_get_pauseparam(struct net_device *dev, 1832 struct ethtool_pauseparam *pause) 1833 { 1834 const struct rtase_private *tp = netdev_priv(dev); 1835 u16 value = rtase_r16(tp, RTASE_CPLUS_CMD); 1836 1837 pause->autoneg = AUTONEG_DISABLE; 1838 pause->tx_pause = !!(value & RTASE_FORCE_TXFLOW_EN); 1839 pause->rx_pause = !!(value & RTASE_FORCE_RXFLOW_EN); 1840 } 1841 1842 static int rtase_set_pauseparam(struct net_device *dev, 1843 struct ethtool_pauseparam *pause) 1844 { 1845 const struct rtase_private *tp = netdev_priv(dev); 1846 u16 value = rtase_r16(tp, RTASE_CPLUS_CMD); 1847 1848 if (pause->autoneg) 1849 return -EOPNOTSUPP; 1850 1851 value &= ~(RTASE_FORCE_TXFLOW_EN | RTASE_FORCE_RXFLOW_EN); 1852 1853 if (pause->tx_pause) 1854 value |= RTASE_FORCE_TXFLOW_EN; 1855 1856 if (pause->rx_pause) 1857 value |= RTASE_FORCE_RXFLOW_EN; 1858 1859 rtase_w16(tp, RTASE_CPLUS_CMD, value); 1860 return 0; 1861 } 1862 1863 static void rtase_get_eth_mac_stats(struct net_device *dev, 1864 struct ethtool_eth_mac_stats *stats) 1865 { 1866 struct rtase_private *tp = netdev_priv(dev); 1867 const struct rtase_counters *counters; 1868 1869 counters = tp->tally_vaddr; 1870 1871 rtase_dump_tally_counter(tp); 1872 1873 stats->FramesTransmittedOK = le64_to_cpu(counters->tx_packets); 1874 stats->FramesReceivedOK = le64_to_cpu(counters->rx_packets); 1875 stats->FramesLostDueToIntMACXmitError = 1876 le64_to_cpu(counters->tx_errors); 1877 stats->BroadcastFramesReceivedOK = le64_to_cpu(counters->rx_broadcast); 1878 } 1879 1880 static const struct ethtool_ops rtase_ethtool_ops = { 1881 .get_link = ethtool_op_get_link, 1882 .get_link_ksettings = rtase_get_settings, 1883 .get_pauseparam = rtase_get_pauseparam, 1884 .set_pauseparam = rtase_set_pauseparam, 1885 .get_eth_mac_stats = rtase_get_eth_mac_stats, 1886 .get_ts_info = ethtool_op_get_ts_info, 1887 }; 1888 1889 static void rtase_init_netdev_ops(struct net_device *dev) 1890 { 1891 dev->netdev_ops = &rtase_netdev_ops; 1892 dev->ethtool_ops = &rtase_ethtool_ops; 1893 } 1894 1895 static void rtase_init_napi(struct rtase_private *tp) 1896 { 1897 u16 i; 1898 1899 for (i = 0; i < tp->int_nums; i++) { 1900 netif_napi_add_config(tp->dev, &tp->int_vector[i].napi, 1901 tp->int_vector[i].poll, i); 1902 netif_napi_set_irq(&tp->int_vector[i].napi, 1903 tp->int_vector[i].irq); 1904 } 1905 } 1906 1907 static void rtase_reset_interrupt(struct pci_dev *pdev, 1908 const struct rtase_private *tp) 1909 { 1910 if (tp->sw_flag & RTASE_SWF_MSIX_ENABLED) 1911 pci_disable_msix(pdev); 1912 else 1913 pci_disable_msi(pdev); 1914 } 1915 1916 static int rtase_alloc_msix(struct pci_dev *pdev, struct rtase_private *tp) 1917 { 1918 int ret, irq; 1919 u16 i; 1920 1921 memset(tp->msix_entry, 0x0, RTASE_NUM_MSIX * 1922 sizeof(struct msix_entry)); 1923 1924 for (i = 0; i < RTASE_NUM_MSIX; i++) 1925 tp->msix_entry[i].entry = i; 1926 1927 ret = pci_enable_msix_exact(pdev, tp->msix_entry, tp->int_nums); 1928 1929 if (ret) 1930 return ret; 1931 1932 for (i = 0; i < tp->int_nums; i++) { 1933 irq = pci_irq_vector(pdev, i); 1934 if (irq < 0) { 1935 pci_disable_msix(pdev); 1936 return irq; 1937 } 1938 1939 tp->int_vector[i].irq = irq; 1940 } 1941 1942 return 0; 1943 } 1944 1945 static int rtase_alloc_interrupt(struct pci_dev *pdev, 1946 struct rtase_private *tp) 1947 { 1948 int ret; 1949 1950 ret = rtase_alloc_msix(pdev, tp); 1951 if (ret) { 1952 ret = pci_enable_msi(pdev); 1953 if (ret) { 1954 dev_err(&pdev->dev, 1955 "unable to alloc interrupt.(MSI)\n"); 1956 return ret; 1957 } 1958 1959 tp->sw_flag |= RTASE_SWF_MSI_ENABLED; 1960 } else { 1961 tp->sw_flag |= RTASE_SWF_MSIX_ENABLED; 1962 } 1963 1964 return 0; 1965 } 1966 1967 static void rtase_init_hardware(const struct rtase_private *tp) 1968 { 1969 u16 i; 1970 1971 for (i = 0; i < RTASE_VLAN_FILTER_ENTRY_NUM; i++) 1972 rtase_w32(tp, RTASE_VLAN_ENTRY_0 + i * 4, 0); 1973 } 1974 1975 static void rtase_init_int_vector(struct rtase_private *tp) 1976 { 1977 u16 i; 1978 1979 /* interrupt vector 0 */ 1980 tp->int_vector[0].tp = tp; 1981 tp->int_vector[0].index = 0; 1982 tp->int_vector[0].imr_addr = RTASE_IMR0; 1983 tp->int_vector[0].isr_addr = RTASE_ISR0; 1984 tp->int_vector[0].imr = RTASE_ROK | RTASE_RDU | RTASE_TOK | 1985 RTASE_TOK4 | RTASE_TOK5 | RTASE_TOK6 | 1986 RTASE_TOK7; 1987 tp->int_vector[0].poll = rtase_poll; 1988 1989 memset(tp->int_vector[0].name, 0x0, sizeof(tp->int_vector[0].name)); 1990 INIT_LIST_HEAD(&tp->int_vector[0].ring_list); 1991 1992 /* interrupt vector 1 ~ 3 */ 1993 for (i = 1; i < tp->int_nums; i++) { 1994 tp->int_vector[i].tp = tp; 1995 tp->int_vector[i].index = i; 1996 tp->int_vector[i].imr_addr = RTASE_IMR1 + (i - 1) * 4; 1997 tp->int_vector[i].isr_addr = RTASE_ISR1 + (i - 1) * 4; 1998 tp->int_vector[i].imr = RTASE_Q_ROK | RTASE_Q_RDU | 1999 RTASE_Q_TOK; 2000 tp->int_vector[i].poll = rtase_poll; 2001 2002 memset(tp->int_vector[i].name, 0x0, 2003 sizeof(tp->int_vector[0].name)); 2004 INIT_LIST_HEAD(&tp->int_vector[i].ring_list); 2005 } 2006 } 2007 2008 static u16 rtase_calc_time_mitigation(u32 time_us) 2009 { 2010 u8 msb, time_count, time_unit; 2011 u16 int_miti; 2012 2013 time_us = min(time_us, RTASE_MITI_MAX_TIME); 2014 2015 if (time_us > RTASE_MITI_TIME_COUNT_MASK) { 2016 msb = fls(time_us); 2017 time_unit = msb - RTASE_MITI_COUNT_BIT_NUM; 2018 time_count = time_us >> (msb - RTASE_MITI_COUNT_BIT_NUM); 2019 } else { 2020 time_unit = 0; 2021 time_count = time_us; 2022 } 2023 2024 int_miti = u16_encode_bits(time_count, RTASE_MITI_TIME_COUNT_MASK) | 2025 u16_encode_bits(time_unit, RTASE_MITI_TIME_UNIT_MASK); 2026 2027 return int_miti; 2028 } 2029 2030 static u16 rtase_calc_packet_num_mitigation(u16 pkt_num) 2031 { 2032 u8 msb, pkt_num_count, pkt_num_unit; 2033 u16 int_miti; 2034 2035 pkt_num = min(pkt_num, RTASE_MITI_MAX_PKT_NUM); 2036 2037 if (pkt_num > 60) { 2038 pkt_num_unit = RTASE_MITI_MAX_PKT_NUM_IDX; 2039 pkt_num_count = pkt_num / RTASE_MITI_MAX_PKT_NUM_UNIT; 2040 } else { 2041 msb = fls(pkt_num); 2042 if (msb >= RTASE_MITI_COUNT_BIT_NUM) { 2043 pkt_num_unit = msb - RTASE_MITI_COUNT_BIT_NUM; 2044 pkt_num_count = pkt_num >> (msb - 2045 RTASE_MITI_COUNT_BIT_NUM); 2046 } else { 2047 pkt_num_unit = 0; 2048 pkt_num_count = pkt_num; 2049 } 2050 } 2051 2052 int_miti = u16_encode_bits(pkt_num_count, 2053 RTASE_MITI_PKT_NUM_COUNT_MASK) | 2054 u16_encode_bits(pkt_num_unit, 2055 RTASE_MITI_PKT_NUM_UNIT_MASK); 2056 2057 return int_miti; 2058 } 2059 2060 static void rtase_init_software_variable(struct pci_dev *pdev, 2061 struct rtase_private *tp) 2062 { 2063 u16 int_miti; 2064 2065 tp->tx_queue_ctrl = RTASE_TXQ_CTRL; 2066 tp->func_tx_queue_num = RTASE_FUNC_TXQ_NUM; 2067 tp->func_rx_queue_num = RTASE_FUNC_RXQ_NUM; 2068 tp->int_nums = RTASE_INTERRUPT_NUM; 2069 2070 int_miti = rtase_calc_time_mitigation(RTASE_MITI_DEFAULT_TIME) | 2071 rtase_calc_packet_num_mitigation(RTASE_MITI_DEFAULT_PKT_NUM); 2072 tp->tx_int_mit = int_miti; 2073 tp->rx_int_mit = int_miti; 2074 2075 tp->sw_flag = 0; 2076 2077 rtase_init_int_vector(tp); 2078 2079 /* MTU range: 60 - hw-specific max */ 2080 tp->dev->min_mtu = ETH_ZLEN; 2081 tp->dev->max_mtu = RTASE_MAX_JUMBO_SIZE; 2082 } 2083 2084 static int rtase_check_mac_version_valid(struct rtase_private *tp) 2085 { 2086 int ret = -ENODEV; 2087 2088 tp->hw_ver = rtase_r32(tp, RTASE_TX_CONFIG_0) & RTASE_HW_VER_MASK; 2089 2090 switch (tp->hw_ver) { 2091 case RTASE_HW_VER_906X_7XA: 2092 case RTASE_HW_VER_906X_7XC: 2093 case RTASE_HW_VER_907XD_V1: 2094 case RTASE_HW_VER_907XD_VA: 2095 ret = 0; 2096 break; 2097 } 2098 2099 return ret; 2100 } 2101 2102 static int rtase_init_board(struct pci_dev *pdev, struct net_device **dev_out, 2103 void __iomem **ioaddr_out) 2104 { 2105 struct net_device *dev; 2106 void __iomem *ioaddr; 2107 int ret = -ENOMEM; 2108 2109 /* dev zeroed in alloc_etherdev */ 2110 dev = alloc_etherdev_mq(sizeof(struct rtase_private), 2111 RTASE_FUNC_TXQ_NUM); 2112 if (!dev) 2113 goto err_out; 2114 2115 SET_NETDEV_DEV(dev, &pdev->dev); 2116 2117 ret = pci_enable_device(pdev); 2118 if (ret) 2119 goto err_out_free_dev; 2120 2121 /* make sure PCI base addr 1 is MMIO */ 2122 if (!(pci_resource_flags(pdev, 2) & IORESOURCE_MEM)) { 2123 ret = -ENODEV; 2124 goto err_out_disable; 2125 } 2126 2127 /* check for weird/broken PCI region reporting */ 2128 if (pci_resource_len(pdev, 2) < RTASE_REGS_SIZE) { 2129 ret = -ENODEV; 2130 goto err_out_disable; 2131 } 2132 2133 ret = pci_request_regions(pdev, KBUILD_MODNAME); 2134 if (ret) 2135 goto err_out_disable; 2136 2137 ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); 2138 if (ret) { 2139 dev_err(&pdev->dev, "no usable dma addressing method\n"); 2140 goto err_out_free_res; 2141 } 2142 2143 pci_set_master(pdev); 2144 2145 /* ioremap MMIO region */ 2146 ioaddr = ioremap(pci_resource_start(pdev, 2), 2147 pci_resource_len(pdev, 2)); 2148 if (!ioaddr) { 2149 ret = -EIO; 2150 goto err_out_free_res; 2151 } 2152 2153 *ioaddr_out = ioaddr; 2154 *dev_out = dev; 2155 2156 return ret; 2157 2158 err_out_free_res: 2159 pci_release_regions(pdev); 2160 2161 err_out_disable: 2162 pci_disable_device(pdev); 2163 2164 err_out_free_dev: 2165 free_netdev(dev); 2166 2167 err_out: 2168 *ioaddr_out = NULL; 2169 *dev_out = NULL; 2170 2171 return ret; 2172 } 2173 2174 static void rtase_release_board(struct pci_dev *pdev, struct net_device *dev, 2175 void __iomem *ioaddr) 2176 { 2177 const struct rtase_private *tp = netdev_priv(dev); 2178 2179 rtase_rar_set(tp, tp->dev->perm_addr); 2180 iounmap(ioaddr); 2181 2182 if (tp->sw_flag & RTASE_SWF_MSIX_ENABLED) 2183 pci_disable_msix(pdev); 2184 else 2185 pci_disable_msi(pdev); 2186 2187 pci_release_regions(pdev); 2188 pci_disable_device(pdev); 2189 free_netdev(dev); 2190 } 2191 2192 static int rtase_init_one(struct pci_dev *pdev, 2193 const struct pci_device_id *ent) 2194 { 2195 struct net_device *dev = NULL; 2196 struct rtase_int_vector *ivec; 2197 void __iomem *ioaddr = NULL; 2198 struct rtase_private *tp; 2199 int ret, i; 2200 2201 if (!pdev->is_physfn && pdev->is_virtfn) { 2202 dev_err(&pdev->dev, 2203 "This module does not support a virtual function."); 2204 return -EINVAL; 2205 } 2206 2207 dev_dbg(&pdev->dev, "Automotive Switch Ethernet driver loaded\n"); 2208 2209 ret = rtase_init_board(pdev, &dev, &ioaddr); 2210 if (ret) 2211 return ret; 2212 2213 tp = netdev_priv(dev); 2214 tp->mmio_addr = ioaddr; 2215 tp->dev = dev; 2216 tp->pdev = pdev; 2217 2218 /* identify chip attached to board */ 2219 ret = rtase_check_mac_version_valid(tp); 2220 if (ret) { 2221 dev_err(&pdev->dev, 2222 "unknown chip version: 0x%08x, contact rtase maintainers (see MAINTAINERS file)\n", 2223 tp->hw_ver); 2224 goto err_out_release_board; 2225 } 2226 2227 rtase_init_software_variable(pdev, tp); 2228 rtase_init_hardware(tp); 2229 2230 ret = rtase_alloc_interrupt(pdev, tp); 2231 if (ret) { 2232 dev_err(&pdev->dev, "unable to alloc MSIX/MSI\n"); 2233 goto err_out_del_napi; 2234 } 2235 2236 rtase_init_napi(tp); 2237 2238 rtase_init_netdev_ops(dev); 2239 2240 dev->pcpu_stat_type = NETDEV_PCPU_STAT_TSTATS; 2241 2242 dev->features |= NETIF_F_HW_VLAN_CTAG_TX | NETIF_F_HW_VLAN_CTAG_RX | 2243 NETIF_F_IP_CSUM | NETIF_F_HIGHDMA | 2244 NETIF_F_RXCSUM | NETIF_F_SG | 2245 NETIF_F_TSO | NETIF_F_IPV6_CSUM | 2246 NETIF_F_TSO6; 2247 2248 dev->hw_features = NETIF_F_SG | NETIF_F_IP_CSUM | 2249 NETIF_F_TSO | NETIF_F_RXCSUM | 2250 NETIF_F_HW_VLAN_CTAG_TX | NETIF_F_HW_VLAN_CTAG_RX | 2251 NETIF_F_RXALL | NETIF_F_RXFCS | 2252 NETIF_F_IPV6_CSUM | NETIF_F_TSO6; 2253 2254 dev->vlan_features = NETIF_F_SG | NETIF_F_IP_CSUM | NETIF_F_TSO | 2255 NETIF_F_HIGHDMA; 2256 dev->priv_flags |= IFF_LIVE_ADDR_CHANGE; 2257 netif_set_tso_max_size(dev, RTASE_LSO_64K); 2258 netif_set_tso_max_segs(dev, RTASE_NIC_MAX_PHYS_BUF_COUNT_LSO2); 2259 2260 rtase_get_mac_address(dev); 2261 2262 tp->tally_vaddr = dma_alloc_coherent(&pdev->dev, 2263 sizeof(*tp->tally_vaddr), 2264 &tp->tally_paddr, 2265 GFP_KERNEL); 2266 if (!tp->tally_vaddr) { 2267 ret = -ENOMEM; 2268 goto err_out_free_dma; 2269 } 2270 2271 rtase_tally_counter_clear(tp); 2272 2273 pci_set_drvdata(pdev, dev); 2274 2275 netif_carrier_off(dev); 2276 2277 ret = register_netdev(dev); 2278 if (ret) 2279 goto err_out_free_dma; 2280 2281 netdev_dbg(dev, "%pM, IRQ %d\n", dev->dev_addr, dev->irq); 2282 2283 return 0; 2284 2285 err_out_free_dma: 2286 if (tp->tally_vaddr) { 2287 dma_free_coherent(&pdev->dev, 2288 sizeof(*tp->tally_vaddr), 2289 tp->tally_vaddr, 2290 tp->tally_paddr); 2291 2292 tp->tally_vaddr = NULL; 2293 } 2294 2295 err_out_del_napi: 2296 for (i = 0; i < tp->int_nums; i++) { 2297 ivec = &tp->int_vector[i]; 2298 netif_napi_del(&ivec->napi); 2299 } 2300 2301 err_out_release_board: 2302 rtase_release_board(pdev, dev, ioaddr); 2303 2304 return ret; 2305 } 2306 2307 static void rtase_remove_one(struct pci_dev *pdev) 2308 { 2309 struct net_device *dev = pci_get_drvdata(pdev); 2310 struct rtase_private *tp = netdev_priv(dev); 2311 struct rtase_int_vector *ivec; 2312 u32 i; 2313 2314 unregister_netdev(dev); 2315 2316 for (i = 0; i < tp->int_nums; i++) { 2317 ivec = &tp->int_vector[i]; 2318 netif_napi_del(&ivec->napi); 2319 } 2320 2321 rtase_reset_interrupt(pdev, tp); 2322 if (tp->tally_vaddr) { 2323 dma_free_coherent(&pdev->dev, 2324 sizeof(*tp->tally_vaddr), 2325 tp->tally_vaddr, 2326 tp->tally_paddr); 2327 tp->tally_vaddr = NULL; 2328 } 2329 2330 rtase_release_board(pdev, dev, tp->mmio_addr); 2331 pci_set_drvdata(pdev, NULL); 2332 } 2333 2334 static void rtase_shutdown(struct pci_dev *pdev) 2335 { 2336 struct net_device *dev = pci_get_drvdata(pdev); 2337 const struct rtase_private *tp; 2338 2339 tp = netdev_priv(dev); 2340 2341 if (netif_running(dev)) 2342 rtase_close(dev); 2343 2344 rtase_reset_interrupt(pdev, tp); 2345 } 2346 2347 static int rtase_suspend(struct device *device) 2348 { 2349 struct net_device *dev = dev_get_drvdata(device); 2350 2351 if (netif_running(dev)) { 2352 netif_device_detach(dev); 2353 rtase_hw_reset(dev); 2354 } 2355 2356 return 0; 2357 } 2358 2359 static int rtase_resume(struct device *device) 2360 { 2361 struct net_device *dev = dev_get_drvdata(device); 2362 struct rtase_private *tp = netdev_priv(dev); 2363 int ret; 2364 2365 /* restore last modified mac address */ 2366 rtase_rar_set(tp, dev->dev_addr); 2367 2368 if (!netif_running(dev)) 2369 goto out; 2370 2371 rtase_wait_for_quiescence(dev); 2372 2373 rtase_tx_clear(tp); 2374 rtase_rx_clear(tp); 2375 2376 ret = rtase_init_ring(dev); 2377 if (ret) { 2378 netdev_err(dev, "unable to init ring\n"); 2379 rtase_free_desc(tp); 2380 return -ENOMEM; 2381 } 2382 2383 rtase_hw_config(dev); 2384 /* always link, so start to transmit & receive */ 2385 rtase_hw_start(dev); 2386 2387 netif_device_attach(dev); 2388 out: 2389 2390 return 0; 2391 } 2392 2393 static const struct dev_pm_ops rtase_pm_ops = { 2394 SYSTEM_SLEEP_PM_OPS(rtase_suspend, rtase_resume) 2395 }; 2396 2397 static struct pci_driver rtase_pci_driver = { 2398 .name = KBUILD_MODNAME, 2399 .id_table = rtase_pci_tbl, 2400 .probe = rtase_init_one, 2401 .remove = rtase_remove_one, 2402 .shutdown = rtase_shutdown, 2403 .driver.pm = pm_ptr(&rtase_pm_ops), 2404 }; 2405 2406 module_pci_driver(rtase_pci_driver); 2407