1 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause 2 /* 3 * rtase is the Linux device driver released for Realtek Automotive Switch 4 * controllers with PCI-Express interface. 5 * 6 * Copyright(c) 2024 Realtek Semiconductor Corp. 7 * 8 * Below is a simplified block diagram of the chip and its relevant interfaces. 9 * 10 * ************************* 11 * * * 12 * * CPU network device * 13 * * * 14 * * +-------------+ * 15 * * | PCIE Host | * 16 * ***********++************ 17 * || 18 * PCIE 19 * || 20 * ********************++********************** 21 * * | PCIE Endpoint | * 22 * * +---------------+ * 23 * * | GMAC | * 24 * * +--++--+ Realtek * 25 * * || RTL90xx Series * 26 * * || * 27 * * +-------------++----------------+ * 28 * * | | MAC | | * 29 * * | +-----+ | * 30 * * | | * 31 * * | Ethernet Switch Core | * 32 * * | | * 33 * * | +-----+ +-----+ | * 34 * * | | MAC |...........| MAC | | * 35 * * +---+-----+-----------+-----+---+ * 36 * * | PHY |...........| PHY | * 37 * * +--++-+ +--++-+ * 38 * *************||****************||*********** 39 * 40 * The block of the Realtek RTL90xx series is our entire chip architecture, 41 * the GMAC is connected to the switch core, and there is no PHY in between. 42 * In addition, this driver is mainly used to control GMAC, but does not 43 * control the switch core, so it is not the same as DSA. Linux only plays 44 * the role of a normal leaf node in this model. 45 */ 46 47 #include <linux/crc32.h> 48 #include <linux/dma-mapping.h> 49 #include <linux/etherdevice.h> 50 #include <linux/if_vlan.h> 51 #include <linux/in.h> 52 #include <linux/init.h> 53 #include <linux/interrupt.h> 54 #include <linux/io.h> 55 #include <linux/iopoll.h> 56 #include <linux/ip.h> 57 #include <linux/ipv6.h> 58 #include <linux/mdio.h> 59 #include <linux/module.h> 60 #include <linux/netdevice.h> 61 #include <linux/pci.h> 62 #include <linux/pm_runtime.h> 63 #include <linux/prefetch.h> 64 #include <linux/ptp_classify.h> 65 #include <linux/rtnetlink.h> 66 #include <linux/tcp.h> 67 #include <asm/irq.h> 68 #include <net/ip6_checksum.h> 69 #include <net/netdev_queues.h> 70 #include <net/page_pool/helpers.h> 71 #include <net/pkt_cls.h> 72 73 #include "rtase.h" 74 75 #define RTK_OPTS1_DEBUG_VALUE 0x0BADBEEF 76 #define RTK_MAGIC_NUMBER 0x0BADBADBADBADBAD 77 78 static const struct pci_device_id rtase_pci_tbl[] = { 79 {PCI_VDEVICE(REALTEK, 0x906A)}, 80 {} 81 }; 82 83 MODULE_DEVICE_TABLE(pci, rtase_pci_tbl); 84 85 MODULE_AUTHOR("Realtek ARD Software Team"); 86 MODULE_DESCRIPTION("Network Driver for the PCIe interface of Realtek Automotive Ethernet Switch"); 87 MODULE_LICENSE("Dual BSD/GPL"); 88 89 struct rtase_counters { 90 __le64 tx_packets; 91 __le64 rx_packets; 92 __le64 tx_errors; 93 __le32 rx_errors; 94 __le16 rx_missed; 95 __le16 align_errors; 96 __le32 tx_one_collision; 97 __le32 tx_multi_collision; 98 __le64 rx_unicast; 99 __le64 rx_broadcast; 100 __le32 rx_multicast; 101 __le16 tx_aborted; 102 __le16 tx_underrun; 103 } __packed; 104 105 static void rtase_w8(const struct rtase_private *tp, u16 reg, u8 val8) 106 { 107 writeb(val8, tp->mmio_addr + reg); 108 } 109 110 static void rtase_w16(const struct rtase_private *tp, u16 reg, u16 val16) 111 { 112 writew(val16, tp->mmio_addr + reg); 113 } 114 115 static void rtase_w32(const struct rtase_private *tp, u16 reg, u32 val32) 116 { 117 writel(val32, tp->mmio_addr + reg); 118 } 119 120 static u8 rtase_r8(const struct rtase_private *tp, u16 reg) 121 { 122 return readb(tp->mmio_addr + reg); 123 } 124 125 static u16 rtase_r16(const struct rtase_private *tp, u16 reg) 126 { 127 return readw(tp->mmio_addr + reg); 128 } 129 130 static u32 rtase_r32(const struct rtase_private *tp, u16 reg) 131 { 132 return readl(tp->mmio_addr + reg); 133 } 134 135 static void rtase_free_desc(struct rtase_private *tp) 136 { 137 struct pci_dev *pdev = tp->pdev; 138 u32 i; 139 140 for (i = 0; i < tp->func_tx_queue_num; i++) { 141 if (!tp->tx_ring[i].desc) 142 continue; 143 144 dma_free_coherent(&pdev->dev, RTASE_TX_RING_DESC_SIZE, 145 tp->tx_ring[i].desc, 146 tp->tx_ring[i].phy_addr); 147 tp->tx_ring[i].desc = NULL; 148 } 149 150 for (i = 0; i < tp->func_rx_queue_num; i++) { 151 if (!tp->rx_ring[i].desc) 152 continue; 153 154 dma_free_coherent(&pdev->dev, RTASE_RX_RING_DESC_SIZE, 155 tp->rx_ring[i].desc, 156 tp->rx_ring[i].phy_addr); 157 tp->rx_ring[i].desc = NULL; 158 } 159 } 160 161 static int rtase_alloc_desc(struct rtase_private *tp) 162 { 163 struct pci_dev *pdev = tp->pdev; 164 u32 i; 165 166 /* rx and tx descriptors needs 256 bytes alignment. 167 * dma_alloc_coherent provides more. 168 */ 169 for (i = 0; i < tp->func_tx_queue_num; i++) { 170 tp->tx_ring[i].desc = 171 dma_alloc_coherent(&pdev->dev, 172 RTASE_TX_RING_DESC_SIZE, 173 &tp->tx_ring[i].phy_addr, 174 GFP_KERNEL); 175 if (!tp->tx_ring[i].desc) 176 goto err_out; 177 } 178 179 for (i = 0; i < tp->func_rx_queue_num; i++) { 180 tp->rx_ring[i].desc = 181 dma_alloc_coherent(&pdev->dev, 182 RTASE_RX_RING_DESC_SIZE, 183 &tp->rx_ring[i].phy_addr, 184 GFP_KERNEL); 185 if (!tp->rx_ring[i].desc) 186 goto err_out; 187 } 188 189 return 0; 190 191 err_out: 192 rtase_free_desc(tp); 193 return -ENOMEM; 194 } 195 196 static void rtase_unmap_tx_skb(struct pci_dev *pdev, u32 len, 197 struct rtase_tx_desc *desc) 198 { 199 dma_unmap_single(&pdev->dev, le64_to_cpu(desc->addr), len, 200 DMA_TO_DEVICE); 201 desc->opts1 = cpu_to_le32(RTK_OPTS1_DEBUG_VALUE); 202 desc->opts2 = 0x00; 203 desc->addr = cpu_to_le64(RTK_MAGIC_NUMBER); 204 } 205 206 static void rtase_tx_clear_range(struct rtase_ring *ring, u32 start, u32 n) 207 { 208 struct rtase_tx_desc *desc_base = ring->desc; 209 struct rtase_private *tp = ring->ivec->tp; 210 u32 i; 211 212 for (i = 0; i < n; i++) { 213 u32 entry = (start + i) % RTASE_NUM_DESC; 214 struct rtase_tx_desc *desc = desc_base + entry; 215 u32 len = ring->mis.len[entry]; 216 struct sk_buff *skb; 217 218 if (len == 0) 219 continue; 220 221 rtase_unmap_tx_skb(tp->pdev, len, desc); 222 ring->mis.len[entry] = 0; 223 skb = ring->skbuff[entry]; 224 if (!skb) 225 continue; 226 227 tp->stats.tx_dropped++; 228 dev_kfree_skb_any(skb); 229 ring->skbuff[entry] = NULL; 230 } 231 } 232 233 static void rtase_tx_clear(struct rtase_private *tp) 234 { 235 struct rtase_ring *ring; 236 u16 i; 237 238 for (i = 0; i < tp->func_tx_queue_num; i++) { 239 ring = &tp->tx_ring[i]; 240 rtase_tx_clear_range(ring, ring->dirty_idx, RTASE_NUM_DESC); 241 ring->cur_idx = 0; 242 ring->dirty_idx = 0; 243 244 netdev_tx_reset_subqueue(tp->dev, i); 245 } 246 } 247 248 static void rtase_mark_to_asic(union rtase_rx_desc *desc, u32 rx_buf_sz) 249 { 250 u32 eor = le32_to_cpu(desc->desc_cmd.opts1) & RTASE_RING_END; 251 252 desc->desc_status.opts2 = 0; 253 /* force memory writes to complete before releasing descriptor */ 254 dma_wmb(); 255 WRITE_ONCE(desc->desc_cmd.opts1, 256 cpu_to_le32(RTASE_DESC_OWN | eor | rx_buf_sz)); 257 } 258 259 static u32 rtase_tx_avail(struct rtase_ring *ring) 260 { 261 return READ_ONCE(ring->dirty_idx) + RTASE_NUM_DESC - 262 READ_ONCE(ring->cur_idx); 263 } 264 265 static int tx_handler(struct rtase_ring *ring, int budget) 266 { 267 const struct rtase_private *tp = ring->ivec->tp; 268 struct net_device *dev = tp->dev; 269 u32 dirty_tx, tx_left; 270 u32 bytes_compl = 0; 271 u32 pkts_compl = 0; 272 int workdone = 0; 273 274 dirty_tx = ring->dirty_idx; 275 tx_left = READ_ONCE(ring->cur_idx) - dirty_tx; 276 277 while (tx_left > 0) { 278 u32 entry = dirty_tx % RTASE_NUM_DESC; 279 struct rtase_tx_desc *desc = ring->desc + 280 sizeof(struct rtase_tx_desc) * entry; 281 u32 status; 282 283 status = le32_to_cpu(desc->opts1); 284 285 if (status & RTASE_DESC_OWN) 286 break; 287 288 rtase_unmap_tx_skb(tp->pdev, ring->mis.len[entry], desc); 289 ring->mis.len[entry] = 0; 290 if (ring->skbuff[entry]) { 291 pkts_compl++; 292 bytes_compl += ring->skbuff[entry]->len; 293 napi_consume_skb(ring->skbuff[entry], budget); 294 ring->skbuff[entry] = NULL; 295 } 296 297 dirty_tx++; 298 tx_left--; 299 workdone++; 300 301 if (workdone == RTASE_TX_BUDGET_DEFAULT) 302 break; 303 } 304 305 if (ring->dirty_idx != dirty_tx) { 306 dev_sw_netstats_tx_add(dev, pkts_compl, bytes_compl); 307 WRITE_ONCE(ring->dirty_idx, dirty_tx); 308 309 netif_subqueue_completed_wake(dev, ring->index, pkts_compl, 310 bytes_compl, 311 rtase_tx_avail(ring), 312 RTASE_TX_START_THRS); 313 314 if (ring->cur_idx != dirty_tx) 315 rtase_w8(tp, RTASE_TPPOLL, BIT(ring->index)); 316 } 317 318 return 0; 319 } 320 321 static void rtase_tx_desc_init(struct rtase_private *tp, u16 idx) 322 { 323 struct rtase_ring *ring = &tp->tx_ring[idx]; 324 struct rtase_tx_desc *desc; 325 u32 i; 326 327 memset(ring->desc, 0x0, RTASE_TX_RING_DESC_SIZE); 328 memset(ring->skbuff, 0x0, sizeof(ring->skbuff)); 329 ring->cur_idx = 0; 330 ring->dirty_idx = 0; 331 ring->index = idx; 332 ring->type = NETDEV_QUEUE_TYPE_TX; 333 ring->alloc_fail = 0; 334 335 for (i = 0; i < RTASE_NUM_DESC; i++) { 336 ring->mis.len[i] = 0; 337 if ((RTASE_NUM_DESC - 1) == i) { 338 desc = ring->desc + sizeof(struct rtase_tx_desc) * i; 339 desc->opts1 = cpu_to_le32(RTASE_RING_END); 340 } 341 } 342 343 ring->ring_handler = tx_handler; 344 if (idx < 4) { 345 ring->ivec = &tp->int_vector[idx]; 346 list_add_tail(&ring->ring_entry, 347 &tp->int_vector[idx].ring_list); 348 } else { 349 ring->ivec = &tp->int_vector[0]; 350 list_add_tail(&ring->ring_entry, &tp->int_vector[0].ring_list); 351 } 352 353 netif_queue_set_napi(tp->dev, ring->index, 354 ring->type, &ring->ivec->napi); 355 } 356 357 static void rtase_map_to_asic(union rtase_rx_desc *desc, dma_addr_t mapping, 358 u32 rx_buf_sz) 359 { 360 desc->desc_cmd.addr = cpu_to_le64(mapping); 361 362 rtase_mark_to_asic(desc, rx_buf_sz); 363 } 364 365 static void rtase_make_unusable_by_asic(union rtase_rx_desc *desc) 366 { 367 desc->desc_cmd.addr = cpu_to_le64(RTK_MAGIC_NUMBER); 368 desc->desc_cmd.opts1 &= ~cpu_to_le32(RTASE_DESC_OWN | RSVD_MASK); 369 } 370 371 static int rtase_alloc_rx_data_buf(struct rtase_ring *ring, 372 void **p_data_buf, 373 union rtase_rx_desc *desc, 374 dma_addr_t *rx_phy_addr) 375 { 376 struct rtase_int_vector *ivec = ring->ivec; 377 const struct rtase_private *tp = ivec->tp; 378 dma_addr_t mapping; 379 struct page *page; 380 381 page = page_pool_dev_alloc_pages(tp->page_pool); 382 if (!page) { 383 ring->alloc_fail++; 384 goto err_out; 385 } 386 387 *p_data_buf = page_address(page); 388 mapping = page_pool_get_dma_addr(page); 389 *rx_phy_addr = mapping; 390 rtase_map_to_asic(desc, mapping, tp->rx_buf_sz); 391 392 return 0; 393 394 err_out: 395 rtase_make_unusable_by_asic(desc); 396 397 return -ENOMEM; 398 } 399 400 static u32 rtase_rx_ring_fill(struct rtase_ring *ring, u32 ring_start, 401 u32 ring_end) 402 { 403 union rtase_rx_desc *desc_base = ring->desc; 404 u32 cur; 405 406 for (cur = ring_start; ring_end - cur > 0; cur++) { 407 u32 i = cur % RTASE_NUM_DESC; 408 union rtase_rx_desc *desc = desc_base + i; 409 int ret; 410 411 if (ring->data_buf[i]) 412 continue; 413 414 ret = rtase_alloc_rx_data_buf(ring, &ring->data_buf[i], desc, 415 &ring->mis.data_phy_addr[i]); 416 if (ret) 417 break; 418 } 419 420 return cur - ring_start; 421 } 422 423 static void rtase_mark_as_last_descriptor(union rtase_rx_desc *desc) 424 { 425 desc->desc_cmd.opts1 |= cpu_to_le32(RTASE_RING_END); 426 } 427 428 static void rtase_rx_ring_clear(struct page_pool *page_pool, 429 struct rtase_ring *ring) 430 { 431 union rtase_rx_desc *desc; 432 struct page *page; 433 u32 i; 434 435 for (i = 0; i < RTASE_NUM_DESC; i++) { 436 desc = ring->desc + sizeof(union rtase_rx_desc) * i; 437 page = virt_to_head_page(ring->data_buf[i]); 438 439 if (ring->data_buf[i]) 440 page_pool_put_full_page(page_pool, page, true); 441 442 rtase_make_unusable_by_asic(desc); 443 } 444 } 445 446 static int rtase_fragmented_frame(u32 status) 447 { 448 return (status & (RTASE_RX_FIRST_FRAG | RTASE_RX_LAST_FRAG)) != 449 (RTASE_RX_FIRST_FRAG | RTASE_RX_LAST_FRAG); 450 } 451 452 static void rtase_rx_csum(const struct rtase_private *tp, struct sk_buff *skb, 453 const union rtase_rx_desc *desc) 454 { 455 u32 opts2 = le32_to_cpu(desc->desc_status.opts2); 456 457 /* rx csum offload */ 458 if (((opts2 & RTASE_RX_V4F) && !(opts2 & RTASE_RX_IPF)) || 459 (opts2 & RTASE_RX_V6F)) { 460 if (((opts2 & RTASE_RX_TCPT) && !(opts2 & RTASE_RX_TCPF)) || 461 ((opts2 & RTASE_RX_UDPT) && !(opts2 & RTASE_RX_UDPF))) 462 skb->ip_summed = CHECKSUM_UNNECESSARY; 463 else 464 skb->ip_summed = CHECKSUM_NONE; 465 } else { 466 skb->ip_summed = CHECKSUM_NONE; 467 } 468 } 469 470 static void rtase_rx_vlan_skb(union rtase_rx_desc *desc, struct sk_buff *skb) 471 { 472 u32 opts2 = le32_to_cpu(desc->desc_status.opts2); 473 474 if (!(opts2 & RTASE_RX_VLAN_TAG)) 475 return; 476 477 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), 478 swab16(opts2 & RTASE_VLAN_TAG_MASK)); 479 } 480 481 static void rtase_rx_skb(const struct rtase_ring *ring, struct sk_buff *skb) 482 { 483 struct rtase_int_vector *ivec = ring->ivec; 484 485 napi_gro_receive(&ivec->napi, skb); 486 } 487 488 static int rx_handler(struct rtase_ring *ring, int budget) 489 { 490 union rtase_rx_desc *desc_base = ring->desc; 491 u32 pkt_size, cur_rx, delta, entry, status; 492 struct rtase_private *tp = ring->ivec->tp; 493 struct net_device *dev = tp->dev; 494 union rtase_rx_desc *desc; 495 struct sk_buff *skb; 496 int workdone = 0; 497 498 cur_rx = ring->cur_idx; 499 entry = cur_rx % RTASE_NUM_DESC; 500 desc = &desc_base[entry]; 501 502 while (workdone < budget) { 503 status = le32_to_cpu(desc->desc_status.opts1); 504 505 if (status & RTASE_DESC_OWN) 506 break; 507 508 /* This barrier is needed to keep us from reading 509 * any other fields out of the rx descriptor until 510 * we know the status of RTASE_DESC_OWN 511 */ 512 dma_rmb(); 513 514 if (unlikely(status & RTASE_RX_RES)) { 515 if (net_ratelimit()) 516 netdev_warn(dev, "Rx ERROR. status = %08x\n", 517 status); 518 519 tp->stats.rx_errors++; 520 521 if (status & (RTASE_RX_RWT | RTASE_RX_RUNT)) 522 tp->stats.rx_length_errors++; 523 524 if (status & RTASE_RX_CRC) 525 tp->stats.rx_crc_errors++; 526 527 if (dev->features & NETIF_F_RXALL) 528 goto process_pkt; 529 530 rtase_mark_to_asic(desc, tp->rx_buf_sz); 531 goto skip_process_pkt; 532 } 533 534 process_pkt: 535 pkt_size = status & RTASE_RX_PKT_SIZE_MASK; 536 if (likely(!(dev->features & NETIF_F_RXFCS))) 537 pkt_size -= ETH_FCS_LEN; 538 539 /* The driver does not support incoming fragmented frames. 540 * They are seen as a symptom of over-mtu sized frames. 541 */ 542 if (unlikely(rtase_fragmented_frame(status))) { 543 tp->stats.rx_dropped++; 544 tp->stats.rx_length_errors++; 545 rtase_mark_to_asic(desc, tp->rx_buf_sz); 546 goto skip_process_pkt; 547 } 548 549 dma_sync_single_for_cpu(&tp->pdev->dev, 550 ring->mis.data_phy_addr[entry], 551 tp->rx_buf_sz, DMA_FROM_DEVICE); 552 553 skb = build_skb(ring->data_buf[entry], PAGE_SIZE); 554 if (!skb) { 555 tp->stats.rx_dropped++; 556 rtase_mark_to_asic(desc, tp->rx_buf_sz); 557 goto skip_process_pkt; 558 } 559 ring->data_buf[entry] = NULL; 560 561 if (dev->features & NETIF_F_RXCSUM) 562 rtase_rx_csum(tp, skb, desc); 563 564 skb_put(skb, pkt_size); 565 skb_mark_for_recycle(skb); 566 skb->protocol = eth_type_trans(skb, dev); 567 568 if (skb->pkt_type == PACKET_MULTICAST) 569 tp->stats.multicast++; 570 571 rtase_rx_vlan_skb(desc, skb); 572 rtase_rx_skb(ring, skb); 573 574 dev_sw_netstats_rx_add(dev, pkt_size); 575 576 skip_process_pkt: 577 workdone++; 578 cur_rx++; 579 entry = cur_rx % RTASE_NUM_DESC; 580 desc = ring->desc + sizeof(union rtase_rx_desc) * entry; 581 } 582 583 ring->cur_idx = cur_rx; 584 delta = rtase_rx_ring_fill(ring, ring->dirty_idx, ring->cur_idx); 585 ring->dirty_idx += delta; 586 587 return workdone; 588 } 589 590 static void rtase_rx_desc_init(struct rtase_private *tp, u16 idx) 591 { 592 struct rtase_ring *ring = &tp->rx_ring[idx]; 593 u16 i; 594 595 memset(ring->desc, 0x0, RTASE_RX_RING_DESC_SIZE); 596 memset(ring->data_buf, 0x0, sizeof(ring->data_buf)); 597 ring->cur_idx = 0; 598 ring->dirty_idx = 0; 599 ring->index = idx; 600 ring->type = NETDEV_QUEUE_TYPE_RX; 601 ring->alloc_fail = 0; 602 603 for (i = 0; i < RTASE_NUM_DESC; i++) 604 ring->mis.data_phy_addr[i] = 0; 605 606 ring->ring_handler = rx_handler; 607 ring->ivec = &tp->int_vector[idx]; 608 netif_queue_set_napi(tp->dev, ring->index, 609 ring->type, &ring->ivec->napi); 610 list_add_tail(&ring->ring_entry, &tp->int_vector[idx].ring_list); 611 } 612 613 static void rtase_rx_clear(struct rtase_private *tp) 614 { 615 u32 i; 616 617 for (i = 0; i < tp->func_rx_queue_num; i++) 618 rtase_rx_ring_clear(tp->page_pool, &tp->rx_ring[i]); 619 620 page_pool_destroy(tp->page_pool); 621 tp->page_pool = NULL; 622 } 623 624 static int rtase_init_ring(const struct net_device *dev) 625 { 626 struct rtase_private *tp = netdev_priv(dev); 627 struct page_pool_params pp_params = { 0 }; 628 struct page_pool *page_pool; 629 u32 num; 630 u16 i; 631 632 pp_params.flags = PP_FLAG_DMA_MAP | PP_FLAG_DMA_SYNC_DEV; 633 pp_params.order = 0; 634 pp_params.pool_size = RTASE_NUM_DESC * tp->func_rx_queue_num; 635 pp_params.nid = dev_to_node(&tp->pdev->dev); 636 pp_params.dev = &tp->pdev->dev; 637 pp_params.dma_dir = DMA_FROM_DEVICE; 638 pp_params.max_len = PAGE_SIZE; 639 pp_params.offset = 0; 640 641 page_pool = page_pool_create(&pp_params); 642 if (IS_ERR(page_pool)) { 643 netdev_err(tp->dev, "failed to create page pool\n"); 644 return -ENOMEM; 645 } 646 647 tp->page_pool = page_pool; 648 649 for (i = 0; i < tp->func_tx_queue_num; i++) 650 rtase_tx_desc_init(tp, i); 651 652 for (i = 0; i < tp->func_rx_queue_num; i++) { 653 rtase_rx_desc_init(tp, i); 654 655 num = rtase_rx_ring_fill(&tp->rx_ring[i], 0, RTASE_NUM_DESC); 656 if (num != RTASE_NUM_DESC) 657 goto err_out; 658 659 rtase_mark_as_last_descriptor(tp->rx_ring[i].desc + 660 sizeof(union rtase_rx_desc) * 661 (RTASE_NUM_DESC - 1)); 662 } 663 664 return 0; 665 666 err_out: 667 rtase_rx_clear(tp); 668 return -ENOMEM; 669 } 670 671 static void rtase_interrupt_mitigation(const struct rtase_private *tp) 672 { 673 u32 i; 674 675 for (i = 0; i < tp->func_tx_queue_num; i++) 676 rtase_w16(tp, RTASE_INT_MITI_TX + i * 2, tp->tx_int_mit); 677 678 for (i = 0; i < tp->func_rx_queue_num; i++) 679 rtase_w16(tp, RTASE_INT_MITI_RX + i * 2, tp->rx_int_mit); 680 } 681 682 static void rtase_tally_counter_addr_fill(const struct rtase_private *tp) 683 { 684 rtase_w32(tp, RTASE_DTCCR4, upper_32_bits(tp->tally_paddr)); 685 rtase_w32(tp, RTASE_DTCCR0, lower_32_bits(tp->tally_paddr)); 686 } 687 688 static void rtase_tally_counter_clear(const struct rtase_private *tp) 689 { 690 u32 cmd = lower_32_bits(tp->tally_paddr); 691 692 rtase_w32(tp, RTASE_DTCCR4, upper_32_bits(tp->tally_paddr)); 693 rtase_w32(tp, RTASE_DTCCR0, cmd | RTASE_COUNTER_RESET); 694 } 695 696 static void rtase_desc_addr_fill(const struct rtase_private *tp) 697 { 698 const struct rtase_ring *ring; 699 u16 i, cmd, val; 700 int err; 701 702 for (i = 0; i < tp->func_tx_queue_num; i++) { 703 ring = &tp->tx_ring[i]; 704 705 rtase_w32(tp, RTASE_TX_DESC_ADDR0, 706 lower_32_bits(ring->phy_addr)); 707 rtase_w32(tp, RTASE_TX_DESC_ADDR4, 708 upper_32_bits(ring->phy_addr)); 709 710 cmd = i | RTASE_TX_DESC_CMD_WE | RTASE_TX_DESC_CMD_CS; 711 rtase_w16(tp, RTASE_TX_DESC_COMMAND, cmd); 712 713 err = read_poll_timeout(rtase_r16, val, 714 !(val & RTASE_TX_DESC_CMD_CS), 10, 715 1000, false, tp, 716 RTASE_TX_DESC_COMMAND); 717 718 if (err == -ETIMEDOUT) 719 netdev_err(tp->dev, 720 "error occurred in fill tx descriptor\n"); 721 } 722 723 for (i = 0; i < tp->func_rx_queue_num; i++) { 724 ring = &tp->rx_ring[i]; 725 726 if (i == 0) { 727 rtase_w32(tp, RTASE_Q0_RX_DESC_ADDR0, 728 lower_32_bits(ring->phy_addr)); 729 rtase_w32(tp, RTASE_Q0_RX_DESC_ADDR4, 730 upper_32_bits(ring->phy_addr)); 731 } else { 732 rtase_w32(tp, (RTASE_Q1_RX_DESC_ADDR0 + ((i - 1) * 8)), 733 lower_32_bits(ring->phy_addr)); 734 rtase_w32(tp, (RTASE_Q1_RX_DESC_ADDR4 + ((i - 1) * 8)), 735 upper_32_bits(ring->phy_addr)); 736 } 737 } 738 } 739 740 static void rtase_hw_set_features(const struct net_device *dev, 741 netdev_features_t features) 742 { 743 const struct rtase_private *tp = netdev_priv(dev); 744 u16 rx_config, val; 745 746 rx_config = rtase_r16(tp, RTASE_RX_CONFIG_0); 747 if (features & NETIF_F_RXALL) 748 rx_config |= (RTASE_ACCEPT_ERR | RTASE_ACCEPT_RUNT); 749 else 750 rx_config &= ~(RTASE_ACCEPT_ERR | RTASE_ACCEPT_RUNT); 751 752 rtase_w16(tp, RTASE_RX_CONFIG_0, rx_config); 753 754 val = rtase_r16(tp, RTASE_CPLUS_CMD); 755 if (features & NETIF_F_RXCSUM) 756 rtase_w16(tp, RTASE_CPLUS_CMD, val | RTASE_RX_CHKSUM); 757 else 758 rtase_w16(tp, RTASE_CPLUS_CMD, val & ~RTASE_RX_CHKSUM); 759 760 rx_config = rtase_r16(tp, RTASE_RX_CONFIG_1); 761 if (dev->features & NETIF_F_HW_VLAN_CTAG_RX) 762 rx_config |= (RTASE_INNER_VLAN_DETAG_EN | 763 RTASE_OUTER_VLAN_DETAG_EN); 764 else 765 rx_config &= ~(RTASE_INNER_VLAN_DETAG_EN | 766 RTASE_OUTER_VLAN_DETAG_EN); 767 768 rtase_w16(tp, RTASE_RX_CONFIG_1, rx_config); 769 } 770 771 static void rtase_hw_set_rx_packet_filter(struct net_device *dev) 772 { 773 u32 mc_filter[2] = { 0xFFFFFFFF, 0xFFFFFFFF }; 774 struct rtase_private *tp = netdev_priv(dev); 775 u16 rx_mode; 776 777 rx_mode = rtase_r16(tp, RTASE_RX_CONFIG_0) & ~RTASE_ACCEPT_MASK; 778 rx_mode |= RTASE_ACCEPT_BROADCAST | RTASE_ACCEPT_MYPHYS; 779 780 if (dev->flags & IFF_PROMISC) { 781 rx_mode |= RTASE_ACCEPT_MULTICAST | RTASE_ACCEPT_ALLPHYS; 782 } else if (dev->flags & IFF_ALLMULTI) { 783 rx_mode |= RTASE_ACCEPT_MULTICAST; 784 } else { 785 struct netdev_hw_addr *hw_addr; 786 787 mc_filter[0] = 0; 788 mc_filter[1] = 0; 789 790 netdev_for_each_mc_addr(hw_addr, dev) { 791 u32 bit_nr = eth_hw_addr_crc(hw_addr); 792 u32 idx = u32_get_bits(bit_nr, BIT(31)); 793 u32 bit = u32_get_bits(bit_nr, 794 RTASE_MULTICAST_FILTER_MASK); 795 796 mc_filter[idx] |= BIT(bit); 797 rx_mode |= RTASE_ACCEPT_MULTICAST; 798 } 799 } 800 801 if (dev->features & NETIF_F_RXALL) 802 rx_mode |= RTASE_ACCEPT_ERR | RTASE_ACCEPT_RUNT; 803 804 rtase_w32(tp, RTASE_MAR0, swab32(mc_filter[1])); 805 rtase_w32(tp, RTASE_MAR1, swab32(mc_filter[0])); 806 rtase_w16(tp, RTASE_RX_CONFIG_0, rx_mode); 807 } 808 809 static void rtase_irq_dis_and_clear(const struct rtase_private *tp) 810 { 811 const struct rtase_int_vector *ivec = &tp->int_vector[0]; 812 u32 val1; 813 u16 val2; 814 u8 i; 815 816 rtase_w32(tp, ivec->imr_addr, 0); 817 val1 = rtase_r32(tp, ivec->isr_addr); 818 rtase_w32(tp, ivec->isr_addr, val1); 819 820 for (i = 1; i < tp->int_nums; i++) { 821 ivec = &tp->int_vector[i]; 822 rtase_w16(tp, ivec->imr_addr, 0); 823 val2 = rtase_r16(tp, ivec->isr_addr); 824 rtase_w16(tp, ivec->isr_addr, val2); 825 } 826 } 827 828 static void rtase_poll_timeout(const struct rtase_private *tp, u32 cond, 829 u32 sleep_us, u64 timeout_us, u16 reg) 830 { 831 int err; 832 u8 val; 833 834 err = read_poll_timeout(rtase_r8, val, val & cond, sleep_us, 835 timeout_us, false, tp, reg); 836 837 if (err == -ETIMEDOUT) 838 netdev_err(tp->dev, "poll reg 0x00%x timeout\n", reg); 839 } 840 841 static void rtase_nic_reset(const struct net_device *dev) 842 { 843 const struct rtase_private *tp = netdev_priv(dev); 844 u16 rx_config; 845 u8 val; 846 847 rx_config = rtase_r16(tp, RTASE_RX_CONFIG_0); 848 rtase_w16(tp, RTASE_RX_CONFIG_0, rx_config & ~RTASE_ACCEPT_MASK); 849 850 val = rtase_r8(tp, RTASE_MISC); 851 rtase_w8(tp, RTASE_MISC, val | RTASE_RX_DV_GATE_EN); 852 853 val = rtase_r8(tp, RTASE_CHIP_CMD); 854 rtase_w8(tp, RTASE_CHIP_CMD, val | RTASE_STOP_REQ); 855 mdelay(2); 856 857 rtase_poll_timeout(tp, RTASE_STOP_REQ_DONE, 100, 150000, 858 RTASE_CHIP_CMD); 859 860 rtase_poll_timeout(tp, RTASE_TX_FIFO_EMPTY, 100, 100000, 861 RTASE_FIFOR); 862 863 rtase_poll_timeout(tp, RTASE_RX_FIFO_EMPTY, 100, 100000, 864 RTASE_FIFOR); 865 866 val = rtase_r8(tp, RTASE_CHIP_CMD); 867 rtase_w8(tp, RTASE_CHIP_CMD, val & ~(RTASE_TE | RTASE_RE)); 868 val = rtase_r8(tp, RTASE_CHIP_CMD); 869 rtase_w8(tp, RTASE_CHIP_CMD, val & ~RTASE_STOP_REQ); 870 871 rtase_w16(tp, RTASE_RX_CONFIG_0, rx_config); 872 } 873 874 static void rtase_hw_reset(const struct net_device *dev) 875 { 876 const struct rtase_private *tp = netdev_priv(dev); 877 878 rtase_irq_dis_and_clear(tp); 879 880 rtase_nic_reset(dev); 881 } 882 883 static void rtase_set_rx_queue(const struct rtase_private *tp) 884 { 885 u16 reg_data; 886 887 reg_data = rtase_r16(tp, RTASE_FCR); 888 switch (tp->func_rx_queue_num) { 889 case 1: 890 u16p_replace_bits(®_data, 0x1, RTASE_FCR_RXQ_MASK); 891 break; 892 case 2: 893 u16p_replace_bits(®_data, 0x2, RTASE_FCR_RXQ_MASK); 894 break; 895 case 4: 896 u16p_replace_bits(®_data, 0x3, RTASE_FCR_RXQ_MASK); 897 break; 898 } 899 rtase_w16(tp, RTASE_FCR, reg_data); 900 } 901 902 static void rtase_set_tx_queue(const struct rtase_private *tp) 903 { 904 u16 reg_data; 905 906 reg_data = rtase_r16(tp, RTASE_TX_CONFIG_1); 907 switch (tp->tx_queue_ctrl) { 908 case 1: 909 u16p_replace_bits(®_data, 0x0, RTASE_TC_MODE_MASK); 910 break; 911 case 2: 912 u16p_replace_bits(®_data, 0x1, RTASE_TC_MODE_MASK); 913 break; 914 case 3: 915 case 4: 916 u16p_replace_bits(®_data, 0x2, RTASE_TC_MODE_MASK); 917 break; 918 default: 919 u16p_replace_bits(®_data, 0x3, RTASE_TC_MODE_MASK); 920 break; 921 } 922 rtase_w16(tp, RTASE_TX_CONFIG_1, reg_data); 923 } 924 925 static void rtase_hw_config(struct net_device *dev) 926 { 927 const struct rtase_private *tp = netdev_priv(dev); 928 u32 reg_data32; 929 u16 reg_data16; 930 931 rtase_hw_reset(dev); 932 933 /* set rx dma burst */ 934 reg_data16 = rtase_r16(tp, RTASE_RX_CONFIG_0); 935 reg_data16 &= ~(RTASE_RX_SINGLE_TAG | RTASE_RX_SINGLE_FETCH); 936 u16p_replace_bits(®_data16, RTASE_RX_DMA_BURST_256, 937 RTASE_RX_MX_DMA_MASK); 938 rtase_w16(tp, RTASE_RX_CONFIG_0, reg_data16); 939 940 /* new rx descritpor */ 941 reg_data16 = rtase_r16(tp, RTASE_RX_CONFIG_1); 942 reg_data16 |= RTASE_RX_NEW_DESC_FORMAT_EN | RTASE_PCIE_NEW_FLOW; 943 u16p_replace_bits(®_data16, 0xF, RTASE_RX_MAX_FETCH_DESC_MASK); 944 rtase_w16(tp, RTASE_RX_CONFIG_1, reg_data16); 945 946 rtase_set_rx_queue(tp); 947 948 rtase_interrupt_mitigation(tp); 949 950 /* set tx dma burst size and interframe gap time */ 951 reg_data32 = rtase_r32(tp, RTASE_TX_CONFIG_0); 952 u32p_replace_bits(®_data32, RTASE_TX_DMA_BURST_UNLIMITED, 953 RTASE_TX_DMA_MASK); 954 u32p_replace_bits(®_data32, RTASE_INTERFRAMEGAP, 955 RTASE_TX_INTER_FRAME_GAP_MASK); 956 rtase_w32(tp, RTASE_TX_CONFIG_0, reg_data32); 957 958 /* new tx descriptor */ 959 reg_data16 = rtase_r16(tp, RTASE_TFUN_CTRL); 960 rtase_w16(tp, RTASE_TFUN_CTRL, reg_data16 | 961 RTASE_TX_NEW_DESC_FORMAT_EN); 962 963 /* tx fetch desc number */ 964 rtase_w8(tp, RTASE_TDFNR, 0x10); 965 966 /* tag num select */ 967 reg_data16 = rtase_r16(tp, RTASE_MTPS); 968 u16p_replace_bits(®_data16, 0x4, RTASE_TAG_NUM_SEL_MASK); 969 rtase_w16(tp, RTASE_MTPS, reg_data16); 970 971 rtase_set_tx_queue(tp); 972 973 rtase_w16(tp, RTASE_TOKSEL, 0x5555); 974 975 rtase_tally_counter_addr_fill(tp); 976 rtase_desc_addr_fill(tp); 977 rtase_hw_set_features(dev, dev->features); 978 979 /* enable flow control */ 980 reg_data16 = rtase_r16(tp, RTASE_GPHY_STD_00); 981 reg_data16 &= ~(RTASE_TXFLOW_EN | RTASE_RXFLOW_EN); 982 rtase_w16(tp, RTASE_GPHY_STD_00, reg_data16); 983 reg_data16 = rtase_r16(tp, RTASE_CPLUS_CMD); 984 reg_data16 |= (RTASE_FORCE_TXFLOW_EN | RTASE_FORCE_RXFLOW_EN); 985 rtase_w16(tp, RTASE_CPLUS_CMD, reg_data16); 986 /* set near fifo threshold - rx missed issue. */ 987 rtase_w16(tp, RTASE_RFIFONFULL, 0x190); 988 989 rtase_w16(tp, RTASE_RMS, tp->rx_buf_sz); 990 991 rtase_hw_set_rx_packet_filter(dev); 992 } 993 994 static void rtase_nic_enable(const struct net_device *dev) 995 { 996 const struct rtase_private *tp = netdev_priv(dev); 997 u16 rcr = rtase_r16(tp, RTASE_RX_CONFIG_1); 998 u8 val; 999 1000 rtase_w16(tp, RTASE_RX_CONFIG_1, rcr & ~RTASE_PCIE_RELOAD_EN); 1001 rtase_w16(tp, RTASE_RX_CONFIG_1, rcr | RTASE_PCIE_RELOAD_EN); 1002 1003 val = rtase_r8(tp, RTASE_CHIP_CMD); 1004 rtase_w8(tp, RTASE_CHIP_CMD, val | RTASE_TE | RTASE_RE); 1005 1006 val = rtase_r8(tp, RTASE_MISC); 1007 rtase_w8(tp, RTASE_MISC, val & ~RTASE_RX_DV_GATE_EN); 1008 } 1009 1010 static void rtase_enable_hw_interrupt(const struct rtase_private *tp) 1011 { 1012 const struct rtase_int_vector *ivec = &tp->int_vector[0]; 1013 u32 i; 1014 1015 rtase_w32(tp, ivec->imr_addr, ivec->imr); 1016 1017 for (i = 1; i < tp->int_nums; i++) { 1018 ivec = &tp->int_vector[i]; 1019 rtase_w16(tp, ivec->imr_addr, ivec->imr); 1020 } 1021 } 1022 1023 static void rtase_hw_start(const struct net_device *dev) 1024 { 1025 const struct rtase_private *tp = netdev_priv(dev); 1026 1027 rtase_nic_enable(dev); 1028 rtase_enable_hw_interrupt(tp); 1029 } 1030 1031 /* the interrupt handler does RXQ0 and TXQ0, TXQ4~7 interrutp status 1032 */ 1033 static irqreturn_t rtase_interrupt(int irq, void *dev_instance) 1034 { 1035 const struct rtase_private *tp; 1036 struct rtase_int_vector *ivec; 1037 u32 status; 1038 1039 ivec = dev_instance; 1040 tp = ivec->tp; 1041 status = rtase_r32(tp, ivec->isr_addr); 1042 1043 rtase_w32(tp, ivec->imr_addr, 0x0); 1044 rtase_w32(tp, ivec->isr_addr, status & ~RTASE_FOVW); 1045 1046 if (napi_schedule_prep(&ivec->napi)) 1047 __napi_schedule(&ivec->napi); 1048 1049 return IRQ_HANDLED; 1050 } 1051 1052 /* the interrupt handler does RXQ1&TXQ1 or RXQ2&TXQ2 or RXQ3&TXQ3 interrupt 1053 * status according to interrupt vector 1054 */ 1055 static irqreturn_t rtase_q_interrupt(int irq, void *dev_instance) 1056 { 1057 const struct rtase_private *tp; 1058 struct rtase_int_vector *ivec; 1059 u16 status; 1060 1061 ivec = dev_instance; 1062 tp = ivec->tp; 1063 status = rtase_r16(tp, ivec->isr_addr); 1064 1065 rtase_w16(tp, ivec->imr_addr, 0x0); 1066 rtase_w16(tp, ivec->isr_addr, status); 1067 1068 if (napi_schedule_prep(&ivec->napi)) 1069 __napi_schedule(&ivec->napi); 1070 1071 return IRQ_HANDLED; 1072 } 1073 1074 static int rtase_poll(struct napi_struct *napi, int budget) 1075 { 1076 const struct rtase_int_vector *ivec; 1077 const struct rtase_private *tp; 1078 struct rtase_ring *ring; 1079 int total_workdone = 0; 1080 1081 ivec = container_of(napi, struct rtase_int_vector, napi); 1082 tp = ivec->tp; 1083 1084 list_for_each_entry(ring, &ivec->ring_list, ring_entry) 1085 total_workdone += ring->ring_handler(ring, budget); 1086 1087 if (total_workdone >= budget) 1088 return budget; 1089 1090 if (napi_complete_done(napi, total_workdone)) { 1091 if (!ivec->index) 1092 rtase_w32(tp, ivec->imr_addr, ivec->imr); 1093 else 1094 rtase_w16(tp, ivec->imr_addr, ivec->imr); 1095 } 1096 1097 return total_workdone; 1098 } 1099 1100 static int rtase_open(struct net_device *dev) 1101 { 1102 struct rtase_private *tp = netdev_priv(dev); 1103 const struct pci_dev *pdev = tp->pdev; 1104 struct rtase_int_vector *ivec; 1105 u16 i = 0, j; 1106 int ret; 1107 1108 ivec = &tp->int_vector[0]; 1109 tp->rx_buf_sz = RTASE_RX_BUF_SIZE; 1110 1111 ret = rtase_alloc_desc(tp); 1112 if (ret) 1113 return ret; 1114 1115 ret = rtase_init_ring(dev); 1116 if (ret) 1117 goto err_free_all_allocated_mem; 1118 1119 rtase_hw_config(dev); 1120 1121 if (tp->sw_flag & RTASE_SWF_MSIX_ENABLED) { 1122 ret = request_irq(ivec->irq, rtase_interrupt, 0, 1123 dev->name, ivec); 1124 if (ret) 1125 goto err_free_all_allocated_irq; 1126 1127 /* request other interrupts to handle multiqueue */ 1128 for (i = 1; i < tp->int_nums; i++) { 1129 ivec = &tp->int_vector[i]; 1130 snprintf(ivec->name, sizeof(ivec->name), "%s_int%u", 1131 tp->dev->name, i); 1132 ret = request_irq(ivec->irq, rtase_q_interrupt, 0, 1133 ivec->name, ivec); 1134 if (ret) 1135 goto err_free_all_allocated_irq; 1136 } 1137 } else { 1138 ret = request_irq(pdev->irq, rtase_interrupt, 0, dev->name, 1139 ivec); 1140 if (ret) 1141 goto err_free_all_allocated_mem; 1142 } 1143 1144 rtase_hw_start(dev); 1145 1146 for (i = 0; i < tp->int_nums; i++) { 1147 ivec = &tp->int_vector[i]; 1148 napi_enable(&ivec->napi); 1149 } 1150 1151 netif_carrier_on(dev); 1152 netif_wake_queue(dev); 1153 1154 return 0; 1155 1156 err_free_all_allocated_irq: 1157 for (j = 0; j < i; j++) 1158 free_irq(tp->int_vector[j].irq, &tp->int_vector[j]); 1159 1160 err_free_all_allocated_mem: 1161 rtase_free_desc(tp); 1162 1163 return ret; 1164 } 1165 1166 static void rtase_down(struct net_device *dev) 1167 { 1168 struct rtase_private *tp = netdev_priv(dev); 1169 struct rtase_int_vector *ivec; 1170 struct rtase_ring *ring, *tmp; 1171 u32 i; 1172 1173 for (i = 0; i < tp->int_nums; i++) { 1174 ivec = &tp->int_vector[i]; 1175 napi_disable(&ivec->napi); 1176 list_for_each_entry_safe(ring, tmp, &ivec->ring_list, 1177 ring_entry) { 1178 netif_queue_set_napi(tp->dev, ring->index, 1179 ring->type, NULL); 1180 1181 list_del(&ring->ring_entry); 1182 } 1183 } 1184 1185 netif_tx_disable(dev); 1186 1187 netif_carrier_off(dev); 1188 1189 rtase_hw_reset(dev); 1190 1191 rtase_tx_clear(tp); 1192 1193 rtase_rx_clear(tp); 1194 } 1195 1196 static int rtase_close(struct net_device *dev) 1197 { 1198 struct rtase_private *tp = netdev_priv(dev); 1199 const struct pci_dev *pdev = tp->pdev; 1200 u32 i; 1201 1202 rtase_down(dev); 1203 1204 if (tp->sw_flag & RTASE_SWF_MSIX_ENABLED) { 1205 for (i = 0; i < tp->int_nums; i++) 1206 free_irq(tp->int_vector[i].irq, &tp->int_vector[i]); 1207 1208 } else { 1209 free_irq(pdev->irq, &tp->int_vector[0]); 1210 } 1211 1212 rtase_free_desc(tp); 1213 1214 return 0; 1215 } 1216 1217 static u32 rtase_tx_vlan_tag(const struct rtase_private *tp, 1218 const struct sk_buff *skb) 1219 { 1220 return (skb_vlan_tag_present(skb)) ? 1221 (RTASE_TX_VLAN_TAG | swab16(skb_vlan_tag_get(skb))) : 0x00; 1222 } 1223 1224 static u32 rtase_tx_csum(struct sk_buff *skb, const struct net_device *dev) 1225 { 1226 u32 csum_cmd = 0; 1227 u8 ip_protocol; 1228 1229 switch (vlan_get_protocol(skb)) { 1230 case htons(ETH_P_IP): 1231 csum_cmd = RTASE_TX_IPCS_C; 1232 ip_protocol = ip_hdr(skb)->protocol; 1233 break; 1234 1235 case htons(ETH_P_IPV6): 1236 csum_cmd = RTASE_TX_IPV6F_C; 1237 ip_protocol = ipv6_hdr(skb)->nexthdr; 1238 break; 1239 1240 default: 1241 ip_protocol = IPPROTO_RAW; 1242 break; 1243 } 1244 1245 if (ip_protocol == IPPROTO_TCP) 1246 csum_cmd |= RTASE_TX_TCPCS_C; 1247 else if (ip_protocol == IPPROTO_UDP) 1248 csum_cmd |= RTASE_TX_UDPCS_C; 1249 1250 csum_cmd |= u32_encode_bits(skb_transport_offset(skb), 1251 RTASE_TCPHO_MASK); 1252 1253 return csum_cmd; 1254 } 1255 1256 static enum rtase_parse_result rtase_get_l3_proto(struct sk_buff *skb, 1257 __be16 *proto, 1258 u32 *network_offset) 1259 { 1260 struct vlan_hdr *vh, _vh; 1261 struct ethhdr *eh, _eh; 1262 u32 offset = ETH_HLEN; 1263 1264 eh = skb_header_pointer(skb, 0, sizeof(_eh), &_eh); 1265 if (!eh) 1266 return RTASE_PARSE_DROP; 1267 1268 *proto = eh->h_proto; 1269 1270 while (eth_type_vlan(*proto)) { 1271 vh = skb_header_pointer(skb, offset, sizeof(_vh), &_vh); 1272 if (!vh) 1273 return RTASE_PARSE_DROP; 1274 1275 *proto = vh->h_vlan_encapsulated_proto; 1276 offset += VLAN_HLEN; 1277 } 1278 1279 *network_offset = offset; 1280 1281 return RTASE_PARSE_OK; 1282 } 1283 1284 static bool rtase_pad_to_transport_len(struct sk_buff *skb, 1285 u32 transport_offset, 1286 u32 pad_to_len) 1287 { 1288 u32 trans_data_len; 1289 u32 pad_len; 1290 1291 trans_data_len = skb->len - transport_offset; 1292 if (trans_data_len >= pad_to_len) 1293 return true; 1294 1295 if (skb_is_nonlinear(skb)) { 1296 if (skb_linearize(skb)) 1297 return false; 1298 } 1299 1300 pad_len = pad_to_len - trans_data_len; 1301 if (__skb_put_padto(skb, skb->len + pad_len, false)) 1302 return false; 1303 1304 return true; 1305 } 1306 1307 static enum rtase_parse_result rtase_get_transport_offset(struct sk_buff *skb, 1308 u32 *transport_offset, 1309 u8 *transport_proto, 1310 u32 *pad_to_len) 1311 { 1312 enum rtase_parse_result ret; 1313 struct ipv6hdr *i6h, _i6h; 1314 struct iphdr *ih, _ih; 1315 bool non_first_frag; 1316 __be16 proto; 1317 u32 offset; 1318 u32 no; 1319 1320 ret = rtase_get_l3_proto(skb, &proto, &no); 1321 if (ret != RTASE_PARSE_OK) 1322 return ret; 1323 1324 switch (proto) { 1325 case htons(ETH_P_IP): 1326 ih = skb_header_pointer(skb, no, sizeof(_ih), &_ih); 1327 if (!ih) 1328 return RTASE_PARSE_DROP; 1329 1330 if (ih->ihl < 5) 1331 return RTASE_PARSE_DROP; 1332 1333 offset = no + ih->ihl * 4; 1334 if (offset > skb->len) 1335 return RTASE_PARSE_DROP; 1336 1337 non_first_frag = ntohs(ih->frag_off) & IP_OFFSET; 1338 1339 if (ih->protocol == IPPROTO_TCP) { 1340 if (skb->len - offset < sizeof(struct tcphdr)) { 1341 if (non_first_frag) { 1342 *transport_offset = offset; 1343 *transport_proto = IPPROTO_TCP; 1344 *pad_to_len = sizeof(struct tcphdr); 1345 1346 return RTASE_PARSE_OK; 1347 } 1348 1349 return RTASE_PARSE_DROP; 1350 } 1351 1352 return RTASE_PARSE_SKIP; 1353 } 1354 1355 if (ih->protocol != IPPROTO_UDP) 1356 return RTASE_PARSE_SKIP; 1357 1358 *transport_offset = offset; 1359 *transport_proto = IPPROTO_UDP; 1360 1361 if (skb->len - offset < sizeof(struct udphdr)) { 1362 if (non_first_frag) { 1363 *pad_to_len = sizeof(struct udphdr); 1364 1365 return RTASE_PARSE_OK; 1366 } 1367 1368 return RTASE_PARSE_DROP; 1369 } 1370 1371 return RTASE_PARSE_OK; 1372 1373 case htons(ETH_P_IPV6): 1374 i6h = skb_header_pointer(skb, no, sizeof(_i6h), &_i6h); 1375 if (!i6h) 1376 return RTASE_PARSE_DROP; 1377 1378 offset = no + sizeof(*i6h); 1379 1380 if (i6h->nexthdr == IPPROTO_TCP) { 1381 if (skb->len - offset < sizeof(struct tcphdr)) 1382 return RTASE_PARSE_DROP; 1383 1384 return RTASE_PARSE_SKIP; 1385 } 1386 1387 if (i6h->nexthdr != IPPROTO_UDP) 1388 return RTASE_PARSE_SKIP; 1389 1390 if (skb->len - offset < sizeof(struct udphdr)) 1391 return RTASE_PARSE_DROP; 1392 1393 *transport_offset = offset; 1394 *transport_proto = IPPROTO_UDP; 1395 1396 return RTASE_PARSE_OK; 1397 1398 default: 1399 return RTASE_PARSE_SKIP; 1400 } 1401 } 1402 1403 static bool rtase_skb_pad(struct sk_buff *skb) 1404 { 1405 enum rtase_parse_result ret; 1406 u32 transport_offset; 1407 __be16 *dest, _dest; 1408 u32 trans_data_len; 1409 u32 pad_to_len = 0; 1410 u8 transport_proto; 1411 u16 dest_port; 1412 1413 ret = rtase_get_transport_offset(skb, &transport_offset, 1414 &transport_proto, &pad_to_len); 1415 if (ret == RTASE_PARSE_SKIP) { 1416 return true; 1417 } else if (ret == RTASE_PARSE_DROP) { 1418 netdev_dbg(skb->dev, "drop malformed packet\n"); 1419 return false; 1420 } 1421 1422 if (pad_to_len && 1423 !rtase_pad_to_transport_len(skb, transport_offset, pad_to_len)) 1424 return false; 1425 1426 if (transport_proto != IPPROTO_UDP) 1427 return true; 1428 1429 trans_data_len = skb->len - transport_offset; 1430 if (trans_data_len < offsetof(struct udphdr, len) || 1431 trans_data_len >= RTASE_MIN_PAD_LEN) 1432 return true; 1433 1434 dest = skb_header_pointer(skb, 1435 transport_offset + 1436 offsetof(struct udphdr, dest), 1437 sizeof(_dest), &_dest); 1438 if (!dest) 1439 return true; 1440 1441 dest_port = ntohs(*dest); 1442 if (dest_port != PTP_EV_PORT && dest_port != PTP_GEN_PORT) 1443 return true; 1444 1445 return rtase_pad_to_transport_len(skb, transport_offset, 1446 RTASE_MIN_PAD_LEN); 1447 } 1448 1449 static int rtase_xmit_frags(struct rtase_ring *ring, struct sk_buff *skb, 1450 u32 opts1, u32 opts2) 1451 { 1452 const struct skb_shared_info *info = skb_shinfo(skb); 1453 const struct rtase_private *tp = ring->ivec->tp; 1454 const u8 nr_frags = info->nr_frags; 1455 struct rtase_tx_desc *txd = NULL; 1456 u32 cur_frag, entry; 1457 1458 entry = ring->cur_idx; 1459 for (cur_frag = 0; cur_frag < nr_frags; cur_frag++) { 1460 const skb_frag_t *frag = &info->frags[cur_frag]; 1461 dma_addr_t mapping; 1462 u32 status, len; 1463 void *addr; 1464 1465 entry = (entry + 1) % RTASE_NUM_DESC; 1466 1467 txd = ring->desc + sizeof(struct rtase_tx_desc) * entry; 1468 len = skb_frag_size(frag); 1469 addr = skb_frag_address(frag); 1470 mapping = dma_map_single(&tp->pdev->dev, addr, len, 1471 DMA_TO_DEVICE); 1472 1473 if (unlikely(dma_mapping_error(&tp->pdev->dev, mapping))) { 1474 if (unlikely(net_ratelimit())) 1475 netdev_err(tp->dev, 1476 "Failed to map TX fragments DMA!\n"); 1477 1478 goto err_out; 1479 } 1480 1481 if (((entry + 1) % RTASE_NUM_DESC) == 0) 1482 status = (opts1 | len | RTASE_RING_END); 1483 else 1484 status = opts1 | len; 1485 1486 if (cur_frag == (nr_frags - 1)) { 1487 ring->skbuff[entry] = skb; 1488 status |= RTASE_TX_LAST_FRAG; 1489 } 1490 1491 ring->mis.len[entry] = len; 1492 txd->addr = cpu_to_le64(mapping); 1493 txd->opts2 = cpu_to_le32(opts2); 1494 1495 /* make sure the operating fields have been updated */ 1496 dma_wmb(); 1497 txd->opts1 = cpu_to_le32(status); 1498 } 1499 1500 return cur_frag; 1501 1502 err_out: 1503 rtase_tx_clear_range(ring, ring->cur_idx + 1, cur_frag); 1504 return -EIO; 1505 } 1506 1507 static netdev_tx_t rtase_start_xmit(struct sk_buff *skb, 1508 struct net_device *dev) 1509 { 1510 struct skb_shared_info *shinfo = skb_shinfo(skb); 1511 struct rtase_private *tp = netdev_priv(dev); 1512 u32 q_idx, entry, len, opts1, opts2; 1513 struct netdev_queue *tx_queue; 1514 bool stop_queue, door_bell; 1515 u32 mss = shinfo->gso_size; 1516 struct rtase_tx_desc *txd; 1517 struct rtase_ring *ring; 1518 dma_addr_t mapping; 1519 int frags; 1520 1521 /* multiqueues */ 1522 q_idx = skb_get_queue_mapping(skb); 1523 ring = &tp->tx_ring[q_idx]; 1524 tx_queue = netdev_get_tx_queue(dev, q_idx); 1525 1526 if (unlikely(!rtase_tx_avail(ring))) { 1527 if (net_ratelimit()) 1528 netdev_err(dev, 1529 "BUG! Tx Ring full when queue awake!\n"); 1530 1531 netif_stop_queue(dev); 1532 return NETDEV_TX_BUSY; 1533 } 1534 1535 entry = ring->cur_idx % RTASE_NUM_DESC; 1536 txd = ring->desc + sizeof(struct rtase_tx_desc) * entry; 1537 1538 opts1 = RTASE_DESC_OWN; 1539 opts2 = rtase_tx_vlan_tag(tp, skb); 1540 1541 /* tcp segmentation offload (or tcp large send) */ 1542 if (mss) { 1543 if (shinfo->gso_type & SKB_GSO_TCPV4) { 1544 opts1 |= RTASE_GIANT_SEND_V4; 1545 } else if (shinfo->gso_type & SKB_GSO_TCPV6) { 1546 if (skb_cow_head(skb, 0)) 1547 goto err_dma_0; 1548 1549 tcp_v6_gso_csum_prep(skb); 1550 opts1 |= RTASE_GIANT_SEND_V6; 1551 } else { 1552 WARN_ON_ONCE(1); 1553 } 1554 1555 opts1 |= u32_encode_bits(skb_transport_offset(skb), 1556 RTASE_TCPHO_MASK); 1557 opts2 |= u32_encode_bits(mss, RTASE_MSS_MASK); 1558 } else if (skb->ip_summed == CHECKSUM_PARTIAL) { 1559 opts2 |= rtase_tx_csum(skb, dev); 1560 } 1561 1562 if (!rtase_skb_pad(skb)) 1563 goto err_dma_0; 1564 1565 frags = rtase_xmit_frags(ring, skb, opts1, opts2); 1566 if (unlikely(frags < 0)) 1567 goto err_dma_0; 1568 1569 if (frags) { 1570 len = skb_headlen(skb); 1571 opts1 |= RTASE_TX_FIRST_FRAG; 1572 } else { 1573 len = skb->len; 1574 ring->skbuff[entry] = skb; 1575 opts1 |= RTASE_TX_FIRST_FRAG | RTASE_TX_LAST_FRAG; 1576 } 1577 1578 if (((entry + 1) % RTASE_NUM_DESC) == 0) 1579 opts1 |= (len | RTASE_RING_END); 1580 else 1581 opts1 |= len; 1582 1583 mapping = dma_map_single(&tp->pdev->dev, skb->data, len, 1584 DMA_TO_DEVICE); 1585 1586 if (unlikely(dma_mapping_error(&tp->pdev->dev, mapping))) { 1587 if (unlikely(net_ratelimit())) 1588 netdev_err(dev, "Failed to map TX DMA!\n"); 1589 1590 goto err_dma_1; 1591 } 1592 1593 ring->mis.len[entry] = len; 1594 txd->addr = cpu_to_le64(mapping); 1595 txd->opts2 = cpu_to_le32(opts2); 1596 txd->opts1 = cpu_to_le32(opts1 & ~RTASE_DESC_OWN); 1597 1598 /* make sure the operating fields have been updated */ 1599 dma_wmb(); 1600 1601 door_bell = __netdev_tx_sent_queue(tx_queue, skb->len, 1602 netdev_xmit_more()); 1603 1604 txd->opts1 = cpu_to_le32(opts1); 1605 1606 skb_tx_timestamp(skb); 1607 1608 /* tx needs to see descriptor changes before updated cur_idx */ 1609 smp_wmb(); 1610 1611 WRITE_ONCE(ring->cur_idx, ring->cur_idx + frags + 1); 1612 1613 stop_queue = !netif_subqueue_maybe_stop(dev, ring->index, 1614 rtase_tx_avail(ring), 1615 RTASE_TX_STOP_THRS, 1616 RTASE_TX_START_THRS); 1617 1618 if (door_bell || stop_queue) 1619 rtase_w8(tp, RTASE_TPPOLL, BIT(ring->index)); 1620 1621 return NETDEV_TX_OK; 1622 1623 err_dma_1: 1624 ring->skbuff[entry] = NULL; 1625 rtase_tx_clear_range(ring, ring->cur_idx + 1, frags); 1626 if (frags) 1627 /* the frags were cleared above, along with the skb */ 1628 return NETDEV_TX_OK; 1629 1630 err_dma_0: 1631 tp->stats.tx_dropped++; 1632 dev_kfree_skb_any(skb); 1633 return NETDEV_TX_OK; 1634 } 1635 1636 static void rtase_set_rx_mode(struct net_device *dev) 1637 { 1638 rtase_hw_set_rx_packet_filter(dev); 1639 } 1640 1641 static void rtase_enable_eem_write(const struct rtase_private *tp) 1642 { 1643 u8 val; 1644 1645 val = rtase_r8(tp, RTASE_EEM); 1646 rtase_w8(tp, RTASE_EEM, val | RTASE_EEM_UNLOCK); 1647 } 1648 1649 static void rtase_disable_eem_write(const struct rtase_private *tp) 1650 { 1651 u8 val; 1652 1653 val = rtase_r8(tp, RTASE_EEM); 1654 rtase_w8(tp, RTASE_EEM, val & ~RTASE_EEM_UNLOCK); 1655 } 1656 1657 static void rtase_rar_set(const struct rtase_private *tp, const u8 *addr) 1658 { 1659 u32 rar_low, rar_high; 1660 1661 rar_low = (u32)addr[0] | ((u32)addr[1] << 8) | 1662 ((u32)addr[2] << 16) | ((u32)addr[3] << 24); 1663 1664 rar_high = (u32)addr[4] | ((u32)addr[5] << 8); 1665 1666 rtase_enable_eem_write(tp); 1667 rtase_w32(tp, RTASE_MAC0, rar_low); 1668 rtase_w32(tp, RTASE_MAC4, rar_high); 1669 rtase_disable_eem_write(tp); 1670 rtase_w16(tp, RTASE_LBK_CTRL, RTASE_LBK_ATLD | RTASE_LBK_CLR); 1671 } 1672 1673 static int rtase_set_mac_address(struct net_device *dev, void *p) 1674 { 1675 struct rtase_private *tp = netdev_priv(dev); 1676 int ret; 1677 1678 ret = eth_mac_addr(dev, p); 1679 if (ret) 1680 return ret; 1681 1682 rtase_rar_set(tp, dev->dev_addr); 1683 1684 return 0; 1685 } 1686 1687 static int rtase_change_mtu(struct net_device *dev, int new_mtu) 1688 { 1689 dev->mtu = new_mtu; 1690 1691 netdev_update_features(dev); 1692 1693 return 0; 1694 } 1695 1696 static void rtase_wait_for_quiescence(const struct net_device *dev) 1697 { 1698 struct rtase_private *tp = netdev_priv(dev); 1699 struct rtase_int_vector *ivec; 1700 u32 i; 1701 1702 for (i = 0; i < tp->int_nums; i++) { 1703 ivec = &tp->int_vector[i]; 1704 synchronize_irq(ivec->irq); 1705 /* wait for any pending NAPI task to complete */ 1706 napi_disable(&ivec->napi); 1707 } 1708 1709 rtase_irq_dis_and_clear(tp); 1710 1711 for (i = 0; i < tp->int_nums; i++) { 1712 ivec = &tp->int_vector[i]; 1713 napi_enable(&ivec->napi); 1714 } 1715 } 1716 1717 static void rtase_sw_reset(struct net_device *dev) 1718 { 1719 struct rtase_private *tp = netdev_priv(dev); 1720 struct rtase_ring *ring, *tmp; 1721 struct rtase_int_vector *ivec; 1722 int ret; 1723 u32 i; 1724 1725 netif_stop_queue(dev); 1726 netif_carrier_off(dev); 1727 rtase_hw_reset(dev); 1728 1729 /* let's wait a bit while any (async) irq lands on */ 1730 rtase_wait_for_quiescence(dev); 1731 rtase_tx_clear(tp); 1732 rtase_rx_clear(tp); 1733 1734 for (i = 0; i < tp->int_nums; i++) { 1735 ivec = &tp->int_vector[i]; 1736 list_for_each_entry_safe(ring, tmp, &ivec->ring_list, 1737 ring_entry) { 1738 netif_queue_set_napi(tp->dev, ring->index, 1739 ring->type, NULL); 1740 1741 list_del(&ring->ring_entry); 1742 } 1743 } 1744 1745 ret = rtase_init_ring(dev); 1746 if (ret) { 1747 netdev_err(dev, "unable to init ring\n"); 1748 rtase_free_desc(tp); 1749 return; 1750 } 1751 1752 rtase_hw_config(dev); 1753 /* always link, so start to transmit & receive */ 1754 rtase_hw_start(dev); 1755 1756 netif_carrier_on(dev); 1757 netif_wake_queue(dev); 1758 } 1759 1760 static void rtase_dump_tally_counter(const struct rtase_private *tp) 1761 { 1762 dma_addr_t paddr = tp->tally_paddr; 1763 u32 cmd = lower_32_bits(paddr); 1764 u32 val; 1765 int err; 1766 1767 rtase_w32(tp, RTASE_DTCCR4, upper_32_bits(paddr)); 1768 rtase_w32(tp, RTASE_DTCCR0, cmd); 1769 rtase_w32(tp, RTASE_DTCCR0, cmd | RTASE_COUNTER_DUMP); 1770 1771 err = read_poll_timeout_atomic(rtase_r32, val, 1772 !(val & RTASE_COUNTER_DUMP), 1773 10, 250, false, tp, RTASE_DTCCR0); 1774 1775 if (err == -ETIMEDOUT) 1776 netdev_err(tp->dev, "error occurred in dump tally counter\n"); 1777 } 1778 1779 static void rtase_dump_state(const struct net_device *dev) 1780 { 1781 const struct rtase_private *tp = netdev_priv(dev); 1782 int max_reg_size = RTASE_PCI_REGS_SIZE; 1783 const struct rtase_counters *counters; 1784 const struct rtase_ring *ring; 1785 u32 dword_rd; 1786 int n = 0; 1787 1788 ring = &tp->tx_ring[0]; 1789 netdev_err(dev, "Tx descriptor info:\n"); 1790 netdev_err(dev, "Tx curIdx = 0x%x\n", ring->cur_idx); 1791 netdev_err(dev, "Tx dirtyIdx = 0x%x\n", ring->dirty_idx); 1792 netdev_err(dev, "Tx phyAddr = %pad\n", &ring->phy_addr); 1793 1794 ring = &tp->rx_ring[0]; 1795 netdev_err(dev, "Rx descriptor info:\n"); 1796 netdev_err(dev, "Rx curIdx = 0x%x\n", ring->cur_idx); 1797 netdev_err(dev, "Rx dirtyIdx = 0x%x\n", ring->dirty_idx); 1798 netdev_err(dev, "Rx phyAddr = %pad\n", &ring->phy_addr); 1799 1800 netdev_err(dev, "Device Registers:\n"); 1801 netdev_err(dev, "Chip Command = 0x%02x\n", 1802 rtase_r8(tp, RTASE_CHIP_CMD)); 1803 netdev_err(dev, "IMR = %08x\n", rtase_r32(tp, RTASE_IMR0)); 1804 netdev_err(dev, "ISR = %08x\n", rtase_r32(tp, RTASE_ISR0)); 1805 netdev_err(dev, "Boot Ctrl Reg(0xE004) = %04x\n", 1806 rtase_r16(tp, RTASE_BOOT_CTL)); 1807 netdev_err(dev, "EPHY ISR(0xE014) = %04x\n", 1808 rtase_r16(tp, RTASE_EPHY_ISR)); 1809 netdev_err(dev, "EPHY IMR(0xE016) = %04x\n", 1810 rtase_r16(tp, RTASE_EPHY_IMR)); 1811 netdev_err(dev, "CLKSW SET REG(0xE018) = %04x\n", 1812 rtase_r16(tp, RTASE_CLKSW_SET)); 1813 1814 netdev_err(dev, "Dump PCI Registers:\n"); 1815 1816 while (n < max_reg_size) { 1817 if ((n % RTASE_DWORD_MOD) == 0) 1818 netdev_err(tp->dev, "0x%03x:\n", n); 1819 1820 pci_read_config_dword(tp->pdev, n, &dword_rd); 1821 netdev_err(tp->dev, "%08x\n", dword_rd); 1822 n += 4; 1823 } 1824 1825 netdev_err(dev, "Dump tally counter:\n"); 1826 counters = tp->tally_vaddr; 1827 rtase_dump_tally_counter(tp); 1828 1829 netdev_err(dev, "tx_packets %lld\n", 1830 le64_to_cpu(counters->tx_packets)); 1831 netdev_err(dev, "rx_packets %lld\n", 1832 le64_to_cpu(counters->rx_packets)); 1833 netdev_err(dev, "tx_errors %lld\n", 1834 le64_to_cpu(counters->tx_errors)); 1835 netdev_err(dev, "rx_errors %d\n", 1836 le32_to_cpu(counters->rx_errors)); 1837 netdev_err(dev, "rx_missed %d\n", 1838 le16_to_cpu(counters->rx_missed)); 1839 netdev_err(dev, "align_errors %d\n", 1840 le16_to_cpu(counters->align_errors)); 1841 netdev_err(dev, "tx_one_collision %d\n", 1842 le32_to_cpu(counters->tx_one_collision)); 1843 netdev_err(dev, "tx_multi_collision %d\n", 1844 le32_to_cpu(counters->tx_multi_collision)); 1845 netdev_err(dev, "rx_unicast %lld\n", 1846 le64_to_cpu(counters->rx_unicast)); 1847 netdev_err(dev, "rx_broadcast %lld\n", 1848 le64_to_cpu(counters->rx_broadcast)); 1849 netdev_err(dev, "rx_multicast %d\n", 1850 le32_to_cpu(counters->rx_multicast)); 1851 netdev_err(dev, "tx_aborted %d\n", 1852 le16_to_cpu(counters->tx_aborted)); 1853 netdev_err(dev, "tx_underrun %d\n", 1854 le16_to_cpu(counters->tx_underrun)); 1855 } 1856 1857 static void rtase_tx_timeout(struct net_device *dev, unsigned int txqueue) 1858 { 1859 rtase_dump_state(dev); 1860 rtase_sw_reset(dev); 1861 } 1862 1863 static void rtase_get_stats64(struct net_device *dev, 1864 struct rtnl_link_stats64 *stats) 1865 { 1866 const struct rtase_private *tp = netdev_priv(dev); 1867 const struct rtase_counters *counters; 1868 1869 counters = tp->tally_vaddr; 1870 1871 dev_fetch_sw_netstats(stats, dev->tstats); 1872 1873 /* fetch additional counter values missing in stats collected by driver 1874 * from tally counter 1875 */ 1876 rtase_dump_tally_counter(tp); 1877 stats->rx_errors = tp->stats.rx_errors; 1878 stats->tx_errors = le64_to_cpu(counters->tx_errors); 1879 stats->rx_dropped = tp->stats.rx_dropped; 1880 stats->tx_dropped = tp->stats.tx_dropped; 1881 stats->multicast = tp->stats.multicast; 1882 stats->rx_length_errors = tp->stats.rx_length_errors; 1883 } 1884 1885 static void rtase_set_hw_cbs(const struct rtase_private *tp, u32 queue) 1886 { 1887 u32 idle = tp->tx_qos[queue].idleslope * RTASE_1T_CLOCK; 1888 u32 val, i; 1889 1890 val = u32_encode_bits(idle / RTASE_1T_POWER, RTASE_IDLESLOPE_INT_MASK); 1891 idle %= RTASE_1T_POWER; 1892 1893 for (i = 1; i <= RTASE_IDLESLOPE_INT_SHIFT; i++) { 1894 idle *= 2; 1895 if ((idle / RTASE_1T_POWER) == 1) 1896 val |= BIT(RTASE_IDLESLOPE_INT_SHIFT - i); 1897 1898 idle %= RTASE_1T_POWER; 1899 } 1900 1901 rtase_w32(tp, RTASE_TXQCRDT_0 + queue * 4, val); 1902 } 1903 1904 static int rtase_setup_tc_cbs(struct rtase_private *tp, 1905 const struct tc_cbs_qopt_offload *qopt) 1906 { 1907 int queue = qopt->queue; 1908 1909 if (queue < 0 || queue >= tp->func_tx_queue_num) 1910 return -EINVAL; 1911 1912 if (!qopt->enable) { 1913 tp->tx_qos[queue].hicredit = 0; 1914 tp->tx_qos[queue].locredit = 0; 1915 tp->tx_qos[queue].idleslope = 0; 1916 tp->tx_qos[queue].sendslope = 0; 1917 1918 rtase_w32(tp, RTASE_TXQCRDT_0 + queue * 4, 0); 1919 } else { 1920 tp->tx_qos[queue].hicredit = qopt->hicredit; 1921 tp->tx_qos[queue].locredit = qopt->locredit; 1922 tp->tx_qos[queue].idleslope = qopt->idleslope; 1923 tp->tx_qos[queue].sendslope = qopt->sendslope; 1924 1925 rtase_set_hw_cbs(tp, queue); 1926 } 1927 1928 return 0; 1929 } 1930 1931 static int rtase_setup_tc(struct net_device *dev, enum tc_setup_type type, 1932 void *type_data) 1933 { 1934 struct rtase_private *tp = netdev_priv(dev); 1935 1936 switch (type) { 1937 case TC_SETUP_QDISC_CBS: 1938 return rtase_setup_tc_cbs(tp, type_data); 1939 default: 1940 return -EOPNOTSUPP; 1941 } 1942 } 1943 1944 static netdev_features_t rtase_fix_features(struct net_device *dev, 1945 netdev_features_t features) 1946 { 1947 netdev_features_t features_fix = features; 1948 1949 /* not support TSO for jumbo frames */ 1950 if (dev->mtu > ETH_DATA_LEN) 1951 features_fix &= ~NETIF_F_ALL_TSO; 1952 1953 return features_fix; 1954 } 1955 1956 static int rtase_set_features(struct net_device *dev, 1957 netdev_features_t features) 1958 { 1959 netdev_features_t features_set = features; 1960 1961 features_set &= NETIF_F_RXALL | NETIF_F_RXCSUM | 1962 NETIF_F_HW_VLAN_CTAG_RX; 1963 1964 if (features_set ^ dev->features) 1965 rtase_hw_set_features(dev, features_set); 1966 1967 return 0; 1968 } 1969 1970 static const struct net_device_ops rtase_netdev_ops = { 1971 .ndo_open = rtase_open, 1972 .ndo_stop = rtase_close, 1973 .ndo_start_xmit = rtase_start_xmit, 1974 .ndo_set_rx_mode = rtase_set_rx_mode, 1975 .ndo_set_mac_address = rtase_set_mac_address, 1976 .ndo_change_mtu = rtase_change_mtu, 1977 .ndo_tx_timeout = rtase_tx_timeout, 1978 .ndo_get_stats64 = rtase_get_stats64, 1979 .ndo_setup_tc = rtase_setup_tc, 1980 .ndo_fix_features = rtase_fix_features, 1981 .ndo_set_features = rtase_set_features, 1982 }; 1983 1984 static void rtase_get_mac_address(struct net_device *dev) 1985 { 1986 struct rtase_private *tp = netdev_priv(dev); 1987 u8 mac_addr[ETH_ALEN] __aligned(2) = {}; 1988 u32 i; 1989 1990 for (i = 0; i < ETH_ALEN; i++) 1991 mac_addr[i] = rtase_r8(tp, RTASE_MAC0 + i); 1992 1993 if (!is_valid_ether_addr(mac_addr)) { 1994 eth_hw_addr_random(dev); 1995 netdev_warn(dev, "Random ether addr %pM\n", dev->dev_addr); 1996 } else { 1997 eth_hw_addr_set(dev, mac_addr); 1998 ether_addr_copy(dev->perm_addr, dev->dev_addr); 1999 } 2000 2001 rtase_rar_set(tp, dev->dev_addr); 2002 } 2003 2004 static int rtase_get_settings(struct net_device *dev, 2005 struct ethtool_link_ksettings *cmd) 2006 { 2007 u32 supported = SUPPORTED_MII | SUPPORTED_Pause | SUPPORTED_Asym_Pause; 2008 const struct rtase_private *tp = netdev_priv(dev); 2009 2010 ethtool_convert_legacy_u32_to_link_mode(cmd->link_modes.supported, 2011 supported); 2012 2013 switch (tp->hw_ver) { 2014 case RTASE_HW_VER_906X_7XA: 2015 case RTASE_HW_VER_906X_7XC: 2016 cmd->base.speed = SPEED_5000; 2017 break; 2018 case RTASE_HW_VER_907XD_V1: 2019 case RTASE_HW_VER_907XD_VA: 2020 cmd->base.speed = SPEED_10000; 2021 break; 2022 } 2023 2024 cmd->base.duplex = DUPLEX_FULL; 2025 cmd->base.port = PORT_MII; 2026 cmd->base.autoneg = AUTONEG_DISABLE; 2027 2028 return 0; 2029 } 2030 2031 static void rtase_get_pauseparam(struct net_device *dev, 2032 struct ethtool_pauseparam *pause) 2033 { 2034 const struct rtase_private *tp = netdev_priv(dev); 2035 u16 value = rtase_r16(tp, RTASE_CPLUS_CMD); 2036 2037 pause->autoneg = AUTONEG_DISABLE; 2038 pause->tx_pause = !!(value & RTASE_FORCE_TXFLOW_EN); 2039 pause->rx_pause = !!(value & RTASE_FORCE_RXFLOW_EN); 2040 } 2041 2042 static int rtase_set_pauseparam(struct net_device *dev, 2043 struct ethtool_pauseparam *pause) 2044 { 2045 const struct rtase_private *tp = netdev_priv(dev); 2046 u16 value = rtase_r16(tp, RTASE_CPLUS_CMD); 2047 2048 if (pause->autoneg) 2049 return -EOPNOTSUPP; 2050 2051 value &= ~(RTASE_FORCE_TXFLOW_EN | RTASE_FORCE_RXFLOW_EN); 2052 2053 if (pause->tx_pause) 2054 value |= RTASE_FORCE_TXFLOW_EN; 2055 2056 if (pause->rx_pause) 2057 value |= RTASE_FORCE_RXFLOW_EN; 2058 2059 rtase_w16(tp, RTASE_CPLUS_CMD, value); 2060 return 0; 2061 } 2062 2063 static void rtase_get_eth_mac_stats(struct net_device *dev, 2064 struct ethtool_eth_mac_stats *stats) 2065 { 2066 struct rtase_private *tp = netdev_priv(dev); 2067 const struct rtase_counters *counters; 2068 2069 counters = tp->tally_vaddr; 2070 2071 rtase_dump_tally_counter(tp); 2072 2073 stats->FramesTransmittedOK = le64_to_cpu(counters->tx_packets); 2074 stats->FramesReceivedOK = le64_to_cpu(counters->rx_packets); 2075 stats->FramesLostDueToIntMACXmitError = 2076 le64_to_cpu(counters->tx_errors); 2077 stats->BroadcastFramesReceivedOK = le64_to_cpu(counters->rx_broadcast); 2078 } 2079 2080 static const struct ethtool_ops rtase_ethtool_ops = { 2081 .get_link = ethtool_op_get_link, 2082 .get_link_ksettings = rtase_get_settings, 2083 .get_pauseparam = rtase_get_pauseparam, 2084 .set_pauseparam = rtase_set_pauseparam, 2085 .get_eth_mac_stats = rtase_get_eth_mac_stats, 2086 .get_ts_info = ethtool_op_get_ts_info, 2087 }; 2088 2089 static void rtase_init_netdev_ops(struct net_device *dev) 2090 { 2091 dev->netdev_ops = &rtase_netdev_ops; 2092 dev->ethtool_ops = &rtase_ethtool_ops; 2093 } 2094 2095 static void rtase_init_napi(struct rtase_private *tp) 2096 { 2097 u16 i; 2098 2099 for (i = 0; i < tp->int_nums; i++) { 2100 netif_napi_add_config(tp->dev, &tp->int_vector[i].napi, 2101 tp->int_vector[i].poll, i); 2102 netif_napi_set_irq(&tp->int_vector[i].napi, 2103 tp->int_vector[i].irq); 2104 } 2105 } 2106 2107 static void rtase_reset_interrupt(struct pci_dev *pdev, 2108 const struct rtase_private *tp) 2109 { 2110 if (tp->sw_flag & RTASE_SWF_MSIX_ENABLED) 2111 pci_disable_msix(pdev); 2112 else 2113 pci_disable_msi(pdev); 2114 } 2115 2116 static int rtase_alloc_msix(struct pci_dev *pdev, struct rtase_private *tp) 2117 { 2118 int ret, irq; 2119 u16 i; 2120 2121 memset(tp->msix_entry, 0x0, RTASE_NUM_MSIX * 2122 sizeof(struct msix_entry)); 2123 2124 for (i = 0; i < RTASE_NUM_MSIX; i++) 2125 tp->msix_entry[i].entry = i; 2126 2127 ret = pci_enable_msix_exact(pdev, tp->msix_entry, tp->int_nums); 2128 2129 if (ret) 2130 return ret; 2131 2132 for (i = 0; i < tp->int_nums; i++) { 2133 irq = pci_irq_vector(pdev, i); 2134 if (irq < 0) { 2135 pci_disable_msix(pdev); 2136 return irq; 2137 } 2138 2139 tp->int_vector[i].irq = irq; 2140 } 2141 2142 return 0; 2143 } 2144 2145 static int rtase_alloc_interrupt(struct pci_dev *pdev, 2146 struct rtase_private *tp) 2147 { 2148 int ret; 2149 2150 ret = rtase_alloc_msix(pdev, tp); 2151 if (ret) { 2152 ret = pci_enable_msi(pdev); 2153 if (ret) { 2154 dev_err(&pdev->dev, 2155 "unable to alloc interrupt.(MSI)\n"); 2156 return ret; 2157 } 2158 2159 tp->sw_flag |= RTASE_SWF_MSI_ENABLED; 2160 } else { 2161 tp->sw_flag |= RTASE_SWF_MSIX_ENABLED; 2162 } 2163 2164 return 0; 2165 } 2166 2167 static void rtase_init_hardware(const struct rtase_private *tp) 2168 { 2169 u16 i; 2170 2171 for (i = 0; i < RTASE_VLAN_FILTER_ENTRY_NUM; i++) 2172 rtase_w32(tp, RTASE_VLAN_ENTRY_0 + i * 4, 0); 2173 } 2174 2175 static void rtase_init_int_vector(struct rtase_private *tp) 2176 { 2177 u16 i; 2178 2179 /* interrupt vector 0 */ 2180 tp->int_vector[0].tp = tp; 2181 tp->int_vector[0].index = 0; 2182 tp->int_vector[0].imr_addr = RTASE_IMR0; 2183 tp->int_vector[0].isr_addr = RTASE_ISR0; 2184 tp->int_vector[0].imr = RTASE_ROK | RTASE_RDU | RTASE_TOK | 2185 RTASE_TOK4 | RTASE_TOK5 | RTASE_TOK6 | 2186 RTASE_TOK7; 2187 tp->int_vector[0].poll = rtase_poll; 2188 2189 memset(tp->int_vector[0].name, 0x0, sizeof(tp->int_vector[0].name)); 2190 INIT_LIST_HEAD(&tp->int_vector[0].ring_list); 2191 2192 /* interrupt vector 1 ~ 3 */ 2193 for (i = 1; i < tp->int_nums; i++) { 2194 tp->int_vector[i].tp = tp; 2195 tp->int_vector[i].index = i; 2196 tp->int_vector[i].imr_addr = RTASE_IMR1 + (i - 1) * 4; 2197 tp->int_vector[i].isr_addr = RTASE_ISR1 + (i - 1) * 4; 2198 tp->int_vector[i].imr = RTASE_Q_ROK | RTASE_Q_RDU | 2199 RTASE_Q_TOK; 2200 tp->int_vector[i].poll = rtase_poll; 2201 2202 memset(tp->int_vector[i].name, 0x0, 2203 sizeof(tp->int_vector[0].name)); 2204 INIT_LIST_HEAD(&tp->int_vector[i].ring_list); 2205 } 2206 } 2207 2208 static u16 rtase_calc_time_mitigation(u32 time_us) 2209 { 2210 u8 msb, time_count, time_unit; 2211 u16 int_miti; 2212 2213 time_us = min(time_us, RTASE_MITI_MAX_TIME); 2214 2215 if (time_us > RTASE_MITI_TIME_COUNT_MASK) { 2216 msb = fls(time_us); 2217 time_unit = msb - RTASE_MITI_COUNT_BIT_NUM; 2218 time_count = time_us >> (msb - RTASE_MITI_COUNT_BIT_NUM); 2219 } else { 2220 time_unit = 0; 2221 time_count = time_us; 2222 } 2223 2224 int_miti = u16_encode_bits(time_count, RTASE_MITI_TIME_COUNT_MASK) | 2225 u16_encode_bits(time_unit, RTASE_MITI_TIME_UNIT_MASK); 2226 2227 return int_miti; 2228 } 2229 2230 static u16 rtase_calc_packet_num_mitigation(u16 pkt_num) 2231 { 2232 u8 msb, pkt_num_count, pkt_num_unit; 2233 u16 int_miti; 2234 2235 pkt_num = min(pkt_num, RTASE_MITI_MAX_PKT_NUM); 2236 2237 if (pkt_num > 60) { 2238 pkt_num_unit = RTASE_MITI_MAX_PKT_NUM_IDX; 2239 pkt_num_count = pkt_num / RTASE_MITI_MAX_PKT_NUM_UNIT; 2240 } else { 2241 msb = fls(pkt_num); 2242 if (msb >= RTASE_MITI_COUNT_BIT_NUM) { 2243 pkt_num_unit = msb - RTASE_MITI_COUNT_BIT_NUM; 2244 pkt_num_count = pkt_num >> (msb - 2245 RTASE_MITI_COUNT_BIT_NUM); 2246 } else { 2247 pkt_num_unit = 0; 2248 pkt_num_count = pkt_num; 2249 } 2250 } 2251 2252 int_miti = u16_encode_bits(pkt_num_count, 2253 RTASE_MITI_PKT_NUM_COUNT_MASK) | 2254 u16_encode_bits(pkt_num_unit, 2255 RTASE_MITI_PKT_NUM_UNIT_MASK); 2256 2257 return int_miti; 2258 } 2259 2260 static void rtase_init_software_variable(struct pci_dev *pdev, 2261 struct rtase_private *tp) 2262 { 2263 u16 int_miti; 2264 2265 tp->tx_queue_ctrl = RTASE_TXQ_CTRL; 2266 tp->func_tx_queue_num = RTASE_FUNC_TXQ_NUM; 2267 tp->func_rx_queue_num = RTASE_FUNC_RXQ_NUM; 2268 tp->int_nums = RTASE_INTERRUPT_NUM; 2269 2270 int_miti = rtase_calc_time_mitigation(RTASE_MITI_DEFAULT_TIME) | 2271 rtase_calc_packet_num_mitigation(RTASE_MITI_DEFAULT_PKT_NUM); 2272 tp->tx_int_mit = int_miti; 2273 tp->rx_int_mit = int_miti; 2274 2275 tp->sw_flag = 0; 2276 2277 rtase_init_int_vector(tp); 2278 2279 /* MTU range: 60 - hw-specific max */ 2280 tp->dev->min_mtu = ETH_ZLEN; 2281 tp->dev->max_mtu = RTASE_MAX_JUMBO_SIZE; 2282 } 2283 2284 static int rtase_check_mac_version_valid(struct rtase_private *tp) 2285 { 2286 int ret = -ENODEV; 2287 2288 tp->hw_ver = rtase_r32(tp, RTASE_TX_CONFIG_0) & RTASE_HW_VER_MASK; 2289 2290 switch (tp->hw_ver) { 2291 case RTASE_HW_VER_906X_7XA: 2292 case RTASE_HW_VER_906X_7XC: 2293 case RTASE_HW_VER_907XD_V1: 2294 case RTASE_HW_VER_907XD_VA: 2295 ret = 0; 2296 break; 2297 } 2298 2299 return ret; 2300 } 2301 2302 static int rtase_init_board(struct pci_dev *pdev, struct net_device **dev_out, 2303 void __iomem **ioaddr_out) 2304 { 2305 struct net_device *dev; 2306 void __iomem *ioaddr; 2307 int ret = -ENOMEM; 2308 2309 /* dev zeroed in alloc_etherdev */ 2310 dev = alloc_etherdev_mq(sizeof(struct rtase_private), 2311 RTASE_FUNC_TXQ_NUM); 2312 if (!dev) 2313 goto err_out; 2314 2315 SET_NETDEV_DEV(dev, &pdev->dev); 2316 2317 ret = pci_enable_device(pdev); 2318 if (ret) 2319 goto err_out_free_dev; 2320 2321 /* make sure PCI base addr 1 is MMIO */ 2322 if (!(pci_resource_flags(pdev, 2) & IORESOURCE_MEM)) { 2323 ret = -ENODEV; 2324 goto err_out_disable; 2325 } 2326 2327 /* check for weird/broken PCI region reporting */ 2328 if (pci_resource_len(pdev, 2) < RTASE_REGS_SIZE) { 2329 ret = -ENODEV; 2330 goto err_out_disable; 2331 } 2332 2333 ret = pci_request_regions(pdev, KBUILD_MODNAME); 2334 if (ret) 2335 goto err_out_disable; 2336 2337 ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); 2338 if (ret) { 2339 dev_err(&pdev->dev, "no usable dma addressing method\n"); 2340 goto err_out_free_res; 2341 } 2342 2343 pci_set_master(pdev); 2344 2345 /* ioremap MMIO region */ 2346 ioaddr = ioremap(pci_resource_start(pdev, 2), 2347 pci_resource_len(pdev, 2)); 2348 if (!ioaddr) { 2349 ret = -EIO; 2350 goto err_out_free_res; 2351 } 2352 2353 *ioaddr_out = ioaddr; 2354 *dev_out = dev; 2355 2356 return ret; 2357 2358 err_out_free_res: 2359 pci_release_regions(pdev); 2360 2361 err_out_disable: 2362 pci_disable_device(pdev); 2363 2364 err_out_free_dev: 2365 free_netdev(dev); 2366 2367 err_out: 2368 *ioaddr_out = NULL; 2369 *dev_out = NULL; 2370 2371 return ret; 2372 } 2373 2374 static void rtase_release_board(struct pci_dev *pdev, struct net_device *dev, 2375 void __iomem *ioaddr) 2376 { 2377 const struct rtase_private *tp = netdev_priv(dev); 2378 2379 rtase_rar_set(tp, tp->dev->perm_addr); 2380 iounmap(ioaddr); 2381 2382 if (tp->sw_flag & RTASE_SWF_MSIX_ENABLED) 2383 pci_disable_msix(pdev); 2384 else 2385 pci_disable_msi(pdev); 2386 2387 pci_release_regions(pdev); 2388 pci_disable_device(pdev); 2389 free_netdev(dev); 2390 } 2391 2392 static int rtase_init_one(struct pci_dev *pdev, 2393 const struct pci_device_id *ent) 2394 { 2395 struct net_device *dev = NULL; 2396 struct rtase_int_vector *ivec; 2397 void __iomem *ioaddr = NULL; 2398 struct rtase_private *tp; 2399 int ret, i; 2400 2401 if (!pdev->is_physfn && pdev->is_virtfn) { 2402 dev_err(&pdev->dev, 2403 "This module does not support a virtual function."); 2404 return -EINVAL; 2405 } 2406 2407 dev_dbg(&pdev->dev, "Automotive Switch Ethernet driver loaded\n"); 2408 2409 ret = rtase_init_board(pdev, &dev, &ioaddr); 2410 if (ret) 2411 return ret; 2412 2413 tp = netdev_priv(dev); 2414 tp->mmio_addr = ioaddr; 2415 tp->dev = dev; 2416 tp->pdev = pdev; 2417 2418 /* identify chip attached to board */ 2419 ret = rtase_check_mac_version_valid(tp); 2420 if (ret) { 2421 dev_err(&pdev->dev, 2422 "unknown chip version: 0x%08x, contact rtase maintainers (see MAINTAINERS file)\n", 2423 tp->hw_ver); 2424 goto err_out_release_board; 2425 } 2426 2427 rtase_init_software_variable(pdev, tp); 2428 rtase_init_hardware(tp); 2429 2430 ret = rtase_alloc_interrupt(pdev, tp); 2431 if (ret) { 2432 dev_err(&pdev->dev, "unable to alloc MSIX/MSI\n"); 2433 goto err_out_del_napi; 2434 } 2435 2436 rtase_init_napi(tp); 2437 2438 rtase_init_netdev_ops(dev); 2439 2440 dev->pcpu_stat_type = NETDEV_PCPU_STAT_TSTATS; 2441 2442 dev->features |= NETIF_F_HW_VLAN_CTAG_TX | NETIF_F_HW_VLAN_CTAG_RX | 2443 NETIF_F_IP_CSUM | NETIF_F_HIGHDMA | 2444 NETIF_F_RXCSUM | NETIF_F_SG | 2445 NETIF_F_TSO | NETIF_F_IPV6_CSUM | 2446 NETIF_F_TSO6; 2447 2448 dev->hw_features = NETIF_F_SG | NETIF_F_IP_CSUM | 2449 NETIF_F_TSO | NETIF_F_RXCSUM | 2450 NETIF_F_HW_VLAN_CTAG_TX | NETIF_F_HW_VLAN_CTAG_RX | 2451 NETIF_F_RXALL | NETIF_F_RXFCS | 2452 NETIF_F_IPV6_CSUM | NETIF_F_TSO6; 2453 2454 dev->vlan_features = NETIF_F_SG | NETIF_F_IP_CSUM | NETIF_F_TSO | 2455 NETIF_F_HIGHDMA; 2456 dev->priv_flags |= IFF_LIVE_ADDR_CHANGE; 2457 netif_set_tso_max_size(dev, RTASE_LSO_64K); 2458 netif_set_tso_max_segs(dev, RTASE_NIC_MAX_PHYS_BUF_COUNT_LSO2); 2459 2460 rtase_get_mac_address(dev); 2461 2462 tp->tally_vaddr = dma_alloc_coherent(&pdev->dev, 2463 sizeof(*tp->tally_vaddr), 2464 &tp->tally_paddr, 2465 GFP_KERNEL); 2466 if (!tp->tally_vaddr) { 2467 ret = -ENOMEM; 2468 goto err_out_free_dma; 2469 } 2470 2471 rtase_tally_counter_clear(tp); 2472 2473 pci_set_drvdata(pdev, dev); 2474 2475 netif_carrier_off(dev); 2476 2477 ret = register_netdev(dev); 2478 if (ret) 2479 goto err_out_free_dma; 2480 2481 netdev_dbg(dev, "%pM, IRQ %d\n", dev->dev_addr, dev->irq); 2482 2483 return 0; 2484 2485 err_out_free_dma: 2486 if (tp->tally_vaddr) { 2487 dma_free_coherent(&pdev->dev, 2488 sizeof(*tp->tally_vaddr), 2489 tp->tally_vaddr, 2490 tp->tally_paddr); 2491 2492 tp->tally_vaddr = NULL; 2493 } 2494 2495 err_out_del_napi: 2496 for (i = 0; i < tp->int_nums; i++) { 2497 ivec = &tp->int_vector[i]; 2498 netif_napi_del(&ivec->napi); 2499 } 2500 2501 err_out_release_board: 2502 rtase_release_board(pdev, dev, ioaddr); 2503 2504 return ret; 2505 } 2506 2507 static void rtase_remove_one(struct pci_dev *pdev) 2508 { 2509 struct net_device *dev = pci_get_drvdata(pdev); 2510 struct rtase_private *tp = netdev_priv(dev); 2511 struct rtase_int_vector *ivec; 2512 u32 i; 2513 2514 unregister_netdev(dev); 2515 2516 for (i = 0; i < tp->int_nums; i++) { 2517 ivec = &tp->int_vector[i]; 2518 netif_napi_del(&ivec->napi); 2519 } 2520 2521 rtase_reset_interrupt(pdev, tp); 2522 if (tp->tally_vaddr) { 2523 dma_free_coherent(&pdev->dev, 2524 sizeof(*tp->tally_vaddr), 2525 tp->tally_vaddr, 2526 tp->tally_paddr); 2527 tp->tally_vaddr = NULL; 2528 } 2529 2530 rtase_release_board(pdev, dev, tp->mmio_addr); 2531 pci_set_drvdata(pdev, NULL); 2532 } 2533 2534 static void rtase_shutdown(struct pci_dev *pdev) 2535 { 2536 struct net_device *dev = pci_get_drvdata(pdev); 2537 const struct rtase_private *tp; 2538 2539 tp = netdev_priv(dev); 2540 2541 if (netif_running(dev)) 2542 rtase_close(dev); 2543 2544 rtase_reset_interrupt(pdev, tp); 2545 } 2546 2547 static int rtase_suspend(struct device *device) 2548 { 2549 struct net_device *dev = dev_get_drvdata(device); 2550 2551 if (netif_running(dev)) { 2552 netif_device_detach(dev); 2553 rtase_hw_reset(dev); 2554 } 2555 2556 return 0; 2557 } 2558 2559 static int rtase_resume(struct device *device) 2560 { 2561 struct net_device *dev = dev_get_drvdata(device); 2562 struct rtase_private *tp = netdev_priv(dev); 2563 int ret; 2564 2565 /* restore last modified mac address */ 2566 rtase_rar_set(tp, dev->dev_addr); 2567 2568 if (!netif_running(dev)) 2569 goto out; 2570 2571 rtase_wait_for_quiescence(dev); 2572 2573 rtase_tx_clear(tp); 2574 rtase_rx_clear(tp); 2575 2576 ret = rtase_init_ring(dev); 2577 if (ret) { 2578 netdev_err(dev, "unable to init ring\n"); 2579 rtase_free_desc(tp); 2580 return -ENOMEM; 2581 } 2582 2583 rtase_hw_config(dev); 2584 /* always link, so start to transmit & receive */ 2585 rtase_hw_start(dev); 2586 2587 netif_device_attach(dev); 2588 out: 2589 2590 return 0; 2591 } 2592 2593 static const struct dev_pm_ops rtase_pm_ops = { 2594 SYSTEM_SLEEP_PM_OPS(rtase_suspend, rtase_resume) 2595 }; 2596 2597 static struct pci_driver rtase_pci_driver = { 2598 .name = KBUILD_MODNAME, 2599 .id_table = rtase_pci_tbl, 2600 .probe = rtase_init_one, 2601 .remove = rtase_remove_one, 2602 .shutdown = rtase_shutdown, 2603 .driver.pm = pm_ptr(&rtase_pm_ops), 2604 }; 2605 2606 module_pci_driver(rtase_pci_driver); 2607