1 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
2 /*
3 * rtase is the Linux device driver released for Realtek Automotive Switch
4 * controllers with PCI-Express interface.
5 *
6 * Copyright(c) 2024 Realtek Semiconductor Corp.
7 *
8 * Below is a simplified block diagram of the chip and its relevant interfaces.
9 *
10 * *************************
11 * * *
12 * * CPU network device *
13 * * *
14 * * +-------------+ *
15 * * | PCIE Host | *
16 * ***********++************
17 * ||
18 * PCIE
19 * ||
20 * ********************++**********************
21 * * | PCIE Endpoint | *
22 * * +---------------+ *
23 * * | GMAC | *
24 * * +--++--+ Realtek *
25 * * || RTL90xx Series *
26 * * || *
27 * * +-------------++----------------+ *
28 * * | | MAC | | *
29 * * | +-----+ | *
30 * * | | *
31 * * | Ethernet Switch Core | *
32 * * | | *
33 * * | +-----+ +-----+ | *
34 * * | | MAC |...........| MAC | | *
35 * * +---+-----+-----------+-----+---+ *
36 * * | PHY |...........| PHY | *
37 * * +--++-+ +--++-+ *
38 * *************||****************||***********
39 *
40 * The block of the Realtek RTL90xx series is our entire chip architecture,
41 * the GMAC is connected to the switch core, and there is no PHY in between.
42 * In addition, this driver is mainly used to control GMAC, but does not
43 * control the switch core, so it is not the same as DSA. Linux only plays
44 * the role of a normal leaf node in this model.
45 */
46
47 #include <linux/crc32.h>
48 #include <linux/dma-mapping.h>
49 #include <linux/etherdevice.h>
50 #include <linux/if_vlan.h>
51 #include <linux/in.h>
52 #include <linux/init.h>
53 #include <linux/interrupt.h>
54 #include <linux/io.h>
55 #include <linux/iopoll.h>
56 #include <linux/ip.h>
57 #include <linux/ipv6.h>
58 #include <linux/mdio.h>
59 #include <linux/module.h>
60 #include <linux/netdevice.h>
61 #include <linux/pci.h>
62 #include <linux/pm_runtime.h>
63 #include <linux/prefetch.h>
64 #include <linux/ptp_classify.h>
65 #include <linux/rtnetlink.h>
66 #include <linux/tcp.h>
67 #include <asm/irq.h>
68 #include <net/ip6_checksum.h>
69 #include <net/netdev_queues.h>
70 #include <net/page_pool/helpers.h>
71 #include <net/pkt_cls.h>
72
73 #include "rtase.h"
74
75 #define RTK_OPTS1_DEBUG_VALUE 0x0BADBEEF
76 #define RTK_MAGIC_NUMBER 0x0BADBADBADBADBAD
77
78 static const struct pci_device_id rtase_pci_tbl[] = {
79 {PCI_VDEVICE(REALTEK, 0x906A)},
80 {}
81 };
82
83 MODULE_DEVICE_TABLE(pci, rtase_pci_tbl);
84
85 MODULE_AUTHOR("Realtek ARD Software Team");
86 MODULE_DESCRIPTION("Network Driver for the PCIe interface of Realtek Automotive Ethernet Switch");
87 MODULE_LICENSE("Dual BSD/GPL");
88
89 struct rtase_counters {
90 __le64 tx_packets;
91 __le64 rx_packets;
92 __le64 tx_errors;
93 __le32 rx_errors;
94 __le16 rx_missed;
95 __le16 align_errors;
96 __le32 tx_one_collision;
97 __le32 tx_multi_collision;
98 __le64 rx_unicast;
99 __le64 rx_broadcast;
100 __le32 rx_multicast;
101 __le16 tx_aborted;
102 __le16 tx_underrun;
103 } __packed;
104
rtase_w8(const struct rtase_private * tp,u16 reg,u8 val8)105 static void rtase_w8(const struct rtase_private *tp, u16 reg, u8 val8)
106 {
107 writeb(val8, tp->mmio_addr + reg);
108 }
109
rtase_w16(const struct rtase_private * tp,u16 reg,u16 val16)110 static void rtase_w16(const struct rtase_private *tp, u16 reg, u16 val16)
111 {
112 writew(val16, tp->mmio_addr + reg);
113 }
114
rtase_w32(const struct rtase_private * tp,u16 reg,u32 val32)115 static void rtase_w32(const struct rtase_private *tp, u16 reg, u32 val32)
116 {
117 writel(val32, tp->mmio_addr + reg);
118 }
119
rtase_r8(const struct rtase_private * tp,u16 reg)120 static u8 rtase_r8(const struct rtase_private *tp, u16 reg)
121 {
122 return readb(tp->mmio_addr + reg);
123 }
124
rtase_r16(const struct rtase_private * tp,u16 reg)125 static u16 rtase_r16(const struct rtase_private *tp, u16 reg)
126 {
127 return readw(tp->mmio_addr + reg);
128 }
129
rtase_r32(const struct rtase_private * tp,u16 reg)130 static u32 rtase_r32(const struct rtase_private *tp, u16 reg)
131 {
132 return readl(tp->mmio_addr + reg);
133 }
134
rtase_free_desc(struct rtase_private * tp)135 static void rtase_free_desc(struct rtase_private *tp)
136 {
137 struct pci_dev *pdev = tp->pdev;
138 u32 i;
139
140 for (i = 0; i < tp->func_tx_queue_num; i++) {
141 if (!tp->tx_ring[i].desc)
142 continue;
143
144 dma_free_coherent(&pdev->dev, RTASE_TX_RING_DESC_SIZE,
145 tp->tx_ring[i].desc,
146 tp->tx_ring[i].phy_addr);
147 tp->tx_ring[i].desc = NULL;
148 }
149
150 for (i = 0; i < tp->func_rx_queue_num; i++) {
151 if (!tp->rx_ring[i].desc)
152 continue;
153
154 dma_free_coherent(&pdev->dev, RTASE_RX_RING_DESC_SIZE,
155 tp->rx_ring[i].desc,
156 tp->rx_ring[i].phy_addr);
157 tp->rx_ring[i].desc = NULL;
158 }
159 }
160
rtase_alloc_desc(struct rtase_private * tp)161 static int rtase_alloc_desc(struct rtase_private *tp)
162 {
163 struct pci_dev *pdev = tp->pdev;
164 u32 i;
165
166 /* rx and tx descriptors needs 256 bytes alignment.
167 * dma_alloc_coherent provides more.
168 */
169 for (i = 0; i < tp->func_tx_queue_num; i++) {
170 tp->tx_ring[i].desc =
171 dma_alloc_coherent(&pdev->dev,
172 RTASE_TX_RING_DESC_SIZE,
173 &tp->tx_ring[i].phy_addr,
174 GFP_KERNEL);
175 if (!tp->tx_ring[i].desc)
176 goto err_out;
177 }
178
179 for (i = 0; i < tp->func_rx_queue_num; i++) {
180 tp->rx_ring[i].desc =
181 dma_alloc_coherent(&pdev->dev,
182 RTASE_RX_RING_DESC_SIZE,
183 &tp->rx_ring[i].phy_addr,
184 GFP_KERNEL);
185 if (!tp->rx_ring[i].desc)
186 goto err_out;
187 }
188
189 return 0;
190
191 err_out:
192 rtase_free_desc(tp);
193 return -ENOMEM;
194 }
195
rtase_unmap_tx_skb(struct pci_dev * pdev,u32 len,struct rtase_tx_desc * desc)196 static void rtase_unmap_tx_skb(struct pci_dev *pdev, u32 len,
197 struct rtase_tx_desc *desc)
198 {
199 dma_unmap_single(&pdev->dev, le64_to_cpu(desc->addr), len,
200 DMA_TO_DEVICE);
201 desc->opts1 = cpu_to_le32(RTK_OPTS1_DEBUG_VALUE);
202 desc->opts2 = 0x00;
203 desc->addr = cpu_to_le64(RTK_MAGIC_NUMBER);
204 }
205
rtase_tx_clear_range(struct rtase_ring * ring,u32 start,u32 n)206 static void rtase_tx_clear_range(struct rtase_ring *ring, u32 start, u32 n)
207 {
208 struct rtase_tx_desc *desc_base = ring->desc;
209 struct rtase_private *tp = ring->ivec->tp;
210 u32 i;
211
212 for (i = 0; i < n; i++) {
213 u32 entry = (start + i) % RTASE_NUM_DESC;
214 struct rtase_tx_desc *desc = desc_base + entry;
215 u32 len = ring->mis.len[entry];
216 struct sk_buff *skb;
217
218 if (len == 0)
219 continue;
220
221 rtase_unmap_tx_skb(tp->pdev, len, desc);
222 ring->mis.len[entry] = 0;
223 skb = ring->skbuff[entry];
224 if (!skb)
225 continue;
226
227 tp->stats.tx_dropped++;
228 dev_kfree_skb_any(skb);
229 ring->skbuff[entry] = NULL;
230 }
231 }
232
rtase_tx_clear(struct rtase_private * tp)233 static void rtase_tx_clear(struct rtase_private *tp)
234 {
235 struct rtase_ring *ring;
236 u16 i;
237
238 for (i = 0; i < tp->func_tx_queue_num; i++) {
239 ring = &tp->tx_ring[i];
240 rtase_tx_clear_range(ring, ring->dirty_idx, RTASE_NUM_DESC);
241 ring->cur_idx = 0;
242 ring->dirty_idx = 0;
243
244 netdev_tx_reset_subqueue(tp->dev, i);
245 }
246 }
247
rtase_mark_to_asic(union rtase_rx_desc * desc,u32 rx_buf_sz)248 static void rtase_mark_to_asic(union rtase_rx_desc *desc, u32 rx_buf_sz)
249 {
250 u32 eor = le32_to_cpu(desc->desc_cmd.opts1) & RTASE_RING_END;
251
252 desc->desc_status.opts2 = 0;
253 /* force memory writes to complete before releasing descriptor */
254 dma_wmb();
255 WRITE_ONCE(desc->desc_cmd.opts1,
256 cpu_to_le32(RTASE_DESC_OWN | eor | rx_buf_sz));
257 }
258
rtase_tx_avail(struct rtase_ring * ring)259 static u32 rtase_tx_avail(struct rtase_ring *ring)
260 {
261 return READ_ONCE(ring->dirty_idx) + RTASE_NUM_DESC -
262 READ_ONCE(ring->cur_idx);
263 }
264
tx_handler(struct rtase_ring * ring,int budget)265 static int tx_handler(struct rtase_ring *ring, int budget)
266 {
267 const struct rtase_private *tp = ring->ivec->tp;
268 struct net_device *dev = tp->dev;
269 u32 dirty_tx, tx_left;
270 u32 bytes_compl = 0;
271 u32 pkts_compl = 0;
272 int workdone = 0;
273
274 dirty_tx = ring->dirty_idx;
275 tx_left = READ_ONCE(ring->cur_idx) - dirty_tx;
276
277 while (tx_left > 0) {
278 u32 entry = dirty_tx % RTASE_NUM_DESC;
279 struct rtase_tx_desc *desc = ring->desc +
280 sizeof(struct rtase_tx_desc) * entry;
281 u32 status;
282
283 status = le32_to_cpu(desc->opts1);
284
285 if (status & RTASE_DESC_OWN)
286 break;
287
288 rtase_unmap_tx_skb(tp->pdev, ring->mis.len[entry], desc);
289 ring->mis.len[entry] = 0;
290 if (ring->skbuff[entry]) {
291 pkts_compl++;
292 bytes_compl += ring->skbuff[entry]->len;
293 napi_consume_skb(ring->skbuff[entry], budget);
294 ring->skbuff[entry] = NULL;
295 }
296
297 dirty_tx++;
298 tx_left--;
299 workdone++;
300
301 if (workdone == RTASE_TX_BUDGET_DEFAULT)
302 break;
303 }
304
305 if (ring->dirty_idx != dirty_tx) {
306 dev_sw_netstats_tx_add(dev, pkts_compl, bytes_compl);
307 WRITE_ONCE(ring->dirty_idx, dirty_tx);
308
309 netif_subqueue_completed_wake(dev, ring->index, pkts_compl,
310 bytes_compl,
311 rtase_tx_avail(ring),
312 RTASE_TX_START_THRS);
313
314 if (ring->cur_idx != dirty_tx)
315 rtase_w8(tp, RTASE_TPPOLL, BIT(ring->index));
316 }
317
318 return 0;
319 }
320
rtase_tx_desc_init(struct rtase_private * tp,u16 idx)321 static void rtase_tx_desc_init(struct rtase_private *tp, u16 idx)
322 {
323 struct rtase_ring *ring = &tp->tx_ring[idx];
324 struct rtase_tx_desc *desc;
325 u32 i;
326
327 memset(ring->desc, 0x0, RTASE_TX_RING_DESC_SIZE);
328 memset(ring->skbuff, 0x0, sizeof(ring->skbuff));
329 ring->cur_idx = 0;
330 ring->dirty_idx = 0;
331 ring->index = idx;
332 ring->type = NETDEV_QUEUE_TYPE_TX;
333 ring->alloc_fail = 0;
334
335 for (i = 0; i < RTASE_NUM_DESC; i++) {
336 ring->mis.len[i] = 0;
337 if ((RTASE_NUM_DESC - 1) == i) {
338 desc = ring->desc + sizeof(struct rtase_tx_desc) * i;
339 desc->opts1 = cpu_to_le32(RTASE_RING_END);
340 }
341 }
342
343 ring->ring_handler = tx_handler;
344 if (idx < 4) {
345 ring->ivec = &tp->int_vector[idx];
346 list_add_tail(&ring->ring_entry,
347 &tp->int_vector[idx].ring_list);
348 } else {
349 ring->ivec = &tp->int_vector[0];
350 list_add_tail(&ring->ring_entry, &tp->int_vector[0].ring_list);
351 }
352
353 netif_queue_set_napi(tp->dev, ring->index,
354 ring->type, &ring->ivec->napi);
355 }
356
rtase_map_to_asic(union rtase_rx_desc * desc,dma_addr_t mapping,u32 rx_buf_sz)357 static void rtase_map_to_asic(union rtase_rx_desc *desc, dma_addr_t mapping,
358 u32 rx_buf_sz)
359 {
360 desc->desc_cmd.addr = cpu_to_le64(mapping);
361
362 rtase_mark_to_asic(desc, rx_buf_sz);
363 }
364
rtase_make_unusable_by_asic(union rtase_rx_desc * desc)365 static void rtase_make_unusable_by_asic(union rtase_rx_desc *desc)
366 {
367 desc->desc_cmd.addr = cpu_to_le64(RTK_MAGIC_NUMBER);
368 desc->desc_cmd.opts1 &= ~cpu_to_le32(RTASE_DESC_OWN | RSVD_MASK);
369 }
370
rtase_alloc_rx_data_buf(struct rtase_ring * ring,void ** p_data_buf,union rtase_rx_desc * desc,dma_addr_t * rx_phy_addr)371 static int rtase_alloc_rx_data_buf(struct rtase_ring *ring,
372 void **p_data_buf,
373 union rtase_rx_desc *desc,
374 dma_addr_t *rx_phy_addr)
375 {
376 struct rtase_int_vector *ivec = ring->ivec;
377 const struct rtase_private *tp = ivec->tp;
378 dma_addr_t mapping;
379 struct page *page;
380
381 page = page_pool_dev_alloc_pages(tp->page_pool);
382 if (!page) {
383 ring->alloc_fail++;
384 goto err_out;
385 }
386
387 *p_data_buf = page_address(page);
388 mapping = page_pool_get_dma_addr(page);
389 *rx_phy_addr = mapping;
390 rtase_map_to_asic(desc, mapping, tp->rx_buf_sz);
391
392 return 0;
393
394 err_out:
395 rtase_make_unusable_by_asic(desc);
396
397 return -ENOMEM;
398 }
399
rtase_rx_ring_fill(struct rtase_ring * ring,u32 ring_start,u32 ring_end)400 static u32 rtase_rx_ring_fill(struct rtase_ring *ring, u32 ring_start,
401 u32 ring_end)
402 {
403 union rtase_rx_desc *desc_base = ring->desc;
404 u32 cur;
405
406 for (cur = ring_start; ring_end - cur > 0; cur++) {
407 u32 i = cur % RTASE_NUM_DESC;
408 union rtase_rx_desc *desc = desc_base + i;
409 int ret;
410
411 if (ring->data_buf[i])
412 continue;
413
414 ret = rtase_alloc_rx_data_buf(ring, &ring->data_buf[i], desc,
415 &ring->mis.data_phy_addr[i]);
416 if (ret)
417 break;
418 }
419
420 return cur - ring_start;
421 }
422
rtase_mark_as_last_descriptor(union rtase_rx_desc * desc)423 static void rtase_mark_as_last_descriptor(union rtase_rx_desc *desc)
424 {
425 desc->desc_cmd.opts1 |= cpu_to_le32(RTASE_RING_END);
426 }
427
rtase_rx_ring_clear(struct page_pool * page_pool,struct rtase_ring * ring)428 static void rtase_rx_ring_clear(struct page_pool *page_pool,
429 struct rtase_ring *ring)
430 {
431 union rtase_rx_desc *desc;
432 struct page *page;
433 u32 i;
434
435 for (i = 0; i < RTASE_NUM_DESC; i++) {
436 desc = ring->desc + sizeof(union rtase_rx_desc) * i;
437 page = virt_to_head_page(ring->data_buf[i]);
438
439 if (ring->data_buf[i])
440 page_pool_put_full_page(page_pool, page, true);
441
442 rtase_make_unusable_by_asic(desc);
443 }
444 }
445
rtase_fragmented_frame(u32 status)446 static int rtase_fragmented_frame(u32 status)
447 {
448 return (status & (RTASE_RX_FIRST_FRAG | RTASE_RX_LAST_FRAG)) !=
449 (RTASE_RX_FIRST_FRAG | RTASE_RX_LAST_FRAG);
450 }
451
rtase_rx_csum(const struct rtase_private * tp,struct sk_buff * skb,const union rtase_rx_desc * desc)452 static void rtase_rx_csum(const struct rtase_private *tp, struct sk_buff *skb,
453 const union rtase_rx_desc *desc)
454 {
455 u32 opts2 = le32_to_cpu(desc->desc_status.opts2);
456
457 /* rx csum offload */
458 if (((opts2 & RTASE_RX_V4F) && !(opts2 & RTASE_RX_IPF)) ||
459 (opts2 & RTASE_RX_V6F)) {
460 if (((opts2 & RTASE_RX_TCPT) && !(opts2 & RTASE_RX_TCPF)) ||
461 ((opts2 & RTASE_RX_UDPT) && !(opts2 & RTASE_RX_UDPF)))
462 skb->ip_summed = CHECKSUM_UNNECESSARY;
463 else
464 skb->ip_summed = CHECKSUM_NONE;
465 } else {
466 skb->ip_summed = CHECKSUM_NONE;
467 }
468 }
469
rtase_rx_vlan_skb(union rtase_rx_desc * desc,struct sk_buff * skb)470 static void rtase_rx_vlan_skb(union rtase_rx_desc *desc, struct sk_buff *skb)
471 {
472 u32 opts2 = le32_to_cpu(desc->desc_status.opts2);
473
474 if (!(opts2 & RTASE_RX_VLAN_TAG))
475 return;
476
477 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q),
478 swab16(opts2 & RTASE_VLAN_TAG_MASK));
479 }
480
rtase_rx_skb(const struct rtase_ring * ring,struct sk_buff * skb)481 static void rtase_rx_skb(const struct rtase_ring *ring, struct sk_buff *skb)
482 {
483 struct rtase_int_vector *ivec = ring->ivec;
484
485 napi_gro_receive(&ivec->napi, skb);
486 }
487
rx_handler(struct rtase_ring * ring,int budget)488 static int rx_handler(struct rtase_ring *ring, int budget)
489 {
490 union rtase_rx_desc *desc_base = ring->desc;
491 u32 pkt_size, cur_rx, delta, entry, status;
492 struct rtase_private *tp = ring->ivec->tp;
493 struct net_device *dev = tp->dev;
494 union rtase_rx_desc *desc;
495 struct sk_buff *skb;
496 int workdone = 0;
497
498 cur_rx = ring->cur_idx;
499 entry = cur_rx % RTASE_NUM_DESC;
500 desc = &desc_base[entry];
501
502 while (workdone < budget) {
503 status = le32_to_cpu(desc->desc_status.opts1);
504
505 if (status & RTASE_DESC_OWN)
506 break;
507
508 /* This barrier is needed to keep us from reading
509 * any other fields out of the rx descriptor until
510 * we know the status of RTASE_DESC_OWN
511 */
512 dma_rmb();
513
514 if (unlikely(status & RTASE_RX_RES)) {
515 if (net_ratelimit())
516 netdev_warn(dev, "Rx ERROR. status = %08x\n",
517 status);
518
519 tp->stats.rx_errors++;
520
521 if (status & (RTASE_RX_RWT | RTASE_RX_RUNT))
522 tp->stats.rx_length_errors++;
523
524 if (status & RTASE_RX_CRC)
525 tp->stats.rx_crc_errors++;
526
527 if (dev->features & NETIF_F_RXALL)
528 goto process_pkt;
529
530 rtase_mark_to_asic(desc, tp->rx_buf_sz);
531 goto skip_process_pkt;
532 }
533
534 process_pkt:
535 pkt_size = status & RTASE_RX_PKT_SIZE_MASK;
536 if (likely(!(dev->features & NETIF_F_RXFCS)))
537 pkt_size -= ETH_FCS_LEN;
538
539 /* The driver does not support incoming fragmented frames.
540 * They are seen as a symptom of over-mtu sized frames.
541 */
542 if (unlikely(rtase_fragmented_frame(status))) {
543 tp->stats.rx_dropped++;
544 tp->stats.rx_length_errors++;
545 rtase_mark_to_asic(desc, tp->rx_buf_sz);
546 goto skip_process_pkt;
547 }
548
549 dma_sync_single_for_cpu(&tp->pdev->dev,
550 ring->mis.data_phy_addr[entry],
551 tp->rx_buf_sz, DMA_FROM_DEVICE);
552
553 skb = build_skb(ring->data_buf[entry], PAGE_SIZE);
554 if (!skb) {
555 tp->stats.rx_dropped++;
556 rtase_mark_to_asic(desc, tp->rx_buf_sz);
557 goto skip_process_pkt;
558 }
559 ring->data_buf[entry] = NULL;
560
561 if (dev->features & NETIF_F_RXCSUM)
562 rtase_rx_csum(tp, skb, desc);
563
564 skb_put(skb, pkt_size);
565 skb_mark_for_recycle(skb);
566 skb->protocol = eth_type_trans(skb, dev);
567
568 if (skb->pkt_type == PACKET_MULTICAST)
569 tp->stats.multicast++;
570
571 rtase_rx_vlan_skb(desc, skb);
572 rtase_rx_skb(ring, skb);
573
574 dev_sw_netstats_rx_add(dev, pkt_size);
575
576 skip_process_pkt:
577 workdone++;
578 cur_rx++;
579 entry = cur_rx % RTASE_NUM_DESC;
580 desc = ring->desc + sizeof(union rtase_rx_desc) * entry;
581 }
582
583 ring->cur_idx = cur_rx;
584 delta = rtase_rx_ring_fill(ring, ring->dirty_idx, ring->cur_idx);
585 ring->dirty_idx += delta;
586
587 return workdone;
588 }
589
rtase_rx_desc_init(struct rtase_private * tp,u16 idx)590 static void rtase_rx_desc_init(struct rtase_private *tp, u16 idx)
591 {
592 struct rtase_ring *ring = &tp->rx_ring[idx];
593 u16 i;
594
595 memset(ring->desc, 0x0, RTASE_RX_RING_DESC_SIZE);
596 memset(ring->data_buf, 0x0, sizeof(ring->data_buf));
597 ring->cur_idx = 0;
598 ring->dirty_idx = 0;
599 ring->index = idx;
600 ring->type = NETDEV_QUEUE_TYPE_RX;
601 ring->alloc_fail = 0;
602
603 for (i = 0; i < RTASE_NUM_DESC; i++)
604 ring->mis.data_phy_addr[i] = 0;
605
606 ring->ring_handler = rx_handler;
607 ring->ivec = &tp->int_vector[idx];
608 netif_queue_set_napi(tp->dev, ring->index,
609 ring->type, &ring->ivec->napi);
610 list_add_tail(&ring->ring_entry, &tp->int_vector[idx].ring_list);
611 }
612
rtase_rx_clear(struct rtase_private * tp)613 static void rtase_rx_clear(struct rtase_private *tp)
614 {
615 u32 i;
616
617 for (i = 0; i < tp->func_rx_queue_num; i++)
618 rtase_rx_ring_clear(tp->page_pool, &tp->rx_ring[i]);
619
620 page_pool_destroy(tp->page_pool);
621 tp->page_pool = NULL;
622 }
623
rtase_init_ring(const struct net_device * dev)624 static int rtase_init_ring(const struct net_device *dev)
625 {
626 struct rtase_private *tp = netdev_priv(dev);
627 struct page_pool_params pp_params = { 0 };
628 struct page_pool *page_pool;
629 u32 num;
630 u16 i;
631
632 pp_params.flags = PP_FLAG_DMA_MAP | PP_FLAG_DMA_SYNC_DEV;
633 pp_params.order = 0;
634 pp_params.pool_size = RTASE_NUM_DESC * tp->func_rx_queue_num;
635 pp_params.nid = dev_to_node(&tp->pdev->dev);
636 pp_params.dev = &tp->pdev->dev;
637 pp_params.dma_dir = DMA_FROM_DEVICE;
638 pp_params.max_len = PAGE_SIZE;
639 pp_params.offset = 0;
640
641 page_pool = page_pool_create(&pp_params);
642 if (IS_ERR(page_pool)) {
643 netdev_err(tp->dev, "failed to create page pool\n");
644 return -ENOMEM;
645 }
646
647 tp->page_pool = page_pool;
648
649 for (i = 0; i < tp->func_tx_queue_num; i++)
650 rtase_tx_desc_init(tp, i);
651
652 for (i = 0; i < tp->func_rx_queue_num; i++) {
653 rtase_rx_desc_init(tp, i);
654
655 num = rtase_rx_ring_fill(&tp->rx_ring[i], 0, RTASE_NUM_DESC);
656 if (num != RTASE_NUM_DESC)
657 goto err_out;
658
659 rtase_mark_as_last_descriptor(tp->rx_ring[i].desc +
660 sizeof(union rtase_rx_desc) *
661 (RTASE_NUM_DESC - 1));
662 }
663
664 return 0;
665
666 err_out:
667 rtase_rx_clear(tp);
668 return -ENOMEM;
669 }
670
rtase_interrupt_mitigation(const struct rtase_private * tp)671 static void rtase_interrupt_mitigation(const struct rtase_private *tp)
672 {
673 u32 i;
674
675 for (i = 0; i < tp->func_tx_queue_num; i++)
676 rtase_w16(tp, RTASE_INT_MITI_TX + i * 2, tp->tx_int_mit);
677
678 for (i = 0; i < tp->func_rx_queue_num; i++)
679 rtase_w16(tp, RTASE_INT_MITI_RX + i * 2, tp->rx_int_mit);
680 }
681
rtase_tally_counter_addr_fill(const struct rtase_private * tp)682 static void rtase_tally_counter_addr_fill(const struct rtase_private *tp)
683 {
684 rtase_w32(tp, RTASE_DTCCR4, upper_32_bits(tp->tally_paddr));
685 rtase_w32(tp, RTASE_DTCCR0, lower_32_bits(tp->tally_paddr));
686 }
687
rtase_tally_counter_clear(const struct rtase_private * tp)688 static void rtase_tally_counter_clear(const struct rtase_private *tp)
689 {
690 u32 cmd = lower_32_bits(tp->tally_paddr);
691
692 rtase_w32(tp, RTASE_DTCCR4, upper_32_bits(tp->tally_paddr));
693 rtase_w32(tp, RTASE_DTCCR0, cmd | RTASE_COUNTER_RESET);
694 }
695
rtase_desc_addr_fill(const struct rtase_private * tp)696 static void rtase_desc_addr_fill(const struct rtase_private *tp)
697 {
698 const struct rtase_ring *ring;
699 u16 i, cmd, val;
700 int err;
701
702 for (i = 0; i < tp->func_tx_queue_num; i++) {
703 ring = &tp->tx_ring[i];
704
705 rtase_w32(tp, RTASE_TX_DESC_ADDR0,
706 lower_32_bits(ring->phy_addr));
707 rtase_w32(tp, RTASE_TX_DESC_ADDR4,
708 upper_32_bits(ring->phy_addr));
709
710 cmd = i | RTASE_TX_DESC_CMD_WE | RTASE_TX_DESC_CMD_CS;
711 rtase_w16(tp, RTASE_TX_DESC_COMMAND, cmd);
712
713 err = read_poll_timeout(rtase_r16, val,
714 !(val & RTASE_TX_DESC_CMD_CS), 10,
715 1000, false, tp,
716 RTASE_TX_DESC_COMMAND);
717
718 if (err == -ETIMEDOUT)
719 netdev_err(tp->dev,
720 "error occurred in fill tx descriptor\n");
721 }
722
723 for (i = 0; i < tp->func_rx_queue_num; i++) {
724 ring = &tp->rx_ring[i];
725
726 if (i == 0) {
727 rtase_w32(tp, RTASE_Q0_RX_DESC_ADDR0,
728 lower_32_bits(ring->phy_addr));
729 rtase_w32(tp, RTASE_Q0_RX_DESC_ADDR4,
730 upper_32_bits(ring->phy_addr));
731 } else {
732 rtase_w32(tp, (RTASE_Q1_RX_DESC_ADDR0 + ((i - 1) * 8)),
733 lower_32_bits(ring->phy_addr));
734 rtase_w32(tp, (RTASE_Q1_RX_DESC_ADDR4 + ((i - 1) * 8)),
735 upper_32_bits(ring->phy_addr));
736 }
737 }
738 }
739
rtase_hw_set_features(const struct net_device * dev,netdev_features_t features)740 static void rtase_hw_set_features(const struct net_device *dev,
741 netdev_features_t features)
742 {
743 const struct rtase_private *tp = netdev_priv(dev);
744 u16 rx_config, val;
745
746 rx_config = rtase_r16(tp, RTASE_RX_CONFIG_0);
747 if (features & NETIF_F_RXALL)
748 rx_config |= (RTASE_ACCEPT_ERR | RTASE_ACCEPT_RUNT);
749 else
750 rx_config &= ~(RTASE_ACCEPT_ERR | RTASE_ACCEPT_RUNT);
751
752 rtase_w16(tp, RTASE_RX_CONFIG_0, rx_config);
753
754 val = rtase_r16(tp, RTASE_CPLUS_CMD);
755 if (features & NETIF_F_RXCSUM)
756 rtase_w16(tp, RTASE_CPLUS_CMD, val | RTASE_RX_CHKSUM);
757 else
758 rtase_w16(tp, RTASE_CPLUS_CMD, val & ~RTASE_RX_CHKSUM);
759
760 rx_config = rtase_r16(tp, RTASE_RX_CONFIG_1);
761 if (dev->features & NETIF_F_HW_VLAN_CTAG_RX)
762 rx_config |= (RTASE_INNER_VLAN_DETAG_EN |
763 RTASE_OUTER_VLAN_DETAG_EN);
764 else
765 rx_config &= ~(RTASE_INNER_VLAN_DETAG_EN |
766 RTASE_OUTER_VLAN_DETAG_EN);
767
768 rtase_w16(tp, RTASE_RX_CONFIG_1, rx_config);
769 }
770
rtase_hw_set_rx_packet_filter(struct net_device * dev)771 static void rtase_hw_set_rx_packet_filter(struct net_device *dev)
772 {
773 u32 mc_filter[2] = { 0xFFFFFFFF, 0xFFFFFFFF };
774 struct rtase_private *tp = netdev_priv(dev);
775 u16 rx_mode;
776
777 rx_mode = rtase_r16(tp, RTASE_RX_CONFIG_0) & ~RTASE_ACCEPT_MASK;
778 rx_mode |= RTASE_ACCEPT_BROADCAST | RTASE_ACCEPT_MYPHYS;
779
780 if (dev->flags & IFF_PROMISC) {
781 rx_mode |= RTASE_ACCEPT_MULTICAST | RTASE_ACCEPT_ALLPHYS;
782 } else if (dev->flags & IFF_ALLMULTI) {
783 rx_mode |= RTASE_ACCEPT_MULTICAST;
784 } else {
785 struct netdev_hw_addr *hw_addr;
786
787 mc_filter[0] = 0;
788 mc_filter[1] = 0;
789
790 netdev_for_each_mc_addr(hw_addr, dev) {
791 u32 bit_nr = eth_hw_addr_crc(hw_addr);
792 u32 idx = u32_get_bits(bit_nr, BIT(31));
793 u32 bit = u32_get_bits(bit_nr,
794 RTASE_MULTICAST_FILTER_MASK);
795
796 mc_filter[idx] |= BIT(bit);
797 rx_mode |= RTASE_ACCEPT_MULTICAST;
798 }
799 }
800
801 if (dev->features & NETIF_F_RXALL)
802 rx_mode |= RTASE_ACCEPT_ERR | RTASE_ACCEPT_RUNT;
803
804 rtase_w32(tp, RTASE_MAR0, swab32(mc_filter[1]));
805 rtase_w32(tp, RTASE_MAR1, swab32(mc_filter[0]));
806 rtase_w16(tp, RTASE_RX_CONFIG_0, rx_mode);
807 }
808
rtase_irq_dis_and_clear(const struct rtase_private * tp)809 static void rtase_irq_dis_and_clear(const struct rtase_private *tp)
810 {
811 const struct rtase_int_vector *ivec = &tp->int_vector[0];
812 u32 val1;
813 u16 val2;
814 u8 i;
815
816 rtase_w32(tp, ivec->imr_addr, 0);
817 val1 = rtase_r32(tp, ivec->isr_addr);
818 rtase_w32(tp, ivec->isr_addr, val1);
819
820 for (i = 1; i < tp->int_nums; i++) {
821 ivec = &tp->int_vector[i];
822 rtase_w16(tp, ivec->imr_addr, 0);
823 val2 = rtase_r16(tp, ivec->isr_addr);
824 rtase_w16(tp, ivec->isr_addr, val2);
825 }
826 }
827
rtase_poll_timeout(const struct rtase_private * tp,u32 cond,u32 sleep_us,u64 timeout_us,u16 reg)828 static void rtase_poll_timeout(const struct rtase_private *tp, u32 cond,
829 u32 sleep_us, u64 timeout_us, u16 reg)
830 {
831 int err;
832 u8 val;
833
834 err = read_poll_timeout(rtase_r8, val, val & cond, sleep_us,
835 timeout_us, false, tp, reg);
836
837 if (err == -ETIMEDOUT)
838 netdev_err(tp->dev, "poll reg 0x00%x timeout\n", reg);
839 }
840
rtase_nic_reset(const struct net_device * dev)841 static void rtase_nic_reset(const struct net_device *dev)
842 {
843 const struct rtase_private *tp = netdev_priv(dev);
844 u16 rx_config;
845 u8 val;
846
847 rx_config = rtase_r16(tp, RTASE_RX_CONFIG_0);
848 rtase_w16(tp, RTASE_RX_CONFIG_0, rx_config & ~RTASE_ACCEPT_MASK);
849
850 val = rtase_r8(tp, RTASE_MISC);
851 rtase_w8(tp, RTASE_MISC, val | RTASE_RX_DV_GATE_EN);
852
853 val = rtase_r8(tp, RTASE_CHIP_CMD);
854 rtase_w8(tp, RTASE_CHIP_CMD, val | RTASE_STOP_REQ);
855 mdelay(2);
856
857 rtase_poll_timeout(tp, RTASE_STOP_REQ_DONE, 100, 150000,
858 RTASE_CHIP_CMD);
859
860 rtase_poll_timeout(tp, RTASE_TX_FIFO_EMPTY, 100, 100000,
861 RTASE_FIFOR);
862
863 rtase_poll_timeout(tp, RTASE_RX_FIFO_EMPTY, 100, 100000,
864 RTASE_FIFOR);
865
866 val = rtase_r8(tp, RTASE_CHIP_CMD);
867 rtase_w8(tp, RTASE_CHIP_CMD, val & ~(RTASE_TE | RTASE_RE));
868 val = rtase_r8(tp, RTASE_CHIP_CMD);
869 rtase_w8(tp, RTASE_CHIP_CMD, val & ~RTASE_STOP_REQ);
870
871 rtase_w16(tp, RTASE_RX_CONFIG_0, rx_config);
872 }
873
rtase_hw_reset(const struct net_device * dev)874 static void rtase_hw_reset(const struct net_device *dev)
875 {
876 const struct rtase_private *tp = netdev_priv(dev);
877
878 rtase_irq_dis_and_clear(tp);
879
880 rtase_nic_reset(dev);
881 }
882
rtase_set_rx_queue(const struct rtase_private * tp)883 static void rtase_set_rx_queue(const struct rtase_private *tp)
884 {
885 u16 reg_data;
886
887 reg_data = rtase_r16(tp, RTASE_FCR);
888 switch (tp->func_rx_queue_num) {
889 case 1:
890 u16p_replace_bits(®_data, 0x1, RTASE_FCR_RXQ_MASK);
891 break;
892 case 2:
893 u16p_replace_bits(®_data, 0x2, RTASE_FCR_RXQ_MASK);
894 break;
895 case 4:
896 u16p_replace_bits(®_data, 0x3, RTASE_FCR_RXQ_MASK);
897 break;
898 }
899 rtase_w16(tp, RTASE_FCR, reg_data);
900 }
901
rtase_set_tx_queue(const struct rtase_private * tp)902 static void rtase_set_tx_queue(const struct rtase_private *tp)
903 {
904 u16 reg_data;
905
906 reg_data = rtase_r16(tp, RTASE_TX_CONFIG_1);
907 switch (tp->tx_queue_ctrl) {
908 case 1:
909 u16p_replace_bits(®_data, 0x0, RTASE_TC_MODE_MASK);
910 break;
911 case 2:
912 u16p_replace_bits(®_data, 0x1, RTASE_TC_MODE_MASK);
913 break;
914 case 3:
915 case 4:
916 u16p_replace_bits(®_data, 0x2, RTASE_TC_MODE_MASK);
917 break;
918 default:
919 u16p_replace_bits(®_data, 0x3, RTASE_TC_MODE_MASK);
920 break;
921 }
922 rtase_w16(tp, RTASE_TX_CONFIG_1, reg_data);
923 }
924
rtase_hw_config(struct net_device * dev)925 static void rtase_hw_config(struct net_device *dev)
926 {
927 const struct rtase_private *tp = netdev_priv(dev);
928 u32 reg_data32;
929 u16 reg_data16;
930
931 rtase_hw_reset(dev);
932
933 /* set rx dma burst */
934 reg_data16 = rtase_r16(tp, RTASE_RX_CONFIG_0);
935 reg_data16 &= ~(RTASE_RX_SINGLE_TAG | RTASE_RX_SINGLE_FETCH);
936 u16p_replace_bits(®_data16, RTASE_RX_DMA_BURST_256,
937 RTASE_RX_MX_DMA_MASK);
938 rtase_w16(tp, RTASE_RX_CONFIG_0, reg_data16);
939
940 /* new rx descritpor */
941 reg_data16 = rtase_r16(tp, RTASE_RX_CONFIG_1);
942 reg_data16 |= RTASE_RX_NEW_DESC_FORMAT_EN | RTASE_PCIE_NEW_FLOW;
943 u16p_replace_bits(®_data16, 0xF, RTASE_RX_MAX_FETCH_DESC_MASK);
944 rtase_w16(tp, RTASE_RX_CONFIG_1, reg_data16);
945
946 rtase_set_rx_queue(tp);
947
948 rtase_interrupt_mitigation(tp);
949
950 /* set tx dma burst size and interframe gap time */
951 reg_data32 = rtase_r32(tp, RTASE_TX_CONFIG_0);
952 u32p_replace_bits(®_data32, RTASE_TX_DMA_BURST_UNLIMITED,
953 RTASE_TX_DMA_MASK);
954 u32p_replace_bits(®_data32, RTASE_INTERFRAMEGAP,
955 RTASE_TX_INTER_FRAME_GAP_MASK);
956 rtase_w32(tp, RTASE_TX_CONFIG_0, reg_data32);
957
958 /* new tx descriptor */
959 reg_data16 = rtase_r16(tp, RTASE_TFUN_CTRL);
960 rtase_w16(tp, RTASE_TFUN_CTRL, reg_data16 |
961 RTASE_TX_NEW_DESC_FORMAT_EN);
962
963 /* tx fetch desc number */
964 rtase_w8(tp, RTASE_TDFNR, 0x10);
965
966 /* tag num select */
967 reg_data16 = rtase_r16(tp, RTASE_MTPS);
968 u16p_replace_bits(®_data16, 0x4, RTASE_TAG_NUM_SEL_MASK);
969 rtase_w16(tp, RTASE_MTPS, reg_data16);
970
971 rtase_set_tx_queue(tp);
972
973 rtase_w16(tp, RTASE_TOKSEL, 0x5555);
974
975 rtase_tally_counter_addr_fill(tp);
976 rtase_desc_addr_fill(tp);
977 rtase_hw_set_features(dev, dev->features);
978
979 /* enable flow control */
980 reg_data16 = rtase_r16(tp, RTASE_GPHY_STD_00);
981 reg_data16 &= ~(RTASE_TXFLOW_EN | RTASE_RXFLOW_EN);
982 rtase_w16(tp, RTASE_GPHY_STD_00, reg_data16);
983 reg_data16 = rtase_r16(tp, RTASE_CPLUS_CMD);
984 reg_data16 |= (RTASE_FORCE_TXFLOW_EN | RTASE_FORCE_RXFLOW_EN);
985 rtase_w16(tp, RTASE_CPLUS_CMD, reg_data16);
986 /* set near fifo threshold - rx missed issue. */
987 rtase_w16(tp, RTASE_RFIFONFULL, 0x190);
988
989 rtase_w16(tp, RTASE_RMS, tp->rx_buf_sz);
990
991 rtase_hw_set_rx_packet_filter(dev);
992 }
993
rtase_nic_enable(const struct net_device * dev)994 static void rtase_nic_enable(const struct net_device *dev)
995 {
996 const struct rtase_private *tp = netdev_priv(dev);
997 u16 rcr = rtase_r16(tp, RTASE_RX_CONFIG_1);
998 u8 val;
999
1000 rtase_w16(tp, RTASE_RX_CONFIG_1, rcr & ~RTASE_PCIE_RELOAD_EN);
1001 rtase_w16(tp, RTASE_RX_CONFIG_1, rcr | RTASE_PCIE_RELOAD_EN);
1002
1003 val = rtase_r8(tp, RTASE_CHIP_CMD);
1004 rtase_w8(tp, RTASE_CHIP_CMD, val | RTASE_TE | RTASE_RE);
1005
1006 val = rtase_r8(tp, RTASE_MISC);
1007 rtase_w8(tp, RTASE_MISC, val & ~RTASE_RX_DV_GATE_EN);
1008 }
1009
rtase_enable_hw_interrupt(const struct rtase_private * tp)1010 static void rtase_enable_hw_interrupt(const struct rtase_private *tp)
1011 {
1012 const struct rtase_int_vector *ivec = &tp->int_vector[0];
1013 u32 i;
1014
1015 rtase_w32(tp, ivec->imr_addr, ivec->imr);
1016
1017 for (i = 1; i < tp->int_nums; i++) {
1018 ivec = &tp->int_vector[i];
1019 rtase_w16(tp, ivec->imr_addr, ivec->imr);
1020 }
1021 }
1022
rtase_hw_start(const struct net_device * dev)1023 static void rtase_hw_start(const struct net_device *dev)
1024 {
1025 const struct rtase_private *tp = netdev_priv(dev);
1026
1027 rtase_nic_enable(dev);
1028 rtase_enable_hw_interrupt(tp);
1029 }
1030
1031 /* the interrupt handler does RXQ0 and TXQ0, TXQ4~7 interrutp status
1032 */
rtase_interrupt(int irq,void * dev_instance)1033 static irqreturn_t rtase_interrupt(int irq, void *dev_instance)
1034 {
1035 const struct rtase_private *tp;
1036 struct rtase_int_vector *ivec;
1037 u32 status;
1038
1039 ivec = dev_instance;
1040 tp = ivec->tp;
1041 status = rtase_r32(tp, ivec->isr_addr);
1042
1043 rtase_w32(tp, ivec->imr_addr, 0x0);
1044 rtase_w32(tp, ivec->isr_addr, status & ~RTASE_FOVW);
1045
1046 if (napi_schedule_prep(&ivec->napi))
1047 __napi_schedule(&ivec->napi);
1048
1049 return IRQ_HANDLED;
1050 }
1051
1052 /* the interrupt handler does RXQ1&TXQ1 or RXQ2&TXQ2 or RXQ3&TXQ3 interrupt
1053 * status according to interrupt vector
1054 */
rtase_q_interrupt(int irq,void * dev_instance)1055 static irqreturn_t rtase_q_interrupt(int irq, void *dev_instance)
1056 {
1057 const struct rtase_private *tp;
1058 struct rtase_int_vector *ivec;
1059 u16 status;
1060
1061 ivec = dev_instance;
1062 tp = ivec->tp;
1063 status = rtase_r16(tp, ivec->isr_addr);
1064
1065 rtase_w16(tp, ivec->imr_addr, 0x0);
1066 rtase_w16(tp, ivec->isr_addr, status);
1067
1068 if (napi_schedule_prep(&ivec->napi))
1069 __napi_schedule(&ivec->napi);
1070
1071 return IRQ_HANDLED;
1072 }
1073
rtase_poll(struct napi_struct * napi,int budget)1074 static int rtase_poll(struct napi_struct *napi, int budget)
1075 {
1076 const struct rtase_int_vector *ivec;
1077 const struct rtase_private *tp;
1078 struct rtase_ring *ring;
1079 int total_workdone = 0;
1080
1081 ivec = container_of(napi, struct rtase_int_vector, napi);
1082 tp = ivec->tp;
1083
1084 list_for_each_entry(ring, &ivec->ring_list, ring_entry)
1085 total_workdone += ring->ring_handler(ring, budget);
1086
1087 if (total_workdone >= budget)
1088 return budget;
1089
1090 if (napi_complete_done(napi, total_workdone)) {
1091 if (!ivec->index)
1092 rtase_w32(tp, ivec->imr_addr, ivec->imr);
1093 else
1094 rtase_w16(tp, ivec->imr_addr, ivec->imr);
1095 }
1096
1097 return total_workdone;
1098 }
1099
rtase_open(struct net_device * dev)1100 static int rtase_open(struct net_device *dev)
1101 {
1102 struct rtase_private *tp = netdev_priv(dev);
1103 const struct pci_dev *pdev = tp->pdev;
1104 struct rtase_int_vector *ivec;
1105 u16 i = 0, j;
1106 int ret;
1107
1108 ivec = &tp->int_vector[0];
1109 tp->rx_buf_sz = RTASE_RX_BUF_SIZE;
1110
1111 ret = rtase_alloc_desc(tp);
1112 if (ret)
1113 return ret;
1114
1115 ret = rtase_init_ring(dev);
1116 if (ret)
1117 goto err_free_all_allocated_mem;
1118
1119 rtase_hw_config(dev);
1120
1121 if (tp->sw_flag & RTASE_SWF_MSIX_ENABLED) {
1122 ret = request_irq(ivec->irq, rtase_interrupt, 0,
1123 dev->name, ivec);
1124 if (ret)
1125 goto err_free_all_allocated_irq;
1126
1127 /* request other interrupts to handle multiqueue */
1128 for (i = 1; i < tp->int_nums; i++) {
1129 ivec = &tp->int_vector[i];
1130 snprintf(ivec->name, sizeof(ivec->name), "%s_int%u",
1131 tp->dev->name, i);
1132 ret = request_irq(ivec->irq, rtase_q_interrupt, 0,
1133 ivec->name, ivec);
1134 if (ret)
1135 goto err_free_all_allocated_irq;
1136 }
1137 } else {
1138 ret = request_irq(pdev->irq, rtase_interrupt, 0, dev->name,
1139 ivec);
1140 if (ret)
1141 goto err_free_all_allocated_mem;
1142 }
1143
1144 rtase_hw_start(dev);
1145
1146 for (i = 0; i < tp->int_nums; i++) {
1147 ivec = &tp->int_vector[i];
1148 napi_enable(&ivec->napi);
1149 }
1150
1151 netif_carrier_on(dev);
1152 netif_wake_queue(dev);
1153
1154 return 0;
1155
1156 err_free_all_allocated_irq:
1157 for (j = 0; j < i; j++)
1158 free_irq(tp->int_vector[j].irq, &tp->int_vector[j]);
1159
1160 err_free_all_allocated_mem:
1161 rtase_free_desc(tp);
1162
1163 return ret;
1164 }
1165
rtase_down(struct net_device * dev)1166 static void rtase_down(struct net_device *dev)
1167 {
1168 struct rtase_private *tp = netdev_priv(dev);
1169 struct rtase_int_vector *ivec;
1170 struct rtase_ring *ring, *tmp;
1171 u32 i;
1172
1173 for (i = 0; i < tp->int_nums; i++) {
1174 ivec = &tp->int_vector[i];
1175 napi_disable(&ivec->napi);
1176 list_for_each_entry_safe(ring, tmp, &ivec->ring_list,
1177 ring_entry) {
1178 netif_queue_set_napi(tp->dev, ring->index,
1179 ring->type, NULL);
1180
1181 list_del(&ring->ring_entry);
1182 }
1183 }
1184
1185 netif_tx_disable(dev);
1186
1187 netif_carrier_off(dev);
1188
1189 rtase_hw_reset(dev);
1190
1191 rtase_tx_clear(tp);
1192
1193 rtase_rx_clear(tp);
1194 }
1195
rtase_close(struct net_device * dev)1196 static int rtase_close(struct net_device *dev)
1197 {
1198 struct rtase_private *tp = netdev_priv(dev);
1199 const struct pci_dev *pdev = tp->pdev;
1200 u32 i;
1201
1202 rtase_down(dev);
1203
1204 if (tp->sw_flag & RTASE_SWF_MSIX_ENABLED) {
1205 for (i = 0; i < tp->int_nums; i++)
1206 free_irq(tp->int_vector[i].irq, &tp->int_vector[i]);
1207
1208 } else {
1209 free_irq(pdev->irq, &tp->int_vector[0]);
1210 }
1211
1212 rtase_free_desc(tp);
1213
1214 return 0;
1215 }
1216
rtase_tx_vlan_tag(const struct rtase_private * tp,const struct sk_buff * skb)1217 static u32 rtase_tx_vlan_tag(const struct rtase_private *tp,
1218 const struct sk_buff *skb)
1219 {
1220 return (skb_vlan_tag_present(skb)) ?
1221 (RTASE_TX_VLAN_TAG | swab16(skb_vlan_tag_get(skb))) : 0x00;
1222 }
1223
rtase_tx_csum(struct sk_buff * skb,const struct net_device * dev)1224 static u32 rtase_tx_csum(struct sk_buff *skb, const struct net_device *dev)
1225 {
1226 u32 csum_cmd = 0;
1227 u8 ip_protocol;
1228
1229 switch (vlan_get_protocol(skb)) {
1230 case htons(ETH_P_IP):
1231 csum_cmd = RTASE_TX_IPCS_C;
1232 ip_protocol = ip_hdr(skb)->protocol;
1233 break;
1234
1235 case htons(ETH_P_IPV6):
1236 csum_cmd = RTASE_TX_IPV6F_C;
1237 ip_protocol = ipv6_hdr(skb)->nexthdr;
1238 break;
1239
1240 default:
1241 ip_protocol = IPPROTO_RAW;
1242 break;
1243 }
1244
1245 if (ip_protocol == IPPROTO_TCP)
1246 csum_cmd |= RTASE_TX_TCPCS_C;
1247 else if (ip_protocol == IPPROTO_UDP)
1248 csum_cmd |= RTASE_TX_UDPCS_C;
1249
1250 csum_cmd |= u32_encode_bits(skb_transport_offset(skb),
1251 RTASE_TCPHO_MASK);
1252
1253 return csum_cmd;
1254 }
1255
rtase_get_l3_proto(struct sk_buff * skb,__be16 * proto,u32 * network_offset)1256 static enum rtase_parse_result rtase_get_l3_proto(struct sk_buff *skb,
1257 __be16 *proto,
1258 u32 *network_offset)
1259 {
1260 struct vlan_hdr *vh, _vh;
1261 struct ethhdr *eh, _eh;
1262 u32 offset = ETH_HLEN;
1263
1264 eh = skb_header_pointer(skb, 0, sizeof(_eh), &_eh);
1265 if (!eh)
1266 return RTASE_PARSE_DROP;
1267
1268 *proto = eh->h_proto;
1269
1270 while (eth_type_vlan(*proto)) {
1271 vh = skb_header_pointer(skb, offset, sizeof(_vh), &_vh);
1272 if (!vh)
1273 return RTASE_PARSE_DROP;
1274
1275 *proto = vh->h_vlan_encapsulated_proto;
1276 offset += VLAN_HLEN;
1277 }
1278
1279 *network_offset = offset;
1280
1281 return RTASE_PARSE_OK;
1282 }
1283
rtase_pad_to_transport_len(struct sk_buff * skb,u32 transport_offset,u32 pad_to_len)1284 static bool rtase_pad_to_transport_len(struct sk_buff *skb,
1285 u32 transport_offset,
1286 u32 pad_to_len)
1287 {
1288 u32 trans_data_len;
1289 u32 pad_len;
1290
1291 trans_data_len = skb->len - transport_offset;
1292 if (trans_data_len >= pad_to_len)
1293 return true;
1294
1295 if (skb_is_nonlinear(skb)) {
1296 if (skb_linearize(skb))
1297 return false;
1298 }
1299
1300 pad_len = pad_to_len - trans_data_len;
1301 if (__skb_put_padto(skb, skb->len + pad_len, false))
1302 return false;
1303
1304 return true;
1305 }
1306
rtase_get_transport_offset(struct sk_buff * skb,u32 * transport_offset,u8 * transport_proto,u32 * pad_to_len)1307 static enum rtase_parse_result rtase_get_transport_offset(struct sk_buff *skb,
1308 u32 *transport_offset,
1309 u8 *transport_proto,
1310 u32 *pad_to_len)
1311 {
1312 enum rtase_parse_result ret;
1313 struct ipv6hdr *i6h, _i6h;
1314 struct iphdr *ih, _ih;
1315 bool non_first_frag;
1316 __be16 proto;
1317 u32 offset;
1318 u32 no;
1319
1320 ret = rtase_get_l3_proto(skb, &proto, &no);
1321 if (ret != RTASE_PARSE_OK)
1322 return ret;
1323
1324 switch (proto) {
1325 case htons(ETH_P_IP):
1326 ih = skb_header_pointer(skb, no, sizeof(_ih), &_ih);
1327 if (!ih)
1328 return RTASE_PARSE_DROP;
1329
1330 if (ih->ihl < 5)
1331 return RTASE_PARSE_DROP;
1332
1333 offset = no + ih->ihl * 4;
1334 if (offset > skb->len)
1335 return RTASE_PARSE_DROP;
1336
1337 non_first_frag = ntohs(ih->frag_off) & IP_OFFSET;
1338
1339 if (ih->protocol == IPPROTO_TCP) {
1340 if (skb->len - offset < sizeof(struct tcphdr)) {
1341 if (non_first_frag) {
1342 *transport_offset = offset;
1343 *transport_proto = IPPROTO_TCP;
1344 *pad_to_len = sizeof(struct tcphdr);
1345
1346 return RTASE_PARSE_OK;
1347 }
1348
1349 return RTASE_PARSE_DROP;
1350 }
1351
1352 return RTASE_PARSE_SKIP;
1353 }
1354
1355 if (ih->protocol != IPPROTO_UDP)
1356 return RTASE_PARSE_SKIP;
1357
1358 *transport_offset = offset;
1359 *transport_proto = IPPROTO_UDP;
1360
1361 if (skb->len - offset < sizeof(struct udphdr)) {
1362 if (non_first_frag) {
1363 *pad_to_len = sizeof(struct udphdr);
1364
1365 return RTASE_PARSE_OK;
1366 }
1367
1368 return RTASE_PARSE_DROP;
1369 }
1370
1371 return RTASE_PARSE_OK;
1372
1373 case htons(ETH_P_IPV6):
1374 i6h = skb_header_pointer(skb, no, sizeof(_i6h), &_i6h);
1375 if (!i6h)
1376 return RTASE_PARSE_DROP;
1377
1378 offset = no + sizeof(*i6h);
1379
1380 if (i6h->nexthdr == IPPROTO_TCP) {
1381 if (skb->len - offset < sizeof(struct tcphdr))
1382 return RTASE_PARSE_DROP;
1383
1384 return RTASE_PARSE_SKIP;
1385 }
1386
1387 if (i6h->nexthdr != IPPROTO_UDP)
1388 return RTASE_PARSE_SKIP;
1389
1390 if (skb->len - offset < sizeof(struct udphdr))
1391 return RTASE_PARSE_DROP;
1392
1393 *transport_offset = offset;
1394 *transport_proto = IPPROTO_UDP;
1395
1396 return RTASE_PARSE_OK;
1397
1398 default:
1399 return RTASE_PARSE_SKIP;
1400 }
1401 }
1402
rtase_skb_pad(struct sk_buff * skb)1403 static bool rtase_skb_pad(struct sk_buff *skb)
1404 {
1405 enum rtase_parse_result ret;
1406 u32 transport_offset;
1407 __be16 *dest, _dest;
1408 u32 trans_data_len;
1409 u32 pad_to_len = 0;
1410 u8 transport_proto;
1411 u16 dest_port;
1412
1413 ret = rtase_get_transport_offset(skb, &transport_offset,
1414 &transport_proto, &pad_to_len);
1415 if (ret == RTASE_PARSE_SKIP) {
1416 return true;
1417 } else if (ret == RTASE_PARSE_DROP) {
1418 netdev_dbg(skb->dev, "drop malformed packet\n");
1419 return false;
1420 }
1421
1422 if (pad_to_len &&
1423 !rtase_pad_to_transport_len(skb, transport_offset, pad_to_len))
1424 return false;
1425
1426 if (transport_proto != IPPROTO_UDP)
1427 return true;
1428
1429 trans_data_len = skb->len - transport_offset;
1430 if (trans_data_len < offsetof(struct udphdr, len) ||
1431 trans_data_len >= RTASE_MIN_PAD_LEN)
1432 return true;
1433
1434 dest = skb_header_pointer(skb,
1435 transport_offset +
1436 offsetof(struct udphdr, dest),
1437 sizeof(_dest), &_dest);
1438 if (!dest)
1439 return true;
1440
1441 dest_port = ntohs(*dest);
1442 if (dest_port != PTP_EV_PORT && dest_port != PTP_GEN_PORT)
1443 return true;
1444
1445 return rtase_pad_to_transport_len(skb, transport_offset,
1446 RTASE_MIN_PAD_LEN);
1447 }
1448
rtase_xmit_frags(struct rtase_ring * ring,struct sk_buff * skb,u32 opts1,u32 opts2)1449 static int rtase_xmit_frags(struct rtase_ring *ring, struct sk_buff *skb,
1450 u32 opts1, u32 opts2)
1451 {
1452 const struct skb_shared_info *info = skb_shinfo(skb);
1453 const struct rtase_private *tp = ring->ivec->tp;
1454 const u8 nr_frags = info->nr_frags;
1455 struct rtase_tx_desc *txd = NULL;
1456 u32 cur_frag, entry;
1457
1458 entry = ring->cur_idx;
1459 for (cur_frag = 0; cur_frag < nr_frags; cur_frag++) {
1460 const skb_frag_t *frag = &info->frags[cur_frag];
1461 dma_addr_t mapping;
1462 u32 status, len;
1463 void *addr;
1464
1465 entry = (entry + 1) % RTASE_NUM_DESC;
1466
1467 txd = ring->desc + sizeof(struct rtase_tx_desc) * entry;
1468 len = skb_frag_size(frag);
1469 addr = skb_frag_address(frag);
1470 mapping = dma_map_single(&tp->pdev->dev, addr, len,
1471 DMA_TO_DEVICE);
1472
1473 if (unlikely(dma_mapping_error(&tp->pdev->dev, mapping))) {
1474 if (unlikely(net_ratelimit()))
1475 netdev_err(tp->dev,
1476 "Failed to map TX fragments DMA!\n");
1477
1478 goto err_out;
1479 }
1480
1481 if (((entry + 1) % RTASE_NUM_DESC) == 0)
1482 status = (opts1 | len | RTASE_RING_END);
1483 else
1484 status = opts1 | len;
1485
1486 if (cur_frag == (nr_frags - 1)) {
1487 ring->skbuff[entry] = skb;
1488 status |= RTASE_TX_LAST_FRAG;
1489 }
1490
1491 ring->mis.len[entry] = len;
1492 txd->addr = cpu_to_le64(mapping);
1493 txd->opts2 = cpu_to_le32(opts2);
1494
1495 /* make sure the operating fields have been updated */
1496 dma_wmb();
1497 txd->opts1 = cpu_to_le32(status);
1498 }
1499
1500 return cur_frag;
1501
1502 err_out:
1503 rtase_tx_clear_range(ring, ring->cur_idx + 1, cur_frag);
1504 return -EIO;
1505 }
1506
rtase_start_xmit(struct sk_buff * skb,struct net_device * dev)1507 static netdev_tx_t rtase_start_xmit(struct sk_buff *skb,
1508 struct net_device *dev)
1509 {
1510 struct skb_shared_info *shinfo = skb_shinfo(skb);
1511 struct rtase_private *tp = netdev_priv(dev);
1512 u32 q_idx, entry, len, opts1, opts2;
1513 struct netdev_queue *tx_queue;
1514 bool stop_queue, door_bell;
1515 u32 mss = shinfo->gso_size;
1516 struct rtase_tx_desc *txd;
1517 struct rtase_ring *ring;
1518 dma_addr_t mapping;
1519 int frags;
1520
1521 /* multiqueues */
1522 q_idx = skb_get_queue_mapping(skb);
1523 ring = &tp->tx_ring[q_idx];
1524 tx_queue = netdev_get_tx_queue(dev, q_idx);
1525
1526 if (unlikely(!rtase_tx_avail(ring))) {
1527 if (net_ratelimit())
1528 netdev_err(dev,
1529 "BUG! Tx Ring full when queue awake!\n");
1530
1531 netif_stop_queue(dev);
1532 return NETDEV_TX_BUSY;
1533 }
1534
1535 entry = ring->cur_idx % RTASE_NUM_DESC;
1536 txd = ring->desc + sizeof(struct rtase_tx_desc) * entry;
1537
1538 opts1 = RTASE_DESC_OWN;
1539 opts2 = rtase_tx_vlan_tag(tp, skb);
1540
1541 /* tcp segmentation offload (or tcp large send) */
1542 if (mss) {
1543 if (shinfo->gso_type & SKB_GSO_TCPV4) {
1544 opts1 |= RTASE_GIANT_SEND_V4;
1545 } else if (shinfo->gso_type & SKB_GSO_TCPV6) {
1546 if (skb_cow_head(skb, 0))
1547 goto err_dma_0;
1548
1549 tcp_v6_gso_csum_prep(skb);
1550 opts1 |= RTASE_GIANT_SEND_V6;
1551 } else {
1552 WARN_ON_ONCE(1);
1553 }
1554
1555 opts1 |= u32_encode_bits(skb_transport_offset(skb),
1556 RTASE_TCPHO_MASK);
1557 opts2 |= u32_encode_bits(mss, RTASE_MSS_MASK);
1558 } else if (skb->ip_summed == CHECKSUM_PARTIAL) {
1559 opts2 |= rtase_tx_csum(skb, dev);
1560 }
1561
1562 if (!rtase_skb_pad(skb))
1563 goto err_dma_0;
1564
1565 frags = rtase_xmit_frags(ring, skb, opts1, opts2);
1566 if (unlikely(frags < 0))
1567 goto err_dma_0;
1568
1569 if (frags) {
1570 len = skb_headlen(skb);
1571 opts1 |= RTASE_TX_FIRST_FRAG;
1572 } else {
1573 len = skb->len;
1574 ring->skbuff[entry] = skb;
1575 opts1 |= RTASE_TX_FIRST_FRAG | RTASE_TX_LAST_FRAG;
1576 }
1577
1578 if (((entry + 1) % RTASE_NUM_DESC) == 0)
1579 opts1 |= (len | RTASE_RING_END);
1580 else
1581 opts1 |= len;
1582
1583 mapping = dma_map_single(&tp->pdev->dev, skb->data, len,
1584 DMA_TO_DEVICE);
1585
1586 if (unlikely(dma_mapping_error(&tp->pdev->dev, mapping))) {
1587 if (unlikely(net_ratelimit()))
1588 netdev_err(dev, "Failed to map TX DMA!\n");
1589
1590 goto err_dma_1;
1591 }
1592
1593 ring->mis.len[entry] = len;
1594 txd->addr = cpu_to_le64(mapping);
1595 txd->opts2 = cpu_to_le32(opts2);
1596 txd->opts1 = cpu_to_le32(opts1 & ~RTASE_DESC_OWN);
1597
1598 /* make sure the operating fields have been updated */
1599 dma_wmb();
1600
1601 door_bell = __netdev_tx_sent_queue(tx_queue, skb->len,
1602 netdev_xmit_more());
1603
1604 txd->opts1 = cpu_to_le32(opts1);
1605
1606 skb_tx_timestamp(skb);
1607
1608 /* tx needs to see descriptor changes before updated cur_idx */
1609 smp_wmb();
1610
1611 WRITE_ONCE(ring->cur_idx, ring->cur_idx + frags + 1);
1612
1613 stop_queue = !netif_subqueue_maybe_stop(dev, ring->index,
1614 rtase_tx_avail(ring),
1615 RTASE_TX_STOP_THRS,
1616 RTASE_TX_START_THRS);
1617
1618 if (door_bell || stop_queue)
1619 rtase_w8(tp, RTASE_TPPOLL, BIT(ring->index));
1620
1621 return NETDEV_TX_OK;
1622
1623 err_dma_1:
1624 ring->skbuff[entry] = NULL;
1625 rtase_tx_clear_range(ring, ring->cur_idx + 1, frags);
1626 if (frags)
1627 /* the frags were cleared above, along with the skb */
1628 return NETDEV_TX_OK;
1629
1630 err_dma_0:
1631 tp->stats.tx_dropped++;
1632 dev_kfree_skb_any(skb);
1633 return NETDEV_TX_OK;
1634 }
1635
rtase_set_rx_mode(struct net_device * dev)1636 static void rtase_set_rx_mode(struct net_device *dev)
1637 {
1638 rtase_hw_set_rx_packet_filter(dev);
1639 }
1640
rtase_enable_eem_write(const struct rtase_private * tp)1641 static void rtase_enable_eem_write(const struct rtase_private *tp)
1642 {
1643 u8 val;
1644
1645 val = rtase_r8(tp, RTASE_EEM);
1646 rtase_w8(tp, RTASE_EEM, val | RTASE_EEM_UNLOCK);
1647 }
1648
rtase_disable_eem_write(const struct rtase_private * tp)1649 static void rtase_disable_eem_write(const struct rtase_private *tp)
1650 {
1651 u8 val;
1652
1653 val = rtase_r8(tp, RTASE_EEM);
1654 rtase_w8(tp, RTASE_EEM, val & ~RTASE_EEM_UNLOCK);
1655 }
1656
rtase_rar_set(const struct rtase_private * tp,const u8 * addr)1657 static void rtase_rar_set(const struct rtase_private *tp, const u8 *addr)
1658 {
1659 u32 rar_low, rar_high;
1660
1661 rar_low = (u32)addr[0] | ((u32)addr[1] << 8) |
1662 ((u32)addr[2] << 16) | ((u32)addr[3] << 24);
1663
1664 rar_high = (u32)addr[4] | ((u32)addr[5] << 8);
1665
1666 rtase_enable_eem_write(tp);
1667 rtase_w32(tp, RTASE_MAC0, rar_low);
1668 rtase_w32(tp, RTASE_MAC4, rar_high);
1669 rtase_disable_eem_write(tp);
1670 rtase_w16(tp, RTASE_LBK_CTRL, RTASE_LBK_ATLD | RTASE_LBK_CLR);
1671 }
1672
rtase_set_mac_address(struct net_device * dev,void * p)1673 static int rtase_set_mac_address(struct net_device *dev, void *p)
1674 {
1675 struct rtase_private *tp = netdev_priv(dev);
1676 int ret;
1677
1678 ret = eth_mac_addr(dev, p);
1679 if (ret)
1680 return ret;
1681
1682 rtase_rar_set(tp, dev->dev_addr);
1683
1684 return 0;
1685 }
1686
rtase_change_mtu(struct net_device * dev,int new_mtu)1687 static int rtase_change_mtu(struct net_device *dev, int new_mtu)
1688 {
1689 dev->mtu = new_mtu;
1690
1691 netdev_update_features(dev);
1692
1693 return 0;
1694 }
1695
rtase_wait_for_quiescence(const struct net_device * dev)1696 static void rtase_wait_for_quiescence(const struct net_device *dev)
1697 {
1698 struct rtase_private *tp = netdev_priv(dev);
1699 struct rtase_int_vector *ivec;
1700 u32 i;
1701
1702 for (i = 0; i < tp->int_nums; i++) {
1703 ivec = &tp->int_vector[i];
1704 synchronize_irq(ivec->irq);
1705 /* wait for any pending NAPI task to complete */
1706 napi_disable(&ivec->napi);
1707 }
1708
1709 rtase_irq_dis_and_clear(tp);
1710
1711 for (i = 0; i < tp->int_nums; i++) {
1712 ivec = &tp->int_vector[i];
1713 napi_enable(&ivec->napi);
1714 }
1715 }
1716
rtase_sw_reset(struct net_device * dev)1717 static void rtase_sw_reset(struct net_device *dev)
1718 {
1719 struct rtase_private *tp = netdev_priv(dev);
1720 struct rtase_ring *ring, *tmp;
1721 struct rtase_int_vector *ivec;
1722 int ret;
1723 u32 i;
1724
1725 netif_stop_queue(dev);
1726 netif_carrier_off(dev);
1727 rtase_hw_reset(dev);
1728
1729 /* let's wait a bit while any (async) irq lands on */
1730 rtase_wait_for_quiescence(dev);
1731 rtase_tx_clear(tp);
1732 rtase_rx_clear(tp);
1733
1734 for (i = 0; i < tp->int_nums; i++) {
1735 ivec = &tp->int_vector[i];
1736 list_for_each_entry_safe(ring, tmp, &ivec->ring_list,
1737 ring_entry) {
1738 netif_queue_set_napi(tp->dev, ring->index,
1739 ring->type, NULL);
1740
1741 list_del(&ring->ring_entry);
1742 }
1743 }
1744
1745 ret = rtase_init_ring(dev);
1746 if (ret) {
1747 netdev_err(dev, "unable to init ring\n");
1748 rtase_free_desc(tp);
1749 return;
1750 }
1751
1752 rtase_hw_config(dev);
1753 /* always link, so start to transmit & receive */
1754 rtase_hw_start(dev);
1755
1756 netif_carrier_on(dev);
1757 netif_wake_queue(dev);
1758 }
1759
rtase_dump_tally_counter(const struct rtase_private * tp)1760 static void rtase_dump_tally_counter(const struct rtase_private *tp)
1761 {
1762 dma_addr_t paddr = tp->tally_paddr;
1763 u32 cmd = lower_32_bits(paddr);
1764 u32 val;
1765 int err;
1766
1767 rtase_w32(tp, RTASE_DTCCR4, upper_32_bits(paddr));
1768 rtase_w32(tp, RTASE_DTCCR0, cmd);
1769 rtase_w32(tp, RTASE_DTCCR0, cmd | RTASE_COUNTER_DUMP);
1770
1771 err = read_poll_timeout_atomic(rtase_r32, val,
1772 !(val & RTASE_COUNTER_DUMP),
1773 10, 250, false, tp, RTASE_DTCCR0);
1774
1775 if (err == -ETIMEDOUT)
1776 netdev_err(tp->dev, "error occurred in dump tally counter\n");
1777 }
1778
rtase_dump_state(const struct net_device * dev)1779 static void rtase_dump_state(const struct net_device *dev)
1780 {
1781 const struct rtase_private *tp = netdev_priv(dev);
1782 int max_reg_size = RTASE_PCI_REGS_SIZE;
1783 const struct rtase_counters *counters;
1784 const struct rtase_ring *ring;
1785 u32 dword_rd;
1786 int n = 0;
1787
1788 ring = &tp->tx_ring[0];
1789 netdev_err(dev, "Tx descriptor info:\n");
1790 netdev_err(dev, "Tx curIdx = 0x%x\n", ring->cur_idx);
1791 netdev_err(dev, "Tx dirtyIdx = 0x%x\n", ring->dirty_idx);
1792 netdev_err(dev, "Tx phyAddr = %pad\n", &ring->phy_addr);
1793
1794 ring = &tp->rx_ring[0];
1795 netdev_err(dev, "Rx descriptor info:\n");
1796 netdev_err(dev, "Rx curIdx = 0x%x\n", ring->cur_idx);
1797 netdev_err(dev, "Rx dirtyIdx = 0x%x\n", ring->dirty_idx);
1798 netdev_err(dev, "Rx phyAddr = %pad\n", &ring->phy_addr);
1799
1800 netdev_err(dev, "Device Registers:\n");
1801 netdev_err(dev, "Chip Command = 0x%02x\n",
1802 rtase_r8(tp, RTASE_CHIP_CMD));
1803 netdev_err(dev, "IMR = %08x\n", rtase_r32(tp, RTASE_IMR0));
1804 netdev_err(dev, "ISR = %08x\n", rtase_r32(tp, RTASE_ISR0));
1805 netdev_err(dev, "Boot Ctrl Reg(0xE004) = %04x\n",
1806 rtase_r16(tp, RTASE_BOOT_CTL));
1807 netdev_err(dev, "EPHY ISR(0xE014) = %04x\n",
1808 rtase_r16(tp, RTASE_EPHY_ISR));
1809 netdev_err(dev, "EPHY IMR(0xE016) = %04x\n",
1810 rtase_r16(tp, RTASE_EPHY_IMR));
1811 netdev_err(dev, "CLKSW SET REG(0xE018) = %04x\n",
1812 rtase_r16(tp, RTASE_CLKSW_SET));
1813
1814 netdev_err(dev, "Dump PCI Registers:\n");
1815
1816 while (n < max_reg_size) {
1817 if ((n % RTASE_DWORD_MOD) == 0)
1818 netdev_err(tp->dev, "0x%03x:\n", n);
1819
1820 pci_read_config_dword(tp->pdev, n, &dword_rd);
1821 netdev_err(tp->dev, "%08x\n", dword_rd);
1822 n += 4;
1823 }
1824
1825 netdev_err(dev, "Dump tally counter:\n");
1826 counters = tp->tally_vaddr;
1827 rtase_dump_tally_counter(tp);
1828
1829 netdev_err(dev, "tx_packets %lld\n",
1830 le64_to_cpu(counters->tx_packets));
1831 netdev_err(dev, "rx_packets %lld\n",
1832 le64_to_cpu(counters->rx_packets));
1833 netdev_err(dev, "tx_errors %lld\n",
1834 le64_to_cpu(counters->tx_errors));
1835 netdev_err(dev, "rx_errors %d\n",
1836 le32_to_cpu(counters->rx_errors));
1837 netdev_err(dev, "rx_missed %d\n",
1838 le16_to_cpu(counters->rx_missed));
1839 netdev_err(dev, "align_errors %d\n",
1840 le16_to_cpu(counters->align_errors));
1841 netdev_err(dev, "tx_one_collision %d\n",
1842 le32_to_cpu(counters->tx_one_collision));
1843 netdev_err(dev, "tx_multi_collision %d\n",
1844 le32_to_cpu(counters->tx_multi_collision));
1845 netdev_err(dev, "rx_unicast %lld\n",
1846 le64_to_cpu(counters->rx_unicast));
1847 netdev_err(dev, "rx_broadcast %lld\n",
1848 le64_to_cpu(counters->rx_broadcast));
1849 netdev_err(dev, "rx_multicast %d\n",
1850 le32_to_cpu(counters->rx_multicast));
1851 netdev_err(dev, "tx_aborted %d\n",
1852 le16_to_cpu(counters->tx_aborted));
1853 netdev_err(dev, "tx_underrun %d\n",
1854 le16_to_cpu(counters->tx_underrun));
1855 }
1856
rtase_tx_timeout(struct net_device * dev,unsigned int txqueue)1857 static void rtase_tx_timeout(struct net_device *dev, unsigned int txqueue)
1858 {
1859 rtase_dump_state(dev);
1860 rtase_sw_reset(dev);
1861 }
1862
rtase_get_stats64(struct net_device * dev,struct rtnl_link_stats64 * stats)1863 static void rtase_get_stats64(struct net_device *dev,
1864 struct rtnl_link_stats64 *stats)
1865 {
1866 const struct rtase_private *tp = netdev_priv(dev);
1867 const struct rtase_counters *counters;
1868
1869 counters = tp->tally_vaddr;
1870
1871 dev_fetch_sw_netstats(stats, dev->tstats);
1872
1873 /* fetch additional counter values missing in stats collected by driver
1874 * from tally counter
1875 */
1876 rtase_dump_tally_counter(tp);
1877 stats->rx_errors = tp->stats.rx_errors;
1878 stats->tx_errors = le64_to_cpu(counters->tx_errors);
1879 stats->rx_dropped = tp->stats.rx_dropped;
1880 stats->tx_dropped = tp->stats.tx_dropped;
1881 stats->multicast = tp->stats.multicast;
1882 stats->rx_length_errors = tp->stats.rx_length_errors;
1883 }
1884
rtase_set_hw_cbs(const struct rtase_private * tp,u32 queue)1885 static void rtase_set_hw_cbs(const struct rtase_private *tp, u32 queue)
1886 {
1887 u32 idle = tp->tx_qos[queue].idleslope * RTASE_1T_CLOCK;
1888 u32 val, i;
1889
1890 val = u32_encode_bits(idle / RTASE_1T_POWER, RTASE_IDLESLOPE_INT_MASK);
1891 idle %= RTASE_1T_POWER;
1892
1893 for (i = 1; i <= RTASE_IDLESLOPE_INT_SHIFT; i++) {
1894 idle *= 2;
1895 if ((idle / RTASE_1T_POWER) == 1)
1896 val |= BIT(RTASE_IDLESLOPE_INT_SHIFT - i);
1897
1898 idle %= RTASE_1T_POWER;
1899 }
1900
1901 rtase_w32(tp, RTASE_TXQCRDT_0 + queue * 4, val);
1902 }
1903
rtase_setup_tc_cbs(struct rtase_private * tp,const struct tc_cbs_qopt_offload * qopt)1904 static int rtase_setup_tc_cbs(struct rtase_private *tp,
1905 const struct tc_cbs_qopt_offload *qopt)
1906 {
1907 int queue = qopt->queue;
1908
1909 if (queue < 0 || queue >= tp->func_tx_queue_num)
1910 return -EINVAL;
1911
1912 if (!qopt->enable) {
1913 tp->tx_qos[queue].hicredit = 0;
1914 tp->tx_qos[queue].locredit = 0;
1915 tp->tx_qos[queue].idleslope = 0;
1916 tp->tx_qos[queue].sendslope = 0;
1917
1918 rtase_w32(tp, RTASE_TXQCRDT_0 + queue * 4, 0);
1919 } else {
1920 tp->tx_qos[queue].hicredit = qopt->hicredit;
1921 tp->tx_qos[queue].locredit = qopt->locredit;
1922 tp->tx_qos[queue].idleslope = qopt->idleslope;
1923 tp->tx_qos[queue].sendslope = qopt->sendslope;
1924
1925 rtase_set_hw_cbs(tp, queue);
1926 }
1927
1928 return 0;
1929 }
1930
rtase_setup_tc(struct net_device * dev,enum tc_setup_type type,void * type_data)1931 static int rtase_setup_tc(struct net_device *dev, enum tc_setup_type type,
1932 void *type_data)
1933 {
1934 struct rtase_private *tp = netdev_priv(dev);
1935
1936 switch (type) {
1937 case TC_SETUP_QDISC_CBS:
1938 return rtase_setup_tc_cbs(tp, type_data);
1939 default:
1940 return -EOPNOTSUPP;
1941 }
1942 }
1943
rtase_fix_features(struct net_device * dev,netdev_features_t features)1944 static netdev_features_t rtase_fix_features(struct net_device *dev,
1945 netdev_features_t features)
1946 {
1947 netdev_features_t features_fix = features;
1948
1949 /* not support TSO for jumbo frames */
1950 if (dev->mtu > ETH_DATA_LEN)
1951 features_fix &= ~NETIF_F_ALL_TSO;
1952
1953 return features_fix;
1954 }
1955
rtase_set_features(struct net_device * dev,netdev_features_t features)1956 static int rtase_set_features(struct net_device *dev,
1957 netdev_features_t features)
1958 {
1959 netdev_features_t features_set = features;
1960
1961 features_set &= NETIF_F_RXALL | NETIF_F_RXCSUM |
1962 NETIF_F_HW_VLAN_CTAG_RX;
1963
1964 if (features_set ^ dev->features)
1965 rtase_hw_set_features(dev, features_set);
1966
1967 return 0;
1968 }
1969
1970 static const struct net_device_ops rtase_netdev_ops = {
1971 .ndo_open = rtase_open,
1972 .ndo_stop = rtase_close,
1973 .ndo_start_xmit = rtase_start_xmit,
1974 .ndo_set_rx_mode = rtase_set_rx_mode,
1975 .ndo_set_mac_address = rtase_set_mac_address,
1976 .ndo_change_mtu = rtase_change_mtu,
1977 .ndo_tx_timeout = rtase_tx_timeout,
1978 .ndo_get_stats64 = rtase_get_stats64,
1979 .ndo_setup_tc = rtase_setup_tc,
1980 .ndo_fix_features = rtase_fix_features,
1981 .ndo_set_features = rtase_set_features,
1982 };
1983
rtase_get_mac_address(struct net_device * dev)1984 static void rtase_get_mac_address(struct net_device *dev)
1985 {
1986 struct rtase_private *tp = netdev_priv(dev);
1987 u8 mac_addr[ETH_ALEN] __aligned(2) = {};
1988 u32 i;
1989
1990 for (i = 0; i < ETH_ALEN; i++)
1991 mac_addr[i] = rtase_r8(tp, RTASE_MAC0 + i);
1992
1993 if (!is_valid_ether_addr(mac_addr)) {
1994 eth_hw_addr_random(dev);
1995 netdev_warn(dev, "Random ether addr %pM\n", dev->dev_addr);
1996 } else {
1997 eth_hw_addr_set(dev, mac_addr);
1998 ether_addr_copy(dev->perm_addr, dev->dev_addr);
1999 }
2000
2001 rtase_rar_set(tp, dev->dev_addr);
2002 }
2003
rtase_get_settings(struct net_device * dev,struct ethtool_link_ksettings * cmd)2004 static int rtase_get_settings(struct net_device *dev,
2005 struct ethtool_link_ksettings *cmd)
2006 {
2007 u32 supported = SUPPORTED_MII | SUPPORTED_Pause | SUPPORTED_Asym_Pause;
2008 const struct rtase_private *tp = netdev_priv(dev);
2009
2010 ethtool_convert_legacy_u32_to_link_mode(cmd->link_modes.supported,
2011 supported);
2012
2013 switch (tp->hw_ver) {
2014 case RTASE_HW_VER_906X_7XA:
2015 case RTASE_HW_VER_906X_7XC:
2016 cmd->base.speed = SPEED_5000;
2017 break;
2018 case RTASE_HW_VER_907XD_V1:
2019 case RTASE_HW_VER_907XD_VA:
2020 cmd->base.speed = SPEED_10000;
2021 break;
2022 }
2023
2024 cmd->base.duplex = DUPLEX_FULL;
2025 cmd->base.port = PORT_MII;
2026 cmd->base.autoneg = AUTONEG_DISABLE;
2027
2028 return 0;
2029 }
2030
rtase_get_pauseparam(struct net_device * dev,struct ethtool_pauseparam * pause)2031 static void rtase_get_pauseparam(struct net_device *dev,
2032 struct ethtool_pauseparam *pause)
2033 {
2034 const struct rtase_private *tp = netdev_priv(dev);
2035 u16 value = rtase_r16(tp, RTASE_CPLUS_CMD);
2036
2037 pause->autoneg = AUTONEG_DISABLE;
2038 pause->tx_pause = !!(value & RTASE_FORCE_TXFLOW_EN);
2039 pause->rx_pause = !!(value & RTASE_FORCE_RXFLOW_EN);
2040 }
2041
rtase_set_pauseparam(struct net_device * dev,struct ethtool_pauseparam * pause)2042 static int rtase_set_pauseparam(struct net_device *dev,
2043 struct ethtool_pauseparam *pause)
2044 {
2045 const struct rtase_private *tp = netdev_priv(dev);
2046 u16 value = rtase_r16(tp, RTASE_CPLUS_CMD);
2047
2048 if (pause->autoneg)
2049 return -EOPNOTSUPP;
2050
2051 value &= ~(RTASE_FORCE_TXFLOW_EN | RTASE_FORCE_RXFLOW_EN);
2052
2053 if (pause->tx_pause)
2054 value |= RTASE_FORCE_TXFLOW_EN;
2055
2056 if (pause->rx_pause)
2057 value |= RTASE_FORCE_RXFLOW_EN;
2058
2059 rtase_w16(tp, RTASE_CPLUS_CMD, value);
2060 return 0;
2061 }
2062
rtase_get_eth_mac_stats(struct net_device * dev,struct ethtool_eth_mac_stats * stats)2063 static void rtase_get_eth_mac_stats(struct net_device *dev,
2064 struct ethtool_eth_mac_stats *stats)
2065 {
2066 struct rtase_private *tp = netdev_priv(dev);
2067 const struct rtase_counters *counters;
2068
2069 counters = tp->tally_vaddr;
2070
2071 rtase_dump_tally_counter(tp);
2072
2073 stats->FramesTransmittedOK = le64_to_cpu(counters->tx_packets);
2074 stats->FramesReceivedOK = le64_to_cpu(counters->rx_packets);
2075 stats->FramesLostDueToIntMACXmitError =
2076 le64_to_cpu(counters->tx_errors);
2077 stats->BroadcastFramesReceivedOK = le64_to_cpu(counters->rx_broadcast);
2078 }
2079
2080 static const struct ethtool_ops rtase_ethtool_ops = {
2081 .get_link = ethtool_op_get_link,
2082 .get_link_ksettings = rtase_get_settings,
2083 .get_pauseparam = rtase_get_pauseparam,
2084 .set_pauseparam = rtase_set_pauseparam,
2085 .get_eth_mac_stats = rtase_get_eth_mac_stats,
2086 .get_ts_info = ethtool_op_get_ts_info,
2087 };
2088
rtase_init_netdev_ops(struct net_device * dev)2089 static void rtase_init_netdev_ops(struct net_device *dev)
2090 {
2091 dev->netdev_ops = &rtase_netdev_ops;
2092 dev->ethtool_ops = &rtase_ethtool_ops;
2093 }
2094
rtase_init_napi(struct rtase_private * tp)2095 static void rtase_init_napi(struct rtase_private *tp)
2096 {
2097 u16 i;
2098
2099 for (i = 0; i < tp->int_nums; i++) {
2100 netif_napi_add_config(tp->dev, &tp->int_vector[i].napi,
2101 tp->int_vector[i].poll, i);
2102 netif_napi_set_irq(&tp->int_vector[i].napi,
2103 tp->int_vector[i].irq);
2104 }
2105 }
2106
rtase_reset_interrupt(struct pci_dev * pdev,const struct rtase_private * tp)2107 static void rtase_reset_interrupt(struct pci_dev *pdev,
2108 const struct rtase_private *tp)
2109 {
2110 if (tp->sw_flag & RTASE_SWF_MSIX_ENABLED)
2111 pci_disable_msix(pdev);
2112 else
2113 pci_disable_msi(pdev);
2114 }
2115
rtase_alloc_msix(struct pci_dev * pdev,struct rtase_private * tp)2116 static int rtase_alloc_msix(struct pci_dev *pdev, struct rtase_private *tp)
2117 {
2118 int ret, irq;
2119 u16 i;
2120
2121 memset(tp->msix_entry, 0x0, RTASE_NUM_MSIX *
2122 sizeof(struct msix_entry));
2123
2124 for (i = 0; i < RTASE_NUM_MSIX; i++)
2125 tp->msix_entry[i].entry = i;
2126
2127 ret = pci_enable_msix_exact(pdev, tp->msix_entry, tp->int_nums);
2128
2129 if (ret)
2130 return ret;
2131
2132 for (i = 0; i < tp->int_nums; i++) {
2133 irq = pci_irq_vector(pdev, i);
2134 if (irq < 0) {
2135 pci_disable_msix(pdev);
2136 return irq;
2137 }
2138
2139 tp->int_vector[i].irq = irq;
2140 }
2141
2142 return 0;
2143 }
2144
rtase_alloc_interrupt(struct pci_dev * pdev,struct rtase_private * tp)2145 static int rtase_alloc_interrupt(struct pci_dev *pdev,
2146 struct rtase_private *tp)
2147 {
2148 int ret;
2149
2150 ret = rtase_alloc_msix(pdev, tp);
2151 if (ret) {
2152 ret = pci_enable_msi(pdev);
2153 if (ret) {
2154 dev_err(&pdev->dev,
2155 "unable to alloc interrupt.(MSI)\n");
2156 return ret;
2157 }
2158
2159 tp->sw_flag |= RTASE_SWF_MSI_ENABLED;
2160 } else {
2161 tp->sw_flag |= RTASE_SWF_MSIX_ENABLED;
2162 }
2163
2164 return 0;
2165 }
2166
rtase_init_hardware(const struct rtase_private * tp)2167 static void rtase_init_hardware(const struct rtase_private *tp)
2168 {
2169 u16 i;
2170
2171 for (i = 0; i < RTASE_VLAN_FILTER_ENTRY_NUM; i++)
2172 rtase_w32(tp, RTASE_VLAN_ENTRY_0 + i * 4, 0);
2173 }
2174
rtase_init_int_vector(struct rtase_private * tp)2175 static void rtase_init_int_vector(struct rtase_private *tp)
2176 {
2177 u16 i;
2178
2179 /* interrupt vector 0 */
2180 tp->int_vector[0].tp = tp;
2181 tp->int_vector[0].index = 0;
2182 tp->int_vector[0].imr_addr = RTASE_IMR0;
2183 tp->int_vector[0].isr_addr = RTASE_ISR0;
2184 tp->int_vector[0].imr = RTASE_ROK | RTASE_RDU | RTASE_TOK |
2185 RTASE_TOK4 | RTASE_TOK5 | RTASE_TOK6 |
2186 RTASE_TOK7;
2187 tp->int_vector[0].poll = rtase_poll;
2188
2189 memset(tp->int_vector[0].name, 0x0, sizeof(tp->int_vector[0].name));
2190 INIT_LIST_HEAD(&tp->int_vector[0].ring_list);
2191
2192 /* interrupt vector 1 ~ 3 */
2193 for (i = 1; i < tp->int_nums; i++) {
2194 tp->int_vector[i].tp = tp;
2195 tp->int_vector[i].index = i;
2196 tp->int_vector[i].imr_addr = RTASE_IMR1 + (i - 1) * 4;
2197 tp->int_vector[i].isr_addr = RTASE_ISR1 + (i - 1) * 4;
2198 tp->int_vector[i].imr = RTASE_Q_ROK | RTASE_Q_RDU |
2199 RTASE_Q_TOK;
2200 tp->int_vector[i].poll = rtase_poll;
2201
2202 memset(tp->int_vector[i].name, 0x0,
2203 sizeof(tp->int_vector[0].name));
2204 INIT_LIST_HEAD(&tp->int_vector[i].ring_list);
2205 }
2206 }
2207
rtase_calc_time_mitigation(u32 time_us)2208 static u16 rtase_calc_time_mitigation(u32 time_us)
2209 {
2210 u8 msb, time_count, time_unit;
2211 u16 int_miti;
2212
2213 time_us = min(time_us, RTASE_MITI_MAX_TIME);
2214
2215 if (time_us > RTASE_MITI_TIME_COUNT_MASK) {
2216 msb = fls(time_us);
2217 time_unit = msb - RTASE_MITI_COUNT_BIT_NUM;
2218 time_count = time_us >> (msb - RTASE_MITI_COUNT_BIT_NUM);
2219 } else {
2220 time_unit = 0;
2221 time_count = time_us;
2222 }
2223
2224 int_miti = u16_encode_bits(time_count, RTASE_MITI_TIME_COUNT_MASK) |
2225 u16_encode_bits(time_unit, RTASE_MITI_TIME_UNIT_MASK);
2226
2227 return int_miti;
2228 }
2229
rtase_calc_packet_num_mitigation(u16 pkt_num)2230 static u16 rtase_calc_packet_num_mitigation(u16 pkt_num)
2231 {
2232 u8 msb, pkt_num_count, pkt_num_unit;
2233 u16 int_miti;
2234
2235 pkt_num = min(pkt_num, RTASE_MITI_MAX_PKT_NUM);
2236
2237 if (pkt_num > 60) {
2238 pkt_num_unit = RTASE_MITI_MAX_PKT_NUM_IDX;
2239 pkt_num_count = pkt_num / RTASE_MITI_MAX_PKT_NUM_UNIT;
2240 } else {
2241 msb = fls(pkt_num);
2242 if (msb >= RTASE_MITI_COUNT_BIT_NUM) {
2243 pkt_num_unit = msb - RTASE_MITI_COUNT_BIT_NUM;
2244 pkt_num_count = pkt_num >> (msb -
2245 RTASE_MITI_COUNT_BIT_NUM);
2246 } else {
2247 pkt_num_unit = 0;
2248 pkt_num_count = pkt_num;
2249 }
2250 }
2251
2252 int_miti = u16_encode_bits(pkt_num_count,
2253 RTASE_MITI_PKT_NUM_COUNT_MASK) |
2254 u16_encode_bits(pkt_num_unit,
2255 RTASE_MITI_PKT_NUM_UNIT_MASK);
2256
2257 return int_miti;
2258 }
2259
rtase_init_software_variable(struct pci_dev * pdev,struct rtase_private * tp)2260 static void rtase_init_software_variable(struct pci_dev *pdev,
2261 struct rtase_private *tp)
2262 {
2263 u16 int_miti;
2264
2265 tp->tx_queue_ctrl = RTASE_TXQ_CTRL;
2266 tp->func_tx_queue_num = RTASE_FUNC_TXQ_NUM;
2267 tp->func_rx_queue_num = RTASE_FUNC_RXQ_NUM;
2268 tp->int_nums = RTASE_INTERRUPT_NUM;
2269
2270 int_miti = rtase_calc_time_mitigation(RTASE_MITI_DEFAULT_TIME) |
2271 rtase_calc_packet_num_mitigation(RTASE_MITI_DEFAULT_PKT_NUM);
2272 tp->tx_int_mit = int_miti;
2273 tp->rx_int_mit = int_miti;
2274
2275 tp->sw_flag = 0;
2276
2277 rtase_init_int_vector(tp);
2278
2279 /* MTU range: 60 - hw-specific max */
2280 tp->dev->min_mtu = ETH_ZLEN;
2281 tp->dev->max_mtu = RTASE_MAX_JUMBO_SIZE;
2282 }
2283
rtase_check_mac_version_valid(struct rtase_private * tp)2284 static int rtase_check_mac_version_valid(struct rtase_private *tp)
2285 {
2286 int ret = -ENODEV;
2287
2288 tp->hw_ver = rtase_r32(tp, RTASE_TX_CONFIG_0) & RTASE_HW_VER_MASK;
2289
2290 switch (tp->hw_ver) {
2291 case RTASE_HW_VER_906X_7XA:
2292 case RTASE_HW_VER_906X_7XC:
2293 case RTASE_HW_VER_907XD_V1:
2294 case RTASE_HW_VER_907XD_VA:
2295 ret = 0;
2296 break;
2297 }
2298
2299 return ret;
2300 }
2301
rtase_init_board(struct pci_dev * pdev,struct net_device ** dev_out,void __iomem ** ioaddr_out)2302 static int rtase_init_board(struct pci_dev *pdev, struct net_device **dev_out,
2303 void __iomem **ioaddr_out)
2304 {
2305 struct net_device *dev;
2306 void __iomem *ioaddr;
2307 int ret = -ENOMEM;
2308
2309 /* dev zeroed in alloc_etherdev */
2310 dev = alloc_etherdev_mq(sizeof(struct rtase_private),
2311 RTASE_FUNC_TXQ_NUM);
2312 if (!dev)
2313 goto err_out;
2314
2315 SET_NETDEV_DEV(dev, &pdev->dev);
2316
2317 ret = pci_enable_device(pdev);
2318 if (ret)
2319 goto err_out_free_dev;
2320
2321 /* make sure PCI base addr 1 is MMIO */
2322 if (!(pci_resource_flags(pdev, 2) & IORESOURCE_MEM)) {
2323 ret = -ENODEV;
2324 goto err_out_disable;
2325 }
2326
2327 /* check for weird/broken PCI region reporting */
2328 if (pci_resource_len(pdev, 2) < RTASE_REGS_SIZE) {
2329 ret = -ENODEV;
2330 goto err_out_disable;
2331 }
2332
2333 ret = pci_request_regions(pdev, KBUILD_MODNAME);
2334 if (ret)
2335 goto err_out_disable;
2336
2337 ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
2338 if (ret) {
2339 dev_err(&pdev->dev, "no usable dma addressing method\n");
2340 goto err_out_free_res;
2341 }
2342
2343 pci_set_master(pdev);
2344
2345 /* ioremap MMIO region */
2346 ioaddr = ioremap(pci_resource_start(pdev, 2),
2347 pci_resource_len(pdev, 2));
2348 if (!ioaddr) {
2349 ret = -EIO;
2350 goto err_out_free_res;
2351 }
2352
2353 *ioaddr_out = ioaddr;
2354 *dev_out = dev;
2355
2356 return ret;
2357
2358 err_out_free_res:
2359 pci_release_regions(pdev);
2360
2361 err_out_disable:
2362 pci_disable_device(pdev);
2363
2364 err_out_free_dev:
2365 free_netdev(dev);
2366
2367 err_out:
2368 *ioaddr_out = NULL;
2369 *dev_out = NULL;
2370
2371 return ret;
2372 }
2373
rtase_release_board(struct pci_dev * pdev,struct net_device * dev,void __iomem * ioaddr)2374 static void rtase_release_board(struct pci_dev *pdev, struct net_device *dev,
2375 void __iomem *ioaddr)
2376 {
2377 const struct rtase_private *tp = netdev_priv(dev);
2378
2379 rtase_rar_set(tp, tp->dev->perm_addr);
2380 iounmap(ioaddr);
2381
2382 if (tp->sw_flag & RTASE_SWF_MSIX_ENABLED)
2383 pci_disable_msix(pdev);
2384 else
2385 pci_disable_msi(pdev);
2386
2387 pci_release_regions(pdev);
2388 pci_disable_device(pdev);
2389 free_netdev(dev);
2390 }
2391
rtase_init_one(struct pci_dev * pdev,const struct pci_device_id * ent)2392 static int rtase_init_one(struct pci_dev *pdev,
2393 const struct pci_device_id *ent)
2394 {
2395 struct net_device *dev = NULL;
2396 struct rtase_int_vector *ivec;
2397 void __iomem *ioaddr = NULL;
2398 struct rtase_private *tp;
2399 int ret, i;
2400
2401 if (!pdev->is_physfn && pdev->is_virtfn) {
2402 dev_err(&pdev->dev,
2403 "This module does not support a virtual function.");
2404 return -EINVAL;
2405 }
2406
2407 dev_dbg(&pdev->dev, "Automotive Switch Ethernet driver loaded\n");
2408
2409 ret = rtase_init_board(pdev, &dev, &ioaddr);
2410 if (ret)
2411 return ret;
2412
2413 tp = netdev_priv(dev);
2414 tp->mmio_addr = ioaddr;
2415 tp->dev = dev;
2416 tp->pdev = pdev;
2417
2418 /* identify chip attached to board */
2419 ret = rtase_check_mac_version_valid(tp);
2420 if (ret) {
2421 dev_err(&pdev->dev,
2422 "unknown chip version: 0x%08x, contact rtase maintainers (see MAINTAINERS file)\n",
2423 tp->hw_ver);
2424 goto err_out_release_board;
2425 }
2426
2427 rtase_init_software_variable(pdev, tp);
2428 rtase_init_hardware(tp);
2429
2430 ret = rtase_alloc_interrupt(pdev, tp);
2431 if (ret) {
2432 dev_err(&pdev->dev, "unable to alloc MSIX/MSI\n");
2433 goto err_out_del_napi;
2434 }
2435
2436 rtase_init_napi(tp);
2437
2438 rtase_init_netdev_ops(dev);
2439
2440 dev->pcpu_stat_type = NETDEV_PCPU_STAT_TSTATS;
2441
2442 dev->features |= NETIF_F_HW_VLAN_CTAG_TX | NETIF_F_HW_VLAN_CTAG_RX |
2443 NETIF_F_IP_CSUM | NETIF_F_HIGHDMA |
2444 NETIF_F_RXCSUM | NETIF_F_SG |
2445 NETIF_F_TSO | NETIF_F_IPV6_CSUM |
2446 NETIF_F_TSO6;
2447
2448 dev->hw_features = NETIF_F_SG | NETIF_F_IP_CSUM |
2449 NETIF_F_TSO | NETIF_F_RXCSUM |
2450 NETIF_F_HW_VLAN_CTAG_TX | NETIF_F_HW_VLAN_CTAG_RX |
2451 NETIF_F_RXALL | NETIF_F_RXFCS |
2452 NETIF_F_IPV6_CSUM | NETIF_F_TSO6;
2453
2454 dev->vlan_features = NETIF_F_SG | NETIF_F_IP_CSUM | NETIF_F_TSO |
2455 NETIF_F_HIGHDMA;
2456 dev->priv_flags |= IFF_LIVE_ADDR_CHANGE;
2457 netif_set_tso_max_size(dev, RTASE_LSO_64K);
2458 netif_set_tso_max_segs(dev, RTASE_NIC_MAX_PHYS_BUF_COUNT_LSO2);
2459
2460 rtase_get_mac_address(dev);
2461
2462 tp->tally_vaddr = dma_alloc_coherent(&pdev->dev,
2463 sizeof(*tp->tally_vaddr),
2464 &tp->tally_paddr,
2465 GFP_KERNEL);
2466 if (!tp->tally_vaddr) {
2467 ret = -ENOMEM;
2468 goto err_out_free_dma;
2469 }
2470
2471 rtase_tally_counter_clear(tp);
2472
2473 pci_set_drvdata(pdev, dev);
2474
2475 netif_carrier_off(dev);
2476
2477 ret = register_netdev(dev);
2478 if (ret)
2479 goto err_out_free_dma;
2480
2481 netdev_dbg(dev, "%pM, IRQ %d\n", dev->dev_addr, dev->irq);
2482
2483 return 0;
2484
2485 err_out_free_dma:
2486 if (tp->tally_vaddr) {
2487 dma_free_coherent(&pdev->dev,
2488 sizeof(*tp->tally_vaddr),
2489 tp->tally_vaddr,
2490 tp->tally_paddr);
2491
2492 tp->tally_vaddr = NULL;
2493 }
2494
2495 err_out_del_napi:
2496 for (i = 0; i < tp->int_nums; i++) {
2497 ivec = &tp->int_vector[i];
2498 netif_napi_del(&ivec->napi);
2499 }
2500
2501 err_out_release_board:
2502 rtase_release_board(pdev, dev, ioaddr);
2503
2504 return ret;
2505 }
2506
rtase_remove_one(struct pci_dev * pdev)2507 static void rtase_remove_one(struct pci_dev *pdev)
2508 {
2509 struct net_device *dev = pci_get_drvdata(pdev);
2510 struct rtase_private *tp = netdev_priv(dev);
2511 struct rtase_int_vector *ivec;
2512 u32 i;
2513
2514 unregister_netdev(dev);
2515
2516 for (i = 0; i < tp->int_nums; i++) {
2517 ivec = &tp->int_vector[i];
2518 netif_napi_del(&ivec->napi);
2519 }
2520
2521 rtase_reset_interrupt(pdev, tp);
2522 if (tp->tally_vaddr) {
2523 dma_free_coherent(&pdev->dev,
2524 sizeof(*tp->tally_vaddr),
2525 tp->tally_vaddr,
2526 tp->tally_paddr);
2527 tp->tally_vaddr = NULL;
2528 }
2529
2530 rtase_release_board(pdev, dev, tp->mmio_addr);
2531 pci_set_drvdata(pdev, NULL);
2532 }
2533
rtase_shutdown(struct pci_dev * pdev)2534 static void rtase_shutdown(struct pci_dev *pdev)
2535 {
2536 struct net_device *dev = pci_get_drvdata(pdev);
2537 const struct rtase_private *tp;
2538
2539 tp = netdev_priv(dev);
2540
2541 if (netif_running(dev))
2542 rtase_close(dev);
2543
2544 rtase_reset_interrupt(pdev, tp);
2545 }
2546
rtase_suspend(struct device * device)2547 static int rtase_suspend(struct device *device)
2548 {
2549 struct net_device *dev = dev_get_drvdata(device);
2550
2551 if (netif_running(dev)) {
2552 netif_device_detach(dev);
2553 rtase_hw_reset(dev);
2554 }
2555
2556 return 0;
2557 }
2558
rtase_resume(struct device * device)2559 static int rtase_resume(struct device *device)
2560 {
2561 struct net_device *dev = dev_get_drvdata(device);
2562 struct rtase_private *tp = netdev_priv(dev);
2563 int ret;
2564
2565 /* restore last modified mac address */
2566 rtase_rar_set(tp, dev->dev_addr);
2567
2568 if (!netif_running(dev))
2569 goto out;
2570
2571 rtase_wait_for_quiescence(dev);
2572
2573 rtase_tx_clear(tp);
2574 rtase_rx_clear(tp);
2575
2576 ret = rtase_init_ring(dev);
2577 if (ret) {
2578 netdev_err(dev, "unable to init ring\n");
2579 rtase_free_desc(tp);
2580 return -ENOMEM;
2581 }
2582
2583 rtase_hw_config(dev);
2584 /* always link, so start to transmit & receive */
2585 rtase_hw_start(dev);
2586
2587 netif_device_attach(dev);
2588 out:
2589
2590 return 0;
2591 }
2592
2593 static const struct dev_pm_ops rtase_pm_ops = {
2594 SYSTEM_SLEEP_PM_OPS(rtase_suspend, rtase_resume)
2595 };
2596
2597 static struct pci_driver rtase_pci_driver = {
2598 .name = KBUILD_MODNAME,
2599 .id_table = rtase_pci_tbl,
2600 .probe = rtase_init_one,
2601 .remove = rtase_remove_one,
2602 .shutdown = rtase_shutdown,
2603 .driver.pm = pm_ptr(&rtase_pm_ops),
2604 };
2605
2606 module_pci_driver(rtase_pci_driver);
2607