1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Cadence MACB/GEM Ethernet Controller driver
4 *
5 * Copyright (C) 2004-2006 Atmel Corporation
6 */
7
8 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
9 #include <linux/clk.h>
10 #include <linux/clk-provider.h>
11 #include <linux/crc32.h>
12 #include <linux/module.h>
13 #include <linux/moduleparam.h>
14 #include <linux/kernel.h>
15 #include <linux/types.h>
16 #include <linux/circ_buf.h>
17 #include <linux/slab.h>
18 #include <linux/init.h>
19 #include <linux/io.h>
20 #include <linux/gpio.h>
21 #include <linux/gpio/consumer.h>
22 #include <linux/interrupt.h>
23 #include <linux/netdevice.h>
24 #include <linux/etherdevice.h>
25 #include <linux/dma-mapping.h>
26 #include <linux/platform_device.h>
27 #include <linux/phylink.h>
28 #include <linux/of.h>
29 #include <linux/of_gpio.h>
30 #include <linux/of_mdio.h>
31 #include <linux/of_net.h>
32 #include <linux/ip.h>
33 #include <linux/udp.h>
34 #include <linux/tcp.h>
35 #include <linux/iopoll.h>
36 #include <linux/phy/phy.h>
37 #include <linux/pm_runtime.h>
38 #include <linux/ptp_classify.h>
39 #include <linux/reset.h>
40 #include <linux/firmware/xlnx-zynqmp.h>
41 #include <linux/inetdevice.h>
42 #include "macb.h"
43
44 /* This structure is only used for MACB on SiFive FU540 devices */
45 struct sifive_fu540_macb_mgmt {
46 void __iomem *reg;
47 unsigned long rate;
48 struct clk_hw hw;
49 };
50
51 #define MACB_RX_BUFFER_SIZE 128
52 #define RX_BUFFER_MULTIPLE 64 /* bytes */
53
54 #define DEFAULT_RX_RING_SIZE 512 /* must be power of 2 */
55 #define MIN_RX_RING_SIZE 64
56 #define MAX_RX_RING_SIZE 8192
57 #define RX_RING_BYTES(bp) (macb_dma_desc_get_size(bp) \
58 * (bp)->rx_ring_size)
59
60 #define DEFAULT_TX_RING_SIZE 512 /* must be power of 2 */
61 #define MIN_TX_RING_SIZE 64
62 #define MAX_TX_RING_SIZE 4096
63 #define TX_RING_BYTES(bp) (macb_dma_desc_get_size(bp) \
64 * (bp)->tx_ring_size)
65
66 /* level of occupied TX descriptors under which we wake up TX process */
67 #define MACB_TX_WAKEUP_THRESH(bp) (3 * (bp)->tx_ring_size / 4)
68
69 #define MACB_RX_INT_FLAGS (MACB_BIT(RCOMP) | MACB_BIT(ISR_ROVR))
70 #define MACB_TX_ERR_FLAGS (MACB_BIT(ISR_TUND) \
71 | MACB_BIT(ISR_RLE) \
72 | MACB_BIT(TXERR))
73 #define MACB_TX_INT_FLAGS (MACB_TX_ERR_FLAGS | MACB_BIT(TCOMP) \
74 | MACB_BIT(TXUBR))
75
76 /* Max length of transmit frame must be a multiple of 8 bytes */
77 #define MACB_TX_LEN_ALIGN 8
78 #define MACB_MAX_TX_LEN ((unsigned int)((1 << MACB_TX_FRMLEN_SIZE) - 1) & ~((unsigned int)(MACB_TX_LEN_ALIGN - 1)))
79 /* Limit maximum TX length as per Cadence TSO errata. This is to avoid a
80 * false amba_error in TX path from the DMA assuming there is not enough
81 * space in the SRAM (16KB) even when there is.
82 */
83 #define GEM_MAX_TX_LEN (unsigned int)(0x3FC0)
84
85 #define GEM_MTU_MIN_SIZE ETH_MIN_MTU
86 #define MACB_NETIF_LSO NETIF_F_TSO
87
88 #define MACB_WOL_ENABLED BIT(0)
89
90 #define HS_SPEED_10000M 4
91 #define MACB_SERDES_RATE_10G 1
92
93 /* Graceful stop timeouts in us. We should allow up to
94 * 1 frame time (10 Mbits/s, full-duplex, ignoring collisions)
95 */
96 #define MACB_HALT_TIMEOUT 14000
97 #define MACB_PM_TIMEOUT 100 /* ms */
98
99 #define MACB_MDIO_TIMEOUT 1000000 /* in usecs */
100
101 /* DMA buffer descriptor might be different size
102 * depends on hardware configuration:
103 *
104 * 1. dma address width 32 bits:
105 * word 1: 32 bit address of Data Buffer
106 * word 2: control
107 *
108 * 2. dma address width 64 bits:
109 * word 1: 32 bit address of Data Buffer
110 * word 2: control
111 * word 3: upper 32 bit address of Data Buffer
112 * word 4: unused
113 *
114 * 3. dma address width 32 bits with hardware timestamping:
115 * word 1: 32 bit address of Data Buffer
116 * word 2: control
117 * word 3: timestamp word 1
118 * word 4: timestamp word 2
119 *
120 * 4. dma address width 64 bits with hardware timestamping:
121 * word 1: 32 bit address of Data Buffer
122 * word 2: control
123 * word 3: upper 32 bit address of Data Buffer
124 * word 4: unused
125 * word 5: timestamp word 1
126 * word 6: timestamp word 2
127 */
macb_dma_desc_get_size(struct macb * bp)128 static unsigned int macb_dma_desc_get_size(struct macb *bp)
129 {
130 #ifdef MACB_EXT_DESC
131 unsigned int desc_size;
132
133 switch (bp->hw_dma_cap) {
134 case HW_DMA_CAP_64B:
135 desc_size = sizeof(struct macb_dma_desc)
136 + sizeof(struct macb_dma_desc_64);
137 break;
138 case HW_DMA_CAP_PTP:
139 desc_size = sizeof(struct macb_dma_desc)
140 + sizeof(struct macb_dma_desc_ptp);
141 break;
142 case HW_DMA_CAP_64B_PTP:
143 desc_size = sizeof(struct macb_dma_desc)
144 + sizeof(struct macb_dma_desc_64)
145 + sizeof(struct macb_dma_desc_ptp);
146 break;
147 default:
148 desc_size = sizeof(struct macb_dma_desc);
149 }
150 return desc_size;
151 #endif
152 return sizeof(struct macb_dma_desc);
153 }
154
macb_adj_dma_desc_idx(struct macb * bp,unsigned int desc_idx)155 static unsigned int macb_adj_dma_desc_idx(struct macb *bp, unsigned int desc_idx)
156 {
157 #ifdef MACB_EXT_DESC
158 switch (bp->hw_dma_cap) {
159 case HW_DMA_CAP_64B:
160 case HW_DMA_CAP_PTP:
161 desc_idx <<= 1;
162 break;
163 case HW_DMA_CAP_64B_PTP:
164 desc_idx *= 3;
165 break;
166 default:
167 break;
168 }
169 #endif
170 return desc_idx;
171 }
172
173 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT
macb_64b_desc(struct macb * bp,struct macb_dma_desc * desc)174 static struct macb_dma_desc_64 *macb_64b_desc(struct macb *bp, struct macb_dma_desc *desc)
175 {
176 return (struct macb_dma_desc_64 *)((void *)desc
177 + sizeof(struct macb_dma_desc));
178 }
179 #endif
180
181 /* Ring buffer accessors */
macb_tx_ring_wrap(struct macb * bp,unsigned int index)182 static unsigned int macb_tx_ring_wrap(struct macb *bp, unsigned int index)
183 {
184 return index & (bp->tx_ring_size - 1);
185 }
186
macb_tx_desc(struct macb_queue * queue,unsigned int index)187 static struct macb_dma_desc *macb_tx_desc(struct macb_queue *queue,
188 unsigned int index)
189 {
190 index = macb_tx_ring_wrap(queue->bp, index);
191 index = macb_adj_dma_desc_idx(queue->bp, index);
192 return &queue->tx_ring[index];
193 }
194
macb_tx_skb(struct macb_queue * queue,unsigned int index)195 static struct macb_tx_skb *macb_tx_skb(struct macb_queue *queue,
196 unsigned int index)
197 {
198 return &queue->tx_skb[macb_tx_ring_wrap(queue->bp, index)];
199 }
200
macb_tx_dma(struct macb_queue * queue,unsigned int index)201 static dma_addr_t macb_tx_dma(struct macb_queue *queue, unsigned int index)
202 {
203 dma_addr_t offset;
204
205 offset = macb_tx_ring_wrap(queue->bp, index) *
206 macb_dma_desc_get_size(queue->bp);
207
208 return queue->tx_ring_dma + offset;
209 }
210
macb_rx_ring_wrap(struct macb * bp,unsigned int index)211 static unsigned int macb_rx_ring_wrap(struct macb *bp, unsigned int index)
212 {
213 return index & (bp->rx_ring_size - 1);
214 }
215
macb_rx_desc(struct macb_queue * queue,unsigned int index)216 static struct macb_dma_desc *macb_rx_desc(struct macb_queue *queue, unsigned int index)
217 {
218 index = macb_rx_ring_wrap(queue->bp, index);
219 index = macb_adj_dma_desc_idx(queue->bp, index);
220 return &queue->rx_ring[index];
221 }
222
macb_rx_buffer(struct macb_queue * queue,unsigned int index)223 static void *macb_rx_buffer(struct macb_queue *queue, unsigned int index)
224 {
225 return queue->rx_buffers + queue->bp->rx_buffer_size *
226 macb_rx_ring_wrap(queue->bp, index);
227 }
228
229 /* I/O accessors */
hw_readl_native(struct macb * bp,int offset)230 static u32 hw_readl_native(struct macb *bp, int offset)
231 {
232 return __raw_readl(bp->regs + offset);
233 }
234
hw_writel_native(struct macb * bp,int offset,u32 value)235 static void hw_writel_native(struct macb *bp, int offset, u32 value)
236 {
237 __raw_writel(value, bp->regs + offset);
238 }
239
hw_readl(struct macb * bp,int offset)240 static u32 hw_readl(struct macb *bp, int offset)
241 {
242 return readl_relaxed(bp->regs + offset);
243 }
244
hw_writel(struct macb * bp,int offset,u32 value)245 static void hw_writel(struct macb *bp, int offset, u32 value)
246 {
247 writel_relaxed(value, bp->regs + offset);
248 }
249
250 /* Find the CPU endianness by using the loopback bit of NCR register. When the
251 * CPU is in big endian we need to program swapped mode for management
252 * descriptor access.
253 */
hw_is_native_io(void __iomem * addr)254 static bool hw_is_native_io(void __iomem *addr)
255 {
256 u32 value = MACB_BIT(LLB);
257
258 __raw_writel(value, addr + MACB_NCR);
259 value = __raw_readl(addr + MACB_NCR);
260
261 /* Write 0 back to disable everything */
262 __raw_writel(0, addr + MACB_NCR);
263
264 return value == MACB_BIT(LLB);
265 }
266
hw_is_gem(void __iomem * addr,bool native_io)267 static bool hw_is_gem(void __iomem *addr, bool native_io)
268 {
269 u32 id;
270
271 if (native_io)
272 id = __raw_readl(addr + MACB_MID);
273 else
274 id = readl_relaxed(addr + MACB_MID);
275
276 return MACB_BFEXT(IDNUM, id) >= 0x2;
277 }
278
macb_set_hwaddr(struct macb * bp)279 static void macb_set_hwaddr(struct macb *bp)
280 {
281 u32 bottom;
282 u16 top;
283
284 bottom = cpu_to_le32(*((u32 *)bp->dev->dev_addr));
285 macb_or_gem_writel(bp, SA1B, bottom);
286 top = cpu_to_le16(*((u16 *)(bp->dev->dev_addr + 4)));
287 macb_or_gem_writel(bp, SA1T, top);
288
289 if (gem_has_ptp(bp)) {
290 gem_writel(bp, RXPTPUNI, bottom);
291 gem_writel(bp, TXPTPUNI, bottom);
292 }
293
294 /* Clear unused address register sets */
295 macb_or_gem_writel(bp, SA2B, 0);
296 macb_or_gem_writel(bp, SA2T, 0);
297 macb_or_gem_writel(bp, SA3B, 0);
298 macb_or_gem_writel(bp, SA3T, 0);
299 macb_or_gem_writel(bp, SA4B, 0);
300 macb_or_gem_writel(bp, SA4T, 0);
301 }
302
macb_get_hwaddr(struct macb * bp)303 static void macb_get_hwaddr(struct macb *bp)
304 {
305 u32 bottom;
306 u16 top;
307 u8 addr[6];
308 int i;
309
310 /* Check all 4 address register for valid address */
311 for (i = 0; i < 4; i++) {
312 bottom = macb_or_gem_readl(bp, SA1B + i * 8);
313 top = macb_or_gem_readl(bp, SA1T + i * 8);
314
315 addr[0] = bottom & 0xff;
316 addr[1] = (bottom >> 8) & 0xff;
317 addr[2] = (bottom >> 16) & 0xff;
318 addr[3] = (bottom >> 24) & 0xff;
319 addr[4] = top & 0xff;
320 addr[5] = (top >> 8) & 0xff;
321
322 if (is_valid_ether_addr(addr)) {
323 eth_hw_addr_set(bp->dev, addr);
324 return;
325 }
326 }
327
328 dev_info(&bp->pdev->dev, "invalid hw address, using random\n");
329 eth_hw_addr_random(bp->dev);
330 }
331
macb_mdio_wait_for_idle(struct macb * bp)332 static int macb_mdio_wait_for_idle(struct macb *bp)
333 {
334 u32 val;
335
336 return readx_poll_timeout(MACB_READ_NSR, bp, val, val & MACB_BIT(IDLE),
337 1, MACB_MDIO_TIMEOUT);
338 }
339
macb_mdio_read_c22(struct mii_bus * bus,int mii_id,int regnum)340 static int macb_mdio_read_c22(struct mii_bus *bus, int mii_id, int regnum)
341 {
342 struct macb *bp = bus->priv;
343 int status;
344
345 status = pm_runtime_resume_and_get(&bp->pdev->dev);
346 if (status < 0)
347 goto mdio_pm_exit;
348
349 status = macb_mdio_wait_for_idle(bp);
350 if (status < 0)
351 goto mdio_read_exit;
352
353 macb_writel(bp, MAN, (MACB_BF(SOF, MACB_MAN_C22_SOF)
354 | MACB_BF(RW, MACB_MAN_C22_READ)
355 | MACB_BF(PHYA, mii_id)
356 | MACB_BF(REGA, regnum)
357 | MACB_BF(CODE, MACB_MAN_C22_CODE)));
358
359 status = macb_mdio_wait_for_idle(bp);
360 if (status < 0)
361 goto mdio_read_exit;
362
363 status = MACB_BFEXT(DATA, macb_readl(bp, MAN));
364
365 mdio_read_exit:
366 pm_runtime_mark_last_busy(&bp->pdev->dev);
367 pm_runtime_put_autosuspend(&bp->pdev->dev);
368 mdio_pm_exit:
369 return status;
370 }
371
macb_mdio_read_c45(struct mii_bus * bus,int mii_id,int devad,int regnum)372 static int macb_mdio_read_c45(struct mii_bus *bus, int mii_id, int devad,
373 int regnum)
374 {
375 struct macb *bp = bus->priv;
376 int status;
377
378 status = pm_runtime_get_sync(&bp->pdev->dev);
379 if (status < 0) {
380 pm_runtime_put_noidle(&bp->pdev->dev);
381 goto mdio_pm_exit;
382 }
383
384 status = macb_mdio_wait_for_idle(bp);
385 if (status < 0)
386 goto mdio_read_exit;
387
388 macb_writel(bp, MAN, (MACB_BF(SOF, MACB_MAN_C45_SOF)
389 | MACB_BF(RW, MACB_MAN_C45_ADDR)
390 | MACB_BF(PHYA, mii_id)
391 | MACB_BF(REGA, devad & 0x1F)
392 | MACB_BF(DATA, regnum & 0xFFFF)
393 | MACB_BF(CODE, MACB_MAN_C45_CODE)));
394
395 status = macb_mdio_wait_for_idle(bp);
396 if (status < 0)
397 goto mdio_read_exit;
398
399 macb_writel(bp, MAN, (MACB_BF(SOF, MACB_MAN_C45_SOF)
400 | MACB_BF(RW, MACB_MAN_C45_READ)
401 | MACB_BF(PHYA, mii_id)
402 | MACB_BF(REGA, devad & 0x1F)
403 | MACB_BF(CODE, MACB_MAN_C45_CODE)));
404
405 status = macb_mdio_wait_for_idle(bp);
406 if (status < 0)
407 goto mdio_read_exit;
408
409 status = MACB_BFEXT(DATA, macb_readl(bp, MAN));
410
411 mdio_read_exit:
412 pm_runtime_mark_last_busy(&bp->pdev->dev);
413 pm_runtime_put_autosuspend(&bp->pdev->dev);
414 mdio_pm_exit:
415 return status;
416 }
417
macb_mdio_write_c22(struct mii_bus * bus,int mii_id,int regnum,u16 value)418 static int macb_mdio_write_c22(struct mii_bus *bus, int mii_id, int regnum,
419 u16 value)
420 {
421 struct macb *bp = bus->priv;
422 int status;
423
424 status = pm_runtime_resume_and_get(&bp->pdev->dev);
425 if (status < 0)
426 goto mdio_pm_exit;
427
428 status = macb_mdio_wait_for_idle(bp);
429 if (status < 0)
430 goto mdio_write_exit;
431
432 macb_writel(bp, MAN, (MACB_BF(SOF, MACB_MAN_C22_SOF)
433 | MACB_BF(RW, MACB_MAN_C22_WRITE)
434 | MACB_BF(PHYA, mii_id)
435 | MACB_BF(REGA, regnum)
436 | MACB_BF(CODE, MACB_MAN_C22_CODE)
437 | MACB_BF(DATA, value)));
438
439 status = macb_mdio_wait_for_idle(bp);
440 if (status < 0)
441 goto mdio_write_exit;
442
443 mdio_write_exit:
444 pm_runtime_mark_last_busy(&bp->pdev->dev);
445 pm_runtime_put_autosuspend(&bp->pdev->dev);
446 mdio_pm_exit:
447 return status;
448 }
449
macb_mdio_write_c45(struct mii_bus * bus,int mii_id,int devad,int regnum,u16 value)450 static int macb_mdio_write_c45(struct mii_bus *bus, int mii_id,
451 int devad, int regnum,
452 u16 value)
453 {
454 struct macb *bp = bus->priv;
455 int status;
456
457 status = pm_runtime_get_sync(&bp->pdev->dev);
458 if (status < 0) {
459 pm_runtime_put_noidle(&bp->pdev->dev);
460 goto mdio_pm_exit;
461 }
462
463 status = macb_mdio_wait_for_idle(bp);
464 if (status < 0)
465 goto mdio_write_exit;
466
467 macb_writel(bp, MAN, (MACB_BF(SOF, MACB_MAN_C45_SOF)
468 | MACB_BF(RW, MACB_MAN_C45_ADDR)
469 | MACB_BF(PHYA, mii_id)
470 | MACB_BF(REGA, devad & 0x1F)
471 | MACB_BF(DATA, regnum & 0xFFFF)
472 | MACB_BF(CODE, MACB_MAN_C45_CODE)));
473
474 status = macb_mdio_wait_for_idle(bp);
475 if (status < 0)
476 goto mdio_write_exit;
477
478 macb_writel(bp, MAN, (MACB_BF(SOF, MACB_MAN_C45_SOF)
479 | MACB_BF(RW, MACB_MAN_C45_WRITE)
480 | MACB_BF(PHYA, mii_id)
481 | MACB_BF(REGA, devad & 0x1F)
482 | MACB_BF(CODE, MACB_MAN_C45_CODE)
483 | MACB_BF(DATA, value)));
484
485 status = macb_mdio_wait_for_idle(bp);
486 if (status < 0)
487 goto mdio_write_exit;
488
489 mdio_write_exit:
490 pm_runtime_mark_last_busy(&bp->pdev->dev);
491 pm_runtime_put_autosuspend(&bp->pdev->dev);
492 mdio_pm_exit:
493 return status;
494 }
495
macb_init_buffers(struct macb * bp)496 static void macb_init_buffers(struct macb *bp)
497 {
498 struct macb_queue *queue;
499 unsigned int q;
500
501 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) {
502 queue_writel(queue, RBQP, lower_32_bits(queue->rx_ring_dma));
503 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT
504 if (bp->hw_dma_cap & HW_DMA_CAP_64B)
505 queue_writel(queue, RBQPH,
506 upper_32_bits(queue->rx_ring_dma));
507 #endif
508 queue_writel(queue, TBQP, lower_32_bits(queue->tx_ring_dma));
509 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT
510 if (bp->hw_dma_cap & HW_DMA_CAP_64B)
511 queue_writel(queue, TBQPH,
512 upper_32_bits(queue->tx_ring_dma));
513 #endif
514 }
515 }
516
517 /**
518 * macb_set_tx_clk() - Set a clock to a new frequency
519 * @bp: pointer to struct macb
520 * @speed: New frequency in Hz
521 */
macb_set_tx_clk(struct macb * bp,int speed)522 static void macb_set_tx_clk(struct macb *bp, int speed)
523 {
524 long ferr, rate, rate_rounded;
525
526 if (!bp->tx_clk || (bp->caps & MACB_CAPS_CLK_HW_CHG))
527 return;
528
529 /* In case of MII the PHY is the clock master */
530 if (bp->phy_interface == PHY_INTERFACE_MODE_MII)
531 return;
532
533 switch (speed) {
534 case SPEED_10:
535 rate = 2500000;
536 break;
537 case SPEED_100:
538 rate = 25000000;
539 break;
540 case SPEED_1000:
541 rate = 125000000;
542 break;
543 default:
544 return;
545 }
546
547 rate_rounded = clk_round_rate(bp->tx_clk, rate);
548 if (rate_rounded < 0)
549 return;
550
551 /* RGMII allows 50 ppm frequency error. Test and warn if this limit
552 * is not satisfied.
553 */
554 ferr = abs(rate_rounded - rate);
555 ferr = DIV_ROUND_UP(ferr, rate / 100000);
556 if (ferr > 5)
557 netdev_warn(bp->dev,
558 "unable to generate target frequency: %ld Hz\n",
559 rate);
560
561 if (clk_set_rate(bp->tx_clk, rate_rounded))
562 netdev_err(bp->dev, "adjusting tx_clk failed.\n");
563 }
564
macb_usx_pcs_link_up(struct phylink_pcs * pcs,unsigned int neg_mode,phy_interface_t interface,int speed,int duplex)565 static void macb_usx_pcs_link_up(struct phylink_pcs *pcs, unsigned int neg_mode,
566 phy_interface_t interface, int speed,
567 int duplex)
568 {
569 struct macb *bp = container_of(pcs, struct macb, phylink_usx_pcs);
570 u32 config;
571
572 config = gem_readl(bp, USX_CONTROL);
573 config = GEM_BFINS(SERDES_RATE, MACB_SERDES_RATE_10G, config);
574 config = GEM_BFINS(USX_CTRL_SPEED, HS_SPEED_10000M, config);
575 config &= ~(GEM_BIT(TX_SCR_BYPASS) | GEM_BIT(RX_SCR_BYPASS));
576 config |= GEM_BIT(TX_EN);
577 gem_writel(bp, USX_CONTROL, config);
578 }
579
macb_usx_pcs_get_state(struct phylink_pcs * pcs,struct phylink_link_state * state)580 static void macb_usx_pcs_get_state(struct phylink_pcs *pcs,
581 struct phylink_link_state *state)
582 {
583 struct macb *bp = container_of(pcs, struct macb, phylink_usx_pcs);
584 u32 val;
585
586 state->speed = SPEED_10000;
587 state->duplex = 1;
588 state->an_complete = 1;
589
590 val = gem_readl(bp, USX_STATUS);
591 state->link = !!(val & GEM_BIT(USX_BLOCK_LOCK));
592 val = gem_readl(bp, NCFGR);
593 if (val & GEM_BIT(PAE))
594 state->pause = MLO_PAUSE_RX;
595 }
596
macb_usx_pcs_config(struct phylink_pcs * pcs,unsigned int neg_mode,phy_interface_t interface,const unsigned long * advertising,bool permit_pause_to_mac)597 static int macb_usx_pcs_config(struct phylink_pcs *pcs,
598 unsigned int neg_mode,
599 phy_interface_t interface,
600 const unsigned long *advertising,
601 bool permit_pause_to_mac)
602 {
603 struct macb *bp = container_of(pcs, struct macb, phylink_usx_pcs);
604
605 gem_writel(bp, USX_CONTROL, gem_readl(bp, USX_CONTROL) |
606 GEM_BIT(SIGNAL_OK));
607
608 return 0;
609 }
610
macb_pcs_get_state(struct phylink_pcs * pcs,struct phylink_link_state * state)611 static void macb_pcs_get_state(struct phylink_pcs *pcs,
612 struct phylink_link_state *state)
613 {
614 state->link = 0;
615 }
616
macb_pcs_an_restart(struct phylink_pcs * pcs)617 static void macb_pcs_an_restart(struct phylink_pcs *pcs)
618 {
619 /* Not supported */
620 }
621
macb_pcs_config(struct phylink_pcs * pcs,unsigned int neg_mode,phy_interface_t interface,const unsigned long * advertising,bool permit_pause_to_mac)622 static int macb_pcs_config(struct phylink_pcs *pcs,
623 unsigned int neg_mode,
624 phy_interface_t interface,
625 const unsigned long *advertising,
626 bool permit_pause_to_mac)
627 {
628 return 0;
629 }
630
631 static const struct phylink_pcs_ops macb_phylink_usx_pcs_ops = {
632 .pcs_get_state = macb_usx_pcs_get_state,
633 .pcs_config = macb_usx_pcs_config,
634 .pcs_link_up = macb_usx_pcs_link_up,
635 };
636
637 static const struct phylink_pcs_ops macb_phylink_pcs_ops = {
638 .pcs_get_state = macb_pcs_get_state,
639 .pcs_an_restart = macb_pcs_an_restart,
640 .pcs_config = macb_pcs_config,
641 };
642
macb_mac_config(struct phylink_config * config,unsigned int mode,const struct phylink_link_state * state)643 static void macb_mac_config(struct phylink_config *config, unsigned int mode,
644 const struct phylink_link_state *state)
645 {
646 struct net_device *ndev = to_net_dev(config->dev);
647 struct macb *bp = netdev_priv(ndev);
648 unsigned long flags;
649 u32 old_ctrl, ctrl;
650 u32 old_ncr, ncr;
651
652 spin_lock_irqsave(&bp->lock, flags);
653
654 old_ctrl = ctrl = macb_or_gem_readl(bp, NCFGR);
655 old_ncr = ncr = macb_or_gem_readl(bp, NCR);
656
657 if (bp->caps & MACB_CAPS_MACB_IS_EMAC) {
658 if (state->interface == PHY_INTERFACE_MODE_RMII)
659 ctrl |= MACB_BIT(RM9200_RMII);
660 } else if (macb_is_gem(bp)) {
661 ctrl &= ~(GEM_BIT(SGMIIEN) | GEM_BIT(PCSSEL));
662 ncr &= ~GEM_BIT(ENABLE_HS_MAC);
663
664 if (state->interface == PHY_INTERFACE_MODE_SGMII) {
665 ctrl |= GEM_BIT(SGMIIEN) | GEM_BIT(PCSSEL);
666 } else if (state->interface == PHY_INTERFACE_MODE_10GBASER) {
667 ctrl |= GEM_BIT(PCSSEL);
668 ncr |= GEM_BIT(ENABLE_HS_MAC);
669 } else if (bp->caps & MACB_CAPS_MIIONRGMII &&
670 bp->phy_interface == PHY_INTERFACE_MODE_MII) {
671 ncr |= MACB_BIT(MIIONRGMII);
672 }
673 }
674
675 /* Apply the new configuration, if any */
676 if (old_ctrl ^ ctrl)
677 macb_or_gem_writel(bp, NCFGR, ctrl);
678
679 if (old_ncr ^ ncr)
680 macb_or_gem_writel(bp, NCR, ncr);
681
682 /* Disable AN for SGMII fixed link configuration, enable otherwise.
683 * Must be written after PCSSEL is set in NCFGR,
684 * otherwise writes will not take effect.
685 */
686 if (macb_is_gem(bp) && state->interface == PHY_INTERFACE_MODE_SGMII) {
687 u32 pcsctrl, old_pcsctrl;
688
689 old_pcsctrl = gem_readl(bp, PCSCNTRL);
690 if (mode == MLO_AN_FIXED)
691 pcsctrl = old_pcsctrl & ~GEM_BIT(PCSAUTONEG);
692 else
693 pcsctrl = old_pcsctrl | GEM_BIT(PCSAUTONEG);
694 if (old_pcsctrl != pcsctrl)
695 gem_writel(bp, PCSCNTRL, pcsctrl);
696 }
697
698 spin_unlock_irqrestore(&bp->lock, flags);
699 }
700
macb_mac_link_down(struct phylink_config * config,unsigned int mode,phy_interface_t interface)701 static void macb_mac_link_down(struct phylink_config *config, unsigned int mode,
702 phy_interface_t interface)
703 {
704 struct net_device *ndev = to_net_dev(config->dev);
705 struct macb *bp = netdev_priv(ndev);
706 struct macb_queue *queue;
707 unsigned int q;
708 u32 ctrl;
709
710 if (!(bp->caps & MACB_CAPS_MACB_IS_EMAC))
711 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue)
712 queue_writel(queue, IDR,
713 bp->rx_intr_mask | MACB_TX_INT_FLAGS | MACB_BIT(HRESP));
714
715 /* Disable Rx and Tx */
716 ctrl = macb_readl(bp, NCR) & ~(MACB_BIT(RE) | MACB_BIT(TE));
717 macb_writel(bp, NCR, ctrl);
718
719 netif_tx_stop_all_queues(ndev);
720 }
721
macb_mac_link_up(struct phylink_config * config,struct phy_device * phy,unsigned int mode,phy_interface_t interface,int speed,int duplex,bool tx_pause,bool rx_pause)722 static void macb_mac_link_up(struct phylink_config *config,
723 struct phy_device *phy,
724 unsigned int mode, phy_interface_t interface,
725 int speed, int duplex,
726 bool tx_pause, bool rx_pause)
727 {
728 struct net_device *ndev = to_net_dev(config->dev);
729 struct macb *bp = netdev_priv(ndev);
730 struct macb_queue *queue;
731 unsigned long flags;
732 unsigned int q;
733 u32 ctrl;
734
735 spin_lock_irqsave(&bp->lock, flags);
736
737 ctrl = macb_or_gem_readl(bp, NCFGR);
738
739 ctrl &= ~(MACB_BIT(SPD) | MACB_BIT(FD));
740
741 if (speed == SPEED_100)
742 ctrl |= MACB_BIT(SPD);
743
744 if (duplex)
745 ctrl |= MACB_BIT(FD);
746
747 if (!(bp->caps & MACB_CAPS_MACB_IS_EMAC)) {
748 ctrl &= ~MACB_BIT(PAE);
749 if (macb_is_gem(bp)) {
750 ctrl &= ~GEM_BIT(GBE);
751
752 if (speed == SPEED_1000)
753 ctrl |= GEM_BIT(GBE);
754 }
755
756 if (rx_pause)
757 ctrl |= MACB_BIT(PAE);
758
759 /* Initialize rings & buffers as clearing MACB_BIT(TE) in link down
760 * cleared the pipeline and control registers.
761 */
762 bp->macbgem_ops.mog_init_rings(bp);
763 macb_init_buffers(bp);
764
765 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue)
766 queue_writel(queue, IER,
767 bp->rx_intr_mask | MACB_TX_INT_FLAGS | MACB_BIT(HRESP));
768 }
769
770 macb_or_gem_writel(bp, NCFGR, ctrl);
771
772 if (bp->phy_interface == PHY_INTERFACE_MODE_10GBASER)
773 gem_writel(bp, HS_MAC_CONFIG, GEM_BFINS(HS_MAC_SPEED, HS_SPEED_10000M,
774 gem_readl(bp, HS_MAC_CONFIG)));
775
776 spin_unlock_irqrestore(&bp->lock, flags);
777
778 if (!(bp->caps & MACB_CAPS_MACB_IS_EMAC))
779 macb_set_tx_clk(bp, speed);
780
781 /* Enable Rx and Tx; Enable PTP unicast */
782 ctrl = macb_readl(bp, NCR);
783 if (gem_has_ptp(bp))
784 ctrl |= MACB_BIT(PTPUNI);
785
786 macb_writel(bp, NCR, ctrl | MACB_BIT(RE) | MACB_BIT(TE));
787
788 netif_tx_wake_all_queues(ndev);
789 }
790
macb_mac_select_pcs(struct phylink_config * config,phy_interface_t interface)791 static struct phylink_pcs *macb_mac_select_pcs(struct phylink_config *config,
792 phy_interface_t interface)
793 {
794 struct net_device *ndev = to_net_dev(config->dev);
795 struct macb *bp = netdev_priv(ndev);
796
797 if (interface == PHY_INTERFACE_MODE_10GBASER)
798 return &bp->phylink_usx_pcs;
799 else if (interface == PHY_INTERFACE_MODE_SGMII)
800 return &bp->phylink_sgmii_pcs;
801 else
802 return NULL;
803 }
804
805 static const struct phylink_mac_ops macb_phylink_ops = {
806 .mac_select_pcs = macb_mac_select_pcs,
807 .mac_config = macb_mac_config,
808 .mac_link_down = macb_mac_link_down,
809 .mac_link_up = macb_mac_link_up,
810 };
811
macb_phy_handle_exists(struct device_node * dn)812 static bool macb_phy_handle_exists(struct device_node *dn)
813 {
814 dn = of_parse_phandle(dn, "phy-handle", 0);
815 of_node_put(dn);
816 return dn != NULL;
817 }
818
macb_phylink_connect(struct macb * bp)819 static int macb_phylink_connect(struct macb *bp)
820 {
821 struct device_node *dn = bp->pdev->dev.of_node;
822 struct net_device *dev = bp->dev;
823 struct phy_device *phydev;
824 int ret;
825
826 if (dn)
827 ret = phylink_of_phy_connect(bp->phylink, dn, 0);
828
829 if (!dn || (ret && !macb_phy_handle_exists(dn))) {
830 phydev = phy_find_first(bp->mii_bus);
831 if (!phydev) {
832 netdev_err(dev, "no PHY found\n");
833 return -ENXIO;
834 }
835
836 /* attach the mac to the phy */
837 ret = phylink_connect_phy(bp->phylink, phydev);
838 }
839
840 if (ret) {
841 netdev_err(dev, "Could not attach PHY (%d)\n", ret);
842 return ret;
843 }
844
845 phylink_start(bp->phylink);
846
847 return 0;
848 }
849
macb_get_pcs_fixed_state(struct phylink_config * config,struct phylink_link_state * state)850 static void macb_get_pcs_fixed_state(struct phylink_config *config,
851 struct phylink_link_state *state)
852 {
853 struct net_device *ndev = to_net_dev(config->dev);
854 struct macb *bp = netdev_priv(ndev);
855
856 state->link = (macb_readl(bp, NSR) & MACB_BIT(NSR_LINK)) != 0;
857 }
858
859 /* based on au1000_eth. c*/
macb_mii_probe(struct net_device * dev)860 static int macb_mii_probe(struct net_device *dev)
861 {
862 struct macb *bp = netdev_priv(dev);
863
864 bp->phylink_sgmii_pcs.ops = &macb_phylink_pcs_ops;
865 bp->phylink_sgmii_pcs.neg_mode = true;
866 bp->phylink_usx_pcs.ops = &macb_phylink_usx_pcs_ops;
867 bp->phylink_usx_pcs.neg_mode = true;
868
869 bp->phylink_config.dev = &dev->dev;
870 bp->phylink_config.type = PHYLINK_NETDEV;
871 bp->phylink_config.mac_managed_pm = true;
872
873 if (bp->phy_interface == PHY_INTERFACE_MODE_SGMII) {
874 bp->phylink_config.poll_fixed_state = true;
875 bp->phylink_config.get_fixed_state = macb_get_pcs_fixed_state;
876 }
877
878 bp->phylink_config.mac_capabilities = MAC_ASYM_PAUSE |
879 MAC_10 | MAC_100;
880
881 __set_bit(PHY_INTERFACE_MODE_MII,
882 bp->phylink_config.supported_interfaces);
883 __set_bit(PHY_INTERFACE_MODE_RMII,
884 bp->phylink_config.supported_interfaces);
885
886 /* Determine what modes are supported */
887 if (macb_is_gem(bp) && (bp->caps & MACB_CAPS_GIGABIT_MODE_AVAILABLE)) {
888 bp->phylink_config.mac_capabilities |= MAC_1000FD;
889 if (!(bp->caps & MACB_CAPS_NO_GIGABIT_HALF))
890 bp->phylink_config.mac_capabilities |= MAC_1000HD;
891
892 __set_bit(PHY_INTERFACE_MODE_GMII,
893 bp->phylink_config.supported_interfaces);
894 phy_interface_set_rgmii(bp->phylink_config.supported_interfaces);
895
896 if (bp->caps & MACB_CAPS_PCS)
897 __set_bit(PHY_INTERFACE_MODE_SGMII,
898 bp->phylink_config.supported_interfaces);
899
900 if (bp->caps & MACB_CAPS_HIGH_SPEED) {
901 __set_bit(PHY_INTERFACE_MODE_10GBASER,
902 bp->phylink_config.supported_interfaces);
903 bp->phylink_config.mac_capabilities |= MAC_10000FD;
904 }
905 }
906
907 bp->phylink = phylink_create(&bp->phylink_config, bp->pdev->dev.fwnode,
908 bp->phy_interface, &macb_phylink_ops);
909 if (IS_ERR(bp->phylink)) {
910 netdev_err(dev, "Could not create a phylink instance (%ld)\n",
911 PTR_ERR(bp->phylink));
912 return PTR_ERR(bp->phylink);
913 }
914
915 return 0;
916 }
917
macb_mdiobus_register(struct macb * bp)918 static int macb_mdiobus_register(struct macb *bp)
919 {
920 struct device_node *child, *np = bp->pdev->dev.of_node;
921
922 /* If we have a child named mdio, probe it instead of looking for PHYs
923 * directly under the MAC node
924 */
925 child = of_get_child_by_name(np, "mdio");
926 if (child) {
927 int ret = of_mdiobus_register(bp->mii_bus, child);
928
929 of_node_put(child);
930 return ret;
931 }
932
933 /* Only create the PHY from the device tree if at least one PHY is
934 * described. Otherwise scan the entire MDIO bus. We do this to support
935 * old device tree that did not follow the best practices and did not
936 * describe their network PHYs.
937 */
938 for_each_available_child_of_node(np, child)
939 if (of_mdiobus_child_is_phy(child)) {
940 /* The loop increments the child refcount,
941 * decrement it before returning.
942 */
943 of_node_put(child);
944
945 return of_mdiobus_register(bp->mii_bus, np);
946 }
947
948 return mdiobus_register(bp->mii_bus);
949 }
950
macb_mii_init(struct macb * bp)951 static int macb_mii_init(struct macb *bp)
952 {
953 struct device_node *child, *np = bp->pdev->dev.of_node;
954 int err = -ENXIO;
955
956 /* With fixed-link, we don't need to register the MDIO bus,
957 * except if we have a child named "mdio" in the device tree.
958 * In that case, some devices may be attached to the MACB's MDIO bus.
959 */
960 child = of_get_child_by_name(np, "mdio");
961 if (child)
962 of_node_put(child);
963 else if (of_phy_is_fixed_link(np))
964 return macb_mii_probe(bp->dev);
965
966 /* Enable management port */
967 macb_writel(bp, NCR, MACB_BIT(MPE));
968
969 bp->mii_bus = mdiobus_alloc();
970 if (!bp->mii_bus) {
971 err = -ENOMEM;
972 goto err_out;
973 }
974
975 bp->mii_bus->name = "MACB_mii_bus";
976 bp->mii_bus->read = &macb_mdio_read_c22;
977 bp->mii_bus->write = &macb_mdio_write_c22;
978 bp->mii_bus->read_c45 = &macb_mdio_read_c45;
979 bp->mii_bus->write_c45 = &macb_mdio_write_c45;
980 snprintf(bp->mii_bus->id, MII_BUS_ID_SIZE, "%s-%x",
981 bp->pdev->name, bp->pdev->id);
982 bp->mii_bus->priv = bp;
983 bp->mii_bus->parent = &bp->pdev->dev;
984
985 dev_set_drvdata(&bp->dev->dev, bp->mii_bus);
986
987 err = macb_mdiobus_register(bp);
988 if (err)
989 goto err_out_free_mdiobus;
990
991 err = macb_mii_probe(bp->dev);
992 if (err)
993 goto err_out_unregister_bus;
994
995 return 0;
996
997 err_out_unregister_bus:
998 mdiobus_unregister(bp->mii_bus);
999 err_out_free_mdiobus:
1000 mdiobus_free(bp->mii_bus);
1001 err_out:
1002 return err;
1003 }
1004
macb_update_stats(struct macb * bp)1005 static void macb_update_stats(struct macb *bp)
1006 {
1007 u32 *p = &bp->hw_stats.macb.rx_pause_frames;
1008 u32 *end = &bp->hw_stats.macb.tx_pause_frames + 1;
1009 int offset = MACB_PFR;
1010
1011 WARN_ON((unsigned long)(end - p - 1) != (MACB_TPF - MACB_PFR) / 4);
1012
1013 for (; p < end; p++, offset += 4)
1014 *p += bp->macb_reg_readl(bp, offset);
1015 }
1016
macb_halt_tx(struct macb * bp)1017 static int macb_halt_tx(struct macb *bp)
1018 {
1019 unsigned long halt_time, timeout;
1020 u32 status;
1021
1022 macb_writel(bp, NCR, macb_readl(bp, NCR) | MACB_BIT(THALT));
1023
1024 timeout = jiffies + usecs_to_jiffies(MACB_HALT_TIMEOUT);
1025 do {
1026 halt_time = jiffies;
1027 status = macb_readl(bp, TSR);
1028 if (!(status & MACB_BIT(TGO)))
1029 return 0;
1030
1031 udelay(250);
1032 } while (time_before(halt_time, timeout));
1033
1034 return -ETIMEDOUT;
1035 }
1036
macb_tx_unmap(struct macb * bp,struct macb_tx_skb * tx_skb,int budget)1037 static void macb_tx_unmap(struct macb *bp, struct macb_tx_skb *tx_skb, int budget)
1038 {
1039 if (tx_skb->mapping) {
1040 if (tx_skb->mapped_as_page)
1041 dma_unmap_page(&bp->pdev->dev, tx_skb->mapping,
1042 tx_skb->size, DMA_TO_DEVICE);
1043 else
1044 dma_unmap_single(&bp->pdev->dev, tx_skb->mapping,
1045 tx_skb->size, DMA_TO_DEVICE);
1046 tx_skb->mapping = 0;
1047 }
1048
1049 if (tx_skb->skb) {
1050 napi_consume_skb(tx_skb->skb, budget);
1051 tx_skb->skb = NULL;
1052 }
1053 }
1054
macb_set_addr(struct macb * bp,struct macb_dma_desc * desc,dma_addr_t addr)1055 static void macb_set_addr(struct macb *bp, struct macb_dma_desc *desc, dma_addr_t addr)
1056 {
1057 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT
1058 struct macb_dma_desc_64 *desc_64;
1059
1060 if (bp->hw_dma_cap & HW_DMA_CAP_64B) {
1061 desc_64 = macb_64b_desc(bp, desc);
1062 desc_64->addrh = upper_32_bits(addr);
1063 /* The low bits of RX address contain the RX_USED bit, clearing
1064 * of which allows packet RX. Make sure the high bits are also
1065 * visible to HW at that point.
1066 */
1067 dma_wmb();
1068 }
1069 #endif
1070 desc->addr = lower_32_bits(addr);
1071 }
1072
macb_get_addr(struct macb * bp,struct macb_dma_desc * desc)1073 static dma_addr_t macb_get_addr(struct macb *bp, struct macb_dma_desc *desc)
1074 {
1075 dma_addr_t addr = 0;
1076 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT
1077 struct macb_dma_desc_64 *desc_64;
1078
1079 if (bp->hw_dma_cap & HW_DMA_CAP_64B) {
1080 desc_64 = macb_64b_desc(bp, desc);
1081 addr = ((u64)(desc_64->addrh) << 32);
1082 }
1083 #endif
1084 addr |= MACB_BF(RX_WADDR, MACB_BFEXT(RX_WADDR, desc->addr));
1085 #ifdef CONFIG_MACB_USE_HWSTAMP
1086 if (bp->hw_dma_cap & HW_DMA_CAP_PTP)
1087 addr &= ~GEM_BIT(DMA_RXVALID);
1088 #endif
1089 return addr;
1090 }
1091
macb_tx_error_task(struct work_struct * work)1092 static void macb_tx_error_task(struct work_struct *work)
1093 {
1094 struct macb_queue *queue = container_of(work, struct macb_queue,
1095 tx_error_task);
1096 bool halt_timeout = false;
1097 struct macb *bp = queue->bp;
1098 struct macb_tx_skb *tx_skb;
1099 struct macb_dma_desc *desc;
1100 struct sk_buff *skb;
1101 unsigned int tail;
1102 unsigned long flags;
1103
1104 netdev_vdbg(bp->dev, "macb_tx_error_task: q = %u, t = %u, h = %u\n",
1105 (unsigned int)(queue - bp->queues),
1106 queue->tx_tail, queue->tx_head);
1107
1108 /* Prevent the queue NAPI TX poll from running, as it calls
1109 * macb_tx_complete(), which in turn may call netif_wake_subqueue().
1110 * As explained below, we have to halt the transmission before updating
1111 * TBQP registers so we call netif_tx_stop_all_queues() to notify the
1112 * network engine about the macb/gem being halted.
1113 */
1114 napi_disable(&queue->napi_tx);
1115 spin_lock_irqsave(&bp->lock, flags);
1116
1117 /* Make sure nobody is trying to queue up new packets */
1118 netif_tx_stop_all_queues(bp->dev);
1119
1120 /* Stop transmission now
1121 * (in case we have just queued new packets)
1122 * macb/gem must be halted to write TBQP register
1123 */
1124 if (macb_halt_tx(bp)) {
1125 netdev_err(bp->dev, "BUG: halt tx timed out\n");
1126 macb_writel(bp, NCR, macb_readl(bp, NCR) & (~MACB_BIT(TE)));
1127 halt_timeout = true;
1128 }
1129
1130 /* Treat frames in TX queue including the ones that caused the error.
1131 * Free transmit buffers in upper layer.
1132 */
1133 for (tail = queue->tx_tail; tail != queue->tx_head; tail++) {
1134 u32 ctrl;
1135
1136 desc = macb_tx_desc(queue, tail);
1137 ctrl = desc->ctrl;
1138 tx_skb = macb_tx_skb(queue, tail);
1139 skb = tx_skb->skb;
1140
1141 if (ctrl & MACB_BIT(TX_USED)) {
1142 /* skb is set for the last buffer of the frame */
1143 while (!skb) {
1144 macb_tx_unmap(bp, tx_skb, 0);
1145 tail++;
1146 tx_skb = macb_tx_skb(queue, tail);
1147 skb = tx_skb->skb;
1148 }
1149
1150 /* ctrl still refers to the first buffer descriptor
1151 * since it's the only one written back by the hardware
1152 */
1153 if (!(ctrl & MACB_BIT(TX_BUF_EXHAUSTED))) {
1154 netdev_vdbg(bp->dev, "txerr skb %u (data %p) TX complete\n",
1155 macb_tx_ring_wrap(bp, tail),
1156 skb->data);
1157 bp->dev->stats.tx_packets++;
1158 queue->stats.tx_packets++;
1159 bp->dev->stats.tx_bytes += skb->len;
1160 queue->stats.tx_bytes += skb->len;
1161 }
1162 } else {
1163 /* "Buffers exhausted mid-frame" errors may only happen
1164 * if the driver is buggy, so complain loudly about
1165 * those. Statistics are updated by hardware.
1166 */
1167 if (ctrl & MACB_BIT(TX_BUF_EXHAUSTED))
1168 netdev_err(bp->dev,
1169 "BUG: TX buffers exhausted mid-frame\n");
1170
1171 desc->ctrl = ctrl | MACB_BIT(TX_USED);
1172 }
1173
1174 macb_tx_unmap(bp, tx_skb, 0);
1175 }
1176
1177 /* Set end of TX queue */
1178 desc = macb_tx_desc(queue, 0);
1179 macb_set_addr(bp, desc, 0);
1180 desc->ctrl = MACB_BIT(TX_USED);
1181
1182 /* Make descriptor updates visible to hardware */
1183 wmb();
1184
1185 /* Reinitialize the TX desc queue */
1186 queue_writel(queue, TBQP, lower_32_bits(queue->tx_ring_dma));
1187 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT
1188 if (bp->hw_dma_cap & HW_DMA_CAP_64B)
1189 queue_writel(queue, TBQPH, upper_32_bits(queue->tx_ring_dma));
1190 #endif
1191 /* Make TX ring reflect state of hardware */
1192 queue->tx_head = 0;
1193 queue->tx_tail = 0;
1194
1195 /* Housework before enabling TX IRQ */
1196 macb_writel(bp, TSR, macb_readl(bp, TSR));
1197 queue_writel(queue, IER, MACB_TX_INT_FLAGS);
1198
1199 if (halt_timeout)
1200 macb_writel(bp, NCR, macb_readl(bp, NCR) | MACB_BIT(TE));
1201
1202 /* Now we are ready to start transmission again */
1203 netif_tx_start_all_queues(bp->dev);
1204 macb_writel(bp, NCR, macb_readl(bp, NCR) | MACB_BIT(TSTART));
1205
1206 spin_unlock_irqrestore(&bp->lock, flags);
1207 napi_enable(&queue->napi_tx);
1208 }
1209
ptp_one_step_sync(struct sk_buff * skb)1210 static bool ptp_one_step_sync(struct sk_buff *skb)
1211 {
1212 struct ptp_header *hdr;
1213 unsigned int ptp_class;
1214 u8 msgtype;
1215
1216 /* No need to parse packet if PTP TS is not involved */
1217 if (likely(!(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)))
1218 goto not_oss;
1219
1220 /* Identify and return whether PTP one step sync is being processed */
1221 ptp_class = ptp_classify_raw(skb);
1222 if (ptp_class == PTP_CLASS_NONE)
1223 goto not_oss;
1224
1225 hdr = ptp_parse_header(skb, ptp_class);
1226 if (!hdr)
1227 goto not_oss;
1228
1229 if (hdr->flag_field[0] & PTP_FLAG_TWOSTEP)
1230 goto not_oss;
1231
1232 msgtype = ptp_get_msgtype(hdr, ptp_class);
1233 if (msgtype == PTP_MSGTYPE_SYNC)
1234 return true;
1235
1236 not_oss:
1237 return false;
1238 }
1239
macb_tx_complete(struct macb_queue * queue,int budget)1240 static int macb_tx_complete(struct macb_queue *queue, int budget)
1241 {
1242 struct macb *bp = queue->bp;
1243 u16 queue_index = queue - bp->queues;
1244 unsigned int tail;
1245 unsigned int head;
1246 int packets = 0;
1247
1248 spin_lock(&queue->tx_ptr_lock);
1249 head = queue->tx_head;
1250 for (tail = queue->tx_tail; tail != head && packets < budget; tail++) {
1251 struct macb_tx_skb *tx_skb;
1252 struct sk_buff *skb;
1253 struct macb_dma_desc *desc;
1254 u32 ctrl;
1255
1256 desc = macb_tx_desc(queue, tail);
1257
1258 /* Make hw descriptor updates visible to CPU */
1259 rmb();
1260
1261 ctrl = desc->ctrl;
1262
1263 /* TX_USED bit is only set by hardware on the very first buffer
1264 * descriptor of the transmitted frame.
1265 */
1266 if (!(ctrl & MACB_BIT(TX_USED)))
1267 break;
1268
1269 /* Process all buffers of the current transmitted frame */
1270 for (;; tail++) {
1271 tx_skb = macb_tx_skb(queue, tail);
1272 skb = tx_skb->skb;
1273
1274 /* First, update TX stats if needed */
1275 if (skb) {
1276 if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) &&
1277 !ptp_one_step_sync(skb))
1278 gem_ptp_do_txstamp(bp, skb, desc);
1279
1280 netdev_vdbg(bp->dev, "skb %u (data %p) TX complete\n",
1281 macb_tx_ring_wrap(bp, tail),
1282 skb->data);
1283 bp->dev->stats.tx_packets++;
1284 queue->stats.tx_packets++;
1285 bp->dev->stats.tx_bytes += skb->len;
1286 queue->stats.tx_bytes += skb->len;
1287 packets++;
1288 }
1289
1290 /* Now we can safely release resources */
1291 macb_tx_unmap(bp, tx_skb, budget);
1292
1293 /* skb is set only for the last buffer of the frame.
1294 * WARNING: at this point skb has been freed by
1295 * macb_tx_unmap().
1296 */
1297 if (skb)
1298 break;
1299 }
1300 }
1301
1302 queue->tx_tail = tail;
1303 if (__netif_subqueue_stopped(bp->dev, queue_index) &&
1304 CIRC_CNT(queue->tx_head, queue->tx_tail,
1305 bp->tx_ring_size) <= MACB_TX_WAKEUP_THRESH(bp))
1306 netif_wake_subqueue(bp->dev, queue_index);
1307 spin_unlock(&queue->tx_ptr_lock);
1308
1309 return packets;
1310 }
1311
gem_rx_refill(struct macb_queue * queue)1312 static void gem_rx_refill(struct macb_queue *queue)
1313 {
1314 unsigned int entry;
1315 struct sk_buff *skb;
1316 dma_addr_t paddr;
1317 struct macb *bp = queue->bp;
1318 struct macb_dma_desc *desc;
1319
1320 while (CIRC_SPACE(queue->rx_prepared_head, queue->rx_tail,
1321 bp->rx_ring_size) > 0) {
1322 entry = macb_rx_ring_wrap(bp, queue->rx_prepared_head);
1323
1324 /* Make hw descriptor updates visible to CPU */
1325 rmb();
1326
1327 desc = macb_rx_desc(queue, entry);
1328
1329 if (!queue->rx_skbuff[entry]) {
1330 /* allocate sk_buff for this free entry in ring */
1331 skb = netdev_alloc_skb(bp->dev, bp->rx_buffer_size);
1332 if (unlikely(!skb)) {
1333 netdev_err(bp->dev,
1334 "Unable to allocate sk_buff\n");
1335 break;
1336 }
1337
1338 /* now fill corresponding descriptor entry */
1339 paddr = dma_map_single(&bp->pdev->dev, skb->data,
1340 bp->rx_buffer_size,
1341 DMA_FROM_DEVICE);
1342 if (dma_mapping_error(&bp->pdev->dev, paddr)) {
1343 dev_kfree_skb(skb);
1344 break;
1345 }
1346
1347 queue->rx_skbuff[entry] = skb;
1348
1349 if (entry == bp->rx_ring_size - 1)
1350 paddr |= MACB_BIT(RX_WRAP);
1351 desc->ctrl = 0;
1352 /* Setting addr clears RX_USED and allows reception,
1353 * make sure ctrl is cleared first to avoid a race.
1354 */
1355 dma_wmb();
1356 macb_set_addr(bp, desc, paddr);
1357
1358 /* properly align Ethernet header */
1359 skb_reserve(skb, NET_IP_ALIGN);
1360 } else {
1361 desc->ctrl = 0;
1362 dma_wmb();
1363 desc->addr &= ~MACB_BIT(RX_USED);
1364 }
1365 queue->rx_prepared_head++;
1366 }
1367
1368 /* Make descriptor updates visible to hardware */
1369 wmb();
1370
1371 netdev_vdbg(bp->dev, "rx ring: queue: %p, prepared head %d, tail %d\n",
1372 queue, queue->rx_prepared_head, queue->rx_tail);
1373 }
1374
1375 /* Mark DMA descriptors from begin up to and not including end as unused */
discard_partial_frame(struct macb_queue * queue,unsigned int begin,unsigned int end)1376 static void discard_partial_frame(struct macb_queue *queue, unsigned int begin,
1377 unsigned int end)
1378 {
1379 unsigned int frag;
1380
1381 for (frag = begin; frag != end; frag++) {
1382 struct macb_dma_desc *desc = macb_rx_desc(queue, frag);
1383
1384 desc->addr &= ~MACB_BIT(RX_USED);
1385 }
1386
1387 /* Make descriptor updates visible to hardware */
1388 wmb();
1389
1390 /* When this happens, the hardware stats registers for
1391 * whatever caused this is updated, so we don't have to record
1392 * anything.
1393 */
1394 }
1395
gem_rx(struct macb_queue * queue,struct napi_struct * napi,int budget)1396 static int gem_rx(struct macb_queue *queue, struct napi_struct *napi,
1397 int budget)
1398 {
1399 struct macb *bp = queue->bp;
1400 unsigned int len;
1401 unsigned int entry;
1402 struct sk_buff *skb;
1403 struct macb_dma_desc *desc;
1404 int count = 0;
1405
1406 while (count < budget) {
1407 u32 ctrl;
1408 dma_addr_t addr;
1409 bool rxused;
1410
1411 entry = macb_rx_ring_wrap(bp, queue->rx_tail);
1412 desc = macb_rx_desc(queue, entry);
1413
1414 /* Make hw descriptor updates visible to CPU */
1415 rmb();
1416
1417 rxused = (desc->addr & MACB_BIT(RX_USED)) ? true : false;
1418 addr = macb_get_addr(bp, desc);
1419
1420 if (!rxused)
1421 break;
1422
1423 /* Ensure ctrl is at least as up-to-date as rxused */
1424 dma_rmb();
1425
1426 ctrl = desc->ctrl;
1427
1428 queue->rx_tail++;
1429 count++;
1430
1431 if (!(ctrl & MACB_BIT(RX_SOF) && ctrl & MACB_BIT(RX_EOF))) {
1432 netdev_err(bp->dev,
1433 "not whole frame pointed by descriptor\n");
1434 bp->dev->stats.rx_dropped++;
1435 queue->stats.rx_dropped++;
1436 break;
1437 }
1438 skb = queue->rx_skbuff[entry];
1439 if (unlikely(!skb)) {
1440 netdev_err(bp->dev,
1441 "inconsistent Rx descriptor chain\n");
1442 bp->dev->stats.rx_dropped++;
1443 queue->stats.rx_dropped++;
1444 break;
1445 }
1446 /* now everything is ready for receiving packet */
1447 queue->rx_skbuff[entry] = NULL;
1448 len = ctrl & bp->rx_frm_len_mask;
1449
1450 netdev_vdbg(bp->dev, "gem_rx %u (len %u)\n", entry, len);
1451
1452 skb_put(skb, len);
1453 dma_unmap_single(&bp->pdev->dev, addr,
1454 bp->rx_buffer_size, DMA_FROM_DEVICE);
1455
1456 skb->protocol = eth_type_trans(skb, bp->dev);
1457 skb_checksum_none_assert(skb);
1458 if (bp->dev->features & NETIF_F_RXCSUM &&
1459 !(bp->dev->flags & IFF_PROMISC) &&
1460 GEM_BFEXT(RX_CSUM, ctrl) & GEM_RX_CSUM_CHECKED_MASK)
1461 skb->ip_summed = CHECKSUM_UNNECESSARY;
1462
1463 bp->dev->stats.rx_packets++;
1464 queue->stats.rx_packets++;
1465 bp->dev->stats.rx_bytes += skb->len;
1466 queue->stats.rx_bytes += skb->len;
1467
1468 gem_ptp_do_rxstamp(bp, skb, desc);
1469
1470 #if defined(DEBUG) && defined(VERBOSE_DEBUG)
1471 netdev_vdbg(bp->dev, "received skb of length %u, csum: %08x\n",
1472 skb->len, skb->csum);
1473 print_hex_dump(KERN_DEBUG, " mac: ", DUMP_PREFIX_ADDRESS, 16, 1,
1474 skb_mac_header(skb), 16, true);
1475 print_hex_dump(KERN_DEBUG, "data: ", DUMP_PREFIX_ADDRESS, 16, 1,
1476 skb->data, 32, true);
1477 #endif
1478
1479 napi_gro_receive(napi, skb);
1480 }
1481
1482 gem_rx_refill(queue);
1483
1484 return count;
1485 }
1486
macb_rx_frame(struct macb_queue * queue,struct napi_struct * napi,unsigned int first_frag,unsigned int last_frag)1487 static int macb_rx_frame(struct macb_queue *queue, struct napi_struct *napi,
1488 unsigned int first_frag, unsigned int last_frag)
1489 {
1490 unsigned int len;
1491 unsigned int frag;
1492 unsigned int offset;
1493 struct sk_buff *skb;
1494 struct macb_dma_desc *desc;
1495 struct macb *bp = queue->bp;
1496
1497 desc = macb_rx_desc(queue, last_frag);
1498 len = desc->ctrl & bp->rx_frm_len_mask;
1499
1500 netdev_vdbg(bp->dev, "macb_rx_frame frags %u - %u (len %u)\n",
1501 macb_rx_ring_wrap(bp, first_frag),
1502 macb_rx_ring_wrap(bp, last_frag), len);
1503
1504 /* The ethernet header starts NET_IP_ALIGN bytes into the
1505 * first buffer. Since the header is 14 bytes, this makes the
1506 * payload word-aligned.
1507 *
1508 * Instead of calling skb_reserve(NET_IP_ALIGN), we just copy
1509 * the two padding bytes into the skb so that we avoid hitting
1510 * the slowpath in memcpy(), and pull them off afterwards.
1511 */
1512 skb = netdev_alloc_skb(bp->dev, len + NET_IP_ALIGN);
1513 if (!skb) {
1514 bp->dev->stats.rx_dropped++;
1515 for (frag = first_frag; ; frag++) {
1516 desc = macb_rx_desc(queue, frag);
1517 desc->addr &= ~MACB_BIT(RX_USED);
1518 if (frag == last_frag)
1519 break;
1520 }
1521
1522 /* Make descriptor updates visible to hardware */
1523 wmb();
1524
1525 return 1;
1526 }
1527
1528 offset = 0;
1529 len += NET_IP_ALIGN;
1530 skb_checksum_none_assert(skb);
1531 skb_put(skb, len);
1532
1533 for (frag = first_frag; ; frag++) {
1534 unsigned int frag_len = bp->rx_buffer_size;
1535
1536 if (offset + frag_len > len) {
1537 if (unlikely(frag != last_frag)) {
1538 dev_kfree_skb_any(skb);
1539 return -1;
1540 }
1541 frag_len = len - offset;
1542 }
1543 skb_copy_to_linear_data_offset(skb, offset,
1544 macb_rx_buffer(queue, frag),
1545 frag_len);
1546 offset += bp->rx_buffer_size;
1547 desc = macb_rx_desc(queue, frag);
1548 desc->addr &= ~MACB_BIT(RX_USED);
1549
1550 if (frag == last_frag)
1551 break;
1552 }
1553
1554 /* Make descriptor updates visible to hardware */
1555 wmb();
1556
1557 __skb_pull(skb, NET_IP_ALIGN);
1558 skb->protocol = eth_type_trans(skb, bp->dev);
1559
1560 bp->dev->stats.rx_packets++;
1561 bp->dev->stats.rx_bytes += skb->len;
1562 netdev_vdbg(bp->dev, "received skb of length %u, csum: %08x\n",
1563 skb->len, skb->csum);
1564 napi_gro_receive(napi, skb);
1565
1566 return 0;
1567 }
1568
macb_init_rx_ring(struct macb_queue * queue)1569 static inline void macb_init_rx_ring(struct macb_queue *queue)
1570 {
1571 struct macb *bp = queue->bp;
1572 dma_addr_t addr;
1573 struct macb_dma_desc *desc = NULL;
1574 int i;
1575
1576 addr = queue->rx_buffers_dma;
1577 for (i = 0; i < bp->rx_ring_size; i++) {
1578 desc = macb_rx_desc(queue, i);
1579 macb_set_addr(bp, desc, addr);
1580 desc->ctrl = 0;
1581 addr += bp->rx_buffer_size;
1582 }
1583 desc->addr |= MACB_BIT(RX_WRAP);
1584 queue->rx_tail = 0;
1585 }
1586
macb_rx(struct macb_queue * queue,struct napi_struct * napi,int budget)1587 static int macb_rx(struct macb_queue *queue, struct napi_struct *napi,
1588 int budget)
1589 {
1590 struct macb *bp = queue->bp;
1591 bool reset_rx_queue = false;
1592 int received = 0;
1593 unsigned int tail;
1594 int first_frag = -1;
1595
1596 for (tail = queue->rx_tail; budget > 0; tail++) {
1597 struct macb_dma_desc *desc = macb_rx_desc(queue, tail);
1598 u32 ctrl;
1599
1600 /* Make hw descriptor updates visible to CPU */
1601 rmb();
1602
1603 if (!(desc->addr & MACB_BIT(RX_USED)))
1604 break;
1605
1606 /* Ensure ctrl is at least as up-to-date as addr */
1607 dma_rmb();
1608
1609 ctrl = desc->ctrl;
1610
1611 if (ctrl & MACB_BIT(RX_SOF)) {
1612 if (first_frag != -1)
1613 discard_partial_frame(queue, first_frag, tail);
1614 first_frag = tail;
1615 }
1616
1617 if (ctrl & MACB_BIT(RX_EOF)) {
1618 int dropped;
1619
1620 if (unlikely(first_frag == -1)) {
1621 reset_rx_queue = true;
1622 continue;
1623 }
1624
1625 dropped = macb_rx_frame(queue, napi, first_frag, tail);
1626 first_frag = -1;
1627 if (unlikely(dropped < 0)) {
1628 reset_rx_queue = true;
1629 continue;
1630 }
1631 if (!dropped) {
1632 received++;
1633 budget--;
1634 }
1635 }
1636 }
1637
1638 if (unlikely(reset_rx_queue)) {
1639 unsigned long flags;
1640 u32 ctrl;
1641
1642 netdev_err(bp->dev, "RX queue corruption: reset it\n");
1643
1644 spin_lock_irqsave(&bp->lock, flags);
1645
1646 ctrl = macb_readl(bp, NCR);
1647 macb_writel(bp, NCR, ctrl & ~MACB_BIT(RE));
1648
1649 macb_init_rx_ring(queue);
1650 queue_writel(queue, RBQP, queue->rx_ring_dma);
1651
1652 macb_writel(bp, NCR, ctrl | MACB_BIT(RE));
1653
1654 spin_unlock_irqrestore(&bp->lock, flags);
1655 return received;
1656 }
1657
1658 if (first_frag != -1)
1659 queue->rx_tail = first_frag;
1660 else
1661 queue->rx_tail = tail;
1662
1663 return received;
1664 }
1665
macb_rx_pending(struct macb_queue * queue)1666 static bool macb_rx_pending(struct macb_queue *queue)
1667 {
1668 struct macb *bp = queue->bp;
1669 unsigned int entry;
1670 struct macb_dma_desc *desc;
1671
1672 entry = macb_rx_ring_wrap(bp, queue->rx_tail);
1673 desc = macb_rx_desc(queue, entry);
1674
1675 /* Make hw descriptor updates visible to CPU */
1676 rmb();
1677
1678 return (desc->addr & MACB_BIT(RX_USED)) != 0;
1679 }
1680
macb_rx_poll(struct napi_struct * napi,int budget)1681 static int macb_rx_poll(struct napi_struct *napi, int budget)
1682 {
1683 struct macb_queue *queue = container_of(napi, struct macb_queue, napi_rx);
1684 struct macb *bp = queue->bp;
1685 int work_done;
1686
1687 work_done = bp->macbgem_ops.mog_rx(queue, napi, budget);
1688
1689 netdev_vdbg(bp->dev, "RX poll: queue = %u, work_done = %d, budget = %d\n",
1690 (unsigned int)(queue - bp->queues), work_done, budget);
1691
1692 if (work_done < budget && napi_complete_done(napi, work_done)) {
1693 queue_writel(queue, IER, bp->rx_intr_mask);
1694
1695 /* Packet completions only seem to propagate to raise
1696 * interrupts when interrupts are enabled at the time, so if
1697 * packets were received while interrupts were disabled,
1698 * they will not cause another interrupt to be generated when
1699 * interrupts are re-enabled.
1700 * Check for this case here to avoid losing a wakeup. This can
1701 * potentially race with the interrupt handler doing the same
1702 * actions if an interrupt is raised just after enabling them,
1703 * but this should be harmless.
1704 */
1705 if (macb_rx_pending(queue)) {
1706 queue_writel(queue, IDR, bp->rx_intr_mask);
1707 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE)
1708 queue_writel(queue, ISR, MACB_BIT(RCOMP));
1709 netdev_vdbg(bp->dev, "poll: packets pending, reschedule\n");
1710 napi_schedule(napi);
1711 }
1712 }
1713
1714 /* TODO: Handle errors */
1715
1716 return work_done;
1717 }
1718
macb_tx_restart(struct macb_queue * queue)1719 static void macb_tx_restart(struct macb_queue *queue)
1720 {
1721 struct macb *bp = queue->bp;
1722 unsigned int head_idx, tbqp;
1723
1724 spin_lock(&queue->tx_ptr_lock);
1725
1726 if (queue->tx_head == queue->tx_tail)
1727 goto out_tx_ptr_unlock;
1728
1729 tbqp = queue_readl(queue, TBQP) / macb_dma_desc_get_size(bp);
1730 tbqp = macb_adj_dma_desc_idx(bp, macb_tx_ring_wrap(bp, tbqp));
1731 head_idx = macb_adj_dma_desc_idx(bp, macb_tx_ring_wrap(bp, queue->tx_head));
1732
1733 if (tbqp == head_idx)
1734 goto out_tx_ptr_unlock;
1735
1736 spin_lock_irq(&bp->lock);
1737 macb_writel(bp, NCR, macb_readl(bp, NCR) | MACB_BIT(TSTART));
1738 spin_unlock_irq(&bp->lock);
1739
1740 out_tx_ptr_unlock:
1741 spin_unlock(&queue->tx_ptr_lock);
1742 }
1743
macb_tx_complete_pending(struct macb_queue * queue)1744 static bool macb_tx_complete_pending(struct macb_queue *queue)
1745 {
1746 bool retval = false;
1747
1748 spin_lock(&queue->tx_ptr_lock);
1749 if (queue->tx_head != queue->tx_tail) {
1750 /* Make hw descriptor updates visible to CPU */
1751 rmb();
1752
1753 if (macb_tx_desc(queue, queue->tx_tail)->ctrl & MACB_BIT(TX_USED))
1754 retval = true;
1755 }
1756 spin_unlock(&queue->tx_ptr_lock);
1757 return retval;
1758 }
1759
macb_tx_poll(struct napi_struct * napi,int budget)1760 static int macb_tx_poll(struct napi_struct *napi, int budget)
1761 {
1762 struct macb_queue *queue = container_of(napi, struct macb_queue, napi_tx);
1763 struct macb *bp = queue->bp;
1764 int work_done;
1765
1766 work_done = macb_tx_complete(queue, budget);
1767
1768 rmb(); // ensure txubr_pending is up to date
1769 if (queue->txubr_pending) {
1770 queue->txubr_pending = false;
1771 netdev_vdbg(bp->dev, "poll: tx restart\n");
1772 macb_tx_restart(queue);
1773 }
1774
1775 netdev_vdbg(bp->dev, "TX poll: queue = %u, work_done = %d, budget = %d\n",
1776 (unsigned int)(queue - bp->queues), work_done, budget);
1777
1778 if (work_done < budget && napi_complete_done(napi, work_done)) {
1779 queue_writel(queue, IER, MACB_BIT(TCOMP));
1780
1781 /* Packet completions only seem to propagate to raise
1782 * interrupts when interrupts are enabled at the time, so if
1783 * packets were sent while interrupts were disabled,
1784 * they will not cause another interrupt to be generated when
1785 * interrupts are re-enabled.
1786 * Check for this case here to avoid losing a wakeup. This can
1787 * potentially race with the interrupt handler doing the same
1788 * actions if an interrupt is raised just after enabling them,
1789 * but this should be harmless.
1790 */
1791 if (macb_tx_complete_pending(queue)) {
1792 queue_writel(queue, IDR, MACB_BIT(TCOMP));
1793 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE)
1794 queue_writel(queue, ISR, MACB_BIT(TCOMP));
1795 netdev_vdbg(bp->dev, "TX poll: packets pending, reschedule\n");
1796 napi_schedule(napi);
1797 }
1798 }
1799
1800 return work_done;
1801 }
1802
macb_hresp_error_task(struct work_struct * work)1803 static void macb_hresp_error_task(struct work_struct *work)
1804 {
1805 struct macb *bp = from_work(bp, work, hresp_err_bh_work);
1806 struct net_device *dev = bp->dev;
1807 struct macb_queue *queue;
1808 unsigned int q;
1809 u32 ctrl;
1810
1811 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) {
1812 queue_writel(queue, IDR, bp->rx_intr_mask |
1813 MACB_TX_INT_FLAGS |
1814 MACB_BIT(HRESP));
1815 }
1816 ctrl = macb_readl(bp, NCR);
1817 ctrl &= ~(MACB_BIT(RE) | MACB_BIT(TE));
1818 macb_writel(bp, NCR, ctrl);
1819
1820 netif_tx_stop_all_queues(dev);
1821 netif_carrier_off(dev);
1822
1823 bp->macbgem_ops.mog_init_rings(bp);
1824
1825 /* Initialize TX and RX buffers */
1826 macb_init_buffers(bp);
1827
1828 /* Enable interrupts */
1829 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue)
1830 queue_writel(queue, IER,
1831 bp->rx_intr_mask |
1832 MACB_TX_INT_FLAGS |
1833 MACB_BIT(HRESP));
1834
1835 ctrl |= MACB_BIT(RE) | MACB_BIT(TE);
1836 macb_writel(bp, NCR, ctrl);
1837
1838 netif_carrier_on(dev);
1839 netif_tx_start_all_queues(dev);
1840 }
1841
macb_wol_interrupt(int irq,void * dev_id)1842 static irqreturn_t macb_wol_interrupt(int irq, void *dev_id)
1843 {
1844 struct macb_queue *queue = dev_id;
1845 struct macb *bp = queue->bp;
1846 u32 status;
1847
1848 status = queue_readl(queue, ISR);
1849
1850 if (unlikely(!status))
1851 return IRQ_NONE;
1852
1853 spin_lock(&bp->lock);
1854
1855 if (status & MACB_BIT(WOL)) {
1856 queue_writel(queue, IDR, MACB_BIT(WOL));
1857 macb_writel(bp, WOL, 0);
1858 netdev_vdbg(bp->dev, "MACB WoL: queue = %u, isr = 0x%08lx\n",
1859 (unsigned int)(queue - bp->queues),
1860 (unsigned long)status);
1861 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE)
1862 queue_writel(queue, ISR, MACB_BIT(WOL));
1863 pm_wakeup_event(&bp->pdev->dev, 0);
1864 }
1865
1866 spin_unlock(&bp->lock);
1867
1868 return IRQ_HANDLED;
1869 }
1870
gem_wol_interrupt(int irq,void * dev_id)1871 static irqreturn_t gem_wol_interrupt(int irq, void *dev_id)
1872 {
1873 struct macb_queue *queue = dev_id;
1874 struct macb *bp = queue->bp;
1875 u32 status;
1876
1877 status = queue_readl(queue, ISR);
1878
1879 if (unlikely(!status))
1880 return IRQ_NONE;
1881
1882 spin_lock(&bp->lock);
1883
1884 if (status & GEM_BIT(WOL)) {
1885 queue_writel(queue, IDR, GEM_BIT(WOL));
1886 gem_writel(bp, WOL, 0);
1887 netdev_vdbg(bp->dev, "GEM WoL: queue = %u, isr = 0x%08lx\n",
1888 (unsigned int)(queue - bp->queues),
1889 (unsigned long)status);
1890 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE)
1891 queue_writel(queue, ISR, GEM_BIT(WOL));
1892 pm_wakeup_event(&bp->pdev->dev, 0);
1893 }
1894
1895 spin_unlock(&bp->lock);
1896
1897 return IRQ_HANDLED;
1898 }
1899
macb_interrupt(int irq,void * dev_id)1900 static irqreturn_t macb_interrupt(int irq, void *dev_id)
1901 {
1902 struct macb_queue *queue = dev_id;
1903 struct macb *bp = queue->bp;
1904 struct net_device *dev = bp->dev;
1905 u32 status, ctrl;
1906
1907 status = queue_readl(queue, ISR);
1908
1909 if (unlikely(!status))
1910 return IRQ_NONE;
1911
1912 spin_lock(&bp->lock);
1913
1914 while (status) {
1915 /* close possible race with dev_close */
1916 if (unlikely(!netif_running(dev))) {
1917 queue_writel(queue, IDR, -1);
1918 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE)
1919 queue_writel(queue, ISR, -1);
1920 break;
1921 }
1922
1923 netdev_vdbg(bp->dev, "queue = %u, isr = 0x%08lx\n",
1924 (unsigned int)(queue - bp->queues),
1925 (unsigned long)status);
1926
1927 if (status & bp->rx_intr_mask) {
1928 /* There's no point taking any more interrupts
1929 * until we have processed the buffers. The
1930 * scheduling call may fail if the poll routine
1931 * is already scheduled, so disable interrupts
1932 * now.
1933 */
1934 queue_writel(queue, IDR, bp->rx_intr_mask);
1935 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE)
1936 queue_writel(queue, ISR, MACB_BIT(RCOMP));
1937
1938 if (napi_schedule_prep(&queue->napi_rx)) {
1939 netdev_vdbg(bp->dev, "scheduling RX softirq\n");
1940 __napi_schedule(&queue->napi_rx);
1941 }
1942 }
1943
1944 if (status & (MACB_BIT(TCOMP) |
1945 MACB_BIT(TXUBR))) {
1946 queue_writel(queue, IDR, MACB_BIT(TCOMP));
1947 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE)
1948 queue_writel(queue, ISR, MACB_BIT(TCOMP) |
1949 MACB_BIT(TXUBR));
1950
1951 if (status & MACB_BIT(TXUBR)) {
1952 queue->txubr_pending = true;
1953 wmb(); // ensure softirq can see update
1954 }
1955
1956 if (napi_schedule_prep(&queue->napi_tx)) {
1957 netdev_vdbg(bp->dev, "scheduling TX softirq\n");
1958 __napi_schedule(&queue->napi_tx);
1959 }
1960 }
1961
1962 if (unlikely(status & (MACB_TX_ERR_FLAGS))) {
1963 queue_writel(queue, IDR, MACB_TX_INT_FLAGS);
1964 schedule_work(&queue->tx_error_task);
1965
1966 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE)
1967 queue_writel(queue, ISR, MACB_TX_ERR_FLAGS);
1968
1969 break;
1970 }
1971
1972 /* Link change detection isn't possible with RMII, so we'll
1973 * add that if/when we get our hands on a full-blown MII PHY.
1974 */
1975
1976 /* There is a hardware issue under heavy load where DMA can
1977 * stop, this causes endless "used buffer descriptor read"
1978 * interrupts but it can be cleared by re-enabling RX. See
1979 * the at91rm9200 manual, section 41.3.1 or the Zynq manual
1980 * section 16.7.4 for details. RXUBR is only enabled for
1981 * these two versions.
1982 */
1983 if (status & MACB_BIT(RXUBR)) {
1984 ctrl = macb_readl(bp, NCR);
1985 macb_writel(bp, NCR, ctrl & ~MACB_BIT(RE));
1986 wmb();
1987 macb_writel(bp, NCR, ctrl | MACB_BIT(RE));
1988
1989 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE)
1990 queue_writel(queue, ISR, MACB_BIT(RXUBR));
1991 }
1992
1993 if (status & MACB_BIT(ISR_ROVR)) {
1994 /* We missed at least one packet */
1995 if (macb_is_gem(bp))
1996 bp->hw_stats.gem.rx_overruns++;
1997 else
1998 bp->hw_stats.macb.rx_overruns++;
1999
2000 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE)
2001 queue_writel(queue, ISR, MACB_BIT(ISR_ROVR));
2002 }
2003
2004 if (status & MACB_BIT(HRESP)) {
2005 queue_work(system_bh_wq, &bp->hresp_err_bh_work);
2006 netdev_err(dev, "DMA bus error: HRESP not OK\n");
2007
2008 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE)
2009 queue_writel(queue, ISR, MACB_BIT(HRESP));
2010 }
2011 status = queue_readl(queue, ISR);
2012 }
2013
2014 spin_unlock(&bp->lock);
2015
2016 return IRQ_HANDLED;
2017 }
2018
2019 #ifdef CONFIG_NET_POLL_CONTROLLER
2020 /* Polling receive - used by netconsole and other diagnostic tools
2021 * to allow network i/o with interrupts disabled.
2022 */
macb_poll_controller(struct net_device * dev)2023 static void macb_poll_controller(struct net_device *dev)
2024 {
2025 struct macb *bp = netdev_priv(dev);
2026 struct macb_queue *queue;
2027 unsigned long flags;
2028 unsigned int q;
2029
2030 local_irq_save(flags);
2031 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue)
2032 macb_interrupt(dev->irq, queue);
2033 local_irq_restore(flags);
2034 }
2035 #endif
2036
macb_tx_map(struct macb * bp,struct macb_queue * queue,struct sk_buff * skb,unsigned int hdrlen)2037 static unsigned int macb_tx_map(struct macb *bp,
2038 struct macb_queue *queue,
2039 struct sk_buff *skb,
2040 unsigned int hdrlen)
2041 {
2042 dma_addr_t mapping;
2043 unsigned int len, entry, i, tx_head = queue->tx_head;
2044 struct macb_tx_skb *tx_skb = NULL;
2045 struct macb_dma_desc *desc;
2046 unsigned int offset, size, count = 0;
2047 unsigned int f, nr_frags = skb_shinfo(skb)->nr_frags;
2048 unsigned int eof = 1, mss_mfs = 0;
2049 u32 ctrl, lso_ctrl = 0, seq_ctrl = 0;
2050
2051 /* LSO */
2052 if (skb_shinfo(skb)->gso_size != 0) {
2053 if (ip_hdr(skb)->protocol == IPPROTO_UDP)
2054 /* UDP - UFO */
2055 lso_ctrl = MACB_LSO_UFO_ENABLE;
2056 else
2057 /* TCP - TSO */
2058 lso_ctrl = MACB_LSO_TSO_ENABLE;
2059 }
2060
2061 /* First, map non-paged data */
2062 len = skb_headlen(skb);
2063
2064 /* first buffer length */
2065 size = hdrlen;
2066
2067 offset = 0;
2068 while (len) {
2069 entry = macb_tx_ring_wrap(bp, tx_head);
2070 tx_skb = &queue->tx_skb[entry];
2071
2072 mapping = dma_map_single(&bp->pdev->dev,
2073 skb->data + offset,
2074 size, DMA_TO_DEVICE);
2075 if (dma_mapping_error(&bp->pdev->dev, mapping))
2076 goto dma_error;
2077
2078 /* Save info to properly release resources */
2079 tx_skb->skb = NULL;
2080 tx_skb->mapping = mapping;
2081 tx_skb->size = size;
2082 tx_skb->mapped_as_page = false;
2083
2084 len -= size;
2085 offset += size;
2086 count++;
2087 tx_head++;
2088
2089 size = min(len, bp->max_tx_length);
2090 }
2091
2092 /* Then, map paged data from fragments */
2093 for (f = 0; f < nr_frags; f++) {
2094 const skb_frag_t *frag = &skb_shinfo(skb)->frags[f];
2095
2096 len = skb_frag_size(frag);
2097 offset = 0;
2098 while (len) {
2099 size = min(len, bp->max_tx_length);
2100 entry = macb_tx_ring_wrap(bp, tx_head);
2101 tx_skb = &queue->tx_skb[entry];
2102
2103 mapping = skb_frag_dma_map(&bp->pdev->dev, frag,
2104 offset, size, DMA_TO_DEVICE);
2105 if (dma_mapping_error(&bp->pdev->dev, mapping))
2106 goto dma_error;
2107
2108 /* Save info to properly release resources */
2109 tx_skb->skb = NULL;
2110 tx_skb->mapping = mapping;
2111 tx_skb->size = size;
2112 tx_skb->mapped_as_page = true;
2113
2114 len -= size;
2115 offset += size;
2116 count++;
2117 tx_head++;
2118 }
2119 }
2120
2121 /* Should never happen */
2122 if (unlikely(!tx_skb)) {
2123 netdev_err(bp->dev, "BUG! empty skb!\n");
2124 return 0;
2125 }
2126
2127 /* This is the last buffer of the frame: save socket buffer */
2128 tx_skb->skb = skb;
2129
2130 /* Update TX ring: update buffer descriptors in reverse order
2131 * to avoid race condition
2132 */
2133
2134 /* Set 'TX_USED' bit in buffer descriptor at tx_head position
2135 * to set the end of TX queue
2136 */
2137 i = tx_head;
2138 entry = macb_tx_ring_wrap(bp, i);
2139 ctrl = MACB_BIT(TX_USED);
2140 desc = macb_tx_desc(queue, entry);
2141 desc->ctrl = ctrl;
2142
2143 if (lso_ctrl) {
2144 if (lso_ctrl == MACB_LSO_UFO_ENABLE)
2145 /* include header and FCS in value given to h/w */
2146 mss_mfs = skb_shinfo(skb)->gso_size +
2147 skb_transport_offset(skb) +
2148 ETH_FCS_LEN;
2149 else /* TSO */ {
2150 mss_mfs = skb_shinfo(skb)->gso_size;
2151 /* TCP Sequence Number Source Select
2152 * can be set only for TSO
2153 */
2154 seq_ctrl = 0;
2155 }
2156 }
2157
2158 do {
2159 i--;
2160 entry = macb_tx_ring_wrap(bp, i);
2161 tx_skb = &queue->tx_skb[entry];
2162 desc = macb_tx_desc(queue, entry);
2163
2164 ctrl = (u32)tx_skb->size;
2165 if (eof) {
2166 ctrl |= MACB_BIT(TX_LAST);
2167 eof = 0;
2168 }
2169 if (unlikely(entry == (bp->tx_ring_size - 1)))
2170 ctrl |= MACB_BIT(TX_WRAP);
2171
2172 /* First descriptor is header descriptor */
2173 if (i == queue->tx_head) {
2174 ctrl |= MACB_BF(TX_LSO, lso_ctrl);
2175 ctrl |= MACB_BF(TX_TCP_SEQ_SRC, seq_ctrl);
2176 if ((bp->dev->features & NETIF_F_HW_CSUM) &&
2177 skb->ip_summed != CHECKSUM_PARTIAL && !lso_ctrl &&
2178 !ptp_one_step_sync(skb))
2179 ctrl |= MACB_BIT(TX_NOCRC);
2180 } else
2181 /* Only set MSS/MFS on payload descriptors
2182 * (second or later descriptor)
2183 */
2184 ctrl |= MACB_BF(MSS_MFS, mss_mfs);
2185
2186 /* Set TX buffer descriptor */
2187 macb_set_addr(bp, desc, tx_skb->mapping);
2188 /* desc->addr must be visible to hardware before clearing
2189 * 'TX_USED' bit in desc->ctrl.
2190 */
2191 wmb();
2192 desc->ctrl = ctrl;
2193 } while (i != queue->tx_head);
2194
2195 queue->tx_head = tx_head;
2196
2197 return count;
2198
2199 dma_error:
2200 netdev_err(bp->dev, "TX DMA map failed\n");
2201
2202 for (i = queue->tx_head; i != tx_head; i++) {
2203 tx_skb = macb_tx_skb(queue, i);
2204
2205 macb_tx_unmap(bp, tx_skb, 0);
2206 }
2207
2208 return 0;
2209 }
2210
macb_features_check(struct sk_buff * skb,struct net_device * dev,netdev_features_t features)2211 static netdev_features_t macb_features_check(struct sk_buff *skb,
2212 struct net_device *dev,
2213 netdev_features_t features)
2214 {
2215 unsigned int nr_frags, f;
2216 unsigned int hdrlen;
2217
2218 /* Validate LSO compatibility */
2219
2220 /* there is only one buffer or protocol is not UDP */
2221 if (!skb_is_nonlinear(skb) || (ip_hdr(skb)->protocol != IPPROTO_UDP))
2222 return features;
2223
2224 /* length of header */
2225 hdrlen = skb_transport_offset(skb);
2226
2227 /* For UFO only:
2228 * When software supplies two or more payload buffers all payload buffers
2229 * apart from the last must be a multiple of 8 bytes in size.
2230 */
2231 if (!IS_ALIGNED(skb_headlen(skb) - hdrlen, MACB_TX_LEN_ALIGN))
2232 return features & ~MACB_NETIF_LSO;
2233
2234 nr_frags = skb_shinfo(skb)->nr_frags;
2235 /* No need to check last fragment */
2236 nr_frags--;
2237 for (f = 0; f < nr_frags; f++) {
2238 const skb_frag_t *frag = &skb_shinfo(skb)->frags[f];
2239
2240 if (!IS_ALIGNED(skb_frag_size(frag), MACB_TX_LEN_ALIGN))
2241 return features & ~MACB_NETIF_LSO;
2242 }
2243 return features;
2244 }
2245
macb_clear_csum(struct sk_buff * skb)2246 static inline int macb_clear_csum(struct sk_buff *skb)
2247 {
2248 /* no change for packets without checksum offloading */
2249 if (skb->ip_summed != CHECKSUM_PARTIAL)
2250 return 0;
2251
2252 /* make sure we can modify the header */
2253 if (unlikely(skb_cow_head(skb, 0)))
2254 return -1;
2255
2256 /* initialize checksum field
2257 * This is required - at least for Zynq, which otherwise calculates
2258 * wrong UDP header checksums for UDP packets with UDP data len <=2
2259 */
2260 *(__sum16 *)(skb_checksum_start(skb) + skb->csum_offset) = 0;
2261 return 0;
2262 }
2263
macb_pad_and_fcs(struct sk_buff ** skb,struct net_device * ndev)2264 static int macb_pad_and_fcs(struct sk_buff **skb, struct net_device *ndev)
2265 {
2266 bool cloned = skb_cloned(*skb) || skb_header_cloned(*skb) ||
2267 skb_is_nonlinear(*skb);
2268 int padlen = ETH_ZLEN - (*skb)->len;
2269 int tailroom = skb_tailroom(*skb);
2270 struct sk_buff *nskb;
2271 u32 fcs;
2272
2273 if (!(ndev->features & NETIF_F_HW_CSUM) ||
2274 !((*skb)->ip_summed != CHECKSUM_PARTIAL) ||
2275 skb_shinfo(*skb)->gso_size || ptp_one_step_sync(*skb))
2276 return 0;
2277
2278 if (padlen <= 0) {
2279 /* FCS could be appeded to tailroom. */
2280 if (tailroom >= ETH_FCS_LEN)
2281 goto add_fcs;
2282 /* No room for FCS, need to reallocate skb. */
2283 else
2284 padlen = ETH_FCS_LEN;
2285 } else {
2286 /* Add room for FCS. */
2287 padlen += ETH_FCS_LEN;
2288 }
2289
2290 if (cloned || tailroom < padlen) {
2291 nskb = skb_copy_expand(*skb, 0, padlen, GFP_ATOMIC);
2292 if (!nskb)
2293 return -ENOMEM;
2294
2295 dev_consume_skb_any(*skb);
2296 *skb = nskb;
2297 }
2298
2299 if (padlen > ETH_FCS_LEN)
2300 skb_put_zero(*skb, padlen - ETH_FCS_LEN);
2301
2302 add_fcs:
2303 /* set FCS to packet */
2304 fcs = crc32_le(~0, (*skb)->data, (*skb)->len);
2305 fcs = ~fcs;
2306
2307 skb_put_u8(*skb, fcs & 0xff);
2308 skb_put_u8(*skb, (fcs >> 8) & 0xff);
2309 skb_put_u8(*skb, (fcs >> 16) & 0xff);
2310 skb_put_u8(*skb, (fcs >> 24) & 0xff);
2311
2312 return 0;
2313 }
2314
macb_start_xmit(struct sk_buff * skb,struct net_device * dev)2315 static netdev_tx_t macb_start_xmit(struct sk_buff *skb, struct net_device *dev)
2316 {
2317 u16 queue_index = skb_get_queue_mapping(skb);
2318 struct macb *bp = netdev_priv(dev);
2319 struct macb_queue *queue = &bp->queues[queue_index];
2320 unsigned int desc_cnt, nr_frags, frag_size, f;
2321 unsigned int hdrlen;
2322 bool is_lso;
2323 netdev_tx_t ret = NETDEV_TX_OK;
2324
2325 if (macb_clear_csum(skb)) {
2326 dev_kfree_skb_any(skb);
2327 return ret;
2328 }
2329
2330 if (macb_pad_and_fcs(&skb, dev)) {
2331 dev_kfree_skb_any(skb);
2332 return ret;
2333 }
2334
2335 #ifdef CONFIG_MACB_USE_HWSTAMP
2336 if ((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) &&
2337 (bp->hw_dma_cap & HW_DMA_CAP_PTP))
2338 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
2339 #endif
2340
2341 is_lso = (skb_shinfo(skb)->gso_size != 0);
2342
2343 if (is_lso) {
2344 /* length of headers */
2345 if (ip_hdr(skb)->protocol == IPPROTO_UDP)
2346 /* only queue eth + ip headers separately for UDP */
2347 hdrlen = skb_transport_offset(skb);
2348 else
2349 hdrlen = skb_tcp_all_headers(skb);
2350 if (skb_headlen(skb) < hdrlen) {
2351 netdev_err(bp->dev, "Error - LSO headers fragmented!!!\n");
2352 /* if this is required, would need to copy to single buffer */
2353 return NETDEV_TX_BUSY;
2354 }
2355 } else
2356 hdrlen = min(skb_headlen(skb), bp->max_tx_length);
2357
2358 #if defined(DEBUG) && defined(VERBOSE_DEBUG)
2359 netdev_vdbg(bp->dev,
2360 "start_xmit: queue %hu len %u head %p data %p tail %p end %p\n",
2361 queue_index, skb->len, skb->head, skb->data,
2362 skb_tail_pointer(skb), skb_end_pointer(skb));
2363 print_hex_dump(KERN_DEBUG, "data: ", DUMP_PREFIX_OFFSET, 16, 1,
2364 skb->data, 16, true);
2365 #endif
2366
2367 /* Count how many TX buffer descriptors are needed to send this
2368 * socket buffer: skb fragments of jumbo frames may need to be
2369 * split into many buffer descriptors.
2370 */
2371 if (is_lso && (skb_headlen(skb) > hdrlen))
2372 /* extra header descriptor if also payload in first buffer */
2373 desc_cnt = DIV_ROUND_UP((skb_headlen(skb) - hdrlen), bp->max_tx_length) + 1;
2374 else
2375 desc_cnt = DIV_ROUND_UP(skb_headlen(skb), bp->max_tx_length);
2376 nr_frags = skb_shinfo(skb)->nr_frags;
2377 for (f = 0; f < nr_frags; f++) {
2378 frag_size = skb_frag_size(&skb_shinfo(skb)->frags[f]);
2379 desc_cnt += DIV_ROUND_UP(frag_size, bp->max_tx_length);
2380 }
2381
2382 spin_lock_bh(&queue->tx_ptr_lock);
2383
2384 /* This is a hard error, log it. */
2385 if (CIRC_SPACE(queue->tx_head, queue->tx_tail,
2386 bp->tx_ring_size) < desc_cnt) {
2387 netif_stop_subqueue(dev, queue_index);
2388 netdev_dbg(bp->dev, "tx_head = %u, tx_tail = %u\n",
2389 queue->tx_head, queue->tx_tail);
2390 ret = NETDEV_TX_BUSY;
2391 goto unlock;
2392 }
2393
2394 /* Map socket buffer for DMA transfer */
2395 if (!macb_tx_map(bp, queue, skb, hdrlen)) {
2396 dev_kfree_skb_any(skb);
2397 goto unlock;
2398 }
2399
2400 /* Make newly initialized descriptor visible to hardware */
2401 wmb();
2402 skb_tx_timestamp(skb);
2403
2404 spin_lock_irq(&bp->lock);
2405 macb_writel(bp, NCR, macb_readl(bp, NCR) | MACB_BIT(TSTART));
2406 spin_unlock_irq(&bp->lock);
2407
2408 if (CIRC_SPACE(queue->tx_head, queue->tx_tail, bp->tx_ring_size) < 1)
2409 netif_stop_subqueue(dev, queue_index);
2410
2411 unlock:
2412 spin_unlock_bh(&queue->tx_ptr_lock);
2413
2414 return ret;
2415 }
2416
macb_init_rx_buffer_size(struct macb * bp,size_t size)2417 static void macb_init_rx_buffer_size(struct macb *bp, size_t size)
2418 {
2419 if (!macb_is_gem(bp)) {
2420 bp->rx_buffer_size = MACB_RX_BUFFER_SIZE;
2421 } else {
2422 bp->rx_buffer_size = size;
2423
2424 if (bp->rx_buffer_size % RX_BUFFER_MULTIPLE) {
2425 netdev_dbg(bp->dev,
2426 "RX buffer must be multiple of %d bytes, expanding\n",
2427 RX_BUFFER_MULTIPLE);
2428 bp->rx_buffer_size =
2429 roundup(bp->rx_buffer_size, RX_BUFFER_MULTIPLE);
2430 }
2431 }
2432
2433 netdev_dbg(bp->dev, "mtu [%u] rx_buffer_size [%zu]\n",
2434 bp->dev->mtu, bp->rx_buffer_size);
2435 }
2436
gem_free_rx_buffers(struct macb * bp)2437 static void gem_free_rx_buffers(struct macb *bp)
2438 {
2439 struct sk_buff *skb;
2440 struct macb_dma_desc *desc;
2441 struct macb_queue *queue;
2442 dma_addr_t addr;
2443 unsigned int q;
2444 int i;
2445
2446 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) {
2447 if (!queue->rx_skbuff)
2448 continue;
2449
2450 for (i = 0; i < bp->rx_ring_size; i++) {
2451 skb = queue->rx_skbuff[i];
2452
2453 if (!skb)
2454 continue;
2455
2456 desc = macb_rx_desc(queue, i);
2457 addr = macb_get_addr(bp, desc);
2458
2459 dma_unmap_single(&bp->pdev->dev, addr, bp->rx_buffer_size,
2460 DMA_FROM_DEVICE);
2461 dev_kfree_skb_any(skb);
2462 skb = NULL;
2463 }
2464
2465 kfree(queue->rx_skbuff);
2466 queue->rx_skbuff = NULL;
2467 }
2468 }
2469
macb_free_rx_buffers(struct macb * bp)2470 static void macb_free_rx_buffers(struct macb *bp)
2471 {
2472 struct macb_queue *queue = &bp->queues[0];
2473
2474 if (queue->rx_buffers) {
2475 dma_free_coherent(&bp->pdev->dev,
2476 bp->rx_ring_size * bp->rx_buffer_size,
2477 queue->rx_buffers, queue->rx_buffers_dma);
2478 queue->rx_buffers = NULL;
2479 }
2480 }
2481
macb_free_consistent(struct macb * bp)2482 static void macb_free_consistent(struct macb *bp)
2483 {
2484 struct macb_queue *queue;
2485 unsigned int q;
2486 int size;
2487
2488 if (bp->rx_ring_tieoff) {
2489 dma_free_coherent(&bp->pdev->dev, macb_dma_desc_get_size(bp),
2490 bp->rx_ring_tieoff, bp->rx_ring_tieoff_dma);
2491 bp->rx_ring_tieoff = NULL;
2492 }
2493
2494 bp->macbgem_ops.mog_free_rx_buffers(bp);
2495
2496 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) {
2497 kfree(queue->tx_skb);
2498 queue->tx_skb = NULL;
2499 if (queue->tx_ring) {
2500 size = TX_RING_BYTES(bp) + bp->tx_bd_rd_prefetch;
2501 dma_free_coherent(&bp->pdev->dev, size,
2502 queue->tx_ring, queue->tx_ring_dma);
2503 queue->tx_ring = NULL;
2504 }
2505 if (queue->rx_ring) {
2506 size = RX_RING_BYTES(bp) + bp->rx_bd_rd_prefetch;
2507 dma_free_coherent(&bp->pdev->dev, size,
2508 queue->rx_ring, queue->rx_ring_dma);
2509 queue->rx_ring = NULL;
2510 }
2511 }
2512 }
2513
gem_alloc_rx_buffers(struct macb * bp)2514 static int gem_alloc_rx_buffers(struct macb *bp)
2515 {
2516 struct macb_queue *queue;
2517 unsigned int q;
2518 int size;
2519
2520 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) {
2521 size = bp->rx_ring_size * sizeof(struct sk_buff *);
2522 queue->rx_skbuff = kzalloc(size, GFP_KERNEL);
2523 if (!queue->rx_skbuff)
2524 return -ENOMEM;
2525 else
2526 netdev_dbg(bp->dev,
2527 "Allocated %d RX struct sk_buff entries at %p\n",
2528 bp->rx_ring_size, queue->rx_skbuff);
2529 }
2530 return 0;
2531 }
2532
macb_alloc_rx_buffers(struct macb * bp)2533 static int macb_alloc_rx_buffers(struct macb *bp)
2534 {
2535 struct macb_queue *queue = &bp->queues[0];
2536 int size;
2537
2538 size = bp->rx_ring_size * bp->rx_buffer_size;
2539 queue->rx_buffers = dma_alloc_coherent(&bp->pdev->dev, size,
2540 &queue->rx_buffers_dma, GFP_KERNEL);
2541 if (!queue->rx_buffers)
2542 return -ENOMEM;
2543
2544 netdev_dbg(bp->dev,
2545 "Allocated RX buffers of %d bytes at %08lx (mapped %p)\n",
2546 size, (unsigned long)queue->rx_buffers_dma, queue->rx_buffers);
2547 return 0;
2548 }
2549
macb_alloc_consistent(struct macb * bp)2550 static int macb_alloc_consistent(struct macb *bp)
2551 {
2552 struct macb_queue *queue;
2553 unsigned int q;
2554 int size;
2555
2556 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) {
2557 size = TX_RING_BYTES(bp) + bp->tx_bd_rd_prefetch;
2558 queue->tx_ring = dma_alloc_coherent(&bp->pdev->dev, size,
2559 &queue->tx_ring_dma,
2560 GFP_KERNEL);
2561 if (!queue->tx_ring)
2562 goto out_err;
2563 netdev_dbg(bp->dev,
2564 "Allocated TX ring for queue %u of %d bytes at %08lx (mapped %p)\n",
2565 q, size, (unsigned long)queue->tx_ring_dma,
2566 queue->tx_ring);
2567
2568 size = bp->tx_ring_size * sizeof(struct macb_tx_skb);
2569 queue->tx_skb = kmalloc(size, GFP_KERNEL);
2570 if (!queue->tx_skb)
2571 goto out_err;
2572
2573 size = RX_RING_BYTES(bp) + bp->rx_bd_rd_prefetch;
2574 queue->rx_ring = dma_alloc_coherent(&bp->pdev->dev, size,
2575 &queue->rx_ring_dma, GFP_KERNEL);
2576 if (!queue->rx_ring)
2577 goto out_err;
2578 netdev_dbg(bp->dev,
2579 "Allocated RX ring of %d bytes at %08lx (mapped %p)\n",
2580 size, (unsigned long)queue->rx_ring_dma, queue->rx_ring);
2581 }
2582 if (bp->macbgem_ops.mog_alloc_rx_buffers(bp))
2583 goto out_err;
2584
2585 /* Required for tie off descriptor for PM cases */
2586 if (!(bp->caps & MACB_CAPS_QUEUE_DISABLE)) {
2587 bp->rx_ring_tieoff = dma_alloc_coherent(&bp->pdev->dev,
2588 macb_dma_desc_get_size(bp),
2589 &bp->rx_ring_tieoff_dma,
2590 GFP_KERNEL);
2591 if (!bp->rx_ring_tieoff)
2592 goto out_err;
2593 }
2594
2595 return 0;
2596
2597 out_err:
2598 macb_free_consistent(bp);
2599 return -ENOMEM;
2600 }
2601
macb_init_tieoff(struct macb * bp)2602 static void macb_init_tieoff(struct macb *bp)
2603 {
2604 struct macb_dma_desc *desc = bp->rx_ring_tieoff;
2605
2606 if (bp->caps & MACB_CAPS_QUEUE_DISABLE)
2607 return;
2608 /* Setup a wrapping descriptor with no free slots
2609 * (WRAP and USED) to tie off/disable unused RX queues.
2610 */
2611 macb_set_addr(bp, desc, MACB_BIT(RX_WRAP) | MACB_BIT(RX_USED));
2612 desc->ctrl = 0;
2613 }
2614
gem_init_rings(struct macb * bp)2615 static void gem_init_rings(struct macb *bp)
2616 {
2617 struct macb_queue *queue;
2618 struct macb_dma_desc *desc = NULL;
2619 unsigned int q;
2620 int i;
2621
2622 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) {
2623 for (i = 0; i < bp->tx_ring_size; i++) {
2624 desc = macb_tx_desc(queue, i);
2625 macb_set_addr(bp, desc, 0);
2626 desc->ctrl = MACB_BIT(TX_USED);
2627 }
2628 desc->ctrl |= MACB_BIT(TX_WRAP);
2629 queue->tx_head = 0;
2630 queue->tx_tail = 0;
2631
2632 queue->rx_tail = 0;
2633 queue->rx_prepared_head = 0;
2634
2635 gem_rx_refill(queue);
2636 }
2637
2638 macb_init_tieoff(bp);
2639 }
2640
macb_init_rings(struct macb * bp)2641 static void macb_init_rings(struct macb *bp)
2642 {
2643 int i;
2644 struct macb_dma_desc *desc = NULL;
2645
2646 macb_init_rx_ring(&bp->queues[0]);
2647
2648 for (i = 0; i < bp->tx_ring_size; i++) {
2649 desc = macb_tx_desc(&bp->queues[0], i);
2650 macb_set_addr(bp, desc, 0);
2651 desc->ctrl = MACB_BIT(TX_USED);
2652 }
2653 bp->queues[0].tx_head = 0;
2654 bp->queues[0].tx_tail = 0;
2655 desc->ctrl |= MACB_BIT(TX_WRAP);
2656
2657 macb_init_tieoff(bp);
2658 }
2659
macb_reset_hw(struct macb * bp)2660 static void macb_reset_hw(struct macb *bp)
2661 {
2662 struct macb_queue *queue;
2663 unsigned int q;
2664 u32 ctrl = macb_readl(bp, NCR);
2665
2666 /* Disable RX and TX (XXX: Should we halt the transmission
2667 * more gracefully?)
2668 */
2669 ctrl &= ~(MACB_BIT(RE) | MACB_BIT(TE));
2670
2671 /* Clear the stats registers (XXX: Update stats first?) */
2672 ctrl |= MACB_BIT(CLRSTAT);
2673
2674 macb_writel(bp, NCR, ctrl);
2675
2676 /* Clear all status flags */
2677 macb_writel(bp, TSR, -1);
2678 macb_writel(bp, RSR, -1);
2679
2680 /* Disable RX partial store and forward and reset watermark value */
2681 gem_writel(bp, PBUFRXCUT, 0);
2682
2683 /* Disable all interrupts */
2684 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) {
2685 queue_writel(queue, IDR, -1);
2686 queue_readl(queue, ISR);
2687 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE)
2688 queue_writel(queue, ISR, -1);
2689 }
2690 }
2691
gem_mdc_clk_div(struct macb * bp)2692 static u32 gem_mdc_clk_div(struct macb *bp)
2693 {
2694 u32 config;
2695 unsigned long pclk_hz = clk_get_rate(bp->pclk);
2696
2697 if (pclk_hz <= 20000000)
2698 config = GEM_BF(CLK, GEM_CLK_DIV8);
2699 else if (pclk_hz <= 40000000)
2700 config = GEM_BF(CLK, GEM_CLK_DIV16);
2701 else if (pclk_hz <= 80000000)
2702 config = GEM_BF(CLK, GEM_CLK_DIV32);
2703 else if (pclk_hz <= 120000000)
2704 config = GEM_BF(CLK, GEM_CLK_DIV48);
2705 else if (pclk_hz <= 160000000)
2706 config = GEM_BF(CLK, GEM_CLK_DIV64);
2707 else if (pclk_hz <= 240000000)
2708 config = GEM_BF(CLK, GEM_CLK_DIV96);
2709 else if (pclk_hz <= 320000000)
2710 config = GEM_BF(CLK, GEM_CLK_DIV128);
2711 else
2712 config = GEM_BF(CLK, GEM_CLK_DIV224);
2713
2714 return config;
2715 }
2716
macb_mdc_clk_div(struct macb * bp)2717 static u32 macb_mdc_clk_div(struct macb *bp)
2718 {
2719 u32 config;
2720 unsigned long pclk_hz;
2721
2722 if (macb_is_gem(bp))
2723 return gem_mdc_clk_div(bp);
2724
2725 pclk_hz = clk_get_rate(bp->pclk);
2726 if (pclk_hz <= 20000000)
2727 config = MACB_BF(CLK, MACB_CLK_DIV8);
2728 else if (pclk_hz <= 40000000)
2729 config = MACB_BF(CLK, MACB_CLK_DIV16);
2730 else if (pclk_hz <= 80000000)
2731 config = MACB_BF(CLK, MACB_CLK_DIV32);
2732 else
2733 config = MACB_BF(CLK, MACB_CLK_DIV64);
2734
2735 return config;
2736 }
2737
2738 /* Get the DMA bus width field of the network configuration register that we
2739 * should program. We find the width from decoding the design configuration
2740 * register to find the maximum supported data bus width.
2741 */
macb_dbw(struct macb * bp)2742 static u32 macb_dbw(struct macb *bp)
2743 {
2744 if (!macb_is_gem(bp))
2745 return 0;
2746
2747 switch (GEM_BFEXT(DBWDEF, gem_readl(bp, DCFG1))) {
2748 case 4:
2749 return GEM_BF(DBW, GEM_DBW128);
2750 case 2:
2751 return GEM_BF(DBW, GEM_DBW64);
2752 case 1:
2753 default:
2754 return GEM_BF(DBW, GEM_DBW32);
2755 }
2756 }
2757
2758 /* Configure the receive DMA engine
2759 * - use the correct receive buffer size
2760 * - set best burst length for DMA operations
2761 * (if not supported by FIFO, it will fallback to default)
2762 * - set both rx/tx packet buffers to full memory size
2763 * These are configurable parameters for GEM.
2764 */
macb_configure_dma(struct macb * bp)2765 static void macb_configure_dma(struct macb *bp)
2766 {
2767 struct macb_queue *queue;
2768 u32 buffer_size;
2769 unsigned int q;
2770 u32 dmacfg;
2771
2772 buffer_size = bp->rx_buffer_size / RX_BUFFER_MULTIPLE;
2773 if (macb_is_gem(bp)) {
2774 dmacfg = gem_readl(bp, DMACFG) & ~GEM_BF(RXBS, -1L);
2775 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) {
2776 if (q)
2777 queue_writel(queue, RBQS, buffer_size);
2778 else
2779 dmacfg |= GEM_BF(RXBS, buffer_size);
2780 }
2781 if (bp->dma_burst_length)
2782 dmacfg = GEM_BFINS(FBLDO, bp->dma_burst_length, dmacfg);
2783 dmacfg |= GEM_BIT(TXPBMS) | GEM_BF(RXBMS, -1L);
2784 dmacfg &= ~GEM_BIT(ENDIA_PKT);
2785
2786 if (bp->native_io)
2787 dmacfg &= ~GEM_BIT(ENDIA_DESC);
2788 else
2789 dmacfg |= GEM_BIT(ENDIA_DESC); /* CPU in big endian */
2790
2791 if (bp->dev->features & NETIF_F_HW_CSUM)
2792 dmacfg |= GEM_BIT(TXCOEN);
2793 else
2794 dmacfg &= ~GEM_BIT(TXCOEN);
2795
2796 dmacfg &= ~GEM_BIT(ADDR64);
2797 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT
2798 if (bp->hw_dma_cap & HW_DMA_CAP_64B)
2799 dmacfg |= GEM_BIT(ADDR64);
2800 #endif
2801 #ifdef CONFIG_MACB_USE_HWSTAMP
2802 if (bp->hw_dma_cap & HW_DMA_CAP_PTP)
2803 dmacfg |= GEM_BIT(RXEXT) | GEM_BIT(TXEXT);
2804 #endif
2805 netdev_dbg(bp->dev, "Cadence configure DMA with 0x%08x\n",
2806 dmacfg);
2807 gem_writel(bp, DMACFG, dmacfg);
2808 }
2809 }
2810
macb_init_hw(struct macb * bp)2811 static void macb_init_hw(struct macb *bp)
2812 {
2813 u32 config;
2814
2815 macb_reset_hw(bp);
2816 macb_set_hwaddr(bp);
2817
2818 config = macb_mdc_clk_div(bp);
2819 config |= MACB_BF(RBOF, NET_IP_ALIGN); /* Make eth data aligned */
2820 config |= MACB_BIT(DRFCS); /* Discard Rx FCS */
2821 if (bp->caps & MACB_CAPS_JUMBO)
2822 config |= MACB_BIT(JFRAME); /* Enable jumbo frames */
2823 else
2824 config |= MACB_BIT(BIG); /* Receive oversized frames */
2825 if (bp->dev->flags & IFF_PROMISC)
2826 config |= MACB_BIT(CAF); /* Copy All Frames */
2827 else if (macb_is_gem(bp) && bp->dev->features & NETIF_F_RXCSUM)
2828 config |= GEM_BIT(RXCOEN);
2829 if (!(bp->dev->flags & IFF_BROADCAST))
2830 config |= MACB_BIT(NBC); /* No BroadCast */
2831 config |= macb_dbw(bp);
2832 macb_writel(bp, NCFGR, config);
2833 if ((bp->caps & MACB_CAPS_JUMBO) && bp->jumbo_max_len)
2834 gem_writel(bp, JML, bp->jumbo_max_len);
2835 bp->rx_frm_len_mask = MACB_RX_FRMLEN_MASK;
2836 if (bp->caps & MACB_CAPS_JUMBO)
2837 bp->rx_frm_len_mask = MACB_RX_JFRMLEN_MASK;
2838
2839 macb_configure_dma(bp);
2840
2841 /* Enable RX partial store and forward and set watermark */
2842 if (bp->rx_watermark)
2843 gem_writel(bp, PBUFRXCUT, (bp->rx_watermark | GEM_BIT(ENCUTTHRU)));
2844 }
2845
2846 /* The hash address register is 64 bits long and takes up two
2847 * locations in the memory map. The least significant bits are stored
2848 * in EMAC_HSL and the most significant bits in EMAC_HSH.
2849 *
2850 * The unicast hash enable and the multicast hash enable bits in the
2851 * network configuration register enable the reception of hash matched
2852 * frames. The destination address is reduced to a 6 bit index into
2853 * the 64 bit hash register using the following hash function. The
2854 * hash function is an exclusive or of every sixth bit of the
2855 * destination address.
2856 *
2857 * hi[5] = da[5] ^ da[11] ^ da[17] ^ da[23] ^ da[29] ^ da[35] ^ da[41] ^ da[47]
2858 * hi[4] = da[4] ^ da[10] ^ da[16] ^ da[22] ^ da[28] ^ da[34] ^ da[40] ^ da[46]
2859 * hi[3] = da[3] ^ da[09] ^ da[15] ^ da[21] ^ da[27] ^ da[33] ^ da[39] ^ da[45]
2860 * hi[2] = da[2] ^ da[08] ^ da[14] ^ da[20] ^ da[26] ^ da[32] ^ da[38] ^ da[44]
2861 * hi[1] = da[1] ^ da[07] ^ da[13] ^ da[19] ^ da[25] ^ da[31] ^ da[37] ^ da[43]
2862 * hi[0] = da[0] ^ da[06] ^ da[12] ^ da[18] ^ da[24] ^ da[30] ^ da[36] ^ da[42]
2863 *
2864 * da[0] represents the least significant bit of the first byte
2865 * received, that is, the multicast/unicast indicator, and da[47]
2866 * represents the most significant bit of the last byte received. If
2867 * the hash index, hi[n], points to a bit that is set in the hash
2868 * register then the frame will be matched according to whether the
2869 * frame is multicast or unicast. A multicast match will be signalled
2870 * if the multicast hash enable bit is set, da[0] is 1 and the hash
2871 * index points to a bit set in the hash register. A unicast match
2872 * will be signalled if the unicast hash enable bit is set, da[0] is 0
2873 * and the hash index points to a bit set in the hash register. To
2874 * receive all multicast frames, the hash register should be set with
2875 * all ones and the multicast hash enable bit should be set in the
2876 * network configuration register.
2877 */
2878
hash_bit_value(int bitnr,__u8 * addr)2879 static inline int hash_bit_value(int bitnr, __u8 *addr)
2880 {
2881 if (addr[bitnr / 8] & (1 << (bitnr % 8)))
2882 return 1;
2883 return 0;
2884 }
2885
2886 /* Return the hash index value for the specified address. */
hash_get_index(__u8 * addr)2887 static int hash_get_index(__u8 *addr)
2888 {
2889 int i, j, bitval;
2890 int hash_index = 0;
2891
2892 for (j = 0; j < 6; j++) {
2893 for (i = 0, bitval = 0; i < 8; i++)
2894 bitval ^= hash_bit_value(i * 6 + j, addr);
2895
2896 hash_index |= (bitval << j);
2897 }
2898
2899 return hash_index;
2900 }
2901
2902 /* Add multicast addresses to the internal multicast-hash table. */
macb_sethashtable(struct net_device * dev)2903 static void macb_sethashtable(struct net_device *dev)
2904 {
2905 struct netdev_hw_addr *ha;
2906 unsigned long mc_filter[2];
2907 unsigned int bitnr;
2908 struct macb *bp = netdev_priv(dev);
2909
2910 mc_filter[0] = 0;
2911 mc_filter[1] = 0;
2912
2913 netdev_for_each_mc_addr(ha, dev) {
2914 bitnr = hash_get_index(ha->addr);
2915 mc_filter[bitnr >> 5] |= 1 << (bitnr & 31);
2916 }
2917
2918 macb_or_gem_writel(bp, HRB, mc_filter[0]);
2919 macb_or_gem_writel(bp, HRT, mc_filter[1]);
2920 }
2921
2922 /* Enable/Disable promiscuous and multicast modes. */
macb_set_rx_mode(struct net_device * dev)2923 static void macb_set_rx_mode(struct net_device *dev)
2924 {
2925 unsigned long cfg;
2926 struct macb *bp = netdev_priv(dev);
2927
2928 cfg = macb_readl(bp, NCFGR);
2929
2930 if (dev->flags & IFF_PROMISC) {
2931 /* Enable promiscuous mode */
2932 cfg |= MACB_BIT(CAF);
2933
2934 /* Disable RX checksum offload */
2935 if (macb_is_gem(bp))
2936 cfg &= ~GEM_BIT(RXCOEN);
2937 } else {
2938 /* Disable promiscuous mode */
2939 cfg &= ~MACB_BIT(CAF);
2940
2941 /* Enable RX checksum offload only if requested */
2942 if (macb_is_gem(bp) && dev->features & NETIF_F_RXCSUM)
2943 cfg |= GEM_BIT(RXCOEN);
2944 }
2945
2946 if (dev->flags & IFF_ALLMULTI) {
2947 /* Enable all multicast mode */
2948 macb_or_gem_writel(bp, HRB, -1);
2949 macb_or_gem_writel(bp, HRT, -1);
2950 cfg |= MACB_BIT(NCFGR_MTI);
2951 } else if (!netdev_mc_empty(dev)) {
2952 /* Enable specific multicasts */
2953 macb_sethashtable(dev);
2954 cfg |= MACB_BIT(NCFGR_MTI);
2955 } else if (dev->flags & (~IFF_ALLMULTI)) {
2956 /* Disable all multicast mode */
2957 macb_or_gem_writel(bp, HRB, 0);
2958 macb_or_gem_writel(bp, HRT, 0);
2959 cfg &= ~MACB_BIT(NCFGR_MTI);
2960 }
2961
2962 macb_writel(bp, NCFGR, cfg);
2963 }
2964
macb_open(struct net_device * dev)2965 static int macb_open(struct net_device *dev)
2966 {
2967 size_t bufsz = dev->mtu + ETH_HLEN + ETH_FCS_LEN + NET_IP_ALIGN;
2968 struct macb *bp = netdev_priv(dev);
2969 struct macb_queue *queue;
2970 unsigned int q;
2971 int err;
2972
2973 netdev_dbg(bp->dev, "open\n");
2974
2975 err = pm_runtime_resume_and_get(&bp->pdev->dev);
2976 if (err < 0)
2977 return err;
2978
2979 /* RX buffers initialization */
2980 macb_init_rx_buffer_size(bp, bufsz);
2981
2982 err = macb_alloc_consistent(bp);
2983 if (err) {
2984 netdev_err(dev, "Unable to allocate DMA memory (error %d)\n",
2985 err);
2986 goto pm_exit;
2987 }
2988
2989 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) {
2990 napi_enable(&queue->napi_rx);
2991 napi_enable(&queue->napi_tx);
2992 }
2993
2994 macb_init_hw(bp);
2995
2996 err = phy_power_on(bp->sgmii_phy);
2997 if (err)
2998 goto reset_hw;
2999
3000 err = macb_phylink_connect(bp);
3001 if (err)
3002 goto phy_off;
3003
3004 netif_tx_start_all_queues(dev);
3005
3006 if (bp->ptp_info)
3007 bp->ptp_info->ptp_init(dev);
3008
3009 return 0;
3010
3011 phy_off:
3012 phy_power_off(bp->sgmii_phy);
3013
3014 reset_hw:
3015 macb_reset_hw(bp);
3016 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) {
3017 napi_disable(&queue->napi_rx);
3018 napi_disable(&queue->napi_tx);
3019 }
3020 macb_free_consistent(bp);
3021 pm_exit:
3022 pm_runtime_put_sync(&bp->pdev->dev);
3023 return err;
3024 }
3025
macb_close(struct net_device * dev)3026 static int macb_close(struct net_device *dev)
3027 {
3028 struct macb *bp = netdev_priv(dev);
3029 struct macb_queue *queue;
3030 unsigned long flags;
3031 unsigned int q;
3032
3033 netif_tx_stop_all_queues(dev);
3034
3035 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) {
3036 napi_disable(&queue->napi_rx);
3037 napi_disable(&queue->napi_tx);
3038 }
3039
3040 phylink_stop(bp->phylink);
3041 phylink_disconnect_phy(bp->phylink);
3042
3043 phy_power_off(bp->sgmii_phy);
3044
3045 spin_lock_irqsave(&bp->lock, flags);
3046 macb_reset_hw(bp);
3047 netif_carrier_off(dev);
3048 spin_unlock_irqrestore(&bp->lock, flags);
3049
3050 macb_free_consistent(bp);
3051
3052 if (bp->ptp_info)
3053 bp->ptp_info->ptp_remove(dev);
3054
3055 pm_runtime_put(&bp->pdev->dev);
3056
3057 return 0;
3058 }
3059
macb_change_mtu(struct net_device * dev,int new_mtu)3060 static int macb_change_mtu(struct net_device *dev, int new_mtu)
3061 {
3062 if (netif_running(dev))
3063 return -EBUSY;
3064
3065 WRITE_ONCE(dev->mtu, new_mtu);
3066
3067 return 0;
3068 }
3069
macb_set_mac_addr(struct net_device * dev,void * addr)3070 static int macb_set_mac_addr(struct net_device *dev, void *addr)
3071 {
3072 int err;
3073
3074 err = eth_mac_addr(dev, addr);
3075 if (err < 0)
3076 return err;
3077
3078 macb_set_hwaddr(netdev_priv(dev));
3079 return 0;
3080 }
3081
gem_update_stats(struct macb * bp)3082 static void gem_update_stats(struct macb *bp)
3083 {
3084 struct macb_queue *queue;
3085 unsigned int i, q, idx;
3086 unsigned long *stat;
3087
3088 u32 *p = &bp->hw_stats.gem.tx_octets_31_0;
3089
3090 for (i = 0; i < GEM_STATS_LEN; ++i, ++p) {
3091 u32 offset = gem_statistics[i].offset;
3092 u64 val = bp->macb_reg_readl(bp, offset);
3093
3094 bp->ethtool_stats[i] += val;
3095 *p += val;
3096
3097 if (offset == GEM_OCTTXL || offset == GEM_OCTRXL) {
3098 /* Add GEM_OCTTXH, GEM_OCTRXH */
3099 val = bp->macb_reg_readl(bp, offset + 4);
3100 bp->ethtool_stats[i] += ((u64)val) << 32;
3101 *(++p) += val;
3102 }
3103 }
3104
3105 idx = GEM_STATS_LEN;
3106 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue)
3107 for (i = 0, stat = &queue->stats.first; i < QUEUE_STATS_LEN; ++i, ++stat)
3108 bp->ethtool_stats[idx++] = *stat;
3109 }
3110
gem_get_stats(struct macb * bp)3111 static struct net_device_stats *gem_get_stats(struct macb *bp)
3112 {
3113 struct gem_stats *hwstat = &bp->hw_stats.gem;
3114 struct net_device_stats *nstat = &bp->dev->stats;
3115
3116 if (!netif_running(bp->dev))
3117 return nstat;
3118
3119 gem_update_stats(bp);
3120
3121 nstat->rx_errors = (hwstat->rx_frame_check_sequence_errors +
3122 hwstat->rx_alignment_errors +
3123 hwstat->rx_resource_errors +
3124 hwstat->rx_overruns +
3125 hwstat->rx_oversize_frames +
3126 hwstat->rx_jabbers +
3127 hwstat->rx_undersized_frames +
3128 hwstat->rx_length_field_frame_errors);
3129 nstat->tx_errors = (hwstat->tx_late_collisions +
3130 hwstat->tx_excessive_collisions +
3131 hwstat->tx_underrun +
3132 hwstat->tx_carrier_sense_errors);
3133 nstat->multicast = hwstat->rx_multicast_frames;
3134 nstat->collisions = (hwstat->tx_single_collision_frames +
3135 hwstat->tx_multiple_collision_frames +
3136 hwstat->tx_excessive_collisions);
3137 nstat->rx_length_errors = (hwstat->rx_oversize_frames +
3138 hwstat->rx_jabbers +
3139 hwstat->rx_undersized_frames +
3140 hwstat->rx_length_field_frame_errors);
3141 nstat->rx_over_errors = hwstat->rx_resource_errors;
3142 nstat->rx_crc_errors = hwstat->rx_frame_check_sequence_errors;
3143 nstat->rx_frame_errors = hwstat->rx_alignment_errors;
3144 nstat->rx_fifo_errors = hwstat->rx_overruns;
3145 nstat->tx_aborted_errors = hwstat->tx_excessive_collisions;
3146 nstat->tx_carrier_errors = hwstat->tx_carrier_sense_errors;
3147 nstat->tx_fifo_errors = hwstat->tx_underrun;
3148
3149 return nstat;
3150 }
3151
gem_get_ethtool_stats(struct net_device * dev,struct ethtool_stats * stats,u64 * data)3152 static void gem_get_ethtool_stats(struct net_device *dev,
3153 struct ethtool_stats *stats, u64 *data)
3154 {
3155 struct macb *bp;
3156
3157 bp = netdev_priv(dev);
3158 gem_update_stats(bp);
3159 memcpy(data, &bp->ethtool_stats, sizeof(u64)
3160 * (GEM_STATS_LEN + QUEUE_STATS_LEN * MACB_MAX_QUEUES));
3161 }
3162
gem_get_sset_count(struct net_device * dev,int sset)3163 static int gem_get_sset_count(struct net_device *dev, int sset)
3164 {
3165 struct macb *bp = netdev_priv(dev);
3166
3167 switch (sset) {
3168 case ETH_SS_STATS:
3169 return GEM_STATS_LEN + bp->num_queues * QUEUE_STATS_LEN;
3170 default:
3171 return -EOPNOTSUPP;
3172 }
3173 }
3174
gem_get_ethtool_strings(struct net_device * dev,u32 sset,u8 * p)3175 static void gem_get_ethtool_strings(struct net_device *dev, u32 sset, u8 *p)
3176 {
3177 char stat_string[ETH_GSTRING_LEN];
3178 struct macb *bp = netdev_priv(dev);
3179 struct macb_queue *queue;
3180 unsigned int i;
3181 unsigned int q;
3182
3183 switch (sset) {
3184 case ETH_SS_STATS:
3185 for (i = 0; i < GEM_STATS_LEN; i++, p += ETH_GSTRING_LEN)
3186 memcpy(p, gem_statistics[i].stat_string,
3187 ETH_GSTRING_LEN);
3188
3189 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) {
3190 for (i = 0; i < QUEUE_STATS_LEN; i++, p += ETH_GSTRING_LEN) {
3191 snprintf(stat_string, ETH_GSTRING_LEN, "q%d_%s",
3192 q, queue_statistics[i].stat_string);
3193 memcpy(p, stat_string, ETH_GSTRING_LEN);
3194 }
3195 }
3196 break;
3197 }
3198 }
3199
macb_get_stats(struct net_device * dev)3200 static struct net_device_stats *macb_get_stats(struct net_device *dev)
3201 {
3202 struct macb *bp = netdev_priv(dev);
3203 struct net_device_stats *nstat = &bp->dev->stats;
3204 struct macb_stats *hwstat = &bp->hw_stats.macb;
3205
3206 if (macb_is_gem(bp))
3207 return gem_get_stats(bp);
3208
3209 /* read stats from hardware */
3210 macb_update_stats(bp);
3211
3212 /* Convert HW stats into netdevice stats */
3213 nstat->rx_errors = (hwstat->rx_fcs_errors +
3214 hwstat->rx_align_errors +
3215 hwstat->rx_resource_errors +
3216 hwstat->rx_overruns +
3217 hwstat->rx_oversize_pkts +
3218 hwstat->rx_jabbers +
3219 hwstat->rx_undersize_pkts +
3220 hwstat->rx_length_mismatch);
3221 nstat->tx_errors = (hwstat->tx_late_cols +
3222 hwstat->tx_excessive_cols +
3223 hwstat->tx_underruns +
3224 hwstat->tx_carrier_errors +
3225 hwstat->sqe_test_errors);
3226 nstat->collisions = (hwstat->tx_single_cols +
3227 hwstat->tx_multiple_cols +
3228 hwstat->tx_excessive_cols);
3229 nstat->rx_length_errors = (hwstat->rx_oversize_pkts +
3230 hwstat->rx_jabbers +
3231 hwstat->rx_undersize_pkts +
3232 hwstat->rx_length_mismatch);
3233 nstat->rx_over_errors = hwstat->rx_resource_errors +
3234 hwstat->rx_overruns;
3235 nstat->rx_crc_errors = hwstat->rx_fcs_errors;
3236 nstat->rx_frame_errors = hwstat->rx_align_errors;
3237 nstat->rx_fifo_errors = hwstat->rx_overruns;
3238 /* XXX: What does "missed" mean? */
3239 nstat->tx_aborted_errors = hwstat->tx_excessive_cols;
3240 nstat->tx_carrier_errors = hwstat->tx_carrier_errors;
3241 nstat->tx_fifo_errors = hwstat->tx_underruns;
3242 /* Don't know about heartbeat or window errors... */
3243
3244 return nstat;
3245 }
3246
macb_get_regs_len(struct net_device * netdev)3247 static int macb_get_regs_len(struct net_device *netdev)
3248 {
3249 return MACB_GREGS_NBR * sizeof(u32);
3250 }
3251
macb_get_regs(struct net_device * dev,struct ethtool_regs * regs,void * p)3252 static void macb_get_regs(struct net_device *dev, struct ethtool_regs *regs,
3253 void *p)
3254 {
3255 struct macb *bp = netdev_priv(dev);
3256 unsigned int tail, head;
3257 u32 *regs_buff = p;
3258
3259 regs->version = (macb_readl(bp, MID) & ((1 << MACB_REV_SIZE) - 1))
3260 | MACB_GREGS_VERSION;
3261
3262 tail = macb_tx_ring_wrap(bp, bp->queues[0].tx_tail);
3263 head = macb_tx_ring_wrap(bp, bp->queues[0].tx_head);
3264
3265 regs_buff[0] = macb_readl(bp, NCR);
3266 regs_buff[1] = macb_or_gem_readl(bp, NCFGR);
3267 regs_buff[2] = macb_readl(bp, NSR);
3268 regs_buff[3] = macb_readl(bp, TSR);
3269 regs_buff[4] = macb_readl(bp, RBQP);
3270 regs_buff[5] = macb_readl(bp, TBQP);
3271 regs_buff[6] = macb_readl(bp, RSR);
3272 regs_buff[7] = macb_readl(bp, IMR);
3273
3274 regs_buff[8] = tail;
3275 regs_buff[9] = head;
3276 regs_buff[10] = macb_tx_dma(&bp->queues[0], tail);
3277 regs_buff[11] = macb_tx_dma(&bp->queues[0], head);
3278
3279 if (!(bp->caps & MACB_CAPS_USRIO_DISABLED))
3280 regs_buff[12] = macb_or_gem_readl(bp, USRIO);
3281 if (macb_is_gem(bp))
3282 regs_buff[13] = gem_readl(bp, DMACFG);
3283 }
3284
macb_get_wol(struct net_device * netdev,struct ethtool_wolinfo * wol)3285 static void macb_get_wol(struct net_device *netdev, struct ethtool_wolinfo *wol)
3286 {
3287 struct macb *bp = netdev_priv(netdev);
3288
3289 phylink_ethtool_get_wol(bp->phylink, wol);
3290 wol->supported |= (WAKE_MAGIC | WAKE_ARP);
3291
3292 /* Add macb wolopts to phy wolopts */
3293 wol->wolopts |= bp->wolopts;
3294 }
3295
macb_set_wol(struct net_device * netdev,struct ethtool_wolinfo * wol)3296 static int macb_set_wol(struct net_device *netdev, struct ethtool_wolinfo *wol)
3297 {
3298 struct macb *bp = netdev_priv(netdev);
3299 int ret;
3300
3301 /* Pass the order to phylink layer */
3302 ret = phylink_ethtool_set_wol(bp->phylink, wol);
3303 /* Don't manage WoL on MAC, if PHY set_wol() fails */
3304 if (ret && ret != -EOPNOTSUPP)
3305 return ret;
3306
3307 bp->wolopts = (wol->wolopts & WAKE_MAGIC) ? WAKE_MAGIC : 0;
3308 bp->wolopts |= (wol->wolopts & WAKE_ARP) ? WAKE_ARP : 0;
3309 bp->wol = (wol->wolopts) ? MACB_WOL_ENABLED : 0;
3310
3311 device_set_wakeup_enable(&bp->pdev->dev, bp->wol);
3312
3313 return 0;
3314 }
3315
macb_get_link_ksettings(struct net_device * netdev,struct ethtool_link_ksettings * kset)3316 static int macb_get_link_ksettings(struct net_device *netdev,
3317 struct ethtool_link_ksettings *kset)
3318 {
3319 struct macb *bp = netdev_priv(netdev);
3320
3321 return phylink_ethtool_ksettings_get(bp->phylink, kset);
3322 }
3323
macb_set_link_ksettings(struct net_device * netdev,const struct ethtool_link_ksettings * kset)3324 static int macb_set_link_ksettings(struct net_device *netdev,
3325 const struct ethtool_link_ksettings *kset)
3326 {
3327 struct macb *bp = netdev_priv(netdev);
3328
3329 return phylink_ethtool_ksettings_set(bp->phylink, kset);
3330 }
3331
macb_get_ringparam(struct net_device * netdev,struct ethtool_ringparam * ring,struct kernel_ethtool_ringparam * kernel_ring,struct netlink_ext_ack * extack)3332 static void macb_get_ringparam(struct net_device *netdev,
3333 struct ethtool_ringparam *ring,
3334 struct kernel_ethtool_ringparam *kernel_ring,
3335 struct netlink_ext_ack *extack)
3336 {
3337 struct macb *bp = netdev_priv(netdev);
3338
3339 ring->rx_max_pending = MAX_RX_RING_SIZE;
3340 ring->tx_max_pending = MAX_TX_RING_SIZE;
3341
3342 ring->rx_pending = bp->rx_ring_size;
3343 ring->tx_pending = bp->tx_ring_size;
3344 }
3345
macb_set_ringparam(struct net_device * netdev,struct ethtool_ringparam * ring,struct kernel_ethtool_ringparam * kernel_ring,struct netlink_ext_ack * extack)3346 static int macb_set_ringparam(struct net_device *netdev,
3347 struct ethtool_ringparam *ring,
3348 struct kernel_ethtool_ringparam *kernel_ring,
3349 struct netlink_ext_ack *extack)
3350 {
3351 struct macb *bp = netdev_priv(netdev);
3352 u32 new_rx_size, new_tx_size;
3353 unsigned int reset = 0;
3354
3355 if ((ring->rx_mini_pending) || (ring->rx_jumbo_pending))
3356 return -EINVAL;
3357
3358 new_rx_size = clamp_t(u32, ring->rx_pending,
3359 MIN_RX_RING_SIZE, MAX_RX_RING_SIZE);
3360 new_rx_size = roundup_pow_of_two(new_rx_size);
3361
3362 new_tx_size = clamp_t(u32, ring->tx_pending,
3363 MIN_TX_RING_SIZE, MAX_TX_RING_SIZE);
3364 new_tx_size = roundup_pow_of_two(new_tx_size);
3365
3366 if ((new_tx_size == bp->tx_ring_size) &&
3367 (new_rx_size == bp->rx_ring_size)) {
3368 /* nothing to do */
3369 return 0;
3370 }
3371
3372 if (netif_running(bp->dev)) {
3373 reset = 1;
3374 macb_close(bp->dev);
3375 }
3376
3377 bp->rx_ring_size = new_rx_size;
3378 bp->tx_ring_size = new_tx_size;
3379
3380 if (reset)
3381 macb_open(bp->dev);
3382
3383 return 0;
3384 }
3385
3386 #ifdef CONFIG_MACB_USE_HWSTAMP
gem_get_tsu_rate(struct macb * bp)3387 static unsigned int gem_get_tsu_rate(struct macb *bp)
3388 {
3389 struct clk *tsu_clk;
3390 unsigned int tsu_rate;
3391
3392 tsu_clk = devm_clk_get(&bp->pdev->dev, "tsu_clk");
3393 if (!IS_ERR(tsu_clk))
3394 tsu_rate = clk_get_rate(tsu_clk);
3395 /* try pclk instead */
3396 else if (!IS_ERR(bp->pclk)) {
3397 tsu_clk = bp->pclk;
3398 tsu_rate = clk_get_rate(tsu_clk);
3399 } else
3400 return -ENOTSUPP;
3401 return tsu_rate;
3402 }
3403
gem_get_ptp_max_adj(void)3404 static s32 gem_get_ptp_max_adj(void)
3405 {
3406 return 64000000;
3407 }
3408
gem_get_ts_info(struct net_device * dev,struct kernel_ethtool_ts_info * info)3409 static int gem_get_ts_info(struct net_device *dev,
3410 struct kernel_ethtool_ts_info *info)
3411 {
3412 struct macb *bp = netdev_priv(dev);
3413
3414 if ((bp->hw_dma_cap & HW_DMA_CAP_PTP) == 0) {
3415 ethtool_op_get_ts_info(dev, info);
3416 return 0;
3417 }
3418
3419 info->so_timestamping =
3420 SOF_TIMESTAMPING_TX_SOFTWARE |
3421 SOF_TIMESTAMPING_TX_HARDWARE |
3422 SOF_TIMESTAMPING_RX_HARDWARE |
3423 SOF_TIMESTAMPING_RAW_HARDWARE;
3424 info->tx_types =
3425 (1 << HWTSTAMP_TX_ONESTEP_SYNC) |
3426 (1 << HWTSTAMP_TX_OFF) |
3427 (1 << HWTSTAMP_TX_ON);
3428 info->rx_filters =
3429 (1 << HWTSTAMP_FILTER_NONE) |
3430 (1 << HWTSTAMP_FILTER_ALL);
3431
3432 if (bp->ptp_clock)
3433 info->phc_index = ptp_clock_index(bp->ptp_clock);
3434
3435 return 0;
3436 }
3437
3438 static struct macb_ptp_info gem_ptp_info = {
3439 .ptp_init = gem_ptp_init,
3440 .ptp_remove = gem_ptp_remove,
3441 .get_ptp_max_adj = gem_get_ptp_max_adj,
3442 .get_tsu_rate = gem_get_tsu_rate,
3443 .get_ts_info = gem_get_ts_info,
3444 .get_hwtst = gem_get_hwtst,
3445 .set_hwtst = gem_set_hwtst,
3446 };
3447 #endif
3448
macb_get_ts_info(struct net_device * netdev,struct kernel_ethtool_ts_info * info)3449 static int macb_get_ts_info(struct net_device *netdev,
3450 struct kernel_ethtool_ts_info *info)
3451 {
3452 struct macb *bp = netdev_priv(netdev);
3453
3454 if (bp->ptp_info)
3455 return bp->ptp_info->get_ts_info(netdev, info);
3456
3457 return ethtool_op_get_ts_info(netdev, info);
3458 }
3459
gem_enable_flow_filters(struct macb * bp,bool enable)3460 static void gem_enable_flow_filters(struct macb *bp, bool enable)
3461 {
3462 struct net_device *netdev = bp->dev;
3463 struct ethtool_rx_fs_item *item;
3464 u32 t2_scr;
3465 int num_t2_scr;
3466
3467 if (!(netdev->features & NETIF_F_NTUPLE))
3468 return;
3469
3470 num_t2_scr = GEM_BFEXT(T2SCR, gem_readl(bp, DCFG8));
3471
3472 list_for_each_entry(item, &bp->rx_fs_list.list, list) {
3473 struct ethtool_rx_flow_spec *fs = &item->fs;
3474 struct ethtool_tcpip4_spec *tp4sp_m;
3475
3476 if (fs->location >= num_t2_scr)
3477 continue;
3478
3479 t2_scr = gem_readl_n(bp, SCRT2, fs->location);
3480
3481 /* enable/disable screener regs for the flow entry */
3482 t2_scr = GEM_BFINS(ETHTEN, enable, t2_scr);
3483
3484 /* only enable fields with no masking */
3485 tp4sp_m = &(fs->m_u.tcp_ip4_spec);
3486
3487 if (enable && (tp4sp_m->ip4src == 0xFFFFFFFF))
3488 t2_scr = GEM_BFINS(CMPAEN, 1, t2_scr);
3489 else
3490 t2_scr = GEM_BFINS(CMPAEN, 0, t2_scr);
3491
3492 if (enable && (tp4sp_m->ip4dst == 0xFFFFFFFF))
3493 t2_scr = GEM_BFINS(CMPBEN, 1, t2_scr);
3494 else
3495 t2_scr = GEM_BFINS(CMPBEN, 0, t2_scr);
3496
3497 if (enable && ((tp4sp_m->psrc == 0xFFFF) || (tp4sp_m->pdst == 0xFFFF)))
3498 t2_scr = GEM_BFINS(CMPCEN, 1, t2_scr);
3499 else
3500 t2_scr = GEM_BFINS(CMPCEN, 0, t2_scr);
3501
3502 gem_writel_n(bp, SCRT2, fs->location, t2_scr);
3503 }
3504 }
3505
gem_prog_cmp_regs(struct macb * bp,struct ethtool_rx_flow_spec * fs)3506 static void gem_prog_cmp_regs(struct macb *bp, struct ethtool_rx_flow_spec *fs)
3507 {
3508 struct ethtool_tcpip4_spec *tp4sp_v, *tp4sp_m;
3509 uint16_t index = fs->location;
3510 u32 w0, w1, t2_scr;
3511 bool cmp_a = false;
3512 bool cmp_b = false;
3513 bool cmp_c = false;
3514
3515 if (!macb_is_gem(bp))
3516 return;
3517
3518 tp4sp_v = &(fs->h_u.tcp_ip4_spec);
3519 tp4sp_m = &(fs->m_u.tcp_ip4_spec);
3520
3521 /* ignore field if any masking set */
3522 if (tp4sp_m->ip4src == 0xFFFFFFFF) {
3523 /* 1st compare reg - IP source address */
3524 w0 = 0;
3525 w1 = 0;
3526 w0 = tp4sp_v->ip4src;
3527 w1 = GEM_BFINS(T2DISMSK, 1, w1); /* 32-bit compare */
3528 w1 = GEM_BFINS(T2CMPOFST, GEM_T2COMPOFST_ETYPE, w1);
3529 w1 = GEM_BFINS(T2OFST, ETYPE_SRCIP_OFFSET, w1);
3530 gem_writel_n(bp, T2CMPW0, T2CMP_OFST(GEM_IP4SRC_CMP(index)), w0);
3531 gem_writel_n(bp, T2CMPW1, T2CMP_OFST(GEM_IP4SRC_CMP(index)), w1);
3532 cmp_a = true;
3533 }
3534
3535 /* ignore field if any masking set */
3536 if (tp4sp_m->ip4dst == 0xFFFFFFFF) {
3537 /* 2nd compare reg - IP destination address */
3538 w0 = 0;
3539 w1 = 0;
3540 w0 = tp4sp_v->ip4dst;
3541 w1 = GEM_BFINS(T2DISMSK, 1, w1); /* 32-bit compare */
3542 w1 = GEM_BFINS(T2CMPOFST, GEM_T2COMPOFST_ETYPE, w1);
3543 w1 = GEM_BFINS(T2OFST, ETYPE_DSTIP_OFFSET, w1);
3544 gem_writel_n(bp, T2CMPW0, T2CMP_OFST(GEM_IP4DST_CMP(index)), w0);
3545 gem_writel_n(bp, T2CMPW1, T2CMP_OFST(GEM_IP4DST_CMP(index)), w1);
3546 cmp_b = true;
3547 }
3548
3549 /* ignore both port fields if masking set in both */
3550 if ((tp4sp_m->psrc == 0xFFFF) || (tp4sp_m->pdst == 0xFFFF)) {
3551 /* 3rd compare reg - source port, destination port */
3552 w0 = 0;
3553 w1 = 0;
3554 w1 = GEM_BFINS(T2CMPOFST, GEM_T2COMPOFST_IPHDR, w1);
3555 if (tp4sp_m->psrc == tp4sp_m->pdst) {
3556 w0 = GEM_BFINS(T2MASK, tp4sp_v->psrc, w0);
3557 w0 = GEM_BFINS(T2CMP, tp4sp_v->pdst, w0);
3558 w1 = GEM_BFINS(T2DISMSK, 1, w1); /* 32-bit compare */
3559 w1 = GEM_BFINS(T2OFST, IPHDR_SRCPORT_OFFSET, w1);
3560 } else {
3561 /* only one port definition */
3562 w1 = GEM_BFINS(T2DISMSK, 0, w1); /* 16-bit compare */
3563 w0 = GEM_BFINS(T2MASK, 0xFFFF, w0);
3564 if (tp4sp_m->psrc == 0xFFFF) { /* src port */
3565 w0 = GEM_BFINS(T2CMP, tp4sp_v->psrc, w0);
3566 w1 = GEM_BFINS(T2OFST, IPHDR_SRCPORT_OFFSET, w1);
3567 } else { /* dst port */
3568 w0 = GEM_BFINS(T2CMP, tp4sp_v->pdst, w0);
3569 w1 = GEM_BFINS(T2OFST, IPHDR_DSTPORT_OFFSET, w1);
3570 }
3571 }
3572 gem_writel_n(bp, T2CMPW0, T2CMP_OFST(GEM_PORT_CMP(index)), w0);
3573 gem_writel_n(bp, T2CMPW1, T2CMP_OFST(GEM_PORT_CMP(index)), w1);
3574 cmp_c = true;
3575 }
3576
3577 t2_scr = 0;
3578 t2_scr = GEM_BFINS(QUEUE, (fs->ring_cookie) & 0xFF, t2_scr);
3579 t2_scr = GEM_BFINS(ETHT2IDX, SCRT2_ETHT, t2_scr);
3580 if (cmp_a)
3581 t2_scr = GEM_BFINS(CMPA, GEM_IP4SRC_CMP(index), t2_scr);
3582 if (cmp_b)
3583 t2_scr = GEM_BFINS(CMPB, GEM_IP4DST_CMP(index), t2_scr);
3584 if (cmp_c)
3585 t2_scr = GEM_BFINS(CMPC, GEM_PORT_CMP(index), t2_scr);
3586 gem_writel_n(bp, SCRT2, index, t2_scr);
3587 }
3588
gem_add_flow_filter(struct net_device * netdev,struct ethtool_rxnfc * cmd)3589 static int gem_add_flow_filter(struct net_device *netdev,
3590 struct ethtool_rxnfc *cmd)
3591 {
3592 struct macb *bp = netdev_priv(netdev);
3593 struct ethtool_rx_flow_spec *fs = &cmd->fs;
3594 struct ethtool_rx_fs_item *item, *newfs;
3595 unsigned long flags;
3596 int ret = -EINVAL;
3597 bool added = false;
3598
3599 newfs = kmalloc(sizeof(*newfs), GFP_KERNEL);
3600 if (newfs == NULL)
3601 return -ENOMEM;
3602 memcpy(&newfs->fs, fs, sizeof(newfs->fs));
3603
3604 netdev_dbg(netdev,
3605 "Adding flow filter entry,type=%u,queue=%u,loc=%u,src=%08X,dst=%08X,ps=%u,pd=%u\n",
3606 fs->flow_type, (int)fs->ring_cookie, fs->location,
3607 htonl(fs->h_u.tcp_ip4_spec.ip4src),
3608 htonl(fs->h_u.tcp_ip4_spec.ip4dst),
3609 be16_to_cpu(fs->h_u.tcp_ip4_spec.psrc),
3610 be16_to_cpu(fs->h_u.tcp_ip4_spec.pdst));
3611
3612 spin_lock_irqsave(&bp->rx_fs_lock, flags);
3613
3614 /* find correct place to add in list */
3615 list_for_each_entry(item, &bp->rx_fs_list.list, list) {
3616 if (item->fs.location > newfs->fs.location) {
3617 list_add_tail(&newfs->list, &item->list);
3618 added = true;
3619 break;
3620 } else if (item->fs.location == fs->location) {
3621 netdev_err(netdev, "Rule not added: location %d not free!\n",
3622 fs->location);
3623 ret = -EBUSY;
3624 goto err;
3625 }
3626 }
3627 if (!added)
3628 list_add_tail(&newfs->list, &bp->rx_fs_list.list);
3629
3630 gem_prog_cmp_regs(bp, fs);
3631 bp->rx_fs_list.count++;
3632 /* enable filtering if NTUPLE on */
3633 gem_enable_flow_filters(bp, 1);
3634
3635 spin_unlock_irqrestore(&bp->rx_fs_lock, flags);
3636 return 0;
3637
3638 err:
3639 spin_unlock_irqrestore(&bp->rx_fs_lock, flags);
3640 kfree(newfs);
3641 return ret;
3642 }
3643
gem_del_flow_filter(struct net_device * netdev,struct ethtool_rxnfc * cmd)3644 static int gem_del_flow_filter(struct net_device *netdev,
3645 struct ethtool_rxnfc *cmd)
3646 {
3647 struct macb *bp = netdev_priv(netdev);
3648 struct ethtool_rx_fs_item *item;
3649 struct ethtool_rx_flow_spec *fs;
3650 unsigned long flags;
3651
3652 spin_lock_irqsave(&bp->rx_fs_lock, flags);
3653
3654 list_for_each_entry(item, &bp->rx_fs_list.list, list) {
3655 if (item->fs.location == cmd->fs.location) {
3656 /* disable screener regs for the flow entry */
3657 fs = &(item->fs);
3658 netdev_dbg(netdev,
3659 "Deleting flow filter entry,type=%u,queue=%u,loc=%u,src=%08X,dst=%08X,ps=%u,pd=%u\n",
3660 fs->flow_type, (int)fs->ring_cookie, fs->location,
3661 htonl(fs->h_u.tcp_ip4_spec.ip4src),
3662 htonl(fs->h_u.tcp_ip4_spec.ip4dst),
3663 be16_to_cpu(fs->h_u.tcp_ip4_spec.psrc),
3664 be16_to_cpu(fs->h_u.tcp_ip4_spec.pdst));
3665
3666 gem_writel_n(bp, SCRT2, fs->location, 0);
3667
3668 list_del(&item->list);
3669 bp->rx_fs_list.count--;
3670 spin_unlock_irqrestore(&bp->rx_fs_lock, flags);
3671 kfree(item);
3672 return 0;
3673 }
3674 }
3675
3676 spin_unlock_irqrestore(&bp->rx_fs_lock, flags);
3677 return -EINVAL;
3678 }
3679
gem_get_flow_entry(struct net_device * netdev,struct ethtool_rxnfc * cmd)3680 static int gem_get_flow_entry(struct net_device *netdev,
3681 struct ethtool_rxnfc *cmd)
3682 {
3683 struct macb *bp = netdev_priv(netdev);
3684 struct ethtool_rx_fs_item *item;
3685
3686 list_for_each_entry(item, &bp->rx_fs_list.list, list) {
3687 if (item->fs.location == cmd->fs.location) {
3688 memcpy(&cmd->fs, &item->fs, sizeof(cmd->fs));
3689 return 0;
3690 }
3691 }
3692 return -EINVAL;
3693 }
3694
gem_get_all_flow_entries(struct net_device * netdev,struct ethtool_rxnfc * cmd,u32 * rule_locs)3695 static int gem_get_all_flow_entries(struct net_device *netdev,
3696 struct ethtool_rxnfc *cmd, u32 *rule_locs)
3697 {
3698 struct macb *bp = netdev_priv(netdev);
3699 struct ethtool_rx_fs_item *item;
3700 uint32_t cnt = 0;
3701
3702 list_for_each_entry(item, &bp->rx_fs_list.list, list) {
3703 if (cnt == cmd->rule_cnt)
3704 return -EMSGSIZE;
3705 rule_locs[cnt] = item->fs.location;
3706 cnt++;
3707 }
3708 cmd->data = bp->max_tuples;
3709 cmd->rule_cnt = cnt;
3710
3711 return 0;
3712 }
3713
gem_get_rxnfc(struct net_device * netdev,struct ethtool_rxnfc * cmd,u32 * rule_locs)3714 static int gem_get_rxnfc(struct net_device *netdev, struct ethtool_rxnfc *cmd,
3715 u32 *rule_locs)
3716 {
3717 struct macb *bp = netdev_priv(netdev);
3718 int ret = 0;
3719
3720 switch (cmd->cmd) {
3721 case ETHTOOL_GRXRINGS:
3722 cmd->data = bp->num_queues;
3723 break;
3724 case ETHTOOL_GRXCLSRLCNT:
3725 cmd->rule_cnt = bp->rx_fs_list.count;
3726 break;
3727 case ETHTOOL_GRXCLSRULE:
3728 ret = gem_get_flow_entry(netdev, cmd);
3729 break;
3730 case ETHTOOL_GRXCLSRLALL:
3731 ret = gem_get_all_flow_entries(netdev, cmd, rule_locs);
3732 break;
3733 default:
3734 netdev_err(netdev,
3735 "Command parameter %d is not supported\n", cmd->cmd);
3736 ret = -EOPNOTSUPP;
3737 }
3738
3739 return ret;
3740 }
3741
gem_set_rxnfc(struct net_device * netdev,struct ethtool_rxnfc * cmd)3742 static int gem_set_rxnfc(struct net_device *netdev, struct ethtool_rxnfc *cmd)
3743 {
3744 struct macb *bp = netdev_priv(netdev);
3745 int ret;
3746
3747 switch (cmd->cmd) {
3748 case ETHTOOL_SRXCLSRLINS:
3749 if ((cmd->fs.location >= bp->max_tuples)
3750 || (cmd->fs.ring_cookie >= bp->num_queues)) {
3751 ret = -EINVAL;
3752 break;
3753 }
3754 ret = gem_add_flow_filter(netdev, cmd);
3755 break;
3756 case ETHTOOL_SRXCLSRLDEL:
3757 ret = gem_del_flow_filter(netdev, cmd);
3758 break;
3759 default:
3760 netdev_err(netdev,
3761 "Command parameter %d is not supported\n", cmd->cmd);
3762 ret = -EOPNOTSUPP;
3763 }
3764
3765 return ret;
3766 }
3767
3768 static const struct ethtool_ops macb_ethtool_ops = {
3769 .get_regs_len = macb_get_regs_len,
3770 .get_regs = macb_get_regs,
3771 .get_link = ethtool_op_get_link,
3772 .get_ts_info = ethtool_op_get_ts_info,
3773 .get_wol = macb_get_wol,
3774 .set_wol = macb_set_wol,
3775 .get_link_ksettings = macb_get_link_ksettings,
3776 .set_link_ksettings = macb_set_link_ksettings,
3777 .get_ringparam = macb_get_ringparam,
3778 .set_ringparam = macb_set_ringparam,
3779 };
3780
3781 static const struct ethtool_ops gem_ethtool_ops = {
3782 .get_regs_len = macb_get_regs_len,
3783 .get_regs = macb_get_regs,
3784 .get_wol = macb_get_wol,
3785 .set_wol = macb_set_wol,
3786 .get_link = ethtool_op_get_link,
3787 .get_ts_info = macb_get_ts_info,
3788 .get_ethtool_stats = gem_get_ethtool_stats,
3789 .get_strings = gem_get_ethtool_strings,
3790 .get_sset_count = gem_get_sset_count,
3791 .get_link_ksettings = macb_get_link_ksettings,
3792 .set_link_ksettings = macb_set_link_ksettings,
3793 .get_ringparam = macb_get_ringparam,
3794 .set_ringparam = macb_set_ringparam,
3795 .get_rxnfc = gem_get_rxnfc,
3796 .set_rxnfc = gem_set_rxnfc,
3797 };
3798
macb_ioctl(struct net_device * dev,struct ifreq * rq,int cmd)3799 static int macb_ioctl(struct net_device *dev, struct ifreq *rq, int cmd)
3800 {
3801 struct macb *bp = netdev_priv(dev);
3802
3803 if (!netif_running(dev))
3804 return -EINVAL;
3805
3806 return phylink_mii_ioctl(bp->phylink, rq, cmd);
3807 }
3808
macb_hwtstamp_get(struct net_device * dev,struct kernel_hwtstamp_config * cfg)3809 static int macb_hwtstamp_get(struct net_device *dev,
3810 struct kernel_hwtstamp_config *cfg)
3811 {
3812 struct macb *bp = netdev_priv(dev);
3813
3814 if (!netif_running(dev))
3815 return -EINVAL;
3816
3817 if (!bp->ptp_info)
3818 return -EOPNOTSUPP;
3819
3820 return bp->ptp_info->get_hwtst(dev, cfg);
3821 }
3822
macb_hwtstamp_set(struct net_device * dev,struct kernel_hwtstamp_config * cfg,struct netlink_ext_ack * extack)3823 static int macb_hwtstamp_set(struct net_device *dev,
3824 struct kernel_hwtstamp_config *cfg,
3825 struct netlink_ext_ack *extack)
3826 {
3827 struct macb *bp = netdev_priv(dev);
3828
3829 if (!netif_running(dev))
3830 return -EINVAL;
3831
3832 if (!bp->ptp_info)
3833 return -EOPNOTSUPP;
3834
3835 return bp->ptp_info->set_hwtst(dev, cfg, extack);
3836 }
3837
macb_set_txcsum_feature(struct macb * bp,netdev_features_t features)3838 static inline void macb_set_txcsum_feature(struct macb *bp,
3839 netdev_features_t features)
3840 {
3841 u32 val;
3842
3843 if (!macb_is_gem(bp))
3844 return;
3845
3846 val = gem_readl(bp, DMACFG);
3847 if (features & NETIF_F_HW_CSUM)
3848 val |= GEM_BIT(TXCOEN);
3849 else
3850 val &= ~GEM_BIT(TXCOEN);
3851
3852 gem_writel(bp, DMACFG, val);
3853 }
3854
macb_set_rxcsum_feature(struct macb * bp,netdev_features_t features)3855 static inline void macb_set_rxcsum_feature(struct macb *bp,
3856 netdev_features_t features)
3857 {
3858 struct net_device *netdev = bp->dev;
3859 u32 val;
3860
3861 if (!macb_is_gem(bp))
3862 return;
3863
3864 val = gem_readl(bp, NCFGR);
3865 if ((features & NETIF_F_RXCSUM) && !(netdev->flags & IFF_PROMISC))
3866 val |= GEM_BIT(RXCOEN);
3867 else
3868 val &= ~GEM_BIT(RXCOEN);
3869
3870 gem_writel(bp, NCFGR, val);
3871 }
3872
macb_set_rxflow_feature(struct macb * bp,netdev_features_t features)3873 static inline void macb_set_rxflow_feature(struct macb *bp,
3874 netdev_features_t features)
3875 {
3876 if (!macb_is_gem(bp))
3877 return;
3878
3879 gem_enable_flow_filters(bp, !!(features & NETIF_F_NTUPLE));
3880 }
3881
macb_set_features(struct net_device * netdev,netdev_features_t features)3882 static int macb_set_features(struct net_device *netdev,
3883 netdev_features_t features)
3884 {
3885 struct macb *bp = netdev_priv(netdev);
3886 netdev_features_t changed = features ^ netdev->features;
3887
3888 /* TX checksum offload */
3889 if (changed & NETIF_F_HW_CSUM)
3890 macb_set_txcsum_feature(bp, features);
3891
3892 /* RX checksum offload */
3893 if (changed & NETIF_F_RXCSUM)
3894 macb_set_rxcsum_feature(bp, features);
3895
3896 /* RX Flow Filters */
3897 if (changed & NETIF_F_NTUPLE)
3898 macb_set_rxflow_feature(bp, features);
3899
3900 return 0;
3901 }
3902
macb_restore_features(struct macb * bp)3903 static void macb_restore_features(struct macb *bp)
3904 {
3905 struct net_device *netdev = bp->dev;
3906 netdev_features_t features = netdev->features;
3907 struct ethtool_rx_fs_item *item;
3908
3909 /* TX checksum offload */
3910 macb_set_txcsum_feature(bp, features);
3911
3912 /* RX checksum offload */
3913 macb_set_rxcsum_feature(bp, features);
3914
3915 /* RX Flow Filters */
3916 list_for_each_entry(item, &bp->rx_fs_list.list, list)
3917 gem_prog_cmp_regs(bp, &item->fs);
3918
3919 macb_set_rxflow_feature(bp, features);
3920 }
3921
3922 static const struct net_device_ops macb_netdev_ops = {
3923 .ndo_open = macb_open,
3924 .ndo_stop = macb_close,
3925 .ndo_start_xmit = macb_start_xmit,
3926 .ndo_set_rx_mode = macb_set_rx_mode,
3927 .ndo_get_stats = macb_get_stats,
3928 .ndo_eth_ioctl = macb_ioctl,
3929 .ndo_validate_addr = eth_validate_addr,
3930 .ndo_change_mtu = macb_change_mtu,
3931 .ndo_set_mac_address = macb_set_mac_addr,
3932 #ifdef CONFIG_NET_POLL_CONTROLLER
3933 .ndo_poll_controller = macb_poll_controller,
3934 #endif
3935 .ndo_set_features = macb_set_features,
3936 .ndo_features_check = macb_features_check,
3937 .ndo_hwtstamp_set = macb_hwtstamp_set,
3938 .ndo_hwtstamp_get = macb_hwtstamp_get,
3939 };
3940
3941 /* Configure peripheral capabilities according to device tree
3942 * and integration options used
3943 */
macb_configure_caps(struct macb * bp,const struct macb_config * dt_conf)3944 static void macb_configure_caps(struct macb *bp,
3945 const struct macb_config *dt_conf)
3946 {
3947 u32 dcfg;
3948
3949 if (dt_conf)
3950 bp->caps = dt_conf->caps;
3951
3952 if (hw_is_gem(bp->regs, bp->native_io)) {
3953 bp->caps |= MACB_CAPS_MACB_IS_GEM;
3954
3955 dcfg = gem_readl(bp, DCFG1);
3956 if (GEM_BFEXT(IRQCOR, dcfg) == 0)
3957 bp->caps |= MACB_CAPS_ISR_CLEAR_ON_WRITE;
3958 if (GEM_BFEXT(NO_PCS, dcfg) == 0)
3959 bp->caps |= MACB_CAPS_PCS;
3960 dcfg = gem_readl(bp, DCFG12);
3961 if (GEM_BFEXT(HIGH_SPEED, dcfg) == 1)
3962 bp->caps |= MACB_CAPS_HIGH_SPEED;
3963 dcfg = gem_readl(bp, DCFG2);
3964 if ((dcfg & (GEM_BIT(RX_PKT_BUFF) | GEM_BIT(TX_PKT_BUFF))) == 0)
3965 bp->caps |= MACB_CAPS_FIFO_MODE;
3966 if (gem_has_ptp(bp)) {
3967 if (!GEM_BFEXT(TSU, gem_readl(bp, DCFG5)))
3968 dev_err(&bp->pdev->dev,
3969 "GEM doesn't support hardware ptp.\n");
3970 else {
3971 #ifdef CONFIG_MACB_USE_HWSTAMP
3972 bp->hw_dma_cap |= HW_DMA_CAP_PTP;
3973 bp->ptp_info = &gem_ptp_info;
3974 #endif
3975 }
3976 }
3977 }
3978
3979 dev_dbg(&bp->pdev->dev, "Cadence caps 0x%08x\n", bp->caps);
3980 }
3981
macb_probe_queues(void __iomem * mem,bool native_io,unsigned int * queue_mask,unsigned int * num_queues)3982 static void macb_probe_queues(void __iomem *mem,
3983 bool native_io,
3984 unsigned int *queue_mask,
3985 unsigned int *num_queues)
3986 {
3987 *queue_mask = 0x1;
3988 *num_queues = 1;
3989
3990 /* is it macb or gem ?
3991 *
3992 * We need to read directly from the hardware here because
3993 * we are early in the probe process and don't have the
3994 * MACB_CAPS_MACB_IS_GEM flag positioned
3995 */
3996 if (!hw_is_gem(mem, native_io))
3997 return;
3998
3999 /* bit 0 is never set but queue 0 always exists */
4000 *queue_mask |= readl_relaxed(mem + GEM_DCFG6) & 0xff;
4001 *num_queues = hweight32(*queue_mask);
4002 }
4003
macb_clks_disable(struct clk * pclk,struct clk * hclk,struct clk * tx_clk,struct clk * rx_clk,struct clk * tsu_clk)4004 static void macb_clks_disable(struct clk *pclk, struct clk *hclk, struct clk *tx_clk,
4005 struct clk *rx_clk, struct clk *tsu_clk)
4006 {
4007 struct clk_bulk_data clks[] = {
4008 { .clk = tsu_clk, },
4009 { .clk = rx_clk, },
4010 { .clk = pclk, },
4011 { .clk = hclk, },
4012 { .clk = tx_clk },
4013 };
4014
4015 clk_bulk_disable_unprepare(ARRAY_SIZE(clks), clks);
4016 }
4017
macb_clk_init(struct platform_device * pdev,struct clk ** pclk,struct clk ** hclk,struct clk ** tx_clk,struct clk ** rx_clk,struct clk ** tsu_clk)4018 static int macb_clk_init(struct platform_device *pdev, struct clk **pclk,
4019 struct clk **hclk, struct clk **tx_clk,
4020 struct clk **rx_clk, struct clk **tsu_clk)
4021 {
4022 struct macb_platform_data *pdata;
4023 int err;
4024
4025 pdata = dev_get_platdata(&pdev->dev);
4026 if (pdata) {
4027 *pclk = pdata->pclk;
4028 *hclk = pdata->hclk;
4029 } else {
4030 *pclk = devm_clk_get(&pdev->dev, "pclk");
4031 *hclk = devm_clk_get(&pdev->dev, "hclk");
4032 }
4033
4034 if (IS_ERR_OR_NULL(*pclk))
4035 return dev_err_probe(&pdev->dev,
4036 IS_ERR(*pclk) ? PTR_ERR(*pclk) : -ENODEV,
4037 "failed to get pclk\n");
4038
4039 if (IS_ERR_OR_NULL(*hclk))
4040 return dev_err_probe(&pdev->dev,
4041 IS_ERR(*hclk) ? PTR_ERR(*hclk) : -ENODEV,
4042 "failed to get hclk\n");
4043
4044 *tx_clk = devm_clk_get_optional(&pdev->dev, "tx_clk");
4045 if (IS_ERR(*tx_clk))
4046 return PTR_ERR(*tx_clk);
4047
4048 *rx_clk = devm_clk_get_optional(&pdev->dev, "rx_clk");
4049 if (IS_ERR(*rx_clk))
4050 return PTR_ERR(*rx_clk);
4051
4052 *tsu_clk = devm_clk_get_optional(&pdev->dev, "tsu_clk");
4053 if (IS_ERR(*tsu_clk))
4054 return PTR_ERR(*tsu_clk);
4055
4056 err = clk_prepare_enable(*pclk);
4057 if (err) {
4058 dev_err(&pdev->dev, "failed to enable pclk (%d)\n", err);
4059 return err;
4060 }
4061
4062 err = clk_prepare_enable(*hclk);
4063 if (err) {
4064 dev_err(&pdev->dev, "failed to enable hclk (%d)\n", err);
4065 goto err_disable_pclk;
4066 }
4067
4068 err = clk_prepare_enable(*tx_clk);
4069 if (err) {
4070 dev_err(&pdev->dev, "failed to enable tx_clk (%d)\n", err);
4071 goto err_disable_hclk;
4072 }
4073
4074 err = clk_prepare_enable(*rx_clk);
4075 if (err) {
4076 dev_err(&pdev->dev, "failed to enable rx_clk (%d)\n", err);
4077 goto err_disable_txclk;
4078 }
4079
4080 err = clk_prepare_enable(*tsu_clk);
4081 if (err) {
4082 dev_err(&pdev->dev, "failed to enable tsu_clk (%d)\n", err);
4083 goto err_disable_rxclk;
4084 }
4085
4086 return 0;
4087
4088 err_disable_rxclk:
4089 clk_disable_unprepare(*rx_clk);
4090
4091 err_disable_txclk:
4092 clk_disable_unprepare(*tx_clk);
4093
4094 err_disable_hclk:
4095 clk_disable_unprepare(*hclk);
4096
4097 err_disable_pclk:
4098 clk_disable_unprepare(*pclk);
4099
4100 return err;
4101 }
4102
macb_init(struct platform_device * pdev)4103 static int macb_init(struct platform_device *pdev)
4104 {
4105 struct net_device *dev = platform_get_drvdata(pdev);
4106 unsigned int hw_q, q;
4107 struct macb *bp = netdev_priv(dev);
4108 struct macb_queue *queue;
4109 int err;
4110 u32 val, reg;
4111
4112 bp->tx_ring_size = DEFAULT_TX_RING_SIZE;
4113 bp->rx_ring_size = DEFAULT_RX_RING_SIZE;
4114
4115 /* set the queue register mapping once for all: queue0 has a special
4116 * register mapping but we don't want to test the queue index then
4117 * compute the corresponding register offset at run time.
4118 */
4119 for (hw_q = 0, q = 0; hw_q < MACB_MAX_QUEUES; ++hw_q) {
4120 if (!(bp->queue_mask & (1 << hw_q)))
4121 continue;
4122
4123 queue = &bp->queues[q];
4124 queue->bp = bp;
4125 spin_lock_init(&queue->tx_ptr_lock);
4126 netif_napi_add(dev, &queue->napi_rx, macb_rx_poll);
4127 netif_napi_add(dev, &queue->napi_tx, macb_tx_poll);
4128 if (hw_q) {
4129 queue->ISR = GEM_ISR(hw_q - 1);
4130 queue->IER = GEM_IER(hw_q - 1);
4131 queue->IDR = GEM_IDR(hw_q - 1);
4132 queue->IMR = GEM_IMR(hw_q - 1);
4133 queue->TBQP = GEM_TBQP(hw_q - 1);
4134 queue->RBQP = GEM_RBQP(hw_q - 1);
4135 queue->RBQS = GEM_RBQS(hw_q - 1);
4136 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT
4137 if (bp->hw_dma_cap & HW_DMA_CAP_64B) {
4138 queue->TBQPH = GEM_TBQPH(hw_q - 1);
4139 queue->RBQPH = GEM_RBQPH(hw_q - 1);
4140 }
4141 #endif
4142 } else {
4143 /* queue0 uses legacy registers */
4144 queue->ISR = MACB_ISR;
4145 queue->IER = MACB_IER;
4146 queue->IDR = MACB_IDR;
4147 queue->IMR = MACB_IMR;
4148 queue->TBQP = MACB_TBQP;
4149 queue->RBQP = MACB_RBQP;
4150 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT
4151 if (bp->hw_dma_cap & HW_DMA_CAP_64B) {
4152 queue->TBQPH = MACB_TBQPH;
4153 queue->RBQPH = MACB_RBQPH;
4154 }
4155 #endif
4156 }
4157
4158 /* get irq: here we use the linux queue index, not the hardware
4159 * queue index. the queue irq definitions in the device tree
4160 * must remove the optional gaps that could exist in the
4161 * hardware queue mask.
4162 */
4163 queue->irq = platform_get_irq(pdev, q);
4164 err = devm_request_irq(&pdev->dev, queue->irq, macb_interrupt,
4165 IRQF_SHARED, dev->name, queue);
4166 if (err) {
4167 dev_err(&pdev->dev,
4168 "Unable to request IRQ %d (error %d)\n",
4169 queue->irq, err);
4170 return err;
4171 }
4172
4173 INIT_WORK(&queue->tx_error_task, macb_tx_error_task);
4174 q++;
4175 }
4176
4177 dev->netdev_ops = &macb_netdev_ops;
4178
4179 /* setup appropriated routines according to adapter type */
4180 if (macb_is_gem(bp)) {
4181 bp->macbgem_ops.mog_alloc_rx_buffers = gem_alloc_rx_buffers;
4182 bp->macbgem_ops.mog_free_rx_buffers = gem_free_rx_buffers;
4183 bp->macbgem_ops.mog_init_rings = gem_init_rings;
4184 bp->macbgem_ops.mog_rx = gem_rx;
4185 dev->ethtool_ops = &gem_ethtool_ops;
4186 } else {
4187 bp->macbgem_ops.mog_alloc_rx_buffers = macb_alloc_rx_buffers;
4188 bp->macbgem_ops.mog_free_rx_buffers = macb_free_rx_buffers;
4189 bp->macbgem_ops.mog_init_rings = macb_init_rings;
4190 bp->macbgem_ops.mog_rx = macb_rx;
4191 dev->ethtool_ops = &macb_ethtool_ops;
4192 }
4193
4194 netdev_sw_irq_coalesce_default_on(dev);
4195
4196 dev->priv_flags |= IFF_LIVE_ADDR_CHANGE;
4197
4198 /* Set features */
4199 dev->hw_features = NETIF_F_SG;
4200
4201 /* Check LSO capability */
4202 if (GEM_BFEXT(PBUF_LSO, gem_readl(bp, DCFG6)))
4203 dev->hw_features |= MACB_NETIF_LSO;
4204
4205 /* Checksum offload is only available on gem with packet buffer */
4206 if (macb_is_gem(bp) && !(bp->caps & MACB_CAPS_FIFO_MODE))
4207 dev->hw_features |= NETIF_F_HW_CSUM | NETIF_F_RXCSUM;
4208 if (bp->caps & MACB_CAPS_SG_DISABLED)
4209 dev->hw_features &= ~NETIF_F_SG;
4210 dev->features = dev->hw_features;
4211
4212 /* Check RX Flow Filters support.
4213 * Max Rx flows set by availability of screeners & compare regs:
4214 * each 4-tuple define requires 1 T2 screener reg + 3 compare regs
4215 */
4216 reg = gem_readl(bp, DCFG8);
4217 bp->max_tuples = min((GEM_BFEXT(SCR2CMP, reg) / 3),
4218 GEM_BFEXT(T2SCR, reg));
4219 INIT_LIST_HEAD(&bp->rx_fs_list.list);
4220 if (bp->max_tuples > 0) {
4221 /* also needs one ethtype match to check IPv4 */
4222 if (GEM_BFEXT(SCR2ETH, reg) > 0) {
4223 /* program this reg now */
4224 reg = 0;
4225 reg = GEM_BFINS(ETHTCMP, (uint16_t)ETH_P_IP, reg);
4226 gem_writel_n(bp, ETHT, SCRT2_ETHT, reg);
4227 /* Filtering is supported in hw but don't enable it in kernel now */
4228 dev->hw_features |= NETIF_F_NTUPLE;
4229 /* init Rx flow definitions */
4230 bp->rx_fs_list.count = 0;
4231 spin_lock_init(&bp->rx_fs_lock);
4232 } else
4233 bp->max_tuples = 0;
4234 }
4235
4236 if (!(bp->caps & MACB_CAPS_USRIO_DISABLED)) {
4237 val = 0;
4238 if (phy_interface_mode_is_rgmii(bp->phy_interface))
4239 val = bp->usrio->rgmii;
4240 else if (bp->phy_interface == PHY_INTERFACE_MODE_RMII &&
4241 (bp->caps & MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII))
4242 val = bp->usrio->rmii;
4243 else if (!(bp->caps & MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII))
4244 val = bp->usrio->mii;
4245
4246 if (bp->caps & MACB_CAPS_USRIO_HAS_CLKEN)
4247 val |= bp->usrio->refclk;
4248
4249 macb_or_gem_writel(bp, USRIO, val);
4250 }
4251
4252 /* Set MII management clock divider */
4253 val = macb_mdc_clk_div(bp);
4254 val |= macb_dbw(bp);
4255 if (bp->phy_interface == PHY_INTERFACE_MODE_SGMII)
4256 val |= GEM_BIT(SGMIIEN) | GEM_BIT(PCSSEL);
4257 macb_writel(bp, NCFGR, val);
4258
4259 return 0;
4260 }
4261
4262 static const struct macb_usrio_config macb_default_usrio = {
4263 .mii = MACB_BIT(MII),
4264 .rmii = MACB_BIT(RMII),
4265 .rgmii = GEM_BIT(RGMII),
4266 .refclk = MACB_BIT(CLKEN),
4267 };
4268
4269 #if defined(CONFIG_OF)
4270 /* 1518 rounded up */
4271 #define AT91ETHER_MAX_RBUFF_SZ 0x600
4272 /* max number of receive buffers */
4273 #define AT91ETHER_MAX_RX_DESCR 9
4274
4275 static struct sifive_fu540_macb_mgmt *mgmt;
4276
at91ether_alloc_coherent(struct macb * lp)4277 static int at91ether_alloc_coherent(struct macb *lp)
4278 {
4279 struct macb_queue *q = &lp->queues[0];
4280
4281 q->rx_ring = dma_alloc_coherent(&lp->pdev->dev,
4282 (AT91ETHER_MAX_RX_DESCR *
4283 macb_dma_desc_get_size(lp)),
4284 &q->rx_ring_dma, GFP_KERNEL);
4285 if (!q->rx_ring)
4286 return -ENOMEM;
4287
4288 q->rx_buffers = dma_alloc_coherent(&lp->pdev->dev,
4289 AT91ETHER_MAX_RX_DESCR *
4290 AT91ETHER_MAX_RBUFF_SZ,
4291 &q->rx_buffers_dma, GFP_KERNEL);
4292 if (!q->rx_buffers) {
4293 dma_free_coherent(&lp->pdev->dev,
4294 AT91ETHER_MAX_RX_DESCR *
4295 macb_dma_desc_get_size(lp),
4296 q->rx_ring, q->rx_ring_dma);
4297 q->rx_ring = NULL;
4298 return -ENOMEM;
4299 }
4300
4301 return 0;
4302 }
4303
at91ether_free_coherent(struct macb * lp)4304 static void at91ether_free_coherent(struct macb *lp)
4305 {
4306 struct macb_queue *q = &lp->queues[0];
4307
4308 if (q->rx_ring) {
4309 dma_free_coherent(&lp->pdev->dev,
4310 AT91ETHER_MAX_RX_DESCR *
4311 macb_dma_desc_get_size(lp),
4312 q->rx_ring, q->rx_ring_dma);
4313 q->rx_ring = NULL;
4314 }
4315
4316 if (q->rx_buffers) {
4317 dma_free_coherent(&lp->pdev->dev,
4318 AT91ETHER_MAX_RX_DESCR *
4319 AT91ETHER_MAX_RBUFF_SZ,
4320 q->rx_buffers, q->rx_buffers_dma);
4321 q->rx_buffers = NULL;
4322 }
4323 }
4324
4325 /* Initialize and start the Receiver and Transmit subsystems */
at91ether_start(struct macb * lp)4326 static int at91ether_start(struct macb *lp)
4327 {
4328 struct macb_queue *q = &lp->queues[0];
4329 struct macb_dma_desc *desc;
4330 dma_addr_t addr;
4331 u32 ctl;
4332 int i, ret;
4333
4334 ret = at91ether_alloc_coherent(lp);
4335 if (ret)
4336 return ret;
4337
4338 addr = q->rx_buffers_dma;
4339 for (i = 0; i < AT91ETHER_MAX_RX_DESCR; i++) {
4340 desc = macb_rx_desc(q, i);
4341 macb_set_addr(lp, desc, addr);
4342 desc->ctrl = 0;
4343 addr += AT91ETHER_MAX_RBUFF_SZ;
4344 }
4345
4346 /* Set the Wrap bit on the last descriptor */
4347 desc->addr |= MACB_BIT(RX_WRAP);
4348
4349 /* Reset buffer index */
4350 q->rx_tail = 0;
4351
4352 /* Program address of descriptor list in Rx Buffer Queue register */
4353 macb_writel(lp, RBQP, q->rx_ring_dma);
4354
4355 /* Enable Receive and Transmit */
4356 ctl = macb_readl(lp, NCR);
4357 macb_writel(lp, NCR, ctl | MACB_BIT(RE) | MACB_BIT(TE));
4358
4359 /* Enable MAC interrupts */
4360 macb_writel(lp, IER, MACB_BIT(RCOMP) |
4361 MACB_BIT(RXUBR) |
4362 MACB_BIT(ISR_TUND) |
4363 MACB_BIT(ISR_RLE) |
4364 MACB_BIT(TCOMP) |
4365 MACB_BIT(ISR_ROVR) |
4366 MACB_BIT(HRESP));
4367
4368 return 0;
4369 }
4370
at91ether_stop(struct macb * lp)4371 static void at91ether_stop(struct macb *lp)
4372 {
4373 u32 ctl;
4374
4375 /* Disable MAC interrupts */
4376 macb_writel(lp, IDR, MACB_BIT(RCOMP) |
4377 MACB_BIT(RXUBR) |
4378 MACB_BIT(ISR_TUND) |
4379 MACB_BIT(ISR_RLE) |
4380 MACB_BIT(TCOMP) |
4381 MACB_BIT(ISR_ROVR) |
4382 MACB_BIT(HRESP));
4383
4384 /* Disable Receiver and Transmitter */
4385 ctl = macb_readl(lp, NCR);
4386 macb_writel(lp, NCR, ctl & ~(MACB_BIT(TE) | MACB_BIT(RE)));
4387
4388 /* Free resources. */
4389 at91ether_free_coherent(lp);
4390 }
4391
4392 /* Open the ethernet interface */
at91ether_open(struct net_device * dev)4393 static int at91ether_open(struct net_device *dev)
4394 {
4395 struct macb *lp = netdev_priv(dev);
4396 u32 ctl;
4397 int ret;
4398
4399 ret = pm_runtime_resume_and_get(&lp->pdev->dev);
4400 if (ret < 0)
4401 return ret;
4402
4403 /* Clear internal statistics */
4404 ctl = macb_readl(lp, NCR);
4405 macb_writel(lp, NCR, ctl | MACB_BIT(CLRSTAT));
4406
4407 macb_set_hwaddr(lp);
4408
4409 ret = at91ether_start(lp);
4410 if (ret)
4411 goto pm_exit;
4412
4413 ret = macb_phylink_connect(lp);
4414 if (ret)
4415 goto stop;
4416
4417 netif_start_queue(dev);
4418
4419 return 0;
4420
4421 stop:
4422 at91ether_stop(lp);
4423 pm_exit:
4424 pm_runtime_put_sync(&lp->pdev->dev);
4425 return ret;
4426 }
4427
4428 /* Close the interface */
at91ether_close(struct net_device * dev)4429 static int at91ether_close(struct net_device *dev)
4430 {
4431 struct macb *lp = netdev_priv(dev);
4432
4433 netif_stop_queue(dev);
4434
4435 phylink_stop(lp->phylink);
4436 phylink_disconnect_phy(lp->phylink);
4437
4438 at91ether_stop(lp);
4439
4440 return pm_runtime_put(&lp->pdev->dev);
4441 }
4442
4443 /* Transmit packet */
at91ether_start_xmit(struct sk_buff * skb,struct net_device * dev)4444 static netdev_tx_t at91ether_start_xmit(struct sk_buff *skb,
4445 struct net_device *dev)
4446 {
4447 struct macb *lp = netdev_priv(dev);
4448
4449 if (macb_readl(lp, TSR) & MACB_BIT(RM9200_BNQ)) {
4450 int desc = 0;
4451
4452 netif_stop_queue(dev);
4453
4454 /* Store packet information (to free when Tx completed) */
4455 lp->rm9200_txq[desc].skb = skb;
4456 lp->rm9200_txq[desc].size = skb->len;
4457 lp->rm9200_txq[desc].mapping = dma_map_single(&lp->pdev->dev, skb->data,
4458 skb->len, DMA_TO_DEVICE);
4459 if (dma_mapping_error(&lp->pdev->dev, lp->rm9200_txq[desc].mapping)) {
4460 dev_kfree_skb_any(skb);
4461 dev->stats.tx_dropped++;
4462 netdev_err(dev, "%s: DMA mapping error\n", __func__);
4463 return NETDEV_TX_OK;
4464 }
4465
4466 /* Set address of the data in the Transmit Address register */
4467 macb_writel(lp, TAR, lp->rm9200_txq[desc].mapping);
4468 /* Set length of the packet in the Transmit Control register */
4469 macb_writel(lp, TCR, skb->len);
4470
4471 } else {
4472 netdev_err(dev, "%s called, but device is busy!\n", __func__);
4473 return NETDEV_TX_BUSY;
4474 }
4475
4476 return NETDEV_TX_OK;
4477 }
4478
4479 /* Extract received frame from buffer descriptors and sent to upper layers.
4480 * (Called from interrupt context)
4481 */
at91ether_rx(struct net_device * dev)4482 static void at91ether_rx(struct net_device *dev)
4483 {
4484 struct macb *lp = netdev_priv(dev);
4485 struct macb_queue *q = &lp->queues[0];
4486 struct macb_dma_desc *desc;
4487 unsigned char *p_recv;
4488 struct sk_buff *skb;
4489 unsigned int pktlen;
4490
4491 desc = macb_rx_desc(q, q->rx_tail);
4492 while (desc->addr & MACB_BIT(RX_USED)) {
4493 p_recv = q->rx_buffers + q->rx_tail * AT91ETHER_MAX_RBUFF_SZ;
4494 pktlen = MACB_BF(RX_FRMLEN, desc->ctrl);
4495 skb = netdev_alloc_skb(dev, pktlen + 2);
4496 if (skb) {
4497 skb_reserve(skb, 2);
4498 skb_put_data(skb, p_recv, pktlen);
4499
4500 skb->protocol = eth_type_trans(skb, dev);
4501 dev->stats.rx_packets++;
4502 dev->stats.rx_bytes += pktlen;
4503 netif_rx(skb);
4504 } else {
4505 dev->stats.rx_dropped++;
4506 }
4507
4508 if (desc->ctrl & MACB_BIT(RX_MHASH_MATCH))
4509 dev->stats.multicast++;
4510
4511 /* reset ownership bit */
4512 desc->addr &= ~MACB_BIT(RX_USED);
4513
4514 /* wrap after last buffer */
4515 if (q->rx_tail == AT91ETHER_MAX_RX_DESCR - 1)
4516 q->rx_tail = 0;
4517 else
4518 q->rx_tail++;
4519
4520 desc = macb_rx_desc(q, q->rx_tail);
4521 }
4522 }
4523
4524 /* MAC interrupt handler */
at91ether_interrupt(int irq,void * dev_id)4525 static irqreturn_t at91ether_interrupt(int irq, void *dev_id)
4526 {
4527 struct net_device *dev = dev_id;
4528 struct macb *lp = netdev_priv(dev);
4529 u32 intstatus, ctl;
4530 unsigned int desc;
4531
4532 /* MAC Interrupt Status register indicates what interrupts are pending.
4533 * It is automatically cleared once read.
4534 */
4535 intstatus = macb_readl(lp, ISR);
4536
4537 /* Receive complete */
4538 if (intstatus & MACB_BIT(RCOMP))
4539 at91ether_rx(dev);
4540
4541 /* Transmit complete */
4542 if (intstatus & MACB_BIT(TCOMP)) {
4543 /* The TCOM bit is set even if the transmission failed */
4544 if (intstatus & (MACB_BIT(ISR_TUND) | MACB_BIT(ISR_RLE)))
4545 dev->stats.tx_errors++;
4546
4547 desc = 0;
4548 if (lp->rm9200_txq[desc].skb) {
4549 dev_consume_skb_irq(lp->rm9200_txq[desc].skb);
4550 lp->rm9200_txq[desc].skb = NULL;
4551 dma_unmap_single(&lp->pdev->dev, lp->rm9200_txq[desc].mapping,
4552 lp->rm9200_txq[desc].size, DMA_TO_DEVICE);
4553 dev->stats.tx_packets++;
4554 dev->stats.tx_bytes += lp->rm9200_txq[desc].size;
4555 }
4556 netif_wake_queue(dev);
4557 }
4558
4559 /* Work-around for EMAC Errata section 41.3.1 */
4560 if (intstatus & MACB_BIT(RXUBR)) {
4561 ctl = macb_readl(lp, NCR);
4562 macb_writel(lp, NCR, ctl & ~MACB_BIT(RE));
4563 wmb();
4564 macb_writel(lp, NCR, ctl | MACB_BIT(RE));
4565 }
4566
4567 if (intstatus & MACB_BIT(ISR_ROVR))
4568 netdev_err(dev, "ROVR error\n");
4569
4570 return IRQ_HANDLED;
4571 }
4572
4573 #ifdef CONFIG_NET_POLL_CONTROLLER
at91ether_poll_controller(struct net_device * dev)4574 static void at91ether_poll_controller(struct net_device *dev)
4575 {
4576 unsigned long flags;
4577
4578 local_irq_save(flags);
4579 at91ether_interrupt(dev->irq, dev);
4580 local_irq_restore(flags);
4581 }
4582 #endif
4583
4584 static const struct net_device_ops at91ether_netdev_ops = {
4585 .ndo_open = at91ether_open,
4586 .ndo_stop = at91ether_close,
4587 .ndo_start_xmit = at91ether_start_xmit,
4588 .ndo_get_stats = macb_get_stats,
4589 .ndo_set_rx_mode = macb_set_rx_mode,
4590 .ndo_set_mac_address = eth_mac_addr,
4591 .ndo_eth_ioctl = macb_ioctl,
4592 .ndo_validate_addr = eth_validate_addr,
4593 #ifdef CONFIG_NET_POLL_CONTROLLER
4594 .ndo_poll_controller = at91ether_poll_controller,
4595 #endif
4596 .ndo_hwtstamp_set = macb_hwtstamp_set,
4597 .ndo_hwtstamp_get = macb_hwtstamp_get,
4598 };
4599
at91ether_clk_init(struct platform_device * pdev,struct clk ** pclk,struct clk ** hclk,struct clk ** tx_clk,struct clk ** rx_clk,struct clk ** tsu_clk)4600 static int at91ether_clk_init(struct platform_device *pdev, struct clk **pclk,
4601 struct clk **hclk, struct clk **tx_clk,
4602 struct clk **rx_clk, struct clk **tsu_clk)
4603 {
4604 int err;
4605
4606 *hclk = NULL;
4607 *tx_clk = NULL;
4608 *rx_clk = NULL;
4609 *tsu_clk = NULL;
4610
4611 *pclk = devm_clk_get(&pdev->dev, "ether_clk");
4612 if (IS_ERR(*pclk))
4613 return PTR_ERR(*pclk);
4614
4615 err = clk_prepare_enable(*pclk);
4616 if (err) {
4617 dev_err(&pdev->dev, "failed to enable pclk (%d)\n", err);
4618 return err;
4619 }
4620
4621 return 0;
4622 }
4623
at91ether_init(struct platform_device * pdev)4624 static int at91ether_init(struct platform_device *pdev)
4625 {
4626 struct net_device *dev = platform_get_drvdata(pdev);
4627 struct macb *bp = netdev_priv(dev);
4628 int err;
4629
4630 bp->queues[0].bp = bp;
4631
4632 dev->netdev_ops = &at91ether_netdev_ops;
4633 dev->ethtool_ops = &macb_ethtool_ops;
4634
4635 err = devm_request_irq(&pdev->dev, dev->irq, at91ether_interrupt,
4636 0, dev->name, dev);
4637 if (err)
4638 return err;
4639
4640 macb_writel(bp, NCR, 0);
4641
4642 macb_writel(bp, NCFGR, MACB_BF(CLK, MACB_CLK_DIV32) | MACB_BIT(BIG));
4643
4644 return 0;
4645 }
4646
fu540_macb_tx_recalc_rate(struct clk_hw * hw,unsigned long parent_rate)4647 static unsigned long fu540_macb_tx_recalc_rate(struct clk_hw *hw,
4648 unsigned long parent_rate)
4649 {
4650 return mgmt->rate;
4651 }
4652
fu540_macb_tx_round_rate(struct clk_hw * hw,unsigned long rate,unsigned long * parent_rate)4653 static long fu540_macb_tx_round_rate(struct clk_hw *hw, unsigned long rate,
4654 unsigned long *parent_rate)
4655 {
4656 if (WARN_ON(rate < 2500000))
4657 return 2500000;
4658 else if (rate == 2500000)
4659 return 2500000;
4660 else if (WARN_ON(rate < 13750000))
4661 return 2500000;
4662 else if (WARN_ON(rate < 25000000))
4663 return 25000000;
4664 else if (rate == 25000000)
4665 return 25000000;
4666 else if (WARN_ON(rate < 75000000))
4667 return 25000000;
4668 else if (WARN_ON(rate < 125000000))
4669 return 125000000;
4670 else if (rate == 125000000)
4671 return 125000000;
4672
4673 WARN_ON(rate > 125000000);
4674
4675 return 125000000;
4676 }
4677
fu540_macb_tx_set_rate(struct clk_hw * hw,unsigned long rate,unsigned long parent_rate)4678 static int fu540_macb_tx_set_rate(struct clk_hw *hw, unsigned long rate,
4679 unsigned long parent_rate)
4680 {
4681 rate = fu540_macb_tx_round_rate(hw, rate, &parent_rate);
4682 if (rate != 125000000)
4683 iowrite32(1, mgmt->reg);
4684 else
4685 iowrite32(0, mgmt->reg);
4686 mgmt->rate = rate;
4687
4688 return 0;
4689 }
4690
4691 static const struct clk_ops fu540_c000_ops = {
4692 .recalc_rate = fu540_macb_tx_recalc_rate,
4693 .round_rate = fu540_macb_tx_round_rate,
4694 .set_rate = fu540_macb_tx_set_rate,
4695 };
4696
fu540_c000_clk_init(struct platform_device * pdev,struct clk ** pclk,struct clk ** hclk,struct clk ** tx_clk,struct clk ** rx_clk,struct clk ** tsu_clk)4697 static int fu540_c000_clk_init(struct platform_device *pdev, struct clk **pclk,
4698 struct clk **hclk, struct clk **tx_clk,
4699 struct clk **rx_clk, struct clk **tsu_clk)
4700 {
4701 struct clk_init_data init;
4702 int err = 0;
4703
4704 err = macb_clk_init(pdev, pclk, hclk, tx_clk, rx_clk, tsu_clk);
4705 if (err)
4706 return err;
4707
4708 mgmt = devm_kzalloc(&pdev->dev, sizeof(*mgmt), GFP_KERNEL);
4709 if (!mgmt) {
4710 err = -ENOMEM;
4711 goto err_disable_clks;
4712 }
4713
4714 init.name = "sifive-gemgxl-mgmt";
4715 init.ops = &fu540_c000_ops;
4716 init.flags = 0;
4717 init.num_parents = 0;
4718
4719 mgmt->rate = 0;
4720 mgmt->hw.init = &init;
4721
4722 *tx_clk = devm_clk_register(&pdev->dev, &mgmt->hw);
4723 if (IS_ERR(*tx_clk)) {
4724 err = PTR_ERR(*tx_clk);
4725 goto err_disable_clks;
4726 }
4727
4728 err = clk_prepare_enable(*tx_clk);
4729 if (err) {
4730 dev_err(&pdev->dev, "failed to enable tx_clk (%u)\n", err);
4731 *tx_clk = NULL;
4732 goto err_disable_clks;
4733 } else {
4734 dev_info(&pdev->dev, "Registered clk switch '%s'\n", init.name);
4735 }
4736
4737 return 0;
4738
4739 err_disable_clks:
4740 macb_clks_disable(*pclk, *hclk, *tx_clk, *rx_clk, *tsu_clk);
4741
4742 return err;
4743 }
4744
fu540_c000_init(struct platform_device * pdev)4745 static int fu540_c000_init(struct platform_device *pdev)
4746 {
4747 mgmt->reg = devm_platform_ioremap_resource(pdev, 1);
4748 if (IS_ERR(mgmt->reg))
4749 return PTR_ERR(mgmt->reg);
4750
4751 return macb_init(pdev);
4752 }
4753
init_reset_optional(struct platform_device * pdev)4754 static int init_reset_optional(struct platform_device *pdev)
4755 {
4756 struct net_device *dev = platform_get_drvdata(pdev);
4757 struct macb *bp = netdev_priv(dev);
4758 int ret;
4759
4760 if (bp->phy_interface == PHY_INTERFACE_MODE_SGMII) {
4761 /* Ensure PHY device used in SGMII mode is ready */
4762 bp->sgmii_phy = devm_phy_optional_get(&pdev->dev, NULL);
4763
4764 if (IS_ERR(bp->sgmii_phy))
4765 return dev_err_probe(&pdev->dev, PTR_ERR(bp->sgmii_phy),
4766 "failed to get SGMII PHY\n");
4767
4768 ret = phy_init(bp->sgmii_phy);
4769 if (ret)
4770 return dev_err_probe(&pdev->dev, ret,
4771 "failed to init SGMII PHY\n");
4772
4773 ret = zynqmp_pm_is_function_supported(PM_IOCTL, IOCTL_SET_GEM_CONFIG);
4774 if (!ret) {
4775 u32 pm_info[2];
4776
4777 ret = of_property_read_u32_array(pdev->dev.of_node, "power-domains",
4778 pm_info, ARRAY_SIZE(pm_info));
4779 if (ret) {
4780 dev_err(&pdev->dev, "Failed to read power management information\n");
4781 goto err_out_phy_exit;
4782 }
4783 ret = zynqmp_pm_set_gem_config(pm_info[1], GEM_CONFIG_FIXED, 0);
4784 if (ret)
4785 goto err_out_phy_exit;
4786
4787 ret = zynqmp_pm_set_gem_config(pm_info[1], GEM_CONFIG_SGMII_MODE, 1);
4788 if (ret)
4789 goto err_out_phy_exit;
4790 }
4791
4792 }
4793
4794 /* Fully reset controller at hardware level if mapped in device tree */
4795 ret = device_reset_optional(&pdev->dev);
4796 if (ret) {
4797 phy_exit(bp->sgmii_phy);
4798 return dev_err_probe(&pdev->dev, ret, "failed to reset controller");
4799 }
4800
4801 ret = macb_init(pdev);
4802
4803 err_out_phy_exit:
4804 if (ret)
4805 phy_exit(bp->sgmii_phy);
4806
4807 return ret;
4808 }
4809
4810 static const struct macb_usrio_config sama7g5_usrio = {
4811 .mii = 0,
4812 .rmii = 1,
4813 .rgmii = 2,
4814 .refclk = BIT(2),
4815 .hdfctlen = BIT(6),
4816 };
4817
4818 static const struct macb_config fu540_c000_config = {
4819 .caps = MACB_CAPS_GIGABIT_MODE_AVAILABLE | MACB_CAPS_JUMBO |
4820 MACB_CAPS_GEM_HAS_PTP,
4821 .dma_burst_length = 16,
4822 .clk_init = fu540_c000_clk_init,
4823 .init = fu540_c000_init,
4824 .jumbo_max_len = 10240,
4825 .usrio = &macb_default_usrio,
4826 };
4827
4828 static const struct macb_config at91sam9260_config = {
4829 .caps = MACB_CAPS_USRIO_HAS_CLKEN | MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII,
4830 .clk_init = macb_clk_init,
4831 .init = macb_init,
4832 .usrio = &macb_default_usrio,
4833 };
4834
4835 static const struct macb_config sama5d3macb_config = {
4836 .caps = MACB_CAPS_SG_DISABLED |
4837 MACB_CAPS_USRIO_HAS_CLKEN | MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII,
4838 .clk_init = macb_clk_init,
4839 .init = macb_init,
4840 .usrio = &macb_default_usrio,
4841 };
4842
4843 static const struct macb_config pc302gem_config = {
4844 .caps = MACB_CAPS_SG_DISABLED | MACB_CAPS_GIGABIT_MODE_AVAILABLE,
4845 .dma_burst_length = 16,
4846 .clk_init = macb_clk_init,
4847 .init = macb_init,
4848 .usrio = &macb_default_usrio,
4849 };
4850
4851 static const struct macb_config sama5d2_config = {
4852 .caps = MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII,
4853 .dma_burst_length = 16,
4854 .clk_init = macb_clk_init,
4855 .init = macb_init,
4856 .usrio = &macb_default_usrio,
4857 };
4858
4859 static const struct macb_config sama5d29_config = {
4860 .caps = MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII | MACB_CAPS_GEM_HAS_PTP,
4861 .dma_burst_length = 16,
4862 .clk_init = macb_clk_init,
4863 .init = macb_init,
4864 .usrio = &macb_default_usrio,
4865 };
4866
4867 static const struct macb_config sama5d3_config = {
4868 .caps = MACB_CAPS_SG_DISABLED | MACB_CAPS_GIGABIT_MODE_AVAILABLE |
4869 MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII | MACB_CAPS_JUMBO,
4870 .dma_burst_length = 16,
4871 .clk_init = macb_clk_init,
4872 .init = macb_init,
4873 .jumbo_max_len = 10240,
4874 .usrio = &macb_default_usrio,
4875 };
4876
4877 static const struct macb_config sama5d4_config = {
4878 .caps = MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII,
4879 .dma_burst_length = 4,
4880 .clk_init = macb_clk_init,
4881 .init = macb_init,
4882 .usrio = &macb_default_usrio,
4883 };
4884
4885 static const struct macb_config emac_config = {
4886 .caps = MACB_CAPS_NEEDS_RSTONUBR | MACB_CAPS_MACB_IS_EMAC,
4887 .clk_init = at91ether_clk_init,
4888 .init = at91ether_init,
4889 .usrio = &macb_default_usrio,
4890 };
4891
4892 static const struct macb_config np4_config = {
4893 .caps = MACB_CAPS_USRIO_DISABLED,
4894 .clk_init = macb_clk_init,
4895 .init = macb_init,
4896 .usrio = &macb_default_usrio,
4897 };
4898
4899 static const struct macb_config zynqmp_config = {
4900 .caps = MACB_CAPS_GIGABIT_MODE_AVAILABLE |
4901 MACB_CAPS_JUMBO |
4902 MACB_CAPS_GEM_HAS_PTP | MACB_CAPS_BD_RD_PREFETCH,
4903 .dma_burst_length = 16,
4904 .clk_init = macb_clk_init,
4905 .init = init_reset_optional,
4906 .jumbo_max_len = 10240,
4907 .usrio = &macb_default_usrio,
4908 };
4909
4910 static const struct macb_config zynq_config = {
4911 .caps = MACB_CAPS_GIGABIT_MODE_AVAILABLE | MACB_CAPS_NO_GIGABIT_HALF |
4912 MACB_CAPS_NEEDS_RSTONUBR,
4913 .dma_burst_length = 16,
4914 .clk_init = macb_clk_init,
4915 .init = macb_init,
4916 .usrio = &macb_default_usrio,
4917 };
4918
4919 static const struct macb_config mpfs_config = {
4920 .caps = MACB_CAPS_GIGABIT_MODE_AVAILABLE |
4921 MACB_CAPS_JUMBO |
4922 MACB_CAPS_GEM_HAS_PTP,
4923 .dma_burst_length = 16,
4924 .clk_init = macb_clk_init,
4925 .init = init_reset_optional,
4926 .usrio = &macb_default_usrio,
4927 .max_tx_length = 4040, /* Cadence Erratum 1686 */
4928 .jumbo_max_len = 4040,
4929 };
4930
4931 static const struct macb_config sama7g5_gem_config = {
4932 .caps = MACB_CAPS_GIGABIT_MODE_AVAILABLE | MACB_CAPS_CLK_HW_CHG |
4933 MACB_CAPS_MIIONRGMII | MACB_CAPS_GEM_HAS_PTP,
4934 .dma_burst_length = 16,
4935 .clk_init = macb_clk_init,
4936 .init = macb_init,
4937 .usrio = &sama7g5_usrio,
4938 };
4939
4940 static const struct macb_config sama7g5_emac_config = {
4941 .caps = MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII |
4942 MACB_CAPS_USRIO_HAS_CLKEN | MACB_CAPS_MIIONRGMII |
4943 MACB_CAPS_GEM_HAS_PTP,
4944 .dma_burst_length = 16,
4945 .clk_init = macb_clk_init,
4946 .init = macb_init,
4947 .usrio = &sama7g5_usrio,
4948 };
4949
4950 static const struct macb_config versal_config = {
4951 .caps = MACB_CAPS_GIGABIT_MODE_AVAILABLE | MACB_CAPS_JUMBO |
4952 MACB_CAPS_GEM_HAS_PTP | MACB_CAPS_BD_RD_PREFETCH | MACB_CAPS_NEED_TSUCLK |
4953 MACB_CAPS_QUEUE_DISABLE,
4954 .dma_burst_length = 16,
4955 .clk_init = macb_clk_init,
4956 .init = init_reset_optional,
4957 .jumbo_max_len = 10240,
4958 .usrio = &macb_default_usrio,
4959 };
4960
4961 static const struct of_device_id macb_dt_ids[] = {
4962 { .compatible = "cdns,at91sam9260-macb", .data = &at91sam9260_config },
4963 { .compatible = "cdns,macb" },
4964 { .compatible = "cdns,np4-macb", .data = &np4_config },
4965 { .compatible = "cdns,pc302-gem", .data = &pc302gem_config },
4966 { .compatible = "cdns,gem", .data = &pc302gem_config },
4967 { .compatible = "cdns,sam9x60-macb", .data = &at91sam9260_config },
4968 { .compatible = "atmel,sama5d2-gem", .data = &sama5d2_config },
4969 { .compatible = "atmel,sama5d29-gem", .data = &sama5d29_config },
4970 { .compatible = "atmel,sama5d3-gem", .data = &sama5d3_config },
4971 { .compatible = "atmel,sama5d3-macb", .data = &sama5d3macb_config },
4972 { .compatible = "atmel,sama5d4-gem", .data = &sama5d4_config },
4973 { .compatible = "cdns,at91rm9200-emac", .data = &emac_config },
4974 { .compatible = "cdns,emac", .data = &emac_config },
4975 { .compatible = "cdns,zynqmp-gem", .data = &zynqmp_config}, /* deprecated */
4976 { .compatible = "cdns,zynq-gem", .data = &zynq_config }, /* deprecated */
4977 { .compatible = "sifive,fu540-c000-gem", .data = &fu540_c000_config },
4978 { .compatible = "microchip,mpfs-macb", .data = &mpfs_config },
4979 { .compatible = "microchip,sama7g5-gem", .data = &sama7g5_gem_config },
4980 { .compatible = "microchip,sama7g5-emac", .data = &sama7g5_emac_config },
4981 { .compatible = "xlnx,zynqmp-gem", .data = &zynqmp_config},
4982 { .compatible = "xlnx,zynq-gem", .data = &zynq_config },
4983 { .compatible = "xlnx,versal-gem", .data = &versal_config},
4984 { /* sentinel */ }
4985 };
4986 MODULE_DEVICE_TABLE(of, macb_dt_ids);
4987 #endif /* CONFIG_OF */
4988
4989 static const struct macb_config default_gem_config = {
4990 .caps = MACB_CAPS_GIGABIT_MODE_AVAILABLE |
4991 MACB_CAPS_JUMBO |
4992 MACB_CAPS_GEM_HAS_PTP,
4993 .dma_burst_length = 16,
4994 .clk_init = macb_clk_init,
4995 .init = macb_init,
4996 .usrio = &macb_default_usrio,
4997 .jumbo_max_len = 10240,
4998 };
4999
macb_probe(struct platform_device * pdev)5000 static int macb_probe(struct platform_device *pdev)
5001 {
5002 const struct macb_config *macb_config = &default_gem_config;
5003 int (*clk_init)(struct platform_device *, struct clk **,
5004 struct clk **, struct clk **, struct clk **,
5005 struct clk **) = macb_config->clk_init;
5006 int (*init)(struct platform_device *) = macb_config->init;
5007 struct device_node *np = pdev->dev.of_node;
5008 struct clk *pclk, *hclk = NULL, *tx_clk = NULL, *rx_clk = NULL;
5009 struct clk *tsu_clk = NULL;
5010 unsigned int queue_mask, num_queues;
5011 bool native_io;
5012 phy_interface_t interface;
5013 struct net_device *dev;
5014 struct resource *regs;
5015 u32 wtrmrk_rst_val;
5016 void __iomem *mem;
5017 struct macb *bp;
5018 int err, val;
5019
5020 mem = devm_platform_get_and_ioremap_resource(pdev, 0, ®s);
5021 if (IS_ERR(mem))
5022 return PTR_ERR(mem);
5023
5024 if (np) {
5025 const struct of_device_id *match;
5026
5027 match = of_match_node(macb_dt_ids, np);
5028 if (match && match->data) {
5029 macb_config = match->data;
5030 clk_init = macb_config->clk_init;
5031 init = macb_config->init;
5032 }
5033 }
5034
5035 err = clk_init(pdev, &pclk, &hclk, &tx_clk, &rx_clk, &tsu_clk);
5036 if (err)
5037 return err;
5038
5039 pm_runtime_set_autosuspend_delay(&pdev->dev, MACB_PM_TIMEOUT);
5040 pm_runtime_use_autosuspend(&pdev->dev);
5041 pm_runtime_get_noresume(&pdev->dev);
5042 pm_runtime_set_active(&pdev->dev);
5043 pm_runtime_enable(&pdev->dev);
5044 native_io = hw_is_native_io(mem);
5045
5046 macb_probe_queues(mem, native_io, &queue_mask, &num_queues);
5047 dev = alloc_etherdev_mq(sizeof(*bp), num_queues);
5048 if (!dev) {
5049 err = -ENOMEM;
5050 goto err_disable_clocks;
5051 }
5052
5053 dev->base_addr = regs->start;
5054
5055 SET_NETDEV_DEV(dev, &pdev->dev);
5056
5057 bp = netdev_priv(dev);
5058 bp->pdev = pdev;
5059 bp->dev = dev;
5060 bp->regs = mem;
5061 bp->native_io = native_io;
5062 if (native_io) {
5063 bp->macb_reg_readl = hw_readl_native;
5064 bp->macb_reg_writel = hw_writel_native;
5065 } else {
5066 bp->macb_reg_readl = hw_readl;
5067 bp->macb_reg_writel = hw_writel;
5068 }
5069 bp->num_queues = num_queues;
5070 bp->queue_mask = queue_mask;
5071 if (macb_config)
5072 bp->dma_burst_length = macb_config->dma_burst_length;
5073 bp->pclk = pclk;
5074 bp->hclk = hclk;
5075 bp->tx_clk = tx_clk;
5076 bp->rx_clk = rx_clk;
5077 bp->tsu_clk = tsu_clk;
5078 if (macb_config)
5079 bp->jumbo_max_len = macb_config->jumbo_max_len;
5080
5081 if (!hw_is_gem(bp->regs, bp->native_io))
5082 bp->max_tx_length = MACB_MAX_TX_LEN;
5083 else if (macb_config->max_tx_length)
5084 bp->max_tx_length = macb_config->max_tx_length;
5085 else
5086 bp->max_tx_length = GEM_MAX_TX_LEN;
5087
5088 bp->wol = 0;
5089 device_set_wakeup_capable(&pdev->dev, 1);
5090
5091 bp->usrio = macb_config->usrio;
5092
5093 /* By default we set to partial store and forward mode for zynqmp.
5094 * Disable if not set in devicetree.
5095 */
5096 if (GEM_BFEXT(PBUF_CUTTHRU, gem_readl(bp, DCFG6))) {
5097 err = of_property_read_u32(bp->pdev->dev.of_node,
5098 "cdns,rx-watermark",
5099 &bp->rx_watermark);
5100
5101 if (!err) {
5102 /* Disable partial store and forward in case of error or
5103 * invalid watermark value
5104 */
5105 wtrmrk_rst_val = (1 << (GEM_BFEXT(RX_PBUF_ADDR, gem_readl(bp, DCFG2)))) - 1;
5106 if (bp->rx_watermark > wtrmrk_rst_val || !bp->rx_watermark) {
5107 dev_info(&bp->pdev->dev, "Invalid watermark value\n");
5108 bp->rx_watermark = 0;
5109 }
5110 }
5111 }
5112 spin_lock_init(&bp->lock);
5113
5114 /* setup capabilities */
5115 macb_configure_caps(bp, macb_config);
5116
5117 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT
5118 if (GEM_BFEXT(DAW64, gem_readl(bp, DCFG6))) {
5119 dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(44));
5120 bp->hw_dma_cap |= HW_DMA_CAP_64B;
5121 }
5122 #endif
5123 platform_set_drvdata(pdev, dev);
5124
5125 dev->irq = platform_get_irq(pdev, 0);
5126 if (dev->irq < 0) {
5127 err = dev->irq;
5128 goto err_out_free_netdev;
5129 }
5130
5131 /* MTU range: 68 - 1518 or 10240 */
5132 dev->min_mtu = GEM_MTU_MIN_SIZE;
5133 if ((bp->caps & MACB_CAPS_JUMBO) && bp->jumbo_max_len)
5134 dev->max_mtu = bp->jumbo_max_len - ETH_HLEN - ETH_FCS_LEN;
5135 else
5136 dev->max_mtu = 1536 - ETH_HLEN - ETH_FCS_LEN;
5137
5138 if (bp->caps & MACB_CAPS_BD_RD_PREFETCH) {
5139 val = GEM_BFEXT(RXBD_RDBUFF, gem_readl(bp, DCFG10));
5140 if (val)
5141 bp->rx_bd_rd_prefetch = (2 << (val - 1)) *
5142 macb_dma_desc_get_size(bp);
5143
5144 val = GEM_BFEXT(TXBD_RDBUFF, gem_readl(bp, DCFG10));
5145 if (val)
5146 bp->tx_bd_rd_prefetch = (2 << (val - 1)) *
5147 macb_dma_desc_get_size(bp);
5148 }
5149
5150 bp->rx_intr_mask = MACB_RX_INT_FLAGS;
5151 if (bp->caps & MACB_CAPS_NEEDS_RSTONUBR)
5152 bp->rx_intr_mask |= MACB_BIT(RXUBR);
5153
5154 err = of_get_ethdev_address(np, bp->dev);
5155 if (err == -EPROBE_DEFER)
5156 goto err_out_free_netdev;
5157 else if (err)
5158 macb_get_hwaddr(bp);
5159
5160 err = of_get_phy_mode(np, &interface);
5161 if (err)
5162 /* not found in DT, MII by default */
5163 bp->phy_interface = PHY_INTERFACE_MODE_MII;
5164 else
5165 bp->phy_interface = interface;
5166
5167 /* IP specific init */
5168 err = init(pdev);
5169 if (err)
5170 goto err_out_free_netdev;
5171
5172 err = macb_mii_init(bp);
5173 if (err)
5174 goto err_out_phy_exit;
5175
5176 netif_carrier_off(dev);
5177
5178 err = register_netdev(dev);
5179 if (err) {
5180 dev_err(&pdev->dev, "Cannot register net device, aborting.\n");
5181 goto err_out_unregister_mdio;
5182 }
5183
5184 INIT_WORK(&bp->hresp_err_bh_work, macb_hresp_error_task);
5185
5186 netdev_info(dev, "Cadence %s rev 0x%08x at 0x%08lx irq %d (%pM)\n",
5187 macb_is_gem(bp) ? "GEM" : "MACB", macb_readl(bp, MID),
5188 dev->base_addr, dev->irq, dev->dev_addr);
5189
5190 pm_runtime_mark_last_busy(&bp->pdev->dev);
5191 pm_runtime_put_autosuspend(&bp->pdev->dev);
5192
5193 return 0;
5194
5195 err_out_unregister_mdio:
5196 mdiobus_unregister(bp->mii_bus);
5197 mdiobus_free(bp->mii_bus);
5198
5199 err_out_phy_exit:
5200 phy_exit(bp->sgmii_phy);
5201
5202 err_out_free_netdev:
5203 free_netdev(dev);
5204
5205 err_disable_clocks:
5206 macb_clks_disable(pclk, hclk, tx_clk, rx_clk, tsu_clk);
5207 pm_runtime_disable(&pdev->dev);
5208 pm_runtime_set_suspended(&pdev->dev);
5209 pm_runtime_dont_use_autosuspend(&pdev->dev);
5210
5211 return err;
5212 }
5213
macb_remove(struct platform_device * pdev)5214 static void macb_remove(struct platform_device *pdev)
5215 {
5216 struct net_device *dev;
5217 struct macb *bp;
5218
5219 dev = platform_get_drvdata(pdev);
5220
5221 if (dev) {
5222 bp = netdev_priv(dev);
5223 phy_exit(bp->sgmii_phy);
5224 mdiobus_unregister(bp->mii_bus);
5225 mdiobus_free(bp->mii_bus);
5226
5227 unregister_netdev(dev);
5228 cancel_work_sync(&bp->hresp_err_bh_work);
5229 pm_runtime_disable(&pdev->dev);
5230 pm_runtime_dont_use_autosuspend(&pdev->dev);
5231 if (!pm_runtime_suspended(&pdev->dev)) {
5232 macb_clks_disable(bp->pclk, bp->hclk, bp->tx_clk,
5233 bp->rx_clk, bp->tsu_clk);
5234 pm_runtime_set_suspended(&pdev->dev);
5235 }
5236 phylink_destroy(bp->phylink);
5237 free_netdev(dev);
5238 }
5239 }
5240
macb_suspend(struct device * dev)5241 static int __maybe_unused macb_suspend(struct device *dev)
5242 {
5243 struct net_device *netdev = dev_get_drvdata(dev);
5244 struct macb *bp = netdev_priv(netdev);
5245 struct in_ifaddr *ifa = NULL;
5246 struct macb_queue *queue;
5247 struct in_device *idev;
5248 unsigned long flags;
5249 unsigned int q;
5250 int err;
5251 u32 tmp;
5252
5253 if (!device_may_wakeup(&bp->dev->dev))
5254 phy_exit(bp->sgmii_phy);
5255
5256 if (!netif_running(netdev))
5257 return 0;
5258
5259 if (bp->wol & MACB_WOL_ENABLED) {
5260 /* Check for IP address in WOL ARP mode */
5261 idev = __in_dev_get_rcu(bp->dev);
5262 if (idev)
5263 ifa = rcu_dereference(idev->ifa_list);
5264 if ((bp->wolopts & WAKE_ARP) && !ifa) {
5265 netdev_err(netdev, "IP address not assigned as required by WoL walk ARP\n");
5266 return -EOPNOTSUPP;
5267 }
5268 spin_lock_irqsave(&bp->lock, flags);
5269
5270 /* Disable Tx and Rx engines before disabling the queues,
5271 * this is mandatory as per the IP spec sheet
5272 */
5273 tmp = macb_readl(bp, NCR);
5274 macb_writel(bp, NCR, tmp & ~(MACB_BIT(TE) | MACB_BIT(RE)));
5275 for (q = 0, queue = bp->queues; q < bp->num_queues;
5276 ++q, ++queue) {
5277 /* Disable RX queues */
5278 if (bp->caps & MACB_CAPS_QUEUE_DISABLE) {
5279 queue_writel(queue, RBQP, MACB_BIT(QUEUE_DISABLE));
5280 } else {
5281 /* Tie off RX queues */
5282 queue_writel(queue, RBQP,
5283 lower_32_bits(bp->rx_ring_tieoff_dma));
5284 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT
5285 queue_writel(queue, RBQPH,
5286 upper_32_bits(bp->rx_ring_tieoff_dma));
5287 #endif
5288 }
5289 /* Disable all interrupts */
5290 queue_writel(queue, IDR, -1);
5291 queue_readl(queue, ISR);
5292 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE)
5293 queue_writel(queue, ISR, -1);
5294 }
5295 /* Enable Receive engine */
5296 macb_writel(bp, NCR, tmp | MACB_BIT(RE));
5297 /* Flush all status bits */
5298 macb_writel(bp, TSR, -1);
5299 macb_writel(bp, RSR, -1);
5300
5301 tmp = (bp->wolopts & WAKE_MAGIC) ? MACB_BIT(MAG) : 0;
5302 if (bp->wolopts & WAKE_ARP) {
5303 tmp |= MACB_BIT(ARP);
5304 /* write IP address into register */
5305 tmp |= MACB_BFEXT(IP, be32_to_cpu(ifa->ifa_local));
5306 }
5307
5308 /* Change interrupt handler and
5309 * Enable WoL IRQ on queue 0
5310 */
5311 devm_free_irq(dev, bp->queues[0].irq, bp->queues);
5312 if (macb_is_gem(bp)) {
5313 err = devm_request_irq(dev, bp->queues[0].irq, gem_wol_interrupt,
5314 IRQF_SHARED, netdev->name, bp->queues);
5315 if (err) {
5316 dev_err(dev,
5317 "Unable to request IRQ %d (error %d)\n",
5318 bp->queues[0].irq, err);
5319 spin_unlock_irqrestore(&bp->lock, flags);
5320 return err;
5321 }
5322 queue_writel(bp->queues, IER, GEM_BIT(WOL));
5323 gem_writel(bp, WOL, tmp);
5324 } else {
5325 err = devm_request_irq(dev, bp->queues[0].irq, macb_wol_interrupt,
5326 IRQF_SHARED, netdev->name, bp->queues);
5327 if (err) {
5328 dev_err(dev,
5329 "Unable to request IRQ %d (error %d)\n",
5330 bp->queues[0].irq, err);
5331 spin_unlock_irqrestore(&bp->lock, flags);
5332 return err;
5333 }
5334 queue_writel(bp->queues, IER, MACB_BIT(WOL));
5335 macb_writel(bp, WOL, tmp);
5336 }
5337 spin_unlock_irqrestore(&bp->lock, flags);
5338
5339 enable_irq_wake(bp->queues[0].irq);
5340 }
5341
5342 netif_device_detach(netdev);
5343 for (q = 0, queue = bp->queues; q < bp->num_queues;
5344 ++q, ++queue) {
5345 napi_disable(&queue->napi_rx);
5346 napi_disable(&queue->napi_tx);
5347 }
5348
5349 if (!(bp->wol & MACB_WOL_ENABLED)) {
5350 rtnl_lock();
5351 phylink_stop(bp->phylink);
5352 rtnl_unlock();
5353 spin_lock_irqsave(&bp->lock, flags);
5354 macb_reset_hw(bp);
5355 spin_unlock_irqrestore(&bp->lock, flags);
5356 }
5357
5358 if (!(bp->caps & MACB_CAPS_USRIO_DISABLED))
5359 bp->pm_data.usrio = macb_or_gem_readl(bp, USRIO);
5360
5361 if (netdev->hw_features & NETIF_F_NTUPLE)
5362 bp->pm_data.scrt2 = gem_readl_n(bp, ETHT, SCRT2_ETHT);
5363
5364 if (bp->ptp_info)
5365 bp->ptp_info->ptp_remove(netdev);
5366 if (!device_may_wakeup(dev))
5367 pm_runtime_force_suspend(dev);
5368
5369 return 0;
5370 }
5371
macb_resume(struct device * dev)5372 static int __maybe_unused macb_resume(struct device *dev)
5373 {
5374 struct net_device *netdev = dev_get_drvdata(dev);
5375 struct macb *bp = netdev_priv(netdev);
5376 struct macb_queue *queue;
5377 unsigned long flags;
5378 unsigned int q;
5379 int err;
5380
5381 if (!device_may_wakeup(&bp->dev->dev))
5382 phy_init(bp->sgmii_phy);
5383
5384 if (!netif_running(netdev))
5385 return 0;
5386
5387 if (!device_may_wakeup(dev))
5388 pm_runtime_force_resume(dev);
5389
5390 if (bp->wol & MACB_WOL_ENABLED) {
5391 spin_lock_irqsave(&bp->lock, flags);
5392 /* Disable WoL */
5393 if (macb_is_gem(bp)) {
5394 queue_writel(bp->queues, IDR, GEM_BIT(WOL));
5395 gem_writel(bp, WOL, 0);
5396 } else {
5397 queue_writel(bp->queues, IDR, MACB_BIT(WOL));
5398 macb_writel(bp, WOL, 0);
5399 }
5400 /* Clear ISR on queue 0 */
5401 queue_readl(bp->queues, ISR);
5402 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE)
5403 queue_writel(bp->queues, ISR, -1);
5404 /* Replace interrupt handler on queue 0 */
5405 devm_free_irq(dev, bp->queues[0].irq, bp->queues);
5406 err = devm_request_irq(dev, bp->queues[0].irq, macb_interrupt,
5407 IRQF_SHARED, netdev->name, bp->queues);
5408 if (err) {
5409 dev_err(dev,
5410 "Unable to request IRQ %d (error %d)\n",
5411 bp->queues[0].irq, err);
5412 spin_unlock_irqrestore(&bp->lock, flags);
5413 return err;
5414 }
5415 spin_unlock_irqrestore(&bp->lock, flags);
5416
5417 disable_irq_wake(bp->queues[0].irq);
5418
5419 /* Now make sure we disable phy before moving
5420 * to common restore path
5421 */
5422 rtnl_lock();
5423 phylink_stop(bp->phylink);
5424 rtnl_unlock();
5425 }
5426
5427 for (q = 0, queue = bp->queues; q < bp->num_queues;
5428 ++q, ++queue) {
5429 napi_enable(&queue->napi_rx);
5430 napi_enable(&queue->napi_tx);
5431 }
5432
5433 if (netdev->hw_features & NETIF_F_NTUPLE)
5434 gem_writel_n(bp, ETHT, SCRT2_ETHT, bp->pm_data.scrt2);
5435
5436 if (!(bp->caps & MACB_CAPS_USRIO_DISABLED))
5437 macb_or_gem_writel(bp, USRIO, bp->pm_data.usrio);
5438
5439 macb_writel(bp, NCR, MACB_BIT(MPE));
5440 macb_init_hw(bp);
5441 macb_set_rx_mode(netdev);
5442 macb_restore_features(bp);
5443 rtnl_lock();
5444
5445 phylink_start(bp->phylink);
5446 rtnl_unlock();
5447
5448 netif_device_attach(netdev);
5449 if (bp->ptp_info)
5450 bp->ptp_info->ptp_init(netdev);
5451
5452 return 0;
5453 }
5454
macb_runtime_suspend(struct device * dev)5455 static int __maybe_unused macb_runtime_suspend(struct device *dev)
5456 {
5457 struct net_device *netdev = dev_get_drvdata(dev);
5458 struct macb *bp = netdev_priv(netdev);
5459
5460 if (!(device_may_wakeup(dev)))
5461 macb_clks_disable(bp->pclk, bp->hclk, bp->tx_clk, bp->rx_clk, bp->tsu_clk);
5462 else if (!(bp->caps & MACB_CAPS_NEED_TSUCLK))
5463 macb_clks_disable(NULL, NULL, NULL, NULL, bp->tsu_clk);
5464
5465 return 0;
5466 }
5467
macb_runtime_resume(struct device * dev)5468 static int __maybe_unused macb_runtime_resume(struct device *dev)
5469 {
5470 struct net_device *netdev = dev_get_drvdata(dev);
5471 struct macb *bp = netdev_priv(netdev);
5472
5473 if (!(device_may_wakeup(dev))) {
5474 clk_prepare_enable(bp->pclk);
5475 clk_prepare_enable(bp->hclk);
5476 clk_prepare_enable(bp->tx_clk);
5477 clk_prepare_enable(bp->rx_clk);
5478 clk_prepare_enable(bp->tsu_clk);
5479 } else if (!(bp->caps & MACB_CAPS_NEED_TSUCLK)) {
5480 clk_prepare_enable(bp->tsu_clk);
5481 }
5482
5483 return 0;
5484 }
5485
5486 static const struct dev_pm_ops macb_pm_ops = {
5487 SET_SYSTEM_SLEEP_PM_OPS(macb_suspend, macb_resume)
5488 SET_RUNTIME_PM_OPS(macb_runtime_suspend, macb_runtime_resume, NULL)
5489 };
5490
5491 static struct platform_driver macb_driver = {
5492 .probe = macb_probe,
5493 .remove_new = macb_remove,
5494 .driver = {
5495 .name = "macb",
5496 .of_match_table = of_match_ptr(macb_dt_ids),
5497 .pm = &macb_pm_ops,
5498 },
5499 };
5500
5501 module_platform_driver(macb_driver);
5502
5503 MODULE_LICENSE("GPL");
5504 MODULE_DESCRIPTION("Cadence MACB/GEM Ethernet driver");
5505 MODULE_AUTHOR("Haavard Skinnemoen (Atmel)");
5506 MODULE_ALIAS("platform:macb");
5507