1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Cadence MACB/GEM Ethernet Controller driver
4 *
5 * Copyright (C) 2004-2006 Atmel Corporation
6 */
7
8 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
9 #include <linux/clk.h>
10 #include <linux/clk-provider.h>
11 #include <linux/crc32.h>
12 #include <linux/module.h>
13 #include <linux/moduleparam.h>
14 #include <linux/kernel.h>
15 #include <linux/types.h>
16 #include <linux/circ_buf.h>
17 #include <linux/slab.h>
18 #include <linux/init.h>
19 #include <linux/io.h>
20 #include <linux/gpio.h>
21 #include <linux/gpio/consumer.h>
22 #include <linux/interrupt.h>
23 #include <linux/netdevice.h>
24 #include <linux/etherdevice.h>
25 #include <linux/dma-mapping.h>
26 #include <linux/platform_device.h>
27 #include <linux/phylink.h>
28 #include <linux/of.h>
29 #include <linux/of_gpio.h>
30 #include <linux/of_mdio.h>
31 #include <linux/of_net.h>
32 #include <linux/ip.h>
33 #include <linux/udp.h>
34 #include <linux/tcp.h>
35 #include <linux/iopoll.h>
36 #include <linux/phy/phy.h>
37 #include <linux/pm_runtime.h>
38 #include <linux/ptp_classify.h>
39 #include <linux/reset.h>
40 #include <linux/firmware/xlnx-zynqmp.h>
41 #include <linux/inetdevice.h>
42 #include "macb.h"
43
44 /* This structure is only used for MACB on SiFive FU540 devices */
45 struct sifive_fu540_macb_mgmt {
46 void __iomem *reg;
47 unsigned long rate;
48 struct clk_hw hw;
49 };
50
51 #define MACB_RX_BUFFER_SIZE 128
52 #define RX_BUFFER_MULTIPLE 64 /* bytes */
53
54 #define DEFAULT_RX_RING_SIZE 512 /* must be power of 2 */
55 #define MIN_RX_RING_SIZE 64
56 #define MAX_RX_RING_SIZE 8192
57 #define RX_RING_BYTES(bp) (macb_dma_desc_get_size(bp) \
58 * (bp)->rx_ring_size)
59
60 #define DEFAULT_TX_RING_SIZE 512 /* must be power of 2 */
61 #define MIN_TX_RING_SIZE 64
62 #define MAX_TX_RING_SIZE 4096
63 #define TX_RING_BYTES(bp) (macb_dma_desc_get_size(bp) \
64 * (bp)->tx_ring_size)
65
66 /* level of occupied TX descriptors under which we wake up TX process */
67 #define MACB_TX_WAKEUP_THRESH(bp) (3 * (bp)->tx_ring_size / 4)
68
69 #define MACB_RX_INT_FLAGS (MACB_BIT(RCOMP) | MACB_BIT(ISR_ROVR))
70 #define MACB_TX_ERR_FLAGS (MACB_BIT(ISR_TUND) \
71 | MACB_BIT(ISR_RLE) \
72 | MACB_BIT(TXERR))
73 #define MACB_TX_INT_FLAGS (MACB_TX_ERR_FLAGS | MACB_BIT(TCOMP) \
74 | MACB_BIT(TXUBR))
75
76 /* Max length of transmit frame must be a multiple of 8 bytes */
77 #define MACB_TX_LEN_ALIGN 8
78 #define MACB_MAX_TX_LEN ((unsigned int)((1 << MACB_TX_FRMLEN_SIZE) - 1) & ~((unsigned int)(MACB_TX_LEN_ALIGN - 1)))
79 /* Limit maximum TX length as per Cadence TSO errata. This is to avoid a
80 * false amba_error in TX path from the DMA assuming there is not enough
81 * space in the SRAM (16KB) even when there is.
82 */
83 #define GEM_MAX_TX_LEN (unsigned int)(0x3FC0)
84
85 #define GEM_MTU_MIN_SIZE ETH_MIN_MTU
86 #define MACB_NETIF_LSO NETIF_F_TSO
87
88 #define MACB_WOL_ENABLED BIT(0)
89
90 #define HS_SPEED_10000M 4
91 #define MACB_SERDES_RATE_10G 1
92
93 /* Graceful stop timeouts in us. We should allow up to
94 * 1 frame time (10 Mbits/s, full-duplex, ignoring collisions)
95 */
96 #define MACB_HALT_TIMEOUT 14000
97 #define MACB_PM_TIMEOUT 100 /* ms */
98
99 #define MACB_MDIO_TIMEOUT 1000000 /* in usecs */
100
101 /* DMA buffer descriptor might be different size
102 * depends on hardware configuration:
103 *
104 * 1. dma address width 32 bits:
105 * word 1: 32 bit address of Data Buffer
106 * word 2: control
107 *
108 * 2. dma address width 64 bits:
109 * word 1: 32 bit address of Data Buffer
110 * word 2: control
111 * word 3: upper 32 bit address of Data Buffer
112 * word 4: unused
113 *
114 * 3. dma address width 32 bits with hardware timestamping:
115 * word 1: 32 bit address of Data Buffer
116 * word 2: control
117 * word 3: timestamp word 1
118 * word 4: timestamp word 2
119 *
120 * 4. dma address width 64 bits with hardware timestamping:
121 * word 1: 32 bit address of Data Buffer
122 * word 2: control
123 * word 3: upper 32 bit address of Data Buffer
124 * word 4: unused
125 * word 5: timestamp word 1
126 * word 6: timestamp word 2
127 */
macb_dma_desc_get_size(struct macb * bp)128 static unsigned int macb_dma_desc_get_size(struct macb *bp)
129 {
130 #ifdef MACB_EXT_DESC
131 unsigned int desc_size;
132
133 switch (bp->hw_dma_cap) {
134 case HW_DMA_CAP_64B:
135 desc_size = sizeof(struct macb_dma_desc)
136 + sizeof(struct macb_dma_desc_64);
137 break;
138 case HW_DMA_CAP_PTP:
139 desc_size = sizeof(struct macb_dma_desc)
140 + sizeof(struct macb_dma_desc_ptp);
141 break;
142 case HW_DMA_CAP_64B_PTP:
143 desc_size = sizeof(struct macb_dma_desc)
144 + sizeof(struct macb_dma_desc_64)
145 + sizeof(struct macb_dma_desc_ptp);
146 break;
147 default:
148 desc_size = sizeof(struct macb_dma_desc);
149 }
150 return desc_size;
151 #endif
152 return sizeof(struct macb_dma_desc);
153 }
154
macb_adj_dma_desc_idx(struct macb * bp,unsigned int desc_idx)155 static unsigned int macb_adj_dma_desc_idx(struct macb *bp, unsigned int desc_idx)
156 {
157 #ifdef MACB_EXT_DESC
158 switch (bp->hw_dma_cap) {
159 case HW_DMA_CAP_64B:
160 case HW_DMA_CAP_PTP:
161 desc_idx <<= 1;
162 break;
163 case HW_DMA_CAP_64B_PTP:
164 desc_idx *= 3;
165 break;
166 default:
167 break;
168 }
169 #endif
170 return desc_idx;
171 }
172
173 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT
macb_64b_desc(struct macb * bp,struct macb_dma_desc * desc)174 static struct macb_dma_desc_64 *macb_64b_desc(struct macb *bp, struct macb_dma_desc *desc)
175 {
176 return (struct macb_dma_desc_64 *)((void *)desc
177 + sizeof(struct macb_dma_desc));
178 }
179 #endif
180
181 /* Ring buffer accessors */
macb_tx_ring_wrap(struct macb * bp,unsigned int index)182 static unsigned int macb_tx_ring_wrap(struct macb *bp, unsigned int index)
183 {
184 return index & (bp->tx_ring_size - 1);
185 }
186
macb_tx_desc(struct macb_queue * queue,unsigned int index)187 static struct macb_dma_desc *macb_tx_desc(struct macb_queue *queue,
188 unsigned int index)
189 {
190 index = macb_tx_ring_wrap(queue->bp, index);
191 index = macb_adj_dma_desc_idx(queue->bp, index);
192 return &queue->tx_ring[index];
193 }
194
macb_tx_skb(struct macb_queue * queue,unsigned int index)195 static struct macb_tx_skb *macb_tx_skb(struct macb_queue *queue,
196 unsigned int index)
197 {
198 return &queue->tx_skb[macb_tx_ring_wrap(queue->bp, index)];
199 }
200
macb_tx_dma(struct macb_queue * queue,unsigned int index)201 static dma_addr_t macb_tx_dma(struct macb_queue *queue, unsigned int index)
202 {
203 dma_addr_t offset;
204
205 offset = macb_tx_ring_wrap(queue->bp, index) *
206 macb_dma_desc_get_size(queue->bp);
207
208 return queue->tx_ring_dma + offset;
209 }
210
macb_rx_ring_wrap(struct macb * bp,unsigned int index)211 static unsigned int macb_rx_ring_wrap(struct macb *bp, unsigned int index)
212 {
213 return index & (bp->rx_ring_size - 1);
214 }
215
macb_rx_desc(struct macb_queue * queue,unsigned int index)216 static struct macb_dma_desc *macb_rx_desc(struct macb_queue *queue, unsigned int index)
217 {
218 index = macb_rx_ring_wrap(queue->bp, index);
219 index = macb_adj_dma_desc_idx(queue->bp, index);
220 return &queue->rx_ring[index];
221 }
222
macb_rx_buffer(struct macb_queue * queue,unsigned int index)223 static void *macb_rx_buffer(struct macb_queue *queue, unsigned int index)
224 {
225 return queue->rx_buffers + queue->bp->rx_buffer_size *
226 macb_rx_ring_wrap(queue->bp, index);
227 }
228
229 /* I/O accessors */
hw_readl_native(struct macb * bp,int offset)230 static u32 hw_readl_native(struct macb *bp, int offset)
231 {
232 return __raw_readl(bp->regs + offset);
233 }
234
hw_writel_native(struct macb * bp,int offset,u32 value)235 static void hw_writel_native(struct macb *bp, int offset, u32 value)
236 {
237 __raw_writel(value, bp->regs + offset);
238 }
239
hw_readl(struct macb * bp,int offset)240 static u32 hw_readl(struct macb *bp, int offset)
241 {
242 return readl_relaxed(bp->regs + offset);
243 }
244
hw_writel(struct macb * bp,int offset,u32 value)245 static void hw_writel(struct macb *bp, int offset, u32 value)
246 {
247 writel_relaxed(value, bp->regs + offset);
248 }
249
250 /* Find the CPU endianness by using the loopback bit of NCR register. When the
251 * CPU is in big endian we need to program swapped mode for management
252 * descriptor access.
253 */
hw_is_native_io(void __iomem * addr)254 static bool hw_is_native_io(void __iomem *addr)
255 {
256 u32 value = MACB_BIT(LLB);
257
258 __raw_writel(value, addr + MACB_NCR);
259 value = __raw_readl(addr + MACB_NCR);
260
261 /* Write 0 back to disable everything */
262 __raw_writel(0, addr + MACB_NCR);
263
264 return value == MACB_BIT(LLB);
265 }
266
hw_is_gem(void __iomem * addr,bool native_io)267 static bool hw_is_gem(void __iomem *addr, bool native_io)
268 {
269 u32 id;
270
271 if (native_io)
272 id = __raw_readl(addr + MACB_MID);
273 else
274 id = readl_relaxed(addr + MACB_MID);
275
276 return MACB_BFEXT(IDNUM, id) >= 0x2;
277 }
278
macb_set_hwaddr(struct macb * bp)279 static void macb_set_hwaddr(struct macb *bp)
280 {
281 u32 bottom;
282 u16 top;
283
284 bottom = cpu_to_le32(*((u32 *)bp->dev->dev_addr));
285 macb_or_gem_writel(bp, SA1B, bottom);
286 top = cpu_to_le16(*((u16 *)(bp->dev->dev_addr + 4)));
287 macb_or_gem_writel(bp, SA1T, top);
288
289 if (gem_has_ptp(bp)) {
290 gem_writel(bp, RXPTPUNI, bottom);
291 gem_writel(bp, TXPTPUNI, bottom);
292 }
293
294 /* Clear unused address register sets */
295 macb_or_gem_writel(bp, SA2B, 0);
296 macb_or_gem_writel(bp, SA2T, 0);
297 macb_or_gem_writel(bp, SA3B, 0);
298 macb_or_gem_writel(bp, SA3T, 0);
299 macb_or_gem_writel(bp, SA4B, 0);
300 macb_or_gem_writel(bp, SA4T, 0);
301 }
302
macb_get_hwaddr(struct macb * bp)303 static void macb_get_hwaddr(struct macb *bp)
304 {
305 u32 bottom;
306 u16 top;
307 u8 addr[6];
308 int i;
309
310 /* Check all 4 address register for valid address */
311 for (i = 0; i < 4; i++) {
312 bottom = macb_or_gem_readl(bp, SA1B + i * 8);
313 top = macb_or_gem_readl(bp, SA1T + i * 8);
314
315 addr[0] = bottom & 0xff;
316 addr[1] = (bottom >> 8) & 0xff;
317 addr[2] = (bottom >> 16) & 0xff;
318 addr[3] = (bottom >> 24) & 0xff;
319 addr[4] = top & 0xff;
320 addr[5] = (top >> 8) & 0xff;
321
322 if (is_valid_ether_addr(addr)) {
323 eth_hw_addr_set(bp->dev, addr);
324 return;
325 }
326 }
327
328 dev_info(&bp->pdev->dev, "invalid hw address, using random\n");
329 eth_hw_addr_random(bp->dev);
330 }
331
macb_mdio_wait_for_idle(struct macb * bp)332 static int macb_mdio_wait_for_idle(struct macb *bp)
333 {
334 u32 val;
335
336 return readx_poll_timeout(MACB_READ_NSR, bp, val, val & MACB_BIT(IDLE),
337 1, MACB_MDIO_TIMEOUT);
338 }
339
macb_mdio_read_c22(struct mii_bus * bus,int mii_id,int regnum)340 static int macb_mdio_read_c22(struct mii_bus *bus, int mii_id, int regnum)
341 {
342 struct macb *bp = bus->priv;
343 int status;
344
345 status = pm_runtime_resume_and_get(&bp->pdev->dev);
346 if (status < 0)
347 goto mdio_pm_exit;
348
349 status = macb_mdio_wait_for_idle(bp);
350 if (status < 0)
351 goto mdio_read_exit;
352
353 macb_writel(bp, MAN, (MACB_BF(SOF, MACB_MAN_C22_SOF)
354 | MACB_BF(RW, MACB_MAN_C22_READ)
355 | MACB_BF(PHYA, mii_id)
356 | MACB_BF(REGA, regnum)
357 | MACB_BF(CODE, MACB_MAN_C22_CODE)));
358
359 status = macb_mdio_wait_for_idle(bp);
360 if (status < 0)
361 goto mdio_read_exit;
362
363 status = MACB_BFEXT(DATA, macb_readl(bp, MAN));
364
365 mdio_read_exit:
366 pm_runtime_mark_last_busy(&bp->pdev->dev);
367 pm_runtime_put_autosuspend(&bp->pdev->dev);
368 mdio_pm_exit:
369 return status;
370 }
371
macb_mdio_read_c45(struct mii_bus * bus,int mii_id,int devad,int regnum)372 static int macb_mdio_read_c45(struct mii_bus *bus, int mii_id, int devad,
373 int regnum)
374 {
375 struct macb *bp = bus->priv;
376 int status;
377
378 status = pm_runtime_get_sync(&bp->pdev->dev);
379 if (status < 0) {
380 pm_runtime_put_noidle(&bp->pdev->dev);
381 goto mdio_pm_exit;
382 }
383
384 status = macb_mdio_wait_for_idle(bp);
385 if (status < 0)
386 goto mdio_read_exit;
387
388 macb_writel(bp, MAN, (MACB_BF(SOF, MACB_MAN_C45_SOF)
389 | MACB_BF(RW, MACB_MAN_C45_ADDR)
390 | MACB_BF(PHYA, mii_id)
391 | MACB_BF(REGA, devad & 0x1F)
392 | MACB_BF(DATA, regnum & 0xFFFF)
393 | MACB_BF(CODE, MACB_MAN_C45_CODE)));
394
395 status = macb_mdio_wait_for_idle(bp);
396 if (status < 0)
397 goto mdio_read_exit;
398
399 macb_writel(bp, MAN, (MACB_BF(SOF, MACB_MAN_C45_SOF)
400 | MACB_BF(RW, MACB_MAN_C45_READ)
401 | MACB_BF(PHYA, mii_id)
402 | MACB_BF(REGA, devad & 0x1F)
403 | MACB_BF(CODE, MACB_MAN_C45_CODE)));
404
405 status = macb_mdio_wait_for_idle(bp);
406 if (status < 0)
407 goto mdio_read_exit;
408
409 status = MACB_BFEXT(DATA, macb_readl(bp, MAN));
410
411 mdio_read_exit:
412 pm_runtime_mark_last_busy(&bp->pdev->dev);
413 pm_runtime_put_autosuspend(&bp->pdev->dev);
414 mdio_pm_exit:
415 return status;
416 }
417
macb_mdio_write_c22(struct mii_bus * bus,int mii_id,int regnum,u16 value)418 static int macb_mdio_write_c22(struct mii_bus *bus, int mii_id, int regnum,
419 u16 value)
420 {
421 struct macb *bp = bus->priv;
422 int status;
423
424 status = pm_runtime_resume_and_get(&bp->pdev->dev);
425 if (status < 0)
426 goto mdio_pm_exit;
427
428 status = macb_mdio_wait_for_idle(bp);
429 if (status < 0)
430 goto mdio_write_exit;
431
432 macb_writel(bp, MAN, (MACB_BF(SOF, MACB_MAN_C22_SOF)
433 | MACB_BF(RW, MACB_MAN_C22_WRITE)
434 | MACB_BF(PHYA, mii_id)
435 | MACB_BF(REGA, regnum)
436 | MACB_BF(CODE, MACB_MAN_C22_CODE)
437 | MACB_BF(DATA, value)));
438
439 status = macb_mdio_wait_for_idle(bp);
440 if (status < 0)
441 goto mdio_write_exit;
442
443 mdio_write_exit:
444 pm_runtime_mark_last_busy(&bp->pdev->dev);
445 pm_runtime_put_autosuspend(&bp->pdev->dev);
446 mdio_pm_exit:
447 return status;
448 }
449
macb_mdio_write_c45(struct mii_bus * bus,int mii_id,int devad,int regnum,u16 value)450 static int macb_mdio_write_c45(struct mii_bus *bus, int mii_id,
451 int devad, int regnum,
452 u16 value)
453 {
454 struct macb *bp = bus->priv;
455 int status;
456
457 status = pm_runtime_get_sync(&bp->pdev->dev);
458 if (status < 0) {
459 pm_runtime_put_noidle(&bp->pdev->dev);
460 goto mdio_pm_exit;
461 }
462
463 status = macb_mdio_wait_for_idle(bp);
464 if (status < 0)
465 goto mdio_write_exit;
466
467 macb_writel(bp, MAN, (MACB_BF(SOF, MACB_MAN_C45_SOF)
468 | MACB_BF(RW, MACB_MAN_C45_ADDR)
469 | MACB_BF(PHYA, mii_id)
470 | MACB_BF(REGA, devad & 0x1F)
471 | MACB_BF(DATA, regnum & 0xFFFF)
472 | MACB_BF(CODE, MACB_MAN_C45_CODE)));
473
474 status = macb_mdio_wait_for_idle(bp);
475 if (status < 0)
476 goto mdio_write_exit;
477
478 macb_writel(bp, MAN, (MACB_BF(SOF, MACB_MAN_C45_SOF)
479 | MACB_BF(RW, MACB_MAN_C45_WRITE)
480 | MACB_BF(PHYA, mii_id)
481 | MACB_BF(REGA, devad & 0x1F)
482 | MACB_BF(CODE, MACB_MAN_C45_CODE)
483 | MACB_BF(DATA, value)));
484
485 status = macb_mdio_wait_for_idle(bp);
486 if (status < 0)
487 goto mdio_write_exit;
488
489 mdio_write_exit:
490 pm_runtime_mark_last_busy(&bp->pdev->dev);
491 pm_runtime_put_autosuspend(&bp->pdev->dev);
492 mdio_pm_exit:
493 return status;
494 }
495
macb_init_buffers(struct macb * bp)496 static void macb_init_buffers(struct macb *bp)
497 {
498 struct macb_queue *queue;
499 unsigned int q;
500
501 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) {
502 queue_writel(queue, RBQP, lower_32_bits(queue->rx_ring_dma));
503 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT
504 if (bp->hw_dma_cap & HW_DMA_CAP_64B)
505 queue_writel(queue, RBQPH,
506 upper_32_bits(queue->rx_ring_dma));
507 #endif
508 queue_writel(queue, TBQP, lower_32_bits(queue->tx_ring_dma));
509 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT
510 if (bp->hw_dma_cap & HW_DMA_CAP_64B)
511 queue_writel(queue, TBQPH,
512 upper_32_bits(queue->tx_ring_dma));
513 #endif
514 }
515 }
516
517 /**
518 * macb_set_tx_clk() - Set a clock to a new frequency
519 * @bp: pointer to struct macb
520 * @speed: New frequency in Hz
521 */
macb_set_tx_clk(struct macb * bp,int speed)522 static void macb_set_tx_clk(struct macb *bp, int speed)
523 {
524 long ferr, rate, rate_rounded;
525
526 if (!bp->tx_clk || (bp->caps & MACB_CAPS_CLK_HW_CHG))
527 return;
528
529 /* In case of MII the PHY is the clock master */
530 if (bp->phy_interface == PHY_INTERFACE_MODE_MII)
531 return;
532
533 switch (speed) {
534 case SPEED_10:
535 rate = 2500000;
536 break;
537 case SPEED_100:
538 rate = 25000000;
539 break;
540 case SPEED_1000:
541 rate = 125000000;
542 break;
543 default:
544 return;
545 }
546
547 rate_rounded = clk_round_rate(bp->tx_clk, rate);
548 if (rate_rounded < 0)
549 return;
550
551 /* RGMII allows 50 ppm frequency error. Test and warn if this limit
552 * is not satisfied.
553 */
554 ferr = abs(rate_rounded - rate);
555 ferr = DIV_ROUND_UP(ferr, rate / 100000);
556 if (ferr > 5)
557 netdev_warn(bp->dev,
558 "unable to generate target frequency: %ld Hz\n",
559 rate);
560
561 if (clk_set_rate(bp->tx_clk, rate_rounded))
562 netdev_err(bp->dev, "adjusting tx_clk failed.\n");
563 }
564
macb_usx_pcs_link_up(struct phylink_pcs * pcs,unsigned int neg_mode,phy_interface_t interface,int speed,int duplex)565 static void macb_usx_pcs_link_up(struct phylink_pcs *pcs, unsigned int neg_mode,
566 phy_interface_t interface, int speed,
567 int duplex)
568 {
569 struct macb *bp = container_of(pcs, struct macb, phylink_usx_pcs);
570 u32 config;
571
572 config = gem_readl(bp, USX_CONTROL);
573 config = GEM_BFINS(SERDES_RATE, MACB_SERDES_RATE_10G, config);
574 config = GEM_BFINS(USX_CTRL_SPEED, HS_SPEED_10000M, config);
575 config &= ~(GEM_BIT(TX_SCR_BYPASS) | GEM_BIT(RX_SCR_BYPASS));
576 config |= GEM_BIT(TX_EN);
577 gem_writel(bp, USX_CONTROL, config);
578 }
579
macb_usx_pcs_get_state(struct phylink_pcs * pcs,struct phylink_link_state * state)580 static void macb_usx_pcs_get_state(struct phylink_pcs *pcs,
581 struct phylink_link_state *state)
582 {
583 struct macb *bp = container_of(pcs, struct macb, phylink_usx_pcs);
584 u32 val;
585
586 state->speed = SPEED_10000;
587 state->duplex = 1;
588 state->an_complete = 1;
589
590 val = gem_readl(bp, USX_STATUS);
591 state->link = !!(val & GEM_BIT(USX_BLOCK_LOCK));
592 val = gem_readl(bp, NCFGR);
593 if (val & GEM_BIT(PAE))
594 state->pause = MLO_PAUSE_RX;
595 }
596
macb_usx_pcs_config(struct phylink_pcs * pcs,unsigned int neg_mode,phy_interface_t interface,const unsigned long * advertising,bool permit_pause_to_mac)597 static int macb_usx_pcs_config(struct phylink_pcs *pcs,
598 unsigned int neg_mode,
599 phy_interface_t interface,
600 const unsigned long *advertising,
601 bool permit_pause_to_mac)
602 {
603 struct macb *bp = container_of(pcs, struct macb, phylink_usx_pcs);
604
605 gem_writel(bp, USX_CONTROL, gem_readl(bp, USX_CONTROL) |
606 GEM_BIT(SIGNAL_OK));
607
608 return 0;
609 }
610
macb_pcs_get_state(struct phylink_pcs * pcs,struct phylink_link_state * state)611 static void macb_pcs_get_state(struct phylink_pcs *pcs,
612 struct phylink_link_state *state)
613 {
614 state->link = 0;
615 }
616
macb_pcs_an_restart(struct phylink_pcs * pcs)617 static void macb_pcs_an_restart(struct phylink_pcs *pcs)
618 {
619 /* Not supported */
620 }
621
macb_pcs_config(struct phylink_pcs * pcs,unsigned int neg_mode,phy_interface_t interface,const unsigned long * advertising,bool permit_pause_to_mac)622 static int macb_pcs_config(struct phylink_pcs *pcs,
623 unsigned int neg_mode,
624 phy_interface_t interface,
625 const unsigned long *advertising,
626 bool permit_pause_to_mac)
627 {
628 return 0;
629 }
630
631 static const struct phylink_pcs_ops macb_phylink_usx_pcs_ops = {
632 .pcs_get_state = macb_usx_pcs_get_state,
633 .pcs_config = macb_usx_pcs_config,
634 .pcs_link_up = macb_usx_pcs_link_up,
635 };
636
637 static const struct phylink_pcs_ops macb_phylink_pcs_ops = {
638 .pcs_get_state = macb_pcs_get_state,
639 .pcs_an_restart = macb_pcs_an_restart,
640 .pcs_config = macb_pcs_config,
641 };
642
macb_mac_config(struct phylink_config * config,unsigned int mode,const struct phylink_link_state * state)643 static void macb_mac_config(struct phylink_config *config, unsigned int mode,
644 const struct phylink_link_state *state)
645 {
646 struct net_device *ndev = to_net_dev(config->dev);
647 struct macb *bp = netdev_priv(ndev);
648 unsigned long flags;
649 u32 old_ctrl, ctrl;
650 u32 old_ncr, ncr;
651
652 spin_lock_irqsave(&bp->lock, flags);
653
654 old_ctrl = ctrl = macb_or_gem_readl(bp, NCFGR);
655 old_ncr = ncr = macb_or_gem_readl(bp, NCR);
656
657 if (bp->caps & MACB_CAPS_MACB_IS_EMAC) {
658 if (state->interface == PHY_INTERFACE_MODE_RMII)
659 ctrl |= MACB_BIT(RM9200_RMII);
660 } else if (macb_is_gem(bp)) {
661 ctrl &= ~(GEM_BIT(SGMIIEN) | GEM_BIT(PCSSEL));
662 ncr &= ~GEM_BIT(ENABLE_HS_MAC);
663
664 if (state->interface == PHY_INTERFACE_MODE_SGMII) {
665 ctrl |= GEM_BIT(SGMIIEN) | GEM_BIT(PCSSEL);
666 } else if (state->interface == PHY_INTERFACE_MODE_10GBASER) {
667 ctrl |= GEM_BIT(PCSSEL);
668 ncr |= GEM_BIT(ENABLE_HS_MAC);
669 } else if (bp->caps & MACB_CAPS_MIIONRGMII &&
670 bp->phy_interface == PHY_INTERFACE_MODE_MII) {
671 ncr |= MACB_BIT(MIIONRGMII);
672 }
673 }
674
675 /* Apply the new configuration, if any */
676 if (old_ctrl ^ ctrl)
677 macb_or_gem_writel(bp, NCFGR, ctrl);
678
679 if (old_ncr ^ ncr)
680 macb_or_gem_writel(bp, NCR, ncr);
681
682 /* Disable AN for SGMII fixed link configuration, enable otherwise.
683 * Must be written after PCSSEL is set in NCFGR,
684 * otherwise writes will not take effect.
685 */
686 if (macb_is_gem(bp) && state->interface == PHY_INTERFACE_MODE_SGMII) {
687 u32 pcsctrl, old_pcsctrl;
688
689 old_pcsctrl = gem_readl(bp, PCSCNTRL);
690 if (mode == MLO_AN_FIXED)
691 pcsctrl = old_pcsctrl & ~GEM_BIT(PCSAUTONEG);
692 else
693 pcsctrl = old_pcsctrl | GEM_BIT(PCSAUTONEG);
694 if (old_pcsctrl != pcsctrl)
695 gem_writel(bp, PCSCNTRL, pcsctrl);
696 }
697
698 spin_unlock_irqrestore(&bp->lock, flags);
699 }
700
macb_mac_link_down(struct phylink_config * config,unsigned int mode,phy_interface_t interface)701 static void macb_mac_link_down(struct phylink_config *config, unsigned int mode,
702 phy_interface_t interface)
703 {
704 struct net_device *ndev = to_net_dev(config->dev);
705 struct macb *bp = netdev_priv(ndev);
706 struct macb_queue *queue;
707 unsigned int q;
708 u32 ctrl;
709
710 if (!(bp->caps & MACB_CAPS_MACB_IS_EMAC))
711 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue)
712 queue_writel(queue, IDR,
713 bp->rx_intr_mask | MACB_TX_INT_FLAGS | MACB_BIT(HRESP));
714
715 /* Disable Rx and Tx */
716 ctrl = macb_readl(bp, NCR) & ~(MACB_BIT(RE) | MACB_BIT(TE));
717 macb_writel(bp, NCR, ctrl);
718
719 netif_tx_stop_all_queues(ndev);
720 }
721
macb_mac_link_up(struct phylink_config * config,struct phy_device * phy,unsigned int mode,phy_interface_t interface,int speed,int duplex,bool tx_pause,bool rx_pause)722 static void macb_mac_link_up(struct phylink_config *config,
723 struct phy_device *phy,
724 unsigned int mode, phy_interface_t interface,
725 int speed, int duplex,
726 bool tx_pause, bool rx_pause)
727 {
728 struct net_device *ndev = to_net_dev(config->dev);
729 struct macb *bp = netdev_priv(ndev);
730 struct macb_queue *queue;
731 unsigned long flags;
732 unsigned int q;
733 u32 ctrl;
734
735 spin_lock_irqsave(&bp->lock, flags);
736
737 ctrl = macb_or_gem_readl(bp, NCFGR);
738
739 ctrl &= ~(MACB_BIT(SPD) | MACB_BIT(FD));
740
741 if (speed == SPEED_100)
742 ctrl |= MACB_BIT(SPD);
743
744 if (duplex)
745 ctrl |= MACB_BIT(FD);
746
747 if (!(bp->caps & MACB_CAPS_MACB_IS_EMAC)) {
748 ctrl &= ~MACB_BIT(PAE);
749 if (macb_is_gem(bp)) {
750 ctrl &= ~GEM_BIT(GBE);
751
752 if (speed == SPEED_1000)
753 ctrl |= GEM_BIT(GBE);
754 }
755
756 if (rx_pause)
757 ctrl |= MACB_BIT(PAE);
758
759 /* Initialize rings & buffers as clearing MACB_BIT(TE) in link down
760 * cleared the pipeline and control registers.
761 */
762 bp->macbgem_ops.mog_init_rings(bp);
763 macb_init_buffers(bp);
764
765 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue)
766 queue_writel(queue, IER,
767 bp->rx_intr_mask | MACB_TX_INT_FLAGS | MACB_BIT(HRESP));
768 }
769
770 macb_or_gem_writel(bp, NCFGR, ctrl);
771
772 if (bp->phy_interface == PHY_INTERFACE_MODE_10GBASER)
773 gem_writel(bp, HS_MAC_CONFIG, GEM_BFINS(HS_MAC_SPEED, HS_SPEED_10000M,
774 gem_readl(bp, HS_MAC_CONFIG)));
775
776 spin_unlock_irqrestore(&bp->lock, flags);
777
778 if (!(bp->caps & MACB_CAPS_MACB_IS_EMAC))
779 macb_set_tx_clk(bp, speed);
780
781 /* Enable Rx and Tx; Enable PTP unicast */
782 ctrl = macb_readl(bp, NCR);
783 if (gem_has_ptp(bp))
784 ctrl |= MACB_BIT(PTPUNI);
785
786 macb_writel(bp, NCR, ctrl | MACB_BIT(RE) | MACB_BIT(TE));
787
788 netif_tx_wake_all_queues(ndev);
789 }
790
macb_mac_select_pcs(struct phylink_config * config,phy_interface_t interface)791 static struct phylink_pcs *macb_mac_select_pcs(struct phylink_config *config,
792 phy_interface_t interface)
793 {
794 struct net_device *ndev = to_net_dev(config->dev);
795 struct macb *bp = netdev_priv(ndev);
796
797 if (interface == PHY_INTERFACE_MODE_10GBASER)
798 return &bp->phylink_usx_pcs;
799 else if (interface == PHY_INTERFACE_MODE_SGMII)
800 return &bp->phylink_sgmii_pcs;
801 else
802 return NULL;
803 }
804
805 static const struct phylink_mac_ops macb_phylink_ops = {
806 .mac_select_pcs = macb_mac_select_pcs,
807 .mac_config = macb_mac_config,
808 .mac_link_down = macb_mac_link_down,
809 .mac_link_up = macb_mac_link_up,
810 };
811
macb_phy_handle_exists(struct device_node * dn)812 static bool macb_phy_handle_exists(struct device_node *dn)
813 {
814 dn = of_parse_phandle(dn, "phy-handle", 0);
815 of_node_put(dn);
816 return dn != NULL;
817 }
818
macb_phylink_connect(struct macb * bp)819 static int macb_phylink_connect(struct macb *bp)
820 {
821 struct device_node *dn = bp->pdev->dev.of_node;
822 struct net_device *dev = bp->dev;
823 struct phy_device *phydev;
824 int ret;
825
826 if (dn)
827 ret = phylink_of_phy_connect(bp->phylink, dn, 0);
828
829 if (!dn || (ret && !macb_phy_handle_exists(dn))) {
830 phydev = phy_find_first(bp->mii_bus);
831 if (!phydev) {
832 netdev_err(dev, "no PHY found\n");
833 return -ENXIO;
834 }
835
836 /* attach the mac to the phy */
837 ret = phylink_connect_phy(bp->phylink, phydev);
838 }
839
840 if (ret) {
841 netdev_err(dev, "Could not attach PHY (%d)\n", ret);
842 return ret;
843 }
844
845 phylink_start(bp->phylink);
846
847 return 0;
848 }
849
macb_get_pcs_fixed_state(struct phylink_config * config,struct phylink_link_state * state)850 static void macb_get_pcs_fixed_state(struct phylink_config *config,
851 struct phylink_link_state *state)
852 {
853 struct net_device *ndev = to_net_dev(config->dev);
854 struct macb *bp = netdev_priv(ndev);
855
856 state->link = (macb_readl(bp, NSR) & MACB_BIT(NSR_LINK)) != 0;
857 }
858
859 /* based on au1000_eth. c*/
macb_mii_probe(struct net_device * dev)860 static int macb_mii_probe(struct net_device *dev)
861 {
862 struct macb *bp = netdev_priv(dev);
863
864 bp->phylink_sgmii_pcs.ops = &macb_phylink_pcs_ops;
865 bp->phylink_sgmii_pcs.neg_mode = true;
866 bp->phylink_usx_pcs.ops = &macb_phylink_usx_pcs_ops;
867 bp->phylink_usx_pcs.neg_mode = true;
868
869 bp->phylink_config.dev = &dev->dev;
870 bp->phylink_config.type = PHYLINK_NETDEV;
871 bp->phylink_config.mac_managed_pm = true;
872
873 if (bp->phy_interface == PHY_INTERFACE_MODE_SGMII) {
874 bp->phylink_config.poll_fixed_state = true;
875 bp->phylink_config.get_fixed_state = macb_get_pcs_fixed_state;
876 }
877
878 bp->phylink_config.mac_capabilities = MAC_ASYM_PAUSE |
879 MAC_10 | MAC_100;
880
881 __set_bit(PHY_INTERFACE_MODE_MII,
882 bp->phylink_config.supported_interfaces);
883 __set_bit(PHY_INTERFACE_MODE_RMII,
884 bp->phylink_config.supported_interfaces);
885
886 /* Determine what modes are supported */
887 if (macb_is_gem(bp) && (bp->caps & MACB_CAPS_GIGABIT_MODE_AVAILABLE)) {
888 bp->phylink_config.mac_capabilities |= MAC_1000FD;
889 if (!(bp->caps & MACB_CAPS_NO_GIGABIT_HALF))
890 bp->phylink_config.mac_capabilities |= MAC_1000HD;
891
892 __set_bit(PHY_INTERFACE_MODE_GMII,
893 bp->phylink_config.supported_interfaces);
894 phy_interface_set_rgmii(bp->phylink_config.supported_interfaces);
895
896 if (bp->caps & MACB_CAPS_PCS)
897 __set_bit(PHY_INTERFACE_MODE_SGMII,
898 bp->phylink_config.supported_interfaces);
899
900 if (bp->caps & MACB_CAPS_HIGH_SPEED) {
901 __set_bit(PHY_INTERFACE_MODE_10GBASER,
902 bp->phylink_config.supported_interfaces);
903 bp->phylink_config.mac_capabilities |= MAC_10000FD;
904 }
905 }
906
907 bp->phylink = phylink_create(&bp->phylink_config, bp->pdev->dev.fwnode,
908 bp->phy_interface, &macb_phylink_ops);
909 if (IS_ERR(bp->phylink)) {
910 netdev_err(dev, "Could not create a phylink instance (%ld)\n",
911 PTR_ERR(bp->phylink));
912 return PTR_ERR(bp->phylink);
913 }
914
915 return 0;
916 }
917
macb_mdiobus_register(struct macb * bp,struct device_node * mdio_np)918 static int macb_mdiobus_register(struct macb *bp, struct device_node *mdio_np)
919 {
920 struct device_node *child, *np = bp->pdev->dev.of_node;
921
922 /* If we have a child named mdio, probe it instead of looking for PHYs
923 * directly under the MAC node
924 */
925 if (mdio_np)
926 return of_mdiobus_register(bp->mii_bus, mdio_np);
927
928 /* Only create the PHY from the device tree if at least one PHY is
929 * described. Otherwise scan the entire MDIO bus. We do this to support
930 * old device tree that did not follow the best practices and did not
931 * describe their network PHYs.
932 */
933 for_each_available_child_of_node(np, child)
934 if (of_mdiobus_child_is_phy(child)) {
935 /* The loop increments the child refcount,
936 * decrement it before returning.
937 */
938 of_node_put(child);
939
940 return of_mdiobus_register(bp->mii_bus, np);
941 }
942
943 return mdiobus_register(bp->mii_bus);
944 }
945
macb_mii_init(struct macb * bp)946 static int macb_mii_init(struct macb *bp)
947 {
948 struct device_node *mdio_np, *np = bp->pdev->dev.of_node;
949 int err = -ENXIO;
950
951 /* With fixed-link, we don't need to register the MDIO bus,
952 * except if we have a child named "mdio" in the device tree.
953 * In that case, some devices may be attached to the MACB's MDIO bus.
954 */
955 mdio_np = of_get_child_by_name(np, "mdio");
956 if (!mdio_np && of_phy_is_fixed_link(np))
957 return macb_mii_probe(bp->dev);
958
959 /* Enable management port */
960 macb_writel(bp, NCR, MACB_BIT(MPE));
961
962 bp->mii_bus = mdiobus_alloc();
963 if (!bp->mii_bus) {
964 err = -ENOMEM;
965 goto err_out;
966 }
967
968 bp->mii_bus->name = "MACB_mii_bus";
969 bp->mii_bus->read = &macb_mdio_read_c22;
970 bp->mii_bus->write = &macb_mdio_write_c22;
971 bp->mii_bus->read_c45 = &macb_mdio_read_c45;
972 bp->mii_bus->write_c45 = &macb_mdio_write_c45;
973 snprintf(bp->mii_bus->id, MII_BUS_ID_SIZE, "%s-%x",
974 bp->pdev->name, bp->pdev->id);
975 bp->mii_bus->priv = bp;
976 bp->mii_bus->parent = &bp->pdev->dev;
977
978 dev_set_drvdata(&bp->dev->dev, bp->mii_bus);
979
980 err = macb_mdiobus_register(bp, mdio_np);
981 if (err)
982 goto err_out_free_mdiobus;
983
984 err = macb_mii_probe(bp->dev);
985 if (err)
986 goto err_out_unregister_bus;
987
988 return 0;
989
990 err_out_unregister_bus:
991 mdiobus_unregister(bp->mii_bus);
992 err_out_free_mdiobus:
993 mdiobus_free(bp->mii_bus);
994 err_out:
995 of_node_put(mdio_np);
996
997 return err;
998 }
999
macb_update_stats(struct macb * bp)1000 static void macb_update_stats(struct macb *bp)
1001 {
1002 u32 *p = &bp->hw_stats.macb.rx_pause_frames;
1003 u32 *end = &bp->hw_stats.macb.tx_pause_frames + 1;
1004 int offset = MACB_PFR;
1005
1006 WARN_ON((unsigned long)(end - p - 1) != (MACB_TPF - MACB_PFR) / 4);
1007
1008 for (; p < end; p++, offset += 4)
1009 *p += bp->macb_reg_readl(bp, offset);
1010 }
1011
macb_halt_tx(struct macb * bp)1012 static int macb_halt_tx(struct macb *bp)
1013 {
1014 unsigned long halt_time, timeout;
1015 u32 status;
1016
1017 macb_writel(bp, NCR, macb_readl(bp, NCR) | MACB_BIT(THALT));
1018
1019 timeout = jiffies + usecs_to_jiffies(MACB_HALT_TIMEOUT);
1020 do {
1021 halt_time = jiffies;
1022 status = macb_readl(bp, TSR);
1023 if (!(status & MACB_BIT(TGO)))
1024 return 0;
1025
1026 udelay(250);
1027 } while (time_before(halt_time, timeout));
1028
1029 return -ETIMEDOUT;
1030 }
1031
macb_tx_unmap(struct macb * bp,struct macb_tx_skb * tx_skb,int budget)1032 static void macb_tx_unmap(struct macb *bp, struct macb_tx_skb *tx_skb, int budget)
1033 {
1034 if (tx_skb->mapping) {
1035 if (tx_skb->mapped_as_page)
1036 dma_unmap_page(&bp->pdev->dev, tx_skb->mapping,
1037 tx_skb->size, DMA_TO_DEVICE);
1038 else
1039 dma_unmap_single(&bp->pdev->dev, tx_skb->mapping,
1040 tx_skb->size, DMA_TO_DEVICE);
1041 tx_skb->mapping = 0;
1042 }
1043
1044 if (tx_skb->skb) {
1045 napi_consume_skb(tx_skb->skb, budget);
1046 tx_skb->skb = NULL;
1047 }
1048 }
1049
macb_set_addr(struct macb * bp,struct macb_dma_desc * desc,dma_addr_t addr)1050 static void macb_set_addr(struct macb *bp, struct macb_dma_desc *desc, dma_addr_t addr)
1051 {
1052 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT
1053 struct macb_dma_desc_64 *desc_64;
1054
1055 if (bp->hw_dma_cap & HW_DMA_CAP_64B) {
1056 desc_64 = macb_64b_desc(bp, desc);
1057 desc_64->addrh = upper_32_bits(addr);
1058 /* The low bits of RX address contain the RX_USED bit, clearing
1059 * of which allows packet RX. Make sure the high bits are also
1060 * visible to HW at that point.
1061 */
1062 dma_wmb();
1063 }
1064 #endif
1065 desc->addr = lower_32_bits(addr);
1066 }
1067
macb_get_addr(struct macb * bp,struct macb_dma_desc * desc)1068 static dma_addr_t macb_get_addr(struct macb *bp, struct macb_dma_desc *desc)
1069 {
1070 dma_addr_t addr = 0;
1071 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT
1072 struct macb_dma_desc_64 *desc_64;
1073
1074 if (bp->hw_dma_cap & HW_DMA_CAP_64B) {
1075 desc_64 = macb_64b_desc(bp, desc);
1076 addr = ((u64)(desc_64->addrh) << 32);
1077 }
1078 #endif
1079 addr |= MACB_BF(RX_WADDR, MACB_BFEXT(RX_WADDR, desc->addr));
1080 #ifdef CONFIG_MACB_USE_HWSTAMP
1081 if (bp->hw_dma_cap & HW_DMA_CAP_PTP)
1082 addr &= ~GEM_BIT(DMA_RXVALID);
1083 #endif
1084 return addr;
1085 }
1086
macb_tx_error_task(struct work_struct * work)1087 static void macb_tx_error_task(struct work_struct *work)
1088 {
1089 struct macb_queue *queue = container_of(work, struct macb_queue,
1090 tx_error_task);
1091 bool halt_timeout = false;
1092 struct macb *bp = queue->bp;
1093 struct macb_tx_skb *tx_skb;
1094 struct macb_dma_desc *desc;
1095 struct sk_buff *skb;
1096 unsigned int tail;
1097 unsigned long flags;
1098
1099 netdev_vdbg(bp->dev, "macb_tx_error_task: q = %u, t = %u, h = %u\n",
1100 (unsigned int)(queue - bp->queues),
1101 queue->tx_tail, queue->tx_head);
1102
1103 /* Prevent the queue NAPI TX poll from running, as it calls
1104 * macb_tx_complete(), which in turn may call netif_wake_subqueue().
1105 * As explained below, we have to halt the transmission before updating
1106 * TBQP registers so we call netif_tx_stop_all_queues() to notify the
1107 * network engine about the macb/gem being halted.
1108 */
1109 napi_disable(&queue->napi_tx);
1110 spin_lock_irqsave(&bp->lock, flags);
1111
1112 /* Make sure nobody is trying to queue up new packets */
1113 netif_tx_stop_all_queues(bp->dev);
1114
1115 /* Stop transmission now
1116 * (in case we have just queued new packets)
1117 * macb/gem must be halted to write TBQP register
1118 */
1119 if (macb_halt_tx(bp)) {
1120 netdev_err(bp->dev, "BUG: halt tx timed out\n");
1121 macb_writel(bp, NCR, macb_readl(bp, NCR) & (~MACB_BIT(TE)));
1122 halt_timeout = true;
1123 }
1124
1125 /* Treat frames in TX queue including the ones that caused the error.
1126 * Free transmit buffers in upper layer.
1127 */
1128 for (tail = queue->tx_tail; tail != queue->tx_head; tail++) {
1129 u32 ctrl;
1130
1131 desc = macb_tx_desc(queue, tail);
1132 ctrl = desc->ctrl;
1133 tx_skb = macb_tx_skb(queue, tail);
1134 skb = tx_skb->skb;
1135
1136 if (ctrl & MACB_BIT(TX_USED)) {
1137 /* skb is set for the last buffer of the frame */
1138 while (!skb) {
1139 macb_tx_unmap(bp, tx_skb, 0);
1140 tail++;
1141 tx_skb = macb_tx_skb(queue, tail);
1142 skb = tx_skb->skb;
1143 }
1144
1145 /* ctrl still refers to the first buffer descriptor
1146 * since it's the only one written back by the hardware
1147 */
1148 if (!(ctrl & MACB_BIT(TX_BUF_EXHAUSTED))) {
1149 netdev_vdbg(bp->dev, "txerr skb %u (data %p) TX complete\n",
1150 macb_tx_ring_wrap(bp, tail),
1151 skb->data);
1152 bp->dev->stats.tx_packets++;
1153 queue->stats.tx_packets++;
1154 bp->dev->stats.tx_bytes += skb->len;
1155 queue->stats.tx_bytes += skb->len;
1156 }
1157 } else {
1158 /* "Buffers exhausted mid-frame" errors may only happen
1159 * if the driver is buggy, so complain loudly about
1160 * those. Statistics are updated by hardware.
1161 */
1162 if (ctrl & MACB_BIT(TX_BUF_EXHAUSTED))
1163 netdev_err(bp->dev,
1164 "BUG: TX buffers exhausted mid-frame\n");
1165
1166 desc->ctrl = ctrl | MACB_BIT(TX_USED);
1167 }
1168
1169 macb_tx_unmap(bp, tx_skb, 0);
1170 }
1171
1172 /* Set end of TX queue */
1173 desc = macb_tx_desc(queue, 0);
1174 macb_set_addr(bp, desc, 0);
1175 desc->ctrl = MACB_BIT(TX_USED);
1176
1177 /* Make descriptor updates visible to hardware */
1178 wmb();
1179
1180 /* Reinitialize the TX desc queue */
1181 queue_writel(queue, TBQP, lower_32_bits(queue->tx_ring_dma));
1182 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT
1183 if (bp->hw_dma_cap & HW_DMA_CAP_64B)
1184 queue_writel(queue, TBQPH, upper_32_bits(queue->tx_ring_dma));
1185 #endif
1186 /* Make TX ring reflect state of hardware */
1187 queue->tx_head = 0;
1188 queue->tx_tail = 0;
1189
1190 /* Housework before enabling TX IRQ */
1191 macb_writel(bp, TSR, macb_readl(bp, TSR));
1192 queue_writel(queue, IER, MACB_TX_INT_FLAGS);
1193
1194 if (halt_timeout)
1195 macb_writel(bp, NCR, macb_readl(bp, NCR) | MACB_BIT(TE));
1196
1197 /* Now we are ready to start transmission again */
1198 netif_tx_start_all_queues(bp->dev);
1199 macb_writel(bp, NCR, macb_readl(bp, NCR) | MACB_BIT(TSTART));
1200
1201 spin_unlock_irqrestore(&bp->lock, flags);
1202 napi_enable(&queue->napi_tx);
1203 }
1204
ptp_one_step_sync(struct sk_buff * skb)1205 static bool ptp_one_step_sync(struct sk_buff *skb)
1206 {
1207 struct ptp_header *hdr;
1208 unsigned int ptp_class;
1209 u8 msgtype;
1210
1211 /* No need to parse packet if PTP TS is not involved */
1212 if (likely(!(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)))
1213 goto not_oss;
1214
1215 /* Identify and return whether PTP one step sync is being processed */
1216 ptp_class = ptp_classify_raw(skb);
1217 if (ptp_class == PTP_CLASS_NONE)
1218 goto not_oss;
1219
1220 hdr = ptp_parse_header(skb, ptp_class);
1221 if (!hdr)
1222 goto not_oss;
1223
1224 if (hdr->flag_field[0] & PTP_FLAG_TWOSTEP)
1225 goto not_oss;
1226
1227 msgtype = ptp_get_msgtype(hdr, ptp_class);
1228 if (msgtype == PTP_MSGTYPE_SYNC)
1229 return true;
1230
1231 not_oss:
1232 return false;
1233 }
1234
macb_tx_complete(struct macb_queue * queue,int budget)1235 static int macb_tx_complete(struct macb_queue *queue, int budget)
1236 {
1237 struct macb *bp = queue->bp;
1238 u16 queue_index = queue - bp->queues;
1239 unsigned int tail;
1240 unsigned int head;
1241 int packets = 0;
1242
1243 spin_lock(&queue->tx_ptr_lock);
1244 head = queue->tx_head;
1245 for (tail = queue->tx_tail; tail != head && packets < budget; tail++) {
1246 struct macb_tx_skb *tx_skb;
1247 struct sk_buff *skb;
1248 struct macb_dma_desc *desc;
1249 u32 ctrl;
1250
1251 desc = macb_tx_desc(queue, tail);
1252
1253 /* Make hw descriptor updates visible to CPU */
1254 rmb();
1255
1256 ctrl = desc->ctrl;
1257
1258 /* TX_USED bit is only set by hardware on the very first buffer
1259 * descriptor of the transmitted frame.
1260 */
1261 if (!(ctrl & MACB_BIT(TX_USED)))
1262 break;
1263
1264 /* Process all buffers of the current transmitted frame */
1265 for (;; tail++) {
1266 tx_skb = macb_tx_skb(queue, tail);
1267 skb = tx_skb->skb;
1268
1269 /* First, update TX stats if needed */
1270 if (skb) {
1271 if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) &&
1272 !ptp_one_step_sync(skb))
1273 gem_ptp_do_txstamp(bp, skb, desc);
1274
1275 netdev_vdbg(bp->dev, "skb %u (data %p) TX complete\n",
1276 macb_tx_ring_wrap(bp, tail),
1277 skb->data);
1278 bp->dev->stats.tx_packets++;
1279 queue->stats.tx_packets++;
1280 bp->dev->stats.tx_bytes += skb->len;
1281 queue->stats.tx_bytes += skb->len;
1282 packets++;
1283 }
1284
1285 /* Now we can safely release resources */
1286 macb_tx_unmap(bp, tx_skb, budget);
1287
1288 /* skb is set only for the last buffer of the frame.
1289 * WARNING: at this point skb has been freed by
1290 * macb_tx_unmap().
1291 */
1292 if (skb)
1293 break;
1294 }
1295 }
1296
1297 queue->tx_tail = tail;
1298 if (__netif_subqueue_stopped(bp->dev, queue_index) &&
1299 CIRC_CNT(queue->tx_head, queue->tx_tail,
1300 bp->tx_ring_size) <= MACB_TX_WAKEUP_THRESH(bp))
1301 netif_wake_subqueue(bp->dev, queue_index);
1302 spin_unlock(&queue->tx_ptr_lock);
1303
1304 return packets;
1305 }
1306
gem_rx_refill(struct macb_queue * queue)1307 static void gem_rx_refill(struct macb_queue *queue)
1308 {
1309 unsigned int entry;
1310 struct sk_buff *skb;
1311 dma_addr_t paddr;
1312 struct macb *bp = queue->bp;
1313 struct macb_dma_desc *desc;
1314
1315 while (CIRC_SPACE(queue->rx_prepared_head, queue->rx_tail,
1316 bp->rx_ring_size) > 0) {
1317 entry = macb_rx_ring_wrap(bp, queue->rx_prepared_head);
1318
1319 /* Make hw descriptor updates visible to CPU */
1320 rmb();
1321
1322 desc = macb_rx_desc(queue, entry);
1323
1324 if (!queue->rx_skbuff[entry]) {
1325 /* allocate sk_buff for this free entry in ring */
1326 skb = netdev_alloc_skb(bp->dev, bp->rx_buffer_size);
1327 if (unlikely(!skb)) {
1328 netdev_err(bp->dev,
1329 "Unable to allocate sk_buff\n");
1330 break;
1331 }
1332
1333 /* now fill corresponding descriptor entry */
1334 paddr = dma_map_single(&bp->pdev->dev, skb->data,
1335 bp->rx_buffer_size,
1336 DMA_FROM_DEVICE);
1337 if (dma_mapping_error(&bp->pdev->dev, paddr)) {
1338 dev_kfree_skb(skb);
1339 break;
1340 }
1341
1342 queue->rx_skbuff[entry] = skb;
1343
1344 if (entry == bp->rx_ring_size - 1)
1345 paddr |= MACB_BIT(RX_WRAP);
1346 desc->ctrl = 0;
1347 /* Setting addr clears RX_USED and allows reception,
1348 * make sure ctrl is cleared first to avoid a race.
1349 */
1350 dma_wmb();
1351 macb_set_addr(bp, desc, paddr);
1352
1353 /* properly align Ethernet header */
1354 skb_reserve(skb, NET_IP_ALIGN);
1355 } else {
1356 desc->ctrl = 0;
1357 dma_wmb();
1358 desc->addr &= ~MACB_BIT(RX_USED);
1359 }
1360 queue->rx_prepared_head++;
1361 }
1362
1363 /* Make descriptor updates visible to hardware */
1364 wmb();
1365
1366 netdev_vdbg(bp->dev, "rx ring: queue: %p, prepared head %d, tail %d\n",
1367 queue, queue->rx_prepared_head, queue->rx_tail);
1368 }
1369
1370 /* Mark DMA descriptors from begin up to and not including end as unused */
discard_partial_frame(struct macb_queue * queue,unsigned int begin,unsigned int end)1371 static void discard_partial_frame(struct macb_queue *queue, unsigned int begin,
1372 unsigned int end)
1373 {
1374 unsigned int frag;
1375
1376 for (frag = begin; frag != end; frag++) {
1377 struct macb_dma_desc *desc = macb_rx_desc(queue, frag);
1378
1379 desc->addr &= ~MACB_BIT(RX_USED);
1380 }
1381
1382 /* Make descriptor updates visible to hardware */
1383 wmb();
1384
1385 /* When this happens, the hardware stats registers for
1386 * whatever caused this is updated, so we don't have to record
1387 * anything.
1388 */
1389 }
1390
gem_rx(struct macb_queue * queue,struct napi_struct * napi,int budget)1391 static int gem_rx(struct macb_queue *queue, struct napi_struct *napi,
1392 int budget)
1393 {
1394 struct macb *bp = queue->bp;
1395 unsigned int len;
1396 unsigned int entry;
1397 struct sk_buff *skb;
1398 struct macb_dma_desc *desc;
1399 int count = 0;
1400
1401 while (count < budget) {
1402 u32 ctrl;
1403 dma_addr_t addr;
1404 bool rxused;
1405
1406 entry = macb_rx_ring_wrap(bp, queue->rx_tail);
1407 desc = macb_rx_desc(queue, entry);
1408
1409 /* Make hw descriptor updates visible to CPU */
1410 rmb();
1411
1412 rxused = (desc->addr & MACB_BIT(RX_USED)) ? true : false;
1413 addr = macb_get_addr(bp, desc);
1414
1415 if (!rxused)
1416 break;
1417
1418 /* Ensure ctrl is at least as up-to-date as rxused */
1419 dma_rmb();
1420
1421 ctrl = desc->ctrl;
1422
1423 queue->rx_tail++;
1424 count++;
1425
1426 if (!(ctrl & MACB_BIT(RX_SOF) && ctrl & MACB_BIT(RX_EOF))) {
1427 netdev_err(bp->dev,
1428 "not whole frame pointed by descriptor\n");
1429 bp->dev->stats.rx_dropped++;
1430 queue->stats.rx_dropped++;
1431 break;
1432 }
1433 skb = queue->rx_skbuff[entry];
1434 if (unlikely(!skb)) {
1435 netdev_err(bp->dev,
1436 "inconsistent Rx descriptor chain\n");
1437 bp->dev->stats.rx_dropped++;
1438 queue->stats.rx_dropped++;
1439 break;
1440 }
1441 /* now everything is ready for receiving packet */
1442 queue->rx_skbuff[entry] = NULL;
1443 len = ctrl & bp->rx_frm_len_mask;
1444
1445 netdev_vdbg(bp->dev, "gem_rx %u (len %u)\n", entry, len);
1446
1447 skb_put(skb, len);
1448 dma_unmap_single(&bp->pdev->dev, addr,
1449 bp->rx_buffer_size, DMA_FROM_DEVICE);
1450
1451 skb->protocol = eth_type_trans(skb, bp->dev);
1452 skb_checksum_none_assert(skb);
1453 if (bp->dev->features & NETIF_F_RXCSUM &&
1454 !(bp->dev->flags & IFF_PROMISC) &&
1455 GEM_BFEXT(RX_CSUM, ctrl) & GEM_RX_CSUM_CHECKED_MASK)
1456 skb->ip_summed = CHECKSUM_UNNECESSARY;
1457
1458 bp->dev->stats.rx_packets++;
1459 queue->stats.rx_packets++;
1460 bp->dev->stats.rx_bytes += skb->len;
1461 queue->stats.rx_bytes += skb->len;
1462
1463 gem_ptp_do_rxstamp(bp, skb, desc);
1464
1465 #if defined(DEBUG) && defined(VERBOSE_DEBUG)
1466 netdev_vdbg(bp->dev, "received skb of length %u, csum: %08x\n",
1467 skb->len, skb->csum);
1468 print_hex_dump(KERN_DEBUG, " mac: ", DUMP_PREFIX_ADDRESS, 16, 1,
1469 skb_mac_header(skb), 16, true);
1470 print_hex_dump(KERN_DEBUG, "data: ", DUMP_PREFIX_ADDRESS, 16, 1,
1471 skb->data, 32, true);
1472 #endif
1473
1474 napi_gro_receive(napi, skb);
1475 }
1476
1477 gem_rx_refill(queue);
1478
1479 return count;
1480 }
1481
macb_rx_frame(struct macb_queue * queue,struct napi_struct * napi,unsigned int first_frag,unsigned int last_frag)1482 static int macb_rx_frame(struct macb_queue *queue, struct napi_struct *napi,
1483 unsigned int first_frag, unsigned int last_frag)
1484 {
1485 unsigned int len;
1486 unsigned int frag;
1487 unsigned int offset;
1488 struct sk_buff *skb;
1489 struct macb_dma_desc *desc;
1490 struct macb *bp = queue->bp;
1491
1492 desc = macb_rx_desc(queue, last_frag);
1493 len = desc->ctrl & bp->rx_frm_len_mask;
1494
1495 netdev_vdbg(bp->dev, "macb_rx_frame frags %u - %u (len %u)\n",
1496 macb_rx_ring_wrap(bp, first_frag),
1497 macb_rx_ring_wrap(bp, last_frag), len);
1498
1499 /* The ethernet header starts NET_IP_ALIGN bytes into the
1500 * first buffer. Since the header is 14 bytes, this makes the
1501 * payload word-aligned.
1502 *
1503 * Instead of calling skb_reserve(NET_IP_ALIGN), we just copy
1504 * the two padding bytes into the skb so that we avoid hitting
1505 * the slowpath in memcpy(), and pull them off afterwards.
1506 */
1507 skb = netdev_alloc_skb(bp->dev, len + NET_IP_ALIGN);
1508 if (!skb) {
1509 bp->dev->stats.rx_dropped++;
1510 for (frag = first_frag; ; frag++) {
1511 desc = macb_rx_desc(queue, frag);
1512 desc->addr &= ~MACB_BIT(RX_USED);
1513 if (frag == last_frag)
1514 break;
1515 }
1516
1517 /* Make descriptor updates visible to hardware */
1518 wmb();
1519
1520 return 1;
1521 }
1522
1523 offset = 0;
1524 len += NET_IP_ALIGN;
1525 skb_checksum_none_assert(skb);
1526 skb_put(skb, len);
1527
1528 for (frag = first_frag; ; frag++) {
1529 unsigned int frag_len = bp->rx_buffer_size;
1530
1531 if (offset + frag_len > len) {
1532 if (unlikely(frag != last_frag)) {
1533 dev_kfree_skb_any(skb);
1534 return -1;
1535 }
1536 frag_len = len - offset;
1537 }
1538 skb_copy_to_linear_data_offset(skb, offset,
1539 macb_rx_buffer(queue, frag),
1540 frag_len);
1541 offset += bp->rx_buffer_size;
1542 desc = macb_rx_desc(queue, frag);
1543 desc->addr &= ~MACB_BIT(RX_USED);
1544
1545 if (frag == last_frag)
1546 break;
1547 }
1548
1549 /* Make descriptor updates visible to hardware */
1550 wmb();
1551
1552 __skb_pull(skb, NET_IP_ALIGN);
1553 skb->protocol = eth_type_trans(skb, bp->dev);
1554
1555 bp->dev->stats.rx_packets++;
1556 bp->dev->stats.rx_bytes += skb->len;
1557 netdev_vdbg(bp->dev, "received skb of length %u, csum: %08x\n",
1558 skb->len, skb->csum);
1559 napi_gro_receive(napi, skb);
1560
1561 return 0;
1562 }
1563
macb_init_rx_ring(struct macb_queue * queue)1564 static inline void macb_init_rx_ring(struct macb_queue *queue)
1565 {
1566 struct macb *bp = queue->bp;
1567 dma_addr_t addr;
1568 struct macb_dma_desc *desc = NULL;
1569 int i;
1570
1571 addr = queue->rx_buffers_dma;
1572 for (i = 0; i < bp->rx_ring_size; i++) {
1573 desc = macb_rx_desc(queue, i);
1574 macb_set_addr(bp, desc, addr);
1575 desc->ctrl = 0;
1576 addr += bp->rx_buffer_size;
1577 }
1578 desc->addr |= MACB_BIT(RX_WRAP);
1579 queue->rx_tail = 0;
1580 }
1581
macb_rx(struct macb_queue * queue,struct napi_struct * napi,int budget)1582 static int macb_rx(struct macb_queue *queue, struct napi_struct *napi,
1583 int budget)
1584 {
1585 struct macb *bp = queue->bp;
1586 bool reset_rx_queue = false;
1587 int received = 0;
1588 unsigned int tail;
1589 int first_frag = -1;
1590
1591 for (tail = queue->rx_tail; budget > 0; tail++) {
1592 struct macb_dma_desc *desc = macb_rx_desc(queue, tail);
1593 u32 ctrl;
1594
1595 /* Make hw descriptor updates visible to CPU */
1596 rmb();
1597
1598 if (!(desc->addr & MACB_BIT(RX_USED)))
1599 break;
1600
1601 /* Ensure ctrl is at least as up-to-date as addr */
1602 dma_rmb();
1603
1604 ctrl = desc->ctrl;
1605
1606 if (ctrl & MACB_BIT(RX_SOF)) {
1607 if (first_frag != -1)
1608 discard_partial_frame(queue, first_frag, tail);
1609 first_frag = tail;
1610 }
1611
1612 if (ctrl & MACB_BIT(RX_EOF)) {
1613 int dropped;
1614
1615 if (unlikely(first_frag == -1)) {
1616 reset_rx_queue = true;
1617 continue;
1618 }
1619
1620 dropped = macb_rx_frame(queue, napi, first_frag, tail);
1621 first_frag = -1;
1622 if (unlikely(dropped < 0)) {
1623 reset_rx_queue = true;
1624 continue;
1625 }
1626 if (!dropped) {
1627 received++;
1628 budget--;
1629 }
1630 }
1631 }
1632
1633 if (unlikely(reset_rx_queue)) {
1634 unsigned long flags;
1635 u32 ctrl;
1636
1637 netdev_err(bp->dev, "RX queue corruption: reset it\n");
1638
1639 spin_lock_irqsave(&bp->lock, flags);
1640
1641 ctrl = macb_readl(bp, NCR);
1642 macb_writel(bp, NCR, ctrl & ~MACB_BIT(RE));
1643
1644 macb_init_rx_ring(queue);
1645 queue_writel(queue, RBQP, queue->rx_ring_dma);
1646
1647 macb_writel(bp, NCR, ctrl | MACB_BIT(RE));
1648
1649 spin_unlock_irqrestore(&bp->lock, flags);
1650 return received;
1651 }
1652
1653 if (first_frag != -1)
1654 queue->rx_tail = first_frag;
1655 else
1656 queue->rx_tail = tail;
1657
1658 return received;
1659 }
1660
macb_rx_pending(struct macb_queue * queue)1661 static bool macb_rx_pending(struct macb_queue *queue)
1662 {
1663 struct macb *bp = queue->bp;
1664 unsigned int entry;
1665 struct macb_dma_desc *desc;
1666
1667 entry = macb_rx_ring_wrap(bp, queue->rx_tail);
1668 desc = macb_rx_desc(queue, entry);
1669
1670 /* Make hw descriptor updates visible to CPU */
1671 rmb();
1672
1673 return (desc->addr & MACB_BIT(RX_USED)) != 0;
1674 }
1675
macb_rx_poll(struct napi_struct * napi,int budget)1676 static int macb_rx_poll(struct napi_struct *napi, int budget)
1677 {
1678 struct macb_queue *queue = container_of(napi, struct macb_queue, napi_rx);
1679 struct macb *bp = queue->bp;
1680 int work_done;
1681
1682 work_done = bp->macbgem_ops.mog_rx(queue, napi, budget);
1683
1684 netdev_vdbg(bp->dev, "RX poll: queue = %u, work_done = %d, budget = %d\n",
1685 (unsigned int)(queue - bp->queues), work_done, budget);
1686
1687 if (work_done < budget && napi_complete_done(napi, work_done)) {
1688 queue_writel(queue, IER, bp->rx_intr_mask);
1689
1690 /* Packet completions only seem to propagate to raise
1691 * interrupts when interrupts are enabled at the time, so if
1692 * packets were received while interrupts were disabled,
1693 * they will not cause another interrupt to be generated when
1694 * interrupts are re-enabled.
1695 * Check for this case here to avoid losing a wakeup. This can
1696 * potentially race with the interrupt handler doing the same
1697 * actions if an interrupt is raised just after enabling them,
1698 * but this should be harmless.
1699 */
1700 if (macb_rx_pending(queue)) {
1701 queue_writel(queue, IDR, bp->rx_intr_mask);
1702 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE)
1703 queue_writel(queue, ISR, MACB_BIT(RCOMP));
1704 netdev_vdbg(bp->dev, "poll: packets pending, reschedule\n");
1705 napi_schedule(napi);
1706 }
1707 }
1708
1709 /* TODO: Handle errors */
1710
1711 return work_done;
1712 }
1713
macb_tx_restart(struct macb_queue * queue)1714 static void macb_tx_restart(struct macb_queue *queue)
1715 {
1716 struct macb *bp = queue->bp;
1717 unsigned int head_idx, tbqp;
1718
1719 spin_lock(&queue->tx_ptr_lock);
1720
1721 if (queue->tx_head == queue->tx_tail)
1722 goto out_tx_ptr_unlock;
1723
1724 tbqp = queue_readl(queue, TBQP) / macb_dma_desc_get_size(bp);
1725 tbqp = macb_adj_dma_desc_idx(bp, macb_tx_ring_wrap(bp, tbqp));
1726 head_idx = macb_adj_dma_desc_idx(bp, macb_tx_ring_wrap(bp, queue->tx_head));
1727
1728 if (tbqp == head_idx)
1729 goto out_tx_ptr_unlock;
1730
1731 spin_lock_irq(&bp->lock);
1732 macb_writel(bp, NCR, macb_readl(bp, NCR) | MACB_BIT(TSTART));
1733 spin_unlock_irq(&bp->lock);
1734
1735 out_tx_ptr_unlock:
1736 spin_unlock(&queue->tx_ptr_lock);
1737 }
1738
macb_tx_complete_pending(struct macb_queue * queue)1739 static bool macb_tx_complete_pending(struct macb_queue *queue)
1740 {
1741 bool retval = false;
1742
1743 spin_lock(&queue->tx_ptr_lock);
1744 if (queue->tx_head != queue->tx_tail) {
1745 /* Make hw descriptor updates visible to CPU */
1746 rmb();
1747
1748 if (macb_tx_desc(queue, queue->tx_tail)->ctrl & MACB_BIT(TX_USED))
1749 retval = true;
1750 }
1751 spin_unlock(&queue->tx_ptr_lock);
1752 return retval;
1753 }
1754
macb_tx_poll(struct napi_struct * napi,int budget)1755 static int macb_tx_poll(struct napi_struct *napi, int budget)
1756 {
1757 struct macb_queue *queue = container_of(napi, struct macb_queue, napi_tx);
1758 struct macb *bp = queue->bp;
1759 int work_done;
1760
1761 work_done = macb_tx_complete(queue, budget);
1762
1763 rmb(); // ensure txubr_pending is up to date
1764 if (queue->txubr_pending) {
1765 queue->txubr_pending = false;
1766 netdev_vdbg(bp->dev, "poll: tx restart\n");
1767 macb_tx_restart(queue);
1768 }
1769
1770 netdev_vdbg(bp->dev, "TX poll: queue = %u, work_done = %d, budget = %d\n",
1771 (unsigned int)(queue - bp->queues), work_done, budget);
1772
1773 if (work_done < budget && napi_complete_done(napi, work_done)) {
1774 queue_writel(queue, IER, MACB_BIT(TCOMP));
1775
1776 /* Packet completions only seem to propagate to raise
1777 * interrupts when interrupts are enabled at the time, so if
1778 * packets were sent while interrupts were disabled,
1779 * they will not cause another interrupt to be generated when
1780 * interrupts are re-enabled.
1781 * Check for this case here to avoid losing a wakeup. This can
1782 * potentially race with the interrupt handler doing the same
1783 * actions if an interrupt is raised just after enabling them,
1784 * but this should be harmless.
1785 */
1786 if (macb_tx_complete_pending(queue)) {
1787 queue_writel(queue, IDR, MACB_BIT(TCOMP));
1788 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE)
1789 queue_writel(queue, ISR, MACB_BIT(TCOMP));
1790 netdev_vdbg(bp->dev, "TX poll: packets pending, reschedule\n");
1791 napi_schedule(napi);
1792 }
1793 }
1794
1795 return work_done;
1796 }
1797
macb_hresp_error_task(struct work_struct * work)1798 static void macb_hresp_error_task(struct work_struct *work)
1799 {
1800 struct macb *bp = from_work(bp, work, hresp_err_bh_work);
1801 struct net_device *dev = bp->dev;
1802 struct macb_queue *queue;
1803 unsigned int q;
1804 u32 ctrl;
1805
1806 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) {
1807 queue_writel(queue, IDR, bp->rx_intr_mask |
1808 MACB_TX_INT_FLAGS |
1809 MACB_BIT(HRESP));
1810 }
1811 ctrl = macb_readl(bp, NCR);
1812 ctrl &= ~(MACB_BIT(RE) | MACB_BIT(TE));
1813 macb_writel(bp, NCR, ctrl);
1814
1815 netif_tx_stop_all_queues(dev);
1816 netif_carrier_off(dev);
1817
1818 bp->macbgem_ops.mog_init_rings(bp);
1819
1820 /* Initialize TX and RX buffers */
1821 macb_init_buffers(bp);
1822
1823 /* Enable interrupts */
1824 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue)
1825 queue_writel(queue, IER,
1826 bp->rx_intr_mask |
1827 MACB_TX_INT_FLAGS |
1828 MACB_BIT(HRESP));
1829
1830 ctrl |= MACB_BIT(RE) | MACB_BIT(TE);
1831 macb_writel(bp, NCR, ctrl);
1832
1833 netif_carrier_on(dev);
1834 netif_tx_start_all_queues(dev);
1835 }
1836
macb_wol_interrupt(int irq,void * dev_id)1837 static irqreturn_t macb_wol_interrupt(int irq, void *dev_id)
1838 {
1839 struct macb_queue *queue = dev_id;
1840 struct macb *bp = queue->bp;
1841 u32 status;
1842
1843 status = queue_readl(queue, ISR);
1844
1845 if (unlikely(!status))
1846 return IRQ_NONE;
1847
1848 spin_lock(&bp->lock);
1849
1850 if (status & MACB_BIT(WOL)) {
1851 queue_writel(queue, IDR, MACB_BIT(WOL));
1852 macb_writel(bp, WOL, 0);
1853 netdev_vdbg(bp->dev, "MACB WoL: queue = %u, isr = 0x%08lx\n",
1854 (unsigned int)(queue - bp->queues),
1855 (unsigned long)status);
1856 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE)
1857 queue_writel(queue, ISR, MACB_BIT(WOL));
1858 pm_wakeup_event(&bp->pdev->dev, 0);
1859 }
1860
1861 spin_unlock(&bp->lock);
1862
1863 return IRQ_HANDLED;
1864 }
1865
gem_wol_interrupt(int irq,void * dev_id)1866 static irqreturn_t gem_wol_interrupt(int irq, void *dev_id)
1867 {
1868 struct macb_queue *queue = dev_id;
1869 struct macb *bp = queue->bp;
1870 u32 status;
1871
1872 status = queue_readl(queue, ISR);
1873
1874 if (unlikely(!status))
1875 return IRQ_NONE;
1876
1877 spin_lock(&bp->lock);
1878
1879 if (status & GEM_BIT(WOL)) {
1880 queue_writel(queue, IDR, GEM_BIT(WOL));
1881 gem_writel(bp, WOL, 0);
1882 netdev_vdbg(bp->dev, "GEM WoL: queue = %u, isr = 0x%08lx\n",
1883 (unsigned int)(queue - bp->queues),
1884 (unsigned long)status);
1885 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE)
1886 queue_writel(queue, ISR, GEM_BIT(WOL));
1887 pm_wakeup_event(&bp->pdev->dev, 0);
1888 }
1889
1890 spin_unlock(&bp->lock);
1891
1892 return IRQ_HANDLED;
1893 }
1894
macb_interrupt(int irq,void * dev_id)1895 static irqreturn_t macb_interrupt(int irq, void *dev_id)
1896 {
1897 struct macb_queue *queue = dev_id;
1898 struct macb *bp = queue->bp;
1899 struct net_device *dev = bp->dev;
1900 u32 status, ctrl;
1901
1902 status = queue_readl(queue, ISR);
1903
1904 if (unlikely(!status))
1905 return IRQ_NONE;
1906
1907 spin_lock(&bp->lock);
1908
1909 while (status) {
1910 /* close possible race with dev_close */
1911 if (unlikely(!netif_running(dev))) {
1912 queue_writel(queue, IDR, -1);
1913 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE)
1914 queue_writel(queue, ISR, -1);
1915 break;
1916 }
1917
1918 netdev_vdbg(bp->dev, "queue = %u, isr = 0x%08lx\n",
1919 (unsigned int)(queue - bp->queues),
1920 (unsigned long)status);
1921
1922 if (status & bp->rx_intr_mask) {
1923 /* There's no point taking any more interrupts
1924 * until we have processed the buffers. The
1925 * scheduling call may fail if the poll routine
1926 * is already scheduled, so disable interrupts
1927 * now.
1928 */
1929 queue_writel(queue, IDR, bp->rx_intr_mask);
1930 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE)
1931 queue_writel(queue, ISR, MACB_BIT(RCOMP));
1932
1933 if (napi_schedule_prep(&queue->napi_rx)) {
1934 netdev_vdbg(bp->dev, "scheduling RX softirq\n");
1935 __napi_schedule(&queue->napi_rx);
1936 }
1937 }
1938
1939 if (status & (MACB_BIT(TCOMP) |
1940 MACB_BIT(TXUBR))) {
1941 queue_writel(queue, IDR, MACB_BIT(TCOMP));
1942 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE)
1943 queue_writel(queue, ISR, MACB_BIT(TCOMP) |
1944 MACB_BIT(TXUBR));
1945
1946 if (status & MACB_BIT(TXUBR)) {
1947 queue->txubr_pending = true;
1948 wmb(); // ensure softirq can see update
1949 }
1950
1951 if (napi_schedule_prep(&queue->napi_tx)) {
1952 netdev_vdbg(bp->dev, "scheduling TX softirq\n");
1953 __napi_schedule(&queue->napi_tx);
1954 }
1955 }
1956
1957 if (unlikely(status & (MACB_TX_ERR_FLAGS))) {
1958 queue_writel(queue, IDR, MACB_TX_INT_FLAGS);
1959 schedule_work(&queue->tx_error_task);
1960
1961 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE)
1962 queue_writel(queue, ISR, MACB_TX_ERR_FLAGS);
1963
1964 break;
1965 }
1966
1967 /* Link change detection isn't possible with RMII, so we'll
1968 * add that if/when we get our hands on a full-blown MII PHY.
1969 */
1970
1971 /* There is a hardware issue under heavy load where DMA can
1972 * stop, this causes endless "used buffer descriptor read"
1973 * interrupts but it can be cleared by re-enabling RX. See
1974 * the at91rm9200 manual, section 41.3.1 or the Zynq manual
1975 * section 16.7.4 for details. RXUBR is only enabled for
1976 * these two versions.
1977 */
1978 if (status & MACB_BIT(RXUBR)) {
1979 ctrl = macb_readl(bp, NCR);
1980 macb_writel(bp, NCR, ctrl & ~MACB_BIT(RE));
1981 wmb();
1982 macb_writel(bp, NCR, ctrl | MACB_BIT(RE));
1983
1984 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE)
1985 queue_writel(queue, ISR, MACB_BIT(RXUBR));
1986 }
1987
1988 if (status & MACB_BIT(ISR_ROVR)) {
1989 /* We missed at least one packet */
1990 if (macb_is_gem(bp))
1991 bp->hw_stats.gem.rx_overruns++;
1992 else
1993 bp->hw_stats.macb.rx_overruns++;
1994
1995 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE)
1996 queue_writel(queue, ISR, MACB_BIT(ISR_ROVR));
1997 }
1998
1999 if (status & MACB_BIT(HRESP)) {
2000 queue_work(system_bh_wq, &bp->hresp_err_bh_work);
2001 netdev_err(dev, "DMA bus error: HRESP not OK\n");
2002
2003 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE)
2004 queue_writel(queue, ISR, MACB_BIT(HRESP));
2005 }
2006 status = queue_readl(queue, ISR);
2007 }
2008
2009 spin_unlock(&bp->lock);
2010
2011 return IRQ_HANDLED;
2012 }
2013
2014 #ifdef CONFIG_NET_POLL_CONTROLLER
2015 /* Polling receive - used by netconsole and other diagnostic tools
2016 * to allow network i/o with interrupts disabled.
2017 */
macb_poll_controller(struct net_device * dev)2018 static void macb_poll_controller(struct net_device *dev)
2019 {
2020 struct macb *bp = netdev_priv(dev);
2021 struct macb_queue *queue;
2022 unsigned long flags;
2023 unsigned int q;
2024
2025 local_irq_save(flags);
2026 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue)
2027 macb_interrupt(dev->irq, queue);
2028 local_irq_restore(flags);
2029 }
2030 #endif
2031
macb_tx_map(struct macb * bp,struct macb_queue * queue,struct sk_buff * skb,unsigned int hdrlen)2032 static unsigned int macb_tx_map(struct macb *bp,
2033 struct macb_queue *queue,
2034 struct sk_buff *skb,
2035 unsigned int hdrlen)
2036 {
2037 dma_addr_t mapping;
2038 unsigned int len, entry, i, tx_head = queue->tx_head;
2039 struct macb_tx_skb *tx_skb = NULL;
2040 struct macb_dma_desc *desc;
2041 unsigned int offset, size, count = 0;
2042 unsigned int f, nr_frags = skb_shinfo(skb)->nr_frags;
2043 unsigned int eof = 1, mss_mfs = 0;
2044 u32 ctrl, lso_ctrl = 0, seq_ctrl = 0;
2045
2046 /* LSO */
2047 if (skb_shinfo(skb)->gso_size != 0) {
2048 if (ip_hdr(skb)->protocol == IPPROTO_UDP)
2049 /* UDP - UFO */
2050 lso_ctrl = MACB_LSO_UFO_ENABLE;
2051 else
2052 /* TCP - TSO */
2053 lso_ctrl = MACB_LSO_TSO_ENABLE;
2054 }
2055
2056 /* First, map non-paged data */
2057 len = skb_headlen(skb);
2058
2059 /* first buffer length */
2060 size = hdrlen;
2061
2062 offset = 0;
2063 while (len) {
2064 entry = macb_tx_ring_wrap(bp, tx_head);
2065 tx_skb = &queue->tx_skb[entry];
2066
2067 mapping = dma_map_single(&bp->pdev->dev,
2068 skb->data + offset,
2069 size, DMA_TO_DEVICE);
2070 if (dma_mapping_error(&bp->pdev->dev, mapping))
2071 goto dma_error;
2072
2073 /* Save info to properly release resources */
2074 tx_skb->skb = NULL;
2075 tx_skb->mapping = mapping;
2076 tx_skb->size = size;
2077 tx_skb->mapped_as_page = false;
2078
2079 len -= size;
2080 offset += size;
2081 count++;
2082 tx_head++;
2083
2084 size = min(len, bp->max_tx_length);
2085 }
2086
2087 /* Then, map paged data from fragments */
2088 for (f = 0; f < nr_frags; f++) {
2089 const skb_frag_t *frag = &skb_shinfo(skb)->frags[f];
2090
2091 len = skb_frag_size(frag);
2092 offset = 0;
2093 while (len) {
2094 size = min(len, bp->max_tx_length);
2095 entry = macb_tx_ring_wrap(bp, tx_head);
2096 tx_skb = &queue->tx_skb[entry];
2097
2098 mapping = skb_frag_dma_map(&bp->pdev->dev, frag,
2099 offset, size, DMA_TO_DEVICE);
2100 if (dma_mapping_error(&bp->pdev->dev, mapping))
2101 goto dma_error;
2102
2103 /* Save info to properly release resources */
2104 tx_skb->skb = NULL;
2105 tx_skb->mapping = mapping;
2106 tx_skb->size = size;
2107 tx_skb->mapped_as_page = true;
2108
2109 len -= size;
2110 offset += size;
2111 count++;
2112 tx_head++;
2113 }
2114 }
2115
2116 /* Should never happen */
2117 if (unlikely(!tx_skb)) {
2118 netdev_err(bp->dev, "BUG! empty skb!\n");
2119 return 0;
2120 }
2121
2122 /* This is the last buffer of the frame: save socket buffer */
2123 tx_skb->skb = skb;
2124
2125 /* Update TX ring: update buffer descriptors in reverse order
2126 * to avoid race condition
2127 */
2128
2129 /* Set 'TX_USED' bit in buffer descriptor at tx_head position
2130 * to set the end of TX queue
2131 */
2132 i = tx_head;
2133 entry = macb_tx_ring_wrap(bp, i);
2134 ctrl = MACB_BIT(TX_USED);
2135 desc = macb_tx_desc(queue, entry);
2136 desc->ctrl = ctrl;
2137
2138 if (lso_ctrl) {
2139 if (lso_ctrl == MACB_LSO_UFO_ENABLE)
2140 /* include header and FCS in value given to h/w */
2141 mss_mfs = skb_shinfo(skb)->gso_size +
2142 skb_transport_offset(skb) +
2143 ETH_FCS_LEN;
2144 else /* TSO */ {
2145 mss_mfs = skb_shinfo(skb)->gso_size;
2146 /* TCP Sequence Number Source Select
2147 * can be set only for TSO
2148 */
2149 seq_ctrl = 0;
2150 }
2151 }
2152
2153 do {
2154 i--;
2155 entry = macb_tx_ring_wrap(bp, i);
2156 tx_skb = &queue->tx_skb[entry];
2157 desc = macb_tx_desc(queue, entry);
2158
2159 ctrl = (u32)tx_skb->size;
2160 if (eof) {
2161 ctrl |= MACB_BIT(TX_LAST);
2162 eof = 0;
2163 }
2164 if (unlikely(entry == (bp->tx_ring_size - 1)))
2165 ctrl |= MACB_BIT(TX_WRAP);
2166
2167 /* First descriptor is header descriptor */
2168 if (i == queue->tx_head) {
2169 ctrl |= MACB_BF(TX_LSO, lso_ctrl);
2170 ctrl |= MACB_BF(TX_TCP_SEQ_SRC, seq_ctrl);
2171 if ((bp->dev->features & NETIF_F_HW_CSUM) &&
2172 skb->ip_summed != CHECKSUM_PARTIAL && !lso_ctrl &&
2173 !ptp_one_step_sync(skb))
2174 ctrl |= MACB_BIT(TX_NOCRC);
2175 } else
2176 /* Only set MSS/MFS on payload descriptors
2177 * (second or later descriptor)
2178 */
2179 ctrl |= MACB_BF(MSS_MFS, mss_mfs);
2180
2181 /* Set TX buffer descriptor */
2182 macb_set_addr(bp, desc, tx_skb->mapping);
2183 /* desc->addr must be visible to hardware before clearing
2184 * 'TX_USED' bit in desc->ctrl.
2185 */
2186 wmb();
2187 desc->ctrl = ctrl;
2188 } while (i != queue->tx_head);
2189
2190 queue->tx_head = tx_head;
2191
2192 return count;
2193
2194 dma_error:
2195 netdev_err(bp->dev, "TX DMA map failed\n");
2196
2197 for (i = queue->tx_head; i != tx_head; i++) {
2198 tx_skb = macb_tx_skb(queue, i);
2199
2200 macb_tx_unmap(bp, tx_skb, 0);
2201 }
2202
2203 return 0;
2204 }
2205
macb_features_check(struct sk_buff * skb,struct net_device * dev,netdev_features_t features)2206 static netdev_features_t macb_features_check(struct sk_buff *skb,
2207 struct net_device *dev,
2208 netdev_features_t features)
2209 {
2210 unsigned int nr_frags, f;
2211 unsigned int hdrlen;
2212
2213 /* Validate LSO compatibility */
2214
2215 /* there is only one buffer or protocol is not UDP */
2216 if (!skb_is_nonlinear(skb) || (ip_hdr(skb)->protocol != IPPROTO_UDP))
2217 return features;
2218
2219 /* length of header */
2220 hdrlen = skb_transport_offset(skb);
2221
2222 /* For UFO only:
2223 * When software supplies two or more payload buffers all payload buffers
2224 * apart from the last must be a multiple of 8 bytes in size.
2225 */
2226 if (!IS_ALIGNED(skb_headlen(skb) - hdrlen, MACB_TX_LEN_ALIGN))
2227 return features & ~MACB_NETIF_LSO;
2228
2229 nr_frags = skb_shinfo(skb)->nr_frags;
2230 /* No need to check last fragment */
2231 nr_frags--;
2232 for (f = 0; f < nr_frags; f++) {
2233 const skb_frag_t *frag = &skb_shinfo(skb)->frags[f];
2234
2235 if (!IS_ALIGNED(skb_frag_size(frag), MACB_TX_LEN_ALIGN))
2236 return features & ~MACB_NETIF_LSO;
2237 }
2238 return features;
2239 }
2240
macb_clear_csum(struct sk_buff * skb)2241 static inline int macb_clear_csum(struct sk_buff *skb)
2242 {
2243 /* no change for packets without checksum offloading */
2244 if (skb->ip_summed != CHECKSUM_PARTIAL)
2245 return 0;
2246
2247 /* make sure we can modify the header */
2248 if (unlikely(skb_cow_head(skb, 0)))
2249 return -1;
2250
2251 /* initialize checksum field
2252 * This is required - at least for Zynq, which otherwise calculates
2253 * wrong UDP header checksums for UDP packets with UDP data len <=2
2254 */
2255 *(__sum16 *)(skb_checksum_start(skb) + skb->csum_offset) = 0;
2256 return 0;
2257 }
2258
macb_pad_and_fcs(struct sk_buff ** skb,struct net_device * ndev)2259 static int macb_pad_and_fcs(struct sk_buff **skb, struct net_device *ndev)
2260 {
2261 bool cloned = skb_cloned(*skb) || skb_header_cloned(*skb) ||
2262 skb_is_nonlinear(*skb);
2263 int padlen = ETH_ZLEN - (*skb)->len;
2264 int tailroom = skb_tailroom(*skb);
2265 struct sk_buff *nskb;
2266 u32 fcs;
2267
2268 if (!(ndev->features & NETIF_F_HW_CSUM) ||
2269 !((*skb)->ip_summed != CHECKSUM_PARTIAL) ||
2270 skb_shinfo(*skb)->gso_size || ptp_one_step_sync(*skb))
2271 return 0;
2272
2273 if (padlen <= 0) {
2274 /* FCS could be appeded to tailroom. */
2275 if (tailroom >= ETH_FCS_LEN)
2276 goto add_fcs;
2277 /* No room for FCS, need to reallocate skb. */
2278 else
2279 padlen = ETH_FCS_LEN;
2280 } else {
2281 /* Add room for FCS. */
2282 padlen += ETH_FCS_LEN;
2283 }
2284
2285 if (cloned || tailroom < padlen) {
2286 nskb = skb_copy_expand(*skb, 0, padlen, GFP_ATOMIC);
2287 if (!nskb)
2288 return -ENOMEM;
2289
2290 dev_consume_skb_any(*skb);
2291 *skb = nskb;
2292 }
2293
2294 if (padlen > ETH_FCS_LEN)
2295 skb_put_zero(*skb, padlen - ETH_FCS_LEN);
2296
2297 add_fcs:
2298 /* set FCS to packet */
2299 fcs = crc32_le(~0, (*skb)->data, (*skb)->len);
2300 fcs = ~fcs;
2301
2302 skb_put_u8(*skb, fcs & 0xff);
2303 skb_put_u8(*skb, (fcs >> 8) & 0xff);
2304 skb_put_u8(*skb, (fcs >> 16) & 0xff);
2305 skb_put_u8(*skb, (fcs >> 24) & 0xff);
2306
2307 return 0;
2308 }
2309
macb_start_xmit(struct sk_buff * skb,struct net_device * dev)2310 static netdev_tx_t macb_start_xmit(struct sk_buff *skb, struct net_device *dev)
2311 {
2312 u16 queue_index = skb_get_queue_mapping(skb);
2313 struct macb *bp = netdev_priv(dev);
2314 struct macb_queue *queue = &bp->queues[queue_index];
2315 unsigned int desc_cnt, nr_frags, frag_size, f;
2316 unsigned int hdrlen;
2317 bool is_lso;
2318 netdev_tx_t ret = NETDEV_TX_OK;
2319
2320 if (macb_clear_csum(skb)) {
2321 dev_kfree_skb_any(skb);
2322 return ret;
2323 }
2324
2325 if (macb_pad_and_fcs(&skb, dev)) {
2326 dev_kfree_skb_any(skb);
2327 return ret;
2328 }
2329
2330 #ifdef CONFIG_MACB_USE_HWSTAMP
2331 if ((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) &&
2332 (bp->hw_dma_cap & HW_DMA_CAP_PTP))
2333 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
2334 #endif
2335
2336 is_lso = (skb_shinfo(skb)->gso_size != 0);
2337
2338 if (is_lso) {
2339 /* length of headers */
2340 if (ip_hdr(skb)->protocol == IPPROTO_UDP)
2341 /* only queue eth + ip headers separately for UDP */
2342 hdrlen = skb_transport_offset(skb);
2343 else
2344 hdrlen = skb_tcp_all_headers(skb);
2345 if (skb_headlen(skb) < hdrlen) {
2346 netdev_err(bp->dev, "Error - LSO headers fragmented!!!\n");
2347 /* if this is required, would need to copy to single buffer */
2348 return NETDEV_TX_BUSY;
2349 }
2350 } else
2351 hdrlen = min(skb_headlen(skb), bp->max_tx_length);
2352
2353 #if defined(DEBUG) && defined(VERBOSE_DEBUG)
2354 netdev_vdbg(bp->dev,
2355 "start_xmit: queue %hu len %u head %p data %p tail %p end %p\n",
2356 queue_index, skb->len, skb->head, skb->data,
2357 skb_tail_pointer(skb), skb_end_pointer(skb));
2358 print_hex_dump(KERN_DEBUG, "data: ", DUMP_PREFIX_OFFSET, 16, 1,
2359 skb->data, 16, true);
2360 #endif
2361
2362 /* Count how many TX buffer descriptors are needed to send this
2363 * socket buffer: skb fragments of jumbo frames may need to be
2364 * split into many buffer descriptors.
2365 */
2366 if (is_lso && (skb_headlen(skb) > hdrlen))
2367 /* extra header descriptor if also payload in first buffer */
2368 desc_cnt = DIV_ROUND_UP((skb_headlen(skb) - hdrlen), bp->max_tx_length) + 1;
2369 else
2370 desc_cnt = DIV_ROUND_UP(skb_headlen(skb), bp->max_tx_length);
2371 nr_frags = skb_shinfo(skb)->nr_frags;
2372 for (f = 0; f < nr_frags; f++) {
2373 frag_size = skb_frag_size(&skb_shinfo(skb)->frags[f]);
2374 desc_cnt += DIV_ROUND_UP(frag_size, bp->max_tx_length);
2375 }
2376
2377 spin_lock_bh(&queue->tx_ptr_lock);
2378
2379 /* This is a hard error, log it. */
2380 if (CIRC_SPACE(queue->tx_head, queue->tx_tail,
2381 bp->tx_ring_size) < desc_cnt) {
2382 netif_stop_subqueue(dev, queue_index);
2383 netdev_dbg(bp->dev, "tx_head = %u, tx_tail = %u\n",
2384 queue->tx_head, queue->tx_tail);
2385 ret = NETDEV_TX_BUSY;
2386 goto unlock;
2387 }
2388
2389 /* Map socket buffer for DMA transfer */
2390 if (!macb_tx_map(bp, queue, skb, hdrlen)) {
2391 dev_kfree_skb_any(skb);
2392 goto unlock;
2393 }
2394
2395 /* Make newly initialized descriptor visible to hardware */
2396 wmb();
2397 skb_tx_timestamp(skb);
2398
2399 spin_lock_irq(&bp->lock);
2400 macb_writel(bp, NCR, macb_readl(bp, NCR) | MACB_BIT(TSTART));
2401 spin_unlock_irq(&bp->lock);
2402
2403 if (CIRC_SPACE(queue->tx_head, queue->tx_tail, bp->tx_ring_size) < 1)
2404 netif_stop_subqueue(dev, queue_index);
2405
2406 unlock:
2407 spin_unlock_bh(&queue->tx_ptr_lock);
2408
2409 return ret;
2410 }
2411
macb_init_rx_buffer_size(struct macb * bp,size_t size)2412 static void macb_init_rx_buffer_size(struct macb *bp, size_t size)
2413 {
2414 if (!macb_is_gem(bp)) {
2415 bp->rx_buffer_size = MACB_RX_BUFFER_SIZE;
2416 } else {
2417 bp->rx_buffer_size = size;
2418
2419 if (bp->rx_buffer_size % RX_BUFFER_MULTIPLE) {
2420 netdev_dbg(bp->dev,
2421 "RX buffer must be multiple of %d bytes, expanding\n",
2422 RX_BUFFER_MULTIPLE);
2423 bp->rx_buffer_size =
2424 roundup(bp->rx_buffer_size, RX_BUFFER_MULTIPLE);
2425 }
2426 }
2427
2428 netdev_dbg(bp->dev, "mtu [%u] rx_buffer_size [%zu]\n",
2429 bp->dev->mtu, bp->rx_buffer_size);
2430 }
2431
gem_free_rx_buffers(struct macb * bp)2432 static void gem_free_rx_buffers(struct macb *bp)
2433 {
2434 struct sk_buff *skb;
2435 struct macb_dma_desc *desc;
2436 struct macb_queue *queue;
2437 dma_addr_t addr;
2438 unsigned int q;
2439 int i;
2440
2441 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) {
2442 if (!queue->rx_skbuff)
2443 continue;
2444
2445 for (i = 0; i < bp->rx_ring_size; i++) {
2446 skb = queue->rx_skbuff[i];
2447
2448 if (!skb)
2449 continue;
2450
2451 desc = macb_rx_desc(queue, i);
2452 addr = macb_get_addr(bp, desc);
2453
2454 dma_unmap_single(&bp->pdev->dev, addr, bp->rx_buffer_size,
2455 DMA_FROM_DEVICE);
2456 dev_kfree_skb_any(skb);
2457 skb = NULL;
2458 }
2459
2460 kfree(queue->rx_skbuff);
2461 queue->rx_skbuff = NULL;
2462 }
2463 }
2464
macb_free_rx_buffers(struct macb * bp)2465 static void macb_free_rx_buffers(struct macb *bp)
2466 {
2467 struct macb_queue *queue = &bp->queues[0];
2468
2469 if (queue->rx_buffers) {
2470 dma_free_coherent(&bp->pdev->dev,
2471 bp->rx_ring_size * bp->rx_buffer_size,
2472 queue->rx_buffers, queue->rx_buffers_dma);
2473 queue->rx_buffers = NULL;
2474 }
2475 }
2476
macb_free_consistent(struct macb * bp)2477 static void macb_free_consistent(struct macb *bp)
2478 {
2479 struct macb_queue *queue;
2480 unsigned int q;
2481 int size;
2482
2483 if (bp->rx_ring_tieoff) {
2484 dma_free_coherent(&bp->pdev->dev, macb_dma_desc_get_size(bp),
2485 bp->rx_ring_tieoff, bp->rx_ring_tieoff_dma);
2486 bp->rx_ring_tieoff = NULL;
2487 }
2488
2489 bp->macbgem_ops.mog_free_rx_buffers(bp);
2490
2491 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) {
2492 kfree(queue->tx_skb);
2493 queue->tx_skb = NULL;
2494 if (queue->tx_ring) {
2495 size = TX_RING_BYTES(bp) + bp->tx_bd_rd_prefetch;
2496 dma_free_coherent(&bp->pdev->dev, size,
2497 queue->tx_ring, queue->tx_ring_dma);
2498 queue->tx_ring = NULL;
2499 }
2500 if (queue->rx_ring) {
2501 size = RX_RING_BYTES(bp) + bp->rx_bd_rd_prefetch;
2502 dma_free_coherent(&bp->pdev->dev, size,
2503 queue->rx_ring, queue->rx_ring_dma);
2504 queue->rx_ring = NULL;
2505 }
2506 }
2507 }
2508
gem_alloc_rx_buffers(struct macb * bp)2509 static int gem_alloc_rx_buffers(struct macb *bp)
2510 {
2511 struct macb_queue *queue;
2512 unsigned int q;
2513 int size;
2514
2515 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) {
2516 size = bp->rx_ring_size * sizeof(struct sk_buff *);
2517 queue->rx_skbuff = kzalloc(size, GFP_KERNEL);
2518 if (!queue->rx_skbuff)
2519 return -ENOMEM;
2520 else
2521 netdev_dbg(bp->dev,
2522 "Allocated %d RX struct sk_buff entries at %p\n",
2523 bp->rx_ring_size, queue->rx_skbuff);
2524 }
2525 return 0;
2526 }
2527
macb_alloc_rx_buffers(struct macb * bp)2528 static int macb_alloc_rx_buffers(struct macb *bp)
2529 {
2530 struct macb_queue *queue = &bp->queues[0];
2531 int size;
2532
2533 size = bp->rx_ring_size * bp->rx_buffer_size;
2534 queue->rx_buffers = dma_alloc_coherent(&bp->pdev->dev, size,
2535 &queue->rx_buffers_dma, GFP_KERNEL);
2536 if (!queue->rx_buffers)
2537 return -ENOMEM;
2538
2539 netdev_dbg(bp->dev,
2540 "Allocated RX buffers of %d bytes at %08lx (mapped %p)\n",
2541 size, (unsigned long)queue->rx_buffers_dma, queue->rx_buffers);
2542 return 0;
2543 }
2544
macb_alloc_consistent(struct macb * bp)2545 static int macb_alloc_consistent(struct macb *bp)
2546 {
2547 struct macb_queue *queue;
2548 unsigned int q;
2549 int size;
2550
2551 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) {
2552 size = TX_RING_BYTES(bp) + bp->tx_bd_rd_prefetch;
2553 queue->tx_ring = dma_alloc_coherent(&bp->pdev->dev, size,
2554 &queue->tx_ring_dma,
2555 GFP_KERNEL);
2556 if (!queue->tx_ring)
2557 goto out_err;
2558 netdev_dbg(bp->dev,
2559 "Allocated TX ring for queue %u of %d bytes at %08lx (mapped %p)\n",
2560 q, size, (unsigned long)queue->tx_ring_dma,
2561 queue->tx_ring);
2562
2563 size = bp->tx_ring_size * sizeof(struct macb_tx_skb);
2564 queue->tx_skb = kmalloc(size, GFP_KERNEL);
2565 if (!queue->tx_skb)
2566 goto out_err;
2567
2568 size = RX_RING_BYTES(bp) + bp->rx_bd_rd_prefetch;
2569 queue->rx_ring = dma_alloc_coherent(&bp->pdev->dev, size,
2570 &queue->rx_ring_dma, GFP_KERNEL);
2571 if (!queue->rx_ring)
2572 goto out_err;
2573 netdev_dbg(bp->dev,
2574 "Allocated RX ring of %d bytes at %08lx (mapped %p)\n",
2575 size, (unsigned long)queue->rx_ring_dma, queue->rx_ring);
2576 }
2577 if (bp->macbgem_ops.mog_alloc_rx_buffers(bp))
2578 goto out_err;
2579
2580 /* Required for tie off descriptor for PM cases */
2581 if (!(bp->caps & MACB_CAPS_QUEUE_DISABLE)) {
2582 bp->rx_ring_tieoff = dma_alloc_coherent(&bp->pdev->dev,
2583 macb_dma_desc_get_size(bp),
2584 &bp->rx_ring_tieoff_dma,
2585 GFP_KERNEL);
2586 if (!bp->rx_ring_tieoff)
2587 goto out_err;
2588 }
2589
2590 return 0;
2591
2592 out_err:
2593 macb_free_consistent(bp);
2594 return -ENOMEM;
2595 }
2596
macb_init_tieoff(struct macb * bp)2597 static void macb_init_tieoff(struct macb *bp)
2598 {
2599 struct macb_dma_desc *desc = bp->rx_ring_tieoff;
2600
2601 if (bp->caps & MACB_CAPS_QUEUE_DISABLE)
2602 return;
2603 /* Setup a wrapping descriptor with no free slots
2604 * (WRAP and USED) to tie off/disable unused RX queues.
2605 */
2606 macb_set_addr(bp, desc, MACB_BIT(RX_WRAP) | MACB_BIT(RX_USED));
2607 desc->ctrl = 0;
2608 }
2609
gem_init_rings(struct macb * bp)2610 static void gem_init_rings(struct macb *bp)
2611 {
2612 struct macb_queue *queue;
2613 struct macb_dma_desc *desc = NULL;
2614 unsigned int q;
2615 int i;
2616
2617 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) {
2618 for (i = 0; i < bp->tx_ring_size; i++) {
2619 desc = macb_tx_desc(queue, i);
2620 macb_set_addr(bp, desc, 0);
2621 desc->ctrl = MACB_BIT(TX_USED);
2622 }
2623 desc->ctrl |= MACB_BIT(TX_WRAP);
2624 queue->tx_head = 0;
2625 queue->tx_tail = 0;
2626
2627 queue->rx_tail = 0;
2628 queue->rx_prepared_head = 0;
2629
2630 gem_rx_refill(queue);
2631 }
2632
2633 macb_init_tieoff(bp);
2634 }
2635
macb_init_rings(struct macb * bp)2636 static void macb_init_rings(struct macb *bp)
2637 {
2638 int i;
2639 struct macb_dma_desc *desc = NULL;
2640
2641 macb_init_rx_ring(&bp->queues[0]);
2642
2643 for (i = 0; i < bp->tx_ring_size; i++) {
2644 desc = macb_tx_desc(&bp->queues[0], i);
2645 macb_set_addr(bp, desc, 0);
2646 desc->ctrl = MACB_BIT(TX_USED);
2647 }
2648 bp->queues[0].tx_head = 0;
2649 bp->queues[0].tx_tail = 0;
2650 desc->ctrl |= MACB_BIT(TX_WRAP);
2651
2652 macb_init_tieoff(bp);
2653 }
2654
macb_reset_hw(struct macb * bp)2655 static void macb_reset_hw(struct macb *bp)
2656 {
2657 struct macb_queue *queue;
2658 unsigned int q;
2659 u32 ctrl = macb_readl(bp, NCR);
2660
2661 /* Disable RX and TX (XXX: Should we halt the transmission
2662 * more gracefully?)
2663 */
2664 ctrl &= ~(MACB_BIT(RE) | MACB_BIT(TE));
2665
2666 /* Clear the stats registers (XXX: Update stats first?) */
2667 ctrl |= MACB_BIT(CLRSTAT);
2668
2669 macb_writel(bp, NCR, ctrl);
2670
2671 /* Clear all status flags */
2672 macb_writel(bp, TSR, -1);
2673 macb_writel(bp, RSR, -1);
2674
2675 /* Disable RX partial store and forward and reset watermark value */
2676 gem_writel(bp, PBUFRXCUT, 0);
2677
2678 /* Disable all interrupts */
2679 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) {
2680 queue_writel(queue, IDR, -1);
2681 queue_readl(queue, ISR);
2682 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE)
2683 queue_writel(queue, ISR, -1);
2684 }
2685 }
2686
gem_mdc_clk_div(struct macb * bp)2687 static u32 gem_mdc_clk_div(struct macb *bp)
2688 {
2689 u32 config;
2690 unsigned long pclk_hz = clk_get_rate(bp->pclk);
2691
2692 if (pclk_hz <= 20000000)
2693 config = GEM_BF(CLK, GEM_CLK_DIV8);
2694 else if (pclk_hz <= 40000000)
2695 config = GEM_BF(CLK, GEM_CLK_DIV16);
2696 else if (pclk_hz <= 80000000)
2697 config = GEM_BF(CLK, GEM_CLK_DIV32);
2698 else if (pclk_hz <= 120000000)
2699 config = GEM_BF(CLK, GEM_CLK_DIV48);
2700 else if (pclk_hz <= 160000000)
2701 config = GEM_BF(CLK, GEM_CLK_DIV64);
2702 else if (pclk_hz <= 240000000)
2703 config = GEM_BF(CLK, GEM_CLK_DIV96);
2704 else if (pclk_hz <= 320000000)
2705 config = GEM_BF(CLK, GEM_CLK_DIV128);
2706 else
2707 config = GEM_BF(CLK, GEM_CLK_DIV224);
2708
2709 return config;
2710 }
2711
macb_mdc_clk_div(struct macb * bp)2712 static u32 macb_mdc_clk_div(struct macb *bp)
2713 {
2714 u32 config;
2715 unsigned long pclk_hz;
2716
2717 if (macb_is_gem(bp))
2718 return gem_mdc_clk_div(bp);
2719
2720 pclk_hz = clk_get_rate(bp->pclk);
2721 if (pclk_hz <= 20000000)
2722 config = MACB_BF(CLK, MACB_CLK_DIV8);
2723 else if (pclk_hz <= 40000000)
2724 config = MACB_BF(CLK, MACB_CLK_DIV16);
2725 else if (pclk_hz <= 80000000)
2726 config = MACB_BF(CLK, MACB_CLK_DIV32);
2727 else
2728 config = MACB_BF(CLK, MACB_CLK_DIV64);
2729
2730 return config;
2731 }
2732
2733 /* Get the DMA bus width field of the network configuration register that we
2734 * should program. We find the width from decoding the design configuration
2735 * register to find the maximum supported data bus width.
2736 */
macb_dbw(struct macb * bp)2737 static u32 macb_dbw(struct macb *bp)
2738 {
2739 if (!macb_is_gem(bp))
2740 return 0;
2741
2742 switch (GEM_BFEXT(DBWDEF, gem_readl(bp, DCFG1))) {
2743 case 4:
2744 return GEM_BF(DBW, GEM_DBW128);
2745 case 2:
2746 return GEM_BF(DBW, GEM_DBW64);
2747 case 1:
2748 default:
2749 return GEM_BF(DBW, GEM_DBW32);
2750 }
2751 }
2752
2753 /* Configure the receive DMA engine
2754 * - use the correct receive buffer size
2755 * - set best burst length for DMA operations
2756 * (if not supported by FIFO, it will fallback to default)
2757 * - set both rx/tx packet buffers to full memory size
2758 * These are configurable parameters for GEM.
2759 */
macb_configure_dma(struct macb * bp)2760 static void macb_configure_dma(struct macb *bp)
2761 {
2762 struct macb_queue *queue;
2763 u32 buffer_size;
2764 unsigned int q;
2765 u32 dmacfg;
2766
2767 buffer_size = bp->rx_buffer_size / RX_BUFFER_MULTIPLE;
2768 if (macb_is_gem(bp)) {
2769 dmacfg = gem_readl(bp, DMACFG) & ~GEM_BF(RXBS, -1L);
2770 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) {
2771 if (q)
2772 queue_writel(queue, RBQS, buffer_size);
2773 else
2774 dmacfg |= GEM_BF(RXBS, buffer_size);
2775 }
2776 if (bp->dma_burst_length)
2777 dmacfg = GEM_BFINS(FBLDO, bp->dma_burst_length, dmacfg);
2778 dmacfg |= GEM_BIT(TXPBMS) | GEM_BF(RXBMS, -1L);
2779 dmacfg &= ~GEM_BIT(ENDIA_PKT);
2780
2781 if (bp->native_io)
2782 dmacfg &= ~GEM_BIT(ENDIA_DESC);
2783 else
2784 dmacfg |= GEM_BIT(ENDIA_DESC); /* CPU in big endian */
2785
2786 if (bp->dev->features & NETIF_F_HW_CSUM)
2787 dmacfg |= GEM_BIT(TXCOEN);
2788 else
2789 dmacfg &= ~GEM_BIT(TXCOEN);
2790
2791 dmacfg &= ~GEM_BIT(ADDR64);
2792 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT
2793 if (bp->hw_dma_cap & HW_DMA_CAP_64B)
2794 dmacfg |= GEM_BIT(ADDR64);
2795 #endif
2796 #ifdef CONFIG_MACB_USE_HWSTAMP
2797 if (bp->hw_dma_cap & HW_DMA_CAP_PTP)
2798 dmacfg |= GEM_BIT(RXEXT) | GEM_BIT(TXEXT);
2799 #endif
2800 netdev_dbg(bp->dev, "Cadence configure DMA with 0x%08x\n",
2801 dmacfg);
2802 gem_writel(bp, DMACFG, dmacfg);
2803 }
2804 }
2805
macb_init_hw(struct macb * bp)2806 static void macb_init_hw(struct macb *bp)
2807 {
2808 u32 config;
2809
2810 macb_reset_hw(bp);
2811 macb_set_hwaddr(bp);
2812
2813 config = macb_mdc_clk_div(bp);
2814 config |= MACB_BF(RBOF, NET_IP_ALIGN); /* Make eth data aligned */
2815 config |= MACB_BIT(DRFCS); /* Discard Rx FCS */
2816 if (bp->caps & MACB_CAPS_JUMBO)
2817 config |= MACB_BIT(JFRAME); /* Enable jumbo frames */
2818 else
2819 config |= MACB_BIT(BIG); /* Receive oversized frames */
2820 if (bp->dev->flags & IFF_PROMISC)
2821 config |= MACB_BIT(CAF); /* Copy All Frames */
2822 else if (macb_is_gem(bp) && bp->dev->features & NETIF_F_RXCSUM)
2823 config |= GEM_BIT(RXCOEN);
2824 if (!(bp->dev->flags & IFF_BROADCAST))
2825 config |= MACB_BIT(NBC); /* No BroadCast */
2826 config |= macb_dbw(bp);
2827 macb_writel(bp, NCFGR, config);
2828 if ((bp->caps & MACB_CAPS_JUMBO) && bp->jumbo_max_len)
2829 gem_writel(bp, JML, bp->jumbo_max_len);
2830 bp->rx_frm_len_mask = MACB_RX_FRMLEN_MASK;
2831 if (bp->caps & MACB_CAPS_JUMBO)
2832 bp->rx_frm_len_mask = MACB_RX_JFRMLEN_MASK;
2833
2834 macb_configure_dma(bp);
2835
2836 /* Enable RX partial store and forward and set watermark */
2837 if (bp->rx_watermark)
2838 gem_writel(bp, PBUFRXCUT, (bp->rx_watermark | GEM_BIT(ENCUTTHRU)));
2839 }
2840
2841 /* The hash address register is 64 bits long and takes up two
2842 * locations in the memory map. The least significant bits are stored
2843 * in EMAC_HSL and the most significant bits in EMAC_HSH.
2844 *
2845 * The unicast hash enable and the multicast hash enable bits in the
2846 * network configuration register enable the reception of hash matched
2847 * frames. The destination address is reduced to a 6 bit index into
2848 * the 64 bit hash register using the following hash function. The
2849 * hash function is an exclusive or of every sixth bit of the
2850 * destination address.
2851 *
2852 * hi[5] = da[5] ^ da[11] ^ da[17] ^ da[23] ^ da[29] ^ da[35] ^ da[41] ^ da[47]
2853 * hi[4] = da[4] ^ da[10] ^ da[16] ^ da[22] ^ da[28] ^ da[34] ^ da[40] ^ da[46]
2854 * hi[3] = da[3] ^ da[09] ^ da[15] ^ da[21] ^ da[27] ^ da[33] ^ da[39] ^ da[45]
2855 * hi[2] = da[2] ^ da[08] ^ da[14] ^ da[20] ^ da[26] ^ da[32] ^ da[38] ^ da[44]
2856 * hi[1] = da[1] ^ da[07] ^ da[13] ^ da[19] ^ da[25] ^ da[31] ^ da[37] ^ da[43]
2857 * hi[0] = da[0] ^ da[06] ^ da[12] ^ da[18] ^ da[24] ^ da[30] ^ da[36] ^ da[42]
2858 *
2859 * da[0] represents the least significant bit of the first byte
2860 * received, that is, the multicast/unicast indicator, and da[47]
2861 * represents the most significant bit of the last byte received. If
2862 * the hash index, hi[n], points to a bit that is set in the hash
2863 * register then the frame will be matched according to whether the
2864 * frame is multicast or unicast. A multicast match will be signalled
2865 * if the multicast hash enable bit is set, da[0] is 1 and the hash
2866 * index points to a bit set in the hash register. A unicast match
2867 * will be signalled if the unicast hash enable bit is set, da[0] is 0
2868 * and the hash index points to a bit set in the hash register. To
2869 * receive all multicast frames, the hash register should be set with
2870 * all ones and the multicast hash enable bit should be set in the
2871 * network configuration register.
2872 */
2873
hash_bit_value(int bitnr,__u8 * addr)2874 static inline int hash_bit_value(int bitnr, __u8 *addr)
2875 {
2876 if (addr[bitnr / 8] & (1 << (bitnr % 8)))
2877 return 1;
2878 return 0;
2879 }
2880
2881 /* Return the hash index value for the specified address. */
hash_get_index(__u8 * addr)2882 static int hash_get_index(__u8 *addr)
2883 {
2884 int i, j, bitval;
2885 int hash_index = 0;
2886
2887 for (j = 0; j < 6; j++) {
2888 for (i = 0, bitval = 0; i < 8; i++)
2889 bitval ^= hash_bit_value(i * 6 + j, addr);
2890
2891 hash_index |= (bitval << j);
2892 }
2893
2894 return hash_index;
2895 }
2896
2897 /* Add multicast addresses to the internal multicast-hash table. */
macb_sethashtable(struct net_device * dev)2898 static void macb_sethashtable(struct net_device *dev)
2899 {
2900 struct netdev_hw_addr *ha;
2901 unsigned long mc_filter[2];
2902 unsigned int bitnr;
2903 struct macb *bp = netdev_priv(dev);
2904
2905 mc_filter[0] = 0;
2906 mc_filter[1] = 0;
2907
2908 netdev_for_each_mc_addr(ha, dev) {
2909 bitnr = hash_get_index(ha->addr);
2910 mc_filter[bitnr >> 5] |= 1 << (bitnr & 31);
2911 }
2912
2913 macb_or_gem_writel(bp, HRB, mc_filter[0]);
2914 macb_or_gem_writel(bp, HRT, mc_filter[1]);
2915 }
2916
2917 /* Enable/Disable promiscuous and multicast modes. */
macb_set_rx_mode(struct net_device * dev)2918 static void macb_set_rx_mode(struct net_device *dev)
2919 {
2920 unsigned long cfg;
2921 struct macb *bp = netdev_priv(dev);
2922
2923 cfg = macb_readl(bp, NCFGR);
2924
2925 if (dev->flags & IFF_PROMISC) {
2926 /* Enable promiscuous mode */
2927 cfg |= MACB_BIT(CAF);
2928
2929 /* Disable RX checksum offload */
2930 if (macb_is_gem(bp))
2931 cfg &= ~GEM_BIT(RXCOEN);
2932 } else {
2933 /* Disable promiscuous mode */
2934 cfg &= ~MACB_BIT(CAF);
2935
2936 /* Enable RX checksum offload only if requested */
2937 if (macb_is_gem(bp) && dev->features & NETIF_F_RXCSUM)
2938 cfg |= GEM_BIT(RXCOEN);
2939 }
2940
2941 if (dev->flags & IFF_ALLMULTI) {
2942 /* Enable all multicast mode */
2943 macb_or_gem_writel(bp, HRB, -1);
2944 macb_or_gem_writel(bp, HRT, -1);
2945 cfg |= MACB_BIT(NCFGR_MTI);
2946 } else if (!netdev_mc_empty(dev)) {
2947 /* Enable specific multicasts */
2948 macb_sethashtable(dev);
2949 cfg |= MACB_BIT(NCFGR_MTI);
2950 } else if (dev->flags & (~IFF_ALLMULTI)) {
2951 /* Disable all multicast mode */
2952 macb_or_gem_writel(bp, HRB, 0);
2953 macb_or_gem_writel(bp, HRT, 0);
2954 cfg &= ~MACB_BIT(NCFGR_MTI);
2955 }
2956
2957 macb_writel(bp, NCFGR, cfg);
2958 }
2959
macb_open(struct net_device * dev)2960 static int macb_open(struct net_device *dev)
2961 {
2962 size_t bufsz = dev->mtu + ETH_HLEN + ETH_FCS_LEN + NET_IP_ALIGN;
2963 struct macb *bp = netdev_priv(dev);
2964 struct macb_queue *queue;
2965 unsigned int q;
2966 int err;
2967
2968 netdev_dbg(bp->dev, "open\n");
2969
2970 err = pm_runtime_resume_and_get(&bp->pdev->dev);
2971 if (err < 0)
2972 return err;
2973
2974 /* RX buffers initialization */
2975 macb_init_rx_buffer_size(bp, bufsz);
2976
2977 err = macb_alloc_consistent(bp);
2978 if (err) {
2979 netdev_err(dev, "Unable to allocate DMA memory (error %d)\n",
2980 err);
2981 goto pm_exit;
2982 }
2983
2984 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) {
2985 napi_enable(&queue->napi_rx);
2986 napi_enable(&queue->napi_tx);
2987 }
2988
2989 macb_init_hw(bp);
2990
2991 err = phy_power_on(bp->sgmii_phy);
2992 if (err)
2993 goto reset_hw;
2994
2995 err = macb_phylink_connect(bp);
2996 if (err)
2997 goto phy_off;
2998
2999 netif_tx_start_all_queues(dev);
3000
3001 if (bp->ptp_info)
3002 bp->ptp_info->ptp_init(dev);
3003
3004 return 0;
3005
3006 phy_off:
3007 phy_power_off(bp->sgmii_phy);
3008
3009 reset_hw:
3010 macb_reset_hw(bp);
3011 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) {
3012 napi_disable(&queue->napi_rx);
3013 napi_disable(&queue->napi_tx);
3014 }
3015 macb_free_consistent(bp);
3016 pm_exit:
3017 pm_runtime_put_sync(&bp->pdev->dev);
3018 return err;
3019 }
3020
macb_close(struct net_device * dev)3021 static int macb_close(struct net_device *dev)
3022 {
3023 struct macb *bp = netdev_priv(dev);
3024 struct macb_queue *queue;
3025 unsigned long flags;
3026 unsigned int q;
3027
3028 netif_tx_stop_all_queues(dev);
3029
3030 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) {
3031 napi_disable(&queue->napi_rx);
3032 napi_disable(&queue->napi_tx);
3033 }
3034
3035 phylink_stop(bp->phylink);
3036 phylink_disconnect_phy(bp->phylink);
3037
3038 phy_power_off(bp->sgmii_phy);
3039
3040 spin_lock_irqsave(&bp->lock, flags);
3041 macb_reset_hw(bp);
3042 netif_carrier_off(dev);
3043 spin_unlock_irqrestore(&bp->lock, flags);
3044
3045 macb_free_consistent(bp);
3046
3047 if (bp->ptp_info)
3048 bp->ptp_info->ptp_remove(dev);
3049
3050 pm_runtime_put(&bp->pdev->dev);
3051
3052 return 0;
3053 }
3054
macb_change_mtu(struct net_device * dev,int new_mtu)3055 static int macb_change_mtu(struct net_device *dev, int new_mtu)
3056 {
3057 if (netif_running(dev))
3058 return -EBUSY;
3059
3060 WRITE_ONCE(dev->mtu, new_mtu);
3061
3062 return 0;
3063 }
3064
macb_set_mac_addr(struct net_device * dev,void * addr)3065 static int macb_set_mac_addr(struct net_device *dev, void *addr)
3066 {
3067 int err;
3068
3069 err = eth_mac_addr(dev, addr);
3070 if (err < 0)
3071 return err;
3072
3073 macb_set_hwaddr(netdev_priv(dev));
3074 return 0;
3075 }
3076
gem_update_stats(struct macb * bp)3077 static void gem_update_stats(struct macb *bp)
3078 {
3079 struct macb_queue *queue;
3080 unsigned int i, q, idx;
3081 unsigned long *stat;
3082
3083 u32 *p = &bp->hw_stats.gem.tx_octets_31_0;
3084
3085 for (i = 0; i < GEM_STATS_LEN; ++i, ++p) {
3086 u32 offset = gem_statistics[i].offset;
3087 u64 val = bp->macb_reg_readl(bp, offset);
3088
3089 bp->ethtool_stats[i] += val;
3090 *p += val;
3091
3092 if (offset == GEM_OCTTXL || offset == GEM_OCTRXL) {
3093 /* Add GEM_OCTTXH, GEM_OCTRXH */
3094 val = bp->macb_reg_readl(bp, offset + 4);
3095 bp->ethtool_stats[i] += ((u64)val) << 32;
3096 *(++p) += val;
3097 }
3098 }
3099
3100 idx = GEM_STATS_LEN;
3101 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue)
3102 for (i = 0, stat = &queue->stats.first; i < QUEUE_STATS_LEN; ++i, ++stat)
3103 bp->ethtool_stats[idx++] = *stat;
3104 }
3105
gem_get_stats(struct macb * bp)3106 static struct net_device_stats *gem_get_stats(struct macb *bp)
3107 {
3108 struct gem_stats *hwstat = &bp->hw_stats.gem;
3109 struct net_device_stats *nstat = &bp->dev->stats;
3110
3111 if (!netif_running(bp->dev))
3112 return nstat;
3113
3114 gem_update_stats(bp);
3115
3116 nstat->rx_errors = (hwstat->rx_frame_check_sequence_errors +
3117 hwstat->rx_alignment_errors +
3118 hwstat->rx_resource_errors +
3119 hwstat->rx_overruns +
3120 hwstat->rx_oversize_frames +
3121 hwstat->rx_jabbers +
3122 hwstat->rx_undersized_frames +
3123 hwstat->rx_length_field_frame_errors);
3124 nstat->tx_errors = (hwstat->tx_late_collisions +
3125 hwstat->tx_excessive_collisions +
3126 hwstat->tx_underrun +
3127 hwstat->tx_carrier_sense_errors);
3128 nstat->multicast = hwstat->rx_multicast_frames;
3129 nstat->collisions = (hwstat->tx_single_collision_frames +
3130 hwstat->tx_multiple_collision_frames +
3131 hwstat->tx_excessive_collisions);
3132 nstat->rx_length_errors = (hwstat->rx_oversize_frames +
3133 hwstat->rx_jabbers +
3134 hwstat->rx_undersized_frames +
3135 hwstat->rx_length_field_frame_errors);
3136 nstat->rx_over_errors = hwstat->rx_resource_errors;
3137 nstat->rx_crc_errors = hwstat->rx_frame_check_sequence_errors;
3138 nstat->rx_frame_errors = hwstat->rx_alignment_errors;
3139 nstat->rx_fifo_errors = hwstat->rx_overruns;
3140 nstat->tx_aborted_errors = hwstat->tx_excessive_collisions;
3141 nstat->tx_carrier_errors = hwstat->tx_carrier_sense_errors;
3142 nstat->tx_fifo_errors = hwstat->tx_underrun;
3143
3144 return nstat;
3145 }
3146
gem_get_ethtool_stats(struct net_device * dev,struct ethtool_stats * stats,u64 * data)3147 static void gem_get_ethtool_stats(struct net_device *dev,
3148 struct ethtool_stats *stats, u64 *data)
3149 {
3150 struct macb *bp;
3151
3152 bp = netdev_priv(dev);
3153 gem_update_stats(bp);
3154 memcpy(data, &bp->ethtool_stats, sizeof(u64)
3155 * (GEM_STATS_LEN + QUEUE_STATS_LEN * MACB_MAX_QUEUES));
3156 }
3157
gem_get_sset_count(struct net_device * dev,int sset)3158 static int gem_get_sset_count(struct net_device *dev, int sset)
3159 {
3160 struct macb *bp = netdev_priv(dev);
3161
3162 switch (sset) {
3163 case ETH_SS_STATS:
3164 return GEM_STATS_LEN + bp->num_queues * QUEUE_STATS_LEN;
3165 default:
3166 return -EOPNOTSUPP;
3167 }
3168 }
3169
gem_get_ethtool_strings(struct net_device * dev,u32 sset,u8 * p)3170 static void gem_get_ethtool_strings(struct net_device *dev, u32 sset, u8 *p)
3171 {
3172 char stat_string[ETH_GSTRING_LEN];
3173 struct macb *bp = netdev_priv(dev);
3174 struct macb_queue *queue;
3175 unsigned int i;
3176 unsigned int q;
3177
3178 switch (sset) {
3179 case ETH_SS_STATS:
3180 for (i = 0; i < GEM_STATS_LEN; i++, p += ETH_GSTRING_LEN)
3181 memcpy(p, gem_statistics[i].stat_string,
3182 ETH_GSTRING_LEN);
3183
3184 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) {
3185 for (i = 0; i < QUEUE_STATS_LEN; i++, p += ETH_GSTRING_LEN) {
3186 snprintf(stat_string, ETH_GSTRING_LEN, "q%d_%s",
3187 q, queue_statistics[i].stat_string);
3188 memcpy(p, stat_string, ETH_GSTRING_LEN);
3189 }
3190 }
3191 break;
3192 }
3193 }
3194
macb_get_stats(struct net_device * dev)3195 static struct net_device_stats *macb_get_stats(struct net_device *dev)
3196 {
3197 struct macb *bp = netdev_priv(dev);
3198 struct net_device_stats *nstat = &bp->dev->stats;
3199 struct macb_stats *hwstat = &bp->hw_stats.macb;
3200
3201 if (macb_is_gem(bp))
3202 return gem_get_stats(bp);
3203
3204 /* read stats from hardware */
3205 macb_update_stats(bp);
3206
3207 /* Convert HW stats into netdevice stats */
3208 nstat->rx_errors = (hwstat->rx_fcs_errors +
3209 hwstat->rx_align_errors +
3210 hwstat->rx_resource_errors +
3211 hwstat->rx_overruns +
3212 hwstat->rx_oversize_pkts +
3213 hwstat->rx_jabbers +
3214 hwstat->rx_undersize_pkts +
3215 hwstat->rx_length_mismatch);
3216 nstat->tx_errors = (hwstat->tx_late_cols +
3217 hwstat->tx_excessive_cols +
3218 hwstat->tx_underruns +
3219 hwstat->tx_carrier_errors +
3220 hwstat->sqe_test_errors);
3221 nstat->collisions = (hwstat->tx_single_cols +
3222 hwstat->tx_multiple_cols +
3223 hwstat->tx_excessive_cols);
3224 nstat->rx_length_errors = (hwstat->rx_oversize_pkts +
3225 hwstat->rx_jabbers +
3226 hwstat->rx_undersize_pkts +
3227 hwstat->rx_length_mismatch);
3228 nstat->rx_over_errors = hwstat->rx_resource_errors +
3229 hwstat->rx_overruns;
3230 nstat->rx_crc_errors = hwstat->rx_fcs_errors;
3231 nstat->rx_frame_errors = hwstat->rx_align_errors;
3232 nstat->rx_fifo_errors = hwstat->rx_overruns;
3233 /* XXX: What does "missed" mean? */
3234 nstat->tx_aborted_errors = hwstat->tx_excessive_cols;
3235 nstat->tx_carrier_errors = hwstat->tx_carrier_errors;
3236 nstat->tx_fifo_errors = hwstat->tx_underruns;
3237 /* Don't know about heartbeat or window errors... */
3238
3239 return nstat;
3240 }
3241
macb_get_regs_len(struct net_device * netdev)3242 static int macb_get_regs_len(struct net_device *netdev)
3243 {
3244 return MACB_GREGS_NBR * sizeof(u32);
3245 }
3246
macb_get_regs(struct net_device * dev,struct ethtool_regs * regs,void * p)3247 static void macb_get_regs(struct net_device *dev, struct ethtool_regs *regs,
3248 void *p)
3249 {
3250 struct macb *bp = netdev_priv(dev);
3251 unsigned int tail, head;
3252 u32 *regs_buff = p;
3253
3254 regs->version = (macb_readl(bp, MID) & ((1 << MACB_REV_SIZE) - 1))
3255 | MACB_GREGS_VERSION;
3256
3257 tail = macb_tx_ring_wrap(bp, bp->queues[0].tx_tail);
3258 head = macb_tx_ring_wrap(bp, bp->queues[0].tx_head);
3259
3260 regs_buff[0] = macb_readl(bp, NCR);
3261 regs_buff[1] = macb_or_gem_readl(bp, NCFGR);
3262 regs_buff[2] = macb_readl(bp, NSR);
3263 regs_buff[3] = macb_readl(bp, TSR);
3264 regs_buff[4] = macb_readl(bp, RBQP);
3265 regs_buff[5] = macb_readl(bp, TBQP);
3266 regs_buff[6] = macb_readl(bp, RSR);
3267 regs_buff[7] = macb_readl(bp, IMR);
3268
3269 regs_buff[8] = tail;
3270 regs_buff[9] = head;
3271 regs_buff[10] = macb_tx_dma(&bp->queues[0], tail);
3272 regs_buff[11] = macb_tx_dma(&bp->queues[0], head);
3273
3274 if (!(bp->caps & MACB_CAPS_USRIO_DISABLED))
3275 regs_buff[12] = macb_or_gem_readl(bp, USRIO);
3276 if (macb_is_gem(bp))
3277 regs_buff[13] = gem_readl(bp, DMACFG);
3278 }
3279
macb_get_wol(struct net_device * netdev,struct ethtool_wolinfo * wol)3280 static void macb_get_wol(struct net_device *netdev, struct ethtool_wolinfo *wol)
3281 {
3282 struct macb *bp = netdev_priv(netdev);
3283
3284 phylink_ethtool_get_wol(bp->phylink, wol);
3285 wol->supported |= (WAKE_MAGIC | WAKE_ARP);
3286
3287 /* Add macb wolopts to phy wolopts */
3288 wol->wolopts |= bp->wolopts;
3289 }
3290
macb_set_wol(struct net_device * netdev,struct ethtool_wolinfo * wol)3291 static int macb_set_wol(struct net_device *netdev, struct ethtool_wolinfo *wol)
3292 {
3293 struct macb *bp = netdev_priv(netdev);
3294 int ret;
3295
3296 /* Pass the order to phylink layer */
3297 ret = phylink_ethtool_set_wol(bp->phylink, wol);
3298 /* Don't manage WoL on MAC, if PHY set_wol() fails */
3299 if (ret && ret != -EOPNOTSUPP)
3300 return ret;
3301
3302 bp->wolopts = (wol->wolopts & WAKE_MAGIC) ? WAKE_MAGIC : 0;
3303 bp->wolopts |= (wol->wolopts & WAKE_ARP) ? WAKE_ARP : 0;
3304 bp->wol = (wol->wolopts) ? MACB_WOL_ENABLED : 0;
3305
3306 device_set_wakeup_enable(&bp->pdev->dev, bp->wol);
3307
3308 return 0;
3309 }
3310
macb_get_link_ksettings(struct net_device * netdev,struct ethtool_link_ksettings * kset)3311 static int macb_get_link_ksettings(struct net_device *netdev,
3312 struct ethtool_link_ksettings *kset)
3313 {
3314 struct macb *bp = netdev_priv(netdev);
3315
3316 return phylink_ethtool_ksettings_get(bp->phylink, kset);
3317 }
3318
macb_set_link_ksettings(struct net_device * netdev,const struct ethtool_link_ksettings * kset)3319 static int macb_set_link_ksettings(struct net_device *netdev,
3320 const struct ethtool_link_ksettings *kset)
3321 {
3322 struct macb *bp = netdev_priv(netdev);
3323
3324 return phylink_ethtool_ksettings_set(bp->phylink, kset);
3325 }
3326
macb_get_ringparam(struct net_device * netdev,struct ethtool_ringparam * ring,struct kernel_ethtool_ringparam * kernel_ring,struct netlink_ext_ack * extack)3327 static void macb_get_ringparam(struct net_device *netdev,
3328 struct ethtool_ringparam *ring,
3329 struct kernel_ethtool_ringparam *kernel_ring,
3330 struct netlink_ext_ack *extack)
3331 {
3332 struct macb *bp = netdev_priv(netdev);
3333
3334 ring->rx_max_pending = MAX_RX_RING_SIZE;
3335 ring->tx_max_pending = MAX_TX_RING_SIZE;
3336
3337 ring->rx_pending = bp->rx_ring_size;
3338 ring->tx_pending = bp->tx_ring_size;
3339 }
3340
macb_set_ringparam(struct net_device * netdev,struct ethtool_ringparam * ring,struct kernel_ethtool_ringparam * kernel_ring,struct netlink_ext_ack * extack)3341 static int macb_set_ringparam(struct net_device *netdev,
3342 struct ethtool_ringparam *ring,
3343 struct kernel_ethtool_ringparam *kernel_ring,
3344 struct netlink_ext_ack *extack)
3345 {
3346 struct macb *bp = netdev_priv(netdev);
3347 u32 new_rx_size, new_tx_size;
3348 unsigned int reset = 0;
3349
3350 if ((ring->rx_mini_pending) || (ring->rx_jumbo_pending))
3351 return -EINVAL;
3352
3353 new_rx_size = clamp_t(u32, ring->rx_pending,
3354 MIN_RX_RING_SIZE, MAX_RX_RING_SIZE);
3355 new_rx_size = roundup_pow_of_two(new_rx_size);
3356
3357 new_tx_size = clamp_t(u32, ring->tx_pending,
3358 MIN_TX_RING_SIZE, MAX_TX_RING_SIZE);
3359 new_tx_size = roundup_pow_of_two(new_tx_size);
3360
3361 if ((new_tx_size == bp->tx_ring_size) &&
3362 (new_rx_size == bp->rx_ring_size)) {
3363 /* nothing to do */
3364 return 0;
3365 }
3366
3367 if (netif_running(bp->dev)) {
3368 reset = 1;
3369 macb_close(bp->dev);
3370 }
3371
3372 bp->rx_ring_size = new_rx_size;
3373 bp->tx_ring_size = new_tx_size;
3374
3375 if (reset)
3376 macb_open(bp->dev);
3377
3378 return 0;
3379 }
3380
3381 #ifdef CONFIG_MACB_USE_HWSTAMP
gem_get_tsu_rate(struct macb * bp)3382 static unsigned int gem_get_tsu_rate(struct macb *bp)
3383 {
3384 struct clk *tsu_clk;
3385 unsigned int tsu_rate;
3386
3387 tsu_clk = devm_clk_get(&bp->pdev->dev, "tsu_clk");
3388 if (!IS_ERR(tsu_clk))
3389 tsu_rate = clk_get_rate(tsu_clk);
3390 /* try pclk instead */
3391 else if (!IS_ERR(bp->pclk)) {
3392 tsu_clk = bp->pclk;
3393 tsu_rate = clk_get_rate(tsu_clk);
3394 } else
3395 return -ENOTSUPP;
3396 return tsu_rate;
3397 }
3398
gem_get_ptp_max_adj(void)3399 static s32 gem_get_ptp_max_adj(void)
3400 {
3401 return 64000000;
3402 }
3403
gem_get_ts_info(struct net_device * dev,struct kernel_ethtool_ts_info * info)3404 static int gem_get_ts_info(struct net_device *dev,
3405 struct kernel_ethtool_ts_info *info)
3406 {
3407 struct macb *bp = netdev_priv(dev);
3408
3409 if ((bp->hw_dma_cap & HW_DMA_CAP_PTP) == 0) {
3410 ethtool_op_get_ts_info(dev, info);
3411 return 0;
3412 }
3413
3414 info->so_timestamping =
3415 SOF_TIMESTAMPING_TX_SOFTWARE |
3416 SOF_TIMESTAMPING_TX_HARDWARE |
3417 SOF_TIMESTAMPING_RX_HARDWARE |
3418 SOF_TIMESTAMPING_RAW_HARDWARE;
3419 info->tx_types =
3420 (1 << HWTSTAMP_TX_ONESTEP_SYNC) |
3421 (1 << HWTSTAMP_TX_OFF) |
3422 (1 << HWTSTAMP_TX_ON);
3423 info->rx_filters =
3424 (1 << HWTSTAMP_FILTER_NONE) |
3425 (1 << HWTSTAMP_FILTER_ALL);
3426
3427 if (bp->ptp_clock)
3428 info->phc_index = ptp_clock_index(bp->ptp_clock);
3429
3430 return 0;
3431 }
3432
3433 static struct macb_ptp_info gem_ptp_info = {
3434 .ptp_init = gem_ptp_init,
3435 .ptp_remove = gem_ptp_remove,
3436 .get_ptp_max_adj = gem_get_ptp_max_adj,
3437 .get_tsu_rate = gem_get_tsu_rate,
3438 .get_ts_info = gem_get_ts_info,
3439 .get_hwtst = gem_get_hwtst,
3440 .set_hwtst = gem_set_hwtst,
3441 };
3442 #endif
3443
macb_get_ts_info(struct net_device * netdev,struct kernel_ethtool_ts_info * info)3444 static int macb_get_ts_info(struct net_device *netdev,
3445 struct kernel_ethtool_ts_info *info)
3446 {
3447 struct macb *bp = netdev_priv(netdev);
3448
3449 if (bp->ptp_info)
3450 return bp->ptp_info->get_ts_info(netdev, info);
3451
3452 return ethtool_op_get_ts_info(netdev, info);
3453 }
3454
gem_enable_flow_filters(struct macb * bp,bool enable)3455 static void gem_enable_flow_filters(struct macb *bp, bool enable)
3456 {
3457 struct net_device *netdev = bp->dev;
3458 struct ethtool_rx_fs_item *item;
3459 u32 t2_scr;
3460 int num_t2_scr;
3461
3462 if (!(netdev->features & NETIF_F_NTUPLE))
3463 return;
3464
3465 num_t2_scr = GEM_BFEXT(T2SCR, gem_readl(bp, DCFG8));
3466
3467 list_for_each_entry(item, &bp->rx_fs_list.list, list) {
3468 struct ethtool_rx_flow_spec *fs = &item->fs;
3469 struct ethtool_tcpip4_spec *tp4sp_m;
3470
3471 if (fs->location >= num_t2_scr)
3472 continue;
3473
3474 t2_scr = gem_readl_n(bp, SCRT2, fs->location);
3475
3476 /* enable/disable screener regs for the flow entry */
3477 t2_scr = GEM_BFINS(ETHTEN, enable, t2_scr);
3478
3479 /* only enable fields with no masking */
3480 tp4sp_m = &(fs->m_u.tcp_ip4_spec);
3481
3482 if (enable && (tp4sp_m->ip4src == 0xFFFFFFFF))
3483 t2_scr = GEM_BFINS(CMPAEN, 1, t2_scr);
3484 else
3485 t2_scr = GEM_BFINS(CMPAEN, 0, t2_scr);
3486
3487 if (enable && (tp4sp_m->ip4dst == 0xFFFFFFFF))
3488 t2_scr = GEM_BFINS(CMPBEN, 1, t2_scr);
3489 else
3490 t2_scr = GEM_BFINS(CMPBEN, 0, t2_scr);
3491
3492 if (enable && ((tp4sp_m->psrc == 0xFFFF) || (tp4sp_m->pdst == 0xFFFF)))
3493 t2_scr = GEM_BFINS(CMPCEN, 1, t2_scr);
3494 else
3495 t2_scr = GEM_BFINS(CMPCEN, 0, t2_scr);
3496
3497 gem_writel_n(bp, SCRT2, fs->location, t2_scr);
3498 }
3499 }
3500
gem_prog_cmp_regs(struct macb * bp,struct ethtool_rx_flow_spec * fs)3501 static void gem_prog_cmp_regs(struct macb *bp, struct ethtool_rx_flow_spec *fs)
3502 {
3503 struct ethtool_tcpip4_spec *tp4sp_v, *tp4sp_m;
3504 uint16_t index = fs->location;
3505 u32 w0, w1, t2_scr;
3506 bool cmp_a = false;
3507 bool cmp_b = false;
3508 bool cmp_c = false;
3509
3510 if (!macb_is_gem(bp))
3511 return;
3512
3513 tp4sp_v = &(fs->h_u.tcp_ip4_spec);
3514 tp4sp_m = &(fs->m_u.tcp_ip4_spec);
3515
3516 /* ignore field if any masking set */
3517 if (tp4sp_m->ip4src == 0xFFFFFFFF) {
3518 /* 1st compare reg - IP source address */
3519 w0 = 0;
3520 w1 = 0;
3521 w0 = tp4sp_v->ip4src;
3522 w1 = GEM_BFINS(T2DISMSK, 1, w1); /* 32-bit compare */
3523 w1 = GEM_BFINS(T2CMPOFST, GEM_T2COMPOFST_ETYPE, w1);
3524 w1 = GEM_BFINS(T2OFST, ETYPE_SRCIP_OFFSET, w1);
3525 gem_writel_n(bp, T2CMPW0, T2CMP_OFST(GEM_IP4SRC_CMP(index)), w0);
3526 gem_writel_n(bp, T2CMPW1, T2CMP_OFST(GEM_IP4SRC_CMP(index)), w1);
3527 cmp_a = true;
3528 }
3529
3530 /* ignore field if any masking set */
3531 if (tp4sp_m->ip4dst == 0xFFFFFFFF) {
3532 /* 2nd compare reg - IP destination address */
3533 w0 = 0;
3534 w1 = 0;
3535 w0 = tp4sp_v->ip4dst;
3536 w1 = GEM_BFINS(T2DISMSK, 1, w1); /* 32-bit compare */
3537 w1 = GEM_BFINS(T2CMPOFST, GEM_T2COMPOFST_ETYPE, w1);
3538 w1 = GEM_BFINS(T2OFST, ETYPE_DSTIP_OFFSET, w1);
3539 gem_writel_n(bp, T2CMPW0, T2CMP_OFST(GEM_IP4DST_CMP(index)), w0);
3540 gem_writel_n(bp, T2CMPW1, T2CMP_OFST(GEM_IP4DST_CMP(index)), w1);
3541 cmp_b = true;
3542 }
3543
3544 /* ignore both port fields if masking set in both */
3545 if ((tp4sp_m->psrc == 0xFFFF) || (tp4sp_m->pdst == 0xFFFF)) {
3546 /* 3rd compare reg - source port, destination port */
3547 w0 = 0;
3548 w1 = 0;
3549 w1 = GEM_BFINS(T2CMPOFST, GEM_T2COMPOFST_IPHDR, w1);
3550 if (tp4sp_m->psrc == tp4sp_m->pdst) {
3551 w0 = GEM_BFINS(T2MASK, tp4sp_v->psrc, w0);
3552 w0 = GEM_BFINS(T2CMP, tp4sp_v->pdst, w0);
3553 w1 = GEM_BFINS(T2DISMSK, 1, w1); /* 32-bit compare */
3554 w1 = GEM_BFINS(T2OFST, IPHDR_SRCPORT_OFFSET, w1);
3555 } else {
3556 /* only one port definition */
3557 w1 = GEM_BFINS(T2DISMSK, 0, w1); /* 16-bit compare */
3558 w0 = GEM_BFINS(T2MASK, 0xFFFF, w0);
3559 if (tp4sp_m->psrc == 0xFFFF) { /* src port */
3560 w0 = GEM_BFINS(T2CMP, tp4sp_v->psrc, w0);
3561 w1 = GEM_BFINS(T2OFST, IPHDR_SRCPORT_OFFSET, w1);
3562 } else { /* dst port */
3563 w0 = GEM_BFINS(T2CMP, tp4sp_v->pdst, w0);
3564 w1 = GEM_BFINS(T2OFST, IPHDR_DSTPORT_OFFSET, w1);
3565 }
3566 }
3567 gem_writel_n(bp, T2CMPW0, T2CMP_OFST(GEM_PORT_CMP(index)), w0);
3568 gem_writel_n(bp, T2CMPW1, T2CMP_OFST(GEM_PORT_CMP(index)), w1);
3569 cmp_c = true;
3570 }
3571
3572 t2_scr = 0;
3573 t2_scr = GEM_BFINS(QUEUE, (fs->ring_cookie) & 0xFF, t2_scr);
3574 t2_scr = GEM_BFINS(ETHT2IDX, SCRT2_ETHT, t2_scr);
3575 if (cmp_a)
3576 t2_scr = GEM_BFINS(CMPA, GEM_IP4SRC_CMP(index), t2_scr);
3577 if (cmp_b)
3578 t2_scr = GEM_BFINS(CMPB, GEM_IP4DST_CMP(index), t2_scr);
3579 if (cmp_c)
3580 t2_scr = GEM_BFINS(CMPC, GEM_PORT_CMP(index), t2_scr);
3581 gem_writel_n(bp, SCRT2, index, t2_scr);
3582 }
3583
gem_add_flow_filter(struct net_device * netdev,struct ethtool_rxnfc * cmd)3584 static int gem_add_flow_filter(struct net_device *netdev,
3585 struct ethtool_rxnfc *cmd)
3586 {
3587 struct macb *bp = netdev_priv(netdev);
3588 struct ethtool_rx_flow_spec *fs = &cmd->fs;
3589 struct ethtool_rx_fs_item *item, *newfs;
3590 unsigned long flags;
3591 int ret = -EINVAL;
3592 bool added = false;
3593
3594 newfs = kmalloc(sizeof(*newfs), GFP_KERNEL);
3595 if (newfs == NULL)
3596 return -ENOMEM;
3597 memcpy(&newfs->fs, fs, sizeof(newfs->fs));
3598
3599 netdev_dbg(netdev,
3600 "Adding flow filter entry,type=%u,queue=%u,loc=%u,src=%08X,dst=%08X,ps=%u,pd=%u\n",
3601 fs->flow_type, (int)fs->ring_cookie, fs->location,
3602 htonl(fs->h_u.tcp_ip4_spec.ip4src),
3603 htonl(fs->h_u.tcp_ip4_spec.ip4dst),
3604 be16_to_cpu(fs->h_u.tcp_ip4_spec.psrc),
3605 be16_to_cpu(fs->h_u.tcp_ip4_spec.pdst));
3606
3607 spin_lock_irqsave(&bp->rx_fs_lock, flags);
3608
3609 /* find correct place to add in list */
3610 list_for_each_entry(item, &bp->rx_fs_list.list, list) {
3611 if (item->fs.location > newfs->fs.location) {
3612 list_add_tail(&newfs->list, &item->list);
3613 added = true;
3614 break;
3615 } else if (item->fs.location == fs->location) {
3616 netdev_err(netdev, "Rule not added: location %d not free!\n",
3617 fs->location);
3618 ret = -EBUSY;
3619 goto err;
3620 }
3621 }
3622 if (!added)
3623 list_add_tail(&newfs->list, &bp->rx_fs_list.list);
3624
3625 gem_prog_cmp_regs(bp, fs);
3626 bp->rx_fs_list.count++;
3627 /* enable filtering if NTUPLE on */
3628 gem_enable_flow_filters(bp, 1);
3629
3630 spin_unlock_irqrestore(&bp->rx_fs_lock, flags);
3631 return 0;
3632
3633 err:
3634 spin_unlock_irqrestore(&bp->rx_fs_lock, flags);
3635 kfree(newfs);
3636 return ret;
3637 }
3638
gem_del_flow_filter(struct net_device * netdev,struct ethtool_rxnfc * cmd)3639 static int gem_del_flow_filter(struct net_device *netdev,
3640 struct ethtool_rxnfc *cmd)
3641 {
3642 struct macb *bp = netdev_priv(netdev);
3643 struct ethtool_rx_fs_item *item;
3644 struct ethtool_rx_flow_spec *fs;
3645 unsigned long flags;
3646
3647 spin_lock_irqsave(&bp->rx_fs_lock, flags);
3648
3649 list_for_each_entry(item, &bp->rx_fs_list.list, list) {
3650 if (item->fs.location == cmd->fs.location) {
3651 /* disable screener regs for the flow entry */
3652 fs = &(item->fs);
3653 netdev_dbg(netdev,
3654 "Deleting flow filter entry,type=%u,queue=%u,loc=%u,src=%08X,dst=%08X,ps=%u,pd=%u\n",
3655 fs->flow_type, (int)fs->ring_cookie, fs->location,
3656 htonl(fs->h_u.tcp_ip4_spec.ip4src),
3657 htonl(fs->h_u.tcp_ip4_spec.ip4dst),
3658 be16_to_cpu(fs->h_u.tcp_ip4_spec.psrc),
3659 be16_to_cpu(fs->h_u.tcp_ip4_spec.pdst));
3660
3661 gem_writel_n(bp, SCRT2, fs->location, 0);
3662
3663 list_del(&item->list);
3664 bp->rx_fs_list.count--;
3665 spin_unlock_irqrestore(&bp->rx_fs_lock, flags);
3666 kfree(item);
3667 return 0;
3668 }
3669 }
3670
3671 spin_unlock_irqrestore(&bp->rx_fs_lock, flags);
3672 return -EINVAL;
3673 }
3674
gem_get_flow_entry(struct net_device * netdev,struct ethtool_rxnfc * cmd)3675 static int gem_get_flow_entry(struct net_device *netdev,
3676 struct ethtool_rxnfc *cmd)
3677 {
3678 struct macb *bp = netdev_priv(netdev);
3679 struct ethtool_rx_fs_item *item;
3680
3681 list_for_each_entry(item, &bp->rx_fs_list.list, list) {
3682 if (item->fs.location == cmd->fs.location) {
3683 memcpy(&cmd->fs, &item->fs, sizeof(cmd->fs));
3684 return 0;
3685 }
3686 }
3687 return -EINVAL;
3688 }
3689
gem_get_all_flow_entries(struct net_device * netdev,struct ethtool_rxnfc * cmd,u32 * rule_locs)3690 static int gem_get_all_flow_entries(struct net_device *netdev,
3691 struct ethtool_rxnfc *cmd, u32 *rule_locs)
3692 {
3693 struct macb *bp = netdev_priv(netdev);
3694 struct ethtool_rx_fs_item *item;
3695 uint32_t cnt = 0;
3696
3697 list_for_each_entry(item, &bp->rx_fs_list.list, list) {
3698 if (cnt == cmd->rule_cnt)
3699 return -EMSGSIZE;
3700 rule_locs[cnt] = item->fs.location;
3701 cnt++;
3702 }
3703 cmd->data = bp->max_tuples;
3704 cmd->rule_cnt = cnt;
3705
3706 return 0;
3707 }
3708
gem_get_rxnfc(struct net_device * netdev,struct ethtool_rxnfc * cmd,u32 * rule_locs)3709 static int gem_get_rxnfc(struct net_device *netdev, struct ethtool_rxnfc *cmd,
3710 u32 *rule_locs)
3711 {
3712 struct macb *bp = netdev_priv(netdev);
3713 int ret = 0;
3714
3715 switch (cmd->cmd) {
3716 case ETHTOOL_GRXRINGS:
3717 cmd->data = bp->num_queues;
3718 break;
3719 case ETHTOOL_GRXCLSRLCNT:
3720 cmd->rule_cnt = bp->rx_fs_list.count;
3721 break;
3722 case ETHTOOL_GRXCLSRULE:
3723 ret = gem_get_flow_entry(netdev, cmd);
3724 break;
3725 case ETHTOOL_GRXCLSRLALL:
3726 ret = gem_get_all_flow_entries(netdev, cmd, rule_locs);
3727 break;
3728 default:
3729 netdev_err(netdev,
3730 "Command parameter %d is not supported\n", cmd->cmd);
3731 ret = -EOPNOTSUPP;
3732 }
3733
3734 return ret;
3735 }
3736
gem_set_rxnfc(struct net_device * netdev,struct ethtool_rxnfc * cmd)3737 static int gem_set_rxnfc(struct net_device *netdev, struct ethtool_rxnfc *cmd)
3738 {
3739 struct macb *bp = netdev_priv(netdev);
3740 int ret;
3741
3742 switch (cmd->cmd) {
3743 case ETHTOOL_SRXCLSRLINS:
3744 if ((cmd->fs.location >= bp->max_tuples)
3745 || (cmd->fs.ring_cookie >= bp->num_queues)) {
3746 ret = -EINVAL;
3747 break;
3748 }
3749 ret = gem_add_flow_filter(netdev, cmd);
3750 break;
3751 case ETHTOOL_SRXCLSRLDEL:
3752 ret = gem_del_flow_filter(netdev, cmd);
3753 break;
3754 default:
3755 netdev_err(netdev,
3756 "Command parameter %d is not supported\n", cmd->cmd);
3757 ret = -EOPNOTSUPP;
3758 }
3759
3760 return ret;
3761 }
3762
3763 static const struct ethtool_ops macb_ethtool_ops = {
3764 .get_regs_len = macb_get_regs_len,
3765 .get_regs = macb_get_regs,
3766 .get_link = ethtool_op_get_link,
3767 .get_ts_info = ethtool_op_get_ts_info,
3768 .get_wol = macb_get_wol,
3769 .set_wol = macb_set_wol,
3770 .get_link_ksettings = macb_get_link_ksettings,
3771 .set_link_ksettings = macb_set_link_ksettings,
3772 .get_ringparam = macb_get_ringparam,
3773 .set_ringparam = macb_set_ringparam,
3774 };
3775
3776 static const struct ethtool_ops gem_ethtool_ops = {
3777 .get_regs_len = macb_get_regs_len,
3778 .get_regs = macb_get_regs,
3779 .get_wol = macb_get_wol,
3780 .set_wol = macb_set_wol,
3781 .get_link = ethtool_op_get_link,
3782 .get_ts_info = macb_get_ts_info,
3783 .get_ethtool_stats = gem_get_ethtool_stats,
3784 .get_strings = gem_get_ethtool_strings,
3785 .get_sset_count = gem_get_sset_count,
3786 .get_link_ksettings = macb_get_link_ksettings,
3787 .set_link_ksettings = macb_set_link_ksettings,
3788 .get_ringparam = macb_get_ringparam,
3789 .set_ringparam = macb_set_ringparam,
3790 .get_rxnfc = gem_get_rxnfc,
3791 .set_rxnfc = gem_set_rxnfc,
3792 };
3793
macb_ioctl(struct net_device * dev,struct ifreq * rq,int cmd)3794 static int macb_ioctl(struct net_device *dev, struct ifreq *rq, int cmd)
3795 {
3796 struct macb *bp = netdev_priv(dev);
3797
3798 if (!netif_running(dev))
3799 return -EINVAL;
3800
3801 return phylink_mii_ioctl(bp->phylink, rq, cmd);
3802 }
3803
macb_hwtstamp_get(struct net_device * dev,struct kernel_hwtstamp_config * cfg)3804 static int macb_hwtstamp_get(struct net_device *dev,
3805 struct kernel_hwtstamp_config *cfg)
3806 {
3807 struct macb *bp = netdev_priv(dev);
3808
3809 if (!netif_running(dev))
3810 return -EINVAL;
3811
3812 if (!bp->ptp_info)
3813 return -EOPNOTSUPP;
3814
3815 return bp->ptp_info->get_hwtst(dev, cfg);
3816 }
3817
macb_hwtstamp_set(struct net_device * dev,struct kernel_hwtstamp_config * cfg,struct netlink_ext_ack * extack)3818 static int macb_hwtstamp_set(struct net_device *dev,
3819 struct kernel_hwtstamp_config *cfg,
3820 struct netlink_ext_ack *extack)
3821 {
3822 struct macb *bp = netdev_priv(dev);
3823
3824 if (!netif_running(dev))
3825 return -EINVAL;
3826
3827 if (!bp->ptp_info)
3828 return -EOPNOTSUPP;
3829
3830 return bp->ptp_info->set_hwtst(dev, cfg, extack);
3831 }
3832
macb_set_txcsum_feature(struct macb * bp,netdev_features_t features)3833 static inline void macb_set_txcsum_feature(struct macb *bp,
3834 netdev_features_t features)
3835 {
3836 u32 val;
3837
3838 if (!macb_is_gem(bp))
3839 return;
3840
3841 val = gem_readl(bp, DMACFG);
3842 if (features & NETIF_F_HW_CSUM)
3843 val |= GEM_BIT(TXCOEN);
3844 else
3845 val &= ~GEM_BIT(TXCOEN);
3846
3847 gem_writel(bp, DMACFG, val);
3848 }
3849
macb_set_rxcsum_feature(struct macb * bp,netdev_features_t features)3850 static inline void macb_set_rxcsum_feature(struct macb *bp,
3851 netdev_features_t features)
3852 {
3853 struct net_device *netdev = bp->dev;
3854 u32 val;
3855
3856 if (!macb_is_gem(bp))
3857 return;
3858
3859 val = gem_readl(bp, NCFGR);
3860 if ((features & NETIF_F_RXCSUM) && !(netdev->flags & IFF_PROMISC))
3861 val |= GEM_BIT(RXCOEN);
3862 else
3863 val &= ~GEM_BIT(RXCOEN);
3864
3865 gem_writel(bp, NCFGR, val);
3866 }
3867
macb_set_rxflow_feature(struct macb * bp,netdev_features_t features)3868 static inline void macb_set_rxflow_feature(struct macb *bp,
3869 netdev_features_t features)
3870 {
3871 if (!macb_is_gem(bp))
3872 return;
3873
3874 gem_enable_flow_filters(bp, !!(features & NETIF_F_NTUPLE));
3875 }
3876
macb_set_features(struct net_device * netdev,netdev_features_t features)3877 static int macb_set_features(struct net_device *netdev,
3878 netdev_features_t features)
3879 {
3880 struct macb *bp = netdev_priv(netdev);
3881 netdev_features_t changed = features ^ netdev->features;
3882
3883 /* TX checksum offload */
3884 if (changed & NETIF_F_HW_CSUM)
3885 macb_set_txcsum_feature(bp, features);
3886
3887 /* RX checksum offload */
3888 if (changed & NETIF_F_RXCSUM)
3889 macb_set_rxcsum_feature(bp, features);
3890
3891 /* RX Flow Filters */
3892 if (changed & NETIF_F_NTUPLE)
3893 macb_set_rxflow_feature(bp, features);
3894
3895 return 0;
3896 }
3897
macb_restore_features(struct macb * bp)3898 static void macb_restore_features(struct macb *bp)
3899 {
3900 struct net_device *netdev = bp->dev;
3901 netdev_features_t features = netdev->features;
3902 struct ethtool_rx_fs_item *item;
3903
3904 /* TX checksum offload */
3905 macb_set_txcsum_feature(bp, features);
3906
3907 /* RX checksum offload */
3908 macb_set_rxcsum_feature(bp, features);
3909
3910 /* RX Flow Filters */
3911 list_for_each_entry(item, &bp->rx_fs_list.list, list)
3912 gem_prog_cmp_regs(bp, &item->fs);
3913
3914 macb_set_rxflow_feature(bp, features);
3915 }
3916
3917 static const struct net_device_ops macb_netdev_ops = {
3918 .ndo_open = macb_open,
3919 .ndo_stop = macb_close,
3920 .ndo_start_xmit = macb_start_xmit,
3921 .ndo_set_rx_mode = macb_set_rx_mode,
3922 .ndo_get_stats = macb_get_stats,
3923 .ndo_eth_ioctl = macb_ioctl,
3924 .ndo_validate_addr = eth_validate_addr,
3925 .ndo_change_mtu = macb_change_mtu,
3926 .ndo_set_mac_address = macb_set_mac_addr,
3927 #ifdef CONFIG_NET_POLL_CONTROLLER
3928 .ndo_poll_controller = macb_poll_controller,
3929 #endif
3930 .ndo_set_features = macb_set_features,
3931 .ndo_features_check = macb_features_check,
3932 .ndo_hwtstamp_set = macb_hwtstamp_set,
3933 .ndo_hwtstamp_get = macb_hwtstamp_get,
3934 };
3935
3936 /* Configure peripheral capabilities according to device tree
3937 * and integration options used
3938 */
macb_configure_caps(struct macb * bp,const struct macb_config * dt_conf)3939 static void macb_configure_caps(struct macb *bp,
3940 const struct macb_config *dt_conf)
3941 {
3942 u32 dcfg;
3943
3944 if (dt_conf)
3945 bp->caps = dt_conf->caps;
3946
3947 if (hw_is_gem(bp->regs, bp->native_io)) {
3948 bp->caps |= MACB_CAPS_MACB_IS_GEM;
3949
3950 dcfg = gem_readl(bp, DCFG1);
3951 if (GEM_BFEXT(IRQCOR, dcfg) == 0)
3952 bp->caps |= MACB_CAPS_ISR_CLEAR_ON_WRITE;
3953 if (GEM_BFEXT(NO_PCS, dcfg) == 0)
3954 bp->caps |= MACB_CAPS_PCS;
3955 dcfg = gem_readl(bp, DCFG12);
3956 if (GEM_BFEXT(HIGH_SPEED, dcfg) == 1)
3957 bp->caps |= MACB_CAPS_HIGH_SPEED;
3958 dcfg = gem_readl(bp, DCFG2);
3959 if ((dcfg & (GEM_BIT(RX_PKT_BUFF) | GEM_BIT(TX_PKT_BUFF))) == 0)
3960 bp->caps |= MACB_CAPS_FIFO_MODE;
3961 if (gem_has_ptp(bp)) {
3962 if (!GEM_BFEXT(TSU, gem_readl(bp, DCFG5)))
3963 dev_err(&bp->pdev->dev,
3964 "GEM doesn't support hardware ptp.\n");
3965 else {
3966 #ifdef CONFIG_MACB_USE_HWSTAMP
3967 bp->hw_dma_cap |= HW_DMA_CAP_PTP;
3968 bp->ptp_info = &gem_ptp_info;
3969 #endif
3970 }
3971 }
3972 }
3973
3974 dev_dbg(&bp->pdev->dev, "Cadence caps 0x%08x\n", bp->caps);
3975 }
3976
macb_probe_queues(void __iomem * mem,bool native_io,unsigned int * queue_mask,unsigned int * num_queues)3977 static void macb_probe_queues(void __iomem *mem,
3978 bool native_io,
3979 unsigned int *queue_mask,
3980 unsigned int *num_queues)
3981 {
3982 *queue_mask = 0x1;
3983 *num_queues = 1;
3984
3985 /* is it macb or gem ?
3986 *
3987 * We need to read directly from the hardware here because
3988 * we are early in the probe process and don't have the
3989 * MACB_CAPS_MACB_IS_GEM flag positioned
3990 */
3991 if (!hw_is_gem(mem, native_io))
3992 return;
3993
3994 /* bit 0 is never set but queue 0 always exists */
3995 *queue_mask |= readl_relaxed(mem + GEM_DCFG6) & 0xff;
3996 *num_queues = hweight32(*queue_mask);
3997 }
3998
macb_clks_disable(struct clk * pclk,struct clk * hclk,struct clk * tx_clk,struct clk * rx_clk,struct clk * tsu_clk)3999 static void macb_clks_disable(struct clk *pclk, struct clk *hclk, struct clk *tx_clk,
4000 struct clk *rx_clk, struct clk *tsu_clk)
4001 {
4002 struct clk_bulk_data clks[] = {
4003 { .clk = tsu_clk, },
4004 { .clk = rx_clk, },
4005 { .clk = pclk, },
4006 { .clk = hclk, },
4007 { .clk = tx_clk },
4008 };
4009
4010 clk_bulk_disable_unprepare(ARRAY_SIZE(clks), clks);
4011 }
4012
macb_clk_init(struct platform_device * pdev,struct clk ** pclk,struct clk ** hclk,struct clk ** tx_clk,struct clk ** rx_clk,struct clk ** tsu_clk)4013 static int macb_clk_init(struct platform_device *pdev, struct clk **pclk,
4014 struct clk **hclk, struct clk **tx_clk,
4015 struct clk **rx_clk, struct clk **tsu_clk)
4016 {
4017 struct macb_platform_data *pdata;
4018 int err;
4019
4020 pdata = dev_get_platdata(&pdev->dev);
4021 if (pdata) {
4022 *pclk = pdata->pclk;
4023 *hclk = pdata->hclk;
4024 } else {
4025 *pclk = devm_clk_get(&pdev->dev, "pclk");
4026 *hclk = devm_clk_get(&pdev->dev, "hclk");
4027 }
4028
4029 if (IS_ERR_OR_NULL(*pclk))
4030 return dev_err_probe(&pdev->dev,
4031 IS_ERR(*pclk) ? PTR_ERR(*pclk) : -ENODEV,
4032 "failed to get pclk\n");
4033
4034 if (IS_ERR_OR_NULL(*hclk))
4035 return dev_err_probe(&pdev->dev,
4036 IS_ERR(*hclk) ? PTR_ERR(*hclk) : -ENODEV,
4037 "failed to get hclk\n");
4038
4039 *tx_clk = devm_clk_get_optional(&pdev->dev, "tx_clk");
4040 if (IS_ERR(*tx_clk))
4041 return PTR_ERR(*tx_clk);
4042
4043 *rx_clk = devm_clk_get_optional(&pdev->dev, "rx_clk");
4044 if (IS_ERR(*rx_clk))
4045 return PTR_ERR(*rx_clk);
4046
4047 *tsu_clk = devm_clk_get_optional(&pdev->dev, "tsu_clk");
4048 if (IS_ERR(*tsu_clk))
4049 return PTR_ERR(*tsu_clk);
4050
4051 err = clk_prepare_enable(*pclk);
4052 if (err) {
4053 dev_err(&pdev->dev, "failed to enable pclk (%d)\n", err);
4054 return err;
4055 }
4056
4057 err = clk_prepare_enable(*hclk);
4058 if (err) {
4059 dev_err(&pdev->dev, "failed to enable hclk (%d)\n", err);
4060 goto err_disable_pclk;
4061 }
4062
4063 err = clk_prepare_enable(*tx_clk);
4064 if (err) {
4065 dev_err(&pdev->dev, "failed to enable tx_clk (%d)\n", err);
4066 goto err_disable_hclk;
4067 }
4068
4069 err = clk_prepare_enable(*rx_clk);
4070 if (err) {
4071 dev_err(&pdev->dev, "failed to enable rx_clk (%d)\n", err);
4072 goto err_disable_txclk;
4073 }
4074
4075 err = clk_prepare_enable(*tsu_clk);
4076 if (err) {
4077 dev_err(&pdev->dev, "failed to enable tsu_clk (%d)\n", err);
4078 goto err_disable_rxclk;
4079 }
4080
4081 return 0;
4082
4083 err_disable_rxclk:
4084 clk_disable_unprepare(*rx_clk);
4085
4086 err_disable_txclk:
4087 clk_disable_unprepare(*tx_clk);
4088
4089 err_disable_hclk:
4090 clk_disable_unprepare(*hclk);
4091
4092 err_disable_pclk:
4093 clk_disable_unprepare(*pclk);
4094
4095 return err;
4096 }
4097
macb_init(struct platform_device * pdev)4098 static int macb_init(struct platform_device *pdev)
4099 {
4100 struct net_device *dev = platform_get_drvdata(pdev);
4101 unsigned int hw_q, q;
4102 struct macb *bp = netdev_priv(dev);
4103 struct macb_queue *queue;
4104 int err;
4105 u32 val, reg;
4106
4107 bp->tx_ring_size = DEFAULT_TX_RING_SIZE;
4108 bp->rx_ring_size = DEFAULT_RX_RING_SIZE;
4109
4110 /* set the queue register mapping once for all: queue0 has a special
4111 * register mapping but we don't want to test the queue index then
4112 * compute the corresponding register offset at run time.
4113 */
4114 for (hw_q = 0, q = 0; hw_q < MACB_MAX_QUEUES; ++hw_q) {
4115 if (!(bp->queue_mask & (1 << hw_q)))
4116 continue;
4117
4118 queue = &bp->queues[q];
4119 queue->bp = bp;
4120 spin_lock_init(&queue->tx_ptr_lock);
4121 netif_napi_add(dev, &queue->napi_rx, macb_rx_poll);
4122 netif_napi_add(dev, &queue->napi_tx, macb_tx_poll);
4123 if (hw_q) {
4124 queue->ISR = GEM_ISR(hw_q - 1);
4125 queue->IER = GEM_IER(hw_q - 1);
4126 queue->IDR = GEM_IDR(hw_q - 1);
4127 queue->IMR = GEM_IMR(hw_q - 1);
4128 queue->TBQP = GEM_TBQP(hw_q - 1);
4129 queue->RBQP = GEM_RBQP(hw_q - 1);
4130 queue->RBQS = GEM_RBQS(hw_q - 1);
4131 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT
4132 if (bp->hw_dma_cap & HW_DMA_CAP_64B) {
4133 queue->TBQPH = GEM_TBQPH(hw_q - 1);
4134 queue->RBQPH = GEM_RBQPH(hw_q - 1);
4135 }
4136 #endif
4137 } else {
4138 /* queue0 uses legacy registers */
4139 queue->ISR = MACB_ISR;
4140 queue->IER = MACB_IER;
4141 queue->IDR = MACB_IDR;
4142 queue->IMR = MACB_IMR;
4143 queue->TBQP = MACB_TBQP;
4144 queue->RBQP = MACB_RBQP;
4145 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT
4146 if (bp->hw_dma_cap & HW_DMA_CAP_64B) {
4147 queue->TBQPH = MACB_TBQPH;
4148 queue->RBQPH = MACB_RBQPH;
4149 }
4150 #endif
4151 }
4152
4153 /* get irq: here we use the linux queue index, not the hardware
4154 * queue index. the queue irq definitions in the device tree
4155 * must remove the optional gaps that could exist in the
4156 * hardware queue mask.
4157 */
4158 queue->irq = platform_get_irq(pdev, q);
4159 err = devm_request_irq(&pdev->dev, queue->irq, macb_interrupt,
4160 IRQF_SHARED, dev->name, queue);
4161 if (err) {
4162 dev_err(&pdev->dev,
4163 "Unable to request IRQ %d (error %d)\n",
4164 queue->irq, err);
4165 return err;
4166 }
4167
4168 INIT_WORK(&queue->tx_error_task, macb_tx_error_task);
4169 q++;
4170 }
4171
4172 dev->netdev_ops = &macb_netdev_ops;
4173
4174 /* setup appropriated routines according to adapter type */
4175 if (macb_is_gem(bp)) {
4176 bp->macbgem_ops.mog_alloc_rx_buffers = gem_alloc_rx_buffers;
4177 bp->macbgem_ops.mog_free_rx_buffers = gem_free_rx_buffers;
4178 bp->macbgem_ops.mog_init_rings = gem_init_rings;
4179 bp->macbgem_ops.mog_rx = gem_rx;
4180 dev->ethtool_ops = &gem_ethtool_ops;
4181 } else {
4182 bp->macbgem_ops.mog_alloc_rx_buffers = macb_alloc_rx_buffers;
4183 bp->macbgem_ops.mog_free_rx_buffers = macb_free_rx_buffers;
4184 bp->macbgem_ops.mog_init_rings = macb_init_rings;
4185 bp->macbgem_ops.mog_rx = macb_rx;
4186 dev->ethtool_ops = &macb_ethtool_ops;
4187 }
4188
4189 netdev_sw_irq_coalesce_default_on(dev);
4190
4191 dev->priv_flags |= IFF_LIVE_ADDR_CHANGE;
4192
4193 /* Set features */
4194 dev->hw_features = NETIF_F_SG;
4195
4196 /* Check LSO capability */
4197 if (GEM_BFEXT(PBUF_LSO, gem_readl(bp, DCFG6)))
4198 dev->hw_features |= MACB_NETIF_LSO;
4199
4200 /* Checksum offload is only available on gem with packet buffer */
4201 if (macb_is_gem(bp) && !(bp->caps & MACB_CAPS_FIFO_MODE))
4202 dev->hw_features |= NETIF_F_HW_CSUM | NETIF_F_RXCSUM;
4203 if (bp->caps & MACB_CAPS_SG_DISABLED)
4204 dev->hw_features &= ~NETIF_F_SG;
4205 dev->features = dev->hw_features;
4206
4207 /* Check RX Flow Filters support.
4208 * Max Rx flows set by availability of screeners & compare regs:
4209 * each 4-tuple define requires 1 T2 screener reg + 3 compare regs
4210 */
4211 reg = gem_readl(bp, DCFG8);
4212 bp->max_tuples = min((GEM_BFEXT(SCR2CMP, reg) / 3),
4213 GEM_BFEXT(T2SCR, reg));
4214 INIT_LIST_HEAD(&bp->rx_fs_list.list);
4215 if (bp->max_tuples > 0) {
4216 /* also needs one ethtype match to check IPv4 */
4217 if (GEM_BFEXT(SCR2ETH, reg) > 0) {
4218 /* program this reg now */
4219 reg = 0;
4220 reg = GEM_BFINS(ETHTCMP, (uint16_t)ETH_P_IP, reg);
4221 gem_writel_n(bp, ETHT, SCRT2_ETHT, reg);
4222 /* Filtering is supported in hw but don't enable it in kernel now */
4223 dev->hw_features |= NETIF_F_NTUPLE;
4224 /* init Rx flow definitions */
4225 bp->rx_fs_list.count = 0;
4226 spin_lock_init(&bp->rx_fs_lock);
4227 } else
4228 bp->max_tuples = 0;
4229 }
4230
4231 if (!(bp->caps & MACB_CAPS_USRIO_DISABLED)) {
4232 val = 0;
4233 if (phy_interface_mode_is_rgmii(bp->phy_interface))
4234 val = bp->usrio->rgmii;
4235 else if (bp->phy_interface == PHY_INTERFACE_MODE_RMII &&
4236 (bp->caps & MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII))
4237 val = bp->usrio->rmii;
4238 else if (!(bp->caps & MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII))
4239 val = bp->usrio->mii;
4240
4241 if (bp->caps & MACB_CAPS_USRIO_HAS_CLKEN)
4242 val |= bp->usrio->refclk;
4243
4244 macb_or_gem_writel(bp, USRIO, val);
4245 }
4246
4247 /* Set MII management clock divider */
4248 val = macb_mdc_clk_div(bp);
4249 val |= macb_dbw(bp);
4250 if (bp->phy_interface == PHY_INTERFACE_MODE_SGMII)
4251 val |= GEM_BIT(SGMIIEN) | GEM_BIT(PCSSEL);
4252 macb_writel(bp, NCFGR, val);
4253
4254 return 0;
4255 }
4256
4257 static const struct macb_usrio_config macb_default_usrio = {
4258 .mii = MACB_BIT(MII),
4259 .rmii = MACB_BIT(RMII),
4260 .rgmii = GEM_BIT(RGMII),
4261 .refclk = MACB_BIT(CLKEN),
4262 };
4263
4264 #if defined(CONFIG_OF)
4265 /* 1518 rounded up */
4266 #define AT91ETHER_MAX_RBUFF_SZ 0x600
4267 /* max number of receive buffers */
4268 #define AT91ETHER_MAX_RX_DESCR 9
4269
4270 static struct sifive_fu540_macb_mgmt *mgmt;
4271
at91ether_alloc_coherent(struct macb * lp)4272 static int at91ether_alloc_coherent(struct macb *lp)
4273 {
4274 struct macb_queue *q = &lp->queues[0];
4275
4276 q->rx_ring = dma_alloc_coherent(&lp->pdev->dev,
4277 (AT91ETHER_MAX_RX_DESCR *
4278 macb_dma_desc_get_size(lp)),
4279 &q->rx_ring_dma, GFP_KERNEL);
4280 if (!q->rx_ring)
4281 return -ENOMEM;
4282
4283 q->rx_buffers = dma_alloc_coherent(&lp->pdev->dev,
4284 AT91ETHER_MAX_RX_DESCR *
4285 AT91ETHER_MAX_RBUFF_SZ,
4286 &q->rx_buffers_dma, GFP_KERNEL);
4287 if (!q->rx_buffers) {
4288 dma_free_coherent(&lp->pdev->dev,
4289 AT91ETHER_MAX_RX_DESCR *
4290 macb_dma_desc_get_size(lp),
4291 q->rx_ring, q->rx_ring_dma);
4292 q->rx_ring = NULL;
4293 return -ENOMEM;
4294 }
4295
4296 return 0;
4297 }
4298
at91ether_free_coherent(struct macb * lp)4299 static void at91ether_free_coherent(struct macb *lp)
4300 {
4301 struct macb_queue *q = &lp->queues[0];
4302
4303 if (q->rx_ring) {
4304 dma_free_coherent(&lp->pdev->dev,
4305 AT91ETHER_MAX_RX_DESCR *
4306 macb_dma_desc_get_size(lp),
4307 q->rx_ring, q->rx_ring_dma);
4308 q->rx_ring = NULL;
4309 }
4310
4311 if (q->rx_buffers) {
4312 dma_free_coherent(&lp->pdev->dev,
4313 AT91ETHER_MAX_RX_DESCR *
4314 AT91ETHER_MAX_RBUFF_SZ,
4315 q->rx_buffers, q->rx_buffers_dma);
4316 q->rx_buffers = NULL;
4317 }
4318 }
4319
4320 /* Initialize and start the Receiver and Transmit subsystems */
at91ether_start(struct macb * lp)4321 static int at91ether_start(struct macb *lp)
4322 {
4323 struct macb_queue *q = &lp->queues[0];
4324 struct macb_dma_desc *desc;
4325 dma_addr_t addr;
4326 u32 ctl;
4327 int i, ret;
4328
4329 ret = at91ether_alloc_coherent(lp);
4330 if (ret)
4331 return ret;
4332
4333 addr = q->rx_buffers_dma;
4334 for (i = 0; i < AT91ETHER_MAX_RX_DESCR; i++) {
4335 desc = macb_rx_desc(q, i);
4336 macb_set_addr(lp, desc, addr);
4337 desc->ctrl = 0;
4338 addr += AT91ETHER_MAX_RBUFF_SZ;
4339 }
4340
4341 /* Set the Wrap bit on the last descriptor */
4342 desc->addr |= MACB_BIT(RX_WRAP);
4343
4344 /* Reset buffer index */
4345 q->rx_tail = 0;
4346
4347 /* Program address of descriptor list in Rx Buffer Queue register */
4348 macb_writel(lp, RBQP, q->rx_ring_dma);
4349
4350 /* Enable Receive and Transmit */
4351 ctl = macb_readl(lp, NCR);
4352 macb_writel(lp, NCR, ctl | MACB_BIT(RE) | MACB_BIT(TE));
4353
4354 /* Enable MAC interrupts */
4355 macb_writel(lp, IER, MACB_BIT(RCOMP) |
4356 MACB_BIT(RXUBR) |
4357 MACB_BIT(ISR_TUND) |
4358 MACB_BIT(ISR_RLE) |
4359 MACB_BIT(TCOMP) |
4360 MACB_BIT(ISR_ROVR) |
4361 MACB_BIT(HRESP));
4362
4363 return 0;
4364 }
4365
at91ether_stop(struct macb * lp)4366 static void at91ether_stop(struct macb *lp)
4367 {
4368 u32 ctl;
4369
4370 /* Disable MAC interrupts */
4371 macb_writel(lp, IDR, MACB_BIT(RCOMP) |
4372 MACB_BIT(RXUBR) |
4373 MACB_BIT(ISR_TUND) |
4374 MACB_BIT(ISR_RLE) |
4375 MACB_BIT(TCOMP) |
4376 MACB_BIT(ISR_ROVR) |
4377 MACB_BIT(HRESP));
4378
4379 /* Disable Receiver and Transmitter */
4380 ctl = macb_readl(lp, NCR);
4381 macb_writel(lp, NCR, ctl & ~(MACB_BIT(TE) | MACB_BIT(RE)));
4382
4383 /* Free resources. */
4384 at91ether_free_coherent(lp);
4385 }
4386
4387 /* Open the ethernet interface */
at91ether_open(struct net_device * dev)4388 static int at91ether_open(struct net_device *dev)
4389 {
4390 struct macb *lp = netdev_priv(dev);
4391 u32 ctl;
4392 int ret;
4393
4394 ret = pm_runtime_resume_and_get(&lp->pdev->dev);
4395 if (ret < 0)
4396 return ret;
4397
4398 /* Clear internal statistics */
4399 ctl = macb_readl(lp, NCR);
4400 macb_writel(lp, NCR, ctl | MACB_BIT(CLRSTAT));
4401
4402 macb_set_hwaddr(lp);
4403
4404 ret = at91ether_start(lp);
4405 if (ret)
4406 goto pm_exit;
4407
4408 ret = macb_phylink_connect(lp);
4409 if (ret)
4410 goto stop;
4411
4412 netif_start_queue(dev);
4413
4414 return 0;
4415
4416 stop:
4417 at91ether_stop(lp);
4418 pm_exit:
4419 pm_runtime_put_sync(&lp->pdev->dev);
4420 return ret;
4421 }
4422
4423 /* Close the interface */
at91ether_close(struct net_device * dev)4424 static int at91ether_close(struct net_device *dev)
4425 {
4426 struct macb *lp = netdev_priv(dev);
4427
4428 netif_stop_queue(dev);
4429
4430 phylink_stop(lp->phylink);
4431 phylink_disconnect_phy(lp->phylink);
4432
4433 at91ether_stop(lp);
4434
4435 return pm_runtime_put(&lp->pdev->dev);
4436 }
4437
4438 /* Transmit packet */
at91ether_start_xmit(struct sk_buff * skb,struct net_device * dev)4439 static netdev_tx_t at91ether_start_xmit(struct sk_buff *skb,
4440 struct net_device *dev)
4441 {
4442 struct macb *lp = netdev_priv(dev);
4443
4444 if (macb_readl(lp, TSR) & MACB_BIT(RM9200_BNQ)) {
4445 int desc = 0;
4446
4447 netif_stop_queue(dev);
4448
4449 /* Store packet information (to free when Tx completed) */
4450 lp->rm9200_txq[desc].skb = skb;
4451 lp->rm9200_txq[desc].size = skb->len;
4452 lp->rm9200_txq[desc].mapping = dma_map_single(&lp->pdev->dev, skb->data,
4453 skb->len, DMA_TO_DEVICE);
4454 if (dma_mapping_error(&lp->pdev->dev, lp->rm9200_txq[desc].mapping)) {
4455 dev_kfree_skb_any(skb);
4456 dev->stats.tx_dropped++;
4457 netdev_err(dev, "%s: DMA mapping error\n", __func__);
4458 return NETDEV_TX_OK;
4459 }
4460
4461 /* Set address of the data in the Transmit Address register */
4462 macb_writel(lp, TAR, lp->rm9200_txq[desc].mapping);
4463 /* Set length of the packet in the Transmit Control register */
4464 macb_writel(lp, TCR, skb->len);
4465
4466 } else {
4467 netdev_err(dev, "%s called, but device is busy!\n", __func__);
4468 return NETDEV_TX_BUSY;
4469 }
4470
4471 return NETDEV_TX_OK;
4472 }
4473
4474 /* Extract received frame from buffer descriptors and sent to upper layers.
4475 * (Called from interrupt context)
4476 */
at91ether_rx(struct net_device * dev)4477 static void at91ether_rx(struct net_device *dev)
4478 {
4479 struct macb *lp = netdev_priv(dev);
4480 struct macb_queue *q = &lp->queues[0];
4481 struct macb_dma_desc *desc;
4482 unsigned char *p_recv;
4483 struct sk_buff *skb;
4484 unsigned int pktlen;
4485
4486 desc = macb_rx_desc(q, q->rx_tail);
4487 while (desc->addr & MACB_BIT(RX_USED)) {
4488 p_recv = q->rx_buffers + q->rx_tail * AT91ETHER_MAX_RBUFF_SZ;
4489 pktlen = MACB_BF(RX_FRMLEN, desc->ctrl);
4490 skb = netdev_alloc_skb(dev, pktlen + 2);
4491 if (skb) {
4492 skb_reserve(skb, 2);
4493 skb_put_data(skb, p_recv, pktlen);
4494
4495 skb->protocol = eth_type_trans(skb, dev);
4496 dev->stats.rx_packets++;
4497 dev->stats.rx_bytes += pktlen;
4498 netif_rx(skb);
4499 } else {
4500 dev->stats.rx_dropped++;
4501 }
4502
4503 if (desc->ctrl & MACB_BIT(RX_MHASH_MATCH))
4504 dev->stats.multicast++;
4505
4506 /* reset ownership bit */
4507 desc->addr &= ~MACB_BIT(RX_USED);
4508
4509 /* wrap after last buffer */
4510 if (q->rx_tail == AT91ETHER_MAX_RX_DESCR - 1)
4511 q->rx_tail = 0;
4512 else
4513 q->rx_tail++;
4514
4515 desc = macb_rx_desc(q, q->rx_tail);
4516 }
4517 }
4518
4519 /* MAC interrupt handler */
at91ether_interrupt(int irq,void * dev_id)4520 static irqreturn_t at91ether_interrupt(int irq, void *dev_id)
4521 {
4522 struct net_device *dev = dev_id;
4523 struct macb *lp = netdev_priv(dev);
4524 u32 intstatus, ctl;
4525 unsigned int desc;
4526
4527 /* MAC Interrupt Status register indicates what interrupts are pending.
4528 * It is automatically cleared once read.
4529 */
4530 intstatus = macb_readl(lp, ISR);
4531
4532 /* Receive complete */
4533 if (intstatus & MACB_BIT(RCOMP))
4534 at91ether_rx(dev);
4535
4536 /* Transmit complete */
4537 if (intstatus & MACB_BIT(TCOMP)) {
4538 /* The TCOM bit is set even if the transmission failed */
4539 if (intstatus & (MACB_BIT(ISR_TUND) | MACB_BIT(ISR_RLE)))
4540 dev->stats.tx_errors++;
4541
4542 desc = 0;
4543 if (lp->rm9200_txq[desc].skb) {
4544 dev_consume_skb_irq(lp->rm9200_txq[desc].skb);
4545 lp->rm9200_txq[desc].skb = NULL;
4546 dma_unmap_single(&lp->pdev->dev, lp->rm9200_txq[desc].mapping,
4547 lp->rm9200_txq[desc].size, DMA_TO_DEVICE);
4548 dev->stats.tx_packets++;
4549 dev->stats.tx_bytes += lp->rm9200_txq[desc].size;
4550 }
4551 netif_wake_queue(dev);
4552 }
4553
4554 /* Work-around for EMAC Errata section 41.3.1 */
4555 if (intstatus & MACB_BIT(RXUBR)) {
4556 ctl = macb_readl(lp, NCR);
4557 macb_writel(lp, NCR, ctl & ~MACB_BIT(RE));
4558 wmb();
4559 macb_writel(lp, NCR, ctl | MACB_BIT(RE));
4560 }
4561
4562 if (intstatus & MACB_BIT(ISR_ROVR))
4563 netdev_err(dev, "ROVR error\n");
4564
4565 return IRQ_HANDLED;
4566 }
4567
4568 #ifdef CONFIG_NET_POLL_CONTROLLER
at91ether_poll_controller(struct net_device * dev)4569 static void at91ether_poll_controller(struct net_device *dev)
4570 {
4571 unsigned long flags;
4572
4573 local_irq_save(flags);
4574 at91ether_interrupt(dev->irq, dev);
4575 local_irq_restore(flags);
4576 }
4577 #endif
4578
4579 static const struct net_device_ops at91ether_netdev_ops = {
4580 .ndo_open = at91ether_open,
4581 .ndo_stop = at91ether_close,
4582 .ndo_start_xmit = at91ether_start_xmit,
4583 .ndo_get_stats = macb_get_stats,
4584 .ndo_set_rx_mode = macb_set_rx_mode,
4585 .ndo_set_mac_address = eth_mac_addr,
4586 .ndo_eth_ioctl = macb_ioctl,
4587 .ndo_validate_addr = eth_validate_addr,
4588 #ifdef CONFIG_NET_POLL_CONTROLLER
4589 .ndo_poll_controller = at91ether_poll_controller,
4590 #endif
4591 .ndo_hwtstamp_set = macb_hwtstamp_set,
4592 .ndo_hwtstamp_get = macb_hwtstamp_get,
4593 };
4594
at91ether_clk_init(struct platform_device * pdev,struct clk ** pclk,struct clk ** hclk,struct clk ** tx_clk,struct clk ** rx_clk,struct clk ** tsu_clk)4595 static int at91ether_clk_init(struct platform_device *pdev, struct clk **pclk,
4596 struct clk **hclk, struct clk **tx_clk,
4597 struct clk **rx_clk, struct clk **tsu_clk)
4598 {
4599 int err;
4600
4601 *hclk = NULL;
4602 *tx_clk = NULL;
4603 *rx_clk = NULL;
4604 *tsu_clk = NULL;
4605
4606 *pclk = devm_clk_get(&pdev->dev, "ether_clk");
4607 if (IS_ERR(*pclk))
4608 return PTR_ERR(*pclk);
4609
4610 err = clk_prepare_enable(*pclk);
4611 if (err) {
4612 dev_err(&pdev->dev, "failed to enable pclk (%d)\n", err);
4613 return err;
4614 }
4615
4616 return 0;
4617 }
4618
at91ether_init(struct platform_device * pdev)4619 static int at91ether_init(struct platform_device *pdev)
4620 {
4621 struct net_device *dev = platform_get_drvdata(pdev);
4622 struct macb *bp = netdev_priv(dev);
4623 int err;
4624
4625 bp->queues[0].bp = bp;
4626
4627 dev->netdev_ops = &at91ether_netdev_ops;
4628 dev->ethtool_ops = &macb_ethtool_ops;
4629
4630 err = devm_request_irq(&pdev->dev, dev->irq, at91ether_interrupt,
4631 0, dev->name, dev);
4632 if (err)
4633 return err;
4634
4635 macb_writel(bp, NCR, 0);
4636
4637 macb_writel(bp, NCFGR, MACB_BF(CLK, MACB_CLK_DIV32) | MACB_BIT(BIG));
4638
4639 return 0;
4640 }
4641
fu540_macb_tx_recalc_rate(struct clk_hw * hw,unsigned long parent_rate)4642 static unsigned long fu540_macb_tx_recalc_rate(struct clk_hw *hw,
4643 unsigned long parent_rate)
4644 {
4645 return mgmt->rate;
4646 }
4647
fu540_macb_tx_round_rate(struct clk_hw * hw,unsigned long rate,unsigned long * parent_rate)4648 static long fu540_macb_tx_round_rate(struct clk_hw *hw, unsigned long rate,
4649 unsigned long *parent_rate)
4650 {
4651 if (WARN_ON(rate < 2500000))
4652 return 2500000;
4653 else if (rate == 2500000)
4654 return 2500000;
4655 else if (WARN_ON(rate < 13750000))
4656 return 2500000;
4657 else if (WARN_ON(rate < 25000000))
4658 return 25000000;
4659 else if (rate == 25000000)
4660 return 25000000;
4661 else if (WARN_ON(rate < 75000000))
4662 return 25000000;
4663 else if (WARN_ON(rate < 125000000))
4664 return 125000000;
4665 else if (rate == 125000000)
4666 return 125000000;
4667
4668 WARN_ON(rate > 125000000);
4669
4670 return 125000000;
4671 }
4672
fu540_macb_tx_set_rate(struct clk_hw * hw,unsigned long rate,unsigned long parent_rate)4673 static int fu540_macb_tx_set_rate(struct clk_hw *hw, unsigned long rate,
4674 unsigned long parent_rate)
4675 {
4676 rate = fu540_macb_tx_round_rate(hw, rate, &parent_rate);
4677 if (rate != 125000000)
4678 iowrite32(1, mgmt->reg);
4679 else
4680 iowrite32(0, mgmt->reg);
4681 mgmt->rate = rate;
4682
4683 return 0;
4684 }
4685
4686 static const struct clk_ops fu540_c000_ops = {
4687 .recalc_rate = fu540_macb_tx_recalc_rate,
4688 .round_rate = fu540_macb_tx_round_rate,
4689 .set_rate = fu540_macb_tx_set_rate,
4690 };
4691
fu540_c000_clk_init(struct platform_device * pdev,struct clk ** pclk,struct clk ** hclk,struct clk ** tx_clk,struct clk ** rx_clk,struct clk ** tsu_clk)4692 static int fu540_c000_clk_init(struct platform_device *pdev, struct clk **pclk,
4693 struct clk **hclk, struct clk **tx_clk,
4694 struct clk **rx_clk, struct clk **tsu_clk)
4695 {
4696 struct clk_init_data init;
4697 int err = 0;
4698
4699 err = macb_clk_init(pdev, pclk, hclk, tx_clk, rx_clk, tsu_clk);
4700 if (err)
4701 return err;
4702
4703 mgmt = devm_kzalloc(&pdev->dev, sizeof(*mgmt), GFP_KERNEL);
4704 if (!mgmt) {
4705 err = -ENOMEM;
4706 goto err_disable_clks;
4707 }
4708
4709 init.name = "sifive-gemgxl-mgmt";
4710 init.ops = &fu540_c000_ops;
4711 init.flags = 0;
4712 init.num_parents = 0;
4713
4714 mgmt->rate = 0;
4715 mgmt->hw.init = &init;
4716
4717 *tx_clk = devm_clk_register(&pdev->dev, &mgmt->hw);
4718 if (IS_ERR(*tx_clk)) {
4719 err = PTR_ERR(*tx_clk);
4720 goto err_disable_clks;
4721 }
4722
4723 err = clk_prepare_enable(*tx_clk);
4724 if (err) {
4725 dev_err(&pdev->dev, "failed to enable tx_clk (%u)\n", err);
4726 *tx_clk = NULL;
4727 goto err_disable_clks;
4728 } else {
4729 dev_info(&pdev->dev, "Registered clk switch '%s'\n", init.name);
4730 }
4731
4732 return 0;
4733
4734 err_disable_clks:
4735 macb_clks_disable(*pclk, *hclk, *tx_clk, *rx_clk, *tsu_clk);
4736
4737 return err;
4738 }
4739
fu540_c000_init(struct platform_device * pdev)4740 static int fu540_c000_init(struct platform_device *pdev)
4741 {
4742 mgmt->reg = devm_platform_ioremap_resource(pdev, 1);
4743 if (IS_ERR(mgmt->reg))
4744 return PTR_ERR(mgmt->reg);
4745
4746 return macb_init(pdev);
4747 }
4748
init_reset_optional(struct platform_device * pdev)4749 static int init_reset_optional(struct platform_device *pdev)
4750 {
4751 struct net_device *dev = platform_get_drvdata(pdev);
4752 struct macb *bp = netdev_priv(dev);
4753 int ret;
4754
4755 if (bp->phy_interface == PHY_INTERFACE_MODE_SGMII) {
4756 /* Ensure PHY device used in SGMII mode is ready */
4757 bp->sgmii_phy = devm_phy_optional_get(&pdev->dev, NULL);
4758
4759 if (IS_ERR(bp->sgmii_phy))
4760 return dev_err_probe(&pdev->dev, PTR_ERR(bp->sgmii_phy),
4761 "failed to get SGMII PHY\n");
4762
4763 ret = phy_init(bp->sgmii_phy);
4764 if (ret)
4765 return dev_err_probe(&pdev->dev, ret,
4766 "failed to init SGMII PHY\n");
4767
4768 ret = zynqmp_pm_is_function_supported(PM_IOCTL, IOCTL_SET_GEM_CONFIG);
4769 if (!ret) {
4770 u32 pm_info[2];
4771
4772 ret = of_property_read_u32_array(pdev->dev.of_node, "power-domains",
4773 pm_info, ARRAY_SIZE(pm_info));
4774 if (ret) {
4775 dev_err(&pdev->dev, "Failed to read power management information\n");
4776 goto err_out_phy_exit;
4777 }
4778 ret = zynqmp_pm_set_gem_config(pm_info[1], GEM_CONFIG_FIXED, 0);
4779 if (ret)
4780 goto err_out_phy_exit;
4781
4782 ret = zynqmp_pm_set_gem_config(pm_info[1], GEM_CONFIG_SGMII_MODE, 1);
4783 if (ret)
4784 goto err_out_phy_exit;
4785 }
4786
4787 }
4788
4789 /* Fully reset controller at hardware level if mapped in device tree */
4790 ret = device_reset_optional(&pdev->dev);
4791 if (ret) {
4792 phy_exit(bp->sgmii_phy);
4793 return dev_err_probe(&pdev->dev, ret, "failed to reset controller");
4794 }
4795
4796 ret = macb_init(pdev);
4797
4798 err_out_phy_exit:
4799 if (ret)
4800 phy_exit(bp->sgmii_phy);
4801
4802 return ret;
4803 }
4804
4805 static const struct macb_usrio_config sama7g5_usrio = {
4806 .mii = 0,
4807 .rmii = 1,
4808 .rgmii = 2,
4809 .refclk = BIT(2),
4810 .hdfctlen = BIT(6),
4811 };
4812
4813 static const struct macb_config fu540_c000_config = {
4814 .caps = MACB_CAPS_GIGABIT_MODE_AVAILABLE | MACB_CAPS_JUMBO |
4815 MACB_CAPS_GEM_HAS_PTP,
4816 .dma_burst_length = 16,
4817 .clk_init = fu540_c000_clk_init,
4818 .init = fu540_c000_init,
4819 .jumbo_max_len = 10240,
4820 .usrio = &macb_default_usrio,
4821 };
4822
4823 static const struct macb_config at91sam9260_config = {
4824 .caps = MACB_CAPS_USRIO_HAS_CLKEN | MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII,
4825 .clk_init = macb_clk_init,
4826 .init = macb_init,
4827 .usrio = &macb_default_usrio,
4828 };
4829
4830 static const struct macb_config sama5d3macb_config = {
4831 .caps = MACB_CAPS_SG_DISABLED |
4832 MACB_CAPS_USRIO_HAS_CLKEN | MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII,
4833 .clk_init = macb_clk_init,
4834 .init = macb_init,
4835 .usrio = &macb_default_usrio,
4836 };
4837
4838 static const struct macb_config pc302gem_config = {
4839 .caps = MACB_CAPS_SG_DISABLED | MACB_CAPS_GIGABIT_MODE_AVAILABLE,
4840 .dma_burst_length = 16,
4841 .clk_init = macb_clk_init,
4842 .init = macb_init,
4843 .usrio = &macb_default_usrio,
4844 };
4845
4846 static const struct macb_config sama5d2_config = {
4847 .caps = MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII | MACB_CAPS_JUMBO,
4848 .dma_burst_length = 16,
4849 .clk_init = macb_clk_init,
4850 .init = macb_init,
4851 .jumbo_max_len = 10240,
4852 .usrio = &macb_default_usrio,
4853 };
4854
4855 static const struct macb_config sama5d29_config = {
4856 .caps = MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII | MACB_CAPS_GEM_HAS_PTP,
4857 .dma_burst_length = 16,
4858 .clk_init = macb_clk_init,
4859 .init = macb_init,
4860 .usrio = &macb_default_usrio,
4861 };
4862
4863 static const struct macb_config sama5d3_config = {
4864 .caps = MACB_CAPS_SG_DISABLED | MACB_CAPS_GIGABIT_MODE_AVAILABLE |
4865 MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII | MACB_CAPS_JUMBO,
4866 .dma_burst_length = 16,
4867 .clk_init = macb_clk_init,
4868 .init = macb_init,
4869 .jumbo_max_len = 10240,
4870 .usrio = &macb_default_usrio,
4871 };
4872
4873 static const struct macb_config sama5d4_config = {
4874 .caps = MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII,
4875 .dma_burst_length = 4,
4876 .clk_init = macb_clk_init,
4877 .init = macb_init,
4878 .usrio = &macb_default_usrio,
4879 };
4880
4881 static const struct macb_config emac_config = {
4882 .caps = MACB_CAPS_NEEDS_RSTONUBR | MACB_CAPS_MACB_IS_EMAC,
4883 .clk_init = at91ether_clk_init,
4884 .init = at91ether_init,
4885 .usrio = &macb_default_usrio,
4886 };
4887
4888 static const struct macb_config np4_config = {
4889 .caps = MACB_CAPS_USRIO_DISABLED,
4890 .clk_init = macb_clk_init,
4891 .init = macb_init,
4892 .usrio = &macb_default_usrio,
4893 };
4894
4895 static const struct macb_config zynqmp_config = {
4896 .caps = MACB_CAPS_GIGABIT_MODE_AVAILABLE |
4897 MACB_CAPS_JUMBO |
4898 MACB_CAPS_GEM_HAS_PTP | MACB_CAPS_BD_RD_PREFETCH,
4899 .dma_burst_length = 16,
4900 .clk_init = macb_clk_init,
4901 .init = init_reset_optional,
4902 .jumbo_max_len = 10240,
4903 .usrio = &macb_default_usrio,
4904 };
4905
4906 static const struct macb_config zynq_config = {
4907 .caps = MACB_CAPS_GIGABIT_MODE_AVAILABLE | MACB_CAPS_NO_GIGABIT_HALF |
4908 MACB_CAPS_NEEDS_RSTONUBR,
4909 .dma_burst_length = 16,
4910 .clk_init = macb_clk_init,
4911 .init = macb_init,
4912 .usrio = &macb_default_usrio,
4913 };
4914
4915 static const struct macb_config mpfs_config = {
4916 .caps = MACB_CAPS_GIGABIT_MODE_AVAILABLE |
4917 MACB_CAPS_JUMBO |
4918 MACB_CAPS_GEM_HAS_PTP,
4919 .dma_burst_length = 16,
4920 .clk_init = macb_clk_init,
4921 .init = init_reset_optional,
4922 .usrio = &macb_default_usrio,
4923 .max_tx_length = 4040, /* Cadence Erratum 1686 */
4924 .jumbo_max_len = 4040,
4925 };
4926
4927 static const struct macb_config sama7g5_gem_config = {
4928 .caps = MACB_CAPS_GIGABIT_MODE_AVAILABLE | MACB_CAPS_CLK_HW_CHG |
4929 MACB_CAPS_MIIONRGMII | MACB_CAPS_GEM_HAS_PTP,
4930 .dma_burst_length = 16,
4931 .clk_init = macb_clk_init,
4932 .init = macb_init,
4933 .usrio = &sama7g5_usrio,
4934 };
4935
4936 static const struct macb_config sama7g5_emac_config = {
4937 .caps = MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII |
4938 MACB_CAPS_USRIO_HAS_CLKEN | MACB_CAPS_MIIONRGMII |
4939 MACB_CAPS_GEM_HAS_PTP,
4940 .dma_burst_length = 16,
4941 .clk_init = macb_clk_init,
4942 .init = macb_init,
4943 .usrio = &sama7g5_usrio,
4944 };
4945
4946 static const struct macb_config versal_config = {
4947 .caps = MACB_CAPS_GIGABIT_MODE_AVAILABLE | MACB_CAPS_JUMBO |
4948 MACB_CAPS_GEM_HAS_PTP | MACB_CAPS_BD_RD_PREFETCH | MACB_CAPS_NEED_TSUCLK |
4949 MACB_CAPS_QUEUE_DISABLE,
4950 .dma_burst_length = 16,
4951 .clk_init = macb_clk_init,
4952 .init = init_reset_optional,
4953 .jumbo_max_len = 10240,
4954 .usrio = &macb_default_usrio,
4955 };
4956
4957 static const struct of_device_id macb_dt_ids[] = {
4958 { .compatible = "cdns,at91sam9260-macb", .data = &at91sam9260_config },
4959 { .compatible = "cdns,macb" },
4960 { .compatible = "cdns,np4-macb", .data = &np4_config },
4961 { .compatible = "cdns,pc302-gem", .data = &pc302gem_config },
4962 { .compatible = "cdns,gem", .data = &pc302gem_config },
4963 { .compatible = "cdns,sam9x60-macb", .data = &at91sam9260_config },
4964 { .compatible = "atmel,sama5d2-gem", .data = &sama5d2_config },
4965 { .compatible = "atmel,sama5d29-gem", .data = &sama5d29_config },
4966 { .compatible = "atmel,sama5d3-gem", .data = &sama5d3_config },
4967 { .compatible = "atmel,sama5d3-macb", .data = &sama5d3macb_config },
4968 { .compatible = "atmel,sama5d4-gem", .data = &sama5d4_config },
4969 { .compatible = "cdns,at91rm9200-emac", .data = &emac_config },
4970 { .compatible = "cdns,emac", .data = &emac_config },
4971 { .compatible = "cdns,zynqmp-gem", .data = &zynqmp_config}, /* deprecated */
4972 { .compatible = "cdns,zynq-gem", .data = &zynq_config }, /* deprecated */
4973 { .compatible = "sifive,fu540-c000-gem", .data = &fu540_c000_config },
4974 { .compatible = "microchip,mpfs-macb", .data = &mpfs_config },
4975 { .compatible = "microchip,sama7g5-gem", .data = &sama7g5_gem_config },
4976 { .compatible = "microchip,sama7g5-emac", .data = &sama7g5_emac_config },
4977 { .compatible = "xlnx,zynqmp-gem", .data = &zynqmp_config},
4978 { .compatible = "xlnx,zynq-gem", .data = &zynq_config },
4979 { .compatible = "xlnx,versal-gem", .data = &versal_config},
4980 { /* sentinel */ }
4981 };
4982 MODULE_DEVICE_TABLE(of, macb_dt_ids);
4983 #endif /* CONFIG_OF */
4984
4985 static const struct macb_config default_gem_config = {
4986 .caps = MACB_CAPS_GIGABIT_MODE_AVAILABLE |
4987 MACB_CAPS_JUMBO |
4988 MACB_CAPS_GEM_HAS_PTP,
4989 .dma_burst_length = 16,
4990 .clk_init = macb_clk_init,
4991 .init = macb_init,
4992 .usrio = &macb_default_usrio,
4993 .jumbo_max_len = 10240,
4994 };
4995
macb_probe(struct platform_device * pdev)4996 static int macb_probe(struct platform_device *pdev)
4997 {
4998 const struct macb_config *macb_config = &default_gem_config;
4999 int (*clk_init)(struct platform_device *, struct clk **,
5000 struct clk **, struct clk **, struct clk **,
5001 struct clk **) = macb_config->clk_init;
5002 int (*init)(struct platform_device *) = macb_config->init;
5003 struct device_node *np = pdev->dev.of_node;
5004 struct clk *pclk, *hclk = NULL, *tx_clk = NULL, *rx_clk = NULL;
5005 struct clk *tsu_clk = NULL;
5006 unsigned int queue_mask, num_queues;
5007 bool native_io;
5008 phy_interface_t interface;
5009 struct net_device *dev;
5010 struct resource *regs;
5011 u32 wtrmrk_rst_val;
5012 void __iomem *mem;
5013 struct macb *bp;
5014 int err, val;
5015
5016 mem = devm_platform_get_and_ioremap_resource(pdev, 0, ®s);
5017 if (IS_ERR(mem))
5018 return PTR_ERR(mem);
5019
5020 if (np) {
5021 const struct of_device_id *match;
5022
5023 match = of_match_node(macb_dt_ids, np);
5024 if (match && match->data) {
5025 macb_config = match->data;
5026 clk_init = macb_config->clk_init;
5027 init = macb_config->init;
5028 }
5029 }
5030
5031 err = clk_init(pdev, &pclk, &hclk, &tx_clk, &rx_clk, &tsu_clk);
5032 if (err)
5033 return err;
5034
5035 pm_runtime_set_autosuspend_delay(&pdev->dev, MACB_PM_TIMEOUT);
5036 pm_runtime_use_autosuspend(&pdev->dev);
5037 pm_runtime_get_noresume(&pdev->dev);
5038 pm_runtime_set_active(&pdev->dev);
5039 pm_runtime_enable(&pdev->dev);
5040 native_io = hw_is_native_io(mem);
5041
5042 macb_probe_queues(mem, native_io, &queue_mask, &num_queues);
5043 dev = alloc_etherdev_mq(sizeof(*bp), num_queues);
5044 if (!dev) {
5045 err = -ENOMEM;
5046 goto err_disable_clocks;
5047 }
5048
5049 dev->base_addr = regs->start;
5050
5051 SET_NETDEV_DEV(dev, &pdev->dev);
5052
5053 bp = netdev_priv(dev);
5054 bp->pdev = pdev;
5055 bp->dev = dev;
5056 bp->regs = mem;
5057 bp->native_io = native_io;
5058 if (native_io) {
5059 bp->macb_reg_readl = hw_readl_native;
5060 bp->macb_reg_writel = hw_writel_native;
5061 } else {
5062 bp->macb_reg_readl = hw_readl;
5063 bp->macb_reg_writel = hw_writel;
5064 }
5065 bp->num_queues = num_queues;
5066 bp->queue_mask = queue_mask;
5067 if (macb_config)
5068 bp->dma_burst_length = macb_config->dma_burst_length;
5069 bp->pclk = pclk;
5070 bp->hclk = hclk;
5071 bp->tx_clk = tx_clk;
5072 bp->rx_clk = rx_clk;
5073 bp->tsu_clk = tsu_clk;
5074 if (macb_config)
5075 bp->jumbo_max_len = macb_config->jumbo_max_len;
5076
5077 if (!hw_is_gem(bp->regs, bp->native_io))
5078 bp->max_tx_length = MACB_MAX_TX_LEN;
5079 else if (macb_config->max_tx_length)
5080 bp->max_tx_length = macb_config->max_tx_length;
5081 else
5082 bp->max_tx_length = GEM_MAX_TX_LEN;
5083
5084 bp->wol = 0;
5085 device_set_wakeup_capable(&pdev->dev, 1);
5086
5087 bp->usrio = macb_config->usrio;
5088
5089 /* By default we set to partial store and forward mode for zynqmp.
5090 * Disable if not set in devicetree.
5091 */
5092 if (GEM_BFEXT(PBUF_CUTTHRU, gem_readl(bp, DCFG6))) {
5093 err = of_property_read_u32(bp->pdev->dev.of_node,
5094 "cdns,rx-watermark",
5095 &bp->rx_watermark);
5096
5097 if (!err) {
5098 /* Disable partial store and forward in case of error or
5099 * invalid watermark value
5100 */
5101 wtrmrk_rst_val = (1 << (GEM_BFEXT(RX_PBUF_ADDR, gem_readl(bp, DCFG2)))) - 1;
5102 if (bp->rx_watermark > wtrmrk_rst_val || !bp->rx_watermark) {
5103 dev_info(&bp->pdev->dev, "Invalid watermark value\n");
5104 bp->rx_watermark = 0;
5105 }
5106 }
5107 }
5108 spin_lock_init(&bp->lock);
5109
5110 /* setup capabilities */
5111 macb_configure_caps(bp, macb_config);
5112
5113 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT
5114 if (GEM_BFEXT(DAW64, gem_readl(bp, DCFG6))) {
5115 dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(44));
5116 bp->hw_dma_cap |= HW_DMA_CAP_64B;
5117 }
5118 #endif
5119 platform_set_drvdata(pdev, dev);
5120
5121 dev->irq = platform_get_irq(pdev, 0);
5122 if (dev->irq < 0) {
5123 err = dev->irq;
5124 goto err_out_free_netdev;
5125 }
5126
5127 /* MTU range: 68 - 1518 or 10240 */
5128 dev->min_mtu = GEM_MTU_MIN_SIZE;
5129 if ((bp->caps & MACB_CAPS_JUMBO) && bp->jumbo_max_len)
5130 dev->max_mtu = bp->jumbo_max_len - ETH_HLEN - ETH_FCS_LEN;
5131 else
5132 dev->max_mtu = 1536 - ETH_HLEN - ETH_FCS_LEN;
5133
5134 if (bp->caps & MACB_CAPS_BD_RD_PREFETCH) {
5135 val = GEM_BFEXT(RXBD_RDBUFF, gem_readl(bp, DCFG10));
5136 if (val)
5137 bp->rx_bd_rd_prefetch = (2 << (val - 1)) *
5138 macb_dma_desc_get_size(bp);
5139
5140 val = GEM_BFEXT(TXBD_RDBUFF, gem_readl(bp, DCFG10));
5141 if (val)
5142 bp->tx_bd_rd_prefetch = (2 << (val - 1)) *
5143 macb_dma_desc_get_size(bp);
5144 }
5145
5146 bp->rx_intr_mask = MACB_RX_INT_FLAGS;
5147 if (bp->caps & MACB_CAPS_NEEDS_RSTONUBR)
5148 bp->rx_intr_mask |= MACB_BIT(RXUBR);
5149
5150 err = of_get_ethdev_address(np, bp->dev);
5151 if (err == -EPROBE_DEFER)
5152 goto err_out_free_netdev;
5153 else if (err)
5154 macb_get_hwaddr(bp);
5155
5156 err = of_get_phy_mode(np, &interface);
5157 if (err)
5158 /* not found in DT, MII by default */
5159 bp->phy_interface = PHY_INTERFACE_MODE_MII;
5160 else
5161 bp->phy_interface = interface;
5162
5163 /* IP specific init */
5164 err = init(pdev);
5165 if (err)
5166 goto err_out_free_netdev;
5167
5168 err = macb_mii_init(bp);
5169 if (err)
5170 goto err_out_phy_exit;
5171
5172 netif_carrier_off(dev);
5173
5174 err = register_netdev(dev);
5175 if (err) {
5176 dev_err(&pdev->dev, "Cannot register net device, aborting.\n");
5177 goto err_out_unregister_mdio;
5178 }
5179
5180 INIT_WORK(&bp->hresp_err_bh_work, macb_hresp_error_task);
5181
5182 netdev_info(dev, "Cadence %s rev 0x%08x at 0x%08lx irq %d (%pM)\n",
5183 macb_is_gem(bp) ? "GEM" : "MACB", macb_readl(bp, MID),
5184 dev->base_addr, dev->irq, dev->dev_addr);
5185
5186 pm_runtime_mark_last_busy(&bp->pdev->dev);
5187 pm_runtime_put_autosuspend(&bp->pdev->dev);
5188
5189 return 0;
5190
5191 err_out_unregister_mdio:
5192 mdiobus_unregister(bp->mii_bus);
5193 mdiobus_free(bp->mii_bus);
5194
5195 err_out_phy_exit:
5196 phy_exit(bp->sgmii_phy);
5197
5198 err_out_free_netdev:
5199 free_netdev(dev);
5200
5201 err_disable_clocks:
5202 macb_clks_disable(pclk, hclk, tx_clk, rx_clk, tsu_clk);
5203 pm_runtime_disable(&pdev->dev);
5204 pm_runtime_set_suspended(&pdev->dev);
5205 pm_runtime_dont_use_autosuspend(&pdev->dev);
5206
5207 return err;
5208 }
5209
macb_remove(struct platform_device * pdev)5210 static void macb_remove(struct platform_device *pdev)
5211 {
5212 struct net_device *dev;
5213 struct macb *bp;
5214
5215 dev = platform_get_drvdata(pdev);
5216
5217 if (dev) {
5218 bp = netdev_priv(dev);
5219 phy_exit(bp->sgmii_phy);
5220 mdiobus_unregister(bp->mii_bus);
5221 mdiobus_free(bp->mii_bus);
5222
5223 unregister_netdev(dev);
5224 cancel_work_sync(&bp->hresp_err_bh_work);
5225 pm_runtime_disable(&pdev->dev);
5226 pm_runtime_dont_use_autosuspend(&pdev->dev);
5227 if (!pm_runtime_suspended(&pdev->dev)) {
5228 macb_clks_disable(bp->pclk, bp->hclk, bp->tx_clk,
5229 bp->rx_clk, bp->tsu_clk);
5230 pm_runtime_set_suspended(&pdev->dev);
5231 }
5232 phylink_destroy(bp->phylink);
5233 free_netdev(dev);
5234 }
5235 }
5236
macb_suspend(struct device * dev)5237 static int __maybe_unused macb_suspend(struct device *dev)
5238 {
5239 struct net_device *netdev = dev_get_drvdata(dev);
5240 struct macb *bp = netdev_priv(netdev);
5241 struct in_ifaddr *ifa = NULL;
5242 struct macb_queue *queue;
5243 struct in_device *idev;
5244 unsigned long flags;
5245 unsigned int q;
5246 int err;
5247 u32 tmp;
5248
5249 if (!device_may_wakeup(&bp->dev->dev))
5250 phy_exit(bp->sgmii_phy);
5251
5252 if (!netif_running(netdev))
5253 return 0;
5254
5255 if (bp->wol & MACB_WOL_ENABLED) {
5256 /* Check for IP address in WOL ARP mode */
5257 idev = __in_dev_get_rcu(bp->dev);
5258 if (idev)
5259 ifa = rcu_dereference(idev->ifa_list);
5260 if ((bp->wolopts & WAKE_ARP) && !ifa) {
5261 netdev_err(netdev, "IP address not assigned as required by WoL walk ARP\n");
5262 return -EOPNOTSUPP;
5263 }
5264 spin_lock_irqsave(&bp->lock, flags);
5265
5266 /* Disable Tx and Rx engines before disabling the queues,
5267 * this is mandatory as per the IP spec sheet
5268 */
5269 tmp = macb_readl(bp, NCR);
5270 macb_writel(bp, NCR, tmp & ~(MACB_BIT(TE) | MACB_BIT(RE)));
5271 for (q = 0, queue = bp->queues; q < bp->num_queues;
5272 ++q, ++queue) {
5273 /* Disable RX queues */
5274 if (bp->caps & MACB_CAPS_QUEUE_DISABLE) {
5275 queue_writel(queue, RBQP, MACB_BIT(QUEUE_DISABLE));
5276 } else {
5277 /* Tie off RX queues */
5278 queue_writel(queue, RBQP,
5279 lower_32_bits(bp->rx_ring_tieoff_dma));
5280 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT
5281 queue_writel(queue, RBQPH,
5282 upper_32_bits(bp->rx_ring_tieoff_dma));
5283 #endif
5284 }
5285 /* Disable all interrupts */
5286 queue_writel(queue, IDR, -1);
5287 queue_readl(queue, ISR);
5288 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE)
5289 queue_writel(queue, ISR, -1);
5290 }
5291 /* Enable Receive engine */
5292 macb_writel(bp, NCR, tmp | MACB_BIT(RE));
5293 /* Flush all status bits */
5294 macb_writel(bp, TSR, -1);
5295 macb_writel(bp, RSR, -1);
5296
5297 tmp = (bp->wolopts & WAKE_MAGIC) ? MACB_BIT(MAG) : 0;
5298 if (bp->wolopts & WAKE_ARP) {
5299 tmp |= MACB_BIT(ARP);
5300 /* write IP address into register */
5301 tmp |= MACB_BFEXT(IP, be32_to_cpu(ifa->ifa_local));
5302 }
5303
5304 /* Change interrupt handler and
5305 * Enable WoL IRQ on queue 0
5306 */
5307 devm_free_irq(dev, bp->queues[0].irq, bp->queues);
5308 if (macb_is_gem(bp)) {
5309 err = devm_request_irq(dev, bp->queues[0].irq, gem_wol_interrupt,
5310 IRQF_SHARED, netdev->name, bp->queues);
5311 if (err) {
5312 dev_err(dev,
5313 "Unable to request IRQ %d (error %d)\n",
5314 bp->queues[0].irq, err);
5315 spin_unlock_irqrestore(&bp->lock, flags);
5316 return err;
5317 }
5318 queue_writel(bp->queues, IER, GEM_BIT(WOL));
5319 gem_writel(bp, WOL, tmp);
5320 } else {
5321 err = devm_request_irq(dev, bp->queues[0].irq, macb_wol_interrupt,
5322 IRQF_SHARED, netdev->name, bp->queues);
5323 if (err) {
5324 dev_err(dev,
5325 "Unable to request IRQ %d (error %d)\n",
5326 bp->queues[0].irq, err);
5327 spin_unlock_irqrestore(&bp->lock, flags);
5328 return err;
5329 }
5330 queue_writel(bp->queues, IER, MACB_BIT(WOL));
5331 macb_writel(bp, WOL, tmp);
5332 }
5333 spin_unlock_irqrestore(&bp->lock, flags);
5334
5335 enable_irq_wake(bp->queues[0].irq);
5336 }
5337
5338 netif_device_detach(netdev);
5339 for (q = 0, queue = bp->queues; q < bp->num_queues;
5340 ++q, ++queue) {
5341 napi_disable(&queue->napi_rx);
5342 napi_disable(&queue->napi_tx);
5343 }
5344
5345 if (!(bp->wol & MACB_WOL_ENABLED)) {
5346 rtnl_lock();
5347 phylink_stop(bp->phylink);
5348 rtnl_unlock();
5349 spin_lock_irqsave(&bp->lock, flags);
5350 macb_reset_hw(bp);
5351 spin_unlock_irqrestore(&bp->lock, flags);
5352 }
5353
5354 if (!(bp->caps & MACB_CAPS_USRIO_DISABLED))
5355 bp->pm_data.usrio = macb_or_gem_readl(bp, USRIO);
5356
5357 if (netdev->hw_features & NETIF_F_NTUPLE)
5358 bp->pm_data.scrt2 = gem_readl_n(bp, ETHT, SCRT2_ETHT);
5359
5360 if (bp->ptp_info)
5361 bp->ptp_info->ptp_remove(netdev);
5362 if (!device_may_wakeup(dev))
5363 pm_runtime_force_suspend(dev);
5364
5365 return 0;
5366 }
5367
macb_resume(struct device * dev)5368 static int __maybe_unused macb_resume(struct device *dev)
5369 {
5370 struct net_device *netdev = dev_get_drvdata(dev);
5371 struct macb *bp = netdev_priv(netdev);
5372 struct macb_queue *queue;
5373 unsigned long flags;
5374 unsigned int q;
5375 int err;
5376
5377 if (!device_may_wakeup(&bp->dev->dev))
5378 phy_init(bp->sgmii_phy);
5379
5380 if (!netif_running(netdev))
5381 return 0;
5382
5383 if (!device_may_wakeup(dev))
5384 pm_runtime_force_resume(dev);
5385
5386 if (bp->wol & MACB_WOL_ENABLED) {
5387 spin_lock_irqsave(&bp->lock, flags);
5388 /* Disable WoL */
5389 if (macb_is_gem(bp)) {
5390 queue_writel(bp->queues, IDR, GEM_BIT(WOL));
5391 gem_writel(bp, WOL, 0);
5392 } else {
5393 queue_writel(bp->queues, IDR, MACB_BIT(WOL));
5394 macb_writel(bp, WOL, 0);
5395 }
5396 /* Clear ISR on queue 0 */
5397 queue_readl(bp->queues, ISR);
5398 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE)
5399 queue_writel(bp->queues, ISR, -1);
5400 /* Replace interrupt handler on queue 0 */
5401 devm_free_irq(dev, bp->queues[0].irq, bp->queues);
5402 err = devm_request_irq(dev, bp->queues[0].irq, macb_interrupt,
5403 IRQF_SHARED, netdev->name, bp->queues);
5404 if (err) {
5405 dev_err(dev,
5406 "Unable to request IRQ %d (error %d)\n",
5407 bp->queues[0].irq, err);
5408 spin_unlock_irqrestore(&bp->lock, flags);
5409 return err;
5410 }
5411 spin_unlock_irqrestore(&bp->lock, flags);
5412
5413 disable_irq_wake(bp->queues[0].irq);
5414
5415 /* Now make sure we disable phy before moving
5416 * to common restore path
5417 */
5418 rtnl_lock();
5419 phylink_stop(bp->phylink);
5420 rtnl_unlock();
5421 }
5422
5423 for (q = 0, queue = bp->queues; q < bp->num_queues;
5424 ++q, ++queue) {
5425 napi_enable(&queue->napi_rx);
5426 napi_enable(&queue->napi_tx);
5427 }
5428
5429 if (netdev->hw_features & NETIF_F_NTUPLE)
5430 gem_writel_n(bp, ETHT, SCRT2_ETHT, bp->pm_data.scrt2);
5431
5432 if (!(bp->caps & MACB_CAPS_USRIO_DISABLED))
5433 macb_or_gem_writel(bp, USRIO, bp->pm_data.usrio);
5434
5435 macb_writel(bp, NCR, MACB_BIT(MPE));
5436 macb_init_hw(bp);
5437 macb_set_rx_mode(netdev);
5438 macb_restore_features(bp);
5439 rtnl_lock();
5440
5441 phylink_start(bp->phylink);
5442 rtnl_unlock();
5443
5444 netif_device_attach(netdev);
5445 if (bp->ptp_info)
5446 bp->ptp_info->ptp_init(netdev);
5447
5448 return 0;
5449 }
5450
macb_runtime_suspend(struct device * dev)5451 static int __maybe_unused macb_runtime_suspend(struct device *dev)
5452 {
5453 struct net_device *netdev = dev_get_drvdata(dev);
5454 struct macb *bp = netdev_priv(netdev);
5455
5456 if (!(device_may_wakeup(dev)))
5457 macb_clks_disable(bp->pclk, bp->hclk, bp->tx_clk, bp->rx_clk, bp->tsu_clk);
5458 else if (!(bp->caps & MACB_CAPS_NEED_TSUCLK))
5459 macb_clks_disable(NULL, NULL, NULL, NULL, bp->tsu_clk);
5460
5461 return 0;
5462 }
5463
macb_runtime_resume(struct device * dev)5464 static int __maybe_unused macb_runtime_resume(struct device *dev)
5465 {
5466 struct net_device *netdev = dev_get_drvdata(dev);
5467 struct macb *bp = netdev_priv(netdev);
5468
5469 if (!(device_may_wakeup(dev))) {
5470 clk_prepare_enable(bp->pclk);
5471 clk_prepare_enable(bp->hclk);
5472 clk_prepare_enable(bp->tx_clk);
5473 clk_prepare_enable(bp->rx_clk);
5474 clk_prepare_enable(bp->tsu_clk);
5475 } else if (!(bp->caps & MACB_CAPS_NEED_TSUCLK)) {
5476 clk_prepare_enable(bp->tsu_clk);
5477 }
5478
5479 return 0;
5480 }
5481
5482 static const struct dev_pm_ops macb_pm_ops = {
5483 SET_SYSTEM_SLEEP_PM_OPS(macb_suspend, macb_resume)
5484 SET_RUNTIME_PM_OPS(macb_runtime_suspend, macb_runtime_resume, NULL)
5485 };
5486
5487 static struct platform_driver macb_driver = {
5488 .probe = macb_probe,
5489 .remove = macb_remove,
5490 .driver = {
5491 .name = "macb",
5492 .of_match_table = of_match_ptr(macb_dt_ids),
5493 .pm = &macb_pm_ops,
5494 },
5495 };
5496
5497 module_platform_driver(macb_driver);
5498
5499 MODULE_LICENSE("GPL");
5500 MODULE_DESCRIPTION("Cadence MACB/GEM Ethernet driver");
5501 MODULE_AUTHOR("Haavard Skinnemoen (Atmel)");
5502 MODULE_ALIAS("platform:macb");
5503