1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Broadcom GENET (Gigabit Ethernet) controller driver
4 *
5 * Copyright (c) 2014-2025 Broadcom
6 */
7
8 #define pr_fmt(fmt) "bcmgenet: " fmt
9
10 #include <linux/acpi.h>
11 #include <linux/kernel.h>
12 #include <linux/module.h>
13 #include <linux/sched.h>
14 #include <linux/types.h>
15 #include <linux/fcntl.h>
16 #include <linux/interrupt.h>
17 #include <linux/string.h>
18 #include <linux/if_ether.h>
19 #include <linux/init.h>
20 #include <linux/errno.h>
21 #include <linux/delay.h>
22 #include <linux/platform_device.h>
23 #include <linux/dma-mapping.h>
24 #include <linux/pm.h>
25 #include <linux/clk.h>
26 #include <net/arp.h>
27
28 #include <linux/mii.h>
29 #include <linux/ethtool.h>
30 #include <linux/netdevice.h>
31 #include <linux/inetdevice.h>
32 #include <linux/etherdevice.h>
33 #include <linux/skbuff.h>
34 #include <linux/in.h>
35 #include <linux/ip.h>
36 #include <linux/ipv6.h>
37 #include <linux/phy.h>
38
39 #include <linux/unaligned.h>
40
41 #include "bcmgenet.h"
42
43 #define GENET_Q0_WEIGHT 1
44 #define GENET_Q1_WEIGHT 4
45
46 #define GENET_Q0_RX_BD_CNT \
47 (TOTAL_DESC - priv->hw_params->rx_queues * priv->hw_params->rx_bds_per_q)
48 #define GENET_Q0_TX_BD_CNT \
49 (TOTAL_DESC - priv->hw_params->tx_queues * priv->hw_params->tx_bds_per_q)
50
51 #define RX_BUF_LENGTH 2048
52 #define SKB_ALIGNMENT 32
53
54 /* Page pool RX buffer layout:
55 * RSB(64) + pad(2) | frame data | skb_shared_info
56 * The HW writes the 64B RSB + 2B alignment padding before the frame.
57 */
58 #define GENET_RSB_PAD (sizeof(struct status_64) + 2)
59
60 /* RX buffer plus the skb_shared_info napi_build_skb() places behind it */
61 #define GENET_RX_BUF_SIZE SKB_HEAD_ALIGN(RX_BUF_LENGTH)
62
63 /* Tx/Rx DMA register offset, skip 256 descriptors */
64 #define WORDS_PER_BD(p) (p->hw_params->words_per_bd)
65 #define DMA_DESC_SIZE (WORDS_PER_BD(priv) * sizeof(u32))
66
67 #define GENET_TDMA_REG_OFF (priv->hw_params->tdma_offset + \
68 TOTAL_DESC * DMA_DESC_SIZE)
69
70 #define GENET_RDMA_REG_OFF (priv->hw_params->rdma_offset + \
71 TOTAL_DESC * DMA_DESC_SIZE)
72
73 /* Forward declarations */
74 static void bcmgenet_set_rx_mode(struct net_device *dev);
75
bcmgenet_writel(u32 value,void __iomem * offset)76 static inline void bcmgenet_writel(u32 value, void __iomem *offset)
77 {
78 /* MIPS chips strapped for BE will automagically configure the
79 * peripheral registers for CPU-native byte order.
80 */
81 if (IS_ENABLED(CONFIG_MIPS) && IS_ENABLED(CONFIG_CPU_BIG_ENDIAN))
82 __raw_writel(value, offset);
83 else
84 writel_relaxed(value, offset);
85 }
86
bcmgenet_readl(void __iomem * offset)87 static inline u32 bcmgenet_readl(void __iomem *offset)
88 {
89 if (IS_ENABLED(CONFIG_MIPS) && IS_ENABLED(CONFIG_CPU_BIG_ENDIAN))
90 return __raw_readl(offset);
91 else
92 return readl_relaxed(offset);
93 }
94
dmadesc_set_length_status(struct bcmgenet_priv * priv,void __iomem * d,u32 value)95 static inline void dmadesc_set_length_status(struct bcmgenet_priv *priv,
96 void __iomem *d, u32 value)
97 {
98 bcmgenet_writel(value, d + DMA_DESC_LENGTH_STATUS);
99 }
100
dmadesc_set_addr(struct bcmgenet_priv * priv,void __iomem * d,dma_addr_t addr)101 static inline void dmadesc_set_addr(struct bcmgenet_priv *priv,
102 void __iomem *d,
103 dma_addr_t addr)
104 {
105 bcmgenet_writel(lower_32_bits(addr), d + DMA_DESC_ADDRESS_LO);
106
107 /* Register writes to GISB bus can take couple hundred nanoseconds
108 * and are done for each packet, save these expensive writes unless
109 * the platform is explicitly configured for 64-bits/LPAE.
110 */
111 #ifdef CONFIG_PHYS_ADDR_T_64BIT
112 if (bcmgenet_has_40bits(priv))
113 bcmgenet_writel(upper_32_bits(addr), d + DMA_DESC_ADDRESS_HI);
114 #endif
115 }
116
117 /* Combined address + length/status setter */
dmadesc_set(struct bcmgenet_priv * priv,void __iomem * d,dma_addr_t addr,u32 val)118 static inline void dmadesc_set(struct bcmgenet_priv *priv,
119 void __iomem *d, dma_addr_t addr, u32 val)
120 {
121 dmadesc_set_addr(priv, d, addr);
122 dmadesc_set_length_status(priv, d, val);
123 }
124
125 #define GENET_VER_FMT "%1d.%1d EPHY: 0x%04x"
126
127 #define GENET_MSG_DEFAULT (NETIF_MSG_DRV | NETIF_MSG_PROBE | \
128 NETIF_MSG_LINK)
129
bcmgenet_rbuf_ctrl_get(struct bcmgenet_priv * priv)130 static inline u32 bcmgenet_rbuf_ctrl_get(struct bcmgenet_priv *priv)
131 {
132 if (GENET_IS_V1(priv))
133 return bcmgenet_rbuf_readl(priv, RBUF_FLUSH_CTRL_V1);
134 else
135 return bcmgenet_sys_readl(priv, SYS_RBUF_FLUSH_CTRL);
136 }
137
bcmgenet_rbuf_ctrl_set(struct bcmgenet_priv * priv,u32 val)138 static inline void bcmgenet_rbuf_ctrl_set(struct bcmgenet_priv *priv, u32 val)
139 {
140 if (GENET_IS_V1(priv))
141 bcmgenet_rbuf_writel(priv, val, RBUF_FLUSH_CTRL_V1);
142 else
143 bcmgenet_sys_writel(priv, val, SYS_RBUF_FLUSH_CTRL);
144 }
145
146 /* These macros are defined to deal with register map change
147 * between GENET1.1 and GENET2. Only those currently being used
148 * by driver are defined.
149 */
bcmgenet_tbuf_ctrl_get(struct bcmgenet_priv * priv)150 static inline u32 bcmgenet_tbuf_ctrl_get(struct bcmgenet_priv *priv)
151 {
152 if (GENET_IS_V1(priv))
153 return bcmgenet_rbuf_readl(priv, TBUF_CTRL_V1);
154 else
155 return bcmgenet_readl(priv->base +
156 priv->hw_params->tbuf_offset + TBUF_CTRL);
157 }
158
bcmgenet_tbuf_ctrl_set(struct bcmgenet_priv * priv,u32 val)159 static inline void bcmgenet_tbuf_ctrl_set(struct bcmgenet_priv *priv, u32 val)
160 {
161 if (GENET_IS_V1(priv))
162 bcmgenet_rbuf_writel(priv, val, TBUF_CTRL_V1);
163 else
164 bcmgenet_writel(val, priv->base +
165 priv->hw_params->tbuf_offset + TBUF_CTRL);
166 }
167
bcmgenet_bp_mc_get(struct bcmgenet_priv * priv)168 static inline u32 bcmgenet_bp_mc_get(struct bcmgenet_priv *priv)
169 {
170 if (GENET_IS_V1(priv))
171 return bcmgenet_rbuf_readl(priv, TBUF_BP_MC_V1);
172 else
173 return bcmgenet_readl(priv->base +
174 priv->hw_params->tbuf_offset + TBUF_BP_MC);
175 }
176
bcmgenet_bp_mc_set(struct bcmgenet_priv * priv,u32 val)177 static inline void bcmgenet_bp_mc_set(struct bcmgenet_priv *priv, u32 val)
178 {
179 if (GENET_IS_V1(priv))
180 bcmgenet_rbuf_writel(priv, val, TBUF_BP_MC_V1);
181 else
182 bcmgenet_writel(val, priv->base +
183 priv->hw_params->tbuf_offset + TBUF_BP_MC);
184 }
185
186 /* RX/TX DMA register accessors */
187 enum dma_reg {
188 DMA_RING_CFG = 0,
189 DMA_CTRL,
190 DMA_STATUS,
191 DMA_SCB_BURST_SIZE,
192 DMA_ARB_CTRL,
193 DMA_PRIORITY_0,
194 DMA_PRIORITY_1,
195 DMA_PRIORITY_2,
196 DMA_INDEX2RING_0,
197 DMA_INDEX2RING_1,
198 DMA_INDEX2RING_2,
199 DMA_INDEX2RING_3,
200 DMA_INDEX2RING_4,
201 DMA_INDEX2RING_5,
202 DMA_INDEX2RING_6,
203 DMA_INDEX2RING_7,
204 DMA_RING0_TIMEOUT,
205 DMA_RING1_TIMEOUT,
206 DMA_RING2_TIMEOUT,
207 DMA_RING3_TIMEOUT,
208 DMA_RING4_TIMEOUT,
209 DMA_RING5_TIMEOUT,
210 DMA_RING6_TIMEOUT,
211 DMA_RING7_TIMEOUT,
212 DMA_RING8_TIMEOUT,
213 DMA_RING9_TIMEOUT,
214 DMA_RING10_TIMEOUT,
215 DMA_RING11_TIMEOUT,
216 DMA_RING12_TIMEOUT,
217 DMA_RING13_TIMEOUT,
218 DMA_RING14_TIMEOUT,
219 DMA_RING15_TIMEOUT,
220 DMA_RING16_TIMEOUT,
221 };
222
223 static const u8 bcmgenet_dma_regs_v3plus[] = {
224 [DMA_RING_CFG] = 0x00,
225 [DMA_CTRL] = 0x04,
226 [DMA_STATUS] = 0x08,
227 [DMA_SCB_BURST_SIZE] = 0x0C,
228 [DMA_ARB_CTRL] = 0x2C,
229 [DMA_PRIORITY_0] = 0x30,
230 [DMA_PRIORITY_1] = 0x34,
231 [DMA_PRIORITY_2] = 0x38,
232 [DMA_RING0_TIMEOUT] = 0x2C,
233 [DMA_RING1_TIMEOUT] = 0x30,
234 [DMA_RING2_TIMEOUT] = 0x34,
235 [DMA_RING3_TIMEOUT] = 0x38,
236 [DMA_RING4_TIMEOUT] = 0x3c,
237 [DMA_RING5_TIMEOUT] = 0x40,
238 [DMA_RING6_TIMEOUT] = 0x44,
239 [DMA_RING7_TIMEOUT] = 0x48,
240 [DMA_RING8_TIMEOUT] = 0x4c,
241 [DMA_RING9_TIMEOUT] = 0x50,
242 [DMA_RING10_TIMEOUT] = 0x54,
243 [DMA_RING11_TIMEOUT] = 0x58,
244 [DMA_RING12_TIMEOUT] = 0x5c,
245 [DMA_RING13_TIMEOUT] = 0x60,
246 [DMA_RING14_TIMEOUT] = 0x64,
247 [DMA_RING15_TIMEOUT] = 0x68,
248 [DMA_RING16_TIMEOUT] = 0x6C,
249 [DMA_INDEX2RING_0] = 0x70,
250 [DMA_INDEX2RING_1] = 0x74,
251 [DMA_INDEX2RING_2] = 0x78,
252 [DMA_INDEX2RING_3] = 0x7C,
253 [DMA_INDEX2RING_4] = 0x80,
254 [DMA_INDEX2RING_5] = 0x84,
255 [DMA_INDEX2RING_6] = 0x88,
256 [DMA_INDEX2RING_7] = 0x8C,
257 };
258
259 static const u8 bcmgenet_dma_regs_v2[] = {
260 [DMA_RING_CFG] = 0x00,
261 [DMA_CTRL] = 0x04,
262 [DMA_STATUS] = 0x08,
263 [DMA_SCB_BURST_SIZE] = 0x0C,
264 [DMA_ARB_CTRL] = 0x30,
265 [DMA_PRIORITY_0] = 0x34,
266 [DMA_PRIORITY_1] = 0x38,
267 [DMA_PRIORITY_2] = 0x3C,
268 [DMA_RING0_TIMEOUT] = 0x2C,
269 [DMA_RING1_TIMEOUT] = 0x30,
270 [DMA_RING2_TIMEOUT] = 0x34,
271 [DMA_RING3_TIMEOUT] = 0x38,
272 [DMA_RING4_TIMEOUT] = 0x3c,
273 [DMA_RING5_TIMEOUT] = 0x40,
274 [DMA_RING6_TIMEOUT] = 0x44,
275 [DMA_RING7_TIMEOUT] = 0x48,
276 [DMA_RING8_TIMEOUT] = 0x4c,
277 [DMA_RING9_TIMEOUT] = 0x50,
278 [DMA_RING10_TIMEOUT] = 0x54,
279 [DMA_RING11_TIMEOUT] = 0x58,
280 [DMA_RING12_TIMEOUT] = 0x5c,
281 [DMA_RING13_TIMEOUT] = 0x60,
282 [DMA_RING14_TIMEOUT] = 0x64,
283 [DMA_RING15_TIMEOUT] = 0x68,
284 [DMA_RING16_TIMEOUT] = 0x6C,
285 };
286
287 static const u8 bcmgenet_dma_regs_v1[] = {
288 [DMA_CTRL] = 0x00,
289 [DMA_STATUS] = 0x04,
290 [DMA_SCB_BURST_SIZE] = 0x0C,
291 [DMA_ARB_CTRL] = 0x30,
292 [DMA_PRIORITY_0] = 0x34,
293 [DMA_PRIORITY_1] = 0x38,
294 [DMA_PRIORITY_2] = 0x3C,
295 [DMA_RING0_TIMEOUT] = 0x2C,
296 [DMA_RING1_TIMEOUT] = 0x30,
297 [DMA_RING2_TIMEOUT] = 0x34,
298 [DMA_RING3_TIMEOUT] = 0x38,
299 [DMA_RING4_TIMEOUT] = 0x3c,
300 [DMA_RING5_TIMEOUT] = 0x40,
301 [DMA_RING6_TIMEOUT] = 0x44,
302 [DMA_RING7_TIMEOUT] = 0x48,
303 [DMA_RING8_TIMEOUT] = 0x4c,
304 [DMA_RING9_TIMEOUT] = 0x50,
305 [DMA_RING10_TIMEOUT] = 0x54,
306 [DMA_RING11_TIMEOUT] = 0x58,
307 [DMA_RING12_TIMEOUT] = 0x5c,
308 [DMA_RING13_TIMEOUT] = 0x60,
309 [DMA_RING14_TIMEOUT] = 0x64,
310 [DMA_RING15_TIMEOUT] = 0x68,
311 [DMA_RING16_TIMEOUT] = 0x6C,
312 };
313
314 /* Set at runtime once bcmgenet version is known */
315 static const u8 *bcmgenet_dma_regs;
316
dev_to_priv(struct device * dev)317 static inline struct bcmgenet_priv *dev_to_priv(struct device *dev)
318 {
319 return netdev_priv(dev_get_drvdata(dev));
320 }
321
bcmgenet_tdma_readl(struct bcmgenet_priv * priv,enum dma_reg r)322 static inline u32 bcmgenet_tdma_readl(struct bcmgenet_priv *priv,
323 enum dma_reg r)
324 {
325 return bcmgenet_readl(priv->base + GENET_TDMA_REG_OFF +
326 DMA_RINGS_SIZE + bcmgenet_dma_regs[r]);
327 }
328
bcmgenet_tdma_writel(struct bcmgenet_priv * priv,u32 val,enum dma_reg r)329 static inline void bcmgenet_tdma_writel(struct bcmgenet_priv *priv,
330 u32 val, enum dma_reg r)
331 {
332 bcmgenet_writel(val, priv->base + GENET_TDMA_REG_OFF +
333 DMA_RINGS_SIZE + bcmgenet_dma_regs[r]);
334 }
335
bcmgenet_rdma_readl(struct bcmgenet_priv * priv,enum dma_reg r)336 static inline u32 bcmgenet_rdma_readl(struct bcmgenet_priv *priv,
337 enum dma_reg r)
338 {
339 return bcmgenet_readl(priv->base + GENET_RDMA_REG_OFF +
340 DMA_RINGS_SIZE + bcmgenet_dma_regs[r]);
341 }
342
bcmgenet_rdma_writel(struct bcmgenet_priv * priv,u32 val,enum dma_reg r)343 static inline void bcmgenet_rdma_writel(struct bcmgenet_priv *priv,
344 u32 val, enum dma_reg r)
345 {
346 bcmgenet_writel(val, priv->base + GENET_RDMA_REG_OFF +
347 DMA_RINGS_SIZE + bcmgenet_dma_regs[r]);
348 }
349
350 /* RDMA/TDMA ring registers and accessors
351 * we merge the common fields and just prefix with T/D the registers
352 * having different meaning depending on the direction
353 */
354 enum dma_ring_reg {
355 TDMA_READ_PTR = 0,
356 RDMA_WRITE_PTR = TDMA_READ_PTR,
357 TDMA_READ_PTR_HI,
358 RDMA_WRITE_PTR_HI = TDMA_READ_PTR_HI,
359 TDMA_CONS_INDEX,
360 RDMA_PROD_INDEX = TDMA_CONS_INDEX,
361 TDMA_PROD_INDEX,
362 RDMA_CONS_INDEX = TDMA_PROD_INDEX,
363 DMA_RING_BUF_SIZE,
364 DMA_START_ADDR,
365 DMA_START_ADDR_HI,
366 DMA_END_ADDR,
367 DMA_END_ADDR_HI,
368 DMA_MBUF_DONE_THRESH,
369 TDMA_FLOW_PERIOD,
370 RDMA_XON_XOFF_THRESH = TDMA_FLOW_PERIOD,
371 TDMA_WRITE_PTR,
372 RDMA_READ_PTR = TDMA_WRITE_PTR,
373 TDMA_WRITE_PTR_HI,
374 RDMA_READ_PTR_HI = TDMA_WRITE_PTR_HI
375 };
376
377 /* GENET v4 supports 40-bits pointer addressing
378 * for obvious reasons the LO and HI word parts
379 * are contiguous, but this offsets the other
380 * registers.
381 */
382 static const u8 genet_dma_ring_regs_v4[] = {
383 [TDMA_READ_PTR] = 0x00,
384 [TDMA_READ_PTR_HI] = 0x04,
385 [TDMA_CONS_INDEX] = 0x08,
386 [TDMA_PROD_INDEX] = 0x0C,
387 [DMA_RING_BUF_SIZE] = 0x10,
388 [DMA_START_ADDR] = 0x14,
389 [DMA_START_ADDR_HI] = 0x18,
390 [DMA_END_ADDR] = 0x1C,
391 [DMA_END_ADDR_HI] = 0x20,
392 [DMA_MBUF_DONE_THRESH] = 0x24,
393 [TDMA_FLOW_PERIOD] = 0x28,
394 [TDMA_WRITE_PTR] = 0x2C,
395 [TDMA_WRITE_PTR_HI] = 0x30,
396 };
397
398 static const u8 genet_dma_ring_regs_v123[] = {
399 [TDMA_READ_PTR] = 0x00,
400 [TDMA_CONS_INDEX] = 0x04,
401 [TDMA_PROD_INDEX] = 0x08,
402 [DMA_RING_BUF_SIZE] = 0x0C,
403 [DMA_START_ADDR] = 0x10,
404 [DMA_END_ADDR] = 0x14,
405 [DMA_MBUF_DONE_THRESH] = 0x18,
406 [TDMA_FLOW_PERIOD] = 0x1C,
407 [TDMA_WRITE_PTR] = 0x20,
408 };
409
410 /* Set at runtime once GENET version is known */
411 static const u8 *genet_dma_ring_regs;
412
bcmgenet_tdma_ring_readl(struct bcmgenet_priv * priv,unsigned int ring,enum dma_ring_reg r)413 static inline u32 bcmgenet_tdma_ring_readl(struct bcmgenet_priv *priv,
414 unsigned int ring,
415 enum dma_ring_reg r)
416 {
417 return bcmgenet_readl(priv->base + GENET_TDMA_REG_OFF +
418 (DMA_RING_SIZE * ring) +
419 genet_dma_ring_regs[r]);
420 }
421
bcmgenet_tdma_ring_writel(struct bcmgenet_priv * priv,unsigned int ring,u32 val,enum dma_ring_reg r)422 static inline void bcmgenet_tdma_ring_writel(struct bcmgenet_priv *priv,
423 unsigned int ring, u32 val,
424 enum dma_ring_reg r)
425 {
426 bcmgenet_writel(val, priv->base + GENET_TDMA_REG_OFF +
427 (DMA_RING_SIZE * ring) +
428 genet_dma_ring_regs[r]);
429 }
430
bcmgenet_rdma_ring_readl(struct bcmgenet_priv * priv,unsigned int ring,enum dma_ring_reg r)431 static inline u32 bcmgenet_rdma_ring_readl(struct bcmgenet_priv *priv,
432 unsigned int ring,
433 enum dma_ring_reg r)
434 {
435 return bcmgenet_readl(priv->base + GENET_RDMA_REG_OFF +
436 (DMA_RING_SIZE * ring) +
437 genet_dma_ring_regs[r]);
438 }
439
bcmgenet_rdma_ring_writel(struct bcmgenet_priv * priv,unsigned int ring,u32 val,enum dma_ring_reg r)440 static inline void bcmgenet_rdma_ring_writel(struct bcmgenet_priv *priv,
441 unsigned int ring, u32 val,
442 enum dma_ring_reg r)
443 {
444 bcmgenet_writel(val, priv->base + GENET_RDMA_REG_OFF +
445 (DMA_RING_SIZE * ring) +
446 genet_dma_ring_regs[r]);
447 }
448
bcmgenet_hfb_enable_filter(struct bcmgenet_priv * priv,u32 f_index)449 static void bcmgenet_hfb_enable_filter(struct bcmgenet_priv *priv, u32 f_index)
450 {
451 u32 offset;
452 u32 reg;
453
454 if (GENET_IS_V1(priv) || GENET_IS_V2(priv)) {
455 reg = bcmgenet_hfb_reg_readl(priv, HFB_CTRL);
456 reg |= (1 << ((f_index % 32) + RBUF_HFB_FILTER_EN_SHIFT)) |
457 RBUF_HFB_EN;
458 bcmgenet_hfb_reg_writel(priv, reg, HFB_CTRL);
459 } else {
460 offset = HFB_FLT_ENABLE_V3PLUS + (f_index < 32) * sizeof(u32);
461 reg = bcmgenet_hfb_reg_readl(priv, offset);
462 reg |= (1 << (f_index % 32));
463 bcmgenet_hfb_reg_writel(priv, reg, offset);
464 reg = bcmgenet_hfb_reg_readl(priv, HFB_CTRL);
465 reg |= RBUF_HFB_EN;
466 bcmgenet_hfb_reg_writel(priv, reg, HFB_CTRL);
467 }
468 }
469
bcmgenet_hfb_disable_filter(struct bcmgenet_priv * priv,u32 f_index)470 static void bcmgenet_hfb_disable_filter(struct bcmgenet_priv *priv, u32 f_index)
471 {
472 u32 offset, reg, reg1;
473
474 if (GENET_IS_V1(priv) || GENET_IS_V2(priv)) {
475 reg = bcmgenet_hfb_reg_readl(priv, HFB_CTRL);
476 reg &= ~(1 << ((f_index % 32) + RBUF_HFB_FILTER_EN_SHIFT));
477 if (!(reg & RBUF_HFB_FILTER_EN_MASK))
478 reg &= ~RBUF_HFB_EN;
479 bcmgenet_hfb_reg_writel(priv, reg, HFB_CTRL);
480 } else {
481 offset = HFB_FLT_ENABLE_V3PLUS;
482 reg = bcmgenet_hfb_reg_readl(priv, offset);
483 reg1 = bcmgenet_hfb_reg_readl(priv, offset + sizeof(u32));
484 if (f_index < 32) {
485 reg1 &= ~(1 << (f_index % 32));
486 bcmgenet_hfb_reg_writel(priv, reg1, offset + sizeof(u32));
487 } else {
488 reg &= ~(1 << (f_index % 32));
489 bcmgenet_hfb_reg_writel(priv, reg, offset);
490 }
491 if (!reg && !reg1) {
492 reg = bcmgenet_hfb_reg_readl(priv, HFB_CTRL);
493 reg &= ~RBUF_HFB_EN;
494 bcmgenet_hfb_reg_writel(priv, reg, HFB_CTRL);
495 }
496 }
497 }
498
bcmgenet_hfb_set_filter_rx_queue_mapping(struct bcmgenet_priv * priv,u32 f_index,u32 rx_queue)499 static void bcmgenet_hfb_set_filter_rx_queue_mapping(struct bcmgenet_priv *priv,
500 u32 f_index, u32 rx_queue)
501 {
502 u32 offset;
503 u32 reg;
504
505 if (GENET_IS_V1(priv) || GENET_IS_V2(priv))
506 return;
507
508 offset = f_index / 8;
509 reg = bcmgenet_rdma_readl(priv, DMA_INDEX2RING_0 + offset);
510 reg &= ~(0xF << (4 * (f_index % 8)));
511 reg |= ((rx_queue & 0xF) << (4 * (f_index % 8)));
512 bcmgenet_rdma_writel(priv, reg, DMA_INDEX2RING_0 + offset);
513 }
514
bcmgenet_hfb_set_filter_length(struct bcmgenet_priv * priv,u32 f_index,u32 f_length)515 static void bcmgenet_hfb_set_filter_length(struct bcmgenet_priv *priv,
516 u32 f_index, u32 f_length)
517 {
518 u32 offset;
519 u32 reg;
520
521 if (GENET_IS_V1(priv) || GENET_IS_V2(priv))
522 offset = HFB_FLT_LEN_V2;
523 else
524 offset = HFB_FLT_LEN_V3PLUS;
525
526 offset += sizeof(u32) *
527 ((priv->hw_params->hfb_filter_cnt - 1 - f_index) / 4);
528 reg = bcmgenet_hfb_reg_readl(priv, offset);
529 reg &= ~(0xFF << (8 * (f_index % 4)));
530 reg |= ((f_length & 0xFF) << (8 * (f_index % 4)));
531 bcmgenet_hfb_reg_writel(priv, reg, offset);
532 }
533
bcmgenet_hfb_validate_mask(void * mask,size_t size)534 static int bcmgenet_hfb_validate_mask(void *mask, size_t size)
535 {
536 while (size) {
537 switch (*(unsigned char *)mask++) {
538 case 0x00:
539 case 0x0f:
540 case 0xf0:
541 case 0xff:
542 size--;
543 continue;
544 default:
545 return -EINVAL;
546 }
547 }
548
549 return 0;
550 }
551
552 #define VALIDATE_MASK(x) \
553 bcmgenet_hfb_validate_mask(&(x), sizeof(x))
554
bcmgenet_hfb_insert_data(struct bcmgenet_priv * priv,u32 f_index,u32 offset,void * val,void * mask,size_t size)555 static int bcmgenet_hfb_insert_data(struct bcmgenet_priv *priv, u32 f_index,
556 u32 offset, void *val, void *mask,
557 size_t size)
558 {
559 u32 index, tmp;
560
561 index = f_index * priv->hw_params->hfb_filter_size + offset / 2;
562 tmp = bcmgenet_hfb_readl(priv, index * sizeof(u32));
563
564 while (size--) {
565 if (offset++ & 1) {
566 tmp &= ~0x300FF;
567 tmp |= (*(unsigned char *)val++);
568 switch ((*(unsigned char *)mask++)) {
569 case 0xFF:
570 tmp |= 0x30000;
571 break;
572 case 0xF0:
573 tmp |= 0x20000;
574 break;
575 case 0x0F:
576 tmp |= 0x10000;
577 break;
578 }
579 bcmgenet_hfb_writel(priv, tmp, index++ * sizeof(u32));
580 if (size)
581 tmp = bcmgenet_hfb_readl(priv,
582 index * sizeof(u32));
583 } else {
584 tmp &= ~0xCFF00;
585 tmp |= (*(unsigned char *)val++) << 8;
586 switch ((*(unsigned char *)mask++)) {
587 case 0xFF:
588 tmp |= 0xC0000;
589 break;
590 case 0xF0:
591 tmp |= 0x80000;
592 break;
593 case 0x0F:
594 tmp |= 0x40000;
595 break;
596 }
597 if (!size)
598 bcmgenet_hfb_writel(priv, tmp, index * sizeof(u32));
599 }
600 }
601
602 return 0;
603 }
604
bcmgenet_hfb_create_rxnfc_filter(struct bcmgenet_priv * priv,struct bcmgenet_rxnfc_rule * rule)605 static void bcmgenet_hfb_create_rxnfc_filter(struct bcmgenet_priv *priv,
606 struct bcmgenet_rxnfc_rule *rule)
607 {
608 struct ethtool_rx_flow_spec *fs = &rule->fs;
609 u32 offset = 0, f_length = 0, f, q;
610 u8 val_8, mask_8;
611 __be16 val_16;
612 u16 mask_16;
613 size_t size;
614
615 f = fs->location + 1;
616 if (fs->flow_type & FLOW_MAC_EXT) {
617 bcmgenet_hfb_insert_data(priv, f, 0,
618 &fs->h_ext.h_dest, &fs->m_ext.h_dest,
619 sizeof(fs->h_ext.h_dest));
620 }
621
622 if (fs->flow_type & FLOW_EXT) {
623 if (fs->m_ext.vlan_etype ||
624 fs->m_ext.vlan_tci) {
625 bcmgenet_hfb_insert_data(priv, f, 12,
626 &fs->h_ext.vlan_etype,
627 &fs->m_ext.vlan_etype,
628 sizeof(fs->h_ext.vlan_etype));
629 bcmgenet_hfb_insert_data(priv, f, 14,
630 &fs->h_ext.vlan_tci,
631 &fs->m_ext.vlan_tci,
632 sizeof(fs->h_ext.vlan_tci));
633 offset += VLAN_HLEN;
634 f_length += DIV_ROUND_UP(VLAN_HLEN, 2);
635 }
636 }
637
638 switch (fs->flow_type & ~(FLOW_EXT | FLOW_MAC_EXT)) {
639 case ETHER_FLOW:
640 f_length += DIV_ROUND_UP(ETH_HLEN, 2);
641 bcmgenet_hfb_insert_data(priv, f, 0,
642 &fs->h_u.ether_spec.h_dest,
643 &fs->m_u.ether_spec.h_dest,
644 sizeof(fs->h_u.ether_spec.h_dest));
645 bcmgenet_hfb_insert_data(priv, f, ETH_ALEN,
646 &fs->h_u.ether_spec.h_source,
647 &fs->m_u.ether_spec.h_source,
648 sizeof(fs->h_u.ether_spec.h_source));
649 bcmgenet_hfb_insert_data(priv, f, (2 * ETH_ALEN) + offset,
650 &fs->h_u.ether_spec.h_proto,
651 &fs->m_u.ether_spec.h_proto,
652 sizeof(fs->h_u.ether_spec.h_proto));
653 break;
654 case IP_USER_FLOW:
655 f_length += DIV_ROUND_UP(ETH_HLEN + 20, 2);
656 /* Specify IP Ether Type */
657 val_16 = htons(ETH_P_IP);
658 mask_16 = 0xFFFF;
659 bcmgenet_hfb_insert_data(priv, f, (2 * ETH_ALEN) + offset,
660 &val_16, &mask_16, sizeof(val_16));
661 bcmgenet_hfb_insert_data(priv, f, 15 + offset,
662 &fs->h_u.usr_ip4_spec.tos,
663 &fs->m_u.usr_ip4_spec.tos,
664 sizeof(fs->h_u.usr_ip4_spec.tos));
665 bcmgenet_hfb_insert_data(priv, f, 23 + offset,
666 &fs->h_u.usr_ip4_spec.proto,
667 &fs->m_u.usr_ip4_spec.proto,
668 sizeof(fs->h_u.usr_ip4_spec.proto));
669 bcmgenet_hfb_insert_data(priv, f, 26 + offset,
670 &fs->h_u.usr_ip4_spec.ip4src,
671 &fs->m_u.usr_ip4_spec.ip4src,
672 sizeof(fs->h_u.usr_ip4_spec.ip4src));
673 bcmgenet_hfb_insert_data(priv, f, 30 + offset,
674 &fs->h_u.usr_ip4_spec.ip4dst,
675 &fs->m_u.usr_ip4_spec.ip4dst,
676 sizeof(fs->h_u.usr_ip4_spec.ip4dst));
677 if (!fs->m_u.usr_ip4_spec.l4_4_bytes)
678 break;
679
680 /* Only supports 20 byte IPv4 header */
681 val_8 = 0x45;
682 mask_8 = 0xFF;
683 bcmgenet_hfb_insert_data(priv, f, ETH_HLEN + offset,
684 &val_8, &mask_8,
685 sizeof(val_8));
686 size = sizeof(fs->h_u.usr_ip4_spec.l4_4_bytes);
687 bcmgenet_hfb_insert_data(priv, f,
688 ETH_HLEN + 20 + offset,
689 &fs->h_u.usr_ip4_spec.l4_4_bytes,
690 &fs->m_u.usr_ip4_spec.l4_4_bytes,
691 size);
692 f_length += DIV_ROUND_UP(size, 2);
693 break;
694 }
695
696 bcmgenet_hfb_set_filter_length(priv, f, 2 * f_length);
697 if (fs->ring_cookie == RX_CLS_FLOW_WAKE)
698 q = 0;
699 else if (fs->ring_cookie == RX_CLS_FLOW_DISC)
700 q = priv->hw_params->rx_queues + 1;
701 else
702 /* Other Rx rings are direct mapped here */
703 q = fs->ring_cookie;
704 bcmgenet_hfb_set_filter_rx_queue_mapping(priv, f, q);
705 bcmgenet_hfb_enable_filter(priv, f);
706 rule->state = BCMGENET_RXNFC_STATE_ENABLED;
707 }
708
709 /* bcmgenet_hfb_clear
710 *
711 * Clear Hardware Filter Block and disable all filtering.
712 */
bcmgenet_hfb_clear_filter(struct bcmgenet_priv * priv,u32 f_index)713 static void bcmgenet_hfb_clear_filter(struct bcmgenet_priv *priv, u32 f_index)
714 {
715 u32 base, i;
716
717 bcmgenet_hfb_set_filter_length(priv, f_index, 0);
718 base = f_index * priv->hw_params->hfb_filter_size;
719 for (i = 0; i < priv->hw_params->hfb_filter_size; i++)
720 bcmgenet_hfb_writel(priv, 0x0, (base + i) * sizeof(u32));
721 }
722
bcmgenet_hfb_clear(struct bcmgenet_priv * priv)723 static void bcmgenet_hfb_clear(struct bcmgenet_priv *priv)
724 {
725 u32 i;
726
727 bcmgenet_hfb_reg_writel(priv, 0, HFB_CTRL);
728
729 if (!GENET_IS_V1(priv) && !GENET_IS_V2(priv)) {
730 bcmgenet_hfb_reg_writel(priv, 0,
731 HFB_FLT_ENABLE_V3PLUS);
732 bcmgenet_hfb_reg_writel(priv, 0,
733 HFB_FLT_ENABLE_V3PLUS + 4);
734 for (i = DMA_INDEX2RING_0; i <= DMA_INDEX2RING_7; i++)
735 bcmgenet_rdma_writel(priv, 0, i);
736 }
737
738 for (i = 0; i < priv->hw_params->hfb_filter_cnt; i++)
739 bcmgenet_hfb_clear_filter(priv, i);
740
741 /* Enable filter 0 to send default flow to ring 0 */
742 bcmgenet_hfb_set_filter_length(priv, 0, 4);
743 bcmgenet_hfb_enable_filter(priv, 0);
744 }
745
bcmgenet_hfb_init(struct bcmgenet_priv * priv)746 static void bcmgenet_hfb_init(struct bcmgenet_priv *priv)
747 {
748 int i;
749
750 INIT_LIST_HEAD(&priv->rxnfc_list);
751 for (i = 0; i < MAX_NUM_OF_FS_RULES; i++) {
752 INIT_LIST_HEAD(&priv->rxnfc_rules[i].list);
753 priv->rxnfc_rules[i].state = BCMGENET_RXNFC_STATE_UNUSED;
754 }
755 }
756
bcmgenet_hfb_restore(struct bcmgenet_priv * priv)757 static void bcmgenet_hfb_restore(struct bcmgenet_priv *priv)
758 {
759 struct bcmgenet_rxnfc_rule *rule;
760
761 bcmgenet_hfb_clear(priv);
762
763 list_for_each_entry(rule, &priv->rxnfc_list, list)
764 if (rule->state != BCMGENET_RXNFC_STATE_UNUSED)
765 bcmgenet_hfb_create_rxnfc_filter(priv, rule);
766 }
767
bcmgenet_begin(struct net_device * dev)768 static int bcmgenet_begin(struct net_device *dev)
769 {
770 struct bcmgenet_priv *priv = netdev_priv(dev);
771
772 /* Turn on the clock */
773 return clk_prepare_enable(priv->clk);
774 }
775
bcmgenet_complete(struct net_device * dev)776 static void bcmgenet_complete(struct net_device *dev)
777 {
778 struct bcmgenet_priv *priv = netdev_priv(dev);
779
780 /* Turn off the clock */
781 clk_disable_unprepare(priv->clk);
782 }
783
bcmgenet_get_link_ksettings(struct net_device * dev,struct ethtool_link_ksettings * cmd)784 static int bcmgenet_get_link_ksettings(struct net_device *dev,
785 struct ethtool_link_ksettings *cmd)
786 {
787 if (!netif_running(dev))
788 return -EINVAL;
789
790 if (!dev->phydev)
791 return -ENODEV;
792
793 phy_ethtool_ksettings_get(dev->phydev, cmd);
794
795 return 0;
796 }
797
bcmgenet_set_link_ksettings(struct net_device * dev,const struct ethtool_link_ksettings * cmd)798 static int bcmgenet_set_link_ksettings(struct net_device *dev,
799 const struct ethtool_link_ksettings *cmd)
800 {
801 if (!netif_running(dev))
802 return -EINVAL;
803
804 if (!dev->phydev)
805 return -ENODEV;
806
807 return phy_ethtool_ksettings_set(dev->phydev, cmd);
808 }
809
bcmgenet_set_features(struct net_device * dev,netdev_features_t features)810 static int bcmgenet_set_features(struct net_device *dev,
811 netdev_features_t features)
812 {
813 struct bcmgenet_priv *priv = netdev_priv(dev);
814 u32 reg;
815 int ret;
816
817 ret = clk_prepare_enable(priv->clk);
818 if (ret)
819 return ret;
820
821 /* Make sure we reflect the value of CRC_CMD_FWD */
822 reg = bcmgenet_umac_readl(priv, UMAC_CMD);
823 priv->crc_fwd_en = !!(reg & CMD_CRC_FWD);
824
825 clk_disable_unprepare(priv->clk);
826
827 return ret;
828 }
829
bcmgenet_get_msglevel(struct net_device * dev)830 static u32 bcmgenet_get_msglevel(struct net_device *dev)
831 {
832 struct bcmgenet_priv *priv = netdev_priv(dev);
833
834 return priv->msg_enable;
835 }
836
bcmgenet_set_msglevel(struct net_device * dev,u32 level)837 static void bcmgenet_set_msglevel(struct net_device *dev, u32 level)
838 {
839 struct bcmgenet_priv *priv = netdev_priv(dev);
840
841 priv->msg_enable = level;
842 }
843
bcmgenet_get_coalesce(struct net_device * dev,struct ethtool_coalesce * ec,struct kernel_ethtool_coalesce * kernel_coal,struct netlink_ext_ack * extack)844 static int bcmgenet_get_coalesce(struct net_device *dev,
845 struct ethtool_coalesce *ec,
846 struct kernel_ethtool_coalesce *kernel_coal,
847 struct netlink_ext_ack *extack)
848 {
849 struct bcmgenet_priv *priv = netdev_priv(dev);
850 struct bcmgenet_rx_ring *ring;
851 unsigned int i;
852
853 ec->tx_max_coalesced_frames =
854 bcmgenet_tdma_ring_readl(priv, 0, DMA_MBUF_DONE_THRESH);
855 ec->rx_max_coalesced_frames =
856 bcmgenet_rdma_ring_readl(priv, 0, DMA_MBUF_DONE_THRESH);
857 ec->rx_coalesce_usecs =
858 (bcmgenet_rdma_readl(priv, DMA_RING0_TIMEOUT) &
859 DMA_TIMEOUT_MASK) * 8192 / 1000;
860
861 for (i = 0; i <= priv->hw_params->rx_queues; i++) {
862 ring = &priv->rx_rings[i];
863 ec->use_adaptive_rx_coalesce |= ring->dim.use_dim;
864 }
865
866 return 0;
867 }
868
bcmgenet_set_rx_coalesce(struct bcmgenet_rx_ring * ring,u32 usecs,u32 pkts)869 static void bcmgenet_set_rx_coalesce(struct bcmgenet_rx_ring *ring,
870 u32 usecs, u32 pkts)
871 {
872 struct bcmgenet_priv *priv = ring->priv;
873 unsigned int i = ring->index;
874 u32 reg;
875
876 bcmgenet_rdma_ring_writel(priv, i, pkts, DMA_MBUF_DONE_THRESH);
877
878 reg = bcmgenet_rdma_readl(priv, DMA_RING0_TIMEOUT + i);
879 reg &= ~DMA_TIMEOUT_MASK;
880 reg |= DIV_ROUND_UP(usecs * 1000, 8192);
881 bcmgenet_rdma_writel(priv, reg, DMA_RING0_TIMEOUT + i);
882 }
883
bcmgenet_set_ring_rx_coalesce(struct bcmgenet_rx_ring * ring,struct ethtool_coalesce * ec)884 static void bcmgenet_set_ring_rx_coalesce(struct bcmgenet_rx_ring *ring,
885 struct ethtool_coalesce *ec)
886 {
887 struct dim_cq_moder moder;
888 u32 usecs, pkts;
889
890 ring->rx_coalesce_usecs = ec->rx_coalesce_usecs;
891 ring->rx_max_coalesced_frames = ec->rx_max_coalesced_frames;
892 usecs = ring->rx_coalesce_usecs;
893 pkts = ring->rx_max_coalesced_frames;
894
895 if (ec->use_adaptive_rx_coalesce && !ring->dim.use_dim) {
896 moder = net_dim_get_def_rx_moderation(ring->dim.dim.mode);
897 usecs = moder.usec;
898 pkts = moder.pkts;
899 }
900
901 ring->dim.use_dim = ec->use_adaptive_rx_coalesce;
902 bcmgenet_set_rx_coalesce(ring, usecs, pkts);
903 }
904
bcmgenet_set_coalesce(struct net_device * dev,struct ethtool_coalesce * ec,struct kernel_ethtool_coalesce * kernel_coal,struct netlink_ext_ack * extack)905 static int bcmgenet_set_coalesce(struct net_device *dev,
906 struct ethtool_coalesce *ec,
907 struct kernel_ethtool_coalesce *kernel_coal,
908 struct netlink_ext_ack *extack)
909 {
910 struct bcmgenet_priv *priv = netdev_priv(dev);
911 unsigned int i;
912
913 /* Base system clock is 125Mhz, DMA timeout is this reference clock
914 * divided by 1024, which yields roughly 8.192us, our maximum value
915 * has to fit in the DMA_TIMEOUT_MASK (16 bits)
916 */
917 if (ec->tx_max_coalesced_frames > DMA_INTR_THRESHOLD_MASK ||
918 ec->tx_max_coalesced_frames == 0 ||
919 ec->rx_max_coalesced_frames > DMA_INTR_THRESHOLD_MASK ||
920 ec->rx_coalesce_usecs > (DMA_TIMEOUT_MASK * 8) + 1)
921 return -EINVAL;
922
923 if (ec->rx_coalesce_usecs == 0 && ec->rx_max_coalesced_frames == 0)
924 return -EINVAL;
925
926 /* GENET TDMA hardware does not support a configurable timeout, but will
927 * always generate an interrupt either after MBDONE packets have been
928 * transmitted, or when the ring is empty.
929 */
930
931 /* Program all TX queues with the same values, as there is no
932 * ethtool knob to do coalescing on a per-queue basis
933 */
934 for (i = 0; i <= priv->hw_params->tx_queues; i++)
935 bcmgenet_tdma_ring_writel(priv, i,
936 ec->tx_max_coalesced_frames,
937 DMA_MBUF_DONE_THRESH);
938
939 for (i = 0; i <= priv->hw_params->rx_queues; i++)
940 bcmgenet_set_ring_rx_coalesce(&priv->rx_rings[i], ec);
941
942 return 0;
943 }
944
bcmgenet_get_pauseparam(struct net_device * dev,struct ethtool_pauseparam * epause)945 static void bcmgenet_get_pauseparam(struct net_device *dev,
946 struct ethtool_pauseparam *epause)
947 {
948 struct bcmgenet_priv *priv;
949 u32 umac_cmd;
950
951 priv = netdev_priv(dev);
952
953 epause->autoneg = priv->autoneg_pause;
954
955 if (netif_carrier_ok(dev)) {
956 /* report active state when link is up */
957 umac_cmd = bcmgenet_umac_readl(priv, UMAC_CMD);
958 epause->tx_pause = !(umac_cmd & CMD_TX_PAUSE_IGNORE);
959 epause->rx_pause = !(umac_cmd & CMD_RX_PAUSE_IGNORE);
960 } else {
961 /* otherwise report stored settings */
962 epause->tx_pause = priv->tx_pause;
963 epause->rx_pause = priv->rx_pause;
964 }
965 }
966
bcmgenet_set_pauseparam(struct net_device * dev,struct ethtool_pauseparam * epause)967 static int bcmgenet_set_pauseparam(struct net_device *dev,
968 struct ethtool_pauseparam *epause)
969 {
970 struct bcmgenet_priv *priv = netdev_priv(dev);
971
972 if (!dev->phydev)
973 return -ENODEV;
974
975 if (!phy_validate_pause(dev->phydev, epause))
976 return -EINVAL;
977
978 priv->autoneg_pause = !!epause->autoneg;
979 priv->tx_pause = !!epause->tx_pause;
980 priv->rx_pause = !!epause->rx_pause;
981
982 bcmgenet_phy_pause_set(dev, priv->rx_pause, priv->tx_pause);
983
984 return 0;
985 }
986
987 /* standard ethtool support functions. */
988 enum bcmgenet_stat_type {
989 BCMGENET_STAT_RTNL = -1,
990 BCMGENET_STAT_MIB_RX,
991 BCMGENET_STAT_MIB_TX,
992 BCMGENET_STAT_RUNT,
993 BCMGENET_STAT_MISC,
994 BCMGENET_STAT_SOFT,
995 BCMGENET_STAT_SOFT64,
996 };
997
998 struct bcmgenet_stats {
999 char stat_string[ETH_GSTRING_LEN];
1000 int stat_sizeof;
1001 int stat_offset;
1002 enum bcmgenet_stat_type type;
1003 /* reg offset from UMAC base for misc counters */
1004 u16 reg_offset;
1005 /* sync for u64 stats counters */
1006 int syncp_offset;
1007 };
1008
1009 #define STAT_RTNL(m) { \
1010 .stat_string = __stringify(m), \
1011 .stat_sizeof = sizeof(((struct rtnl_link_stats64 *)0)->m), \
1012 .stat_offset = offsetof(struct rtnl_link_stats64, m), \
1013 .type = BCMGENET_STAT_RTNL, \
1014 }
1015
1016 #define STAT_GENET_MIB(str, m, _type) { \
1017 .stat_string = str, \
1018 .stat_sizeof = sizeof(((struct bcmgenet_priv *)0)->m), \
1019 .stat_offset = offsetof(struct bcmgenet_priv, m), \
1020 .type = _type, \
1021 }
1022
1023 #define STAT_GENET_SOFT_MIB64(str, s, m) { \
1024 .stat_string = str, \
1025 .stat_sizeof = sizeof(((struct bcmgenet_priv *)0)->s.m), \
1026 .stat_offset = offsetof(struct bcmgenet_priv, s.m), \
1027 .type = BCMGENET_STAT_SOFT64, \
1028 .syncp_offset = offsetof(struct bcmgenet_priv, s.syncp), \
1029 }
1030
1031 #define STAT_GENET_MIB_RX(str, m) STAT_GENET_MIB(str, m, BCMGENET_STAT_MIB_RX)
1032 #define STAT_GENET_MIB_TX(str, m) STAT_GENET_MIB(str, m, BCMGENET_STAT_MIB_TX)
1033 #define STAT_GENET_RUNT(str, m) STAT_GENET_MIB(str, m, BCMGENET_STAT_RUNT)
1034 #define STAT_GENET_SOFT_MIB(str, m) STAT_GENET_MIB(str, m, BCMGENET_STAT_SOFT)
1035
1036 #define STAT_GENET_MISC(str, m, offset) { \
1037 .stat_string = str, \
1038 .stat_sizeof = sizeof(((struct bcmgenet_priv *)0)->m), \
1039 .stat_offset = offsetof(struct bcmgenet_priv, m), \
1040 .type = BCMGENET_STAT_MISC, \
1041 .reg_offset = offset, \
1042 }
1043
1044 #define STAT_GENET_Q(num) \
1045 STAT_GENET_SOFT_MIB64("txq" __stringify(num) "_packets", \
1046 tx_rings[num].stats64, packets), \
1047 STAT_GENET_SOFT_MIB64("txq" __stringify(num) "_bytes", \
1048 tx_rings[num].stats64, bytes), \
1049 STAT_GENET_SOFT_MIB64("txq" __stringify(num) "_errors", \
1050 tx_rings[num].stats64, errors), \
1051 STAT_GENET_SOFT_MIB64("txq" __stringify(num) "_dropped", \
1052 tx_rings[num].stats64, dropped), \
1053 STAT_GENET_SOFT_MIB64("rxq" __stringify(num) "_bytes", \
1054 rx_rings[num].stats64, bytes), \
1055 STAT_GENET_SOFT_MIB64("rxq" __stringify(num) "_packets", \
1056 rx_rings[num].stats64, packets), \
1057 STAT_GENET_SOFT_MIB64("rxq" __stringify(num) "_errors", \
1058 rx_rings[num].stats64, errors), \
1059 STAT_GENET_SOFT_MIB64("rxq" __stringify(num) "_dropped", \
1060 rx_rings[num].stats64, dropped), \
1061 STAT_GENET_SOFT_MIB64("rxq" __stringify(num) "_multicast", \
1062 rx_rings[num].stats64, multicast), \
1063 STAT_GENET_SOFT_MIB64("rxq" __stringify(num) "_missed", \
1064 rx_rings[num].stats64, missed), \
1065 STAT_GENET_SOFT_MIB64("rxq" __stringify(num) "_length_errors", \
1066 rx_rings[num].stats64, length_errors), \
1067 STAT_GENET_SOFT_MIB64("rxq" __stringify(num) "_over_errors", \
1068 rx_rings[num].stats64, over_errors), \
1069 STAT_GENET_SOFT_MIB64("rxq" __stringify(num) "_crc_errors", \
1070 rx_rings[num].stats64, crc_errors), \
1071 STAT_GENET_SOFT_MIB64("rxq" __stringify(num) "_frame_errors", \
1072 rx_rings[num].stats64, frame_errors), \
1073 STAT_GENET_SOFT_MIB64("rxq" __stringify(num) "_fragmented_errors", \
1074 rx_rings[num].stats64, fragmented_errors), \
1075 STAT_GENET_SOFT_MIB64("rxq" __stringify(num) "_broadcast", \
1076 rx_rings[num].stats64, broadcast)
1077
1078 /* There is a 0xC gap between the end of RX and beginning of TX stats and then
1079 * between the end of TX stats and the beginning of the RX RUNT
1080 */
1081 #define BCMGENET_STAT_OFFSET 0xc
1082
1083 /* Hardware counters must be kept in sync because the order/offset
1084 * is important here (order in structure declaration = order in hardware)
1085 */
1086 static const struct bcmgenet_stats bcmgenet_gstrings_stats[] = {
1087 /* general stats */
1088 STAT_RTNL(rx_packets),
1089 STAT_RTNL(tx_packets),
1090 STAT_RTNL(rx_bytes),
1091 STAT_RTNL(tx_bytes),
1092 STAT_RTNL(rx_errors),
1093 STAT_RTNL(tx_errors),
1094 STAT_RTNL(rx_dropped),
1095 STAT_RTNL(tx_dropped),
1096 STAT_RTNL(multicast),
1097 STAT_RTNL(rx_missed_errors),
1098 STAT_RTNL(rx_length_errors),
1099 STAT_RTNL(rx_over_errors),
1100 STAT_RTNL(rx_crc_errors),
1101 STAT_RTNL(rx_frame_errors),
1102 /* UniMAC RSV counters */
1103 STAT_GENET_MIB_RX("rx_64_octets", mib.rx.pkt_cnt.cnt_64),
1104 STAT_GENET_MIB_RX("rx_65_127_oct", mib.rx.pkt_cnt.cnt_127),
1105 STAT_GENET_MIB_RX("rx_128_255_oct", mib.rx.pkt_cnt.cnt_255),
1106 STAT_GENET_MIB_RX("rx_256_511_oct", mib.rx.pkt_cnt.cnt_511),
1107 STAT_GENET_MIB_RX("rx_512_1023_oct", mib.rx.pkt_cnt.cnt_1023),
1108 STAT_GENET_MIB_RX("rx_1024_1518_oct", mib.rx.pkt_cnt.cnt_1518),
1109 STAT_GENET_MIB_RX("rx_vlan_1519_1522_oct", mib.rx.pkt_cnt.cnt_mgv),
1110 STAT_GENET_MIB_RX("rx_1522_2047_oct", mib.rx.pkt_cnt.cnt_2047),
1111 STAT_GENET_MIB_RX("rx_2048_4095_oct", mib.rx.pkt_cnt.cnt_4095),
1112 STAT_GENET_MIB_RX("rx_4096_9216_oct", mib.rx.pkt_cnt.cnt_9216),
1113 STAT_GENET_MIB_RX("rx_pkts", mib.rx.pkt),
1114 STAT_GENET_MIB_RX("rx_bytes", mib.rx.bytes),
1115 STAT_GENET_MIB_RX("rx_multicast", mib.rx.mca),
1116 STAT_GENET_MIB_RX("rx_broadcast", mib.rx.bca),
1117 STAT_GENET_MIB_RX("rx_fcs", mib.rx.fcs),
1118 STAT_GENET_MIB_RX("rx_control", mib.rx.cf),
1119 STAT_GENET_MIB_RX("rx_pause", mib.rx.pf),
1120 STAT_GENET_MIB_RX("rx_unknown", mib.rx.uo),
1121 STAT_GENET_MIB_RX("rx_align", mib.rx.aln),
1122 STAT_GENET_MIB_RX("rx_outrange", mib.rx.flr),
1123 STAT_GENET_MIB_RX("rx_code", mib.rx.cde),
1124 STAT_GENET_MIB_RX("rx_carrier", mib.rx.fcr),
1125 STAT_GENET_MIB_RX("rx_oversize", mib.rx.ovr),
1126 STAT_GENET_MIB_RX("rx_jabber", mib.rx.jbr),
1127 STAT_GENET_MIB_RX("rx_mtu_err", mib.rx.mtue),
1128 STAT_GENET_MIB_RX("rx_good_pkts", mib.rx.pok),
1129 STAT_GENET_MIB_RX("rx_unicast", mib.rx.uc),
1130 STAT_GENET_MIB_RX("rx_ppp", mib.rx.ppp),
1131 STAT_GENET_MIB_RX("rx_crc", mib.rx.rcrc),
1132 /* UniMAC TSV counters */
1133 STAT_GENET_MIB_TX("tx_64_octets", mib.tx.pkt_cnt.cnt_64),
1134 STAT_GENET_MIB_TX("tx_65_127_oct", mib.tx.pkt_cnt.cnt_127),
1135 STAT_GENET_MIB_TX("tx_128_255_oct", mib.tx.pkt_cnt.cnt_255),
1136 STAT_GENET_MIB_TX("tx_256_511_oct", mib.tx.pkt_cnt.cnt_511),
1137 STAT_GENET_MIB_TX("tx_512_1023_oct", mib.tx.pkt_cnt.cnt_1023),
1138 STAT_GENET_MIB_TX("tx_1024_1518_oct", mib.tx.pkt_cnt.cnt_1518),
1139 STAT_GENET_MIB_TX("tx_vlan_1519_1522_oct", mib.tx.pkt_cnt.cnt_mgv),
1140 STAT_GENET_MIB_TX("tx_1522_2047_oct", mib.tx.pkt_cnt.cnt_2047),
1141 STAT_GENET_MIB_TX("tx_2048_4095_oct", mib.tx.pkt_cnt.cnt_4095),
1142 STAT_GENET_MIB_TX("tx_4096_9216_oct", mib.tx.pkt_cnt.cnt_9216),
1143 STAT_GENET_MIB_TX("tx_pkts", mib.tx.pkts),
1144 STAT_GENET_MIB_TX("tx_multicast", mib.tx.mca),
1145 STAT_GENET_MIB_TX("tx_broadcast", mib.tx.bca),
1146 STAT_GENET_MIB_TX("tx_pause", mib.tx.pf),
1147 STAT_GENET_MIB_TX("tx_control", mib.tx.cf),
1148 STAT_GENET_MIB_TX("tx_fcs_err", mib.tx.fcs),
1149 STAT_GENET_MIB_TX("tx_oversize", mib.tx.ovr),
1150 STAT_GENET_MIB_TX("tx_defer", mib.tx.drf),
1151 STAT_GENET_MIB_TX("tx_excess_defer", mib.tx.edf),
1152 STAT_GENET_MIB_TX("tx_single_col", mib.tx.scl),
1153 STAT_GENET_MIB_TX("tx_multi_col", mib.tx.mcl),
1154 STAT_GENET_MIB_TX("tx_late_col", mib.tx.lcl),
1155 STAT_GENET_MIB_TX("tx_excess_col", mib.tx.ecl),
1156 STAT_GENET_MIB_TX("tx_frags", mib.tx.frg),
1157 STAT_GENET_MIB_TX("tx_total_col", mib.tx.ncl),
1158 STAT_GENET_MIB_TX("tx_jabber", mib.tx.jbr),
1159 STAT_GENET_MIB_TX("tx_bytes", mib.tx.bytes),
1160 STAT_GENET_MIB_TX("tx_good_pkts", mib.tx.pok),
1161 STAT_GENET_MIB_TX("tx_unicast", mib.tx.uc),
1162 /* UniMAC RUNT counters */
1163 STAT_GENET_RUNT("rx_runt_pkts", mib.rx_runt_cnt),
1164 STAT_GENET_RUNT("rx_runt_valid_fcs", mib.rx_runt_fcs),
1165 STAT_GENET_RUNT("rx_runt_inval_fcs_align", mib.rx_runt_fcs_align),
1166 STAT_GENET_RUNT("rx_runt_bytes", mib.rx_runt_bytes),
1167 /* Misc UniMAC counters */
1168 STAT_GENET_MISC("rbuf_ovflow_cnt", mib.rbuf_ovflow_cnt,
1169 UMAC_RBUF_OVFL_CNT_V1),
1170 STAT_GENET_MISC("rbuf_err_cnt", mib.rbuf_err_cnt,
1171 UMAC_RBUF_ERR_CNT_V1),
1172 STAT_GENET_MISC("mdf_err_cnt", mib.mdf_err_cnt, UMAC_MDF_ERR_CNT),
1173 STAT_GENET_SOFT_MIB("alloc_rx_buff_failed", mib.alloc_rx_buff_failed),
1174 STAT_GENET_SOFT_MIB("tx_dma_failed", mib.tx_dma_failed),
1175 STAT_GENET_SOFT_MIB("tx_realloc_tsb", mib.tx_realloc_tsb),
1176 STAT_GENET_SOFT_MIB("tx_realloc_tsb_failed",
1177 mib.tx_realloc_tsb_failed),
1178 /* Per TX queues */
1179 STAT_GENET_Q(0),
1180 STAT_GENET_Q(1),
1181 STAT_GENET_Q(2),
1182 STAT_GENET_Q(3),
1183 STAT_GENET_Q(4),
1184 };
1185
1186 #define BCMGENET_STATS_LEN ARRAY_SIZE(bcmgenet_gstrings_stats)
1187
1188 #define BCMGENET_STATS64_ADD(stats, m, v) \
1189 do { \
1190 u64_stats_update_begin(&stats->syncp); \
1191 u64_stats_add(&stats->m, v); \
1192 u64_stats_update_end(&stats->syncp); \
1193 } while (0)
1194
1195 #define BCMGENET_STATS64_INC(stats, m) \
1196 do { \
1197 u64_stats_update_begin(&stats->syncp); \
1198 u64_stats_inc(&stats->m); \
1199 u64_stats_update_end(&stats->syncp); \
1200 } while (0)
1201
bcmgenet_get_drvinfo(struct net_device * dev,struct ethtool_drvinfo * info)1202 static void bcmgenet_get_drvinfo(struct net_device *dev,
1203 struct ethtool_drvinfo *info)
1204 {
1205 strscpy(info->driver, "bcmgenet", sizeof(info->driver));
1206 }
1207
bcmgenet_get_sset_count(struct net_device * dev,int string_set)1208 static int bcmgenet_get_sset_count(struct net_device *dev, int string_set)
1209 {
1210 switch (string_set) {
1211 case ETH_SS_STATS:
1212 return BCMGENET_STATS_LEN;
1213 default:
1214 return -EOPNOTSUPP;
1215 }
1216 }
1217
bcmgenet_get_strings(struct net_device * dev,u32 stringset,u8 * data)1218 static void bcmgenet_get_strings(struct net_device *dev, u32 stringset,
1219 u8 *data)
1220 {
1221 const char *str;
1222 int i;
1223
1224 switch (stringset) {
1225 case ETH_SS_STATS:
1226 for (i = 0; i < BCMGENET_STATS_LEN; i++) {
1227 str = bcmgenet_gstrings_stats[i].stat_string;
1228 ethtool_puts(&data, str);
1229 }
1230 break;
1231 }
1232 }
1233
bcmgenet_update_stat_misc(struct bcmgenet_priv * priv,u16 offset)1234 static u32 bcmgenet_update_stat_misc(struct bcmgenet_priv *priv, u16 offset)
1235 {
1236 u16 new_offset;
1237 u32 val;
1238
1239 switch (offset) {
1240 case UMAC_RBUF_OVFL_CNT_V1:
1241 if (GENET_IS_V2(priv))
1242 new_offset = RBUF_OVFL_CNT_V2;
1243 else
1244 new_offset = RBUF_OVFL_CNT_V3PLUS;
1245
1246 val = bcmgenet_rbuf_readl(priv, new_offset);
1247 /* clear if overflowed */
1248 if (val == ~0)
1249 bcmgenet_rbuf_writel(priv, 0, new_offset);
1250 break;
1251 case UMAC_RBUF_ERR_CNT_V1:
1252 if (GENET_IS_V2(priv))
1253 new_offset = RBUF_ERR_CNT_V2;
1254 else
1255 new_offset = RBUF_ERR_CNT_V3PLUS;
1256
1257 val = bcmgenet_rbuf_readl(priv, new_offset);
1258 /* clear if overflowed */
1259 if (val == ~0)
1260 bcmgenet_rbuf_writel(priv, 0, new_offset);
1261 break;
1262 default:
1263 val = bcmgenet_umac_readl(priv, offset);
1264 /* clear if overflowed */
1265 if (val == ~0)
1266 bcmgenet_umac_writel(priv, 0, offset);
1267 break;
1268 }
1269
1270 return val;
1271 }
1272
bcmgenet_update_mib_counters(struct bcmgenet_priv * priv)1273 static void bcmgenet_update_mib_counters(struct bcmgenet_priv *priv)
1274 {
1275 int i, j = 0;
1276
1277 for (i = 0; i < BCMGENET_STATS_LEN; i++) {
1278 const struct bcmgenet_stats *s;
1279 u8 offset = 0;
1280 u32 val = 0;
1281 char *p;
1282
1283 s = &bcmgenet_gstrings_stats[i];
1284 switch (s->type) {
1285 case BCMGENET_STAT_RTNL:
1286 case BCMGENET_STAT_SOFT:
1287 case BCMGENET_STAT_SOFT64:
1288 continue;
1289 case BCMGENET_STAT_RUNT:
1290 offset += BCMGENET_STAT_OFFSET;
1291 fallthrough;
1292 case BCMGENET_STAT_MIB_TX:
1293 offset += BCMGENET_STAT_OFFSET;
1294 fallthrough;
1295 case BCMGENET_STAT_MIB_RX:
1296 val = bcmgenet_umac_readl(priv,
1297 UMAC_MIB_START + j + offset);
1298 offset = 0; /* Reset Offset */
1299 break;
1300 case BCMGENET_STAT_MISC:
1301 if (GENET_IS_V1(priv)) {
1302 val = bcmgenet_umac_readl(priv, s->reg_offset);
1303 /* clear if overflowed */
1304 if (val == ~0)
1305 bcmgenet_umac_writel(priv, 0,
1306 s->reg_offset);
1307 } else {
1308 val = bcmgenet_update_stat_misc(priv,
1309 s->reg_offset);
1310 }
1311 break;
1312 }
1313
1314 j += s->stat_sizeof;
1315 p = (char *)priv + s->stat_offset;
1316 *(u32 *)p = val;
1317 }
1318 }
1319
bcmgenet_get_ethtool_stats(struct net_device * dev,struct ethtool_stats * stats,u64 * data)1320 static void bcmgenet_get_ethtool_stats(struct net_device *dev,
1321 struct ethtool_stats *stats,
1322 u64 *data)
1323 {
1324 struct bcmgenet_priv *priv = netdev_priv(dev);
1325 struct rtnl_link_stats64 stats64;
1326 struct u64_stats_sync *syncp;
1327 unsigned int start;
1328 int i;
1329
1330 if (netif_running(dev))
1331 bcmgenet_update_mib_counters(priv);
1332
1333 dev_get_stats(dev, &stats64);
1334
1335 for (i = 0; i < BCMGENET_STATS_LEN; i++) {
1336 const struct bcmgenet_stats *s;
1337 char *p;
1338
1339 s = &bcmgenet_gstrings_stats[i];
1340 p = (char *)priv;
1341
1342 if (s->type == BCMGENET_STAT_SOFT64) {
1343 syncp = (struct u64_stats_sync *)(p + s->syncp_offset);
1344 do {
1345 start = u64_stats_fetch_begin(syncp);
1346 data[i] = u64_stats_read((u64_stats_t *)(p + s->stat_offset));
1347 } while (u64_stats_fetch_retry(syncp, start));
1348 } else {
1349 if (s->type == BCMGENET_STAT_RTNL)
1350 p = (char *)&stats64;
1351
1352 p += s->stat_offset;
1353 if (s->stat_sizeof == sizeof(u64))
1354 data[i] = *(u64 *)p;
1355 else
1356 data[i] = *(u32 *)p;
1357 }
1358 }
1359 }
1360
bcmgenet_eee_enable_set(struct net_device * dev,bool enable)1361 void bcmgenet_eee_enable_set(struct net_device *dev, bool enable)
1362 {
1363 struct bcmgenet_priv *priv = netdev_priv(dev);
1364 u32 off = priv->hw_params->tbuf_offset + TBUF_ENERGY_CTRL;
1365 u32 reg;
1366
1367 if (enable && !priv->clk_eee_enabled) {
1368 clk_prepare_enable(priv->clk_eee);
1369 priv->clk_eee_enabled = true;
1370 }
1371
1372 reg = bcmgenet_umac_readl(priv, UMAC_EEE_CTRL);
1373 if (enable)
1374 reg |= EEE_EN;
1375 else
1376 reg &= ~EEE_EN;
1377 bcmgenet_umac_writel(priv, reg, UMAC_EEE_CTRL);
1378
1379 /* Enable EEE and switch to a 27Mhz clock automatically */
1380 reg = bcmgenet_readl(priv->base + off);
1381 if (enable)
1382 reg |= TBUF_EEE_EN | TBUF_PM_EN;
1383 else
1384 reg &= ~(TBUF_EEE_EN | TBUF_PM_EN);
1385 bcmgenet_writel(reg, priv->base + off);
1386
1387 /* RBUF EEE/PM can break the RX path on GENET. Keep it disabled. */
1388 reg = bcmgenet_rbuf_readl(priv, RBUF_ENERGY_CTRL);
1389 if (reg & (RBUF_EEE_EN | RBUF_PM_EN)) {
1390 reg &= ~(RBUF_EEE_EN | RBUF_PM_EN);
1391 bcmgenet_rbuf_writel(priv, reg, RBUF_ENERGY_CTRL);
1392 }
1393
1394 if (!enable && priv->clk_eee_enabled) {
1395 clk_disable_unprepare(priv->clk_eee);
1396 priv->clk_eee_enabled = false;
1397 }
1398
1399 }
1400
bcmgenet_get_eee(struct net_device * dev,struct ethtool_keee * e)1401 static int bcmgenet_get_eee(struct net_device *dev, struct ethtool_keee *e)
1402 {
1403 struct bcmgenet_priv *priv = netdev_priv(dev);
1404 int ret;
1405
1406 if (GENET_IS_V1(priv))
1407 return -EOPNOTSUPP;
1408
1409 if (!dev->phydev)
1410 return -ENODEV;
1411
1412 ret = phy_ethtool_get_eee(dev->phydev, e);
1413 if (ret)
1414 return ret;
1415
1416 /* tx_lpi_timer is maintained by the MAC hardware register; the
1417 * PHY-level eee_cfg timer is not set for GENET.
1418 */
1419 e->tx_lpi_timer = bcmgenet_umac_readl(priv, UMAC_EEE_LPI_TIMER);
1420
1421 return 0;
1422 }
1423
bcmgenet_set_eee(struct net_device * dev,struct ethtool_keee * e)1424 static int bcmgenet_set_eee(struct net_device *dev, struct ethtool_keee *e)
1425 {
1426 struct bcmgenet_priv *priv = netdev_priv(dev);
1427
1428 if (GENET_IS_V1(priv))
1429 return -EOPNOTSUPP;
1430
1431 if (!dev->phydev)
1432 return -ENODEV;
1433
1434 bcmgenet_umac_writel(priv, e->tx_lpi_timer, UMAC_EEE_LPI_TIMER);
1435
1436 return phy_ethtool_set_eee(dev->phydev, e);
1437 }
1438
bcmgenet_validate_flow(struct net_device * dev,struct ethtool_rxnfc * cmd)1439 static int bcmgenet_validate_flow(struct net_device *dev,
1440 struct ethtool_rxnfc *cmd)
1441 {
1442 struct ethtool_usrip4_spec *l4_mask;
1443 struct ethhdr *eth_mask;
1444
1445 if (cmd->fs.location >= MAX_NUM_OF_FS_RULES &&
1446 cmd->fs.location != RX_CLS_LOC_ANY) {
1447 netdev_err(dev, "rxnfc: Invalid location (%d)\n",
1448 cmd->fs.location);
1449 return -EINVAL;
1450 }
1451
1452 switch (cmd->fs.flow_type & ~(FLOW_EXT | FLOW_MAC_EXT)) {
1453 case IP_USER_FLOW:
1454 l4_mask = &cmd->fs.m_u.usr_ip4_spec;
1455 /* don't allow mask which isn't valid */
1456 if (VALIDATE_MASK(l4_mask->ip4src) ||
1457 VALIDATE_MASK(l4_mask->ip4dst) ||
1458 VALIDATE_MASK(l4_mask->l4_4_bytes) ||
1459 VALIDATE_MASK(l4_mask->proto) ||
1460 VALIDATE_MASK(l4_mask->ip_ver) ||
1461 VALIDATE_MASK(l4_mask->tos)) {
1462 netdev_err(dev, "rxnfc: Unsupported mask\n");
1463 return -EINVAL;
1464 }
1465 break;
1466 case ETHER_FLOW:
1467 eth_mask = &cmd->fs.m_u.ether_spec;
1468 /* don't allow mask which isn't valid */
1469 if (VALIDATE_MASK(eth_mask->h_dest) ||
1470 VALIDATE_MASK(eth_mask->h_source) ||
1471 VALIDATE_MASK(eth_mask->h_proto)) {
1472 netdev_err(dev, "rxnfc: Unsupported mask\n");
1473 return -EINVAL;
1474 }
1475 break;
1476 default:
1477 netdev_err(dev, "rxnfc: Unsupported flow type (0x%x)\n",
1478 cmd->fs.flow_type);
1479 return -EINVAL;
1480 }
1481
1482 if ((cmd->fs.flow_type & FLOW_EXT)) {
1483 /* don't allow mask which isn't valid */
1484 if (VALIDATE_MASK(cmd->fs.m_ext.vlan_etype) ||
1485 VALIDATE_MASK(cmd->fs.m_ext.vlan_tci)) {
1486 netdev_err(dev, "rxnfc: Unsupported mask\n");
1487 return -EINVAL;
1488 }
1489 if (cmd->fs.m_ext.data[0] || cmd->fs.m_ext.data[1]) {
1490 netdev_err(dev, "rxnfc: user-def not supported\n");
1491 return -EINVAL;
1492 }
1493 }
1494
1495 if ((cmd->fs.flow_type & FLOW_MAC_EXT)) {
1496 /* don't allow mask which isn't valid */
1497 if (VALIDATE_MASK(cmd->fs.m_ext.h_dest)) {
1498 netdev_err(dev, "rxnfc: Unsupported mask\n");
1499 return -EINVAL;
1500 }
1501 }
1502
1503 return 0;
1504 }
1505
bcmgenet_insert_flow(struct net_device * dev,struct ethtool_rxnfc * cmd)1506 static int bcmgenet_insert_flow(struct net_device *dev,
1507 struct ethtool_rxnfc *cmd)
1508 {
1509 struct bcmgenet_priv *priv = netdev_priv(dev);
1510 struct bcmgenet_rxnfc_rule *loc_rule;
1511 int err, i;
1512
1513 if (priv->hw_params->hfb_filter_size < 128) {
1514 netdev_err(dev, "rxnfc: Not supported by this device\n");
1515 return -EINVAL;
1516 }
1517
1518 if (cmd->fs.ring_cookie > priv->hw_params->rx_queues &&
1519 cmd->fs.ring_cookie != RX_CLS_FLOW_WAKE &&
1520 cmd->fs.ring_cookie != RX_CLS_FLOW_DISC) {
1521 netdev_err(dev, "rxnfc: Unsupported action (%llu)\n",
1522 cmd->fs.ring_cookie);
1523 return -EINVAL;
1524 }
1525
1526 err = bcmgenet_validate_flow(dev, cmd);
1527 if (err)
1528 return err;
1529
1530 if (cmd->fs.location == RX_CLS_LOC_ANY) {
1531 list_for_each_entry(loc_rule, &priv->rxnfc_list, list) {
1532 cmd->fs.location = loc_rule->fs.location;
1533 err = memcmp(&loc_rule->fs, &cmd->fs,
1534 sizeof(struct ethtool_rx_flow_spec));
1535 if (!err)
1536 /* rule exists so return current location */
1537 return 0;
1538 }
1539 for (i = 0; i < MAX_NUM_OF_FS_RULES; i++) {
1540 loc_rule = &priv->rxnfc_rules[i];
1541 if (loc_rule->state == BCMGENET_RXNFC_STATE_UNUSED) {
1542 cmd->fs.location = i;
1543 break;
1544 }
1545 }
1546 if (i == MAX_NUM_OF_FS_RULES) {
1547 cmd->fs.location = RX_CLS_LOC_ANY;
1548 return -ENOSPC;
1549 }
1550 } else {
1551 loc_rule = &priv->rxnfc_rules[cmd->fs.location];
1552 }
1553 if (loc_rule->state == BCMGENET_RXNFC_STATE_ENABLED)
1554 bcmgenet_hfb_disable_filter(priv, cmd->fs.location + 1);
1555 if (loc_rule->state != BCMGENET_RXNFC_STATE_UNUSED) {
1556 list_del(&loc_rule->list);
1557 bcmgenet_hfb_clear_filter(priv, cmd->fs.location + 1);
1558 }
1559 loc_rule->state = BCMGENET_RXNFC_STATE_UNUSED;
1560 memcpy(&loc_rule->fs, &cmd->fs,
1561 sizeof(struct ethtool_rx_flow_spec));
1562
1563 bcmgenet_hfb_create_rxnfc_filter(priv, loc_rule);
1564
1565 list_add_tail(&loc_rule->list, &priv->rxnfc_list);
1566
1567 return 0;
1568 }
1569
bcmgenet_delete_flow(struct net_device * dev,struct ethtool_rxnfc * cmd)1570 static int bcmgenet_delete_flow(struct net_device *dev,
1571 struct ethtool_rxnfc *cmd)
1572 {
1573 struct bcmgenet_priv *priv = netdev_priv(dev);
1574 struct bcmgenet_rxnfc_rule *rule;
1575 int err = 0;
1576
1577 if (cmd->fs.location >= MAX_NUM_OF_FS_RULES)
1578 return -EINVAL;
1579
1580 rule = &priv->rxnfc_rules[cmd->fs.location];
1581 if (rule->state == BCMGENET_RXNFC_STATE_UNUSED) {
1582 err = -ENOENT;
1583 goto out;
1584 }
1585
1586 if (rule->state == BCMGENET_RXNFC_STATE_ENABLED)
1587 bcmgenet_hfb_disable_filter(priv, cmd->fs.location + 1);
1588 if (rule->state != BCMGENET_RXNFC_STATE_UNUSED) {
1589 list_del(&rule->list);
1590 bcmgenet_hfb_clear_filter(priv, cmd->fs.location + 1);
1591 }
1592 rule->state = BCMGENET_RXNFC_STATE_UNUSED;
1593 memset(&rule->fs, 0, sizeof(struct ethtool_rx_flow_spec));
1594
1595 out:
1596 return err;
1597 }
1598
bcmgenet_set_rxnfc(struct net_device * dev,struct ethtool_rxnfc * cmd)1599 static int bcmgenet_set_rxnfc(struct net_device *dev, struct ethtool_rxnfc *cmd)
1600 {
1601 struct bcmgenet_priv *priv = netdev_priv(dev);
1602 int err = 0;
1603
1604 switch (cmd->cmd) {
1605 case ETHTOOL_SRXCLSRLINS:
1606 err = bcmgenet_insert_flow(dev, cmd);
1607 break;
1608 case ETHTOOL_SRXCLSRLDEL:
1609 err = bcmgenet_delete_flow(dev, cmd);
1610 break;
1611 default:
1612 netdev_warn(priv->dev, "Unsupported ethtool command. (%d)\n",
1613 cmd->cmd);
1614 return -EINVAL;
1615 }
1616
1617 return err;
1618 }
1619
bcmgenet_get_flow(struct net_device * dev,struct ethtool_rxnfc * cmd,int loc)1620 static int bcmgenet_get_flow(struct net_device *dev, struct ethtool_rxnfc *cmd,
1621 int loc)
1622 {
1623 struct bcmgenet_priv *priv = netdev_priv(dev);
1624 struct bcmgenet_rxnfc_rule *rule;
1625 int err = 0;
1626
1627 if (loc < 0 || loc >= MAX_NUM_OF_FS_RULES)
1628 return -EINVAL;
1629
1630 rule = &priv->rxnfc_rules[loc];
1631 if (rule->state == BCMGENET_RXNFC_STATE_UNUSED)
1632 err = -ENOENT;
1633 else
1634 memcpy(&cmd->fs, &rule->fs,
1635 sizeof(struct ethtool_rx_flow_spec));
1636
1637 return err;
1638 }
1639
bcmgenet_get_num_flows(struct bcmgenet_priv * priv)1640 static int bcmgenet_get_num_flows(struct bcmgenet_priv *priv)
1641 {
1642 struct list_head *pos;
1643 int res = 0;
1644
1645 list_for_each(pos, &priv->rxnfc_list)
1646 res++;
1647
1648 return res;
1649 }
1650
bcmgenet_get_rx_ring_count(struct net_device * dev)1651 static u32 bcmgenet_get_rx_ring_count(struct net_device *dev)
1652 {
1653 struct bcmgenet_priv *priv = netdev_priv(dev);
1654
1655 return priv->hw_params->rx_queues ?: 1;
1656 }
1657
bcmgenet_get_rxnfc(struct net_device * dev,struct ethtool_rxnfc * cmd,u32 * rule_locs)1658 static int bcmgenet_get_rxnfc(struct net_device *dev, struct ethtool_rxnfc *cmd,
1659 u32 *rule_locs)
1660 {
1661 struct bcmgenet_priv *priv = netdev_priv(dev);
1662 struct bcmgenet_rxnfc_rule *rule;
1663 int err = 0;
1664 int i = 0;
1665
1666 switch (cmd->cmd) {
1667 case ETHTOOL_GRXCLSRLCNT:
1668 cmd->rule_cnt = bcmgenet_get_num_flows(priv);
1669 cmd->data = MAX_NUM_OF_FS_RULES | RX_CLS_LOC_SPECIAL;
1670 break;
1671 case ETHTOOL_GRXCLSRULE:
1672 err = bcmgenet_get_flow(dev, cmd, cmd->fs.location);
1673 break;
1674 case ETHTOOL_GRXCLSRLALL:
1675 list_for_each_entry(rule, &priv->rxnfc_list, list)
1676 if (i < cmd->rule_cnt)
1677 rule_locs[i++] = rule->fs.location;
1678 cmd->rule_cnt = i;
1679 cmd->data = MAX_NUM_OF_FS_RULES;
1680 break;
1681 default:
1682 err = -EOPNOTSUPP;
1683 break;
1684 }
1685
1686 return err;
1687 }
1688
1689 /* standard ethtool support functions. */
1690 static const struct ethtool_ops bcmgenet_ethtool_ops = {
1691 .supported_coalesce_params = ETHTOOL_COALESCE_RX_USECS |
1692 ETHTOOL_COALESCE_MAX_FRAMES |
1693 ETHTOOL_COALESCE_USE_ADAPTIVE_RX,
1694 .begin = bcmgenet_begin,
1695 .complete = bcmgenet_complete,
1696 .get_strings = bcmgenet_get_strings,
1697 .get_sset_count = bcmgenet_get_sset_count,
1698 .get_ethtool_stats = bcmgenet_get_ethtool_stats,
1699 .get_drvinfo = bcmgenet_get_drvinfo,
1700 .get_link = ethtool_op_get_link,
1701 .get_msglevel = bcmgenet_get_msglevel,
1702 .set_msglevel = bcmgenet_set_msglevel,
1703 .get_wol = bcmgenet_get_wol,
1704 .set_wol = bcmgenet_set_wol,
1705 .get_eee = bcmgenet_get_eee,
1706 .set_eee = bcmgenet_set_eee,
1707 .nway_reset = phy_ethtool_nway_reset,
1708 .get_coalesce = bcmgenet_get_coalesce,
1709 .set_coalesce = bcmgenet_set_coalesce,
1710 .get_link_ksettings = bcmgenet_get_link_ksettings,
1711 .set_link_ksettings = bcmgenet_set_link_ksettings,
1712 .get_ts_info = ethtool_op_get_ts_info,
1713 .get_rxnfc = bcmgenet_get_rxnfc,
1714 .set_rxnfc = bcmgenet_set_rxnfc,
1715 .get_rx_ring_count = bcmgenet_get_rx_ring_count,
1716 .get_pauseparam = bcmgenet_get_pauseparam,
1717 .set_pauseparam = bcmgenet_set_pauseparam,
1718 };
1719
1720 /* Power down the unimac, based on mode. */
bcmgenet_power_down(struct bcmgenet_priv * priv,enum bcmgenet_power_mode mode)1721 static int bcmgenet_power_down(struct bcmgenet_priv *priv,
1722 enum bcmgenet_power_mode mode)
1723 {
1724 int ret = 0;
1725 u32 reg;
1726
1727 switch (mode) {
1728 case GENET_POWER_CABLE_SENSE:
1729 phy_detach(priv->dev->phydev);
1730 break;
1731
1732 case GENET_POWER_WOL_MAGIC:
1733 ret = bcmgenet_wol_power_down_cfg(priv, mode);
1734 break;
1735
1736 case GENET_POWER_PASSIVE:
1737 /* Power down LED */
1738 if (bcmgenet_has_ext(priv)) {
1739 reg = bcmgenet_ext_readl(priv, EXT_EXT_PWR_MGMT);
1740 if (GENET_IS_V5(priv) && !bcmgenet_has_ephy_16nm(priv))
1741 reg |= EXT_PWR_DOWN_PHY_EN |
1742 EXT_PWR_DOWN_PHY_RD |
1743 EXT_PWR_DOWN_PHY_SD |
1744 EXT_PWR_DOWN_PHY_RX |
1745 EXT_PWR_DOWN_PHY_TX |
1746 EXT_IDDQ_GLBL_PWR;
1747 else
1748 reg |= EXT_PWR_DOWN_PHY;
1749
1750 reg |= (EXT_PWR_DOWN_DLL | EXT_PWR_DOWN_BIAS);
1751 bcmgenet_ext_writel(priv, reg, EXT_EXT_PWR_MGMT);
1752
1753 bcmgenet_phy_power_set(priv->dev, false);
1754 }
1755 break;
1756 default:
1757 break;
1758 }
1759
1760 return ret;
1761 }
1762
bcmgenet_power_up(struct bcmgenet_priv * priv,enum bcmgenet_power_mode mode)1763 static int bcmgenet_power_up(struct bcmgenet_priv *priv,
1764 enum bcmgenet_power_mode mode)
1765 {
1766 int ret = 0;
1767 u32 reg;
1768
1769 switch (mode) {
1770 case GENET_POWER_PASSIVE:
1771 if (!bcmgenet_has_ext(priv))
1772 break;
1773
1774 reg = bcmgenet_ext_readl(priv, EXT_EXT_PWR_MGMT);
1775 reg &= ~(EXT_PWR_DOWN_DLL | EXT_PWR_DOWN_BIAS |
1776 EXT_ENERGY_DET_MASK);
1777 if (GENET_IS_V5(priv) && !bcmgenet_has_ephy_16nm(priv)) {
1778 reg &= ~(EXT_PWR_DOWN_PHY_EN |
1779 EXT_PWR_DOWN_PHY_RD |
1780 EXT_PWR_DOWN_PHY_SD |
1781 EXT_PWR_DOWN_PHY_RX |
1782 EXT_PWR_DOWN_PHY_TX |
1783 EXT_IDDQ_GLBL_PWR);
1784 reg |= EXT_PHY_RESET;
1785 bcmgenet_ext_writel(priv, reg, EXT_EXT_PWR_MGMT);
1786 mdelay(1);
1787
1788 reg &= ~EXT_PHY_RESET;
1789 } else {
1790 reg &= ~EXT_PWR_DOWN_PHY;
1791 reg |= EXT_PWR_DN_EN_LD;
1792 }
1793 bcmgenet_ext_writel(priv, reg, EXT_EXT_PWR_MGMT);
1794 bcmgenet_phy_power_set(priv->dev, true);
1795 break;
1796
1797 case GENET_POWER_CABLE_SENSE:
1798 if (!bcmgenet_has_ext(priv))
1799 break;
1800
1801 /* enable APD */
1802 if (!GENET_IS_V5(priv)) {
1803 reg = bcmgenet_ext_readl(priv, EXT_EXT_PWR_MGMT);
1804 reg |= EXT_PWR_DN_EN_LD;
1805 bcmgenet_ext_writel(priv, reg, EXT_EXT_PWR_MGMT);
1806 }
1807 break;
1808 case GENET_POWER_WOL_MAGIC:
1809 ret = bcmgenet_wol_power_up_cfg(priv, mode);
1810 break;
1811 default:
1812 break;
1813 }
1814
1815 return ret;
1816 }
1817
bcmgenet_get_txcb(struct bcmgenet_priv * priv,struct bcmgenet_tx_ring * ring)1818 static struct enet_cb *bcmgenet_get_txcb(struct bcmgenet_priv *priv,
1819 struct bcmgenet_tx_ring *ring)
1820 {
1821 struct enet_cb *tx_cb_ptr;
1822
1823 tx_cb_ptr = ring->cbs;
1824 tx_cb_ptr += ring->write_ptr - ring->cb_ptr;
1825
1826 /* Advancing local write pointer */
1827 if (ring->write_ptr == ring->end_ptr)
1828 ring->write_ptr = ring->cb_ptr;
1829 else
1830 ring->write_ptr++;
1831
1832 return tx_cb_ptr;
1833 }
1834
bcmgenet_put_txcb(struct bcmgenet_priv * priv,struct bcmgenet_tx_ring * ring)1835 static struct enet_cb *bcmgenet_put_txcb(struct bcmgenet_priv *priv,
1836 struct bcmgenet_tx_ring *ring)
1837 {
1838 struct enet_cb *tx_cb_ptr;
1839
1840 /* Rewinding local write pointer */
1841 if (ring->write_ptr == ring->cb_ptr)
1842 ring->write_ptr = ring->end_ptr;
1843 else
1844 ring->write_ptr--;
1845
1846 tx_cb_ptr = ring->cbs;
1847 tx_cb_ptr += ring->write_ptr - ring->cb_ptr;
1848
1849 return tx_cb_ptr;
1850 }
1851
bcmgenet_rx_ring_int_disable(struct bcmgenet_rx_ring * ring)1852 static inline void bcmgenet_rx_ring_int_disable(struct bcmgenet_rx_ring *ring)
1853 {
1854 bcmgenet_intrl2_1_writel(ring->priv,
1855 1 << (UMAC_IRQ1_RX_INTR_SHIFT + ring->index),
1856 INTRL2_CPU_MASK_SET);
1857 }
1858
bcmgenet_rx_ring_int_enable(struct bcmgenet_rx_ring * ring)1859 static inline void bcmgenet_rx_ring_int_enable(struct bcmgenet_rx_ring *ring)
1860 {
1861 bcmgenet_intrl2_1_writel(ring->priv,
1862 1 << (UMAC_IRQ1_RX_INTR_SHIFT + ring->index),
1863 INTRL2_CPU_MASK_CLEAR);
1864 }
1865
bcmgenet_tx_ring_int_enable(struct bcmgenet_tx_ring * ring)1866 static inline void bcmgenet_tx_ring_int_enable(struct bcmgenet_tx_ring *ring)
1867 {
1868 bcmgenet_intrl2_1_writel(ring->priv, 1 << ring->index,
1869 INTRL2_CPU_MASK_CLEAR);
1870 }
1871
bcmgenet_tx_ring_int_disable(struct bcmgenet_tx_ring * ring)1872 static inline void bcmgenet_tx_ring_int_disable(struct bcmgenet_tx_ring *ring)
1873 {
1874 bcmgenet_intrl2_1_writel(ring->priv, 1 << ring->index,
1875 INTRL2_CPU_MASK_SET);
1876 }
1877
1878 /* Simple helper to free a transmit control block's resources
1879 * Returns an skb when the last transmit control block associated with the
1880 * skb is freed. The skb should be freed by the caller if necessary.
1881 */
bcmgenet_free_tx_cb(struct device * dev,struct enet_cb * cb)1882 static struct sk_buff *bcmgenet_free_tx_cb(struct device *dev,
1883 struct enet_cb *cb)
1884 {
1885 struct sk_buff *skb;
1886
1887 skb = cb->skb;
1888
1889 if (skb) {
1890 cb->skb = NULL;
1891 if (cb == GENET_CB(skb)->first_cb)
1892 dma_unmap_single(dev, dma_unmap_addr(cb, dma_addr),
1893 dma_unmap_len(cb, dma_len),
1894 DMA_TO_DEVICE);
1895 else
1896 dma_unmap_page(dev, dma_unmap_addr(cb, dma_addr),
1897 dma_unmap_len(cb, dma_len),
1898 DMA_TO_DEVICE);
1899 dma_unmap_addr_set(cb, dma_addr, 0);
1900
1901 if (cb == GENET_CB(skb)->last_cb)
1902 return skb;
1903
1904 } else if (dma_unmap_addr(cb, dma_addr)) {
1905 dma_unmap_page(dev,
1906 dma_unmap_addr(cb, dma_addr),
1907 dma_unmap_len(cb, dma_len),
1908 DMA_TO_DEVICE);
1909 dma_unmap_addr_set(cb, dma_addr, 0);
1910 }
1911
1912 return NULL;
1913 }
1914
1915 /* Simple helper to free a receive control block's resources */
bcmgenet_free_rx_cb(struct enet_cb * cb,struct page_pool * pool)1916 static void bcmgenet_free_rx_cb(struct enet_cb *cb,
1917 struct page_pool *pool)
1918 {
1919 if (cb->rx_page) {
1920 page_pool_put_full_page(pool, cb->rx_page, false);
1921 cb->rx_page = NULL;
1922 }
1923 }
1924
1925 /* Unlocked version of the reclaim routine */
__bcmgenet_tx_reclaim(struct net_device * dev,struct bcmgenet_tx_ring * ring)1926 static unsigned int __bcmgenet_tx_reclaim(struct net_device *dev,
1927 struct bcmgenet_tx_ring *ring)
1928 {
1929 struct bcmgenet_tx_stats64 *stats = &ring->stats64;
1930 struct bcmgenet_priv *priv = netdev_priv(dev);
1931 unsigned int txbds_processed = 0;
1932 unsigned int bytes_compl = 0;
1933 unsigned int pkts_compl = 0;
1934 unsigned int txbds_ready;
1935 unsigned int c_index;
1936 struct sk_buff *skb;
1937
1938 /* Clear status before servicing to reduce spurious interrupts */
1939 bcmgenet_intrl2_1_writel(priv, (1 << ring->index), INTRL2_CPU_CLEAR);
1940
1941 /* Compute how many buffers are transmitted since last xmit call */
1942 c_index = bcmgenet_tdma_ring_readl(priv, ring->index, TDMA_CONS_INDEX)
1943 & DMA_C_INDEX_MASK;
1944 txbds_ready = (c_index - ring->c_index) & DMA_C_INDEX_MASK;
1945
1946 netif_dbg(priv, tx_done, dev,
1947 "%s ring=%d old_c_index=%u c_index=%u txbds_ready=%u\n",
1948 __func__, ring->index, ring->c_index, c_index, txbds_ready);
1949
1950 /* Reclaim transmitted buffers */
1951 while (txbds_processed < txbds_ready) {
1952 skb = bcmgenet_free_tx_cb(&priv->pdev->dev,
1953 &priv->tx_cbs[ring->clean_ptr]);
1954 if (skb) {
1955 pkts_compl++;
1956 bytes_compl += GENET_CB(skb)->bytes_sent;
1957 dev_consume_skb_any(skb);
1958 }
1959
1960 txbds_processed++;
1961 if (likely(ring->clean_ptr < ring->end_ptr))
1962 ring->clean_ptr++;
1963 else
1964 ring->clean_ptr = ring->cb_ptr;
1965 }
1966
1967 ring->free_bds += txbds_processed;
1968 ring->c_index = c_index;
1969
1970 u64_stats_update_begin(&stats->syncp);
1971 u64_stats_add(&stats->packets, pkts_compl);
1972 u64_stats_add(&stats->bytes, bytes_compl);
1973 u64_stats_update_end(&stats->syncp);
1974
1975 netdev_tx_completed_queue(netdev_get_tx_queue(dev, ring->index),
1976 pkts_compl, bytes_compl);
1977
1978 return txbds_processed;
1979 }
1980
bcmgenet_tx_reclaim(struct net_device * dev,struct bcmgenet_tx_ring * ring,bool all)1981 static unsigned int bcmgenet_tx_reclaim(struct net_device *dev,
1982 struct bcmgenet_tx_ring *ring,
1983 bool all)
1984 {
1985 struct bcmgenet_priv *priv = netdev_priv(dev);
1986 struct device *kdev = &priv->pdev->dev;
1987 unsigned int released, drop, wr_ptr;
1988 struct enet_cb *cb_ptr;
1989 struct sk_buff *skb;
1990
1991 spin_lock_bh(&ring->lock);
1992 released = __bcmgenet_tx_reclaim(dev, ring);
1993 if (all) {
1994 skb = NULL;
1995 drop = (ring->prod_index - ring->c_index) & DMA_C_INDEX_MASK;
1996 released += drop;
1997 ring->prod_index = ring->c_index & DMA_C_INDEX_MASK;
1998 ring->free_bds += drop;
1999 while (drop--) {
2000 cb_ptr = bcmgenet_put_txcb(priv, ring);
2001 skb = cb_ptr->skb;
2002 bcmgenet_free_tx_cb(kdev, cb_ptr);
2003 if (skb && cb_ptr == GENET_CB(skb)->first_cb) {
2004 dev_consume_skb_any(skb);
2005 skb = NULL;
2006 }
2007 }
2008 if (skb)
2009 dev_consume_skb_any(skb);
2010 netdev_tx_reset_queue(netdev_get_tx_queue(dev, ring->index));
2011 bcmgenet_tdma_ring_writel(priv, ring->index,
2012 ring->prod_index, TDMA_PROD_INDEX);
2013 wr_ptr = ring->write_ptr * WORDS_PER_BD(priv);
2014 bcmgenet_tdma_ring_writel(priv, ring->index, wr_ptr,
2015 TDMA_WRITE_PTR);
2016 }
2017 spin_unlock_bh(&ring->lock);
2018
2019 return released;
2020 }
2021
bcmgenet_tx_poll(struct napi_struct * napi,int budget)2022 static int bcmgenet_tx_poll(struct napi_struct *napi, int budget)
2023 {
2024 struct bcmgenet_tx_ring *ring =
2025 container_of(napi, struct bcmgenet_tx_ring, napi);
2026 unsigned int work_done = 0;
2027 struct netdev_queue *txq;
2028
2029 spin_lock(&ring->lock);
2030 work_done = __bcmgenet_tx_reclaim(ring->priv->dev, ring);
2031 if (ring->free_bds > (MAX_SKB_FRAGS + 1)) {
2032 txq = netdev_get_tx_queue(ring->priv->dev, ring->index);
2033 netif_tx_wake_queue(txq);
2034 }
2035 spin_unlock(&ring->lock);
2036
2037 if (work_done == 0) {
2038 napi_complete(napi);
2039 bcmgenet_tx_ring_int_enable(ring);
2040
2041 return 0;
2042 }
2043
2044 return budget;
2045 }
2046
bcmgenet_tx_reclaim_all(struct net_device * dev)2047 static void bcmgenet_tx_reclaim_all(struct net_device *dev)
2048 {
2049 struct bcmgenet_priv *priv = netdev_priv(dev);
2050 int i = 0;
2051
2052 do {
2053 bcmgenet_tx_reclaim(dev, &priv->tx_rings[i++], true);
2054 } while (i <= priv->hw_params->tx_queues && netif_is_multiqueue(dev));
2055 }
2056
2057 /* Reallocate the SKB to put enough headroom in front of it and insert
2058 * the transmit checksum offsets in the descriptors
2059 */
bcmgenet_add_tsb(struct net_device * dev,struct sk_buff * skb,struct bcmgenet_tx_ring * ring)2060 static struct sk_buff *bcmgenet_add_tsb(struct net_device *dev,
2061 struct sk_buff *skb,
2062 struct bcmgenet_tx_ring *ring)
2063 {
2064 struct bcmgenet_tx_stats64 *stats = &ring->stats64;
2065 struct bcmgenet_priv *priv = netdev_priv(dev);
2066 struct status_64 *status = NULL;
2067 struct sk_buff *new_skb;
2068 u16 offset;
2069 u8 ip_proto;
2070 __be16 ip_ver;
2071 u32 tx_csum_info;
2072
2073 if (unlikely(skb_headroom(skb) < sizeof(*status))) {
2074 /* If 64 byte status block enabled, must make sure skb has
2075 * enough headroom for us to insert 64B status block.
2076 */
2077 new_skb = skb_realloc_headroom(skb, sizeof(*status));
2078 if (!new_skb) {
2079 dev_kfree_skb_any(skb);
2080 priv->mib.tx_realloc_tsb_failed++;
2081 BCMGENET_STATS64_INC(stats, dropped);
2082 return NULL;
2083 }
2084 dev_consume_skb_any(skb);
2085 skb = new_skb;
2086 priv->mib.tx_realloc_tsb++;
2087 }
2088
2089 skb_push(skb, sizeof(*status));
2090 status = (struct status_64 *)skb->data;
2091
2092 if (skb->ip_summed == CHECKSUM_PARTIAL) {
2093 ip_ver = skb->protocol;
2094 switch (ip_ver) {
2095 case htons(ETH_P_IP):
2096 ip_proto = ip_hdr(skb)->protocol;
2097 break;
2098 case htons(ETH_P_IPV6):
2099 ip_proto = ipv6_hdr(skb)->nexthdr;
2100 break;
2101 default:
2102 /* don't use UDP flag */
2103 ip_proto = 0;
2104 break;
2105 }
2106
2107 offset = skb_checksum_start_offset(skb) - sizeof(*status);
2108 tx_csum_info = (offset << STATUS_TX_CSUM_START_SHIFT) |
2109 (offset + skb->csum_offset) |
2110 STATUS_TX_CSUM_LV;
2111
2112 /* Set the special UDP flag for UDP */
2113 if (ip_proto == IPPROTO_UDP)
2114 tx_csum_info |= STATUS_TX_CSUM_PROTO_UDP;
2115
2116 status->tx_csum_info = tx_csum_info;
2117 }
2118
2119 return skb;
2120 }
2121
bcmgenet_hide_tsb(struct sk_buff * skb)2122 static void bcmgenet_hide_tsb(struct sk_buff *skb)
2123 {
2124 __skb_pull(skb, sizeof(struct status_64));
2125 }
2126
bcmgenet_xmit(struct sk_buff * skb,struct net_device * dev)2127 static netdev_tx_t bcmgenet_xmit(struct sk_buff *skb, struct net_device *dev)
2128 {
2129 struct bcmgenet_priv *priv = netdev_priv(dev);
2130 struct device *kdev = &priv->pdev->dev;
2131 struct bcmgenet_tx_ring *ring = NULL;
2132 struct enet_cb *tx_cb_ptr;
2133 struct netdev_queue *txq;
2134 int nr_frags, index;
2135 dma_addr_t mapping;
2136 unsigned int size;
2137 skb_frag_t *frag;
2138 u32 len_stat;
2139 int ret;
2140 int i;
2141
2142 index = skb_get_queue_mapping(skb);
2143 ring = &priv->tx_rings[index];
2144 txq = netdev_get_tx_queue(dev, index);
2145
2146 nr_frags = skb_shinfo(skb)->nr_frags;
2147
2148 spin_lock(&ring->lock);
2149 if (ring->free_bds <= (nr_frags + 1)) {
2150 if (!netif_tx_queue_stopped(txq))
2151 netif_tx_stop_queue(txq);
2152 ret = NETDEV_TX_BUSY;
2153 goto out;
2154 }
2155
2156 /* We store Tx state in the control block, so the skb must not be
2157 * shared, but ether_setup() leaves IFF_TX_SKB_SHARING set.
2158 */
2159 skb = skb_share_check(skb, GFP_ATOMIC);
2160 if (!skb) {
2161 BCMGENET_STATS64_INC((&ring->stats64), dropped);
2162 ret = NETDEV_TX_OK;
2163 goto out;
2164 }
2165
2166 /* Retain how many bytes will be sent on the wire, without TSB inserted
2167 * by transmit checksum offload
2168 */
2169 GENET_CB(skb)->bytes_sent = skb->len;
2170
2171 /* add the Transmit Status Block */
2172 skb = bcmgenet_add_tsb(dev, skb, ring);
2173 if (!skb) {
2174 ret = NETDEV_TX_OK;
2175 goto out;
2176 }
2177
2178 for (i = 0; i <= nr_frags; i++) {
2179 tx_cb_ptr = bcmgenet_get_txcb(priv, ring);
2180
2181 BUG_ON(!tx_cb_ptr);
2182
2183 if (!i) {
2184 /* Transmit single SKB or head of fragment list */
2185 GENET_CB(skb)->first_cb = tx_cb_ptr;
2186 size = skb_headlen(skb);
2187 mapping = dma_map_single(kdev, skb->data, size,
2188 DMA_TO_DEVICE);
2189 } else {
2190 /* xmit fragment */
2191 frag = &skb_shinfo(skb)->frags[i - 1];
2192 size = skb_frag_size(frag);
2193 mapping = skb_frag_dma_map(kdev, frag, 0, size,
2194 DMA_TO_DEVICE);
2195 }
2196
2197 ret = dma_mapping_error(kdev, mapping);
2198 if (ret) {
2199 priv->mib.tx_dma_failed++;
2200 netif_err(priv, tx_err, dev, "Tx DMA map failed\n");
2201 ret = NETDEV_TX_OK;
2202 goto out_unmap_frags;
2203 }
2204 dma_unmap_addr_set(tx_cb_ptr, dma_addr, mapping);
2205 dma_unmap_len_set(tx_cb_ptr, dma_len, size);
2206
2207 tx_cb_ptr->skb = skb;
2208
2209 len_stat = (size << DMA_BUFLENGTH_SHIFT) |
2210 (priv->hw_params->qtag_mask << DMA_TX_QTAG_SHIFT);
2211
2212 /* Note: if we ever change from DMA_TX_APPEND_CRC below we
2213 * will need to restore software padding of "runt" packets
2214 */
2215 len_stat |= DMA_TX_APPEND_CRC;
2216
2217 if (!i) {
2218 len_stat |= DMA_SOP;
2219 if (skb->ip_summed == CHECKSUM_PARTIAL)
2220 len_stat |= DMA_TX_DO_CSUM;
2221 }
2222 if (i == nr_frags)
2223 len_stat |= DMA_EOP;
2224
2225 dmadesc_set(priv, tx_cb_ptr->bd_addr, mapping, len_stat);
2226 }
2227
2228 GENET_CB(skb)->last_cb = tx_cb_ptr;
2229
2230 bcmgenet_hide_tsb(skb);
2231 skb_tx_timestamp(skb);
2232
2233 /* Decrement total BD count and advance our write pointer */
2234 ring->free_bds -= nr_frags + 1;
2235 ring->prod_index += nr_frags + 1;
2236 ring->prod_index &= DMA_P_INDEX_MASK;
2237
2238 netdev_tx_sent_queue(txq, GENET_CB(skb)->bytes_sent);
2239
2240 if (ring->free_bds <= (MAX_SKB_FRAGS + 1))
2241 netif_tx_stop_queue(txq);
2242
2243 if (!netdev_xmit_more() || netif_xmit_stopped(txq))
2244 /* Packets are ready, update producer index */
2245 bcmgenet_tdma_ring_writel(priv, ring->index,
2246 ring->prod_index, TDMA_PROD_INDEX);
2247 out:
2248 spin_unlock(&ring->lock);
2249
2250 return ret;
2251
2252 out_unmap_frags:
2253 /* Back up for failed control block mapping */
2254 bcmgenet_put_txcb(priv, ring);
2255
2256 /* Unmap successfully mapped control blocks */
2257 while (i-- > 0) {
2258 tx_cb_ptr = bcmgenet_put_txcb(priv, ring);
2259 bcmgenet_free_tx_cb(kdev, tx_cb_ptr);
2260 }
2261
2262 dev_kfree_skb(skb);
2263 goto out;
2264 }
2265
bcmgenet_rx_refill(struct bcmgenet_rx_ring * ring,struct enet_cb * cb)2266 static int bcmgenet_rx_refill(struct bcmgenet_rx_ring *ring,
2267 struct enet_cb *cb)
2268 {
2269 struct bcmgenet_priv *priv = ring->priv;
2270 unsigned int size = GENET_RX_BUF_SIZE;
2271 unsigned int offset;
2272 dma_addr_t mapping;
2273 struct page *page;
2274
2275 page = page_pool_dev_alloc(ring->page_pool, &offset, &size);
2276 if (!page) {
2277 priv->mib.alloc_rx_buff_failed++;
2278 netif_err(priv, rx_err, priv->dev,
2279 "%s: Rx page allocation failed\n", __func__);
2280 return -ENOMEM;
2281 }
2282
2283 /* page_pool handles DMA mapping via PP_FLAG_DMA_MAP */
2284 mapping = page_pool_get_dma_addr(page) + offset;
2285 dma_sync_single_for_device(&priv->pdev->dev, mapping, RX_BUF_LENGTH,
2286 DMA_FROM_DEVICE);
2287
2288 cb->rx_page = page;
2289 cb->rx_offset = offset;
2290 cb->rx_size = size;
2291 dmadesc_set_addr(priv, cb->bd_addr, mapping);
2292
2293 return 0;
2294 }
2295
2296 /* bcmgenet_desc_rx - descriptor based rx process.
2297 * this could be called from bottom half, or from NAPI polling method.
2298 */
bcmgenet_desc_rx(struct bcmgenet_rx_ring * ring,unsigned int budget)2299 static unsigned int bcmgenet_desc_rx(struct bcmgenet_rx_ring *ring,
2300 unsigned int budget)
2301 {
2302 struct bcmgenet_rx_stats64 *stats = &ring->stats64;
2303 struct bcmgenet_priv *priv = ring->priv;
2304 struct net_device *dev = priv->dev;
2305 struct enet_cb *cb;
2306 struct sk_buff *skb;
2307 u32 dma_length_status;
2308 unsigned long dma_flag;
2309 int len;
2310 unsigned int rxpktprocessed = 0, rxpkttoprocess;
2311 unsigned int bytes_processed = 0;
2312 unsigned int p_index, mask;
2313 unsigned int discards;
2314
2315 /* Clear status before servicing to reduce spurious interrupts */
2316 mask = 1 << (UMAC_IRQ1_RX_INTR_SHIFT + ring->index);
2317 bcmgenet_intrl2_1_writel(priv, mask, INTRL2_CPU_CLEAR);
2318
2319 p_index = bcmgenet_rdma_ring_readl(priv, ring->index, RDMA_PROD_INDEX);
2320
2321 discards = (p_index >> DMA_P_INDEX_DISCARD_CNT_SHIFT) &
2322 DMA_P_INDEX_DISCARD_CNT_MASK;
2323 if (discards > ring->old_discards) {
2324 discards = discards - ring->old_discards;
2325 BCMGENET_STATS64_ADD(stats, missed, discards);
2326 ring->old_discards += discards;
2327
2328 /* Clear HW register when we reach 75% of maximum 0xFFFF */
2329 if (ring->old_discards >= 0xC000) {
2330 ring->old_discards = 0;
2331 bcmgenet_rdma_ring_writel(priv, ring->index, 0,
2332 RDMA_PROD_INDEX);
2333 }
2334 }
2335
2336 p_index &= DMA_P_INDEX_MASK;
2337 rxpkttoprocess = (p_index - ring->c_index) & DMA_C_INDEX_MASK;
2338
2339 netif_dbg(priv, rx_status, dev,
2340 "RDMA: rxpkttoprocess=%d\n", rxpkttoprocess);
2341
2342 while ((rxpktprocessed < rxpkttoprocess) &&
2343 (rxpktprocessed < budget)) {
2344 unsigned int rx_offset, rx_size;
2345 struct status_64 *status;
2346 struct page *rx_page;
2347 void *hard_start;
2348 __be16 rx_csum;
2349
2350 cb = &priv->rx_cbs[ring->read_ptr];
2351
2352 /* Save the received page before refilling */
2353 rx_page = cb->rx_page;
2354 rx_offset = cb->rx_offset;
2355 rx_size = cb->rx_size;
2356
2357 if (bcmgenet_rx_refill(ring, cb)) {
2358 BCMGENET_STATS64_INC(stats, dropped);
2359 goto next;
2360 }
2361
2362 /* Sync the full buffer; the HW may have written anywhere
2363 * up to RX_BUF_LENGTH.
2364 */
2365 page_pool_dma_sync_for_cpu(ring->page_pool, rx_page, rx_offset,
2366 RX_BUF_LENGTH);
2367
2368 hard_start = page_address(rx_page) + rx_offset;
2369 status = (struct status_64 *)hard_start;
2370 dma_length_status = status->length_status;
2371
2372 /* DMA flags and length are still valid no matter how
2373 * we got the Receive Status Vector (64B RSB or register)
2374 */
2375 dma_flag = dma_length_status & 0xffff;
2376 len = dma_length_status >> DMA_BUFLENGTH_SHIFT;
2377
2378 netif_dbg(priv, rx_status, dev,
2379 "%s:p_ind=%d c_ind=%d read_ptr=%d len_stat=0x%08x\n",
2380 __func__, p_index, ring->c_index,
2381 ring->read_ptr, dma_length_status);
2382
2383 /* Reject lengths that would underflow the SKB build path. */
2384 if (unlikely(len > RX_BUF_LENGTH || len < GENET_RSB_PAD)) {
2385 netif_err(priv, rx_status, dev,
2386 "invalid packet length %d\n", len);
2387 BCMGENET_STATS64_INC(stats, length_errors);
2388 page_pool_put_full_page(ring->page_pool, rx_page,
2389 true);
2390 goto next;
2391 }
2392
2393 if (unlikely(!(dma_flag & DMA_EOP) || !(dma_flag & DMA_SOP))) {
2394 netif_err(priv, rx_status, dev,
2395 "dropping fragmented packet!\n");
2396 BCMGENET_STATS64_INC(stats, fragmented_errors);
2397 page_pool_put_full_page(ring->page_pool, rx_page,
2398 true);
2399 goto next;
2400 }
2401
2402 /* report errors */
2403 if (unlikely(dma_flag & (DMA_RX_CRC_ERROR |
2404 DMA_RX_OV |
2405 DMA_RX_NO |
2406 DMA_RX_LG |
2407 DMA_RX_RXER))) {
2408 netif_err(priv, rx_status, dev, "dma_flag=0x%x\n",
2409 (unsigned int)dma_flag);
2410 u64_stats_update_begin(&stats->syncp);
2411 if (dma_flag & DMA_RX_CRC_ERROR)
2412 u64_stats_inc(&stats->crc_errors);
2413 if (dma_flag & DMA_RX_OV)
2414 u64_stats_inc(&stats->over_errors);
2415 if (dma_flag & DMA_RX_NO)
2416 u64_stats_inc(&stats->frame_errors);
2417 if (dma_flag & DMA_RX_LG)
2418 u64_stats_inc(&stats->length_errors);
2419 if ((dma_flag & (DMA_RX_CRC_ERROR |
2420 DMA_RX_OV |
2421 DMA_RX_NO |
2422 DMA_RX_LG |
2423 DMA_RX_RXER)) == DMA_RX_RXER)
2424 u64_stats_inc(&stats->errors);
2425 u64_stats_update_end(&stats->syncp);
2426 page_pool_put_full_page(ring->page_pool, rx_page,
2427 true);
2428 goto next;
2429 } /* error packet */
2430
2431 /* Build SKB from the page - data starts at hard_start,
2432 * frame begins after RSB(64) + pad(2) = 66 bytes.
2433 */
2434 skb = napi_build_skb(hard_start, rx_size);
2435 if (unlikely(!skb)) {
2436 BCMGENET_STATS64_INC(stats, dropped);
2437 page_pool_put_full_page(ring->page_pool, rx_page,
2438 true);
2439 goto next;
2440 }
2441
2442 skb_mark_for_recycle(skb);
2443
2444 /* Reserve the RSB + pad, then set the data length */
2445 skb_reserve(skb, GENET_RSB_PAD);
2446 __skb_put(skb, len - GENET_RSB_PAD);
2447
2448 if (priv->crc_fwd_en) {
2449 skb_trim(skb, skb->len - ETH_FCS_LEN);
2450 }
2451
2452 /* Set up checksum offload */
2453 if (dev->features & NETIF_F_RXCSUM) {
2454 rx_csum = (__force __be16)(status->rx_csum & 0xffff);
2455 if (rx_csum) {
2456 skb->csum = (__force __wsum)ntohs(rx_csum);
2457 skb->ip_summed = CHECKSUM_COMPLETE;
2458 }
2459 }
2460
2461 len = skb->len;
2462 bytes_processed += len;
2463
2464 /*Finish setting up the received SKB and send it to the kernel*/
2465 skb->protocol = eth_type_trans(skb, priv->dev);
2466
2467 u64_stats_update_begin(&stats->syncp);
2468 u64_stats_inc(&stats->packets);
2469 u64_stats_add(&stats->bytes, len);
2470 if (dma_flag & DMA_RX_MULT)
2471 u64_stats_inc(&stats->multicast);
2472 else if (dma_flag & DMA_RX_BRDCAST)
2473 u64_stats_inc(&stats->broadcast);
2474 u64_stats_update_end(&stats->syncp);
2475
2476 /* Notify kernel */
2477 napi_gro_receive(&ring->napi, skb);
2478 netif_dbg(priv, rx_status, dev, "pushed up to kernel\n");
2479
2480 next:
2481 rxpktprocessed++;
2482 if (likely(ring->read_ptr < ring->end_ptr))
2483 ring->read_ptr++;
2484 else
2485 ring->read_ptr = ring->cb_ptr;
2486
2487 ring->c_index = (ring->c_index + 1) & DMA_C_INDEX_MASK;
2488 bcmgenet_rdma_ring_writel(priv, ring->index, ring->c_index, RDMA_CONS_INDEX);
2489 }
2490
2491 ring->dim.bytes = bytes_processed;
2492 ring->dim.packets = rxpktprocessed;
2493
2494 return rxpktprocessed;
2495 }
2496
2497 /* Rx NAPI polling method */
bcmgenet_rx_poll(struct napi_struct * napi,int budget)2498 static int bcmgenet_rx_poll(struct napi_struct *napi, int budget)
2499 {
2500 struct bcmgenet_rx_ring *ring = container_of(napi,
2501 struct bcmgenet_rx_ring, napi);
2502 struct dim_sample dim_sample = {};
2503 unsigned int work_done;
2504
2505 work_done = bcmgenet_desc_rx(ring, budget);
2506
2507 if (work_done < budget && napi_complete_done(napi, work_done))
2508 bcmgenet_rx_ring_int_enable(ring);
2509
2510 if (ring->dim.use_dim) {
2511 dim_update_sample(ring->dim.event_ctr, ring->dim.packets,
2512 ring->dim.bytes, &dim_sample);
2513 net_dim(&ring->dim.dim, &dim_sample);
2514 }
2515
2516 return work_done;
2517 }
2518
bcmgenet_dim_work(struct work_struct * work)2519 static void bcmgenet_dim_work(struct work_struct *work)
2520 {
2521 struct dim *dim = container_of(work, struct dim, work);
2522 struct bcmgenet_net_dim *ndim =
2523 container_of(dim, struct bcmgenet_net_dim, dim);
2524 struct bcmgenet_rx_ring *ring =
2525 container_of(ndim, struct bcmgenet_rx_ring, dim);
2526 struct dim_cq_moder cur_profile =
2527 net_dim_get_rx_moderation(dim->mode, dim->profile_ix);
2528
2529 bcmgenet_set_rx_coalesce(ring, cur_profile.usec, cur_profile.pkts);
2530 dim->state = DIM_START_MEASURE;
2531 }
2532
2533 /* Assign page_pool pages to RX DMA descriptors. */
bcmgenet_alloc_rx_buffers(struct bcmgenet_priv * priv,struct bcmgenet_rx_ring * ring)2534 static int bcmgenet_alloc_rx_buffers(struct bcmgenet_priv *priv,
2535 struct bcmgenet_rx_ring *ring)
2536 {
2537 struct enet_cb *cb;
2538 int i;
2539
2540 netif_dbg(priv, hw, priv->dev, "%s\n", __func__);
2541
2542 /* loop here for each buffer needing assign */
2543 for (i = 0; i < ring->size; i++) {
2544 cb = ring->cbs + i;
2545 if (bcmgenet_rx_refill(ring, cb))
2546 return -ENOMEM;
2547 }
2548
2549 return 0;
2550 }
2551
bcmgenet_free_rx_buffers(struct bcmgenet_priv * priv)2552 static void bcmgenet_free_rx_buffers(struct bcmgenet_priv *priv)
2553 {
2554 struct bcmgenet_rx_ring *ring;
2555 struct enet_cb *cb;
2556 int q, i;
2557
2558 for (q = 0; q <= priv->hw_params->rx_queues; q++) {
2559 ring = &priv->rx_rings[q];
2560 if (!ring->page_pool)
2561 continue;
2562 for (i = 0; i < ring->size; i++) {
2563 cb = ring->cbs + i;
2564 bcmgenet_free_rx_cb(cb, ring->page_pool);
2565 }
2566 }
2567 }
2568
umac_enable_set(struct bcmgenet_priv * priv,u32 mask,bool enable)2569 static void umac_enable_set(struct bcmgenet_priv *priv, u32 mask, bool enable)
2570 {
2571 u32 reg;
2572
2573 spin_lock_bh(&priv->reg_lock);
2574 reg = bcmgenet_umac_readl(priv, UMAC_CMD);
2575 if (reg & CMD_SW_RESET) {
2576 spin_unlock_bh(&priv->reg_lock);
2577 return;
2578 }
2579 if (enable)
2580 reg |= mask;
2581 else
2582 reg &= ~mask;
2583 bcmgenet_umac_writel(priv, reg, UMAC_CMD);
2584 spin_unlock_bh(&priv->reg_lock);
2585
2586 /* UniMAC stops on a packet boundary, wait for a full-size packet
2587 * to be processed
2588 */
2589 if (enable == 0)
2590 usleep_range(1000, 2000);
2591 }
2592
reset_umac(struct bcmgenet_priv * priv)2593 static void reset_umac(struct bcmgenet_priv *priv)
2594 {
2595 /* 7358a0/7552a0: bad default in RBUF_FLUSH_CTRL.umac_sw_rst */
2596 bcmgenet_rbuf_ctrl_set(priv, 0);
2597 udelay(10);
2598
2599 /* issue soft reset and disable MAC while updating its registers */
2600 spin_lock_bh(&priv->reg_lock);
2601 bcmgenet_umac_writel(priv, CMD_SW_RESET, UMAC_CMD);
2602 udelay(2);
2603 spin_unlock_bh(&priv->reg_lock);
2604 }
2605
bcmgenet_intr_disable(struct bcmgenet_priv * priv)2606 static void bcmgenet_intr_disable(struct bcmgenet_priv *priv)
2607 {
2608 /* Mask all interrupts.*/
2609 bcmgenet_intrl2_0_writel(priv, 0xFFFFFFFF, INTRL2_CPU_MASK_SET);
2610 bcmgenet_intrl2_0_writel(priv, 0xFFFFFFFF, INTRL2_CPU_CLEAR);
2611 bcmgenet_intrl2_1_writel(priv, 0xFFFFFFFF, INTRL2_CPU_MASK_SET);
2612 bcmgenet_intrl2_1_writel(priv, 0xFFFFFFFF, INTRL2_CPU_CLEAR);
2613 }
2614
bcmgenet_link_intr_enable(struct bcmgenet_priv * priv)2615 static void bcmgenet_link_intr_enable(struct bcmgenet_priv *priv)
2616 {
2617 u32 int0_enable = 0;
2618
2619 /* Monitor cable plug/unplugged event for internal PHY, external PHY
2620 * and MoCA PHY
2621 */
2622 if (priv->internal_phy) {
2623 int0_enable |= UMAC_IRQ_LINK_EVENT;
2624 if (GENET_IS_V1(priv) || GENET_IS_V2(priv) || GENET_IS_V3(priv))
2625 int0_enable |= UMAC_IRQ_PHY_DET_R;
2626 } else if (priv->ext_phy) {
2627 int0_enable |= UMAC_IRQ_LINK_EVENT;
2628 } else if (priv->phy_interface == PHY_INTERFACE_MODE_MOCA) {
2629 if (bcmgenet_has_moca_link_det(priv))
2630 int0_enable |= UMAC_IRQ_LINK_EVENT;
2631 }
2632 bcmgenet_intrl2_0_writel(priv, int0_enable, INTRL2_CPU_MASK_CLEAR);
2633 }
2634
init_umac(struct bcmgenet_priv * priv)2635 static void init_umac(struct bcmgenet_priv *priv)
2636 {
2637 struct device *kdev = &priv->pdev->dev;
2638 u32 reg;
2639 u32 int0_enable = 0;
2640
2641 dev_dbg(&priv->pdev->dev, "bcmgenet: init_umac\n");
2642
2643 reset_umac(priv);
2644
2645 /* clear tx/rx counter */
2646 bcmgenet_umac_writel(priv,
2647 MIB_RESET_RX | MIB_RESET_TX | MIB_RESET_RUNT,
2648 UMAC_MIB_CTRL);
2649 bcmgenet_umac_writel(priv, 0, UMAC_MIB_CTRL);
2650
2651 bcmgenet_umac_writel(priv, ENET_MAX_MTU_SIZE, UMAC_MAX_FRAME_LEN);
2652
2653 /* init tx registers, enable TSB */
2654 reg = bcmgenet_tbuf_ctrl_get(priv);
2655 reg |= TBUF_64B_EN;
2656 bcmgenet_tbuf_ctrl_set(priv, reg);
2657
2658 /* init rx registers, enable ip header optimization and RSB */
2659 reg = bcmgenet_rbuf_readl(priv, RBUF_CTRL);
2660 reg |= RBUF_ALIGN_2B | RBUF_64B_EN;
2661 bcmgenet_rbuf_writel(priv, reg, RBUF_CTRL);
2662
2663 /* enable rx checksumming */
2664 reg = bcmgenet_rbuf_readl(priv, RBUF_CHK_CTRL);
2665 reg |= RBUF_RXCHK_EN | RBUF_L3_PARSE_DIS;
2666 /* If UniMAC forwards CRC, we need to skip over it to get
2667 * a valid CHK bit to be set in the per-packet status word
2668 */
2669 if (priv->crc_fwd_en)
2670 reg |= RBUF_SKIP_FCS;
2671 else
2672 reg &= ~RBUF_SKIP_FCS;
2673 bcmgenet_rbuf_writel(priv, reg, RBUF_CHK_CTRL);
2674
2675 if (!GENET_IS_V1(priv) && !GENET_IS_V2(priv))
2676 bcmgenet_rbuf_writel(priv, 1, RBUF_TBUF_SIZE_CTRL);
2677
2678 bcmgenet_intr_disable(priv);
2679
2680 /* Configure backpressure vectors for MoCA */
2681 if (priv->phy_interface == PHY_INTERFACE_MODE_MOCA) {
2682 reg = bcmgenet_bp_mc_get(priv);
2683 reg |= BIT(priv->hw_params->bp_in_en_shift);
2684
2685 /* bp_mask: back pressure mask */
2686 if (netif_is_multiqueue(priv->dev))
2687 reg |= priv->hw_params->bp_in_mask;
2688 else
2689 reg &= ~priv->hw_params->bp_in_mask;
2690 bcmgenet_bp_mc_set(priv, reg);
2691 }
2692
2693 /* Enable MDIO interrupts on GENET v3+ */
2694 if (bcmgenet_has_mdio_intr(priv))
2695 int0_enable |= UMAC_IRQ_MDIO_EVENT;
2696
2697 bcmgenet_intrl2_0_writel(priv, int0_enable, INTRL2_CPU_MASK_CLEAR);
2698
2699 dev_dbg(kdev, "done init umac\n");
2700 }
2701
bcmgenet_init_dim(struct bcmgenet_rx_ring * ring,void (* cb)(struct work_struct * work))2702 static void bcmgenet_init_dim(struct bcmgenet_rx_ring *ring,
2703 void (*cb)(struct work_struct *work))
2704 {
2705 struct bcmgenet_net_dim *dim = &ring->dim;
2706
2707 INIT_WORK(&dim->dim.work, cb);
2708 dim->dim.mode = DIM_CQ_PERIOD_MODE_START_FROM_EQE;
2709 dim->event_ctr = 0;
2710 dim->packets = 0;
2711 dim->bytes = 0;
2712 }
2713
bcmgenet_init_rx_coalesce(struct bcmgenet_rx_ring * ring)2714 static void bcmgenet_init_rx_coalesce(struct bcmgenet_rx_ring *ring)
2715 {
2716 struct bcmgenet_net_dim *dim = &ring->dim;
2717 struct dim_cq_moder moder;
2718 u32 usecs, pkts;
2719
2720 usecs = ring->rx_coalesce_usecs;
2721 pkts = ring->rx_max_coalesced_frames;
2722
2723 /* If DIM was enabled, re-apply default parameters */
2724 if (dim->use_dim) {
2725 moder = net_dim_get_def_rx_moderation(dim->dim.mode);
2726 usecs = moder.usec;
2727 pkts = moder.pkts;
2728 }
2729
2730 bcmgenet_set_rx_coalesce(ring, usecs, pkts);
2731 }
2732
2733 /* Initialize a Tx ring along with corresponding hardware registers */
bcmgenet_init_tx_ring(struct bcmgenet_priv * priv,unsigned int index,unsigned int size,unsigned int start_ptr,unsigned int end_ptr)2734 static void bcmgenet_init_tx_ring(struct bcmgenet_priv *priv,
2735 unsigned int index, unsigned int size,
2736 unsigned int start_ptr, unsigned int end_ptr)
2737 {
2738 struct bcmgenet_tx_ring *ring = &priv->tx_rings[index];
2739 u32 words_per_bd = WORDS_PER_BD(priv);
2740 u32 flow_period_val = 0;
2741
2742 spin_lock_init(&ring->lock);
2743 ring->priv = priv;
2744 ring->index = index;
2745 ring->cbs = priv->tx_cbs + start_ptr;
2746 ring->size = size;
2747 ring->clean_ptr = start_ptr;
2748 ring->c_index = 0;
2749 ring->free_bds = size;
2750 ring->write_ptr = start_ptr;
2751 ring->cb_ptr = start_ptr;
2752 ring->end_ptr = end_ptr - 1;
2753 ring->prod_index = 0;
2754
2755 /* Set flow period for ring != 0 */
2756 if (index)
2757 flow_period_val = ENET_MAX_MTU_SIZE << 16;
2758
2759 bcmgenet_tdma_ring_writel(priv, index, 0, TDMA_PROD_INDEX);
2760 bcmgenet_tdma_ring_writel(priv, index, 0, TDMA_CONS_INDEX);
2761 bcmgenet_tdma_ring_writel(priv, index, 1, DMA_MBUF_DONE_THRESH);
2762 /* Disable rate control for now */
2763 bcmgenet_tdma_ring_writel(priv, index, flow_period_val,
2764 TDMA_FLOW_PERIOD);
2765 bcmgenet_tdma_ring_writel(priv, index,
2766 ((size << DMA_RING_SIZE_SHIFT) |
2767 RX_BUF_LENGTH), DMA_RING_BUF_SIZE);
2768
2769 /* Set start and end address, read and write pointers */
2770 bcmgenet_tdma_ring_writel(priv, index, start_ptr * words_per_bd,
2771 DMA_START_ADDR);
2772 bcmgenet_tdma_ring_writel(priv, index, start_ptr * words_per_bd,
2773 TDMA_READ_PTR);
2774 bcmgenet_tdma_ring_writel(priv, index, start_ptr * words_per_bd,
2775 TDMA_WRITE_PTR);
2776 bcmgenet_tdma_ring_writel(priv, index, end_ptr * words_per_bd - 1,
2777 DMA_END_ADDR);
2778
2779 /* Initialize Tx NAPI */
2780 netif_napi_add_tx(priv->dev, &ring->napi, bcmgenet_tx_poll);
2781 }
2782
bcmgenet_rx_ring_create_pool(struct bcmgenet_priv * priv,struct bcmgenet_rx_ring * ring)2783 static int bcmgenet_rx_ring_create_pool(struct bcmgenet_priv *priv,
2784 struct bcmgenet_rx_ring *ring)
2785 {
2786 /* Buffers share a page. bcmgenet_rx_refill() syncs each one for the
2787 * device, PP_FLAG_DMA_SYNC_DEV would sync the whole page.
2788 */
2789 struct page_pool_params pp_params = {
2790 .order = 0,
2791 .flags = PP_FLAG_DMA_MAP,
2792 .pool_size = ring->size,
2793 .nid = NUMA_NO_NODE,
2794 .dev = &priv->pdev->dev,
2795 .dma_dir = DMA_FROM_DEVICE,
2796 };
2797 int err;
2798
2799 ring->page_pool = page_pool_create(&pp_params);
2800 if (IS_ERR(ring->page_pool)) {
2801 err = PTR_ERR(ring->page_pool);
2802 ring->page_pool = NULL;
2803 return err;
2804 }
2805
2806 return 0;
2807 }
2808
2809 /* Initialize a RDMA ring */
bcmgenet_init_rx_ring(struct bcmgenet_priv * priv,unsigned int index,unsigned int size,unsigned int start_ptr,unsigned int end_ptr)2810 static int bcmgenet_init_rx_ring(struct bcmgenet_priv *priv,
2811 unsigned int index, unsigned int size,
2812 unsigned int start_ptr, unsigned int end_ptr)
2813 {
2814 struct bcmgenet_rx_ring *ring = &priv->rx_rings[index];
2815 u32 words_per_bd = WORDS_PER_BD(priv);
2816 int ret, i;
2817
2818 ring->priv = priv;
2819 ring->index = index;
2820 ring->cbs = priv->rx_cbs + start_ptr;
2821 ring->size = size;
2822 ring->c_index = 0;
2823 ring->read_ptr = start_ptr;
2824 ring->cb_ptr = start_ptr;
2825 ring->end_ptr = end_ptr - 1;
2826
2827 ret = bcmgenet_rx_ring_create_pool(priv, ring);
2828 if (ret)
2829 return ret;
2830
2831 ret = bcmgenet_alloc_rx_buffers(priv, ring);
2832 if (ret) {
2833 for (i = 0; i < ring->size; i++)
2834 bcmgenet_free_rx_cb(ring->cbs + i, ring->page_pool);
2835 page_pool_destroy(ring->page_pool);
2836 ring->page_pool = NULL;
2837 return ret;
2838 }
2839
2840 bcmgenet_init_dim(ring, bcmgenet_dim_work);
2841 bcmgenet_init_rx_coalesce(ring);
2842
2843 /* Initialize Rx NAPI */
2844 netif_napi_add(priv->dev, &ring->napi, bcmgenet_rx_poll);
2845
2846 bcmgenet_rdma_ring_writel(priv, index, 0, RDMA_PROD_INDEX);
2847 bcmgenet_rdma_ring_writel(priv, index, 0, RDMA_CONS_INDEX);
2848 bcmgenet_rdma_ring_writel(priv, index,
2849 ((size << DMA_RING_SIZE_SHIFT) |
2850 RX_BUF_LENGTH), DMA_RING_BUF_SIZE);
2851 bcmgenet_rdma_ring_writel(priv, index,
2852 (DMA_FC_THRESH_LO <<
2853 DMA_XOFF_THRESHOLD_SHIFT) |
2854 DMA_FC_THRESH_HI, RDMA_XON_XOFF_THRESH);
2855
2856 /* Set start and end address, read and write pointers */
2857 bcmgenet_rdma_ring_writel(priv, index, start_ptr * words_per_bd,
2858 DMA_START_ADDR);
2859 bcmgenet_rdma_ring_writel(priv, index, start_ptr * words_per_bd,
2860 RDMA_READ_PTR);
2861 bcmgenet_rdma_ring_writel(priv, index, start_ptr * words_per_bd,
2862 RDMA_WRITE_PTR);
2863 bcmgenet_rdma_ring_writel(priv, index, end_ptr * words_per_bd - 1,
2864 DMA_END_ADDR);
2865
2866 return ret;
2867 }
2868
bcmgenet_enable_tx_napi(struct bcmgenet_priv * priv)2869 static void bcmgenet_enable_tx_napi(struct bcmgenet_priv *priv)
2870 {
2871 unsigned int i;
2872 struct bcmgenet_tx_ring *ring;
2873
2874 for (i = 0; i <= priv->hw_params->tx_queues; ++i) {
2875 ring = &priv->tx_rings[i];
2876 napi_enable(&ring->napi);
2877 bcmgenet_tx_ring_int_enable(ring);
2878 }
2879 }
2880
bcmgenet_disable_tx_napi(struct bcmgenet_priv * priv)2881 static void bcmgenet_disable_tx_napi(struct bcmgenet_priv *priv)
2882 {
2883 unsigned int i;
2884 struct bcmgenet_tx_ring *ring;
2885
2886 for (i = 0; i <= priv->hw_params->tx_queues; ++i) {
2887 ring = &priv->tx_rings[i];
2888 napi_disable(&ring->napi);
2889 }
2890 }
2891
bcmgenet_fini_tx_napi(struct bcmgenet_priv * priv)2892 static void bcmgenet_fini_tx_napi(struct bcmgenet_priv *priv)
2893 {
2894 unsigned int i;
2895 struct bcmgenet_tx_ring *ring;
2896
2897 for (i = 0; i <= priv->hw_params->tx_queues; ++i) {
2898 ring = &priv->tx_rings[i];
2899 netif_napi_del(&ring->napi);
2900 }
2901 }
2902
bcmgenet_tdma_disable(struct bcmgenet_priv * priv)2903 static int bcmgenet_tdma_disable(struct bcmgenet_priv *priv)
2904 {
2905 int timeout = 0;
2906 u32 reg, mask;
2907
2908 reg = bcmgenet_tdma_readl(priv, DMA_CTRL);
2909 mask = (1 << (priv->hw_params->tx_queues + 1)) - 1;
2910 mask = (mask << DMA_RING_BUF_EN_SHIFT) | DMA_EN;
2911 reg &= ~mask;
2912 bcmgenet_tdma_writel(priv, reg, DMA_CTRL);
2913
2914 /* Check DMA status register to confirm DMA is disabled */
2915 while (timeout++ < DMA_TIMEOUT_VAL) {
2916 reg = bcmgenet_tdma_readl(priv, DMA_STATUS);
2917 if ((reg & mask) == mask)
2918 return 0;
2919
2920 udelay(1);
2921 }
2922
2923 return -ETIMEDOUT;
2924 }
2925
bcmgenet_rdma_disable(struct bcmgenet_priv * priv)2926 static int bcmgenet_rdma_disable(struct bcmgenet_priv *priv)
2927 {
2928 int timeout = 0;
2929 u32 reg, mask;
2930
2931 reg = bcmgenet_rdma_readl(priv, DMA_CTRL);
2932 mask = (1 << (priv->hw_params->rx_queues + 1)) - 1;
2933 mask = (mask << DMA_RING_BUF_EN_SHIFT) | DMA_EN;
2934 reg &= ~mask;
2935 bcmgenet_rdma_writel(priv, reg, DMA_CTRL);
2936
2937 /* Check DMA status register to confirm DMA is disabled */
2938 while (timeout++ < DMA_TIMEOUT_VAL) {
2939 reg = bcmgenet_rdma_readl(priv, DMA_STATUS);
2940 if ((reg & mask) == mask)
2941 return 0;
2942
2943 udelay(1);
2944 }
2945
2946 return -ETIMEDOUT;
2947 }
2948
2949 /* Initialize Tx queues
2950 *
2951 * Queues 1-4 are the priority queues, each one has 32 descriptors.
2952 * The weighted round-robin arbiter gives them a larger share of TX
2953 * bandwidth than the default queue 0.
2954 *
2955 * Queue 0 is the default Tx queue with
2956 * GENET_Q0_TX_BD_CNT = 256 - 4 * 32 = 128 descriptors.
2957 *
2958 * The transmit control block pool is then partitioned as follows:
2959 * - Tx queue 0 uses tx_cbs[0..127]
2960 * - Tx queue 1 uses tx_cbs[128..159]
2961 * - Tx queue 2 uses tx_cbs[160..191]
2962 * - Tx queue 3 uses tx_cbs[192..223]
2963 * - Tx queue 4 uses tx_cbs[224..255]
2964 */
bcmgenet_init_tx_queues(struct net_device * dev)2965 static void bcmgenet_init_tx_queues(struct net_device *dev)
2966 {
2967 struct bcmgenet_priv *priv = netdev_priv(dev);
2968 unsigned int start = 0, end = GENET_Q0_TX_BD_CNT;
2969 u32 i, ring_mask, dma_priority[3] = {0, 0, 0};
2970
2971 /* Enable Weighted Round-Robin arbiter mode */
2972 bcmgenet_tdma_writel(priv, DMA_ARBITER_WRR, DMA_ARB_CTRL);
2973
2974 /* Initialize Tx priority queues */
2975 for (i = 0; i <= priv->hw_params->tx_queues; i++) {
2976 bcmgenet_init_tx_ring(priv, i, end - start, start, end);
2977 start = end;
2978 end += priv->hw_params->tx_bds_per_q;
2979 dma_priority[DMA_PRIO_REG_INDEX(i)] |=
2980 (i ? GENET_Q1_WEIGHT : GENET_Q0_WEIGHT)
2981 << DMA_PRIO_REG_SHIFT(i);
2982 }
2983
2984 /* Set Tx queue priorities */
2985 bcmgenet_tdma_writel(priv, dma_priority[0], DMA_PRIORITY_0);
2986 bcmgenet_tdma_writel(priv, dma_priority[1], DMA_PRIORITY_1);
2987 bcmgenet_tdma_writel(priv, dma_priority[2], DMA_PRIORITY_2);
2988
2989 /* Configure Tx queues as descriptor rings */
2990 ring_mask = (1 << (priv->hw_params->tx_queues + 1)) - 1;
2991 bcmgenet_tdma_writel(priv, ring_mask, DMA_RING_CFG);
2992
2993 /* Enable Tx rings */
2994 ring_mask <<= DMA_RING_BUF_EN_SHIFT;
2995 bcmgenet_tdma_writel(priv, ring_mask, DMA_CTRL);
2996 }
2997
bcmgenet_enable_rx_napi(struct bcmgenet_priv * priv)2998 static void bcmgenet_enable_rx_napi(struct bcmgenet_priv *priv)
2999 {
3000 unsigned int i;
3001 struct bcmgenet_rx_ring *ring;
3002
3003 for (i = 0; i <= priv->hw_params->rx_queues; ++i) {
3004 ring = &priv->rx_rings[i];
3005 napi_enable(&ring->napi);
3006 bcmgenet_rx_ring_int_enable(ring);
3007 }
3008 }
3009
bcmgenet_disable_rx_napi(struct bcmgenet_priv * priv)3010 static void bcmgenet_disable_rx_napi(struct bcmgenet_priv *priv)
3011 {
3012 unsigned int i;
3013 struct bcmgenet_rx_ring *ring;
3014
3015 for (i = 0; i <= priv->hw_params->rx_queues; ++i) {
3016 ring = &priv->rx_rings[i];
3017 napi_disable(&ring->napi);
3018 cancel_work_sync(&ring->dim.dim.work);
3019 }
3020 }
3021
bcmgenet_fini_rx_napi(struct bcmgenet_priv * priv)3022 static void bcmgenet_fini_rx_napi(struct bcmgenet_priv *priv)
3023 {
3024 unsigned int i;
3025 struct bcmgenet_rx_ring *ring;
3026
3027 for (i = 0; i <= priv->hw_params->rx_queues; ++i) {
3028 ring = &priv->rx_rings[i];
3029 netif_napi_del(&ring->napi);
3030 }
3031 }
3032
bcmgenet_destroy_rx_page_pools(struct bcmgenet_priv * priv)3033 static void bcmgenet_destroy_rx_page_pools(struct bcmgenet_priv *priv)
3034 {
3035 struct bcmgenet_rx_ring *ring;
3036 unsigned int i;
3037
3038 for (i = 0; i <= priv->hw_params->rx_queues; ++i) {
3039 ring = &priv->rx_rings[i];
3040 if (ring->page_pool) {
3041 page_pool_destroy(ring->page_pool);
3042 ring->page_pool = NULL;
3043 }
3044 }
3045 }
3046
3047 /* Initialize Rx queues
3048 *
3049 * Queues 0-15 are priority queues. Hardware Filtering Block (HFB) can be
3050 * used to direct traffic to these queues.
3051 *
3052 * Queue 0 is also the default Rx queue with GENET_Q0_RX_BD_CNT descriptors.
3053 */
bcmgenet_init_rx_queues(struct net_device * dev)3054 static int bcmgenet_init_rx_queues(struct net_device *dev)
3055 {
3056 struct bcmgenet_priv *priv = netdev_priv(dev);
3057 unsigned int start = 0, end = GENET_Q0_RX_BD_CNT;
3058 u32 i, ring_mask;
3059 int ret;
3060
3061 /* Initialize Rx priority queues */
3062 for (i = 0; i <= priv->hw_params->rx_queues; i++) {
3063 ret = bcmgenet_init_rx_ring(priv, i, end - start, start, end);
3064 if (ret)
3065 return ret;
3066
3067 start = end;
3068 end += priv->hw_params->rx_bds_per_q;
3069 }
3070
3071 /* Configure Rx queues as descriptor rings */
3072 ring_mask = (1 << (priv->hw_params->rx_queues + 1)) - 1;
3073 bcmgenet_rdma_writel(priv, ring_mask, DMA_RING_CFG);
3074
3075 /* Enable Rx rings */
3076 ring_mask <<= DMA_RING_BUF_EN_SHIFT;
3077 bcmgenet_rdma_writel(priv, ring_mask, DMA_CTRL);
3078
3079 return 0;
3080 }
3081
bcmgenet_dma_teardown(struct bcmgenet_priv * priv)3082 static int bcmgenet_dma_teardown(struct bcmgenet_priv *priv)
3083 {
3084 int ret = 0;
3085
3086 /* Disable TDMA to stop add more frames in TX DMA */
3087 if (-ETIMEDOUT == bcmgenet_tdma_disable(priv)) {
3088 netdev_warn(priv->dev, "Timed out while disabling TX DMA\n");
3089 ret = -ETIMEDOUT;
3090 }
3091
3092 /* Wait 10ms for packet drain in both tx and rx dma */
3093 usleep_range(10000, 20000);
3094
3095 /* Disable RDMA */
3096 if (-ETIMEDOUT == bcmgenet_rdma_disable(priv)) {
3097 netdev_warn(priv->dev, "Timed out while disabling RX DMA\n");
3098 ret = -ETIMEDOUT;
3099 }
3100
3101 return ret;
3102 }
3103
bcmgenet_fini_dma(struct bcmgenet_priv * priv)3104 static void bcmgenet_fini_dma(struct bcmgenet_priv *priv)
3105 {
3106 struct netdev_queue *txq;
3107 int i;
3108
3109 bcmgenet_fini_rx_napi(priv);
3110 bcmgenet_fini_tx_napi(priv);
3111
3112 for (i = 0; i <= priv->hw_params->tx_queues; i++) {
3113 txq = netdev_get_tx_queue(priv->dev, i);
3114 netdev_tx_reset_queue(txq);
3115 }
3116
3117 bcmgenet_free_rx_buffers(priv);
3118 bcmgenet_destroy_rx_page_pools(priv);
3119 kfree(priv->rx_cbs);
3120 kfree(priv->tx_cbs);
3121 }
3122
3123 /* init_edma: Initialize DMA control register */
bcmgenet_init_dma(struct bcmgenet_priv * priv,bool flush_rx)3124 static int bcmgenet_init_dma(struct bcmgenet_priv *priv, bool flush_rx)
3125 {
3126 struct enet_cb *cb;
3127 unsigned int i;
3128 int ret;
3129 u32 reg;
3130
3131 netif_dbg(priv, hw, priv->dev, "%s\n", __func__);
3132
3133 /* Disable TX DMA */
3134 ret = bcmgenet_tdma_disable(priv);
3135 if (ret) {
3136 netdev_err(priv->dev, "failed to halt Tx DMA\n");
3137 return ret;
3138 }
3139
3140 /* Disable RX DMA */
3141 ret = bcmgenet_rdma_disable(priv);
3142 if (ret) {
3143 netdev_err(priv->dev, "failed to halt Rx DMA\n");
3144 return ret;
3145 }
3146
3147 /* Flush TX queues */
3148 bcmgenet_umac_writel(priv, 1, UMAC_TX_FLUSH);
3149 udelay(10);
3150 bcmgenet_umac_writel(priv, 0, UMAC_TX_FLUSH);
3151
3152 if (flush_rx) {
3153 reg = bcmgenet_rbuf_ctrl_get(priv);
3154 bcmgenet_rbuf_ctrl_set(priv, reg | BIT(0));
3155 udelay(10);
3156 bcmgenet_rbuf_ctrl_set(priv, reg);
3157 udelay(10);
3158 }
3159
3160 /* Initialize common Rx ring structures */
3161 priv->rx_bds = priv->base + priv->hw_params->rdma_offset;
3162 priv->num_rx_bds = TOTAL_DESC;
3163 priv->rx_cbs = kzalloc_objs(struct enet_cb, priv->num_rx_bds);
3164 if (!priv->rx_cbs)
3165 return -ENOMEM;
3166
3167 for (i = 0; i < priv->num_rx_bds; i++) {
3168 cb = priv->rx_cbs + i;
3169 cb->bd_addr = priv->rx_bds + i * DMA_DESC_SIZE;
3170 }
3171
3172 /* Initialize common TX ring structures */
3173 priv->tx_bds = priv->base + priv->hw_params->tdma_offset;
3174 priv->num_tx_bds = TOTAL_DESC;
3175 priv->tx_cbs = kzalloc_objs(struct enet_cb, priv->num_tx_bds);
3176 if (!priv->tx_cbs) {
3177 kfree(priv->rx_cbs);
3178 return -ENOMEM;
3179 }
3180
3181 for (i = 0; i < priv->num_tx_bds; i++) {
3182 cb = priv->tx_cbs + i;
3183 cb->bd_addr = priv->tx_bds + i * DMA_DESC_SIZE;
3184 }
3185
3186 /* Init rDma */
3187 bcmgenet_rdma_writel(priv, priv->dma_max_burst_length,
3188 DMA_SCB_BURST_SIZE);
3189
3190 /* Initialize Rx queues */
3191 ret = bcmgenet_init_rx_queues(priv->dev);
3192 if (ret) {
3193 netdev_err(priv->dev, "failed to initialize Rx queues\n");
3194 bcmgenet_free_rx_buffers(priv);
3195 bcmgenet_destroy_rx_page_pools(priv);
3196 kfree(priv->rx_cbs);
3197 kfree(priv->tx_cbs);
3198 return ret;
3199 }
3200
3201 /* Init tDma */
3202 bcmgenet_tdma_writel(priv, priv->dma_max_burst_length,
3203 DMA_SCB_BURST_SIZE);
3204
3205 /* Initialize Tx queues */
3206 bcmgenet_init_tx_queues(priv->dev);
3207
3208 /* Enable RX/TX DMA */
3209 reg = bcmgenet_rdma_readl(priv, DMA_CTRL);
3210 reg |= DMA_EN;
3211 bcmgenet_rdma_writel(priv, reg, DMA_CTRL);
3212
3213 reg = bcmgenet_tdma_readl(priv, DMA_CTRL);
3214 reg |= DMA_EN;
3215 bcmgenet_tdma_writel(priv, reg, DMA_CTRL);
3216
3217 return 0;
3218 }
3219
3220 /* Interrupt bottom half */
bcmgenet_irq_task(struct work_struct * work)3221 static void bcmgenet_irq_task(struct work_struct *work)
3222 {
3223 unsigned int status;
3224 struct bcmgenet_priv *priv = container_of(
3225 work, struct bcmgenet_priv, bcmgenet_irq_work);
3226
3227 netif_dbg(priv, intr, priv->dev, "%s\n", __func__);
3228
3229 spin_lock_irq(&priv->lock);
3230 status = priv->irq0_stat;
3231 priv->irq0_stat = 0;
3232 spin_unlock_irq(&priv->lock);
3233
3234 if (status & UMAC_IRQ_PHY_DET_R &&
3235 priv->dev->phydev->autoneg != AUTONEG_ENABLE) {
3236 phy_init_hw(priv->dev->phydev);
3237 genphy_config_aneg(priv->dev->phydev);
3238 }
3239
3240 /* Link UP/DOWN event */
3241 if (status & UMAC_IRQ_LINK_EVENT)
3242 phy_mac_interrupt(priv->dev->phydev);
3243
3244 }
3245
3246 /* bcmgenet_isr1: handle Rx and Tx queues */
bcmgenet_isr1(int irq,void * dev_id)3247 static irqreturn_t bcmgenet_isr1(int irq, void *dev_id)
3248 {
3249 struct bcmgenet_priv *priv = dev_id;
3250 struct bcmgenet_rx_ring *rx_ring;
3251 struct bcmgenet_tx_ring *tx_ring;
3252 unsigned int index, status;
3253
3254 /* Read irq status */
3255 status = bcmgenet_intrl2_1_readl(priv, INTRL2_CPU_STAT) &
3256 ~bcmgenet_intrl2_1_readl(priv, INTRL2_CPU_MASK_STATUS);
3257
3258 /* clear interrupts */
3259 bcmgenet_intrl2_1_writel(priv, status, INTRL2_CPU_CLEAR);
3260
3261 netif_dbg(priv, intr, priv->dev,
3262 "%s: IRQ=0x%x\n", __func__, status);
3263
3264 /* Check Rx priority queue interrupts */
3265 for (index = 0; index <= priv->hw_params->rx_queues; index++) {
3266 if (!(status & BIT(UMAC_IRQ1_RX_INTR_SHIFT + index)))
3267 continue;
3268
3269 rx_ring = &priv->rx_rings[index];
3270 rx_ring->dim.event_ctr++;
3271
3272 if (likely(napi_schedule_prep(&rx_ring->napi))) {
3273 bcmgenet_rx_ring_int_disable(rx_ring);
3274 __napi_schedule_irqoff(&rx_ring->napi);
3275 }
3276 }
3277
3278 /* Check Tx priority queue interrupts */
3279 for (index = 0; index <= priv->hw_params->tx_queues; index++) {
3280 if (!(status & BIT(index)))
3281 continue;
3282
3283 tx_ring = &priv->tx_rings[index];
3284
3285 if (likely(napi_schedule_prep(&tx_ring->napi))) {
3286 bcmgenet_tx_ring_int_disable(tx_ring);
3287 __napi_schedule_irqoff(&tx_ring->napi);
3288 }
3289 }
3290
3291 return IRQ_HANDLED;
3292 }
3293
3294 /* bcmgenet_isr0: handle other stuff */
bcmgenet_isr0(int irq,void * dev_id)3295 static irqreturn_t bcmgenet_isr0(int irq, void *dev_id)
3296 {
3297 struct bcmgenet_priv *priv = dev_id;
3298 unsigned int status;
3299 unsigned long flags;
3300
3301 /* Read irq status */
3302 status = bcmgenet_intrl2_0_readl(priv, INTRL2_CPU_STAT) &
3303 ~bcmgenet_intrl2_0_readl(priv, INTRL2_CPU_MASK_STATUS);
3304
3305 /* clear interrupts */
3306 bcmgenet_intrl2_0_writel(priv, status, INTRL2_CPU_CLEAR);
3307
3308 netif_dbg(priv, intr, priv->dev,
3309 "IRQ=0x%x\n", status);
3310
3311 if (bcmgenet_has_mdio_intr(priv) && status & UMAC_IRQ_MDIO_EVENT)
3312 wake_up(&priv->wq);
3313
3314 /* all other interested interrupts handled in bottom half */
3315 status &= (UMAC_IRQ_LINK_EVENT | UMAC_IRQ_PHY_DET_R);
3316 if (status) {
3317 /* Save irq status for bottom-half processing. */
3318 spin_lock_irqsave(&priv->lock, flags);
3319 priv->irq0_stat |= status;
3320 spin_unlock_irqrestore(&priv->lock, flags);
3321
3322 schedule_work(&priv->bcmgenet_irq_work);
3323 }
3324
3325 return IRQ_HANDLED;
3326 }
3327
bcmgenet_wol_isr(int irq,void * dev_id)3328 static irqreturn_t bcmgenet_wol_isr(int irq, void *dev_id)
3329 {
3330 /* Acknowledge the interrupt */
3331 return IRQ_HANDLED;
3332 }
3333
bcmgenet_umac_reset(struct bcmgenet_priv * priv)3334 static void bcmgenet_umac_reset(struct bcmgenet_priv *priv)
3335 {
3336 u32 reg;
3337
3338 reg = bcmgenet_rbuf_ctrl_get(priv);
3339 reg |= BIT(1);
3340 bcmgenet_rbuf_ctrl_set(priv, reg);
3341 udelay(10);
3342
3343 reg &= ~BIT(1);
3344 bcmgenet_rbuf_ctrl_set(priv, reg);
3345 udelay(10);
3346 }
3347
bcmgenet_set_hw_addr(struct bcmgenet_priv * priv,const unsigned char * addr)3348 static void bcmgenet_set_hw_addr(struct bcmgenet_priv *priv,
3349 const unsigned char *addr)
3350 {
3351 bcmgenet_umac_writel(priv, get_unaligned_be32(&addr[0]), UMAC_MAC0);
3352 bcmgenet_umac_writel(priv, get_unaligned_be16(&addr[4]), UMAC_MAC1);
3353 }
3354
bcmgenet_get_hw_addr(struct bcmgenet_priv * priv,unsigned char * addr)3355 static void bcmgenet_get_hw_addr(struct bcmgenet_priv *priv,
3356 unsigned char *addr)
3357 {
3358 u32 addr_tmp;
3359
3360 addr_tmp = bcmgenet_umac_readl(priv, UMAC_MAC0);
3361 put_unaligned_be32(addr_tmp, &addr[0]);
3362 addr_tmp = bcmgenet_umac_readl(priv, UMAC_MAC1);
3363 put_unaligned_be16(addr_tmp, &addr[4]);
3364 }
3365
bcmgenet_netif_start(struct net_device * dev)3366 static void bcmgenet_netif_start(struct net_device *dev)
3367 {
3368 struct bcmgenet_priv *priv = netdev_priv(dev);
3369
3370 /* Start the network engine */
3371 netif_addr_lock_bh(dev);
3372 bcmgenet_set_rx_mode(dev);
3373 netif_addr_unlock_bh(dev);
3374 bcmgenet_enable_rx_napi(priv);
3375
3376 umac_enable_set(priv, CMD_TX_EN | CMD_RX_EN, true);
3377
3378 bcmgenet_enable_tx_napi(priv);
3379
3380 /* Monitor link interrupts now */
3381 bcmgenet_link_intr_enable(priv);
3382
3383 phy_start(dev->phydev);
3384 }
3385
bcmgenet_open(struct net_device * dev)3386 static int bcmgenet_open(struct net_device *dev)
3387 {
3388 struct bcmgenet_priv *priv = netdev_priv(dev);
3389 int ret;
3390
3391 netif_dbg(priv, ifup, dev, "bcmgenet_open\n");
3392
3393 /* Turn on the clock */
3394 clk_prepare_enable(priv->clk);
3395
3396 /* If this is an internal GPHY, power it back on now, before UniMAC is
3397 * brought out of reset as absolutely no UniMAC activity is allowed
3398 */
3399 if (priv->internal_phy)
3400 bcmgenet_power_up(priv, GENET_POWER_PASSIVE);
3401
3402 /* take MAC out of reset */
3403 bcmgenet_umac_reset(priv);
3404
3405 init_umac(priv);
3406
3407 /* Apply features again in case we changed them while interface was
3408 * down
3409 */
3410 bcmgenet_set_features(dev, dev->features);
3411
3412 bcmgenet_set_hw_addr(priv, dev->dev_addr);
3413
3414 /* Restore the filters, the MAC was reset above */
3415 bcmgenet_hfb_restore(priv);
3416
3417 /* Reinitialize TDMA and RDMA and SW housekeeping */
3418 ret = bcmgenet_init_dma(priv, true);
3419 if (ret) {
3420 netdev_err(dev, "failed to initialize DMA\n");
3421 goto err_clk_disable;
3422 }
3423
3424 ret = request_irq(priv->irq0, bcmgenet_isr0, IRQF_SHARED,
3425 dev->name, priv);
3426 if (ret < 0) {
3427 netdev_err(dev, "can't request IRQ %d\n", priv->irq0);
3428 goto err_fini_dma;
3429 }
3430
3431 ret = request_irq(priv->irq1, bcmgenet_isr1, IRQF_SHARED,
3432 dev->name, priv);
3433 if (ret < 0) {
3434 netdev_err(dev, "can't request IRQ %d\n", priv->irq1);
3435 goto err_irq0;
3436 }
3437
3438 ret = bcmgenet_mii_probe(dev);
3439 if (ret) {
3440 netdev_err(dev, "failed to connect to PHY\n");
3441 goto err_irq1;
3442 }
3443
3444 bcmgenet_phy_pause_set(dev, priv->rx_pause, priv->tx_pause);
3445
3446 bcmgenet_netif_start(dev);
3447
3448 netif_tx_start_all_queues(dev);
3449
3450 return 0;
3451
3452 err_irq1:
3453 free_irq(priv->irq1, priv);
3454 err_irq0:
3455 free_irq(priv->irq0, priv);
3456 err_fini_dma:
3457 bcmgenet_dma_teardown(priv);
3458 bcmgenet_fini_dma(priv);
3459 err_clk_disable:
3460 if (priv->internal_phy)
3461 bcmgenet_power_down(priv, GENET_POWER_PASSIVE);
3462 clk_disable_unprepare(priv->clk);
3463 return ret;
3464 }
3465
bcmgenet_netif_stop(struct net_device * dev,bool stop_phy)3466 static void bcmgenet_netif_stop(struct net_device *dev, bool stop_phy)
3467 {
3468 struct bcmgenet_priv *priv = netdev_priv(dev);
3469
3470 /* Stop completion polling before it can wake a stopped queue */
3471 bcmgenet_disable_tx_napi(priv);
3472 netif_tx_disable(dev);
3473
3474 /* Disable MAC receive */
3475 bcmgenet_hfb_reg_writel(priv, 0, HFB_CTRL);
3476 umac_enable_set(priv, CMD_RX_EN, false);
3477
3478 if (stop_phy)
3479 phy_stop(dev->phydev);
3480
3481 bcmgenet_dma_teardown(priv);
3482
3483 /* Disable MAC transmit. TX DMA disabled must be done before this */
3484 umac_enable_set(priv, CMD_TX_EN, false);
3485
3486 bcmgenet_disable_rx_napi(priv);
3487 bcmgenet_intr_disable(priv);
3488
3489 /* Wait for pending work items to complete. Since interrupts are
3490 * disabled no new work will be scheduled.
3491 */
3492 cancel_work_sync(&priv->bcmgenet_irq_work);
3493
3494 /* tx reclaim */
3495 bcmgenet_tx_reclaim_all(dev);
3496 bcmgenet_fini_dma(priv);
3497 }
3498
bcmgenet_close(struct net_device * dev)3499 static int bcmgenet_close(struct net_device *dev)
3500 {
3501 struct bcmgenet_priv *priv = netdev_priv(dev);
3502 int ret = 0;
3503
3504 netif_dbg(priv, ifdown, dev, "bcmgenet_close\n");
3505
3506 bcmgenet_netif_stop(dev, false);
3507
3508 /* Really kill the PHY state machine and disconnect from it */
3509 phy_disconnect(dev->phydev);
3510
3511 free_irq(priv->irq0, priv);
3512 free_irq(priv->irq1, priv);
3513
3514 if (priv->internal_phy)
3515 ret = bcmgenet_power_down(priv, GENET_POWER_PASSIVE);
3516
3517 clk_disable_unprepare(priv->clk);
3518
3519 return ret;
3520 }
3521
bcmgenet_dump_tx_queue(struct bcmgenet_tx_ring * ring)3522 static void bcmgenet_dump_tx_queue(struct bcmgenet_tx_ring *ring)
3523 {
3524 struct bcmgenet_priv *priv = ring->priv;
3525 u32 p_index, c_index, intsts, intmsk;
3526 struct netdev_queue *txq;
3527 unsigned int free_bds;
3528 bool txq_stopped;
3529
3530 if (!netif_msg_tx_err(priv))
3531 return;
3532
3533 txq = netdev_get_tx_queue(priv->dev, ring->index);
3534
3535 spin_lock(&ring->lock);
3536 intsts = ~bcmgenet_intrl2_1_readl(priv, INTRL2_CPU_MASK_STATUS);
3537 intmsk = 1 << ring->index;
3538 c_index = bcmgenet_tdma_ring_readl(priv, ring->index, TDMA_CONS_INDEX);
3539 p_index = bcmgenet_tdma_ring_readl(priv, ring->index, TDMA_PROD_INDEX);
3540 txq_stopped = netif_tx_queue_stopped(txq);
3541 free_bds = ring->free_bds;
3542 spin_unlock(&ring->lock);
3543
3544 netif_err(priv, tx_err, priv->dev, "Ring %d queue %d status summary\n"
3545 "TX queue status: %s, interrupts: %s\n"
3546 "(sw)free_bds: %d (sw)size: %d\n"
3547 "(sw)p_index: %d (hw)p_index: %d\n"
3548 "(sw)c_index: %d (hw)c_index: %d\n"
3549 "(sw)clean_p: %d (sw)write_p: %d\n"
3550 "(sw)cb_ptr: %d (sw)end_ptr: %d\n",
3551 ring->index, ring->index,
3552 txq_stopped ? "stopped" : "active",
3553 intsts & intmsk ? "enabled" : "disabled",
3554 free_bds, ring->size,
3555 ring->prod_index, p_index & DMA_P_INDEX_MASK,
3556 ring->c_index, c_index & DMA_C_INDEX_MASK,
3557 ring->clean_ptr, ring->write_ptr,
3558 ring->cb_ptr, ring->end_ptr);
3559 }
3560
bcmgenet_timeout(struct net_device * dev,unsigned int txqueue)3561 static void bcmgenet_timeout(struct net_device *dev, unsigned int txqueue)
3562 {
3563 struct bcmgenet_priv *priv = netdev_priv(dev);
3564 struct bcmgenet_tx_ring *ring = &priv->tx_rings[txqueue];
3565 struct netdev_queue *txq = netdev_get_tx_queue(dev, txqueue);
3566
3567 netif_dbg(priv, tx_err, dev, "bcmgenet_timeout\n");
3568
3569 bcmgenet_dump_tx_queue(ring);
3570
3571 bcmgenet_tx_reclaim(dev, ring, true);
3572
3573 /* Re-enable the TX interrupt for this ring */
3574 bcmgenet_intrl2_1_writel(priv, 1 << txqueue, INTRL2_CPU_MASK_CLEAR);
3575
3576 txq_trans_cond_update(txq);
3577
3578 BCMGENET_STATS64_INC((&ring->stats64), errors);
3579
3580 netif_tx_wake_queue(txq);
3581 }
3582
3583 #define MAX_MDF_FILTER 17
3584
bcmgenet_set_mdf_addr(struct bcmgenet_priv * priv,const unsigned char * addr,int * i)3585 static inline void bcmgenet_set_mdf_addr(struct bcmgenet_priv *priv,
3586 const unsigned char *addr,
3587 int *i)
3588 {
3589 bcmgenet_umac_writel(priv, addr[0] << 8 | addr[1],
3590 UMAC_MDF_ADDR + (*i * 4));
3591 bcmgenet_umac_writel(priv, addr[2] << 24 | addr[3] << 16 |
3592 addr[4] << 8 | addr[5],
3593 UMAC_MDF_ADDR + ((*i + 1) * 4));
3594 *i += 2;
3595 }
3596
bcmgenet_set_rx_mode(struct net_device * dev)3597 static void bcmgenet_set_rx_mode(struct net_device *dev)
3598 {
3599 struct bcmgenet_priv *priv = netdev_priv(dev);
3600 struct netdev_hw_addr *ha;
3601 int i, nfilter;
3602 u32 reg;
3603
3604 netif_dbg(priv, hw, dev, "%s: %08X\n", __func__, dev->flags);
3605
3606 /* Number of filters needed */
3607 nfilter = netdev_uc_count(dev) + netdev_mc_count(dev) + 2;
3608
3609 /*
3610 * Turn on promicuous mode for three scenarios
3611 * 1. IFF_PROMISC flag is set
3612 * 2. IFF_ALLMULTI flag is set
3613 * 3. The number of filters needed exceeds the number filters
3614 * supported by the hardware.
3615 */
3616 spin_lock(&priv->reg_lock);
3617 reg = bcmgenet_umac_readl(priv, UMAC_CMD);
3618 if ((dev->flags & (IFF_PROMISC | IFF_ALLMULTI)) ||
3619 (nfilter > MAX_MDF_FILTER)) {
3620 reg |= CMD_PROMISC;
3621 bcmgenet_umac_writel(priv, reg, UMAC_CMD);
3622 spin_unlock(&priv->reg_lock);
3623 bcmgenet_umac_writel(priv, 0, UMAC_MDF_CTRL);
3624 return;
3625 } else {
3626 reg &= ~CMD_PROMISC;
3627 bcmgenet_umac_writel(priv, reg, UMAC_CMD);
3628 spin_unlock(&priv->reg_lock);
3629 }
3630
3631 /* update MDF filter */
3632 i = 0;
3633 /* Broadcast */
3634 bcmgenet_set_mdf_addr(priv, dev->broadcast, &i);
3635 /* my own address.*/
3636 bcmgenet_set_mdf_addr(priv, dev->dev_addr, &i);
3637
3638 /* Unicast */
3639 netdev_for_each_uc_addr(ha, dev)
3640 bcmgenet_set_mdf_addr(priv, ha->addr, &i);
3641
3642 /* Multicast */
3643 netdev_for_each_mc_addr(ha, dev)
3644 bcmgenet_set_mdf_addr(priv, ha->addr, &i);
3645
3646 /* Enable filters */
3647 reg = GENMASK(MAX_MDF_FILTER - 1, MAX_MDF_FILTER - nfilter);
3648 bcmgenet_umac_writel(priv, reg, UMAC_MDF_CTRL);
3649 }
3650
3651 /* Set the hardware MAC address. */
bcmgenet_set_mac_addr(struct net_device * dev,void * p)3652 static int bcmgenet_set_mac_addr(struct net_device *dev, void *p)
3653 {
3654 struct sockaddr *addr = p;
3655
3656 /* Setting the MAC address at the hardware level is not possible
3657 * without disabling the UniMAC RX/TX enable bits.
3658 */
3659 if (netif_running(dev))
3660 return -EBUSY;
3661
3662 if (!is_valid_ether_addr(addr->sa_data))
3663 return -EADDRNOTAVAIL;
3664
3665 eth_hw_addr_set(dev, addr->sa_data);
3666
3667 return 0;
3668 }
3669
bcmgenet_get_stats64(struct net_device * dev,struct rtnl_link_stats64 * stats)3670 static void bcmgenet_get_stats64(struct net_device *dev,
3671 struct rtnl_link_stats64 *stats)
3672 {
3673 struct bcmgenet_priv *priv = netdev_priv(dev);
3674 struct bcmgenet_tx_stats64 *tx_stats;
3675 struct bcmgenet_rx_stats64 *rx_stats;
3676 u64 rx_length_errors, rx_over_errors;
3677 u64 rx_missed, rx_fragmented_errors;
3678 u64 rx_crc_errors, rx_frame_errors;
3679 u64 tx_errors, tx_dropped;
3680 u64 rx_errors, rx_dropped;
3681 u64 tx_bytes, tx_packets;
3682 u64 rx_bytes, rx_packets;
3683 unsigned int start;
3684 unsigned int q;
3685 u64 multicast;
3686
3687 for (q = 0; q <= priv->hw_params->tx_queues; q++) {
3688 tx_stats = &priv->tx_rings[q].stats64;
3689 do {
3690 start = u64_stats_fetch_begin(&tx_stats->syncp);
3691 tx_bytes = u64_stats_read(&tx_stats->bytes);
3692 tx_packets = u64_stats_read(&tx_stats->packets);
3693 tx_errors = u64_stats_read(&tx_stats->errors);
3694 tx_dropped = u64_stats_read(&tx_stats->dropped);
3695 } while (u64_stats_fetch_retry(&tx_stats->syncp, start));
3696
3697 stats->tx_bytes += tx_bytes;
3698 stats->tx_packets += tx_packets;
3699 stats->tx_errors += tx_errors;
3700 stats->tx_dropped += tx_dropped;
3701 }
3702
3703 for (q = 0; q <= priv->hw_params->rx_queues; q++) {
3704 rx_stats = &priv->rx_rings[q].stats64;
3705 do {
3706 start = u64_stats_fetch_begin(&rx_stats->syncp);
3707 rx_bytes = u64_stats_read(&rx_stats->bytes);
3708 rx_packets = u64_stats_read(&rx_stats->packets);
3709 rx_errors = u64_stats_read(&rx_stats->errors);
3710 rx_dropped = u64_stats_read(&rx_stats->dropped);
3711 rx_missed = u64_stats_read(&rx_stats->missed);
3712 rx_length_errors = u64_stats_read(&rx_stats->length_errors);
3713 rx_over_errors = u64_stats_read(&rx_stats->over_errors);
3714 rx_crc_errors = u64_stats_read(&rx_stats->crc_errors);
3715 rx_frame_errors = u64_stats_read(&rx_stats->frame_errors);
3716 rx_fragmented_errors = u64_stats_read(&rx_stats->fragmented_errors);
3717 multicast = u64_stats_read(&rx_stats->multicast);
3718 } while (u64_stats_fetch_retry(&rx_stats->syncp, start));
3719
3720 rx_errors += rx_length_errors;
3721 rx_errors += rx_crc_errors;
3722 rx_errors += rx_frame_errors;
3723 rx_errors += rx_fragmented_errors;
3724
3725 stats->rx_bytes += rx_bytes;
3726 stats->rx_packets += rx_packets;
3727 stats->rx_errors += rx_errors;
3728 stats->rx_dropped += rx_dropped;
3729 stats->rx_missed_errors += rx_missed;
3730 stats->rx_length_errors += rx_length_errors;
3731 stats->rx_over_errors += rx_over_errors;
3732 stats->rx_crc_errors += rx_crc_errors;
3733 stats->rx_frame_errors += rx_frame_errors;
3734 stats->multicast += multicast;
3735 }
3736 }
3737
bcmgenet_change_carrier(struct net_device * dev,bool new_carrier)3738 static int bcmgenet_change_carrier(struct net_device *dev, bool new_carrier)
3739 {
3740 struct bcmgenet_priv *priv = netdev_priv(dev);
3741
3742 if (!dev->phydev || !phy_is_pseudo_fixed_link(dev->phydev) ||
3743 priv->phy_interface != PHY_INTERFACE_MODE_MOCA)
3744 return -EOPNOTSUPP;
3745
3746 if (new_carrier)
3747 netif_carrier_on(dev);
3748 else
3749 netif_carrier_off(dev);
3750
3751 return 0;
3752 }
3753
3754 static const struct net_device_ops bcmgenet_netdev_ops = {
3755 .ndo_open = bcmgenet_open,
3756 .ndo_stop = bcmgenet_close,
3757 .ndo_start_xmit = bcmgenet_xmit,
3758 .ndo_tx_timeout = bcmgenet_timeout,
3759 .ndo_set_rx_mode = bcmgenet_set_rx_mode,
3760 .ndo_set_mac_address = bcmgenet_set_mac_addr,
3761 .ndo_eth_ioctl = phy_do_ioctl_running,
3762 .ndo_set_features = bcmgenet_set_features,
3763 .ndo_get_stats64 = bcmgenet_get_stats64,
3764 .ndo_change_carrier = bcmgenet_change_carrier,
3765 };
3766
3767 /* GENET hardware parameters/characteristics */
3768 static const struct bcmgenet_hw_params bcmgenet_hw_params_v1 = {
3769 .tx_queues = 0,
3770 .tx_bds_per_q = 0,
3771 .rx_queues = 0,
3772 .rx_bds_per_q = 0,
3773 .bp_in_en_shift = 16,
3774 .bp_in_mask = 0xffff,
3775 .hfb_filter_cnt = 16,
3776 .hfb_filter_size = 64,
3777 .qtag_mask = 0x1F,
3778 .hfb_offset = 0x1000,
3779 .hfb_reg_offset = GENET_RBUF_OFF + RBUF_HFB_CTRL_V1,
3780 .rdma_offset = 0x2000,
3781 .tdma_offset = 0x3000,
3782 .words_per_bd = 2,
3783 };
3784
3785 static const struct bcmgenet_hw_params bcmgenet_hw_params_v2 = {
3786 .tx_queues = 4,
3787 .tx_bds_per_q = 32,
3788 .rx_queues = 0,
3789 .rx_bds_per_q = 0,
3790 .bp_in_en_shift = 16,
3791 .bp_in_mask = 0xffff,
3792 .hfb_filter_cnt = 16,
3793 .hfb_filter_size = 64,
3794 .qtag_mask = 0x1F,
3795 .tbuf_offset = 0x0600,
3796 .hfb_offset = 0x1000,
3797 .hfb_reg_offset = 0x2000,
3798 .rdma_offset = 0x3000,
3799 .tdma_offset = 0x4000,
3800 .words_per_bd = 2,
3801 };
3802
3803 static const struct bcmgenet_hw_params bcmgenet_hw_params_v3 = {
3804 .tx_queues = 4,
3805 .tx_bds_per_q = 32,
3806 .rx_queues = 0,
3807 .rx_bds_per_q = 0,
3808 .bp_in_en_shift = 17,
3809 .bp_in_mask = 0x1ffff,
3810 .hfb_filter_cnt = 48,
3811 .hfb_filter_size = 128,
3812 .qtag_mask = 0x3F,
3813 .tbuf_offset = 0x0600,
3814 .hfb_offset = 0x8000,
3815 .hfb_reg_offset = 0xfc00,
3816 .rdma_offset = 0x10000,
3817 .tdma_offset = 0x11000,
3818 .words_per_bd = 2,
3819 };
3820
3821 static const struct bcmgenet_hw_params bcmgenet_hw_params_v4 = {
3822 .tx_queues = 4,
3823 .tx_bds_per_q = 32,
3824 .rx_queues = 0,
3825 .rx_bds_per_q = 0,
3826 .bp_in_en_shift = 17,
3827 .bp_in_mask = 0x1ffff,
3828 .hfb_filter_cnt = 48,
3829 .hfb_filter_size = 128,
3830 .qtag_mask = 0x3F,
3831 .tbuf_offset = 0x0600,
3832 .hfb_offset = 0x8000,
3833 .hfb_reg_offset = 0xfc00,
3834 .rdma_offset = 0x2000,
3835 .tdma_offset = 0x4000,
3836 .words_per_bd = 3,
3837 };
3838
3839 /* Infer hardware parameters from the detected GENET version */
bcmgenet_set_hw_params(struct bcmgenet_priv * priv)3840 static void bcmgenet_set_hw_params(struct bcmgenet_priv *priv)
3841 {
3842 const struct bcmgenet_hw_params *params;
3843 u32 reg;
3844 u8 major;
3845 u16 gphy_rev;
3846
3847 /* default to latest values */
3848 params = &bcmgenet_hw_params_v4;
3849 bcmgenet_dma_regs = bcmgenet_dma_regs_v3plus;
3850 genet_dma_ring_regs = genet_dma_ring_regs_v4;
3851 if (GENET_IS_V3(priv)) {
3852 params = &bcmgenet_hw_params_v3;
3853 bcmgenet_dma_regs = bcmgenet_dma_regs_v3plus;
3854 genet_dma_ring_regs = genet_dma_ring_regs_v123;
3855 } else if (GENET_IS_V2(priv)) {
3856 params = &bcmgenet_hw_params_v2;
3857 bcmgenet_dma_regs = bcmgenet_dma_regs_v2;
3858 genet_dma_ring_regs = genet_dma_ring_regs_v123;
3859 } else if (GENET_IS_V1(priv)) {
3860 params = &bcmgenet_hw_params_v1;
3861 bcmgenet_dma_regs = bcmgenet_dma_regs_v1;
3862 genet_dma_ring_regs = genet_dma_ring_regs_v123;
3863 }
3864 priv->hw_params = params;
3865
3866 /* Read GENET HW version */
3867 reg = bcmgenet_sys_readl(priv, SYS_REV_CTRL);
3868 major = (reg >> 24 & 0x0f);
3869 if (major == 6 || major == 7)
3870 major = 5;
3871 else if (major == 5)
3872 major = 4;
3873 else if (major == 0)
3874 major = 1;
3875 if (major != priv->version) {
3876 dev_err(&priv->pdev->dev,
3877 "GENET version mismatch, got: %d, configured for: %d\n",
3878 major, priv->version);
3879 }
3880
3881 /* Print the GENET core version */
3882 dev_info(&priv->pdev->dev, "GENET " GENET_VER_FMT,
3883 major, (reg >> 16) & 0x0f, reg & 0xffff);
3884
3885 /* Store the integrated PHY revision for the MDIO probing function
3886 * to pass this information to the PHY driver. The PHY driver expects
3887 * to find the PHY major revision in bits 15:8 while the GENET register
3888 * stores that information in bits 7:0, account for that.
3889 *
3890 * On newer chips, starting with PHY revision G0, a new scheme is
3891 * deployed similar to the Starfighter 2 switch with GPHY major
3892 * revision in bits 15:8 and patch level in bits 7:0. Major revision 0
3893 * is reserved as well as special value 0x01ff, we have a small
3894 * heuristic to check for the new GPHY revision and re-arrange things
3895 * so the GPHY driver is happy.
3896 */
3897 gphy_rev = reg & 0xffff;
3898
3899 if (GENET_IS_V5(priv)) {
3900 /* The EPHY revision should come from the MDIO registers of
3901 * the PHY not from GENET.
3902 */
3903 if (gphy_rev != 0) {
3904 pr_warn("GENET is reporting EPHY revision: 0x%04x\n",
3905 gphy_rev);
3906 }
3907 /* This is reserved so should require special treatment */
3908 } else if (gphy_rev == 0 || gphy_rev == 0x01ff) {
3909 pr_warn("Invalid GPHY revision detected: 0x%04x\n", gphy_rev);
3910 return;
3911 /* This is the good old scheme, just GPHY major, no minor nor patch */
3912 } else if ((gphy_rev & 0xf0) != 0) {
3913 priv->gphy_rev = gphy_rev << 8;
3914 /* This is the new scheme, GPHY major rolls over with 0x10 = rev G0 */
3915 } else if ((gphy_rev & 0xff00) != 0) {
3916 priv->gphy_rev = gphy_rev;
3917 }
3918
3919 #ifdef CONFIG_PHYS_ADDR_T_64BIT
3920 if (!bcmgenet_has_40bits(priv))
3921 pr_warn("GENET does not support 40-bits PA\n");
3922 #endif
3923
3924 pr_debug("Configuration for version: %d\n"
3925 "TXq: %1d, TXqBDs: %1d, RXq: %1d, RXqBDs: %1d\n"
3926 "BP << en: %2d, BP msk: 0x%05x\n"
3927 "HFB count: %2d, QTAQ msk: 0x%05x\n"
3928 "TBUF: 0x%04x, HFB: 0x%04x, HFBreg: 0x%04x\n"
3929 "RDMA: 0x%05x, TDMA: 0x%05x\n"
3930 "Words/BD: %d\n",
3931 priv->version,
3932 params->tx_queues, params->tx_bds_per_q,
3933 params->rx_queues, params->rx_bds_per_q,
3934 params->bp_in_en_shift, params->bp_in_mask,
3935 params->hfb_filter_cnt, params->qtag_mask,
3936 params->tbuf_offset, params->hfb_offset,
3937 params->hfb_reg_offset,
3938 params->rdma_offset, params->tdma_offset,
3939 params->words_per_bd);
3940 }
3941
3942 struct bcmgenet_plat_data {
3943 enum bcmgenet_version version;
3944 u32 dma_max_burst_length;
3945 u32 flags;
3946 };
3947
3948 static const struct bcmgenet_plat_data v1_plat_data = {
3949 .version = GENET_V1,
3950 .dma_max_burst_length = DMA_MAX_BURST_LENGTH,
3951 };
3952
3953 static const struct bcmgenet_plat_data v2_plat_data = {
3954 .version = GENET_V2,
3955 .dma_max_burst_length = DMA_MAX_BURST_LENGTH,
3956 .flags = GENET_HAS_EXT,
3957 };
3958
3959 static const struct bcmgenet_plat_data v3_plat_data = {
3960 .version = GENET_V3,
3961 .dma_max_burst_length = DMA_MAX_BURST_LENGTH,
3962 .flags = GENET_HAS_EXT | GENET_HAS_MDIO_INTR |
3963 GENET_HAS_MOCA_LINK_DET,
3964 };
3965
3966 static const struct bcmgenet_plat_data v4_plat_data = {
3967 .version = GENET_V4,
3968 .dma_max_burst_length = DMA_MAX_BURST_LENGTH,
3969 .flags = GENET_HAS_40BITS | GENET_HAS_EXT |
3970 GENET_HAS_MDIO_INTR | GENET_HAS_MOCA_LINK_DET,
3971 };
3972
3973 static const struct bcmgenet_plat_data v5_plat_data = {
3974 .version = GENET_V5,
3975 .dma_max_burst_length = DMA_MAX_BURST_LENGTH,
3976 .flags = GENET_HAS_40BITS | GENET_HAS_EXT |
3977 GENET_HAS_MDIO_INTR | GENET_HAS_MOCA_LINK_DET,
3978 };
3979
3980 static const struct bcmgenet_plat_data bcm2711_plat_data = {
3981 .version = GENET_V5,
3982 .dma_max_burst_length = 0x08,
3983 .flags = GENET_HAS_40BITS | GENET_HAS_EXT |
3984 GENET_HAS_MDIO_INTR | GENET_HAS_MOCA_LINK_DET,
3985 };
3986
3987 static const struct bcmgenet_plat_data bcm7712_plat_data = {
3988 .version = GENET_V5,
3989 .dma_max_burst_length = DMA_MAX_BURST_LENGTH,
3990 .flags = GENET_HAS_40BITS | GENET_HAS_EXT |
3991 GENET_HAS_MDIO_INTR | GENET_HAS_MOCA_LINK_DET |
3992 GENET_HAS_EPHY_16NM,
3993 };
3994
3995 static const struct of_device_id bcmgenet_match[] = {
3996 { .compatible = "brcm,genet-v1", .data = &v1_plat_data },
3997 { .compatible = "brcm,genet-v2", .data = &v2_plat_data },
3998 { .compatible = "brcm,genet-v3", .data = &v3_plat_data },
3999 { .compatible = "brcm,genet-v4", .data = &v4_plat_data },
4000 { .compatible = "brcm,genet-v5", .data = &v5_plat_data },
4001 { .compatible = "brcm,bcm2711-genet-v5", .data = &bcm2711_plat_data },
4002 { .compatible = "brcm,bcm7712-genet-v5", .data = &bcm7712_plat_data },
4003 { },
4004 };
4005 MODULE_DEVICE_TABLE(of, bcmgenet_match);
4006
bcmgenet_probe(struct platform_device * pdev)4007 static int bcmgenet_probe(struct platform_device *pdev)
4008 {
4009 const struct bcmgenet_plat_data *pdata;
4010 struct bcmgenet_priv *priv;
4011 struct net_device *dev;
4012 unsigned int i;
4013 int err = -EIO;
4014
4015 /* Up to GENET_MAX_MQ_CNT + 1 TX queues and RX queues */
4016 dev = alloc_etherdev_mqs(sizeof(*priv), GENET_MAX_MQ_CNT + 1,
4017 GENET_MAX_MQ_CNT + 1);
4018 if (!dev) {
4019 dev_err(&pdev->dev, "can't allocate net device\n");
4020 return -ENOMEM;
4021 }
4022
4023 priv = netdev_priv(dev);
4024 priv->irq0 = platform_get_irq(pdev, 0);
4025 if (priv->irq0 < 0) {
4026 err = priv->irq0;
4027 goto err;
4028 }
4029 priv->irq1 = platform_get_irq(pdev, 1);
4030 if (priv->irq1 < 0) {
4031 err = priv->irq1;
4032 goto err;
4033 }
4034 priv->wol_irq = platform_get_irq_optional(pdev, 2);
4035 if (priv->wol_irq == -EPROBE_DEFER) {
4036 err = priv->wol_irq;
4037 goto err;
4038 }
4039
4040 priv->base = devm_platform_ioremap_resource(pdev, 0);
4041 if (IS_ERR(priv->base)) {
4042 err = PTR_ERR(priv->base);
4043 goto err;
4044 }
4045
4046 spin_lock_init(&priv->reg_lock);
4047 spin_lock_init(&priv->lock);
4048
4049 /* Set default pause parameters */
4050 priv->autoneg_pause = 1;
4051 priv->tx_pause = 1;
4052 priv->rx_pause = 1;
4053
4054 SET_NETDEV_DEV(dev, &pdev->dev);
4055 dev_set_drvdata(&pdev->dev, dev);
4056 dev->watchdog_timeo = 2 * HZ;
4057 dev->ethtool_ops = &bcmgenet_ethtool_ops;
4058 dev->netdev_ops = &bcmgenet_netdev_ops;
4059
4060 priv->msg_enable = netif_msg_init(-1, GENET_MSG_DEFAULT);
4061
4062 /* Set default features */
4063 dev->features |= NETIF_F_SG | NETIF_F_HIGHDMA | NETIF_F_HW_CSUM |
4064 NETIF_F_RXCSUM;
4065 dev->hw_features |= dev->features;
4066 dev->vlan_features |= dev->features;
4067
4068 netdev_sw_irq_coalesce_default_on(dev);
4069
4070 /* Request the WOL interrupt and advertise suspend if available */
4071 priv->wol_irq_disabled = true;
4072 if (priv->wol_irq > 0) {
4073 err = devm_request_irq(&pdev->dev, priv->wol_irq,
4074 bcmgenet_wol_isr, 0, dev->name, priv);
4075 if (!err)
4076 device_set_wakeup_capable(&pdev->dev, 1);
4077 }
4078
4079 /* Set the needed headroom to account for any possible
4080 * features enabling/disabling at runtime
4081 */
4082 dev->needed_headroom += 64;
4083
4084 priv->dev = dev;
4085 priv->pdev = pdev;
4086
4087 pdata = device_get_match_data(&pdev->dev);
4088 if (pdata) {
4089 priv->version = pdata->version;
4090 priv->dma_max_burst_length = pdata->dma_max_burst_length;
4091 priv->flags = pdata->flags;
4092 }
4093
4094 priv->clk = devm_clk_get_optional(&priv->pdev->dev, "enet");
4095 if (IS_ERR(priv->clk)) {
4096 dev_dbg(&priv->pdev->dev, "failed to get enet clock\n");
4097 err = PTR_ERR(priv->clk);
4098 goto err;
4099 }
4100
4101 err = clk_prepare_enable(priv->clk);
4102 if (err)
4103 goto err;
4104
4105 bcmgenet_set_hw_params(priv);
4106
4107 err = -EIO;
4108 if (bcmgenet_has_40bits(priv))
4109 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(40));
4110 if (err)
4111 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
4112 if (err)
4113 goto err_clk_disable;
4114
4115 /* Mii wait queue */
4116 init_waitqueue_head(&priv->wq);
4117 bcmgenet_hfb_init(priv);
4118 INIT_WORK(&priv->bcmgenet_irq_work, bcmgenet_irq_task);
4119
4120 priv->clk_wol = devm_clk_get_optional(&priv->pdev->dev, "enet-wol");
4121 if (IS_ERR(priv->clk_wol)) {
4122 dev_dbg(&priv->pdev->dev, "failed to get enet-wol clock\n");
4123 err = PTR_ERR(priv->clk_wol);
4124 goto err_clk_disable;
4125 }
4126
4127 priv->clk_eee = devm_clk_get_optional(&priv->pdev->dev, "enet-eee");
4128 if (IS_ERR(priv->clk_eee)) {
4129 dev_dbg(&priv->pdev->dev, "failed to get enet-eee clock\n");
4130 err = PTR_ERR(priv->clk_eee);
4131 goto err_clk_disable;
4132 }
4133
4134 /* If this is an internal GPHY, power it on now, before UniMAC is
4135 * brought out of reset as absolutely no UniMAC activity is allowed
4136 */
4137 if (device_get_phy_mode(&pdev->dev) == PHY_INTERFACE_MODE_INTERNAL)
4138 bcmgenet_power_up(priv, GENET_POWER_PASSIVE);
4139
4140 if (device_get_ethdev_address(&pdev->dev, dev))
4141 if (has_acpi_companion(&pdev->dev)) {
4142 u8 addr[ETH_ALEN];
4143
4144 bcmgenet_get_hw_addr(priv, addr);
4145 eth_hw_addr_set(dev, addr);
4146 }
4147
4148 if (!is_valid_ether_addr(dev->dev_addr)) {
4149 dev_warn(&pdev->dev, "using random Ethernet MAC\n");
4150 eth_hw_addr_random(dev);
4151 }
4152
4153 reset_umac(priv);
4154
4155 err = bcmgenet_mii_init(dev);
4156 if (err)
4157 goto err_clk_disable;
4158
4159 /* setup number of real queues + 1 */
4160 netif_set_real_num_tx_queues(priv->dev, priv->hw_params->tx_queues + 1);
4161 netif_set_real_num_rx_queues(priv->dev, priv->hw_params->rx_queues + 1);
4162
4163 /* Set default coalescing parameters */
4164 for (i = 0; i <= priv->hw_params->rx_queues; i++) {
4165 priv->rx_rings[i].priv = priv;
4166 priv->rx_rings[i].index = i;
4167 priv->rx_rings[i].rx_max_coalesced_frames = 1;
4168 }
4169
4170 /* Initialize u64 stats seq counter for 32bit machines */
4171 for (i = 0; i <= GENET_MAX_MQ_CNT; i++) {
4172 u64_stats_init(&priv->rx_rings[i].stats64.syncp);
4173 u64_stats_init(&priv->tx_rings[i].stats64.syncp);
4174 }
4175
4176 /* libphy will determine the link state */
4177 netif_carrier_off(dev);
4178
4179 /* Turn off the main clock, WOL clock is handled separately */
4180 clk_disable_unprepare(priv->clk);
4181
4182 err = register_netdev(dev);
4183 if (err) {
4184 bcmgenet_mii_exit(dev);
4185 goto err;
4186 }
4187
4188 return err;
4189
4190 err_clk_disable:
4191 clk_disable_unprepare(priv->clk);
4192 err:
4193 free_netdev(dev);
4194 return err;
4195 }
4196
bcmgenet_remove(struct platform_device * pdev)4197 static void bcmgenet_remove(struct platform_device *pdev)
4198 {
4199 struct bcmgenet_priv *priv = dev_to_priv(&pdev->dev);
4200
4201 dev_set_drvdata(&pdev->dev, NULL);
4202 unregister_netdev(priv->dev);
4203 bcmgenet_mii_exit(priv->dev);
4204 free_netdev(priv->dev);
4205 }
4206
bcmgenet_shutdown(struct platform_device * pdev)4207 static void bcmgenet_shutdown(struct platform_device *pdev)
4208 {
4209 bcmgenet_remove(pdev);
4210 }
4211
4212 #ifdef CONFIG_PM_SLEEP
bcmgenet_resume_noirq(struct device * d)4213 static int bcmgenet_resume_noirq(struct device *d)
4214 {
4215 struct net_device *dev = dev_get_drvdata(d);
4216 struct bcmgenet_priv *priv = netdev_priv(dev);
4217 int ret;
4218 u32 reg;
4219
4220 if (!netif_running(dev))
4221 return 0;
4222
4223 /* Turn on the clock */
4224 ret = clk_prepare_enable(priv->clk);
4225 if (ret)
4226 return ret;
4227
4228 if (device_may_wakeup(d) && priv->wolopts) {
4229 /* Account for Wake-on-LAN events and clear those events
4230 * (Some devices need more time between enabling the clocks
4231 * and the interrupt register reflecting the wake event so
4232 * read the register twice)
4233 */
4234 reg = bcmgenet_intrl2_0_readl(priv, INTRL2_CPU_STAT);
4235 reg = bcmgenet_intrl2_0_readl(priv, INTRL2_CPU_STAT);
4236 if (reg & UMAC_IRQ_WAKE_EVENT)
4237 pm_wakeup_event(&priv->pdev->dev, 0);
4238
4239 /* From WOL-enabled suspend, switch to regular clock */
4240 if (!bcmgenet_power_up(priv, GENET_POWER_WOL_MAGIC))
4241 return 0;
4242
4243 /* Failed so fall through to reset MAC */
4244 }
4245
4246 /* If this is an internal GPHY, power it back on now, before UniMAC is
4247 * brought out of reset as absolutely no UniMAC activity is allowed
4248 */
4249 if (priv->internal_phy)
4250 bcmgenet_power_up(priv, GENET_POWER_PASSIVE);
4251
4252 /* take MAC out of reset */
4253 bcmgenet_umac_reset(priv);
4254
4255 return 0;
4256 }
4257
bcmgenet_resume(struct device * d)4258 static int bcmgenet_resume(struct device *d)
4259 {
4260 struct net_device *dev = dev_get_drvdata(d);
4261 struct bcmgenet_priv *priv = netdev_priv(dev);
4262 struct bcmgenet_rxnfc_rule *rule;
4263 int ret;
4264 u32 reg;
4265
4266 if (!netif_running(dev))
4267 return 0;
4268
4269 if (device_may_wakeup(d) && priv->wolopts) {
4270 reg = bcmgenet_umac_readl(priv, UMAC_CMD);
4271 if (reg & CMD_RX_EN) {
4272 /* Successfully exited WoL, just resume data flows */
4273 list_for_each_entry(rule, &priv->rxnfc_list, list)
4274 if (rule->state == BCMGENET_RXNFC_STATE_ENABLED)
4275 bcmgenet_hfb_enable_filter(priv,
4276 rule->fs.location + 1);
4277 bcmgenet_hfb_enable_filter(priv, 0);
4278 bcmgenet_set_rx_mode(dev);
4279 bcmgenet_enable_rx_napi(priv);
4280
4281 /* Reinitialize Tx flows */
4282 bcmgenet_tdma_disable(priv);
4283 bcmgenet_init_tx_queues(priv->dev);
4284 reg = bcmgenet_tdma_readl(priv, DMA_CTRL);
4285 reg |= DMA_EN;
4286 bcmgenet_tdma_writel(priv, reg, DMA_CTRL);
4287 bcmgenet_enable_tx_napi(priv);
4288
4289 bcmgenet_link_intr_enable(priv);
4290 phy_start_machine(dev->phydev);
4291
4292 netif_device_attach(dev);
4293 enable_irq(priv->irq1);
4294 return 0;
4295 }
4296 /* MAC was reset so complete bcmgenet_netif_stop() */
4297 umac_enable_set(priv, CMD_RX_EN | CMD_TX_EN, false);
4298 bcmgenet_rdma_disable(priv);
4299 bcmgenet_intr_disable(priv);
4300 bcmgenet_fini_dma(priv);
4301 enable_irq(priv->irq1);
4302 }
4303
4304 init_umac(priv);
4305
4306 phy_init_hw(dev->phydev);
4307
4308 /* Speed settings must be restored */
4309 genphy_config_aneg(dev->phydev);
4310 bcmgenet_mii_config(priv->dev, false);
4311
4312 /* Restore enabled features */
4313 bcmgenet_set_features(dev, dev->features);
4314
4315 bcmgenet_set_hw_addr(priv, dev->dev_addr);
4316
4317 /* Restore hardware filters */
4318 bcmgenet_hfb_restore(priv);
4319
4320 /* Reinitialize TDMA and RDMA and SW housekeeping */
4321 ret = bcmgenet_init_dma(priv, false);
4322 if (ret) {
4323 netdev_err(dev, "failed to initialize DMA\n");
4324 goto out_clk_disable;
4325 }
4326
4327 if (!device_may_wakeup(d))
4328 phy_resume(dev->phydev);
4329
4330 bcmgenet_netif_start(dev);
4331
4332 netif_device_attach(dev);
4333
4334 return 0;
4335
4336 out_clk_disable:
4337 if (priv->internal_phy)
4338 bcmgenet_power_down(priv, GENET_POWER_PASSIVE);
4339 clk_disable_unprepare(priv->clk);
4340 return ret;
4341 }
4342
bcmgenet_suspend(struct device * d)4343 static int bcmgenet_suspend(struct device *d)
4344 {
4345 struct net_device *dev = dev_get_drvdata(d);
4346 struct bcmgenet_priv *priv = netdev_priv(dev);
4347 struct bcmgenet_rxnfc_rule *rule;
4348 u32 reg, hfb_enable = 0;
4349
4350 if (!netif_running(dev))
4351 return 0;
4352
4353 netif_device_detach(dev);
4354
4355 if (device_may_wakeup(d) && priv->wolopts) {
4356 /* Stop completion polling before it can wake a stopped queue */
4357 bcmgenet_disable_tx_napi(priv);
4358 netif_tx_disable(dev);
4359
4360 /* Suspend non-wake Rx data flows */
4361 if (priv->wolopts & WAKE_FILTER)
4362 list_for_each_entry(rule, &priv->rxnfc_list, list)
4363 if (rule->fs.ring_cookie == RX_CLS_FLOW_WAKE &&
4364 rule->state == BCMGENET_RXNFC_STATE_ENABLED)
4365 hfb_enable |= 1 << rule->fs.location;
4366 reg = bcmgenet_hfb_reg_readl(priv, HFB_CTRL);
4367 if (GENET_IS_V1(priv) || GENET_IS_V2(priv)) {
4368 reg &= ~RBUF_HFB_FILTER_EN_MASK;
4369 reg |= hfb_enable << (RBUF_HFB_FILTER_EN_SHIFT + 1);
4370 } else {
4371 bcmgenet_hfb_reg_writel(priv, hfb_enable << 1,
4372 HFB_FLT_ENABLE_V3PLUS + 4);
4373 }
4374 if (!hfb_enable)
4375 reg &= ~RBUF_HFB_EN;
4376 bcmgenet_hfb_reg_writel(priv, reg, HFB_CTRL);
4377
4378 /* Clear any old filter matches so only new matches wake */
4379 bcmgenet_intrl2_0_writel(priv, 0xFFFFFFFF, INTRL2_CPU_MASK_SET);
4380 bcmgenet_intrl2_0_writel(priv, 0xFFFFFFFF, INTRL2_CPU_CLEAR);
4381
4382 if (-ETIMEDOUT == bcmgenet_tdma_disable(priv))
4383 netdev_warn(priv->dev,
4384 "Timed out while disabling TX DMA\n");
4385
4386 bcmgenet_disable_rx_napi(priv);
4387 disable_irq(priv->irq1);
4388 bcmgenet_tx_reclaim_all(dev);
4389 bcmgenet_fini_tx_napi(priv);
4390 } else {
4391 /* Teardown the interface */
4392 bcmgenet_netif_stop(dev, true);
4393 }
4394
4395 return 0;
4396 }
4397
bcmgenet_suspend_noirq(struct device * d)4398 static int bcmgenet_suspend_noirq(struct device *d)
4399 {
4400 struct net_device *dev = dev_get_drvdata(d);
4401 struct bcmgenet_priv *priv = netdev_priv(dev);
4402 int ret = 0;
4403
4404 if (!netif_running(dev))
4405 return 0;
4406
4407 /* Prepare the device for Wake-on-LAN and switch to the slow clock */
4408 if (device_may_wakeup(d) && priv->wolopts)
4409 ret = bcmgenet_power_down(priv, GENET_POWER_WOL_MAGIC);
4410 else if (priv->internal_phy)
4411 ret = bcmgenet_power_down(priv, GENET_POWER_PASSIVE);
4412
4413 /* Let the framework handle resumption and leave the clocks on */
4414 if (ret)
4415 return ret;
4416
4417 /* Turn off the clocks */
4418 clk_disable_unprepare(priv->clk);
4419
4420 return 0;
4421 }
4422 #else
4423 #define bcmgenet_suspend NULL
4424 #define bcmgenet_suspend_noirq NULL
4425 #define bcmgenet_resume NULL
4426 #define bcmgenet_resume_noirq NULL
4427 #endif /* CONFIG_PM_SLEEP */
4428
4429 static const struct dev_pm_ops bcmgenet_pm_ops = {
4430 .suspend = bcmgenet_suspend,
4431 .suspend_noirq = bcmgenet_suspend_noirq,
4432 .resume = bcmgenet_resume,
4433 .resume_noirq = bcmgenet_resume_noirq,
4434 };
4435
4436 static const struct acpi_device_id genet_acpi_match[] = {
4437 { "BCM6E4E", (kernel_ulong_t)&bcm2711_plat_data },
4438 { },
4439 };
4440 MODULE_DEVICE_TABLE(acpi, genet_acpi_match);
4441
4442 static struct platform_driver bcmgenet_driver = {
4443 .probe = bcmgenet_probe,
4444 .remove = bcmgenet_remove,
4445 .shutdown = bcmgenet_shutdown,
4446 .driver = {
4447 .name = "bcmgenet",
4448 .of_match_table = bcmgenet_match,
4449 .pm = &bcmgenet_pm_ops,
4450 .acpi_match_table = genet_acpi_match,
4451 },
4452 };
4453 module_platform_driver(bcmgenet_driver);
4454
4455 MODULE_AUTHOR("Broadcom Corporation");
4456 MODULE_DESCRIPTION("Broadcom GENET Ethernet controller driver");
4457 MODULE_ALIAS("platform:bcmgenet");
4458 MODULE_LICENSE("GPL");
4459 MODULE_SOFTDEP("pre: mdio-bcm-unimac");
4460