1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Broadcom GENET (Gigabit Ethernet) controller driver
4 *
5 * Copyright (c) 2014-2025 Broadcom
6 */
7
8 #define pr_fmt(fmt) "bcmgenet: " fmt
9
10 #include <linux/acpi.h>
11 #include <linux/kernel.h>
12 #include <linux/module.h>
13 #include <linux/sched.h>
14 #include <linux/types.h>
15 #include <linux/fcntl.h>
16 #include <linux/interrupt.h>
17 #include <linux/string.h>
18 #include <linux/if_ether.h>
19 #include <linux/init.h>
20 #include <linux/errno.h>
21 #include <linux/delay.h>
22 #include <linux/platform_device.h>
23 #include <linux/dma-mapping.h>
24 #include <linux/pm.h>
25 #include <linux/clk.h>
26 #include <net/arp.h>
27
28 #include <linux/mii.h>
29 #include <linux/ethtool.h>
30 #include <linux/netdevice.h>
31 #include <linux/inetdevice.h>
32 #include <linux/etherdevice.h>
33 #include <linux/skbuff.h>
34 #include <linux/in.h>
35 #include <linux/ip.h>
36 #include <linux/ipv6.h>
37 #include <linux/phy.h>
38
39 #include <linux/unaligned.h>
40
41 #include "bcmgenet.h"
42
43 /* Default highest priority queue for multi queue support */
44 #define GENET_Q1_PRIORITY 0
45 #define GENET_Q0_PRIORITY 1
46
47 #define GENET_Q0_RX_BD_CNT \
48 (TOTAL_DESC - priv->hw_params->rx_queues * priv->hw_params->rx_bds_per_q)
49 #define GENET_Q0_TX_BD_CNT \
50 (TOTAL_DESC - priv->hw_params->tx_queues * priv->hw_params->tx_bds_per_q)
51
52 #define RX_BUF_LENGTH 2048
53 #define SKB_ALIGNMENT 32
54
55 /* Tx/Rx DMA register offset, skip 256 descriptors */
56 #define WORDS_PER_BD(p) (p->hw_params->words_per_bd)
57 #define DMA_DESC_SIZE (WORDS_PER_BD(priv) * sizeof(u32))
58
59 #define GENET_TDMA_REG_OFF (priv->hw_params->tdma_offset + \
60 TOTAL_DESC * DMA_DESC_SIZE)
61
62 #define GENET_RDMA_REG_OFF (priv->hw_params->rdma_offset + \
63 TOTAL_DESC * DMA_DESC_SIZE)
64
65 /* Forward declarations */
66 static void bcmgenet_set_rx_mode(struct net_device *dev);
67
bcmgenet_writel(u32 value,void __iomem * offset)68 static inline void bcmgenet_writel(u32 value, void __iomem *offset)
69 {
70 /* MIPS chips strapped for BE will automagically configure the
71 * peripheral registers for CPU-native byte order.
72 */
73 if (IS_ENABLED(CONFIG_MIPS) && IS_ENABLED(CONFIG_CPU_BIG_ENDIAN))
74 __raw_writel(value, offset);
75 else
76 writel_relaxed(value, offset);
77 }
78
bcmgenet_readl(void __iomem * offset)79 static inline u32 bcmgenet_readl(void __iomem *offset)
80 {
81 if (IS_ENABLED(CONFIG_MIPS) && IS_ENABLED(CONFIG_CPU_BIG_ENDIAN))
82 return __raw_readl(offset);
83 else
84 return readl_relaxed(offset);
85 }
86
dmadesc_set_length_status(struct bcmgenet_priv * priv,void __iomem * d,u32 value)87 static inline void dmadesc_set_length_status(struct bcmgenet_priv *priv,
88 void __iomem *d, u32 value)
89 {
90 bcmgenet_writel(value, d + DMA_DESC_LENGTH_STATUS);
91 }
92
dmadesc_set_addr(struct bcmgenet_priv * priv,void __iomem * d,dma_addr_t addr)93 static inline void dmadesc_set_addr(struct bcmgenet_priv *priv,
94 void __iomem *d,
95 dma_addr_t addr)
96 {
97 bcmgenet_writel(lower_32_bits(addr), d + DMA_DESC_ADDRESS_LO);
98
99 /* Register writes to GISB bus can take couple hundred nanoseconds
100 * and are done for each packet, save these expensive writes unless
101 * the platform is explicitly configured for 64-bits/LPAE.
102 */
103 #ifdef CONFIG_PHYS_ADDR_T_64BIT
104 if (bcmgenet_has_40bits(priv))
105 bcmgenet_writel(upper_32_bits(addr), d + DMA_DESC_ADDRESS_HI);
106 #endif
107 }
108
109 /* Combined address + length/status setter */
dmadesc_set(struct bcmgenet_priv * priv,void __iomem * d,dma_addr_t addr,u32 val)110 static inline void dmadesc_set(struct bcmgenet_priv *priv,
111 void __iomem *d, dma_addr_t addr, u32 val)
112 {
113 dmadesc_set_addr(priv, d, addr);
114 dmadesc_set_length_status(priv, d, val);
115 }
116
117 #define GENET_VER_FMT "%1d.%1d EPHY: 0x%04x"
118
119 #define GENET_MSG_DEFAULT (NETIF_MSG_DRV | NETIF_MSG_PROBE | \
120 NETIF_MSG_LINK)
121
bcmgenet_rbuf_ctrl_get(struct bcmgenet_priv * priv)122 static inline u32 bcmgenet_rbuf_ctrl_get(struct bcmgenet_priv *priv)
123 {
124 if (GENET_IS_V1(priv))
125 return bcmgenet_rbuf_readl(priv, RBUF_FLUSH_CTRL_V1);
126 else
127 return bcmgenet_sys_readl(priv, SYS_RBUF_FLUSH_CTRL);
128 }
129
bcmgenet_rbuf_ctrl_set(struct bcmgenet_priv * priv,u32 val)130 static inline void bcmgenet_rbuf_ctrl_set(struct bcmgenet_priv *priv, u32 val)
131 {
132 if (GENET_IS_V1(priv))
133 bcmgenet_rbuf_writel(priv, val, RBUF_FLUSH_CTRL_V1);
134 else
135 bcmgenet_sys_writel(priv, val, SYS_RBUF_FLUSH_CTRL);
136 }
137
138 /* These macros are defined to deal with register map change
139 * between GENET1.1 and GENET2. Only those currently being used
140 * by driver are defined.
141 */
bcmgenet_tbuf_ctrl_get(struct bcmgenet_priv * priv)142 static inline u32 bcmgenet_tbuf_ctrl_get(struct bcmgenet_priv *priv)
143 {
144 if (GENET_IS_V1(priv))
145 return bcmgenet_rbuf_readl(priv, TBUF_CTRL_V1);
146 else
147 return bcmgenet_readl(priv->base +
148 priv->hw_params->tbuf_offset + TBUF_CTRL);
149 }
150
bcmgenet_tbuf_ctrl_set(struct bcmgenet_priv * priv,u32 val)151 static inline void bcmgenet_tbuf_ctrl_set(struct bcmgenet_priv *priv, u32 val)
152 {
153 if (GENET_IS_V1(priv))
154 bcmgenet_rbuf_writel(priv, val, TBUF_CTRL_V1);
155 else
156 bcmgenet_writel(val, priv->base +
157 priv->hw_params->tbuf_offset + TBUF_CTRL);
158 }
159
bcmgenet_bp_mc_get(struct bcmgenet_priv * priv)160 static inline u32 bcmgenet_bp_mc_get(struct bcmgenet_priv *priv)
161 {
162 if (GENET_IS_V1(priv))
163 return bcmgenet_rbuf_readl(priv, TBUF_BP_MC_V1);
164 else
165 return bcmgenet_readl(priv->base +
166 priv->hw_params->tbuf_offset + TBUF_BP_MC);
167 }
168
bcmgenet_bp_mc_set(struct bcmgenet_priv * priv,u32 val)169 static inline void bcmgenet_bp_mc_set(struct bcmgenet_priv *priv, u32 val)
170 {
171 if (GENET_IS_V1(priv))
172 bcmgenet_rbuf_writel(priv, val, TBUF_BP_MC_V1);
173 else
174 bcmgenet_writel(val, priv->base +
175 priv->hw_params->tbuf_offset + TBUF_BP_MC);
176 }
177
178 /* RX/TX DMA register accessors */
179 enum dma_reg {
180 DMA_RING_CFG = 0,
181 DMA_CTRL,
182 DMA_STATUS,
183 DMA_SCB_BURST_SIZE,
184 DMA_ARB_CTRL,
185 DMA_PRIORITY_0,
186 DMA_PRIORITY_1,
187 DMA_PRIORITY_2,
188 DMA_INDEX2RING_0,
189 DMA_INDEX2RING_1,
190 DMA_INDEX2RING_2,
191 DMA_INDEX2RING_3,
192 DMA_INDEX2RING_4,
193 DMA_INDEX2RING_5,
194 DMA_INDEX2RING_6,
195 DMA_INDEX2RING_7,
196 DMA_RING0_TIMEOUT,
197 DMA_RING1_TIMEOUT,
198 DMA_RING2_TIMEOUT,
199 DMA_RING3_TIMEOUT,
200 DMA_RING4_TIMEOUT,
201 DMA_RING5_TIMEOUT,
202 DMA_RING6_TIMEOUT,
203 DMA_RING7_TIMEOUT,
204 DMA_RING8_TIMEOUT,
205 DMA_RING9_TIMEOUT,
206 DMA_RING10_TIMEOUT,
207 DMA_RING11_TIMEOUT,
208 DMA_RING12_TIMEOUT,
209 DMA_RING13_TIMEOUT,
210 DMA_RING14_TIMEOUT,
211 DMA_RING15_TIMEOUT,
212 DMA_RING16_TIMEOUT,
213 };
214
215 static const u8 bcmgenet_dma_regs_v3plus[] = {
216 [DMA_RING_CFG] = 0x00,
217 [DMA_CTRL] = 0x04,
218 [DMA_STATUS] = 0x08,
219 [DMA_SCB_BURST_SIZE] = 0x0C,
220 [DMA_ARB_CTRL] = 0x2C,
221 [DMA_PRIORITY_0] = 0x30,
222 [DMA_PRIORITY_1] = 0x34,
223 [DMA_PRIORITY_2] = 0x38,
224 [DMA_RING0_TIMEOUT] = 0x2C,
225 [DMA_RING1_TIMEOUT] = 0x30,
226 [DMA_RING2_TIMEOUT] = 0x34,
227 [DMA_RING3_TIMEOUT] = 0x38,
228 [DMA_RING4_TIMEOUT] = 0x3c,
229 [DMA_RING5_TIMEOUT] = 0x40,
230 [DMA_RING6_TIMEOUT] = 0x44,
231 [DMA_RING7_TIMEOUT] = 0x48,
232 [DMA_RING8_TIMEOUT] = 0x4c,
233 [DMA_RING9_TIMEOUT] = 0x50,
234 [DMA_RING10_TIMEOUT] = 0x54,
235 [DMA_RING11_TIMEOUT] = 0x58,
236 [DMA_RING12_TIMEOUT] = 0x5c,
237 [DMA_RING13_TIMEOUT] = 0x60,
238 [DMA_RING14_TIMEOUT] = 0x64,
239 [DMA_RING15_TIMEOUT] = 0x68,
240 [DMA_RING16_TIMEOUT] = 0x6C,
241 [DMA_INDEX2RING_0] = 0x70,
242 [DMA_INDEX2RING_1] = 0x74,
243 [DMA_INDEX2RING_2] = 0x78,
244 [DMA_INDEX2RING_3] = 0x7C,
245 [DMA_INDEX2RING_4] = 0x80,
246 [DMA_INDEX2RING_5] = 0x84,
247 [DMA_INDEX2RING_6] = 0x88,
248 [DMA_INDEX2RING_7] = 0x8C,
249 };
250
251 static const u8 bcmgenet_dma_regs_v2[] = {
252 [DMA_RING_CFG] = 0x00,
253 [DMA_CTRL] = 0x04,
254 [DMA_STATUS] = 0x08,
255 [DMA_SCB_BURST_SIZE] = 0x0C,
256 [DMA_ARB_CTRL] = 0x30,
257 [DMA_PRIORITY_0] = 0x34,
258 [DMA_PRIORITY_1] = 0x38,
259 [DMA_PRIORITY_2] = 0x3C,
260 [DMA_RING0_TIMEOUT] = 0x2C,
261 [DMA_RING1_TIMEOUT] = 0x30,
262 [DMA_RING2_TIMEOUT] = 0x34,
263 [DMA_RING3_TIMEOUT] = 0x38,
264 [DMA_RING4_TIMEOUT] = 0x3c,
265 [DMA_RING5_TIMEOUT] = 0x40,
266 [DMA_RING6_TIMEOUT] = 0x44,
267 [DMA_RING7_TIMEOUT] = 0x48,
268 [DMA_RING8_TIMEOUT] = 0x4c,
269 [DMA_RING9_TIMEOUT] = 0x50,
270 [DMA_RING10_TIMEOUT] = 0x54,
271 [DMA_RING11_TIMEOUT] = 0x58,
272 [DMA_RING12_TIMEOUT] = 0x5c,
273 [DMA_RING13_TIMEOUT] = 0x60,
274 [DMA_RING14_TIMEOUT] = 0x64,
275 [DMA_RING15_TIMEOUT] = 0x68,
276 [DMA_RING16_TIMEOUT] = 0x6C,
277 };
278
279 static const u8 bcmgenet_dma_regs_v1[] = {
280 [DMA_CTRL] = 0x00,
281 [DMA_STATUS] = 0x04,
282 [DMA_SCB_BURST_SIZE] = 0x0C,
283 [DMA_ARB_CTRL] = 0x30,
284 [DMA_PRIORITY_0] = 0x34,
285 [DMA_PRIORITY_1] = 0x38,
286 [DMA_PRIORITY_2] = 0x3C,
287 [DMA_RING0_TIMEOUT] = 0x2C,
288 [DMA_RING1_TIMEOUT] = 0x30,
289 [DMA_RING2_TIMEOUT] = 0x34,
290 [DMA_RING3_TIMEOUT] = 0x38,
291 [DMA_RING4_TIMEOUT] = 0x3c,
292 [DMA_RING5_TIMEOUT] = 0x40,
293 [DMA_RING6_TIMEOUT] = 0x44,
294 [DMA_RING7_TIMEOUT] = 0x48,
295 [DMA_RING8_TIMEOUT] = 0x4c,
296 [DMA_RING9_TIMEOUT] = 0x50,
297 [DMA_RING10_TIMEOUT] = 0x54,
298 [DMA_RING11_TIMEOUT] = 0x58,
299 [DMA_RING12_TIMEOUT] = 0x5c,
300 [DMA_RING13_TIMEOUT] = 0x60,
301 [DMA_RING14_TIMEOUT] = 0x64,
302 [DMA_RING15_TIMEOUT] = 0x68,
303 [DMA_RING16_TIMEOUT] = 0x6C,
304 };
305
306 /* Set at runtime once bcmgenet version is known */
307 static const u8 *bcmgenet_dma_regs;
308
dev_to_priv(struct device * dev)309 static inline struct bcmgenet_priv *dev_to_priv(struct device *dev)
310 {
311 return netdev_priv(dev_get_drvdata(dev));
312 }
313
bcmgenet_tdma_readl(struct bcmgenet_priv * priv,enum dma_reg r)314 static inline u32 bcmgenet_tdma_readl(struct bcmgenet_priv *priv,
315 enum dma_reg r)
316 {
317 return bcmgenet_readl(priv->base + GENET_TDMA_REG_OFF +
318 DMA_RINGS_SIZE + bcmgenet_dma_regs[r]);
319 }
320
bcmgenet_tdma_writel(struct bcmgenet_priv * priv,u32 val,enum dma_reg r)321 static inline void bcmgenet_tdma_writel(struct bcmgenet_priv *priv,
322 u32 val, enum dma_reg r)
323 {
324 bcmgenet_writel(val, priv->base + GENET_TDMA_REG_OFF +
325 DMA_RINGS_SIZE + bcmgenet_dma_regs[r]);
326 }
327
bcmgenet_rdma_readl(struct bcmgenet_priv * priv,enum dma_reg r)328 static inline u32 bcmgenet_rdma_readl(struct bcmgenet_priv *priv,
329 enum dma_reg r)
330 {
331 return bcmgenet_readl(priv->base + GENET_RDMA_REG_OFF +
332 DMA_RINGS_SIZE + bcmgenet_dma_regs[r]);
333 }
334
bcmgenet_rdma_writel(struct bcmgenet_priv * priv,u32 val,enum dma_reg r)335 static inline void bcmgenet_rdma_writel(struct bcmgenet_priv *priv,
336 u32 val, enum dma_reg r)
337 {
338 bcmgenet_writel(val, priv->base + GENET_RDMA_REG_OFF +
339 DMA_RINGS_SIZE + bcmgenet_dma_regs[r]);
340 }
341
342 /* RDMA/TDMA ring registers and accessors
343 * we merge the common fields and just prefix with T/D the registers
344 * having different meaning depending on the direction
345 */
346 enum dma_ring_reg {
347 TDMA_READ_PTR = 0,
348 RDMA_WRITE_PTR = TDMA_READ_PTR,
349 TDMA_READ_PTR_HI,
350 RDMA_WRITE_PTR_HI = TDMA_READ_PTR_HI,
351 TDMA_CONS_INDEX,
352 RDMA_PROD_INDEX = TDMA_CONS_INDEX,
353 TDMA_PROD_INDEX,
354 RDMA_CONS_INDEX = TDMA_PROD_INDEX,
355 DMA_RING_BUF_SIZE,
356 DMA_START_ADDR,
357 DMA_START_ADDR_HI,
358 DMA_END_ADDR,
359 DMA_END_ADDR_HI,
360 DMA_MBUF_DONE_THRESH,
361 TDMA_FLOW_PERIOD,
362 RDMA_XON_XOFF_THRESH = TDMA_FLOW_PERIOD,
363 TDMA_WRITE_PTR,
364 RDMA_READ_PTR = TDMA_WRITE_PTR,
365 TDMA_WRITE_PTR_HI,
366 RDMA_READ_PTR_HI = TDMA_WRITE_PTR_HI
367 };
368
369 /* GENET v4 supports 40-bits pointer addressing
370 * for obvious reasons the LO and HI word parts
371 * are contiguous, but this offsets the other
372 * registers.
373 */
374 static const u8 genet_dma_ring_regs_v4[] = {
375 [TDMA_READ_PTR] = 0x00,
376 [TDMA_READ_PTR_HI] = 0x04,
377 [TDMA_CONS_INDEX] = 0x08,
378 [TDMA_PROD_INDEX] = 0x0C,
379 [DMA_RING_BUF_SIZE] = 0x10,
380 [DMA_START_ADDR] = 0x14,
381 [DMA_START_ADDR_HI] = 0x18,
382 [DMA_END_ADDR] = 0x1C,
383 [DMA_END_ADDR_HI] = 0x20,
384 [DMA_MBUF_DONE_THRESH] = 0x24,
385 [TDMA_FLOW_PERIOD] = 0x28,
386 [TDMA_WRITE_PTR] = 0x2C,
387 [TDMA_WRITE_PTR_HI] = 0x30,
388 };
389
390 static const u8 genet_dma_ring_regs_v123[] = {
391 [TDMA_READ_PTR] = 0x00,
392 [TDMA_CONS_INDEX] = 0x04,
393 [TDMA_PROD_INDEX] = 0x08,
394 [DMA_RING_BUF_SIZE] = 0x0C,
395 [DMA_START_ADDR] = 0x10,
396 [DMA_END_ADDR] = 0x14,
397 [DMA_MBUF_DONE_THRESH] = 0x18,
398 [TDMA_FLOW_PERIOD] = 0x1C,
399 [TDMA_WRITE_PTR] = 0x20,
400 };
401
402 /* Set at runtime once GENET version is known */
403 static const u8 *genet_dma_ring_regs;
404
bcmgenet_tdma_ring_readl(struct bcmgenet_priv * priv,unsigned int ring,enum dma_ring_reg r)405 static inline u32 bcmgenet_tdma_ring_readl(struct bcmgenet_priv *priv,
406 unsigned int ring,
407 enum dma_ring_reg r)
408 {
409 return bcmgenet_readl(priv->base + GENET_TDMA_REG_OFF +
410 (DMA_RING_SIZE * ring) +
411 genet_dma_ring_regs[r]);
412 }
413
bcmgenet_tdma_ring_writel(struct bcmgenet_priv * priv,unsigned int ring,u32 val,enum dma_ring_reg r)414 static inline void bcmgenet_tdma_ring_writel(struct bcmgenet_priv *priv,
415 unsigned int ring, u32 val,
416 enum dma_ring_reg r)
417 {
418 bcmgenet_writel(val, priv->base + GENET_TDMA_REG_OFF +
419 (DMA_RING_SIZE * ring) +
420 genet_dma_ring_regs[r]);
421 }
422
bcmgenet_rdma_ring_readl(struct bcmgenet_priv * priv,unsigned int ring,enum dma_ring_reg r)423 static inline u32 bcmgenet_rdma_ring_readl(struct bcmgenet_priv *priv,
424 unsigned int ring,
425 enum dma_ring_reg r)
426 {
427 return bcmgenet_readl(priv->base + GENET_RDMA_REG_OFF +
428 (DMA_RING_SIZE * ring) +
429 genet_dma_ring_regs[r]);
430 }
431
bcmgenet_rdma_ring_writel(struct bcmgenet_priv * priv,unsigned int ring,u32 val,enum dma_ring_reg r)432 static inline void bcmgenet_rdma_ring_writel(struct bcmgenet_priv *priv,
433 unsigned int ring, u32 val,
434 enum dma_ring_reg r)
435 {
436 bcmgenet_writel(val, priv->base + GENET_RDMA_REG_OFF +
437 (DMA_RING_SIZE * ring) +
438 genet_dma_ring_regs[r]);
439 }
440
bcmgenet_hfb_enable_filter(struct bcmgenet_priv * priv,u32 f_index)441 static void bcmgenet_hfb_enable_filter(struct bcmgenet_priv *priv, u32 f_index)
442 {
443 u32 offset;
444 u32 reg;
445
446 if (GENET_IS_V1(priv) || GENET_IS_V2(priv)) {
447 reg = bcmgenet_hfb_reg_readl(priv, HFB_CTRL);
448 reg |= (1 << ((f_index % 32) + RBUF_HFB_FILTER_EN_SHIFT)) |
449 RBUF_HFB_EN;
450 bcmgenet_hfb_reg_writel(priv, reg, HFB_CTRL);
451 } else {
452 offset = HFB_FLT_ENABLE_V3PLUS + (f_index < 32) * sizeof(u32);
453 reg = bcmgenet_hfb_reg_readl(priv, offset);
454 reg |= (1 << (f_index % 32));
455 bcmgenet_hfb_reg_writel(priv, reg, offset);
456 reg = bcmgenet_hfb_reg_readl(priv, HFB_CTRL);
457 reg |= RBUF_HFB_EN;
458 bcmgenet_hfb_reg_writel(priv, reg, HFB_CTRL);
459 }
460 }
461
bcmgenet_hfb_disable_filter(struct bcmgenet_priv * priv,u32 f_index)462 static void bcmgenet_hfb_disable_filter(struct bcmgenet_priv *priv, u32 f_index)
463 {
464 u32 offset, reg, reg1;
465
466 if (GENET_IS_V1(priv) || GENET_IS_V2(priv)) {
467 reg = bcmgenet_hfb_reg_readl(priv, HFB_CTRL);
468 reg &= ~(1 << ((f_index % 32) + RBUF_HFB_FILTER_EN_SHIFT));
469 if (!(reg & RBUF_HFB_FILTER_EN_MASK))
470 reg &= ~RBUF_HFB_EN;
471 bcmgenet_hfb_reg_writel(priv, reg, HFB_CTRL);
472 } else {
473 offset = HFB_FLT_ENABLE_V3PLUS;
474 reg = bcmgenet_hfb_reg_readl(priv, offset);
475 reg1 = bcmgenet_hfb_reg_readl(priv, offset + sizeof(u32));
476 if (f_index < 32) {
477 reg1 &= ~(1 << (f_index % 32));
478 bcmgenet_hfb_reg_writel(priv, reg1, offset + sizeof(u32));
479 } else {
480 reg &= ~(1 << (f_index % 32));
481 bcmgenet_hfb_reg_writel(priv, reg, offset);
482 }
483 if (!reg && !reg1) {
484 reg = bcmgenet_hfb_reg_readl(priv, HFB_CTRL);
485 reg &= ~RBUF_HFB_EN;
486 bcmgenet_hfb_reg_writel(priv, reg, HFB_CTRL);
487 }
488 }
489 }
490
bcmgenet_hfb_set_filter_rx_queue_mapping(struct bcmgenet_priv * priv,u32 f_index,u32 rx_queue)491 static void bcmgenet_hfb_set_filter_rx_queue_mapping(struct bcmgenet_priv *priv,
492 u32 f_index, u32 rx_queue)
493 {
494 u32 offset;
495 u32 reg;
496
497 if (GENET_IS_V1(priv) || GENET_IS_V2(priv))
498 return;
499
500 offset = f_index / 8;
501 reg = bcmgenet_rdma_readl(priv, DMA_INDEX2RING_0 + offset);
502 reg &= ~(0xF << (4 * (f_index % 8)));
503 reg |= ((rx_queue & 0xF) << (4 * (f_index % 8)));
504 bcmgenet_rdma_writel(priv, reg, DMA_INDEX2RING_0 + offset);
505 }
506
bcmgenet_hfb_set_filter_length(struct bcmgenet_priv * priv,u32 f_index,u32 f_length)507 static void bcmgenet_hfb_set_filter_length(struct bcmgenet_priv *priv,
508 u32 f_index, u32 f_length)
509 {
510 u32 offset;
511 u32 reg;
512
513 if (GENET_IS_V1(priv) || GENET_IS_V2(priv))
514 offset = HFB_FLT_LEN_V2;
515 else
516 offset = HFB_FLT_LEN_V3PLUS;
517
518 offset += sizeof(u32) *
519 ((priv->hw_params->hfb_filter_cnt - 1 - f_index) / 4);
520 reg = bcmgenet_hfb_reg_readl(priv, offset);
521 reg &= ~(0xFF << (8 * (f_index % 4)));
522 reg |= ((f_length & 0xFF) << (8 * (f_index % 4)));
523 bcmgenet_hfb_reg_writel(priv, reg, offset);
524 }
525
bcmgenet_hfb_validate_mask(void * mask,size_t size)526 static int bcmgenet_hfb_validate_mask(void *mask, size_t size)
527 {
528 while (size) {
529 switch (*(unsigned char *)mask++) {
530 case 0x00:
531 case 0x0f:
532 case 0xf0:
533 case 0xff:
534 size--;
535 continue;
536 default:
537 return -EINVAL;
538 }
539 }
540
541 return 0;
542 }
543
544 #define VALIDATE_MASK(x) \
545 bcmgenet_hfb_validate_mask(&(x), sizeof(x))
546
bcmgenet_hfb_insert_data(struct bcmgenet_priv * priv,u32 f_index,u32 offset,void * val,void * mask,size_t size)547 static int bcmgenet_hfb_insert_data(struct bcmgenet_priv *priv, u32 f_index,
548 u32 offset, void *val, void *mask,
549 size_t size)
550 {
551 u32 index, tmp;
552
553 index = f_index * priv->hw_params->hfb_filter_size + offset / 2;
554 tmp = bcmgenet_hfb_readl(priv, index * sizeof(u32));
555
556 while (size--) {
557 if (offset++ & 1) {
558 tmp &= ~0x300FF;
559 tmp |= (*(unsigned char *)val++);
560 switch ((*(unsigned char *)mask++)) {
561 case 0xFF:
562 tmp |= 0x30000;
563 break;
564 case 0xF0:
565 tmp |= 0x20000;
566 break;
567 case 0x0F:
568 tmp |= 0x10000;
569 break;
570 }
571 bcmgenet_hfb_writel(priv, tmp, index++ * sizeof(u32));
572 if (size)
573 tmp = bcmgenet_hfb_readl(priv,
574 index * sizeof(u32));
575 } else {
576 tmp &= ~0xCFF00;
577 tmp |= (*(unsigned char *)val++) << 8;
578 switch ((*(unsigned char *)mask++)) {
579 case 0xFF:
580 tmp |= 0xC0000;
581 break;
582 case 0xF0:
583 tmp |= 0x80000;
584 break;
585 case 0x0F:
586 tmp |= 0x40000;
587 break;
588 }
589 if (!size)
590 bcmgenet_hfb_writel(priv, tmp, index * sizeof(u32));
591 }
592 }
593
594 return 0;
595 }
596
bcmgenet_hfb_create_rxnfc_filter(struct bcmgenet_priv * priv,struct bcmgenet_rxnfc_rule * rule)597 static void bcmgenet_hfb_create_rxnfc_filter(struct bcmgenet_priv *priv,
598 struct bcmgenet_rxnfc_rule *rule)
599 {
600 struct ethtool_rx_flow_spec *fs = &rule->fs;
601 u32 offset = 0, f_length = 0, f, q;
602 u8 val_8, mask_8;
603 __be16 val_16;
604 u16 mask_16;
605 size_t size;
606
607 f = fs->location + 1;
608 if (fs->flow_type & FLOW_MAC_EXT) {
609 bcmgenet_hfb_insert_data(priv, f, 0,
610 &fs->h_ext.h_dest, &fs->m_ext.h_dest,
611 sizeof(fs->h_ext.h_dest));
612 }
613
614 if (fs->flow_type & FLOW_EXT) {
615 if (fs->m_ext.vlan_etype ||
616 fs->m_ext.vlan_tci) {
617 bcmgenet_hfb_insert_data(priv, f, 12,
618 &fs->h_ext.vlan_etype,
619 &fs->m_ext.vlan_etype,
620 sizeof(fs->h_ext.vlan_etype));
621 bcmgenet_hfb_insert_data(priv, f, 14,
622 &fs->h_ext.vlan_tci,
623 &fs->m_ext.vlan_tci,
624 sizeof(fs->h_ext.vlan_tci));
625 offset += VLAN_HLEN;
626 f_length += DIV_ROUND_UP(VLAN_HLEN, 2);
627 }
628 }
629
630 switch (fs->flow_type & ~(FLOW_EXT | FLOW_MAC_EXT)) {
631 case ETHER_FLOW:
632 f_length += DIV_ROUND_UP(ETH_HLEN, 2);
633 bcmgenet_hfb_insert_data(priv, f, 0,
634 &fs->h_u.ether_spec.h_dest,
635 &fs->m_u.ether_spec.h_dest,
636 sizeof(fs->h_u.ether_spec.h_dest));
637 bcmgenet_hfb_insert_data(priv, f, ETH_ALEN,
638 &fs->h_u.ether_spec.h_source,
639 &fs->m_u.ether_spec.h_source,
640 sizeof(fs->h_u.ether_spec.h_source));
641 bcmgenet_hfb_insert_data(priv, f, (2 * ETH_ALEN) + offset,
642 &fs->h_u.ether_spec.h_proto,
643 &fs->m_u.ether_spec.h_proto,
644 sizeof(fs->h_u.ether_spec.h_proto));
645 break;
646 case IP_USER_FLOW:
647 f_length += DIV_ROUND_UP(ETH_HLEN + 20, 2);
648 /* Specify IP Ether Type */
649 val_16 = htons(ETH_P_IP);
650 mask_16 = 0xFFFF;
651 bcmgenet_hfb_insert_data(priv, f, (2 * ETH_ALEN) + offset,
652 &val_16, &mask_16, sizeof(val_16));
653 bcmgenet_hfb_insert_data(priv, f, 15 + offset,
654 &fs->h_u.usr_ip4_spec.tos,
655 &fs->m_u.usr_ip4_spec.tos,
656 sizeof(fs->h_u.usr_ip4_spec.tos));
657 bcmgenet_hfb_insert_data(priv, f, 23 + offset,
658 &fs->h_u.usr_ip4_spec.proto,
659 &fs->m_u.usr_ip4_spec.proto,
660 sizeof(fs->h_u.usr_ip4_spec.proto));
661 bcmgenet_hfb_insert_data(priv, f, 26 + offset,
662 &fs->h_u.usr_ip4_spec.ip4src,
663 &fs->m_u.usr_ip4_spec.ip4src,
664 sizeof(fs->h_u.usr_ip4_spec.ip4src));
665 bcmgenet_hfb_insert_data(priv, f, 30 + offset,
666 &fs->h_u.usr_ip4_spec.ip4dst,
667 &fs->m_u.usr_ip4_spec.ip4dst,
668 sizeof(fs->h_u.usr_ip4_spec.ip4dst));
669 if (!fs->m_u.usr_ip4_spec.l4_4_bytes)
670 break;
671
672 /* Only supports 20 byte IPv4 header */
673 val_8 = 0x45;
674 mask_8 = 0xFF;
675 bcmgenet_hfb_insert_data(priv, f, ETH_HLEN + offset,
676 &val_8, &mask_8,
677 sizeof(val_8));
678 size = sizeof(fs->h_u.usr_ip4_spec.l4_4_bytes);
679 bcmgenet_hfb_insert_data(priv, f,
680 ETH_HLEN + 20 + offset,
681 &fs->h_u.usr_ip4_spec.l4_4_bytes,
682 &fs->m_u.usr_ip4_spec.l4_4_bytes,
683 size);
684 f_length += DIV_ROUND_UP(size, 2);
685 break;
686 }
687
688 bcmgenet_hfb_set_filter_length(priv, f, 2 * f_length);
689 if (fs->ring_cookie == RX_CLS_FLOW_WAKE)
690 q = 0;
691 else if (fs->ring_cookie == RX_CLS_FLOW_DISC)
692 q = priv->hw_params->rx_queues + 1;
693 else
694 /* Other Rx rings are direct mapped here */
695 q = fs->ring_cookie;
696 bcmgenet_hfb_set_filter_rx_queue_mapping(priv, f, q);
697 bcmgenet_hfb_enable_filter(priv, f);
698 rule->state = BCMGENET_RXNFC_STATE_ENABLED;
699 }
700
701 /* bcmgenet_hfb_clear
702 *
703 * Clear Hardware Filter Block and disable all filtering.
704 */
bcmgenet_hfb_clear_filter(struct bcmgenet_priv * priv,u32 f_index)705 static void bcmgenet_hfb_clear_filter(struct bcmgenet_priv *priv, u32 f_index)
706 {
707 u32 base, i;
708
709 bcmgenet_hfb_set_filter_length(priv, f_index, 0);
710 base = f_index * priv->hw_params->hfb_filter_size;
711 for (i = 0; i < priv->hw_params->hfb_filter_size; i++)
712 bcmgenet_hfb_writel(priv, 0x0, (base + i) * sizeof(u32));
713 }
714
bcmgenet_hfb_clear(struct bcmgenet_priv * priv)715 static void bcmgenet_hfb_clear(struct bcmgenet_priv *priv)
716 {
717 u32 i;
718
719 bcmgenet_hfb_reg_writel(priv, 0, HFB_CTRL);
720
721 if (!GENET_IS_V1(priv) && !GENET_IS_V2(priv)) {
722 bcmgenet_hfb_reg_writel(priv, 0,
723 HFB_FLT_ENABLE_V3PLUS);
724 bcmgenet_hfb_reg_writel(priv, 0,
725 HFB_FLT_ENABLE_V3PLUS + 4);
726 for (i = DMA_INDEX2RING_0; i <= DMA_INDEX2RING_7; i++)
727 bcmgenet_rdma_writel(priv, 0, i);
728 }
729
730 for (i = 0; i < priv->hw_params->hfb_filter_cnt; i++)
731 bcmgenet_hfb_clear_filter(priv, i);
732
733 /* Enable filter 0 to send default flow to ring 0 */
734 bcmgenet_hfb_set_filter_length(priv, 0, 4);
735 bcmgenet_hfb_enable_filter(priv, 0);
736 }
737
bcmgenet_hfb_init(struct bcmgenet_priv * priv)738 static void bcmgenet_hfb_init(struct bcmgenet_priv *priv)
739 {
740 int i;
741
742 INIT_LIST_HEAD(&priv->rxnfc_list);
743 for (i = 0; i < MAX_NUM_OF_FS_RULES; i++) {
744 INIT_LIST_HEAD(&priv->rxnfc_rules[i].list);
745 priv->rxnfc_rules[i].state = BCMGENET_RXNFC_STATE_UNUSED;
746 }
747
748 bcmgenet_hfb_clear(priv);
749 }
750
bcmgenet_begin(struct net_device * dev)751 static int bcmgenet_begin(struct net_device *dev)
752 {
753 struct bcmgenet_priv *priv = netdev_priv(dev);
754
755 /* Turn on the clock */
756 return clk_prepare_enable(priv->clk);
757 }
758
bcmgenet_complete(struct net_device * dev)759 static void bcmgenet_complete(struct net_device *dev)
760 {
761 struct bcmgenet_priv *priv = netdev_priv(dev);
762
763 /* Turn off the clock */
764 clk_disable_unprepare(priv->clk);
765 }
766
bcmgenet_get_link_ksettings(struct net_device * dev,struct ethtool_link_ksettings * cmd)767 static int bcmgenet_get_link_ksettings(struct net_device *dev,
768 struct ethtool_link_ksettings *cmd)
769 {
770 if (!netif_running(dev))
771 return -EINVAL;
772
773 if (!dev->phydev)
774 return -ENODEV;
775
776 phy_ethtool_ksettings_get(dev->phydev, cmd);
777
778 return 0;
779 }
780
bcmgenet_set_link_ksettings(struct net_device * dev,const struct ethtool_link_ksettings * cmd)781 static int bcmgenet_set_link_ksettings(struct net_device *dev,
782 const struct ethtool_link_ksettings *cmd)
783 {
784 if (!netif_running(dev))
785 return -EINVAL;
786
787 if (!dev->phydev)
788 return -ENODEV;
789
790 return phy_ethtool_ksettings_set(dev->phydev, cmd);
791 }
792
bcmgenet_set_features(struct net_device * dev,netdev_features_t features)793 static int bcmgenet_set_features(struct net_device *dev,
794 netdev_features_t features)
795 {
796 struct bcmgenet_priv *priv = netdev_priv(dev);
797 u32 reg;
798 int ret;
799
800 ret = clk_prepare_enable(priv->clk);
801 if (ret)
802 return ret;
803
804 /* Make sure we reflect the value of CRC_CMD_FWD */
805 reg = bcmgenet_umac_readl(priv, UMAC_CMD);
806 priv->crc_fwd_en = !!(reg & CMD_CRC_FWD);
807
808 clk_disable_unprepare(priv->clk);
809
810 return ret;
811 }
812
bcmgenet_get_msglevel(struct net_device * dev)813 static u32 bcmgenet_get_msglevel(struct net_device *dev)
814 {
815 struct bcmgenet_priv *priv = netdev_priv(dev);
816
817 return priv->msg_enable;
818 }
819
bcmgenet_set_msglevel(struct net_device * dev,u32 level)820 static void bcmgenet_set_msglevel(struct net_device *dev, u32 level)
821 {
822 struct bcmgenet_priv *priv = netdev_priv(dev);
823
824 priv->msg_enable = level;
825 }
826
bcmgenet_get_coalesce(struct net_device * dev,struct ethtool_coalesce * ec,struct kernel_ethtool_coalesce * kernel_coal,struct netlink_ext_ack * extack)827 static int bcmgenet_get_coalesce(struct net_device *dev,
828 struct ethtool_coalesce *ec,
829 struct kernel_ethtool_coalesce *kernel_coal,
830 struct netlink_ext_ack *extack)
831 {
832 struct bcmgenet_priv *priv = netdev_priv(dev);
833 struct bcmgenet_rx_ring *ring;
834 unsigned int i;
835
836 ec->tx_max_coalesced_frames =
837 bcmgenet_tdma_ring_readl(priv, 0, DMA_MBUF_DONE_THRESH);
838 ec->rx_max_coalesced_frames =
839 bcmgenet_rdma_ring_readl(priv, 0, DMA_MBUF_DONE_THRESH);
840 ec->rx_coalesce_usecs =
841 bcmgenet_rdma_readl(priv, DMA_RING0_TIMEOUT) * 8192 / 1000;
842
843 for (i = 0; i <= priv->hw_params->rx_queues; i++) {
844 ring = &priv->rx_rings[i];
845 ec->use_adaptive_rx_coalesce |= ring->dim.use_dim;
846 }
847
848 return 0;
849 }
850
bcmgenet_set_rx_coalesce(struct bcmgenet_rx_ring * ring,u32 usecs,u32 pkts)851 static void bcmgenet_set_rx_coalesce(struct bcmgenet_rx_ring *ring,
852 u32 usecs, u32 pkts)
853 {
854 struct bcmgenet_priv *priv = ring->priv;
855 unsigned int i = ring->index;
856 u32 reg;
857
858 bcmgenet_rdma_ring_writel(priv, i, pkts, DMA_MBUF_DONE_THRESH);
859
860 reg = bcmgenet_rdma_readl(priv, DMA_RING0_TIMEOUT + i);
861 reg &= ~DMA_TIMEOUT_MASK;
862 reg |= DIV_ROUND_UP(usecs * 1000, 8192);
863 bcmgenet_rdma_writel(priv, reg, DMA_RING0_TIMEOUT + i);
864 }
865
bcmgenet_set_ring_rx_coalesce(struct bcmgenet_rx_ring * ring,struct ethtool_coalesce * ec)866 static void bcmgenet_set_ring_rx_coalesce(struct bcmgenet_rx_ring *ring,
867 struct ethtool_coalesce *ec)
868 {
869 struct dim_cq_moder moder;
870 u32 usecs, pkts;
871
872 ring->rx_coalesce_usecs = ec->rx_coalesce_usecs;
873 ring->rx_max_coalesced_frames = ec->rx_max_coalesced_frames;
874 usecs = ring->rx_coalesce_usecs;
875 pkts = ring->rx_max_coalesced_frames;
876
877 if (ec->use_adaptive_rx_coalesce && !ring->dim.use_dim) {
878 moder = net_dim_get_def_rx_moderation(ring->dim.dim.mode);
879 usecs = moder.usec;
880 pkts = moder.pkts;
881 }
882
883 ring->dim.use_dim = ec->use_adaptive_rx_coalesce;
884 bcmgenet_set_rx_coalesce(ring, usecs, pkts);
885 }
886
bcmgenet_set_coalesce(struct net_device * dev,struct ethtool_coalesce * ec,struct kernel_ethtool_coalesce * kernel_coal,struct netlink_ext_ack * extack)887 static int bcmgenet_set_coalesce(struct net_device *dev,
888 struct ethtool_coalesce *ec,
889 struct kernel_ethtool_coalesce *kernel_coal,
890 struct netlink_ext_ack *extack)
891 {
892 struct bcmgenet_priv *priv = netdev_priv(dev);
893 unsigned int i;
894
895 /* Base system clock is 125Mhz, DMA timeout is this reference clock
896 * divided by 1024, which yields roughly 8.192us, our maximum value
897 * has to fit in the DMA_TIMEOUT_MASK (16 bits)
898 */
899 if (ec->tx_max_coalesced_frames > DMA_INTR_THRESHOLD_MASK ||
900 ec->tx_max_coalesced_frames == 0 ||
901 ec->rx_max_coalesced_frames > DMA_INTR_THRESHOLD_MASK ||
902 ec->rx_coalesce_usecs > (DMA_TIMEOUT_MASK * 8) + 1)
903 return -EINVAL;
904
905 if (ec->rx_coalesce_usecs == 0 && ec->rx_max_coalesced_frames == 0)
906 return -EINVAL;
907
908 /* GENET TDMA hardware does not support a configurable timeout, but will
909 * always generate an interrupt either after MBDONE packets have been
910 * transmitted, or when the ring is empty.
911 */
912
913 /* Program all TX queues with the same values, as there is no
914 * ethtool knob to do coalescing on a per-queue basis
915 */
916 for (i = 0; i <= priv->hw_params->tx_queues; i++)
917 bcmgenet_tdma_ring_writel(priv, i,
918 ec->tx_max_coalesced_frames,
919 DMA_MBUF_DONE_THRESH);
920
921 for (i = 0; i <= priv->hw_params->rx_queues; i++)
922 bcmgenet_set_ring_rx_coalesce(&priv->rx_rings[i], ec);
923
924 return 0;
925 }
926
bcmgenet_get_pauseparam(struct net_device * dev,struct ethtool_pauseparam * epause)927 static void bcmgenet_get_pauseparam(struct net_device *dev,
928 struct ethtool_pauseparam *epause)
929 {
930 struct bcmgenet_priv *priv;
931 u32 umac_cmd;
932
933 priv = netdev_priv(dev);
934
935 epause->autoneg = priv->autoneg_pause;
936
937 if (netif_carrier_ok(dev)) {
938 /* report active state when link is up */
939 umac_cmd = bcmgenet_umac_readl(priv, UMAC_CMD);
940 epause->tx_pause = !(umac_cmd & CMD_TX_PAUSE_IGNORE);
941 epause->rx_pause = !(umac_cmd & CMD_RX_PAUSE_IGNORE);
942 } else {
943 /* otherwise report stored settings */
944 epause->tx_pause = priv->tx_pause;
945 epause->rx_pause = priv->rx_pause;
946 }
947 }
948
bcmgenet_set_pauseparam(struct net_device * dev,struct ethtool_pauseparam * epause)949 static int bcmgenet_set_pauseparam(struct net_device *dev,
950 struct ethtool_pauseparam *epause)
951 {
952 struct bcmgenet_priv *priv = netdev_priv(dev);
953
954 if (!dev->phydev)
955 return -ENODEV;
956
957 if (!phy_validate_pause(dev->phydev, epause))
958 return -EINVAL;
959
960 priv->autoneg_pause = !!epause->autoneg;
961 priv->tx_pause = !!epause->tx_pause;
962 priv->rx_pause = !!epause->rx_pause;
963
964 bcmgenet_phy_pause_set(dev, priv->rx_pause, priv->tx_pause);
965
966 return 0;
967 }
968
969 /* standard ethtool support functions. */
970 enum bcmgenet_stat_type {
971 BCMGENET_STAT_RTNL = -1,
972 BCMGENET_STAT_MIB_RX,
973 BCMGENET_STAT_MIB_TX,
974 BCMGENET_STAT_RUNT,
975 BCMGENET_STAT_MISC,
976 BCMGENET_STAT_SOFT,
977 BCMGENET_STAT_SOFT64,
978 };
979
980 struct bcmgenet_stats {
981 char stat_string[ETH_GSTRING_LEN];
982 int stat_sizeof;
983 int stat_offset;
984 enum bcmgenet_stat_type type;
985 /* reg offset from UMAC base for misc counters */
986 u16 reg_offset;
987 /* sync for u64 stats counters */
988 int syncp_offset;
989 };
990
991 #define STAT_RTNL(m) { \
992 .stat_string = __stringify(m), \
993 .stat_sizeof = sizeof(((struct rtnl_link_stats64 *)0)->m), \
994 .stat_offset = offsetof(struct rtnl_link_stats64, m), \
995 .type = BCMGENET_STAT_RTNL, \
996 }
997
998 #define STAT_GENET_MIB(str, m, _type) { \
999 .stat_string = str, \
1000 .stat_sizeof = sizeof(((struct bcmgenet_priv *)0)->m), \
1001 .stat_offset = offsetof(struct bcmgenet_priv, m), \
1002 .type = _type, \
1003 }
1004
1005 #define STAT_GENET_SOFT_MIB64(str, s, m) { \
1006 .stat_string = str, \
1007 .stat_sizeof = sizeof(((struct bcmgenet_priv *)0)->s.m), \
1008 .stat_offset = offsetof(struct bcmgenet_priv, s.m), \
1009 .type = BCMGENET_STAT_SOFT64, \
1010 .syncp_offset = offsetof(struct bcmgenet_priv, s.syncp), \
1011 }
1012
1013 #define STAT_GENET_MIB_RX(str, m) STAT_GENET_MIB(str, m, BCMGENET_STAT_MIB_RX)
1014 #define STAT_GENET_MIB_TX(str, m) STAT_GENET_MIB(str, m, BCMGENET_STAT_MIB_TX)
1015 #define STAT_GENET_RUNT(str, m) STAT_GENET_MIB(str, m, BCMGENET_STAT_RUNT)
1016 #define STAT_GENET_SOFT_MIB(str, m) STAT_GENET_MIB(str, m, BCMGENET_STAT_SOFT)
1017
1018 #define STAT_GENET_MISC(str, m, offset) { \
1019 .stat_string = str, \
1020 .stat_sizeof = sizeof(((struct bcmgenet_priv *)0)->m), \
1021 .stat_offset = offsetof(struct bcmgenet_priv, m), \
1022 .type = BCMGENET_STAT_MISC, \
1023 .reg_offset = offset, \
1024 }
1025
1026 #define STAT_GENET_Q(num) \
1027 STAT_GENET_SOFT_MIB64("txq" __stringify(num) "_packets", \
1028 tx_rings[num].stats64, packets), \
1029 STAT_GENET_SOFT_MIB64("txq" __stringify(num) "_bytes", \
1030 tx_rings[num].stats64, bytes), \
1031 STAT_GENET_SOFT_MIB64("txq" __stringify(num) "_errors", \
1032 tx_rings[num].stats64, errors), \
1033 STAT_GENET_SOFT_MIB64("txq" __stringify(num) "_dropped", \
1034 tx_rings[num].stats64, dropped), \
1035 STAT_GENET_SOFT_MIB64("rxq" __stringify(num) "_bytes", \
1036 rx_rings[num].stats64, bytes), \
1037 STAT_GENET_SOFT_MIB64("rxq" __stringify(num) "_packets", \
1038 rx_rings[num].stats64, packets), \
1039 STAT_GENET_SOFT_MIB64("rxq" __stringify(num) "_errors", \
1040 rx_rings[num].stats64, errors), \
1041 STAT_GENET_SOFT_MIB64("rxq" __stringify(num) "_dropped", \
1042 rx_rings[num].stats64, dropped), \
1043 STAT_GENET_SOFT_MIB64("rxq" __stringify(num) "_multicast", \
1044 rx_rings[num].stats64, multicast), \
1045 STAT_GENET_SOFT_MIB64("rxq" __stringify(num) "_missed", \
1046 rx_rings[num].stats64, missed), \
1047 STAT_GENET_SOFT_MIB64("rxq" __stringify(num) "_length_errors", \
1048 rx_rings[num].stats64, length_errors), \
1049 STAT_GENET_SOFT_MIB64("rxq" __stringify(num) "_over_errors", \
1050 rx_rings[num].stats64, over_errors), \
1051 STAT_GENET_SOFT_MIB64("rxq" __stringify(num) "_crc_errors", \
1052 rx_rings[num].stats64, crc_errors), \
1053 STAT_GENET_SOFT_MIB64("rxq" __stringify(num) "_frame_errors", \
1054 rx_rings[num].stats64, frame_errors), \
1055 STAT_GENET_SOFT_MIB64("rxq" __stringify(num) "_fragmented_errors", \
1056 rx_rings[num].stats64, fragmented_errors), \
1057 STAT_GENET_SOFT_MIB64("rxq" __stringify(num) "_broadcast", \
1058 rx_rings[num].stats64, broadcast)
1059
1060 /* There is a 0xC gap between the end of RX and beginning of TX stats and then
1061 * between the end of TX stats and the beginning of the RX RUNT
1062 */
1063 #define BCMGENET_STAT_OFFSET 0xc
1064
1065 /* Hardware counters must be kept in sync because the order/offset
1066 * is important here (order in structure declaration = order in hardware)
1067 */
1068 static const struct bcmgenet_stats bcmgenet_gstrings_stats[] = {
1069 /* general stats */
1070 STAT_RTNL(rx_packets),
1071 STAT_RTNL(tx_packets),
1072 STAT_RTNL(rx_bytes),
1073 STAT_RTNL(tx_bytes),
1074 STAT_RTNL(rx_errors),
1075 STAT_RTNL(tx_errors),
1076 STAT_RTNL(rx_dropped),
1077 STAT_RTNL(tx_dropped),
1078 STAT_RTNL(multicast),
1079 STAT_RTNL(rx_missed_errors),
1080 STAT_RTNL(rx_length_errors),
1081 STAT_RTNL(rx_over_errors),
1082 STAT_RTNL(rx_crc_errors),
1083 STAT_RTNL(rx_frame_errors),
1084 /* UniMAC RSV counters */
1085 STAT_GENET_MIB_RX("rx_64_octets", mib.rx.pkt_cnt.cnt_64),
1086 STAT_GENET_MIB_RX("rx_65_127_oct", mib.rx.pkt_cnt.cnt_127),
1087 STAT_GENET_MIB_RX("rx_128_255_oct", mib.rx.pkt_cnt.cnt_255),
1088 STAT_GENET_MIB_RX("rx_256_511_oct", mib.rx.pkt_cnt.cnt_511),
1089 STAT_GENET_MIB_RX("rx_512_1023_oct", mib.rx.pkt_cnt.cnt_1023),
1090 STAT_GENET_MIB_RX("rx_1024_1518_oct", mib.rx.pkt_cnt.cnt_1518),
1091 STAT_GENET_MIB_RX("rx_vlan_1519_1522_oct", mib.rx.pkt_cnt.cnt_mgv),
1092 STAT_GENET_MIB_RX("rx_1522_2047_oct", mib.rx.pkt_cnt.cnt_2047),
1093 STAT_GENET_MIB_RX("rx_2048_4095_oct", mib.rx.pkt_cnt.cnt_4095),
1094 STAT_GENET_MIB_RX("rx_4096_9216_oct", mib.rx.pkt_cnt.cnt_9216),
1095 STAT_GENET_MIB_RX("rx_pkts", mib.rx.pkt),
1096 STAT_GENET_MIB_RX("rx_bytes", mib.rx.bytes),
1097 STAT_GENET_MIB_RX("rx_multicast", mib.rx.mca),
1098 STAT_GENET_MIB_RX("rx_broadcast", mib.rx.bca),
1099 STAT_GENET_MIB_RX("rx_fcs", mib.rx.fcs),
1100 STAT_GENET_MIB_RX("rx_control", mib.rx.cf),
1101 STAT_GENET_MIB_RX("rx_pause", mib.rx.pf),
1102 STAT_GENET_MIB_RX("rx_unknown", mib.rx.uo),
1103 STAT_GENET_MIB_RX("rx_align", mib.rx.aln),
1104 STAT_GENET_MIB_RX("rx_outrange", mib.rx.flr),
1105 STAT_GENET_MIB_RX("rx_code", mib.rx.cde),
1106 STAT_GENET_MIB_RX("rx_carrier", mib.rx.fcr),
1107 STAT_GENET_MIB_RX("rx_oversize", mib.rx.ovr),
1108 STAT_GENET_MIB_RX("rx_jabber", mib.rx.jbr),
1109 STAT_GENET_MIB_RX("rx_mtu_err", mib.rx.mtue),
1110 STAT_GENET_MIB_RX("rx_good_pkts", mib.rx.pok),
1111 STAT_GENET_MIB_RX("rx_unicast", mib.rx.uc),
1112 STAT_GENET_MIB_RX("rx_ppp", mib.rx.ppp),
1113 STAT_GENET_MIB_RX("rx_crc", mib.rx.rcrc),
1114 /* UniMAC TSV counters */
1115 STAT_GENET_MIB_TX("tx_64_octets", mib.tx.pkt_cnt.cnt_64),
1116 STAT_GENET_MIB_TX("tx_65_127_oct", mib.tx.pkt_cnt.cnt_127),
1117 STAT_GENET_MIB_TX("tx_128_255_oct", mib.tx.pkt_cnt.cnt_255),
1118 STAT_GENET_MIB_TX("tx_256_511_oct", mib.tx.pkt_cnt.cnt_511),
1119 STAT_GENET_MIB_TX("tx_512_1023_oct", mib.tx.pkt_cnt.cnt_1023),
1120 STAT_GENET_MIB_TX("tx_1024_1518_oct", mib.tx.pkt_cnt.cnt_1518),
1121 STAT_GENET_MIB_TX("tx_vlan_1519_1522_oct", mib.tx.pkt_cnt.cnt_mgv),
1122 STAT_GENET_MIB_TX("tx_1522_2047_oct", mib.tx.pkt_cnt.cnt_2047),
1123 STAT_GENET_MIB_TX("tx_2048_4095_oct", mib.tx.pkt_cnt.cnt_4095),
1124 STAT_GENET_MIB_TX("tx_4096_9216_oct", mib.tx.pkt_cnt.cnt_9216),
1125 STAT_GENET_MIB_TX("tx_pkts", mib.tx.pkts),
1126 STAT_GENET_MIB_TX("tx_multicast", mib.tx.mca),
1127 STAT_GENET_MIB_TX("tx_broadcast", mib.tx.bca),
1128 STAT_GENET_MIB_TX("tx_pause", mib.tx.pf),
1129 STAT_GENET_MIB_TX("tx_control", mib.tx.cf),
1130 STAT_GENET_MIB_TX("tx_fcs_err", mib.tx.fcs),
1131 STAT_GENET_MIB_TX("tx_oversize", mib.tx.ovr),
1132 STAT_GENET_MIB_TX("tx_defer", mib.tx.drf),
1133 STAT_GENET_MIB_TX("tx_excess_defer", mib.tx.edf),
1134 STAT_GENET_MIB_TX("tx_single_col", mib.tx.scl),
1135 STAT_GENET_MIB_TX("tx_multi_col", mib.tx.mcl),
1136 STAT_GENET_MIB_TX("tx_late_col", mib.tx.lcl),
1137 STAT_GENET_MIB_TX("tx_excess_col", mib.tx.ecl),
1138 STAT_GENET_MIB_TX("tx_frags", mib.tx.frg),
1139 STAT_GENET_MIB_TX("tx_total_col", mib.tx.ncl),
1140 STAT_GENET_MIB_TX("tx_jabber", mib.tx.jbr),
1141 STAT_GENET_MIB_TX("tx_bytes", mib.tx.bytes),
1142 STAT_GENET_MIB_TX("tx_good_pkts", mib.tx.pok),
1143 STAT_GENET_MIB_TX("tx_unicast", mib.tx.uc),
1144 /* UniMAC RUNT counters */
1145 STAT_GENET_RUNT("rx_runt_pkts", mib.rx_runt_cnt),
1146 STAT_GENET_RUNT("rx_runt_valid_fcs", mib.rx_runt_fcs),
1147 STAT_GENET_RUNT("rx_runt_inval_fcs_align", mib.rx_runt_fcs_align),
1148 STAT_GENET_RUNT("rx_runt_bytes", mib.rx_runt_bytes),
1149 /* Misc UniMAC counters */
1150 STAT_GENET_MISC("rbuf_ovflow_cnt", mib.rbuf_ovflow_cnt,
1151 UMAC_RBUF_OVFL_CNT_V1),
1152 STAT_GENET_MISC("rbuf_err_cnt", mib.rbuf_err_cnt,
1153 UMAC_RBUF_ERR_CNT_V1),
1154 STAT_GENET_MISC("mdf_err_cnt", mib.mdf_err_cnt, UMAC_MDF_ERR_CNT),
1155 STAT_GENET_SOFT_MIB("alloc_rx_buff_failed", mib.alloc_rx_buff_failed),
1156 STAT_GENET_SOFT_MIB("rx_dma_failed", mib.rx_dma_failed),
1157 STAT_GENET_SOFT_MIB("tx_dma_failed", mib.tx_dma_failed),
1158 STAT_GENET_SOFT_MIB("tx_realloc_tsb", mib.tx_realloc_tsb),
1159 STAT_GENET_SOFT_MIB("tx_realloc_tsb_failed",
1160 mib.tx_realloc_tsb_failed),
1161 /* Per TX queues */
1162 STAT_GENET_Q(0),
1163 STAT_GENET_Q(1),
1164 STAT_GENET_Q(2),
1165 STAT_GENET_Q(3),
1166 STAT_GENET_Q(4),
1167 };
1168
1169 #define BCMGENET_STATS_LEN ARRAY_SIZE(bcmgenet_gstrings_stats)
1170
1171 #define BCMGENET_STATS64_ADD(stats, m, v) \
1172 do { \
1173 u64_stats_update_begin(&stats->syncp); \
1174 u64_stats_add(&stats->m, v); \
1175 u64_stats_update_end(&stats->syncp); \
1176 } while (0)
1177
1178 #define BCMGENET_STATS64_INC(stats, m) \
1179 do { \
1180 u64_stats_update_begin(&stats->syncp); \
1181 u64_stats_inc(&stats->m); \
1182 u64_stats_update_end(&stats->syncp); \
1183 } while (0)
1184
bcmgenet_get_drvinfo(struct net_device * dev,struct ethtool_drvinfo * info)1185 static void bcmgenet_get_drvinfo(struct net_device *dev,
1186 struct ethtool_drvinfo *info)
1187 {
1188 strscpy(info->driver, "bcmgenet", sizeof(info->driver));
1189 }
1190
bcmgenet_get_sset_count(struct net_device * dev,int string_set)1191 static int bcmgenet_get_sset_count(struct net_device *dev, int string_set)
1192 {
1193 switch (string_set) {
1194 case ETH_SS_STATS:
1195 return BCMGENET_STATS_LEN;
1196 default:
1197 return -EOPNOTSUPP;
1198 }
1199 }
1200
bcmgenet_get_strings(struct net_device * dev,u32 stringset,u8 * data)1201 static void bcmgenet_get_strings(struct net_device *dev, u32 stringset,
1202 u8 *data)
1203 {
1204 const char *str;
1205 int i;
1206
1207 switch (stringset) {
1208 case ETH_SS_STATS:
1209 for (i = 0; i < BCMGENET_STATS_LEN; i++) {
1210 str = bcmgenet_gstrings_stats[i].stat_string;
1211 ethtool_puts(&data, str);
1212 }
1213 break;
1214 }
1215 }
1216
bcmgenet_update_stat_misc(struct bcmgenet_priv * priv,u16 offset)1217 static u32 bcmgenet_update_stat_misc(struct bcmgenet_priv *priv, u16 offset)
1218 {
1219 u16 new_offset;
1220 u32 val;
1221
1222 switch (offset) {
1223 case UMAC_RBUF_OVFL_CNT_V1:
1224 if (GENET_IS_V2(priv))
1225 new_offset = RBUF_OVFL_CNT_V2;
1226 else
1227 new_offset = RBUF_OVFL_CNT_V3PLUS;
1228
1229 val = bcmgenet_rbuf_readl(priv, new_offset);
1230 /* clear if overflowed */
1231 if (val == ~0)
1232 bcmgenet_rbuf_writel(priv, 0, new_offset);
1233 break;
1234 case UMAC_RBUF_ERR_CNT_V1:
1235 if (GENET_IS_V2(priv))
1236 new_offset = RBUF_ERR_CNT_V2;
1237 else
1238 new_offset = RBUF_ERR_CNT_V3PLUS;
1239
1240 val = bcmgenet_rbuf_readl(priv, new_offset);
1241 /* clear if overflowed */
1242 if (val == ~0)
1243 bcmgenet_rbuf_writel(priv, 0, new_offset);
1244 break;
1245 default:
1246 val = bcmgenet_umac_readl(priv, offset);
1247 /* clear if overflowed */
1248 if (val == ~0)
1249 bcmgenet_umac_writel(priv, 0, offset);
1250 break;
1251 }
1252
1253 return val;
1254 }
1255
bcmgenet_update_mib_counters(struct bcmgenet_priv * priv)1256 static void bcmgenet_update_mib_counters(struct bcmgenet_priv *priv)
1257 {
1258 int i, j = 0;
1259
1260 for (i = 0; i < BCMGENET_STATS_LEN; i++) {
1261 const struct bcmgenet_stats *s;
1262 u8 offset = 0;
1263 u32 val = 0;
1264 char *p;
1265
1266 s = &bcmgenet_gstrings_stats[i];
1267 switch (s->type) {
1268 case BCMGENET_STAT_RTNL:
1269 case BCMGENET_STAT_SOFT:
1270 case BCMGENET_STAT_SOFT64:
1271 continue;
1272 case BCMGENET_STAT_RUNT:
1273 offset += BCMGENET_STAT_OFFSET;
1274 fallthrough;
1275 case BCMGENET_STAT_MIB_TX:
1276 offset += BCMGENET_STAT_OFFSET;
1277 fallthrough;
1278 case BCMGENET_STAT_MIB_RX:
1279 val = bcmgenet_umac_readl(priv,
1280 UMAC_MIB_START + j + offset);
1281 offset = 0; /* Reset Offset */
1282 break;
1283 case BCMGENET_STAT_MISC:
1284 if (GENET_IS_V1(priv)) {
1285 val = bcmgenet_umac_readl(priv, s->reg_offset);
1286 /* clear if overflowed */
1287 if (val == ~0)
1288 bcmgenet_umac_writel(priv, 0,
1289 s->reg_offset);
1290 } else {
1291 val = bcmgenet_update_stat_misc(priv,
1292 s->reg_offset);
1293 }
1294 break;
1295 }
1296
1297 j += s->stat_sizeof;
1298 p = (char *)priv + s->stat_offset;
1299 *(u32 *)p = val;
1300 }
1301 }
1302
bcmgenet_get_ethtool_stats(struct net_device * dev,struct ethtool_stats * stats,u64 * data)1303 static void bcmgenet_get_ethtool_stats(struct net_device *dev,
1304 struct ethtool_stats *stats,
1305 u64 *data)
1306 {
1307 struct bcmgenet_priv *priv = netdev_priv(dev);
1308 struct rtnl_link_stats64 stats64;
1309 struct u64_stats_sync *syncp;
1310 unsigned int start;
1311 int i;
1312
1313 if (netif_running(dev))
1314 bcmgenet_update_mib_counters(priv);
1315
1316 dev_get_stats(dev, &stats64);
1317
1318 for (i = 0; i < BCMGENET_STATS_LEN; i++) {
1319 const struct bcmgenet_stats *s;
1320 char *p;
1321
1322 s = &bcmgenet_gstrings_stats[i];
1323 p = (char *)priv;
1324
1325 if (s->type == BCMGENET_STAT_SOFT64) {
1326 syncp = (struct u64_stats_sync *)(p + s->syncp_offset);
1327 do {
1328 start = u64_stats_fetch_begin(syncp);
1329 data[i] = u64_stats_read((u64_stats_t *)(p + s->stat_offset));
1330 } while (u64_stats_fetch_retry(syncp, start));
1331 } else {
1332 if (s->type == BCMGENET_STAT_RTNL)
1333 p = (char *)&stats64;
1334
1335 p += s->stat_offset;
1336 if (sizeof(unsigned long) != sizeof(u32) &&
1337 s->stat_sizeof == sizeof(unsigned long))
1338 data[i] = *(unsigned long *)p;
1339 else
1340 data[i] = *(u32 *)p;
1341 }
1342 }
1343 }
1344
bcmgenet_eee_enable_set(struct net_device * dev,bool enable)1345 void bcmgenet_eee_enable_set(struct net_device *dev, bool enable)
1346 {
1347 struct bcmgenet_priv *priv = netdev_priv(dev);
1348 u32 off = priv->hw_params->tbuf_offset + TBUF_ENERGY_CTRL;
1349 u32 reg;
1350
1351 if (enable && !priv->clk_eee_enabled) {
1352 clk_prepare_enable(priv->clk_eee);
1353 priv->clk_eee_enabled = true;
1354 }
1355
1356 reg = bcmgenet_umac_readl(priv, UMAC_EEE_CTRL);
1357 if (enable)
1358 reg |= EEE_EN;
1359 else
1360 reg &= ~EEE_EN;
1361 bcmgenet_umac_writel(priv, reg, UMAC_EEE_CTRL);
1362
1363 /* Enable EEE and switch to a 27Mhz clock automatically */
1364 reg = bcmgenet_readl(priv->base + off);
1365 if (enable)
1366 reg |= TBUF_EEE_EN | TBUF_PM_EN;
1367 else
1368 reg &= ~(TBUF_EEE_EN | TBUF_PM_EN);
1369 bcmgenet_writel(reg, priv->base + off);
1370
1371 /* RBUF EEE/PM can break the RX path on GENET. Keep it disabled. */
1372 reg = bcmgenet_rbuf_readl(priv, RBUF_ENERGY_CTRL);
1373 if (reg & (RBUF_EEE_EN | RBUF_PM_EN)) {
1374 reg &= ~(RBUF_EEE_EN | RBUF_PM_EN);
1375 bcmgenet_rbuf_writel(priv, reg, RBUF_ENERGY_CTRL);
1376 }
1377
1378 if (!enable && priv->clk_eee_enabled) {
1379 clk_disable_unprepare(priv->clk_eee);
1380 priv->clk_eee_enabled = false;
1381 }
1382
1383 }
1384
bcmgenet_get_eee(struct net_device * dev,struct ethtool_keee * e)1385 static int bcmgenet_get_eee(struct net_device *dev, struct ethtool_keee *e)
1386 {
1387 struct bcmgenet_priv *priv = netdev_priv(dev);
1388 int ret;
1389
1390 if (GENET_IS_V1(priv))
1391 return -EOPNOTSUPP;
1392
1393 if (!dev->phydev)
1394 return -ENODEV;
1395
1396 ret = phy_ethtool_get_eee(dev->phydev, e);
1397 if (ret)
1398 return ret;
1399
1400 /* tx_lpi_timer is maintained by the MAC hardware register; the
1401 * PHY-level eee_cfg timer is not set for GENET.
1402 */
1403 e->tx_lpi_timer = bcmgenet_umac_readl(priv, UMAC_EEE_LPI_TIMER);
1404
1405 return 0;
1406 }
1407
bcmgenet_set_eee(struct net_device * dev,struct ethtool_keee * e)1408 static int bcmgenet_set_eee(struct net_device *dev, struct ethtool_keee *e)
1409 {
1410 struct bcmgenet_priv *priv = netdev_priv(dev);
1411
1412 if (GENET_IS_V1(priv))
1413 return -EOPNOTSUPP;
1414
1415 if (!dev->phydev)
1416 return -ENODEV;
1417
1418 bcmgenet_umac_writel(priv, e->tx_lpi_timer, UMAC_EEE_LPI_TIMER);
1419
1420 return phy_ethtool_set_eee(dev->phydev, e);
1421 }
1422
bcmgenet_validate_flow(struct net_device * dev,struct ethtool_rxnfc * cmd)1423 static int bcmgenet_validate_flow(struct net_device *dev,
1424 struct ethtool_rxnfc *cmd)
1425 {
1426 struct ethtool_usrip4_spec *l4_mask;
1427 struct ethhdr *eth_mask;
1428
1429 if (cmd->fs.location >= MAX_NUM_OF_FS_RULES &&
1430 cmd->fs.location != RX_CLS_LOC_ANY) {
1431 netdev_err(dev, "rxnfc: Invalid location (%d)\n",
1432 cmd->fs.location);
1433 return -EINVAL;
1434 }
1435
1436 switch (cmd->fs.flow_type & ~(FLOW_EXT | FLOW_MAC_EXT)) {
1437 case IP_USER_FLOW:
1438 l4_mask = &cmd->fs.m_u.usr_ip4_spec;
1439 /* don't allow mask which isn't valid */
1440 if (VALIDATE_MASK(l4_mask->ip4src) ||
1441 VALIDATE_MASK(l4_mask->ip4dst) ||
1442 VALIDATE_MASK(l4_mask->l4_4_bytes) ||
1443 VALIDATE_MASK(l4_mask->proto) ||
1444 VALIDATE_MASK(l4_mask->ip_ver) ||
1445 VALIDATE_MASK(l4_mask->tos)) {
1446 netdev_err(dev, "rxnfc: Unsupported mask\n");
1447 return -EINVAL;
1448 }
1449 break;
1450 case ETHER_FLOW:
1451 eth_mask = &cmd->fs.m_u.ether_spec;
1452 /* don't allow mask which isn't valid */
1453 if (VALIDATE_MASK(eth_mask->h_dest) ||
1454 VALIDATE_MASK(eth_mask->h_source) ||
1455 VALIDATE_MASK(eth_mask->h_proto)) {
1456 netdev_err(dev, "rxnfc: Unsupported mask\n");
1457 return -EINVAL;
1458 }
1459 break;
1460 default:
1461 netdev_err(dev, "rxnfc: Unsupported flow type (0x%x)\n",
1462 cmd->fs.flow_type);
1463 return -EINVAL;
1464 }
1465
1466 if ((cmd->fs.flow_type & FLOW_EXT)) {
1467 /* don't allow mask which isn't valid */
1468 if (VALIDATE_MASK(cmd->fs.m_ext.vlan_etype) ||
1469 VALIDATE_MASK(cmd->fs.m_ext.vlan_tci)) {
1470 netdev_err(dev, "rxnfc: Unsupported mask\n");
1471 return -EINVAL;
1472 }
1473 if (cmd->fs.m_ext.data[0] || cmd->fs.m_ext.data[1]) {
1474 netdev_err(dev, "rxnfc: user-def not supported\n");
1475 return -EINVAL;
1476 }
1477 }
1478
1479 if ((cmd->fs.flow_type & FLOW_MAC_EXT)) {
1480 /* don't allow mask which isn't valid */
1481 if (VALIDATE_MASK(cmd->fs.m_ext.h_dest)) {
1482 netdev_err(dev, "rxnfc: Unsupported mask\n");
1483 return -EINVAL;
1484 }
1485 }
1486
1487 return 0;
1488 }
1489
bcmgenet_insert_flow(struct net_device * dev,struct ethtool_rxnfc * cmd)1490 static int bcmgenet_insert_flow(struct net_device *dev,
1491 struct ethtool_rxnfc *cmd)
1492 {
1493 struct bcmgenet_priv *priv = netdev_priv(dev);
1494 struct bcmgenet_rxnfc_rule *loc_rule;
1495 int err, i;
1496
1497 if (priv->hw_params->hfb_filter_size < 128) {
1498 netdev_err(dev, "rxnfc: Not supported by this device\n");
1499 return -EINVAL;
1500 }
1501
1502 if (cmd->fs.ring_cookie > priv->hw_params->rx_queues &&
1503 cmd->fs.ring_cookie != RX_CLS_FLOW_WAKE &&
1504 cmd->fs.ring_cookie != RX_CLS_FLOW_DISC) {
1505 netdev_err(dev, "rxnfc: Unsupported action (%llu)\n",
1506 cmd->fs.ring_cookie);
1507 return -EINVAL;
1508 }
1509
1510 err = bcmgenet_validate_flow(dev, cmd);
1511 if (err)
1512 return err;
1513
1514 if (cmd->fs.location == RX_CLS_LOC_ANY) {
1515 list_for_each_entry(loc_rule, &priv->rxnfc_list, list) {
1516 cmd->fs.location = loc_rule->fs.location;
1517 err = memcmp(&loc_rule->fs, &cmd->fs,
1518 sizeof(struct ethtool_rx_flow_spec));
1519 if (!err)
1520 /* rule exists so return current location */
1521 return 0;
1522 }
1523 for (i = 0; i < MAX_NUM_OF_FS_RULES; i++) {
1524 loc_rule = &priv->rxnfc_rules[i];
1525 if (loc_rule->state == BCMGENET_RXNFC_STATE_UNUSED) {
1526 cmd->fs.location = i;
1527 break;
1528 }
1529 }
1530 if (i == MAX_NUM_OF_FS_RULES) {
1531 cmd->fs.location = RX_CLS_LOC_ANY;
1532 return -ENOSPC;
1533 }
1534 } else {
1535 loc_rule = &priv->rxnfc_rules[cmd->fs.location];
1536 }
1537 if (loc_rule->state == BCMGENET_RXNFC_STATE_ENABLED)
1538 bcmgenet_hfb_disable_filter(priv, cmd->fs.location + 1);
1539 if (loc_rule->state != BCMGENET_RXNFC_STATE_UNUSED) {
1540 list_del(&loc_rule->list);
1541 bcmgenet_hfb_clear_filter(priv, cmd->fs.location + 1);
1542 }
1543 loc_rule->state = BCMGENET_RXNFC_STATE_UNUSED;
1544 memcpy(&loc_rule->fs, &cmd->fs,
1545 sizeof(struct ethtool_rx_flow_spec));
1546
1547 bcmgenet_hfb_create_rxnfc_filter(priv, loc_rule);
1548
1549 list_add_tail(&loc_rule->list, &priv->rxnfc_list);
1550
1551 return 0;
1552 }
1553
bcmgenet_delete_flow(struct net_device * dev,struct ethtool_rxnfc * cmd)1554 static int bcmgenet_delete_flow(struct net_device *dev,
1555 struct ethtool_rxnfc *cmd)
1556 {
1557 struct bcmgenet_priv *priv = netdev_priv(dev);
1558 struct bcmgenet_rxnfc_rule *rule;
1559 int err = 0;
1560
1561 if (cmd->fs.location >= MAX_NUM_OF_FS_RULES)
1562 return -EINVAL;
1563
1564 rule = &priv->rxnfc_rules[cmd->fs.location];
1565 if (rule->state == BCMGENET_RXNFC_STATE_UNUSED) {
1566 err = -ENOENT;
1567 goto out;
1568 }
1569
1570 if (rule->state == BCMGENET_RXNFC_STATE_ENABLED)
1571 bcmgenet_hfb_disable_filter(priv, cmd->fs.location + 1);
1572 if (rule->state != BCMGENET_RXNFC_STATE_UNUSED) {
1573 list_del(&rule->list);
1574 bcmgenet_hfb_clear_filter(priv, cmd->fs.location + 1);
1575 }
1576 rule->state = BCMGENET_RXNFC_STATE_UNUSED;
1577 memset(&rule->fs, 0, sizeof(struct ethtool_rx_flow_spec));
1578
1579 out:
1580 return err;
1581 }
1582
bcmgenet_set_rxnfc(struct net_device * dev,struct ethtool_rxnfc * cmd)1583 static int bcmgenet_set_rxnfc(struct net_device *dev, struct ethtool_rxnfc *cmd)
1584 {
1585 struct bcmgenet_priv *priv = netdev_priv(dev);
1586 int err = 0;
1587
1588 switch (cmd->cmd) {
1589 case ETHTOOL_SRXCLSRLINS:
1590 err = bcmgenet_insert_flow(dev, cmd);
1591 break;
1592 case ETHTOOL_SRXCLSRLDEL:
1593 err = bcmgenet_delete_flow(dev, cmd);
1594 break;
1595 default:
1596 netdev_warn(priv->dev, "Unsupported ethtool command. (%d)\n",
1597 cmd->cmd);
1598 return -EINVAL;
1599 }
1600
1601 return err;
1602 }
1603
bcmgenet_get_flow(struct net_device * dev,struct ethtool_rxnfc * cmd,int loc)1604 static int bcmgenet_get_flow(struct net_device *dev, struct ethtool_rxnfc *cmd,
1605 int loc)
1606 {
1607 struct bcmgenet_priv *priv = netdev_priv(dev);
1608 struct bcmgenet_rxnfc_rule *rule;
1609 int err = 0;
1610
1611 if (loc < 0 || loc >= MAX_NUM_OF_FS_RULES)
1612 return -EINVAL;
1613
1614 rule = &priv->rxnfc_rules[loc];
1615 if (rule->state == BCMGENET_RXNFC_STATE_UNUSED)
1616 err = -ENOENT;
1617 else
1618 memcpy(&cmd->fs, &rule->fs,
1619 sizeof(struct ethtool_rx_flow_spec));
1620
1621 return err;
1622 }
1623
bcmgenet_get_num_flows(struct bcmgenet_priv * priv)1624 static int bcmgenet_get_num_flows(struct bcmgenet_priv *priv)
1625 {
1626 struct list_head *pos;
1627 int res = 0;
1628
1629 list_for_each(pos, &priv->rxnfc_list)
1630 res++;
1631
1632 return res;
1633 }
1634
bcmgenet_get_rx_ring_count(struct net_device * dev)1635 static u32 bcmgenet_get_rx_ring_count(struct net_device *dev)
1636 {
1637 struct bcmgenet_priv *priv = netdev_priv(dev);
1638
1639 return priv->hw_params->rx_queues ?: 1;
1640 }
1641
bcmgenet_get_rxnfc(struct net_device * dev,struct ethtool_rxnfc * cmd,u32 * rule_locs)1642 static int bcmgenet_get_rxnfc(struct net_device *dev, struct ethtool_rxnfc *cmd,
1643 u32 *rule_locs)
1644 {
1645 struct bcmgenet_priv *priv = netdev_priv(dev);
1646 struct bcmgenet_rxnfc_rule *rule;
1647 int err = 0;
1648 int i = 0;
1649
1650 switch (cmd->cmd) {
1651 case ETHTOOL_GRXCLSRLCNT:
1652 cmd->rule_cnt = bcmgenet_get_num_flows(priv);
1653 cmd->data = MAX_NUM_OF_FS_RULES | RX_CLS_LOC_SPECIAL;
1654 break;
1655 case ETHTOOL_GRXCLSRULE:
1656 err = bcmgenet_get_flow(dev, cmd, cmd->fs.location);
1657 break;
1658 case ETHTOOL_GRXCLSRLALL:
1659 list_for_each_entry(rule, &priv->rxnfc_list, list)
1660 if (i < cmd->rule_cnt)
1661 rule_locs[i++] = rule->fs.location;
1662 cmd->rule_cnt = i;
1663 cmd->data = MAX_NUM_OF_FS_RULES;
1664 break;
1665 default:
1666 err = -EOPNOTSUPP;
1667 break;
1668 }
1669
1670 return err;
1671 }
1672
1673 /* standard ethtool support functions. */
1674 static const struct ethtool_ops bcmgenet_ethtool_ops = {
1675 .supported_coalesce_params = ETHTOOL_COALESCE_RX_USECS |
1676 ETHTOOL_COALESCE_MAX_FRAMES |
1677 ETHTOOL_COALESCE_USE_ADAPTIVE_RX,
1678 .begin = bcmgenet_begin,
1679 .complete = bcmgenet_complete,
1680 .get_strings = bcmgenet_get_strings,
1681 .get_sset_count = bcmgenet_get_sset_count,
1682 .get_ethtool_stats = bcmgenet_get_ethtool_stats,
1683 .get_drvinfo = bcmgenet_get_drvinfo,
1684 .get_link = ethtool_op_get_link,
1685 .get_msglevel = bcmgenet_get_msglevel,
1686 .set_msglevel = bcmgenet_set_msglevel,
1687 .get_wol = bcmgenet_get_wol,
1688 .set_wol = bcmgenet_set_wol,
1689 .get_eee = bcmgenet_get_eee,
1690 .set_eee = bcmgenet_set_eee,
1691 .nway_reset = phy_ethtool_nway_reset,
1692 .get_coalesce = bcmgenet_get_coalesce,
1693 .set_coalesce = bcmgenet_set_coalesce,
1694 .get_link_ksettings = bcmgenet_get_link_ksettings,
1695 .set_link_ksettings = bcmgenet_set_link_ksettings,
1696 .get_ts_info = ethtool_op_get_ts_info,
1697 .get_rxnfc = bcmgenet_get_rxnfc,
1698 .set_rxnfc = bcmgenet_set_rxnfc,
1699 .get_rx_ring_count = bcmgenet_get_rx_ring_count,
1700 .get_pauseparam = bcmgenet_get_pauseparam,
1701 .set_pauseparam = bcmgenet_set_pauseparam,
1702 };
1703
1704 /* Power down the unimac, based on mode. */
bcmgenet_power_down(struct bcmgenet_priv * priv,enum bcmgenet_power_mode mode)1705 static int bcmgenet_power_down(struct bcmgenet_priv *priv,
1706 enum bcmgenet_power_mode mode)
1707 {
1708 int ret = 0;
1709 u32 reg;
1710
1711 switch (mode) {
1712 case GENET_POWER_CABLE_SENSE:
1713 phy_detach(priv->dev->phydev);
1714 break;
1715
1716 case GENET_POWER_WOL_MAGIC:
1717 ret = bcmgenet_wol_power_down_cfg(priv, mode);
1718 break;
1719
1720 case GENET_POWER_PASSIVE:
1721 /* Power down LED */
1722 if (bcmgenet_has_ext(priv)) {
1723 reg = bcmgenet_ext_readl(priv, EXT_EXT_PWR_MGMT);
1724 if (GENET_IS_V5(priv) && !bcmgenet_has_ephy_16nm(priv))
1725 reg |= EXT_PWR_DOWN_PHY_EN |
1726 EXT_PWR_DOWN_PHY_RD |
1727 EXT_PWR_DOWN_PHY_SD |
1728 EXT_PWR_DOWN_PHY_RX |
1729 EXT_PWR_DOWN_PHY_TX |
1730 EXT_IDDQ_GLBL_PWR;
1731 else
1732 reg |= EXT_PWR_DOWN_PHY;
1733
1734 reg |= (EXT_PWR_DOWN_DLL | EXT_PWR_DOWN_BIAS);
1735 bcmgenet_ext_writel(priv, reg, EXT_EXT_PWR_MGMT);
1736
1737 bcmgenet_phy_power_set(priv->dev, false);
1738 }
1739 break;
1740 default:
1741 break;
1742 }
1743
1744 return ret;
1745 }
1746
bcmgenet_power_up(struct bcmgenet_priv * priv,enum bcmgenet_power_mode mode)1747 static int bcmgenet_power_up(struct bcmgenet_priv *priv,
1748 enum bcmgenet_power_mode mode)
1749 {
1750 int ret = 0;
1751 u32 reg;
1752
1753 if (!bcmgenet_has_ext(priv))
1754 return ret;
1755
1756 reg = bcmgenet_ext_readl(priv, EXT_EXT_PWR_MGMT);
1757
1758 switch (mode) {
1759 case GENET_POWER_PASSIVE:
1760 reg &= ~(EXT_PWR_DOWN_DLL | EXT_PWR_DOWN_BIAS |
1761 EXT_ENERGY_DET_MASK);
1762 if (GENET_IS_V5(priv) && !bcmgenet_has_ephy_16nm(priv)) {
1763 reg &= ~(EXT_PWR_DOWN_PHY_EN |
1764 EXT_PWR_DOWN_PHY_RD |
1765 EXT_PWR_DOWN_PHY_SD |
1766 EXT_PWR_DOWN_PHY_RX |
1767 EXT_PWR_DOWN_PHY_TX |
1768 EXT_IDDQ_GLBL_PWR);
1769 reg |= EXT_PHY_RESET;
1770 bcmgenet_ext_writel(priv, reg, EXT_EXT_PWR_MGMT);
1771 mdelay(1);
1772
1773 reg &= ~EXT_PHY_RESET;
1774 } else {
1775 reg &= ~EXT_PWR_DOWN_PHY;
1776 reg |= EXT_PWR_DN_EN_LD;
1777 }
1778 bcmgenet_ext_writel(priv, reg, EXT_EXT_PWR_MGMT);
1779 bcmgenet_phy_power_set(priv->dev, true);
1780 break;
1781
1782 case GENET_POWER_CABLE_SENSE:
1783 /* enable APD */
1784 if (!GENET_IS_V5(priv)) {
1785 reg |= EXT_PWR_DN_EN_LD;
1786 bcmgenet_ext_writel(priv, reg, EXT_EXT_PWR_MGMT);
1787 }
1788 break;
1789 case GENET_POWER_WOL_MAGIC:
1790 ret = bcmgenet_wol_power_up_cfg(priv, mode);
1791 break;
1792 default:
1793 break;
1794 }
1795
1796 return ret;
1797 }
1798
bcmgenet_get_txcb(struct bcmgenet_priv * priv,struct bcmgenet_tx_ring * ring)1799 static struct enet_cb *bcmgenet_get_txcb(struct bcmgenet_priv *priv,
1800 struct bcmgenet_tx_ring *ring)
1801 {
1802 struct enet_cb *tx_cb_ptr;
1803
1804 tx_cb_ptr = ring->cbs;
1805 tx_cb_ptr += ring->write_ptr - ring->cb_ptr;
1806
1807 /* Advancing local write pointer */
1808 if (ring->write_ptr == ring->end_ptr)
1809 ring->write_ptr = ring->cb_ptr;
1810 else
1811 ring->write_ptr++;
1812
1813 return tx_cb_ptr;
1814 }
1815
bcmgenet_put_txcb(struct bcmgenet_priv * priv,struct bcmgenet_tx_ring * ring)1816 static struct enet_cb *bcmgenet_put_txcb(struct bcmgenet_priv *priv,
1817 struct bcmgenet_tx_ring *ring)
1818 {
1819 struct enet_cb *tx_cb_ptr;
1820
1821 /* Rewinding local write pointer */
1822 if (ring->write_ptr == ring->cb_ptr)
1823 ring->write_ptr = ring->end_ptr;
1824 else
1825 ring->write_ptr--;
1826
1827 tx_cb_ptr = ring->cbs;
1828 tx_cb_ptr += ring->write_ptr - ring->cb_ptr;
1829
1830 return tx_cb_ptr;
1831 }
1832
bcmgenet_rx_ring_int_disable(struct bcmgenet_rx_ring * ring)1833 static inline void bcmgenet_rx_ring_int_disable(struct bcmgenet_rx_ring *ring)
1834 {
1835 bcmgenet_intrl2_1_writel(ring->priv,
1836 1 << (UMAC_IRQ1_RX_INTR_SHIFT + ring->index),
1837 INTRL2_CPU_MASK_SET);
1838 }
1839
bcmgenet_rx_ring_int_enable(struct bcmgenet_rx_ring * ring)1840 static inline void bcmgenet_rx_ring_int_enable(struct bcmgenet_rx_ring *ring)
1841 {
1842 bcmgenet_intrl2_1_writel(ring->priv,
1843 1 << (UMAC_IRQ1_RX_INTR_SHIFT + ring->index),
1844 INTRL2_CPU_MASK_CLEAR);
1845 }
1846
bcmgenet_tx_ring_int_enable(struct bcmgenet_tx_ring * ring)1847 static inline void bcmgenet_tx_ring_int_enable(struct bcmgenet_tx_ring *ring)
1848 {
1849 bcmgenet_intrl2_1_writel(ring->priv, 1 << ring->index,
1850 INTRL2_CPU_MASK_CLEAR);
1851 }
1852
bcmgenet_tx_ring_int_disable(struct bcmgenet_tx_ring * ring)1853 static inline void bcmgenet_tx_ring_int_disable(struct bcmgenet_tx_ring *ring)
1854 {
1855 bcmgenet_intrl2_1_writel(ring->priv, 1 << ring->index,
1856 INTRL2_CPU_MASK_SET);
1857 }
1858
1859 /* Simple helper to free a transmit control block's resources
1860 * Returns an skb when the last transmit control block associated with the
1861 * skb is freed. The skb should be freed by the caller if necessary.
1862 */
bcmgenet_free_tx_cb(struct device * dev,struct enet_cb * cb)1863 static struct sk_buff *bcmgenet_free_tx_cb(struct device *dev,
1864 struct enet_cb *cb)
1865 {
1866 struct sk_buff *skb;
1867
1868 skb = cb->skb;
1869
1870 if (skb) {
1871 cb->skb = NULL;
1872 if (cb == GENET_CB(skb)->first_cb)
1873 dma_unmap_single(dev, dma_unmap_addr(cb, dma_addr),
1874 dma_unmap_len(cb, dma_len),
1875 DMA_TO_DEVICE);
1876 else
1877 dma_unmap_page(dev, dma_unmap_addr(cb, dma_addr),
1878 dma_unmap_len(cb, dma_len),
1879 DMA_TO_DEVICE);
1880 dma_unmap_addr_set(cb, dma_addr, 0);
1881
1882 if (cb == GENET_CB(skb)->last_cb)
1883 return skb;
1884
1885 } else if (dma_unmap_addr(cb, dma_addr)) {
1886 dma_unmap_page(dev,
1887 dma_unmap_addr(cb, dma_addr),
1888 dma_unmap_len(cb, dma_len),
1889 DMA_TO_DEVICE);
1890 dma_unmap_addr_set(cb, dma_addr, 0);
1891 }
1892
1893 return NULL;
1894 }
1895
1896 /* Simple helper to free a receive control block's resources */
bcmgenet_free_rx_cb(struct device * dev,struct enet_cb * cb)1897 static struct sk_buff *bcmgenet_free_rx_cb(struct device *dev,
1898 struct enet_cb *cb)
1899 {
1900 struct sk_buff *skb;
1901
1902 skb = cb->skb;
1903 cb->skb = NULL;
1904
1905 if (dma_unmap_addr(cb, dma_addr)) {
1906 dma_unmap_single(dev, dma_unmap_addr(cb, dma_addr),
1907 dma_unmap_len(cb, dma_len), DMA_FROM_DEVICE);
1908 dma_unmap_addr_set(cb, dma_addr, 0);
1909 }
1910
1911 return skb;
1912 }
1913
1914 /* Unlocked version of the reclaim routine */
__bcmgenet_tx_reclaim(struct net_device * dev,struct bcmgenet_tx_ring * ring)1915 static unsigned int __bcmgenet_tx_reclaim(struct net_device *dev,
1916 struct bcmgenet_tx_ring *ring)
1917 {
1918 struct bcmgenet_tx_stats64 *stats = &ring->stats64;
1919 struct bcmgenet_priv *priv = netdev_priv(dev);
1920 unsigned int txbds_processed = 0;
1921 unsigned int bytes_compl = 0;
1922 unsigned int pkts_compl = 0;
1923 unsigned int txbds_ready;
1924 unsigned int c_index;
1925 struct sk_buff *skb;
1926
1927 /* Clear status before servicing to reduce spurious interrupts */
1928 bcmgenet_intrl2_1_writel(priv, (1 << ring->index), INTRL2_CPU_CLEAR);
1929
1930 /* Compute how many buffers are transmitted since last xmit call */
1931 c_index = bcmgenet_tdma_ring_readl(priv, ring->index, TDMA_CONS_INDEX)
1932 & DMA_C_INDEX_MASK;
1933 txbds_ready = (c_index - ring->c_index) & DMA_C_INDEX_MASK;
1934
1935 netif_dbg(priv, tx_done, dev,
1936 "%s ring=%d old_c_index=%u c_index=%u txbds_ready=%u\n",
1937 __func__, ring->index, ring->c_index, c_index, txbds_ready);
1938
1939 /* Reclaim transmitted buffers */
1940 while (txbds_processed < txbds_ready) {
1941 skb = bcmgenet_free_tx_cb(&priv->pdev->dev,
1942 &priv->tx_cbs[ring->clean_ptr]);
1943 if (skb) {
1944 pkts_compl++;
1945 bytes_compl += GENET_CB(skb)->bytes_sent;
1946 dev_consume_skb_any(skb);
1947 }
1948
1949 txbds_processed++;
1950 if (likely(ring->clean_ptr < ring->end_ptr))
1951 ring->clean_ptr++;
1952 else
1953 ring->clean_ptr = ring->cb_ptr;
1954 }
1955
1956 ring->free_bds += txbds_processed;
1957 ring->c_index = c_index;
1958
1959 u64_stats_update_begin(&stats->syncp);
1960 u64_stats_add(&stats->packets, pkts_compl);
1961 u64_stats_add(&stats->bytes, bytes_compl);
1962 u64_stats_update_end(&stats->syncp);
1963
1964 netdev_tx_completed_queue(netdev_get_tx_queue(dev, ring->index),
1965 pkts_compl, bytes_compl);
1966
1967 return txbds_processed;
1968 }
1969
bcmgenet_tx_reclaim(struct net_device * dev,struct bcmgenet_tx_ring * ring,bool all)1970 static unsigned int bcmgenet_tx_reclaim(struct net_device *dev,
1971 struct bcmgenet_tx_ring *ring,
1972 bool all)
1973 {
1974 struct bcmgenet_priv *priv = netdev_priv(dev);
1975 struct device *kdev = &priv->pdev->dev;
1976 unsigned int released, drop, wr_ptr;
1977 struct enet_cb *cb_ptr;
1978 struct sk_buff *skb;
1979
1980 spin_lock_bh(&ring->lock);
1981 released = __bcmgenet_tx_reclaim(dev, ring);
1982 if (all) {
1983 skb = NULL;
1984 drop = (ring->prod_index - ring->c_index) & DMA_C_INDEX_MASK;
1985 released += drop;
1986 ring->prod_index = ring->c_index & DMA_C_INDEX_MASK;
1987 ring->free_bds += drop;
1988 while (drop--) {
1989 cb_ptr = bcmgenet_put_txcb(priv, ring);
1990 skb = cb_ptr->skb;
1991 bcmgenet_free_tx_cb(kdev, cb_ptr);
1992 if (skb && cb_ptr == GENET_CB(skb)->first_cb) {
1993 dev_consume_skb_any(skb);
1994 skb = NULL;
1995 }
1996 }
1997 if (skb)
1998 dev_consume_skb_any(skb);
1999 netdev_tx_reset_queue(netdev_get_tx_queue(dev, ring->index));
2000 bcmgenet_tdma_ring_writel(priv, ring->index,
2001 ring->prod_index, TDMA_PROD_INDEX);
2002 wr_ptr = ring->write_ptr * WORDS_PER_BD(priv);
2003 bcmgenet_tdma_ring_writel(priv, ring->index, wr_ptr,
2004 TDMA_WRITE_PTR);
2005 }
2006 spin_unlock_bh(&ring->lock);
2007
2008 return released;
2009 }
2010
bcmgenet_tx_poll(struct napi_struct * napi,int budget)2011 static int bcmgenet_tx_poll(struct napi_struct *napi, int budget)
2012 {
2013 struct bcmgenet_tx_ring *ring =
2014 container_of(napi, struct bcmgenet_tx_ring, napi);
2015 unsigned int work_done = 0;
2016 struct netdev_queue *txq;
2017
2018 spin_lock(&ring->lock);
2019 work_done = __bcmgenet_tx_reclaim(ring->priv->dev, ring);
2020 if (ring->free_bds > (MAX_SKB_FRAGS + 1)) {
2021 txq = netdev_get_tx_queue(ring->priv->dev, ring->index);
2022 netif_tx_wake_queue(txq);
2023 }
2024 spin_unlock(&ring->lock);
2025
2026 if (work_done == 0) {
2027 napi_complete(napi);
2028 bcmgenet_tx_ring_int_enable(ring);
2029
2030 return 0;
2031 }
2032
2033 return budget;
2034 }
2035
bcmgenet_tx_reclaim_all(struct net_device * dev)2036 static void bcmgenet_tx_reclaim_all(struct net_device *dev)
2037 {
2038 struct bcmgenet_priv *priv = netdev_priv(dev);
2039 int i = 0;
2040
2041 do {
2042 bcmgenet_tx_reclaim(dev, &priv->tx_rings[i++], true);
2043 } while (i <= priv->hw_params->tx_queues && netif_is_multiqueue(dev));
2044 }
2045
2046 /* Reallocate the SKB to put enough headroom in front of it and insert
2047 * the transmit checksum offsets in the descriptors
2048 */
bcmgenet_add_tsb(struct net_device * dev,struct sk_buff * skb,struct bcmgenet_tx_ring * ring)2049 static struct sk_buff *bcmgenet_add_tsb(struct net_device *dev,
2050 struct sk_buff *skb,
2051 struct bcmgenet_tx_ring *ring)
2052 {
2053 struct bcmgenet_tx_stats64 *stats = &ring->stats64;
2054 struct bcmgenet_priv *priv = netdev_priv(dev);
2055 struct status_64 *status = NULL;
2056 struct sk_buff *new_skb;
2057 u16 offset;
2058 u8 ip_proto;
2059 __be16 ip_ver;
2060 u32 tx_csum_info;
2061
2062 if (unlikely(skb_headroom(skb) < sizeof(*status))) {
2063 /* If 64 byte status block enabled, must make sure skb has
2064 * enough headroom for us to insert 64B status block.
2065 */
2066 new_skb = skb_realloc_headroom(skb, sizeof(*status));
2067 if (!new_skb) {
2068 dev_kfree_skb_any(skb);
2069 priv->mib.tx_realloc_tsb_failed++;
2070 BCMGENET_STATS64_INC(stats, dropped);
2071 return NULL;
2072 }
2073 dev_consume_skb_any(skb);
2074 skb = new_skb;
2075 priv->mib.tx_realloc_tsb++;
2076 }
2077
2078 skb_push(skb, sizeof(*status));
2079 status = (struct status_64 *)skb->data;
2080
2081 if (skb->ip_summed == CHECKSUM_PARTIAL) {
2082 ip_ver = skb->protocol;
2083 switch (ip_ver) {
2084 case htons(ETH_P_IP):
2085 ip_proto = ip_hdr(skb)->protocol;
2086 break;
2087 case htons(ETH_P_IPV6):
2088 ip_proto = ipv6_hdr(skb)->nexthdr;
2089 break;
2090 default:
2091 /* don't use UDP flag */
2092 ip_proto = 0;
2093 break;
2094 }
2095
2096 offset = skb_checksum_start_offset(skb) - sizeof(*status);
2097 tx_csum_info = (offset << STATUS_TX_CSUM_START_SHIFT) |
2098 (offset + skb->csum_offset) |
2099 STATUS_TX_CSUM_LV;
2100
2101 /* Set the special UDP flag for UDP */
2102 if (ip_proto == IPPROTO_UDP)
2103 tx_csum_info |= STATUS_TX_CSUM_PROTO_UDP;
2104
2105 status->tx_csum_info = tx_csum_info;
2106 }
2107
2108 return skb;
2109 }
2110
bcmgenet_hide_tsb(struct sk_buff * skb)2111 static void bcmgenet_hide_tsb(struct sk_buff *skb)
2112 {
2113 __skb_pull(skb, sizeof(struct status_64));
2114 }
2115
bcmgenet_xmit(struct sk_buff * skb,struct net_device * dev)2116 static netdev_tx_t bcmgenet_xmit(struct sk_buff *skb, struct net_device *dev)
2117 {
2118 struct bcmgenet_priv *priv = netdev_priv(dev);
2119 struct device *kdev = &priv->pdev->dev;
2120 struct bcmgenet_tx_ring *ring = NULL;
2121 struct enet_cb *tx_cb_ptr;
2122 struct netdev_queue *txq;
2123 int nr_frags, index;
2124 dma_addr_t mapping;
2125 unsigned int size;
2126 skb_frag_t *frag;
2127 u32 len_stat;
2128 int ret;
2129 int i;
2130
2131 index = skb_get_queue_mapping(skb);
2132 /* Mapping strategy:
2133 * queue_mapping = 0, unclassified, packet xmited through ring 0
2134 * queue_mapping = 1, goes to ring 1. (highest priority queue)
2135 * queue_mapping = 2, goes to ring 2.
2136 * queue_mapping = 3, goes to ring 3.
2137 * queue_mapping = 4, goes to ring 4.
2138 */
2139 ring = &priv->tx_rings[index];
2140 txq = netdev_get_tx_queue(dev, index);
2141
2142 nr_frags = skb_shinfo(skb)->nr_frags;
2143
2144 spin_lock(&ring->lock);
2145 if (ring->free_bds <= (nr_frags + 1)) {
2146 if (!netif_tx_queue_stopped(txq))
2147 netif_tx_stop_queue(txq);
2148 ret = NETDEV_TX_BUSY;
2149 goto out;
2150 }
2151
2152 /* Retain how many bytes will be sent on the wire, without TSB inserted
2153 * by transmit checksum offload
2154 */
2155 GENET_CB(skb)->bytes_sent = skb->len;
2156
2157 /* add the Transmit Status Block */
2158 skb = bcmgenet_add_tsb(dev, skb, ring);
2159 if (!skb) {
2160 ret = NETDEV_TX_OK;
2161 goto out;
2162 }
2163
2164 for (i = 0; i <= nr_frags; i++) {
2165 tx_cb_ptr = bcmgenet_get_txcb(priv, ring);
2166
2167 BUG_ON(!tx_cb_ptr);
2168
2169 if (!i) {
2170 /* Transmit single SKB or head of fragment list */
2171 GENET_CB(skb)->first_cb = tx_cb_ptr;
2172 size = skb_headlen(skb);
2173 mapping = dma_map_single(kdev, skb->data, size,
2174 DMA_TO_DEVICE);
2175 } else {
2176 /* xmit fragment */
2177 frag = &skb_shinfo(skb)->frags[i - 1];
2178 size = skb_frag_size(frag);
2179 mapping = skb_frag_dma_map(kdev, frag, 0, size,
2180 DMA_TO_DEVICE);
2181 }
2182
2183 ret = dma_mapping_error(kdev, mapping);
2184 if (ret) {
2185 priv->mib.tx_dma_failed++;
2186 netif_err(priv, tx_err, dev, "Tx DMA map failed\n");
2187 ret = NETDEV_TX_OK;
2188 goto out_unmap_frags;
2189 }
2190 dma_unmap_addr_set(tx_cb_ptr, dma_addr, mapping);
2191 dma_unmap_len_set(tx_cb_ptr, dma_len, size);
2192
2193 tx_cb_ptr->skb = skb;
2194
2195 len_stat = (size << DMA_BUFLENGTH_SHIFT) |
2196 (priv->hw_params->qtag_mask << DMA_TX_QTAG_SHIFT);
2197
2198 /* Note: if we ever change from DMA_TX_APPEND_CRC below we
2199 * will need to restore software padding of "runt" packets
2200 */
2201 len_stat |= DMA_TX_APPEND_CRC;
2202
2203 if (!i) {
2204 len_stat |= DMA_SOP;
2205 if (skb->ip_summed == CHECKSUM_PARTIAL)
2206 len_stat |= DMA_TX_DO_CSUM;
2207 }
2208 if (i == nr_frags)
2209 len_stat |= DMA_EOP;
2210
2211 dmadesc_set(priv, tx_cb_ptr->bd_addr, mapping, len_stat);
2212 }
2213
2214 GENET_CB(skb)->last_cb = tx_cb_ptr;
2215
2216 bcmgenet_hide_tsb(skb);
2217 skb_tx_timestamp(skb);
2218
2219 /* Decrement total BD count and advance our write pointer */
2220 ring->free_bds -= nr_frags + 1;
2221 ring->prod_index += nr_frags + 1;
2222 ring->prod_index &= DMA_P_INDEX_MASK;
2223
2224 netdev_tx_sent_queue(txq, GENET_CB(skb)->bytes_sent);
2225
2226 if (ring->free_bds <= (MAX_SKB_FRAGS + 1))
2227 netif_tx_stop_queue(txq);
2228
2229 if (!netdev_xmit_more() || netif_xmit_stopped(txq))
2230 /* Packets are ready, update producer index */
2231 bcmgenet_tdma_ring_writel(priv, ring->index,
2232 ring->prod_index, TDMA_PROD_INDEX);
2233 out:
2234 spin_unlock(&ring->lock);
2235
2236 return ret;
2237
2238 out_unmap_frags:
2239 /* Back up for failed control block mapping */
2240 bcmgenet_put_txcb(priv, ring);
2241
2242 /* Unmap successfully mapped control blocks */
2243 while (i-- > 0) {
2244 tx_cb_ptr = bcmgenet_put_txcb(priv, ring);
2245 bcmgenet_free_tx_cb(kdev, tx_cb_ptr);
2246 }
2247
2248 dev_kfree_skb(skb);
2249 goto out;
2250 }
2251
bcmgenet_rx_refill(struct bcmgenet_priv * priv,struct enet_cb * cb)2252 static struct sk_buff *bcmgenet_rx_refill(struct bcmgenet_priv *priv,
2253 struct enet_cb *cb)
2254 {
2255 struct device *kdev = &priv->pdev->dev;
2256 struct sk_buff *skb;
2257 struct sk_buff *rx_skb;
2258 dma_addr_t mapping;
2259
2260 /* Allocate a new Rx skb */
2261 skb = __netdev_alloc_skb(priv->dev, priv->rx_buf_len + SKB_ALIGNMENT,
2262 GFP_ATOMIC | __GFP_NOWARN);
2263 if (!skb) {
2264 priv->mib.alloc_rx_buff_failed++;
2265 netif_err(priv, rx_err, priv->dev,
2266 "%s: Rx skb allocation failed\n", __func__);
2267 return NULL;
2268 }
2269
2270 /* DMA-map the new Rx skb */
2271 mapping = dma_map_single(kdev, skb->data, priv->rx_buf_len,
2272 DMA_FROM_DEVICE);
2273 if (dma_mapping_error(kdev, mapping)) {
2274 priv->mib.rx_dma_failed++;
2275 dev_kfree_skb_any(skb);
2276 netif_err(priv, rx_err, priv->dev,
2277 "%s: Rx skb DMA mapping failed\n", __func__);
2278 return NULL;
2279 }
2280
2281 /* Grab the current Rx skb from the ring and DMA-unmap it */
2282 rx_skb = bcmgenet_free_rx_cb(kdev, cb);
2283
2284 /* Put the new Rx skb on the ring */
2285 cb->skb = skb;
2286 dma_unmap_addr_set(cb, dma_addr, mapping);
2287 dma_unmap_len_set(cb, dma_len, priv->rx_buf_len);
2288 dmadesc_set_addr(priv, cb->bd_addr, mapping);
2289
2290 /* Return the current Rx skb to caller */
2291 return rx_skb;
2292 }
2293
2294 /* bcmgenet_desc_rx - descriptor based rx process.
2295 * this could be called from bottom half, or from NAPI polling method.
2296 */
bcmgenet_desc_rx(struct bcmgenet_rx_ring * ring,unsigned int budget)2297 static unsigned int bcmgenet_desc_rx(struct bcmgenet_rx_ring *ring,
2298 unsigned int budget)
2299 {
2300 struct bcmgenet_rx_stats64 *stats = &ring->stats64;
2301 struct bcmgenet_priv *priv = ring->priv;
2302 struct net_device *dev = priv->dev;
2303 struct enet_cb *cb;
2304 struct sk_buff *skb;
2305 u32 dma_length_status;
2306 unsigned long dma_flag;
2307 int len;
2308 unsigned int rxpktprocessed = 0, rxpkttoprocess;
2309 unsigned int bytes_processed = 0;
2310 unsigned int p_index, mask;
2311 unsigned int discards;
2312
2313 /* Clear status before servicing to reduce spurious interrupts */
2314 mask = 1 << (UMAC_IRQ1_RX_INTR_SHIFT + ring->index);
2315 bcmgenet_intrl2_1_writel(priv, mask, INTRL2_CPU_CLEAR);
2316
2317 p_index = bcmgenet_rdma_ring_readl(priv, ring->index, RDMA_PROD_INDEX);
2318
2319 discards = (p_index >> DMA_P_INDEX_DISCARD_CNT_SHIFT) &
2320 DMA_P_INDEX_DISCARD_CNT_MASK;
2321 if (discards > ring->old_discards) {
2322 discards = discards - ring->old_discards;
2323 BCMGENET_STATS64_ADD(stats, missed, discards);
2324 ring->old_discards += discards;
2325
2326 /* Clear HW register when we reach 75% of maximum 0xFFFF */
2327 if (ring->old_discards >= 0xC000) {
2328 ring->old_discards = 0;
2329 bcmgenet_rdma_ring_writel(priv, ring->index, 0,
2330 RDMA_PROD_INDEX);
2331 }
2332 }
2333
2334 p_index &= DMA_P_INDEX_MASK;
2335 rxpkttoprocess = (p_index - ring->c_index) & DMA_C_INDEX_MASK;
2336
2337 netif_dbg(priv, rx_status, dev,
2338 "RDMA: rxpkttoprocess=%d\n", rxpkttoprocess);
2339
2340 while ((rxpktprocessed < rxpkttoprocess) &&
2341 (rxpktprocessed < budget)) {
2342 struct status_64 *status;
2343 __be16 rx_csum;
2344
2345 cb = &priv->rx_cbs[ring->read_ptr];
2346 skb = bcmgenet_rx_refill(priv, cb);
2347
2348 if (unlikely(!skb)) {
2349 BCMGENET_STATS64_INC(stats, dropped);
2350 goto next;
2351 }
2352
2353 status = (struct status_64 *)skb->data;
2354 dma_length_status = status->length_status;
2355 if (dev->features & NETIF_F_RXCSUM) {
2356 rx_csum = (__force __be16)(status->rx_csum & 0xffff);
2357 if (rx_csum) {
2358 skb->csum = (__force __wsum)ntohs(rx_csum);
2359 skb->ip_summed = CHECKSUM_COMPLETE;
2360 }
2361 }
2362
2363 /* DMA flags and length are still valid no matter how
2364 * we got the Receive Status Vector (64B RSB or register)
2365 */
2366 dma_flag = dma_length_status & 0xffff;
2367 len = dma_length_status >> DMA_BUFLENGTH_SHIFT;
2368
2369 netif_dbg(priv, rx_status, dev,
2370 "%s:p_ind=%d c_ind=%d read_ptr=%d len_stat=0x%08x\n",
2371 __func__, p_index, ring->c_index,
2372 ring->read_ptr, dma_length_status);
2373
2374 if (unlikely(len > RX_BUF_LENGTH)) {
2375 netif_err(priv, rx_status, dev, "oversized packet\n");
2376 BCMGENET_STATS64_INC(stats, length_errors);
2377 dev_kfree_skb_any(skb);
2378 goto next;
2379 }
2380
2381 if (unlikely(!(dma_flag & DMA_EOP) || !(dma_flag & DMA_SOP))) {
2382 netif_err(priv, rx_status, dev,
2383 "dropping fragmented packet!\n");
2384 BCMGENET_STATS64_INC(stats, fragmented_errors);
2385 dev_kfree_skb_any(skb);
2386 goto next;
2387 }
2388
2389 /* report errors */
2390 if (unlikely(dma_flag & (DMA_RX_CRC_ERROR |
2391 DMA_RX_OV |
2392 DMA_RX_NO |
2393 DMA_RX_LG |
2394 DMA_RX_RXER))) {
2395 netif_err(priv, rx_status, dev, "dma_flag=0x%x\n",
2396 (unsigned int)dma_flag);
2397 u64_stats_update_begin(&stats->syncp);
2398 if (dma_flag & DMA_RX_CRC_ERROR)
2399 u64_stats_inc(&stats->crc_errors);
2400 if (dma_flag & DMA_RX_OV)
2401 u64_stats_inc(&stats->over_errors);
2402 if (dma_flag & DMA_RX_NO)
2403 u64_stats_inc(&stats->frame_errors);
2404 if (dma_flag & DMA_RX_LG)
2405 u64_stats_inc(&stats->length_errors);
2406 if ((dma_flag & (DMA_RX_CRC_ERROR |
2407 DMA_RX_OV |
2408 DMA_RX_NO |
2409 DMA_RX_LG |
2410 DMA_RX_RXER)) == DMA_RX_RXER)
2411 u64_stats_inc(&stats->errors);
2412 u64_stats_update_end(&stats->syncp);
2413 dev_kfree_skb_any(skb);
2414 goto next;
2415 } /* error packet */
2416
2417 skb_put(skb, len);
2418
2419 /* remove RSB and hardware 2bytes added for IP alignment */
2420 skb_pull(skb, 66);
2421 len -= 66;
2422
2423 if (priv->crc_fwd_en) {
2424 skb_trim(skb, len - ETH_FCS_LEN);
2425 len -= ETH_FCS_LEN;
2426 }
2427
2428 bytes_processed += len;
2429
2430 /*Finish setting up the received SKB and send it to the kernel*/
2431 skb->protocol = eth_type_trans(skb, priv->dev);
2432
2433 u64_stats_update_begin(&stats->syncp);
2434 u64_stats_inc(&stats->packets);
2435 u64_stats_add(&stats->bytes, len);
2436 if (dma_flag & DMA_RX_MULT)
2437 u64_stats_inc(&stats->multicast);
2438 else if (dma_flag & DMA_RX_BRDCAST)
2439 u64_stats_inc(&stats->broadcast);
2440 u64_stats_update_end(&stats->syncp);
2441
2442 /* Notify kernel */
2443 napi_gro_receive(&ring->napi, skb);
2444 netif_dbg(priv, rx_status, dev, "pushed up to kernel\n");
2445
2446 next:
2447 rxpktprocessed++;
2448 if (likely(ring->read_ptr < ring->end_ptr))
2449 ring->read_ptr++;
2450 else
2451 ring->read_ptr = ring->cb_ptr;
2452
2453 ring->c_index = (ring->c_index + 1) & DMA_C_INDEX_MASK;
2454 bcmgenet_rdma_ring_writel(priv, ring->index, ring->c_index, RDMA_CONS_INDEX);
2455 }
2456
2457 ring->dim.bytes = bytes_processed;
2458 ring->dim.packets = rxpktprocessed;
2459
2460 return rxpktprocessed;
2461 }
2462
2463 /* Rx NAPI polling method */
bcmgenet_rx_poll(struct napi_struct * napi,int budget)2464 static int bcmgenet_rx_poll(struct napi_struct *napi, int budget)
2465 {
2466 struct bcmgenet_rx_ring *ring = container_of(napi,
2467 struct bcmgenet_rx_ring, napi);
2468 struct dim_sample dim_sample = {};
2469 unsigned int work_done;
2470
2471 work_done = bcmgenet_desc_rx(ring, budget);
2472
2473 if (work_done < budget && napi_complete_done(napi, work_done))
2474 bcmgenet_rx_ring_int_enable(ring);
2475
2476 if (ring->dim.use_dim) {
2477 dim_update_sample(ring->dim.event_ctr, ring->dim.packets,
2478 ring->dim.bytes, &dim_sample);
2479 net_dim(&ring->dim.dim, &dim_sample);
2480 }
2481
2482 return work_done;
2483 }
2484
bcmgenet_dim_work(struct work_struct * work)2485 static void bcmgenet_dim_work(struct work_struct *work)
2486 {
2487 struct dim *dim = container_of(work, struct dim, work);
2488 struct bcmgenet_net_dim *ndim =
2489 container_of(dim, struct bcmgenet_net_dim, dim);
2490 struct bcmgenet_rx_ring *ring =
2491 container_of(ndim, struct bcmgenet_rx_ring, dim);
2492 struct dim_cq_moder cur_profile =
2493 net_dim_get_rx_moderation(dim->mode, dim->profile_ix);
2494
2495 bcmgenet_set_rx_coalesce(ring, cur_profile.usec, cur_profile.pkts);
2496 dim->state = DIM_START_MEASURE;
2497 }
2498
2499 /* Assign skb to RX DMA descriptor. */
bcmgenet_alloc_rx_buffers(struct bcmgenet_priv * priv,struct bcmgenet_rx_ring * ring)2500 static int bcmgenet_alloc_rx_buffers(struct bcmgenet_priv *priv,
2501 struct bcmgenet_rx_ring *ring)
2502 {
2503 struct enet_cb *cb;
2504 struct sk_buff *skb;
2505 int i;
2506
2507 netif_dbg(priv, hw, priv->dev, "%s\n", __func__);
2508
2509 /* loop here for each buffer needing assign */
2510 for (i = 0; i < ring->size; i++) {
2511 cb = ring->cbs + i;
2512 skb = bcmgenet_rx_refill(priv, cb);
2513 if (skb)
2514 dev_consume_skb_any(skb);
2515 if (!cb->skb)
2516 return -ENOMEM;
2517 }
2518
2519 return 0;
2520 }
2521
bcmgenet_free_rx_buffers(struct bcmgenet_priv * priv)2522 static void bcmgenet_free_rx_buffers(struct bcmgenet_priv *priv)
2523 {
2524 struct sk_buff *skb;
2525 struct enet_cb *cb;
2526 int i;
2527
2528 for (i = 0; i < priv->num_rx_bds; i++) {
2529 cb = &priv->rx_cbs[i];
2530
2531 skb = bcmgenet_free_rx_cb(&priv->pdev->dev, cb);
2532 if (skb)
2533 dev_consume_skb_any(skb);
2534 }
2535 }
2536
umac_enable_set(struct bcmgenet_priv * priv,u32 mask,bool enable)2537 static void umac_enable_set(struct bcmgenet_priv *priv, u32 mask, bool enable)
2538 {
2539 u32 reg;
2540
2541 spin_lock_bh(&priv->reg_lock);
2542 reg = bcmgenet_umac_readl(priv, UMAC_CMD);
2543 if (reg & CMD_SW_RESET) {
2544 spin_unlock_bh(&priv->reg_lock);
2545 return;
2546 }
2547 if (enable)
2548 reg |= mask;
2549 else
2550 reg &= ~mask;
2551 bcmgenet_umac_writel(priv, reg, UMAC_CMD);
2552 spin_unlock_bh(&priv->reg_lock);
2553
2554 /* UniMAC stops on a packet boundary, wait for a full-size packet
2555 * to be processed
2556 */
2557 if (enable == 0)
2558 usleep_range(1000, 2000);
2559 }
2560
reset_umac(struct bcmgenet_priv * priv)2561 static void reset_umac(struct bcmgenet_priv *priv)
2562 {
2563 /* 7358a0/7552a0: bad default in RBUF_FLUSH_CTRL.umac_sw_rst */
2564 bcmgenet_rbuf_ctrl_set(priv, 0);
2565 udelay(10);
2566
2567 /* issue soft reset and disable MAC while updating its registers */
2568 spin_lock_bh(&priv->reg_lock);
2569 bcmgenet_umac_writel(priv, CMD_SW_RESET, UMAC_CMD);
2570 udelay(2);
2571 spin_unlock_bh(&priv->reg_lock);
2572 }
2573
bcmgenet_intr_disable(struct bcmgenet_priv * priv)2574 static void bcmgenet_intr_disable(struct bcmgenet_priv *priv)
2575 {
2576 /* Mask all interrupts.*/
2577 bcmgenet_intrl2_0_writel(priv, 0xFFFFFFFF, INTRL2_CPU_MASK_SET);
2578 bcmgenet_intrl2_0_writel(priv, 0xFFFFFFFF, INTRL2_CPU_CLEAR);
2579 bcmgenet_intrl2_1_writel(priv, 0xFFFFFFFF, INTRL2_CPU_MASK_SET);
2580 bcmgenet_intrl2_1_writel(priv, 0xFFFFFFFF, INTRL2_CPU_CLEAR);
2581 }
2582
bcmgenet_link_intr_enable(struct bcmgenet_priv * priv)2583 static void bcmgenet_link_intr_enable(struct bcmgenet_priv *priv)
2584 {
2585 u32 int0_enable = 0;
2586
2587 /* Monitor cable plug/unplugged event for internal PHY, external PHY
2588 * and MoCA PHY
2589 */
2590 if (priv->internal_phy) {
2591 int0_enable |= UMAC_IRQ_LINK_EVENT;
2592 if (GENET_IS_V1(priv) || GENET_IS_V2(priv) || GENET_IS_V3(priv))
2593 int0_enable |= UMAC_IRQ_PHY_DET_R;
2594 } else if (priv->ext_phy) {
2595 int0_enable |= UMAC_IRQ_LINK_EVENT;
2596 } else if (priv->phy_interface == PHY_INTERFACE_MODE_MOCA) {
2597 if (bcmgenet_has_moca_link_det(priv))
2598 int0_enable |= UMAC_IRQ_LINK_EVENT;
2599 }
2600 bcmgenet_intrl2_0_writel(priv, int0_enable, INTRL2_CPU_MASK_CLEAR);
2601 }
2602
init_umac(struct bcmgenet_priv * priv)2603 static void init_umac(struct bcmgenet_priv *priv)
2604 {
2605 struct device *kdev = &priv->pdev->dev;
2606 u32 reg;
2607 u32 int0_enable = 0;
2608
2609 dev_dbg(&priv->pdev->dev, "bcmgenet: init_umac\n");
2610
2611 reset_umac(priv);
2612
2613 /* clear tx/rx counter */
2614 bcmgenet_umac_writel(priv,
2615 MIB_RESET_RX | MIB_RESET_TX | MIB_RESET_RUNT,
2616 UMAC_MIB_CTRL);
2617 bcmgenet_umac_writel(priv, 0, UMAC_MIB_CTRL);
2618
2619 bcmgenet_umac_writel(priv, ENET_MAX_MTU_SIZE, UMAC_MAX_FRAME_LEN);
2620
2621 /* init tx registers, enable TSB */
2622 reg = bcmgenet_tbuf_ctrl_get(priv);
2623 reg |= TBUF_64B_EN;
2624 bcmgenet_tbuf_ctrl_set(priv, reg);
2625
2626 /* init rx registers, enable ip header optimization and RSB */
2627 reg = bcmgenet_rbuf_readl(priv, RBUF_CTRL);
2628 reg |= RBUF_ALIGN_2B | RBUF_64B_EN;
2629 bcmgenet_rbuf_writel(priv, reg, RBUF_CTRL);
2630
2631 /* enable rx checksumming */
2632 reg = bcmgenet_rbuf_readl(priv, RBUF_CHK_CTRL);
2633 reg |= RBUF_RXCHK_EN | RBUF_L3_PARSE_DIS;
2634 /* If UniMAC forwards CRC, we need to skip over it to get
2635 * a valid CHK bit to be set in the per-packet status word
2636 */
2637 if (priv->crc_fwd_en)
2638 reg |= RBUF_SKIP_FCS;
2639 else
2640 reg &= ~RBUF_SKIP_FCS;
2641 bcmgenet_rbuf_writel(priv, reg, RBUF_CHK_CTRL);
2642
2643 if (!GENET_IS_V1(priv) && !GENET_IS_V2(priv))
2644 bcmgenet_rbuf_writel(priv, 1, RBUF_TBUF_SIZE_CTRL);
2645
2646 bcmgenet_intr_disable(priv);
2647
2648 /* Configure backpressure vectors for MoCA */
2649 if (priv->phy_interface == PHY_INTERFACE_MODE_MOCA) {
2650 reg = bcmgenet_bp_mc_get(priv);
2651 reg |= BIT(priv->hw_params->bp_in_en_shift);
2652
2653 /* bp_mask: back pressure mask */
2654 if (netif_is_multiqueue(priv->dev))
2655 reg |= priv->hw_params->bp_in_mask;
2656 else
2657 reg &= ~priv->hw_params->bp_in_mask;
2658 bcmgenet_bp_mc_set(priv, reg);
2659 }
2660
2661 /* Enable MDIO interrupts on GENET v3+ */
2662 if (bcmgenet_has_mdio_intr(priv))
2663 int0_enable |= UMAC_IRQ_MDIO_EVENT;
2664
2665 bcmgenet_intrl2_0_writel(priv, int0_enable, INTRL2_CPU_MASK_CLEAR);
2666
2667 dev_dbg(kdev, "done init umac\n");
2668 }
2669
bcmgenet_init_dim(struct bcmgenet_rx_ring * ring,void (* cb)(struct work_struct * work))2670 static void bcmgenet_init_dim(struct bcmgenet_rx_ring *ring,
2671 void (*cb)(struct work_struct *work))
2672 {
2673 struct bcmgenet_net_dim *dim = &ring->dim;
2674
2675 INIT_WORK(&dim->dim.work, cb);
2676 dim->dim.mode = DIM_CQ_PERIOD_MODE_START_FROM_EQE;
2677 dim->event_ctr = 0;
2678 dim->packets = 0;
2679 dim->bytes = 0;
2680 }
2681
bcmgenet_init_rx_coalesce(struct bcmgenet_rx_ring * ring)2682 static void bcmgenet_init_rx_coalesce(struct bcmgenet_rx_ring *ring)
2683 {
2684 struct bcmgenet_net_dim *dim = &ring->dim;
2685 struct dim_cq_moder moder;
2686 u32 usecs, pkts;
2687
2688 usecs = ring->rx_coalesce_usecs;
2689 pkts = ring->rx_max_coalesced_frames;
2690
2691 /* If DIM was enabled, re-apply default parameters */
2692 if (dim->use_dim) {
2693 moder = net_dim_get_def_rx_moderation(dim->dim.mode);
2694 usecs = moder.usec;
2695 pkts = moder.pkts;
2696 }
2697
2698 bcmgenet_set_rx_coalesce(ring, usecs, pkts);
2699 }
2700
2701 /* Initialize a Tx ring along with corresponding hardware registers */
bcmgenet_init_tx_ring(struct bcmgenet_priv * priv,unsigned int index,unsigned int size,unsigned int start_ptr,unsigned int end_ptr)2702 static void bcmgenet_init_tx_ring(struct bcmgenet_priv *priv,
2703 unsigned int index, unsigned int size,
2704 unsigned int start_ptr, unsigned int end_ptr)
2705 {
2706 struct bcmgenet_tx_ring *ring = &priv->tx_rings[index];
2707 u32 words_per_bd = WORDS_PER_BD(priv);
2708 u32 flow_period_val = 0;
2709
2710 spin_lock_init(&ring->lock);
2711 ring->priv = priv;
2712 ring->index = index;
2713 ring->cbs = priv->tx_cbs + start_ptr;
2714 ring->size = size;
2715 ring->clean_ptr = start_ptr;
2716 ring->c_index = 0;
2717 ring->free_bds = size;
2718 ring->write_ptr = start_ptr;
2719 ring->cb_ptr = start_ptr;
2720 ring->end_ptr = end_ptr - 1;
2721 ring->prod_index = 0;
2722
2723 /* Set flow period for ring != 0 */
2724 if (index)
2725 flow_period_val = ENET_MAX_MTU_SIZE << 16;
2726
2727 bcmgenet_tdma_ring_writel(priv, index, 0, TDMA_PROD_INDEX);
2728 bcmgenet_tdma_ring_writel(priv, index, 0, TDMA_CONS_INDEX);
2729 bcmgenet_tdma_ring_writel(priv, index, 1, DMA_MBUF_DONE_THRESH);
2730 /* Disable rate control for now */
2731 bcmgenet_tdma_ring_writel(priv, index, flow_period_val,
2732 TDMA_FLOW_PERIOD);
2733 bcmgenet_tdma_ring_writel(priv, index,
2734 ((size << DMA_RING_SIZE_SHIFT) |
2735 RX_BUF_LENGTH), DMA_RING_BUF_SIZE);
2736
2737 /* Set start and end address, read and write pointers */
2738 bcmgenet_tdma_ring_writel(priv, index, start_ptr * words_per_bd,
2739 DMA_START_ADDR);
2740 bcmgenet_tdma_ring_writel(priv, index, start_ptr * words_per_bd,
2741 TDMA_READ_PTR);
2742 bcmgenet_tdma_ring_writel(priv, index, start_ptr * words_per_bd,
2743 TDMA_WRITE_PTR);
2744 bcmgenet_tdma_ring_writel(priv, index, end_ptr * words_per_bd - 1,
2745 DMA_END_ADDR);
2746
2747 /* Initialize Tx NAPI */
2748 netif_napi_add_tx(priv->dev, &ring->napi, bcmgenet_tx_poll);
2749 }
2750
2751 /* Initialize a RDMA ring */
bcmgenet_init_rx_ring(struct bcmgenet_priv * priv,unsigned int index,unsigned int size,unsigned int start_ptr,unsigned int end_ptr)2752 static int bcmgenet_init_rx_ring(struct bcmgenet_priv *priv,
2753 unsigned int index, unsigned int size,
2754 unsigned int start_ptr, unsigned int end_ptr)
2755 {
2756 struct bcmgenet_rx_ring *ring = &priv->rx_rings[index];
2757 u32 words_per_bd = WORDS_PER_BD(priv);
2758 int ret;
2759
2760 ring->priv = priv;
2761 ring->index = index;
2762 ring->cbs = priv->rx_cbs + start_ptr;
2763 ring->size = size;
2764 ring->c_index = 0;
2765 ring->read_ptr = start_ptr;
2766 ring->cb_ptr = start_ptr;
2767 ring->end_ptr = end_ptr - 1;
2768
2769 ret = bcmgenet_alloc_rx_buffers(priv, ring);
2770 if (ret)
2771 return ret;
2772
2773 bcmgenet_init_dim(ring, bcmgenet_dim_work);
2774 bcmgenet_init_rx_coalesce(ring);
2775
2776 /* Initialize Rx NAPI */
2777 netif_napi_add(priv->dev, &ring->napi, bcmgenet_rx_poll);
2778
2779 bcmgenet_rdma_ring_writel(priv, index, 0, RDMA_PROD_INDEX);
2780 bcmgenet_rdma_ring_writel(priv, index, 0, RDMA_CONS_INDEX);
2781 bcmgenet_rdma_ring_writel(priv, index,
2782 ((size << DMA_RING_SIZE_SHIFT) |
2783 RX_BUF_LENGTH), DMA_RING_BUF_SIZE);
2784 bcmgenet_rdma_ring_writel(priv, index,
2785 (DMA_FC_THRESH_LO <<
2786 DMA_XOFF_THRESHOLD_SHIFT) |
2787 DMA_FC_THRESH_HI, RDMA_XON_XOFF_THRESH);
2788
2789 /* Set start and end address, read and write pointers */
2790 bcmgenet_rdma_ring_writel(priv, index, start_ptr * words_per_bd,
2791 DMA_START_ADDR);
2792 bcmgenet_rdma_ring_writel(priv, index, start_ptr * words_per_bd,
2793 RDMA_READ_PTR);
2794 bcmgenet_rdma_ring_writel(priv, index, start_ptr * words_per_bd,
2795 RDMA_WRITE_PTR);
2796 bcmgenet_rdma_ring_writel(priv, index, end_ptr * words_per_bd - 1,
2797 DMA_END_ADDR);
2798
2799 return ret;
2800 }
2801
bcmgenet_enable_tx_napi(struct bcmgenet_priv * priv)2802 static void bcmgenet_enable_tx_napi(struct bcmgenet_priv *priv)
2803 {
2804 unsigned int i;
2805 struct bcmgenet_tx_ring *ring;
2806
2807 for (i = 0; i <= priv->hw_params->tx_queues; ++i) {
2808 ring = &priv->tx_rings[i];
2809 napi_enable(&ring->napi);
2810 bcmgenet_tx_ring_int_enable(ring);
2811 }
2812 }
2813
bcmgenet_disable_tx_napi(struct bcmgenet_priv * priv)2814 static void bcmgenet_disable_tx_napi(struct bcmgenet_priv *priv)
2815 {
2816 unsigned int i;
2817 struct bcmgenet_tx_ring *ring;
2818
2819 for (i = 0; i <= priv->hw_params->tx_queues; ++i) {
2820 ring = &priv->tx_rings[i];
2821 napi_disable(&ring->napi);
2822 }
2823 }
2824
bcmgenet_fini_tx_napi(struct bcmgenet_priv * priv)2825 static void bcmgenet_fini_tx_napi(struct bcmgenet_priv *priv)
2826 {
2827 unsigned int i;
2828 struct bcmgenet_tx_ring *ring;
2829
2830 for (i = 0; i <= priv->hw_params->tx_queues; ++i) {
2831 ring = &priv->tx_rings[i];
2832 netif_napi_del(&ring->napi);
2833 }
2834 }
2835
bcmgenet_tdma_disable(struct bcmgenet_priv * priv)2836 static int bcmgenet_tdma_disable(struct bcmgenet_priv *priv)
2837 {
2838 int timeout = 0;
2839 u32 reg, mask;
2840
2841 reg = bcmgenet_tdma_readl(priv, DMA_CTRL);
2842 mask = (1 << (priv->hw_params->tx_queues + 1)) - 1;
2843 mask = (mask << DMA_RING_BUF_EN_SHIFT) | DMA_EN;
2844 reg &= ~mask;
2845 bcmgenet_tdma_writel(priv, reg, DMA_CTRL);
2846
2847 /* Check DMA status register to confirm DMA is disabled */
2848 while (timeout++ < DMA_TIMEOUT_VAL) {
2849 reg = bcmgenet_tdma_readl(priv, DMA_STATUS);
2850 if ((reg & mask) == mask)
2851 return 0;
2852
2853 udelay(1);
2854 }
2855
2856 return -ETIMEDOUT;
2857 }
2858
bcmgenet_rdma_disable(struct bcmgenet_priv * priv)2859 static int bcmgenet_rdma_disable(struct bcmgenet_priv *priv)
2860 {
2861 int timeout = 0;
2862 u32 reg, mask;
2863
2864 reg = bcmgenet_rdma_readl(priv, DMA_CTRL);
2865 mask = (1 << (priv->hw_params->rx_queues + 1)) - 1;
2866 mask = (mask << DMA_RING_BUF_EN_SHIFT) | DMA_EN;
2867 reg &= ~mask;
2868 bcmgenet_rdma_writel(priv, reg, DMA_CTRL);
2869
2870 /* Check DMA status register to confirm DMA is disabled */
2871 while (timeout++ < DMA_TIMEOUT_VAL) {
2872 reg = bcmgenet_rdma_readl(priv, DMA_STATUS);
2873 if ((reg & mask) == mask)
2874 return 0;
2875
2876 udelay(1);
2877 }
2878
2879 return -ETIMEDOUT;
2880 }
2881
2882 /* Initialize Tx queues
2883 *
2884 * Queues 1-4 are priority-based, each one has 32 descriptors,
2885 * with queue 1 being the highest priority queue.
2886 *
2887 * Queue 0 is the default Tx queue with
2888 * GENET_Q0_TX_BD_CNT = 256 - 4 * 32 = 128 descriptors.
2889 *
2890 * The transmit control block pool is then partitioned as follows:
2891 * - Tx queue 0 uses tx_cbs[0..127]
2892 * - Tx queue 1 uses tx_cbs[128..159]
2893 * - Tx queue 2 uses tx_cbs[160..191]
2894 * - Tx queue 3 uses tx_cbs[192..223]
2895 * - Tx queue 4 uses tx_cbs[224..255]
2896 */
bcmgenet_init_tx_queues(struct net_device * dev)2897 static void bcmgenet_init_tx_queues(struct net_device *dev)
2898 {
2899 struct bcmgenet_priv *priv = netdev_priv(dev);
2900 unsigned int start = 0, end = GENET_Q0_TX_BD_CNT;
2901 u32 i, ring_mask, dma_priority[3] = {0, 0, 0};
2902
2903 /* Enable strict priority arbiter mode */
2904 bcmgenet_tdma_writel(priv, DMA_ARBITER_SP, DMA_ARB_CTRL);
2905
2906 /* Initialize Tx priority queues */
2907 for (i = 0; i <= priv->hw_params->tx_queues; i++) {
2908 bcmgenet_init_tx_ring(priv, i, end - start, start, end);
2909 start = end;
2910 end += priv->hw_params->tx_bds_per_q;
2911 dma_priority[DMA_PRIO_REG_INDEX(i)] |=
2912 (i ? GENET_Q1_PRIORITY : GENET_Q0_PRIORITY)
2913 << DMA_PRIO_REG_SHIFT(i);
2914 }
2915
2916 /* Set Tx queue priorities */
2917 bcmgenet_tdma_writel(priv, dma_priority[0], DMA_PRIORITY_0);
2918 bcmgenet_tdma_writel(priv, dma_priority[1], DMA_PRIORITY_1);
2919 bcmgenet_tdma_writel(priv, dma_priority[2], DMA_PRIORITY_2);
2920
2921 /* Configure Tx queues as descriptor rings */
2922 ring_mask = (1 << (priv->hw_params->tx_queues + 1)) - 1;
2923 bcmgenet_tdma_writel(priv, ring_mask, DMA_RING_CFG);
2924
2925 /* Enable Tx rings */
2926 ring_mask <<= DMA_RING_BUF_EN_SHIFT;
2927 bcmgenet_tdma_writel(priv, ring_mask, DMA_CTRL);
2928 }
2929
bcmgenet_enable_rx_napi(struct bcmgenet_priv * priv)2930 static void bcmgenet_enable_rx_napi(struct bcmgenet_priv *priv)
2931 {
2932 unsigned int i;
2933 struct bcmgenet_rx_ring *ring;
2934
2935 for (i = 0; i <= priv->hw_params->rx_queues; ++i) {
2936 ring = &priv->rx_rings[i];
2937 napi_enable(&ring->napi);
2938 bcmgenet_rx_ring_int_enable(ring);
2939 }
2940 }
2941
bcmgenet_disable_rx_napi(struct bcmgenet_priv * priv)2942 static void bcmgenet_disable_rx_napi(struct bcmgenet_priv *priv)
2943 {
2944 unsigned int i;
2945 struct bcmgenet_rx_ring *ring;
2946
2947 for (i = 0; i <= priv->hw_params->rx_queues; ++i) {
2948 ring = &priv->rx_rings[i];
2949 napi_disable(&ring->napi);
2950 cancel_work_sync(&ring->dim.dim.work);
2951 }
2952 }
2953
bcmgenet_fini_rx_napi(struct bcmgenet_priv * priv)2954 static void bcmgenet_fini_rx_napi(struct bcmgenet_priv *priv)
2955 {
2956 unsigned int i;
2957 struct bcmgenet_rx_ring *ring;
2958
2959 for (i = 0; i <= priv->hw_params->rx_queues; ++i) {
2960 ring = &priv->rx_rings[i];
2961 netif_napi_del(&ring->napi);
2962 }
2963 }
2964
2965 /* Initialize Rx queues
2966 *
2967 * Queues 0-15 are priority queues. Hardware Filtering Block (HFB) can be
2968 * used to direct traffic to these queues.
2969 *
2970 * Queue 0 is also the default Rx queue with GENET_Q0_RX_BD_CNT descriptors.
2971 */
bcmgenet_init_rx_queues(struct net_device * dev)2972 static int bcmgenet_init_rx_queues(struct net_device *dev)
2973 {
2974 struct bcmgenet_priv *priv = netdev_priv(dev);
2975 unsigned int start = 0, end = GENET_Q0_RX_BD_CNT;
2976 u32 i, ring_mask;
2977 int ret;
2978
2979 /* Initialize Rx priority queues */
2980 for (i = 0; i <= priv->hw_params->rx_queues; i++) {
2981 ret = bcmgenet_init_rx_ring(priv, i, end - start, start, end);
2982 if (ret)
2983 return ret;
2984
2985 start = end;
2986 end += priv->hw_params->rx_bds_per_q;
2987 }
2988
2989 /* Configure Rx queues as descriptor rings */
2990 ring_mask = (1 << (priv->hw_params->rx_queues + 1)) - 1;
2991 bcmgenet_rdma_writel(priv, ring_mask, DMA_RING_CFG);
2992
2993 /* Enable Rx rings */
2994 ring_mask <<= DMA_RING_BUF_EN_SHIFT;
2995 bcmgenet_rdma_writel(priv, ring_mask, DMA_CTRL);
2996
2997 return 0;
2998 }
2999
bcmgenet_dma_teardown(struct bcmgenet_priv * priv)3000 static int bcmgenet_dma_teardown(struct bcmgenet_priv *priv)
3001 {
3002 int ret = 0;
3003
3004 /* Disable TDMA to stop add more frames in TX DMA */
3005 if (-ETIMEDOUT == bcmgenet_tdma_disable(priv)) {
3006 netdev_warn(priv->dev, "Timed out while disabling TX DMA\n");
3007 ret = -ETIMEDOUT;
3008 }
3009
3010 /* Wait 10ms for packet drain in both tx and rx dma */
3011 usleep_range(10000, 20000);
3012
3013 /* Disable RDMA */
3014 if (-ETIMEDOUT == bcmgenet_rdma_disable(priv)) {
3015 netdev_warn(priv->dev, "Timed out while disabling RX DMA\n");
3016 ret = -ETIMEDOUT;
3017 }
3018
3019 return ret;
3020 }
3021
bcmgenet_fini_dma(struct bcmgenet_priv * priv)3022 static void bcmgenet_fini_dma(struct bcmgenet_priv *priv)
3023 {
3024 struct netdev_queue *txq;
3025 int i;
3026
3027 bcmgenet_fini_rx_napi(priv);
3028 bcmgenet_fini_tx_napi(priv);
3029
3030 for (i = 0; i <= priv->hw_params->tx_queues; i++) {
3031 txq = netdev_get_tx_queue(priv->dev, i);
3032 netdev_tx_reset_queue(txq);
3033 }
3034
3035 bcmgenet_free_rx_buffers(priv);
3036 kfree(priv->rx_cbs);
3037 kfree(priv->tx_cbs);
3038 }
3039
3040 /* init_edma: Initialize DMA control register */
bcmgenet_init_dma(struct bcmgenet_priv * priv,bool flush_rx)3041 static int bcmgenet_init_dma(struct bcmgenet_priv *priv, bool flush_rx)
3042 {
3043 struct enet_cb *cb;
3044 unsigned int i;
3045 int ret;
3046 u32 reg;
3047
3048 netif_dbg(priv, hw, priv->dev, "%s\n", __func__);
3049
3050 /* Disable TX DMA */
3051 ret = bcmgenet_tdma_disable(priv);
3052 if (ret) {
3053 netdev_err(priv->dev, "failed to halt Tx DMA\n");
3054 return ret;
3055 }
3056
3057 /* Disable RX DMA */
3058 ret = bcmgenet_rdma_disable(priv);
3059 if (ret) {
3060 netdev_err(priv->dev, "failed to halt Rx DMA\n");
3061 return ret;
3062 }
3063
3064 /* Flush TX queues */
3065 bcmgenet_umac_writel(priv, 1, UMAC_TX_FLUSH);
3066 udelay(10);
3067 bcmgenet_umac_writel(priv, 0, UMAC_TX_FLUSH);
3068
3069 if (flush_rx) {
3070 reg = bcmgenet_rbuf_ctrl_get(priv);
3071 bcmgenet_rbuf_ctrl_set(priv, reg | BIT(0));
3072 udelay(10);
3073 bcmgenet_rbuf_ctrl_set(priv, reg);
3074 udelay(10);
3075 }
3076
3077 /* Initialize common Rx ring structures */
3078 priv->rx_bds = priv->base + priv->hw_params->rdma_offset;
3079 priv->num_rx_bds = TOTAL_DESC;
3080 priv->rx_cbs = kzalloc_objs(struct enet_cb, priv->num_rx_bds);
3081 if (!priv->rx_cbs)
3082 return -ENOMEM;
3083
3084 for (i = 0; i < priv->num_rx_bds; i++) {
3085 cb = priv->rx_cbs + i;
3086 cb->bd_addr = priv->rx_bds + i * DMA_DESC_SIZE;
3087 }
3088
3089 /* Initialize common TX ring structures */
3090 priv->tx_bds = priv->base + priv->hw_params->tdma_offset;
3091 priv->num_tx_bds = TOTAL_DESC;
3092 priv->tx_cbs = kzalloc_objs(struct enet_cb, priv->num_tx_bds);
3093 if (!priv->tx_cbs) {
3094 kfree(priv->rx_cbs);
3095 return -ENOMEM;
3096 }
3097
3098 for (i = 0; i < priv->num_tx_bds; i++) {
3099 cb = priv->tx_cbs + i;
3100 cb->bd_addr = priv->tx_bds + i * DMA_DESC_SIZE;
3101 }
3102
3103 /* Init rDma */
3104 bcmgenet_rdma_writel(priv, priv->dma_max_burst_length,
3105 DMA_SCB_BURST_SIZE);
3106
3107 /* Initialize Rx queues */
3108 ret = bcmgenet_init_rx_queues(priv->dev);
3109 if (ret) {
3110 netdev_err(priv->dev, "failed to initialize Rx queues\n");
3111 bcmgenet_free_rx_buffers(priv);
3112 kfree(priv->rx_cbs);
3113 kfree(priv->tx_cbs);
3114 return ret;
3115 }
3116
3117 /* Init tDma */
3118 bcmgenet_tdma_writel(priv, priv->dma_max_burst_length,
3119 DMA_SCB_BURST_SIZE);
3120
3121 /* Initialize Tx queues */
3122 bcmgenet_init_tx_queues(priv->dev);
3123
3124 /* Enable RX/TX DMA */
3125 reg = bcmgenet_rdma_readl(priv, DMA_CTRL);
3126 reg |= DMA_EN;
3127 bcmgenet_rdma_writel(priv, reg, DMA_CTRL);
3128
3129 reg = bcmgenet_tdma_readl(priv, DMA_CTRL);
3130 reg |= DMA_EN;
3131 bcmgenet_tdma_writel(priv, reg, DMA_CTRL);
3132
3133 return 0;
3134 }
3135
3136 /* Interrupt bottom half */
bcmgenet_irq_task(struct work_struct * work)3137 static void bcmgenet_irq_task(struct work_struct *work)
3138 {
3139 unsigned int status;
3140 struct bcmgenet_priv *priv = container_of(
3141 work, struct bcmgenet_priv, bcmgenet_irq_work);
3142
3143 netif_dbg(priv, intr, priv->dev, "%s\n", __func__);
3144
3145 spin_lock_irq(&priv->lock);
3146 status = priv->irq0_stat;
3147 priv->irq0_stat = 0;
3148 spin_unlock_irq(&priv->lock);
3149
3150 if (status & UMAC_IRQ_PHY_DET_R &&
3151 priv->dev->phydev->autoneg != AUTONEG_ENABLE) {
3152 phy_init_hw(priv->dev->phydev);
3153 genphy_config_aneg(priv->dev->phydev);
3154 }
3155
3156 /* Link UP/DOWN event */
3157 if (status & UMAC_IRQ_LINK_EVENT)
3158 phy_mac_interrupt(priv->dev->phydev);
3159
3160 }
3161
3162 /* bcmgenet_isr1: handle Rx and Tx queues */
bcmgenet_isr1(int irq,void * dev_id)3163 static irqreturn_t bcmgenet_isr1(int irq, void *dev_id)
3164 {
3165 struct bcmgenet_priv *priv = dev_id;
3166 struct bcmgenet_rx_ring *rx_ring;
3167 struct bcmgenet_tx_ring *tx_ring;
3168 unsigned int index, status;
3169
3170 /* Read irq status */
3171 status = bcmgenet_intrl2_1_readl(priv, INTRL2_CPU_STAT) &
3172 ~bcmgenet_intrl2_1_readl(priv, INTRL2_CPU_MASK_STATUS);
3173
3174 /* clear interrupts */
3175 bcmgenet_intrl2_1_writel(priv, status, INTRL2_CPU_CLEAR);
3176
3177 netif_dbg(priv, intr, priv->dev,
3178 "%s: IRQ=0x%x\n", __func__, status);
3179
3180 /* Check Rx priority queue interrupts */
3181 for (index = 0; index <= priv->hw_params->rx_queues; index++) {
3182 if (!(status & BIT(UMAC_IRQ1_RX_INTR_SHIFT + index)))
3183 continue;
3184
3185 rx_ring = &priv->rx_rings[index];
3186 rx_ring->dim.event_ctr++;
3187
3188 if (likely(napi_schedule_prep(&rx_ring->napi))) {
3189 bcmgenet_rx_ring_int_disable(rx_ring);
3190 __napi_schedule_irqoff(&rx_ring->napi);
3191 }
3192 }
3193
3194 /* Check Tx priority queue interrupts */
3195 for (index = 0; index <= priv->hw_params->tx_queues; index++) {
3196 if (!(status & BIT(index)))
3197 continue;
3198
3199 tx_ring = &priv->tx_rings[index];
3200
3201 if (likely(napi_schedule_prep(&tx_ring->napi))) {
3202 bcmgenet_tx_ring_int_disable(tx_ring);
3203 __napi_schedule_irqoff(&tx_ring->napi);
3204 }
3205 }
3206
3207 return IRQ_HANDLED;
3208 }
3209
3210 /* bcmgenet_isr0: handle other stuff */
bcmgenet_isr0(int irq,void * dev_id)3211 static irqreturn_t bcmgenet_isr0(int irq, void *dev_id)
3212 {
3213 struct bcmgenet_priv *priv = dev_id;
3214 unsigned int status;
3215 unsigned long flags;
3216
3217 /* Read irq status */
3218 status = bcmgenet_intrl2_0_readl(priv, INTRL2_CPU_STAT) &
3219 ~bcmgenet_intrl2_0_readl(priv, INTRL2_CPU_MASK_STATUS);
3220
3221 /* clear interrupts */
3222 bcmgenet_intrl2_0_writel(priv, status, INTRL2_CPU_CLEAR);
3223
3224 netif_dbg(priv, intr, priv->dev,
3225 "IRQ=0x%x\n", status);
3226
3227 if (bcmgenet_has_mdio_intr(priv) && status & UMAC_IRQ_MDIO_EVENT)
3228 wake_up(&priv->wq);
3229
3230 /* all other interested interrupts handled in bottom half */
3231 status &= (UMAC_IRQ_LINK_EVENT | UMAC_IRQ_PHY_DET_R);
3232 if (status) {
3233 /* Save irq status for bottom-half processing. */
3234 spin_lock_irqsave(&priv->lock, flags);
3235 priv->irq0_stat |= status;
3236 spin_unlock_irqrestore(&priv->lock, flags);
3237
3238 schedule_work(&priv->bcmgenet_irq_work);
3239 }
3240
3241 return IRQ_HANDLED;
3242 }
3243
bcmgenet_wol_isr(int irq,void * dev_id)3244 static irqreturn_t bcmgenet_wol_isr(int irq, void *dev_id)
3245 {
3246 /* Acknowledge the interrupt */
3247 return IRQ_HANDLED;
3248 }
3249
bcmgenet_umac_reset(struct bcmgenet_priv * priv)3250 static void bcmgenet_umac_reset(struct bcmgenet_priv *priv)
3251 {
3252 u32 reg;
3253
3254 reg = bcmgenet_rbuf_ctrl_get(priv);
3255 reg |= BIT(1);
3256 bcmgenet_rbuf_ctrl_set(priv, reg);
3257 udelay(10);
3258
3259 reg &= ~BIT(1);
3260 bcmgenet_rbuf_ctrl_set(priv, reg);
3261 udelay(10);
3262 }
3263
bcmgenet_set_hw_addr(struct bcmgenet_priv * priv,const unsigned char * addr)3264 static void bcmgenet_set_hw_addr(struct bcmgenet_priv *priv,
3265 const unsigned char *addr)
3266 {
3267 bcmgenet_umac_writel(priv, get_unaligned_be32(&addr[0]), UMAC_MAC0);
3268 bcmgenet_umac_writel(priv, get_unaligned_be16(&addr[4]), UMAC_MAC1);
3269 }
3270
bcmgenet_get_hw_addr(struct bcmgenet_priv * priv,unsigned char * addr)3271 static void bcmgenet_get_hw_addr(struct bcmgenet_priv *priv,
3272 unsigned char *addr)
3273 {
3274 u32 addr_tmp;
3275
3276 addr_tmp = bcmgenet_umac_readl(priv, UMAC_MAC0);
3277 put_unaligned_be32(addr_tmp, &addr[0]);
3278 addr_tmp = bcmgenet_umac_readl(priv, UMAC_MAC1);
3279 put_unaligned_be16(addr_tmp, &addr[4]);
3280 }
3281
bcmgenet_netif_start(struct net_device * dev)3282 static void bcmgenet_netif_start(struct net_device *dev)
3283 {
3284 struct bcmgenet_priv *priv = netdev_priv(dev);
3285
3286 /* Start the network engine */
3287 netif_addr_lock_bh(dev);
3288 bcmgenet_set_rx_mode(dev);
3289 netif_addr_unlock_bh(dev);
3290 bcmgenet_enable_rx_napi(priv);
3291
3292 umac_enable_set(priv, CMD_TX_EN | CMD_RX_EN, true);
3293
3294 bcmgenet_enable_tx_napi(priv);
3295
3296 /* Monitor link interrupts now */
3297 bcmgenet_link_intr_enable(priv);
3298
3299 phy_start(dev->phydev);
3300 }
3301
bcmgenet_open(struct net_device * dev)3302 static int bcmgenet_open(struct net_device *dev)
3303 {
3304 struct bcmgenet_priv *priv = netdev_priv(dev);
3305 int ret;
3306
3307 netif_dbg(priv, ifup, dev, "bcmgenet_open\n");
3308
3309 /* Turn on the clock */
3310 clk_prepare_enable(priv->clk);
3311
3312 /* If this is an internal GPHY, power it back on now, before UniMAC is
3313 * brought out of reset as absolutely no UniMAC activity is allowed
3314 */
3315 if (priv->internal_phy)
3316 bcmgenet_power_up(priv, GENET_POWER_PASSIVE);
3317
3318 /* take MAC out of reset */
3319 bcmgenet_umac_reset(priv);
3320
3321 init_umac(priv);
3322
3323 /* Apply features again in case we changed them while interface was
3324 * down
3325 */
3326 bcmgenet_set_features(dev, dev->features);
3327
3328 bcmgenet_set_hw_addr(priv, dev->dev_addr);
3329
3330 /* HFB init */
3331 bcmgenet_hfb_init(priv);
3332
3333 /* Reinitialize TDMA and RDMA and SW housekeeping */
3334 ret = bcmgenet_init_dma(priv, true);
3335 if (ret) {
3336 netdev_err(dev, "failed to initialize DMA\n");
3337 goto err_clk_disable;
3338 }
3339
3340 ret = request_irq(priv->irq0, bcmgenet_isr0, IRQF_SHARED,
3341 dev->name, priv);
3342 if (ret < 0) {
3343 netdev_err(dev, "can't request IRQ %d\n", priv->irq0);
3344 goto err_fini_dma;
3345 }
3346
3347 ret = request_irq(priv->irq1, bcmgenet_isr1, IRQF_SHARED,
3348 dev->name, priv);
3349 if (ret < 0) {
3350 netdev_err(dev, "can't request IRQ %d\n", priv->irq1);
3351 goto err_irq0;
3352 }
3353
3354 ret = bcmgenet_mii_probe(dev);
3355 if (ret) {
3356 netdev_err(dev, "failed to connect to PHY\n");
3357 goto err_irq1;
3358 }
3359
3360 bcmgenet_phy_pause_set(dev, priv->rx_pause, priv->tx_pause);
3361
3362 bcmgenet_netif_start(dev);
3363
3364 netif_tx_start_all_queues(dev);
3365
3366 return 0;
3367
3368 err_irq1:
3369 free_irq(priv->irq1, priv);
3370 err_irq0:
3371 free_irq(priv->irq0, priv);
3372 err_fini_dma:
3373 bcmgenet_dma_teardown(priv);
3374 bcmgenet_fini_dma(priv);
3375 err_clk_disable:
3376 if (priv->internal_phy)
3377 bcmgenet_power_down(priv, GENET_POWER_PASSIVE);
3378 clk_disable_unprepare(priv->clk);
3379 return ret;
3380 }
3381
bcmgenet_netif_stop(struct net_device * dev,bool stop_phy)3382 static void bcmgenet_netif_stop(struct net_device *dev, bool stop_phy)
3383 {
3384 struct bcmgenet_priv *priv = netdev_priv(dev);
3385
3386 netif_tx_disable(dev);
3387
3388 /* Disable MAC receive */
3389 bcmgenet_hfb_reg_writel(priv, 0, HFB_CTRL);
3390 umac_enable_set(priv, CMD_RX_EN, false);
3391
3392 if (stop_phy)
3393 phy_stop(dev->phydev);
3394
3395 bcmgenet_dma_teardown(priv);
3396
3397 /* Disable MAC transmit. TX DMA disabled must be done before this */
3398 umac_enable_set(priv, CMD_TX_EN, false);
3399
3400 bcmgenet_disable_tx_napi(priv);
3401 bcmgenet_disable_rx_napi(priv);
3402 bcmgenet_intr_disable(priv);
3403
3404 /* Wait for pending work items to complete. Since interrupts are
3405 * disabled no new work will be scheduled.
3406 */
3407 cancel_work_sync(&priv->bcmgenet_irq_work);
3408
3409 /* tx reclaim */
3410 bcmgenet_tx_reclaim_all(dev);
3411 bcmgenet_fini_dma(priv);
3412 }
3413
bcmgenet_close(struct net_device * dev)3414 static int bcmgenet_close(struct net_device *dev)
3415 {
3416 struct bcmgenet_priv *priv = netdev_priv(dev);
3417 int ret = 0;
3418
3419 netif_dbg(priv, ifdown, dev, "bcmgenet_close\n");
3420
3421 bcmgenet_netif_stop(dev, false);
3422
3423 /* Really kill the PHY state machine and disconnect from it */
3424 phy_disconnect(dev->phydev);
3425
3426 free_irq(priv->irq0, priv);
3427 free_irq(priv->irq1, priv);
3428
3429 if (priv->internal_phy)
3430 ret = bcmgenet_power_down(priv, GENET_POWER_PASSIVE);
3431
3432 clk_disable_unprepare(priv->clk);
3433
3434 return ret;
3435 }
3436
bcmgenet_dump_tx_queue(struct bcmgenet_tx_ring * ring)3437 static void bcmgenet_dump_tx_queue(struct bcmgenet_tx_ring *ring)
3438 {
3439 struct bcmgenet_priv *priv = ring->priv;
3440 u32 p_index, c_index, intsts, intmsk;
3441 struct netdev_queue *txq;
3442 unsigned int free_bds;
3443 bool txq_stopped;
3444
3445 if (!netif_msg_tx_err(priv))
3446 return;
3447
3448 txq = netdev_get_tx_queue(priv->dev, ring->index);
3449
3450 spin_lock(&ring->lock);
3451 intsts = ~bcmgenet_intrl2_1_readl(priv, INTRL2_CPU_MASK_STATUS);
3452 intmsk = 1 << ring->index;
3453 c_index = bcmgenet_tdma_ring_readl(priv, ring->index, TDMA_CONS_INDEX);
3454 p_index = bcmgenet_tdma_ring_readl(priv, ring->index, TDMA_PROD_INDEX);
3455 txq_stopped = netif_tx_queue_stopped(txq);
3456 free_bds = ring->free_bds;
3457 spin_unlock(&ring->lock);
3458
3459 netif_err(priv, tx_err, priv->dev, "Ring %d queue %d status summary\n"
3460 "TX queue status: %s, interrupts: %s\n"
3461 "(sw)free_bds: %d (sw)size: %d\n"
3462 "(sw)p_index: %d (hw)p_index: %d\n"
3463 "(sw)c_index: %d (hw)c_index: %d\n"
3464 "(sw)clean_p: %d (sw)write_p: %d\n"
3465 "(sw)cb_ptr: %d (sw)end_ptr: %d\n",
3466 ring->index, ring->index,
3467 txq_stopped ? "stopped" : "active",
3468 intsts & intmsk ? "enabled" : "disabled",
3469 free_bds, ring->size,
3470 ring->prod_index, p_index & DMA_P_INDEX_MASK,
3471 ring->c_index, c_index & DMA_C_INDEX_MASK,
3472 ring->clean_ptr, ring->write_ptr,
3473 ring->cb_ptr, ring->end_ptr);
3474 }
3475
bcmgenet_timeout(struct net_device * dev,unsigned int txqueue)3476 static void bcmgenet_timeout(struct net_device *dev, unsigned int txqueue)
3477 {
3478 struct bcmgenet_priv *priv = netdev_priv(dev);
3479 struct bcmgenet_tx_ring *ring = &priv->tx_rings[txqueue];
3480 struct netdev_queue *txq = netdev_get_tx_queue(dev, txqueue);
3481
3482 netif_dbg(priv, tx_err, dev, "bcmgenet_timeout\n");
3483
3484 bcmgenet_dump_tx_queue(ring);
3485
3486 bcmgenet_tx_reclaim(dev, ring, true);
3487
3488 /* Re-enable the TX interrupt for this ring */
3489 bcmgenet_intrl2_1_writel(priv, 1 << txqueue, INTRL2_CPU_MASK_CLEAR);
3490
3491 txq_trans_cond_update(txq);
3492
3493 BCMGENET_STATS64_INC((&ring->stats64), errors);
3494
3495 netif_tx_wake_queue(txq);
3496 }
3497
3498 #define MAX_MDF_FILTER 17
3499
bcmgenet_set_mdf_addr(struct bcmgenet_priv * priv,const unsigned char * addr,int * i)3500 static inline void bcmgenet_set_mdf_addr(struct bcmgenet_priv *priv,
3501 const unsigned char *addr,
3502 int *i)
3503 {
3504 bcmgenet_umac_writel(priv, addr[0] << 8 | addr[1],
3505 UMAC_MDF_ADDR + (*i * 4));
3506 bcmgenet_umac_writel(priv, addr[2] << 24 | addr[3] << 16 |
3507 addr[4] << 8 | addr[5],
3508 UMAC_MDF_ADDR + ((*i + 1) * 4));
3509 *i += 2;
3510 }
3511
bcmgenet_set_rx_mode(struct net_device * dev)3512 static void bcmgenet_set_rx_mode(struct net_device *dev)
3513 {
3514 struct bcmgenet_priv *priv = netdev_priv(dev);
3515 struct netdev_hw_addr *ha;
3516 int i, nfilter;
3517 u32 reg;
3518
3519 netif_dbg(priv, hw, dev, "%s: %08X\n", __func__, dev->flags);
3520
3521 /* Number of filters needed */
3522 nfilter = netdev_uc_count(dev) + netdev_mc_count(dev) + 2;
3523
3524 /*
3525 * Turn on promicuous mode for three scenarios
3526 * 1. IFF_PROMISC flag is set
3527 * 2. IFF_ALLMULTI flag is set
3528 * 3. The number of filters needed exceeds the number filters
3529 * supported by the hardware.
3530 */
3531 spin_lock(&priv->reg_lock);
3532 reg = bcmgenet_umac_readl(priv, UMAC_CMD);
3533 if ((dev->flags & (IFF_PROMISC | IFF_ALLMULTI)) ||
3534 (nfilter > MAX_MDF_FILTER)) {
3535 reg |= CMD_PROMISC;
3536 bcmgenet_umac_writel(priv, reg, UMAC_CMD);
3537 spin_unlock(&priv->reg_lock);
3538 bcmgenet_umac_writel(priv, 0, UMAC_MDF_CTRL);
3539 return;
3540 } else {
3541 reg &= ~CMD_PROMISC;
3542 bcmgenet_umac_writel(priv, reg, UMAC_CMD);
3543 spin_unlock(&priv->reg_lock);
3544 }
3545
3546 /* update MDF filter */
3547 i = 0;
3548 /* Broadcast */
3549 bcmgenet_set_mdf_addr(priv, dev->broadcast, &i);
3550 /* my own address.*/
3551 bcmgenet_set_mdf_addr(priv, dev->dev_addr, &i);
3552
3553 /* Unicast */
3554 netdev_for_each_uc_addr(ha, dev)
3555 bcmgenet_set_mdf_addr(priv, ha->addr, &i);
3556
3557 /* Multicast */
3558 netdev_for_each_mc_addr(ha, dev)
3559 bcmgenet_set_mdf_addr(priv, ha->addr, &i);
3560
3561 /* Enable filters */
3562 reg = GENMASK(MAX_MDF_FILTER - 1, MAX_MDF_FILTER - nfilter);
3563 bcmgenet_umac_writel(priv, reg, UMAC_MDF_CTRL);
3564 }
3565
3566 /* Set the hardware MAC address. */
bcmgenet_set_mac_addr(struct net_device * dev,void * p)3567 static int bcmgenet_set_mac_addr(struct net_device *dev, void *p)
3568 {
3569 struct sockaddr *addr = p;
3570
3571 /* Setting the MAC address at the hardware level is not possible
3572 * without disabling the UniMAC RX/TX enable bits.
3573 */
3574 if (netif_running(dev))
3575 return -EBUSY;
3576
3577 eth_hw_addr_set(dev, addr->sa_data);
3578
3579 return 0;
3580 }
3581
bcmgenet_get_stats64(struct net_device * dev,struct rtnl_link_stats64 * stats)3582 static void bcmgenet_get_stats64(struct net_device *dev,
3583 struct rtnl_link_stats64 *stats)
3584 {
3585 struct bcmgenet_priv *priv = netdev_priv(dev);
3586 struct bcmgenet_tx_stats64 *tx_stats;
3587 struct bcmgenet_rx_stats64 *rx_stats;
3588 u64 rx_length_errors, rx_over_errors;
3589 u64 rx_missed, rx_fragmented_errors;
3590 u64 rx_crc_errors, rx_frame_errors;
3591 u64 tx_errors, tx_dropped;
3592 u64 rx_errors, rx_dropped;
3593 u64 tx_bytes, tx_packets;
3594 u64 rx_bytes, rx_packets;
3595 unsigned int start;
3596 unsigned int q;
3597 u64 multicast;
3598
3599 for (q = 0; q <= priv->hw_params->tx_queues; q++) {
3600 tx_stats = &priv->tx_rings[q].stats64;
3601 do {
3602 start = u64_stats_fetch_begin(&tx_stats->syncp);
3603 tx_bytes = u64_stats_read(&tx_stats->bytes);
3604 tx_packets = u64_stats_read(&tx_stats->packets);
3605 tx_errors = u64_stats_read(&tx_stats->errors);
3606 tx_dropped = u64_stats_read(&tx_stats->dropped);
3607 } while (u64_stats_fetch_retry(&tx_stats->syncp, start));
3608
3609 stats->tx_bytes += tx_bytes;
3610 stats->tx_packets += tx_packets;
3611 stats->tx_errors += tx_errors;
3612 stats->tx_dropped += tx_dropped;
3613 }
3614
3615 for (q = 0; q <= priv->hw_params->rx_queues; q++) {
3616 rx_stats = &priv->rx_rings[q].stats64;
3617 do {
3618 start = u64_stats_fetch_begin(&rx_stats->syncp);
3619 rx_bytes = u64_stats_read(&rx_stats->bytes);
3620 rx_packets = u64_stats_read(&rx_stats->packets);
3621 rx_errors = u64_stats_read(&rx_stats->errors);
3622 rx_dropped = u64_stats_read(&rx_stats->dropped);
3623 rx_missed = u64_stats_read(&rx_stats->missed);
3624 rx_length_errors = u64_stats_read(&rx_stats->length_errors);
3625 rx_over_errors = u64_stats_read(&rx_stats->over_errors);
3626 rx_crc_errors = u64_stats_read(&rx_stats->crc_errors);
3627 rx_frame_errors = u64_stats_read(&rx_stats->frame_errors);
3628 rx_fragmented_errors = u64_stats_read(&rx_stats->fragmented_errors);
3629 multicast = u64_stats_read(&rx_stats->multicast);
3630 } while (u64_stats_fetch_retry(&rx_stats->syncp, start));
3631
3632 rx_errors += rx_length_errors;
3633 rx_errors += rx_crc_errors;
3634 rx_errors += rx_frame_errors;
3635 rx_errors += rx_fragmented_errors;
3636
3637 stats->rx_bytes += rx_bytes;
3638 stats->rx_packets += rx_packets;
3639 stats->rx_errors += rx_errors;
3640 stats->rx_dropped += rx_dropped;
3641 stats->rx_missed_errors += rx_missed;
3642 stats->rx_length_errors += rx_length_errors;
3643 stats->rx_over_errors += rx_over_errors;
3644 stats->rx_crc_errors += rx_crc_errors;
3645 stats->rx_frame_errors += rx_frame_errors;
3646 stats->multicast += multicast;
3647 }
3648 }
3649
bcmgenet_change_carrier(struct net_device * dev,bool new_carrier)3650 static int bcmgenet_change_carrier(struct net_device *dev, bool new_carrier)
3651 {
3652 struct bcmgenet_priv *priv = netdev_priv(dev);
3653
3654 if (!dev->phydev || !phy_is_pseudo_fixed_link(dev->phydev) ||
3655 priv->phy_interface != PHY_INTERFACE_MODE_MOCA)
3656 return -EOPNOTSUPP;
3657
3658 if (new_carrier)
3659 netif_carrier_on(dev);
3660 else
3661 netif_carrier_off(dev);
3662
3663 return 0;
3664 }
3665
3666 static const struct net_device_ops bcmgenet_netdev_ops = {
3667 .ndo_open = bcmgenet_open,
3668 .ndo_stop = bcmgenet_close,
3669 .ndo_start_xmit = bcmgenet_xmit,
3670 .ndo_tx_timeout = bcmgenet_timeout,
3671 .ndo_set_rx_mode = bcmgenet_set_rx_mode,
3672 .ndo_set_mac_address = bcmgenet_set_mac_addr,
3673 .ndo_eth_ioctl = phy_do_ioctl_running,
3674 .ndo_set_features = bcmgenet_set_features,
3675 .ndo_get_stats64 = bcmgenet_get_stats64,
3676 .ndo_change_carrier = bcmgenet_change_carrier,
3677 };
3678
3679 /* GENET hardware parameters/characteristics */
3680 static const struct bcmgenet_hw_params bcmgenet_hw_params_v1 = {
3681 .tx_queues = 0,
3682 .tx_bds_per_q = 0,
3683 .rx_queues = 0,
3684 .rx_bds_per_q = 0,
3685 .bp_in_en_shift = 16,
3686 .bp_in_mask = 0xffff,
3687 .hfb_filter_cnt = 16,
3688 .hfb_filter_size = 64,
3689 .qtag_mask = 0x1F,
3690 .hfb_offset = 0x1000,
3691 .hfb_reg_offset = GENET_RBUF_OFF + RBUF_HFB_CTRL_V1,
3692 .rdma_offset = 0x2000,
3693 .tdma_offset = 0x3000,
3694 .words_per_bd = 2,
3695 };
3696
3697 static const struct bcmgenet_hw_params bcmgenet_hw_params_v2 = {
3698 .tx_queues = 4,
3699 .tx_bds_per_q = 32,
3700 .rx_queues = 0,
3701 .rx_bds_per_q = 0,
3702 .bp_in_en_shift = 16,
3703 .bp_in_mask = 0xffff,
3704 .hfb_filter_cnt = 16,
3705 .hfb_filter_size = 64,
3706 .qtag_mask = 0x1F,
3707 .tbuf_offset = 0x0600,
3708 .hfb_offset = 0x1000,
3709 .hfb_reg_offset = 0x2000,
3710 .rdma_offset = 0x3000,
3711 .tdma_offset = 0x4000,
3712 .words_per_bd = 2,
3713 };
3714
3715 static const struct bcmgenet_hw_params bcmgenet_hw_params_v3 = {
3716 .tx_queues = 4,
3717 .tx_bds_per_q = 32,
3718 .rx_queues = 0,
3719 .rx_bds_per_q = 0,
3720 .bp_in_en_shift = 17,
3721 .bp_in_mask = 0x1ffff,
3722 .hfb_filter_cnt = 48,
3723 .hfb_filter_size = 128,
3724 .qtag_mask = 0x3F,
3725 .tbuf_offset = 0x0600,
3726 .hfb_offset = 0x8000,
3727 .hfb_reg_offset = 0xfc00,
3728 .rdma_offset = 0x10000,
3729 .tdma_offset = 0x11000,
3730 .words_per_bd = 2,
3731 };
3732
3733 static const struct bcmgenet_hw_params bcmgenet_hw_params_v4 = {
3734 .tx_queues = 4,
3735 .tx_bds_per_q = 32,
3736 .rx_queues = 0,
3737 .rx_bds_per_q = 0,
3738 .bp_in_en_shift = 17,
3739 .bp_in_mask = 0x1ffff,
3740 .hfb_filter_cnt = 48,
3741 .hfb_filter_size = 128,
3742 .qtag_mask = 0x3F,
3743 .tbuf_offset = 0x0600,
3744 .hfb_offset = 0x8000,
3745 .hfb_reg_offset = 0xfc00,
3746 .rdma_offset = 0x2000,
3747 .tdma_offset = 0x4000,
3748 .words_per_bd = 3,
3749 };
3750
3751 /* Infer hardware parameters from the detected GENET version */
bcmgenet_set_hw_params(struct bcmgenet_priv * priv)3752 static void bcmgenet_set_hw_params(struct bcmgenet_priv *priv)
3753 {
3754 const struct bcmgenet_hw_params *params;
3755 u32 reg;
3756 u8 major;
3757 u16 gphy_rev;
3758
3759 /* default to latest values */
3760 params = &bcmgenet_hw_params_v4;
3761 bcmgenet_dma_regs = bcmgenet_dma_regs_v3plus;
3762 genet_dma_ring_regs = genet_dma_ring_regs_v4;
3763 if (GENET_IS_V3(priv)) {
3764 params = &bcmgenet_hw_params_v3;
3765 bcmgenet_dma_regs = bcmgenet_dma_regs_v3plus;
3766 genet_dma_ring_regs = genet_dma_ring_regs_v123;
3767 } else if (GENET_IS_V2(priv)) {
3768 params = &bcmgenet_hw_params_v2;
3769 bcmgenet_dma_regs = bcmgenet_dma_regs_v2;
3770 genet_dma_ring_regs = genet_dma_ring_regs_v123;
3771 } else if (GENET_IS_V1(priv)) {
3772 params = &bcmgenet_hw_params_v1;
3773 bcmgenet_dma_regs = bcmgenet_dma_regs_v1;
3774 genet_dma_ring_regs = genet_dma_ring_regs_v123;
3775 }
3776 priv->hw_params = params;
3777
3778 /* Read GENET HW version */
3779 reg = bcmgenet_sys_readl(priv, SYS_REV_CTRL);
3780 major = (reg >> 24 & 0x0f);
3781 if (major == 6 || major == 7)
3782 major = 5;
3783 else if (major == 5)
3784 major = 4;
3785 else if (major == 0)
3786 major = 1;
3787 if (major != priv->version) {
3788 dev_err(&priv->pdev->dev,
3789 "GENET version mismatch, got: %d, configured for: %d\n",
3790 major, priv->version);
3791 }
3792
3793 /* Print the GENET core version */
3794 dev_info(&priv->pdev->dev, "GENET " GENET_VER_FMT,
3795 major, (reg >> 16) & 0x0f, reg & 0xffff);
3796
3797 /* Store the integrated PHY revision for the MDIO probing function
3798 * to pass this information to the PHY driver. The PHY driver expects
3799 * to find the PHY major revision in bits 15:8 while the GENET register
3800 * stores that information in bits 7:0, account for that.
3801 *
3802 * On newer chips, starting with PHY revision G0, a new scheme is
3803 * deployed similar to the Starfighter 2 switch with GPHY major
3804 * revision in bits 15:8 and patch level in bits 7:0. Major revision 0
3805 * is reserved as well as special value 0x01ff, we have a small
3806 * heuristic to check for the new GPHY revision and re-arrange things
3807 * so the GPHY driver is happy.
3808 */
3809 gphy_rev = reg & 0xffff;
3810
3811 if (GENET_IS_V5(priv)) {
3812 /* The EPHY revision should come from the MDIO registers of
3813 * the PHY not from GENET.
3814 */
3815 if (gphy_rev != 0) {
3816 pr_warn("GENET is reporting EPHY revision: 0x%04x\n",
3817 gphy_rev);
3818 }
3819 /* This is reserved so should require special treatment */
3820 } else if (gphy_rev == 0 || gphy_rev == 0x01ff) {
3821 pr_warn("Invalid GPHY revision detected: 0x%04x\n", gphy_rev);
3822 return;
3823 /* This is the good old scheme, just GPHY major, no minor nor patch */
3824 } else if ((gphy_rev & 0xf0) != 0) {
3825 priv->gphy_rev = gphy_rev << 8;
3826 /* This is the new scheme, GPHY major rolls over with 0x10 = rev G0 */
3827 } else if ((gphy_rev & 0xff00) != 0) {
3828 priv->gphy_rev = gphy_rev;
3829 }
3830
3831 #ifdef CONFIG_PHYS_ADDR_T_64BIT
3832 if (!bcmgenet_has_40bits(priv))
3833 pr_warn("GENET does not support 40-bits PA\n");
3834 #endif
3835
3836 pr_debug("Configuration for version: %d\n"
3837 "TXq: %1d, TXqBDs: %1d, RXq: %1d, RXqBDs: %1d\n"
3838 "BP << en: %2d, BP msk: 0x%05x\n"
3839 "HFB count: %2d, QTAQ msk: 0x%05x\n"
3840 "TBUF: 0x%04x, HFB: 0x%04x, HFBreg: 0x%04x\n"
3841 "RDMA: 0x%05x, TDMA: 0x%05x\n"
3842 "Words/BD: %d\n",
3843 priv->version,
3844 params->tx_queues, params->tx_bds_per_q,
3845 params->rx_queues, params->rx_bds_per_q,
3846 params->bp_in_en_shift, params->bp_in_mask,
3847 params->hfb_filter_cnt, params->qtag_mask,
3848 params->tbuf_offset, params->hfb_offset,
3849 params->hfb_reg_offset,
3850 params->rdma_offset, params->tdma_offset,
3851 params->words_per_bd);
3852 }
3853
3854 struct bcmgenet_plat_data {
3855 enum bcmgenet_version version;
3856 u32 dma_max_burst_length;
3857 u32 flags;
3858 };
3859
3860 static const struct bcmgenet_plat_data v1_plat_data = {
3861 .version = GENET_V1,
3862 .dma_max_burst_length = DMA_MAX_BURST_LENGTH,
3863 };
3864
3865 static const struct bcmgenet_plat_data v2_plat_data = {
3866 .version = GENET_V2,
3867 .dma_max_burst_length = DMA_MAX_BURST_LENGTH,
3868 .flags = GENET_HAS_EXT,
3869 };
3870
3871 static const struct bcmgenet_plat_data v3_plat_data = {
3872 .version = GENET_V3,
3873 .dma_max_burst_length = DMA_MAX_BURST_LENGTH,
3874 .flags = GENET_HAS_EXT | GENET_HAS_MDIO_INTR |
3875 GENET_HAS_MOCA_LINK_DET,
3876 };
3877
3878 static const struct bcmgenet_plat_data v4_plat_data = {
3879 .version = GENET_V4,
3880 .dma_max_burst_length = DMA_MAX_BURST_LENGTH,
3881 .flags = GENET_HAS_40BITS | GENET_HAS_EXT |
3882 GENET_HAS_MDIO_INTR | GENET_HAS_MOCA_LINK_DET,
3883 };
3884
3885 static const struct bcmgenet_plat_data v5_plat_data = {
3886 .version = GENET_V5,
3887 .dma_max_burst_length = DMA_MAX_BURST_LENGTH,
3888 .flags = GENET_HAS_40BITS | GENET_HAS_EXT |
3889 GENET_HAS_MDIO_INTR | GENET_HAS_MOCA_LINK_DET,
3890 };
3891
3892 static const struct bcmgenet_plat_data bcm2711_plat_data = {
3893 .version = GENET_V5,
3894 .dma_max_burst_length = 0x08,
3895 .flags = GENET_HAS_40BITS | GENET_HAS_EXT |
3896 GENET_HAS_MDIO_INTR | GENET_HAS_MOCA_LINK_DET,
3897 };
3898
3899 static const struct bcmgenet_plat_data bcm7712_plat_data = {
3900 .version = GENET_V5,
3901 .dma_max_burst_length = DMA_MAX_BURST_LENGTH,
3902 .flags = GENET_HAS_40BITS | GENET_HAS_EXT |
3903 GENET_HAS_MDIO_INTR | GENET_HAS_MOCA_LINK_DET |
3904 GENET_HAS_EPHY_16NM,
3905 };
3906
3907 static const struct of_device_id bcmgenet_match[] = {
3908 { .compatible = "brcm,genet-v1", .data = &v1_plat_data },
3909 { .compatible = "brcm,genet-v2", .data = &v2_plat_data },
3910 { .compatible = "brcm,genet-v3", .data = &v3_plat_data },
3911 { .compatible = "brcm,genet-v4", .data = &v4_plat_data },
3912 { .compatible = "brcm,genet-v5", .data = &v5_plat_data },
3913 { .compatible = "brcm,bcm2711-genet-v5", .data = &bcm2711_plat_data },
3914 { .compatible = "brcm,bcm7712-genet-v5", .data = &bcm7712_plat_data },
3915 { },
3916 };
3917 MODULE_DEVICE_TABLE(of, bcmgenet_match);
3918
bcmgenet_probe(struct platform_device * pdev)3919 static int bcmgenet_probe(struct platform_device *pdev)
3920 {
3921 const struct bcmgenet_plat_data *pdata;
3922 struct bcmgenet_priv *priv;
3923 struct net_device *dev;
3924 unsigned int i;
3925 int err = -EIO;
3926
3927 /* Up to GENET_MAX_MQ_CNT + 1 TX queues and RX queues */
3928 dev = alloc_etherdev_mqs(sizeof(*priv), GENET_MAX_MQ_CNT + 1,
3929 GENET_MAX_MQ_CNT + 1);
3930 if (!dev) {
3931 dev_err(&pdev->dev, "can't allocate net device\n");
3932 return -ENOMEM;
3933 }
3934
3935 priv = netdev_priv(dev);
3936 priv->irq0 = platform_get_irq(pdev, 0);
3937 if (priv->irq0 < 0) {
3938 err = priv->irq0;
3939 goto err;
3940 }
3941 priv->irq1 = platform_get_irq(pdev, 1);
3942 if (priv->irq1 < 0) {
3943 err = priv->irq1;
3944 goto err;
3945 }
3946 priv->wol_irq = platform_get_irq_optional(pdev, 2);
3947 if (priv->wol_irq == -EPROBE_DEFER) {
3948 err = priv->wol_irq;
3949 goto err;
3950 }
3951
3952 priv->base = devm_platform_ioremap_resource(pdev, 0);
3953 if (IS_ERR(priv->base)) {
3954 err = PTR_ERR(priv->base);
3955 goto err;
3956 }
3957
3958 spin_lock_init(&priv->reg_lock);
3959 spin_lock_init(&priv->lock);
3960
3961 /* Set default pause parameters */
3962 priv->autoneg_pause = 1;
3963 priv->tx_pause = 1;
3964 priv->rx_pause = 1;
3965
3966 SET_NETDEV_DEV(dev, &pdev->dev);
3967 dev_set_drvdata(&pdev->dev, dev);
3968 dev->watchdog_timeo = 2 * HZ;
3969 dev->ethtool_ops = &bcmgenet_ethtool_ops;
3970 dev->netdev_ops = &bcmgenet_netdev_ops;
3971
3972 priv->msg_enable = netif_msg_init(-1, GENET_MSG_DEFAULT);
3973
3974 /* Set default features */
3975 dev->features |= NETIF_F_SG | NETIF_F_HIGHDMA | NETIF_F_HW_CSUM |
3976 NETIF_F_RXCSUM;
3977 dev->hw_features |= dev->features;
3978 dev->vlan_features |= dev->features;
3979
3980 netdev_sw_irq_coalesce_default_on(dev);
3981
3982 /* Request the WOL interrupt and advertise suspend if available */
3983 priv->wol_irq_disabled = true;
3984 if (priv->wol_irq > 0) {
3985 err = devm_request_irq(&pdev->dev, priv->wol_irq,
3986 bcmgenet_wol_isr, 0, dev->name, priv);
3987 if (!err)
3988 device_set_wakeup_capable(&pdev->dev, 1);
3989 }
3990
3991 /* Set the needed headroom to account for any possible
3992 * features enabling/disabling at runtime
3993 */
3994 dev->needed_headroom += 64;
3995
3996 priv->dev = dev;
3997 priv->pdev = pdev;
3998
3999 pdata = device_get_match_data(&pdev->dev);
4000 if (pdata) {
4001 priv->version = pdata->version;
4002 priv->dma_max_burst_length = pdata->dma_max_burst_length;
4003 priv->flags = pdata->flags;
4004 }
4005
4006 priv->clk = devm_clk_get_optional(&priv->pdev->dev, "enet");
4007 if (IS_ERR(priv->clk)) {
4008 dev_dbg(&priv->pdev->dev, "failed to get enet clock\n");
4009 err = PTR_ERR(priv->clk);
4010 goto err;
4011 }
4012
4013 err = clk_prepare_enable(priv->clk);
4014 if (err)
4015 goto err;
4016
4017 bcmgenet_set_hw_params(priv);
4018
4019 err = -EIO;
4020 if (bcmgenet_has_40bits(priv))
4021 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(40));
4022 if (err)
4023 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
4024 if (err)
4025 goto err_clk_disable;
4026
4027 /* Mii wait queue */
4028 init_waitqueue_head(&priv->wq);
4029 /* Always use RX_BUF_LENGTH (2KB) buffer for all chips */
4030 priv->rx_buf_len = RX_BUF_LENGTH;
4031 INIT_WORK(&priv->bcmgenet_irq_work, bcmgenet_irq_task);
4032
4033 priv->clk_wol = devm_clk_get_optional(&priv->pdev->dev, "enet-wol");
4034 if (IS_ERR(priv->clk_wol)) {
4035 dev_dbg(&priv->pdev->dev, "failed to get enet-wol clock\n");
4036 err = PTR_ERR(priv->clk_wol);
4037 goto err_clk_disable;
4038 }
4039
4040 priv->clk_eee = devm_clk_get_optional(&priv->pdev->dev, "enet-eee");
4041 if (IS_ERR(priv->clk_eee)) {
4042 dev_dbg(&priv->pdev->dev, "failed to get enet-eee clock\n");
4043 err = PTR_ERR(priv->clk_eee);
4044 goto err_clk_disable;
4045 }
4046
4047 /* If this is an internal GPHY, power it on now, before UniMAC is
4048 * brought out of reset as absolutely no UniMAC activity is allowed
4049 */
4050 if (device_get_phy_mode(&pdev->dev) == PHY_INTERFACE_MODE_INTERNAL)
4051 bcmgenet_power_up(priv, GENET_POWER_PASSIVE);
4052
4053 if (device_get_ethdev_address(&pdev->dev, dev))
4054 if (has_acpi_companion(&pdev->dev)) {
4055 u8 addr[ETH_ALEN];
4056
4057 bcmgenet_get_hw_addr(priv, addr);
4058 eth_hw_addr_set(dev, addr);
4059 }
4060
4061 if (!is_valid_ether_addr(dev->dev_addr)) {
4062 dev_warn(&pdev->dev, "using random Ethernet MAC\n");
4063 eth_hw_addr_random(dev);
4064 }
4065
4066 reset_umac(priv);
4067
4068 err = bcmgenet_mii_init(dev);
4069 if (err)
4070 goto err_clk_disable;
4071
4072 /* setup number of real queues + 1 */
4073 netif_set_real_num_tx_queues(priv->dev, priv->hw_params->tx_queues + 1);
4074 netif_set_real_num_rx_queues(priv->dev, priv->hw_params->rx_queues + 1);
4075
4076 /* Set default coalescing parameters */
4077 for (i = 0; i <= priv->hw_params->rx_queues; i++)
4078 priv->rx_rings[i].rx_max_coalesced_frames = 1;
4079
4080 /* Initialize u64 stats seq counter for 32bit machines */
4081 for (i = 0; i <= priv->hw_params->rx_queues; i++)
4082 u64_stats_init(&priv->rx_rings[i].stats64.syncp);
4083 for (i = 0; i <= priv->hw_params->tx_queues; i++)
4084 u64_stats_init(&priv->tx_rings[i].stats64.syncp);
4085
4086 /* libphy will determine the link state */
4087 netif_carrier_off(dev);
4088
4089 /* Turn off the main clock, WOL clock is handled separately */
4090 clk_disable_unprepare(priv->clk);
4091
4092 err = register_netdev(dev);
4093 if (err) {
4094 bcmgenet_mii_exit(dev);
4095 goto err;
4096 }
4097
4098 return err;
4099
4100 err_clk_disable:
4101 clk_disable_unprepare(priv->clk);
4102 err:
4103 free_netdev(dev);
4104 return err;
4105 }
4106
bcmgenet_remove(struct platform_device * pdev)4107 static void bcmgenet_remove(struct platform_device *pdev)
4108 {
4109 struct bcmgenet_priv *priv = dev_to_priv(&pdev->dev);
4110
4111 dev_set_drvdata(&pdev->dev, NULL);
4112 unregister_netdev(priv->dev);
4113 bcmgenet_mii_exit(priv->dev);
4114 free_netdev(priv->dev);
4115 }
4116
bcmgenet_shutdown(struct platform_device * pdev)4117 static void bcmgenet_shutdown(struct platform_device *pdev)
4118 {
4119 bcmgenet_remove(pdev);
4120 }
4121
4122 #ifdef CONFIG_PM_SLEEP
bcmgenet_resume_noirq(struct device * d)4123 static int bcmgenet_resume_noirq(struct device *d)
4124 {
4125 struct net_device *dev = dev_get_drvdata(d);
4126 struct bcmgenet_priv *priv = netdev_priv(dev);
4127 int ret;
4128 u32 reg;
4129
4130 if (!netif_running(dev))
4131 return 0;
4132
4133 /* Turn on the clock */
4134 ret = clk_prepare_enable(priv->clk);
4135 if (ret)
4136 return ret;
4137
4138 if (device_may_wakeup(d) && priv->wolopts) {
4139 /* Account for Wake-on-LAN events and clear those events
4140 * (Some devices need more time between enabling the clocks
4141 * and the interrupt register reflecting the wake event so
4142 * read the register twice)
4143 */
4144 reg = bcmgenet_intrl2_0_readl(priv, INTRL2_CPU_STAT);
4145 reg = bcmgenet_intrl2_0_readl(priv, INTRL2_CPU_STAT);
4146 if (reg & UMAC_IRQ_WAKE_EVENT)
4147 pm_wakeup_event(&priv->pdev->dev, 0);
4148
4149 /* From WOL-enabled suspend, switch to regular clock */
4150 if (!bcmgenet_power_up(priv, GENET_POWER_WOL_MAGIC))
4151 return 0;
4152
4153 /* Failed so fall through to reset MAC */
4154 }
4155
4156 /* If this is an internal GPHY, power it back on now, before UniMAC is
4157 * brought out of reset as absolutely no UniMAC activity is allowed
4158 */
4159 if (priv->internal_phy)
4160 bcmgenet_power_up(priv, GENET_POWER_PASSIVE);
4161
4162 /* take MAC out of reset */
4163 bcmgenet_umac_reset(priv);
4164
4165 return 0;
4166 }
4167
bcmgenet_resume(struct device * d)4168 static int bcmgenet_resume(struct device *d)
4169 {
4170 struct net_device *dev = dev_get_drvdata(d);
4171 struct bcmgenet_priv *priv = netdev_priv(dev);
4172 struct bcmgenet_rxnfc_rule *rule;
4173 int ret;
4174 u32 reg;
4175
4176 if (!netif_running(dev))
4177 return 0;
4178
4179 if (device_may_wakeup(d) && priv->wolopts) {
4180 reg = bcmgenet_umac_readl(priv, UMAC_CMD);
4181 if (reg & CMD_RX_EN) {
4182 /* Successfully exited WoL, just resume data flows */
4183 list_for_each_entry(rule, &priv->rxnfc_list, list)
4184 if (rule->state == BCMGENET_RXNFC_STATE_ENABLED)
4185 bcmgenet_hfb_enable_filter(priv,
4186 rule->fs.location + 1);
4187 bcmgenet_hfb_enable_filter(priv, 0);
4188 bcmgenet_set_rx_mode(dev);
4189 bcmgenet_enable_rx_napi(priv);
4190
4191 /* Reinitialize Tx flows */
4192 bcmgenet_tdma_disable(priv);
4193 bcmgenet_init_tx_queues(priv->dev);
4194 reg = bcmgenet_tdma_readl(priv, DMA_CTRL);
4195 reg |= DMA_EN;
4196 bcmgenet_tdma_writel(priv, reg, DMA_CTRL);
4197 bcmgenet_enable_tx_napi(priv);
4198
4199 bcmgenet_link_intr_enable(priv);
4200 phy_start_machine(dev->phydev);
4201
4202 netif_device_attach(dev);
4203 enable_irq(priv->irq1);
4204 return 0;
4205 }
4206 /* MAC was reset so complete bcmgenet_netif_stop() */
4207 umac_enable_set(priv, CMD_RX_EN | CMD_TX_EN, false);
4208 bcmgenet_rdma_disable(priv);
4209 bcmgenet_intr_disable(priv);
4210 bcmgenet_fini_dma(priv);
4211 enable_irq(priv->irq1);
4212 }
4213
4214 init_umac(priv);
4215
4216 phy_init_hw(dev->phydev);
4217
4218 /* Speed settings must be restored */
4219 genphy_config_aneg(dev->phydev);
4220 bcmgenet_mii_config(priv->dev, false);
4221
4222 /* Restore enabled features */
4223 bcmgenet_set_features(dev, dev->features);
4224
4225 bcmgenet_set_hw_addr(priv, dev->dev_addr);
4226
4227 /* Restore hardware filters */
4228 bcmgenet_hfb_clear(priv);
4229 list_for_each_entry(rule, &priv->rxnfc_list, list)
4230 if (rule->state != BCMGENET_RXNFC_STATE_UNUSED)
4231 bcmgenet_hfb_create_rxnfc_filter(priv, rule);
4232
4233 /* Reinitialize TDMA and RDMA and SW housekeeping */
4234 ret = bcmgenet_init_dma(priv, false);
4235 if (ret) {
4236 netdev_err(dev, "failed to initialize DMA\n");
4237 goto out_clk_disable;
4238 }
4239
4240 if (!device_may_wakeup(d))
4241 phy_resume(dev->phydev);
4242
4243 bcmgenet_netif_start(dev);
4244
4245 netif_device_attach(dev);
4246
4247 return 0;
4248
4249 out_clk_disable:
4250 if (priv->internal_phy)
4251 bcmgenet_power_down(priv, GENET_POWER_PASSIVE);
4252 clk_disable_unprepare(priv->clk);
4253 return ret;
4254 }
4255
bcmgenet_suspend(struct device * d)4256 static int bcmgenet_suspend(struct device *d)
4257 {
4258 struct net_device *dev = dev_get_drvdata(d);
4259 struct bcmgenet_priv *priv = netdev_priv(dev);
4260 struct bcmgenet_rxnfc_rule *rule;
4261 u32 reg, hfb_enable = 0;
4262
4263 if (!netif_running(dev))
4264 return 0;
4265
4266 netif_device_detach(dev);
4267
4268 if (device_may_wakeup(d) && priv->wolopts) {
4269 netif_tx_disable(dev);
4270
4271 /* Suspend non-wake Rx data flows */
4272 if (priv->wolopts & WAKE_FILTER)
4273 list_for_each_entry(rule, &priv->rxnfc_list, list)
4274 if (rule->fs.ring_cookie == RX_CLS_FLOW_WAKE &&
4275 rule->state == BCMGENET_RXNFC_STATE_ENABLED)
4276 hfb_enable |= 1 << rule->fs.location;
4277 reg = bcmgenet_hfb_reg_readl(priv, HFB_CTRL);
4278 if (GENET_IS_V1(priv) || GENET_IS_V2(priv)) {
4279 reg &= ~RBUF_HFB_FILTER_EN_MASK;
4280 reg |= hfb_enable << (RBUF_HFB_FILTER_EN_SHIFT + 1);
4281 } else {
4282 bcmgenet_hfb_reg_writel(priv, hfb_enable << 1,
4283 HFB_FLT_ENABLE_V3PLUS + 4);
4284 }
4285 if (!hfb_enable)
4286 reg &= ~RBUF_HFB_EN;
4287 bcmgenet_hfb_reg_writel(priv, reg, HFB_CTRL);
4288
4289 /* Clear any old filter matches so only new matches wake */
4290 bcmgenet_intrl2_0_writel(priv, 0xFFFFFFFF, INTRL2_CPU_MASK_SET);
4291 bcmgenet_intrl2_0_writel(priv, 0xFFFFFFFF, INTRL2_CPU_CLEAR);
4292
4293 if (-ETIMEDOUT == bcmgenet_tdma_disable(priv))
4294 netdev_warn(priv->dev,
4295 "Timed out while disabling TX DMA\n");
4296
4297 bcmgenet_disable_tx_napi(priv);
4298 bcmgenet_disable_rx_napi(priv);
4299 disable_irq(priv->irq1);
4300 bcmgenet_tx_reclaim_all(dev);
4301 bcmgenet_fini_tx_napi(priv);
4302 } else {
4303 /* Teardown the interface */
4304 bcmgenet_netif_stop(dev, true);
4305 }
4306
4307 return 0;
4308 }
4309
bcmgenet_suspend_noirq(struct device * d)4310 static int bcmgenet_suspend_noirq(struct device *d)
4311 {
4312 struct net_device *dev = dev_get_drvdata(d);
4313 struct bcmgenet_priv *priv = netdev_priv(dev);
4314 int ret = 0;
4315
4316 if (!netif_running(dev))
4317 return 0;
4318
4319 /* Prepare the device for Wake-on-LAN and switch to the slow clock */
4320 if (device_may_wakeup(d) && priv->wolopts)
4321 ret = bcmgenet_power_down(priv, GENET_POWER_WOL_MAGIC);
4322 else if (priv->internal_phy)
4323 ret = bcmgenet_power_down(priv, GENET_POWER_PASSIVE);
4324
4325 /* Let the framework handle resumption and leave the clocks on */
4326 if (ret)
4327 return ret;
4328
4329 /* Turn off the clocks */
4330 clk_disable_unprepare(priv->clk);
4331
4332 return 0;
4333 }
4334 #else
4335 #define bcmgenet_suspend NULL
4336 #define bcmgenet_suspend_noirq NULL
4337 #define bcmgenet_resume NULL
4338 #define bcmgenet_resume_noirq NULL
4339 #endif /* CONFIG_PM_SLEEP */
4340
4341 static const struct dev_pm_ops bcmgenet_pm_ops = {
4342 .suspend = bcmgenet_suspend,
4343 .suspend_noirq = bcmgenet_suspend_noirq,
4344 .resume = bcmgenet_resume,
4345 .resume_noirq = bcmgenet_resume_noirq,
4346 };
4347
4348 static const struct acpi_device_id genet_acpi_match[] = {
4349 { "BCM6E4E", (kernel_ulong_t)&bcm2711_plat_data },
4350 { },
4351 };
4352 MODULE_DEVICE_TABLE(acpi, genet_acpi_match);
4353
4354 static struct platform_driver bcmgenet_driver = {
4355 .probe = bcmgenet_probe,
4356 .remove = bcmgenet_remove,
4357 .shutdown = bcmgenet_shutdown,
4358 .driver = {
4359 .name = "bcmgenet",
4360 .of_match_table = bcmgenet_match,
4361 .pm = &bcmgenet_pm_ops,
4362 .acpi_match_table = genet_acpi_match,
4363 },
4364 };
4365 module_platform_driver(bcmgenet_driver);
4366
4367 MODULE_AUTHOR("Broadcom Corporation");
4368 MODULE_DESCRIPTION("Broadcom GENET Ethernet controller driver");
4369 MODULE_ALIAS("platform:bcmgenet");
4370 MODULE_LICENSE("GPL");
4371 MODULE_SOFTDEP("pre: mdio-bcm-unimac");
4372