xref: /linux/drivers/net/ethernet/marvell/mvneta.c (revision 9410645520e9b820069761f3450ef6661418e279)
1 /*
2  * Driver for Marvell NETA network card for Armada XP and Armada 370 SoCs.
3  *
4  * Copyright (C) 2012 Marvell
5  *
6  * Rami Rosen <rosenr@marvell.com>
7  * Thomas Petazzoni <thomas.petazzoni@free-electrons.com>
8  *
9  * This file is licensed under the terms of the GNU General Public
10  * License version 2. This program is licensed "as is" without any
11  * warranty of any kind, whether express or implied.
12  */
13 
14 #include <linux/clk.h>
15 #include <linux/cpu.h>
16 #include <linux/etherdevice.h>
17 #include <linux/if_vlan.h>
18 #include <linux/inetdevice.h>
19 #include <linux/interrupt.h>
20 #include <linux/io.h>
21 #include <linux/kernel.h>
22 #include <linux/mbus.h>
23 #include <linux/module.h>
24 #include <linux/netdevice.h>
25 #include <linux/of.h>
26 #include <linux/of_address.h>
27 #include <linux/of_irq.h>
28 #include <linux/of_mdio.h>
29 #include <linux/of_net.h>
30 #include <linux/phy/phy.h>
31 #include <linux/phy.h>
32 #include <linux/phylink.h>
33 #include <linux/platform_device.h>
34 #include <linux/skbuff.h>
35 #include <net/hwbm.h>
36 #include "mvneta_bm.h"
37 #include <net/ip.h>
38 #include <net/ipv6.h>
39 #include <net/tso.h>
40 #include <net/page_pool/helpers.h>
41 #include <net/pkt_sched.h>
42 #include <linux/bpf_trace.h>
43 
44 /* Registers */
45 #define MVNETA_RXQ_CONFIG_REG(q)                (0x1400 + ((q) << 2))
46 #define      MVNETA_RXQ_HW_BUF_ALLOC            BIT(0)
47 #define      MVNETA_RXQ_SHORT_POOL_ID_SHIFT	4
48 #define      MVNETA_RXQ_SHORT_POOL_ID_MASK	0x30
49 #define      MVNETA_RXQ_LONG_POOL_ID_SHIFT	6
50 #define      MVNETA_RXQ_LONG_POOL_ID_MASK	0xc0
51 #define      MVNETA_RXQ_PKT_OFFSET_ALL_MASK     (0xf    << 8)
52 #define      MVNETA_RXQ_PKT_OFFSET_MASK(offs)   ((offs) << 8)
53 #define MVNETA_RXQ_THRESHOLD_REG(q)             (0x14c0 + ((q) << 2))
54 #define      MVNETA_RXQ_NON_OCCUPIED(v)         ((v) << 16)
55 #define MVNETA_RXQ_BASE_ADDR_REG(q)             (0x1480 + ((q) << 2))
56 #define MVNETA_RXQ_SIZE_REG(q)                  (0x14a0 + ((q) << 2))
57 #define      MVNETA_RXQ_BUF_SIZE_SHIFT          19
58 #define      MVNETA_RXQ_BUF_SIZE_MASK           (0x1fff << 19)
59 #define MVNETA_RXQ_STATUS_REG(q)                (0x14e0 + ((q) << 2))
60 #define      MVNETA_RXQ_OCCUPIED_ALL_MASK       0x3fff
61 #define MVNETA_RXQ_STATUS_UPDATE_REG(q)         (0x1500 + ((q) << 2))
62 #define      MVNETA_RXQ_ADD_NON_OCCUPIED_SHIFT  16
63 #define      MVNETA_RXQ_ADD_NON_OCCUPIED_MAX    255
64 #define MVNETA_PORT_POOL_BUFFER_SZ_REG(pool)	(0x1700 + ((pool) << 2))
65 #define      MVNETA_PORT_POOL_BUFFER_SZ_SHIFT	3
66 #define      MVNETA_PORT_POOL_BUFFER_SZ_MASK	0xfff8
67 #define MVNETA_PORT_RX_RESET                    0x1cc0
68 #define      MVNETA_PORT_RX_DMA_RESET           BIT(0)
69 #define MVNETA_PHY_ADDR                         0x2000
70 #define      MVNETA_PHY_ADDR_MASK               0x1f
71 #define MVNETA_MBUS_RETRY                       0x2010
72 #define MVNETA_UNIT_INTR_CAUSE                  0x2080
73 #define MVNETA_UNIT_CONTROL                     0x20B0
74 #define      MVNETA_PHY_POLLING_ENABLE          BIT(1)
75 #define MVNETA_WIN_BASE(w)                      (0x2200 + ((w) << 3))
76 #define MVNETA_WIN_SIZE(w)                      (0x2204 + ((w) << 3))
77 #define MVNETA_WIN_REMAP(w)                     (0x2280 + ((w) << 2))
78 #define MVNETA_BASE_ADDR_ENABLE                 0x2290
79 #define      MVNETA_AC5_CNM_DDR_TARGET		0x2
80 #define      MVNETA_AC5_CNM_DDR_ATTR		0xb
81 #define MVNETA_ACCESS_PROTECT_ENABLE            0x2294
82 #define MVNETA_PORT_CONFIG                      0x2400
83 #define      MVNETA_UNI_PROMISC_MODE            BIT(0)
84 #define      MVNETA_DEF_RXQ(q)                  ((q) << 1)
85 #define      MVNETA_DEF_RXQ_ARP(q)              ((q) << 4)
86 #define      MVNETA_TX_UNSET_ERR_SUM            BIT(12)
87 #define      MVNETA_DEF_RXQ_TCP(q)              ((q) << 16)
88 #define      MVNETA_DEF_RXQ_UDP(q)              ((q) << 19)
89 #define      MVNETA_DEF_RXQ_BPDU(q)             ((q) << 22)
90 #define      MVNETA_RX_CSUM_WITH_PSEUDO_HDR     BIT(25)
91 #define      MVNETA_PORT_CONFIG_DEFL_VALUE(q)   (MVNETA_DEF_RXQ(q)       | \
92 						 MVNETA_DEF_RXQ_ARP(q)	 | \
93 						 MVNETA_DEF_RXQ_TCP(q)	 | \
94 						 MVNETA_DEF_RXQ_UDP(q)	 | \
95 						 MVNETA_DEF_RXQ_BPDU(q)	 | \
96 						 MVNETA_TX_UNSET_ERR_SUM | \
97 						 MVNETA_RX_CSUM_WITH_PSEUDO_HDR)
98 #define MVNETA_PORT_CONFIG_EXTEND                0x2404
99 #define MVNETA_MAC_ADDR_LOW                      0x2414
100 #define MVNETA_MAC_ADDR_HIGH                     0x2418
101 #define MVNETA_SDMA_CONFIG                       0x241c
102 #define      MVNETA_SDMA_BRST_SIZE_16            4
103 #define      MVNETA_RX_BRST_SZ_MASK(burst)       ((burst) << 1)
104 #define      MVNETA_RX_NO_DATA_SWAP              BIT(4)
105 #define      MVNETA_TX_NO_DATA_SWAP              BIT(5)
106 #define      MVNETA_DESC_SWAP                    BIT(6)
107 #define      MVNETA_TX_BRST_SZ_MASK(burst)       ((burst) << 22)
108 #define	MVNETA_VLAN_PRIO_TO_RXQ			 0x2440
109 #define      MVNETA_VLAN_PRIO_RXQ_MAP(prio, rxq) ((rxq) << ((prio) * 3))
110 #define MVNETA_PORT_STATUS                       0x2444
111 #define      MVNETA_TX_IN_PRGRS                  BIT(0)
112 #define      MVNETA_TX_FIFO_EMPTY                BIT(8)
113 #define MVNETA_RX_MIN_FRAME_SIZE                 0x247c
114 /* Only exists on Armada XP and Armada 370 */
115 #define MVNETA_SERDES_CFG			 0x24A0
116 #define      MVNETA_SGMII_SERDES_PROTO		 0x0cc7
117 #define      MVNETA_QSGMII_SERDES_PROTO		 0x0667
118 #define      MVNETA_HSGMII_SERDES_PROTO		 0x1107
119 #define MVNETA_TYPE_PRIO                         0x24bc
120 #define      MVNETA_FORCE_UNI                    BIT(21)
121 #define MVNETA_TXQ_CMD_1                         0x24e4
122 #define MVNETA_TXQ_CMD                           0x2448
123 #define      MVNETA_TXQ_DISABLE_SHIFT            8
124 #define      MVNETA_TXQ_ENABLE_MASK              0x000000ff
125 #define MVNETA_RX_DISCARD_FRAME_COUNT		 0x2484
126 #define MVNETA_OVERRUN_FRAME_COUNT		 0x2488
127 #define MVNETA_GMAC_CLOCK_DIVIDER                0x24f4
128 #define      MVNETA_GMAC_1MS_CLOCK_ENABLE        BIT(31)
129 #define MVNETA_ACC_MODE                          0x2500
130 #define MVNETA_BM_ADDRESS                        0x2504
131 #define MVNETA_CPU_MAP(cpu)                      (0x2540 + ((cpu) << 2))
132 #define      MVNETA_CPU_RXQ_ACCESS_ALL_MASK      0x000000ff
133 #define      MVNETA_CPU_TXQ_ACCESS_ALL_MASK      0x0000ff00
134 #define      MVNETA_CPU_RXQ_ACCESS(rxq)		 BIT(rxq)
135 #define      MVNETA_CPU_TXQ_ACCESS(txq)		 BIT(txq + 8)
136 #define MVNETA_RXQ_TIME_COAL_REG(q)              (0x2580 + ((q) << 2))
137 
138 /* Exception Interrupt Port/Queue Cause register
139  *
140  * Their behavior depend of the mapping done using the PCPX2Q
141  * registers. For a given CPU if the bit associated to a queue is not
142  * set, then for the register a read from this CPU will always return
143  * 0 and a write won't do anything
144  */
145 
146 #define MVNETA_INTR_NEW_CAUSE                    0x25a0
147 #define MVNETA_INTR_NEW_MASK                     0x25a4
148 
149 /* bits  0..7  = TXQ SENT, one bit per queue.
150  * bits  8..15 = RXQ OCCUP, one bit per queue.
151  * bits 16..23 = RXQ FREE, one bit per queue.
152  * bit  29 = OLD_REG_SUM, see old reg ?
153  * bit  30 = TX_ERR_SUM, one bit for 4 ports
154  * bit  31 = MISC_SUM,   one bit for 4 ports
155  */
156 #define      MVNETA_TX_INTR_MASK(nr_txqs)        (((1 << nr_txqs) - 1) << 0)
157 #define      MVNETA_TX_INTR_MASK_ALL             (0xff << 0)
158 #define      MVNETA_RX_INTR_MASK(nr_rxqs)        (((1 << nr_rxqs) - 1) << 8)
159 #define      MVNETA_RX_INTR_MASK_ALL             (0xff << 8)
160 #define      MVNETA_MISCINTR_INTR_MASK           BIT(31)
161 
162 #define MVNETA_INTR_OLD_CAUSE                    0x25a8
163 #define MVNETA_INTR_OLD_MASK                     0x25ac
164 
165 /* Data Path Port/Queue Cause Register */
166 #define MVNETA_INTR_MISC_CAUSE                   0x25b0
167 #define MVNETA_INTR_MISC_MASK                    0x25b4
168 
169 #define      MVNETA_CAUSE_PHY_STATUS_CHANGE      BIT(0)
170 #define      MVNETA_CAUSE_LINK_CHANGE            BIT(1)
171 #define      MVNETA_CAUSE_PTP                    BIT(4)
172 
173 #define      MVNETA_CAUSE_INTERNAL_ADDR_ERR      BIT(7)
174 #define      MVNETA_CAUSE_RX_OVERRUN             BIT(8)
175 #define      MVNETA_CAUSE_RX_CRC_ERROR           BIT(9)
176 #define      MVNETA_CAUSE_RX_LARGE_PKT           BIT(10)
177 #define      MVNETA_CAUSE_TX_UNDERUN             BIT(11)
178 #define      MVNETA_CAUSE_PRBS_ERR               BIT(12)
179 #define      MVNETA_CAUSE_PSC_SYNC_CHANGE        BIT(13)
180 #define      MVNETA_CAUSE_SERDES_SYNC_ERR        BIT(14)
181 
182 #define      MVNETA_CAUSE_BMU_ALLOC_ERR_SHIFT    16
183 #define      MVNETA_CAUSE_BMU_ALLOC_ERR_ALL_MASK   (0xF << MVNETA_CAUSE_BMU_ALLOC_ERR_SHIFT)
184 #define      MVNETA_CAUSE_BMU_ALLOC_ERR_MASK(pool) (1 << (MVNETA_CAUSE_BMU_ALLOC_ERR_SHIFT + (pool)))
185 
186 #define      MVNETA_CAUSE_TXQ_ERROR_SHIFT        24
187 #define      MVNETA_CAUSE_TXQ_ERROR_ALL_MASK     (0xFF << MVNETA_CAUSE_TXQ_ERROR_SHIFT)
188 #define      MVNETA_CAUSE_TXQ_ERROR_MASK(q)      (1 << (MVNETA_CAUSE_TXQ_ERROR_SHIFT + (q)))
189 
190 #define MVNETA_INTR_ENABLE                       0x25b8
191 #define      MVNETA_TXQ_INTR_ENABLE_ALL_MASK     0x0000ff00
192 #define      MVNETA_RXQ_INTR_ENABLE_ALL_MASK     0x000000ff
193 
194 #define MVNETA_RXQ_CMD                           0x2680
195 #define      MVNETA_RXQ_DISABLE_SHIFT            8
196 #define      MVNETA_RXQ_ENABLE_MASK              0x000000ff
197 #define MVETH_TXQ_TOKEN_COUNT_REG(q)             (0x2700 + ((q) << 4))
198 #define MVETH_TXQ_TOKEN_CFG_REG(q)               (0x2704 + ((q) << 4))
199 #define MVNETA_GMAC_CTRL_0                       0x2c00
200 #define      MVNETA_GMAC_MAX_RX_SIZE_SHIFT       2
201 #define      MVNETA_GMAC_MAX_RX_SIZE_MASK        0x7ffc
202 #define      MVNETA_GMAC0_PORT_1000BASE_X        BIT(1)
203 #define      MVNETA_GMAC0_PORT_ENABLE            BIT(0)
204 #define MVNETA_GMAC_CTRL_2                       0x2c08
205 #define      MVNETA_GMAC2_INBAND_AN_ENABLE       BIT(0)
206 #define      MVNETA_GMAC2_PCS_ENABLE             BIT(3)
207 #define      MVNETA_GMAC2_PORT_RGMII             BIT(4)
208 #define      MVNETA_GMAC2_PORT_RESET             BIT(6)
209 #define MVNETA_GMAC_STATUS                       0x2c10
210 #define      MVNETA_GMAC_LINK_UP                 BIT(0)
211 #define      MVNETA_GMAC_SPEED_1000              BIT(1)
212 #define      MVNETA_GMAC_SPEED_100               BIT(2)
213 #define      MVNETA_GMAC_FULL_DUPLEX             BIT(3)
214 #define      MVNETA_GMAC_RX_FLOW_CTRL_ENABLE     BIT(4)
215 #define      MVNETA_GMAC_TX_FLOW_CTRL_ENABLE     BIT(5)
216 #define      MVNETA_GMAC_RX_FLOW_CTRL_ACTIVE     BIT(6)
217 #define      MVNETA_GMAC_TX_FLOW_CTRL_ACTIVE     BIT(7)
218 #define      MVNETA_GMAC_AN_COMPLETE             BIT(11)
219 #define      MVNETA_GMAC_SYNC_OK                 BIT(14)
220 #define MVNETA_GMAC_AUTONEG_CONFIG               0x2c0c
221 #define      MVNETA_GMAC_FORCE_LINK_DOWN         BIT(0)
222 #define      MVNETA_GMAC_FORCE_LINK_PASS         BIT(1)
223 #define      MVNETA_GMAC_INBAND_AN_ENABLE        BIT(2)
224 #define      MVNETA_GMAC_AN_BYPASS_ENABLE        BIT(3)
225 #define      MVNETA_GMAC_INBAND_RESTART_AN       BIT(4)
226 #define      MVNETA_GMAC_CONFIG_MII_SPEED        BIT(5)
227 #define      MVNETA_GMAC_CONFIG_GMII_SPEED       BIT(6)
228 #define      MVNETA_GMAC_AN_SPEED_EN             BIT(7)
229 #define      MVNETA_GMAC_CONFIG_FLOW_CTRL        BIT(8)
230 #define      MVNETA_GMAC_ADVERT_SYM_FLOW_CTRL    BIT(9)
231 #define      MVNETA_GMAC_AN_FLOW_CTRL_EN         BIT(11)
232 #define      MVNETA_GMAC_CONFIG_FULL_DUPLEX      BIT(12)
233 #define      MVNETA_GMAC_AN_DUPLEX_EN            BIT(13)
234 #define MVNETA_GMAC_CTRL_4                       0x2c90
235 #define      MVNETA_GMAC4_SHORT_PREAMBLE_ENABLE  BIT(1)
236 #define MVNETA_MIB_COUNTERS_BASE                 0x3000
237 #define      MVNETA_MIB_LATE_COLLISION           0x7c
238 #define MVNETA_DA_FILT_SPEC_MCAST                0x3400
239 #define MVNETA_DA_FILT_OTH_MCAST                 0x3500
240 #define MVNETA_DA_FILT_UCAST_BASE                0x3600
241 #define MVNETA_TXQ_BASE_ADDR_REG(q)              (0x3c00 + ((q) << 2))
242 #define MVNETA_TXQ_SIZE_REG(q)                   (0x3c20 + ((q) << 2))
243 #define      MVNETA_TXQ_SENT_THRESH_ALL_MASK     0x3fff0000
244 #define      MVNETA_TXQ_SENT_THRESH_MASK(coal)   ((coal) << 16)
245 #define MVNETA_TXQ_UPDATE_REG(q)                 (0x3c60 + ((q) << 2))
246 #define      MVNETA_TXQ_DEC_SENT_SHIFT           16
247 #define      MVNETA_TXQ_DEC_SENT_MASK            0xff
248 #define MVNETA_TXQ_STATUS_REG(q)                 (0x3c40 + ((q) << 2))
249 #define      MVNETA_TXQ_SENT_DESC_SHIFT          16
250 #define      MVNETA_TXQ_SENT_DESC_MASK           0x3fff0000
251 #define MVNETA_PORT_TX_RESET                     0x3cf0
252 #define      MVNETA_PORT_TX_DMA_RESET            BIT(0)
253 #define MVNETA_TXQ_CMD1_REG			 0x3e00
254 #define      MVNETA_TXQ_CMD1_BW_LIM_SEL_V1	 BIT(3)
255 #define      MVNETA_TXQ_CMD1_BW_LIM_EN		 BIT(0)
256 #define MVNETA_REFILL_NUM_CLK_REG		 0x3e08
257 #define      MVNETA_REFILL_MAX_NUM_CLK		 0x0000ffff
258 #define MVNETA_TX_MTU                            0x3e0c
259 #define MVNETA_TX_TOKEN_SIZE                     0x3e14
260 #define      MVNETA_TX_TOKEN_SIZE_MAX            0xffffffff
261 #define MVNETA_TXQ_BUCKET_REFILL_REG(q)		 (0x3e20 + ((q) << 2))
262 #define      MVNETA_TXQ_BUCKET_REFILL_PERIOD_MASK	0x3ff00000
263 #define      MVNETA_TXQ_BUCKET_REFILL_PERIOD_SHIFT	20
264 #define      MVNETA_TXQ_BUCKET_REFILL_VALUE_MAX	 0x0007ffff
265 #define MVNETA_TXQ_TOKEN_SIZE_REG(q)             (0x3e40 + ((q) << 2))
266 #define      MVNETA_TXQ_TOKEN_SIZE_MAX           0x7fffffff
267 
268 /* The values of the bucket refill base period and refill period are taken from
269  * the reference manual, and adds up to a base resolution of 10Kbps. This allows
270  * to cover all rate-limit values from 10Kbps up to 5Gbps
271  */
272 
273 /* Base period for the rate limit algorithm */
274 #define MVNETA_TXQ_BUCKET_REFILL_BASE_PERIOD_NS	100
275 
276 /* Number of Base Period to wait between each bucket refill */
277 #define MVNETA_TXQ_BUCKET_REFILL_PERIOD	1000
278 
279 /* The base resolution for rate limiting, in bps. Any max_rate value should be
280  * a multiple of that value.
281  */
282 #define MVNETA_TXQ_RATE_LIMIT_RESOLUTION (NSEC_PER_SEC / \
283 					 (MVNETA_TXQ_BUCKET_REFILL_BASE_PERIOD_NS * \
284 					  MVNETA_TXQ_BUCKET_REFILL_PERIOD))
285 
286 #define MVNETA_LPI_CTRL_0                        0x2cc0
287 #define MVNETA_LPI_CTRL_1                        0x2cc4
288 #define      MVNETA_LPI_REQUEST_ENABLE           BIT(0)
289 #define MVNETA_LPI_CTRL_2                        0x2cc8
290 #define MVNETA_LPI_STATUS                        0x2ccc
291 
292 #define MVNETA_CAUSE_TXQ_SENT_DESC_ALL_MASK	 0xff
293 
294 /* Descriptor ring Macros */
295 #define MVNETA_QUEUE_NEXT_DESC(q, index)	\
296 	(((index) < (q)->last_desc) ? ((index) + 1) : 0)
297 
298 /* Various constants */
299 
300 /* Coalescing */
301 #define MVNETA_TXDONE_COAL_PKTS		0	/* interrupt per packet */
302 #define MVNETA_RX_COAL_PKTS		32
303 #define MVNETA_RX_COAL_USEC		100
304 
305 /* The two bytes Marvell header. Either contains a special value used
306  * by Marvell switches when a specific hardware mode is enabled (not
307  * supported by this driver) or is filled automatically by zeroes on
308  * the RX side. Those two bytes being at the front of the Ethernet
309  * header, they allow to have the IP header aligned on a 4 bytes
310  * boundary automatically: the hardware skips those two bytes on its
311  * own.
312  */
313 #define MVNETA_MH_SIZE			2
314 
315 #define MVNETA_VLAN_TAG_LEN             4
316 
317 #define MVNETA_TX_CSUM_DEF_SIZE		1600
318 #define MVNETA_TX_CSUM_MAX_SIZE		9800
319 #define MVNETA_ACC_MODE_EXT1		1
320 #define MVNETA_ACC_MODE_EXT2		2
321 
322 #define MVNETA_MAX_DECODE_WIN		6
323 
324 /* Timeout constants */
325 #define MVNETA_TX_DISABLE_TIMEOUT_MSEC	1000
326 #define MVNETA_RX_DISABLE_TIMEOUT_MSEC	1000
327 #define MVNETA_TX_FIFO_EMPTY_TIMEOUT	10000
328 
329 #define MVNETA_TX_MTU_MAX		0x3ffff
330 
331 /* The RSS lookup table actually has 256 entries but we do not use
332  * them yet
333  */
334 #define MVNETA_RSS_LU_TABLE_SIZE	1
335 
336 /* Max number of Rx descriptors */
337 #define MVNETA_MAX_RXD 512
338 
339 /* Max number of Tx descriptors */
340 #define MVNETA_MAX_TXD 1024
341 
342 /* Max number of allowed TCP segments for software TSO */
343 #define MVNETA_MAX_TSO_SEGS 100
344 
345 #define MVNETA_MAX_SKB_DESCS (MVNETA_MAX_TSO_SEGS * 2 + MAX_SKB_FRAGS)
346 
347 /* The size of a TSO header page */
348 #define MVNETA_TSO_PAGE_SIZE (2 * PAGE_SIZE)
349 
350 /* Number of TSO headers per page. This should be a power of 2 */
351 #define MVNETA_TSO_PER_PAGE (MVNETA_TSO_PAGE_SIZE / TSO_HEADER_SIZE)
352 
353 /* Maximum number of TSO header pages */
354 #define MVNETA_MAX_TSO_PAGES (MVNETA_MAX_TXD / MVNETA_TSO_PER_PAGE)
355 
356 /* descriptor aligned size */
357 #define MVNETA_DESC_ALIGNED_SIZE	32
358 
359 /* Number of bytes to be taken into account by HW when putting incoming data
360  * to the buffers. It is needed in case NET_SKB_PAD exceeds maximum packet
361  * offset supported in MVNETA_RXQ_CONFIG_REG(q) registers.
362  */
363 #define MVNETA_RX_PKT_OFFSET_CORRECTION		64
364 
365 #define MVNETA_RX_PKT_SIZE(mtu) \
366 	ALIGN((mtu) + MVNETA_MH_SIZE + MVNETA_VLAN_TAG_LEN + \
367 	      ETH_HLEN + ETH_FCS_LEN,			     \
368 	      cache_line_size())
369 
370 /* Driver assumes that the last 3 bits are 0 */
371 #define MVNETA_SKB_HEADROOM	ALIGN(max(NET_SKB_PAD, XDP_PACKET_HEADROOM), 8)
372 #define MVNETA_SKB_PAD	(SKB_DATA_ALIGN(sizeof(struct skb_shared_info) + \
373 			 MVNETA_SKB_HEADROOM))
374 #define MVNETA_MAX_RX_BUF_SIZE	(PAGE_SIZE - MVNETA_SKB_PAD)
375 
376 #define MVNETA_RX_GET_BM_POOL_ID(rxd) \
377 	(((rxd)->status & MVNETA_RXD_BM_POOL_MASK) >> MVNETA_RXD_BM_POOL_SHIFT)
378 
379 enum {
380 	ETHTOOL_STAT_EEE_WAKEUP,
381 	ETHTOOL_STAT_SKB_ALLOC_ERR,
382 	ETHTOOL_STAT_REFILL_ERR,
383 	ETHTOOL_XDP_REDIRECT,
384 	ETHTOOL_XDP_PASS,
385 	ETHTOOL_XDP_DROP,
386 	ETHTOOL_XDP_TX,
387 	ETHTOOL_XDP_TX_ERR,
388 	ETHTOOL_XDP_XMIT,
389 	ETHTOOL_XDP_XMIT_ERR,
390 	ETHTOOL_MAX_STATS,
391 };
392 
393 struct mvneta_statistic {
394 	unsigned short offset;
395 	unsigned short type;
396 	const char name[ETH_GSTRING_LEN];
397 };
398 
399 #define T_REG_32	32
400 #define T_REG_64	64
401 #define T_SW		1
402 
403 #define MVNETA_XDP_PASS		0
404 #define MVNETA_XDP_DROPPED	BIT(0)
405 #define MVNETA_XDP_TX		BIT(1)
406 #define MVNETA_XDP_REDIR	BIT(2)
407 
408 static const struct mvneta_statistic mvneta_statistics[] = {
409 	{ 0x3000, T_REG_64, "good_octets_received", },
410 	{ 0x3010, T_REG_32, "good_frames_received", },
411 	{ 0x3008, T_REG_32, "bad_octets_received", },
412 	{ 0x3014, T_REG_32, "bad_frames_received", },
413 	{ 0x3018, T_REG_32, "broadcast_frames_received", },
414 	{ 0x301c, T_REG_32, "multicast_frames_received", },
415 	{ 0x3050, T_REG_32, "unrec_mac_control_received", },
416 	{ 0x3058, T_REG_32, "good_fc_received", },
417 	{ 0x305c, T_REG_32, "bad_fc_received", },
418 	{ 0x3060, T_REG_32, "undersize_received", },
419 	{ 0x3064, T_REG_32, "fragments_received", },
420 	{ 0x3068, T_REG_32, "oversize_received", },
421 	{ 0x306c, T_REG_32, "jabber_received", },
422 	{ 0x3070, T_REG_32, "mac_receive_error", },
423 	{ 0x3074, T_REG_32, "bad_crc_event", },
424 	{ 0x3078, T_REG_32, "collision", },
425 	{ 0x307c, T_REG_32, "late_collision", },
426 	{ 0x2484, T_REG_32, "rx_discard", },
427 	{ 0x2488, T_REG_32, "rx_overrun", },
428 	{ 0x3020, T_REG_32, "frames_64_octets", },
429 	{ 0x3024, T_REG_32, "frames_65_to_127_octets", },
430 	{ 0x3028, T_REG_32, "frames_128_to_255_octets", },
431 	{ 0x302c, T_REG_32, "frames_256_to_511_octets", },
432 	{ 0x3030, T_REG_32, "frames_512_to_1023_octets", },
433 	{ 0x3034, T_REG_32, "frames_1024_to_max_octets", },
434 	{ 0x3038, T_REG_64, "good_octets_sent", },
435 	{ 0x3040, T_REG_32, "good_frames_sent", },
436 	{ 0x3044, T_REG_32, "excessive_collision", },
437 	{ 0x3048, T_REG_32, "multicast_frames_sent", },
438 	{ 0x304c, T_REG_32, "broadcast_frames_sent", },
439 	{ 0x3054, T_REG_32, "fc_sent", },
440 	{ 0x300c, T_REG_32, "internal_mac_transmit_err", },
441 	{ ETHTOOL_STAT_EEE_WAKEUP, T_SW, "eee_wakeup_errors", },
442 	{ ETHTOOL_STAT_SKB_ALLOC_ERR, T_SW, "skb_alloc_errors", },
443 	{ ETHTOOL_STAT_REFILL_ERR, T_SW, "refill_errors", },
444 	{ ETHTOOL_XDP_REDIRECT, T_SW, "rx_xdp_redirect", },
445 	{ ETHTOOL_XDP_PASS, T_SW, "rx_xdp_pass", },
446 	{ ETHTOOL_XDP_DROP, T_SW, "rx_xdp_drop", },
447 	{ ETHTOOL_XDP_TX, T_SW, "rx_xdp_tx", },
448 	{ ETHTOOL_XDP_TX_ERR, T_SW, "rx_xdp_tx_errors", },
449 	{ ETHTOOL_XDP_XMIT, T_SW, "tx_xdp_xmit", },
450 	{ ETHTOOL_XDP_XMIT_ERR, T_SW, "tx_xdp_xmit_errors", },
451 };
452 
453 struct mvneta_stats {
454 	u64	rx_packets;
455 	u64	rx_bytes;
456 	u64	tx_packets;
457 	u64	tx_bytes;
458 	/* xdp */
459 	u64	xdp_redirect;
460 	u64	xdp_pass;
461 	u64	xdp_drop;
462 	u64	xdp_xmit;
463 	u64	xdp_xmit_err;
464 	u64	xdp_tx;
465 	u64	xdp_tx_err;
466 };
467 
468 struct mvneta_ethtool_stats {
469 	struct mvneta_stats ps;
470 	u64	skb_alloc_error;
471 	u64	refill_error;
472 };
473 
474 struct mvneta_pcpu_stats {
475 	struct u64_stats_sync syncp;
476 
477 	struct mvneta_ethtool_stats es;
478 	u64	rx_dropped;
479 	u64	rx_errors;
480 };
481 
482 struct mvneta_pcpu_port {
483 	/* Pointer to the shared port */
484 	struct mvneta_port	*pp;
485 
486 	/* Pointer to the CPU-local NAPI struct */
487 	struct napi_struct	napi;
488 
489 	/* Cause of the previous interrupt */
490 	u32			cause_rx_tx;
491 };
492 
493 enum {
494 	__MVNETA_DOWN,
495 };
496 
497 struct mvneta_port {
498 	u8 id;
499 	struct mvneta_pcpu_port __percpu	*ports;
500 	struct mvneta_pcpu_stats __percpu	*stats;
501 
502 	unsigned long state;
503 
504 	int pkt_size;
505 	void __iomem *base;
506 	struct mvneta_rx_queue *rxqs;
507 	struct mvneta_tx_queue *txqs;
508 	struct net_device *dev;
509 	struct hlist_node node_online;
510 	struct hlist_node node_dead;
511 	int rxq_def;
512 	/* Protect the access to the percpu interrupt registers,
513 	 * ensuring that the configuration remains coherent.
514 	 */
515 	spinlock_t lock;
516 	bool is_stopped;
517 
518 	u32 cause_rx_tx;
519 	struct napi_struct napi;
520 
521 	struct bpf_prog *xdp_prog;
522 
523 	/* Core clock */
524 	struct clk *clk;
525 	/* AXI clock */
526 	struct clk *clk_bus;
527 	u8 mcast_count[256];
528 	u16 tx_ring_size;
529 	u16 rx_ring_size;
530 
531 	phy_interface_t phy_interface;
532 	struct device_node *dn;
533 	unsigned int tx_csum_limit;
534 	struct phylink *phylink;
535 	struct phylink_config phylink_config;
536 	struct phylink_pcs phylink_pcs;
537 	struct phy *comphy;
538 
539 	struct mvneta_bm *bm_priv;
540 	struct mvneta_bm_pool *pool_long;
541 	struct mvneta_bm_pool *pool_short;
542 	int bm_win_id;
543 
544 	bool eee_enabled;
545 	bool eee_active;
546 	bool tx_lpi_enabled;
547 
548 	u64 ethtool_stats[ARRAY_SIZE(mvneta_statistics)];
549 
550 	u32 indir[MVNETA_RSS_LU_TABLE_SIZE];
551 
552 	/* Flags for special SoC configurations */
553 	bool neta_armada3700;
554 	bool neta_ac5;
555 	u16 rx_offset_correction;
556 	const struct mbus_dram_target_info *dram_target_info;
557 };
558 
559 /* The mvneta_tx_desc and mvneta_rx_desc structures describe the
560  * layout of the transmit and reception DMA descriptors, and their
561  * layout is therefore defined by the hardware design
562  */
563 
564 #define MVNETA_TX_L3_OFF_SHIFT	0
565 #define MVNETA_TX_IP_HLEN_SHIFT	8
566 #define MVNETA_TX_L4_UDP	BIT(16)
567 #define MVNETA_TX_L3_IP6	BIT(17)
568 #define MVNETA_TXD_IP_CSUM	BIT(18)
569 #define MVNETA_TXD_Z_PAD	BIT(19)
570 #define MVNETA_TXD_L_DESC	BIT(20)
571 #define MVNETA_TXD_F_DESC	BIT(21)
572 #define MVNETA_TXD_FLZ_DESC	(MVNETA_TXD_Z_PAD  | \
573 				 MVNETA_TXD_L_DESC | \
574 				 MVNETA_TXD_F_DESC)
575 #define MVNETA_TX_L4_CSUM_FULL	BIT(30)
576 #define MVNETA_TX_L4_CSUM_NOT	BIT(31)
577 
578 #define MVNETA_RXD_ERR_CRC		0x0
579 #define MVNETA_RXD_BM_POOL_SHIFT	13
580 #define MVNETA_RXD_BM_POOL_MASK		(BIT(13) | BIT(14))
581 #define MVNETA_RXD_ERR_SUMMARY		BIT(16)
582 #define MVNETA_RXD_ERR_OVERRUN		BIT(17)
583 #define MVNETA_RXD_ERR_LEN		BIT(18)
584 #define MVNETA_RXD_ERR_RESOURCE		(BIT(17) | BIT(18))
585 #define MVNETA_RXD_ERR_CODE_MASK	(BIT(17) | BIT(18))
586 #define MVNETA_RXD_L3_IP4		BIT(25)
587 #define MVNETA_RXD_LAST_DESC		BIT(26)
588 #define MVNETA_RXD_FIRST_DESC		BIT(27)
589 #define MVNETA_RXD_FIRST_LAST_DESC	(MVNETA_RXD_FIRST_DESC | \
590 					 MVNETA_RXD_LAST_DESC)
591 #define MVNETA_RXD_L4_CSUM_OK		BIT(30)
592 
593 #if defined(__LITTLE_ENDIAN)
594 struct mvneta_tx_desc {
595 	u32  command;		/* Options used by HW for packet transmitting.*/
596 	u16  reserved1;		/* csum_l4 (for future use)		*/
597 	u16  data_size;		/* Data size of transmitted packet in bytes */
598 	u32  buf_phys_addr;	/* Physical addr of transmitted buffer	*/
599 	u32  reserved2;		/* hw_cmd - (for future use, PMT)	*/
600 	u32  reserved3[4];	/* Reserved - (for future use)		*/
601 };
602 
603 struct mvneta_rx_desc {
604 	u32  status;		/* Info about received packet		*/
605 	u16  reserved1;		/* pnc_info - (for future use, PnC)	*/
606 	u16  data_size;		/* Size of received packet in bytes	*/
607 
608 	u32  buf_phys_addr;	/* Physical address of the buffer	*/
609 	u32  reserved2;		/* pnc_flow_id  (for future use, PnC)	*/
610 
611 	u32  buf_cookie;	/* cookie for access to RX buffer in rx path */
612 	u16  reserved3;		/* prefetch_cmd, for future use		*/
613 	u16  reserved4;		/* csum_l4 - (for future use, PnC)	*/
614 
615 	u32  reserved5;		/* pnc_extra PnC (for future use, PnC)	*/
616 	u32  reserved6;		/* hw_cmd (for future use, PnC and HWF)	*/
617 };
618 #else
619 struct mvneta_tx_desc {
620 	u16  data_size;		/* Data size of transmitted packet in bytes */
621 	u16  reserved1;		/* csum_l4 (for future use)		*/
622 	u32  command;		/* Options used by HW for packet transmitting.*/
623 	u32  reserved2;		/* hw_cmd - (for future use, PMT)	*/
624 	u32  buf_phys_addr;	/* Physical addr of transmitted buffer	*/
625 	u32  reserved3[4];	/* Reserved - (for future use)		*/
626 };
627 
628 struct mvneta_rx_desc {
629 	u16  data_size;		/* Size of received packet in bytes	*/
630 	u16  reserved1;		/* pnc_info - (for future use, PnC)	*/
631 	u32  status;		/* Info about received packet		*/
632 
633 	u32  reserved2;		/* pnc_flow_id  (for future use, PnC)	*/
634 	u32  buf_phys_addr;	/* Physical address of the buffer	*/
635 
636 	u16  reserved4;		/* csum_l4 - (for future use, PnC)	*/
637 	u16  reserved3;		/* prefetch_cmd, for future use		*/
638 	u32  buf_cookie;	/* cookie for access to RX buffer in rx path */
639 
640 	u32  reserved5;		/* pnc_extra PnC (for future use, PnC)	*/
641 	u32  reserved6;		/* hw_cmd (for future use, PnC and HWF)	*/
642 };
643 #endif
644 
645 enum mvneta_tx_buf_type {
646 	MVNETA_TYPE_TSO,
647 	MVNETA_TYPE_SKB,
648 	MVNETA_TYPE_XDP_TX,
649 	MVNETA_TYPE_XDP_NDO,
650 };
651 
652 struct mvneta_tx_buf {
653 	enum mvneta_tx_buf_type type;
654 	union {
655 		struct xdp_frame *xdpf;
656 		struct sk_buff *skb;
657 	};
658 };
659 
660 struct mvneta_tx_queue {
661 	/* Number of this TX queue, in the range 0-7 */
662 	u8 id;
663 
664 	/* Number of TX DMA descriptors in the descriptor ring */
665 	int size;
666 
667 	/* Number of currently used TX DMA descriptor in the
668 	 * descriptor ring
669 	 */
670 	int count;
671 	int pending;
672 	int tx_stop_threshold;
673 	int tx_wake_threshold;
674 
675 	/* Array of transmitted buffers */
676 	struct mvneta_tx_buf *buf;
677 
678 	/* Index of last TX DMA descriptor that was inserted */
679 	int txq_put_index;
680 
681 	/* Index of the TX DMA descriptor to be cleaned up */
682 	int txq_get_index;
683 
684 	u32 done_pkts_coal;
685 
686 	/* Virtual address of the TX DMA descriptors array */
687 	struct mvneta_tx_desc *descs;
688 
689 	/* DMA address of the TX DMA descriptors array */
690 	dma_addr_t descs_phys;
691 
692 	/* Index of the last TX DMA descriptor */
693 	int last_desc;
694 
695 	/* Index of the next TX DMA descriptor to process */
696 	int next_desc_to_proc;
697 
698 	/* DMA buffers for TSO headers */
699 	char *tso_hdrs[MVNETA_MAX_TSO_PAGES];
700 
701 	/* DMA address of TSO headers */
702 	dma_addr_t tso_hdrs_phys[MVNETA_MAX_TSO_PAGES];
703 
704 	/* Affinity mask for CPUs*/
705 	cpumask_t affinity_mask;
706 };
707 
708 struct mvneta_rx_queue {
709 	/* rx queue number, in the range 0-7 */
710 	u8 id;
711 
712 	/* num of rx descriptors in the rx descriptor ring */
713 	int size;
714 
715 	u32 pkts_coal;
716 	u32 time_coal;
717 
718 	/* page_pool */
719 	struct page_pool *page_pool;
720 	struct xdp_rxq_info xdp_rxq;
721 
722 	/* Virtual address of the RX buffer */
723 	void  **buf_virt_addr;
724 
725 	/* Virtual address of the RX DMA descriptors array */
726 	struct mvneta_rx_desc *descs;
727 
728 	/* DMA address of the RX DMA descriptors array */
729 	dma_addr_t descs_phys;
730 
731 	/* Index of the last RX DMA descriptor */
732 	int last_desc;
733 
734 	/* Index of the next RX DMA descriptor to process */
735 	int next_desc_to_proc;
736 
737 	/* Index of first RX DMA descriptor to refill */
738 	int first_to_refill;
739 	u32 refill_num;
740 };
741 
742 static enum cpuhp_state online_hpstate;
743 /* The hardware supports eight (8) rx queues, but we are only allowing
744  * the first one to be used. Therefore, let's just allocate one queue.
745  */
746 static int rxq_number = 8;
747 static int txq_number = 8;
748 
749 static int rxq_def;
750 
751 static int rx_copybreak __read_mostly = 256;
752 
753 /* HW BM need that each port be identify by a unique ID */
754 static int global_port_id;
755 
756 #define MVNETA_DRIVER_NAME "mvneta"
757 #define MVNETA_DRIVER_VERSION "1.0"
758 
759 /* Utility/helper methods */
760 
761 /* Write helper method */
mvreg_write(struct mvneta_port * pp,u32 offset,u32 data)762 static void mvreg_write(struct mvneta_port *pp, u32 offset, u32 data)
763 {
764 	writel(data, pp->base + offset);
765 }
766 
767 /* Read helper method */
mvreg_read(struct mvneta_port * pp,u32 offset)768 static u32 mvreg_read(struct mvneta_port *pp, u32 offset)
769 {
770 	return readl(pp->base + offset);
771 }
772 
773 /* Increment txq get counter */
mvneta_txq_inc_get(struct mvneta_tx_queue * txq)774 static void mvneta_txq_inc_get(struct mvneta_tx_queue *txq)
775 {
776 	txq->txq_get_index++;
777 	if (txq->txq_get_index == txq->size)
778 		txq->txq_get_index = 0;
779 }
780 
781 /* Increment txq put counter */
mvneta_txq_inc_put(struct mvneta_tx_queue * txq)782 static void mvneta_txq_inc_put(struct mvneta_tx_queue *txq)
783 {
784 	txq->txq_put_index++;
785 	if (txq->txq_put_index == txq->size)
786 		txq->txq_put_index = 0;
787 }
788 
789 
790 /* Clear all MIB counters */
mvneta_mib_counters_clear(struct mvneta_port * pp)791 static void mvneta_mib_counters_clear(struct mvneta_port *pp)
792 {
793 	int i;
794 
795 	/* Perform dummy reads from MIB counters */
796 	for (i = 0; i < MVNETA_MIB_LATE_COLLISION; i += 4)
797 		mvreg_read(pp, (MVNETA_MIB_COUNTERS_BASE + i));
798 	mvreg_read(pp, MVNETA_RX_DISCARD_FRAME_COUNT);
799 	mvreg_read(pp, MVNETA_OVERRUN_FRAME_COUNT);
800 }
801 
802 /* Get System Network Statistics */
803 static void
mvneta_get_stats64(struct net_device * dev,struct rtnl_link_stats64 * stats)804 mvneta_get_stats64(struct net_device *dev,
805 		   struct rtnl_link_stats64 *stats)
806 {
807 	struct mvneta_port *pp = netdev_priv(dev);
808 	unsigned int start;
809 	int cpu;
810 
811 	for_each_possible_cpu(cpu) {
812 		struct mvneta_pcpu_stats *cpu_stats;
813 		u64 rx_packets;
814 		u64 rx_bytes;
815 		u64 rx_dropped;
816 		u64 rx_errors;
817 		u64 tx_packets;
818 		u64 tx_bytes;
819 
820 		cpu_stats = per_cpu_ptr(pp->stats, cpu);
821 		do {
822 			start = u64_stats_fetch_begin(&cpu_stats->syncp);
823 			rx_packets = cpu_stats->es.ps.rx_packets;
824 			rx_bytes   = cpu_stats->es.ps.rx_bytes;
825 			rx_dropped = cpu_stats->rx_dropped;
826 			rx_errors  = cpu_stats->rx_errors;
827 			tx_packets = cpu_stats->es.ps.tx_packets;
828 			tx_bytes   = cpu_stats->es.ps.tx_bytes;
829 		} while (u64_stats_fetch_retry(&cpu_stats->syncp, start));
830 
831 		stats->rx_packets += rx_packets;
832 		stats->rx_bytes   += rx_bytes;
833 		stats->rx_dropped += rx_dropped;
834 		stats->rx_errors  += rx_errors;
835 		stats->tx_packets += tx_packets;
836 		stats->tx_bytes   += tx_bytes;
837 	}
838 
839 	stats->tx_dropped	= dev->stats.tx_dropped;
840 }
841 
842 /* Rx descriptors helper methods */
843 
844 /* Checks whether the RX descriptor having this status is both the first
845  * and the last descriptor for the RX packet. Each RX packet is currently
846  * received through a single RX descriptor, so not having each RX
847  * descriptor with its first and last bits set is an error
848  */
mvneta_rxq_desc_is_first_last(u32 status)849 static int mvneta_rxq_desc_is_first_last(u32 status)
850 {
851 	return (status & MVNETA_RXD_FIRST_LAST_DESC) ==
852 		MVNETA_RXD_FIRST_LAST_DESC;
853 }
854 
855 /* Add number of descriptors ready to receive new packets */
mvneta_rxq_non_occup_desc_add(struct mvneta_port * pp,struct mvneta_rx_queue * rxq,int ndescs)856 static void mvneta_rxq_non_occup_desc_add(struct mvneta_port *pp,
857 					  struct mvneta_rx_queue *rxq,
858 					  int ndescs)
859 {
860 	/* Only MVNETA_RXQ_ADD_NON_OCCUPIED_MAX (255) descriptors can
861 	 * be added at once
862 	 */
863 	while (ndescs > MVNETA_RXQ_ADD_NON_OCCUPIED_MAX) {
864 		mvreg_write(pp, MVNETA_RXQ_STATUS_UPDATE_REG(rxq->id),
865 			    (MVNETA_RXQ_ADD_NON_OCCUPIED_MAX <<
866 			     MVNETA_RXQ_ADD_NON_OCCUPIED_SHIFT));
867 		ndescs -= MVNETA_RXQ_ADD_NON_OCCUPIED_MAX;
868 	}
869 
870 	mvreg_write(pp, MVNETA_RXQ_STATUS_UPDATE_REG(rxq->id),
871 		    (ndescs << MVNETA_RXQ_ADD_NON_OCCUPIED_SHIFT));
872 }
873 
874 /* Get number of RX descriptors occupied by received packets */
mvneta_rxq_busy_desc_num_get(struct mvneta_port * pp,struct mvneta_rx_queue * rxq)875 static int mvneta_rxq_busy_desc_num_get(struct mvneta_port *pp,
876 					struct mvneta_rx_queue *rxq)
877 {
878 	u32 val;
879 
880 	val = mvreg_read(pp, MVNETA_RXQ_STATUS_REG(rxq->id));
881 	return val & MVNETA_RXQ_OCCUPIED_ALL_MASK;
882 }
883 
884 /* Update num of rx desc called upon return from rx path or
885  * from mvneta_rxq_drop_pkts().
886  */
mvneta_rxq_desc_num_update(struct mvneta_port * pp,struct mvneta_rx_queue * rxq,int rx_done,int rx_filled)887 static void mvneta_rxq_desc_num_update(struct mvneta_port *pp,
888 				       struct mvneta_rx_queue *rxq,
889 				       int rx_done, int rx_filled)
890 {
891 	u32 val;
892 
893 	if ((rx_done <= 0xff) && (rx_filled <= 0xff)) {
894 		val = rx_done |
895 		  (rx_filled << MVNETA_RXQ_ADD_NON_OCCUPIED_SHIFT);
896 		mvreg_write(pp, MVNETA_RXQ_STATUS_UPDATE_REG(rxq->id), val);
897 		return;
898 	}
899 
900 	/* Only 255 descriptors can be added at once */
901 	while ((rx_done > 0) || (rx_filled > 0)) {
902 		if (rx_done <= 0xff) {
903 			val = rx_done;
904 			rx_done = 0;
905 		} else {
906 			val = 0xff;
907 			rx_done -= 0xff;
908 		}
909 		if (rx_filled <= 0xff) {
910 			val |= rx_filled << MVNETA_RXQ_ADD_NON_OCCUPIED_SHIFT;
911 			rx_filled = 0;
912 		} else {
913 			val |= 0xff << MVNETA_RXQ_ADD_NON_OCCUPIED_SHIFT;
914 			rx_filled -= 0xff;
915 		}
916 		mvreg_write(pp, MVNETA_RXQ_STATUS_UPDATE_REG(rxq->id), val);
917 	}
918 }
919 
920 /* Get pointer to next RX descriptor to be processed by SW */
921 static struct mvneta_rx_desc *
mvneta_rxq_next_desc_get(struct mvneta_rx_queue * rxq)922 mvneta_rxq_next_desc_get(struct mvneta_rx_queue *rxq)
923 {
924 	int rx_desc = rxq->next_desc_to_proc;
925 
926 	rxq->next_desc_to_proc = MVNETA_QUEUE_NEXT_DESC(rxq, rx_desc);
927 	prefetch(rxq->descs + rxq->next_desc_to_proc);
928 	return rxq->descs + rx_desc;
929 }
930 
931 /* Change maximum receive size of the port. */
mvneta_max_rx_size_set(struct mvneta_port * pp,int max_rx_size)932 static void mvneta_max_rx_size_set(struct mvneta_port *pp, int max_rx_size)
933 {
934 	u32 val;
935 
936 	val =  mvreg_read(pp, MVNETA_GMAC_CTRL_0);
937 	val &= ~MVNETA_GMAC_MAX_RX_SIZE_MASK;
938 	val |= ((max_rx_size - MVNETA_MH_SIZE) / 2) <<
939 		MVNETA_GMAC_MAX_RX_SIZE_SHIFT;
940 	mvreg_write(pp, MVNETA_GMAC_CTRL_0, val);
941 }
942 
943 
944 /* Set rx queue offset */
mvneta_rxq_offset_set(struct mvneta_port * pp,struct mvneta_rx_queue * rxq,int offset)945 static void mvneta_rxq_offset_set(struct mvneta_port *pp,
946 				  struct mvneta_rx_queue *rxq,
947 				  int offset)
948 {
949 	u32 val;
950 
951 	val = mvreg_read(pp, MVNETA_RXQ_CONFIG_REG(rxq->id));
952 	val &= ~MVNETA_RXQ_PKT_OFFSET_ALL_MASK;
953 
954 	/* Offset is in */
955 	val |= MVNETA_RXQ_PKT_OFFSET_MASK(offset >> 3);
956 	mvreg_write(pp, MVNETA_RXQ_CONFIG_REG(rxq->id), val);
957 }
958 
959 
960 /* Tx descriptors helper methods */
961 
962 /* Update HW with number of TX descriptors to be sent */
mvneta_txq_pend_desc_add(struct mvneta_port * pp,struct mvneta_tx_queue * txq,int pend_desc)963 static void mvneta_txq_pend_desc_add(struct mvneta_port *pp,
964 				     struct mvneta_tx_queue *txq,
965 				     int pend_desc)
966 {
967 	u32 val;
968 
969 	pend_desc += txq->pending;
970 
971 	/* Only 255 Tx descriptors can be added at once */
972 	do {
973 		val = min(pend_desc, 255);
974 		mvreg_write(pp, MVNETA_TXQ_UPDATE_REG(txq->id), val);
975 		pend_desc -= val;
976 	} while (pend_desc > 0);
977 	txq->pending = 0;
978 }
979 
980 /* Get pointer to next TX descriptor to be processed (send) by HW */
981 static struct mvneta_tx_desc *
mvneta_txq_next_desc_get(struct mvneta_tx_queue * txq)982 mvneta_txq_next_desc_get(struct mvneta_tx_queue *txq)
983 {
984 	int tx_desc = txq->next_desc_to_proc;
985 
986 	txq->next_desc_to_proc = MVNETA_QUEUE_NEXT_DESC(txq, tx_desc);
987 	return txq->descs + tx_desc;
988 }
989 
990 /* Release the last allocated TX descriptor. Useful to handle DMA
991  * mapping failures in the TX path.
992  */
mvneta_txq_desc_put(struct mvneta_tx_queue * txq)993 static void mvneta_txq_desc_put(struct mvneta_tx_queue *txq)
994 {
995 	if (txq->next_desc_to_proc == 0)
996 		txq->next_desc_to_proc = txq->last_desc - 1;
997 	else
998 		txq->next_desc_to_proc--;
999 }
1000 
1001 /* Set rxq buf size */
mvneta_rxq_buf_size_set(struct mvneta_port * pp,struct mvneta_rx_queue * rxq,int buf_size)1002 static void mvneta_rxq_buf_size_set(struct mvneta_port *pp,
1003 				    struct mvneta_rx_queue *rxq,
1004 				    int buf_size)
1005 {
1006 	u32 val;
1007 
1008 	val = mvreg_read(pp, MVNETA_RXQ_SIZE_REG(rxq->id));
1009 
1010 	val &= ~MVNETA_RXQ_BUF_SIZE_MASK;
1011 	val |= ((buf_size >> 3) << MVNETA_RXQ_BUF_SIZE_SHIFT);
1012 
1013 	mvreg_write(pp, MVNETA_RXQ_SIZE_REG(rxq->id), val);
1014 }
1015 
1016 /* Disable buffer management (BM) */
mvneta_rxq_bm_disable(struct mvneta_port * pp,struct mvneta_rx_queue * rxq)1017 static void mvneta_rxq_bm_disable(struct mvneta_port *pp,
1018 				  struct mvneta_rx_queue *rxq)
1019 {
1020 	u32 val;
1021 
1022 	val = mvreg_read(pp, MVNETA_RXQ_CONFIG_REG(rxq->id));
1023 	val &= ~MVNETA_RXQ_HW_BUF_ALLOC;
1024 	mvreg_write(pp, MVNETA_RXQ_CONFIG_REG(rxq->id), val);
1025 }
1026 
1027 /* Enable buffer management (BM) */
mvneta_rxq_bm_enable(struct mvneta_port * pp,struct mvneta_rx_queue * rxq)1028 static void mvneta_rxq_bm_enable(struct mvneta_port *pp,
1029 				 struct mvneta_rx_queue *rxq)
1030 {
1031 	u32 val;
1032 
1033 	val = mvreg_read(pp, MVNETA_RXQ_CONFIG_REG(rxq->id));
1034 	val |= MVNETA_RXQ_HW_BUF_ALLOC;
1035 	mvreg_write(pp, MVNETA_RXQ_CONFIG_REG(rxq->id), val);
1036 }
1037 
1038 /* Notify HW about port's assignment of pool for bigger packets */
mvneta_rxq_long_pool_set(struct mvneta_port * pp,struct mvneta_rx_queue * rxq)1039 static void mvneta_rxq_long_pool_set(struct mvneta_port *pp,
1040 				     struct mvneta_rx_queue *rxq)
1041 {
1042 	u32 val;
1043 
1044 	val = mvreg_read(pp, MVNETA_RXQ_CONFIG_REG(rxq->id));
1045 	val &= ~MVNETA_RXQ_LONG_POOL_ID_MASK;
1046 	val |= (pp->pool_long->id << MVNETA_RXQ_LONG_POOL_ID_SHIFT);
1047 
1048 	mvreg_write(pp, MVNETA_RXQ_CONFIG_REG(rxq->id), val);
1049 }
1050 
1051 /* Notify HW about port's assignment of pool for smaller packets */
mvneta_rxq_short_pool_set(struct mvneta_port * pp,struct mvneta_rx_queue * rxq)1052 static void mvneta_rxq_short_pool_set(struct mvneta_port *pp,
1053 				      struct mvneta_rx_queue *rxq)
1054 {
1055 	u32 val;
1056 
1057 	val = mvreg_read(pp, MVNETA_RXQ_CONFIG_REG(rxq->id));
1058 	val &= ~MVNETA_RXQ_SHORT_POOL_ID_MASK;
1059 	val |= (pp->pool_short->id << MVNETA_RXQ_SHORT_POOL_ID_SHIFT);
1060 
1061 	mvreg_write(pp, MVNETA_RXQ_CONFIG_REG(rxq->id), val);
1062 }
1063 
1064 /* Set port's receive buffer size for assigned BM pool */
mvneta_bm_pool_bufsize_set(struct mvneta_port * pp,int buf_size,u8 pool_id)1065 static inline void mvneta_bm_pool_bufsize_set(struct mvneta_port *pp,
1066 					      int buf_size,
1067 					      u8 pool_id)
1068 {
1069 	u32 val;
1070 
1071 	if (!IS_ALIGNED(buf_size, 8)) {
1072 		dev_warn(pp->dev->dev.parent,
1073 			 "illegal buf_size value %d, round to %d\n",
1074 			 buf_size, ALIGN(buf_size, 8));
1075 		buf_size = ALIGN(buf_size, 8);
1076 	}
1077 
1078 	val = mvreg_read(pp, MVNETA_PORT_POOL_BUFFER_SZ_REG(pool_id));
1079 	val |= buf_size & MVNETA_PORT_POOL_BUFFER_SZ_MASK;
1080 	mvreg_write(pp, MVNETA_PORT_POOL_BUFFER_SZ_REG(pool_id), val);
1081 }
1082 
1083 /* Configure MBUS window in order to enable access BM internal SRAM */
mvneta_mbus_io_win_set(struct mvneta_port * pp,u32 base,u32 wsize,u8 target,u8 attr)1084 static int mvneta_mbus_io_win_set(struct mvneta_port *pp, u32 base, u32 wsize,
1085 				  u8 target, u8 attr)
1086 {
1087 	u32 win_enable, win_protect;
1088 	int i;
1089 
1090 	win_enable = mvreg_read(pp, MVNETA_BASE_ADDR_ENABLE);
1091 
1092 	if (pp->bm_win_id < 0) {
1093 		/* Find first not occupied window */
1094 		for (i = 0; i < MVNETA_MAX_DECODE_WIN; i++) {
1095 			if (win_enable & (1 << i)) {
1096 				pp->bm_win_id = i;
1097 				break;
1098 			}
1099 		}
1100 		if (i == MVNETA_MAX_DECODE_WIN)
1101 			return -ENOMEM;
1102 	} else {
1103 		i = pp->bm_win_id;
1104 	}
1105 
1106 	mvreg_write(pp, MVNETA_WIN_BASE(i), 0);
1107 	mvreg_write(pp, MVNETA_WIN_SIZE(i), 0);
1108 
1109 	if (i < 4)
1110 		mvreg_write(pp, MVNETA_WIN_REMAP(i), 0);
1111 
1112 	mvreg_write(pp, MVNETA_WIN_BASE(i), (base & 0xffff0000) |
1113 		    (attr << 8) | target);
1114 
1115 	mvreg_write(pp, MVNETA_WIN_SIZE(i), (wsize - 1) & 0xffff0000);
1116 
1117 	win_protect = mvreg_read(pp, MVNETA_ACCESS_PROTECT_ENABLE);
1118 	win_protect |= 3 << (2 * i);
1119 	mvreg_write(pp, MVNETA_ACCESS_PROTECT_ENABLE, win_protect);
1120 
1121 	win_enable &= ~(1 << i);
1122 	mvreg_write(pp, MVNETA_BASE_ADDR_ENABLE, win_enable);
1123 
1124 	return 0;
1125 }
1126 
mvneta_bm_port_mbus_init(struct mvneta_port * pp)1127 static int mvneta_bm_port_mbus_init(struct mvneta_port *pp)
1128 {
1129 	u32 wsize;
1130 	u8 target, attr;
1131 	int err;
1132 
1133 	/* Get BM window information */
1134 	err = mvebu_mbus_get_io_win_info(pp->bm_priv->bppi_phys_addr, &wsize,
1135 					 &target, &attr);
1136 	if (err < 0)
1137 		return err;
1138 
1139 	pp->bm_win_id = -1;
1140 
1141 	/* Open NETA -> BM window */
1142 	err = mvneta_mbus_io_win_set(pp, pp->bm_priv->bppi_phys_addr, wsize,
1143 				     target, attr);
1144 	if (err < 0) {
1145 		netdev_info(pp->dev, "fail to configure mbus window to BM\n");
1146 		return err;
1147 	}
1148 	return 0;
1149 }
1150 
1151 /* Assign and initialize pools for port. In case of fail
1152  * buffer manager will remain disabled for current port.
1153  */
mvneta_bm_port_init(struct platform_device * pdev,struct mvneta_port * pp)1154 static int mvneta_bm_port_init(struct platform_device *pdev,
1155 			       struct mvneta_port *pp)
1156 {
1157 	struct device_node *dn = pdev->dev.of_node;
1158 	u32 long_pool_id, short_pool_id;
1159 
1160 	if (!pp->neta_armada3700) {
1161 		int ret;
1162 
1163 		ret = mvneta_bm_port_mbus_init(pp);
1164 		if (ret)
1165 			return ret;
1166 	}
1167 
1168 	if (of_property_read_u32(dn, "bm,pool-long", &long_pool_id)) {
1169 		netdev_info(pp->dev, "missing long pool id\n");
1170 		return -EINVAL;
1171 	}
1172 
1173 	/* Create port's long pool depending on mtu */
1174 	pp->pool_long = mvneta_bm_pool_use(pp->bm_priv, long_pool_id,
1175 					   MVNETA_BM_LONG, pp->id,
1176 					   MVNETA_RX_PKT_SIZE(pp->dev->mtu));
1177 	if (!pp->pool_long) {
1178 		netdev_info(pp->dev, "fail to obtain long pool for port\n");
1179 		return -ENOMEM;
1180 	}
1181 
1182 	pp->pool_long->port_map |= 1 << pp->id;
1183 
1184 	mvneta_bm_pool_bufsize_set(pp, pp->pool_long->buf_size,
1185 				   pp->pool_long->id);
1186 
1187 	/* If short pool id is not defined, assume using single pool */
1188 	if (of_property_read_u32(dn, "bm,pool-short", &short_pool_id))
1189 		short_pool_id = long_pool_id;
1190 
1191 	/* Create port's short pool */
1192 	pp->pool_short = mvneta_bm_pool_use(pp->bm_priv, short_pool_id,
1193 					    MVNETA_BM_SHORT, pp->id,
1194 					    MVNETA_BM_SHORT_PKT_SIZE);
1195 	if (!pp->pool_short) {
1196 		netdev_info(pp->dev, "fail to obtain short pool for port\n");
1197 		mvneta_bm_pool_destroy(pp->bm_priv, pp->pool_long, 1 << pp->id);
1198 		return -ENOMEM;
1199 	}
1200 
1201 	if (short_pool_id != long_pool_id) {
1202 		pp->pool_short->port_map |= 1 << pp->id;
1203 		mvneta_bm_pool_bufsize_set(pp, pp->pool_short->buf_size,
1204 					   pp->pool_short->id);
1205 	}
1206 
1207 	return 0;
1208 }
1209 
1210 /* Update settings of a pool for bigger packets */
mvneta_bm_update_mtu(struct mvneta_port * pp,int mtu)1211 static void mvneta_bm_update_mtu(struct mvneta_port *pp, int mtu)
1212 {
1213 	struct mvneta_bm_pool *bm_pool = pp->pool_long;
1214 	struct hwbm_pool *hwbm_pool = &bm_pool->hwbm_pool;
1215 	int num;
1216 
1217 	/* Release all buffers from long pool */
1218 	mvneta_bm_bufs_free(pp->bm_priv, bm_pool, 1 << pp->id);
1219 	if (hwbm_pool->buf_num) {
1220 		WARN(1, "cannot free all buffers in pool %d\n",
1221 		     bm_pool->id);
1222 		goto bm_mtu_err;
1223 	}
1224 
1225 	bm_pool->pkt_size = MVNETA_RX_PKT_SIZE(mtu);
1226 	bm_pool->buf_size = MVNETA_RX_BUF_SIZE(bm_pool->pkt_size);
1227 	hwbm_pool->frag_size = SKB_DATA_ALIGN(sizeof(struct skb_shared_info)) +
1228 			SKB_DATA_ALIGN(MVNETA_RX_BUF_SIZE(bm_pool->pkt_size));
1229 
1230 	/* Fill entire long pool */
1231 	num = hwbm_pool_add(hwbm_pool, hwbm_pool->size);
1232 	if (num != hwbm_pool->size) {
1233 		WARN(1, "pool %d: %d of %d allocated\n",
1234 		     bm_pool->id, num, hwbm_pool->size);
1235 		goto bm_mtu_err;
1236 	}
1237 	mvneta_bm_pool_bufsize_set(pp, bm_pool->buf_size, bm_pool->id);
1238 
1239 	return;
1240 
1241 bm_mtu_err:
1242 	mvneta_bm_pool_destroy(pp->bm_priv, pp->pool_long, 1 << pp->id);
1243 	mvneta_bm_pool_destroy(pp->bm_priv, pp->pool_short, 1 << pp->id);
1244 
1245 	pp->bm_priv = NULL;
1246 	pp->rx_offset_correction = MVNETA_SKB_HEADROOM;
1247 	mvreg_write(pp, MVNETA_ACC_MODE, MVNETA_ACC_MODE_EXT1);
1248 	netdev_info(pp->dev, "fail to update MTU, fall back to software BM\n");
1249 }
1250 
1251 /* Start the Ethernet port RX and TX activity */
mvneta_port_up(struct mvneta_port * pp)1252 static void mvneta_port_up(struct mvneta_port *pp)
1253 {
1254 	int queue;
1255 	u32 q_map;
1256 
1257 	/* Enable all initialized TXs. */
1258 	q_map = 0;
1259 	for (queue = 0; queue < txq_number; queue++) {
1260 		struct mvneta_tx_queue *txq = &pp->txqs[queue];
1261 		if (txq->descs)
1262 			q_map |= (1 << queue);
1263 	}
1264 	mvreg_write(pp, MVNETA_TXQ_CMD, q_map);
1265 
1266 	q_map = 0;
1267 	/* Enable all initialized RXQs. */
1268 	for (queue = 0; queue < rxq_number; queue++) {
1269 		struct mvneta_rx_queue *rxq = &pp->rxqs[queue];
1270 
1271 		if (rxq->descs)
1272 			q_map |= (1 << queue);
1273 	}
1274 	mvreg_write(pp, MVNETA_RXQ_CMD, q_map);
1275 }
1276 
1277 /* Stop the Ethernet port activity */
mvneta_port_down(struct mvneta_port * pp)1278 static void mvneta_port_down(struct mvneta_port *pp)
1279 {
1280 	u32 val;
1281 	int count;
1282 
1283 	/* Stop Rx port activity. Check port Rx activity. */
1284 	val = mvreg_read(pp, MVNETA_RXQ_CMD) & MVNETA_RXQ_ENABLE_MASK;
1285 
1286 	/* Issue stop command for active channels only */
1287 	if (val != 0)
1288 		mvreg_write(pp, MVNETA_RXQ_CMD,
1289 			    val << MVNETA_RXQ_DISABLE_SHIFT);
1290 
1291 	/* Wait for all Rx activity to terminate. */
1292 	count = 0;
1293 	do {
1294 		if (count++ >= MVNETA_RX_DISABLE_TIMEOUT_MSEC) {
1295 			netdev_warn(pp->dev,
1296 				    "TIMEOUT for RX stopped ! rx_queue_cmd: 0x%08x\n",
1297 				    val);
1298 			break;
1299 		}
1300 		mdelay(1);
1301 
1302 		val = mvreg_read(pp, MVNETA_RXQ_CMD);
1303 	} while (val & MVNETA_RXQ_ENABLE_MASK);
1304 
1305 	/* Stop Tx port activity. Check port Tx activity. Issue stop
1306 	 * command for active channels only
1307 	 */
1308 	val = (mvreg_read(pp, MVNETA_TXQ_CMD)) & MVNETA_TXQ_ENABLE_MASK;
1309 
1310 	if (val != 0)
1311 		mvreg_write(pp, MVNETA_TXQ_CMD,
1312 			    (val << MVNETA_TXQ_DISABLE_SHIFT));
1313 
1314 	/* Wait for all Tx activity to terminate. */
1315 	count = 0;
1316 	do {
1317 		if (count++ >= MVNETA_TX_DISABLE_TIMEOUT_MSEC) {
1318 			netdev_warn(pp->dev,
1319 				    "TIMEOUT for TX stopped status=0x%08x\n",
1320 				    val);
1321 			break;
1322 		}
1323 		mdelay(1);
1324 
1325 		/* Check TX Command reg that all Txqs are stopped */
1326 		val = mvreg_read(pp, MVNETA_TXQ_CMD);
1327 
1328 	} while (val & MVNETA_TXQ_ENABLE_MASK);
1329 
1330 	/* Double check to verify that TX FIFO is empty */
1331 	count = 0;
1332 	do {
1333 		if (count++ >= MVNETA_TX_FIFO_EMPTY_TIMEOUT) {
1334 			netdev_warn(pp->dev,
1335 				    "TX FIFO empty timeout status=0x%08x\n",
1336 				    val);
1337 			break;
1338 		}
1339 		mdelay(1);
1340 
1341 		val = mvreg_read(pp, MVNETA_PORT_STATUS);
1342 	} while (!(val & MVNETA_TX_FIFO_EMPTY) &&
1343 		 (val & MVNETA_TX_IN_PRGRS));
1344 
1345 	udelay(200);
1346 }
1347 
1348 /* Enable the port by setting the port enable bit of the MAC control register */
mvneta_port_enable(struct mvneta_port * pp)1349 static void mvneta_port_enable(struct mvneta_port *pp)
1350 {
1351 	u32 val;
1352 
1353 	/* Enable port */
1354 	val = mvreg_read(pp, MVNETA_GMAC_CTRL_0);
1355 	val |= MVNETA_GMAC0_PORT_ENABLE;
1356 	mvreg_write(pp, MVNETA_GMAC_CTRL_0, val);
1357 }
1358 
1359 /* Disable the port and wait for about 200 usec before retuning */
mvneta_port_disable(struct mvneta_port * pp)1360 static void mvneta_port_disable(struct mvneta_port *pp)
1361 {
1362 	u32 val;
1363 
1364 	/* Reset the Enable bit in the Serial Control Register */
1365 	val = mvreg_read(pp, MVNETA_GMAC_CTRL_0);
1366 	val &= ~MVNETA_GMAC0_PORT_ENABLE;
1367 	mvreg_write(pp, MVNETA_GMAC_CTRL_0, val);
1368 
1369 	udelay(200);
1370 }
1371 
1372 /* Multicast tables methods */
1373 
1374 /* Set all entries in Unicast MAC Table; queue==-1 means reject all */
mvneta_set_ucast_table(struct mvneta_port * pp,int queue)1375 static void mvneta_set_ucast_table(struct mvneta_port *pp, int queue)
1376 {
1377 	int offset;
1378 	u32 val;
1379 
1380 	if (queue == -1) {
1381 		val = 0;
1382 	} else {
1383 		val = 0x1 | (queue << 1);
1384 		val |= (val << 24) | (val << 16) | (val << 8);
1385 	}
1386 
1387 	for (offset = 0; offset <= 0xc; offset += 4)
1388 		mvreg_write(pp, MVNETA_DA_FILT_UCAST_BASE + offset, val);
1389 }
1390 
1391 /* Set all entries in Special Multicast MAC Table; queue==-1 means reject all */
mvneta_set_special_mcast_table(struct mvneta_port * pp,int queue)1392 static void mvneta_set_special_mcast_table(struct mvneta_port *pp, int queue)
1393 {
1394 	int offset;
1395 	u32 val;
1396 
1397 	if (queue == -1) {
1398 		val = 0;
1399 	} else {
1400 		val = 0x1 | (queue << 1);
1401 		val |= (val << 24) | (val << 16) | (val << 8);
1402 	}
1403 
1404 	for (offset = 0; offset <= 0xfc; offset += 4)
1405 		mvreg_write(pp, MVNETA_DA_FILT_SPEC_MCAST + offset, val);
1406 
1407 }
1408 
1409 /* Set all entries in Other Multicast MAC Table. queue==-1 means reject all */
mvneta_set_other_mcast_table(struct mvneta_port * pp,int queue)1410 static void mvneta_set_other_mcast_table(struct mvneta_port *pp, int queue)
1411 {
1412 	int offset;
1413 	u32 val;
1414 
1415 	if (queue == -1) {
1416 		memset(pp->mcast_count, 0, sizeof(pp->mcast_count));
1417 		val = 0;
1418 	} else {
1419 		memset(pp->mcast_count, 1, sizeof(pp->mcast_count));
1420 		val = 0x1 | (queue << 1);
1421 		val |= (val << 24) | (val << 16) | (val << 8);
1422 	}
1423 
1424 	for (offset = 0; offset <= 0xfc; offset += 4)
1425 		mvreg_write(pp, MVNETA_DA_FILT_OTH_MCAST + offset, val);
1426 }
1427 
mvneta_percpu_unmask_interrupt(void * arg)1428 static void mvneta_percpu_unmask_interrupt(void *arg)
1429 {
1430 	struct mvneta_port *pp = arg;
1431 
1432 	/* All the queue are unmasked, but actually only the ones
1433 	 * mapped to this CPU will be unmasked
1434 	 */
1435 	mvreg_write(pp, MVNETA_INTR_NEW_MASK,
1436 		    MVNETA_RX_INTR_MASK_ALL |
1437 		    MVNETA_TX_INTR_MASK_ALL |
1438 		    MVNETA_MISCINTR_INTR_MASK);
1439 }
1440 
mvneta_percpu_mask_interrupt(void * arg)1441 static void mvneta_percpu_mask_interrupt(void *arg)
1442 {
1443 	struct mvneta_port *pp = arg;
1444 
1445 	/* All the queue are masked, but actually only the ones
1446 	 * mapped to this CPU will be masked
1447 	 */
1448 	mvreg_write(pp, MVNETA_INTR_NEW_MASK, 0);
1449 	mvreg_write(pp, MVNETA_INTR_OLD_MASK, 0);
1450 	mvreg_write(pp, MVNETA_INTR_MISC_MASK, 0);
1451 }
1452 
mvneta_percpu_clear_intr_cause(void * arg)1453 static void mvneta_percpu_clear_intr_cause(void *arg)
1454 {
1455 	struct mvneta_port *pp = arg;
1456 
1457 	/* All the queue are cleared, but actually only the ones
1458 	 * mapped to this CPU will be cleared
1459 	 */
1460 	mvreg_write(pp, MVNETA_INTR_NEW_CAUSE, 0);
1461 	mvreg_write(pp, MVNETA_INTR_MISC_CAUSE, 0);
1462 	mvreg_write(pp, MVNETA_INTR_OLD_CAUSE, 0);
1463 }
1464 
1465 /* This method sets defaults to the NETA port:
1466  *	Clears interrupt Cause and Mask registers.
1467  *	Clears all MAC tables.
1468  *	Sets defaults to all registers.
1469  *	Resets RX and TX descriptor rings.
1470  *	Resets PHY.
1471  * This method can be called after mvneta_port_down() to return the port
1472  *	settings to defaults.
1473  */
mvneta_defaults_set(struct mvneta_port * pp)1474 static void mvneta_defaults_set(struct mvneta_port *pp)
1475 {
1476 	int cpu;
1477 	int queue;
1478 	u32 val;
1479 	int max_cpu = num_present_cpus();
1480 
1481 	/* Clear all Cause registers */
1482 	on_each_cpu(mvneta_percpu_clear_intr_cause, pp, true);
1483 
1484 	/* Mask all interrupts */
1485 	on_each_cpu(mvneta_percpu_mask_interrupt, pp, true);
1486 	mvreg_write(pp, MVNETA_INTR_ENABLE, 0);
1487 
1488 	/* Enable MBUS Retry bit16 */
1489 	mvreg_write(pp, MVNETA_MBUS_RETRY, 0x20);
1490 
1491 	/* Set CPU queue access map. CPUs are assigned to the RX and
1492 	 * TX queues modulo their number. If there is only one TX
1493 	 * queue then it is assigned to the CPU associated to the
1494 	 * default RX queue.
1495 	 */
1496 	for_each_present_cpu(cpu) {
1497 		int rxq_map = 0, txq_map = 0;
1498 		int rxq, txq;
1499 		if (!pp->neta_armada3700) {
1500 			for (rxq = 0; rxq < rxq_number; rxq++)
1501 				if ((rxq % max_cpu) == cpu)
1502 					rxq_map |= MVNETA_CPU_RXQ_ACCESS(rxq);
1503 
1504 			for (txq = 0; txq < txq_number; txq++)
1505 				if ((txq % max_cpu) == cpu)
1506 					txq_map |= MVNETA_CPU_TXQ_ACCESS(txq);
1507 
1508 			/* With only one TX queue we configure a special case
1509 			 * which will allow to get all the irq on a single
1510 			 * CPU
1511 			 */
1512 			if (txq_number == 1)
1513 				txq_map = (cpu == pp->rxq_def) ?
1514 					MVNETA_CPU_TXQ_ACCESS(0) : 0;
1515 
1516 		} else {
1517 			txq_map = MVNETA_CPU_TXQ_ACCESS_ALL_MASK;
1518 			rxq_map = MVNETA_CPU_RXQ_ACCESS_ALL_MASK;
1519 		}
1520 
1521 		mvreg_write(pp, MVNETA_CPU_MAP(cpu), rxq_map | txq_map);
1522 	}
1523 
1524 	/* Reset RX and TX DMAs */
1525 	mvreg_write(pp, MVNETA_PORT_RX_RESET, MVNETA_PORT_RX_DMA_RESET);
1526 	mvreg_write(pp, MVNETA_PORT_TX_RESET, MVNETA_PORT_TX_DMA_RESET);
1527 
1528 	/* Disable Legacy WRR, Disable EJP, Release from reset */
1529 	mvreg_write(pp, MVNETA_TXQ_CMD_1, 0);
1530 	for (queue = 0; queue < txq_number; queue++) {
1531 		mvreg_write(pp, MVETH_TXQ_TOKEN_COUNT_REG(queue), 0);
1532 		mvreg_write(pp, MVETH_TXQ_TOKEN_CFG_REG(queue), 0);
1533 	}
1534 
1535 	mvreg_write(pp, MVNETA_PORT_TX_RESET, 0);
1536 	mvreg_write(pp, MVNETA_PORT_RX_RESET, 0);
1537 
1538 	/* Set Port Acceleration Mode */
1539 	if (pp->bm_priv)
1540 		/* HW buffer management + legacy parser */
1541 		val = MVNETA_ACC_MODE_EXT2;
1542 	else
1543 		/* SW buffer management + legacy parser */
1544 		val = MVNETA_ACC_MODE_EXT1;
1545 	mvreg_write(pp, MVNETA_ACC_MODE, val);
1546 
1547 	if (pp->bm_priv)
1548 		mvreg_write(pp, MVNETA_BM_ADDRESS, pp->bm_priv->bppi_phys_addr);
1549 
1550 	/* Update val of portCfg register accordingly with all RxQueue types */
1551 	val = MVNETA_PORT_CONFIG_DEFL_VALUE(pp->rxq_def);
1552 	mvreg_write(pp, MVNETA_PORT_CONFIG, val);
1553 
1554 	val = 0;
1555 	mvreg_write(pp, MVNETA_PORT_CONFIG_EXTEND, val);
1556 	mvreg_write(pp, MVNETA_RX_MIN_FRAME_SIZE, 64);
1557 
1558 	/* Build PORT_SDMA_CONFIG_REG */
1559 	val = 0;
1560 
1561 	/* Default burst size */
1562 	val |= MVNETA_TX_BRST_SZ_MASK(MVNETA_SDMA_BRST_SIZE_16);
1563 	val |= MVNETA_RX_BRST_SZ_MASK(MVNETA_SDMA_BRST_SIZE_16);
1564 	val |= MVNETA_RX_NO_DATA_SWAP | MVNETA_TX_NO_DATA_SWAP;
1565 
1566 #if defined(__BIG_ENDIAN)
1567 	val |= MVNETA_DESC_SWAP;
1568 #endif
1569 
1570 	/* Assign port SDMA configuration */
1571 	mvreg_write(pp, MVNETA_SDMA_CONFIG, val);
1572 
1573 	/* Disable PHY polling in hardware, since we're using the
1574 	 * kernel phylib to do this.
1575 	 */
1576 	val = mvreg_read(pp, MVNETA_UNIT_CONTROL);
1577 	val &= ~MVNETA_PHY_POLLING_ENABLE;
1578 	mvreg_write(pp, MVNETA_UNIT_CONTROL, val);
1579 
1580 	mvneta_set_ucast_table(pp, -1);
1581 	mvneta_set_special_mcast_table(pp, -1);
1582 	mvneta_set_other_mcast_table(pp, -1);
1583 
1584 	/* Set port interrupt enable register - default enable all */
1585 	mvreg_write(pp, MVNETA_INTR_ENABLE,
1586 		    (MVNETA_RXQ_INTR_ENABLE_ALL_MASK
1587 		     | MVNETA_TXQ_INTR_ENABLE_ALL_MASK));
1588 
1589 	mvneta_mib_counters_clear(pp);
1590 }
1591 
1592 /* Set max sizes for tx queues */
mvneta_txq_max_tx_size_set(struct mvneta_port * pp,int max_tx_size)1593 static void mvneta_txq_max_tx_size_set(struct mvneta_port *pp, int max_tx_size)
1594 
1595 {
1596 	u32 val, size, mtu;
1597 	int queue;
1598 
1599 	mtu = max_tx_size * 8;
1600 	if (mtu > MVNETA_TX_MTU_MAX)
1601 		mtu = MVNETA_TX_MTU_MAX;
1602 
1603 	/* Set MTU */
1604 	val = mvreg_read(pp, MVNETA_TX_MTU);
1605 	val &= ~MVNETA_TX_MTU_MAX;
1606 	val |= mtu;
1607 	mvreg_write(pp, MVNETA_TX_MTU, val);
1608 
1609 	/* TX token size and all TXQs token size must be larger that MTU */
1610 	val = mvreg_read(pp, MVNETA_TX_TOKEN_SIZE);
1611 
1612 	size = val & MVNETA_TX_TOKEN_SIZE_MAX;
1613 	if (size < mtu) {
1614 		size = mtu;
1615 		val &= ~MVNETA_TX_TOKEN_SIZE_MAX;
1616 		val |= size;
1617 		mvreg_write(pp, MVNETA_TX_TOKEN_SIZE, val);
1618 	}
1619 	for (queue = 0; queue < txq_number; queue++) {
1620 		val = mvreg_read(pp, MVNETA_TXQ_TOKEN_SIZE_REG(queue));
1621 
1622 		size = val & MVNETA_TXQ_TOKEN_SIZE_MAX;
1623 		if (size < mtu) {
1624 			size = mtu;
1625 			val &= ~MVNETA_TXQ_TOKEN_SIZE_MAX;
1626 			val |= size;
1627 			mvreg_write(pp, MVNETA_TXQ_TOKEN_SIZE_REG(queue), val);
1628 		}
1629 	}
1630 }
1631 
1632 /* Set unicast address */
mvneta_set_ucast_addr(struct mvneta_port * pp,u8 last_nibble,int queue)1633 static void mvneta_set_ucast_addr(struct mvneta_port *pp, u8 last_nibble,
1634 				  int queue)
1635 {
1636 	unsigned int unicast_reg;
1637 	unsigned int tbl_offset;
1638 	unsigned int reg_offset;
1639 
1640 	/* Locate the Unicast table entry */
1641 	last_nibble = (0xf & last_nibble);
1642 
1643 	/* offset from unicast tbl base */
1644 	tbl_offset = (last_nibble / 4) * 4;
1645 
1646 	/* offset within the above reg  */
1647 	reg_offset = last_nibble % 4;
1648 
1649 	unicast_reg = mvreg_read(pp, (MVNETA_DA_FILT_UCAST_BASE + tbl_offset));
1650 
1651 	if (queue == -1) {
1652 		/* Clear accepts frame bit at specified unicast DA tbl entry */
1653 		unicast_reg &= ~(0xff << (8 * reg_offset));
1654 	} else {
1655 		unicast_reg &= ~(0xff << (8 * reg_offset));
1656 		unicast_reg |= ((0x01 | (queue << 1)) << (8 * reg_offset));
1657 	}
1658 
1659 	mvreg_write(pp, (MVNETA_DA_FILT_UCAST_BASE + tbl_offset), unicast_reg);
1660 }
1661 
1662 /* Set mac address */
mvneta_mac_addr_set(struct mvneta_port * pp,const unsigned char * addr,int queue)1663 static void mvneta_mac_addr_set(struct mvneta_port *pp,
1664 				const unsigned char *addr, int queue)
1665 {
1666 	unsigned int mac_h;
1667 	unsigned int mac_l;
1668 
1669 	if (queue != -1) {
1670 		mac_l = (addr[4] << 8) | (addr[5]);
1671 		mac_h = (addr[0] << 24) | (addr[1] << 16) |
1672 			(addr[2] << 8) | (addr[3] << 0);
1673 
1674 		mvreg_write(pp, MVNETA_MAC_ADDR_LOW, mac_l);
1675 		mvreg_write(pp, MVNETA_MAC_ADDR_HIGH, mac_h);
1676 	}
1677 
1678 	/* Accept frames of this address */
1679 	mvneta_set_ucast_addr(pp, addr[5], queue);
1680 }
1681 
1682 /* Set the number of packets that will be received before RX interrupt
1683  * will be generated by HW.
1684  */
mvneta_rx_pkts_coal_set(struct mvneta_port * pp,struct mvneta_rx_queue * rxq,u32 value)1685 static void mvneta_rx_pkts_coal_set(struct mvneta_port *pp,
1686 				    struct mvneta_rx_queue *rxq, u32 value)
1687 {
1688 	mvreg_write(pp, MVNETA_RXQ_THRESHOLD_REG(rxq->id),
1689 		    value | MVNETA_RXQ_NON_OCCUPIED(0));
1690 }
1691 
1692 /* Set the time delay in usec before RX interrupt will be generated by
1693  * HW.
1694  */
mvneta_rx_time_coal_set(struct mvneta_port * pp,struct mvneta_rx_queue * rxq,u32 value)1695 static void mvneta_rx_time_coal_set(struct mvneta_port *pp,
1696 				    struct mvneta_rx_queue *rxq, u32 value)
1697 {
1698 	u32 val;
1699 	unsigned long clk_rate;
1700 
1701 	clk_rate = clk_get_rate(pp->clk);
1702 	val = (clk_rate / 1000000) * value;
1703 
1704 	mvreg_write(pp, MVNETA_RXQ_TIME_COAL_REG(rxq->id), val);
1705 }
1706 
1707 /* Set threshold for TX_DONE pkts coalescing */
mvneta_tx_done_pkts_coal_set(struct mvneta_port * pp,struct mvneta_tx_queue * txq,u32 value)1708 static void mvneta_tx_done_pkts_coal_set(struct mvneta_port *pp,
1709 					 struct mvneta_tx_queue *txq, u32 value)
1710 {
1711 	u32 val;
1712 
1713 	val = mvreg_read(pp, MVNETA_TXQ_SIZE_REG(txq->id));
1714 
1715 	val &= ~MVNETA_TXQ_SENT_THRESH_ALL_MASK;
1716 	val |= MVNETA_TXQ_SENT_THRESH_MASK(value);
1717 
1718 	mvreg_write(pp, MVNETA_TXQ_SIZE_REG(txq->id), val);
1719 }
1720 
1721 /* Handle rx descriptor fill by setting buf_cookie and buf_phys_addr */
mvneta_rx_desc_fill(struct mvneta_rx_desc * rx_desc,u32 phys_addr,void * virt_addr,struct mvneta_rx_queue * rxq)1722 static void mvneta_rx_desc_fill(struct mvneta_rx_desc *rx_desc,
1723 				u32 phys_addr, void *virt_addr,
1724 				struct mvneta_rx_queue *rxq)
1725 {
1726 	int i;
1727 
1728 	rx_desc->buf_phys_addr = phys_addr;
1729 	i = rx_desc - rxq->descs;
1730 	rxq->buf_virt_addr[i] = virt_addr;
1731 }
1732 
1733 /* Decrement sent descriptors counter */
mvneta_txq_sent_desc_dec(struct mvneta_port * pp,struct mvneta_tx_queue * txq,int sent_desc)1734 static void mvneta_txq_sent_desc_dec(struct mvneta_port *pp,
1735 				     struct mvneta_tx_queue *txq,
1736 				     int sent_desc)
1737 {
1738 	u32 val;
1739 
1740 	/* Only 255 TX descriptors can be updated at once */
1741 	while (sent_desc > 0xff) {
1742 		val = 0xff << MVNETA_TXQ_DEC_SENT_SHIFT;
1743 		mvreg_write(pp, MVNETA_TXQ_UPDATE_REG(txq->id), val);
1744 		sent_desc = sent_desc - 0xff;
1745 	}
1746 
1747 	val = sent_desc << MVNETA_TXQ_DEC_SENT_SHIFT;
1748 	mvreg_write(pp, MVNETA_TXQ_UPDATE_REG(txq->id), val);
1749 }
1750 
1751 /* Get number of TX descriptors already sent by HW */
mvneta_txq_sent_desc_num_get(struct mvneta_port * pp,struct mvneta_tx_queue * txq)1752 static int mvneta_txq_sent_desc_num_get(struct mvneta_port *pp,
1753 					struct mvneta_tx_queue *txq)
1754 {
1755 	u32 val;
1756 	int sent_desc;
1757 
1758 	val = mvreg_read(pp, MVNETA_TXQ_STATUS_REG(txq->id));
1759 	sent_desc = (val & MVNETA_TXQ_SENT_DESC_MASK) >>
1760 		MVNETA_TXQ_SENT_DESC_SHIFT;
1761 
1762 	return sent_desc;
1763 }
1764 
1765 /* Get number of sent descriptors and decrement counter.
1766  *  The number of sent descriptors is returned.
1767  */
mvneta_txq_sent_desc_proc(struct mvneta_port * pp,struct mvneta_tx_queue * txq)1768 static int mvneta_txq_sent_desc_proc(struct mvneta_port *pp,
1769 				     struct mvneta_tx_queue *txq)
1770 {
1771 	int sent_desc;
1772 
1773 	/* Get number of sent descriptors */
1774 	sent_desc = mvneta_txq_sent_desc_num_get(pp, txq);
1775 
1776 	/* Decrement sent descriptors counter */
1777 	if (sent_desc)
1778 		mvneta_txq_sent_desc_dec(pp, txq, sent_desc);
1779 
1780 	return sent_desc;
1781 }
1782 
1783 /* Set TXQ descriptors fields relevant for CSUM calculation */
mvneta_txq_desc_csum(int l3_offs,__be16 l3_proto,int ip_hdr_len,int l4_proto)1784 static u32 mvneta_txq_desc_csum(int l3_offs, __be16 l3_proto,
1785 				int ip_hdr_len, int l4_proto)
1786 {
1787 	u32 command;
1788 
1789 	/* Fields: L3_offset, IP_hdrlen, L3_type, G_IPv4_chk,
1790 	 * G_L4_chk, L4_type; required only for checksum
1791 	 * calculation
1792 	 */
1793 	command =  l3_offs    << MVNETA_TX_L3_OFF_SHIFT;
1794 	command |= ip_hdr_len << MVNETA_TX_IP_HLEN_SHIFT;
1795 
1796 	if (l3_proto == htons(ETH_P_IP))
1797 		command |= MVNETA_TXD_IP_CSUM;
1798 	else
1799 		command |= MVNETA_TX_L3_IP6;
1800 
1801 	if (l4_proto == IPPROTO_TCP)
1802 		command |=  MVNETA_TX_L4_CSUM_FULL;
1803 	else if (l4_proto == IPPROTO_UDP)
1804 		command |= MVNETA_TX_L4_UDP | MVNETA_TX_L4_CSUM_FULL;
1805 	else
1806 		command |= MVNETA_TX_L4_CSUM_NOT;
1807 
1808 	return command;
1809 }
1810 
1811 
1812 /* Display more error info */
mvneta_rx_error(struct mvneta_port * pp,struct mvneta_rx_desc * rx_desc)1813 static void mvneta_rx_error(struct mvneta_port *pp,
1814 			    struct mvneta_rx_desc *rx_desc)
1815 {
1816 	struct mvneta_pcpu_stats *stats = this_cpu_ptr(pp->stats);
1817 	u32 status = rx_desc->status;
1818 
1819 	/* update per-cpu counter */
1820 	u64_stats_update_begin(&stats->syncp);
1821 	stats->rx_errors++;
1822 	u64_stats_update_end(&stats->syncp);
1823 
1824 	switch (status & MVNETA_RXD_ERR_CODE_MASK) {
1825 	case MVNETA_RXD_ERR_CRC:
1826 		netdev_err(pp->dev, "bad rx status %08x (crc error), size=%d\n",
1827 			   status, rx_desc->data_size);
1828 		break;
1829 	case MVNETA_RXD_ERR_OVERRUN:
1830 		netdev_err(pp->dev, "bad rx status %08x (overrun error), size=%d\n",
1831 			   status, rx_desc->data_size);
1832 		break;
1833 	case MVNETA_RXD_ERR_LEN:
1834 		netdev_err(pp->dev, "bad rx status %08x (max frame length error), size=%d\n",
1835 			   status, rx_desc->data_size);
1836 		break;
1837 	case MVNETA_RXD_ERR_RESOURCE:
1838 		netdev_err(pp->dev, "bad rx status %08x (resource error), size=%d\n",
1839 			   status, rx_desc->data_size);
1840 		break;
1841 	}
1842 }
1843 
1844 /* Handle RX checksum offload based on the descriptor's status */
mvneta_rx_csum(struct mvneta_port * pp,u32 status)1845 static int mvneta_rx_csum(struct mvneta_port *pp, u32 status)
1846 {
1847 	if ((pp->dev->features & NETIF_F_RXCSUM) &&
1848 	    (status & MVNETA_RXD_L3_IP4) &&
1849 	    (status & MVNETA_RXD_L4_CSUM_OK))
1850 		return CHECKSUM_UNNECESSARY;
1851 
1852 	return CHECKSUM_NONE;
1853 }
1854 
1855 /* Return tx queue pointer (find last set bit) according to <cause> returned
1856  * form tx_done reg. <cause> must not be null. The return value is always a
1857  * valid queue for matching the first one found in <cause>.
1858  */
mvneta_tx_done_policy(struct mvneta_port * pp,u32 cause)1859 static struct mvneta_tx_queue *mvneta_tx_done_policy(struct mvneta_port *pp,
1860 						     u32 cause)
1861 {
1862 	int queue = fls(cause) - 1;
1863 
1864 	return &pp->txqs[queue];
1865 }
1866 
1867 /* Free tx queue skbuffs */
mvneta_txq_bufs_free(struct mvneta_port * pp,struct mvneta_tx_queue * txq,int num,struct netdev_queue * nq,bool napi)1868 static void mvneta_txq_bufs_free(struct mvneta_port *pp,
1869 				 struct mvneta_tx_queue *txq, int num,
1870 				 struct netdev_queue *nq, bool napi)
1871 {
1872 	unsigned int bytes_compl = 0, pkts_compl = 0;
1873 	struct xdp_frame_bulk bq;
1874 	int i;
1875 
1876 	xdp_frame_bulk_init(&bq);
1877 
1878 	rcu_read_lock(); /* need for xdp_return_frame_bulk */
1879 
1880 	for (i = 0; i < num; i++) {
1881 		struct mvneta_tx_buf *buf = &txq->buf[txq->txq_get_index];
1882 		struct mvneta_tx_desc *tx_desc = txq->descs +
1883 			txq->txq_get_index;
1884 
1885 		mvneta_txq_inc_get(txq);
1886 
1887 		if (buf->type == MVNETA_TYPE_XDP_NDO ||
1888 		    buf->type == MVNETA_TYPE_SKB)
1889 			dma_unmap_single(pp->dev->dev.parent,
1890 					 tx_desc->buf_phys_addr,
1891 					 tx_desc->data_size, DMA_TO_DEVICE);
1892 		if ((buf->type == MVNETA_TYPE_TSO ||
1893 		     buf->type == MVNETA_TYPE_SKB) && buf->skb) {
1894 			bytes_compl += buf->skb->len;
1895 			pkts_compl++;
1896 			dev_kfree_skb_any(buf->skb);
1897 		} else if ((buf->type == MVNETA_TYPE_XDP_TX ||
1898 			    buf->type == MVNETA_TYPE_XDP_NDO) && buf->xdpf) {
1899 			if (napi && buf->type == MVNETA_TYPE_XDP_TX)
1900 				xdp_return_frame_rx_napi(buf->xdpf);
1901 			else
1902 				xdp_return_frame_bulk(buf->xdpf, &bq);
1903 		}
1904 	}
1905 	xdp_flush_frame_bulk(&bq);
1906 
1907 	rcu_read_unlock();
1908 
1909 	netdev_tx_completed_queue(nq, pkts_compl, bytes_compl);
1910 }
1911 
1912 /* Handle end of transmission */
mvneta_txq_done(struct mvneta_port * pp,struct mvneta_tx_queue * txq)1913 static void mvneta_txq_done(struct mvneta_port *pp,
1914 			   struct mvneta_tx_queue *txq)
1915 {
1916 	struct netdev_queue *nq = netdev_get_tx_queue(pp->dev, txq->id);
1917 	int tx_done;
1918 
1919 	tx_done = mvneta_txq_sent_desc_proc(pp, txq);
1920 	if (!tx_done)
1921 		return;
1922 
1923 	mvneta_txq_bufs_free(pp, txq, tx_done, nq, true);
1924 
1925 	txq->count -= tx_done;
1926 
1927 	if (netif_tx_queue_stopped(nq)) {
1928 		if (txq->count <= txq->tx_wake_threshold)
1929 			netif_tx_wake_queue(nq);
1930 	}
1931 }
1932 
1933 /* Refill processing for SW buffer management */
1934 /* Allocate page per descriptor */
mvneta_rx_refill(struct mvneta_port * pp,struct mvneta_rx_desc * rx_desc,struct mvneta_rx_queue * rxq,gfp_t gfp_mask)1935 static int mvneta_rx_refill(struct mvneta_port *pp,
1936 			    struct mvneta_rx_desc *rx_desc,
1937 			    struct mvneta_rx_queue *rxq,
1938 			    gfp_t gfp_mask)
1939 {
1940 	dma_addr_t phys_addr;
1941 	struct page *page;
1942 
1943 	page = page_pool_alloc_pages(rxq->page_pool,
1944 				     gfp_mask | __GFP_NOWARN);
1945 	if (!page)
1946 		return -ENOMEM;
1947 
1948 	phys_addr = page_pool_get_dma_addr(page) + pp->rx_offset_correction;
1949 	mvneta_rx_desc_fill(rx_desc, phys_addr, page, rxq);
1950 
1951 	return 0;
1952 }
1953 
1954 /* Handle tx checksum */
mvneta_skb_tx_csum(struct sk_buff * skb)1955 static u32 mvneta_skb_tx_csum(struct sk_buff *skb)
1956 {
1957 	if (skb->ip_summed == CHECKSUM_PARTIAL) {
1958 		int ip_hdr_len = 0;
1959 		__be16 l3_proto = vlan_get_protocol(skb);
1960 		u8 l4_proto;
1961 
1962 		if (l3_proto == htons(ETH_P_IP)) {
1963 			struct iphdr *ip4h = ip_hdr(skb);
1964 
1965 			/* Calculate IPv4 checksum and L4 checksum */
1966 			ip_hdr_len = ip4h->ihl;
1967 			l4_proto = ip4h->protocol;
1968 		} else if (l3_proto == htons(ETH_P_IPV6)) {
1969 			struct ipv6hdr *ip6h = ipv6_hdr(skb);
1970 
1971 			/* Read l4_protocol from one of IPv6 extra headers */
1972 			if (skb_network_header_len(skb) > 0)
1973 				ip_hdr_len = (skb_network_header_len(skb) >> 2);
1974 			l4_proto = ip6h->nexthdr;
1975 		} else
1976 			return MVNETA_TX_L4_CSUM_NOT;
1977 
1978 		return mvneta_txq_desc_csum(skb_network_offset(skb),
1979 					    l3_proto, ip_hdr_len, l4_proto);
1980 	}
1981 
1982 	return MVNETA_TX_L4_CSUM_NOT;
1983 }
1984 
1985 /* Drop packets received by the RXQ and free buffers */
mvneta_rxq_drop_pkts(struct mvneta_port * pp,struct mvneta_rx_queue * rxq)1986 static void mvneta_rxq_drop_pkts(struct mvneta_port *pp,
1987 				 struct mvneta_rx_queue *rxq)
1988 {
1989 	int rx_done, i;
1990 
1991 	rx_done = mvneta_rxq_busy_desc_num_get(pp, rxq);
1992 	if (rx_done)
1993 		mvneta_rxq_desc_num_update(pp, rxq, rx_done, rx_done);
1994 
1995 	if (pp->bm_priv) {
1996 		for (i = 0; i < rx_done; i++) {
1997 			struct mvneta_rx_desc *rx_desc =
1998 						  mvneta_rxq_next_desc_get(rxq);
1999 			u8 pool_id = MVNETA_RX_GET_BM_POOL_ID(rx_desc);
2000 			struct mvneta_bm_pool *bm_pool;
2001 
2002 			bm_pool = &pp->bm_priv->bm_pools[pool_id];
2003 			/* Return dropped buffer to the pool */
2004 			mvneta_bm_pool_put_bp(pp->bm_priv, bm_pool,
2005 					      rx_desc->buf_phys_addr);
2006 		}
2007 		return;
2008 	}
2009 
2010 	for (i = 0; i < rxq->size; i++) {
2011 		struct mvneta_rx_desc *rx_desc = rxq->descs + i;
2012 		void *data = rxq->buf_virt_addr[i];
2013 		if (!data || !(rx_desc->buf_phys_addr))
2014 			continue;
2015 
2016 		page_pool_put_full_page(rxq->page_pool, data, false);
2017 	}
2018 	if (xdp_rxq_info_is_reg(&rxq->xdp_rxq))
2019 		xdp_rxq_info_unreg(&rxq->xdp_rxq);
2020 	page_pool_destroy(rxq->page_pool);
2021 	rxq->page_pool = NULL;
2022 }
2023 
2024 static void
mvneta_update_stats(struct mvneta_port * pp,struct mvneta_stats * ps)2025 mvneta_update_stats(struct mvneta_port *pp,
2026 		    struct mvneta_stats *ps)
2027 {
2028 	struct mvneta_pcpu_stats *stats = this_cpu_ptr(pp->stats);
2029 
2030 	u64_stats_update_begin(&stats->syncp);
2031 	stats->es.ps.rx_packets += ps->rx_packets;
2032 	stats->es.ps.rx_bytes += ps->rx_bytes;
2033 	/* xdp */
2034 	stats->es.ps.xdp_redirect += ps->xdp_redirect;
2035 	stats->es.ps.xdp_pass += ps->xdp_pass;
2036 	stats->es.ps.xdp_drop += ps->xdp_drop;
2037 	u64_stats_update_end(&stats->syncp);
2038 }
2039 
2040 static inline
mvneta_rx_refill_queue(struct mvneta_port * pp,struct mvneta_rx_queue * rxq)2041 int mvneta_rx_refill_queue(struct mvneta_port *pp, struct mvneta_rx_queue *rxq)
2042 {
2043 	struct mvneta_rx_desc *rx_desc;
2044 	int curr_desc = rxq->first_to_refill;
2045 	int i;
2046 
2047 	for (i = 0; (i < rxq->refill_num) && (i < 64); i++) {
2048 		rx_desc = rxq->descs + curr_desc;
2049 		if (!(rx_desc->buf_phys_addr)) {
2050 			if (mvneta_rx_refill(pp, rx_desc, rxq, GFP_ATOMIC)) {
2051 				struct mvneta_pcpu_stats *stats;
2052 
2053 				pr_err("Can't refill queue %d. Done %d from %d\n",
2054 				       rxq->id, i, rxq->refill_num);
2055 
2056 				stats = this_cpu_ptr(pp->stats);
2057 				u64_stats_update_begin(&stats->syncp);
2058 				stats->es.refill_error++;
2059 				u64_stats_update_end(&stats->syncp);
2060 				break;
2061 			}
2062 		}
2063 		curr_desc = MVNETA_QUEUE_NEXT_DESC(rxq, curr_desc);
2064 	}
2065 	rxq->refill_num -= i;
2066 	rxq->first_to_refill = curr_desc;
2067 
2068 	return i;
2069 }
2070 
2071 static void
mvneta_xdp_put_buff(struct mvneta_port * pp,struct mvneta_rx_queue * rxq,struct xdp_buff * xdp,int sync_len)2072 mvneta_xdp_put_buff(struct mvneta_port *pp, struct mvneta_rx_queue *rxq,
2073 		    struct xdp_buff *xdp, int sync_len)
2074 {
2075 	struct skb_shared_info *sinfo = xdp_get_shared_info_from_buff(xdp);
2076 	int i;
2077 
2078 	if (likely(!xdp_buff_has_frags(xdp)))
2079 		goto out;
2080 
2081 	for (i = 0; i < sinfo->nr_frags; i++)
2082 		page_pool_put_full_page(rxq->page_pool,
2083 					skb_frag_page(&sinfo->frags[i]), true);
2084 
2085 out:
2086 	page_pool_put_page(rxq->page_pool, virt_to_head_page(xdp->data),
2087 			   sync_len, true);
2088 }
2089 
2090 static int
mvneta_xdp_submit_frame(struct mvneta_port * pp,struct mvneta_tx_queue * txq,struct xdp_frame * xdpf,int * nxmit_byte,bool dma_map)2091 mvneta_xdp_submit_frame(struct mvneta_port *pp, struct mvneta_tx_queue *txq,
2092 			struct xdp_frame *xdpf, int *nxmit_byte, bool dma_map)
2093 {
2094 	struct skb_shared_info *sinfo = xdp_get_shared_info_from_frame(xdpf);
2095 	struct device *dev = pp->dev->dev.parent;
2096 	struct mvneta_tx_desc *tx_desc;
2097 	int i, num_frames = 1;
2098 	struct page *page;
2099 
2100 	if (unlikely(xdp_frame_has_frags(xdpf)))
2101 		num_frames += sinfo->nr_frags;
2102 
2103 	if (txq->count + num_frames >= txq->size)
2104 		return MVNETA_XDP_DROPPED;
2105 
2106 	for (i = 0; i < num_frames; i++) {
2107 		struct mvneta_tx_buf *buf = &txq->buf[txq->txq_put_index];
2108 		skb_frag_t *frag = NULL;
2109 		int len = xdpf->len;
2110 		dma_addr_t dma_addr;
2111 
2112 		if (unlikely(i)) { /* paged area */
2113 			frag = &sinfo->frags[i - 1];
2114 			len = skb_frag_size(frag);
2115 		}
2116 
2117 		tx_desc = mvneta_txq_next_desc_get(txq);
2118 		if (dma_map) {
2119 			/* ndo_xdp_xmit */
2120 			void *data;
2121 
2122 			data = unlikely(frag) ? skb_frag_address(frag)
2123 					      : xdpf->data;
2124 			dma_addr = dma_map_single(dev, data, len,
2125 						  DMA_TO_DEVICE);
2126 			if (dma_mapping_error(dev, dma_addr)) {
2127 				mvneta_txq_desc_put(txq);
2128 				goto unmap;
2129 			}
2130 
2131 			buf->type = MVNETA_TYPE_XDP_NDO;
2132 		} else {
2133 			page = unlikely(frag) ? skb_frag_page(frag)
2134 					      : virt_to_page(xdpf->data);
2135 			dma_addr = page_pool_get_dma_addr(page);
2136 			if (unlikely(frag))
2137 				dma_addr += skb_frag_off(frag);
2138 			else
2139 				dma_addr += sizeof(*xdpf) + xdpf->headroom;
2140 			dma_sync_single_for_device(dev, dma_addr, len,
2141 						   DMA_BIDIRECTIONAL);
2142 			buf->type = MVNETA_TYPE_XDP_TX;
2143 		}
2144 		buf->xdpf = unlikely(i) ? NULL : xdpf;
2145 
2146 		tx_desc->command = unlikely(i) ? 0 : MVNETA_TXD_F_DESC;
2147 		tx_desc->buf_phys_addr = dma_addr;
2148 		tx_desc->data_size = len;
2149 		*nxmit_byte += len;
2150 
2151 		mvneta_txq_inc_put(txq);
2152 	}
2153 	/*last descriptor */
2154 	tx_desc->command |= MVNETA_TXD_L_DESC | MVNETA_TXD_Z_PAD;
2155 
2156 	txq->pending += num_frames;
2157 	txq->count += num_frames;
2158 
2159 	return MVNETA_XDP_TX;
2160 
2161 unmap:
2162 	for (i--; i >= 0; i--) {
2163 		mvneta_txq_desc_put(txq);
2164 		tx_desc = txq->descs + txq->next_desc_to_proc;
2165 		dma_unmap_single(dev, tx_desc->buf_phys_addr,
2166 				 tx_desc->data_size,
2167 				 DMA_TO_DEVICE);
2168 	}
2169 
2170 	return MVNETA_XDP_DROPPED;
2171 }
2172 
2173 static int
mvneta_xdp_xmit_back(struct mvneta_port * pp,struct xdp_buff * xdp)2174 mvneta_xdp_xmit_back(struct mvneta_port *pp, struct xdp_buff *xdp)
2175 {
2176 	struct mvneta_pcpu_stats *stats = this_cpu_ptr(pp->stats);
2177 	struct mvneta_tx_queue *txq;
2178 	struct netdev_queue *nq;
2179 	int cpu, nxmit_byte = 0;
2180 	struct xdp_frame *xdpf;
2181 	u32 ret;
2182 
2183 	xdpf = xdp_convert_buff_to_frame(xdp);
2184 	if (unlikely(!xdpf))
2185 		return MVNETA_XDP_DROPPED;
2186 
2187 	cpu = smp_processor_id();
2188 	txq = &pp->txqs[cpu % txq_number];
2189 	nq = netdev_get_tx_queue(pp->dev, txq->id);
2190 
2191 	__netif_tx_lock(nq, cpu);
2192 	ret = mvneta_xdp_submit_frame(pp, txq, xdpf, &nxmit_byte, false);
2193 	if (ret == MVNETA_XDP_TX) {
2194 		u64_stats_update_begin(&stats->syncp);
2195 		stats->es.ps.tx_bytes += nxmit_byte;
2196 		stats->es.ps.tx_packets++;
2197 		stats->es.ps.xdp_tx++;
2198 		u64_stats_update_end(&stats->syncp);
2199 
2200 		mvneta_txq_pend_desc_add(pp, txq, 0);
2201 	} else {
2202 		u64_stats_update_begin(&stats->syncp);
2203 		stats->es.ps.xdp_tx_err++;
2204 		u64_stats_update_end(&stats->syncp);
2205 	}
2206 	__netif_tx_unlock(nq);
2207 
2208 	return ret;
2209 }
2210 
2211 static int
mvneta_xdp_xmit(struct net_device * dev,int num_frame,struct xdp_frame ** frames,u32 flags)2212 mvneta_xdp_xmit(struct net_device *dev, int num_frame,
2213 		struct xdp_frame **frames, u32 flags)
2214 {
2215 	struct mvneta_port *pp = netdev_priv(dev);
2216 	struct mvneta_pcpu_stats *stats = this_cpu_ptr(pp->stats);
2217 	int i, nxmit_byte = 0, nxmit = 0;
2218 	int cpu = smp_processor_id();
2219 	struct mvneta_tx_queue *txq;
2220 	struct netdev_queue *nq;
2221 	u32 ret;
2222 
2223 	if (unlikely(test_bit(__MVNETA_DOWN, &pp->state)))
2224 		return -ENETDOWN;
2225 
2226 	if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK))
2227 		return -EINVAL;
2228 
2229 	txq = &pp->txqs[cpu % txq_number];
2230 	nq = netdev_get_tx_queue(pp->dev, txq->id);
2231 
2232 	__netif_tx_lock(nq, cpu);
2233 	for (i = 0; i < num_frame; i++) {
2234 		ret = mvneta_xdp_submit_frame(pp, txq, frames[i], &nxmit_byte,
2235 					      true);
2236 		if (ret != MVNETA_XDP_TX)
2237 			break;
2238 
2239 		nxmit++;
2240 	}
2241 
2242 	if (unlikely(flags & XDP_XMIT_FLUSH))
2243 		mvneta_txq_pend_desc_add(pp, txq, 0);
2244 	__netif_tx_unlock(nq);
2245 
2246 	u64_stats_update_begin(&stats->syncp);
2247 	stats->es.ps.tx_bytes += nxmit_byte;
2248 	stats->es.ps.tx_packets += nxmit;
2249 	stats->es.ps.xdp_xmit += nxmit;
2250 	stats->es.ps.xdp_xmit_err += num_frame - nxmit;
2251 	u64_stats_update_end(&stats->syncp);
2252 
2253 	return nxmit;
2254 }
2255 
2256 static int
mvneta_run_xdp(struct mvneta_port * pp,struct mvneta_rx_queue * rxq,struct bpf_prog * prog,struct xdp_buff * xdp,u32 frame_sz,struct mvneta_stats * stats)2257 mvneta_run_xdp(struct mvneta_port *pp, struct mvneta_rx_queue *rxq,
2258 	       struct bpf_prog *prog, struct xdp_buff *xdp,
2259 	       u32 frame_sz, struct mvneta_stats *stats)
2260 {
2261 	unsigned int len, data_len, sync;
2262 	u32 ret, act;
2263 
2264 	len = xdp->data_end - xdp->data_hard_start - pp->rx_offset_correction;
2265 	data_len = xdp->data_end - xdp->data;
2266 	act = bpf_prog_run_xdp(prog, xdp);
2267 
2268 	/* Due xdp_adjust_tail: DMA sync for_device cover max len CPU touch */
2269 	sync = xdp->data_end - xdp->data_hard_start - pp->rx_offset_correction;
2270 	sync = max(sync, len);
2271 
2272 	switch (act) {
2273 	case XDP_PASS:
2274 		stats->xdp_pass++;
2275 		return MVNETA_XDP_PASS;
2276 	case XDP_REDIRECT: {
2277 		int err;
2278 
2279 		err = xdp_do_redirect(pp->dev, xdp, prog);
2280 		if (unlikely(err)) {
2281 			mvneta_xdp_put_buff(pp, rxq, xdp, sync);
2282 			ret = MVNETA_XDP_DROPPED;
2283 		} else {
2284 			ret = MVNETA_XDP_REDIR;
2285 			stats->xdp_redirect++;
2286 		}
2287 		break;
2288 	}
2289 	case XDP_TX:
2290 		ret = mvneta_xdp_xmit_back(pp, xdp);
2291 		if (ret != MVNETA_XDP_TX)
2292 			mvneta_xdp_put_buff(pp, rxq, xdp, sync);
2293 		break;
2294 	default:
2295 		bpf_warn_invalid_xdp_action(pp->dev, prog, act);
2296 		fallthrough;
2297 	case XDP_ABORTED:
2298 		trace_xdp_exception(pp->dev, prog, act);
2299 		fallthrough;
2300 	case XDP_DROP:
2301 		mvneta_xdp_put_buff(pp, rxq, xdp, sync);
2302 		ret = MVNETA_XDP_DROPPED;
2303 		stats->xdp_drop++;
2304 		break;
2305 	}
2306 
2307 	stats->rx_bytes += frame_sz + xdp->data_end - xdp->data - data_len;
2308 	stats->rx_packets++;
2309 
2310 	return ret;
2311 }
2312 
2313 static void
mvneta_swbm_rx_frame(struct mvneta_port * pp,struct mvneta_rx_desc * rx_desc,struct mvneta_rx_queue * rxq,struct xdp_buff * xdp,int * size,struct page * page)2314 mvneta_swbm_rx_frame(struct mvneta_port *pp,
2315 		     struct mvneta_rx_desc *rx_desc,
2316 		     struct mvneta_rx_queue *rxq,
2317 		     struct xdp_buff *xdp, int *size,
2318 		     struct page *page)
2319 {
2320 	unsigned char *data = page_address(page);
2321 	int data_len = -MVNETA_MH_SIZE, len;
2322 	struct net_device *dev = pp->dev;
2323 	enum dma_data_direction dma_dir;
2324 
2325 	if (*size > MVNETA_MAX_RX_BUF_SIZE) {
2326 		len = MVNETA_MAX_RX_BUF_SIZE;
2327 		data_len += len;
2328 	} else {
2329 		len = *size;
2330 		data_len += len - ETH_FCS_LEN;
2331 	}
2332 	*size = *size - len;
2333 
2334 	dma_dir = page_pool_get_dma_dir(rxq->page_pool);
2335 	dma_sync_single_for_cpu(dev->dev.parent,
2336 				rx_desc->buf_phys_addr,
2337 				len, dma_dir);
2338 
2339 	rx_desc->buf_phys_addr = 0;
2340 
2341 	/* Prefetch header */
2342 	prefetch(data);
2343 	xdp_buff_clear_frags_flag(xdp);
2344 	xdp_prepare_buff(xdp, data, pp->rx_offset_correction + MVNETA_MH_SIZE,
2345 			 data_len, false);
2346 }
2347 
2348 static void
mvneta_swbm_add_rx_fragment(struct mvneta_port * pp,struct mvneta_rx_desc * rx_desc,struct mvneta_rx_queue * rxq,struct xdp_buff * xdp,int * size,struct page * page)2349 mvneta_swbm_add_rx_fragment(struct mvneta_port *pp,
2350 			    struct mvneta_rx_desc *rx_desc,
2351 			    struct mvneta_rx_queue *rxq,
2352 			    struct xdp_buff *xdp, int *size,
2353 			    struct page *page)
2354 {
2355 	struct skb_shared_info *sinfo = xdp_get_shared_info_from_buff(xdp);
2356 	struct net_device *dev = pp->dev;
2357 	enum dma_data_direction dma_dir;
2358 	int data_len, len;
2359 
2360 	if (*size > MVNETA_MAX_RX_BUF_SIZE) {
2361 		len = MVNETA_MAX_RX_BUF_SIZE;
2362 		data_len = len;
2363 	} else {
2364 		len = *size;
2365 		data_len = len - ETH_FCS_LEN;
2366 	}
2367 	dma_dir = page_pool_get_dma_dir(rxq->page_pool);
2368 	dma_sync_single_for_cpu(dev->dev.parent,
2369 				rx_desc->buf_phys_addr,
2370 				len, dma_dir);
2371 	rx_desc->buf_phys_addr = 0;
2372 
2373 	if (!xdp_buff_has_frags(xdp))
2374 		sinfo->nr_frags = 0;
2375 
2376 	if (data_len > 0 && sinfo->nr_frags < MAX_SKB_FRAGS) {
2377 		skb_frag_t *frag = &sinfo->frags[sinfo->nr_frags++];
2378 
2379 		skb_frag_fill_page_desc(frag, page,
2380 					pp->rx_offset_correction, data_len);
2381 
2382 		if (!xdp_buff_has_frags(xdp)) {
2383 			sinfo->xdp_frags_size = *size;
2384 			xdp_buff_set_frags_flag(xdp);
2385 		}
2386 		if (page_is_pfmemalloc(page))
2387 			xdp_buff_set_frag_pfmemalloc(xdp);
2388 	} else {
2389 		page_pool_put_full_page(rxq->page_pool, page, true);
2390 	}
2391 	*size -= len;
2392 }
2393 
2394 static struct sk_buff *
mvneta_swbm_build_skb(struct mvneta_port * pp,struct page_pool * pool,struct xdp_buff * xdp,u32 desc_status)2395 mvneta_swbm_build_skb(struct mvneta_port *pp, struct page_pool *pool,
2396 		      struct xdp_buff *xdp, u32 desc_status)
2397 {
2398 	struct skb_shared_info *sinfo = xdp_get_shared_info_from_buff(xdp);
2399 	struct sk_buff *skb;
2400 	u8 num_frags;
2401 
2402 	if (unlikely(xdp_buff_has_frags(xdp)))
2403 		num_frags = sinfo->nr_frags;
2404 
2405 	skb = build_skb(xdp->data_hard_start, PAGE_SIZE);
2406 	if (!skb)
2407 		return ERR_PTR(-ENOMEM);
2408 
2409 	skb_mark_for_recycle(skb);
2410 
2411 	skb_reserve(skb, xdp->data - xdp->data_hard_start);
2412 	skb_put(skb, xdp->data_end - xdp->data);
2413 	skb->ip_summed = mvneta_rx_csum(pp, desc_status);
2414 
2415 	if (unlikely(xdp_buff_has_frags(xdp)))
2416 		xdp_update_skb_shared_info(skb, num_frags,
2417 					   sinfo->xdp_frags_size,
2418 					   num_frags * xdp->frame_sz,
2419 					   xdp_buff_is_frag_pfmemalloc(xdp));
2420 
2421 	return skb;
2422 }
2423 
2424 /* Main rx processing when using software buffer management */
mvneta_rx_swbm(struct napi_struct * napi,struct mvneta_port * pp,int budget,struct mvneta_rx_queue * rxq)2425 static int mvneta_rx_swbm(struct napi_struct *napi,
2426 			  struct mvneta_port *pp, int budget,
2427 			  struct mvneta_rx_queue *rxq)
2428 {
2429 	int rx_proc = 0, rx_todo, refill, size = 0;
2430 	struct net_device *dev = pp->dev;
2431 	struct mvneta_stats ps = {};
2432 	struct bpf_prog *xdp_prog;
2433 	u32 desc_status, frame_sz;
2434 	struct xdp_buff xdp_buf;
2435 
2436 	xdp_init_buff(&xdp_buf, PAGE_SIZE, &rxq->xdp_rxq);
2437 	xdp_buf.data_hard_start = NULL;
2438 
2439 	/* Get number of received packets */
2440 	rx_todo = mvneta_rxq_busy_desc_num_get(pp, rxq);
2441 
2442 	xdp_prog = READ_ONCE(pp->xdp_prog);
2443 
2444 	/* Fairness NAPI loop */
2445 	while (rx_proc < budget && rx_proc < rx_todo) {
2446 		struct mvneta_rx_desc *rx_desc = mvneta_rxq_next_desc_get(rxq);
2447 		u32 rx_status, index;
2448 		struct sk_buff *skb;
2449 		struct page *page;
2450 
2451 		index = rx_desc - rxq->descs;
2452 		page = (struct page *)rxq->buf_virt_addr[index];
2453 
2454 		rx_status = rx_desc->status;
2455 		rx_proc++;
2456 		rxq->refill_num++;
2457 
2458 		if (rx_status & MVNETA_RXD_FIRST_DESC) {
2459 			/* Check errors only for FIRST descriptor */
2460 			if (rx_status & MVNETA_RXD_ERR_SUMMARY) {
2461 				mvneta_rx_error(pp, rx_desc);
2462 				goto next;
2463 			}
2464 
2465 			size = rx_desc->data_size;
2466 			frame_sz = size - ETH_FCS_LEN;
2467 			desc_status = rx_status;
2468 
2469 			mvneta_swbm_rx_frame(pp, rx_desc, rxq, &xdp_buf,
2470 					     &size, page);
2471 		} else {
2472 			if (unlikely(!xdp_buf.data_hard_start)) {
2473 				rx_desc->buf_phys_addr = 0;
2474 				page_pool_put_full_page(rxq->page_pool, page,
2475 							true);
2476 				goto next;
2477 			}
2478 
2479 			mvneta_swbm_add_rx_fragment(pp, rx_desc, rxq, &xdp_buf,
2480 						    &size, page);
2481 		} /* Middle or Last descriptor */
2482 
2483 		if (!(rx_status & MVNETA_RXD_LAST_DESC))
2484 			/* no last descriptor this time */
2485 			continue;
2486 
2487 		if (size) {
2488 			mvneta_xdp_put_buff(pp, rxq, &xdp_buf, -1);
2489 			goto next;
2490 		}
2491 
2492 		if (xdp_prog &&
2493 		    mvneta_run_xdp(pp, rxq, xdp_prog, &xdp_buf, frame_sz, &ps))
2494 			goto next;
2495 
2496 		skb = mvneta_swbm_build_skb(pp, rxq->page_pool, &xdp_buf, desc_status);
2497 		if (IS_ERR(skb)) {
2498 			struct mvneta_pcpu_stats *stats = this_cpu_ptr(pp->stats);
2499 
2500 			mvneta_xdp_put_buff(pp, rxq, &xdp_buf, -1);
2501 
2502 			u64_stats_update_begin(&stats->syncp);
2503 			stats->es.skb_alloc_error++;
2504 			stats->rx_dropped++;
2505 			u64_stats_update_end(&stats->syncp);
2506 
2507 			goto next;
2508 		}
2509 
2510 		ps.rx_bytes += skb->len;
2511 		ps.rx_packets++;
2512 
2513 		skb->protocol = eth_type_trans(skb, dev);
2514 		napi_gro_receive(napi, skb);
2515 next:
2516 		xdp_buf.data_hard_start = NULL;
2517 	}
2518 
2519 	if (xdp_buf.data_hard_start)
2520 		mvneta_xdp_put_buff(pp, rxq, &xdp_buf, -1);
2521 
2522 	if (ps.xdp_redirect)
2523 		xdp_do_flush();
2524 
2525 	if (ps.rx_packets)
2526 		mvneta_update_stats(pp, &ps);
2527 
2528 	/* return some buffers to hardware queue, one at a time is too slow */
2529 	refill = mvneta_rx_refill_queue(pp, rxq);
2530 
2531 	/* Update rxq management counters */
2532 	mvneta_rxq_desc_num_update(pp, rxq, rx_proc, refill);
2533 
2534 	return ps.rx_packets;
2535 }
2536 
2537 /* Main rx processing when using hardware buffer management */
mvneta_rx_hwbm(struct napi_struct * napi,struct mvneta_port * pp,int rx_todo,struct mvneta_rx_queue * rxq)2538 static int mvneta_rx_hwbm(struct napi_struct *napi,
2539 			  struct mvneta_port *pp, int rx_todo,
2540 			  struct mvneta_rx_queue *rxq)
2541 {
2542 	struct net_device *dev = pp->dev;
2543 	int rx_done;
2544 	u32 rcvd_pkts = 0;
2545 	u32 rcvd_bytes = 0;
2546 
2547 	/* Get number of received packets */
2548 	rx_done = mvneta_rxq_busy_desc_num_get(pp, rxq);
2549 
2550 	if (rx_todo > rx_done)
2551 		rx_todo = rx_done;
2552 
2553 	rx_done = 0;
2554 
2555 	/* Fairness NAPI loop */
2556 	while (rx_done < rx_todo) {
2557 		struct mvneta_rx_desc *rx_desc = mvneta_rxq_next_desc_get(rxq);
2558 		struct mvneta_bm_pool *bm_pool = NULL;
2559 		struct sk_buff *skb;
2560 		unsigned char *data;
2561 		dma_addr_t phys_addr;
2562 		u32 rx_status, frag_size;
2563 		int rx_bytes, err;
2564 		u8 pool_id;
2565 
2566 		rx_done++;
2567 		rx_status = rx_desc->status;
2568 		rx_bytes = rx_desc->data_size - (ETH_FCS_LEN + MVNETA_MH_SIZE);
2569 		data = (u8 *)(uintptr_t)rx_desc->buf_cookie;
2570 		phys_addr = rx_desc->buf_phys_addr;
2571 		pool_id = MVNETA_RX_GET_BM_POOL_ID(rx_desc);
2572 		bm_pool = &pp->bm_priv->bm_pools[pool_id];
2573 
2574 		if (!mvneta_rxq_desc_is_first_last(rx_status) ||
2575 		    (rx_status & MVNETA_RXD_ERR_SUMMARY)) {
2576 err_drop_frame_ret_pool:
2577 			/* Return the buffer to the pool */
2578 			mvneta_bm_pool_put_bp(pp->bm_priv, bm_pool,
2579 					      rx_desc->buf_phys_addr);
2580 err_drop_frame:
2581 			mvneta_rx_error(pp, rx_desc);
2582 			/* leave the descriptor untouched */
2583 			continue;
2584 		}
2585 
2586 		if (rx_bytes <= rx_copybreak) {
2587 			/* better copy a small frame and not unmap the DMA region */
2588 			skb = netdev_alloc_skb_ip_align(dev, rx_bytes);
2589 			if (unlikely(!skb))
2590 				goto err_drop_frame_ret_pool;
2591 
2592 			dma_sync_single_range_for_cpu(&pp->bm_priv->pdev->dev,
2593 			                              rx_desc->buf_phys_addr,
2594 			                              MVNETA_MH_SIZE + NET_SKB_PAD,
2595 			                              rx_bytes,
2596 			                              DMA_FROM_DEVICE);
2597 			skb_put_data(skb, data + MVNETA_MH_SIZE + NET_SKB_PAD,
2598 				     rx_bytes);
2599 
2600 			skb->protocol = eth_type_trans(skb, dev);
2601 			skb->ip_summed = mvneta_rx_csum(pp, rx_status);
2602 			napi_gro_receive(napi, skb);
2603 
2604 			rcvd_pkts++;
2605 			rcvd_bytes += rx_bytes;
2606 
2607 			/* Return the buffer to the pool */
2608 			mvneta_bm_pool_put_bp(pp->bm_priv, bm_pool,
2609 					      rx_desc->buf_phys_addr);
2610 
2611 			/* leave the descriptor and buffer untouched */
2612 			continue;
2613 		}
2614 
2615 		/* Refill processing */
2616 		err = hwbm_pool_refill(&bm_pool->hwbm_pool, GFP_ATOMIC);
2617 		if (err) {
2618 			struct mvneta_pcpu_stats *stats;
2619 
2620 			netdev_err(dev, "Linux processing - Can't refill\n");
2621 
2622 			stats = this_cpu_ptr(pp->stats);
2623 			u64_stats_update_begin(&stats->syncp);
2624 			stats->es.refill_error++;
2625 			u64_stats_update_end(&stats->syncp);
2626 
2627 			goto err_drop_frame_ret_pool;
2628 		}
2629 
2630 		frag_size = bm_pool->hwbm_pool.frag_size;
2631 
2632 		skb = build_skb(data, frag_size > PAGE_SIZE ? 0 : frag_size);
2633 
2634 		/* After refill old buffer has to be unmapped regardless
2635 		 * the skb is successfully built or not.
2636 		 */
2637 		dma_unmap_single(&pp->bm_priv->pdev->dev, phys_addr,
2638 				 bm_pool->buf_size, DMA_FROM_DEVICE);
2639 		if (!skb)
2640 			goto err_drop_frame;
2641 
2642 		rcvd_pkts++;
2643 		rcvd_bytes += rx_bytes;
2644 
2645 		/* Linux processing */
2646 		skb_reserve(skb, MVNETA_MH_SIZE + NET_SKB_PAD);
2647 		skb_put(skb, rx_bytes);
2648 
2649 		skb->protocol = eth_type_trans(skb, dev);
2650 		skb->ip_summed = mvneta_rx_csum(pp, rx_status);
2651 
2652 		napi_gro_receive(napi, skb);
2653 	}
2654 
2655 	if (rcvd_pkts) {
2656 		struct mvneta_pcpu_stats *stats = this_cpu_ptr(pp->stats);
2657 
2658 		u64_stats_update_begin(&stats->syncp);
2659 		stats->es.ps.rx_packets += rcvd_pkts;
2660 		stats->es.ps.rx_bytes += rcvd_bytes;
2661 		u64_stats_update_end(&stats->syncp);
2662 	}
2663 
2664 	/* Update rxq management counters */
2665 	mvneta_rxq_desc_num_update(pp, rxq, rx_done, rx_done);
2666 
2667 	return rx_done;
2668 }
2669 
mvneta_free_tso_hdrs(struct mvneta_port * pp,struct mvneta_tx_queue * txq)2670 static void mvneta_free_tso_hdrs(struct mvneta_port *pp,
2671 				 struct mvneta_tx_queue *txq)
2672 {
2673 	struct device *dev = pp->dev->dev.parent;
2674 	int i;
2675 
2676 	for (i = 0; i < MVNETA_MAX_TSO_PAGES; i++) {
2677 		if (txq->tso_hdrs[i]) {
2678 			dma_free_coherent(dev, MVNETA_TSO_PAGE_SIZE,
2679 					  txq->tso_hdrs[i],
2680 					  txq->tso_hdrs_phys[i]);
2681 			txq->tso_hdrs[i] = NULL;
2682 		}
2683 	}
2684 }
2685 
mvneta_alloc_tso_hdrs(struct mvneta_port * pp,struct mvneta_tx_queue * txq)2686 static int mvneta_alloc_tso_hdrs(struct mvneta_port *pp,
2687 				 struct mvneta_tx_queue *txq)
2688 {
2689 	struct device *dev = pp->dev->dev.parent;
2690 	int i, num;
2691 
2692 	num = DIV_ROUND_UP(txq->size, MVNETA_TSO_PER_PAGE);
2693 	for (i = 0; i < num; i++) {
2694 		txq->tso_hdrs[i] = dma_alloc_coherent(dev, MVNETA_TSO_PAGE_SIZE,
2695 						      &txq->tso_hdrs_phys[i],
2696 						      GFP_KERNEL);
2697 		if (!txq->tso_hdrs[i]) {
2698 			mvneta_free_tso_hdrs(pp, txq);
2699 			return -ENOMEM;
2700 		}
2701 	}
2702 
2703 	return 0;
2704 }
2705 
mvneta_get_tso_hdr(struct mvneta_tx_queue * txq,dma_addr_t * dma)2706 static char *mvneta_get_tso_hdr(struct mvneta_tx_queue *txq, dma_addr_t *dma)
2707 {
2708 	int index, offset;
2709 
2710 	index = txq->txq_put_index / MVNETA_TSO_PER_PAGE;
2711 	offset = (txq->txq_put_index % MVNETA_TSO_PER_PAGE) * TSO_HEADER_SIZE;
2712 
2713 	*dma = txq->tso_hdrs_phys[index] + offset;
2714 
2715 	return txq->tso_hdrs[index] + offset;
2716 }
2717 
mvneta_tso_put_hdr(struct sk_buff * skb,struct mvneta_tx_queue * txq,struct tso_t * tso,int size,bool is_last)2718 static void mvneta_tso_put_hdr(struct sk_buff *skb, struct mvneta_tx_queue *txq,
2719 			       struct tso_t *tso, int size, bool is_last)
2720 {
2721 	struct mvneta_tx_buf *buf = &txq->buf[txq->txq_put_index];
2722 	int hdr_len = skb_tcp_all_headers(skb);
2723 	struct mvneta_tx_desc *tx_desc;
2724 	dma_addr_t hdr_phys;
2725 	char *hdr;
2726 
2727 	hdr = mvneta_get_tso_hdr(txq, &hdr_phys);
2728 	tso_build_hdr(skb, hdr, tso, size, is_last);
2729 
2730 	tx_desc = mvneta_txq_next_desc_get(txq);
2731 	tx_desc->data_size = hdr_len;
2732 	tx_desc->command = mvneta_skb_tx_csum(skb);
2733 	tx_desc->command |= MVNETA_TXD_F_DESC;
2734 	tx_desc->buf_phys_addr = hdr_phys;
2735 	buf->type = MVNETA_TYPE_TSO;
2736 	buf->skb = NULL;
2737 
2738 	mvneta_txq_inc_put(txq);
2739 }
2740 
2741 static inline int
mvneta_tso_put_data(struct net_device * dev,struct mvneta_tx_queue * txq,struct sk_buff * skb,char * data,int size,bool last_tcp,bool is_last)2742 mvneta_tso_put_data(struct net_device *dev, struct mvneta_tx_queue *txq,
2743 		    struct sk_buff *skb, char *data, int size,
2744 		    bool last_tcp, bool is_last)
2745 {
2746 	struct mvneta_tx_buf *buf = &txq->buf[txq->txq_put_index];
2747 	struct mvneta_tx_desc *tx_desc;
2748 
2749 	tx_desc = mvneta_txq_next_desc_get(txq);
2750 	tx_desc->data_size = size;
2751 	tx_desc->buf_phys_addr = dma_map_single(dev->dev.parent, data,
2752 						size, DMA_TO_DEVICE);
2753 	if (unlikely(dma_mapping_error(dev->dev.parent,
2754 		     tx_desc->buf_phys_addr))) {
2755 		mvneta_txq_desc_put(txq);
2756 		return -ENOMEM;
2757 	}
2758 
2759 	tx_desc->command = 0;
2760 	buf->type = MVNETA_TYPE_SKB;
2761 	buf->skb = NULL;
2762 
2763 	if (last_tcp) {
2764 		/* last descriptor in the TCP packet */
2765 		tx_desc->command = MVNETA_TXD_L_DESC;
2766 
2767 		/* last descriptor in SKB */
2768 		if (is_last)
2769 			buf->skb = skb;
2770 	}
2771 	mvneta_txq_inc_put(txq);
2772 	return 0;
2773 }
2774 
mvneta_release_descs(struct mvneta_port * pp,struct mvneta_tx_queue * txq,int first,int num)2775 static void mvneta_release_descs(struct mvneta_port *pp,
2776 				 struct mvneta_tx_queue *txq,
2777 				 int first, int num)
2778 {
2779 	int desc_idx, i;
2780 
2781 	desc_idx = first + num;
2782 	if (desc_idx >= txq->size)
2783 		desc_idx -= txq->size;
2784 
2785 	for (i = num; i >= 0; i--) {
2786 		struct mvneta_tx_desc *tx_desc = txq->descs + desc_idx;
2787 		struct mvneta_tx_buf *buf = &txq->buf[desc_idx];
2788 
2789 		if (buf->type == MVNETA_TYPE_SKB)
2790 			dma_unmap_single(pp->dev->dev.parent,
2791 					 tx_desc->buf_phys_addr,
2792 					 tx_desc->data_size,
2793 					 DMA_TO_DEVICE);
2794 
2795 		mvneta_txq_desc_put(txq);
2796 
2797 		if (desc_idx == 0)
2798 			desc_idx = txq->size;
2799 		desc_idx -= 1;
2800 	}
2801 }
2802 
mvneta_tx_tso(struct sk_buff * skb,struct net_device * dev,struct mvneta_tx_queue * txq)2803 static int mvneta_tx_tso(struct sk_buff *skb, struct net_device *dev,
2804 			 struct mvneta_tx_queue *txq)
2805 {
2806 	int hdr_len, total_len, data_left;
2807 	int first_desc, desc_count = 0;
2808 	struct mvneta_port *pp = netdev_priv(dev);
2809 	struct tso_t tso;
2810 
2811 	/* Count needed descriptors */
2812 	if ((txq->count + tso_count_descs(skb)) >= txq->size)
2813 		return 0;
2814 
2815 	if (skb_headlen(skb) < skb_tcp_all_headers(skb)) {
2816 		pr_info("*** Is this even possible?\n");
2817 		return 0;
2818 	}
2819 
2820 	first_desc = txq->txq_put_index;
2821 
2822 	/* Initialize the TSO handler, and prepare the first payload */
2823 	hdr_len = tso_start(skb, &tso);
2824 
2825 	total_len = skb->len - hdr_len;
2826 	while (total_len > 0) {
2827 		data_left = min_t(int, skb_shinfo(skb)->gso_size, total_len);
2828 		total_len -= data_left;
2829 		desc_count++;
2830 
2831 		/* prepare packet headers: MAC + IP + TCP */
2832 		mvneta_tso_put_hdr(skb, txq, &tso, data_left, total_len == 0);
2833 
2834 		while (data_left > 0) {
2835 			int size;
2836 			desc_count++;
2837 
2838 			size = min_t(int, tso.size, data_left);
2839 
2840 			if (mvneta_tso_put_data(dev, txq, skb,
2841 						 tso.data, size,
2842 						 size == data_left,
2843 						 total_len == 0))
2844 				goto err_release;
2845 			data_left -= size;
2846 
2847 			tso_build_data(skb, &tso, size);
2848 		}
2849 	}
2850 
2851 	return desc_count;
2852 
2853 err_release:
2854 	/* Release all used data descriptors; header descriptors must not
2855 	 * be DMA-unmapped.
2856 	 */
2857 	mvneta_release_descs(pp, txq, first_desc, desc_count - 1);
2858 	return 0;
2859 }
2860 
2861 /* Handle tx fragmentation processing */
mvneta_tx_frag_process(struct mvneta_port * pp,struct sk_buff * skb,struct mvneta_tx_queue * txq)2862 static int mvneta_tx_frag_process(struct mvneta_port *pp, struct sk_buff *skb,
2863 				  struct mvneta_tx_queue *txq)
2864 {
2865 	struct mvneta_tx_desc *tx_desc;
2866 	int i, nr_frags = skb_shinfo(skb)->nr_frags;
2867 	int first_desc = txq->txq_put_index;
2868 
2869 	for (i = 0; i < nr_frags; i++) {
2870 		struct mvneta_tx_buf *buf = &txq->buf[txq->txq_put_index];
2871 		skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
2872 		void *addr = skb_frag_address(frag);
2873 
2874 		tx_desc = mvneta_txq_next_desc_get(txq);
2875 		tx_desc->data_size = skb_frag_size(frag);
2876 
2877 		tx_desc->buf_phys_addr =
2878 			dma_map_single(pp->dev->dev.parent, addr,
2879 				       tx_desc->data_size, DMA_TO_DEVICE);
2880 
2881 		if (dma_mapping_error(pp->dev->dev.parent,
2882 				      tx_desc->buf_phys_addr)) {
2883 			mvneta_txq_desc_put(txq);
2884 			goto error;
2885 		}
2886 
2887 		if (i == nr_frags - 1) {
2888 			/* Last descriptor */
2889 			tx_desc->command = MVNETA_TXD_L_DESC | MVNETA_TXD_Z_PAD;
2890 			buf->skb = skb;
2891 		} else {
2892 			/* Descriptor in the middle: Not First, Not Last */
2893 			tx_desc->command = 0;
2894 			buf->skb = NULL;
2895 		}
2896 		buf->type = MVNETA_TYPE_SKB;
2897 		mvneta_txq_inc_put(txq);
2898 	}
2899 
2900 	return 0;
2901 
2902 error:
2903 	/* Release all descriptors that were used to map fragments of
2904 	 * this packet, as well as the corresponding DMA mappings
2905 	 */
2906 	mvneta_release_descs(pp, txq, first_desc, i - 1);
2907 	return -ENOMEM;
2908 }
2909 
2910 /* Main tx processing */
mvneta_tx(struct sk_buff * skb,struct net_device * dev)2911 static netdev_tx_t mvneta_tx(struct sk_buff *skb, struct net_device *dev)
2912 {
2913 	struct mvneta_port *pp = netdev_priv(dev);
2914 	u16 txq_id = skb_get_queue_mapping(skb);
2915 	struct mvneta_tx_queue *txq = &pp->txqs[txq_id];
2916 	struct mvneta_tx_buf *buf = &txq->buf[txq->txq_put_index];
2917 	struct mvneta_tx_desc *tx_desc;
2918 	int len = skb->len;
2919 	int frags = 0;
2920 	u32 tx_cmd;
2921 
2922 	if (!netif_running(dev))
2923 		goto out;
2924 
2925 	if (skb_is_gso(skb)) {
2926 		frags = mvneta_tx_tso(skb, dev, txq);
2927 		goto out;
2928 	}
2929 
2930 	frags = skb_shinfo(skb)->nr_frags + 1;
2931 
2932 	/* Get a descriptor for the first part of the packet */
2933 	tx_desc = mvneta_txq_next_desc_get(txq);
2934 
2935 	tx_cmd = mvneta_skb_tx_csum(skb);
2936 
2937 	tx_desc->data_size = skb_headlen(skb);
2938 
2939 	tx_desc->buf_phys_addr = dma_map_single(dev->dev.parent, skb->data,
2940 						tx_desc->data_size,
2941 						DMA_TO_DEVICE);
2942 	if (unlikely(dma_mapping_error(dev->dev.parent,
2943 				       tx_desc->buf_phys_addr))) {
2944 		mvneta_txq_desc_put(txq);
2945 		frags = 0;
2946 		goto out;
2947 	}
2948 
2949 	buf->type = MVNETA_TYPE_SKB;
2950 	if (frags == 1) {
2951 		/* First and Last descriptor */
2952 		tx_cmd |= MVNETA_TXD_FLZ_DESC;
2953 		tx_desc->command = tx_cmd;
2954 		buf->skb = skb;
2955 		mvneta_txq_inc_put(txq);
2956 	} else {
2957 		/* First but not Last */
2958 		tx_cmd |= MVNETA_TXD_F_DESC;
2959 		buf->skb = NULL;
2960 		mvneta_txq_inc_put(txq);
2961 		tx_desc->command = tx_cmd;
2962 		/* Continue with other skb fragments */
2963 		if (mvneta_tx_frag_process(pp, skb, txq)) {
2964 			dma_unmap_single(dev->dev.parent,
2965 					 tx_desc->buf_phys_addr,
2966 					 tx_desc->data_size,
2967 					 DMA_TO_DEVICE);
2968 			mvneta_txq_desc_put(txq);
2969 			frags = 0;
2970 			goto out;
2971 		}
2972 	}
2973 
2974 out:
2975 	if (frags > 0) {
2976 		struct netdev_queue *nq = netdev_get_tx_queue(dev, txq_id);
2977 		struct mvneta_pcpu_stats *stats = this_cpu_ptr(pp->stats);
2978 
2979 		netdev_tx_sent_queue(nq, len);
2980 
2981 		txq->count += frags;
2982 		if (txq->count >= txq->tx_stop_threshold)
2983 			netif_tx_stop_queue(nq);
2984 
2985 		if (!netdev_xmit_more() || netif_xmit_stopped(nq) ||
2986 		    txq->pending + frags > MVNETA_TXQ_DEC_SENT_MASK)
2987 			mvneta_txq_pend_desc_add(pp, txq, frags);
2988 		else
2989 			txq->pending += frags;
2990 
2991 		u64_stats_update_begin(&stats->syncp);
2992 		stats->es.ps.tx_bytes += len;
2993 		stats->es.ps.tx_packets++;
2994 		u64_stats_update_end(&stats->syncp);
2995 	} else {
2996 		dev->stats.tx_dropped++;
2997 		dev_kfree_skb_any(skb);
2998 	}
2999 
3000 	return NETDEV_TX_OK;
3001 }
3002 
3003 
3004 /* Free tx resources, when resetting a port */
mvneta_txq_done_force(struct mvneta_port * pp,struct mvneta_tx_queue * txq)3005 static void mvneta_txq_done_force(struct mvneta_port *pp,
3006 				  struct mvneta_tx_queue *txq)
3007 
3008 {
3009 	struct netdev_queue *nq = netdev_get_tx_queue(pp->dev, txq->id);
3010 	int tx_done = txq->count;
3011 
3012 	mvneta_txq_bufs_free(pp, txq, tx_done, nq, false);
3013 
3014 	/* reset txq */
3015 	txq->count = 0;
3016 	txq->txq_put_index = 0;
3017 	txq->txq_get_index = 0;
3018 }
3019 
3020 /* Handle tx done - called in softirq context. The <cause_tx_done> argument
3021  * must be a valid cause according to MVNETA_TXQ_INTR_MASK_ALL.
3022  */
mvneta_tx_done_gbe(struct mvneta_port * pp,u32 cause_tx_done)3023 static void mvneta_tx_done_gbe(struct mvneta_port *pp, u32 cause_tx_done)
3024 {
3025 	struct mvneta_tx_queue *txq;
3026 	struct netdev_queue *nq;
3027 	int cpu = smp_processor_id();
3028 
3029 	while (cause_tx_done) {
3030 		txq = mvneta_tx_done_policy(pp, cause_tx_done);
3031 
3032 		nq = netdev_get_tx_queue(pp->dev, txq->id);
3033 		__netif_tx_lock(nq, cpu);
3034 
3035 		if (txq->count)
3036 			mvneta_txq_done(pp, txq);
3037 
3038 		__netif_tx_unlock(nq);
3039 		cause_tx_done &= ~((1 << txq->id));
3040 	}
3041 }
3042 
3043 /* Compute crc8 of the specified address, using a unique algorithm ,
3044  * according to hw spec, different than generic crc8 algorithm
3045  */
mvneta_addr_crc(unsigned char * addr)3046 static int mvneta_addr_crc(unsigned char *addr)
3047 {
3048 	int crc = 0;
3049 	int i;
3050 
3051 	for (i = 0; i < ETH_ALEN; i++) {
3052 		int j;
3053 
3054 		crc = (crc ^ addr[i]) << 8;
3055 		for (j = 7; j >= 0; j--) {
3056 			if (crc & (0x100 << j))
3057 				crc ^= 0x107 << j;
3058 		}
3059 	}
3060 
3061 	return crc;
3062 }
3063 
3064 /* This method controls the net device special MAC multicast support.
3065  * The Special Multicast Table for MAC addresses supports MAC of the form
3066  * 0x01-00-5E-00-00-XX (where XX is between 0x00 and 0xFF).
3067  * The MAC DA[7:0] bits are used as a pointer to the Special Multicast
3068  * Table entries in the DA-Filter table. This method set the Special
3069  * Multicast Table appropriate entry.
3070  */
mvneta_set_special_mcast_addr(struct mvneta_port * pp,unsigned char last_byte,int queue)3071 static void mvneta_set_special_mcast_addr(struct mvneta_port *pp,
3072 					  unsigned char last_byte,
3073 					  int queue)
3074 {
3075 	unsigned int smc_table_reg;
3076 	unsigned int tbl_offset;
3077 	unsigned int reg_offset;
3078 
3079 	/* Register offset from SMC table base    */
3080 	tbl_offset = (last_byte / 4);
3081 	/* Entry offset within the above reg */
3082 	reg_offset = last_byte % 4;
3083 
3084 	smc_table_reg = mvreg_read(pp, (MVNETA_DA_FILT_SPEC_MCAST
3085 					+ tbl_offset * 4));
3086 
3087 	if (queue == -1)
3088 		smc_table_reg &= ~(0xff << (8 * reg_offset));
3089 	else {
3090 		smc_table_reg &= ~(0xff << (8 * reg_offset));
3091 		smc_table_reg |= ((0x01 | (queue << 1)) << (8 * reg_offset));
3092 	}
3093 
3094 	mvreg_write(pp, MVNETA_DA_FILT_SPEC_MCAST + tbl_offset * 4,
3095 		    smc_table_reg);
3096 }
3097 
3098 /* This method controls the network device Other MAC multicast support.
3099  * The Other Multicast Table is used for multicast of another type.
3100  * A CRC-8 is used as an index to the Other Multicast Table entries
3101  * in the DA-Filter table.
3102  * The method gets the CRC-8 value from the calling routine and
3103  * sets the Other Multicast Table appropriate entry according to the
3104  * specified CRC-8 .
3105  */
mvneta_set_other_mcast_addr(struct mvneta_port * pp,unsigned char crc8,int queue)3106 static void mvneta_set_other_mcast_addr(struct mvneta_port *pp,
3107 					unsigned char crc8,
3108 					int queue)
3109 {
3110 	unsigned int omc_table_reg;
3111 	unsigned int tbl_offset;
3112 	unsigned int reg_offset;
3113 
3114 	tbl_offset = (crc8 / 4) * 4; /* Register offset from OMC table base */
3115 	reg_offset = crc8 % 4;	     /* Entry offset within the above reg   */
3116 
3117 	omc_table_reg = mvreg_read(pp, MVNETA_DA_FILT_OTH_MCAST + tbl_offset);
3118 
3119 	if (queue == -1) {
3120 		/* Clear accepts frame bit at specified Other DA table entry */
3121 		omc_table_reg &= ~(0xff << (8 * reg_offset));
3122 	} else {
3123 		omc_table_reg &= ~(0xff << (8 * reg_offset));
3124 		omc_table_reg |= ((0x01 | (queue << 1)) << (8 * reg_offset));
3125 	}
3126 
3127 	mvreg_write(pp, MVNETA_DA_FILT_OTH_MCAST + tbl_offset, omc_table_reg);
3128 }
3129 
3130 /* The network device supports multicast using two tables:
3131  *    1) Special Multicast Table for MAC addresses of the form
3132  *       0x01-00-5E-00-00-XX (where XX is between 0x00 and 0xFF).
3133  *       The MAC DA[7:0] bits are used as a pointer to the Special Multicast
3134  *       Table entries in the DA-Filter table.
3135  *    2) Other Multicast Table for multicast of another type. A CRC-8 value
3136  *       is used as an index to the Other Multicast Table entries in the
3137  *       DA-Filter table.
3138  */
mvneta_mcast_addr_set(struct mvneta_port * pp,unsigned char * p_addr,int queue)3139 static int mvneta_mcast_addr_set(struct mvneta_port *pp, unsigned char *p_addr,
3140 				 int queue)
3141 {
3142 	unsigned char crc_result = 0;
3143 
3144 	if (memcmp(p_addr, "\x01\x00\x5e\x00\x00", 5) == 0) {
3145 		mvneta_set_special_mcast_addr(pp, p_addr[5], queue);
3146 		return 0;
3147 	}
3148 
3149 	crc_result = mvneta_addr_crc(p_addr);
3150 	if (queue == -1) {
3151 		if (pp->mcast_count[crc_result] == 0) {
3152 			netdev_info(pp->dev, "No valid Mcast for crc8=0x%02x\n",
3153 				    crc_result);
3154 			return -EINVAL;
3155 		}
3156 
3157 		pp->mcast_count[crc_result]--;
3158 		if (pp->mcast_count[crc_result] != 0) {
3159 			netdev_info(pp->dev,
3160 				    "After delete there are %d valid Mcast for crc8=0x%02x\n",
3161 				    pp->mcast_count[crc_result], crc_result);
3162 			return -EINVAL;
3163 		}
3164 	} else
3165 		pp->mcast_count[crc_result]++;
3166 
3167 	mvneta_set_other_mcast_addr(pp, crc_result, queue);
3168 
3169 	return 0;
3170 }
3171 
3172 /* Configure Fitering mode of Ethernet port */
mvneta_rx_unicast_promisc_set(struct mvneta_port * pp,int is_promisc)3173 static void mvneta_rx_unicast_promisc_set(struct mvneta_port *pp,
3174 					  int is_promisc)
3175 {
3176 	u32 port_cfg_reg, val;
3177 
3178 	port_cfg_reg = mvreg_read(pp, MVNETA_PORT_CONFIG);
3179 
3180 	val = mvreg_read(pp, MVNETA_TYPE_PRIO);
3181 
3182 	/* Set / Clear UPM bit in port configuration register */
3183 	if (is_promisc) {
3184 		/* Accept all Unicast addresses */
3185 		port_cfg_reg |= MVNETA_UNI_PROMISC_MODE;
3186 		val |= MVNETA_FORCE_UNI;
3187 		mvreg_write(pp, MVNETA_MAC_ADDR_LOW, 0xffff);
3188 		mvreg_write(pp, MVNETA_MAC_ADDR_HIGH, 0xffffffff);
3189 	} else {
3190 		/* Reject all Unicast addresses */
3191 		port_cfg_reg &= ~MVNETA_UNI_PROMISC_MODE;
3192 		val &= ~MVNETA_FORCE_UNI;
3193 	}
3194 
3195 	mvreg_write(pp, MVNETA_PORT_CONFIG, port_cfg_reg);
3196 	mvreg_write(pp, MVNETA_TYPE_PRIO, val);
3197 }
3198 
3199 /* register unicast and multicast addresses */
mvneta_set_rx_mode(struct net_device * dev)3200 static void mvneta_set_rx_mode(struct net_device *dev)
3201 {
3202 	struct mvneta_port *pp = netdev_priv(dev);
3203 	struct netdev_hw_addr *ha;
3204 
3205 	if (dev->flags & IFF_PROMISC) {
3206 		/* Accept all: Multicast + Unicast */
3207 		mvneta_rx_unicast_promisc_set(pp, 1);
3208 		mvneta_set_ucast_table(pp, pp->rxq_def);
3209 		mvneta_set_special_mcast_table(pp, pp->rxq_def);
3210 		mvneta_set_other_mcast_table(pp, pp->rxq_def);
3211 	} else {
3212 		/* Accept single Unicast */
3213 		mvneta_rx_unicast_promisc_set(pp, 0);
3214 		mvneta_set_ucast_table(pp, -1);
3215 		mvneta_mac_addr_set(pp, dev->dev_addr, pp->rxq_def);
3216 
3217 		if (dev->flags & IFF_ALLMULTI) {
3218 			/* Accept all multicast */
3219 			mvneta_set_special_mcast_table(pp, pp->rxq_def);
3220 			mvneta_set_other_mcast_table(pp, pp->rxq_def);
3221 		} else {
3222 			/* Accept only initialized multicast */
3223 			mvneta_set_special_mcast_table(pp, -1);
3224 			mvneta_set_other_mcast_table(pp, -1);
3225 
3226 			if (!netdev_mc_empty(dev)) {
3227 				netdev_for_each_mc_addr(ha, dev) {
3228 					mvneta_mcast_addr_set(pp, ha->addr,
3229 							      pp->rxq_def);
3230 				}
3231 			}
3232 		}
3233 	}
3234 }
3235 
3236 /* Interrupt handling - the callback for request_irq() */
mvneta_isr(int irq,void * dev_id)3237 static irqreturn_t mvneta_isr(int irq, void *dev_id)
3238 {
3239 	struct mvneta_port *pp = (struct mvneta_port *)dev_id;
3240 
3241 	mvreg_write(pp, MVNETA_INTR_NEW_MASK, 0);
3242 	napi_schedule(&pp->napi);
3243 
3244 	return IRQ_HANDLED;
3245 }
3246 
3247 /* Interrupt handling - the callback for request_percpu_irq() */
mvneta_percpu_isr(int irq,void * dev_id)3248 static irqreturn_t mvneta_percpu_isr(int irq, void *dev_id)
3249 {
3250 	struct mvneta_pcpu_port *port = (struct mvneta_pcpu_port *)dev_id;
3251 
3252 	disable_percpu_irq(port->pp->dev->irq);
3253 	napi_schedule(&port->napi);
3254 
3255 	return IRQ_HANDLED;
3256 }
3257 
mvneta_link_change(struct mvneta_port * pp)3258 static void mvneta_link_change(struct mvneta_port *pp)
3259 {
3260 	u32 gmac_stat = mvreg_read(pp, MVNETA_GMAC_STATUS);
3261 
3262 	phylink_pcs_change(&pp->phylink_pcs,
3263 			   !!(gmac_stat & MVNETA_GMAC_LINK_UP));
3264 }
3265 
3266 /* NAPI handler
3267  * Bits 0 - 7 of the causeRxTx register indicate that are transmitted
3268  * packets on the corresponding TXQ (Bit 0 is for TX queue 1).
3269  * Bits 8 -15 of the cause Rx Tx register indicate that are received
3270  * packets on the corresponding RXQ (Bit 8 is for RX queue 0).
3271  * Each CPU has its own causeRxTx register
3272  */
mvneta_poll(struct napi_struct * napi,int budget)3273 static int mvneta_poll(struct napi_struct *napi, int budget)
3274 {
3275 	int rx_done = 0;
3276 	u32 cause_rx_tx;
3277 	int rx_queue;
3278 	struct mvneta_port *pp = netdev_priv(napi->dev);
3279 	struct mvneta_pcpu_port *port = this_cpu_ptr(pp->ports);
3280 
3281 	if (!netif_running(pp->dev)) {
3282 		napi_complete(napi);
3283 		return rx_done;
3284 	}
3285 
3286 	/* Read cause register */
3287 	cause_rx_tx = mvreg_read(pp, MVNETA_INTR_NEW_CAUSE);
3288 	if (cause_rx_tx & MVNETA_MISCINTR_INTR_MASK) {
3289 		u32 cause_misc = mvreg_read(pp, MVNETA_INTR_MISC_CAUSE);
3290 
3291 		mvreg_write(pp, MVNETA_INTR_MISC_CAUSE, 0);
3292 
3293 		if (cause_misc & (MVNETA_CAUSE_PHY_STATUS_CHANGE |
3294 				  MVNETA_CAUSE_LINK_CHANGE))
3295 			mvneta_link_change(pp);
3296 	}
3297 
3298 	/* Release Tx descriptors */
3299 	if (cause_rx_tx & MVNETA_TX_INTR_MASK_ALL) {
3300 		mvneta_tx_done_gbe(pp, (cause_rx_tx & MVNETA_TX_INTR_MASK_ALL));
3301 		cause_rx_tx &= ~MVNETA_TX_INTR_MASK_ALL;
3302 	}
3303 
3304 	/* For the case where the last mvneta_poll did not process all
3305 	 * RX packets
3306 	 */
3307 	cause_rx_tx |= pp->neta_armada3700 ? pp->cause_rx_tx :
3308 		port->cause_rx_tx;
3309 
3310 	rx_queue = fls(((cause_rx_tx >> 8) & 0xff));
3311 	if (rx_queue) {
3312 		rx_queue = rx_queue - 1;
3313 		if (pp->bm_priv)
3314 			rx_done = mvneta_rx_hwbm(napi, pp, budget,
3315 						 &pp->rxqs[rx_queue]);
3316 		else
3317 			rx_done = mvneta_rx_swbm(napi, pp, budget,
3318 						 &pp->rxqs[rx_queue]);
3319 	}
3320 
3321 	if (rx_done < budget) {
3322 		cause_rx_tx = 0;
3323 		napi_complete_done(napi, rx_done);
3324 
3325 		if (pp->neta_armada3700) {
3326 			unsigned long flags;
3327 
3328 			local_irq_save(flags);
3329 			mvreg_write(pp, MVNETA_INTR_NEW_MASK,
3330 				    MVNETA_RX_INTR_MASK(rxq_number) |
3331 				    MVNETA_TX_INTR_MASK(txq_number) |
3332 				    MVNETA_MISCINTR_INTR_MASK);
3333 			local_irq_restore(flags);
3334 		} else {
3335 			enable_percpu_irq(pp->dev->irq, 0);
3336 		}
3337 	}
3338 
3339 	if (pp->neta_armada3700)
3340 		pp->cause_rx_tx = cause_rx_tx;
3341 	else
3342 		port->cause_rx_tx = cause_rx_tx;
3343 
3344 	return rx_done;
3345 }
3346 
mvneta_create_page_pool(struct mvneta_port * pp,struct mvneta_rx_queue * rxq,int size)3347 static int mvneta_create_page_pool(struct mvneta_port *pp,
3348 				   struct mvneta_rx_queue *rxq, int size)
3349 {
3350 	struct bpf_prog *xdp_prog = READ_ONCE(pp->xdp_prog);
3351 	struct page_pool_params pp_params = {
3352 		.order = 0,
3353 		.flags = PP_FLAG_DMA_MAP | PP_FLAG_DMA_SYNC_DEV,
3354 		.pool_size = size,
3355 		.nid = NUMA_NO_NODE,
3356 		.dev = pp->dev->dev.parent,
3357 		.dma_dir = xdp_prog ? DMA_BIDIRECTIONAL : DMA_FROM_DEVICE,
3358 		.offset = pp->rx_offset_correction,
3359 		.max_len = MVNETA_MAX_RX_BUF_SIZE,
3360 	};
3361 	int err;
3362 
3363 	rxq->page_pool = page_pool_create(&pp_params);
3364 	if (IS_ERR(rxq->page_pool)) {
3365 		err = PTR_ERR(rxq->page_pool);
3366 		rxq->page_pool = NULL;
3367 		return err;
3368 	}
3369 
3370 	err = __xdp_rxq_info_reg(&rxq->xdp_rxq, pp->dev, rxq->id, 0,
3371 				 PAGE_SIZE);
3372 	if (err < 0)
3373 		goto err_free_pp;
3374 
3375 	err = xdp_rxq_info_reg_mem_model(&rxq->xdp_rxq, MEM_TYPE_PAGE_POOL,
3376 					 rxq->page_pool);
3377 	if (err)
3378 		goto err_unregister_rxq;
3379 
3380 	return 0;
3381 
3382 err_unregister_rxq:
3383 	xdp_rxq_info_unreg(&rxq->xdp_rxq);
3384 err_free_pp:
3385 	page_pool_destroy(rxq->page_pool);
3386 	rxq->page_pool = NULL;
3387 	return err;
3388 }
3389 
3390 /* Handle rxq fill: allocates rxq skbs; called when initializing a port */
mvneta_rxq_fill(struct mvneta_port * pp,struct mvneta_rx_queue * rxq,int num)3391 static int mvneta_rxq_fill(struct mvneta_port *pp, struct mvneta_rx_queue *rxq,
3392 			   int num)
3393 {
3394 	int i, err;
3395 
3396 	err = mvneta_create_page_pool(pp, rxq, num);
3397 	if (err < 0)
3398 		return err;
3399 
3400 	for (i = 0; i < num; i++) {
3401 		memset(rxq->descs + i, 0, sizeof(struct mvneta_rx_desc));
3402 		if (mvneta_rx_refill(pp, rxq->descs + i, rxq,
3403 				     GFP_KERNEL) != 0) {
3404 			netdev_err(pp->dev,
3405 				   "%s:rxq %d, %d of %d buffs  filled\n",
3406 				   __func__, rxq->id, i, num);
3407 			break;
3408 		}
3409 	}
3410 
3411 	/* Add this number of RX descriptors as non occupied (ready to
3412 	 * get packets)
3413 	 */
3414 	mvneta_rxq_non_occup_desc_add(pp, rxq, i);
3415 
3416 	return i;
3417 }
3418 
3419 /* Free all packets pending transmit from all TXQs and reset TX port */
mvneta_tx_reset(struct mvneta_port * pp)3420 static void mvneta_tx_reset(struct mvneta_port *pp)
3421 {
3422 	int queue;
3423 
3424 	/* free the skb's in the tx ring */
3425 	for (queue = 0; queue < txq_number; queue++)
3426 		mvneta_txq_done_force(pp, &pp->txqs[queue]);
3427 
3428 	mvreg_write(pp, MVNETA_PORT_TX_RESET, MVNETA_PORT_TX_DMA_RESET);
3429 	mvreg_write(pp, MVNETA_PORT_TX_RESET, 0);
3430 }
3431 
mvneta_rx_reset(struct mvneta_port * pp)3432 static void mvneta_rx_reset(struct mvneta_port *pp)
3433 {
3434 	mvreg_write(pp, MVNETA_PORT_RX_RESET, MVNETA_PORT_RX_DMA_RESET);
3435 	mvreg_write(pp, MVNETA_PORT_RX_RESET, 0);
3436 }
3437 
3438 /* Rx/Tx queue initialization/cleanup methods */
3439 
mvneta_rxq_sw_init(struct mvneta_port * pp,struct mvneta_rx_queue * rxq)3440 static int mvneta_rxq_sw_init(struct mvneta_port *pp,
3441 			      struct mvneta_rx_queue *rxq)
3442 {
3443 	rxq->size = pp->rx_ring_size;
3444 
3445 	/* Allocate memory for RX descriptors */
3446 	rxq->descs = dma_alloc_coherent(pp->dev->dev.parent,
3447 					rxq->size * MVNETA_DESC_ALIGNED_SIZE,
3448 					&rxq->descs_phys, GFP_KERNEL);
3449 	if (!rxq->descs)
3450 		return -ENOMEM;
3451 
3452 	rxq->last_desc = rxq->size - 1;
3453 
3454 	return 0;
3455 }
3456 
mvneta_rxq_hw_init(struct mvneta_port * pp,struct mvneta_rx_queue * rxq)3457 static void mvneta_rxq_hw_init(struct mvneta_port *pp,
3458 			       struct mvneta_rx_queue *rxq)
3459 {
3460 	/* Set Rx descriptors queue starting address */
3461 	mvreg_write(pp, MVNETA_RXQ_BASE_ADDR_REG(rxq->id), rxq->descs_phys);
3462 	mvreg_write(pp, MVNETA_RXQ_SIZE_REG(rxq->id), rxq->size);
3463 
3464 	/* Set coalescing pkts and time */
3465 	mvneta_rx_pkts_coal_set(pp, rxq, rxq->pkts_coal);
3466 	mvneta_rx_time_coal_set(pp, rxq, rxq->time_coal);
3467 
3468 	if (!pp->bm_priv) {
3469 		/* Set Offset */
3470 		mvneta_rxq_offset_set(pp, rxq, 0);
3471 		mvneta_rxq_buf_size_set(pp, rxq, PAGE_SIZE < SZ_64K ?
3472 					MVNETA_MAX_RX_BUF_SIZE :
3473 					MVNETA_RX_BUF_SIZE(pp->pkt_size));
3474 		mvneta_rxq_bm_disable(pp, rxq);
3475 		mvneta_rxq_fill(pp, rxq, rxq->size);
3476 	} else {
3477 		/* Set Offset */
3478 		mvneta_rxq_offset_set(pp, rxq,
3479 				      NET_SKB_PAD - pp->rx_offset_correction);
3480 
3481 		mvneta_rxq_bm_enable(pp, rxq);
3482 		/* Fill RXQ with buffers from RX pool */
3483 		mvneta_rxq_long_pool_set(pp, rxq);
3484 		mvneta_rxq_short_pool_set(pp, rxq);
3485 		mvneta_rxq_non_occup_desc_add(pp, rxq, rxq->size);
3486 	}
3487 }
3488 
3489 /* Create a specified RX queue */
mvneta_rxq_init(struct mvneta_port * pp,struct mvneta_rx_queue * rxq)3490 static int mvneta_rxq_init(struct mvneta_port *pp,
3491 			   struct mvneta_rx_queue *rxq)
3492 
3493 {
3494 	int ret;
3495 
3496 	ret = mvneta_rxq_sw_init(pp, rxq);
3497 	if (ret < 0)
3498 		return ret;
3499 
3500 	mvneta_rxq_hw_init(pp, rxq);
3501 
3502 	return 0;
3503 }
3504 
3505 /* Cleanup Rx queue */
mvneta_rxq_deinit(struct mvneta_port * pp,struct mvneta_rx_queue * rxq)3506 static void mvneta_rxq_deinit(struct mvneta_port *pp,
3507 			      struct mvneta_rx_queue *rxq)
3508 {
3509 	mvneta_rxq_drop_pkts(pp, rxq);
3510 
3511 	if (rxq->descs)
3512 		dma_free_coherent(pp->dev->dev.parent,
3513 				  rxq->size * MVNETA_DESC_ALIGNED_SIZE,
3514 				  rxq->descs,
3515 				  rxq->descs_phys);
3516 
3517 	rxq->descs             = NULL;
3518 	rxq->last_desc         = 0;
3519 	rxq->next_desc_to_proc = 0;
3520 	rxq->descs_phys        = 0;
3521 	rxq->first_to_refill   = 0;
3522 	rxq->refill_num        = 0;
3523 }
3524 
mvneta_txq_sw_init(struct mvneta_port * pp,struct mvneta_tx_queue * txq)3525 static int mvneta_txq_sw_init(struct mvneta_port *pp,
3526 			      struct mvneta_tx_queue *txq)
3527 {
3528 	int cpu, err;
3529 
3530 	txq->size = pp->tx_ring_size;
3531 
3532 	/* A queue must always have room for at least one skb.
3533 	 * Therefore, stop the queue when the free entries reaches
3534 	 * the maximum number of descriptors per skb.
3535 	 */
3536 	txq->tx_stop_threshold = txq->size - MVNETA_MAX_SKB_DESCS;
3537 	txq->tx_wake_threshold = txq->tx_stop_threshold / 2;
3538 
3539 	/* Allocate memory for TX descriptors */
3540 	txq->descs = dma_alloc_coherent(pp->dev->dev.parent,
3541 					txq->size * MVNETA_DESC_ALIGNED_SIZE,
3542 					&txq->descs_phys, GFP_KERNEL);
3543 	if (!txq->descs)
3544 		return -ENOMEM;
3545 
3546 	txq->last_desc = txq->size - 1;
3547 
3548 	txq->buf = kmalloc_array(txq->size, sizeof(*txq->buf), GFP_KERNEL);
3549 	if (!txq->buf)
3550 		return -ENOMEM;
3551 
3552 	/* Allocate DMA buffers for TSO MAC/IP/TCP headers */
3553 	err = mvneta_alloc_tso_hdrs(pp, txq);
3554 	if (err)
3555 		return err;
3556 
3557 	/* Setup XPS mapping */
3558 	if (pp->neta_armada3700)
3559 		cpu = 0;
3560 	else if (txq_number > 1)
3561 		cpu = txq->id % num_present_cpus();
3562 	else
3563 		cpu = pp->rxq_def % num_present_cpus();
3564 	cpumask_set_cpu(cpu, &txq->affinity_mask);
3565 	netif_set_xps_queue(pp->dev, &txq->affinity_mask, txq->id);
3566 
3567 	return 0;
3568 }
3569 
mvneta_txq_hw_init(struct mvneta_port * pp,struct mvneta_tx_queue * txq)3570 static void mvneta_txq_hw_init(struct mvneta_port *pp,
3571 			       struct mvneta_tx_queue *txq)
3572 {
3573 	/* Set maximum bandwidth for enabled TXQs */
3574 	mvreg_write(pp, MVETH_TXQ_TOKEN_CFG_REG(txq->id), 0x03ffffff);
3575 	mvreg_write(pp, MVETH_TXQ_TOKEN_COUNT_REG(txq->id), 0x3fffffff);
3576 
3577 	/* Set Tx descriptors queue starting address */
3578 	mvreg_write(pp, MVNETA_TXQ_BASE_ADDR_REG(txq->id), txq->descs_phys);
3579 	mvreg_write(pp, MVNETA_TXQ_SIZE_REG(txq->id), txq->size);
3580 
3581 	mvneta_tx_done_pkts_coal_set(pp, txq, txq->done_pkts_coal);
3582 }
3583 
3584 /* Create and initialize a tx queue */
mvneta_txq_init(struct mvneta_port * pp,struct mvneta_tx_queue * txq)3585 static int mvneta_txq_init(struct mvneta_port *pp,
3586 			   struct mvneta_tx_queue *txq)
3587 {
3588 	int ret;
3589 
3590 	ret = mvneta_txq_sw_init(pp, txq);
3591 	if (ret < 0)
3592 		return ret;
3593 
3594 	mvneta_txq_hw_init(pp, txq);
3595 
3596 	return 0;
3597 }
3598 
3599 /* Free allocated resources when mvneta_txq_init() fails to allocate memory*/
mvneta_txq_sw_deinit(struct mvneta_port * pp,struct mvneta_tx_queue * txq)3600 static void mvneta_txq_sw_deinit(struct mvneta_port *pp,
3601 				 struct mvneta_tx_queue *txq)
3602 {
3603 	struct netdev_queue *nq = netdev_get_tx_queue(pp->dev, txq->id);
3604 
3605 	kfree(txq->buf);
3606 
3607 	mvneta_free_tso_hdrs(pp, txq);
3608 	if (txq->descs)
3609 		dma_free_coherent(pp->dev->dev.parent,
3610 				  txq->size * MVNETA_DESC_ALIGNED_SIZE,
3611 				  txq->descs, txq->descs_phys);
3612 
3613 	netdev_tx_reset_queue(nq);
3614 
3615 	txq->buf               = NULL;
3616 	txq->descs             = NULL;
3617 	txq->last_desc         = 0;
3618 	txq->next_desc_to_proc = 0;
3619 	txq->descs_phys        = 0;
3620 }
3621 
mvneta_txq_hw_deinit(struct mvneta_port * pp,struct mvneta_tx_queue * txq)3622 static void mvneta_txq_hw_deinit(struct mvneta_port *pp,
3623 				 struct mvneta_tx_queue *txq)
3624 {
3625 	/* Set minimum bandwidth for disabled TXQs */
3626 	mvreg_write(pp, MVETH_TXQ_TOKEN_CFG_REG(txq->id), 0);
3627 	mvreg_write(pp, MVETH_TXQ_TOKEN_COUNT_REG(txq->id), 0);
3628 
3629 	/* Set Tx descriptors queue starting address and size */
3630 	mvreg_write(pp, MVNETA_TXQ_BASE_ADDR_REG(txq->id), 0);
3631 	mvreg_write(pp, MVNETA_TXQ_SIZE_REG(txq->id), 0);
3632 }
3633 
mvneta_txq_deinit(struct mvneta_port * pp,struct mvneta_tx_queue * txq)3634 static void mvneta_txq_deinit(struct mvneta_port *pp,
3635 			      struct mvneta_tx_queue *txq)
3636 {
3637 	mvneta_txq_sw_deinit(pp, txq);
3638 	mvneta_txq_hw_deinit(pp, txq);
3639 }
3640 
3641 /* Cleanup all Tx queues */
mvneta_cleanup_txqs(struct mvneta_port * pp)3642 static void mvneta_cleanup_txqs(struct mvneta_port *pp)
3643 {
3644 	int queue;
3645 
3646 	for (queue = 0; queue < txq_number; queue++)
3647 		mvneta_txq_deinit(pp, &pp->txqs[queue]);
3648 }
3649 
3650 /* Cleanup all Rx queues */
mvneta_cleanup_rxqs(struct mvneta_port * pp)3651 static void mvneta_cleanup_rxqs(struct mvneta_port *pp)
3652 {
3653 	int queue;
3654 
3655 	for (queue = 0; queue < rxq_number; queue++)
3656 		mvneta_rxq_deinit(pp, &pp->rxqs[queue]);
3657 }
3658 
3659 
3660 /* Init all Rx queues */
mvneta_setup_rxqs(struct mvneta_port * pp)3661 static int mvneta_setup_rxqs(struct mvneta_port *pp)
3662 {
3663 	int queue;
3664 
3665 	for (queue = 0; queue < rxq_number; queue++) {
3666 		int err = mvneta_rxq_init(pp, &pp->rxqs[queue]);
3667 
3668 		if (err) {
3669 			netdev_err(pp->dev, "%s: can't create rxq=%d\n",
3670 				   __func__, queue);
3671 			mvneta_cleanup_rxqs(pp);
3672 			return err;
3673 		}
3674 	}
3675 
3676 	return 0;
3677 }
3678 
3679 /* Init all tx queues */
mvneta_setup_txqs(struct mvneta_port * pp)3680 static int mvneta_setup_txqs(struct mvneta_port *pp)
3681 {
3682 	int queue;
3683 
3684 	for (queue = 0; queue < txq_number; queue++) {
3685 		int err = mvneta_txq_init(pp, &pp->txqs[queue]);
3686 		if (err) {
3687 			netdev_err(pp->dev, "%s: can't create txq=%d\n",
3688 				   __func__, queue);
3689 			mvneta_cleanup_txqs(pp);
3690 			return err;
3691 		}
3692 	}
3693 
3694 	return 0;
3695 }
3696 
mvneta_comphy_init(struct mvneta_port * pp,phy_interface_t interface)3697 static int mvneta_comphy_init(struct mvneta_port *pp, phy_interface_t interface)
3698 {
3699 	int ret;
3700 
3701 	ret = phy_set_mode_ext(pp->comphy, PHY_MODE_ETHERNET, interface);
3702 	if (ret)
3703 		return ret;
3704 
3705 	return phy_power_on(pp->comphy);
3706 }
3707 
mvneta_config_interface(struct mvneta_port * pp,phy_interface_t interface)3708 static int mvneta_config_interface(struct mvneta_port *pp,
3709 				   phy_interface_t interface)
3710 {
3711 	int ret = 0;
3712 
3713 	if (pp->comphy) {
3714 		if (interface == PHY_INTERFACE_MODE_SGMII ||
3715 		    interface == PHY_INTERFACE_MODE_1000BASEX ||
3716 		    interface == PHY_INTERFACE_MODE_2500BASEX) {
3717 			ret = mvneta_comphy_init(pp, interface);
3718 		}
3719 	} else {
3720 		switch (interface) {
3721 		case PHY_INTERFACE_MODE_QSGMII:
3722 			mvreg_write(pp, MVNETA_SERDES_CFG,
3723 				    MVNETA_QSGMII_SERDES_PROTO);
3724 			break;
3725 
3726 		case PHY_INTERFACE_MODE_SGMII:
3727 		case PHY_INTERFACE_MODE_1000BASEX:
3728 			mvreg_write(pp, MVNETA_SERDES_CFG,
3729 				    MVNETA_SGMII_SERDES_PROTO);
3730 			break;
3731 
3732 		case PHY_INTERFACE_MODE_2500BASEX:
3733 			mvreg_write(pp, MVNETA_SERDES_CFG,
3734 				    MVNETA_HSGMII_SERDES_PROTO);
3735 			break;
3736 		default:
3737 			break;
3738 		}
3739 	}
3740 
3741 	pp->phy_interface = interface;
3742 
3743 	return ret;
3744 }
3745 
mvneta_start_dev(struct mvneta_port * pp)3746 static void mvneta_start_dev(struct mvneta_port *pp)
3747 {
3748 	int cpu;
3749 
3750 	WARN_ON(mvneta_config_interface(pp, pp->phy_interface));
3751 
3752 	mvneta_max_rx_size_set(pp, pp->pkt_size);
3753 	mvneta_txq_max_tx_size_set(pp, pp->pkt_size);
3754 
3755 	/* start the Rx/Tx activity */
3756 	mvneta_port_enable(pp);
3757 
3758 	if (!pp->neta_armada3700) {
3759 		/* Enable polling on the port */
3760 		for_each_online_cpu(cpu) {
3761 			struct mvneta_pcpu_port *port =
3762 				per_cpu_ptr(pp->ports, cpu);
3763 
3764 			napi_enable(&port->napi);
3765 		}
3766 	} else {
3767 		napi_enable(&pp->napi);
3768 	}
3769 
3770 	/* Unmask interrupts. It has to be done from each CPU */
3771 	on_each_cpu(mvneta_percpu_unmask_interrupt, pp, true);
3772 
3773 	mvreg_write(pp, MVNETA_INTR_MISC_MASK,
3774 		    MVNETA_CAUSE_PHY_STATUS_CHANGE |
3775 		    MVNETA_CAUSE_LINK_CHANGE);
3776 
3777 	phylink_start(pp->phylink);
3778 
3779 	/* We may have called phylink_speed_down before */
3780 	phylink_speed_up(pp->phylink);
3781 
3782 	netif_tx_start_all_queues(pp->dev);
3783 
3784 	clear_bit(__MVNETA_DOWN, &pp->state);
3785 }
3786 
mvneta_stop_dev(struct mvneta_port * pp)3787 static void mvneta_stop_dev(struct mvneta_port *pp)
3788 {
3789 	unsigned int cpu;
3790 
3791 	set_bit(__MVNETA_DOWN, &pp->state);
3792 
3793 	if (device_may_wakeup(&pp->dev->dev))
3794 		phylink_speed_down(pp->phylink, false);
3795 
3796 	phylink_stop(pp->phylink);
3797 
3798 	if (!pp->neta_armada3700) {
3799 		for_each_online_cpu(cpu) {
3800 			struct mvneta_pcpu_port *port =
3801 				per_cpu_ptr(pp->ports, cpu);
3802 
3803 			napi_disable(&port->napi);
3804 		}
3805 	} else {
3806 		napi_disable(&pp->napi);
3807 	}
3808 
3809 	netif_carrier_off(pp->dev);
3810 
3811 	mvneta_port_down(pp);
3812 	netif_tx_stop_all_queues(pp->dev);
3813 
3814 	/* Stop the port activity */
3815 	mvneta_port_disable(pp);
3816 
3817 	/* Clear all ethernet port interrupts */
3818 	on_each_cpu(mvneta_percpu_clear_intr_cause, pp, true);
3819 
3820 	/* Mask all ethernet port interrupts */
3821 	on_each_cpu(mvneta_percpu_mask_interrupt, pp, true);
3822 
3823 	mvneta_tx_reset(pp);
3824 	mvneta_rx_reset(pp);
3825 
3826 	WARN_ON(phy_power_off(pp->comphy));
3827 }
3828 
mvneta_percpu_enable(void * arg)3829 static void mvneta_percpu_enable(void *arg)
3830 {
3831 	struct mvneta_port *pp = arg;
3832 
3833 	enable_percpu_irq(pp->dev->irq, IRQ_TYPE_NONE);
3834 }
3835 
mvneta_percpu_disable(void * arg)3836 static void mvneta_percpu_disable(void *arg)
3837 {
3838 	struct mvneta_port *pp = arg;
3839 
3840 	disable_percpu_irq(pp->dev->irq);
3841 }
3842 
3843 /* Change the device mtu */
mvneta_change_mtu(struct net_device * dev,int mtu)3844 static int mvneta_change_mtu(struct net_device *dev, int mtu)
3845 {
3846 	struct mvneta_port *pp = netdev_priv(dev);
3847 	struct bpf_prog *prog = pp->xdp_prog;
3848 	int ret;
3849 
3850 	if (!IS_ALIGNED(MVNETA_RX_PKT_SIZE(mtu), 8)) {
3851 		netdev_info(dev, "Illegal MTU value %d, rounding to %d\n",
3852 			    mtu, ALIGN(MVNETA_RX_PKT_SIZE(mtu), 8));
3853 		mtu = ALIGN(MVNETA_RX_PKT_SIZE(mtu), 8);
3854 	}
3855 
3856 	if (prog && !prog->aux->xdp_has_frags &&
3857 	    mtu > MVNETA_MAX_RX_BUF_SIZE) {
3858 		netdev_info(dev, "Illegal MTU %d for XDP prog without frags\n",
3859 			    mtu);
3860 
3861 		return -EINVAL;
3862 	}
3863 
3864 	WRITE_ONCE(dev->mtu, mtu);
3865 
3866 	if (!netif_running(dev)) {
3867 		if (pp->bm_priv)
3868 			mvneta_bm_update_mtu(pp, mtu);
3869 
3870 		netdev_update_features(dev);
3871 		return 0;
3872 	}
3873 
3874 	/* The interface is running, so we have to force a
3875 	 * reallocation of the queues
3876 	 */
3877 	mvneta_stop_dev(pp);
3878 	on_each_cpu(mvneta_percpu_disable, pp, true);
3879 
3880 	mvneta_cleanup_txqs(pp);
3881 	mvneta_cleanup_rxqs(pp);
3882 
3883 	if (pp->bm_priv)
3884 		mvneta_bm_update_mtu(pp, mtu);
3885 
3886 	pp->pkt_size = MVNETA_RX_PKT_SIZE(dev->mtu);
3887 
3888 	ret = mvneta_setup_rxqs(pp);
3889 	if (ret) {
3890 		netdev_err(dev, "unable to setup rxqs after MTU change\n");
3891 		return ret;
3892 	}
3893 
3894 	ret = mvneta_setup_txqs(pp);
3895 	if (ret) {
3896 		netdev_err(dev, "unable to setup txqs after MTU change\n");
3897 		return ret;
3898 	}
3899 
3900 	on_each_cpu(mvneta_percpu_enable, pp, true);
3901 	mvneta_start_dev(pp);
3902 
3903 	netdev_update_features(dev);
3904 
3905 	return 0;
3906 }
3907 
mvneta_fix_features(struct net_device * dev,netdev_features_t features)3908 static netdev_features_t mvneta_fix_features(struct net_device *dev,
3909 					     netdev_features_t features)
3910 {
3911 	struct mvneta_port *pp = netdev_priv(dev);
3912 
3913 	if (pp->tx_csum_limit && dev->mtu > pp->tx_csum_limit) {
3914 		features &= ~(NETIF_F_IP_CSUM | NETIF_F_TSO);
3915 		netdev_info(dev,
3916 			    "Disable IP checksum for MTU greater than %dB\n",
3917 			    pp->tx_csum_limit);
3918 	}
3919 
3920 	return features;
3921 }
3922 
3923 /* Get mac address */
mvneta_get_mac_addr(struct mvneta_port * pp,unsigned char * addr)3924 static void mvneta_get_mac_addr(struct mvneta_port *pp, unsigned char *addr)
3925 {
3926 	u32 mac_addr_l, mac_addr_h;
3927 
3928 	mac_addr_l = mvreg_read(pp, MVNETA_MAC_ADDR_LOW);
3929 	mac_addr_h = mvreg_read(pp, MVNETA_MAC_ADDR_HIGH);
3930 	addr[0] = (mac_addr_h >> 24) & 0xFF;
3931 	addr[1] = (mac_addr_h >> 16) & 0xFF;
3932 	addr[2] = (mac_addr_h >> 8) & 0xFF;
3933 	addr[3] = mac_addr_h & 0xFF;
3934 	addr[4] = (mac_addr_l >> 8) & 0xFF;
3935 	addr[5] = mac_addr_l & 0xFF;
3936 }
3937 
3938 /* Handle setting mac address */
mvneta_set_mac_addr(struct net_device * dev,void * addr)3939 static int mvneta_set_mac_addr(struct net_device *dev, void *addr)
3940 {
3941 	struct mvneta_port *pp = netdev_priv(dev);
3942 	struct sockaddr *sockaddr = addr;
3943 	int ret;
3944 
3945 	ret = eth_prepare_mac_addr_change(dev, addr);
3946 	if (ret < 0)
3947 		return ret;
3948 	/* Remove previous address table entry */
3949 	mvneta_mac_addr_set(pp, dev->dev_addr, -1);
3950 
3951 	/* Set new addr in hw */
3952 	mvneta_mac_addr_set(pp, sockaddr->sa_data, pp->rxq_def);
3953 
3954 	eth_commit_mac_addr_change(dev, addr);
3955 	return 0;
3956 }
3957 
mvneta_pcs_to_port(struct phylink_pcs * pcs)3958 static struct mvneta_port *mvneta_pcs_to_port(struct phylink_pcs *pcs)
3959 {
3960 	return container_of(pcs, struct mvneta_port, phylink_pcs);
3961 }
3962 
mvneta_pcs_validate(struct phylink_pcs * pcs,unsigned long * supported,const struct phylink_link_state * state)3963 static int mvneta_pcs_validate(struct phylink_pcs *pcs,
3964 			       unsigned long *supported,
3965 			       const struct phylink_link_state *state)
3966 {
3967 	/* We only support QSGMII, SGMII, 802.3z and RGMII modes.
3968 	 * When in 802.3z mode, we must have AN enabled:
3969 	 * "Bit 2 Field InBandAnEn In-band Auto-Negotiation enable. ...
3970 	 * When <PortType> = 1 (1000BASE-X) this field must be set to 1."
3971 	 */
3972 	if (phy_interface_mode_is_8023z(state->interface) &&
3973 	    !phylink_test(state->advertising, Autoneg))
3974 		return -EINVAL;
3975 
3976 	return 0;
3977 }
3978 
mvneta_pcs_get_state(struct phylink_pcs * pcs,struct phylink_link_state * state)3979 static void mvneta_pcs_get_state(struct phylink_pcs *pcs,
3980 				 struct phylink_link_state *state)
3981 {
3982 	struct mvneta_port *pp = mvneta_pcs_to_port(pcs);
3983 	u32 gmac_stat;
3984 
3985 	gmac_stat = mvreg_read(pp, MVNETA_GMAC_STATUS);
3986 
3987 	if (gmac_stat & MVNETA_GMAC_SPEED_1000)
3988 		state->speed =
3989 			state->interface == PHY_INTERFACE_MODE_2500BASEX ?
3990 			SPEED_2500 : SPEED_1000;
3991 	else if (gmac_stat & MVNETA_GMAC_SPEED_100)
3992 		state->speed = SPEED_100;
3993 	else
3994 		state->speed = SPEED_10;
3995 
3996 	state->an_complete = !!(gmac_stat & MVNETA_GMAC_AN_COMPLETE);
3997 	state->link = !!(gmac_stat & MVNETA_GMAC_LINK_UP);
3998 	state->duplex = !!(gmac_stat & MVNETA_GMAC_FULL_DUPLEX);
3999 
4000 	if (gmac_stat & MVNETA_GMAC_RX_FLOW_CTRL_ENABLE)
4001 		state->pause |= MLO_PAUSE_RX;
4002 	if (gmac_stat & MVNETA_GMAC_TX_FLOW_CTRL_ENABLE)
4003 		state->pause |= MLO_PAUSE_TX;
4004 }
4005 
mvneta_pcs_config(struct phylink_pcs * pcs,unsigned int neg_mode,phy_interface_t interface,const unsigned long * advertising,bool permit_pause_to_mac)4006 static int mvneta_pcs_config(struct phylink_pcs *pcs, unsigned int neg_mode,
4007 			     phy_interface_t interface,
4008 			     const unsigned long *advertising,
4009 			     bool permit_pause_to_mac)
4010 {
4011 	struct mvneta_port *pp = mvneta_pcs_to_port(pcs);
4012 	u32 mask, val, an, old_an, changed;
4013 
4014 	mask = MVNETA_GMAC_INBAND_AN_ENABLE |
4015 	       MVNETA_GMAC_INBAND_RESTART_AN |
4016 	       MVNETA_GMAC_AN_SPEED_EN |
4017 	       MVNETA_GMAC_AN_FLOW_CTRL_EN |
4018 	       MVNETA_GMAC_AN_DUPLEX_EN;
4019 
4020 	if (neg_mode == PHYLINK_PCS_NEG_INBAND_ENABLED) {
4021 		mask |= MVNETA_GMAC_CONFIG_MII_SPEED |
4022 			MVNETA_GMAC_CONFIG_GMII_SPEED |
4023 			MVNETA_GMAC_CONFIG_FULL_DUPLEX;
4024 		val = MVNETA_GMAC_INBAND_AN_ENABLE;
4025 
4026 		if (interface == PHY_INTERFACE_MODE_SGMII) {
4027 			/* SGMII mode receives the speed and duplex from PHY */
4028 			val |= MVNETA_GMAC_AN_SPEED_EN |
4029 			       MVNETA_GMAC_AN_DUPLEX_EN;
4030 		} else {
4031 			/* 802.3z mode has fixed speed and duplex */
4032 			val |= MVNETA_GMAC_CONFIG_GMII_SPEED |
4033 			       MVNETA_GMAC_CONFIG_FULL_DUPLEX;
4034 
4035 			/* The FLOW_CTRL_EN bit selects either the hardware
4036 			 * automatically or the CONFIG_FLOW_CTRL manually
4037 			 * controls the GMAC pause mode.
4038 			 */
4039 			if (permit_pause_to_mac)
4040 				val |= MVNETA_GMAC_AN_FLOW_CTRL_EN;
4041 
4042 			/* Update the advertisement bits */
4043 			mask |= MVNETA_GMAC_ADVERT_SYM_FLOW_CTRL;
4044 			if (phylink_test(advertising, Pause))
4045 				val |= MVNETA_GMAC_ADVERT_SYM_FLOW_CTRL;
4046 		}
4047 	} else {
4048 		/* Phy or fixed speed - disable in-band AN modes */
4049 		val = 0;
4050 	}
4051 
4052 	old_an = an = mvreg_read(pp, MVNETA_GMAC_AUTONEG_CONFIG);
4053 	an = (an & ~mask) | val;
4054 	changed = old_an ^ an;
4055 	if (changed)
4056 		mvreg_write(pp, MVNETA_GMAC_AUTONEG_CONFIG, an);
4057 
4058 	/* We are only interested in the advertisement bits changing */
4059 	return !!(changed & MVNETA_GMAC_ADVERT_SYM_FLOW_CTRL);
4060 }
4061 
mvneta_pcs_an_restart(struct phylink_pcs * pcs)4062 static void mvneta_pcs_an_restart(struct phylink_pcs *pcs)
4063 {
4064 	struct mvneta_port *pp = mvneta_pcs_to_port(pcs);
4065 	u32 gmac_an = mvreg_read(pp, MVNETA_GMAC_AUTONEG_CONFIG);
4066 
4067 	mvreg_write(pp, MVNETA_GMAC_AUTONEG_CONFIG,
4068 		    gmac_an | MVNETA_GMAC_INBAND_RESTART_AN);
4069 	mvreg_write(pp, MVNETA_GMAC_AUTONEG_CONFIG,
4070 		    gmac_an & ~MVNETA_GMAC_INBAND_RESTART_AN);
4071 }
4072 
4073 static const struct phylink_pcs_ops mvneta_phylink_pcs_ops = {
4074 	.pcs_validate = mvneta_pcs_validate,
4075 	.pcs_get_state = mvneta_pcs_get_state,
4076 	.pcs_config = mvneta_pcs_config,
4077 	.pcs_an_restart = mvneta_pcs_an_restart,
4078 };
4079 
mvneta_mac_select_pcs(struct phylink_config * config,phy_interface_t interface)4080 static struct phylink_pcs *mvneta_mac_select_pcs(struct phylink_config *config,
4081 						 phy_interface_t interface)
4082 {
4083 	struct net_device *ndev = to_net_dev(config->dev);
4084 	struct mvneta_port *pp = netdev_priv(ndev);
4085 
4086 	return &pp->phylink_pcs;
4087 }
4088 
mvneta_mac_prepare(struct phylink_config * config,unsigned int mode,phy_interface_t interface)4089 static int mvneta_mac_prepare(struct phylink_config *config, unsigned int mode,
4090 			      phy_interface_t interface)
4091 {
4092 	struct net_device *ndev = to_net_dev(config->dev);
4093 	struct mvneta_port *pp = netdev_priv(ndev);
4094 	u32 val;
4095 
4096 	if (pp->phy_interface != interface ||
4097 	    phylink_autoneg_inband(mode)) {
4098 		/* Force the link down when changing the interface or if in
4099 		 * in-band mode. According to Armada 370 documentation, we
4100 		 * can only change the port mode and in-band enable when the
4101 		 * link is down.
4102 		 */
4103 		val = mvreg_read(pp, MVNETA_GMAC_AUTONEG_CONFIG);
4104 		val &= ~MVNETA_GMAC_FORCE_LINK_PASS;
4105 		val |= MVNETA_GMAC_FORCE_LINK_DOWN;
4106 		mvreg_write(pp, MVNETA_GMAC_AUTONEG_CONFIG, val);
4107 	}
4108 
4109 	if (pp->phy_interface != interface)
4110 		WARN_ON(phy_power_off(pp->comphy));
4111 
4112 	/* Enable the 1ms clock */
4113 	if (phylink_autoneg_inband(mode)) {
4114 		unsigned long rate = clk_get_rate(pp->clk);
4115 
4116 		mvreg_write(pp, MVNETA_GMAC_CLOCK_DIVIDER,
4117 			    MVNETA_GMAC_1MS_CLOCK_ENABLE | (rate / 1000));
4118 	}
4119 
4120 	return 0;
4121 }
4122 
mvneta_mac_config(struct phylink_config * config,unsigned int mode,const struct phylink_link_state * state)4123 static void mvneta_mac_config(struct phylink_config *config, unsigned int mode,
4124 			      const struct phylink_link_state *state)
4125 {
4126 	struct net_device *ndev = to_net_dev(config->dev);
4127 	struct mvneta_port *pp = netdev_priv(ndev);
4128 	u32 new_ctrl0, gmac_ctrl0 = mvreg_read(pp, MVNETA_GMAC_CTRL_0);
4129 	u32 new_ctrl2, gmac_ctrl2 = mvreg_read(pp, MVNETA_GMAC_CTRL_2);
4130 	u32 new_ctrl4, gmac_ctrl4 = mvreg_read(pp, MVNETA_GMAC_CTRL_4);
4131 
4132 	new_ctrl0 = gmac_ctrl0 & ~MVNETA_GMAC0_PORT_1000BASE_X;
4133 	new_ctrl2 = gmac_ctrl2 & ~(MVNETA_GMAC2_INBAND_AN_ENABLE |
4134 				   MVNETA_GMAC2_PORT_RESET);
4135 	new_ctrl4 = gmac_ctrl4 & ~(MVNETA_GMAC4_SHORT_PREAMBLE_ENABLE);
4136 
4137 	/* Even though it might look weird, when we're configured in
4138 	 * SGMII or QSGMII mode, the RGMII bit needs to be set.
4139 	 */
4140 	new_ctrl2 |= MVNETA_GMAC2_PORT_RGMII;
4141 
4142 	if (state->interface == PHY_INTERFACE_MODE_QSGMII ||
4143 	    state->interface == PHY_INTERFACE_MODE_SGMII ||
4144 	    phy_interface_mode_is_8023z(state->interface))
4145 		new_ctrl2 |= MVNETA_GMAC2_PCS_ENABLE;
4146 
4147 	if (!phylink_autoneg_inband(mode)) {
4148 		/* Phy or fixed speed - nothing to do, leave the
4149 		 * configured speed, duplex and flow control as-is.
4150 		 */
4151 	} else if (state->interface == PHY_INTERFACE_MODE_SGMII) {
4152 		/* SGMII mode receives the state from the PHY */
4153 		new_ctrl2 |= MVNETA_GMAC2_INBAND_AN_ENABLE;
4154 	} else {
4155 		/* 802.3z negotiation - only 1000base-X */
4156 		new_ctrl0 |= MVNETA_GMAC0_PORT_1000BASE_X;
4157 	}
4158 
4159 	/* When at 2.5G, the link partner can send frames with shortened
4160 	 * preambles.
4161 	 */
4162 	if (state->interface == PHY_INTERFACE_MODE_2500BASEX)
4163 		new_ctrl4 |= MVNETA_GMAC4_SHORT_PREAMBLE_ENABLE;
4164 
4165 	if (new_ctrl0 != gmac_ctrl0)
4166 		mvreg_write(pp, MVNETA_GMAC_CTRL_0, new_ctrl0);
4167 	if (new_ctrl2 != gmac_ctrl2)
4168 		mvreg_write(pp, MVNETA_GMAC_CTRL_2, new_ctrl2);
4169 	if (new_ctrl4 != gmac_ctrl4)
4170 		mvreg_write(pp, MVNETA_GMAC_CTRL_4, new_ctrl4);
4171 
4172 	if (gmac_ctrl2 & MVNETA_GMAC2_PORT_RESET) {
4173 		while ((mvreg_read(pp, MVNETA_GMAC_CTRL_2) &
4174 			MVNETA_GMAC2_PORT_RESET) != 0)
4175 			continue;
4176 	}
4177 }
4178 
mvneta_mac_finish(struct phylink_config * config,unsigned int mode,phy_interface_t interface)4179 static int mvneta_mac_finish(struct phylink_config *config, unsigned int mode,
4180 			     phy_interface_t interface)
4181 {
4182 	struct net_device *ndev = to_net_dev(config->dev);
4183 	struct mvneta_port *pp = netdev_priv(ndev);
4184 	u32 val, clk;
4185 
4186 	/* Disable 1ms clock if not in in-band mode */
4187 	if (!phylink_autoneg_inband(mode)) {
4188 		clk = mvreg_read(pp, MVNETA_GMAC_CLOCK_DIVIDER);
4189 		clk &= ~MVNETA_GMAC_1MS_CLOCK_ENABLE;
4190 		mvreg_write(pp, MVNETA_GMAC_CLOCK_DIVIDER, clk);
4191 	}
4192 
4193 	if (pp->phy_interface != interface)
4194 		/* Enable the Serdes PHY */
4195 		WARN_ON(mvneta_config_interface(pp, interface));
4196 
4197 	/* Allow the link to come up if in in-band mode, otherwise the
4198 	 * link is forced via mac_link_down()/mac_link_up()
4199 	 */
4200 	if (phylink_autoneg_inband(mode)) {
4201 		val = mvreg_read(pp, MVNETA_GMAC_AUTONEG_CONFIG);
4202 		val &= ~MVNETA_GMAC_FORCE_LINK_DOWN;
4203 		mvreg_write(pp, MVNETA_GMAC_AUTONEG_CONFIG, val);
4204 	}
4205 
4206 	return 0;
4207 }
4208 
mvneta_set_eee(struct mvneta_port * pp,bool enable)4209 static void mvneta_set_eee(struct mvneta_port *pp, bool enable)
4210 {
4211 	u32 lpi_ctl1;
4212 
4213 	lpi_ctl1 = mvreg_read(pp, MVNETA_LPI_CTRL_1);
4214 	if (enable)
4215 		lpi_ctl1 |= MVNETA_LPI_REQUEST_ENABLE;
4216 	else
4217 		lpi_ctl1 &= ~MVNETA_LPI_REQUEST_ENABLE;
4218 	mvreg_write(pp, MVNETA_LPI_CTRL_1, lpi_ctl1);
4219 }
4220 
mvneta_mac_link_down(struct phylink_config * config,unsigned int mode,phy_interface_t interface)4221 static void mvneta_mac_link_down(struct phylink_config *config,
4222 				 unsigned int mode, phy_interface_t interface)
4223 {
4224 	struct net_device *ndev = to_net_dev(config->dev);
4225 	struct mvneta_port *pp = netdev_priv(ndev);
4226 	u32 val;
4227 
4228 	mvneta_port_down(pp);
4229 
4230 	if (!phylink_autoneg_inband(mode)) {
4231 		val = mvreg_read(pp, MVNETA_GMAC_AUTONEG_CONFIG);
4232 		val &= ~MVNETA_GMAC_FORCE_LINK_PASS;
4233 		val |= MVNETA_GMAC_FORCE_LINK_DOWN;
4234 		mvreg_write(pp, MVNETA_GMAC_AUTONEG_CONFIG, val);
4235 	}
4236 
4237 	pp->eee_active = false;
4238 	mvneta_set_eee(pp, false);
4239 }
4240 
mvneta_mac_link_up(struct phylink_config * config,struct phy_device * phy,unsigned int mode,phy_interface_t interface,int speed,int duplex,bool tx_pause,bool rx_pause)4241 static void mvneta_mac_link_up(struct phylink_config *config,
4242 			       struct phy_device *phy,
4243 			       unsigned int mode, phy_interface_t interface,
4244 			       int speed, int duplex,
4245 			       bool tx_pause, bool rx_pause)
4246 {
4247 	struct net_device *ndev = to_net_dev(config->dev);
4248 	struct mvneta_port *pp = netdev_priv(ndev);
4249 	u32 val;
4250 
4251 	if (!phylink_autoneg_inband(mode)) {
4252 		val = mvreg_read(pp, MVNETA_GMAC_AUTONEG_CONFIG);
4253 		val &= ~(MVNETA_GMAC_FORCE_LINK_DOWN |
4254 			 MVNETA_GMAC_CONFIG_MII_SPEED |
4255 			 MVNETA_GMAC_CONFIG_GMII_SPEED |
4256 			 MVNETA_GMAC_CONFIG_FLOW_CTRL |
4257 			 MVNETA_GMAC_CONFIG_FULL_DUPLEX);
4258 		val |= MVNETA_GMAC_FORCE_LINK_PASS;
4259 
4260 		if (speed == SPEED_1000 || speed == SPEED_2500)
4261 			val |= MVNETA_GMAC_CONFIG_GMII_SPEED;
4262 		else if (speed == SPEED_100)
4263 			val |= MVNETA_GMAC_CONFIG_MII_SPEED;
4264 
4265 		if (duplex == DUPLEX_FULL)
4266 			val |= MVNETA_GMAC_CONFIG_FULL_DUPLEX;
4267 
4268 		if (tx_pause || rx_pause)
4269 			val |= MVNETA_GMAC_CONFIG_FLOW_CTRL;
4270 
4271 		mvreg_write(pp, MVNETA_GMAC_AUTONEG_CONFIG, val);
4272 	} else {
4273 		/* When inband doesn't cover flow control or flow control is
4274 		 * disabled, we need to manually configure it. This bit will
4275 		 * only have effect if MVNETA_GMAC_AN_FLOW_CTRL_EN is unset.
4276 		 */
4277 		val = mvreg_read(pp, MVNETA_GMAC_AUTONEG_CONFIG);
4278 		val &= ~MVNETA_GMAC_CONFIG_FLOW_CTRL;
4279 
4280 		if (tx_pause || rx_pause)
4281 			val |= MVNETA_GMAC_CONFIG_FLOW_CTRL;
4282 
4283 		mvreg_write(pp, MVNETA_GMAC_AUTONEG_CONFIG, val);
4284 	}
4285 
4286 	mvneta_port_up(pp);
4287 
4288 	if (phy && pp->eee_enabled) {
4289 		pp->eee_active = phy_init_eee(phy, false) >= 0;
4290 		mvneta_set_eee(pp, pp->eee_active && pp->tx_lpi_enabled);
4291 	}
4292 }
4293 
4294 static const struct phylink_mac_ops mvneta_phylink_ops = {
4295 	.mac_select_pcs = mvneta_mac_select_pcs,
4296 	.mac_prepare = mvneta_mac_prepare,
4297 	.mac_config = mvneta_mac_config,
4298 	.mac_finish = mvneta_mac_finish,
4299 	.mac_link_down = mvneta_mac_link_down,
4300 	.mac_link_up = mvneta_mac_link_up,
4301 };
4302 
mvneta_mdio_probe(struct mvneta_port * pp)4303 static int mvneta_mdio_probe(struct mvneta_port *pp)
4304 {
4305 	struct ethtool_wolinfo wol = { .cmd = ETHTOOL_GWOL };
4306 	int err = phylink_of_phy_connect(pp->phylink, pp->dn, 0);
4307 
4308 	if (err)
4309 		netdev_err(pp->dev, "could not attach PHY: %d\n", err);
4310 
4311 	phylink_ethtool_get_wol(pp->phylink, &wol);
4312 	device_set_wakeup_capable(&pp->dev->dev, !!wol.supported);
4313 
4314 	/* PHY WoL may be enabled but device wakeup disabled */
4315 	if (wol.supported)
4316 		device_set_wakeup_enable(&pp->dev->dev, !!wol.wolopts);
4317 
4318 	return err;
4319 }
4320 
mvneta_mdio_remove(struct mvneta_port * pp)4321 static void mvneta_mdio_remove(struct mvneta_port *pp)
4322 {
4323 	phylink_disconnect_phy(pp->phylink);
4324 }
4325 
4326 /* Electing a CPU must be done in an atomic way: it should be done
4327  * after or before the removal/insertion of a CPU and this function is
4328  * not reentrant.
4329  */
mvneta_percpu_elect(struct mvneta_port * pp)4330 static void mvneta_percpu_elect(struct mvneta_port *pp)
4331 {
4332 	int elected_cpu = 0, max_cpu, cpu;
4333 
4334 	/* Use the cpu associated to the rxq when it is online, in all
4335 	 * the other cases, use the cpu 0 which can't be offline.
4336 	 */
4337 	if (pp->rxq_def < nr_cpu_ids && cpu_online(pp->rxq_def))
4338 		elected_cpu = pp->rxq_def;
4339 
4340 	max_cpu = num_present_cpus();
4341 
4342 	for_each_online_cpu(cpu) {
4343 		int rxq_map = 0, txq_map = 0;
4344 		int rxq;
4345 
4346 		for (rxq = 0; rxq < rxq_number; rxq++)
4347 			if ((rxq % max_cpu) == cpu)
4348 				rxq_map |= MVNETA_CPU_RXQ_ACCESS(rxq);
4349 
4350 		if (cpu == elected_cpu)
4351 			/* Map the default receive queue to the elected CPU */
4352 			rxq_map |= MVNETA_CPU_RXQ_ACCESS(pp->rxq_def);
4353 
4354 		/* We update the TX queue map only if we have one
4355 		 * queue. In this case we associate the TX queue to
4356 		 * the CPU bound to the default RX queue
4357 		 */
4358 		if (txq_number == 1)
4359 			txq_map = (cpu == elected_cpu) ?
4360 				MVNETA_CPU_TXQ_ACCESS(0) : 0;
4361 		else
4362 			txq_map = mvreg_read(pp, MVNETA_CPU_MAP(cpu)) &
4363 				MVNETA_CPU_TXQ_ACCESS_ALL_MASK;
4364 
4365 		mvreg_write(pp, MVNETA_CPU_MAP(cpu), rxq_map | txq_map);
4366 
4367 		/* Update the interrupt mask on each CPU according the
4368 		 * new mapping
4369 		 */
4370 		smp_call_function_single(cpu, mvneta_percpu_unmask_interrupt,
4371 					 pp, true);
4372 	}
4373 };
4374 
mvneta_cpu_online(unsigned int cpu,struct hlist_node * node)4375 static int mvneta_cpu_online(unsigned int cpu, struct hlist_node *node)
4376 {
4377 	int other_cpu;
4378 	struct mvneta_port *pp = hlist_entry_safe(node, struct mvneta_port,
4379 						  node_online);
4380 	struct mvneta_pcpu_port *port = per_cpu_ptr(pp->ports, cpu);
4381 
4382 	/* Armada 3700's per-cpu interrupt for mvneta is broken, all interrupts
4383 	 * are routed to CPU 0, so we don't need all the cpu-hotplug support
4384 	 */
4385 	if (pp->neta_armada3700)
4386 		return 0;
4387 
4388 	spin_lock(&pp->lock);
4389 	/*
4390 	 * Configuring the driver for a new CPU while the driver is
4391 	 * stopping is racy, so just avoid it.
4392 	 */
4393 	if (pp->is_stopped) {
4394 		spin_unlock(&pp->lock);
4395 		return 0;
4396 	}
4397 	netif_tx_stop_all_queues(pp->dev);
4398 
4399 	/*
4400 	 * We have to synchronise on tha napi of each CPU except the one
4401 	 * just being woken up
4402 	 */
4403 	for_each_online_cpu(other_cpu) {
4404 		if (other_cpu != cpu) {
4405 			struct mvneta_pcpu_port *other_port =
4406 				per_cpu_ptr(pp->ports, other_cpu);
4407 
4408 			napi_synchronize(&other_port->napi);
4409 		}
4410 	}
4411 
4412 	/* Mask all ethernet port interrupts */
4413 	on_each_cpu(mvneta_percpu_mask_interrupt, pp, true);
4414 	napi_enable(&port->napi);
4415 
4416 	/*
4417 	 * Enable per-CPU interrupts on the CPU that is
4418 	 * brought up.
4419 	 */
4420 	mvneta_percpu_enable(pp);
4421 
4422 	/*
4423 	 * Enable per-CPU interrupt on the one CPU we care
4424 	 * about.
4425 	 */
4426 	mvneta_percpu_elect(pp);
4427 
4428 	/* Unmask all ethernet port interrupts */
4429 	on_each_cpu(mvneta_percpu_unmask_interrupt, pp, true);
4430 	mvreg_write(pp, MVNETA_INTR_MISC_MASK,
4431 		    MVNETA_CAUSE_PHY_STATUS_CHANGE |
4432 		    MVNETA_CAUSE_LINK_CHANGE);
4433 	netif_tx_start_all_queues(pp->dev);
4434 	spin_unlock(&pp->lock);
4435 	return 0;
4436 }
4437 
mvneta_cpu_down_prepare(unsigned int cpu,struct hlist_node * node)4438 static int mvneta_cpu_down_prepare(unsigned int cpu, struct hlist_node *node)
4439 {
4440 	struct mvneta_port *pp = hlist_entry_safe(node, struct mvneta_port,
4441 						  node_online);
4442 	struct mvneta_pcpu_port *port = per_cpu_ptr(pp->ports, cpu);
4443 
4444 	/*
4445 	 * Thanks to this lock we are sure that any pending cpu election is
4446 	 * done.
4447 	 */
4448 	spin_lock(&pp->lock);
4449 	/* Mask all ethernet port interrupts */
4450 	on_each_cpu(mvneta_percpu_mask_interrupt, pp, true);
4451 	spin_unlock(&pp->lock);
4452 
4453 	napi_synchronize(&port->napi);
4454 	napi_disable(&port->napi);
4455 	/* Disable per-CPU interrupts on the CPU that is brought down. */
4456 	mvneta_percpu_disable(pp);
4457 	return 0;
4458 }
4459 
mvneta_cpu_dead(unsigned int cpu,struct hlist_node * node)4460 static int mvneta_cpu_dead(unsigned int cpu, struct hlist_node *node)
4461 {
4462 	struct mvneta_port *pp = hlist_entry_safe(node, struct mvneta_port,
4463 						  node_dead);
4464 
4465 	/* Check if a new CPU must be elected now this on is down */
4466 	spin_lock(&pp->lock);
4467 	mvneta_percpu_elect(pp);
4468 	spin_unlock(&pp->lock);
4469 	/* Unmask all ethernet port interrupts */
4470 	on_each_cpu(mvneta_percpu_unmask_interrupt, pp, true);
4471 	mvreg_write(pp, MVNETA_INTR_MISC_MASK,
4472 		    MVNETA_CAUSE_PHY_STATUS_CHANGE |
4473 		    MVNETA_CAUSE_LINK_CHANGE);
4474 	netif_tx_start_all_queues(pp->dev);
4475 	return 0;
4476 }
4477 
mvneta_open(struct net_device * dev)4478 static int mvneta_open(struct net_device *dev)
4479 {
4480 	struct mvneta_port *pp = netdev_priv(dev);
4481 	int ret;
4482 
4483 	pp->pkt_size = MVNETA_RX_PKT_SIZE(pp->dev->mtu);
4484 
4485 	ret = mvneta_setup_rxqs(pp);
4486 	if (ret)
4487 		return ret;
4488 
4489 	ret = mvneta_setup_txqs(pp);
4490 	if (ret)
4491 		goto err_cleanup_rxqs;
4492 
4493 	/* Connect to port interrupt line */
4494 	if (pp->neta_armada3700)
4495 		ret = request_irq(pp->dev->irq, mvneta_isr, 0,
4496 				  dev->name, pp);
4497 	else
4498 		ret = request_percpu_irq(pp->dev->irq, mvneta_percpu_isr,
4499 					 dev->name, pp->ports);
4500 	if (ret) {
4501 		netdev_err(pp->dev, "cannot request irq %d\n", pp->dev->irq);
4502 		goto err_cleanup_txqs;
4503 	}
4504 
4505 	if (!pp->neta_armada3700) {
4506 		/* Enable per-CPU interrupt on all the CPU to handle our RX
4507 		 * queue interrupts
4508 		 */
4509 		on_each_cpu(mvneta_percpu_enable, pp, true);
4510 
4511 		pp->is_stopped = false;
4512 		/* Register a CPU notifier to handle the case where our CPU
4513 		 * might be taken offline.
4514 		 */
4515 		ret = cpuhp_state_add_instance_nocalls(online_hpstate,
4516 						       &pp->node_online);
4517 		if (ret)
4518 			goto err_free_irq;
4519 
4520 		ret = cpuhp_state_add_instance_nocalls(CPUHP_NET_MVNETA_DEAD,
4521 						       &pp->node_dead);
4522 		if (ret)
4523 			goto err_free_online_hp;
4524 	}
4525 
4526 	ret = mvneta_mdio_probe(pp);
4527 	if (ret < 0) {
4528 		netdev_err(dev, "cannot probe MDIO bus\n");
4529 		goto err_free_dead_hp;
4530 	}
4531 
4532 	mvneta_start_dev(pp);
4533 
4534 	return 0;
4535 
4536 err_free_dead_hp:
4537 	if (!pp->neta_armada3700)
4538 		cpuhp_state_remove_instance_nocalls(CPUHP_NET_MVNETA_DEAD,
4539 						    &pp->node_dead);
4540 err_free_online_hp:
4541 	if (!pp->neta_armada3700)
4542 		cpuhp_state_remove_instance_nocalls(online_hpstate,
4543 						    &pp->node_online);
4544 err_free_irq:
4545 	if (pp->neta_armada3700) {
4546 		free_irq(pp->dev->irq, pp);
4547 	} else {
4548 		on_each_cpu(mvneta_percpu_disable, pp, true);
4549 		free_percpu_irq(pp->dev->irq, pp->ports);
4550 	}
4551 err_cleanup_txqs:
4552 	mvneta_cleanup_txqs(pp);
4553 err_cleanup_rxqs:
4554 	mvneta_cleanup_rxqs(pp);
4555 	return ret;
4556 }
4557 
4558 /* Stop the port, free port interrupt line */
mvneta_stop(struct net_device * dev)4559 static int mvneta_stop(struct net_device *dev)
4560 {
4561 	struct mvneta_port *pp = netdev_priv(dev);
4562 
4563 	if (!pp->neta_armada3700) {
4564 		/* Inform that we are stopping so we don't want to setup the
4565 		 * driver for new CPUs in the notifiers. The code of the
4566 		 * notifier for CPU online is protected by the same spinlock,
4567 		 * so when we get the lock, the notifer work is done.
4568 		 */
4569 		spin_lock(&pp->lock);
4570 		pp->is_stopped = true;
4571 		spin_unlock(&pp->lock);
4572 
4573 		mvneta_stop_dev(pp);
4574 		mvneta_mdio_remove(pp);
4575 
4576 		cpuhp_state_remove_instance_nocalls(online_hpstate,
4577 						    &pp->node_online);
4578 		cpuhp_state_remove_instance_nocalls(CPUHP_NET_MVNETA_DEAD,
4579 						    &pp->node_dead);
4580 		on_each_cpu(mvneta_percpu_disable, pp, true);
4581 		free_percpu_irq(dev->irq, pp->ports);
4582 	} else {
4583 		mvneta_stop_dev(pp);
4584 		mvneta_mdio_remove(pp);
4585 		free_irq(dev->irq, pp);
4586 	}
4587 
4588 	mvneta_cleanup_rxqs(pp);
4589 	mvneta_cleanup_txqs(pp);
4590 
4591 	return 0;
4592 }
4593 
mvneta_ioctl(struct net_device * dev,struct ifreq * ifr,int cmd)4594 static int mvneta_ioctl(struct net_device *dev, struct ifreq *ifr, int cmd)
4595 {
4596 	struct mvneta_port *pp = netdev_priv(dev);
4597 
4598 	return phylink_mii_ioctl(pp->phylink, ifr, cmd);
4599 }
4600 
mvneta_xdp_setup(struct net_device * dev,struct bpf_prog * prog,struct netlink_ext_ack * extack)4601 static int mvneta_xdp_setup(struct net_device *dev, struct bpf_prog *prog,
4602 			    struct netlink_ext_ack *extack)
4603 {
4604 	bool need_update, running = netif_running(dev);
4605 	struct mvneta_port *pp = netdev_priv(dev);
4606 	struct bpf_prog *old_prog;
4607 
4608 	if (prog && !prog->aux->xdp_has_frags &&
4609 	    dev->mtu > MVNETA_MAX_RX_BUF_SIZE) {
4610 		NL_SET_ERR_MSG_MOD(extack, "prog does not support XDP frags");
4611 		return -EOPNOTSUPP;
4612 	}
4613 
4614 	if (pp->bm_priv) {
4615 		NL_SET_ERR_MSG_MOD(extack,
4616 				   "Hardware Buffer Management not supported on XDP");
4617 		return -EOPNOTSUPP;
4618 	}
4619 
4620 	need_update = !!pp->xdp_prog != !!prog;
4621 	if (running && need_update)
4622 		mvneta_stop(dev);
4623 
4624 	old_prog = xchg(&pp->xdp_prog, prog);
4625 	if (old_prog)
4626 		bpf_prog_put(old_prog);
4627 
4628 	if (running && need_update)
4629 		return mvneta_open(dev);
4630 
4631 	return 0;
4632 }
4633 
mvneta_xdp(struct net_device * dev,struct netdev_bpf * xdp)4634 static int mvneta_xdp(struct net_device *dev, struct netdev_bpf *xdp)
4635 {
4636 	switch (xdp->command) {
4637 	case XDP_SETUP_PROG:
4638 		return mvneta_xdp_setup(dev, xdp->prog, xdp->extack);
4639 	default:
4640 		return -EINVAL;
4641 	}
4642 }
4643 
4644 /* Ethtool methods */
4645 
4646 /* Set link ksettings (phy address, speed) for ethtools */
4647 static int
mvneta_ethtool_set_link_ksettings(struct net_device * ndev,const struct ethtool_link_ksettings * cmd)4648 mvneta_ethtool_set_link_ksettings(struct net_device *ndev,
4649 				  const struct ethtool_link_ksettings *cmd)
4650 {
4651 	struct mvneta_port *pp = netdev_priv(ndev);
4652 
4653 	return phylink_ethtool_ksettings_set(pp->phylink, cmd);
4654 }
4655 
4656 /* Get link ksettings for ethtools */
4657 static int
mvneta_ethtool_get_link_ksettings(struct net_device * ndev,struct ethtool_link_ksettings * cmd)4658 mvneta_ethtool_get_link_ksettings(struct net_device *ndev,
4659 				  struct ethtool_link_ksettings *cmd)
4660 {
4661 	struct mvneta_port *pp = netdev_priv(ndev);
4662 
4663 	return phylink_ethtool_ksettings_get(pp->phylink, cmd);
4664 }
4665 
mvneta_ethtool_nway_reset(struct net_device * dev)4666 static int mvneta_ethtool_nway_reset(struct net_device *dev)
4667 {
4668 	struct mvneta_port *pp = netdev_priv(dev);
4669 
4670 	return phylink_ethtool_nway_reset(pp->phylink);
4671 }
4672 
4673 /* Set interrupt coalescing for ethtools */
4674 static int
mvneta_ethtool_set_coalesce(struct net_device * dev,struct ethtool_coalesce * c,struct kernel_ethtool_coalesce * kernel_coal,struct netlink_ext_ack * extack)4675 mvneta_ethtool_set_coalesce(struct net_device *dev,
4676 			    struct ethtool_coalesce *c,
4677 			    struct kernel_ethtool_coalesce *kernel_coal,
4678 			    struct netlink_ext_ack *extack)
4679 {
4680 	struct mvneta_port *pp = netdev_priv(dev);
4681 	int queue;
4682 
4683 	for (queue = 0; queue < rxq_number; queue++) {
4684 		struct mvneta_rx_queue *rxq = &pp->rxqs[queue];
4685 		rxq->time_coal = c->rx_coalesce_usecs;
4686 		rxq->pkts_coal = c->rx_max_coalesced_frames;
4687 		mvneta_rx_pkts_coal_set(pp, rxq, rxq->pkts_coal);
4688 		mvneta_rx_time_coal_set(pp, rxq, rxq->time_coal);
4689 	}
4690 
4691 	for (queue = 0; queue < txq_number; queue++) {
4692 		struct mvneta_tx_queue *txq = &pp->txqs[queue];
4693 		txq->done_pkts_coal = c->tx_max_coalesced_frames;
4694 		mvneta_tx_done_pkts_coal_set(pp, txq, txq->done_pkts_coal);
4695 	}
4696 
4697 	return 0;
4698 }
4699 
4700 /* get coalescing for ethtools */
4701 static int
mvneta_ethtool_get_coalesce(struct net_device * dev,struct ethtool_coalesce * c,struct kernel_ethtool_coalesce * kernel_coal,struct netlink_ext_ack * extack)4702 mvneta_ethtool_get_coalesce(struct net_device *dev,
4703 			    struct ethtool_coalesce *c,
4704 			    struct kernel_ethtool_coalesce *kernel_coal,
4705 			    struct netlink_ext_ack *extack)
4706 {
4707 	struct mvneta_port *pp = netdev_priv(dev);
4708 
4709 	c->rx_coalesce_usecs        = pp->rxqs[0].time_coal;
4710 	c->rx_max_coalesced_frames  = pp->rxqs[0].pkts_coal;
4711 
4712 	c->tx_max_coalesced_frames =  pp->txqs[0].done_pkts_coal;
4713 	return 0;
4714 }
4715 
4716 
mvneta_ethtool_get_drvinfo(struct net_device * dev,struct ethtool_drvinfo * drvinfo)4717 static void mvneta_ethtool_get_drvinfo(struct net_device *dev,
4718 				    struct ethtool_drvinfo *drvinfo)
4719 {
4720 	strscpy(drvinfo->driver, MVNETA_DRIVER_NAME,
4721 		sizeof(drvinfo->driver));
4722 	strscpy(drvinfo->version, MVNETA_DRIVER_VERSION,
4723 		sizeof(drvinfo->version));
4724 	strscpy(drvinfo->bus_info, dev_name(&dev->dev),
4725 		sizeof(drvinfo->bus_info));
4726 }
4727 
4728 
4729 static void
mvneta_ethtool_get_ringparam(struct net_device * netdev,struct ethtool_ringparam * ring,struct kernel_ethtool_ringparam * kernel_ring,struct netlink_ext_ack * extack)4730 mvneta_ethtool_get_ringparam(struct net_device *netdev,
4731 			     struct ethtool_ringparam *ring,
4732 			     struct kernel_ethtool_ringparam *kernel_ring,
4733 			     struct netlink_ext_ack *extack)
4734 {
4735 	struct mvneta_port *pp = netdev_priv(netdev);
4736 
4737 	ring->rx_max_pending = MVNETA_MAX_RXD;
4738 	ring->tx_max_pending = MVNETA_MAX_TXD;
4739 	ring->rx_pending = pp->rx_ring_size;
4740 	ring->tx_pending = pp->tx_ring_size;
4741 }
4742 
4743 static int
mvneta_ethtool_set_ringparam(struct net_device * dev,struct ethtool_ringparam * ring,struct kernel_ethtool_ringparam * kernel_ring,struct netlink_ext_ack * extack)4744 mvneta_ethtool_set_ringparam(struct net_device *dev,
4745 			     struct ethtool_ringparam *ring,
4746 			     struct kernel_ethtool_ringparam *kernel_ring,
4747 			     struct netlink_ext_ack *extack)
4748 {
4749 	struct mvneta_port *pp = netdev_priv(dev);
4750 
4751 	if ((ring->rx_pending == 0) || (ring->tx_pending == 0))
4752 		return -EINVAL;
4753 	pp->rx_ring_size = ring->rx_pending < MVNETA_MAX_RXD ?
4754 		ring->rx_pending : MVNETA_MAX_RXD;
4755 
4756 	pp->tx_ring_size = clamp_t(u16, ring->tx_pending,
4757 				   MVNETA_MAX_SKB_DESCS * 2, MVNETA_MAX_TXD);
4758 	if (pp->tx_ring_size != ring->tx_pending)
4759 		netdev_warn(dev, "TX queue size set to %u (requested %u)\n",
4760 			    pp->tx_ring_size, ring->tx_pending);
4761 
4762 	if (netif_running(dev)) {
4763 		mvneta_stop(dev);
4764 		if (mvneta_open(dev)) {
4765 			netdev_err(dev,
4766 				   "error on opening device after ring param change\n");
4767 			return -ENOMEM;
4768 		}
4769 	}
4770 
4771 	return 0;
4772 }
4773 
mvneta_ethtool_get_pauseparam(struct net_device * dev,struct ethtool_pauseparam * pause)4774 static void mvneta_ethtool_get_pauseparam(struct net_device *dev,
4775 					  struct ethtool_pauseparam *pause)
4776 {
4777 	struct mvneta_port *pp = netdev_priv(dev);
4778 
4779 	phylink_ethtool_get_pauseparam(pp->phylink, pause);
4780 }
4781 
mvneta_ethtool_set_pauseparam(struct net_device * dev,struct ethtool_pauseparam * pause)4782 static int mvneta_ethtool_set_pauseparam(struct net_device *dev,
4783 					 struct ethtool_pauseparam *pause)
4784 {
4785 	struct mvneta_port *pp = netdev_priv(dev);
4786 
4787 	return phylink_ethtool_set_pauseparam(pp->phylink, pause);
4788 }
4789 
mvneta_ethtool_get_strings(struct net_device * netdev,u32 sset,u8 * data)4790 static void mvneta_ethtool_get_strings(struct net_device *netdev, u32 sset,
4791 				       u8 *data)
4792 {
4793 	if (sset == ETH_SS_STATS) {
4794 		struct mvneta_port *pp = netdev_priv(netdev);
4795 		int i;
4796 
4797 		for (i = 0; i < ARRAY_SIZE(mvneta_statistics); i++)
4798 			memcpy(data + i * ETH_GSTRING_LEN,
4799 			       mvneta_statistics[i].name, ETH_GSTRING_LEN);
4800 
4801 		if (!pp->bm_priv) {
4802 			data += ETH_GSTRING_LEN * ARRAY_SIZE(mvneta_statistics);
4803 			page_pool_ethtool_stats_get_strings(data);
4804 		}
4805 	}
4806 }
4807 
4808 static void
mvneta_ethtool_update_pcpu_stats(struct mvneta_port * pp,struct mvneta_ethtool_stats * es)4809 mvneta_ethtool_update_pcpu_stats(struct mvneta_port *pp,
4810 				 struct mvneta_ethtool_stats *es)
4811 {
4812 	unsigned int start;
4813 	int cpu;
4814 
4815 	for_each_possible_cpu(cpu) {
4816 		struct mvneta_pcpu_stats *stats;
4817 		u64 skb_alloc_error;
4818 		u64 refill_error;
4819 		u64 xdp_redirect;
4820 		u64 xdp_xmit_err;
4821 		u64 xdp_tx_err;
4822 		u64 xdp_pass;
4823 		u64 xdp_drop;
4824 		u64 xdp_xmit;
4825 		u64 xdp_tx;
4826 
4827 		stats = per_cpu_ptr(pp->stats, cpu);
4828 		do {
4829 			start = u64_stats_fetch_begin(&stats->syncp);
4830 			skb_alloc_error = stats->es.skb_alloc_error;
4831 			refill_error = stats->es.refill_error;
4832 			xdp_redirect = stats->es.ps.xdp_redirect;
4833 			xdp_pass = stats->es.ps.xdp_pass;
4834 			xdp_drop = stats->es.ps.xdp_drop;
4835 			xdp_xmit = stats->es.ps.xdp_xmit;
4836 			xdp_xmit_err = stats->es.ps.xdp_xmit_err;
4837 			xdp_tx = stats->es.ps.xdp_tx;
4838 			xdp_tx_err = stats->es.ps.xdp_tx_err;
4839 		} while (u64_stats_fetch_retry(&stats->syncp, start));
4840 
4841 		es->skb_alloc_error += skb_alloc_error;
4842 		es->refill_error += refill_error;
4843 		es->ps.xdp_redirect += xdp_redirect;
4844 		es->ps.xdp_pass += xdp_pass;
4845 		es->ps.xdp_drop += xdp_drop;
4846 		es->ps.xdp_xmit += xdp_xmit;
4847 		es->ps.xdp_xmit_err += xdp_xmit_err;
4848 		es->ps.xdp_tx += xdp_tx;
4849 		es->ps.xdp_tx_err += xdp_tx_err;
4850 	}
4851 }
4852 
mvneta_ethtool_update_stats(struct mvneta_port * pp)4853 static void mvneta_ethtool_update_stats(struct mvneta_port *pp)
4854 {
4855 	struct mvneta_ethtool_stats stats = {};
4856 	const struct mvneta_statistic *s;
4857 	void __iomem *base = pp->base;
4858 	u32 high, low;
4859 	u64 val;
4860 	int i;
4861 
4862 	mvneta_ethtool_update_pcpu_stats(pp, &stats);
4863 	for (i = 0, s = mvneta_statistics;
4864 	     s < mvneta_statistics + ARRAY_SIZE(mvneta_statistics);
4865 	     s++, i++) {
4866 		switch (s->type) {
4867 		case T_REG_32:
4868 			val = readl_relaxed(base + s->offset);
4869 			pp->ethtool_stats[i] += val;
4870 			break;
4871 		case T_REG_64:
4872 			/* Docs say to read low 32-bit then high */
4873 			low = readl_relaxed(base + s->offset);
4874 			high = readl_relaxed(base + s->offset + 4);
4875 			val = (u64)high << 32 | low;
4876 			pp->ethtool_stats[i] += val;
4877 			break;
4878 		case T_SW:
4879 			switch (s->offset) {
4880 			case ETHTOOL_STAT_EEE_WAKEUP:
4881 				val = phylink_get_eee_err(pp->phylink);
4882 				pp->ethtool_stats[i] += val;
4883 				break;
4884 			case ETHTOOL_STAT_SKB_ALLOC_ERR:
4885 				pp->ethtool_stats[i] = stats.skb_alloc_error;
4886 				break;
4887 			case ETHTOOL_STAT_REFILL_ERR:
4888 				pp->ethtool_stats[i] = stats.refill_error;
4889 				break;
4890 			case ETHTOOL_XDP_REDIRECT:
4891 				pp->ethtool_stats[i] = stats.ps.xdp_redirect;
4892 				break;
4893 			case ETHTOOL_XDP_PASS:
4894 				pp->ethtool_stats[i] = stats.ps.xdp_pass;
4895 				break;
4896 			case ETHTOOL_XDP_DROP:
4897 				pp->ethtool_stats[i] = stats.ps.xdp_drop;
4898 				break;
4899 			case ETHTOOL_XDP_TX:
4900 				pp->ethtool_stats[i] = stats.ps.xdp_tx;
4901 				break;
4902 			case ETHTOOL_XDP_TX_ERR:
4903 				pp->ethtool_stats[i] = stats.ps.xdp_tx_err;
4904 				break;
4905 			case ETHTOOL_XDP_XMIT:
4906 				pp->ethtool_stats[i] = stats.ps.xdp_xmit;
4907 				break;
4908 			case ETHTOOL_XDP_XMIT_ERR:
4909 				pp->ethtool_stats[i] = stats.ps.xdp_xmit_err;
4910 				break;
4911 			}
4912 			break;
4913 		}
4914 	}
4915 }
4916 
mvneta_ethtool_pp_stats(struct mvneta_port * pp,u64 * data)4917 static void mvneta_ethtool_pp_stats(struct mvneta_port *pp, u64 *data)
4918 {
4919 	struct page_pool_stats stats = {};
4920 	int i;
4921 
4922 	for (i = 0; i < rxq_number; i++) {
4923 		if (pp->rxqs[i].page_pool)
4924 			page_pool_get_stats(pp->rxqs[i].page_pool, &stats);
4925 	}
4926 
4927 	page_pool_ethtool_stats_get(data, &stats);
4928 }
4929 
mvneta_ethtool_get_stats(struct net_device * dev,struct ethtool_stats * stats,u64 * data)4930 static void mvneta_ethtool_get_stats(struct net_device *dev,
4931 				     struct ethtool_stats *stats, u64 *data)
4932 {
4933 	struct mvneta_port *pp = netdev_priv(dev);
4934 	int i;
4935 
4936 	mvneta_ethtool_update_stats(pp);
4937 
4938 	for (i = 0; i < ARRAY_SIZE(mvneta_statistics); i++)
4939 		*data++ = pp->ethtool_stats[i];
4940 
4941 	if (!pp->bm_priv)
4942 		mvneta_ethtool_pp_stats(pp, data);
4943 }
4944 
mvneta_ethtool_get_sset_count(struct net_device * dev,int sset)4945 static int mvneta_ethtool_get_sset_count(struct net_device *dev, int sset)
4946 {
4947 	if (sset == ETH_SS_STATS) {
4948 		int count = ARRAY_SIZE(mvneta_statistics);
4949 		struct mvneta_port *pp = netdev_priv(dev);
4950 
4951 		if (!pp->bm_priv)
4952 			count += page_pool_ethtool_stats_get_count();
4953 
4954 		return count;
4955 	}
4956 
4957 	return -EOPNOTSUPP;
4958 }
4959 
mvneta_ethtool_get_rxfh_indir_size(struct net_device * dev)4960 static u32 mvneta_ethtool_get_rxfh_indir_size(struct net_device *dev)
4961 {
4962 	return MVNETA_RSS_LU_TABLE_SIZE;
4963 }
4964 
mvneta_ethtool_get_rxnfc(struct net_device * dev,struct ethtool_rxnfc * info,u32 * rules __always_unused)4965 static int mvneta_ethtool_get_rxnfc(struct net_device *dev,
4966 				    struct ethtool_rxnfc *info,
4967 				    u32 *rules __always_unused)
4968 {
4969 	switch (info->cmd) {
4970 	case ETHTOOL_GRXRINGS:
4971 		info->data =  rxq_number;
4972 		return 0;
4973 	case ETHTOOL_GRXFH:
4974 		return -EOPNOTSUPP;
4975 	default:
4976 		return -EOPNOTSUPP;
4977 	}
4978 }
4979 
mvneta_config_rss(struct mvneta_port * pp)4980 static int  mvneta_config_rss(struct mvneta_port *pp)
4981 {
4982 	int cpu;
4983 	u32 val;
4984 
4985 	netif_tx_stop_all_queues(pp->dev);
4986 
4987 	on_each_cpu(mvneta_percpu_mask_interrupt, pp, true);
4988 
4989 	if (!pp->neta_armada3700) {
4990 		/* We have to synchronise on the napi of each CPU */
4991 		for_each_online_cpu(cpu) {
4992 			struct mvneta_pcpu_port *pcpu_port =
4993 				per_cpu_ptr(pp->ports, cpu);
4994 
4995 			napi_synchronize(&pcpu_port->napi);
4996 			napi_disable(&pcpu_port->napi);
4997 		}
4998 	} else {
4999 		napi_synchronize(&pp->napi);
5000 		napi_disable(&pp->napi);
5001 	}
5002 
5003 	pp->rxq_def = pp->indir[0];
5004 
5005 	/* Update unicast mapping */
5006 	mvneta_set_rx_mode(pp->dev);
5007 
5008 	/* Update val of portCfg register accordingly with all RxQueue types */
5009 	val = MVNETA_PORT_CONFIG_DEFL_VALUE(pp->rxq_def);
5010 	mvreg_write(pp, MVNETA_PORT_CONFIG, val);
5011 
5012 	/* Update the elected CPU matching the new rxq_def */
5013 	spin_lock(&pp->lock);
5014 	mvneta_percpu_elect(pp);
5015 	spin_unlock(&pp->lock);
5016 
5017 	if (!pp->neta_armada3700) {
5018 		/* We have to synchronise on the napi of each CPU */
5019 		for_each_online_cpu(cpu) {
5020 			struct mvneta_pcpu_port *pcpu_port =
5021 				per_cpu_ptr(pp->ports, cpu);
5022 
5023 			napi_enable(&pcpu_port->napi);
5024 		}
5025 	} else {
5026 		napi_enable(&pp->napi);
5027 	}
5028 
5029 	netif_tx_start_all_queues(pp->dev);
5030 
5031 	return 0;
5032 }
5033 
mvneta_ethtool_set_rxfh(struct net_device * dev,struct ethtool_rxfh_param * rxfh,struct netlink_ext_ack * extack)5034 static int mvneta_ethtool_set_rxfh(struct net_device *dev,
5035 				   struct ethtool_rxfh_param *rxfh,
5036 				   struct netlink_ext_ack *extack)
5037 {
5038 	struct mvneta_port *pp = netdev_priv(dev);
5039 
5040 	/* Current code for Armada 3700 doesn't support RSS features yet */
5041 	if (pp->neta_armada3700)
5042 		return -EOPNOTSUPP;
5043 
5044 	/* We require at least one supported parameter to be changed
5045 	 * and no change in any of the unsupported parameters
5046 	 */
5047 	if (rxfh->key ||
5048 	    (rxfh->hfunc != ETH_RSS_HASH_NO_CHANGE &&
5049 	     rxfh->hfunc != ETH_RSS_HASH_TOP))
5050 		return -EOPNOTSUPP;
5051 
5052 	if (!rxfh->indir)
5053 		return 0;
5054 
5055 	memcpy(pp->indir, rxfh->indir, MVNETA_RSS_LU_TABLE_SIZE);
5056 
5057 	return mvneta_config_rss(pp);
5058 }
5059 
mvneta_ethtool_get_rxfh(struct net_device * dev,struct ethtool_rxfh_param * rxfh)5060 static int mvneta_ethtool_get_rxfh(struct net_device *dev,
5061 				   struct ethtool_rxfh_param *rxfh)
5062 {
5063 	struct mvneta_port *pp = netdev_priv(dev);
5064 
5065 	/* Current code for Armada 3700 doesn't support RSS features yet */
5066 	if (pp->neta_armada3700)
5067 		return -EOPNOTSUPP;
5068 
5069 	rxfh->hfunc = ETH_RSS_HASH_TOP;
5070 
5071 	if (!rxfh->indir)
5072 		return 0;
5073 
5074 	memcpy(rxfh->indir, pp->indir, MVNETA_RSS_LU_TABLE_SIZE);
5075 
5076 	return 0;
5077 }
5078 
mvneta_ethtool_get_wol(struct net_device * dev,struct ethtool_wolinfo * wol)5079 static void mvneta_ethtool_get_wol(struct net_device *dev,
5080 				   struct ethtool_wolinfo *wol)
5081 {
5082 	struct mvneta_port *pp = netdev_priv(dev);
5083 
5084 	phylink_ethtool_get_wol(pp->phylink, wol);
5085 }
5086 
mvneta_ethtool_set_wol(struct net_device * dev,struct ethtool_wolinfo * wol)5087 static int mvneta_ethtool_set_wol(struct net_device *dev,
5088 				  struct ethtool_wolinfo *wol)
5089 {
5090 	struct mvneta_port *pp = netdev_priv(dev);
5091 	int ret;
5092 
5093 	ret = phylink_ethtool_set_wol(pp->phylink, wol);
5094 	if (!ret)
5095 		device_set_wakeup_enable(&dev->dev, !!wol->wolopts);
5096 
5097 	return ret;
5098 }
5099 
mvneta_ethtool_get_eee(struct net_device * dev,struct ethtool_keee * eee)5100 static int mvneta_ethtool_get_eee(struct net_device *dev,
5101 				  struct ethtool_keee *eee)
5102 {
5103 	struct mvneta_port *pp = netdev_priv(dev);
5104 	u32 lpi_ctl0;
5105 
5106 	lpi_ctl0 = mvreg_read(pp, MVNETA_LPI_CTRL_0);
5107 
5108 	eee->eee_enabled = pp->eee_enabled;
5109 	eee->eee_active = pp->eee_active;
5110 	eee->tx_lpi_enabled = pp->tx_lpi_enabled;
5111 	eee->tx_lpi_timer = (lpi_ctl0) >> 8; // * scale;
5112 
5113 	return phylink_ethtool_get_eee(pp->phylink, eee);
5114 }
5115 
mvneta_ethtool_set_eee(struct net_device * dev,struct ethtool_keee * eee)5116 static int mvneta_ethtool_set_eee(struct net_device *dev,
5117 				  struct ethtool_keee *eee)
5118 {
5119 	struct mvneta_port *pp = netdev_priv(dev);
5120 	u32 lpi_ctl0;
5121 
5122 	/* The Armada 37x documents do not give limits for this other than
5123 	 * it being an 8-bit register.
5124 	 */
5125 	if (eee->tx_lpi_enabled && eee->tx_lpi_timer > 255)
5126 		return -EINVAL;
5127 
5128 	lpi_ctl0 = mvreg_read(pp, MVNETA_LPI_CTRL_0);
5129 	lpi_ctl0 &= ~(0xff << 8);
5130 	lpi_ctl0 |= eee->tx_lpi_timer << 8;
5131 	mvreg_write(pp, MVNETA_LPI_CTRL_0, lpi_ctl0);
5132 
5133 	pp->eee_enabled = eee->eee_enabled;
5134 	pp->tx_lpi_enabled = eee->tx_lpi_enabled;
5135 
5136 	mvneta_set_eee(pp, eee->tx_lpi_enabled && eee->eee_enabled);
5137 
5138 	return phylink_ethtool_set_eee(pp->phylink, eee);
5139 }
5140 
mvneta_clear_rx_prio_map(struct mvneta_port * pp)5141 static void mvneta_clear_rx_prio_map(struct mvneta_port *pp)
5142 {
5143 	mvreg_write(pp, MVNETA_VLAN_PRIO_TO_RXQ, 0);
5144 }
5145 
mvneta_map_vlan_prio_to_rxq(struct mvneta_port * pp,u8 pri,u8 rxq)5146 static void mvneta_map_vlan_prio_to_rxq(struct mvneta_port *pp, u8 pri, u8 rxq)
5147 {
5148 	u32 val = mvreg_read(pp, MVNETA_VLAN_PRIO_TO_RXQ);
5149 
5150 	val &= ~MVNETA_VLAN_PRIO_RXQ_MAP(pri, 0x7);
5151 	val |= MVNETA_VLAN_PRIO_RXQ_MAP(pri, rxq);
5152 
5153 	mvreg_write(pp, MVNETA_VLAN_PRIO_TO_RXQ, val);
5154 }
5155 
mvneta_enable_per_queue_rate_limit(struct mvneta_port * pp)5156 static int mvneta_enable_per_queue_rate_limit(struct mvneta_port *pp)
5157 {
5158 	unsigned long core_clk_rate;
5159 	u32 refill_cycles;
5160 	u32 val;
5161 
5162 	core_clk_rate = clk_get_rate(pp->clk);
5163 	if (!core_clk_rate)
5164 		return -EINVAL;
5165 
5166 	refill_cycles = MVNETA_TXQ_BUCKET_REFILL_BASE_PERIOD_NS /
5167 			(NSEC_PER_SEC / core_clk_rate);
5168 
5169 	if (refill_cycles > MVNETA_REFILL_MAX_NUM_CLK)
5170 		return -EINVAL;
5171 
5172 	/* Enable bw limit algorithm version 3 */
5173 	val = mvreg_read(pp, MVNETA_TXQ_CMD1_REG);
5174 	val &= ~(MVNETA_TXQ_CMD1_BW_LIM_SEL_V1 | MVNETA_TXQ_CMD1_BW_LIM_EN);
5175 	mvreg_write(pp, MVNETA_TXQ_CMD1_REG, val);
5176 
5177 	/* Set the base refill rate */
5178 	mvreg_write(pp, MVNETA_REFILL_NUM_CLK_REG, refill_cycles);
5179 
5180 	return 0;
5181 }
5182 
mvneta_disable_per_queue_rate_limit(struct mvneta_port * pp)5183 static void mvneta_disable_per_queue_rate_limit(struct mvneta_port *pp)
5184 {
5185 	u32 val = mvreg_read(pp, MVNETA_TXQ_CMD1_REG);
5186 
5187 	val |= (MVNETA_TXQ_CMD1_BW_LIM_SEL_V1 | MVNETA_TXQ_CMD1_BW_LIM_EN);
5188 	mvreg_write(pp, MVNETA_TXQ_CMD1_REG, val);
5189 }
5190 
mvneta_setup_queue_rates(struct mvneta_port * pp,int queue,u64 min_rate,u64 max_rate)5191 static int mvneta_setup_queue_rates(struct mvneta_port *pp, int queue,
5192 				    u64 min_rate, u64 max_rate)
5193 {
5194 	u32 refill_val, rem;
5195 	u32 val = 0;
5196 
5197 	/* Convert to from Bps to bps */
5198 	max_rate *= 8;
5199 
5200 	if (min_rate)
5201 		return -EINVAL;
5202 
5203 	refill_val = div_u64_rem(max_rate, MVNETA_TXQ_RATE_LIMIT_RESOLUTION,
5204 				 &rem);
5205 
5206 	if (rem || !refill_val ||
5207 	    refill_val > MVNETA_TXQ_BUCKET_REFILL_VALUE_MAX)
5208 		return -EINVAL;
5209 
5210 	val = refill_val;
5211 	val |= (MVNETA_TXQ_BUCKET_REFILL_PERIOD <<
5212 		MVNETA_TXQ_BUCKET_REFILL_PERIOD_SHIFT);
5213 
5214 	mvreg_write(pp, MVNETA_TXQ_BUCKET_REFILL_REG(queue), val);
5215 
5216 	return 0;
5217 }
5218 
mvneta_setup_mqprio(struct net_device * dev,struct tc_mqprio_qopt_offload * mqprio)5219 static int mvneta_setup_mqprio(struct net_device *dev,
5220 			       struct tc_mqprio_qopt_offload *mqprio)
5221 {
5222 	struct mvneta_port *pp = netdev_priv(dev);
5223 	int rxq, txq, tc, ret;
5224 	u8 num_tc;
5225 
5226 	if (mqprio->qopt.hw != TC_MQPRIO_HW_OFFLOAD_TCS)
5227 		return 0;
5228 
5229 	num_tc = mqprio->qopt.num_tc;
5230 
5231 	if (num_tc > rxq_number)
5232 		return -EINVAL;
5233 
5234 	mvneta_clear_rx_prio_map(pp);
5235 
5236 	if (!num_tc) {
5237 		mvneta_disable_per_queue_rate_limit(pp);
5238 		netdev_reset_tc(dev);
5239 		return 0;
5240 	}
5241 
5242 	netdev_set_num_tc(dev, mqprio->qopt.num_tc);
5243 
5244 	for (tc = 0; tc < mqprio->qopt.num_tc; tc++) {
5245 		netdev_set_tc_queue(dev, tc, mqprio->qopt.count[tc],
5246 				    mqprio->qopt.offset[tc]);
5247 
5248 		for (rxq = mqprio->qopt.offset[tc];
5249 		     rxq < mqprio->qopt.count[tc] + mqprio->qopt.offset[tc];
5250 		     rxq++) {
5251 			if (rxq >= rxq_number)
5252 				return -EINVAL;
5253 
5254 			mvneta_map_vlan_prio_to_rxq(pp, tc, rxq);
5255 		}
5256 	}
5257 
5258 	if (mqprio->shaper != TC_MQPRIO_SHAPER_BW_RATE) {
5259 		mvneta_disable_per_queue_rate_limit(pp);
5260 		return 0;
5261 	}
5262 
5263 	if (mqprio->qopt.num_tc > txq_number)
5264 		return -EINVAL;
5265 
5266 	ret = mvneta_enable_per_queue_rate_limit(pp);
5267 	if (ret)
5268 		return ret;
5269 
5270 	for (tc = 0; tc < mqprio->qopt.num_tc; tc++) {
5271 		for (txq = mqprio->qopt.offset[tc];
5272 		     txq < mqprio->qopt.count[tc] + mqprio->qopt.offset[tc];
5273 		     txq++) {
5274 			if (txq >= txq_number)
5275 				return -EINVAL;
5276 
5277 			ret = mvneta_setup_queue_rates(pp, txq,
5278 						       mqprio->min_rate[tc],
5279 						       mqprio->max_rate[tc]);
5280 			if (ret)
5281 				return ret;
5282 		}
5283 	}
5284 
5285 	return 0;
5286 }
5287 
mvneta_setup_tc(struct net_device * dev,enum tc_setup_type type,void * type_data)5288 static int mvneta_setup_tc(struct net_device *dev, enum tc_setup_type type,
5289 			   void *type_data)
5290 {
5291 	switch (type) {
5292 	case TC_SETUP_QDISC_MQPRIO:
5293 		return mvneta_setup_mqprio(dev, type_data);
5294 	default:
5295 		return -EOPNOTSUPP;
5296 	}
5297 }
5298 
5299 static const struct net_device_ops mvneta_netdev_ops = {
5300 	.ndo_open            = mvneta_open,
5301 	.ndo_stop            = mvneta_stop,
5302 	.ndo_start_xmit      = mvneta_tx,
5303 	.ndo_set_rx_mode     = mvneta_set_rx_mode,
5304 	.ndo_set_mac_address = mvneta_set_mac_addr,
5305 	.ndo_change_mtu      = mvneta_change_mtu,
5306 	.ndo_fix_features    = mvneta_fix_features,
5307 	.ndo_get_stats64     = mvneta_get_stats64,
5308 	.ndo_eth_ioctl        = mvneta_ioctl,
5309 	.ndo_bpf	     = mvneta_xdp,
5310 	.ndo_xdp_xmit        = mvneta_xdp_xmit,
5311 	.ndo_setup_tc	     = mvneta_setup_tc,
5312 };
5313 
5314 static const struct ethtool_ops mvneta_eth_tool_ops = {
5315 	.supported_coalesce_params = ETHTOOL_COALESCE_RX_USECS |
5316 				     ETHTOOL_COALESCE_MAX_FRAMES,
5317 	.nway_reset	= mvneta_ethtool_nway_reset,
5318 	.get_link       = ethtool_op_get_link,
5319 	.set_coalesce   = mvneta_ethtool_set_coalesce,
5320 	.get_coalesce   = mvneta_ethtool_get_coalesce,
5321 	.get_drvinfo    = mvneta_ethtool_get_drvinfo,
5322 	.get_ringparam  = mvneta_ethtool_get_ringparam,
5323 	.set_ringparam	= mvneta_ethtool_set_ringparam,
5324 	.get_pauseparam	= mvneta_ethtool_get_pauseparam,
5325 	.set_pauseparam	= mvneta_ethtool_set_pauseparam,
5326 	.get_strings	= mvneta_ethtool_get_strings,
5327 	.get_ethtool_stats = mvneta_ethtool_get_stats,
5328 	.get_sset_count	= mvneta_ethtool_get_sset_count,
5329 	.get_rxfh_indir_size = mvneta_ethtool_get_rxfh_indir_size,
5330 	.get_rxnfc	= mvneta_ethtool_get_rxnfc,
5331 	.get_rxfh	= mvneta_ethtool_get_rxfh,
5332 	.set_rxfh	= mvneta_ethtool_set_rxfh,
5333 	.get_link_ksettings = mvneta_ethtool_get_link_ksettings,
5334 	.set_link_ksettings = mvneta_ethtool_set_link_ksettings,
5335 	.get_wol        = mvneta_ethtool_get_wol,
5336 	.set_wol        = mvneta_ethtool_set_wol,
5337 	.get_eee	= mvneta_ethtool_get_eee,
5338 	.set_eee	= mvneta_ethtool_set_eee,
5339 };
5340 
5341 /* Initialize hw */
mvneta_init(struct device * dev,struct mvneta_port * pp)5342 static int mvneta_init(struct device *dev, struct mvneta_port *pp)
5343 {
5344 	int queue;
5345 
5346 	/* Disable port */
5347 	mvneta_port_disable(pp);
5348 
5349 	/* Set port default values */
5350 	mvneta_defaults_set(pp);
5351 
5352 	pp->txqs = devm_kcalloc(dev, txq_number, sizeof(*pp->txqs), GFP_KERNEL);
5353 	if (!pp->txqs)
5354 		return -ENOMEM;
5355 
5356 	/* Initialize TX descriptor rings */
5357 	for (queue = 0; queue < txq_number; queue++) {
5358 		struct mvneta_tx_queue *txq = &pp->txqs[queue];
5359 		txq->id = queue;
5360 		txq->size = pp->tx_ring_size;
5361 		txq->done_pkts_coal = MVNETA_TXDONE_COAL_PKTS;
5362 	}
5363 
5364 	pp->rxqs = devm_kcalloc(dev, rxq_number, sizeof(*pp->rxqs), GFP_KERNEL);
5365 	if (!pp->rxqs)
5366 		return -ENOMEM;
5367 
5368 	/* Create Rx descriptor rings */
5369 	for (queue = 0; queue < rxq_number; queue++) {
5370 		struct mvneta_rx_queue *rxq = &pp->rxqs[queue];
5371 		rxq->id = queue;
5372 		rxq->size = pp->rx_ring_size;
5373 		rxq->pkts_coal = MVNETA_RX_COAL_PKTS;
5374 		rxq->time_coal = MVNETA_RX_COAL_USEC;
5375 		rxq->buf_virt_addr
5376 			= devm_kmalloc_array(pp->dev->dev.parent,
5377 					     rxq->size,
5378 					     sizeof(*rxq->buf_virt_addr),
5379 					     GFP_KERNEL);
5380 		if (!rxq->buf_virt_addr)
5381 			return -ENOMEM;
5382 	}
5383 
5384 	return 0;
5385 }
5386 
5387 /* platform glue : initialize decoding windows */
mvneta_conf_mbus_windows(struct mvneta_port * pp,const struct mbus_dram_target_info * dram)5388 static void mvneta_conf_mbus_windows(struct mvneta_port *pp,
5389 				     const struct mbus_dram_target_info *dram)
5390 {
5391 	u32 win_enable;
5392 	u32 win_protect;
5393 	int i;
5394 
5395 	for (i = 0; i < 6; i++) {
5396 		mvreg_write(pp, MVNETA_WIN_BASE(i), 0);
5397 		mvreg_write(pp, MVNETA_WIN_SIZE(i), 0);
5398 
5399 		if (i < 4)
5400 			mvreg_write(pp, MVNETA_WIN_REMAP(i), 0);
5401 	}
5402 
5403 	win_enable = 0x3f;
5404 	win_protect = 0;
5405 
5406 	if (dram) {
5407 		for (i = 0; i < dram->num_cs; i++) {
5408 			const struct mbus_dram_window *cs = dram->cs + i;
5409 
5410 			mvreg_write(pp, MVNETA_WIN_BASE(i),
5411 				    (cs->base & 0xffff0000) |
5412 				    (cs->mbus_attr << 8) |
5413 				    dram->mbus_dram_target_id);
5414 
5415 			mvreg_write(pp, MVNETA_WIN_SIZE(i),
5416 				    (cs->size - 1) & 0xffff0000);
5417 
5418 			win_enable &= ~(1 << i);
5419 			win_protect |= 3 << (2 * i);
5420 		}
5421 	} else {
5422 		if (pp->neta_ac5)
5423 			mvreg_write(pp, MVNETA_WIN_BASE(0),
5424 				    (MVNETA_AC5_CNM_DDR_ATTR << 8) |
5425 				    MVNETA_AC5_CNM_DDR_TARGET);
5426 		/* For Armada3700 open default 4GB Mbus window, leaving
5427 		 * arbitration of target/attribute to a different layer
5428 		 * of configuration.
5429 		 */
5430 		mvreg_write(pp, MVNETA_WIN_SIZE(0), 0xffff0000);
5431 		win_enable &= ~BIT(0);
5432 		win_protect = 3;
5433 	}
5434 
5435 	mvreg_write(pp, MVNETA_BASE_ADDR_ENABLE, win_enable);
5436 	mvreg_write(pp, MVNETA_ACCESS_PROTECT_ENABLE, win_protect);
5437 }
5438 
5439 /* Power up the port */
mvneta_port_power_up(struct mvneta_port * pp,int phy_mode)5440 static int mvneta_port_power_up(struct mvneta_port *pp, int phy_mode)
5441 {
5442 	/* MAC Cause register should be cleared */
5443 	mvreg_write(pp, MVNETA_UNIT_INTR_CAUSE, 0);
5444 
5445 	if (phy_mode != PHY_INTERFACE_MODE_QSGMII &&
5446 	    phy_mode != PHY_INTERFACE_MODE_SGMII &&
5447 	    !phy_interface_mode_is_8023z(phy_mode) &&
5448 	    !phy_interface_mode_is_rgmii(phy_mode))
5449 		return -EINVAL;
5450 
5451 	return 0;
5452 }
5453 
5454 /* Device initialization routine */
mvneta_probe(struct platform_device * pdev)5455 static int mvneta_probe(struct platform_device *pdev)
5456 {
5457 	struct device_node *dn = pdev->dev.of_node;
5458 	struct device_node *bm_node;
5459 	struct mvneta_port *pp;
5460 	struct net_device *dev;
5461 	struct phylink *phylink;
5462 	struct phy *comphy;
5463 	char hw_mac_addr[ETH_ALEN];
5464 	phy_interface_t phy_mode;
5465 	const char *mac_from;
5466 	int tx_csum_limit;
5467 	int err;
5468 	int cpu;
5469 
5470 	dev = devm_alloc_etherdev_mqs(&pdev->dev, sizeof(struct mvneta_port),
5471 				      txq_number, rxq_number);
5472 	if (!dev)
5473 		return -ENOMEM;
5474 
5475 	dev->tx_queue_len = MVNETA_MAX_TXD;
5476 	dev->watchdog_timeo = 5 * HZ;
5477 	dev->netdev_ops = &mvneta_netdev_ops;
5478 	dev->ethtool_ops = &mvneta_eth_tool_ops;
5479 
5480 	pp = netdev_priv(dev);
5481 	spin_lock_init(&pp->lock);
5482 	pp->dn = dn;
5483 
5484 	pp->rxq_def = rxq_def;
5485 	pp->indir[0] = rxq_def;
5486 
5487 	err = of_get_phy_mode(dn, &phy_mode);
5488 	if (err) {
5489 		dev_err(&pdev->dev, "incorrect phy-mode\n");
5490 		return err;
5491 	}
5492 
5493 	pp->phy_interface = phy_mode;
5494 
5495 	comphy = devm_of_phy_get(&pdev->dev, dn, NULL);
5496 	if (comphy == ERR_PTR(-EPROBE_DEFER))
5497 		return -EPROBE_DEFER;
5498 
5499 	if (IS_ERR(comphy))
5500 		comphy = NULL;
5501 
5502 	pp->comphy = comphy;
5503 
5504 	pp->base = devm_platform_ioremap_resource(pdev, 0);
5505 	if (IS_ERR(pp->base))
5506 		return PTR_ERR(pp->base);
5507 
5508 	/* Get special SoC configurations */
5509 	if (of_device_is_compatible(dn, "marvell,armada-3700-neta"))
5510 		pp->neta_armada3700 = true;
5511 	if (of_device_is_compatible(dn, "marvell,armada-ac5-neta")) {
5512 		pp->neta_armada3700 = true;
5513 		pp->neta_ac5 = true;
5514 	}
5515 
5516 	dev->irq = irq_of_parse_and_map(dn, 0);
5517 	if (dev->irq == 0)
5518 		return -EINVAL;
5519 
5520 	pp->clk = devm_clk_get(&pdev->dev, "core");
5521 	if (IS_ERR(pp->clk))
5522 		pp->clk = devm_clk_get(&pdev->dev, NULL);
5523 	if (IS_ERR(pp->clk)) {
5524 		err = PTR_ERR(pp->clk);
5525 		goto err_free_irq;
5526 	}
5527 
5528 	clk_prepare_enable(pp->clk);
5529 
5530 	pp->clk_bus = devm_clk_get(&pdev->dev, "bus");
5531 	if (!IS_ERR(pp->clk_bus))
5532 		clk_prepare_enable(pp->clk_bus);
5533 
5534 	pp->phylink_pcs.ops = &mvneta_phylink_pcs_ops;
5535 	pp->phylink_pcs.neg_mode = true;
5536 
5537 	pp->phylink_config.dev = &dev->dev;
5538 	pp->phylink_config.type = PHYLINK_NETDEV;
5539 	pp->phylink_config.mac_capabilities = MAC_SYM_PAUSE | MAC_10 |
5540 		MAC_100 | MAC_1000FD | MAC_2500FD;
5541 
5542 	phy_interface_set_rgmii(pp->phylink_config.supported_interfaces);
5543 	__set_bit(PHY_INTERFACE_MODE_QSGMII,
5544 		  pp->phylink_config.supported_interfaces);
5545 	if (comphy) {
5546 		/* If a COMPHY is present, we can support any of the serdes
5547 		 * modes and switch between them.
5548 		 */
5549 		__set_bit(PHY_INTERFACE_MODE_SGMII,
5550 			  pp->phylink_config.supported_interfaces);
5551 		__set_bit(PHY_INTERFACE_MODE_1000BASEX,
5552 			  pp->phylink_config.supported_interfaces);
5553 		__set_bit(PHY_INTERFACE_MODE_2500BASEX,
5554 			  pp->phylink_config.supported_interfaces);
5555 	} else if (phy_mode == PHY_INTERFACE_MODE_2500BASEX) {
5556 		/* No COMPHY, with only 2500BASE-X mode supported */
5557 		__set_bit(PHY_INTERFACE_MODE_2500BASEX,
5558 			  pp->phylink_config.supported_interfaces);
5559 	} else if (phy_mode == PHY_INTERFACE_MODE_1000BASEX ||
5560 		   phy_mode == PHY_INTERFACE_MODE_SGMII) {
5561 		/* No COMPHY, we can switch between 1000BASE-X and SGMII */
5562 		__set_bit(PHY_INTERFACE_MODE_1000BASEX,
5563 			  pp->phylink_config.supported_interfaces);
5564 		__set_bit(PHY_INTERFACE_MODE_SGMII,
5565 			  pp->phylink_config.supported_interfaces);
5566 	}
5567 
5568 	phylink = phylink_create(&pp->phylink_config, pdev->dev.fwnode,
5569 				 phy_mode, &mvneta_phylink_ops);
5570 	if (IS_ERR(phylink)) {
5571 		err = PTR_ERR(phylink);
5572 		goto err_clk;
5573 	}
5574 
5575 	pp->phylink = phylink;
5576 
5577 	/* Alloc per-cpu port structure */
5578 	pp->ports = alloc_percpu(struct mvneta_pcpu_port);
5579 	if (!pp->ports) {
5580 		err = -ENOMEM;
5581 		goto err_free_phylink;
5582 	}
5583 
5584 	/* Alloc per-cpu stats */
5585 	pp->stats = netdev_alloc_pcpu_stats(struct mvneta_pcpu_stats);
5586 	if (!pp->stats) {
5587 		err = -ENOMEM;
5588 		goto err_free_ports;
5589 	}
5590 
5591 	err = of_get_ethdev_address(dn, dev);
5592 	if (!err) {
5593 		mac_from = "device tree";
5594 	} else {
5595 		mvneta_get_mac_addr(pp, hw_mac_addr);
5596 		if (is_valid_ether_addr(hw_mac_addr)) {
5597 			mac_from = "hardware";
5598 			eth_hw_addr_set(dev, hw_mac_addr);
5599 		} else {
5600 			mac_from = "random";
5601 			eth_hw_addr_random(dev);
5602 		}
5603 	}
5604 
5605 	if (!of_property_read_u32(dn, "tx-csum-limit", &tx_csum_limit)) {
5606 		if (tx_csum_limit < 0 ||
5607 		    tx_csum_limit > MVNETA_TX_CSUM_MAX_SIZE) {
5608 			tx_csum_limit = MVNETA_TX_CSUM_DEF_SIZE;
5609 			dev_info(&pdev->dev,
5610 				 "Wrong TX csum limit in DT, set to %dB\n",
5611 				 MVNETA_TX_CSUM_DEF_SIZE);
5612 		}
5613 	} else if (of_device_is_compatible(dn, "marvell,armada-370-neta")) {
5614 		tx_csum_limit = MVNETA_TX_CSUM_DEF_SIZE;
5615 	} else {
5616 		tx_csum_limit = MVNETA_TX_CSUM_MAX_SIZE;
5617 	}
5618 
5619 	pp->tx_csum_limit = tx_csum_limit;
5620 
5621 	pp->dram_target_info = mv_mbus_dram_info();
5622 	/* Armada3700 requires setting default configuration of Mbus
5623 	 * windows, however without using filled mbus_dram_target_info
5624 	 * structure.
5625 	 */
5626 	if (pp->dram_target_info || pp->neta_armada3700)
5627 		mvneta_conf_mbus_windows(pp, pp->dram_target_info);
5628 
5629 	pp->tx_ring_size = MVNETA_MAX_TXD;
5630 	pp->rx_ring_size = MVNETA_MAX_RXD;
5631 
5632 	pp->dev = dev;
5633 	SET_NETDEV_DEV(dev, &pdev->dev);
5634 
5635 	pp->id = global_port_id++;
5636 
5637 	/* Obtain access to BM resources if enabled and already initialized */
5638 	bm_node = of_parse_phandle(dn, "buffer-manager", 0);
5639 	if (bm_node) {
5640 		pp->bm_priv = mvneta_bm_get(bm_node);
5641 		if (pp->bm_priv) {
5642 			err = mvneta_bm_port_init(pdev, pp);
5643 			if (err < 0) {
5644 				dev_info(&pdev->dev,
5645 					 "use SW buffer management\n");
5646 				mvneta_bm_put(pp->bm_priv);
5647 				pp->bm_priv = NULL;
5648 			}
5649 		}
5650 		/* Set RX packet offset correction for platforms, whose
5651 		 * NET_SKB_PAD, exceeds 64B. It should be 64B for 64-bit
5652 		 * platforms and 0B for 32-bit ones.
5653 		 */
5654 		pp->rx_offset_correction = max(0,
5655 					       NET_SKB_PAD -
5656 					       MVNETA_RX_PKT_OFFSET_CORRECTION);
5657 	}
5658 	of_node_put(bm_node);
5659 
5660 	/* sw buffer management */
5661 	if (!pp->bm_priv)
5662 		pp->rx_offset_correction = MVNETA_SKB_HEADROOM;
5663 
5664 	err = mvneta_init(&pdev->dev, pp);
5665 	if (err < 0)
5666 		goto err_netdev;
5667 
5668 	err = mvneta_port_power_up(pp, pp->phy_interface);
5669 	if (err < 0) {
5670 		dev_err(&pdev->dev, "can't power up port\n");
5671 		goto err_netdev;
5672 	}
5673 
5674 	/* Armada3700 network controller does not support per-cpu
5675 	 * operation, so only single NAPI should be initialized.
5676 	 */
5677 	if (pp->neta_armada3700) {
5678 		netif_napi_add(dev, &pp->napi, mvneta_poll);
5679 	} else {
5680 		for_each_present_cpu(cpu) {
5681 			struct mvneta_pcpu_port *port =
5682 				per_cpu_ptr(pp->ports, cpu);
5683 
5684 			netif_napi_add(dev, &port->napi, mvneta_poll);
5685 			port->pp = pp;
5686 		}
5687 	}
5688 
5689 	dev->features = NETIF_F_SG | NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM |
5690 			NETIF_F_TSO | NETIF_F_RXCSUM;
5691 	dev->hw_features |= dev->features;
5692 	dev->vlan_features |= dev->features;
5693 	if (!pp->bm_priv)
5694 		dev->xdp_features = NETDEV_XDP_ACT_BASIC |
5695 				    NETDEV_XDP_ACT_REDIRECT |
5696 				    NETDEV_XDP_ACT_NDO_XMIT |
5697 				    NETDEV_XDP_ACT_RX_SG |
5698 				    NETDEV_XDP_ACT_NDO_XMIT_SG;
5699 	dev->priv_flags |= IFF_LIVE_ADDR_CHANGE;
5700 	netif_set_tso_max_segs(dev, MVNETA_MAX_TSO_SEGS);
5701 
5702 	/* MTU range: 68 - 9676 */
5703 	dev->min_mtu = ETH_MIN_MTU;
5704 	/* 9676 == 9700 - 20 and rounding to 8 */
5705 	dev->max_mtu = 9676;
5706 
5707 	err = register_netdev(dev);
5708 	if (err < 0) {
5709 		dev_err(&pdev->dev, "failed to register\n");
5710 		goto err_netdev;
5711 	}
5712 
5713 	netdev_info(dev, "Using %s mac address %pM\n", mac_from,
5714 		    dev->dev_addr);
5715 
5716 	platform_set_drvdata(pdev, pp->dev);
5717 
5718 	return 0;
5719 
5720 err_netdev:
5721 	if (pp->bm_priv) {
5722 		mvneta_bm_pool_destroy(pp->bm_priv, pp->pool_long, 1 << pp->id);
5723 		mvneta_bm_pool_destroy(pp->bm_priv, pp->pool_short,
5724 				       1 << pp->id);
5725 		mvneta_bm_put(pp->bm_priv);
5726 	}
5727 	free_percpu(pp->stats);
5728 err_free_ports:
5729 	free_percpu(pp->ports);
5730 err_free_phylink:
5731 	if (pp->phylink)
5732 		phylink_destroy(pp->phylink);
5733 err_clk:
5734 	clk_disable_unprepare(pp->clk_bus);
5735 	clk_disable_unprepare(pp->clk);
5736 err_free_irq:
5737 	irq_dispose_mapping(dev->irq);
5738 	return err;
5739 }
5740 
5741 /* Device removal routine */
mvneta_remove(struct platform_device * pdev)5742 static void mvneta_remove(struct platform_device *pdev)
5743 {
5744 	struct net_device  *dev = platform_get_drvdata(pdev);
5745 	struct mvneta_port *pp = netdev_priv(dev);
5746 
5747 	unregister_netdev(dev);
5748 	clk_disable_unprepare(pp->clk_bus);
5749 	clk_disable_unprepare(pp->clk);
5750 	free_percpu(pp->ports);
5751 	free_percpu(pp->stats);
5752 	irq_dispose_mapping(dev->irq);
5753 	phylink_destroy(pp->phylink);
5754 
5755 	if (pp->bm_priv) {
5756 		mvneta_bm_pool_destroy(pp->bm_priv, pp->pool_long, 1 << pp->id);
5757 		mvneta_bm_pool_destroy(pp->bm_priv, pp->pool_short,
5758 				       1 << pp->id);
5759 		mvneta_bm_put(pp->bm_priv);
5760 	}
5761 }
5762 
5763 #ifdef CONFIG_PM_SLEEP
mvneta_suspend(struct device * device)5764 static int mvneta_suspend(struct device *device)
5765 {
5766 	int queue;
5767 	struct net_device *dev = dev_get_drvdata(device);
5768 	struct mvneta_port *pp = netdev_priv(dev);
5769 
5770 	if (!netif_running(dev))
5771 		goto clean_exit;
5772 
5773 	if (!pp->neta_armada3700) {
5774 		spin_lock(&pp->lock);
5775 		pp->is_stopped = true;
5776 		spin_unlock(&pp->lock);
5777 
5778 		cpuhp_state_remove_instance_nocalls(online_hpstate,
5779 						    &pp->node_online);
5780 		cpuhp_state_remove_instance_nocalls(CPUHP_NET_MVNETA_DEAD,
5781 						    &pp->node_dead);
5782 	}
5783 
5784 	rtnl_lock();
5785 	mvneta_stop_dev(pp);
5786 	rtnl_unlock();
5787 
5788 	for (queue = 0; queue < rxq_number; queue++) {
5789 		struct mvneta_rx_queue *rxq = &pp->rxqs[queue];
5790 
5791 		mvneta_rxq_drop_pkts(pp, rxq);
5792 	}
5793 
5794 	for (queue = 0; queue < txq_number; queue++) {
5795 		struct mvneta_tx_queue *txq = &pp->txqs[queue];
5796 
5797 		mvneta_txq_hw_deinit(pp, txq);
5798 	}
5799 
5800 clean_exit:
5801 	netif_device_detach(dev);
5802 	clk_disable_unprepare(pp->clk_bus);
5803 	clk_disable_unprepare(pp->clk);
5804 
5805 	return 0;
5806 }
5807 
mvneta_resume(struct device * device)5808 static int mvneta_resume(struct device *device)
5809 {
5810 	struct platform_device *pdev = to_platform_device(device);
5811 	struct net_device *dev = dev_get_drvdata(device);
5812 	struct mvneta_port *pp = netdev_priv(dev);
5813 	int err, queue;
5814 
5815 	clk_prepare_enable(pp->clk);
5816 	if (!IS_ERR(pp->clk_bus))
5817 		clk_prepare_enable(pp->clk_bus);
5818 	if (pp->dram_target_info || pp->neta_armada3700)
5819 		mvneta_conf_mbus_windows(pp, pp->dram_target_info);
5820 	if (pp->bm_priv) {
5821 		err = mvneta_bm_port_init(pdev, pp);
5822 		if (err < 0) {
5823 			dev_info(&pdev->dev, "use SW buffer management\n");
5824 			pp->rx_offset_correction = MVNETA_SKB_HEADROOM;
5825 			pp->bm_priv = NULL;
5826 		}
5827 	}
5828 	mvneta_defaults_set(pp);
5829 	err = mvneta_port_power_up(pp, pp->phy_interface);
5830 	if (err < 0) {
5831 		dev_err(device, "can't power up port\n");
5832 		return err;
5833 	}
5834 
5835 	netif_device_attach(dev);
5836 
5837 	if (!netif_running(dev))
5838 		return 0;
5839 
5840 	for (queue = 0; queue < rxq_number; queue++) {
5841 		struct mvneta_rx_queue *rxq = &pp->rxqs[queue];
5842 
5843 		rxq->next_desc_to_proc = 0;
5844 		mvneta_rxq_hw_init(pp, rxq);
5845 	}
5846 
5847 	for (queue = 0; queue < txq_number; queue++) {
5848 		struct mvneta_tx_queue *txq = &pp->txqs[queue];
5849 
5850 		txq->next_desc_to_proc = 0;
5851 		mvneta_txq_hw_init(pp, txq);
5852 	}
5853 
5854 	if (!pp->neta_armada3700) {
5855 		spin_lock(&pp->lock);
5856 		pp->is_stopped = false;
5857 		spin_unlock(&pp->lock);
5858 		cpuhp_state_add_instance_nocalls(online_hpstate,
5859 						 &pp->node_online);
5860 		cpuhp_state_add_instance_nocalls(CPUHP_NET_MVNETA_DEAD,
5861 						 &pp->node_dead);
5862 	}
5863 
5864 	rtnl_lock();
5865 	mvneta_start_dev(pp);
5866 	rtnl_unlock();
5867 	mvneta_set_rx_mode(dev);
5868 
5869 	return 0;
5870 }
5871 #endif
5872 
5873 static SIMPLE_DEV_PM_OPS(mvneta_pm_ops, mvneta_suspend, mvneta_resume);
5874 
5875 static const struct of_device_id mvneta_match[] = {
5876 	{ .compatible = "marvell,armada-370-neta" },
5877 	{ .compatible = "marvell,armada-xp-neta" },
5878 	{ .compatible = "marvell,armada-3700-neta" },
5879 	{ .compatible = "marvell,armada-ac5-neta" },
5880 	{ }
5881 };
5882 MODULE_DEVICE_TABLE(of, mvneta_match);
5883 
5884 static struct platform_driver mvneta_driver = {
5885 	.probe = mvneta_probe,
5886 	.remove_new = mvneta_remove,
5887 	.driver = {
5888 		.name = MVNETA_DRIVER_NAME,
5889 		.of_match_table = mvneta_match,
5890 		.pm = &mvneta_pm_ops,
5891 	},
5892 };
5893 
mvneta_driver_init(void)5894 static int __init mvneta_driver_init(void)
5895 {
5896 	int ret;
5897 
5898 	BUILD_BUG_ON_NOT_POWER_OF_2(MVNETA_TSO_PER_PAGE);
5899 
5900 	ret = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN, "net/mvneta:online",
5901 				      mvneta_cpu_online,
5902 				      mvneta_cpu_down_prepare);
5903 	if (ret < 0)
5904 		goto out;
5905 	online_hpstate = ret;
5906 	ret = cpuhp_setup_state_multi(CPUHP_NET_MVNETA_DEAD, "net/mvneta:dead",
5907 				      NULL, mvneta_cpu_dead);
5908 	if (ret)
5909 		goto err_dead;
5910 
5911 	ret = platform_driver_register(&mvneta_driver);
5912 	if (ret)
5913 		goto err;
5914 	return 0;
5915 
5916 err:
5917 	cpuhp_remove_multi_state(CPUHP_NET_MVNETA_DEAD);
5918 err_dead:
5919 	cpuhp_remove_multi_state(online_hpstate);
5920 out:
5921 	return ret;
5922 }
5923 module_init(mvneta_driver_init);
5924 
mvneta_driver_exit(void)5925 static void __exit mvneta_driver_exit(void)
5926 {
5927 	platform_driver_unregister(&mvneta_driver);
5928 	cpuhp_remove_multi_state(CPUHP_NET_MVNETA_DEAD);
5929 	cpuhp_remove_multi_state(online_hpstate);
5930 }
5931 module_exit(mvneta_driver_exit);
5932 
5933 MODULE_DESCRIPTION("Marvell NETA Ethernet Driver - www.marvell.com");
5934 MODULE_AUTHOR("Rami Rosen <rosenr@marvell.com>, Thomas Petazzoni <thomas.petazzoni@free-electrons.com>");
5935 MODULE_LICENSE("GPL");
5936 
5937 module_param(rxq_number, int, 0444);
5938 module_param(txq_number, int, 0444);
5939 
5940 module_param(rxq_def, int, 0444);
5941 module_param(rx_copybreak, int, 0644);
5942