xref: /linux/drivers/net/ethernet/realtek/r8169_main.c (revision 26fbb4c8c7c3ee9a4c3b4de555a8587b5a19154e)
1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * r8169.c: RealTek 8169/8168/8101 ethernet driver.
4  *
5  * Copyright (c) 2002 ShuChen <shuchen@realtek.com.tw>
6  * Copyright (c) 2003 - 2007 Francois Romieu <romieu@fr.zoreil.com>
7  * Copyright (c) a lot of people too. Please respect their work.
8  *
9  * See MAINTAINERS file for support contact information.
10  */
11 
12 #include <linux/module.h>
13 #include <linux/pci.h>
14 #include <linux/netdevice.h>
15 #include <linux/etherdevice.h>
16 #include <linux/clk.h>
17 #include <linux/delay.h>
18 #include <linux/ethtool.h>
19 #include <linux/phy.h>
20 #include <linux/if_vlan.h>
21 #include <linux/in.h>
22 #include <linux/io.h>
23 #include <linux/ip.h>
24 #include <linux/tcp.h>
25 #include <linux/interrupt.h>
26 #include <linux/dma-mapping.h>
27 #include <linux/pm_runtime.h>
28 #include <linux/bitfield.h>
29 #include <linux/prefetch.h>
30 #include <linux/ipv6.h>
31 #include <net/ip6_checksum.h>
32 
33 #include "r8169.h"
34 #include "r8169_firmware.h"
35 
36 #define MODULENAME "r8169"
37 
38 #define FIRMWARE_8168D_1	"rtl_nic/rtl8168d-1.fw"
39 #define FIRMWARE_8168D_2	"rtl_nic/rtl8168d-2.fw"
40 #define FIRMWARE_8168E_1	"rtl_nic/rtl8168e-1.fw"
41 #define FIRMWARE_8168E_2	"rtl_nic/rtl8168e-2.fw"
42 #define FIRMWARE_8168E_3	"rtl_nic/rtl8168e-3.fw"
43 #define FIRMWARE_8168F_1	"rtl_nic/rtl8168f-1.fw"
44 #define FIRMWARE_8168F_2	"rtl_nic/rtl8168f-2.fw"
45 #define FIRMWARE_8105E_1	"rtl_nic/rtl8105e-1.fw"
46 #define FIRMWARE_8402_1		"rtl_nic/rtl8402-1.fw"
47 #define FIRMWARE_8411_1		"rtl_nic/rtl8411-1.fw"
48 #define FIRMWARE_8411_2		"rtl_nic/rtl8411-2.fw"
49 #define FIRMWARE_8106E_1	"rtl_nic/rtl8106e-1.fw"
50 #define FIRMWARE_8106E_2	"rtl_nic/rtl8106e-2.fw"
51 #define FIRMWARE_8168G_2	"rtl_nic/rtl8168g-2.fw"
52 #define FIRMWARE_8168G_3	"rtl_nic/rtl8168g-3.fw"
53 #define FIRMWARE_8168H_1	"rtl_nic/rtl8168h-1.fw"
54 #define FIRMWARE_8168H_2	"rtl_nic/rtl8168h-2.fw"
55 #define FIRMWARE_8168FP_3	"rtl_nic/rtl8168fp-3.fw"
56 #define FIRMWARE_8107E_1	"rtl_nic/rtl8107e-1.fw"
57 #define FIRMWARE_8107E_2	"rtl_nic/rtl8107e-2.fw"
58 #define FIRMWARE_8125A_3	"rtl_nic/rtl8125a-3.fw"
59 #define FIRMWARE_8125B_2	"rtl_nic/rtl8125b-2.fw"
60 
61 /* Maximum number of multicast addresses to filter (vs. Rx-all-multicast).
62    The RTL chips use a 64 element hash table based on the Ethernet CRC. */
63 #define	MC_FILTER_LIMIT	32
64 
65 #define TX_DMA_BURST	7	/* Maximum PCI burst, '7' is unlimited */
66 #define InterFrameGap	0x03	/* 3 means InterFrameGap = the shortest one */
67 
68 #define R8169_REGS_SIZE		256
69 #define R8169_RX_BUF_SIZE	(SZ_16K - 1)
70 #define NUM_TX_DESC	256	/* Number of Tx descriptor registers */
71 #define NUM_RX_DESC	256	/* Number of Rx descriptor registers */
72 #define R8169_TX_RING_BYTES	(NUM_TX_DESC * sizeof(struct TxDesc))
73 #define R8169_RX_RING_BYTES	(NUM_RX_DESC * sizeof(struct RxDesc))
74 
75 #define OCP_STD_PHY_BASE	0xa400
76 
77 #define RTL_CFG_NO_GBIT	1
78 
79 /* write/read MMIO register */
80 #define RTL_W8(tp, reg, val8)	writeb((val8), tp->mmio_addr + (reg))
81 #define RTL_W16(tp, reg, val16)	writew((val16), tp->mmio_addr + (reg))
82 #define RTL_W32(tp, reg, val32)	writel((val32), tp->mmio_addr + (reg))
83 #define RTL_R8(tp, reg)		readb(tp->mmio_addr + (reg))
84 #define RTL_R16(tp, reg)		readw(tp->mmio_addr + (reg))
85 #define RTL_R32(tp, reg)		readl(tp->mmio_addr + (reg))
86 
87 #define JUMBO_4K	(4 * SZ_1K - VLAN_ETH_HLEN - ETH_FCS_LEN)
88 #define JUMBO_6K	(6 * SZ_1K - VLAN_ETH_HLEN - ETH_FCS_LEN)
89 #define JUMBO_7K	(7 * SZ_1K - VLAN_ETH_HLEN - ETH_FCS_LEN)
90 #define JUMBO_9K	(9 * SZ_1K - VLAN_ETH_HLEN - ETH_FCS_LEN)
91 
92 static const struct {
93 	const char *name;
94 	const char *fw_name;
95 } rtl_chip_infos[] = {
96 	/* PCI devices. */
97 	[RTL_GIGA_MAC_VER_02] = {"RTL8169s"				},
98 	[RTL_GIGA_MAC_VER_03] = {"RTL8110s"				},
99 	[RTL_GIGA_MAC_VER_04] = {"RTL8169sb/8110sb"			},
100 	[RTL_GIGA_MAC_VER_05] = {"RTL8169sc/8110sc"			},
101 	[RTL_GIGA_MAC_VER_06] = {"RTL8169sc/8110sc"			},
102 	/* PCI-E devices. */
103 	[RTL_GIGA_MAC_VER_07] = {"RTL8102e"				},
104 	[RTL_GIGA_MAC_VER_08] = {"RTL8102e"				},
105 	[RTL_GIGA_MAC_VER_09] = {"RTL8102e/RTL8103e"			},
106 	[RTL_GIGA_MAC_VER_10] = {"RTL8101e"				},
107 	[RTL_GIGA_MAC_VER_11] = {"RTL8168b/8111b"			},
108 	[RTL_GIGA_MAC_VER_12] = {"RTL8168b/8111b"			},
109 	[RTL_GIGA_MAC_VER_13] = {"RTL8101e/RTL8100e"			},
110 	[RTL_GIGA_MAC_VER_14] = {"RTL8401"				},
111 	[RTL_GIGA_MAC_VER_16] = {"RTL8101e"				},
112 	[RTL_GIGA_MAC_VER_17] = {"RTL8168b/8111b"			},
113 	[RTL_GIGA_MAC_VER_18] = {"RTL8168cp/8111cp"			},
114 	[RTL_GIGA_MAC_VER_19] = {"RTL8168c/8111c"			},
115 	[RTL_GIGA_MAC_VER_20] = {"RTL8168c/8111c"			},
116 	[RTL_GIGA_MAC_VER_21] = {"RTL8168c/8111c"			},
117 	[RTL_GIGA_MAC_VER_22] = {"RTL8168c/8111c"			},
118 	[RTL_GIGA_MAC_VER_23] = {"RTL8168cp/8111cp"			},
119 	[RTL_GIGA_MAC_VER_24] = {"RTL8168cp/8111cp"			},
120 	[RTL_GIGA_MAC_VER_25] = {"RTL8168d/8111d",	FIRMWARE_8168D_1},
121 	[RTL_GIGA_MAC_VER_26] = {"RTL8168d/8111d",	FIRMWARE_8168D_2},
122 	[RTL_GIGA_MAC_VER_27] = {"RTL8168dp/8111dp"			},
123 	[RTL_GIGA_MAC_VER_28] = {"RTL8168dp/8111dp"			},
124 	[RTL_GIGA_MAC_VER_29] = {"RTL8105e",		FIRMWARE_8105E_1},
125 	[RTL_GIGA_MAC_VER_30] = {"RTL8105e",		FIRMWARE_8105E_1},
126 	[RTL_GIGA_MAC_VER_31] = {"RTL8168dp/8111dp"			},
127 	[RTL_GIGA_MAC_VER_32] = {"RTL8168e/8111e",	FIRMWARE_8168E_1},
128 	[RTL_GIGA_MAC_VER_33] = {"RTL8168e/8111e",	FIRMWARE_8168E_2},
129 	[RTL_GIGA_MAC_VER_34] = {"RTL8168evl/8111evl",	FIRMWARE_8168E_3},
130 	[RTL_GIGA_MAC_VER_35] = {"RTL8168f/8111f",	FIRMWARE_8168F_1},
131 	[RTL_GIGA_MAC_VER_36] = {"RTL8168f/8111f",	FIRMWARE_8168F_2},
132 	[RTL_GIGA_MAC_VER_37] = {"RTL8402",		FIRMWARE_8402_1 },
133 	[RTL_GIGA_MAC_VER_38] = {"RTL8411",		FIRMWARE_8411_1 },
134 	[RTL_GIGA_MAC_VER_39] = {"RTL8106e",		FIRMWARE_8106E_1},
135 	[RTL_GIGA_MAC_VER_40] = {"RTL8168g/8111g",	FIRMWARE_8168G_2},
136 	[RTL_GIGA_MAC_VER_41] = {"RTL8168g/8111g"			},
137 	[RTL_GIGA_MAC_VER_42] = {"RTL8168gu/8111gu",	FIRMWARE_8168G_3},
138 	[RTL_GIGA_MAC_VER_43] = {"RTL8106eus",		FIRMWARE_8106E_2},
139 	[RTL_GIGA_MAC_VER_44] = {"RTL8411b",		FIRMWARE_8411_2 },
140 	[RTL_GIGA_MAC_VER_45] = {"RTL8168h/8111h",	FIRMWARE_8168H_1},
141 	[RTL_GIGA_MAC_VER_46] = {"RTL8168h/8111h",	FIRMWARE_8168H_2},
142 	[RTL_GIGA_MAC_VER_47] = {"RTL8107e",		FIRMWARE_8107E_1},
143 	[RTL_GIGA_MAC_VER_48] = {"RTL8107e",		FIRMWARE_8107E_2},
144 	[RTL_GIGA_MAC_VER_49] = {"RTL8168ep/8111ep"			},
145 	[RTL_GIGA_MAC_VER_50] = {"RTL8168ep/8111ep"			},
146 	[RTL_GIGA_MAC_VER_51] = {"RTL8168ep/8111ep"			},
147 	[RTL_GIGA_MAC_VER_52] = {"RTL8168fp/RTL8117",  FIRMWARE_8168FP_3},
148 	[RTL_GIGA_MAC_VER_60] = {"RTL8125A"				},
149 	[RTL_GIGA_MAC_VER_61] = {"RTL8125A",		FIRMWARE_8125A_3},
150 	/* reserve 62 for CFG_METHOD_4 in the vendor driver */
151 	[RTL_GIGA_MAC_VER_63] = {"RTL8125B",		FIRMWARE_8125B_2},
152 };
153 
154 static const struct pci_device_id rtl8169_pci_tbl[] = {
155 	{ PCI_VDEVICE(REALTEK,	0x2502) },
156 	{ PCI_VDEVICE(REALTEK,	0x2600) },
157 	{ PCI_VDEVICE(REALTEK,	0x8129) },
158 	{ PCI_VDEVICE(REALTEK,	0x8136), RTL_CFG_NO_GBIT },
159 	{ PCI_VDEVICE(REALTEK,	0x8161) },
160 	{ PCI_VDEVICE(REALTEK,	0x8167) },
161 	{ PCI_VDEVICE(REALTEK,	0x8168) },
162 	{ PCI_VDEVICE(NCUBE,	0x8168) },
163 	{ PCI_VDEVICE(REALTEK,	0x8169) },
164 	{ PCI_VENDOR_ID_DLINK,	0x4300,
165 		PCI_VENDOR_ID_DLINK, 0x4b10, 0, 0 },
166 	{ PCI_VDEVICE(DLINK,	0x4300) },
167 	{ PCI_VDEVICE(DLINK,	0x4302) },
168 	{ PCI_VDEVICE(AT,	0xc107) },
169 	{ PCI_VDEVICE(USR,	0x0116) },
170 	{ PCI_VENDOR_ID_LINKSYS, 0x1032, PCI_ANY_ID, 0x0024 },
171 	{ 0x0001, 0x8168, PCI_ANY_ID, 0x2410 },
172 	{ PCI_VDEVICE(REALTEK,	0x8125) },
173 	{ PCI_VDEVICE(REALTEK,	0x3000) },
174 	{}
175 };
176 
177 MODULE_DEVICE_TABLE(pci, rtl8169_pci_tbl);
178 
179 enum rtl_registers {
180 	MAC0		= 0,	/* Ethernet hardware address. */
181 	MAC4		= 4,
182 	MAR0		= 8,	/* Multicast filter. */
183 	CounterAddrLow		= 0x10,
184 	CounterAddrHigh		= 0x14,
185 	TxDescStartAddrLow	= 0x20,
186 	TxDescStartAddrHigh	= 0x24,
187 	TxHDescStartAddrLow	= 0x28,
188 	TxHDescStartAddrHigh	= 0x2c,
189 	FLASH		= 0x30,
190 	ERSR		= 0x36,
191 	ChipCmd		= 0x37,
192 	TxPoll		= 0x38,
193 	IntrMask	= 0x3c,
194 	IntrStatus	= 0x3e,
195 
196 	TxConfig	= 0x40,
197 #define	TXCFG_AUTO_FIFO			(1 << 7)	/* 8111e-vl */
198 #define	TXCFG_EMPTY			(1 << 11)	/* 8111e-vl */
199 
200 	RxConfig	= 0x44,
201 #define	RX128_INT_EN			(1 << 15)	/* 8111c and later */
202 #define	RX_MULTI_EN			(1 << 14)	/* 8111c only */
203 #define	RXCFG_FIFO_SHIFT		13
204 					/* No threshold before first PCI xfer */
205 #define	RX_FIFO_THRESH			(7 << RXCFG_FIFO_SHIFT)
206 #define	RX_EARLY_OFF			(1 << 11)
207 #define	RXCFG_DMA_SHIFT			8
208 					/* Unlimited maximum PCI burst. */
209 #define	RX_DMA_BURST			(7 << RXCFG_DMA_SHIFT)
210 
211 	Cfg9346		= 0x50,
212 	Config0		= 0x51,
213 	Config1		= 0x52,
214 	Config2		= 0x53,
215 #define PME_SIGNAL			(1 << 5)	/* 8168c and later */
216 
217 	Config3		= 0x54,
218 	Config4		= 0x55,
219 	Config5		= 0x56,
220 	PHYAR		= 0x60,
221 	PHYstatus	= 0x6c,
222 	RxMaxSize	= 0xda,
223 	CPlusCmd	= 0xe0,
224 	IntrMitigate	= 0xe2,
225 
226 #define RTL_COALESCE_TX_USECS	GENMASK(15, 12)
227 #define RTL_COALESCE_TX_FRAMES	GENMASK(11, 8)
228 #define RTL_COALESCE_RX_USECS	GENMASK(7, 4)
229 #define RTL_COALESCE_RX_FRAMES	GENMASK(3, 0)
230 
231 #define RTL_COALESCE_T_MAX	0x0fU
232 #define RTL_COALESCE_FRAME_MAX	(RTL_COALESCE_T_MAX * 4)
233 
234 	RxDescAddrLow	= 0xe4,
235 	RxDescAddrHigh	= 0xe8,
236 	EarlyTxThres	= 0xec,	/* 8169. Unit of 32 bytes. */
237 
238 #define NoEarlyTx	0x3f	/* Max value : no early transmit. */
239 
240 	MaxTxPacketSize	= 0xec,	/* 8101/8168. Unit of 128 bytes. */
241 
242 #define TxPacketMax	(8064 >> 7)
243 #define EarlySize	0x27
244 
245 	FuncEvent	= 0xf0,
246 	FuncEventMask	= 0xf4,
247 	FuncPresetState	= 0xf8,
248 	IBCR0           = 0xf8,
249 	IBCR2           = 0xf9,
250 	IBIMR0          = 0xfa,
251 	IBISR0          = 0xfb,
252 	FuncForceEvent	= 0xfc,
253 };
254 
255 enum rtl8168_8101_registers {
256 	CSIDR			= 0x64,
257 	CSIAR			= 0x68,
258 #define	CSIAR_FLAG			0x80000000
259 #define	CSIAR_WRITE_CMD			0x80000000
260 #define	CSIAR_BYTE_ENABLE		0x0000f000
261 #define	CSIAR_ADDR_MASK			0x00000fff
262 	PMCH			= 0x6f,
263 	EPHYAR			= 0x80,
264 #define	EPHYAR_FLAG			0x80000000
265 #define	EPHYAR_WRITE_CMD		0x80000000
266 #define	EPHYAR_REG_MASK			0x1f
267 #define	EPHYAR_REG_SHIFT		16
268 #define	EPHYAR_DATA_MASK		0xffff
269 	DLLPR			= 0xd0,
270 #define	PFM_EN				(1 << 6)
271 #define	TX_10M_PS_EN			(1 << 7)
272 	DBG_REG			= 0xd1,
273 #define	FIX_NAK_1			(1 << 4)
274 #define	FIX_NAK_2			(1 << 3)
275 	TWSI			= 0xd2,
276 	MCU			= 0xd3,
277 #define	NOW_IS_OOB			(1 << 7)
278 #define	TX_EMPTY			(1 << 5)
279 #define	RX_EMPTY			(1 << 4)
280 #define	RXTX_EMPTY			(TX_EMPTY | RX_EMPTY)
281 #define	EN_NDP				(1 << 3)
282 #define	EN_OOB_RESET			(1 << 2)
283 #define	LINK_LIST_RDY			(1 << 1)
284 	EFUSEAR			= 0xdc,
285 #define	EFUSEAR_FLAG			0x80000000
286 #define	EFUSEAR_WRITE_CMD		0x80000000
287 #define	EFUSEAR_READ_CMD		0x00000000
288 #define	EFUSEAR_REG_MASK		0x03ff
289 #define	EFUSEAR_REG_SHIFT		8
290 #define	EFUSEAR_DATA_MASK		0xff
291 	MISC_1			= 0xf2,
292 #define	PFM_D3COLD_EN			(1 << 6)
293 };
294 
295 enum rtl8168_registers {
296 	LED_FREQ		= 0x1a,
297 	EEE_LED			= 0x1b,
298 	ERIDR			= 0x70,
299 	ERIAR			= 0x74,
300 #define ERIAR_FLAG			0x80000000
301 #define ERIAR_WRITE_CMD			0x80000000
302 #define ERIAR_READ_CMD			0x00000000
303 #define ERIAR_ADDR_BYTE_ALIGN		4
304 #define ERIAR_TYPE_SHIFT		16
305 #define ERIAR_EXGMAC			(0x00 << ERIAR_TYPE_SHIFT)
306 #define ERIAR_MSIX			(0x01 << ERIAR_TYPE_SHIFT)
307 #define ERIAR_ASF			(0x02 << ERIAR_TYPE_SHIFT)
308 #define ERIAR_OOB			(0x02 << ERIAR_TYPE_SHIFT)
309 #define ERIAR_MASK_SHIFT		12
310 #define ERIAR_MASK_0001			(0x1 << ERIAR_MASK_SHIFT)
311 #define ERIAR_MASK_0011			(0x3 << ERIAR_MASK_SHIFT)
312 #define ERIAR_MASK_0100			(0x4 << ERIAR_MASK_SHIFT)
313 #define ERIAR_MASK_0101			(0x5 << ERIAR_MASK_SHIFT)
314 #define ERIAR_MASK_1111			(0xf << ERIAR_MASK_SHIFT)
315 	EPHY_RXER_NUM		= 0x7c,
316 	OCPDR			= 0xb0,	/* OCP GPHY access */
317 #define OCPDR_WRITE_CMD			0x80000000
318 #define OCPDR_READ_CMD			0x00000000
319 #define OCPDR_REG_MASK			0x7f
320 #define OCPDR_GPHY_REG_SHIFT		16
321 #define OCPDR_DATA_MASK			0xffff
322 	OCPAR			= 0xb4,
323 #define OCPAR_FLAG			0x80000000
324 #define OCPAR_GPHY_WRITE_CMD		0x8000f060
325 #define OCPAR_GPHY_READ_CMD		0x0000f060
326 	GPHY_OCP		= 0xb8,
327 	RDSAR1			= 0xd0,	/* 8168c only. Undocumented on 8168dp */
328 	MISC			= 0xf0,	/* 8168e only. */
329 #define TXPLA_RST			(1 << 29)
330 #define DISABLE_LAN_EN			(1 << 23) /* Enable GPIO pin */
331 #define PWM_EN				(1 << 22)
332 #define RXDV_GATED_EN			(1 << 19)
333 #define EARLY_TALLY_EN			(1 << 16)
334 };
335 
336 enum rtl8125_registers {
337 	IntrMask_8125		= 0x38,
338 	IntrStatus_8125		= 0x3c,
339 	TxPoll_8125		= 0x90,
340 	MAC0_BKP		= 0x19e0,
341 	EEE_TXIDLE_TIMER_8125	= 0x6048,
342 };
343 
344 #define RX_VLAN_INNER_8125	BIT(22)
345 #define RX_VLAN_OUTER_8125	BIT(23)
346 #define RX_VLAN_8125		(RX_VLAN_INNER_8125 | RX_VLAN_OUTER_8125)
347 
348 #define RX_FETCH_DFLT_8125	(8 << 27)
349 
350 enum rtl_register_content {
351 	/* InterruptStatusBits */
352 	SYSErr		= 0x8000,
353 	PCSTimeout	= 0x4000,
354 	SWInt		= 0x0100,
355 	TxDescUnavail	= 0x0080,
356 	RxFIFOOver	= 0x0040,
357 	LinkChg		= 0x0020,
358 	RxOverflow	= 0x0010,
359 	TxErr		= 0x0008,
360 	TxOK		= 0x0004,
361 	RxErr		= 0x0002,
362 	RxOK		= 0x0001,
363 
364 	/* RxStatusDesc */
365 	RxRWT	= (1 << 22),
366 	RxRES	= (1 << 21),
367 	RxRUNT	= (1 << 20),
368 	RxCRC	= (1 << 19),
369 
370 	/* ChipCmdBits */
371 	StopReq		= 0x80,
372 	CmdReset	= 0x10,
373 	CmdRxEnb	= 0x08,
374 	CmdTxEnb	= 0x04,
375 	RxBufEmpty	= 0x01,
376 
377 	/* TXPoll register p.5 */
378 	HPQ		= 0x80,		/* Poll cmd on the high prio queue */
379 	NPQ		= 0x40,		/* Poll cmd on the low prio queue */
380 	FSWInt		= 0x01,		/* Forced software interrupt */
381 
382 	/* Cfg9346Bits */
383 	Cfg9346_Lock	= 0x00,
384 	Cfg9346_Unlock	= 0xc0,
385 
386 	/* rx_mode_bits */
387 	AcceptErr	= 0x20,
388 	AcceptRunt	= 0x10,
389 #define RX_CONFIG_ACCEPT_ERR_MASK	0x30
390 	AcceptBroadcast	= 0x08,
391 	AcceptMulticast	= 0x04,
392 	AcceptMyPhys	= 0x02,
393 	AcceptAllPhys	= 0x01,
394 #define RX_CONFIG_ACCEPT_OK_MASK	0x0f
395 #define RX_CONFIG_ACCEPT_MASK		0x3f
396 
397 	/* TxConfigBits */
398 	TxInterFrameGapShift = 24,
399 	TxDMAShift = 8,	/* DMA burst value (0-7) is shift this many bits */
400 
401 	/* Config1 register p.24 */
402 	LEDS1		= (1 << 7),
403 	LEDS0		= (1 << 6),
404 	Speed_down	= (1 << 4),
405 	MEMMAP		= (1 << 3),
406 	IOMAP		= (1 << 2),
407 	VPD		= (1 << 1),
408 	PMEnable	= (1 << 0),	/* Power Management Enable */
409 
410 	/* Config2 register p. 25 */
411 	ClkReqEn	= (1 << 7),	/* Clock Request Enable */
412 	MSIEnable	= (1 << 5),	/* 8169 only. Reserved in the 8168. */
413 	PCI_Clock_66MHz = 0x01,
414 	PCI_Clock_33MHz = 0x00,
415 
416 	/* Config3 register p.25 */
417 	MagicPacket	= (1 << 5),	/* Wake up when receives a Magic Packet */
418 	LinkUp		= (1 << 4),	/* Wake up when the cable connection is re-established */
419 	Jumbo_En0	= (1 << 2),	/* 8168 only. Reserved in the 8168b */
420 	Rdy_to_L23	= (1 << 1),	/* L23 Enable */
421 	Beacon_en	= (1 << 0),	/* 8168 only. Reserved in the 8168b */
422 
423 	/* Config4 register */
424 	Jumbo_En1	= (1 << 1),	/* 8168 only. Reserved in the 8168b */
425 
426 	/* Config5 register p.27 */
427 	BWF		= (1 << 6),	/* Accept Broadcast wakeup frame */
428 	MWF		= (1 << 5),	/* Accept Multicast wakeup frame */
429 	UWF		= (1 << 4),	/* Accept Unicast wakeup frame */
430 	Spi_en		= (1 << 3),
431 	LanWake		= (1 << 1),	/* LanWake enable/disable */
432 	PMEStatus	= (1 << 0),	/* PME status can be reset by PCI RST# */
433 	ASPM_en		= (1 << 0),	/* ASPM enable */
434 
435 	/* CPlusCmd p.31 */
436 	EnableBist	= (1 << 15),	// 8168 8101
437 	Mac_dbgo_oe	= (1 << 14),	// 8168 8101
438 	EnAnaPLL	= (1 << 14),	// 8169
439 	Normal_mode	= (1 << 13),	// unused
440 	Force_half_dup	= (1 << 12),	// 8168 8101
441 	Force_rxflow_en	= (1 << 11),	// 8168 8101
442 	Force_txflow_en	= (1 << 10),	// 8168 8101
443 	Cxpl_dbg_sel	= (1 << 9),	// 8168 8101
444 	ASF		= (1 << 8),	// 8168 8101
445 	PktCntrDisable	= (1 << 7),	// 8168 8101
446 	Mac_dbgo_sel	= 0x001c,	// 8168
447 	RxVlan		= (1 << 6),
448 	RxChkSum	= (1 << 5),
449 	PCIDAC		= (1 << 4),
450 	PCIMulRW	= (1 << 3),
451 #define INTT_MASK	GENMASK(1, 0)
452 #define CPCMD_MASK	(Normal_mode | RxVlan | RxChkSum | INTT_MASK)
453 
454 	/* rtl8169_PHYstatus */
455 	TBI_Enable	= 0x80,
456 	TxFlowCtrl	= 0x40,
457 	RxFlowCtrl	= 0x20,
458 	_1000bpsF	= 0x10,
459 	_100bps		= 0x08,
460 	_10bps		= 0x04,
461 	LinkStatus	= 0x02,
462 	FullDup		= 0x01,
463 
464 	/* ResetCounterCommand */
465 	CounterReset	= 0x1,
466 
467 	/* DumpCounterCommand */
468 	CounterDump	= 0x8,
469 
470 	/* magic enable v2 */
471 	MagicPacket_v2	= (1 << 16),	/* Wake up when receives a Magic Packet */
472 };
473 
474 enum rtl_desc_bit {
475 	/* First doubleword. */
476 	DescOwn		= (1 << 31), /* Descriptor is owned by NIC */
477 	RingEnd		= (1 << 30), /* End of descriptor ring */
478 	FirstFrag	= (1 << 29), /* First segment of a packet */
479 	LastFrag	= (1 << 28), /* Final segment of a packet */
480 };
481 
482 /* Generic case. */
483 enum rtl_tx_desc_bit {
484 	/* First doubleword. */
485 	TD_LSO		= (1 << 27),		/* Large Send Offload */
486 #define TD_MSS_MAX			0x07ffu	/* MSS value */
487 
488 	/* Second doubleword. */
489 	TxVlanTag	= (1 << 17),		/* Add VLAN tag */
490 };
491 
492 /* 8169, 8168b and 810x except 8102e. */
493 enum rtl_tx_desc_bit_0 {
494 	/* First doubleword. */
495 #define TD0_MSS_SHIFT			16	/* MSS position (11 bits) */
496 	TD0_TCP_CS	= (1 << 16),		/* Calculate TCP/IP checksum */
497 	TD0_UDP_CS	= (1 << 17),		/* Calculate UDP/IP checksum */
498 	TD0_IP_CS	= (1 << 18),		/* Calculate IP checksum */
499 };
500 
501 /* 8102e, 8168c and beyond. */
502 enum rtl_tx_desc_bit_1 {
503 	/* First doubleword. */
504 	TD1_GTSENV4	= (1 << 26),		/* Giant Send for IPv4 */
505 	TD1_GTSENV6	= (1 << 25),		/* Giant Send for IPv6 */
506 #define GTTCPHO_SHIFT			18
507 #define GTTCPHO_MAX			0x7f
508 
509 	/* Second doubleword. */
510 #define TCPHO_SHIFT			18
511 #define TCPHO_MAX			0x3ff
512 #define TD1_MSS_SHIFT			18	/* MSS position (11 bits) */
513 	TD1_IPv6_CS	= (1 << 28),		/* Calculate IPv6 checksum */
514 	TD1_IPv4_CS	= (1 << 29),		/* Calculate IPv4 checksum */
515 	TD1_TCP_CS	= (1 << 30),		/* Calculate TCP/IP checksum */
516 	TD1_UDP_CS	= (1 << 31),		/* Calculate UDP/IP checksum */
517 };
518 
519 enum rtl_rx_desc_bit {
520 	/* Rx private */
521 	PID1		= (1 << 18), /* Protocol ID bit 1/2 */
522 	PID0		= (1 << 17), /* Protocol ID bit 0/2 */
523 
524 #define RxProtoUDP	(PID1)
525 #define RxProtoTCP	(PID0)
526 #define RxProtoIP	(PID1 | PID0)
527 #define RxProtoMask	RxProtoIP
528 
529 	IPFail		= (1 << 16), /* IP checksum failed */
530 	UDPFail		= (1 << 15), /* UDP/IP checksum failed */
531 	TCPFail		= (1 << 14), /* TCP/IP checksum failed */
532 	RxVlanTag	= (1 << 16), /* VLAN tag available */
533 };
534 
535 #define RTL_GSO_MAX_SIZE_V1	32000
536 #define RTL_GSO_MAX_SEGS_V1	24
537 #define RTL_GSO_MAX_SIZE_V2	64000
538 #define RTL_GSO_MAX_SEGS_V2	64
539 
540 struct TxDesc {
541 	__le32 opts1;
542 	__le32 opts2;
543 	__le64 addr;
544 };
545 
546 struct RxDesc {
547 	__le32 opts1;
548 	__le32 opts2;
549 	__le64 addr;
550 };
551 
552 struct ring_info {
553 	struct sk_buff	*skb;
554 	u32		len;
555 };
556 
557 struct rtl8169_counters {
558 	__le64	tx_packets;
559 	__le64	rx_packets;
560 	__le64	tx_errors;
561 	__le32	rx_errors;
562 	__le16	rx_missed;
563 	__le16	align_errors;
564 	__le32	tx_one_collision;
565 	__le32	tx_multi_collision;
566 	__le64	rx_unicast;
567 	__le64	rx_broadcast;
568 	__le32	rx_multicast;
569 	__le16	tx_aborted;
570 	__le16	tx_underun;
571 };
572 
573 struct rtl8169_tc_offsets {
574 	bool	inited;
575 	__le64	tx_errors;
576 	__le32	tx_multi_collision;
577 	__le16	tx_aborted;
578 	__le16	rx_missed;
579 };
580 
581 enum rtl_flag {
582 	RTL_FLAG_TASK_ENABLED = 0,
583 	RTL_FLAG_TASK_RESET_PENDING,
584 	RTL_FLAG_MAX
585 };
586 
587 struct rtl8169_private {
588 	void __iomem *mmio_addr;	/* memory map physical address */
589 	struct pci_dev *pci_dev;
590 	struct net_device *dev;
591 	struct phy_device *phydev;
592 	struct napi_struct napi;
593 	enum mac_version mac_version;
594 	u32 cur_rx; /* Index into the Rx descriptor buffer of next Rx pkt. */
595 	u32 cur_tx; /* Index into the Tx descriptor buffer of next Rx pkt. */
596 	u32 dirty_tx;
597 	struct TxDesc *TxDescArray;	/* 256-aligned Tx descriptor ring */
598 	struct RxDesc *RxDescArray;	/* 256-aligned Rx descriptor ring */
599 	dma_addr_t TxPhyAddr;
600 	dma_addr_t RxPhyAddr;
601 	struct page *Rx_databuff[NUM_RX_DESC];	/* Rx data buffers */
602 	struct ring_info tx_skb[NUM_TX_DESC];	/* Tx data buffers */
603 	u16 cp_cmd;
604 	u32 irq_mask;
605 	struct clk *clk;
606 
607 	struct {
608 		DECLARE_BITMAP(flags, RTL_FLAG_MAX);
609 		struct work_struct work;
610 	} wk;
611 
612 	unsigned supports_gmii:1;
613 	unsigned aspm_manageable:1;
614 	dma_addr_t counters_phys_addr;
615 	struct rtl8169_counters *counters;
616 	struct rtl8169_tc_offsets tc_offset;
617 	u32 saved_wolopts;
618 	int eee_adv;
619 
620 	const char *fw_name;
621 	struct rtl_fw *rtl_fw;
622 
623 	u32 ocp_base;
624 };
625 
626 typedef void (*rtl_generic_fct)(struct rtl8169_private *tp);
627 
628 MODULE_AUTHOR("Realtek and the Linux r8169 crew <netdev@vger.kernel.org>");
629 MODULE_DESCRIPTION("RealTek RTL-8169 Gigabit Ethernet driver");
630 MODULE_SOFTDEP("pre: realtek");
631 MODULE_LICENSE("GPL");
632 MODULE_FIRMWARE(FIRMWARE_8168D_1);
633 MODULE_FIRMWARE(FIRMWARE_8168D_2);
634 MODULE_FIRMWARE(FIRMWARE_8168E_1);
635 MODULE_FIRMWARE(FIRMWARE_8168E_2);
636 MODULE_FIRMWARE(FIRMWARE_8168E_3);
637 MODULE_FIRMWARE(FIRMWARE_8105E_1);
638 MODULE_FIRMWARE(FIRMWARE_8168F_1);
639 MODULE_FIRMWARE(FIRMWARE_8168F_2);
640 MODULE_FIRMWARE(FIRMWARE_8402_1);
641 MODULE_FIRMWARE(FIRMWARE_8411_1);
642 MODULE_FIRMWARE(FIRMWARE_8411_2);
643 MODULE_FIRMWARE(FIRMWARE_8106E_1);
644 MODULE_FIRMWARE(FIRMWARE_8106E_2);
645 MODULE_FIRMWARE(FIRMWARE_8168G_2);
646 MODULE_FIRMWARE(FIRMWARE_8168G_3);
647 MODULE_FIRMWARE(FIRMWARE_8168H_1);
648 MODULE_FIRMWARE(FIRMWARE_8168H_2);
649 MODULE_FIRMWARE(FIRMWARE_8168FP_3);
650 MODULE_FIRMWARE(FIRMWARE_8107E_1);
651 MODULE_FIRMWARE(FIRMWARE_8107E_2);
652 MODULE_FIRMWARE(FIRMWARE_8125A_3);
653 MODULE_FIRMWARE(FIRMWARE_8125B_2);
654 
655 static inline struct device *tp_to_dev(struct rtl8169_private *tp)
656 {
657 	return &tp->pci_dev->dev;
658 }
659 
660 static void rtl_lock_config_regs(struct rtl8169_private *tp)
661 {
662 	RTL_W8(tp, Cfg9346, Cfg9346_Lock);
663 }
664 
665 static void rtl_unlock_config_regs(struct rtl8169_private *tp)
666 {
667 	RTL_W8(tp, Cfg9346, Cfg9346_Unlock);
668 }
669 
670 static void rtl_pci_commit(struct rtl8169_private *tp)
671 {
672 	/* Read an arbitrary register to commit a preceding PCI write */
673 	RTL_R8(tp, ChipCmd);
674 }
675 
676 static bool rtl_is_8125(struct rtl8169_private *tp)
677 {
678 	return tp->mac_version >= RTL_GIGA_MAC_VER_60;
679 }
680 
681 static bool rtl_is_8168evl_up(struct rtl8169_private *tp)
682 {
683 	return tp->mac_version >= RTL_GIGA_MAC_VER_34 &&
684 	       tp->mac_version != RTL_GIGA_MAC_VER_39 &&
685 	       tp->mac_version <= RTL_GIGA_MAC_VER_52;
686 }
687 
688 static bool rtl_supports_eee(struct rtl8169_private *tp)
689 {
690 	return tp->mac_version >= RTL_GIGA_MAC_VER_34 &&
691 	       tp->mac_version != RTL_GIGA_MAC_VER_37 &&
692 	       tp->mac_version != RTL_GIGA_MAC_VER_39;
693 }
694 
695 static void rtl_read_mac_from_reg(struct rtl8169_private *tp, u8 *mac, int reg)
696 {
697 	int i;
698 
699 	for (i = 0; i < ETH_ALEN; i++)
700 		mac[i] = RTL_R8(tp, reg + i);
701 }
702 
703 struct rtl_cond {
704 	bool (*check)(struct rtl8169_private *);
705 	const char *msg;
706 };
707 
708 static bool rtl_loop_wait(struct rtl8169_private *tp, const struct rtl_cond *c,
709 			  unsigned long usecs, int n, bool high)
710 {
711 	int i;
712 
713 	for (i = 0; i < n; i++) {
714 		if (c->check(tp) == high)
715 			return true;
716 		fsleep(usecs);
717 	}
718 
719 	if (net_ratelimit())
720 		netdev_err(tp->dev, "%s == %d (loop: %d, delay: %lu).\n",
721 			   c->msg, !high, n, usecs);
722 	return false;
723 }
724 
725 static bool rtl_loop_wait_high(struct rtl8169_private *tp,
726 			       const struct rtl_cond *c,
727 			       unsigned long d, int n)
728 {
729 	return rtl_loop_wait(tp, c, d, n, true);
730 }
731 
732 static bool rtl_loop_wait_low(struct rtl8169_private *tp,
733 			      const struct rtl_cond *c,
734 			      unsigned long d, int n)
735 {
736 	return rtl_loop_wait(tp, c, d, n, false);
737 }
738 
739 #define DECLARE_RTL_COND(name)				\
740 static bool name ## _check(struct rtl8169_private *);	\
741 							\
742 static const struct rtl_cond name = {			\
743 	.check	= name ## _check,			\
744 	.msg	= #name					\
745 };							\
746 							\
747 static bool name ## _check(struct rtl8169_private *tp)
748 
749 static bool rtl_ocp_reg_failure(struct rtl8169_private *tp, u32 reg)
750 {
751 	if (reg & 0xffff0001) {
752 		if (net_ratelimit())
753 			netdev_err(tp->dev, "Invalid ocp reg %x!\n", reg);
754 		return true;
755 	}
756 	return false;
757 }
758 
759 DECLARE_RTL_COND(rtl_ocp_gphy_cond)
760 {
761 	return RTL_R32(tp, GPHY_OCP) & OCPAR_FLAG;
762 }
763 
764 static void r8168_phy_ocp_write(struct rtl8169_private *tp, u32 reg, u32 data)
765 {
766 	if (rtl_ocp_reg_failure(tp, reg))
767 		return;
768 
769 	RTL_W32(tp, GPHY_OCP, OCPAR_FLAG | (reg << 15) | data);
770 
771 	rtl_loop_wait_low(tp, &rtl_ocp_gphy_cond, 25, 10);
772 }
773 
774 static int r8168_phy_ocp_read(struct rtl8169_private *tp, u32 reg)
775 {
776 	if (rtl_ocp_reg_failure(tp, reg))
777 		return 0;
778 
779 	RTL_W32(tp, GPHY_OCP, reg << 15);
780 
781 	return rtl_loop_wait_high(tp, &rtl_ocp_gphy_cond, 25, 10) ?
782 		(RTL_R32(tp, GPHY_OCP) & 0xffff) : -ETIMEDOUT;
783 }
784 
785 static void r8168_mac_ocp_write(struct rtl8169_private *tp, u32 reg, u32 data)
786 {
787 	if (rtl_ocp_reg_failure(tp, reg))
788 		return;
789 
790 	RTL_W32(tp, OCPDR, OCPAR_FLAG | (reg << 15) | data);
791 }
792 
793 static u16 r8168_mac_ocp_read(struct rtl8169_private *tp, u32 reg)
794 {
795 	if (rtl_ocp_reg_failure(tp, reg))
796 		return 0;
797 
798 	RTL_W32(tp, OCPDR, reg << 15);
799 
800 	return RTL_R32(tp, OCPDR);
801 }
802 
803 static void r8168_mac_ocp_modify(struct rtl8169_private *tp, u32 reg, u16 mask,
804 				 u16 set)
805 {
806 	u16 data = r8168_mac_ocp_read(tp, reg);
807 
808 	r8168_mac_ocp_write(tp, reg, (data & ~mask) | set);
809 }
810 
811 static void r8168g_mdio_write(struct rtl8169_private *tp, int reg, int value)
812 {
813 	if (reg == 0x1f) {
814 		tp->ocp_base = value ? value << 4 : OCP_STD_PHY_BASE;
815 		return;
816 	}
817 
818 	if (tp->ocp_base != OCP_STD_PHY_BASE)
819 		reg -= 0x10;
820 
821 	r8168_phy_ocp_write(tp, tp->ocp_base + reg * 2, value);
822 }
823 
824 static int r8168g_mdio_read(struct rtl8169_private *tp, int reg)
825 {
826 	if (reg == 0x1f)
827 		return tp->ocp_base == OCP_STD_PHY_BASE ? 0 : tp->ocp_base >> 4;
828 
829 	if (tp->ocp_base != OCP_STD_PHY_BASE)
830 		reg -= 0x10;
831 
832 	return r8168_phy_ocp_read(tp, tp->ocp_base + reg * 2);
833 }
834 
835 static void mac_mcu_write(struct rtl8169_private *tp, int reg, int value)
836 {
837 	if (reg == 0x1f) {
838 		tp->ocp_base = value << 4;
839 		return;
840 	}
841 
842 	r8168_mac_ocp_write(tp, tp->ocp_base + reg, value);
843 }
844 
845 static int mac_mcu_read(struct rtl8169_private *tp, int reg)
846 {
847 	return r8168_mac_ocp_read(tp, tp->ocp_base + reg);
848 }
849 
850 DECLARE_RTL_COND(rtl_phyar_cond)
851 {
852 	return RTL_R32(tp, PHYAR) & 0x80000000;
853 }
854 
855 static void r8169_mdio_write(struct rtl8169_private *tp, int reg, int value)
856 {
857 	RTL_W32(tp, PHYAR, 0x80000000 | (reg & 0x1f) << 16 | (value & 0xffff));
858 
859 	rtl_loop_wait_low(tp, &rtl_phyar_cond, 25, 20);
860 	/*
861 	 * According to hardware specs a 20us delay is required after write
862 	 * complete indication, but before sending next command.
863 	 */
864 	udelay(20);
865 }
866 
867 static int r8169_mdio_read(struct rtl8169_private *tp, int reg)
868 {
869 	int value;
870 
871 	RTL_W32(tp, PHYAR, 0x0 | (reg & 0x1f) << 16);
872 
873 	value = rtl_loop_wait_high(tp, &rtl_phyar_cond, 25, 20) ?
874 		RTL_R32(tp, PHYAR) & 0xffff : -ETIMEDOUT;
875 
876 	/*
877 	 * According to hardware specs a 20us delay is required after read
878 	 * complete indication, but before sending next command.
879 	 */
880 	udelay(20);
881 
882 	return value;
883 }
884 
885 DECLARE_RTL_COND(rtl_ocpar_cond)
886 {
887 	return RTL_R32(tp, OCPAR) & OCPAR_FLAG;
888 }
889 
890 static void r8168dp_1_mdio_access(struct rtl8169_private *tp, int reg, u32 data)
891 {
892 	RTL_W32(tp, OCPDR, data | ((reg & OCPDR_REG_MASK) << OCPDR_GPHY_REG_SHIFT));
893 	RTL_W32(tp, OCPAR, OCPAR_GPHY_WRITE_CMD);
894 	RTL_W32(tp, EPHY_RXER_NUM, 0);
895 
896 	rtl_loop_wait_low(tp, &rtl_ocpar_cond, 1000, 100);
897 }
898 
899 static void r8168dp_1_mdio_write(struct rtl8169_private *tp, int reg, int value)
900 {
901 	r8168dp_1_mdio_access(tp, reg,
902 			      OCPDR_WRITE_CMD | (value & OCPDR_DATA_MASK));
903 }
904 
905 static int r8168dp_1_mdio_read(struct rtl8169_private *tp, int reg)
906 {
907 	r8168dp_1_mdio_access(tp, reg, OCPDR_READ_CMD);
908 
909 	mdelay(1);
910 	RTL_W32(tp, OCPAR, OCPAR_GPHY_READ_CMD);
911 	RTL_W32(tp, EPHY_RXER_NUM, 0);
912 
913 	return rtl_loop_wait_high(tp, &rtl_ocpar_cond, 1000, 100) ?
914 		RTL_R32(tp, OCPDR) & OCPDR_DATA_MASK : -ETIMEDOUT;
915 }
916 
917 #define R8168DP_1_MDIO_ACCESS_BIT	0x00020000
918 
919 static void r8168dp_2_mdio_start(struct rtl8169_private *tp)
920 {
921 	RTL_W32(tp, 0xd0, RTL_R32(tp, 0xd0) & ~R8168DP_1_MDIO_ACCESS_BIT);
922 }
923 
924 static void r8168dp_2_mdio_stop(struct rtl8169_private *tp)
925 {
926 	RTL_W32(tp, 0xd0, RTL_R32(tp, 0xd0) | R8168DP_1_MDIO_ACCESS_BIT);
927 }
928 
929 static void r8168dp_2_mdio_write(struct rtl8169_private *tp, int reg, int value)
930 {
931 	r8168dp_2_mdio_start(tp);
932 
933 	r8169_mdio_write(tp, reg, value);
934 
935 	r8168dp_2_mdio_stop(tp);
936 }
937 
938 static int r8168dp_2_mdio_read(struct rtl8169_private *tp, int reg)
939 {
940 	int value;
941 
942 	/* Work around issue with chip reporting wrong PHY ID */
943 	if (reg == MII_PHYSID2)
944 		return 0xc912;
945 
946 	r8168dp_2_mdio_start(tp);
947 
948 	value = r8169_mdio_read(tp, reg);
949 
950 	r8168dp_2_mdio_stop(tp);
951 
952 	return value;
953 }
954 
955 static void rtl_writephy(struct rtl8169_private *tp, int location, int val)
956 {
957 	switch (tp->mac_version) {
958 	case RTL_GIGA_MAC_VER_27:
959 		r8168dp_1_mdio_write(tp, location, val);
960 		break;
961 	case RTL_GIGA_MAC_VER_28:
962 	case RTL_GIGA_MAC_VER_31:
963 		r8168dp_2_mdio_write(tp, location, val);
964 		break;
965 	case RTL_GIGA_MAC_VER_40 ... RTL_GIGA_MAC_VER_63:
966 		r8168g_mdio_write(tp, location, val);
967 		break;
968 	default:
969 		r8169_mdio_write(tp, location, val);
970 		break;
971 	}
972 }
973 
974 static int rtl_readphy(struct rtl8169_private *tp, int location)
975 {
976 	switch (tp->mac_version) {
977 	case RTL_GIGA_MAC_VER_27:
978 		return r8168dp_1_mdio_read(tp, location);
979 	case RTL_GIGA_MAC_VER_28:
980 	case RTL_GIGA_MAC_VER_31:
981 		return r8168dp_2_mdio_read(tp, location);
982 	case RTL_GIGA_MAC_VER_40 ... RTL_GIGA_MAC_VER_63:
983 		return r8168g_mdio_read(tp, location);
984 	default:
985 		return r8169_mdio_read(tp, location);
986 	}
987 }
988 
989 DECLARE_RTL_COND(rtl_ephyar_cond)
990 {
991 	return RTL_R32(tp, EPHYAR) & EPHYAR_FLAG;
992 }
993 
994 static void rtl_ephy_write(struct rtl8169_private *tp, int reg_addr, int value)
995 {
996 	RTL_W32(tp, EPHYAR, EPHYAR_WRITE_CMD | (value & EPHYAR_DATA_MASK) |
997 		(reg_addr & EPHYAR_REG_MASK) << EPHYAR_REG_SHIFT);
998 
999 	rtl_loop_wait_low(tp, &rtl_ephyar_cond, 10, 100);
1000 
1001 	udelay(10);
1002 }
1003 
1004 static u16 rtl_ephy_read(struct rtl8169_private *tp, int reg_addr)
1005 {
1006 	RTL_W32(tp, EPHYAR, (reg_addr & EPHYAR_REG_MASK) << EPHYAR_REG_SHIFT);
1007 
1008 	return rtl_loop_wait_high(tp, &rtl_ephyar_cond, 10, 100) ?
1009 		RTL_R32(tp, EPHYAR) & EPHYAR_DATA_MASK : ~0;
1010 }
1011 
1012 static void r8168fp_adjust_ocp_cmd(struct rtl8169_private *tp, u32 *cmd, int type)
1013 {
1014 	/* based on RTL8168FP_OOBMAC_BASE in vendor driver */
1015 	if (tp->mac_version == RTL_GIGA_MAC_VER_52 && type == ERIAR_OOB)
1016 		*cmd |= 0x7f0 << 18;
1017 }
1018 
1019 DECLARE_RTL_COND(rtl_eriar_cond)
1020 {
1021 	return RTL_R32(tp, ERIAR) & ERIAR_FLAG;
1022 }
1023 
1024 static void _rtl_eri_write(struct rtl8169_private *tp, int addr, u32 mask,
1025 			   u32 val, int type)
1026 {
1027 	u32 cmd = ERIAR_WRITE_CMD | type | mask | addr;
1028 
1029 	BUG_ON((addr & 3) || (mask == 0));
1030 	RTL_W32(tp, ERIDR, val);
1031 	r8168fp_adjust_ocp_cmd(tp, &cmd, type);
1032 	RTL_W32(tp, ERIAR, cmd);
1033 
1034 	rtl_loop_wait_low(tp, &rtl_eriar_cond, 100, 100);
1035 }
1036 
1037 static void rtl_eri_write(struct rtl8169_private *tp, int addr, u32 mask,
1038 			  u32 val)
1039 {
1040 	_rtl_eri_write(tp, addr, mask, val, ERIAR_EXGMAC);
1041 }
1042 
1043 static u32 _rtl_eri_read(struct rtl8169_private *tp, int addr, int type)
1044 {
1045 	u32 cmd = ERIAR_READ_CMD | type | ERIAR_MASK_1111 | addr;
1046 
1047 	r8168fp_adjust_ocp_cmd(tp, &cmd, type);
1048 	RTL_W32(tp, ERIAR, cmd);
1049 
1050 	return rtl_loop_wait_high(tp, &rtl_eriar_cond, 100, 100) ?
1051 		RTL_R32(tp, ERIDR) : ~0;
1052 }
1053 
1054 static u32 rtl_eri_read(struct rtl8169_private *tp, int addr)
1055 {
1056 	return _rtl_eri_read(tp, addr, ERIAR_EXGMAC);
1057 }
1058 
1059 static void rtl_w0w1_eri(struct rtl8169_private *tp, int addr, u32 p, u32 m)
1060 {
1061 	u32 val = rtl_eri_read(tp, addr);
1062 
1063 	rtl_eri_write(tp, addr, ERIAR_MASK_1111, (val & ~m) | p);
1064 }
1065 
1066 static void rtl_eri_set_bits(struct rtl8169_private *tp, int addr, u32 p)
1067 {
1068 	rtl_w0w1_eri(tp, addr, p, 0);
1069 }
1070 
1071 static void rtl_eri_clear_bits(struct rtl8169_private *tp, int addr, u32 m)
1072 {
1073 	rtl_w0w1_eri(tp, addr, 0, m);
1074 }
1075 
1076 static u32 r8168dp_ocp_read(struct rtl8169_private *tp, u16 reg)
1077 {
1078 	RTL_W32(tp, OCPAR, 0x0fu << 12 | (reg & 0x0fff));
1079 	return rtl_loop_wait_high(tp, &rtl_ocpar_cond, 100, 20) ?
1080 		RTL_R32(tp, OCPDR) : ~0;
1081 }
1082 
1083 static u32 r8168ep_ocp_read(struct rtl8169_private *tp, u16 reg)
1084 {
1085 	return _rtl_eri_read(tp, reg, ERIAR_OOB);
1086 }
1087 
1088 static void r8168dp_ocp_write(struct rtl8169_private *tp, u8 mask, u16 reg,
1089 			      u32 data)
1090 {
1091 	RTL_W32(tp, OCPDR, data);
1092 	RTL_W32(tp, OCPAR, OCPAR_FLAG | ((u32)mask & 0x0f) << 12 | (reg & 0x0fff));
1093 	rtl_loop_wait_low(tp, &rtl_ocpar_cond, 100, 20);
1094 }
1095 
1096 static void r8168ep_ocp_write(struct rtl8169_private *tp, u8 mask, u16 reg,
1097 			      u32 data)
1098 {
1099 	_rtl_eri_write(tp, reg, ((u32)mask & 0x0f) << ERIAR_MASK_SHIFT,
1100 		       data, ERIAR_OOB);
1101 }
1102 
1103 static void r8168dp_oob_notify(struct rtl8169_private *tp, u8 cmd)
1104 {
1105 	rtl_eri_write(tp, 0xe8, ERIAR_MASK_0001, cmd);
1106 
1107 	r8168dp_ocp_write(tp, 0x1, 0x30, 0x00000001);
1108 }
1109 
1110 #define OOB_CMD_RESET		0x00
1111 #define OOB_CMD_DRIVER_START	0x05
1112 #define OOB_CMD_DRIVER_STOP	0x06
1113 
1114 static u16 rtl8168_get_ocp_reg(struct rtl8169_private *tp)
1115 {
1116 	return (tp->mac_version == RTL_GIGA_MAC_VER_31) ? 0xb8 : 0x10;
1117 }
1118 
1119 DECLARE_RTL_COND(rtl_dp_ocp_read_cond)
1120 {
1121 	u16 reg;
1122 
1123 	reg = rtl8168_get_ocp_reg(tp);
1124 
1125 	return r8168dp_ocp_read(tp, reg) & 0x00000800;
1126 }
1127 
1128 DECLARE_RTL_COND(rtl_ep_ocp_read_cond)
1129 {
1130 	return r8168ep_ocp_read(tp, 0x124) & 0x00000001;
1131 }
1132 
1133 DECLARE_RTL_COND(rtl_ocp_tx_cond)
1134 {
1135 	return RTL_R8(tp, IBISR0) & 0x20;
1136 }
1137 
1138 static void rtl8168ep_stop_cmac(struct rtl8169_private *tp)
1139 {
1140 	RTL_W8(tp, IBCR2, RTL_R8(tp, IBCR2) & ~0x01);
1141 	rtl_loop_wait_high(tp, &rtl_ocp_tx_cond, 50000, 2000);
1142 	RTL_W8(tp, IBISR0, RTL_R8(tp, IBISR0) | 0x20);
1143 	RTL_W8(tp, IBCR0, RTL_R8(tp, IBCR0) & ~0x01);
1144 }
1145 
1146 static void rtl8168dp_driver_start(struct rtl8169_private *tp)
1147 {
1148 	r8168dp_oob_notify(tp, OOB_CMD_DRIVER_START);
1149 	rtl_loop_wait_high(tp, &rtl_dp_ocp_read_cond, 10000, 10);
1150 }
1151 
1152 static void rtl8168ep_driver_start(struct rtl8169_private *tp)
1153 {
1154 	r8168ep_ocp_write(tp, 0x01, 0x180, OOB_CMD_DRIVER_START);
1155 	r8168ep_ocp_write(tp, 0x01, 0x30, r8168ep_ocp_read(tp, 0x30) | 0x01);
1156 	rtl_loop_wait_high(tp, &rtl_ep_ocp_read_cond, 10000, 10);
1157 }
1158 
1159 static void rtl8168_driver_start(struct rtl8169_private *tp)
1160 {
1161 	switch (tp->mac_version) {
1162 	case RTL_GIGA_MAC_VER_27:
1163 	case RTL_GIGA_MAC_VER_28:
1164 	case RTL_GIGA_MAC_VER_31:
1165 		rtl8168dp_driver_start(tp);
1166 		break;
1167 	case RTL_GIGA_MAC_VER_49 ... RTL_GIGA_MAC_VER_52:
1168 		rtl8168ep_driver_start(tp);
1169 		break;
1170 	default:
1171 		BUG();
1172 		break;
1173 	}
1174 }
1175 
1176 static void rtl8168dp_driver_stop(struct rtl8169_private *tp)
1177 {
1178 	r8168dp_oob_notify(tp, OOB_CMD_DRIVER_STOP);
1179 	rtl_loop_wait_low(tp, &rtl_dp_ocp_read_cond, 10000, 10);
1180 }
1181 
1182 static void rtl8168ep_driver_stop(struct rtl8169_private *tp)
1183 {
1184 	rtl8168ep_stop_cmac(tp);
1185 	r8168ep_ocp_write(tp, 0x01, 0x180, OOB_CMD_DRIVER_STOP);
1186 	r8168ep_ocp_write(tp, 0x01, 0x30, r8168ep_ocp_read(tp, 0x30) | 0x01);
1187 	rtl_loop_wait_low(tp, &rtl_ep_ocp_read_cond, 10000, 10);
1188 }
1189 
1190 static void rtl8168_driver_stop(struct rtl8169_private *tp)
1191 {
1192 	switch (tp->mac_version) {
1193 	case RTL_GIGA_MAC_VER_27:
1194 	case RTL_GIGA_MAC_VER_28:
1195 	case RTL_GIGA_MAC_VER_31:
1196 		rtl8168dp_driver_stop(tp);
1197 		break;
1198 	case RTL_GIGA_MAC_VER_49 ... RTL_GIGA_MAC_VER_52:
1199 		rtl8168ep_driver_stop(tp);
1200 		break;
1201 	default:
1202 		BUG();
1203 		break;
1204 	}
1205 }
1206 
1207 static bool r8168dp_check_dash(struct rtl8169_private *tp)
1208 {
1209 	u16 reg = rtl8168_get_ocp_reg(tp);
1210 
1211 	return !!(r8168dp_ocp_read(tp, reg) & 0x00008000);
1212 }
1213 
1214 static bool r8168ep_check_dash(struct rtl8169_private *tp)
1215 {
1216 	return r8168ep_ocp_read(tp, 0x128) & 0x00000001;
1217 }
1218 
1219 static bool r8168_check_dash(struct rtl8169_private *tp)
1220 {
1221 	switch (tp->mac_version) {
1222 	case RTL_GIGA_MAC_VER_27:
1223 	case RTL_GIGA_MAC_VER_28:
1224 	case RTL_GIGA_MAC_VER_31:
1225 		return r8168dp_check_dash(tp);
1226 	case RTL_GIGA_MAC_VER_49 ... RTL_GIGA_MAC_VER_52:
1227 		return r8168ep_check_dash(tp);
1228 	default:
1229 		return false;
1230 	}
1231 }
1232 
1233 static void rtl_reset_packet_filter(struct rtl8169_private *tp)
1234 {
1235 	rtl_eri_clear_bits(tp, 0xdc, BIT(0));
1236 	rtl_eri_set_bits(tp, 0xdc, BIT(0));
1237 }
1238 
1239 DECLARE_RTL_COND(rtl_efusear_cond)
1240 {
1241 	return RTL_R32(tp, EFUSEAR) & EFUSEAR_FLAG;
1242 }
1243 
1244 u8 rtl8168d_efuse_read(struct rtl8169_private *tp, int reg_addr)
1245 {
1246 	RTL_W32(tp, EFUSEAR, (reg_addr & EFUSEAR_REG_MASK) << EFUSEAR_REG_SHIFT);
1247 
1248 	return rtl_loop_wait_high(tp, &rtl_efusear_cond, 100, 300) ?
1249 		RTL_R32(tp, EFUSEAR) & EFUSEAR_DATA_MASK : ~0;
1250 }
1251 
1252 static u32 rtl_get_events(struct rtl8169_private *tp)
1253 {
1254 	if (rtl_is_8125(tp))
1255 		return RTL_R32(tp, IntrStatus_8125);
1256 	else
1257 		return RTL_R16(tp, IntrStatus);
1258 }
1259 
1260 static void rtl_ack_events(struct rtl8169_private *tp, u32 bits)
1261 {
1262 	if (rtl_is_8125(tp))
1263 		RTL_W32(tp, IntrStatus_8125, bits);
1264 	else
1265 		RTL_W16(tp, IntrStatus, bits);
1266 }
1267 
1268 static void rtl_irq_disable(struct rtl8169_private *tp)
1269 {
1270 	if (rtl_is_8125(tp))
1271 		RTL_W32(tp, IntrMask_8125, 0);
1272 	else
1273 		RTL_W16(tp, IntrMask, 0);
1274 }
1275 
1276 static void rtl_irq_enable(struct rtl8169_private *tp)
1277 {
1278 	if (rtl_is_8125(tp))
1279 		RTL_W32(tp, IntrMask_8125, tp->irq_mask);
1280 	else
1281 		RTL_W16(tp, IntrMask, tp->irq_mask);
1282 }
1283 
1284 static void rtl8169_irq_mask_and_ack(struct rtl8169_private *tp)
1285 {
1286 	rtl_irq_disable(tp);
1287 	rtl_ack_events(tp, 0xffffffff);
1288 	rtl_pci_commit(tp);
1289 }
1290 
1291 static void rtl_link_chg_patch(struct rtl8169_private *tp)
1292 {
1293 	struct phy_device *phydev = tp->phydev;
1294 
1295 	if (tp->mac_version == RTL_GIGA_MAC_VER_34 ||
1296 	    tp->mac_version == RTL_GIGA_MAC_VER_38) {
1297 		if (phydev->speed == SPEED_1000) {
1298 			rtl_eri_write(tp, 0x1bc, ERIAR_MASK_1111, 0x00000011);
1299 			rtl_eri_write(tp, 0x1dc, ERIAR_MASK_1111, 0x00000005);
1300 		} else if (phydev->speed == SPEED_100) {
1301 			rtl_eri_write(tp, 0x1bc, ERIAR_MASK_1111, 0x0000001f);
1302 			rtl_eri_write(tp, 0x1dc, ERIAR_MASK_1111, 0x00000005);
1303 		} else {
1304 			rtl_eri_write(tp, 0x1bc, ERIAR_MASK_1111, 0x0000001f);
1305 			rtl_eri_write(tp, 0x1dc, ERIAR_MASK_1111, 0x0000003f);
1306 		}
1307 		rtl_reset_packet_filter(tp);
1308 	} else if (tp->mac_version == RTL_GIGA_MAC_VER_35 ||
1309 		   tp->mac_version == RTL_GIGA_MAC_VER_36) {
1310 		if (phydev->speed == SPEED_1000) {
1311 			rtl_eri_write(tp, 0x1bc, ERIAR_MASK_1111, 0x00000011);
1312 			rtl_eri_write(tp, 0x1dc, ERIAR_MASK_1111, 0x00000005);
1313 		} else {
1314 			rtl_eri_write(tp, 0x1bc, ERIAR_MASK_1111, 0x0000001f);
1315 			rtl_eri_write(tp, 0x1dc, ERIAR_MASK_1111, 0x0000003f);
1316 		}
1317 	} else if (tp->mac_version == RTL_GIGA_MAC_VER_37) {
1318 		if (phydev->speed == SPEED_10) {
1319 			rtl_eri_write(tp, 0x1d0, ERIAR_MASK_0011, 0x4d02);
1320 			rtl_eri_write(tp, 0x1dc, ERIAR_MASK_0011, 0x0060a);
1321 		} else {
1322 			rtl_eri_write(tp, 0x1d0, ERIAR_MASK_0011, 0x0000);
1323 		}
1324 	}
1325 }
1326 
1327 #define WAKE_ANY (WAKE_PHY | WAKE_MAGIC | WAKE_UCAST | WAKE_BCAST | WAKE_MCAST)
1328 
1329 static void rtl8169_get_wol(struct net_device *dev, struct ethtool_wolinfo *wol)
1330 {
1331 	struct rtl8169_private *tp = netdev_priv(dev);
1332 
1333 	wol->supported = WAKE_ANY;
1334 	wol->wolopts = tp->saved_wolopts;
1335 }
1336 
1337 static void __rtl8169_set_wol(struct rtl8169_private *tp, u32 wolopts)
1338 {
1339 	static const struct {
1340 		u32 opt;
1341 		u16 reg;
1342 		u8  mask;
1343 	} cfg[] = {
1344 		{ WAKE_PHY,   Config3, LinkUp },
1345 		{ WAKE_UCAST, Config5, UWF },
1346 		{ WAKE_BCAST, Config5, BWF },
1347 		{ WAKE_MCAST, Config5, MWF },
1348 		{ WAKE_ANY,   Config5, LanWake },
1349 		{ WAKE_MAGIC, Config3, MagicPacket }
1350 	};
1351 	unsigned int i, tmp = ARRAY_SIZE(cfg);
1352 	u8 options;
1353 
1354 	rtl_unlock_config_regs(tp);
1355 
1356 	if (rtl_is_8168evl_up(tp)) {
1357 		tmp--;
1358 		if (wolopts & WAKE_MAGIC)
1359 			rtl_eri_set_bits(tp, 0x0dc, MagicPacket_v2);
1360 		else
1361 			rtl_eri_clear_bits(tp, 0x0dc, MagicPacket_v2);
1362 	} else if (rtl_is_8125(tp)) {
1363 		tmp--;
1364 		if (wolopts & WAKE_MAGIC)
1365 			r8168_mac_ocp_modify(tp, 0xc0b6, 0, BIT(0));
1366 		else
1367 			r8168_mac_ocp_modify(tp, 0xc0b6, BIT(0), 0);
1368 	}
1369 
1370 	for (i = 0; i < tmp; i++) {
1371 		options = RTL_R8(tp, cfg[i].reg) & ~cfg[i].mask;
1372 		if (wolopts & cfg[i].opt)
1373 			options |= cfg[i].mask;
1374 		RTL_W8(tp, cfg[i].reg, options);
1375 	}
1376 
1377 	switch (tp->mac_version) {
1378 	case RTL_GIGA_MAC_VER_02 ... RTL_GIGA_MAC_VER_06:
1379 		options = RTL_R8(tp, Config1) & ~PMEnable;
1380 		if (wolopts)
1381 			options |= PMEnable;
1382 		RTL_W8(tp, Config1, options);
1383 		break;
1384 	case RTL_GIGA_MAC_VER_34:
1385 	case RTL_GIGA_MAC_VER_37:
1386 	case RTL_GIGA_MAC_VER_39 ... RTL_GIGA_MAC_VER_63:
1387 		options = RTL_R8(tp, Config2) & ~PME_SIGNAL;
1388 		if (wolopts)
1389 			options |= PME_SIGNAL;
1390 		RTL_W8(tp, Config2, options);
1391 		break;
1392 	default:
1393 		break;
1394 	}
1395 
1396 	rtl_lock_config_regs(tp);
1397 
1398 	device_set_wakeup_enable(tp_to_dev(tp), wolopts);
1399 	tp->dev->wol_enabled = wolopts ? 1 : 0;
1400 }
1401 
1402 static int rtl8169_set_wol(struct net_device *dev, struct ethtool_wolinfo *wol)
1403 {
1404 	struct rtl8169_private *tp = netdev_priv(dev);
1405 
1406 	if (wol->wolopts & ~WAKE_ANY)
1407 		return -EINVAL;
1408 
1409 	tp->saved_wolopts = wol->wolopts;
1410 	__rtl8169_set_wol(tp, tp->saved_wolopts);
1411 
1412 	return 0;
1413 }
1414 
1415 static void rtl8169_get_drvinfo(struct net_device *dev,
1416 				struct ethtool_drvinfo *info)
1417 {
1418 	struct rtl8169_private *tp = netdev_priv(dev);
1419 	struct rtl_fw *rtl_fw = tp->rtl_fw;
1420 
1421 	strlcpy(info->driver, MODULENAME, sizeof(info->driver));
1422 	strlcpy(info->bus_info, pci_name(tp->pci_dev), sizeof(info->bus_info));
1423 	BUILD_BUG_ON(sizeof(info->fw_version) < sizeof(rtl_fw->version));
1424 	if (rtl_fw)
1425 		strlcpy(info->fw_version, rtl_fw->version,
1426 			sizeof(info->fw_version));
1427 }
1428 
1429 static int rtl8169_get_regs_len(struct net_device *dev)
1430 {
1431 	return R8169_REGS_SIZE;
1432 }
1433 
1434 static netdev_features_t rtl8169_fix_features(struct net_device *dev,
1435 	netdev_features_t features)
1436 {
1437 	struct rtl8169_private *tp = netdev_priv(dev);
1438 
1439 	if (dev->mtu > TD_MSS_MAX)
1440 		features &= ~NETIF_F_ALL_TSO;
1441 
1442 	if (dev->mtu > ETH_DATA_LEN &&
1443 	    tp->mac_version > RTL_GIGA_MAC_VER_06)
1444 		features &= ~(NETIF_F_CSUM_MASK | NETIF_F_ALL_TSO);
1445 
1446 	return features;
1447 }
1448 
1449 static void rtl_set_rx_config_features(struct rtl8169_private *tp,
1450 				       netdev_features_t features)
1451 {
1452 	u32 rx_config = RTL_R32(tp, RxConfig);
1453 
1454 	if (features & NETIF_F_RXALL)
1455 		rx_config |= RX_CONFIG_ACCEPT_ERR_MASK;
1456 	else
1457 		rx_config &= ~RX_CONFIG_ACCEPT_ERR_MASK;
1458 
1459 	if (rtl_is_8125(tp)) {
1460 		if (features & NETIF_F_HW_VLAN_CTAG_RX)
1461 			rx_config |= RX_VLAN_8125;
1462 		else
1463 			rx_config &= ~RX_VLAN_8125;
1464 	}
1465 
1466 	RTL_W32(tp, RxConfig, rx_config);
1467 }
1468 
1469 static int rtl8169_set_features(struct net_device *dev,
1470 				netdev_features_t features)
1471 {
1472 	struct rtl8169_private *tp = netdev_priv(dev);
1473 
1474 	rtl_set_rx_config_features(tp, features);
1475 
1476 	if (features & NETIF_F_RXCSUM)
1477 		tp->cp_cmd |= RxChkSum;
1478 	else
1479 		tp->cp_cmd &= ~RxChkSum;
1480 
1481 	if (!rtl_is_8125(tp)) {
1482 		if (features & NETIF_F_HW_VLAN_CTAG_RX)
1483 			tp->cp_cmd |= RxVlan;
1484 		else
1485 			tp->cp_cmd &= ~RxVlan;
1486 	}
1487 
1488 	RTL_W16(tp, CPlusCmd, tp->cp_cmd);
1489 	rtl_pci_commit(tp);
1490 
1491 	return 0;
1492 }
1493 
1494 static inline u32 rtl8169_tx_vlan_tag(struct sk_buff *skb)
1495 {
1496 	return (skb_vlan_tag_present(skb)) ?
1497 		TxVlanTag | swab16(skb_vlan_tag_get(skb)) : 0x00;
1498 }
1499 
1500 static void rtl8169_rx_vlan_tag(struct RxDesc *desc, struct sk_buff *skb)
1501 {
1502 	u32 opts2 = le32_to_cpu(desc->opts2);
1503 
1504 	if (opts2 & RxVlanTag)
1505 		__vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), swab16(opts2 & 0xffff));
1506 }
1507 
1508 static void rtl8169_get_regs(struct net_device *dev, struct ethtool_regs *regs,
1509 			     void *p)
1510 {
1511 	struct rtl8169_private *tp = netdev_priv(dev);
1512 	u32 __iomem *data = tp->mmio_addr;
1513 	u32 *dw = p;
1514 	int i;
1515 
1516 	for (i = 0; i < R8169_REGS_SIZE; i += 4)
1517 		memcpy_fromio(dw++, data++, 4);
1518 }
1519 
1520 static const char rtl8169_gstrings[][ETH_GSTRING_LEN] = {
1521 	"tx_packets",
1522 	"rx_packets",
1523 	"tx_errors",
1524 	"rx_errors",
1525 	"rx_missed",
1526 	"align_errors",
1527 	"tx_single_collisions",
1528 	"tx_multi_collisions",
1529 	"unicast",
1530 	"broadcast",
1531 	"multicast",
1532 	"tx_aborted",
1533 	"tx_underrun",
1534 };
1535 
1536 static int rtl8169_get_sset_count(struct net_device *dev, int sset)
1537 {
1538 	switch (sset) {
1539 	case ETH_SS_STATS:
1540 		return ARRAY_SIZE(rtl8169_gstrings);
1541 	default:
1542 		return -EOPNOTSUPP;
1543 	}
1544 }
1545 
1546 DECLARE_RTL_COND(rtl_counters_cond)
1547 {
1548 	return RTL_R32(tp, CounterAddrLow) & (CounterReset | CounterDump);
1549 }
1550 
1551 static void rtl8169_do_counters(struct rtl8169_private *tp, u32 counter_cmd)
1552 {
1553 	dma_addr_t paddr = tp->counters_phys_addr;
1554 	u32 cmd;
1555 
1556 	RTL_W32(tp, CounterAddrHigh, (u64)paddr >> 32);
1557 	rtl_pci_commit(tp);
1558 	cmd = (u64)paddr & DMA_BIT_MASK(32);
1559 	RTL_W32(tp, CounterAddrLow, cmd);
1560 	RTL_W32(tp, CounterAddrLow, cmd | counter_cmd);
1561 
1562 	rtl_loop_wait_low(tp, &rtl_counters_cond, 10, 1000);
1563 }
1564 
1565 static void rtl8169_update_counters(struct rtl8169_private *tp)
1566 {
1567 	u8 val = RTL_R8(tp, ChipCmd);
1568 
1569 	/*
1570 	 * Some chips are unable to dump tally counters when the receiver
1571 	 * is disabled. If 0xff chip may be in a PCI power-save state.
1572 	 */
1573 	if (val & CmdRxEnb && val != 0xff)
1574 		rtl8169_do_counters(tp, CounterDump);
1575 }
1576 
1577 static void rtl8169_init_counter_offsets(struct rtl8169_private *tp)
1578 {
1579 	struct rtl8169_counters *counters = tp->counters;
1580 
1581 	/*
1582 	 * rtl8169_init_counter_offsets is called from rtl_open.  On chip
1583 	 * versions prior to RTL_GIGA_MAC_VER_19 the tally counters are only
1584 	 * reset by a power cycle, while the counter values collected by the
1585 	 * driver are reset at every driver unload/load cycle.
1586 	 *
1587 	 * To make sure the HW values returned by @get_stats64 match the SW
1588 	 * values, we collect the initial values at first open(*) and use them
1589 	 * as offsets to normalize the values returned by @get_stats64.
1590 	 *
1591 	 * (*) We can't call rtl8169_init_counter_offsets from rtl_init_one
1592 	 * for the reason stated in rtl8169_update_counters; CmdRxEnb is only
1593 	 * set at open time by rtl_hw_start.
1594 	 */
1595 
1596 	if (tp->tc_offset.inited)
1597 		return;
1598 
1599 	if (tp->mac_version >= RTL_GIGA_MAC_VER_19) {
1600 		rtl8169_do_counters(tp, CounterReset);
1601 	} else {
1602 		rtl8169_update_counters(tp);
1603 		tp->tc_offset.tx_errors = counters->tx_errors;
1604 		tp->tc_offset.tx_multi_collision = counters->tx_multi_collision;
1605 		tp->tc_offset.tx_aborted = counters->tx_aborted;
1606 		tp->tc_offset.rx_missed = counters->rx_missed;
1607 	}
1608 
1609 	tp->tc_offset.inited = true;
1610 }
1611 
1612 static void rtl8169_get_ethtool_stats(struct net_device *dev,
1613 				      struct ethtool_stats *stats, u64 *data)
1614 {
1615 	struct rtl8169_private *tp = netdev_priv(dev);
1616 	struct rtl8169_counters *counters;
1617 
1618 	counters = tp->counters;
1619 	rtl8169_update_counters(tp);
1620 
1621 	data[0] = le64_to_cpu(counters->tx_packets);
1622 	data[1] = le64_to_cpu(counters->rx_packets);
1623 	data[2] = le64_to_cpu(counters->tx_errors);
1624 	data[3] = le32_to_cpu(counters->rx_errors);
1625 	data[4] = le16_to_cpu(counters->rx_missed);
1626 	data[5] = le16_to_cpu(counters->align_errors);
1627 	data[6] = le32_to_cpu(counters->tx_one_collision);
1628 	data[7] = le32_to_cpu(counters->tx_multi_collision);
1629 	data[8] = le64_to_cpu(counters->rx_unicast);
1630 	data[9] = le64_to_cpu(counters->rx_broadcast);
1631 	data[10] = le32_to_cpu(counters->rx_multicast);
1632 	data[11] = le16_to_cpu(counters->tx_aborted);
1633 	data[12] = le16_to_cpu(counters->tx_underun);
1634 }
1635 
1636 static void rtl8169_get_strings(struct net_device *dev, u32 stringset, u8 *data)
1637 {
1638 	switch(stringset) {
1639 	case ETH_SS_STATS:
1640 		memcpy(data, *rtl8169_gstrings, sizeof(rtl8169_gstrings));
1641 		break;
1642 	}
1643 }
1644 
1645 /*
1646  * Interrupt coalescing
1647  *
1648  * > 1 - the availability of the IntrMitigate (0xe2) register through the
1649  * >     8169, 8168 and 810x line of chipsets
1650  *
1651  * 8169, 8168, and 8136(810x) serial chipsets support it.
1652  *
1653  * > 2 - the Tx timer unit at gigabit speed
1654  *
1655  * The unit of the timer depends on both the speed and the setting of CPlusCmd
1656  * (0xe0) bit 1 and bit 0.
1657  *
1658  * For 8169
1659  * bit[1:0] \ speed        1000M           100M            10M
1660  * 0 0                     320ns           2.56us          40.96us
1661  * 0 1                     2.56us          20.48us         327.7us
1662  * 1 0                     5.12us          40.96us         655.4us
1663  * 1 1                     10.24us         81.92us         1.31ms
1664  *
1665  * For the other
1666  * bit[1:0] \ speed        1000M           100M            10M
1667  * 0 0                     5us             2.56us          40.96us
1668  * 0 1                     40us            20.48us         327.7us
1669  * 1 0                     80us            40.96us         655.4us
1670  * 1 1                     160us           81.92us         1.31ms
1671  */
1672 
1673 /* rx/tx scale factors for all CPlusCmd[0:1] cases */
1674 struct rtl_coalesce_info {
1675 	u32 speed;
1676 	u32 scale_nsecs[4];
1677 };
1678 
1679 /* produce array with base delay *1, *8, *8*2, *8*2*2 */
1680 #define COALESCE_DELAY(d) { (d), 8 * (d), 16 * (d), 32 * (d) }
1681 
1682 static const struct rtl_coalesce_info rtl_coalesce_info_8169[] = {
1683 	{ SPEED_1000,	COALESCE_DELAY(320) },
1684 	{ SPEED_100,	COALESCE_DELAY(2560) },
1685 	{ SPEED_10,	COALESCE_DELAY(40960) },
1686 	{ 0 },
1687 };
1688 
1689 static const struct rtl_coalesce_info rtl_coalesce_info_8168_8136[] = {
1690 	{ SPEED_1000,	COALESCE_DELAY(5000) },
1691 	{ SPEED_100,	COALESCE_DELAY(2560) },
1692 	{ SPEED_10,	COALESCE_DELAY(40960) },
1693 	{ 0 },
1694 };
1695 #undef COALESCE_DELAY
1696 
1697 /* get rx/tx scale vector corresponding to current speed */
1698 static const struct rtl_coalesce_info *
1699 rtl_coalesce_info(struct rtl8169_private *tp)
1700 {
1701 	const struct rtl_coalesce_info *ci;
1702 
1703 	if (tp->mac_version <= RTL_GIGA_MAC_VER_06)
1704 		ci = rtl_coalesce_info_8169;
1705 	else
1706 		ci = rtl_coalesce_info_8168_8136;
1707 
1708 	/* if speed is unknown assume highest one */
1709 	if (tp->phydev->speed == SPEED_UNKNOWN)
1710 		return ci;
1711 
1712 	for (; ci->speed; ci++) {
1713 		if (tp->phydev->speed == ci->speed)
1714 			return ci;
1715 	}
1716 
1717 	return ERR_PTR(-ELNRNG);
1718 }
1719 
1720 static int rtl_get_coalesce(struct net_device *dev, struct ethtool_coalesce *ec)
1721 {
1722 	struct rtl8169_private *tp = netdev_priv(dev);
1723 	const struct rtl_coalesce_info *ci;
1724 	u32 scale, c_us, c_fr;
1725 	u16 intrmit;
1726 
1727 	if (rtl_is_8125(tp))
1728 		return -EOPNOTSUPP;
1729 
1730 	memset(ec, 0, sizeof(*ec));
1731 
1732 	/* get rx/tx scale corresponding to current speed and CPlusCmd[0:1] */
1733 	ci = rtl_coalesce_info(tp);
1734 	if (IS_ERR(ci))
1735 		return PTR_ERR(ci);
1736 
1737 	scale = ci->scale_nsecs[tp->cp_cmd & INTT_MASK];
1738 
1739 	intrmit = RTL_R16(tp, IntrMitigate);
1740 
1741 	c_us = FIELD_GET(RTL_COALESCE_TX_USECS, intrmit);
1742 	ec->tx_coalesce_usecs = DIV_ROUND_UP(c_us * scale, 1000);
1743 
1744 	c_fr = FIELD_GET(RTL_COALESCE_TX_FRAMES, intrmit);
1745 	/* ethtool_coalesce states usecs and max_frames must not both be 0 */
1746 	ec->tx_max_coalesced_frames = (c_us || c_fr) ? c_fr * 4 : 1;
1747 
1748 	c_us = FIELD_GET(RTL_COALESCE_RX_USECS, intrmit);
1749 	ec->rx_coalesce_usecs = DIV_ROUND_UP(c_us * scale, 1000);
1750 
1751 	c_fr = FIELD_GET(RTL_COALESCE_RX_FRAMES, intrmit);
1752 	ec->rx_max_coalesced_frames = (c_us || c_fr) ? c_fr * 4 : 1;
1753 
1754 	return 0;
1755 }
1756 
1757 /* choose appropriate scale factor and CPlusCmd[0:1] for (speed, usec) */
1758 static int rtl_coalesce_choose_scale(struct rtl8169_private *tp, u32 usec,
1759 				     u16 *cp01)
1760 {
1761 	const struct rtl_coalesce_info *ci;
1762 	u16 i;
1763 
1764 	ci = rtl_coalesce_info(tp);
1765 	if (IS_ERR(ci))
1766 		return PTR_ERR(ci);
1767 
1768 	for (i = 0; i < 4; i++) {
1769 		if (usec <= ci->scale_nsecs[i] * RTL_COALESCE_T_MAX / 1000U) {
1770 			*cp01 = i;
1771 			return ci->scale_nsecs[i];
1772 		}
1773 	}
1774 
1775 	return -ERANGE;
1776 }
1777 
1778 static int rtl_set_coalesce(struct net_device *dev, struct ethtool_coalesce *ec)
1779 {
1780 	struct rtl8169_private *tp = netdev_priv(dev);
1781 	u32 tx_fr = ec->tx_max_coalesced_frames;
1782 	u32 rx_fr = ec->rx_max_coalesced_frames;
1783 	u32 coal_usec_max, units;
1784 	u16 w = 0, cp01 = 0;
1785 	int scale;
1786 
1787 	if (rtl_is_8125(tp))
1788 		return -EOPNOTSUPP;
1789 
1790 	if (rx_fr > RTL_COALESCE_FRAME_MAX || tx_fr > RTL_COALESCE_FRAME_MAX)
1791 		return -ERANGE;
1792 
1793 	coal_usec_max = max(ec->rx_coalesce_usecs, ec->tx_coalesce_usecs);
1794 	scale = rtl_coalesce_choose_scale(tp, coal_usec_max, &cp01);
1795 	if (scale < 0)
1796 		return scale;
1797 
1798 	/* Accept max_frames=1 we returned in rtl_get_coalesce. Accept it
1799 	 * not only when usecs=0 because of e.g. the following scenario:
1800 	 *
1801 	 * - both rx_usecs=0 & rx_frames=0 in hardware (no delay on RX)
1802 	 * - rtl_get_coalesce returns rx_usecs=0, rx_frames=1
1803 	 * - then user does `ethtool -C eth0 rx-usecs 100`
1804 	 *
1805 	 * Since ethtool sends to kernel whole ethtool_coalesce settings,
1806 	 * if we want to ignore rx_frames then it has to be set to 0.
1807 	 */
1808 	if (rx_fr == 1)
1809 		rx_fr = 0;
1810 	if (tx_fr == 1)
1811 		tx_fr = 0;
1812 
1813 	/* HW requires time limit to be set if frame limit is set */
1814 	if ((tx_fr && !ec->tx_coalesce_usecs) ||
1815 	    (rx_fr && !ec->rx_coalesce_usecs))
1816 		return -EINVAL;
1817 
1818 	w |= FIELD_PREP(RTL_COALESCE_TX_FRAMES, DIV_ROUND_UP(tx_fr, 4));
1819 	w |= FIELD_PREP(RTL_COALESCE_RX_FRAMES, DIV_ROUND_UP(rx_fr, 4));
1820 
1821 	units = DIV_ROUND_UP(ec->tx_coalesce_usecs * 1000U, scale);
1822 	w |= FIELD_PREP(RTL_COALESCE_TX_USECS, units);
1823 	units = DIV_ROUND_UP(ec->rx_coalesce_usecs * 1000U, scale);
1824 	w |= FIELD_PREP(RTL_COALESCE_RX_USECS, units);
1825 
1826 	RTL_W16(tp, IntrMitigate, w);
1827 
1828 	/* Meaning of PktCntrDisable bit changed from RTL8168e-vl */
1829 	if (rtl_is_8168evl_up(tp)) {
1830 		if (!rx_fr && !tx_fr)
1831 			/* disable packet counter */
1832 			tp->cp_cmd |= PktCntrDisable;
1833 		else
1834 			tp->cp_cmd &= ~PktCntrDisable;
1835 	}
1836 
1837 	tp->cp_cmd = (tp->cp_cmd & ~INTT_MASK) | cp01;
1838 	RTL_W16(tp, CPlusCmd, tp->cp_cmd);
1839 	rtl_pci_commit(tp);
1840 
1841 	return 0;
1842 }
1843 
1844 static int rtl8169_get_eee(struct net_device *dev, struct ethtool_eee *data)
1845 {
1846 	struct rtl8169_private *tp = netdev_priv(dev);
1847 
1848 	if (!rtl_supports_eee(tp))
1849 		return -EOPNOTSUPP;
1850 
1851 	return phy_ethtool_get_eee(tp->phydev, data);
1852 }
1853 
1854 static int rtl8169_set_eee(struct net_device *dev, struct ethtool_eee *data)
1855 {
1856 	struct rtl8169_private *tp = netdev_priv(dev);
1857 	int ret;
1858 
1859 	if (!rtl_supports_eee(tp))
1860 		return -EOPNOTSUPP;
1861 
1862 	ret = phy_ethtool_set_eee(tp->phydev, data);
1863 
1864 	if (!ret)
1865 		tp->eee_adv = phy_read_mmd(dev->phydev, MDIO_MMD_AN,
1866 					   MDIO_AN_EEE_ADV);
1867 	return ret;
1868 }
1869 
1870 static const struct ethtool_ops rtl8169_ethtool_ops = {
1871 	.supported_coalesce_params = ETHTOOL_COALESCE_USECS |
1872 				     ETHTOOL_COALESCE_MAX_FRAMES,
1873 	.get_drvinfo		= rtl8169_get_drvinfo,
1874 	.get_regs_len		= rtl8169_get_regs_len,
1875 	.get_link		= ethtool_op_get_link,
1876 	.get_coalesce		= rtl_get_coalesce,
1877 	.set_coalesce		= rtl_set_coalesce,
1878 	.get_regs		= rtl8169_get_regs,
1879 	.get_wol		= rtl8169_get_wol,
1880 	.set_wol		= rtl8169_set_wol,
1881 	.get_strings		= rtl8169_get_strings,
1882 	.get_sset_count		= rtl8169_get_sset_count,
1883 	.get_ethtool_stats	= rtl8169_get_ethtool_stats,
1884 	.get_ts_info		= ethtool_op_get_ts_info,
1885 	.nway_reset		= phy_ethtool_nway_reset,
1886 	.get_eee		= rtl8169_get_eee,
1887 	.set_eee		= rtl8169_set_eee,
1888 	.get_link_ksettings	= phy_ethtool_get_link_ksettings,
1889 	.set_link_ksettings	= phy_ethtool_set_link_ksettings,
1890 };
1891 
1892 static void rtl_enable_eee(struct rtl8169_private *tp)
1893 {
1894 	struct phy_device *phydev = tp->phydev;
1895 	int adv;
1896 
1897 	/* respect EEE advertisement the user may have set */
1898 	if (tp->eee_adv >= 0)
1899 		adv = tp->eee_adv;
1900 	else
1901 		adv = phy_read_mmd(phydev, MDIO_MMD_PCS, MDIO_PCS_EEE_ABLE);
1902 
1903 	if (adv >= 0)
1904 		phy_write_mmd(phydev, MDIO_MMD_AN, MDIO_AN_EEE_ADV, adv);
1905 }
1906 
1907 static enum mac_version rtl8169_get_mac_version(u16 xid, bool gmii)
1908 {
1909 	/*
1910 	 * The driver currently handles the 8168Bf and the 8168Be identically
1911 	 * but they can be identified more specifically through the test below
1912 	 * if needed:
1913 	 *
1914 	 * (RTL_R32(tp, TxConfig) & 0x700000) == 0x500000 ? 8168Bf : 8168Be
1915 	 *
1916 	 * Same thing for the 8101Eb and the 8101Ec:
1917 	 *
1918 	 * (RTL_R32(tp, TxConfig) & 0x700000) == 0x200000 ? 8101Eb : 8101Ec
1919 	 */
1920 	static const struct rtl_mac_info {
1921 		u16 mask;
1922 		u16 val;
1923 		enum mac_version ver;
1924 	} mac_info[] = {
1925 		/* 8125B family. */
1926 		{ 0x7cf, 0x641,	RTL_GIGA_MAC_VER_63 },
1927 
1928 		/* 8125A family. */
1929 		{ 0x7cf, 0x608,	RTL_GIGA_MAC_VER_60 },
1930 		{ 0x7c8, 0x608,	RTL_GIGA_MAC_VER_61 },
1931 
1932 		/* RTL8117 */
1933 		{ 0x7cf, 0x54a,	RTL_GIGA_MAC_VER_52 },
1934 
1935 		/* 8168EP family. */
1936 		{ 0x7cf, 0x502,	RTL_GIGA_MAC_VER_51 },
1937 		{ 0x7cf, 0x501,	RTL_GIGA_MAC_VER_50 },
1938 		{ 0x7cf, 0x500,	RTL_GIGA_MAC_VER_49 },
1939 
1940 		/* 8168H family. */
1941 		{ 0x7cf, 0x541,	RTL_GIGA_MAC_VER_46 },
1942 		{ 0x7cf, 0x540,	RTL_GIGA_MAC_VER_45 },
1943 
1944 		/* 8168G family. */
1945 		{ 0x7cf, 0x5c8,	RTL_GIGA_MAC_VER_44 },
1946 		{ 0x7cf, 0x509,	RTL_GIGA_MAC_VER_42 },
1947 		{ 0x7cf, 0x4c1,	RTL_GIGA_MAC_VER_41 },
1948 		{ 0x7cf, 0x4c0,	RTL_GIGA_MAC_VER_40 },
1949 
1950 		/* 8168F family. */
1951 		{ 0x7c8, 0x488,	RTL_GIGA_MAC_VER_38 },
1952 		{ 0x7cf, 0x481,	RTL_GIGA_MAC_VER_36 },
1953 		{ 0x7cf, 0x480,	RTL_GIGA_MAC_VER_35 },
1954 
1955 		/* 8168E family. */
1956 		{ 0x7c8, 0x2c8,	RTL_GIGA_MAC_VER_34 },
1957 		{ 0x7cf, 0x2c1,	RTL_GIGA_MAC_VER_32 },
1958 		{ 0x7c8, 0x2c0,	RTL_GIGA_MAC_VER_33 },
1959 
1960 		/* 8168D family. */
1961 		{ 0x7cf, 0x281,	RTL_GIGA_MAC_VER_25 },
1962 		{ 0x7c8, 0x280,	RTL_GIGA_MAC_VER_26 },
1963 
1964 		/* 8168DP family. */
1965 		{ 0x7cf, 0x288,	RTL_GIGA_MAC_VER_27 },
1966 		{ 0x7cf, 0x28a,	RTL_GIGA_MAC_VER_28 },
1967 		{ 0x7cf, 0x28b,	RTL_GIGA_MAC_VER_31 },
1968 
1969 		/* 8168C family. */
1970 		{ 0x7cf, 0x3c9,	RTL_GIGA_MAC_VER_23 },
1971 		{ 0x7cf, 0x3c8,	RTL_GIGA_MAC_VER_18 },
1972 		{ 0x7c8, 0x3c8,	RTL_GIGA_MAC_VER_24 },
1973 		{ 0x7cf, 0x3c0,	RTL_GIGA_MAC_VER_19 },
1974 		{ 0x7cf, 0x3c2,	RTL_GIGA_MAC_VER_20 },
1975 		{ 0x7cf, 0x3c3,	RTL_GIGA_MAC_VER_21 },
1976 		{ 0x7c8, 0x3c0,	RTL_GIGA_MAC_VER_22 },
1977 
1978 		/* 8168B family. */
1979 		{ 0x7cf, 0x380,	RTL_GIGA_MAC_VER_12 },
1980 		{ 0x7c8, 0x380,	RTL_GIGA_MAC_VER_17 },
1981 		{ 0x7c8, 0x300,	RTL_GIGA_MAC_VER_11 },
1982 
1983 		/* 8101 family. */
1984 		{ 0x7c8, 0x448,	RTL_GIGA_MAC_VER_39 },
1985 		{ 0x7c8, 0x440,	RTL_GIGA_MAC_VER_37 },
1986 		{ 0x7cf, 0x409,	RTL_GIGA_MAC_VER_29 },
1987 		{ 0x7c8, 0x408,	RTL_GIGA_MAC_VER_30 },
1988 		{ 0x7cf, 0x349,	RTL_GIGA_MAC_VER_08 },
1989 		{ 0x7cf, 0x249,	RTL_GIGA_MAC_VER_08 },
1990 		{ 0x7cf, 0x348,	RTL_GIGA_MAC_VER_07 },
1991 		{ 0x7cf, 0x248,	RTL_GIGA_MAC_VER_07 },
1992 		{ 0x7cf, 0x340,	RTL_GIGA_MAC_VER_13 },
1993 		{ 0x7cf, 0x240,	RTL_GIGA_MAC_VER_14 },
1994 		{ 0x7cf, 0x343,	RTL_GIGA_MAC_VER_10 },
1995 		{ 0x7cf, 0x342,	RTL_GIGA_MAC_VER_16 },
1996 		{ 0x7c8, 0x348,	RTL_GIGA_MAC_VER_09 },
1997 		{ 0x7c8, 0x248,	RTL_GIGA_MAC_VER_09 },
1998 		{ 0x7c8, 0x340,	RTL_GIGA_MAC_VER_16 },
1999 		/* FIXME: where did these entries come from ? -- FR */
2000 		{ 0xfc8, 0x388,	RTL_GIGA_MAC_VER_13 },
2001 		{ 0xfc8, 0x308,	RTL_GIGA_MAC_VER_13 },
2002 
2003 		/* 8110 family. */
2004 		{ 0xfc8, 0x980,	RTL_GIGA_MAC_VER_06 },
2005 		{ 0xfc8, 0x180,	RTL_GIGA_MAC_VER_05 },
2006 		{ 0xfc8, 0x100,	RTL_GIGA_MAC_VER_04 },
2007 		{ 0xfc8, 0x040,	RTL_GIGA_MAC_VER_03 },
2008 		{ 0xfc8, 0x008,	RTL_GIGA_MAC_VER_02 },
2009 
2010 		/* Catch-all */
2011 		{ 0x000, 0x000,	RTL_GIGA_MAC_NONE   }
2012 	};
2013 	const struct rtl_mac_info *p = mac_info;
2014 	enum mac_version ver;
2015 
2016 	while ((xid & p->mask) != p->val)
2017 		p++;
2018 	ver = p->ver;
2019 
2020 	if (ver != RTL_GIGA_MAC_NONE && !gmii) {
2021 		if (ver == RTL_GIGA_MAC_VER_42)
2022 			ver = RTL_GIGA_MAC_VER_43;
2023 		else if (ver == RTL_GIGA_MAC_VER_45)
2024 			ver = RTL_GIGA_MAC_VER_47;
2025 		else if (ver == RTL_GIGA_MAC_VER_46)
2026 			ver = RTL_GIGA_MAC_VER_48;
2027 	}
2028 
2029 	return ver;
2030 }
2031 
2032 static void rtl_release_firmware(struct rtl8169_private *tp)
2033 {
2034 	if (tp->rtl_fw) {
2035 		rtl_fw_release_firmware(tp->rtl_fw);
2036 		kfree(tp->rtl_fw);
2037 		tp->rtl_fw = NULL;
2038 	}
2039 }
2040 
2041 void r8169_apply_firmware(struct rtl8169_private *tp)
2042 {
2043 	int val;
2044 
2045 	/* TODO: release firmware if rtl_fw_write_firmware signals failure. */
2046 	if (tp->rtl_fw) {
2047 		rtl_fw_write_firmware(tp, tp->rtl_fw);
2048 		/* At least one firmware doesn't reset tp->ocp_base. */
2049 		tp->ocp_base = OCP_STD_PHY_BASE;
2050 
2051 		/* PHY soft reset may still be in progress */
2052 		phy_read_poll_timeout(tp->phydev, MII_BMCR, val,
2053 				      !(val & BMCR_RESET),
2054 				      50000, 600000, true);
2055 	}
2056 }
2057 
2058 static void rtl8168_config_eee_mac(struct rtl8169_private *tp)
2059 {
2060 	/* Adjust EEE LED frequency */
2061 	if (tp->mac_version != RTL_GIGA_MAC_VER_38)
2062 		RTL_W8(tp, EEE_LED, RTL_R8(tp, EEE_LED) & ~0x07);
2063 
2064 	rtl_eri_set_bits(tp, 0x1b0, 0x0003);
2065 }
2066 
2067 static void rtl8125a_config_eee_mac(struct rtl8169_private *tp)
2068 {
2069 	r8168_mac_ocp_modify(tp, 0xe040, 0, BIT(1) | BIT(0));
2070 	r8168_mac_ocp_modify(tp, 0xeb62, 0, BIT(2) | BIT(1));
2071 }
2072 
2073 static void rtl8125_set_eee_txidle_timer(struct rtl8169_private *tp)
2074 {
2075 	RTL_W16(tp, EEE_TXIDLE_TIMER_8125, tp->dev->mtu + ETH_HLEN + 0x20);
2076 }
2077 
2078 static void rtl8125b_config_eee_mac(struct rtl8169_private *tp)
2079 {
2080 	rtl8125_set_eee_txidle_timer(tp);
2081 	r8168_mac_ocp_modify(tp, 0xe040, 0, BIT(1) | BIT(0));
2082 }
2083 
2084 static void rtl_rar_exgmac_set(struct rtl8169_private *tp, u8 *addr)
2085 {
2086 	const u16 w[] = {
2087 		addr[0] | (addr[1] << 8),
2088 		addr[2] | (addr[3] << 8),
2089 		addr[4] | (addr[5] << 8)
2090 	};
2091 
2092 	rtl_eri_write(tp, 0xe0, ERIAR_MASK_1111, w[0] | (w[1] << 16));
2093 	rtl_eri_write(tp, 0xe4, ERIAR_MASK_1111, w[2]);
2094 	rtl_eri_write(tp, 0xf0, ERIAR_MASK_1111, w[0] << 16);
2095 	rtl_eri_write(tp, 0xf4, ERIAR_MASK_1111, w[1] | (w[2] << 16));
2096 }
2097 
2098 u16 rtl8168h_2_get_adc_bias_ioffset(struct rtl8169_private *tp)
2099 {
2100 	u16 data1, data2, ioffset;
2101 
2102 	r8168_mac_ocp_write(tp, 0xdd02, 0x807d);
2103 	data1 = r8168_mac_ocp_read(tp, 0xdd02);
2104 	data2 = r8168_mac_ocp_read(tp, 0xdd00);
2105 
2106 	ioffset = (data2 >> 1) & 0x7ff8;
2107 	ioffset |= data2 & 0x0007;
2108 	if (data1 & BIT(7))
2109 		ioffset |= BIT(15);
2110 
2111 	return ioffset;
2112 }
2113 
2114 static void rtl_schedule_task(struct rtl8169_private *tp, enum rtl_flag flag)
2115 {
2116 	set_bit(flag, tp->wk.flags);
2117 	schedule_work(&tp->wk.work);
2118 }
2119 
2120 static void rtl8169_init_phy(struct rtl8169_private *tp)
2121 {
2122 	r8169_hw_phy_config(tp, tp->phydev, tp->mac_version);
2123 
2124 	if (tp->mac_version <= RTL_GIGA_MAC_VER_06) {
2125 		pci_write_config_byte(tp->pci_dev, PCI_LATENCY_TIMER, 0x40);
2126 		pci_write_config_byte(tp->pci_dev, PCI_CACHE_LINE_SIZE, 0x08);
2127 		/* set undocumented MAC Reg C+CR Offset 0x82h */
2128 		RTL_W8(tp, 0x82, 0x01);
2129 	}
2130 
2131 	if (tp->mac_version == RTL_GIGA_MAC_VER_05 &&
2132 	    tp->pci_dev->subsystem_vendor == PCI_VENDOR_ID_GIGABYTE &&
2133 	    tp->pci_dev->subsystem_device == 0xe000)
2134 		phy_write_paged(tp->phydev, 0x0001, 0x10, 0xf01b);
2135 
2136 	/* We may have called phy_speed_down before */
2137 	phy_speed_up(tp->phydev);
2138 
2139 	if (rtl_supports_eee(tp))
2140 		rtl_enable_eee(tp);
2141 
2142 	genphy_soft_reset(tp->phydev);
2143 }
2144 
2145 static void rtl_rar_set(struct rtl8169_private *tp, u8 *addr)
2146 {
2147 	rtl_unlock_config_regs(tp);
2148 
2149 	RTL_W32(tp, MAC4, addr[4] | addr[5] << 8);
2150 	rtl_pci_commit(tp);
2151 
2152 	RTL_W32(tp, MAC0, addr[0] | addr[1] << 8 | addr[2] << 16 | addr[3] << 24);
2153 	rtl_pci_commit(tp);
2154 
2155 	if (tp->mac_version == RTL_GIGA_MAC_VER_34)
2156 		rtl_rar_exgmac_set(tp, addr);
2157 
2158 	rtl_lock_config_regs(tp);
2159 }
2160 
2161 static int rtl_set_mac_address(struct net_device *dev, void *p)
2162 {
2163 	struct rtl8169_private *tp = netdev_priv(dev);
2164 	int ret;
2165 
2166 	ret = eth_mac_addr(dev, p);
2167 	if (ret)
2168 		return ret;
2169 
2170 	rtl_rar_set(tp, dev->dev_addr);
2171 
2172 	return 0;
2173 }
2174 
2175 static void rtl_wol_suspend_quirk(struct rtl8169_private *tp)
2176 {
2177 	switch (tp->mac_version) {
2178 	case RTL_GIGA_MAC_VER_25:
2179 	case RTL_GIGA_MAC_VER_26:
2180 	case RTL_GIGA_MAC_VER_29:
2181 	case RTL_GIGA_MAC_VER_30:
2182 	case RTL_GIGA_MAC_VER_32:
2183 	case RTL_GIGA_MAC_VER_33:
2184 	case RTL_GIGA_MAC_VER_34:
2185 	case RTL_GIGA_MAC_VER_37 ... RTL_GIGA_MAC_VER_63:
2186 		RTL_W32(tp, RxConfig, RTL_R32(tp, RxConfig) |
2187 			AcceptBroadcast | AcceptMulticast | AcceptMyPhys);
2188 		break;
2189 	default:
2190 		break;
2191 	}
2192 }
2193 
2194 static void rtl_pll_power_down(struct rtl8169_private *tp)
2195 {
2196 	if (r8168_check_dash(tp))
2197 		return;
2198 
2199 	if (tp->mac_version == RTL_GIGA_MAC_VER_32 ||
2200 	    tp->mac_version == RTL_GIGA_MAC_VER_33)
2201 		rtl_ephy_write(tp, 0x19, 0xff64);
2202 
2203 	if (device_may_wakeup(tp_to_dev(tp))) {
2204 		phy_speed_down(tp->phydev, false);
2205 		rtl_wol_suspend_quirk(tp);
2206 		return;
2207 	}
2208 
2209 	switch (tp->mac_version) {
2210 	case RTL_GIGA_MAC_VER_25 ... RTL_GIGA_MAC_VER_26:
2211 	case RTL_GIGA_MAC_VER_32 ... RTL_GIGA_MAC_VER_33:
2212 	case RTL_GIGA_MAC_VER_37:
2213 	case RTL_GIGA_MAC_VER_39:
2214 	case RTL_GIGA_MAC_VER_43:
2215 	case RTL_GIGA_MAC_VER_44:
2216 	case RTL_GIGA_MAC_VER_45:
2217 	case RTL_GIGA_MAC_VER_46:
2218 	case RTL_GIGA_MAC_VER_47:
2219 	case RTL_GIGA_MAC_VER_48:
2220 	case RTL_GIGA_MAC_VER_50 ... RTL_GIGA_MAC_VER_63:
2221 		RTL_W8(tp, PMCH, RTL_R8(tp, PMCH) & ~0x80);
2222 		break;
2223 	case RTL_GIGA_MAC_VER_40:
2224 	case RTL_GIGA_MAC_VER_41:
2225 	case RTL_GIGA_MAC_VER_49:
2226 		rtl_eri_clear_bits(tp, 0x1a8, 0xfc000000);
2227 		RTL_W8(tp, PMCH, RTL_R8(tp, PMCH) & ~0x80);
2228 		break;
2229 	default:
2230 		break;
2231 	}
2232 }
2233 
2234 static void rtl_pll_power_up(struct rtl8169_private *tp)
2235 {
2236 	switch (tp->mac_version) {
2237 	case RTL_GIGA_MAC_VER_25 ... RTL_GIGA_MAC_VER_26:
2238 	case RTL_GIGA_MAC_VER_32 ... RTL_GIGA_MAC_VER_33:
2239 	case RTL_GIGA_MAC_VER_37:
2240 	case RTL_GIGA_MAC_VER_39:
2241 	case RTL_GIGA_MAC_VER_43:
2242 		RTL_W8(tp, PMCH, RTL_R8(tp, PMCH) | 0x80);
2243 		break;
2244 	case RTL_GIGA_MAC_VER_44:
2245 	case RTL_GIGA_MAC_VER_45:
2246 	case RTL_GIGA_MAC_VER_46:
2247 	case RTL_GIGA_MAC_VER_47:
2248 	case RTL_GIGA_MAC_VER_48:
2249 	case RTL_GIGA_MAC_VER_50 ... RTL_GIGA_MAC_VER_63:
2250 		RTL_W8(tp, PMCH, RTL_R8(tp, PMCH) | 0xc0);
2251 		break;
2252 	case RTL_GIGA_MAC_VER_40:
2253 	case RTL_GIGA_MAC_VER_41:
2254 	case RTL_GIGA_MAC_VER_49:
2255 		RTL_W8(tp, PMCH, RTL_R8(tp, PMCH) | 0xc0);
2256 		rtl_eri_set_bits(tp, 0x1a8, 0xfc000000);
2257 		break;
2258 	default:
2259 		break;
2260 	}
2261 
2262 	phy_resume(tp->phydev);
2263 }
2264 
2265 static void rtl_init_rxcfg(struct rtl8169_private *tp)
2266 {
2267 	switch (tp->mac_version) {
2268 	case RTL_GIGA_MAC_VER_02 ... RTL_GIGA_MAC_VER_06:
2269 	case RTL_GIGA_MAC_VER_10 ... RTL_GIGA_MAC_VER_17:
2270 		RTL_W32(tp, RxConfig, RX_FIFO_THRESH | RX_DMA_BURST);
2271 		break;
2272 	case RTL_GIGA_MAC_VER_18 ... RTL_GIGA_MAC_VER_24:
2273 	case RTL_GIGA_MAC_VER_34 ... RTL_GIGA_MAC_VER_36:
2274 	case RTL_GIGA_MAC_VER_38:
2275 		RTL_W32(tp, RxConfig, RX128_INT_EN | RX_MULTI_EN | RX_DMA_BURST);
2276 		break;
2277 	case RTL_GIGA_MAC_VER_40 ... RTL_GIGA_MAC_VER_52:
2278 		RTL_W32(tp, RxConfig, RX128_INT_EN | RX_MULTI_EN | RX_DMA_BURST | RX_EARLY_OFF);
2279 		break;
2280 	case RTL_GIGA_MAC_VER_60 ... RTL_GIGA_MAC_VER_63:
2281 		RTL_W32(tp, RxConfig, RX_FETCH_DFLT_8125 | RX_DMA_BURST);
2282 		break;
2283 	default:
2284 		RTL_W32(tp, RxConfig, RX128_INT_EN | RX_DMA_BURST);
2285 		break;
2286 	}
2287 }
2288 
2289 static void rtl8169_init_ring_indexes(struct rtl8169_private *tp)
2290 {
2291 	tp->dirty_tx = tp->cur_tx = tp->cur_rx = 0;
2292 }
2293 
2294 static void r8168c_hw_jumbo_enable(struct rtl8169_private *tp)
2295 {
2296 	RTL_W8(tp, Config3, RTL_R8(tp, Config3) | Jumbo_En0);
2297 	RTL_W8(tp, Config4, RTL_R8(tp, Config4) | Jumbo_En1);
2298 }
2299 
2300 static void r8168c_hw_jumbo_disable(struct rtl8169_private *tp)
2301 {
2302 	RTL_W8(tp, Config3, RTL_R8(tp, Config3) & ~Jumbo_En0);
2303 	RTL_W8(tp, Config4, RTL_R8(tp, Config4) & ~Jumbo_En1);
2304 }
2305 
2306 static void r8168dp_hw_jumbo_enable(struct rtl8169_private *tp)
2307 {
2308 	RTL_W8(tp, Config3, RTL_R8(tp, Config3) | Jumbo_En0);
2309 }
2310 
2311 static void r8168dp_hw_jumbo_disable(struct rtl8169_private *tp)
2312 {
2313 	RTL_W8(tp, Config3, RTL_R8(tp, Config3) & ~Jumbo_En0);
2314 }
2315 
2316 static void r8168e_hw_jumbo_enable(struct rtl8169_private *tp)
2317 {
2318 	RTL_W8(tp, MaxTxPacketSize, 0x3f);
2319 	RTL_W8(tp, Config3, RTL_R8(tp, Config3) | Jumbo_En0);
2320 	RTL_W8(tp, Config4, RTL_R8(tp, Config4) | 0x01);
2321 }
2322 
2323 static void r8168e_hw_jumbo_disable(struct rtl8169_private *tp)
2324 {
2325 	RTL_W8(tp, MaxTxPacketSize, 0x0c);
2326 	RTL_W8(tp, Config3, RTL_R8(tp, Config3) & ~Jumbo_En0);
2327 	RTL_W8(tp, Config4, RTL_R8(tp, Config4) & ~0x01);
2328 }
2329 
2330 static void r8168b_1_hw_jumbo_enable(struct rtl8169_private *tp)
2331 {
2332 	RTL_W8(tp, Config4, RTL_R8(tp, Config4) | (1 << 0));
2333 }
2334 
2335 static void r8168b_1_hw_jumbo_disable(struct rtl8169_private *tp)
2336 {
2337 	RTL_W8(tp, Config4, RTL_R8(tp, Config4) & ~(1 << 0));
2338 }
2339 
2340 static void rtl_jumbo_config(struct rtl8169_private *tp)
2341 {
2342 	bool jumbo = tp->dev->mtu > ETH_DATA_LEN;
2343 
2344 	rtl_unlock_config_regs(tp);
2345 	switch (tp->mac_version) {
2346 	case RTL_GIGA_MAC_VER_12:
2347 	case RTL_GIGA_MAC_VER_17:
2348 		if (jumbo) {
2349 			pcie_set_readrq(tp->pci_dev, 512);
2350 			r8168b_1_hw_jumbo_enable(tp);
2351 		} else {
2352 			r8168b_1_hw_jumbo_disable(tp);
2353 		}
2354 		break;
2355 	case RTL_GIGA_MAC_VER_18 ... RTL_GIGA_MAC_VER_26:
2356 		if (jumbo) {
2357 			pcie_set_readrq(tp->pci_dev, 512);
2358 			r8168c_hw_jumbo_enable(tp);
2359 		} else {
2360 			r8168c_hw_jumbo_disable(tp);
2361 		}
2362 		break;
2363 	case RTL_GIGA_MAC_VER_27 ... RTL_GIGA_MAC_VER_28:
2364 		if (jumbo)
2365 			r8168dp_hw_jumbo_enable(tp);
2366 		else
2367 			r8168dp_hw_jumbo_disable(tp);
2368 		break;
2369 	case RTL_GIGA_MAC_VER_31 ... RTL_GIGA_MAC_VER_33:
2370 		if (jumbo) {
2371 			pcie_set_readrq(tp->pci_dev, 512);
2372 			r8168e_hw_jumbo_enable(tp);
2373 		} else {
2374 			r8168e_hw_jumbo_disable(tp);
2375 		}
2376 		break;
2377 	default:
2378 		break;
2379 	}
2380 	rtl_lock_config_regs(tp);
2381 
2382 	if (!jumbo && pci_is_pcie(tp->pci_dev) && tp->supports_gmii)
2383 		pcie_set_readrq(tp->pci_dev, 4096);
2384 }
2385 
2386 DECLARE_RTL_COND(rtl_chipcmd_cond)
2387 {
2388 	return RTL_R8(tp, ChipCmd) & CmdReset;
2389 }
2390 
2391 static void rtl_hw_reset(struct rtl8169_private *tp)
2392 {
2393 	RTL_W8(tp, ChipCmd, CmdReset);
2394 
2395 	rtl_loop_wait_low(tp, &rtl_chipcmd_cond, 100, 100);
2396 }
2397 
2398 static void rtl_request_firmware(struct rtl8169_private *tp)
2399 {
2400 	struct rtl_fw *rtl_fw;
2401 
2402 	/* firmware loaded already or no firmware available */
2403 	if (tp->rtl_fw || !tp->fw_name)
2404 		return;
2405 
2406 	rtl_fw = kzalloc(sizeof(*rtl_fw), GFP_KERNEL);
2407 	if (!rtl_fw)
2408 		return;
2409 
2410 	rtl_fw->phy_write = rtl_writephy;
2411 	rtl_fw->phy_read = rtl_readphy;
2412 	rtl_fw->mac_mcu_write = mac_mcu_write;
2413 	rtl_fw->mac_mcu_read = mac_mcu_read;
2414 	rtl_fw->fw_name = tp->fw_name;
2415 	rtl_fw->dev = tp_to_dev(tp);
2416 
2417 	if (rtl_fw_request_firmware(rtl_fw))
2418 		kfree(rtl_fw);
2419 	else
2420 		tp->rtl_fw = rtl_fw;
2421 }
2422 
2423 static void rtl_rx_close(struct rtl8169_private *tp)
2424 {
2425 	RTL_W32(tp, RxConfig, RTL_R32(tp, RxConfig) & ~RX_CONFIG_ACCEPT_MASK);
2426 }
2427 
2428 DECLARE_RTL_COND(rtl_npq_cond)
2429 {
2430 	return RTL_R8(tp, TxPoll) & NPQ;
2431 }
2432 
2433 DECLARE_RTL_COND(rtl_txcfg_empty_cond)
2434 {
2435 	return RTL_R32(tp, TxConfig) & TXCFG_EMPTY;
2436 }
2437 
2438 DECLARE_RTL_COND(rtl_rxtx_empty_cond)
2439 {
2440 	return (RTL_R8(tp, MCU) & RXTX_EMPTY) == RXTX_EMPTY;
2441 }
2442 
2443 DECLARE_RTL_COND(rtl_rxtx_empty_cond_2)
2444 {
2445 	/* IntrMitigate has new functionality on RTL8125 */
2446 	return (RTL_R16(tp, IntrMitigate) & 0x0103) == 0x0103;
2447 }
2448 
2449 static void rtl_wait_txrx_fifo_empty(struct rtl8169_private *tp)
2450 {
2451 	switch (tp->mac_version) {
2452 	case RTL_GIGA_MAC_VER_40 ... RTL_GIGA_MAC_VER_52:
2453 		rtl_loop_wait_high(tp, &rtl_txcfg_empty_cond, 100, 42);
2454 		rtl_loop_wait_high(tp, &rtl_rxtx_empty_cond, 100, 42);
2455 		break;
2456 	case RTL_GIGA_MAC_VER_60 ... RTL_GIGA_MAC_VER_61:
2457 		rtl_loop_wait_high(tp, &rtl_rxtx_empty_cond, 100, 42);
2458 		break;
2459 	case RTL_GIGA_MAC_VER_63:
2460 		RTL_W8(tp, ChipCmd, RTL_R8(tp, ChipCmd) | StopReq);
2461 		rtl_loop_wait_high(tp, &rtl_rxtx_empty_cond, 100, 42);
2462 		rtl_loop_wait_high(tp, &rtl_rxtx_empty_cond_2, 100, 42);
2463 		break;
2464 	default:
2465 		break;
2466 	}
2467 }
2468 
2469 static void rtl_enable_rxdvgate(struct rtl8169_private *tp)
2470 {
2471 	RTL_W32(tp, MISC, RTL_R32(tp, MISC) | RXDV_GATED_EN);
2472 	fsleep(2000);
2473 	rtl_wait_txrx_fifo_empty(tp);
2474 }
2475 
2476 static void rtl_set_tx_config_registers(struct rtl8169_private *tp)
2477 {
2478 	u32 val = TX_DMA_BURST << TxDMAShift |
2479 		  InterFrameGap << TxInterFrameGapShift;
2480 
2481 	if (rtl_is_8168evl_up(tp))
2482 		val |= TXCFG_AUTO_FIFO;
2483 
2484 	RTL_W32(tp, TxConfig, val);
2485 }
2486 
2487 static void rtl_set_rx_max_size(struct rtl8169_private *tp)
2488 {
2489 	/* Low hurts. Let's disable the filtering. */
2490 	RTL_W16(tp, RxMaxSize, R8169_RX_BUF_SIZE + 1);
2491 }
2492 
2493 static void rtl_set_rx_tx_desc_registers(struct rtl8169_private *tp)
2494 {
2495 	/*
2496 	 * Magic spell: some iop3xx ARM board needs the TxDescAddrHigh
2497 	 * register to be written before TxDescAddrLow to work.
2498 	 * Switching from MMIO to I/O access fixes the issue as well.
2499 	 */
2500 	RTL_W32(tp, TxDescStartAddrHigh, ((u64) tp->TxPhyAddr) >> 32);
2501 	RTL_W32(tp, TxDescStartAddrLow, ((u64) tp->TxPhyAddr) & DMA_BIT_MASK(32));
2502 	RTL_W32(tp, RxDescAddrHigh, ((u64) tp->RxPhyAddr) >> 32);
2503 	RTL_W32(tp, RxDescAddrLow, ((u64) tp->RxPhyAddr) & DMA_BIT_MASK(32));
2504 }
2505 
2506 static void rtl8169_set_magic_reg(struct rtl8169_private *tp)
2507 {
2508 	u32 val;
2509 
2510 	if (tp->mac_version == RTL_GIGA_MAC_VER_05)
2511 		val = 0x000fff00;
2512 	else if (tp->mac_version == RTL_GIGA_MAC_VER_06)
2513 		val = 0x00ffff00;
2514 	else
2515 		return;
2516 
2517 	if (RTL_R8(tp, Config2) & PCI_Clock_66MHz)
2518 		val |= 0xff;
2519 
2520 	RTL_W32(tp, 0x7c, val);
2521 }
2522 
2523 static void rtl_set_rx_mode(struct net_device *dev)
2524 {
2525 	u32 rx_mode = AcceptBroadcast | AcceptMyPhys | AcceptMulticast;
2526 	/* Multicast hash filter */
2527 	u32 mc_filter[2] = { 0xffffffff, 0xffffffff };
2528 	struct rtl8169_private *tp = netdev_priv(dev);
2529 	u32 tmp;
2530 
2531 	if (dev->flags & IFF_PROMISC) {
2532 		rx_mode |= AcceptAllPhys;
2533 	} else if (netdev_mc_count(dev) > MC_FILTER_LIMIT ||
2534 		   dev->flags & IFF_ALLMULTI ||
2535 		   tp->mac_version == RTL_GIGA_MAC_VER_35) {
2536 		/* accept all multicasts */
2537 	} else if (netdev_mc_empty(dev)) {
2538 		rx_mode &= ~AcceptMulticast;
2539 	} else {
2540 		struct netdev_hw_addr *ha;
2541 
2542 		mc_filter[1] = mc_filter[0] = 0;
2543 		netdev_for_each_mc_addr(ha, dev) {
2544 			u32 bit_nr = eth_hw_addr_crc(ha) >> 26;
2545 			mc_filter[bit_nr >> 5] |= BIT(bit_nr & 31);
2546 		}
2547 
2548 		if (tp->mac_version > RTL_GIGA_MAC_VER_06) {
2549 			tmp = mc_filter[0];
2550 			mc_filter[0] = swab32(mc_filter[1]);
2551 			mc_filter[1] = swab32(tmp);
2552 		}
2553 	}
2554 
2555 	RTL_W32(tp, MAR0 + 4, mc_filter[1]);
2556 	RTL_W32(tp, MAR0 + 0, mc_filter[0]);
2557 
2558 	tmp = RTL_R32(tp, RxConfig);
2559 	RTL_W32(tp, RxConfig, (tmp & ~RX_CONFIG_ACCEPT_OK_MASK) | rx_mode);
2560 }
2561 
2562 DECLARE_RTL_COND(rtl_csiar_cond)
2563 {
2564 	return RTL_R32(tp, CSIAR) & CSIAR_FLAG;
2565 }
2566 
2567 static void rtl_csi_write(struct rtl8169_private *tp, int addr, int value)
2568 {
2569 	u32 func = PCI_FUNC(tp->pci_dev->devfn);
2570 
2571 	RTL_W32(tp, CSIDR, value);
2572 	RTL_W32(tp, CSIAR, CSIAR_WRITE_CMD | (addr & CSIAR_ADDR_MASK) |
2573 		CSIAR_BYTE_ENABLE | func << 16);
2574 
2575 	rtl_loop_wait_low(tp, &rtl_csiar_cond, 10, 100);
2576 }
2577 
2578 static u32 rtl_csi_read(struct rtl8169_private *tp, int addr)
2579 {
2580 	u32 func = PCI_FUNC(tp->pci_dev->devfn);
2581 
2582 	RTL_W32(tp, CSIAR, (addr & CSIAR_ADDR_MASK) | func << 16 |
2583 		CSIAR_BYTE_ENABLE);
2584 
2585 	return rtl_loop_wait_high(tp, &rtl_csiar_cond, 10, 100) ?
2586 		RTL_R32(tp, CSIDR) : ~0;
2587 }
2588 
2589 static void rtl_csi_access_enable(struct rtl8169_private *tp, u8 val)
2590 {
2591 	struct pci_dev *pdev = tp->pci_dev;
2592 	u32 csi;
2593 
2594 	/* According to Realtek the value at config space address 0x070f
2595 	 * controls the L0s/L1 entrance latency. We try standard ECAM access
2596 	 * first and if it fails fall back to CSI.
2597 	 */
2598 	if (pdev->cfg_size > 0x070f &&
2599 	    pci_write_config_byte(pdev, 0x070f, val) == PCIBIOS_SUCCESSFUL)
2600 		return;
2601 
2602 	netdev_notice_once(tp->dev,
2603 		"No native access to PCI extended config space, falling back to CSI\n");
2604 	csi = rtl_csi_read(tp, 0x070c) & 0x00ffffff;
2605 	rtl_csi_write(tp, 0x070c, csi | val << 24);
2606 }
2607 
2608 static void rtl_set_def_aspm_entry_latency(struct rtl8169_private *tp)
2609 {
2610 	rtl_csi_access_enable(tp, 0x27);
2611 }
2612 
2613 struct ephy_info {
2614 	unsigned int offset;
2615 	u16 mask;
2616 	u16 bits;
2617 };
2618 
2619 static void __rtl_ephy_init(struct rtl8169_private *tp,
2620 			    const struct ephy_info *e, int len)
2621 {
2622 	u16 w;
2623 
2624 	while (len-- > 0) {
2625 		w = (rtl_ephy_read(tp, e->offset) & ~e->mask) | e->bits;
2626 		rtl_ephy_write(tp, e->offset, w);
2627 		e++;
2628 	}
2629 }
2630 
2631 #define rtl_ephy_init(tp, a) __rtl_ephy_init(tp, a, ARRAY_SIZE(a))
2632 
2633 static void rtl_disable_clock_request(struct rtl8169_private *tp)
2634 {
2635 	pcie_capability_clear_word(tp->pci_dev, PCI_EXP_LNKCTL,
2636 				   PCI_EXP_LNKCTL_CLKREQ_EN);
2637 }
2638 
2639 static void rtl_enable_clock_request(struct rtl8169_private *tp)
2640 {
2641 	pcie_capability_set_word(tp->pci_dev, PCI_EXP_LNKCTL,
2642 				 PCI_EXP_LNKCTL_CLKREQ_EN);
2643 }
2644 
2645 static void rtl_pcie_state_l2l3_disable(struct rtl8169_private *tp)
2646 {
2647 	/* work around an issue when PCI reset occurs during L2/L3 state */
2648 	RTL_W8(tp, Config3, RTL_R8(tp, Config3) & ~Rdy_to_L23);
2649 }
2650 
2651 static void rtl_hw_aspm_clkreq_enable(struct rtl8169_private *tp, bool enable)
2652 {
2653 	/* Don't enable ASPM in the chip if OS can't control ASPM */
2654 	if (enable && tp->aspm_manageable) {
2655 		RTL_W8(tp, Config5, RTL_R8(tp, Config5) | ASPM_en);
2656 		RTL_W8(tp, Config2, RTL_R8(tp, Config2) | ClkReqEn);
2657 	} else {
2658 		RTL_W8(tp, Config2, RTL_R8(tp, Config2) & ~ClkReqEn);
2659 		RTL_W8(tp, Config5, RTL_R8(tp, Config5) & ~ASPM_en);
2660 	}
2661 
2662 	udelay(10);
2663 }
2664 
2665 static void rtl_set_fifo_size(struct rtl8169_private *tp, u16 rx_stat,
2666 			      u16 tx_stat, u16 rx_dyn, u16 tx_dyn)
2667 {
2668 	/* Usage of dynamic vs. static FIFO is controlled by bit
2669 	 * TXCFG_AUTO_FIFO. Exact meaning of FIFO values isn't known.
2670 	 */
2671 	rtl_eri_write(tp, 0xc8, ERIAR_MASK_1111, (rx_stat << 16) | rx_dyn);
2672 	rtl_eri_write(tp, 0xe8, ERIAR_MASK_1111, (tx_stat << 16) | tx_dyn);
2673 }
2674 
2675 static void rtl8168g_set_pause_thresholds(struct rtl8169_private *tp,
2676 					  u8 low, u8 high)
2677 {
2678 	/* FIFO thresholds for pause flow control */
2679 	rtl_eri_write(tp, 0xcc, ERIAR_MASK_0001, low);
2680 	rtl_eri_write(tp, 0xd0, ERIAR_MASK_0001, high);
2681 }
2682 
2683 static void rtl_hw_start_8168b(struct rtl8169_private *tp)
2684 {
2685 	RTL_W8(tp, Config3, RTL_R8(tp, Config3) & ~Beacon_en);
2686 }
2687 
2688 static void __rtl_hw_start_8168cp(struct rtl8169_private *tp)
2689 {
2690 	RTL_W8(tp, Config1, RTL_R8(tp, Config1) | Speed_down);
2691 
2692 	RTL_W8(tp, Config3, RTL_R8(tp, Config3) & ~Beacon_en);
2693 
2694 	rtl_disable_clock_request(tp);
2695 }
2696 
2697 static void rtl_hw_start_8168cp_1(struct rtl8169_private *tp)
2698 {
2699 	static const struct ephy_info e_info_8168cp[] = {
2700 		{ 0x01, 0,	0x0001 },
2701 		{ 0x02, 0x0800,	0x1000 },
2702 		{ 0x03, 0,	0x0042 },
2703 		{ 0x06, 0x0080,	0x0000 },
2704 		{ 0x07, 0,	0x2000 }
2705 	};
2706 
2707 	rtl_set_def_aspm_entry_latency(tp);
2708 
2709 	rtl_ephy_init(tp, e_info_8168cp);
2710 
2711 	__rtl_hw_start_8168cp(tp);
2712 }
2713 
2714 static void rtl_hw_start_8168cp_2(struct rtl8169_private *tp)
2715 {
2716 	rtl_set_def_aspm_entry_latency(tp);
2717 
2718 	RTL_W8(tp, Config3, RTL_R8(tp, Config3) & ~Beacon_en);
2719 }
2720 
2721 static void rtl_hw_start_8168cp_3(struct rtl8169_private *tp)
2722 {
2723 	rtl_set_def_aspm_entry_latency(tp);
2724 
2725 	RTL_W8(tp, Config3, RTL_R8(tp, Config3) & ~Beacon_en);
2726 
2727 	/* Magic. */
2728 	RTL_W8(tp, DBG_REG, 0x20);
2729 }
2730 
2731 static void rtl_hw_start_8168c_1(struct rtl8169_private *tp)
2732 {
2733 	static const struct ephy_info e_info_8168c_1[] = {
2734 		{ 0x02, 0x0800,	0x1000 },
2735 		{ 0x03, 0,	0x0002 },
2736 		{ 0x06, 0x0080,	0x0000 }
2737 	};
2738 
2739 	rtl_set_def_aspm_entry_latency(tp);
2740 
2741 	RTL_W8(tp, DBG_REG, 0x06 | FIX_NAK_1 | FIX_NAK_2);
2742 
2743 	rtl_ephy_init(tp, e_info_8168c_1);
2744 
2745 	__rtl_hw_start_8168cp(tp);
2746 }
2747 
2748 static void rtl_hw_start_8168c_2(struct rtl8169_private *tp)
2749 {
2750 	static const struct ephy_info e_info_8168c_2[] = {
2751 		{ 0x01, 0,	0x0001 },
2752 		{ 0x03, 0x0400,	0x0020 }
2753 	};
2754 
2755 	rtl_set_def_aspm_entry_latency(tp);
2756 
2757 	rtl_ephy_init(tp, e_info_8168c_2);
2758 
2759 	__rtl_hw_start_8168cp(tp);
2760 }
2761 
2762 static void rtl_hw_start_8168c_3(struct rtl8169_private *tp)
2763 {
2764 	rtl_hw_start_8168c_2(tp);
2765 }
2766 
2767 static void rtl_hw_start_8168c_4(struct rtl8169_private *tp)
2768 {
2769 	rtl_set_def_aspm_entry_latency(tp);
2770 
2771 	__rtl_hw_start_8168cp(tp);
2772 }
2773 
2774 static void rtl_hw_start_8168d(struct rtl8169_private *tp)
2775 {
2776 	rtl_set_def_aspm_entry_latency(tp);
2777 
2778 	rtl_disable_clock_request(tp);
2779 }
2780 
2781 static void rtl_hw_start_8168d_4(struct rtl8169_private *tp)
2782 {
2783 	static const struct ephy_info e_info_8168d_4[] = {
2784 		{ 0x0b, 0x0000,	0x0048 },
2785 		{ 0x19, 0x0020,	0x0050 },
2786 		{ 0x0c, 0x0100,	0x0020 },
2787 		{ 0x10, 0x0004,	0x0000 },
2788 	};
2789 
2790 	rtl_set_def_aspm_entry_latency(tp);
2791 
2792 	rtl_ephy_init(tp, e_info_8168d_4);
2793 
2794 	rtl_enable_clock_request(tp);
2795 }
2796 
2797 static void rtl_hw_start_8168e_1(struct rtl8169_private *tp)
2798 {
2799 	static const struct ephy_info e_info_8168e_1[] = {
2800 		{ 0x00, 0x0200,	0x0100 },
2801 		{ 0x00, 0x0000,	0x0004 },
2802 		{ 0x06, 0x0002,	0x0001 },
2803 		{ 0x06, 0x0000,	0x0030 },
2804 		{ 0x07, 0x0000,	0x2000 },
2805 		{ 0x00, 0x0000,	0x0020 },
2806 		{ 0x03, 0x5800,	0x2000 },
2807 		{ 0x03, 0x0000,	0x0001 },
2808 		{ 0x01, 0x0800,	0x1000 },
2809 		{ 0x07, 0x0000,	0x4000 },
2810 		{ 0x1e, 0x0000,	0x2000 },
2811 		{ 0x19, 0xffff,	0xfe6c },
2812 		{ 0x0a, 0x0000,	0x0040 }
2813 	};
2814 
2815 	rtl_set_def_aspm_entry_latency(tp);
2816 
2817 	rtl_ephy_init(tp, e_info_8168e_1);
2818 
2819 	rtl_disable_clock_request(tp);
2820 
2821 	/* Reset tx FIFO pointer */
2822 	RTL_W32(tp, MISC, RTL_R32(tp, MISC) | TXPLA_RST);
2823 	RTL_W32(tp, MISC, RTL_R32(tp, MISC) & ~TXPLA_RST);
2824 
2825 	RTL_W8(tp, Config5, RTL_R8(tp, Config5) & ~Spi_en);
2826 }
2827 
2828 static void rtl_hw_start_8168e_2(struct rtl8169_private *tp)
2829 {
2830 	static const struct ephy_info e_info_8168e_2[] = {
2831 		{ 0x09, 0x0000,	0x0080 },
2832 		{ 0x19, 0x0000,	0x0224 },
2833 		{ 0x00, 0x0000,	0x0004 },
2834 		{ 0x0c, 0x3df0,	0x0200 },
2835 	};
2836 
2837 	rtl_set_def_aspm_entry_latency(tp);
2838 
2839 	rtl_ephy_init(tp, e_info_8168e_2);
2840 
2841 	rtl_eri_write(tp, 0xc0, ERIAR_MASK_0011, 0x0000);
2842 	rtl_eri_write(tp, 0xb8, ERIAR_MASK_1111, 0x0000);
2843 	rtl_set_fifo_size(tp, 0x10, 0x10, 0x02, 0x06);
2844 	rtl_eri_set_bits(tp, 0x0d4, 0x1f00);
2845 	rtl_eri_set_bits(tp, 0x1d0, BIT(1));
2846 	rtl_reset_packet_filter(tp);
2847 	rtl_eri_set_bits(tp, 0x1b0, BIT(4));
2848 	rtl_eri_write(tp, 0xcc, ERIAR_MASK_1111, 0x00000050);
2849 	rtl_eri_write(tp, 0xd0, ERIAR_MASK_1111, 0x07ff0060);
2850 
2851 	rtl_disable_clock_request(tp);
2852 
2853 	RTL_W8(tp, MCU, RTL_R8(tp, MCU) & ~NOW_IS_OOB);
2854 
2855 	rtl8168_config_eee_mac(tp);
2856 
2857 	RTL_W8(tp, DLLPR, RTL_R8(tp, DLLPR) | PFM_EN);
2858 	RTL_W32(tp, MISC, RTL_R32(tp, MISC) | PWM_EN);
2859 	RTL_W8(tp, Config5, RTL_R8(tp, Config5) & ~Spi_en);
2860 
2861 	rtl_hw_aspm_clkreq_enable(tp, true);
2862 }
2863 
2864 static void rtl_hw_start_8168f(struct rtl8169_private *tp)
2865 {
2866 	rtl_set_def_aspm_entry_latency(tp);
2867 
2868 	rtl_eri_write(tp, 0xc0, ERIAR_MASK_0011, 0x0000);
2869 	rtl_eri_write(tp, 0xb8, ERIAR_MASK_1111, 0x0000);
2870 	rtl_set_fifo_size(tp, 0x10, 0x10, 0x02, 0x06);
2871 	rtl_reset_packet_filter(tp);
2872 	rtl_eri_set_bits(tp, 0x1b0, BIT(4));
2873 	rtl_eri_set_bits(tp, 0x1d0, BIT(4) | BIT(1));
2874 	rtl_eri_write(tp, 0xcc, ERIAR_MASK_1111, 0x00000050);
2875 	rtl_eri_write(tp, 0xd0, ERIAR_MASK_1111, 0x00000060);
2876 
2877 	rtl_disable_clock_request(tp);
2878 
2879 	RTL_W8(tp, MCU, RTL_R8(tp, MCU) & ~NOW_IS_OOB);
2880 	RTL_W8(tp, DLLPR, RTL_R8(tp, DLLPR) | PFM_EN);
2881 	RTL_W32(tp, MISC, RTL_R32(tp, MISC) | PWM_EN);
2882 	RTL_W8(tp, Config5, RTL_R8(tp, Config5) & ~Spi_en);
2883 
2884 	rtl8168_config_eee_mac(tp);
2885 }
2886 
2887 static void rtl_hw_start_8168f_1(struct rtl8169_private *tp)
2888 {
2889 	static const struct ephy_info e_info_8168f_1[] = {
2890 		{ 0x06, 0x00c0,	0x0020 },
2891 		{ 0x08, 0x0001,	0x0002 },
2892 		{ 0x09, 0x0000,	0x0080 },
2893 		{ 0x19, 0x0000,	0x0224 },
2894 		{ 0x00, 0x0000,	0x0008 },
2895 		{ 0x0c, 0x3df0,	0x0200 },
2896 	};
2897 
2898 	rtl_hw_start_8168f(tp);
2899 
2900 	rtl_ephy_init(tp, e_info_8168f_1);
2901 
2902 	rtl_eri_set_bits(tp, 0x0d4, 0x1f00);
2903 }
2904 
2905 static void rtl_hw_start_8411(struct rtl8169_private *tp)
2906 {
2907 	static const struct ephy_info e_info_8168f_1[] = {
2908 		{ 0x06, 0x00c0,	0x0020 },
2909 		{ 0x0f, 0xffff,	0x5200 },
2910 		{ 0x19, 0x0000,	0x0224 },
2911 		{ 0x00, 0x0000,	0x0008 },
2912 		{ 0x0c, 0x3df0,	0x0200 },
2913 	};
2914 
2915 	rtl_hw_start_8168f(tp);
2916 	rtl_pcie_state_l2l3_disable(tp);
2917 
2918 	rtl_ephy_init(tp, e_info_8168f_1);
2919 
2920 	rtl_eri_set_bits(tp, 0x0d4, 0x0c00);
2921 }
2922 
2923 static void rtl_hw_start_8168g(struct rtl8169_private *tp)
2924 {
2925 	rtl_set_fifo_size(tp, 0x08, 0x10, 0x02, 0x06);
2926 	rtl8168g_set_pause_thresholds(tp, 0x38, 0x48);
2927 
2928 	rtl_set_def_aspm_entry_latency(tp);
2929 
2930 	rtl_reset_packet_filter(tp);
2931 	rtl_eri_write(tp, 0x2f8, ERIAR_MASK_0011, 0x1d8f);
2932 
2933 	RTL_W32(tp, MISC, RTL_R32(tp, MISC) & ~RXDV_GATED_EN);
2934 
2935 	rtl_eri_write(tp, 0xc0, ERIAR_MASK_0011, 0x0000);
2936 	rtl_eri_write(tp, 0xb8, ERIAR_MASK_0011, 0x0000);
2937 	rtl_eri_set_bits(tp, 0x0d4, 0x1f80);
2938 
2939 	rtl8168_config_eee_mac(tp);
2940 
2941 	rtl_w0w1_eri(tp, 0x2fc, 0x01, 0x06);
2942 	rtl_eri_clear_bits(tp, 0x1b0, BIT(12));
2943 
2944 	rtl_pcie_state_l2l3_disable(tp);
2945 }
2946 
2947 static void rtl_hw_start_8168g_1(struct rtl8169_private *tp)
2948 {
2949 	static const struct ephy_info e_info_8168g_1[] = {
2950 		{ 0x00, 0x0008,	0x0000 },
2951 		{ 0x0c, 0x3ff0,	0x0820 },
2952 		{ 0x1e, 0x0000,	0x0001 },
2953 		{ 0x19, 0x8000,	0x0000 }
2954 	};
2955 
2956 	rtl_hw_start_8168g(tp);
2957 
2958 	/* disable aspm and clock request before access ephy */
2959 	rtl_hw_aspm_clkreq_enable(tp, false);
2960 	rtl_ephy_init(tp, e_info_8168g_1);
2961 	rtl_hw_aspm_clkreq_enable(tp, true);
2962 }
2963 
2964 static void rtl_hw_start_8168g_2(struct rtl8169_private *tp)
2965 {
2966 	static const struct ephy_info e_info_8168g_2[] = {
2967 		{ 0x00, 0x0008,	0x0000 },
2968 		{ 0x0c, 0x3ff0,	0x0820 },
2969 		{ 0x19, 0xffff,	0x7c00 },
2970 		{ 0x1e, 0xffff,	0x20eb },
2971 		{ 0x0d, 0xffff,	0x1666 },
2972 		{ 0x00, 0xffff,	0x10a3 },
2973 		{ 0x06, 0xffff,	0xf050 },
2974 		{ 0x04, 0x0000,	0x0010 },
2975 		{ 0x1d, 0x4000,	0x0000 },
2976 	};
2977 
2978 	rtl_hw_start_8168g(tp);
2979 
2980 	/* disable aspm and clock request before access ephy */
2981 	rtl_hw_aspm_clkreq_enable(tp, false);
2982 	rtl_ephy_init(tp, e_info_8168g_2);
2983 }
2984 
2985 static void rtl_hw_start_8411_2(struct rtl8169_private *tp)
2986 {
2987 	static const struct ephy_info e_info_8411_2[] = {
2988 		{ 0x00, 0x0008,	0x0000 },
2989 		{ 0x0c, 0x37d0,	0x0820 },
2990 		{ 0x1e, 0x0000,	0x0001 },
2991 		{ 0x19, 0x8021,	0x0000 },
2992 		{ 0x1e, 0x0000,	0x2000 },
2993 		{ 0x0d, 0x0100,	0x0200 },
2994 		{ 0x00, 0x0000,	0x0080 },
2995 		{ 0x06, 0x0000,	0x0010 },
2996 		{ 0x04, 0x0000,	0x0010 },
2997 		{ 0x1d, 0x0000,	0x4000 },
2998 	};
2999 
3000 	rtl_hw_start_8168g(tp);
3001 
3002 	/* disable aspm and clock request before access ephy */
3003 	rtl_hw_aspm_clkreq_enable(tp, false);
3004 	rtl_ephy_init(tp, e_info_8411_2);
3005 
3006 	/* The following Realtek-provided magic fixes an issue with the RX unit
3007 	 * getting confused after the PHY having been powered-down.
3008 	 */
3009 	r8168_mac_ocp_write(tp, 0xFC28, 0x0000);
3010 	r8168_mac_ocp_write(tp, 0xFC2A, 0x0000);
3011 	r8168_mac_ocp_write(tp, 0xFC2C, 0x0000);
3012 	r8168_mac_ocp_write(tp, 0xFC2E, 0x0000);
3013 	r8168_mac_ocp_write(tp, 0xFC30, 0x0000);
3014 	r8168_mac_ocp_write(tp, 0xFC32, 0x0000);
3015 	r8168_mac_ocp_write(tp, 0xFC34, 0x0000);
3016 	r8168_mac_ocp_write(tp, 0xFC36, 0x0000);
3017 	mdelay(3);
3018 	r8168_mac_ocp_write(tp, 0xFC26, 0x0000);
3019 
3020 	r8168_mac_ocp_write(tp, 0xF800, 0xE008);
3021 	r8168_mac_ocp_write(tp, 0xF802, 0xE00A);
3022 	r8168_mac_ocp_write(tp, 0xF804, 0xE00C);
3023 	r8168_mac_ocp_write(tp, 0xF806, 0xE00E);
3024 	r8168_mac_ocp_write(tp, 0xF808, 0xE027);
3025 	r8168_mac_ocp_write(tp, 0xF80A, 0xE04F);
3026 	r8168_mac_ocp_write(tp, 0xF80C, 0xE05E);
3027 	r8168_mac_ocp_write(tp, 0xF80E, 0xE065);
3028 	r8168_mac_ocp_write(tp, 0xF810, 0xC602);
3029 	r8168_mac_ocp_write(tp, 0xF812, 0xBE00);
3030 	r8168_mac_ocp_write(tp, 0xF814, 0x0000);
3031 	r8168_mac_ocp_write(tp, 0xF816, 0xC502);
3032 	r8168_mac_ocp_write(tp, 0xF818, 0xBD00);
3033 	r8168_mac_ocp_write(tp, 0xF81A, 0x074C);
3034 	r8168_mac_ocp_write(tp, 0xF81C, 0xC302);
3035 	r8168_mac_ocp_write(tp, 0xF81E, 0xBB00);
3036 	r8168_mac_ocp_write(tp, 0xF820, 0x080A);
3037 	r8168_mac_ocp_write(tp, 0xF822, 0x6420);
3038 	r8168_mac_ocp_write(tp, 0xF824, 0x48C2);
3039 	r8168_mac_ocp_write(tp, 0xF826, 0x8C20);
3040 	r8168_mac_ocp_write(tp, 0xF828, 0xC516);
3041 	r8168_mac_ocp_write(tp, 0xF82A, 0x64A4);
3042 	r8168_mac_ocp_write(tp, 0xF82C, 0x49C0);
3043 	r8168_mac_ocp_write(tp, 0xF82E, 0xF009);
3044 	r8168_mac_ocp_write(tp, 0xF830, 0x74A2);
3045 	r8168_mac_ocp_write(tp, 0xF832, 0x8CA5);
3046 	r8168_mac_ocp_write(tp, 0xF834, 0x74A0);
3047 	r8168_mac_ocp_write(tp, 0xF836, 0xC50E);
3048 	r8168_mac_ocp_write(tp, 0xF838, 0x9CA2);
3049 	r8168_mac_ocp_write(tp, 0xF83A, 0x1C11);
3050 	r8168_mac_ocp_write(tp, 0xF83C, 0x9CA0);
3051 	r8168_mac_ocp_write(tp, 0xF83E, 0xE006);
3052 	r8168_mac_ocp_write(tp, 0xF840, 0x74F8);
3053 	r8168_mac_ocp_write(tp, 0xF842, 0x48C4);
3054 	r8168_mac_ocp_write(tp, 0xF844, 0x8CF8);
3055 	r8168_mac_ocp_write(tp, 0xF846, 0xC404);
3056 	r8168_mac_ocp_write(tp, 0xF848, 0xBC00);
3057 	r8168_mac_ocp_write(tp, 0xF84A, 0xC403);
3058 	r8168_mac_ocp_write(tp, 0xF84C, 0xBC00);
3059 	r8168_mac_ocp_write(tp, 0xF84E, 0x0BF2);
3060 	r8168_mac_ocp_write(tp, 0xF850, 0x0C0A);
3061 	r8168_mac_ocp_write(tp, 0xF852, 0xE434);
3062 	r8168_mac_ocp_write(tp, 0xF854, 0xD3C0);
3063 	r8168_mac_ocp_write(tp, 0xF856, 0x49D9);
3064 	r8168_mac_ocp_write(tp, 0xF858, 0xF01F);
3065 	r8168_mac_ocp_write(tp, 0xF85A, 0xC526);
3066 	r8168_mac_ocp_write(tp, 0xF85C, 0x64A5);
3067 	r8168_mac_ocp_write(tp, 0xF85E, 0x1400);
3068 	r8168_mac_ocp_write(tp, 0xF860, 0xF007);
3069 	r8168_mac_ocp_write(tp, 0xF862, 0x0C01);
3070 	r8168_mac_ocp_write(tp, 0xF864, 0x8CA5);
3071 	r8168_mac_ocp_write(tp, 0xF866, 0x1C15);
3072 	r8168_mac_ocp_write(tp, 0xF868, 0xC51B);
3073 	r8168_mac_ocp_write(tp, 0xF86A, 0x9CA0);
3074 	r8168_mac_ocp_write(tp, 0xF86C, 0xE013);
3075 	r8168_mac_ocp_write(tp, 0xF86E, 0xC519);
3076 	r8168_mac_ocp_write(tp, 0xF870, 0x74A0);
3077 	r8168_mac_ocp_write(tp, 0xF872, 0x48C4);
3078 	r8168_mac_ocp_write(tp, 0xF874, 0x8CA0);
3079 	r8168_mac_ocp_write(tp, 0xF876, 0xC516);
3080 	r8168_mac_ocp_write(tp, 0xF878, 0x74A4);
3081 	r8168_mac_ocp_write(tp, 0xF87A, 0x48C8);
3082 	r8168_mac_ocp_write(tp, 0xF87C, 0x48CA);
3083 	r8168_mac_ocp_write(tp, 0xF87E, 0x9CA4);
3084 	r8168_mac_ocp_write(tp, 0xF880, 0xC512);
3085 	r8168_mac_ocp_write(tp, 0xF882, 0x1B00);
3086 	r8168_mac_ocp_write(tp, 0xF884, 0x9BA0);
3087 	r8168_mac_ocp_write(tp, 0xF886, 0x1B1C);
3088 	r8168_mac_ocp_write(tp, 0xF888, 0x483F);
3089 	r8168_mac_ocp_write(tp, 0xF88A, 0x9BA2);
3090 	r8168_mac_ocp_write(tp, 0xF88C, 0x1B04);
3091 	r8168_mac_ocp_write(tp, 0xF88E, 0xC508);
3092 	r8168_mac_ocp_write(tp, 0xF890, 0x9BA0);
3093 	r8168_mac_ocp_write(tp, 0xF892, 0xC505);
3094 	r8168_mac_ocp_write(tp, 0xF894, 0xBD00);
3095 	r8168_mac_ocp_write(tp, 0xF896, 0xC502);
3096 	r8168_mac_ocp_write(tp, 0xF898, 0xBD00);
3097 	r8168_mac_ocp_write(tp, 0xF89A, 0x0300);
3098 	r8168_mac_ocp_write(tp, 0xF89C, 0x051E);
3099 	r8168_mac_ocp_write(tp, 0xF89E, 0xE434);
3100 	r8168_mac_ocp_write(tp, 0xF8A0, 0xE018);
3101 	r8168_mac_ocp_write(tp, 0xF8A2, 0xE092);
3102 	r8168_mac_ocp_write(tp, 0xF8A4, 0xDE20);
3103 	r8168_mac_ocp_write(tp, 0xF8A6, 0xD3C0);
3104 	r8168_mac_ocp_write(tp, 0xF8A8, 0xC50F);
3105 	r8168_mac_ocp_write(tp, 0xF8AA, 0x76A4);
3106 	r8168_mac_ocp_write(tp, 0xF8AC, 0x49E3);
3107 	r8168_mac_ocp_write(tp, 0xF8AE, 0xF007);
3108 	r8168_mac_ocp_write(tp, 0xF8B0, 0x49C0);
3109 	r8168_mac_ocp_write(tp, 0xF8B2, 0xF103);
3110 	r8168_mac_ocp_write(tp, 0xF8B4, 0xC607);
3111 	r8168_mac_ocp_write(tp, 0xF8B6, 0xBE00);
3112 	r8168_mac_ocp_write(tp, 0xF8B8, 0xC606);
3113 	r8168_mac_ocp_write(tp, 0xF8BA, 0xBE00);
3114 	r8168_mac_ocp_write(tp, 0xF8BC, 0xC602);
3115 	r8168_mac_ocp_write(tp, 0xF8BE, 0xBE00);
3116 	r8168_mac_ocp_write(tp, 0xF8C0, 0x0C4C);
3117 	r8168_mac_ocp_write(tp, 0xF8C2, 0x0C28);
3118 	r8168_mac_ocp_write(tp, 0xF8C4, 0x0C2C);
3119 	r8168_mac_ocp_write(tp, 0xF8C6, 0xDC00);
3120 	r8168_mac_ocp_write(tp, 0xF8C8, 0xC707);
3121 	r8168_mac_ocp_write(tp, 0xF8CA, 0x1D00);
3122 	r8168_mac_ocp_write(tp, 0xF8CC, 0x8DE2);
3123 	r8168_mac_ocp_write(tp, 0xF8CE, 0x48C1);
3124 	r8168_mac_ocp_write(tp, 0xF8D0, 0xC502);
3125 	r8168_mac_ocp_write(tp, 0xF8D2, 0xBD00);
3126 	r8168_mac_ocp_write(tp, 0xF8D4, 0x00AA);
3127 	r8168_mac_ocp_write(tp, 0xF8D6, 0xE0C0);
3128 	r8168_mac_ocp_write(tp, 0xF8D8, 0xC502);
3129 	r8168_mac_ocp_write(tp, 0xF8DA, 0xBD00);
3130 	r8168_mac_ocp_write(tp, 0xF8DC, 0x0132);
3131 
3132 	r8168_mac_ocp_write(tp, 0xFC26, 0x8000);
3133 
3134 	r8168_mac_ocp_write(tp, 0xFC2A, 0x0743);
3135 	r8168_mac_ocp_write(tp, 0xFC2C, 0x0801);
3136 	r8168_mac_ocp_write(tp, 0xFC2E, 0x0BE9);
3137 	r8168_mac_ocp_write(tp, 0xFC30, 0x02FD);
3138 	r8168_mac_ocp_write(tp, 0xFC32, 0x0C25);
3139 	r8168_mac_ocp_write(tp, 0xFC34, 0x00A9);
3140 	r8168_mac_ocp_write(tp, 0xFC36, 0x012D);
3141 
3142 	rtl_hw_aspm_clkreq_enable(tp, true);
3143 }
3144 
3145 static void rtl_hw_start_8168h_1(struct rtl8169_private *tp)
3146 {
3147 	static const struct ephy_info e_info_8168h_1[] = {
3148 		{ 0x1e, 0x0800,	0x0001 },
3149 		{ 0x1d, 0x0000,	0x0800 },
3150 		{ 0x05, 0xffff,	0x2089 },
3151 		{ 0x06, 0xffff,	0x5881 },
3152 		{ 0x04, 0xffff,	0x854a },
3153 		{ 0x01, 0xffff,	0x068b }
3154 	};
3155 	int rg_saw_cnt;
3156 
3157 	/* disable aspm and clock request before access ephy */
3158 	rtl_hw_aspm_clkreq_enable(tp, false);
3159 	rtl_ephy_init(tp, e_info_8168h_1);
3160 
3161 	rtl_set_fifo_size(tp, 0x08, 0x10, 0x02, 0x06);
3162 	rtl8168g_set_pause_thresholds(tp, 0x38, 0x48);
3163 
3164 	rtl_set_def_aspm_entry_latency(tp);
3165 
3166 	rtl_reset_packet_filter(tp);
3167 
3168 	rtl_eri_set_bits(tp, 0xd4, 0x1f00);
3169 	rtl_eri_set_bits(tp, 0xdc, 0x001c);
3170 
3171 	rtl_eri_write(tp, 0x5f0, ERIAR_MASK_0011, 0x4f87);
3172 
3173 	RTL_W32(tp, MISC, RTL_R32(tp, MISC) & ~RXDV_GATED_EN);
3174 
3175 	rtl_eri_write(tp, 0xc0, ERIAR_MASK_0011, 0x0000);
3176 	rtl_eri_write(tp, 0xb8, ERIAR_MASK_0011, 0x0000);
3177 
3178 	rtl8168_config_eee_mac(tp);
3179 
3180 	RTL_W8(tp, DLLPR, RTL_R8(tp, DLLPR) & ~PFM_EN);
3181 	RTL_W8(tp, MISC_1, RTL_R8(tp, MISC_1) & ~PFM_D3COLD_EN);
3182 
3183 	RTL_W8(tp, DLLPR, RTL_R8(tp, DLLPR) & ~TX_10M_PS_EN);
3184 
3185 	rtl_eri_clear_bits(tp, 0x1b0, BIT(12));
3186 
3187 	rtl_pcie_state_l2l3_disable(tp);
3188 
3189 	rg_saw_cnt = phy_read_paged(tp->phydev, 0x0c42, 0x13) & 0x3fff;
3190 	if (rg_saw_cnt > 0) {
3191 		u16 sw_cnt_1ms_ini;
3192 
3193 		sw_cnt_1ms_ini = 16000000/rg_saw_cnt;
3194 		sw_cnt_1ms_ini &= 0x0fff;
3195 		r8168_mac_ocp_modify(tp, 0xd412, 0x0fff, sw_cnt_1ms_ini);
3196 	}
3197 
3198 	r8168_mac_ocp_modify(tp, 0xe056, 0x00f0, 0x0070);
3199 	r8168_mac_ocp_modify(tp, 0xe052, 0x6000, 0x8008);
3200 	r8168_mac_ocp_modify(tp, 0xe0d6, 0x01ff, 0x017f);
3201 	r8168_mac_ocp_modify(tp, 0xd420, 0x0fff, 0x047f);
3202 
3203 	r8168_mac_ocp_write(tp, 0xe63e, 0x0001);
3204 	r8168_mac_ocp_write(tp, 0xe63e, 0x0000);
3205 	r8168_mac_ocp_write(tp, 0xc094, 0x0000);
3206 	r8168_mac_ocp_write(tp, 0xc09e, 0x0000);
3207 
3208 	rtl_hw_aspm_clkreq_enable(tp, true);
3209 }
3210 
3211 static void rtl_hw_start_8168ep(struct rtl8169_private *tp)
3212 {
3213 	rtl8168ep_stop_cmac(tp);
3214 
3215 	rtl_set_fifo_size(tp, 0x08, 0x10, 0x02, 0x06);
3216 	rtl8168g_set_pause_thresholds(tp, 0x2f, 0x5f);
3217 
3218 	rtl_set_def_aspm_entry_latency(tp);
3219 
3220 	rtl_reset_packet_filter(tp);
3221 
3222 	rtl_eri_set_bits(tp, 0xd4, 0x1f80);
3223 
3224 	rtl_eri_write(tp, 0x5f0, ERIAR_MASK_0011, 0x4f87);
3225 
3226 	RTL_W32(tp, MISC, RTL_R32(tp, MISC) & ~RXDV_GATED_EN);
3227 
3228 	rtl_eri_write(tp, 0xc0, ERIAR_MASK_0011, 0x0000);
3229 	rtl_eri_write(tp, 0xb8, ERIAR_MASK_0011, 0x0000);
3230 
3231 	rtl8168_config_eee_mac(tp);
3232 
3233 	rtl_w0w1_eri(tp, 0x2fc, 0x01, 0x06);
3234 
3235 	RTL_W8(tp, DLLPR, RTL_R8(tp, DLLPR) & ~TX_10M_PS_EN);
3236 
3237 	rtl_pcie_state_l2l3_disable(tp);
3238 }
3239 
3240 static void rtl_hw_start_8168ep_1(struct rtl8169_private *tp)
3241 {
3242 	static const struct ephy_info e_info_8168ep_1[] = {
3243 		{ 0x00, 0xffff,	0x10ab },
3244 		{ 0x06, 0xffff,	0xf030 },
3245 		{ 0x08, 0xffff,	0x2006 },
3246 		{ 0x0d, 0xffff,	0x1666 },
3247 		{ 0x0c, 0x3ff0,	0x0000 }
3248 	};
3249 
3250 	/* disable aspm and clock request before access ephy */
3251 	rtl_hw_aspm_clkreq_enable(tp, false);
3252 	rtl_ephy_init(tp, e_info_8168ep_1);
3253 
3254 	rtl_hw_start_8168ep(tp);
3255 
3256 	rtl_hw_aspm_clkreq_enable(tp, true);
3257 }
3258 
3259 static void rtl_hw_start_8168ep_2(struct rtl8169_private *tp)
3260 {
3261 	static const struct ephy_info e_info_8168ep_2[] = {
3262 		{ 0x00, 0xffff,	0x10a3 },
3263 		{ 0x19, 0xffff,	0xfc00 },
3264 		{ 0x1e, 0xffff,	0x20ea }
3265 	};
3266 
3267 	/* disable aspm and clock request before access ephy */
3268 	rtl_hw_aspm_clkreq_enable(tp, false);
3269 	rtl_ephy_init(tp, e_info_8168ep_2);
3270 
3271 	rtl_hw_start_8168ep(tp);
3272 
3273 	RTL_W8(tp, DLLPR, RTL_R8(tp, DLLPR) & ~PFM_EN);
3274 	RTL_W8(tp, MISC_1, RTL_R8(tp, MISC_1) & ~PFM_D3COLD_EN);
3275 
3276 	rtl_hw_aspm_clkreq_enable(tp, true);
3277 }
3278 
3279 static void rtl_hw_start_8168ep_3(struct rtl8169_private *tp)
3280 {
3281 	static const struct ephy_info e_info_8168ep_3[] = {
3282 		{ 0x00, 0x0000,	0x0080 },
3283 		{ 0x0d, 0x0100,	0x0200 },
3284 		{ 0x19, 0x8021,	0x0000 },
3285 		{ 0x1e, 0x0000,	0x2000 },
3286 	};
3287 
3288 	/* disable aspm and clock request before access ephy */
3289 	rtl_hw_aspm_clkreq_enable(tp, false);
3290 	rtl_ephy_init(tp, e_info_8168ep_3);
3291 
3292 	rtl_hw_start_8168ep(tp);
3293 
3294 	RTL_W8(tp, DLLPR, RTL_R8(tp, DLLPR) & ~PFM_EN);
3295 	RTL_W8(tp, MISC_1, RTL_R8(tp, MISC_1) & ~PFM_D3COLD_EN);
3296 
3297 	r8168_mac_ocp_modify(tp, 0xd3e2, 0x0fff, 0x0271);
3298 	r8168_mac_ocp_modify(tp, 0xd3e4, 0x00ff, 0x0000);
3299 	r8168_mac_ocp_modify(tp, 0xe860, 0x0000, 0x0080);
3300 
3301 	rtl_hw_aspm_clkreq_enable(tp, true);
3302 }
3303 
3304 static void rtl_hw_start_8117(struct rtl8169_private *tp)
3305 {
3306 	static const struct ephy_info e_info_8117[] = {
3307 		{ 0x19, 0x0040,	0x1100 },
3308 		{ 0x59, 0x0040,	0x1100 },
3309 	};
3310 	int rg_saw_cnt;
3311 
3312 	rtl8168ep_stop_cmac(tp);
3313 
3314 	/* disable aspm and clock request before access ephy */
3315 	rtl_hw_aspm_clkreq_enable(tp, false);
3316 	rtl_ephy_init(tp, e_info_8117);
3317 
3318 	rtl_set_fifo_size(tp, 0x08, 0x10, 0x02, 0x06);
3319 	rtl8168g_set_pause_thresholds(tp, 0x2f, 0x5f);
3320 
3321 	rtl_set_def_aspm_entry_latency(tp);
3322 
3323 	rtl_reset_packet_filter(tp);
3324 
3325 	rtl_eri_set_bits(tp, 0xd4, 0x1f90);
3326 
3327 	rtl_eri_write(tp, 0x5f0, ERIAR_MASK_0011, 0x4f87);
3328 
3329 	RTL_W32(tp, MISC, RTL_R32(tp, MISC) & ~RXDV_GATED_EN);
3330 
3331 	rtl_eri_write(tp, 0xc0, ERIAR_MASK_0011, 0x0000);
3332 	rtl_eri_write(tp, 0xb8, ERIAR_MASK_0011, 0x0000);
3333 
3334 	rtl8168_config_eee_mac(tp);
3335 
3336 	RTL_W8(tp, DLLPR, RTL_R8(tp, DLLPR) & ~PFM_EN);
3337 	RTL_W8(tp, MISC_1, RTL_R8(tp, MISC_1) & ~PFM_D3COLD_EN);
3338 
3339 	RTL_W8(tp, DLLPR, RTL_R8(tp, DLLPR) & ~TX_10M_PS_EN);
3340 
3341 	rtl_eri_clear_bits(tp, 0x1b0, BIT(12));
3342 
3343 	rtl_pcie_state_l2l3_disable(tp);
3344 
3345 	rg_saw_cnt = phy_read_paged(tp->phydev, 0x0c42, 0x13) & 0x3fff;
3346 	if (rg_saw_cnt > 0) {
3347 		u16 sw_cnt_1ms_ini;
3348 
3349 		sw_cnt_1ms_ini = (16000000 / rg_saw_cnt) & 0x0fff;
3350 		r8168_mac_ocp_modify(tp, 0xd412, 0x0fff, sw_cnt_1ms_ini);
3351 	}
3352 
3353 	r8168_mac_ocp_modify(tp, 0xe056, 0x00f0, 0x0070);
3354 	r8168_mac_ocp_write(tp, 0xea80, 0x0003);
3355 	r8168_mac_ocp_modify(tp, 0xe052, 0x0000, 0x0009);
3356 	r8168_mac_ocp_modify(tp, 0xd420, 0x0fff, 0x047f);
3357 
3358 	r8168_mac_ocp_write(tp, 0xe63e, 0x0001);
3359 	r8168_mac_ocp_write(tp, 0xe63e, 0x0000);
3360 	r8168_mac_ocp_write(tp, 0xc094, 0x0000);
3361 	r8168_mac_ocp_write(tp, 0xc09e, 0x0000);
3362 
3363 	/* firmware is for MAC only */
3364 	r8169_apply_firmware(tp);
3365 
3366 	rtl_hw_aspm_clkreq_enable(tp, true);
3367 }
3368 
3369 static void rtl_hw_start_8102e_1(struct rtl8169_private *tp)
3370 {
3371 	static const struct ephy_info e_info_8102e_1[] = {
3372 		{ 0x01,	0, 0x6e65 },
3373 		{ 0x02,	0, 0x091f },
3374 		{ 0x03,	0, 0xc2f9 },
3375 		{ 0x06,	0, 0xafb5 },
3376 		{ 0x07,	0, 0x0e00 },
3377 		{ 0x19,	0, 0xec80 },
3378 		{ 0x01,	0, 0x2e65 },
3379 		{ 0x01,	0, 0x6e65 }
3380 	};
3381 	u8 cfg1;
3382 
3383 	rtl_set_def_aspm_entry_latency(tp);
3384 
3385 	RTL_W8(tp, DBG_REG, FIX_NAK_1);
3386 
3387 	RTL_W8(tp, Config1,
3388 	       LEDS1 | LEDS0 | Speed_down | MEMMAP | IOMAP | VPD | PMEnable);
3389 	RTL_W8(tp, Config3, RTL_R8(tp, Config3) & ~Beacon_en);
3390 
3391 	cfg1 = RTL_R8(tp, Config1);
3392 	if ((cfg1 & LEDS0) && (cfg1 & LEDS1))
3393 		RTL_W8(tp, Config1, cfg1 & ~LEDS0);
3394 
3395 	rtl_ephy_init(tp, e_info_8102e_1);
3396 }
3397 
3398 static void rtl_hw_start_8102e_2(struct rtl8169_private *tp)
3399 {
3400 	rtl_set_def_aspm_entry_latency(tp);
3401 
3402 	RTL_W8(tp, Config1, MEMMAP | IOMAP | VPD | PMEnable);
3403 	RTL_W8(tp, Config3, RTL_R8(tp, Config3) & ~Beacon_en);
3404 }
3405 
3406 static void rtl_hw_start_8102e_3(struct rtl8169_private *tp)
3407 {
3408 	rtl_hw_start_8102e_2(tp);
3409 
3410 	rtl_ephy_write(tp, 0x03, 0xc2f9);
3411 }
3412 
3413 static void rtl_hw_start_8401(struct rtl8169_private *tp)
3414 {
3415 	static const struct ephy_info e_info_8401[] = {
3416 		{ 0x01,	0xffff, 0x6fe5 },
3417 		{ 0x03,	0xffff, 0x0599 },
3418 		{ 0x06,	0xffff, 0xaf25 },
3419 		{ 0x07,	0xffff, 0x8e68 },
3420 	};
3421 
3422 	rtl_ephy_init(tp, e_info_8401);
3423 	RTL_W8(tp, Config3, RTL_R8(tp, Config3) & ~Beacon_en);
3424 }
3425 
3426 static void rtl_hw_start_8105e_1(struct rtl8169_private *tp)
3427 {
3428 	static const struct ephy_info e_info_8105e_1[] = {
3429 		{ 0x07,	0, 0x4000 },
3430 		{ 0x19,	0, 0x0200 },
3431 		{ 0x19,	0, 0x0020 },
3432 		{ 0x1e,	0, 0x2000 },
3433 		{ 0x03,	0, 0x0001 },
3434 		{ 0x19,	0, 0x0100 },
3435 		{ 0x19,	0, 0x0004 },
3436 		{ 0x0a,	0, 0x0020 }
3437 	};
3438 
3439 	/* Force LAN exit from ASPM if Rx/Tx are not idle */
3440 	RTL_W32(tp, FuncEvent, RTL_R32(tp, FuncEvent) | 0x002800);
3441 
3442 	/* Disable Early Tally Counter */
3443 	RTL_W32(tp, FuncEvent, RTL_R32(tp, FuncEvent) & ~0x010000);
3444 
3445 	RTL_W8(tp, MCU, RTL_R8(tp, MCU) | EN_NDP | EN_OOB_RESET);
3446 	RTL_W8(tp, DLLPR, RTL_R8(tp, DLLPR) | PFM_EN);
3447 
3448 	rtl_ephy_init(tp, e_info_8105e_1);
3449 
3450 	rtl_pcie_state_l2l3_disable(tp);
3451 }
3452 
3453 static void rtl_hw_start_8105e_2(struct rtl8169_private *tp)
3454 {
3455 	rtl_hw_start_8105e_1(tp);
3456 	rtl_ephy_write(tp, 0x1e, rtl_ephy_read(tp, 0x1e) | 0x8000);
3457 }
3458 
3459 static void rtl_hw_start_8402(struct rtl8169_private *tp)
3460 {
3461 	static const struct ephy_info e_info_8402[] = {
3462 		{ 0x19,	0xffff, 0xff64 },
3463 		{ 0x1e,	0, 0x4000 }
3464 	};
3465 
3466 	rtl_set_def_aspm_entry_latency(tp);
3467 
3468 	/* Force LAN exit from ASPM if Rx/Tx are not idle */
3469 	RTL_W32(tp, FuncEvent, RTL_R32(tp, FuncEvent) | 0x002800);
3470 
3471 	RTL_W8(tp, MCU, RTL_R8(tp, MCU) & ~NOW_IS_OOB);
3472 
3473 	rtl_ephy_init(tp, e_info_8402);
3474 
3475 	rtl_set_fifo_size(tp, 0x00, 0x00, 0x02, 0x06);
3476 	rtl_reset_packet_filter(tp);
3477 	rtl_eri_write(tp, 0xc0, ERIAR_MASK_0011, 0x0000);
3478 	rtl_eri_write(tp, 0xb8, ERIAR_MASK_0011, 0x0000);
3479 	rtl_w0w1_eri(tp, 0x0d4, 0x0e00, 0xff00);
3480 
3481 	/* disable EEE */
3482 	rtl_eri_write(tp, 0x1b0, ERIAR_MASK_0011, 0x0000);
3483 
3484 	rtl_pcie_state_l2l3_disable(tp);
3485 }
3486 
3487 static void rtl_hw_start_8106(struct rtl8169_private *tp)
3488 {
3489 	rtl_hw_aspm_clkreq_enable(tp, false);
3490 
3491 	/* Force LAN exit from ASPM if Rx/Tx are not idle */
3492 	RTL_W32(tp, FuncEvent, RTL_R32(tp, FuncEvent) | 0x002800);
3493 
3494 	RTL_W32(tp, MISC, (RTL_R32(tp, MISC) | DISABLE_LAN_EN) & ~EARLY_TALLY_EN);
3495 	RTL_W8(tp, MCU, RTL_R8(tp, MCU) | EN_NDP | EN_OOB_RESET);
3496 	RTL_W8(tp, DLLPR, RTL_R8(tp, DLLPR) & ~PFM_EN);
3497 
3498 	rtl_eri_write(tp, 0x1d0, ERIAR_MASK_0011, 0x0000);
3499 
3500 	/* disable EEE */
3501 	rtl_eri_write(tp, 0x1b0, ERIAR_MASK_0011, 0x0000);
3502 
3503 	rtl_pcie_state_l2l3_disable(tp);
3504 	rtl_hw_aspm_clkreq_enable(tp, true);
3505 }
3506 
3507 DECLARE_RTL_COND(rtl_mac_ocp_e00e_cond)
3508 {
3509 	return r8168_mac_ocp_read(tp, 0xe00e) & BIT(13);
3510 }
3511 
3512 static void rtl_hw_start_8125_common(struct rtl8169_private *tp)
3513 {
3514 	rtl_pcie_state_l2l3_disable(tp);
3515 
3516 	RTL_W16(tp, 0x382, 0x221b);
3517 	RTL_W8(tp, 0x4500, 0);
3518 	RTL_W16(tp, 0x4800, 0);
3519 
3520 	/* disable UPS */
3521 	r8168_mac_ocp_modify(tp, 0xd40a, 0x0010, 0x0000);
3522 
3523 	RTL_W8(tp, Config1, RTL_R8(tp, Config1) & ~0x10);
3524 
3525 	r8168_mac_ocp_write(tp, 0xc140, 0xffff);
3526 	r8168_mac_ocp_write(tp, 0xc142, 0xffff);
3527 
3528 	r8168_mac_ocp_modify(tp, 0xd3e2, 0x0fff, 0x03a9);
3529 	r8168_mac_ocp_modify(tp, 0xd3e4, 0x00ff, 0x0000);
3530 	r8168_mac_ocp_modify(tp, 0xe860, 0x0000, 0x0080);
3531 
3532 	/* disable new tx descriptor format */
3533 	r8168_mac_ocp_modify(tp, 0xeb58, 0x0001, 0x0000);
3534 
3535 	if (tp->mac_version == RTL_GIGA_MAC_VER_63)
3536 		r8168_mac_ocp_modify(tp, 0xe614, 0x0700, 0x0200);
3537 	else
3538 		r8168_mac_ocp_modify(tp, 0xe614, 0x0700, 0x0400);
3539 
3540 	if (tp->mac_version == RTL_GIGA_MAC_VER_63)
3541 		r8168_mac_ocp_modify(tp, 0xe63e, 0x0c30, 0x0000);
3542 	else
3543 		r8168_mac_ocp_modify(tp, 0xe63e, 0x0c30, 0x0020);
3544 
3545 	r8168_mac_ocp_modify(tp, 0xc0b4, 0x0000, 0x000c);
3546 	r8168_mac_ocp_modify(tp, 0xeb6a, 0x00ff, 0x0033);
3547 	r8168_mac_ocp_modify(tp, 0xeb50, 0x03e0, 0x0040);
3548 	r8168_mac_ocp_modify(tp, 0xe056, 0x00f0, 0x0030);
3549 	r8168_mac_ocp_modify(tp, 0xe040, 0x1000, 0x0000);
3550 	r8168_mac_ocp_modify(tp, 0xea1c, 0x0003, 0x0001);
3551 	r8168_mac_ocp_modify(tp, 0xe0c0, 0x4f0f, 0x4403);
3552 	r8168_mac_ocp_modify(tp, 0xe052, 0x0080, 0x0068);
3553 	r8168_mac_ocp_modify(tp, 0xc0ac, 0x0080, 0x1f00);
3554 	r8168_mac_ocp_modify(tp, 0xd430, 0x0fff, 0x047f);
3555 
3556 	r8168_mac_ocp_modify(tp, 0xea1c, 0x0004, 0x0000);
3557 	r8168_mac_ocp_modify(tp, 0xeb54, 0x0000, 0x0001);
3558 	udelay(1);
3559 	r8168_mac_ocp_modify(tp, 0xeb54, 0x0001, 0x0000);
3560 	RTL_W16(tp, 0x1880, RTL_R16(tp, 0x1880) & ~0x0030);
3561 
3562 	r8168_mac_ocp_write(tp, 0xe098, 0xc302);
3563 
3564 	rtl_loop_wait_low(tp, &rtl_mac_ocp_e00e_cond, 1000, 10);
3565 
3566 	if (tp->mac_version == RTL_GIGA_MAC_VER_63)
3567 		rtl8125b_config_eee_mac(tp);
3568 	else
3569 		rtl8125a_config_eee_mac(tp);
3570 
3571 	RTL_W32(tp, MISC, RTL_R32(tp, MISC) & ~RXDV_GATED_EN);
3572 	udelay(10);
3573 }
3574 
3575 static void rtl_hw_start_8125a_1(struct rtl8169_private *tp)
3576 {
3577 	static const struct ephy_info e_info_8125a_1[] = {
3578 		{ 0x01, 0xffff, 0xa812 },
3579 		{ 0x09, 0xffff, 0x520c },
3580 		{ 0x04, 0xffff, 0xd000 },
3581 		{ 0x0d, 0xffff, 0xf702 },
3582 		{ 0x0a, 0xffff, 0x8653 },
3583 		{ 0x06, 0xffff, 0x001e },
3584 		{ 0x08, 0xffff, 0x3595 },
3585 		{ 0x20, 0xffff, 0x9455 },
3586 		{ 0x21, 0xffff, 0x99ff },
3587 		{ 0x02, 0xffff, 0x6046 },
3588 		{ 0x29, 0xffff, 0xfe00 },
3589 		{ 0x23, 0xffff, 0xab62 },
3590 
3591 		{ 0x41, 0xffff, 0xa80c },
3592 		{ 0x49, 0xffff, 0x520c },
3593 		{ 0x44, 0xffff, 0xd000 },
3594 		{ 0x4d, 0xffff, 0xf702 },
3595 		{ 0x4a, 0xffff, 0x8653 },
3596 		{ 0x46, 0xffff, 0x001e },
3597 		{ 0x48, 0xffff, 0x3595 },
3598 		{ 0x60, 0xffff, 0x9455 },
3599 		{ 0x61, 0xffff, 0x99ff },
3600 		{ 0x42, 0xffff, 0x6046 },
3601 		{ 0x69, 0xffff, 0xfe00 },
3602 		{ 0x63, 0xffff, 0xab62 },
3603 	};
3604 
3605 	rtl_set_def_aspm_entry_latency(tp);
3606 
3607 	/* disable aspm and clock request before access ephy */
3608 	rtl_hw_aspm_clkreq_enable(tp, false);
3609 	rtl_ephy_init(tp, e_info_8125a_1);
3610 
3611 	rtl_hw_start_8125_common(tp);
3612 	rtl_hw_aspm_clkreq_enable(tp, true);
3613 }
3614 
3615 static void rtl_hw_start_8125a_2(struct rtl8169_private *tp)
3616 {
3617 	static const struct ephy_info e_info_8125a_2[] = {
3618 		{ 0x04, 0xffff, 0xd000 },
3619 		{ 0x0a, 0xffff, 0x8653 },
3620 		{ 0x23, 0xffff, 0xab66 },
3621 		{ 0x20, 0xffff, 0x9455 },
3622 		{ 0x21, 0xffff, 0x99ff },
3623 		{ 0x29, 0xffff, 0xfe04 },
3624 
3625 		{ 0x44, 0xffff, 0xd000 },
3626 		{ 0x4a, 0xffff, 0x8653 },
3627 		{ 0x63, 0xffff, 0xab66 },
3628 		{ 0x60, 0xffff, 0x9455 },
3629 		{ 0x61, 0xffff, 0x99ff },
3630 		{ 0x69, 0xffff, 0xfe04 },
3631 	};
3632 
3633 	rtl_set_def_aspm_entry_latency(tp);
3634 
3635 	/* disable aspm and clock request before access ephy */
3636 	rtl_hw_aspm_clkreq_enable(tp, false);
3637 	rtl_ephy_init(tp, e_info_8125a_2);
3638 
3639 	rtl_hw_start_8125_common(tp);
3640 	rtl_hw_aspm_clkreq_enable(tp, true);
3641 }
3642 
3643 static void rtl_hw_start_8125b(struct rtl8169_private *tp)
3644 {
3645 	static const struct ephy_info e_info_8125b[] = {
3646 		{ 0x0b, 0xffff, 0xa908 },
3647 		{ 0x1e, 0xffff, 0x20eb },
3648 		{ 0x4b, 0xffff, 0xa908 },
3649 		{ 0x5e, 0xffff, 0x20eb },
3650 		{ 0x22, 0x0030, 0x0020 },
3651 		{ 0x62, 0x0030, 0x0020 },
3652 	};
3653 
3654 	rtl_set_def_aspm_entry_latency(tp);
3655 	rtl_hw_aspm_clkreq_enable(tp, false);
3656 
3657 	rtl_ephy_init(tp, e_info_8125b);
3658 	rtl_hw_start_8125_common(tp);
3659 
3660 	rtl_hw_aspm_clkreq_enable(tp, true);
3661 }
3662 
3663 static void rtl_hw_config(struct rtl8169_private *tp)
3664 {
3665 	static const rtl_generic_fct hw_configs[] = {
3666 		[RTL_GIGA_MAC_VER_07] = rtl_hw_start_8102e_1,
3667 		[RTL_GIGA_MAC_VER_08] = rtl_hw_start_8102e_3,
3668 		[RTL_GIGA_MAC_VER_09] = rtl_hw_start_8102e_2,
3669 		[RTL_GIGA_MAC_VER_10] = NULL,
3670 		[RTL_GIGA_MAC_VER_11] = rtl_hw_start_8168b,
3671 		[RTL_GIGA_MAC_VER_12] = rtl_hw_start_8168b,
3672 		[RTL_GIGA_MAC_VER_13] = NULL,
3673 		[RTL_GIGA_MAC_VER_14] = rtl_hw_start_8401,
3674 		[RTL_GIGA_MAC_VER_16] = NULL,
3675 		[RTL_GIGA_MAC_VER_17] = rtl_hw_start_8168b,
3676 		[RTL_GIGA_MAC_VER_18] = rtl_hw_start_8168cp_1,
3677 		[RTL_GIGA_MAC_VER_19] = rtl_hw_start_8168c_1,
3678 		[RTL_GIGA_MAC_VER_20] = rtl_hw_start_8168c_2,
3679 		[RTL_GIGA_MAC_VER_21] = rtl_hw_start_8168c_3,
3680 		[RTL_GIGA_MAC_VER_22] = rtl_hw_start_8168c_4,
3681 		[RTL_GIGA_MAC_VER_23] = rtl_hw_start_8168cp_2,
3682 		[RTL_GIGA_MAC_VER_24] = rtl_hw_start_8168cp_3,
3683 		[RTL_GIGA_MAC_VER_25] = rtl_hw_start_8168d,
3684 		[RTL_GIGA_MAC_VER_26] = rtl_hw_start_8168d,
3685 		[RTL_GIGA_MAC_VER_27] = rtl_hw_start_8168d,
3686 		[RTL_GIGA_MAC_VER_28] = rtl_hw_start_8168d_4,
3687 		[RTL_GIGA_MAC_VER_29] = rtl_hw_start_8105e_1,
3688 		[RTL_GIGA_MAC_VER_30] = rtl_hw_start_8105e_2,
3689 		[RTL_GIGA_MAC_VER_31] = rtl_hw_start_8168d,
3690 		[RTL_GIGA_MAC_VER_32] = rtl_hw_start_8168e_1,
3691 		[RTL_GIGA_MAC_VER_33] = rtl_hw_start_8168e_1,
3692 		[RTL_GIGA_MAC_VER_34] = rtl_hw_start_8168e_2,
3693 		[RTL_GIGA_MAC_VER_35] = rtl_hw_start_8168f_1,
3694 		[RTL_GIGA_MAC_VER_36] = rtl_hw_start_8168f_1,
3695 		[RTL_GIGA_MAC_VER_37] = rtl_hw_start_8402,
3696 		[RTL_GIGA_MAC_VER_38] = rtl_hw_start_8411,
3697 		[RTL_GIGA_MAC_VER_39] = rtl_hw_start_8106,
3698 		[RTL_GIGA_MAC_VER_40] = rtl_hw_start_8168g_1,
3699 		[RTL_GIGA_MAC_VER_41] = rtl_hw_start_8168g_1,
3700 		[RTL_GIGA_MAC_VER_42] = rtl_hw_start_8168g_2,
3701 		[RTL_GIGA_MAC_VER_43] = rtl_hw_start_8168g_2,
3702 		[RTL_GIGA_MAC_VER_44] = rtl_hw_start_8411_2,
3703 		[RTL_GIGA_MAC_VER_45] = rtl_hw_start_8168h_1,
3704 		[RTL_GIGA_MAC_VER_46] = rtl_hw_start_8168h_1,
3705 		[RTL_GIGA_MAC_VER_47] = rtl_hw_start_8168h_1,
3706 		[RTL_GIGA_MAC_VER_48] = rtl_hw_start_8168h_1,
3707 		[RTL_GIGA_MAC_VER_49] = rtl_hw_start_8168ep_1,
3708 		[RTL_GIGA_MAC_VER_50] = rtl_hw_start_8168ep_2,
3709 		[RTL_GIGA_MAC_VER_51] = rtl_hw_start_8168ep_3,
3710 		[RTL_GIGA_MAC_VER_52] = rtl_hw_start_8117,
3711 		[RTL_GIGA_MAC_VER_60] = rtl_hw_start_8125a_1,
3712 		[RTL_GIGA_MAC_VER_61] = rtl_hw_start_8125a_2,
3713 		[RTL_GIGA_MAC_VER_63] = rtl_hw_start_8125b,
3714 	};
3715 
3716 	if (hw_configs[tp->mac_version])
3717 		hw_configs[tp->mac_version](tp);
3718 }
3719 
3720 static void rtl_hw_start_8125(struct rtl8169_private *tp)
3721 {
3722 	int i;
3723 
3724 	/* disable interrupt coalescing */
3725 	for (i = 0xa00; i < 0xb00; i += 4)
3726 		RTL_W32(tp, i, 0);
3727 
3728 	rtl_hw_config(tp);
3729 }
3730 
3731 static void rtl_hw_start_8168(struct rtl8169_private *tp)
3732 {
3733 	if (rtl_is_8168evl_up(tp))
3734 		RTL_W8(tp, MaxTxPacketSize, EarlySize);
3735 	else
3736 		RTL_W8(tp, MaxTxPacketSize, TxPacketMax);
3737 
3738 	rtl_hw_config(tp);
3739 
3740 	/* disable interrupt coalescing */
3741 	RTL_W16(tp, IntrMitigate, 0x0000);
3742 }
3743 
3744 static void rtl_hw_start_8169(struct rtl8169_private *tp)
3745 {
3746 	RTL_W8(tp, EarlyTxThres, NoEarlyTx);
3747 
3748 	tp->cp_cmd |= PCIMulRW;
3749 
3750 	if (tp->mac_version == RTL_GIGA_MAC_VER_02 ||
3751 	    tp->mac_version == RTL_GIGA_MAC_VER_03)
3752 		tp->cp_cmd |= EnAnaPLL;
3753 
3754 	RTL_W16(tp, CPlusCmd, tp->cp_cmd);
3755 
3756 	rtl8169_set_magic_reg(tp);
3757 
3758 	/* disable interrupt coalescing */
3759 	RTL_W16(tp, IntrMitigate, 0x0000);
3760 }
3761 
3762 static void rtl_hw_start(struct  rtl8169_private *tp)
3763 {
3764 	rtl_unlock_config_regs(tp);
3765 
3766 	RTL_W16(tp, CPlusCmd, tp->cp_cmd);
3767 
3768 	if (tp->mac_version <= RTL_GIGA_MAC_VER_06)
3769 		rtl_hw_start_8169(tp);
3770 	else if (rtl_is_8125(tp))
3771 		rtl_hw_start_8125(tp);
3772 	else
3773 		rtl_hw_start_8168(tp);
3774 
3775 	rtl_set_rx_max_size(tp);
3776 	rtl_set_rx_tx_desc_registers(tp);
3777 	rtl_lock_config_regs(tp);
3778 
3779 	rtl_jumbo_config(tp);
3780 
3781 	/* Initially a 10 us delay. Turned it into a PCI commit. - FR */
3782 	rtl_pci_commit(tp);
3783 
3784 	RTL_W8(tp, ChipCmd, CmdTxEnb | CmdRxEnb);
3785 	rtl_init_rxcfg(tp);
3786 	rtl_set_tx_config_registers(tp);
3787 	rtl_set_rx_config_features(tp, tp->dev->features);
3788 	rtl_set_rx_mode(tp->dev);
3789 	rtl_irq_enable(tp);
3790 }
3791 
3792 static int rtl8169_change_mtu(struct net_device *dev, int new_mtu)
3793 {
3794 	struct rtl8169_private *tp = netdev_priv(dev);
3795 
3796 	dev->mtu = new_mtu;
3797 	netdev_update_features(dev);
3798 	rtl_jumbo_config(tp);
3799 
3800 	switch (tp->mac_version) {
3801 	case RTL_GIGA_MAC_VER_61:
3802 	case RTL_GIGA_MAC_VER_63:
3803 		rtl8125_set_eee_txidle_timer(tp);
3804 		break;
3805 	default:
3806 		break;
3807 	}
3808 
3809 	return 0;
3810 }
3811 
3812 static void rtl8169_mark_to_asic(struct RxDesc *desc)
3813 {
3814 	u32 eor = le32_to_cpu(desc->opts1) & RingEnd;
3815 
3816 	desc->opts2 = 0;
3817 	/* Force memory writes to complete before releasing descriptor */
3818 	dma_wmb();
3819 	WRITE_ONCE(desc->opts1, cpu_to_le32(DescOwn | eor | R8169_RX_BUF_SIZE));
3820 }
3821 
3822 static struct page *rtl8169_alloc_rx_data(struct rtl8169_private *tp,
3823 					  struct RxDesc *desc)
3824 {
3825 	struct device *d = tp_to_dev(tp);
3826 	int node = dev_to_node(d);
3827 	dma_addr_t mapping;
3828 	struct page *data;
3829 
3830 	data = alloc_pages_node(node, GFP_KERNEL, get_order(R8169_RX_BUF_SIZE));
3831 	if (!data)
3832 		return NULL;
3833 
3834 	mapping = dma_map_page(d, data, 0, R8169_RX_BUF_SIZE, DMA_FROM_DEVICE);
3835 	if (unlikely(dma_mapping_error(d, mapping))) {
3836 		netdev_err(tp->dev, "Failed to map RX DMA!\n");
3837 		__free_pages(data, get_order(R8169_RX_BUF_SIZE));
3838 		return NULL;
3839 	}
3840 
3841 	desc->addr = cpu_to_le64(mapping);
3842 	rtl8169_mark_to_asic(desc);
3843 
3844 	return data;
3845 }
3846 
3847 static void rtl8169_rx_clear(struct rtl8169_private *tp)
3848 {
3849 	int i;
3850 
3851 	for (i = 0; i < NUM_RX_DESC && tp->Rx_databuff[i]; i++) {
3852 		dma_unmap_page(tp_to_dev(tp),
3853 			       le64_to_cpu(tp->RxDescArray[i].addr),
3854 			       R8169_RX_BUF_SIZE, DMA_FROM_DEVICE);
3855 		__free_pages(tp->Rx_databuff[i], get_order(R8169_RX_BUF_SIZE));
3856 		tp->Rx_databuff[i] = NULL;
3857 		tp->RxDescArray[i].addr = 0;
3858 		tp->RxDescArray[i].opts1 = 0;
3859 	}
3860 }
3861 
3862 static int rtl8169_rx_fill(struct rtl8169_private *tp)
3863 {
3864 	int i;
3865 
3866 	for (i = 0; i < NUM_RX_DESC; i++) {
3867 		struct page *data;
3868 
3869 		data = rtl8169_alloc_rx_data(tp, tp->RxDescArray + i);
3870 		if (!data) {
3871 			rtl8169_rx_clear(tp);
3872 			return -ENOMEM;
3873 		}
3874 		tp->Rx_databuff[i] = data;
3875 	}
3876 
3877 	/* mark as last descriptor in the ring */
3878 	tp->RxDescArray[NUM_RX_DESC - 1].opts1 |= cpu_to_le32(RingEnd);
3879 
3880 	return 0;
3881 }
3882 
3883 static int rtl8169_init_ring(struct rtl8169_private *tp)
3884 {
3885 	rtl8169_init_ring_indexes(tp);
3886 
3887 	memset(tp->tx_skb, 0, sizeof(tp->tx_skb));
3888 	memset(tp->Rx_databuff, 0, sizeof(tp->Rx_databuff));
3889 
3890 	return rtl8169_rx_fill(tp);
3891 }
3892 
3893 static void rtl8169_unmap_tx_skb(struct rtl8169_private *tp, unsigned int entry)
3894 {
3895 	struct ring_info *tx_skb = tp->tx_skb + entry;
3896 	struct TxDesc *desc = tp->TxDescArray + entry;
3897 
3898 	dma_unmap_single(tp_to_dev(tp), le64_to_cpu(desc->addr), tx_skb->len,
3899 			 DMA_TO_DEVICE);
3900 	memset(desc, 0, sizeof(*desc));
3901 	memset(tx_skb, 0, sizeof(*tx_skb));
3902 }
3903 
3904 static void rtl8169_tx_clear_range(struct rtl8169_private *tp, u32 start,
3905 				   unsigned int n)
3906 {
3907 	unsigned int i;
3908 
3909 	for (i = 0; i < n; i++) {
3910 		unsigned int entry = (start + i) % NUM_TX_DESC;
3911 		struct ring_info *tx_skb = tp->tx_skb + entry;
3912 		unsigned int len = tx_skb->len;
3913 
3914 		if (len) {
3915 			struct sk_buff *skb = tx_skb->skb;
3916 
3917 			rtl8169_unmap_tx_skb(tp, entry);
3918 			if (skb)
3919 				dev_consume_skb_any(skb);
3920 		}
3921 	}
3922 }
3923 
3924 static void rtl8169_tx_clear(struct rtl8169_private *tp)
3925 {
3926 	rtl8169_tx_clear_range(tp, tp->dirty_tx, NUM_TX_DESC);
3927 	netdev_reset_queue(tp->dev);
3928 }
3929 
3930 static void rtl8169_cleanup(struct rtl8169_private *tp, bool going_down)
3931 {
3932 	napi_disable(&tp->napi);
3933 
3934 	/* Give a racing hard_start_xmit a few cycles to complete. */
3935 	synchronize_net();
3936 
3937 	/* Disable interrupts */
3938 	rtl8169_irq_mask_and_ack(tp);
3939 
3940 	rtl_rx_close(tp);
3941 
3942 	if (going_down && tp->dev->wol_enabled)
3943 		goto no_reset;
3944 
3945 	switch (tp->mac_version) {
3946 	case RTL_GIGA_MAC_VER_27:
3947 	case RTL_GIGA_MAC_VER_28:
3948 	case RTL_GIGA_MAC_VER_31:
3949 		rtl_loop_wait_low(tp, &rtl_npq_cond, 20, 2000);
3950 		break;
3951 	case RTL_GIGA_MAC_VER_34 ... RTL_GIGA_MAC_VER_38:
3952 		RTL_W8(tp, ChipCmd, RTL_R8(tp, ChipCmd) | StopReq);
3953 		rtl_loop_wait_high(tp, &rtl_txcfg_empty_cond, 100, 666);
3954 		break;
3955 	case RTL_GIGA_MAC_VER_40 ... RTL_GIGA_MAC_VER_63:
3956 		rtl_enable_rxdvgate(tp);
3957 		fsleep(2000);
3958 		break;
3959 	default:
3960 		RTL_W8(tp, ChipCmd, RTL_R8(tp, ChipCmd) | StopReq);
3961 		fsleep(100);
3962 		break;
3963 	}
3964 
3965 	rtl_hw_reset(tp);
3966 no_reset:
3967 	rtl8169_tx_clear(tp);
3968 	rtl8169_init_ring_indexes(tp);
3969 }
3970 
3971 static void rtl_reset_work(struct rtl8169_private *tp)
3972 {
3973 	int i;
3974 
3975 	netif_stop_queue(tp->dev);
3976 
3977 	rtl8169_cleanup(tp, false);
3978 
3979 	for (i = 0; i < NUM_RX_DESC; i++)
3980 		rtl8169_mark_to_asic(tp->RxDescArray + i);
3981 
3982 	napi_enable(&tp->napi);
3983 	rtl_hw_start(tp);
3984 }
3985 
3986 static void rtl8169_tx_timeout(struct net_device *dev, unsigned int txqueue)
3987 {
3988 	struct rtl8169_private *tp = netdev_priv(dev);
3989 
3990 	rtl_schedule_task(tp, RTL_FLAG_TASK_RESET_PENDING);
3991 }
3992 
3993 static int rtl8169_tx_map(struct rtl8169_private *tp, const u32 *opts, u32 len,
3994 			  void *addr, unsigned int entry, bool desc_own)
3995 {
3996 	struct TxDesc *txd = tp->TxDescArray + entry;
3997 	struct device *d = tp_to_dev(tp);
3998 	dma_addr_t mapping;
3999 	u32 opts1;
4000 	int ret;
4001 
4002 	mapping = dma_map_single(d, addr, len, DMA_TO_DEVICE);
4003 	ret = dma_mapping_error(d, mapping);
4004 	if (unlikely(ret)) {
4005 		if (net_ratelimit())
4006 			netdev_err(tp->dev, "Failed to map TX data!\n");
4007 		return ret;
4008 	}
4009 
4010 	txd->addr = cpu_to_le64(mapping);
4011 	txd->opts2 = cpu_to_le32(opts[1]);
4012 
4013 	opts1 = opts[0] | len;
4014 	if (entry == NUM_TX_DESC - 1)
4015 		opts1 |= RingEnd;
4016 	if (desc_own)
4017 		opts1 |= DescOwn;
4018 	txd->opts1 = cpu_to_le32(opts1);
4019 
4020 	tp->tx_skb[entry].len = len;
4021 
4022 	return 0;
4023 }
4024 
4025 static int rtl8169_xmit_frags(struct rtl8169_private *tp, struct sk_buff *skb,
4026 			      const u32 *opts, unsigned int entry)
4027 {
4028 	struct skb_shared_info *info = skb_shinfo(skb);
4029 	unsigned int cur_frag;
4030 
4031 	for (cur_frag = 0; cur_frag < info->nr_frags; cur_frag++) {
4032 		const skb_frag_t *frag = info->frags + cur_frag;
4033 		void *addr = skb_frag_address(frag);
4034 		u32 len = skb_frag_size(frag);
4035 
4036 		entry = (entry + 1) % NUM_TX_DESC;
4037 
4038 		if (unlikely(rtl8169_tx_map(tp, opts, len, addr, entry, true)))
4039 			goto err_out;
4040 	}
4041 
4042 	return 0;
4043 
4044 err_out:
4045 	rtl8169_tx_clear_range(tp, tp->cur_tx + 1, cur_frag);
4046 	return -EIO;
4047 }
4048 
4049 static bool rtl_test_hw_pad_bug(struct rtl8169_private *tp)
4050 {
4051 	switch (tp->mac_version) {
4052 	case RTL_GIGA_MAC_VER_34:
4053 	case RTL_GIGA_MAC_VER_60:
4054 	case RTL_GIGA_MAC_VER_61:
4055 	case RTL_GIGA_MAC_VER_63:
4056 		return true;
4057 	default:
4058 		return false;
4059 	}
4060 }
4061 
4062 static void rtl8169_tso_csum_v1(struct sk_buff *skb, u32 *opts)
4063 {
4064 	u32 mss = skb_shinfo(skb)->gso_size;
4065 
4066 	if (mss) {
4067 		opts[0] |= TD_LSO;
4068 		opts[0] |= mss << TD0_MSS_SHIFT;
4069 	} else if (skb->ip_summed == CHECKSUM_PARTIAL) {
4070 		const struct iphdr *ip = ip_hdr(skb);
4071 
4072 		if (ip->protocol == IPPROTO_TCP)
4073 			opts[0] |= TD0_IP_CS | TD0_TCP_CS;
4074 		else if (ip->protocol == IPPROTO_UDP)
4075 			opts[0] |= TD0_IP_CS | TD0_UDP_CS;
4076 		else
4077 			WARN_ON_ONCE(1);
4078 	}
4079 }
4080 
4081 static bool rtl8169_tso_csum_v2(struct rtl8169_private *tp,
4082 				struct sk_buff *skb, u32 *opts)
4083 {
4084 	u32 transport_offset = (u32)skb_transport_offset(skb);
4085 	struct skb_shared_info *shinfo = skb_shinfo(skb);
4086 	u32 mss = shinfo->gso_size;
4087 
4088 	if (mss) {
4089 		if (shinfo->gso_type & SKB_GSO_TCPV4) {
4090 			opts[0] |= TD1_GTSENV4;
4091 		} else if (shinfo->gso_type & SKB_GSO_TCPV6) {
4092 			if (skb_cow_head(skb, 0))
4093 				return false;
4094 
4095 			tcp_v6_gso_csum_prep(skb);
4096 			opts[0] |= TD1_GTSENV6;
4097 		} else {
4098 			WARN_ON_ONCE(1);
4099 		}
4100 
4101 		opts[0] |= transport_offset << GTTCPHO_SHIFT;
4102 		opts[1] |= mss << TD1_MSS_SHIFT;
4103 	} else if (skb->ip_summed == CHECKSUM_PARTIAL) {
4104 		u8 ip_protocol;
4105 
4106 		switch (vlan_get_protocol(skb)) {
4107 		case htons(ETH_P_IP):
4108 			opts[1] |= TD1_IPv4_CS;
4109 			ip_protocol = ip_hdr(skb)->protocol;
4110 			break;
4111 
4112 		case htons(ETH_P_IPV6):
4113 			opts[1] |= TD1_IPv6_CS;
4114 			ip_protocol = ipv6_hdr(skb)->nexthdr;
4115 			break;
4116 
4117 		default:
4118 			ip_protocol = IPPROTO_RAW;
4119 			break;
4120 		}
4121 
4122 		if (ip_protocol == IPPROTO_TCP)
4123 			opts[1] |= TD1_TCP_CS;
4124 		else if (ip_protocol == IPPROTO_UDP)
4125 			opts[1] |= TD1_UDP_CS;
4126 		else
4127 			WARN_ON_ONCE(1);
4128 
4129 		opts[1] |= transport_offset << TCPHO_SHIFT;
4130 	} else {
4131 		if (unlikely(skb->len < ETH_ZLEN && rtl_test_hw_pad_bug(tp)))
4132 			/* eth_skb_pad would free the skb on error */
4133 			return !__skb_put_padto(skb, ETH_ZLEN, false);
4134 	}
4135 
4136 	return true;
4137 }
4138 
4139 static bool rtl_tx_slots_avail(struct rtl8169_private *tp)
4140 {
4141 	unsigned int slots_avail = READ_ONCE(tp->dirty_tx) + NUM_TX_DESC
4142 					- READ_ONCE(tp->cur_tx);
4143 
4144 	/* A skbuff with nr_frags needs nr_frags+1 entries in the tx queue */
4145 	return slots_avail > MAX_SKB_FRAGS;
4146 }
4147 
4148 /* Versions RTL8102e and from RTL8168c onwards support csum_v2 */
4149 static bool rtl_chip_supports_csum_v2(struct rtl8169_private *tp)
4150 {
4151 	switch (tp->mac_version) {
4152 	case RTL_GIGA_MAC_VER_02 ... RTL_GIGA_MAC_VER_06:
4153 	case RTL_GIGA_MAC_VER_10 ... RTL_GIGA_MAC_VER_17:
4154 		return false;
4155 	default:
4156 		return true;
4157 	}
4158 }
4159 
4160 static void rtl8169_doorbell(struct rtl8169_private *tp)
4161 {
4162 	if (rtl_is_8125(tp))
4163 		RTL_W16(tp, TxPoll_8125, BIT(0));
4164 	else
4165 		RTL_W8(tp, TxPoll, NPQ);
4166 }
4167 
4168 static netdev_tx_t rtl8169_start_xmit(struct sk_buff *skb,
4169 				      struct net_device *dev)
4170 {
4171 	unsigned int frags = skb_shinfo(skb)->nr_frags;
4172 	struct rtl8169_private *tp = netdev_priv(dev);
4173 	unsigned int entry = tp->cur_tx % NUM_TX_DESC;
4174 	struct TxDesc *txd_first, *txd_last;
4175 	bool stop_queue, door_bell;
4176 	u32 opts[2];
4177 
4178 	if (unlikely(!rtl_tx_slots_avail(tp))) {
4179 		if (net_ratelimit())
4180 			netdev_err(dev, "BUG! Tx Ring full when queue awake!\n");
4181 		goto err_stop_0;
4182 	}
4183 
4184 	opts[1] = rtl8169_tx_vlan_tag(skb);
4185 	opts[0] = 0;
4186 
4187 	if (!rtl_chip_supports_csum_v2(tp))
4188 		rtl8169_tso_csum_v1(skb, opts);
4189 	else if (!rtl8169_tso_csum_v2(tp, skb, opts))
4190 		goto err_dma_0;
4191 
4192 	if (unlikely(rtl8169_tx_map(tp, opts, skb_headlen(skb), skb->data,
4193 				    entry, false)))
4194 		goto err_dma_0;
4195 
4196 	txd_first = tp->TxDescArray + entry;
4197 
4198 	if (frags) {
4199 		if (rtl8169_xmit_frags(tp, skb, opts, entry))
4200 			goto err_dma_1;
4201 		entry = (entry + frags) % NUM_TX_DESC;
4202 	}
4203 
4204 	txd_last = tp->TxDescArray + entry;
4205 	txd_last->opts1 |= cpu_to_le32(LastFrag);
4206 	tp->tx_skb[entry].skb = skb;
4207 
4208 	skb_tx_timestamp(skb);
4209 
4210 	/* Force memory writes to complete before releasing descriptor */
4211 	dma_wmb();
4212 
4213 	door_bell = __netdev_sent_queue(dev, skb->len, netdev_xmit_more());
4214 
4215 	txd_first->opts1 |= cpu_to_le32(DescOwn | FirstFrag);
4216 
4217 	/* rtl_tx needs to see descriptor changes before updated tp->cur_tx */
4218 	smp_wmb();
4219 
4220 	WRITE_ONCE(tp->cur_tx, tp->cur_tx + frags + 1);
4221 
4222 	stop_queue = !rtl_tx_slots_avail(tp);
4223 	if (unlikely(stop_queue)) {
4224 		/* Avoid wrongly optimistic queue wake-up: rtl_tx thread must
4225 		 * not miss a ring update when it notices a stopped queue.
4226 		 */
4227 		smp_wmb();
4228 		netif_stop_queue(dev);
4229 		/* Sync with rtl_tx:
4230 		 * - publish queue status and cur_tx ring index (write barrier)
4231 		 * - refresh dirty_tx ring index (read barrier).
4232 		 * May the current thread have a pessimistic view of the ring
4233 		 * status and forget to wake up queue, a racing rtl_tx thread
4234 		 * can't.
4235 		 */
4236 		smp_mb__after_atomic();
4237 		if (rtl_tx_slots_avail(tp))
4238 			netif_start_queue(dev);
4239 		door_bell = true;
4240 	}
4241 
4242 	if (door_bell)
4243 		rtl8169_doorbell(tp);
4244 
4245 	return NETDEV_TX_OK;
4246 
4247 err_dma_1:
4248 	rtl8169_unmap_tx_skb(tp, entry);
4249 err_dma_0:
4250 	dev_kfree_skb_any(skb);
4251 	dev->stats.tx_dropped++;
4252 	return NETDEV_TX_OK;
4253 
4254 err_stop_0:
4255 	netif_stop_queue(dev);
4256 	dev->stats.tx_dropped++;
4257 	return NETDEV_TX_BUSY;
4258 }
4259 
4260 static unsigned int rtl_last_frag_len(struct sk_buff *skb)
4261 {
4262 	struct skb_shared_info *info = skb_shinfo(skb);
4263 	unsigned int nr_frags = info->nr_frags;
4264 
4265 	if (!nr_frags)
4266 		return UINT_MAX;
4267 
4268 	return skb_frag_size(info->frags + nr_frags - 1);
4269 }
4270 
4271 /* Workaround for hw issues with TSO on RTL8168evl */
4272 static netdev_features_t rtl8168evl_fix_tso(struct sk_buff *skb,
4273 					    netdev_features_t features)
4274 {
4275 	/* IPv4 header has options field */
4276 	if (vlan_get_protocol(skb) == htons(ETH_P_IP) &&
4277 	    ip_hdrlen(skb) > sizeof(struct iphdr))
4278 		features &= ~NETIF_F_ALL_TSO;
4279 
4280 	/* IPv4 TCP header has options field */
4281 	else if (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV4 &&
4282 		 tcp_hdrlen(skb) > sizeof(struct tcphdr))
4283 		features &= ~NETIF_F_ALL_TSO;
4284 
4285 	else if (rtl_last_frag_len(skb) <= 6)
4286 		features &= ~NETIF_F_ALL_TSO;
4287 
4288 	return features;
4289 }
4290 
4291 static netdev_features_t rtl8169_features_check(struct sk_buff *skb,
4292 						struct net_device *dev,
4293 						netdev_features_t features)
4294 {
4295 	int transport_offset = skb_transport_offset(skb);
4296 	struct rtl8169_private *tp = netdev_priv(dev);
4297 
4298 	if (skb_is_gso(skb)) {
4299 		if (tp->mac_version == RTL_GIGA_MAC_VER_34)
4300 			features = rtl8168evl_fix_tso(skb, features);
4301 
4302 		if (transport_offset > GTTCPHO_MAX &&
4303 		    rtl_chip_supports_csum_v2(tp))
4304 			features &= ~NETIF_F_ALL_TSO;
4305 	} else if (skb->ip_summed == CHECKSUM_PARTIAL) {
4306 		/* work around hw bug on some chip versions */
4307 		if (skb->len < ETH_ZLEN)
4308 			features &= ~NETIF_F_CSUM_MASK;
4309 
4310 		if (transport_offset > TCPHO_MAX &&
4311 		    rtl_chip_supports_csum_v2(tp))
4312 			features &= ~NETIF_F_CSUM_MASK;
4313 	}
4314 
4315 	return vlan_features_check(skb, features);
4316 }
4317 
4318 static void rtl8169_pcierr_interrupt(struct net_device *dev)
4319 {
4320 	struct rtl8169_private *tp = netdev_priv(dev);
4321 	struct pci_dev *pdev = tp->pci_dev;
4322 	int pci_status_errs;
4323 	u16 pci_cmd;
4324 
4325 	pci_read_config_word(pdev, PCI_COMMAND, &pci_cmd);
4326 
4327 	pci_status_errs = pci_status_get_and_clear_errors(pdev);
4328 
4329 	if (net_ratelimit())
4330 		netdev_err(dev, "PCI error (cmd = 0x%04x, status_errs = 0x%04x)\n",
4331 			   pci_cmd, pci_status_errs);
4332 	/*
4333 	 * The recovery sequence below admits a very elaborated explanation:
4334 	 * - it seems to work;
4335 	 * - I did not see what else could be done;
4336 	 * - it makes iop3xx happy.
4337 	 *
4338 	 * Feel free to adjust to your needs.
4339 	 */
4340 	if (pdev->broken_parity_status)
4341 		pci_cmd &= ~PCI_COMMAND_PARITY;
4342 	else
4343 		pci_cmd |= PCI_COMMAND_SERR | PCI_COMMAND_PARITY;
4344 
4345 	pci_write_config_word(pdev, PCI_COMMAND, pci_cmd);
4346 
4347 	rtl_schedule_task(tp, RTL_FLAG_TASK_RESET_PENDING);
4348 }
4349 
4350 static void rtl_tx(struct net_device *dev, struct rtl8169_private *tp,
4351 		   int budget)
4352 {
4353 	unsigned int dirty_tx, bytes_compl = 0, pkts_compl = 0;
4354 	struct sk_buff *skb;
4355 
4356 	dirty_tx = tp->dirty_tx;
4357 
4358 	while (READ_ONCE(tp->cur_tx) != dirty_tx) {
4359 		unsigned int entry = dirty_tx % NUM_TX_DESC;
4360 		u32 status;
4361 
4362 		status = le32_to_cpu(tp->TxDescArray[entry].opts1);
4363 		if (status & DescOwn)
4364 			break;
4365 
4366 		skb = tp->tx_skb[entry].skb;
4367 		rtl8169_unmap_tx_skb(tp, entry);
4368 
4369 		if (skb) {
4370 			pkts_compl++;
4371 			bytes_compl += skb->len;
4372 			napi_consume_skb(skb, budget);
4373 		}
4374 		dirty_tx++;
4375 	}
4376 
4377 	if (tp->dirty_tx != dirty_tx) {
4378 		netdev_completed_queue(dev, pkts_compl, bytes_compl);
4379 		dev_sw_netstats_tx_add(dev, pkts_compl, bytes_compl);
4380 
4381 		/* Sync with rtl8169_start_xmit:
4382 		 * - publish dirty_tx ring index (write barrier)
4383 		 * - refresh cur_tx ring index and queue status (read barrier)
4384 		 * May the current thread miss the stopped queue condition,
4385 		 * a racing xmit thread can only have a right view of the
4386 		 * ring status.
4387 		 */
4388 		smp_store_mb(tp->dirty_tx, dirty_tx);
4389 		if (netif_queue_stopped(dev) && rtl_tx_slots_avail(tp))
4390 			netif_wake_queue(dev);
4391 		/*
4392 		 * 8168 hack: TxPoll requests are lost when the Tx packets are
4393 		 * too close. Let's kick an extra TxPoll request when a burst
4394 		 * of start_xmit activity is detected (if it is not detected,
4395 		 * it is slow enough). -- FR
4396 		 * If skb is NULL then we come here again once a tx irq is
4397 		 * triggered after the last fragment is marked transmitted.
4398 		 */
4399 		if (tp->cur_tx != dirty_tx && skb)
4400 			rtl8169_doorbell(tp);
4401 	}
4402 }
4403 
4404 static inline int rtl8169_fragmented_frame(u32 status)
4405 {
4406 	return (status & (FirstFrag | LastFrag)) != (FirstFrag | LastFrag);
4407 }
4408 
4409 static inline void rtl8169_rx_csum(struct sk_buff *skb, u32 opts1)
4410 {
4411 	u32 status = opts1 & RxProtoMask;
4412 
4413 	if (((status == RxProtoTCP) && !(opts1 & TCPFail)) ||
4414 	    ((status == RxProtoUDP) && !(opts1 & UDPFail)))
4415 		skb->ip_summed = CHECKSUM_UNNECESSARY;
4416 	else
4417 		skb_checksum_none_assert(skb);
4418 }
4419 
4420 static int rtl_rx(struct net_device *dev, struct rtl8169_private *tp, int budget)
4421 {
4422 	struct device *d = tp_to_dev(tp);
4423 	int count;
4424 
4425 	for (count = 0; count < budget; count++, tp->cur_rx++) {
4426 		unsigned int pkt_size, entry = tp->cur_rx % NUM_RX_DESC;
4427 		struct RxDesc *desc = tp->RxDescArray + entry;
4428 		struct sk_buff *skb;
4429 		const void *rx_buf;
4430 		dma_addr_t addr;
4431 		u32 status;
4432 
4433 		status = le32_to_cpu(desc->opts1);
4434 		if (status & DescOwn)
4435 			break;
4436 
4437 		/* This barrier is needed to keep us from reading
4438 		 * any other fields out of the Rx descriptor until
4439 		 * we know the status of DescOwn
4440 		 */
4441 		dma_rmb();
4442 
4443 		if (unlikely(status & RxRES)) {
4444 			if (net_ratelimit())
4445 				netdev_warn(dev, "Rx ERROR. status = %08x\n",
4446 					    status);
4447 			dev->stats.rx_errors++;
4448 			if (status & (RxRWT | RxRUNT))
4449 				dev->stats.rx_length_errors++;
4450 			if (status & RxCRC)
4451 				dev->stats.rx_crc_errors++;
4452 
4453 			if (!(dev->features & NETIF_F_RXALL))
4454 				goto release_descriptor;
4455 			else if (status & RxRWT || !(status & (RxRUNT | RxCRC)))
4456 				goto release_descriptor;
4457 		}
4458 
4459 		pkt_size = status & GENMASK(13, 0);
4460 		if (likely(!(dev->features & NETIF_F_RXFCS)))
4461 			pkt_size -= ETH_FCS_LEN;
4462 
4463 		/* The driver does not support incoming fragmented frames.
4464 		 * They are seen as a symptom of over-mtu sized frames.
4465 		 */
4466 		if (unlikely(rtl8169_fragmented_frame(status))) {
4467 			dev->stats.rx_dropped++;
4468 			dev->stats.rx_length_errors++;
4469 			goto release_descriptor;
4470 		}
4471 
4472 		skb = napi_alloc_skb(&tp->napi, pkt_size);
4473 		if (unlikely(!skb)) {
4474 			dev->stats.rx_dropped++;
4475 			goto release_descriptor;
4476 		}
4477 
4478 		addr = le64_to_cpu(desc->addr);
4479 		rx_buf = page_address(tp->Rx_databuff[entry]);
4480 
4481 		dma_sync_single_for_cpu(d, addr, pkt_size, DMA_FROM_DEVICE);
4482 		prefetch(rx_buf);
4483 		skb_copy_to_linear_data(skb, rx_buf, pkt_size);
4484 		skb->tail += pkt_size;
4485 		skb->len = pkt_size;
4486 		dma_sync_single_for_device(d, addr, pkt_size, DMA_FROM_DEVICE);
4487 
4488 		rtl8169_rx_csum(skb, status);
4489 		skb->protocol = eth_type_trans(skb, dev);
4490 
4491 		rtl8169_rx_vlan_tag(desc, skb);
4492 
4493 		if (skb->pkt_type == PACKET_MULTICAST)
4494 			dev->stats.multicast++;
4495 
4496 		napi_gro_receive(&tp->napi, skb);
4497 
4498 		dev_sw_netstats_rx_add(dev, pkt_size);
4499 release_descriptor:
4500 		rtl8169_mark_to_asic(desc);
4501 	}
4502 
4503 	return count;
4504 }
4505 
4506 static irqreturn_t rtl8169_interrupt(int irq, void *dev_instance)
4507 {
4508 	struct rtl8169_private *tp = dev_instance;
4509 	u32 status = rtl_get_events(tp);
4510 
4511 	if ((status & 0xffff) == 0xffff || !(status & tp->irq_mask))
4512 		return IRQ_NONE;
4513 
4514 	if (unlikely(status & SYSErr)) {
4515 		rtl8169_pcierr_interrupt(tp->dev);
4516 		goto out;
4517 	}
4518 
4519 	if (status & LinkChg)
4520 		phy_mac_interrupt(tp->phydev);
4521 
4522 	if (unlikely(status & RxFIFOOver &&
4523 	    tp->mac_version == RTL_GIGA_MAC_VER_11)) {
4524 		netif_stop_queue(tp->dev);
4525 		rtl_schedule_task(tp, RTL_FLAG_TASK_RESET_PENDING);
4526 	}
4527 
4528 	rtl_irq_disable(tp);
4529 	napi_schedule(&tp->napi);
4530 out:
4531 	rtl_ack_events(tp, status);
4532 
4533 	return IRQ_HANDLED;
4534 }
4535 
4536 static void rtl_task(struct work_struct *work)
4537 {
4538 	struct rtl8169_private *tp =
4539 		container_of(work, struct rtl8169_private, wk.work);
4540 
4541 	rtnl_lock();
4542 
4543 	if (!netif_running(tp->dev) ||
4544 	    !test_bit(RTL_FLAG_TASK_ENABLED, tp->wk.flags))
4545 		goto out_unlock;
4546 
4547 	if (test_and_clear_bit(RTL_FLAG_TASK_RESET_PENDING, tp->wk.flags)) {
4548 		rtl_reset_work(tp);
4549 		netif_wake_queue(tp->dev);
4550 	}
4551 out_unlock:
4552 	rtnl_unlock();
4553 }
4554 
4555 static int rtl8169_poll(struct napi_struct *napi, int budget)
4556 {
4557 	struct rtl8169_private *tp = container_of(napi, struct rtl8169_private, napi);
4558 	struct net_device *dev = tp->dev;
4559 	int work_done;
4560 
4561 	work_done = rtl_rx(dev, tp, budget);
4562 
4563 	rtl_tx(dev, tp, budget);
4564 
4565 	if (work_done < budget && napi_complete_done(napi, work_done))
4566 		rtl_irq_enable(tp);
4567 
4568 	return work_done;
4569 }
4570 
4571 static void r8169_phylink_handler(struct net_device *ndev)
4572 {
4573 	struct rtl8169_private *tp = netdev_priv(ndev);
4574 
4575 	if (netif_carrier_ok(ndev)) {
4576 		rtl_link_chg_patch(tp);
4577 		pm_request_resume(&tp->pci_dev->dev);
4578 	} else {
4579 		pm_runtime_idle(&tp->pci_dev->dev);
4580 	}
4581 
4582 	if (net_ratelimit())
4583 		phy_print_status(tp->phydev);
4584 }
4585 
4586 static int r8169_phy_connect(struct rtl8169_private *tp)
4587 {
4588 	struct phy_device *phydev = tp->phydev;
4589 	phy_interface_t phy_mode;
4590 	int ret;
4591 
4592 	phy_mode = tp->supports_gmii ? PHY_INTERFACE_MODE_GMII :
4593 		   PHY_INTERFACE_MODE_MII;
4594 
4595 	ret = phy_connect_direct(tp->dev, phydev, r8169_phylink_handler,
4596 				 phy_mode);
4597 	if (ret)
4598 		return ret;
4599 
4600 	if (!tp->supports_gmii)
4601 		phy_set_max_speed(phydev, SPEED_100);
4602 
4603 	phy_support_asym_pause(phydev);
4604 
4605 	phy_attached_info(phydev);
4606 
4607 	return 0;
4608 }
4609 
4610 static void rtl8169_down(struct rtl8169_private *tp)
4611 {
4612 	/* Clear all task flags */
4613 	bitmap_zero(tp->wk.flags, RTL_FLAG_MAX);
4614 
4615 	phy_stop(tp->phydev);
4616 
4617 	rtl8169_update_counters(tp);
4618 
4619 	rtl8169_cleanup(tp, true);
4620 
4621 	rtl_pll_power_down(tp);
4622 }
4623 
4624 static void rtl8169_up(struct rtl8169_private *tp)
4625 {
4626 	rtl_pll_power_up(tp);
4627 	rtl8169_init_phy(tp);
4628 	napi_enable(&tp->napi);
4629 	set_bit(RTL_FLAG_TASK_ENABLED, tp->wk.flags);
4630 	rtl_reset_work(tp);
4631 
4632 	phy_start(tp->phydev);
4633 }
4634 
4635 static int rtl8169_close(struct net_device *dev)
4636 {
4637 	struct rtl8169_private *tp = netdev_priv(dev);
4638 	struct pci_dev *pdev = tp->pci_dev;
4639 
4640 	pm_runtime_get_sync(&pdev->dev);
4641 
4642 	netif_stop_queue(dev);
4643 	rtl8169_down(tp);
4644 	rtl8169_rx_clear(tp);
4645 
4646 	cancel_work_sync(&tp->wk.work);
4647 
4648 	phy_disconnect(tp->phydev);
4649 
4650 	free_irq(pci_irq_vector(pdev, 0), tp);
4651 
4652 	dma_free_coherent(&pdev->dev, R8169_RX_RING_BYTES, tp->RxDescArray,
4653 			  tp->RxPhyAddr);
4654 	dma_free_coherent(&pdev->dev, R8169_TX_RING_BYTES, tp->TxDescArray,
4655 			  tp->TxPhyAddr);
4656 	tp->TxDescArray = NULL;
4657 	tp->RxDescArray = NULL;
4658 
4659 	pm_runtime_put_sync(&pdev->dev);
4660 
4661 	return 0;
4662 }
4663 
4664 #ifdef CONFIG_NET_POLL_CONTROLLER
4665 static void rtl8169_netpoll(struct net_device *dev)
4666 {
4667 	struct rtl8169_private *tp = netdev_priv(dev);
4668 
4669 	rtl8169_interrupt(pci_irq_vector(tp->pci_dev, 0), tp);
4670 }
4671 #endif
4672 
4673 static int rtl_open(struct net_device *dev)
4674 {
4675 	struct rtl8169_private *tp = netdev_priv(dev);
4676 	struct pci_dev *pdev = tp->pci_dev;
4677 	unsigned long irqflags;
4678 	int retval = -ENOMEM;
4679 
4680 	pm_runtime_get_sync(&pdev->dev);
4681 
4682 	/*
4683 	 * Rx and Tx descriptors needs 256 bytes alignment.
4684 	 * dma_alloc_coherent provides more.
4685 	 */
4686 	tp->TxDescArray = dma_alloc_coherent(&pdev->dev, R8169_TX_RING_BYTES,
4687 					     &tp->TxPhyAddr, GFP_KERNEL);
4688 	if (!tp->TxDescArray)
4689 		goto out;
4690 
4691 	tp->RxDescArray = dma_alloc_coherent(&pdev->dev, R8169_RX_RING_BYTES,
4692 					     &tp->RxPhyAddr, GFP_KERNEL);
4693 	if (!tp->RxDescArray)
4694 		goto err_free_tx_0;
4695 
4696 	retval = rtl8169_init_ring(tp);
4697 	if (retval < 0)
4698 		goto err_free_rx_1;
4699 
4700 	rtl_request_firmware(tp);
4701 
4702 	irqflags = pci_dev_msi_enabled(pdev) ? IRQF_NO_THREAD : IRQF_SHARED;
4703 	retval = request_irq(pci_irq_vector(pdev, 0), rtl8169_interrupt,
4704 			     irqflags, dev->name, tp);
4705 	if (retval < 0)
4706 		goto err_release_fw_2;
4707 
4708 	retval = r8169_phy_connect(tp);
4709 	if (retval)
4710 		goto err_free_irq;
4711 
4712 	rtl8169_up(tp);
4713 	rtl8169_init_counter_offsets(tp);
4714 	netif_start_queue(dev);
4715 out:
4716 	pm_runtime_put_sync(&pdev->dev);
4717 
4718 	return retval;
4719 
4720 err_free_irq:
4721 	free_irq(pci_irq_vector(pdev, 0), tp);
4722 err_release_fw_2:
4723 	rtl_release_firmware(tp);
4724 	rtl8169_rx_clear(tp);
4725 err_free_rx_1:
4726 	dma_free_coherent(&pdev->dev, R8169_RX_RING_BYTES, tp->RxDescArray,
4727 			  tp->RxPhyAddr);
4728 	tp->RxDescArray = NULL;
4729 err_free_tx_0:
4730 	dma_free_coherent(&pdev->dev, R8169_TX_RING_BYTES, tp->TxDescArray,
4731 			  tp->TxPhyAddr);
4732 	tp->TxDescArray = NULL;
4733 	goto out;
4734 }
4735 
4736 static void
4737 rtl8169_get_stats64(struct net_device *dev, struct rtnl_link_stats64 *stats)
4738 {
4739 	struct rtl8169_private *tp = netdev_priv(dev);
4740 	struct pci_dev *pdev = tp->pci_dev;
4741 	struct rtl8169_counters *counters = tp->counters;
4742 
4743 	pm_runtime_get_noresume(&pdev->dev);
4744 
4745 	netdev_stats_to_stats64(stats, &dev->stats);
4746 	dev_fetch_sw_netstats(stats, dev->tstats);
4747 
4748 	/*
4749 	 * Fetch additional counter values missing in stats collected by driver
4750 	 * from tally counters.
4751 	 */
4752 	if (pm_runtime_active(&pdev->dev))
4753 		rtl8169_update_counters(tp);
4754 
4755 	/*
4756 	 * Subtract values fetched during initalization.
4757 	 * See rtl8169_init_counter_offsets for a description why we do that.
4758 	 */
4759 	stats->tx_errors = le64_to_cpu(counters->tx_errors) -
4760 		le64_to_cpu(tp->tc_offset.tx_errors);
4761 	stats->collisions = le32_to_cpu(counters->tx_multi_collision) -
4762 		le32_to_cpu(tp->tc_offset.tx_multi_collision);
4763 	stats->tx_aborted_errors = le16_to_cpu(counters->tx_aborted) -
4764 		le16_to_cpu(tp->tc_offset.tx_aborted);
4765 	stats->rx_missed_errors = le16_to_cpu(counters->rx_missed) -
4766 		le16_to_cpu(tp->tc_offset.rx_missed);
4767 
4768 	pm_runtime_put_noidle(&pdev->dev);
4769 }
4770 
4771 static void rtl8169_net_suspend(struct rtl8169_private *tp)
4772 {
4773 	netif_device_detach(tp->dev);
4774 
4775 	if (netif_running(tp->dev))
4776 		rtl8169_down(tp);
4777 }
4778 
4779 #ifdef CONFIG_PM
4780 
4781 static int rtl8169_net_resume(struct rtl8169_private *tp)
4782 {
4783 	rtl_rar_set(tp, tp->dev->dev_addr);
4784 
4785 	if (tp->TxDescArray)
4786 		rtl8169_up(tp);
4787 
4788 	netif_device_attach(tp->dev);
4789 
4790 	return 0;
4791 }
4792 
4793 static int __maybe_unused rtl8169_suspend(struct device *device)
4794 {
4795 	struct rtl8169_private *tp = dev_get_drvdata(device);
4796 
4797 	rtnl_lock();
4798 	rtl8169_net_suspend(tp);
4799 	if (!device_may_wakeup(tp_to_dev(tp)))
4800 		clk_disable_unprepare(tp->clk);
4801 	rtnl_unlock();
4802 
4803 	return 0;
4804 }
4805 
4806 static int __maybe_unused rtl8169_resume(struct device *device)
4807 {
4808 	struct rtl8169_private *tp = dev_get_drvdata(device);
4809 
4810 	if (!device_may_wakeup(tp_to_dev(tp)))
4811 		clk_prepare_enable(tp->clk);
4812 
4813 	/* Reportedly at least Asus X453MA truncates packets otherwise */
4814 	if (tp->mac_version == RTL_GIGA_MAC_VER_37)
4815 		rtl_init_rxcfg(tp);
4816 
4817 	return rtl8169_net_resume(tp);
4818 }
4819 
4820 static int rtl8169_runtime_suspend(struct device *device)
4821 {
4822 	struct rtl8169_private *tp = dev_get_drvdata(device);
4823 
4824 	if (!tp->TxDescArray) {
4825 		netif_device_detach(tp->dev);
4826 		return 0;
4827 	}
4828 
4829 	rtnl_lock();
4830 	__rtl8169_set_wol(tp, WAKE_PHY);
4831 	rtl8169_net_suspend(tp);
4832 	rtnl_unlock();
4833 
4834 	return 0;
4835 }
4836 
4837 static int rtl8169_runtime_resume(struct device *device)
4838 {
4839 	struct rtl8169_private *tp = dev_get_drvdata(device);
4840 
4841 	__rtl8169_set_wol(tp, tp->saved_wolopts);
4842 
4843 	return rtl8169_net_resume(tp);
4844 }
4845 
4846 static int rtl8169_runtime_idle(struct device *device)
4847 {
4848 	struct rtl8169_private *tp = dev_get_drvdata(device);
4849 
4850 	if (!netif_running(tp->dev) || !netif_carrier_ok(tp->dev))
4851 		pm_schedule_suspend(device, 10000);
4852 
4853 	return -EBUSY;
4854 }
4855 
4856 static const struct dev_pm_ops rtl8169_pm_ops = {
4857 	SET_SYSTEM_SLEEP_PM_OPS(rtl8169_suspend, rtl8169_resume)
4858 	SET_RUNTIME_PM_OPS(rtl8169_runtime_suspend, rtl8169_runtime_resume,
4859 			   rtl8169_runtime_idle)
4860 };
4861 
4862 #endif /* CONFIG_PM */
4863 
4864 static void rtl_wol_shutdown_quirk(struct rtl8169_private *tp)
4865 {
4866 	/* WoL fails with 8168b when the receiver is disabled. */
4867 	switch (tp->mac_version) {
4868 	case RTL_GIGA_MAC_VER_11:
4869 	case RTL_GIGA_MAC_VER_12:
4870 	case RTL_GIGA_MAC_VER_17:
4871 		pci_clear_master(tp->pci_dev);
4872 
4873 		RTL_W8(tp, ChipCmd, CmdRxEnb);
4874 		rtl_pci_commit(tp);
4875 		break;
4876 	default:
4877 		break;
4878 	}
4879 }
4880 
4881 static void rtl_shutdown(struct pci_dev *pdev)
4882 {
4883 	struct rtl8169_private *tp = pci_get_drvdata(pdev);
4884 
4885 	rtnl_lock();
4886 	rtl8169_net_suspend(tp);
4887 	rtnl_unlock();
4888 
4889 	/* Restore original MAC address */
4890 	rtl_rar_set(tp, tp->dev->perm_addr);
4891 
4892 	if (system_state == SYSTEM_POWER_OFF) {
4893 		if (tp->saved_wolopts) {
4894 			rtl_wol_suspend_quirk(tp);
4895 			rtl_wol_shutdown_quirk(tp);
4896 		}
4897 
4898 		pci_wake_from_d3(pdev, true);
4899 		pci_set_power_state(pdev, PCI_D3hot);
4900 	}
4901 }
4902 
4903 static void rtl_remove_one(struct pci_dev *pdev)
4904 {
4905 	struct rtl8169_private *tp = pci_get_drvdata(pdev);
4906 
4907 	if (pci_dev_run_wake(pdev))
4908 		pm_runtime_get_noresume(&pdev->dev);
4909 
4910 	unregister_netdev(tp->dev);
4911 
4912 	if (r8168_check_dash(tp))
4913 		rtl8168_driver_stop(tp);
4914 
4915 	rtl_release_firmware(tp);
4916 
4917 	/* restore original MAC address */
4918 	rtl_rar_set(tp, tp->dev->perm_addr);
4919 }
4920 
4921 static const struct net_device_ops rtl_netdev_ops = {
4922 	.ndo_open		= rtl_open,
4923 	.ndo_stop		= rtl8169_close,
4924 	.ndo_get_stats64	= rtl8169_get_stats64,
4925 	.ndo_start_xmit		= rtl8169_start_xmit,
4926 	.ndo_features_check	= rtl8169_features_check,
4927 	.ndo_tx_timeout		= rtl8169_tx_timeout,
4928 	.ndo_validate_addr	= eth_validate_addr,
4929 	.ndo_change_mtu		= rtl8169_change_mtu,
4930 	.ndo_fix_features	= rtl8169_fix_features,
4931 	.ndo_set_features	= rtl8169_set_features,
4932 	.ndo_set_mac_address	= rtl_set_mac_address,
4933 	.ndo_do_ioctl		= phy_do_ioctl_running,
4934 	.ndo_set_rx_mode	= rtl_set_rx_mode,
4935 #ifdef CONFIG_NET_POLL_CONTROLLER
4936 	.ndo_poll_controller	= rtl8169_netpoll,
4937 #endif
4938 
4939 };
4940 
4941 static void rtl_set_irq_mask(struct rtl8169_private *tp)
4942 {
4943 	tp->irq_mask = RxOK | RxErr | TxOK | TxErr | LinkChg;
4944 
4945 	if (tp->mac_version <= RTL_GIGA_MAC_VER_06)
4946 		tp->irq_mask |= SYSErr | RxOverflow | RxFIFOOver;
4947 	else if (tp->mac_version == RTL_GIGA_MAC_VER_11)
4948 		/* special workaround needed */
4949 		tp->irq_mask |= RxFIFOOver;
4950 	else
4951 		tp->irq_mask |= RxOverflow;
4952 }
4953 
4954 static int rtl_alloc_irq(struct rtl8169_private *tp)
4955 {
4956 	unsigned int flags;
4957 
4958 	switch (tp->mac_version) {
4959 	case RTL_GIGA_MAC_VER_02 ... RTL_GIGA_MAC_VER_06:
4960 		rtl_unlock_config_regs(tp);
4961 		RTL_W8(tp, Config2, RTL_R8(tp, Config2) & ~MSIEnable);
4962 		rtl_lock_config_regs(tp);
4963 		fallthrough;
4964 	case RTL_GIGA_MAC_VER_07 ... RTL_GIGA_MAC_VER_17:
4965 		flags = PCI_IRQ_LEGACY;
4966 		break;
4967 	default:
4968 		flags = PCI_IRQ_ALL_TYPES;
4969 		break;
4970 	}
4971 
4972 	return pci_alloc_irq_vectors(tp->pci_dev, 1, 1, flags);
4973 }
4974 
4975 static void rtl_read_mac_address(struct rtl8169_private *tp,
4976 				 u8 mac_addr[ETH_ALEN])
4977 {
4978 	/* Get MAC address */
4979 	if (rtl_is_8168evl_up(tp) && tp->mac_version != RTL_GIGA_MAC_VER_34) {
4980 		u32 value = rtl_eri_read(tp, 0xe0);
4981 
4982 		mac_addr[0] = (value >>  0) & 0xff;
4983 		mac_addr[1] = (value >>  8) & 0xff;
4984 		mac_addr[2] = (value >> 16) & 0xff;
4985 		mac_addr[3] = (value >> 24) & 0xff;
4986 
4987 		value = rtl_eri_read(tp, 0xe4);
4988 		mac_addr[4] = (value >>  0) & 0xff;
4989 		mac_addr[5] = (value >>  8) & 0xff;
4990 	} else if (rtl_is_8125(tp)) {
4991 		rtl_read_mac_from_reg(tp, mac_addr, MAC0_BKP);
4992 	}
4993 }
4994 
4995 DECLARE_RTL_COND(rtl_link_list_ready_cond)
4996 {
4997 	return RTL_R8(tp, MCU) & LINK_LIST_RDY;
4998 }
4999 
5000 static void r8168g_wait_ll_share_fifo_ready(struct rtl8169_private *tp)
5001 {
5002 	rtl_loop_wait_high(tp, &rtl_link_list_ready_cond, 100, 42);
5003 }
5004 
5005 static int r8169_mdio_read_reg(struct mii_bus *mii_bus, int phyaddr, int phyreg)
5006 {
5007 	struct rtl8169_private *tp = mii_bus->priv;
5008 
5009 	if (phyaddr > 0)
5010 		return -ENODEV;
5011 
5012 	return rtl_readphy(tp, phyreg);
5013 }
5014 
5015 static int r8169_mdio_write_reg(struct mii_bus *mii_bus, int phyaddr,
5016 				int phyreg, u16 val)
5017 {
5018 	struct rtl8169_private *tp = mii_bus->priv;
5019 
5020 	if (phyaddr > 0)
5021 		return -ENODEV;
5022 
5023 	rtl_writephy(tp, phyreg, val);
5024 
5025 	return 0;
5026 }
5027 
5028 static int r8169_mdio_register(struct rtl8169_private *tp)
5029 {
5030 	struct pci_dev *pdev = tp->pci_dev;
5031 	struct mii_bus *new_bus;
5032 	int ret;
5033 
5034 	new_bus = devm_mdiobus_alloc(&pdev->dev);
5035 	if (!new_bus)
5036 		return -ENOMEM;
5037 
5038 	new_bus->name = "r8169";
5039 	new_bus->priv = tp;
5040 	new_bus->parent = &pdev->dev;
5041 	new_bus->irq[0] = PHY_IGNORE_INTERRUPT;
5042 	snprintf(new_bus->id, MII_BUS_ID_SIZE, "r8169-%x", pci_dev_id(pdev));
5043 
5044 	new_bus->read = r8169_mdio_read_reg;
5045 	new_bus->write = r8169_mdio_write_reg;
5046 
5047 	ret = devm_mdiobus_register(&pdev->dev, new_bus);
5048 	if (ret)
5049 		return ret;
5050 
5051 	tp->phydev = mdiobus_get_phy(new_bus, 0);
5052 	if (!tp->phydev) {
5053 		return -ENODEV;
5054 	} else if (!tp->phydev->drv) {
5055 		/* Most chip versions fail with the genphy driver.
5056 		 * Therefore ensure that the dedicated PHY driver is loaded.
5057 		 */
5058 		dev_err(&pdev->dev, "no dedicated PHY driver found for PHY ID 0x%08x, maybe realtek.ko needs to be added to initramfs?\n",
5059 			tp->phydev->phy_id);
5060 		return -EUNATCH;
5061 	}
5062 
5063 	/* PHY will be woken up in rtl_open() */
5064 	phy_suspend(tp->phydev);
5065 
5066 	return 0;
5067 }
5068 
5069 static void rtl_hw_init_8168g(struct rtl8169_private *tp)
5070 {
5071 	rtl_enable_rxdvgate(tp);
5072 
5073 	RTL_W8(tp, ChipCmd, RTL_R8(tp, ChipCmd) & ~(CmdTxEnb | CmdRxEnb));
5074 	msleep(1);
5075 	RTL_W8(tp, MCU, RTL_R8(tp, MCU) & ~NOW_IS_OOB);
5076 
5077 	r8168_mac_ocp_modify(tp, 0xe8de, BIT(14), 0);
5078 	r8168g_wait_ll_share_fifo_ready(tp);
5079 
5080 	r8168_mac_ocp_modify(tp, 0xe8de, 0, BIT(15));
5081 	r8168g_wait_ll_share_fifo_ready(tp);
5082 }
5083 
5084 static void rtl_hw_init_8125(struct rtl8169_private *tp)
5085 {
5086 	rtl_enable_rxdvgate(tp);
5087 
5088 	RTL_W8(tp, ChipCmd, RTL_R8(tp, ChipCmd) & ~(CmdTxEnb | CmdRxEnb));
5089 	msleep(1);
5090 	RTL_W8(tp, MCU, RTL_R8(tp, MCU) & ~NOW_IS_OOB);
5091 
5092 	r8168_mac_ocp_modify(tp, 0xe8de, BIT(14), 0);
5093 	r8168g_wait_ll_share_fifo_ready(tp);
5094 
5095 	r8168_mac_ocp_write(tp, 0xc0aa, 0x07d0);
5096 	r8168_mac_ocp_write(tp, 0xc0a6, 0x0150);
5097 	r8168_mac_ocp_write(tp, 0xc01e, 0x5555);
5098 	r8168g_wait_ll_share_fifo_ready(tp);
5099 }
5100 
5101 static void rtl_hw_initialize(struct rtl8169_private *tp)
5102 {
5103 	switch (tp->mac_version) {
5104 	case RTL_GIGA_MAC_VER_49 ... RTL_GIGA_MAC_VER_52:
5105 		rtl8168ep_stop_cmac(tp);
5106 		fallthrough;
5107 	case RTL_GIGA_MAC_VER_40 ... RTL_GIGA_MAC_VER_48:
5108 		rtl_hw_init_8168g(tp);
5109 		break;
5110 	case RTL_GIGA_MAC_VER_60 ... RTL_GIGA_MAC_VER_63:
5111 		rtl_hw_init_8125(tp);
5112 		break;
5113 	default:
5114 		break;
5115 	}
5116 }
5117 
5118 static int rtl_jumbo_max(struct rtl8169_private *tp)
5119 {
5120 	/* Non-GBit versions don't support jumbo frames */
5121 	if (!tp->supports_gmii)
5122 		return 0;
5123 
5124 	switch (tp->mac_version) {
5125 	/* RTL8169 */
5126 	case RTL_GIGA_MAC_VER_02 ... RTL_GIGA_MAC_VER_06:
5127 		return JUMBO_7K;
5128 	/* RTL8168b */
5129 	case RTL_GIGA_MAC_VER_11:
5130 	case RTL_GIGA_MAC_VER_12:
5131 	case RTL_GIGA_MAC_VER_17:
5132 		return JUMBO_4K;
5133 	/* RTL8168c */
5134 	case RTL_GIGA_MAC_VER_18 ... RTL_GIGA_MAC_VER_24:
5135 		return JUMBO_6K;
5136 	default:
5137 		return JUMBO_9K;
5138 	}
5139 }
5140 
5141 static void rtl_disable_clk(void *data)
5142 {
5143 	clk_disable_unprepare(data);
5144 }
5145 
5146 static int rtl_get_ether_clk(struct rtl8169_private *tp)
5147 {
5148 	struct device *d = tp_to_dev(tp);
5149 	struct clk *clk;
5150 	int rc;
5151 
5152 	clk = devm_clk_get(d, "ether_clk");
5153 	if (IS_ERR(clk)) {
5154 		rc = PTR_ERR(clk);
5155 		if (rc == -ENOENT)
5156 			/* clk-core allows NULL (for suspend / resume) */
5157 			rc = 0;
5158 		else
5159 			dev_err_probe(d, rc, "failed to get clk\n");
5160 	} else {
5161 		tp->clk = clk;
5162 		rc = clk_prepare_enable(clk);
5163 		if (rc)
5164 			dev_err(d, "failed to enable clk: %d\n", rc);
5165 		else
5166 			rc = devm_add_action_or_reset(d, rtl_disable_clk, clk);
5167 	}
5168 
5169 	return rc;
5170 }
5171 
5172 static void rtl_init_mac_address(struct rtl8169_private *tp)
5173 {
5174 	struct net_device *dev = tp->dev;
5175 	u8 *mac_addr = dev->dev_addr;
5176 	int rc;
5177 
5178 	rc = eth_platform_get_mac_address(tp_to_dev(tp), mac_addr);
5179 	if (!rc)
5180 		goto done;
5181 
5182 	rtl_read_mac_address(tp, mac_addr);
5183 	if (is_valid_ether_addr(mac_addr))
5184 		goto done;
5185 
5186 	rtl_read_mac_from_reg(tp, mac_addr, MAC0);
5187 	if (is_valid_ether_addr(mac_addr))
5188 		goto done;
5189 
5190 	eth_hw_addr_random(dev);
5191 	dev_warn(tp_to_dev(tp), "can't read MAC address, setting random one\n");
5192 done:
5193 	rtl_rar_set(tp, mac_addr);
5194 }
5195 
5196 static int rtl_init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
5197 {
5198 	struct rtl8169_private *tp;
5199 	int jumbo_max, region, rc;
5200 	enum mac_version chipset;
5201 	struct net_device *dev;
5202 	u16 xid;
5203 
5204 	dev = devm_alloc_etherdev(&pdev->dev, sizeof (*tp));
5205 	if (!dev)
5206 		return -ENOMEM;
5207 
5208 	SET_NETDEV_DEV(dev, &pdev->dev);
5209 	dev->netdev_ops = &rtl_netdev_ops;
5210 	tp = netdev_priv(dev);
5211 	tp->dev = dev;
5212 	tp->pci_dev = pdev;
5213 	tp->supports_gmii = ent->driver_data == RTL_CFG_NO_GBIT ? 0 : 1;
5214 	tp->eee_adv = -1;
5215 	tp->ocp_base = OCP_STD_PHY_BASE;
5216 
5217 	dev->tstats = devm_netdev_alloc_pcpu_stats(&pdev->dev,
5218 						   struct pcpu_sw_netstats);
5219 	if (!dev->tstats)
5220 		return -ENOMEM;
5221 
5222 	/* Get the *optional* external "ether_clk" used on some boards */
5223 	rc = rtl_get_ether_clk(tp);
5224 	if (rc)
5225 		return rc;
5226 
5227 	/* Disable ASPM completely as that cause random device stop working
5228 	 * problems as well as full system hangs for some PCIe devices users.
5229 	 */
5230 	rc = pci_disable_link_state(pdev, PCIE_LINK_STATE_L0S |
5231 					  PCIE_LINK_STATE_L1);
5232 	tp->aspm_manageable = !rc;
5233 
5234 	/* enable device (incl. PCI PM wakeup and hotplug setup) */
5235 	rc = pcim_enable_device(pdev);
5236 	if (rc < 0) {
5237 		dev_err(&pdev->dev, "enable failure\n");
5238 		return rc;
5239 	}
5240 
5241 	if (pcim_set_mwi(pdev) < 0)
5242 		dev_info(&pdev->dev, "Mem-Wr-Inval unavailable\n");
5243 
5244 	/* use first MMIO region */
5245 	region = ffs(pci_select_bars(pdev, IORESOURCE_MEM)) - 1;
5246 	if (region < 0) {
5247 		dev_err(&pdev->dev, "no MMIO resource found\n");
5248 		return -ENODEV;
5249 	}
5250 
5251 	/* check for weird/broken PCI region reporting */
5252 	if (pci_resource_len(pdev, region) < R8169_REGS_SIZE) {
5253 		dev_err(&pdev->dev, "Invalid PCI region size(s), aborting\n");
5254 		return -ENODEV;
5255 	}
5256 
5257 	rc = pcim_iomap_regions(pdev, BIT(region), MODULENAME);
5258 	if (rc < 0) {
5259 		dev_err(&pdev->dev, "cannot remap MMIO, aborting\n");
5260 		return rc;
5261 	}
5262 
5263 	tp->mmio_addr = pcim_iomap_table(pdev)[region];
5264 
5265 	xid = (RTL_R32(tp, TxConfig) >> 20) & 0xfcf;
5266 
5267 	/* Identify chip attached to board */
5268 	chipset = rtl8169_get_mac_version(xid, tp->supports_gmii);
5269 	if (chipset == RTL_GIGA_MAC_NONE) {
5270 		dev_err(&pdev->dev, "unknown chip XID %03x\n", xid);
5271 		return -ENODEV;
5272 	}
5273 
5274 	tp->mac_version = chipset;
5275 
5276 	tp->cp_cmd = RTL_R16(tp, CPlusCmd) & CPCMD_MASK;
5277 
5278 	if (sizeof(dma_addr_t) > 4 && tp->mac_version >= RTL_GIGA_MAC_VER_18 &&
5279 	    !dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)))
5280 		dev->features |= NETIF_F_HIGHDMA;
5281 
5282 	rtl_init_rxcfg(tp);
5283 
5284 	rtl8169_irq_mask_and_ack(tp);
5285 
5286 	rtl_hw_initialize(tp);
5287 
5288 	rtl_hw_reset(tp);
5289 
5290 	pci_set_master(pdev);
5291 
5292 	rc = rtl_alloc_irq(tp);
5293 	if (rc < 0) {
5294 		dev_err(&pdev->dev, "Can't allocate interrupt\n");
5295 		return rc;
5296 	}
5297 
5298 	INIT_WORK(&tp->wk.work, rtl_task);
5299 
5300 	rtl_init_mac_address(tp);
5301 
5302 	dev->ethtool_ops = &rtl8169_ethtool_ops;
5303 
5304 	netif_napi_add(dev, &tp->napi, rtl8169_poll, NAPI_POLL_WEIGHT);
5305 
5306 	dev->hw_features = NETIF_F_IP_CSUM | NETIF_F_RXCSUM |
5307 			   NETIF_F_HW_VLAN_CTAG_TX | NETIF_F_HW_VLAN_CTAG_RX;
5308 	dev->vlan_features = NETIF_F_SG | NETIF_F_IP_CSUM | NETIF_F_TSO;
5309 	dev->priv_flags |= IFF_LIVE_ADDR_CHANGE;
5310 
5311 	/*
5312 	 * Pretend we are using VLANs; This bypasses a nasty bug where
5313 	 * Interrupts stop flowing on high load on 8110SCd controllers.
5314 	 */
5315 	if (tp->mac_version == RTL_GIGA_MAC_VER_05)
5316 		/* Disallow toggling */
5317 		dev->hw_features &= ~NETIF_F_HW_VLAN_CTAG_RX;
5318 
5319 	if (rtl_chip_supports_csum_v2(tp))
5320 		dev->hw_features |= NETIF_F_IPV6_CSUM;
5321 
5322 	dev->features |= dev->hw_features;
5323 
5324 	/* There has been a number of reports that using SG/TSO results in
5325 	 * tx timeouts. However for a lot of people SG/TSO works fine.
5326 	 * Therefore disable both features by default, but allow users to
5327 	 * enable them. Use at own risk!
5328 	 */
5329 	if (rtl_chip_supports_csum_v2(tp)) {
5330 		dev->hw_features |= NETIF_F_SG | NETIF_F_TSO | NETIF_F_TSO6;
5331 		dev->gso_max_size = RTL_GSO_MAX_SIZE_V2;
5332 		dev->gso_max_segs = RTL_GSO_MAX_SEGS_V2;
5333 	} else {
5334 		dev->hw_features |= NETIF_F_SG | NETIF_F_TSO;
5335 		dev->gso_max_size = RTL_GSO_MAX_SIZE_V1;
5336 		dev->gso_max_segs = RTL_GSO_MAX_SEGS_V1;
5337 	}
5338 
5339 	dev->hw_features |= NETIF_F_RXALL;
5340 	dev->hw_features |= NETIF_F_RXFCS;
5341 
5342 	/* configure chip for default features */
5343 	rtl8169_set_features(dev, dev->features);
5344 
5345 	jumbo_max = rtl_jumbo_max(tp);
5346 	if (jumbo_max)
5347 		dev->max_mtu = jumbo_max;
5348 
5349 	rtl_set_irq_mask(tp);
5350 
5351 	tp->fw_name = rtl_chip_infos[chipset].fw_name;
5352 
5353 	tp->counters = dmam_alloc_coherent (&pdev->dev, sizeof(*tp->counters),
5354 					    &tp->counters_phys_addr,
5355 					    GFP_KERNEL);
5356 	if (!tp->counters)
5357 		return -ENOMEM;
5358 
5359 	pci_set_drvdata(pdev, tp);
5360 
5361 	rc = r8169_mdio_register(tp);
5362 	if (rc)
5363 		return rc;
5364 
5365 	/* chip gets powered up in rtl_open() */
5366 	rtl_pll_power_down(tp);
5367 
5368 	rc = register_netdev(dev);
5369 	if (rc)
5370 		return rc;
5371 
5372 	netdev_info(dev, "%s, %pM, XID %03x, IRQ %d\n",
5373 		    rtl_chip_infos[chipset].name, dev->dev_addr, xid,
5374 		    pci_irq_vector(pdev, 0));
5375 
5376 	if (jumbo_max)
5377 		netdev_info(dev, "jumbo features [frames: %d bytes, tx checksumming: %s]\n",
5378 			    jumbo_max, tp->mac_version <= RTL_GIGA_MAC_VER_06 ?
5379 			    "ok" : "ko");
5380 
5381 	if (r8168_check_dash(tp)) {
5382 		netdev_info(dev, "DASH enabled\n");
5383 		rtl8168_driver_start(tp);
5384 	}
5385 
5386 	if (pci_dev_run_wake(pdev))
5387 		pm_runtime_put_sync(&pdev->dev);
5388 
5389 	return 0;
5390 }
5391 
5392 static struct pci_driver rtl8169_pci_driver = {
5393 	.name		= MODULENAME,
5394 	.id_table	= rtl8169_pci_tbl,
5395 	.probe		= rtl_init_one,
5396 	.remove		= rtl_remove_one,
5397 	.shutdown	= rtl_shutdown,
5398 #ifdef CONFIG_PM
5399 	.driver.pm	= &rtl8169_pm_ops,
5400 #endif
5401 };
5402 
5403 module_pci_driver(rtl8169_pci_driver);
5404