xref: /linux/drivers/net/ethernet/marvell/mv643xx_eth.c (revision 4413e16d9d21673bb5048a2e542f1aaa00015c2e)
1 /*
2  * Driver for Marvell Discovery (MV643XX) and Marvell Orion ethernet ports
3  * Copyright (C) 2002 Matthew Dharm <mdharm@momenco.com>
4  *
5  * Based on the 64360 driver from:
6  * Copyright (C) 2002 Rabeeh Khoury <rabeeh@galileo.co.il>
7  *		      Rabeeh Khoury <rabeeh@marvell.com>
8  *
9  * Copyright (C) 2003 PMC-Sierra, Inc.,
10  *	written by Manish Lachwani
11  *
12  * Copyright (C) 2003 Ralf Baechle <ralf@linux-mips.org>
13  *
14  * Copyright (C) 2004-2006 MontaVista Software, Inc.
15  *			   Dale Farnsworth <dale@farnsworth.org>
16  *
17  * Copyright (C) 2004 Steven J. Hill <sjhill1@rockwellcollins.com>
18  *				     <sjhill@realitydiluted.com>
19  *
20  * Copyright (C) 2007-2008 Marvell Semiconductor
21  *			   Lennert Buytenhek <buytenh@marvell.com>
22  *
23  * This program is free software; you can redistribute it and/or
24  * modify it under the terms of the GNU General Public License
25  * as published by the Free Software Foundation; either version 2
26  * of the License, or (at your option) any later version.
27  *
28  * This program is distributed in the hope that it will be useful,
29  * but WITHOUT ANY WARRANTY; without even the implied warranty of
30  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
31  * GNU General Public License for more details.
32  *
33  * You should have received a copy of the GNU General Public License
34  * along with this program; if not, write to the Free Software
35  * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
36  */
37 
38 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
39 
40 #include <linux/init.h>
41 #include <linux/dma-mapping.h>
42 #include <linux/in.h>
43 #include <linux/ip.h>
44 #include <linux/tcp.h>
45 #include <linux/udp.h>
46 #include <linux/etherdevice.h>
47 #include <linux/delay.h>
48 #include <linux/ethtool.h>
49 #include <linux/platform_device.h>
50 #include <linux/module.h>
51 #include <linux/kernel.h>
52 #include <linux/spinlock.h>
53 #include <linux/workqueue.h>
54 #include <linux/phy.h>
55 #include <linux/mv643xx_eth.h>
56 #include <linux/io.h>
57 #include <linux/types.h>
58 #include <linux/inet_lro.h>
59 #include <linux/slab.h>
60 #include <linux/clk.h>
61 
62 static char mv643xx_eth_driver_name[] = "mv643xx_eth";
63 static char mv643xx_eth_driver_version[] = "1.4";
64 
65 
66 /*
67  * Registers shared between all ports.
68  */
69 #define PHY_ADDR			0x0000
70 #define SMI_REG				0x0004
71 #define  SMI_BUSY			0x10000000
72 #define  SMI_READ_VALID			0x08000000
73 #define  SMI_OPCODE_READ		0x04000000
74 #define  SMI_OPCODE_WRITE		0x00000000
75 #define ERR_INT_CAUSE			0x0080
76 #define  ERR_INT_SMI_DONE		0x00000010
77 #define ERR_INT_MASK			0x0084
78 #define WINDOW_BASE(w)			(0x0200 + ((w) << 3))
79 #define WINDOW_SIZE(w)			(0x0204 + ((w) << 3))
80 #define WINDOW_REMAP_HIGH(w)		(0x0280 + ((w) << 2))
81 #define WINDOW_BAR_ENABLE		0x0290
82 #define WINDOW_PROTECT(w)		(0x0294 + ((w) << 4))
83 
84 /*
85  * Main per-port registers.  These live at offset 0x0400 for
86  * port #0, 0x0800 for port #1, and 0x0c00 for port #2.
87  */
88 #define PORT_CONFIG			0x0000
89 #define  UNICAST_PROMISCUOUS_MODE	0x00000001
90 #define PORT_CONFIG_EXT			0x0004
91 #define MAC_ADDR_LOW			0x0014
92 #define MAC_ADDR_HIGH			0x0018
93 #define SDMA_CONFIG			0x001c
94 #define  TX_BURST_SIZE_16_64BIT		0x01000000
95 #define  TX_BURST_SIZE_4_64BIT		0x00800000
96 #define  BLM_TX_NO_SWAP			0x00000020
97 #define  BLM_RX_NO_SWAP			0x00000010
98 #define  RX_BURST_SIZE_16_64BIT		0x00000008
99 #define  RX_BURST_SIZE_4_64BIT		0x00000004
100 #define PORT_SERIAL_CONTROL		0x003c
101 #define  SET_MII_SPEED_TO_100		0x01000000
102 #define  SET_GMII_SPEED_TO_1000		0x00800000
103 #define  SET_FULL_DUPLEX_MODE		0x00200000
104 #define  MAX_RX_PACKET_9700BYTE		0x000a0000
105 #define  DISABLE_AUTO_NEG_SPEED_GMII	0x00002000
106 #define  DO_NOT_FORCE_LINK_FAIL		0x00000400
107 #define  SERIAL_PORT_CONTROL_RESERVED	0x00000200
108 #define  DISABLE_AUTO_NEG_FOR_FLOW_CTRL	0x00000008
109 #define  DISABLE_AUTO_NEG_FOR_DUPLEX	0x00000004
110 #define  FORCE_LINK_PASS		0x00000002
111 #define  SERIAL_PORT_ENABLE		0x00000001
112 #define PORT_STATUS			0x0044
113 #define  TX_FIFO_EMPTY			0x00000400
114 #define  TX_IN_PROGRESS			0x00000080
115 #define  PORT_SPEED_MASK		0x00000030
116 #define  PORT_SPEED_1000		0x00000010
117 #define  PORT_SPEED_100			0x00000020
118 #define  PORT_SPEED_10			0x00000000
119 #define  FLOW_CONTROL_ENABLED		0x00000008
120 #define  FULL_DUPLEX			0x00000004
121 #define  LINK_UP			0x00000002
122 #define TXQ_COMMAND			0x0048
123 #define TXQ_FIX_PRIO_CONF		0x004c
124 #define TX_BW_RATE			0x0050
125 #define TX_BW_MTU			0x0058
126 #define TX_BW_BURST			0x005c
127 #define INT_CAUSE			0x0060
128 #define  INT_TX_END			0x07f80000
129 #define  INT_TX_END_0			0x00080000
130 #define  INT_RX				0x000003fc
131 #define  INT_RX_0			0x00000004
132 #define  INT_EXT			0x00000002
133 #define INT_CAUSE_EXT			0x0064
134 #define  INT_EXT_LINK_PHY		0x00110000
135 #define  INT_EXT_TX			0x000000ff
136 #define INT_MASK			0x0068
137 #define INT_MASK_EXT			0x006c
138 #define TX_FIFO_URGENT_THRESHOLD	0x0074
139 #define RX_DISCARD_FRAME_CNT		0x0084
140 #define RX_OVERRUN_FRAME_CNT		0x0088
141 #define TXQ_FIX_PRIO_CONF_MOVED		0x00dc
142 #define TX_BW_RATE_MOVED		0x00e0
143 #define TX_BW_MTU_MOVED			0x00e8
144 #define TX_BW_BURST_MOVED		0x00ec
145 #define RXQ_CURRENT_DESC_PTR(q)		(0x020c + ((q) << 4))
146 #define RXQ_COMMAND			0x0280
147 #define TXQ_CURRENT_DESC_PTR(q)		(0x02c0 + ((q) << 2))
148 #define TXQ_BW_TOKENS(q)		(0x0300 + ((q) << 4))
149 #define TXQ_BW_CONF(q)			(0x0304 + ((q) << 4))
150 #define TXQ_BW_WRR_CONF(q)		(0x0308 + ((q) << 4))
151 
152 /*
153  * Misc per-port registers.
154  */
155 #define MIB_COUNTERS(p)			(0x1000 + ((p) << 7))
156 #define SPECIAL_MCAST_TABLE(p)		(0x1400 + ((p) << 10))
157 #define OTHER_MCAST_TABLE(p)		(0x1500 + ((p) << 10))
158 #define UNICAST_TABLE(p)		(0x1600 + ((p) << 10))
159 
160 
161 /*
162  * SDMA configuration register default value.
163  */
164 #if defined(__BIG_ENDIAN)
165 #define PORT_SDMA_CONFIG_DEFAULT_VALUE		\
166 		(RX_BURST_SIZE_4_64BIT	|	\
167 		 TX_BURST_SIZE_4_64BIT)
168 #elif defined(__LITTLE_ENDIAN)
169 #define PORT_SDMA_CONFIG_DEFAULT_VALUE		\
170 		(RX_BURST_SIZE_4_64BIT	|	\
171 		 BLM_RX_NO_SWAP		|	\
172 		 BLM_TX_NO_SWAP		|	\
173 		 TX_BURST_SIZE_4_64BIT)
174 #else
175 #error One of __BIG_ENDIAN or __LITTLE_ENDIAN must be defined
176 #endif
177 
178 
179 /*
180  * Misc definitions.
181  */
182 #define DEFAULT_RX_QUEUE_SIZE	128
183 #define DEFAULT_TX_QUEUE_SIZE	256
184 #define SKB_DMA_REALIGN		((PAGE_SIZE - NET_SKB_PAD) % SMP_CACHE_BYTES)
185 
186 
187 /*
188  * RX/TX descriptors.
189  */
190 #if defined(__BIG_ENDIAN)
191 struct rx_desc {
192 	u16 byte_cnt;		/* Descriptor buffer byte count		*/
193 	u16 buf_size;		/* Buffer size				*/
194 	u32 cmd_sts;		/* Descriptor command status		*/
195 	u32 next_desc_ptr;	/* Next descriptor pointer		*/
196 	u32 buf_ptr;		/* Descriptor buffer pointer		*/
197 };
198 
199 struct tx_desc {
200 	u16 byte_cnt;		/* buffer byte count			*/
201 	u16 l4i_chk;		/* CPU provided TCP checksum		*/
202 	u32 cmd_sts;		/* Command/status field			*/
203 	u32 next_desc_ptr;	/* Pointer to next descriptor		*/
204 	u32 buf_ptr;		/* pointer to buffer for this descriptor*/
205 };
206 #elif defined(__LITTLE_ENDIAN)
207 struct rx_desc {
208 	u32 cmd_sts;		/* Descriptor command status		*/
209 	u16 buf_size;		/* Buffer size				*/
210 	u16 byte_cnt;		/* Descriptor buffer byte count		*/
211 	u32 buf_ptr;		/* Descriptor buffer pointer		*/
212 	u32 next_desc_ptr;	/* Next descriptor pointer		*/
213 };
214 
215 struct tx_desc {
216 	u32 cmd_sts;		/* Command/status field			*/
217 	u16 l4i_chk;		/* CPU provided TCP checksum		*/
218 	u16 byte_cnt;		/* buffer byte count			*/
219 	u32 buf_ptr;		/* pointer to buffer for this descriptor*/
220 	u32 next_desc_ptr;	/* Pointer to next descriptor		*/
221 };
222 #else
223 #error One of __BIG_ENDIAN or __LITTLE_ENDIAN must be defined
224 #endif
225 
226 /* RX & TX descriptor command */
227 #define BUFFER_OWNED_BY_DMA		0x80000000
228 
229 /* RX & TX descriptor status */
230 #define ERROR_SUMMARY			0x00000001
231 
232 /* RX descriptor status */
233 #define LAYER_4_CHECKSUM_OK		0x40000000
234 #define RX_ENABLE_INTERRUPT		0x20000000
235 #define RX_FIRST_DESC			0x08000000
236 #define RX_LAST_DESC			0x04000000
237 #define RX_IP_HDR_OK			0x02000000
238 #define RX_PKT_IS_IPV4			0x01000000
239 #define RX_PKT_IS_ETHERNETV2		0x00800000
240 #define RX_PKT_LAYER4_TYPE_MASK		0x00600000
241 #define RX_PKT_LAYER4_TYPE_TCP_IPV4	0x00000000
242 #define RX_PKT_IS_VLAN_TAGGED		0x00080000
243 
244 /* TX descriptor command */
245 #define TX_ENABLE_INTERRUPT		0x00800000
246 #define GEN_CRC				0x00400000
247 #define TX_FIRST_DESC			0x00200000
248 #define TX_LAST_DESC			0x00100000
249 #define ZERO_PADDING			0x00080000
250 #define GEN_IP_V4_CHECKSUM		0x00040000
251 #define GEN_TCP_UDP_CHECKSUM		0x00020000
252 #define UDP_FRAME			0x00010000
253 #define MAC_HDR_EXTRA_4_BYTES		0x00008000
254 #define MAC_HDR_EXTRA_8_BYTES		0x00000200
255 
256 #define TX_IHL_SHIFT			11
257 
258 
259 /* global *******************************************************************/
260 struct mv643xx_eth_shared_private {
261 	/*
262 	 * Ethernet controller base address.
263 	 */
264 	void __iomem *base;
265 
266 	/*
267 	 * Points at the right SMI instance to use.
268 	 */
269 	struct mv643xx_eth_shared_private *smi;
270 
271 	/*
272 	 * Provides access to local SMI interface.
273 	 */
274 	struct mii_bus *smi_bus;
275 
276 	/*
277 	 * If we have access to the error interrupt pin (which is
278 	 * somewhat misnamed as it not only reflects internal errors
279 	 * but also reflects SMI completion), use that to wait for
280 	 * SMI access completion instead of polling the SMI busy bit.
281 	 */
282 	int err_interrupt;
283 	wait_queue_head_t smi_busy_wait;
284 
285 	/*
286 	 * Per-port MBUS window access register value.
287 	 */
288 	u32 win_protect;
289 
290 	/*
291 	 * Hardware-specific parameters.
292 	 */
293 	int extended_rx_coal_limit;
294 	int tx_bw_control;
295 	int tx_csum_limit;
296 
297 };
298 
299 #define TX_BW_CONTROL_ABSENT		0
300 #define TX_BW_CONTROL_OLD_LAYOUT	1
301 #define TX_BW_CONTROL_NEW_LAYOUT	2
302 
303 static int mv643xx_eth_open(struct net_device *dev);
304 static int mv643xx_eth_stop(struct net_device *dev);
305 
306 
307 /* per-port *****************************************************************/
308 struct mib_counters {
309 	u64 good_octets_received;
310 	u32 bad_octets_received;
311 	u32 internal_mac_transmit_err;
312 	u32 good_frames_received;
313 	u32 bad_frames_received;
314 	u32 broadcast_frames_received;
315 	u32 multicast_frames_received;
316 	u32 frames_64_octets;
317 	u32 frames_65_to_127_octets;
318 	u32 frames_128_to_255_octets;
319 	u32 frames_256_to_511_octets;
320 	u32 frames_512_to_1023_octets;
321 	u32 frames_1024_to_max_octets;
322 	u64 good_octets_sent;
323 	u32 good_frames_sent;
324 	u32 excessive_collision;
325 	u32 multicast_frames_sent;
326 	u32 broadcast_frames_sent;
327 	u32 unrec_mac_control_received;
328 	u32 fc_sent;
329 	u32 good_fc_received;
330 	u32 bad_fc_received;
331 	u32 undersize_received;
332 	u32 fragments_received;
333 	u32 oversize_received;
334 	u32 jabber_received;
335 	u32 mac_receive_error;
336 	u32 bad_crc_event;
337 	u32 collision;
338 	u32 late_collision;
339 	/* Non MIB hardware counters */
340 	u32 rx_discard;
341 	u32 rx_overrun;
342 };
343 
344 struct lro_counters {
345 	u32 lro_aggregated;
346 	u32 lro_flushed;
347 	u32 lro_no_desc;
348 };
349 
350 struct rx_queue {
351 	int index;
352 
353 	int rx_ring_size;
354 
355 	int rx_desc_count;
356 	int rx_curr_desc;
357 	int rx_used_desc;
358 
359 	struct rx_desc *rx_desc_area;
360 	dma_addr_t rx_desc_dma;
361 	int rx_desc_area_size;
362 	struct sk_buff **rx_skb;
363 
364 	struct net_lro_mgr lro_mgr;
365 	struct net_lro_desc lro_arr[8];
366 };
367 
368 struct tx_queue {
369 	int index;
370 
371 	int tx_ring_size;
372 
373 	int tx_desc_count;
374 	int tx_curr_desc;
375 	int tx_used_desc;
376 
377 	struct tx_desc *tx_desc_area;
378 	dma_addr_t tx_desc_dma;
379 	int tx_desc_area_size;
380 
381 	struct sk_buff_head tx_skb;
382 
383 	unsigned long tx_packets;
384 	unsigned long tx_bytes;
385 	unsigned long tx_dropped;
386 };
387 
388 struct mv643xx_eth_private {
389 	struct mv643xx_eth_shared_private *shared;
390 	void __iomem *base;
391 	int port_num;
392 
393 	struct net_device *dev;
394 
395 	struct phy_device *phy;
396 
397 	struct timer_list mib_counters_timer;
398 	spinlock_t mib_counters_lock;
399 	struct mib_counters mib_counters;
400 
401 	struct lro_counters lro_counters;
402 
403 	struct work_struct tx_timeout_task;
404 
405 	struct napi_struct napi;
406 	u32 int_mask;
407 	u8 oom;
408 	u8 work_link;
409 	u8 work_tx;
410 	u8 work_tx_end;
411 	u8 work_rx;
412 	u8 work_rx_refill;
413 
414 	int skb_size;
415 	struct sk_buff_head rx_recycle;
416 
417 	/*
418 	 * RX state.
419 	 */
420 	int rx_ring_size;
421 	unsigned long rx_desc_sram_addr;
422 	int rx_desc_sram_size;
423 	int rxq_count;
424 	struct timer_list rx_oom;
425 	struct rx_queue rxq[8];
426 
427 	/*
428 	 * TX state.
429 	 */
430 	int tx_ring_size;
431 	unsigned long tx_desc_sram_addr;
432 	int tx_desc_sram_size;
433 	int txq_count;
434 	struct tx_queue txq[8];
435 
436 	/*
437 	 * Hardware-specific parameters.
438 	 */
439 #if defined(CONFIG_HAVE_CLK)
440 	struct clk *clk;
441 #endif
442 	unsigned int t_clk;
443 };
444 
445 
446 /* port register accessors **************************************************/
447 static inline u32 rdl(struct mv643xx_eth_private *mp, int offset)
448 {
449 	return readl(mp->shared->base + offset);
450 }
451 
452 static inline u32 rdlp(struct mv643xx_eth_private *mp, int offset)
453 {
454 	return readl(mp->base + offset);
455 }
456 
457 static inline void wrl(struct mv643xx_eth_private *mp, int offset, u32 data)
458 {
459 	writel(data, mp->shared->base + offset);
460 }
461 
462 static inline void wrlp(struct mv643xx_eth_private *mp, int offset, u32 data)
463 {
464 	writel(data, mp->base + offset);
465 }
466 
467 
468 /* rxq/txq helper functions *************************************************/
469 static struct mv643xx_eth_private *rxq_to_mp(struct rx_queue *rxq)
470 {
471 	return container_of(rxq, struct mv643xx_eth_private, rxq[rxq->index]);
472 }
473 
474 static struct mv643xx_eth_private *txq_to_mp(struct tx_queue *txq)
475 {
476 	return container_of(txq, struct mv643xx_eth_private, txq[txq->index]);
477 }
478 
479 static void rxq_enable(struct rx_queue *rxq)
480 {
481 	struct mv643xx_eth_private *mp = rxq_to_mp(rxq);
482 	wrlp(mp, RXQ_COMMAND, 1 << rxq->index);
483 }
484 
485 static void rxq_disable(struct rx_queue *rxq)
486 {
487 	struct mv643xx_eth_private *mp = rxq_to_mp(rxq);
488 	u8 mask = 1 << rxq->index;
489 
490 	wrlp(mp, RXQ_COMMAND, mask << 8);
491 	while (rdlp(mp, RXQ_COMMAND) & mask)
492 		udelay(10);
493 }
494 
495 static void txq_reset_hw_ptr(struct tx_queue *txq)
496 {
497 	struct mv643xx_eth_private *mp = txq_to_mp(txq);
498 	u32 addr;
499 
500 	addr = (u32)txq->tx_desc_dma;
501 	addr += txq->tx_curr_desc * sizeof(struct tx_desc);
502 	wrlp(mp, TXQ_CURRENT_DESC_PTR(txq->index), addr);
503 }
504 
505 static void txq_enable(struct tx_queue *txq)
506 {
507 	struct mv643xx_eth_private *mp = txq_to_mp(txq);
508 	wrlp(mp, TXQ_COMMAND, 1 << txq->index);
509 }
510 
511 static void txq_disable(struct tx_queue *txq)
512 {
513 	struct mv643xx_eth_private *mp = txq_to_mp(txq);
514 	u8 mask = 1 << txq->index;
515 
516 	wrlp(mp, TXQ_COMMAND, mask << 8);
517 	while (rdlp(mp, TXQ_COMMAND) & mask)
518 		udelay(10);
519 }
520 
521 static void txq_maybe_wake(struct tx_queue *txq)
522 {
523 	struct mv643xx_eth_private *mp = txq_to_mp(txq);
524 	struct netdev_queue *nq = netdev_get_tx_queue(mp->dev, txq->index);
525 
526 	if (netif_tx_queue_stopped(nq)) {
527 		__netif_tx_lock(nq, smp_processor_id());
528 		if (txq->tx_ring_size - txq->tx_desc_count >= MAX_SKB_FRAGS + 1)
529 			netif_tx_wake_queue(nq);
530 		__netif_tx_unlock(nq);
531 	}
532 }
533 
534 
535 /* rx napi ******************************************************************/
536 static int
537 mv643xx_get_skb_header(struct sk_buff *skb, void **iphdr, void **tcph,
538 		       u64 *hdr_flags, void *priv)
539 {
540 	unsigned long cmd_sts = (unsigned long)priv;
541 
542 	/*
543 	 * Make sure that this packet is Ethernet II, is not VLAN
544 	 * tagged, is IPv4, has a valid IP header, and is TCP.
545 	 */
546 	if ((cmd_sts & (RX_IP_HDR_OK | RX_PKT_IS_IPV4 |
547 		       RX_PKT_IS_ETHERNETV2 | RX_PKT_LAYER4_TYPE_MASK |
548 		       RX_PKT_IS_VLAN_TAGGED)) !=
549 	    (RX_IP_HDR_OK | RX_PKT_IS_IPV4 |
550 	     RX_PKT_IS_ETHERNETV2 | RX_PKT_LAYER4_TYPE_TCP_IPV4))
551 		return -1;
552 
553 	skb_reset_network_header(skb);
554 	skb_set_transport_header(skb, ip_hdrlen(skb));
555 	*iphdr = ip_hdr(skb);
556 	*tcph = tcp_hdr(skb);
557 	*hdr_flags = LRO_IPV4 | LRO_TCP;
558 
559 	return 0;
560 }
561 
562 static int rxq_process(struct rx_queue *rxq, int budget)
563 {
564 	struct mv643xx_eth_private *mp = rxq_to_mp(rxq);
565 	struct net_device_stats *stats = &mp->dev->stats;
566 	int lro_flush_needed;
567 	int rx;
568 
569 	lro_flush_needed = 0;
570 	rx = 0;
571 	while (rx < budget && rxq->rx_desc_count) {
572 		struct rx_desc *rx_desc;
573 		unsigned int cmd_sts;
574 		struct sk_buff *skb;
575 		u16 byte_cnt;
576 
577 		rx_desc = &rxq->rx_desc_area[rxq->rx_curr_desc];
578 
579 		cmd_sts = rx_desc->cmd_sts;
580 		if (cmd_sts & BUFFER_OWNED_BY_DMA)
581 			break;
582 		rmb();
583 
584 		skb = rxq->rx_skb[rxq->rx_curr_desc];
585 		rxq->rx_skb[rxq->rx_curr_desc] = NULL;
586 
587 		rxq->rx_curr_desc++;
588 		if (rxq->rx_curr_desc == rxq->rx_ring_size)
589 			rxq->rx_curr_desc = 0;
590 
591 		dma_unmap_single(mp->dev->dev.parent, rx_desc->buf_ptr,
592 				 rx_desc->buf_size, DMA_FROM_DEVICE);
593 		rxq->rx_desc_count--;
594 		rx++;
595 
596 		mp->work_rx_refill |= 1 << rxq->index;
597 
598 		byte_cnt = rx_desc->byte_cnt;
599 
600 		/*
601 		 * Update statistics.
602 		 *
603 		 * Note that the descriptor byte count includes 2 dummy
604 		 * bytes automatically inserted by the hardware at the
605 		 * start of the packet (which we don't count), and a 4
606 		 * byte CRC at the end of the packet (which we do count).
607 		 */
608 		stats->rx_packets++;
609 		stats->rx_bytes += byte_cnt - 2;
610 
611 		/*
612 		 * In case we received a packet without first / last bits
613 		 * on, or the error summary bit is set, the packet needs
614 		 * to be dropped.
615 		 */
616 		if ((cmd_sts & (RX_FIRST_DESC | RX_LAST_DESC | ERROR_SUMMARY))
617 			!= (RX_FIRST_DESC | RX_LAST_DESC))
618 			goto err;
619 
620 		/*
621 		 * The -4 is for the CRC in the trailer of the
622 		 * received packet
623 		 */
624 		skb_put(skb, byte_cnt - 2 - 4);
625 
626 		if (cmd_sts & LAYER_4_CHECKSUM_OK)
627 			skb->ip_summed = CHECKSUM_UNNECESSARY;
628 		skb->protocol = eth_type_trans(skb, mp->dev);
629 
630 		if (skb->dev->features & NETIF_F_LRO &&
631 		    skb->ip_summed == CHECKSUM_UNNECESSARY) {
632 			lro_receive_skb(&rxq->lro_mgr, skb, (void *)cmd_sts);
633 			lro_flush_needed = 1;
634 		} else
635 			netif_receive_skb(skb);
636 
637 		continue;
638 
639 err:
640 		stats->rx_dropped++;
641 
642 		if ((cmd_sts & (RX_FIRST_DESC | RX_LAST_DESC)) !=
643 			(RX_FIRST_DESC | RX_LAST_DESC)) {
644 			if (net_ratelimit())
645 				netdev_err(mp->dev,
646 					   "received packet spanning multiple descriptors\n");
647 		}
648 
649 		if (cmd_sts & ERROR_SUMMARY)
650 			stats->rx_errors++;
651 
652 		dev_kfree_skb(skb);
653 	}
654 
655 	if (lro_flush_needed)
656 		lro_flush_all(&rxq->lro_mgr);
657 
658 	if (rx < budget)
659 		mp->work_rx &= ~(1 << rxq->index);
660 
661 	return rx;
662 }
663 
664 static int rxq_refill(struct rx_queue *rxq, int budget)
665 {
666 	struct mv643xx_eth_private *mp = rxq_to_mp(rxq);
667 	int refilled;
668 
669 	refilled = 0;
670 	while (refilled < budget && rxq->rx_desc_count < rxq->rx_ring_size) {
671 		struct sk_buff *skb;
672 		int rx;
673 		struct rx_desc *rx_desc;
674 		int size;
675 
676 		skb = __skb_dequeue(&mp->rx_recycle);
677 		if (skb == NULL)
678 			skb = netdev_alloc_skb(mp->dev, mp->skb_size);
679 
680 		if (skb == NULL) {
681 			mp->oom = 1;
682 			goto oom;
683 		}
684 
685 		if (SKB_DMA_REALIGN)
686 			skb_reserve(skb, SKB_DMA_REALIGN);
687 
688 		refilled++;
689 		rxq->rx_desc_count++;
690 
691 		rx = rxq->rx_used_desc++;
692 		if (rxq->rx_used_desc == rxq->rx_ring_size)
693 			rxq->rx_used_desc = 0;
694 
695 		rx_desc = rxq->rx_desc_area + rx;
696 
697 		size = skb->end - skb->data;
698 		rx_desc->buf_ptr = dma_map_single(mp->dev->dev.parent,
699 						  skb->data, size,
700 						  DMA_FROM_DEVICE);
701 		rx_desc->buf_size = size;
702 		rxq->rx_skb[rx] = skb;
703 		wmb();
704 		rx_desc->cmd_sts = BUFFER_OWNED_BY_DMA | RX_ENABLE_INTERRUPT;
705 		wmb();
706 
707 		/*
708 		 * The hardware automatically prepends 2 bytes of
709 		 * dummy data to each received packet, so that the
710 		 * IP header ends up 16-byte aligned.
711 		 */
712 		skb_reserve(skb, 2);
713 	}
714 
715 	if (refilled < budget)
716 		mp->work_rx_refill &= ~(1 << rxq->index);
717 
718 oom:
719 	return refilled;
720 }
721 
722 
723 /* tx ***********************************************************************/
724 static inline unsigned int has_tiny_unaligned_frags(struct sk_buff *skb)
725 {
726 	int frag;
727 
728 	for (frag = 0; frag < skb_shinfo(skb)->nr_frags; frag++) {
729 		const skb_frag_t *fragp = &skb_shinfo(skb)->frags[frag];
730 
731 		if (skb_frag_size(fragp) <= 8 && fragp->page_offset & 7)
732 			return 1;
733 	}
734 
735 	return 0;
736 }
737 
738 static void txq_submit_frag_skb(struct tx_queue *txq, struct sk_buff *skb)
739 {
740 	struct mv643xx_eth_private *mp = txq_to_mp(txq);
741 	int nr_frags = skb_shinfo(skb)->nr_frags;
742 	int frag;
743 
744 	for (frag = 0; frag < nr_frags; frag++) {
745 		skb_frag_t *this_frag;
746 		int tx_index;
747 		struct tx_desc *desc;
748 
749 		this_frag = &skb_shinfo(skb)->frags[frag];
750 		tx_index = txq->tx_curr_desc++;
751 		if (txq->tx_curr_desc == txq->tx_ring_size)
752 			txq->tx_curr_desc = 0;
753 		desc = &txq->tx_desc_area[tx_index];
754 
755 		/*
756 		 * The last fragment will generate an interrupt
757 		 * which will free the skb on TX completion.
758 		 */
759 		if (frag == nr_frags - 1) {
760 			desc->cmd_sts = BUFFER_OWNED_BY_DMA |
761 					ZERO_PADDING | TX_LAST_DESC |
762 					TX_ENABLE_INTERRUPT;
763 		} else {
764 			desc->cmd_sts = BUFFER_OWNED_BY_DMA;
765 		}
766 
767 		desc->l4i_chk = 0;
768 		desc->byte_cnt = skb_frag_size(this_frag);
769 		desc->buf_ptr = skb_frag_dma_map(mp->dev->dev.parent,
770 						 this_frag, 0,
771 						 skb_frag_size(this_frag),
772 						 DMA_TO_DEVICE);
773 	}
774 }
775 
776 static inline __be16 sum16_as_be(__sum16 sum)
777 {
778 	return (__force __be16)sum;
779 }
780 
781 static int txq_submit_skb(struct tx_queue *txq, struct sk_buff *skb)
782 {
783 	struct mv643xx_eth_private *mp = txq_to_mp(txq);
784 	int nr_frags = skb_shinfo(skb)->nr_frags;
785 	int tx_index;
786 	struct tx_desc *desc;
787 	u32 cmd_sts;
788 	u16 l4i_chk;
789 	int length;
790 
791 	cmd_sts = TX_FIRST_DESC | GEN_CRC | BUFFER_OWNED_BY_DMA;
792 	l4i_chk = 0;
793 
794 	if (skb->ip_summed == CHECKSUM_PARTIAL) {
795 		int hdr_len;
796 		int tag_bytes;
797 
798 		BUG_ON(skb->protocol != htons(ETH_P_IP) &&
799 		       skb->protocol != htons(ETH_P_8021Q));
800 
801 		hdr_len = (void *)ip_hdr(skb) - (void *)skb->data;
802 		tag_bytes = hdr_len - ETH_HLEN;
803 		if (skb->len - hdr_len > mp->shared->tx_csum_limit ||
804 		    unlikely(tag_bytes & ~12)) {
805 			if (skb_checksum_help(skb) == 0)
806 				goto no_csum;
807 			kfree_skb(skb);
808 			return 1;
809 		}
810 
811 		if (tag_bytes & 4)
812 			cmd_sts |= MAC_HDR_EXTRA_4_BYTES;
813 		if (tag_bytes & 8)
814 			cmd_sts |= MAC_HDR_EXTRA_8_BYTES;
815 
816 		cmd_sts |= GEN_TCP_UDP_CHECKSUM |
817 			   GEN_IP_V4_CHECKSUM   |
818 			   ip_hdr(skb)->ihl << TX_IHL_SHIFT;
819 
820 		switch (ip_hdr(skb)->protocol) {
821 		case IPPROTO_UDP:
822 			cmd_sts |= UDP_FRAME;
823 			l4i_chk = ntohs(sum16_as_be(udp_hdr(skb)->check));
824 			break;
825 		case IPPROTO_TCP:
826 			l4i_chk = ntohs(sum16_as_be(tcp_hdr(skb)->check));
827 			break;
828 		default:
829 			BUG();
830 		}
831 	} else {
832 no_csum:
833 		/* Errata BTS #50, IHL must be 5 if no HW checksum */
834 		cmd_sts |= 5 << TX_IHL_SHIFT;
835 	}
836 
837 	tx_index = txq->tx_curr_desc++;
838 	if (txq->tx_curr_desc == txq->tx_ring_size)
839 		txq->tx_curr_desc = 0;
840 	desc = &txq->tx_desc_area[tx_index];
841 
842 	if (nr_frags) {
843 		txq_submit_frag_skb(txq, skb);
844 		length = skb_headlen(skb);
845 	} else {
846 		cmd_sts |= ZERO_PADDING | TX_LAST_DESC | TX_ENABLE_INTERRUPT;
847 		length = skb->len;
848 	}
849 
850 	desc->l4i_chk = l4i_chk;
851 	desc->byte_cnt = length;
852 	desc->buf_ptr = dma_map_single(mp->dev->dev.parent, skb->data,
853 				       length, DMA_TO_DEVICE);
854 
855 	__skb_queue_tail(&txq->tx_skb, skb);
856 
857 	skb_tx_timestamp(skb);
858 
859 	/* ensure all other descriptors are written before first cmd_sts */
860 	wmb();
861 	desc->cmd_sts = cmd_sts;
862 
863 	/* clear TX_END status */
864 	mp->work_tx_end &= ~(1 << txq->index);
865 
866 	/* ensure all descriptors are written before poking hardware */
867 	wmb();
868 	txq_enable(txq);
869 
870 	txq->tx_desc_count += nr_frags + 1;
871 
872 	return 0;
873 }
874 
875 static netdev_tx_t mv643xx_eth_xmit(struct sk_buff *skb, struct net_device *dev)
876 {
877 	struct mv643xx_eth_private *mp = netdev_priv(dev);
878 	int length, queue;
879 	struct tx_queue *txq;
880 	struct netdev_queue *nq;
881 
882 	queue = skb_get_queue_mapping(skb);
883 	txq = mp->txq + queue;
884 	nq = netdev_get_tx_queue(dev, queue);
885 
886 	if (has_tiny_unaligned_frags(skb) && __skb_linearize(skb)) {
887 		txq->tx_dropped++;
888 		netdev_printk(KERN_DEBUG, dev,
889 			      "failed to linearize skb with tiny unaligned fragment\n");
890 		return NETDEV_TX_BUSY;
891 	}
892 
893 	if (txq->tx_ring_size - txq->tx_desc_count < MAX_SKB_FRAGS + 1) {
894 		if (net_ratelimit())
895 			netdev_err(dev, "tx queue full?!\n");
896 		kfree_skb(skb);
897 		return NETDEV_TX_OK;
898 	}
899 
900 	length = skb->len;
901 
902 	if (!txq_submit_skb(txq, skb)) {
903 		int entries_left;
904 
905 		txq->tx_bytes += length;
906 		txq->tx_packets++;
907 
908 		entries_left = txq->tx_ring_size - txq->tx_desc_count;
909 		if (entries_left < MAX_SKB_FRAGS + 1)
910 			netif_tx_stop_queue(nq);
911 	}
912 
913 	return NETDEV_TX_OK;
914 }
915 
916 
917 /* tx napi ******************************************************************/
918 static void txq_kick(struct tx_queue *txq)
919 {
920 	struct mv643xx_eth_private *mp = txq_to_mp(txq);
921 	struct netdev_queue *nq = netdev_get_tx_queue(mp->dev, txq->index);
922 	u32 hw_desc_ptr;
923 	u32 expected_ptr;
924 
925 	__netif_tx_lock(nq, smp_processor_id());
926 
927 	if (rdlp(mp, TXQ_COMMAND) & (1 << txq->index))
928 		goto out;
929 
930 	hw_desc_ptr = rdlp(mp, TXQ_CURRENT_DESC_PTR(txq->index));
931 	expected_ptr = (u32)txq->tx_desc_dma +
932 				txq->tx_curr_desc * sizeof(struct tx_desc);
933 
934 	if (hw_desc_ptr != expected_ptr)
935 		txq_enable(txq);
936 
937 out:
938 	__netif_tx_unlock(nq);
939 
940 	mp->work_tx_end &= ~(1 << txq->index);
941 }
942 
943 static int txq_reclaim(struct tx_queue *txq, int budget, int force)
944 {
945 	struct mv643xx_eth_private *mp = txq_to_mp(txq);
946 	struct netdev_queue *nq = netdev_get_tx_queue(mp->dev, txq->index);
947 	int reclaimed;
948 
949 	__netif_tx_lock(nq, smp_processor_id());
950 
951 	reclaimed = 0;
952 	while (reclaimed < budget && txq->tx_desc_count > 0) {
953 		int tx_index;
954 		struct tx_desc *desc;
955 		u32 cmd_sts;
956 		struct sk_buff *skb;
957 
958 		tx_index = txq->tx_used_desc;
959 		desc = &txq->tx_desc_area[tx_index];
960 		cmd_sts = desc->cmd_sts;
961 
962 		if (cmd_sts & BUFFER_OWNED_BY_DMA) {
963 			if (!force)
964 				break;
965 			desc->cmd_sts = cmd_sts & ~BUFFER_OWNED_BY_DMA;
966 		}
967 
968 		txq->tx_used_desc = tx_index + 1;
969 		if (txq->tx_used_desc == txq->tx_ring_size)
970 			txq->tx_used_desc = 0;
971 
972 		reclaimed++;
973 		txq->tx_desc_count--;
974 
975 		skb = NULL;
976 		if (cmd_sts & TX_LAST_DESC)
977 			skb = __skb_dequeue(&txq->tx_skb);
978 
979 		if (cmd_sts & ERROR_SUMMARY) {
980 			netdev_info(mp->dev, "tx error\n");
981 			mp->dev->stats.tx_errors++;
982 		}
983 
984 		if (cmd_sts & TX_FIRST_DESC) {
985 			dma_unmap_single(mp->dev->dev.parent, desc->buf_ptr,
986 					 desc->byte_cnt, DMA_TO_DEVICE);
987 		} else {
988 			dma_unmap_page(mp->dev->dev.parent, desc->buf_ptr,
989 				       desc->byte_cnt, DMA_TO_DEVICE);
990 		}
991 
992 		if (skb != NULL) {
993 			if (skb_queue_len(&mp->rx_recycle) <
994 					mp->rx_ring_size &&
995 			    skb_recycle_check(skb, mp->skb_size))
996 				__skb_queue_head(&mp->rx_recycle, skb);
997 			else
998 				dev_kfree_skb(skb);
999 		}
1000 	}
1001 
1002 	__netif_tx_unlock(nq);
1003 
1004 	if (reclaimed < budget)
1005 		mp->work_tx &= ~(1 << txq->index);
1006 
1007 	return reclaimed;
1008 }
1009 
1010 
1011 /* tx rate control **********************************************************/
1012 /*
1013  * Set total maximum TX rate (shared by all TX queues for this port)
1014  * to 'rate' bits per second, with a maximum burst of 'burst' bytes.
1015  */
1016 static void tx_set_rate(struct mv643xx_eth_private *mp, int rate, int burst)
1017 {
1018 	int token_rate;
1019 	int mtu;
1020 	int bucket_size;
1021 
1022 	token_rate = ((rate / 1000) * 64) / (mp->t_clk / 1000);
1023 	if (token_rate > 1023)
1024 		token_rate = 1023;
1025 
1026 	mtu = (mp->dev->mtu + 255) >> 8;
1027 	if (mtu > 63)
1028 		mtu = 63;
1029 
1030 	bucket_size = (burst + 255) >> 8;
1031 	if (bucket_size > 65535)
1032 		bucket_size = 65535;
1033 
1034 	switch (mp->shared->tx_bw_control) {
1035 	case TX_BW_CONTROL_OLD_LAYOUT:
1036 		wrlp(mp, TX_BW_RATE, token_rate);
1037 		wrlp(mp, TX_BW_MTU, mtu);
1038 		wrlp(mp, TX_BW_BURST, bucket_size);
1039 		break;
1040 	case TX_BW_CONTROL_NEW_LAYOUT:
1041 		wrlp(mp, TX_BW_RATE_MOVED, token_rate);
1042 		wrlp(mp, TX_BW_MTU_MOVED, mtu);
1043 		wrlp(mp, TX_BW_BURST_MOVED, bucket_size);
1044 		break;
1045 	}
1046 }
1047 
1048 static void txq_set_rate(struct tx_queue *txq, int rate, int burst)
1049 {
1050 	struct mv643xx_eth_private *mp = txq_to_mp(txq);
1051 	int token_rate;
1052 	int bucket_size;
1053 
1054 	token_rate = ((rate / 1000) * 64) / (mp->t_clk / 1000);
1055 	if (token_rate > 1023)
1056 		token_rate = 1023;
1057 
1058 	bucket_size = (burst + 255) >> 8;
1059 	if (bucket_size > 65535)
1060 		bucket_size = 65535;
1061 
1062 	wrlp(mp, TXQ_BW_TOKENS(txq->index), token_rate << 14);
1063 	wrlp(mp, TXQ_BW_CONF(txq->index), (bucket_size << 10) | token_rate);
1064 }
1065 
1066 static void txq_set_fixed_prio_mode(struct tx_queue *txq)
1067 {
1068 	struct mv643xx_eth_private *mp = txq_to_mp(txq);
1069 	int off;
1070 	u32 val;
1071 
1072 	/*
1073 	 * Turn on fixed priority mode.
1074 	 */
1075 	off = 0;
1076 	switch (mp->shared->tx_bw_control) {
1077 	case TX_BW_CONTROL_OLD_LAYOUT:
1078 		off = TXQ_FIX_PRIO_CONF;
1079 		break;
1080 	case TX_BW_CONTROL_NEW_LAYOUT:
1081 		off = TXQ_FIX_PRIO_CONF_MOVED;
1082 		break;
1083 	}
1084 
1085 	if (off) {
1086 		val = rdlp(mp, off);
1087 		val |= 1 << txq->index;
1088 		wrlp(mp, off, val);
1089 	}
1090 }
1091 
1092 
1093 /* mii management interface *************************************************/
1094 static irqreturn_t mv643xx_eth_err_irq(int irq, void *dev_id)
1095 {
1096 	struct mv643xx_eth_shared_private *msp = dev_id;
1097 
1098 	if (readl(msp->base + ERR_INT_CAUSE) & ERR_INT_SMI_DONE) {
1099 		writel(~ERR_INT_SMI_DONE, msp->base + ERR_INT_CAUSE);
1100 		wake_up(&msp->smi_busy_wait);
1101 		return IRQ_HANDLED;
1102 	}
1103 
1104 	return IRQ_NONE;
1105 }
1106 
1107 static int smi_is_done(struct mv643xx_eth_shared_private *msp)
1108 {
1109 	return !(readl(msp->base + SMI_REG) & SMI_BUSY);
1110 }
1111 
1112 static int smi_wait_ready(struct mv643xx_eth_shared_private *msp)
1113 {
1114 	if (msp->err_interrupt == NO_IRQ) {
1115 		int i;
1116 
1117 		for (i = 0; !smi_is_done(msp); i++) {
1118 			if (i == 10)
1119 				return -ETIMEDOUT;
1120 			msleep(10);
1121 		}
1122 
1123 		return 0;
1124 	}
1125 
1126 	if (!smi_is_done(msp)) {
1127 		wait_event_timeout(msp->smi_busy_wait, smi_is_done(msp),
1128 				   msecs_to_jiffies(100));
1129 		if (!smi_is_done(msp))
1130 			return -ETIMEDOUT;
1131 	}
1132 
1133 	return 0;
1134 }
1135 
1136 static int smi_bus_read(struct mii_bus *bus, int addr, int reg)
1137 {
1138 	struct mv643xx_eth_shared_private *msp = bus->priv;
1139 	void __iomem *smi_reg = msp->base + SMI_REG;
1140 	int ret;
1141 
1142 	if (smi_wait_ready(msp)) {
1143 		pr_warn("SMI bus busy timeout\n");
1144 		return -ETIMEDOUT;
1145 	}
1146 
1147 	writel(SMI_OPCODE_READ | (reg << 21) | (addr << 16), smi_reg);
1148 
1149 	if (smi_wait_ready(msp)) {
1150 		pr_warn("SMI bus busy timeout\n");
1151 		return -ETIMEDOUT;
1152 	}
1153 
1154 	ret = readl(smi_reg);
1155 	if (!(ret & SMI_READ_VALID)) {
1156 		pr_warn("SMI bus read not valid\n");
1157 		return -ENODEV;
1158 	}
1159 
1160 	return ret & 0xffff;
1161 }
1162 
1163 static int smi_bus_write(struct mii_bus *bus, int addr, int reg, u16 val)
1164 {
1165 	struct mv643xx_eth_shared_private *msp = bus->priv;
1166 	void __iomem *smi_reg = msp->base + SMI_REG;
1167 
1168 	if (smi_wait_ready(msp)) {
1169 		pr_warn("SMI bus busy timeout\n");
1170 		return -ETIMEDOUT;
1171 	}
1172 
1173 	writel(SMI_OPCODE_WRITE | (reg << 21) |
1174 		(addr << 16) | (val & 0xffff), smi_reg);
1175 
1176 	if (smi_wait_ready(msp)) {
1177 		pr_warn("SMI bus busy timeout\n");
1178 		return -ETIMEDOUT;
1179 	}
1180 
1181 	return 0;
1182 }
1183 
1184 
1185 /* statistics ***************************************************************/
1186 static struct net_device_stats *mv643xx_eth_get_stats(struct net_device *dev)
1187 {
1188 	struct mv643xx_eth_private *mp = netdev_priv(dev);
1189 	struct net_device_stats *stats = &dev->stats;
1190 	unsigned long tx_packets = 0;
1191 	unsigned long tx_bytes = 0;
1192 	unsigned long tx_dropped = 0;
1193 	int i;
1194 
1195 	for (i = 0; i < mp->txq_count; i++) {
1196 		struct tx_queue *txq = mp->txq + i;
1197 
1198 		tx_packets += txq->tx_packets;
1199 		tx_bytes += txq->tx_bytes;
1200 		tx_dropped += txq->tx_dropped;
1201 	}
1202 
1203 	stats->tx_packets = tx_packets;
1204 	stats->tx_bytes = tx_bytes;
1205 	stats->tx_dropped = tx_dropped;
1206 
1207 	return stats;
1208 }
1209 
1210 static void mv643xx_eth_grab_lro_stats(struct mv643xx_eth_private *mp)
1211 {
1212 	u32 lro_aggregated = 0;
1213 	u32 lro_flushed = 0;
1214 	u32 lro_no_desc = 0;
1215 	int i;
1216 
1217 	for (i = 0; i < mp->rxq_count; i++) {
1218 		struct rx_queue *rxq = mp->rxq + i;
1219 
1220 		lro_aggregated += rxq->lro_mgr.stats.aggregated;
1221 		lro_flushed += rxq->lro_mgr.stats.flushed;
1222 		lro_no_desc += rxq->lro_mgr.stats.no_desc;
1223 	}
1224 
1225 	mp->lro_counters.lro_aggregated = lro_aggregated;
1226 	mp->lro_counters.lro_flushed = lro_flushed;
1227 	mp->lro_counters.lro_no_desc = lro_no_desc;
1228 }
1229 
1230 static inline u32 mib_read(struct mv643xx_eth_private *mp, int offset)
1231 {
1232 	return rdl(mp, MIB_COUNTERS(mp->port_num) + offset);
1233 }
1234 
1235 static void mib_counters_clear(struct mv643xx_eth_private *mp)
1236 {
1237 	int i;
1238 
1239 	for (i = 0; i < 0x80; i += 4)
1240 		mib_read(mp, i);
1241 
1242 	/* Clear non MIB hw counters also */
1243 	rdlp(mp, RX_DISCARD_FRAME_CNT);
1244 	rdlp(mp, RX_OVERRUN_FRAME_CNT);
1245 }
1246 
1247 static void mib_counters_update(struct mv643xx_eth_private *mp)
1248 {
1249 	struct mib_counters *p = &mp->mib_counters;
1250 
1251 	spin_lock_bh(&mp->mib_counters_lock);
1252 	p->good_octets_received += mib_read(mp, 0x00);
1253 	p->bad_octets_received += mib_read(mp, 0x08);
1254 	p->internal_mac_transmit_err += mib_read(mp, 0x0c);
1255 	p->good_frames_received += mib_read(mp, 0x10);
1256 	p->bad_frames_received += mib_read(mp, 0x14);
1257 	p->broadcast_frames_received += mib_read(mp, 0x18);
1258 	p->multicast_frames_received += mib_read(mp, 0x1c);
1259 	p->frames_64_octets += mib_read(mp, 0x20);
1260 	p->frames_65_to_127_octets += mib_read(mp, 0x24);
1261 	p->frames_128_to_255_octets += mib_read(mp, 0x28);
1262 	p->frames_256_to_511_octets += mib_read(mp, 0x2c);
1263 	p->frames_512_to_1023_octets += mib_read(mp, 0x30);
1264 	p->frames_1024_to_max_octets += mib_read(mp, 0x34);
1265 	p->good_octets_sent += mib_read(mp, 0x38);
1266 	p->good_frames_sent += mib_read(mp, 0x40);
1267 	p->excessive_collision += mib_read(mp, 0x44);
1268 	p->multicast_frames_sent += mib_read(mp, 0x48);
1269 	p->broadcast_frames_sent += mib_read(mp, 0x4c);
1270 	p->unrec_mac_control_received += mib_read(mp, 0x50);
1271 	p->fc_sent += mib_read(mp, 0x54);
1272 	p->good_fc_received += mib_read(mp, 0x58);
1273 	p->bad_fc_received += mib_read(mp, 0x5c);
1274 	p->undersize_received += mib_read(mp, 0x60);
1275 	p->fragments_received += mib_read(mp, 0x64);
1276 	p->oversize_received += mib_read(mp, 0x68);
1277 	p->jabber_received += mib_read(mp, 0x6c);
1278 	p->mac_receive_error += mib_read(mp, 0x70);
1279 	p->bad_crc_event += mib_read(mp, 0x74);
1280 	p->collision += mib_read(mp, 0x78);
1281 	p->late_collision += mib_read(mp, 0x7c);
1282 	/* Non MIB hardware counters */
1283 	p->rx_discard += rdlp(mp, RX_DISCARD_FRAME_CNT);
1284 	p->rx_overrun += rdlp(mp, RX_OVERRUN_FRAME_CNT);
1285 	spin_unlock_bh(&mp->mib_counters_lock);
1286 
1287 	mod_timer(&mp->mib_counters_timer, jiffies + 30 * HZ);
1288 }
1289 
1290 static void mib_counters_timer_wrapper(unsigned long _mp)
1291 {
1292 	struct mv643xx_eth_private *mp = (void *)_mp;
1293 
1294 	mib_counters_update(mp);
1295 }
1296 
1297 
1298 /* interrupt coalescing *****************************************************/
1299 /*
1300  * Hardware coalescing parameters are set in units of 64 t_clk
1301  * cycles.  I.e.:
1302  *
1303  *	coal_delay_in_usec = 64000000 * register_value / t_clk_rate
1304  *
1305  *	register_value = coal_delay_in_usec * t_clk_rate / 64000000
1306  *
1307  * In the ->set*() methods, we round the computed register value
1308  * to the nearest integer.
1309  */
1310 static unsigned int get_rx_coal(struct mv643xx_eth_private *mp)
1311 {
1312 	u32 val = rdlp(mp, SDMA_CONFIG);
1313 	u64 temp;
1314 
1315 	if (mp->shared->extended_rx_coal_limit)
1316 		temp = ((val & 0x02000000) >> 10) | ((val & 0x003fff80) >> 7);
1317 	else
1318 		temp = (val & 0x003fff00) >> 8;
1319 
1320 	temp *= 64000000;
1321 	do_div(temp, mp->t_clk);
1322 
1323 	return (unsigned int)temp;
1324 }
1325 
1326 static void set_rx_coal(struct mv643xx_eth_private *mp, unsigned int usec)
1327 {
1328 	u64 temp;
1329 	u32 val;
1330 
1331 	temp = (u64)usec * mp->t_clk;
1332 	temp += 31999999;
1333 	do_div(temp, 64000000);
1334 
1335 	val = rdlp(mp, SDMA_CONFIG);
1336 	if (mp->shared->extended_rx_coal_limit) {
1337 		if (temp > 0xffff)
1338 			temp = 0xffff;
1339 		val &= ~0x023fff80;
1340 		val |= (temp & 0x8000) << 10;
1341 		val |= (temp & 0x7fff) << 7;
1342 	} else {
1343 		if (temp > 0x3fff)
1344 			temp = 0x3fff;
1345 		val &= ~0x003fff00;
1346 		val |= (temp & 0x3fff) << 8;
1347 	}
1348 	wrlp(mp, SDMA_CONFIG, val);
1349 }
1350 
1351 static unsigned int get_tx_coal(struct mv643xx_eth_private *mp)
1352 {
1353 	u64 temp;
1354 
1355 	temp = (rdlp(mp, TX_FIFO_URGENT_THRESHOLD) & 0x3fff0) >> 4;
1356 	temp *= 64000000;
1357 	do_div(temp, mp->t_clk);
1358 
1359 	return (unsigned int)temp;
1360 }
1361 
1362 static void set_tx_coal(struct mv643xx_eth_private *mp, unsigned int usec)
1363 {
1364 	u64 temp;
1365 
1366 	temp = (u64)usec * mp->t_clk;
1367 	temp += 31999999;
1368 	do_div(temp, 64000000);
1369 
1370 	if (temp > 0x3fff)
1371 		temp = 0x3fff;
1372 
1373 	wrlp(mp, TX_FIFO_URGENT_THRESHOLD, temp << 4);
1374 }
1375 
1376 
1377 /* ethtool ******************************************************************/
1378 struct mv643xx_eth_stats {
1379 	char stat_string[ETH_GSTRING_LEN];
1380 	int sizeof_stat;
1381 	int netdev_off;
1382 	int mp_off;
1383 };
1384 
1385 #define SSTAT(m)						\
1386 	{ #m, FIELD_SIZEOF(struct net_device_stats, m),		\
1387 	  offsetof(struct net_device, stats.m), -1 }
1388 
1389 #define MIBSTAT(m)						\
1390 	{ #m, FIELD_SIZEOF(struct mib_counters, m),		\
1391 	  -1, offsetof(struct mv643xx_eth_private, mib_counters.m) }
1392 
1393 #define LROSTAT(m)						\
1394 	{ #m, FIELD_SIZEOF(struct lro_counters, m),		\
1395 	  -1, offsetof(struct mv643xx_eth_private, lro_counters.m) }
1396 
1397 static const struct mv643xx_eth_stats mv643xx_eth_stats[] = {
1398 	SSTAT(rx_packets),
1399 	SSTAT(tx_packets),
1400 	SSTAT(rx_bytes),
1401 	SSTAT(tx_bytes),
1402 	SSTAT(rx_errors),
1403 	SSTAT(tx_errors),
1404 	SSTAT(rx_dropped),
1405 	SSTAT(tx_dropped),
1406 	MIBSTAT(good_octets_received),
1407 	MIBSTAT(bad_octets_received),
1408 	MIBSTAT(internal_mac_transmit_err),
1409 	MIBSTAT(good_frames_received),
1410 	MIBSTAT(bad_frames_received),
1411 	MIBSTAT(broadcast_frames_received),
1412 	MIBSTAT(multicast_frames_received),
1413 	MIBSTAT(frames_64_octets),
1414 	MIBSTAT(frames_65_to_127_octets),
1415 	MIBSTAT(frames_128_to_255_octets),
1416 	MIBSTAT(frames_256_to_511_octets),
1417 	MIBSTAT(frames_512_to_1023_octets),
1418 	MIBSTAT(frames_1024_to_max_octets),
1419 	MIBSTAT(good_octets_sent),
1420 	MIBSTAT(good_frames_sent),
1421 	MIBSTAT(excessive_collision),
1422 	MIBSTAT(multicast_frames_sent),
1423 	MIBSTAT(broadcast_frames_sent),
1424 	MIBSTAT(unrec_mac_control_received),
1425 	MIBSTAT(fc_sent),
1426 	MIBSTAT(good_fc_received),
1427 	MIBSTAT(bad_fc_received),
1428 	MIBSTAT(undersize_received),
1429 	MIBSTAT(fragments_received),
1430 	MIBSTAT(oversize_received),
1431 	MIBSTAT(jabber_received),
1432 	MIBSTAT(mac_receive_error),
1433 	MIBSTAT(bad_crc_event),
1434 	MIBSTAT(collision),
1435 	MIBSTAT(late_collision),
1436 	MIBSTAT(rx_discard),
1437 	MIBSTAT(rx_overrun),
1438 	LROSTAT(lro_aggregated),
1439 	LROSTAT(lro_flushed),
1440 	LROSTAT(lro_no_desc),
1441 };
1442 
1443 static int
1444 mv643xx_eth_get_settings_phy(struct mv643xx_eth_private *mp,
1445 			     struct ethtool_cmd *cmd)
1446 {
1447 	int err;
1448 
1449 	err = phy_read_status(mp->phy);
1450 	if (err == 0)
1451 		err = phy_ethtool_gset(mp->phy, cmd);
1452 
1453 	/*
1454 	 * The MAC does not support 1000baseT_Half.
1455 	 */
1456 	cmd->supported &= ~SUPPORTED_1000baseT_Half;
1457 	cmd->advertising &= ~ADVERTISED_1000baseT_Half;
1458 
1459 	return err;
1460 }
1461 
1462 static int
1463 mv643xx_eth_get_settings_phyless(struct mv643xx_eth_private *mp,
1464 				 struct ethtool_cmd *cmd)
1465 {
1466 	u32 port_status;
1467 
1468 	port_status = rdlp(mp, PORT_STATUS);
1469 
1470 	cmd->supported = SUPPORTED_MII;
1471 	cmd->advertising = ADVERTISED_MII;
1472 	switch (port_status & PORT_SPEED_MASK) {
1473 	case PORT_SPEED_10:
1474 		ethtool_cmd_speed_set(cmd, SPEED_10);
1475 		break;
1476 	case PORT_SPEED_100:
1477 		ethtool_cmd_speed_set(cmd, SPEED_100);
1478 		break;
1479 	case PORT_SPEED_1000:
1480 		ethtool_cmd_speed_set(cmd, SPEED_1000);
1481 		break;
1482 	default:
1483 		cmd->speed = -1;
1484 		break;
1485 	}
1486 	cmd->duplex = (port_status & FULL_DUPLEX) ? DUPLEX_FULL : DUPLEX_HALF;
1487 	cmd->port = PORT_MII;
1488 	cmd->phy_address = 0;
1489 	cmd->transceiver = XCVR_INTERNAL;
1490 	cmd->autoneg = AUTONEG_DISABLE;
1491 	cmd->maxtxpkt = 1;
1492 	cmd->maxrxpkt = 1;
1493 
1494 	return 0;
1495 }
1496 
1497 static int
1498 mv643xx_eth_get_settings(struct net_device *dev, struct ethtool_cmd *cmd)
1499 {
1500 	struct mv643xx_eth_private *mp = netdev_priv(dev);
1501 
1502 	if (mp->phy != NULL)
1503 		return mv643xx_eth_get_settings_phy(mp, cmd);
1504 	else
1505 		return mv643xx_eth_get_settings_phyless(mp, cmd);
1506 }
1507 
1508 static int
1509 mv643xx_eth_set_settings(struct net_device *dev, struct ethtool_cmd *cmd)
1510 {
1511 	struct mv643xx_eth_private *mp = netdev_priv(dev);
1512 
1513 	if (mp->phy == NULL)
1514 		return -EINVAL;
1515 
1516 	/*
1517 	 * The MAC does not support 1000baseT_Half.
1518 	 */
1519 	cmd->advertising &= ~ADVERTISED_1000baseT_Half;
1520 
1521 	return phy_ethtool_sset(mp->phy, cmd);
1522 }
1523 
1524 static void mv643xx_eth_get_drvinfo(struct net_device *dev,
1525 				    struct ethtool_drvinfo *drvinfo)
1526 {
1527 	strlcpy(drvinfo->driver, mv643xx_eth_driver_name,
1528 		sizeof(drvinfo->driver));
1529 	strlcpy(drvinfo->version, mv643xx_eth_driver_version,
1530 		sizeof(drvinfo->version));
1531 	strlcpy(drvinfo->fw_version, "N/A", sizeof(drvinfo->fw_version));
1532 	strlcpy(drvinfo->bus_info, "platform", sizeof(drvinfo->bus_info));
1533 	drvinfo->n_stats = ARRAY_SIZE(mv643xx_eth_stats);
1534 }
1535 
1536 static int mv643xx_eth_nway_reset(struct net_device *dev)
1537 {
1538 	struct mv643xx_eth_private *mp = netdev_priv(dev);
1539 
1540 	if (mp->phy == NULL)
1541 		return -EINVAL;
1542 
1543 	return genphy_restart_aneg(mp->phy);
1544 }
1545 
1546 static int
1547 mv643xx_eth_get_coalesce(struct net_device *dev, struct ethtool_coalesce *ec)
1548 {
1549 	struct mv643xx_eth_private *mp = netdev_priv(dev);
1550 
1551 	ec->rx_coalesce_usecs = get_rx_coal(mp);
1552 	ec->tx_coalesce_usecs = get_tx_coal(mp);
1553 
1554 	return 0;
1555 }
1556 
1557 static int
1558 mv643xx_eth_set_coalesce(struct net_device *dev, struct ethtool_coalesce *ec)
1559 {
1560 	struct mv643xx_eth_private *mp = netdev_priv(dev);
1561 
1562 	set_rx_coal(mp, ec->rx_coalesce_usecs);
1563 	set_tx_coal(mp, ec->tx_coalesce_usecs);
1564 
1565 	return 0;
1566 }
1567 
1568 static void
1569 mv643xx_eth_get_ringparam(struct net_device *dev, struct ethtool_ringparam *er)
1570 {
1571 	struct mv643xx_eth_private *mp = netdev_priv(dev);
1572 
1573 	er->rx_max_pending = 4096;
1574 	er->tx_max_pending = 4096;
1575 
1576 	er->rx_pending = mp->rx_ring_size;
1577 	er->tx_pending = mp->tx_ring_size;
1578 }
1579 
1580 static int
1581 mv643xx_eth_set_ringparam(struct net_device *dev, struct ethtool_ringparam *er)
1582 {
1583 	struct mv643xx_eth_private *mp = netdev_priv(dev);
1584 
1585 	if (er->rx_mini_pending || er->rx_jumbo_pending)
1586 		return -EINVAL;
1587 
1588 	mp->rx_ring_size = er->rx_pending < 4096 ? er->rx_pending : 4096;
1589 	mp->tx_ring_size = er->tx_pending < 4096 ? er->tx_pending : 4096;
1590 
1591 	if (netif_running(dev)) {
1592 		mv643xx_eth_stop(dev);
1593 		if (mv643xx_eth_open(dev)) {
1594 			netdev_err(dev,
1595 				   "fatal error on re-opening device after ring param change\n");
1596 			return -ENOMEM;
1597 		}
1598 	}
1599 
1600 	return 0;
1601 }
1602 
1603 
1604 static int
1605 mv643xx_eth_set_features(struct net_device *dev, netdev_features_t features)
1606 {
1607 	struct mv643xx_eth_private *mp = netdev_priv(dev);
1608 	bool rx_csum = features & NETIF_F_RXCSUM;
1609 
1610 	wrlp(mp, PORT_CONFIG, rx_csum ? 0x02000000 : 0x00000000);
1611 
1612 	return 0;
1613 }
1614 
1615 static void mv643xx_eth_get_strings(struct net_device *dev,
1616 				    uint32_t stringset, uint8_t *data)
1617 {
1618 	int i;
1619 
1620 	if (stringset == ETH_SS_STATS) {
1621 		for (i = 0; i < ARRAY_SIZE(mv643xx_eth_stats); i++) {
1622 			memcpy(data + i * ETH_GSTRING_LEN,
1623 				mv643xx_eth_stats[i].stat_string,
1624 				ETH_GSTRING_LEN);
1625 		}
1626 	}
1627 }
1628 
1629 static void mv643xx_eth_get_ethtool_stats(struct net_device *dev,
1630 					  struct ethtool_stats *stats,
1631 					  uint64_t *data)
1632 {
1633 	struct mv643xx_eth_private *mp = netdev_priv(dev);
1634 	int i;
1635 
1636 	mv643xx_eth_get_stats(dev);
1637 	mib_counters_update(mp);
1638 	mv643xx_eth_grab_lro_stats(mp);
1639 
1640 	for (i = 0; i < ARRAY_SIZE(mv643xx_eth_stats); i++) {
1641 		const struct mv643xx_eth_stats *stat;
1642 		void *p;
1643 
1644 		stat = mv643xx_eth_stats + i;
1645 
1646 		if (stat->netdev_off >= 0)
1647 			p = ((void *)mp->dev) + stat->netdev_off;
1648 		else
1649 			p = ((void *)mp) + stat->mp_off;
1650 
1651 		data[i] = (stat->sizeof_stat == 8) ?
1652 				*(uint64_t *)p : *(uint32_t *)p;
1653 	}
1654 }
1655 
1656 static int mv643xx_eth_get_sset_count(struct net_device *dev, int sset)
1657 {
1658 	if (sset == ETH_SS_STATS)
1659 		return ARRAY_SIZE(mv643xx_eth_stats);
1660 
1661 	return -EOPNOTSUPP;
1662 }
1663 
1664 static const struct ethtool_ops mv643xx_eth_ethtool_ops = {
1665 	.get_settings		= mv643xx_eth_get_settings,
1666 	.set_settings		= mv643xx_eth_set_settings,
1667 	.get_drvinfo		= mv643xx_eth_get_drvinfo,
1668 	.nway_reset		= mv643xx_eth_nway_reset,
1669 	.get_link		= ethtool_op_get_link,
1670 	.get_coalesce		= mv643xx_eth_get_coalesce,
1671 	.set_coalesce		= mv643xx_eth_set_coalesce,
1672 	.get_ringparam		= mv643xx_eth_get_ringparam,
1673 	.set_ringparam		= mv643xx_eth_set_ringparam,
1674 	.get_strings		= mv643xx_eth_get_strings,
1675 	.get_ethtool_stats	= mv643xx_eth_get_ethtool_stats,
1676 	.get_sset_count		= mv643xx_eth_get_sset_count,
1677 	.get_ts_info		= ethtool_op_get_ts_info,
1678 };
1679 
1680 
1681 /* address handling *********************************************************/
1682 static void uc_addr_get(struct mv643xx_eth_private *mp, unsigned char *addr)
1683 {
1684 	unsigned int mac_h = rdlp(mp, MAC_ADDR_HIGH);
1685 	unsigned int mac_l = rdlp(mp, MAC_ADDR_LOW);
1686 
1687 	addr[0] = (mac_h >> 24) & 0xff;
1688 	addr[1] = (mac_h >> 16) & 0xff;
1689 	addr[2] = (mac_h >> 8) & 0xff;
1690 	addr[3] = mac_h & 0xff;
1691 	addr[4] = (mac_l >> 8) & 0xff;
1692 	addr[5] = mac_l & 0xff;
1693 }
1694 
1695 static void uc_addr_set(struct mv643xx_eth_private *mp, unsigned char *addr)
1696 {
1697 	wrlp(mp, MAC_ADDR_HIGH,
1698 		(addr[0] << 24) | (addr[1] << 16) | (addr[2] << 8) | addr[3]);
1699 	wrlp(mp, MAC_ADDR_LOW, (addr[4] << 8) | addr[5]);
1700 }
1701 
1702 static u32 uc_addr_filter_mask(struct net_device *dev)
1703 {
1704 	struct netdev_hw_addr *ha;
1705 	u32 nibbles;
1706 
1707 	if (dev->flags & IFF_PROMISC)
1708 		return 0;
1709 
1710 	nibbles = 1 << (dev->dev_addr[5] & 0x0f);
1711 	netdev_for_each_uc_addr(ha, dev) {
1712 		if (memcmp(dev->dev_addr, ha->addr, 5))
1713 			return 0;
1714 		if ((dev->dev_addr[5] ^ ha->addr[5]) & 0xf0)
1715 			return 0;
1716 
1717 		nibbles |= 1 << (ha->addr[5] & 0x0f);
1718 	}
1719 
1720 	return nibbles;
1721 }
1722 
1723 static void mv643xx_eth_program_unicast_filter(struct net_device *dev)
1724 {
1725 	struct mv643xx_eth_private *mp = netdev_priv(dev);
1726 	u32 port_config;
1727 	u32 nibbles;
1728 	int i;
1729 
1730 	uc_addr_set(mp, dev->dev_addr);
1731 
1732 	port_config = rdlp(mp, PORT_CONFIG) & ~UNICAST_PROMISCUOUS_MODE;
1733 
1734 	nibbles = uc_addr_filter_mask(dev);
1735 	if (!nibbles) {
1736 		port_config |= UNICAST_PROMISCUOUS_MODE;
1737 		nibbles = 0xffff;
1738 	}
1739 
1740 	for (i = 0; i < 16; i += 4) {
1741 		int off = UNICAST_TABLE(mp->port_num) + i;
1742 		u32 v;
1743 
1744 		v = 0;
1745 		if (nibbles & 1)
1746 			v |= 0x00000001;
1747 		if (nibbles & 2)
1748 			v |= 0x00000100;
1749 		if (nibbles & 4)
1750 			v |= 0x00010000;
1751 		if (nibbles & 8)
1752 			v |= 0x01000000;
1753 		nibbles >>= 4;
1754 
1755 		wrl(mp, off, v);
1756 	}
1757 
1758 	wrlp(mp, PORT_CONFIG, port_config);
1759 }
1760 
1761 static int addr_crc(unsigned char *addr)
1762 {
1763 	int crc = 0;
1764 	int i;
1765 
1766 	for (i = 0; i < 6; i++) {
1767 		int j;
1768 
1769 		crc = (crc ^ addr[i]) << 8;
1770 		for (j = 7; j >= 0; j--) {
1771 			if (crc & (0x100 << j))
1772 				crc ^= 0x107 << j;
1773 		}
1774 	}
1775 
1776 	return crc;
1777 }
1778 
1779 static void mv643xx_eth_program_multicast_filter(struct net_device *dev)
1780 {
1781 	struct mv643xx_eth_private *mp = netdev_priv(dev);
1782 	u32 *mc_spec;
1783 	u32 *mc_other;
1784 	struct netdev_hw_addr *ha;
1785 	int i;
1786 
1787 	if (dev->flags & (IFF_PROMISC | IFF_ALLMULTI)) {
1788 		int port_num;
1789 		u32 accept;
1790 
1791 oom:
1792 		port_num = mp->port_num;
1793 		accept = 0x01010101;
1794 		for (i = 0; i < 0x100; i += 4) {
1795 			wrl(mp, SPECIAL_MCAST_TABLE(port_num) + i, accept);
1796 			wrl(mp, OTHER_MCAST_TABLE(port_num) + i, accept);
1797 		}
1798 		return;
1799 	}
1800 
1801 	mc_spec = kmalloc(0x200, GFP_ATOMIC);
1802 	if (mc_spec == NULL)
1803 		goto oom;
1804 	mc_other = mc_spec + (0x100 >> 2);
1805 
1806 	memset(mc_spec, 0, 0x100);
1807 	memset(mc_other, 0, 0x100);
1808 
1809 	netdev_for_each_mc_addr(ha, dev) {
1810 		u8 *a = ha->addr;
1811 		u32 *table;
1812 		int entry;
1813 
1814 		if (memcmp(a, "\x01\x00\x5e\x00\x00", 5) == 0) {
1815 			table = mc_spec;
1816 			entry = a[5];
1817 		} else {
1818 			table = mc_other;
1819 			entry = addr_crc(a);
1820 		}
1821 
1822 		table[entry >> 2] |= 1 << (8 * (entry & 3));
1823 	}
1824 
1825 	for (i = 0; i < 0x100; i += 4) {
1826 		wrl(mp, SPECIAL_MCAST_TABLE(mp->port_num) + i, mc_spec[i >> 2]);
1827 		wrl(mp, OTHER_MCAST_TABLE(mp->port_num) + i, mc_other[i >> 2]);
1828 	}
1829 
1830 	kfree(mc_spec);
1831 }
1832 
1833 static void mv643xx_eth_set_rx_mode(struct net_device *dev)
1834 {
1835 	mv643xx_eth_program_unicast_filter(dev);
1836 	mv643xx_eth_program_multicast_filter(dev);
1837 }
1838 
1839 static int mv643xx_eth_set_mac_address(struct net_device *dev, void *addr)
1840 {
1841 	struct sockaddr *sa = addr;
1842 
1843 	if (!is_valid_ether_addr(sa->sa_data))
1844 		return -EADDRNOTAVAIL;
1845 
1846 	memcpy(dev->dev_addr, sa->sa_data, ETH_ALEN);
1847 
1848 	netif_addr_lock_bh(dev);
1849 	mv643xx_eth_program_unicast_filter(dev);
1850 	netif_addr_unlock_bh(dev);
1851 
1852 	return 0;
1853 }
1854 
1855 
1856 /* rx/tx queue initialisation ***********************************************/
1857 static int rxq_init(struct mv643xx_eth_private *mp, int index)
1858 {
1859 	struct rx_queue *rxq = mp->rxq + index;
1860 	struct rx_desc *rx_desc;
1861 	int size;
1862 	int i;
1863 
1864 	rxq->index = index;
1865 
1866 	rxq->rx_ring_size = mp->rx_ring_size;
1867 
1868 	rxq->rx_desc_count = 0;
1869 	rxq->rx_curr_desc = 0;
1870 	rxq->rx_used_desc = 0;
1871 
1872 	size = rxq->rx_ring_size * sizeof(struct rx_desc);
1873 
1874 	if (index == 0 && size <= mp->rx_desc_sram_size) {
1875 		rxq->rx_desc_area = ioremap(mp->rx_desc_sram_addr,
1876 						mp->rx_desc_sram_size);
1877 		rxq->rx_desc_dma = mp->rx_desc_sram_addr;
1878 	} else {
1879 		rxq->rx_desc_area = dma_alloc_coherent(mp->dev->dev.parent,
1880 						       size, &rxq->rx_desc_dma,
1881 						       GFP_KERNEL);
1882 	}
1883 
1884 	if (rxq->rx_desc_area == NULL) {
1885 		netdev_err(mp->dev,
1886 			   "can't allocate rx ring (%d bytes)\n", size);
1887 		goto out;
1888 	}
1889 	memset(rxq->rx_desc_area, 0, size);
1890 
1891 	rxq->rx_desc_area_size = size;
1892 	rxq->rx_skb = kmalloc(rxq->rx_ring_size * sizeof(*rxq->rx_skb),
1893 								GFP_KERNEL);
1894 	if (rxq->rx_skb == NULL) {
1895 		netdev_err(mp->dev, "can't allocate rx skb ring\n");
1896 		goto out_free;
1897 	}
1898 
1899 	rx_desc = rxq->rx_desc_area;
1900 	for (i = 0; i < rxq->rx_ring_size; i++) {
1901 		int nexti;
1902 
1903 		nexti = i + 1;
1904 		if (nexti == rxq->rx_ring_size)
1905 			nexti = 0;
1906 
1907 		rx_desc[i].next_desc_ptr = rxq->rx_desc_dma +
1908 					nexti * sizeof(struct rx_desc);
1909 	}
1910 
1911 	rxq->lro_mgr.dev = mp->dev;
1912 	memset(&rxq->lro_mgr.stats, 0, sizeof(rxq->lro_mgr.stats));
1913 	rxq->lro_mgr.features = LRO_F_NAPI;
1914 	rxq->lro_mgr.ip_summed = CHECKSUM_UNNECESSARY;
1915 	rxq->lro_mgr.ip_summed_aggr = CHECKSUM_UNNECESSARY;
1916 	rxq->lro_mgr.max_desc = ARRAY_SIZE(rxq->lro_arr);
1917 	rxq->lro_mgr.max_aggr = 32;
1918 	rxq->lro_mgr.frag_align_pad = 0;
1919 	rxq->lro_mgr.lro_arr = rxq->lro_arr;
1920 	rxq->lro_mgr.get_skb_header = mv643xx_get_skb_header;
1921 
1922 	memset(&rxq->lro_arr, 0, sizeof(rxq->lro_arr));
1923 
1924 	return 0;
1925 
1926 
1927 out_free:
1928 	if (index == 0 && size <= mp->rx_desc_sram_size)
1929 		iounmap(rxq->rx_desc_area);
1930 	else
1931 		dma_free_coherent(mp->dev->dev.parent, size,
1932 				  rxq->rx_desc_area,
1933 				  rxq->rx_desc_dma);
1934 
1935 out:
1936 	return -ENOMEM;
1937 }
1938 
1939 static void rxq_deinit(struct rx_queue *rxq)
1940 {
1941 	struct mv643xx_eth_private *mp = rxq_to_mp(rxq);
1942 	int i;
1943 
1944 	rxq_disable(rxq);
1945 
1946 	for (i = 0; i < rxq->rx_ring_size; i++) {
1947 		if (rxq->rx_skb[i]) {
1948 			dev_kfree_skb(rxq->rx_skb[i]);
1949 			rxq->rx_desc_count--;
1950 		}
1951 	}
1952 
1953 	if (rxq->rx_desc_count) {
1954 		netdev_err(mp->dev, "error freeing rx ring -- %d skbs stuck\n",
1955 			   rxq->rx_desc_count);
1956 	}
1957 
1958 	if (rxq->index == 0 &&
1959 	    rxq->rx_desc_area_size <= mp->rx_desc_sram_size)
1960 		iounmap(rxq->rx_desc_area);
1961 	else
1962 		dma_free_coherent(mp->dev->dev.parent, rxq->rx_desc_area_size,
1963 				  rxq->rx_desc_area, rxq->rx_desc_dma);
1964 
1965 	kfree(rxq->rx_skb);
1966 }
1967 
1968 static int txq_init(struct mv643xx_eth_private *mp, int index)
1969 {
1970 	struct tx_queue *txq = mp->txq + index;
1971 	struct tx_desc *tx_desc;
1972 	int size;
1973 	int i;
1974 
1975 	txq->index = index;
1976 
1977 	txq->tx_ring_size = mp->tx_ring_size;
1978 
1979 	txq->tx_desc_count = 0;
1980 	txq->tx_curr_desc = 0;
1981 	txq->tx_used_desc = 0;
1982 
1983 	size = txq->tx_ring_size * sizeof(struct tx_desc);
1984 
1985 	if (index == 0 && size <= mp->tx_desc_sram_size) {
1986 		txq->tx_desc_area = ioremap(mp->tx_desc_sram_addr,
1987 						mp->tx_desc_sram_size);
1988 		txq->tx_desc_dma = mp->tx_desc_sram_addr;
1989 	} else {
1990 		txq->tx_desc_area = dma_alloc_coherent(mp->dev->dev.parent,
1991 						       size, &txq->tx_desc_dma,
1992 						       GFP_KERNEL);
1993 	}
1994 
1995 	if (txq->tx_desc_area == NULL) {
1996 		netdev_err(mp->dev,
1997 			   "can't allocate tx ring (%d bytes)\n", size);
1998 		return -ENOMEM;
1999 	}
2000 	memset(txq->tx_desc_area, 0, size);
2001 
2002 	txq->tx_desc_area_size = size;
2003 
2004 	tx_desc = txq->tx_desc_area;
2005 	for (i = 0; i < txq->tx_ring_size; i++) {
2006 		struct tx_desc *txd = tx_desc + i;
2007 		int nexti;
2008 
2009 		nexti = i + 1;
2010 		if (nexti == txq->tx_ring_size)
2011 			nexti = 0;
2012 
2013 		txd->cmd_sts = 0;
2014 		txd->next_desc_ptr = txq->tx_desc_dma +
2015 					nexti * sizeof(struct tx_desc);
2016 	}
2017 
2018 	skb_queue_head_init(&txq->tx_skb);
2019 
2020 	return 0;
2021 }
2022 
2023 static void txq_deinit(struct tx_queue *txq)
2024 {
2025 	struct mv643xx_eth_private *mp = txq_to_mp(txq);
2026 
2027 	txq_disable(txq);
2028 	txq_reclaim(txq, txq->tx_ring_size, 1);
2029 
2030 	BUG_ON(txq->tx_used_desc != txq->tx_curr_desc);
2031 
2032 	if (txq->index == 0 &&
2033 	    txq->tx_desc_area_size <= mp->tx_desc_sram_size)
2034 		iounmap(txq->tx_desc_area);
2035 	else
2036 		dma_free_coherent(mp->dev->dev.parent, txq->tx_desc_area_size,
2037 				  txq->tx_desc_area, txq->tx_desc_dma);
2038 }
2039 
2040 
2041 /* netdev ops and related ***************************************************/
2042 static int mv643xx_eth_collect_events(struct mv643xx_eth_private *mp)
2043 {
2044 	u32 int_cause;
2045 	u32 int_cause_ext;
2046 
2047 	int_cause = rdlp(mp, INT_CAUSE) & mp->int_mask;
2048 	if (int_cause == 0)
2049 		return 0;
2050 
2051 	int_cause_ext = 0;
2052 	if (int_cause & INT_EXT) {
2053 		int_cause &= ~INT_EXT;
2054 		int_cause_ext = rdlp(mp, INT_CAUSE_EXT);
2055 	}
2056 
2057 	if (int_cause) {
2058 		wrlp(mp, INT_CAUSE, ~int_cause);
2059 		mp->work_tx_end |= ((int_cause & INT_TX_END) >> 19) &
2060 				~(rdlp(mp, TXQ_COMMAND) & 0xff);
2061 		mp->work_rx |= (int_cause & INT_RX) >> 2;
2062 	}
2063 
2064 	int_cause_ext &= INT_EXT_LINK_PHY | INT_EXT_TX;
2065 	if (int_cause_ext) {
2066 		wrlp(mp, INT_CAUSE_EXT, ~int_cause_ext);
2067 		if (int_cause_ext & INT_EXT_LINK_PHY)
2068 			mp->work_link = 1;
2069 		mp->work_tx |= int_cause_ext & INT_EXT_TX;
2070 	}
2071 
2072 	return 1;
2073 }
2074 
2075 static irqreturn_t mv643xx_eth_irq(int irq, void *dev_id)
2076 {
2077 	struct net_device *dev = (struct net_device *)dev_id;
2078 	struct mv643xx_eth_private *mp = netdev_priv(dev);
2079 
2080 	if (unlikely(!mv643xx_eth_collect_events(mp)))
2081 		return IRQ_NONE;
2082 
2083 	wrlp(mp, INT_MASK, 0);
2084 	napi_schedule(&mp->napi);
2085 
2086 	return IRQ_HANDLED;
2087 }
2088 
2089 static void handle_link_event(struct mv643xx_eth_private *mp)
2090 {
2091 	struct net_device *dev = mp->dev;
2092 	u32 port_status;
2093 	int speed;
2094 	int duplex;
2095 	int fc;
2096 
2097 	port_status = rdlp(mp, PORT_STATUS);
2098 	if (!(port_status & LINK_UP)) {
2099 		if (netif_carrier_ok(dev)) {
2100 			int i;
2101 
2102 			netdev_info(dev, "link down\n");
2103 
2104 			netif_carrier_off(dev);
2105 
2106 			for (i = 0; i < mp->txq_count; i++) {
2107 				struct tx_queue *txq = mp->txq + i;
2108 
2109 				txq_reclaim(txq, txq->tx_ring_size, 1);
2110 				txq_reset_hw_ptr(txq);
2111 			}
2112 		}
2113 		return;
2114 	}
2115 
2116 	switch (port_status & PORT_SPEED_MASK) {
2117 	case PORT_SPEED_10:
2118 		speed = 10;
2119 		break;
2120 	case PORT_SPEED_100:
2121 		speed = 100;
2122 		break;
2123 	case PORT_SPEED_1000:
2124 		speed = 1000;
2125 		break;
2126 	default:
2127 		speed = -1;
2128 		break;
2129 	}
2130 	duplex = (port_status & FULL_DUPLEX) ? 1 : 0;
2131 	fc = (port_status & FLOW_CONTROL_ENABLED) ? 1 : 0;
2132 
2133 	netdev_info(dev, "link up, %d Mb/s, %s duplex, flow control %sabled\n",
2134 		    speed, duplex ? "full" : "half", fc ? "en" : "dis");
2135 
2136 	if (!netif_carrier_ok(dev))
2137 		netif_carrier_on(dev);
2138 }
2139 
2140 static int mv643xx_eth_poll(struct napi_struct *napi, int budget)
2141 {
2142 	struct mv643xx_eth_private *mp;
2143 	int work_done;
2144 
2145 	mp = container_of(napi, struct mv643xx_eth_private, napi);
2146 
2147 	if (unlikely(mp->oom)) {
2148 		mp->oom = 0;
2149 		del_timer(&mp->rx_oom);
2150 	}
2151 
2152 	work_done = 0;
2153 	while (work_done < budget) {
2154 		u8 queue_mask;
2155 		int queue;
2156 		int work_tbd;
2157 
2158 		if (mp->work_link) {
2159 			mp->work_link = 0;
2160 			handle_link_event(mp);
2161 			work_done++;
2162 			continue;
2163 		}
2164 
2165 		queue_mask = mp->work_tx | mp->work_tx_end | mp->work_rx;
2166 		if (likely(!mp->oom))
2167 			queue_mask |= mp->work_rx_refill;
2168 
2169 		if (!queue_mask) {
2170 			if (mv643xx_eth_collect_events(mp))
2171 				continue;
2172 			break;
2173 		}
2174 
2175 		queue = fls(queue_mask) - 1;
2176 		queue_mask = 1 << queue;
2177 
2178 		work_tbd = budget - work_done;
2179 		if (work_tbd > 16)
2180 			work_tbd = 16;
2181 
2182 		if (mp->work_tx_end & queue_mask) {
2183 			txq_kick(mp->txq + queue);
2184 		} else if (mp->work_tx & queue_mask) {
2185 			work_done += txq_reclaim(mp->txq + queue, work_tbd, 0);
2186 			txq_maybe_wake(mp->txq + queue);
2187 		} else if (mp->work_rx & queue_mask) {
2188 			work_done += rxq_process(mp->rxq + queue, work_tbd);
2189 		} else if (!mp->oom && (mp->work_rx_refill & queue_mask)) {
2190 			work_done += rxq_refill(mp->rxq + queue, work_tbd);
2191 		} else {
2192 			BUG();
2193 		}
2194 	}
2195 
2196 	if (work_done < budget) {
2197 		if (mp->oom)
2198 			mod_timer(&mp->rx_oom, jiffies + (HZ / 10));
2199 		napi_complete(napi);
2200 		wrlp(mp, INT_MASK, mp->int_mask);
2201 	}
2202 
2203 	return work_done;
2204 }
2205 
2206 static inline void oom_timer_wrapper(unsigned long data)
2207 {
2208 	struct mv643xx_eth_private *mp = (void *)data;
2209 
2210 	napi_schedule(&mp->napi);
2211 }
2212 
2213 static void phy_reset(struct mv643xx_eth_private *mp)
2214 {
2215 	int data;
2216 
2217 	data = phy_read(mp->phy, MII_BMCR);
2218 	if (data < 0)
2219 		return;
2220 
2221 	data |= BMCR_RESET;
2222 	if (phy_write(mp->phy, MII_BMCR, data) < 0)
2223 		return;
2224 
2225 	do {
2226 		data = phy_read(mp->phy, MII_BMCR);
2227 	} while (data >= 0 && data & BMCR_RESET);
2228 }
2229 
2230 static void port_start(struct mv643xx_eth_private *mp)
2231 {
2232 	u32 pscr;
2233 	int i;
2234 
2235 	/*
2236 	 * Perform PHY reset, if there is a PHY.
2237 	 */
2238 	if (mp->phy != NULL) {
2239 		struct ethtool_cmd cmd;
2240 
2241 		mv643xx_eth_get_settings(mp->dev, &cmd);
2242 		phy_reset(mp);
2243 		mv643xx_eth_set_settings(mp->dev, &cmd);
2244 	}
2245 
2246 	/*
2247 	 * Configure basic link parameters.
2248 	 */
2249 	pscr = rdlp(mp, PORT_SERIAL_CONTROL);
2250 
2251 	pscr |= SERIAL_PORT_ENABLE;
2252 	wrlp(mp, PORT_SERIAL_CONTROL, pscr);
2253 
2254 	pscr |= DO_NOT_FORCE_LINK_FAIL;
2255 	if (mp->phy == NULL)
2256 		pscr |= FORCE_LINK_PASS;
2257 	wrlp(mp, PORT_SERIAL_CONTROL, pscr);
2258 
2259 	/*
2260 	 * Configure TX path and queues.
2261 	 */
2262 	tx_set_rate(mp, 1000000000, 16777216);
2263 	for (i = 0; i < mp->txq_count; i++) {
2264 		struct tx_queue *txq = mp->txq + i;
2265 
2266 		txq_reset_hw_ptr(txq);
2267 		txq_set_rate(txq, 1000000000, 16777216);
2268 		txq_set_fixed_prio_mode(txq);
2269 	}
2270 
2271 	/*
2272 	 * Receive all unmatched unicast, TCP, UDP, BPDU and broadcast
2273 	 * frames to RX queue #0, and include the pseudo-header when
2274 	 * calculating receive checksums.
2275 	 */
2276 	mv643xx_eth_set_features(mp->dev, mp->dev->features);
2277 
2278 	/*
2279 	 * Treat BPDUs as normal multicasts, and disable partition mode.
2280 	 */
2281 	wrlp(mp, PORT_CONFIG_EXT, 0x00000000);
2282 
2283 	/*
2284 	 * Add configured unicast addresses to address filter table.
2285 	 */
2286 	mv643xx_eth_program_unicast_filter(mp->dev);
2287 
2288 	/*
2289 	 * Enable the receive queues.
2290 	 */
2291 	for (i = 0; i < mp->rxq_count; i++) {
2292 		struct rx_queue *rxq = mp->rxq + i;
2293 		u32 addr;
2294 
2295 		addr = (u32)rxq->rx_desc_dma;
2296 		addr += rxq->rx_curr_desc * sizeof(struct rx_desc);
2297 		wrlp(mp, RXQ_CURRENT_DESC_PTR(i), addr);
2298 
2299 		rxq_enable(rxq);
2300 	}
2301 }
2302 
2303 static void mv643xx_eth_recalc_skb_size(struct mv643xx_eth_private *mp)
2304 {
2305 	int skb_size;
2306 
2307 	/*
2308 	 * Reserve 2+14 bytes for an ethernet header (the hardware
2309 	 * automatically prepends 2 bytes of dummy data to each
2310 	 * received packet), 16 bytes for up to four VLAN tags, and
2311 	 * 4 bytes for the trailing FCS -- 36 bytes total.
2312 	 */
2313 	skb_size = mp->dev->mtu + 36;
2314 
2315 	/*
2316 	 * Make sure that the skb size is a multiple of 8 bytes, as
2317 	 * the lower three bits of the receive descriptor's buffer
2318 	 * size field are ignored by the hardware.
2319 	 */
2320 	mp->skb_size = (skb_size + 7) & ~7;
2321 
2322 	/*
2323 	 * If NET_SKB_PAD is smaller than a cache line,
2324 	 * netdev_alloc_skb() will cause skb->data to be misaligned
2325 	 * to a cache line boundary.  If this is the case, include
2326 	 * some extra space to allow re-aligning the data area.
2327 	 */
2328 	mp->skb_size += SKB_DMA_REALIGN;
2329 }
2330 
2331 static int mv643xx_eth_open(struct net_device *dev)
2332 {
2333 	struct mv643xx_eth_private *mp = netdev_priv(dev);
2334 	int err;
2335 	int i;
2336 
2337 	wrlp(mp, INT_CAUSE, 0);
2338 	wrlp(mp, INT_CAUSE_EXT, 0);
2339 	rdlp(mp, INT_CAUSE_EXT);
2340 
2341 	err = request_irq(dev->irq, mv643xx_eth_irq,
2342 			  IRQF_SHARED, dev->name, dev);
2343 	if (err) {
2344 		netdev_err(dev, "can't assign irq\n");
2345 		return -EAGAIN;
2346 	}
2347 
2348 	mv643xx_eth_recalc_skb_size(mp);
2349 
2350 	napi_enable(&mp->napi);
2351 
2352 	skb_queue_head_init(&mp->rx_recycle);
2353 
2354 	mp->int_mask = INT_EXT;
2355 
2356 	for (i = 0; i < mp->rxq_count; i++) {
2357 		err = rxq_init(mp, i);
2358 		if (err) {
2359 			while (--i >= 0)
2360 				rxq_deinit(mp->rxq + i);
2361 			goto out;
2362 		}
2363 
2364 		rxq_refill(mp->rxq + i, INT_MAX);
2365 		mp->int_mask |= INT_RX_0 << i;
2366 	}
2367 
2368 	if (mp->oom) {
2369 		mp->rx_oom.expires = jiffies + (HZ / 10);
2370 		add_timer(&mp->rx_oom);
2371 	}
2372 
2373 	for (i = 0; i < mp->txq_count; i++) {
2374 		err = txq_init(mp, i);
2375 		if (err) {
2376 			while (--i >= 0)
2377 				txq_deinit(mp->txq + i);
2378 			goto out_free;
2379 		}
2380 		mp->int_mask |= INT_TX_END_0 << i;
2381 	}
2382 
2383 	port_start(mp);
2384 
2385 	wrlp(mp, INT_MASK_EXT, INT_EXT_LINK_PHY | INT_EXT_TX);
2386 	wrlp(mp, INT_MASK, mp->int_mask);
2387 
2388 	return 0;
2389 
2390 
2391 out_free:
2392 	for (i = 0; i < mp->rxq_count; i++)
2393 		rxq_deinit(mp->rxq + i);
2394 out:
2395 	free_irq(dev->irq, dev);
2396 
2397 	return err;
2398 }
2399 
2400 static void port_reset(struct mv643xx_eth_private *mp)
2401 {
2402 	unsigned int data;
2403 	int i;
2404 
2405 	for (i = 0; i < mp->rxq_count; i++)
2406 		rxq_disable(mp->rxq + i);
2407 	for (i = 0; i < mp->txq_count; i++)
2408 		txq_disable(mp->txq + i);
2409 
2410 	while (1) {
2411 		u32 ps = rdlp(mp, PORT_STATUS);
2412 
2413 		if ((ps & (TX_IN_PROGRESS | TX_FIFO_EMPTY)) == TX_FIFO_EMPTY)
2414 			break;
2415 		udelay(10);
2416 	}
2417 
2418 	/* Reset the Enable bit in the Configuration Register */
2419 	data = rdlp(mp, PORT_SERIAL_CONTROL);
2420 	data &= ~(SERIAL_PORT_ENABLE		|
2421 		  DO_NOT_FORCE_LINK_FAIL	|
2422 		  FORCE_LINK_PASS);
2423 	wrlp(mp, PORT_SERIAL_CONTROL, data);
2424 }
2425 
2426 static int mv643xx_eth_stop(struct net_device *dev)
2427 {
2428 	struct mv643xx_eth_private *mp = netdev_priv(dev);
2429 	int i;
2430 
2431 	wrlp(mp, INT_MASK_EXT, 0x00000000);
2432 	wrlp(mp, INT_MASK, 0x00000000);
2433 	rdlp(mp, INT_MASK);
2434 
2435 	napi_disable(&mp->napi);
2436 
2437 	del_timer_sync(&mp->rx_oom);
2438 
2439 	netif_carrier_off(dev);
2440 
2441 	free_irq(dev->irq, dev);
2442 
2443 	port_reset(mp);
2444 	mv643xx_eth_get_stats(dev);
2445 	mib_counters_update(mp);
2446 	del_timer_sync(&mp->mib_counters_timer);
2447 
2448 	skb_queue_purge(&mp->rx_recycle);
2449 
2450 	for (i = 0; i < mp->rxq_count; i++)
2451 		rxq_deinit(mp->rxq + i);
2452 	for (i = 0; i < mp->txq_count; i++)
2453 		txq_deinit(mp->txq + i);
2454 
2455 	return 0;
2456 }
2457 
2458 static int mv643xx_eth_ioctl(struct net_device *dev, struct ifreq *ifr, int cmd)
2459 {
2460 	struct mv643xx_eth_private *mp = netdev_priv(dev);
2461 
2462 	if (mp->phy != NULL)
2463 		return phy_mii_ioctl(mp->phy, ifr, cmd);
2464 
2465 	return -EOPNOTSUPP;
2466 }
2467 
2468 static int mv643xx_eth_change_mtu(struct net_device *dev, int new_mtu)
2469 {
2470 	struct mv643xx_eth_private *mp = netdev_priv(dev);
2471 
2472 	if (new_mtu < 64 || new_mtu > 9500)
2473 		return -EINVAL;
2474 
2475 	dev->mtu = new_mtu;
2476 	mv643xx_eth_recalc_skb_size(mp);
2477 	tx_set_rate(mp, 1000000000, 16777216);
2478 
2479 	if (!netif_running(dev))
2480 		return 0;
2481 
2482 	/*
2483 	 * Stop and then re-open the interface. This will allocate RX
2484 	 * skbs of the new MTU.
2485 	 * There is a possible danger that the open will not succeed,
2486 	 * due to memory being full.
2487 	 */
2488 	mv643xx_eth_stop(dev);
2489 	if (mv643xx_eth_open(dev)) {
2490 		netdev_err(dev,
2491 			   "fatal error on re-opening device after MTU change\n");
2492 	}
2493 
2494 	return 0;
2495 }
2496 
2497 static void tx_timeout_task(struct work_struct *ugly)
2498 {
2499 	struct mv643xx_eth_private *mp;
2500 
2501 	mp = container_of(ugly, struct mv643xx_eth_private, tx_timeout_task);
2502 	if (netif_running(mp->dev)) {
2503 		netif_tx_stop_all_queues(mp->dev);
2504 		port_reset(mp);
2505 		port_start(mp);
2506 		netif_tx_wake_all_queues(mp->dev);
2507 	}
2508 }
2509 
2510 static void mv643xx_eth_tx_timeout(struct net_device *dev)
2511 {
2512 	struct mv643xx_eth_private *mp = netdev_priv(dev);
2513 
2514 	netdev_info(dev, "tx timeout\n");
2515 
2516 	schedule_work(&mp->tx_timeout_task);
2517 }
2518 
2519 #ifdef CONFIG_NET_POLL_CONTROLLER
2520 static void mv643xx_eth_netpoll(struct net_device *dev)
2521 {
2522 	struct mv643xx_eth_private *mp = netdev_priv(dev);
2523 
2524 	wrlp(mp, INT_MASK, 0x00000000);
2525 	rdlp(mp, INT_MASK);
2526 
2527 	mv643xx_eth_irq(dev->irq, dev);
2528 
2529 	wrlp(mp, INT_MASK, mp->int_mask);
2530 }
2531 #endif
2532 
2533 
2534 /* platform glue ************************************************************/
2535 static void
2536 mv643xx_eth_conf_mbus_windows(struct mv643xx_eth_shared_private *msp,
2537 			      const struct mbus_dram_target_info *dram)
2538 {
2539 	void __iomem *base = msp->base;
2540 	u32 win_enable;
2541 	u32 win_protect;
2542 	int i;
2543 
2544 	for (i = 0; i < 6; i++) {
2545 		writel(0, base + WINDOW_BASE(i));
2546 		writel(0, base + WINDOW_SIZE(i));
2547 		if (i < 4)
2548 			writel(0, base + WINDOW_REMAP_HIGH(i));
2549 	}
2550 
2551 	win_enable = 0x3f;
2552 	win_protect = 0;
2553 
2554 	for (i = 0; i < dram->num_cs; i++) {
2555 		const struct mbus_dram_window *cs = dram->cs + i;
2556 
2557 		writel((cs->base & 0xffff0000) |
2558 			(cs->mbus_attr << 8) |
2559 			dram->mbus_dram_target_id, base + WINDOW_BASE(i));
2560 		writel((cs->size - 1) & 0xffff0000, base + WINDOW_SIZE(i));
2561 
2562 		win_enable &= ~(1 << i);
2563 		win_protect |= 3 << (2 * i);
2564 	}
2565 
2566 	writel(win_enable, base + WINDOW_BAR_ENABLE);
2567 	msp->win_protect = win_protect;
2568 }
2569 
2570 static void infer_hw_params(struct mv643xx_eth_shared_private *msp)
2571 {
2572 	/*
2573 	 * Check whether we have a 14-bit coal limit field in bits
2574 	 * [21:8], or a 16-bit coal limit in bits [25,21:7] of the
2575 	 * SDMA config register.
2576 	 */
2577 	writel(0x02000000, msp->base + 0x0400 + SDMA_CONFIG);
2578 	if (readl(msp->base + 0x0400 + SDMA_CONFIG) & 0x02000000)
2579 		msp->extended_rx_coal_limit = 1;
2580 	else
2581 		msp->extended_rx_coal_limit = 0;
2582 
2583 	/*
2584 	 * Check whether the MAC supports TX rate control, and if
2585 	 * yes, whether its associated registers are in the old or
2586 	 * the new place.
2587 	 */
2588 	writel(1, msp->base + 0x0400 + TX_BW_MTU_MOVED);
2589 	if (readl(msp->base + 0x0400 + TX_BW_MTU_MOVED) & 1) {
2590 		msp->tx_bw_control = TX_BW_CONTROL_NEW_LAYOUT;
2591 	} else {
2592 		writel(7, msp->base + 0x0400 + TX_BW_RATE);
2593 		if (readl(msp->base + 0x0400 + TX_BW_RATE) & 7)
2594 			msp->tx_bw_control = TX_BW_CONTROL_OLD_LAYOUT;
2595 		else
2596 			msp->tx_bw_control = TX_BW_CONTROL_ABSENT;
2597 	}
2598 }
2599 
2600 static int mv643xx_eth_shared_probe(struct platform_device *pdev)
2601 {
2602 	static int mv643xx_eth_version_printed;
2603 	struct mv643xx_eth_shared_platform_data *pd = pdev->dev.platform_data;
2604 	struct mv643xx_eth_shared_private *msp;
2605 	const struct mbus_dram_target_info *dram;
2606 	struct resource *res;
2607 	int ret;
2608 
2609 	if (!mv643xx_eth_version_printed++)
2610 		pr_notice("MV-643xx 10/100/1000 ethernet driver version %s\n",
2611 			  mv643xx_eth_driver_version);
2612 
2613 	ret = -EINVAL;
2614 	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
2615 	if (res == NULL)
2616 		goto out;
2617 
2618 	ret = -ENOMEM;
2619 	msp = kzalloc(sizeof(*msp), GFP_KERNEL);
2620 	if (msp == NULL)
2621 		goto out;
2622 
2623 	msp->base = ioremap(res->start, resource_size(res));
2624 	if (msp->base == NULL)
2625 		goto out_free;
2626 
2627 	/*
2628 	 * Set up and register SMI bus.
2629 	 */
2630 	if (pd == NULL || pd->shared_smi == NULL) {
2631 		msp->smi_bus = mdiobus_alloc();
2632 		if (msp->smi_bus == NULL)
2633 			goto out_unmap;
2634 
2635 		msp->smi_bus->priv = msp;
2636 		msp->smi_bus->name = "mv643xx_eth smi";
2637 		msp->smi_bus->read = smi_bus_read;
2638 		msp->smi_bus->write = smi_bus_write,
2639 		snprintf(msp->smi_bus->id, MII_BUS_ID_SIZE, "%s-%d",
2640 			pdev->name, pdev->id);
2641 		msp->smi_bus->parent = &pdev->dev;
2642 		msp->smi_bus->phy_mask = 0xffffffff;
2643 		if (mdiobus_register(msp->smi_bus) < 0)
2644 			goto out_free_mii_bus;
2645 		msp->smi = msp;
2646 	} else {
2647 		msp->smi = platform_get_drvdata(pd->shared_smi);
2648 	}
2649 
2650 	msp->err_interrupt = NO_IRQ;
2651 	init_waitqueue_head(&msp->smi_busy_wait);
2652 
2653 	/*
2654 	 * Check whether the error interrupt is hooked up.
2655 	 */
2656 	res = platform_get_resource(pdev, IORESOURCE_IRQ, 0);
2657 	if (res != NULL) {
2658 		int err;
2659 
2660 		err = request_irq(res->start, mv643xx_eth_err_irq,
2661 				  IRQF_SHARED, "mv643xx_eth", msp);
2662 		if (!err) {
2663 			writel(ERR_INT_SMI_DONE, msp->base + ERR_INT_MASK);
2664 			msp->err_interrupt = res->start;
2665 		}
2666 	}
2667 
2668 	/*
2669 	 * (Re-)program MBUS remapping windows if we are asked to.
2670 	 */
2671 	dram = mv_mbus_dram_info();
2672 	if (dram)
2673 		mv643xx_eth_conf_mbus_windows(msp, dram);
2674 
2675 	msp->tx_csum_limit = (pd != NULL && pd->tx_csum_limit) ?
2676 					pd->tx_csum_limit : 9 * 1024;
2677 	infer_hw_params(msp);
2678 
2679 	platform_set_drvdata(pdev, msp);
2680 
2681 	return 0;
2682 
2683 out_free_mii_bus:
2684 	mdiobus_free(msp->smi_bus);
2685 out_unmap:
2686 	iounmap(msp->base);
2687 out_free:
2688 	kfree(msp);
2689 out:
2690 	return ret;
2691 }
2692 
2693 static int mv643xx_eth_shared_remove(struct platform_device *pdev)
2694 {
2695 	struct mv643xx_eth_shared_private *msp = platform_get_drvdata(pdev);
2696 	struct mv643xx_eth_shared_platform_data *pd = pdev->dev.platform_data;
2697 
2698 	if (pd == NULL || pd->shared_smi == NULL) {
2699 		mdiobus_unregister(msp->smi_bus);
2700 		mdiobus_free(msp->smi_bus);
2701 	}
2702 	if (msp->err_interrupt != NO_IRQ)
2703 		free_irq(msp->err_interrupt, msp);
2704 	iounmap(msp->base);
2705 	kfree(msp);
2706 
2707 	return 0;
2708 }
2709 
2710 static struct platform_driver mv643xx_eth_shared_driver = {
2711 	.probe		= mv643xx_eth_shared_probe,
2712 	.remove		= mv643xx_eth_shared_remove,
2713 	.driver = {
2714 		.name	= MV643XX_ETH_SHARED_NAME,
2715 		.owner	= THIS_MODULE,
2716 	},
2717 };
2718 
2719 static void phy_addr_set(struct mv643xx_eth_private *mp, int phy_addr)
2720 {
2721 	int addr_shift = 5 * mp->port_num;
2722 	u32 data;
2723 
2724 	data = rdl(mp, PHY_ADDR);
2725 	data &= ~(0x1f << addr_shift);
2726 	data |= (phy_addr & 0x1f) << addr_shift;
2727 	wrl(mp, PHY_ADDR, data);
2728 }
2729 
2730 static int phy_addr_get(struct mv643xx_eth_private *mp)
2731 {
2732 	unsigned int data;
2733 
2734 	data = rdl(mp, PHY_ADDR);
2735 
2736 	return (data >> (5 * mp->port_num)) & 0x1f;
2737 }
2738 
2739 static void set_params(struct mv643xx_eth_private *mp,
2740 		       struct mv643xx_eth_platform_data *pd)
2741 {
2742 	struct net_device *dev = mp->dev;
2743 
2744 	if (is_valid_ether_addr(pd->mac_addr))
2745 		memcpy(dev->dev_addr, pd->mac_addr, 6);
2746 	else
2747 		uc_addr_get(mp, dev->dev_addr);
2748 
2749 	mp->rx_ring_size = DEFAULT_RX_QUEUE_SIZE;
2750 	if (pd->rx_queue_size)
2751 		mp->rx_ring_size = pd->rx_queue_size;
2752 	mp->rx_desc_sram_addr = pd->rx_sram_addr;
2753 	mp->rx_desc_sram_size = pd->rx_sram_size;
2754 
2755 	mp->rxq_count = pd->rx_queue_count ? : 1;
2756 
2757 	mp->tx_ring_size = DEFAULT_TX_QUEUE_SIZE;
2758 	if (pd->tx_queue_size)
2759 		mp->tx_ring_size = pd->tx_queue_size;
2760 	mp->tx_desc_sram_addr = pd->tx_sram_addr;
2761 	mp->tx_desc_sram_size = pd->tx_sram_size;
2762 
2763 	mp->txq_count = pd->tx_queue_count ? : 1;
2764 }
2765 
2766 static struct phy_device *phy_scan(struct mv643xx_eth_private *mp,
2767 				   int phy_addr)
2768 {
2769 	struct mii_bus *bus = mp->shared->smi->smi_bus;
2770 	struct phy_device *phydev;
2771 	int start;
2772 	int num;
2773 	int i;
2774 
2775 	if (phy_addr == MV643XX_ETH_PHY_ADDR_DEFAULT) {
2776 		start = phy_addr_get(mp) & 0x1f;
2777 		num = 32;
2778 	} else {
2779 		start = phy_addr & 0x1f;
2780 		num = 1;
2781 	}
2782 
2783 	phydev = NULL;
2784 	for (i = 0; i < num; i++) {
2785 		int addr = (start + i) & 0x1f;
2786 
2787 		if (bus->phy_map[addr] == NULL)
2788 			mdiobus_scan(bus, addr);
2789 
2790 		if (phydev == NULL) {
2791 			phydev = bus->phy_map[addr];
2792 			if (phydev != NULL)
2793 				phy_addr_set(mp, addr);
2794 		}
2795 	}
2796 
2797 	return phydev;
2798 }
2799 
2800 static void phy_init(struct mv643xx_eth_private *mp, int speed, int duplex)
2801 {
2802 	struct phy_device *phy = mp->phy;
2803 
2804 	phy_reset(mp);
2805 
2806 	phy_attach(mp->dev, dev_name(&phy->dev), 0, PHY_INTERFACE_MODE_GMII);
2807 
2808 	if (speed == 0) {
2809 		phy->autoneg = AUTONEG_ENABLE;
2810 		phy->speed = 0;
2811 		phy->duplex = 0;
2812 		phy->advertising = phy->supported | ADVERTISED_Autoneg;
2813 	} else {
2814 		phy->autoneg = AUTONEG_DISABLE;
2815 		phy->advertising = 0;
2816 		phy->speed = speed;
2817 		phy->duplex = duplex;
2818 	}
2819 	phy_start_aneg(phy);
2820 }
2821 
2822 static void init_pscr(struct mv643xx_eth_private *mp, int speed, int duplex)
2823 {
2824 	u32 pscr;
2825 
2826 	pscr = rdlp(mp, PORT_SERIAL_CONTROL);
2827 	if (pscr & SERIAL_PORT_ENABLE) {
2828 		pscr &= ~SERIAL_PORT_ENABLE;
2829 		wrlp(mp, PORT_SERIAL_CONTROL, pscr);
2830 	}
2831 
2832 	pscr = MAX_RX_PACKET_9700BYTE | SERIAL_PORT_CONTROL_RESERVED;
2833 	if (mp->phy == NULL) {
2834 		pscr |= DISABLE_AUTO_NEG_SPEED_GMII;
2835 		if (speed == SPEED_1000)
2836 			pscr |= SET_GMII_SPEED_TO_1000;
2837 		else if (speed == SPEED_100)
2838 			pscr |= SET_MII_SPEED_TO_100;
2839 
2840 		pscr |= DISABLE_AUTO_NEG_FOR_FLOW_CTRL;
2841 
2842 		pscr |= DISABLE_AUTO_NEG_FOR_DUPLEX;
2843 		if (duplex == DUPLEX_FULL)
2844 			pscr |= SET_FULL_DUPLEX_MODE;
2845 	}
2846 
2847 	wrlp(mp, PORT_SERIAL_CONTROL, pscr);
2848 }
2849 
2850 static const struct net_device_ops mv643xx_eth_netdev_ops = {
2851 	.ndo_open		= mv643xx_eth_open,
2852 	.ndo_stop		= mv643xx_eth_stop,
2853 	.ndo_start_xmit		= mv643xx_eth_xmit,
2854 	.ndo_set_rx_mode	= mv643xx_eth_set_rx_mode,
2855 	.ndo_set_mac_address	= mv643xx_eth_set_mac_address,
2856 	.ndo_validate_addr	= eth_validate_addr,
2857 	.ndo_do_ioctl		= mv643xx_eth_ioctl,
2858 	.ndo_change_mtu		= mv643xx_eth_change_mtu,
2859 	.ndo_set_features	= mv643xx_eth_set_features,
2860 	.ndo_tx_timeout		= mv643xx_eth_tx_timeout,
2861 	.ndo_get_stats		= mv643xx_eth_get_stats,
2862 #ifdef CONFIG_NET_POLL_CONTROLLER
2863 	.ndo_poll_controller	= mv643xx_eth_netpoll,
2864 #endif
2865 };
2866 
2867 static int mv643xx_eth_probe(struct platform_device *pdev)
2868 {
2869 	struct mv643xx_eth_platform_data *pd;
2870 	struct mv643xx_eth_private *mp;
2871 	struct net_device *dev;
2872 	struct resource *res;
2873 	int err;
2874 
2875 	pd = pdev->dev.platform_data;
2876 	if (pd == NULL) {
2877 		dev_err(&pdev->dev, "no mv643xx_eth_platform_data\n");
2878 		return -ENODEV;
2879 	}
2880 
2881 	if (pd->shared == NULL) {
2882 		dev_err(&pdev->dev, "no mv643xx_eth_platform_data->shared\n");
2883 		return -ENODEV;
2884 	}
2885 
2886 	dev = alloc_etherdev_mq(sizeof(struct mv643xx_eth_private), 8);
2887 	if (!dev)
2888 		return -ENOMEM;
2889 
2890 	mp = netdev_priv(dev);
2891 	platform_set_drvdata(pdev, mp);
2892 
2893 	mp->shared = platform_get_drvdata(pd->shared);
2894 	mp->base = mp->shared->base + 0x0400 + (pd->port_number << 10);
2895 	mp->port_num = pd->port_number;
2896 
2897 	mp->dev = dev;
2898 
2899 	/*
2900 	 * Start with a default rate, and if there is a clock, allow
2901 	 * it to override the default.
2902 	 */
2903 	mp->t_clk = 133000000;
2904 #if defined(CONFIG_HAVE_CLK)
2905 	mp->clk = clk_get(&pdev->dev, (pdev->id ? "1" : "0"));
2906 	if (!IS_ERR(mp->clk)) {
2907 		clk_prepare_enable(mp->clk);
2908 		mp->t_clk = clk_get_rate(mp->clk);
2909 	}
2910 #endif
2911 	set_params(mp, pd);
2912 	netif_set_real_num_tx_queues(dev, mp->txq_count);
2913 	netif_set_real_num_rx_queues(dev, mp->rxq_count);
2914 
2915 	if (pd->phy_addr != MV643XX_ETH_PHY_NONE)
2916 		mp->phy = phy_scan(mp, pd->phy_addr);
2917 
2918 	if (mp->phy != NULL)
2919 		phy_init(mp, pd->speed, pd->duplex);
2920 
2921 	SET_ETHTOOL_OPS(dev, &mv643xx_eth_ethtool_ops);
2922 
2923 	init_pscr(mp, pd->speed, pd->duplex);
2924 
2925 
2926 	mib_counters_clear(mp);
2927 
2928 	init_timer(&mp->mib_counters_timer);
2929 	mp->mib_counters_timer.data = (unsigned long)mp;
2930 	mp->mib_counters_timer.function = mib_counters_timer_wrapper;
2931 	mp->mib_counters_timer.expires = jiffies + 30 * HZ;
2932 	add_timer(&mp->mib_counters_timer);
2933 
2934 	spin_lock_init(&mp->mib_counters_lock);
2935 
2936 	INIT_WORK(&mp->tx_timeout_task, tx_timeout_task);
2937 
2938 	netif_napi_add(dev, &mp->napi, mv643xx_eth_poll, 128);
2939 
2940 	init_timer(&mp->rx_oom);
2941 	mp->rx_oom.data = (unsigned long)mp;
2942 	mp->rx_oom.function = oom_timer_wrapper;
2943 
2944 
2945 	res = platform_get_resource(pdev, IORESOURCE_IRQ, 0);
2946 	BUG_ON(!res);
2947 	dev->irq = res->start;
2948 
2949 	dev->netdev_ops = &mv643xx_eth_netdev_ops;
2950 
2951 	dev->watchdog_timeo = 2 * HZ;
2952 	dev->base_addr = 0;
2953 
2954 	dev->hw_features = NETIF_F_SG | NETIF_F_IP_CSUM |
2955 		NETIF_F_RXCSUM | NETIF_F_LRO;
2956 	dev->features = NETIF_F_SG | NETIF_F_IP_CSUM | NETIF_F_RXCSUM;
2957 	dev->vlan_features = NETIF_F_SG | NETIF_F_IP_CSUM;
2958 
2959 	dev->priv_flags |= IFF_UNICAST_FLT;
2960 
2961 	SET_NETDEV_DEV(dev, &pdev->dev);
2962 
2963 	if (mp->shared->win_protect)
2964 		wrl(mp, WINDOW_PROTECT(mp->port_num), mp->shared->win_protect);
2965 
2966 	netif_carrier_off(dev);
2967 
2968 	wrlp(mp, SDMA_CONFIG, PORT_SDMA_CONFIG_DEFAULT_VALUE);
2969 
2970 	set_rx_coal(mp, 250);
2971 	set_tx_coal(mp, 0);
2972 
2973 	err = register_netdev(dev);
2974 	if (err)
2975 		goto out;
2976 
2977 	netdev_notice(dev, "port %d with MAC address %pM\n",
2978 		      mp->port_num, dev->dev_addr);
2979 
2980 	if (mp->tx_desc_sram_size > 0)
2981 		netdev_notice(dev, "configured with sram\n");
2982 
2983 	return 0;
2984 
2985 out:
2986 #if defined(CONFIG_HAVE_CLK)
2987 	if (!IS_ERR(mp->clk)) {
2988 		clk_disable_unprepare(mp->clk);
2989 		clk_put(mp->clk);
2990 	}
2991 #endif
2992 	free_netdev(dev);
2993 
2994 	return err;
2995 }
2996 
2997 static int mv643xx_eth_remove(struct platform_device *pdev)
2998 {
2999 	struct mv643xx_eth_private *mp = platform_get_drvdata(pdev);
3000 
3001 	unregister_netdev(mp->dev);
3002 	if (mp->phy != NULL)
3003 		phy_detach(mp->phy);
3004 	cancel_work_sync(&mp->tx_timeout_task);
3005 
3006 #if defined(CONFIG_HAVE_CLK)
3007 	if (!IS_ERR(mp->clk)) {
3008 		clk_disable_unprepare(mp->clk);
3009 		clk_put(mp->clk);
3010 	}
3011 #endif
3012 
3013 	free_netdev(mp->dev);
3014 
3015 	platform_set_drvdata(pdev, NULL);
3016 
3017 	return 0;
3018 }
3019 
3020 static void mv643xx_eth_shutdown(struct platform_device *pdev)
3021 {
3022 	struct mv643xx_eth_private *mp = platform_get_drvdata(pdev);
3023 
3024 	/* Mask all interrupts on ethernet port */
3025 	wrlp(mp, INT_MASK, 0);
3026 	rdlp(mp, INT_MASK);
3027 
3028 	if (netif_running(mp->dev))
3029 		port_reset(mp);
3030 }
3031 
3032 static struct platform_driver mv643xx_eth_driver = {
3033 	.probe		= mv643xx_eth_probe,
3034 	.remove		= mv643xx_eth_remove,
3035 	.shutdown	= mv643xx_eth_shutdown,
3036 	.driver = {
3037 		.name	= MV643XX_ETH_NAME,
3038 		.owner	= THIS_MODULE,
3039 	},
3040 };
3041 
3042 static int __init mv643xx_eth_init_module(void)
3043 {
3044 	int rc;
3045 
3046 	rc = platform_driver_register(&mv643xx_eth_shared_driver);
3047 	if (!rc) {
3048 		rc = platform_driver_register(&mv643xx_eth_driver);
3049 		if (rc)
3050 			platform_driver_unregister(&mv643xx_eth_shared_driver);
3051 	}
3052 
3053 	return rc;
3054 }
3055 module_init(mv643xx_eth_init_module);
3056 
3057 static void __exit mv643xx_eth_cleanup_module(void)
3058 {
3059 	platform_driver_unregister(&mv643xx_eth_driver);
3060 	platform_driver_unregister(&mv643xx_eth_shared_driver);
3061 }
3062 module_exit(mv643xx_eth_cleanup_module);
3063 
3064 MODULE_AUTHOR("Rabeeh Khoury, Assaf Hoffman, Matthew Dharm, "
3065 	      "Manish Lachwani, Dale Farnsworth and Lennert Buytenhek");
3066 MODULE_DESCRIPTION("Ethernet driver for Marvell MV643XX");
3067 MODULE_LICENSE("GPL");
3068 MODULE_ALIAS("platform:" MV643XX_ETH_SHARED_NAME);
3069 MODULE_ALIAS("platform:" MV643XX_ETH_NAME);
3070