xref: /linux/drivers/net/ethernet/intel/e1000/e1000_main.c (revision 0b8061c340b643e01da431dd60c75a41bb1d31ec)
1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright(c) 1999 - 2006 Intel Corporation. */
3 
4 #include "e1000.h"
5 #include <net/ip6_checksum.h>
6 #include <linux/io.h>
7 #include <linux/prefetch.h>
8 #include <linux/bitops.h>
9 #include <linux/if_vlan.h>
10 
11 char e1000_driver_name[] = "e1000";
12 static char e1000_driver_string[] = "Intel(R) PRO/1000 Network Driver";
13 static const char e1000_copyright[] = "Copyright (c) 1999-2006 Intel Corporation.";
14 
15 /* e1000_pci_tbl - PCI Device ID Table
16  *
17  * Last entry must be all 0s
18  *
19  * Macro expands to...
20  *   {PCI_DEVICE(PCI_VENDOR_ID_INTEL, device_id)}
21  */
22 static const struct pci_device_id e1000_pci_tbl[] = {
23 	INTEL_E1000_ETHERNET_DEVICE(0x1000),
24 	INTEL_E1000_ETHERNET_DEVICE(0x1001),
25 	INTEL_E1000_ETHERNET_DEVICE(0x1004),
26 	INTEL_E1000_ETHERNET_DEVICE(0x1008),
27 	INTEL_E1000_ETHERNET_DEVICE(0x1009),
28 	INTEL_E1000_ETHERNET_DEVICE(0x100C),
29 	INTEL_E1000_ETHERNET_DEVICE(0x100D),
30 	INTEL_E1000_ETHERNET_DEVICE(0x100E),
31 	INTEL_E1000_ETHERNET_DEVICE(0x100F),
32 	INTEL_E1000_ETHERNET_DEVICE(0x1010),
33 	INTEL_E1000_ETHERNET_DEVICE(0x1011),
34 	INTEL_E1000_ETHERNET_DEVICE(0x1012),
35 	INTEL_E1000_ETHERNET_DEVICE(0x1013),
36 	INTEL_E1000_ETHERNET_DEVICE(0x1014),
37 	INTEL_E1000_ETHERNET_DEVICE(0x1015),
38 	INTEL_E1000_ETHERNET_DEVICE(0x1016),
39 	INTEL_E1000_ETHERNET_DEVICE(0x1017),
40 	INTEL_E1000_ETHERNET_DEVICE(0x1018),
41 	INTEL_E1000_ETHERNET_DEVICE(0x1019),
42 	INTEL_E1000_ETHERNET_DEVICE(0x101A),
43 	INTEL_E1000_ETHERNET_DEVICE(0x101D),
44 	INTEL_E1000_ETHERNET_DEVICE(0x101E),
45 	INTEL_E1000_ETHERNET_DEVICE(0x1026),
46 	INTEL_E1000_ETHERNET_DEVICE(0x1027),
47 	INTEL_E1000_ETHERNET_DEVICE(0x1028),
48 	INTEL_E1000_ETHERNET_DEVICE(0x1075),
49 	INTEL_E1000_ETHERNET_DEVICE(0x1076),
50 	INTEL_E1000_ETHERNET_DEVICE(0x1077),
51 	INTEL_E1000_ETHERNET_DEVICE(0x1078),
52 	INTEL_E1000_ETHERNET_DEVICE(0x1079),
53 	INTEL_E1000_ETHERNET_DEVICE(0x107A),
54 	INTEL_E1000_ETHERNET_DEVICE(0x107B),
55 	INTEL_E1000_ETHERNET_DEVICE(0x107C),
56 	INTEL_E1000_ETHERNET_DEVICE(0x108A),
57 	INTEL_E1000_ETHERNET_DEVICE(0x1099),
58 	INTEL_E1000_ETHERNET_DEVICE(0x10B5),
59 	INTEL_E1000_ETHERNET_DEVICE(0x2E6E),
60 	/* required last entry */
61 	{0,}
62 };
63 
64 MODULE_DEVICE_TABLE(pci, e1000_pci_tbl);
65 
66 int e1000_up(struct e1000_adapter *adapter);
67 void e1000_down(struct e1000_adapter *adapter);
68 void e1000_reinit_locked(struct e1000_adapter *adapter);
69 void e1000_reset(struct e1000_adapter *adapter);
70 int e1000_setup_all_tx_resources(struct e1000_adapter *adapter);
71 int e1000_setup_all_rx_resources(struct e1000_adapter *adapter);
72 void e1000_free_all_tx_resources(struct e1000_adapter *adapter);
73 void e1000_free_all_rx_resources(struct e1000_adapter *adapter);
74 static int e1000_setup_tx_resources(struct e1000_adapter *adapter,
75 				    struct e1000_tx_ring *txdr);
76 static int e1000_setup_rx_resources(struct e1000_adapter *adapter,
77 				    struct e1000_rx_ring *rxdr);
78 static void e1000_free_tx_resources(struct e1000_adapter *adapter,
79 				    struct e1000_tx_ring *tx_ring);
80 static void e1000_free_rx_resources(struct e1000_adapter *adapter,
81 				    struct e1000_rx_ring *rx_ring);
82 void e1000_update_stats(struct e1000_adapter *adapter);
83 
84 static int e1000_init_module(void);
85 static void e1000_exit_module(void);
86 static int e1000_probe(struct pci_dev *pdev, const struct pci_device_id *ent);
87 static void e1000_remove(struct pci_dev *pdev);
88 static int e1000_alloc_queues(struct e1000_adapter *adapter);
89 static int e1000_sw_init(struct e1000_adapter *adapter);
90 int e1000_open(struct net_device *netdev);
91 int e1000_close(struct net_device *netdev);
92 static void e1000_configure_tx(struct e1000_adapter *adapter);
93 static void e1000_configure_rx(struct e1000_adapter *adapter);
94 static void e1000_setup_rctl(struct e1000_adapter *adapter);
95 static void e1000_clean_all_tx_rings(struct e1000_adapter *adapter);
96 static void e1000_clean_all_rx_rings(struct e1000_adapter *adapter);
97 static void e1000_clean_tx_ring(struct e1000_adapter *adapter,
98 				struct e1000_tx_ring *tx_ring);
99 static void e1000_clean_rx_ring(struct e1000_adapter *adapter,
100 				struct e1000_rx_ring *rx_ring);
101 static void e1000_set_rx_mode(struct net_device *netdev);
102 static void e1000_update_phy_info_task(struct work_struct *work);
103 static void e1000_watchdog(struct work_struct *work);
104 static void e1000_82547_tx_fifo_stall_task(struct work_struct *work);
105 static netdev_tx_t e1000_xmit_frame(struct sk_buff *skb,
106 				    struct net_device *netdev);
107 static int e1000_change_mtu(struct net_device *netdev, int new_mtu);
108 static int e1000_set_mac(struct net_device *netdev, void *p);
109 static irqreturn_t e1000_intr(int irq, void *data);
110 static bool e1000_clean_tx_irq(struct e1000_adapter *adapter,
111 			       struct e1000_tx_ring *tx_ring);
112 static int e1000_clean(struct napi_struct *napi, int budget);
113 static bool e1000_clean_rx_irq(struct e1000_adapter *adapter,
114 			       struct e1000_rx_ring *rx_ring,
115 			       int *work_done, int work_to_do);
116 static bool e1000_clean_jumbo_rx_irq(struct e1000_adapter *adapter,
117 				     struct e1000_rx_ring *rx_ring,
118 				     int *work_done, int work_to_do);
119 static void e1000_alloc_dummy_rx_buffers(struct e1000_adapter *adapter,
120 					 struct e1000_rx_ring *rx_ring,
121 					 int cleaned_count)
122 {
123 }
124 static void e1000_alloc_rx_buffers(struct e1000_adapter *adapter,
125 				   struct e1000_rx_ring *rx_ring,
126 				   int cleaned_count);
127 static void e1000_alloc_jumbo_rx_buffers(struct e1000_adapter *adapter,
128 					 struct e1000_rx_ring *rx_ring,
129 					 int cleaned_count);
130 static int e1000_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd);
131 static int e1000_mii_ioctl(struct net_device *netdev, struct ifreq *ifr,
132 			   int cmd);
133 static void e1000_enter_82542_rst(struct e1000_adapter *adapter);
134 static void e1000_leave_82542_rst(struct e1000_adapter *adapter);
135 static void e1000_tx_timeout(struct net_device *dev, unsigned int txqueue);
136 static void e1000_reset_task(struct work_struct *work);
137 static void e1000_smartspeed(struct e1000_adapter *adapter);
138 static int e1000_82547_fifo_workaround(struct e1000_adapter *adapter,
139 				       struct sk_buff *skb);
140 
141 static bool e1000_vlan_used(struct e1000_adapter *adapter);
142 static void e1000_vlan_mode(struct net_device *netdev,
143 			    netdev_features_t features);
144 static void e1000_vlan_filter_on_off(struct e1000_adapter *adapter,
145 				     bool filter_on);
146 static int e1000_vlan_rx_add_vid(struct net_device *netdev,
147 				 __be16 proto, u16 vid);
148 static int e1000_vlan_rx_kill_vid(struct net_device *netdev,
149 				  __be16 proto, u16 vid);
150 static void e1000_restore_vlan(struct e1000_adapter *adapter);
151 
152 static int __maybe_unused e1000_suspend(struct device *dev);
153 static int __maybe_unused e1000_resume(struct device *dev);
154 static void e1000_shutdown(struct pci_dev *pdev);
155 
156 #ifdef CONFIG_NET_POLL_CONTROLLER
157 /* for netdump / net console */
158 static void e1000_netpoll (struct net_device *netdev);
159 #endif
160 
161 #define COPYBREAK_DEFAULT 256
162 static unsigned int copybreak __read_mostly = COPYBREAK_DEFAULT;
163 module_param(copybreak, uint, 0644);
164 MODULE_PARM_DESC(copybreak,
165 	"Maximum size of packet that is copied to a new buffer on receive");
166 
167 static pci_ers_result_t e1000_io_error_detected(struct pci_dev *pdev,
168 						pci_channel_state_t state);
169 static pci_ers_result_t e1000_io_slot_reset(struct pci_dev *pdev);
170 static void e1000_io_resume(struct pci_dev *pdev);
171 
172 static const struct pci_error_handlers e1000_err_handler = {
173 	.error_detected = e1000_io_error_detected,
174 	.slot_reset = e1000_io_slot_reset,
175 	.resume = e1000_io_resume,
176 };
177 
178 static SIMPLE_DEV_PM_OPS(e1000_pm_ops, e1000_suspend, e1000_resume);
179 
180 static struct pci_driver e1000_driver = {
181 	.name     = e1000_driver_name,
182 	.id_table = e1000_pci_tbl,
183 	.probe    = e1000_probe,
184 	.remove   = e1000_remove,
185 	.driver = {
186 		.pm = &e1000_pm_ops,
187 	},
188 	.shutdown = e1000_shutdown,
189 	.err_handler = &e1000_err_handler
190 };
191 
192 MODULE_AUTHOR("Intel Corporation, <linux.nics@intel.com>");
193 MODULE_DESCRIPTION("Intel(R) PRO/1000 Network Driver");
194 MODULE_LICENSE("GPL v2");
195 
196 #define DEFAULT_MSG_ENABLE (NETIF_MSG_DRV|NETIF_MSG_PROBE|NETIF_MSG_LINK)
197 static int debug = -1;
198 module_param(debug, int, 0);
199 MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all)");
200 
201 /**
202  * e1000_get_hw_dev - helper function for getting netdev
203  * @hw: pointer to HW struct
204  *
205  * return device used by hardware layer to print debugging information
206  *
207  **/
208 struct net_device *e1000_get_hw_dev(struct e1000_hw *hw)
209 {
210 	struct e1000_adapter *adapter = hw->back;
211 	return adapter->netdev;
212 }
213 
214 /**
215  * e1000_init_module - Driver Registration Routine
216  *
217  * e1000_init_module is the first routine called when the driver is
218  * loaded. All it does is register with the PCI subsystem.
219  **/
220 static int __init e1000_init_module(void)
221 {
222 	int ret;
223 	pr_info("%s\n", e1000_driver_string);
224 
225 	pr_info("%s\n", e1000_copyright);
226 
227 	ret = pci_register_driver(&e1000_driver);
228 	if (copybreak != COPYBREAK_DEFAULT) {
229 		if (copybreak == 0)
230 			pr_info("copybreak disabled\n");
231 		else
232 			pr_info("copybreak enabled for "
233 				   "packets <= %u bytes\n", copybreak);
234 	}
235 	return ret;
236 }
237 
238 module_init(e1000_init_module);
239 
240 /**
241  * e1000_exit_module - Driver Exit Cleanup Routine
242  *
243  * e1000_exit_module is called just before the driver is removed
244  * from memory.
245  **/
246 static void __exit e1000_exit_module(void)
247 {
248 	pci_unregister_driver(&e1000_driver);
249 }
250 
251 module_exit(e1000_exit_module);
252 
253 static int e1000_request_irq(struct e1000_adapter *adapter)
254 {
255 	struct net_device *netdev = adapter->netdev;
256 	irq_handler_t handler = e1000_intr;
257 	int irq_flags = IRQF_SHARED;
258 	int err;
259 
260 	err = request_irq(adapter->pdev->irq, handler, irq_flags, netdev->name,
261 			  netdev);
262 	if (err) {
263 		e_err(probe, "Unable to allocate interrupt Error: %d\n", err);
264 	}
265 
266 	return err;
267 }
268 
269 static void e1000_free_irq(struct e1000_adapter *adapter)
270 {
271 	struct net_device *netdev = adapter->netdev;
272 
273 	free_irq(adapter->pdev->irq, netdev);
274 }
275 
276 /**
277  * e1000_irq_disable - Mask off interrupt generation on the NIC
278  * @adapter: board private structure
279  **/
280 static void e1000_irq_disable(struct e1000_adapter *adapter)
281 {
282 	struct e1000_hw *hw = &adapter->hw;
283 
284 	ew32(IMC, ~0);
285 	E1000_WRITE_FLUSH();
286 	synchronize_irq(adapter->pdev->irq);
287 }
288 
289 /**
290  * e1000_irq_enable - Enable default interrupt generation settings
291  * @adapter: board private structure
292  **/
293 static void e1000_irq_enable(struct e1000_adapter *adapter)
294 {
295 	struct e1000_hw *hw = &adapter->hw;
296 
297 	ew32(IMS, IMS_ENABLE_MASK);
298 	E1000_WRITE_FLUSH();
299 }
300 
301 static void e1000_update_mng_vlan(struct e1000_adapter *adapter)
302 {
303 	struct e1000_hw *hw = &adapter->hw;
304 	struct net_device *netdev = adapter->netdev;
305 	u16 vid = hw->mng_cookie.vlan_id;
306 	u16 old_vid = adapter->mng_vlan_id;
307 
308 	if (!e1000_vlan_used(adapter))
309 		return;
310 
311 	if (!test_bit(vid, adapter->active_vlans)) {
312 		if (hw->mng_cookie.status &
313 		    E1000_MNG_DHCP_COOKIE_STATUS_VLAN_SUPPORT) {
314 			e1000_vlan_rx_add_vid(netdev, htons(ETH_P_8021Q), vid);
315 			adapter->mng_vlan_id = vid;
316 		} else {
317 			adapter->mng_vlan_id = E1000_MNG_VLAN_NONE;
318 		}
319 		if ((old_vid != (u16)E1000_MNG_VLAN_NONE) &&
320 		    (vid != old_vid) &&
321 		    !test_bit(old_vid, adapter->active_vlans))
322 			e1000_vlan_rx_kill_vid(netdev, htons(ETH_P_8021Q),
323 					       old_vid);
324 	} else {
325 		adapter->mng_vlan_id = vid;
326 	}
327 }
328 
329 static void e1000_init_manageability(struct e1000_adapter *adapter)
330 {
331 	struct e1000_hw *hw = &adapter->hw;
332 
333 	if (adapter->en_mng_pt) {
334 		u32 manc = er32(MANC);
335 
336 		/* disable hardware interception of ARP */
337 		manc &= ~(E1000_MANC_ARP_EN);
338 
339 		ew32(MANC, manc);
340 	}
341 }
342 
343 static void e1000_release_manageability(struct e1000_adapter *adapter)
344 {
345 	struct e1000_hw *hw = &adapter->hw;
346 
347 	if (adapter->en_mng_pt) {
348 		u32 manc = er32(MANC);
349 
350 		/* re-enable hardware interception of ARP */
351 		manc |= E1000_MANC_ARP_EN;
352 
353 		ew32(MANC, manc);
354 	}
355 }
356 
357 /**
358  * e1000_configure - configure the hardware for RX and TX
359  * @adapter: private board structure
360  **/
361 static void e1000_configure(struct e1000_adapter *adapter)
362 {
363 	struct net_device *netdev = adapter->netdev;
364 	int i;
365 
366 	e1000_set_rx_mode(netdev);
367 
368 	e1000_restore_vlan(adapter);
369 	e1000_init_manageability(adapter);
370 
371 	e1000_configure_tx(adapter);
372 	e1000_setup_rctl(adapter);
373 	e1000_configure_rx(adapter);
374 	/* call E1000_DESC_UNUSED which always leaves
375 	 * at least 1 descriptor unused to make sure
376 	 * next_to_use != next_to_clean
377 	 */
378 	for (i = 0; i < adapter->num_rx_queues; i++) {
379 		struct e1000_rx_ring *ring = &adapter->rx_ring[i];
380 		adapter->alloc_rx_buf(adapter, ring,
381 				      E1000_DESC_UNUSED(ring));
382 	}
383 }
384 
385 int e1000_up(struct e1000_adapter *adapter)
386 {
387 	struct e1000_hw *hw = &adapter->hw;
388 
389 	/* hardware has been reset, we need to reload some things */
390 	e1000_configure(adapter);
391 
392 	clear_bit(__E1000_DOWN, &adapter->flags);
393 
394 	napi_enable(&adapter->napi);
395 
396 	e1000_irq_enable(adapter);
397 
398 	netif_wake_queue(adapter->netdev);
399 
400 	/* fire a link change interrupt to start the watchdog */
401 	ew32(ICS, E1000_ICS_LSC);
402 	return 0;
403 }
404 
405 /**
406  * e1000_power_up_phy - restore link in case the phy was powered down
407  * @adapter: address of board private structure
408  *
409  * The phy may be powered down to save power and turn off link when the
410  * driver is unloaded and wake on lan is not enabled (among others)
411  * *** this routine MUST be followed by a call to e1000_reset ***
412  **/
413 void e1000_power_up_phy(struct e1000_adapter *adapter)
414 {
415 	struct e1000_hw *hw = &adapter->hw;
416 	u16 mii_reg = 0;
417 
418 	/* Just clear the power down bit to wake the phy back up */
419 	if (hw->media_type == e1000_media_type_copper) {
420 		/* according to the manual, the phy will retain its
421 		 * settings across a power-down/up cycle
422 		 */
423 		e1000_read_phy_reg(hw, PHY_CTRL, &mii_reg);
424 		mii_reg &= ~MII_CR_POWER_DOWN;
425 		e1000_write_phy_reg(hw, PHY_CTRL, mii_reg);
426 	}
427 }
428 
429 static void e1000_power_down_phy(struct e1000_adapter *adapter)
430 {
431 	struct e1000_hw *hw = &adapter->hw;
432 
433 	/* Power down the PHY so no link is implied when interface is down *
434 	 * The PHY cannot be powered down if any of the following is true *
435 	 * (a) WoL is enabled
436 	 * (b) AMT is active
437 	 * (c) SoL/IDER session is active
438 	 */
439 	if (!adapter->wol && hw->mac_type >= e1000_82540 &&
440 	   hw->media_type == e1000_media_type_copper) {
441 		u16 mii_reg = 0;
442 
443 		switch (hw->mac_type) {
444 		case e1000_82540:
445 		case e1000_82545:
446 		case e1000_82545_rev_3:
447 		case e1000_82546:
448 		case e1000_ce4100:
449 		case e1000_82546_rev_3:
450 		case e1000_82541:
451 		case e1000_82541_rev_2:
452 		case e1000_82547:
453 		case e1000_82547_rev_2:
454 			if (er32(MANC) & E1000_MANC_SMBUS_EN)
455 				goto out;
456 			break;
457 		default:
458 			goto out;
459 		}
460 		e1000_read_phy_reg(hw, PHY_CTRL, &mii_reg);
461 		mii_reg |= MII_CR_POWER_DOWN;
462 		e1000_write_phy_reg(hw, PHY_CTRL, mii_reg);
463 		msleep(1);
464 	}
465 out:
466 	return;
467 }
468 
469 static void e1000_down_and_stop(struct e1000_adapter *adapter)
470 {
471 	set_bit(__E1000_DOWN, &adapter->flags);
472 
473 	cancel_delayed_work_sync(&adapter->watchdog_task);
474 
475 	/*
476 	 * Since the watchdog task can reschedule other tasks, we should cancel
477 	 * it first, otherwise we can run into the situation when a work is
478 	 * still running after the adapter has been turned down.
479 	 */
480 
481 	cancel_delayed_work_sync(&adapter->phy_info_task);
482 	cancel_delayed_work_sync(&adapter->fifo_stall_task);
483 
484 	/* Only kill reset task if adapter is not resetting */
485 	if (!test_bit(__E1000_RESETTING, &adapter->flags))
486 		cancel_work_sync(&adapter->reset_task);
487 }
488 
489 void e1000_down(struct e1000_adapter *adapter)
490 {
491 	struct e1000_hw *hw = &adapter->hw;
492 	struct net_device *netdev = adapter->netdev;
493 	u32 rctl, tctl;
494 
495 	/* disable receives in the hardware */
496 	rctl = er32(RCTL);
497 	ew32(RCTL, rctl & ~E1000_RCTL_EN);
498 	/* flush and sleep below */
499 
500 	netif_tx_disable(netdev);
501 
502 	/* disable transmits in the hardware */
503 	tctl = er32(TCTL);
504 	tctl &= ~E1000_TCTL_EN;
505 	ew32(TCTL, tctl);
506 	/* flush both disables and wait for them to finish */
507 	E1000_WRITE_FLUSH();
508 	msleep(10);
509 
510 	/* Set the carrier off after transmits have been disabled in the
511 	 * hardware, to avoid race conditions with e1000_watchdog() (which
512 	 * may be running concurrently to us, checking for the carrier
513 	 * bit to decide whether it should enable transmits again). Such
514 	 * a race condition would result into transmission being disabled
515 	 * in the hardware until the next IFF_DOWN+IFF_UP cycle.
516 	 */
517 	netif_carrier_off(netdev);
518 
519 	napi_disable(&adapter->napi);
520 
521 	e1000_irq_disable(adapter);
522 
523 	/* Setting DOWN must be after irq_disable to prevent
524 	 * a screaming interrupt.  Setting DOWN also prevents
525 	 * tasks from rescheduling.
526 	 */
527 	e1000_down_and_stop(adapter);
528 
529 	adapter->link_speed = 0;
530 	adapter->link_duplex = 0;
531 
532 	e1000_reset(adapter);
533 	e1000_clean_all_tx_rings(adapter);
534 	e1000_clean_all_rx_rings(adapter);
535 }
536 
537 void e1000_reinit_locked(struct e1000_adapter *adapter)
538 {
539 	while (test_and_set_bit(__E1000_RESETTING, &adapter->flags))
540 		msleep(1);
541 
542 	/* only run the task if not already down */
543 	if (!test_bit(__E1000_DOWN, &adapter->flags)) {
544 		e1000_down(adapter);
545 		e1000_up(adapter);
546 	}
547 
548 	clear_bit(__E1000_RESETTING, &adapter->flags);
549 }
550 
551 void e1000_reset(struct e1000_adapter *adapter)
552 {
553 	struct e1000_hw *hw = &adapter->hw;
554 	u32 pba = 0, tx_space, min_tx_space, min_rx_space;
555 	bool legacy_pba_adjust = false;
556 	u16 hwm;
557 
558 	/* Repartition Pba for greater than 9k mtu
559 	 * To take effect CTRL.RST is required.
560 	 */
561 
562 	switch (hw->mac_type) {
563 	case e1000_82542_rev2_0:
564 	case e1000_82542_rev2_1:
565 	case e1000_82543:
566 	case e1000_82544:
567 	case e1000_82540:
568 	case e1000_82541:
569 	case e1000_82541_rev_2:
570 		legacy_pba_adjust = true;
571 		pba = E1000_PBA_48K;
572 		break;
573 	case e1000_82545:
574 	case e1000_82545_rev_3:
575 	case e1000_82546:
576 	case e1000_ce4100:
577 	case e1000_82546_rev_3:
578 		pba = E1000_PBA_48K;
579 		break;
580 	case e1000_82547:
581 	case e1000_82547_rev_2:
582 		legacy_pba_adjust = true;
583 		pba = E1000_PBA_30K;
584 		break;
585 	case e1000_undefined:
586 	case e1000_num_macs:
587 		break;
588 	}
589 
590 	if (legacy_pba_adjust) {
591 		if (hw->max_frame_size > E1000_RXBUFFER_8192)
592 			pba -= 8; /* allocate more FIFO for Tx */
593 
594 		if (hw->mac_type == e1000_82547) {
595 			adapter->tx_fifo_head = 0;
596 			adapter->tx_head_addr = pba << E1000_TX_HEAD_ADDR_SHIFT;
597 			adapter->tx_fifo_size =
598 				(E1000_PBA_40K - pba) << E1000_PBA_BYTES_SHIFT;
599 			atomic_set(&adapter->tx_fifo_stall, 0);
600 		}
601 	} else if (hw->max_frame_size >  ETH_FRAME_LEN + ETH_FCS_LEN) {
602 		/* adjust PBA for jumbo frames */
603 		ew32(PBA, pba);
604 
605 		/* To maintain wire speed transmits, the Tx FIFO should be
606 		 * large enough to accommodate two full transmit packets,
607 		 * rounded up to the next 1KB and expressed in KB.  Likewise,
608 		 * the Rx FIFO should be large enough to accommodate at least
609 		 * one full receive packet and is similarly rounded up and
610 		 * expressed in KB.
611 		 */
612 		pba = er32(PBA);
613 		/* upper 16 bits has Tx packet buffer allocation size in KB */
614 		tx_space = pba >> 16;
615 		/* lower 16 bits has Rx packet buffer allocation size in KB */
616 		pba &= 0xffff;
617 		/* the Tx fifo also stores 16 bytes of information about the Tx
618 		 * but don't include ethernet FCS because hardware appends it
619 		 */
620 		min_tx_space = (hw->max_frame_size +
621 				sizeof(struct e1000_tx_desc) -
622 				ETH_FCS_LEN) * 2;
623 		min_tx_space = ALIGN(min_tx_space, 1024);
624 		min_tx_space >>= 10;
625 		/* software strips receive CRC, so leave room for it */
626 		min_rx_space = hw->max_frame_size;
627 		min_rx_space = ALIGN(min_rx_space, 1024);
628 		min_rx_space >>= 10;
629 
630 		/* If current Tx allocation is less than the min Tx FIFO size,
631 		 * and the min Tx FIFO size is less than the current Rx FIFO
632 		 * allocation, take space away from current Rx allocation
633 		 */
634 		if (tx_space < min_tx_space &&
635 		    ((min_tx_space - tx_space) < pba)) {
636 			pba = pba - (min_tx_space - tx_space);
637 
638 			/* PCI/PCIx hardware has PBA alignment constraints */
639 			switch (hw->mac_type) {
640 			case e1000_82545 ... e1000_82546_rev_3:
641 				pba &= ~(E1000_PBA_8K - 1);
642 				break;
643 			default:
644 				break;
645 			}
646 
647 			/* if short on Rx space, Rx wins and must trump Tx
648 			 * adjustment or use Early Receive if available
649 			 */
650 			if (pba < min_rx_space)
651 				pba = min_rx_space;
652 		}
653 	}
654 
655 	ew32(PBA, pba);
656 
657 	/* flow control settings:
658 	 * The high water mark must be low enough to fit one full frame
659 	 * (or the size used for early receive) above it in the Rx FIFO.
660 	 * Set it to the lower of:
661 	 * - 90% of the Rx FIFO size, and
662 	 * - the full Rx FIFO size minus the early receive size (for parts
663 	 *   with ERT support assuming ERT set to E1000_ERT_2048), or
664 	 * - the full Rx FIFO size minus one full frame
665 	 */
666 	hwm = min(((pba << 10) * 9 / 10),
667 		  ((pba << 10) - hw->max_frame_size));
668 
669 	hw->fc_high_water = hwm & 0xFFF8;	/* 8-byte granularity */
670 	hw->fc_low_water = hw->fc_high_water - 8;
671 	hw->fc_pause_time = E1000_FC_PAUSE_TIME;
672 	hw->fc_send_xon = 1;
673 	hw->fc = hw->original_fc;
674 
675 	/* Allow time for pending master requests to run */
676 	e1000_reset_hw(hw);
677 	if (hw->mac_type >= e1000_82544)
678 		ew32(WUC, 0);
679 
680 	if (e1000_init_hw(hw))
681 		e_dev_err("Hardware Error\n");
682 	e1000_update_mng_vlan(adapter);
683 
684 	/* if (adapter->hwflags & HWFLAGS_PHY_PWR_BIT) { */
685 	if (hw->mac_type >= e1000_82544 &&
686 	    hw->autoneg == 1 &&
687 	    hw->autoneg_advertised == ADVERTISE_1000_FULL) {
688 		u32 ctrl = er32(CTRL);
689 		/* clear phy power management bit if we are in gig only mode,
690 		 * which if enabled will attempt negotiation to 100Mb, which
691 		 * can cause a loss of link at power off or driver unload
692 		 */
693 		ctrl &= ~E1000_CTRL_SWDPIN3;
694 		ew32(CTRL, ctrl);
695 	}
696 
697 	/* Enable h/w to recognize an 802.1Q VLAN Ethernet packet */
698 	ew32(VET, ETHERNET_IEEE_VLAN_TYPE);
699 
700 	e1000_reset_adaptive(hw);
701 	e1000_phy_get_info(hw, &adapter->phy_info);
702 
703 	e1000_release_manageability(adapter);
704 }
705 
706 /* Dump the eeprom for users having checksum issues */
707 static void e1000_dump_eeprom(struct e1000_adapter *adapter)
708 {
709 	struct net_device *netdev = adapter->netdev;
710 	struct ethtool_eeprom eeprom;
711 	const struct ethtool_ops *ops = netdev->ethtool_ops;
712 	u8 *data;
713 	int i;
714 	u16 csum_old, csum_new = 0;
715 
716 	eeprom.len = ops->get_eeprom_len(netdev);
717 	eeprom.offset = 0;
718 
719 	data = kmalloc(eeprom.len, GFP_KERNEL);
720 	if (!data)
721 		return;
722 
723 	ops->get_eeprom(netdev, &eeprom, data);
724 
725 	csum_old = (data[EEPROM_CHECKSUM_REG * 2]) +
726 		   (data[EEPROM_CHECKSUM_REG * 2 + 1] << 8);
727 	for (i = 0; i < EEPROM_CHECKSUM_REG * 2; i += 2)
728 		csum_new += data[i] + (data[i + 1] << 8);
729 	csum_new = EEPROM_SUM - csum_new;
730 
731 	pr_err("/*********************/\n");
732 	pr_err("Current EEPROM Checksum : 0x%04x\n", csum_old);
733 	pr_err("Calculated              : 0x%04x\n", csum_new);
734 
735 	pr_err("Offset    Values\n");
736 	pr_err("========  ======\n");
737 	print_hex_dump(KERN_ERR, "", DUMP_PREFIX_OFFSET, 16, 1, data, 128, 0);
738 
739 	pr_err("Include this output when contacting your support provider.\n");
740 	pr_err("This is not a software error! Something bad happened to\n");
741 	pr_err("your hardware or EEPROM image. Ignoring this problem could\n");
742 	pr_err("result in further problems, possibly loss of data,\n");
743 	pr_err("corruption or system hangs!\n");
744 	pr_err("The MAC Address will be reset to 00:00:00:00:00:00,\n");
745 	pr_err("which is invalid and requires you to set the proper MAC\n");
746 	pr_err("address manually before continuing to enable this network\n");
747 	pr_err("device. Please inspect the EEPROM dump and report the\n");
748 	pr_err("issue to your hardware vendor or Intel Customer Support.\n");
749 	pr_err("/*********************/\n");
750 
751 	kfree(data);
752 }
753 
754 /**
755  * e1000_is_need_ioport - determine if an adapter needs ioport resources or not
756  * @pdev: PCI device information struct
757  *
758  * Return true if an adapter needs ioport resources
759  **/
760 static int e1000_is_need_ioport(struct pci_dev *pdev)
761 {
762 	switch (pdev->device) {
763 	case E1000_DEV_ID_82540EM:
764 	case E1000_DEV_ID_82540EM_LOM:
765 	case E1000_DEV_ID_82540EP:
766 	case E1000_DEV_ID_82540EP_LOM:
767 	case E1000_DEV_ID_82540EP_LP:
768 	case E1000_DEV_ID_82541EI:
769 	case E1000_DEV_ID_82541EI_MOBILE:
770 	case E1000_DEV_ID_82541ER:
771 	case E1000_DEV_ID_82541ER_LOM:
772 	case E1000_DEV_ID_82541GI:
773 	case E1000_DEV_ID_82541GI_LF:
774 	case E1000_DEV_ID_82541GI_MOBILE:
775 	case E1000_DEV_ID_82544EI_COPPER:
776 	case E1000_DEV_ID_82544EI_FIBER:
777 	case E1000_DEV_ID_82544GC_COPPER:
778 	case E1000_DEV_ID_82544GC_LOM:
779 	case E1000_DEV_ID_82545EM_COPPER:
780 	case E1000_DEV_ID_82545EM_FIBER:
781 	case E1000_DEV_ID_82546EB_COPPER:
782 	case E1000_DEV_ID_82546EB_FIBER:
783 	case E1000_DEV_ID_82546EB_QUAD_COPPER:
784 		return true;
785 	default:
786 		return false;
787 	}
788 }
789 
790 static netdev_features_t e1000_fix_features(struct net_device *netdev,
791 	netdev_features_t features)
792 {
793 	/* Since there is no support for separate Rx/Tx vlan accel
794 	 * enable/disable make sure Tx flag is always in same state as Rx.
795 	 */
796 	if (features & NETIF_F_HW_VLAN_CTAG_RX)
797 		features |= NETIF_F_HW_VLAN_CTAG_TX;
798 	else
799 		features &= ~NETIF_F_HW_VLAN_CTAG_TX;
800 
801 	return features;
802 }
803 
804 static int e1000_set_features(struct net_device *netdev,
805 	netdev_features_t features)
806 {
807 	struct e1000_adapter *adapter = netdev_priv(netdev);
808 	netdev_features_t changed = features ^ netdev->features;
809 
810 	if (changed & NETIF_F_HW_VLAN_CTAG_RX)
811 		e1000_vlan_mode(netdev, features);
812 
813 	if (!(changed & (NETIF_F_RXCSUM | NETIF_F_RXALL)))
814 		return 0;
815 
816 	netdev->features = features;
817 	adapter->rx_csum = !!(features & NETIF_F_RXCSUM);
818 
819 	if (netif_running(netdev))
820 		e1000_reinit_locked(adapter);
821 	else
822 		e1000_reset(adapter);
823 
824 	return 1;
825 }
826 
827 static const struct net_device_ops e1000_netdev_ops = {
828 	.ndo_open		= e1000_open,
829 	.ndo_stop		= e1000_close,
830 	.ndo_start_xmit		= e1000_xmit_frame,
831 	.ndo_set_rx_mode	= e1000_set_rx_mode,
832 	.ndo_set_mac_address	= e1000_set_mac,
833 	.ndo_tx_timeout		= e1000_tx_timeout,
834 	.ndo_change_mtu		= e1000_change_mtu,
835 	.ndo_do_ioctl		= e1000_ioctl,
836 	.ndo_validate_addr	= eth_validate_addr,
837 	.ndo_vlan_rx_add_vid	= e1000_vlan_rx_add_vid,
838 	.ndo_vlan_rx_kill_vid	= e1000_vlan_rx_kill_vid,
839 #ifdef CONFIG_NET_POLL_CONTROLLER
840 	.ndo_poll_controller	= e1000_netpoll,
841 #endif
842 	.ndo_fix_features	= e1000_fix_features,
843 	.ndo_set_features	= e1000_set_features,
844 };
845 
846 /**
847  * e1000_init_hw_struct - initialize members of hw struct
848  * @adapter: board private struct
849  * @hw: structure used by e1000_hw.c
850  *
851  * Factors out initialization of the e1000_hw struct to its own function
852  * that can be called very early at init (just after struct allocation).
853  * Fields are initialized based on PCI device information and
854  * OS network device settings (MTU size).
855  * Returns negative error codes if MAC type setup fails.
856  */
857 static int e1000_init_hw_struct(struct e1000_adapter *adapter,
858 				struct e1000_hw *hw)
859 {
860 	struct pci_dev *pdev = adapter->pdev;
861 
862 	/* PCI config space info */
863 	hw->vendor_id = pdev->vendor;
864 	hw->device_id = pdev->device;
865 	hw->subsystem_vendor_id = pdev->subsystem_vendor;
866 	hw->subsystem_id = pdev->subsystem_device;
867 	hw->revision_id = pdev->revision;
868 
869 	pci_read_config_word(pdev, PCI_COMMAND, &hw->pci_cmd_word);
870 
871 	hw->max_frame_size = adapter->netdev->mtu +
872 			     ENET_HEADER_SIZE + ETHERNET_FCS_SIZE;
873 	hw->min_frame_size = MINIMUM_ETHERNET_FRAME_SIZE;
874 
875 	/* identify the MAC */
876 	if (e1000_set_mac_type(hw)) {
877 		e_err(probe, "Unknown MAC Type\n");
878 		return -EIO;
879 	}
880 
881 	switch (hw->mac_type) {
882 	default:
883 		break;
884 	case e1000_82541:
885 	case e1000_82547:
886 	case e1000_82541_rev_2:
887 	case e1000_82547_rev_2:
888 		hw->phy_init_script = 1;
889 		break;
890 	}
891 
892 	e1000_set_media_type(hw);
893 	e1000_get_bus_info(hw);
894 
895 	hw->wait_autoneg_complete = false;
896 	hw->tbi_compatibility_en = true;
897 	hw->adaptive_ifs = true;
898 
899 	/* Copper options */
900 
901 	if (hw->media_type == e1000_media_type_copper) {
902 		hw->mdix = AUTO_ALL_MODES;
903 		hw->disable_polarity_correction = false;
904 		hw->master_slave = E1000_MASTER_SLAVE;
905 	}
906 
907 	return 0;
908 }
909 
910 /**
911  * e1000_probe - Device Initialization Routine
912  * @pdev: PCI device information struct
913  * @ent: entry in e1000_pci_tbl
914  *
915  * Returns 0 on success, negative on failure
916  *
917  * e1000_probe initializes an adapter identified by a pci_dev structure.
918  * The OS initialization, configuring of the adapter private structure,
919  * and a hardware reset occur.
920  **/
921 static int e1000_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
922 {
923 	struct net_device *netdev;
924 	struct e1000_adapter *adapter = NULL;
925 	struct e1000_hw *hw;
926 
927 	static int cards_found;
928 	static int global_quad_port_a; /* global ksp3 port a indication */
929 	int i, err, pci_using_dac;
930 	u16 eeprom_data = 0;
931 	u16 tmp = 0;
932 	u16 eeprom_apme_mask = E1000_EEPROM_APME;
933 	int bars, need_ioport;
934 	bool disable_dev = false;
935 
936 	/* do not allocate ioport bars when not needed */
937 	need_ioport = e1000_is_need_ioport(pdev);
938 	if (need_ioport) {
939 		bars = pci_select_bars(pdev, IORESOURCE_MEM | IORESOURCE_IO);
940 		err = pci_enable_device(pdev);
941 	} else {
942 		bars = pci_select_bars(pdev, IORESOURCE_MEM);
943 		err = pci_enable_device_mem(pdev);
944 	}
945 	if (err)
946 		return err;
947 
948 	err = pci_request_selected_regions(pdev, bars, e1000_driver_name);
949 	if (err)
950 		goto err_pci_reg;
951 
952 	pci_set_master(pdev);
953 	err = pci_save_state(pdev);
954 	if (err)
955 		goto err_alloc_etherdev;
956 
957 	err = -ENOMEM;
958 	netdev = alloc_etherdev(sizeof(struct e1000_adapter));
959 	if (!netdev)
960 		goto err_alloc_etherdev;
961 
962 	SET_NETDEV_DEV(netdev, &pdev->dev);
963 
964 	pci_set_drvdata(pdev, netdev);
965 	adapter = netdev_priv(netdev);
966 	adapter->netdev = netdev;
967 	adapter->pdev = pdev;
968 	adapter->msg_enable = netif_msg_init(debug, DEFAULT_MSG_ENABLE);
969 	adapter->bars = bars;
970 	adapter->need_ioport = need_ioport;
971 
972 	hw = &adapter->hw;
973 	hw->back = adapter;
974 
975 	err = -EIO;
976 	hw->hw_addr = pci_ioremap_bar(pdev, BAR_0);
977 	if (!hw->hw_addr)
978 		goto err_ioremap;
979 
980 	if (adapter->need_ioport) {
981 		for (i = BAR_1; i < PCI_STD_NUM_BARS; i++) {
982 			if (pci_resource_len(pdev, i) == 0)
983 				continue;
984 			if (pci_resource_flags(pdev, i) & IORESOURCE_IO) {
985 				hw->io_base = pci_resource_start(pdev, i);
986 				break;
987 			}
988 		}
989 	}
990 
991 	/* make ready for any if (hw->...) below */
992 	err = e1000_init_hw_struct(adapter, hw);
993 	if (err)
994 		goto err_sw_init;
995 
996 	/* there is a workaround being applied below that limits
997 	 * 64-bit DMA addresses to 64-bit hardware.  There are some
998 	 * 32-bit adapters that Tx hang when given 64-bit DMA addresses
999 	 */
1000 	pci_using_dac = 0;
1001 	if ((hw->bus_type == e1000_bus_type_pcix) &&
1002 	    !dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64))) {
1003 		pci_using_dac = 1;
1004 	} else {
1005 		err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
1006 		if (err) {
1007 			pr_err("No usable DMA config, aborting\n");
1008 			goto err_dma;
1009 		}
1010 	}
1011 
1012 	netdev->netdev_ops = &e1000_netdev_ops;
1013 	e1000_set_ethtool_ops(netdev);
1014 	netdev->watchdog_timeo = 5 * HZ;
1015 	netif_napi_add(netdev, &adapter->napi, e1000_clean, 64);
1016 
1017 	strncpy(netdev->name, pci_name(pdev), sizeof(netdev->name) - 1);
1018 
1019 	adapter->bd_number = cards_found;
1020 
1021 	/* setup the private structure */
1022 
1023 	err = e1000_sw_init(adapter);
1024 	if (err)
1025 		goto err_sw_init;
1026 
1027 	err = -EIO;
1028 	if (hw->mac_type == e1000_ce4100) {
1029 		hw->ce4100_gbe_mdio_base_virt =
1030 					ioremap(pci_resource_start(pdev, BAR_1),
1031 						pci_resource_len(pdev, BAR_1));
1032 
1033 		if (!hw->ce4100_gbe_mdio_base_virt)
1034 			goto err_mdio_ioremap;
1035 	}
1036 
1037 	if (hw->mac_type >= e1000_82543) {
1038 		netdev->hw_features = NETIF_F_SG |
1039 				   NETIF_F_HW_CSUM |
1040 				   NETIF_F_HW_VLAN_CTAG_RX;
1041 		netdev->features = NETIF_F_HW_VLAN_CTAG_TX |
1042 				   NETIF_F_HW_VLAN_CTAG_FILTER;
1043 	}
1044 
1045 	if ((hw->mac_type >= e1000_82544) &&
1046 	   (hw->mac_type != e1000_82547))
1047 		netdev->hw_features |= NETIF_F_TSO;
1048 
1049 	netdev->priv_flags |= IFF_SUPP_NOFCS;
1050 
1051 	netdev->features |= netdev->hw_features;
1052 	netdev->hw_features |= (NETIF_F_RXCSUM |
1053 				NETIF_F_RXALL |
1054 				NETIF_F_RXFCS);
1055 
1056 	if (pci_using_dac) {
1057 		netdev->features |= NETIF_F_HIGHDMA;
1058 		netdev->vlan_features |= NETIF_F_HIGHDMA;
1059 	}
1060 
1061 	netdev->vlan_features |= (NETIF_F_TSO |
1062 				  NETIF_F_HW_CSUM |
1063 				  NETIF_F_SG);
1064 
1065 	/* Do not set IFF_UNICAST_FLT for VMWare's 82545EM */
1066 	if (hw->device_id != E1000_DEV_ID_82545EM_COPPER ||
1067 	    hw->subsystem_vendor_id != PCI_VENDOR_ID_VMWARE)
1068 		netdev->priv_flags |= IFF_UNICAST_FLT;
1069 
1070 	/* MTU range: 46 - 16110 */
1071 	netdev->min_mtu = ETH_ZLEN - ETH_HLEN;
1072 	netdev->max_mtu = MAX_JUMBO_FRAME_SIZE - (ETH_HLEN + ETH_FCS_LEN);
1073 
1074 	adapter->en_mng_pt = e1000_enable_mng_pass_thru(hw);
1075 
1076 	/* initialize eeprom parameters */
1077 	if (e1000_init_eeprom_params(hw)) {
1078 		e_err(probe, "EEPROM initialization failed\n");
1079 		goto err_eeprom;
1080 	}
1081 
1082 	/* before reading the EEPROM, reset the controller to
1083 	 * put the device in a known good starting state
1084 	 */
1085 
1086 	e1000_reset_hw(hw);
1087 
1088 	/* make sure the EEPROM is good */
1089 	if (e1000_validate_eeprom_checksum(hw) < 0) {
1090 		e_err(probe, "The EEPROM Checksum Is Not Valid\n");
1091 		e1000_dump_eeprom(adapter);
1092 		/* set MAC address to all zeroes to invalidate and temporary
1093 		 * disable this device for the user. This blocks regular
1094 		 * traffic while still permitting ethtool ioctls from reaching
1095 		 * the hardware as well as allowing the user to run the
1096 		 * interface after manually setting a hw addr using
1097 		 * `ip set address`
1098 		 */
1099 		memset(hw->mac_addr, 0, netdev->addr_len);
1100 	} else {
1101 		/* copy the MAC address out of the EEPROM */
1102 		if (e1000_read_mac_addr(hw))
1103 			e_err(probe, "EEPROM Read Error\n");
1104 	}
1105 	/* don't block initialization here due to bad MAC address */
1106 	memcpy(netdev->dev_addr, hw->mac_addr, netdev->addr_len);
1107 
1108 	if (!is_valid_ether_addr(netdev->dev_addr))
1109 		e_err(probe, "Invalid MAC Address\n");
1110 
1111 
1112 	INIT_DELAYED_WORK(&adapter->watchdog_task, e1000_watchdog);
1113 	INIT_DELAYED_WORK(&adapter->fifo_stall_task,
1114 			  e1000_82547_tx_fifo_stall_task);
1115 	INIT_DELAYED_WORK(&adapter->phy_info_task, e1000_update_phy_info_task);
1116 	INIT_WORK(&adapter->reset_task, e1000_reset_task);
1117 
1118 	e1000_check_options(adapter);
1119 
1120 	/* Initial Wake on LAN setting
1121 	 * If APM wake is enabled in the EEPROM,
1122 	 * enable the ACPI Magic Packet filter
1123 	 */
1124 
1125 	switch (hw->mac_type) {
1126 	case e1000_82542_rev2_0:
1127 	case e1000_82542_rev2_1:
1128 	case e1000_82543:
1129 		break;
1130 	case e1000_82544:
1131 		e1000_read_eeprom(hw,
1132 			EEPROM_INIT_CONTROL2_REG, 1, &eeprom_data);
1133 		eeprom_apme_mask = E1000_EEPROM_82544_APM;
1134 		break;
1135 	case e1000_82546:
1136 	case e1000_82546_rev_3:
1137 		if (er32(STATUS) & E1000_STATUS_FUNC_1) {
1138 			e1000_read_eeprom(hw,
1139 				EEPROM_INIT_CONTROL3_PORT_B, 1, &eeprom_data);
1140 			break;
1141 		}
1142 		fallthrough;
1143 	default:
1144 		e1000_read_eeprom(hw,
1145 			EEPROM_INIT_CONTROL3_PORT_A, 1, &eeprom_data);
1146 		break;
1147 	}
1148 	if (eeprom_data & eeprom_apme_mask)
1149 		adapter->eeprom_wol |= E1000_WUFC_MAG;
1150 
1151 	/* now that we have the eeprom settings, apply the special cases
1152 	 * where the eeprom may be wrong or the board simply won't support
1153 	 * wake on lan on a particular port
1154 	 */
1155 	switch (pdev->device) {
1156 	case E1000_DEV_ID_82546GB_PCIE:
1157 		adapter->eeprom_wol = 0;
1158 		break;
1159 	case E1000_DEV_ID_82546EB_FIBER:
1160 	case E1000_DEV_ID_82546GB_FIBER:
1161 		/* Wake events only supported on port A for dual fiber
1162 		 * regardless of eeprom setting
1163 		 */
1164 		if (er32(STATUS) & E1000_STATUS_FUNC_1)
1165 			adapter->eeprom_wol = 0;
1166 		break;
1167 	case E1000_DEV_ID_82546GB_QUAD_COPPER_KSP3:
1168 		/* if quad port adapter, disable WoL on all but port A */
1169 		if (global_quad_port_a != 0)
1170 			adapter->eeprom_wol = 0;
1171 		else
1172 			adapter->quad_port_a = true;
1173 		/* Reset for multiple quad port adapters */
1174 		if (++global_quad_port_a == 4)
1175 			global_quad_port_a = 0;
1176 		break;
1177 	}
1178 
1179 	/* initialize the wol settings based on the eeprom settings */
1180 	adapter->wol = adapter->eeprom_wol;
1181 	device_set_wakeup_enable(&adapter->pdev->dev, adapter->wol);
1182 
1183 	/* Auto detect PHY address */
1184 	if (hw->mac_type == e1000_ce4100) {
1185 		for (i = 0; i < 32; i++) {
1186 			hw->phy_addr = i;
1187 			e1000_read_phy_reg(hw, PHY_ID2, &tmp);
1188 
1189 			if (tmp != 0 && tmp != 0xFF)
1190 				break;
1191 		}
1192 
1193 		if (i >= 32)
1194 			goto err_eeprom;
1195 	}
1196 
1197 	/* reset the hardware with the new settings */
1198 	e1000_reset(adapter);
1199 
1200 	strcpy(netdev->name, "eth%d");
1201 	err = register_netdev(netdev);
1202 	if (err)
1203 		goto err_register;
1204 
1205 	e1000_vlan_filter_on_off(adapter, false);
1206 
1207 	/* print bus type/speed/width info */
1208 	e_info(probe, "(PCI%s:%dMHz:%d-bit) %pM\n",
1209 	       ((hw->bus_type == e1000_bus_type_pcix) ? "-X" : ""),
1210 	       ((hw->bus_speed == e1000_bus_speed_133) ? 133 :
1211 		(hw->bus_speed == e1000_bus_speed_120) ? 120 :
1212 		(hw->bus_speed == e1000_bus_speed_100) ? 100 :
1213 		(hw->bus_speed == e1000_bus_speed_66) ? 66 : 33),
1214 	       ((hw->bus_width == e1000_bus_width_64) ? 64 : 32),
1215 	       netdev->dev_addr);
1216 
1217 	/* carrier off reporting is important to ethtool even BEFORE open */
1218 	netif_carrier_off(netdev);
1219 
1220 	e_info(probe, "Intel(R) PRO/1000 Network Connection\n");
1221 
1222 	cards_found++;
1223 	return 0;
1224 
1225 err_register:
1226 err_eeprom:
1227 	e1000_phy_hw_reset(hw);
1228 
1229 	if (hw->flash_address)
1230 		iounmap(hw->flash_address);
1231 	kfree(adapter->tx_ring);
1232 	kfree(adapter->rx_ring);
1233 err_dma:
1234 err_sw_init:
1235 err_mdio_ioremap:
1236 	iounmap(hw->ce4100_gbe_mdio_base_virt);
1237 	iounmap(hw->hw_addr);
1238 err_ioremap:
1239 	disable_dev = !test_and_set_bit(__E1000_DISABLED, &adapter->flags);
1240 	free_netdev(netdev);
1241 err_alloc_etherdev:
1242 	pci_release_selected_regions(pdev, bars);
1243 err_pci_reg:
1244 	if (!adapter || disable_dev)
1245 		pci_disable_device(pdev);
1246 	return err;
1247 }
1248 
1249 /**
1250  * e1000_remove - Device Removal Routine
1251  * @pdev: PCI device information struct
1252  *
1253  * e1000_remove is called by the PCI subsystem to alert the driver
1254  * that it should release a PCI device. That could be caused by a
1255  * Hot-Plug event, or because the driver is going to be removed from
1256  * memory.
1257  **/
1258 static void e1000_remove(struct pci_dev *pdev)
1259 {
1260 	struct net_device *netdev = pci_get_drvdata(pdev);
1261 	struct e1000_adapter *adapter = netdev_priv(netdev);
1262 	struct e1000_hw *hw = &adapter->hw;
1263 	bool disable_dev;
1264 
1265 	e1000_down_and_stop(adapter);
1266 	e1000_release_manageability(adapter);
1267 
1268 	unregister_netdev(netdev);
1269 
1270 	e1000_phy_hw_reset(hw);
1271 
1272 	kfree(adapter->tx_ring);
1273 	kfree(adapter->rx_ring);
1274 
1275 	if (hw->mac_type == e1000_ce4100)
1276 		iounmap(hw->ce4100_gbe_mdio_base_virt);
1277 	iounmap(hw->hw_addr);
1278 	if (hw->flash_address)
1279 		iounmap(hw->flash_address);
1280 	pci_release_selected_regions(pdev, adapter->bars);
1281 
1282 	disable_dev = !test_and_set_bit(__E1000_DISABLED, &adapter->flags);
1283 	free_netdev(netdev);
1284 
1285 	if (disable_dev)
1286 		pci_disable_device(pdev);
1287 }
1288 
1289 /**
1290  * e1000_sw_init - Initialize general software structures (struct e1000_adapter)
1291  * @adapter: board private structure to initialize
1292  *
1293  * e1000_sw_init initializes the Adapter private data structure.
1294  * e1000_init_hw_struct MUST be called before this function
1295  **/
1296 static int e1000_sw_init(struct e1000_adapter *adapter)
1297 {
1298 	adapter->rx_buffer_len = MAXIMUM_ETHERNET_VLAN_SIZE;
1299 
1300 	adapter->num_tx_queues = 1;
1301 	adapter->num_rx_queues = 1;
1302 
1303 	if (e1000_alloc_queues(adapter)) {
1304 		e_err(probe, "Unable to allocate memory for queues\n");
1305 		return -ENOMEM;
1306 	}
1307 
1308 	/* Explicitly disable IRQ since the NIC can be in any state. */
1309 	e1000_irq_disable(adapter);
1310 
1311 	spin_lock_init(&adapter->stats_lock);
1312 
1313 	set_bit(__E1000_DOWN, &adapter->flags);
1314 
1315 	return 0;
1316 }
1317 
1318 /**
1319  * e1000_alloc_queues - Allocate memory for all rings
1320  * @adapter: board private structure to initialize
1321  *
1322  * We allocate one ring per queue at run-time since we don't know the
1323  * number of queues at compile-time.
1324  **/
1325 static int e1000_alloc_queues(struct e1000_adapter *adapter)
1326 {
1327 	adapter->tx_ring = kcalloc(adapter->num_tx_queues,
1328 				   sizeof(struct e1000_tx_ring), GFP_KERNEL);
1329 	if (!adapter->tx_ring)
1330 		return -ENOMEM;
1331 
1332 	adapter->rx_ring = kcalloc(adapter->num_rx_queues,
1333 				   sizeof(struct e1000_rx_ring), GFP_KERNEL);
1334 	if (!adapter->rx_ring) {
1335 		kfree(adapter->tx_ring);
1336 		return -ENOMEM;
1337 	}
1338 
1339 	return E1000_SUCCESS;
1340 }
1341 
1342 /**
1343  * e1000_open - Called when a network interface is made active
1344  * @netdev: network interface device structure
1345  *
1346  * Returns 0 on success, negative value on failure
1347  *
1348  * The open entry point is called when a network interface is made
1349  * active by the system (IFF_UP).  At this point all resources needed
1350  * for transmit and receive operations are allocated, the interrupt
1351  * handler is registered with the OS, the watchdog task is started,
1352  * and the stack is notified that the interface is ready.
1353  **/
1354 int e1000_open(struct net_device *netdev)
1355 {
1356 	struct e1000_adapter *adapter = netdev_priv(netdev);
1357 	struct e1000_hw *hw = &adapter->hw;
1358 	int err;
1359 
1360 	/* disallow open during test */
1361 	if (test_bit(__E1000_TESTING, &adapter->flags))
1362 		return -EBUSY;
1363 
1364 	netif_carrier_off(netdev);
1365 
1366 	/* allocate transmit descriptors */
1367 	err = e1000_setup_all_tx_resources(adapter);
1368 	if (err)
1369 		goto err_setup_tx;
1370 
1371 	/* allocate receive descriptors */
1372 	err = e1000_setup_all_rx_resources(adapter);
1373 	if (err)
1374 		goto err_setup_rx;
1375 
1376 	e1000_power_up_phy(adapter);
1377 
1378 	adapter->mng_vlan_id = E1000_MNG_VLAN_NONE;
1379 	if ((hw->mng_cookie.status &
1380 			  E1000_MNG_DHCP_COOKIE_STATUS_VLAN_SUPPORT)) {
1381 		e1000_update_mng_vlan(adapter);
1382 	}
1383 
1384 	/* before we allocate an interrupt, we must be ready to handle it.
1385 	 * Setting DEBUG_SHIRQ in the kernel makes it fire an interrupt
1386 	 * as soon as we call pci_request_irq, so we have to setup our
1387 	 * clean_rx handler before we do so.
1388 	 */
1389 	e1000_configure(adapter);
1390 
1391 	err = e1000_request_irq(adapter);
1392 	if (err)
1393 		goto err_req_irq;
1394 
1395 	/* From here on the code is the same as e1000_up() */
1396 	clear_bit(__E1000_DOWN, &adapter->flags);
1397 
1398 	napi_enable(&adapter->napi);
1399 
1400 	e1000_irq_enable(adapter);
1401 
1402 	netif_start_queue(netdev);
1403 
1404 	/* fire a link status change interrupt to start the watchdog */
1405 	ew32(ICS, E1000_ICS_LSC);
1406 
1407 	return E1000_SUCCESS;
1408 
1409 err_req_irq:
1410 	e1000_power_down_phy(adapter);
1411 	e1000_free_all_rx_resources(adapter);
1412 err_setup_rx:
1413 	e1000_free_all_tx_resources(adapter);
1414 err_setup_tx:
1415 	e1000_reset(adapter);
1416 
1417 	return err;
1418 }
1419 
1420 /**
1421  * e1000_close - Disables a network interface
1422  * @netdev: network interface device structure
1423  *
1424  * Returns 0, this is not allowed to fail
1425  *
1426  * The close entry point is called when an interface is de-activated
1427  * by the OS.  The hardware is still under the drivers control, but
1428  * needs to be disabled.  A global MAC reset is issued to stop the
1429  * hardware, and all transmit and receive resources are freed.
1430  **/
1431 int e1000_close(struct net_device *netdev)
1432 {
1433 	struct e1000_adapter *adapter = netdev_priv(netdev);
1434 	struct e1000_hw *hw = &adapter->hw;
1435 	int count = E1000_CHECK_RESET_COUNT;
1436 
1437 	while (test_and_set_bit(__E1000_RESETTING, &adapter->flags) && count--)
1438 		usleep_range(10000, 20000);
1439 
1440 	WARN_ON(count < 0);
1441 
1442 	/* signal that we're down so that the reset task will no longer run */
1443 	set_bit(__E1000_DOWN, &adapter->flags);
1444 	clear_bit(__E1000_RESETTING, &adapter->flags);
1445 
1446 	e1000_down(adapter);
1447 	e1000_power_down_phy(adapter);
1448 	e1000_free_irq(adapter);
1449 
1450 	e1000_free_all_tx_resources(adapter);
1451 	e1000_free_all_rx_resources(adapter);
1452 
1453 	/* kill manageability vlan ID if supported, but not if a vlan with
1454 	 * the same ID is registered on the host OS (let 8021q kill it)
1455 	 */
1456 	if ((hw->mng_cookie.status &
1457 	     E1000_MNG_DHCP_COOKIE_STATUS_VLAN_SUPPORT) &&
1458 	    !test_bit(adapter->mng_vlan_id, adapter->active_vlans)) {
1459 		e1000_vlan_rx_kill_vid(netdev, htons(ETH_P_8021Q),
1460 				       adapter->mng_vlan_id);
1461 	}
1462 
1463 	return 0;
1464 }
1465 
1466 /**
1467  * e1000_check_64k_bound - check that memory doesn't cross 64kB boundary
1468  * @adapter: address of board private structure
1469  * @start: address of beginning of memory
1470  * @len: length of memory
1471  **/
1472 static bool e1000_check_64k_bound(struct e1000_adapter *adapter, void *start,
1473 				  unsigned long len)
1474 {
1475 	struct e1000_hw *hw = &adapter->hw;
1476 	unsigned long begin = (unsigned long)start;
1477 	unsigned long end = begin + len;
1478 
1479 	/* First rev 82545 and 82546 need to not allow any memory
1480 	 * write location to cross 64k boundary due to errata 23
1481 	 */
1482 	if (hw->mac_type == e1000_82545 ||
1483 	    hw->mac_type == e1000_ce4100 ||
1484 	    hw->mac_type == e1000_82546) {
1485 		return ((begin ^ (end - 1)) >> 16) == 0;
1486 	}
1487 
1488 	return true;
1489 }
1490 
1491 /**
1492  * e1000_setup_tx_resources - allocate Tx resources (Descriptors)
1493  * @adapter: board private structure
1494  * @txdr:    tx descriptor ring (for a specific queue) to setup
1495  *
1496  * Return 0 on success, negative on failure
1497  **/
1498 static int e1000_setup_tx_resources(struct e1000_adapter *adapter,
1499 				    struct e1000_tx_ring *txdr)
1500 {
1501 	struct pci_dev *pdev = adapter->pdev;
1502 	int size;
1503 
1504 	size = sizeof(struct e1000_tx_buffer) * txdr->count;
1505 	txdr->buffer_info = vzalloc(size);
1506 	if (!txdr->buffer_info)
1507 		return -ENOMEM;
1508 
1509 	/* round up to nearest 4K */
1510 
1511 	txdr->size = txdr->count * sizeof(struct e1000_tx_desc);
1512 	txdr->size = ALIGN(txdr->size, 4096);
1513 
1514 	txdr->desc = dma_alloc_coherent(&pdev->dev, txdr->size, &txdr->dma,
1515 					GFP_KERNEL);
1516 	if (!txdr->desc) {
1517 setup_tx_desc_die:
1518 		vfree(txdr->buffer_info);
1519 		return -ENOMEM;
1520 	}
1521 
1522 	/* Fix for errata 23, can't cross 64kB boundary */
1523 	if (!e1000_check_64k_bound(adapter, txdr->desc, txdr->size)) {
1524 		void *olddesc = txdr->desc;
1525 		dma_addr_t olddma = txdr->dma;
1526 		e_err(tx_err, "txdr align check failed: %u bytes at %p\n",
1527 		      txdr->size, txdr->desc);
1528 		/* Try again, without freeing the previous */
1529 		txdr->desc = dma_alloc_coherent(&pdev->dev, txdr->size,
1530 						&txdr->dma, GFP_KERNEL);
1531 		/* Failed allocation, critical failure */
1532 		if (!txdr->desc) {
1533 			dma_free_coherent(&pdev->dev, txdr->size, olddesc,
1534 					  olddma);
1535 			goto setup_tx_desc_die;
1536 		}
1537 
1538 		if (!e1000_check_64k_bound(adapter, txdr->desc, txdr->size)) {
1539 			/* give up */
1540 			dma_free_coherent(&pdev->dev, txdr->size, txdr->desc,
1541 					  txdr->dma);
1542 			dma_free_coherent(&pdev->dev, txdr->size, olddesc,
1543 					  olddma);
1544 			e_err(probe, "Unable to allocate aligned memory "
1545 			      "for the transmit descriptor ring\n");
1546 			vfree(txdr->buffer_info);
1547 			return -ENOMEM;
1548 		} else {
1549 			/* Free old allocation, new allocation was successful */
1550 			dma_free_coherent(&pdev->dev, txdr->size, olddesc,
1551 					  olddma);
1552 		}
1553 	}
1554 	memset(txdr->desc, 0, txdr->size);
1555 
1556 	txdr->next_to_use = 0;
1557 	txdr->next_to_clean = 0;
1558 
1559 	return 0;
1560 }
1561 
1562 /**
1563  * e1000_setup_all_tx_resources - wrapper to allocate Tx resources
1564  * 				  (Descriptors) for all queues
1565  * @adapter: board private structure
1566  *
1567  * Return 0 on success, negative on failure
1568  **/
1569 int e1000_setup_all_tx_resources(struct e1000_adapter *adapter)
1570 {
1571 	int i, err = 0;
1572 
1573 	for (i = 0; i < adapter->num_tx_queues; i++) {
1574 		err = e1000_setup_tx_resources(adapter, &adapter->tx_ring[i]);
1575 		if (err) {
1576 			e_err(probe, "Allocation for Tx Queue %u failed\n", i);
1577 			for (i-- ; i >= 0; i--)
1578 				e1000_free_tx_resources(adapter,
1579 							&adapter->tx_ring[i]);
1580 			break;
1581 		}
1582 	}
1583 
1584 	return err;
1585 }
1586 
1587 /**
1588  * e1000_configure_tx - Configure 8254x Transmit Unit after Reset
1589  * @adapter: board private structure
1590  *
1591  * Configure the Tx unit of the MAC after a reset.
1592  **/
1593 static void e1000_configure_tx(struct e1000_adapter *adapter)
1594 {
1595 	u64 tdba;
1596 	struct e1000_hw *hw = &adapter->hw;
1597 	u32 tdlen, tctl, tipg;
1598 	u32 ipgr1, ipgr2;
1599 
1600 	/* Setup the HW Tx Head and Tail descriptor pointers */
1601 
1602 	switch (adapter->num_tx_queues) {
1603 	case 1:
1604 	default:
1605 		tdba = adapter->tx_ring[0].dma;
1606 		tdlen = adapter->tx_ring[0].count *
1607 			sizeof(struct e1000_tx_desc);
1608 		ew32(TDLEN, tdlen);
1609 		ew32(TDBAH, (tdba >> 32));
1610 		ew32(TDBAL, (tdba & 0x00000000ffffffffULL));
1611 		ew32(TDT, 0);
1612 		ew32(TDH, 0);
1613 		adapter->tx_ring[0].tdh = ((hw->mac_type >= e1000_82543) ?
1614 					   E1000_TDH : E1000_82542_TDH);
1615 		adapter->tx_ring[0].tdt = ((hw->mac_type >= e1000_82543) ?
1616 					   E1000_TDT : E1000_82542_TDT);
1617 		break;
1618 	}
1619 
1620 	/* Set the default values for the Tx Inter Packet Gap timer */
1621 	if ((hw->media_type == e1000_media_type_fiber ||
1622 	     hw->media_type == e1000_media_type_internal_serdes))
1623 		tipg = DEFAULT_82543_TIPG_IPGT_FIBER;
1624 	else
1625 		tipg = DEFAULT_82543_TIPG_IPGT_COPPER;
1626 
1627 	switch (hw->mac_type) {
1628 	case e1000_82542_rev2_0:
1629 	case e1000_82542_rev2_1:
1630 		tipg = DEFAULT_82542_TIPG_IPGT;
1631 		ipgr1 = DEFAULT_82542_TIPG_IPGR1;
1632 		ipgr2 = DEFAULT_82542_TIPG_IPGR2;
1633 		break;
1634 	default:
1635 		ipgr1 = DEFAULT_82543_TIPG_IPGR1;
1636 		ipgr2 = DEFAULT_82543_TIPG_IPGR2;
1637 		break;
1638 	}
1639 	tipg |= ipgr1 << E1000_TIPG_IPGR1_SHIFT;
1640 	tipg |= ipgr2 << E1000_TIPG_IPGR2_SHIFT;
1641 	ew32(TIPG, tipg);
1642 
1643 	/* Set the Tx Interrupt Delay register */
1644 
1645 	ew32(TIDV, adapter->tx_int_delay);
1646 	if (hw->mac_type >= e1000_82540)
1647 		ew32(TADV, adapter->tx_abs_int_delay);
1648 
1649 	/* Program the Transmit Control Register */
1650 
1651 	tctl = er32(TCTL);
1652 	tctl &= ~E1000_TCTL_CT;
1653 	tctl |= E1000_TCTL_PSP | E1000_TCTL_RTLC |
1654 		(E1000_COLLISION_THRESHOLD << E1000_CT_SHIFT);
1655 
1656 	e1000_config_collision_dist(hw);
1657 
1658 	/* Setup Transmit Descriptor Settings for eop descriptor */
1659 	adapter->txd_cmd = E1000_TXD_CMD_EOP | E1000_TXD_CMD_IFCS;
1660 
1661 	/* only set IDE if we are delaying interrupts using the timers */
1662 	if (adapter->tx_int_delay)
1663 		adapter->txd_cmd |= E1000_TXD_CMD_IDE;
1664 
1665 	if (hw->mac_type < e1000_82543)
1666 		adapter->txd_cmd |= E1000_TXD_CMD_RPS;
1667 	else
1668 		adapter->txd_cmd |= E1000_TXD_CMD_RS;
1669 
1670 	/* Cache if we're 82544 running in PCI-X because we'll
1671 	 * need this to apply a workaround later in the send path.
1672 	 */
1673 	if (hw->mac_type == e1000_82544 &&
1674 	    hw->bus_type == e1000_bus_type_pcix)
1675 		adapter->pcix_82544 = true;
1676 
1677 	ew32(TCTL, tctl);
1678 
1679 }
1680 
1681 /**
1682  * e1000_setup_rx_resources - allocate Rx resources (Descriptors)
1683  * @adapter: board private structure
1684  * @rxdr:    rx descriptor ring (for a specific queue) to setup
1685  *
1686  * Returns 0 on success, negative on failure
1687  **/
1688 static int e1000_setup_rx_resources(struct e1000_adapter *adapter,
1689 				    struct e1000_rx_ring *rxdr)
1690 {
1691 	struct pci_dev *pdev = adapter->pdev;
1692 	int size, desc_len;
1693 
1694 	size = sizeof(struct e1000_rx_buffer) * rxdr->count;
1695 	rxdr->buffer_info = vzalloc(size);
1696 	if (!rxdr->buffer_info)
1697 		return -ENOMEM;
1698 
1699 	desc_len = sizeof(struct e1000_rx_desc);
1700 
1701 	/* Round up to nearest 4K */
1702 
1703 	rxdr->size = rxdr->count * desc_len;
1704 	rxdr->size = ALIGN(rxdr->size, 4096);
1705 
1706 	rxdr->desc = dma_alloc_coherent(&pdev->dev, rxdr->size, &rxdr->dma,
1707 					GFP_KERNEL);
1708 	if (!rxdr->desc) {
1709 setup_rx_desc_die:
1710 		vfree(rxdr->buffer_info);
1711 		return -ENOMEM;
1712 	}
1713 
1714 	/* Fix for errata 23, can't cross 64kB boundary */
1715 	if (!e1000_check_64k_bound(adapter, rxdr->desc, rxdr->size)) {
1716 		void *olddesc = rxdr->desc;
1717 		dma_addr_t olddma = rxdr->dma;
1718 		e_err(rx_err, "rxdr align check failed: %u bytes at %p\n",
1719 		      rxdr->size, rxdr->desc);
1720 		/* Try again, without freeing the previous */
1721 		rxdr->desc = dma_alloc_coherent(&pdev->dev, rxdr->size,
1722 						&rxdr->dma, GFP_KERNEL);
1723 		/* Failed allocation, critical failure */
1724 		if (!rxdr->desc) {
1725 			dma_free_coherent(&pdev->dev, rxdr->size, olddesc,
1726 					  olddma);
1727 			goto setup_rx_desc_die;
1728 		}
1729 
1730 		if (!e1000_check_64k_bound(adapter, rxdr->desc, rxdr->size)) {
1731 			/* give up */
1732 			dma_free_coherent(&pdev->dev, rxdr->size, rxdr->desc,
1733 					  rxdr->dma);
1734 			dma_free_coherent(&pdev->dev, rxdr->size, olddesc,
1735 					  olddma);
1736 			e_err(probe, "Unable to allocate aligned memory for "
1737 			      "the Rx descriptor ring\n");
1738 			goto setup_rx_desc_die;
1739 		} else {
1740 			/* Free old allocation, new allocation was successful */
1741 			dma_free_coherent(&pdev->dev, rxdr->size, olddesc,
1742 					  olddma);
1743 		}
1744 	}
1745 	memset(rxdr->desc, 0, rxdr->size);
1746 
1747 	rxdr->next_to_clean = 0;
1748 	rxdr->next_to_use = 0;
1749 	rxdr->rx_skb_top = NULL;
1750 
1751 	return 0;
1752 }
1753 
1754 /**
1755  * e1000_setup_all_rx_resources - wrapper to allocate Rx resources
1756  * 				  (Descriptors) for all queues
1757  * @adapter: board private structure
1758  *
1759  * Return 0 on success, negative on failure
1760  **/
1761 int e1000_setup_all_rx_resources(struct e1000_adapter *adapter)
1762 {
1763 	int i, err = 0;
1764 
1765 	for (i = 0; i < adapter->num_rx_queues; i++) {
1766 		err = e1000_setup_rx_resources(adapter, &adapter->rx_ring[i]);
1767 		if (err) {
1768 			e_err(probe, "Allocation for Rx Queue %u failed\n", i);
1769 			for (i-- ; i >= 0; i--)
1770 				e1000_free_rx_resources(adapter,
1771 							&adapter->rx_ring[i]);
1772 			break;
1773 		}
1774 	}
1775 
1776 	return err;
1777 }
1778 
1779 /**
1780  * e1000_setup_rctl - configure the receive control registers
1781  * @adapter: Board private structure
1782  **/
1783 static void e1000_setup_rctl(struct e1000_adapter *adapter)
1784 {
1785 	struct e1000_hw *hw = &adapter->hw;
1786 	u32 rctl;
1787 
1788 	rctl = er32(RCTL);
1789 
1790 	rctl &= ~(3 << E1000_RCTL_MO_SHIFT);
1791 
1792 	rctl |= E1000_RCTL_BAM | E1000_RCTL_LBM_NO |
1793 		E1000_RCTL_RDMTS_HALF |
1794 		(hw->mc_filter_type << E1000_RCTL_MO_SHIFT);
1795 
1796 	if (hw->tbi_compatibility_on == 1)
1797 		rctl |= E1000_RCTL_SBP;
1798 	else
1799 		rctl &= ~E1000_RCTL_SBP;
1800 
1801 	if (adapter->netdev->mtu <= ETH_DATA_LEN)
1802 		rctl &= ~E1000_RCTL_LPE;
1803 	else
1804 		rctl |= E1000_RCTL_LPE;
1805 
1806 	/* Setup buffer sizes */
1807 	rctl &= ~E1000_RCTL_SZ_4096;
1808 	rctl |= E1000_RCTL_BSEX;
1809 	switch (adapter->rx_buffer_len) {
1810 	case E1000_RXBUFFER_2048:
1811 	default:
1812 		rctl |= E1000_RCTL_SZ_2048;
1813 		rctl &= ~E1000_RCTL_BSEX;
1814 		break;
1815 	case E1000_RXBUFFER_4096:
1816 		rctl |= E1000_RCTL_SZ_4096;
1817 		break;
1818 	case E1000_RXBUFFER_8192:
1819 		rctl |= E1000_RCTL_SZ_8192;
1820 		break;
1821 	case E1000_RXBUFFER_16384:
1822 		rctl |= E1000_RCTL_SZ_16384;
1823 		break;
1824 	}
1825 
1826 	/* This is useful for sniffing bad packets. */
1827 	if (adapter->netdev->features & NETIF_F_RXALL) {
1828 		/* UPE and MPE will be handled by normal PROMISC logic
1829 		 * in e1000e_set_rx_mode
1830 		 */
1831 		rctl |= (E1000_RCTL_SBP | /* Receive bad packets */
1832 			 E1000_RCTL_BAM | /* RX All Bcast Pkts */
1833 			 E1000_RCTL_PMCF); /* RX All MAC Ctrl Pkts */
1834 
1835 		rctl &= ~(E1000_RCTL_VFE | /* Disable VLAN filter */
1836 			  E1000_RCTL_DPF | /* Allow filtered pause */
1837 			  E1000_RCTL_CFIEN); /* Dis VLAN CFIEN Filter */
1838 		/* Do not mess with E1000_CTRL_VME, it affects transmit as well,
1839 		 * and that breaks VLANs.
1840 		 */
1841 	}
1842 
1843 	ew32(RCTL, rctl);
1844 }
1845 
1846 /**
1847  * e1000_configure_rx - Configure 8254x Receive Unit after Reset
1848  * @adapter: board private structure
1849  *
1850  * Configure the Rx unit of the MAC after a reset.
1851  **/
1852 static void e1000_configure_rx(struct e1000_adapter *adapter)
1853 {
1854 	u64 rdba;
1855 	struct e1000_hw *hw = &adapter->hw;
1856 	u32 rdlen, rctl, rxcsum;
1857 
1858 	if (adapter->netdev->mtu > ETH_DATA_LEN) {
1859 		rdlen = adapter->rx_ring[0].count *
1860 			sizeof(struct e1000_rx_desc);
1861 		adapter->clean_rx = e1000_clean_jumbo_rx_irq;
1862 		adapter->alloc_rx_buf = e1000_alloc_jumbo_rx_buffers;
1863 	} else {
1864 		rdlen = adapter->rx_ring[0].count *
1865 			sizeof(struct e1000_rx_desc);
1866 		adapter->clean_rx = e1000_clean_rx_irq;
1867 		adapter->alloc_rx_buf = e1000_alloc_rx_buffers;
1868 	}
1869 
1870 	/* disable receives while setting up the descriptors */
1871 	rctl = er32(RCTL);
1872 	ew32(RCTL, rctl & ~E1000_RCTL_EN);
1873 
1874 	/* set the Receive Delay Timer Register */
1875 	ew32(RDTR, adapter->rx_int_delay);
1876 
1877 	if (hw->mac_type >= e1000_82540) {
1878 		ew32(RADV, adapter->rx_abs_int_delay);
1879 		if (adapter->itr_setting != 0)
1880 			ew32(ITR, 1000000000 / (adapter->itr * 256));
1881 	}
1882 
1883 	/* Setup the HW Rx Head and Tail Descriptor Pointers and
1884 	 * the Base and Length of the Rx Descriptor Ring
1885 	 */
1886 	switch (adapter->num_rx_queues) {
1887 	case 1:
1888 	default:
1889 		rdba = adapter->rx_ring[0].dma;
1890 		ew32(RDLEN, rdlen);
1891 		ew32(RDBAH, (rdba >> 32));
1892 		ew32(RDBAL, (rdba & 0x00000000ffffffffULL));
1893 		ew32(RDT, 0);
1894 		ew32(RDH, 0);
1895 		adapter->rx_ring[0].rdh = ((hw->mac_type >= e1000_82543) ?
1896 					   E1000_RDH : E1000_82542_RDH);
1897 		adapter->rx_ring[0].rdt = ((hw->mac_type >= e1000_82543) ?
1898 					   E1000_RDT : E1000_82542_RDT);
1899 		break;
1900 	}
1901 
1902 	/* Enable 82543 Receive Checksum Offload for TCP and UDP */
1903 	if (hw->mac_type >= e1000_82543) {
1904 		rxcsum = er32(RXCSUM);
1905 		if (adapter->rx_csum)
1906 			rxcsum |= E1000_RXCSUM_TUOFL;
1907 		else
1908 			/* don't need to clear IPPCSE as it defaults to 0 */
1909 			rxcsum &= ~E1000_RXCSUM_TUOFL;
1910 		ew32(RXCSUM, rxcsum);
1911 	}
1912 
1913 	/* Enable Receives */
1914 	ew32(RCTL, rctl | E1000_RCTL_EN);
1915 }
1916 
1917 /**
1918  * e1000_free_tx_resources - Free Tx Resources per Queue
1919  * @adapter: board private structure
1920  * @tx_ring: Tx descriptor ring for a specific queue
1921  *
1922  * Free all transmit software resources
1923  **/
1924 static void e1000_free_tx_resources(struct e1000_adapter *adapter,
1925 				    struct e1000_tx_ring *tx_ring)
1926 {
1927 	struct pci_dev *pdev = adapter->pdev;
1928 
1929 	e1000_clean_tx_ring(adapter, tx_ring);
1930 
1931 	vfree(tx_ring->buffer_info);
1932 	tx_ring->buffer_info = NULL;
1933 
1934 	dma_free_coherent(&pdev->dev, tx_ring->size, tx_ring->desc,
1935 			  tx_ring->dma);
1936 
1937 	tx_ring->desc = NULL;
1938 }
1939 
1940 /**
1941  * e1000_free_all_tx_resources - Free Tx Resources for All Queues
1942  * @adapter: board private structure
1943  *
1944  * Free all transmit software resources
1945  **/
1946 void e1000_free_all_tx_resources(struct e1000_adapter *adapter)
1947 {
1948 	int i;
1949 
1950 	for (i = 0; i < adapter->num_tx_queues; i++)
1951 		e1000_free_tx_resources(adapter, &adapter->tx_ring[i]);
1952 }
1953 
1954 static void
1955 e1000_unmap_and_free_tx_resource(struct e1000_adapter *adapter,
1956 				 struct e1000_tx_buffer *buffer_info)
1957 {
1958 	if (buffer_info->dma) {
1959 		if (buffer_info->mapped_as_page)
1960 			dma_unmap_page(&adapter->pdev->dev, buffer_info->dma,
1961 				       buffer_info->length, DMA_TO_DEVICE);
1962 		else
1963 			dma_unmap_single(&adapter->pdev->dev, buffer_info->dma,
1964 					 buffer_info->length,
1965 					 DMA_TO_DEVICE);
1966 		buffer_info->dma = 0;
1967 	}
1968 	if (buffer_info->skb) {
1969 		dev_kfree_skb_any(buffer_info->skb);
1970 		buffer_info->skb = NULL;
1971 	}
1972 	buffer_info->time_stamp = 0;
1973 	/* buffer_info must be completely set up in the transmit path */
1974 }
1975 
1976 /**
1977  * e1000_clean_tx_ring - Free Tx Buffers
1978  * @adapter: board private structure
1979  * @tx_ring: ring to be cleaned
1980  **/
1981 static void e1000_clean_tx_ring(struct e1000_adapter *adapter,
1982 				struct e1000_tx_ring *tx_ring)
1983 {
1984 	struct e1000_hw *hw = &adapter->hw;
1985 	struct e1000_tx_buffer *buffer_info;
1986 	unsigned long size;
1987 	unsigned int i;
1988 
1989 	/* Free all the Tx ring sk_buffs */
1990 
1991 	for (i = 0; i < tx_ring->count; i++) {
1992 		buffer_info = &tx_ring->buffer_info[i];
1993 		e1000_unmap_and_free_tx_resource(adapter, buffer_info);
1994 	}
1995 
1996 	netdev_reset_queue(adapter->netdev);
1997 	size = sizeof(struct e1000_tx_buffer) * tx_ring->count;
1998 	memset(tx_ring->buffer_info, 0, size);
1999 
2000 	/* Zero out the descriptor ring */
2001 
2002 	memset(tx_ring->desc, 0, tx_ring->size);
2003 
2004 	tx_ring->next_to_use = 0;
2005 	tx_ring->next_to_clean = 0;
2006 	tx_ring->last_tx_tso = false;
2007 
2008 	writel(0, hw->hw_addr + tx_ring->tdh);
2009 	writel(0, hw->hw_addr + tx_ring->tdt);
2010 }
2011 
2012 /**
2013  * e1000_clean_all_tx_rings - Free Tx Buffers for all queues
2014  * @adapter: board private structure
2015  **/
2016 static void e1000_clean_all_tx_rings(struct e1000_adapter *adapter)
2017 {
2018 	int i;
2019 
2020 	for (i = 0; i < adapter->num_tx_queues; i++)
2021 		e1000_clean_tx_ring(adapter, &adapter->tx_ring[i]);
2022 }
2023 
2024 /**
2025  * e1000_free_rx_resources - Free Rx Resources
2026  * @adapter: board private structure
2027  * @rx_ring: ring to clean the resources from
2028  *
2029  * Free all receive software resources
2030  **/
2031 static void e1000_free_rx_resources(struct e1000_adapter *adapter,
2032 				    struct e1000_rx_ring *rx_ring)
2033 {
2034 	struct pci_dev *pdev = adapter->pdev;
2035 
2036 	e1000_clean_rx_ring(adapter, rx_ring);
2037 
2038 	vfree(rx_ring->buffer_info);
2039 	rx_ring->buffer_info = NULL;
2040 
2041 	dma_free_coherent(&pdev->dev, rx_ring->size, rx_ring->desc,
2042 			  rx_ring->dma);
2043 
2044 	rx_ring->desc = NULL;
2045 }
2046 
2047 /**
2048  * e1000_free_all_rx_resources - Free Rx Resources for All Queues
2049  * @adapter: board private structure
2050  *
2051  * Free all receive software resources
2052  **/
2053 void e1000_free_all_rx_resources(struct e1000_adapter *adapter)
2054 {
2055 	int i;
2056 
2057 	for (i = 0; i < adapter->num_rx_queues; i++)
2058 		e1000_free_rx_resources(adapter, &adapter->rx_ring[i]);
2059 }
2060 
2061 #define E1000_HEADROOM (NET_SKB_PAD + NET_IP_ALIGN)
2062 static unsigned int e1000_frag_len(const struct e1000_adapter *a)
2063 {
2064 	return SKB_DATA_ALIGN(a->rx_buffer_len + E1000_HEADROOM) +
2065 		SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
2066 }
2067 
2068 static void *e1000_alloc_frag(const struct e1000_adapter *a)
2069 {
2070 	unsigned int len = e1000_frag_len(a);
2071 	u8 *data = netdev_alloc_frag(len);
2072 
2073 	if (likely(data))
2074 		data += E1000_HEADROOM;
2075 	return data;
2076 }
2077 
2078 /**
2079  * e1000_clean_rx_ring - Free Rx Buffers per Queue
2080  * @adapter: board private structure
2081  * @rx_ring: ring to free buffers from
2082  **/
2083 static void e1000_clean_rx_ring(struct e1000_adapter *adapter,
2084 				struct e1000_rx_ring *rx_ring)
2085 {
2086 	struct e1000_hw *hw = &adapter->hw;
2087 	struct e1000_rx_buffer *buffer_info;
2088 	struct pci_dev *pdev = adapter->pdev;
2089 	unsigned long size;
2090 	unsigned int i;
2091 
2092 	/* Free all the Rx netfrags */
2093 	for (i = 0; i < rx_ring->count; i++) {
2094 		buffer_info = &rx_ring->buffer_info[i];
2095 		if (adapter->clean_rx == e1000_clean_rx_irq) {
2096 			if (buffer_info->dma)
2097 				dma_unmap_single(&pdev->dev, buffer_info->dma,
2098 						 adapter->rx_buffer_len,
2099 						 DMA_FROM_DEVICE);
2100 			if (buffer_info->rxbuf.data) {
2101 				skb_free_frag(buffer_info->rxbuf.data);
2102 				buffer_info->rxbuf.data = NULL;
2103 			}
2104 		} else if (adapter->clean_rx == e1000_clean_jumbo_rx_irq) {
2105 			if (buffer_info->dma)
2106 				dma_unmap_page(&pdev->dev, buffer_info->dma,
2107 					       adapter->rx_buffer_len,
2108 					       DMA_FROM_DEVICE);
2109 			if (buffer_info->rxbuf.page) {
2110 				put_page(buffer_info->rxbuf.page);
2111 				buffer_info->rxbuf.page = NULL;
2112 			}
2113 		}
2114 
2115 		buffer_info->dma = 0;
2116 	}
2117 
2118 	/* there also may be some cached data from a chained receive */
2119 	napi_free_frags(&adapter->napi);
2120 	rx_ring->rx_skb_top = NULL;
2121 
2122 	size = sizeof(struct e1000_rx_buffer) * rx_ring->count;
2123 	memset(rx_ring->buffer_info, 0, size);
2124 
2125 	/* Zero out the descriptor ring */
2126 	memset(rx_ring->desc, 0, rx_ring->size);
2127 
2128 	rx_ring->next_to_clean = 0;
2129 	rx_ring->next_to_use = 0;
2130 
2131 	writel(0, hw->hw_addr + rx_ring->rdh);
2132 	writel(0, hw->hw_addr + rx_ring->rdt);
2133 }
2134 
2135 /**
2136  * e1000_clean_all_rx_rings - Free Rx Buffers for all queues
2137  * @adapter: board private structure
2138  **/
2139 static void e1000_clean_all_rx_rings(struct e1000_adapter *adapter)
2140 {
2141 	int i;
2142 
2143 	for (i = 0; i < adapter->num_rx_queues; i++)
2144 		e1000_clean_rx_ring(adapter, &adapter->rx_ring[i]);
2145 }
2146 
2147 /* The 82542 2.0 (revision 2) needs to have the receive unit in reset
2148  * and memory write and invalidate disabled for certain operations
2149  */
2150 static void e1000_enter_82542_rst(struct e1000_adapter *adapter)
2151 {
2152 	struct e1000_hw *hw = &adapter->hw;
2153 	struct net_device *netdev = adapter->netdev;
2154 	u32 rctl;
2155 
2156 	e1000_pci_clear_mwi(hw);
2157 
2158 	rctl = er32(RCTL);
2159 	rctl |= E1000_RCTL_RST;
2160 	ew32(RCTL, rctl);
2161 	E1000_WRITE_FLUSH();
2162 	mdelay(5);
2163 
2164 	if (netif_running(netdev))
2165 		e1000_clean_all_rx_rings(adapter);
2166 }
2167 
2168 static void e1000_leave_82542_rst(struct e1000_adapter *adapter)
2169 {
2170 	struct e1000_hw *hw = &adapter->hw;
2171 	struct net_device *netdev = adapter->netdev;
2172 	u32 rctl;
2173 
2174 	rctl = er32(RCTL);
2175 	rctl &= ~E1000_RCTL_RST;
2176 	ew32(RCTL, rctl);
2177 	E1000_WRITE_FLUSH();
2178 	mdelay(5);
2179 
2180 	if (hw->pci_cmd_word & PCI_COMMAND_INVALIDATE)
2181 		e1000_pci_set_mwi(hw);
2182 
2183 	if (netif_running(netdev)) {
2184 		/* No need to loop, because 82542 supports only 1 queue */
2185 		struct e1000_rx_ring *ring = &adapter->rx_ring[0];
2186 		e1000_configure_rx(adapter);
2187 		adapter->alloc_rx_buf(adapter, ring, E1000_DESC_UNUSED(ring));
2188 	}
2189 }
2190 
2191 /**
2192  * e1000_set_mac - Change the Ethernet Address of the NIC
2193  * @netdev: network interface device structure
2194  * @p: pointer to an address structure
2195  *
2196  * Returns 0 on success, negative on failure
2197  **/
2198 static int e1000_set_mac(struct net_device *netdev, void *p)
2199 {
2200 	struct e1000_adapter *adapter = netdev_priv(netdev);
2201 	struct e1000_hw *hw = &adapter->hw;
2202 	struct sockaddr *addr = p;
2203 
2204 	if (!is_valid_ether_addr(addr->sa_data))
2205 		return -EADDRNOTAVAIL;
2206 
2207 	/* 82542 2.0 needs to be in reset to write receive address registers */
2208 
2209 	if (hw->mac_type == e1000_82542_rev2_0)
2210 		e1000_enter_82542_rst(adapter);
2211 
2212 	memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len);
2213 	memcpy(hw->mac_addr, addr->sa_data, netdev->addr_len);
2214 
2215 	e1000_rar_set(hw, hw->mac_addr, 0);
2216 
2217 	if (hw->mac_type == e1000_82542_rev2_0)
2218 		e1000_leave_82542_rst(adapter);
2219 
2220 	return 0;
2221 }
2222 
2223 /**
2224  * e1000_set_rx_mode - Secondary Unicast, Multicast and Promiscuous mode set
2225  * @netdev: network interface device structure
2226  *
2227  * The set_rx_mode entry point is called whenever the unicast or multicast
2228  * address lists or the network interface flags are updated. This routine is
2229  * responsible for configuring the hardware for proper unicast, multicast,
2230  * promiscuous mode, and all-multi behavior.
2231  **/
2232 static void e1000_set_rx_mode(struct net_device *netdev)
2233 {
2234 	struct e1000_adapter *adapter = netdev_priv(netdev);
2235 	struct e1000_hw *hw = &adapter->hw;
2236 	struct netdev_hw_addr *ha;
2237 	bool use_uc = false;
2238 	u32 rctl;
2239 	u32 hash_value;
2240 	int i, rar_entries = E1000_RAR_ENTRIES;
2241 	int mta_reg_count = E1000_NUM_MTA_REGISTERS;
2242 	u32 *mcarray = kcalloc(mta_reg_count, sizeof(u32), GFP_ATOMIC);
2243 
2244 	if (!mcarray)
2245 		return;
2246 
2247 	/* Check for Promiscuous and All Multicast modes */
2248 
2249 	rctl = er32(RCTL);
2250 
2251 	if (netdev->flags & IFF_PROMISC) {
2252 		rctl |= (E1000_RCTL_UPE | E1000_RCTL_MPE);
2253 		rctl &= ~E1000_RCTL_VFE;
2254 	} else {
2255 		if (netdev->flags & IFF_ALLMULTI)
2256 			rctl |= E1000_RCTL_MPE;
2257 		else
2258 			rctl &= ~E1000_RCTL_MPE;
2259 		/* Enable VLAN filter if there is a VLAN */
2260 		if (e1000_vlan_used(adapter))
2261 			rctl |= E1000_RCTL_VFE;
2262 	}
2263 
2264 	if (netdev_uc_count(netdev) > rar_entries - 1) {
2265 		rctl |= E1000_RCTL_UPE;
2266 	} else if (!(netdev->flags & IFF_PROMISC)) {
2267 		rctl &= ~E1000_RCTL_UPE;
2268 		use_uc = true;
2269 	}
2270 
2271 	ew32(RCTL, rctl);
2272 
2273 	/* 82542 2.0 needs to be in reset to write receive address registers */
2274 
2275 	if (hw->mac_type == e1000_82542_rev2_0)
2276 		e1000_enter_82542_rst(adapter);
2277 
2278 	/* load the first 14 addresses into the exact filters 1-14. Unicast
2279 	 * addresses take precedence to avoid disabling unicast filtering
2280 	 * when possible.
2281 	 *
2282 	 * RAR 0 is used for the station MAC address
2283 	 * if there are not 14 addresses, go ahead and clear the filters
2284 	 */
2285 	i = 1;
2286 	if (use_uc)
2287 		netdev_for_each_uc_addr(ha, netdev) {
2288 			if (i == rar_entries)
2289 				break;
2290 			e1000_rar_set(hw, ha->addr, i++);
2291 		}
2292 
2293 	netdev_for_each_mc_addr(ha, netdev) {
2294 		if (i == rar_entries) {
2295 			/* load any remaining addresses into the hash table */
2296 			u32 hash_reg, hash_bit, mta;
2297 			hash_value = e1000_hash_mc_addr(hw, ha->addr);
2298 			hash_reg = (hash_value >> 5) & 0x7F;
2299 			hash_bit = hash_value & 0x1F;
2300 			mta = (1 << hash_bit);
2301 			mcarray[hash_reg] |= mta;
2302 		} else {
2303 			e1000_rar_set(hw, ha->addr, i++);
2304 		}
2305 	}
2306 
2307 	for (; i < rar_entries; i++) {
2308 		E1000_WRITE_REG_ARRAY(hw, RA, i << 1, 0);
2309 		E1000_WRITE_FLUSH();
2310 		E1000_WRITE_REG_ARRAY(hw, RA, (i << 1) + 1, 0);
2311 		E1000_WRITE_FLUSH();
2312 	}
2313 
2314 	/* write the hash table completely, write from bottom to avoid
2315 	 * both stupid write combining chipsets, and flushing each write
2316 	 */
2317 	for (i = mta_reg_count - 1; i >= 0 ; i--) {
2318 		/* If we are on an 82544 has an errata where writing odd
2319 		 * offsets overwrites the previous even offset, but writing
2320 		 * backwards over the range solves the issue by always
2321 		 * writing the odd offset first
2322 		 */
2323 		E1000_WRITE_REG_ARRAY(hw, MTA, i, mcarray[i]);
2324 	}
2325 	E1000_WRITE_FLUSH();
2326 
2327 	if (hw->mac_type == e1000_82542_rev2_0)
2328 		e1000_leave_82542_rst(adapter);
2329 
2330 	kfree(mcarray);
2331 }
2332 
2333 /**
2334  * e1000_update_phy_info_task - get phy info
2335  * @work: work struct contained inside adapter struct
2336  *
2337  * Need to wait a few seconds after link up to get diagnostic information from
2338  * the phy
2339  */
2340 static void e1000_update_phy_info_task(struct work_struct *work)
2341 {
2342 	struct e1000_adapter *adapter = container_of(work,
2343 						     struct e1000_adapter,
2344 						     phy_info_task.work);
2345 
2346 	e1000_phy_get_info(&adapter->hw, &adapter->phy_info);
2347 }
2348 
2349 /**
2350  * e1000_82547_tx_fifo_stall_task - task to complete work
2351  * @work: work struct contained inside adapter struct
2352  **/
2353 static void e1000_82547_tx_fifo_stall_task(struct work_struct *work)
2354 {
2355 	struct e1000_adapter *adapter = container_of(work,
2356 						     struct e1000_adapter,
2357 						     fifo_stall_task.work);
2358 	struct e1000_hw *hw = &adapter->hw;
2359 	struct net_device *netdev = adapter->netdev;
2360 	u32 tctl;
2361 
2362 	if (atomic_read(&adapter->tx_fifo_stall)) {
2363 		if ((er32(TDT) == er32(TDH)) &&
2364 		   (er32(TDFT) == er32(TDFH)) &&
2365 		   (er32(TDFTS) == er32(TDFHS))) {
2366 			tctl = er32(TCTL);
2367 			ew32(TCTL, tctl & ~E1000_TCTL_EN);
2368 			ew32(TDFT, adapter->tx_head_addr);
2369 			ew32(TDFH, adapter->tx_head_addr);
2370 			ew32(TDFTS, adapter->tx_head_addr);
2371 			ew32(TDFHS, adapter->tx_head_addr);
2372 			ew32(TCTL, tctl);
2373 			E1000_WRITE_FLUSH();
2374 
2375 			adapter->tx_fifo_head = 0;
2376 			atomic_set(&adapter->tx_fifo_stall, 0);
2377 			netif_wake_queue(netdev);
2378 		} else if (!test_bit(__E1000_DOWN, &adapter->flags)) {
2379 			schedule_delayed_work(&adapter->fifo_stall_task, 1);
2380 		}
2381 	}
2382 }
2383 
2384 bool e1000_has_link(struct e1000_adapter *adapter)
2385 {
2386 	struct e1000_hw *hw = &adapter->hw;
2387 	bool link_active = false;
2388 
2389 	/* get_link_status is set on LSC (link status) interrupt or rx
2390 	 * sequence error interrupt (except on intel ce4100).
2391 	 * get_link_status will stay false until the
2392 	 * e1000_check_for_link establishes link for copper adapters
2393 	 * ONLY
2394 	 */
2395 	switch (hw->media_type) {
2396 	case e1000_media_type_copper:
2397 		if (hw->mac_type == e1000_ce4100)
2398 			hw->get_link_status = 1;
2399 		if (hw->get_link_status) {
2400 			e1000_check_for_link(hw);
2401 			link_active = !hw->get_link_status;
2402 		} else {
2403 			link_active = true;
2404 		}
2405 		break;
2406 	case e1000_media_type_fiber:
2407 		e1000_check_for_link(hw);
2408 		link_active = !!(er32(STATUS) & E1000_STATUS_LU);
2409 		break;
2410 	case e1000_media_type_internal_serdes:
2411 		e1000_check_for_link(hw);
2412 		link_active = hw->serdes_has_link;
2413 		break;
2414 	default:
2415 		break;
2416 	}
2417 
2418 	return link_active;
2419 }
2420 
2421 /**
2422  * e1000_watchdog - work function
2423  * @work: work struct contained inside adapter struct
2424  **/
2425 static void e1000_watchdog(struct work_struct *work)
2426 {
2427 	struct e1000_adapter *adapter = container_of(work,
2428 						     struct e1000_adapter,
2429 						     watchdog_task.work);
2430 	struct e1000_hw *hw = &adapter->hw;
2431 	struct net_device *netdev = adapter->netdev;
2432 	struct e1000_tx_ring *txdr = adapter->tx_ring;
2433 	u32 link, tctl;
2434 
2435 	link = e1000_has_link(adapter);
2436 	if ((netif_carrier_ok(netdev)) && link)
2437 		goto link_up;
2438 
2439 	if (link) {
2440 		if (!netif_carrier_ok(netdev)) {
2441 			u32 ctrl;
2442 			/* update snapshot of PHY registers on LSC */
2443 			e1000_get_speed_and_duplex(hw,
2444 						   &adapter->link_speed,
2445 						   &adapter->link_duplex);
2446 
2447 			ctrl = er32(CTRL);
2448 			pr_info("%s NIC Link is Up %d Mbps %s, "
2449 				"Flow Control: %s\n",
2450 				netdev->name,
2451 				adapter->link_speed,
2452 				adapter->link_duplex == FULL_DUPLEX ?
2453 				"Full Duplex" : "Half Duplex",
2454 				((ctrl & E1000_CTRL_TFCE) && (ctrl &
2455 				E1000_CTRL_RFCE)) ? "RX/TX" : ((ctrl &
2456 				E1000_CTRL_RFCE) ? "RX" : ((ctrl &
2457 				E1000_CTRL_TFCE) ? "TX" : "None")));
2458 
2459 			/* adjust timeout factor according to speed/duplex */
2460 			adapter->tx_timeout_factor = 1;
2461 			switch (adapter->link_speed) {
2462 			case SPEED_10:
2463 				adapter->tx_timeout_factor = 16;
2464 				break;
2465 			case SPEED_100:
2466 				/* maybe add some timeout factor ? */
2467 				break;
2468 			}
2469 
2470 			/* enable transmits in the hardware */
2471 			tctl = er32(TCTL);
2472 			tctl |= E1000_TCTL_EN;
2473 			ew32(TCTL, tctl);
2474 
2475 			netif_carrier_on(netdev);
2476 			if (!test_bit(__E1000_DOWN, &adapter->flags))
2477 				schedule_delayed_work(&adapter->phy_info_task,
2478 						      2 * HZ);
2479 			adapter->smartspeed = 0;
2480 		}
2481 	} else {
2482 		if (netif_carrier_ok(netdev)) {
2483 			adapter->link_speed = 0;
2484 			adapter->link_duplex = 0;
2485 			pr_info("%s NIC Link is Down\n",
2486 				netdev->name);
2487 			netif_carrier_off(netdev);
2488 
2489 			if (!test_bit(__E1000_DOWN, &adapter->flags))
2490 				schedule_delayed_work(&adapter->phy_info_task,
2491 						      2 * HZ);
2492 		}
2493 
2494 		e1000_smartspeed(adapter);
2495 	}
2496 
2497 link_up:
2498 	e1000_update_stats(adapter);
2499 
2500 	hw->tx_packet_delta = adapter->stats.tpt - adapter->tpt_old;
2501 	adapter->tpt_old = adapter->stats.tpt;
2502 	hw->collision_delta = adapter->stats.colc - adapter->colc_old;
2503 	adapter->colc_old = adapter->stats.colc;
2504 
2505 	adapter->gorcl = adapter->stats.gorcl - adapter->gorcl_old;
2506 	adapter->gorcl_old = adapter->stats.gorcl;
2507 	adapter->gotcl = adapter->stats.gotcl - adapter->gotcl_old;
2508 	adapter->gotcl_old = adapter->stats.gotcl;
2509 
2510 	e1000_update_adaptive(hw);
2511 
2512 	if (!netif_carrier_ok(netdev)) {
2513 		if (E1000_DESC_UNUSED(txdr) + 1 < txdr->count) {
2514 			/* We've lost link, so the controller stops DMA,
2515 			 * but we've got queued Tx work that's never going
2516 			 * to get done, so reset controller to flush Tx.
2517 			 * (Do the reset outside of interrupt context).
2518 			 */
2519 			adapter->tx_timeout_count++;
2520 			schedule_work(&adapter->reset_task);
2521 			/* exit immediately since reset is imminent */
2522 			return;
2523 		}
2524 	}
2525 
2526 	/* Simple mode for Interrupt Throttle Rate (ITR) */
2527 	if (hw->mac_type >= e1000_82540 && adapter->itr_setting == 4) {
2528 		/* Symmetric Tx/Rx gets a reduced ITR=2000;
2529 		 * Total asymmetrical Tx or Rx gets ITR=8000;
2530 		 * everyone else is between 2000-8000.
2531 		 */
2532 		u32 goc = (adapter->gotcl + adapter->gorcl) / 10000;
2533 		u32 dif = (adapter->gotcl > adapter->gorcl ?
2534 			    adapter->gotcl - adapter->gorcl :
2535 			    adapter->gorcl - adapter->gotcl) / 10000;
2536 		u32 itr = goc > 0 ? (dif * 6000 / goc + 2000) : 8000;
2537 
2538 		ew32(ITR, 1000000000 / (itr * 256));
2539 	}
2540 
2541 	/* Cause software interrupt to ensure rx ring is cleaned */
2542 	ew32(ICS, E1000_ICS_RXDMT0);
2543 
2544 	/* Force detection of hung controller every watchdog period */
2545 	adapter->detect_tx_hung = true;
2546 
2547 	/* Reschedule the task */
2548 	if (!test_bit(__E1000_DOWN, &adapter->flags))
2549 		schedule_delayed_work(&adapter->watchdog_task, 2 * HZ);
2550 }
2551 
2552 enum latency_range {
2553 	lowest_latency = 0,
2554 	low_latency = 1,
2555 	bulk_latency = 2,
2556 	latency_invalid = 255
2557 };
2558 
2559 /**
2560  * e1000_update_itr - update the dynamic ITR value based on statistics
2561  * @adapter: pointer to adapter
2562  * @itr_setting: current adapter->itr
2563  * @packets: the number of packets during this measurement interval
2564  * @bytes: the number of bytes during this measurement interval
2565  *
2566  *      Stores a new ITR value based on packets and byte
2567  *      counts during the last interrupt.  The advantage of per interrupt
2568  *      computation is faster updates and more accurate ITR for the current
2569  *      traffic pattern.  Constants in this function were computed
2570  *      based on theoretical maximum wire speed and thresholds were set based
2571  *      on testing data as well as attempting to minimize response time
2572  *      while increasing bulk throughput.
2573  *      this functionality is controlled by the InterruptThrottleRate module
2574  *      parameter (see e1000_param.c)
2575  **/
2576 static unsigned int e1000_update_itr(struct e1000_adapter *adapter,
2577 				     u16 itr_setting, int packets, int bytes)
2578 {
2579 	unsigned int retval = itr_setting;
2580 	struct e1000_hw *hw = &adapter->hw;
2581 
2582 	if (unlikely(hw->mac_type < e1000_82540))
2583 		goto update_itr_done;
2584 
2585 	if (packets == 0)
2586 		goto update_itr_done;
2587 
2588 	switch (itr_setting) {
2589 	case lowest_latency:
2590 		/* jumbo frames get bulk treatment*/
2591 		if (bytes/packets > 8000)
2592 			retval = bulk_latency;
2593 		else if ((packets < 5) && (bytes > 512))
2594 			retval = low_latency;
2595 		break;
2596 	case low_latency:  /* 50 usec aka 20000 ints/s */
2597 		if (bytes > 10000) {
2598 			/* jumbo frames need bulk latency setting */
2599 			if (bytes/packets > 8000)
2600 				retval = bulk_latency;
2601 			else if ((packets < 10) || ((bytes/packets) > 1200))
2602 				retval = bulk_latency;
2603 			else if ((packets > 35))
2604 				retval = lowest_latency;
2605 		} else if (bytes/packets > 2000)
2606 			retval = bulk_latency;
2607 		else if (packets <= 2 && bytes < 512)
2608 			retval = lowest_latency;
2609 		break;
2610 	case bulk_latency: /* 250 usec aka 4000 ints/s */
2611 		if (bytes > 25000) {
2612 			if (packets > 35)
2613 				retval = low_latency;
2614 		} else if (bytes < 6000) {
2615 			retval = low_latency;
2616 		}
2617 		break;
2618 	}
2619 
2620 update_itr_done:
2621 	return retval;
2622 }
2623 
2624 static void e1000_set_itr(struct e1000_adapter *adapter)
2625 {
2626 	struct e1000_hw *hw = &adapter->hw;
2627 	u16 current_itr;
2628 	u32 new_itr = adapter->itr;
2629 
2630 	if (unlikely(hw->mac_type < e1000_82540))
2631 		return;
2632 
2633 	/* for non-gigabit speeds, just fix the interrupt rate at 4000 */
2634 	if (unlikely(adapter->link_speed != SPEED_1000)) {
2635 		new_itr = 4000;
2636 		goto set_itr_now;
2637 	}
2638 
2639 	adapter->tx_itr = e1000_update_itr(adapter, adapter->tx_itr,
2640 					   adapter->total_tx_packets,
2641 					   adapter->total_tx_bytes);
2642 	/* conservative mode (itr 3) eliminates the lowest_latency setting */
2643 	if (adapter->itr_setting == 3 && adapter->tx_itr == lowest_latency)
2644 		adapter->tx_itr = low_latency;
2645 
2646 	adapter->rx_itr = e1000_update_itr(adapter, adapter->rx_itr,
2647 					   adapter->total_rx_packets,
2648 					   adapter->total_rx_bytes);
2649 	/* conservative mode (itr 3) eliminates the lowest_latency setting */
2650 	if (adapter->itr_setting == 3 && adapter->rx_itr == lowest_latency)
2651 		adapter->rx_itr = low_latency;
2652 
2653 	current_itr = max(adapter->rx_itr, adapter->tx_itr);
2654 
2655 	switch (current_itr) {
2656 	/* counts and packets in update_itr are dependent on these numbers */
2657 	case lowest_latency:
2658 		new_itr = 70000;
2659 		break;
2660 	case low_latency:
2661 		new_itr = 20000; /* aka hwitr = ~200 */
2662 		break;
2663 	case bulk_latency:
2664 		new_itr = 4000;
2665 		break;
2666 	default:
2667 		break;
2668 	}
2669 
2670 set_itr_now:
2671 	if (new_itr != adapter->itr) {
2672 		/* this attempts to bias the interrupt rate towards Bulk
2673 		 * by adding intermediate steps when interrupt rate is
2674 		 * increasing
2675 		 */
2676 		new_itr = new_itr > adapter->itr ?
2677 			  min(adapter->itr + (new_itr >> 2), new_itr) :
2678 			  new_itr;
2679 		adapter->itr = new_itr;
2680 		ew32(ITR, 1000000000 / (new_itr * 256));
2681 	}
2682 }
2683 
2684 #define E1000_TX_FLAGS_CSUM		0x00000001
2685 #define E1000_TX_FLAGS_VLAN		0x00000002
2686 #define E1000_TX_FLAGS_TSO		0x00000004
2687 #define E1000_TX_FLAGS_IPV4		0x00000008
2688 #define E1000_TX_FLAGS_NO_FCS		0x00000010
2689 #define E1000_TX_FLAGS_VLAN_MASK	0xffff0000
2690 #define E1000_TX_FLAGS_VLAN_SHIFT	16
2691 
2692 static int e1000_tso(struct e1000_adapter *adapter,
2693 		     struct e1000_tx_ring *tx_ring, struct sk_buff *skb,
2694 		     __be16 protocol)
2695 {
2696 	struct e1000_context_desc *context_desc;
2697 	struct e1000_tx_buffer *buffer_info;
2698 	unsigned int i;
2699 	u32 cmd_length = 0;
2700 	u16 ipcse = 0, tucse, mss;
2701 	u8 ipcss, ipcso, tucss, tucso, hdr_len;
2702 
2703 	if (skb_is_gso(skb)) {
2704 		int err;
2705 
2706 		err = skb_cow_head(skb, 0);
2707 		if (err < 0)
2708 			return err;
2709 
2710 		hdr_len = skb_transport_offset(skb) + tcp_hdrlen(skb);
2711 		mss = skb_shinfo(skb)->gso_size;
2712 		if (protocol == htons(ETH_P_IP)) {
2713 			struct iphdr *iph = ip_hdr(skb);
2714 			iph->tot_len = 0;
2715 			iph->check = 0;
2716 			tcp_hdr(skb)->check = ~csum_tcpudp_magic(iph->saddr,
2717 								 iph->daddr, 0,
2718 								 IPPROTO_TCP,
2719 								 0);
2720 			cmd_length = E1000_TXD_CMD_IP;
2721 			ipcse = skb_transport_offset(skb) - 1;
2722 		} else if (skb_is_gso_v6(skb)) {
2723 			tcp_v6_gso_csum_prep(skb);
2724 			ipcse = 0;
2725 		}
2726 		ipcss = skb_network_offset(skb);
2727 		ipcso = (void *)&(ip_hdr(skb)->check) - (void *)skb->data;
2728 		tucss = skb_transport_offset(skb);
2729 		tucso = (void *)&(tcp_hdr(skb)->check) - (void *)skb->data;
2730 		tucse = 0;
2731 
2732 		cmd_length |= (E1000_TXD_CMD_DEXT | E1000_TXD_CMD_TSE |
2733 			       E1000_TXD_CMD_TCP | (skb->len - (hdr_len)));
2734 
2735 		i = tx_ring->next_to_use;
2736 		context_desc = E1000_CONTEXT_DESC(*tx_ring, i);
2737 		buffer_info = &tx_ring->buffer_info[i];
2738 
2739 		context_desc->lower_setup.ip_fields.ipcss  = ipcss;
2740 		context_desc->lower_setup.ip_fields.ipcso  = ipcso;
2741 		context_desc->lower_setup.ip_fields.ipcse  = cpu_to_le16(ipcse);
2742 		context_desc->upper_setup.tcp_fields.tucss = tucss;
2743 		context_desc->upper_setup.tcp_fields.tucso = tucso;
2744 		context_desc->upper_setup.tcp_fields.tucse = cpu_to_le16(tucse);
2745 		context_desc->tcp_seg_setup.fields.mss     = cpu_to_le16(mss);
2746 		context_desc->tcp_seg_setup.fields.hdr_len = hdr_len;
2747 		context_desc->cmd_and_length = cpu_to_le32(cmd_length);
2748 
2749 		buffer_info->time_stamp = jiffies;
2750 		buffer_info->next_to_watch = i;
2751 
2752 		if (++i == tx_ring->count)
2753 			i = 0;
2754 
2755 		tx_ring->next_to_use = i;
2756 
2757 		return true;
2758 	}
2759 	return false;
2760 }
2761 
2762 static bool e1000_tx_csum(struct e1000_adapter *adapter,
2763 			  struct e1000_tx_ring *tx_ring, struct sk_buff *skb,
2764 			  __be16 protocol)
2765 {
2766 	struct e1000_context_desc *context_desc;
2767 	struct e1000_tx_buffer *buffer_info;
2768 	unsigned int i;
2769 	u8 css;
2770 	u32 cmd_len = E1000_TXD_CMD_DEXT;
2771 
2772 	if (skb->ip_summed != CHECKSUM_PARTIAL)
2773 		return false;
2774 
2775 	switch (protocol) {
2776 	case cpu_to_be16(ETH_P_IP):
2777 		if (ip_hdr(skb)->protocol == IPPROTO_TCP)
2778 			cmd_len |= E1000_TXD_CMD_TCP;
2779 		break;
2780 	case cpu_to_be16(ETH_P_IPV6):
2781 		/* XXX not handling all IPV6 headers */
2782 		if (ipv6_hdr(skb)->nexthdr == IPPROTO_TCP)
2783 			cmd_len |= E1000_TXD_CMD_TCP;
2784 		break;
2785 	default:
2786 		if (unlikely(net_ratelimit()))
2787 			e_warn(drv, "checksum_partial proto=%x!\n",
2788 			       skb->protocol);
2789 		break;
2790 	}
2791 
2792 	css = skb_checksum_start_offset(skb);
2793 
2794 	i = tx_ring->next_to_use;
2795 	buffer_info = &tx_ring->buffer_info[i];
2796 	context_desc = E1000_CONTEXT_DESC(*tx_ring, i);
2797 
2798 	context_desc->lower_setup.ip_config = 0;
2799 	context_desc->upper_setup.tcp_fields.tucss = css;
2800 	context_desc->upper_setup.tcp_fields.tucso =
2801 		css + skb->csum_offset;
2802 	context_desc->upper_setup.tcp_fields.tucse = 0;
2803 	context_desc->tcp_seg_setup.data = 0;
2804 	context_desc->cmd_and_length = cpu_to_le32(cmd_len);
2805 
2806 	buffer_info->time_stamp = jiffies;
2807 	buffer_info->next_to_watch = i;
2808 
2809 	if (unlikely(++i == tx_ring->count))
2810 		i = 0;
2811 
2812 	tx_ring->next_to_use = i;
2813 
2814 	return true;
2815 }
2816 
2817 #define E1000_MAX_TXD_PWR	12
2818 #define E1000_MAX_DATA_PER_TXD	(1<<E1000_MAX_TXD_PWR)
2819 
2820 static int e1000_tx_map(struct e1000_adapter *adapter,
2821 			struct e1000_tx_ring *tx_ring,
2822 			struct sk_buff *skb, unsigned int first,
2823 			unsigned int max_per_txd, unsigned int nr_frags,
2824 			unsigned int mss)
2825 {
2826 	struct e1000_hw *hw = &adapter->hw;
2827 	struct pci_dev *pdev = adapter->pdev;
2828 	struct e1000_tx_buffer *buffer_info;
2829 	unsigned int len = skb_headlen(skb);
2830 	unsigned int offset = 0, size, count = 0, i;
2831 	unsigned int f, bytecount, segs;
2832 
2833 	i = tx_ring->next_to_use;
2834 
2835 	while (len) {
2836 		buffer_info = &tx_ring->buffer_info[i];
2837 		size = min(len, max_per_txd);
2838 		/* Workaround for Controller erratum --
2839 		 * descriptor for non-tso packet in a linear SKB that follows a
2840 		 * tso gets written back prematurely before the data is fully
2841 		 * DMA'd to the controller
2842 		 */
2843 		if (!skb->data_len && tx_ring->last_tx_tso &&
2844 		    !skb_is_gso(skb)) {
2845 			tx_ring->last_tx_tso = false;
2846 			size -= 4;
2847 		}
2848 
2849 		/* Workaround for premature desc write-backs
2850 		 * in TSO mode.  Append 4-byte sentinel desc
2851 		 */
2852 		if (unlikely(mss && !nr_frags && size == len && size > 8))
2853 			size -= 4;
2854 		/* work-around for errata 10 and it applies
2855 		 * to all controllers in PCI-X mode
2856 		 * The fix is to make sure that the first descriptor of a
2857 		 * packet is smaller than 2048 - 16 - 16 (or 2016) bytes
2858 		 */
2859 		if (unlikely((hw->bus_type == e1000_bus_type_pcix) &&
2860 			     (size > 2015) && count == 0))
2861 			size = 2015;
2862 
2863 		/* Workaround for potential 82544 hang in PCI-X.  Avoid
2864 		 * terminating buffers within evenly-aligned dwords.
2865 		 */
2866 		if (unlikely(adapter->pcix_82544 &&
2867 		   !((unsigned long)(skb->data + offset + size - 1) & 4) &&
2868 		   size > 4))
2869 			size -= 4;
2870 
2871 		buffer_info->length = size;
2872 		/* set time_stamp *before* dma to help avoid a possible race */
2873 		buffer_info->time_stamp = jiffies;
2874 		buffer_info->mapped_as_page = false;
2875 		buffer_info->dma = dma_map_single(&pdev->dev,
2876 						  skb->data + offset,
2877 						  size, DMA_TO_DEVICE);
2878 		if (dma_mapping_error(&pdev->dev, buffer_info->dma))
2879 			goto dma_error;
2880 		buffer_info->next_to_watch = i;
2881 
2882 		len -= size;
2883 		offset += size;
2884 		count++;
2885 		if (len) {
2886 			i++;
2887 			if (unlikely(i == tx_ring->count))
2888 				i = 0;
2889 		}
2890 	}
2891 
2892 	for (f = 0; f < nr_frags; f++) {
2893 		const skb_frag_t *frag = &skb_shinfo(skb)->frags[f];
2894 
2895 		len = skb_frag_size(frag);
2896 		offset = 0;
2897 
2898 		while (len) {
2899 			unsigned long bufend;
2900 			i++;
2901 			if (unlikely(i == tx_ring->count))
2902 				i = 0;
2903 
2904 			buffer_info = &tx_ring->buffer_info[i];
2905 			size = min(len, max_per_txd);
2906 			/* Workaround for premature desc write-backs
2907 			 * in TSO mode.  Append 4-byte sentinel desc
2908 			 */
2909 			if (unlikely(mss && f == (nr_frags-1) &&
2910 			    size == len && size > 8))
2911 				size -= 4;
2912 			/* Workaround for potential 82544 hang in PCI-X.
2913 			 * Avoid terminating buffers within evenly-aligned
2914 			 * dwords.
2915 			 */
2916 			bufend = (unsigned long)
2917 				page_to_phys(skb_frag_page(frag));
2918 			bufend += offset + size - 1;
2919 			if (unlikely(adapter->pcix_82544 &&
2920 				     !(bufend & 4) &&
2921 				     size > 4))
2922 				size -= 4;
2923 
2924 			buffer_info->length = size;
2925 			buffer_info->time_stamp = jiffies;
2926 			buffer_info->mapped_as_page = true;
2927 			buffer_info->dma = skb_frag_dma_map(&pdev->dev, frag,
2928 						offset, size, DMA_TO_DEVICE);
2929 			if (dma_mapping_error(&pdev->dev, buffer_info->dma))
2930 				goto dma_error;
2931 			buffer_info->next_to_watch = i;
2932 
2933 			len -= size;
2934 			offset += size;
2935 			count++;
2936 		}
2937 	}
2938 
2939 	segs = skb_shinfo(skb)->gso_segs ?: 1;
2940 	/* multiply data chunks by size of headers */
2941 	bytecount = ((segs - 1) * skb_headlen(skb)) + skb->len;
2942 
2943 	tx_ring->buffer_info[i].skb = skb;
2944 	tx_ring->buffer_info[i].segs = segs;
2945 	tx_ring->buffer_info[i].bytecount = bytecount;
2946 	tx_ring->buffer_info[first].next_to_watch = i;
2947 
2948 	return count;
2949 
2950 dma_error:
2951 	dev_err(&pdev->dev, "TX DMA map failed\n");
2952 	buffer_info->dma = 0;
2953 	if (count)
2954 		count--;
2955 
2956 	while (count--) {
2957 		if (i == 0)
2958 			i += tx_ring->count;
2959 		i--;
2960 		buffer_info = &tx_ring->buffer_info[i];
2961 		e1000_unmap_and_free_tx_resource(adapter, buffer_info);
2962 	}
2963 
2964 	return 0;
2965 }
2966 
2967 static void e1000_tx_queue(struct e1000_adapter *adapter,
2968 			   struct e1000_tx_ring *tx_ring, int tx_flags,
2969 			   int count)
2970 {
2971 	struct e1000_tx_desc *tx_desc = NULL;
2972 	struct e1000_tx_buffer *buffer_info;
2973 	u32 txd_upper = 0, txd_lower = E1000_TXD_CMD_IFCS;
2974 	unsigned int i;
2975 
2976 	if (likely(tx_flags & E1000_TX_FLAGS_TSO)) {
2977 		txd_lower |= E1000_TXD_CMD_DEXT | E1000_TXD_DTYP_D |
2978 			     E1000_TXD_CMD_TSE;
2979 		txd_upper |= E1000_TXD_POPTS_TXSM << 8;
2980 
2981 		if (likely(tx_flags & E1000_TX_FLAGS_IPV4))
2982 			txd_upper |= E1000_TXD_POPTS_IXSM << 8;
2983 	}
2984 
2985 	if (likely(tx_flags & E1000_TX_FLAGS_CSUM)) {
2986 		txd_lower |= E1000_TXD_CMD_DEXT | E1000_TXD_DTYP_D;
2987 		txd_upper |= E1000_TXD_POPTS_TXSM << 8;
2988 	}
2989 
2990 	if (unlikely(tx_flags & E1000_TX_FLAGS_VLAN)) {
2991 		txd_lower |= E1000_TXD_CMD_VLE;
2992 		txd_upper |= (tx_flags & E1000_TX_FLAGS_VLAN_MASK);
2993 	}
2994 
2995 	if (unlikely(tx_flags & E1000_TX_FLAGS_NO_FCS))
2996 		txd_lower &= ~(E1000_TXD_CMD_IFCS);
2997 
2998 	i = tx_ring->next_to_use;
2999 
3000 	while (count--) {
3001 		buffer_info = &tx_ring->buffer_info[i];
3002 		tx_desc = E1000_TX_DESC(*tx_ring, i);
3003 		tx_desc->buffer_addr = cpu_to_le64(buffer_info->dma);
3004 		tx_desc->lower.data =
3005 			cpu_to_le32(txd_lower | buffer_info->length);
3006 		tx_desc->upper.data = cpu_to_le32(txd_upper);
3007 		if (unlikely(++i == tx_ring->count))
3008 			i = 0;
3009 	}
3010 
3011 	tx_desc->lower.data |= cpu_to_le32(adapter->txd_cmd);
3012 
3013 	/* txd_cmd re-enables FCS, so we'll re-disable it here as desired. */
3014 	if (unlikely(tx_flags & E1000_TX_FLAGS_NO_FCS))
3015 		tx_desc->lower.data &= ~(cpu_to_le32(E1000_TXD_CMD_IFCS));
3016 
3017 	/* Force memory writes to complete before letting h/w
3018 	 * know there are new descriptors to fetch.  (Only
3019 	 * applicable for weak-ordered memory model archs,
3020 	 * such as IA-64).
3021 	 */
3022 	dma_wmb();
3023 
3024 	tx_ring->next_to_use = i;
3025 }
3026 
3027 /* 82547 workaround to avoid controller hang in half-duplex environment.
3028  * The workaround is to avoid queuing a large packet that would span
3029  * the internal Tx FIFO ring boundary by notifying the stack to resend
3030  * the packet at a later time.  This gives the Tx FIFO an opportunity to
3031  * flush all packets.  When that occurs, we reset the Tx FIFO pointers
3032  * to the beginning of the Tx FIFO.
3033  */
3034 
3035 #define E1000_FIFO_HDR			0x10
3036 #define E1000_82547_PAD_LEN		0x3E0
3037 
3038 static int e1000_82547_fifo_workaround(struct e1000_adapter *adapter,
3039 				       struct sk_buff *skb)
3040 {
3041 	u32 fifo_space = adapter->tx_fifo_size - adapter->tx_fifo_head;
3042 	u32 skb_fifo_len = skb->len + E1000_FIFO_HDR;
3043 
3044 	skb_fifo_len = ALIGN(skb_fifo_len, E1000_FIFO_HDR);
3045 
3046 	if (adapter->link_duplex != HALF_DUPLEX)
3047 		goto no_fifo_stall_required;
3048 
3049 	if (atomic_read(&adapter->tx_fifo_stall))
3050 		return 1;
3051 
3052 	if (skb_fifo_len >= (E1000_82547_PAD_LEN + fifo_space)) {
3053 		atomic_set(&adapter->tx_fifo_stall, 1);
3054 		return 1;
3055 	}
3056 
3057 no_fifo_stall_required:
3058 	adapter->tx_fifo_head += skb_fifo_len;
3059 	if (adapter->tx_fifo_head >= adapter->tx_fifo_size)
3060 		adapter->tx_fifo_head -= adapter->tx_fifo_size;
3061 	return 0;
3062 }
3063 
3064 static int __e1000_maybe_stop_tx(struct net_device *netdev, int size)
3065 {
3066 	struct e1000_adapter *adapter = netdev_priv(netdev);
3067 	struct e1000_tx_ring *tx_ring = adapter->tx_ring;
3068 
3069 	netif_stop_queue(netdev);
3070 	/* Herbert's original patch had:
3071 	 *  smp_mb__after_netif_stop_queue();
3072 	 * but since that doesn't exist yet, just open code it.
3073 	 */
3074 	smp_mb();
3075 
3076 	/* We need to check again in a case another CPU has just
3077 	 * made room available.
3078 	 */
3079 	if (likely(E1000_DESC_UNUSED(tx_ring) < size))
3080 		return -EBUSY;
3081 
3082 	/* A reprieve! */
3083 	netif_start_queue(netdev);
3084 	++adapter->restart_queue;
3085 	return 0;
3086 }
3087 
3088 static int e1000_maybe_stop_tx(struct net_device *netdev,
3089 			       struct e1000_tx_ring *tx_ring, int size)
3090 {
3091 	if (likely(E1000_DESC_UNUSED(tx_ring) >= size))
3092 		return 0;
3093 	return __e1000_maybe_stop_tx(netdev, size);
3094 }
3095 
3096 #define TXD_USE_COUNT(S, X) (((S) + ((1 << (X)) - 1)) >> (X))
3097 static netdev_tx_t e1000_xmit_frame(struct sk_buff *skb,
3098 				    struct net_device *netdev)
3099 {
3100 	struct e1000_adapter *adapter = netdev_priv(netdev);
3101 	struct e1000_hw *hw = &adapter->hw;
3102 	struct e1000_tx_ring *tx_ring;
3103 	unsigned int first, max_per_txd = E1000_MAX_DATA_PER_TXD;
3104 	unsigned int max_txd_pwr = E1000_MAX_TXD_PWR;
3105 	unsigned int tx_flags = 0;
3106 	unsigned int len = skb_headlen(skb);
3107 	unsigned int nr_frags;
3108 	unsigned int mss;
3109 	int count = 0;
3110 	int tso;
3111 	unsigned int f;
3112 	__be16 protocol = vlan_get_protocol(skb);
3113 
3114 	/* This goes back to the question of how to logically map a Tx queue
3115 	 * to a flow.  Right now, performance is impacted slightly negatively
3116 	 * if using multiple Tx queues.  If the stack breaks away from a
3117 	 * single qdisc implementation, we can look at this again.
3118 	 */
3119 	tx_ring = adapter->tx_ring;
3120 
3121 	/* On PCI/PCI-X HW, if packet size is less than ETH_ZLEN,
3122 	 * packets may get corrupted during padding by HW.
3123 	 * To WA this issue, pad all small packets manually.
3124 	 */
3125 	if (eth_skb_pad(skb))
3126 		return NETDEV_TX_OK;
3127 
3128 	mss = skb_shinfo(skb)->gso_size;
3129 	/* The controller does a simple calculation to
3130 	 * make sure there is enough room in the FIFO before
3131 	 * initiating the DMA for each buffer.  The calc is:
3132 	 * 4 = ceil(buffer len/mss).  To make sure we don't
3133 	 * overrun the FIFO, adjust the max buffer len if mss
3134 	 * drops.
3135 	 */
3136 	if (mss) {
3137 		u8 hdr_len;
3138 		max_per_txd = min(mss << 2, max_per_txd);
3139 		max_txd_pwr = fls(max_per_txd) - 1;
3140 
3141 		hdr_len = skb_transport_offset(skb) + tcp_hdrlen(skb);
3142 		if (skb->data_len && hdr_len == len) {
3143 			switch (hw->mac_type) {
3144 			case e1000_82544: {
3145 				unsigned int pull_size;
3146 
3147 				/* Make sure we have room to chop off 4 bytes,
3148 				 * and that the end alignment will work out to
3149 				 * this hardware's requirements
3150 				 * NOTE: this is a TSO only workaround
3151 				 * if end byte alignment not correct move us
3152 				 * into the next dword
3153 				 */
3154 				if ((unsigned long)(skb_tail_pointer(skb) - 1)
3155 				    & 4)
3156 					break;
3157 				pull_size = min((unsigned int)4, skb->data_len);
3158 				if (!__pskb_pull_tail(skb, pull_size)) {
3159 					e_err(drv, "__pskb_pull_tail "
3160 					      "failed.\n");
3161 					dev_kfree_skb_any(skb);
3162 					return NETDEV_TX_OK;
3163 				}
3164 				len = skb_headlen(skb);
3165 				break;
3166 			}
3167 			default:
3168 				/* do nothing */
3169 				break;
3170 			}
3171 		}
3172 	}
3173 
3174 	/* reserve a descriptor for the offload context */
3175 	if ((mss) || (skb->ip_summed == CHECKSUM_PARTIAL))
3176 		count++;
3177 	count++;
3178 
3179 	/* Controller Erratum workaround */
3180 	if (!skb->data_len && tx_ring->last_tx_tso && !skb_is_gso(skb))
3181 		count++;
3182 
3183 	count += TXD_USE_COUNT(len, max_txd_pwr);
3184 
3185 	if (adapter->pcix_82544)
3186 		count++;
3187 
3188 	/* work-around for errata 10 and it applies to all controllers
3189 	 * in PCI-X mode, so add one more descriptor to the count
3190 	 */
3191 	if (unlikely((hw->bus_type == e1000_bus_type_pcix) &&
3192 			(len > 2015)))
3193 		count++;
3194 
3195 	nr_frags = skb_shinfo(skb)->nr_frags;
3196 	for (f = 0; f < nr_frags; f++)
3197 		count += TXD_USE_COUNT(skb_frag_size(&skb_shinfo(skb)->frags[f]),
3198 				       max_txd_pwr);
3199 	if (adapter->pcix_82544)
3200 		count += nr_frags;
3201 
3202 	/* need: count + 2 desc gap to keep tail from touching
3203 	 * head, otherwise try next time
3204 	 */
3205 	if (unlikely(e1000_maybe_stop_tx(netdev, tx_ring, count + 2)))
3206 		return NETDEV_TX_BUSY;
3207 
3208 	if (unlikely((hw->mac_type == e1000_82547) &&
3209 		     (e1000_82547_fifo_workaround(adapter, skb)))) {
3210 		netif_stop_queue(netdev);
3211 		if (!test_bit(__E1000_DOWN, &adapter->flags))
3212 			schedule_delayed_work(&adapter->fifo_stall_task, 1);
3213 		return NETDEV_TX_BUSY;
3214 	}
3215 
3216 	if (skb_vlan_tag_present(skb)) {
3217 		tx_flags |= E1000_TX_FLAGS_VLAN;
3218 		tx_flags |= (skb_vlan_tag_get(skb) <<
3219 			     E1000_TX_FLAGS_VLAN_SHIFT);
3220 	}
3221 
3222 	first = tx_ring->next_to_use;
3223 
3224 	tso = e1000_tso(adapter, tx_ring, skb, protocol);
3225 	if (tso < 0) {
3226 		dev_kfree_skb_any(skb);
3227 		return NETDEV_TX_OK;
3228 	}
3229 
3230 	if (likely(tso)) {
3231 		if (likely(hw->mac_type != e1000_82544))
3232 			tx_ring->last_tx_tso = true;
3233 		tx_flags |= E1000_TX_FLAGS_TSO;
3234 	} else if (likely(e1000_tx_csum(adapter, tx_ring, skb, protocol)))
3235 		tx_flags |= E1000_TX_FLAGS_CSUM;
3236 
3237 	if (protocol == htons(ETH_P_IP))
3238 		tx_flags |= E1000_TX_FLAGS_IPV4;
3239 
3240 	if (unlikely(skb->no_fcs))
3241 		tx_flags |= E1000_TX_FLAGS_NO_FCS;
3242 
3243 	count = e1000_tx_map(adapter, tx_ring, skb, first, max_per_txd,
3244 			     nr_frags, mss);
3245 
3246 	if (count) {
3247 		/* The descriptors needed is higher than other Intel drivers
3248 		 * due to a number of workarounds.  The breakdown is below:
3249 		 * Data descriptors: MAX_SKB_FRAGS + 1
3250 		 * Context Descriptor: 1
3251 		 * Keep head from touching tail: 2
3252 		 * Workarounds: 3
3253 		 */
3254 		int desc_needed = MAX_SKB_FRAGS + 7;
3255 
3256 		netdev_sent_queue(netdev, skb->len);
3257 		skb_tx_timestamp(skb);
3258 
3259 		e1000_tx_queue(adapter, tx_ring, tx_flags, count);
3260 
3261 		/* 82544 potentially requires twice as many data descriptors
3262 		 * in order to guarantee buffers don't end on evenly-aligned
3263 		 * dwords
3264 		 */
3265 		if (adapter->pcix_82544)
3266 			desc_needed += MAX_SKB_FRAGS + 1;
3267 
3268 		/* Make sure there is space in the ring for the next send. */
3269 		e1000_maybe_stop_tx(netdev, tx_ring, desc_needed);
3270 
3271 		if (!netdev_xmit_more() ||
3272 		    netif_xmit_stopped(netdev_get_tx_queue(netdev, 0))) {
3273 			writel(tx_ring->next_to_use, hw->hw_addr + tx_ring->tdt);
3274 		}
3275 	} else {
3276 		dev_kfree_skb_any(skb);
3277 		tx_ring->buffer_info[first].time_stamp = 0;
3278 		tx_ring->next_to_use = first;
3279 	}
3280 
3281 	return NETDEV_TX_OK;
3282 }
3283 
3284 #define NUM_REGS 38 /* 1 based count */
3285 static void e1000_regdump(struct e1000_adapter *adapter)
3286 {
3287 	struct e1000_hw *hw = &adapter->hw;
3288 	u32 regs[NUM_REGS];
3289 	u32 *regs_buff = regs;
3290 	int i = 0;
3291 
3292 	static const char * const reg_name[] = {
3293 		"CTRL",  "STATUS",
3294 		"RCTL", "RDLEN", "RDH", "RDT", "RDTR",
3295 		"TCTL", "TDBAL", "TDBAH", "TDLEN", "TDH", "TDT",
3296 		"TIDV", "TXDCTL", "TADV", "TARC0",
3297 		"TDBAL1", "TDBAH1", "TDLEN1", "TDH1", "TDT1",
3298 		"TXDCTL1", "TARC1",
3299 		"CTRL_EXT", "ERT", "RDBAL", "RDBAH",
3300 		"TDFH", "TDFT", "TDFHS", "TDFTS", "TDFPC",
3301 		"RDFH", "RDFT", "RDFHS", "RDFTS", "RDFPC"
3302 	};
3303 
3304 	regs_buff[0]  = er32(CTRL);
3305 	regs_buff[1]  = er32(STATUS);
3306 
3307 	regs_buff[2]  = er32(RCTL);
3308 	regs_buff[3]  = er32(RDLEN);
3309 	regs_buff[4]  = er32(RDH);
3310 	regs_buff[5]  = er32(RDT);
3311 	regs_buff[6]  = er32(RDTR);
3312 
3313 	regs_buff[7]  = er32(TCTL);
3314 	regs_buff[8]  = er32(TDBAL);
3315 	regs_buff[9]  = er32(TDBAH);
3316 	regs_buff[10] = er32(TDLEN);
3317 	regs_buff[11] = er32(TDH);
3318 	regs_buff[12] = er32(TDT);
3319 	regs_buff[13] = er32(TIDV);
3320 	regs_buff[14] = er32(TXDCTL);
3321 	regs_buff[15] = er32(TADV);
3322 	regs_buff[16] = er32(TARC0);
3323 
3324 	regs_buff[17] = er32(TDBAL1);
3325 	regs_buff[18] = er32(TDBAH1);
3326 	regs_buff[19] = er32(TDLEN1);
3327 	regs_buff[20] = er32(TDH1);
3328 	regs_buff[21] = er32(TDT1);
3329 	regs_buff[22] = er32(TXDCTL1);
3330 	regs_buff[23] = er32(TARC1);
3331 	regs_buff[24] = er32(CTRL_EXT);
3332 	regs_buff[25] = er32(ERT);
3333 	regs_buff[26] = er32(RDBAL0);
3334 	regs_buff[27] = er32(RDBAH0);
3335 	regs_buff[28] = er32(TDFH);
3336 	regs_buff[29] = er32(TDFT);
3337 	regs_buff[30] = er32(TDFHS);
3338 	regs_buff[31] = er32(TDFTS);
3339 	regs_buff[32] = er32(TDFPC);
3340 	regs_buff[33] = er32(RDFH);
3341 	regs_buff[34] = er32(RDFT);
3342 	regs_buff[35] = er32(RDFHS);
3343 	regs_buff[36] = er32(RDFTS);
3344 	regs_buff[37] = er32(RDFPC);
3345 
3346 	pr_info("Register dump\n");
3347 	for (i = 0; i < NUM_REGS; i++)
3348 		pr_info("%-15s  %08x\n", reg_name[i], regs_buff[i]);
3349 }
3350 
3351 /*
3352  * e1000_dump: Print registers, tx ring and rx ring
3353  */
3354 static void e1000_dump(struct e1000_adapter *adapter)
3355 {
3356 	/* this code doesn't handle multiple rings */
3357 	struct e1000_tx_ring *tx_ring = adapter->tx_ring;
3358 	struct e1000_rx_ring *rx_ring = adapter->rx_ring;
3359 	int i;
3360 
3361 	if (!netif_msg_hw(adapter))
3362 		return;
3363 
3364 	/* Print Registers */
3365 	e1000_regdump(adapter);
3366 
3367 	/* transmit dump */
3368 	pr_info("TX Desc ring0 dump\n");
3369 
3370 	/* Transmit Descriptor Formats - DEXT[29] is 0 (Legacy) or 1 (Extended)
3371 	 *
3372 	 * Legacy Transmit Descriptor
3373 	 *   +--------------------------------------------------------------+
3374 	 * 0 |         Buffer Address [63:0] (Reserved on Write Back)       |
3375 	 *   +--------------------------------------------------------------+
3376 	 * 8 | Special  |    CSS     | Status |  CMD    |  CSO   |  Length  |
3377 	 *   +--------------------------------------------------------------+
3378 	 *   63       48 47        36 35    32 31     24 23    16 15        0
3379 	 *
3380 	 * Extended Context Descriptor (DTYP=0x0) for TSO or checksum offload
3381 	 *   63      48 47    40 39       32 31             16 15    8 7      0
3382 	 *   +----------------------------------------------------------------+
3383 	 * 0 |  TUCSE  | TUCS0  |   TUCSS   |     IPCSE       | IPCS0 | IPCSS |
3384 	 *   +----------------------------------------------------------------+
3385 	 * 8 |   MSS   | HDRLEN | RSV | STA | TUCMD | DTYP |      PAYLEN      |
3386 	 *   +----------------------------------------------------------------+
3387 	 *   63      48 47    40 39 36 35 32 31   24 23  20 19                0
3388 	 *
3389 	 * Extended Data Descriptor (DTYP=0x1)
3390 	 *   +----------------------------------------------------------------+
3391 	 * 0 |                     Buffer Address [63:0]                      |
3392 	 *   +----------------------------------------------------------------+
3393 	 * 8 | VLAN tag |  POPTS  | Rsvd | Status | Command | DTYP |  DTALEN  |
3394 	 *   +----------------------------------------------------------------+
3395 	 *   63       48 47     40 39  36 35    32 31     24 23  20 19        0
3396 	 */
3397 	pr_info("Tc[desc]     [Ce CoCsIpceCoS] [MssHlRSCm0Plen] [bi->dma       ] leng  ntw timestmp         bi->skb\n");
3398 	pr_info("Td[desc]     [address 63:0  ] [VlaPoRSCm1Dlen] [bi->dma       ] leng  ntw timestmp         bi->skb\n");
3399 
3400 	if (!netif_msg_tx_done(adapter))
3401 		goto rx_ring_summary;
3402 
3403 	for (i = 0; tx_ring->desc && (i < tx_ring->count); i++) {
3404 		struct e1000_tx_desc *tx_desc = E1000_TX_DESC(*tx_ring, i);
3405 		struct e1000_tx_buffer *buffer_info = &tx_ring->buffer_info[i];
3406 		struct my_u { __le64 a; __le64 b; };
3407 		struct my_u *u = (struct my_u *)tx_desc;
3408 		const char *type;
3409 
3410 		if (i == tx_ring->next_to_use && i == tx_ring->next_to_clean)
3411 			type = "NTC/U";
3412 		else if (i == tx_ring->next_to_use)
3413 			type = "NTU";
3414 		else if (i == tx_ring->next_to_clean)
3415 			type = "NTC";
3416 		else
3417 			type = "";
3418 
3419 		pr_info("T%c[0x%03X]    %016llX %016llX %016llX %04X  %3X %016llX %p %s\n",
3420 			((le64_to_cpu(u->b) & (1<<20)) ? 'd' : 'c'), i,
3421 			le64_to_cpu(u->a), le64_to_cpu(u->b),
3422 			(u64)buffer_info->dma, buffer_info->length,
3423 			buffer_info->next_to_watch,
3424 			(u64)buffer_info->time_stamp, buffer_info->skb, type);
3425 	}
3426 
3427 rx_ring_summary:
3428 	/* receive dump */
3429 	pr_info("\nRX Desc ring dump\n");
3430 
3431 	/* Legacy Receive Descriptor Format
3432 	 *
3433 	 * +-----------------------------------------------------+
3434 	 * |                Buffer Address [63:0]                |
3435 	 * +-----------------------------------------------------+
3436 	 * | VLAN Tag | Errors | Status 0 | Packet csum | Length |
3437 	 * +-----------------------------------------------------+
3438 	 * 63       48 47    40 39      32 31         16 15      0
3439 	 */
3440 	pr_info("R[desc]      [address 63:0  ] [vl er S cks ln] [bi->dma       ] [bi->skb]\n");
3441 
3442 	if (!netif_msg_rx_status(adapter))
3443 		goto exit;
3444 
3445 	for (i = 0; rx_ring->desc && (i < rx_ring->count); i++) {
3446 		struct e1000_rx_desc *rx_desc = E1000_RX_DESC(*rx_ring, i);
3447 		struct e1000_rx_buffer *buffer_info = &rx_ring->buffer_info[i];
3448 		struct my_u { __le64 a; __le64 b; };
3449 		struct my_u *u = (struct my_u *)rx_desc;
3450 		const char *type;
3451 
3452 		if (i == rx_ring->next_to_use)
3453 			type = "NTU";
3454 		else if (i == rx_ring->next_to_clean)
3455 			type = "NTC";
3456 		else
3457 			type = "";
3458 
3459 		pr_info("R[0x%03X]     %016llX %016llX %016llX %p %s\n",
3460 			i, le64_to_cpu(u->a), le64_to_cpu(u->b),
3461 			(u64)buffer_info->dma, buffer_info->rxbuf.data, type);
3462 	} /* for */
3463 
3464 	/* dump the descriptor caches */
3465 	/* rx */
3466 	pr_info("Rx descriptor cache in 64bit format\n");
3467 	for (i = 0x6000; i <= 0x63FF ; i += 0x10) {
3468 		pr_info("R%04X: %08X|%08X %08X|%08X\n",
3469 			i,
3470 			readl(adapter->hw.hw_addr + i+4),
3471 			readl(adapter->hw.hw_addr + i),
3472 			readl(adapter->hw.hw_addr + i+12),
3473 			readl(adapter->hw.hw_addr + i+8));
3474 	}
3475 	/* tx */
3476 	pr_info("Tx descriptor cache in 64bit format\n");
3477 	for (i = 0x7000; i <= 0x73FF ; i += 0x10) {
3478 		pr_info("T%04X: %08X|%08X %08X|%08X\n",
3479 			i,
3480 			readl(adapter->hw.hw_addr + i+4),
3481 			readl(adapter->hw.hw_addr + i),
3482 			readl(adapter->hw.hw_addr + i+12),
3483 			readl(adapter->hw.hw_addr + i+8));
3484 	}
3485 exit:
3486 	return;
3487 }
3488 
3489 /**
3490  * e1000_tx_timeout - Respond to a Tx Hang
3491  * @netdev: network interface device structure
3492  * @txqueue: number of the Tx queue that hung (unused)
3493  **/
3494 static void e1000_tx_timeout(struct net_device *netdev, unsigned int __always_unused txqueue)
3495 {
3496 	struct e1000_adapter *adapter = netdev_priv(netdev);
3497 
3498 	/* Do the reset outside of interrupt context */
3499 	adapter->tx_timeout_count++;
3500 	schedule_work(&adapter->reset_task);
3501 }
3502 
3503 static void e1000_reset_task(struct work_struct *work)
3504 {
3505 	struct e1000_adapter *adapter =
3506 		container_of(work, struct e1000_adapter, reset_task);
3507 
3508 	e_err(drv, "Reset adapter\n");
3509 	e1000_reinit_locked(adapter);
3510 }
3511 
3512 /**
3513  * e1000_change_mtu - Change the Maximum Transfer Unit
3514  * @netdev: network interface device structure
3515  * @new_mtu: new value for maximum frame size
3516  *
3517  * Returns 0 on success, negative on failure
3518  **/
3519 static int e1000_change_mtu(struct net_device *netdev, int new_mtu)
3520 {
3521 	struct e1000_adapter *adapter = netdev_priv(netdev);
3522 	struct e1000_hw *hw = &adapter->hw;
3523 	int max_frame = new_mtu + ETH_HLEN + ETH_FCS_LEN;
3524 
3525 	/* Adapter-specific max frame size limits. */
3526 	switch (hw->mac_type) {
3527 	case e1000_undefined ... e1000_82542_rev2_1:
3528 		if (max_frame > (ETH_FRAME_LEN + ETH_FCS_LEN)) {
3529 			e_err(probe, "Jumbo Frames not supported.\n");
3530 			return -EINVAL;
3531 		}
3532 		break;
3533 	default:
3534 		/* Capable of supporting up to MAX_JUMBO_FRAME_SIZE limit. */
3535 		break;
3536 	}
3537 
3538 	while (test_and_set_bit(__E1000_RESETTING, &adapter->flags))
3539 		msleep(1);
3540 	/* e1000_down has a dependency on max_frame_size */
3541 	hw->max_frame_size = max_frame;
3542 	if (netif_running(netdev)) {
3543 		/* prevent buffers from being reallocated */
3544 		adapter->alloc_rx_buf = e1000_alloc_dummy_rx_buffers;
3545 		e1000_down(adapter);
3546 	}
3547 
3548 	/* NOTE: netdev_alloc_skb reserves 16 bytes, and typically NET_IP_ALIGN
3549 	 * means we reserve 2 more, this pushes us to allocate from the next
3550 	 * larger slab size.
3551 	 * i.e. RXBUFFER_2048 --> size-4096 slab
3552 	 * however with the new *_jumbo_rx* routines, jumbo receives will use
3553 	 * fragmented skbs
3554 	 */
3555 
3556 	if (max_frame <= E1000_RXBUFFER_2048)
3557 		adapter->rx_buffer_len = E1000_RXBUFFER_2048;
3558 	else
3559 #if (PAGE_SIZE >= E1000_RXBUFFER_16384)
3560 		adapter->rx_buffer_len = E1000_RXBUFFER_16384;
3561 #elif (PAGE_SIZE >= E1000_RXBUFFER_4096)
3562 		adapter->rx_buffer_len = PAGE_SIZE;
3563 #endif
3564 
3565 	/* adjust allocation if LPE protects us, and we aren't using SBP */
3566 	if (!hw->tbi_compatibility_on &&
3567 	    ((max_frame == (ETH_FRAME_LEN + ETH_FCS_LEN)) ||
3568 	     (max_frame == MAXIMUM_ETHERNET_VLAN_SIZE)))
3569 		adapter->rx_buffer_len = MAXIMUM_ETHERNET_VLAN_SIZE;
3570 
3571 	netdev_dbg(netdev, "changing MTU from %d to %d\n",
3572 		   netdev->mtu, new_mtu);
3573 	netdev->mtu = new_mtu;
3574 
3575 	if (netif_running(netdev))
3576 		e1000_up(adapter);
3577 	else
3578 		e1000_reset(adapter);
3579 
3580 	clear_bit(__E1000_RESETTING, &adapter->flags);
3581 
3582 	return 0;
3583 }
3584 
3585 /**
3586  * e1000_update_stats - Update the board statistics counters
3587  * @adapter: board private structure
3588  **/
3589 void e1000_update_stats(struct e1000_adapter *adapter)
3590 {
3591 	struct net_device *netdev = adapter->netdev;
3592 	struct e1000_hw *hw = &adapter->hw;
3593 	struct pci_dev *pdev = adapter->pdev;
3594 	unsigned long flags;
3595 	u16 phy_tmp;
3596 
3597 #define PHY_IDLE_ERROR_COUNT_MASK 0x00FF
3598 
3599 	/* Prevent stats update while adapter is being reset, or if the pci
3600 	 * connection is down.
3601 	 */
3602 	if (adapter->link_speed == 0)
3603 		return;
3604 	if (pci_channel_offline(pdev))
3605 		return;
3606 
3607 	spin_lock_irqsave(&adapter->stats_lock, flags);
3608 
3609 	/* these counters are modified from e1000_tbi_adjust_stats,
3610 	 * called from the interrupt context, so they must only
3611 	 * be written while holding adapter->stats_lock
3612 	 */
3613 
3614 	adapter->stats.crcerrs += er32(CRCERRS);
3615 	adapter->stats.gprc += er32(GPRC);
3616 	adapter->stats.gorcl += er32(GORCL);
3617 	adapter->stats.gorch += er32(GORCH);
3618 	adapter->stats.bprc += er32(BPRC);
3619 	adapter->stats.mprc += er32(MPRC);
3620 	adapter->stats.roc += er32(ROC);
3621 
3622 	adapter->stats.prc64 += er32(PRC64);
3623 	adapter->stats.prc127 += er32(PRC127);
3624 	adapter->stats.prc255 += er32(PRC255);
3625 	adapter->stats.prc511 += er32(PRC511);
3626 	adapter->stats.prc1023 += er32(PRC1023);
3627 	adapter->stats.prc1522 += er32(PRC1522);
3628 
3629 	adapter->stats.symerrs += er32(SYMERRS);
3630 	adapter->stats.mpc += er32(MPC);
3631 	adapter->stats.scc += er32(SCC);
3632 	adapter->stats.ecol += er32(ECOL);
3633 	adapter->stats.mcc += er32(MCC);
3634 	adapter->stats.latecol += er32(LATECOL);
3635 	adapter->stats.dc += er32(DC);
3636 	adapter->stats.sec += er32(SEC);
3637 	adapter->stats.rlec += er32(RLEC);
3638 	adapter->stats.xonrxc += er32(XONRXC);
3639 	adapter->stats.xontxc += er32(XONTXC);
3640 	adapter->stats.xoffrxc += er32(XOFFRXC);
3641 	adapter->stats.xofftxc += er32(XOFFTXC);
3642 	adapter->stats.fcruc += er32(FCRUC);
3643 	adapter->stats.gptc += er32(GPTC);
3644 	adapter->stats.gotcl += er32(GOTCL);
3645 	adapter->stats.gotch += er32(GOTCH);
3646 	adapter->stats.rnbc += er32(RNBC);
3647 	adapter->stats.ruc += er32(RUC);
3648 	adapter->stats.rfc += er32(RFC);
3649 	adapter->stats.rjc += er32(RJC);
3650 	adapter->stats.torl += er32(TORL);
3651 	adapter->stats.torh += er32(TORH);
3652 	adapter->stats.totl += er32(TOTL);
3653 	adapter->stats.toth += er32(TOTH);
3654 	adapter->stats.tpr += er32(TPR);
3655 
3656 	adapter->stats.ptc64 += er32(PTC64);
3657 	adapter->stats.ptc127 += er32(PTC127);
3658 	adapter->stats.ptc255 += er32(PTC255);
3659 	adapter->stats.ptc511 += er32(PTC511);
3660 	adapter->stats.ptc1023 += er32(PTC1023);
3661 	adapter->stats.ptc1522 += er32(PTC1522);
3662 
3663 	adapter->stats.mptc += er32(MPTC);
3664 	adapter->stats.bptc += er32(BPTC);
3665 
3666 	/* used for adaptive IFS */
3667 
3668 	hw->tx_packet_delta = er32(TPT);
3669 	adapter->stats.tpt += hw->tx_packet_delta;
3670 	hw->collision_delta = er32(COLC);
3671 	adapter->stats.colc += hw->collision_delta;
3672 
3673 	if (hw->mac_type >= e1000_82543) {
3674 		adapter->stats.algnerrc += er32(ALGNERRC);
3675 		adapter->stats.rxerrc += er32(RXERRC);
3676 		adapter->stats.tncrs += er32(TNCRS);
3677 		adapter->stats.cexterr += er32(CEXTERR);
3678 		adapter->stats.tsctc += er32(TSCTC);
3679 		adapter->stats.tsctfc += er32(TSCTFC);
3680 	}
3681 
3682 	/* Fill out the OS statistics structure */
3683 	netdev->stats.multicast = adapter->stats.mprc;
3684 	netdev->stats.collisions = adapter->stats.colc;
3685 
3686 	/* Rx Errors */
3687 
3688 	/* RLEC on some newer hardware can be incorrect so build
3689 	 * our own version based on RUC and ROC
3690 	 */
3691 	netdev->stats.rx_errors = adapter->stats.rxerrc +
3692 		adapter->stats.crcerrs + adapter->stats.algnerrc +
3693 		adapter->stats.ruc + adapter->stats.roc +
3694 		adapter->stats.cexterr;
3695 	adapter->stats.rlerrc = adapter->stats.ruc + adapter->stats.roc;
3696 	netdev->stats.rx_length_errors = adapter->stats.rlerrc;
3697 	netdev->stats.rx_crc_errors = adapter->stats.crcerrs;
3698 	netdev->stats.rx_frame_errors = adapter->stats.algnerrc;
3699 	netdev->stats.rx_missed_errors = adapter->stats.mpc;
3700 
3701 	/* Tx Errors */
3702 	adapter->stats.txerrc = adapter->stats.ecol + adapter->stats.latecol;
3703 	netdev->stats.tx_errors = adapter->stats.txerrc;
3704 	netdev->stats.tx_aborted_errors = adapter->stats.ecol;
3705 	netdev->stats.tx_window_errors = adapter->stats.latecol;
3706 	netdev->stats.tx_carrier_errors = adapter->stats.tncrs;
3707 	if (hw->bad_tx_carr_stats_fd &&
3708 	    adapter->link_duplex == FULL_DUPLEX) {
3709 		netdev->stats.tx_carrier_errors = 0;
3710 		adapter->stats.tncrs = 0;
3711 	}
3712 
3713 	/* Tx Dropped needs to be maintained elsewhere */
3714 
3715 	/* Phy Stats */
3716 	if (hw->media_type == e1000_media_type_copper) {
3717 		if ((adapter->link_speed == SPEED_1000) &&
3718 		   (!e1000_read_phy_reg(hw, PHY_1000T_STATUS, &phy_tmp))) {
3719 			phy_tmp &= PHY_IDLE_ERROR_COUNT_MASK;
3720 			adapter->phy_stats.idle_errors += phy_tmp;
3721 		}
3722 
3723 		if ((hw->mac_type <= e1000_82546) &&
3724 		   (hw->phy_type == e1000_phy_m88) &&
3725 		   !e1000_read_phy_reg(hw, M88E1000_RX_ERR_CNTR, &phy_tmp))
3726 			adapter->phy_stats.receive_errors += phy_tmp;
3727 	}
3728 
3729 	/* Management Stats */
3730 	if (hw->has_smbus) {
3731 		adapter->stats.mgptc += er32(MGTPTC);
3732 		adapter->stats.mgprc += er32(MGTPRC);
3733 		adapter->stats.mgpdc += er32(MGTPDC);
3734 	}
3735 
3736 	spin_unlock_irqrestore(&adapter->stats_lock, flags);
3737 }
3738 
3739 /**
3740  * e1000_intr - Interrupt Handler
3741  * @irq: interrupt number
3742  * @data: pointer to a network interface device structure
3743  **/
3744 static irqreturn_t e1000_intr(int irq, void *data)
3745 {
3746 	struct net_device *netdev = data;
3747 	struct e1000_adapter *adapter = netdev_priv(netdev);
3748 	struct e1000_hw *hw = &adapter->hw;
3749 	u32 icr = er32(ICR);
3750 
3751 	if (unlikely((!icr)))
3752 		return IRQ_NONE;  /* Not our interrupt */
3753 
3754 	/* we might have caused the interrupt, but the above
3755 	 * read cleared it, and just in case the driver is
3756 	 * down there is nothing to do so return handled
3757 	 */
3758 	if (unlikely(test_bit(__E1000_DOWN, &adapter->flags)))
3759 		return IRQ_HANDLED;
3760 
3761 	if (unlikely(icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC))) {
3762 		hw->get_link_status = 1;
3763 		/* guard against interrupt when we're going down */
3764 		if (!test_bit(__E1000_DOWN, &adapter->flags))
3765 			schedule_delayed_work(&adapter->watchdog_task, 1);
3766 	}
3767 
3768 	/* disable interrupts, without the synchronize_irq bit */
3769 	ew32(IMC, ~0);
3770 	E1000_WRITE_FLUSH();
3771 
3772 	if (likely(napi_schedule_prep(&adapter->napi))) {
3773 		adapter->total_tx_bytes = 0;
3774 		adapter->total_tx_packets = 0;
3775 		adapter->total_rx_bytes = 0;
3776 		adapter->total_rx_packets = 0;
3777 		__napi_schedule(&adapter->napi);
3778 	} else {
3779 		/* this really should not happen! if it does it is basically a
3780 		 * bug, but not a hard error, so enable ints and continue
3781 		 */
3782 		if (!test_bit(__E1000_DOWN, &adapter->flags))
3783 			e1000_irq_enable(adapter);
3784 	}
3785 
3786 	return IRQ_HANDLED;
3787 }
3788 
3789 /**
3790  * e1000_clean - NAPI Rx polling callback
3791  * @napi: napi struct containing references to driver info
3792  * @budget: budget given to driver for receive packets
3793  **/
3794 static int e1000_clean(struct napi_struct *napi, int budget)
3795 {
3796 	struct e1000_adapter *adapter = container_of(napi, struct e1000_adapter,
3797 						     napi);
3798 	int tx_clean_complete = 0, work_done = 0;
3799 
3800 	tx_clean_complete = e1000_clean_tx_irq(adapter, &adapter->tx_ring[0]);
3801 
3802 	adapter->clean_rx(adapter, &adapter->rx_ring[0], &work_done, budget);
3803 
3804 	if (!tx_clean_complete || work_done == budget)
3805 		return budget;
3806 
3807 	/* Exit the polling mode, but don't re-enable interrupts if stack might
3808 	 * poll us due to busy-polling
3809 	 */
3810 	if (likely(napi_complete_done(napi, work_done))) {
3811 		if (likely(adapter->itr_setting & 3))
3812 			e1000_set_itr(adapter);
3813 		if (!test_bit(__E1000_DOWN, &adapter->flags))
3814 			e1000_irq_enable(adapter);
3815 	}
3816 
3817 	return work_done;
3818 }
3819 
3820 /**
3821  * e1000_clean_tx_irq - Reclaim resources after transmit completes
3822  * @adapter: board private structure
3823  * @tx_ring: ring to clean
3824  **/
3825 static bool e1000_clean_tx_irq(struct e1000_adapter *adapter,
3826 			       struct e1000_tx_ring *tx_ring)
3827 {
3828 	struct e1000_hw *hw = &adapter->hw;
3829 	struct net_device *netdev = adapter->netdev;
3830 	struct e1000_tx_desc *tx_desc, *eop_desc;
3831 	struct e1000_tx_buffer *buffer_info;
3832 	unsigned int i, eop;
3833 	unsigned int count = 0;
3834 	unsigned int total_tx_bytes = 0, total_tx_packets = 0;
3835 	unsigned int bytes_compl = 0, pkts_compl = 0;
3836 
3837 	i = tx_ring->next_to_clean;
3838 	eop = tx_ring->buffer_info[i].next_to_watch;
3839 	eop_desc = E1000_TX_DESC(*tx_ring, eop);
3840 
3841 	while ((eop_desc->upper.data & cpu_to_le32(E1000_TXD_STAT_DD)) &&
3842 	       (count < tx_ring->count)) {
3843 		bool cleaned = false;
3844 		dma_rmb();	/* read buffer_info after eop_desc */
3845 		for ( ; !cleaned; count++) {
3846 			tx_desc = E1000_TX_DESC(*tx_ring, i);
3847 			buffer_info = &tx_ring->buffer_info[i];
3848 			cleaned = (i == eop);
3849 
3850 			if (cleaned) {
3851 				total_tx_packets += buffer_info->segs;
3852 				total_tx_bytes += buffer_info->bytecount;
3853 				if (buffer_info->skb) {
3854 					bytes_compl += buffer_info->skb->len;
3855 					pkts_compl++;
3856 				}
3857 
3858 			}
3859 			e1000_unmap_and_free_tx_resource(adapter, buffer_info);
3860 			tx_desc->upper.data = 0;
3861 
3862 			if (unlikely(++i == tx_ring->count))
3863 				i = 0;
3864 		}
3865 
3866 		eop = tx_ring->buffer_info[i].next_to_watch;
3867 		eop_desc = E1000_TX_DESC(*tx_ring, eop);
3868 	}
3869 
3870 	/* Synchronize with E1000_DESC_UNUSED called from e1000_xmit_frame,
3871 	 * which will reuse the cleaned buffers.
3872 	 */
3873 	smp_store_release(&tx_ring->next_to_clean, i);
3874 
3875 	netdev_completed_queue(netdev, pkts_compl, bytes_compl);
3876 
3877 #define TX_WAKE_THRESHOLD 32
3878 	if (unlikely(count && netif_carrier_ok(netdev) &&
3879 		     E1000_DESC_UNUSED(tx_ring) >= TX_WAKE_THRESHOLD)) {
3880 		/* Make sure that anybody stopping the queue after this
3881 		 * sees the new next_to_clean.
3882 		 */
3883 		smp_mb();
3884 
3885 		if (netif_queue_stopped(netdev) &&
3886 		    !(test_bit(__E1000_DOWN, &adapter->flags))) {
3887 			netif_wake_queue(netdev);
3888 			++adapter->restart_queue;
3889 		}
3890 	}
3891 
3892 	if (adapter->detect_tx_hung) {
3893 		/* Detect a transmit hang in hardware, this serializes the
3894 		 * check with the clearing of time_stamp and movement of i
3895 		 */
3896 		adapter->detect_tx_hung = false;
3897 		if (tx_ring->buffer_info[eop].time_stamp &&
3898 		    time_after(jiffies, tx_ring->buffer_info[eop].time_stamp +
3899 			       (adapter->tx_timeout_factor * HZ)) &&
3900 		    !(er32(STATUS) & E1000_STATUS_TXOFF)) {
3901 
3902 			/* detected Tx unit hang */
3903 			e_err(drv, "Detected Tx Unit Hang\n"
3904 			      "  Tx Queue             <%lu>\n"
3905 			      "  TDH                  <%x>\n"
3906 			      "  TDT                  <%x>\n"
3907 			      "  next_to_use          <%x>\n"
3908 			      "  next_to_clean        <%x>\n"
3909 			      "buffer_info[next_to_clean]\n"
3910 			      "  time_stamp           <%lx>\n"
3911 			      "  next_to_watch        <%x>\n"
3912 			      "  jiffies              <%lx>\n"
3913 			      "  next_to_watch.status <%x>\n",
3914 				(unsigned long)(tx_ring - adapter->tx_ring),
3915 				readl(hw->hw_addr + tx_ring->tdh),
3916 				readl(hw->hw_addr + tx_ring->tdt),
3917 				tx_ring->next_to_use,
3918 				tx_ring->next_to_clean,
3919 				tx_ring->buffer_info[eop].time_stamp,
3920 				eop,
3921 				jiffies,
3922 				eop_desc->upper.fields.status);
3923 			e1000_dump(adapter);
3924 			netif_stop_queue(netdev);
3925 		}
3926 	}
3927 	adapter->total_tx_bytes += total_tx_bytes;
3928 	adapter->total_tx_packets += total_tx_packets;
3929 	netdev->stats.tx_bytes += total_tx_bytes;
3930 	netdev->stats.tx_packets += total_tx_packets;
3931 	return count < tx_ring->count;
3932 }
3933 
3934 /**
3935  * e1000_rx_checksum - Receive Checksum Offload for 82543
3936  * @adapter:     board private structure
3937  * @status_err:  receive descriptor status and error fields
3938  * @csum:        receive descriptor csum field
3939  * @skb:         socket buffer with received data
3940  **/
3941 static void e1000_rx_checksum(struct e1000_adapter *adapter, u32 status_err,
3942 			      u32 csum, struct sk_buff *skb)
3943 {
3944 	struct e1000_hw *hw = &adapter->hw;
3945 	u16 status = (u16)status_err;
3946 	u8 errors = (u8)(status_err >> 24);
3947 
3948 	skb_checksum_none_assert(skb);
3949 
3950 	/* 82543 or newer only */
3951 	if (unlikely(hw->mac_type < e1000_82543))
3952 		return;
3953 	/* Ignore Checksum bit is set */
3954 	if (unlikely(status & E1000_RXD_STAT_IXSM))
3955 		return;
3956 	/* TCP/UDP checksum error bit is set */
3957 	if (unlikely(errors & E1000_RXD_ERR_TCPE)) {
3958 		/* let the stack verify checksum errors */
3959 		adapter->hw_csum_err++;
3960 		return;
3961 	}
3962 	/* TCP/UDP Checksum has not been calculated */
3963 	if (!(status & E1000_RXD_STAT_TCPCS))
3964 		return;
3965 
3966 	/* It must be a TCP or UDP packet with a valid checksum */
3967 	if (likely(status & E1000_RXD_STAT_TCPCS)) {
3968 		/* TCP checksum is good */
3969 		skb->ip_summed = CHECKSUM_UNNECESSARY;
3970 	}
3971 	adapter->hw_csum_good++;
3972 }
3973 
3974 /**
3975  * e1000_consume_page - helper function for jumbo Rx path
3976  * @bi: software descriptor shadow data
3977  * @skb: skb being modified
3978  * @length: length of data being added
3979  **/
3980 static void e1000_consume_page(struct e1000_rx_buffer *bi, struct sk_buff *skb,
3981 			       u16 length)
3982 {
3983 	bi->rxbuf.page = NULL;
3984 	skb->len += length;
3985 	skb->data_len += length;
3986 	skb->truesize += PAGE_SIZE;
3987 }
3988 
3989 /**
3990  * e1000_receive_skb - helper function to handle rx indications
3991  * @adapter: board private structure
3992  * @status: descriptor status field as written by hardware
3993  * @vlan: descriptor vlan field as written by hardware (no le/be conversion)
3994  * @skb: pointer to sk_buff to be indicated to stack
3995  */
3996 static void e1000_receive_skb(struct e1000_adapter *adapter, u8 status,
3997 			      __le16 vlan, struct sk_buff *skb)
3998 {
3999 	skb->protocol = eth_type_trans(skb, adapter->netdev);
4000 
4001 	if (status & E1000_RXD_STAT_VP) {
4002 		u16 vid = le16_to_cpu(vlan) & E1000_RXD_SPC_VLAN_MASK;
4003 
4004 		__vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vid);
4005 	}
4006 	napi_gro_receive(&adapter->napi, skb);
4007 }
4008 
4009 /**
4010  * e1000_tbi_adjust_stats
4011  * @hw: Struct containing variables accessed by shared code
4012  * @stats: point to stats struct
4013  * @frame_len: The length of the frame in question
4014  * @mac_addr: The Ethernet destination address of the frame in question
4015  *
4016  * Adjusts the statistic counters when a frame is accepted by TBI_ACCEPT
4017  */
4018 static void e1000_tbi_adjust_stats(struct e1000_hw *hw,
4019 				   struct e1000_hw_stats *stats,
4020 				   u32 frame_len, const u8 *mac_addr)
4021 {
4022 	u64 carry_bit;
4023 
4024 	/* First adjust the frame length. */
4025 	frame_len--;
4026 	/* We need to adjust the statistics counters, since the hardware
4027 	 * counters overcount this packet as a CRC error and undercount
4028 	 * the packet as a good packet
4029 	 */
4030 	/* This packet should not be counted as a CRC error. */
4031 	stats->crcerrs--;
4032 	/* This packet does count as a Good Packet Received. */
4033 	stats->gprc++;
4034 
4035 	/* Adjust the Good Octets received counters */
4036 	carry_bit = 0x80000000 & stats->gorcl;
4037 	stats->gorcl += frame_len;
4038 	/* If the high bit of Gorcl (the low 32 bits of the Good Octets
4039 	 * Received Count) was one before the addition,
4040 	 * AND it is zero after, then we lost the carry out,
4041 	 * need to add one to Gorch (Good Octets Received Count High).
4042 	 * This could be simplified if all environments supported
4043 	 * 64-bit integers.
4044 	 */
4045 	if (carry_bit && ((stats->gorcl & 0x80000000) == 0))
4046 		stats->gorch++;
4047 	/* Is this a broadcast or multicast?  Check broadcast first,
4048 	 * since the test for a multicast frame will test positive on
4049 	 * a broadcast frame.
4050 	 */
4051 	if (is_broadcast_ether_addr(mac_addr))
4052 		stats->bprc++;
4053 	else if (is_multicast_ether_addr(mac_addr))
4054 		stats->mprc++;
4055 
4056 	if (frame_len == hw->max_frame_size) {
4057 		/* In this case, the hardware has overcounted the number of
4058 		 * oversize frames.
4059 		 */
4060 		if (stats->roc > 0)
4061 			stats->roc--;
4062 	}
4063 
4064 	/* Adjust the bin counters when the extra byte put the frame in the
4065 	 * wrong bin. Remember that the frame_len was adjusted above.
4066 	 */
4067 	if (frame_len == 64) {
4068 		stats->prc64++;
4069 		stats->prc127--;
4070 	} else if (frame_len == 127) {
4071 		stats->prc127++;
4072 		stats->prc255--;
4073 	} else if (frame_len == 255) {
4074 		stats->prc255++;
4075 		stats->prc511--;
4076 	} else if (frame_len == 511) {
4077 		stats->prc511++;
4078 		stats->prc1023--;
4079 	} else if (frame_len == 1023) {
4080 		stats->prc1023++;
4081 		stats->prc1522--;
4082 	} else if (frame_len == 1522) {
4083 		stats->prc1522++;
4084 	}
4085 }
4086 
4087 static bool e1000_tbi_should_accept(struct e1000_adapter *adapter,
4088 				    u8 status, u8 errors,
4089 				    u32 length, const u8 *data)
4090 {
4091 	struct e1000_hw *hw = &adapter->hw;
4092 	u8 last_byte = *(data + length - 1);
4093 
4094 	if (TBI_ACCEPT(hw, status, errors, length, last_byte)) {
4095 		unsigned long irq_flags;
4096 
4097 		spin_lock_irqsave(&adapter->stats_lock, irq_flags);
4098 		e1000_tbi_adjust_stats(hw, &adapter->stats, length, data);
4099 		spin_unlock_irqrestore(&adapter->stats_lock, irq_flags);
4100 
4101 		return true;
4102 	}
4103 
4104 	return false;
4105 }
4106 
4107 static struct sk_buff *e1000_alloc_rx_skb(struct e1000_adapter *adapter,
4108 					  unsigned int bufsz)
4109 {
4110 	struct sk_buff *skb = napi_alloc_skb(&adapter->napi, bufsz);
4111 
4112 	if (unlikely(!skb))
4113 		adapter->alloc_rx_buff_failed++;
4114 	return skb;
4115 }
4116 
4117 /**
4118  * e1000_clean_jumbo_rx_irq - Send received data up the network stack; legacy
4119  * @adapter: board private structure
4120  * @rx_ring: ring to clean
4121  * @work_done: amount of napi work completed this call
4122  * @work_to_do: max amount of work allowed for this call to do
4123  *
4124  * the return value indicates whether actual cleaning was done, there
4125  * is no guarantee that everything was cleaned
4126  */
4127 static bool e1000_clean_jumbo_rx_irq(struct e1000_adapter *adapter,
4128 				     struct e1000_rx_ring *rx_ring,
4129 				     int *work_done, int work_to_do)
4130 {
4131 	struct net_device *netdev = adapter->netdev;
4132 	struct pci_dev *pdev = adapter->pdev;
4133 	struct e1000_rx_desc *rx_desc, *next_rxd;
4134 	struct e1000_rx_buffer *buffer_info, *next_buffer;
4135 	u32 length;
4136 	unsigned int i;
4137 	int cleaned_count = 0;
4138 	bool cleaned = false;
4139 	unsigned int total_rx_bytes = 0, total_rx_packets = 0;
4140 
4141 	i = rx_ring->next_to_clean;
4142 	rx_desc = E1000_RX_DESC(*rx_ring, i);
4143 	buffer_info = &rx_ring->buffer_info[i];
4144 
4145 	while (rx_desc->status & E1000_RXD_STAT_DD) {
4146 		struct sk_buff *skb;
4147 		u8 status;
4148 
4149 		if (*work_done >= work_to_do)
4150 			break;
4151 		(*work_done)++;
4152 		dma_rmb(); /* read descriptor and rx_buffer_info after status DD */
4153 
4154 		status = rx_desc->status;
4155 
4156 		if (++i == rx_ring->count)
4157 			i = 0;
4158 
4159 		next_rxd = E1000_RX_DESC(*rx_ring, i);
4160 		prefetch(next_rxd);
4161 
4162 		next_buffer = &rx_ring->buffer_info[i];
4163 
4164 		cleaned = true;
4165 		cleaned_count++;
4166 		dma_unmap_page(&pdev->dev, buffer_info->dma,
4167 			       adapter->rx_buffer_len, DMA_FROM_DEVICE);
4168 		buffer_info->dma = 0;
4169 
4170 		length = le16_to_cpu(rx_desc->length);
4171 
4172 		/* errors is only valid for DD + EOP descriptors */
4173 		if (unlikely((status & E1000_RXD_STAT_EOP) &&
4174 		    (rx_desc->errors & E1000_RXD_ERR_FRAME_ERR_MASK))) {
4175 			u8 *mapped = page_address(buffer_info->rxbuf.page);
4176 
4177 			if (e1000_tbi_should_accept(adapter, status,
4178 						    rx_desc->errors,
4179 						    length, mapped)) {
4180 				length--;
4181 			} else if (netdev->features & NETIF_F_RXALL) {
4182 				goto process_skb;
4183 			} else {
4184 				/* an error means any chain goes out the window
4185 				 * too
4186 				 */
4187 				dev_kfree_skb(rx_ring->rx_skb_top);
4188 				rx_ring->rx_skb_top = NULL;
4189 				goto next_desc;
4190 			}
4191 		}
4192 
4193 #define rxtop rx_ring->rx_skb_top
4194 process_skb:
4195 		if (!(status & E1000_RXD_STAT_EOP)) {
4196 			/* this descriptor is only the beginning (or middle) */
4197 			if (!rxtop) {
4198 				/* this is the beginning of a chain */
4199 				rxtop = napi_get_frags(&adapter->napi);
4200 				if (!rxtop)
4201 					break;
4202 
4203 				skb_fill_page_desc(rxtop, 0,
4204 						   buffer_info->rxbuf.page,
4205 						   0, length);
4206 			} else {
4207 				/* this is the middle of a chain */
4208 				skb_fill_page_desc(rxtop,
4209 				    skb_shinfo(rxtop)->nr_frags,
4210 				    buffer_info->rxbuf.page, 0, length);
4211 			}
4212 			e1000_consume_page(buffer_info, rxtop, length);
4213 			goto next_desc;
4214 		} else {
4215 			if (rxtop) {
4216 				/* end of the chain */
4217 				skb_fill_page_desc(rxtop,
4218 				    skb_shinfo(rxtop)->nr_frags,
4219 				    buffer_info->rxbuf.page, 0, length);
4220 				skb = rxtop;
4221 				rxtop = NULL;
4222 				e1000_consume_page(buffer_info, skb, length);
4223 			} else {
4224 				struct page *p;
4225 				/* no chain, got EOP, this buf is the packet
4226 				 * copybreak to save the put_page/alloc_page
4227 				 */
4228 				p = buffer_info->rxbuf.page;
4229 				if (length <= copybreak) {
4230 					u8 *vaddr;
4231 
4232 					if (likely(!(netdev->features & NETIF_F_RXFCS)))
4233 						length -= 4;
4234 					skb = e1000_alloc_rx_skb(adapter,
4235 								 length);
4236 					if (!skb)
4237 						break;
4238 
4239 					vaddr = kmap_atomic(p);
4240 					memcpy(skb_tail_pointer(skb), vaddr,
4241 					       length);
4242 					kunmap_atomic(vaddr);
4243 					/* re-use the page, so don't erase
4244 					 * buffer_info->rxbuf.page
4245 					 */
4246 					skb_put(skb, length);
4247 					e1000_rx_checksum(adapter,
4248 							  status | rx_desc->errors << 24,
4249 							  le16_to_cpu(rx_desc->csum), skb);
4250 
4251 					total_rx_bytes += skb->len;
4252 					total_rx_packets++;
4253 
4254 					e1000_receive_skb(adapter, status,
4255 							  rx_desc->special, skb);
4256 					goto next_desc;
4257 				} else {
4258 					skb = napi_get_frags(&adapter->napi);
4259 					if (!skb) {
4260 						adapter->alloc_rx_buff_failed++;
4261 						break;
4262 					}
4263 					skb_fill_page_desc(skb, 0, p, 0,
4264 							   length);
4265 					e1000_consume_page(buffer_info, skb,
4266 							   length);
4267 				}
4268 			}
4269 		}
4270 
4271 		/* Receive Checksum Offload XXX recompute due to CRC strip? */
4272 		e1000_rx_checksum(adapter,
4273 				  (u32)(status) |
4274 				  ((u32)(rx_desc->errors) << 24),
4275 				  le16_to_cpu(rx_desc->csum), skb);
4276 
4277 		total_rx_bytes += (skb->len - 4); /* don't count FCS */
4278 		if (likely(!(netdev->features & NETIF_F_RXFCS)))
4279 			pskb_trim(skb, skb->len - 4);
4280 		total_rx_packets++;
4281 
4282 		if (status & E1000_RXD_STAT_VP) {
4283 			__le16 vlan = rx_desc->special;
4284 			u16 vid = le16_to_cpu(vlan) & E1000_RXD_SPC_VLAN_MASK;
4285 
4286 			__vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vid);
4287 		}
4288 
4289 		napi_gro_frags(&adapter->napi);
4290 
4291 next_desc:
4292 		rx_desc->status = 0;
4293 
4294 		/* return some buffers to hardware, one at a time is too slow */
4295 		if (unlikely(cleaned_count >= E1000_RX_BUFFER_WRITE)) {
4296 			adapter->alloc_rx_buf(adapter, rx_ring, cleaned_count);
4297 			cleaned_count = 0;
4298 		}
4299 
4300 		/* use prefetched values */
4301 		rx_desc = next_rxd;
4302 		buffer_info = next_buffer;
4303 	}
4304 	rx_ring->next_to_clean = i;
4305 
4306 	cleaned_count = E1000_DESC_UNUSED(rx_ring);
4307 	if (cleaned_count)
4308 		adapter->alloc_rx_buf(adapter, rx_ring, cleaned_count);
4309 
4310 	adapter->total_rx_packets += total_rx_packets;
4311 	adapter->total_rx_bytes += total_rx_bytes;
4312 	netdev->stats.rx_bytes += total_rx_bytes;
4313 	netdev->stats.rx_packets += total_rx_packets;
4314 	return cleaned;
4315 }
4316 
4317 /* this should improve performance for small packets with large amounts
4318  * of reassembly being done in the stack
4319  */
4320 static struct sk_buff *e1000_copybreak(struct e1000_adapter *adapter,
4321 				       struct e1000_rx_buffer *buffer_info,
4322 				       u32 length, const void *data)
4323 {
4324 	struct sk_buff *skb;
4325 
4326 	if (length > copybreak)
4327 		return NULL;
4328 
4329 	skb = e1000_alloc_rx_skb(adapter, length);
4330 	if (!skb)
4331 		return NULL;
4332 
4333 	dma_sync_single_for_cpu(&adapter->pdev->dev, buffer_info->dma,
4334 				length, DMA_FROM_DEVICE);
4335 
4336 	skb_put_data(skb, data, length);
4337 
4338 	return skb;
4339 }
4340 
4341 /**
4342  * e1000_clean_rx_irq - Send received data up the network stack; legacy
4343  * @adapter: board private structure
4344  * @rx_ring: ring to clean
4345  * @work_done: amount of napi work completed this call
4346  * @work_to_do: max amount of work allowed for this call to do
4347  */
4348 static bool e1000_clean_rx_irq(struct e1000_adapter *adapter,
4349 			       struct e1000_rx_ring *rx_ring,
4350 			       int *work_done, int work_to_do)
4351 {
4352 	struct net_device *netdev = adapter->netdev;
4353 	struct pci_dev *pdev = adapter->pdev;
4354 	struct e1000_rx_desc *rx_desc, *next_rxd;
4355 	struct e1000_rx_buffer *buffer_info, *next_buffer;
4356 	u32 length;
4357 	unsigned int i;
4358 	int cleaned_count = 0;
4359 	bool cleaned = false;
4360 	unsigned int total_rx_bytes = 0, total_rx_packets = 0;
4361 
4362 	i = rx_ring->next_to_clean;
4363 	rx_desc = E1000_RX_DESC(*rx_ring, i);
4364 	buffer_info = &rx_ring->buffer_info[i];
4365 
4366 	while (rx_desc->status & E1000_RXD_STAT_DD) {
4367 		struct sk_buff *skb;
4368 		u8 *data;
4369 		u8 status;
4370 
4371 		if (*work_done >= work_to_do)
4372 			break;
4373 		(*work_done)++;
4374 		dma_rmb(); /* read descriptor and rx_buffer_info after status DD */
4375 
4376 		status = rx_desc->status;
4377 		length = le16_to_cpu(rx_desc->length);
4378 
4379 		data = buffer_info->rxbuf.data;
4380 		prefetch(data);
4381 		skb = e1000_copybreak(adapter, buffer_info, length, data);
4382 		if (!skb) {
4383 			unsigned int frag_len = e1000_frag_len(adapter);
4384 
4385 			skb = build_skb(data - E1000_HEADROOM, frag_len);
4386 			if (!skb) {
4387 				adapter->alloc_rx_buff_failed++;
4388 				break;
4389 			}
4390 
4391 			skb_reserve(skb, E1000_HEADROOM);
4392 			dma_unmap_single(&pdev->dev, buffer_info->dma,
4393 					 adapter->rx_buffer_len,
4394 					 DMA_FROM_DEVICE);
4395 			buffer_info->dma = 0;
4396 			buffer_info->rxbuf.data = NULL;
4397 		}
4398 
4399 		if (++i == rx_ring->count)
4400 			i = 0;
4401 
4402 		next_rxd = E1000_RX_DESC(*rx_ring, i);
4403 		prefetch(next_rxd);
4404 
4405 		next_buffer = &rx_ring->buffer_info[i];
4406 
4407 		cleaned = true;
4408 		cleaned_count++;
4409 
4410 		/* !EOP means multiple descriptors were used to store a single
4411 		 * packet, if thats the case we need to toss it.  In fact, we
4412 		 * to toss every packet with the EOP bit clear and the next
4413 		 * frame that _does_ have the EOP bit set, as it is by
4414 		 * definition only a frame fragment
4415 		 */
4416 		if (unlikely(!(status & E1000_RXD_STAT_EOP)))
4417 			adapter->discarding = true;
4418 
4419 		if (adapter->discarding) {
4420 			/* All receives must fit into a single buffer */
4421 			netdev_dbg(netdev, "Receive packet consumed multiple buffers\n");
4422 			dev_kfree_skb(skb);
4423 			if (status & E1000_RXD_STAT_EOP)
4424 				adapter->discarding = false;
4425 			goto next_desc;
4426 		}
4427 
4428 		if (unlikely(rx_desc->errors & E1000_RXD_ERR_FRAME_ERR_MASK)) {
4429 			if (e1000_tbi_should_accept(adapter, status,
4430 						    rx_desc->errors,
4431 						    length, data)) {
4432 				length--;
4433 			} else if (netdev->features & NETIF_F_RXALL) {
4434 				goto process_skb;
4435 			} else {
4436 				dev_kfree_skb(skb);
4437 				goto next_desc;
4438 			}
4439 		}
4440 
4441 process_skb:
4442 		total_rx_bytes += (length - 4); /* don't count FCS */
4443 		total_rx_packets++;
4444 
4445 		if (likely(!(netdev->features & NETIF_F_RXFCS)))
4446 			/* adjust length to remove Ethernet CRC, this must be
4447 			 * done after the TBI_ACCEPT workaround above
4448 			 */
4449 			length -= 4;
4450 
4451 		if (buffer_info->rxbuf.data == NULL)
4452 			skb_put(skb, length);
4453 		else /* copybreak skb */
4454 			skb_trim(skb, length);
4455 
4456 		/* Receive Checksum Offload */
4457 		e1000_rx_checksum(adapter,
4458 				  (u32)(status) |
4459 				  ((u32)(rx_desc->errors) << 24),
4460 				  le16_to_cpu(rx_desc->csum), skb);
4461 
4462 		e1000_receive_skb(adapter, status, rx_desc->special, skb);
4463 
4464 next_desc:
4465 		rx_desc->status = 0;
4466 
4467 		/* return some buffers to hardware, one at a time is too slow */
4468 		if (unlikely(cleaned_count >= E1000_RX_BUFFER_WRITE)) {
4469 			adapter->alloc_rx_buf(adapter, rx_ring, cleaned_count);
4470 			cleaned_count = 0;
4471 		}
4472 
4473 		/* use prefetched values */
4474 		rx_desc = next_rxd;
4475 		buffer_info = next_buffer;
4476 	}
4477 	rx_ring->next_to_clean = i;
4478 
4479 	cleaned_count = E1000_DESC_UNUSED(rx_ring);
4480 	if (cleaned_count)
4481 		adapter->alloc_rx_buf(adapter, rx_ring, cleaned_count);
4482 
4483 	adapter->total_rx_packets += total_rx_packets;
4484 	adapter->total_rx_bytes += total_rx_bytes;
4485 	netdev->stats.rx_bytes += total_rx_bytes;
4486 	netdev->stats.rx_packets += total_rx_packets;
4487 	return cleaned;
4488 }
4489 
4490 /**
4491  * e1000_alloc_jumbo_rx_buffers - Replace used jumbo receive buffers
4492  * @adapter: address of board private structure
4493  * @rx_ring: pointer to receive ring structure
4494  * @cleaned_count: number of buffers to allocate this pass
4495  **/
4496 static void
4497 e1000_alloc_jumbo_rx_buffers(struct e1000_adapter *adapter,
4498 			     struct e1000_rx_ring *rx_ring, int cleaned_count)
4499 {
4500 	struct pci_dev *pdev = adapter->pdev;
4501 	struct e1000_rx_desc *rx_desc;
4502 	struct e1000_rx_buffer *buffer_info;
4503 	unsigned int i;
4504 
4505 	i = rx_ring->next_to_use;
4506 	buffer_info = &rx_ring->buffer_info[i];
4507 
4508 	while (cleaned_count--) {
4509 		/* allocate a new page if necessary */
4510 		if (!buffer_info->rxbuf.page) {
4511 			buffer_info->rxbuf.page = alloc_page(GFP_ATOMIC);
4512 			if (unlikely(!buffer_info->rxbuf.page)) {
4513 				adapter->alloc_rx_buff_failed++;
4514 				break;
4515 			}
4516 		}
4517 
4518 		if (!buffer_info->dma) {
4519 			buffer_info->dma = dma_map_page(&pdev->dev,
4520 							buffer_info->rxbuf.page, 0,
4521 							adapter->rx_buffer_len,
4522 							DMA_FROM_DEVICE);
4523 			if (dma_mapping_error(&pdev->dev, buffer_info->dma)) {
4524 				put_page(buffer_info->rxbuf.page);
4525 				buffer_info->rxbuf.page = NULL;
4526 				buffer_info->dma = 0;
4527 				adapter->alloc_rx_buff_failed++;
4528 				break;
4529 			}
4530 		}
4531 
4532 		rx_desc = E1000_RX_DESC(*rx_ring, i);
4533 		rx_desc->buffer_addr = cpu_to_le64(buffer_info->dma);
4534 
4535 		if (unlikely(++i == rx_ring->count))
4536 			i = 0;
4537 		buffer_info = &rx_ring->buffer_info[i];
4538 	}
4539 
4540 	if (likely(rx_ring->next_to_use != i)) {
4541 		rx_ring->next_to_use = i;
4542 		if (unlikely(i-- == 0))
4543 			i = (rx_ring->count - 1);
4544 
4545 		/* Force memory writes to complete before letting h/w
4546 		 * know there are new descriptors to fetch.  (Only
4547 		 * applicable for weak-ordered memory model archs,
4548 		 * such as IA-64).
4549 		 */
4550 		dma_wmb();
4551 		writel(i, adapter->hw.hw_addr + rx_ring->rdt);
4552 	}
4553 }
4554 
4555 /**
4556  * e1000_alloc_rx_buffers - Replace used receive buffers; legacy & extended
4557  * @adapter: address of board private structure
4558  * @rx_ring: pointer to ring struct
4559  * @cleaned_count: number of new Rx buffers to try to allocate
4560  **/
4561 static void e1000_alloc_rx_buffers(struct e1000_adapter *adapter,
4562 				   struct e1000_rx_ring *rx_ring,
4563 				   int cleaned_count)
4564 {
4565 	struct e1000_hw *hw = &adapter->hw;
4566 	struct pci_dev *pdev = adapter->pdev;
4567 	struct e1000_rx_desc *rx_desc;
4568 	struct e1000_rx_buffer *buffer_info;
4569 	unsigned int i;
4570 	unsigned int bufsz = adapter->rx_buffer_len;
4571 
4572 	i = rx_ring->next_to_use;
4573 	buffer_info = &rx_ring->buffer_info[i];
4574 
4575 	while (cleaned_count--) {
4576 		void *data;
4577 
4578 		if (buffer_info->rxbuf.data)
4579 			goto skip;
4580 
4581 		data = e1000_alloc_frag(adapter);
4582 		if (!data) {
4583 			/* Better luck next round */
4584 			adapter->alloc_rx_buff_failed++;
4585 			break;
4586 		}
4587 
4588 		/* Fix for errata 23, can't cross 64kB boundary */
4589 		if (!e1000_check_64k_bound(adapter, data, bufsz)) {
4590 			void *olddata = data;
4591 			e_err(rx_err, "skb align check failed: %u bytes at "
4592 			      "%p\n", bufsz, data);
4593 			/* Try again, without freeing the previous */
4594 			data = e1000_alloc_frag(adapter);
4595 			/* Failed allocation, critical failure */
4596 			if (!data) {
4597 				skb_free_frag(olddata);
4598 				adapter->alloc_rx_buff_failed++;
4599 				break;
4600 			}
4601 
4602 			if (!e1000_check_64k_bound(adapter, data, bufsz)) {
4603 				/* give up */
4604 				skb_free_frag(data);
4605 				skb_free_frag(olddata);
4606 				adapter->alloc_rx_buff_failed++;
4607 				break;
4608 			}
4609 
4610 			/* Use new allocation */
4611 			skb_free_frag(olddata);
4612 		}
4613 		buffer_info->dma = dma_map_single(&pdev->dev,
4614 						  data,
4615 						  adapter->rx_buffer_len,
4616 						  DMA_FROM_DEVICE);
4617 		if (dma_mapping_error(&pdev->dev, buffer_info->dma)) {
4618 			skb_free_frag(data);
4619 			buffer_info->dma = 0;
4620 			adapter->alloc_rx_buff_failed++;
4621 			break;
4622 		}
4623 
4624 		/* XXX if it was allocated cleanly it will never map to a
4625 		 * boundary crossing
4626 		 */
4627 
4628 		/* Fix for errata 23, can't cross 64kB boundary */
4629 		if (!e1000_check_64k_bound(adapter,
4630 					(void *)(unsigned long)buffer_info->dma,
4631 					adapter->rx_buffer_len)) {
4632 			e_err(rx_err, "dma align check failed: %u bytes at "
4633 			      "%p\n", adapter->rx_buffer_len,
4634 			      (void *)(unsigned long)buffer_info->dma);
4635 
4636 			dma_unmap_single(&pdev->dev, buffer_info->dma,
4637 					 adapter->rx_buffer_len,
4638 					 DMA_FROM_DEVICE);
4639 
4640 			skb_free_frag(data);
4641 			buffer_info->rxbuf.data = NULL;
4642 			buffer_info->dma = 0;
4643 
4644 			adapter->alloc_rx_buff_failed++;
4645 			break;
4646 		}
4647 		buffer_info->rxbuf.data = data;
4648  skip:
4649 		rx_desc = E1000_RX_DESC(*rx_ring, i);
4650 		rx_desc->buffer_addr = cpu_to_le64(buffer_info->dma);
4651 
4652 		if (unlikely(++i == rx_ring->count))
4653 			i = 0;
4654 		buffer_info = &rx_ring->buffer_info[i];
4655 	}
4656 
4657 	if (likely(rx_ring->next_to_use != i)) {
4658 		rx_ring->next_to_use = i;
4659 		if (unlikely(i-- == 0))
4660 			i = (rx_ring->count - 1);
4661 
4662 		/* Force memory writes to complete before letting h/w
4663 		 * know there are new descriptors to fetch.  (Only
4664 		 * applicable for weak-ordered memory model archs,
4665 		 * such as IA-64).
4666 		 */
4667 		dma_wmb();
4668 		writel(i, hw->hw_addr + rx_ring->rdt);
4669 	}
4670 }
4671 
4672 /**
4673  * e1000_smartspeed - Workaround for SmartSpeed on 82541 and 82547 controllers.
4674  * @adapter: address of board private structure
4675  **/
4676 static void e1000_smartspeed(struct e1000_adapter *adapter)
4677 {
4678 	struct e1000_hw *hw = &adapter->hw;
4679 	u16 phy_status;
4680 	u16 phy_ctrl;
4681 
4682 	if ((hw->phy_type != e1000_phy_igp) || !hw->autoneg ||
4683 	   !(hw->autoneg_advertised & ADVERTISE_1000_FULL))
4684 		return;
4685 
4686 	if (adapter->smartspeed == 0) {
4687 		/* If Master/Slave config fault is asserted twice,
4688 		 * we assume back-to-back
4689 		 */
4690 		e1000_read_phy_reg(hw, PHY_1000T_STATUS, &phy_status);
4691 		if (!(phy_status & SR_1000T_MS_CONFIG_FAULT))
4692 			return;
4693 		e1000_read_phy_reg(hw, PHY_1000T_STATUS, &phy_status);
4694 		if (!(phy_status & SR_1000T_MS_CONFIG_FAULT))
4695 			return;
4696 		e1000_read_phy_reg(hw, PHY_1000T_CTRL, &phy_ctrl);
4697 		if (phy_ctrl & CR_1000T_MS_ENABLE) {
4698 			phy_ctrl &= ~CR_1000T_MS_ENABLE;
4699 			e1000_write_phy_reg(hw, PHY_1000T_CTRL,
4700 					    phy_ctrl);
4701 			adapter->smartspeed++;
4702 			if (!e1000_phy_setup_autoneg(hw) &&
4703 			   !e1000_read_phy_reg(hw, PHY_CTRL,
4704 					       &phy_ctrl)) {
4705 				phy_ctrl |= (MII_CR_AUTO_NEG_EN |
4706 					     MII_CR_RESTART_AUTO_NEG);
4707 				e1000_write_phy_reg(hw, PHY_CTRL,
4708 						    phy_ctrl);
4709 			}
4710 		}
4711 		return;
4712 	} else if (adapter->smartspeed == E1000_SMARTSPEED_DOWNSHIFT) {
4713 		/* If still no link, perhaps using 2/3 pair cable */
4714 		e1000_read_phy_reg(hw, PHY_1000T_CTRL, &phy_ctrl);
4715 		phy_ctrl |= CR_1000T_MS_ENABLE;
4716 		e1000_write_phy_reg(hw, PHY_1000T_CTRL, phy_ctrl);
4717 		if (!e1000_phy_setup_autoneg(hw) &&
4718 		   !e1000_read_phy_reg(hw, PHY_CTRL, &phy_ctrl)) {
4719 			phy_ctrl |= (MII_CR_AUTO_NEG_EN |
4720 				     MII_CR_RESTART_AUTO_NEG);
4721 			e1000_write_phy_reg(hw, PHY_CTRL, phy_ctrl);
4722 		}
4723 	}
4724 	/* Restart process after E1000_SMARTSPEED_MAX iterations */
4725 	if (adapter->smartspeed++ == E1000_SMARTSPEED_MAX)
4726 		adapter->smartspeed = 0;
4727 }
4728 
4729 /**
4730  * e1000_ioctl - handle ioctl calls
4731  * @netdev: pointer to our netdev
4732  * @ifr: pointer to interface request structure
4733  * @cmd: ioctl data
4734  **/
4735 static int e1000_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
4736 {
4737 	switch (cmd) {
4738 	case SIOCGMIIPHY:
4739 	case SIOCGMIIREG:
4740 	case SIOCSMIIREG:
4741 		return e1000_mii_ioctl(netdev, ifr, cmd);
4742 	default:
4743 		return -EOPNOTSUPP;
4744 	}
4745 }
4746 
4747 /**
4748  * e1000_mii_ioctl -
4749  * @netdev: pointer to our netdev
4750  * @ifr: pointer to interface request structure
4751  * @cmd: ioctl data
4752  **/
4753 static int e1000_mii_ioctl(struct net_device *netdev, struct ifreq *ifr,
4754 			   int cmd)
4755 {
4756 	struct e1000_adapter *adapter = netdev_priv(netdev);
4757 	struct e1000_hw *hw = &adapter->hw;
4758 	struct mii_ioctl_data *data = if_mii(ifr);
4759 	int retval;
4760 	u16 mii_reg;
4761 	unsigned long flags;
4762 
4763 	if (hw->media_type != e1000_media_type_copper)
4764 		return -EOPNOTSUPP;
4765 
4766 	switch (cmd) {
4767 	case SIOCGMIIPHY:
4768 		data->phy_id = hw->phy_addr;
4769 		break;
4770 	case SIOCGMIIREG:
4771 		spin_lock_irqsave(&adapter->stats_lock, flags);
4772 		if (e1000_read_phy_reg(hw, data->reg_num & 0x1F,
4773 				   &data->val_out)) {
4774 			spin_unlock_irqrestore(&adapter->stats_lock, flags);
4775 			return -EIO;
4776 		}
4777 		spin_unlock_irqrestore(&adapter->stats_lock, flags);
4778 		break;
4779 	case SIOCSMIIREG:
4780 		if (data->reg_num & ~(0x1F))
4781 			return -EFAULT;
4782 		mii_reg = data->val_in;
4783 		spin_lock_irqsave(&adapter->stats_lock, flags);
4784 		if (e1000_write_phy_reg(hw, data->reg_num,
4785 					mii_reg)) {
4786 			spin_unlock_irqrestore(&adapter->stats_lock, flags);
4787 			return -EIO;
4788 		}
4789 		spin_unlock_irqrestore(&adapter->stats_lock, flags);
4790 		if (hw->media_type == e1000_media_type_copper) {
4791 			switch (data->reg_num) {
4792 			case PHY_CTRL:
4793 				if (mii_reg & MII_CR_POWER_DOWN)
4794 					break;
4795 				if (mii_reg & MII_CR_AUTO_NEG_EN) {
4796 					hw->autoneg = 1;
4797 					hw->autoneg_advertised = 0x2F;
4798 				} else {
4799 					u32 speed;
4800 					if (mii_reg & 0x40)
4801 						speed = SPEED_1000;
4802 					else if (mii_reg & 0x2000)
4803 						speed = SPEED_100;
4804 					else
4805 						speed = SPEED_10;
4806 					retval = e1000_set_spd_dplx(
4807 						adapter, speed,
4808 						((mii_reg & 0x100)
4809 						 ? DUPLEX_FULL :
4810 						 DUPLEX_HALF));
4811 					if (retval)
4812 						return retval;
4813 				}
4814 				if (netif_running(adapter->netdev))
4815 					e1000_reinit_locked(adapter);
4816 				else
4817 					e1000_reset(adapter);
4818 				break;
4819 			case M88E1000_PHY_SPEC_CTRL:
4820 			case M88E1000_EXT_PHY_SPEC_CTRL:
4821 				if (e1000_phy_reset(hw))
4822 					return -EIO;
4823 				break;
4824 			}
4825 		} else {
4826 			switch (data->reg_num) {
4827 			case PHY_CTRL:
4828 				if (mii_reg & MII_CR_POWER_DOWN)
4829 					break;
4830 				if (netif_running(adapter->netdev))
4831 					e1000_reinit_locked(adapter);
4832 				else
4833 					e1000_reset(adapter);
4834 				break;
4835 			}
4836 		}
4837 		break;
4838 	default:
4839 		return -EOPNOTSUPP;
4840 	}
4841 	return E1000_SUCCESS;
4842 }
4843 
4844 void e1000_pci_set_mwi(struct e1000_hw *hw)
4845 {
4846 	struct e1000_adapter *adapter = hw->back;
4847 	int ret_val = pci_set_mwi(adapter->pdev);
4848 
4849 	if (ret_val)
4850 		e_err(probe, "Error in setting MWI\n");
4851 }
4852 
4853 void e1000_pci_clear_mwi(struct e1000_hw *hw)
4854 {
4855 	struct e1000_adapter *adapter = hw->back;
4856 
4857 	pci_clear_mwi(adapter->pdev);
4858 }
4859 
4860 int e1000_pcix_get_mmrbc(struct e1000_hw *hw)
4861 {
4862 	struct e1000_adapter *adapter = hw->back;
4863 	return pcix_get_mmrbc(adapter->pdev);
4864 }
4865 
4866 void e1000_pcix_set_mmrbc(struct e1000_hw *hw, int mmrbc)
4867 {
4868 	struct e1000_adapter *adapter = hw->back;
4869 	pcix_set_mmrbc(adapter->pdev, mmrbc);
4870 }
4871 
4872 void e1000_io_write(struct e1000_hw *hw, unsigned long port, u32 value)
4873 {
4874 	outl(value, port);
4875 }
4876 
4877 static bool e1000_vlan_used(struct e1000_adapter *adapter)
4878 {
4879 	u16 vid;
4880 
4881 	for_each_set_bit(vid, adapter->active_vlans, VLAN_N_VID)
4882 		return true;
4883 	return false;
4884 }
4885 
4886 static void __e1000_vlan_mode(struct e1000_adapter *adapter,
4887 			      netdev_features_t features)
4888 {
4889 	struct e1000_hw *hw = &adapter->hw;
4890 	u32 ctrl;
4891 
4892 	ctrl = er32(CTRL);
4893 	if (features & NETIF_F_HW_VLAN_CTAG_RX) {
4894 		/* enable VLAN tag insert/strip */
4895 		ctrl |= E1000_CTRL_VME;
4896 	} else {
4897 		/* disable VLAN tag insert/strip */
4898 		ctrl &= ~E1000_CTRL_VME;
4899 	}
4900 	ew32(CTRL, ctrl);
4901 }
4902 static void e1000_vlan_filter_on_off(struct e1000_adapter *adapter,
4903 				     bool filter_on)
4904 {
4905 	struct e1000_hw *hw = &adapter->hw;
4906 	u32 rctl;
4907 
4908 	if (!test_bit(__E1000_DOWN, &adapter->flags))
4909 		e1000_irq_disable(adapter);
4910 
4911 	__e1000_vlan_mode(adapter, adapter->netdev->features);
4912 	if (filter_on) {
4913 		/* enable VLAN receive filtering */
4914 		rctl = er32(RCTL);
4915 		rctl &= ~E1000_RCTL_CFIEN;
4916 		if (!(adapter->netdev->flags & IFF_PROMISC))
4917 			rctl |= E1000_RCTL_VFE;
4918 		ew32(RCTL, rctl);
4919 		e1000_update_mng_vlan(adapter);
4920 	} else {
4921 		/* disable VLAN receive filtering */
4922 		rctl = er32(RCTL);
4923 		rctl &= ~E1000_RCTL_VFE;
4924 		ew32(RCTL, rctl);
4925 	}
4926 
4927 	if (!test_bit(__E1000_DOWN, &adapter->flags))
4928 		e1000_irq_enable(adapter);
4929 }
4930 
4931 static void e1000_vlan_mode(struct net_device *netdev,
4932 			    netdev_features_t features)
4933 {
4934 	struct e1000_adapter *adapter = netdev_priv(netdev);
4935 
4936 	if (!test_bit(__E1000_DOWN, &adapter->flags))
4937 		e1000_irq_disable(adapter);
4938 
4939 	__e1000_vlan_mode(adapter, features);
4940 
4941 	if (!test_bit(__E1000_DOWN, &adapter->flags))
4942 		e1000_irq_enable(adapter);
4943 }
4944 
4945 static int e1000_vlan_rx_add_vid(struct net_device *netdev,
4946 				 __be16 proto, u16 vid)
4947 {
4948 	struct e1000_adapter *adapter = netdev_priv(netdev);
4949 	struct e1000_hw *hw = &adapter->hw;
4950 	u32 vfta, index;
4951 
4952 	if ((hw->mng_cookie.status &
4953 	     E1000_MNG_DHCP_COOKIE_STATUS_VLAN_SUPPORT) &&
4954 	    (vid == adapter->mng_vlan_id))
4955 		return 0;
4956 
4957 	if (!e1000_vlan_used(adapter))
4958 		e1000_vlan_filter_on_off(adapter, true);
4959 
4960 	/* add VID to filter table */
4961 	index = (vid >> 5) & 0x7F;
4962 	vfta = E1000_READ_REG_ARRAY(hw, VFTA, index);
4963 	vfta |= (1 << (vid & 0x1F));
4964 	e1000_write_vfta(hw, index, vfta);
4965 
4966 	set_bit(vid, adapter->active_vlans);
4967 
4968 	return 0;
4969 }
4970 
4971 static int e1000_vlan_rx_kill_vid(struct net_device *netdev,
4972 				  __be16 proto, u16 vid)
4973 {
4974 	struct e1000_adapter *adapter = netdev_priv(netdev);
4975 	struct e1000_hw *hw = &adapter->hw;
4976 	u32 vfta, index;
4977 
4978 	if (!test_bit(__E1000_DOWN, &adapter->flags))
4979 		e1000_irq_disable(adapter);
4980 	if (!test_bit(__E1000_DOWN, &adapter->flags))
4981 		e1000_irq_enable(adapter);
4982 
4983 	/* remove VID from filter table */
4984 	index = (vid >> 5) & 0x7F;
4985 	vfta = E1000_READ_REG_ARRAY(hw, VFTA, index);
4986 	vfta &= ~(1 << (vid & 0x1F));
4987 	e1000_write_vfta(hw, index, vfta);
4988 
4989 	clear_bit(vid, adapter->active_vlans);
4990 
4991 	if (!e1000_vlan_used(adapter))
4992 		e1000_vlan_filter_on_off(adapter, false);
4993 
4994 	return 0;
4995 }
4996 
4997 static void e1000_restore_vlan(struct e1000_adapter *adapter)
4998 {
4999 	u16 vid;
5000 
5001 	if (!e1000_vlan_used(adapter))
5002 		return;
5003 
5004 	e1000_vlan_filter_on_off(adapter, true);
5005 	for_each_set_bit(vid, adapter->active_vlans, VLAN_N_VID)
5006 		e1000_vlan_rx_add_vid(adapter->netdev, htons(ETH_P_8021Q), vid);
5007 }
5008 
5009 int e1000_set_spd_dplx(struct e1000_adapter *adapter, u32 spd, u8 dplx)
5010 {
5011 	struct e1000_hw *hw = &adapter->hw;
5012 
5013 	hw->autoneg = 0;
5014 
5015 	/* Make sure dplx is at most 1 bit and lsb of speed is not set
5016 	 * for the switch() below to work
5017 	 */
5018 	if ((spd & 1) || (dplx & ~1))
5019 		goto err_inval;
5020 
5021 	/* Fiber NICs only allow 1000 gbps Full duplex */
5022 	if ((hw->media_type == e1000_media_type_fiber) &&
5023 	    spd != SPEED_1000 &&
5024 	    dplx != DUPLEX_FULL)
5025 		goto err_inval;
5026 
5027 	switch (spd + dplx) {
5028 	case SPEED_10 + DUPLEX_HALF:
5029 		hw->forced_speed_duplex = e1000_10_half;
5030 		break;
5031 	case SPEED_10 + DUPLEX_FULL:
5032 		hw->forced_speed_duplex = e1000_10_full;
5033 		break;
5034 	case SPEED_100 + DUPLEX_HALF:
5035 		hw->forced_speed_duplex = e1000_100_half;
5036 		break;
5037 	case SPEED_100 + DUPLEX_FULL:
5038 		hw->forced_speed_duplex = e1000_100_full;
5039 		break;
5040 	case SPEED_1000 + DUPLEX_FULL:
5041 		hw->autoneg = 1;
5042 		hw->autoneg_advertised = ADVERTISE_1000_FULL;
5043 		break;
5044 	case SPEED_1000 + DUPLEX_HALF: /* not supported */
5045 	default:
5046 		goto err_inval;
5047 	}
5048 
5049 	/* clear MDI, MDI(-X) override is only allowed when autoneg enabled */
5050 	hw->mdix = AUTO_ALL_MODES;
5051 
5052 	return 0;
5053 
5054 err_inval:
5055 	e_err(probe, "Unsupported Speed/Duplex configuration\n");
5056 	return -EINVAL;
5057 }
5058 
5059 static int __e1000_shutdown(struct pci_dev *pdev, bool *enable_wake)
5060 {
5061 	struct net_device *netdev = pci_get_drvdata(pdev);
5062 	struct e1000_adapter *adapter = netdev_priv(netdev);
5063 	struct e1000_hw *hw = &adapter->hw;
5064 	u32 ctrl, ctrl_ext, rctl, status;
5065 	u32 wufc = adapter->wol;
5066 
5067 	netif_device_detach(netdev);
5068 
5069 	if (netif_running(netdev)) {
5070 		int count = E1000_CHECK_RESET_COUNT;
5071 
5072 		while (test_bit(__E1000_RESETTING, &adapter->flags) && count--)
5073 			usleep_range(10000, 20000);
5074 
5075 		WARN_ON(test_bit(__E1000_RESETTING, &adapter->flags));
5076 		e1000_down(adapter);
5077 	}
5078 
5079 	status = er32(STATUS);
5080 	if (status & E1000_STATUS_LU)
5081 		wufc &= ~E1000_WUFC_LNKC;
5082 
5083 	if (wufc) {
5084 		e1000_setup_rctl(adapter);
5085 		e1000_set_rx_mode(netdev);
5086 
5087 		rctl = er32(RCTL);
5088 
5089 		/* turn on all-multi mode if wake on multicast is enabled */
5090 		if (wufc & E1000_WUFC_MC)
5091 			rctl |= E1000_RCTL_MPE;
5092 
5093 		/* enable receives in the hardware */
5094 		ew32(RCTL, rctl | E1000_RCTL_EN);
5095 
5096 		if (hw->mac_type >= e1000_82540) {
5097 			ctrl = er32(CTRL);
5098 			/* advertise wake from D3Cold */
5099 			#define E1000_CTRL_ADVD3WUC 0x00100000
5100 			/* phy power management enable */
5101 			#define E1000_CTRL_EN_PHY_PWR_MGMT 0x00200000
5102 			ctrl |= E1000_CTRL_ADVD3WUC |
5103 				E1000_CTRL_EN_PHY_PWR_MGMT;
5104 			ew32(CTRL, ctrl);
5105 		}
5106 
5107 		if (hw->media_type == e1000_media_type_fiber ||
5108 		    hw->media_type == e1000_media_type_internal_serdes) {
5109 			/* keep the laser running in D3 */
5110 			ctrl_ext = er32(CTRL_EXT);
5111 			ctrl_ext |= E1000_CTRL_EXT_SDP7_DATA;
5112 			ew32(CTRL_EXT, ctrl_ext);
5113 		}
5114 
5115 		ew32(WUC, E1000_WUC_PME_EN);
5116 		ew32(WUFC, wufc);
5117 	} else {
5118 		ew32(WUC, 0);
5119 		ew32(WUFC, 0);
5120 	}
5121 
5122 	e1000_release_manageability(adapter);
5123 
5124 	*enable_wake = !!wufc;
5125 
5126 	/* make sure adapter isn't asleep if manageability is enabled */
5127 	if (adapter->en_mng_pt)
5128 		*enable_wake = true;
5129 
5130 	if (netif_running(netdev))
5131 		e1000_free_irq(adapter);
5132 
5133 	if (!test_and_set_bit(__E1000_DISABLED, &adapter->flags))
5134 		pci_disable_device(pdev);
5135 
5136 	return 0;
5137 }
5138 
5139 static int __maybe_unused e1000_suspend(struct device *dev)
5140 {
5141 	int retval;
5142 	struct pci_dev *pdev = to_pci_dev(dev);
5143 	bool wake;
5144 
5145 	retval = __e1000_shutdown(pdev, &wake);
5146 	device_set_wakeup_enable(dev, wake);
5147 
5148 	return retval;
5149 }
5150 
5151 static int __maybe_unused e1000_resume(struct device *dev)
5152 {
5153 	struct pci_dev *pdev = to_pci_dev(dev);
5154 	struct net_device *netdev = pci_get_drvdata(pdev);
5155 	struct e1000_adapter *adapter = netdev_priv(netdev);
5156 	struct e1000_hw *hw = &adapter->hw;
5157 	u32 err;
5158 
5159 	if (adapter->need_ioport)
5160 		err = pci_enable_device(pdev);
5161 	else
5162 		err = pci_enable_device_mem(pdev);
5163 	if (err) {
5164 		pr_err("Cannot enable PCI device from suspend\n");
5165 		return err;
5166 	}
5167 
5168 	/* flush memory to make sure state is correct */
5169 	smp_mb__before_atomic();
5170 	clear_bit(__E1000_DISABLED, &adapter->flags);
5171 	pci_set_master(pdev);
5172 
5173 	pci_enable_wake(pdev, PCI_D3hot, 0);
5174 	pci_enable_wake(pdev, PCI_D3cold, 0);
5175 
5176 	if (netif_running(netdev)) {
5177 		err = e1000_request_irq(adapter);
5178 		if (err)
5179 			return err;
5180 	}
5181 
5182 	e1000_power_up_phy(adapter);
5183 	e1000_reset(adapter);
5184 	ew32(WUS, ~0);
5185 
5186 	e1000_init_manageability(adapter);
5187 
5188 	if (netif_running(netdev))
5189 		e1000_up(adapter);
5190 
5191 	netif_device_attach(netdev);
5192 
5193 	return 0;
5194 }
5195 
5196 static void e1000_shutdown(struct pci_dev *pdev)
5197 {
5198 	bool wake;
5199 
5200 	__e1000_shutdown(pdev, &wake);
5201 
5202 	if (system_state == SYSTEM_POWER_OFF) {
5203 		pci_wake_from_d3(pdev, wake);
5204 		pci_set_power_state(pdev, PCI_D3hot);
5205 	}
5206 }
5207 
5208 #ifdef CONFIG_NET_POLL_CONTROLLER
5209 /* Polling 'interrupt' - used by things like netconsole to send skbs
5210  * without having to re-enable interrupts. It's not called while
5211  * the interrupt routine is executing.
5212  */
5213 static void e1000_netpoll(struct net_device *netdev)
5214 {
5215 	struct e1000_adapter *adapter = netdev_priv(netdev);
5216 
5217 	if (disable_hardirq(adapter->pdev->irq))
5218 		e1000_intr(adapter->pdev->irq, netdev);
5219 	enable_irq(adapter->pdev->irq);
5220 }
5221 #endif
5222 
5223 /**
5224  * e1000_io_error_detected - called when PCI error is detected
5225  * @pdev: Pointer to PCI device
5226  * @state: The current pci connection state
5227  *
5228  * This function is called after a PCI bus error affecting
5229  * this device has been detected.
5230  */
5231 static pci_ers_result_t e1000_io_error_detected(struct pci_dev *pdev,
5232 						pci_channel_state_t state)
5233 {
5234 	struct net_device *netdev = pci_get_drvdata(pdev);
5235 	struct e1000_adapter *adapter = netdev_priv(netdev);
5236 
5237 	netif_device_detach(netdev);
5238 
5239 	if (state == pci_channel_io_perm_failure)
5240 		return PCI_ERS_RESULT_DISCONNECT;
5241 
5242 	if (netif_running(netdev))
5243 		e1000_down(adapter);
5244 
5245 	if (!test_and_set_bit(__E1000_DISABLED, &adapter->flags))
5246 		pci_disable_device(pdev);
5247 
5248 	/* Request a slot slot reset. */
5249 	return PCI_ERS_RESULT_NEED_RESET;
5250 }
5251 
5252 /**
5253  * e1000_io_slot_reset - called after the pci bus has been reset.
5254  * @pdev: Pointer to PCI device
5255  *
5256  * Restart the card from scratch, as if from a cold-boot. Implementation
5257  * resembles the first-half of the e1000_resume routine.
5258  */
5259 static pci_ers_result_t e1000_io_slot_reset(struct pci_dev *pdev)
5260 {
5261 	struct net_device *netdev = pci_get_drvdata(pdev);
5262 	struct e1000_adapter *adapter = netdev_priv(netdev);
5263 	struct e1000_hw *hw = &adapter->hw;
5264 	int err;
5265 
5266 	if (adapter->need_ioport)
5267 		err = pci_enable_device(pdev);
5268 	else
5269 		err = pci_enable_device_mem(pdev);
5270 	if (err) {
5271 		pr_err("Cannot re-enable PCI device after reset.\n");
5272 		return PCI_ERS_RESULT_DISCONNECT;
5273 	}
5274 
5275 	/* flush memory to make sure state is correct */
5276 	smp_mb__before_atomic();
5277 	clear_bit(__E1000_DISABLED, &adapter->flags);
5278 	pci_set_master(pdev);
5279 
5280 	pci_enable_wake(pdev, PCI_D3hot, 0);
5281 	pci_enable_wake(pdev, PCI_D3cold, 0);
5282 
5283 	e1000_reset(adapter);
5284 	ew32(WUS, ~0);
5285 
5286 	return PCI_ERS_RESULT_RECOVERED;
5287 }
5288 
5289 /**
5290  * e1000_io_resume - called when traffic can start flowing again.
5291  * @pdev: Pointer to PCI device
5292  *
5293  * This callback is called when the error recovery driver tells us that
5294  * its OK to resume normal operation. Implementation resembles the
5295  * second-half of the e1000_resume routine.
5296  */
5297 static void e1000_io_resume(struct pci_dev *pdev)
5298 {
5299 	struct net_device *netdev = pci_get_drvdata(pdev);
5300 	struct e1000_adapter *adapter = netdev_priv(netdev);
5301 
5302 	e1000_init_manageability(adapter);
5303 
5304 	if (netif_running(netdev)) {
5305 		if (e1000_up(adapter)) {
5306 			pr_info("can't bring device back up after reset\n");
5307 			return;
5308 		}
5309 	}
5310 
5311 	netif_device_attach(netdev);
5312 }
5313 
5314 /* e1000_main.c */
5315