xref: /titanic_41/usr/src/uts/common/io/igb/igb_main.c (revision c6a664189ff58eecb65704097a089f0bb4d35523)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 
22 /*
23  * Copyright (c) 2007-2012 Intel Corporation. All rights reserved.
24  */
25 
26 /*
27  * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved.
28  * Copyright 2013, Nexenta Systems, Inc. All rights reserved.
29  */
30 
31 #include "igb_sw.h"
32 
33 static char ident[] = "Intel 1Gb Ethernet";
34 static char igb_version[] = "igb 2.3.8-ish";
35 
36 /*
37  * Local function protoypes
38  */
39 static int igb_register_mac(igb_t *);
40 static int igb_identify_hardware(igb_t *);
41 static int igb_regs_map(igb_t *);
42 static void igb_init_properties(igb_t *);
43 static int igb_init_driver_settings(igb_t *);
44 static void igb_init_locks(igb_t *);
45 static void igb_destroy_locks(igb_t *);
46 static int igb_init_mac_address(igb_t *);
47 static int igb_init(igb_t *);
48 static int igb_init_adapter(igb_t *);
49 static void igb_stop_adapter(igb_t *);
50 static int igb_reset(igb_t *);
51 static void igb_tx_clean(igb_t *);
52 static boolean_t igb_tx_drain(igb_t *);
53 static boolean_t igb_rx_drain(igb_t *);
54 static int igb_alloc_rings(igb_t *);
55 static int igb_alloc_rx_data(igb_t *);
56 static void igb_free_rx_data(igb_t *);
57 static void igb_free_rings(igb_t *);
58 static void igb_setup_rings(igb_t *);
59 static void igb_setup_rx(igb_t *);
60 static void igb_setup_tx(igb_t *);
61 static void igb_setup_rx_ring(igb_rx_ring_t *);
62 static void igb_setup_tx_ring(igb_tx_ring_t *);
63 static void igb_setup_rss(igb_t *);
64 static void igb_setup_mac_rss_classify(igb_t *);
65 static void igb_setup_mac_classify(igb_t *);
66 static void igb_init_unicst(igb_t *);
67 static void igb_setup_multicst(igb_t *);
68 static void igb_get_phy_state(igb_t *);
69 static void igb_param_sync(igb_t *);
70 static void igb_get_conf(igb_t *);
71 static int igb_get_prop(igb_t *, char *, int, int, int);
72 static boolean_t igb_is_link_up(igb_t *);
73 static boolean_t igb_link_check(igb_t *);
74 static void igb_local_timer(void *);
75 static void igb_link_timer(void *);
76 static void igb_arm_watchdog_timer(igb_t *);
77 static void igb_start_watchdog_timer(igb_t *);
78 static void igb_restart_watchdog_timer(igb_t *);
79 static void igb_stop_watchdog_timer(igb_t *);
80 static void igb_start_link_timer(igb_t *);
81 static void igb_stop_link_timer(igb_t *);
82 static void igb_disable_adapter_interrupts(igb_t *);
83 static void igb_enable_adapter_interrupts_82575(igb_t *);
84 static void igb_enable_adapter_interrupts_82576(igb_t *);
85 static void igb_enable_adapter_interrupts_82580(igb_t *);
86 static boolean_t is_valid_mac_addr(uint8_t *);
87 static boolean_t igb_stall_check(igb_t *);
88 static boolean_t igb_set_loopback_mode(igb_t *, uint32_t);
89 static void igb_set_external_loopback(igb_t *);
90 static void igb_set_internal_phy_loopback(igb_t *);
91 static void igb_set_internal_serdes_loopback(igb_t *);
92 static boolean_t igb_find_mac_address(igb_t *);
93 static int igb_alloc_intrs(igb_t *);
94 static int igb_alloc_intr_handles(igb_t *, int);
95 static int igb_add_intr_handlers(igb_t *);
96 static void igb_rem_intr_handlers(igb_t *);
97 static void igb_rem_intrs(igb_t *);
98 static int igb_enable_intrs(igb_t *);
99 static int igb_disable_intrs(igb_t *);
100 static void igb_setup_msix_82575(igb_t *);
101 static void igb_setup_msix_82576(igb_t *);
102 static void igb_setup_msix_82580(igb_t *);
103 static uint_t igb_intr_legacy(void *, void *);
104 static uint_t igb_intr_msi(void *, void *);
105 static uint_t igb_intr_rx(void *, void *);
106 static uint_t igb_intr_tx(void *, void *);
107 static uint_t igb_intr_tx_other(void *, void *);
108 static void igb_intr_rx_work(igb_rx_ring_t *);
109 static void igb_intr_tx_work(igb_tx_ring_t *);
110 static void igb_intr_link_work(igb_t *);
111 static void igb_get_driver_control(struct e1000_hw *);
112 static void igb_release_driver_control(struct e1000_hw *);
113 
114 static int igb_attach(dev_info_t *, ddi_attach_cmd_t);
115 static int igb_detach(dev_info_t *, ddi_detach_cmd_t);
116 static int igb_resume(dev_info_t *);
117 static int igb_suspend(dev_info_t *);
118 static int igb_quiesce(dev_info_t *);
119 static void igb_unconfigure(dev_info_t *, igb_t *);
120 static int igb_fm_error_cb(dev_info_t *, ddi_fm_error_t *,
121     const void *);
122 static void igb_fm_init(igb_t *);
123 static void igb_fm_fini(igb_t *);
124 static void igb_release_multicast(igb_t *);
125 
126 char *igb_priv_props[] = {
127 	"_eee_support",
128 	"_tx_copy_thresh",
129 	"_tx_recycle_thresh",
130 	"_tx_overload_thresh",
131 	"_tx_resched_thresh",
132 	"_rx_copy_thresh",
133 	"_rx_limit_per_intr",
134 	"_intr_throttling",
135 	"_adv_pause_cap",
136 	"_adv_asym_pause_cap",
137 	NULL
138 };
139 
140 static struct cb_ops igb_cb_ops = {
141 	nulldev,		/* cb_open */
142 	nulldev,		/* cb_close */
143 	nodev,			/* cb_strategy */
144 	nodev,			/* cb_print */
145 	nodev,			/* cb_dump */
146 	nodev,			/* cb_read */
147 	nodev,			/* cb_write */
148 	nodev,			/* cb_ioctl */
149 	nodev,			/* cb_devmap */
150 	nodev,			/* cb_mmap */
151 	nodev,			/* cb_segmap */
152 	nochpoll,		/* cb_chpoll */
153 	ddi_prop_op,		/* cb_prop_op */
154 	NULL,			/* cb_stream */
155 	D_MP | D_HOTPLUG,	/* cb_flag */
156 	CB_REV,			/* cb_rev */
157 	nodev,			/* cb_aread */
158 	nodev			/* cb_awrite */
159 };
160 
161 static struct dev_ops igb_dev_ops = {
162 	DEVO_REV,		/* devo_rev */
163 	0,			/* devo_refcnt */
164 	NULL,			/* devo_getinfo */
165 	nulldev,		/* devo_identify */
166 	nulldev,		/* devo_probe */
167 	igb_attach,		/* devo_attach */
168 	igb_detach,		/* devo_detach */
169 	nodev,			/* devo_reset */
170 	&igb_cb_ops,		/* devo_cb_ops */
171 	NULL,			/* devo_bus_ops */
172 	ddi_power,		/* devo_power */
173 	igb_quiesce,	/* devo_quiesce */
174 };
175 
176 static struct modldrv igb_modldrv = {
177 	&mod_driverops,		/* Type of module.  This one is a driver */
178 	ident,			/* Discription string */
179 	&igb_dev_ops,		/* driver ops */
180 };
181 
182 static struct modlinkage igb_modlinkage = {
183 	MODREV_1, &igb_modldrv, NULL
184 };
185 
186 /* Access attributes for register mapping */
187 ddi_device_acc_attr_t igb_regs_acc_attr = {
188 	DDI_DEVICE_ATTR_V1,
189 	DDI_STRUCTURE_LE_ACC,
190 	DDI_STRICTORDER_ACC,
191 	DDI_FLAGERR_ACC
192 };
193 
194 #define	IGB_M_CALLBACK_FLAGS \
195 	(MC_IOCTL | MC_GETCAPAB | MC_SETPROP | MC_GETPROP | MC_PROPINFO)
196 
197 static mac_callbacks_t igb_m_callbacks = {
198 	IGB_M_CALLBACK_FLAGS,
199 	igb_m_stat,
200 	igb_m_start,
201 	igb_m_stop,
202 	igb_m_promisc,
203 	igb_m_multicst,
204 	NULL,
205 	NULL,
206 	NULL,
207 	igb_m_ioctl,
208 	igb_m_getcapab,
209 	NULL,
210 	NULL,
211 	igb_m_setprop,
212 	igb_m_getprop,
213 	igb_m_propinfo
214 };
215 
216 /*
217  * Initialize capabilities of each supported adapter type
218  */
219 static adapter_info_t igb_82575_cap = {
220 	/* limits */
221 	4,		/* maximum number of rx queues */
222 	1,		/* minimum number of rx queues */
223 	4,		/* default number of rx queues */
224 	4,		/* maximum number of tx queues */
225 	1,		/* minimum number of tx queues */
226 	4,		/* default number of tx queues */
227 	65535,		/* maximum interrupt throttle rate */
228 	0,		/* minimum interrupt throttle rate */
229 	200,		/* default interrupt throttle rate */
230 
231 	/* function pointers */
232 	igb_enable_adapter_interrupts_82575,
233 	igb_setup_msix_82575,
234 
235 	/* capabilities */
236 	(IGB_FLAG_HAS_DCA |	/* capability flags */
237 	IGB_FLAG_VMDQ_POOL),
238 
239 	0xffc00000		/* mask for RXDCTL register */
240 };
241 
242 static adapter_info_t igb_82576_cap = {
243 	/* limits */
244 	16,		/* maximum number of rx queues */
245 	1,		/* minimum number of rx queues */
246 	4,		/* default number of rx queues */
247 	16,		/* maximum number of tx queues */
248 	1,		/* minimum number of tx queues */
249 	4,		/* default number of tx queues */
250 	65535,		/* maximum interrupt throttle rate */
251 	0,		/* minimum interrupt throttle rate */
252 	200,		/* default interrupt throttle rate */
253 
254 	/* function pointers */
255 	igb_enable_adapter_interrupts_82576,
256 	igb_setup_msix_82576,
257 
258 	/* capabilities */
259 	(IGB_FLAG_HAS_DCA |	/* capability flags */
260 	IGB_FLAG_VMDQ_POOL |
261 	IGB_FLAG_NEED_CTX_IDX),
262 
263 	0xffe00000		/* mask for RXDCTL register */
264 };
265 
266 static adapter_info_t igb_82580_cap = {
267 	/* limits */
268 	8,		/* maximum number of rx queues */
269 	1,		/* minimum number of rx queues */
270 	4,		/* default number of rx queues */
271 	8,		/* maximum number of tx queues */
272 	1,		/* minimum number of tx queues */
273 	4,		/* default number of tx queues */
274 	65535,		/* maximum interrupt throttle rate */
275 	0,		/* minimum interrupt throttle rate */
276 	200,		/* default interrupt throttle rate */
277 
278 	/* function pointers */
279 	igb_enable_adapter_interrupts_82580,
280 	igb_setup_msix_82580,
281 
282 	/* capabilities */
283 	(IGB_FLAG_HAS_DCA |	/* capability flags */
284 	IGB_FLAG_VMDQ_POOL |
285 	IGB_FLAG_NEED_CTX_IDX),
286 
287 	0xffe00000		/* mask for RXDCTL register */
288 };
289 
290 static adapter_info_t igb_i350_cap = {
291 	/* limits */
292 	8,		/* maximum number of rx queues */
293 	1,		/* minimum number of rx queues */
294 	4,		/* default number of rx queues */
295 	8,		/* maximum number of tx queues */
296 	1,		/* minimum number of tx queues */
297 	4,		/* default number of tx queues */
298 	65535,		/* maximum interrupt throttle rate */
299 	0,		/* minimum interrupt throttle rate */
300 	200,		/* default interrupt throttle rate */
301 
302 	/* function pointers */
303 	igb_enable_adapter_interrupts_82580,
304 	igb_setup_msix_82580,
305 
306 	/* capabilities */
307 	(IGB_FLAG_HAS_DCA |	/* capability flags */
308 	IGB_FLAG_VMDQ_POOL |
309 	IGB_FLAG_NEED_CTX_IDX),
310 
311 	0xffe00000		/* mask for RXDCTL register */
312 };
313 
314 static adapter_info_t igb_i210_cap = {
315 	/* limits */
316 	4,		/* maximum number of rx queues */
317 	1,		/* minimum number of rx queues */
318 	4,		/* default number of rx queues */
319 	4,		/* maximum number of tx queues */
320 	1,		/* minimum number of tx queues */
321 	4,		/* default number of tx queues */
322 	65535,		/* maximum interrupt throttle rate */
323 	0,		/* minimum interrupt throttle rate */
324 	200,		/* default interrupt throttle rate */
325 
326 	/* function pointers */
327 	igb_enable_adapter_interrupts_82580,
328 	igb_setup_msix_82580,
329 
330 	/* capabilities */
331 	(IGB_FLAG_HAS_DCA |	/* capability flags */
332 	IGB_FLAG_VMDQ_POOL |
333 	IGB_FLAG_NEED_CTX_IDX),
334 
335 	0xfff00000		/* mask for RXDCTL register */
336 };
337 
338 static adapter_info_t igb_i354_cap = {
339 	/* limits */
340 	8,		/* maximum number of rx queues */
341 	1,		/* minimum number of rx queues */
342 	4,		/* default number of rx queues */
343 	8,		/* maximum number of tx queues */
344 	1,		/* minimum number of tx queues */
345 	4,		/* default number of tx queues */
346 	65535,		/* maximum interrupt throttle rate */
347 	0,		/* minimum interrupt throttle rate */
348 	200,		/* default interrupt throttle rate */
349 
350 	/* function pointers */
351 	igb_enable_adapter_interrupts_82580,
352 	igb_setup_msix_82580,
353 
354 	/* capabilities */
355 	(IGB_FLAG_HAS_DCA |	/* capability flags */
356 	IGB_FLAG_VMDQ_POOL |
357 	IGB_FLAG_NEED_CTX_IDX),
358 
359 	0xfff00000		/* mask for RXDCTL register */
360 };
361 
362 /*
363  * Module Initialization Functions
364  */
365 
366 int
_init(void)367 _init(void)
368 {
369 	int status;
370 
371 	mac_init_ops(&igb_dev_ops, MODULE_NAME);
372 
373 	status = mod_install(&igb_modlinkage);
374 
375 	if (status != DDI_SUCCESS) {
376 		mac_fini_ops(&igb_dev_ops);
377 	}
378 
379 	return (status);
380 }
381 
382 int
_fini(void)383 _fini(void)
384 {
385 	int status;
386 
387 	status = mod_remove(&igb_modlinkage);
388 
389 	if (status == DDI_SUCCESS) {
390 		mac_fini_ops(&igb_dev_ops);
391 	}
392 
393 	return (status);
394 
395 }
396 
397 int
_info(struct modinfo * modinfop)398 _info(struct modinfo *modinfop)
399 {
400 	int status;
401 
402 	status = mod_info(&igb_modlinkage, modinfop);
403 
404 	return (status);
405 }
406 
407 /*
408  * igb_attach - driver attach
409  *
410  * This function is the device specific initialization entry
411  * point. This entry point is required and must be written.
412  * The DDI_ATTACH command must be provided in the attach entry
413  * point. When attach() is called with cmd set to DDI_ATTACH,
414  * all normal kernel services (such as kmem_alloc(9F)) are
415  * available for use by the driver.
416  *
417  * The attach() function will be called once for each instance
418  * of  the  device  on  the  system with cmd set to DDI_ATTACH.
419  * Until attach() succeeds, the only driver entry points which
420  * may be called are open(9E) and getinfo(9E).
421  */
422 static int
igb_attach(dev_info_t * devinfo,ddi_attach_cmd_t cmd)423 igb_attach(dev_info_t *devinfo, ddi_attach_cmd_t cmd)
424 {
425 	igb_t *igb;
426 	struct igb_osdep *osdep;
427 	struct e1000_hw *hw;
428 	int instance;
429 
430 	/*
431 	 * Check the command and perform corresponding operations
432 	 */
433 	switch (cmd) {
434 	default:
435 		return (DDI_FAILURE);
436 
437 	case DDI_RESUME:
438 		return (igb_resume(devinfo));
439 
440 	case DDI_ATTACH:
441 		break;
442 	}
443 
444 	/* Get the device instance */
445 	instance = ddi_get_instance(devinfo);
446 
447 	/* Allocate memory for the instance data structure */
448 	igb = kmem_zalloc(sizeof (igb_t), KM_SLEEP);
449 
450 	igb->dip = devinfo;
451 	igb->instance = instance;
452 
453 	hw = &igb->hw;
454 	osdep = &igb->osdep;
455 	hw->back = osdep;
456 	osdep->igb = igb;
457 
458 	/* Attach the instance pointer to the dev_info data structure */
459 	ddi_set_driver_private(devinfo, igb);
460 
461 
462 	/* Initialize for fma support */
463 	igb->fm_capabilities = igb_get_prop(igb, "fm-capable",
464 	    0, 0x0f,
465 	    DDI_FM_EREPORT_CAPABLE | DDI_FM_ACCCHK_CAPABLE |
466 	    DDI_FM_DMACHK_CAPABLE | DDI_FM_ERRCB_CAPABLE);
467 	igb_fm_init(igb);
468 	igb->attach_progress |= ATTACH_PROGRESS_FMINIT;
469 
470 	/*
471 	 * Map PCI config space registers
472 	 */
473 	if (pci_config_setup(devinfo, &osdep->cfg_handle) != DDI_SUCCESS) {
474 		igb_error(igb, "Failed to map PCI configurations");
475 		goto attach_fail;
476 	}
477 	igb->attach_progress |= ATTACH_PROGRESS_PCI_CONFIG;
478 
479 	/*
480 	 * Identify the chipset family
481 	 */
482 	if (igb_identify_hardware(igb) != IGB_SUCCESS) {
483 		igb_error(igb, "Failed to identify hardware");
484 		goto attach_fail;
485 	}
486 
487 	/*
488 	 * Map device registers
489 	 */
490 	if (igb_regs_map(igb) != IGB_SUCCESS) {
491 		igb_error(igb, "Failed to map device registers");
492 		goto attach_fail;
493 	}
494 	igb->attach_progress |= ATTACH_PROGRESS_REGS_MAP;
495 
496 	/*
497 	 * Initialize driver parameters
498 	 */
499 	igb_init_properties(igb);
500 	igb->attach_progress |= ATTACH_PROGRESS_PROPS;
501 
502 	/*
503 	 * Allocate interrupts
504 	 */
505 	if (igb_alloc_intrs(igb) != IGB_SUCCESS) {
506 		igb_error(igb, "Failed to allocate interrupts");
507 		goto attach_fail;
508 	}
509 	igb->attach_progress |= ATTACH_PROGRESS_ALLOC_INTR;
510 
511 	/*
512 	 * Allocate rx/tx rings based on the ring numbers.
513 	 * The actual numbers of rx/tx rings are decided by the number of
514 	 * allocated interrupt vectors, so we should allocate the rings after
515 	 * interrupts are allocated.
516 	 */
517 	if (igb_alloc_rings(igb) != IGB_SUCCESS) {
518 		igb_error(igb, "Failed to allocate rx/tx rings or groups");
519 		goto attach_fail;
520 	}
521 	igb->attach_progress |= ATTACH_PROGRESS_ALLOC_RINGS;
522 
523 	/*
524 	 * Add interrupt handlers
525 	 */
526 	if (igb_add_intr_handlers(igb) != IGB_SUCCESS) {
527 		igb_error(igb, "Failed to add interrupt handlers");
528 		goto attach_fail;
529 	}
530 	igb->attach_progress |= ATTACH_PROGRESS_ADD_INTR;
531 
532 	/*
533 	 * Initialize driver parameters
534 	 */
535 	if (igb_init_driver_settings(igb) != IGB_SUCCESS) {
536 		igb_error(igb, "Failed to initialize driver settings");
537 		goto attach_fail;
538 	}
539 
540 	if (igb_check_acc_handle(igb->osdep.cfg_handle) != DDI_FM_OK) {
541 		ddi_fm_service_impact(igb->dip, DDI_SERVICE_LOST);
542 		goto attach_fail;
543 	}
544 
545 	/*
546 	 * Initialize mutexes for this device.
547 	 * Do this before enabling the interrupt handler and
548 	 * register the softint to avoid the condition where
549 	 * interrupt handler can try using uninitialized mutex
550 	 */
551 	igb_init_locks(igb);
552 	igb->attach_progress |= ATTACH_PROGRESS_LOCKS;
553 
554 	/*
555 	 * Initialize the adapter
556 	 */
557 	if (igb_init(igb) != IGB_SUCCESS) {
558 		igb_error(igb, "Failed to initialize adapter");
559 		goto attach_fail;
560 	}
561 	igb->attach_progress |= ATTACH_PROGRESS_INIT_ADAPTER;
562 
563 	/*
564 	 * Initialize statistics
565 	 */
566 	if (igb_init_stats(igb) != IGB_SUCCESS) {
567 		igb_error(igb, "Failed to initialize statistics");
568 		goto attach_fail;
569 	}
570 	igb->attach_progress |= ATTACH_PROGRESS_STATS;
571 
572 	/*
573 	 * Register the driver to the MAC
574 	 */
575 	if (igb_register_mac(igb) != IGB_SUCCESS) {
576 		igb_error(igb, "Failed to register MAC");
577 		goto attach_fail;
578 	}
579 	igb->attach_progress |= ATTACH_PROGRESS_MAC;
580 
581 	/*
582 	 * Now that mutex locks are initialized, and the chip is also
583 	 * initialized, enable interrupts.
584 	 */
585 	if (igb_enable_intrs(igb) != IGB_SUCCESS) {
586 		igb_error(igb, "Failed to enable DDI interrupts");
587 		goto attach_fail;
588 	}
589 	igb->attach_progress |= ATTACH_PROGRESS_ENABLE_INTR;
590 
591 	igb_log(igb, "%s", igb_version);
592 	atomic_or_32(&igb->igb_state, IGB_INITIALIZED);
593 
594 	/*
595 	 * Newer models have Energy Efficient Ethernet, let's disable this by
596 	 * default.
597 	 */
598 	if (igb->hw.mac.type == e1000_i350)
599 		(void) e1000_set_eee_i350(&igb->hw);
600 	else if (igb->hw.mac.type == e1000_i354)
601 		(void) e1000_set_eee_i354(&igb->hw);
602 
603 	return (DDI_SUCCESS);
604 
605 attach_fail:
606 	igb_unconfigure(devinfo, igb);
607 	return (DDI_FAILURE);
608 }
609 
610 /*
611  * igb_detach - driver detach
612  *
613  * The detach() function is the complement of the attach routine.
614  * If cmd is set to DDI_DETACH, detach() is used to remove  the
615  * state  associated  with  a  given  instance of a device node
616  * prior to the removal of that instance from the system.
617  *
618  * The detach() function will be called once for each  instance
619  * of the device for which there has been a successful attach()
620  * once there are no longer  any  opens  on  the  device.
621  *
622  * Interrupts routine are disabled, All memory allocated by this
623  * driver are freed.
624  */
625 static int
igb_detach(dev_info_t * devinfo,ddi_detach_cmd_t cmd)626 igb_detach(dev_info_t *devinfo, ddi_detach_cmd_t cmd)
627 {
628 	igb_t *igb;
629 
630 	/*
631 	 * Check detach command
632 	 */
633 	switch (cmd) {
634 	default:
635 		return (DDI_FAILURE);
636 
637 	case DDI_SUSPEND:
638 		return (igb_suspend(devinfo));
639 
640 	case DDI_DETACH:
641 		break;
642 	}
643 
644 
645 	/*
646 	 * Get the pointer to the driver private data structure
647 	 */
648 	igb = (igb_t *)ddi_get_driver_private(devinfo);
649 	if (igb == NULL)
650 		return (DDI_FAILURE);
651 
652 	/*
653 	 * Unregister MAC. If failed, we have to fail the detach
654 	 */
655 	if (mac_unregister(igb->mac_hdl) != 0) {
656 		igb_error(igb, "Failed to unregister MAC");
657 		return (DDI_FAILURE);
658 	}
659 	igb->attach_progress &= ~ATTACH_PROGRESS_MAC;
660 
661 	/*
662 	 * If the device is still running, it needs to be stopped first.
663 	 * This check is necessary because under some specific circumstances,
664 	 * the detach routine can be called without stopping the interface
665 	 * first.
666 	 */
667 	mutex_enter(&igb->gen_lock);
668 	if (igb->igb_state & IGB_STARTED) {
669 		atomic_and_32(&igb->igb_state, ~IGB_STARTED);
670 		igb_stop(igb, B_TRUE);
671 		mutex_exit(&igb->gen_lock);
672 		/* Disable and stop the watchdog timer */
673 		igb_disable_watchdog_timer(igb);
674 	} else
675 		mutex_exit(&igb->gen_lock);
676 
677 	/*
678 	 * Check if there are still rx buffers held by the upper layer.
679 	 * If so, fail the detach.
680 	 */
681 	if (!igb_rx_drain(igb))
682 		return (DDI_FAILURE);
683 
684 	/*
685 	 * Do the remaining unconfigure routines
686 	 */
687 	igb_unconfigure(devinfo, igb);
688 
689 	return (DDI_SUCCESS);
690 }
691 
692 /*
693  * quiesce(9E) entry point.
694  *
695  * This function is called when the system is single-threaded at high
696  * PIL with preemption disabled. Therefore, this function must not be
697  * blocked.
698  *
699  * This function returns DDI_SUCCESS on success, or DDI_FAILURE on failure.
700  * DDI_FAILURE indicates an error condition and should almost never happen.
701  */
702 static int
igb_quiesce(dev_info_t * devinfo)703 igb_quiesce(dev_info_t *devinfo)
704 {
705 	igb_t *igb;
706 	struct e1000_hw *hw;
707 
708 	igb = (igb_t *)ddi_get_driver_private(devinfo);
709 
710 	if (igb == NULL)
711 		return (DDI_FAILURE);
712 
713 	hw = &igb->hw;
714 
715 	/*
716 	 * Disable the adapter interrupts
717 	 */
718 	igb_disable_adapter_interrupts(igb);
719 
720 	/* Tell firmware driver is no longer in control */
721 	igb_release_driver_control(hw);
722 
723 	/*
724 	 * Reset the chipset
725 	 */
726 	(void) e1000_reset_hw(hw);
727 
728 	/*
729 	 * Reset PHY if possible
730 	 */
731 	if (e1000_check_reset_block(hw) == E1000_SUCCESS)
732 		(void) e1000_phy_hw_reset(hw);
733 
734 	return (DDI_SUCCESS);
735 }
736 
737 /*
738  * igb_unconfigure - release all resources held by this instance
739  */
740 static void
igb_unconfigure(dev_info_t * devinfo,igb_t * igb)741 igb_unconfigure(dev_info_t *devinfo, igb_t *igb)
742 {
743 	/*
744 	 * Disable interrupt
745 	 */
746 	if (igb->attach_progress & ATTACH_PROGRESS_ENABLE_INTR) {
747 		(void) igb_disable_intrs(igb);
748 	}
749 
750 	/*
751 	 * Unregister MAC
752 	 */
753 	if (igb->attach_progress & ATTACH_PROGRESS_MAC) {
754 		(void) mac_unregister(igb->mac_hdl);
755 	}
756 
757 	/*
758 	 * Free statistics
759 	 */
760 	if (igb->attach_progress & ATTACH_PROGRESS_STATS) {
761 		kstat_delete((kstat_t *)igb->igb_ks);
762 	}
763 
764 	/*
765 	 * Remove interrupt handlers
766 	 */
767 	if (igb->attach_progress & ATTACH_PROGRESS_ADD_INTR) {
768 		igb_rem_intr_handlers(igb);
769 	}
770 
771 	/*
772 	 * Remove interrupts
773 	 */
774 	if (igb->attach_progress & ATTACH_PROGRESS_ALLOC_INTR) {
775 		igb_rem_intrs(igb);
776 	}
777 
778 	/*
779 	 * Remove driver properties
780 	 */
781 	if (igb->attach_progress & ATTACH_PROGRESS_PROPS) {
782 		(void) ddi_prop_remove_all(devinfo);
783 	}
784 
785 	/*
786 	 * Stop the adapter
787 	 */
788 	if (igb->attach_progress & ATTACH_PROGRESS_INIT_ADAPTER) {
789 		mutex_enter(&igb->gen_lock);
790 		igb_stop_adapter(igb);
791 		mutex_exit(&igb->gen_lock);
792 		if (igb_check_acc_handle(igb->osdep.reg_handle) != DDI_FM_OK)
793 			ddi_fm_service_impact(igb->dip, DDI_SERVICE_UNAFFECTED);
794 	}
795 
796 	/*
797 	 * Free multicast table
798 	 */
799 	igb_release_multicast(igb);
800 
801 	/*
802 	 * Free register handle
803 	 */
804 	if (igb->attach_progress & ATTACH_PROGRESS_REGS_MAP) {
805 		if (igb->osdep.reg_handle != NULL)
806 			ddi_regs_map_free(&igb->osdep.reg_handle);
807 	}
808 
809 	/*
810 	 * Free PCI config handle
811 	 */
812 	if (igb->attach_progress & ATTACH_PROGRESS_PCI_CONFIG) {
813 		if (igb->osdep.cfg_handle != NULL)
814 			pci_config_teardown(&igb->osdep.cfg_handle);
815 	}
816 
817 	/*
818 	 * Free locks
819 	 */
820 	if (igb->attach_progress & ATTACH_PROGRESS_LOCKS) {
821 		igb_destroy_locks(igb);
822 	}
823 
824 	/*
825 	 * Free the rx/tx rings
826 	 */
827 	if (igb->attach_progress & ATTACH_PROGRESS_ALLOC_RINGS) {
828 		igb_free_rings(igb);
829 	}
830 
831 	/*
832 	 * Remove FMA
833 	 */
834 	if (igb->attach_progress & ATTACH_PROGRESS_FMINIT) {
835 		igb_fm_fini(igb);
836 	}
837 
838 	/*
839 	 * Free the driver data structure
840 	 */
841 	kmem_free(igb, sizeof (igb_t));
842 
843 	ddi_set_driver_private(devinfo, NULL);
844 }
845 
846 /*
847  * igb_register_mac - Register the driver and its function pointers with
848  * the GLD interface
849  */
850 static int
igb_register_mac(igb_t * igb)851 igb_register_mac(igb_t *igb)
852 {
853 	struct e1000_hw *hw = &igb->hw;
854 	mac_register_t *mac;
855 	int status;
856 
857 	if ((mac = mac_alloc(MAC_VERSION)) == NULL)
858 		return (IGB_FAILURE);
859 
860 	mac->m_type_ident = MAC_PLUGIN_IDENT_ETHER;
861 	mac->m_driver = igb;
862 	mac->m_dip = igb->dip;
863 	mac->m_src_addr = hw->mac.addr;
864 	mac->m_callbacks = &igb_m_callbacks;
865 	mac->m_min_sdu = 0;
866 	mac->m_max_sdu = igb->max_frame_size -
867 	    sizeof (struct ether_vlan_header) - ETHERFCSL;
868 	mac->m_margin = VLAN_TAGSZ;
869 	mac->m_priv_props = igb_priv_props;
870 	mac->m_v12n = MAC_VIRT_LEVEL1;
871 
872 	status = mac_register(mac, &igb->mac_hdl);
873 
874 	mac_free(mac);
875 
876 	return ((status == 0) ? IGB_SUCCESS : IGB_FAILURE);
877 }
878 
879 /*
880  * igb_identify_hardware - Identify the type of the chipset
881  */
882 static int
igb_identify_hardware(igb_t * igb)883 igb_identify_hardware(igb_t *igb)
884 {
885 	struct e1000_hw *hw = &igb->hw;
886 	struct igb_osdep *osdep = &igb->osdep;
887 
888 	/*
889 	 * Get the device id
890 	 */
891 	hw->vendor_id =
892 	    pci_config_get16(osdep->cfg_handle, PCI_CONF_VENID);
893 	hw->device_id =
894 	    pci_config_get16(osdep->cfg_handle, PCI_CONF_DEVID);
895 	hw->revision_id =
896 	    pci_config_get8(osdep->cfg_handle, PCI_CONF_REVID);
897 	hw->subsystem_device_id =
898 	    pci_config_get16(osdep->cfg_handle, PCI_CONF_SUBSYSID);
899 	hw->subsystem_vendor_id =
900 	    pci_config_get16(osdep->cfg_handle, PCI_CONF_SUBVENID);
901 
902 	/*
903 	 * Set the mac type of the adapter based on the device id
904 	 */
905 	if (e1000_set_mac_type(hw) != E1000_SUCCESS) {
906 		return (IGB_FAILURE);
907 	}
908 
909 	/*
910 	 * Install adapter capabilities based on mac type
911 	 */
912 	switch (hw->mac.type) {
913 	case e1000_82575:
914 		igb->capab = &igb_82575_cap;
915 		break;
916 	case e1000_82576:
917 		igb->capab = &igb_82576_cap;
918 		break;
919 	case e1000_82580:
920 		igb->capab = &igb_82580_cap;
921 		break;
922 	case e1000_i350:
923 		igb->capab = &igb_i350_cap;
924 		break;
925 	case e1000_i210:
926 	case e1000_i211:
927 		igb->capab = &igb_i210_cap;
928 		break;
929 	case e1000_i354:
930 		igb->capab = &igb_i354_cap;
931 		break;
932 	default:
933 		return (IGB_FAILURE);
934 	}
935 
936 	return (IGB_SUCCESS);
937 }
938 
939 /*
940  * igb_regs_map - Map the device registers
941  */
942 static int
igb_regs_map(igb_t * igb)943 igb_regs_map(igb_t *igb)
944 {
945 	dev_info_t *devinfo = igb->dip;
946 	struct e1000_hw *hw = &igb->hw;
947 	struct igb_osdep *osdep = &igb->osdep;
948 	off_t mem_size;
949 
950 	/*
951 	 * First get the size of device registers to be mapped.
952 	 */
953 	if (ddi_dev_regsize(devinfo, IGB_ADAPTER_REGSET, &mem_size) !=
954 	    DDI_SUCCESS) {
955 		return (IGB_FAILURE);
956 	}
957 
958 	/*
959 	 * Call ddi_regs_map_setup() to map registers
960 	 */
961 	if ((ddi_regs_map_setup(devinfo, IGB_ADAPTER_REGSET,
962 	    (caddr_t *)&hw->hw_addr, 0,
963 	    mem_size, &igb_regs_acc_attr,
964 	    &osdep->reg_handle)) != DDI_SUCCESS) {
965 		return (IGB_FAILURE);
966 	}
967 
968 	return (IGB_SUCCESS);
969 }
970 
971 /*
972  * igb_init_properties - Initialize driver properties
973  */
974 static void
igb_init_properties(igb_t * igb)975 igb_init_properties(igb_t *igb)
976 {
977 	/*
978 	 * Get conf file properties, including link settings
979 	 * jumbo frames, ring number, descriptor number, etc.
980 	 */
981 	igb_get_conf(igb);
982 }
983 
984 /*
985  * igb_init_driver_settings - Initialize driver settings
986  *
987  * The settings include hardware function pointers, bus information,
988  * rx/tx rings settings, link state, and any other parameters that
989  * need to be setup during driver initialization.
990  */
991 static int
igb_init_driver_settings(igb_t * igb)992 igb_init_driver_settings(igb_t *igb)
993 {
994 	struct e1000_hw *hw = &igb->hw;
995 	igb_rx_ring_t *rx_ring;
996 	igb_tx_ring_t *tx_ring;
997 	uint32_t rx_size;
998 	uint32_t tx_size;
999 	int i;
1000 
1001 	/*
1002 	 * Initialize chipset specific hardware function pointers
1003 	 */
1004 	if (e1000_setup_init_funcs(hw, B_TRUE) != E1000_SUCCESS) {
1005 		return (IGB_FAILURE);
1006 	}
1007 
1008 	/*
1009 	 * Get bus information
1010 	 */
1011 	if (e1000_get_bus_info(hw) != E1000_SUCCESS) {
1012 		return (IGB_FAILURE);
1013 	}
1014 
1015 	/*
1016 	 * Get the system page size
1017 	 */
1018 	igb->page_size = ddi_ptob(igb->dip, (ulong_t)1);
1019 
1020 	/*
1021 	 * Set rx buffer size
1022 	 * The IP header alignment room is counted in the calculation.
1023 	 * The rx buffer size is in unit of 1K that is required by the
1024 	 * chipset hardware.
1025 	 */
1026 	rx_size = igb->max_frame_size + IPHDR_ALIGN_ROOM;
1027 	igb->rx_buf_size = ((rx_size >> 10) +
1028 	    ((rx_size & (((uint32_t)1 << 10) - 1)) > 0 ? 1 : 0)) << 10;
1029 
1030 	/*
1031 	 * Set tx buffer size
1032 	 */
1033 	tx_size = igb->max_frame_size;
1034 	igb->tx_buf_size = ((tx_size >> 10) +
1035 	    ((tx_size & (((uint32_t)1 << 10) - 1)) > 0 ? 1 : 0)) << 10;
1036 
1037 	/*
1038 	 * Initialize rx/tx rings parameters
1039 	 */
1040 	for (i = 0; i < igb->num_rx_rings; i++) {
1041 		rx_ring = &igb->rx_rings[i];
1042 		rx_ring->index = i;
1043 		rx_ring->igb = igb;
1044 	}
1045 
1046 	for (i = 0; i < igb->num_tx_rings; i++) {
1047 		tx_ring = &igb->tx_rings[i];
1048 		tx_ring->index = i;
1049 		tx_ring->igb = igb;
1050 		if (igb->tx_head_wb_enable)
1051 			tx_ring->tx_recycle = igb_tx_recycle_head_wb;
1052 		else
1053 			tx_ring->tx_recycle = igb_tx_recycle_legacy;
1054 
1055 		tx_ring->ring_size = igb->tx_ring_size;
1056 		tx_ring->free_list_size = igb->tx_ring_size +
1057 		    (igb->tx_ring_size >> 1);
1058 	}
1059 
1060 	/*
1061 	 * Initialize values of interrupt throttling rates
1062 	 */
1063 	for (i = 1; i < MAX_NUM_EITR; i++)
1064 		igb->intr_throttling[i] = igb->intr_throttling[0];
1065 
1066 	/*
1067 	 * The initial link state should be "unknown"
1068 	 */
1069 	igb->link_state = LINK_STATE_UNKNOWN;
1070 
1071 	return (IGB_SUCCESS);
1072 }
1073 
1074 /*
1075  * igb_init_locks - Initialize locks
1076  */
1077 static void
igb_init_locks(igb_t * igb)1078 igb_init_locks(igb_t *igb)
1079 {
1080 	igb_rx_ring_t *rx_ring;
1081 	igb_tx_ring_t *tx_ring;
1082 	int i;
1083 
1084 	for (i = 0; i < igb->num_rx_rings; i++) {
1085 		rx_ring = &igb->rx_rings[i];
1086 		mutex_init(&rx_ring->rx_lock, NULL,
1087 		    MUTEX_DRIVER, DDI_INTR_PRI(igb->intr_pri));
1088 	}
1089 
1090 	for (i = 0; i < igb->num_tx_rings; i++) {
1091 		tx_ring = &igb->tx_rings[i];
1092 		mutex_init(&tx_ring->tx_lock, NULL,
1093 		    MUTEX_DRIVER, DDI_INTR_PRI(igb->intr_pri));
1094 		mutex_init(&tx_ring->recycle_lock, NULL,
1095 		    MUTEX_DRIVER, DDI_INTR_PRI(igb->intr_pri));
1096 		mutex_init(&tx_ring->tcb_head_lock, NULL,
1097 		    MUTEX_DRIVER, DDI_INTR_PRI(igb->intr_pri));
1098 		mutex_init(&tx_ring->tcb_tail_lock, NULL,
1099 		    MUTEX_DRIVER, DDI_INTR_PRI(igb->intr_pri));
1100 	}
1101 
1102 	mutex_init(&igb->gen_lock, NULL,
1103 	    MUTEX_DRIVER, DDI_INTR_PRI(igb->intr_pri));
1104 
1105 	mutex_init(&igb->watchdog_lock, NULL,
1106 	    MUTEX_DRIVER, DDI_INTR_PRI(igb->intr_pri));
1107 
1108 	mutex_init(&igb->link_lock, NULL,
1109 	    MUTEX_DRIVER, DDI_INTR_PRI(igb->intr_pri));
1110 }
1111 
1112 /*
1113  * igb_destroy_locks - Destroy locks
1114  */
1115 static void
igb_destroy_locks(igb_t * igb)1116 igb_destroy_locks(igb_t *igb)
1117 {
1118 	igb_rx_ring_t *rx_ring;
1119 	igb_tx_ring_t *tx_ring;
1120 	int i;
1121 
1122 	for (i = 0; i < igb->num_rx_rings; i++) {
1123 		rx_ring = &igb->rx_rings[i];
1124 		mutex_destroy(&rx_ring->rx_lock);
1125 	}
1126 
1127 	for (i = 0; i < igb->num_tx_rings; i++) {
1128 		tx_ring = &igb->tx_rings[i];
1129 		mutex_destroy(&tx_ring->tx_lock);
1130 		mutex_destroy(&tx_ring->recycle_lock);
1131 		mutex_destroy(&tx_ring->tcb_head_lock);
1132 		mutex_destroy(&tx_ring->tcb_tail_lock);
1133 	}
1134 
1135 	mutex_destroy(&igb->gen_lock);
1136 	mutex_destroy(&igb->watchdog_lock);
1137 	mutex_destroy(&igb->link_lock);
1138 }
1139 
1140 static int
igb_resume(dev_info_t * devinfo)1141 igb_resume(dev_info_t *devinfo)
1142 {
1143 	igb_t *igb;
1144 
1145 	igb = (igb_t *)ddi_get_driver_private(devinfo);
1146 	if (igb == NULL)
1147 		return (DDI_FAILURE);
1148 
1149 	mutex_enter(&igb->gen_lock);
1150 
1151 	/*
1152 	 * Enable interrupts
1153 	 */
1154 	if (igb->attach_progress & ATTACH_PROGRESS_ENABLE_INTR) {
1155 		if (igb_enable_intrs(igb) != IGB_SUCCESS) {
1156 			igb_error(igb, "Failed to enable DDI interrupts");
1157 			mutex_exit(&igb->gen_lock);
1158 			return (DDI_FAILURE);
1159 		}
1160 	}
1161 
1162 	if (igb->igb_state & IGB_STARTED) {
1163 		if (igb_start(igb, B_FALSE) != IGB_SUCCESS) {
1164 			mutex_exit(&igb->gen_lock);
1165 			return (DDI_FAILURE);
1166 		}
1167 
1168 		/*
1169 		 * Enable and start the watchdog timer
1170 		 */
1171 		igb_enable_watchdog_timer(igb);
1172 	}
1173 
1174 	atomic_and_32(&igb->igb_state, ~IGB_SUSPENDED);
1175 
1176 	mutex_exit(&igb->gen_lock);
1177 
1178 	return (DDI_SUCCESS);
1179 }
1180 
1181 static int
igb_suspend(dev_info_t * devinfo)1182 igb_suspend(dev_info_t *devinfo)
1183 {
1184 	igb_t *igb;
1185 
1186 	igb = (igb_t *)ddi_get_driver_private(devinfo);
1187 	if (igb == NULL)
1188 		return (DDI_FAILURE);
1189 
1190 	mutex_enter(&igb->gen_lock);
1191 
1192 	atomic_or_32(&igb->igb_state, IGB_SUSPENDED);
1193 
1194 	/*
1195 	 * Disable interrupts
1196 	 */
1197 	if (igb->attach_progress & ATTACH_PROGRESS_ENABLE_INTR) {
1198 		(void) igb_disable_intrs(igb);
1199 	}
1200 
1201 	if (!(igb->igb_state & IGB_STARTED)) {
1202 		mutex_exit(&igb->gen_lock);
1203 		return (DDI_SUCCESS);
1204 	}
1205 
1206 	igb_stop(igb, B_FALSE);
1207 
1208 	mutex_exit(&igb->gen_lock);
1209 
1210 	/*
1211 	 * Disable and stop the watchdog timer
1212 	 */
1213 	igb_disable_watchdog_timer(igb);
1214 
1215 	return (DDI_SUCCESS);
1216 }
1217 
1218 static int
igb_init(igb_t * igb)1219 igb_init(igb_t *igb)
1220 {
1221 	mutex_enter(&igb->gen_lock);
1222 
1223 	/*
1224 	 * Initilize the adapter
1225 	 */
1226 	if (igb_init_adapter(igb) != IGB_SUCCESS) {
1227 		mutex_exit(&igb->gen_lock);
1228 		igb_fm_ereport(igb, DDI_FM_DEVICE_INVAL_STATE);
1229 		ddi_fm_service_impact(igb->dip, DDI_SERVICE_LOST);
1230 		return (IGB_FAILURE);
1231 	}
1232 
1233 	mutex_exit(&igb->gen_lock);
1234 
1235 	return (IGB_SUCCESS);
1236 }
1237 
1238 /*
1239  * igb_init_mac_address - Initialize the default MAC address
1240  *
1241  * On success, the MAC address is entered in the igb->hw.mac.addr
1242  * and hw->mac.perm_addr fields and the adapter's RAR(0) receive
1243  * address register.
1244  *
1245  * Important side effects:
1246  * 1. adapter is reset - this is required to put it in a known state.
1247  * 2. all of non-volatile memory (NVM) is read & checksummed - NVM is where
1248  * MAC address and all default settings are stored, so a valid checksum
1249  * is required.
1250  */
1251 static int
igb_init_mac_address(igb_t * igb)1252 igb_init_mac_address(igb_t *igb)
1253 {
1254 	struct e1000_hw *hw = &igb->hw;
1255 
1256 	ASSERT(mutex_owned(&igb->gen_lock));
1257 
1258 	/*
1259 	 * Reset chipset to put the hardware in a known state
1260 	 * before we try to get MAC address from NVM.
1261 	 */
1262 	if (e1000_reset_hw(hw) != E1000_SUCCESS) {
1263 		igb_error(igb, "Adapter reset failed.");
1264 		goto init_mac_fail;
1265 	}
1266 
1267 	/*
1268 	 * NVM validation
1269 	 */
1270 	if (((igb->hw.mac.type != e1000_i210) &&
1271 	    (igb->hw.mac.type != e1000_i211)) &&
1272 	    (e1000_validate_nvm_checksum(hw) < 0)) {
1273 		/*
1274 		 * Some PCI-E parts fail the first check due to
1275 		 * the link being in sleep state.  Call it again,
1276 		 * if it fails a second time its a real issue.
1277 		 */
1278 		if (e1000_validate_nvm_checksum(hw) < 0) {
1279 			igb_error(igb,
1280 			    "Invalid NVM checksum. Please contact "
1281 			    "the vendor to update the NVM.");
1282 			goto init_mac_fail;
1283 		}
1284 	}
1285 
1286 	/*
1287 	 * Get the mac address
1288 	 * This function should handle SPARC case correctly.
1289 	 */
1290 	if (!igb_find_mac_address(igb)) {
1291 		igb_error(igb, "Failed to get the mac address");
1292 		goto init_mac_fail;
1293 	}
1294 
1295 	/* Validate mac address */
1296 	if (!is_valid_mac_addr(hw->mac.addr)) {
1297 		igb_error(igb, "Invalid mac address");
1298 		goto init_mac_fail;
1299 	}
1300 
1301 	return (IGB_SUCCESS);
1302 
1303 init_mac_fail:
1304 	return (IGB_FAILURE);
1305 }
1306 
1307 /*
1308  * igb_init_adapter - Initialize the adapter
1309  */
1310 static int
igb_init_adapter(igb_t * igb)1311 igb_init_adapter(igb_t *igb)
1312 {
1313 	struct e1000_hw *hw = &igb->hw;
1314 	uint32_t pba;
1315 	int oemid[2];
1316 	uint16_t nvmword;
1317 	uint32_t hwm;
1318 	uint32_t default_mtu;
1319 	u8 pbanum[E1000_PBANUM_LENGTH];
1320 	char eepromver[5];	/* f.ff */
1321 	int i;
1322 
1323 	ASSERT(mutex_owned(&igb->gen_lock));
1324 
1325 	/*
1326 	 * In order to obtain the default MAC address, this will reset the
1327 	 * adapter and validate the NVM that the address and many other
1328 	 * default settings come from.
1329 	 */
1330 	if (igb_init_mac_address(igb) != IGB_SUCCESS) {
1331 		igb_error(igb, "Failed to initialize MAC address");
1332 		goto init_adapter_fail;
1333 	}
1334 
1335 	/*
1336 	 * Packet Buffer Allocation (PBA)
1337 	 * Writing PBA sets the receive portion of the buffer
1338 	 * the remainder is used for the transmit buffer.
1339 	 */
1340 	switch (hw->mac.type) {
1341 	case e1000_82575:
1342 		pba = E1000_PBA_32K;
1343 		break;
1344 	case e1000_82576:
1345 		pba = E1000_READ_REG(hw, E1000_RXPBS);
1346 		pba &= E1000_RXPBS_SIZE_MASK_82576;
1347 		break;
1348 	case e1000_82580:
1349 	case e1000_i350:
1350 	case e1000_i354:
1351 		pba = E1000_READ_REG(hw, E1000_RXPBS);
1352 		pba = e1000_rxpbs_adjust_82580(pba);
1353 		break;
1354 	case e1000_i210:
1355 	case e1000_i211:
1356 		pba = E1000_PBA_34K;
1357 	default:
1358 		break;
1359 	}
1360 
1361 	/* Special needs in case of Jumbo frames */
1362 	default_mtu = igb_get_prop(igb, PROP_DEFAULT_MTU,
1363 	    MIN_MTU, MAX_MTU, DEFAULT_MTU);
1364 	if ((hw->mac.type == e1000_82575) && (default_mtu > ETHERMTU)) {
1365 		u32 tx_space, min_tx, min_rx;
1366 		pba = E1000_READ_REG(hw, E1000_PBA);
1367 		tx_space = pba >> 16;
1368 		pba &= 0xffff;
1369 		min_tx = (igb->max_frame_size +
1370 		    sizeof (struct e1000_tx_desc) - ETHERNET_FCS_SIZE) * 2;
1371 		min_tx = roundup(min_tx, 1024);
1372 		min_tx >>= 10;
1373 		min_rx = igb->max_frame_size;
1374 		min_rx = roundup(min_rx, 1024);
1375 		min_rx >>= 10;
1376 		if (tx_space < min_tx &&
1377 		    ((min_tx - tx_space) < pba)) {
1378 			pba = pba - (min_tx - tx_space);
1379 			/*
1380 			 * if short on rx space, rx wins
1381 			 * and must trump tx adjustment
1382 			 */
1383 			if (pba < min_rx)
1384 				pba = min_rx;
1385 		}
1386 		E1000_WRITE_REG(hw, E1000_PBA, pba);
1387 	}
1388 
1389 	DEBUGOUT1("igb_init: pba=%dK", pba);
1390 
1391 	/*
1392 	 * These parameters control the automatic generation (Tx) and
1393 	 * response (Rx) to Ethernet PAUSE frames.
1394 	 * - High water mark should allow for at least two frames to be
1395 	 *   received after sending an XOFF.
1396 	 * - Low water mark works best when it is very near the high water mark.
1397 	 *   This allows the receiver to restart by sending XON when it has
1398 	 *   drained a bit.
1399 	 */
1400 	hwm = min(((pba << 10) * 9 / 10),
1401 	    ((pba << 10) - 2 * igb->max_frame_size));
1402 
1403 	if (hw->mac.type < e1000_82576) {
1404 		hw->fc.high_water = hwm & 0xFFF8;  /* 8-byte granularity */
1405 		hw->fc.low_water = hw->fc.high_water - 8;
1406 	} else {
1407 		hw->fc.high_water = hwm & 0xFFF0;  /* 16-byte granularity */
1408 		hw->fc.low_water = hw->fc.high_water - 16;
1409 	}
1410 
1411 	hw->fc.pause_time = E1000_FC_PAUSE_TIME;
1412 	hw->fc.send_xon = B_TRUE;
1413 
1414 	(void) e1000_validate_mdi_setting(hw);
1415 
1416 	/*
1417 	 * Reset the chipset hardware the second time to put PBA settings
1418 	 * into effect.
1419 	 */
1420 	if (e1000_reset_hw(hw) != E1000_SUCCESS) {
1421 		igb_error(igb, "Second reset failed");
1422 		goto init_adapter_fail;
1423 	}
1424 
1425 	/*
1426 	 * Don't wait for auto-negotiation to complete
1427 	 */
1428 	hw->phy.autoneg_wait_to_complete = B_FALSE;
1429 
1430 	/*
1431 	 * Copper options
1432 	 */
1433 	if (hw->phy.media_type == e1000_media_type_copper) {
1434 		hw->phy.mdix = 0;	/* AUTO_ALL_MODES */
1435 		hw->phy.disable_polarity_correction = B_FALSE;
1436 		hw->phy.ms_type = e1000_ms_hw_default; /* E1000_MASTER_SLAVE */
1437 	}
1438 
1439 	/*
1440 	 * Initialize link settings
1441 	 */
1442 	(void) igb_setup_link(igb, B_FALSE);
1443 
1444 	/*
1445 	 * Configure/Initialize hardware
1446 	 */
1447 	if (e1000_init_hw(hw) != E1000_SUCCESS) {
1448 		igb_error(igb, "Failed to initialize hardware");
1449 		goto init_adapter_fail;
1450 	}
1451 
1452 	/*
1453 	 *  Start the link setup timer
1454 	 */
1455 	igb_start_link_timer(igb);
1456 
1457 	/*
1458 	 * Disable wakeup control by default
1459 	 */
1460 	E1000_WRITE_REG(hw, E1000_WUC, 0);
1461 
1462 	/*
1463 	 * Record phy info in hw struct
1464 	 */
1465 	(void) e1000_get_phy_info(hw);
1466 
1467 	/*
1468 	 * Make sure driver has control
1469 	 */
1470 	igb_get_driver_control(hw);
1471 
1472 	/*
1473 	 * Restore LED settings to the default from EEPROM
1474 	 * to meet the standard for Sun platforms.
1475 	 */
1476 	(void) e1000_cleanup_led(hw);
1477 
1478 	/*
1479 	 * Setup MSI-X interrupts
1480 	 */
1481 	if (igb->intr_type == DDI_INTR_TYPE_MSIX)
1482 		igb->capab->setup_msix(igb);
1483 
1484 	/*
1485 	 * Initialize unicast addresses.
1486 	 */
1487 	igb_init_unicst(igb);
1488 
1489 	/*
1490 	 * Setup and initialize the mctable structures.
1491 	 */
1492 	igb_setup_multicst(igb);
1493 
1494 	/*
1495 	 * Set interrupt throttling rate
1496 	 */
1497 	for (i = 0; i < igb->intr_cnt; i++)
1498 		E1000_WRITE_REG(hw, E1000_EITR(i), igb->intr_throttling[i]);
1499 
1500 	/*
1501 	 * Read identifying information and place in devinfo.
1502 	 */
1503 	nvmword = 0xffff;
1504 	(void) e1000_read_nvm(&igb->hw, NVM_OEM_OFFSET_0, 1, &nvmword);
1505 	oemid[0] = (int)nvmword;
1506 	(void) e1000_read_nvm(&igb->hw, NVM_OEM_OFFSET_1, 1, &nvmword);
1507 	oemid[1] = (int)nvmword;
1508 	(void) ddi_prop_update_int_array(DDI_DEV_T_NONE, igb->dip,
1509 	    "oem-identifier", oemid, 2);
1510 
1511 	pbanum[0] = '\0';
1512 	(void) e1000_read_pba_string(&igb->hw, pbanum, sizeof (pbanum));
1513 	if (*pbanum != '\0') {
1514 		(void) ddi_prop_update_string(DDI_DEV_T_NONE, igb->dip,
1515 		    "printed-board-assembly", (char *)pbanum);
1516 	}
1517 
1518 	nvmword = 0xffff;
1519 	(void) e1000_read_nvm(&igb->hw, NVM_VERSION, 1, &nvmword);
1520 	if ((nvmword & 0xf00) == 0) {
1521 		(void) snprintf(eepromver, sizeof (eepromver), "%x.%x",
1522 		    (nvmword & 0xf000) >> 12, (nvmword & 0xff));
1523 		(void) ddi_prop_update_string(DDI_DEV_T_NONE, igb->dip,
1524 		    "nvm-version", eepromver);
1525 	}
1526 
1527 	/*
1528 	 * Save the state of the phy
1529 	 */
1530 	igb_get_phy_state(igb);
1531 
1532 	igb_param_sync(igb);
1533 
1534 	return (IGB_SUCCESS);
1535 
1536 init_adapter_fail:
1537 	/*
1538 	 * Reset PHY if possible
1539 	 */
1540 	if (e1000_check_reset_block(hw) == E1000_SUCCESS)
1541 		(void) e1000_phy_hw_reset(hw);
1542 
1543 	return (IGB_FAILURE);
1544 }
1545 
1546 /*
1547  * igb_stop_adapter - Stop the adapter
1548  */
1549 static void
igb_stop_adapter(igb_t * igb)1550 igb_stop_adapter(igb_t *igb)
1551 {
1552 	struct e1000_hw *hw = &igb->hw;
1553 
1554 	ASSERT(mutex_owned(&igb->gen_lock));
1555 
1556 	/* Stop the link setup timer */
1557 	igb_stop_link_timer(igb);
1558 
1559 	/* Tell firmware driver is no longer in control */
1560 	igb_release_driver_control(hw);
1561 
1562 	/*
1563 	 * Reset the chipset
1564 	 */
1565 	if (e1000_reset_hw(hw) != E1000_SUCCESS) {
1566 		igb_fm_ereport(igb, DDI_FM_DEVICE_INVAL_STATE);
1567 		ddi_fm_service_impact(igb->dip, DDI_SERVICE_LOST);
1568 	}
1569 
1570 	/*
1571 	 * e1000_phy_hw_reset is not needed here, MAC reset above is sufficient
1572 	 */
1573 }
1574 
1575 /*
1576  * igb_reset - Reset the chipset and restart the driver.
1577  *
1578  * It involves stopping and re-starting the chipset,
1579  * and re-configuring the rx/tx rings.
1580  */
1581 static int
igb_reset(igb_t * igb)1582 igb_reset(igb_t *igb)
1583 {
1584 	int i;
1585 
1586 	mutex_enter(&igb->gen_lock);
1587 
1588 	ASSERT(igb->igb_state & IGB_STARTED);
1589 	atomic_and_32(&igb->igb_state, ~IGB_STARTED);
1590 
1591 	/*
1592 	 * Disable the adapter interrupts to stop any rx/tx activities
1593 	 * before draining pending data and resetting hardware.
1594 	 */
1595 	igb_disable_adapter_interrupts(igb);
1596 
1597 	/*
1598 	 * Drain the pending transmit packets
1599 	 */
1600 	(void) igb_tx_drain(igb);
1601 
1602 	for (i = 0; i < igb->num_rx_rings; i++)
1603 		mutex_enter(&igb->rx_rings[i].rx_lock);
1604 	for (i = 0; i < igb->num_tx_rings; i++)
1605 		mutex_enter(&igb->tx_rings[i].tx_lock);
1606 
1607 	/*
1608 	 * Stop the adapter
1609 	 */
1610 	igb_stop_adapter(igb);
1611 
1612 	/*
1613 	 * Clean the pending tx data/resources
1614 	 */
1615 	igb_tx_clean(igb);
1616 
1617 	/*
1618 	 * Start the adapter
1619 	 */
1620 	if (igb_init_adapter(igb) != IGB_SUCCESS) {
1621 		igb_fm_ereport(igb, DDI_FM_DEVICE_INVAL_STATE);
1622 		goto reset_failure;
1623 	}
1624 
1625 	/*
1626 	 * Setup the rx/tx rings
1627 	 */
1628 	igb->tx_ring_init = B_FALSE;
1629 	igb_setup_rings(igb);
1630 
1631 	atomic_and_32(&igb->igb_state, ~(IGB_ERROR | IGB_STALL));
1632 
1633 	/*
1634 	 * Enable adapter interrupts
1635 	 * The interrupts must be enabled after the driver state is START
1636 	 */
1637 	igb->capab->enable_intr(igb);
1638 
1639 	if (igb_check_acc_handle(igb->osdep.cfg_handle) != DDI_FM_OK)
1640 		goto reset_failure;
1641 
1642 	if (igb_check_acc_handle(igb->osdep.reg_handle) != DDI_FM_OK)
1643 		goto reset_failure;
1644 
1645 	for (i = igb->num_tx_rings - 1; i >= 0; i--)
1646 		mutex_exit(&igb->tx_rings[i].tx_lock);
1647 	for (i = igb->num_rx_rings - 1; i >= 0; i--)
1648 		mutex_exit(&igb->rx_rings[i].rx_lock);
1649 
1650 	atomic_or_32(&igb->igb_state, IGB_STARTED);
1651 
1652 	mutex_exit(&igb->gen_lock);
1653 
1654 	return (IGB_SUCCESS);
1655 
1656 reset_failure:
1657 	for (i = igb->num_tx_rings - 1; i >= 0; i--)
1658 		mutex_exit(&igb->tx_rings[i].tx_lock);
1659 	for (i = igb->num_rx_rings - 1; i >= 0; i--)
1660 		mutex_exit(&igb->rx_rings[i].rx_lock);
1661 
1662 	mutex_exit(&igb->gen_lock);
1663 
1664 	ddi_fm_service_impact(igb->dip, DDI_SERVICE_LOST);
1665 
1666 	return (IGB_FAILURE);
1667 }
1668 
1669 /*
1670  * igb_tx_clean - Clean the pending transmit packets and DMA resources
1671  */
1672 static void
igb_tx_clean(igb_t * igb)1673 igb_tx_clean(igb_t *igb)
1674 {
1675 	igb_tx_ring_t *tx_ring;
1676 	tx_control_block_t *tcb;
1677 	link_list_t pending_list;
1678 	uint32_t desc_num;
1679 	int i, j;
1680 
1681 	LINK_LIST_INIT(&pending_list);
1682 
1683 	for (i = 0; i < igb->num_tx_rings; i++) {
1684 		tx_ring = &igb->tx_rings[i];
1685 
1686 		mutex_enter(&tx_ring->recycle_lock);
1687 
1688 		/*
1689 		 * Clean the pending tx data - the pending packets in the
1690 		 * work_list that have no chances to be transmitted again.
1691 		 *
1692 		 * We must ensure the chipset is stopped or the link is down
1693 		 * before cleaning the transmit packets.
1694 		 */
1695 		desc_num = 0;
1696 		for (j = 0; j < tx_ring->ring_size; j++) {
1697 			tcb = tx_ring->work_list[j];
1698 			if (tcb != NULL) {
1699 				desc_num += tcb->desc_num;
1700 
1701 				tx_ring->work_list[j] = NULL;
1702 
1703 				igb_free_tcb(tcb);
1704 
1705 				LIST_PUSH_TAIL(&pending_list, &tcb->link);
1706 			}
1707 		}
1708 
1709 		if (desc_num > 0) {
1710 			atomic_add_32(&tx_ring->tbd_free, desc_num);
1711 			ASSERT(tx_ring->tbd_free == tx_ring->ring_size);
1712 
1713 			/*
1714 			 * Reset the head and tail pointers of the tbd ring;
1715 			 * Reset the head write-back if it is enabled.
1716 			 */
1717 			tx_ring->tbd_head = 0;
1718 			tx_ring->tbd_tail = 0;
1719 			if (igb->tx_head_wb_enable)
1720 				*tx_ring->tbd_head_wb = 0;
1721 
1722 			E1000_WRITE_REG(&igb->hw, E1000_TDH(tx_ring->index), 0);
1723 			E1000_WRITE_REG(&igb->hw, E1000_TDT(tx_ring->index), 0);
1724 		}
1725 
1726 		mutex_exit(&tx_ring->recycle_lock);
1727 
1728 		/*
1729 		 * Add the tx control blocks in the pending list to
1730 		 * the free list.
1731 		 */
1732 		igb_put_free_list(tx_ring, &pending_list);
1733 	}
1734 }
1735 
1736 /*
1737  * igb_tx_drain - Drain the tx rings to allow pending packets to be transmitted
1738  */
1739 static boolean_t
igb_tx_drain(igb_t * igb)1740 igb_tx_drain(igb_t *igb)
1741 {
1742 	igb_tx_ring_t *tx_ring;
1743 	boolean_t done;
1744 	int i, j;
1745 
1746 	/*
1747 	 * Wait for a specific time to allow pending tx packets
1748 	 * to be transmitted.
1749 	 *
1750 	 * Check the counter tbd_free to see if transmission is done.
1751 	 * No lock protection is needed here.
1752 	 *
1753 	 * Return B_TRUE if all pending packets have been transmitted;
1754 	 * Otherwise return B_FALSE;
1755 	 */
1756 	for (i = 0; i < TX_DRAIN_TIME; i++) {
1757 
1758 		done = B_TRUE;
1759 		for (j = 0; j < igb->num_tx_rings; j++) {
1760 			tx_ring = &igb->tx_rings[j];
1761 			done = done &&
1762 			    (tx_ring->tbd_free == tx_ring->ring_size);
1763 		}
1764 
1765 		if (done)
1766 			break;
1767 
1768 		msec_delay(1);
1769 	}
1770 
1771 	return (done);
1772 }
1773 
1774 /*
1775  * igb_rx_drain - Wait for all rx buffers to be released by upper layer
1776  */
1777 static boolean_t
igb_rx_drain(igb_t * igb)1778 igb_rx_drain(igb_t *igb)
1779 {
1780 	boolean_t done;
1781 	int i;
1782 
1783 	/*
1784 	 * Polling the rx free list to check if those rx buffers held by
1785 	 * the upper layer are released.
1786 	 *
1787 	 * Check the counter rcb_free to see if all pending buffers are
1788 	 * released. No lock protection is needed here.
1789 	 *
1790 	 * Return B_TRUE if all pending buffers have been released;
1791 	 * Otherwise return B_FALSE;
1792 	 */
1793 	for (i = 0; i < RX_DRAIN_TIME; i++) {
1794 		done = (igb->rcb_pending == 0);
1795 
1796 		if (done)
1797 			break;
1798 
1799 		msec_delay(1);
1800 	}
1801 
1802 	return (done);
1803 }
1804 
1805 /*
1806  * igb_start - Start the driver/chipset
1807  */
1808 int
igb_start(igb_t * igb,boolean_t alloc_buffer)1809 igb_start(igb_t *igb, boolean_t alloc_buffer)
1810 {
1811 	int i;
1812 
1813 	ASSERT(mutex_owned(&igb->gen_lock));
1814 
1815 	if (alloc_buffer) {
1816 		if (igb_alloc_rx_data(igb) != IGB_SUCCESS) {
1817 			igb_error(igb,
1818 			    "Failed to allocate software receive rings");
1819 			return (IGB_FAILURE);
1820 		}
1821 
1822 		/* Allocate buffers for all the rx/tx rings */
1823 		if (igb_alloc_dma(igb) != IGB_SUCCESS) {
1824 			igb_error(igb, "Failed to allocate DMA resource");
1825 			return (IGB_FAILURE);
1826 		}
1827 
1828 		igb->tx_ring_init = B_TRUE;
1829 	} else {
1830 		igb->tx_ring_init = B_FALSE;
1831 	}
1832 
1833 	for (i = 0; i < igb->num_rx_rings; i++)
1834 		mutex_enter(&igb->rx_rings[i].rx_lock);
1835 	for (i = 0; i < igb->num_tx_rings; i++)
1836 		mutex_enter(&igb->tx_rings[i].tx_lock);
1837 
1838 	/*
1839 	 * Start the adapter
1840 	 */
1841 	if ((igb->attach_progress & ATTACH_PROGRESS_INIT_ADAPTER) == 0) {
1842 		if (igb_init_adapter(igb) != IGB_SUCCESS) {
1843 			igb_fm_ereport(igb, DDI_FM_DEVICE_INVAL_STATE);
1844 			goto start_failure;
1845 		}
1846 		igb->attach_progress |= ATTACH_PROGRESS_INIT_ADAPTER;
1847 	}
1848 
1849 	/*
1850 	 * Setup the rx/tx rings
1851 	 */
1852 	igb_setup_rings(igb);
1853 
1854 	/*
1855 	 * Enable adapter interrupts
1856 	 * The interrupts must be enabled after the driver state is START
1857 	 */
1858 	igb->capab->enable_intr(igb);
1859 
1860 	if (igb_check_acc_handle(igb->osdep.cfg_handle) != DDI_FM_OK)
1861 		goto start_failure;
1862 
1863 	if (igb_check_acc_handle(igb->osdep.reg_handle) != DDI_FM_OK)
1864 		goto start_failure;
1865 
1866 	if (igb->hw.mac.type == e1000_i350)
1867 		(void) e1000_set_eee_i350(&igb->hw);
1868 	else if (igb->hw.mac.type == e1000_i354)
1869 		(void) e1000_set_eee_i354(&igb->hw);
1870 
1871 	for (i = igb->num_tx_rings - 1; i >= 0; i--)
1872 		mutex_exit(&igb->tx_rings[i].tx_lock);
1873 	for (i = igb->num_rx_rings - 1; i >= 0; i--)
1874 		mutex_exit(&igb->rx_rings[i].rx_lock);
1875 
1876 	return (IGB_SUCCESS);
1877 
1878 start_failure:
1879 	for (i = igb->num_tx_rings - 1; i >= 0; i--)
1880 		mutex_exit(&igb->tx_rings[i].tx_lock);
1881 	for (i = igb->num_rx_rings - 1; i >= 0; i--)
1882 		mutex_exit(&igb->rx_rings[i].rx_lock);
1883 
1884 	ddi_fm_service_impact(igb->dip, DDI_SERVICE_LOST);
1885 
1886 	return (IGB_FAILURE);
1887 }
1888 
1889 /*
1890  * igb_stop - Stop the driver/chipset
1891  */
1892 void
igb_stop(igb_t * igb,boolean_t free_buffer)1893 igb_stop(igb_t *igb, boolean_t free_buffer)
1894 {
1895 	int i;
1896 
1897 	ASSERT(mutex_owned(&igb->gen_lock));
1898 
1899 	igb->attach_progress &= ~ATTACH_PROGRESS_INIT_ADAPTER;
1900 
1901 	/*
1902 	 * Disable the adapter interrupts
1903 	 */
1904 	igb_disable_adapter_interrupts(igb);
1905 
1906 	/*
1907 	 * Drain the pending tx packets
1908 	 */
1909 	(void) igb_tx_drain(igb);
1910 
1911 	for (i = 0; i < igb->num_rx_rings; i++)
1912 		mutex_enter(&igb->rx_rings[i].rx_lock);
1913 	for (i = 0; i < igb->num_tx_rings; i++)
1914 		mutex_enter(&igb->tx_rings[i].tx_lock);
1915 
1916 	/*
1917 	 * Stop the adapter
1918 	 */
1919 	igb_stop_adapter(igb);
1920 
1921 	/*
1922 	 * Clean the pending tx data/resources
1923 	 */
1924 	igb_tx_clean(igb);
1925 
1926 	for (i = igb->num_tx_rings - 1; i >= 0; i--)
1927 		mutex_exit(&igb->tx_rings[i].tx_lock);
1928 	for (i = igb->num_rx_rings - 1; i >= 0; i--)
1929 		mutex_exit(&igb->rx_rings[i].rx_lock);
1930 
1931 	if (igb_check_acc_handle(igb->osdep.reg_handle) != DDI_FM_OK)
1932 		ddi_fm_service_impact(igb->dip, DDI_SERVICE_LOST);
1933 
1934 	if (igb->link_state == LINK_STATE_UP) {
1935 		igb->link_state = LINK_STATE_UNKNOWN;
1936 		mac_link_update(igb->mac_hdl, igb->link_state);
1937 	}
1938 
1939 	if (free_buffer) {
1940 		/*
1941 		 * Release the DMA/memory resources of rx/tx rings
1942 		 */
1943 		igb_free_dma(igb);
1944 		igb_free_rx_data(igb);
1945 	}
1946 }
1947 
1948 /*
1949  * igb_alloc_rings - Allocate memory space for rx/tx rings
1950  */
1951 static int
igb_alloc_rings(igb_t * igb)1952 igb_alloc_rings(igb_t *igb)
1953 {
1954 	/*
1955 	 * Allocate memory space for rx rings
1956 	 */
1957 	igb->rx_rings = kmem_zalloc(
1958 	    sizeof (igb_rx_ring_t) * igb->num_rx_rings,
1959 	    KM_NOSLEEP);
1960 
1961 	if (igb->rx_rings == NULL) {
1962 		return (IGB_FAILURE);
1963 	}
1964 
1965 	/*
1966 	 * Allocate memory space for tx rings
1967 	 */
1968 	igb->tx_rings = kmem_zalloc(
1969 	    sizeof (igb_tx_ring_t) * igb->num_tx_rings,
1970 	    KM_NOSLEEP);
1971 
1972 	if (igb->tx_rings == NULL) {
1973 		kmem_free(igb->rx_rings,
1974 		    sizeof (igb_rx_ring_t) * igb->num_rx_rings);
1975 		igb->rx_rings = NULL;
1976 		return (IGB_FAILURE);
1977 	}
1978 
1979 	/*
1980 	 * Allocate memory space for rx ring groups
1981 	 */
1982 	igb->rx_groups = kmem_zalloc(
1983 	    sizeof (igb_rx_group_t) * igb->num_rx_groups,
1984 	    KM_NOSLEEP);
1985 
1986 	if (igb->rx_groups == NULL) {
1987 		kmem_free(igb->rx_rings,
1988 		    sizeof (igb_rx_ring_t) * igb->num_rx_rings);
1989 		kmem_free(igb->tx_rings,
1990 		    sizeof (igb_tx_ring_t) * igb->num_tx_rings);
1991 		igb->rx_rings = NULL;
1992 		igb->tx_rings = NULL;
1993 		return (IGB_FAILURE);
1994 	}
1995 
1996 	return (IGB_SUCCESS);
1997 }
1998 
1999 /*
2000  * igb_free_rings - Free the memory space of rx/tx rings.
2001  */
2002 static void
igb_free_rings(igb_t * igb)2003 igb_free_rings(igb_t *igb)
2004 {
2005 	if (igb->rx_rings != NULL) {
2006 		kmem_free(igb->rx_rings,
2007 		    sizeof (igb_rx_ring_t) * igb->num_rx_rings);
2008 		igb->rx_rings = NULL;
2009 	}
2010 
2011 	if (igb->tx_rings != NULL) {
2012 		kmem_free(igb->tx_rings,
2013 		    sizeof (igb_tx_ring_t) * igb->num_tx_rings);
2014 		igb->tx_rings = NULL;
2015 	}
2016 
2017 	if (igb->rx_groups != NULL) {
2018 		kmem_free(igb->rx_groups,
2019 		    sizeof (igb_rx_group_t) * igb->num_rx_groups);
2020 		igb->rx_groups = NULL;
2021 	}
2022 }
2023 
2024 static int
igb_alloc_rx_data(igb_t * igb)2025 igb_alloc_rx_data(igb_t *igb)
2026 {
2027 	igb_rx_ring_t *rx_ring;
2028 	int i;
2029 
2030 	for (i = 0; i < igb->num_rx_rings; i++) {
2031 		rx_ring = &igb->rx_rings[i];
2032 		if (igb_alloc_rx_ring_data(rx_ring) != IGB_SUCCESS)
2033 			goto alloc_rx_rings_failure;
2034 	}
2035 	return (IGB_SUCCESS);
2036 
2037 alloc_rx_rings_failure:
2038 	igb_free_rx_data(igb);
2039 	return (IGB_FAILURE);
2040 }
2041 
2042 static void
igb_free_rx_data(igb_t * igb)2043 igb_free_rx_data(igb_t *igb)
2044 {
2045 	igb_rx_ring_t *rx_ring;
2046 	igb_rx_data_t *rx_data;
2047 	int i;
2048 
2049 	for (i = 0; i < igb->num_rx_rings; i++) {
2050 		rx_ring = &igb->rx_rings[i];
2051 
2052 		mutex_enter(&igb->rx_pending_lock);
2053 		rx_data = rx_ring->rx_data;
2054 
2055 		if (rx_data != NULL) {
2056 			rx_data->flag |= IGB_RX_STOPPED;
2057 
2058 			if (rx_data->rcb_pending == 0) {
2059 				igb_free_rx_ring_data(rx_data);
2060 				rx_ring->rx_data = NULL;
2061 			}
2062 		}
2063 
2064 		mutex_exit(&igb->rx_pending_lock);
2065 	}
2066 }
2067 
2068 /*
2069  * igb_setup_rings - Setup rx/tx rings
2070  */
2071 static void
igb_setup_rings(igb_t * igb)2072 igb_setup_rings(igb_t *igb)
2073 {
2074 	/*
2075 	 * Setup the rx/tx rings, including the following:
2076 	 *
2077 	 * 1. Setup the descriptor ring and the control block buffers;
2078 	 * 2. Initialize necessary registers for receive/transmit;
2079 	 * 3. Initialize software pointers/parameters for receive/transmit;
2080 	 */
2081 	igb_setup_rx(igb);
2082 
2083 	igb_setup_tx(igb);
2084 }
2085 
2086 static void
igb_setup_rx_ring(igb_rx_ring_t * rx_ring)2087 igb_setup_rx_ring(igb_rx_ring_t *rx_ring)
2088 {
2089 	igb_t *igb = rx_ring->igb;
2090 	igb_rx_data_t *rx_data = rx_ring->rx_data;
2091 	struct e1000_hw *hw = &igb->hw;
2092 	rx_control_block_t *rcb;
2093 	union e1000_adv_rx_desc	*rbd;
2094 	uint32_t size;
2095 	uint32_t buf_low;
2096 	uint32_t buf_high;
2097 	uint32_t rxdctl;
2098 	int i;
2099 
2100 	ASSERT(mutex_owned(&rx_ring->rx_lock));
2101 	ASSERT(mutex_owned(&igb->gen_lock));
2102 
2103 	/*
2104 	 * Initialize descriptor ring with buffer addresses
2105 	 */
2106 	for (i = 0; i < igb->rx_ring_size; i++) {
2107 		rcb = rx_data->work_list[i];
2108 		rbd = &rx_data->rbd_ring[i];
2109 
2110 		rbd->read.pkt_addr = rcb->rx_buf.dma_address;
2111 		rbd->read.hdr_addr = NULL;
2112 	}
2113 
2114 	/*
2115 	 * Initialize the base address registers
2116 	 */
2117 	buf_low = (uint32_t)rx_data->rbd_area.dma_address;
2118 	buf_high = (uint32_t)(rx_data->rbd_area.dma_address >> 32);
2119 	E1000_WRITE_REG(hw, E1000_RDBAH(rx_ring->index), buf_high);
2120 	E1000_WRITE_REG(hw, E1000_RDBAL(rx_ring->index), buf_low);
2121 
2122 	/*
2123 	 * Initialize the length register
2124 	 */
2125 	size = rx_data->ring_size * sizeof (union e1000_adv_rx_desc);
2126 	E1000_WRITE_REG(hw, E1000_RDLEN(rx_ring->index), size);
2127 
2128 	/*
2129 	 * Initialize buffer size & descriptor type
2130 	 */
2131 	E1000_WRITE_REG(hw, E1000_SRRCTL(rx_ring->index),
2132 	    ((igb->rx_buf_size >> E1000_SRRCTL_BSIZEPKT_SHIFT) |
2133 	    E1000_SRRCTL_DESCTYPE_ADV_ONEBUF));
2134 
2135 	/*
2136 	 * Setup the Receive Descriptor Control Register (RXDCTL)
2137 	 */
2138 	rxdctl = E1000_READ_REG(hw, E1000_RXDCTL(rx_ring->index));
2139 	rxdctl &= igb->capab->rxdctl_mask;
2140 	rxdctl |= E1000_RXDCTL_QUEUE_ENABLE;
2141 	rxdctl |= 16;		/* pthresh */
2142 	rxdctl |= 8 << 8;	/* hthresh */
2143 	rxdctl |= 1 << 16;	/* wthresh */
2144 	E1000_WRITE_REG(hw, E1000_RXDCTL(rx_ring->index), rxdctl);
2145 
2146 	rx_data->rbd_next = 0;
2147 }
2148 
2149 static void
igb_setup_rx(igb_t * igb)2150 igb_setup_rx(igb_t *igb)
2151 {
2152 	igb_rx_ring_t *rx_ring;
2153 	igb_rx_data_t *rx_data;
2154 	igb_rx_group_t *rx_group;
2155 	struct e1000_hw *hw = &igb->hw;
2156 	uint32_t rctl, rxcsum;
2157 	uint32_t ring_per_group;
2158 	int i;
2159 
2160 	/*
2161 	 * Setup the Receive Control Register (RCTL), and enable the
2162 	 * receiver. The initial configuration is to: enable the receiver,
2163 	 * accept broadcasts, discard bad packets, accept long packets,
2164 	 * disable VLAN filter checking, and set receive buffer size to
2165 	 * 2k.  For 82575, also set the receive descriptor minimum
2166 	 * threshold size to 1/2 the ring.
2167 	 */
2168 	rctl = E1000_READ_REG(hw, E1000_RCTL);
2169 
2170 	/*
2171 	 * Clear the field used for wakeup control.  This driver doesn't do
2172 	 * wakeup but leave this here for completeness.
2173 	 */
2174 	rctl &= ~(3 << E1000_RCTL_MO_SHIFT);
2175 	rctl &= ~(E1000_RCTL_LBM_TCVR | E1000_RCTL_LBM_MAC);
2176 
2177 	rctl |= (E1000_RCTL_EN |	/* Enable Receive Unit */
2178 	    E1000_RCTL_BAM |		/* Accept Broadcast Packets */
2179 	    E1000_RCTL_LPE |		/* Large Packet Enable */
2180 					/* Multicast filter offset */
2181 	    (hw->mac.mc_filter_type << E1000_RCTL_MO_SHIFT) |
2182 	    E1000_RCTL_RDMTS_HALF |	/* rx descriptor threshold */
2183 	    E1000_RCTL_SECRC);		/* Strip Ethernet CRC */
2184 
2185 	for (i = 0; i < igb->num_rx_groups; i++) {
2186 		rx_group = &igb->rx_groups[i];
2187 		rx_group->index = i;
2188 		rx_group->igb = igb;
2189 	}
2190 
2191 	/*
2192 	 * Set up all rx descriptor rings - must be called before receive unit
2193 	 * enabled.
2194 	 */
2195 	ring_per_group = igb->num_rx_rings / igb->num_rx_groups;
2196 	for (i = 0; i < igb->num_rx_rings; i++) {
2197 		rx_ring = &igb->rx_rings[i];
2198 		igb_setup_rx_ring(rx_ring);
2199 
2200 		/*
2201 		 * Map a ring to a group by assigning a group index
2202 		 */
2203 		rx_ring->group_index = i / ring_per_group;
2204 	}
2205 
2206 	/*
2207 	 * Setup the Rx Long Packet Max Length register
2208 	 */
2209 	E1000_WRITE_REG(hw, E1000_RLPML, igb->max_frame_size);
2210 
2211 	/*
2212 	 * Hardware checksum settings
2213 	 */
2214 	if (igb->rx_hcksum_enable) {
2215 		rxcsum =
2216 		    E1000_RXCSUM_TUOFL |	/* TCP/UDP checksum */
2217 		    E1000_RXCSUM_IPOFL;		/* IP checksum */
2218 
2219 		E1000_WRITE_REG(hw, E1000_RXCSUM, rxcsum);
2220 	}
2221 
2222 	/*
2223 	 * Setup classify and RSS for multiple receive queues
2224 	 */
2225 	switch (igb->vmdq_mode) {
2226 	case E1000_VMDQ_OFF:
2227 		/*
2228 		 * One ring group, only RSS is needed when more than
2229 		 * one ring enabled.
2230 		 */
2231 		if (igb->num_rx_rings > 1)
2232 			igb_setup_rss(igb);
2233 		break;
2234 	case E1000_VMDQ_MAC:
2235 		/*
2236 		 * Multiple groups, each group has one ring,
2237 		 * only the MAC classification is needed.
2238 		 */
2239 		igb_setup_mac_classify(igb);
2240 		break;
2241 	case E1000_VMDQ_MAC_RSS:
2242 		/*
2243 		 * Multiple groups and multiple rings, both
2244 		 * MAC classification and RSS are needed.
2245 		 */
2246 		igb_setup_mac_rss_classify(igb);
2247 		break;
2248 	}
2249 
2250 	/*
2251 	 * Enable the receive unit - must be done after all
2252 	 * the rx setup above.
2253 	 */
2254 	E1000_WRITE_REG(hw, E1000_RCTL, rctl);
2255 
2256 	/*
2257 	 * Initialize all adapter ring head & tail pointers - must
2258 	 * be done after receive unit is enabled
2259 	 */
2260 	for (i = 0; i < igb->num_rx_rings; i++) {
2261 		rx_ring = &igb->rx_rings[i];
2262 		rx_data = rx_ring->rx_data;
2263 		E1000_WRITE_REG(hw, E1000_RDH(i), 0);
2264 		E1000_WRITE_REG(hw, E1000_RDT(i), rx_data->ring_size - 1);
2265 	}
2266 
2267 	/*
2268 	 * 82575 with manageability enabled needs a special flush to make
2269 	 * sure the fifos start clean.
2270 	 */
2271 	if ((hw->mac.type == e1000_82575) &&
2272 	    (E1000_READ_REG(hw, E1000_MANC) & E1000_MANC_RCV_TCO_EN)) {
2273 		e1000_rx_fifo_flush_82575(hw);
2274 	}
2275 }
2276 
2277 static void
igb_setup_tx_ring(igb_tx_ring_t * tx_ring)2278 igb_setup_tx_ring(igb_tx_ring_t *tx_ring)
2279 {
2280 	igb_t *igb = tx_ring->igb;
2281 	struct e1000_hw *hw = &igb->hw;
2282 	uint32_t size;
2283 	uint32_t buf_low;
2284 	uint32_t buf_high;
2285 	uint32_t reg_val;
2286 
2287 	ASSERT(mutex_owned(&tx_ring->tx_lock));
2288 	ASSERT(mutex_owned(&igb->gen_lock));
2289 
2290 
2291 	/*
2292 	 * Initialize the length register
2293 	 */
2294 	size = tx_ring->ring_size * sizeof (union e1000_adv_tx_desc);
2295 	E1000_WRITE_REG(hw, E1000_TDLEN(tx_ring->index), size);
2296 
2297 	/*
2298 	 * Initialize the base address registers
2299 	 */
2300 	buf_low = (uint32_t)tx_ring->tbd_area.dma_address;
2301 	buf_high = (uint32_t)(tx_ring->tbd_area.dma_address >> 32);
2302 	E1000_WRITE_REG(hw, E1000_TDBAL(tx_ring->index), buf_low);
2303 	E1000_WRITE_REG(hw, E1000_TDBAH(tx_ring->index), buf_high);
2304 
2305 	/*
2306 	 * Setup head & tail pointers
2307 	 */
2308 	E1000_WRITE_REG(hw, E1000_TDH(tx_ring->index), 0);
2309 	E1000_WRITE_REG(hw, E1000_TDT(tx_ring->index), 0);
2310 
2311 	/*
2312 	 * Setup head write-back
2313 	 */
2314 	if (igb->tx_head_wb_enable) {
2315 		/*
2316 		 * The memory of the head write-back is allocated using
2317 		 * the extra tbd beyond the tail of the tbd ring.
2318 		 */
2319 		tx_ring->tbd_head_wb = (uint32_t *)
2320 		    ((uintptr_t)tx_ring->tbd_area.address + size);
2321 		*tx_ring->tbd_head_wb = 0;
2322 
2323 		buf_low = (uint32_t)
2324 		    (tx_ring->tbd_area.dma_address + size);
2325 		buf_high = (uint32_t)
2326 		    ((tx_ring->tbd_area.dma_address + size) >> 32);
2327 
2328 		/* Set the head write-back enable bit */
2329 		buf_low |= E1000_TX_HEAD_WB_ENABLE;
2330 
2331 		E1000_WRITE_REG(hw, E1000_TDWBAL(tx_ring->index), buf_low);
2332 		E1000_WRITE_REG(hw, E1000_TDWBAH(tx_ring->index), buf_high);
2333 
2334 		/*
2335 		 * Turn off relaxed ordering for head write back or it will
2336 		 * cause problems with the tx recycling
2337 		 */
2338 		reg_val = E1000_READ_REG(hw,
2339 		    E1000_DCA_TXCTRL(tx_ring->index));
2340 		reg_val &= ~E1000_DCA_TXCTRL_TX_WB_RO_EN;
2341 		E1000_WRITE_REG(hw,
2342 		    E1000_DCA_TXCTRL(tx_ring->index), reg_val);
2343 	} else {
2344 		tx_ring->tbd_head_wb = NULL;
2345 	}
2346 
2347 	tx_ring->tbd_head = 0;
2348 	tx_ring->tbd_tail = 0;
2349 	tx_ring->tbd_free = tx_ring->ring_size;
2350 
2351 	if (igb->tx_ring_init == B_TRUE) {
2352 		tx_ring->tcb_head = 0;
2353 		tx_ring->tcb_tail = 0;
2354 		tx_ring->tcb_free = tx_ring->free_list_size;
2355 	}
2356 
2357 	/*
2358 	 * Enable TXDCTL per queue
2359 	 */
2360 	reg_val = E1000_READ_REG(hw, E1000_TXDCTL(tx_ring->index));
2361 	reg_val |= E1000_TXDCTL_QUEUE_ENABLE;
2362 	E1000_WRITE_REG(hw, E1000_TXDCTL(tx_ring->index), reg_val);
2363 
2364 	/*
2365 	 * Initialize hardware checksum offload settings
2366 	 */
2367 	bzero(&tx_ring->tx_context, sizeof (tx_context_t));
2368 }
2369 
2370 static void
igb_setup_tx(igb_t * igb)2371 igb_setup_tx(igb_t *igb)
2372 {
2373 	igb_tx_ring_t *tx_ring;
2374 	struct e1000_hw *hw = &igb->hw;
2375 	uint32_t reg_val;
2376 	int i;
2377 
2378 	for (i = 0; i < igb->num_tx_rings; i++) {
2379 		tx_ring = &igb->tx_rings[i];
2380 		igb_setup_tx_ring(tx_ring);
2381 	}
2382 
2383 	/*
2384 	 * Setup the Transmit Control Register (TCTL)
2385 	 */
2386 	reg_val = E1000_READ_REG(hw, E1000_TCTL);
2387 	reg_val &= ~E1000_TCTL_CT;
2388 	reg_val |= E1000_TCTL_PSP | E1000_TCTL_RTLC |
2389 	    (E1000_COLLISION_THRESHOLD << E1000_CT_SHIFT);
2390 
2391 	/* Enable transmits */
2392 	reg_val |= E1000_TCTL_EN;
2393 
2394 	E1000_WRITE_REG(hw, E1000_TCTL, reg_val);
2395 }
2396 
2397 /*
2398  * igb_setup_rss - Setup receive-side scaling feature
2399  */
2400 static void
igb_setup_rss(igb_t * igb)2401 igb_setup_rss(igb_t *igb)
2402 {
2403 	struct e1000_hw *hw = &igb->hw;
2404 	uint32_t i, mrqc, rxcsum;
2405 	int shift = 0;
2406 	uint32_t random;
2407 	union e1000_reta {
2408 		uint32_t	dword;
2409 		uint8_t		bytes[4];
2410 	} reta;
2411 
2412 	/* Setup the Redirection Table */
2413 	if (hw->mac.type == e1000_82576) {
2414 		shift = 3;
2415 	} else if (hw->mac.type == e1000_82575) {
2416 		shift = 6;
2417 	}
2418 	for (i = 0; i < (32 * 4); i++) {
2419 		reta.bytes[i & 3] = (i % igb->num_rx_rings) << shift;
2420 		if ((i & 3) == 3) {
2421 			E1000_WRITE_REG(hw,
2422 			    (E1000_RETA(0) + (i & ~3)), reta.dword);
2423 		}
2424 	}
2425 
2426 	/* Fill out hash function seeds */
2427 	for (i = 0; i < 10; i++) {
2428 		(void) random_get_pseudo_bytes((uint8_t *)&random,
2429 		    sizeof (uint32_t));
2430 		E1000_WRITE_REG(hw, E1000_RSSRK(i), random);
2431 	}
2432 
2433 	/* Setup the Multiple Receive Queue Control register */
2434 	mrqc = E1000_MRQC_ENABLE_RSS_4Q;
2435 	mrqc |= (E1000_MRQC_RSS_FIELD_IPV4 |
2436 	    E1000_MRQC_RSS_FIELD_IPV4_TCP |
2437 	    E1000_MRQC_RSS_FIELD_IPV6 |
2438 	    E1000_MRQC_RSS_FIELD_IPV6_TCP |
2439 	    E1000_MRQC_RSS_FIELD_IPV4_UDP |
2440 	    E1000_MRQC_RSS_FIELD_IPV6_UDP |
2441 	    E1000_MRQC_RSS_FIELD_IPV6_UDP_EX |
2442 	    E1000_MRQC_RSS_FIELD_IPV6_TCP_EX);
2443 
2444 	E1000_WRITE_REG(hw, E1000_MRQC, mrqc);
2445 
2446 	/*
2447 	 * Disable Packet Checksum to enable RSS for multiple receive queues.
2448 	 *
2449 	 * The Packet Checksum is not ethernet CRC. It is another kind of
2450 	 * checksum offloading provided by the 82575 chipset besides the IP
2451 	 * header checksum offloading and the TCP/UDP checksum offloading.
2452 	 * The Packet Checksum is by default computed over the entire packet
2453 	 * from the first byte of the DA through the last byte of the CRC,
2454 	 * including the Ethernet and IP headers.
2455 	 *
2456 	 * It is a hardware limitation that Packet Checksum is mutually
2457 	 * exclusive with RSS.
2458 	 */
2459 	rxcsum = E1000_READ_REG(hw, E1000_RXCSUM);
2460 	rxcsum |= E1000_RXCSUM_PCSD;
2461 	E1000_WRITE_REG(hw, E1000_RXCSUM, rxcsum);
2462 }
2463 
2464 /*
2465  * igb_setup_mac_rss_classify - Setup MAC classification and rss
2466  */
2467 static void
igb_setup_mac_rss_classify(igb_t * igb)2468 igb_setup_mac_rss_classify(igb_t *igb)
2469 {
2470 	struct e1000_hw *hw = &igb->hw;
2471 	uint32_t i, mrqc, vmdctl, rxcsum;
2472 	uint32_t ring_per_group;
2473 	int shift_group0, shift_group1;
2474 	uint32_t random;
2475 	union e1000_reta {
2476 		uint32_t	dword;
2477 		uint8_t		bytes[4];
2478 	} reta;
2479 
2480 	ring_per_group = igb->num_rx_rings / igb->num_rx_groups;
2481 
2482 	/* Setup the Redirection Table, it is shared between two groups */
2483 	shift_group0 = 2;
2484 	shift_group1 = 6;
2485 	for (i = 0; i < (32 * 4); i++) {
2486 		reta.bytes[i & 3] = ((i % ring_per_group) << shift_group0) |
2487 		    ((ring_per_group + (i % ring_per_group)) << shift_group1);
2488 		if ((i & 3) == 3) {
2489 			E1000_WRITE_REG(hw,
2490 			    (E1000_RETA(0) + (i & ~3)), reta.dword);
2491 		}
2492 	}
2493 
2494 	/* Fill out hash function seeds */
2495 	for (i = 0; i < 10; i++) {
2496 		(void) random_get_pseudo_bytes((uint8_t *)&random,
2497 		    sizeof (uint32_t));
2498 		E1000_WRITE_REG(hw, E1000_RSSRK(i), random);
2499 	}
2500 
2501 	/*
2502 	 * Setup the Multiple Receive Queue Control register,
2503 	 * enable VMDq based on packet destination MAC address and RSS.
2504 	 */
2505 	mrqc = E1000_MRQC_ENABLE_VMDQ_MAC_RSS_GROUP;
2506 	mrqc |= (E1000_MRQC_RSS_FIELD_IPV4 |
2507 	    E1000_MRQC_RSS_FIELD_IPV4_TCP |
2508 	    E1000_MRQC_RSS_FIELD_IPV6 |
2509 	    E1000_MRQC_RSS_FIELD_IPV6_TCP |
2510 	    E1000_MRQC_RSS_FIELD_IPV4_UDP |
2511 	    E1000_MRQC_RSS_FIELD_IPV6_UDP |
2512 	    E1000_MRQC_RSS_FIELD_IPV6_UDP_EX |
2513 	    E1000_MRQC_RSS_FIELD_IPV6_TCP_EX);
2514 
2515 	E1000_WRITE_REG(hw, E1000_MRQC, mrqc);
2516 
2517 
2518 	/* Define the default group and default queues */
2519 	vmdctl = E1000_VMDQ_MAC_GROUP_DEFAULT_QUEUE;
2520 	E1000_WRITE_REG(hw, E1000_VT_CTL, vmdctl);
2521 
2522 	/*
2523 	 * Disable Packet Checksum to enable RSS for multiple receive queues.
2524 	 *
2525 	 * The Packet Checksum is not ethernet CRC. It is another kind of
2526 	 * checksum offloading provided by the 82575 chipset besides the IP
2527 	 * header checksum offloading and the TCP/UDP checksum offloading.
2528 	 * The Packet Checksum is by default computed over the entire packet
2529 	 * from the first byte of the DA through the last byte of the CRC,
2530 	 * including the Ethernet and IP headers.
2531 	 *
2532 	 * It is a hardware limitation that Packet Checksum is mutually
2533 	 * exclusive with RSS.
2534 	 */
2535 	rxcsum = E1000_READ_REG(hw, E1000_RXCSUM);
2536 	rxcsum |= E1000_RXCSUM_PCSD;
2537 	E1000_WRITE_REG(hw, E1000_RXCSUM, rxcsum);
2538 }
2539 
2540 /*
2541  * igb_setup_mac_classify - Setup MAC classification feature
2542  */
2543 static void
igb_setup_mac_classify(igb_t * igb)2544 igb_setup_mac_classify(igb_t *igb)
2545 {
2546 	struct e1000_hw *hw = &igb->hw;
2547 	uint32_t mrqc, rxcsum;
2548 
2549 	/*
2550 	 * Setup the Multiple Receive Queue Control register,
2551 	 * enable VMDq based on packet destination MAC address.
2552 	 */
2553 	mrqc = E1000_MRQC_ENABLE_VMDQ_MAC_GROUP;
2554 	E1000_WRITE_REG(hw, E1000_MRQC, mrqc);
2555 
2556 	/*
2557 	 * Disable Packet Checksum to enable RSS for multiple receive queues.
2558 	 *
2559 	 * The Packet Checksum is not ethernet CRC. It is another kind of
2560 	 * checksum offloading provided by the 82575 chipset besides the IP
2561 	 * header checksum offloading and the TCP/UDP checksum offloading.
2562 	 * The Packet Checksum is by default computed over the entire packet
2563 	 * from the first byte of the DA through the last byte of the CRC,
2564 	 * including the Ethernet and IP headers.
2565 	 *
2566 	 * It is a hardware limitation that Packet Checksum is mutually
2567 	 * exclusive with RSS.
2568 	 */
2569 	rxcsum = E1000_READ_REG(hw, E1000_RXCSUM);
2570 	rxcsum |= E1000_RXCSUM_PCSD;
2571 	E1000_WRITE_REG(hw, E1000_RXCSUM, rxcsum);
2572 
2573 }
2574 
2575 /*
2576  * igb_init_unicst - Initialize the unicast addresses
2577  */
2578 static void
igb_init_unicst(igb_t * igb)2579 igb_init_unicst(igb_t *igb)
2580 {
2581 	struct e1000_hw *hw = &igb->hw;
2582 	int slot;
2583 
2584 	/*
2585 	 * Here we should consider two situations:
2586 	 *
2587 	 * 1. Chipset is initialized the first time
2588 	 *    Initialize the multiple unicast addresses, and
2589 	 *    save the default MAC address.
2590 	 *
2591 	 * 2. Chipset is reset
2592 	 *    Recover the multiple unicast addresses from the
2593 	 *    software data structure to the RAR registers.
2594 	 */
2595 
2596 	/*
2597 	 * Clear the default MAC address in the RAR0 rgister,
2598 	 * which is loaded from EEPROM when system boot or chipreset,
2599 	 * this will cause the conficts with add_mac/rem_mac entry
2600 	 * points when VMDq is enabled. For this reason, the RAR0
2601 	 * must be cleared for both cases mentioned above.
2602 	 */
2603 	e1000_rar_clear(hw, 0);
2604 
2605 	if (!igb->unicst_init) {
2606 
2607 		/* Initialize the multiple unicast addresses */
2608 		igb->unicst_total = MAX_NUM_UNICAST_ADDRESSES;
2609 		igb->unicst_avail = igb->unicst_total;
2610 
2611 		for (slot = 0; slot < igb->unicst_total; slot++)
2612 			igb->unicst_addr[slot].mac.set = 0;
2613 
2614 		igb->unicst_init = B_TRUE;
2615 	} else {
2616 		/* Re-configure the RAR registers */
2617 		for (slot = 0; slot < igb->unicst_total; slot++) {
2618 			(void) e1000_rar_set_vmdq(hw,
2619 			    igb->unicst_addr[slot].mac.addr,
2620 			    slot, igb->vmdq_mode,
2621 			    igb->unicst_addr[slot].mac.group_index);
2622 		}
2623 	}
2624 }
2625 
2626 /*
2627  * igb_unicst_find - Find the slot for the specified unicast address
2628  */
2629 int
igb_unicst_find(igb_t * igb,const uint8_t * mac_addr)2630 igb_unicst_find(igb_t *igb, const uint8_t *mac_addr)
2631 {
2632 	int slot;
2633 
2634 	ASSERT(mutex_owned(&igb->gen_lock));
2635 
2636 	for (slot = 0; slot < igb->unicst_total; slot++) {
2637 		if (bcmp(igb->unicst_addr[slot].mac.addr,
2638 		    mac_addr, ETHERADDRL) == 0)
2639 			return (slot);
2640 	}
2641 
2642 	return (-1);
2643 }
2644 
2645 /*
2646  * igb_unicst_set - Set the unicast address to the specified slot
2647  */
2648 int
igb_unicst_set(igb_t * igb,const uint8_t * mac_addr,int slot)2649 igb_unicst_set(igb_t *igb, const uint8_t *mac_addr,
2650     int slot)
2651 {
2652 	struct e1000_hw *hw = &igb->hw;
2653 
2654 	ASSERT(mutex_owned(&igb->gen_lock));
2655 
2656 	/*
2657 	 * Save the unicast address in the software data structure
2658 	 */
2659 	bcopy(mac_addr, igb->unicst_addr[slot].mac.addr, ETHERADDRL);
2660 
2661 	/*
2662 	 * Set the unicast address to the RAR register
2663 	 */
2664 	(void) e1000_rar_set(hw, (uint8_t *)mac_addr, slot);
2665 
2666 	if (igb_check_acc_handle(igb->osdep.reg_handle) != DDI_FM_OK) {
2667 		ddi_fm_service_impact(igb->dip, DDI_SERVICE_DEGRADED);
2668 		return (EIO);
2669 	}
2670 
2671 	return (0);
2672 }
2673 
2674 /*
2675  * igb_multicst_add - Add a multicst address
2676  */
2677 int
igb_multicst_add(igb_t * igb,const uint8_t * multiaddr)2678 igb_multicst_add(igb_t *igb, const uint8_t *multiaddr)
2679 {
2680 	struct ether_addr *new_table;
2681 	size_t new_len;
2682 	size_t old_len;
2683 
2684 	ASSERT(mutex_owned(&igb->gen_lock));
2685 
2686 	if ((multiaddr[0] & 01) == 0) {
2687 		igb_error(igb, "Illegal multicast address");
2688 		return (EINVAL);
2689 	}
2690 
2691 	if (igb->mcast_count >= igb->mcast_max_num) {
2692 		igb_error(igb, "Adapter requested more than %d mcast addresses",
2693 		    igb->mcast_max_num);
2694 		return (ENOENT);
2695 	}
2696 
2697 	if (igb->mcast_count == igb->mcast_alloc_count) {
2698 		old_len = igb->mcast_alloc_count *
2699 		    sizeof (struct ether_addr);
2700 		new_len = (igb->mcast_alloc_count + MCAST_ALLOC_COUNT) *
2701 		    sizeof (struct ether_addr);
2702 
2703 		new_table = kmem_alloc(new_len, KM_NOSLEEP);
2704 		if (new_table == NULL) {
2705 			igb_error(igb,
2706 			    "Not enough memory to alloc mcast table");
2707 			return (ENOMEM);
2708 		}
2709 
2710 		if (igb->mcast_table != NULL) {
2711 			bcopy(igb->mcast_table, new_table, old_len);
2712 			kmem_free(igb->mcast_table, old_len);
2713 		}
2714 		igb->mcast_alloc_count += MCAST_ALLOC_COUNT;
2715 		igb->mcast_table = new_table;
2716 	}
2717 
2718 	bcopy(multiaddr,
2719 	    &igb->mcast_table[igb->mcast_count], ETHERADDRL);
2720 	igb->mcast_count++;
2721 
2722 	/*
2723 	 * Update the multicast table in the hardware
2724 	 */
2725 	igb_setup_multicst(igb);
2726 
2727 	if (igb_check_acc_handle(igb->osdep.reg_handle) != DDI_FM_OK) {
2728 		ddi_fm_service_impact(igb->dip, DDI_SERVICE_DEGRADED);
2729 		return (EIO);
2730 	}
2731 
2732 	return (0);
2733 }
2734 
2735 /*
2736  * igb_multicst_remove - Remove a multicst address
2737  */
2738 int
igb_multicst_remove(igb_t * igb,const uint8_t * multiaddr)2739 igb_multicst_remove(igb_t *igb, const uint8_t *multiaddr)
2740 {
2741 	struct ether_addr *new_table;
2742 	size_t new_len;
2743 	size_t old_len;
2744 	int i;
2745 
2746 	ASSERT(mutex_owned(&igb->gen_lock));
2747 
2748 	for (i = 0; i < igb->mcast_count; i++) {
2749 		if (bcmp(multiaddr, &igb->mcast_table[i],
2750 		    ETHERADDRL) == 0) {
2751 			for (i++; i < igb->mcast_count; i++) {
2752 				igb->mcast_table[i - 1] =
2753 				    igb->mcast_table[i];
2754 			}
2755 			igb->mcast_count--;
2756 			break;
2757 		}
2758 	}
2759 
2760 	if ((igb->mcast_alloc_count - igb->mcast_count) >
2761 	    MCAST_ALLOC_COUNT) {
2762 		old_len = igb->mcast_alloc_count *
2763 		    sizeof (struct ether_addr);
2764 		new_len = (igb->mcast_alloc_count - MCAST_ALLOC_COUNT) *
2765 		    sizeof (struct ether_addr);
2766 
2767 		new_table = kmem_alloc(new_len, KM_NOSLEEP);
2768 		if (new_table != NULL) {
2769 			bcopy(igb->mcast_table, new_table, new_len);
2770 			kmem_free(igb->mcast_table, old_len);
2771 			igb->mcast_alloc_count -= MCAST_ALLOC_COUNT;
2772 			igb->mcast_table = new_table;
2773 		}
2774 	}
2775 
2776 	/*
2777 	 * Update the multicast table in the hardware
2778 	 */
2779 	igb_setup_multicst(igb);
2780 
2781 	if (igb_check_acc_handle(igb->osdep.reg_handle) != DDI_FM_OK) {
2782 		ddi_fm_service_impact(igb->dip, DDI_SERVICE_DEGRADED);
2783 		return (EIO);
2784 	}
2785 
2786 	return (0);
2787 }
2788 
2789 static void
igb_release_multicast(igb_t * igb)2790 igb_release_multicast(igb_t *igb)
2791 {
2792 	if (igb->mcast_table != NULL) {
2793 		kmem_free(igb->mcast_table,
2794 		    igb->mcast_alloc_count * sizeof (struct ether_addr));
2795 		igb->mcast_table = NULL;
2796 	}
2797 }
2798 
2799 /*
2800  * igb_setup_multicast - setup multicast data structures
2801  *
2802  * This routine initializes all of the multicast related structures
2803  * and save them in the hardware registers.
2804  */
2805 static void
igb_setup_multicst(igb_t * igb)2806 igb_setup_multicst(igb_t *igb)
2807 {
2808 	uint8_t *mc_addr_list;
2809 	uint32_t mc_addr_count;
2810 	struct e1000_hw *hw = &igb->hw;
2811 
2812 	ASSERT(mutex_owned(&igb->gen_lock));
2813 	ASSERT(igb->mcast_count <= igb->mcast_max_num);
2814 
2815 	mc_addr_list = (uint8_t *)igb->mcast_table;
2816 	mc_addr_count = igb->mcast_count;
2817 
2818 	/*
2819 	 * Update the multicase addresses to the MTA registers
2820 	 */
2821 	e1000_update_mc_addr_list(hw, mc_addr_list, mc_addr_count);
2822 }
2823 
2824 /*
2825  * igb_get_conf - Get driver configurations set in driver.conf
2826  *
2827  * This routine gets user-configured values out of the configuration
2828  * file igb.conf.
2829  *
2830  * For each configurable value, there is a minimum, a maximum, and a
2831  * default.
2832  * If user does not configure a value, use the default.
2833  * If user configures below the minimum, use the minumum.
2834  * If user configures above the maximum, use the maxumum.
2835  */
2836 static void
igb_get_conf(igb_t * igb)2837 igb_get_conf(igb_t *igb)
2838 {
2839 	struct e1000_hw *hw = &igb->hw;
2840 	uint32_t default_mtu;
2841 	uint32_t flow_control;
2842 	uint32_t ring_per_group;
2843 	int i;
2844 
2845 	/*
2846 	 * igb driver supports the following user configurations:
2847 	 *
2848 	 * Link configurations:
2849 	 *    adv_autoneg_cap
2850 	 *    adv_1000fdx_cap
2851 	 *    adv_100fdx_cap
2852 	 *    adv_100hdx_cap
2853 	 *    adv_10fdx_cap
2854 	 *    adv_10hdx_cap
2855 	 * Note: 1000hdx is not supported.
2856 	 *
2857 	 * Jumbo frame configuration:
2858 	 *    default_mtu
2859 	 *
2860 	 * Ethernet flow control configuration:
2861 	 *    flow_control
2862 	 *
2863 	 * Multiple rings configurations:
2864 	 *    tx_queue_number
2865 	 *    tx_ring_size
2866 	 *    rx_queue_number
2867 	 *    rx_ring_size
2868 	 *
2869 	 * Call igb_get_prop() to get the value for a specific
2870 	 * configuration parameter.
2871 	 */
2872 
2873 	/*
2874 	 * Link configurations
2875 	 */
2876 	igb->param_adv_autoneg_cap = igb_get_prop(igb,
2877 	    PROP_ADV_AUTONEG_CAP, 0, 1, 1);
2878 	igb->param_adv_1000fdx_cap = igb_get_prop(igb,
2879 	    PROP_ADV_1000FDX_CAP, 0, 1, 1);
2880 	igb->param_adv_100fdx_cap = igb_get_prop(igb,
2881 	    PROP_ADV_100FDX_CAP, 0, 1, 1);
2882 	igb->param_adv_100hdx_cap = igb_get_prop(igb,
2883 	    PROP_ADV_100HDX_CAP, 0, 1, 1);
2884 	igb->param_adv_10fdx_cap = igb_get_prop(igb,
2885 	    PROP_ADV_10FDX_CAP, 0, 1, 1);
2886 	igb->param_adv_10hdx_cap = igb_get_prop(igb,
2887 	    PROP_ADV_10HDX_CAP, 0, 1, 1);
2888 
2889 	/*
2890 	 * Jumbo frame configurations
2891 	 */
2892 	default_mtu = igb_get_prop(igb, PROP_DEFAULT_MTU,
2893 	    MIN_MTU, MAX_MTU, DEFAULT_MTU);
2894 
2895 	igb->max_frame_size = default_mtu +
2896 	    sizeof (struct ether_vlan_header) + ETHERFCSL;
2897 
2898 	/*
2899 	 * Ethernet flow control configuration
2900 	 */
2901 	flow_control = igb_get_prop(igb, PROP_FLOW_CONTROL,
2902 	    e1000_fc_none, 4, e1000_fc_full);
2903 	if (flow_control == 4)
2904 		flow_control = e1000_fc_default;
2905 
2906 	hw->fc.requested_mode = flow_control;
2907 
2908 	/*
2909 	 * Multiple rings configurations
2910 	 */
2911 	igb->tx_ring_size = igb_get_prop(igb, PROP_TX_RING_SIZE,
2912 	    MIN_TX_RING_SIZE, MAX_TX_RING_SIZE, DEFAULT_TX_RING_SIZE);
2913 	igb->rx_ring_size = igb_get_prop(igb, PROP_RX_RING_SIZE,
2914 	    MIN_RX_RING_SIZE, MAX_RX_RING_SIZE, DEFAULT_RX_RING_SIZE);
2915 
2916 	igb->mr_enable = igb_get_prop(igb, PROP_MR_ENABLE, 0, 1, 0);
2917 	igb->num_rx_groups = igb_get_prop(igb, PROP_RX_GROUP_NUM,
2918 	    MIN_RX_GROUP_NUM, MAX_RX_GROUP_NUM, DEFAULT_RX_GROUP_NUM);
2919 	/*
2920 	 * Currently we do not support VMDq for 82576 and 82580.
2921 	 * If it is e1000_82576, set num_rx_groups to 1.
2922 	 */
2923 	if (hw->mac.type >= e1000_82576)
2924 		igb->num_rx_groups = 1;
2925 
2926 	if (igb->mr_enable) {
2927 		igb->num_tx_rings = igb->capab->def_tx_que_num;
2928 		igb->num_rx_rings = igb->capab->def_rx_que_num;
2929 	} else {
2930 		igb->num_tx_rings = 1;
2931 		igb->num_rx_rings = 1;
2932 
2933 		if (igb->num_rx_groups > 1) {
2934 			igb_error(igb,
2935 			    "Invalid rx groups number. Please enable multiple "
2936 			    "rings first");
2937 			igb->num_rx_groups = 1;
2938 		}
2939 	}
2940 
2941 	/*
2942 	 * Check the divisibility between rx rings and rx groups.
2943 	 */
2944 	for (i = igb->num_rx_groups; i > 0; i--) {
2945 		if ((igb->num_rx_rings % i) == 0)
2946 			break;
2947 	}
2948 	if (i != igb->num_rx_groups) {
2949 		igb_error(igb,
2950 		    "Invalid rx groups number. Downgrade the rx group "
2951 		    "number to %d.", i);
2952 		igb->num_rx_groups = i;
2953 	}
2954 
2955 	/*
2956 	 * Get the ring number per group.
2957 	 */
2958 	ring_per_group = igb->num_rx_rings / igb->num_rx_groups;
2959 
2960 	if (igb->num_rx_groups == 1) {
2961 		/*
2962 		 * One rx ring group, the rx ring number is num_rx_rings.
2963 		 */
2964 		igb->vmdq_mode = E1000_VMDQ_OFF;
2965 	} else if (ring_per_group == 1) {
2966 		/*
2967 		 * Multiple rx groups, each group has one rx ring.
2968 		 */
2969 		igb->vmdq_mode = E1000_VMDQ_MAC;
2970 	} else {
2971 		/*
2972 		 * Multiple groups and multiple rings.
2973 		 */
2974 		igb->vmdq_mode = E1000_VMDQ_MAC_RSS;
2975 	}
2976 
2977 	/*
2978 	 * Tunable used to force an interrupt type. The only use is
2979 	 * for testing of the lesser interrupt types.
2980 	 * 0 = don't force interrupt type
2981 	 * 1 = force interrupt type MSIX
2982 	 * 2 = force interrupt type MSI
2983 	 * 3 = force interrupt type Legacy
2984 	 */
2985 	igb->intr_force = igb_get_prop(igb, PROP_INTR_FORCE,
2986 	    IGB_INTR_NONE, IGB_INTR_LEGACY, IGB_INTR_NONE);
2987 
2988 	igb->tx_hcksum_enable = igb_get_prop(igb, PROP_TX_HCKSUM_ENABLE,
2989 	    0, 1, 1);
2990 	igb->rx_hcksum_enable = igb_get_prop(igb, PROP_RX_HCKSUM_ENABLE,
2991 	    0, 1, 1);
2992 	igb->lso_enable = igb_get_prop(igb, PROP_LSO_ENABLE,
2993 	    0, 1, 1);
2994 	igb->tx_head_wb_enable = igb_get_prop(igb, PROP_TX_HEAD_WB_ENABLE,
2995 	    0, 1, 1);
2996 
2997 	/*
2998 	 * igb LSO needs the tx h/w checksum support.
2999 	 * Here LSO will be disabled if tx h/w checksum has been disabled.
3000 	 */
3001 	if (igb->tx_hcksum_enable == B_FALSE)
3002 		igb->lso_enable = B_FALSE;
3003 
3004 	igb->tx_copy_thresh = igb_get_prop(igb, PROP_TX_COPY_THRESHOLD,
3005 	    MIN_TX_COPY_THRESHOLD, MAX_TX_COPY_THRESHOLD,
3006 	    DEFAULT_TX_COPY_THRESHOLD);
3007 	igb->tx_recycle_thresh = igb_get_prop(igb, PROP_TX_RECYCLE_THRESHOLD,
3008 	    MIN_TX_RECYCLE_THRESHOLD, MAX_TX_RECYCLE_THRESHOLD,
3009 	    DEFAULT_TX_RECYCLE_THRESHOLD);
3010 	igb->tx_overload_thresh = igb_get_prop(igb, PROP_TX_OVERLOAD_THRESHOLD,
3011 	    MIN_TX_OVERLOAD_THRESHOLD, MAX_TX_OVERLOAD_THRESHOLD,
3012 	    DEFAULT_TX_OVERLOAD_THRESHOLD);
3013 	igb->tx_resched_thresh = igb_get_prop(igb, PROP_TX_RESCHED_THRESHOLD,
3014 	    MIN_TX_RESCHED_THRESHOLD,
3015 	    MIN(igb->tx_ring_size, MAX_TX_RESCHED_THRESHOLD),
3016 	    igb->tx_ring_size > DEFAULT_TX_RESCHED_THRESHOLD ?
3017 	    DEFAULT_TX_RESCHED_THRESHOLD : DEFAULT_TX_RESCHED_THRESHOLD_LOW);
3018 
3019 	igb->rx_copy_thresh = igb_get_prop(igb, PROP_RX_COPY_THRESHOLD,
3020 	    MIN_RX_COPY_THRESHOLD, MAX_RX_COPY_THRESHOLD,
3021 	    DEFAULT_RX_COPY_THRESHOLD);
3022 	igb->rx_limit_per_intr = igb_get_prop(igb, PROP_RX_LIMIT_PER_INTR,
3023 	    MIN_RX_LIMIT_PER_INTR, MAX_RX_LIMIT_PER_INTR,
3024 	    DEFAULT_RX_LIMIT_PER_INTR);
3025 
3026 	igb->intr_throttling[0] = igb_get_prop(igb, PROP_INTR_THROTTLING,
3027 	    igb->capab->min_intr_throttle,
3028 	    igb->capab->max_intr_throttle,
3029 	    igb->capab->def_intr_throttle);
3030 
3031 	/*
3032 	 * Max number of multicast addresses
3033 	 */
3034 	igb->mcast_max_num =
3035 	    igb_get_prop(igb, PROP_MCAST_MAX_NUM,
3036 	    MIN_MCAST_NUM, MAX_MCAST_NUM, DEFAULT_MCAST_NUM);
3037 }
3038 
3039 /*
3040  * igb_get_prop - Get a property value out of the configuration file igb.conf
3041  *
3042  * Caller provides the name of the property, a default value, a minimum
3043  * value, and a maximum value.
3044  *
3045  * Return configured value of the property, with default, minimum and
3046  * maximum properly applied.
3047  */
3048 static int
igb_get_prop(igb_t * igb,char * propname,int minval,int maxval,int defval)3049 igb_get_prop(igb_t *igb,
3050     char *propname,	/* name of the property */
3051     int minval,		/* minimum acceptable value */
3052     int maxval,		/* maximim acceptable value */
3053     int defval)		/* default value */
3054 {
3055 	int value;
3056 
3057 	/*
3058 	 * Call ddi_prop_get_int() to read the conf settings
3059 	 */
3060 	value = ddi_prop_get_int(DDI_DEV_T_ANY, igb->dip,
3061 	    DDI_PROP_DONTPASS, propname, defval);
3062 
3063 	if (value > maxval)
3064 		value = maxval;
3065 
3066 	if (value < minval)
3067 		value = minval;
3068 
3069 	return (value);
3070 }
3071 
3072 /*
3073  * igb_setup_link - Using the link properties to setup the link
3074  */
3075 int
igb_setup_link(igb_t * igb,boolean_t setup_hw)3076 igb_setup_link(igb_t *igb, boolean_t setup_hw)
3077 {
3078 	struct e1000_mac_info *mac;
3079 	struct e1000_phy_info *phy;
3080 	boolean_t invalid;
3081 
3082 	mac = &igb->hw.mac;
3083 	phy = &igb->hw.phy;
3084 	invalid = B_FALSE;
3085 
3086 	if (igb->param_adv_autoneg_cap == 1) {
3087 		mac->autoneg = B_TRUE;
3088 		phy->autoneg_advertised = 0;
3089 
3090 		/*
3091 		 * 1000hdx is not supported for autonegotiation
3092 		 */
3093 		if (igb->param_adv_1000fdx_cap == 1)
3094 			phy->autoneg_advertised |= ADVERTISE_1000_FULL;
3095 
3096 		if (igb->param_adv_100fdx_cap == 1)
3097 			phy->autoneg_advertised |= ADVERTISE_100_FULL;
3098 
3099 		if (igb->param_adv_100hdx_cap == 1)
3100 			phy->autoneg_advertised |= ADVERTISE_100_HALF;
3101 
3102 		if (igb->param_adv_10fdx_cap == 1)
3103 			phy->autoneg_advertised |= ADVERTISE_10_FULL;
3104 
3105 		if (igb->param_adv_10hdx_cap == 1)
3106 			phy->autoneg_advertised |= ADVERTISE_10_HALF;
3107 
3108 		if (phy->autoneg_advertised == 0)
3109 			invalid = B_TRUE;
3110 	} else {
3111 		mac->autoneg = B_FALSE;
3112 
3113 		/*
3114 		 * 1000fdx and 1000hdx are not supported for forced link
3115 		 */
3116 		if (igb->param_adv_100fdx_cap == 1)
3117 			mac->forced_speed_duplex = ADVERTISE_100_FULL;
3118 		else if (igb->param_adv_100hdx_cap == 1)
3119 			mac->forced_speed_duplex = ADVERTISE_100_HALF;
3120 		else if (igb->param_adv_10fdx_cap == 1)
3121 			mac->forced_speed_duplex = ADVERTISE_10_FULL;
3122 		else if (igb->param_adv_10hdx_cap == 1)
3123 			mac->forced_speed_duplex = ADVERTISE_10_HALF;
3124 		else
3125 			invalid = B_TRUE;
3126 	}
3127 
3128 	if (invalid) {
3129 		igb_notice(igb, "Invalid link settings. Setup link to "
3130 		    "autonegotiation with full link capabilities.");
3131 		mac->autoneg = B_TRUE;
3132 		phy->autoneg_advertised = ADVERTISE_1000_FULL |
3133 		    ADVERTISE_100_FULL | ADVERTISE_100_HALF |
3134 		    ADVERTISE_10_FULL | ADVERTISE_10_HALF;
3135 	}
3136 
3137 	if (setup_hw) {
3138 		if (e1000_setup_link(&igb->hw) != E1000_SUCCESS)
3139 			return (IGB_FAILURE);
3140 	}
3141 
3142 	return (IGB_SUCCESS);
3143 }
3144 
3145 
3146 /*
3147  * igb_is_link_up - Check if the link is up
3148  */
3149 static boolean_t
igb_is_link_up(igb_t * igb)3150 igb_is_link_up(igb_t *igb)
3151 {
3152 	struct e1000_hw *hw = &igb->hw;
3153 	boolean_t link_up = B_FALSE;
3154 
3155 	ASSERT(mutex_owned(&igb->gen_lock));
3156 
3157 	/*
3158 	 * get_link_status is set in the interrupt handler on link-status-change
3159 	 * or rx sequence error interrupt.  get_link_status will stay
3160 	 * false until the e1000_check_for_link establishes link only
3161 	 * for copper adapters.
3162 	 */
3163 	switch (hw->phy.media_type) {
3164 	case e1000_media_type_copper:
3165 		if (hw->mac.get_link_status) {
3166 			(void) e1000_check_for_link(hw);
3167 			link_up = !hw->mac.get_link_status;
3168 		} else {
3169 			link_up = B_TRUE;
3170 		}
3171 		break;
3172 	case e1000_media_type_fiber:
3173 		(void) e1000_check_for_link(hw);
3174 		link_up = (E1000_READ_REG(hw, E1000_STATUS) & E1000_STATUS_LU);
3175 		break;
3176 	case e1000_media_type_internal_serdes:
3177 		(void) e1000_check_for_link(hw);
3178 		link_up = hw->mac.serdes_has_link;
3179 		break;
3180 	}
3181 
3182 	return (link_up);
3183 }
3184 
3185 /*
3186  * igb_link_check - Link status processing
3187  */
3188 static boolean_t
igb_link_check(igb_t * igb)3189 igb_link_check(igb_t *igb)
3190 {
3191 	struct e1000_hw *hw = &igb->hw;
3192 	uint16_t speed = 0, duplex = 0;
3193 	boolean_t link_changed = B_FALSE;
3194 
3195 	ASSERT(mutex_owned(&igb->gen_lock));
3196 
3197 	if (igb_is_link_up(igb)) {
3198 		/*
3199 		 * The Link is up, check whether it was marked as down earlier
3200 		 */
3201 		if (igb->link_state != LINK_STATE_UP) {
3202 			(void) e1000_get_speed_and_duplex(hw, &speed, &duplex);
3203 			igb->link_speed = speed;
3204 			igb->link_duplex = duplex;
3205 			igb->link_state = LINK_STATE_UP;
3206 			link_changed = B_TRUE;
3207 			if (!igb->link_complete)
3208 				igb_stop_link_timer(igb);
3209 		}
3210 	} else if (igb->link_complete) {
3211 		if (igb->link_state != LINK_STATE_DOWN) {
3212 			igb->link_speed = 0;
3213 			igb->link_duplex = 0;
3214 			igb->link_state = LINK_STATE_DOWN;
3215 			link_changed = B_TRUE;
3216 		}
3217 	}
3218 
3219 	if (igb_check_acc_handle(igb->osdep.reg_handle) != DDI_FM_OK) {
3220 		ddi_fm_service_impact(igb->dip, DDI_SERVICE_DEGRADED);
3221 		return (B_FALSE);
3222 	}
3223 
3224 	return (link_changed);
3225 }
3226 
3227 /*
3228  * igb_local_timer - driver watchdog function
3229  *
3230  * This function will handle the hardware stall check, link status
3231  * check and other routines.
3232  */
3233 static void
igb_local_timer(void * arg)3234 igb_local_timer(void *arg)
3235 {
3236 	igb_t *igb = (igb_t *)arg;
3237 	boolean_t link_changed = B_FALSE;
3238 
3239 	if (igb->igb_state & IGB_ERROR) {
3240 		igb->reset_count++;
3241 		if (igb_reset(igb) == IGB_SUCCESS)
3242 			ddi_fm_service_impact(igb->dip, DDI_SERVICE_RESTORED);
3243 
3244 		igb_restart_watchdog_timer(igb);
3245 		return;
3246 	}
3247 
3248 	if (igb_stall_check(igb) || (igb->igb_state & IGB_STALL)) {
3249 		igb_fm_ereport(igb, DDI_FM_DEVICE_STALL);
3250 		ddi_fm_service_impact(igb->dip, DDI_SERVICE_LOST);
3251 		igb->reset_count++;
3252 		if (igb_reset(igb) == IGB_SUCCESS)
3253 			ddi_fm_service_impact(igb->dip, DDI_SERVICE_RESTORED);
3254 
3255 		igb_restart_watchdog_timer(igb);
3256 		return;
3257 	}
3258 
3259 	mutex_enter(&igb->gen_lock);
3260 	if (!(igb->igb_state & IGB_SUSPENDED) && (igb->igb_state & IGB_STARTED))
3261 		link_changed = igb_link_check(igb);
3262 	mutex_exit(&igb->gen_lock);
3263 
3264 	if (link_changed)
3265 		mac_link_update(igb->mac_hdl, igb->link_state);
3266 
3267 	igb_restart_watchdog_timer(igb);
3268 }
3269 
3270 /*
3271  * igb_link_timer - link setup timer function
3272  *
3273  * It is called when the timer for link setup is expired, which indicates
3274  * the completion of the link setup. The link state will not be updated
3275  * until the link setup is completed. And the link state will not be sent
3276  * to the upper layer through mac_link_update() in this function. It will
3277  * be updated in the local timer routine or the interrupts service routine
3278  * after the interface is started (plumbed).
3279  */
3280 static void
igb_link_timer(void * arg)3281 igb_link_timer(void *arg)
3282 {
3283 	igb_t *igb = (igb_t *)arg;
3284 
3285 	mutex_enter(&igb->link_lock);
3286 	igb->link_complete = B_TRUE;
3287 	igb->link_tid = 0;
3288 	mutex_exit(&igb->link_lock);
3289 }
3290 /*
3291  * igb_stall_check - check for transmit stall
3292  *
3293  * This function checks if the adapter is stalled (in transmit).
3294  *
3295  * It is called each time the watchdog timeout is invoked.
3296  * If the transmit descriptor reclaim continuously fails,
3297  * the watchdog value will increment by 1. If the watchdog
3298  * value exceeds the threshold, the igb is assumed to
3299  * have stalled and need to be reset.
3300  */
3301 static boolean_t
igb_stall_check(igb_t * igb)3302 igb_stall_check(igb_t *igb)
3303 {
3304 	igb_tx_ring_t *tx_ring;
3305 	struct e1000_hw *hw = &igb->hw;
3306 	boolean_t result;
3307 	int i;
3308 
3309 	if (igb->link_state != LINK_STATE_UP)
3310 		return (B_FALSE);
3311 
3312 	/*
3313 	 * If any tx ring is stalled, we'll reset the chipset
3314 	 */
3315 	result = B_FALSE;
3316 	for (i = 0; i < igb->num_tx_rings; i++) {
3317 		tx_ring = &igb->tx_rings[i];
3318 
3319 		if (tx_ring->recycle_fail > 0)
3320 			tx_ring->stall_watchdog++;
3321 		else
3322 			tx_ring->stall_watchdog = 0;
3323 
3324 		if (tx_ring->stall_watchdog >= STALL_WATCHDOG_TIMEOUT) {
3325 			result = B_TRUE;
3326 			if (hw->mac.type == e1000_82580) {
3327 				hw->dev_spec._82575.global_device_reset
3328 				    = B_TRUE;
3329 			}
3330 			break;
3331 		}
3332 	}
3333 
3334 	if (result) {
3335 		tx_ring->stall_watchdog = 0;
3336 		tx_ring->recycle_fail = 0;
3337 	}
3338 
3339 	return (result);
3340 }
3341 
3342 
3343 /*
3344  * is_valid_mac_addr - Check if the mac address is valid
3345  */
3346 static boolean_t
is_valid_mac_addr(uint8_t * mac_addr)3347 is_valid_mac_addr(uint8_t *mac_addr)
3348 {
3349 	const uint8_t addr_test1[6] = { 0, 0, 0, 0, 0, 0 };
3350 	const uint8_t addr_test2[6] =
3351 	    { 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF };
3352 
3353 	if (!(bcmp(addr_test1, mac_addr, ETHERADDRL)) ||
3354 	    !(bcmp(addr_test2, mac_addr, ETHERADDRL)))
3355 		return (B_FALSE);
3356 
3357 	return (B_TRUE);
3358 }
3359 
3360 static boolean_t
igb_find_mac_address(igb_t * igb)3361 igb_find_mac_address(igb_t *igb)
3362 {
3363 	struct e1000_hw *hw = &igb->hw;
3364 #ifdef __sparc
3365 	uchar_t *bytes;
3366 	struct ether_addr sysaddr;
3367 	uint_t nelts;
3368 	int err;
3369 	boolean_t found = B_FALSE;
3370 
3371 	/*
3372 	 * The "vendor's factory-set address" may already have
3373 	 * been extracted from the chip, but if the property
3374 	 * "local-mac-address" is set we use that instead.
3375 	 *
3376 	 * We check whether it looks like an array of 6
3377 	 * bytes (which it should, if OBP set it).  If we can't
3378 	 * make sense of it this way, we'll ignore it.
3379 	 */
3380 	err = ddi_prop_lookup_byte_array(DDI_DEV_T_ANY, igb->dip,
3381 	    DDI_PROP_DONTPASS, "local-mac-address", &bytes, &nelts);
3382 	if (err == DDI_PROP_SUCCESS) {
3383 		if (nelts == ETHERADDRL) {
3384 			while (nelts--)
3385 				hw->mac.addr[nelts] = bytes[nelts];
3386 			found = B_TRUE;
3387 		}
3388 		ddi_prop_free(bytes);
3389 	}
3390 
3391 	/*
3392 	 * Look up the OBP property "local-mac-address?". If the user has set
3393 	 * 'local-mac-address? = false', use "the system address" instead.
3394 	 */
3395 	if (ddi_prop_lookup_byte_array(DDI_DEV_T_ANY, igb->dip, 0,
3396 	    "local-mac-address?", &bytes, &nelts) == DDI_PROP_SUCCESS) {
3397 		if (strncmp("false", (caddr_t)bytes, (size_t)nelts) == 0) {
3398 			if (localetheraddr(NULL, &sysaddr) != 0) {
3399 				bcopy(&sysaddr, hw->mac.addr, ETHERADDRL);
3400 				found = B_TRUE;
3401 			}
3402 		}
3403 		ddi_prop_free(bytes);
3404 	}
3405 
3406 	/*
3407 	 * Finally(!), if there's a valid "mac-address" property (created
3408 	 * if we netbooted from this interface), we must use this instead
3409 	 * of any of the above to ensure that the NFS/install server doesn't
3410 	 * get confused by the address changing as Solaris takes over!
3411 	 */
3412 	err = ddi_prop_lookup_byte_array(DDI_DEV_T_ANY, igb->dip,
3413 	    DDI_PROP_DONTPASS, "mac-address", &bytes, &nelts);
3414 	if (err == DDI_PROP_SUCCESS) {
3415 		if (nelts == ETHERADDRL) {
3416 			while (nelts--)
3417 				hw->mac.addr[nelts] = bytes[nelts];
3418 			found = B_TRUE;
3419 		}
3420 		ddi_prop_free(bytes);
3421 	}
3422 
3423 	if (found) {
3424 		bcopy(hw->mac.addr, hw->mac.perm_addr, ETHERADDRL);
3425 		return (B_TRUE);
3426 	}
3427 #endif
3428 
3429 	/*
3430 	 * Read the device MAC address from the EEPROM
3431 	 */
3432 	if (e1000_read_mac_addr(hw) != E1000_SUCCESS)
3433 		return (B_FALSE);
3434 
3435 	return (B_TRUE);
3436 }
3437 
3438 #pragma inline(igb_arm_watchdog_timer)
3439 
3440 static void
igb_arm_watchdog_timer(igb_t * igb)3441 igb_arm_watchdog_timer(igb_t *igb)
3442 {
3443 	/*
3444 	 * Fire a watchdog timer
3445 	 */
3446 	igb->watchdog_tid =
3447 	    timeout(igb_local_timer,
3448 	    (void *)igb, 1 * drv_usectohz(1000000));
3449 
3450 }
3451 
3452 /*
3453  * igb_enable_watchdog_timer - Enable and start the driver watchdog timer
3454  */
3455 void
igb_enable_watchdog_timer(igb_t * igb)3456 igb_enable_watchdog_timer(igb_t *igb)
3457 {
3458 	mutex_enter(&igb->watchdog_lock);
3459 
3460 	if (!igb->watchdog_enable) {
3461 		igb->watchdog_enable = B_TRUE;
3462 		igb->watchdog_start = B_TRUE;
3463 		igb_arm_watchdog_timer(igb);
3464 	}
3465 
3466 	mutex_exit(&igb->watchdog_lock);
3467 
3468 }
3469 
3470 /*
3471  * igb_disable_watchdog_timer - Disable and stop the driver watchdog timer
3472  */
3473 void
igb_disable_watchdog_timer(igb_t * igb)3474 igb_disable_watchdog_timer(igb_t *igb)
3475 {
3476 	timeout_id_t tid;
3477 
3478 	mutex_enter(&igb->watchdog_lock);
3479 
3480 	igb->watchdog_enable = B_FALSE;
3481 	igb->watchdog_start = B_FALSE;
3482 	tid = igb->watchdog_tid;
3483 	igb->watchdog_tid = 0;
3484 
3485 	mutex_exit(&igb->watchdog_lock);
3486 
3487 	if (tid != 0)
3488 		(void) untimeout(tid);
3489 
3490 }
3491 
3492 /*
3493  * igb_start_watchdog_timer - Start the driver watchdog timer
3494  */
3495 static void
igb_start_watchdog_timer(igb_t * igb)3496 igb_start_watchdog_timer(igb_t *igb)
3497 {
3498 	mutex_enter(&igb->watchdog_lock);
3499 
3500 	if (igb->watchdog_enable) {
3501 		if (!igb->watchdog_start) {
3502 			igb->watchdog_start = B_TRUE;
3503 			igb_arm_watchdog_timer(igb);
3504 		}
3505 	}
3506 
3507 	mutex_exit(&igb->watchdog_lock);
3508 }
3509 
3510 /*
3511  * igb_restart_watchdog_timer - Restart the driver watchdog timer
3512  */
3513 static void
igb_restart_watchdog_timer(igb_t * igb)3514 igb_restart_watchdog_timer(igb_t *igb)
3515 {
3516 	mutex_enter(&igb->watchdog_lock);
3517 
3518 	if (igb->watchdog_start)
3519 		igb_arm_watchdog_timer(igb);
3520 
3521 	mutex_exit(&igb->watchdog_lock);
3522 }
3523 
3524 /*
3525  * igb_stop_watchdog_timer - Stop the driver watchdog timer
3526  */
3527 static void
igb_stop_watchdog_timer(igb_t * igb)3528 igb_stop_watchdog_timer(igb_t *igb)
3529 {
3530 	timeout_id_t tid;
3531 
3532 	mutex_enter(&igb->watchdog_lock);
3533 
3534 	igb->watchdog_start = B_FALSE;
3535 	tid = igb->watchdog_tid;
3536 	igb->watchdog_tid = 0;
3537 
3538 	mutex_exit(&igb->watchdog_lock);
3539 
3540 	if (tid != 0)
3541 		(void) untimeout(tid);
3542 }
3543 
3544 /*
3545  * igb_start_link_timer - Start the link setup timer
3546  */
3547 static void
igb_start_link_timer(struct igb * igb)3548 igb_start_link_timer(struct igb *igb)
3549 {
3550 	struct e1000_hw *hw = &igb->hw;
3551 	clock_t link_timeout;
3552 
3553 	if (hw->mac.autoneg)
3554 		link_timeout = PHY_AUTO_NEG_LIMIT *
3555 		    drv_usectohz(100000);
3556 	else
3557 		link_timeout = PHY_FORCE_LIMIT * drv_usectohz(100000);
3558 
3559 	mutex_enter(&igb->link_lock);
3560 	if (hw->phy.autoneg_wait_to_complete) {
3561 		igb->link_complete = B_TRUE;
3562 	} else {
3563 		igb->link_complete = B_FALSE;
3564 		igb->link_tid = timeout(igb_link_timer, (void *)igb,
3565 		    link_timeout);
3566 	}
3567 	mutex_exit(&igb->link_lock);
3568 }
3569 
3570 /*
3571  * igb_stop_link_timer - Stop the link setup timer
3572  */
3573 static void
igb_stop_link_timer(struct igb * igb)3574 igb_stop_link_timer(struct igb *igb)
3575 {
3576 	timeout_id_t tid;
3577 
3578 	mutex_enter(&igb->link_lock);
3579 	igb->link_complete = B_TRUE;
3580 	tid = igb->link_tid;
3581 	igb->link_tid = 0;
3582 	mutex_exit(&igb->link_lock);
3583 
3584 	if (tid != 0)
3585 		(void) untimeout(tid);
3586 }
3587 
3588 /*
3589  * igb_disable_adapter_interrupts - Clear/disable all hardware interrupts
3590  */
3591 static void
igb_disable_adapter_interrupts(igb_t * igb)3592 igb_disable_adapter_interrupts(igb_t *igb)
3593 {
3594 	struct e1000_hw *hw = &igb->hw;
3595 
3596 	/*
3597 	 * Set the IMC register to mask all the interrupts,
3598 	 * including the tx interrupts.
3599 	 */
3600 	E1000_WRITE_REG(hw, E1000_IMC, ~0);
3601 	E1000_WRITE_REG(hw, E1000_IAM, 0);
3602 
3603 	/*
3604 	 * Additional disabling for MSI-X
3605 	 */
3606 	if (igb->intr_type == DDI_INTR_TYPE_MSIX) {
3607 		E1000_WRITE_REG(hw, E1000_EIMC, ~0);
3608 		E1000_WRITE_REG(hw, E1000_EIAC, 0);
3609 		E1000_WRITE_REG(hw, E1000_EIAM, 0);
3610 	}
3611 
3612 	E1000_WRITE_FLUSH(hw);
3613 }
3614 
3615 /*
3616  * igb_enable_adapter_interrupts_82580 - Enable NIC interrupts for 82580
3617  */
3618 static void
igb_enable_adapter_interrupts_82580(igb_t * igb)3619 igb_enable_adapter_interrupts_82580(igb_t *igb)
3620 {
3621 	struct e1000_hw *hw = &igb->hw;
3622 
3623 	/* Clear any pending interrupts */
3624 	(void) E1000_READ_REG(hw, E1000_ICR);
3625 	igb->ims_mask |= E1000_IMS_DRSTA;
3626 
3627 	if (igb->intr_type == DDI_INTR_TYPE_MSIX) {
3628 
3629 		/* Interrupt enabling for MSI-X */
3630 		E1000_WRITE_REG(hw, E1000_EIMS, igb->eims_mask);
3631 		E1000_WRITE_REG(hw, E1000_EIAC, igb->eims_mask);
3632 		igb->ims_mask = (E1000_IMS_LSC | E1000_IMS_DRSTA);
3633 		E1000_WRITE_REG(hw, E1000_IMS, igb->ims_mask);
3634 	} else { /* Interrupt enabling for MSI and legacy */
3635 		E1000_WRITE_REG(hw, E1000_IVAR0, E1000_IVAR_VALID);
3636 		igb->ims_mask = IMS_ENABLE_MASK | E1000_IMS_TXQE;
3637 		igb->ims_mask |= E1000_IMS_DRSTA;
3638 		E1000_WRITE_REG(hw, E1000_IMS, igb->ims_mask);
3639 	}
3640 
3641 	/* Disable auto-mask for ICR interrupt bits */
3642 	E1000_WRITE_REG(hw, E1000_IAM, 0);
3643 
3644 	E1000_WRITE_FLUSH(hw);
3645 }
3646 
3647 /*
3648  * igb_enable_adapter_interrupts_82576 - Enable NIC interrupts for 82576
3649  */
3650 static void
igb_enable_adapter_interrupts_82576(igb_t * igb)3651 igb_enable_adapter_interrupts_82576(igb_t *igb)
3652 {
3653 	struct e1000_hw *hw = &igb->hw;
3654 
3655 	/* Clear any pending interrupts */
3656 	(void) E1000_READ_REG(hw, E1000_ICR);
3657 
3658 	if (igb->intr_type == DDI_INTR_TYPE_MSIX) {
3659 
3660 		/* Interrupt enabling for MSI-X */
3661 		E1000_WRITE_REG(hw, E1000_EIMS, igb->eims_mask);
3662 		E1000_WRITE_REG(hw, E1000_EIAC, igb->eims_mask);
3663 		igb->ims_mask = E1000_IMS_LSC;
3664 		E1000_WRITE_REG(hw, E1000_IMS, E1000_IMS_LSC);
3665 	} else {
3666 		/* Interrupt enabling for MSI and legacy */
3667 		E1000_WRITE_REG(hw, E1000_IVAR0, E1000_IVAR_VALID);
3668 		igb->ims_mask = IMS_ENABLE_MASK | E1000_IMS_TXQE;
3669 		E1000_WRITE_REG(hw, E1000_IMS,
3670 		    (IMS_ENABLE_MASK | E1000_IMS_TXQE));
3671 	}
3672 
3673 	/* Disable auto-mask for ICR interrupt bits */
3674 	E1000_WRITE_REG(hw, E1000_IAM, 0);
3675 
3676 	E1000_WRITE_FLUSH(hw);
3677 }
3678 
3679 /*
3680  * igb_enable_adapter_interrupts_82575 - Enable NIC interrupts for 82575
3681  */
3682 static void
igb_enable_adapter_interrupts_82575(igb_t * igb)3683 igb_enable_adapter_interrupts_82575(igb_t *igb)
3684 {
3685 	struct e1000_hw *hw = &igb->hw;
3686 	uint32_t reg;
3687 
3688 	/* Clear any pending interrupts */
3689 	(void) E1000_READ_REG(hw, E1000_ICR);
3690 
3691 	if (igb->intr_type == DDI_INTR_TYPE_MSIX) {
3692 		/* Interrupt enabling for MSI-X */
3693 		E1000_WRITE_REG(hw, E1000_EIMS, igb->eims_mask);
3694 		E1000_WRITE_REG(hw, E1000_EIAC, igb->eims_mask);
3695 		igb->ims_mask = E1000_IMS_LSC;
3696 		E1000_WRITE_REG(hw, E1000_IMS, E1000_IMS_LSC);
3697 
3698 		/* Enable MSI-X PBA support */
3699 		reg = E1000_READ_REG(hw, E1000_CTRL_EXT);
3700 		reg |= E1000_CTRL_EXT_PBA_CLR;
3701 
3702 		/* Non-selective interrupt clear-on-read */
3703 		reg |= E1000_CTRL_EXT_IRCA;	/* Called NSICR in the EAS */
3704 
3705 		E1000_WRITE_REG(hw, E1000_CTRL_EXT, reg);
3706 	} else {
3707 		/* Interrupt enabling for MSI and legacy */
3708 		igb->ims_mask = IMS_ENABLE_MASK;
3709 		E1000_WRITE_REG(hw, E1000_IMS, IMS_ENABLE_MASK);
3710 	}
3711 
3712 	E1000_WRITE_FLUSH(hw);
3713 }
3714 
3715 /*
3716  * Loopback Support
3717  */
3718 static lb_property_t lb_normal =
3719 	{ normal,	"normal",	IGB_LB_NONE		};
3720 static lb_property_t lb_external =
3721 	{ external,	"External",	IGB_LB_EXTERNAL		};
3722 static lb_property_t lb_phy =
3723 	{ internal,	"PHY",		IGB_LB_INTERNAL_PHY	};
3724 static lb_property_t lb_serdes =
3725 	{ internal,	"SerDes",	IGB_LB_INTERNAL_SERDES	};
3726 
3727 enum ioc_reply
igb_loopback_ioctl(igb_t * igb,struct iocblk * iocp,mblk_t * mp)3728 igb_loopback_ioctl(igb_t *igb, struct iocblk *iocp, mblk_t *mp)
3729 {
3730 	lb_info_sz_t *lbsp;
3731 	lb_property_t *lbpp;
3732 	struct e1000_hw *hw;
3733 	uint32_t *lbmp;
3734 	uint32_t size;
3735 	uint32_t value;
3736 
3737 	hw = &igb->hw;
3738 
3739 	if (mp->b_cont == NULL)
3740 		return (IOC_INVAL);
3741 
3742 	switch (iocp->ioc_cmd) {
3743 	default:
3744 		return (IOC_INVAL);
3745 
3746 	case LB_GET_INFO_SIZE:
3747 		size = sizeof (lb_info_sz_t);
3748 		if (iocp->ioc_count != size)
3749 			return (IOC_INVAL);
3750 
3751 		value = sizeof (lb_normal);
3752 		if (hw->phy.media_type == e1000_media_type_copper)
3753 			value += sizeof (lb_phy);
3754 		else
3755 			value += sizeof (lb_serdes);
3756 		value += sizeof (lb_external);
3757 
3758 		lbsp = (lb_info_sz_t *)(uintptr_t)mp->b_cont->b_rptr;
3759 		*lbsp = value;
3760 		break;
3761 
3762 	case LB_GET_INFO:
3763 		value = sizeof (lb_normal);
3764 		if (hw->phy.media_type == e1000_media_type_copper)
3765 			value += sizeof (lb_phy);
3766 		else
3767 			value += sizeof (lb_serdes);
3768 		value += sizeof (lb_external);
3769 
3770 		size = value;
3771 		if (iocp->ioc_count != size)
3772 			return (IOC_INVAL);
3773 
3774 		value = 0;
3775 		lbpp = (lb_property_t *)(uintptr_t)mp->b_cont->b_rptr;
3776 
3777 		lbpp[value++] = lb_normal;
3778 		if (hw->phy.media_type == e1000_media_type_copper)
3779 			lbpp[value++] = lb_phy;
3780 		else
3781 			lbpp[value++] = lb_serdes;
3782 		lbpp[value++] = lb_external;
3783 		break;
3784 
3785 	case LB_GET_MODE:
3786 		size = sizeof (uint32_t);
3787 		if (iocp->ioc_count != size)
3788 			return (IOC_INVAL);
3789 
3790 		lbmp = (uint32_t *)(uintptr_t)mp->b_cont->b_rptr;
3791 		*lbmp = igb->loopback_mode;
3792 		break;
3793 
3794 	case LB_SET_MODE:
3795 		size = 0;
3796 		if (iocp->ioc_count != sizeof (uint32_t))
3797 			return (IOC_INVAL);
3798 
3799 		lbmp = (uint32_t *)(uintptr_t)mp->b_cont->b_rptr;
3800 		if (!igb_set_loopback_mode(igb, *lbmp))
3801 			return (IOC_INVAL);
3802 		break;
3803 	}
3804 
3805 	iocp->ioc_count = size;
3806 	iocp->ioc_error = 0;
3807 
3808 	if (igb_check_acc_handle(igb->osdep.reg_handle) != DDI_FM_OK) {
3809 		ddi_fm_service_impact(igb->dip, DDI_SERVICE_DEGRADED);
3810 		return (IOC_INVAL);
3811 	}
3812 
3813 	return (IOC_REPLY);
3814 }
3815 
3816 /*
3817  * igb_set_loopback_mode - Setup loopback based on the loopback mode
3818  */
3819 static boolean_t
igb_set_loopback_mode(igb_t * igb,uint32_t mode)3820 igb_set_loopback_mode(igb_t *igb, uint32_t mode)
3821 {
3822 	struct e1000_hw *hw;
3823 	int i;
3824 
3825 	if (mode == igb->loopback_mode)
3826 		return (B_TRUE);
3827 
3828 	hw = &igb->hw;
3829 
3830 	igb->loopback_mode = mode;
3831 
3832 	if (mode == IGB_LB_NONE) {
3833 		/* Reset the chip */
3834 		hw->phy.autoneg_wait_to_complete = B_TRUE;
3835 		(void) igb_reset(igb);
3836 		hw->phy.autoneg_wait_to_complete = B_FALSE;
3837 		return (B_TRUE);
3838 	}
3839 
3840 	mutex_enter(&igb->gen_lock);
3841 
3842 	switch (mode) {
3843 	default:
3844 		mutex_exit(&igb->gen_lock);
3845 		return (B_FALSE);
3846 
3847 	case IGB_LB_EXTERNAL:
3848 		igb_set_external_loopback(igb);
3849 		break;
3850 
3851 	case IGB_LB_INTERNAL_PHY:
3852 		igb_set_internal_phy_loopback(igb);
3853 		break;
3854 
3855 	case IGB_LB_INTERNAL_SERDES:
3856 		igb_set_internal_serdes_loopback(igb);
3857 		break;
3858 	}
3859 
3860 	mutex_exit(&igb->gen_lock);
3861 
3862 	/*
3863 	 * When external loopback is set, wait up to 1000ms to get the link up.
3864 	 * According to test, 1000ms can work and it's an experimental value.
3865 	 */
3866 	if (mode == IGB_LB_EXTERNAL) {
3867 		for (i = 0; i <= 10; i++) {
3868 			mutex_enter(&igb->gen_lock);
3869 			(void) igb_link_check(igb);
3870 			mutex_exit(&igb->gen_lock);
3871 
3872 			if (igb->link_state == LINK_STATE_UP)
3873 				break;
3874 
3875 			msec_delay(100);
3876 		}
3877 
3878 		if (igb->link_state != LINK_STATE_UP) {
3879 			/*
3880 			 * Does not support external loopback.
3881 			 * Reset driver to loopback none.
3882 			 */
3883 			igb->loopback_mode = IGB_LB_NONE;
3884 
3885 			/* Reset the chip */
3886 			hw->phy.autoneg_wait_to_complete = B_TRUE;
3887 			(void) igb_reset(igb);
3888 			hw->phy.autoneg_wait_to_complete = B_FALSE;
3889 
3890 			IGB_DEBUGLOG_0(igb, "Set external loopback failed, "
3891 			    "reset to loopback none.");
3892 
3893 			return (B_FALSE);
3894 		}
3895 	}
3896 
3897 	return (B_TRUE);
3898 }
3899 
3900 /*
3901  * igb_set_external_loopback - Set the external loopback mode
3902  */
3903 static void
igb_set_external_loopback(igb_t * igb)3904 igb_set_external_loopback(igb_t *igb)
3905 {
3906 	struct e1000_hw *hw;
3907 	uint32_t ctrl_ext;
3908 
3909 	hw = &igb->hw;
3910 
3911 	/* Set link mode to PHY (00b) in the Extended Control register */
3912 	ctrl_ext = E1000_READ_REG(hw, E1000_CTRL_EXT);
3913 	ctrl_ext &= ~E1000_CTRL_EXT_LINK_MODE_MASK;
3914 	E1000_WRITE_REG(hw, E1000_CTRL_EXT, ctrl_ext);
3915 
3916 	(void) e1000_write_phy_reg(hw, 0x0, 0x0140);
3917 	(void) e1000_write_phy_reg(hw, 0x9, 0x1a00);
3918 	(void) e1000_write_phy_reg(hw, 0x12, 0x1610);
3919 	(void) e1000_write_phy_reg(hw, 0x1f37, 0x3f1c);
3920 }
3921 
3922 /*
3923  * igb_set_internal_phy_loopback - Set the internal PHY loopback mode
3924  */
3925 static void
igb_set_internal_phy_loopback(igb_t * igb)3926 igb_set_internal_phy_loopback(igb_t *igb)
3927 {
3928 	struct e1000_hw *hw;
3929 	uint32_t ctrl_ext;
3930 	uint16_t phy_ctrl;
3931 	uint16_t phy_pconf;
3932 
3933 	hw = &igb->hw;
3934 
3935 	/* Set link mode to PHY (00b) in the Extended Control register */
3936 	ctrl_ext = E1000_READ_REG(hw, E1000_CTRL_EXT);
3937 	ctrl_ext &= ~E1000_CTRL_EXT_LINK_MODE_MASK;
3938 	E1000_WRITE_REG(hw, E1000_CTRL_EXT, ctrl_ext);
3939 
3940 	/*
3941 	 * Set PHY control register (0x4140):
3942 	 *    Set full duplex mode
3943 	 *    Set loopback bit
3944 	 *    Clear auto-neg enable bit
3945 	 *    Set PHY speed
3946 	 */
3947 	phy_ctrl = MII_CR_FULL_DUPLEX | MII_CR_SPEED_1000 | MII_CR_LOOPBACK;
3948 	(void) e1000_write_phy_reg(hw, PHY_CONTROL, phy_ctrl);
3949 
3950 	/* Set the link disable bit in the Port Configuration register */
3951 	(void) e1000_read_phy_reg(hw, 0x10, &phy_pconf);
3952 	phy_pconf |= (uint16_t)1 << 14;
3953 	(void) e1000_write_phy_reg(hw, 0x10, phy_pconf);
3954 }
3955 
3956 /*
3957  * igb_set_internal_serdes_loopback - Set the internal SerDes loopback mode
3958  */
3959 static void
igb_set_internal_serdes_loopback(igb_t * igb)3960 igb_set_internal_serdes_loopback(igb_t *igb)
3961 {
3962 	struct e1000_hw *hw;
3963 	uint32_t ctrl_ext;
3964 	uint32_t ctrl;
3965 	uint32_t pcs_lctl;
3966 	uint32_t connsw;
3967 
3968 	hw = &igb->hw;
3969 
3970 	/* Set link mode to SerDes (11b) in the Extended Control register */
3971 	ctrl_ext = E1000_READ_REG(hw, E1000_CTRL_EXT);
3972 	ctrl_ext |= E1000_CTRL_EXT_LINK_MODE_PCIE_SERDES;
3973 	E1000_WRITE_REG(hw, E1000_CTRL_EXT, ctrl_ext);
3974 
3975 	/* Configure the SerDes to loopback */
3976 	E1000_WRITE_REG(hw, E1000_SCTL, 0x410);
3977 
3978 	/* Set Device Control register */
3979 	ctrl = E1000_READ_REG(hw, E1000_CTRL);
3980 	ctrl |= (E1000_CTRL_FD |	/* Force full duplex */
3981 	    E1000_CTRL_SLU);		/* Force link up */
3982 	ctrl &= ~(E1000_CTRL_RFCE |	/* Disable receive flow control */
3983 	    E1000_CTRL_TFCE |		/* Disable transmit flow control */
3984 	    E1000_CTRL_LRST);		/* Clear link reset */
3985 	E1000_WRITE_REG(hw, E1000_CTRL, ctrl);
3986 
3987 	/* Set PCS Link Control register */
3988 	pcs_lctl = E1000_READ_REG(hw, E1000_PCS_LCTL);
3989 	pcs_lctl |= (E1000_PCS_LCTL_FORCE_LINK |
3990 	    E1000_PCS_LCTL_FSD |
3991 	    E1000_PCS_LCTL_FDV_FULL |
3992 	    E1000_PCS_LCTL_FLV_LINK_UP);
3993 	pcs_lctl &= ~E1000_PCS_LCTL_AN_ENABLE;
3994 	E1000_WRITE_REG(hw, E1000_PCS_LCTL, pcs_lctl);
3995 
3996 	/* Set the Copper/Fiber Switch Control - CONNSW register */
3997 	connsw = E1000_READ_REG(hw, E1000_CONNSW);
3998 	connsw &= ~E1000_CONNSW_ENRGSRC;
3999 	E1000_WRITE_REG(hw, E1000_CONNSW, connsw);
4000 }
4001 
4002 #pragma inline(igb_intr_rx_work)
4003 /*
4004  * igb_intr_rx_work - rx processing of ISR
4005  */
4006 static void
igb_intr_rx_work(igb_rx_ring_t * rx_ring)4007 igb_intr_rx_work(igb_rx_ring_t *rx_ring)
4008 {
4009 	mblk_t *mp;
4010 
4011 	mutex_enter(&rx_ring->rx_lock);
4012 	mp = igb_rx(rx_ring, IGB_NO_POLL);
4013 	mutex_exit(&rx_ring->rx_lock);
4014 
4015 	if (mp != NULL)
4016 		mac_rx_ring(rx_ring->igb->mac_hdl, rx_ring->ring_handle, mp,
4017 		    rx_ring->ring_gen_num);
4018 }
4019 
4020 #pragma inline(igb_intr_tx_work)
4021 /*
4022  * igb_intr_tx_work - tx processing of ISR
4023  */
4024 static void
igb_intr_tx_work(igb_tx_ring_t * tx_ring)4025 igb_intr_tx_work(igb_tx_ring_t *tx_ring)
4026 {
4027 	igb_t *igb = tx_ring->igb;
4028 
4029 	/* Recycle the tx descriptors */
4030 	tx_ring->tx_recycle(tx_ring);
4031 
4032 	/* Schedule the re-transmit */
4033 	if (tx_ring->reschedule &&
4034 	    (tx_ring->tbd_free >= igb->tx_resched_thresh)) {
4035 		tx_ring->reschedule = B_FALSE;
4036 		mac_tx_ring_update(tx_ring->igb->mac_hdl, tx_ring->ring_handle);
4037 		IGB_DEBUG_STAT(tx_ring->stat_reschedule);
4038 	}
4039 }
4040 
4041 #pragma inline(igb_intr_link_work)
4042 /*
4043  * igb_intr_link_work - link-status-change processing of ISR
4044  */
4045 static void
igb_intr_link_work(igb_t * igb)4046 igb_intr_link_work(igb_t *igb)
4047 {
4048 	boolean_t link_changed;
4049 
4050 	igb_stop_watchdog_timer(igb);
4051 
4052 	mutex_enter(&igb->gen_lock);
4053 
4054 	/*
4055 	 * Because we got a link-status-change interrupt, force
4056 	 * e1000_check_for_link() to look at phy
4057 	 */
4058 	igb->hw.mac.get_link_status = B_TRUE;
4059 
4060 	/* igb_link_check takes care of link status change */
4061 	link_changed = igb_link_check(igb);
4062 
4063 	/* Get new phy state */
4064 	igb_get_phy_state(igb);
4065 
4066 	mutex_exit(&igb->gen_lock);
4067 
4068 	if (link_changed)
4069 		mac_link_update(igb->mac_hdl, igb->link_state);
4070 
4071 	igb_start_watchdog_timer(igb);
4072 }
4073 
4074 /*
4075  * igb_intr_legacy - Interrupt handler for legacy interrupts
4076  */
4077 static uint_t
igb_intr_legacy(void * arg1,void * arg2)4078 igb_intr_legacy(void *arg1, void *arg2)
4079 {
4080 	igb_t *igb = (igb_t *)arg1;
4081 	igb_tx_ring_t *tx_ring;
4082 	uint32_t icr;
4083 	mblk_t *mp;
4084 	boolean_t tx_reschedule;
4085 	boolean_t link_changed;
4086 	uint_t result;
4087 
4088 	_NOTE(ARGUNUSED(arg2));
4089 
4090 	mutex_enter(&igb->gen_lock);
4091 
4092 	if (igb->igb_state & IGB_SUSPENDED) {
4093 		mutex_exit(&igb->gen_lock);
4094 		return (DDI_INTR_UNCLAIMED);
4095 	}
4096 
4097 	mp = NULL;
4098 	tx_reschedule = B_FALSE;
4099 	link_changed = B_FALSE;
4100 	icr = E1000_READ_REG(&igb->hw, E1000_ICR);
4101 
4102 	if (igb_check_acc_handle(igb->osdep.reg_handle) != DDI_FM_OK) {
4103 		mutex_exit(&igb->gen_lock);
4104 		ddi_fm_service_impact(igb->dip, DDI_SERVICE_DEGRADED);
4105 		atomic_or_32(&igb->igb_state, IGB_ERROR);
4106 		return (DDI_INTR_UNCLAIMED);
4107 	}
4108 
4109 	if (icr & E1000_ICR_INT_ASSERTED) {
4110 		/*
4111 		 * E1000_ICR_INT_ASSERTED bit was set:
4112 		 * Read(Clear) the ICR, claim this interrupt,
4113 		 * look for work to do.
4114 		 */
4115 		ASSERT(igb->num_rx_rings == 1);
4116 		ASSERT(igb->num_tx_rings == 1);
4117 
4118 		/* Make sure all interrupt causes cleared */
4119 		(void) E1000_READ_REG(&igb->hw, E1000_EICR);
4120 
4121 		if (icr & E1000_ICR_RXT0) {
4122 			mp = igb_rx(&igb->rx_rings[0], IGB_NO_POLL);
4123 		}
4124 
4125 		if (icr & E1000_ICR_TXDW) {
4126 			tx_ring = &igb->tx_rings[0];
4127 
4128 			/* Recycle the tx descriptors */
4129 			tx_ring->tx_recycle(tx_ring);
4130 
4131 			/* Schedule the re-transmit */
4132 			tx_reschedule = (tx_ring->reschedule &&
4133 			    (tx_ring->tbd_free >= igb->tx_resched_thresh));
4134 		}
4135 
4136 		if (icr & E1000_ICR_LSC) {
4137 			/*
4138 			 * Because we got a link-status-change interrupt, force
4139 			 * e1000_check_for_link() to look at phy
4140 			 */
4141 			igb->hw.mac.get_link_status = B_TRUE;
4142 
4143 			/* igb_link_check takes care of link status change */
4144 			link_changed = igb_link_check(igb);
4145 
4146 			/* Get new phy state */
4147 			igb_get_phy_state(igb);
4148 		}
4149 
4150 		if (icr & E1000_ICR_DRSTA) {
4151 			/* 82580 Full Device Reset needed */
4152 			atomic_or_32(&igb->igb_state, IGB_STALL);
4153 		}
4154 
4155 		result = DDI_INTR_CLAIMED;
4156 	} else {
4157 		/*
4158 		 * E1000_ICR_INT_ASSERTED bit was not set:
4159 		 * Don't claim this interrupt.
4160 		 */
4161 		result = DDI_INTR_UNCLAIMED;
4162 	}
4163 
4164 	mutex_exit(&igb->gen_lock);
4165 
4166 	/*
4167 	 * Do the following work outside of the gen_lock
4168 	 */
4169 	if (mp != NULL)
4170 		mac_rx(igb->mac_hdl, NULL, mp);
4171 
4172 	if (tx_reschedule)  {
4173 		tx_ring->reschedule = B_FALSE;
4174 		mac_tx_ring_update(igb->mac_hdl, tx_ring->ring_handle);
4175 		IGB_DEBUG_STAT(tx_ring->stat_reschedule);
4176 	}
4177 
4178 	if (link_changed)
4179 		mac_link_update(igb->mac_hdl, igb->link_state);
4180 
4181 	return (result);
4182 }
4183 
4184 /*
4185  * igb_intr_msi - Interrupt handler for MSI
4186  */
4187 static uint_t
igb_intr_msi(void * arg1,void * arg2)4188 igb_intr_msi(void *arg1, void *arg2)
4189 {
4190 	igb_t *igb = (igb_t *)arg1;
4191 	uint32_t icr;
4192 
4193 	_NOTE(ARGUNUSED(arg2));
4194 
4195 	icr = E1000_READ_REG(&igb->hw, E1000_ICR);
4196 
4197 	if (igb_check_acc_handle(igb->osdep.reg_handle) != DDI_FM_OK) {
4198 		ddi_fm_service_impact(igb->dip, DDI_SERVICE_DEGRADED);
4199 		atomic_or_32(&igb->igb_state, IGB_ERROR);
4200 		return (DDI_INTR_CLAIMED);
4201 	}
4202 
4203 	/* Make sure all interrupt causes cleared */
4204 	(void) E1000_READ_REG(&igb->hw, E1000_EICR);
4205 
4206 	/*
4207 	 * For MSI interrupt, we have only one vector,
4208 	 * so we have only one rx ring and one tx ring enabled.
4209 	 */
4210 	ASSERT(igb->num_rx_rings == 1);
4211 	ASSERT(igb->num_tx_rings == 1);
4212 
4213 	if (icr & E1000_ICR_RXT0) {
4214 		igb_intr_rx_work(&igb->rx_rings[0]);
4215 	}
4216 
4217 	if (icr & E1000_ICR_TXDW) {
4218 		igb_intr_tx_work(&igb->tx_rings[0]);
4219 	}
4220 
4221 	if (icr & E1000_ICR_LSC) {
4222 		igb_intr_link_work(igb);
4223 	}
4224 
4225 	if (icr & E1000_ICR_DRSTA) {
4226 		/* 82580 Full Device Reset needed */
4227 		atomic_or_32(&igb->igb_state, IGB_STALL);
4228 	}
4229 
4230 	return (DDI_INTR_CLAIMED);
4231 }
4232 
4233 /*
4234  * igb_intr_rx - Interrupt handler for rx
4235  */
4236 static uint_t
igb_intr_rx(void * arg1,void * arg2)4237 igb_intr_rx(void *arg1, void *arg2)
4238 {
4239 	igb_rx_ring_t *rx_ring = (igb_rx_ring_t *)arg1;
4240 
4241 	_NOTE(ARGUNUSED(arg2));
4242 
4243 	/*
4244 	 * Only used via MSI-X vector so don't check cause bits
4245 	 * and only clean the given ring.
4246 	 */
4247 	igb_intr_rx_work(rx_ring);
4248 
4249 	return (DDI_INTR_CLAIMED);
4250 }
4251 
4252 /*
4253  * igb_intr_tx - Interrupt handler for tx
4254  */
4255 static uint_t
igb_intr_tx(void * arg1,void * arg2)4256 igb_intr_tx(void *arg1, void *arg2)
4257 {
4258 	igb_tx_ring_t *tx_ring = (igb_tx_ring_t *)arg1;
4259 
4260 	_NOTE(ARGUNUSED(arg2));
4261 
4262 	/*
4263 	 * Only used via MSI-X vector so don't check cause bits
4264 	 * and only clean the given ring.
4265 	 */
4266 	igb_intr_tx_work(tx_ring);
4267 
4268 	return (DDI_INTR_CLAIMED);
4269 }
4270 
4271 /*
4272  * igb_intr_tx_other - Interrupt handler for both tx and other
4273  *
4274  */
4275 static uint_t
igb_intr_tx_other(void * arg1,void * arg2)4276 igb_intr_tx_other(void *arg1, void *arg2)
4277 {
4278 	igb_t *igb = (igb_t *)arg1;
4279 	uint32_t icr;
4280 
4281 	_NOTE(ARGUNUSED(arg2));
4282 
4283 	icr = E1000_READ_REG(&igb->hw, E1000_ICR);
4284 
4285 	if (igb_check_acc_handle(igb->osdep.reg_handle) != DDI_FM_OK) {
4286 		ddi_fm_service_impact(igb->dip, DDI_SERVICE_DEGRADED);
4287 		atomic_or_32(&igb->igb_state, IGB_ERROR);
4288 		return (DDI_INTR_CLAIMED);
4289 	}
4290 
4291 	/*
4292 	 * Look for tx reclaiming work first. Remember, in the
4293 	 * case of only interrupt sharing, only one tx ring is
4294 	 * used
4295 	 */
4296 	igb_intr_tx_work(&igb->tx_rings[0]);
4297 
4298 	/*
4299 	 * Check for "other" causes.
4300 	 */
4301 	if (icr & E1000_ICR_LSC) {
4302 		igb_intr_link_work(igb);
4303 	}
4304 
4305 	/*
4306 	 * The DOUTSYNC bit indicates a tx packet dropped because
4307 	 * DMA engine gets "out of sync". There isn't a real fix
4308 	 * for this. The Intel recommendation is to count the number
4309 	 * of occurrences so user can detect when it is happening.
4310 	 * The issue is non-fatal and there's no recovery action
4311 	 * available.
4312 	 */
4313 	if (icr & E1000_ICR_DOUTSYNC) {
4314 		IGB_STAT(igb->dout_sync);
4315 	}
4316 
4317 	if (icr & E1000_ICR_DRSTA) {
4318 		/* 82580 Full Device Reset needed */
4319 		atomic_or_32(&igb->igb_state, IGB_STALL);
4320 	}
4321 
4322 	return (DDI_INTR_CLAIMED);
4323 }
4324 
4325 /*
4326  * igb_alloc_intrs - Allocate interrupts for the driver
4327  *
4328  * Normal sequence is to try MSI-X; if not sucessful, try MSI;
4329  * if not successful, try Legacy.
4330  * igb->intr_force can be used to force sequence to start with
4331  * any of the 3 types.
4332  * If MSI-X is not used, number of tx/rx rings is forced to 1.
4333  */
4334 static int
igb_alloc_intrs(igb_t * igb)4335 igb_alloc_intrs(igb_t *igb)
4336 {
4337 	dev_info_t *devinfo;
4338 	int intr_types;
4339 	int rc;
4340 
4341 	devinfo = igb->dip;
4342 
4343 	/* Get supported interrupt types */
4344 	rc = ddi_intr_get_supported_types(devinfo, &intr_types);
4345 
4346 	if (rc != DDI_SUCCESS) {
4347 		igb_log(igb,
4348 		    "Get supported interrupt types failed: %d", rc);
4349 		return (IGB_FAILURE);
4350 	}
4351 	IGB_DEBUGLOG_1(igb, "Supported interrupt types: %x", intr_types);
4352 
4353 	igb->intr_type = 0;
4354 
4355 	/* Install MSI-X interrupts */
4356 	if ((intr_types & DDI_INTR_TYPE_MSIX) &&
4357 	    (igb->intr_force <= IGB_INTR_MSIX)) {
4358 		rc = igb_alloc_intr_handles(igb, DDI_INTR_TYPE_MSIX);
4359 
4360 		if (rc == IGB_SUCCESS)
4361 			return (IGB_SUCCESS);
4362 
4363 		igb_log(igb,
4364 		    "Allocate MSI-X failed, trying MSI interrupts...");
4365 	}
4366 
4367 	/* MSI-X not used, force rings to 1 */
4368 	igb->num_rx_rings = 1;
4369 	igb->num_tx_rings = 1;
4370 	igb_log(igb,
4371 	    "MSI-X not used, force rx and tx queue number to 1");
4372 
4373 	/* Install MSI interrupts */
4374 	if ((intr_types & DDI_INTR_TYPE_MSI) &&
4375 	    (igb->intr_force <= IGB_INTR_MSI)) {
4376 		rc = igb_alloc_intr_handles(igb, DDI_INTR_TYPE_MSI);
4377 
4378 		if (rc == IGB_SUCCESS)
4379 			return (IGB_SUCCESS);
4380 
4381 		igb_log(igb,
4382 		    "Allocate MSI failed, trying Legacy interrupts...");
4383 	}
4384 
4385 	/* Install legacy interrupts */
4386 	if (intr_types & DDI_INTR_TYPE_FIXED) {
4387 		rc = igb_alloc_intr_handles(igb, DDI_INTR_TYPE_FIXED);
4388 
4389 		if (rc == IGB_SUCCESS)
4390 			return (IGB_SUCCESS);
4391 
4392 		igb_log(igb,
4393 		    "Allocate Legacy interrupts failed");
4394 	}
4395 
4396 	/* If none of the 3 types succeeded, return failure */
4397 	return (IGB_FAILURE);
4398 }
4399 
4400 /*
4401  * igb_alloc_intr_handles - Allocate interrupt handles.
4402  *
4403  * For legacy and MSI, only 1 handle is needed.  For MSI-X,
4404  * if fewer than 2 handles are available, return failure.
4405  * Upon success, this sets the number of Rx rings to a number that
4406  * matches the handles available for Rx interrupts.
4407  */
4408 static int
igb_alloc_intr_handles(igb_t * igb,int intr_type)4409 igb_alloc_intr_handles(igb_t *igb, int intr_type)
4410 {
4411 	dev_info_t *devinfo;
4412 	int orig, request, count, avail, actual;
4413 	int diff, minimum;
4414 	int rc;
4415 
4416 	devinfo = igb->dip;
4417 
4418 	switch (intr_type) {
4419 	case DDI_INTR_TYPE_FIXED:
4420 		request = 1;	/* Request 1 legacy interrupt handle */
4421 		minimum = 1;
4422 		IGB_DEBUGLOG_0(igb, "interrupt type: legacy");
4423 		break;
4424 
4425 	case DDI_INTR_TYPE_MSI:
4426 		request = 1;	/* Request 1 MSI interrupt handle */
4427 		minimum = 1;
4428 		IGB_DEBUGLOG_0(igb, "interrupt type: MSI");
4429 		break;
4430 
4431 	case DDI_INTR_TYPE_MSIX:
4432 		/*
4433 		 * Number of vectors for the adapter is
4434 		 * # rx rings + # tx rings
4435 		 * One of tx vectors is for tx & other
4436 		 */
4437 		request = igb->num_rx_rings + igb->num_tx_rings;
4438 		orig = request;
4439 		minimum = 2;
4440 		IGB_DEBUGLOG_0(igb, "interrupt type: MSI-X");
4441 		break;
4442 
4443 	default:
4444 		igb_log(igb,
4445 		    "invalid call to igb_alloc_intr_handles(): %d\n",
4446 		    intr_type);
4447 		return (IGB_FAILURE);
4448 	}
4449 	IGB_DEBUGLOG_2(igb, "interrupt handles requested: %d  minimum: %d",
4450 	    request, minimum);
4451 
4452 	/*
4453 	 * Get number of supported interrupts
4454 	 */
4455 	rc = ddi_intr_get_nintrs(devinfo, intr_type, &count);
4456 	if ((rc != DDI_SUCCESS) || (count < minimum)) {
4457 		igb_log(igb,
4458 		    "Get supported interrupt number failed. "
4459 		    "Return: %d, count: %d", rc, count);
4460 		return (IGB_FAILURE);
4461 	}
4462 	IGB_DEBUGLOG_1(igb, "interrupts supported: %d", count);
4463 
4464 	/*
4465 	 * Get number of available interrupts
4466 	 */
4467 	rc = ddi_intr_get_navail(devinfo, intr_type, &avail);
4468 	if ((rc != DDI_SUCCESS) || (avail < minimum)) {
4469 		igb_log(igb,
4470 		    "Get available interrupt number failed. "
4471 		    "Return: %d, available: %d", rc, avail);
4472 		return (IGB_FAILURE);
4473 	}
4474 	IGB_DEBUGLOG_1(igb, "interrupts available: %d", avail);
4475 
4476 	if (avail < request) {
4477 		igb_log(igb, "Request %d handles, %d available",
4478 		    request, avail);
4479 		request = avail;
4480 	}
4481 
4482 	actual = 0;
4483 	igb->intr_cnt = 0;
4484 
4485 	/*
4486 	 * Allocate an array of interrupt handles
4487 	 */
4488 	igb->intr_size = request * sizeof (ddi_intr_handle_t);
4489 	igb->htable = kmem_alloc(igb->intr_size, KM_SLEEP);
4490 
4491 	rc = ddi_intr_alloc(devinfo, igb->htable, intr_type, 0,
4492 	    request, &actual, DDI_INTR_ALLOC_NORMAL);
4493 	if (rc != DDI_SUCCESS) {
4494 		igb_log(igb, "Allocate interrupts failed. "
4495 		    "return: %d, request: %d, actual: %d",
4496 		    rc, request, actual);
4497 		goto alloc_handle_fail;
4498 	}
4499 	IGB_DEBUGLOG_1(igb, "interrupts actually allocated: %d", actual);
4500 
4501 	igb->intr_cnt = actual;
4502 
4503 	if (actual < minimum) {
4504 		igb_log(igb, "Insufficient interrupt handles allocated: %d",
4505 		    actual);
4506 		goto alloc_handle_fail;
4507 	}
4508 
4509 	/*
4510 	 * For MSI-X, actual might force us to reduce number of tx & rx rings
4511 	 */
4512 	if ((intr_type == DDI_INTR_TYPE_MSIX) && (orig > actual)) {
4513 		diff = orig - actual;
4514 		if (diff < igb->num_tx_rings) {
4515 			igb_log(igb,
4516 			    "MSI-X vectors force Tx queue number to %d",
4517 			    igb->num_tx_rings - diff);
4518 			igb->num_tx_rings -= diff;
4519 		} else {
4520 			igb_log(igb,
4521 			    "MSI-X vectors force Tx queue number to 1");
4522 			igb->num_tx_rings = 1;
4523 
4524 			igb_log(igb,
4525 			    "MSI-X vectors force Rx queue number to %d",
4526 			    actual - 1);
4527 			igb->num_rx_rings = actual - 1;
4528 		}
4529 	}
4530 
4531 	/*
4532 	 * Get priority for first vector, assume remaining are all the same
4533 	 */
4534 	rc = ddi_intr_get_pri(igb->htable[0], &igb->intr_pri);
4535 	if (rc != DDI_SUCCESS) {
4536 		igb_log(igb,
4537 		    "Get interrupt priority failed: %d", rc);
4538 		goto alloc_handle_fail;
4539 	}
4540 
4541 	rc = ddi_intr_get_cap(igb->htable[0], &igb->intr_cap);
4542 	if (rc != DDI_SUCCESS) {
4543 		igb_log(igb,
4544 		    "Get interrupt cap failed: %d", rc);
4545 		goto alloc_handle_fail;
4546 	}
4547 
4548 	igb->intr_type = intr_type;
4549 
4550 	return (IGB_SUCCESS);
4551 
4552 alloc_handle_fail:
4553 	igb_rem_intrs(igb);
4554 
4555 	return (IGB_FAILURE);
4556 }
4557 
4558 /*
4559  * igb_add_intr_handlers - Add interrupt handlers based on the interrupt type
4560  *
4561  * Before adding the interrupt handlers, the interrupt vectors have
4562  * been allocated, and the rx/tx rings have also been allocated.
4563  */
4564 static int
igb_add_intr_handlers(igb_t * igb)4565 igb_add_intr_handlers(igb_t *igb)
4566 {
4567 	igb_rx_ring_t *rx_ring;
4568 	igb_tx_ring_t *tx_ring;
4569 	int vector;
4570 	int rc;
4571 	int i;
4572 
4573 	vector = 0;
4574 
4575 	switch (igb->intr_type) {
4576 	case DDI_INTR_TYPE_MSIX:
4577 		/* Add interrupt handler for tx + other */
4578 		tx_ring = &igb->tx_rings[0];
4579 		rc = ddi_intr_add_handler(igb->htable[vector],
4580 		    (ddi_intr_handler_t *)igb_intr_tx_other,
4581 		    (void *)igb, NULL);
4582 
4583 		if (rc != DDI_SUCCESS) {
4584 			igb_log(igb,
4585 			    "Add tx/other interrupt handler failed: %d", rc);
4586 			return (IGB_FAILURE);
4587 		}
4588 		tx_ring->intr_vector = vector;
4589 		vector++;
4590 
4591 		/* Add interrupt handler for each rx ring */
4592 		for (i = 0; i < igb->num_rx_rings; i++) {
4593 			rx_ring = &igb->rx_rings[i];
4594 
4595 			rc = ddi_intr_add_handler(igb->htable[vector],
4596 			    (ddi_intr_handler_t *)igb_intr_rx,
4597 			    (void *)rx_ring, NULL);
4598 
4599 			if (rc != DDI_SUCCESS) {
4600 				igb_log(igb,
4601 				    "Add rx interrupt handler failed. "
4602 				    "return: %d, rx ring: %d", rc, i);
4603 				for (vector--; vector >= 0; vector--) {
4604 					(void) ddi_intr_remove_handler(
4605 					    igb->htable[vector]);
4606 				}
4607 				return (IGB_FAILURE);
4608 			}
4609 
4610 			rx_ring->intr_vector = vector;
4611 
4612 			vector++;
4613 		}
4614 
4615 		/* Add interrupt handler for each tx ring from 2nd ring */
4616 		for (i = 1; i < igb->num_tx_rings; i++) {
4617 			tx_ring = &igb->tx_rings[i];
4618 
4619 			rc = ddi_intr_add_handler(igb->htable[vector],
4620 			    (ddi_intr_handler_t *)igb_intr_tx,
4621 			    (void *)tx_ring, NULL);
4622 
4623 			if (rc != DDI_SUCCESS) {
4624 				igb_log(igb,
4625 				    "Add tx interrupt handler failed. "
4626 				    "return: %d, tx ring: %d", rc, i);
4627 				for (vector--; vector >= 0; vector--) {
4628 					(void) ddi_intr_remove_handler(
4629 					    igb->htable[vector]);
4630 				}
4631 				return (IGB_FAILURE);
4632 			}
4633 
4634 			tx_ring->intr_vector = vector;
4635 
4636 			vector++;
4637 		}
4638 
4639 		break;
4640 
4641 	case DDI_INTR_TYPE_MSI:
4642 		/* Add interrupt handlers for the only vector */
4643 		rc = ddi_intr_add_handler(igb->htable[vector],
4644 		    (ddi_intr_handler_t *)igb_intr_msi,
4645 		    (void *)igb, NULL);
4646 
4647 		if (rc != DDI_SUCCESS) {
4648 			igb_log(igb,
4649 			    "Add MSI interrupt handler failed: %d", rc);
4650 			return (IGB_FAILURE);
4651 		}
4652 
4653 		rx_ring = &igb->rx_rings[0];
4654 		rx_ring->intr_vector = vector;
4655 
4656 		vector++;
4657 		break;
4658 
4659 	case DDI_INTR_TYPE_FIXED:
4660 		/* Add interrupt handlers for the only vector */
4661 		rc = ddi_intr_add_handler(igb->htable[vector],
4662 		    (ddi_intr_handler_t *)igb_intr_legacy,
4663 		    (void *)igb, NULL);
4664 
4665 		if (rc != DDI_SUCCESS) {
4666 			igb_log(igb,
4667 			    "Add legacy interrupt handler failed: %d", rc);
4668 			return (IGB_FAILURE);
4669 		}
4670 
4671 		rx_ring = &igb->rx_rings[0];
4672 		rx_ring->intr_vector = vector;
4673 
4674 		vector++;
4675 		break;
4676 
4677 	default:
4678 		return (IGB_FAILURE);
4679 	}
4680 
4681 	ASSERT(vector == igb->intr_cnt);
4682 
4683 	return (IGB_SUCCESS);
4684 }
4685 
4686 /*
4687  * igb_setup_msix_82575 - setup 82575 adapter to use MSI-X interrupts
4688  *
4689  * For each vector enabled on the adapter, Set the MSIXBM register accordingly
4690  */
4691 static void
igb_setup_msix_82575(igb_t * igb)4692 igb_setup_msix_82575(igb_t *igb)
4693 {
4694 	uint32_t eims = 0;
4695 	int i, vector;
4696 	struct e1000_hw *hw = &igb->hw;
4697 
4698 	/*
4699 	 * Set vector for tx ring 0 and other causes.
4700 	 * NOTE assumption that it is vector 0.
4701 	 */
4702 	vector = 0;
4703 
4704 	igb->eims_mask = E1000_EICR_TX_QUEUE0 | E1000_EICR_OTHER;
4705 	E1000_WRITE_REG(hw, E1000_MSIXBM(vector), igb->eims_mask);
4706 	vector++;
4707 
4708 	for (i = 0; i < igb->num_rx_rings; i++) {
4709 		/*
4710 		 * Set vector for each rx ring
4711 		 */
4712 		eims = (E1000_EICR_RX_QUEUE0 << i);
4713 		E1000_WRITE_REG(hw, E1000_MSIXBM(vector), eims);
4714 
4715 		/*
4716 		 * Accumulate bits to enable in
4717 		 * igb_enable_adapter_interrupts_82575()
4718 		 */
4719 		igb->eims_mask |= eims;
4720 
4721 		vector++;
4722 	}
4723 
4724 	for (i = 1; i < igb->num_tx_rings; i++) {
4725 		/*
4726 		 * Set vector for each tx ring from 2nd tx ring
4727 		 */
4728 		eims = (E1000_EICR_TX_QUEUE0 << i);
4729 		E1000_WRITE_REG(hw, E1000_MSIXBM(vector), eims);
4730 
4731 		/*
4732 		 * Accumulate bits to enable in
4733 		 * igb_enable_adapter_interrupts_82575()
4734 		 */
4735 		igb->eims_mask |= eims;
4736 
4737 		vector++;
4738 	}
4739 
4740 	ASSERT(vector == igb->intr_cnt);
4741 
4742 	/*
4743 	 * Disable IAM for ICR interrupt bits
4744 	 */
4745 	E1000_WRITE_REG(hw, E1000_IAM, 0);
4746 	E1000_WRITE_FLUSH(hw);
4747 }
4748 
4749 /*
4750  * igb_setup_msix_82576 - setup 82576 adapter to use MSI-X interrupts
4751  *
4752  * 82576 uses a table based method for assigning vectors.  Each queue has a
4753  * single entry in the table to which we write a vector number along with a
4754  * "valid" bit.  The entry is a single byte in a 4-byte register.  Vectors
4755  * take a different position in the 4-byte register depending on whether
4756  * they are numbered above or below 8.
4757  */
4758 static void
igb_setup_msix_82576(igb_t * igb)4759 igb_setup_msix_82576(igb_t *igb)
4760 {
4761 	struct e1000_hw *hw = &igb->hw;
4762 	uint32_t ivar, index, vector;
4763 	int i;
4764 
4765 	/* must enable msi-x capability before IVAR settings */
4766 	E1000_WRITE_REG(hw, E1000_GPIE,
4767 	    (E1000_GPIE_MSIX_MODE | E1000_GPIE_PBA | E1000_GPIE_NSICR));
4768 
4769 	/*
4770 	 * Set vector for tx ring 0 and other causes.
4771 	 * NOTE assumption that it is vector 0.
4772 	 * This is also interdependent with installation of interrupt service
4773 	 * routines in igb_add_intr_handlers().
4774 	 */
4775 
4776 	/* assign "other" causes to vector 0 */
4777 	vector = 0;
4778 	ivar = ((vector | E1000_IVAR_VALID) << 8);
4779 	E1000_WRITE_REG(hw, E1000_IVAR_MISC, ivar);
4780 
4781 	/* assign tx ring 0 to vector 0 */
4782 	ivar = ((vector | E1000_IVAR_VALID) << 8);
4783 	E1000_WRITE_REG(hw, E1000_IVAR0, ivar);
4784 
4785 	/* prepare to enable tx & other interrupt causes */
4786 	igb->eims_mask = (1 << vector);
4787 
4788 	vector ++;
4789 	for (i = 0; i < igb->num_rx_rings; i++) {
4790 		/*
4791 		 * Set vector for each rx ring
4792 		 */
4793 		index = (i & 0x7);
4794 		ivar = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index);
4795 
4796 		if (i < 8) {
4797 			/* vector goes into low byte of register */
4798 			ivar = ivar & 0xFFFFFF00;
4799 			ivar |= (vector | E1000_IVAR_VALID);
4800 		} else {
4801 			/* vector goes into third byte of register */
4802 			ivar = ivar & 0xFF00FFFF;
4803 			ivar |= ((vector | E1000_IVAR_VALID) << 16);
4804 		}
4805 		E1000_WRITE_REG_ARRAY(hw, E1000_IVAR0, index, ivar);
4806 
4807 		/* Accumulate interrupt-cause bits to enable */
4808 		igb->eims_mask |= (1 << vector);
4809 
4810 		vector ++;
4811 	}
4812 
4813 	for (i = 1; i < igb->num_tx_rings; i++) {
4814 		/*
4815 		 * Set vector for each tx ring from 2nd tx ring.
4816 		 * Note assumption that tx vectors numericall follow rx vectors.
4817 		 */
4818 		index = (i & 0x7);
4819 		ivar = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index);
4820 
4821 		if (i < 8) {
4822 			/* vector goes into second byte of register */
4823 			ivar = ivar & 0xFFFF00FF;
4824 			ivar |= ((vector | E1000_IVAR_VALID) << 8);
4825 		} else {
4826 			/* vector goes into fourth byte of register */
4827 			ivar = ivar & 0x00FFFFFF;
4828 			ivar |= (vector | E1000_IVAR_VALID) << 24;
4829 		}
4830 		E1000_WRITE_REG_ARRAY(hw, E1000_IVAR0, index, ivar);
4831 
4832 		/* Accumulate interrupt-cause bits to enable */
4833 		igb->eims_mask |= (1 << vector);
4834 
4835 		vector ++;
4836 	}
4837 
4838 	ASSERT(vector == igb->intr_cnt);
4839 }
4840 
4841 /*
4842  * igb_setup_msix_82580 - setup 82580 adapter to use MSI-X interrupts
4843  *
4844  * 82580 uses same table approach at 82576 but has fewer entries.  Each
4845  * queue has a single entry in the table to which we write a vector number
4846  * along with a "valid" bit.  Vectors take a different position in the
4847  * register depending on * whether * they are numbered above or below 4.
4848  */
4849 static void
igb_setup_msix_82580(igb_t * igb)4850 igb_setup_msix_82580(igb_t *igb)
4851 {
4852 	struct e1000_hw *hw = &igb->hw;
4853 	uint32_t ivar, index, vector;
4854 	int i;
4855 
4856 	/* must enable msi-x capability before IVAR settings */
4857 	E1000_WRITE_REG(hw, E1000_GPIE, (E1000_GPIE_MSIX_MODE |
4858 	    E1000_GPIE_PBA | E1000_GPIE_NSICR | E1000_GPIE_EIAME));
4859 	/*
4860 	 * Set vector for tx ring 0 and other causes.
4861 	 * NOTE assumption that it is vector 0.
4862 	 * This is also interdependent with installation of interrupt service
4863 	 * routines in igb_add_intr_handlers().
4864 	 */
4865 
4866 	/* assign "other" causes to vector 0 */
4867 	vector = 0;
4868 	ivar = ((vector | E1000_IVAR_VALID) << 8);
4869 	E1000_WRITE_REG(hw, E1000_IVAR_MISC, ivar);
4870 
4871 	/* assign tx ring 0 to vector 0 */
4872 	ivar = ((vector | E1000_IVAR_VALID) << 8);
4873 	E1000_WRITE_REG(hw, E1000_IVAR0, ivar);
4874 
4875 	/* prepare to enable tx & other interrupt causes */
4876 	igb->eims_mask = (1 << vector);
4877 
4878 	vector ++;
4879 
4880 	for (i = 0; i < igb->num_rx_rings; i++) {
4881 		/*
4882 		 * Set vector for each rx ring
4883 		 */
4884 		index = (i >> 1);
4885 		ivar = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index);
4886 
4887 		if (i & 1) {
4888 			/* vector goes into third byte of register */
4889 			ivar = ivar & 0xFF00FFFF;
4890 			ivar |= ((vector | E1000_IVAR_VALID) << 16);
4891 		} else {
4892 			/* vector goes into low byte of register */
4893 			ivar = ivar & 0xFFFFFF00;
4894 			ivar |= (vector | E1000_IVAR_VALID);
4895 		}
4896 		E1000_WRITE_REG_ARRAY(hw, E1000_IVAR0, index, ivar);
4897 
4898 		/* Accumulate interrupt-cause bits to enable */
4899 		igb->eims_mask |= (1 << vector);
4900 
4901 		vector ++;
4902 	}
4903 
4904 	for (i = 1; i < igb->num_tx_rings; i++) {
4905 		/*
4906 		 * Set vector for each tx ring from 2nd tx ring.
4907 		 * Note assumption that tx vectors numericall follow rx vectors.
4908 		 */
4909 		index = (i >> 1);
4910 		ivar = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index);
4911 
4912 		if (i & 1) {
4913 			/* vector goes into high byte of register */
4914 			ivar = ivar & 0x00FFFFFF;
4915 			ivar |= ((vector | E1000_IVAR_VALID) << 24);
4916 		} else {
4917 			/* vector goes into second byte of register */
4918 			ivar = ivar & 0xFFFF00FF;
4919 			ivar |= (vector | E1000_IVAR_VALID) << 8;
4920 		}
4921 		E1000_WRITE_REG_ARRAY(hw, E1000_IVAR0, index, ivar);
4922 
4923 		/* Accumulate interrupt-cause bits to enable */
4924 		igb->eims_mask |= (1 << vector);
4925 
4926 		vector ++;
4927 	}
4928 	ASSERT(vector == igb->intr_cnt);
4929 }
4930 
4931 /*
4932  * igb_rem_intr_handlers - remove the interrupt handlers
4933  */
4934 static void
igb_rem_intr_handlers(igb_t * igb)4935 igb_rem_intr_handlers(igb_t *igb)
4936 {
4937 	int i;
4938 	int rc;
4939 
4940 	for (i = 0; i < igb->intr_cnt; i++) {
4941 		rc = ddi_intr_remove_handler(igb->htable[i]);
4942 		if (rc != DDI_SUCCESS) {
4943 			IGB_DEBUGLOG_1(igb,
4944 			    "Remove intr handler failed: %d", rc);
4945 		}
4946 	}
4947 }
4948 
4949 /*
4950  * igb_rem_intrs - remove the allocated interrupts
4951  */
4952 static void
igb_rem_intrs(igb_t * igb)4953 igb_rem_intrs(igb_t *igb)
4954 {
4955 	int i;
4956 	int rc;
4957 
4958 	for (i = 0; i < igb->intr_cnt; i++) {
4959 		rc = ddi_intr_free(igb->htable[i]);
4960 		if (rc != DDI_SUCCESS) {
4961 			IGB_DEBUGLOG_1(igb,
4962 			    "Free intr failed: %d", rc);
4963 		}
4964 	}
4965 
4966 	kmem_free(igb->htable, igb->intr_size);
4967 	igb->htable = NULL;
4968 }
4969 
4970 /*
4971  * igb_enable_intrs - enable all the ddi interrupts
4972  */
4973 static int
igb_enable_intrs(igb_t * igb)4974 igb_enable_intrs(igb_t *igb)
4975 {
4976 	int i;
4977 	int rc;
4978 
4979 	/* Enable interrupts */
4980 	if (igb->intr_cap & DDI_INTR_FLAG_BLOCK) {
4981 		/* Call ddi_intr_block_enable() for MSI */
4982 		rc = ddi_intr_block_enable(igb->htable, igb->intr_cnt);
4983 		if (rc != DDI_SUCCESS) {
4984 			igb_log(igb,
4985 			    "Enable block intr failed: %d", rc);
4986 			return (IGB_FAILURE);
4987 		}
4988 	} else {
4989 		/* Call ddi_intr_enable() for Legacy/MSI non block enable */
4990 		for (i = 0; i < igb->intr_cnt; i++) {
4991 			rc = ddi_intr_enable(igb->htable[i]);
4992 			if (rc != DDI_SUCCESS) {
4993 				igb_log(igb,
4994 				    "Enable intr failed: %d", rc);
4995 				return (IGB_FAILURE);
4996 			}
4997 		}
4998 	}
4999 
5000 	return (IGB_SUCCESS);
5001 }
5002 
5003 /*
5004  * igb_disable_intrs - disable all the ddi interrupts
5005  */
5006 static int
igb_disable_intrs(igb_t * igb)5007 igb_disable_intrs(igb_t *igb)
5008 {
5009 	int i;
5010 	int rc;
5011 
5012 	/* Disable all interrupts */
5013 	if (igb->intr_cap & DDI_INTR_FLAG_BLOCK) {
5014 		rc = ddi_intr_block_disable(igb->htable, igb->intr_cnt);
5015 		if (rc != DDI_SUCCESS) {
5016 			igb_log(igb,
5017 			    "Disable block intr failed: %d", rc);
5018 			return (IGB_FAILURE);
5019 		}
5020 	} else {
5021 		for (i = 0; i < igb->intr_cnt; i++) {
5022 			rc = ddi_intr_disable(igb->htable[i]);
5023 			if (rc != DDI_SUCCESS) {
5024 				igb_log(igb,
5025 				    "Disable intr failed: %d", rc);
5026 				return (IGB_FAILURE);
5027 			}
5028 		}
5029 	}
5030 
5031 	return (IGB_SUCCESS);
5032 }
5033 
5034 /*
5035  * igb_get_phy_state - Get and save the parameters read from PHY registers
5036  */
5037 static void
igb_get_phy_state(igb_t * igb)5038 igb_get_phy_state(igb_t *igb)
5039 {
5040 	struct e1000_hw *hw = &igb->hw;
5041 	uint16_t phy_ctrl;
5042 	uint16_t phy_status;
5043 	uint16_t phy_an_adv;
5044 	uint16_t phy_an_exp;
5045 	uint16_t phy_ext_status;
5046 	uint16_t phy_1000t_ctrl;
5047 	uint16_t phy_1000t_status;
5048 	uint16_t phy_lp_able;
5049 
5050 	ASSERT(mutex_owned(&igb->gen_lock));
5051 
5052 	if (hw->phy.media_type == e1000_media_type_copper) {
5053 		(void) e1000_read_phy_reg(hw, PHY_CONTROL, &phy_ctrl);
5054 		(void) e1000_read_phy_reg(hw, PHY_STATUS, &phy_status);
5055 		(void) e1000_read_phy_reg(hw, PHY_AUTONEG_ADV, &phy_an_adv);
5056 		(void) e1000_read_phy_reg(hw, PHY_AUTONEG_EXP, &phy_an_exp);
5057 		(void) e1000_read_phy_reg(hw, PHY_EXT_STATUS, &phy_ext_status);
5058 		(void) e1000_read_phy_reg(hw, PHY_1000T_CTRL, &phy_1000t_ctrl);
5059 		(void) e1000_read_phy_reg(hw,
5060 		    PHY_1000T_STATUS, &phy_1000t_status);
5061 		(void) e1000_read_phy_reg(hw, PHY_LP_ABILITY, &phy_lp_able);
5062 
5063 		igb->param_autoneg_cap =
5064 		    (phy_status & MII_SR_AUTONEG_CAPS) ? 1 : 0;
5065 		igb->param_pause_cap =
5066 		    (phy_an_adv & NWAY_AR_PAUSE) ? 1 : 0;
5067 		igb->param_asym_pause_cap =
5068 		    (phy_an_adv & NWAY_AR_ASM_DIR) ? 1 : 0;
5069 		igb->param_1000fdx_cap =
5070 		    ((phy_ext_status & IEEE_ESR_1000T_FD_CAPS) ||
5071 		    (phy_ext_status & IEEE_ESR_1000X_FD_CAPS)) ? 1 : 0;
5072 		igb->param_1000hdx_cap =
5073 		    ((phy_ext_status & IEEE_ESR_1000T_HD_CAPS) ||
5074 		    (phy_ext_status & IEEE_ESR_1000X_HD_CAPS)) ? 1 : 0;
5075 		igb->param_100t4_cap =
5076 		    (phy_status & MII_SR_100T4_CAPS) ? 1 : 0;
5077 		igb->param_100fdx_cap = ((phy_status & MII_SR_100X_FD_CAPS) ||
5078 		    (phy_status & MII_SR_100T2_FD_CAPS)) ? 1 : 0;
5079 		igb->param_100hdx_cap = ((phy_status & MII_SR_100X_HD_CAPS) ||
5080 		    (phy_status & MII_SR_100T2_HD_CAPS)) ? 1 : 0;
5081 		igb->param_10fdx_cap =
5082 		    (phy_status & MII_SR_10T_FD_CAPS) ? 1 : 0;
5083 		igb->param_10hdx_cap =
5084 		    (phy_status & MII_SR_10T_HD_CAPS) ? 1 : 0;
5085 		igb->param_rem_fault =
5086 		    (phy_status & MII_SR_REMOTE_FAULT) ? 1 : 0;
5087 
5088 		igb->param_adv_autoneg_cap = hw->mac.autoneg;
5089 		igb->param_adv_pause_cap =
5090 		    (phy_an_adv & NWAY_AR_PAUSE) ? 1 : 0;
5091 		igb->param_adv_asym_pause_cap =
5092 		    (phy_an_adv & NWAY_AR_ASM_DIR) ? 1 : 0;
5093 		igb->param_adv_1000hdx_cap =
5094 		    (phy_1000t_ctrl & CR_1000T_HD_CAPS) ? 1 : 0;
5095 		igb->param_adv_100t4_cap =
5096 		    (phy_an_adv & NWAY_AR_100T4_CAPS) ? 1 : 0;
5097 		igb->param_adv_rem_fault =
5098 		    (phy_an_adv & NWAY_AR_REMOTE_FAULT) ? 1 : 0;
5099 		if (igb->param_adv_autoneg_cap == 1) {
5100 			igb->param_adv_1000fdx_cap =
5101 			    (phy_1000t_ctrl & CR_1000T_FD_CAPS) ? 1 : 0;
5102 			igb->param_adv_100fdx_cap =
5103 			    (phy_an_adv & NWAY_AR_100TX_FD_CAPS) ? 1 : 0;
5104 			igb->param_adv_100hdx_cap =
5105 			    (phy_an_adv & NWAY_AR_100TX_HD_CAPS) ? 1 : 0;
5106 			igb->param_adv_10fdx_cap =
5107 			    (phy_an_adv & NWAY_AR_10T_FD_CAPS) ? 1 : 0;
5108 			igb->param_adv_10hdx_cap =
5109 			    (phy_an_adv & NWAY_AR_10T_HD_CAPS) ? 1 : 0;
5110 		}
5111 
5112 		igb->param_lp_autoneg_cap =
5113 		    (phy_an_exp & NWAY_ER_LP_NWAY_CAPS) ? 1 : 0;
5114 		igb->param_lp_pause_cap =
5115 		    (phy_lp_able & NWAY_LPAR_PAUSE) ? 1 : 0;
5116 		igb->param_lp_asym_pause_cap =
5117 		    (phy_lp_able & NWAY_LPAR_ASM_DIR) ? 1 : 0;
5118 		igb->param_lp_1000fdx_cap =
5119 		    (phy_1000t_status & SR_1000T_LP_FD_CAPS) ? 1 : 0;
5120 		igb->param_lp_1000hdx_cap =
5121 		    (phy_1000t_status & SR_1000T_LP_HD_CAPS) ? 1 : 0;
5122 		igb->param_lp_100t4_cap =
5123 		    (phy_lp_able & NWAY_LPAR_100T4_CAPS) ? 1 : 0;
5124 		igb->param_lp_100fdx_cap =
5125 		    (phy_lp_able & NWAY_LPAR_100TX_FD_CAPS) ? 1 : 0;
5126 		igb->param_lp_100hdx_cap =
5127 		    (phy_lp_able & NWAY_LPAR_100TX_HD_CAPS) ? 1 : 0;
5128 		igb->param_lp_10fdx_cap =
5129 		    (phy_lp_able & NWAY_LPAR_10T_FD_CAPS) ? 1 : 0;
5130 		igb->param_lp_10hdx_cap =
5131 		    (phy_lp_able & NWAY_LPAR_10T_HD_CAPS) ? 1 : 0;
5132 		igb->param_lp_rem_fault =
5133 		    (phy_lp_able & NWAY_LPAR_REMOTE_FAULT) ? 1 : 0;
5134 	} else {
5135 		/*
5136 		 * 1Gig Fiber adapter only offers 1Gig Full Duplex.
5137 		 */
5138 		igb->param_autoneg_cap = 0;
5139 		igb->param_pause_cap = 1;
5140 		igb->param_asym_pause_cap = 1;
5141 		igb->param_1000fdx_cap = 1;
5142 		igb->param_1000hdx_cap = 0;
5143 		igb->param_100t4_cap = 0;
5144 		igb->param_100fdx_cap = 0;
5145 		igb->param_100hdx_cap = 0;
5146 		igb->param_10fdx_cap = 0;
5147 		igb->param_10hdx_cap = 0;
5148 
5149 		igb->param_adv_autoneg_cap = 0;
5150 		igb->param_adv_pause_cap = 1;
5151 		igb->param_adv_asym_pause_cap = 1;
5152 		igb->param_adv_1000fdx_cap = 1;
5153 		igb->param_adv_1000hdx_cap = 0;
5154 		igb->param_adv_100t4_cap = 0;
5155 		igb->param_adv_100fdx_cap = 0;
5156 		igb->param_adv_100hdx_cap = 0;
5157 		igb->param_adv_10fdx_cap = 0;
5158 		igb->param_adv_10hdx_cap = 0;
5159 
5160 		igb->param_lp_autoneg_cap = 0;
5161 		igb->param_lp_pause_cap = 0;
5162 		igb->param_lp_asym_pause_cap = 0;
5163 		igb->param_lp_1000fdx_cap = 0;
5164 		igb->param_lp_1000hdx_cap = 0;
5165 		igb->param_lp_100t4_cap = 0;
5166 		igb->param_lp_100fdx_cap = 0;
5167 		igb->param_lp_100hdx_cap = 0;
5168 		igb->param_lp_10fdx_cap = 0;
5169 		igb->param_lp_10hdx_cap = 0;
5170 		igb->param_lp_rem_fault = 0;
5171 	}
5172 }
5173 
5174 /*
5175  * synchronize the adv* and en* parameters.
5176  *
5177  * See comments in <sys/dld.h> for details of the *_en_*
5178  * parameters. The usage of ndd for setting adv parameters will
5179  * synchronize all the en parameters with the e1000g parameters,
5180  * implicitly disabling any settings made via dladm.
5181  */
5182 static void
igb_param_sync(igb_t * igb)5183 igb_param_sync(igb_t *igb)
5184 {
5185 	igb->param_en_1000fdx_cap = igb->param_adv_1000fdx_cap;
5186 	igb->param_en_1000hdx_cap = igb->param_adv_1000hdx_cap;
5187 	igb->param_en_100t4_cap = igb->param_adv_100t4_cap;
5188 	igb->param_en_100fdx_cap = igb->param_adv_100fdx_cap;
5189 	igb->param_en_100hdx_cap = igb->param_adv_100hdx_cap;
5190 	igb->param_en_10fdx_cap = igb->param_adv_10fdx_cap;
5191 	igb->param_en_10hdx_cap = igb->param_adv_10hdx_cap;
5192 }
5193 
5194 /*
5195  * igb_get_driver_control
5196  */
5197 static void
igb_get_driver_control(struct e1000_hw * hw)5198 igb_get_driver_control(struct e1000_hw *hw)
5199 {
5200 	uint32_t ctrl_ext;
5201 
5202 	/* Notify firmware that driver is in control of device */
5203 	ctrl_ext = E1000_READ_REG(hw, E1000_CTRL_EXT);
5204 	ctrl_ext |= E1000_CTRL_EXT_DRV_LOAD;
5205 	E1000_WRITE_REG(hw, E1000_CTRL_EXT, ctrl_ext);
5206 }
5207 
5208 /*
5209  * igb_release_driver_control
5210  */
5211 static void
igb_release_driver_control(struct e1000_hw * hw)5212 igb_release_driver_control(struct e1000_hw *hw)
5213 {
5214 	uint32_t ctrl_ext;
5215 
5216 	/* Notify firmware that driver is no longer in control of device */
5217 	ctrl_ext = E1000_READ_REG(hw, E1000_CTRL_EXT);
5218 	ctrl_ext &= ~E1000_CTRL_EXT_DRV_LOAD;
5219 	E1000_WRITE_REG(hw, E1000_CTRL_EXT, ctrl_ext);
5220 }
5221 
5222 /*
5223  * igb_atomic_reserve - Atomic decrease operation
5224  */
5225 int
igb_atomic_reserve(uint32_t * count_p,uint32_t n)5226 igb_atomic_reserve(uint32_t *count_p, uint32_t n)
5227 {
5228 	uint32_t oldval;
5229 	uint32_t newval;
5230 
5231 	/* ATOMICALLY */
5232 	do {
5233 		oldval = *count_p;
5234 		if (oldval < n)
5235 			return (-1);
5236 		newval = oldval - n;
5237 	} while (atomic_cas_32(count_p, oldval, newval) != oldval);
5238 
5239 	return (newval);
5240 }
5241 
5242 /*
5243  * FMA support
5244  */
5245 
5246 int
igb_check_acc_handle(ddi_acc_handle_t handle)5247 igb_check_acc_handle(ddi_acc_handle_t handle)
5248 {
5249 	ddi_fm_error_t de;
5250 
5251 	ddi_fm_acc_err_get(handle, &de, DDI_FME_VERSION);
5252 	ddi_fm_acc_err_clear(handle, DDI_FME_VERSION);
5253 	return (de.fme_status);
5254 }
5255 
5256 int
igb_check_dma_handle(ddi_dma_handle_t handle)5257 igb_check_dma_handle(ddi_dma_handle_t handle)
5258 {
5259 	ddi_fm_error_t de;
5260 
5261 	ddi_fm_dma_err_get(handle, &de, DDI_FME_VERSION);
5262 	return (de.fme_status);
5263 }
5264 
5265 /*
5266  * The IO fault service error handling callback function
5267  */
5268 /*ARGSUSED*/
5269 static int
igb_fm_error_cb(dev_info_t * dip,ddi_fm_error_t * err,const void * impl_data)5270 igb_fm_error_cb(dev_info_t *dip, ddi_fm_error_t *err, const void *impl_data)
5271 {
5272 	/*
5273 	 * as the driver can always deal with an error in any dma or
5274 	 * access handle, we can just return the fme_status value.
5275 	 */
5276 	pci_ereport_post(dip, err, NULL);
5277 	return (err->fme_status);
5278 }
5279 
5280 static void
igb_fm_init(igb_t * igb)5281 igb_fm_init(igb_t *igb)
5282 {
5283 	ddi_iblock_cookie_t iblk;
5284 	int fma_dma_flag;
5285 
5286 	/* Only register with IO Fault Services if we have some capability */
5287 	if (igb->fm_capabilities & DDI_FM_ACCCHK_CAPABLE) {
5288 		igb_regs_acc_attr.devacc_attr_access = DDI_FLAGERR_ACC;
5289 	} else {
5290 		igb_regs_acc_attr.devacc_attr_access = DDI_DEFAULT_ACC;
5291 	}
5292 
5293 	if (igb->fm_capabilities & DDI_FM_DMACHK_CAPABLE) {
5294 		fma_dma_flag = 1;
5295 	} else {
5296 		fma_dma_flag = 0;
5297 	}
5298 
5299 	(void) igb_set_fma_flags(fma_dma_flag);
5300 
5301 	if (igb->fm_capabilities) {
5302 
5303 		/* Register capabilities with IO Fault Services */
5304 		ddi_fm_init(igb->dip, &igb->fm_capabilities, &iblk);
5305 
5306 		/*
5307 		 * Initialize pci ereport capabilities if ereport capable
5308 		 */
5309 		if (DDI_FM_EREPORT_CAP(igb->fm_capabilities) ||
5310 		    DDI_FM_ERRCB_CAP(igb->fm_capabilities))
5311 			pci_ereport_setup(igb->dip);
5312 
5313 		/*
5314 		 * Register error callback if error callback capable
5315 		 */
5316 		if (DDI_FM_ERRCB_CAP(igb->fm_capabilities))
5317 			ddi_fm_handler_register(igb->dip,
5318 			    igb_fm_error_cb, (void*) igb);
5319 	}
5320 }
5321 
5322 static void
igb_fm_fini(igb_t * igb)5323 igb_fm_fini(igb_t *igb)
5324 {
5325 	/* Only unregister FMA capabilities if we registered some */
5326 	if (igb->fm_capabilities) {
5327 
5328 		/*
5329 		 * Release any resources allocated by pci_ereport_setup()
5330 		 */
5331 		if (DDI_FM_EREPORT_CAP(igb->fm_capabilities) ||
5332 		    DDI_FM_ERRCB_CAP(igb->fm_capabilities))
5333 			pci_ereport_teardown(igb->dip);
5334 
5335 		/*
5336 		 * Un-register error callback if error callback capable
5337 		 */
5338 		if (DDI_FM_ERRCB_CAP(igb->fm_capabilities))
5339 			ddi_fm_handler_unregister(igb->dip);
5340 
5341 		/* Unregister from IO Fault Services */
5342 		ddi_fm_fini(igb->dip);
5343 	}
5344 }
5345 
5346 void
igb_fm_ereport(igb_t * igb,char * detail)5347 igb_fm_ereport(igb_t *igb, char *detail)
5348 {
5349 	uint64_t ena;
5350 	char buf[FM_MAX_CLASS];
5351 
5352 	(void) snprintf(buf, FM_MAX_CLASS, "%s.%s", DDI_FM_DEVICE, detail);
5353 	ena = fm_ena_generate(0, FM_ENA_FMT1);
5354 	if (DDI_FM_EREPORT_CAP(igb->fm_capabilities)) {
5355 		ddi_fm_ereport_post(igb->dip, buf, ena, DDI_NOSLEEP,
5356 		    FM_VERSION, DATA_TYPE_UINT8, FM_EREPORT_VERS0, NULL);
5357 	}
5358 }
5359