1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22 /*
23 * Copyright(c) 2007-2010 Intel Corporation. All rights reserved.
24 */
25
26 /*
27 * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved.
28 * Copyright 2020 Joyent, Inc.
29 * Copyright 2012 Nexenta Systems, Inc. All rights reserved.
30 * Copyright (c) 2013 Saso Kiselkov. All rights reserved.
31 * Copyright (c) 2013 OSN Online Service Nuernberg GmbH. All rights reserved.
32 * Copyright 2016 OmniTI Computer Consulting, Inc. All rights reserved.
33 * Copyright 2020 Oxide Computer Company
34 */
35
36 #include "ixgbe_sw.h"
37
38 static char ixgbe_ident[] = "Intel 10Gb Ethernet";
39
40 /*
41 * Local function protoypes
42 */
43 static int ixgbe_register_mac(ixgbe_t *);
44 static int ixgbe_identify_hardware(ixgbe_t *);
45 static int ixgbe_regs_map(ixgbe_t *);
46 static void ixgbe_init_properties(ixgbe_t *);
47 static int ixgbe_init_driver_settings(ixgbe_t *);
48 static void ixgbe_init_locks(ixgbe_t *);
49 static void ixgbe_destroy_locks(ixgbe_t *);
50 static int ixgbe_init(ixgbe_t *);
51 static int ixgbe_chip_start(ixgbe_t *);
52 static void ixgbe_chip_stop(ixgbe_t *);
53 static int ixgbe_reset(ixgbe_t *);
54 static void ixgbe_tx_clean(ixgbe_t *);
55 static boolean_t ixgbe_tx_drain(ixgbe_t *);
56 static boolean_t ixgbe_rx_drain(ixgbe_t *);
57 static int ixgbe_alloc_rings(ixgbe_t *);
58 static void ixgbe_free_rings(ixgbe_t *);
59 static int ixgbe_alloc_rx_data(ixgbe_t *);
60 static void ixgbe_free_rx_data(ixgbe_t *);
61 static int ixgbe_setup_rings(ixgbe_t *);
62 static int ixgbe_setup_rx(ixgbe_t *);
63 static void ixgbe_setup_tx(ixgbe_t *);
64 static void ixgbe_setup_rx_ring(ixgbe_rx_ring_t *);
65 static void ixgbe_setup_tx_ring(ixgbe_tx_ring_t *);
66 static void ixgbe_setup_rss(ixgbe_t *);
67 static void ixgbe_setup_vmdq(ixgbe_t *);
68 static void ixgbe_setup_vmdq_rss(ixgbe_t *);
69 static void ixgbe_setup_rss_table(ixgbe_t *);
70 static void ixgbe_init_unicst(ixgbe_t *);
71 static int ixgbe_init_vlan(ixgbe_t *);
72 static int ixgbe_unicst_find(ixgbe_t *, const uint8_t *);
73 static void ixgbe_setup_multicst(ixgbe_t *);
74 static void ixgbe_get_hw_state(ixgbe_t *);
75 static void ixgbe_setup_vmdq_rss_conf(ixgbe_t *ixgbe);
76 static void ixgbe_get_conf(ixgbe_t *);
77 static void ixgbe_init_params(ixgbe_t *);
78 static int ixgbe_get_prop(ixgbe_t *, char *, int, int, int);
79 static void ixgbe_driver_link_check(ixgbe_t *);
80 static void ixgbe_sfp_check(void *);
81 static void ixgbe_overtemp_check(void *);
82 static void ixgbe_phy_check(void *);
83 static void ixgbe_link_timer(void *);
84 static void ixgbe_local_timer(void *);
85 static void ixgbe_arm_watchdog_timer(ixgbe_t *);
86 static void ixgbe_restart_watchdog_timer(ixgbe_t *);
87 static void ixgbe_disable_adapter_interrupts(ixgbe_t *);
88 static void ixgbe_enable_adapter_interrupts(ixgbe_t *);
89 static boolean_t is_valid_mac_addr(uint8_t *);
90 static boolean_t ixgbe_stall_check(ixgbe_t *);
91 static boolean_t ixgbe_set_loopback_mode(ixgbe_t *, uint32_t);
92 static void ixgbe_set_internal_mac_loopback(ixgbe_t *);
93 static boolean_t ixgbe_find_mac_address(ixgbe_t *);
94 static int ixgbe_alloc_intrs(ixgbe_t *);
95 static int ixgbe_alloc_intr_handles(ixgbe_t *, int);
96 static int ixgbe_add_intr_handlers(ixgbe_t *);
97 static void ixgbe_map_rxring_to_vector(ixgbe_t *, int, int);
98 static void ixgbe_map_txring_to_vector(ixgbe_t *, int, int);
99 static void ixgbe_setup_ivar(ixgbe_t *, uint16_t, uint8_t, int8_t);
100 static void ixgbe_enable_ivar(ixgbe_t *, uint16_t, int8_t);
101 static void ixgbe_disable_ivar(ixgbe_t *, uint16_t, int8_t);
102 static uint32_t ixgbe_get_hw_rx_index(ixgbe_t *ixgbe, uint32_t sw_rx_index);
103 static int ixgbe_map_intrs_to_vectors(ixgbe_t *);
104 static void ixgbe_setup_adapter_vector(ixgbe_t *);
105 static void ixgbe_rem_intr_handlers(ixgbe_t *);
106 static void ixgbe_rem_intrs(ixgbe_t *);
107 static int ixgbe_enable_intrs(ixgbe_t *);
108 static int ixgbe_disable_intrs(ixgbe_t *);
109 static uint_t ixgbe_intr_legacy(void *, void *);
110 static uint_t ixgbe_intr_msi(void *, void *);
111 static uint_t ixgbe_intr_msix(void *, void *);
112 static void ixgbe_intr_rx_work(ixgbe_rx_ring_t *);
113 static void ixgbe_intr_tx_work(ixgbe_tx_ring_t *);
114 static void ixgbe_intr_other_work(ixgbe_t *, uint32_t);
115 static void ixgbe_get_driver_control(struct ixgbe_hw *);
116 static int ixgbe_addmac(void *, const uint8_t *);
117 static int ixgbe_remmac(void *, const uint8_t *);
118 static int ixgbe_addvlan(mac_group_driver_t, uint16_t);
119 static int ixgbe_remvlan(mac_group_driver_t, uint16_t);
120 static void ixgbe_release_driver_control(struct ixgbe_hw *);
121
122 static int ixgbe_attach(dev_info_t *, ddi_attach_cmd_t);
123 static int ixgbe_detach(dev_info_t *, ddi_detach_cmd_t);
124 static int ixgbe_resume(dev_info_t *);
125 static int ixgbe_suspend(dev_info_t *);
126 static int ixgbe_quiesce(dev_info_t *);
127 static void ixgbe_unconfigure(dev_info_t *, ixgbe_t *);
128 static uint8_t *ixgbe_mc_table_itr(struct ixgbe_hw *, uint8_t **, uint32_t *);
129 static int ixgbe_cbfunc(dev_info_t *, ddi_cb_action_t, void *, void *, void *);
130 static int ixgbe_intr_cb_register(ixgbe_t *);
131 static int ixgbe_intr_adjust(ixgbe_t *, ddi_cb_action_t, int);
132
133 static int ixgbe_fm_error_cb(dev_info_t *dip, ddi_fm_error_t *err,
134 const void *impl_data);
135 static void ixgbe_fm_init(ixgbe_t *);
136 static void ixgbe_fm_fini(ixgbe_t *);
137 static int ixgbe_ufm_fill_image(ddi_ufm_handle_t *, void *arg, uint_t,
138 ddi_ufm_image_t *);
139 static int ixgbe_ufm_fill_slot(ddi_ufm_handle_t *, void *, uint_t, uint_t,
140 ddi_ufm_slot_t *);
141 static int ixgbe_ufm_getcaps(ddi_ufm_handle_t *, void *, ddi_ufm_cap_t *);
142 static int ixgbe_ufm_readimg(ddi_ufm_handle_t *, void *, uint_t, uint_t,
143 uint64_t, uint64_t, void *, uint64_t *);
144
145 char *ixgbe_priv_props[] = {
146 "_tx_copy_thresh",
147 "_tx_recycle_thresh",
148 "_tx_overload_thresh",
149 "_tx_resched_thresh",
150 "_rx_copy_thresh",
151 "_rx_limit_per_intr",
152 "_intr_throttling",
153 "_adv_pause_cap",
154 "_adv_asym_pause_cap",
155 NULL
156 };
157
158 #define IXGBE_MAX_PRIV_PROPS \
159 (sizeof (ixgbe_priv_props) / sizeof (mac_priv_prop_t))
160
161 static struct cb_ops ixgbe_cb_ops = {
162 nulldev, /* cb_open */
163 nulldev, /* cb_close */
164 nodev, /* cb_strategy */
165 nodev, /* cb_print */
166 nodev, /* cb_dump */
167 nodev, /* cb_read */
168 nodev, /* cb_write */
169 nodev, /* cb_ioctl */
170 nodev, /* cb_devmap */
171 nodev, /* cb_mmap */
172 nodev, /* cb_segmap */
173 nochpoll, /* cb_chpoll */
174 ddi_prop_op, /* cb_prop_op */
175 NULL, /* cb_stream */
176 D_MP | D_HOTPLUG, /* cb_flag */
177 CB_REV, /* cb_rev */
178 nodev, /* cb_aread */
179 nodev /* cb_awrite */
180 };
181
182 static struct dev_ops ixgbe_dev_ops = {
183 DEVO_REV, /* devo_rev */
184 0, /* devo_refcnt */
185 NULL, /* devo_getinfo */
186 nulldev, /* devo_identify */
187 nulldev, /* devo_probe */
188 ixgbe_attach, /* devo_attach */
189 ixgbe_detach, /* devo_detach */
190 nodev, /* devo_reset */
191 &ixgbe_cb_ops, /* devo_cb_ops */
192 NULL, /* devo_bus_ops */
193 ddi_power, /* devo_power */
194 ixgbe_quiesce, /* devo_quiesce */
195 };
196
197 static struct modldrv ixgbe_modldrv = {
198 &mod_driverops, /* Type of module. This one is a driver */
199 ixgbe_ident, /* Discription string */
200 &ixgbe_dev_ops /* driver ops */
201 };
202
203 static struct modlinkage ixgbe_modlinkage = {
204 MODREV_1, &ixgbe_modldrv, NULL
205 };
206
207 /*
208 * Access attributes for register mapping
209 */
210 ddi_device_acc_attr_t ixgbe_regs_acc_attr = {
211 DDI_DEVICE_ATTR_V1,
212 DDI_STRUCTURE_LE_ACC,
213 DDI_STRICTORDER_ACC,
214 DDI_FLAGERR_ACC
215 };
216
217 /*
218 * Loopback property
219 */
220 static lb_property_t lb_normal = {
221 normal, "normal", IXGBE_LB_NONE
222 };
223
224 static lb_property_t lb_mac = {
225 internal, "MAC", IXGBE_LB_INTERNAL_MAC
226 };
227
228 static lb_property_t lb_external = {
229 external, "External", IXGBE_LB_EXTERNAL
230 };
231
232 #define IXGBE_M_CALLBACK_FLAGS \
233 (MC_IOCTL | MC_GETCAPAB | MC_SETPROP | MC_GETPROP | MC_PROPINFO)
234
235 static mac_callbacks_t ixgbe_m_callbacks = {
236 IXGBE_M_CALLBACK_FLAGS,
237 ixgbe_m_stat,
238 ixgbe_m_start,
239 ixgbe_m_stop,
240 ixgbe_m_promisc,
241 ixgbe_m_multicst,
242 NULL,
243 NULL,
244 NULL,
245 ixgbe_m_ioctl,
246 ixgbe_m_getcapab,
247 NULL,
248 NULL,
249 ixgbe_m_setprop,
250 ixgbe_m_getprop,
251 ixgbe_m_propinfo
252 };
253
254 /*
255 * Initialize capabilities of each supported adapter type
256 */
257 static adapter_info_t ixgbe_82598eb_cap = {
258 64, /* maximum number of rx queues */
259 1, /* minimum number of rx queues */
260 64, /* default number of rx queues */
261 16, /* maximum number of rx groups */
262 1, /* minimum number of rx groups */
263 1, /* default number of rx groups */
264 32, /* maximum number of tx queues */
265 1, /* minimum number of tx queues */
266 8, /* default number of tx queues */
267 16366, /* maximum MTU size */
268 0xFFFF, /* maximum interrupt throttle rate */
269 0, /* minimum interrupt throttle rate */
270 200, /* default interrupt throttle rate */
271 18, /* maximum total msix vectors */
272 16, /* maximum number of ring vectors */
273 2, /* maximum number of other vectors */
274 IXGBE_EICR_LSC, /* "other" interrupt types handled */
275 0, /* "other" interrupt types enable mask */
276 (IXGBE_FLAG_DCA_CAPABLE /* capability flags */
277 | IXGBE_FLAG_RSS_CAPABLE
278 | IXGBE_FLAG_VMDQ_CAPABLE)
279 };
280
281 static adapter_info_t ixgbe_82599eb_cap = {
282 128, /* maximum number of rx queues */
283 1, /* minimum number of rx queues */
284 128, /* default number of rx queues */
285 64, /* maximum number of rx groups */
286 1, /* minimum number of rx groups */
287 1, /* default number of rx groups */
288 128, /* maximum number of tx queues */
289 1, /* minimum number of tx queues */
290 8, /* default number of tx queues */
291 15500, /* maximum MTU size */
292 0xFF8, /* maximum interrupt throttle rate */
293 0, /* minimum interrupt throttle rate */
294 200, /* default interrupt throttle rate */
295 64, /* maximum total msix vectors */
296 16, /* maximum number of ring vectors */
297 2, /* maximum number of other vectors */
298 (IXGBE_EICR_LSC
299 | IXGBE_EICR_GPI_SDP1
300 | IXGBE_EICR_GPI_SDP2), /* "other" interrupt types handled */
301
302 (IXGBE_SDP1_GPIEN
303 | IXGBE_SDP2_GPIEN), /* "other" interrupt types enable mask */
304
305 (IXGBE_FLAG_DCA_CAPABLE
306 | IXGBE_FLAG_RSS_CAPABLE
307 | IXGBE_FLAG_VMDQ_CAPABLE
308 | IXGBE_FLAG_RSC_CAPABLE
309 | IXGBE_FLAG_SFP_PLUG_CAPABLE) /* capability flags */
310 };
311
312 static adapter_info_t ixgbe_X540_cap = {
313 128, /* maximum number of rx queues */
314 1, /* minimum number of rx queues */
315 128, /* default number of rx queues */
316 64, /* maximum number of rx groups */
317 1, /* minimum number of rx groups */
318 1, /* default number of rx groups */
319 128, /* maximum number of tx queues */
320 1, /* minimum number of tx queues */
321 8, /* default number of tx queues */
322 15500, /* maximum MTU size */
323 0xFF8, /* maximum interrupt throttle rate */
324 0, /* minimum interrupt throttle rate */
325 200, /* default interrupt throttle rate */
326 64, /* maximum total msix vectors */
327 16, /* maximum number of ring vectors */
328 2, /* maximum number of other vectors */
329 (IXGBE_EICR_LSC
330 | IXGBE_EICR_GPI_SDP1_X540
331 | IXGBE_EICR_GPI_SDP2_X540), /* "other" interrupt types handled */
332
333 (IXGBE_SDP1_GPIEN_X540
334 | IXGBE_SDP2_GPIEN_X540), /* "other" interrupt types enable mask */
335
336 (IXGBE_FLAG_DCA_CAPABLE
337 | IXGBE_FLAG_RSS_CAPABLE
338 | IXGBE_FLAG_VMDQ_CAPABLE
339 | IXGBE_FLAG_RSC_CAPABLE) /* capability flags */
340 };
341
342 static adapter_info_t ixgbe_X550_cap = {
343 128, /* maximum number of rx queues */
344 1, /* minimum number of rx queues */
345 128, /* default number of rx queues */
346 64, /* maximum number of rx groups */
347 1, /* minimum number of rx groups */
348 1, /* default number of rx groups */
349 128, /* maximum number of tx queues */
350 1, /* minimum number of tx queues */
351 8, /* default number of tx queues */
352 15500, /* maximum MTU size */
353 0xFF8, /* maximum interrupt throttle rate */
354 0, /* minimum interrupt throttle rate */
355 0x200, /* default interrupt throttle rate */
356 64, /* maximum total msix vectors */
357 16, /* maximum number of ring vectors */
358 2, /* maximum number of other vectors */
359 IXGBE_EICR_LSC, /* "other" interrupt types handled */
360 0, /* "other" interrupt types enable mask */
361 (IXGBE_FLAG_RSS_CAPABLE
362 | IXGBE_FLAG_VMDQ_CAPABLE
363 | IXGBE_FLAG_RSC_CAPABLE) /* capability flags */
364 };
365
366 static ddi_ufm_ops_t ixgbe_ufm_ops = {
367 .ddi_ufm_op_fill_image = ixgbe_ufm_fill_image,
368 .ddi_ufm_op_fill_slot = ixgbe_ufm_fill_slot,
369 .ddi_ufm_op_getcaps = ixgbe_ufm_getcaps,
370 .ddi_ufm_op_readimg = ixgbe_ufm_readimg
371 };
372
373
374 /*
375 * Module Initialization Functions.
376 */
377
378 int
_init(void)379 _init(void)
380 {
381 int status;
382
383 mac_init_ops(&ixgbe_dev_ops, MODULE_NAME);
384
385 status = mod_install(&ixgbe_modlinkage);
386
387 if (status != DDI_SUCCESS) {
388 mac_fini_ops(&ixgbe_dev_ops);
389 }
390
391 return (status);
392 }
393
394 int
_fini(void)395 _fini(void)
396 {
397 int status;
398
399 status = mod_remove(&ixgbe_modlinkage);
400
401 if (status == DDI_SUCCESS) {
402 mac_fini_ops(&ixgbe_dev_ops);
403 }
404
405 return (status);
406 }
407
408 int
_info(struct modinfo * modinfop)409 _info(struct modinfo *modinfop)
410 {
411 int status;
412
413 status = mod_info(&ixgbe_modlinkage, modinfop);
414
415 return (status);
416 }
417
418 /*
419 * ixgbe_attach - Driver attach.
420 *
421 * This function is the device specific initialization entry
422 * point. This entry point is required and must be written.
423 * The DDI_ATTACH command must be provided in the attach entry
424 * point. When attach() is called with cmd set to DDI_ATTACH,
425 * all normal kernel services (such as kmem_alloc(9F)) are
426 * available for use by the driver.
427 *
428 * The attach() function will be called once for each instance
429 * of the device on the system with cmd set to DDI_ATTACH.
430 * Until attach() succeeds, the only driver entry points which
431 * may be called are open(9E) and getinfo(9E).
432 */
433 static int
ixgbe_attach(dev_info_t * devinfo,ddi_attach_cmd_t cmd)434 ixgbe_attach(dev_info_t *devinfo, ddi_attach_cmd_t cmd)
435 {
436 ixgbe_t *ixgbe;
437 struct ixgbe_osdep *osdep;
438 struct ixgbe_hw *hw;
439 int instance;
440 char taskqname[32];
441
442 /*
443 * Check the command and perform corresponding operations
444 */
445 switch (cmd) {
446 default:
447 return (DDI_FAILURE);
448
449 case DDI_RESUME:
450 return (ixgbe_resume(devinfo));
451
452 case DDI_ATTACH:
453 break;
454 }
455
456 /* Get the device instance */
457 instance = ddi_get_instance(devinfo);
458
459 /* Allocate memory for the instance data structure */
460 ixgbe = kmem_zalloc(sizeof (ixgbe_t), KM_SLEEP);
461
462 ixgbe->dip = devinfo;
463 ixgbe->instance = instance;
464
465 hw = &ixgbe->hw;
466 osdep = &ixgbe->osdep;
467 hw->back = osdep;
468 osdep->ixgbe = ixgbe;
469
470 /* Attach the instance pointer to the dev_info data structure */
471 ddi_set_driver_private(devinfo, ixgbe);
472
473 /*
474 * Initialize for FMA support
475 */
476 ixgbe->fm_capabilities = ixgbe_get_prop(ixgbe, PROP_FM_CAPABLE,
477 0, 0x0f, DDI_FM_EREPORT_CAPABLE | DDI_FM_ACCCHK_CAPABLE |
478 DDI_FM_DMACHK_CAPABLE | DDI_FM_ERRCB_CAPABLE);
479 ixgbe_fm_init(ixgbe);
480 ixgbe->attach_progress |= ATTACH_PROGRESS_FM_INIT;
481
482 /*
483 * Map PCI config space registers
484 */
485 if (pci_config_setup(devinfo, &osdep->cfg_handle) != DDI_SUCCESS) {
486 ixgbe_error(ixgbe, "Failed to map PCI configurations");
487 goto attach_fail;
488 }
489 ixgbe->attach_progress |= ATTACH_PROGRESS_PCI_CONFIG;
490
491 /*
492 * Identify the chipset family
493 */
494 if (ixgbe_identify_hardware(ixgbe) != IXGBE_SUCCESS) {
495 ixgbe_error(ixgbe, "Failed to identify hardware");
496 goto attach_fail;
497 }
498
499 /*
500 * Map device registers
501 */
502 if (ixgbe_regs_map(ixgbe) != IXGBE_SUCCESS) {
503 ixgbe_error(ixgbe, "Failed to map device registers");
504 goto attach_fail;
505 }
506 ixgbe->attach_progress |= ATTACH_PROGRESS_REGS_MAP;
507
508 /*
509 * Initialize driver parameters
510 */
511 ixgbe_init_properties(ixgbe);
512 ixgbe->attach_progress |= ATTACH_PROGRESS_PROPS;
513
514 /*
515 * Register interrupt callback
516 */
517 if (ixgbe_intr_cb_register(ixgbe) != IXGBE_SUCCESS) {
518 ixgbe_error(ixgbe, "Failed to register interrupt callback");
519 goto attach_fail;
520 }
521
522 /*
523 * Allocate interrupts
524 */
525 if (ixgbe_alloc_intrs(ixgbe) != IXGBE_SUCCESS) {
526 ixgbe_error(ixgbe, "Failed to allocate interrupts");
527 goto attach_fail;
528 }
529 ixgbe->attach_progress |= ATTACH_PROGRESS_ALLOC_INTR;
530
531 /*
532 * Allocate rx/tx rings based on the ring numbers.
533 * The actual numbers of rx/tx rings are decided by the number of
534 * allocated interrupt vectors, so we should allocate the rings after
535 * interrupts are allocated.
536 */
537 if (ixgbe_alloc_rings(ixgbe) != IXGBE_SUCCESS) {
538 ixgbe_error(ixgbe, "Failed to allocate rx and tx rings");
539 goto attach_fail;
540 }
541 ixgbe->attach_progress |= ATTACH_PROGRESS_ALLOC_RINGS;
542
543 /*
544 * Map rings to interrupt vectors
545 */
546 if (ixgbe_map_intrs_to_vectors(ixgbe) != IXGBE_SUCCESS) {
547 ixgbe_error(ixgbe, "Failed to map interrupts to vectors");
548 goto attach_fail;
549 }
550
551 /*
552 * Add interrupt handlers
553 */
554 if (ixgbe_add_intr_handlers(ixgbe) != IXGBE_SUCCESS) {
555 ixgbe_error(ixgbe, "Failed to add interrupt handlers");
556 goto attach_fail;
557 }
558 ixgbe->attach_progress |= ATTACH_PROGRESS_ADD_INTR;
559
560 /*
561 * Create a taskq for sfp-change
562 */
563 (void) sprintf(taskqname, "ixgbe%d_sfp_taskq", instance);
564 if ((ixgbe->sfp_taskq = ddi_taskq_create(devinfo, taskqname,
565 1, TASKQ_DEFAULTPRI, 0)) == NULL) {
566 ixgbe_error(ixgbe, "sfp_taskq create failed");
567 goto attach_fail;
568 }
569 ixgbe->attach_progress |= ATTACH_PROGRESS_SFP_TASKQ;
570
571 /*
572 * Create a taskq for over-temp
573 */
574 (void) sprintf(taskqname, "ixgbe%d_overtemp_taskq", instance);
575 if ((ixgbe->overtemp_taskq = ddi_taskq_create(devinfo, taskqname,
576 1, TASKQ_DEFAULTPRI, 0)) == NULL) {
577 ixgbe_error(ixgbe, "overtemp_taskq create failed");
578 goto attach_fail;
579 }
580 ixgbe->attach_progress |= ATTACH_PROGRESS_OVERTEMP_TASKQ;
581
582 /*
583 * Create a taskq for processing external PHY interrupts
584 */
585 (void) sprintf(taskqname, "ixgbe%d_phy_taskq", instance);
586 if ((ixgbe->phy_taskq = ddi_taskq_create(devinfo, taskqname,
587 1, TASKQ_DEFAULTPRI, 0)) == NULL) {
588 ixgbe_error(ixgbe, "phy_taskq create failed");
589 goto attach_fail;
590 }
591 ixgbe->attach_progress |= ATTACH_PROGRESS_PHY_TASKQ;
592
593 /*
594 * Initialize driver parameters
595 */
596 if (ixgbe_init_driver_settings(ixgbe) != IXGBE_SUCCESS) {
597 ixgbe_error(ixgbe, "Failed to initialize driver settings");
598 goto attach_fail;
599 }
600
601 /*
602 * Initialize mutexes for this device.
603 * Do this before enabling the interrupt handler and
604 * register the softint to avoid the condition where
605 * interrupt handler can try using uninitialized mutex.
606 */
607 ixgbe_init_locks(ixgbe);
608 ixgbe->attach_progress |= ATTACH_PROGRESS_LOCKS;
609
610 /*
611 * Initialize chipset hardware
612 */
613 if (ixgbe_init(ixgbe) != IXGBE_SUCCESS) {
614 ixgbe_error(ixgbe, "Failed to initialize adapter");
615 goto attach_fail;
616 }
617 ixgbe->link_check_complete = B_FALSE;
618 ixgbe->link_check_hrtime = gethrtime() +
619 (IXGBE_LINK_UP_TIME * 100000000ULL);
620 ixgbe->attach_progress |= ATTACH_PROGRESS_INIT;
621
622 if (ixgbe_check_acc_handle(ixgbe->osdep.cfg_handle) != DDI_FM_OK) {
623 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_LOST);
624 goto attach_fail;
625 }
626
627 /*
628 * Initialize adapter capabilities
629 */
630 ixgbe_init_params(ixgbe);
631
632 /*
633 * Initialize statistics
634 */
635 if (ixgbe_init_stats(ixgbe) != IXGBE_SUCCESS) {
636 ixgbe_error(ixgbe, "Failed to initialize statistics");
637 goto attach_fail;
638 }
639 ixgbe->attach_progress |= ATTACH_PROGRESS_STATS;
640
641 /*
642 * Register the driver to the MAC
643 */
644 if (ixgbe_register_mac(ixgbe) != IXGBE_SUCCESS) {
645 ixgbe_error(ixgbe, "Failed to register MAC");
646 goto attach_fail;
647 }
648 mac_link_update(ixgbe->mac_hdl, LINK_STATE_UNKNOWN);
649 ixgbe->attach_progress |= ATTACH_PROGRESS_MAC;
650
651 ixgbe->periodic_id = ddi_periodic_add(ixgbe_link_timer, ixgbe,
652 IXGBE_CYCLIC_PERIOD, DDI_IPL_0);
653 if (ixgbe->periodic_id == 0) {
654 ixgbe_error(ixgbe, "Failed to add the link check timer");
655 goto attach_fail;
656 }
657 ixgbe->attach_progress |= ATTACH_PROGRESS_LINK_TIMER;
658
659 /*
660 * Now that mutex locks are initialized, and the chip is also
661 * initialized, enable interrupts.
662 */
663 if (ixgbe_enable_intrs(ixgbe) != IXGBE_SUCCESS) {
664 ixgbe_error(ixgbe, "Failed to enable DDI interrupts");
665 goto attach_fail;
666 }
667 ixgbe->attach_progress |= ATTACH_PROGRESS_ENABLE_INTR;
668
669 if (ixgbe->hw.bus.func == 0) {
670 if (ddi_ufm_init(devinfo, DDI_UFM_CURRENT_VERSION,
671 &ixgbe_ufm_ops, &ixgbe->ixgbe_ufmh, ixgbe) != 0) {
672 ixgbe_error(ixgbe, "Failed to enable DDI UFM support");
673 goto attach_fail;
674 }
675 ixgbe->attach_progress |= ATTACH_PROGRESS_UFM;
676 ddi_ufm_update(ixgbe->ixgbe_ufmh);
677 }
678
679 ixgbe_log(ixgbe, "%s", ixgbe_ident);
680 atomic_or_32(&ixgbe->ixgbe_state, IXGBE_INITIALIZED);
681
682 return (DDI_SUCCESS);
683
684 attach_fail:
685 ixgbe_unconfigure(devinfo, ixgbe);
686 return (DDI_FAILURE);
687 }
688
689 /*
690 * ixgbe_detach - Driver detach.
691 *
692 * The detach() function is the complement of the attach routine.
693 * If cmd is set to DDI_DETACH, detach() is used to remove the
694 * state associated with a given instance of a device node
695 * prior to the removal of that instance from the system.
696 *
697 * The detach() function will be called once for each instance
698 * of the device for which there has been a successful attach()
699 * once there are no longer any opens on the device.
700 *
701 * Interrupts routine are disabled, All memory allocated by this
702 * driver are freed.
703 */
704 static int
ixgbe_detach(dev_info_t * devinfo,ddi_detach_cmd_t cmd)705 ixgbe_detach(dev_info_t *devinfo, ddi_detach_cmd_t cmd)
706 {
707 ixgbe_t *ixgbe;
708
709 /*
710 * Check detach command
711 */
712 switch (cmd) {
713 default:
714 return (DDI_FAILURE);
715
716 case DDI_SUSPEND:
717 return (ixgbe_suspend(devinfo));
718
719 case DDI_DETACH:
720 break;
721 }
722
723 /*
724 * Get the pointer to the driver private data structure
725 */
726 ixgbe = (ixgbe_t *)ddi_get_driver_private(devinfo);
727 if (ixgbe == NULL)
728 return (DDI_FAILURE);
729
730 /*
731 * If the device is still running, it needs to be stopped first.
732 * This check is necessary because under some specific circumstances,
733 * the detach routine can be called without stopping the interface
734 * first.
735 */
736 if (ixgbe->ixgbe_state & IXGBE_STARTED) {
737 atomic_and_32(&ixgbe->ixgbe_state, ~IXGBE_STARTED);
738 mutex_enter(&ixgbe->gen_lock);
739 ixgbe_stop(ixgbe, B_TRUE);
740 mutex_exit(&ixgbe->gen_lock);
741 /* Disable and stop the watchdog timer */
742 ixgbe_disable_watchdog_timer(ixgbe);
743 }
744
745 /*
746 * Check if there are still rx buffers held by the upper layer.
747 * If so, fail the detach.
748 */
749 if (!ixgbe_rx_drain(ixgbe))
750 return (DDI_FAILURE);
751
752 /*
753 * Do the remaining unconfigure routines
754 */
755 ixgbe_unconfigure(devinfo, ixgbe);
756
757 return (DDI_SUCCESS);
758 }
759
760 /*
761 * quiesce(9E) entry point.
762 *
763 * This function is called when the system is single-threaded at high
764 * PIL with preemption disabled. Therefore, this function must not be
765 * blocked.
766 *
767 * This function returns DDI_SUCCESS on success, or DDI_FAILURE on failure.
768 * DDI_FAILURE indicates an error condition and should almost never happen.
769 */
770 static int
ixgbe_quiesce(dev_info_t * devinfo)771 ixgbe_quiesce(dev_info_t *devinfo)
772 {
773 ixgbe_t *ixgbe;
774 struct ixgbe_hw *hw;
775
776 ixgbe = (ixgbe_t *)ddi_get_driver_private(devinfo);
777
778 if (ixgbe == NULL)
779 return (DDI_FAILURE);
780
781 hw = &ixgbe->hw;
782
783 /*
784 * Disable the adapter interrupts
785 */
786 ixgbe_disable_adapter_interrupts(ixgbe);
787
788 /*
789 * Tell firmware driver is no longer in control
790 */
791 ixgbe_release_driver_control(hw);
792
793 /*
794 * Reset the chipset
795 */
796 (void) ixgbe_reset_hw(hw);
797
798 /*
799 * Reset PHY
800 */
801 (void) ixgbe_reset_phy(hw);
802
803 return (DDI_SUCCESS);
804 }
805
806 static void
ixgbe_unconfigure(dev_info_t * devinfo,ixgbe_t * ixgbe)807 ixgbe_unconfigure(dev_info_t *devinfo, ixgbe_t *ixgbe)
808 {
809 /*
810 * Disable interrupt
811 */
812 if (ixgbe->attach_progress & ATTACH_PROGRESS_ENABLE_INTR) {
813 (void) ixgbe_disable_intrs(ixgbe);
814 }
815
816 /*
817 * remove the link check timer
818 */
819 if (ixgbe->attach_progress & ATTACH_PROGRESS_LINK_TIMER) {
820 if (ixgbe->periodic_id != NULL) {
821 ddi_periodic_delete(ixgbe->periodic_id);
822 ixgbe->periodic_id = NULL;
823 }
824 }
825
826 /*
827 * Clean up the UFM subsystem. Note this only is set on function 0.
828 */
829 if (ixgbe->attach_progress & ATTACH_PROGRESS_UFM) {
830 ddi_ufm_fini(ixgbe->ixgbe_ufmh);
831 }
832
833 /*
834 * Unregister MAC
835 */
836 if (ixgbe->attach_progress & ATTACH_PROGRESS_MAC) {
837 (void) mac_unregister(ixgbe->mac_hdl);
838 }
839
840 /*
841 * Free statistics
842 */
843 if (ixgbe->attach_progress & ATTACH_PROGRESS_STATS) {
844 kstat_delete((kstat_t *)ixgbe->ixgbe_ks);
845 }
846
847 /*
848 * Remove interrupt handlers
849 */
850 if (ixgbe->attach_progress & ATTACH_PROGRESS_ADD_INTR) {
851 ixgbe_rem_intr_handlers(ixgbe);
852 }
853
854 /*
855 * Remove taskq for sfp-status-change
856 */
857 if (ixgbe->attach_progress & ATTACH_PROGRESS_SFP_TASKQ) {
858 ddi_taskq_destroy(ixgbe->sfp_taskq);
859 }
860
861 /*
862 * Remove taskq for over-temp
863 */
864 if (ixgbe->attach_progress & ATTACH_PROGRESS_OVERTEMP_TASKQ) {
865 ddi_taskq_destroy(ixgbe->overtemp_taskq);
866 }
867
868 /*
869 * Remove taskq for external PHYs
870 */
871 if (ixgbe->attach_progress & ATTACH_PROGRESS_PHY_TASKQ) {
872 ddi_taskq_destroy(ixgbe->phy_taskq);
873 }
874
875 /*
876 * Remove interrupts
877 */
878 if (ixgbe->attach_progress & ATTACH_PROGRESS_ALLOC_INTR) {
879 ixgbe_rem_intrs(ixgbe);
880 }
881
882 /*
883 * Unregister interrupt callback handler
884 */
885 if (ixgbe->cb_hdl != NULL) {
886 (void) ddi_cb_unregister(ixgbe->cb_hdl);
887 }
888
889 /*
890 * Remove driver properties
891 */
892 if (ixgbe->attach_progress & ATTACH_PROGRESS_PROPS) {
893 (void) ddi_prop_remove_all(devinfo);
894 }
895
896 /*
897 * Stop the chipset
898 */
899 if (ixgbe->attach_progress & ATTACH_PROGRESS_INIT) {
900 mutex_enter(&ixgbe->gen_lock);
901 ixgbe_chip_stop(ixgbe);
902 mutex_exit(&ixgbe->gen_lock);
903 }
904
905 /*
906 * Free register handle
907 */
908 if (ixgbe->attach_progress & ATTACH_PROGRESS_REGS_MAP) {
909 if (ixgbe->osdep.reg_handle != NULL)
910 ddi_regs_map_free(&ixgbe->osdep.reg_handle);
911 }
912
913 /*
914 * Free PCI config handle
915 */
916 if (ixgbe->attach_progress & ATTACH_PROGRESS_PCI_CONFIG) {
917 if (ixgbe->osdep.cfg_handle != NULL)
918 pci_config_teardown(&ixgbe->osdep.cfg_handle);
919 }
920
921 /*
922 * Free locks
923 */
924 if (ixgbe->attach_progress & ATTACH_PROGRESS_LOCKS) {
925 ixgbe_destroy_locks(ixgbe);
926 }
927
928 /*
929 * Free the rx/tx rings
930 */
931 if (ixgbe->attach_progress & ATTACH_PROGRESS_ALLOC_RINGS) {
932 ixgbe_free_rings(ixgbe);
933 }
934
935 /*
936 * Unregister FMA capabilities
937 */
938 if (ixgbe->attach_progress & ATTACH_PROGRESS_FM_INIT) {
939 ixgbe_fm_fini(ixgbe);
940 }
941
942 /*
943 * Free the driver data structure
944 */
945 kmem_free(ixgbe, sizeof (ixgbe_t));
946
947 ddi_set_driver_private(devinfo, NULL);
948 }
949
950 /*
951 * ixgbe_register_mac - Register the driver and its function pointers with
952 * the GLD interface.
953 */
954 static int
ixgbe_register_mac(ixgbe_t * ixgbe)955 ixgbe_register_mac(ixgbe_t *ixgbe)
956 {
957 struct ixgbe_hw *hw = &ixgbe->hw;
958 mac_register_t *mac;
959 int status;
960
961 if ((mac = mac_alloc(MAC_VERSION)) == NULL)
962 return (IXGBE_FAILURE);
963
964 mac->m_type_ident = MAC_PLUGIN_IDENT_ETHER;
965 mac->m_driver = ixgbe;
966 mac->m_dip = ixgbe->dip;
967 mac->m_src_addr = hw->mac.addr;
968 mac->m_callbacks = &ixgbe_m_callbacks;
969 mac->m_min_sdu = 0;
970 mac->m_max_sdu = ixgbe->default_mtu;
971 mac->m_margin = VLAN_TAGSZ;
972 mac->m_priv_props = ixgbe_priv_props;
973 mac->m_v12n = MAC_VIRT_LEVEL1;
974
975 status = mac_register(mac, &ixgbe->mac_hdl);
976
977 mac_free(mac);
978
979 return ((status == 0) ? IXGBE_SUCCESS : IXGBE_FAILURE);
980 }
981
982 /*
983 * ixgbe_identify_hardware - Identify the type of the chipset.
984 */
985 static int
ixgbe_identify_hardware(ixgbe_t * ixgbe)986 ixgbe_identify_hardware(ixgbe_t *ixgbe)
987 {
988 struct ixgbe_hw *hw = &ixgbe->hw;
989 struct ixgbe_osdep *osdep = &ixgbe->osdep;
990
991 /*
992 * Get the device id
993 */
994 hw->vendor_id =
995 pci_config_get16(osdep->cfg_handle, PCI_CONF_VENID);
996 hw->device_id =
997 pci_config_get16(osdep->cfg_handle, PCI_CONF_DEVID);
998 hw->revision_id =
999 pci_config_get8(osdep->cfg_handle, PCI_CONF_REVID);
1000 hw->subsystem_device_id =
1001 pci_config_get16(osdep->cfg_handle, PCI_CONF_SUBSYSID);
1002 hw->subsystem_vendor_id =
1003 pci_config_get16(osdep->cfg_handle, PCI_CONF_SUBVENID);
1004
1005 /*
1006 * Set the mac type of the adapter based on the device id
1007 */
1008 if (ixgbe_set_mac_type(hw) != IXGBE_SUCCESS) {
1009 return (IXGBE_FAILURE);
1010 }
1011
1012 /*
1013 * Install adapter capabilities
1014 */
1015 switch (hw->mac.type) {
1016 case ixgbe_mac_82598EB:
1017 IXGBE_DEBUGLOG_0(ixgbe, "identify 82598 adapter\n");
1018 ixgbe->capab = &ixgbe_82598eb_cap;
1019
1020 if (ixgbe_get_media_type(hw) == ixgbe_media_type_copper) {
1021 ixgbe->capab->flags |= IXGBE_FLAG_FAN_FAIL_CAPABLE;
1022 ixgbe->capab->other_intr |= IXGBE_EICR_GPI_SDP1;
1023 ixgbe->capab->other_gpie |= IXGBE_SDP1_GPIEN;
1024 }
1025 break;
1026
1027 case ixgbe_mac_82599EB:
1028 IXGBE_DEBUGLOG_0(ixgbe, "identify 82599 adapter\n");
1029 ixgbe->capab = &ixgbe_82599eb_cap;
1030
1031 if (hw->device_id == IXGBE_DEV_ID_82599_T3_LOM) {
1032 ixgbe->capab->flags |= IXGBE_FLAG_TEMP_SENSOR_CAPABLE;
1033 ixgbe->capab->other_intr |= IXGBE_EICR_GPI_SDP0;
1034 ixgbe->capab->other_gpie |= IXGBE_SDP0_GPIEN;
1035 }
1036 break;
1037
1038 case ixgbe_mac_X540:
1039 IXGBE_DEBUGLOG_0(ixgbe, "identify X540 adapter\n");
1040 ixgbe->capab = &ixgbe_X540_cap;
1041 /*
1042 * For now, X540 is all set in its capab structure.
1043 * As other X540 variants show up, things can change here.
1044 */
1045 break;
1046
1047 case ixgbe_mac_X550:
1048 case ixgbe_mac_X550EM_x:
1049 case ixgbe_mac_X550EM_a:
1050 IXGBE_DEBUGLOG_0(ixgbe, "identify X550 adapter\n");
1051 ixgbe->capab = &ixgbe_X550_cap;
1052
1053 if (hw->device_id == IXGBE_DEV_ID_X550EM_X_SFP ||
1054 hw->device_id == IXGBE_DEV_ID_X550EM_A_SFP ||
1055 hw->device_id == IXGBE_DEV_ID_X550EM_A_SFP_N ||
1056 hw->device_id == IXGBE_DEV_ID_X550EM_A_QSFP ||
1057 hw->device_id == IXGBE_DEV_ID_X550EM_A_QSFP_N) {
1058 ixgbe->capab->flags |= IXGBE_FLAG_SFP_PLUG_CAPABLE;
1059 }
1060
1061 /*
1062 * Link detection on X552 SFP+ and X552/X557-AT
1063 */
1064 if (hw->device_id == IXGBE_DEV_ID_X550EM_X_SFP ||
1065 hw->device_id == IXGBE_DEV_ID_X550EM_A_SFP ||
1066 hw->device_id == IXGBE_DEV_ID_X550EM_A_SFP_N ||
1067 hw->device_id == IXGBE_DEV_ID_X550EM_X_10G_T) {
1068 ixgbe->capab->other_intr |=
1069 IXGBE_EIMS_GPI_SDP0_BY_MAC(hw);
1070 }
1071 if (hw->phy.type == ixgbe_phy_x550em_ext_t) {
1072 ixgbe->capab->other_gpie |= IXGBE_SDP0_GPIEN_X540;
1073 }
1074 break;
1075
1076 default:
1077 IXGBE_DEBUGLOG_1(ixgbe,
1078 "adapter not supported in ixgbe_identify_hardware(): %d\n",
1079 hw->mac.type);
1080 return (IXGBE_FAILURE);
1081 }
1082
1083 return (IXGBE_SUCCESS);
1084 }
1085
1086 /*
1087 * ixgbe_regs_map - Map the device registers.
1088 *
1089 */
1090 static int
ixgbe_regs_map(ixgbe_t * ixgbe)1091 ixgbe_regs_map(ixgbe_t *ixgbe)
1092 {
1093 dev_info_t *devinfo = ixgbe->dip;
1094 struct ixgbe_hw *hw = &ixgbe->hw;
1095 struct ixgbe_osdep *osdep = &ixgbe->osdep;
1096 off_t mem_size;
1097
1098 /*
1099 * First get the size of device registers to be mapped.
1100 */
1101 if (ddi_dev_regsize(devinfo, IXGBE_ADAPTER_REGSET, &mem_size)
1102 != DDI_SUCCESS) {
1103 return (IXGBE_FAILURE);
1104 }
1105
1106 /*
1107 * Call ddi_regs_map_setup() to map registers
1108 */
1109 if ((ddi_regs_map_setup(devinfo, IXGBE_ADAPTER_REGSET,
1110 (caddr_t *)&hw->hw_addr, 0,
1111 mem_size, &ixgbe_regs_acc_attr,
1112 &osdep->reg_handle)) != DDI_SUCCESS) {
1113 return (IXGBE_FAILURE);
1114 }
1115
1116 return (IXGBE_SUCCESS);
1117 }
1118
1119 /*
1120 * ixgbe_init_properties - Initialize driver properties.
1121 */
1122 static void
ixgbe_init_properties(ixgbe_t * ixgbe)1123 ixgbe_init_properties(ixgbe_t *ixgbe)
1124 {
1125 /*
1126 * Get conf file properties, including link settings
1127 * jumbo frames, ring number, descriptor number, etc.
1128 */
1129 ixgbe_get_conf(ixgbe);
1130 }
1131
1132 /*
1133 * ixgbe_init_driver_settings - Initialize driver settings.
1134 *
1135 * The settings include hardware function pointers, bus information,
1136 * rx/tx rings settings, link state, and any other parameters that
1137 * need to be setup during driver initialization.
1138 */
1139 static int
ixgbe_init_driver_settings(ixgbe_t * ixgbe)1140 ixgbe_init_driver_settings(ixgbe_t *ixgbe)
1141 {
1142 struct ixgbe_hw *hw = &ixgbe->hw;
1143 dev_info_t *devinfo = ixgbe->dip;
1144 ixgbe_rx_ring_t *rx_ring;
1145 ixgbe_rx_group_t *rx_group;
1146 ixgbe_tx_ring_t *tx_ring;
1147 uint32_t rx_size;
1148 uint32_t tx_size;
1149 uint32_t ring_per_group;
1150 int i;
1151
1152 /*
1153 * Initialize chipset specific hardware function pointers
1154 */
1155 if (ixgbe_init_shared_code(hw) != IXGBE_SUCCESS) {
1156 return (IXGBE_FAILURE);
1157 }
1158
1159 /*
1160 * Get the system page size
1161 */
1162 ixgbe->sys_page_size = ddi_ptob(devinfo, (ulong_t)1);
1163
1164 /*
1165 * Set rx buffer size
1166 *
1167 * The IP header alignment room is counted in the calculation.
1168 * The rx buffer size is in unit of 1K that is required by the
1169 * chipset hardware.
1170 */
1171 rx_size = ixgbe->max_frame_size + IPHDR_ALIGN_ROOM;
1172 ixgbe->rx_buf_size = ((rx_size >> 10) +
1173 ((rx_size & (((uint32_t)1 << 10) - 1)) > 0 ? 1 : 0)) << 10;
1174
1175 /*
1176 * Set tx buffer size
1177 */
1178 tx_size = ixgbe->max_frame_size;
1179 ixgbe->tx_buf_size = ((tx_size >> 10) +
1180 ((tx_size & (((uint32_t)1 << 10) - 1)) > 0 ? 1 : 0)) << 10;
1181
1182 /*
1183 * Initialize rx/tx rings/groups parameters
1184 */
1185 ring_per_group = ixgbe->num_rx_rings / ixgbe->num_rx_groups;
1186 for (i = 0; i < ixgbe->num_rx_rings; i++) {
1187 rx_ring = &ixgbe->rx_rings[i];
1188 rx_ring->index = i;
1189 rx_ring->ixgbe = ixgbe;
1190 rx_ring->group_index = i / ring_per_group;
1191 rx_ring->hw_index = ixgbe_get_hw_rx_index(ixgbe, i);
1192 }
1193
1194 for (i = 0; i < ixgbe->num_rx_groups; i++) {
1195 rx_group = &ixgbe->rx_groups[i];
1196 rx_group->index = i;
1197 rx_group->ixgbe = ixgbe;
1198 list_create(&rx_group->vlans, sizeof (ixgbe_vlan_t),
1199 offsetof(ixgbe_vlan_t, ixvl_link));
1200 }
1201
1202 for (i = 0; i < ixgbe->num_tx_rings; i++) {
1203 tx_ring = &ixgbe->tx_rings[i];
1204 tx_ring->index = i;
1205 tx_ring->ixgbe = ixgbe;
1206 if (ixgbe->tx_head_wb_enable)
1207 tx_ring->tx_recycle = ixgbe_tx_recycle_head_wb;
1208 else
1209 tx_ring->tx_recycle = ixgbe_tx_recycle_legacy;
1210
1211 tx_ring->ring_size = ixgbe->tx_ring_size;
1212 tx_ring->free_list_size = ixgbe->tx_ring_size +
1213 (ixgbe->tx_ring_size >> 1);
1214 }
1215
1216 /*
1217 * Initialize values of interrupt throttling rate
1218 */
1219 for (i = 1; i < MAX_INTR_VECTOR; i++)
1220 ixgbe->intr_throttling[i] = ixgbe->intr_throttling[0];
1221
1222 /*
1223 * The initial link state should be "unknown"
1224 */
1225 ixgbe->link_state = LINK_STATE_UNKNOWN;
1226
1227 return (IXGBE_SUCCESS);
1228 }
1229
1230 /*
1231 * ixgbe_init_locks - Initialize locks.
1232 */
1233 static void
ixgbe_init_locks(ixgbe_t * ixgbe)1234 ixgbe_init_locks(ixgbe_t *ixgbe)
1235 {
1236 ixgbe_rx_ring_t *rx_ring;
1237 ixgbe_tx_ring_t *tx_ring;
1238 int i;
1239
1240 for (i = 0; i < ixgbe->num_rx_rings; i++) {
1241 rx_ring = &ixgbe->rx_rings[i];
1242 mutex_init(&rx_ring->rx_lock, NULL,
1243 MUTEX_DRIVER, DDI_INTR_PRI(ixgbe->intr_pri));
1244 }
1245
1246 for (i = 0; i < ixgbe->num_tx_rings; i++) {
1247 tx_ring = &ixgbe->tx_rings[i];
1248 mutex_init(&tx_ring->tx_lock, NULL,
1249 MUTEX_DRIVER, DDI_INTR_PRI(ixgbe->intr_pri));
1250 mutex_init(&tx_ring->recycle_lock, NULL,
1251 MUTEX_DRIVER, DDI_INTR_PRI(ixgbe->intr_pri));
1252 mutex_init(&tx_ring->tcb_head_lock, NULL,
1253 MUTEX_DRIVER, DDI_INTR_PRI(ixgbe->intr_pri));
1254 mutex_init(&tx_ring->tcb_tail_lock, NULL,
1255 MUTEX_DRIVER, DDI_INTR_PRI(ixgbe->intr_pri));
1256 }
1257
1258 mutex_init(&ixgbe->gen_lock, NULL,
1259 MUTEX_DRIVER, DDI_INTR_PRI(ixgbe->intr_pri));
1260
1261 mutex_init(&ixgbe->watchdog_lock, NULL,
1262 MUTEX_DRIVER, DDI_INTR_PRI(ixgbe->intr_pri));
1263 }
1264
1265 /*
1266 * ixgbe_destroy_locks - Destroy locks.
1267 */
1268 static void
ixgbe_destroy_locks(ixgbe_t * ixgbe)1269 ixgbe_destroy_locks(ixgbe_t *ixgbe)
1270 {
1271 ixgbe_rx_ring_t *rx_ring;
1272 ixgbe_tx_ring_t *tx_ring;
1273 int i;
1274
1275 for (i = 0; i < ixgbe->num_rx_rings; i++) {
1276 rx_ring = &ixgbe->rx_rings[i];
1277 mutex_destroy(&rx_ring->rx_lock);
1278 }
1279
1280 for (i = 0; i < ixgbe->num_tx_rings; i++) {
1281 tx_ring = &ixgbe->tx_rings[i];
1282 mutex_destroy(&tx_ring->tx_lock);
1283 mutex_destroy(&tx_ring->recycle_lock);
1284 mutex_destroy(&tx_ring->tcb_head_lock);
1285 mutex_destroy(&tx_ring->tcb_tail_lock);
1286 }
1287
1288 mutex_destroy(&ixgbe->gen_lock);
1289 mutex_destroy(&ixgbe->watchdog_lock);
1290 }
1291
1292 /*
1293 * We need to try and determine which LED index in hardware corresponds to the
1294 * link/activity LED. This is the one that'll be overwritten when we perform
1295 * GLDv3 LED activity.
1296 */
1297 static void
ixgbe_led_init(ixgbe_t * ixgbe)1298 ixgbe_led_init(ixgbe_t *ixgbe)
1299 {
1300 uint32_t reg, i;
1301 struct ixgbe_hw *hw = &ixgbe->hw;
1302
1303 reg = IXGBE_READ_REG(hw, IXGBE_LEDCTL);
1304 for (i = 0; i < 4; i++) {
1305 if (((reg >> IXGBE_LED_MODE_SHIFT(i)) &
1306 IXGBE_LED_MODE_MASK_BASE) == IXGBE_LED_LINK_ACTIVE) {
1307 ixgbe->ixgbe_led_index = i;
1308 return;
1309 }
1310 }
1311
1312 /*
1313 * If we couldn't determine this, we use the default for various MACs
1314 * based on information Intel has inserted into other drivers over the
1315 * years.
1316 */
1317 switch (hw->mac.type) {
1318 case ixgbe_mac_X550EM_a:
1319 ixgbe->ixgbe_led_index = 0;
1320 break;
1321 case ixgbe_mac_X550EM_x:
1322 ixgbe->ixgbe_led_index = 1;
1323 break;
1324 default:
1325 ixgbe->ixgbe_led_index = 2;
1326 break;
1327 }
1328 }
1329
1330 static int
ixgbe_resume(dev_info_t * devinfo)1331 ixgbe_resume(dev_info_t *devinfo)
1332 {
1333 ixgbe_t *ixgbe;
1334 int i;
1335
1336 ixgbe = (ixgbe_t *)ddi_get_driver_private(devinfo);
1337 if (ixgbe == NULL)
1338 return (DDI_FAILURE);
1339
1340 mutex_enter(&ixgbe->gen_lock);
1341
1342 if (ixgbe->ixgbe_state & IXGBE_STARTED) {
1343 if (ixgbe_start(ixgbe, B_FALSE) != IXGBE_SUCCESS) {
1344 mutex_exit(&ixgbe->gen_lock);
1345 return (DDI_FAILURE);
1346 }
1347
1348 /*
1349 * Enable and start the watchdog timer
1350 */
1351 ixgbe_enable_watchdog_timer(ixgbe);
1352 }
1353
1354 atomic_and_32(&ixgbe->ixgbe_state, ~IXGBE_SUSPENDED);
1355
1356 if (ixgbe->ixgbe_state & IXGBE_STARTED) {
1357 for (i = 0; i < ixgbe->num_tx_rings; i++) {
1358 mac_tx_ring_update(ixgbe->mac_hdl,
1359 ixgbe->tx_rings[i].ring_handle);
1360 }
1361 }
1362
1363 mutex_exit(&ixgbe->gen_lock);
1364
1365 return (DDI_SUCCESS);
1366 }
1367
1368 static int
ixgbe_suspend(dev_info_t * devinfo)1369 ixgbe_suspend(dev_info_t *devinfo)
1370 {
1371 ixgbe_t *ixgbe;
1372
1373 ixgbe = (ixgbe_t *)ddi_get_driver_private(devinfo);
1374 if (ixgbe == NULL)
1375 return (DDI_FAILURE);
1376
1377 mutex_enter(&ixgbe->gen_lock);
1378
1379 atomic_or_32(&ixgbe->ixgbe_state, IXGBE_SUSPENDED);
1380 if (!(ixgbe->ixgbe_state & IXGBE_STARTED)) {
1381 mutex_exit(&ixgbe->gen_lock);
1382 return (DDI_SUCCESS);
1383 }
1384 ixgbe_stop(ixgbe, B_FALSE);
1385
1386 mutex_exit(&ixgbe->gen_lock);
1387
1388 /*
1389 * Disable and stop the watchdog timer
1390 */
1391 ixgbe_disable_watchdog_timer(ixgbe);
1392
1393 return (DDI_SUCCESS);
1394 }
1395
1396 /*
1397 * ixgbe_init - Initialize the device.
1398 */
1399 static int
ixgbe_init(ixgbe_t * ixgbe)1400 ixgbe_init(ixgbe_t *ixgbe)
1401 {
1402 struct ixgbe_hw *hw = &ixgbe->hw;
1403 u8 pbanum[IXGBE_PBANUM_LENGTH];
1404 int rv;
1405
1406 mutex_enter(&ixgbe->gen_lock);
1407
1408 /*
1409 * Configure/Initialize hardware
1410 */
1411 rv = ixgbe_init_hw(hw);
1412 if (rv != IXGBE_SUCCESS) {
1413 switch (rv) {
1414
1415 /*
1416 * The first three errors are not prohibitive to us progressing
1417 * further, and are maily advisory in nature. In the case of a
1418 * SFP module not being present or not deemed supported by the
1419 * common code, we adivse the operator of this fact but carry on
1420 * instead of failing hard, as SFPs can be inserted or replaced
1421 * while the driver is running. In the case of a unknown error,
1422 * we fail-hard, logging the reason and emitting a FMA event.
1423 */
1424 case IXGBE_ERR_EEPROM_VERSION:
1425 ixgbe_error(ixgbe,
1426 "This Intel 10Gb Ethernet device is pre-release and"
1427 " contains outdated firmware. Please contact your"
1428 " hardware vendor for a replacement.");
1429 break;
1430 case IXGBE_ERR_SFP_NOT_PRESENT:
1431 ixgbe_error(ixgbe,
1432 "No SFP+ module detected on this interface. Please "
1433 "install a supported SFP+ module for this "
1434 "interface to become operational.");
1435 break;
1436 case IXGBE_ERR_SFP_NOT_SUPPORTED:
1437 ixgbe_error(ixgbe,
1438 "Unsupported SFP+ module detected. Please replace "
1439 "it with a supported SFP+ module per Intel "
1440 "documentation, or bypass this check with "
1441 "allow_unsupported_sfp=1 in ixgbe.conf.");
1442 break;
1443 default:
1444 ixgbe_error(ixgbe,
1445 "Failed to initialize hardware. ixgbe_init_hw "
1446 "returned %d", rv);
1447 ixgbe_fm_ereport(ixgbe, DDI_FM_DEVICE_INVAL_STATE);
1448 goto init_fail;
1449 }
1450 }
1451
1452 /*
1453 * Need to init eeprom before validating the checksum.
1454 */
1455 if (ixgbe_init_eeprom_params(hw) < 0) {
1456 ixgbe_error(ixgbe,
1457 "Unable to intitialize the eeprom interface.");
1458 ixgbe_fm_ereport(ixgbe, DDI_FM_DEVICE_INVAL_STATE);
1459 goto init_fail;
1460 }
1461
1462 /*
1463 * NVM validation
1464 */
1465 if (ixgbe_validate_eeprom_checksum(hw, NULL) < 0) {
1466 /*
1467 * Some PCI-E parts fail the first check due to
1468 * the link being in sleep state. Call it again,
1469 * if it fails a second time it's a real issue.
1470 */
1471 if (ixgbe_validate_eeprom_checksum(hw, NULL) < 0) {
1472 ixgbe_error(ixgbe,
1473 "Invalid NVM checksum. Please contact "
1474 "the vendor to update the NVM.");
1475 ixgbe_fm_ereport(ixgbe, DDI_FM_DEVICE_INVAL_STATE);
1476 goto init_fail;
1477 }
1478 }
1479
1480 /*
1481 * Setup default flow control thresholds - enable/disable
1482 * & flow control type is controlled by ixgbe.conf
1483 */
1484 hw->fc.high_water[0] = DEFAULT_FCRTH;
1485 hw->fc.low_water[0] = DEFAULT_FCRTL;
1486 hw->fc.pause_time = DEFAULT_FCPAUSE;
1487 hw->fc.send_xon = B_TRUE;
1488
1489 /*
1490 * Initialize flow control
1491 */
1492 (void) ixgbe_start_hw(hw);
1493
1494 /*
1495 * Initialize link settings
1496 */
1497 (void) ixgbe_driver_setup_link(ixgbe, B_FALSE);
1498
1499 /*
1500 * Initialize the chipset hardware
1501 */
1502 if (ixgbe_chip_start(ixgbe) != IXGBE_SUCCESS) {
1503 ixgbe_fm_ereport(ixgbe, DDI_FM_DEVICE_INVAL_STATE);
1504 goto init_fail;
1505 }
1506
1507 /*
1508 * Read identifying information and place in devinfo.
1509 */
1510 pbanum[0] = '\0';
1511 (void) ixgbe_read_pba_string(hw, pbanum, sizeof (pbanum));
1512 if (*pbanum != '\0') {
1513 (void) ddi_prop_update_string(DDI_DEV_T_NONE, ixgbe->dip,
1514 "printed-board-assembly", (char *)pbanum);
1515 }
1516
1517 /*
1518 * Determine LED index.
1519 */
1520 ixgbe_led_init(ixgbe);
1521
1522 if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != DDI_FM_OK) {
1523 goto init_fail;
1524 }
1525
1526 mutex_exit(&ixgbe->gen_lock);
1527 return (IXGBE_SUCCESS);
1528
1529 init_fail:
1530 /*
1531 * Reset PHY
1532 */
1533 (void) ixgbe_reset_phy(hw);
1534
1535 mutex_exit(&ixgbe->gen_lock);
1536 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_LOST);
1537 return (IXGBE_FAILURE);
1538 }
1539
1540 /*
1541 * ixgbe_chip_start - Initialize and start the chipset hardware.
1542 */
1543 static int
ixgbe_chip_start(ixgbe_t * ixgbe)1544 ixgbe_chip_start(ixgbe_t *ixgbe)
1545 {
1546 struct ixgbe_hw *hw = &ixgbe->hw;
1547 int i;
1548
1549 ASSERT(mutex_owned(&ixgbe->gen_lock));
1550
1551 /*
1552 * Get the mac address
1553 * This function should handle SPARC case correctly.
1554 */
1555 if (!ixgbe_find_mac_address(ixgbe)) {
1556 ixgbe_error(ixgbe, "Failed to get the mac address");
1557 return (IXGBE_FAILURE);
1558 }
1559
1560 /*
1561 * Validate the mac address
1562 */
1563 (void) ixgbe_init_rx_addrs(hw);
1564 if (!is_valid_mac_addr(hw->mac.addr)) {
1565 ixgbe_error(ixgbe, "Invalid mac address");
1566 return (IXGBE_FAILURE);
1567 }
1568
1569 /*
1570 * Re-enable relaxed ordering for performance. It is disabled
1571 * by default in the hardware init.
1572 */
1573 if (ixgbe->relax_order_enable == B_TRUE)
1574 ixgbe_enable_relaxed_ordering(hw);
1575
1576 /*
1577 * Setup adapter interrupt vectors
1578 */
1579 ixgbe_setup_adapter_vector(ixgbe);
1580
1581 /*
1582 * Initialize unicast addresses.
1583 */
1584 ixgbe_init_unicst(ixgbe);
1585
1586 /*
1587 * Setup and initialize the mctable structures.
1588 */
1589 ixgbe_setup_multicst(ixgbe);
1590
1591 /*
1592 * Set interrupt throttling rate
1593 */
1594 for (i = 0; i < ixgbe->intr_cnt; i++) {
1595 IXGBE_WRITE_REG(hw, IXGBE_EITR(i), ixgbe->intr_throttling[i]);
1596 }
1597
1598 /*
1599 * Disable Wake-on-LAN
1600 */
1601 IXGBE_WRITE_REG(hw, IXGBE_WUC, 0);
1602
1603 /*
1604 * Some adapters offer Energy Efficient Ethernet (EEE) support.
1605 * Due to issues with EEE in e1000g/igb, we disable this by default
1606 * as a precautionary measure.
1607 *
1608 * Currently, this is present on a number of the X550 family parts.
1609 */
1610 (void) ixgbe_setup_eee(hw, B_FALSE);
1611
1612 /*
1613 * Turn on any present SFP Tx laser
1614 */
1615 ixgbe_enable_tx_laser(hw);
1616
1617 /*
1618 * Power on the PHY
1619 */
1620 (void) ixgbe_set_phy_power(hw, B_TRUE);
1621
1622 /*
1623 * Save the state of the PHY
1624 */
1625 ixgbe_get_hw_state(ixgbe);
1626
1627 /*
1628 * Make sure driver has control
1629 */
1630 ixgbe_get_driver_control(hw);
1631
1632 return (IXGBE_SUCCESS);
1633 }
1634
1635 /*
1636 * ixgbe_chip_stop - Stop the chipset hardware
1637 */
1638 static void
ixgbe_chip_stop(ixgbe_t * ixgbe)1639 ixgbe_chip_stop(ixgbe_t *ixgbe)
1640 {
1641 struct ixgbe_hw *hw = &ixgbe->hw;
1642 int rv;
1643
1644 ASSERT(mutex_owned(&ixgbe->gen_lock));
1645
1646 /*
1647 * Stop interupt generation and disable Tx unit
1648 */
1649 hw->adapter_stopped = B_FALSE;
1650 (void) ixgbe_stop_adapter(hw);
1651
1652 /*
1653 * Reset the chipset
1654 */
1655 (void) ixgbe_reset_hw(hw);
1656
1657 /*
1658 * Reset PHY
1659 */
1660 (void) ixgbe_reset_phy(hw);
1661
1662 /*
1663 * Enter LPLU (Low Power, Link Up) mode, if available. Avoid resetting
1664 * the PHY while doing so. Else, just power down the PHY.
1665 */
1666 if (hw->phy.ops.enter_lplu != NULL) {
1667 hw->phy.reset_disable = B_TRUE;
1668 rv = hw->phy.ops.enter_lplu(hw);
1669 if (rv != IXGBE_SUCCESS)
1670 ixgbe_error(ixgbe, "Error while entering LPLU: %d", rv);
1671 hw->phy.reset_disable = B_FALSE;
1672 } else {
1673 (void) ixgbe_set_phy_power(hw, B_FALSE);
1674 }
1675
1676 /*
1677 * Turn off any present SFP Tx laser
1678 * Expected for health and safety reasons
1679 */
1680 ixgbe_disable_tx_laser(hw);
1681
1682 /*
1683 * Tell firmware driver is no longer in control
1684 */
1685 ixgbe_release_driver_control(hw);
1686
1687 }
1688
1689 /*
1690 * ixgbe_reset - Reset the chipset and re-start the driver.
1691 *
1692 * It involves stopping and re-starting the chipset,
1693 * and re-configuring the rx/tx rings.
1694 */
1695 static int
ixgbe_reset(ixgbe_t * ixgbe)1696 ixgbe_reset(ixgbe_t *ixgbe)
1697 {
1698 int i;
1699
1700 /*
1701 * Disable and stop the watchdog timer
1702 */
1703 ixgbe_disable_watchdog_timer(ixgbe);
1704
1705 mutex_enter(&ixgbe->gen_lock);
1706
1707 ASSERT(ixgbe->ixgbe_state & IXGBE_STARTED);
1708 atomic_and_32(&ixgbe->ixgbe_state, ~IXGBE_STARTED);
1709
1710 ixgbe_stop(ixgbe, B_FALSE);
1711
1712 if (ixgbe_start(ixgbe, B_FALSE) != IXGBE_SUCCESS) {
1713 mutex_exit(&ixgbe->gen_lock);
1714 return (IXGBE_FAILURE);
1715 }
1716
1717 /*
1718 * After resetting, need to recheck the link status.
1719 */
1720 ixgbe->link_check_complete = B_FALSE;
1721 ixgbe->link_check_hrtime = gethrtime() +
1722 (IXGBE_LINK_UP_TIME * 100000000ULL);
1723
1724 atomic_or_32(&ixgbe->ixgbe_state, IXGBE_STARTED);
1725
1726 if (!(ixgbe->ixgbe_state & IXGBE_SUSPENDED)) {
1727 for (i = 0; i < ixgbe->num_tx_rings; i++) {
1728 mac_tx_ring_update(ixgbe->mac_hdl,
1729 ixgbe->tx_rings[i].ring_handle);
1730 }
1731 }
1732
1733 mutex_exit(&ixgbe->gen_lock);
1734
1735 /*
1736 * Enable and start the watchdog timer
1737 */
1738 ixgbe_enable_watchdog_timer(ixgbe);
1739
1740 return (IXGBE_SUCCESS);
1741 }
1742
1743 /*
1744 * ixgbe_tx_clean - Clean the pending transmit packets and DMA resources.
1745 */
1746 static void
ixgbe_tx_clean(ixgbe_t * ixgbe)1747 ixgbe_tx_clean(ixgbe_t *ixgbe)
1748 {
1749 ixgbe_tx_ring_t *tx_ring;
1750 tx_control_block_t *tcb;
1751 link_list_t pending_list;
1752 uint32_t desc_num;
1753 int i, j;
1754
1755 LINK_LIST_INIT(&pending_list);
1756
1757 for (i = 0; i < ixgbe->num_tx_rings; i++) {
1758 tx_ring = &ixgbe->tx_rings[i];
1759
1760 mutex_enter(&tx_ring->recycle_lock);
1761
1762 /*
1763 * Clean the pending tx data - the pending packets in the
1764 * work_list that have no chances to be transmitted again.
1765 *
1766 * We must ensure the chipset is stopped or the link is down
1767 * before cleaning the transmit packets.
1768 */
1769 desc_num = 0;
1770 for (j = 0; j < tx_ring->ring_size; j++) {
1771 tcb = tx_ring->work_list[j];
1772 if (tcb != NULL) {
1773 desc_num += tcb->desc_num;
1774
1775 tx_ring->work_list[j] = NULL;
1776
1777 ixgbe_free_tcb(tcb);
1778
1779 LIST_PUSH_TAIL(&pending_list, &tcb->link);
1780 }
1781 }
1782
1783 if (desc_num > 0) {
1784 atomic_add_32(&tx_ring->tbd_free, desc_num);
1785 ASSERT(tx_ring->tbd_free == tx_ring->ring_size);
1786
1787 /*
1788 * Reset the head and tail pointers of the tbd ring;
1789 * Reset the writeback head if it's enable.
1790 */
1791 tx_ring->tbd_head = 0;
1792 tx_ring->tbd_tail = 0;
1793 if (ixgbe->tx_head_wb_enable)
1794 *tx_ring->tbd_head_wb = 0;
1795
1796 IXGBE_WRITE_REG(&ixgbe->hw,
1797 IXGBE_TDH(tx_ring->index), 0);
1798 IXGBE_WRITE_REG(&ixgbe->hw,
1799 IXGBE_TDT(tx_ring->index), 0);
1800 }
1801
1802 mutex_exit(&tx_ring->recycle_lock);
1803
1804 /*
1805 * Add the tx control blocks in the pending list to
1806 * the free list.
1807 */
1808 ixgbe_put_free_list(tx_ring, &pending_list);
1809 }
1810 }
1811
1812 /*
1813 * ixgbe_tx_drain - Drain the tx rings to allow pending packets to be
1814 * transmitted.
1815 */
1816 static boolean_t
ixgbe_tx_drain(ixgbe_t * ixgbe)1817 ixgbe_tx_drain(ixgbe_t *ixgbe)
1818 {
1819 ixgbe_tx_ring_t *tx_ring;
1820 boolean_t done;
1821 int i, j;
1822
1823 /*
1824 * Wait for a specific time to allow pending tx packets
1825 * to be transmitted.
1826 *
1827 * Check the counter tbd_free to see if transmission is done.
1828 * No lock protection is needed here.
1829 *
1830 * Return B_TRUE if all pending packets have been transmitted;
1831 * Otherwise return B_FALSE;
1832 */
1833 for (i = 0; i < TX_DRAIN_TIME; i++) {
1834
1835 done = B_TRUE;
1836 for (j = 0; j < ixgbe->num_tx_rings; j++) {
1837 tx_ring = &ixgbe->tx_rings[j];
1838 done = done &&
1839 (tx_ring->tbd_free == tx_ring->ring_size);
1840 }
1841
1842 if (done)
1843 break;
1844
1845 msec_delay(1);
1846 }
1847
1848 return (done);
1849 }
1850
1851 /*
1852 * ixgbe_rx_drain - Wait for all rx buffers to be released by upper layer.
1853 */
1854 static boolean_t
ixgbe_rx_drain(ixgbe_t * ixgbe)1855 ixgbe_rx_drain(ixgbe_t *ixgbe)
1856 {
1857 boolean_t done = B_TRUE;
1858 int i;
1859
1860 /*
1861 * Polling the rx free list to check if those rx buffers held by
1862 * the upper layer are released.
1863 *
1864 * Check the counter rcb_free to see if all pending buffers are
1865 * released. No lock protection is needed here.
1866 *
1867 * Return B_TRUE if all pending buffers have been released;
1868 * Otherwise return B_FALSE;
1869 */
1870 for (i = 0; i < RX_DRAIN_TIME; i++) {
1871 done = (ixgbe->rcb_pending == 0);
1872
1873 if (done)
1874 break;
1875
1876 msec_delay(1);
1877 }
1878
1879 return (done);
1880 }
1881
1882 /*
1883 * ixgbe_start - Start the driver/chipset.
1884 */
1885 int
ixgbe_start(ixgbe_t * ixgbe,boolean_t alloc_buffer)1886 ixgbe_start(ixgbe_t *ixgbe, boolean_t alloc_buffer)
1887 {
1888 struct ixgbe_hw *hw = &ixgbe->hw;
1889 int i;
1890
1891 ASSERT(mutex_owned(&ixgbe->gen_lock));
1892
1893 if (alloc_buffer) {
1894 if (ixgbe_alloc_rx_data(ixgbe) != IXGBE_SUCCESS) {
1895 ixgbe_error(ixgbe,
1896 "Failed to allocate software receive rings");
1897 return (IXGBE_FAILURE);
1898 }
1899
1900 /* Allocate buffers for all the rx/tx rings */
1901 if (ixgbe_alloc_dma(ixgbe) != IXGBE_SUCCESS) {
1902 ixgbe_error(ixgbe, "Failed to allocate DMA resource");
1903 return (IXGBE_FAILURE);
1904 }
1905
1906 ixgbe->tx_ring_init = B_TRUE;
1907 } else {
1908 ixgbe->tx_ring_init = B_FALSE;
1909 }
1910
1911 for (i = 0; i < ixgbe->num_rx_rings; i++)
1912 mutex_enter(&ixgbe->rx_rings[i].rx_lock);
1913 for (i = 0; i < ixgbe->num_tx_rings; i++)
1914 mutex_enter(&ixgbe->tx_rings[i].tx_lock);
1915
1916 /*
1917 * Start the chipset hardware
1918 */
1919 if (ixgbe_chip_start(ixgbe) != IXGBE_SUCCESS) {
1920 ixgbe_fm_ereport(ixgbe, DDI_FM_DEVICE_INVAL_STATE);
1921 goto start_failure;
1922 }
1923
1924 /*
1925 * Configure link now for X550
1926 *
1927 * X550 possesses a LPLU (Low-Power Link Up) mode which keeps the
1928 * resting state of the adapter at a 1Gb FDX speed. Prior to the X550,
1929 * the resting state of the link would be the maximum speed that
1930 * autonegotiation will allow (usually 10Gb, infrastructure allowing)
1931 * so we never bothered with explicitly setting the link to 10Gb as it
1932 * would already be at that state on driver attach. With X550, we must
1933 * trigger a re-negotiation of the link in order to switch from a LPLU
1934 * 1Gb link to 10Gb (cable and link partner permitting.)
1935 */
1936 if (hw->mac.type == ixgbe_mac_X550 ||
1937 hw->mac.type == ixgbe_mac_X550EM_a ||
1938 hw->mac.type == ixgbe_mac_X550EM_x) {
1939 (void) ixgbe_driver_setup_link(ixgbe, B_TRUE);
1940 ixgbe_get_hw_state(ixgbe);
1941 }
1942
1943 if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != DDI_FM_OK) {
1944 goto start_failure;
1945 }
1946
1947 /*
1948 * Setup the rx/tx rings
1949 */
1950 if (ixgbe_setup_rings(ixgbe) != IXGBE_SUCCESS)
1951 goto start_failure;
1952
1953 /*
1954 * ixgbe_start() will be called when resetting, however if reset
1955 * happens, we need to clear the ERROR, STALL and OVERTEMP flags
1956 * before enabling the interrupts.
1957 */
1958 atomic_and_32(&ixgbe->ixgbe_state, ~(IXGBE_ERROR
1959 | IXGBE_STALL| IXGBE_OVERTEMP));
1960
1961 /*
1962 * Enable adapter interrupts
1963 * The interrupts must be enabled after the driver state is START
1964 */
1965 ixgbe_enable_adapter_interrupts(ixgbe);
1966
1967 for (i = ixgbe->num_tx_rings - 1; i >= 0; i--)
1968 mutex_exit(&ixgbe->tx_rings[i].tx_lock);
1969 for (i = ixgbe->num_rx_rings - 1; i >= 0; i--)
1970 mutex_exit(&ixgbe->rx_rings[i].rx_lock);
1971
1972 return (IXGBE_SUCCESS);
1973
1974 start_failure:
1975 for (i = ixgbe->num_tx_rings - 1; i >= 0; i--)
1976 mutex_exit(&ixgbe->tx_rings[i].tx_lock);
1977 for (i = ixgbe->num_rx_rings - 1; i >= 0; i--)
1978 mutex_exit(&ixgbe->rx_rings[i].rx_lock);
1979
1980 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_LOST);
1981
1982 return (IXGBE_FAILURE);
1983 }
1984
1985 /*
1986 * ixgbe_stop - Stop the driver/chipset.
1987 */
1988 void
ixgbe_stop(ixgbe_t * ixgbe,boolean_t free_buffer)1989 ixgbe_stop(ixgbe_t *ixgbe, boolean_t free_buffer)
1990 {
1991 int i;
1992
1993 ASSERT(mutex_owned(&ixgbe->gen_lock));
1994
1995 /*
1996 * Disable the adapter interrupts
1997 */
1998 ixgbe_disable_adapter_interrupts(ixgbe);
1999
2000 /*
2001 * Drain the pending tx packets
2002 */
2003 (void) ixgbe_tx_drain(ixgbe);
2004
2005 for (i = 0; i < ixgbe->num_rx_rings; i++)
2006 mutex_enter(&ixgbe->rx_rings[i].rx_lock);
2007 for (i = 0; i < ixgbe->num_tx_rings; i++)
2008 mutex_enter(&ixgbe->tx_rings[i].tx_lock);
2009
2010 /*
2011 * Stop the chipset hardware
2012 */
2013 ixgbe_chip_stop(ixgbe);
2014
2015 if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != DDI_FM_OK) {
2016 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_LOST);
2017 }
2018
2019 /*
2020 * Clean the pending tx data/resources
2021 */
2022 ixgbe_tx_clean(ixgbe);
2023
2024 for (i = ixgbe->num_tx_rings - 1; i >= 0; i--)
2025 mutex_exit(&ixgbe->tx_rings[i].tx_lock);
2026 for (i = ixgbe->num_rx_rings - 1; i >= 0; i--)
2027 mutex_exit(&ixgbe->rx_rings[i].rx_lock);
2028
2029 if (ixgbe->link_state == LINK_STATE_UP) {
2030 ixgbe->link_state = LINK_STATE_UNKNOWN;
2031 mac_link_update(ixgbe->mac_hdl, ixgbe->link_state);
2032 }
2033
2034 if (free_buffer) {
2035 /*
2036 * Release the DMA/memory resources of rx/tx rings
2037 */
2038 ixgbe_free_dma(ixgbe);
2039 ixgbe_free_rx_data(ixgbe);
2040 }
2041 }
2042
2043 /*
2044 * ixgbe_cbfunc - Driver interface for generic DDI callbacks
2045 */
2046 /* ARGSUSED */
2047 static int
ixgbe_cbfunc(dev_info_t * dip,ddi_cb_action_t cbaction,void * cbarg,void * arg1,void * arg2)2048 ixgbe_cbfunc(dev_info_t *dip, ddi_cb_action_t cbaction, void *cbarg,
2049 void *arg1, void *arg2)
2050 {
2051 ixgbe_t *ixgbe = (ixgbe_t *)arg1;
2052
2053 switch (cbaction) {
2054 /* IRM callback */
2055 int count;
2056 case DDI_CB_INTR_ADD:
2057 case DDI_CB_INTR_REMOVE:
2058 count = (int)(uintptr_t)cbarg;
2059 ASSERT(ixgbe->intr_type == DDI_INTR_TYPE_MSIX);
2060 DTRACE_PROBE2(ixgbe__irm__callback, int, count,
2061 int, ixgbe->intr_cnt);
2062 if (ixgbe_intr_adjust(ixgbe, cbaction, count) !=
2063 DDI_SUCCESS) {
2064 ixgbe_error(ixgbe,
2065 "IRM CB: Failed to adjust interrupts");
2066 goto cb_fail;
2067 }
2068 break;
2069 default:
2070 IXGBE_DEBUGLOG_1(ixgbe, "DDI CB: action 0x%x NOT supported",
2071 cbaction);
2072 return (DDI_ENOTSUP);
2073 }
2074 return (DDI_SUCCESS);
2075 cb_fail:
2076 return (DDI_FAILURE);
2077 }
2078
2079 /*
2080 * ixgbe_intr_adjust - Adjust interrupt to respond to IRM request.
2081 */
2082 static int
ixgbe_intr_adjust(ixgbe_t * ixgbe,ddi_cb_action_t cbaction,int count)2083 ixgbe_intr_adjust(ixgbe_t *ixgbe, ddi_cb_action_t cbaction, int count)
2084 {
2085 int i, rc, actual;
2086
2087 if (count == 0)
2088 return (DDI_SUCCESS);
2089
2090 if ((cbaction == DDI_CB_INTR_ADD &&
2091 ixgbe->intr_cnt + count > ixgbe->intr_cnt_max) ||
2092 (cbaction == DDI_CB_INTR_REMOVE &&
2093 ixgbe->intr_cnt - count < ixgbe->intr_cnt_min))
2094 return (DDI_FAILURE);
2095
2096 if (!(ixgbe->ixgbe_state & IXGBE_STARTED)) {
2097 return (DDI_FAILURE);
2098 }
2099
2100 for (i = 0; i < ixgbe->num_rx_rings; i++)
2101 mac_ring_intr_set(ixgbe->rx_rings[i].ring_handle, NULL);
2102 for (i = 0; i < ixgbe->num_tx_rings; i++)
2103 mac_ring_intr_set(ixgbe->tx_rings[i].ring_handle, NULL);
2104
2105 mutex_enter(&ixgbe->gen_lock);
2106 ixgbe->ixgbe_state &= ~IXGBE_STARTED;
2107 ixgbe->ixgbe_state |= IXGBE_INTR_ADJUST;
2108 ixgbe->ixgbe_state |= IXGBE_SUSPENDED;
2109 mac_link_update(ixgbe->mac_hdl, LINK_STATE_UNKNOWN);
2110
2111 ixgbe_stop(ixgbe, B_FALSE);
2112 /*
2113 * Disable interrupts
2114 */
2115 if (ixgbe->attach_progress & ATTACH_PROGRESS_ENABLE_INTR) {
2116 rc = ixgbe_disable_intrs(ixgbe);
2117 ASSERT(rc == IXGBE_SUCCESS);
2118 }
2119 ixgbe->attach_progress &= ~ATTACH_PROGRESS_ENABLE_INTR;
2120
2121 /*
2122 * Remove interrupt handlers
2123 */
2124 if (ixgbe->attach_progress & ATTACH_PROGRESS_ADD_INTR) {
2125 ixgbe_rem_intr_handlers(ixgbe);
2126 }
2127 ixgbe->attach_progress &= ~ATTACH_PROGRESS_ADD_INTR;
2128
2129 /*
2130 * Clear vect_map
2131 */
2132 bzero(&ixgbe->vect_map, sizeof (ixgbe->vect_map));
2133 switch (cbaction) {
2134 case DDI_CB_INTR_ADD:
2135 rc = ddi_intr_alloc(ixgbe->dip, ixgbe->htable,
2136 DDI_INTR_TYPE_MSIX, ixgbe->intr_cnt, count, &actual,
2137 DDI_INTR_ALLOC_NORMAL);
2138 if (rc != DDI_SUCCESS || actual != count) {
2139 ixgbe_log(ixgbe, "Adjust interrupts failed."
2140 "return: %d, irm cb size: %d, actual: %d",
2141 rc, count, actual);
2142 goto intr_adjust_fail;
2143 }
2144 ixgbe->intr_cnt += count;
2145 break;
2146
2147 case DDI_CB_INTR_REMOVE:
2148 for (i = ixgbe->intr_cnt - count;
2149 i < ixgbe->intr_cnt; i ++) {
2150 rc = ddi_intr_free(ixgbe->htable[i]);
2151 ixgbe->htable[i] = NULL;
2152 if (rc != DDI_SUCCESS) {
2153 ixgbe_log(ixgbe, "Adjust interrupts failed."
2154 "return: %d, irm cb size: %d, actual: %d",
2155 rc, count, actual);
2156 goto intr_adjust_fail;
2157 }
2158 }
2159 ixgbe->intr_cnt -= count;
2160 break;
2161 }
2162
2163 /*
2164 * Get priority for first vector, assume remaining are all the same
2165 */
2166 rc = ddi_intr_get_pri(ixgbe->htable[0], &ixgbe->intr_pri);
2167 if (rc != DDI_SUCCESS) {
2168 ixgbe_log(ixgbe,
2169 "Get interrupt priority failed: %d", rc);
2170 goto intr_adjust_fail;
2171 }
2172 rc = ddi_intr_get_cap(ixgbe->htable[0], &ixgbe->intr_cap);
2173 if (rc != DDI_SUCCESS) {
2174 ixgbe_log(ixgbe, "Get interrupt cap failed: %d", rc);
2175 goto intr_adjust_fail;
2176 }
2177 ixgbe->attach_progress |= ATTACH_PROGRESS_ALLOC_INTR;
2178
2179 /*
2180 * Map rings to interrupt vectors
2181 */
2182 if (ixgbe_map_intrs_to_vectors(ixgbe) != IXGBE_SUCCESS) {
2183 ixgbe_error(ixgbe,
2184 "IRM CB: Failed to map interrupts to vectors");
2185 goto intr_adjust_fail;
2186 }
2187
2188 /*
2189 * Add interrupt handlers
2190 */
2191 if (ixgbe_add_intr_handlers(ixgbe) != IXGBE_SUCCESS) {
2192 ixgbe_error(ixgbe, "IRM CB: Failed to add interrupt handlers");
2193 goto intr_adjust_fail;
2194 }
2195 ixgbe->attach_progress |= ATTACH_PROGRESS_ADD_INTR;
2196
2197 /*
2198 * Now that mutex locks are initialized, and the chip is also
2199 * initialized, enable interrupts.
2200 */
2201 if (ixgbe_enable_intrs(ixgbe) != IXGBE_SUCCESS) {
2202 ixgbe_error(ixgbe, "IRM CB: Failed to enable DDI interrupts");
2203 goto intr_adjust_fail;
2204 }
2205 ixgbe->attach_progress |= ATTACH_PROGRESS_ENABLE_INTR;
2206 if (ixgbe_start(ixgbe, B_FALSE) != IXGBE_SUCCESS) {
2207 ixgbe_error(ixgbe, "IRM CB: Failed to start");
2208 goto intr_adjust_fail;
2209 }
2210 ixgbe->ixgbe_state &= ~IXGBE_INTR_ADJUST;
2211 ixgbe->ixgbe_state &= ~IXGBE_SUSPENDED;
2212 ixgbe->ixgbe_state |= IXGBE_STARTED;
2213 mutex_exit(&ixgbe->gen_lock);
2214
2215 for (i = 0; i < ixgbe->num_rx_rings; i++) {
2216 mac_ring_intr_set(ixgbe->rx_rings[i].ring_handle,
2217 ixgbe->htable[ixgbe->rx_rings[i].intr_vector]);
2218 }
2219 for (i = 0; i < ixgbe->num_tx_rings; i++) {
2220 mac_ring_intr_set(ixgbe->tx_rings[i].ring_handle,
2221 ixgbe->htable[ixgbe->tx_rings[i].intr_vector]);
2222 }
2223
2224 /* Wakeup all Tx rings */
2225 for (i = 0; i < ixgbe->num_tx_rings; i++) {
2226 mac_tx_ring_update(ixgbe->mac_hdl,
2227 ixgbe->tx_rings[i].ring_handle);
2228 }
2229
2230 IXGBE_DEBUGLOG_3(ixgbe,
2231 "IRM CB: interrupts new value: 0x%x(0x%x:0x%x).",
2232 ixgbe->intr_cnt, ixgbe->intr_cnt_min, ixgbe->intr_cnt_max);
2233 return (DDI_SUCCESS);
2234
2235 intr_adjust_fail:
2236 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_LOST);
2237 mutex_exit(&ixgbe->gen_lock);
2238 return (DDI_FAILURE);
2239 }
2240
2241 /*
2242 * ixgbe_intr_cb_register - Register interrupt callback function.
2243 */
2244 static int
ixgbe_intr_cb_register(ixgbe_t * ixgbe)2245 ixgbe_intr_cb_register(ixgbe_t *ixgbe)
2246 {
2247 if (ddi_cb_register(ixgbe->dip, DDI_CB_FLAG_INTR, ixgbe_cbfunc,
2248 ixgbe, NULL, &ixgbe->cb_hdl) != DDI_SUCCESS) {
2249 return (IXGBE_FAILURE);
2250 }
2251 IXGBE_DEBUGLOG_0(ixgbe, "Interrupt callback function registered.");
2252 return (IXGBE_SUCCESS);
2253 }
2254
2255 /*
2256 * ixgbe_alloc_rings - Allocate memory space for rx/tx rings.
2257 */
2258 static int
ixgbe_alloc_rings(ixgbe_t * ixgbe)2259 ixgbe_alloc_rings(ixgbe_t *ixgbe)
2260 {
2261 /*
2262 * Allocate memory space for rx rings
2263 */
2264 ixgbe->rx_rings = kmem_zalloc(
2265 sizeof (ixgbe_rx_ring_t) * ixgbe->num_rx_rings,
2266 KM_NOSLEEP);
2267
2268 if (ixgbe->rx_rings == NULL) {
2269 return (IXGBE_FAILURE);
2270 }
2271
2272 /*
2273 * Allocate memory space for tx rings
2274 */
2275 ixgbe->tx_rings = kmem_zalloc(
2276 sizeof (ixgbe_tx_ring_t) * ixgbe->num_tx_rings,
2277 KM_NOSLEEP);
2278
2279 if (ixgbe->tx_rings == NULL) {
2280 kmem_free(ixgbe->rx_rings,
2281 sizeof (ixgbe_rx_ring_t) * ixgbe->num_rx_rings);
2282 ixgbe->rx_rings = NULL;
2283 return (IXGBE_FAILURE);
2284 }
2285
2286 /*
2287 * Allocate memory space for rx ring groups
2288 */
2289 ixgbe->rx_groups = kmem_zalloc(
2290 sizeof (ixgbe_rx_group_t) * ixgbe->num_rx_groups,
2291 KM_NOSLEEP);
2292
2293 if (ixgbe->rx_groups == NULL) {
2294 kmem_free(ixgbe->rx_rings,
2295 sizeof (ixgbe_rx_ring_t) * ixgbe->num_rx_rings);
2296 kmem_free(ixgbe->tx_rings,
2297 sizeof (ixgbe_tx_ring_t) * ixgbe->num_tx_rings);
2298 ixgbe->rx_rings = NULL;
2299 ixgbe->tx_rings = NULL;
2300 return (IXGBE_FAILURE);
2301 }
2302
2303 return (IXGBE_SUCCESS);
2304 }
2305
2306 /*
2307 * ixgbe_free_rings - Free the memory space of rx/tx rings.
2308 */
2309 static void
ixgbe_free_rings(ixgbe_t * ixgbe)2310 ixgbe_free_rings(ixgbe_t *ixgbe)
2311 {
2312 if (ixgbe->rx_rings != NULL) {
2313 kmem_free(ixgbe->rx_rings,
2314 sizeof (ixgbe_rx_ring_t) * ixgbe->num_rx_rings);
2315 ixgbe->rx_rings = NULL;
2316 }
2317
2318 if (ixgbe->tx_rings != NULL) {
2319 kmem_free(ixgbe->tx_rings,
2320 sizeof (ixgbe_tx_ring_t) * ixgbe->num_tx_rings);
2321 ixgbe->tx_rings = NULL;
2322 }
2323
2324 for (uint_t i = 0; i < ixgbe->num_rx_groups; i++) {
2325 ixgbe_vlan_t *vlp;
2326 ixgbe_rx_group_t *rx_group = &ixgbe->rx_groups[i];
2327
2328 while ((vlp = list_remove_head(&rx_group->vlans)) != NULL)
2329 kmem_free(vlp, sizeof (ixgbe_vlan_t));
2330
2331 list_destroy(&rx_group->vlans);
2332 }
2333
2334 if (ixgbe->rx_groups != NULL) {
2335 kmem_free(ixgbe->rx_groups,
2336 sizeof (ixgbe_rx_group_t) * ixgbe->num_rx_groups);
2337 ixgbe->rx_groups = NULL;
2338 }
2339 }
2340
2341 static int
ixgbe_alloc_rx_data(ixgbe_t * ixgbe)2342 ixgbe_alloc_rx_data(ixgbe_t *ixgbe)
2343 {
2344 ixgbe_rx_ring_t *rx_ring;
2345 int i;
2346
2347 for (i = 0; i < ixgbe->num_rx_rings; i++) {
2348 rx_ring = &ixgbe->rx_rings[i];
2349 if (ixgbe_alloc_rx_ring_data(rx_ring) != IXGBE_SUCCESS)
2350 goto alloc_rx_rings_failure;
2351 }
2352 return (IXGBE_SUCCESS);
2353
2354 alloc_rx_rings_failure:
2355 ixgbe_free_rx_data(ixgbe);
2356 return (IXGBE_FAILURE);
2357 }
2358
2359 static void
ixgbe_free_rx_data(ixgbe_t * ixgbe)2360 ixgbe_free_rx_data(ixgbe_t *ixgbe)
2361 {
2362 ixgbe_rx_ring_t *rx_ring;
2363 ixgbe_rx_data_t *rx_data;
2364 int i;
2365
2366 for (i = 0; i < ixgbe->num_rx_rings; i++) {
2367 rx_ring = &ixgbe->rx_rings[i];
2368
2369 mutex_enter(&ixgbe->rx_pending_lock);
2370 rx_data = rx_ring->rx_data;
2371
2372 if (rx_data != NULL) {
2373 rx_data->flag |= IXGBE_RX_STOPPED;
2374
2375 if (rx_data->rcb_pending == 0) {
2376 ixgbe_free_rx_ring_data(rx_data);
2377 rx_ring->rx_data = NULL;
2378 }
2379 }
2380
2381 mutex_exit(&ixgbe->rx_pending_lock);
2382 }
2383 }
2384
2385 /*
2386 * ixgbe_setup_rings - Setup rx/tx rings.
2387 */
2388 static int
ixgbe_setup_rings(ixgbe_t * ixgbe)2389 ixgbe_setup_rings(ixgbe_t *ixgbe)
2390 {
2391 /*
2392 * Setup the rx/tx rings, including the following:
2393 *
2394 * 1. Setup the descriptor ring and the control block buffers;
2395 * 2. Initialize necessary registers for receive/transmit;
2396 * 3. Initialize software pointers/parameters for receive/transmit;
2397 */
2398 if (ixgbe_setup_rx(ixgbe) != IXGBE_SUCCESS)
2399 return (IXGBE_FAILURE);
2400
2401 ixgbe_setup_tx(ixgbe);
2402
2403 return (IXGBE_SUCCESS);
2404 }
2405
2406 static void
ixgbe_setup_rx_ring(ixgbe_rx_ring_t * rx_ring)2407 ixgbe_setup_rx_ring(ixgbe_rx_ring_t *rx_ring)
2408 {
2409 ixgbe_t *ixgbe = rx_ring->ixgbe;
2410 ixgbe_rx_data_t *rx_data = rx_ring->rx_data;
2411 struct ixgbe_hw *hw = &ixgbe->hw;
2412 rx_control_block_t *rcb;
2413 union ixgbe_adv_rx_desc *rbd;
2414 uint32_t size;
2415 uint32_t buf_low;
2416 uint32_t buf_high;
2417 uint32_t reg_val;
2418 int i;
2419
2420 ASSERT(mutex_owned(&rx_ring->rx_lock));
2421 ASSERT(mutex_owned(&ixgbe->gen_lock));
2422
2423 for (i = 0; i < ixgbe->rx_ring_size; i++) {
2424 rcb = rx_data->work_list[i];
2425 rbd = &rx_data->rbd_ring[i];
2426
2427 rbd->read.pkt_addr = rcb->rx_buf.dma_address;
2428 rbd->read.hdr_addr = 0;
2429 }
2430
2431 /*
2432 * Initialize the length register
2433 */
2434 size = rx_data->ring_size * sizeof (union ixgbe_adv_rx_desc);
2435 IXGBE_WRITE_REG(hw, IXGBE_RDLEN(rx_ring->hw_index), size);
2436
2437 /*
2438 * Initialize the base address registers
2439 */
2440 buf_low = (uint32_t)rx_data->rbd_area.dma_address;
2441 buf_high = (uint32_t)(rx_data->rbd_area.dma_address >> 32);
2442 IXGBE_WRITE_REG(hw, IXGBE_RDBAH(rx_ring->hw_index), buf_high);
2443 IXGBE_WRITE_REG(hw, IXGBE_RDBAL(rx_ring->hw_index), buf_low);
2444
2445 /*
2446 * Setup head & tail pointers
2447 */
2448 IXGBE_WRITE_REG(hw, IXGBE_RDT(rx_ring->hw_index),
2449 rx_data->ring_size - 1);
2450 IXGBE_WRITE_REG(hw, IXGBE_RDH(rx_ring->hw_index), 0);
2451
2452 rx_data->rbd_next = 0;
2453 rx_data->lro_first = 0;
2454
2455 /*
2456 * Setup the Receive Descriptor Control Register (RXDCTL)
2457 * PTHRESH=32 descriptors (half the internal cache)
2458 * HTHRESH=0 descriptors (to minimize latency on fetch)
2459 * WTHRESH defaults to 1 (writeback each descriptor)
2460 */
2461 reg_val = IXGBE_READ_REG(hw, IXGBE_RXDCTL(rx_ring->hw_index));
2462 reg_val |= IXGBE_RXDCTL_ENABLE; /* enable queue */
2463
2464 /* Not a valid value for 82599, X540 or X550 */
2465 if (hw->mac.type == ixgbe_mac_82598EB) {
2466 reg_val |= 0x0020; /* pthresh */
2467 }
2468 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(rx_ring->hw_index), reg_val);
2469
2470 if (hw->mac.type == ixgbe_mac_82599EB ||
2471 hw->mac.type == ixgbe_mac_X540 ||
2472 hw->mac.type == ixgbe_mac_X550 ||
2473 hw->mac.type == ixgbe_mac_X550EM_x ||
2474 hw->mac.type == ixgbe_mac_X550EM_a) {
2475 reg_val = IXGBE_READ_REG(hw, IXGBE_RDRXCTL);
2476 reg_val |= (IXGBE_RDRXCTL_CRCSTRIP | IXGBE_RDRXCTL_AGGDIS);
2477 IXGBE_WRITE_REG(hw, IXGBE_RDRXCTL, reg_val);
2478 }
2479
2480 /*
2481 * Setup the Split and Replication Receive Control Register.
2482 * Set the rx buffer size and the advanced descriptor type.
2483 */
2484 reg_val = (ixgbe->rx_buf_size >> IXGBE_SRRCTL_BSIZEPKT_SHIFT) |
2485 IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF;
2486 reg_val |= IXGBE_SRRCTL_DROP_EN;
2487 IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(rx_ring->hw_index), reg_val);
2488 }
2489
2490 static int
ixgbe_setup_rx(ixgbe_t * ixgbe)2491 ixgbe_setup_rx(ixgbe_t *ixgbe)
2492 {
2493 ixgbe_rx_ring_t *rx_ring;
2494 struct ixgbe_hw *hw = &ixgbe->hw;
2495 uint32_t reg_val;
2496 uint32_t i;
2497 uint32_t psrtype_rss_bit;
2498
2499 /*
2500 * Ensure that Rx is disabled while setting up
2501 * the Rx unit and Rx descriptor ring(s)
2502 */
2503 ixgbe_disable_rx(hw);
2504
2505 /* PSRTYPE must be configured for 82599 */
2506 if (ixgbe->classify_mode != IXGBE_CLASSIFY_VMDQ &&
2507 ixgbe->classify_mode != IXGBE_CLASSIFY_VMDQ_RSS) {
2508 reg_val = IXGBE_PSRTYPE_TCPHDR | IXGBE_PSRTYPE_UDPHDR |
2509 IXGBE_PSRTYPE_IPV4HDR | IXGBE_PSRTYPE_IPV6HDR;
2510 reg_val |= IXGBE_PSRTYPE_L2HDR;
2511 reg_val |= 0x80000000;
2512 IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(0), reg_val);
2513 } else {
2514 if (ixgbe->num_rx_groups > 32) {
2515 psrtype_rss_bit = 0x20000000;
2516 } else {
2517 psrtype_rss_bit = 0x40000000;
2518 }
2519 for (i = 0; i < ixgbe->capab->max_rx_grp_num; i++) {
2520 reg_val = IXGBE_PSRTYPE_TCPHDR | IXGBE_PSRTYPE_UDPHDR |
2521 IXGBE_PSRTYPE_IPV4HDR | IXGBE_PSRTYPE_IPV6HDR;
2522 reg_val |= IXGBE_PSRTYPE_L2HDR;
2523 reg_val |= psrtype_rss_bit;
2524 IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(i), reg_val);
2525 }
2526 }
2527
2528 /*
2529 * Set filter control in FCTRL to determine types of packets are passed
2530 * up to the driver.
2531 * - Pass broadcast packets.
2532 * - Do not pass flow control pause frames (82598-specific)
2533 */
2534 reg_val = IXGBE_READ_REG(hw, IXGBE_FCTRL);
2535 reg_val |= IXGBE_FCTRL_BAM; /* Broadcast Accept Mode */
2536 if (hw->mac.type == ixgbe_mac_82598EB) {
2537 reg_val |= IXGBE_FCTRL_DPF; /* Discard Pause Frames */
2538 }
2539 IXGBE_WRITE_REG(hw, IXGBE_FCTRL, reg_val);
2540
2541 /*
2542 * Hardware checksum settings
2543 */
2544 if (ixgbe->rx_hcksum_enable) {
2545 reg_val = IXGBE_READ_REG(hw, IXGBE_RXCSUM);
2546 reg_val |= IXGBE_RXCSUM_IPPCSE; /* IP checksum */
2547 IXGBE_WRITE_REG(hw, IXGBE_RXCSUM, reg_val);
2548 }
2549
2550 /*
2551 * Setup VMDq and RSS for multiple receive queues
2552 */
2553 switch (ixgbe->classify_mode) {
2554 case IXGBE_CLASSIFY_RSS:
2555 /*
2556 * One group, only RSS is needed when more than
2557 * one ring enabled.
2558 */
2559 ixgbe_setup_rss(ixgbe);
2560 break;
2561
2562 case IXGBE_CLASSIFY_VMDQ:
2563 /*
2564 * Multiple groups, each group has one ring,
2565 * only VMDq is needed.
2566 */
2567 ixgbe_setup_vmdq(ixgbe);
2568 break;
2569
2570 case IXGBE_CLASSIFY_VMDQ_RSS:
2571 /*
2572 * Multiple groups and multiple rings, both
2573 * VMDq and RSS are needed.
2574 */
2575 ixgbe_setup_vmdq_rss(ixgbe);
2576 break;
2577
2578 default:
2579 break;
2580 }
2581
2582 /*
2583 * Initialize VLAN SW and HW state if VLAN filtering is
2584 * enabled.
2585 */
2586 if (ixgbe->vlft_enabled) {
2587 if (ixgbe_init_vlan(ixgbe) != IXGBE_SUCCESS)
2588 return (IXGBE_FAILURE);
2589 }
2590
2591 /*
2592 * Enable the receive unit. This must be done after filter
2593 * control is set in FCTRL. On 82598, we disable the descriptor monitor.
2594 * 82598 is the only adapter which defines this RXCTRL option.
2595 */
2596 reg_val = IXGBE_READ_REG(hw, IXGBE_RXCTRL);
2597 if (hw->mac.type == ixgbe_mac_82598EB)
2598 reg_val |= IXGBE_RXCTRL_DMBYPS; /* descriptor monitor bypass */
2599 reg_val |= IXGBE_RXCTRL_RXEN;
2600 (void) ixgbe_enable_rx_dma(hw, reg_val);
2601
2602 /*
2603 * ixgbe_setup_rx_ring must be called after configuring RXCTRL
2604 */
2605 for (i = 0; i < ixgbe->num_rx_rings; i++) {
2606 rx_ring = &ixgbe->rx_rings[i];
2607 ixgbe_setup_rx_ring(rx_ring);
2608 }
2609
2610 /*
2611 * The 82598 controller gives us the RNBC (Receive No Buffer
2612 * Count) register to determine the number of frames dropped
2613 * due to no available descriptors on the destination queue.
2614 * However, this register was removed starting with 82599 and
2615 * it was replaced with the RQSMR/QPRDC registers. The nice
2616 * thing about the new registers is that they allow you to map
2617 * groups of queues to specific stat registers. The bad thing
2618 * is there are only 16 slots in the stat registers, so this
2619 * won't work when we have 32 Rx groups. Instead, we map all
2620 * queues to the zero slot of the stat registers, giving us a
2621 * global counter at QPRDC[0] (with the equivalent semantics
2622 * of RNBC). Perhaps future controllers will have more slots
2623 * and we can implement per-group counters.
2624 */
2625 for (i = 0; i < ixgbe->num_rx_rings; i++) {
2626 uint32_t index = ixgbe->rx_rings[i].hw_index;
2627 IXGBE_WRITE_REG(hw, IXGBE_RQSMR(index >> 2), 0);
2628 }
2629
2630 /*
2631 * The Max Frame Size in MHADD/MAXFRS will be internally increased
2632 * by four bytes if the packet has a VLAN field, so includes MTU,
2633 * ethernet header and frame check sequence.
2634 * Register is MAXFRS in 82599.
2635 */
2636 reg_val = IXGBE_READ_REG(hw, IXGBE_MHADD);
2637 reg_val &= ~IXGBE_MHADD_MFS_MASK;
2638 reg_val |= (ixgbe->default_mtu + sizeof (struct ether_header)
2639 + ETHERFCSL) << IXGBE_MHADD_MFS_SHIFT;
2640 IXGBE_WRITE_REG(hw, IXGBE_MHADD, reg_val);
2641
2642 /*
2643 * Setup Jumbo Frame enable bit
2644 */
2645 reg_val = IXGBE_READ_REG(hw, IXGBE_HLREG0);
2646 if (ixgbe->default_mtu > ETHERMTU)
2647 reg_val |= IXGBE_HLREG0_JUMBOEN;
2648 else
2649 reg_val &= ~IXGBE_HLREG0_JUMBOEN;
2650 IXGBE_WRITE_REG(hw, IXGBE_HLREG0, reg_val);
2651
2652 /*
2653 * Setup RSC for multiple receive queues.
2654 */
2655 if (ixgbe->lro_enable) {
2656 for (i = 0; i < ixgbe->num_rx_rings; i++) {
2657 /*
2658 * Make sure rx_buf_size * MAXDESC not greater
2659 * than 65535.
2660 * Intel recommends 4 for MAXDESC field value.
2661 */
2662 reg_val = IXGBE_READ_REG(hw, IXGBE_RSCCTL(i));
2663 reg_val |= IXGBE_RSCCTL_RSCEN;
2664 if (ixgbe->rx_buf_size == IXGBE_PKG_BUF_16k)
2665 reg_val |= IXGBE_RSCCTL_MAXDESC_1;
2666 else
2667 reg_val |= IXGBE_RSCCTL_MAXDESC_4;
2668 IXGBE_WRITE_REG(hw, IXGBE_RSCCTL(i), reg_val);
2669 }
2670
2671 reg_val = IXGBE_READ_REG(hw, IXGBE_RSCDBU);
2672 reg_val |= IXGBE_RSCDBU_RSCACKDIS;
2673 IXGBE_WRITE_REG(hw, IXGBE_RSCDBU, reg_val);
2674
2675 reg_val = IXGBE_READ_REG(hw, IXGBE_RDRXCTL);
2676 reg_val |= IXGBE_RDRXCTL_RSCACKC;
2677 reg_val |= IXGBE_RDRXCTL_FCOE_WRFIX;
2678 reg_val &= ~IXGBE_RDRXCTL_RSCFRSTSIZE;
2679
2680 IXGBE_WRITE_REG(hw, IXGBE_RDRXCTL, reg_val);
2681 }
2682
2683 return (IXGBE_SUCCESS);
2684 }
2685
2686 static void
ixgbe_setup_tx_ring(ixgbe_tx_ring_t * tx_ring)2687 ixgbe_setup_tx_ring(ixgbe_tx_ring_t *tx_ring)
2688 {
2689 ixgbe_t *ixgbe = tx_ring->ixgbe;
2690 struct ixgbe_hw *hw = &ixgbe->hw;
2691 uint32_t size;
2692 uint32_t buf_low;
2693 uint32_t buf_high;
2694 uint32_t reg_val;
2695
2696 ASSERT(mutex_owned(&tx_ring->tx_lock));
2697 ASSERT(mutex_owned(&ixgbe->gen_lock));
2698
2699 /*
2700 * Initialize the length register
2701 */
2702 size = tx_ring->ring_size * sizeof (union ixgbe_adv_tx_desc);
2703 IXGBE_WRITE_REG(hw, IXGBE_TDLEN(tx_ring->index), size);
2704
2705 /*
2706 * Initialize the base address registers
2707 */
2708 buf_low = (uint32_t)tx_ring->tbd_area.dma_address;
2709 buf_high = (uint32_t)(tx_ring->tbd_area.dma_address >> 32);
2710 IXGBE_WRITE_REG(hw, IXGBE_TDBAL(tx_ring->index), buf_low);
2711 IXGBE_WRITE_REG(hw, IXGBE_TDBAH(tx_ring->index), buf_high);
2712
2713 /*
2714 * Setup head & tail pointers
2715 */
2716 IXGBE_WRITE_REG(hw, IXGBE_TDH(tx_ring->index), 0);
2717 IXGBE_WRITE_REG(hw, IXGBE_TDT(tx_ring->index), 0);
2718
2719 /*
2720 * Setup head write-back
2721 */
2722 if (ixgbe->tx_head_wb_enable) {
2723 /*
2724 * The memory of the head write-back is allocated using
2725 * the extra tbd beyond the tail of the tbd ring.
2726 */
2727 tx_ring->tbd_head_wb = (uint32_t *)
2728 ((uintptr_t)tx_ring->tbd_area.address + size);
2729 *tx_ring->tbd_head_wb = 0;
2730
2731 buf_low = (uint32_t)
2732 (tx_ring->tbd_area.dma_address + size);
2733 buf_high = (uint32_t)
2734 ((tx_ring->tbd_area.dma_address + size) >> 32);
2735
2736 /* Set the head write-back enable bit */
2737 buf_low |= IXGBE_TDWBAL_HEAD_WB_ENABLE;
2738
2739 IXGBE_WRITE_REG(hw, IXGBE_TDWBAL(tx_ring->index), buf_low);
2740 IXGBE_WRITE_REG(hw, IXGBE_TDWBAH(tx_ring->index), buf_high);
2741
2742 /*
2743 * Turn off relaxed ordering for head write back or it will
2744 * cause problems with the tx recycling
2745 */
2746
2747 reg_val = (hw->mac.type == ixgbe_mac_82598EB) ?
2748 IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL(tx_ring->index)) :
2749 IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL_82599(tx_ring->index));
2750 reg_val &= ~IXGBE_DCA_TXCTRL_DESC_WRO_EN;
2751 if (hw->mac.type == ixgbe_mac_82598EB) {
2752 IXGBE_WRITE_REG(hw,
2753 IXGBE_DCA_TXCTRL(tx_ring->index), reg_val);
2754 } else {
2755 IXGBE_WRITE_REG(hw,
2756 IXGBE_DCA_TXCTRL_82599(tx_ring->index), reg_val);
2757 }
2758 } else {
2759 tx_ring->tbd_head_wb = NULL;
2760 }
2761
2762 tx_ring->tbd_head = 0;
2763 tx_ring->tbd_tail = 0;
2764 tx_ring->tbd_free = tx_ring->ring_size;
2765
2766 if (ixgbe->tx_ring_init == B_TRUE) {
2767 tx_ring->tcb_head = 0;
2768 tx_ring->tcb_tail = 0;
2769 tx_ring->tcb_free = tx_ring->free_list_size;
2770 }
2771
2772 /*
2773 * Initialize the s/w context structure
2774 */
2775 bzero(&tx_ring->tx_context, sizeof (ixgbe_tx_context_t));
2776 }
2777
2778 static void
ixgbe_setup_tx(ixgbe_t * ixgbe)2779 ixgbe_setup_tx(ixgbe_t *ixgbe)
2780 {
2781 struct ixgbe_hw *hw = &ixgbe->hw;
2782 ixgbe_tx_ring_t *tx_ring;
2783 uint32_t reg_val;
2784 int i;
2785
2786 for (i = 0; i < ixgbe->num_tx_rings; i++) {
2787 tx_ring = &ixgbe->tx_rings[i];
2788 ixgbe_setup_tx_ring(tx_ring);
2789 }
2790
2791 /*
2792 * Setup the per-ring statistics mapping. We map all Tx queues
2793 * to slot 0 to stay consistent with Rx.
2794 */
2795 for (i = 0; i < ixgbe->num_tx_rings; i++) {
2796 switch (hw->mac.type) {
2797 case ixgbe_mac_82598EB:
2798 IXGBE_WRITE_REG(hw, IXGBE_TQSMR(i >> 2), 0);
2799 break;
2800
2801 default:
2802 IXGBE_WRITE_REG(hw, IXGBE_TQSM(i >> 2), 0);
2803 break;
2804 }
2805 }
2806
2807 /*
2808 * Enable CRC appending and TX padding (for short tx frames)
2809 */
2810 reg_val = IXGBE_READ_REG(hw, IXGBE_HLREG0);
2811 reg_val |= IXGBE_HLREG0_TXCRCEN | IXGBE_HLREG0_TXPADEN;
2812 IXGBE_WRITE_REG(hw, IXGBE_HLREG0, reg_val);
2813
2814 /*
2815 * enable DMA for 82599, X540 and X550 parts
2816 */
2817 if (hw->mac.type == ixgbe_mac_82599EB ||
2818 hw->mac.type == ixgbe_mac_X540 ||
2819 hw->mac.type == ixgbe_mac_X550 ||
2820 hw->mac.type == ixgbe_mac_X550EM_x ||
2821 hw->mac.type == ixgbe_mac_X550EM_a) {
2822 /* DMATXCTL.TE must be set after all Tx config is complete */
2823 reg_val = IXGBE_READ_REG(hw, IXGBE_DMATXCTL);
2824 reg_val |= IXGBE_DMATXCTL_TE;
2825 IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL, reg_val);
2826
2827 /* Disable arbiter to set MTQC */
2828 reg_val = IXGBE_READ_REG(hw, IXGBE_RTTDCS);
2829 reg_val |= IXGBE_RTTDCS_ARBDIS;
2830 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, reg_val);
2831 IXGBE_WRITE_REG(hw, IXGBE_MTQC, IXGBE_MTQC_64Q_1PB);
2832 reg_val &= ~IXGBE_RTTDCS_ARBDIS;
2833 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, reg_val);
2834 }
2835
2836 /*
2837 * Enabling tx queues ..
2838 * For 82599 must be done after DMATXCTL.TE is set
2839 */
2840 for (i = 0; i < ixgbe->num_tx_rings; i++) {
2841 tx_ring = &ixgbe->tx_rings[i];
2842 reg_val = IXGBE_READ_REG(hw, IXGBE_TXDCTL(tx_ring->index));
2843 reg_val |= IXGBE_TXDCTL_ENABLE;
2844 IXGBE_WRITE_REG(hw, IXGBE_TXDCTL(tx_ring->index), reg_val);
2845 }
2846 }
2847
2848 /*
2849 * ixgbe_setup_rss - Setup receive-side scaling feature.
2850 */
2851 static void
ixgbe_setup_rss(ixgbe_t * ixgbe)2852 ixgbe_setup_rss(ixgbe_t *ixgbe)
2853 {
2854 struct ixgbe_hw *hw = &ixgbe->hw;
2855 uint32_t mrqc;
2856
2857 /*
2858 * Initialize RETA/ERETA table
2859 */
2860 ixgbe_setup_rss_table(ixgbe);
2861
2862 /*
2863 * Enable RSS & perform hash on these packet types
2864 */
2865 mrqc = IXGBE_MRQC_RSSEN |
2866 IXGBE_MRQC_RSS_FIELD_IPV4 |
2867 IXGBE_MRQC_RSS_FIELD_IPV4_TCP |
2868 IXGBE_MRQC_RSS_FIELD_IPV4_UDP |
2869 IXGBE_MRQC_RSS_FIELD_IPV6_EX_TCP |
2870 IXGBE_MRQC_RSS_FIELD_IPV6_EX |
2871 IXGBE_MRQC_RSS_FIELD_IPV6 |
2872 IXGBE_MRQC_RSS_FIELD_IPV6_TCP |
2873 IXGBE_MRQC_RSS_FIELD_IPV6_UDP |
2874 IXGBE_MRQC_RSS_FIELD_IPV6_EX_UDP;
2875 IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc);
2876 }
2877
2878 /*
2879 * ixgbe_setup_vmdq - Setup MAC classification feature
2880 */
2881 static void
ixgbe_setup_vmdq(ixgbe_t * ixgbe)2882 ixgbe_setup_vmdq(ixgbe_t *ixgbe)
2883 {
2884 struct ixgbe_hw *hw = &ixgbe->hw;
2885 uint32_t vmdctl, i, vtctl, vlnctl;
2886
2887 /*
2888 * Setup the VMDq Control register, enable VMDq based on
2889 * packet destination MAC address:
2890 */
2891 switch (hw->mac.type) {
2892 case ixgbe_mac_82598EB:
2893 /*
2894 * VMDq Enable = 1;
2895 * VMDq Filter = 0; MAC filtering
2896 * Default VMDq output index = 0;
2897 */
2898 vmdctl = IXGBE_VMD_CTL_VMDQ_EN;
2899 IXGBE_WRITE_REG(hw, IXGBE_VMD_CTL, vmdctl);
2900 break;
2901
2902 case ixgbe_mac_82599EB:
2903 case ixgbe_mac_X540:
2904 case ixgbe_mac_X550:
2905 case ixgbe_mac_X550EM_x:
2906 case ixgbe_mac_X550EM_a:
2907 /*
2908 * Enable VMDq-only.
2909 */
2910 vmdctl = IXGBE_MRQC_VMDQEN;
2911 IXGBE_WRITE_REG(hw, IXGBE_MRQC, vmdctl);
2912
2913 for (i = 0; i < hw->mac.num_rar_entries; i++) {
2914 IXGBE_WRITE_REG(hw, IXGBE_MPSAR_LO(i), 0);
2915 IXGBE_WRITE_REG(hw, IXGBE_MPSAR_HI(i), 0);
2916 }
2917
2918 /*
2919 * Enable Virtualization and Replication.
2920 */
2921 vtctl = IXGBE_READ_REG(hw, IXGBE_VT_CTL);
2922 ixgbe->rx_def_group = vtctl & IXGBE_VT_CTL_POOL_MASK;
2923 vtctl |= IXGBE_VT_CTL_VT_ENABLE | IXGBE_VT_CTL_REPLEN;
2924 IXGBE_WRITE_REG(hw, IXGBE_VT_CTL, vtctl);
2925
2926 /*
2927 * Enable VLAN filtering and switching (VFTA and VLVF).
2928 */
2929 vlnctl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
2930 vlnctl |= IXGBE_VLNCTRL_VFE;
2931 IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, vlnctl);
2932 ixgbe->vlft_enabled = B_TRUE;
2933
2934 /*
2935 * Enable receiving packets to all VFs
2936 */
2937 IXGBE_WRITE_REG(hw, IXGBE_VFRE(0), IXGBE_VFRE_ENABLE_ALL);
2938 IXGBE_WRITE_REG(hw, IXGBE_VFRE(1), IXGBE_VFRE_ENABLE_ALL);
2939 break;
2940
2941 default:
2942 break;
2943 }
2944 }
2945
2946 /*
2947 * ixgbe_setup_vmdq_rss - Setup both vmdq feature and rss feature.
2948 */
2949 static void
ixgbe_setup_vmdq_rss(ixgbe_t * ixgbe)2950 ixgbe_setup_vmdq_rss(ixgbe_t *ixgbe)
2951 {
2952 struct ixgbe_hw *hw = &ixgbe->hw;
2953 uint32_t i, mrqc;
2954 uint32_t vtctl, vmdctl, vlnctl;
2955
2956 /*
2957 * Initialize RETA/ERETA table
2958 */
2959 ixgbe_setup_rss_table(ixgbe);
2960
2961 /*
2962 * Enable and setup RSS and VMDq
2963 */
2964 switch (hw->mac.type) {
2965 case ixgbe_mac_82598EB:
2966 /*
2967 * Enable RSS & Setup RSS Hash functions
2968 */
2969 mrqc = IXGBE_MRQC_RSSEN |
2970 IXGBE_MRQC_RSS_FIELD_IPV4 |
2971 IXGBE_MRQC_RSS_FIELD_IPV4_TCP |
2972 IXGBE_MRQC_RSS_FIELD_IPV4_UDP |
2973 IXGBE_MRQC_RSS_FIELD_IPV6_EX_TCP |
2974 IXGBE_MRQC_RSS_FIELD_IPV6_EX |
2975 IXGBE_MRQC_RSS_FIELD_IPV6 |
2976 IXGBE_MRQC_RSS_FIELD_IPV6_TCP |
2977 IXGBE_MRQC_RSS_FIELD_IPV6_UDP |
2978 IXGBE_MRQC_RSS_FIELD_IPV6_EX_UDP;
2979 IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc);
2980
2981 /*
2982 * Enable and Setup VMDq
2983 * VMDq Filter = 0; MAC filtering
2984 * Default VMDq output index = 0;
2985 */
2986 vmdctl = IXGBE_VMD_CTL_VMDQ_EN;
2987 IXGBE_WRITE_REG(hw, IXGBE_VMD_CTL, vmdctl);
2988 break;
2989
2990 case ixgbe_mac_82599EB:
2991 case ixgbe_mac_X540:
2992 case ixgbe_mac_X550:
2993 case ixgbe_mac_X550EM_x:
2994 case ixgbe_mac_X550EM_a:
2995 /*
2996 * Enable RSS & Setup RSS Hash functions
2997 */
2998 mrqc = IXGBE_MRQC_RSS_FIELD_IPV4 |
2999 IXGBE_MRQC_RSS_FIELD_IPV4_TCP |
3000 IXGBE_MRQC_RSS_FIELD_IPV4_UDP |
3001 IXGBE_MRQC_RSS_FIELD_IPV6_EX_TCP |
3002 IXGBE_MRQC_RSS_FIELD_IPV6_EX |
3003 IXGBE_MRQC_RSS_FIELD_IPV6 |
3004 IXGBE_MRQC_RSS_FIELD_IPV6_TCP |
3005 IXGBE_MRQC_RSS_FIELD_IPV6_UDP |
3006 IXGBE_MRQC_RSS_FIELD_IPV6_EX_UDP;
3007
3008 /*
3009 * Enable VMDq+RSS.
3010 */
3011 if (ixgbe->num_rx_groups > 32) {
3012 mrqc = mrqc | IXGBE_MRQC_VMDQRSS64EN;
3013 } else {
3014 mrqc = mrqc | IXGBE_MRQC_VMDQRSS32EN;
3015 }
3016
3017 IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc);
3018
3019 for (i = 0; i < hw->mac.num_rar_entries; i++) {
3020 IXGBE_WRITE_REG(hw, IXGBE_MPSAR_LO(i), 0);
3021 IXGBE_WRITE_REG(hw, IXGBE_MPSAR_HI(i), 0);
3022 }
3023 break;
3024
3025 default:
3026 break;
3027
3028 }
3029
3030 if (hw->mac.type == ixgbe_mac_82599EB ||
3031 hw->mac.type == ixgbe_mac_X540 ||
3032 hw->mac.type == ixgbe_mac_X550 ||
3033 hw->mac.type == ixgbe_mac_X550EM_x ||
3034 hw->mac.type == ixgbe_mac_X550EM_a) {
3035 /*
3036 * Enable Virtualization and Replication.
3037 */
3038 vtctl = IXGBE_READ_REG(hw, IXGBE_VT_CTL);
3039 ixgbe->rx_def_group = vtctl & IXGBE_VT_CTL_POOL_MASK;
3040 vtctl |= IXGBE_VT_CTL_VT_ENABLE | IXGBE_VT_CTL_REPLEN;
3041 vtctl = IXGBE_VT_CTL_VT_ENABLE | IXGBE_VT_CTL_REPLEN;
3042 IXGBE_WRITE_REG(hw, IXGBE_VT_CTL, vtctl);
3043
3044 /*
3045 * Enable VLAN filtering and switching (VFTA and VLVF).
3046 */
3047 vlnctl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
3048 vlnctl |= IXGBE_VLNCTRL_VFE;
3049 IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, vlnctl);
3050 ixgbe->vlft_enabled = B_TRUE;
3051
3052 /*
3053 * Enable receiving packets to all VFs
3054 */
3055 IXGBE_WRITE_REG(hw, IXGBE_VFRE(0), IXGBE_VFRE_ENABLE_ALL);
3056 IXGBE_WRITE_REG(hw, IXGBE_VFRE(1), IXGBE_VFRE_ENABLE_ALL);
3057 }
3058 }
3059
3060 /*
3061 * ixgbe_setup_rss_table - Setup RSS table
3062 */
3063 static void
ixgbe_setup_rss_table(ixgbe_t * ixgbe)3064 ixgbe_setup_rss_table(ixgbe_t *ixgbe)
3065 {
3066 struct ixgbe_hw *hw = &ixgbe->hw;
3067 uint32_t i, j;
3068 uint32_t random;
3069 uint32_t reta;
3070 uint32_t ring_per_group;
3071 uint32_t ring;
3072 uint32_t table_size;
3073 uint32_t index_mult;
3074 uint32_t rxcsum;
3075
3076 /*
3077 * Set multiplier for RETA setup and table size based on MAC type.
3078 * RETA table sizes vary by model:
3079 *
3080 * 82598, 82599, X540: 128 table entries.
3081 * X550: 512 table entries.
3082 */
3083 index_mult = 0x1;
3084 table_size = 128;
3085 switch (ixgbe->hw.mac.type) {
3086 case ixgbe_mac_82598EB:
3087 index_mult = 0x11;
3088 break;
3089 case ixgbe_mac_X550:
3090 case ixgbe_mac_X550EM_x:
3091 case ixgbe_mac_X550EM_a:
3092 table_size = 512;
3093 break;
3094 default:
3095 break;
3096 }
3097
3098 /*
3099 * Fill out RSS redirection table. The configuation of the indices is
3100 * hardware-dependent.
3101 *
3102 * 82598: 8 bits wide containing two 4 bit RSS indices
3103 * 82599, X540: 8 bits wide containing one 4 bit RSS index
3104 * X550: 8 bits wide containing one 6 bit RSS index
3105 */
3106 reta = 0;
3107 ring_per_group = ixgbe->num_rx_rings / ixgbe->num_rx_groups;
3108
3109 for (i = 0, j = 0; i < table_size; i++, j++) {
3110 if (j == ring_per_group) j = 0;
3111
3112 /*
3113 * The low 8 bits are for hash value (n+0);
3114 * The next 8 bits are for hash value (n+1), etc.
3115 */
3116 ring = (j * index_mult);
3117 reta = reta >> 8;
3118 reta = reta | (((uint32_t)ring) << 24);
3119
3120 if ((i & 3) == 3) {
3121 /*
3122 * The first 128 table entries are programmed into the
3123 * RETA register, with any beyond that (eg; on X550)
3124 * into ERETA.
3125 */
3126 if (i < 128)
3127 IXGBE_WRITE_REG(hw, IXGBE_RETA(i >> 2), reta);
3128 else
3129 IXGBE_WRITE_REG(hw, IXGBE_ERETA((i >> 2) - 32),
3130 reta);
3131 reta = 0;
3132 }
3133 }
3134
3135 /*
3136 * Fill out hash function seeds with a random constant
3137 */
3138 for (i = 0; i < 10; i++) {
3139 (void) random_get_pseudo_bytes((uint8_t *)&random,
3140 sizeof (uint32_t));
3141 IXGBE_WRITE_REG(hw, IXGBE_RSSRK(i), random);
3142 }
3143
3144 /*
3145 * Disable Packet Checksum to enable RSS for multiple receive queues.
3146 * It is an adapter hardware limitation that Packet Checksum is
3147 * mutually exclusive with RSS.
3148 */
3149 rxcsum = IXGBE_READ_REG(hw, IXGBE_RXCSUM);
3150 rxcsum |= IXGBE_RXCSUM_PCSD;
3151 rxcsum &= ~IXGBE_RXCSUM_IPPCSE;
3152 IXGBE_WRITE_REG(hw, IXGBE_RXCSUM, rxcsum);
3153 }
3154
3155 /*
3156 * ixgbe_init_unicst - Initialize the unicast addresses.
3157 */
3158 static void
ixgbe_init_unicst(ixgbe_t * ixgbe)3159 ixgbe_init_unicst(ixgbe_t *ixgbe)
3160 {
3161 struct ixgbe_hw *hw = &ixgbe->hw;
3162 uint8_t *mac_addr;
3163 int slot;
3164 /*
3165 * Here we should consider two situations:
3166 *
3167 * 1. Chipset is initialized at the first time,
3168 * Clear all the multiple unicast addresses.
3169 *
3170 * 2. Chipset is reset
3171 * Recover the multiple unicast addresses from the
3172 * software data structure to the RAR registers.
3173 */
3174 if (!ixgbe->unicst_init) {
3175 /*
3176 * Initialize the multiple unicast addresses
3177 */
3178 ixgbe->unicst_total = hw->mac.num_rar_entries;
3179 ixgbe->unicst_avail = ixgbe->unicst_total;
3180 for (slot = 0; slot < ixgbe->unicst_total; slot++) {
3181 mac_addr = ixgbe->unicst_addr[slot].mac.addr;
3182 bzero(mac_addr, ETHERADDRL);
3183 (void) ixgbe_set_rar(hw, slot, mac_addr, 0, 0);
3184 ixgbe->unicst_addr[slot].mac.set = 0;
3185 }
3186 ixgbe->unicst_init = B_TRUE;
3187 } else {
3188 /* Re-configure the RAR registers */
3189 for (slot = 0; slot < ixgbe->unicst_total; slot++) {
3190 mac_addr = ixgbe->unicst_addr[slot].mac.addr;
3191 if (ixgbe->unicst_addr[slot].mac.set == 1) {
3192 (void) ixgbe_set_rar(hw, slot, mac_addr,
3193 ixgbe->unicst_addr[slot].mac.group_index,
3194 IXGBE_RAH_AV);
3195 } else {
3196 bzero(mac_addr, ETHERADDRL);
3197 (void) ixgbe_set_rar(hw, slot, mac_addr, 0, 0);
3198 }
3199 }
3200 }
3201 }
3202
3203 /*
3204 * ixgbe_unicst_find - Find the slot for the specified unicast address
3205 */
3206 int
ixgbe_unicst_find(ixgbe_t * ixgbe,const uint8_t * mac_addr)3207 ixgbe_unicst_find(ixgbe_t *ixgbe, const uint8_t *mac_addr)
3208 {
3209 int slot;
3210
3211 ASSERT(mutex_owned(&ixgbe->gen_lock));
3212
3213 for (slot = 0; slot < ixgbe->unicst_total; slot++) {
3214 if (bcmp(ixgbe->unicst_addr[slot].mac.addr,
3215 mac_addr, ETHERADDRL) == 0)
3216 return (slot);
3217 }
3218
3219 return (-1);
3220 }
3221
3222 /*
3223 * Restore the HW state to match the SW state during restart.
3224 */
3225 static int
ixgbe_init_vlan(ixgbe_t * ixgbe)3226 ixgbe_init_vlan(ixgbe_t *ixgbe)
3227 {
3228 /*
3229 * The device is starting for the first time; there is nothing
3230 * to do.
3231 */
3232 if (!ixgbe->vlft_init) {
3233 ixgbe->vlft_init = B_TRUE;
3234 return (IXGBE_SUCCESS);
3235 }
3236
3237 for (uint_t i = 0; i < ixgbe->num_rx_groups; i++) {
3238 int ret;
3239 boolean_t vlvf_bypass;
3240 ixgbe_rx_group_t *rxg = &ixgbe->rx_groups[i];
3241 struct ixgbe_hw *hw = &ixgbe->hw;
3242
3243 if (rxg->aupe) {
3244 uint32_t vml2flt;
3245
3246 vml2flt = IXGBE_READ_REG(hw, IXGBE_VMOLR(rxg->index));
3247 vml2flt |= IXGBE_VMOLR_AUPE;
3248 IXGBE_WRITE_REG(hw, IXGBE_VMOLR(rxg->index), vml2flt);
3249 }
3250
3251 vlvf_bypass = (rxg->index == ixgbe->rx_def_group);
3252 for (ixgbe_vlan_t *vlp = list_head(&rxg->vlans); vlp != NULL;
3253 vlp = list_next(&rxg->vlans, vlp)) {
3254 ret = ixgbe_set_vfta(hw, vlp->ixvl_vid, rxg->index,
3255 B_TRUE, vlvf_bypass);
3256
3257 if (ret != IXGBE_SUCCESS) {
3258 ixgbe_error(ixgbe, "Failed to program VFTA"
3259 " for group %u, VID: %u, ret: %d.",
3260 rxg->index, vlp->ixvl_vid, ret);
3261 return (IXGBE_FAILURE);
3262 }
3263 }
3264 }
3265
3266 return (IXGBE_SUCCESS);
3267 }
3268
3269 /*
3270 * ixgbe_multicst_add - Add a multicst address.
3271 */
3272 int
ixgbe_multicst_add(ixgbe_t * ixgbe,const uint8_t * multiaddr)3273 ixgbe_multicst_add(ixgbe_t *ixgbe, const uint8_t *multiaddr)
3274 {
3275 ASSERT(mutex_owned(&ixgbe->gen_lock));
3276
3277 if ((multiaddr[0] & 01) == 0) {
3278 return (EINVAL);
3279 }
3280
3281 if (ixgbe->mcast_count >= MAX_NUM_MULTICAST_ADDRESSES) {
3282 return (ENOENT);
3283 }
3284
3285 bcopy(multiaddr,
3286 &ixgbe->mcast_table[ixgbe->mcast_count], ETHERADDRL);
3287 ixgbe->mcast_count++;
3288
3289 /*
3290 * Update the multicast table in the hardware
3291 */
3292 ixgbe_setup_multicst(ixgbe);
3293
3294 if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != DDI_FM_OK) {
3295 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_DEGRADED);
3296 return (EIO);
3297 }
3298
3299 return (0);
3300 }
3301
3302 /*
3303 * ixgbe_multicst_remove - Remove a multicst address.
3304 */
3305 int
ixgbe_multicst_remove(ixgbe_t * ixgbe,const uint8_t * multiaddr)3306 ixgbe_multicst_remove(ixgbe_t *ixgbe, const uint8_t *multiaddr)
3307 {
3308 int i;
3309
3310 ASSERT(mutex_owned(&ixgbe->gen_lock));
3311
3312 for (i = 0; i < ixgbe->mcast_count; i++) {
3313 if (bcmp(multiaddr, &ixgbe->mcast_table[i],
3314 ETHERADDRL) == 0) {
3315 for (i++; i < ixgbe->mcast_count; i++) {
3316 ixgbe->mcast_table[i - 1] =
3317 ixgbe->mcast_table[i];
3318 }
3319 ixgbe->mcast_count--;
3320 break;
3321 }
3322 }
3323
3324 /*
3325 * Update the multicast table in the hardware
3326 */
3327 ixgbe_setup_multicst(ixgbe);
3328
3329 if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != DDI_FM_OK) {
3330 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_DEGRADED);
3331 return (EIO);
3332 }
3333
3334 return (0);
3335 }
3336
3337 /*
3338 * ixgbe_setup_multicast - Setup multicast data structures.
3339 *
3340 * This routine initializes all of the multicast related structures
3341 * and save them in the hardware registers.
3342 */
3343 static void
ixgbe_setup_multicst(ixgbe_t * ixgbe)3344 ixgbe_setup_multicst(ixgbe_t *ixgbe)
3345 {
3346 uint8_t *mc_addr_list;
3347 uint32_t mc_addr_count;
3348 struct ixgbe_hw *hw = &ixgbe->hw;
3349
3350 ASSERT(mutex_owned(&ixgbe->gen_lock));
3351
3352 ASSERT(ixgbe->mcast_count <= MAX_NUM_MULTICAST_ADDRESSES);
3353
3354 mc_addr_list = (uint8_t *)ixgbe->mcast_table;
3355 mc_addr_count = ixgbe->mcast_count;
3356
3357 /*
3358 * Update the multicast addresses to the MTA registers
3359 */
3360 (void) ixgbe_update_mc_addr_list(hw, mc_addr_list, mc_addr_count,
3361 ixgbe_mc_table_itr, TRUE);
3362 }
3363
3364 /*
3365 * ixgbe_setup_vmdq_rss_conf - Configure vmdq and rss (number and mode).
3366 *
3367 * Configure the rx classification mode (vmdq & rss) and vmdq & rss numbers.
3368 * Different chipsets may have different allowed configuration of vmdq and rss.
3369 */
3370 static void
ixgbe_setup_vmdq_rss_conf(ixgbe_t * ixgbe)3371 ixgbe_setup_vmdq_rss_conf(ixgbe_t *ixgbe)
3372 {
3373 struct ixgbe_hw *hw = &ixgbe->hw;
3374 uint32_t ring_per_group;
3375
3376 switch (hw->mac.type) {
3377 case ixgbe_mac_82598EB:
3378 /*
3379 * 82598 supports the following combination:
3380 * vmdq no. x rss no.
3381 * [5..16] x 1
3382 * [1..4] x [1..16]
3383 * However 8 rss queue per pool (vmdq) is sufficient for
3384 * most cases.
3385 */
3386 ring_per_group = ixgbe->num_rx_rings / ixgbe->num_rx_groups;
3387 if (ixgbe->num_rx_groups > 4) {
3388 ixgbe->num_rx_rings = ixgbe->num_rx_groups;
3389 } else {
3390 ixgbe->num_rx_rings = ixgbe->num_rx_groups *
3391 min(8, ring_per_group);
3392 }
3393
3394 break;
3395
3396 case ixgbe_mac_82599EB:
3397 case ixgbe_mac_X540:
3398 case ixgbe_mac_X550:
3399 case ixgbe_mac_X550EM_x:
3400 case ixgbe_mac_X550EM_a:
3401 /*
3402 * 82599 supports the following combination:
3403 * vmdq no. x rss no.
3404 * [33..64] x [1..2]
3405 * [2..32] x [1..4]
3406 * 1 x [1..16]
3407 * However 8 rss queue per pool (vmdq) is sufficient for
3408 * most cases.
3409 *
3410 * For now, treat X540 and X550 like the 82599.
3411 */
3412 ring_per_group = ixgbe->num_rx_rings / ixgbe->num_rx_groups;
3413 if (ixgbe->num_rx_groups == 1) {
3414 ixgbe->num_rx_rings = min(8, ring_per_group);
3415 } else if (ixgbe->num_rx_groups <= 32) {
3416 ixgbe->num_rx_rings = ixgbe->num_rx_groups *
3417 min(4, ring_per_group);
3418 } else if (ixgbe->num_rx_groups <= 64) {
3419 ixgbe->num_rx_rings = ixgbe->num_rx_groups *
3420 min(2, ring_per_group);
3421 }
3422 break;
3423
3424 default:
3425 break;
3426 }
3427
3428 ring_per_group = ixgbe->num_rx_rings / ixgbe->num_rx_groups;
3429
3430 if (ixgbe->num_rx_groups == 1 && ring_per_group == 1) {
3431 ixgbe->classify_mode = IXGBE_CLASSIFY_NONE;
3432 } else if (ixgbe->num_rx_groups != 1 && ring_per_group == 1) {
3433 ixgbe->classify_mode = IXGBE_CLASSIFY_VMDQ;
3434 } else if (ixgbe->num_rx_groups != 1 && ring_per_group != 1) {
3435 ixgbe->classify_mode = IXGBE_CLASSIFY_VMDQ_RSS;
3436 } else {
3437 ixgbe->classify_mode = IXGBE_CLASSIFY_RSS;
3438 }
3439
3440 IXGBE_DEBUGLOG_2(ixgbe, "rx group number:%d, rx ring number:%d",
3441 ixgbe->num_rx_groups, ixgbe->num_rx_rings);
3442 }
3443
3444 /*
3445 * ixgbe_get_conf - Get driver configurations set in driver.conf.
3446 *
3447 * This routine gets user-configured values out of the configuration
3448 * file ixgbe.conf.
3449 *
3450 * For each configurable value, there is a minimum, a maximum, and a
3451 * default.
3452 * If user does not configure a value, use the default.
3453 * If user configures below the minimum, use the minumum.
3454 * If user configures above the maximum, use the maxumum.
3455 */
3456 static void
ixgbe_get_conf(ixgbe_t * ixgbe)3457 ixgbe_get_conf(ixgbe_t *ixgbe)
3458 {
3459 struct ixgbe_hw *hw = &ixgbe->hw;
3460 uint32_t flow_control;
3461
3462 /*
3463 * ixgbe driver supports the following user configurations:
3464 *
3465 * Jumbo frame configuration:
3466 * default_mtu
3467 *
3468 * Ethernet flow control configuration:
3469 * flow_control
3470 *
3471 * Multiple rings configurations:
3472 * tx_queue_number
3473 * tx_ring_size
3474 * rx_queue_number
3475 * rx_ring_size
3476 *
3477 * Call ixgbe_get_prop() to get the value for a specific
3478 * configuration parameter.
3479 */
3480
3481 /*
3482 * Jumbo frame configuration - max_frame_size controls host buffer
3483 * allocation, so includes MTU, ethernet header, vlan tag and
3484 * frame check sequence.
3485 */
3486 ixgbe->default_mtu = ixgbe_get_prop(ixgbe, PROP_DEFAULT_MTU,
3487 MIN_MTU, ixgbe->capab->max_mtu, DEFAULT_MTU);
3488
3489 ixgbe->max_frame_size = ixgbe->default_mtu +
3490 sizeof (struct ether_vlan_header) + ETHERFCSL;
3491
3492 /*
3493 * Ethernet flow control configuration
3494 */
3495 flow_control = ixgbe_get_prop(ixgbe, PROP_FLOW_CONTROL,
3496 ixgbe_fc_none, 3, ixgbe_fc_none);
3497 if (flow_control == 3)
3498 flow_control = ixgbe_fc_default;
3499
3500 /*
3501 * fc.requested mode is what the user requests. After autoneg,
3502 * fc.current_mode will be the flow_control mode that was negotiated.
3503 */
3504 hw->fc.requested_mode = flow_control;
3505
3506 /*
3507 * Multiple rings configurations
3508 */
3509 ixgbe->num_tx_rings = ixgbe_get_prop(ixgbe, PROP_TX_QUEUE_NUM,
3510 ixgbe->capab->min_tx_que_num,
3511 ixgbe->capab->max_tx_que_num,
3512 ixgbe->capab->def_tx_que_num);
3513 ixgbe->tx_ring_size = ixgbe_get_prop(ixgbe, PROP_TX_RING_SIZE,
3514 MIN_TX_RING_SIZE, MAX_TX_RING_SIZE, DEFAULT_TX_RING_SIZE);
3515
3516 ixgbe->num_rx_rings = ixgbe_get_prop(ixgbe, PROP_RX_QUEUE_NUM,
3517 ixgbe->capab->min_rx_que_num,
3518 ixgbe->capab->max_rx_que_num,
3519 ixgbe->capab->def_rx_que_num);
3520 ixgbe->rx_ring_size = ixgbe_get_prop(ixgbe, PROP_RX_RING_SIZE,
3521 MIN_RX_RING_SIZE, MAX_RX_RING_SIZE, DEFAULT_RX_RING_SIZE);
3522
3523 /*
3524 * Multiple groups configuration
3525 */
3526 ixgbe->num_rx_groups = ixgbe_get_prop(ixgbe, PROP_RX_GROUP_NUM,
3527 ixgbe->capab->min_rx_grp_num, ixgbe->capab->max_rx_grp_num,
3528 ixgbe->capab->def_rx_grp_num);
3529
3530 ixgbe->mr_enable = ixgbe_get_prop(ixgbe, PROP_MR_ENABLE,
3531 0, 1, DEFAULT_MR_ENABLE);
3532
3533 if (ixgbe->mr_enable == B_FALSE) {
3534 ixgbe->num_tx_rings = 1;
3535 ixgbe->num_rx_rings = 1;
3536 ixgbe->num_rx_groups = 1;
3537 ixgbe->classify_mode = IXGBE_CLASSIFY_NONE;
3538 } else {
3539 ixgbe->num_rx_rings = ixgbe->num_rx_groups *
3540 max(ixgbe->num_rx_rings / ixgbe->num_rx_groups, 1);
3541 /*
3542 * The combination of num_rx_rings and num_rx_groups
3543 * may be not supported by h/w. We need to adjust
3544 * them to appropriate values.
3545 */
3546 ixgbe_setup_vmdq_rss_conf(ixgbe);
3547 }
3548
3549 /*
3550 * Tunable used to force an interrupt type. The only use is
3551 * for testing of the lesser interrupt types.
3552 * 0 = don't force interrupt type
3553 * 1 = force interrupt type MSI-X
3554 * 2 = force interrupt type MSI
3555 * 3 = force interrupt type Legacy
3556 */
3557 ixgbe->intr_force = ixgbe_get_prop(ixgbe, PROP_INTR_FORCE,
3558 IXGBE_INTR_NONE, IXGBE_INTR_LEGACY, IXGBE_INTR_NONE);
3559
3560 ixgbe->tx_hcksum_enable = ixgbe_get_prop(ixgbe, PROP_TX_HCKSUM_ENABLE,
3561 0, 1, DEFAULT_TX_HCKSUM_ENABLE);
3562 ixgbe->rx_hcksum_enable = ixgbe_get_prop(ixgbe, PROP_RX_HCKSUM_ENABLE,
3563 0, 1, DEFAULT_RX_HCKSUM_ENABLE);
3564 ixgbe->lso_enable = ixgbe_get_prop(ixgbe, PROP_LSO_ENABLE,
3565 0, 1, DEFAULT_LSO_ENABLE);
3566 ixgbe->lro_enable = ixgbe_get_prop(ixgbe, PROP_LRO_ENABLE,
3567 0, 1, DEFAULT_LRO_ENABLE);
3568 ixgbe->tx_head_wb_enable = ixgbe_get_prop(ixgbe, PROP_TX_HEAD_WB_ENABLE,
3569 0, 1, DEFAULT_TX_HEAD_WB_ENABLE);
3570 ixgbe->relax_order_enable = ixgbe_get_prop(ixgbe,
3571 PROP_RELAX_ORDER_ENABLE, 0, 1, DEFAULT_RELAX_ORDER_ENABLE);
3572
3573 /* Head Write Back not recommended for 82599, X540 and X550 */
3574 if (hw->mac.type == ixgbe_mac_82599EB ||
3575 hw->mac.type == ixgbe_mac_X540 ||
3576 hw->mac.type == ixgbe_mac_X550 ||
3577 hw->mac.type == ixgbe_mac_X550EM_x ||
3578 hw->mac.type == ixgbe_mac_X550EM_a) {
3579 ixgbe->tx_head_wb_enable = B_FALSE;
3580 }
3581
3582 /*
3583 * ixgbe LSO needs the tx h/w checksum support.
3584 * LSO will be disabled if tx h/w checksum is not
3585 * enabled.
3586 */
3587 if (ixgbe->tx_hcksum_enable == B_FALSE) {
3588 ixgbe->lso_enable = B_FALSE;
3589 }
3590
3591 /*
3592 * ixgbe LRO needs the rx h/w checksum support.
3593 * LRO will be disabled if rx h/w checksum is not
3594 * enabled.
3595 */
3596 if (ixgbe->rx_hcksum_enable == B_FALSE) {
3597 ixgbe->lro_enable = B_FALSE;
3598 }
3599
3600 /*
3601 * ixgbe LRO only supported by 82599, X540 and X550
3602 */
3603 if (hw->mac.type == ixgbe_mac_82598EB) {
3604 ixgbe->lro_enable = B_FALSE;
3605 }
3606 ixgbe->tx_copy_thresh = ixgbe_get_prop(ixgbe, PROP_TX_COPY_THRESHOLD,
3607 MIN_TX_COPY_THRESHOLD, MAX_TX_COPY_THRESHOLD,
3608 DEFAULT_TX_COPY_THRESHOLD);
3609 ixgbe->tx_recycle_thresh = ixgbe_get_prop(ixgbe,
3610 PROP_TX_RECYCLE_THRESHOLD, MIN_TX_RECYCLE_THRESHOLD,
3611 MAX_TX_RECYCLE_THRESHOLD, DEFAULT_TX_RECYCLE_THRESHOLD);
3612 ixgbe->tx_overload_thresh = ixgbe_get_prop(ixgbe,
3613 PROP_TX_OVERLOAD_THRESHOLD, MIN_TX_OVERLOAD_THRESHOLD,
3614 MAX_TX_OVERLOAD_THRESHOLD, DEFAULT_TX_OVERLOAD_THRESHOLD);
3615 ixgbe->tx_resched_thresh = ixgbe_get_prop(ixgbe,
3616 PROP_TX_RESCHED_THRESHOLD, MIN_TX_RESCHED_THRESHOLD,
3617 MAX_TX_RESCHED_THRESHOLD, DEFAULT_TX_RESCHED_THRESHOLD);
3618
3619 ixgbe->rx_copy_thresh = ixgbe_get_prop(ixgbe, PROP_RX_COPY_THRESHOLD,
3620 MIN_RX_COPY_THRESHOLD, MAX_RX_COPY_THRESHOLD,
3621 DEFAULT_RX_COPY_THRESHOLD);
3622 ixgbe->rx_limit_per_intr = ixgbe_get_prop(ixgbe, PROP_RX_LIMIT_PER_INTR,
3623 MIN_RX_LIMIT_PER_INTR, MAX_RX_LIMIT_PER_INTR,
3624 DEFAULT_RX_LIMIT_PER_INTR);
3625
3626 ixgbe->intr_throttling[0] = ixgbe_get_prop(ixgbe, PROP_INTR_THROTTLING,
3627 ixgbe->capab->min_intr_throttle,
3628 ixgbe->capab->max_intr_throttle,
3629 ixgbe->capab->def_intr_throttle);
3630 /*
3631 * 82599, X540 and X550 require the interrupt throttling rate is
3632 * a multiple of 8. This is enforced by the register definiton.
3633 */
3634 if (hw->mac.type == ixgbe_mac_82599EB ||
3635 hw->mac.type == ixgbe_mac_X540 ||
3636 hw->mac.type == ixgbe_mac_X550 ||
3637 hw->mac.type == ixgbe_mac_X550EM_x ||
3638 hw->mac.type == ixgbe_mac_X550EM_a)
3639 ixgbe->intr_throttling[0] = ixgbe->intr_throttling[0] & 0xFF8;
3640
3641 hw->allow_unsupported_sfp = ixgbe_get_prop(ixgbe,
3642 PROP_ALLOW_UNSUPPORTED_SFP, 0, 1, DEFAULT_ALLOW_UNSUPPORTED_SFP);
3643 }
3644
3645 static void
ixgbe_init_params(ixgbe_t * ixgbe)3646 ixgbe_init_params(ixgbe_t *ixgbe)
3647 {
3648 struct ixgbe_hw *hw = &ixgbe->hw;
3649 ixgbe_link_speed speeds_supported = 0;
3650 boolean_t negotiate;
3651
3652 /*
3653 * Get a list of speeds the adapter supports. If the hw struct hasn't
3654 * been populated with this information yet, retrieve it from the
3655 * adapter and save it to our own variable.
3656 *
3657 * On certain adapters, such as ones which use SFPs, the contents of
3658 * hw->phy.speeds_supported (and hw->phy.autoneg_advertised) are not
3659 * updated, so we must rely on calling ixgbe_get_link_capabilities()
3660 * in order to ascertain the speeds which we are capable of supporting,
3661 * and in the case of SFP-equipped adapters, which speed we are
3662 * advertising. If ixgbe_get_link_capabilities() fails for some reason,
3663 * we'll go with a default list of speeds as a last resort.
3664 */
3665 speeds_supported = hw->phy.speeds_supported;
3666
3667 if (speeds_supported == 0) {
3668 if (ixgbe_get_link_capabilities(hw, &speeds_supported,
3669 &negotiate) != IXGBE_SUCCESS) {
3670 if (hw->mac.type == ixgbe_mac_82598EB) {
3671 speeds_supported =
3672 IXGBE_LINK_SPEED_82598_AUTONEG;
3673 } else {
3674 speeds_supported =
3675 IXGBE_LINK_SPEED_82599_AUTONEG;
3676 }
3677 }
3678 }
3679 ixgbe->speeds_supported = speeds_supported;
3680
3681 /*
3682 * By default, all supported speeds are enabled and advertised.
3683 */
3684 if (speeds_supported & IXGBE_LINK_SPEED_10GB_FULL) {
3685 ixgbe->param_en_10000fdx_cap = 1;
3686 ixgbe->param_adv_10000fdx_cap = 1;
3687 } else {
3688 ixgbe->param_en_10000fdx_cap = 0;
3689 ixgbe->param_adv_10000fdx_cap = 0;
3690 }
3691
3692 if (speeds_supported & IXGBE_LINK_SPEED_5GB_FULL) {
3693 ixgbe->param_en_5000fdx_cap = 1;
3694 ixgbe->param_adv_5000fdx_cap = 1;
3695 } else {
3696 ixgbe->param_en_5000fdx_cap = 0;
3697 ixgbe->param_adv_5000fdx_cap = 0;
3698 }
3699
3700 if (speeds_supported & IXGBE_LINK_SPEED_2_5GB_FULL) {
3701 ixgbe->param_en_2500fdx_cap = 1;
3702 ixgbe->param_adv_2500fdx_cap = 1;
3703 } else {
3704 ixgbe->param_en_2500fdx_cap = 0;
3705 ixgbe->param_adv_2500fdx_cap = 0;
3706 }
3707
3708 if (speeds_supported & IXGBE_LINK_SPEED_1GB_FULL) {
3709 ixgbe->param_en_1000fdx_cap = 1;
3710 ixgbe->param_adv_1000fdx_cap = 1;
3711 } else {
3712 ixgbe->param_en_1000fdx_cap = 0;
3713 ixgbe->param_adv_1000fdx_cap = 0;
3714 }
3715
3716 if (speeds_supported & IXGBE_LINK_SPEED_100_FULL) {
3717 ixgbe->param_en_100fdx_cap = 1;
3718 ixgbe->param_adv_100fdx_cap = 1;
3719 } else {
3720 ixgbe->param_en_100fdx_cap = 0;
3721 ixgbe->param_adv_100fdx_cap = 0;
3722 }
3723
3724 ixgbe->param_pause_cap = 1;
3725 ixgbe->param_asym_pause_cap = 1;
3726 ixgbe->param_rem_fault = 0;
3727
3728 ixgbe->param_adv_autoneg_cap = 1;
3729 ixgbe->param_adv_pause_cap = 1;
3730 ixgbe->param_adv_asym_pause_cap = 1;
3731 ixgbe->param_adv_rem_fault = 0;
3732
3733 ixgbe->param_lp_10000fdx_cap = 0;
3734 ixgbe->param_lp_5000fdx_cap = 0;
3735 ixgbe->param_lp_2500fdx_cap = 0;
3736 ixgbe->param_lp_1000fdx_cap = 0;
3737 ixgbe->param_lp_100fdx_cap = 0;
3738 ixgbe->param_lp_autoneg_cap = 0;
3739 ixgbe->param_lp_pause_cap = 0;
3740 ixgbe->param_lp_asym_pause_cap = 0;
3741 ixgbe->param_lp_rem_fault = 0;
3742 }
3743
3744 /*
3745 * ixgbe_get_prop - Get a property value out of the configuration file
3746 * ixgbe.conf.
3747 *
3748 * Caller provides the name of the property, a default value, a minimum
3749 * value, and a maximum value.
3750 *
3751 * Return configured value of the property, with default, minimum and
3752 * maximum properly applied.
3753 */
3754 static int
ixgbe_get_prop(ixgbe_t * ixgbe,char * propname,int minval,int maxval,int defval)3755 ixgbe_get_prop(ixgbe_t *ixgbe,
3756 char *propname, /* name of the property */
3757 int minval, /* minimum acceptable value */
3758 int maxval, /* maximim acceptable value */
3759 int defval) /* default value */
3760 {
3761 int value;
3762
3763 /*
3764 * Call ddi_prop_get_int() to read the conf settings
3765 */
3766 value = ddi_prop_get_int(DDI_DEV_T_ANY, ixgbe->dip,
3767 DDI_PROP_DONTPASS, propname, defval);
3768 if (value > maxval)
3769 value = maxval;
3770
3771 if (value < minval)
3772 value = minval;
3773
3774 return (value);
3775 }
3776
3777 /*
3778 * ixgbe_driver_setup_link - Using the link properties to setup the link.
3779 */
3780 int
ixgbe_driver_setup_link(ixgbe_t * ixgbe,boolean_t setup_hw)3781 ixgbe_driver_setup_link(ixgbe_t *ixgbe, boolean_t setup_hw)
3782 {
3783 struct ixgbe_hw *hw = &ixgbe->hw;
3784 ixgbe_link_speed advertised = 0;
3785
3786 /*
3787 * Assemble a list of enabled speeds to auto-negotiate with.
3788 */
3789 if (ixgbe->param_en_10000fdx_cap == 1)
3790 advertised |= IXGBE_LINK_SPEED_10GB_FULL;
3791
3792 if (ixgbe->param_en_5000fdx_cap == 1)
3793 advertised |= IXGBE_LINK_SPEED_5GB_FULL;
3794
3795 if (ixgbe->param_en_2500fdx_cap == 1)
3796 advertised |= IXGBE_LINK_SPEED_2_5GB_FULL;
3797
3798 if (ixgbe->param_en_1000fdx_cap == 1)
3799 advertised |= IXGBE_LINK_SPEED_1GB_FULL;
3800
3801 if (ixgbe->param_en_100fdx_cap == 1)
3802 advertised |= IXGBE_LINK_SPEED_100_FULL;
3803
3804 /*
3805 * As a last resort, autoneg with a default list of speeds.
3806 */
3807 if (ixgbe->param_adv_autoneg_cap == 1 && advertised == 0) {
3808 ixgbe_notice(ixgbe, "Invalid link settings. Setting link "
3809 "to autonegotiate with full capabilities.");
3810
3811 if (hw->mac.type == ixgbe_mac_82598EB)
3812 advertised = IXGBE_LINK_SPEED_82598_AUTONEG;
3813 else
3814 advertised = IXGBE_LINK_SPEED_82599_AUTONEG;
3815 }
3816
3817 if (setup_hw) {
3818 if (ixgbe_setup_link(&ixgbe->hw, advertised,
3819 ixgbe->param_adv_autoneg_cap) != IXGBE_SUCCESS) {
3820 ixgbe_notice(ixgbe, "Setup link failed on this "
3821 "device.");
3822 return (IXGBE_FAILURE);
3823 }
3824 }
3825
3826 return (IXGBE_SUCCESS);
3827 }
3828
3829 /*
3830 * ixgbe_driver_link_check - Link status processing.
3831 *
3832 * This function can be called in both kernel context and interrupt context
3833 */
3834 static void
ixgbe_driver_link_check(ixgbe_t * ixgbe)3835 ixgbe_driver_link_check(ixgbe_t *ixgbe)
3836 {
3837 struct ixgbe_hw *hw = &ixgbe->hw;
3838 ixgbe_link_speed speed = IXGBE_LINK_SPEED_UNKNOWN;
3839 boolean_t link_up = B_FALSE;
3840 boolean_t link_changed = B_FALSE;
3841
3842 ASSERT(mutex_owned(&ixgbe->gen_lock));
3843
3844 (void) ixgbe_check_link(hw, &speed, &link_up, B_FALSE);
3845 if (link_up) {
3846 ixgbe->link_check_complete = B_TRUE;
3847
3848 /* Link is up, enable flow control settings */
3849 (void) ixgbe_fc_enable(hw);
3850
3851 /*
3852 * The Link is up, check whether it was marked as down earlier
3853 */
3854 if (ixgbe->link_state != LINK_STATE_UP) {
3855 switch (speed) {
3856 case IXGBE_LINK_SPEED_10GB_FULL:
3857 ixgbe->link_speed = SPEED_10GB;
3858 break;
3859 case IXGBE_LINK_SPEED_5GB_FULL:
3860 ixgbe->link_speed = SPEED_5GB;
3861 break;
3862 case IXGBE_LINK_SPEED_2_5GB_FULL:
3863 ixgbe->link_speed = SPEED_2_5GB;
3864 break;
3865 case IXGBE_LINK_SPEED_1GB_FULL:
3866 ixgbe->link_speed = SPEED_1GB;
3867 break;
3868 case IXGBE_LINK_SPEED_100_FULL:
3869 ixgbe->link_speed = SPEED_100;
3870 }
3871 ixgbe->link_duplex = LINK_DUPLEX_FULL;
3872 ixgbe->link_state = LINK_STATE_UP;
3873 link_changed = B_TRUE;
3874 }
3875 } else {
3876 if (ixgbe->link_check_complete == B_TRUE ||
3877 (ixgbe->link_check_complete == B_FALSE &&
3878 gethrtime() >= ixgbe->link_check_hrtime)) {
3879 /*
3880 * The link is really down
3881 */
3882 ixgbe->link_check_complete = B_TRUE;
3883
3884 if (ixgbe->link_state != LINK_STATE_DOWN) {
3885 ixgbe->link_speed = 0;
3886 ixgbe->link_duplex = LINK_DUPLEX_UNKNOWN;
3887 ixgbe->link_state = LINK_STATE_DOWN;
3888 link_changed = B_TRUE;
3889 }
3890 }
3891 }
3892
3893 /*
3894 * If we are in an interrupt context, need to re-enable the
3895 * interrupt, which was automasked
3896 */
3897 if (servicing_interrupt() != 0) {
3898 ixgbe->eims |= IXGBE_EICR_LSC;
3899 IXGBE_WRITE_REG(hw, IXGBE_EIMS, ixgbe->eims);
3900 }
3901
3902 if (link_changed) {
3903 mac_link_update(ixgbe->mac_hdl, ixgbe->link_state);
3904 }
3905 }
3906
3907 /*
3908 * ixgbe_sfp_check - sfp module processing done in taskq only for 82599.
3909 */
3910 static void
ixgbe_sfp_check(void * arg)3911 ixgbe_sfp_check(void *arg)
3912 {
3913 ixgbe_t *ixgbe = (ixgbe_t *)arg;
3914 uint32_t eicr = ixgbe->eicr;
3915 struct ixgbe_hw *hw = &ixgbe->hw;
3916
3917 mutex_enter(&ixgbe->gen_lock);
3918 (void) hw->phy.ops.identify_sfp(hw);
3919 if (eicr & IXGBE_EICR_GPI_SDP1_BY_MAC(hw)) {
3920 /* clear the interrupt */
3921 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP1_BY_MAC(hw));
3922
3923 /* if link up, do multispeed fiber setup */
3924 (void) ixgbe_setup_link(hw, IXGBE_LINK_SPEED_82599_AUTONEG,
3925 B_TRUE);
3926 ixgbe_driver_link_check(ixgbe);
3927 ixgbe_get_hw_state(ixgbe);
3928 } else if (eicr & IXGBE_EICR_GPI_SDP2_BY_MAC(hw)) {
3929 /* clear the interrupt */
3930 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP2_BY_MAC(hw));
3931
3932 /* if link up, do sfp module setup */
3933 (void) hw->mac.ops.setup_sfp(hw);
3934
3935 /* do multispeed fiber setup */
3936 (void) ixgbe_setup_link(hw, IXGBE_LINK_SPEED_82599_AUTONEG,
3937 B_TRUE);
3938 ixgbe_driver_link_check(ixgbe);
3939 ixgbe_get_hw_state(ixgbe);
3940 }
3941 mutex_exit(&ixgbe->gen_lock);
3942
3943 /*
3944 * We need to fully re-check the link later.
3945 */
3946 ixgbe->link_check_complete = B_FALSE;
3947 ixgbe->link_check_hrtime = gethrtime() +
3948 (IXGBE_LINK_UP_TIME * 100000000ULL);
3949 }
3950
3951 /*
3952 * ixgbe_overtemp_check - overtemp module processing done in taskq
3953 *
3954 * This routine will only be called on adapters with temperature sensor.
3955 * The indication of over-temperature can be either SDP0 interrupt or the link
3956 * status change interrupt.
3957 */
3958 static void
ixgbe_overtemp_check(void * arg)3959 ixgbe_overtemp_check(void *arg)
3960 {
3961 ixgbe_t *ixgbe = (ixgbe_t *)arg;
3962 struct ixgbe_hw *hw = &ixgbe->hw;
3963 uint32_t eicr = ixgbe->eicr;
3964 ixgbe_link_speed speed;
3965 boolean_t link_up;
3966
3967 mutex_enter(&ixgbe->gen_lock);
3968
3969 /* make sure we know current state of link */
3970 (void) ixgbe_check_link(hw, &speed, &link_up, B_FALSE);
3971
3972 /* check over-temp condition */
3973 if (((eicr & IXGBE_EICR_GPI_SDP0_BY_MAC(hw)) && (!link_up)) ||
3974 (eicr & IXGBE_EICR_LSC)) {
3975 if (hw->phy.ops.check_overtemp(hw) == IXGBE_ERR_OVERTEMP) {
3976 atomic_or_32(&ixgbe->ixgbe_state, IXGBE_OVERTEMP);
3977
3978 /*
3979 * Disable the adapter interrupts
3980 */
3981 ixgbe_disable_adapter_interrupts(ixgbe);
3982
3983 /*
3984 * Disable Rx/Tx units
3985 */
3986 (void) ixgbe_stop_adapter(hw);
3987
3988 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_LOST);
3989 ixgbe_error(ixgbe,
3990 "Problem: Network adapter has been stopped "
3991 "because it has overheated");
3992 ixgbe_error(ixgbe,
3993 "Action: Restart the computer. "
3994 "If the problem persists, power off the system "
3995 "and replace the adapter");
3996 }
3997 }
3998
3999 /* write to clear the interrupt */
4000 IXGBE_WRITE_REG(hw, IXGBE_EICR, eicr);
4001
4002 mutex_exit(&ixgbe->gen_lock);
4003 }
4004
4005 /*
4006 * ixgbe_phy_check - taskq to process interrupts from an external PHY
4007 *
4008 * This routine will only be called on adapters with external PHYs
4009 * (such as X550) that may be trying to raise our attention to some event.
4010 * Currently, this is limited to claiming PHY overtemperature and link status
4011 * change (LSC) events, however this may expand to include other things in
4012 * future adapters.
4013 */
4014 static void
ixgbe_phy_check(void * arg)4015 ixgbe_phy_check(void *arg)
4016 {
4017 ixgbe_t *ixgbe = (ixgbe_t *)arg;
4018 struct ixgbe_hw *hw = &ixgbe->hw;
4019 int rv;
4020
4021 mutex_enter(&ixgbe->gen_lock);
4022
4023 /*
4024 * X550 baseT PHY overtemp and LSC events are handled here.
4025 *
4026 * If an overtemp event occurs, it will be reflected in the
4027 * return value of phy.ops.handle_lasi() and the common code will
4028 * automatically power off the baseT PHY. This is our cue to trigger
4029 * an FMA event.
4030 *
4031 * If a link status change event occurs, phy.ops.handle_lasi() will
4032 * automatically initiate a link setup between the integrated KR PHY
4033 * and the external X557 PHY to ensure that the link speed between
4034 * them matches the link speed of the baseT link.
4035 */
4036 rv = ixgbe_handle_lasi(hw);
4037
4038 if (rv == IXGBE_ERR_OVERTEMP) {
4039 atomic_or_32(&ixgbe->ixgbe_state, IXGBE_OVERTEMP);
4040
4041 /*
4042 * Disable the adapter interrupts
4043 */
4044 ixgbe_disable_adapter_interrupts(ixgbe);
4045
4046 /*
4047 * Disable Rx/Tx units
4048 */
4049 (void) ixgbe_stop_adapter(hw);
4050
4051 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_LOST);
4052 ixgbe_error(ixgbe,
4053 "Problem: Network adapter has been stopped due to a "
4054 "overtemperature event being detected.");
4055 ixgbe_error(ixgbe,
4056 "Action: Shut down or restart the computer. If the issue "
4057 "persists, please take action in accordance with the "
4058 "recommendations from your system vendor.");
4059 }
4060
4061 mutex_exit(&ixgbe->gen_lock);
4062 }
4063
4064 /*
4065 * ixgbe_link_timer - timer for link status detection
4066 */
4067 static void
ixgbe_link_timer(void * arg)4068 ixgbe_link_timer(void *arg)
4069 {
4070 ixgbe_t *ixgbe = (ixgbe_t *)arg;
4071
4072 mutex_enter(&ixgbe->gen_lock);
4073 ixgbe_driver_link_check(ixgbe);
4074 mutex_exit(&ixgbe->gen_lock);
4075 }
4076
4077 /*
4078 * ixgbe_local_timer - Driver watchdog function.
4079 *
4080 * This function will handle the transmit stall check and other routines.
4081 */
4082 static void
ixgbe_local_timer(void * arg)4083 ixgbe_local_timer(void *arg)
4084 {
4085 ixgbe_t *ixgbe = (ixgbe_t *)arg;
4086
4087 if (ixgbe->ixgbe_state & IXGBE_OVERTEMP)
4088 goto out;
4089
4090 if (ixgbe->ixgbe_state & IXGBE_ERROR) {
4091 ixgbe->reset_count++;
4092 if (ixgbe_reset(ixgbe) == IXGBE_SUCCESS)
4093 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_RESTORED);
4094 goto out;
4095 }
4096
4097 if (ixgbe_stall_check(ixgbe)) {
4098 atomic_or_32(&ixgbe->ixgbe_state, IXGBE_STALL);
4099 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_DEGRADED);
4100
4101 ixgbe->reset_count++;
4102 if (ixgbe_reset(ixgbe) == IXGBE_SUCCESS)
4103 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_RESTORED);
4104 }
4105
4106 out:
4107 ixgbe_restart_watchdog_timer(ixgbe);
4108 }
4109
4110 /*
4111 * ixgbe_stall_check - Check for transmit stall.
4112 *
4113 * This function checks if the adapter is stalled (in transmit).
4114 *
4115 * It is called each time the watchdog timeout is invoked.
4116 * If the transmit descriptor reclaim continuously fails,
4117 * the watchdog value will increment by 1. If the watchdog
4118 * value exceeds the threshold, the ixgbe is assumed to
4119 * have stalled and need to be reset.
4120 */
4121 static boolean_t
ixgbe_stall_check(ixgbe_t * ixgbe)4122 ixgbe_stall_check(ixgbe_t *ixgbe)
4123 {
4124 ixgbe_tx_ring_t *tx_ring;
4125 boolean_t result;
4126 int i;
4127
4128 if (ixgbe->link_state != LINK_STATE_UP)
4129 return (B_FALSE);
4130
4131 /*
4132 * If any tx ring is stalled, we'll reset the chipset
4133 */
4134 result = B_FALSE;
4135 for (i = 0; i < ixgbe->num_tx_rings; i++) {
4136 tx_ring = &ixgbe->tx_rings[i];
4137 if (tx_ring->tbd_free <= ixgbe->tx_recycle_thresh) {
4138 tx_ring->tx_recycle(tx_ring);
4139 }
4140
4141 if (tx_ring->recycle_fail > 0)
4142 tx_ring->stall_watchdog++;
4143 else
4144 tx_ring->stall_watchdog = 0;
4145
4146 if (tx_ring->stall_watchdog >= STALL_WATCHDOG_TIMEOUT) {
4147 result = B_TRUE;
4148 break;
4149 }
4150 }
4151
4152 if (result) {
4153 tx_ring->stall_watchdog = 0;
4154 tx_ring->recycle_fail = 0;
4155 }
4156
4157 return (result);
4158 }
4159
4160
4161 /*
4162 * is_valid_mac_addr - Check if the mac address is valid.
4163 */
4164 static boolean_t
is_valid_mac_addr(uint8_t * mac_addr)4165 is_valid_mac_addr(uint8_t *mac_addr)
4166 {
4167 const uint8_t addr_test1[6] = { 0, 0, 0, 0, 0, 0 };
4168 const uint8_t addr_test2[6] =
4169 { 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF };
4170
4171 if (!(bcmp(addr_test1, mac_addr, ETHERADDRL)) ||
4172 !(bcmp(addr_test2, mac_addr, ETHERADDRL)))
4173 return (B_FALSE);
4174
4175 return (B_TRUE);
4176 }
4177
4178 static boolean_t
ixgbe_find_mac_address(ixgbe_t * ixgbe)4179 ixgbe_find_mac_address(ixgbe_t *ixgbe)
4180 {
4181 #ifdef __sparc
4182 struct ixgbe_hw *hw = &ixgbe->hw;
4183 uchar_t *bytes;
4184 struct ether_addr sysaddr;
4185 uint_t nelts;
4186 int err;
4187 boolean_t found = B_FALSE;
4188
4189 /*
4190 * The "vendor's factory-set address" may already have
4191 * been extracted from the chip, but if the property
4192 * "local-mac-address" is set we use that instead.
4193 *
4194 * We check whether it looks like an array of 6
4195 * bytes (which it should, if OBP set it). If we can't
4196 * make sense of it this way, we'll ignore it.
4197 */
4198 err = ddi_prop_lookup_byte_array(DDI_DEV_T_ANY, ixgbe->dip,
4199 DDI_PROP_DONTPASS, "local-mac-address", &bytes, &nelts);
4200 if (err == DDI_PROP_SUCCESS) {
4201 if (nelts == ETHERADDRL) {
4202 while (nelts--)
4203 hw->mac.addr[nelts] = bytes[nelts];
4204 found = B_TRUE;
4205 }
4206 ddi_prop_free(bytes);
4207 }
4208
4209 /*
4210 * Look up the OBP property "local-mac-address?". If the user has set
4211 * 'local-mac-address? = false', use "the system address" instead.
4212 */
4213 if (ddi_prop_lookup_byte_array(DDI_DEV_T_ANY, ixgbe->dip, 0,
4214 "local-mac-address?", &bytes, &nelts) == DDI_PROP_SUCCESS) {
4215 if (strncmp("false", (caddr_t)bytes, (size_t)nelts) == 0) {
4216 if (localetheraddr(NULL, &sysaddr) != 0) {
4217 bcopy(&sysaddr, hw->mac.addr, ETHERADDRL);
4218 found = B_TRUE;
4219 }
4220 }
4221 ddi_prop_free(bytes);
4222 }
4223
4224 /*
4225 * Finally(!), if there's a valid "mac-address" property (created
4226 * if we netbooted from this interface), we must use this instead
4227 * of any of the above to ensure that the NFS/install server doesn't
4228 * get confused by the address changing as illumos takes over!
4229 */
4230 err = ddi_prop_lookup_byte_array(DDI_DEV_T_ANY, ixgbe->dip,
4231 DDI_PROP_DONTPASS, "mac-address", &bytes, &nelts);
4232 if (err == DDI_PROP_SUCCESS) {
4233 if (nelts == ETHERADDRL) {
4234 while (nelts--)
4235 hw->mac.addr[nelts] = bytes[nelts];
4236 found = B_TRUE;
4237 }
4238 ddi_prop_free(bytes);
4239 }
4240
4241 if (found) {
4242 bcopy(hw->mac.addr, hw->mac.perm_addr, ETHERADDRL);
4243 return (B_TRUE);
4244 }
4245 #else
4246 _NOTE(ARGUNUSED(ixgbe));
4247 #endif
4248
4249 return (B_TRUE);
4250 }
4251
4252 #pragma inline(ixgbe_arm_watchdog_timer)
4253 static void
ixgbe_arm_watchdog_timer(ixgbe_t * ixgbe)4254 ixgbe_arm_watchdog_timer(ixgbe_t *ixgbe)
4255 {
4256 /*
4257 * Fire a watchdog timer
4258 */
4259 ixgbe->watchdog_tid =
4260 timeout(ixgbe_local_timer,
4261 (void *)ixgbe, 1 * drv_usectohz(1000000));
4262
4263 }
4264
4265 /*
4266 * ixgbe_enable_watchdog_timer - Enable and start the driver watchdog timer.
4267 */
4268 void
ixgbe_enable_watchdog_timer(ixgbe_t * ixgbe)4269 ixgbe_enable_watchdog_timer(ixgbe_t *ixgbe)
4270 {
4271 mutex_enter(&ixgbe->watchdog_lock);
4272
4273 if (!ixgbe->watchdog_enable) {
4274 ixgbe->watchdog_enable = B_TRUE;
4275 ixgbe->watchdog_start = B_TRUE;
4276 ixgbe_arm_watchdog_timer(ixgbe);
4277 }
4278
4279 mutex_exit(&ixgbe->watchdog_lock);
4280 }
4281
4282 /*
4283 * ixgbe_disable_watchdog_timer - Disable and stop the driver watchdog timer.
4284 */
4285 void
ixgbe_disable_watchdog_timer(ixgbe_t * ixgbe)4286 ixgbe_disable_watchdog_timer(ixgbe_t *ixgbe)
4287 {
4288 timeout_id_t tid;
4289
4290 mutex_enter(&ixgbe->watchdog_lock);
4291
4292 ixgbe->watchdog_enable = B_FALSE;
4293 ixgbe->watchdog_start = B_FALSE;
4294 tid = ixgbe->watchdog_tid;
4295 ixgbe->watchdog_tid = 0;
4296
4297 mutex_exit(&ixgbe->watchdog_lock);
4298
4299 if (tid != 0)
4300 (void) untimeout(tid);
4301 }
4302
4303 /*
4304 * ixgbe_start_watchdog_timer - Start the driver watchdog timer.
4305 */
4306 void
ixgbe_start_watchdog_timer(ixgbe_t * ixgbe)4307 ixgbe_start_watchdog_timer(ixgbe_t *ixgbe)
4308 {
4309 mutex_enter(&ixgbe->watchdog_lock);
4310
4311 if (ixgbe->watchdog_enable) {
4312 if (!ixgbe->watchdog_start) {
4313 ixgbe->watchdog_start = B_TRUE;
4314 ixgbe_arm_watchdog_timer(ixgbe);
4315 }
4316 }
4317
4318 mutex_exit(&ixgbe->watchdog_lock);
4319 }
4320
4321 /*
4322 * ixgbe_restart_watchdog_timer - Restart the driver watchdog timer.
4323 */
4324 static void
ixgbe_restart_watchdog_timer(ixgbe_t * ixgbe)4325 ixgbe_restart_watchdog_timer(ixgbe_t *ixgbe)
4326 {
4327 mutex_enter(&ixgbe->watchdog_lock);
4328
4329 if (ixgbe->watchdog_start)
4330 ixgbe_arm_watchdog_timer(ixgbe);
4331
4332 mutex_exit(&ixgbe->watchdog_lock);
4333 }
4334
4335 /*
4336 * ixgbe_stop_watchdog_timer - Stop the driver watchdog timer.
4337 */
4338 void
ixgbe_stop_watchdog_timer(ixgbe_t * ixgbe)4339 ixgbe_stop_watchdog_timer(ixgbe_t *ixgbe)
4340 {
4341 timeout_id_t tid;
4342
4343 mutex_enter(&ixgbe->watchdog_lock);
4344
4345 ixgbe->watchdog_start = B_FALSE;
4346 tid = ixgbe->watchdog_tid;
4347 ixgbe->watchdog_tid = 0;
4348
4349 mutex_exit(&ixgbe->watchdog_lock);
4350
4351 if (tid != 0)
4352 (void) untimeout(tid);
4353 }
4354
4355 /*
4356 * ixgbe_disable_adapter_interrupts - Disable all adapter interrupts.
4357 */
4358 static void
ixgbe_disable_adapter_interrupts(ixgbe_t * ixgbe)4359 ixgbe_disable_adapter_interrupts(ixgbe_t *ixgbe)
4360 {
4361 struct ixgbe_hw *hw = &ixgbe->hw;
4362
4363 /*
4364 * mask all interrupts off
4365 */
4366 IXGBE_WRITE_REG(hw, IXGBE_EIMC, 0xffffffff);
4367
4368 /*
4369 * for MSI-X, also disable autoclear
4370 */
4371 if (ixgbe->intr_type == DDI_INTR_TYPE_MSIX) {
4372 IXGBE_WRITE_REG(hw, IXGBE_EIAC, 0x0);
4373 }
4374
4375 IXGBE_WRITE_FLUSH(hw);
4376 }
4377
4378 /*
4379 * ixgbe_enable_adapter_interrupts - Enable all hardware interrupts.
4380 */
4381 static void
ixgbe_enable_adapter_interrupts(ixgbe_t * ixgbe)4382 ixgbe_enable_adapter_interrupts(ixgbe_t *ixgbe)
4383 {
4384 struct ixgbe_hw *hw = &ixgbe->hw;
4385 uint32_t eiac, eiam;
4386 uint32_t gpie = IXGBE_READ_REG(hw, IXGBE_GPIE);
4387
4388 /* interrupt types to enable */
4389 ixgbe->eims = IXGBE_EIMS_ENABLE_MASK; /* shared code default */
4390 ixgbe->eims &= ~IXGBE_EIMS_TCP_TIMER; /* minus tcp timer */
4391 ixgbe->eims |= ixgbe->capab->other_intr; /* "other" interrupt types */
4392
4393 /* enable automask on "other" causes that this adapter can generate */
4394 eiam = ixgbe->capab->other_intr;
4395
4396 /*
4397 * msi-x mode
4398 */
4399 if (ixgbe->intr_type == DDI_INTR_TYPE_MSIX) {
4400 /* enable autoclear but not on bits 29:20 */
4401 eiac = (ixgbe->eims & ~IXGBE_OTHER_INTR);
4402
4403 /* general purpose interrupt enable */
4404 gpie |= (IXGBE_GPIE_MSIX_MODE
4405 | IXGBE_GPIE_PBA_SUPPORT
4406 | IXGBE_GPIE_OCD
4407 | IXGBE_GPIE_EIAME);
4408 /*
4409 * non-msi-x mode
4410 */
4411 } else {
4412
4413 /* disable autoclear, leave gpie at default */
4414 eiac = 0;
4415
4416 /*
4417 * General purpose interrupt enable.
4418 * For 82599, X540 and X550, extended interrupt
4419 * automask enable only in MSI or MSI-X mode
4420 */
4421 if ((hw->mac.type == ixgbe_mac_82598EB) ||
4422 (ixgbe->intr_type == DDI_INTR_TYPE_MSI)) {
4423 gpie |= IXGBE_GPIE_EIAME;
4424 }
4425 }
4426
4427 /* Enable specific "other" interrupt types */
4428 switch (hw->mac.type) {
4429 case ixgbe_mac_82598EB:
4430 gpie |= ixgbe->capab->other_gpie;
4431 break;
4432
4433 case ixgbe_mac_82599EB:
4434 case ixgbe_mac_X540:
4435 case ixgbe_mac_X550:
4436 case ixgbe_mac_X550EM_x:
4437 case ixgbe_mac_X550EM_a:
4438 gpie |= ixgbe->capab->other_gpie;
4439
4440 /* Enable RSC Delay 8us when LRO enabled */
4441 if (ixgbe->lro_enable) {
4442 gpie |= (1 << IXGBE_GPIE_RSC_DELAY_SHIFT);
4443 }
4444 break;
4445
4446 default:
4447 break;
4448 }
4449
4450 /* write to interrupt control registers */
4451 IXGBE_WRITE_REG(hw, IXGBE_EIMS, ixgbe->eims);
4452 IXGBE_WRITE_REG(hw, IXGBE_EIAC, eiac);
4453 IXGBE_WRITE_REG(hw, IXGBE_EIAM, eiam);
4454 IXGBE_WRITE_REG(hw, IXGBE_GPIE, gpie);
4455 IXGBE_WRITE_FLUSH(hw);
4456 }
4457
4458 /*
4459 * ixgbe_loopback_ioctl - Loopback support.
4460 */
4461 enum ioc_reply
ixgbe_loopback_ioctl(ixgbe_t * ixgbe,struct iocblk * iocp,mblk_t * mp)4462 ixgbe_loopback_ioctl(ixgbe_t *ixgbe, struct iocblk *iocp, mblk_t *mp)
4463 {
4464 lb_info_sz_t *lbsp;
4465 lb_property_t *lbpp;
4466 uint32_t *lbmp;
4467 uint32_t size;
4468 uint32_t value;
4469
4470 if (mp->b_cont == NULL)
4471 return (IOC_INVAL);
4472
4473 switch (iocp->ioc_cmd) {
4474 default:
4475 return (IOC_INVAL);
4476
4477 case LB_GET_INFO_SIZE:
4478 size = sizeof (lb_info_sz_t);
4479 if (iocp->ioc_count != size)
4480 return (IOC_INVAL);
4481
4482 value = sizeof (lb_normal);
4483 value += sizeof (lb_mac);
4484 value += sizeof (lb_external);
4485
4486 lbsp = (lb_info_sz_t *)(uintptr_t)mp->b_cont->b_rptr;
4487 *lbsp = value;
4488 break;
4489
4490 case LB_GET_INFO:
4491 value = sizeof (lb_normal);
4492 value += sizeof (lb_mac);
4493 value += sizeof (lb_external);
4494
4495 size = value;
4496 if (iocp->ioc_count != size)
4497 return (IOC_INVAL);
4498
4499 value = 0;
4500 lbpp = (lb_property_t *)(uintptr_t)mp->b_cont->b_rptr;
4501
4502 lbpp[value++] = lb_normal;
4503 lbpp[value++] = lb_mac;
4504 lbpp[value++] = lb_external;
4505 break;
4506
4507 case LB_GET_MODE:
4508 size = sizeof (uint32_t);
4509 if (iocp->ioc_count != size)
4510 return (IOC_INVAL);
4511
4512 lbmp = (uint32_t *)(uintptr_t)mp->b_cont->b_rptr;
4513 *lbmp = ixgbe->loopback_mode;
4514 break;
4515
4516 case LB_SET_MODE:
4517 size = 0;
4518 if (iocp->ioc_count != sizeof (uint32_t))
4519 return (IOC_INVAL);
4520
4521 lbmp = (uint32_t *)(uintptr_t)mp->b_cont->b_rptr;
4522 if (!ixgbe_set_loopback_mode(ixgbe, *lbmp))
4523 return (IOC_INVAL);
4524 break;
4525 }
4526
4527 iocp->ioc_count = size;
4528 iocp->ioc_error = 0;
4529
4530 if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != DDI_FM_OK) {
4531 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_DEGRADED);
4532 return (IOC_INVAL);
4533 }
4534
4535 return (IOC_REPLY);
4536 }
4537
4538 /*
4539 * ixgbe_set_loopback_mode - Setup loopback based on the loopback mode.
4540 */
4541 static boolean_t
ixgbe_set_loopback_mode(ixgbe_t * ixgbe,uint32_t mode)4542 ixgbe_set_loopback_mode(ixgbe_t *ixgbe, uint32_t mode)
4543 {
4544 if (mode == ixgbe->loopback_mode)
4545 return (B_TRUE);
4546
4547 ixgbe->loopback_mode = mode;
4548
4549 if (mode == IXGBE_LB_NONE) {
4550 /*
4551 * Reset the chip
4552 */
4553 (void) ixgbe_reset(ixgbe);
4554 return (B_TRUE);
4555 }
4556
4557 mutex_enter(&ixgbe->gen_lock);
4558
4559 switch (mode) {
4560 default:
4561 mutex_exit(&ixgbe->gen_lock);
4562 return (B_FALSE);
4563
4564 case IXGBE_LB_EXTERNAL:
4565 break;
4566
4567 case IXGBE_LB_INTERNAL_MAC:
4568 ixgbe_set_internal_mac_loopback(ixgbe);
4569 break;
4570 }
4571
4572 mutex_exit(&ixgbe->gen_lock);
4573
4574 return (B_TRUE);
4575 }
4576
4577 /*
4578 * ixgbe_set_internal_mac_loopback - Set the internal MAC loopback mode.
4579 */
4580 static void
ixgbe_set_internal_mac_loopback(ixgbe_t * ixgbe)4581 ixgbe_set_internal_mac_loopback(ixgbe_t *ixgbe)
4582 {
4583 struct ixgbe_hw *hw;
4584 uint32_t reg;
4585 uint8_t atlas;
4586
4587 hw = &ixgbe->hw;
4588
4589 /*
4590 * Setup MAC loopback
4591 */
4592 reg = IXGBE_READ_REG(&ixgbe->hw, IXGBE_HLREG0);
4593 reg |= IXGBE_HLREG0_LPBK;
4594 IXGBE_WRITE_REG(&ixgbe->hw, IXGBE_HLREG0, reg);
4595
4596 reg = IXGBE_READ_REG(&ixgbe->hw, IXGBE_AUTOC);
4597 reg &= ~IXGBE_AUTOC_LMS_MASK;
4598 IXGBE_WRITE_REG(&ixgbe->hw, IXGBE_AUTOC, reg);
4599
4600 /*
4601 * Disable Atlas Tx lanes to keep packets in loopback and not on wire
4602 */
4603 switch (hw->mac.type) {
4604 case ixgbe_mac_82598EB:
4605 (void) ixgbe_read_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_LPBK,
4606 &atlas);
4607 atlas |= IXGBE_ATLAS_PDN_TX_REG_EN;
4608 (void) ixgbe_write_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_LPBK,
4609 atlas);
4610
4611 (void) ixgbe_read_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_10G,
4612 &atlas);
4613 atlas |= IXGBE_ATLAS_PDN_TX_10G_QL_ALL;
4614 (void) ixgbe_write_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_10G,
4615 atlas);
4616
4617 (void) ixgbe_read_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_1G,
4618 &atlas);
4619 atlas |= IXGBE_ATLAS_PDN_TX_1G_QL_ALL;
4620 (void) ixgbe_write_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_1G,
4621 atlas);
4622
4623 (void) ixgbe_read_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_AN,
4624 &atlas);
4625 atlas |= IXGBE_ATLAS_PDN_TX_AN_QL_ALL;
4626 (void) ixgbe_write_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_AN,
4627 atlas);
4628 break;
4629
4630 case ixgbe_mac_82599EB:
4631 case ixgbe_mac_X540:
4632 case ixgbe_mac_X550:
4633 case ixgbe_mac_X550EM_x:
4634 case ixgbe_mac_X550EM_a:
4635 reg = IXGBE_READ_REG(&ixgbe->hw, IXGBE_AUTOC);
4636 reg |= (IXGBE_AUTOC_FLU |
4637 IXGBE_AUTOC_10G_KX4);
4638 IXGBE_WRITE_REG(&ixgbe->hw, IXGBE_AUTOC, reg);
4639
4640 (void) ixgbe_setup_link(&ixgbe->hw, IXGBE_LINK_SPEED_10GB_FULL,
4641 B_FALSE);
4642 break;
4643
4644 default:
4645 break;
4646 }
4647 }
4648
4649 #pragma inline(ixgbe_intr_rx_work)
4650 /*
4651 * ixgbe_intr_rx_work - RX processing of ISR.
4652 */
4653 static void
ixgbe_intr_rx_work(ixgbe_rx_ring_t * rx_ring)4654 ixgbe_intr_rx_work(ixgbe_rx_ring_t *rx_ring)
4655 {
4656 mblk_t *mp;
4657
4658 mutex_enter(&rx_ring->rx_lock);
4659
4660 mp = ixgbe_ring_rx(rx_ring, IXGBE_POLL_NULL);
4661 mutex_exit(&rx_ring->rx_lock);
4662
4663 if (mp != NULL)
4664 mac_rx_ring(rx_ring->ixgbe->mac_hdl, rx_ring->ring_handle, mp,
4665 rx_ring->ring_gen_num);
4666 }
4667
4668 #pragma inline(ixgbe_intr_tx_work)
4669 /*
4670 * ixgbe_intr_tx_work - TX processing of ISR.
4671 */
4672 static void
ixgbe_intr_tx_work(ixgbe_tx_ring_t * tx_ring)4673 ixgbe_intr_tx_work(ixgbe_tx_ring_t *tx_ring)
4674 {
4675 ixgbe_t *ixgbe = tx_ring->ixgbe;
4676
4677 /*
4678 * Recycle the tx descriptors
4679 */
4680 tx_ring->tx_recycle(tx_ring);
4681
4682 /*
4683 * Schedule the re-transmit
4684 */
4685 if (tx_ring->reschedule &&
4686 (tx_ring->tbd_free >= ixgbe->tx_resched_thresh)) {
4687 tx_ring->reschedule = B_FALSE;
4688 mac_tx_ring_update(tx_ring->ixgbe->mac_hdl,
4689 tx_ring->ring_handle);
4690 tx_ring->stat_reschedule++;
4691 }
4692 }
4693
4694 #pragma inline(ixgbe_intr_other_work)
4695 /*
4696 * ixgbe_intr_other_work - Process interrupt types other than tx/rx
4697 */
4698 static void
ixgbe_intr_other_work(ixgbe_t * ixgbe,uint32_t eicr)4699 ixgbe_intr_other_work(ixgbe_t *ixgbe, uint32_t eicr)
4700 {
4701 struct ixgbe_hw *hw = &ixgbe->hw;
4702
4703 ASSERT(mutex_owned(&ixgbe->gen_lock));
4704
4705 /*
4706 * handle link status change
4707 */
4708 if (eicr & IXGBE_EICR_LSC) {
4709 ixgbe_driver_link_check(ixgbe);
4710 ixgbe_get_hw_state(ixgbe);
4711 }
4712
4713 /*
4714 * check for fan failure on adapters with fans
4715 */
4716 if ((ixgbe->capab->flags & IXGBE_FLAG_FAN_FAIL_CAPABLE) &&
4717 (eicr & IXGBE_EICR_GPI_SDP1)) {
4718 atomic_or_32(&ixgbe->ixgbe_state, IXGBE_OVERTEMP);
4719
4720 /*
4721 * Disable the adapter interrupts
4722 */
4723 ixgbe_disable_adapter_interrupts(ixgbe);
4724
4725 /*
4726 * Disable Rx/Tx units
4727 */
4728 (void) ixgbe_stop_adapter(&ixgbe->hw);
4729
4730 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_LOST);
4731 ixgbe_error(ixgbe,
4732 "Problem: Network adapter has been stopped "
4733 "because the fan has stopped.\n");
4734 ixgbe_error(ixgbe,
4735 "Action: Replace the adapter.\n");
4736
4737 /* re-enable the interrupt, which was automasked */
4738 ixgbe->eims |= IXGBE_EICR_GPI_SDP1;
4739 }
4740
4741 /*
4742 * Do SFP check for adapters with hot-plug capability
4743 */
4744 if ((ixgbe->capab->flags & IXGBE_FLAG_SFP_PLUG_CAPABLE) &&
4745 ((eicr & IXGBE_EICR_GPI_SDP1_BY_MAC(hw)) ||
4746 (eicr & IXGBE_EICR_GPI_SDP2_BY_MAC(hw)))) {
4747 ixgbe->eicr = eicr;
4748 if ((ddi_taskq_dispatch(ixgbe->sfp_taskq,
4749 ixgbe_sfp_check, (void *)ixgbe,
4750 DDI_NOSLEEP)) != DDI_SUCCESS) {
4751 ixgbe_log(ixgbe, "No memory available to dispatch "
4752 "taskq for SFP check");
4753 }
4754 }
4755
4756 /*
4757 * Do over-temperature check for adapters with temp sensor
4758 */
4759 if ((ixgbe->capab->flags & IXGBE_FLAG_TEMP_SENSOR_CAPABLE) &&
4760 ((eicr & IXGBE_EICR_GPI_SDP0_BY_MAC(hw)) ||
4761 (eicr & IXGBE_EICR_LSC))) {
4762 ixgbe->eicr = eicr;
4763 if ((ddi_taskq_dispatch(ixgbe->overtemp_taskq,
4764 ixgbe_overtemp_check, (void *)ixgbe,
4765 DDI_NOSLEEP)) != DDI_SUCCESS) {
4766 ixgbe_log(ixgbe, "No memory available to dispatch "
4767 "taskq for overtemp check");
4768 }
4769 }
4770
4771 /*
4772 * Process an external PHY interrupt
4773 */
4774 if (hw->device_id == IXGBE_DEV_ID_X550EM_X_10G_T &&
4775 (eicr & IXGBE_EICR_GPI_SDP0_X540)) {
4776 ixgbe->eicr = eicr;
4777 if ((ddi_taskq_dispatch(ixgbe->phy_taskq,
4778 ixgbe_phy_check, (void *)ixgbe,
4779 DDI_NOSLEEP)) != DDI_SUCCESS) {
4780 ixgbe_log(ixgbe, "No memory available to dispatch "
4781 "taskq for PHY check");
4782 }
4783 }
4784 }
4785
4786 /*
4787 * ixgbe_intr_legacy - Interrupt handler for legacy interrupts.
4788 */
4789 static uint_t
ixgbe_intr_legacy(void * arg1,void * arg2)4790 ixgbe_intr_legacy(void *arg1, void *arg2)
4791 {
4792 ixgbe_t *ixgbe = (ixgbe_t *)arg1;
4793 struct ixgbe_hw *hw = &ixgbe->hw;
4794 ixgbe_tx_ring_t *tx_ring;
4795 ixgbe_rx_ring_t *rx_ring;
4796 uint32_t eicr;
4797 mblk_t *mp;
4798 boolean_t tx_reschedule;
4799 uint_t result;
4800
4801 _NOTE(ARGUNUSED(arg2));
4802
4803 mutex_enter(&ixgbe->gen_lock);
4804 if (ixgbe->ixgbe_state & IXGBE_SUSPENDED) {
4805 mutex_exit(&ixgbe->gen_lock);
4806 return (DDI_INTR_UNCLAIMED);
4807 }
4808
4809 mp = NULL;
4810 tx_reschedule = B_FALSE;
4811
4812 /*
4813 * Any bit set in eicr: claim this interrupt
4814 */
4815 eicr = IXGBE_READ_REG(hw, IXGBE_EICR);
4816
4817 if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != DDI_FM_OK) {
4818 mutex_exit(&ixgbe->gen_lock);
4819 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_DEGRADED);
4820 atomic_or_32(&ixgbe->ixgbe_state, IXGBE_ERROR);
4821 return (DDI_INTR_CLAIMED);
4822 }
4823
4824 if (eicr) {
4825 /*
4826 * For legacy interrupt, we have only one interrupt,
4827 * so we have only one rx ring and one tx ring enabled.
4828 */
4829 ASSERT(ixgbe->num_rx_rings == 1);
4830 ASSERT(ixgbe->num_tx_rings == 1);
4831
4832 /*
4833 * For legacy interrupt, rx rings[0] will use RTxQ[0].
4834 */
4835 if (eicr & 0x1) {
4836 ixgbe->eimc |= IXGBE_EICR_RTX_QUEUE;
4837 IXGBE_WRITE_REG(hw, IXGBE_EIMC, ixgbe->eimc);
4838 ixgbe->eims |= IXGBE_EICR_RTX_QUEUE;
4839 /*
4840 * Clean the rx descriptors
4841 */
4842 rx_ring = &ixgbe->rx_rings[0];
4843 mp = ixgbe_ring_rx(rx_ring, IXGBE_POLL_NULL);
4844 }
4845
4846 /*
4847 * For legacy interrupt, tx rings[0] will use RTxQ[1].
4848 */
4849 if (eicr & 0x2) {
4850 /*
4851 * Recycle the tx descriptors
4852 */
4853 tx_ring = &ixgbe->tx_rings[0];
4854 tx_ring->tx_recycle(tx_ring);
4855
4856 /*
4857 * Schedule the re-transmit
4858 */
4859 tx_reschedule = (tx_ring->reschedule &&
4860 (tx_ring->tbd_free >= ixgbe->tx_resched_thresh));
4861 }
4862
4863 /* any interrupt type other than tx/rx */
4864 if (eicr & ixgbe->capab->other_intr) {
4865 switch (hw->mac.type) {
4866 case ixgbe_mac_82598EB:
4867 ixgbe->eims &= ~(eicr & IXGBE_OTHER_INTR);
4868 break;
4869
4870 case ixgbe_mac_82599EB:
4871 case ixgbe_mac_X540:
4872 case ixgbe_mac_X550:
4873 case ixgbe_mac_X550EM_x:
4874 case ixgbe_mac_X550EM_a:
4875 ixgbe->eimc = IXGBE_82599_OTHER_INTR;
4876 IXGBE_WRITE_REG(hw, IXGBE_EIMC, ixgbe->eimc);
4877 break;
4878
4879 default:
4880 break;
4881 }
4882 ixgbe_intr_other_work(ixgbe, eicr);
4883 ixgbe->eims &= ~(eicr & IXGBE_OTHER_INTR);
4884 }
4885
4886 mutex_exit(&ixgbe->gen_lock);
4887
4888 result = DDI_INTR_CLAIMED;
4889 } else {
4890 mutex_exit(&ixgbe->gen_lock);
4891
4892 /*
4893 * No interrupt cause bits set: don't claim this interrupt.
4894 */
4895 result = DDI_INTR_UNCLAIMED;
4896 }
4897
4898 /* re-enable the interrupts which were automasked */
4899 IXGBE_WRITE_REG(hw, IXGBE_EIMS, ixgbe->eims);
4900
4901 /*
4902 * Do the following work outside of the gen_lock
4903 */
4904 if (mp != NULL) {
4905 mac_rx_ring(rx_ring->ixgbe->mac_hdl, rx_ring->ring_handle, mp,
4906 rx_ring->ring_gen_num);
4907 }
4908
4909 if (tx_reschedule) {
4910 tx_ring->reschedule = B_FALSE;
4911 mac_tx_ring_update(ixgbe->mac_hdl, tx_ring->ring_handle);
4912 tx_ring->stat_reschedule++;
4913 }
4914
4915 return (result);
4916 }
4917
4918 /*
4919 * ixgbe_intr_msi - Interrupt handler for MSI.
4920 */
4921 static uint_t
ixgbe_intr_msi(void * arg1,void * arg2)4922 ixgbe_intr_msi(void *arg1, void *arg2)
4923 {
4924 ixgbe_t *ixgbe = (ixgbe_t *)arg1;
4925 struct ixgbe_hw *hw = &ixgbe->hw;
4926 uint32_t eicr;
4927
4928 _NOTE(ARGUNUSED(arg2));
4929
4930 eicr = IXGBE_READ_REG(hw, IXGBE_EICR);
4931
4932 if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != DDI_FM_OK) {
4933 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_DEGRADED);
4934 atomic_or_32(&ixgbe->ixgbe_state, IXGBE_ERROR);
4935 return (DDI_INTR_CLAIMED);
4936 }
4937
4938 /*
4939 * For MSI interrupt, we have only one vector,
4940 * so we have only one rx ring and one tx ring enabled.
4941 */
4942 ASSERT(ixgbe->num_rx_rings == 1);
4943 ASSERT(ixgbe->num_tx_rings == 1);
4944
4945 /*
4946 * For MSI interrupt, rx rings[0] will use RTxQ[0].
4947 */
4948 if (eicr & 0x1) {
4949 ixgbe_intr_rx_work(&ixgbe->rx_rings[0]);
4950 }
4951
4952 /*
4953 * For MSI interrupt, tx rings[0] will use RTxQ[1].
4954 */
4955 if (eicr & 0x2) {
4956 ixgbe_intr_tx_work(&ixgbe->tx_rings[0]);
4957 }
4958
4959 /* any interrupt type other than tx/rx */
4960 if (eicr & ixgbe->capab->other_intr) {
4961 mutex_enter(&ixgbe->gen_lock);
4962 switch (hw->mac.type) {
4963 case ixgbe_mac_82598EB:
4964 ixgbe->eims &= ~(eicr & IXGBE_OTHER_INTR);
4965 break;
4966
4967 case ixgbe_mac_82599EB:
4968 case ixgbe_mac_X540:
4969 case ixgbe_mac_X550:
4970 case ixgbe_mac_X550EM_x:
4971 case ixgbe_mac_X550EM_a:
4972 ixgbe->eimc = IXGBE_82599_OTHER_INTR;
4973 IXGBE_WRITE_REG(hw, IXGBE_EIMC, ixgbe->eimc);
4974 break;
4975
4976 default:
4977 break;
4978 }
4979 ixgbe_intr_other_work(ixgbe, eicr);
4980 ixgbe->eims &= ~(eicr & IXGBE_OTHER_INTR);
4981 mutex_exit(&ixgbe->gen_lock);
4982 }
4983
4984 /* re-enable the interrupts which were automasked */
4985 IXGBE_WRITE_REG(hw, IXGBE_EIMS, ixgbe->eims);
4986
4987 return (DDI_INTR_CLAIMED);
4988 }
4989
4990 /*
4991 * ixgbe_intr_msix - Interrupt handler for MSI-X.
4992 */
4993 static uint_t
ixgbe_intr_msix(void * arg1,void * arg2)4994 ixgbe_intr_msix(void *arg1, void *arg2)
4995 {
4996 ixgbe_intr_vector_t *vect = (ixgbe_intr_vector_t *)arg1;
4997 ixgbe_t *ixgbe = vect->ixgbe;
4998 struct ixgbe_hw *hw = &ixgbe->hw;
4999 uint32_t eicr;
5000 int r_idx = 0;
5001
5002 _NOTE(ARGUNUSED(arg2));
5003
5004 /*
5005 * Clean each rx ring that has its bit set in the map
5006 */
5007 r_idx = bt_getlowbit(vect->rx_map, 0, (ixgbe->num_rx_rings - 1));
5008 while (r_idx >= 0) {
5009 ixgbe_intr_rx_work(&ixgbe->rx_rings[r_idx]);
5010 r_idx = bt_getlowbit(vect->rx_map, (r_idx + 1),
5011 (ixgbe->num_rx_rings - 1));
5012 }
5013
5014 /*
5015 * Clean each tx ring that has its bit set in the map
5016 */
5017 r_idx = bt_getlowbit(vect->tx_map, 0, (ixgbe->num_tx_rings - 1));
5018 while (r_idx >= 0) {
5019 ixgbe_intr_tx_work(&ixgbe->tx_rings[r_idx]);
5020 r_idx = bt_getlowbit(vect->tx_map, (r_idx + 1),
5021 (ixgbe->num_tx_rings - 1));
5022 }
5023
5024
5025 /*
5026 * Clean other interrupt (link change) that has its bit set in the map
5027 */
5028 if (BT_TEST(vect->other_map, 0) == 1) {
5029 eicr = IXGBE_READ_REG(hw, IXGBE_EICR);
5030
5031 if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) !=
5032 DDI_FM_OK) {
5033 ddi_fm_service_impact(ixgbe->dip,
5034 DDI_SERVICE_DEGRADED);
5035 atomic_or_32(&ixgbe->ixgbe_state, IXGBE_ERROR);
5036 return (DDI_INTR_CLAIMED);
5037 }
5038
5039 /*
5040 * Check "other" cause bits: any interrupt type other than tx/rx
5041 */
5042 if (eicr & ixgbe->capab->other_intr) {
5043 mutex_enter(&ixgbe->gen_lock);
5044 switch (hw->mac.type) {
5045 case ixgbe_mac_82598EB:
5046 ixgbe->eims &= ~(eicr & IXGBE_OTHER_INTR);
5047 ixgbe_intr_other_work(ixgbe, eicr);
5048 break;
5049
5050 case ixgbe_mac_82599EB:
5051 case ixgbe_mac_X540:
5052 case ixgbe_mac_X550:
5053 case ixgbe_mac_X550EM_x:
5054 case ixgbe_mac_X550EM_a:
5055 ixgbe->eims |= IXGBE_EICR_RTX_QUEUE;
5056 ixgbe_intr_other_work(ixgbe, eicr);
5057 break;
5058
5059 default:
5060 break;
5061 }
5062 mutex_exit(&ixgbe->gen_lock);
5063 }
5064
5065 /* re-enable the interrupts which were automasked */
5066 IXGBE_WRITE_REG(hw, IXGBE_EIMS, ixgbe->eims);
5067 }
5068
5069 return (DDI_INTR_CLAIMED);
5070 }
5071
5072 /*
5073 * ixgbe_alloc_intrs - Allocate interrupts for the driver.
5074 *
5075 * Normal sequence is to try MSI-X; if not sucessful, try MSI;
5076 * if not successful, try Legacy.
5077 * ixgbe->intr_force can be used to force sequence to start with
5078 * any of the 3 types.
5079 * If MSI-X is not used, number of tx/rx rings is forced to 1.
5080 */
5081 static int
ixgbe_alloc_intrs(ixgbe_t * ixgbe)5082 ixgbe_alloc_intrs(ixgbe_t *ixgbe)
5083 {
5084 dev_info_t *devinfo;
5085 int intr_types;
5086 int rc;
5087
5088 devinfo = ixgbe->dip;
5089
5090 /*
5091 * Get supported interrupt types
5092 */
5093 rc = ddi_intr_get_supported_types(devinfo, &intr_types);
5094
5095 if (rc != DDI_SUCCESS) {
5096 ixgbe_log(ixgbe,
5097 "Get supported interrupt types failed: %d", rc);
5098 return (IXGBE_FAILURE);
5099 }
5100 IXGBE_DEBUGLOG_1(ixgbe, "Supported interrupt types: %x", intr_types);
5101
5102 ixgbe->intr_type = 0;
5103
5104 /*
5105 * Install MSI-X interrupts
5106 */
5107 if ((intr_types & DDI_INTR_TYPE_MSIX) &&
5108 (ixgbe->intr_force <= IXGBE_INTR_MSIX)) {
5109 rc = ixgbe_alloc_intr_handles(ixgbe, DDI_INTR_TYPE_MSIX);
5110 if (rc == IXGBE_SUCCESS)
5111 return (IXGBE_SUCCESS);
5112
5113 ixgbe_log(ixgbe,
5114 "Allocate MSI-X failed, trying MSI interrupts...");
5115 }
5116
5117 /*
5118 * MSI-X not used, force rings and groups to 1
5119 */
5120 ixgbe->num_rx_rings = 1;
5121 ixgbe->num_rx_groups = 1;
5122 ixgbe->num_tx_rings = 1;
5123 ixgbe->classify_mode = IXGBE_CLASSIFY_NONE;
5124 ixgbe_log(ixgbe,
5125 "MSI-X not used, force rings and groups number to 1");
5126
5127 /*
5128 * Install MSI interrupts
5129 */
5130 if ((intr_types & DDI_INTR_TYPE_MSI) &&
5131 (ixgbe->intr_force <= IXGBE_INTR_MSI)) {
5132 rc = ixgbe_alloc_intr_handles(ixgbe, DDI_INTR_TYPE_MSI);
5133 if (rc == IXGBE_SUCCESS)
5134 return (IXGBE_SUCCESS);
5135
5136 ixgbe_log(ixgbe,
5137 "Allocate MSI failed, trying Legacy interrupts...");
5138 }
5139
5140 /*
5141 * Install legacy interrupts
5142 */
5143 if (intr_types & DDI_INTR_TYPE_FIXED) {
5144 /*
5145 * Disallow legacy interrupts for X550. X550 has a silicon
5146 * bug which prevents Shared Legacy interrupts from working.
5147 * For details, please reference:
5148 *
5149 * Intel Ethernet Controller X550 Specification Update rev. 2.1
5150 * May 2016, erratum 22: PCIe Interrupt Status Bit
5151 */
5152 if (ixgbe->hw.mac.type == ixgbe_mac_X550 ||
5153 ixgbe->hw.mac.type == ixgbe_mac_X550EM_x ||
5154 ixgbe->hw.mac.type == ixgbe_mac_X550EM_a ||
5155 ixgbe->hw.mac.type == ixgbe_mac_X550_vf ||
5156 ixgbe->hw.mac.type == ixgbe_mac_X550EM_x_vf ||
5157 ixgbe->hw.mac.type == ixgbe_mac_X550EM_a_vf) {
5158 ixgbe_log(ixgbe,
5159 "Legacy interrupts are not supported on this "
5160 "adapter. Please use MSI or MSI-X instead.");
5161 return (IXGBE_FAILURE);
5162 }
5163 rc = ixgbe_alloc_intr_handles(ixgbe, DDI_INTR_TYPE_FIXED);
5164 if (rc == IXGBE_SUCCESS)
5165 return (IXGBE_SUCCESS);
5166
5167 ixgbe_log(ixgbe,
5168 "Allocate Legacy interrupts failed");
5169 }
5170
5171 /*
5172 * If none of the 3 types succeeded, return failure
5173 */
5174 return (IXGBE_FAILURE);
5175 }
5176
5177 /*
5178 * ixgbe_alloc_intr_handles - Allocate interrupt handles.
5179 *
5180 * For legacy and MSI, only 1 handle is needed. For MSI-X,
5181 * if fewer than 2 handles are available, return failure.
5182 * Upon success, this maps the vectors to rx and tx rings for
5183 * interrupts.
5184 */
5185 static int
ixgbe_alloc_intr_handles(ixgbe_t * ixgbe,int intr_type)5186 ixgbe_alloc_intr_handles(ixgbe_t *ixgbe, int intr_type)
5187 {
5188 dev_info_t *devinfo;
5189 int request, count, actual;
5190 int minimum;
5191 int rc;
5192 uint32_t ring_per_group;
5193
5194 devinfo = ixgbe->dip;
5195
5196 switch (intr_type) {
5197 case DDI_INTR_TYPE_FIXED:
5198 request = 1; /* Request 1 legacy interrupt handle */
5199 minimum = 1;
5200 IXGBE_DEBUGLOG_0(ixgbe, "interrupt type: legacy");
5201 break;
5202
5203 case DDI_INTR_TYPE_MSI:
5204 request = 1; /* Request 1 MSI interrupt handle */
5205 minimum = 1;
5206 IXGBE_DEBUGLOG_0(ixgbe, "interrupt type: MSI");
5207 break;
5208
5209 case DDI_INTR_TYPE_MSIX:
5210 /*
5211 * Best number of vectors for the adapter is
5212 * (# rx rings + # tx rings), however we will
5213 * limit the request number.
5214 */
5215 request = min(16, ixgbe->num_rx_rings + ixgbe->num_tx_rings);
5216 if (request > ixgbe->capab->max_ring_vect)
5217 request = ixgbe->capab->max_ring_vect;
5218 minimum = 1;
5219 IXGBE_DEBUGLOG_0(ixgbe, "interrupt type: MSI-X");
5220 break;
5221
5222 default:
5223 ixgbe_log(ixgbe,
5224 "invalid call to ixgbe_alloc_intr_handles(): %d\n",
5225 intr_type);
5226 return (IXGBE_FAILURE);
5227 }
5228 IXGBE_DEBUGLOG_2(ixgbe, "interrupt handles requested: %d minimum: %d",
5229 request, minimum);
5230
5231 /*
5232 * Get number of supported interrupts
5233 */
5234 rc = ddi_intr_get_nintrs(devinfo, intr_type, &count);
5235 if ((rc != DDI_SUCCESS) || (count < minimum)) {
5236 ixgbe_log(ixgbe,
5237 "Get interrupt number failed. Return: %d, count: %d",
5238 rc, count);
5239 return (IXGBE_FAILURE);
5240 }
5241 IXGBE_DEBUGLOG_1(ixgbe, "interrupts supported: %d", count);
5242
5243 actual = 0;
5244 ixgbe->intr_cnt = 0;
5245 ixgbe->intr_cnt_max = 0;
5246 ixgbe->intr_cnt_min = 0;
5247
5248 /*
5249 * Allocate an array of interrupt handles
5250 */
5251 ixgbe->intr_size = request * sizeof (ddi_intr_handle_t);
5252 ixgbe->htable = kmem_alloc(ixgbe->intr_size, KM_SLEEP);
5253
5254 rc = ddi_intr_alloc(devinfo, ixgbe->htable, intr_type, 0,
5255 request, &actual, DDI_INTR_ALLOC_NORMAL);
5256 if (rc != DDI_SUCCESS) {
5257 ixgbe_log(ixgbe, "Allocate interrupts failed. "
5258 "return: %d, request: %d, actual: %d",
5259 rc, request, actual);
5260 goto alloc_handle_fail;
5261 }
5262 IXGBE_DEBUGLOG_1(ixgbe, "interrupts actually allocated: %d", actual);
5263
5264 /*
5265 * upper/lower limit of interrupts
5266 */
5267 ixgbe->intr_cnt = actual;
5268 ixgbe->intr_cnt_max = request;
5269 ixgbe->intr_cnt_min = minimum;
5270
5271 /*
5272 * rss number per group should not exceed the rx interrupt number,
5273 * else need to adjust rx ring number.
5274 */
5275 ring_per_group = ixgbe->num_rx_rings / ixgbe->num_rx_groups;
5276 ASSERT((ixgbe->num_rx_rings % ixgbe->num_rx_groups) == 0);
5277 if (actual < ring_per_group) {
5278 ixgbe->num_rx_rings = ixgbe->num_rx_groups * actual;
5279 ixgbe_setup_vmdq_rss_conf(ixgbe);
5280 }
5281
5282 /*
5283 * Now we know the actual number of vectors. Here we map the vector
5284 * to other, rx rings and tx ring.
5285 */
5286 if (actual < minimum) {
5287 ixgbe_log(ixgbe, "Insufficient interrupt handles available: %d",
5288 actual);
5289 goto alloc_handle_fail;
5290 }
5291
5292 /*
5293 * Get priority for first vector, assume remaining are all the same
5294 */
5295 rc = ddi_intr_get_pri(ixgbe->htable[0], &ixgbe->intr_pri);
5296 if (rc != DDI_SUCCESS) {
5297 ixgbe_log(ixgbe,
5298 "Get interrupt priority failed: %d", rc);
5299 goto alloc_handle_fail;
5300 }
5301
5302 rc = ddi_intr_get_cap(ixgbe->htable[0], &ixgbe->intr_cap);
5303 if (rc != DDI_SUCCESS) {
5304 ixgbe_log(ixgbe,
5305 "Get interrupt cap failed: %d", rc);
5306 goto alloc_handle_fail;
5307 }
5308
5309 ixgbe->intr_type = intr_type;
5310
5311 return (IXGBE_SUCCESS);
5312
5313 alloc_handle_fail:
5314 ixgbe_rem_intrs(ixgbe);
5315
5316 return (IXGBE_FAILURE);
5317 }
5318
5319 /*
5320 * ixgbe_add_intr_handlers - Add interrupt handlers based on the interrupt type.
5321 *
5322 * Before adding the interrupt handlers, the interrupt vectors have
5323 * been allocated, and the rx/tx rings have also been allocated.
5324 */
5325 static int
ixgbe_add_intr_handlers(ixgbe_t * ixgbe)5326 ixgbe_add_intr_handlers(ixgbe_t *ixgbe)
5327 {
5328 int vector = 0;
5329 int rc;
5330
5331 switch (ixgbe->intr_type) {
5332 case DDI_INTR_TYPE_MSIX:
5333 /*
5334 * Add interrupt handler for all vectors
5335 */
5336 for (vector = 0; vector < ixgbe->intr_cnt; vector++) {
5337 /*
5338 * install pointer to vect_map[vector]
5339 */
5340 rc = ddi_intr_add_handler(ixgbe->htable[vector],
5341 (ddi_intr_handler_t *)ixgbe_intr_msix,
5342 (void *)&ixgbe->vect_map[vector], NULL);
5343
5344 if (rc != DDI_SUCCESS) {
5345 ixgbe_log(ixgbe,
5346 "Add interrupt handler failed. "
5347 "return: %d, vector: %d", rc, vector);
5348 for (vector--; vector >= 0; vector--) {
5349 (void) ddi_intr_remove_handler(
5350 ixgbe->htable[vector]);
5351 }
5352 return (IXGBE_FAILURE);
5353 }
5354 }
5355
5356 break;
5357
5358 case DDI_INTR_TYPE_MSI:
5359 /*
5360 * Add interrupt handlers for the only vector
5361 */
5362 rc = ddi_intr_add_handler(ixgbe->htable[vector],
5363 (ddi_intr_handler_t *)ixgbe_intr_msi,
5364 (void *)ixgbe, NULL);
5365
5366 if (rc != DDI_SUCCESS) {
5367 ixgbe_log(ixgbe,
5368 "Add MSI interrupt handler failed: %d", rc);
5369 return (IXGBE_FAILURE);
5370 }
5371
5372 break;
5373
5374 case DDI_INTR_TYPE_FIXED:
5375 /*
5376 * Add interrupt handlers for the only vector
5377 */
5378 rc = ddi_intr_add_handler(ixgbe->htable[vector],
5379 (ddi_intr_handler_t *)ixgbe_intr_legacy,
5380 (void *)ixgbe, NULL);
5381
5382 if (rc != DDI_SUCCESS) {
5383 ixgbe_log(ixgbe,
5384 "Add legacy interrupt handler failed: %d", rc);
5385 return (IXGBE_FAILURE);
5386 }
5387
5388 break;
5389
5390 default:
5391 return (IXGBE_FAILURE);
5392 }
5393
5394 return (IXGBE_SUCCESS);
5395 }
5396
5397 #pragma inline(ixgbe_map_rxring_to_vector)
5398 /*
5399 * ixgbe_map_rxring_to_vector - Map given rx ring to given interrupt vector.
5400 */
5401 static void
ixgbe_map_rxring_to_vector(ixgbe_t * ixgbe,int r_idx,int v_idx)5402 ixgbe_map_rxring_to_vector(ixgbe_t *ixgbe, int r_idx, int v_idx)
5403 {
5404 /*
5405 * Set bit in map
5406 */
5407 BT_SET(ixgbe->vect_map[v_idx].rx_map, r_idx);
5408
5409 /*
5410 * Count bits set
5411 */
5412 ixgbe->vect_map[v_idx].rxr_cnt++;
5413
5414 /*
5415 * Remember bit position
5416 */
5417 ixgbe->rx_rings[r_idx].intr_vector = v_idx;
5418 ixgbe->rx_rings[r_idx].vect_bit = 1 << v_idx;
5419 }
5420
5421 #pragma inline(ixgbe_map_txring_to_vector)
5422 /*
5423 * ixgbe_map_txring_to_vector - Map given tx ring to given interrupt vector.
5424 */
5425 static void
ixgbe_map_txring_to_vector(ixgbe_t * ixgbe,int t_idx,int v_idx)5426 ixgbe_map_txring_to_vector(ixgbe_t *ixgbe, int t_idx, int v_idx)
5427 {
5428 /*
5429 * Set bit in map
5430 */
5431 BT_SET(ixgbe->vect_map[v_idx].tx_map, t_idx);
5432
5433 /*
5434 * Count bits set
5435 */
5436 ixgbe->vect_map[v_idx].txr_cnt++;
5437
5438 /*
5439 * Remember bit position
5440 */
5441 ixgbe->tx_rings[t_idx].intr_vector = v_idx;
5442 ixgbe->tx_rings[t_idx].vect_bit = 1 << v_idx;
5443 }
5444
5445 /*
5446 * ixgbe_setup_ivar - Set the given entry in the given interrupt vector
5447 * allocation register (IVAR).
5448 * cause:
5449 * -1 : other cause
5450 * 0 : rx
5451 * 1 : tx
5452 */
5453 static void
ixgbe_setup_ivar(ixgbe_t * ixgbe,uint16_t intr_alloc_entry,uint8_t msix_vector,int8_t cause)5454 ixgbe_setup_ivar(ixgbe_t *ixgbe, uint16_t intr_alloc_entry, uint8_t msix_vector,
5455 int8_t cause)
5456 {
5457 struct ixgbe_hw *hw = &ixgbe->hw;
5458 u32 ivar, index;
5459
5460 switch (hw->mac.type) {
5461 case ixgbe_mac_82598EB:
5462 msix_vector |= IXGBE_IVAR_ALLOC_VAL;
5463 if (cause == -1) {
5464 cause = 0;
5465 }
5466 index = (((cause * 64) + intr_alloc_entry) >> 2) & 0x1F;
5467 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(index));
5468 ivar &= ~(0xFF << (8 * (intr_alloc_entry & 0x3)));
5469 ivar |= (msix_vector << (8 * (intr_alloc_entry & 0x3)));
5470 IXGBE_WRITE_REG(hw, IXGBE_IVAR(index), ivar);
5471 break;
5472
5473 case ixgbe_mac_82599EB:
5474 case ixgbe_mac_X540:
5475 case ixgbe_mac_X550:
5476 case ixgbe_mac_X550EM_x:
5477 case ixgbe_mac_X550EM_a:
5478 if (cause == -1) {
5479 /* other causes */
5480 msix_vector |= IXGBE_IVAR_ALLOC_VAL;
5481 index = (intr_alloc_entry & 1) * 8;
5482 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR_MISC);
5483 ivar &= ~(0xFF << index);
5484 ivar |= (msix_vector << index);
5485 IXGBE_WRITE_REG(hw, IXGBE_IVAR_MISC, ivar);
5486 } else {
5487 /* tx or rx causes */
5488 msix_vector |= IXGBE_IVAR_ALLOC_VAL;
5489 index = ((16 * (intr_alloc_entry & 1)) + (8 * cause));
5490 ivar = IXGBE_READ_REG(hw,
5491 IXGBE_IVAR(intr_alloc_entry >> 1));
5492 ivar &= ~(0xFF << index);
5493 ivar |= (msix_vector << index);
5494 IXGBE_WRITE_REG(hw, IXGBE_IVAR(intr_alloc_entry >> 1),
5495 ivar);
5496 }
5497 break;
5498
5499 default:
5500 break;
5501 }
5502 }
5503
5504 /*
5505 * ixgbe_enable_ivar - Enable the given entry by setting the VAL bit of
5506 * given interrupt vector allocation register (IVAR).
5507 * cause:
5508 * -1 : other cause
5509 * 0 : rx
5510 * 1 : tx
5511 */
5512 static void
ixgbe_enable_ivar(ixgbe_t * ixgbe,uint16_t intr_alloc_entry,int8_t cause)5513 ixgbe_enable_ivar(ixgbe_t *ixgbe, uint16_t intr_alloc_entry, int8_t cause)
5514 {
5515 struct ixgbe_hw *hw = &ixgbe->hw;
5516 u32 ivar, index;
5517
5518 switch (hw->mac.type) {
5519 case ixgbe_mac_82598EB:
5520 if (cause == -1) {
5521 cause = 0;
5522 }
5523 index = (((cause * 64) + intr_alloc_entry) >> 2) & 0x1F;
5524 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(index));
5525 ivar |= (IXGBE_IVAR_ALLOC_VAL << (8 *
5526 (intr_alloc_entry & 0x3)));
5527 IXGBE_WRITE_REG(hw, IXGBE_IVAR(index), ivar);
5528 break;
5529
5530 case ixgbe_mac_82599EB:
5531 case ixgbe_mac_X540:
5532 case ixgbe_mac_X550:
5533 case ixgbe_mac_X550EM_x:
5534 case ixgbe_mac_X550EM_a:
5535 if (cause == -1) {
5536 /* other causes */
5537 index = (intr_alloc_entry & 1) * 8;
5538 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR_MISC);
5539 ivar |= (IXGBE_IVAR_ALLOC_VAL << index);
5540 IXGBE_WRITE_REG(hw, IXGBE_IVAR_MISC, ivar);
5541 } else {
5542 /* tx or rx causes */
5543 index = ((16 * (intr_alloc_entry & 1)) + (8 * cause));
5544 ivar = IXGBE_READ_REG(hw,
5545 IXGBE_IVAR(intr_alloc_entry >> 1));
5546 ivar |= (IXGBE_IVAR_ALLOC_VAL << index);
5547 IXGBE_WRITE_REG(hw, IXGBE_IVAR(intr_alloc_entry >> 1),
5548 ivar);
5549 }
5550 break;
5551
5552 default:
5553 break;
5554 }
5555 }
5556
5557 /*
5558 * ixgbe_disable_ivar - Disble the given entry by clearing the VAL bit of
5559 * given interrupt vector allocation register (IVAR).
5560 * cause:
5561 * -1 : other cause
5562 * 0 : rx
5563 * 1 : tx
5564 */
5565 static void
ixgbe_disable_ivar(ixgbe_t * ixgbe,uint16_t intr_alloc_entry,int8_t cause)5566 ixgbe_disable_ivar(ixgbe_t *ixgbe, uint16_t intr_alloc_entry, int8_t cause)
5567 {
5568 struct ixgbe_hw *hw = &ixgbe->hw;
5569 u32 ivar, index;
5570
5571 switch (hw->mac.type) {
5572 case ixgbe_mac_82598EB:
5573 if (cause == -1) {
5574 cause = 0;
5575 }
5576 index = (((cause * 64) + intr_alloc_entry) >> 2) & 0x1F;
5577 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(index));
5578 ivar &= ~(IXGBE_IVAR_ALLOC_VAL<< (8 *
5579 (intr_alloc_entry & 0x3)));
5580 IXGBE_WRITE_REG(hw, IXGBE_IVAR(index), ivar);
5581 break;
5582
5583 case ixgbe_mac_82599EB:
5584 case ixgbe_mac_X540:
5585 case ixgbe_mac_X550:
5586 case ixgbe_mac_X550EM_x:
5587 case ixgbe_mac_X550EM_a:
5588 if (cause == -1) {
5589 /* other causes */
5590 index = (intr_alloc_entry & 1) * 8;
5591 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR_MISC);
5592 ivar &= ~(IXGBE_IVAR_ALLOC_VAL << index);
5593 IXGBE_WRITE_REG(hw, IXGBE_IVAR_MISC, ivar);
5594 } else {
5595 /* tx or rx causes */
5596 index = ((16 * (intr_alloc_entry & 1)) + (8 * cause));
5597 ivar = IXGBE_READ_REG(hw,
5598 IXGBE_IVAR(intr_alloc_entry >> 1));
5599 ivar &= ~(IXGBE_IVAR_ALLOC_VAL << index);
5600 IXGBE_WRITE_REG(hw, IXGBE_IVAR(intr_alloc_entry >> 1),
5601 ivar);
5602 }
5603 break;
5604
5605 default:
5606 break;
5607 }
5608 }
5609
5610 /*
5611 * Convert the rx ring index driver maintained to the rx ring index
5612 * in h/w.
5613 */
5614 static uint32_t
ixgbe_get_hw_rx_index(ixgbe_t * ixgbe,uint32_t sw_rx_index)5615 ixgbe_get_hw_rx_index(ixgbe_t *ixgbe, uint32_t sw_rx_index)
5616 {
5617
5618 struct ixgbe_hw *hw = &ixgbe->hw;
5619 uint32_t rx_ring_per_group, hw_rx_index;
5620
5621 if (ixgbe->classify_mode == IXGBE_CLASSIFY_RSS ||
5622 ixgbe->classify_mode == IXGBE_CLASSIFY_NONE) {
5623 return (sw_rx_index);
5624 } else if (ixgbe->classify_mode == IXGBE_CLASSIFY_VMDQ) {
5625 switch (hw->mac.type) {
5626 case ixgbe_mac_82598EB:
5627 return (sw_rx_index);
5628
5629 case ixgbe_mac_82599EB:
5630 case ixgbe_mac_X540:
5631 case ixgbe_mac_X550:
5632 case ixgbe_mac_X550EM_x:
5633 case ixgbe_mac_X550EM_a:
5634 return (sw_rx_index * 2);
5635
5636 default:
5637 break;
5638 }
5639 } else if (ixgbe->classify_mode == IXGBE_CLASSIFY_VMDQ_RSS) {
5640 rx_ring_per_group = ixgbe->num_rx_rings / ixgbe->num_rx_groups;
5641
5642 switch (hw->mac.type) {
5643 case ixgbe_mac_82598EB:
5644 hw_rx_index = (sw_rx_index / rx_ring_per_group) *
5645 16 + (sw_rx_index % rx_ring_per_group);
5646 return (hw_rx_index);
5647
5648 case ixgbe_mac_82599EB:
5649 case ixgbe_mac_X540:
5650 case ixgbe_mac_X550:
5651 case ixgbe_mac_X550EM_x:
5652 case ixgbe_mac_X550EM_a:
5653 if (ixgbe->num_rx_groups > 32) {
5654 hw_rx_index = (sw_rx_index /
5655 rx_ring_per_group) * 2 +
5656 (sw_rx_index % rx_ring_per_group);
5657 } else {
5658 hw_rx_index = (sw_rx_index /
5659 rx_ring_per_group) * 4 +
5660 (sw_rx_index % rx_ring_per_group);
5661 }
5662 return (hw_rx_index);
5663
5664 default:
5665 break;
5666 }
5667 }
5668
5669 /*
5670 * Should never reach. Just to make compiler happy.
5671 */
5672 return (sw_rx_index);
5673 }
5674
5675 /*
5676 * ixgbe_map_intrs_to_vectors - Map different interrupts to MSI-X vectors.
5677 *
5678 * For MSI-X, here will map rx interrupt, tx interrupt and other interrupt
5679 * to vector[0 - (intr_cnt -1)].
5680 */
5681 static int
ixgbe_map_intrs_to_vectors(ixgbe_t * ixgbe)5682 ixgbe_map_intrs_to_vectors(ixgbe_t *ixgbe)
5683 {
5684 int i, vector = 0;
5685
5686 /* initialize vector map */
5687 bzero(&ixgbe->vect_map, sizeof (ixgbe->vect_map));
5688 for (i = 0; i < ixgbe->intr_cnt; i++) {
5689 ixgbe->vect_map[i].ixgbe = ixgbe;
5690 }
5691
5692 /*
5693 * non-MSI-X case is very simple: rx rings[0] on RTxQ[0],
5694 * tx rings[0] on RTxQ[1].
5695 */
5696 if (ixgbe->intr_type != DDI_INTR_TYPE_MSIX) {
5697 ixgbe_map_rxring_to_vector(ixgbe, 0, 0);
5698 ixgbe_map_txring_to_vector(ixgbe, 0, 1);
5699 return (IXGBE_SUCCESS);
5700 }
5701
5702 /*
5703 * Interrupts/vectors mapping for MSI-X
5704 */
5705
5706 /*
5707 * Map other interrupt to vector 0,
5708 * Set bit in map and count the bits set.
5709 */
5710 BT_SET(ixgbe->vect_map[vector].other_map, 0);
5711 ixgbe->vect_map[vector].other_cnt++;
5712
5713 /*
5714 * Map rx ring interrupts to vectors
5715 */
5716 for (i = 0; i < ixgbe->num_rx_rings; i++) {
5717 ixgbe_map_rxring_to_vector(ixgbe, i, vector);
5718 vector = (vector +1) % ixgbe->intr_cnt;
5719 }
5720
5721 /*
5722 * Map tx ring interrupts to vectors
5723 */
5724 for (i = 0; i < ixgbe->num_tx_rings; i++) {
5725 ixgbe_map_txring_to_vector(ixgbe, i, vector);
5726 vector = (vector +1) % ixgbe->intr_cnt;
5727 }
5728
5729 return (IXGBE_SUCCESS);
5730 }
5731
5732 /*
5733 * ixgbe_setup_adapter_vector - Setup the adapter interrupt vector(s).
5734 *
5735 * This relies on ring/vector mapping already set up in the
5736 * vect_map[] structures
5737 */
5738 static void
ixgbe_setup_adapter_vector(ixgbe_t * ixgbe)5739 ixgbe_setup_adapter_vector(ixgbe_t *ixgbe)
5740 {
5741 struct ixgbe_hw *hw = &ixgbe->hw;
5742 ixgbe_intr_vector_t *vect; /* vector bitmap */
5743 int r_idx; /* ring index */
5744 int v_idx; /* vector index */
5745 uint32_t hw_index;
5746
5747 /*
5748 * Clear any previous entries
5749 */
5750 switch (hw->mac.type) {
5751 case ixgbe_mac_82598EB:
5752 for (v_idx = 0; v_idx < 25; v_idx++)
5753 IXGBE_WRITE_REG(hw, IXGBE_IVAR(v_idx), 0);
5754 break;
5755
5756 case ixgbe_mac_82599EB:
5757 case ixgbe_mac_X540:
5758 case ixgbe_mac_X550:
5759 case ixgbe_mac_X550EM_x:
5760 case ixgbe_mac_X550EM_a:
5761 for (v_idx = 0; v_idx < 64; v_idx++)
5762 IXGBE_WRITE_REG(hw, IXGBE_IVAR(v_idx), 0);
5763 IXGBE_WRITE_REG(hw, IXGBE_IVAR_MISC, 0);
5764 break;
5765
5766 default:
5767 break;
5768 }
5769
5770 /*
5771 * For non MSI-X interrupt, rx rings[0] will use RTxQ[0], and
5772 * tx rings[0] will use RTxQ[1].
5773 */
5774 if (ixgbe->intr_type != DDI_INTR_TYPE_MSIX) {
5775 ixgbe_setup_ivar(ixgbe, 0, 0, 0);
5776 ixgbe_setup_ivar(ixgbe, 0, 1, 1);
5777 return;
5778 }
5779
5780 /*
5781 * For MSI-X interrupt, "Other" is always on vector[0].
5782 */
5783 ixgbe_setup_ivar(ixgbe, IXGBE_IVAR_OTHER_CAUSES_INDEX, 0, -1);
5784
5785 /*
5786 * For each interrupt vector, populate the IVAR table
5787 */
5788 for (v_idx = 0; v_idx < ixgbe->intr_cnt; v_idx++) {
5789 vect = &ixgbe->vect_map[v_idx];
5790
5791 /*
5792 * For each rx ring bit set
5793 */
5794 r_idx = bt_getlowbit(vect->rx_map, 0,
5795 (ixgbe->num_rx_rings - 1));
5796
5797 while (r_idx >= 0) {
5798 hw_index = ixgbe->rx_rings[r_idx].hw_index;
5799 ixgbe_setup_ivar(ixgbe, hw_index, v_idx, 0);
5800 r_idx = bt_getlowbit(vect->rx_map, (r_idx + 1),
5801 (ixgbe->num_rx_rings - 1));
5802 }
5803
5804 /*
5805 * For each tx ring bit set
5806 */
5807 r_idx = bt_getlowbit(vect->tx_map, 0,
5808 (ixgbe->num_tx_rings - 1));
5809
5810 while (r_idx >= 0) {
5811 ixgbe_setup_ivar(ixgbe, r_idx, v_idx, 1);
5812 r_idx = bt_getlowbit(vect->tx_map, (r_idx + 1),
5813 (ixgbe->num_tx_rings - 1));
5814 }
5815 }
5816 }
5817
5818 /*
5819 * ixgbe_rem_intr_handlers - Remove the interrupt handlers.
5820 */
5821 static void
ixgbe_rem_intr_handlers(ixgbe_t * ixgbe)5822 ixgbe_rem_intr_handlers(ixgbe_t *ixgbe)
5823 {
5824 int i;
5825 int rc;
5826
5827 for (i = 0; i < ixgbe->intr_cnt; i++) {
5828 rc = ddi_intr_remove_handler(ixgbe->htable[i]);
5829 if (rc != DDI_SUCCESS) {
5830 IXGBE_DEBUGLOG_1(ixgbe,
5831 "Remove intr handler failed: %d", rc);
5832 }
5833 }
5834 }
5835
5836 /*
5837 * ixgbe_rem_intrs - Remove the allocated interrupts.
5838 */
5839 static void
ixgbe_rem_intrs(ixgbe_t * ixgbe)5840 ixgbe_rem_intrs(ixgbe_t *ixgbe)
5841 {
5842 int i;
5843 int rc;
5844
5845 for (i = 0; i < ixgbe->intr_cnt; i++) {
5846 rc = ddi_intr_free(ixgbe->htable[i]);
5847 if (rc != DDI_SUCCESS) {
5848 IXGBE_DEBUGLOG_1(ixgbe,
5849 "Free intr failed: %d", rc);
5850 }
5851 }
5852
5853 kmem_free(ixgbe->htable, ixgbe->intr_size);
5854 ixgbe->htable = NULL;
5855 }
5856
5857 /*
5858 * ixgbe_enable_intrs - Enable all the ddi interrupts.
5859 */
5860 static int
ixgbe_enable_intrs(ixgbe_t * ixgbe)5861 ixgbe_enable_intrs(ixgbe_t *ixgbe)
5862 {
5863 int i;
5864 int rc;
5865
5866 /*
5867 * Enable interrupts
5868 */
5869 if (ixgbe->intr_cap & DDI_INTR_FLAG_BLOCK) {
5870 /*
5871 * Call ddi_intr_block_enable() for MSI
5872 */
5873 rc = ddi_intr_block_enable(ixgbe->htable, ixgbe->intr_cnt);
5874 if (rc != DDI_SUCCESS) {
5875 ixgbe_log(ixgbe,
5876 "Enable block intr failed: %d", rc);
5877 return (IXGBE_FAILURE);
5878 }
5879 } else {
5880 /*
5881 * Call ddi_intr_enable() for Legacy/MSI non block enable
5882 */
5883 for (i = 0; i < ixgbe->intr_cnt; i++) {
5884 rc = ddi_intr_enable(ixgbe->htable[i]);
5885 if (rc != DDI_SUCCESS) {
5886 ixgbe_log(ixgbe,
5887 "Enable intr failed: %d", rc);
5888 return (IXGBE_FAILURE);
5889 }
5890 }
5891 }
5892
5893 return (IXGBE_SUCCESS);
5894 }
5895
5896 /*
5897 * ixgbe_disable_intrs - Disable all the interrupts.
5898 */
5899 static int
ixgbe_disable_intrs(ixgbe_t * ixgbe)5900 ixgbe_disable_intrs(ixgbe_t *ixgbe)
5901 {
5902 int i;
5903 int rc;
5904
5905 /*
5906 * Disable all interrupts
5907 */
5908 if (ixgbe->intr_cap & DDI_INTR_FLAG_BLOCK) {
5909 rc = ddi_intr_block_disable(ixgbe->htable, ixgbe->intr_cnt);
5910 if (rc != DDI_SUCCESS) {
5911 ixgbe_log(ixgbe,
5912 "Disable block intr failed: %d", rc);
5913 return (IXGBE_FAILURE);
5914 }
5915 } else {
5916 for (i = 0; i < ixgbe->intr_cnt; i++) {
5917 rc = ddi_intr_disable(ixgbe->htable[i]);
5918 if (rc != DDI_SUCCESS) {
5919 ixgbe_log(ixgbe,
5920 "Disable intr failed: %d", rc);
5921 return (IXGBE_FAILURE);
5922 }
5923 }
5924 }
5925
5926 return (IXGBE_SUCCESS);
5927 }
5928
5929 /*
5930 * ixgbe_get_hw_state - Get and save parameters related to adapter hardware.
5931 */
5932 static void
ixgbe_get_hw_state(ixgbe_t * ixgbe)5933 ixgbe_get_hw_state(ixgbe_t *ixgbe)
5934 {
5935 struct ixgbe_hw *hw = &ixgbe->hw;
5936 ixgbe_link_speed speed = 0;
5937 boolean_t link_up = B_FALSE;
5938 uint32_t pcs1g_anlp = 0;
5939
5940 ASSERT(mutex_owned(&ixgbe->gen_lock));
5941 ixgbe->param_lp_1000fdx_cap = 0;
5942 ixgbe->param_lp_100fdx_cap = 0;
5943
5944 /* check for link, don't wait */
5945 (void) ixgbe_check_link(hw, &speed, &link_up, B_FALSE);
5946 ixgbe->phys_supported = ixgbe_get_supported_physical_layer(hw);
5947
5948 /*
5949 * Update the observed Link Partner's capabilities. Not all adapters
5950 * can provide full information on the LP's capable speeds, so we
5951 * provide what we can.
5952 */
5953 if (link_up) {
5954 pcs1g_anlp = IXGBE_READ_REG(hw, IXGBE_PCS1GANLP);
5955
5956 ixgbe->param_lp_1000fdx_cap =
5957 (pcs1g_anlp & IXGBE_PCS1GANLP_LPFD) ? 1 : 0;
5958 ixgbe->param_lp_100fdx_cap =
5959 (pcs1g_anlp & IXGBE_PCS1GANLP_LPFD) ? 1 : 0;
5960 }
5961
5962 /*
5963 * Update GLD's notion of the adapter's currently advertised speeds.
5964 * Since the common code doesn't always record the current autonegotiate
5965 * settings in the phy struct for all parts (specifically, adapters with
5966 * SFPs) we first test to see if it is 0, and if so, we fall back to
5967 * using the adapter's speed capabilities which we saved during instance
5968 * init in ixgbe_init_params().
5969 *
5970 * Adapters with SFPs will always be shown as advertising all of their
5971 * supported speeds, and adapters with baseT PHYs (where the phy struct
5972 * is maintained by the common code) will always have a factual view of
5973 * their currently-advertised speeds. In the case of SFPs, this is
5974 * acceptable as we default to advertising all speeds that the adapter
5975 * claims to support, and those properties are immutable; unlike on
5976 * baseT (copper) PHYs, where speeds can be enabled or disabled at will.
5977 */
5978 speed = hw->phy.autoneg_advertised;
5979 if (speed == 0)
5980 speed = ixgbe->speeds_supported;
5981
5982 ixgbe->param_adv_10000fdx_cap =
5983 (speed & IXGBE_LINK_SPEED_10GB_FULL) ? 1 : 0;
5984 ixgbe->param_adv_5000fdx_cap =
5985 (speed & IXGBE_LINK_SPEED_5GB_FULL) ? 1 : 0;
5986 ixgbe->param_adv_2500fdx_cap =
5987 (speed & IXGBE_LINK_SPEED_2_5GB_FULL) ? 1 : 0;
5988 ixgbe->param_adv_1000fdx_cap =
5989 (speed & IXGBE_LINK_SPEED_1GB_FULL) ? 1 : 0;
5990 ixgbe->param_adv_100fdx_cap =
5991 (speed & IXGBE_LINK_SPEED_100_FULL) ? 1 : 0;
5992 }
5993
5994 /*
5995 * ixgbe_get_driver_control - Notify that driver is in control of device.
5996 */
5997 static void
ixgbe_get_driver_control(struct ixgbe_hw * hw)5998 ixgbe_get_driver_control(struct ixgbe_hw *hw)
5999 {
6000 uint32_t ctrl_ext;
6001
6002 /*
6003 * Notify firmware that driver is in control of device
6004 */
6005 ctrl_ext = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT);
6006 ctrl_ext |= IXGBE_CTRL_EXT_DRV_LOAD;
6007 IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl_ext);
6008 }
6009
6010 /*
6011 * ixgbe_release_driver_control - Notify that driver is no longer in control
6012 * of device.
6013 */
6014 static void
ixgbe_release_driver_control(struct ixgbe_hw * hw)6015 ixgbe_release_driver_control(struct ixgbe_hw *hw)
6016 {
6017 uint32_t ctrl_ext;
6018
6019 /*
6020 * Notify firmware that driver is no longer in control of device
6021 */
6022 ctrl_ext = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT);
6023 ctrl_ext &= ~IXGBE_CTRL_EXT_DRV_LOAD;
6024 IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl_ext);
6025 }
6026
6027 /*
6028 * ixgbe_atomic_reserve - Atomic decrease operation.
6029 */
6030 int
ixgbe_atomic_reserve(uint32_t * count_p,uint32_t n)6031 ixgbe_atomic_reserve(uint32_t *count_p, uint32_t n)
6032 {
6033 uint32_t oldval;
6034 uint32_t newval;
6035
6036 /*
6037 * ATOMICALLY
6038 */
6039 do {
6040 oldval = *count_p;
6041 if (oldval < n)
6042 return (-1);
6043 newval = oldval - n;
6044 } while (atomic_cas_32(count_p, oldval, newval) != oldval);
6045
6046 return (newval);
6047 }
6048
6049 /*
6050 * ixgbe_mc_table_itr - Traverse the entries in the multicast table.
6051 */
6052 static uint8_t *
ixgbe_mc_table_itr(struct ixgbe_hw * hw,uint8_t ** upd_ptr,uint32_t * vmdq)6053 ixgbe_mc_table_itr(struct ixgbe_hw *hw, uint8_t **upd_ptr, uint32_t *vmdq)
6054 {
6055 uint8_t *addr = *upd_ptr;
6056 uint8_t *new_ptr;
6057
6058 _NOTE(ARGUNUSED(hw));
6059 _NOTE(ARGUNUSED(vmdq));
6060
6061 new_ptr = addr + IXGBE_ETH_LENGTH_OF_ADDRESS;
6062 *upd_ptr = new_ptr;
6063 return (addr);
6064 }
6065
6066 /*
6067 * FMA support
6068 */
6069 int
ixgbe_check_acc_handle(ddi_acc_handle_t handle)6070 ixgbe_check_acc_handle(ddi_acc_handle_t handle)
6071 {
6072 ddi_fm_error_t de;
6073
6074 ddi_fm_acc_err_get(handle, &de, DDI_FME_VERSION);
6075 ddi_fm_acc_err_clear(handle, DDI_FME_VERSION);
6076 return (de.fme_status);
6077 }
6078
6079 int
ixgbe_check_dma_handle(ddi_dma_handle_t handle)6080 ixgbe_check_dma_handle(ddi_dma_handle_t handle)
6081 {
6082 ddi_fm_error_t de;
6083
6084 ddi_fm_dma_err_get(handle, &de, DDI_FME_VERSION);
6085 return (de.fme_status);
6086 }
6087
6088 /*
6089 * ixgbe_fm_error_cb - The IO fault service error handling callback function.
6090 */
6091 static int
ixgbe_fm_error_cb(dev_info_t * dip,ddi_fm_error_t * err,const void * impl_data)6092 ixgbe_fm_error_cb(dev_info_t *dip, ddi_fm_error_t *err, const void *impl_data)
6093 {
6094 _NOTE(ARGUNUSED(impl_data));
6095 /*
6096 * as the driver can always deal with an error in any dma or
6097 * access handle, we can just return the fme_status value.
6098 */
6099 pci_ereport_post(dip, err, NULL);
6100 return (err->fme_status);
6101 }
6102
6103 static void
ixgbe_fm_init(ixgbe_t * ixgbe)6104 ixgbe_fm_init(ixgbe_t *ixgbe)
6105 {
6106 ddi_iblock_cookie_t iblk;
6107 int fma_dma_flag;
6108
6109 /*
6110 * Only register with IO Fault Services if we have some capability
6111 */
6112 if (ixgbe->fm_capabilities & DDI_FM_ACCCHK_CAPABLE) {
6113 ixgbe_regs_acc_attr.devacc_attr_access = DDI_FLAGERR_ACC;
6114 } else {
6115 ixgbe_regs_acc_attr.devacc_attr_access = DDI_DEFAULT_ACC;
6116 }
6117
6118 if (ixgbe->fm_capabilities & DDI_FM_DMACHK_CAPABLE) {
6119 fma_dma_flag = 1;
6120 } else {
6121 fma_dma_flag = 0;
6122 }
6123
6124 ixgbe_set_fma_flags(fma_dma_flag);
6125
6126 if (ixgbe->fm_capabilities) {
6127
6128 /*
6129 * Register capabilities with IO Fault Services
6130 */
6131 ddi_fm_init(ixgbe->dip, &ixgbe->fm_capabilities, &iblk);
6132
6133 /*
6134 * Initialize pci ereport capabilities if ereport capable
6135 */
6136 if (DDI_FM_EREPORT_CAP(ixgbe->fm_capabilities) ||
6137 DDI_FM_ERRCB_CAP(ixgbe->fm_capabilities))
6138 pci_ereport_setup(ixgbe->dip);
6139
6140 /*
6141 * Register error callback if error callback capable
6142 */
6143 if (DDI_FM_ERRCB_CAP(ixgbe->fm_capabilities))
6144 ddi_fm_handler_register(ixgbe->dip,
6145 ixgbe_fm_error_cb, (void*) ixgbe);
6146 }
6147 }
6148
6149 static void
ixgbe_fm_fini(ixgbe_t * ixgbe)6150 ixgbe_fm_fini(ixgbe_t *ixgbe)
6151 {
6152 /*
6153 * Only unregister FMA capabilities if they are registered
6154 */
6155 if (ixgbe->fm_capabilities) {
6156
6157 /*
6158 * Release any resources allocated by pci_ereport_setup()
6159 */
6160 if (DDI_FM_EREPORT_CAP(ixgbe->fm_capabilities) ||
6161 DDI_FM_ERRCB_CAP(ixgbe->fm_capabilities))
6162 pci_ereport_teardown(ixgbe->dip);
6163
6164 /*
6165 * Un-register error callback if error callback capable
6166 */
6167 if (DDI_FM_ERRCB_CAP(ixgbe->fm_capabilities))
6168 ddi_fm_handler_unregister(ixgbe->dip);
6169
6170 /*
6171 * Unregister from IO Fault Service
6172 */
6173 ddi_fm_fini(ixgbe->dip);
6174 }
6175 }
6176
6177 void
ixgbe_fm_ereport(ixgbe_t * ixgbe,char * detail)6178 ixgbe_fm_ereport(ixgbe_t *ixgbe, char *detail)
6179 {
6180 uint64_t ena;
6181 char buf[FM_MAX_CLASS];
6182
6183 (void) snprintf(buf, FM_MAX_CLASS, "%s.%s", DDI_FM_DEVICE, detail);
6184 ena = fm_ena_generate(0, FM_ENA_FMT1);
6185 if (DDI_FM_EREPORT_CAP(ixgbe->fm_capabilities)) {
6186 ddi_fm_ereport_post(ixgbe->dip, buf, ena, DDI_NOSLEEP,
6187 FM_VERSION, DATA_TYPE_UINT8, FM_EREPORT_VERS0, NULL);
6188 }
6189 }
6190
6191 static int
ixgbe_ring_start(mac_ring_driver_t rh,uint64_t mr_gen_num)6192 ixgbe_ring_start(mac_ring_driver_t rh, uint64_t mr_gen_num)
6193 {
6194 ixgbe_rx_ring_t *rx_ring = (ixgbe_rx_ring_t *)rh;
6195
6196 mutex_enter(&rx_ring->rx_lock);
6197 rx_ring->ring_gen_num = mr_gen_num;
6198 mutex_exit(&rx_ring->rx_lock);
6199 return (0);
6200 }
6201
6202 /*
6203 * Get the global ring index by a ring index within a group.
6204 */
6205 static int
ixgbe_get_rx_ring_index(ixgbe_t * ixgbe,int gindex,int rindex)6206 ixgbe_get_rx_ring_index(ixgbe_t *ixgbe, int gindex, int rindex)
6207 {
6208 ixgbe_rx_ring_t *rx_ring;
6209 int i;
6210
6211 for (i = 0; i < ixgbe->num_rx_rings; i++) {
6212 rx_ring = &ixgbe->rx_rings[i];
6213 if (rx_ring->group_index == gindex)
6214 rindex--;
6215 if (rindex < 0)
6216 return (i);
6217 }
6218
6219 return (-1);
6220 }
6221
6222 /*
6223 * Callback funtion for MAC layer to register all rings.
6224 */
6225 /* ARGSUSED */
6226 void
ixgbe_fill_ring(void * arg,mac_ring_type_t rtype,const int group_index,const int ring_index,mac_ring_info_t * infop,mac_ring_handle_t rh)6227 ixgbe_fill_ring(void *arg, mac_ring_type_t rtype, const int group_index,
6228 const int ring_index, mac_ring_info_t *infop, mac_ring_handle_t rh)
6229 {
6230 ixgbe_t *ixgbe = (ixgbe_t *)arg;
6231 mac_intr_t *mintr = &infop->mri_intr;
6232
6233 switch (rtype) {
6234 case MAC_RING_TYPE_RX: {
6235 /*
6236 * 'index' is the ring index within the group.
6237 * Need to get the global ring index by searching in groups.
6238 */
6239 int global_ring_index = ixgbe_get_rx_ring_index(
6240 ixgbe, group_index, ring_index);
6241
6242 ASSERT(global_ring_index >= 0);
6243
6244 ixgbe_rx_ring_t *rx_ring = &ixgbe->rx_rings[global_ring_index];
6245 rx_ring->ring_handle = rh;
6246
6247 infop->mri_driver = (mac_ring_driver_t)rx_ring;
6248 infop->mri_start = ixgbe_ring_start;
6249 infop->mri_stop = NULL;
6250 infop->mri_poll = ixgbe_ring_rx_poll;
6251 infop->mri_stat = ixgbe_rx_ring_stat;
6252
6253 mintr->mi_handle = (mac_intr_handle_t)rx_ring;
6254 mintr->mi_enable = ixgbe_rx_ring_intr_enable;
6255 mintr->mi_disable = ixgbe_rx_ring_intr_disable;
6256 if (ixgbe->intr_type &
6257 (DDI_INTR_TYPE_MSIX | DDI_INTR_TYPE_MSI)) {
6258 mintr->mi_ddi_handle =
6259 ixgbe->htable[rx_ring->intr_vector];
6260 }
6261
6262 break;
6263 }
6264 case MAC_RING_TYPE_TX: {
6265 ASSERT(group_index == -1);
6266 ASSERT(ring_index < ixgbe->num_tx_rings);
6267
6268 ixgbe_tx_ring_t *tx_ring = &ixgbe->tx_rings[ring_index];
6269 tx_ring->ring_handle = rh;
6270
6271 infop->mri_driver = (mac_ring_driver_t)tx_ring;
6272 infop->mri_start = NULL;
6273 infop->mri_stop = NULL;
6274 infop->mri_tx = ixgbe_ring_tx;
6275 infop->mri_stat = ixgbe_tx_ring_stat;
6276 if (ixgbe->intr_type &
6277 (DDI_INTR_TYPE_MSIX | DDI_INTR_TYPE_MSI)) {
6278 mintr->mi_ddi_handle =
6279 ixgbe->htable[tx_ring->intr_vector];
6280 }
6281 break;
6282 }
6283 default:
6284 break;
6285 }
6286 }
6287
6288 /*
6289 * Callback funtion for MAC layer to register all groups.
6290 */
6291 void
ixgbe_fill_group(void * arg,mac_ring_type_t rtype,const int index,mac_group_info_t * infop,mac_group_handle_t gh)6292 ixgbe_fill_group(void *arg, mac_ring_type_t rtype, const int index,
6293 mac_group_info_t *infop, mac_group_handle_t gh)
6294 {
6295 ixgbe_t *ixgbe = (ixgbe_t *)arg;
6296 struct ixgbe_hw *hw = &ixgbe->hw;
6297
6298 switch (rtype) {
6299 case MAC_RING_TYPE_RX: {
6300 ixgbe_rx_group_t *rx_group;
6301
6302 rx_group = &ixgbe->rx_groups[index];
6303 rx_group->group_handle = gh;
6304
6305 infop->mgi_driver = (mac_group_driver_t)rx_group;
6306 infop->mgi_start = NULL;
6307 infop->mgi_stop = NULL;
6308 infop->mgi_addmac = ixgbe_addmac;
6309 infop->mgi_remmac = ixgbe_remmac;
6310
6311 if ((ixgbe->classify_mode == IXGBE_CLASSIFY_VMDQ ||
6312 ixgbe->classify_mode == IXGBE_CLASSIFY_VMDQ_RSS) &&
6313 (hw->mac.type == ixgbe_mac_82599EB ||
6314 hw->mac.type == ixgbe_mac_X540 ||
6315 hw->mac.type == ixgbe_mac_X550 ||
6316 hw->mac.type == ixgbe_mac_X550EM_x)) {
6317 infop->mgi_addvlan = ixgbe_addvlan;
6318 infop->mgi_remvlan = ixgbe_remvlan;
6319 } else {
6320 infop->mgi_addvlan = NULL;
6321 infop->mgi_remvlan = NULL;
6322 }
6323
6324 infop->mgi_count = (ixgbe->num_rx_rings / ixgbe->num_rx_groups);
6325
6326 break;
6327 }
6328 case MAC_RING_TYPE_TX:
6329 break;
6330 default:
6331 break;
6332 }
6333 }
6334
6335 /*
6336 * Enable interrupt on the specificed rx ring.
6337 */
6338 int
ixgbe_rx_ring_intr_enable(mac_intr_handle_t intrh)6339 ixgbe_rx_ring_intr_enable(mac_intr_handle_t intrh)
6340 {
6341 ixgbe_rx_ring_t *rx_ring = (ixgbe_rx_ring_t *)intrh;
6342 ixgbe_t *ixgbe = rx_ring->ixgbe;
6343 int r_idx = rx_ring->index;
6344 int hw_r_idx = rx_ring->hw_index;
6345 int v_idx = rx_ring->intr_vector;
6346
6347 mutex_enter(&ixgbe->gen_lock);
6348 if (ixgbe->ixgbe_state & IXGBE_INTR_ADJUST) {
6349 mutex_exit(&ixgbe->gen_lock);
6350 /*
6351 * Simply return 0.
6352 * Interrupts are being adjusted. ixgbe_intr_adjust()
6353 * will eventually re-enable the interrupt when it's
6354 * done with the adjustment.
6355 */
6356 return (0);
6357 }
6358
6359 /*
6360 * To enable interrupt by setting the VAL bit of given interrupt
6361 * vector allocation register (IVAR).
6362 */
6363 ixgbe_enable_ivar(ixgbe, hw_r_idx, 0);
6364
6365 BT_SET(ixgbe->vect_map[v_idx].rx_map, r_idx);
6366
6367 /*
6368 * Trigger a Rx interrupt on this ring
6369 */
6370 IXGBE_WRITE_REG(&ixgbe->hw, IXGBE_EICS, (1 << v_idx));
6371 IXGBE_WRITE_FLUSH(&ixgbe->hw);
6372
6373 mutex_exit(&ixgbe->gen_lock);
6374
6375 return (0);
6376 }
6377
6378 /*
6379 * Disable interrupt on the specificed rx ring.
6380 */
6381 int
ixgbe_rx_ring_intr_disable(mac_intr_handle_t intrh)6382 ixgbe_rx_ring_intr_disable(mac_intr_handle_t intrh)
6383 {
6384 ixgbe_rx_ring_t *rx_ring = (ixgbe_rx_ring_t *)intrh;
6385 ixgbe_t *ixgbe = rx_ring->ixgbe;
6386 int r_idx = rx_ring->index;
6387 int hw_r_idx = rx_ring->hw_index;
6388 int v_idx = rx_ring->intr_vector;
6389
6390 mutex_enter(&ixgbe->gen_lock);
6391 if (ixgbe->ixgbe_state & IXGBE_INTR_ADJUST) {
6392 mutex_exit(&ixgbe->gen_lock);
6393 /*
6394 * Simply return 0.
6395 * In the rare case where an interrupt is being
6396 * disabled while interrupts are being adjusted,
6397 * we don't fail the operation. No interrupts will
6398 * be generated while they are adjusted, and
6399 * ixgbe_intr_adjust() will cause the interrupts
6400 * to be re-enabled once it completes. Note that
6401 * in this case, packets may be delivered to the
6402 * stack via interrupts before xgbe_rx_ring_intr_enable()
6403 * is called again. This is acceptable since interrupt
6404 * adjustment is infrequent, and the stack will be
6405 * able to handle these packets.
6406 */
6407 return (0);
6408 }
6409
6410 /*
6411 * To disable interrupt by clearing the VAL bit of given interrupt
6412 * vector allocation register (IVAR).
6413 */
6414 ixgbe_disable_ivar(ixgbe, hw_r_idx, 0);
6415
6416 BT_CLEAR(ixgbe->vect_map[v_idx].rx_map, r_idx);
6417
6418 mutex_exit(&ixgbe->gen_lock);
6419
6420 return (0);
6421 }
6422
6423 static ixgbe_vlan_t *
ixgbe_find_vlan(ixgbe_rx_group_t * rx_group,uint16_t vid)6424 ixgbe_find_vlan(ixgbe_rx_group_t *rx_group, uint16_t vid)
6425 {
6426 for (ixgbe_vlan_t *vlp = list_head(&rx_group->vlans); vlp != NULL;
6427 vlp = list_next(&rx_group->vlans, vlp)) {
6428 if (vlp->ixvl_vid == vid)
6429 return (vlp);
6430 }
6431
6432 return (NULL);
6433 }
6434
6435 /*
6436 * Attempt to use a VLAN HW filter for this group. If the group is
6437 * interested in untagged packets then set AUPE only. If the group is
6438 * the default then only set the VFTA. Leave the VLVF slots open for
6439 * reserved groups to guarantee their use of HW filtering.
6440 */
6441 static int
ixgbe_addvlan(mac_group_driver_t gdriver,uint16_t vid)6442 ixgbe_addvlan(mac_group_driver_t gdriver, uint16_t vid)
6443 {
6444 ixgbe_rx_group_t *rx_group = (ixgbe_rx_group_t *)gdriver;
6445 ixgbe_t *ixgbe = rx_group->ixgbe;
6446 struct ixgbe_hw *hw = &ixgbe->hw;
6447 ixgbe_vlan_t *vlp;
6448 int ret;
6449 boolean_t is_def_grp;
6450
6451 mutex_enter(&ixgbe->gen_lock);
6452
6453 if (ixgbe->ixgbe_state & IXGBE_SUSPENDED) {
6454 mutex_exit(&ixgbe->gen_lock);
6455 return (ECANCELED);
6456 }
6457
6458 /*
6459 * Let's be sure VLAN filtering is enabled.
6460 */
6461 VERIFY3B(ixgbe->vlft_enabled, ==, B_TRUE);
6462 is_def_grp = (rx_group->index == ixgbe->rx_def_group);
6463
6464 /*
6465 * VLAN filtering is enabled but we want to receive untagged
6466 * traffic on this group -- set the AUPE bit on the group and
6467 * leave the VLAN tables alone.
6468 */
6469 if (vid == MAC_VLAN_UNTAGGED) {
6470 /*
6471 * We never enable AUPE on the default group; it is
6472 * redundant. Untagged traffic which passes L2
6473 * filtering is delivered to the default group if no
6474 * other group is interested.
6475 */
6476 if (!is_def_grp) {
6477 uint32_t vml2flt;
6478
6479 vml2flt = IXGBE_READ_REG(hw,
6480 IXGBE_VMOLR(rx_group->index));
6481 vml2flt |= IXGBE_VMOLR_AUPE;
6482 IXGBE_WRITE_REG(hw, IXGBE_VMOLR(rx_group->index),
6483 vml2flt);
6484 rx_group->aupe = B_TRUE;
6485 }
6486
6487 mutex_exit(&ixgbe->gen_lock);
6488 return (0);
6489 }
6490
6491 vlp = ixgbe_find_vlan(rx_group, vid);
6492 if (vlp != NULL) {
6493 /* Only the default group supports multiple clients. */
6494 VERIFY3B(is_def_grp, ==, B_TRUE);
6495 vlp->ixvl_refs++;
6496 mutex_exit(&ixgbe->gen_lock);
6497 return (0);
6498 }
6499
6500 /*
6501 * The default group doesn't require a VLVF entry, only a VFTA
6502 * entry. All traffic passing L2 filtering (MPSAR + VFTA) is
6503 * delivered to the default group if no other group is
6504 * interested. The fourth argument, vlvf_bypass, tells the
6505 * ixgbe common code to avoid using a VLVF slot if one isn't
6506 * already allocated to this VLAN.
6507 *
6508 * This logic is meant to reserve VLVF slots for use by
6509 * reserved groups: guaranteeing their use of HW filtering.
6510 */
6511 ret = ixgbe_set_vfta(hw, vid, rx_group->index, B_TRUE, is_def_grp);
6512
6513 if (ret == IXGBE_SUCCESS) {
6514 vlp = kmem_zalloc(sizeof (ixgbe_vlan_t), KM_SLEEP);
6515 vlp->ixvl_vid = vid;
6516 vlp->ixvl_refs = 1;
6517 list_insert_tail(&rx_group->vlans, vlp);
6518 mutex_exit(&ixgbe->gen_lock);
6519 return (0);
6520 }
6521
6522 /*
6523 * We should actually never return ENOSPC because we've set
6524 * things up so that every reserved group is guaranteed to
6525 * have a VLVF slot.
6526 */
6527 if (ret == IXGBE_ERR_PARAM)
6528 ret = EINVAL;
6529 else if (ret == IXGBE_ERR_NO_SPACE)
6530 ret = ENOSPC;
6531 else
6532 ret = EIO;
6533
6534 mutex_exit(&ixgbe->gen_lock);
6535 return (ret);
6536 }
6537
6538 /*
6539 * Attempt to remove the VLAN HW filter associated with this group. If
6540 * we are removing a HW filter for the default group then we know only
6541 * the VFTA was set (VLVF is reserved for non-default/reserved
6542 * groups). If the group wishes to stop receiving untagged traffic
6543 * then clear the AUPE but leave the VLAN filters alone.
6544 */
6545 static int
ixgbe_remvlan(mac_group_driver_t gdriver,uint16_t vid)6546 ixgbe_remvlan(mac_group_driver_t gdriver, uint16_t vid)
6547 {
6548 ixgbe_rx_group_t *rx_group = (ixgbe_rx_group_t *)gdriver;
6549 ixgbe_t *ixgbe = rx_group->ixgbe;
6550 struct ixgbe_hw *hw = &ixgbe->hw;
6551 int ret;
6552 ixgbe_vlan_t *vlp;
6553 boolean_t is_def_grp;
6554
6555 mutex_enter(&ixgbe->gen_lock);
6556
6557 if (ixgbe->ixgbe_state & IXGBE_SUSPENDED) {
6558 mutex_exit(&ixgbe->gen_lock);
6559 return (ECANCELED);
6560 }
6561
6562 is_def_grp = (rx_group->index == ixgbe->rx_def_group);
6563
6564 /* See the AUPE comment in ixgbe_addvlan(). */
6565 if (vid == MAC_VLAN_UNTAGGED) {
6566 if (!is_def_grp) {
6567 uint32_t vml2flt;
6568
6569 vml2flt = IXGBE_READ_REG(hw,
6570 IXGBE_VMOLR(rx_group->index));
6571 vml2flt &= ~IXGBE_VMOLR_AUPE;
6572 IXGBE_WRITE_REG(hw,
6573 IXGBE_VMOLR(rx_group->index), vml2flt);
6574 rx_group->aupe = B_FALSE;
6575 }
6576 mutex_exit(&ixgbe->gen_lock);
6577 return (0);
6578 }
6579
6580 vlp = ixgbe_find_vlan(rx_group, vid);
6581 if (vlp == NULL) {
6582 mutex_exit(&ixgbe->gen_lock);
6583 return (ENOENT);
6584 }
6585
6586 /*
6587 * See the comment in ixgbe_addvlan() about is_def_grp and
6588 * vlvf_bypass.
6589 */
6590 if (vlp->ixvl_refs == 1) {
6591 ret = ixgbe_set_vfta(hw, vid, rx_group->index, B_FALSE,
6592 is_def_grp);
6593 } else {
6594 /*
6595 * Only the default group can have multiple clients.
6596 * If there is more than one client, leave the
6597 * VFTA[vid] bit alone.
6598 */
6599 VERIFY3B(is_def_grp, ==, B_TRUE);
6600 VERIFY3U(vlp->ixvl_refs, >, 1);
6601 vlp->ixvl_refs--;
6602 mutex_exit(&ixgbe->gen_lock);
6603 return (0);
6604 }
6605
6606 if (ret != IXGBE_SUCCESS) {
6607 mutex_exit(&ixgbe->gen_lock);
6608 /* IXGBE_ERR_PARAM should be the only possible error here. */
6609 if (ret == IXGBE_ERR_PARAM)
6610 return (EINVAL);
6611 else
6612 return (EIO);
6613 }
6614
6615 VERIFY3U(vlp->ixvl_refs, ==, 1);
6616 vlp->ixvl_refs = 0;
6617 list_remove(&rx_group->vlans, vlp);
6618 kmem_free(vlp, sizeof (ixgbe_vlan_t));
6619
6620 /*
6621 * Calling ixgbe_set_vfta() on a non-default group may have
6622 * cleared the VFTA[vid] bit even though the default group
6623 * still has clients using the vid. This happens because the
6624 * ixgbe common code doesn't ref count the use of VLANs. Check
6625 * for any use of vid on the default group and make sure the
6626 * VFTA[vid] bit is set. This operation is idempotent: setting
6627 * VFTA[vid] to true if already true won't hurt anything.
6628 */
6629 if (!is_def_grp) {
6630 ixgbe_rx_group_t *defgrp;
6631
6632 defgrp = &ixgbe->rx_groups[ixgbe->rx_def_group];
6633 vlp = ixgbe_find_vlan(defgrp, vid);
6634 if (vlp != NULL) {
6635 /* This shouldn't fail, but if it does return EIO. */
6636 ret = ixgbe_set_vfta(hw, vid, rx_group->index, B_TRUE,
6637 B_TRUE);
6638 if (ret != IXGBE_SUCCESS) {
6639 mutex_exit(&ixgbe->gen_lock);
6640 return (EIO);
6641 }
6642 }
6643 }
6644
6645 mutex_exit(&ixgbe->gen_lock);
6646 return (0);
6647 }
6648
6649 /*
6650 * Add a mac address.
6651 */
6652 static int
ixgbe_addmac(void * arg,const uint8_t * mac_addr)6653 ixgbe_addmac(void *arg, const uint8_t *mac_addr)
6654 {
6655 ixgbe_rx_group_t *rx_group = (ixgbe_rx_group_t *)arg;
6656 ixgbe_t *ixgbe = rx_group->ixgbe;
6657 struct ixgbe_hw *hw = &ixgbe->hw;
6658 int slot, i;
6659
6660 mutex_enter(&ixgbe->gen_lock);
6661
6662 if (ixgbe->ixgbe_state & IXGBE_SUSPENDED) {
6663 mutex_exit(&ixgbe->gen_lock);
6664 return (ECANCELED);
6665 }
6666
6667 if (ixgbe->unicst_avail == 0) {
6668 /* no slots available */
6669 mutex_exit(&ixgbe->gen_lock);
6670 return (ENOSPC);
6671 }
6672
6673 /*
6674 * The first ixgbe->num_rx_groups slots are reserved for each respective
6675 * group. The rest slots are shared by all groups. While adding a
6676 * MAC address, reserved slots are firstly checked then the shared
6677 * slots are searched.
6678 */
6679 slot = -1;
6680 if (ixgbe->unicst_addr[rx_group->index].mac.set == 1) {
6681 for (i = ixgbe->num_rx_groups; i < ixgbe->unicst_total; i++) {
6682 if (ixgbe->unicst_addr[i].mac.set == 0) {
6683 slot = i;
6684 break;
6685 }
6686 }
6687 } else {
6688 slot = rx_group->index;
6689 }
6690
6691 if (slot == -1) {
6692 /* no slots available */
6693 mutex_exit(&ixgbe->gen_lock);
6694 return (ENOSPC);
6695 }
6696
6697 bcopy(mac_addr, ixgbe->unicst_addr[slot].mac.addr, ETHERADDRL);
6698 (void) ixgbe_set_rar(hw, slot, ixgbe->unicst_addr[slot].mac.addr,
6699 rx_group->index, IXGBE_RAH_AV);
6700 ixgbe->unicst_addr[slot].mac.set = 1;
6701 ixgbe->unicst_addr[slot].mac.group_index = rx_group->index;
6702 ixgbe->unicst_avail--;
6703
6704 mutex_exit(&ixgbe->gen_lock);
6705
6706 return (0);
6707 }
6708
6709 /*
6710 * Remove a mac address.
6711 */
6712 static int
ixgbe_remmac(void * arg,const uint8_t * mac_addr)6713 ixgbe_remmac(void *arg, const uint8_t *mac_addr)
6714 {
6715 ixgbe_rx_group_t *rx_group = (ixgbe_rx_group_t *)arg;
6716 ixgbe_t *ixgbe = rx_group->ixgbe;
6717 struct ixgbe_hw *hw = &ixgbe->hw;
6718 int slot;
6719
6720 mutex_enter(&ixgbe->gen_lock);
6721
6722 if (ixgbe->ixgbe_state & IXGBE_SUSPENDED) {
6723 mutex_exit(&ixgbe->gen_lock);
6724 return (ECANCELED);
6725 }
6726
6727 slot = ixgbe_unicst_find(ixgbe, mac_addr);
6728 if (slot == -1) {
6729 mutex_exit(&ixgbe->gen_lock);
6730 return (EINVAL);
6731 }
6732
6733 if (ixgbe->unicst_addr[slot].mac.set == 0) {
6734 mutex_exit(&ixgbe->gen_lock);
6735 return (EINVAL);
6736 }
6737
6738 bzero(ixgbe->unicst_addr[slot].mac.addr, ETHERADDRL);
6739 (void) ixgbe_clear_rar(hw, slot);
6740 ixgbe->unicst_addr[slot].mac.set = 0;
6741 ixgbe->unicst_avail++;
6742
6743 mutex_exit(&ixgbe->gen_lock);
6744
6745 return (0);
6746 }
6747
6748 static int
ixgbe_ufm_fill_image(ddi_ufm_handle_t * ufmh,void * arg,uint_t imgno,ddi_ufm_image_t * imgp)6749 ixgbe_ufm_fill_image(ddi_ufm_handle_t *ufmh, void *arg, uint_t imgno,
6750 ddi_ufm_image_t *imgp)
6751 {
6752 ixgbe_t *ixgbe = arg;
6753 const char *type;
6754
6755 if (imgno != 0) {
6756 return (EINVAL);
6757 }
6758
6759 ddi_ufm_image_set_desc(imgp, "NVM");
6760 ddi_ufm_image_set_nslots(imgp, 1);
6761 switch (ixgbe->hw.eeprom.type) {
6762 case ixgbe_eeprom_spi:
6763 type = "SPI EEPROM";
6764 break;
6765 case ixgbe_flash:
6766 type = "Flash";
6767 break;
6768 default:
6769 type = NULL;
6770 break;
6771 }
6772
6773 if (type != NULL) {
6774 nvlist_t *nvl;
6775
6776 nvl = fnvlist_alloc();
6777 fnvlist_add_string(nvl, "image-type", type);
6778 /*
6779 * The DDI takes ownership of the nvlist_t at this point.
6780 */
6781 ddi_ufm_image_set_misc(imgp, nvl);
6782 }
6783
6784 return (0);
6785 }
6786
6787 static int
ixgbe_ufm_fill_slot(ddi_ufm_handle_t * ufmh,void * arg,uint_t imgno,uint_t slotno,ddi_ufm_slot_t * slotp)6788 ixgbe_ufm_fill_slot(ddi_ufm_handle_t *ufmh, void *arg, uint_t imgno,
6789 uint_t slotno, ddi_ufm_slot_t *slotp)
6790 {
6791 ixgbe_t *ixgbe = arg;
6792
6793 if (imgno != 0 || slotno != 0) {
6794 return (EINVAL);
6795 }
6796
6797 /*
6798 * Unfortunately there is no generic versioning in the ixgbe family
6799 * eeprom parts.
6800 */
6801 ddi_ufm_slot_set_version(slotp, "unknown");
6802 ddi_ufm_slot_set_attrs(slotp, DDI_UFM_ATTR_ACTIVE |
6803 DDI_UFM_ATTR_READABLE | DDI_UFM_ATTR_WRITEABLE);
6804 ddi_ufm_slot_set_imgsize(slotp, ixgbe->hw.eeprom.word_size * 2);
6805
6806 return (0);
6807 }
6808
6809 static int
ixgbe_ufm_getcaps(ddi_ufm_handle_t * ufmh,void * arg,ddi_ufm_cap_t * caps)6810 ixgbe_ufm_getcaps(ddi_ufm_handle_t *ufmh, void *arg, ddi_ufm_cap_t *caps)
6811 {
6812 ixgbe_t *ixgbe = arg;
6813
6814 *caps = 0;
6815 switch (ixgbe->hw.eeprom.type) {
6816 case ixgbe_eeprom_spi:
6817 case ixgbe_flash:
6818 *caps |= DDI_UFM_CAP_REPORT;
6819 if (ixgbe->hw.eeprom.ops.read_buffer != NULL) {
6820 *caps |= DDI_UFM_CAP_READIMG;
6821 }
6822 break;
6823 default:
6824 break;
6825 }
6826
6827 return (0);
6828 }
6829
6830 static int
ixgbe_ufm_readimg(ddi_ufm_handle_t * ufmh,void * arg,uint_t imgno,uint_t slotno,uint64_t len,uint64_t offset,void * buf,uint64_t * nread)6831 ixgbe_ufm_readimg(ddi_ufm_handle_t *ufmh, void *arg, uint_t imgno,
6832 uint_t slotno, uint64_t len, uint64_t offset, void *buf, uint64_t *nread)
6833 {
6834 int ret;
6835 uint16_t wordoff, nwords, *buf16 = buf;
6836 ixgbe_t *ixgbe = arg;
6837 uint32_t imgsize = ixgbe->hw.eeprom.word_size * 2;
6838
6839 if (imgno != 0 || slotno != 0) {
6840 return (EINVAL);
6841 }
6842
6843 if (len > imgsize || offset > imgsize || len + offset > imgsize) {
6844 return (EINVAL);
6845 }
6846
6847 if (ixgbe->hw.eeprom.ops.read_buffer == NULL) {
6848 return (ENOTSUP);
6849 }
6850
6851 /*
6852 * Hardware provides us a means to read 16-bit words. For the time
6853 * being, restrict offset and length to be 2 byte aligned. We should
6854 * probably reduce this restriction. We could probably just use a bounce
6855 * buffer.
6856 */
6857 if ((offset % 2) != 0 || (len % 2) != 0) {
6858 return (EINVAL);
6859 }
6860
6861 wordoff = offset >> 1;
6862 nwords = len >> 1;
6863 mutex_enter(&ixgbe->gen_lock);
6864 ret = ixgbe_read_eeprom_buffer(&ixgbe->hw, wordoff, nwords, buf16);
6865 mutex_exit(&ixgbe->gen_lock);
6866
6867 if (ret == 0) {
6868 uint16_t i;
6869 *nread = len;
6870 for (i = 0; i < nwords; i++) {
6871 buf16[i] = LE_16(buf16[i]);
6872 }
6873 } else {
6874 ret = EIO;
6875 }
6876
6877 return (ret);
6878 }
6879