1 /****************************************************************************** 2 3 Copyright (c) 2013-2015, Intel Corporation 4 All rights reserved. 5 6 Redistribution and use in source and binary forms, with or without 7 modification, are permitted provided that the following conditions are met: 8 9 1. Redistributions of source code must retain the above copyright notice, 10 this list of conditions and the following disclaimer. 11 12 2. Redistributions in binary form must reproduce the above copyright 13 notice, this list of conditions and the following disclaimer in the 14 documentation and/or other materials provided with the distribution. 15 16 3. Neither the name of the Intel Corporation nor the names of its 17 contributors may be used to endorse or promote products derived from 18 this software without specific prior written permission. 19 20 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 23 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 24 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 25 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 26 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 27 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 28 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 29 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 30 POSSIBILITY OF SUCH DAMAGE. 31 32 ******************************************************************************/ 33 /*$FreeBSD$*/ 34 35 #ifndef IXL_STANDALONE_BUILD 36 #include "opt_inet.h" 37 #include "opt_inet6.h" 38 #include "opt_rss.h" 39 #endif 40 41 #include "ixl.h" 42 #include "ixl_pf.h" 43 44 #ifdef RSS 45 #include <net/rss_config.h> 46 #endif 47 48 /********************************************************************* 49 * Driver version 50 *********************************************************************/ 51 char ixl_driver_version[] = "1.4.3"; 52 53 /********************************************************************* 54 * PCI Device ID Table 55 * 56 * Used by probe to select devices to load on 57 * Last field stores an index into ixl_strings 58 * Last entry must be all 0s 59 * 60 * { Vendor ID, Device ID, SubVendor ID, SubDevice ID, String Index } 61 *********************************************************************/ 62 63 static ixl_vendor_info_t ixl_vendor_info_array[] = 64 { 65 {I40E_INTEL_VENDOR_ID, I40E_DEV_ID_SFP_XL710, 0, 0, 0}, 66 {I40E_INTEL_VENDOR_ID, I40E_DEV_ID_KX_B, 0, 0, 0}, 67 {I40E_INTEL_VENDOR_ID, I40E_DEV_ID_KX_C, 0, 0, 0}, 68 {I40E_INTEL_VENDOR_ID, I40E_DEV_ID_QSFP_A, 0, 0, 0}, 69 {I40E_INTEL_VENDOR_ID, I40E_DEV_ID_QSFP_B, 0, 0, 0}, 70 {I40E_INTEL_VENDOR_ID, I40E_DEV_ID_QSFP_C, 0, 0, 0}, 71 {I40E_INTEL_VENDOR_ID, I40E_DEV_ID_10G_BASE_T, 0, 0, 0}, 72 {I40E_INTEL_VENDOR_ID, I40E_DEV_ID_10G_BASE_T4, 0, 0, 0}, 73 #ifdef X722_SUPPORT 74 {I40E_INTEL_VENDOR_ID, I40E_DEV_ID_SFP_X722, 0, 0, 0}, 75 {I40E_INTEL_VENDOR_ID, I40E_DEV_ID_1G_BASE_T_X722, 0, 0, 0}, 76 {I40E_INTEL_VENDOR_ID, I40E_DEV_ID_10G_BASE_T_X722, 0, 0, 0}, 77 #endif 78 /* required last entry */ 79 {0, 0, 0, 0, 0} 80 }; 81 82 /********************************************************************* 83 * Table of branding strings 84 *********************************************************************/ 85 86 static char *ixl_strings[] = { 87 "Intel(R) Ethernet Connection XL710 Driver" 88 }; 89 90 91 /********************************************************************* 92 * Function prototypes 93 *********************************************************************/ 94 static int ixl_probe(device_t); 95 static int ixl_attach(device_t); 96 static int ixl_detach(device_t); 97 static int ixl_shutdown(device_t); 98 static int ixl_get_hw_capabilities(struct ixl_pf *); 99 static void ixl_cap_txcsum_tso(struct ixl_vsi *, struct ifnet *, int); 100 static int ixl_ioctl(struct ifnet *, u_long, caddr_t); 101 static void ixl_init(void *); 102 static void ixl_init_locked(struct ixl_pf *); 103 static void ixl_stop(struct ixl_pf *); 104 static void ixl_media_status(struct ifnet *, struct ifmediareq *); 105 static int ixl_media_change(struct ifnet *); 106 static void ixl_update_link_status(struct ixl_pf *); 107 static int ixl_allocate_pci_resources(struct ixl_pf *); 108 static u16 ixl_get_bus_info(struct i40e_hw *, device_t); 109 static int ixl_setup_stations(struct ixl_pf *); 110 static int ixl_switch_config(struct ixl_pf *); 111 static int ixl_initialize_vsi(struct ixl_vsi *); 112 static int ixl_assign_vsi_msix(struct ixl_pf *); 113 static int ixl_assign_vsi_legacy(struct ixl_pf *); 114 static int ixl_init_msix(struct ixl_pf *); 115 static void ixl_configure_msix(struct ixl_pf *); 116 static void ixl_configure_itr(struct ixl_pf *); 117 static void ixl_configure_legacy(struct ixl_pf *); 118 static void ixl_init_taskqueues(struct ixl_pf *); 119 static void ixl_free_taskqueues(struct ixl_pf *); 120 static void ixl_free_pci_resources(struct ixl_pf *); 121 static void ixl_local_timer(void *); 122 static int ixl_setup_interface(device_t, struct ixl_vsi *); 123 static void ixl_link_event(struct ixl_pf *, struct i40e_arq_event_info *); 124 static void ixl_config_rss(struct ixl_vsi *); 125 static void ixl_set_queue_rx_itr(struct ixl_queue *); 126 static void ixl_set_queue_tx_itr(struct ixl_queue *); 127 static int ixl_set_advertised_speeds(struct ixl_pf *, int); 128 129 static int ixl_enable_rings(struct ixl_vsi *); 130 static int ixl_disable_rings(struct ixl_vsi *); 131 static void ixl_enable_intr(struct ixl_vsi *); 132 static void ixl_disable_intr(struct ixl_vsi *); 133 static void ixl_disable_rings_intr(struct ixl_vsi *); 134 135 static void ixl_enable_adminq(struct i40e_hw *); 136 static void ixl_disable_adminq(struct i40e_hw *); 137 static void ixl_enable_queue(struct i40e_hw *, int); 138 static void ixl_disable_queue(struct i40e_hw *, int); 139 static void ixl_enable_legacy(struct i40e_hw *); 140 static void ixl_disable_legacy(struct i40e_hw *); 141 142 static void ixl_set_promisc(struct ixl_vsi *); 143 static void ixl_add_multi(struct ixl_vsi *); 144 static void ixl_del_multi(struct ixl_vsi *); 145 static void ixl_register_vlan(void *, struct ifnet *, u16); 146 static void ixl_unregister_vlan(void *, struct ifnet *, u16); 147 static void ixl_setup_vlan_filters(struct ixl_vsi *); 148 149 static void ixl_init_filters(struct ixl_vsi *); 150 static void ixl_reconfigure_filters(struct ixl_vsi *vsi); 151 static void ixl_add_filter(struct ixl_vsi *, u8 *, s16 vlan); 152 static void ixl_del_filter(struct ixl_vsi *, u8 *, s16 vlan); 153 static void ixl_add_hw_filters(struct ixl_vsi *, int, int); 154 static void ixl_del_hw_filters(struct ixl_vsi *, int); 155 static struct ixl_mac_filter * 156 ixl_find_filter(struct ixl_vsi *, u8 *, s16); 157 static void ixl_add_mc_filter(struct ixl_vsi *, u8 *); 158 static void ixl_free_mac_filters(struct ixl_vsi *vsi); 159 160 161 /* Sysctl debug interface */ 162 #ifdef IXL_DEBUG_SYSCTL 163 static int ixl_debug_info(SYSCTL_HANDLER_ARGS); 164 static void ixl_print_debug_info(struct ixl_pf *); 165 #endif 166 167 /* The MSI/X Interrupt handlers */ 168 static void ixl_intr(void *); 169 static void ixl_msix_que(void *); 170 static void ixl_msix_adminq(void *); 171 static void ixl_handle_mdd_event(struct ixl_pf *); 172 173 /* Deferred interrupt tasklets */ 174 static void ixl_do_adminq(void *, int); 175 176 /* Sysctl handlers */ 177 static int ixl_set_flowcntl(SYSCTL_HANDLER_ARGS); 178 static int ixl_set_advertise(SYSCTL_HANDLER_ARGS); 179 static int ixl_current_speed(SYSCTL_HANDLER_ARGS); 180 static int ixl_sysctl_show_fw(SYSCTL_HANDLER_ARGS); 181 182 /* Statistics */ 183 static void ixl_add_hw_stats(struct ixl_pf *); 184 static void ixl_add_sysctls_mac_stats(struct sysctl_ctx_list *, 185 struct sysctl_oid_list *, struct i40e_hw_port_stats *); 186 static void ixl_add_sysctls_eth_stats(struct sysctl_ctx_list *, 187 struct sysctl_oid_list *, 188 struct i40e_eth_stats *); 189 static void ixl_update_stats_counters(struct ixl_pf *); 190 static void ixl_update_eth_stats(struct ixl_vsi *); 191 static void ixl_update_vsi_stats(struct ixl_vsi *); 192 static void ixl_pf_reset_stats(struct ixl_pf *); 193 static void ixl_vsi_reset_stats(struct ixl_vsi *); 194 static void ixl_stat_update48(struct i40e_hw *, u32, u32, bool, 195 u64 *, u64 *); 196 static void ixl_stat_update32(struct i40e_hw *, u32, bool, 197 u64 *, u64 *); 198 199 #ifdef IXL_DEBUG_SYSCTL 200 static int ixl_sysctl_link_status(SYSCTL_HANDLER_ARGS); 201 static int ixl_sysctl_phy_abilities(SYSCTL_HANDLER_ARGS); 202 static int ixl_sysctl_sw_filter_list(SYSCTL_HANDLER_ARGS); 203 static int ixl_sysctl_hw_res_alloc(SYSCTL_HANDLER_ARGS); 204 static int ixl_sysctl_switch_config(SYSCTL_HANDLER_ARGS); 205 #endif 206 207 #ifdef PCI_IOV 208 static int ixl_adminq_err_to_errno(enum i40e_admin_queue_err err); 209 210 static int ixl_iov_init(device_t dev, uint16_t num_vfs, const nvlist_t*); 211 static void ixl_iov_uninit(device_t dev); 212 static int ixl_add_vf(device_t dev, uint16_t vfnum, const nvlist_t*); 213 214 static void ixl_handle_vf_msg(struct ixl_pf *, 215 struct i40e_arq_event_info *); 216 static void ixl_handle_vflr(void *arg, int pending); 217 218 static void ixl_reset_vf(struct ixl_pf *pf, struct ixl_vf *vf); 219 static void ixl_reinit_vf(struct ixl_pf *pf, struct ixl_vf *vf); 220 #endif 221 222 /********************************************************************* 223 * FreeBSD Device Interface Entry Points 224 *********************************************************************/ 225 226 static device_method_t ixl_methods[] = { 227 /* Device interface */ 228 DEVMETHOD(device_probe, ixl_probe), 229 DEVMETHOD(device_attach, ixl_attach), 230 DEVMETHOD(device_detach, ixl_detach), 231 DEVMETHOD(device_shutdown, ixl_shutdown), 232 #ifdef PCI_IOV 233 DEVMETHOD(pci_iov_init, ixl_iov_init), 234 DEVMETHOD(pci_iov_uninit, ixl_iov_uninit), 235 DEVMETHOD(pci_iov_add_vf, ixl_add_vf), 236 #endif 237 {0, 0} 238 }; 239 240 static driver_t ixl_driver = { 241 "ixl", ixl_methods, sizeof(struct ixl_pf), 242 }; 243 244 devclass_t ixl_devclass; 245 DRIVER_MODULE(ixl, pci, ixl_driver, ixl_devclass, 0, 0); 246 247 MODULE_DEPEND(ixl, pci, 1, 1, 1); 248 MODULE_DEPEND(ixl, ether, 1, 1, 1); 249 #ifdef DEV_NETMAP 250 MODULE_DEPEND(ixl, netmap, 1, 1, 1); 251 #endif /* DEV_NETMAP */ 252 253 /* 254 ** Global reset mutex 255 */ 256 static struct mtx ixl_reset_mtx; 257 258 /* 259 ** TUNEABLE PARAMETERS: 260 */ 261 262 static SYSCTL_NODE(_hw, OID_AUTO, ixl, CTLFLAG_RD, 0, 263 "IXL driver parameters"); 264 265 /* 266 * MSIX should be the default for best performance, 267 * but this allows it to be forced off for testing. 268 */ 269 static int ixl_enable_msix = 1; 270 TUNABLE_INT("hw.ixl.enable_msix", &ixl_enable_msix); 271 SYSCTL_INT(_hw_ixl, OID_AUTO, enable_msix, CTLFLAG_RDTUN, &ixl_enable_msix, 0, 272 "Enable MSI-X interrupts"); 273 274 /* 275 ** Number of descriptors per ring: 276 ** - TX and RX are the same size 277 */ 278 static int ixl_ringsz = DEFAULT_RING; 279 TUNABLE_INT("hw.ixl.ringsz", &ixl_ringsz); 280 SYSCTL_INT(_hw_ixl, OID_AUTO, ring_size, CTLFLAG_RDTUN, 281 &ixl_ringsz, 0, "Descriptor Ring Size"); 282 283 /* 284 ** This can be set manually, if left as 0 the 285 ** number of queues will be calculated based 286 ** on cpus and msix vectors available. 287 */ 288 int ixl_max_queues = 0; 289 TUNABLE_INT("hw.ixl.max_queues", &ixl_max_queues); 290 SYSCTL_INT(_hw_ixl, OID_AUTO, max_queues, CTLFLAG_RDTUN, 291 &ixl_max_queues, 0, "Number of Queues"); 292 293 /* 294 ** Controls for Interrupt Throttling 295 ** - true/false for dynamic adjustment 296 ** - default values for static ITR 297 */ 298 int ixl_dynamic_rx_itr = 0; 299 TUNABLE_INT("hw.ixl.dynamic_rx_itr", &ixl_dynamic_rx_itr); 300 SYSCTL_INT(_hw_ixl, OID_AUTO, dynamic_rx_itr, CTLFLAG_RDTUN, 301 &ixl_dynamic_rx_itr, 0, "Dynamic RX Interrupt Rate"); 302 303 int ixl_dynamic_tx_itr = 0; 304 TUNABLE_INT("hw.ixl.dynamic_tx_itr", &ixl_dynamic_tx_itr); 305 SYSCTL_INT(_hw_ixl, OID_AUTO, dynamic_tx_itr, CTLFLAG_RDTUN, 306 &ixl_dynamic_tx_itr, 0, "Dynamic TX Interrupt Rate"); 307 308 int ixl_rx_itr = IXL_ITR_8K; 309 TUNABLE_INT("hw.ixl.rx_itr", &ixl_rx_itr); 310 SYSCTL_INT(_hw_ixl, OID_AUTO, rx_itr, CTLFLAG_RDTUN, 311 &ixl_rx_itr, 0, "RX Interrupt Rate"); 312 313 int ixl_tx_itr = IXL_ITR_4K; 314 TUNABLE_INT("hw.ixl.tx_itr", &ixl_tx_itr); 315 SYSCTL_INT(_hw_ixl, OID_AUTO, tx_itr, CTLFLAG_RDTUN, 316 &ixl_tx_itr, 0, "TX Interrupt Rate"); 317 318 #ifdef IXL_FDIR 319 static int ixl_enable_fdir = 1; 320 TUNABLE_INT("hw.ixl.enable_fdir", &ixl_enable_fdir); 321 /* Rate at which we sample */ 322 int ixl_atr_rate = 20; 323 TUNABLE_INT("hw.ixl.atr_rate", &ixl_atr_rate); 324 #endif 325 326 #ifdef DEV_NETMAP 327 #define NETMAP_IXL_MAIN /* only bring in one part of the netmap code */ 328 #include <dev/netmap/if_ixl_netmap.h> 329 #endif /* DEV_NETMAP */ 330 331 static char *ixl_fc_string[6] = { 332 "None", 333 "Rx", 334 "Tx", 335 "Full", 336 "Priority", 337 "Default" 338 }; 339 340 static MALLOC_DEFINE(M_IXL, "ixl", "ixl driver allocations"); 341 342 static uint8_t ixl_bcast_addr[ETHER_ADDR_LEN] = 343 {0xff, 0xff, 0xff, 0xff, 0xff, 0xff}; 344 345 /********************************************************************* 346 * Device identification routine 347 * 348 * ixl_probe determines if the driver should be loaded on 349 * the hardware based on PCI vendor/device id of the device. 350 * 351 * return BUS_PROBE_DEFAULT on success, positive on failure 352 *********************************************************************/ 353 354 static int 355 ixl_probe(device_t dev) 356 { 357 ixl_vendor_info_t *ent; 358 359 u16 pci_vendor_id, pci_device_id; 360 u16 pci_subvendor_id, pci_subdevice_id; 361 char device_name[256]; 362 static bool lock_init = FALSE; 363 364 INIT_DEBUGOUT("ixl_probe: begin"); 365 366 pci_vendor_id = pci_get_vendor(dev); 367 if (pci_vendor_id != I40E_INTEL_VENDOR_ID) 368 return (ENXIO); 369 370 pci_device_id = pci_get_device(dev); 371 pci_subvendor_id = pci_get_subvendor(dev); 372 pci_subdevice_id = pci_get_subdevice(dev); 373 374 ent = ixl_vendor_info_array; 375 while (ent->vendor_id != 0) { 376 if ((pci_vendor_id == ent->vendor_id) && 377 (pci_device_id == ent->device_id) && 378 379 ((pci_subvendor_id == ent->subvendor_id) || 380 (ent->subvendor_id == 0)) && 381 382 ((pci_subdevice_id == ent->subdevice_id) || 383 (ent->subdevice_id == 0))) { 384 sprintf(device_name, "%s, Version - %s", 385 ixl_strings[ent->index], 386 ixl_driver_version); 387 device_set_desc_copy(dev, device_name); 388 /* One shot mutex init */ 389 if (lock_init == FALSE) { 390 lock_init = TRUE; 391 mtx_init(&ixl_reset_mtx, 392 "ixl_reset", 393 "IXL RESET Lock", MTX_DEF); 394 } 395 return (BUS_PROBE_DEFAULT); 396 } 397 ent++; 398 } 399 return (ENXIO); 400 } 401 402 /********************************************************************* 403 * Device initialization routine 404 * 405 * The attach entry point is called when the driver is being loaded. 406 * This routine identifies the type of hardware, allocates all resources 407 * and initializes the hardware. 408 * 409 * return 0 on success, positive on failure 410 *********************************************************************/ 411 412 static int 413 ixl_attach(device_t dev) 414 { 415 struct ixl_pf *pf; 416 struct i40e_hw *hw; 417 struct ixl_vsi *vsi; 418 u16 bus; 419 int error = 0; 420 #ifdef PCI_IOV 421 nvlist_t *pf_schema, *vf_schema; 422 int iov_error; 423 #endif 424 425 INIT_DEBUGOUT("ixl_attach: begin"); 426 427 /* Allocate, clear, and link in our primary soft structure */ 428 pf = device_get_softc(dev); 429 pf->dev = pf->osdep.dev = dev; 430 hw = &pf->hw; 431 432 /* 433 ** Note this assumes we have a single embedded VSI, 434 ** this could be enhanced later to allocate multiple 435 */ 436 vsi = &pf->vsi; 437 vsi->dev = pf->dev; 438 439 /* Core Lock Init*/ 440 IXL_PF_LOCK_INIT(pf, device_get_nameunit(dev)); 441 442 /* Set up the timer callout */ 443 callout_init_mtx(&pf->timer, &pf->pf_mtx, 0); 444 445 /* Set up sysctls */ 446 SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev), 447 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), 448 OID_AUTO, "fc", CTLTYPE_INT | CTLFLAG_RW, 449 pf, 0, ixl_set_flowcntl, "I", "Flow Control"); 450 451 SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev), 452 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), 453 OID_AUTO, "advertise_speed", CTLTYPE_INT | CTLFLAG_RW, 454 pf, 0, ixl_set_advertise, "I", "Advertised Speed"); 455 456 SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev), 457 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), 458 OID_AUTO, "current_speed", CTLTYPE_STRING | CTLFLAG_RD, 459 pf, 0, ixl_current_speed, "A", "Current Port Speed"); 460 461 SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev), 462 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), 463 OID_AUTO, "fw_version", CTLTYPE_STRING | CTLFLAG_RD, 464 pf, 0, ixl_sysctl_show_fw, "A", "Firmware version"); 465 466 SYSCTL_ADD_INT(device_get_sysctl_ctx(dev), 467 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), 468 OID_AUTO, "rx_itr", CTLFLAG_RW, 469 &ixl_rx_itr, IXL_ITR_8K, "RX ITR"); 470 471 SYSCTL_ADD_INT(device_get_sysctl_ctx(dev), 472 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), 473 OID_AUTO, "dynamic_rx_itr", CTLFLAG_RW, 474 &ixl_dynamic_rx_itr, 0, "Dynamic RX ITR"); 475 476 SYSCTL_ADD_INT(device_get_sysctl_ctx(dev), 477 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), 478 OID_AUTO, "tx_itr", CTLFLAG_RW, 479 &ixl_tx_itr, IXL_ITR_4K, "TX ITR"); 480 481 SYSCTL_ADD_INT(device_get_sysctl_ctx(dev), 482 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), 483 OID_AUTO, "dynamic_tx_itr", CTLFLAG_RW, 484 &ixl_dynamic_tx_itr, 0, "Dynamic TX ITR"); 485 486 #ifdef IXL_DEBUG_SYSCTL 487 SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev), 488 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), 489 OID_AUTO, "debug", CTLTYPE_INT|CTLFLAG_RW, pf, 0, 490 ixl_debug_info, "I", "Debug Information"); 491 492 /* Debug shared-code message level */ 493 SYSCTL_ADD_UINT(device_get_sysctl_ctx(dev), 494 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), 495 OID_AUTO, "debug_mask", CTLFLAG_RW, 496 &pf->hw.debug_mask, 0, "Debug Message Level"); 497 498 SYSCTL_ADD_UINT(device_get_sysctl_ctx(dev), 499 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), 500 OID_AUTO, "vc_debug_level", CTLFLAG_RW, &pf->vc_debug_lvl, 501 0, "PF/VF Virtual Channel debug level"); 502 503 SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev), 504 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), 505 OID_AUTO, "link_status", CTLTYPE_STRING | CTLFLAG_RD, 506 pf, 0, ixl_sysctl_link_status, "A", "Current Link Status"); 507 508 SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev), 509 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), 510 OID_AUTO, "phy_abilities", CTLTYPE_STRING | CTLFLAG_RD, 511 pf, 0, ixl_sysctl_phy_abilities, "A", "PHY Abilities"); 512 513 SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev), 514 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), 515 OID_AUTO, "filter_list", CTLTYPE_STRING | CTLFLAG_RD, 516 pf, 0, ixl_sysctl_sw_filter_list, "A", "SW Filter List"); 517 518 SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev), 519 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), 520 OID_AUTO, "hw_res_alloc", CTLTYPE_STRING | CTLFLAG_RD, 521 pf, 0, ixl_sysctl_hw_res_alloc, "A", "HW Resource Allocation"); 522 523 SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev), 524 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), 525 OID_AUTO, "switch_config", CTLTYPE_STRING | CTLFLAG_RD, 526 pf, 0, ixl_sysctl_switch_config, "A", "HW Switch Configuration"); 527 #endif 528 529 /* Save off the PCI information */ 530 hw->vendor_id = pci_get_vendor(dev); 531 hw->device_id = pci_get_device(dev); 532 hw->revision_id = pci_read_config(dev, PCIR_REVID, 1); 533 hw->subsystem_vendor_id = 534 pci_read_config(dev, PCIR_SUBVEND_0, 2); 535 hw->subsystem_device_id = 536 pci_read_config(dev, PCIR_SUBDEV_0, 2); 537 538 hw->bus.device = pci_get_slot(dev); 539 hw->bus.func = pci_get_function(dev); 540 541 pf->vc_debug_lvl = 1; 542 543 /* Do PCI setup - map BAR0, etc */ 544 if (ixl_allocate_pci_resources(pf)) { 545 device_printf(dev, "Allocation of PCI resources failed\n"); 546 error = ENXIO; 547 goto err_out; 548 } 549 550 /* Establish a clean starting point */ 551 i40e_clear_hw(hw); 552 error = i40e_pf_reset(hw); 553 if (error) { 554 device_printf(dev,"PF reset failure %x\n", error); 555 error = EIO; 556 goto err_out; 557 } 558 559 /* Set admin queue parameters */ 560 hw->aq.num_arq_entries = IXL_AQ_LEN; 561 hw->aq.num_asq_entries = IXL_AQ_LEN; 562 hw->aq.arq_buf_size = IXL_AQ_BUFSZ; 563 hw->aq.asq_buf_size = IXL_AQ_BUFSZ; 564 565 /* Initialize the shared code */ 566 error = i40e_init_shared_code(hw); 567 if (error) { 568 device_printf(dev,"Unable to initialize the shared code\n"); 569 error = EIO; 570 goto err_out; 571 } 572 573 /* Set up the admin queue */ 574 error = i40e_init_adminq(hw); 575 if (error) { 576 device_printf(dev, "The driver for the device stopped " 577 "because the NVM image is newer than expected.\n" 578 "You must install the most recent version of " 579 " the network driver.\n"); 580 goto err_out; 581 } 582 device_printf(dev, "%s\n", ixl_fw_version_str(hw)); 583 584 if (hw->aq.api_maj_ver == I40E_FW_API_VERSION_MAJOR && 585 hw->aq.api_min_ver > I40E_FW_API_VERSION_MINOR) 586 device_printf(dev, "The driver for the device detected " 587 "a newer version of the NVM image than expected.\n" 588 "Please install the most recent version of the network driver.\n"); 589 else if (hw->aq.api_maj_ver < I40E_FW_API_VERSION_MAJOR || 590 hw->aq.api_min_ver < (I40E_FW_API_VERSION_MINOR - 1)) 591 device_printf(dev, "The driver for the device detected " 592 "an older version of the NVM image than expected.\n" 593 "Please update the NVM image.\n"); 594 595 /* Clear PXE mode */ 596 i40e_clear_pxe_mode(hw); 597 598 /* Get capabilities from the device */ 599 error = ixl_get_hw_capabilities(pf); 600 if (error) { 601 device_printf(dev, "HW capabilities failure!\n"); 602 goto err_get_cap; 603 } 604 605 /* Set up host memory cache */ 606 error = i40e_init_lan_hmc(hw, hw->func_caps.num_tx_qp, 607 hw->func_caps.num_rx_qp, 0, 0); 608 if (error) { 609 device_printf(dev, "init_lan_hmc failed: %d\n", error); 610 goto err_get_cap; 611 } 612 613 error = i40e_configure_lan_hmc(hw, I40E_HMC_MODEL_DIRECT_ONLY); 614 if (error) { 615 device_printf(dev, "configure_lan_hmc failed: %d\n", error); 616 goto err_mac_hmc; 617 } 618 619 /* Disable LLDP from the firmware */ 620 i40e_aq_stop_lldp(hw, TRUE, NULL); 621 622 i40e_get_mac_addr(hw, hw->mac.addr); 623 error = i40e_validate_mac_addr(hw->mac.addr); 624 if (error) { 625 device_printf(dev, "validate_mac_addr failed: %d\n", error); 626 goto err_mac_hmc; 627 } 628 bcopy(hw->mac.addr, hw->mac.perm_addr, ETHER_ADDR_LEN); 629 i40e_get_port_mac_addr(hw, hw->mac.port_addr); 630 631 /* Set up VSI and queues */ 632 if (ixl_setup_stations(pf) != 0) { 633 device_printf(dev, "setup stations failed!\n"); 634 error = ENOMEM; 635 goto err_mac_hmc; 636 } 637 638 /* Initialize mac filter list for VSI */ 639 SLIST_INIT(&vsi->ftl); 640 641 /* Set up interrupt routing here */ 642 if (pf->msix > 1) 643 error = ixl_assign_vsi_msix(pf); 644 else 645 error = ixl_assign_vsi_legacy(pf); 646 if (error) 647 goto err_mac_hmc; 648 649 if (((hw->aq.fw_maj_ver == 4) && (hw->aq.fw_min_ver < 33)) || 650 (hw->aq.fw_maj_ver < 4)) { 651 i40e_msec_delay(75); 652 error = i40e_aq_set_link_restart_an(hw, TRUE, NULL); 653 if (error) 654 device_printf(dev, "link restart failed, aq_err=%d\n", 655 pf->hw.aq.asq_last_status); 656 } 657 658 /* Determine link state */ 659 i40e_aq_get_link_info(hw, TRUE, NULL, NULL); 660 i40e_get_link_status(hw, &pf->link_up); 661 662 /* Setup OS specific network interface */ 663 if (ixl_setup_interface(dev, vsi) != 0) { 664 device_printf(dev, "interface setup failed!\n"); 665 error = EIO; 666 goto err_late; 667 } 668 669 error = ixl_switch_config(pf); 670 if (error) { 671 device_printf(dev, "Initial switch config failed: %d\n", error); 672 goto err_late; 673 } 674 675 /* Limit phy interrupts to link and modules failure */ 676 error = i40e_aq_set_phy_int_mask(hw, ~(I40E_AQ_EVENT_LINK_UPDOWN | 677 I40E_AQ_EVENT_MODULE_QUAL_FAIL), NULL); 678 if (error) 679 device_printf(dev, "set phy mask failed: %d\n", error); 680 681 /* Get the bus configuration and set the shared code */ 682 bus = ixl_get_bus_info(hw, dev); 683 i40e_set_pci_config_data(hw, bus); 684 685 /* Initialize taskqueues */ 686 ixl_init_taskqueues(pf); 687 688 /* Initialize statistics */ 689 ixl_pf_reset_stats(pf); 690 ixl_update_stats_counters(pf); 691 ixl_add_hw_stats(pf); 692 693 /* Register for VLAN events */ 694 vsi->vlan_attach = EVENTHANDLER_REGISTER(vlan_config, 695 ixl_register_vlan, vsi, EVENTHANDLER_PRI_FIRST); 696 vsi->vlan_detach = EVENTHANDLER_REGISTER(vlan_unconfig, 697 ixl_unregister_vlan, vsi, EVENTHANDLER_PRI_FIRST); 698 699 #ifdef PCI_IOV 700 /* SR-IOV is only supported when MSI-X is in use. */ 701 if (pf->msix > 1) { 702 pf_schema = pci_iov_schema_alloc_node(); 703 vf_schema = pci_iov_schema_alloc_node(); 704 pci_iov_schema_add_unicast_mac(vf_schema, "mac-addr", 0, NULL); 705 pci_iov_schema_add_bool(vf_schema, "mac-anti-spoof", 706 IOV_SCHEMA_HASDEFAULT, TRUE); 707 pci_iov_schema_add_bool(vf_schema, "allow-set-mac", 708 IOV_SCHEMA_HASDEFAULT, FALSE); 709 pci_iov_schema_add_bool(vf_schema, "allow-promisc", 710 IOV_SCHEMA_HASDEFAULT, FALSE); 711 712 iov_error = pci_iov_attach(dev, pf_schema, vf_schema); 713 if (iov_error != 0) 714 device_printf(dev, 715 "Failed to initialize SR-IOV (error=%d)\n", 716 iov_error); 717 } 718 #endif 719 720 #ifdef DEV_NETMAP 721 ixl_netmap_attach(vsi); 722 #endif /* DEV_NETMAP */ 723 INIT_DEBUGOUT("ixl_attach: end"); 724 return (0); 725 726 err_late: 727 if (vsi->ifp != NULL) 728 if_free(vsi->ifp); 729 err_mac_hmc: 730 i40e_shutdown_lan_hmc(hw); 731 err_get_cap: 732 i40e_shutdown_adminq(hw); 733 err_out: 734 ixl_free_pci_resources(pf); 735 ixl_free_vsi(vsi); 736 IXL_PF_LOCK_DESTROY(pf); 737 return (error); 738 } 739 740 /********************************************************************* 741 * Device removal routine 742 * 743 * The detach entry point is called when the driver is being removed. 744 * This routine stops the adapter and deallocates all the resources 745 * that were allocated for driver operation. 746 * 747 * return 0 on success, positive on failure 748 *********************************************************************/ 749 750 static int 751 ixl_detach(device_t dev) 752 { 753 struct ixl_pf *pf = device_get_softc(dev); 754 struct i40e_hw *hw = &pf->hw; 755 struct ixl_vsi *vsi = &pf->vsi; 756 i40e_status status; 757 #ifdef PCI_IOV 758 int error; 759 #endif 760 761 INIT_DEBUGOUT("ixl_detach: begin"); 762 763 /* Make sure VLANS are not using driver */ 764 if (vsi->ifp->if_vlantrunk != NULL) { 765 device_printf(dev,"Vlan in use, detach first\n"); 766 return (EBUSY); 767 } 768 769 #ifdef PCI_IOV 770 error = pci_iov_detach(dev); 771 if (error != 0) { 772 device_printf(dev, "SR-IOV in use; detach first.\n"); 773 return (error); 774 } 775 #endif 776 777 ether_ifdetach(vsi->ifp); 778 if (vsi->ifp->if_drv_flags & IFF_DRV_RUNNING) { 779 IXL_PF_LOCK(pf); 780 ixl_stop(pf); 781 IXL_PF_UNLOCK(pf); 782 } 783 784 ixl_free_taskqueues(pf); 785 786 /* Shutdown LAN HMC */ 787 status = i40e_shutdown_lan_hmc(hw); 788 if (status) 789 device_printf(dev, 790 "Shutdown LAN HMC failed with code %d\n", status); 791 792 /* Shutdown admin queue */ 793 status = i40e_shutdown_adminq(hw); 794 if (status) 795 device_printf(dev, 796 "Shutdown Admin queue failed with code %d\n", status); 797 798 /* Unregister VLAN events */ 799 if (vsi->vlan_attach != NULL) 800 EVENTHANDLER_DEREGISTER(vlan_config, vsi->vlan_attach); 801 if (vsi->vlan_detach != NULL) 802 EVENTHANDLER_DEREGISTER(vlan_unconfig, vsi->vlan_detach); 803 804 callout_drain(&pf->timer); 805 #ifdef DEV_NETMAP 806 netmap_detach(vsi->ifp); 807 #endif /* DEV_NETMAP */ 808 ixl_free_pci_resources(pf); 809 bus_generic_detach(dev); 810 if_free(vsi->ifp); 811 ixl_free_vsi(vsi); 812 IXL_PF_LOCK_DESTROY(pf); 813 return (0); 814 } 815 816 /********************************************************************* 817 * 818 * Shutdown entry point 819 * 820 **********************************************************************/ 821 822 static int 823 ixl_shutdown(device_t dev) 824 { 825 struct ixl_pf *pf = device_get_softc(dev); 826 IXL_PF_LOCK(pf); 827 ixl_stop(pf); 828 IXL_PF_UNLOCK(pf); 829 return (0); 830 } 831 832 833 /********************************************************************* 834 * 835 * Get the hardware capabilities 836 * 837 **********************************************************************/ 838 839 static int 840 ixl_get_hw_capabilities(struct ixl_pf *pf) 841 { 842 struct i40e_aqc_list_capabilities_element_resp *buf; 843 struct i40e_hw *hw = &pf->hw; 844 device_t dev = pf->dev; 845 int error, len; 846 u16 needed; 847 bool again = TRUE; 848 849 len = 40 * sizeof(struct i40e_aqc_list_capabilities_element_resp); 850 retry: 851 if (!(buf = (struct i40e_aqc_list_capabilities_element_resp *) 852 malloc(len, M_DEVBUF, M_NOWAIT | M_ZERO))) { 853 device_printf(dev, "Unable to allocate cap memory\n"); 854 return (ENOMEM); 855 } 856 857 /* This populates the hw struct */ 858 error = i40e_aq_discover_capabilities(hw, buf, len, 859 &needed, i40e_aqc_opc_list_func_capabilities, NULL); 860 free(buf, M_DEVBUF); 861 if ((pf->hw.aq.asq_last_status == I40E_AQ_RC_ENOMEM) && 862 (again == TRUE)) { 863 /* retry once with a larger buffer */ 864 again = FALSE; 865 len = needed; 866 goto retry; 867 } else if (pf->hw.aq.asq_last_status != I40E_AQ_RC_OK) { 868 device_printf(dev, "capability discovery failed: %d\n", 869 pf->hw.aq.asq_last_status); 870 return (ENODEV); 871 } 872 873 /* Capture this PF's starting queue pair */ 874 pf->qbase = hw->func_caps.base_queue; 875 876 #ifdef IXL_DEBUG 877 device_printf(dev,"pf_id=%d, num_vfs=%d, msix_pf=%d, " 878 "msix_vf=%d, fd_g=%d, fd_b=%d, tx_qp=%d rx_qp=%d qbase=%d\n", 879 hw->pf_id, hw->func_caps.num_vfs, 880 hw->func_caps.num_msix_vectors, 881 hw->func_caps.num_msix_vectors_vf, 882 hw->func_caps.fd_filters_guaranteed, 883 hw->func_caps.fd_filters_best_effort, 884 hw->func_caps.num_tx_qp, 885 hw->func_caps.num_rx_qp, 886 hw->func_caps.base_queue); 887 #endif 888 return (error); 889 } 890 891 static void 892 ixl_cap_txcsum_tso(struct ixl_vsi *vsi, struct ifnet *ifp, int mask) 893 { 894 device_t dev = vsi->dev; 895 896 /* Enable/disable TXCSUM/TSO4 */ 897 if (!(ifp->if_capenable & IFCAP_TXCSUM) 898 && !(ifp->if_capenable & IFCAP_TSO4)) { 899 if (mask & IFCAP_TXCSUM) { 900 ifp->if_capenable |= IFCAP_TXCSUM; 901 /* enable TXCSUM, restore TSO if previously enabled */ 902 if (vsi->flags & IXL_FLAGS_KEEP_TSO4) { 903 vsi->flags &= ~IXL_FLAGS_KEEP_TSO4; 904 ifp->if_capenable |= IFCAP_TSO4; 905 } 906 } 907 else if (mask & IFCAP_TSO4) { 908 ifp->if_capenable |= (IFCAP_TXCSUM | IFCAP_TSO4); 909 vsi->flags &= ~IXL_FLAGS_KEEP_TSO4; 910 device_printf(dev, 911 "TSO4 requires txcsum, enabling both...\n"); 912 } 913 } else if((ifp->if_capenable & IFCAP_TXCSUM) 914 && !(ifp->if_capenable & IFCAP_TSO4)) { 915 if (mask & IFCAP_TXCSUM) 916 ifp->if_capenable &= ~IFCAP_TXCSUM; 917 else if (mask & IFCAP_TSO4) 918 ifp->if_capenable |= IFCAP_TSO4; 919 } else if((ifp->if_capenable & IFCAP_TXCSUM) 920 && (ifp->if_capenable & IFCAP_TSO4)) { 921 if (mask & IFCAP_TXCSUM) { 922 vsi->flags |= IXL_FLAGS_KEEP_TSO4; 923 ifp->if_capenable &= ~(IFCAP_TXCSUM | IFCAP_TSO4); 924 device_printf(dev, 925 "TSO4 requires txcsum, disabling both...\n"); 926 } else if (mask & IFCAP_TSO4) 927 ifp->if_capenable &= ~IFCAP_TSO4; 928 } 929 930 /* Enable/disable TXCSUM_IPV6/TSO6 */ 931 if (!(ifp->if_capenable & IFCAP_TXCSUM_IPV6) 932 && !(ifp->if_capenable & IFCAP_TSO6)) { 933 if (mask & IFCAP_TXCSUM_IPV6) { 934 ifp->if_capenable |= IFCAP_TXCSUM_IPV6; 935 if (vsi->flags & IXL_FLAGS_KEEP_TSO6) { 936 vsi->flags &= ~IXL_FLAGS_KEEP_TSO6; 937 ifp->if_capenable |= IFCAP_TSO6; 938 } 939 } else if (mask & IFCAP_TSO6) { 940 ifp->if_capenable |= (IFCAP_TXCSUM_IPV6 | IFCAP_TSO6); 941 vsi->flags &= ~IXL_FLAGS_KEEP_TSO6; 942 device_printf(dev, 943 "TSO6 requires txcsum6, enabling both...\n"); 944 } 945 } else if((ifp->if_capenable & IFCAP_TXCSUM_IPV6) 946 && !(ifp->if_capenable & IFCAP_TSO6)) { 947 if (mask & IFCAP_TXCSUM_IPV6) 948 ifp->if_capenable &= ~IFCAP_TXCSUM_IPV6; 949 else if (mask & IFCAP_TSO6) 950 ifp->if_capenable |= IFCAP_TSO6; 951 } else if ((ifp->if_capenable & IFCAP_TXCSUM_IPV6) 952 && (ifp->if_capenable & IFCAP_TSO6)) { 953 if (mask & IFCAP_TXCSUM_IPV6) { 954 vsi->flags |= IXL_FLAGS_KEEP_TSO6; 955 ifp->if_capenable &= ~(IFCAP_TXCSUM_IPV6 | IFCAP_TSO6); 956 device_printf(dev, 957 "TSO6 requires txcsum6, disabling both...\n"); 958 } else if (mask & IFCAP_TSO6) 959 ifp->if_capenable &= ~IFCAP_TSO6; 960 } 961 } 962 963 /********************************************************************* 964 * Ioctl entry point 965 * 966 * ixl_ioctl is called when the user wants to configure the 967 * interface. 968 * 969 * return 0 on success, positive on failure 970 **********************************************************************/ 971 972 static int 973 ixl_ioctl(struct ifnet * ifp, u_long command, caddr_t data) 974 { 975 struct ixl_vsi *vsi = ifp->if_softc; 976 struct ixl_pf *pf = vsi->back; 977 struct ifreq *ifr = (struct ifreq *) data; 978 #if defined(INET) || defined(INET6) 979 struct ifaddr *ifa = (struct ifaddr *)data; 980 bool avoid_reset = FALSE; 981 #endif 982 int error = 0; 983 984 switch (command) { 985 986 case SIOCSIFADDR: 987 #ifdef INET 988 if (ifa->ifa_addr->sa_family == AF_INET) 989 avoid_reset = TRUE; 990 #endif 991 #ifdef INET6 992 if (ifa->ifa_addr->sa_family == AF_INET6) 993 avoid_reset = TRUE; 994 #endif 995 #if defined(INET) || defined(INET6) 996 /* 997 ** Calling init results in link renegotiation, 998 ** so we avoid doing it when possible. 999 */ 1000 if (avoid_reset) { 1001 ifp->if_flags |= IFF_UP; 1002 if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) 1003 ixl_init(pf); 1004 #ifdef INET 1005 if (!(ifp->if_flags & IFF_NOARP)) 1006 arp_ifinit(ifp, ifa); 1007 #endif 1008 } else 1009 error = ether_ioctl(ifp, command, data); 1010 break; 1011 #endif 1012 case SIOCSIFMTU: 1013 IOCTL_DEBUGOUT("ioctl: SIOCSIFMTU (Set Interface MTU)"); 1014 if (ifr->ifr_mtu > IXL_MAX_FRAME - 1015 ETHER_HDR_LEN - ETHER_CRC_LEN - ETHER_VLAN_ENCAP_LEN) { 1016 error = EINVAL; 1017 } else { 1018 IXL_PF_LOCK(pf); 1019 ifp->if_mtu = ifr->ifr_mtu; 1020 vsi->max_frame_size = 1021 ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN 1022 + ETHER_VLAN_ENCAP_LEN; 1023 ixl_init_locked(pf); 1024 IXL_PF_UNLOCK(pf); 1025 } 1026 break; 1027 case SIOCSIFFLAGS: 1028 IOCTL_DEBUGOUT("ioctl: SIOCSIFFLAGS (Set Interface Flags)"); 1029 IXL_PF_LOCK(pf); 1030 if (ifp->if_flags & IFF_UP) { 1031 if ((ifp->if_drv_flags & IFF_DRV_RUNNING)) { 1032 if ((ifp->if_flags ^ pf->if_flags) & 1033 (IFF_PROMISC | IFF_ALLMULTI)) { 1034 ixl_set_promisc(vsi); 1035 } 1036 } else 1037 ixl_init_locked(pf); 1038 } else 1039 if (ifp->if_drv_flags & IFF_DRV_RUNNING) 1040 ixl_stop(pf); 1041 pf->if_flags = ifp->if_flags; 1042 IXL_PF_UNLOCK(pf); 1043 break; 1044 case SIOCADDMULTI: 1045 IOCTL_DEBUGOUT("ioctl: SIOCADDMULTI"); 1046 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 1047 IXL_PF_LOCK(pf); 1048 ixl_disable_intr(vsi); 1049 ixl_add_multi(vsi); 1050 ixl_enable_intr(vsi); 1051 IXL_PF_UNLOCK(pf); 1052 } 1053 break; 1054 case SIOCDELMULTI: 1055 IOCTL_DEBUGOUT("ioctl: SIOCDELMULTI"); 1056 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 1057 IXL_PF_LOCK(pf); 1058 ixl_disable_intr(vsi); 1059 ixl_del_multi(vsi); 1060 ixl_enable_intr(vsi); 1061 IXL_PF_UNLOCK(pf); 1062 } 1063 break; 1064 case SIOCSIFMEDIA: 1065 case SIOCGIFMEDIA: 1066 #ifdef IFM_ETH_XTYPE 1067 case SIOCGIFXMEDIA: 1068 #endif 1069 IOCTL_DEBUGOUT("ioctl: SIOCxIFMEDIA (Get/Set Interface Media)"); 1070 error = ifmedia_ioctl(ifp, ifr, &vsi->media, command); 1071 break; 1072 case SIOCSIFCAP: 1073 { 1074 int mask = ifr->ifr_reqcap ^ ifp->if_capenable; 1075 IOCTL_DEBUGOUT("ioctl: SIOCSIFCAP (Set Capabilities)"); 1076 1077 ixl_cap_txcsum_tso(vsi, ifp, mask); 1078 1079 if (mask & IFCAP_RXCSUM) 1080 ifp->if_capenable ^= IFCAP_RXCSUM; 1081 if (mask & IFCAP_RXCSUM_IPV6) 1082 ifp->if_capenable ^= IFCAP_RXCSUM_IPV6; 1083 if (mask & IFCAP_LRO) 1084 ifp->if_capenable ^= IFCAP_LRO; 1085 if (mask & IFCAP_VLAN_HWTAGGING) 1086 ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING; 1087 if (mask & IFCAP_VLAN_HWFILTER) 1088 ifp->if_capenable ^= IFCAP_VLAN_HWFILTER; 1089 if (mask & IFCAP_VLAN_HWTSO) 1090 ifp->if_capenable ^= IFCAP_VLAN_HWTSO; 1091 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 1092 IXL_PF_LOCK(pf); 1093 ixl_init_locked(pf); 1094 IXL_PF_UNLOCK(pf); 1095 } 1096 VLAN_CAPABILITIES(ifp); 1097 1098 break; 1099 } 1100 1101 default: 1102 IOCTL_DEBUGOUT("ioctl: UNKNOWN (0x%X)\n", (int)command); 1103 error = ether_ioctl(ifp, command, data); 1104 break; 1105 } 1106 1107 return (error); 1108 } 1109 1110 1111 /********************************************************************* 1112 * Init entry point 1113 * 1114 * This routine is used in two ways. It is used by the stack as 1115 * init entry point in network interface structure. It is also used 1116 * by the driver as a hw/sw initialization routine to get to a 1117 * consistent state. 1118 * 1119 * return 0 on success, positive on failure 1120 **********************************************************************/ 1121 1122 static void 1123 ixl_init_locked(struct ixl_pf *pf) 1124 { 1125 struct i40e_hw *hw = &pf->hw; 1126 struct ixl_vsi *vsi = &pf->vsi; 1127 struct ifnet *ifp = vsi->ifp; 1128 device_t dev = pf->dev; 1129 struct i40e_filter_control_settings filter; 1130 u8 tmpaddr[ETHER_ADDR_LEN]; 1131 int ret; 1132 1133 mtx_assert(&pf->pf_mtx, MA_OWNED); 1134 INIT_DEBUGOUT("ixl_init: begin"); 1135 ixl_stop(pf); 1136 1137 /* Get the latest mac address... User might use a LAA */ 1138 bcopy(IF_LLADDR(vsi->ifp), tmpaddr, 1139 I40E_ETH_LENGTH_OF_ADDRESS); 1140 if (!cmp_etheraddr(hw->mac.addr, tmpaddr) && 1141 (i40e_validate_mac_addr(tmpaddr) == I40E_SUCCESS)) { 1142 ixl_del_filter(vsi, hw->mac.addr, IXL_VLAN_ANY); 1143 bcopy(tmpaddr, hw->mac.addr, 1144 I40E_ETH_LENGTH_OF_ADDRESS); 1145 ret = i40e_aq_mac_address_write(hw, 1146 I40E_AQC_WRITE_TYPE_LAA_ONLY, 1147 hw->mac.addr, NULL); 1148 if (ret) { 1149 device_printf(dev, "LLA address" 1150 "change failed!!\n"); 1151 return; 1152 } else { 1153 ixl_add_filter(vsi, hw->mac.addr, IXL_VLAN_ANY); 1154 } 1155 } 1156 1157 /* Set the various hardware offload abilities */ 1158 ifp->if_hwassist = 0; 1159 if (ifp->if_capenable & IFCAP_TSO) 1160 ifp->if_hwassist |= CSUM_TSO; 1161 if (ifp->if_capenable & IFCAP_TXCSUM) 1162 ifp->if_hwassist |= (CSUM_TCP | CSUM_UDP); 1163 if (ifp->if_capenable & IFCAP_TXCSUM_IPV6) 1164 ifp->if_hwassist |= (CSUM_TCP_IPV6 | CSUM_UDP_IPV6); 1165 1166 /* Set up the device filtering */ 1167 bzero(&filter, sizeof(filter)); 1168 filter.enable_ethtype = TRUE; 1169 filter.enable_macvlan = TRUE; 1170 #ifdef IXL_FDIR 1171 filter.enable_fdir = TRUE; 1172 #endif 1173 filter.hash_lut_size = I40E_HASH_LUT_SIZE_512; 1174 if (i40e_set_filter_control(hw, &filter)) 1175 device_printf(dev, "set_filter_control() failed\n"); 1176 1177 /* Set up RSS */ 1178 ixl_config_rss(vsi); 1179 1180 /* 1181 ** Prepare the VSI: rings, hmc contexts, etc... 1182 */ 1183 if (ixl_initialize_vsi(vsi)) { 1184 device_printf(dev, "initialize vsi failed!!\n"); 1185 return; 1186 } 1187 1188 /* Add protocol filters to list */ 1189 ixl_init_filters(vsi); 1190 1191 /* Setup vlan's if needed */ 1192 ixl_setup_vlan_filters(vsi); 1193 1194 /* Start the local timer */ 1195 callout_reset(&pf->timer, hz, ixl_local_timer, pf); 1196 1197 /* Set up MSI/X routing and the ITR settings */ 1198 if (ixl_enable_msix) { 1199 ixl_configure_msix(pf); 1200 ixl_configure_itr(pf); 1201 } else 1202 ixl_configure_legacy(pf); 1203 1204 ixl_enable_rings(vsi); 1205 1206 i40e_aq_set_default_vsi(hw, vsi->seid, NULL); 1207 1208 ixl_reconfigure_filters(vsi); 1209 1210 /* Set MTU in hardware*/ 1211 int aq_error = i40e_aq_set_mac_config(hw, vsi->max_frame_size, 1212 TRUE, 0, NULL); 1213 if (aq_error) 1214 device_printf(vsi->dev, 1215 "aq_set_mac_config in init error, code %d\n", 1216 aq_error); 1217 1218 /* And now turn on interrupts */ 1219 ixl_enable_intr(vsi); 1220 1221 /* Now inform the stack we're ready */ 1222 ifp->if_drv_flags |= IFF_DRV_RUNNING; 1223 ifp->if_drv_flags &= ~IFF_DRV_OACTIVE; 1224 1225 return; 1226 } 1227 1228 static void 1229 ixl_init(void *arg) 1230 { 1231 struct ixl_pf *pf = arg; 1232 1233 IXL_PF_LOCK(pf); 1234 ixl_init_locked(pf); 1235 IXL_PF_UNLOCK(pf); 1236 return; 1237 } 1238 1239 /* 1240 ** 1241 ** MSIX Interrupt Handlers and Tasklets 1242 ** 1243 */ 1244 static void 1245 ixl_handle_que(void *context, int pending) 1246 { 1247 struct ixl_queue *que = context; 1248 struct ixl_vsi *vsi = que->vsi; 1249 struct i40e_hw *hw = vsi->hw; 1250 struct tx_ring *txr = &que->txr; 1251 struct ifnet *ifp = vsi->ifp; 1252 bool more; 1253 1254 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 1255 more = ixl_rxeof(que, IXL_RX_LIMIT); 1256 IXL_TX_LOCK(txr); 1257 ixl_txeof(que); 1258 if (!drbr_empty(ifp, txr->br)) 1259 ixl_mq_start_locked(ifp, txr); 1260 IXL_TX_UNLOCK(txr); 1261 if (more) { 1262 taskqueue_enqueue(que->tq, &que->task); 1263 return; 1264 } 1265 } 1266 1267 /* Reenable this interrupt - hmmm */ 1268 ixl_enable_queue(hw, que->me); 1269 return; 1270 } 1271 1272 1273 /********************************************************************* 1274 * 1275 * Legacy Interrupt Service routine 1276 * 1277 **********************************************************************/ 1278 void 1279 ixl_intr(void *arg) 1280 { 1281 struct ixl_pf *pf = arg; 1282 struct i40e_hw *hw = &pf->hw; 1283 struct ixl_vsi *vsi = &pf->vsi; 1284 struct ixl_queue *que = vsi->queues; 1285 struct ifnet *ifp = vsi->ifp; 1286 struct tx_ring *txr = &que->txr; 1287 u32 reg, icr0, mask; 1288 bool more_tx, more_rx; 1289 1290 ++que->irqs; 1291 1292 /* Protect against spurious interrupts */ 1293 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) 1294 return; 1295 1296 icr0 = rd32(hw, I40E_PFINT_ICR0); 1297 1298 reg = rd32(hw, I40E_PFINT_DYN_CTL0); 1299 reg = reg | I40E_PFINT_DYN_CTL0_CLEARPBA_MASK; 1300 wr32(hw, I40E_PFINT_DYN_CTL0, reg); 1301 1302 mask = rd32(hw, I40E_PFINT_ICR0_ENA); 1303 1304 #ifdef PCI_IOV 1305 if (icr0 & I40E_PFINT_ICR0_VFLR_MASK) 1306 taskqueue_enqueue(pf->tq, &pf->vflr_task); 1307 #endif 1308 1309 if (icr0 & I40E_PFINT_ICR0_ADMINQ_MASK) { 1310 taskqueue_enqueue(pf->tq, &pf->adminq); 1311 return; 1312 } 1313 1314 more_rx = ixl_rxeof(que, IXL_RX_LIMIT); 1315 1316 IXL_TX_LOCK(txr); 1317 more_tx = ixl_txeof(que); 1318 if (!drbr_empty(vsi->ifp, txr->br)) 1319 more_tx = 1; 1320 IXL_TX_UNLOCK(txr); 1321 1322 /* re-enable other interrupt causes */ 1323 wr32(hw, I40E_PFINT_ICR0_ENA, mask); 1324 1325 /* And now the queues */ 1326 reg = rd32(hw, I40E_QINT_RQCTL(0)); 1327 reg |= I40E_QINT_RQCTL_CAUSE_ENA_MASK; 1328 wr32(hw, I40E_QINT_RQCTL(0), reg); 1329 1330 reg = rd32(hw, I40E_QINT_TQCTL(0)); 1331 reg |= I40E_QINT_TQCTL_CAUSE_ENA_MASK; 1332 reg &= ~I40E_PFINT_ICR0_INTEVENT_MASK; 1333 wr32(hw, I40E_QINT_TQCTL(0), reg); 1334 1335 ixl_enable_legacy(hw); 1336 1337 return; 1338 } 1339 1340 1341 /********************************************************************* 1342 * 1343 * MSIX VSI Interrupt Service routine 1344 * 1345 **********************************************************************/ 1346 void 1347 ixl_msix_que(void *arg) 1348 { 1349 struct ixl_queue *que = arg; 1350 struct ixl_vsi *vsi = que->vsi; 1351 struct i40e_hw *hw = vsi->hw; 1352 struct tx_ring *txr = &que->txr; 1353 bool more_tx, more_rx; 1354 1355 /* Protect against spurious interrupts */ 1356 if (!(vsi->ifp->if_drv_flags & IFF_DRV_RUNNING)) 1357 return; 1358 1359 ++que->irqs; 1360 1361 more_rx = ixl_rxeof(que, IXL_RX_LIMIT); 1362 1363 IXL_TX_LOCK(txr); 1364 more_tx = ixl_txeof(que); 1365 /* 1366 ** Make certain that if the stack 1367 ** has anything queued the task gets 1368 ** scheduled to handle it. 1369 */ 1370 if (!drbr_empty(vsi->ifp, txr->br)) 1371 more_tx = 1; 1372 IXL_TX_UNLOCK(txr); 1373 1374 ixl_set_queue_rx_itr(que); 1375 ixl_set_queue_tx_itr(que); 1376 1377 if (more_tx || more_rx) 1378 taskqueue_enqueue(que->tq, &que->task); 1379 else 1380 ixl_enable_queue(hw, que->me); 1381 1382 return; 1383 } 1384 1385 1386 /********************************************************************* 1387 * 1388 * MSIX Admin Queue Interrupt Service routine 1389 * 1390 **********************************************************************/ 1391 static void 1392 ixl_msix_adminq(void *arg) 1393 { 1394 struct ixl_pf *pf = arg; 1395 struct i40e_hw *hw = &pf->hw; 1396 u32 reg, mask; 1397 1398 ++pf->admin_irq; 1399 1400 reg = rd32(hw, I40E_PFINT_ICR0); 1401 mask = rd32(hw, I40E_PFINT_ICR0_ENA); 1402 1403 /* Check on the cause */ 1404 if (reg & I40E_PFINT_ICR0_ADMINQ_MASK) 1405 mask &= ~I40E_PFINT_ICR0_ENA_ADMINQ_MASK; 1406 1407 if (reg & I40E_PFINT_ICR0_MAL_DETECT_MASK) { 1408 ixl_handle_mdd_event(pf); 1409 mask &= ~I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK; 1410 } 1411 1412 #ifdef PCI_IOV 1413 if (reg & I40E_PFINT_ICR0_VFLR_MASK) { 1414 mask &= ~I40E_PFINT_ICR0_ENA_VFLR_MASK; 1415 taskqueue_enqueue(pf->tq, &pf->vflr_task); 1416 } 1417 #endif 1418 1419 reg = rd32(hw, I40E_PFINT_DYN_CTL0); 1420 reg = reg | I40E_PFINT_DYN_CTL0_CLEARPBA_MASK; 1421 wr32(hw, I40E_PFINT_DYN_CTL0, reg); 1422 1423 taskqueue_enqueue(pf->tq, &pf->adminq); 1424 return; 1425 } 1426 1427 /********************************************************************* 1428 * 1429 * Media Ioctl callback 1430 * 1431 * This routine is called whenever the user queries the status of 1432 * the interface using ifconfig. 1433 * 1434 **********************************************************************/ 1435 static void 1436 ixl_media_status(struct ifnet * ifp, struct ifmediareq * ifmr) 1437 { 1438 struct ixl_vsi *vsi = ifp->if_softc; 1439 struct ixl_pf *pf = vsi->back; 1440 struct i40e_hw *hw = &pf->hw; 1441 1442 INIT_DEBUGOUT("ixl_media_status: begin"); 1443 IXL_PF_LOCK(pf); 1444 1445 hw->phy.get_link_info = TRUE; 1446 i40e_get_link_status(hw, &pf->link_up); 1447 ixl_update_link_status(pf); 1448 1449 ifmr->ifm_status = IFM_AVALID; 1450 ifmr->ifm_active = IFM_ETHER; 1451 1452 if (!pf->link_up) { 1453 IXL_PF_UNLOCK(pf); 1454 return; 1455 } 1456 1457 ifmr->ifm_status |= IFM_ACTIVE; 1458 /* Hardware is always full-duplex */ 1459 ifmr->ifm_active |= IFM_FDX; 1460 1461 switch (hw->phy.link_info.phy_type) { 1462 /* 100 M */ 1463 case I40E_PHY_TYPE_100BASE_TX: 1464 ifmr->ifm_active |= IFM_100_TX; 1465 break; 1466 /* 1 G */ 1467 case I40E_PHY_TYPE_1000BASE_T: 1468 ifmr->ifm_active |= IFM_1000_T; 1469 break; 1470 case I40E_PHY_TYPE_1000BASE_SX: 1471 ifmr->ifm_active |= IFM_1000_SX; 1472 break; 1473 case I40E_PHY_TYPE_1000BASE_LX: 1474 ifmr->ifm_active |= IFM_1000_LX; 1475 break; 1476 /* 10 G */ 1477 case I40E_PHY_TYPE_10GBASE_SFPP_CU: 1478 ifmr->ifm_active |= IFM_10G_TWINAX; 1479 break; 1480 case I40E_PHY_TYPE_10GBASE_SR: 1481 ifmr->ifm_active |= IFM_10G_SR; 1482 break; 1483 case I40E_PHY_TYPE_10GBASE_LR: 1484 ifmr->ifm_active |= IFM_10G_LR; 1485 break; 1486 case I40E_PHY_TYPE_10GBASE_T: 1487 ifmr->ifm_active |= IFM_10G_T; 1488 break; 1489 /* 40 G */ 1490 case I40E_PHY_TYPE_40GBASE_CR4: 1491 case I40E_PHY_TYPE_40GBASE_CR4_CU: 1492 ifmr->ifm_active |= IFM_40G_CR4; 1493 break; 1494 case I40E_PHY_TYPE_40GBASE_SR4: 1495 ifmr->ifm_active |= IFM_40G_SR4; 1496 break; 1497 case I40E_PHY_TYPE_40GBASE_LR4: 1498 ifmr->ifm_active |= IFM_40G_LR4; 1499 break; 1500 #ifndef IFM_ETH_XTYPE 1501 case I40E_PHY_TYPE_1000BASE_KX: 1502 ifmr->ifm_active |= IFM_1000_CX; 1503 break; 1504 case I40E_PHY_TYPE_10GBASE_CR1_CU: 1505 case I40E_PHY_TYPE_10GBASE_CR1: 1506 ifmr->ifm_active |= IFM_10G_TWINAX; 1507 break; 1508 case I40E_PHY_TYPE_10GBASE_KX4: 1509 ifmr->ifm_active |= IFM_10G_CX4; 1510 break; 1511 case I40E_PHY_TYPE_10GBASE_KR: 1512 ifmr->ifm_active |= IFM_10G_SR; 1513 break; 1514 case I40E_PHY_TYPE_40GBASE_KR4: 1515 case I40E_PHY_TYPE_XLPPI: 1516 ifmr->ifm_active |= IFM_40G_SR4; 1517 break; 1518 #else 1519 case I40E_PHY_TYPE_1000BASE_KX: 1520 ifmr->ifm_active |= IFM_1000_KX; 1521 break; 1522 /* ERJ: What's the difference between these? */ 1523 case I40E_PHY_TYPE_10GBASE_CR1_CU: 1524 case I40E_PHY_TYPE_10GBASE_CR1: 1525 ifmr->ifm_active |= IFM_10G_CR1; 1526 break; 1527 case I40E_PHY_TYPE_10GBASE_KX4: 1528 ifmr->ifm_active |= IFM_10G_KX4; 1529 break; 1530 case I40E_PHY_TYPE_10GBASE_KR: 1531 ifmr->ifm_active |= IFM_10G_KR; 1532 break; 1533 case I40E_PHY_TYPE_20GBASE_KR2: 1534 ifmr->ifm_active |= IFM_20G_KR2; 1535 break; 1536 case I40E_PHY_TYPE_40GBASE_KR4: 1537 ifmr->ifm_active |= IFM_40G_KR4; 1538 break; 1539 case I40E_PHY_TYPE_XLPPI: 1540 ifmr->ifm_active |= IFM_40G_XLPPI; 1541 break; 1542 #endif 1543 default: 1544 ifmr->ifm_active |= IFM_UNKNOWN; 1545 break; 1546 } 1547 /* Report flow control status as well */ 1548 if (hw->phy.link_info.an_info & I40E_AQ_LINK_PAUSE_TX) 1549 ifmr->ifm_active |= IFM_ETH_TXPAUSE; 1550 if (hw->phy.link_info.an_info & I40E_AQ_LINK_PAUSE_RX) 1551 ifmr->ifm_active |= IFM_ETH_RXPAUSE; 1552 1553 IXL_PF_UNLOCK(pf); 1554 1555 return; 1556 } 1557 1558 /********************************************************************* 1559 * 1560 * Media Ioctl callback 1561 * 1562 * This routine is called when the user changes speed/duplex using 1563 * media/mediopt option with ifconfig. 1564 * 1565 **********************************************************************/ 1566 static int 1567 ixl_media_change(struct ifnet * ifp) 1568 { 1569 struct ixl_vsi *vsi = ifp->if_softc; 1570 struct ifmedia *ifm = &vsi->media; 1571 1572 INIT_DEBUGOUT("ixl_media_change: begin"); 1573 1574 if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER) 1575 return (EINVAL); 1576 1577 if_printf(ifp, "Media change is currently not supported.\n"); 1578 1579 return (ENODEV); 1580 } 1581 1582 1583 #ifdef IXL_FDIR 1584 /* 1585 ** ATR: Application Targetted Receive - creates a filter 1586 ** based on TX flow info that will keep the receive 1587 ** portion of the flow on the same queue. Based on the 1588 ** implementation this is only available for TCP connections 1589 */ 1590 void 1591 ixl_atr(struct ixl_queue *que, struct tcphdr *th, int etype) 1592 { 1593 struct ixl_vsi *vsi = que->vsi; 1594 struct tx_ring *txr = &que->txr; 1595 struct i40e_filter_program_desc *FDIR; 1596 u32 ptype, dtype; 1597 int idx; 1598 1599 /* check if ATR is enabled and sample rate */ 1600 if ((!ixl_enable_fdir) || (!txr->atr_rate)) 1601 return; 1602 /* 1603 ** We sample all TCP SYN/FIN packets, 1604 ** or at the selected sample rate 1605 */ 1606 txr->atr_count++; 1607 if (((th->th_flags & (TH_FIN | TH_SYN)) == 0) && 1608 (txr->atr_count < txr->atr_rate)) 1609 return; 1610 txr->atr_count = 0; 1611 1612 /* Get a descriptor to use */ 1613 idx = txr->next_avail; 1614 FDIR = (struct i40e_filter_program_desc *) &txr->base[idx]; 1615 if (++idx == que->num_desc) 1616 idx = 0; 1617 txr->avail--; 1618 txr->next_avail = idx; 1619 1620 ptype = (que->me << I40E_TXD_FLTR_QW0_QINDEX_SHIFT) & 1621 I40E_TXD_FLTR_QW0_QINDEX_MASK; 1622 1623 ptype |= (etype == ETHERTYPE_IP) ? 1624 (I40E_FILTER_PCTYPE_NONF_IPV4_TCP << 1625 I40E_TXD_FLTR_QW0_PCTYPE_SHIFT) : 1626 (I40E_FILTER_PCTYPE_NONF_IPV6_TCP << 1627 I40E_TXD_FLTR_QW0_PCTYPE_SHIFT); 1628 1629 ptype |= vsi->id << I40E_TXD_FLTR_QW0_DEST_VSI_SHIFT; 1630 1631 dtype = I40E_TX_DESC_DTYPE_FILTER_PROG; 1632 1633 /* 1634 ** We use the TCP TH_FIN as a trigger to remove 1635 ** the filter, otherwise its an update. 1636 */ 1637 dtype |= (th->th_flags & TH_FIN) ? 1638 (I40E_FILTER_PROGRAM_DESC_PCMD_REMOVE << 1639 I40E_TXD_FLTR_QW1_PCMD_SHIFT) : 1640 (I40E_FILTER_PROGRAM_DESC_PCMD_ADD_UPDATE << 1641 I40E_TXD_FLTR_QW1_PCMD_SHIFT); 1642 1643 dtype |= I40E_FILTER_PROGRAM_DESC_DEST_DIRECT_PACKET_QINDEX << 1644 I40E_TXD_FLTR_QW1_DEST_SHIFT; 1645 1646 dtype |= I40E_FILTER_PROGRAM_DESC_FD_STATUS_FD_ID << 1647 I40E_TXD_FLTR_QW1_FD_STATUS_SHIFT; 1648 1649 FDIR->qindex_flex_ptype_vsi = htole32(ptype); 1650 FDIR->dtype_cmd_cntindex = htole32(dtype); 1651 return; 1652 } 1653 #endif 1654 1655 1656 static void 1657 ixl_set_promisc(struct ixl_vsi *vsi) 1658 { 1659 struct ifnet *ifp = vsi->ifp; 1660 struct i40e_hw *hw = vsi->hw; 1661 int err, mcnt = 0; 1662 bool uni = FALSE, multi = FALSE; 1663 1664 if (ifp->if_flags & IFF_ALLMULTI) 1665 multi = TRUE; 1666 else { /* Need to count the multicast addresses */ 1667 struct ifmultiaddr *ifma; 1668 if_maddr_rlock(ifp); 1669 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { 1670 if (ifma->ifma_addr->sa_family != AF_LINK) 1671 continue; 1672 if (mcnt == MAX_MULTICAST_ADDR) 1673 break; 1674 mcnt++; 1675 } 1676 if_maddr_runlock(ifp); 1677 } 1678 1679 if (mcnt >= MAX_MULTICAST_ADDR) 1680 multi = TRUE; 1681 if (ifp->if_flags & IFF_PROMISC) 1682 uni = TRUE; 1683 1684 err = i40e_aq_set_vsi_unicast_promiscuous(hw, 1685 vsi->seid, uni, NULL); 1686 err = i40e_aq_set_vsi_multicast_promiscuous(hw, 1687 vsi->seid, multi, NULL); 1688 return; 1689 } 1690 1691 /********************************************************************* 1692 * Filter Routines 1693 * 1694 * Routines for multicast and vlan filter management. 1695 * 1696 *********************************************************************/ 1697 static void 1698 ixl_add_multi(struct ixl_vsi *vsi) 1699 { 1700 struct ifmultiaddr *ifma; 1701 struct ifnet *ifp = vsi->ifp; 1702 struct i40e_hw *hw = vsi->hw; 1703 int mcnt = 0, flags; 1704 1705 IOCTL_DEBUGOUT("ixl_add_multi: begin"); 1706 1707 if_maddr_rlock(ifp); 1708 /* 1709 ** First just get a count, to decide if we 1710 ** we simply use multicast promiscuous. 1711 */ 1712 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { 1713 if (ifma->ifma_addr->sa_family != AF_LINK) 1714 continue; 1715 mcnt++; 1716 } 1717 if_maddr_runlock(ifp); 1718 1719 if (__predict_false(mcnt >= MAX_MULTICAST_ADDR)) { 1720 /* delete existing MC filters */ 1721 ixl_del_hw_filters(vsi, mcnt); 1722 i40e_aq_set_vsi_multicast_promiscuous(hw, 1723 vsi->seid, TRUE, NULL); 1724 return; 1725 } 1726 1727 mcnt = 0; 1728 if_maddr_rlock(ifp); 1729 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { 1730 if (ifma->ifma_addr->sa_family != AF_LINK) 1731 continue; 1732 ixl_add_mc_filter(vsi, 1733 (u8*)LLADDR((struct sockaddr_dl *) ifma->ifma_addr)); 1734 mcnt++; 1735 } 1736 if_maddr_runlock(ifp); 1737 if (mcnt > 0) { 1738 flags = (IXL_FILTER_ADD | IXL_FILTER_USED | IXL_FILTER_MC); 1739 ixl_add_hw_filters(vsi, flags, mcnt); 1740 } 1741 1742 IOCTL_DEBUGOUT("ixl_add_multi: end"); 1743 return; 1744 } 1745 1746 static void 1747 ixl_del_multi(struct ixl_vsi *vsi) 1748 { 1749 struct ifnet *ifp = vsi->ifp; 1750 struct ifmultiaddr *ifma; 1751 struct ixl_mac_filter *f; 1752 int mcnt = 0; 1753 bool match = FALSE; 1754 1755 IOCTL_DEBUGOUT("ixl_del_multi: begin"); 1756 1757 /* Search for removed multicast addresses */ 1758 if_maddr_rlock(ifp); 1759 SLIST_FOREACH(f, &vsi->ftl, next) { 1760 if ((f->flags & IXL_FILTER_USED) && (f->flags & IXL_FILTER_MC)) { 1761 match = FALSE; 1762 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { 1763 if (ifma->ifma_addr->sa_family != AF_LINK) 1764 continue; 1765 u8 *mc_addr = (u8 *)LLADDR((struct sockaddr_dl *)ifma->ifma_addr); 1766 if (cmp_etheraddr(f->macaddr, mc_addr)) { 1767 match = TRUE; 1768 break; 1769 } 1770 } 1771 if (match == FALSE) { 1772 f->flags |= IXL_FILTER_DEL; 1773 mcnt++; 1774 } 1775 } 1776 } 1777 if_maddr_runlock(ifp); 1778 1779 if (mcnt > 0) 1780 ixl_del_hw_filters(vsi, mcnt); 1781 } 1782 1783 1784 /********************************************************************* 1785 * Timer routine 1786 * 1787 * This routine checks for link status,updates statistics, 1788 * and runs the watchdog check. 1789 * 1790 **********************************************************************/ 1791 1792 static void 1793 ixl_local_timer(void *arg) 1794 { 1795 struct ixl_pf *pf = arg; 1796 struct i40e_hw *hw = &pf->hw; 1797 struct ixl_vsi *vsi = &pf->vsi; 1798 struct ixl_queue *que = vsi->queues; 1799 device_t dev = pf->dev; 1800 int hung = 0; 1801 u32 mask; 1802 1803 mtx_assert(&pf->pf_mtx, MA_OWNED); 1804 1805 /* Fire off the adminq task */ 1806 taskqueue_enqueue(pf->tq, &pf->adminq); 1807 1808 /* Update stats */ 1809 ixl_update_stats_counters(pf); 1810 1811 /* 1812 ** Check status of the queues 1813 */ 1814 mask = (I40E_PFINT_DYN_CTLN_INTENA_MASK | 1815 I40E_PFINT_DYN_CTLN_SWINT_TRIG_MASK); 1816 1817 for (int i = 0; i < vsi->num_queues; i++,que++) { 1818 /* Any queues with outstanding work get a sw irq */ 1819 if (que->busy) 1820 wr32(hw, I40E_PFINT_DYN_CTLN(que->me), mask); 1821 /* 1822 ** Each time txeof runs without cleaning, but there 1823 ** are uncleaned descriptors it increments busy. If 1824 ** we get to 5 we declare it hung. 1825 */ 1826 if (que->busy == IXL_QUEUE_HUNG) { 1827 ++hung; 1828 /* Mark the queue as inactive */ 1829 vsi->active_queues &= ~((u64)1 << que->me); 1830 continue; 1831 } else { 1832 /* Check if we've come back from hung */ 1833 if ((vsi->active_queues & ((u64)1 << que->me)) == 0) 1834 vsi->active_queues |= ((u64)1 << que->me); 1835 } 1836 if (que->busy >= IXL_MAX_TX_BUSY) { 1837 #ifdef IXL_DEBUG 1838 device_printf(dev,"Warning queue %d " 1839 "appears to be hung!\n", i); 1840 #endif 1841 que->busy = IXL_QUEUE_HUNG; 1842 ++hung; 1843 } 1844 } 1845 /* Only reinit if all queues show hung */ 1846 if (hung == vsi->num_queues) 1847 goto hung; 1848 1849 callout_reset(&pf->timer, hz, ixl_local_timer, pf); 1850 return; 1851 1852 hung: 1853 device_printf(dev, "Local Timer: HANG DETECT - Resetting!!\n"); 1854 ixl_init_locked(pf); 1855 } 1856 1857 /* 1858 ** Note: this routine updates the OS on the link state 1859 ** the real check of the hardware only happens with 1860 ** a link interrupt. 1861 */ 1862 static void 1863 ixl_update_link_status(struct ixl_pf *pf) 1864 { 1865 struct ixl_vsi *vsi = &pf->vsi; 1866 struct i40e_hw *hw = &pf->hw; 1867 struct ifnet *ifp = vsi->ifp; 1868 device_t dev = pf->dev; 1869 1870 if (pf->link_up){ 1871 if (vsi->link_active == FALSE) { 1872 pf->fc = hw->fc.current_mode; 1873 if (bootverbose) { 1874 device_printf(dev,"Link is up %d Gbps %s," 1875 " Flow Control: %s\n", 1876 ((pf->link_speed == 1877 I40E_LINK_SPEED_40GB)? 40:10), 1878 "Full Duplex", ixl_fc_string[pf->fc]); 1879 } 1880 vsi->link_active = TRUE; 1881 /* 1882 ** Warn user if link speed on NPAR enabled 1883 ** partition is not at least 10GB 1884 */ 1885 if (hw->func_caps.npar_enable && 1886 (hw->phy.link_info.link_speed == 1887 I40E_LINK_SPEED_1GB || 1888 hw->phy.link_info.link_speed == 1889 I40E_LINK_SPEED_100MB)) 1890 device_printf(dev, "The partition detected" 1891 "link speed that is less than 10Gbps\n"); 1892 if_link_state_change(ifp, LINK_STATE_UP); 1893 } 1894 } else { /* Link down */ 1895 if (vsi->link_active == TRUE) { 1896 if (bootverbose) 1897 device_printf(dev,"Link is Down\n"); 1898 if_link_state_change(ifp, LINK_STATE_DOWN); 1899 vsi->link_active = FALSE; 1900 } 1901 } 1902 1903 return; 1904 } 1905 1906 /********************************************************************* 1907 * 1908 * This routine disables all traffic on the adapter by issuing a 1909 * global reset on the MAC and deallocates TX/RX buffers. 1910 * 1911 **********************************************************************/ 1912 1913 static void 1914 ixl_stop(struct ixl_pf *pf) 1915 { 1916 struct ixl_vsi *vsi = &pf->vsi; 1917 struct ifnet *ifp = vsi->ifp; 1918 1919 mtx_assert(&pf->pf_mtx, MA_OWNED); 1920 1921 INIT_DEBUGOUT("ixl_stop: begin\n"); 1922 if (pf->num_vfs == 0) 1923 ixl_disable_intr(vsi); 1924 else 1925 ixl_disable_rings_intr(vsi); 1926 ixl_disable_rings(vsi); 1927 1928 /* Tell the stack that the interface is no longer active */ 1929 ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE); 1930 1931 /* Stop the local timer */ 1932 callout_stop(&pf->timer); 1933 1934 return; 1935 } 1936 1937 1938 /********************************************************************* 1939 * 1940 * Setup MSIX Interrupt resources and handlers for the VSI 1941 * 1942 **********************************************************************/ 1943 static int 1944 ixl_assign_vsi_legacy(struct ixl_pf *pf) 1945 { 1946 device_t dev = pf->dev; 1947 struct ixl_vsi *vsi = &pf->vsi; 1948 struct ixl_queue *que = vsi->queues; 1949 int error, rid = 0; 1950 1951 if (pf->msix == 1) 1952 rid = 1; 1953 pf->res = bus_alloc_resource_any(dev, SYS_RES_IRQ, 1954 &rid, RF_SHAREABLE | RF_ACTIVE); 1955 if (pf->res == NULL) { 1956 device_printf(dev,"Unable to allocate" 1957 " bus resource: vsi legacy/msi interrupt\n"); 1958 return (ENXIO); 1959 } 1960 1961 /* Set the handler function */ 1962 error = bus_setup_intr(dev, pf->res, 1963 INTR_TYPE_NET | INTR_MPSAFE, NULL, 1964 ixl_intr, pf, &pf->tag); 1965 if (error) { 1966 pf->res = NULL; 1967 device_printf(dev, "Failed to register legacy/msi handler"); 1968 return (error); 1969 } 1970 bus_describe_intr(dev, pf->res, pf->tag, "irq0"); 1971 TASK_INIT(&que->tx_task, 0, ixl_deferred_mq_start, que); 1972 TASK_INIT(&que->task, 0, ixl_handle_que, que); 1973 que->tq = taskqueue_create_fast("ixl_que", M_NOWAIT, 1974 taskqueue_thread_enqueue, &que->tq); 1975 taskqueue_start_threads(&que->tq, 1, PI_NET, "%s que", 1976 device_get_nameunit(dev)); 1977 TASK_INIT(&pf->adminq, 0, ixl_do_adminq, pf); 1978 1979 #ifdef PCI_IOV 1980 TASK_INIT(&pf->vflr_task, 0, ixl_handle_vflr, pf); 1981 #endif 1982 1983 pf->tq = taskqueue_create_fast("ixl_adm", M_NOWAIT, 1984 taskqueue_thread_enqueue, &pf->tq); 1985 taskqueue_start_threads(&pf->tq, 1, PI_NET, "%s adminq", 1986 device_get_nameunit(dev)); 1987 1988 return (0); 1989 } 1990 1991 static void 1992 ixl_init_taskqueues(struct ixl_pf *pf) 1993 { 1994 struct ixl_vsi *vsi = &pf->vsi; 1995 struct ixl_queue *que = vsi->queues; 1996 device_t dev = pf->dev; 1997 #ifdef RSS 1998 int cpu_id; 1999 cpuset_t cpu_mask; 2000 #endif 2001 2002 /* Tasklet for Admin Queue */ 2003 TASK_INIT(&pf->adminq, 0, ixl_do_adminq, pf); 2004 #ifdef PCI_IOV 2005 /* VFLR Tasklet */ 2006 TASK_INIT(&pf->vflr_task, 0, ixl_handle_vflr, pf); 2007 #endif 2008 2009 /* Create and start PF taskqueue */ 2010 pf->tq = taskqueue_create_fast("ixl_adm", M_NOWAIT, 2011 taskqueue_thread_enqueue, &pf->tq); 2012 taskqueue_start_threads(&pf->tq, 1, PI_NET, "%s adminq", 2013 device_get_nameunit(dev)); 2014 2015 /* Create queue tasks and start queue taskqueues */ 2016 for (int i = 0; i < vsi->num_queues; i++, que++) { 2017 TASK_INIT(&que->tx_task, 0, ixl_deferred_mq_start, que); 2018 TASK_INIT(&que->task, 0, ixl_handle_que, que); 2019 que->tq = taskqueue_create_fast("ixl_que", M_NOWAIT, 2020 taskqueue_thread_enqueue, &que->tq); 2021 #ifdef RSS 2022 cpu_id = rss_getcpu(i % rss_getnumbuckets()); 2023 CPU_SETOF(cpu_id, &cpu_mask); 2024 taskqueue_start_threads_cpuset(&que->tq, 1, PI_NET, 2025 &cpu_mask, "%s (bucket %d)", 2026 device_get_nameunit(dev), cpu_id); 2027 #else 2028 taskqueue_start_threads(&que->tq, 1, PI_NET, 2029 "%s (que %d)", device_get_nameunit(dev), que->me); 2030 #endif 2031 } 2032 } 2033 2034 static void 2035 ixl_free_taskqueues(struct ixl_pf *pf) 2036 { 2037 struct ixl_vsi *vsi = &pf->vsi; 2038 struct ixl_queue *que = vsi->queues; 2039 2040 if (pf->tq) 2041 taskqueue_free(pf->tq); 2042 for (int i = 0; i < vsi->num_queues; i++, que++) { 2043 if (que->tq) 2044 taskqueue_free(que->tq); 2045 } 2046 } 2047 2048 /********************************************************************* 2049 * 2050 * Setup MSIX Interrupt resources and handlers for the VSI 2051 * 2052 **********************************************************************/ 2053 static int 2054 ixl_assign_vsi_msix(struct ixl_pf *pf) 2055 { 2056 device_t dev = pf->dev; 2057 struct ixl_vsi *vsi = &pf->vsi; 2058 struct ixl_queue *que = vsi->queues; 2059 struct tx_ring *txr; 2060 int error, rid, vector = 0; 2061 2062 /* Admin Que is vector 0*/ 2063 rid = vector + 1; 2064 pf->res = bus_alloc_resource_any(dev, 2065 SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE); 2066 if (!pf->res) { 2067 device_printf(dev,"Unable to allocate" 2068 " bus resource: Adminq interrupt [%d]\n", rid); 2069 return (ENXIO); 2070 } 2071 /* Set the adminq vector and handler */ 2072 error = bus_setup_intr(dev, pf->res, 2073 INTR_TYPE_NET | INTR_MPSAFE, NULL, 2074 ixl_msix_adminq, pf, &pf->tag); 2075 if (error) { 2076 pf->res = NULL; 2077 device_printf(dev, "Failed to register Admin que handler"); 2078 return (error); 2079 } 2080 bus_describe_intr(dev, pf->res, pf->tag, "aq"); 2081 pf->admvec = vector; 2082 ++vector; 2083 2084 /* Now set up the stations */ 2085 for (int i = 0; i < vsi->num_queues; i++, vector++, que++) { 2086 int cpu_id = i; 2087 rid = vector + 1; 2088 txr = &que->txr; 2089 que->res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid, 2090 RF_SHAREABLE | RF_ACTIVE); 2091 if (que->res == NULL) { 2092 device_printf(dev,"Unable to allocate" 2093 " bus resource: que interrupt [%d]\n", vector); 2094 return (ENXIO); 2095 } 2096 /* Set the handler function */ 2097 error = bus_setup_intr(dev, que->res, 2098 INTR_TYPE_NET | INTR_MPSAFE, NULL, 2099 ixl_msix_que, que, &que->tag); 2100 if (error) { 2101 que->res = NULL; 2102 device_printf(dev, "Failed to register que handler"); 2103 return (error); 2104 } 2105 bus_describe_intr(dev, que->res, que->tag, "q%d", i); 2106 /* Bind the vector to a CPU */ 2107 #ifdef RSS 2108 cpu_id = rss_getcpu(i % rss_getnumbuckets()); 2109 #endif 2110 bus_bind_intr(dev, que->res, cpu_id); 2111 que->msix = vector; 2112 } 2113 2114 return (0); 2115 } 2116 2117 2118 /* 2119 * Allocate MSI/X vectors 2120 */ 2121 static int 2122 ixl_init_msix(struct ixl_pf *pf) 2123 { 2124 device_t dev = pf->dev; 2125 int rid, want, vectors, queues, available; 2126 2127 /* Override by tuneable */ 2128 if (ixl_enable_msix == 0) 2129 goto msi; 2130 2131 /* 2132 ** When used in a virtualized environment 2133 ** PCI BUSMASTER capability may not be set 2134 ** so explicity set it here and rewrite 2135 ** the ENABLE in the MSIX control register 2136 ** at this point to cause the host to 2137 ** successfully initialize us. 2138 */ 2139 { 2140 u16 pci_cmd_word; 2141 int msix_ctrl; 2142 pci_cmd_word = pci_read_config(dev, PCIR_COMMAND, 2); 2143 pci_cmd_word |= PCIM_CMD_BUSMASTEREN; 2144 pci_write_config(dev, PCIR_COMMAND, pci_cmd_word, 2); 2145 pci_find_cap(dev, PCIY_MSIX, &rid); 2146 rid += PCIR_MSIX_CTRL; 2147 msix_ctrl = pci_read_config(dev, rid, 2); 2148 msix_ctrl |= PCIM_MSIXCTRL_MSIX_ENABLE; 2149 pci_write_config(dev, rid, msix_ctrl, 2); 2150 } 2151 2152 /* First try MSI/X */ 2153 rid = PCIR_BAR(IXL_BAR); 2154 pf->msix_mem = bus_alloc_resource_any(dev, 2155 SYS_RES_MEMORY, &rid, RF_ACTIVE); 2156 if (!pf->msix_mem) { 2157 /* May not be enabled */ 2158 device_printf(pf->dev, 2159 "Unable to map MSIX table \n"); 2160 goto msi; 2161 } 2162 2163 available = pci_msix_count(dev); 2164 if (available == 0) { /* system has msix disabled */ 2165 bus_release_resource(dev, SYS_RES_MEMORY, 2166 rid, pf->msix_mem); 2167 pf->msix_mem = NULL; 2168 goto msi; 2169 } 2170 2171 /* Figure out a reasonable auto config value */ 2172 queues = (mp_ncpus > (available - 1)) ? (available - 1) : mp_ncpus; 2173 2174 /* Override with hardcoded value if it's less than autoconfig count */ 2175 if ((ixl_max_queues != 0) && (ixl_max_queues <= queues)) 2176 queues = ixl_max_queues; 2177 else if ((ixl_max_queues != 0) && (ixl_max_queues > queues)) 2178 device_printf(dev, "ixl_max_queues > # of cpus, using " 2179 "autoconfig amount...\n"); 2180 /* Or limit maximum auto-configured queues to 8 */ 2181 else if ((ixl_max_queues == 0) && (queues > 8)) 2182 queues = 8; 2183 2184 #ifdef RSS 2185 /* If we're doing RSS, clamp at the number of RSS buckets */ 2186 if (queues > rss_getnumbuckets()) 2187 queues = rss_getnumbuckets(); 2188 #endif 2189 2190 /* 2191 ** Want one vector (RX/TX pair) per queue 2192 ** plus an additional for the admin queue. 2193 */ 2194 want = queues + 1; 2195 if (want <= available) /* Have enough */ 2196 vectors = want; 2197 else { 2198 device_printf(pf->dev, 2199 "MSIX Configuration Problem, " 2200 "%d vectors available but %d wanted!\n", 2201 available, want); 2202 return (0); /* Will go to Legacy setup */ 2203 } 2204 2205 if (pci_alloc_msix(dev, &vectors) == 0) { 2206 device_printf(pf->dev, 2207 "Using MSIX interrupts with %d vectors\n", vectors); 2208 pf->msix = vectors; 2209 pf->vsi.num_queues = queues; 2210 #ifdef RSS 2211 /* 2212 * If we're doing RSS, the number of queues needs to 2213 * match the number of RSS buckets that are configured. 2214 * 2215 * + If there's more queues than RSS buckets, we'll end 2216 * up with queues that get no traffic. 2217 * 2218 * + If there's more RSS buckets than queues, we'll end 2219 * up having multiple RSS buckets map to the same queue, 2220 * so there'll be some contention. 2221 */ 2222 if (queues != rss_getnumbuckets()) { 2223 device_printf(dev, 2224 "%s: queues (%d) != RSS buckets (%d)" 2225 "; performance will be impacted.\n", 2226 __func__, queues, rss_getnumbuckets()); 2227 } 2228 #endif 2229 return (vectors); 2230 } 2231 msi: 2232 vectors = pci_msi_count(dev); 2233 pf->vsi.num_queues = 1; 2234 pf->msix = 1; 2235 ixl_max_queues = 1; 2236 ixl_enable_msix = 0; 2237 if (vectors == 1 && pci_alloc_msi(dev, &vectors) == 0) 2238 device_printf(pf->dev,"Using an MSI interrupt\n"); 2239 else { 2240 pf->msix = 0; 2241 device_printf(pf->dev,"Using a Legacy interrupt\n"); 2242 } 2243 return (vectors); 2244 } 2245 2246 2247 /* 2248 * Plumb MSI/X vectors 2249 */ 2250 static void 2251 ixl_configure_msix(struct ixl_pf *pf) 2252 { 2253 struct i40e_hw *hw = &pf->hw; 2254 struct ixl_vsi *vsi = &pf->vsi; 2255 u32 reg; 2256 u16 vector = 1; 2257 2258 /* First set up the adminq - vector 0 */ 2259 wr32(hw, I40E_PFINT_ICR0_ENA, 0); /* disable all */ 2260 rd32(hw, I40E_PFINT_ICR0); /* read to clear */ 2261 2262 reg = I40E_PFINT_ICR0_ENA_ECC_ERR_MASK | 2263 I40E_PFINT_ICR0_ENA_GRST_MASK | 2264 I40E_PFINT_ICR0_HMC_ERR_MASK | 2265 I40E_PFINT_ICR0_ENA_ADMINQ_MASK | 2266 I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK | 2267 I40E_PFINT_ICR0_ENA_VFLR_MASK | 2268 I40E_PFINT_ICR0_ENA_PCI_EXCEPTION_MASK; 2269 wr32(hw, I40E_PFINT_ICR0_ENA, reg); 2270 2271 wr32(hw, I40E_PFINT_LNKLST0, 0x7FF); 2272 wr32(hw, I40E_PFINT_ITR0(IXL_RX_ITR), 0x003E); 2273 2274 wr32(hw, I40E_PFINT_DYN_CTL0, 2275 I40E_PFINT_DYN_CTL0_SW_ITR_INDX_MASK | 2276 I40E_PFINT_DYN_CTL0_INTENA_MSK_MASK); 2277 2278 wr32(hw, I40E_PFINT_STAT_CTL0, 0); 2279 2280 /* Next configure the queues */ 2281 for (int i = 0; i < vsi->num_queues; i++, vector++) { 2282 wr32(hw, I40E_PFINT_DYN_CTLN(i), 0); 2283 /* First queue type is RX / type 0 */ 2284 wr32(hw, I40E_PFINT_LNKLSTN(i), i); 2285 2286 reg = I40E_QINT_RQCTL_CAUSE_ENA_MASK | 2287 (IXL_RX_ITR << I40E_QINT_RQCTL_ITR_INDX_SHIFT) | 2288 (vector << I40E_QINT_RQCTL_MSIX_INDX_SHIFT) | 2289 (i << I40E_QINT_RQCTL_NEXTQ_INDX_SHIFT) | 2290 (I40E_QUEUE_TYPE_TX << I40E_QINT_RQCTL_NEXTQ_TYPE_SHIFT); 2291 wr32(hw, I40E_QINT_RQCTL(i), reg); 2292 2293 reg = I40E_QINT_TQCTL_CAUSE_ENA_MASK | 2294 (IXL_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT) | 2295 (vector << I40E_QINT_TQCTL_MSIX_INDX_SHIFT) | 2296 (IXL_QUEUE_EOL << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT) | 2297 (I40E_QUEUE_TYPE_RX << I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT); 2298 wr32(hw, I40E_QINT_TQCTL(i), reg); 2299 } 2300 } 2301 2302 /* 2303 * Configure for MSI single vector operation 2304 */ 2305 static void 2306 ixl_configure_legacy(struct ixl_pf *pf) 2307 { 2308 struct i40e_hw *hw = &pf->hw; 2309 u32 reg; 2310 2311 2312 wr32(hw, I40E_PFINT_ITR0(0), 0); 2313 wr32(hw, I40E_PFINT_ITR0(1), 0); 2314 2315 2316 /* Setup "other" causes */ 2317 reg = I40E_PFINT_ICR0_ENA_ECC_ERR_MASK 2318 | I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK 2319 | I40E_PFINT_ICR0_ENA_GRST_MASK 2320 | I40E_PFINT_ICR0_ENA_PCI_EXCEPTION_MASK 2321 | I40E_PFINT_ICR0_ENA_GPIO_MASK 2322 | I40E_PFINT_ICR0_ENA_LINK_STAT_CHANGE_MASK 2323 | I40E_PFINT_ICR0_ENA_HMC_ERR_MASK 2324 | I40E_PFINT_ICR0_ENA_PE_CRITERR_MASK 2325 | I40E_PFINT_ICR0_ENA_VFLR_MASK 2326 | I40E_PFINT_ICR0_ENA_ADMINQ_MASK 2327 ; 2328 wr32(hw, I40E_PFINT_ICR0_ENA, reg); 2329 2330 /* SW_ITR_IDX = 0, but don't change INTENA */ 2331 wr32(hw, I40E_PFINT_DYN_CTL0, 2332 I40E_PFINT_DYN_CTLN_SW_ITR_INDX_MASK | 2333 I40E_PFINT_DYN_CTLN_INTENA_MSK_MASK); 2334 /* SW_ITR_IDX = 0, OTHER_ITR_IDX = 0 */ 2335 wr32(hw, I40E_PFINT_STAT_CTL0, 0); 2336 2337 /* FIRSTQ_INDX = 0, FIRSTQ_TYPE = 0 (rx) */ 2338 wr32(hw, I40E_PFINT_LNKLST0, 0); 2339 2340 /* Associate the queue pair to the vector and enable the q int */ 2341 reg = I40E_QINT_RQCTL_CAUSE_ENA_MASK 2342 | (IXL_RX_ITR << I40E_QINT_RQCTL_ITR_INDX_SHIFT) 2343 | (I40E_QUEUE_TYPE_TX << I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT); 2344 wr32(hw, I40E_QINT_RQCTL(0), reg); 2345 2346 reg = I40E_QINT_TQCTL_CAUSE_ENA_MASK 2347 | (IXL_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT) 2348 | (IXL_QUEUE_EOL << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT); 2349 wr32(hw, I40E_QINT_TQCTL(0), reg); 2350 2351 /* Next enable the queue pair */ 2352 reg = rd32(hw, I40E_QTX_ENA(0)); 2353 reg |= I40E_QTX_ENA_QENA_REQ_MASK; 2354 wr32(hw, I40E_QTX_ENA(0), reg); 2355 2356 reg = rd32(hw, I40E_QRX_ENA(0)); 2357 reg |= I40E_QRX_ENA_QENA_REQ_MASK; 2358 wr32(hw, I40E_QRX_ENA(0), reg); 2359 } 2360 2361 2362 /* 2363 * Set the Initial ITR state 2364 */ 2365 static void 2366 ixl_configure_itr(struct ixl_pf *pf) 2367 { 2368 struct i40e_hw *hw = &pf->hw; 2369 struct ixl_vsi *vsi = &pf->vsi; 2370 struct ixl_queue *que = vsi->queues; 2371 2372 vsi->rx_itr_setting = ixl_rx_itr; 2373 if (ixl_dynamic_rx_itr) 2374 vsi->rx_itr_setting |= IXL_ITR_DYNAMIC; 2375 vsi->tx_itr_setting = ixl_tx_itr; 2376 if (ixl_dynamic_tx_itr) 2377 vsi->tx_itr_setting |= IXL_ITR_DYNAMIC; 2378 2379 for (int i = 0; i < vsi->num_queues; i++, que++) { 2380 struct tx_ring *txr = &que->txr; 2381 struct rx_ring *rxr = &que->rxr; 2382 2383 wr32(hw, I40E_PFINT_ITRN(IXL_RX_ITR, i), 2384 vsi->rx_itr_setting); 2385 rxr->itr = vsi->rx_itr_setting; 2386 rxr->latency = IXL_AVE_LATENCY; 2387 wr32(hw, I40E_PFINT_ITRN(IXL_TX_ITR, i), 2388 vsi->tx_itr_setting); 2389 txr->itr = vsi->tx_itr_setting; 2390 txr->latency = IXL_AVE_LATENCY; 2391 } 2392 } 2393 2394 2395 static int 2396 ixl_allocate_pci_resources(struct ixl_pf *pf) 2397 { 2398 int rid; 2399 device_t dev = pf->dev; 2400 2401 rid = PCIR_BAR(0); 2402 pf->pci_mem = bus_alloc_resource_any(dev, SYS_RES_MEMORY, 2403 &rid, RF_ACTIVE); 2404 2405 if (!(pf->pci_mem)) { 2406 device_printf(dev,"Unable to allocate bus resource: memory\n"); 2407 return (ENXIO); 2408 } 2409 2410 pf->osdep.mem_bus_space_tag = 2411 rman_get_bustag(pf->pci_mem); 2412 pf->osdep.mem_bus_space_handle = 2413 rman_get_bushandle(pf->pci_mem); 2414 pf->osdep.mem_bus_space_size = rman_get_size(pf->pci_mem); 2415 pf->osdep.flush_reg = I40E_GLGEN_STAT; 2416 pf->hw.hw_addr = (u8 *) &pf->osdep.mem_bus_space_handle; 2417 2418 pf->hw.back = &pf->osdep; 2419 2420 /* 2421 ** Now setup MSI or MSI/X, should 2422 ** return us the number of supported 2423 ** vectors. (Will be 1 for MSI) 2424 */ 2425 pf->msix = ixl_init_msix(pf); 2426 return (0); 2427 } 2428 2429 static void 2430 ixl_free_pci_resources(struct ixl_pf * pf) 2431 { 2432 struct ixl_vsi *vsi = &pf->vsi; 2433 struct ixl_queue *que = vsi->queues; 2434 device_t dev = pf->dev; 2435 int rid, memrid; 2436 2437 memrid = PCIR_BAR(IXL_BAR); 2438 2439 /* We may get here before stations are setup */ 2440 if ((!ixl_enable_msix) || (que == NULL)) 2441 goto early; 2442 2443 /* 2444 ** Release all msix VSI resources: 2445 */ 2446 for (int i = 0; i < vsi->num_queues; i++, que++) { 2447 rid = que->msix + 1; 2448 if (que->tag != NULL) { 2449 bus_teardown_intr(dev, que->res, que->tag); 2450 que->tag = NULL; 2451 } 2452 if (que->res != NULL) 2453 bus_release_resource(dev, SYS_RES_IRQ, rid, que->res); 2454 } 2455 2456 early: 2457 /* Clean the AdminQ interrupt last */ 2458 if (pf->admvec) /* we are doing MSIX */ 2459 rid = pf->admvec + 1; 2460 else 2461 (pf->msix != 0) ? (rid = 1):(rid = 0); 2462 2463 if (pf->tag != NULL) { 2464 bus_teardown_intr(dev, pf->res, pf->tag); 2465 pf->tag = NULL; 2466 } 2467 if (pf->res != NULL) 2468 bus_release_resource(dev, SYS_RES_IRQ, rid, pf->res); 2469 2470 if (pf->msix) 2471 pci_release_msi(dev); 2472 2473 if (pf->msix_mem != NULL) 2474 bus_release_resource(dev, SYS_RES_MEMORY, 2475 memrid, pf->msix_mem); 2476 2477 if (pf->pci_mem != NULL) 2478 bus_release_resource(dev, SYS_RES_MEMORY, 2479 PCIR_BAR(0), pf->pci_mem); 2480 2481 return; 2482 } 2483 2484 static void 2485 ixl_add_ifmedia(struct ixl_vsi *vsi, u32 phy_type) 2486 { 2487 /* Display supported media types */ 2488 if (phy_type & (1 << I40E_PHY_TYPE_100BASE_TX)) 2489 ifmedia_add(&vsi->media, IFM_ETHER | IFM_100_TX, 0, NULL); 2490 2491 if (phy_type & (1 << I40E_PHY_TYPE_1000BASE_T)) 2492 ifmedia_add(&vsi->media, IFM_ETHER | IFM_1000_T, 0, NULL); 2493 if (phy_type & (1 << I40E_PHY_TYPE_1000BASE_SX)) 2494 ifmedia_add(&vsi->media, IFM_ETHER | IFM_1000_SX, 0, NULL); 2495 if (phy_type & (1 << I40E_PHY_TYPE_1000BASE_LX)) 2496 ifmedia_add(&vsi->media, IFM_ETHER | IFM_1000_LX, 0, NULL); 2497 2498 if (phy_type & (1 << I40E_PHY_TYPE_XAUI) || 2499 phy_type & (1 << I40E_PHY_TYPE_XFI) || 2500 phy_type & (1 << I40E_PHY_TYPE_10GBASE_SFPP_CU)) 2501 ifmedia_add(&vsi->media, IFM_ETHER | IFM_10G_TWINAX, 0, NULL); 2502 2503 if (phy_type & (1 << I40E_PHY_TYPE_10GBASE_SR)) 2504 ifmedia_add(&vsi->media, IFM_ETHER | IFM_10G_SR, 0, NULL); 2505 if (phy_type & (1 << I40E_PHY_TYPE_10GBASE_LR)) 2506 ifmedia_add(&vsi->media, IFM_ETHER | IFM_10G_LR, 0, NULL); 2507 if (phy_type & (1 << I40E_PHY_TYPE_10GBASE_T)) 2508 ifmedia_add(&vsi->media, IFM_ETHER | IFM_10G_T, 0, NULL); 2509 2510 if (phy_type & (1 << I40E_PHY_TYPE_40GBASE_CR4) || 2511 phy_type & (1 << I40E_PHY_TYPE_40GBASE_CR4_CU) || 2512 phy_type & (1 << I40E_PHY_TYPE_40GBASE_AOC) || 2513 phy_type & (1 << I40E_PHY_TYPE_XLAUI) || 2514 phy_type & (1 << I40E_PHY_TYPE_40GBASE_KR4)) 2515 ifmedia_add(&vsi->media, IFM_ETHER | IFM_40G_CR4, 0, NULL); 2516 if (phy_type & (1 << I40E_PHY_TYPE_40GBASE_SR4)) 2517 ifmedia_add(&vsi->media, IFM_ETHER | IFM_40G_SR4, 0, NULL); 2518 if (phy_type & (1 << I40E_PHY_TYPE_40GBASE_LR4)) 2519 ifmedia_add(&vsi->media, IFM_ETHER | IFM_40G_LR4, 0, NULL); 2520 2521 #ifndef IFM_ETH_XTYPE 2522 if (phy_type & (1 << I40E_PHY_TYPE_1000BASE_KX)) 2523 ifmedia_add(&vsi->media, IFM_ETHER | IFM_1000_CX, 0, NULL); 2524 2525 if (phy_type & (1 << I40E_PHY_TYPE_10GBASE_CR1_CU) || 2526 phy_type & (1 << I40E_PHY_TYPE_10GBASE_CR1) || 2527 phy_type & (1 << I40E_PHY_TYPE_10GBASE_AOC) || 2528 phy_type & (1 << I40E_PHY_TYPE_SFI)) 2529 ifmedia_add(&vsi->media, IFM_ETHER | IFM_10G_TWINAX, 0, NULL); 2530 if (phy_type & (1 << I40E_PHY_TYPE_10GBASE_KX4)) 2531 ifmedia_add(&vsi->media, IFM_ETHER | IFM_10G_CX4, 0, NULL); 2532 if (phy_type & (1 << I40E_PHY_TYPE_10GBASE_KR)) 2533 ifmedia_add(&vsi->media, IFM_ETHER | IFM_10G_SR, 0, NULL); 2534 2535 if (phy_type & (1 << I40E_PHY_TYPE_40GBASE_KR4)) 2536 ifmedia_add(&vsi->media, IFM_ETHER | IFM_40G_SR4, 0, NULL); 2537 if (phy_type & (1 << I40E_PHY_TYPE_XLPPI)) 2538 ifmedia_add(&vsi->media, IFM_ETHER | IFM_40G_CR4, 0, NULL); 2539 #else 2540 if (phy_type & (1 << I40E_PHY_TYPE_1000BASE_KX)) 2541 ifmedia_add(&vsi->media, IFM_ETHER | IFM_1000_KX, 0, NULL); 2542 2543 if (phy_type & (1 << I40E_PHY_TYPE_10GBASE_CR1_CU) 2544 || phy_type & (1 << I40E_PHY_TYPE_10GBASE_CR1)) 2545 ifmedia_add(&vsi->media, IFM_ETHER | IFM_10G_CR1, 0, NULL); 2546 if (phy_type & (1 << I40E_PHY_TYPE_10GBASE_AOC)) 2547 ifmedia_add(&vsi->media, IFM_ETHER | IFM_10G_TWINAX_LONG, 0, NULL); 2548 if (phy_type & (1 << I40E_PHY_TYPE_SFI)) 2549 ifmedia_add(&vsi->media, IFM_ETHER | IFM_10G_SFI, 0, NULL); 2550 if (phy_type & (1 << I40E_PHY_TYPE_10GBASE_KX4)) 2551 ifmedia_add(&vsi->media, IFM_ETHER | IFM_10G_KX4, 0, NULL); 2552 if (phy_type & (1 << I40E_PHY_TYPE_10GBASE_KR)) 2553 ifmedia_add(&vsi->media, IFM_ETHER | IFM_10G_KR, 0, NULL); 2554 2555 if (phy_type & (1 << I40E_PHY_TYPE_20GBASE_KR2)) 2556 ifmedia_add(&vsi->media, IFM_ETHER | IFM_20G_KR2, 0, NULL); 2557 2558 if (phy_type & (1 << I40E_PHY_TYPE_40GBASE_KR4)) 2559 ifmedia_add(&vsi->media, IFM_ETHER | IFM_40G_KR4, 0, NULL); 2560 if (phy_type & (1 << I40E_PHY_TYPE_XLPPI)) 2561 ifmedia_add(&vsi->media, IFM_ETHER | IFM_40G_XLPPI, 0, NULL); 2562 #endif 2563 } 2564 2565 /********************************************************************* 2566 * 2567 * Setup networking device structure and register an interface. 2568 * 2569 **********************************************************************/ 2570 static int 2571 ixl_setup_interface(device_t dev, struct ixl_vsi *vsi) 2572 { 2573 struct ifnet *ifp; 2574 struct i40e_hw *hw = vsi->hw; 2575 struct ixl_queue *que = vsi->queues; 2576 struct i40e_aq_get_phy_abilities_resp abilities; 2577 enum i40e_status_code aq_error = 0; 2578 2579 INIT_DEBUGOUT("ixl_setup_interface: begin"); 2580 2581 ifp = vsi->ifp = if_alloc(IFT_ETHER); 2582 if (ifp == NULL) { 2583 device_printf(dev, "can not allocate ifnet structure\n"); 2584 return (-1); 2585 } 2586 if_initname(ifp, device_get_name(dev), device_get_unit(dev)); 2587 ifp->if_mtu = ETHERMTU; 2588 ifp->if_baudrate = IF_Gbps(40); 2589 ifp->if_init = ixl_init; 2590 ifp->if_softc = vsi; 2591 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 2592 ifp->if_ioctl = ixl_ioctl; 2593 2594 #if __FreeBSD_version >= 1100036 2595 if_setgetcounterfn(ifp, ixl_get_counter); 2596 #endif 2597 2598 ifp->if_transmit = ixl_mq_start; 2599 2600 ifp->if_qflush = ixl_qflush; 2601 2602 ifp->if_snd.ifq_maxlen = que->num_desc - 2; 2603 2604 vsi->max_frame_size = 2605 ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN 2606 + ETHER_VLAN_ENCAP_LEN; 2607 2608 /* 2609 * Tell the upper layer(s) we support long frames. 2610 */ 2611 ifp->if_hdrlen = sizeof(struct ether_vlan_header); 2612 2613 ifp->if_capabilities |= IFCAP_HWCSUM; 2614 ifp->if_capabilities |= IFCAP_HWCSUM_IPV6; 2615 ifp->if_capabilities |= IFCAP_TSO; 2616 ifp->if_capabilities |= IFCAP_JUMBO_MTU; 2617 ifp->if_capabilities |= IFCAP_LRO; 2618 2619 /* VLAN capabilties */ 2620 ifp->if_capabilities |= IFCAP_VLAN_HWTAGGING 2621 | IFCAP_VLAN_HWTSO 2622 | IFCAP_VLAN_MTU 2623 | IFCAP_VLAN_HWCSUM; 2624 ifp->if_capenable = ifp->if_capabilities; 2625 2626 /* 2627 ** Don't turn this on by default, if vlans are 2628 ** created on another pseudo device (eg. lagg) 2629 ** then vlan events are not passed thru, breaking 2630 ** operation, but with HW FILTER off it works. If 2631 ** using vlans directly on the ixl driver you can 2632 ** enable this and get full hardware tag filtering. 2633 */ 2634 ifp->if_capabilities |= IFCAP_VLAN_HWFILTER; 2635 2636 /* 2637 * Specify the media types supported by this adapter and register 2638 * callbacks to update media and link information 2639 */ 2640 ifmedia_init(&vsi->media, IFM_IMASK, ixl_media_change, 2641 ixl_media_status); 2642 2643 aq_error = i40e_aq_get_phy_capabilities(hw, 2644 FALSE, TRUE, &abilities, NULL); 2645 /* May need delay to detect fiber correctly */ 2646 if (aq_error == I40E_ERR_UNKNOWN_PHY) { 2647 i40e_msec_delay(200); 2648 aq_error = i40e_aq_get_phy_capabilities(hw, FALSE, 2649 TRUE, &abilities, NULL); 2650 } 2651 if (aq_error) { 2652 if (aq_error == I40E_ERR_UNKNOWN_PHY) 2653 device_printf(dev, "Unknown PHY type detected!\n"); 2654 else 2655 device_printf(dev, 2656 "Error getting supported media types, err %d," 2657 " AQ error %d\n", aq_error, hw->aq.asq_last_status); 2658 return (0); 2659 } 2660 2661 ixl_add_ifmedia(vsi, abilities.phy_type); 2662 2663 /* Use autoselect media by default */ 2664 ifmedia_add(&vsi->media, IFM_ETHER | IFM_AUTO, 0, NULL); 2665 ifmedia_set(&vsi->media, IFM_ETHER | IFM_AUTO); 2666 2667 ether_ifattach(ifp, hw->mac.addr); 2668 2669 return (0); 2670 } 2671 2672 /* 2673 ** Run when the Admin Queue gets a 2674 ** link transition interrupt. 2675 */ 2676 static void 2677 ixl_link_event(struct ixl_pf *pf, struct i40e_arq_event_info *e) 2678 { 2679 struct i40e_hw *hw = &pf->hw; 2680 struct i40e_aqc_get_link_status *status = 2681 (struct i40e_aqc_get_link_status *)&e->desc.params.raw; 2682 bool check; 2683 2684 hw->phy.get_link_info = TRUE; 2685 i40e_get_link_status(hw, &check); 2686 pf->link_up = check; 2687 #ifdef IXL_DEBUG 2688 printf("Link is %s\n", check ? "up":"down"); 2689 #endif 2690 /* Report if Unqualified modules are found */ 2691 if ((status->link_info & I40E_AQ_MEDIA_AVAILABLE) && 2692 (!(status->an_info & I40E_AQ_QUALIFIED_MODULE)) && 2693 (!(status->link_info & I40E_AQ_LINK_UP))) 2694 device_printf(pf->dev, "Link failed because " 2695 "an unqualified module was detected\n"); 2696 2697 return; 2698 } 2699 2700 /********************************************************************* 2701 * 2702 * Get Firmware Switch configuration 2703 * - this will need to be more robust when more complex 2704 * switch configurations are enabled. 2705 * 2706 **********************************************************************/ 2707 static int 2708 ixl_switch_config(struct ixl_pf *pf) 2709 { 2710 struct i40e_hw *hw = &pf->hw; 2711 struct ixl_vsi *vsi = &pf->vsi; 2712 device_t dev = vsi->dev; 2713 struct i40e_aqc_get_switch_config_resp *sw_config; 2714 u8 aq_buf[I40E_AQ_LARGE_BUF]; 2715 int ret; 2716 u16 next = 0; 2717 2718 memset(&aq_buf, 0, sizeof(aq_buf)); 2719 sw_config = (struct i40e_aqc_get_switch_config_resp *)aq_buf; 2720 ret = i40e_aq_get_switch_config(hw, sw_config, 2721 sizeof(aq_buf), &next, NULL); 2722 if (ret) { 2723 device_printf(dev,"aq_get_switch_config failed (ret=%d)!!\n", 2724 ret); 2725 return (ret); 2726 } 2727 #ifdef IXL_DEBUG 2728 device_printf(dev, 2729 "Switch config: header reported: %d in structure, %d total\n", 2730 sw_config->header.num_reported, sw_config->header.num_total); 2731 for (int i = 0; i < sw_config->header.num_reported; i++) { 2732 device_printf(dev, 2733 "%d: type=%d seid=%d uplink=%d downlink=%d\n", i, 2734 sw_config->element[i].element_type, 2735 sw_config->element[i].seid, 2736 sw_config->element[i].uplink_seid, 2737 sw_config->element[i].downlink_seid); 2738 } 2739 #endif 2740 /* Simplified due to a single VSI at the moment */ 2741 vsi->uplink_seid = sw_config->element[0].uplink_seid; 2742 vsi->downlink_seid = sw_config->element[0].downlink_seid; 2743 vsi->seid = sw_config->element[0].seid; 2744 return (ret); 2745 } 2746 2747 /********************************************************************* 2748 * 2749 * Initialize the VSI: this handles contexts, which means things 2750 * like the number of descriptors, buffer size, 2751 * plus we init the rings thru this function. 2752 * 2753 **********************************************************************/ 2754 static int 2755 ixl_initialize_vsi(struct ixl_vsi *vsi) 2756 { 2757 struct ixl_pf *pf = vsi->back; 2758 struct ixl_queue *que = vsi->queues; 2759 device_t dev = vsi->dev; 2760 struct i40e_hw *hw = vsi->hw; 2761 struct i40e_vsi_context ctxt; 2762 int err = 0; 2763 2764 memset(&ctxt, 0, sizeof(ctxt)); 2765 ctxt.seid = vsi->seid; 2766 if (pf->veb_seid != 0) 2767 ctxt.uplink_seid = pf->veb_seid; 2768 ctxt.pf_num = hw->pf_id; 2769 err = i40e_aq_get_vsi_params(hw, &ctxt, NULL); 2770 if (err) { 2771 device_printf(dev,"get vsi params failed %x!!\n", err); 2772 return (err); 2773 } 2774 #ifdef IXL_DEBUG 2775 printf("get_vsi_params: seid: %d, uplinkseid: %d, vsi_number: %d, " 2776 "vsis_allocated: %d, vsis_unallocated: %d, flags: 0x%x, " 2777 "pfnum: %d, vfnum: %d, stat idx: %d, enabled: %d\n", ctxt.seid, 2778 ctxt.uplink_seid, ctxt.vsi_number, 2779 ctxt.vsis_allocated, ctxt.vsis_unallocated, 2780 ctxt.flags, ctxt.pf_num, ctxt.vf_num, 2781 ctxt.info.stat_counter_idx, ctxt.info.up_enable_bits); 2782 #endif 2783 /* 2784 ** Set the queue and traffic class bits 2785 ** - when multiple traffic classes are supported 2786 ** this will need to be more robust. 2787 */ 2788 ctxt.info.valid_sections = I40E_AQ_VSI_PROP_QUEUE_MAP_VALID; 2789 ctxt.info.mapping_flags |= I40E_AQ_VSI_QUE_MAP_CONTIG; 2790 /* In contig mode, que_mapping[0] is first queue index used by this VSI */ 2791 ctxt.info.queue_mapping[0] = 0; 2792 /* 2793 * This VSI will only use traffic class 0; start traffic class 0's 2794 * queue allocation at queue 0, and assign it 64 (2^6) queues (though 2795 * the driver may not use all of them). 2796 */ 2797 ctxt.info.tc_mapping[0] = ((0 << I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT) 2798 & I40E_AQ_VSI_TC_QUE_OFFSET_MASK) | 2799 ((6 << I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT) 2800 & I40E_AQ_VSI_TC_QUE_NUMBER_MASK); 2801 2802 /* Set VLAN receive stripping mode */ 2803 ctxt.info.valid_sections |= I40E_AQ_VSI_PROP_VLAN_VALID; 2804 ctxt.info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_ALL; 2805 if (vsi->ifp->if_capenable & IFCAP_VLAN_HWTAGGING) 2806 ctxt.info.port_vlan_flags |= I40E_AQ_VSI_PVLAN_EMOD_STR_BOTH; 2807 else 2808 ctxt.info.port_vlan_flags |= I40E_AQ_VSI_PVLAN_EMOD_NOTHING; 2809 2810 /* Keep copy of VSI info in VSI for statistic counters */ 2811 memcpy(&vsi->info, &ctxt.info, sizeof(ctxt.info)); 2812 2813 /* Reset VSI statistics */ 2814 ixl_vsi_reset_stats(vsi); 2815 vsi->hw_filters_add = 0; 2816 vsi->hw_filters_del = 0; 2817 2818 ctxt.flags = htole16(I40E_AQ_VSI_TYPE_PF); 2819 2820 err = i40e_aq_update_vsi_params(hw, &ctxt, NULL); 2821 if (err) { 2822 device_printf(dev,"update vsi params failed %x!!\n", 2823 hw->aq.asq_last_status); 2824 return (err); 2825 } 2826 2827 for (int i = 0; i < vsi->num_queues; i++, que++) { 2828 struct tx_ring *txr = &que->txr; 2829 struct rx_ring *rxr = &que->rxr; 2830 struct i40e_hmc_obj_txq tctx; 2831 struct i40e_hmc_obj_rxq rctx; 2832 u32 txctl; 2833 u16 size; 2834 2835 2836 /* Setup the HMC TX Context */ 2837 size = que->num_desc * sizeof(struct i40e_tx_desc); 2838 memset(&tctx, 0, sizeof(struct i40e_hmc_obj_txq)); 2839 tctx.new_context = 1; 2840 tctx.base = (txr->dma.pa/IXL_TX_CTX_BASE_UNITS); 2841 tctx.qlen = que->num_desc; 2842 tctx.fc_ena = 0; 2843 tctx.rdylist = vsi->info.qs_handle[0]; /* index is TC */ 2844 /* Enable HEAD writeback */ 2845 tctx.head_wb_ena = 1; 2846 tctx.head_wb_addr = txr->dma.pa + 2847 (que->num_desc * sizeof(struct i40e_tx_desc)); 2848 tctx.rdylist_act = 0; 2849 err = i40e_clear_lan_tx_queue_context(hw, i); 2850 if (err) { 2851 device_printf(dev, "Unable to clear TX context\n"); 2852 break; 2853 } 2854 err = i40e_set_lan_tx_queue_context(hw, i, &tctx); 2855 if (err) { 2856 device_printf(dev, "Unable to set TX context\n"); 2857 break; 2858 } 2859 /* Associate the ring with this PF */ 2860 txctl = I40E_QTX_CTL_PF_QUEUE; 2861 txctl |= ((hw->pf_id << I40E_QTX_CTL_PF_INDX_SHIFT) & 2862 I40E_QTX_CTL_PF_INDX_MASK); 2863 wr32(hw, I40E_QTX_CTL(i), txctl); 2864 ixl_flush(hw); 2865 2866 /* Do ring (re)init */ 2867 ixl_init_tx_ring(que); 2868 2869 /* Next setup the HMC RX Context */ 2870 if (vsi->max_frame_size <= MCLBYTES) 2871 rxr->mbuf_sz = MCLBYTES; 2872 else 2873 rxr->mbuf_sz = MJUMPAGESIZE; 2874 2875 u16 max_rxmax = rxr->mbuf_sz * hw->func_caps.rx_buf_chain_len; 2876 2877 /* Set up an RX context for the HMC */ 2878 memset(&rctx, 0, sizeof(struct i40e_hmc_obj_rxq)); 2879 rctx.dbuff = rxr->mbuf_sz >> I40E_RXQ_CTX_DBUFF_SHIFT; 2880 /* ignore header split for now */ 2881 rctx.hbuff = 0 >> I40E_RXQ_CTX_HBUFF_SHIFT; 2882 rctx.rxmax = (vsi->max_frame_size < max_rxmax) ? 2883 vsi->max_frame_size : max_rxmax; 2884 rctx.dtype = 0; 2885 rctx.dsize = 1; /* do 32byte descriptors */ 2886 rctx.hsplit_0 = 0; /* no HDR split initially */ 2887 rctx.base = (rxr->dma.pa/IXL_RX_CTX_BASE_UNITS); 2888 rctx.qlen = que->num_desc; 2889 rctx.tphrdesc_ena = 1; 2890 rctx.tphwdesc_ena = 1; 2891 rctx.tphdata_ena = 0; 2892 rctx.tphhead_ena = 0; 2893 rctx.lrxqthresh = 2; 2894 rctx.crcstrip = 1; 2895 rctx.l2tsel = 1; 2896 rctx.showiv = 1; 2897 rctx.fc_ena = 0; 2898 rctx.prefena = 1; 2899 2900 err = i40e_clear_lan_rx_queue_context(hw, i); 2901 if (err) { 2902 device_printf(dev, 2903 "Unable to clear RX context %d\n", i); 2904 break; 2905 } 2906 err = i40e_set_lan_rx_queue_context(hw, i, &rctx); 2907 if (err) { 2908 device_printf(dev, "Unable to set RX context %d\n", i); 2909 break; 2910 } 2911 err = ixl_init_rx_ring(que); 2912 if (err) { 2913 device_printf(dev, "Fail in init_rx_ring %d\n", i); 2914 break; 2915 } 2916 #ifdef DEV_NETMAP 2917 /* preserve queue */ 2918 if (vsi->ifp->if_capenable & IFCAP_NETMAP) { 2919 struct netmap_adapter *na = NA(vsi->ifp); 2920 struct netmap_kring *kring = &na->rx_rings[i]; 2921 int t = na->num_rx_desc - 1 - nm_kr_rxspace(kring); 2922 wr32(vsi->hw, I40E_QRX_TAIL(que->me), t); 2923 } else 2924 #endif /* DEV_NETMAP */ 2925 wr32(vsi->hw, I40E_QRX_TAIL(que->me), que->num_desc - 1); 2926 } 2927 return (err); 2928 } 2929 2930 2931 /********************************************************************* 2932 * 2933 * Free all VSI structs. 2934 * 2935 **********************************************************************/ 2936 void 2937 ixl_free_vsi(struct ixl_vsi *vsi) 2938 { 2939 struct ixl_pf *pf = (struct ixl_pf *)vsi->back; 2940 struct ixl_queue *que = vsi->queues; 2941 2942 /* Free station queues */ 2943 for (int i = 0; i < vsi->num_queues; i++, que++) { 2944 struct tx_ring *txr = &que->txr; 2945 struct rx_ring *rxr = &que->rxr; 2946 2947 if (!mtx_initialized(&txr->mtx)) /* uninitialized */ 2948 continue; 2949 IXL_TX_LOCK(txr); 2950 ixl_free_que_tx(que); 2951 if (txr->base) 2952 i40e_free_dma_mem(&pf->hw, &txr->dma); 2953 IXL_TX_UNLOCK(txr); 2954 IXL_TX_LOCK_DESTROY(txr); 2955 2956 if (!mtx_initialized(&rxr->mtx)) /* uninitialized */ 2957 continue; 2958 IXL_RX_LOCK(rxr); 2959 ixl_free_que_rx(que); 2960 if (rxr->base) 2961 i40e_free_dma_mem(&pf->hw, &rxr->dma); 2962 IXL_RX_UNLOCK(rxr); 2963 IXL_RX_LOCK_DESTROY(rxr); 2964 2965 } 2966 free(vsi->queues, M_DEVBUF); 2967 2968 /* Free VSI filter list */ 2969 ixl_free_mac_filters(vsi); 2970 } 2971 2972 static void 2973 ixl_free_mac_filters(struct ixl_vsi *vsi) 2974 { 2975 struct ixl_mac_filter *f; 2976 2977 while (!SLIST_EMPTY(&vsi->ftl)) { 2978 f = SLIST_FIRST(&vsi->ftl); 2979 SLIST_REMOVE_HEAD(&vsi->ftl, next); 2980 free(f, M_DEVBUF); 2981 } 2982 } 2983 2984 2985 /********************************************************************* 2986 * 2987 * Allocate memory for the VSI (virtual station interface) and their 2988 * associated queues, rings and the descriptors associated with each, 2989 * called only once at attach. 2990 * 2991 **********************************************************************/ 2992 static int 2993 ixl_setup_stations(struct ixl_pf *pf) 2994 { 2995 device_t dev = pf->dev; 2996 struct ixl_vsi *vsi; 2997 struct ixl_queue *que; 2998 struct tx_ring *txr; 2999 struct rx_ring *rxr; 3000 int rsize, tsize; 3001 int error = I40E_SUCCESS; 3002 3003 vsi = &pf->vsi; 3004 vsi->back = (void *)pf; 3005 vsi->hw = &pf->hw; 3006 vsi->id = 0; 3007 vsi->num_vlans = 0; 3008 vsi->back = pf; 3009 3010 /* Get memory for the station queues */ 3011 if (!(vsi->queues = 3012 (struct ixl_queue *) malloc(sizeof(struct ixl_queue) * 3013 vsi->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) { 3014 device_printf(dev, "Unable to allocate queue memory\n"); 3015 error = ENOMEM; 3016 goto early; 3017 } 3018 3019 for (int i = 0; i < vsi->num_queues; i++) { 3020 que = &vsi->queues[i]; 3021 que->num_desc = ixl_ringsz; 3022 que->me = i; 3023 que->vsi = vsi; 3024 /* mark the queue as active */ 3025 vsi->active_queues |= (u64)1 << que->me; 3026 txr = &que->txr; 3027 txr->que = que; 3028 txr->tail = I40E_QTX_TAIL(que->me); 3029 3030 /* Initialize the TX lock */ 3031 snprintf(txr->mtx_name, sizeof(txr->mtx_name), "%s:tx(%d)", 3032 device_get_nameunit(dev), que->me); 3033 mtx_init(&txr->mtx, txr->mtx_name, NULL, MTX_DEF); 3034 /* Create the TX descriptor ring */ 3035 tsize = roundup2((que->num_desc * 3036 sizeof(struct i40e_tx_desc)) + 3037 sizeof(u32), DBA_ALIGN); 3038 if (i40e_allocate_dma_mem(&pf->hw, 3039 &txr->dma, i40e_mem_reserved, tsize, DBA_ALIGN)) { 3040 device_printf(dev, 3041 "Unable to allocate TX Descriptor memory\n"); 3042 error = ENOMEM; 3043 goto fail; 3044 } 3045 txr->base = (struct i40e_tx_desc *)txr->dma.va; 3046 bzero((void *)txr->base, tsize); 3047 /* Now allocate transmit soft structs for the ring */ 3048 if (ixl_allocate_tx_data(que)) { 3049 device_printf(dev, 3050 "Critical Failure setting up TX structures\n"); 3051 error = ENOMEM; 3052 goto fail; 3053 } 3054 /* Allocate a buf ring */ 3055 txr->br = buf_ring_alloc(4096, M_DEVBUF, 3056 M_WAITOK, &txr->mtx); 3057 if (txr->br == NULL) { 3058 device_printf(dev, 3059 "Critical Failure setting up TX buf ring\n"); 3060 error = ENOMEM; 3061 goto fail; 3062 } 3063 3064 /* 3065 * Next the RX queues... 3066 */ 3067 rsize = roundup2(que->num_desc * 3068 sizeof(union i40e_rx_desc), DBA_ALIGN); 3069 rxr = &que->rxr; 3070 rxr->que = que; 3071 rxr->tail = I40E_QRX_TAIL(que->me); 3072 3073 /* Initialize the RX side lock */ 3074 snprintf(rxr->mtx_name, sizeof(rxr->mtx_name), "%s:rx(%d)", 3075 device_get_nameunit(dev), que->me); 3076 mtx_init(&rxr->mtx, rxr->mtx_name, NULL, MTX_DEF); 3077 3078 if (i40e_allocate_dma_mem(&pf->hw, 3079 &rxr->dma, i40e_mem_reserved, rsize, 4096)) { 3080 device_printf(dev, 3081 "Unable to allocate RX Descriptor memory\n"); 3082 error = ENOMEM; 3083 goto fail; 3084 } 3085 rxr->base = (union i40e_rx_desc *)rxr->dma.va; 3086 bzero((void *)rxr->base, rsize); 3087 3088 /* Allocate receive soft structs for the ring*/ 3089 if (ixl_allocate_rx_data(que)) { 3090 device_printf(dev, 3091 "Critical Failure setting up receive structs\n"); 3092 error = ENOMEM; 3093 goto fail; 3094 } 3095 } 3096 3097 return (0); 3098 3099 fail: 3100 for (int i = 0; i < vsi->num_queues; i++) { 3101 que = &vsi->queues[i]; 3102 rxr = &que->rxr; 3103 txr = &que->txr; 3104 if (rxr->base) 3105 i40e_free_dma_mem(&pf->hw, &rxr->dma); 3106 if (txr->base) 3107 i40e_free_dma_mem(&pf->hw, &txr->dma); 3108 } 3109 3110 early: 3111 return (error); 3112 } 3113 3114 /* 3115 ** Provide a update to the queue RX 3116 ** interrupt moderation value. 3117 */ 3118 static void 3119 ixl_set_queue_rx_itr(struct ixl_queue *que) 3120 { 3121 struct ixl_vsi *vsi = que->vsi; 3122 struct i40e_hw *hw = vsi->hw; 3123 struct rx_ring *rxr = &que->rxr; 3124 u16 rx_itr; 3125 u16 rx_latency = 0; 3126 int rx_bytes; 3127 3128 3129 /* Idle, do nothing */ 3130 if (rxr->bytes == 0) 3131 return; 3132 3133 if (ixl_dynamic_rx_itr) { 3134 rx_bytes = rxr->bytes/rxr->itr; 3135 rx_itr = rxr->itr; 3136 3137 /* Adjust latency range */ 3138 switch (rxr->latency) { 3139 case IXL_LOW_LATENCY: 3140 if (rx_bytes > 10) { 3141 rx_latency = IXL_AVE_LATENCY; 3142 rx_itr = IXL_ITR_20K; 3143 } 3144 break; 3145 case IXL_AVE_LATENCY: 3146 if (rx_bytes > 20) { 3147 rx_latency = IXL_BULK_LATENCY; 3148 rx_itr = IXL_ITR_8K; 3149 } else if (rx_bytes <= 10) { 3150 rx_latency = IXL_LOW_LATENCY; 3151 rx_itr = IXL_ITR_100K; 3152 } 3153 break; 3154 case IXL_BULK_LATENCY: 3155 if (rx_bytes <= 20) { 3156 rx_latency = IXL_AVE_LATENCY; 3157 rx_itr = IXL_ITR_20K; 3158 } 3159 break; 3160 } 3161 3162 rxr->latency = rx_latency; 3163 3164 if (rx_itr != rxr->itr) { 3165 /* do an exponential smoothing */ 3166 rx_itr = (10 * rx_itr * rxr->itr) / 3167 ((9 * rx_itr) + rxr->itr); 3168 rxr->itr = rx_itr & IXL_MAX_ITR; 3169 wr32(hw, I40E_PFINT_ITRN(IXL_RX_ITR, 3170 que->me), rxr->itr); 3171 } 3172 } else { /* We may have have toggled to non-dynamic */ 3173 if (vsi->rx_itr_setting & IXL_ITR_DYNAMIC) 3174 vsi->rx_itr_setting = ixl_rx_itr; 3175 /* Update the hardware if needed */ 3176 if (rxr->itr != vsi->rx_itr_setting) { 3177 rxr->itr = vsi->rx_itr_setting; 3178 wr32(hw, I40E_PFINT_ITRN(IXL_RX_ITR, 3179 que->me), rxr->itr); 3180 } 3181 } 3182 rxr->bytes = 0; 3183 rxr->packets = 0; 3184 return; 3185 } 3186 3187 3188 /* 3189 ** Provide a update to the queue TX 3190 ** interrupt moderation value. 3191 */ 3192 static void 3193 ixl_set_queue_tx_itr(struct ixl_queue *que) 3194 { 3195 struct ixl_vsi *vsi = que->vsi; 3196 struct i40e_hw *hw = vsi->hw; 3197 struct tx_ring *txr = &que->txr; 3198 u16 tx_itr; 3199 u16 tx_latency = 0; 3200 int tx_bytes; 3201 3202 3203 /* Idle, do nothing */ 3204 if (txr->bytes == 0) 3205 return; 3206 3207 if (ixl_dynamic_tx_itr) { 3208 tx_bytes = txr->bytes/txr->itr; 3209 tx_itr = txr->itr; 3210 3211 switch (txr->latency) { 3212 case IXL_LOW_LATENCY: 3213 if (tx_bytes > 10) { 3214 tx_latency = IXL_AVE_LATENCY; 3215 tx_itr = IXL_ITR_20K; 3216 } 3217 break; 3218 case IXL_AVE_LATENCY: 3219 if (tx_bytes > 20) { 3220 tx_latency = IXL_BULK_LATENCY; 3221 tx_itr = IXL_ITR_8K; 3222 } else if (tx_bytes <= 10) { 3223 tx_latency = IXL_LOW_LATENCY; 3224 tx_itr = IXL_ITR_100K; 3225 } 3226 break; 3227 case IXL_BULK_LATENCY: 3228 if (tx_bytes <= 20) { 3229 tx_latency = IXL_AVE_LATENCY; 3230 tx_itr = IXL_ITR_20K; 3231 } 3232 break; 3233 } 3234 3235 txr->latency = tx_latency; 3236 3237 if (tx_itr != txr->itr) { 3238 /* do an exponential smoothing */ 3239 tx_itr = (10 * tx_itr * txr->itr) / 3240 ((9 * tx_itr) + txr->itr); 3241 txr->itr = tx_itr & IXL_MAX_ITR; 3242 wr32(hw, I40E_PFINT_ITRN(IXL_TX_ITR, 3243 que->me), txr->itr); 3244 } 3245 3246 } else { /* We may have have toggled to non-dynamic */ 3247 if (vsi->tx_itr_setting & IXL_ITR_DYNAMIC) 3248 vsi->tx_itr_setting = ixl_tx_itr; 3249 /* Update the hardware if needed */ 3250 if (txr->itr != vsi->tx_itr_setting) { 3251 txr->itr = vsi->tx_itr_setting; 3252 wr32(hw, I40E_PFINT_ITRN(IXL_TX_ITR, 3253 que->me), txr->itr); 3254 } 3255 } 3256 txr->bytes = 0; 3257 txr->packets = 0; 3258 return; 3259 } 3260 3261 #define QUEUE_NAME_LEN 32 3262 3263 static void 3264 ixl_add_vsi_sysctls(struct ixl_pf *pf, struct ixl_vsi *vsi, 3265 struct sysctl_ctx_list *ctx, const char *sysctl_name) 3266 { 3267 struct sysctl_oid *tree; 3268 struct sysctl_oid_list *child; 3269 struct sysctl_oid_list *vsi_list; 3270 3271 tree = device_get_sysctl_tree(pf->dev); 3272 child = SYSCTL_CHILDREN(tree); 3273 vsi->vsi_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, sysctl_name, 3274 CTLFLAG_RD, NULL, "VSI Number"); 3275 vsi_list = SYSCTL_CHILDREN(vsi->vsi_node); 3276 3277 ixl_add_sysctls_eth_stats(ctx, vsi_list, &vsi->eth_stats); 3278 } 3279 3280 static void 3281 ixl_add_hw_stats(struct ixl_pf *pf) 3282 { 3283 device_t dev = pf->dev; 3284 struct ixl_vsi *vsi = &pf->vsi; 3285 struct ixl_queue *queues = vsi->queues; 3286 struct i40e_hw_port_stats *pf_stats = &pf->stats; 3287 3288 struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(dev); 3289 struct sysctl_oid *tree = device_get_sysctl_tree(dev); 3290 struct sysctl_oid_list *child = SYSCTL_CHILDREN(tree); 3291 struct sysctl_oid_list *vsi_list; 3292 3293 struct sysctl_oid *queue_node; 3294 struct sysctl_oid_list *queue_list; 3295 3296 struct tx_ring *txr; 3297 struct rx_ring *rxr; 3298 char queue_namebuf[QUEUE_NAME_LEN]; 3299 3300 /* Driver statistics */ 3301 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "watchdog_events", 3302 CTLFLAG_RD, &pf->watchdog_events, 3303 "Watchdog timeouts"); 3304 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "admin_irq", 3305 CTLFLAG_RD, &pf->admin_irq, 3306 "Admin Queue IRQ Handled"); 3307 3308 ixl_add_vsi_sysctls(pf, &pf->vsi, ctx, "pf"); 3309 vsi_list = SYSCTL_CHILDREN(pf->vsi.vsi_node); 3310 3311 /* Queue statistics */ 3312 for (int q = 0; q < vsi->num_queues; q++) { 3313 snprintf(queue_namebuf, QUEUE_NAME_LEN, "que%d", q); 3314 queue_node = SYSCTL_ADD_NODE(ctx, vsi_list, 3315 OID_AUTO, queue_namebuf, CTLFLAG_RD, NULL, "Queue #"); 3316 queue_list = SYSCTL_CHILDREN(queue_node); 3317 3318 txr = &(queues[q].txr); 3319 rxr = &(queues[q].rxr); 3320 3321 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "mbuf_defrag_failed", 3322 CTLFLAG_RD, &(queues[q].mbuf_defrag_failed), 3323 "m_defrag() failed"); 3324 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "dropped", 3325 CTLFLAG_RD, &(queues[q].dropped_pkts), 3326 "Driver dropped packets"); 3327 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "irqs", 3328 CTLFLAG_RD, &(queues[q].irqs), 3329 "irqs on this queue"); 3330 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "tso_tx", 3331 CTLFLAG_RD, &(queues[q].tso), 3332 "TSO"); 3333 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "tx_dma_setup", 3334 CTLFLAG_RD, &(queues[q].tx_dma_setup), 3335 "Driver tx dma failure in xmit"); 3336 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "no_desc_avail", 3337 CTLFLAG_RD, &(txr->no_desc), 3338 "Queue No Descriptor Available"); 3339 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "tx_packets", 3340 CTLFLAG_RD, &(txr->total_packets), 3341 "Queue Packets Transmitted"); 3342 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "tx_bytes", 3343 CTLFLAG_RD, &(txr->tx_bytes), 3344 "Queue Bytes Transmitted"); 3345 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "rx_packets", 3346 CTLFLAG_RD, &(rxr->rx_packets), 3347 "Queue Packets Received"); 3348 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "rx_bytes", 3349 CTLFLAG_RD, &(rxr->rx_bytes), 3350 "Queue Bytes Received"); 3351 } 3352 3353 /* MAC stats */ 3354 ixl_add_sysctls_mac_stats(ctx, child, pf_stats); 3355 } 3356 3357 static void 3358 ixl_add_sysctls_eth_stats(struct sysctl_ctx_list *ctx, 3359 struct sysctl_oid_list *child, 3360 struct i40e_eth_stats *eth_stats) 3361 { 3362 struct ixl_sysctl_info ctls[] = 3363 { 3364 {ð_stats->rx_bytes, "good_octets_rcvd", "Good Octets Received"}, 3365 {ð_stats->rx_unicast, "ucast_pkts_rcvd", 3366 "Unicast Packets Received"}, 3367 {ð_stats->rx_multicast, "mcast_pkts_rcvd", 3368 "Multicast Packets Received"}, 3369 {ð_stats->rx_broadcast, "bcast_pkts_rcvd", 3370 "Broadcast Packets Received"}, 3371 {ð_stats->rx_discards, "rx_discards", "Discarded RX packets"}, 3372 {ð_stats->tx_bytes, "good_octets_txd", "Good Octets Transmitted"}, 3373 {ð_stats->tx_unicast, "ucast_pkts_txd", "Unicast Packets Transmitted"}, 3374 {ð_stats->tx_multicast, "mcast_pkts_txd", 3375 "Multicast Packets Transmitted"}, 3376 {ð_stats->tx_broadcast, "bcast_pkts_txd", 3377 "Broadcast Packets Transmitted"}, 3378 // end 3379 {0,0,0} 3380 }; 3381 3382 struct ixl_sysctl_info *entry = ctls; 3383 while (entry->stat != NULL) 3384 { 3385 SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, entry->name, 3386 CTLFLAG_RD, entry->stat, 3387 entry->description); 3388 entry++; 3389 } 3390 } 3391 3392 static void 3393 ixl_add_sysctls_mac_stats(struct sysctl_ctx_list *ctx, 3394 struct sysctl_oid_list *child, 3395 struct i40e_hw_port_stats *stats) 3396 { 3397 struct sysctl_oid *stat_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, "mac", 3398 CTLFLAG_RD, NULL, "Mac Statistics"); 3399 struct sysctl_oid_list *stat_list = SYSCTL_CHILDREN(stat_node); 3400 3401 struct i40e_eth_stats *eth_stats = &stats->eth; 3402 ixl_add_sysctls_eth_stats(ctx, stat_list, eth_stats); 3403 3404 struct ixl_sysctl_info ctls[] = 3405 { 3406 {&stats->crc_errors, "crc_errors", "CRC Errors"}, 3407 {&stats->illegal_bytes, "illegal_bytes", "Illegal Byte Errors"}, 3408 {&stats->mac_local_faults, "local_faults", "MAC Local Faults"}, 3409 {&stats->mac_remote_faults, "remote_faults", "MAC Remote Faults"}, 3410 {&stats->rx_length_errors, "rx_length_errors", "Receive Length Errors"}, 3411 /* Packet Reception Stats */ 3412 {&stats->rx_size_64, "rx_frames_64", "64 byte frames received"}, 3413 {&stats->rx_size_127, "rx_frames_65_127", "65-127 byte frames received"}, 3414 {&stats->rx_size_255, "rx_frames_128_255", "128-255 byte frames received"}, 3415 {&stats->rx_size_511, "rx_frames_256_511", "256-511 byte frames received"}, 3416 {&stats->rx_size_1023, "rx_frames_512_1023", "512-1023 byte frames received"}, 3417 {&stats->rx_size_1522, "rx_frames_1024_1522", "1024-1522 byte frames received"}, 3418 {&stats->rx_size_big, "rx_frames_big", "1523-9522 byte frames received"}, 3419 {&stats->rx_undersize, "rx_undersize", "Undersized packets received"}, 3420 {&stats->rx_fragments, "rx_fragmented", "Fragmented packets received"}, 3421 {&stats->rx_oversize, "rx_oversized", "Oversized packets received"}, 3422 {&stats->rx_jabber, "rx_jabber", "Received Jabber"}, 3423 {&stats->checksum_error, "checksum_errors", "Checksum Errors"}, 3424 /* Packet Transmission Stats */ 3425 {&stats->tx_size_64, "tx_frames_64", "64 byte frames transmitted"}, 3426 {&stats->tx_size_127, "tx_frames_65_127", "65-127 byte frames transmitted"}, 3427 {&stats->tx_size_255, "tx_frames_128_255", "128-255 byte frames transmitted"}, 3428 {&stats->tx_size_511, "tx_frames_256_511", "256-511 byte frames transmitted"}, 3429 {&stats->tx_size_1023, "tx_frames_512_1023", "512-1023 byte frames transmitted"}, 3430 {&stats->tx_size_1522, "tx_frames_1024_1522", "1024-1522 byte frames transmitted"}, 3431 {&stats->tx_size_big, "tx_frames_big", "1523-9522 byte frames transmitted"}, 3432 /* Flow control */ 3433 {&stats->link_xon_tx, "xon_txd", "Link XON transmitted"}, 3434 {&stats->link_xon_rx, "xon_recvd", "Link XON received"}, 3435 {&stats->link_xoff_tx, "xoff_txd", "Link XOFF transmitted"}, 3436 {&stats->link_xoff_rx, "xoff_recvd", "Link XOFF received"}, 3437 /* End */ 3438 {0,0,0} 3439 }; 3440 3441 struct ixl_sysctl_info *entry = ctls; 3442 while (entry->stat != NULL) 3443 { 3444 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, entry->name, 3445 CTLFLAG_RD, entry->stat, 3446 entry->description); 3447 entry++; 3448 } 3449 } 3450 3451 3452 /* 3453 ** ixl_config_rss - setup RSS 3454 ** - note this is done for the single vsi 3455 */ 3456 static void ixl_config_rss(struct ixl_vsi *vsi) 3457 { 3458 struct ixl_pf *pf = (struct ixl_pf *)vsi->back; 3459 struct i40e_hw *hw = vsi->hw; 3460 u32 lut = 0; 3461 u64 set_hena = 0, hena; 3462 int i, j, que_id; 3463 #ifdef RSS 3464 u32 rss_hash_config; 3465 u32 rss_seed[IXL_KEYSZ]; 3466 #else 3467 u32 rss_seed[IXL_KEYSZ] = {0x41b01687, 3468 0x183cfd8c, 0xce880440, 0x580cbc3c, 3469 0x35897377, 0x328b25e1, 0x4fa98922, 3470 0xb7d90c14, 0xd5bad70d, 0xcd15a2c1}; 3471 #endif 3472 3473 #ifdef RSS 3474 /* Fetch the configured RSS key */ 3475 rss_getkey((uint8_t *) &rss_seed); 3476 #endif 3477 3478 /* Fill out hash function seed */ 3479 for (i = 0; i < IXL_KEYSZ; i++) 3480 wr32(hw, I40E_PFQF_HKEY(i), rss_seed[i]); 3481 3482 /* Enable PCTYPES for RSS: */ 3483 #ifdef RSS 3484 rss_hash_config = rss_gethashconfig(); 3485 if (rss_hash_config & RSS_HASHTYPE_RSS_IPV4) 3486 set_hena |= ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV4_OTHER); 3487 if (rss_hash_config & RSS_HASHTYPE_RSS_TCP_IPV4) 3488 set_hena |= ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV4_TCP); 3489 if (rss_hash_config & RSS_HASHTYPE_RSS_UDP_IPV4) 3490 set_hena |= ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV4_UDP); 3491 if (rss_hash_config & RSS_HASHTYPE_RSS_IPV6) 3492 set_hena |= ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV6_OTHER); 3493 if (rss_hash_config & RSS_HASHTYPE_RSS_IPV6_EX) 3494 set_hena |= ((u64)1 << I40E_FILTER_PCTYPE_FRAG_IPV6); 3495 if (rss_hash_config & RSS_HASHTYPE_RSS_TCP_IPV6) 3496 set_hena |= ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV6_TCP); 3497 if (rss_hash_config & RSS_HASHTYPE_RSS_UDP_IPV6) 3498 set_hena |= ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV6_UDP); 3499 #else 3500 set_hena = 3501 ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV4_UDP) | 3502 ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV4_TCP) | 3503 ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV4_SCTP) | 3504 ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV4_OTHER) | 3505 ((u64)1 << I40E_FILTER_PCTYPE_FRAG_IPV4) | 3506 ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV6_UDP) | 3507 ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV6_TCP) | 3508 ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV6_SCTP) | 3509 ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV6_OTHER) | 3510 ((u64)1 << I40E_FILTER_PCTYPE_FRAG_IPV6) | 3511 ((u64)1 << I40E_FILTER_PCTYPE_L2_PAYLOAD); 3512 #endif 3513 hena = (u64)rd32(hw, I40E_PFQF_HENA(0)) | 3514 ((u64)rd32(hw, I40E_PFQF_HENA(1)) << 32); 3515 hena |= set_hena; 3516 wr32(hw, I40E_PFQF_HENA(0), (u32)hena); 3517 wr32(hw, I40E_PFQF_HENA(1), (u32)(hena >> 32)); 3518 3519 /* Populate the LUT with max no. of queues in round robin fashion */ 3520 for (i = j = 0; i < pf->hw.func_caps.rss_table_size; i++, j++) { 3521 if (j == vsi->num_queues) 3522 j = 0; 3523 #ifdef RSS 3524 /* 3525 * Fetch the RSS bucket id for the given indirection entry. 3526 * Cap it at the number of configured buckets (which is 3527 * num_queues.) 3528 */ 3529 que_id = rss_get_indirection_to_bucket(i); 3530 que_id = que_id % vsi->num_queues; 3531 #else 3532 que_id = j; 3533 #endif 3534 /* lut = 4-byte sliding window of 4 lut entries */ 3535 lut = (lut << 8) | (que_id & 3536 ((0x1 << pf->hw.func_caps.rss_table_entry_width) - 1)); 3537 /* On i = 3, we have 4 entries in lut; write to the register */ 3538 if ((i & 3) == 3) 3539 wr32(hw, I40E_PFQF_HLUT(i >> 2), lut); 3540 } 3541 ixl_flush(hw); 3542 } 3543 3544 3545 /* 3546 ** This routine is run via an vlan config EVENT, 3547 ** it enables us to use the HW Filter table since 3548 ** we can get the vlan id. This just creates the 3549 ** entry in the soft version of the VFTA, init will 3550 ** repopulate the real table. 3551 */ 3552 static void 3553 ixl_register_vlan(void *arg, struct ifnet *ifp, u16 vtag) 3554 { 3555 struct ixl_vsi *vsi = ifp->if_softc; 3556 struct i40e_hw *hw = vsi->hw; 3557 struct ixl_pf *pf = (struct ixl_pf *)vsi->back; 3558 3559 if (ifp->if_softc != arg) /* Not our event */ 3560 return; 3561 3562 if ((vtag == 0) || (vtag > 4095)) /* Invalid */ 3563 return; 3564 3565 IXL_PF_LOCK(pf); 3566 ++vsi->num_vlans; 3567 ixl_add_filter(vsi, hw->mac.addr, vtag); 3568 IXL_PF_UNLOCK(pf); 3569 } 3570 3571 /* 3572 ** This routine is run via an vlan 3573 ** unconfig EVENT, remove our entry 3574 ** in the soft vfta. 3575 */ 3576 static void 3577 ixl_unregister_vlan(void *arg, struct ifnet *ifp, u16 vtag) 3578 { 3579 struct ixl_vsi *vsi = ifp->if_softc; 3580 struct i40e_hw *hw = vsi->hw; 3581 struct ixl_pf *pf = (struct ixl_pf *)vsi->back; 3582 3583 if (ifp->if_softc != arg) 3584 return; 3585 3586 if ((vtag == 0) || (vtag > 4095)) /* Invalid */ 3587 return; 3588 3589 IXL_PF_LOCK(pf); 3590 --vsi->num_vlans; 3591 ixl_del_filter(vsi, hw->mac.addr, vtag); 3592 IXL_PF_UNLOCK(pf); 3593 } 3594 3595 /* 3596 ** This routine updates vlan filters, called by init 3597 ** it scans the filter table and then updates the hw 3598 ** after a soft reset. 3599 */ 3600 static void 3601 ixl_setup_vlan_filters(struct ixl_vsi *vsi) 3602 { 3603 struct ixl_mac_filter *f; 3604 int cnt = 0, flags; 3605 3606 if (vsi->num_vlans == 0) 3607 return; 3608 /* 3609 ** Scan the filter list for vlan entries, 3610 ** mark them for addition and then call 3611 ** for the AQ update. 3612 */ 3613 SLIST_FOREACH(f, &vsi->ftl, next) { 3614 if (f->flags & IXL_FILTER_VLAN) { 3615 f->flags |= 3616 (IXL_FILTER_ADD | 3617 IXL_FILTER_USED); 3618 cnt++; 3619 } 3620 } 3621 if (cnt == 0) { 3622 printf("setup vlan: no filters found!\n"); 3623 return; 3624 } 3625 flags = IXL_FILTER_VLAN; 3626 flags |= (IXL_FILTER_ADD | IXL_FILTER_USED); 3627 ixl_add_hw_filters(vsi, flags, cnt); 3628 return; 3629 } 3630 3631 /* 3632 ** Initialize filter list and add filters that the hardware 3633 ** needs to know about. 3634 */ 3635 static void 3636 ixl_init_filters(struct ixl_vsi *vsi) 3637 { 3638 /* Add broadcast address */ 3639 ixl_add_filter(vsi, ixl_bcast_addr, IXL_VLAN_ANY); 3640 } 3641 3642 /* 3643 ** This routine adds mulicast filters 3644 */ 3645 static void 3646 ixl_add_mc_filter(struct ixl_vsi *vsi, u8 *macaddr) 3647 { 3648 struct ixl_mac_filter *f; 3649 3650 /* Does one already exist */ 3651 f = ixl_find_filter(vsi, macaddr, IXL_VLAN_ANY); 3652 if (f != NULL) 3653 return; 3654 3655 f = ixl_get_filter(vsi); 3656 if (f == NULL) { 3657 printf("WARNING: no filter available!!\n"); 3658 return; 3659 } 3660 bcopy(macaddr, f->macaddr, ETHER_ADDR_LEN); 3661 f->vlan = IXL_VLAN_ANY; 3662 f->flags |= (IXL_FILTER_ADD | IXL_FILTER_USED 3663 | IXL_FILTER_MC); 3664 3665 return; 3666 } 3667 3668 static void 3669 ixl_reconfigure_filters(struct ixl_vsi *vsi) 3670 { 3671 3672 ixl_add_hw_filters(vsi, IXL_FILTER_USED, vsi->num_macs); 3673 } 3674 3675 /* 3676 ** This routine adds macvlan filters 3677 */ 3678 static void 3679 ixl_add_filter(struct ixl_vsi *vsi, u8 *macaddr, s16 vlan) 3680 { 3681 struct ixl_mac_filter *f, *tmp; 3682 struct ixl_pf *pf; 3683 device_t dev; 3684 3685 DEBUGOUT("ixl_add_filter: begin"); 3686 3687 pf = vsi->back; 3688 dev = pf->dev; 3689 3690 /* Does one already exist */ 3691 f = ixl_find_filter(vsi, macaddr, vlan); 3692 if (f != NULL) 3693 return; 3694 /* 3695 ** Is this the first vlan being registered, if so we 3696 ** need to remove the ANY filter that indicates we are 3697 ** not in a vlan, and replace that with a 0 filter. 3698 */ 3699 if ((vlan != IXL_VLAN_ANY) && (vsi->num_vlans == 1)) { 3700 tmp = ixl_find_filter(vsi, macaddr, IXL_VLAN_ANY); 3701 if (tmp != NULL) { 3702 ixl_del_filter(vsi, macaddr, IXL_VLAN_ANY); 3703 ixl_add_filter(vsi, macaddr, 0); 3704 } 3705 } 3706 3707 f = ixl_get_filter(vsi); 3708 if (f == NULL) { 3709 device_printf(dev, "WARNING: no filter available!!\n"); 3710 return; 3711 } 3712 bcopy(macaddr, f->macaddr, ETHER_ADDR_LEN); 3713 f->vlan = vlan; 3714 f->flags |= (IXL_FILTER_ADD | IXL_FILTER_USED); 3715 if (f->vlan != IXL_VLAN_ANY) 3716 f->flags |= IXL_FILTER_VLAN; 3717 else 3718 vsi->num_macs++; 3719 3720 ixl_add_hw_filters(vsi, f->flags, 1); 3721 return; 3722 } 3723 3724 static void 3725 ixl_del_filter(struct ixl_vsi *vsi, u8 *macaddr, s16 vlan) 3726 { 3727 struct ixl_mac_filter *f; 3728 3729 f = ixl_find_filter(vsi, macaddr, vlan); 3730 if (f == NULL) 3731 return; 3732 3733 f->flags |= IXL_FILTER_DEL; 3734 ixl_del_hw_filters(vsi, 1); 3735 vsi->num_macs--; 3736 3737 /* Check if this is the last vlan removal */ 3738 if (vlan != IXL_VLAN_ANY && vsi->num_vlans == 0) { 3739 /* Switch back to a non-vlan filter */ 3740 ixl_del_filter(vsi, macaddr, 0); 3741 ixl_add_filter(vsi, macaddr, IXL_VLAN_ANY); 3742 } 3743 return; 3744 } 3745 3746 /* 3747 ** Find the filter with both matching mac addr and vlan id 3748 */ 3749 static struct ixl_mac_filter * 3750 ixl_find_filter(struct ixl_vsi *vsi, u8 *macaddr, s16 vlan) 3751 { 3752 struct ixl_mac_filter *f; 3753 bool match = FALSE; 3754 3755 SLIST_FOREACH(f, &vsi->ftl, next) { 3756 if (!cmp_etheraddr(f->macaddr, macaddr)) 3757 continue; 3758 if (f->vlan == vlan) { 3759 match = TRUE; 3760 break; 3761 } 3762 } 3763 3764 if (!match) 3765 f = NULL; 3766 return (f); 3767 } 3768 3769 /* 3770 ** This routine takes additions to the vsi filter 3771 ** table and creates an Admin Queue call to create 3772 ** the filters in the hardware. 3773 */ 3774 static void 3775 ixl_add_hw_filters(struct ixl_vsi *vsi, int flags, int cnt) 3776 { 3777 struct i40e_aqc_add_macvlan_element_data *a, *b; 3778 struct ixl_mac_filter *f; 3779 struct ixl_pf *pf; 3780 struct i40e_hw *hw; 3781 device_t dev; 3782 int err, j = 0; 3783 3784 pf = vsi->back; 3785 dev = pf->dev; 3786 hw = &pf->hw; 3787 IXL_PF_LOCK_ASSERT(pf); 3788 3789 a = malloc(sizeof(struct i40e_aqc_add_macvlan_element_data) * cnt, 3790 M_DEVBUF, M_NOWAIT | M_ZERO); 3791 if (a == NULL) { 3792 device_printf(dev, "add_hw_filters failed to get memory\n"); 3793 return; 3794 } 3795 3796 /* 3797 ** Scan the filter list, each time we find one 3798 ** we add it to the admin queue array and turn off 3799 ** the add bit. 3800 */ 3801 SLIST_FOREACH(f, &vsi->ftl, next) { 3802 if (f->flags == flags) { 3803 b = &a[j]; // a pox on fvl long names :) 3804 bcopy(f->macaddr, b->mac_addr, ETHER_ADDR_LEN); 3805 if (f->vlan == IXL_VLAN_ANY) { 3806 b->vlan_tag = 0; 3807 b->flags = I40E_AQC_MACVLAN_ADD_IGNORE_VLAN; 3808 } else { 3809 b->vlan_tag = f->vlan; 3810 b->flags = 0; 3811 } 3812 b->flags |= I40E_AQC_MACVLAN_ADD_PERFECT_MATCH; 3813 f->flags &= ~IXL_FILTER_ADD; 3814 j++; 3815 } 3816 if (j == cnt) 3817 break; 3818 } 3819 if (j > 0) { 3820 err = i40e_aq_add_macvlan(hw, vsi->seid, a, j, NULL); 3821 if (err) 3822 device_printf(dev, "aq_add_macvlan err %d, " 3823 "aq_error %d\n", err, hw->aq.asq_last_status); 3824 else 3825 vsi->hw_filters_add += j; 3826 } 3827 free(a, M_DEVBUF); 3828 return; 3829 } 3830 3831 /* 3832 ** This routine takes removals in the vsi filter 3833 ** table and creates an Admin Queue call to delete 3834 ** the filters in the hardware. 3835 */ 3836 static void 3837 ixl_del_hw_filters(struct ixl_vsi *vsi, int cnt) 3838 { 3839 struct i40e_aqc_remove_macvlan_element_data *d, *e; 3840 struct ixl_pf *pf; 3841 struct i40e_hw *hw; 3842 device_t dev; 3843 struct ixl_mac_filter *f, *f_temp; 3844 int err, j = 0; 3845 3846 DEBUGOUT("ixl_del_hw_filters: begin\n"); 3847 3848 pf = vsi->back; 3849 hw = &pf->hw; 3850 dev = pf->dev; 3851 3852 d = malloc(sizeof(struct i40e_aqc_remove_macvlan_element_data) * cnt, 3853 M_DEVBUF, M_NOWAIT | M_ZERO); 3854 if (d == NULL) { 3855 printf("del hw filter failed to get memory\n"); 3856 return; 3857 } 3858 3859 SLIST_FOREACH_SAFE(f, &vsi->ftl, next, f_temp) { 3860 if (f->flags & IXL_FILTER_DEL) { 3861 e = &d[j]; // a pox on fvl long names :) 3862 bcopy(f->macaddr, e->mac_addr, ETHER_ADDR_LEN); 3863 e->vlan_tag = (f->vlan == IXL_VLAN_ANY ? 0 : f->vlan); 3864 e->flags = I40E_AQC_MACVLAN_DEL_PERFECT_MATCH; 3865 /* delete entry from vsi list */ 3866 SLIST_REMOVE(&vsi->ftl, f, ixl_mac_filter, next); 3867 free(f, M_DEVBUF); 3868 j++; 3869 } 3870 if (j == cnt) 3871 break; 3872 } 3873 if (j > 0) { 3874 err = i40e_aq_remove_macvlan(hw, vsi->seid, d, j, NULL); 3875 /* NOTE: returns ENOENT every time but seems to work fine, 3876 so we'll ignore that specific error. */ 3877 // TODO: Does this still occur on current firmwares? 3878 if (err && hw->aq.asq_last_status != I40E_AQ_RC_ENOENT) { 3879 int sc = 0; 3880 for (int i = 0; i < j; i++) 3881 sc += (!d[i].error_code); 3882 vsi->hw_filters_del += sc; 3883 device_printf(dev, 3884 "Failed to remove %d/%d filters, aq error %d\n", 3885 j - sc, j, hw->aq.asq_last_status); 3886 } else 3887 vsi->hw_filters_del += j; 3888 } 3889 free(d, M_DEVBUF); 3890 3891 DEBUGOUT("ixl_del_hw_filters: end\n"); 3892 return; 3893 } 3894 3895 static int 3896 ixl_enable_rings(struct ixl_vsi *vsi) 3897 { 3898 struct ixl_pf *pf = vsi->back; 3899 struct i40e_hw *hw = &pf->hw; 3900 int index, error; 3901 u32 reg; 3902 3903 error = 0; 3904 for (int i = 0; i < vsi->num_queues; i++) { 3905 index = vsi->first_queue + i; 3906 i40e_pre_tx_queue_cfg(hw, index, TRUE); 3907 3908 reg = rd32(hw, I40E_QTX_ENA(index)); 3909 reg |= I40E_QTX_ENA_QENA_REQ_MASK | 3910 I40E_QTX_ENA_QENA_STAT_MASK; 3911 wr32(hw, I40E_QTX_ENA(index), reg); 3912 /* Verify the enable took */ 3913 for (int j = 0; j < 10; j++) { 3914 reg = rd32(hw, I40E_QTX_ENA(index)); 3915 if (reg & I40E_QTX_ENA_QENA_STAT_MASK) 3916 break; 3917 i40e_msec_delay(10); 3918 } 3919 if ((reg & I40E_QTX_ENA_QENA_STAT_MASK) == 0) { 3920 device_printf(pf->dev, "TX queue %d disabled!\n", 3921 index); 3922 error = ETIMEDOUT; 3923 } 3924 3925 reg = rd32(hw, I40E_QRX_ENA(index)); 3926 reg |= I40E_QRX_ENA_QENA_REQ_MASK | 3927 I40E_QRX_ENA_QENA_STAT_MASK; 3928 wr32(hw, I40E_QRX_ENA(index), reg); 3929 /* Verify the enable took */ 3930 for (int j = 0; j < 10; j++) { 3931 reg = rd32(hw, I40E_QRX_ENA(index)); 3932 if (reg & I40E_QRX_ENA_QENA_STAT_MASK) 3933 break; 3934 i40e_msec_delay(10); 3935 } 3936 if ((reg & I40E_QRX_ENA_QENA_STAT_MASK) == 0) { 3937 device_printf(pf->dev, "RX queue %d disabled!\n", 3938 index); 3939 error = ETIMEDOUT; 3940 } 3941 } 3942 3943 return (error); 3944 } 3945 3946 static int 3947 ixl_disable_rings(struct ixl_vsi *vsi) 3948 { 3949 struct ixl_pf *pf = vsi->back; 3950 struct i40e_hw *hw = &pf->hw; 3951 int index, error; 3952 u32 reg; 3953 3954 error = 0; 3955 for (int i = 0; i < vsi->num_queues; i++) { 3956 index = vsi->first_queue + i; 3957 3958 i40e_pre_tx_queue_cfg(hw, index, FALSE); 3959 i40e_usec_delay(500); 3960 3961 reg = rd32(hw, I40E_QTX_ENA(index)); 3962 reg &= ~I40E_QTX_ENA_QENA_REQ_MASK; 3963 wr32(hw, I40E_QTX_ENA(index), reg); 3964 /* Verify the disable took */ 3965 for (int j = 0; j < 10; j++) { 3966 reg = rd32(hw, I40E_QTX_ENA(index)); 3967 if (!(reg & I40E_QTX_ENA_QENA_STAT_MASK)) 3968 break; 3969 i40e_msec_delay(10); 3970 } 3971 if (reg & I40E_QTX_ENA_QENA_STAT_MASK) { 3972 device_printf(pf->dev, "TX queue %d still enabled!\n", 3973 index); 3974 error = ETIMEDOUT; 3975 } 3976 3977 reg = rd32(hw, I40E_QRX_ENA(index)); 3978 reg &= ~I40E_QRX_ENA_QENA_REQ_MASK; 3979 wr32(hw, I40E_QRX_ENA(index), reg); 3980 /* Verify the disable took */ 3981 for (int j = 0; j < 10; j++) { 3982 reg = rd32(hw, I40E_QRX_ENA(index)); 3983 if (!(reg & I40E_QRX_ENA_QENA_STAT_MASK)) 3984 break; 3985 i40e_msec_delay(10); 3986 } 3987 if (reg & I40E_QRX_ENA_QENA_STAT_MASK) { 3988 device_printf(pf->dev, "RX queue %d still enabled!\n", 3989 index); 3990 error = ETIMEDOUT; 3991 } 3992 } 3993 3994 return (error); 3995 } 3996 3997 /** 3998 * ixl_handle_mdd_event 3999 * 4000 * Called from interrupt handler to identify possibly malicious vfs 4001 * (But also detects events from the PF, as well) 4002 **/ 4003 static void ixl_handle_mdd_event(struct ixl_pf *pf) 4004 { 4005 struct i40e_hw *hw = &pf->hw; 4006 device_t dev = pf->dev; 4007 bool mdd_detected = false; 4008 bool pf_mdd_detected = false; 4009 u32 reg; 4010 4011 /* find what triggered the MDD event */ 4012 reg = rd32(hw, I40E_GL_MDET_TX); 4013 if (reg & I40E_GL_MDET_TX_VALID_MASK) { 4014 u8 pf_num = (reg & I40E_GL_MDET_TX_PF_NUM_MASK) >> 4015 I40E_GL_MDET_TX_PF_NUM_SHIFT; 4016 u8 event = (reg & I40E_GL_MDET_TX_EVENT_MASK) >> 4017 I40E_GL_MDET_TX_EVENT_SHIFT; 4018 u8 queue = (reg & I40E_GL_MDET_TX_QUEUE_MASK) >> 4019 I40E_GL_MDET_TX_QUEUE_SHIFT; 4020 device_printf(dev, 4021 "Malicious Driver Detection event 0x%02x" 4022 " on TX queue %d pf number 0x%02x\n", 4023 event, queue, pf_num); 4024 wr32(hw, I40E_GL_MDET_TX, 0xffffffff); 4025 mdd_detected = true; 4026 } 4027 reg = rd32(hw, I40E_GL_MDET_RX); 4028 if (reg & I40E_GL_MDET_RX_VALID_MASK) { 4029 u8 func = (reg & I40E_GL_MDET_RX_FUNCTION_MASK) >> 4030 I40E_GL_MDET_RX_FUNCTION_SHIFT; 4031 u8 event = (reg & I40E_GL_MDET_RX_EVENT_MASK) >> 4032 I40E_GL_MDET_RX_EVENT_SHIFT; 4033 u8 queue = (reg & I40E_GL_MDET_RX_QUEUE_MASK) >> 4034 I40E_GL_MDET_RX_QUEUE_SHIFT; 4035 device_printf(dev, 4036 "Malicious Driver Detection event 0x%02x" 4037 " on RX queue %d of function 0x%02x\n", 4038 event, queue, func); 4039 wr32(hw, I40E_GL_MDET_RX, 0xffffffff); 4040 mdd_detected = true; 4041 } 4042 4043 if (mdd_detected) { 4044 reg = rd32(hw, I40E_PF_MDET_TX); 4045 if (reg & I40E_PF_MDET_TX_VALID_MASK) { 4046 wr32(hw, I40E_PF_MDET_TX, 0xFFFF); 4047 device_printf(dev, 4048 "MDD TX event is for this function 0x%08x", 4049 reg); 4050 pf_mdd_detected = true; 4051 } 4052 reg = rd32(hw, I40E_PF_MDET_RX); 4053 if (reg & I40E_PF_MDET_RX_VALID_MASK) { 4054 wr32(hw, I40E_PF_MDET_RX, 0xFFFF); 4055 device_printf(dev, 4056 "MDD RX event is for this function 0x%08x", 4057 reg); 4058 pf_mdd_detected = true; 4059 } 4060 } 4061 4062 /* re-enable mdd interrupt cause */ 4063 reg = rd32(hw, I40E_PFINT_ICR0_ENA); 4064 reg |= I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK; 4065 wr32(hw, I40E_PFINT_ICR0_ENA, reg); 4066 ixl_flush(hw); 4067 } 4068 4069 static void 4070 ixl_enable_intr(struct ixl_vsi *vsi) 4071 { 4072 struct i40e_hw *hw = vsi->hw; 4073 struct ixl_queue *que = vsi->queues; 4074 4075 if (ixl_enable_msix) { 4076 ixl_enable_adminq(hw); 4077 for (int i = 0; i < vsi->num_queues; i++, que++) 4078 ixl_enable_queue(hw, que->me); 4079 } else 4080 ixl_enable_legacy(hw); 4081 } 4082 4083 static void 4084 ixl_disable_rings_intr(struct ixl_vsi *vsi) 4085 { 4086 struct i40e_hw *hw = vsi->hw; 4087 struct ixl_queue *que = vsi->queues; 4088 4089 for (int i = 0; i < vsi->num_queues; i++, que++) 4090 ixl_disable_queue(hw, que->me); 4091 } 4092 4093 static void 4094 ixl_disable_intr(struct ixl_vsi *vsi) 4095 { 4096 struct i40e_hw *hw = vsi->hw; 4097 4098 if (ixl_enable_msix) 4099 ixl_disable_adminq(hw); 4100 else 4101 ixl_disable_legacy(hw); 4102 } 4103 4104 static void 4105 ixl_enable_adminq(struct i40e_hw *hw) 4106 { 4107 u32 reg; 4108 4109 reg = I40E_PFINT_DYN_CTL0_INTENA_MASK | 4110 I40E_PFINT_DYN_CTL0_CLEARPBA_MASK | 4111 (IXL_ITR_NONE << I40E_PFINT_DYN_CTL0_ITR_INDX_SHIFT); 4112 wr32(hw, I40E_PFINT_DYN_CTL0, reg); 4113 ixl_flush(hw); 4114 return; 4115 } 4116 4117 static void 4118 ixl_disable_adminq(struct i40e_hw *hw) 4119 { 4120 u32 reg; 4121 4122 reg = IXL_ITR_NONE << I40E_PFINT_DYN_CTL0_ITR_INDX_SHIFT; 4123 wr32(hw, I40E_PFINT_DYN_CTL0, reg); 4124 4125 return; 4126 } 4127 4128 static void 4129 ixl_enable_queue(struct i40e_hw *hw, int id) 4130 { 4131 u32 reg; 4132 4133 reg = I40E_PFINT_DYN_CTLN_INTENA_MASK | 4134 I40E_PFINT_DYN_CTLN_CLEARPBA_MASK | 4135 (IXL_ITR_NONE << I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT); 4136 wr32(hw, I40E_PFINT_DYN_CTLN(id), reg); 4137 } 4138 4139 static void 4140 ixl_disable_queue(struct i40e_hw *hw, int id) 4141 { 4142 u32 reg; 4143 4144 reg = IXL_ITR_NONE << I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT; 4145 wr32(hw, I40E_PFINT_DYN_CTLN(id), reg); 4146 4147 return; 4148 } 4149 4150 static void 4151 ixl_enable_legacy(struct i40e_hw *hw) 4152 { 4153 u32 reg; 4154 reg = I40E_PFINT_DYN_CTL0_INTENA_MASK | 4155 I40E_PFINT_DYN_CTL0_CLEARPBA_MASK | 4156 (IXL_ITR_NONE << I40E_PFINT_DYN_CTL0_ITR_INDX_SHIFT); 4157 wr32(hw, I40E_PFINT_DYN_CTL0, reg); 4158 } 4159 4160 static void 4161 ixl_disable_legacy(struct i40e_hw *hw) 4162 { 4163 u32 reg; 4164 4165 reg = IXL_ITR_NONE << I40E_PFINT_DYN_CTL0_ITR_INDX_SHIFT; 4166 wr32(hw, I40E_PFINT_DYN_CTL0, reg); 4167 4168 return; 4169 } 4170 4171 static void 4172 ixl_update_stats_counters(struct ixl_pf *pf) 4173 { 4174 struct i40e_hw *hw = &pf->hw; 4175 struct ixl_vsi *vsi = &pf->vsi; 4176 struct ixl_vf *vf; 4177 4178 struct i40e_hw_port_stats *nsd = &pf->stats; 4179 struct i40e_hw_port_stats *osd = &pf->stats_offsets; 4180 4181 /* Update hw stats */ 4182 ixl_stat_update32(hw, I40E_GLPRT_CRCERRS(hw->port), 4183 pf->stat_offsets_loaded, 4184 &osd->crc_errors, &nsd->crc_errors); 4185 ixl_stat_update32(hw, I40E_GLPRT_ILLERRC(hw->port), 4186 pf->stat_offsets_loaded, 4187 &osd->illegal_bytes, &nsd->illegal_bytes); 4188 ixl_stat_update48(hw, I40E_GLPRT_GORCH(hw->port), 4189 I40E_GLPRT_GORCL(hw->port), 4190 pf->stat_offsets_loaded, 4191 &osd->eth.rx_bytes, &nsd->eth.rx_bytes); 4192 ixl_stat_update48(hw, I40E_GLPRT_GOTCH(hw->port), 4193 I40E_GLPRT_GOTCL(hw->port), 4194 pf->stat_offsets_loaded, 4195 &osd->eth.tx_bytes, &nsd->eth.tx_bytes); 4196 ixl_stat_update32(hw, I40E_GLPRT_RDPC(hw->port), 4197 pf->stat_offsets_loaded, 4198 &osd->eth.rx_discards, 4199 &nsd->eth.rx_discards); 4200 ixl_stat_update48(hw, I40E_GLPRT_UPRCH(hw->port), 4201 I40E_GLPRT_UPRCL(hw->port), 4202 pf->stat_offsets_loaded, 4203 &osd->eth.rx_unicast, 4204 &nsd->eth.rx_unicast); 4205 ixl_stat_update48(hw, I40E_GLPRT_UPTCH(hw->port), 4206 I40E_GLPRT_UPTCL(hw->port), 4207 pf->stat_offsets_loaded, 4208 &osd->eth.tx_unicast, 4209 &nsd->eth.tx_unicast); 4210 ixl_stat_update48(hw, I40E_GLPRT_MPRCH(hw->port), 4211 I40E_GLPRT_MPRCL(hw->port), 4212 pf->stat_offsets_loaded, 4213 &osd->eth.rx_multicast, 4214 &nsd->eth.rx_multicast); 4215 ixl_stat_update48(hw, I40E_GLPRT_MPTCH(hw->port), 4216 I40E_GLPRT_MPTCL(hw->port), 4217 pf->stat_offsets_loaded, 4218 &osd->eth.tx_multicast, 4219 &nsd->eth.tx_multicast); 4220 ixl_stat_update48(hw, I40E_GLPRT_BPRCH(hw->port), 4221 I40E_GLPRT_BPRCL(hw->port), 4222 pf->stat_offsets_loaded, 4223 &osd->eth.rx_broadcast, 4224 &nsd->eth.rx_broadcast); 4225 ixl_stat_update48(hw, I40E_GLPRT_BPTCH(hw->port), 4226 I40E_GLPRT_BPTCL(hw->port), 4227 pf->stat_offsets_loaded, 4228 &osd->eth.tx_broadcast, 4229 &nsd->eth.tx_broadcast); 4230 4231 ixl_stat_update32(hw, I40E_GLPRT_TDOLD(hw->port), 4232 pf->stat_offsets_loaded, 4233 &osd->tx_dropped_link_down, 4234 &nsd->tx_dropped_link_down); 4235 ixl_stat_update32(hw, I40E_GLPRT_MLFC(hw->port), 4236 pf->stat_offsets_loaded, 4237 &osd->mac_local_faults, 4238 &nsd->mac_local_faults); 4239 ixl_stat_update32(hw, I40E_GLPRT_MRFC(hw->port), 4240 pf->stat_offsets_loaded, 4241 &osd->mac_remote_faults, 4242 &nsd->mac_remote_faults); 4243 ixl_stat_update32(hw, I40E_GLPRT_RLEC(hw->port), 4244 pf->stat_offsets_loaded, 4245 &osd->rx_length_errors, 4246 &nsd->rx_length_errors); 4247 4248 /* Flow control (LFC) stats */ 4249 ixl_stat_update32(hw, I40E_GLPRT_LXONRXC(hw->port), 4250 pf->stat_offsets_loaded, 4251 &osd->link_xon_rx, &nsd->link_xon_rx); 4252 ixl_stat_update32(hw, I40E_GLPRT_LXONTXC(hw->port), 4253 pf->stat_offsets_loaded, 4254 &osd->link_xon_tx, &nsd->link_xon_tx); 4255 ixl_stat_update32(hw, I40E_GLPRT_LXOFFRXC(hw->port), 4256 pf->stat_offsets_loaded, 4257 &osd->link_xoff_rx, &nsd->link_xoff_rx); 4258 ixl_stat_update32(hw, I40E_GLPRT_LXOFFTXC(hw->port), 4259 pf->stat_offsets_loaded, 4260 &osd->link_xoff_tx, &nsd->link_xoff_tx); 4261 4262 /* Packet size stats rx */ 4263 ixl_stat_update48(hw, I40E_GLPRT_PRC64H(hw->port), 4264 I40E_GLPRT_PRC64L(hw->port), 4265 pf->stat_offsets_loaded, 4266 &osd->rx_size_64, &nsd->rx_size_64); 4267 ixl_stat_update48(hw, I40E_GLPRT_PRC127H(hw->port), 4268 I40E_GLPRT_PRC127L(hw->port), 4269 pf->stat_offsets_loaded, 4270 &osd->rx_size_127, &nsd->rx_size_127); 4271 ixl_stat_update48(hw, I40E_GLPRT_PRC255H(hw->port), 4272 I40E_GLPRT_PRC255L(hw->port), 4273 pf->stat_offsets_loaded, 4274 &osd->rx_size_255, &nsd->rx_size_255); 4275 ixl_stat_update48(hw, I40E_GLPRT_PRC511H(hw->port), 4276 I40E_GLPRT_PRC511L(hw->port), 4277 pf->stat_offsets_loaded, 4278 &osd->rx_size_511, &nsd->rx_size_511); 4279 ixl_stat_update48(hw, I40E_GLPRT_PRC1023H(hw->port), 4280 I40E_GLPRT_PRC1023L(hw->port), 4281 pf->stat_offsets_loaded, 4282 &osd->rx_size_1023, &nsd->rx_size_1023); 4283 ixl_stat_update48(hw, I40E_GLPRT_PRC1522H(hw->port), 4284 I40E_GLPRT_PRC1522L(hw->port), 4285 pf->stat_offsets_loaded, 4286 &osd->rx_size_1522, &nsd->rx_size_1522); 4287 ixl_stat_update48(hw, I40E_GLPRT_PRC9522H(hw->port), 4288 I40E_GLPRT_PRC9522L(hw->port), 4289 pf->stat_offsets_loaded, 4290 &osd->rx_size_big, &nsd->rx_size_big); 4291 4292 /* Packet size stats tx */ 4293 ixl_stat_update48(hw, I40E_GLPRT_PTC64H(hw->port), 4294 I40E_GLPRT_PTC64L(hw->port), 4295 pf->stat_offsets_loaded, 4296 &osd->tx_size_64, &nsd->tx_size_64); 4297 ixl_stat_update48(hw, I40E_GLPRT_PTC127H(hw->port), 4298 I40E_GLPRT_PTC127L(hw->port), 4299 pf->stat_offsets_loaded, 4300 &osd->tx_size_127, &nsd->tx_size_127); 4301 ixl_stat_update48(hw, I40E_GLPRT_PTC255H(hw->port), 4302 I40E_GLPRT_PTC255L(hw->port), 4303 pf->stat_offsets_loaded, 4304 &osd->tx_size_255, &nsd->tx_size_255); 4305 ixl_stat_update48(hw, I40E_GLPRT_PTC511H(hw->port), 4306 I40E_GLPRT_PTC511L(hw->port), 4307 pf->stat_offsets_loaded, 4308 &osd->tx_size_511, &nsd->tx_size_511); 4309 ixl_stat_update48(hw, I40E_GLPRT_PTC1023H(hw->port), 4310 I40E_GLPRT_PTC1023L(hw->port), 4311 pf->stat_offsets_loaded, 4312 &osd->tx_size_1023, &nsd->tx_size_1023); 4313 ixl_stat_update48(hw, I40E_GLPRT_PTC1522H(hw->port), 4314 I40E_GLPRT_PTC1522L(hw->port), 4315 pf->stat_offsets_loaded, 4316 &osd->tx_size_1522, &nsd->tx_size_1522); 4317 ixl_stat_update48(hw, I40E_GLPRT_PTC9522H(hw->port), 4318 I40E_GLPRT_PTC9522L(hw->port), 4319 pf->stat_offsets_loaded, 4320 &osd->tx_size_big, &nsd->tx_size_big); 4321 4322 ixl_stat_update32(hw, I40E_GLPRT_RUC(hw->port), 4323 pf->stat_offsets_loaded, 4324 &osd->rx_undersize, &nsd->rx_undersize); 4325 ixl_stat_update32(hw, I40E_GLPRT_RFC(hw->port), 4326 pf->stat_offsets_loaded, 4327 &osd->rx_fragments, &nsd->rx_fragments); 4328 ixl_stat_update32(hw, I40E_GLPRT_ROC(hw->port), 4329 pf->stat_offsets_loaded, 4330 &osd->rx_oversize, &nsd->rx_oversize); 4331 ixl_stat_update32(hw, I40E_GLPRT_RJC(hw->port), 4332 pf->stat_offsets_loaded, 4333 &osd->rx_jabber, &nsd->rx_jabber); 4334 pf->stat_offsets_loaded = true; 4335 /* End hw stats */ 4336 4337 /* Update vsi stats */ 4338 ixl_update_vsi_stats(vsi); 4339 4340 for (int i = 0; i < pf->num_vfs; i++) { 4341 vf = &pf->vfs[i]; 4342 if (vf->vf_flags & VF_FLAG_ENABLED) 4343 ixl_update_eth_stats(&pf->vfs[i].vsi); 4344 } 4345 } 4346 4347 /* 4348 ** Tasklet handler for MSIX Adminq interrupts 4349 ** - do outside interrupt since it might sleep 4350 */ 4351 static void 4352 ixl_do_adminq(void *context, int pending) 4353 { 4354 struct ixl_pf *pf = context; 4355 struct i40e_hw *hw = &pf->hw; 4356 struct ixl_vsi *vsi = &pf->vsi; 4357 struct i40e_arq_event_info event; 4358 i40e_status ret; 4359 u32 reg, loop = 0; 4360 u16 opcode, result; 4361 4362 event.buf_len = IXL_AQ_BUF_SZ; 4363 event.msg_buf = malloc(event.buf_len, 4364 M_DEVBUF, M_NOWAIT | M_ZERO); 4365 if (!event.msg_buf) { 4366 printf("Unable to allocate adminq memory\n"); 4367 return; 4368 } 4369 4370 IXL_PF_LOCK(pf); 4371 /* clean and process any events */ 4372 do { 4373 ret = i40e_clean_arq_element(hw, &event, &result); 4374 if (ret) 4375 break; 4376 opcode = LE16_TO_CPU(event.desc.opcode); 4377 switch (opcode) { 4378 case i40e_aqc_opc_get_link_status: 4379 ixl_link_event(pf, &event); 4380 ixl_update_link_status(pf); 4381 break; 4382 case i40e_aqc_opc_send_msg_to_pf: 4383 #ifdef PCI_IOV 4384 ixl_handle_vf_msg(pf, &event); 4385 #endif 4386 break; 4387 case i40e_aqc_opc_event_lan_overflow: 4388 break; 4389 default: 4390 #ifdef IXL_DEBUG 4391 printf("AdminQ unknown event %x\n", opcode); 4392 #endif 4393 break; 4394 } 4395 4396 } while (result && (loop++ < IXL_ADM_LIMIT)); 4397 4398 reg = rd32(hw, I40E_PFINT_ICR0_ENA); 4399 reg |= I40E_PFINT_ICR0_ENA_ADMINQ_MASK; 4400 wr32(hw, I40E_PFINT_ICR0_ENA, reg); 4401 free(event.msg_buf, M_DEVBUF); 4402 4403 /* 4404 * If there are still messages to process, reschedule ourselves. 4405 * Otherwise, re-enable our interrupt and go to sleep. 4406 */ 4407 if (result > 0) 4408 taskqueue_enqueue(pf->tq, &pf->adminq); 4409 else 4410 ixl_enable_intr(vsi); 4411 4412 IXL_PF_UNLOCK(pf); 4413 } 4414 4415 #ifdef IXL_DEBUG_SYSCTL 4416 static int 4417 ixl_debug_info(SYSCTL_HANDLER_ARGS) 4418 { 4419 struct ixl_pf *pf; 4420 int error, input = 0; 4421 4422 error = sysctl_handle_int(oidp, &input, 0, req); 4423 4424 if (error || !req->newptr) 4425 return (error); 4426 4427 if (input == 1) { 4428 pf = (struct ixl_pf *)arg1; 4429 ixl_print_debug_info(pf); 4430 } 4431 4432 return (error); 4433 } 4434 4435 static void 4436 ixl_print_debug_info(struct ixl_pf *pf) 4437 { 4438 struct i40e_hw *hw = &pf->hw; 4439 struct ixl_vsi *vsi = &pf->vsi; 4440 struct ixl_queue *que = vsi->queues; 4441 struct rx_ring *rxr = &que->rxr; 4442 struct tx_ring *txr = &que->txr; 4443 u32 reg; 4444 4445 4446 printf("Queue irqs = %jx\n", (uintmax_t)que->irqs); 4447 printf("AdminQ irqs = %jx\n", (uintmax_t)pf->admin_irq); 4448 printf("RX next check = %x\n", rxr->next_check); 4449 printf("RX not ready = %jx\n", (uintmax_t)rxr->not_done); 4450 printf("RX packets = %jx\n", (uintmax_t)rxr->rx_packets); 4451 printf("TX desc avail = %x\n", txr->avail); 4452 4453 reg = rd32(hw, I40E_GLV_GORCL(0xc)); 4454 printf("RX Bytes = %x\n", reg); 4455 reg = rd32(hw, I40E_GLPRT_GORCL(hw->port)); 4456 printf("Port RX Bytes = %x\n", reg); 4457 reg = rd32(hw, I40E_GLV_RDPC(0xc)); 4458 printf("RX discard = %x\n", reg); 4459 reg = rd32(hw, I40E_GLPRT_RDPC(hw->port)); 4460 printf("Port RX discard = %x\n", reg); 4461 4462 reg = rd32(hw, I40E_GLV_TEPC(0xc)); 4463 printf("TX errors = %x\n", reg); 4464 reg = rd32(hw, I40E_GLV_GOTCL(0xc)); 4465 printf("TX Bytes = %x\n", reg); 4466 4467 reg = rd32(hw, I40E_GLPRT_RUC(hw->port)); 4468 printf("RX undersize = %x\n", reg); 4469 reg = rd32(hw, I40E_GLPRT_RFC(hw->port)); 4470 printf("RX fragments = %x\n", reg); 4471 reg = rd32(hw, I40E_GLPRT_ROC(hw->port)); 4472 printf("RX oversize = %x\n", reg); 4473 reg = rd32(hw, I40E_GLPRT_RLEC(hw->port)); 4474 printf("RX length error = %x\n", reg); 4475 reg = rd32(hw, I40E_GLPRT_MRFC(hw->port)); 4476 printf("mac remote fault = %x\n", reg); 4477 reg = rd32(hw, I40E_GLPRT_MLFC(hw->port)); 4478 printf("mac local fault = %x\n", reg); 4479 } 4480 #endif 4481 4482 /** 4483 * Update VSI-specific ethernet statistics counters. 4484 **/ 4485 void ixl_update_eth_stats(struct ixl_vsi *vsi) 4486 { 4487 struct ixl_pf *pf = (struct ixl_pf *)vsi->back; 4488 struct i40e_hw *hw = &pf->hw; 4489 struct i40e_eth_stats *es; 4490 struct i40e_eth_stats *oes; 4491 struct i40e_hw_port_stats *nsd; 4492 u16 stat_idx = vsi->info.stat_counter_idx; 4493 4494 es = &vsi->eth_stats; 4495 oes = &vsi->eth_stats_offsets; 4496 nsd = &pf->stats; 4497 4498 /* Gather up the stats that the hw collects */ 4499 ixl_stat_update32(hw, I40E_GLV_TEPC(stat_idx), 4500 vsi->stat_offsets_loaded, 4501 &oes->tx_errors, &es->tx_errors); 4502 ixl_stat_update32(hw, I40E_GLV_RDPC(stat_idx), 4503 vsi->stat_offsets_loaded, 4504 &oes->rx_discards, &es->rx_discards); 4505 4506 ixl_stat_update48(hw, I40E_GLV_GORCH(stat_idx), 4507 I40E_GLV_GORCL(stat_idx), 4508 vsi->stat_offsets_loaded, 4509 &oes->rx_bytes, &es->rx_bytes); 4510 ixl_stat_update48(hw, I40E_GLV_UPRCH(stat_idx), 4511 I40E_GLV_UPRCL(stat_idx), 4512 vsi->stat_offsets_loaded, 4513 &oes->rx_unicast, &es->rx_unicast); 4514 ixl_stat_update48(hw, I40E_GLV_MPRCH(stat_idx), 4515 I40E_GLV_MPRCL(stat_idx), 4516 vsi->stat_offsets_loaded, 4517 &oes->rx_multicast, &es->rx_multicast); 4518 ixl_stat_update48(hw, I40E_GLV_BPRCH(stat_idx), 4519 I40E_GLV_BPRCL(stat_idx), 4520 vsi->stat_offsets_loaded, 4521 &oes->rx_broadcast, &es->rx_broadcast); 4522 4523 ixl_stat_update48(hw, I40E_GLV_GOTCH(stat_idx), 4524 I40E_GLV_GOTCL(stat_idx), 4525 vsi->stat_offsets_loaded, 4526 &oes->tx_bytes, &es->tx_bytes); 4527 ixl_stat_update48(hw, I40E_GLV_UPTCH(stat_idx), 4528 I40E_GLV_UPTCL(stat_idx), 4529 vsi->stat_offsets_loaded, 4530 &oes->tx_unicast, &es->tx_unicast); 4531 ixl_stat_update48(hw, I40E_GLV_MPTCH(stat_idx), 4532 I40E_GLV_MPTCL(stat_idx), 4533 vsi->stat_offsets_loaded, 4534 &oes->tx_multicast, &es->tx_multicast); 4535 ixl_stat_update48(hw, I40E_GLV_BPTCH(stat_idx), 4536 I40E_GLV_BPTCL(stat_idx), 4537 vsi->stat_offsets_loaded, 4538 &oes->tx_broadcast, &es->tx_broadcast); 4539 vsi->stat_offsets_loaded = true; 4540 } 4541 4542 static void 4543 ixl_update_vsi_stats(struct ixl_vsi *vsi) 4544 { 4545 struct ixl_pf *pf; 4546 struct ifnet *ifp; 4547 struct i40e_eth_stats *es; 4548 u64 tx_discards; 4549 4550 struct i40e_hw_port_stats *nsd; 4551 4552 pf = vsi->back; 4553 ifp = vsi->ifp; 4554 es = &vsi->eth_stats; 4555 nsd = &pf->stats; 4556 4557 ixl_update_eth_stats(vsi); 4558 4559 tx_discards = es->tx_discards + nsd->tx_dropped_link_down; 4560 for (int i = 0; i < vsi->num_queues; i++) 4561 tx_discards += vsi->queues[i].txr.br->br_drops; 4562 4563 /* Update ifnet stats */ 4564 IXL_SET_IPACKETS(vsi, es->rx_unicast + 4565 es->rx_multicast + 4566 es->rx_broadcast); 4567 IXL_SET_OPACKETS(vsi, es->tx_unicast + 4568 es->tx_multicast + 4569 es->tx_broadcast); 4570 IXL_SET_IBYTES(vsi, es->rx_bytes); 4571 IXL_SET_OBYTES(vsi, es->tx_bytes); 4572 IXL_SET_IMCASTS(vsi, es->rx_multicast); 4573 IXL_SET_OMCASTS(vsi, es->tx_multicast); 4574 4575 IXL_SET_IERRORS(vsi, nsd->crc_errors + nsd->illegal_bytes + 4576 nsd->rx_undersize + nsd->rx_oversize + nsd->rx_fragments + 4577 nsd->rx_jabber); 4578 IXL_SET_OERRORS(vsi, es->tx_errors); 4579 IXL_SET_IQDROPS(vsi, es->rx_discards + nsd->eth.rx_discards); 4580 IXL_SET_OQDROPS(vsi, tx_discards); 4581 IXL_SET_NOPROTO(vsi, es->rx_unknown_protocol); 4582 IXL_SET_COLLISIONS(vsi, 0); 4583 } 4584 4585 /** 4586 * Reset all of the stats for the given pf 4587 **/ 4588 void ixl_pf_reset_stats(struct ixl_pf *pf) 4589 { 4590 bzero(&pf->stats, sizeof(struct i40e_hw_port_stats)); 4591 bzero(&pf->stats_offsets, sizeof(struct i40e_hw_port_stats)); 4592 pf->stat_offsets_loaded = false; 4593 } 4594 4595 /** 4596 * Resets all stats of the given vsi 4597 **/ 4598 void ixl_vsi_reset_stats(struct ixl_vsi *vsi) 4599 { 4600 bzero(&vsi->eth_stats, sizeof(struct i40e_eth_stats)); 4601 bzero(&vsi->eth_stats_offsets, sizeof(struct i40e_eth_stats)); 4602 vsi->stat_offsets_loaded = false; 4603 } 4604 4605 /** 4606 * Read and update a 48 bit stat from the hw 4607 * 4608 * Since the device stats are not reset at PFReset, they likely will not 4609 * be zeroed when the driver starts. We'll save the first values read 4610 * and use them as offsets to be subtracted from the raw values in order 4611 * to report stats that count from zero. 4612 **/ 4613 static void 4614 ixl_stat_update48(struct i40e_hw *hw, u32 hireg, u32 loreg, 4615 bool offset_loaded, u64 *offset, u64 *stat) 4616 { 4617 u64 new_data; 4618 4619 #if defined(__FreeBSD__) && (__FreeBSD_version >= 1000000) && defined(__amd64__) 4620 new_data = rd64(hw, loreg); 4621 #else 4622 /* 4623 * Use two rd32's instead of one rd64; FreeBSD versions before 4624 * 10 don't support 8 byte bus reads/writes. 4625 */ 4626 new_data = rd32(hw, loreg); 4627 new_data |= ((u64)(rd32(hw, hireg) & 0xFFFF)) << 32; 4628 #endif 4629 4630 if (!offset_loaded) 4631 *offset = new_data; 4632 if (new_data >= *offset) 4633 *stat = new_data - *offset; 4634 else 4635 *stat = (new_data + ((u64)1 << 48)) - *offset; 4636 *stat &= 0xFFFFFFFFFFFFULL; 4637 } 4638 4639 /** 4640 * Read and update a 32 bit stat from the hw 4641 **/ 4642 static void 4643 ixl_stat_update32(struct i40e_hw *hw, u32 reg, 4644 bool offset_loaded, u64 *offset, u64 *stat) 4645 { 4646 u32 new_data; 4647 4648 new_data = rd32(hw, reg); 4649 if (!offset_loaded) 4650 *offset = new_data; 4651 if (new_data >= *offset) 4652 *stat = (u32)(new_data - *offset); 4653 else 4654 *stat = (u32)((new_data + ((u64)1 << 32)) - *offset); 4655 } 4656 4657 /* 4658 ** Set flow control using sysctl: 4659 ** 0 - off 4660 ** 1 - rx pause 4661 ** 2 - tx pause 4662 ** 3 - full 4663 */ 4664 static int 4665 ixl_set_flowcntl(SYSCTL_HANDLER_ARGS) 4666 { 4667 /* 4668 * TODO: ensure flow control is disabled if 4669 * priority flow control is enabled 4670 * 4671 * TODO: ensure tx CRC by hardware should be enabled 4672 * if tx flow control is enabled. 4673 */ 4674 struct ixl_pf *pf = (struct ixl_pf *)arg1; 4675 struct i40e_hw *hw = &pf->hw; 4676 device_t dev = pf->dev; 4677 int error = 0; 4678 enum i40e_status_code aq_error = 0; 4679 u8 fc_aq_err = 0; 4680 4681 /* Get request */ 4682 error = sysctl_handle_int(oidp, &pf->fc, 0, req); 4683 if ((error) || (req->newptr == NULL)) 4684 return (error); 4685 if (pf->fc < 0 || pf->fc > 3) { 4686 device_printf(dev, 4687 "Invalid fc mode; valid modes are 0 through 3\n"); 4688 return (EINVAL); 4689 } 4690 4691 /* 4692 ** Changing flow control mode currently does not work on 4693 ** 40GBASE-CR4 PHYs 4694 */ 4695 if (hw->phy.link_info.phy_type == I40E_PHY_TYPE_40GBASE_CR4 4696 || hw->phy.link_info.phy_type == I40E_PHY_TYPE_40GBASE_CR4_CU) { 4697 device_printf(dev, "Changing flow control mode unsupported" 4698 " on 40GBase-CR4 media.\n"); 4699 return (ENODEV); 4700 } 4701 4702 /* Set fc ability for port */ 4703 hw->fc.requested_mode = pf->fc; 4704 aq_error = i40e_set_fc(hw, &fc_aq_err, TRUE); 4705 if (aq_error) { 4706 device_printf(dev, 4707 "%s: Error setting new fc mode %d; fc_err %#x\n", 4708 __func__, aq_error, fc_aq_err); 4709 return (EAGAIN); 4710 } 4711 4712 return (0); 4713 } 4714 4715 static int 4716 ixl_current_speed(SYSCTL_HANDLER_ARGS) 4717 { 4718 struct ixl_pf *pf = (struct ixl_pf *)arg1; 4719 struct i40e_hw *hw = &pf->hw; 4720 int error = 0, index = 0; 4721 4722 char *speeds[] = { 4723 "Unknown", 4724 "100M", 4725 "1G", 4726 "10G", 4727 "40G", 4728 "20G" 4729 }; 4730 4731 ixl_update_link_status(pf); 4732 4733 switch (hw->phy.link_info.link_speed) { 4734 case I40E_LINK_SPEED_100MB: 4735 index = 1; 4736 break; 4737 case I40E_LINK_SPEED_1GB: 4738 index = 2; 4739 break; 4740 case I40E_LINK_SPEED_10GB: 4741 index = 3; 4742 break; 4743 case I40E_LINK_SPEED_40GB: 4744 index = 4; 4745 break; 4746 case I40E_LINK_SPEED_20GB: 4747 index = 5; 4748 break; 4749 case I40E_LINK_SPEED_UNKNOWN: 4750 default: 4751 index = 0; 4752 break; 4753 } 4754 4755 error = sysctl_handle_string(oidp, speeds[index], 4756 strlen(speeds[index]), req); 4757 return (error); 4758 } 4759 4760 static int 4761 ixl_set_advertised_speeds(struct ixl_pf *pf, int speeds) 4762 { 4763 struct i40e_hw *hw = &pf->hw; 4764 device_t dev = pf->dev; 4765 struct i40e_aq_get_phy_abilities_resp abilities; 4766 struct i40e_aq_set_phy_config config; 4767 enum i40e_status_code aq_error = 0; 4768 4769 /* Get current capability information */ 4770 aq_error = i40e_aq_get_phy_capabilities(hw, 4771 FALSE, FALSE, &abilities, NULL); 4772 if (aq_error) { 4773 device_printf(dev, 4774 "%s: Error getting phy capabilities %d," 4775 " aq error: %d\n", __func__, aq_error, 4776 hw->aq.asq_last_status); 4777 return (EAGAIN); 4778 } 4779 4780 /* Prepare new config */ 4781 bzero(&config, sizeof(config)); 4782 config.phy_type = abilities.phy_type; 4783 config.abilities = abilities.abilities 4784 | I40E_AQ_PHY_ENABLE_ATOMIC_LINK; 4785 config.eee_capability = abilities.eee_capability; 4786 config.eeer = abilities.eeer_val; 4787 config.low_power_ctrl = abilities.d3_lpan; 4788 /* Translate into aq cmd link_speed */ 4789 if (speeds & 0x8) 4790 config.link_speed |= I40E_LINK_SPEED_20GB; 4791 if (speeds & 0x4) 4792 config.link_speed |= I40E_LINK_SPEED_10GB; 4793 if (speeds & 0x2) 4794 config.link_speed |= I40E_LINK_SPEED_1GB; 4795 if (speeds & 0x1) 4796 config.link_speed |= I40E_LINK_SPEED_100MB; 4797 4798 /* Do aq command & restart link */ 4799 aq_error = i40e_aq_set_phy_config(hw, &config, NULL); 4800 if (aq_error) { 4801 device_printf(dev, 4802 "%s: Error setting new phy config %d," 4803 " aq error: %d\n", __func__, aq_error, 4804 hw->aq.asq_last_status); 4805 return (EAGAIN); 4806 } 4807 4808 /* 4809 ** This seems a bit heavy handed, but we 4810 ** need to get a reinit on some devices 4811 */ 4812 IXL_PF_LOCK(pf); 4813 ixl_stop(pf); 4814 ixl_init_locked(pf); 4815 IXL_PF_UNLOCK(pf); 4816 4817 return (0); 4818 } 4819 4820 /* 4821 ** Control link advertise speed: 4822 ** Flags: 4823 ** 0x1 - advertise 100 Mb 4824 ** 0x2 - advertise 1G 4825 ** 0x4 - advertise 10G 4826 ** 0x8 - advertise 20G 4827 ** 4828 ** Does not work on 40G devices. 4829 */ 4830 static int 4831 ixl_set_advertise(SYSCTL_HANDLER_ARGS) 4832 { 4833 struct ixl_pf *pf = (struct ixl_pf *)arg1; 4834 struct i40e_hw *hw = &pf->hw; 4835 device_t dev = pf->dev; 4836 int requested_ls = 0; 4837 int error = 0; 4838 4839 /* 4840 ** FW doesn't support changing advertised speed 4841 ** for 40G devices; speed is always 40G. 4842 */ 4843 if (i40e_is_40G_device(hw->device_id)) 4844 return (ENODEV); 4845 4846 /* Read in new mode */ 4847 requested_ls = pf->advertised_speed; 4848 error = sysctl_handle_int(oidp, &requested_ls, 0, req); 4849 if ((error) || (req->newptr == NULL)) 4850 return (error); 4851 /* Check for sane value */ 4852 if (requested_ls < 0x1 || requested_ls > 0xE) { 4853 device_printf(dev, "Invalid advertised speed; " 4854 "valid modes are 0x1 through 0xE\n"); 4855 return (EINVAL); 4856 } 4857 /* Then check for validity based on adapter type */ 4858 switch (hw->device_id) { 4859 case I40E_DEV_ID_10G_BASE_T: 4860 if (requested_ls & 0x8) { 4861 device_printf(dev, 4862 "20Gbs speed not supported on this device.\n"); 4863 return (EINVAL); 4864 } 4865 break; 4866 case I40E_DEV_ID_20G_KR2: 4867 if (requested_ls & 0x1) { 4868 device_printf(dev, 4869 "100Mbs speed not supported on this device.\n"); 4870 return (EINVAL); 4871 } 4872 break; 4873 default: 4874 if (requested_ls & ~0x6) { 4875 device_printf(dev, 4876 "Only 1/10Gbs speeds are supported on this device.\n"); 4877 return (EINVAL); 4878 } 4879 break; 4880 } 4881 4882 /* Exit if no change */ 4883 if (pf->advertised_speed == requested_ls) 4884 return (0); 4885 4886 error = ixl_set_advertised_speeds(pf, requested_ls); 4887 if (error) 4888 return (error); 4889 4890 pf->advertised_speed = requested_ls; 4891 ixl_update_link_status(pf); 4892 return (0); 4893 } 4894 4895 /* 4896 ** Get the width and transaction speed of 4897 ** the bus this adapter is plugged into. 4898 */ 4899 static u16 4900 ixl_get_bus_info(struct i40e_hw *hw, device_t dev) 4901 { 4902 u16 link; 4903 u32 offset; 4904 4905 4906 /* Get the PCI Express Capabilities offset */ 4907 pci_find_cap(dev, PCIY_EXPRESS, &offset); 4908 4909 /* ...and read the Link Status Register */ 4910 link = pci_read_config(dev, offset + PCIER_LINK_STA, 2); 4911 4912 switch (link & I40E_PCI_LINK_WIDTH) { 4913 case I40E_PCI_LINK_WIDTH_1: 4914 hw->bus.width = i40e_bus_width_pcie_x1; 4915 break; 4916 case I40E_PCI_LINK_WIDTH_2: 4917 hw->bus.width = i40e_bus_width_pcie_x2; 4918 break; 4919 case I40E_PCI_LINK_WIDTH_4: 4920 hw->bus.width = i40e_bus_width_pcie_x4; 4921 break; 4922 case I40E_PCI_LINK_WIDTH_8: 4923 hw->bus.width = i40e_bus_width_pcie_x8; 4924 break; 4925 default: 4926 hw->bus.width = i40e_bus_width_unknown; 4927 break; 4928 } 4929 4930 switch (link & I40E_PCI_LINK_SPEED) { 4931 case I40E_PCI_LINK_SPEED_2500: 4932 hw->bus.speed = i40e_bus_speed_2500; 4933 break; 4934 case I40E_PCI_LINK_SPEED_5000: 4935 hw->bus.speed = i40e_bus_speed_5000; 4936 break; 4937 case I40E_PCI_LINK_SPEED_8000: 4938 hw->bus.speed = i40e_bus_speed_8000; 4939 break; 4940 default: 4941 hw->bus.speed = i40e_bus_speed_unknown; 4942 break; 4943 } 4944 4945 4946 device_printf(dev,"PCI Express Bus: Speed %s %s\n", 4947 ((hw->bus.speed == i40e_bus_speed_8000) ? "8.0GT/s": 4948 (hw->bus.speed == i40e_bus_speed_5000) ? "5.0GT/s": 4949 (hw->bus.speed == i40e_bus_speed_2500) ? "2.5GT/s":"Unknown"), 4950 (hw->bus.width == i40e_bus_width_pcie_x8) ? "Width x8" : 4951 (hw->bus.width == i40e_bus_width_pcie_x4) ? "Width x4" : 4952 (hw->bus.width == i40e_bus_width_pcie_x1) ? "Width x1" : 4953 ("Unknown")); 4954 4955 if ((hw->bus.width <= i40e_bus_width_pcie_x8) && 4956 (hw->bus.speed < i40e_bus_speed_8000)) { 4957 device_printf(dev, "PCI-Express bandwidth available" 4958 " for this device\n may be insufficient for" 4959 " optimal performance.\n"); 4960 device_printf(dev, "For expected performance a x8 " 4961 "PCIE Gen3 slot is required.\n"); 4962 } 4963 4964 return (link); 4965 } 4966 4967 static int 4968 ixl_sysctl_show_fw(SYSCTL_HANDLER_ARGS) 4969 { 4970 struct ixl_pf *pf = (struct ixl_pf *)arg1; 4971 struct i40e_hw *hw = &pf->hw; 4972 char buf[32]; 4973 4974 snprintf(buf, sizeof(buf), 4975 "f%d.%d a%d.%d n%02x.%02x e%08x", 4976 hw->aq.fw_maj_ver, hw->aq.fw_min_ver, 4977 hw->aq.api_maj_ver, hw->aq.api_min_ver, 4978 (hw->nvm.version & IXL_NVM_VERSION_HI_MASK) >> 4979 IXL_NVM_VERSION_HI_SHIFT, 4980 (hw->nvm.version & IXL_NVM_VERSION_LO_MASK) >> 4981 IXL_NVM_VERSION_LO_SHIFT, 4982 hw->nvm.eetrack); 4983 return (sysctl_handle_string(oidp, buf, strlen(buf), req)); 4984 } 4985 4986 4987 #ifdef IXL_DEBUG_SYSCTL 4988 static int 4989 ixl_sysctl_link_status(SYSCTL_HANDLER_ARGS) 4990 { 4991 struct ixl_pf *pf = (struct ixl_pf *)arg1; 4992 struct i40e_hw *hw = &pf->hw; 4993 struct i40e_link_status link_status; 4994 char buf[512]; 4995 4996 enum i40e_status_code aq_error = 0; 4997 4998 aq_error = i40e_aq_get_link_info(hw, TRUE, &link_status, NULL); 4999 if (aq_error) { 5000 printf("i40e_aq_get_link_info() error %d\n", aq_error); 5001 return (EPERM); 5002 } 5003 5004 sprintf(buf, "\n" 5005 "PHY Type : %#04x\n" 5006 "Speed : %#04x\n" 5007 "Link info: %#04x\n" 5008 "AN info : %#04x\n" 5009 "Ext info : %#04x", 5010 link_status.phy_type, link_status.link_speed, 5011 link_status.link_info, link_status.an_info, 5012 link_status.ext_info); 5013 5014 return (sysctl_handle_string(oidp, buf, strlen(buf), req)); 5015 } 5016 5017 static int 5018 ixl_sysctl_phy_abilities(SYSCTL_HANDLER_ARGS) 5019 { 5020 struct ixl_pf *pf = (struct ixl_pf *)arg1; 5021 struct i40e_hw *hw = &pf->hw; 5022 char buf[512]; 5023 enum i40e_status_code aq_error = 0; 5024 5025 struct i40e_aq_get_phy_abilities_resp abilities; 5026 5027 aq_error = i40e_aq_get_phy_capabilities(hw, 5028 TRUE, FALSE, &abilities, NULL); 5029 if (aq_error) { 5030 printf("i40e_aq_get_phy_capabilities() error %d\n", aq_error); 5031 return (EPERM); 5032 } 5033 5034 sprintf(buf, "\n" 5035 "PHY Type : %#010x\n" 5036 "Speed : %#04x\n" 5037 "Abilities: %#04x\n" 5038 "EEE cap : %#06x\n" 5039 "EEER reg : %#010x\n" 5040 "D3 Lpan : %#04x", 5041 abilities.phy_type, abilities.link_speed, 5042 abilities.abilities, abilities.eee_capability, 5043 abilities.eeer_val, abilities.d3_lpan); 5044 5045 return (sysctl_handle_string(oidp, buf, strlen(buf), req)); 5046 } 5047 5048 static int 5049 ixl_sysctl_sw_filter_list(SYSCTL_HANDLER_ARGS) 5050 { 5051 struct ixl_pf *pf = (struct ixl_pf *)arg1; 5052 struct ixl_vsi *vsi = &pf->vsi; 5053 struct ixl_mac_filter *f; 5054 char *buf, *buf_i; 5055 5056 int error = 0; 5057 int ftl_len = 0; 5058 int ftl_counter = 0; 5059 int buf_len = 0; 5060 int entry_len = 42; 5061 5062 SLIST_FOREACH(f, &vsi->ftl, next) { 5063 ftl_len++; 5064 } 5065 5066 if (ftl_len < 1) { 5067 sysctl_handle_string(oidp, "(none)", 6, req); 5068 return (0); 5069 } 5070 5071 buf_len = sizeof(char) * (entry_len + 1) * ftl_len + 2; 5072 buf = buf_i = malloc(buf_len, M_DEVBUF, M_NOWAIT); 5073 5074 sprintf(buf_i++, "\n"); 5075 SLIST_FOREACH(f, &vsi->ftl, next) { 5076 sprintf(buf_i, 5077 MAC_FORMAT ", vlan %4d, flags %#06x", 5078 MAC_FORMAT_ARGS(f->macaddr), f->vlan, f->flags); 5079 buf_i += entry_len; 5080 /* don't print '\n' for last entry */ 5081 if (++ftl_counter != ftl_len) { 5082 sprintf(buf_i, "\n"); 5083 buf_i++; 5084 } 5085 } 5086 5087 error = sysctl_handle_string(oidp, buf, strlen(buf), req); 5088 if (error) 5089 printf("sysctl error: %d\n", error); 5090 free(buf, M_DEVBUF); 5091 return error; 5092 } 5093 5094 #define IXL_SW_RES_SIZE 0x14 5095 static int 5096 ixl_res_alloc_cmp(const void *a, const void *b) 5097 { 5098 const struct i40e_aqc_switch_resource_alloc_element_resp *one, *two; 5099 one = (const struct i40e_aqc_switch_resource_alloc_element_resp *)a; 5100 two = (const struct i40e_aqc_switch_resource_alloc_element_resp *)b; 5101 5102 return ((int)one->resource_type - (int)two->resource_type); 5103 } 5104 5105 static int 5106 ixl_sysctl_hw_res_alloc(SYSCTL_HANDLER_ARGS) 5107 { 5108 struct ixl_pf *pf = (struct ixl_pf *)arg1; 5109 struct i40e_hw *hw = &pf->hw; 5110 device_t dev = pf->dev; 5111 struct sbuf *buf; 5112 int error = 0; 5113 5114 u8 num_entries; 5115 struct i40e_aqc_switch_resource_alloc_element_resp resp[IXL_SW_RES_SIZE]; 5116 5117 buf = sbuf_new_for_sysctl(NULL, NULL, 0, req); 5118 if (!buf) { 5119 device_printf(dev, "Could not allocate sbuf for output.\n"); 5120 return (ENOMEM); 5121 } 5122 5123 bzero(resp, sizeof(resp)); 5124 error = i40e_aq_get_switch_resource_alloc(hw, &num_entries, 5125 resp, 5126 IXL_SW_RES_SIZE, 5127 NULL); 5128 if (error) { 5129 device_printf(dev, 5130 "%s: get_switch_resource_alloc() error %d, aq error %d\n", 5131 __func__, error, hw->aq.asq_last_status); 5132 sbuf_delete(buf); 5133 return error; 5134 } 5135 5136 /* Sort entries by type for display */ 5137 qsort(resp, num_entries, 5138 sizeof(struct i40e_aqc_switch_resource_alloc_element_resp), 5139 &ixl_res_alloc_cmp); 5140 5141 sbuf_cat(buf, "\n"); 5142 sbuf_printf(buf, "# of entries: %d\n", num_entries); 5143 sbuf_printf(buf, 5144 "Type | Guaranteed | Total | Used | Un-allocated\n" 5145 " | (this) | (all) | (this) | (all) \n"); 5146 for (int i = 0; i < num_entries; i++) { 5147 sbuf_printf(buf, 5148 "%#4x | %10d %5d %6d %12d", 5149 resp[i].resource_type, 5150 resp[i].guaranteed, 5151 resp[i].total, 5152 resp[i].used, 5153 resp[i].total_unalloced); 5154 if (i < num_entries - 1) 5155 sbuf_cat(buf, "\n"); 5156 } 5157 5158 error = sbuf_finish(buf); 5159 sbuf_delete(buf); 5160 5161 return (error); 5162 } 5163 5164 /* 5165 ** Caller must init and delete sbuf; this function will clear and 5166 ** finish it for caller. 5167 */ 5168 static char * 5169 ixl_switch_element_string(struct sbuf *s, u16 seid, bool uplink) 5170 { 5171 sbuf_clear(s); 5172 5173 if (seid == 0 && uplink) 5174 sbuf_cat(s, "Network"); 5175 else if (seid == 0) 5176 sbuf_cat(s, "Host"); 5177 else if (seid == 1) 5178 sbuf_cat(s, "EMP"); 5179 else if (seid <= 5) 5180 sbuf_printf(s, "MAC %d", seid - 2); 5181 else if (seid <= 15) 5182 sbuf_cat(s, "Reserved"); 5183 else if (seid <= 31) 5184 sbuf_printf(s, "PF %d", seid - 16); 5185 else if (seid <= 159) 5186 sbuf_printf(s, "VF %d", seid - 32); 5187 else if (seid <= 287) 5188 sbuf_cat(s, "Reserved"); 5189 else if (seid <= 511) 5190 sbuf_cat(s, "Other"); // for other structures 5191 else if (seid <= 895) 5192 sbuf_printf(s, "VSI %d", seid - 512); 5193 else if (seid <= 1023) 5194 sbuf_printf(s, "Reserved"); 5195 else 5196 sbuf_cat(s, "Invalid"); 5197 5198 sbuf_finish(s); 5199 return sbuf_data(s); 5200 } 5201 5202 static int 5203 ixl_sysctl_switch_config(SYSCTL_HANDLER_ARGS) 5204 { 5205 struct ixl_pf *pf = (struct ixl_pf *)arg1; 5206 struct i40e_hw *hw = &pf->hw; 5207 device_t dev = pf->dev; 5208 struct sbuf *buf; 5209 struct sbuf *nmbuf; 5210 int error = 0; 5211 u8 aq_buf[I40E_AQ_LARGE_BUF]; 5212 5213 u16 next = 0; 5214 struct i40e_aqc_get_switch_config_resp *sw_config; 5215 sw_config = (struct i40e_aqc_get_switch_config_resp *)aq_buf; 5216 5217 buf = sbuf_new_for_sysctl(NULL, NULL, 0, req); 5218 if (!buf) { 5219 device_printf(dev, "Could not allocate sbuf for sysctl output.\n"); 5220 return (ENOMEM); 5221 } 5222 5223 error = i40e_aq_get_switch_config(hw, sw_config, 5224 sizeof(aq_buf), &next, NULL); 5225 if (error) { 5226 device_printf(dev, 5227 "%s: aq_get_switch_config() error %d, aq error %d\n", 5228 __func__, error, hw->aq.asq_last_status); 5229 sbuf_delete(buf); 5230 return error; 5231 } 5232 5233 nmbuf = sbuf_new_auto(); 5234 if (!nmbuf) { 5235 device_printf(dev, "Could not allocate sbuf for name output.\n"); 5236 return (ENOMEM); 5237 } 5238 5239 sbuf_cat(buf, "\n"); 5240 // Assuming <= 255 elements in switch 5241 sbuf_printf(buf, "# of elements: %d\n", sw_config->header.num_reported); 5242 /* Exclude: 5243 ** Revision -- all elements are revision 1 for now 5244 */ 5245 sbuf_printf(buf, 5246 "SEID ( Name ) | Uplink | Downlink | Conn Type\n" 5247 " | | | (uplink)\n"); 5248 for (int i = 0; i < sw_config->header.num_reported; i++) { 5249 // "%4d (%8s) | %8s %8s %#8x", 5250 sbuf_printf(buf, "%4d", sw_config->element[i].seid); 5251 sbuf_cat(buf, " "); 5252 sbuf_printf(buf, "(%8s)", ixl_switch_element_string(nmbuf, 5253 sw_config->element[i].seid, false)); 5254 sbuf_cat(buf, " | "); 5255 sbuf_printf(buf, "%8s", ixl_switch_element_string(nmbuf, 5256 sw_config->element[i].uplink_seid, true)); 5257 sbuf_cat(buf, " "); 5258 sbuf_printf(buf, "%8s", ixl_switch_element_string(nmbuf, 5259 sw_config->element[i].downlink_seid, false)); 5260 sbuf_cat(buf, " "); 5261 sbuf_printf(buf, "%#8x", sw_config->element[i].connection_type); 5262 if (i < sw_config->header.num_reported - 1) 5263 sbuf_cat(buf, "\n"); 5264 } 5265 sbuf_delete(nmbuf); 5266 5267 error = sbuf_finish(buf); 5268 sbuf_delete(buf); 5269 5270 return (error); 5271 } 5272 #endif /* IXL_DEBUG_SYSCTL */ 5273 5274 5275 #ifdef PCI_IOV 5276 static int 5277 ixl_vf_alloc_vsi(struct ixl_pf *pf, struct ixl_vf *vf) 5278 { 5279 struct i40e_hw *hw; 5280 struct ixl_vsi *vsi; 5281 struct i40e_vsi_context vsi_ctx; 5282 int i; 5283 uint16_t first_queue; 5284 enum i40e_status_code code; 5285 5286 hw = &pf->hw; 5287 vsi = &pf->vsi; 5288 5289 vsi_ctx.pf_num = hw->pf_id; 5290 vsi_ctx.uplink_seid = pf->veb_seid; 5291 vsi_ctx.connection_type = IXL_VSI_DATA_PORT; 5292 vsi_ctx.vf_num = hw->func_caps.vf_base_id + vf->vf_num; 5293 vsi_ctx.flags = I40E_AQ_VSI_TYPE_VF; 5294 5295 bzero(&vsi_ctx.info, sizeof(vsi_ctx.info)); 5296 5297 vsi_ctx.info.valid_sections = htole16(I40E_AQ_VSI_PROP_SWITCH_VALID); 5298 vsi_ctx.info.switch_id = htole16(0); 5299 5300 vsi_ctx.info.valid_sections |= htole16(I40E_AQ_VSI_PROP_SECURITY_VALID); 5301 vsi_ctx.info.sec_flags = 0; 5302 if (vf->vf_flags & VF_FLAG_MAC_ANTI_SPOOF) 5303 vsi_ctx.info.sec_flags |= I40E_AQ_VSI_SEC_FLAG_ENABLE_MAC_CHK; 5304 5305 vsi_ctx.info.valid_sections |= htole16(I40E_AQ_VSI_PROP_VLAN_VALID); 5306 vsi_ctx.info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_ALL | 5307 I40E_AQ_VSI_PVLAN_EMOD_NOTHING; 5308 5309 vsi_ctx.info.valid_sections |= 5310 htole16(I40E_AQ_VSI_PROP_QUEUE_MAP_VALID); 5311 vsi_ctx.info.mapping_flags = htole16(I40E_AQ_VSI_QUE_MAP_NONCONTIG); 5312 first_queue = vsi->num_queues + vf->vf_num * IXLV_MAX_QUEUES; 5313 for (i = 0; i < IXLV_MAX_QUEUES; i++) 5314 vsi_ctx.info.queue_mapping[i] = htole16(first_queue + i); 5315 for (; i < nitems(vsi_ctx.info.queue_mapping); i++) 5316 vsi_ctx.info.queue_mapping[i] = htole16(I40E_AQ_VSI_QUEUE_MASK); 5317 5318 vsi_ctx.info.tc_mapping[0] = htole16( 5319 (0 << I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT) | 5320 (1 << I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT)); 5321 5322 code = i40e_aq_add_vsi(hw, &vsi_ctx, NULL); 5323 if (code != I40E_SUCCESS) 5324 return (ixl_adminq_err_to_errno(hw->aq.asq_last_status)); 5325 vf->vsi.seid = vsi_ctx.seid; 5326 vf->vsi.vsi_num = vsi_ctx.vsi_number; 5327 vf->vsi.first_queue = first_queue; 5328 vf->vsi.num_queues = IXLV_MAX_QUEUES; 5329 5330 code = i40e_aq_get_vsi_params(hw, &vsi_ctx, NULL); 5331 if (code != I40E_SUCCESS) 5332 return (ixl_adminq_err_to_errno(hw->aq.asq_last_status)); 5333 5334 code = i40e_aq_config_vsi_bw_limit(hw, vf->vsi.seid, 0, 0, NULL); 5335 if (code != I40E_SUCCESS) { 5336 device_printf(pf->dev, "Failed to disable BW limit: %d\n", 5337 ixl_adminq_err_to_errno(hw->aq.asq_last_status)); 5338 return (ixl_adminq_err_to_errno(hw->aq.asq_last_status)); 5339 } 5340 5341 memcpy(&vf->vsi.info, &vsi_ctx.info, sizeof(vf->vsi.info)); 5342 return (0); 5343 } 5344 5345 static int 5346 ixl_vf_setup_vsi(struct ixl_pf *pf, struct ixl_vf *vf) 5347 { 5348 struct i40e_hw *hw; 5349 int error; 5350 5351 hw = &pf->hw; 5352 5353 error = ixl_vf_alloc_vsi(pf, vf); 5354 if (error != 0) 5355 return (error); 5356 5357 vf->vsi.hw_filters_add = 0; 5358 vf->vsi.hw_filters_del = 0; 5359 ixl_add_filter(&vf->vsi, ixl_bcast_addr, IXL_VLAN_ANY); 5360 ixl_reconfigure_filters(&vf->vsi); 5361 5362 return (0); 5363 } 5364 5365 static void 5366 ixl_vf_map_vsi_queue(struct i40e_hw *hw, struct ixl_vf *vf, int qnum, 5367 uint32_t val) 5368 { 5369 uint32_t qtable; 5370 int index, shift; 5371 5372 /* 5373 * Two queues are mapped in a single register, so we have to do some 5374 * gymnastics to convert the queue number into a register index and 5375 * shift. 5376 */ 5377 index = qnum / 2; 5378 shift = (qnum % 2) * I40E_VSILAN_QTABLE_QINDEX_1_SHIFT; 5379 5380 qtable = rd32(hw, I40E_VSILAN_QTABLE(index, vf->vsi.vsi_num)); 5381 qtable &= ~(I40E_VSILAN_QTABLE_QINDEX_0_MASK << shift); 5382 qtable |= val << shift; 5383 wr32(hw, I40E_VSILAN_QTABLE(index, vf->vsi.vsi_num), qtable); 5384 } 5385 5386 static void 5387 ixl_vf_map_queues(struct ixl_pf *pf, struct ixl_vf *vf) 5388 { 5389 struct i40e_hw *hw; 5390 uint32_t qtable; 5391 int i; 5392 5393 hw = &pf->hw; 5394 5395 /* 5396 * Contiguous mappings aren't actually supported by the hardware, 5397 * so we have to use non-contiguous mappings. 5398 */ 5399 wr32(hw, I40E_VSILAN_QBASE(vf->vsi.vsi_num), 5400 I40E_VSILAN_QBASE_VSIQTABLE_ENA_MASK); 5401 5402 wr32(hw, I40E_VPLAN_MAPENA(vf->vf_num), 5403 I40E_VPLAN_MAPENA_TXRX_ENA_MASK); 5404 5405 for (i = 0; i < vf->vsi.num_queues; i++) { 5406 qtable = (vf->vsi.first_queue + i) << 5407 I40E_VPLAN_QTABLE_QINDEX_SHIFT; 5408 5409 wr32(hw, I40E_VPLAN_QTABLE(i, vf->vf_num), qtable); 5410 } 5411 5412 /* Map queues allocated to VF to its VSI. */ 5413 for (i = 0; i < vf->vsi.num_queues; i++) 5414 ixl_vf_map_vsi_queue(hw, vf, i, vf->vsi.first_queue + i); 5415 5416 /* Set rest of VSI queues as unused. */ 5417 for (; i < IXL_MAX_VSI_QUEUES; i++) 5418 ixl_vf_map_vsi_queue(hw, vf, i, 5419 I40E_VSILAN_QTABLE_QINDEX_0_MASK); 5420 5421 ixl_flush(hw); 5422 } 5423 5424 static void 5425 ixl_vf_vsi_release(struct ixl_pf *pf, struct ixl_vsi *vsi) 5426 { 5427 struct i40e_hw *hw; 5428 5429 hw = &pf->hw; 5430 5431 if (vsi->seid == 0) 5432 return; 5433 5434 i40e_aq_delete_element(hw, vsi->seid, NULL); 5435 } 5436 5437 static void 5438 ixl_vf_disable_queue_intr(struct i40e_hw *hw, uint32_t vfint_reg) 5439 { 5440 5441 wr32(hw, vfint_reg, I40E_VFINT_DYN_CTLN_CLEARPBA_MASK); 5442 ixl_flush(hw); 5443 } 5444 5445 static void 5446 ixl_vf_unregister_intr(struct i40e_hw *hw, uint32_t vpint_reg) 5447 { 5448 5449 wr32(hw, vpint_reg, I40E_VPINT_LNKLSTN_FIRSTQ_TYPE_MASK | 5450 I40E_VPINT_LNKLSTN_FIRSTQ_INDX_MASK); 5451 ixl_flush(hw); 5452 } 5453 5454 static void 5455 ixl_vf_release_resources(struct ixl_pf *pf, struct ixl_vf *vf) 5456 { 5457 struct i40e_hw *hw; 5458 uint32_t vfint_reg, vpint_reg; 5459 int i; 5460 5461 hw = &pf->hw; 5462 5463 ixl_vf_vsi_release(pf, &vf->vsi); 5464 5465 /* Index 0 has a special register. */ 5466 ixl_vf_disable_queue_intr(hw, I40E_VFINT_DYN_CTL0(vf->vf_num)); 5467 5468 for (i = 1; i < hw->func_caps.num_msix_vectors_vf; i++) { 5469 vfint_reg = IXL_VFINT_DYN_CTLN_REG(hw, i , vf->vf_num); 5470 ixl_vf_disable_queue_intr(hw, vfint_reg); 5471 } 5472 5473 /* Index 0 has a special register. */ 5474 ixl_vf_unregister_intr(hw, I40E_VPINT_LNKLST0(vf->vf_num)); 5475 5476 for (i = 1; i < hw->func_caps.num_msix_vectors_vf; i++) { 5477 vpint_reg = IXL_VPINT_LNKLSTN_REG(hw, i, vf->vf_num); 5478 ixl_vf_unregister_intr(hw, vpint_reg); 5479 } 5480 5481 vf->vsi.num_queues = 0; 5482 } 5483 5484 static int 5485 ixl_flush_pcie(struct ixl_pf *pf, struct ixl_vf *vf) 5486 { 5487 struct i40e_hw *hw; 5488 int i; 5489 uint16_t global_vf_num; 5490 uint32_t ciad; 5491 5492 hw = &pf->hw; 5493 global_vf_num = hw->func_caps.vf_base_id + vf->vf_num; 5494 5495 wr32(hw, I40E_PF_PCI_CIAA, IXL_PF_PCI_CIAA_VF_DEVICE_STATUS | 5496 (global_vf_num << I40E_PF_PCI_CIAA_VF_NUM_SHIFT)); 5497 for (i = 0; i < IXL_VF_RESET_TIMEOUT; i++) { 5498 ciad = rd32(hw, I40E_PF_PCI_CIAD); 5499 if ((ciad & IXL_PF_PCI_CIAD_VF_TRANS_PENDING_MASK) == 0) 5500 return (0); 5501 DELAY(1); 5502 } 5503 5504 return (ETIMEDOUT); 5505 } 5506 5507 static void 5508 ixl_reset_vf(struct ixl_pf *pf, struct ixl_vf *vf) 5509 { 5510 struct i40e_hw *hw; 5511 uint32_t vfrtrig; 5512 5513 hw = &pf->hw; 5514 5515 vfrtrig = rd32(hw, I40E_VPGEN_VFRTRIG(vf->vf_num)); 5516 vfrtrig |= I40E_VPGEN_VFRTRIG_VFSWR_MASK; 5517 wr32(hw, I40E_VPGEN_VFRTRIG(vf->vf_num), vfrtrig); 5518 ixl_flush(hw); 5519 5520 ixl_reinit_vf(pf, vf); 5521 } 5522 5523 static void 5524 ixl_reinit_vf(struct ixl_pf *pf, struct ixl_vf *vf) 5525 { 5526 struct i40e_hw *hw; 5527 uint32_t vfrstat, vfrtrig; 5528 int i, error; 5529 5530 hw = &pf->hw; 5531 5532 error = ixl_flush_pcie(pf, vf); 5533 if (error != 0) 5534 device_printf(pf->dev, 5535 "Timed out waiting for PCIe activity to stop on VF-%d\n", 5536 vf->vf_num); 5537 5538 for (i = 0; i < IXL_VF_RESET_TIMEOUT; i++) { 5539 DELAY(10); 5540 5541 vfrstat = rd32(hw, I40E_VPGEN_VFRSTAT(vf->vf_num)); 5542 if (vfrstat & I40E_VPGEN_VFRSTAT_VFRD_MASK) 5543 break; 5544 } 5545 5546 if (i == IXL_VF_RESET_TIMEOUT) 5547 device_printf(pf->dev, "VF %d failed to reset\n", vf->vf_num); 5548 5549 wr32(hw, I40E_VFGEN_RSTAT1(vf->vf_num), I40E_VFR_COMPLETED); 5550 5551 vfrtrig = rd32(hw, I40E_VPGEN_VFRTRIG(vf->vf_num)); 5552 vfrtrig &= ~I40E_VPGEN_VFRTRIG_VFSWR_MASK; 5553 wr32(hw, I40E_VPGEN_VFRTRIG(vf->vf_num), vfrtrig); 5554 5555 if (vf->vsi.seid != 0) 5556 ixl_disable_rings(&vf->vsi); 5557 5558 ixl_vf_release_resources(pf, vf); 5559 ixl_vf_setup_vsi(pf, vf); 5560 ixl_vf_map_queues(pf, vf); 5561 5562 wr32(hw, I40E_VFGEN_RSTAT1(vf->vf_num), I40E_VFR_VFACTIVE); 5563 ixl_flush(hw); 5564 } 5565 5566 static const char * 5567 ixl_vc_opcode_str(uint16_t op) 5568 { 5569 5570 switch (op) { 5571 case I40E_VIRTCHNL_OP_VERSION: 5572 return ("VERSION"); 5573 case I40E_VIRTCHNL_OP_RESET_VF: 5574 return ("RESET_VF"); 5575 case I40E_VIRTCHNL_OP_GET_VF_RESOURCES: 5576 return ("GET_VF_RESOURCES"); 5577 case I40E_VIRTCHNL_OP_CONFIG_TX_QUEUE: 5578 return ("CONFIG_TX_QUEUE"); 5579 case I40E_VIRTCHNL_OP_CONFIG_RX_QUEUE: 5580 return ("CONFIG_RX_QUEUE"); 5581 case I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES: 5582 return ("CONFIG_VSI_QUEUES"); 5583 case I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP: 5584 return ("CONFIG_IRQ_MAP"); 5585 case I40E_VIRTCHNL_OP_ENABLE_QUEUES: 5586 return ("ENABLE_QUEUES"); 5587 case I40E_VIRTCHNL_OP_DISABLE_QUEUES: 5588 return ("DISABLE_QUEUES"); 5589 case I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS: 5590 return ("ADD_ETHER_ADDRESS"); 5591 case I40E_VIRTCHNL_OP_DEL_ETHER_ADDRESS: 5592 return ("DEL_ETHER_ADDRESS"); 5593 case I40E_VIRTCHNL_OP_ADD_VLAN: 5594 return ("ADD_VLAN"); 5595 case I40E_VIRTCHNL_OP_DEL_VLAN: 5596 return ("DEL_VLAN"); 5597 case I40E_VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE: 5598 return ("CONFIG_PROMISCUOUS_MODE"); 5599 case I40E_VIRTCHNL_OP_GET_STATS: 5600 return ("GET_STATS"); 5601 case I40E_VIRTCHNL_OP_FCOE: 5602 return ("FCOE"); 5603 case I40E_VIRTCHNL_OP_EVENT: 5604 return ("EVENT"); 5605 default: 5606 return ("UNKNOWN"); 5607 } 5608 } 5609 5610 static int 5611 ixl_vc_opcode_level(uint16_t opcode) 5612 { 5613 5614 switch (opcode) { 5615 case I40E_VIRTCHNL_OP_GET_STATS: 5616 return (10); 5617 default: 5618 return (5); 5619 } 5620 } 5621 5622 static void 5623 ixl_send_vf_msg(struct ixl_pf *pf, struct ixl_vf *vf, uint16_t op, 5624 enum i40e_status_code status, void *msg, uint16_t len) 5625 { 5626 struct i40e_hw *hw; 5627 int global_vf_id; 5628 5629 hw = &pf->hw; 5630 global_vf_id = hw->func_caps.vf_base_id + vf->vf_num; 5631 5632 I40E_VC_DEBUG(pf, ixl_vc_opcode_level(op), 5633 "Sending msg (op=%s[%d], status=%d) to VF-%d\n", 5634 ixl_vc_opcode_str(op), op, status, vf->vf_num); 5635 5636 i40e_aq_send_msg_to_vf(hw, global_vf_id, op, status, msg, len, NULL); 5637 } 5638 5639 static void 5640 ixl_send_vf_ack(struct ixl_pf *pf, struct ixl_vf *vf, uint16_t op) 5641 { 5642 5643 ixl_send_vf_msg(pf, vf, op, I40E_SUCCESS, NULL, 0); 5644 } 5645 5646 static void 5647 ixl_send_vf_nack_msg(struct ixl_pf *pf, struct ixl_vf *vf, uint16_t op, 5648 enum i40e_status_code status, const char *file, int line) 5649 { 5650 5651 I40E_VC_DEBUG(pf, 1, 5652 "Sending NACK (op=%s[%d], err=%d) to VF-%d from %s:%d\n", 5653 ixl_vc_opcode_str(op), op, status, vf->vf_num, file, line); 5654 ixl_send_vf_msg(pf, vf, op, status, NULL, 0); 5655 } 5656 5657 static void 5658 ixl_vf_version_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg, 5659 uint16_t msg_size) 5660 { 5661 struct i40e_virtchnl_version_info reply; 5662 5663 if (msg_size != sizeof(struct i40e_virtchnl_version_info)) { 5664 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_VERSION, 5665 I40E_ERR_PARAM); 5666 return; 5667 } 5668 5669 reply.major = I40E_VIRTCHNL_VERSION_MAJOR; 5670 reply.minor = I40E_VIRTCHNL_VERSION_MINOR; 5671 ixl_send_vf_msg(pf, vf, I40E_VIRTCHNL_OP_VERSION, I40E_SUCCESS, &reply, 5672 sizeof(reply)); 5673 } 5674 5675 static void 5676 ixl_vf_reset_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg, 5677 uint16_t msg_size) 5678 { 5679 5680 if (msg_size != 0) { 5681 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_RESET_VF, 5682 I40E_ERR_PARAM); 5683 return; 5684 } 5685 5686 ixl_reset_vf(pf, vf); 5687 5688 /* No response to a reset message. */ 5689 } 5690 5691 static void 5692 ixl_vf_get_resources_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg, 5693 uint16_t msg_size) 5694 { 5695 struct i40e_virtchnl_vf_resource reply; 5696 5697 if (msg_size != 0) { 5698 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_GET_VF_RESOURCES, 5699 I40E_ERR_PARAM); 5700 return; 5701 } 5702 5703 bzero(&reply, sizeof(reply)); 5704 5705 reply.vf_offload_flags = I40E_VIRTCHNL_VF_OFFLOAD_L2; 5706 5707 reply.num_vsis = 1; 5708 reply.num_queue_pairs = vf->vsi.num_queues; 5709 reply.max_vectors = pf->hw.func_caps.num_msix_vectors_vf; 5710 reply.vsi_res[0].vsi_id = vf->vsi.vsi_num; 5711 reply.vsi_res[0].vsi_type = I40E_VSI_SRIOV; 5712 reply.vsi_res[0].num_queue_pairs = vf->vsi.num_queues; 5713 memcpy(reply.vsi_res[0].default_mac_addr, vf->mac, ETHER_ADDR_LEN); 5714 5715 ixl_send_vf_msg(pf, vf, I40E_VIRTCHNL_OP_GET_VF_RESOURCES, 5716 I40E_SUCCESS, &reply, sizeof(reply)); 5717 } 5718 5719 static int 5720 ixl_vf_config_tx_queue(struct ixl_pf *pf, struct ixl_vf *vf, 5721 struct i40e_virtchnl_txq_info *info) 5722 { 5723 struct i40e_hw *hw; 5724 struct i40e_hmc_obj_txq txq; 5725 uint16_t global_queue_num, global_vf_num; 5726 enum i40e_status_code status; 5727 uint32_t qtx_ctl; 5728 5729 hw = &pf->hw; 5730 global_queue_num = vf->vsi.first_queue + info->queue_id; 5731 global_vf_num = hw->func_caps.vf_base_id + vf->vf_num; 5732 bzero(&txq, sizeof(txq)); 5733 5734 status = i40e_clear_lan_tx_queue_context(hw, global_queue_num); 5735 if (status != I40E_SUCCESS) 5736 return (EINVAL); 5737 5738 txq.base = info->dma_ring_addr / IXL_TX_CTX_BASE_UNITS; 5739 5740 txq.head_wb_ena = info->headwb_enabled; 5741 txq.head_wb_addr = info->dma_headwb_addr; 5742 txq.qlen = info->ring_len; 5743 txq.rdylist = le16_to_cpu(vf->vsi.info.qs_handle[0]); 5744 txq.rdylist_act = 0; 5745 5746 status = i40e_set_lan_tx_queue_context(hw, global_queue_num, &txq); 5747 if (status != I40E_SUCCESS) 5748 return (EINVAL); 5749 5750 qtx_ctl = I40E_QTX_CTL_VF_QUEUE | 5751 (hw->pf_id << I40E_QTX_CTL_PF_INDX_SHIFT) | 5752 (global_vf_num << I40E_QTX_CTL_VFVM_INDX_SHIFT); 5753 wr32(hw, I40E_QTX_CTL(global_queue_num), qtx_ctl); 5754 ixl_flush(hw); 5755 5756 return (0); 5757 } 5758 5759 static int 5760 ixl_vf_config_rx_queue(struct ixl_pf *pf, struct ixl_vf *vf, 5761 struct i40e_virtchnl_rxq_info *info) 5762 { 5763 struct i40e_hw *hw; 5764 struct i40e_hmc_obj_rxq rxq; 5765 uint16_t global_queue_num; 5766 enum i40e_status_code status; 5767 5768 hw = &pf->hw; 5769 global_queue_num = vf->vsi.first_queue + info->queue_id; 5770 bzero(&rxq, sizeof(rxq)); 5771 5772 if (info->databuffer_size > IXL_VF_MAX_BUFFER) 5773 return (EINVAL); 5774 5775 if (info->max_pkt_size > IXL_VF_MAX_FRAME || 5776 info->max_pkt_size < ETHER_MIN_LEN) 5777 return (EINVAL); 5778 5779 if (info->splithdr_enabled) { 5780 if (info->hdr_size > IXL_VF_MAX_HDR_BUFFER) 5781 return (EINVAL); 5782 5783 rxq.hsplit_0 = info->rx_split_pos & 5784 (I40E_HMC_OBJ_RX_HSPLIT_0_SPLIT_L2 | 5785 I40E_HMC_OBJ_RX_HSPLIT_0_SPLIT_IP | 5786 I40E_HMC_OBJ_RX_HSPLIT_0_SPLIT_TCP_UDP | 5787 I40E_HMC_OBJ_RX_HSPLIT_0_SPLIT_SCTP); 5788 rxq.hbuff = info->hdr_size >> I40E_RXQ_CTX_HBUFF_SHIFT; 5789 5790 rxq.dtype = 2; 5791 } 5792 5793 status = i40e_clear_lan_rx_queue_context(hw, global_queue_num); 5794 if (status != I40E_SUCCESS) 5795 return (EINVAL); 5796 5797 rxq.base = info->dma_ring_addr / IXL_RX_CTX_BASE_UNITS; 5798 rxq.qlen = info->ring_len; 5799 5800 rxq.dbuff = info->databuffer_size >> I40E_RXQ_CTX_DBUFF_SHIFT; 5801 5802 rxq.dsize = 1; 5803 rxq.crcstrip = 1; 5804 rxq.l2tsel = 1; 5805 5806 rxq.rxmax = info->max_pkt_size; 5807 rxq.tphrdesc_ena = 1; 5808 rxq.tphwdesc_ena = 1; 5809 rxq.tphdata_ena = 1; 5810 rxq.tphhead_ena = 1; 5811 rxq.lrxqthresh = 2; 5812 rxq.prefena = 1; 5813 5814 status = i40e_set_lan_rx_queue_context(hw, global_queue_num, &rxq); 5815 if (status != I40E_SUCCESS) 5816 return (EINVAL); 5817 5818 return (0); 5819 } 5820 5821 static void 5822 ixl_vf_config_vsi_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg, 5823 uint16_t msg_size) 5824 { 5825 struct i40e_virtchnl_vsi_queue_config_info *info; 5826 struct i40e_virtchnl_queue_pair_info *pair; 5827 int i; 5828 5829 if (msg_size < sizeof(*info)) { 5830 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES, 5831 I40E_ERR_PARAM); 5832 return; 5833 } 5834 5835 info = msg; 5836 if (info->num_queue_pairs == 0) { 5837 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES, 5838 I40E_ERR_PARAM); 5839 return; 5840 } 5841 5842 if (msg_size != sizeof(*info) + info->num_queue_pairs * sizeof(*pair)) { 5843 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES, 5844 I40E_ERR_PARAM); 5845 return; 5846 } 5847 5848 if (info->vsi_id != vf->vsi.vsi_num) { 5849 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES, 5850 I40E_ERR_PARAM); 5851 return; 5852 } 5853 5854 for (i = 0; i < info->num_queue_pairs; i++) { 5855 pair = &info->qpair[i]; 5856 5857 if (pair->txq.vsi_id != vf->vsi.vsi_num || 5858 pair->rxq.vsi_id != vf->vsi.vsi_num || 5859 pair->txq.queue_id != pair->rxq.queue_id || 5860 pair->txq.queue_id >= vf->vsi.num_queues) { 5861 5862 i40e_send_vf_nack(pf, vf, 5863 I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES, I40E_ERR_PARAM); 5864 return; 5865 } 5866 5867 if (ixl_vf_config_tx_queue(pf, vf, &pair->txq) != 0) { 5868 i40e_send_vf_nack(pf, vf, 5869 I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES, I40E_ERR_PARAM); 5870 return; 5871 } 5872 5873 if (ixl_vf_config_rx_queue(pf, vf, &pair->rxq) != 0) { 5874 i40e_send_vf_nack(pf, vf, 5875 I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES, I40E_ERR_PARAM); 5876 return; 5877 } 5878 } 5879 5880 ixl_send_vf_ack(pf, vf, I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES); 5881 } 5882 5883 static void 5884 ixl_vf_set_qctl(struct ixl_pf *pf, 5885 const struct i40e_virtchnl_vector_map *vector, 5886 enum i40e_queue_type cur_type, uint16_t cur_queue, 5887 enum i40e_queue_type *last_type, uint16_t *last_queue) 5888 { 5889 uint32_t offset, qctl; 5890 uint16_t itr_indx; 5891 5892 if (cur_type == I40E_QUEUE_TYPE_RX) { 5893 offset = I40E_QINT_RQCTL(cur_queue); 5894 itr_indx = vector->rxitr_idx; 5895 } else { 5896 offset = I40E_QINT_TQCTL(cur_queue); 5897 itr_indx = vector->txitr_idx; 5898 } 5899 5900 qctl = htole32((vector->vector_id << I40E_QINT_RQCTL_MSIX_INDX_SHIFT) | 5901 (*last_type << I40E_QINT_RQCTL_NEXTQ_TYPE_SHIFT) | 5902 (*last_queue << I40E_QINT_RQCTL_NEXTQ_INDX_SHIFT) | 5903 I40E_QINT_RQCTL_CAUSE_ENA_MASK | 5904 (itr_indx << I40E_QINT_RQCTL_ITR_INDX_SHIFT)); 5905 5906 wr32(&pf->hw, offset, qctl); 5907 5908 *last_type = cur_type; 5909 *last_queue = cur_queue; 5910 } 5911 5912 static void 5913 ixl_vf_config_vector(struct ixl_pf *pf, struct ixl_vf *vf, 5914 const struct i40e_virtchnl_vector_map *vector) 5915 { 5916 struct i40e_hw *hw; 5917 u_int qindex; 5918 enum i40e_queue_type type, last_type; 5919 uint32_t lnklst_reg; 5920 uint16_t rxq_map, txq_map, cur_queue, last_queue; 5921 5922 hw = &pf->hw; 5923 5924 rxq_map = vector->rxq_map; 5925 txq_map = vector->txq_map; 5926 5927 last_queue = IXL_END_OF_INTR_LNKLST; 5928 last_type = I40E_QUEUE_TYPE_RX; 5929 5930 /* 5931 * The datasheet says to optimize performance, RX queues and TX queues 5932 * should be interleaved in the interrupt linked list, so we process 5933 * both at once here. 5934 */ 5935 while ((rxq_map != 0) || (txq_map != 0)) { 5936 if (txq_map != 0) { 5937 qindex = ffs(txq_map) - 1; 5938 type = I40E_QUEUE_TYPE_TX; 5939 cur_queue = vf->vsi.first_queue + qindex; 5940 ixl_vf_set_qctl(pf, vector, type, cur_queue, 5941 &last_type, &last_queue); 5942 txq_map &= ~(1 << qindex); 5943 } 5944 5945 if (rxq_map != 0) { 5946 qindex = ffs(rxq_map) - 1; 5947 type = I40E_QUEUE_TYPE_RX; 5948 cur_queue = vf->vsi.first_queue + qindex; 5949 ixl_vf_set_qctl(pf, vector, type, cur_queue, 5950 &last_type, &last_queue); 5951 rxq_map &= ~(1 << qindex); 5952 } 5953 } 5954 5955 if (vector->vector_id == 0) 5956 lnklst_reg = I40E_VPINT_LNKLST0(vf->vf_num); 5957 else 5958 lnklst_reg = IXL_VPINT_LNKLSTN_REG(hw, vector->vector_id, 5959 vf->vf_num); 5960 wr32(hw, lnklst_reg, 5961 (last_queue << I40E_VPINT_LNKLST0_FIRSTQ_INDX_SHIFT) | 5962 (last_type << I40E_VPINT_LNKLST0_FIRSTQ_TYPE_SHIFT)); 5963 5964 ixl_flush(hw); 5965 } 5966 5967 static void 5968 ixl_vf_config_irq_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg, 5969 uint16_t msg_size) 5970 { 5971 struct i40e_virtchnl_irq_map_info *map; 5972 struct i40e_virtchnl_vector_map *vector; 5973 struct i40e_hw *hw; 5974 int i, largest_txq, largest_rxq; 5975 5976 hw = &pf->hw; 5977 5978 if (msg_size < sizeof(*map)) { 5979 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP, 5980 I40E_ERR_PARAM); 5981 return; 5982 } 5983 5984 map = msg; 5985 if (map->num_vectors == 0) { 5986 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP, 5987 I40E_ERR_PARAM); 5988 return; 5989 } 5990 5991 if (msg_size != sizeof(*map) + map->num_vectors * sizeof(*vector)) { 5992 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP, 5993 I40E_ERR_PARAM); 5994 return; 5995 } 5996 5997 for (i = 0; i < map->num_vectors; i++) { 5998 vector = &map->vecmap[i]; 5999 6000 if ((vector->vector_id >= hw->func_caps.num_msix_vectors_vf) || 6001 vector->vsi_id != vf->vsi.vsi_num) { 6002 i40e_send_vf_nack(pf, vf, 6003 I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP, I40E_ERR_PARAM); 6004 return; 6005 } 6006 6007 if (vector->rxq_map != 0) { 6008 largest_rxq = fls(vector->rxq_map) - 1; 6009 if (largest_rxq >= vf->vsi.num_queues) { 6010 i40e_send_vf_nack(pf, vf, 6011 I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP, 6012 I40E_ERR_PARAM); 6013 return; 6014 } 6015 } 6016 6017 if (vector->txq_map != 0) { 6018 largest_txq = fls(vector->txq_map) - 1; 6019 if (largest_txq >= vf->vsi.num_queues) { 6020 i40e_send_vf_nack(pf, vf, 6021 I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP, 6022 I40E_ERR_PARAM); 6023 return; 6024 } 6025 } 6026 6027 if (vector->rxitr_idx > IXL_MAX_ITR_IDX || 6028 vector->txitr_idx > IXL_MAX_ITR_IDX) { 6029 i40e_send_vf_nack(pf, vf, 6030 I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP, 6031 I40E_ERR_PARAM); 6032 return; 6033 } 6034 6035 ixl_vf_config_vector(pf, vf, vector); 6036 } 6037 6038 ixl_send_vf_ack(pf, vf, I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP); 6039 } 6040 6041 static void 6042 ixl_vf_enable_queues_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg, 6043 uint16_t msg_size) 6044 { 6045 struct i40e_virtchnl_queue_select *select; 6046 int error; 6047 6048 if (msg_size != sizeof(*select)) { 6049 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ENABLE_QUEUES, 6050 I40E_ERR_PARAM); 6051 return; 6052 } 6053 6054 select = msg; 6055 if (select->vsi_id != vf->vsi.vsi_num || 6056 select->rx_queues == 0 || select->tx_queues == 0) { 6057 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ENABLE_QUEUES, 6058 I40E_ERR_PARAM); 6059 return; 6060 } 6061 6062 error = ixl_enable_rings(&vf->vsi); 6063 if (error) { 6064 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ENABLE_QUEUES, 6065 I40E_ERR_TIMEOUT); 6066 return; 6067 } 6068 6069 ixl_send_vf_ack(pf, vf, I40E_VIRTCHNL_OP_ENABLE_QUEUES); 6070 } 6071 6072 static void 6073 ixl_vf_disable_queues_msg(struct ixl_pf *pf, struct ixl_vf *vf, 6074 void *msg, uint16_t msg_size) 6075 { 6076 struct i40e_virtchnl_queue_select *select; 6077 int error; 6078 6079 if (msg_size != sizeof(*select)) { 6080 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_DISABLE_QUEUES, 6081 I40E_ERR_PARAM); 6082 return; 6083 } 6084 6085 select = msg; 6086 if (select->vsi_id != vf->vsi.vsi_num || 6087 select->rx_queues == 0 || select->tx_queues == 0) { 6088 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_DISABLE_QUEUES, 6089 I40E_ERR_PARAM); 6090 return; 6091 } 6092 6093 error = ixl_disable_rings(&vf->vsi); 6094 if (error) { 6095 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_DISABLE_QUEUES, 6096 I40E_ERR_TIMEOUT); 6097 return; 6098 } 6099 6100 ixl_send_vf_ack(pf, vf, I40E_VIRTCHNL_OP_DISABLE_QUEUES); 6101 } 6102 6103 static boolean_t 6104 ixl_zero_mac(const uint8_t *addr) 6105 { 6106 uint8_t zero[ETHER_ADDR_LEN] = {0, 0, 0, 0, 0, 0}; 6107 6108 return (cmp_etheraddr(addr, zero)); 6109 } 6110 6111 static boolean_t 6112 ixl_bcast_mac(const uint8_t *addr) 6113 { 6114 6115 return (cmp_etheraddr(addr, ixl_bcast_addr)); 6116 } 6117 6118 static int 6119 ixl_vf_mac_valid(struct ixl_vf *vf, const uint8_t *addr) 6120 { 6121 6122 if (ixl_zero_mac(addr) || ixl_bcast_mac(addr)) 6123 return (EINVAL); 6124 6125 /* 6126 * If the VF is not allowed to change its MAC address, don't let it 6127 * set a MAC filter for an address that is not a multicast address and 6128 * is not its assigned MAC. 6129 */ 6130 if (!(vf->vf_flags & VF_FLAG_SET_MAC_CAP) && 6131 !(ETHER_IS_MULTICAST(addr) || cmp_etheraddr(addr, vf->mac))) 6132 return (EPERM); 6133 6134 return (0); 6135 } 6136 6137 static void 6138 ixl_vf_add_mac_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg, 6139 uint16_t msg_size) 6140 { 6141 struct i40e_virtchnl_ether_addr_list *addr_list; 6142 struct i40e_virtchnl_ether_addr *addr; 6143 struct ixl_vsi *vsi; 6144 int i; 6145 size_t expected_size; 6146 6147 vsi = &vf->vsi; 6148 6149 if (msg_size < sizeof(*addr_list)) { 6150 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS, 6151 I40E_ERR_PARAM); 6152 return; 6153 } 6154 6155 addr_list = msg; 6156 expected_size = sizeof(*addr_list) + 6157 addr_list->num_elements * sizeof(*addr); 6158 6159 if (addr_list->num_elements == 0 || 6160 addr_list->vsi_id != vsi->vsi_num || 6161 msg_size != expected_size) { 6162 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS, 6163 I40E_ERR_PARAM); 6164 return; 6165 } 6166 6167 for (i = 0; i < addr_list->num_elements; i++) { 6168 if (ixl_vf_mac_valid(vf, addr_list->list[i].addr) != 0) { 6169 i40e_send_vf_nack(pf, vf, 6170 I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS, I40E_ERR_PARAM); 6171 return; 6172 } 6173 } 6174 6175 for (i = 0; i < addr_list->num_elements; i++) { 6176 addr = &addr_list->list[i]; 6177 ixl_add_filter(vsi, addr->addr, IXL_VLAN_ANY); 6178 } 6179 6180 ixl_send_vf_ack(pf, vf, I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS); 6181 } 6182 6183 static void 6184 ixl_vf_del_mac_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg, 6185 uint16_t msg_size) 6186 { 6187 struct i40e_virtchnl_ether_addr_list *addr_list; 6188 struct i40e_virtchnl_ether_addr *addr; 6189 size_t expected_size; 6190 int i; 6191 6192 if (msg_size < sizeof(*addr_list)) { 6193 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS, 6194 I40E_ERR_PARAM); 6195 return; 6196 } 6197 6198 addr_list = msg; 6199 expected_size = sizeof(*addr_list) + 6200 addr_list->num_elements * sizeof(*addr); 6201 6202 if (addr_list->num_elements == 0 || 6203 addr_list->vsi_id != vf->vsi.vsi_num || 6204 msg_size != expected_size) { 6205 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS, 6206 I40E_ERR_PARAM); 6207 return; 6208 } 6209 6210 for (i = 0; i < addr_list->num_elements; i++) { 6211 addr = &addr_list->list[i]; 6212 if (ixl_zero_mac(addr->addr) || ixl_bcast_mac(addr->addr)) { 6213 i40e_send_vf_nack(pf, vf, 6214 I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS, I40E_ERR_PARAM); 6215 return; 6216 } 6217 } 6218 6219 for (i = 0; i < addr_list->num_elements; i++) { 6220 addr = &addr_list->list[i]; 6221 ixl_del_filter(&vf->vsi, addr->addr, IXL_VLAN_ANY); 6222 } 6223 6224 ixl_send_vf_ack(pf, vf, I40E_VIRTCHNL_OP_DEL_ETHER_ADDRESS); 6225 } 6226 6227 static enum i40e_status_code 6228 ixl_vf_enable_vlan_strip(struct ixl_pf *pf, struct ixl_vf *vf) 6229 { 6230 struct i40e_vsi_context vsi_ctx; 6231 6232 vsi_ctx.seid = vf->vsi.seid; 6233 6234 bzero(&vsi_ctx.info, sizeof(vsi_ctx.info)); 6235 vsi_ctx.info.valid_sections = htole16(I40E_AQ_VSI_PROP_VLAN_VALID); 6236 vsi_ctx.info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_ALL | 6237 I40E_AQ_VSI_PVLAN_EMOD_STR_BOTH; 6238 return (i40e_aq_update_vsi_params(&pf->hw, &vsi_ctx, NULL)); 6239 } 6240 6241 static void 6242 ixl_vf_add_vlan_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg, 6243 uint16_t msg_size) 6244 { 6245 struct i40e_virtchnl_vlan_filter_list *filter_list; 6246 enum i40e_status_code code; 6247 size_t expected_size; 6248 int i; 6249 6250 if (msg_size < sizeof(*filter_list)) { 6251 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_VLAN, 6252 I40E_ERR_PARAM); 6253 return; 6254 } 6255 6256 filter_list = msg; 6257 expected_size = sizeof(*filter_list) + 6258 filter_list->num_elements * sizeof(uint16_t); 6259 if (filter_list->num_elements == 0 || 6260 filter_list->vsi_id != vf->vsi.vsi_num || 6261 msg_size != expected_size) { 6262 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_VLAN, 6263 I40E_ERR_PARAM); 6264 return; 6265 } 6266 6267 if (!(vf->vf_flags & VF_FLAG_VLAN_CAP)) { 6268 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_VLAN, 6269 I40E_ERR_PARAM); 6270 return; 6271 } 6272 6273 for (i = 0; i < filter_list->num_elements; i++) { 6274 if (filter_list->vlan_id[i] > EVL_VLID_MASK) { 6275 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_VLAN, 6276 I40E_ERR_PARAM); 6277 return; 6278 } 6279 } 6280 6281 code = ixl_vf_enable_vlan_strip(pf, vf); 6282 if (code != I40E_SUCCESS) { 6283 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_VLAN, 6284 I40E_ERR_PARAM); 6285 } 6286 6287 for (i = 0; i < filter_list->num_elements; i++) 6288 ixl_add_filter(&vf->vsi, vf->mac, filter_list->vlan_id[i]); 6289 6290 ixl_send_vf_ack(pf, vf, I40E_VIRTCHNL_OP_ADD_VLAN); 6291 } 6292 6293 static void 6294 ixl_vf_del_vlan_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg, 6295 uint16_t msg_size) 6296 { 6297 struct i40e_virtchnl_vlan_filter_list *filter_list; 6298 int i; 6299 size_t expected_size; 6300 6301 if (msg_size < sizeof(*filter_list)) { 6302 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_DEL_VLAN, 6303 I40E_ERR_PARAM); 6304 return; 6305 } 6306 6307 filter_list = msg; 6308 expected_size = sizeof(*filter_list) + 6309 filter_list->num_elements * sizeof(uint16_t); 6310 if (filter_list->num_elements == 0 || 6311 filter_list->vsi_id != vf->vsi.vsi_num || 6312 msg_size != expected_size) { 6313 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_DEL_VLAN, 6314 I40E_ERR_PARAM); 6315 return; 6316 } 6317 6318 for (i = 0; i < filter_list->num_elements; i++) { 6319 if (filter_list->vlan_id[i] > EVL_VLID_MASK) { 6320 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_VLAN, 6321 I40E_ERR_PARAM); 6322 return; 6323 } 6324 } 6325 6326 if (!(vf->vf_flags & VF_FLAG_VLAN_CAP)) { 6327 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_VLAN, 6328 I40E_ERR_PARAM); 6329 return; 6330 } 6331 6332 for (i = 0; i < filter_list->num_elements; i++) 6333 ixl_del_filter(&vf->vsi, vf->mac, filter_list->vlan_id[i]); 6334 6335 ixl_send_vf_ack(pf, vf, I40E_VIRTCHNL_OP_DEL_VLAN); 6336 } 6337 6338 static void 6339 ixl_vf_config_promisc_msg(struct ixl_pf *pf, struct ixl_vf *vf, 6340 void *msg, uint16_t msg_size) 6341 { 6342 struct i40e_virtchnl_promisc_info *info; 6343 enum i40e_status_code code; 6344 6345 if (msg_size != sizeof(*info)) { 6346 i40e_send_vf_nack(pf, vf, 6347 I40E_VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE, I40E_ERR_PARAM); 6348 return; 6349 } 6350 6351 if (!(vf->vf_flags & VF_FLAG_PROMISC_CAP)) { 6352 i40e_send_vf_nack(pf, vf, 6353 I40E_VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE, I40E_ERR_PARAM); 6354 return; 6355 } 6356 6357 info = msg; 6358 if (info->vsi_id != vf->vsi.vsi_num) { 6359 i40e_send_vf_nack(pf, vf, 6360 I40E_VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE, I40E_ERR_PARAM); 6361 return; 6362 } 6363 6364 code = i40e_aq_set_vsi_unicast_promiscuous(&pf->hw, info->vsi_id, 6365 info->flags & I40E_FLAG_VF_UNICAST_PROMISC, NULL); 6366 if (code != I40E_SUCCESS) { 6367 i40e_send_vf_nack(pf, vf, 6368 I40E_VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE, code); 6369 return; 6370 } 6371 6372 code = i40e_aq_set_vsi_multicast_promiscuous(&pf->hw, info->vsi_id, 6373 info->flags & I40E_FLAG_VF_MULTICAST_PROMISC, NULL); 6374 if (code != I40E_SUCCESS) { 6375 i40e_send_vf_nack(pf, vf, 6376 I40E_VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE, code); 6377 return; 6378 } 6379 6380 ixl_send_vf_ack(pf, vf, I40E_VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE); 6381 } 6382 6383 static void 6384 ixl_vf_get_stats_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg, 6385 uint16_t msg_size) 6386 { 6387 struct i40e_virtchnl_queue_select *queue; 6388 6389 if (msg_size != sizeof(*queue)) { 6390 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_GET_STATS, 6391 I40E_ERR_PARAM); 6392 return; 6393 } 6394 6395 queue = msg; 6396 if (queue->vsi_id != vf->vsi.vsi_num) { 6397 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_GET_STATS, 6398 I40E_ERR_PARAM); 6399 return; 6400 } 6401 6402 ixl_update_eth_stats(&vf->vsi); 6403 6404 ixl_send_vf_msg(pf, vf, I40E_VIRTCHNL_OP_GET_STATS, 6405 I40E_SUCCESS, &vf->vsi.eth_stats, sizeof(vf->vsi.eth_stats)); 6406 } 6407 6408 static void 6409 ixl_handle_vf_msg(struct ixl_pf *pf, struct i40e_arq_event_info *event) 6410 { 6411 struct ixl_vf *vf; 6412 void *msg; 6413 uint16_t vf_num, msg_size; 6414 uint32_t opcode; 6415 6416 vf_num = le16toh(event->desc.retval) - pf->hw.func_caps.vf_base_id; 6417 opcode = le32toh(event->desc.cookie_high); 6418 6419 if (vf_num >= pf->num_vfs) { 6420 device_printf(pf->dev, "Got msg from illegal VF: %d\n", vf_num); 6421 return; 6422 } 6423 6424 vf = &pf->vfs[vf_num]; 6425 msg = event->msg_buf; 6426 msg_size = event->msg_len; 6427 6428 I40E_VC_DEBUG(pf, ixl_vc_opcode_level(opcode), 6429 "Got msg %s(%d) from VF-%d of size %d\n", 6430 ixl_vc_opcode_str(opcode), opcode, vf_num, msg_size); 6431 6432 switch (opcode) { 6433 case I40E_VIRTCHNL_OP_VERSION: 6434 ixl_vf_version_msg(pf, vf, msg, msg_size); 6435 break; 6436 case I40E_VIRTCHNL_OP_RESET_VF: 6437 ixl_vf_reset_msg(pf, vf, msg, msg_size); 6438 break; 6439 case I40E_VIRTCHNL_OP_GET_VF_RESOURCES: 6440 ixl_vf_get_resources_msg(pf, vf, msg, msg_size); 6441 break; 6442 case I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES: 6443 ixl_vf_config_vsi_msg(pf, vf, msg, msg_size); 6444 break; 6445 case I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP: 6446 ixl_vf_config_irq_msg(pf, vf, msg, msg_size); 6447 break; 6448 case I40E_VIRTCHNL_OP_ENABLE_QUEUES: 6449 ixl_vf_enable_queues_msg(pf, vf, msg, msg_size); 6450 break; 6451 case I40E_VIRTCHNL_OP_DISABLE_QUEUES: 6452 ixl_vf_disable_queues_msg(pf, vf, msg, msg_size); 6453 break; 6454 case I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS: 6455 ixl_vf_add_mac_msg(pf, vf, msg, msg_size); 6456 break; 6457 case I40E_VIRTCHNL_OP_DEL_ETHER_ADDRESS: 6458 ixl_vf_del_mac_msg(pf, vf, msg, msg_size); 6459 break; 6460 case I40E_VIRTCHNL_OP_ADD_VLAN: 6461 ixl_vf_add_vlan_msg(pf, vf, msg, msg_size); 6462 break; 6463 case I40E_VIRTCHNL_OP_DEL_VLAN: 6464 ixl_vf_del_vlan_msg(pf, vf, msg, msg_size); 6465 break; 6466 case I40E_VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE: 6467 ixl_vf_config_promisc_msg(pf, vf, msg, msg_size); 6468 break; 6469 case I40E_VIRTCHNL_OP_GET_STATS: 6470 ixl_vf_get_stats_msg(pf, vf, msg, msg_size); 6471 break; 6472 6473 /* These two opcodes have been superseded by CONFIG_VSI_QUEUES. */ 6474 case I40E_VIRTCHNL_OP_CONFIG_TX_QUEUE: 6475 case I40E_VIRTCHNL_OP_CONFIG_RX_QUEUE: 6476 default: 6477 i40e_send_vf_nack(pf, vf, opcode, I40E_ERR_NOT_IMPLEMENTED); 6478 break; 6479 } 6480 } 6481 6482 /* Handle any VFs that have reset themselves via a Function Level Reset(FLR). */ 6483 static void 6484 ixl_handle_vflr(void *arg, int pending) 6485 { 6486 struct ixl_pf *pf; 6487 struct i40e_hw *hw; 6488 uint16_t global_vf_num; 6489 uint32_t vflrstat_index, vflrstat_mask, vflrstat, icr0; 6490 int i; 6491 6492 pf = arg; 6493 hw = &pf->hw; 6494 6495 IXL_PF_LOCK(pf); 6496 for (i = 0; i < pf->num_vfs; i++) { 6497 global_vf_num = hw->func_caps.vf_base_id + i; 6498 6499 vflrstat_index = IXL_GLGEN_VFLRSTAT_INDEX(global_vf_num); 6500 vflrstat_mask = IXL_GLGEN_VFLRSTAT_MASK(global_vf_num); 6501 vflrstat = rd32(hw, I40E_GLGEN_VFLRSTAT(vflrstat_index)); 6502 if (vflrstat & vflrstat_mask) { 6503 wr32(hw, I40E_GLGEN_VFLRSTAT(vflrstat_index), 6504 vflrstat_mask); 6505 6506 ixl_reinit_vf(pf, &pf->vfs[i]); 6507 } 6508 } 6509 6510 icr0 = rd32(hw, I40E_PFINT_ICR0_ENA); 6511 icr0 |= I40E_PFINT_ICR0_ENA_VFLR_MASK; 6512 wr32(hw, I40E_PFINT_ICR0_ENA, icr0); 6513 ixl_flush(hw); 6514 6515 IXL_PF_UNLOCK(pf); 6516 } 6517 6518 static int 6519 ixl_adminq_err_to_errno(enum i40e_admin_queue_err err) 6520 { 6521 6522 switch (err) { 6523 case I40E_AQ_RC_EPERM: 6524 return (EPERM); 6525 case I40E_AQ_RC_ENOENT: 6526 return (ENOENT); 6527 case I40E_AQ_RC_ESRCH: 6528 return (ESRCH); 6529 case I40E_AQ_RC_EINTR: 6530 return (EINTR); 6531 case I40E_AQ_RC_EIO: 6532 return (EIO); 6533 case I40E_AQ_RC_ENXIO: 6534 return (ENXIO); 6535 case I40E_AQ_RC_E2BIG: 6536 return (E2BIG); 6537 case I40E_AQ_RC_EAGAIN: 6538 return (EAGAIN); 6539 case I40E_AQ_RC_ENOMEM: 6540 return (ENOMEM); 6541 case I40E_AQ_RC_EACCES: 6542 return (EACCES); 6543 case I40E_AQ_RC_EFAULT: 6544 return (EFAULT); 6545 case I40E_AQ_RC_EBUSY: 6546 return (EBUSY); 6547 case I40E_AQ_RC_EEXIST: 6548 return (EEXIST); 6549 case I40E_AQ_RC_EINVAL: 6550 return (EINVAL); 6551 case I40E_AQ_RC_ENOTTY: 6552 return (ENOTTY); 6553 case I40E_AQ_RC_ENOSPC: 6554 return (ENOSPC); 6555 case I40E_AQ_RC_ENOSYS: 6556 return (ENOSYS); 6557 case I40E_AQ_RC_ERANGE: 6558 return (ERANGE); 6559 case I40E_AQ_RC_EFLUSHED: 6560 return (EINVAL); /* No exact equivalent in errno.h */ 6561 case I40E_AQ_RC_BAD_ADDR: 6562 return (EFAULT); 6563 case I40E_AQ_RC_EMODE: 6564 return (EPERM); 6565 case I40E_AQ_RC_EFBIG: 6566 return (EFBIG); 6567 default: 6568 return (EINVAL); 6569 } 6570 } 6571 6572 static int 6573 ixl_iov_init(device_t dev, uint16_t num_vfs, const nvlist_t *params) 6574 { 6575 struct ixl_pf *pf; 6576 struct i40e_hw *hw; 6577 struct ixl_vsi *pf_vsi; 6578 enum i40e_status_code ret; 6579 int i, error; 6580 6581 pf = device_get_softc(dev); 6582 hw = &pf->hw; 6583 pf_vsi = &pf->vsi; 6584 6585 IXL_PF_LOCK(pf); 6586 pf->vfs = malloc(sizeof(struct ixl_vf) * num_vfs, M_IXL, M_NOWAIT | 6587 M_ZERO); 6588 6589 if (pf->vfs == NULL) { 6590 error = ENOMEM; 6591 goto fail; 6592 } 6593 6594 for (i = 0; i < num_vfs; i++) 6595 sysctl_ctx_init(&pf->vfs[i].ctx); 6596 6597 ret = i40e_aq_add_veb(hw, pf_vsi->uplink_seid, pf_vsi->seid, 6598 1, FALSE, FALSE, &pf->veb_seid, NULL); 6599 if (ret != I40E_SUCCESS) { 6600 error = ixl_adminq_err_to_errno(hw->aq.asq_last_status); 6601 device_printf(dev, "add_veb failed; code=%d error=%d", ret, 6602 error); 6603 goto fail; 6604 } 6605 6606 ixl_configure_msix(pf); 6607 ixl_enable_adminq(hw); 6608 6609 pf->num_vfs = num_vfs; 6610 IXL_PF_UNLOCK(pf); 6611 return (0); 6612 6613 fail: 6614 free(pf->vfs, M_IXL); 6615 pf->vfs = NULL; 6616 IXL_PF_UNLOCK(pf); 6617 return (error); 6618 } 6619 6620 static void 6621 ixl_iov_uninit(device_t dev) 6622 { 6623 struct ixl_pf *pf; 6624 struct i40e_hw *hw; 6625 struct ixl_vsi *vsi; 6626 struct ifnet *ifp; 6627 struct ixl_vf *vfs; 6628 int i, num_vfs; 6629 6630 pf = device_get_softc(dev); 6631 hw = &pf->hw; 6632 vsi = &pf->vsi; 6633 ifp = vsi->ifp; 6634 6635 IXL_PF_LOCK(pf); 6636 for (i = 0; i < pf->num_vfs; i++) { 6637 if (pf->vfs[i].vsi.seid != 0) 6638 i40e_aq_delete_element(hw, pf->vfs[i].vsi.seid, NULL); 6639 } 6640 6641 if (pf->veb_seid != 0) { 6642 i40e_aq_delete_element(hw, pf->veb_seid, NULL); 6643 pf->veb_seid = 0; 6644 } 6645 6646 #if __FreeBSD_version > 1100022 6647 if ((if_getdrvflags(ifp) & IFF_DRV_RUNNING) == 0) 6648 #else 6649 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) 6650 #endif 6651 ixl_disable_intr(vsi); 6652 6653 vfs = pf->vfs; 6654 num_vfs = pf->num_vfs; 6655 6656 pf->vfs = NULL; 6657 pf->num_vfs = 0; 6658 IXL_PF_UNLOCK(pf); 6659 6660 /* Do this after the unlock as sysctl_ctx_free might sleep. */ 6661 for (i = 0; i < num_vfs; i++) 6662 sysctl_ctx_free(&vfs[i].ctx); 6663 free(vfs, M_IXL); 6664 } 6665 6666 static int 6667 ixl_add_vf(device_t dev, uint16_t vfnum, const nvlist_t *params) 6668 { 6669 char sysctl_name[QUEUE_NAME_LEN]; 6670 struct ixl_pf *pf; 6671 struct ixl_vf *vf; 6672 const void *mac; 6673 size_t size; 6674 int error; 6675 6676 pf = device_get_softc(dev); 6677 vf = &pf->vfs[vfnum]; 6678 6679 IXL_PF_LOCK(pf); 6680 vf->vf_num = vfnum; 6681 6682 vf->vsi.back = pf; 6683 vf->vf_flags = VF_FLAG_ENABLED; 6684 SLIST_INIT(&vf->vsi.ftl); 6685 6686 error = ixl_vf_setup_vsi(pf, vf); 6687 if (error != 0) 6688 goto out; 6689 6690 if (nvlist_exists_binary(params, "mac-addr")) { 6691 mac = nvlist_get_binary(params, "mac-addr", &size); 6692 bcopy(mac, vf->mac, ETHER_ADDR_LEN); 6693 6694 if (nvlist_get_bool(params, "allow-set-mac")) 6695 vf->vf_flags |= VF_FLAG_SET_MAC_CAP; 6696 } else 6697 /* 6698 * If the administrator has not specified a MAC address then 6699 * we must allow the VF to choose one. 6700 */ 6701 vf->vf_flags |= VF_FLAG_SET_MAC_CAP; 6702 6703 if (nvlist_get_bool(params, "mac-anti-spoof")) 6704 vf->vf_flags |= VF_FLAG_MAC_ANTI_SPOOF; 6705 6706 if (nvlist_get_bool(params, "allow-promisc")) 6707 vf->vf_flags |= VF_FLAG_PROMISC_CAP; 6708 6709 vf->vf_flags |= VF_FLAG_VLAN_CAP; 6710 6711 ixl_reset_vf(pf, vf); 6712 out: 6713 IXL_PF_UNLOCK(pf); 6714 if (error == 0) { 6715 snprintf(sysctl_name, sizeof(sysctl_name), "vf%d", vfnum); 6716 ixl_add_vsi_sysctls(pf, &vf->vsi, &vf->ctx, sysctl_name); 6717 } 6718 6719 return (error); 6720 } 6721 #endif /* PCI_IOV */ 6722