1 /****************************************************************************** 2 3 Copyright (c) 2013-2015, Intel Corporation 4 All rights reserved. 5 6 Redistribution and use in source and binary forms, with or without 7 modification, are permitted provided that the following conditions are met: 8 9 1. Redistributions of source code must retain the above copyright notice, 10 this list of conditions and the following disclaimer. 11 12 2. Redistributions in binary form must reproduce the above copyright 13 notice, this list of conditions and the following disclaimer in the 14 documentation and/or other materials provided with the distribution. 15 16 3. Neither the name of the Intel Corporation nor the names of its 17 contributors may be used to endorse or promote products derived from 18 this software without specific prior written permission. 19 20 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 23 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 24 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 25 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 26 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 27 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 28 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 29 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 30 POSSIBILITY OF SUCH DAMAGE. 31 32 ******************************************************************************/ 33 /*$FreeBSD$*/ 34 35 #ifndef IXL_STANDALONE_BUILD 36 #include "opt_inet.h" 37 #include "opt_inet6.h" 38 #include "opt_rss.h" 39 #endif 40 41 #include "ixl.h" 42 #include "ixl_pf.h" 43 44 #ifdef RSS 45 #include <net/rss_config.h> 46 #endif 47 48 /********************************************************************* 49 * Driver version 50 *********************************************************************/ 51 char ixl_driver_version[] = "1.4.1"; 52 53 /********************************************************************* 54 * PCI Device ID Table 55 * 56 * Used by probe to select devices to load on 57 * Last field stores an index into ixl_strings 58 * Last entry must be all 0s 59 * 60 * { Vendor ID, Device ID, SubVendor ID, SubDevice ID, String Index } 61 *********************************************************************/ 62 63 static ixl_vendor_info_t ixl_vendor_info_array[] = 64 { 65 {I40E_INTEL_VENDOR_ID, I40E_DEV_ID_SFP_XL710, 0, 0, 0}, 66 {I40E_INTEL_VENDOR_ID, I40E_DEV_ID_KX_A, 0, 0, 0}, 67 {I40E_INTEL_VENDOR_ID, I40E_DEV_ID_KX_B, 0, 0, 0}, 68 {I40E_INTEL_VENDOR_ID, I40E_DEV_ID_KX_C, 0, 0, 0}, 69 {I40E_INTEL_VENDOR_ID, I40E_DEV_ID_QSFP_A, 0, 0, 0}, 70 {I40E_INTEL_VENDOR_ID, I40E_DEV_ID_QSFP_B, 0, 0, 0}, 71 {I40E_INTEL_VENDOR_ID, I40E_DEV_ID_QSFP_C, 0, 0, 0}, 72 {I40E_INTEL_VENDOR_ID, I40E_DEV_ID_10G_BASE_T, 0, 0, 0}, 73 {I40E_INTEL_VENDOR_ID, I40E_DEV_ID_20G_KR2, 0, 0, 0}, 74 /* required last entry */ 75 {0, 0, 0, 0, 0} 76 }; 77 78 /********************************************************************* 79 * Table of branding strings 80 *********************************************************************/ 81 82 static char *ixl_strings[] = { 83 "Intel(R) Ethernet Connection XL710 Driver" 84 }; 85 86 87 /********************************************************************* 88 * Function prototypes 89 *********************************************************************/ 90 static int ixl_probe(device_t); 91 static int ixl_attach(device_t); 92 static int ixl_detach(device_t); 93 static int ixl_shutdown(device_t); 94 static int ixl_get_hw_capabilities(struct ixl_pf *); 95 static void ixl_cap_txcsum_tso(struct ixl_vsi *, struct ifnet *, int); 96 static int ixl_ioctl(struct ifnet *, u_long, caddr_t); 97 static void ixl_init(void *); 98 static void ixl_init_locked(struct ixl_pf *); 99 static void ixl_stop(struct ixl_pf *); 100 static void ixl_media_status(struct ifnet *, struct ifmediareq *); 101 static int ixl_media_change(struct ifnet *); 102 static void ixl_update_link_status(struct ixl_pf *); 103 static int ixl_allocate_pci_resources(struct ixl_pf *); 104 static u16 ixl_get_bus_info(struct i40e_hw *, device_t); 105 static int ixl_setup_stations(struct ixl_pf *); 106 static int ixl_switch_config(struct ixl_pf *); 107 static int ixl_initialize_vsi(struct ixl_vsi *); 108 static int ixl_assign_vsi_msix(struct ixl_pf *); 109 static int ixl_assign_vsi_legacy(struct ixl_pf *); 110 static int ixl_init_msix(struct ixl_pf *); 111 static void ixl_configure_msix(struct ixl_pf *); 112 static void ixl_configure_itr(struct ixl_pf *); 113 static void ixl_configure_legacy(struct ixl_pf *); 114 static void ixl_free_pci_resources(struct ixl_pf *); 115 static void ixl_local_timer(void *); 116 static int ixl_setup_interface(device_t, struct ixl_vsi *); 117 static void ixl_link_event(struct ixl_pf *, struct i40e_arq_event_info *); 118 static void ixl_config_rss(struct ixl_vsi *); 119 static void ixl_set_queue_rx_itr(struct ixl_queue *); 120 static void ixl_set_queue_tx_itr(struct ixl_queue *); 121 static int ixl_set_advertised_speeds(struct ixl_pf *, int); 122 123 static int ixl_enable_rings(struct ixl_vsi *); 124 static int ixl_disable_rings(struct ixl_vsi *); 125 static void ixl_enable_intr(struct ixl_vsi *); 126 static void ixl_disable_intr(struct ixl_vsi *); 127 static void ixl_disable_rings_intr(struct ixl_vsi *); 128 129 static void ixl_enable_adminq(struct i40e_hw *); 130 static void ixl_disable_adminq(struct i40e_hw *); 131 static void ixl_enable_queue(struct i40e_hw *, int); 132 static void ixl_disable_queue(struct i40e_hw *, int); 133 static void ixl_enable_legacy(struct i40e_hw *); 134 static void ixl_disable_legacy(struct i40e_hw *); 135 136 static void ixl_set_promisc(struct ixl_vsi *); 137 static void ixl_add_multi(struct ixl_vsi *); 138 static void ixl_del_multi(struct ixl_vsi *); 139 static void ixl_register_vlan(void *, struct ifnet *, u16); 140 static void ixl_unregister_vlan(void *, struct ifnet *, u16); 141 static void ixl_setup_vlan_filters(struct ixl_vsi *); 142 143 static void ixl_init_filters(struct ixl_vsi *); 144 static void ixl_reconfigure_filters(struct ixl_vsi *vsi); 145 static void ixl_add_filter(struct ixl_vsi *, u8 *, s16 vlan); 146 static void ixl_del_filter(struct ixl_vsi *, u8 *, s16 vlan); 147 static void ixl_add_hw_filters(struct ixl_vsi *, int, int); 148 static void ixl_del_hw_filters(struct ixl_vsi *, int); 149 static struct ixl_mac_filter * 150 ixl_find_filter(struct ixl_vsi *, u8 *, s16); 151 static void ixl_add_mc_filter(struct ixl_vsi *, u8 *); 152 static void ixl_free_mac_filters(struct ixl_vsi *vsi); 153 154 155 /* Sysctl debug interface */ 156 static int ixl_debug_info(SYSCTL_HANDLER_ARGS); 157 static void ixl_print_debug_info(struct ixl_pf *); 158 159 /* The MSI/X Interrupt handlers */ 160 static void ixl_intr(void *); 161 static void ixl_msix_que(void *); 162 static void ixl_msix_adminq(void *); 163 static void ixl_handle_mdd_event(struct ixl_pf *); 164 165 /* Deferred interrupt tasklets */ 166 static void ixl_do_adminq(void *, int); 167 168 /* Sysctl handlers */ 169 static int ixl_set_flowcntl(SYSCTL_HANDLER_ARGS); 170 static int ixl_set_advertise(SYSCTL_HANDLER_ARGS); 171 static int ixl_current_speed(SYSCTL_HANDLER_ARGS); 172 static int ixl_sysctl_show_fw(SYSCTL_HANDLER_ARGS); 173 174 /* Statistics */ 175 static void ixl_add_hw_stats(struct ixl_pf *); 176 static void ixl_add_sysctls_mac_stats(struct sysctl_ctx_list *, 177 struct sysctl_oid_list *, struct i40e_hw_port_stats *); 178 static void ixl_add_sysctls_eth_stats(struct sysctl_ctx_list *, 179 struct sysctl_oid_list *, 180 struct i40e_eth_stats *); 181 static void ixl_update_stats_counters(struct ixl_pf *); 182 static void ixl_update_eth_stats(struct ixl_vsi *); 183 static void ixl_update_vsi_stats(struct ixl_vsi *); 184 static void ixl_pf_reset_stats(struct ixl_pf *); 185 static void ixl_vsi_reset_stats(struct ixl_vsi *); 186 static void ixl_stat_update48(struct i40e_hw *, u32, u32, bool, 187 u64 *, u64 *); 188 static void ixl_stat_update32(struct i40e_hw *, u32, bool, 189 u64 *, u64 *); 190 191 #ifdef IXL_DEBUG_SYSCTL 192 static int ixl_sysctl_link_status(SYSCTL_HANDLER_ARGS); 193 static int ixl_sysctl_phy_abilities(SYSCTL_HANDLER_ARGS); 194 static int ixl_sysctl_sw_filter_list(SYSCTL_HANDLER_ARGS); 195 static int ixl_sysctl_hw_res_alloc(SYSCTL_HANDLER_ARGS); 196 static int ixl_sysctl_switch_config(SYSCTL_HANDLER_ARGS); 197 #endif 198 199 #ifdef PCI_IOV 200 static int ixl_adminq_err_to_errno(enum i40e_admin_queue_err err); 201 202 static int ixl_init_iov(device_t dev, uint16_t num_vfs, const nvlist_t*); 203 static void ixl_uninit_iov(device_t dev); 204 static int ixl_add_vf(device_t dev, uint16_t vfnum, const nvlist_t*); 205 206 static void ixl_handle_vf_msg(struct ixl_pf *, 207 struct i40e_arq_event_info *); 208 static void ixl_handle_vflr(void *arg, int pending); 209 210 static void ixl_reset_vf(struct ixl_pf *pf, struct ixl_vf *vf); 211 static void ixl_reinit_vf(struct ixl_pf *pf, struct ixl_vf *vf); 212 #endif 213 214 /********************************************************************* 215 * FreeBSD Device Interface Entry Points 216 *********************************************************************/ 217 218 static device_method_t ixl_methods[] = { 219 /* Device interface */ 220 DEVMETHOD(device_probe, ixl_probe), 221 DEVMETHOD(device_attach, ixl_attach), 222 DEVMETHOD(device_detach, ixl_detach), 223 DEVMETHOD(device_shutdown, ixl_shutdown), 224 #ifdef PCI_IOV 225 DEVMETHOD(pci_init_iov, ixl_init_iov), 226 DEVMETHOD(pci_uninit_iov, ixl_uninit_iov), 227 DEVMETHOD(pci_add_vf, ixl_add_vf), 228 #endif 229 {0, 0} 230 }; 231 232 static driver_t ixl_driver = { 233 "ixl", ixl_methods, sizeof(struct ixl_pf), 234 }; 235 236 devclass_t ixl_devclass; 237 DRIVER_MODULE(ixl, pci, ixl_driver, ixl_devclass, 0, 0); 238 239 MODULE_DEPEND(ixl, pci, 1, 1, 1); 240 MODULE_DEPEND(ixl, ether, 1, 1, 1); 241 242 #ifdef DEV_NETMAP 243 MODULE_DEPEND(ixl, netmap, 1, 1, 1); 244 #endif /* DEV_NETMAP */ 245 246 247 /* 248 ** Global reset mutex 249 */ 250 static struct mtx ixl_reset_mtx; 251 252 /* 253 ** TUNEABLE PARAMETERS: 254 */ 255 256 static SYSCTL_NODE(_hw, OID_AUTO, ixl, CTLFLAG_RD, 0, 257 "IXL driver parameters"); 258 259 /* 260 * MSIX should be the default for best performance, 261 * but this allows it to be forced off for testing. 262 */ 263 static int ixl_enable_msix = 1; 264 TUNABLE_INT("hw.ixl.enable_msix", &ixl_enable_msix); 265 SYSCTL_INT(_hw_ixl, OID_AUTO, enable_msix, CTLFLAG_RDTUN, &ixl_enable_msix, 0, 266 "Enable MSI-X interrupts"); 267 268 /* 269 ** Number of descriptors per ring: 270 ** - TX and RX are the same size 271 */ 272 static int ixl_ringsz = DEFAULT_RING; 273 TUNABLE_INT("hw.ixl.ringsz", &ixl_ringsz); 274 SYSCTL_INT(_hw_ixl, OID_AUTO, ring_size, CTLFLAG_RDTUN, 275 &ixl_ringsz, 0, "Descriptor Ring Size"); 276 277 /* 278 ** This can be set manually, if left as 0 the 279 ** number of queues will be calculated based 280 ** on cpus and msix vectors available. 281 */ 282 int ixl_max_queues = 0; 283 TUNABLE_INT("hw.ixl.max_queues", &ixl_max_queues); 284 SYSCTL_INT(_hw_ixl, OID_AUTO, max_queues, CTLFLAG_RDTUN, 285 &ixl_max_queues, 0, "Number of Queues"); 286 287 /* 288 ** Controls for Interrupt Throttling 289 ** - true/false for dynamic adjustment 290 ** - default values for static ITR 291 */ 292 int ixl_dynamic_rx_itr = 0; 293 TUNABLE_INT("hw.ixl.dynamic_rx_itr", &ixl_dynamic_rx_itr); 294 SYSCTL_INT(_hw_ixl, OID_AUTO, dynamic_rx_itr, CTLFLAG_RDTUN, 295 &ixl_dynamic_rx_itr, 0, "Dynamic RX Interrupt Rate"); 296 297 int ixl_dynamic_tx_itr = 0; 298 TUNABLE_INT("hw.ixl.dynamic_tx_itr", &ixl_dynamic_tx_itr); 299 SYSCTL_INT(_hw_ixl, OID_AUTO, dynamic_tx_itr, CTLFLAG_RDTUN, 300 &ixl_dynamic_tx_itr, 0, "Dynamic TX Interrupt Rate"); 301 302 int ixl_rx_itr = IXL_ITR_8K; 303 TUNABLE_INT("hw.ixl.rx_itr", &ixl_rx_itr); 304 SYSCTL_INT(_hw_ixl, OID_AUTO, rx_itr, CTLFLAG_RDTUN, 305 &ixl_rx_itr, 0, "RX Interrupt Rate"); 306 307 int ixl_tx_itr = IXL_ITR_4K; 308 TUNABLE_INT("hw.ixl.tx_itr", &ixl_tx_itr); 309 SYSCTL_INT(_hw_ixl, OID_AUTO, tx_itr, CTLFLAG_RDTUN, 310 &ixl_tx_itr, 0, "TX Interrupt Rate"); 311 312 #ifdef IXL_FDIR 313 static int ixl_enable_fdir = 1; 314 TUNABLE_INT("hw.ixl.enable_fdir", &ixl_enable_fdir); 315 /* Rate at which we sample */ 316 int ixl_atr_rate = 20; 317 TUNABLE_INT("hw.ixl.atr_rate", &ixl_atr_rate); 318 #endif 319 320 #ifdef DEV_NETMAP 321 #define NETMAP_IXL_MAIN /* only bring in one part of the netmap code */ 322 #include <dev/netmap/if_ixl_netmap.h> 323 #endif /* DEV_NETMAP */ 324 325 static char *ixl_fc_string[6] = { 326 "None", 327 "Rx", 328 "Tx", 329 "Full", 330 "Priority", 331 "Default" 332 }; 333 334 static MALLOC_DEFINE(M_IXL, "ixl", "ixl driver allocations"); 335 336 static uint8_t ixl_bcast_addr[ETHER_ADDR_LEN] = 337 {0xff, 0xff, 0xff, 0xff, 0xff, 0xff}; 338 339 /********************************************************************* 340 * Device identification routine 341 * 342 * ixl_probe determines if the driver should be loaded on 343 * the hardware based on PCI vendor/device id of the device. 344 * 345 * return BUS_PROBE_DEFAULT on success, positive on failure 346 *********************************************************************/ 347 348 static int 349 ixl_probe(device_t dev) 350 { 351 ixl_vendor_info_t *ent; 352 353 u16 pci_vendor_id, pci_device_id; 354 u16 pci_subvendor_id, pci_subdevice_id; 355 char device_name[256]; 356 static bool lock_init = FALSE; 357 358 INIT_DEBUGOUT("ixl_probe: begin"); 359 360 pci_vendor_id = pci_get_vendor(dev); 361 if (pci_vendor_id != I40E_INTEL_VENDOR_ID) 362 return (ENXIO); 363 364 pci_device_id = pci_get_device(dev); 365 pci_subvendor_id = pci_get_subvendor(dev); 366 pci_subdevice_id = pci_get_subdevice(dev); 367 368 ent = ixl_vendor_info_array; 369 while (ent->vendor_id != 0) { 370 if ((pci_vendor_id == ent->vendor_id) && 371 (pci_device_id == ent->device_id) && 372 373 ((pci_subvendor_id == ent->subvendor_id) || 374 (ent->subvendor_id == 0)) && 375 376 ((pci_subdevice_id == ent->subdevice_id) || 377 (ent->subdevice_id == 0))) { 378 sprintf(device_name, "%s, Version - %s", 379 ixl_strings[ent->index], 380 ixl_driver_version); 381 device_set_desc_copy(dev, device_name); 382 /* One shot mutex init */ 383 if (lock_init == FALSE) { 384 lock_init = TRUE; 385 mtx_init(&ixl_reset_mtx, 386 "ixl_reset", 387 "IXL RESET Lock", MTX_DEF); 388 } 389 return (BUS_PROBE_DEFAULT); 390 } 391 ent++; 392 } 393 return (ENXIO); 394 } 395 396 /********************************************************************* 397 * Device initialization routine 398 * 399 * The attach entry point is called when the driver is being loaded. 400 * This routine identifies the type of hardware, allocates all resources 401 * and initializes the hardware. 402 * 403 * return 0 on success, positive on failure 404 *********************************************************************/ 405 406 static int 407 ixl_attach(device_t dev) 408 { 409 struct ixl_pf *pf; 410 struct i40e_hw *hw; 411 struct ixl_vsi *vsi; 412 u16 bus; 413 int error = 0; 414 #ifdef PCI_IOV 415 nvlist_t *pf_schema, *vf_schema; 416 int iov_error; 417 #endif 418 419 INIT_DEBUGOUT("ixl_attach: begin"); 420 421 /* Allocate, clear, and link in our primary soft structure */ 422 pf = device_get_softc(dev); 423 pf->dev = pf->osdep.dev = dev; 424 hw = &pf->hw; 425 426 /* 427 ** Note this assumes we have a single embedded VSI, 428 ** this could be enhanced later to allocate multiple 429 */ 430 vsi = &pf->vsi; 431 vsi->dev = pf->dev; 432 433 /* Core Lock Init*/ 434 IXL_PF_LOCK_INIT(pf, device_get_nameunit(dev)); 435 436 /* Set up the timer callout */ 437 callout_init_mtx(&pf->timer, &pf->pf_mtx, 0); 438 439 /* Set up sysctls */ 440 SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev), 441 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), 442 OID_AUTO, "fc", CTLTYPE_INT | CTLFLAG_RW, 443 pf, 0, ixl_set_flowcntl, "I", "Flow Control"); 444 445 SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev), 446 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), 447 OID_AUTO, "advertise_speed", CTLTYPE_INT | CTLFLAG_RW, 448 pf, 0, ixl_set_advertise, "I", "Advertised Speed"); 449 450 SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev), 451 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), 452 OID_AUTO, "current_speed", CTLTYPE_STRING | CTLFLAG_RD, 453 pf, 0, ixl_current_speed, "A", "Current Port Speed"); 454 455 SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev), 456 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), 457 OID_AUTO, "fw_version", CTLTYPE_STRING | CTLFLAG_RD, 458 pf, 0, ixl_sysctl_show_fw, "A", "Firmware version"); 459 460 SYSCTL_ADD_INT(device_get_sysctl_ctx(dev), 461 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), 462 OID_AUTO, "rx_itr", CTLFLAG_RW, 463 &ixl_rx_itr, IXL_ITR_8K, "RX ITR"); 464 465 SYSCTL_ADD_INT(device_get_sysctl_ctx(dev), 466 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), 467 OID_AUTO, "dynamic_rx_itr", CTLFLAG_RW, 468 &ixl_dynamic_rx_itr, 0, "Dynamic RX ITR"); 469 470 SYSCTL_ADD_INT(device_get_sysctl_ctx(dev), 471 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), 472 OID_AUTO, "tx_itr", CTLFLAG_RW, 473 &ixl_tx_itr, IXL_ITR_4K, "TX ITR"); 474 475 SYSCTL_ADD_INT(device_get_sysctl_ctx(dev), 476 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), 477 OID_AUTO, "dynamic_tx_itr", CTLFLAG_RW, 478 &ixl_dynamic_tx_itr, 0, "Dynamic TX ITR"); 479 480 #ifdef IXL_DEBUG_SYSCTL 481 SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev), 482 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), 483 OID_AUTO, "link_status", CTLTYPE_STRING | CTLFLAG_RD, 484 pf, 0, ixl_sysctl_link_status, "A", "Current Link Status"); 485 486 SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev), 487 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), 488 OID_AUTO, "phy_abilities", CTLTYPE_STRING | CTLFLAG_RD, 489 pf, 0, ixl_sysctl_phy_abilities, "A", "PHY Abilities"); 490 491 SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev), 492 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), 493 OID_AUTO, "filter_list", CTLTYPE_STRING | CTLFLAG_RD, 494 pf, 0, ixl_sysctl_sw_filter_list, "A", "SW Filter List"); 495 496 SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev), 497 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), 498 OID_AUTO, "hw_res_alloc", CTLTYPE_STRING | CTLFLAG_RD, 499 pf, 0, ixl_sysctl_hw_res_alloc, "A", "HW Resource Allocation"); 500 501 SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev), 502 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), 503 OID_AUTO, "switch_config", CTLTYPE_STRING | CTLFLAG_RD, 504 pf, 0, ixl_sysctl_switch_config, "A", "HW Switch Configuration"); 505 #endif 506 507 /* Save off the PCI information */ 508 hw->vendor_id = pci_get_vendor(dev); 509 hw->device_id = pci_get_device(dev); 510 hw->revision_id = pci_read_config(dev, PCIR_REVID, 1); 511 hw->subsystem_vendor_id = 512 pci_read_config(dev, PCIR_SUBVEND_0, 2); 513 hw->subsystem_device_id = 514 pci_read_config(dev, PCIR_SUBDEV_0, 2); 515 516 hw->bus.device = pci_get_slot(dev); 517 hw->bus.func = pci_get_function(dev); 518 519 pf->vc_debug_lvl = 1; 520 521 /* Do PCI setup - map BAR0, etc */ 522 if (ixl_allocate_pci_resources(pf)) { 523 device_printf(dev, "Allocation of PCI resources failed\n"); 524 error = ENXIO; 525 goto err_out; 526 } 527 528 /* Create for initial debugging use */ 529 SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev), 530 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), 531 OID_AUTO, "debug", CTLTYPE_INT|CTLFLAG_RW, pf, 0, 532 ixl_debug_info, "I", "Debug Information"); 533 534 535 /* Establish a clean starting point */ 536 i40e_clear_hw(hw); 537 error = i40e_pf_reset(hw); 538 if (error) { 539 device_printf(dev,"PF reset failure %x\n", error); 540 error = EIO; 541 goto err_out; 542 } 543 544 /* Set admin queue parameters */ 545 hw->aq.num_arq_entries = IXL_AQ_LEN; 546 hw->aq.num_asq_entries = IXL_AQ_LEN; 547 hw->aq.arq_buf_size = IXL_AQ_BUFSZ; 548 hw->aq.asq_buf_size = IXL_AQ_BUFSZ; 549 550 /* Initialize the shared code */ 551 error = i40e_init_shared_code(hw); 552 if (error) { 553 device_printf(dev,"Unable to initialize the shared code\n"); 554 error = EIO; 555 goto err_out; 556 } 557 558 /* Set up the admin queue */ 559 error = i40e_init_adminq(hw); 560 if (error) { 561 device_printf(dev, "The driver for the device stopped " 562 "because the NVM image is newer than expected.\n" 563 "You must install the most recent version of " 564 " the network driver.\n"); 565 goto err_out; 566 } 567 device_printf(dev, "%s\n", ixl_fw_version_str(hw)); 568 569 if (hw->aq.api_maj_ver == I40E_FW_API_VERSION_MAJOR && 570 hw->aq.api_min_ver > I40E_FW_API_VERSION_MINOR) 571 device_printf(dev, "The driver for the device detected " 572 "a newer version of the NVM image than expected.\n" 573 "Please install the most recent version of the network driver.\n"); 574 else if (hw->aq.api_maj_ver < I40E_FW_API_VERSION_MAJOR || 575 hw->aq.api_min_ver < (I40E_FW_API_VERSION_MINOR - 1)) 576 device_printf(dev, "The driver for the device detected " 577 "an older version of the NVM image than expected.\n" 578 "Please update the NVM image.\n"); 579 580 /* Clear PXE mode */ 581 i40e_clear_pxe_mode(hw); 582 583 /* Get capabilities from the device */ 584 error = ixl_get_hw_capabilities(pf); 585 if (error) { 586 device_printf(dev, "HW capabilities failure!\n"); 587 goto err_get_cap; 588 } 589 590 /* Set up host memory cache */ 591 error = i40e_init_lan_hmc(hw, hw->func_caps.num_tx_qp, 592 hw->func_caps.num_rx_qp, 0, 0); 593 if (error) { 594 device_printf(dev, "init_lan_hmc failed: %d\n", error); 595 goto err_get_cap; 596 } 597 598 error = i40e_configure_lan_hmc(hw, I40E_HMC_MODEL_DIRECT_ONLY); 599 if (error) { 600 device_printf(dev, "configure_lan_hmc failed: %d\n", error); 601 goto err_mac_hmc; 602 } 603 604 /* Disable LLDP from the firmware */ 605 i40e_aq_stop_lldp(hw, TRUE, NULL); 606 607 i40e_get_mac_addr(hw, hw->mac.addr); 608 error = i40e_validate_mac_addr(hw->mac.addr); 609 if (error) { 610 device_printf(dev, "validate_mac_addr failed: %d\n", error); 611 goto err_mac_hmc; 612 } 613 bcopy(hw->mac.addr, hw->mac.perm_addr, ETHER_ADDR_LEN); 614 i40e_get_port_mac_addr(hw, hw->mac.port_addr); 615 616 /* Set up VSI and queues */ 617 if (ixl_setup_stations(pf) != 0) { 618 device_printf(dev, "setup stations failed!\n"); 619 error = ENOMEM; 620 goto err_mac_hmc; 621 } 622 623 /* Initialize mac filter list for VSI */ 624 SLIST_INIT(&vsi->ftl); 625 626 /* Set up interrupt routing here */ 627 if (pf->msix > 1) 628 error = ixl_assign_vsi_msix(pf); 629 else 630 error = ixl_assign_vsi_legacy(pf); 631 if (error) 632 goto err_late; 633 634 if (((hw->aq.fw_maj_ver == 4) && (hw->aq.fw_min_ver < 33)) || 635 (hw->aq.fw_maj_ver < 4)) { 636 i40e_msec_delay(75); 637 error = i40e_aq_set_link_restart_an(hw, TRUE, NULL); 638 if (error) 639 device_printf(dev, "link restart failed, aq_err=%d\n", 640 pf->hw.aq.asq_last_status); 641 } 642 643 /* Determine link state */ 644 i40e_aq_get_link_info(hw, TRUE, NULL, NULL); 645 pf->link_up = i40e_get_link_status(hw); 646 647 /* Setup OS specific network interface */ 648 if (ixl_setup_interface(dev, vsi) != 0) { 649 device_printf(dev, "interface setup failed!\n"); 650 error = EIO; 651 goto err_late; 652 } 653 654 error = ixl_switch_config(pf); 655 if (error) { 656 device_printf(dev, "Initial switch config failed: %d\n", error); 657 goto err_mac_hmc; 658 } 659 660 /* Limit phy interrupts to link and modules failure */ 661 error = i40e_aq_set_phy_int_mask(hw, 662 I40E_AQ_EVENT_LINK_UPDOWN | I40E_AQ_EVENT_MODULE_QUAL_FAIL, NULL); 663 if (error) 664 device_printf(dev, "set phy mask failed: %d\n", error); 665 666 /* Get the bus configuration and set the shared code */ 667 bus = ixl_get_bus_info(hw, dev); 668 i40e_set_pci_config_data(hw, bus); 669 670 /* Initialize statistics */ 671 ixl_pf_reset_stats(pf); 672 ixl_update_stats_counters(pf); 673 ixl_add_hw_stats(pf); 674 675 /* Register for VLAN events */ 676 vsi->vlan_attach = EVENTHANDLER_REGISTER(vlan_config, 677 ixl_register_vlan, vsi, EVENTHANDLER_PRI_FIRST); 678 vsi->vlan_detach = EVENTHANDLER_REGISTER(vlan_unconfig, 679 ixl_unregister_vlan, vsi, EVENTHANDLER_PRI_FIRST); 680 681 #ifdef PCI_IOV 682 /* SR-IOV is only supported when MSI-X is in use. */ 683 if (pf->msix > 1) { 684 pf_schema = pci_iov_schema_alloc_node(); 685 vf_schema = pci_iov_schema_alloc_node(); 686 pci_iov_schema_add_unicast_mac(vf_schema, "mac-addr", 0, NULL); 687 pci_iov_schema_add_bool(vf_schema, "mac-anti-spoof", 688 IOV_SCHEMA_HASDEFAULT, TRUE); 689 pci_iov_schema_add_bool(vf_schema, "allow-set-mac", 690 IOV_SCHEMA_HASDEFAULT, FALSE); 691 pci_iov_schema_add_bool(vf_schema, "allow-promisc", 692 IOV_SCHEMA_HASDEFAULT, FALSE); 693 694 iov_error = pci_iov_attach(dev, pf_schema, vf_schema); 695 if (iov_error != 0) 696 device_printf(dev, 697 "Failed to initialize SR-IOV (error=%d)\n", 698 iov_error); 699 } 700 #endif 701 702 #ifdef DEV_NETMAP 703 ixl_netmap_attach(vsi); 704 #endif /* DEV_NETMAP */ 705 706 INIT_DEBUGOUT("ixl_attach: end"); 707 return (0); 708 709 err_late: 710 if (vsi->ifp != NULL) 711 if_free(vsi->ifp); 712 err_mac_hmc: 713 i40e_shutdown_lan_hmc(hw); 714 err_get_cap: 715 i40e_shutdown_adminq(hw); 716 err_out: 717 ixl_free_pci_resources(pf); 718 ixl_free_vsi(vsi); 719 IXL_PF_LOCK_DESTROY(pf); 720 return (error); 721 } 722 723 /********************************************************************* 724 * Device removal routine 725 * 726 * The detach entry point is called when the driver is being removed. 727 * This routine stops the adapter and deallocates all the resources 728 * that were allocated for driver operation. 729 * 730 * return 0 on success, positive on failure 731 *********************************************************************/ 732 733 static int 734 ixl_detach(device_t dev) 735 { 736 struct ixl_pf *pf = device_get_softc(dev); 737 struct i40e_hw *hw = &pf->hw; 738 struct ixl_vsi *vsi = &pf->vsi; 739 struct ixl_queue *que = vsi->queues; 740 i40e_status status; 741 #ifdef PCI_IOV 742 int error; 743 #endif 744 745 INIT_DEBUGOUT("ixl_detach: begin"); 746 747 /* Make sure VLANS are not using driver */ 748 if (vsi->ifp->if_vlantrunk != NULL) { 749 device_printf(dev,"Vlan in use, detach first\n"); 750 return (EBUSY); 751 } 752 753 #ifdef PCI_IOV 754 error = pci_iov_detach(dev); 755 if (error != 0) { 756 device_printf(dev, "SR-IOV in use; detach first.\n"); 757 return (error); 758 } 759 #endif 760 761 ether_ifdetach(vsi->ifp); 762 if (vsi->ifp->if_drv_flags & IFF_DRV_RUNNING) { 763 IXL_PF_LOCK(pf); 764 ixl_stop(pf); 765 IXL_PF_UNLOCK(pf); 766 } 767 768 for (int i = 0; i < vsi->num_queues; i++, que++) { 769 if (que->tq) { 770 taskqueue_drain(que->tq, &que->task); 771 taskqueue_drain(que->tq, &que->tx_task); 772 taskqueue_free(que->tq); 773 } 774 } 775 776 /* Shutdown LAN HMC */ 777 status = i40e_shutdown_lan_hmc(hw); 778 if (status) 779 device_printf(dev, 780 "Shutdown LAN HMC failed with code %d\n", status); 781 782 /* Shutdown admin queue */ 783 status = i40e_shutdown_adminq(hw); 784 if (status) 785 device_printf(dev, 786 "Shutdown Admin queue failed with code %d\n", status); 787 788 /* Unregister VLAN events */ 789 if (vsi->vlan_attach != NULL) 790 EVENTHANDLER_DEREGISTER(vlan_config, vsi->vlan_attach); 791 if (vsi->vlan_detach != NULL) 792 EVENTHANDLER_DEREGISTER(vlan_unconfig, vsi->vlan_detach); 793 794 callout_drain(&pf->timer); 795 #ifdef DEV_NETMAP 796 netmap_detach(vsi->ifp); 797 #endif /* DEV_NETMAP */ 798 ixl_free_pci_resources(pf); 799 bus_generic_detach(dev); 800 if_free(vsi->ifp); 801 ixl_free_vsi(vsi); 802 IXL_PF_LOCK_DESTROY(pf); 803 return (0); 804 } 805 806 /********************************************************************* 807 * 808 * Shutdown entry point 809 * 810 **********************************************************************/ 811 812 static int 813 ixl_shutdown(device_t dev) 814 { 815 struct ixl_pf *pf = device_get_softc(dev); 816 IXL_PF_LOCK(pf); 817 ixl_stop(pf); 818 IXL_PF_UNLOCK(pf); 819 return (0); 820 } 821 822 823 /********************************************************************* 824 * 825 * Get the hardware capabilities 826 * 827 **********************************************************************/ 828 829 static int 830 ixl_get_hw_capabilities(struct ixl_pf *pf) 831 { 832 struct i40e_aqc_list_capabilities_element_resp *buf; 833 struct i40e_hw *hw = &pf->hw; 834 device_t dev = pf->dev; 835 int error, len; 836 u16 needed; 837 bool again = TRUE; 838 839 len = 40 * sizeof(struct i40e_aqc_list_capabilities_element_resp); 840 retry: 841 if (!(buf = (struct i40e_aqc_list_capabilities_element_resp *) 842 malloc(len, M_DEVBUF, M_NOWAIT | M_ZERO))) { 843 device_printf(dev, "Unable to allocate cap memory\n"); 844 return (ENOMEM); 845 } 846 847 /* This populates the hw struct */ 848 error = i40e_aq_discover_capabilities(hw, buf, len, 849 &needed, i40e_aqc_opc_list_func_capabilities, NULL); 850 free(buf, M_DEVBUF); 851 if ((pf->hw.aq.asq_last_status == I40E_AQ_RC_ENOMEM) && 852 (again == TRUE)) { 853 /* retry once with a larger buffer */ 854 again = FALSE; 855 len = needed; 856 goto retry; 857 } else if (pf->hw.aq.asq_last_status != I40E_AQ_RC_OK) { 858 device_printf(dev, "capability discovery failed: %d\n", 859 pf->hw.aq.asq_last_status); 860 return (ENODEV); 861 } 862 863 /* Capture this PF's starting queue pair */ 864 pf->qbase = hw->func_caps.base_queue; 865 866 #ifdef IXL_DEBUG 867 device_printf(dev,"pf_id=%d, num_vfs=%d, msix_pf=%d, " 868 "msix_vf=%d, fd_g=%d, fd_b=%d, tx_qp=%d rx_qp=%d qbase=%d\n", 869 hw->pf_id, hw->func_caps.num_vfs, 870 hw->func_caps.num_msix_vectors, 871 hw->func_caps.num_msix_vectors_vf, 872 hw->func_caps.fd_filters_guaranteed, 873 hw->func_caps.fd_filters_best_effort, 874 hw->func_caps.num_tx_qp, 875 hw->func_caps.num_rx_qp, 876 hw->func_caps.base_queue); 877 #endif 878 return (error); 879 } 880 881 static void 882 ixl_cap_txcsum_tso(struct ixl_vsi *vsi, struct ifnet *ifp, int mask) 883 { 884 device_t dev = vsi->dev; 885 886 /* Enable/disable TXCSUM/TSO4 */ 887 if (!(ifp->if_capenable & IFCAP_TXCSUM) 888 && !(ifp->if_capenable & IFCAP_TSO4)) { 889 if (mask & IFCAP_TXCSUM) { 890 ifp->if_capenable |= IFCAP_TXCSUM; 891 /* enable TXCSUM, restore TSO if previously enabled */ 892 if (vsi->flags & IXL_FLAGS_KEEP_TSO4) { 893 vsi->flags &= ~IXL_FLAGS_KEEP_TSO4; 894 ifp->if_capenable |= IFCAP_TSO4; 895 } 896 } 897 else if (mask & IFCAP_TSO4) { 898 ifp->if_capenable |= (IFCAP_TXCSUM | IFCAP_TSO4); 899 vsi->flags &= ~IXL_FLAGS_KEEP_TSO4; 900 device_printf(dev, 901 "TSO4 requires txcsum, enabling both...\n"); 902 } 903 } else if((ifp->if_capenable & IFCAP_TXCSUM) 904 && !(ifp->if_capenable & IFCAP_TSO4)) { 905 if (mask & IFCAP_TXCSUM) 906 ifp->if_capenable &= ~IFCAP_TXCSUM; 907 else if (mask & IFCAP_TSO4) 908 ifp->if_capenable |= IFCAP_TSO4; 909 } else if((ifp->if_capenable & IFCAP_TXCSUM) 910 && (ifp->if_capenable & IFCAP_TSO4)) { 911 if (mask & IFCAP_TXCSUM) { 912 vsi->flags |= IXL_FLAGS_KEEP_TSO4; 913 ifp->if_capenable &= ~(IFCAP_TXCSUM | IFCAP_TSO4); 914 device_printf(dev, 915 "TSO4 requires txcsum, disabling both...\n"); 916 } else if (mask & IFCAP_TSO4) 917 ifp->if_capenable &= ~IFCAP_TSO4; 918 } 919 920 /* Enable/disable TXCSUM_IPV6/TSO6 */ 921 if (!(ifp->if_capenable & IFCAP_TXCSUM_IPV6) 922 && !(ifp->if_capenable & IFCAP_TSO6)) { 923 if (mask & IFCAP_TXCSUM_IPV6) { 924 ifp->if_capenable |= IFCAP_TXCSUM_IPV6; 925 if (vsi->flags & IXL_FLAGS_KEEP_TSO6) { 926 vsi->flags &= ~IXL_FLAGS_KEEP_TSO6; 927 ifp->if_capenable |= IFCAP_TSO6; 928 } 929 } else if (mask & IFCAP_TSO6) { 930 ifp->if_capenable |= (IFCAP_TXCSUM_IPV6 | IFCAP_TSO6); 931 vsi->flags &= ~IXL_FLAGS_KEEP_TSO6; 932 device_printf(dev, 933 "TSO6 requires txcsum6, enabling both...\n"); 934 } 935 } else if((ifp->if_capenable & IFCAP_TXCSUM_IPV6) 936 && !(ifp->if_capenable & IFCAP_TSO6)) { 937 if (mask & IFCAP_TXCSUM_IPV6) 938 ifp->if_capenable &= ~IFCAP_TXCSUM_IPV6; 939 else if (mask & IFCAP_TSO6) 940 ifp->if_capenable |= IFCAP_TSO6; 941 } else if ((ifp->if_capenable & IFCAP_TXCSUM_IPV6) 942 && (ifp->if_capenable & IFCAP_TSO6)) { 943 if (mask & IFCAP_TXCSUM_IPV6) { 944 vsi->flags |= IXL_FLAGS_KEEP_TSO6; 945 ifp->if_capenable &= ~(IFCAP_TXCSUM_IPV6 | IFCAP_TSO6); 946 device_printf(dev, 947 "TSO6 requires txcsum6, disabling both...\n"); 948 } else if (mask & IFCAP_TSO6) 949 ifp->if_capenable &= ~IFCAP_TSO6; 950 } 951 } 952 953 /********************************************************************* 954 * Ioctl entry point 955 * 956 * ixl_ioctl is called when the user wants to configure the 957 * interface. 958 * 959 * return 0 on success, positive on failure 960 **********************************************************************/ 961 962 static int 963 ixl_ioctl(struct ifnet * ifp, u_long command, caddr_t data) 964 { 965 struct ixl_vsi *vsi = ifp->if_softc; 966 struct ixl_pf *pf = vsi->back; 967 struct ifreq *ifr = (struct ifreq *) data; 968 #if defined(INET) || defined(INET6) 969 struct ifaddr *ifa = (struct ifaddr *)data; 970 bool avoid_reset = FALSE; 971 #endif 972 int error = 0; 973 974 switch (command) { 975 976 case SIOCSIFADDR: 977 #ifdef INET 978 if (ifa->ifa_addr->sa_family == AF_INET) 979 avoid_reset = TRUE; 980 #endif 981 #ifdef INET6 982 if (ifa->ifa_addr->sa_family == AF_INET6) 983 avoid_reset = TRUE; 984 #endif 985 #if defined(INET) || defined(INET6) 986 /* 987 ** Calling init results in link renegotiation, 988 ** so we avoid doing it when possible. 989 */ 990 if (avoid_reset) { 991 ifp->if_flags |= IFF_UP; 992 if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) 993 ixl_init(pf); 994 #ifdef INET 995 if (!(ifp->if_flags & IFF_NOARP)) 996 arp_ifinit(ifp, ifa); 997 #endif 998 } else 999 error = ether_ioctl(ifp, command, data); 1000 break; 1001 #endif 1002 case SIOCSIFMTU: 1003 IOCTL_DEBUGOUT("ioctl: SIOCSIFMTU (Set Interface MTU)"); 1004 if (ifr->ifr_mtu > IXL_MAX_FRAME - 1005 ETHER_HDR_LEN - ETHER_CRC_LEN - ETHER_VLAN_ENCAP_LEN) { 1006 error = EINVAL; 1007 } else { 1008 IXL_PF_LOCK(pf); 1009 ifp->if_mtu = ifr->ifr_mtu; 1010 vsi->max_frame_size = 1011 ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN 1012 + ETHER_VLAN_ENCAP_LEN; 1013 ixl_init_locked(pf); 1014 IXL_PF_UNLOCK(pf); 1015 } 1016 break; 1017 case SIOCSIFFLAGS: 1018 IOCTL_DEBUGOUT("ioctl: SIOCSIFFLAGS (Set Interface Flags)"); 1019 IXL_PF_LOCK(pf); 1020 if (ifp->if_flags & IFF_UP) { 1021 if ((ifp->if_drv_flags & IFF_DRV_RUNNING)) { 1022 if ((ifp->if_flags ^ pf->if_flags) & 1023 (IFF_PROMISC | IFF_ALLMULTI)) { 1024 ixl_set_promisc(vsi); 1025 } 1026 } else 1027 ixl_init_locked(pf); 1028 } else 1029 if (ifp->if_drv_flags & IFF_DRV_RUNNING) 1030 ixl_stop(pf); 1031 pf->if_flags = ifp->if_flags; 1032 IXL_PF_UNLOCK(pf); 1033 break; 1034 case SIOCADDMULTI: 1035 IOCTL_DEBUGOUT("ioctl: SIOCADDMULTI"); 1036 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 1037 IXL_PF_LOCK(pf); 1038 ixl_disable_intr(vsi); 1039 ixl_add_multi(vsi); 1040 ixl_enable_intr(vsi); 1041 IXL_PF_UNLOCK(pf); 1042 } 1043 break; 1044 case SIOCDELMULTI: 1045 IOCTL_DEBUGOUT("ioctl: SIOCDELMULTI"); 1046 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 1047 IXL_PF_LOCK(pf); 1048 ixl_disable_intr(vsi); 1049 ixl_del_multi(vsi); 1050 ixl_enable_intr(vsi); 1051 IXL_PF_UNLOCK(pf); 1052 } 1053 break; 1054 case SIOCSIFMEDIA: 1055 case SIOCGIFMEDIA: 1056 IOCTL_DEBUGOUT("ioctl: SIOCxIFMEDIA (Get/Set Interface Media)"); 1057 error = ifmedia_ioctl(ifp, ifr, &vsi->media, command); 1058 break; 1059 case SIOCSIFCAP: 1060 { 1061 int mask = ifr->ifr_reqcap ^ ifp->if_capenable; 1062 IOCTL_DEBUGOUT("ioctl: SIOCSIFCAP (Set Capabilities)"); 1063 1064 ixl_cap_txcsum_tso(vsi, ifp, mask); 1065 1066 if (mask & IFCAP_RXCSUM) 1067 ifp->if_capenable ^= IFCAP_RXCSUM; 1068 if (mask & IFCAP_RXCSUM_IPV6) 1069 ifp->if_capenable ^= IFCAP_RXCSUM_IPV6; 1070 if (mask & IFCAP_LRO) 1071 ifp->if_capenable ^= IFCAP_LRO; 1072 if (mask & IFCAP_VLAN_HWTAGGING) 1073 ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING; 1074 if (mask & IFCAP_VLAN_HWFILTER) 1075 ifp->if_capenable ^= IFCAP_VLAN_HWFILTER; 1076 if (mask & IFCAP_VLAN_HWTSO) 1077 ifp->if_capenable ^= IFCAP_VLAN_HWTSO; 1078 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 1079 IXL_PF_LOCK(pf); 1080 ixl_init_locked(pf); 1081 IXL_PF_UNLOCK(pf); 1082 } 1083 VLAN_CAPABILITIES(ifp); 1084 1085 break; 1086 } 1087 1088 default: 1089 IOCTL_DEBUGOUT("ioctl: UNKNOWN (0x%X)\n", (int)command); 1090 error = ether_ioctl(ifp, command, data); 1091 break; 1092 } 1093 1094 return (error); 1095 } 1096 1097 1098 /********************************************************************* 1099 * Init entry point 1100 * 1101 * This routine is used in two ways. It is used by the stack as 1102 * init entry point in network interface structure. It is also used 1103 * by the driver as a hw/sw initialization routine to get to a 1104 * consistent state. 1105 * 1106 * return 0 on success, positive on failure 1107 **********************************************************************/ 1108 1109 static void 1110 ixl_init_locked(struct ixl_pf *pf) 1111 { 1112 struct i40e_hw *hw = &pf->hw; 1113 struct ixl_vsi *vsi = &pf->vsi; 1114 struct ifnet *ifp = vsi->ifp; 1115 device_t dev = pf->dev; 1116 struct i40e_filter_control_settings filter; 1117 u8 tmpaddr[ETHER_ADDR_LEN]; 1118 int ret; 1119 1120 mtx_assert(&pf->pf_mtx, MA_OWNED); 1121 INIT_DEBUGOUT("ixl_init: begin"); 1122 ixl_stop(pf); 1123 1124 /* Get the latest mac address... User might use a LAA */ 1125 bcopy(IF_LLADDR(vsi->ifp), tmpaddr, 1126 I40E_ETH_LENGTH_OF_ADDRESS); 1127 if (!cmp_etheraddr(hw->mac.addr, tmpaddr) && 1128 i40e_validate_mac_addr(tmpaddr)) { 1129 bcopy(tmpaddr, hw->mac.addr, 1130 I40E_ETH_LENGTH_OF_ADDRESS); 1131 ret = i40e_aq_mac_address_write(hw, 1132 I40E_AQC_WRITE_TYPE_LAA_ONLY, 1133 hw->mac.addr, NULL); 1134 if (ret) { 1135 device_printf(dev, "LLA address" 1136 "change failed!!\n"); 1137 return; 1138 } 1139 } 1140 1141 /* Set the various hardware offload abilities */ 1142 ifp->if_hwassist = 0; 1143 if (ifp->if_capenable & IFCAP_TSO) 1144 ifp->if_hwassist |= CSUM_TSO; 1145 if (ifp->if_capenable & IFCAP_TXCSUM) 1146 ifp->if_hwassist |= (CSUM_TCP | CSUM_UDP); 1147 if (ifp->if_capenable & IFCAP_TXCSUM_IPV6) 1148 ifp->if_hwassist |= (CSUM_TCP_IPV6 | CSUM_UDP_IPV6); 1149 1150 /* Set up the device filtering */ 1151 bzero(&filter, sizeof(filter)); 1152 filter.enable_ethtype = TRUE; 1153 filter.enable_macvlan = TRUE; 1154 #ifdef IXL_FDIR 1155 filter.enable_fdir = TRUE; 1156 #endif 1157 if (i40e_set_filter_control(hw, &filter)) 1158 device_printf(dev, "set_filter_control() failed\n"); 1159 1160 /* Set up RSS */ 1161 ixl_config_rss(vsi); 1162 1163 /* 1164 ** Prepare the VSI: rings, hmc contexts, etc... 1165 */ 1166 if (ixl_initialize_vsi(vsi)) { 1167 device_printf(dev, "initialize vsi failed!!\n"); 1168 return; 1169 } 1170 1171 /* Add protocol filters to list */ 1172 ixl_init_filters(vsi); 1173 1174 /* Setup vlan's if needed */ 1175 ixl_setup_vlan_filters(vsi); 1176 1177 /* Start the local timer */ 1178 callout_reset(&pf->timer, hz, ixl_local_timer, pf); 1179 1180 /* Set up MSI/X routing and the ITR settings */ 1181 if (ixl_enable_msix) { 1182 ixl_configure_msix(pf); 1183 ixl_configure_itr(pf); 1184 } else 1185 ixl_configure_legacy(pf); 1186 1187 ixl_enable_rings(vsi); 1188 1189 i40e_aq_set_default_vsi(hw, vsi->seid, NULL); 1190 1191 ixl_reconfigure_filters(vsi); 1192 1193 /* Set MTU in hardware*/ 1194 int aq_error = i40e_aq_set_mac_config(hw, vsi->max_frame_size, 1195 TRUE, 0, NULL); 1196 if (aq_error) 1197 device_printf(vsi->dev, 1198 "aq_set_mac_config in init error, code %d\n", 1199 aq_error); 1200 1201 /* And now turn on interrupts */ 1202 ixl_enable_intr(vsi); 1203 1204 /* Now inform the stack we're ready */ 1205 ifp->if_drv_flags |= IFF_DRV_RUNNING; 1206 ifp->if_drv_flags &= ~IFF_DRV_OACTIVE; 1207 1208 return; 1209 } 1210 1211 static void 1212 ixl_init(void *arg) 1213 { 1214 struct ixl_pf *pf = arg; 1215 1216 IXL_PF_LOCK(pf); 1217 ixl_init_locked(pf); 1218 IXL_PF_UNLOCK(pf); 1219 return; 1220 } 1221 1222 /* 1223 ** 1224 ** MSIX Interrupt Handlers and Tasklets 1225 ** 1226 */ 1227 static void 1228 ixl_handle_que(void *context, int pending) 1229 { 1230 struct ixl_queue *que = context; 1231 struct ixl_vsi *vsi = que->vsi; 1232 struct i40e_hw *hw = vsi->hw; 1233 struct tx_ring *txr = &que->txr; 1234 struct ifnet *ifp = vsi->ifp; 1235 bool more; 1236 1237 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 1238 more = ixl_rxeof(que, IXL_RX_LIMIT); 1239 IXL_TX_LOCK(txr); 1240 ixl_txeof(que); 1241 if (!drbr_empty(ifp, txr->br)) 1242 ixl_mq_start_locked(ifp, txr); 1243 IXL_TX_UNLOCK(txr); 1244 if (more) { 1245 taskqueue_enqueue(que->tq, &que->task); 1246 return; 1247 } 1248 } 1249 1250 /* Reenable this interrupt - hmmm */ 1251 ixl_enable_queue(hw, que->me); 1252 return; 1253 } 1254 1255 1256 /********************************************************************* 1257 * 1258 * Legacy Interrupt Service routine 1259 * 1260 **********************************************************************/ 1261 void 1262 ixl_intr(void *arg) 1263 { 1264 struct ixl_pf *pf = arg; 1265 struct i40e_hw *hw = &pf->hw; 1266 struct ixl_vsi *vsi = &pf->vsi; 1267 struct ixl_queue *que = vsi->queues; 1268 struct ifnet *ifp = vsi->ifp; 1269 struct tx_ring *txr = &que->txr; 1270 u32 reg, icr0, mask; 1271 bool more_tx, more_rx; 1272 1273 ++que->irqs; 1274 1275 /* Protect against spurious interrupts */ 1276 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) 1277 return; 1278 1279 icr0 = rd32(hw, I40E_PFINT_ICR0); 1280 1281 reg = rd32(hw, I40E_PFINT_DYN_CTL0); 1282 reg = reg | I40E_PFINT_DYN_CTL0_CLEARPBA_MASK; 1283 wr32(hw, I40E_PFINT_DYN_CTL0, reg); 1284 1285 mask = rd32(hw, I40E_PFINT_ICR0_ENA); 1286 1287 #ifdef PCI_IOV 1288 if (icr0 & I40E_PFINT_ICR0_VFLR_MASK) 1289 taskqueue_enqueue(pf->tq, &pf->vflr_task); 1290 #endif 1291 1292 if (icr0 & I40E_PFINT_ICR0_ADMINQ_MASK) { 1293 taskqueue_enqueue(pf->tq, &pf->adminq); 1294 return; 1295 } 1296 1297 more_rx = ixl_rxeof(que, IXL_RX_LIMIT); 1298 1299 IXL_TX_LOCK(txr); 1300 more_tx = ixl_txeof(que); 1301 if (!drbr_empty(vsi->ifp, txr->br)) 1302 more_tx = 1; 1303 IXL_TX_UNLOCK(txr); 1304 1305 /* re-enable other interrupt causes */ 1306 wr32(hw, I40E_PFINT_ICR0_ENA, mask); 1307 1308 /* And now the queues */ 1309 reg = rd32(hw, I40E_QINT_RQCTL(0)); 1310 reg |= I40E_QINT_RQCTL_CAUSE_ENA_MASK; 1311 wr32(hw, I40E_QINT_RQCTL(0), reg); 1312 1313 reg = rd32(hw, I40E_QINT_TQCTL(0)); 1314 reg |= I40E_QINT_TQCTL_CAUSE_ENA_MASK; 1315 reg &= ~I40E_PFINT_ICR0_INTEVENT_MASK; 1316 wr32(hw, I40E_QINT_TQCTL(0), reg); 1317 1318 ixl_enable_legacy(hw); 1319 1320 return; 1321 } 1322 1323 1324 /********************************************************************* 1325 * 1326 * MSIX VSI Interrupt Service routine 1327 * 1328 **********************************************************************/ 1329 void 1330 ixl_msix_que(void *arg) 1331 { 1332 struct ixl_queue *que = arg; 1333 struct ixl_vsi *vsi = que->vsi; 1334 struct i40e_hw *hw = vsi->hw; 1335 struct tx_ring *txr = &que->txr; 1336 bool more_tx, more_rx; 1337 1338 /* Protect against spurious interrupts */ 1339 if (!(vsi->ifp->if_drv_flags & IFF_DRV_RUNNING)) 1340 return; 1341 1342 ++que->irqs; 1343 1344 more_rx = ixl_rxeof(que, IXL_RX_LIMIT); 1345 1346 IXL_TX_LOCK(txr); 1347 more_tx = ixl_txeof(que); 1348 /* 1349 ** Make certain that if the stack 1350 ** has anything queued the task gets 1351 ** scheduled to handle it. 1352 */ 1353 if (!drbr_empty(vsi->ifp, txr->br)) 1354 more_tx = 1; 1355 IXL_TX_UNLOCK(txr); 1356 1357 ixl_set_queue_rx_itr(que); 1358 ixl_set_queue_tx_itr(que); 1359 1360 if (more_tx || more_rx) 1361 taskqueue_enqueue(que->tq, &que->task); 1362 else 1363 ixl_enable_queue(hw, que->me); 1364 1365 return; 1366 } 1367 1368 1369 /********************************************************************* 1370 * 1371 * MSIX Admin Queue Interrupt Service routine 1372 * 1373 **********************************************************************/ 1374 static void 1375 ixl_msix_adminq(void *arg) 1376 { 1377 struct ixl_pf *pf = arg; 1378 struct i40e_hw *hw = &pf->hw; 1379 u32 reg, mask; 1380 1381 ++pf->admin_irq; 1382 1383 reg = rd32(hw, I40E_PFINT_ICR0); 1384 mask = rd32(hw, I40E_PFINT_ICR0_ENA); 1385 1386 /* Check on the cause */ 1387 if (reg & I40E_PFINT_ICR0_ADMINQ_MASK) 1388 mask &= ~I40E_PFINT_ICR0_ENA_ADMINQ_MASK; 1389 1390 if (reg & I40E_PFINT_ICR0_MAL_DETECT_MASK) { 1391 ixl_handle_mdd_event(pf); 1392 mask &= ~I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK; 1393 } 1394 1395 #ifdef PCI_IOV 1396 if (reg & I40E_PFINT_ICR0_VFLR_MASK) { 1397 mask &= ~I40E_PFINT_ICR0_ENA_VFLR_MASK; 1398 taskqueue_enqueue(pf->tq, &pf->vflr_task); 1399 } 1400 #endif 1401 1402 reg = rd32(hw, I40E_PFINT_DYN_CTL0); 1403 reg = reg | I40E_PFINT_DYN_CTL0_CLEARPBA_MASK; 1404 wr32(hw, I40E_PFINT_DYN_CTL0, reg); 1405 1406 taskqueue_enqueue(pf->tq, &pf->adminq); 1407 return; 1408 } 1409 1410 /********************************************************************* 1411 * 1412 * Media Ioctl callback 1413 * 1414 * This routine is called whenever the user queries the status of 1415 * the interface using ifconfig. 1416 * 1417 **********************************************************************/ 1418 static void 1419 ixl_media_status(struct ifnet * ifp, struct ifmediareq * ifmr) 1420 { 1421 struct ixl_vsi *vsi = ifp->if_softc; 1422 struct ixl_pf *pf = vsi->back; 1423 struct i40e_hw *hw = &pf->hw; 1424 1425 INIT_DEBUGOUT("ixl_media_status: begin"); 1426 IXL_PF_LOCK(pf); 1427 1428 hw->phy.get_link_info = TRUE; 1429 pf->link_up = i40e_get_link_status(hw); 1430 ixl_update_link_status(pf); 1431 1432 ifmr->ifm_status = IFM_AVALID; 1433 ifmr->ifm_active = IFM_ETHER; 1434 1435 if (!pf->link_up) { 1436 IXL_PF_UNLOCK(pf); 1437 return; 1438 } 1439 1440 ifmr->ifm_status |= IFM_ACTIVE; 1441 /* Hardware is always full-duplex */ 1442 ifmr->ifm_active |= IFM_FDX; 1443 1444 switch (hw->phy.link_info.phy_type) { 1445 /* 100 M */ 1446 case I40E_PHY_TYPE_100BASE_TX: 1447 ifmr->ifm_active |= IFM_100_TX; 1448 break; 1449 /* 1 G */ 1450 case I40E_PHY_TYPE_1000BASE_T: 1451 ifmr->ifm_active |= IFM_1000_T; 1452 break; 1453 case I40E_PHY_TYPE_1000BASE_SX: 1454 ifmr->ifm_active |= IFM_1000_SX; 1455 break; 1456 case I40E_PHY_TYPE_1000BASE_LX: 1457 ifmr->ifm_active |= IFM_1000_LX; 1458 break; 1459 /* 10 G */ 1460 case I40E_PHY_TYPE_10GBASE_CR1: 1461 case I40E_PHY_TYPE_10GBASE_CR1_CU: 1462 case I40E_PHY_TYPE_10GBASE_SFPP_CU: 1463 /* Using this until a real KR media type */ 1464 case I40E_PHY_TYPE_10GBASE_KR: 1465 case I40E_PHY_TYPE_10GBASE_KX4: 1466 ifmr->ifm_active |= IFM_10G_TWINAX; 1467 break; 1468 case I40E_PHY_TYPE_10GBASE_SR: 1469 ifmr->ifm_active |= IFM_10G_SR; 1470 break; 1471 case I40E_PHY_TYPE_10GBASE_LR: 1472 ifmr->ifm_active |= IFM_10G_LR; 1473 break; 1474 case I40E_PHY_TYPE_10GBASE_T: 1475 ifmr->ifm_active |= IFM_10G_T; 1476 break; 1477 /* 40 G */ 1478 case I40E_PHY_TYPE_40GBASE_CR4: 1479 case I40E_PHY_TYPE_40GBASE_CR4_CU: 1480 ifmr->ifm_active |= IFM_40G_CR4; 1481 break; 1482 case I40E_PHY_TYPE_40GBASE_SR4: 1483 ifmr->ifm_active |= IFM_40G_SR4; 1484 break; 1485 case I40E_PHY_TYPE_40GBASE_LR4: 1486 ifmr->ifm_active |= IFM_40G_LR4; 1487 break; 1488 /* 1489 ** Set these to CR4 because OS does not 1490 ** have types available yet. 1491 */ 1492 case I40E_PHY_TYPE_40GBASE_KR4: 1493 case I40E_PHY_TYPE_XLAUI: 1494 case I40E_PHY_TYPE_XLPPI: 1495 case I40E_PHY_TYPE_40GBASE_AOC: 1496 ifmr->ifm_active |= IFM_40G_CR4; 1497 break; 1498 default: 1499 ifmr->ifm_active |= IFM_UNKNOWN; 1500 break; 1501 } 1502 /* Report flow control status as well */ 1503 if (hw->phy.link_info.an_info & I40E_AQ_LINK_PAUSE_TX) 1504 ifmr->ifm_active |= IFM_ETH_TXPAUSE; 1505 if (hw->phy.link_info.an_info & I40E_AQ_LINK_PAUSE_RX) 1506 ifmr->ifm_active |= IFM_ETH_RXPAUSE; 1507 1508 IXL_PF_UNLOCK(pf); 1509 1510 return; 1511 } 1512 1513 /********************************************************************* 1514 * 1515 * Media Ioctl callback 1516 * 1517 * This routine is called when the user changes speed/duplex using 1518 * media/mediopt option with ifconfig. 1519 * 1520 **********************************************************************/ 1521 static int 1522 ixl_media_change(struct ifnet * ifp) 1523 { 1524 struct ixl_vsi *vsi = ifp->if_softc; 1525 struct ifmedia *ifm = &vsi->media; 1526 1527 INIT_DEBUGOUT("ixl_media_change: begin"); 1528 1529 if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER) 1530 return (EINVAL); 1531 1532 if_printf(ifp, "Media change is currently not supported.\n"); 1533 1534 return (ENODEV); 1535 } 1536 1537 1538 #ifdef IXL_FDIR 1539 /* 1540 ** ATR: Application Targetted Receive - creates a filter 1541 ** based on TX flow info that will keep the receive 1542 ** portion of the flow on the same queue. Based on the 1543 ** implementation this is only available for TCP connections 1544 */ 1545 void 1546 ixl_atr(struct ixl_queue *que, struct tcphdr *th, int etype) 1547 { 1548 struct ixl_vsi *vsi = que->vsi; 1549 struct tx_ring *txr = &que->txr; 1550 struct i40e_filter_program_desc *FDIR; 1551 u32 ptype, dtype; 1552 int idx; 1553 1554 /* check if ATR is enabled and sample rate */ 1555 if ((!ixl_enable_fdir) || (!txr->atr_rate)) 1556 return; 1557 /* 1558 ** We sample all TCP SYN/FIN packets, 1559 ** or at the selected sample rate 1560 */ 1561 txr->atr_count++; 1562 if (((th->th_flags & (TH_FIN | TH_SYN)) == 0) && 1563 (txr->atr_count < txr->atr_rate)) 1564 return; 1565 txr->atr_count = 0; 1566 1567 /* Get a descriptor to use */ 1568 idx = txr->next_avail; 1569 FDIR = (struct i40e_filter_program_desc *) &txr->base[idx]; 1570 if (++idx == que->num_desc) 1571 idx = 0; 1572 txr->avail--; 1573 txr->next_avail = idx; 1574 1575 ptype = (que->me << I40E_TXD_FLTR_QW0_QINDEX_SHIFT) & 1576 I40E_TXD_FLTR_QW0_QINDEX_MASK; 1577 1578 ptype |= (etype == ETHERTYPE_IP) ? 1579 (I40E_FILTER_PCTYPE_NONF_IPV4_TCP << 1580 I40E_TXD_FLTR_QW0_PCTYPE_SHIFT) : 1581 (I40E_FILTER_PCTYPE_NONF_IPV6_TCP << 1582 I40E_TXD_FLTR_QW0_PCTYPE_SHIFT); 1583 1584 ptype |= vsi->id << I40E_TXD_FLTR_QW0_DEST_VSI_SHIFT; 1585 1586 dtype = I40E_TX_DESC_DTYPE_FILTER_PROG; 1587 1588 /* 1589 ** We use the TCP TH_FIN as a trigger to remove 1590 ** the filter, otherwise its an update. 1591 */ 1592 dtype |= (th->th_flags & TH_FIN) ? 1593 (I40E_FILTER_PROGRAM_DESC_PCMD_REMOVE << 1594 I40E_TXD_FLTR_QW1_PCMD_SHIFT) : 1595 (I40E_FILTER_PROGRAM_DESC_PCMD_ADD_UPDATE << 1596 I40E_TXD_FLTR_QW1_PCMD_SHIFT); 1597 1598 dtype |= I40E_FILTER_PROGRAM_DESC_DEST_DIRECT_PACKET_QINDEX << 1599 I40E_TXD_FLTR_QW1_DEST_SHIFT; 1600 1601 dtype |= I40E_FILTER_PROGRAM_DESC_FD_STATUS_FD_ID << 1602 I40E_TXD_FLTR_QW1_FD_STATUS_SHIFT; 1603 1604 FDIR->qindex_flex_ptype_vsi = htole32(ptype); 1605 FDIR->dtype_cmd_cntindex = htole32(dtype); 1606 return; 1607 } 1608 #endif 1609 1610 1611 static void 1612 ixl_set_promisc(struct ixl_vsi *vsi) 1613 { 1614 struct ifnet *ifp = vsi->ifp; 1615 struct i40e_hw *hw = vsi->hw; 1616 int err, mcnt = 0; 1617 bool uni = FALSE, multi = FALSE; 1618 1619 if (ifp->if_flags & IFF_ALLMULTI) 1620 multi = TRUE; 1621 else { /* Need to count the multicast addresses */ 1622 struct ifmultiaddr *ifma; 1623 if_maddr_rlock(ifp); 1624 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { 1625 if (ifma->ifma_addr->sa_family != AF_LINK) 1626 continue; 1627 if (mcnt == MAX_MULTICAST_ADDR) 1628 break; 1629 mcnt++; 1630 } 1631 if_maddr_runlock(ifp); 1632 } 1633 1634 if (mcnt >= MAX_MULTICAST_ADDR) 1635 multi = TRUE; 1636 if (ifp->if_flags & IFF_PROMISC) 1637 uni = TRUE; 1638 1639 err = i40e_aq_set_vsi_unicast_promiscuous(hw, 1640 vsi->seid, uni, NULL); 1641 err = i40e_aq_set_vsi_multicast_promiscuous(hw, 1642 vsi->seid, multi, NULL); 1643 return; 1644 } 1645 1646 /********************************************************************* 1647 * Filter Routines 1648 * 1649 * Routines for multicast and vlan filter management. 1650 * 1651 *********************************************************************/ 1652 static void 1653 ixl_add_multi(struct ixl_vsi *vsi) 1654 { 1655 struct ifmultiaddr *ifma; 1656 struct ifnet *ifp = vsi->ifp; 1657 struct i40e_hw *hw = vsi->hw; 1658 int mcnt = 0, flags; 1659 1660 IOCTL_DEBUGOUT("ixl_add_multi: begin"); 1661 1662 if_maddr_rlock(ifp); 1663 /* 1664 ** First just get a count, to decide if we 1665 ** we simply use multicast promiscuous. 1666 */ 1667 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { 1668 if (ifma->ifma_addr->sa_family != AF_LINK) 1669 continue; 1670 mcnt++; 1671 } 1672 if_maddr_runlock(ifp); 1673 1674 if (__predict_false(mcnt >= MAX_MULTICAST_ADDR)) { 1675 /* delete existing MC filters */ 1676 ixl_del_hw_filters(vsi, mcnt); 1677 i40e_aq_set_vsi_multicast_promiscuous(hw, 1678 vsi->seid, TRUE, NULL); 1679 return; 1680 } 1681 1682 mcnt = 0; 1683 if_maddr_rlock(ifp); 1684 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { 1685 if (ifma->ifma_addr->sa_family != AF_LINK) 1686 continue; 1687 ixl_add_mc_filter(vsi, 1688 (u8*)LLADDR((struct sockaddr_dl *) ifma->ifma_addr)); 1689 mcnt++; 1690 } 1691 if_maddr_runlock(ifp); 1692 if (mcnt > 0) { 1693 flags = (IXL_FILTER_ADD | IXL_FILTER_USED | IXL_FILTER_MC); 1694 ixl_add_hw_filters(vsi, flags, mcnt); 1695 } 1696 1697 IOCTL_DEBUGOUT("ixl_add_multi: end"); 1698 return; 1699 } 1700 1701 static void 1702 ixl_del_multi(struct ixl_vsi *vsi) 1703 { 1704 struct ifnet *ifp = vsi->ifp; 1705 struct ifmultiaddr *ifma; 1706 struct ixl_mac_filter *f; 1707 int mcnt = 0; 1708 bool match = FALSE; 1709 1710 IOCTL_DEBUGOUT("ixl_del_multi: begin"); 1711 1712 /* Search for removed multicast addresses */ 1713 if_maddr_rlock(ifp); 1714 SLIST_FOREACH(f, &vsi->ftl, next) { 1715 if ((f->flags & IXL_FILTER_USED) && (f->flags & IXL_FILTER_MC)) { 1716 match = FALSE; 1717 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { 1718 if (ifma->ifma_addr->sa_family != AF_LINK) 1719 continue; 1720 u8 *mc_addr = (u8 *)LLADDR((struct sockaddr_dl *)ifma->ifma_addr); 1721 if (cmp_etheraddr(f->macaddr, mc_addr)) { 1722 match = TRUE; 1723 break; 1724 } 1725 } 1726 if (match == FALSE) { 1727 f->flags |= IXL_FILTER_DEL; 1728 mcnt++; 1729 } 1730 } 1731 } 1732 if_maddr_runlock(ifp); 1733 1734 if (mcnt > 0) 1735 ixl_del_hw_filters(vsi, mcnt); 1736 } 1737 1738 1739 /********************************************************************* 1740 * Timer routine 1741 * 1742 * This routine checks for link status,updates statistics, 1743 * and runs the watchdog check. 1744 * 1745 **********************************************************************/ 1746 1747 static void 1748 ixl_local_timer(void *arg) 1749 { 1750 struct ixl_pf *pf = arg; 1751 struct i40e_hw *hw = &pf->hw; 1752 struct ixl_vsi *vsi = &pf->vsi; 1753 struct ixl_queue *que = vsi->queues; 1754 device_t dev = pf->dev; 1755 int hung = 0; 1756 u32 mask; 1757 1758 mtx_assert(&pf->pf_mtx, MA_OWNED); 1759 1760 /* Fire off the adminq task */ 1761 taskqueue_enqueue(pf->tq, &pf->adminq); 1762 1763 /* Update stats */ 1764 ixl_update_stats_counters(pf); 1765 1766 /* 1767 ** Check status of the queues 1768 */ 1769 mask = (I40E_PFINT_DYN_CTLN_INTENA_MASK | 1770 I40E_PFINT_DYN_CTLN_SWINT_TRIG_MASK); 1771 1772 for (int i = 0; i < vsi->num_queues; i++,que++) { 1773 /* Any queues with outstanding work get a sw irq */ 1774 if (que->busy) 1775 wr32(hw, I40E_PFINT_DYN_CTLN(que->me), mask); 1776 /* 1777 ** Each time txeof runs without cleaning, but there 1778 ** are uncleaned descriptors it increments busy. If 1779 ** we get to 5 we declare it hung. 1780 */ 1781 if (que->busy == IXL_QUEUE_HUNG) { 1782 ++hung; 1783 /* Mark the queue as inactive */ 1784 vsi->active_queues &= ~((u64)1 << que->me); 1785 continue; 1786 } else { 1787 /* Check if we've come back from hung */ 1788 if ((vsi->active_queues & ((u64)1 << que->me)) == 0) 1789 vsi->active_queues |= ((u64)1 << que->me); 1790 } 1791 if (que->busy >= IXL_MAX_TX_BUSY) { 1792 #ifdef IXL_DEBUG 1793 device_printf(dev,"Warning queue %d " 1794 "appears to be hung!\n", i); 1795 #endif 1796 que->busy = IXL_QUEUE_HUNG; 1797 ++hung; 1798 } 1799 } 1800 /* Only reinit if all queues show hung */ 1801 if (hung == vsi->num_queues) 1802 goto hung; 1803 1804 callout_reset(&pf->timer, hz, ixl_local_timer, pf); 1805 return; 1806 1807 hung: 1808 device_printf(dev, "Local Timer: HANG DETECT - Resetting!!\n"); 1809 ixl_init_locked(pf); 1810 } 1811 1812 /* 1813 ** Note: this routine updates the OS on the link state 1814 ** the real check of the hardware only happens with 1815 ** a link interrupt. 1816 */ 1817 static void 1818 ixl_update_link_status(struct ixl_pf *pf) 1819 { 1820 struct ixl_vsi *vsi = &pf->vsi; 1821 struct i40e_hw *hw = &pf->hw; 1822 struct ifnet *ifp = vsi->ifp; 1823 device_t dev = pf->dev; 1824 1825 if (pf->link_up){ 1826 if (vsi->link_active == FALSE) { 1827 pf->fc = hw->fc.current_mode; 1828 if (bootverbose) { 1829 device_printf(dev,"Link is up %d Gbps %s," 1830 " Flow Control: %s\n", 1831 ((pf->link_speed == 1832 I40E_LINK_SPEED_40GB)? 40:10), 1833 "Full Duplex", ixl_fc_string[pf->fc]); 1834 } 1835 vsi->link_active = TRUE; 1836 /* 1837 ** Warn user if link speed on NPAR enabled 1838 ** partition is not at least 10GB 1839 */ 1840 if (hw->func_caps.npar_enable && 1841 (hw->phy.link_info.link_speed == 1842 I40E_LINK_SPEED_1GB || 1843 hw->phy.link_info.link_speed == 1844 I40E_LINK_SPEED_100MB)) 1845 device_printf(dev, "The partition detected" 1846 "link speed that is less than 10Gbps\n"); 1847 if_link_state_change(ifp, LINK_STATE_UP); 1848 } 1849 } else { /* Link down */ 1850 if (vsi->link_active == TRUE) { 1851 if (bootverbose) 1852 device_printf(dev,"Link is Down\n"); 1853 if_link_state_change(ifp, LINK_STATE_DOWN); 1854 vsi->link_active = FALSE; 1855 } 1856 } 1857 1858 return; 1859 } 1860 1861 /********************************************************************* 1862 * 1863 * This routine disables all traffic on the adapter by issuing a 1864 * global reset on the MAC and deallocates TX/RX buffers. 1865 * 1866 **********************************************************************/ 1867 1868 static void 1869 ixl_stop(struct ixl_pf *pf) 1870 { 1871 struct ixl_vsi *vsi = &pf->vsi; 1872 struct ifnet *ifp = vsi->ifp; 1873 1874 mtx_assert(&pf->pf_mtx, MA_OWNED); 1875 1876 INIT_DEBUGOUT("ixl_stop: begin\n"); 1877 if (pf->num_vfs == 0) 1878 ixl_disable_intr(vsi); 1879 else 1880 ixl_disable_rings_intr(vsi); 1881 ixl_disable_rings(vsi); 1882 1883 /* Tell the stack that the interface is no longer active */ 1884 ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE); 1885 1886 /* Stop the local timer */ 1887 callout_stop(&pf->timer); 1888 1889 return; 1890 } 1891 1892 1893 /********************************************************************* 1894 * 1895 * Setup MSIX Interrupt resources and handlers for the VSI 1896 * 1897 **********************************************************************/ 1898 static int 1899 ixl_assign_vsi_legacy(struct ixl_pf *pf) 1900 { 1901 device_t dev = pf->dev; 1902 struct ixl_vsi *vsi = &pf->vsi; 1903 struct ixl_queue *que = vsi->queues; 1904 int error, rid = 0; 1905 1906 if (pf->msix == 1) 1907 rid = 1; 1908 pf->res = bus_alloc_resource_any(dev, SYS_RES_IRQ, 1909 &rid, RF_SHAREABLE | RF_ACTIVE); 1910 if (pf->res == NULL) { 1911 device_printf(dev,"Unable to allocate" 1912 " bus resource: vsi legacy/msi interrupt\n"); 1913 return (ENXIO); 1914 } 1915 1916 /* Set the handler function */ 1917 error = bus_setup_intr(dev, pf->res, 1918 INTR_TYPE_NET | INTR_MPSAFE, NULL, 1919 ixl_intr, pf, &pf->tag); 1920 if (error) { 1921 pf->res = NULL; 1922 device_printf(dev, "Failed to register legacy/msi handler"); 1923 return (error); 1924 } 1925 bus_describe_intr(dev, pf->res, pf->tag, "irq0"); 1926 TASK_INIT(&que->tx_task, 0, ixl_deferred_mq_start, que); 1927 TASK_INIT(&que->task, 0, ixl_handle_que, que); 1928 que->tq = taskqueue_create_fast("ixl_que", M_NOWAIT, 1929 taskqueue_thread_enqueue, &que->tq); 1930 taskqueue_start_threads(&que->tq, 1, PI_NET, "%s que", 1931 device_get_nameunit(dev)); 1932 TASK_INIT(&pf->adminq, 0, ixl_do_adminq, pf); 1933 1934 #ifdef PCI_IOV 1935 TASK_INIT(&pf->vflr_task, 0, ixl_handle_vflr, pf); 1936 #endif 1937 1938 pf->tq = taskqueue_create_fast("ixl_adm", M_NOWAIT, 1939 taskqueue_thread_enqueue, &pf->tq); 1940 taskqueue_start_threads(&pf->tq, 1, PI_NET, "%s adminq", 1941 device_get_nameunit(dev)); 1942 1943 return (0); 1944 } 1945 1946 1947 /********************************************************************* 1948 * 1949 * Setup MSIX Interrupt resources and handlers for the VSI 1950 * 1951 **********************************************************************/ 1952 static int 1953 ixl_assign_vsi_msix(struct ixl_pf *pf) 1954 { 1955 device_t dev = pf->dev; 1956 struct ixl_vsi *vsi = &pf->vsi; 1957 struct ixl_queue *que = vsi->queues; 1958 struct tx_ring *txr; 1959 int error, rid, vector = 0; 1960 #ifdef RSS 1961 cpuset_t cpu_mask; 1962 #endif 1963 1964 /* Admin Que is vector 0*/ 1965 rid = vector + 1; 1966 pf->res = bus_alloc_resource_any(dev, 1967 SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE); 1968 if (!pf->res) { 1969 device_printf(dev,"Unable to allocate" 1970 " bus resource: Adminq interrupt [%d]\n", rid); 1971 return (ENXIO); 1972 } 1973 /* Set the adminq vector and handler */ 1974 error = bus_setup_intr(dev, pf->res, 1975 INTR_TYPE_NET | INTR_MPSAFE, NULL, 1976 ixl_msix_adminq, pf, &pf->tag); 1977 if (error) { 1978 pf->res = NULL; 1979 device_printf(dev, "Failed to register Admin que handler"); 1980 return (error); 1981 } 1982 bus_describe_intr(dev, pf->res, pf->tag, "aq"); 1983 pf->admvec = vector; 1984 /* Tasklet for Admin Queue */ 1985 TASK_INIT(&pf->adminq, 0, ixl_do_adminq, pf); 1986 1987 #ifdef PCI_IOV 1988 TASK_INIT(&pf->vflr_task, 0, ixl_handle_vflr, pf); 1989 #endif 1990 1991 pf->tq = taskqueue_create_fast("ixl_adm", M_NOWAIT, 1992 taskqueue_thread_enqueue, &pf->tq); 1993 taskqueue_start_threads(&pf->tq, 1, PI_NET, "%s adminq", 1994 device_get_nameunit(pf->dev)); 1995 ++vector; 1996 1997 /* Now set up the stations */ 1998 for (int i = 0; i < vsi->num_queues; i++, vector++, que++) { 1999 int cpu_id = i; 2000 rid = vector + 1; 2001 txr = &que->txr; 2002 que->res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid, 2003 RF_SHAREABLE | RF_ACTIVE); 2004 if (que->res == NULL) { 2005 device_printf(dev,"Unable to allocate" 2006 " bus resource: que interrupt [%d]\n", vector); 2007 return (ENXIO); 2008 } 2009 /* Set the handler function */ 2010 error = bus_setup_intr(dev, que->res, 2011 INTR_TYPE_NET | INTR_MPSAFE, NULL, 2012 ixl_msix_que, que, &que->tag); 2013 if (error) { 2014 que->res = NULL; 2015 device_printf(dev, "Failed to register que handler"); 2016 return (error); 2017 } 2018 bus_describe_intr(dev, que->res, que->tag, "q%d", i); 2019 /* Bind the vector to a CPU */ 2020 #ifdef RSS 2021 cpu_id = rss_getcpu(i % rss_getnumbuckets()); 2022 #endif 2023 bus_bind_intr(dev, que->res, cpu_id); 2024 que->msix = vector; 2025 TASK_INIT(&que->tx_task, 0, ixl_deferred_mq_start, que); 2026 TASK_INIT(&que->task, 0, ixl_handle_que, que); 2027 que->tq = taskqueue_create_fast("ixl_que", M_NOWAIT, 2028 taskqueue_thread_enqueue, &que->tq); 2029 #ifdef RSS 2030 CPU_SETOF(cpu_id, &cpu_mask); 2031 taskqueue_start_threads_cpuset(&que->tq, 1, PI_NET, 2032 &cpu_mask, "%s (bucket %d)", 2033 device_get_nameunit(dev), cpu_id); 2034 #else 2035 taskqueue_start_threads(&que->tq, 1, PI_NET, 2036 "%s que", device_get_nameunit(dev)); 2037 #endif 2038 } 2039 2040 return (0); 2041 } 2042 2043 2044 /* 2045 * Allocate MSI/X vectors 2046 */ 2047 static int 2048 ixl_init_msix(struct ixl_pf *pf) 2049 { 2050 device_t dev = pf->dev; 2051 int rid, want, vectors, queues, available; 2052 2053 /* Override by tuneable */ 2054 if (ixl_enable_msix == 0) 2055 goto msi; 2056 2057 /* 2058 ** When used in a virtualized environment 2059 ** PCI BUSMASTER capability may not be set 2060 ** so explicity set it here and rewrite 2061 ** the ENABLE in the MSIX control register 2062 ** at this point to cause the host to 2063 ** successfully initialize us. 2064 */ 2065 { 2066 u16 pci_cmd_word; 2067 int msix_ctrl; 2068 pci_cmd_word = pci_read_config(dev, PCIR_COMMAND, 2); 2069 pci_cmd_word |= PCIM_CMD_BUSMASTEREN; 2070 pci_write_config(dev, PCIR_COMMAND, pci_cmd_word, 2); 2071 pci_find_cap(dev, PCIY_MSIX, &rid); 2072 rid += PCIR_MSIX_CTRL; 2073 msix_ctrl = pci_read_config(dev, rid, 2); 2074 msix_ctrl |= PCIM_MSIXCTRL_MSIX_ENABLE; 2075 pci_write_config(dev, rid, msix_ctrl, 2); 2076 } 2077 2078 /* First try MSI/X */ 2079 rid = PCIR_BAR(IXL_BAR); 2080 pf->msix_mem = bus_alloc_resource_any(dev, 2081 SYS_RES_MEMORY, &rid, RF_ACTIVE); 2082 if (!pf->msix_mem) { 2083 /* May not be enabled */ 2084 device_printf(pf->dev, 2085 "Unable to map MSIX table \n"); 2086 goto msi; 2087 } 2088 2089 available = pci_msix_count(dev); 2090 if (available == 0) { /* system has msix disabled */ 2091 bus_release_resource(dev, SYS_RES_MEMORY, 2092 rid, pf->msix_mem); 2093 pf->msix_mem = NULL; 2094 goto msi; 2095 } 2096 2097 /* Figure out a reasonable auto config value */ 2098 queues = (mp_ncpus > (available - 1)) ? (available - 1) : mp_ncpus; 2099 2100 /* Override with hardcoded value if sane */ 2101 if ((ixl_max_queues != 0) && (ixl_max_queues <= queues)) 2102 queues = ixl_max_queues; 2103 2104 #ifdef RSS 2105 /* If we're doing RSS, clamp at the number of RSS buckets */ 2106 if (queues > rss_getnumbuckets()) 2107 queues = rss_getnumbuckets(); 2108 #endif 2109 2110 /* 2111 ** Want one vector (RX/TX pair) per queue 2112 ** plus an additional for the admin queue. 2113 */ 2114 want = queues + 1; 2115 if (want <= available) /* Have enough */ 2116 vectors = want; 2117 else { 2118 device_printf(pf->dev, 2119 "MSIX Configuration Problem, " 2120 "%d vectors available but %d wanted!\n", 2121 available, want); 2122 return (0); /* Will go to Legacy setup */ 2123 } 2124 2125 if (pci_alloc_msix(dev, &vectors) == 0) { 2126 device_printf(pf->dev, 2127 "Using MSIX interrupts with %d vectors\n", vectors); 2128 pf->msix = vectors; 2129 pf->vsi.num_queues = queues; 2130 #ifdef RSS 2131 /* 2132 * If we're doing RSS, the number of queues needs to 2133 * match the number of RSS buckets that are configured. 2134 * 2135 * + If there's more queues than RSS buckets, we'll end 2136 * up with queues that get no traffic. 2137 * 2138 * + If there's more RSS buckets than queues, we'll end 2139 * up having multiple RSS buckets map to the same queue, 2140 * so there'll be some contention. 2141 */ 2142 if (queues != rss_getnumbuckets()) { 2143 device_printf(dev, 2144 "%s: queues (%d) != RSS buckets (%d)" 2145 "; performance will be impacted.\n", 2146 __func__, queues, rss_getnumbuckets()); 2147 } 2148 #endif 2149 return (vectors); 2150 } 2151 msi: 2152 vectors = pci_msi_count(dev); 2153 pf->vsi.num_queues = 1; 2154 pf->msix = 1; 2155 ixl_max_queues = 1; 2156 ixl_enable_msix = 0; 2157 if (vectors == 1 && pci_alloc_msi(dev, &vectors) == 0) 2158 device_printf(pf->dev,"Using an MSI interrupt\n"); 2159 else { 2160 pf->msix = 0; 2161 device_printf(pf->dev,"Using a Legacy interrupt\n"); 2162 } 2163 return (vectors); 2164 } 2165 2166 2167 /* 2168 * Plumb MSI/X vectors 2169 */ 2170 static void 2171 ixl_configure_msix(struct ixl_pf *pf) 2172 { 2173 struct i40e_hw *hw = &pf->hw; 2174 struct ixl_vsi *vsi = &pf->vsi; 2175 u32 reg; 2176 u16 vector = 1; 2177 2178 /* First set up the adminq - vector 0 */ 2179 wr32(hw, I40E_PFINT_ICR0_ENA, 0); /* disable all */ 2180 rd32(hw, I40E_PFINT_ICR0); /* read to clear */ 2181 2182 reg = I40E_PFINT_ICR0_ENA_ECC_ERR_MASK | 2183 I40E_PFINT_ICR0_ENA_GRST_MASK | 2184 I40E_PFINT_ICR0_HMC_ERR_MASK | 2185 I40E_PFINT_ICR0_ENA_ADMINQ_MASK | 2186 I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK | 2187 I40E_PFINT_ICR0_ENA_VFLR_MASK | 2188 I40E_PFINT_ICR0_ENA_PCI_EXCEPTION_MASK; 2189 wr32(hw, I40E_PFINT_ICR0_ENA, reg); 2190 2191 wr32(hw, I40E_PFINT_LNKLST0, 0x7FF); 2192 wr32(hw, I40E_PFINT_ITR0(IXL_RX_ITR), 0x003E); 2193 2194 wr32(hw, I40E_PFINT_DYN_CTL0, 2195 I40E_PFINT_DYN_CTL0_SW_ITR_INDX_MASK | 2196 I40E_PFINT_DYN_CTL0_INTENA_MSK_MASK); 2197 2198 wr32(hw, I40E_PFINT_STAT_CTL0, 0); 2199 2200 /* Next configure the queues */ 2201 for (int i = 0; i < vsi->num_queues; i++, vector++) { 2202 wr32(hw, I40E_PFINT_DYN_CTLN(i), i); 2203 wr32(hw, I40E_PFINT_LNKLSTN(i), i); 2204 2205 reg = I40E_QINT_RQCTL_CAUSE_ENA_MASK | 2206 (IXL_RX_ITR << I40E_QINT_RQCTL_ITR_INDX_SHIFT) | 2207 (vector << I40E_QINT_RQCTL_MSIX_INDX_SHIFT) | 2208 (i << I40E_QINT_RQCTL_NEXTQ_INDX_SHIFT) | 2209 (I40E_QUEUE_TYPE_TX << I40E_QINT_RQCTL_NEXTQ_TYPE_SHIFT); 2210 wr32(hw, I40E_QINT_RQCTL(i), reg); 2211 2212 reg = I40E_QINT_TQCTL_CAUSE_ENA_MASK | 2213 (IXL_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT) | 2214 (vector << I40E_QINT_TQCTL_MSIX_INDX_SHIFT) | 2215 ((i+1) << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT) | 2216 (I40E_QUEUE_TYPE_RX << I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT); 2217 if (i == (vsi->num_queues - 1)) 2218 reg |= (IXL_QUEUE_EOL 2219 << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT); 2220 wr32(hw, I40E_QINT_TQCTL(i), reg); 2221 } 2222 } 2223 2224 /* 2225 * Configure for MSI single vector operation 2226 */ 2227 static void 2228 ixl_configure_legacy(struct ixl_pf *pf) 2229 { 2230 struct i40e_hw *hw = &pf->hw; 2231 u32 reg; 2232 2233 2234 wr32(hw, I40E_PFINT_ITR0(0), 0); 2235 wr32(hw, I40E_PFINT_ITR0(1), 0); 2236 2237 2238 /* Setup "other" causes */ 2239 reg = I40E_PFINT_ICR0_ENA_ECC_ERR_MASK 2240 | I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK 2241 | I40E_PFINT_ICR0_ENA_GRST_MASK 2242 | I40E_PFINT_ICR0_ENA_PCI_EXCEPTION_MASK 2243 | I40E_PFINT_ICR0_ENA_GPIO_MASK 2244 | I40E_PFINT_ICR0_ENA_LINK_STAT_CHANGE_MASK 2245 | I40E_PFINT_ICR0_ENA_HMC_ERR_MASK 2246 | I40E_PFINT_ICR0_ENA_PE_CRITERR_MASK 2247 | I40E_PFINT_ICR0_ENA_VFLR_MASK 2248 | I40E_PFINT_ICR0_ENA_ADMINQ_MASK 2249 ; 2250 wr32(hw, I40E_PFINT_ICR0_ENA, reg); 2251 2252 /* SW_ITR_IDX = 0, but don't change INTENA */ 2253 wr32(hw, I40E_PFINT_DYN_CTL0, 2254 I40E_PFINT_DYN_CTLN_SW_ITR_INDX_MASK | 2255 I40E_PFINT_DYN_CTLN_INTENA_MSK_MASK); 2256 /* SW_ITR_IDX = 0, OTHER_ITR_IDX = 0 */ 2257 wr32(hw, I40E_PFINT_STAT_CTL0, 0); 2258 2259 /* FIRSTQ_INDX = 0, FIRSTQ_TYPE = 0 (rx) */ 2260 wr32(hw, I40E_PFINT_LNKLST0, 0); 2261 2262 /* Associate the queue pair to the vector and enable the q int */ 2263 reg = I40E_QINT_RQCTL_CAUSE_ENA_MASK 2264 | (IXL_RX_ITR << I40E_QINT_RQCTL_ITR_INDX_SHIFT) 2265 | (I40E_QUEUE_TYPE_TX << I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT); 2266 wr32(hw, I40E_QINT_RQCTL(0), reg); 2267 2268 reg = I40E_QINT_TQCTL_CAUSE_ENA_MASK 2269 | (IXL_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT) 2270 | (IXL_QUEUE_EOL << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT); 2271 wr32(hw, I40E_QINT_TQCTL(0), reg); 2272 2273 /* Next enable the queue pair */ 2274 reg = rd32(hw, I40E_QTX_ENA(0)); 2275 reg |= I40E_QTX_ENA_QENA_REQ_MASK; 2276 wr32(hw, I40E_QTX_ENA(0), reg); 2277 2278 reg = rd32(hw, I40E_QRX_ENA(0)); 2279 reg |= I40E_QRX_ENA_QENA_REQ_MASK; 2280 wr32(hw, I40E_QRX_ENA(0), reg); 2281 } 2282 2283 2284 /* 2285 * Set the Initial ITR state 2286 */ 2287 static void 2288 ixl_configure_itr(struct ixl_pf *pf) 2289 { 2290 struct i40e_hw *hw = &pf->hw; 2291 struct ixl_vsi *vsi = &pf->vsi; 2292 struct ixl_queue *que = vsi->queues; 2293 2294 vsi->rx_itr_setting = ixl_rx_itr; 2295 if (ixl_dynamic_rx_itr) 2296 vsi->rx_itr_setting |= IXL_ITR_DYNAMIC; 2297 vsi->tx_itr_setting = ixl_tx_itr; 2298 if (ixl_dynamic_tx_itr) 2299 vsi->tx_itr_setting |= IXL_ITR_DYNAMIC; 2300 2301 for (int i = 0; i < vsi->num_queues; i++, que++) { 2302 struct tx_ring *txr = &que->txr; 2303 struct rx_ring *rxr = &que->rxr; 2304 2305 wr32(hw, I40E_PFINT_ITRN(IXL_RX_ITR, i), 2306 vsi->rx_itr_setting); 2307 rxr->itr = vsi->rx_itr_setting; 2308 rxr->latency = IXL_AVE_LATENCY; 2309 wr32(hw, I40E_PFINT_ITRN(IXL_TX_ITR, i), 2310 vsi->tx_itr_setting); 2311 txr->itr = vsi->tx_itr_setting; 2312 txr->latency = IXL_AVE_LATENCY; 2313 } 2314 } 2315 2316 2317 static int 2318 ixl_allocate_pci_resources(struct ixl_pf *pf) 2319 { 2320 int rid; 2321 device_t dev = pf->dev; 2322 2323 rid = PCIR_BAR(0); 2324 pf->pci_mem = bus_alloc_resource_any(dev, SYS_RES_MEMORY, 2325 &rid, RF_ACTIVE); 2326 2327 if (!(pf->pci_mem)) { 2328 device_printf(dev,"Unable to allocate bus resource: memory\n"); 2329 return (ENXIO); 2330 } 2331 2332 pf->osdep.mem_bus_space_tag = 2333 rman_get_bustag(pf->pci_mem); 2334 pf->osdep.mem_bus_space_handle = 2335 rman_get_bushandle(pf->pci_mem); 2336 pf->osdep.mem_bus_space_size = rman_get_size(pf->pci_mem); 2337 pf->osdep.flush_reg = I40E_GLGEN_STAT; 2338 pf->hw.hw_addr = (u8 *) &pf->osdep.mem_bus_space_handle; 2339 2340 pf->hw.back = &pf->osdep; 2341 2342 /* 2343 ** Now setup MSI or MSI/X, should 2344 ** return us the number of supported 2345 ** vectors. (Will be 1 for MSI) 2346 */ 2347 pf->msix = ixl_init_msix(pf); 2348 return (0); 2349 } 2350 2351 static void 2352 ixl_free_pci_resources(struct ixl_pf * pf) 2353 { 2354 struct ixl_vsi *vsi = &pf->vsi; 2355 struct ixl_queue *que = vsi->queues; 2356 device_t dev = pf->dev; 2357 int rid, memrid; 2358 2359 memrid = PCIR_BAR(IXL_BAR); 2360 2361 /* We may get here before stations are setup */ 2362 if ((!ixl_enable_msix) || (que == NULL)) 2363 goto early; 2364 2365 /* 2366 ** Release all msix VSI resources: 2367 */ 2368 for (int i = 0; i < vsi->num_queues; i++, que++) { 2369 rid = que->msix + 1; 2370 if (que->tag != NULL) { 2371 bus_teardown_intr(dev, que->res, que->tag); 2372 que->tag = NULL; 2373 } 2374 if (que->res != NULL) 2375 bus_release_resource(dev, SYS_RES_IRQ, rid, que->res); 2376 } 2377 2378 early: 2379 /* Clean the AdminQ interrupt last */ 2380 if (pf->admvec) /* we are doing MSIX */ 2381 rid = pf->admvec + 1; 2382 else 2383 (pf->msix != 0) ? (rid = 1):(rid = 0); 2384 2385 if (pf->tag != NULL) { 2386 bus_teardown_intr(dev, pf->res, pf->tag); 2387 pf->tag = NULL; 2388 } 2389 if (pf->res != NULL) 2390 bus_release_resource(dev, SYS_RES_IRQ, rid, pf->res); 2391 2392 if (pf->msix) 2393 pci_release_msi(dev); 2394 2395 if (pf->msix_mem != NULL) 2396 bus_release_resource(dev, SYS_RES_MEMORY, 2397 memrid, pf->msix_mem); 2398 2399 if (pf->pci_mem != NULL) 2400 bus_release_resource(dev, SYS_RES_MEMORY, 2401 PCIR_BAR(0), pf->pci_mem); 2402 2403 return; 2404 } 2405 2406 static void 2407 ixl_add_ifmedia(struct ixl_vsi *vsi, u32 phy_type) 2408 { 2409 /* Display supported media types */ 2410 if (phy_type & (1 << I40E_PHY_TYPE_100BASE_TX)) 2411 ifmedia_add(&vsi->media, IFM_ETHER | IFM_100_TX, 0, NULL); 2412 2413 if (phy_type & (1 << I40E_PHY_TYPE_1000BASE_T)) 2414 ifmedia_add(&vsi->media, IFM_ETHER | IFM_1000_T, 0, NULL); 2415 if (phy_type & (1 << I40E_PHY_TYPE_1000BASE_SX)) 2416 ifmedia_add(&vsi->media, IFM_ETHER | IFM_1000_SX, 0, NULL); 2417 if (phy_type & (1 << I40E_PHY_TYPE_1000BASE_LX)) 2418 ifmedia_add(&vsi->media, IFM_ETHER | IFM_1000_LX, 0, NULL); 2419 2420 if (phy_type & (1 << I40E_PHY_TYPE_10GBASE_CR1_CU) || 2421 phy_type & (1 << I40E_PHY_TYPE_10GBASE_KX4) || 2422 phy_type & (1 << I40E_PHY_TYPE_10GBASE_KR) || 2423 phy_type & (1 << I40E_PHY_TYPE_10GBASE_AOC) || 2424 phy_type & (1 << I40E_PHY_TYPE_XAUI) || 2425 phy_type & (1 << I40E_PHY_TYPE_XFI) || 2426 phy_type & (1 << I40E_PHY_TYPE_SFI) || 2427 phy_type & (1 << I40E_PHY_TYPE_10GBASE_SFPP_CU)) 2428 ifmedia_add(&vsi->media, IFM_ETHER | IFM_10G_TWINAX, 0, NULL); 2429 2430 if (phy_type & (1 << I40E_PHY_TYPE_10GBASE_SR)) 2431 ifmedia_add(&vsi->media, IFM_ETHER | IFM_10G_SR, 0, NULL); 2432 if (phy_type & (1 << I40E_PHY_TYPE_10GBASE_LR)) 2433 ifmedia_add(&vsi->media, IFM_ETHER | IFM_10G_LR, 0, NULL); 2434 if (phy_type & (1 << I40E_PHY_TYPE_10GBASE_T)) 2435 ifmedia_add(&vsi->media, IFM_ETHER | IFM_10G_T, 0, NULL); 2436 2437 if (phy_type & (1 << I40E_PHY_TYPE_40GBASE_CR4) || 2438 phy_type & (1 << I40E_PHY_TYPE_40GBASE_CR4_CU) || 2439 phy_type & (1 << I40E_PHY_TYPE_40GBASE_AOC) || 2440 phy_type & (1 << I40E_PHY_TYPE_XLAUI) || 2441 phy_type & (1 << I40E_PHY_TYPE_XLPPI) || 2442 /* KR4 uses CR4 until the OS has the real media type */ 2443 phy_type & (1 << I40E_PHY_TYPE_40GBASE_KR4)) 2444 ifmedia_add(&vsi->media, IFM_ETHER | IFM_40G_CR4, 0, NULL); 2445 2446 if (phy_type & (1 << I40E_PHY_TYPE_40GBASE_SR4)) 2447 ifmedia_add(&vsi->media, IFM_ETHER | IFM_40G_SR4, 0, NULL); 2448 if (phy_type & (1 << I40E_PHY_TYPE_40GBASE_LR4)) 2449 ifmedia_add(&vsi->media, IFM_ETHER | IFM_40G_LR4, 0, NULL); 2450 } 2451 2452 /********************************************************************* 2453 * 2454 * Setup networking device structure and register an interface. 2455 * 2456 **********************************************************************/ 2457 static int 2458 ixl_setup_interface(device_t dev, struct ixl_vsi *vsi) 2459 { 2460 struct ifnet *ifp; 2461 struct i40e_hw *hw = vsi->hw; 2462 struct ixl_queue *que = vsi->queues; 2463 struct i40e_aq_get_phy_abilities_resp abilities; 2464 enum i40e_status_code aq_error = 0; 2465 2466 INIT_DEBUGOUT("ixl_setup_interface: begin"); 2467 2468 ifp = vsi->ifp = if_alloc(IFT_ETHER); 2469 if (ifp == NULL) { 2470 device_printf(dev, "can not allocate ifnet structure\n"); 2471 return (-1); 2472 } 2473 if_initname(ifp, device_get_name(dev), device_get_unit(dev)); 2474 ifp->if_mtu = ETHERMTU; 2475 ifp->if_baudrate = 4000000000; // ?? 2476 ifp->if_init = ixl_init; 2477 ifp->if_softc = vsi; 2478 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 2479 ifp->if_ioctl = ixl_ioctl; 2480 2481 #if __FreeBSD_version >= 1100036 2482 if_setgetcounterfn(ifp, ixl_get_counter); 2483 #endif 2484 2485 ifp->if_transmit = ixl_mq_start; 2486 2487 ifp->if_qflush = ixl_qflush; 2488 2489 ifp->if_snd.ifq_maxlen = que->num_desc - 2; 2490 2491 vsi->max_frame_size = 2492 ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN 2493 + ETHER_VLAN_ENCAP_LEN; 2494 2495 /* 2496 * Tell the upper layer(s) we support long frames. 2497 */ 2498 ifp->if_hdrlen = sizeof(struct ether_vlan_header); 2499 2500 ifp->if_capabilities |= IFCAP_HWCSUM; 2501 ifp->if_capabilities |= IFCAP_HWCSUM_IPV6; 2502 ifp->if_capabilities |= IFCAP_TSO; 2503 ifp->if_capabilities |= IFCAP_JUMBO_MTU; 2504 ifp->if_capabilities |= IFCAP_LRO; 2505 2506 /* VLAN capabilties */ 2507 ifp->if_capabilities |= IFCAP_VLAN_HWTAGGING 2508 | IFCAP_VLAN_HWTSO 2509 | IFCAP_VLAN_MTU 2510 | IFCAP_VLAN_HWCSUM; 2511 ifp->if_capenable = ifp->if_capabilities; 2512 2513 /* 2514 ** Don't turn this on by default, if vlans are 2515 ** created on another pseudo device (eg. lagg) 2516 ** then vlan events are not passed thru, breaking 2517 ** operation, but with HW FILTER off it works. If 2518 ** using vlans directly on the ixl driver you can 2519 ** enable this and get full hardware tag filtering. 2520 */ 2521 ifp->if_capabilities |= IFCAP_VLAN_HWFILTER; 2522 2523 /* 2524 * Specify the media types supported by this adapter and register 2525 * callbacks to update media and link information 2526 */ 2527 ifmedia_init(&vsi->media, IFM_IMASK, ixl_media_change, 2528 ixl_media_status); 2529 2530 aq_error = i40e_aq_get_phy_capabilities(hw, 2531 FALSE, TRUE, &abilities, NULL); 2532 /* May need delay to detect fiber correctly */ 2533 if (aq_error == I40E_ERR_UNKNOWN_PHY) { 2534 i40e_msec_delay(200); 2535 aq_error = i40e_aq_get_phy_capabilities(hw, FALSE, 2536 TRUE, &abilities, NULL); 2537 } 2538 if (aq_error) { 2539 if (aq_error == I40E_ERR_UNKNOWN_PHY) 2540 device_printf(dev, "Unknown PHY type detected!\n"); 2541 else 2542 device_printf(dev, 2543 "Error getting supported media types, err %d," 2544 " AQ error %d\n", aq_error, hw->aq.asq_last_status); 2545 return (0); 2546 } 2547 2548 ixl_add_ifmedia(vsi, abilities.phy_type); 2549 2550 /* Use autoselect media by default */ 2551 ifmedia_add(&vsi->media, IFM_ETHER | IFM_AUTO, 0, NULL); 2552 ifmedia_set(&vsi->media, IFM_ETHER | IFM_AUTO); 2553 2554 ether_ifattach(ifp, hw->mac.addr); 2555 2556 return (0); 2557 } 2558 2559 /* 2560 ** Run when the Admin Queue gets a 2561 ** link transition interrupt. 2562 */ 2563 static void 2564 ixl_link_event(struct ixl_pf *pf, struct i40e_arq_event_info *e) 2565 { 2566 struct i40e_hw *hw = &pf->hw; 2567 struct i40e_aqc_get_link_status *status = 2568 (struct i40e_aqc_get_link_status *)&e->desc.params.raw; 2569 bool check; 2570 2571 hw->phy.get_link_info = TRUE; 2572 check = i40e_get_link_status(hw); 2573 pf->link_up = check; 2574 #ifdef IXL_DEBUG 2575 printf("Link is %s\n", check ? "up":"down"); 2576 #endif 2577 /* Report if Unqualified modules are found */ 2578 if ((status->link_info & I40E_AQ_MEDIA_AVAILABLE) && 2579 (!(status->an_info & I40E_AQ_QUALIFIED_MODULE)) && 2580 (!(status->link_info & I40E_AQ_LINK_UP))) 2581 device_printf(pf->dev, "Link failed because " 2582 "an unqualified module was detected\n"); 2583 2584 return; 2585 } 2586 2587 /********************************************************************* 2588 * 2589 * Get Firmware Switch configuration 2590 * - this will need to be more robust when more complex 2591 * switch configurations are enabled. 2592 * 2593 **********************************************************************/ 2594 static int 2595 ixl_switch_config(struct ixl_pf *pf) 2596 { 2597 struct i40e_hw *hw = &pf->hw; 2598 struct ixl_vsi *vsi = &pf->vsi; 2599 device_t dev = vsi->dev; 2600 struct i40e_aqc_get_switch_config_resp *sw_config; 2601 u8 aq_buf[I40E_AQ_LARGE_BUF]; 2602 int ret; 2603 u16 next = 0; 2604 2605 memset(&aq_buf, 0, sizeof(aq_buf)); 2606 sw_config = (struct i40e_aqc_get_switch_config_resp *)aq_buf; 2607 ret = i40e_aq_get_switch_config(hw, sw_config, 2608 sizeof(aq_buf), &next, NULL); 2609 if (ret) { 2610 device_printf(dev,"aq_get_switch_config failed (ret=%d)!!\n", 2611 ret); 2612 return (ret); 2613 } 2614 #ifdef IXL_DEBUG 2615 device_printf(dev, 2616 "Switch config: header reported: %d in structure, %d total\n", 2617 sw_config->header.num_reported, sw_config->header.num_total); 2618 for (int i = 0; i < sw_config->header.num_reported; i++) { 2619 device_printf(dev, 2620 "%d: type=%d seid=%d uplink=%d downlink=%d\n", i, 2621 sw_config->element[i].element_type, 2622 sw_config->element[i].seid, 2623 sw_config->element[i].uplink_seid, 2624 sw_config->element[i].downlink_seid); 2625 } 2626 #endif 2627 /* Simplified due to a single VSI at the moment */ 2628 vsi->uplink_seid = sw_config->element[0].uplink_seid; 2629 vsi->downlink_seid = sw_config->element[0].downlink_seid; 2630 vsi->seid = sw_config->element[0].seid; 2631 return (ret); 2632 } 2633 2634 /********************************************************************* 2635 * 2636 * Initialize the VSI: this handles contexts, which means things 2637 * like the number of descriptors, buffer size, 2638 * plus we init the rings thru this function. 2639 * 2640 **********************************************************************/ 2641 static int 2642 ixl_initialize_vsi(struct ixl_vsi *vsi) 2643 { 2644 struct ixl_pf *pf = vsi->back; 2645 struct ixl_queue *que = vsi->queues; 2646 device_t dev = vsi->dev; 2647 struct i40e_hw *hw = vsi->hw; 2648 struct i40e_vsi_context ctxt; 2649 int err = 0; 2650 2651 memset(&ctxt, 0, sizeof(ctxt)); 2652 ctxt.seid = vsi->seid; 2653 if (pf->veb_seid != 0) 2654 ctxt.uplink_seid = pf->veb_seid; 2655 ctxt.pf_num = hw->pf_id; 2656 err = i40e_aq_get_vsi_params(hw, &ctxt, NULL); 2657 if (err) { 2658 device_printf(dev,"get vsi params failed %x!!\n", err); 2659 return (err); 2660 } 2661 #ifdef IXL_DEBUG 2662 printf("get_vsi_params: seid: %d, uplinkseid: %d, vsi_number: %d, " 2663 "vsis_allocated: %d, vsis_unallocated: %d, flags: 0x%x, " 2664 "pfnum: %d, vfnum: %d, stat idx: %d, enabled: %d\n", ctxt.seid, 2665 ctxt.uplink_seid, ctxt.vsi_number, 2666 ctxt.vsis_allocated, ctxt.vsis_unallocated, 2667 ctxt.flags, ctxt.pf_num, ctxt.vf_num, 2668 ctxt.info.stat_counter_idx, ctxt.info.up_enable_bits); 2669 #endif 2670 /* 2671 ** Set the queue and traffic class bits 2672 ** - when multiple traffic classes are supported 2673 ** this will need to be more robust. 2674 */ 2675 ctxt.info.valid_sections = I40E_AQ_VSI_PROP_QUEUE_MAP_VALID; 2676 ctxt.info.mapping_flags |= I40E_AQ_VSI_QUE_MAP_CONTIG; 2677 ctxt.info.queue_mapping[0] = 0; 2678 ctxt.info.tc_mapping[0] = 0x0800; 2679 2680 /* Set VLAN receive stripping mode */ 2681 ctxt.info.valid_sections |= I40E_AQ_VSI_PROP_VLAN_VALID; 2682 ctxt.info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_ALL; 2683 if (vsi->ifp->if_capenable & IFCAP_VLAN_HWTAGGING) 2684 ctxt.info.port_vlan_flags |= I40E_AQ_VSI_PVLAN_EMOD_STR_BOTH; 2685 else 2686 ctxt.info.port_vlan_flags |= I40E_AQ_VSI_PVLAN_EMOD_NOTHING; 2687 2688 /* Keep copy of VSI info in VSI for statistic counters */ 2689 memcpy(&vsi->info, &ctxt.info, sizeof(ctxt.info)); 2690 2691 /* Reset VSI statistics */ 2692 ixl_vsi_reset_stats(vsi); 2693 vsi->hw_filters_add = 0; 2694 vsi->hw_filters_del = 0; 2695 2696 ctxt.flags = htole16(I40E_AQ_VSI_TYPE_PF); 2697 2698 err = i40e_aq_update_vsi_params(hw, &ctxt, NULL); 2699 if (err) { 2700 device_printf(dev,"update vsi params failed %x!!\n", 2701 hw->aq.asq_last_status); 2702 return (err); 2703 } 2704 2705 for (int i = 0; i < vsi->num_queues; i++, que++) { 2706 struct tx_ring *txr = &que->txr; 2707 struct rx_ring *rxr = &que->rxr; 2708 struct i40e_hmc_obj_txq tctx; 2709 struct i40e_hmc_obj_rxq rctx; 2710 u32 txctl; 2711 u16 size; 2712 2713 2714 /* Setup the HMC TX Context */ 2715 size = que->num_desc * sizeof(struct i40e_tx_desc); 2716 memset(&tctx, 0, sizeof(struct i40e_hmc_obj_txq)); 2717 tctx.new_context = 1; 2718 tctx.base = (txr->dma.pa/IXL_TX_CTX_BASE_UNITS); 2719 tctx.qlen = que->num_desc; 2720 tctx.fc_ena = 0; 2721 tctx.rdylist = vsi->info.qs_handle[0]; /* index is TC */ 2722 /* Enable HEAD writeback */ 2723 tctx.head_wb_ena = 1; 2724 tctx.head_wb_addr = txr->dma.pa + 2725 (que->num_desc * sizeof(struct i40e_tx_desc)); 2726 tctx.rdylist_act = 0; 2727 err = i40e_clear_lan_tx_queue_context(hw, i); 2728 if (err) { 2729 device_printf(dev, "Unable to clear TX context\n"); 2730 break; 2731 } 2732 err = i40e_set_lan_tx_queue_context(hw, i, &tctx); 2733 if (err) { 2734 device_printf(dev, "Unable to set TX context\n"); 2735 break; 2736 } 2737 /* Associate the ring with this PF */ 2738 txctl = I40E_QTX_CTL_PF_QUEUE; 2739 txctl |= ((hw->pf_id << I40E_QTX_CTL_PF_INDX_SHIFT) & 2740 I40E_QTX_CTL_PF_INDX_MASK); 2741 wr32(hw, I40E_QTX_CTL(i), txctl); 2742 ixl_flush(hw); 2743 2744 /* Do ring (re)init */ 2745 ixl_init_tx_ring(que); 2746 2747 /* Next setup the HMC RX Context */ 2748 if (vsi->max_frame_size <= MCLBYTES) 2749 rxr->mbuf_sz = MCLBYTES; 2750 else 2751 rxr->mbuf_sz = MJUMPAGESIZE; 2752 2753 u16 max_rxmax = rxr->mbuf_sz * hw->func_caps.rx_buf_chain_len; 2754 2755 /* Set up an RX context for the HMC */ 2756 memset(&rctx, 0, sizeof(struct i40e_hmc_obj_rxq)); 2757 rctx.dbuff = rxr->mbuf_sz >> I40E_RXQ_CTX_DBUFF_SHIFT; 2758 /* ignore header split for now */ 2759 rctx.hbuff = 0 >> I40E_RXQ_CTX_HBUFF_SHIFT; 2760 rctx.rxmax = (vsi->max_frame_size < max_rxmax) ? 2761 vsi->max_frame_size : max_rxmax; 2762 rctx.dtype = 0; 2763 rctx.dsize = 1; /* do 32byte descriptors */ 2764 rctx.hsplit_0 = 0; /* no HDR split initially */ 2765 rctx.base = (rxr->dma.pa/IXL_RX_CTX_BASE_UNITS); 2766 rctx.qlen = que->num_desc; 2767 rctx.tphrdesc_ena = 1; 2768 rctx.tphwdesc_ena = 1; 2769 rctx.tphdata_ena = 0; 2770 rctx.tphhead_ena = 0; 2771 rctx.lrxqthresh = 2; 2772 rctx.crcstrip = 1; 2773 rctx.l2tsel = 1; 2774 rctx.showiv = 1; 2775 rctx.fc_ena = 0; 2776 rctx.prefena = 1; 2777 2778 err = i40e_clear_lan_rx_queue_context(hw, i); 2779 if (err) { 2780 device_printf(dev, 2781 "Unable to clear RX context %d\n", i); 2782 break; 2783 } 2784 err = i40e_set_lan_rx_queue_context(hw, i, &rctx); 2785 if (err) { 2786 device_printf(dev, "Unable to set RX context %d\n", i); 2787 break; 2788 } 2789 err = ixl_init_rx_ring(que); 2790 if (err) { 2791 device_printf(dev, "Fail in init_rx_ring %d\n", i); 2792 break; 2793 } 2794 wr32(vsi->hw, I40E_QRX_TAIL(que->me), 0); 2795 #ifdef DEV_NETMAP 2796 /* preserve queue */ 2797 if (vsi->ifp->if_capenable & IFCAP_NETMAP) { 2798 struct netmap_adapter *na = NA(vsi->ifp); 2799 struct netmap_kring *kring = &na->rx_rings[i]; 2800 int t = na->num_rx_desc - 1 - nm_kr_rxspace(kring); 2801 wr32(vsi->hw, I40E_QRX_TAIL(que->me), t); 2802 } else 2803 #endif /* DEV_NETMAP */ 2804 wr32(vsi->hw, I40E_QRX_TAIL(que->me), que->num_desc - 1); 2805 } 2806 return (err); 2807 } 2808 2809 2810 /********************************************************************* 2811 * 2812 * Free all VSI structs. 2813 * 2814 **********************************************************************/ 2815 void 2816 ixl_free_vsi(struct ixl_vsi *vsi) 2817 { 2818 struct ixl_pf *pf = (struct ixl_pf *)vsi->back; 2819 struct ixl_queue *que = vsi->queues; 2820 2821 /* Free station queues */ 2822 for (int i = 0; i < vsi->num_queues; i++, que++) { 2823 struct tx_ring *txr = &que->txr; 2824 struct rx_ring *rxr = &que->rxr; 2825 2826 if (!mtx_initialized(&txr->mtx)) /* uninitialized */ 2827 continue; 2828 IXL_TX_LOCK(txr); 2829 ixl_free_que_tx(que); 2830 if (txr->base) 2831 i40e_free_dma_mem(&pf->hw, &txr->dma); 2832 IXL_TX_UNLOCK(txr); 2833 IXL_TX_LOCK_DESTROY(txr); 2834 2835 if (!mtx_initialized(&rxr->mtx)) /* uninitialized */ 2836 continue; 2837 IXL_RX_LOCK(rxr); 2838 ixl_free_que_rx(que); 2839 if (rxr->base) 2840 i40e_free_dma_mem(&pf->hw, &rxr->dma); 2841 IXL_RX_UNLOCK(rxr); 2842 IXL_RX_LOCK_DESTROY(rxr); 2843 2844 } 2845 free(vsi->queues, M_DEVBUF); 2846 2847 /* Free VSI filter list */ 2848 ixl_free_mac_filters(vsi); 2849 } 2850 2851 static void 2852 ixl_free_mac_filters(struct ixl_vsi *vsi) 2853 { 2854 struct ixl_mac_filter *f; 2855 2856 while (!SLIST_EMPTY(&vsi->ftl)) { 2857 f = SLIST_FIRST(&vsi->ftl); 2858 SLIST_REMOVE_HEAD(&vsi->ftl, next); 2859 free(f, M_DEVBUF); 2860 } 2861 } 2862 2863 2864 /********************************************************************* 2865 * 2866 * Allocate memory for the VSI (virtual station interface) and their 2867 * associated queues, rings and the descriptors associated with each, 2868 * called only once at attach. 2869 * 2870 **********************************************************************/ 2871 static int 2872 ixl_setup_stations(struct ixl_pf *pf) 2873 { 2874 device_t dev = pf->dev; 2875 struct ixl_vsi *vsi; 2876 struct ixl_queue *que; 2877 struct tx_ring *txr; 2878 struct rx_ring *rxr; 2879 int rsize, tsize; 2880 int error = I40E_SUCCESS; 2881 2882 vsi = &pf->vsi; 2883 vsi->back = (void *)pf; 2884 vsi->hw = &pf->hw; 2885 vsi->id = 0; 2886 vsi->num_vlans = 0; 2887 vsi->back = pf; 2888 2889 /* Get memory for the station queues */ 2890 if (!(vsi->queues = 2891 (struct ixl_queue *) malloc(sizeof(struct ixl_queue) * 2892 vsi->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) { 2893 device_printf(dev, "Unable to allocate queue memory\n"); 2894 error = ENOMEM; 2895 goto early; 2896 } 2897 2898 for (int i = 0; i < vsi->num_queues; i++) { 2899 que = &vsi->queues[i]; 2900 que->num_desc = ixl_ringsz; 2901 que->me = i; 2902 que->vsi = vsi; 2903 /* mark the queue as active */ 2904 vsi->active_queues |= (u64)1 << que->me; 2905 txr = &que->txr; 2906 txr->que = que; 2907 txr->tail = I40E_QTX_TAIL(que->me); 2908 2909 /* Initialize the TX lock */ 2910 snprintf(txr->mtx_name, sizeof(txr->mtx_name), "%s:tx(%d)", 2911 device_get_nameunit(dev), que->me); 2912 mtx_init(&txr->mtx, txr->mtx_name, NULL, MTX_DEF); 2913 /* Create the TX descriptor ring */ 2914 tsize = roundup2((que->num_desc * 2915 sizeof(struct i40e_tx_desc)) + 2916 sizeof(u32), DBA_ALIGN); 2917 if (i40e_allocate_dma_mem(&pf->hw, 2918 &txr->dma, i40e_mem_reserved, tsize, DBA_ALIGN)) { 2919 device_printf(dev, 2920 "Unable to allocate TX Descriptor memory\n"); 2921 error = ENOMEM; 2922 goto fail; 2923 } 2924 txr->base = (struct i40e_tx_desc *)txr->dma.va; 2925 bzero((void *)txr->base, tsize); 2926 /* Now allocate transmit soft structs for the ring */ 2927 if (ixl_allocate_tx_data(que)) { 2928 device_printf(dev, 2929 "Critical Failure setting up TX structures\n"); 2930 error = ENOMEM; 2931 goto fail; 2932 } 2933 /* Allocate a buf ring */ 2934 txr->br = buf_ring_alloc(4096, M_DEVBUF, 2935 M_WAITOK, &txr->mtx); 2936 if (txr->br == NULL) { 2937 device_printf(dev, 2938 "Critical Failure setting up TX buf ring\n"); 2939 error = ENOMEM; 2940 goto fail; 2941 } 2942 2943 /* 2944 * Next the RX queues... 2945 */ 2946 rsize = roundup2(que->num_desc * 2947 sizeof(union i40e_rx_desc), DBA_ALIGN); 2948 rxr = &que->rxr; 2949 rxr->que = que; 2950 rxr->tail = I40E_QRX_TAIL(que->me); 2951 2952 /* Initialize the RX side lock */ 2953 snprintf(rxr->mtx_name, sizeof(rxr->mtx_name), "%s:rx(%d)", 2954 device_get_nameunit(dev), que->me); 2955 mtx_init(&rxr->mtx, rxr->mtx_name, NULL, MTX_DEF); 2956 2957 if (i40e_allocate_dma_mem(&pf->hw, 2958 &rxr->dma, i40e_mem_reserved, rsize, 4096)) { 2959 device_printf(dev, 2960 "Unable to allocate RX Descriptor memory\n"); 2961 error = ENOMEM; 2962 goto fail; 2963 } 2964 rxr->base = (union i40e_rx_desc *)rxr->dma.va; 2965 bzero((void *)rxr->base, rsize); 2966 2967 /* Allocate receive soft structs for the ring*/ 2968 if (ixl_allocate_rx_data(que)) { 2969 device_printf(dev, 2970 "Critical Failure setting up receive structs\n"); 2971 error = ENOMEM; 2972 goto fail; 2973 } 2974 } 2975 2976 return (0); 2977 2978 fail: 2979 for (int i = 0; i < vsi->num_queues; i++) { 2980 que = &vsi->queues[i]; 2981 rxr = &que->rxr; 2982 txr = &que->txr; 2983 if (rxr->base) 2984 i40e_free_dma_mem(&pf->hw, &rxr->dma); 2985 if (txr->base) 2986 i40e_free_dma_mem(&pf->hw, &txr->dma); 2987 } 2988 2989 early: 2990 return (error); 2991 } 2992 2993 /* 2994 ** Provide a update to the queue RX 2995 ** interrupt moderation value. 2996 */ 2997 static void 2998 ixl_set_queue_rx_itr(struct ixl_queue *que) 2999 { 3000 struct ixl_vsi *vsi = que->vsi; 3001 struct i40e_hw *hw = vsi->hw; 3002 struct rx_ring *rxr = &que->rxr; 3003 u16 rx_itr; 3004 u16 rx_latency = 0; 3005 int rx_bytes; 3006 3007 3008 /* Idle, do nothing */ 3009 if (rxr->bytes == 0) 3010 return; 3011 3012 if (ixl_dynamic_rx_itr) { 3013 rx_bytes = rxr->bytes/rxr->itr; 3014 rx_itr = rxr->itr; 3015 3016 /* Adjust latency range */ 3017 switch (rxr->latency) { 3018 case IXL_LOW_LATENCY: 3019 if (rx_bytes > 10) { 3020 rx_latency = IXL_AVE_LATENCY; 3021 rx_itr = IXL_ITR_20K; 3022 } 3023 break; 3024 case IXL_AVE_LATENCY: 3025 if (rx_bytes > 20) { 3026 rx_latency = IXL_BULK_LATENCY; 3027 rx_itr = IXL_ITR_8K; 3028 } else if (rx_bytes <= 10) { 3029 rx_latency = IXL_LOW_LATENCY; 3030 rx_itr = IXL_ITR_100K; 3031 } 3032 break; 3033 case IXL_BULK_LATENCY: 3034 if (rx_bytes <= 20) { 3035 rx_latency = IXL_AVE_LATENCY; 3036 rx_itr = IXL_ITR_20K; 3037 } 3038 break; 3039 } 3040 3041 rxr->latency = rx_latency; 3042 3043 if (rx_itr != rxr->itr) { 3044 /* do an exponential smoothing */ 3045 rx_itr = (10 * rx_itr * rxr->itr) / 3046 ((9 * rx_itr) + rxr->itr); 3047 rxr->itr = rx_itr & IXL_MAX_ITR; 3048 wr32(hw, I40E_PFINT_ITRN(IXL_RX_ITR, 3049 que->me), rxr->itr); 3050 } 3051 } else { /* We may have have toggled to non-dynamic */ 3052 if (vsi->rx_itr_setting & IXL_ITR_DYNAMIC) 3053 vsi->rx_itr_setting = ixl_rx_itr; 3054 /* Update the hardware if needed */ 3055 if (rxr->itr != vsi->rx_itr_setting) { 3056 rxr->itr = vsi->rx_itr_setting; 3057 wr32(hw, I40E_PFINT_ITRN(IXL_RX_ITR, 3058 que->me), rxr->itr); 3059 } 3060 } 3061 rxr->bytes = 0; 3062 rxr->packets = 0; 3063 return; 3064 } 3065 3066 3067 /* 3068 ** Provide a update to the queue TX 3069 ** interrupt moderation value. 3070 */ 3071 static void 3072 ixl_set_queue_tx_itr(struct ixl_queue *que) 3073 { 3074 struct ixl_vsi *vsi = que->vsi; 3075 struct i40e_hw *hw = vsi->hw; 3076 struct tx_ring *txr = &que->txr; 3077 u16 tx_itr; 3078 u16 tx_latency = 0; 3079 int tx_bytes; 3080 3081 3082 /* Idle, do nothing */ 3083 if (txr->bytes == 0) 3084 return; 3085 3086 if (ixl_dynamic_tx_itr) { 3087 tx_bytes = txr->bytes/txr->itr; 3088 tx_itr = txr->itr; 3089 3090 switch (txr->latency) { 3091 case IXL_LOW_LATENCY: 3092 if (tx_bytes > 10) { 3093 tx_latency = IXL_AVE_LATENCY; 3094 tx_itr = IXL_ITR_20K; 3095 } 3096 break; 3097 case IXL_AVE_LATENCY: 3098 if (tx_bytes > 20) { 3099 tx_latency = IXL_BULK_LATENCY; 3100 tx_itr = IXL_ITR_8K; 3101 } else if (tx_bytes <= 10) { 3102 tx_latency = IXL_LOW_LATENCY; 3103 tx_itr = IXL_ITR_100K; 3104 } 3105 break; 3106 case IXL_BULK_LATENCY: 3107 if (tx_bytes <= 20) { 3108 tx_latency = IXL_AVE_LATENCY; 3109 tx_itr = IXL_ITR_20K; 3110 } 3111 break; 3112 } 3113 3114 txr->latency = tx_latency; 3115 3116 if (tx_itr != txr->itr) { 3117 /* do an exponential smoothing */ 3118 tx_itr = (10 * tx_itr * txr->itr) / 3119 ((9 * tx_itr) + txr->itr); 3120 txr->itr = tx_itr & IXL_MAX_ITR; 3121 wr32(hw, I40E_PFINT_ITRN(IXL_TX_ITR, 3122 que->me), txr->itr); 3123 } 3124 3125 } else { /* We may have have toggled to non-dynamic */ 3126 if (vsi->tx_itr_setting & IXL_ITR_DYNAMIC) 3127 vsi->tx_itr_setting = ixl_tx_itr; 3128 /* Update the hardware if needed */ 3129 if (txr->itr != vsi->tx_itr_setting) { 3130 txr->itr = vsi->tx_itr_setting; 3131 wr32(hw, I40E_PFINT_ITRN(IXL_TX_ITR, 3132 que->me), txr->itr); 3133 } 3134 } 3135 txr->bytes = 0; 3136 txr->packets = 0; 3137 return; 3138 } 3139 3140 #define QUEUE_NAME_LEN 32 3141 3142 static void 3143 ixl_add_vsi_sysctls(struct ixl_pf *pf, struct ixl_vsi *vsi, 3144 struct sysctl_ctx_list *ctx, const char *sysctl_name) 3145 { 3146 struct sysctl_oid *tree; 3147 struct sysctl_oid_list *child; 3148 struct sysctl_oid_list *vsi_list; 3149 3150 tree = device_get_sysctl_tree(pf->dev); 3151 child = SYSCTL_CHILDREN(tree); 3152 vsi->vsi_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, sysctl_name, 3153 CTLFLAG_RD, NULL, "VSI Number"); 3154 vsi_list = SYSCTL_CHILDREN(vsi->vsi_node); 3155 3156 ixl_add_sysctls_eth_stats(ctx, vsi_list, &vsi->eth_stats); 3157 } 3158 3159 static void 3160 ixl_add_hw_stats(struct ixl_pf *pf) 3161 { 3162 device_t dev = pf->dev; 3163 struct ixl_vsi *vsi = &pf->vsi; 3164 struct ixl_queue *queues = vsi->queues; 3165 struct i40e_hw_port_stats *pf_stats = &pf->stats; 3166 3167 struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(dev); 3168 struct sysctl_oid *tree = device_get_sysctl_tree(dev); 3169 struct sysctl_oid_list *child = SYSCTL_CHILDREN(tree); 3170 struct sysctl_oid_list *vsi_list; 3171 3172 struct sysctl_oid *queue_node; 3173 struct sysctl_oid_list *queue_list; 3174 3175 struct tx_ring *txr; 3176 struct rx_ring *rxr; 3177 char queue_namebuf[QUEUE_NAME_LEN]; 3178 3179 /* Driver statistics */ 3180 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "watchdog_events", 3181 CTLFLAG_RD, &pf->watchdog_events, 3182 "Watchdog timeouts"); 3183 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "admin_irq", 3184 CTLFLAG_RD, &pf->admin_irq, 3185 "Admin Queue IRQ Handled"); 3186 3187 SYSCTL_ADD_INT(ctx, child, OID_AUTO, "vc_debug_level", 3188 CTLFLAG_RW, &pf->vc_debug_lvl, 0, 3189 "PF/VF Virtual Channel debug logging level"); 3190 3191 ixl_add_vsi_sysctls(pf, &pf->vsi, ctx, "pf"); 3192 vsi_list = SYSCTL_CHILDREN(pf->vsi.vsi_node); 3193 3194 /* Queue statistics */ 3195 for (int q = 0; q < vsi->num_queues; q++) { 3196 snprintf(queue_namebuf, QUEUE_NAME_LEN, "que%d", q); 3197 queue_node = SYSCTL_ADD_NODE(ctx, vsi_list, 3198 OID_AUTO, queue_namebuf, CTLFLAG_RD, NULL, "Queue #"); 3199 queue_list = SYSCTL_CHILDREN(queue_node); 3200 3201 txr = &(queues[q].txr); 3202 rxr = &(queues[q].rxr); 3203 3204 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "mbuf_defrag_failed", 3205 CTLFLAG_RD, &(queues[q].mbuf_defrag_failed), 3206 "m_defrag() failed"); 3207 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "dropped", 3208 CTLFLAG_RD, &(queues[q].dropped_pkts), 3209 "Driver dropped packets"); 3210 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "irqs", 3211 CTLFLAG_RD, &(queues[q].irqs), 3212 "irqs on this queue"); 3213 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "tso_tx", 3214 CTLFLAG_RD, &(queues[q].tso), 3215 "TSO"); 3216 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "tx_dma_setup", 3217 CTLFLAG_RD, &(queues[q].tx_dma_setup), 3218 "Driver tx dma failure in xmit"); 3219 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "no_desc_avail", 3220 CTLFLAG_RD, &(txr->no_desc), 3221 "Queue No Descriptor Available"); 3222 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "tx_packets", 3223 CTLFLAG_RD, &(txr->total_packets), 3224 "Queue Packets Transmitted"); 3225 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "tx_bytes", 3226 CTLFLAG_RD, &(txr->tx_bytes), 3227 "Queue Bytes Transmitted"); 3228 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "rx_packets", 3229 CTLFLAG_RD, &(rxr->rx_packets), 3230 "Queue Packets Received"); 3231 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "rx_bytes", 3232 CTLFLAG_RD, &(rxr->rx_bytes), 3233 "Queue Bytes Received"); 3234 } 3235 3236 /* MAC stats */ 3237 ixl_add_sysctls_mac_stats(ctx, child, pf_stats); 3238 } 3239 3240 static void 3241 ixl_add_sysctls_eth_stats(struct sysctl_ctx_list *ctx, 3242 struct sysctl_oid_list *child, 3243 struct i40e_eth_stats *eth_stats) 3244 { 3245 struct ixl_sysctl_info ctls[] = 3246 { 3247 {ð_stats->rx_bytes, "good_octets_rcvd", "Good Octets Received"}, 3248 {ð_stats->rx_unicast, "ucast_pkts_rcvd", 3249 "Unicast Packets Received"}, 3250 {ð_stats->rx_multicast, "mcast_pkts_rcvd", 3251 "Multicast Packets Received"}, 3252 {ð_stats->rx_broadcast, "bcast_pkts_rcvd", 3253 "Broadcast Packets Received"}, 3254 {ð_stats->rx_discards, "rx_discards", "Discarded RX packets"}, 3255 {ð_stats->tx_bytes, "good_octets_txd", "Good Octets Transmitted"}, 3256 {ð_stats->tx_unicast, "ucast_pkts_txd", "Unicast Packets Transmitted"}, 3257 {ð_stats->tx_multicast, "mcast_pkts_txd", 3258 "Multicast Packets Transmitted"}, 3259 {ð_stats->tx_broadcast, "bcast_pkts_txd", 3260 "Broadcast Packets Transmitted"}, 3261 // end 3262 {0,0,0} 3263 }; 3264 3265 struct ixl_sysctl_info *entry = ctls; 3266 while (entry->stat != 0) 3267 { 3268 SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, entry->name, 3269 CTLFLAG_RD, entry->stat, 3270 entry->description); 3271 entry++; 3272 } 3273 } 3274 3275 static void 3276 ixl_add_sysctls_mac_stats(struct sysctl_ctx_list *ctx, 3277 struct sysctl_oid_list *child, 3278 struct i40e_hw_port_stats *stats) 3279 { 3280 struct sysctl_oid *stat_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, "mac", 3281 CTLFLAG_RD, NULL, "Mac Statistics"); 3282 struct sysctl_oid_list *stat_list = SYSCTL_CHILDREN(stat_node); 3283 3284 struct i40e_eth_stats *eth_stats = &stats->eth; 3285 ixl_add_sysctls_eth_stats(ctx, stat_list, eth_stats); 3286 3287 struct ixl_sysctl_info ctls[] = 3288 { 3289 {&stats->crc_errors, "crc_errors", "CRC Errors"}, 3290 {&stats->illegal_bytes, "illegal_bytes", "Illegal Byte Errors"}, 3291 {&stats->mac_local_faults, "local_faults", "MAC Local Faults"}, 3292 {&stats->mac_remote_faults, "remote_faults", "MAC Remote Faults"}, 3293 {&stats->rx_length_errors, "rx_length_errors", "Receive Length Errors"}, 3294 /* Packet Reception Stats */ 3295 {&stats->rx_size_64, "rx_frames_64", "64 byte frames received"}, 3296 {&stats->rx_size_127, "rx_frames_65_127", "65-127 byte frames received"}, 3297 {&stats->rx_size_255, "rx_frames_128_255", "128-255 byte frames received"}, 3298 {&stats->rx_size_511, "rx_frames_256_511", "256-511 byte frames received"}, 3299 {&stats->rx_size_1023, "rx_frames_512_1023", "512-1023 byte frames received"}, 3300 {&stats->rx_size_1522, "rx_frames_1024_1522", "1024-1522 byte frames received"}, 3301 {&stats->rx_size_big, "rx_frames_big", "1523-9522 byte frames received"}, 3302 {&stats->rx_undersize, "rx_undersize", "Undersized packets received"}, 3303 {&stats->rx_fragments, "rx_fragmented", "Fragmented packets received"}, 3304 {&stats->rx_oversize, "rx_oversized", "Oversized packets received"}, 3305 {&stats->rx_jabber, "rx_jabber", "Received Jabber"}, 3306 {&stats->checksum_error, "checksum_errors", "Checksum Errors"}, 3307 /* Packet Transmission Stats */ 3308 {&stats->tx_size_64, "tx_frames_64", "64 byte frames transmitted"}, 3309 {&stats->tx_size_127, "tx_frames_65_127", "65-127 byte frames transmitted"}, 3310 {&stats->tx_size_255, "tx_frames_128_255", "128-255 byte frames transmitted"}, 3311 {&stats->tx_size_511, "tx_frames_256_511", "256-511 byte frames transmitted"}, 3312 {&stats->tx_size_1023, "tx_frames_512_1023", "512-1023 byte frames transmitted"}, 3313 {&stats->tx_size_1522, "tx_frames_1024_1522", "1024-1522 byte frames transmitted"}, 3314 {&stats->tx_size_big, "tx_frames_big", "1523-9522 byte frames transmitted"}, 3315 /* Flow control */ 3316 {&stats->link_xon_tx, "xon_txd", "Link XON transmitted"}, 3317 {&stats->link_xon_rx, "xon_recvd", "Link XON received"}, 3318 {&stats->link_xoff_tx, "xoff_txd", "Link XOFF transmitted"}, 3319 {&stats->link_xoff_rx, "xoff_recvd", "Link XOFF received"}, 3320 /* End */ 3321 {0,0,0} 3322 }; 3323 3324 struct ixl_sysctl_info *entry = ctls; 3325 while (entry->stat != 0) 3326 { 3327 SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, entry->name, 3328 CTLFLAG_RD, entry->stat, 3329 entry->description); 3330 entry++; 3331 } 3332 } 3333 3334 /* 3335 ** ixl_config_rss - setup RSS 3336 ** - note this is done for the single vsi 3337 */ 3338 static void ixl_config_rss(struct ixl_vsi *vsi) 3339 { 3340 struct ixl_pf *pf = (struct ixl_pf *)vsi->back; 3341 struct i40e_hw *hw = vsi->hw; 3342 u32 lut = 0; 3343 u64 set_hena = 0, hena; 3344 int i, j, que_id; 3345 #ifdef RSS 3346 u32 rss_hash_config; 3347 u32 rss_seed[IXL_KEYSZ]; 3348 #else 3349 u32 rss_seed[IXL_KEYSZ] = {0x41b01687, 3350 0x183cfd8c, 0xce880440, 0x580cbc3c, 3351 0x35897377, 0x328b25e1, 0x4fa98922, 3352 0xb7d90c14, 0xd5bad70d, 0xcd15a2c1}; 3353 #endif 3354 3355 #ifdef RSS 3356 /* Fetch the configured RSS key */ 3357 rss_getkey((uint8_t *) &rss_seed); 3358 #endif 3359 3360 /* Fill out hash function seed */ 3361 for (i = 0; i < IXL_KEYSZ; i++) 3362 wr32(hw, I40E_PFQF_HKEY(i), rss_seed[i]); 3363 3364 /* Enable PCTYPES for RSS: */ 3365 #ifdef RSS 3366 rss_hash_config = rss_gethashconfig(); 3367 if (rss_hash_config & RSS_HASHTYPE_RSS_IPV4) 3368 set_hena |= ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV4_OTHER); 3369 if (rss_hash_config & RSS_HASHTYPE_RSS_TCP_IPV4) 3370 set_hena |= ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV4_TCP); 3371 if (rss_hash_config & RSS_HASHTYPE_RSS_UDP_IPV4) 3372 set_hena |= ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV4_UDP); 3373 if (rss_hash_config & RSS_HASHTYPE_RSS_IPV6) 3374 set_hena |= ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV6_OTHER); 3375 if (rss_hash_config & RSS_HASHTYPE_RSS_IPV6_EX) 3376 set_hena |= ((u64)1 << I40E_FILTER_PCTYPE_FRAG_IPV6); 3377 if (rss_hash_config & RSS_HASHTYPE_RSS_TCP_IPV6) 3378 set_hena |= ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV6_TCP); 3379 if (rss_hash_config & RSS_HASHTYPE_RSS_UDP_IPV6) 3380 set_hena |= ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV6_UDP); 3381 #else 3382 set_hena = 3383 ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV4_UDP) | 3384 ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV4_TCP) | 3385 ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV4_SCTP) | 3386 ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV4_OTHER) | 3387 ((u64)1 << I40E_FILTER_PCTYPE_FRAG_IPV4) | 3388 ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV6_UDP) | 3389 ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV6_TCP) | 3390 ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV6_SCTP) | 3391 ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV6_OTHER) | 3392 ((u64)1 << I40E_FILTER_PCTYPE_FRAG_IPV6) | 3393 ((u64)1 << I40E_FILTER_PCTYPE_L2_PAYLOAD); 3394 #endif 3395 hena = (u64)rd32(hw, I40E_PFQF_HENA(0)) | 3396 ((u64)rd32(hw, I40E_PFQF_HENA(1)) << 32); 3397 hena |= set_hena; 3398 wr32(hw, I40E_PFQF_HENA(0), (u32)hena); 3399 wr32(hw, I40E_PFQF_HENA(1), (u32)(hena >> 32)); 3400 3401 /* Populate the LUT with max no. of queues in round robin fashion */ 3402 for (i = j = 0; i < pf->hw.func_caps.rss_table_size; i++, j++) { 3403 if (j == vsi->num_queues) 3404 j = 0; 3405 #ifdef RSS 3406 /* 3407 * Fetch the RSS bucket id for the given indirection entry. 3408 * Cap it at the number of configured buckets (which is 3409 * num_queues.) 3410 */ 3411 que_id = rss_get_indirection_to_bucket(i); 3412 que_id = que_id % vsi->num_queues; 3413 #else 3414 que_id = j; 3415 #endif 3416 /* lut = 4-byte sliding window of 4 lut entries */ 3417 lut = (lut << 8) | (que_id & 3418 ((0x1 << pf->hw.func_caps.rss_table_entry_width) - 1)); 3419 /* On i = 3, we have 4 entries in lut; write to the register */ 3420 if ((i & 3) == 3) 3421 wr32(hw, I40E_PFQF_HLUT(i >> 2), lut); 3422 } 3423 ixl_flush(hw); 3424 } 3425 3426 3427 /* 3428 ** This routine is run via an vlan config EVENT, 3429 ** it enables us to use the HW Filter table since 3430 ** we can get the vlan id. This just creates the 3431 ** entry in the soft version of the VFTA, init will 3432 ** repopulate the real table. 3433 */ 3434 static void 3435 ixl_register_vlan(void *arg, struct ifnet *ifp, u16 vtag) 3436 { 3437 struct ixl_vsi *vsi = ifp->if_softc; 3438 struct i40e_hw *hw = vsi->hw; 3439 struct ixl_pf *pf = (struct ixl_pf *)vsi->back; 3440 3441 if (ifp->if_softc != arg) /* Not our event */ 3442 return; 3443 3444 if ((vtag == 0) || (vtag > 4095)) /* Invalid */ 3445 return; 3446 3447 IXL_PF_LOCK(pf); 3448 ++vsi->num_vlans; 3449 ixl_add_filter(vsi, hw->mac.addr, vtag); 3450 IXL_PF_UNLOCK(pf); 3451 } 3452 3453 /* 3454 ** This routine is run via an vlan 3455 ** unconfig EVENT, remove our entry 3456 ** in the soft vfta. 3457 */ 3458 static void 3459 ixl_unregister_vlan(void *arg, struct ifnet *ifp, u16 vtag) 3460 { 3461 struct ixl_vsi *vsi = ifp->if_softc; 3462 struct i40e_hw *hw = vsi->hw; 3463 struct ixl_pf *pf = (struct ixl_pf *)vsi->back; 3464 3465 if (ifp->if_softc != arg) 3466 return; 3467 3468 if ((vtag == 0) || (vtag > 4095)) /* Invalid */ 3469 return; 3470 3471 IXL_PF_LOCK(pf); 3472 --vsi->num_vlans; 3473 ixl_del_filter(vsi, hw->mac.addr, vtag); 3474 IXL_PF_UNLOCK(pf); 3475 } 3476 3477 /* 3478 ** This routine updates vlan filters, called by init 3479 ** it scans the filter table and then updates the hw 3480 ** after a soft reset. 3481 */ 3482 static void 3483 ixl_setup_vlan_filters(struct ixl_vsi *vsi) 3484 { 3485 struct ixl_mac_filter *f; 3486 int cnt = 0, flags; 3487 3488 if (vsi->num_vlans == 0) 3489 return; 3490 /* 3491 ** Scan the filter list for vlan entries, 3492 ** mark them for addition and then call 3493 ** for the AQ update. 3494 */ 3495 SLIST_FOREACH(f, &vsi->ftl, next) { 3496 if (f->flags & IXL_FILTER_VLAN) { 3497 f->flags |= 3498 (IXL_FILTER_ADD | 3499 IXL_FILTER_USED); 3500 cnt++; 3501 } 3502 } 3503 if (cnt == 0) { 3504 printf("setup vlan: no filters found!\n"); 3505 return; 3506 } 3507 flags = IXL_FILTER_VLAN; 3508 flags |= (IXL_FILTER_ADD | IXL_FILTER_USED); 3509 ixl_add_hw_filters(vsi, flags, cnt); 3510 return; 3511 } 3512 3513 /* 3514 ** Initialize filter list and add filters that the hardware 3515 ** needs to know about. 3516 */ 3517 static void 3518 ixl_init_filters(struct ixl_vsi *vsi) 3519 { 3520 /* Add broadcast address */ 3521 ixl_add_filter(vsi, ixl_bcast_addr, IXL_VLAN_ANY); 3522 } 3523 3524 /* 3525 ** This routine adds mulicast filters 3526 */ 3527 static void 3528 ixl_add_mc_filter(struct ixl_vsi *vsi, u8 *macaddr) 3529 { 3530 struct ixl_mac_filter *f; 3531 3532 /* Does one already exist */ 3533 f = ixl_find_filter(vsi, macaddr, IXL_VLAN_ANY); 3534 if (f != NULL) 3535 return; 3536 3537 f = ixl_get_filter(vsi); 3538 if (f == NULL) { 3539 printf("WARNING: no filter available!!\n"); 3540 return; 3541 } 3542 bcopy(macaddr, f->macaddr, ETHER_ADDR_LEN); 3543 f->vlan = IXL_VLAN_ANY; 3544 f->flags |= (IXL_FILTER_ADD | IXL_FILTER_USED 3545 | IXL_FILTER_MC); 3546 3547 return; 3548 } 3549 3550 static void 3551 ixl_reconfigure_filters(struct ixl_vsi *vsi) 3552 { 3553 3554 ixl_add_hw_filters(vsi, IXL_FILTER_USED, vsi->num_macs); 3555 } 3556 3557 /* 3558 ** This routine adds macvlan filters 3559 */ 3560 static void 3561 ixl_add_filter(struct ixl_vsi *vsi, u8 *macaddr, s16 vlan) 3562 { 3563 struct ixl_mac_filter *f, *tmp; 3564 struct ixl_pf *pf; 3565 device_t dev; 3566 3567 DEBUGOUT("ixl_add_filter: begin"); 3568 3569 pf = vsi->back; 3570 dev = pf->dev; 3571 3572 /* Does one already exist */ 3573 f = ixl_find_filter(vsi, macaddr, vlan); 3574 if (f != NULL) 3575 return; 3576 /* 3577 ** Is this the first vlan being registered, if so we 3578 ** need to remove the ANY filter that indicates we are 3579 ** not in a vlan, and replace that with a 0 filter. 3580 */ 3581 if ((vlan != IXL_VLAN_ANY) && (vsi->num_vlans == 1)) { 3582 tmp = ixl_find_filter(vsi, macaddr, IXL_VLAN_ANY); 3583 if (tmp != NULL) { 3584 ixl_del_filter(vsi, macaddr, IXL_VLAN_ANY); 3585 ixl_add_filter(vsi, macaddr, 0); 3586 } 3587 } 3588 3589 f = ixl_get_filter(vsi); 3590 if (f == NULL) { 3591 device_printf(dev, "WARNING: no filter available!!\n"); 3592 return; 3593 } 3594 bcopy(macaddr, f->macaddr, ETHER_ADDR_LEN); 3595 f->vlan = vlan; 3596 f->flags |= (IXL_FILTER_ADD | IXL_FILTER_USED); 3597 if (f->vlan != IXL_VLAN_ANY) 3598 f->flags |= IXL_FILTER_VLAN; 3599 else 3600 vsi->num_macs++; 3601 3602 ixl_add_hw_filters(vsi, f->flags, 1); 3603 return; 3604 } 3605 3606 static void 3607 ixl_del_filter(struct ixl_vsi *vsi, u8 *macaddr, s16 vlan) 3608 { 3609 struct ixl_mac_filter *f; 3610 3611 f = ixl_find_filter(vsi, macaddr, vlan); 3612 if (f == NULL) 3613 return; 3614 3615 f->flags |= IXL_FILTER_DEL; 3616 ixl_del_hw_filters(vsi, 1); 3617 vsi->num_macs--; 3618 3619 /* Check if this is the last vlan removal */ 3620 if (vlan != IXL_VLAN_ANY && vsi->num_vlans == 0) { 3621 /* Switch back to a non-vlan filter */ 3622 ixl_del_filter(vsi, macaddr, 0); 3623 ixl_add_filter(vsi, macaddr, IXL_VLAN_ANY); 3624 } 3625 return; 3626 } 3627 3628 /* 3629 ** Find the filter with both matching mac addr and vlan id 3630 */ 3631 static struct ixl_mac_filter * 3632 ixl_find_filter(struct ixl_vsi *vsi, u8 *macaddr, s16 vlan) 3633 { 3634 struct ixl_mac_filter *f; 3635 bool match = FALSE; 3636 3637 SLIST_FOREACH(f, &vsi->ftl, next) { 3638 if (!cmp_etheraddr(f->macaddr, macaddr)) 3639 continue; 3640 if (f->vlan == vlan) { 3641 match = TRUE; 3642 break; 3643 } 3644 } 3645 3646 if (!match) 3647 f = NULL; 3648 return (f); 3649 } 3650 3651 /* 3652 ** This routine takes additions to the vsi filter 3653 ** table and creates an Admin Queue call to create 3654 ** the filters in the hardware. 3655 */ 3656 static void 3657 ixl_add_hw_filters(struct ixl_vsi *vsi, int flags, int cnt) 3658 { 3659 struct i40e_aqc_add_macvlan_element_data *a, *b; 3660 struct ixl_mac_filter *f; 3661 struct ixl_pf *pf; 3662 struct i40e_hw *hw; 3663 device_t dev; 3664 int err, j = 0; 3665 3666 pf = vsi->back; 3667 dev = pf->dev; 3668 hw = &pf->hw; 3669 IXL_PF_LOCK_ASSERT(pf); 3670 3671 a = malloc(sizeof(struct i40e_aqc_add_macvlan_element_data) * cnt, 3672 M_DEVBUF, M_NOWAIT | M_ZERO); 3673 if (a == NULL) { 3674 device_printf(dev, "add_hw_filters failed to get memory\n"); 3675 return; 3676 } 3677 3678 /* 3679 ** Scan the filter list, each time we find one 3680 ** we add it to the admin queue array and turn off 3681 ** the add bit. 3682 */ 3683 SLIST_FOREACH(f, &vsi->ftl, next) { 3684 if (f->flags == flags) { 3685 b = &a[j]; // a pox on fvl long names :) 3686 bcopy(f->macaddr, b->mac_addr, ETHER_ADDR_LEN); 3687 if (f->vlan == IXL_VLAN_ANY) { 3688 b->vlan_tag = 0; 3689 b->flags = I40E_AQC_MACVLAN_ADD_IGNORE_VLAN; 3690 } else { 3691 b->vlan_tag = f->vlan; 3692 b->flags = 0; 3693 } 3694 b->flags |= I40E_AQC_MACVLAN_ADD_PERFECT_MATCH; 3695 f->flags &= ~IXL_FILTER_ADD; 3696 j++; 3697 } 3698 if (j == cnt) 3699 break; 3700 } 3701 if (j > 0) { 3702 err = i40e_aq_add_macvlan(hw, vsi->seid, a, j, NULL); 3703 if (err) 3704 device_printf(dev, "aq_add_macvlan err %d, " 3705 "aq_error %d\n", err, hw->aq.asq_last_status); 3706 else 3707 vsi->hw_filters_add += j; 3708 } 3709 free(a, M_DEVBUF); 3710 return; 3711 } 3712 3713 /* 3714 ** This routine takes removals in the vsi filter 3715 ** table and creates an Admin Queue call to delete 3716 ** the filters in the hardware. 3717 */ 3718 static void 3719 ixl_del_hw_filters(struct ixl_vsi *vsi, int cnt) 3720 { 3721 struct i40e_aqc_remove_macvlan_element_data *d, *e; 3722 struct ixl_pf *pf; 3723 struct i40e_hw *hw; 3724 device_t dev; 3725 struct ixl_mac_filter *f, *f_temp; 3726 int err, j = 0; 3727 3728 DEBUGOUT("ixl_del_hw_filters: begin\n"); 3729 3730 pf = vsi->back; 3731 hw = &pf->hw; 3732 dev = pf->dev; 3733 3734 d = malloc(sizeof(struct i40e_aqc_remove_macvlan_element_data) * cnt, 3735 M_DEVBUF, M_NOWAIT | M_ZERO); 3736 if (d == NULL) { 3737 printf("del hw filter failed to get memory\n"); 3738 return; 3739 } 3740 3741 SLIST_FOREACH_SAFE(f, &vsi->ftl, next, f_temp) { 3742 if (f->flags & IXL_FILTER_DEL) { 3743 e = &d[j]; // a pox on fvl long names :) 3744 bcopy(f->macaddr, e->mac_addr, ETHER_ADDR_LEN); 3745 e->vlan_tag = (f->vlan == IXL_VLAN_ANY ? 0 : f->vlan); 3746 e->flags = I40E_AQC_MACVLAN_DEL_PERFECT_MATCH; 3747 /* delete entry from vsi list */ 3748 SLIST_REMOVE(&vsi->ftl, f, ixl_mac_filter, next); 3749 free(f, M_DEVBUF); 3750 j++; 3751 } 3752 if (j == cnt) 3753 break; 3754 } 3755 if (j > 0) { 3756 err = i40e_aq_remove_macvlan(hw, vsi->seid, d, j, NULL); 3757 /* NOTE: returns ENOENT every time but seems to work fine, 3758 so we'll ignore that specific error. */ 3759 // TODO: Does this still occur on current firmwares? 3760 if (err && hw->aq.asq_last_status != I40E_AQ_RC_ENOENT) { 3761 int sc = 0; 3762 for (int i = 0; i < j; i++) 3763 sc += (!d[i].error_code); 3764 vsi->hw_filters_del += sc; 3765 device_printf(dev, 3766 "Failed to remove %d/%d filters, aq error %d\n", 3767 j - sc, j, hw->aq.asq_last_status); 3768 } else 3769 vsi->hw_filters_del += j; 3770 } 3771 free(d, M_DEVBUF); 3772 3773 DEBUGOUT("ixl_del_hw_filters: end\n"); 3774 return; 3775 } 3776 3777 static int 3778 ixl_enable_rings(struct ixl_vsi *vsi) 3779 { 3780 struct ixl_pf *pf = vsi->back; 3781 struct i40e_hw *hw = &pf->hw; 3782 int index, error; 3783 u32 reg; 3784 3785 error = 0; 3786 for (int i = 0; i < vsi->num_queues; i++) { 3787 index = vsi->first_queue + i; 3788 i40e_pre_tx_queue_cfg(hw, index, TRUE); 3789 3790 reg = rd32(hw, I40E_QTX_ENA(index)); 3791 reg |= I40E_QTX_ENA_QENA_REQ_MASK | 3792 I40E_QTX_ENA_QENA_STAT_MASK; 3793 wr32(hw, I40E_QTX_ENA(index), reg); 3794 /* Verify the enable took */ 3795 for (int j = 0; j < 10; j++) { 3796 reg = rd32(hw, I40E_QTX_ENA(index)); 3797 if (reg & I40E_QTX_ENA_QENA_STAT_MASK) 3798 break; 3799 i40e_msec_delay(10); 3800 } 3801 if ((reg & I40E_QTX_ENA_QENA_STAT_MASK) == 0) { 3802 device_printf(pf->dev, "TX queue %d disabled!\n", 3803 index); 3804 error = ETIMEDOUT; 3805 } 3806 3807 reg = rd32(hw, I40E_QRX_ENA(index)); 3808 reg |= I40E_QRX_ENA_QENA_REQ_MASK | 3809 I40E_QRX_ENA_QENA_STAT_MASK; 3810 wr32(hw, I40E_QRX_ENA(index), reg); 3811 /* Verify the enable took */ 3812 for (int j = 0; j < 10; j++) { 3813 reg = rd32(hw, I40E_QRX_ENA(index)); 3814 if (reg & I40E_QRX_ENA_QENA_STAT_MASK) 3815 break; 3816 i40e_msec_delay(10); 3817 } 3818 if ((reg & I40E_QRX_ENA_QENA_STAT_MASK) == 0) { 3819 device_printf(pf->dev, "RX queue %d disabled!\n", 3820 index); 3821 error = ETIMEDOUT; 3822 } 3823 } 3824 3825 return (error); 3826 } 3827 3828 static int 3829 ixl_disable_rings(struct ixl_vsi *vsi) 3830 { 3831 struct ixl_pf *pf = vsi->back; 3832 struct i40e_hw *hw = &pf->hw; 3833 int index, error; 3834 u32 reg; 3835 3836 error = 0; 3837 for (int i = 0; i < vsi->num_queues; i++) { 3838 index = vsi->first_queue + i; 3839 3840 i40e_pre_tx_queue_cfg(hw, index, FALSE); 3841 i40e_usec_delay(500); 3842 3843 reg = rd32(hw, I40E_QTX_ENA(index)); 3844 reg &= ~I40E_QTX_ENA_QENA_REQ_MASK; 3845 wr32(hw, I40E_QTX_ENA(index), reg); 3846 /* Verify the disable took */ 3847 for (int j = 0; j < 10; j++) { 3848 reg = rd32(hw, I40E_QTX_ENA(index)); 3849 if (!(reg & I40E_QTX_ENA_QENA_STAT_MASK)) 3850 break; 3851 i40e_msec_delay(10); 3852 } 3853 if (reg & I40E_QTX_ENA_QENA_STAT_MASK) { 3854 device_printf(pf->dev, "TX queue %d still enabled!\n", 3855 index); 3856 error = ETIMEDOUT; 3857 } 3858 3859 reg = rd32(hw, I40E_QRX_ENA(index)); 3860 reg &= ~I40E_QRX_ENA_QENA_REQ_MASK; 3861 wr32(hw, I40E_QRX_ENA(index), reg); 3862 /* Verify the disable took */ 3863 for (int j = 0; j < 10; j++) { 3864 reg = rd32(hw, I40E_QRX_ENA(index)); 3865 if (!(reg & I40E_QRX_ENA_QENA_STAT_MASK)) 3866 break; 3867 i40e_msec_delay(10); 3868 } 3869 if (reg & I40E_QRX_ENA_QENA_STAT_MASK) { 3870 device_printf(pf->dev, "RX queue %d still enabled!\n", 3871 index); 3872 error = ETIMEDOUT; 3873 } 3874 } 3875 3876 return (error); 3877 } 3878 3879 /** 3880 * ixl_handle_mdd_event 3881 * 3882 * Called from interrupt handler to identify possibly malicious vfs 3883 * (But also detects events from the PF, as well) 3884 **/ 3885 static void ixl_handle_mdd_event(struct ixl_pf *pf) 3886 { 3887 struct i40e_hw *hw = &pf->hw; 3888 device_t dev = pf->dev; 3889 bool mdd_detected = false; 3890 bool pf_mdd_detected = false; 3891 u32 reg; 3892 3893 /* find what triggered the MDD event */ 3894 reg = rd32(hw, I40E_GL_MDET_TX); 3895 if (reg & I40E_GL_MDET_TX_VALID_MASK) { 3896 u8 pf_num = (reg & I40E_GL_MDET_TX_PF_NUM_MASK) >> 3897 I40E_GL_MDET_TX_PF_NUM_SHIFT; 3898 u8 event = (reg & I40E_GL_MDET_TX_EVENT_MASK) >> 3899 I40E_GL_MDET_TX_EVENT_SHIFT; 3900 u8 queue = (reg & I40E_GL_MDET_TX_QUEUE_MASK) >> 3901 I40E_GL_MDET_TX_QUEUE_SHIFT; 3902 device_printf(dev, 3903 "Malicious Driver Detection event 0x%02x" 3904 " on TX queue %d pf number 0x%02x\n", 3905 event, queue, pf_num); 3906 wr32(hw, I40E_GL_MDET_TX, 0xffffffff); 3907 mdd_detected = true; 3908 } 3909 reg = rd32(hw, I40E_GL_MDET_RX); 3910 if (reg & I40E_GL_MDET_RX_VALID_MASK) { 3911 u8 func = (reg & I40E_GL_MDET_RX_FUNCTION_MASK) >> 3912 I40E_GL_MDET_RX_FUNCTION_SHIFT; 3913 u8 event = (reg & I40E_GL_MDET_RX_EVENT_MASK) >> 3914 I40E_GL_MDET_RX_EVENT_SHIFT; 3915 u8 queue = (reg & I40E_GL_MDET_RX_QUEUE_MASK) >> 3916 I40E_GL_MDET_RX_QUEUE_SHIFT; 3917 device_printf(dev, 3918 "Malicious Driver Detection event 0x%02x" 3919 " on RX queue %d of function 0x%02x\n", 3920 event, queue, func); 3921 wr32(hw, I40E_GL_MDET_RX, 0xffffffff); 3922 mdd_detected = true; 3923 } 3924 3925 if (mdd_detected) { 3926 reg = rd32(hw, I40E_PF_MDET_TX); 3927 if (reg & I40E_PF_MDET_TX_VALID_MASK) { 3928 wr32(hw, I40E_PF_MDET_TX, 0xFFFF); 3929 device_printf(dev, 3930 "MDD TX event is for this function 0x%08x", 3931 reg); 3932 pf_mdd_detected = true; 3933 } 3934 reg = rd32(hw, I40E_PF_MDET_RX); 3935 if (reg & I40E_PF_MDET_RX_VALID_MASK) { 3936 wr32(hw, I40E_PF_MDET_RX, 0xFFFF); 3937 device_printf(dev, 3938 "MDD RX event is for this function 0x%08x", 3939 reg); 3940 pf_mdd_detected = true; 3941 } 3942 } 3943 3944 /* re-enable mdd interrupt cause */ 3945 reg = rd32(hw, I40E_PFINT_ICR0_ENA); 3946 reg |= I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK; 3947 wr32(hw, I40E_PFINT_ICR0_ENA, reg); 3948 ixl_flush(hw); 3949 } 3950 3951 static void 3952 ixl_enable_intr(struct ixl_vsi *vsi) 3953 { 3954 struct i40e_hw *hw = vsi->hw; 3955 struct ixl_queue *que = vsi->queues; 3956 3957 if (ixl_enable_msix) { 3958 ixl_enable_adminq(hw); 3959 for (int i = 0; i < vsi->num_queues; i++, que++) 3960 ixl_enable_queue(hw, que->me); 3961 } else 3962 ixl_enable_legacy(hw); 3963 } 3964 3965 static void 3966 ixl_disable_rings_intr(struct ixl_vsi *vsi) 3967 { 3968 struct i40e_hw *hw = vsi->hw; 3969 struct ixl_queue *que = vsi->queues; 3970 3971 for (int i = 0; i < vsi->num_queues; i++, que++) 3972 ixl_disable_queue(hw, que->me); 3973 } 3974 3975 static void 3976 ixl_disable_intr(struct ixl_vsi *vsi) 3977 { 3978 struct i40e_hw *hw = vsi->hw; 3979 3980 if (ixl_enable_msix) 3981 ixl_disable_adminq(hw); 3982 else 3983 ixl_disable_legacy(hw); 3984 } 3985 3986 static void 3987 ixl_enable_adminq(struct i40e_hw *hw) 3988 { 3989 u32 reg; 3990 3991 reg = I40E_PFINT_DYN_CTL0_INTENA_MASK | 3992 I40E_PFINT_DYN_CTL0_CLEARPBA_MASK | 3993 (IXL_ITR_NONE << I40E_PFINT_DYN_CTL0_ITR_INDX_SHIFT); 3994 wr32(hw, I40E_PFINT_DYN_CTL0, reg); 3995 ixl_flush(hw); 3996 return; 3997 } 3998 3999 static void 4000 ixl_disable_adminq(struct i40e_hw *hw) 4001 { 4002 u32 reg; 4003 4004 reg = IXL_ITR_NONE << I40E_PFINT_DYN_CTL0_ITR_INDX_SHIFT; 4005 wr32(hw, I40E_PFINT_DYN_CTL0, reg); 4006 4007 return; 4008 } 4009 4010 static void 4011 ixl_enable_queue(struct i40e_hw *hw, int id) 4012 { 4013 u32 reg; 4014 4015 reg = I40E_PFINT_DYN_CTLN_INTENA_MASK | 4016 I40E_PFINT_DYN_CTLN_CLEARPBA_MASK | 4017 (IXL_ITR_NONE << I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT); 4018 wr32(hw, I40E_PFINT_DYN_CTLN(id), reg); 4019 } 4020 4021 static void 4022 ixl_disable_queue(struct i40e_hw *hw, int id) 4023 { 4024 u32 reg; 4025 4026 reg = IXL_ITR_NONE << I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT; 4027 wr32(hw, I40E_PFINT_DYN_CTLN(id), reg); 4028 4029 return; 4030 } 4031 4032 static void 4033 ixl_enable_legacy(struct i40e_hw *hw) 4034 { 4035 u32 reg; 4036 reg = I40E_PFINT_DYN_CTL0_INTENA_MASK | 4037 I40E_PFINT_DYN_CTL0_CLEARPBA_MASK | 4038 (IXL_ITR_NONE << I40E_PFINT_DYN_CTL0_ITR_INDX_SHIFT); 4039 wr32(hw, I40E_PFINT_DYN_CTL0, reg); 4040 } 4041 4042 static void 4043 ixl_disable_legacy(struct i40e_hw *hw) 4044 { 4045 u32 reg; 4046 4047 reg = IXL_ITR_NONE << I40E_PFINT_DYN_CTL0_ITR_INDX_SHIFT; 4048 wr32(hw, I40E_PFINT_DYN_CTL0, reg); 4049 4050 return; 4051 } 4052 4053 static void 4054 ixl_update_stats_counters(struct ixl_pf *pf) 4055 { 4056 struct i40e_hw *hw = &pf->hw; 4057 struct ixl_vsi *vsi = &pf->vsi; 4058 struct ixl_vf *vf; 4059 4060 struct i40e_hw_port_stats *nsd = &pf->stats; 4061 struct i40e_hw_port_stats *osd = &pf->stats_offsets; 4062 4063 /* Update hw stats */ 4064 ixl_stat_update32(hw, I40E_GLPRT_CRCERRS(hw->port), 4065 pf->stat_offsets_loaded, 4066 &osd->crc_errors, &nsd->crc_errors); 4067 ixl_stat_update32(hw, I40E_GLPRT_ILLERRC(hw->port), 4068 pf->stat_offsets_loaded, 4069 &osd->illegal_bytes, &nsd->illegal_bytes); 4070 ixl_stat_update48(hw, I40E_GLPRT_GORCH(hw->port), 4071 I40E_GLPRT_GORCL(hw->port), 4072 pf->stat_offsets_loaded, 4073 &osd->eth.rx_bytes, &nsd->eth.rx_bytes); 4074 ixl_stat_update48(hw, I40E_GLPRT_GOTCH(hw->port), 4075 I40E_GLPRT_GOTCL(hw->port), 4076 pf->stat_offsets_loaded, 4077 &osd->eth.tx_bytes, &nsd->eth.tx_bytes); 4078 ixl_stat_update32(hw, I40E_GLPRT_RDPC(hw->port), 4079 pf->stat_offsets_loaded, 4080 &osd->eth.rx_discards, 4081 &nsd->eth.rx_discards); 4082 ixl_stat_update48(hw, I40E_GLPRT_UPRCH(hw->port), 4083 I40E_GLPRT_UPRCL(hw->port), 4084 pf->stat_offsets_loaded, 4085 &osd->eth.rx_unicast, 4086 &nsd->eth.rx_unicast); 4087 ixl_stat_update48(hw, I40E_GLPRT_UPTCH(hw->port), 4088 I40E_GLPRT_UPTCL(hw->port), 4089 pf->stat_offsets_loaded, 4090 &osd->eth.tx_unicast, 4091 &nsd->eth.tx_unicast); 4092 ixl_stat_update48(hw, I40E_GLPRT_MPRCH(hw->port), 4093 I40E_GLPRT_MPRCL(hw->port), 4094 pf->stat_offsets_loaded, 4095 &osd->eth.rx_multicast, 4096 &nsd->eth.rx_multicast); 4097 ixl_stat_update48(hw, I40E_GLPRT_MPTCH(hw->port), 4098 I40E_GLPRT_MPTCL(hw->port), 4099 pf->stat_offsets_loaded, 4100 &osd->eth.tx_multicast, 4101 &nsd->eth.tx_multicast); 4102 ixl_stat_update48(hw, I40E_GLPRT_BPRCH(hw->port), 4103 I40E_GLPRT_BPRCL(hw->port), 4104 pf->stat_offsets_loaded, 4105 &osd->eth.rx_broadcast, 4106 &nsd->eth.rx_broadcast); 4107 ixl_stat_update48(hw, I40E_GLPRT_BPTCH(hw->port), 4108 I40E_GLPRT_BPTCL(hw->port), 4109 pf->stat_offsets_loaded, 4110 &osd->eth.tx_broadcast, 4111 &nsd->eth.tx_broadcast); 4112 4113 ixl_stat_update32(hw, I40E_GLPRT_TDOLD(hw->port), 4114 pf->stat_offsets_loaded, 4115 &osd->tx_dropped_link_down, 4116 &nsd->tx_dropped_link_down); 4117 ixl_stat_update32(hw, I40E_GLPRT_MLFC(hw->port), 4118 pf->stat_offsets_loaded, 4119 &osd->mac_local_faults, 4120 &nsd->mac_local_faults); 4121 ixl_stat_update32(hw, I40E_GLPRT_MRFC(hw->port), 4122 pf->stat_offsets_loaded, 4123 &osd->mac_remote_faults, 4124 &nsd->mac_remote_faults); 4125 ixl_stat_update32(hw, I40E_GLPRT_RLEC(hw->port), 4126 pf->stat_offsets_loaded, 4127 &osd->rx_length_errors, 4128 &nsd->rx_length_errors); 4129 4130 /* Flow control (LFC) stats */ 4131 ixl_stat_update32(hw, I40E_GLPRT_LXONRXC(hw->port), 4132 pf->stat_offsets_loaded, 4133 &osd->link_xon_rx, &nsd->link_xon_rx); 4134 ixl_stat_update32(hw, I40E_GLPRT_LXONTXC(hw->port), 4135 pf->stat_offsets_loaded, 4136 &osd->link_xon_tx, &nsd->link_xon_tx); 4137 ixl_stat_update32(hw, I40E_GLPRT_LXOFFRXC(hw->port), 4138 pf->stat_offsets_loaded, 4139 &osd->link_xoff_rx, &nsd->link_xoff_rx); 4140 ixl_stat_update32(hw, I40E_GLPRT_LXOFFTXC(hw->port), 4141 pf->stat_offsets_loaded, 4142 &osd->link_xoff_tx, &nsd->link_xoff_tx); 4143 4144 /* Packet size stats rx */ 4145 ixl_stat_update48(hw, I40E_GLPRT_PRC64H(hw->port), 4146 I40E_GLPRT_PRC64L(hw->port), 4147 pf->stat_offsets_loaded, 4148 &osd->rx_size_64, &nsd->rx_size_64); 4149 ixl_stat_update48(hw, I40E_GLPRT_PRC127H(hw->port), 4150 I40E_GLPRT_PRC127L(hw->port), 4151 pf->stat_offsets_loaded, 4152 &osd->rx_size_127, &nsd->rx_size_127); 4153 ixl_stat_update48(hw, I40E_GLPRT_PRC255H(hw->port), 4154 I40E_GLPRT_PRC255L(hw->port), 4155 pf->stat_offsets_loaded, 4156 &osd->rx_size_255, &nsd->rx_size_255); 4157 ixl_stat_update48(hw, I40E_GLPRT_PRC511H(hw->port), 4158 I40E_GLPRT_PRC511L(hw->port), 4159 pf->stat_offsets_loaded, 4160 &osd->rx_size_511, &nsd->rx_size_511); 4161 ixl_stat_update48(hw, I40E_GLPRT_PRC1023H(hw->port), 4162 I40E_GLPRT_PRC1023L(hw->port), 4163 pf->stat_offsets_loaded, 4164 &osd->rx_size_1023, &nsd->rx_size_1023); 4165 ixl_stat_update48(hw, I40E_GLPRT_PRC1522H(hw->port), 4166 I40E_GLPRT_PRC1522L(hw->port), 4167 pf->stat_offsets_loaded, 4168 &osd->rx_size_1522, &nsd->rx_size_1522); 4169 ixl_stat_update48(hw, I40E_GLPRT_PRC9522H(hw->port), 4170 I40E_GLPRT_PRC9522L(hw->port), 4171 pf->stat_offsets_loaded, 4172 &osd->rx_size_big, &nsd->rx_size_big); 4173 4174 /* Packet size stats tx */ 4175 ixl_stat_update48(hw, I40E_GLPRT_PTC64H(hw->port), 4176 I40E_GLPRT_PTC64L(hw->port), 4177 pf->stat_offsets_loaded, 4178 &osd->tx_size_64, &nsd->tx_size_64); 4179 ixl_stat_update48(hw, I40E_GLPRT_PTC127H(hw->port), 4180 I40E_GLPRT_PTC127L(hw->port), 4181 pf->stat_offsets_loaded, 4182 &osd->tx_size_127, &nsd->tx_size_127); 4183 ixl_stat_update48(hw, I40E_GLPRT_PTC255H(hw->port), 4184 I40E_GLPRT_PTC255L(hw->port), 4185 pf->stat_offsets_loaded, 4186 &osd->tx_size_255, &nsd->tx_size_255); 4187 ixl_stat_update48(hw, I40E_GLPRT_PTC511H(hw->port), 4188 I40E_GLPRT_PTC511L(hw->port), 4189 pf->stat_offsets_loaded, 4190 &osd->tx_size_511, &nsd->tx_size_511); 4191 ixl_stat_update48(hw, I40E_GLPRT_PTC1023H(hw->port), 4192 I40E_GLPRT_PTC1023L(hw->port), 4193 pf->stat_offsets_loaded, 4194 &osd->tx_size_1023, &nsd->tx_size_1023); 4195 ixl_stat_update48(hw, I40E_GLPRT_PTC1522H(hw->port), 4196 I40E_GLPRT_PTC1522L(hw->port), 4197 pf->stat_offsets_loaded, 4198 &osd->tx_size_1522, &nsd->tx_size_1522); 4199 ixl_stat_update48(hw, I40E_GLPRT_PTC9522H(hw->port), 4200 I40E_GLPRT_PTC9522L(hw->port), 4201 pf->stat_offsets_loaded, 4202 &osd->tx_size_big, &nsd->tx_size_big); 4203 4204 ixl_stat_update32(hw, I40E_GLPRT_RUC(hw->port), 4205 pf->stat_offsets_loaded, 4206 &osd->rx_undersize, &nsd->rx_undersize); 4207 ixl_stat_update32(hw, I40E_GLPRT_RFC(hw->port), 4208 pf->stat_offsets_loaded, 4209 &osd->rx_fragments, &nsd->rx_fragments); 4210 ixl_stat_update32(hw, I40E_GLPRT_ROC(hw->port), 4211 pf->stat_offsets_loaded, 4212 &osd->rx_oversize, &nsd->rx_oversize); 4213 ixl_stat_update32(hw, I40E_GLPRT_RJC(hw->port), 4214 pf->stat_offsets_loaded, 4215 &osd->rx_jabber, &nsd->rx_jabber); 4216 pf->stat_offsets_loaded = true; 4217 /* End hw stats */ 4218 4219 /* Update vsi stats */ 4220 ixl_update_vsi_stats(vsi); 4221 4222 for (int i = 0; i < pf->num_vfs; i++) { 4223 vf = &pf->vfs[i]; 4224 if (vf->vf_flags & VF_FLAG_ENABLED) 4225 ixl_update_eth_stats(&pf->vfs[i].vsi); 4226 } 4227 } 4228 4229 /* 4230 ** Tasklet handler for MSIX Adminq interrupts 4231 ** - do outside interrupt since it might sleep 4232 */ 4233 static void 4234 ixl_do_adminq(void *context, int pending) 4235 { 4236 struct ixl_pf *pf = context; 4237 struct i40e_hw *hw = &pf->hw; 4238 struct ixl_vsi *vsi = &pf->vsi; 4239 struct i40e_arq_event_info event; 4240 i40e_status ret; 4241 u32 reg, loop = 0; 4242 u16 opcode, result; 4243 4244 event.buf_len = IXL_AQ_BUF_SZ; 4245 event.msg_buf = malloc(event.buf_len, 4246 M_DEVBUF, M_NOWAIT | M_ZERO); 4247 if (!event.msg_buf) { 4248 printf("Unable to allocate adminq memory\n"); 4249 return; 4250 } 4251 4252 IXL_PF_LOCK(pf); 4253 /* clean and process any events */ 4254 do { 4255 ret = i40e_clean_arq_element(hw, &event, &result); 4256 if (ret) 4257 break; 4258 opcode = LE16_TO_CPU(event.desc.opcode); 4259 switch (opcode) { 4260 case i40e_aqc_opc_get_link_status: 4261 ixl_link_event(pf, &event); 4262 ixl_update_link_status(pf); 4263 break; 4264 case i40e_aqc_opc_send_msg_to_pf: 4265 #ifdef PCI_IOV 4266 ixl_handle_vf_msg(pf, &event); 4267 #endif 4268 break; 4269 case i40e_aqc_opc_event_lan_overflow: 4270 break; 4271 default: 4272 #ifdef IXL_DEBUG 4273 printf("AdminQ unknown event %x\n", opcode); 4274 #endif 4275 break; 4276 } 4277 4278 } while (result && (loop++ < IXL_ADM_LIMIT)); 4279 4280 reg = rd32(hw, I40E_PFINT_ICR0_ENA); 4281 reg |= I40E_PFINT_ICR0_ENA_ADMINQ_MASK; 4282 wr32(hw, I40E_PFINT_ICR0_ENA, reg); 4283 free(event.msg_buf, M_DEVBUF); 4284 4285 /* 4286 * If there are still messages to process, reschedule ourselves. 4287 * Otherwise, re-enable our interrupt and go to sleep. 4288 */ 4289 if (result > 0) 4290 taskqueue_enqueue(pf->tq, &pf->adminq); 4291 else 4292 ixl_enable_intr(vsi); 4293 4294 IXL_PF_UNLOCK(pf); 4295 } 4296 4297 static int 4298 ixl_debug_info(SYSCTL_HANDLER_ARGS) 4299 { 4300 struct ixl_pf *pf; 4301 int error, input = 0; 4302 4303 error = sysctl_handle_int(oidp, &input, 0, req); 4304 4305 if (error || !req->newptr) 4306 return (error); 4307 4308 if (input == 1) { 4309 pf = (struct ixl_pf *)arg1; 4310 ixl_print_debug_info(pf); 4311 } 4312 4313 return (error); 4314 } 4315 4316 static void 4317 ixl_print_debug_info(struct ixl_pf *pf) 4318 { 4319 struct i40e_hw *hw = &pf->hw; 4320 struct ixl_vsi *vsi = &pf->vsi; 4321 struct ixl_queue *que = vsi->queues; 4322 struct rx_ring *rxr = &que->rxr; 4323 struct tx_ring *txr = &que->txr; 4324 u32 reg; 4325 4326 4327 printf("Queue irqs = %jx\n", (uintmax_t)que->irqs); 4328 printf("AdminQ irqs = %jx\n", (uintmax_t)pf->admin_irq); 4329 printf("RX next check = %x\n", rxr->next_check); 4330 printf("RX not ready = %jx\n", (uintmax_t)rxr->not_done); 4331 printf("RX packets = %jx\n", (uintmax_t)rxr->rx_packets); 4332 printf("TX desc avail = %x\n", txr->avail); 4333 4334 reg = rd32(hw, I40E_GLV_GORCL(0xc)); 4335 printf("RX Bytes = %x\n", reg); 4336 reg = rd32(hw, I40E_GLPRT_GORCL(hw->port)); 4337 printf("Port RX Bytes = %x\n", reg); 4338 reg = rd32(hw, I40E_GLV_RDPC(0xc)); 4339 printf("RX discard = %x\n", reg); 4340 reg = rd32(hw, I40E_GLPRT_RDPC(hw->port)); 4341 printf("Port RX discard = %x\n", reg); 4342 4343 reg = rd32(hw, I40E_GLV_TEPC(0xc)); 4344 printf("TX errors = %x\n", reg); 4345 reg = rd32(hw, I40E_GLV_GOTCL(0xc)); 4346 printf("TX Bytes = %x\n", reg); 4347 4348 reg = rd32(hw, I40E_GLPRT_RUC(hw->port)); 4349 printf("RX undersize = %x\n", reg); 4350 reg = rd32(hw, I40E_GLPRT_RFC(hw->port)); 4351 printf("RX fragments = %x\n", reg); 4352 reg = rd32(hw, I40E_GLPRT_ROC(hw->port)); 4353 printf("RX oversize = %x\n", reg); 4354 reg = rd32(hw, I40E_GLPRT_RLEC(hw->port)); 4355 printf("RX length error = %x\n", reg); 4356 reg = rd32(hw, I40E_GLPRT_MRFC(hw->port)); 4357 printf("mac remote fault = %x\n", reg); 4358 reg = rd32(hw, I40E_GLPRT_MLFC(hw->port)); 4359 printf("mac local fault = %x\n", reg); 4360 } 4361 4362 /** 4363 * Update VSI-specific ethernet statistics counters. 4364 **/ 4365 void ixl_update_eth_stats(struct ixl_vsi *vsi) 4366 { 4367 struct ixl_pf *pf = (struct ixl_pf *)vsi->back; 4368 struct i40e_hw *hw = &pf->hw; 4369 struct i40e_eth_stats *es; 4370 struct i40e_eth_stats *oes; 4371 struct i40e_hw_port_stats *nsd; 4372 u16 stat_idx = vsi->info.stat_counter_idx; 4373 4374 es = &vsi->eth_stats; 4375 oes = &vsi->eth_stats_offsets; 4376 nsd = &pf->stats; 4377 4378 /* Gather up the stats that the hw collects */ 4379 ixl_stat_update32(hw, I40E_GLV_TEPC(stat_idx), 4380 vsi->stat_offsets_loaded, 4381 &oes->tx_errors, &es->tx_errors); 4382 ixl_stat_update32(hw, I40E_GLV_RDPC(stat_idx), 4383 vsi->stat_offsets_loaded, 4384 &oes->rx_discards, &es->rx_discards); 4385 4386 ixl_stat_update48(hw, I40E_GLV_GORCH(stat_idx), 4387 I40E_GLV_GORCL(stat_idx), 4388 vsi->stat_offsets_loaded, 4389 &oes->rx_bytes, &es->rx_bytes); 4390 ixl_stat_update48(hw, I40E_GLV_UPRCH(stat_idx), 4391 I40E_GLV_UPRCL(stat_idx), 4392 vsi->stat_offsets_loaded, 4393 &oes->rx_unicast, &es->rx_unicast); 4394 ixl_stat_update48(hw, I40E_GLV_MPRCH(stat_idx), 4395 I40E_GLV_MPRCL(stat_idx), 4396 vsi->stat_offsets_loaded, 4397 &oes->rx_multicast, &es->rx_multicast); 4398 ixl_stat_update48(hw, I40E_GLV_BPRCH(stat_idx), 4399 I40E_GLV_BPRCL(stat_idx), 4400 vsi->stat_offsets_loaded, 4401 &oes->rx_broadcast, &es->rx_broadcast); 4402 4403 ixl_stat_update48(hw, I40E_GLV_GOTCH(stat_idx), 4404 I40E_GLV_GOTCL(stat_idx), 4405 vsi->stat_offsets_loaded, 4406 &oes->tx_bytes, &es->tx_bytes); 4407 ixl_stat_update48(hw, I40E_GLV_UPTCH(stat_idx), 4408 I40E_GLV_UPTCL(stat_idx), 4409 vsi->stat_offsets_loaded, 4410 &oes->tx_unicast, &es->tx_unicast); 4411 ixl_stat_update48(hw, I40E_GLV_MPTCH(stat_idx), 4412 I40E_GLV_MPTCL(stat_idx), 4413 vsi->stat_offsets_loaded, 4414 &oes->tx_multicast, &es->tx_multicast); 4415 ixl_stat_update48(hw, I40E_GLV_BPTCH(stat_idx), 4416 I40E_GLV_BPTCL(stat_idx), 4417 vsi->stat_offsets_loaded, 4418 &oes->tx_broadcast, &es->tx_broadcast); 4419 vsi->stat_offsets_loaded = true; 4420 } 4421 4422 static void 4423 ixl_update_vsi_stats(struct ixl_vsi *vsi) 4424 { 4425 struct ixl_pf *pf; 4426 struct ifnet *ifp; 4427 struct i40e_eth_stats *es; 4428 u64 tx_discards; 4429 4430 struct i40e_hw_port_stats *nsd; 4431 4432 pf = vsi->back; 4433 ifp = vsi->ifp; 4434 es = &vsi->eth_stats; 4435 nsd = &pf->stats; 4436 4437 ixl_update_eth_stats(vsi); 4438 4439 tx_discards = es->tx_discards + nsd->tx_dropped_link_down; 4440 for (int i = 0; i < vsi->num_queues; i++) 4441 tx_discards += vsi->queues[i].txr.br->br_drops; 4442 4443 /* Update ifnet stats */ 4444 IXL_SET_IPACKETS(vsi, es->rx_unicast + 4445 es->rx_multicast + 4446 es->rx_broadcast); 4447 IXL_SET_OPACKETS(vsi, es->tx_unicast + 4448 es->tx_multicast + 4449 es->tx_broadcast); 4450 IXL_SET_IBYTES(vsi, es->rx_bytes); 4451 IXL_SET_OBYTES(vsi, es->tx_bytes); 4452 IXL_SET_IMCASTS(vsi, es->rx_multicast); 4453 IXL_SET_OMCASTS(vsi, es->tx_multicast); 4454 4455 IXL_SET_IERRORS(vsi, nsd->crc_errors + nsd->illegal_bytes + 4456 nsd->rx_undersize + nsd->rx_oversize + nsd->rx_fragments + 4457 nsd->rx_jabber); 4458 IXL_SET_OERRORS(vsi, es->tx_errors); 4459 IXL_SET_IQDROPS(vsi, es->rx_discards + nsd->eth.rx_discards); 4460 IXL_SET_OQDROPS(vsi, tx_discards); 4461 IXL_SET_NOPROTO(vsi, es->rx_unknown_protocol); 4462 IXL_SET_COLLISIONS(vsi, 0); 4463 } 4464 4465 /** 4466 * Reset all of the stats for the given pf 4467 **/ 4468 void ixl_pf_reset_stats(struct ixl_pf *pf) 4469 { 4470 bzero(&pf->stats, sizeof(struct i40e_hw_port_stats)); 4471 bzero(&pf->stats_offsets, sizeof(struct i40e_hw_port_stats)); 4472 pf->stat_offsets_loaded = false; 4473 } 4474 4475 /** 4476 * Resets all stats of the given vsi 4477 **/ 4478 void ixl_vsi_reset_stats(struct ixl_vsi *vsi) 4479 { 4480 bzero(&vsi->eth_stats, sizeof(struct i40e_eth_stats)); 4481 bzero(&vsi->eth_stats_offsets, sizeof(struct i40e_eth_stats)); 4482 vsi->stat_offsets_loaded = false; 4483 } 4484 4485 /** 4486 * Read and update a 48 bit stat from the hw 4487 * 4488 * Since the device stats are not reset at PFReset, they likely will not 4489 * be zeroed when the driver starts. We'll save the first values read 4490 * and use them as offsets to be subtracted from the raw values in order 4491 * to report stats that count from zero. 4492 **/ 4493 static void 4494 ixl_stat_update48(struct i40e_hw *hw, u32 hireg, u32 loreg, 4495 bool offset_loaded, u64 *offset, u64 *stat) 4496 { 4497 u64 new_data; 4498 4499 #if defined(__FreeBSD__) && (__FreeBSD_version >= 1000000) && defined(__amd64__) 4500 new_data = rd64(hw, loreg); 4501 #else 4502 /* 4503 * Use two rd32's instead of one rd64; FreeBSD versions before 4504 * 10 don't support 8 byte bus reads/writes. 4505 */ 4506 new_data = rd32(hw, loreg); 4507 new_data |= ((u64)(rd32(hw, hireg) & 0xFFFF)) << 32; 4508 #endif 4509 4510 if (!offset_loaded) 4511 *offset = new_data; 4512 if (new_data >= *offset) 4513 *stat = new_data - *offset; 4514 else 4515 *stat = (new_data + ((u64)1 << 48)) - *offset; 4516 *stat &= 0xFFFFFFFFFFFFULL; 4517 } 4518 4519 /** 4520 * Read and update a 32 bit stat from the hw 4521 **/ 4522 static void 4523 ixl_stat_update32(struct i40e_hw *hw, u32 reg, 4524 bool offset_loaded, u64 *offset, u64 *stat) 4525 { 4526 u32 new_data; 4527 4528 new_data = rd32(hw, reg); 4529 if (!offset_loaded) 4530 *offset = new_data; 4531 if (new_data >= *offset) 4532 *stat = (u32)(new_data - *offset); 4533 else 4534 *stat = (u32)((new_data + ((u64)1 << 32)) - *offset); 4535 } 4536 4537 /* 4538 ** Set flow control using sysctl: 4539 ** 0 - off 4540 ** 1 - rx pause 4541 ** 2 - tx pause 4542 ** 3 - full 4543 */ 4544 static int 4545 ixl_set_flowcntl(SYSCTL_HANDLER_ARGS) 4546 { 4547 /* 4548 * TODO: ensure flow control is disabled if 4549 * priority flow control is enabled 4550 * 4551 * TODO: ensure tx CRC by hardware should be enabled 4552 * if tx flow control is enabled. 4553 */ 4554 struct ixl_pf *pf = (struct ixl_pf *)arg1; 4555 struct i40e_hw *hw = &pf->hw; 4556 device_t dev = pf->dev; 4557 int error = 0; 4558 enum i40e_status_code aq_error = 0; 4559 u8 fc_aq_err = 0; 4560 4561 /* Get request */ 4562 error = sysctl_handle_int(oidp, &pf->fc, 0, req); 4563 if ((error) || (req->newptr == NULL)) 4564 return (error); 4565 if (pf->fc < 0 || pf->fc > 3) { 4566 device_printf(dev, 4567 "Invalid fc mode; valid modes are 0 through 3\n"); 4568 return (EINVAL); 4569 } 4570 4571 /* 4572 ** Changing flow control mode currently does not work on 4573 ** 40GBASE-CR4 PHYs 4574 */ 4575 if (hw->phy.link_info.phy_type == I40E_PHY_TYPE_40GBASE_CR4 4576 || hw->phy.link_info.phy_type == I40E_PHY_TYPE_40GBASE_CR4_CU) { 4577 device_printf(dev, "Changing flow control mode unsupported" 4578 " on 40GBase-CR4 media.\n"); 4579 return (ENODEV); 4580 } 4581 4582 /* Set fc ability for port */ 4583 hw->fc.requested_mode = pf->fc; 4584 aq_error = i40e_set_fc(hw, &fc_aq_err, TRUE); 4585 if (aq_error) { 4586 device_printf(dev, 4587 "%s: Error setting new fc mode %d; fc_err %#x\n", 4588 __func__, aq_error, fc_aq_err); 4589 return (EAGAIN); 4590 } 4591 4592 return (0); 4593 } 4594 4595 static int 4596 ixl_current_speed(SYSCTL_HANDLER_ARGS) 4597 { 4598 struct ixl_pf *pf = (struct ixl_pf *)arg1; 4599 struct i40e_hw *hw = &pf->hw; 4600 int error = 0, index = 0; 4601 4602 char *speeds[] = { 4603 "Unknown", 4604 "100M", 4605 "1G", 4606 "10G", 4607 "40G", 4608 "20G" 4609 }; 4610 4611 ixl_update_link_status(pf); 4612 4613 switch (hw->phy.link_info.link_speed) { 4614 case I40E_LINK_SPEED_100MB: 4615 index = 1; 4616 break; 4617 case I40E_LINK_SPEED_1GB: 4618 index = 2; 4619 break; 4620 case I40E_LINK_SPEED_10GB: 4621 index = 3; 4622 break; 4623 case I40E_LINK_SPEED_40GB: 4624 index = 4; 4625 break; 4626 case I40E_LINK_SPEED_20GB: 4627 index = 5; 4628 break; 4629 case I40E_LINK_SPEED_UNKNOWN: 4630 default: 4631 index = 0; 4632 break; 4633 } 4634 4635 error = sysctl_handle_string(oidp, speeds[index], 4636 strlen(speeds[index]), req); 4637 return (error); 4638 } 4639 4640 static int 4641 ixl_set_advertised_speeds(struct ixl_pf *pf, int speeds) 4642 { 4643 struct i40e_hw *hw = &pf->hw; 4644 device_t dev = pf->dev; 4645 struct i40e_aq_get_phy_abilities_resp abilities; 4646 struct i40e_aq_set_phy_config config; 4647 enum i40e_status_code aq_error = 0; 4648 4649 /* Get current capability information */ 4650 aq_error = i40e_aq_get_phy_capabilities(hw, 4651 FALSE, FALSE, &abilities, NULL); 4652 if (aq_error) { 4653 device_printf(dev, 4654 "%s: Error getting phy capabilities %d," 4655 " aq error: %d\n", __func__, aq_error, 4656 hw->aq.asq_last_status); 4657 return (EAGAIN); 4658 } 4659 4660 /* Prepare new config */ 4661 bzero(&config, sizeof(config)); 4662 config.phy_type = abilities.phy_type; 4663 config.abilities = abilities.abilities 4664 | I40E_AQ_PHY_ENABLE_ATOMIC_LINK; 4665 config.eee_capability = abilities.eee_capability; 4666 config.eeer = abilities.eeer_val; 4667 config.low_power_ctrl = abilities.d3_lpan; 4668 /* Translate into aq cmd link_speed */ 4669 if (speeds & 0x8) 4670 config.link_speed |= I40E_LINK_SPEED_20GB; 4671 if (speeds & 0x4) 4672 config.link_speed |= I40E_LINK_SPEED_10GB; 4673 if (speeds & 0x2) 4674 config.link_speed |= I40E_LINK_SPEED_1GB; 4675 if (speeds & 0x1) 4676 config.link_speed |= I40E_LINK_SPEED_100MB; 4677 4678 /* Do aq command & restart link */ 4679 aq_error = i40e_aq_set_phy_config(hw, &config, NULL); 4680 if (aq_error) { 4681 device_printf(dev, 4682 "%s: Error setting new phy config %d," 4683 " aq error: %d\n", __func__, aq_error, 4684 hw->aq.asq_last_status); 4685 return (EAGAIN); 4686 } 4687 4688 /* 4689 ** This seems a bit heavy handed, but we 4690 ** need to get a reinit on some devices 4691 */ 4692 IXL_PF_LOCK(pf); 4693 ixl_stop(pf); 4694 ixl_init_locked(pf); 4695 IXL_PF_UNLOCK(pf); 4696 4697 return (0); 4698 } 4699 4700 /* 4701 ** Control link advertise speed: 4702 ** Flags: 4703 ** 0x1 - advertise 100 Mb 4704 ** 0x2 - advertise 1G 4705 ** 0x4 - advertise 10G 4706 ** 0x8 - advertise 20G 4707 ** 4708 ** Does not work on 40G devices. 4709 */ 4710 static int 4711 ixl_set_advertise(SYSCTL_HANDLER_ARGS) 4712 { 4713 struct ixl_pf *pf = (struct ixl_pf *)arg1; 4714 struct i40e_hw *hw = &pf->hw; 4715 device_t dev = pf->dev; 4716 int requested_ls = 0; 4717 int error = 0; 4718 4719 /* 4720 ** FW doesn't support changing advertised speed 4721 ** for 40G devices; speed is always 40G. 4722 */ 4723 if (i40e_is_40G_device(hw->device_id)) 4724 return (ENODEV); 4725 4726 /* Read in new mode */ 4727 requested_ls = pf->advertised_speed; 4728 error = sysctl_handle_int(oidp, &requested_ls, 0, req); 4729 if ((error) || (req->newptr == NULL)) 4730 return (error); 4731 /* Check for sane value */ 4732 if (requested_ls < 0x1 || requested_ls > 0xE) { 4733 device_printf(dev, "Invalid advertised speed; " 4734 "valid modes are 0x1 through 0xE\n"); 4735 return (EINVAL); 4736 } 4737 /* Then check for validity based on adapter type */ 4738 switch (hw->device_id) { 4739 case I40E_DEV_ID_10G_BASE_T: 4740 if (requested_ls & 0x8) { 4741 device_printf(dev, 4742 "20Gbs speed not supported on this device.\n"); 4743 return (EINVAL); 4744 } 4745 break; 4746 case I40E_DEV_ID_20G_KR2: 4747 if (requested_ls & 0x1) { 4748 device_printf(dev, 4749 "100Mbs speed not supported on this device.\n"); 4750 return (EINVAL); 4751 } 4752 break; 4753 default: 4754 if (requested_ls & ~0x6) { 4755 device_printf(dev, 4756 "Only 1/10Gbs speeds are supported on this device.\n"); 4757 return (EINVAL); 4758 } 4759 break; 4760 } 4761 4762 /* Exit if no change */ 4763 if (pf->advertised_speed == requested_ls) 4764 return (0); 4765 4766 error = ixl_set_advertised_speeds(pf, requested_ls); 4767 if (error) 4768 return (error); 4769 4770 pf->advertised_speed = requested_ls; 4771 ixl_update_link_status(pf); 4772 return (0); 4773 } 4774 4775 /* 4776 ** Get the width and transaction speed of 4777 ** the bus this adapter is plugged into. 4778 */ 4779 static u16 4780 ixl_get_bus_info(struct i40e_hw *hw, device_t dev) 4781 { 4782 u16 link; 4783 u32 offset; 4784 4785 4786 /* Get the PCI Express Capabilities offset */ 4787 pci_find_cap(dev, PCIY_EXPRESS, &offset); 4788 4789 /* ...and read the Link Status Register */ 4790 link = pci_read_config(dev, offset + PCIER_LINK_STA, 2); 4791 4792 switch (link & I40E_PCI_LINK_WIDTH) { 4793 case I40E_PCI_LINK_WIDTH_1: 4794 hw->bus.width = i40e_bus_width_pcie_x1; 4795 break; 4796 case I40E_PCI_LINK_WIDTH_2: 4797 hw->bus.width = i40e_bus_width_pcie_x2; 4798 break; 4799 case I40E_PCI_LINK_WIDTH_4: 4800 hw->bus.width = i40e_bus_width_pcie_x4; 4801 break; 4802 case I40E_PCI_LINK_WIDTH_8: 4803 hw->bus.width = i40e_bus_width_pcie_x8; 4804 break; 4805 default: 4806 hw->bus.width = i40e_bus_width_unknown; 4807 break; 4808 } 4809 4810 switch (link & I40E_PCI_LINK_SPEED) { 4811 case I40E_PCI_LINK_SPEED_2500: 4812 hw->bus.speed = i40e_bus_speed_2500; 4813 break; 4814 case I40E_PCI_LINK_SPEED_5000: 4815 hw->bus.speed = i40e_bus_speed_5000; 4816 break; 4817 case I40E_PCI_LINK_SPEED_8000: 4818 hw->bus.speed = i40e_bus_speed_8000; 4819 break; 4820 default: 4821 hw->bus.speed = i40e_bus_speed_unknown; 4822 break; 4823 } 4824 4825 4826 device_printf(dev,"PCI Express Bus: Speed %s %s\n", 4827 ((hw->bus.speed == i40e_bus_speed_8000) ? "8.0GT/s": 4828 (hw->bus.speed == i40e_bus_speed_5000) ? "5.0GT/s": 4829 (hw->bus.speed == i40e_bus_speed_2500) ? "2.5GT/s":"Unknown"), 4830 (hw->bus.width == i40e_bus_width_pcie_x8) ? "Width x8" : 4831 (hw->bus.width == i40e_bus_width_pcie_x4) ? "Width x4" : 4832 (hw->bus.width == i40e_bus_width_pcie_x1) ? "Width x1" : 4833 ("Unknown")); 4834 4835 if ((hw->bus.width <= i40e_bus_width_pcie_x8) && 4836 (hw->bus.speed < i40e_bus_speed_8000)) { 4837 device_printf(dev, "PCI-Express bandwidth available" 4838 " for this device\n may be insufficient for" 4839 " optimal performance.\n"); 4840 device_printf(dev, "For expected performance a x8 " 4841 "PCIE Gen3 slot is required.\n"); 4842 } 4843 4844 return (link); 4845 } 4846 4847 static int 4848 ixl_sysctl_show_fw(SYSCTL_HANDLER_ARGS) 4849 { 4850 struct ixl_pf *pf = (struct ixl_pf *)arg1; 4851 struct i40e_hw *hw = &pf->hw; 4852 char buf[32]; 4853 4854 snprintf(buf, sizeof(buf), 4855 "f%d.%d a%d.%d n%02x.%02x e%08x", 4856 hw->aq.fw_maj_ver, hw->aq.fw_min_ver, 4857 hw->aq.api_maj_ver, hw->aq.api_min_ver, 4858 (hw->nvm.version & IXL_NVM_VERSION_HI_MASK) >> 4859 IXL_NVM_VERSION_HI_SHIFT, 4860 (hw->nvm.version & IXL_NVM_VERSION_LO_MASK) >> 4861 IXL_NVM_VERSION_LO_SHIFT, 4862 hw->nvm.eetrack); 4863 return (sysctl_handle_string(oidp, buf, strlen(buf), req)); 4864 } 4865 4866 4867 #ifdef IXL_DEBUG_SYSCTL 4868 static int 4869 ixl_sysctl_link_status(SYSCTL_HANDLER_ARGS) 4870 { 4871 struct ixl_pf *pf = (struct ixl_pf *)arg1; 4872 struct i40e_hw *hw = &pf->hw; 4873 struct i40e_link_status link_status; 4874 char buf[512]; 4875 4876 enum i40e_status_code aq_error = 0; 4877 4878 aq_error = i40e_aq_get_link_info(hw, TRUE, &link_status, NULL); 4879 if (aq_error) { 4880 printf("i40e_aq_get_link_info() error %d\n", aq_error); 4881 return (EPERM); 4882 } 4883 4884 sprintf(buf, "\n" 4885 "PHY Type : %#04x\n" 4886 "Speed : %#04x\n" 4887 "Link info: %#04x\n" 4888 "AN info : %#04x\n" 4889 "Ext info : %#04x", 4890 link_status.phy_type, link_status.link_speed, 4891 link_status.link_info, link_status.an_info, 4892 link_status.ext_info); 4893 4894 return (sysctl_handle_string(oidp, buf, strlen(buf), req)); 4895 } 4896 4897 static int 4898 ixl_sysctl_phy_abilities(SYSCTL_HANDLER_ARGS) 4899 { 4900 struct ixl_pf *pf = (struct ixl_pf *)arg1; 4901 struct i40e_hw *hw = &pf->hw; 4902 char buf[512]; 4903 enum i40e_status_code aq_error = 0; 4904 4905 struct i40e_aq_get_phy_abilities_resp abilities; 4906 4907 aq_error = i40e_aq_get_phy_capabilities(hw, 4908 TRUE, FALSE, &abilities, NULL); 4909 if (aq_error) { 4910 printf("i40e_aq_get_phy_capabilities() error %d\n", aq_error); 4911 return (EPERM); 4912 } 4913 4914 sprintf(buf, "\n" 4915 "PHY Type : %#010x\n" 4916 "Speed : %#04x\n" 4917 "Abilities: %#04x\n" 4918 "EEE cap : %#06x\n" 4919 "EEER reg : %#010x\n" 4920 "D3 Lpan : %#04x", 4921 abilities.phy_type, abilities.link_speed, 4922 abilities.abilities, abilities.eee_capability, 4923 abilities.eeer_val, abilities.d3_lpan); 4924 4925 return (sysctl_handle_string(oidp, buf, strlen(buf), req)); 4926 } 4927 4928 static int 4929 ixl_sysctl_sw_filter_list(SYSCTL_HANDLER_ARGS) 4930 { 4931 struct ixl_pf *pf = (struct ixl_pf *)arg1; 4932 struct ixl_vsi *vsi = &pf->vsi; 4933 struct ixl_mac_filter *f; 4934 char *buf, *buf_i; 4935 4936 int error = 0; 4937 int ftl_len = 0; 4938 int ftl_counter = 0; 4939 int buf_len = 0; 4940 int entry_len = 42; 4941 4942 SLIST_FOREACH(f, &vsi->ftl, next) { 4943 ftl_len++; 4944 } 4945 4946 if (ftl_len < 1) { 4947 sysctl_handle_string(oidp, "(none)", 6, req); 4948 return (0); 4949 } 4950 4951 buf_len = sizeof(char) * (entry_len + 1) * ftl_len + 2; 4952 buf = buf_i = malloc(buf_len, M_DEVBUF, M_NOWAIT); 4953 4954 sprintf(buf_i++, "\n"); 4955 SLIST_FOREACH(f, &vsi->ftl, next) { 4956 sprintf(buf_i, 4957 MAC_FORMAT ", vlan %4d, flags %#06x", 4958 MAC_FORMAT_ARGS(f->macaddr), f->vlan, f->flags); 4959 buf_i += entry_len; 4960 /* don't print '\n' for last entry */ 4961 if (++ftl_counter != ftl_len) { 4962 sprintf(buf_i, "\n"); 4963 buf_i++; 4964 } 4965 } 4966 4967 error = sysctl_handle_string(oidp, buf, strlen(buf), req); 4968 if (error) 4969 printf("sysctl error: %d\n", error); 4970 free(buf, M_DEVBUF); 4971 return error; 4972 } 4973 4974 #define IXL_SW_RES_SIZE 0x14 4975 static int 4976 ixl_res_alloc_cmp(const void *a, const void *b) 4977 { 4978 const struct i40e_aqc_switch_resource_alloc_element_resp *one, *two; 4979 one = (struct i40e_aqc_switch_resource_alloc_element_resp *)a; 4980 two = (struct i40e_aqc_switch_resource_alloc_element_resp *)b; 4981 4982 return ((int)one->resource_type - (int)two->resource_type); 4983 } 4984 4985 static int 4986 ixl_sysctl_hw_res_alloc(SYSCTL_HANDLER_ARGS) 4987 { 4988 struct ixl_pf *pf = (struct ixl_pf *)arg1; 4989 struct i40e_hw *hw = &pf->hw; 4990 device_t dev = pf->dev; 4991 struct sbuf *buf; 4992 int error = 0; 4993 4994 u8 num_entries; 4995 struct i40e_aqc_switch_resource_alloc_element_resp resp[IXL_SW_RES_SIZE]; 4996 4997 buf = sbuf_new_for_sysctl(NULL, NULL, 0, req); 4998 if (!buf) { 4999 device_printf(dev, "Could not allocate sbuf for output.\n"); 5000 return (ENOMEM); 5001 } 5002 5003 bzero(resp, sizeof(resp)); 5004 error = i40e_aq_get_switch_resource_alloc(hw, &num_entries, 5005 resp, 5006 IXL_SW_RES_SIZE, 5007 NULL); 5008 if (error) { 5009 device_printf(dev, 5010 "%s: get_switch_resource_alloc() error %d, aq error %d\n", 5011 __func__, error, hw->aq.asq_last_status); 5012 sbuf_delete(buf); 5013 return error; 5014 } 5015 5016 /* Sort entries by type for display */ 5017 qsort(resp, num_entries, 5018 sizeof(struct i40e_aqc_switch_resource_alloc_element_resp), 5019 &ixl_res_alloc_cmp); 5020 5021 sbuf_cat(buf, "\n"); 5022 sbuf_printf(buf, "# of entries: %d\n", num_entries); 5023 sbuf_printf(buf, 5024 "Type | Guaranteed | Total | Used | Un-allocated\n" 5025 " | (this) | (all) | (this) | (all) \n"); 5026 for (int i = 0; i < num_entries; i++) { 5027 sbuf_printf(buf, 5028 "%#4x | %10d %5d %6d %12d", 5029 resp[i].resource_type, 5030 resp[i].guaranteed, 5031 resp[i].total, 5032 resp[i].used, 5033 resp[i].total_unalloced); 5034 if (i < num_entries - 1) 5035 sbuf_cat(buf, "\n"); 5036 } 5037 5038 error = sbuf_finish(buf); 5039 if (error) { 5040 device_printf(dev, "Error finishing sbuf: %d\n", error); 5041 sbuf_delete(buf); 5042 return error; 5043 } 5044 5045 error = sysctl_handle_string(oidp, sbuf_data(buf), sbuf_len(buf), req); 5046 if (error) 5047 device_printf(dev, "sysctl error: %d\n", error); 5048 sbuf_delete(buf); 5049 return error; 5050 } 5051 5052 /* 5053 ** Caller must init and delete sbuf; this function will clear and 5054 ** finish it for caller. 5055 */ 5056 static char * 5057 ixl_switch_element_string(struct sbuf *s, u16 seid, bool uplink) 5058 { 5059 sbuf_clear(s); 5060 5061 if (seid == 0 && uplink) 5062 sbuf_cat(s, "Network"); 5063 else if (seid == 0) 5064 sbuf_cat(s, "Host"); 5065 else if (seid == 1) 5066 sbuf_cat(s, "EMP"); 5067 else if (seid <= 5) 5068 sbuf_printf(s, "MAC %d", seid - 2); 5069 else if (seid <= 15) 5070 sbuf_cat(s, "Reserved"); 5071 else if (seid <= 31) 5072 sbuf_printf(s, "PF %d", seid - 16); 5073 else if (seid <= 159) 5074 sbuf_printf(s, "VF %d", seid - 32); 5075 else if (seid <= 287) 5076 sbuf_cat(s, "Reserved"); 5077 else if (seid <= 511) 5078 sbuf_cat(s, "Other"); // for other structures 5079 else if (seid <= 895) 5080 sbuf_printf(s, "VSI %d", seid - 512); 5081 else if (seid <= 1023) 5082 sbuf_printf(s, "Reserved"); 5083 else 5084 sbuf_cat(s, "Invalid"); 5085 5086 sbuf_finish(s); 5087 return sbuf_data(s); 5088 } 5089 5090 static int 5091 ixl_sysctl_switch_config(SYSCTL_HANDLER_ARGS) 5092 { 5093 struct ixl_pf *pf = (struct ixl_pf *)arg1; 5094 struct i40e_hw *hw = &pf->hw; 5095 device_t dev = pf->dev; 5096 struct sbuf *buf; 5097 struct sbuf *nmbuf; 5098 int error = 0; 5099 u8 aq_buf[I40E_AQ_LARGE_BUF]; 5100 5101 u16 next = 0; 5102 struct i40e_aqc_get_switch_config_resp *sw_config; 5103 sw_config = (struct i40e_aqc_get_switch_config_resp *)aq_buf; 5104 5105 buf = sbuf_new_for_sysctl(NULL, NULL, 0, req); 5106 if (!buf) { 5107 device_printf(dev, "Could not allocate sbuf for sysctl output.\n"); 5108 return (ENOMEM); 5109 } 5110 5111 error = i40e_aq_get_switch_config(hw, sw_config, 5112 sizeof(aq_buf), &next, NULL); 5113 if (error) { 5114 device_printf(dev, 5115 "%s: aq_get_switch_config() error %d, aq error %d\n", 5116 __func__, error, hw->aq.asq_last_status); 5117 sbuf_delete(buf); 5118 return error; 5119 } 5120 5121 nmbuf = sbuf_new_auto(); 5122 if (!nmbuf) { 5123 device_printf(dev, "Could not allocate sbuf for name output.\n"); 5124 return (ENOMEM); 5125 } 5126 5127 sbuf_cat(buf, "\n"); 5128 // Assuming <= 255 elements in switch 5129 sbuf_printf(buf, "# of elements: %d\n", sw_config->header.num_reported); 5130 /* Exclude: 5131 ** Revision -- all elements are revision 1 for now 5132 */ 5133 sbuf_printf(buf, 5134 "SEID ( Name ) | Uplink | Downlink | Conn Type\n" 5135 " | | | (uplink)\n"); 5136 for (int i = 0; i < sw_config->header.num_reported; i++) { 5137 // "%4d (%8s) | %8s %8s %#8x", 5138 sbuf_printf(buf, "%4d", sw_config->element[i].seid); 5139 sbuf_cat(buf, " "); 5140 sbuf_printf(buf, "(%8s)", ixl_switch_element_string(nmbuf, 5141 sw_config->element[i].seid, false)); 5142 sbuf_cat(buf, " | "); 5143 sbuf_printf(buf, "%8s", ixl_switch_element_string(nmbuf, 5144 sw_config->element[i].uplink_seid, true)); 5145 sbuf_cat(buf, " "); 5146 sbuf_printf(buf, "%8s", ixl_switch_element_string(nmbuf, 5147 sw_config->element[i].downlink_seid, false)); 5148 sbuf_cat(buf, " "); 5149 sbuf_printf(buf, "%#8x", sw_config->element[i].connection_type); 5150 if (i < sw_config->header.num_reported - 1) 5151 sbuf_cat(buf, "\n"); 5152 } 5153 sbuf_delete(nmbuf); 5154 5155 error = sbuf_finish(buf); 5156 if (error) { 5157 device_printf(dev, "Error finishing sbuf: %d\n", error); 5158 sbuf_delete(buf); 5159 return error; 5160 } 5161 5162 error = sysctl_handle_string(oidp, sbuf_data(buf), sbuf_len(buf), req); 5163 if (error) 5164 device_printf(dev, "sysctl error: %d\n", error); 5165 sbuf_delete(buf); 5166 5167 return (error); 5168 } 5169 #endif /* IXL_DEBUG_SYSCTL */ 5170 5171 5172 #ifdef PCI_IOV 5173 static int 5174 ixl_vf_alloc_vsi(struct ixl_pf *pf, struct ixl_vf *vf) 5175 { 5176 struct i40e_hw *hw; 5177 struct ixl_vsi *vsi; 5178 struct i40e_vsi_context vsi_ctx; 5179 int i; 5180 uint16_t first_queue; 5181 enum i40e_status_code code; 5182 5183 hw = &pf->hw; 5184 vsi = &pf->vsi; 5185 5186 vsi_ctx.pf_num = hw->pf_id; 5187 vsi_ctx.uplink_seid = pf->veb_seid; 5188 vsi_ctx.connection_type = IXL_VSI_DATA_PORT; 5189 vsi_ctx.vf_num = hw->func_caps.vf_base_id + vf->vf_num; 5190 vsi_ctx.flags = I40E_AQ_VSI_TYPE_VF; 5191 5192 bzero(&vsi_ctx.info, sizeof(vsi_ctx.info)); 5193 5194 vsi_ctx.info.valid_sections = htole16(I40E_AQ_VSI_PROP_SWITCH_VALID); 5195 vsi_ctx.info.switch_id = htole16(0); 5196 5197 vsi_ctx.info.valid_sections |= htole16(I40E_AQ_VSI_PROP_SECURITY_VALID); 5198 vsi_ctx.info.sec_flags = 0; 5199 if (vf->vf_flags & VF_FLAG_MAC_ANTI_SPOOF) 5200 vsi_ctx.info.sec_flags |= I40E_AQ_VSI_SEC_FLAG_ENABLE_MAC_CHK; 5201 5202 vsi_ctx.info.valid_sections |= htole16(I40E_AQ_VSI_PROP_VLAN_VALID); 5203 vsi_ctx.info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_ALL | 5204 I40E_AQ_VSI_PVLAN_EMOD_NOTHING; 5205 5206 vsi_ctx.info.valid_sections |= 5207 htole16(I40E_AQ_VSI_PROP_QUEUE_MAP_VALID); 5208 vsi_ctx.info.mapping_flags = htole16(I40E_AQ_VSI_QUE_MAP_NONCONTIG); 5209 first_queue = vsi->num_queues + vf->vf_num * IXLV_MAX_QUEUES; 5210 for (i = 0; i < IXLV_MAX_QUEUES; i++) 5211 vsi_ctx.info.queue_mapping[i] = htole16(first_queue + i); 5212 for (; i < nitems(vsi_ctx.info.queue_mapping); i++) 5213 vsi_ctx.info.queue_mapping[i] = htole16(I40E_AQ_VSI_QUEUE_MASK); 5214 5215 vsi_ctx.info.tc_mapping[0] = htole16( 5216 (0 << I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT) | 5217 (1 << I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT)); 5218 5219 code = i40e_aq_add_vsi(hw, &vsi_ctx, NULL); 5220 if (code != I40E_SUCCESS) 5221 return (ixl_adminq_err_to_errno(hw->aq.asq_last_status)); 5222 vf->vsi.seid = vsi_ctx.seid; 5223 vf->vsi.vsi_num = vsi_ctx.vsi_number; 5224 vf->vsi.first_queue = first_queue; 5225 vf->vsi.num_queues = IXLV_MAX_QUEUES; 5226 5227 code = i40e_aq_get_vsi_params(hw, &vsi_ctx, NULL); 5228 if (code != I40E_SUCCESS) 5229 return (ixl_adminq_err_to_errno(hw->aq.asq_last_status)); 5230 5231 code = i40e_aq_config_vsi_bw_limit(hw, vf->vsi.seid, 0, 0, NULL); 5232 if (code != I40E_SUCCESS) { 5233 device_printf(pf->dev, "Failed to disable BW limit: %d\n", 5234 ixl_adminq_err_to_errno(hw->aq.asq_last_status)); 5235 return (ixl_adminq_err_to_errno(hw->aq.asq_last_status)); 5236 } 5237 5238 memcpy(&vf->vsi.info, &vsi_ctx.info, sizeof(vf->vsi.info)); 5239 return (0); 5240 } 5241 5242 static int 5243 ixl_vf_setup_vsi(struct ixl_pf *pf, struct ixl_vf *vf) 5244 { 5245 struct i40e_hw *hw; 5246 int error; 5247 5248 hw = &pf->hw; 5249 5250 error = ixl_vf_alloc_vsi(pf, vf); 5251 if (error != 0) 5252 return (error); 5253 5254 vf->vsi.hw_filters_add = 0; 5255 vf->vsi.hw_filters_del = 0; 5256 ixl_add_filter(&vf->vsi, ixl_bcast_addr, IXL_VLAN_ANY); 5257 ixl_reconfigure_filters(&vf->vsi); 5258 5259 return (0); 5260 } 5261 5262 static void 5263 ixl_vf_map_vsi_queue(struct i40e_hw *hw, struct ixl_vf *vf, int qnum, 5264 uint32_t val) 5265 { 5266 uint32_t qtable; 5267 int index, shift; 5268 5269 /* 5270 * Two queues are mapped in a single register, so we have to do some 5271 * gymnastics to convert the queue number into a register index and 5272 * shift. 5273 */ 5274 index = qnum / 2; 5275 shift = (qnum % 2) * I40E_VSILAN_QTABLE_QINDEX_1_SHIFT; 5276 5277 qtable = rd32(hw, I40E_VSILAN_QTABLE(index, vf->vsi.vsi_num)); 5278 qtable &= ~(I40E_VSILAN_QTABLE_QINDEX_0_MASK << shift); 5279 qtable |= val << shift; 5280 wr32(hw, I40E_VSILAN_QTABLE(index, vf->vsi.vsi_num), qtable); 5281 } 5282 5283 static void 5284 ixl_vf_map_queues(struct ixl_pf *pf, struct ixl_vf *vf) 5285 { 5286 struct i40e_hw *hw; 5287 uint32_t qtable; 5288 int i; 5289 5290 hw = &pf->hw; 5291 5292 /* 5293 * Contiguous mappings aren't actually supported by the hardware, 5294 * so we have to use non-contiguous mappings. 5295 */ 5296 wr32(hw, I40E_VSILAN_QBASE(vf->vsi.vsi_num), 5297 I40E_VSILAN_QBASE_VSIQTABLE_ENA_MASK); 5298 5299 wr32(hw, I40E_VPLAN_MAPENA(vf->vf_num), 5300 I40E_VPLAN_MAPENA_TXRX_ENA_MASK); 5301 5302 for (i = 0; i < vf->vsi.num_queues; i++) { 5303 qtable = (vf->vsi.first_queue + i) << 5304 I40E_VPLAN_QTABLE_QINDEX_SHIFT; 5305 5306 wr32(hw, I40E_VPLAN_QTABLE(i, vf->vf_num), qtable); 5307 } 5308 5309 /* Map queues allocated to VF to its VSI. */ 5310 for (i = 0; i < vf->vsi.num_queues; i++) 5311 ixl_vf_map_vsi_queue(hw, vf, i, vf->vsi.first_queue + i); 5312 5313 /* Set rest of VSI queues as unused. */ 5314 for (; i < IXL_MAX_VSI_QUEUES; i++) 5315 ixl_vf_map_vsi_queue(hw, vf, i, 5316 I40E_VSILAN_QTABLE_QINDEX_0_MASK); 5317 5318 ixl_flush(hw); 5319 } 5320 5321 static void 5322 ixl_vf_vsi_release(struct ixl_pf *pf, struct ixl_vsi *vsi) 5323 { 5324 struct i40e_hw *hw; 5325 5326 hw = &pf->hw; 5327 5328 if (vsi->seid == 0) 5329 return; 5330 5331 i40e_aq_delete_element(hw, vsi->seid, NULL); 5332 } 5333 5334 static void 5335 ixl_vf_disable_queue_intr(struct i40e_hw *hw, uint32_t vfint_reg) 5336 { 5337 5338 wr32(hw, vfint_reg, I40E_VFINT_DYN_CTLN_CLEARPBA_MASK); 5339 ixl_flush(hw); 5340 } 5341 5342 static void 5343 ixl_vf_unregister_intr(struct i40e_hw *hw, uint32_t vpint_reg) 5344 { 5345 5346 wr32(hw, vpint_reg, I40E_VPINT_LNKLSTN_FIRSTQ_TYPE_MASK | 5347 I40E_VPINT_LNKLSTN_FIRSTQ_INDX_MASK); 5348 ixl_flush(hw); 5349 } 5350 5351 static void 5352 ixl_vf_release_resources(struct ixl_pf *pf, struct ixl_vf *vf) 5353 { 5354 struct i40e_hw *hw; 5355 uint32_t vfint_reg, vpint_reg; 5356 int i; 5357 5358 hw = &pf->hw; 5359 5360 ixl_vf_vsi_release(pf, &vf->vsi); 5361 5362 /* Index 0 has a special register. */ 5363 ixl_vf_disable_queue_intr(hw, I40E_VFINT_DYN_CTL0(vf->vf_num)); 5364 5365 for (i = 1; i < hw->func_caps.num_msix_vectors_vf; i++) { 5366 vfint_reg = IXL_VFINT_DYN_CTLN_REG(hw, i , vf->vf_num); 5367 ixl_vf_disable_queue_intr(hw, vfint_reg); 5368 } 5369 5370 /* Index 0 has a special register. */ 5371 ixl_vf_unregister_intr(hw, I40E_VPINT_LNKLST0(vf->vf_num)); 5372 5373 for (i = 1; i < hw->func_caps.num_msix_vectors_vf; i++) { 5374 vpint_reg = IXL_VPINT_LNKLSTN_REG(hw, i, vf->vf_num); 5375 ixl_vf_unregister_intr(hw, vpint_reg); 5376 } 5377 5378 vf->vsi.num_queues = 0; 5379 } 5380 5381 static int 5382 ixl_flush_pcie(struct ixl_pf *pf, struct ixl_vf *vf) 5383 { 5384 struct i40e_hw *hw; 5385 int i; 5386 uint16_t global_vf_num; 5387 uint32_t ciad; 5388 5389 hw = &pf->hw; 5390 global_vf_num = hw->func_caps.vf_base_id + vf->vf_num; 5391 5392 wr32(hw, I40E_PF_PCI_CIAA, IXL_PF_PCI_CIAA_VF_DEVICE_STATUS | 5393 (global_vf_num << I40E_PF_PCI_CIAA_VF_NUM_SHIFT)); 5394 for (i = 0; i < IXL_VF_RESET_TIMEOUT; i++) { 5395 ciad = rd32(hw, I40E_PF_PCI_CIAD); 5396 if ((ciad & IXL_PF_PCI_CIAD_VF_TRANS_PENDING_MASK) == 0) 5397 return (0); 5398 DELAY(1); 5399 } 5400 5401 return (ETIMEDOUT); 5402 } 5403 5404 static void 5405 ixl_reset_vf(struct ixl_pf *pf, struct ixl_vf *vf) 5406 { 5407 struct i40e_hw *hw; 5408 uint32_t vfrtrig; 5409 5410 hw = &pf->hw; 5411 5412 vfrtrig = rd32(hw, I40E_VPGEN_VFRTRIG(vf->vf_num)); 5413 vfrtrig |= I40E_VPGEN_VFRTRIG_VFSWR_MASK; 5414 wr32(hw, I40E_VPGEN_VFRTRIG(vf->vf_num), vfrtrig); 5415 ixl_flush(hw); 5416 5417 ixl_reinit_vf(pf, vf); 5418 } 5419 5420 static void 5421 ixl_reinit_vf(struct ixl_pf *pf, struct ixl_vf *vf) 5422 { 5423 struct i40e_hw *hw; 5424 uint32_t vfrstat, vfrtrig; 5425 int i, error; 5426 5427 hw = &pf->hw; 5428 5429 error = ixl_flush_pcie(pf, vf); 5430 if (error != 0) 5431 device_printf(pf->dev, 5432 "Timed out waiting for PCIe activity to stop on VF-%d\n", 5433 vf->vf_num); 5434 5435 for (i = 0; i < IXL_VF_RESET_TIMEOUT; i++) { 5436 DELAY(10); 5437 5438 vfrstat = rd32(hw, I40E_VPGEN_VFRSTAT(vf->vf_num)); 5439 if (vfrstat & I40E_VPGEN_VFRSTAT_VFRD_MASK) 5440 break; 5441 } 5442 5443 if (i == IXL_VF_RESET_TIMEOUT) 5444 device_printf(pf->dev, "VF %d failed to reset\n", vf->vf_num); 5445 5446 wr32(hw, I40E_VFGEN_RSTAT1(vf->vf_num), I40E_VFR_COMPLETED); 5447 5448 vfrtrig = rd32(hw, I40E_VPGEN_VFRTRIG(vf->vf_num)); 5449 vfrtrig &= ~I40E_VPGEN_VFRTRIG_VFSWR_MASK; 5450 wr32(hw, I40E_VPGEN_VFRTRIG(vf->vf_num), vfrtrig); 5451 5452 if (vf->vsi.seid != 0) 5453 ixl_disable_rings(&vf->vsi); 5454 5455 ixl_vf_release_resources(pf, vf); 5456 ixl_vf_setup_vsi(pf, vf); 5457 ixl_vf_map_queues(pf, vf); 5458 5459 wr32(hw, I40E_VFGEN_RSTAT1(vf->vf_num), I40E_VFR_VFACTIVE); 5460 ixl_flush(hw); 5461 } 5462 5463 static const char * 5464 ixl_vc_opcode_str(uint16_t op) 5465 { 5466 5467 switch (op) { 5468 case I40E_VIRTCHNL_OP_VERSION: 5469 return ("VERSION"); 5470 case I40E_VIRTCHNL_OP_RESET_VF: 5471 return ("RESET_VF"); 5472 case I40E_VIRTCHNL_OP_GET_VF_RESOURCES: 5473 return ("GET_VF_RESOURCES"); 5474 case I40E_VIRTCHNL_OP_CONFIG_TX_QUEUE: 5475 return ("CONFIG_TX_QUEUE"); 5476 case I40E_VIRTCHNL_OP_CONFIG_RX_QUEUE: 5477 return ("CONFIG_RX_QUEUE"); 5478 case I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES: 5479 return ("CONFIG_VSI_QUEUES"); 5480 case I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP: 5481 return ("CONFIG_IRQ_MAP"); 5482 case I40E_VIRTCHNL_OP_ENABLE_QUEUES: 5483 return ("ENABLE_QUEUES"); 5484 case I40E_VIRTCHNL_OP_DISABLE_QUEUES: 5485 return ("DISABLE_QUEUES"); 5486 case I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS: 5487 return ("ADD_ETHER_ADDRESS"); 5488 case I40E_VIRTCHNL_OP_DEL_ETHER_ADDRESS: 5489 return ("DEL_ETHER_ADDRESS"); 5490 case I40E_VIRTCHNL_OP_ADD_VLAN: 5491 return ("ADD_VLAN"); 5492 case I40E_VIRTCHNL_OP_DEL_VLAN: 5493 return ("DEL_VLAN"); 5494 case I40E_VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE: 5495 return ("CONFIG_PROMISCUOUS_MODE"); 5496 case I40E_VIRTCHNL_OP_GET_STATS: 5497 return ("GET_STATS"); 5498 case I40E_VIRTCHNL_OP_FCOE: 5499 return ("FCOE"); 5500 case I40E_VIRTCHNL_OP_EVENT: 5501 return ("EVENT"); 5502 default: 5503 return ("UNKNOWN"); 5504 } 5505 } 5506 5507 static int 5508 ixl_vc_opcode_level(uint16_t opcode) 5509 { 5510 5511 switch (opcode) { 5512 case I40E_VIRTCHNL_OP_GET_STATS: 5513 return (10); 5514 default: 5515 return (5); 5516 } 5517 } 5518 5519 static void 5520 ixl_send_vf_msg(struct ixl_pf *pf, struct ixl_vf *vf, uint16_t op, 5521 enum i40e_status_code status, void *msg, uint16_t len) 5522 { 5523 struct i40e_hw *hw; 5524 int global_vf_id; 5525 5526 hw = &pf->hw; 5527 global_vf_id = hw->func_caps.vf_base_id + vf->vf_num; 5528 5529 I40E_VC_DEBUG(pf, ixl_vc_opcode_level(op), 5530 "Sending msg (op=%s[%d], status=%d) to VF-%d\n", 5531 ixl_vc_opcode_str(op), op, status, vf->vf_num); 5532 5533 i40e_aq_send_msg_to_vf(hw, global_vf_id, op, status, msg, len, NULL); 5534 } 5535 5536 static void 5537 ixl_send_vf_ack(struct ixl_pf *pf, struct ixl_vf *vf, uint16_t op) 5538 { 5539 5540 ixl_send_vf_msg(pf, vf, op, I40E_SUCCESS, NULL, 0); 5541 } 5542 5543 static void 5544 ixl_send_vf_nack_msg(struct ixl_pf *pf, struct ixl_vf *vf, uint16_t op, 5545 enum i40e_status_code status, const char *file, int line) 5546 { 5547 5548 I40E_VC_DEBUG(pf, 1, 5549 "Sending NACK (op=%s[%d], err=%d) to VF-%d from %s:%d\n", 5550 ixl_vc_opcode_str(op), op, status, vf->vf_num, file, line); 5551 ixl_send_vf_msg(pf, vf, op, status, NULL, 0); 5552 } 5553 5554 static void 5555 ixl_vf_version_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg, 5556 uint16_t msg_size) 5557 { 5558 struct i40e_virtchnl_version_info reply; 5559 5560 if (msg_size != sizeof(struct i40e_virtchnl_version_info)) { 5561 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_VERSION, 5562 I40E_ERR_PARAM); 5563 return; 5564 } 5565 5566 reply.major = I40E_VIRTCHNL_VERSION_MAJOR; 5567 reply.minor = I40E_VIRTCHNL_VERSION_MINOR; 5568 ixl_send_vf_msg(pf, vf, I40E_VIRTCHNL_OP_VERSION, I40E_SUCCESS, &reply, 5569 sizeof(reply)); 5570 } 5571 5572 static void 5573 ixl_vf_reset_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg, 5574 uint16_t msg_size) 5575 { 5576 5577 if (msg_size != 0) { 5578 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_RESET_VF, 5579 I40E_ERR_PARAM); 5580 return; 5581 } 5582 5583 ixl_reset_vf(pf, vf); 5584 5585 /* No response to a reset message. */ 5586 } 5587 5588 static void 5589 ixl_vf_get_resources_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg, 5590 uint16_t msg_size) 5591 { 5592 struct i40e_virtchnl_vf_resource reply; 5593 5594 if (msg_size != 0) { 5595 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_GET_VF_RESOURCES, 5596 I40E_ERR_PARAM); 5597 return; 5598 } 5599 5600 bzero(&reply, sizeof(reply)); 5601 5602 reply.vf_offload_flags = I40E_VIRTCHNL_VF_OFFLOAD_L2; 5603 5604 reply.num_vsis = 1; 5605 reply.num_queue_pairs = vf->vsi.num_queues; 5606 reply.max_vectors = pf->hw.func_caps.num_msix_vectors_vf; 5607 reply.vsi_res[0].vsi_id = vf->vsi.vsi_num; 5608 reply.vsi_res[0].vsi_type = I40E_VSI_SRIOV; 5609 reply.vsi_res[0].num_queue_pairs = vf->vsi.num_queues; 5610 memcpy(reply.vsi_res[0].default_mac_addr, vf->mac, ETHER_ADDR_LEN); 5611 5612 ixl_send_vf_msg(pf, vf, I40E_VIRTCHNL_OP_GET_VF_RESOURCES, 5613 I40E_SUCCESS, &reply, sizeof(reply)); 5614 } 5615 5616 static int 5617 ixl_vf_config_tx_queue(struct ixl_pf *pf, struct ixl_vf *vf, 5618 struct i40e_virtchnl_txq_info *info) 5619 { 5620 struct i40e_hw *hw; 5621 struct i40e_hmc_obj_txq txq; 5622 uint16_t global_queue_num, global_vf_num; 5623 enum i40e_status_code status; 5624 uint32_t qtx_ctl; 5625 5626 hw = &pf->hw; 5627 global_queue_num = vf->vsi.first_queue + info->queue_id; 5628 global_vf_num = hw->func_caps.vf_base_id + vf->vf_num; 5629 bzero(&txq, sizeof(txq)); 5630 5631 status = i40e_clear_lan_tx_queue_context(hw, global_queue_num); 5632 if (status != I40E_SUCCESS) 5633 return (EINVAL); 5634 5635 txq.base = info->dma_ring_addr / IXL_TX_CTX_BASE_UNITS; 5636 5637 txq.head_wb_ena = info->headwb_enabled; 5638 txq.head_wb_addr = info->dma_headwb_addr; 5639 txq.qlen = info->ring_len; 5640 txq.rdylist = le16_to_cpu(vf->vsi.info.qs_handle[0]); 5641 txq.rdylist_act = 0; 5642 5643 status = i40e_set_lan_tx_queue_context(hw, global_queue_num, &txq); 5644 if (status != I40E_SUCCESS) 5645 return (EINVAL); 5646 5647 qtx_ctl = I40E_QTX_CTL_VF_QUEUE | 5648 (hw->pf_id << I40E_QTX_CTL_PF_INDX_SHIFT) | 5649 (global_vf_num << I40E_QTX_CTL_VFVM_INDX_SHIFT); 5650 wr32(hw, I40E_QTX_CTL(global_queue_num), qtx_ctl); 5651 ixl_flush(hw); 5652 5653 return (0); 5654 } 5655 5656 static int 5657 ixl_vf_config_rx_queue(struct ixl_pf *pf, struct ixl_vf *vf, 5658 struct i40e_virtchnl_rxq_info *info) 5659 { 5660 struct i40e_hw *hw; 5661 struct i40e_hmc_obj_rxq rxq; 5662 uint16_t global_queue_num; 5663 enum i40e_status_code status; 5664 5665 hw = &pf->hw; 5666 global_queue_num = vf->vsi.first_queue + info->queue_id; 5667 bzero(&rxq, sizeof(rxq)); 5668 5669 if (info->databuffer_size > IXL_VF_MAX_BUFFER) 5670 return (EINVAL); 5671 5672 if (info->max_pkt_size > IXL_VF_MAX_FRAME || 5673 info->max_pkt_size < ETHER_MIN_LEN) 5674 return (EINVAL); 5675 5676 if (info->splithdr_enabled) { 5677 if (info->hdr_size > IXL_VF_MAX_HDR_BUFFER) 5678 return (EINVAL); 5679 5680 rxq.hsplit_0 = info->rx_split_pos & 5681 (I40E_HMC_OBJ_RX_HSPLIT_0_SPLIT_L2 | 5682 I40E_HMC_OBJ_RX_HSPLIT_0_SPLIT_IP | 5683 I40E_HMC_OBJ_RX_HSPLIT_0_SPLIT_TCP_UDP | 5684 I40E_HMC_OBJ_RX_HSPLIT_0_SPLIT_SCTP); 5685 rxq.hbuff = info->hdr_size >> I40E_RXQ_CTX_HBUFF_SHIFT; 5686 5687 rxq.dtype = 2; 5688 } 5689 5690 status = i40e_clear_lan_rx_queue_context(hw, global_queue_num); 5691 if (status != I40E_SUCCESS) 5692 return (EINVAL); 5693 5694 rxq.base = info->dma_ring_addr / IXL_RX_CTX_BASE_UNITS; 5695 rxq.qlen = info->ring_len; 5696 5697 rxq.dbuff = info->databuffer_size >> I40E_RXQ_CTX_DBUFF_SHIFT; 5698 5699 rxq.dsize = 1; 5700 rxq.crcstrip = 1; 5701 rxq.l2tsel = 1; 5702 5703 rxq.rxmax = info->max_pkt_size; 5704 rxq.tphrdesc_ena = 1; 5705 rxq.tphwdesc_ena = 1; 5706 rxq.tphdata_ena = 1; 5707 rxq.tphhead_ena = 1; 5708 rxq.lrxqthresh = 2; 5709 rxq.prefena = 1; 5710 5711 status = i40e_set_lan_rx_queue_context(hw, global_queue_num, &rxq); 5712 if (status != I40E_SUCCESS) 5713 return (EINVAL); 5714 5715 return (0); 5716 } 5717 5718 static void 5719 ixl_vf_config_vsi_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg, 5720 uint16_t msg_size) 5721 { 5722 struct i40e_virtchnl_vsi_queue_config_info *info; 5723 struct i40e_virtchnl_queue_pair_info *pair; 5724 int i; 5725 5726 if (msg_size < sizeof(*info)) { 5727 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES, 5728 I40E_ERR_PARAM); 5729 return; 5730 } 5731 5732 info = msg; 5733 if (info->num_queue_pairs == 0) { 5734 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES, 5735 I40E_ERR_PARAM); 5736 return; 5737 } 5738 5739 if (msg_size != sizeof(*info) + info->num_queue_pairs * sizeof(*pair)) { 5740 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES, 5741 I40E_ERR_PARAM); 5742 return; 5743 } 5744 5745 if (info->vsi_id != vf->vsi.vsi_num) { 5746 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES, 5747 I40E_ERR_PARAM); 5748 return; 5749 } 5750 5751 for (i = 0; i < info->num_queue_pairs; i++) { 5752 pair = &info->qpair[i]; 5753 5754 if (pair->txq.vsi_id != vf->vsi.vsi_num || 5755 pair->rxq.vsi_id != vf->vsi.vsi_num || 5756 pair->txq.queue_id != pair->rxq.queue_id || 5757 pair->txq.queue_id >= vf->vsi.num_queues) { 5758 5759 i40e_send_vf_nack(pf, vf, 5760 I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES, I40E_ERR_PARAM); 5761 return; 5762 } 5763 5764 if (ixl_vf_config_tx_queue(pf, vf, &pair->txq) != 0) { 5765 i40e_send_vf_nack(pf, vf, 5766 I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES, I40E_ERR_PARAM); 5767 return; 5768 } 5769 5770 if (ixl_vf_config_rx_queue(pf, vf, &pair->rxq) != 0) { 5771 i40e_send_vf_nack(pf, vf, 5772 I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES, I40E_ERR_PARAM); 5773 return; 5774 } 5775 } 5776 5777 ixl_send_vf_ack(pf, vf, I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES); 5778 } 5779 5780 static void 5781 ixl_vf_set_qctl(struct ixl_pf *pf, 5782 const struct i40e_virtchnl_vector_map *vector, 5783 enum i40e_queue_type cur_type, uint16_t cur_queue, 5784 enum i40e_queue_type *last_type, uint16_t *last_queue) 5785 { 5786 uint32_t offset, qctl; 5787 uint16_t itr_indx; 5788 5789 if (cur_type == I40E_QUEUE_TYPE_RX) { 5790 offset = I40E_QINT_RQCTL(cur_queue); 5791 itr_indx = vector->rxitr_idx; 5792 } else { 5793 offset = I40E_QINT_TQCTL(cur_queue); 5794 itr_indx = vector->txitr_idx; 5795 } 5796 5797 qctl = htole32((vector->vector_id << I40E_QINT_RQCTL_MSIX_INDX_SHIFT) | 5798 (*last_type << I40E_QINT_RQCTL_NEXTQ_TYPE_SHIFT) | 5799 (*last_queue << I40E_QINT_RQCTL_NEXTQ_INDX_SHIFT) | 5800 I40E_QINT_RQCTL_CAUSE_ENA_MASK | 5801 (itr_indx << I40E_QINT_RQCTL_ITR_INDX_SHIFT)); 5802 5803 wr32(&pf->hw, offset, qctl); 5804 5805 *last_type = cur_type; 5806 *last_queue = cur_queue; 5807 } 5808 5809 static void 5810 ixl_vf_config_vector(struct ixl_pf *pf, struct ixl_vf *vf, 5811 const struct i40e_virtchnl_vector_map *vector) 5812 { 5813 struct i40e_hw *hw; 5814 u_int qindex; 5815 enum i40e_queue_type type, last_type; 5816 uint32_t lnklst_reg; 5817 uint16_t rxq_map, txq_map, cur_queue, last_queue; 5818 5819 hw = &pf->hw; 5820 5821 rxq_map = vector->rxq_map; 5822 txq_map = vector->txq_map; 5823 5824 last_queue = IXL_END_OF_INTR_LNKLST; 5825 last_type = I40E_QUEUE_TYPE_RX; 5826 5827 /* 5828 * The datasheet says to optimize performance, RX queues and TX queues 5829 * should be interleaved in the interrupt linked list, so we process 5830 * both at once here. 5831 */ 5832 while ((rxq_map != 0) || (txq_map != 0)) { 5833 if (txq_map != 0) { 5834 qindex = ffs(txq_map) - 1; 5835 type = I40E_QUEUE_TYPE_TX; 5836 cur_queue = vf->vsi.first_queue + qindex; 5837 ixl_vf_set_qctl(pf, vector, type, cur_queue, 5838 &last_type, &last_queue); 5839 txq_map &= ~(1 << qindex); 5840 } 5841 5842 if (rxq_map != 0) { 5843 qindex = ffs(rxq_map) - 1; 5844 type = I40E_QUEUE_TYPE_RX; 5845 cur_queue = vf->vsi.first_queue + qindex; 5846 ixl_vf_set_qctl(pf, vector, type, cur_queue, 5847 &last_type, &last_queue); 5848 rxq_map &= ~(1 << qindex); 5849 } 5850 } 5851 5852 if (vector->vector_id == 0) 5853 lnklst_reg = I40E_VPINT_LNKLST0(vf->vf_num); 5854 else 5855 lnklst_reg = IXL_VPINT_LNKLSTN_REG(hw, vector->vector_id, 5856 vf->vf_num); 5857 wr32(hw, lnklst_reg, 5858 (last_queue << I40E_VPINT_LNKLST0_FIRSTQ_INDX_SHIFT) | 5859 (last_type << I40E_VPINT_LNKLST0_FIRSTQ_TYPE_SHIFT)); 5860 5861 ixl_flush(hw); 5862 } 5863 5864 static void 5865 ixl_vf_config_irq_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg, 5866 uint16_t msg_size) 5867 { 5868 struct i40e_virtchnl_irq_map_info *map; 5869 struct i40e_virtchnl_vector_map *vector; 5870 struct i40e_hw *hw; 5871 int i, largest_txq, largest_rxq; 5872 5873 hw = &pf->hw; 5874 5875 if (msg_size < sizeof(*map)) { 5876 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP, 5877 I40E_ERR_PARAM); 5878 return; 5879 } 5880 5881 map = msg; 5882 if (map->num_vectors == 0) { 5883 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP, 5884 I40E_ERR_PARAM); 5885 return; 5886 } 5887 5888 if (msg_size != sizeof(*map) + map->num_vectors * sizeof(*vector)) { 5889 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP, 5890 I40E_ERR_PARAM); 5891 return; 5892 } 5893 5894 for (i = 0; i < map->num_vectors; i++) { 5895 vector = &map->vecmap[i]; 5896 5897 if ((vector->vector_id >= hw->func_caps.num_msix_vectors_vf) || 5898 vector->vsi_id != vf->vsi.vsi_num) { 5899 i40e_send_vf_nack(pf, vf, 5900 I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP, I40E_ERR_PARAM); 5901 return; 5902 } 5903 5904 if (vector->rxq_map != 0) { 5905 largest_rxq = fls(vector->rxq_map) - 1; 5906 if (largest_rxq >= vf->vsi.num_queues) { 5907 i40e_send_vf_nack(pf, vf, 5908 I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP, 5909 I40E_ERR_PARAM); 5910 return; 5911 } 5912 } 5913 5914 if (vector->txq_map != 0) { 5915 largest_txq = fls(vector->txq_map) - 1; 5916 if (largest_txq >= vf->vsi.num_queues) { 5917 i40e_send_vf_nack(pf, vf, 5918 I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP, 5919 I40E_ERR_PARAM); 5920 return; 5921 } 5922 } 5923 5924 if (vector->rxitr_idx > IXL_MAX_ITR_IDX || 5925 vector->txitr_idx > IXL_MAX_ITR_IDX) { 5926 i40e_send_vf_nack(pf, vf, 5927 I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP, 5928 I40E_ERR_PARAM); 5929 return; 5930 } 5931 5932 ixl_vf_config_vector(pf, vf, vector); 5933 } 5934 5935 ixl_send_vf_ack(pf, vf, I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP); 5936 } 5937 5938 static void 5939 ixl_vf_enable_queues_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg, 5940 uint16_t msg_size) 5941 { 5942 struct i40e_virtchnl_queue_select *select; 5943 int error; 5944 5945 if (msg_size != sizeof(*select)) { 5946 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ENABLE_QUEUES, 5947 I40E_ERR_PARAM); 5948 return; 5949 } 5950 5951 select = msg; 5952 if (select->vsi_id != vf->vsi.vsi_num || 5953 select->rx_queues == 0 || select->tx_queues == 0) { 5954 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ENABLE_QUEUES, 5955 I40E_ERR_PARAM); 5956 return; 5957 } 5958 5959 error = ixl_enable_rings(&vf->vsi); 5960 if (error) { 5961 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ENABLE_QUEUES, 5962 I40E_ERR_TIMEOUT); 5963 return; 5964 } 5965 5966 ixl_send_vf_ack(pf, vf, I40E_VIRTCHNL_OP_ENABLE_QUEUES); 5967 } 5968 5969 static void 5970 ixl_vf_disable_queues_msg(struct ixl_pf *pf, struct ixl_vf *vf, 5971 void *msg, uint16_t msg_size) 5972 { 5973 struct i40e_virtchnl_queue_select *select; 5974 int error; 5975 5976 if (msg_size != sizeof(*select)) { 5977 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_DISABLE_QUEUES, 5978 I40E_ERR_PARAM); 5979 return; 5980 } 5981 5982 select = msg; 5983 if (select->vsi_id != vf->vsi.vsi_num || 5984 select->rx_queues == 0 || select->tx_queues == 0) { 5985 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_DISABLE_QUEUES, 5986 I40E_ERR_PARAM); 5987 return; 5988 } 5989 5990 error = ixl_disable_rings(&vf->vsi); 5991 if (error) { 5992 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_DISABLE_QUEUES, 5993 I40E_ERR_TIMEOUT); 5994 return; 5995 } 5996 5997 ixl_send_vf_ack(pf, vf, I40E_VIRTCHNL_OP_DISABLE_QUEUES); 5998 } 5999 6000 static boolean_t 6001 ixl_zero_mac(const uint8_t *addr) 6002 { 6003 uint8_t zero[ETHER_ADDR_LEN] = {0, 0, 0, 0, 0, 0}; 6004 6005 return (cmp_etheraddr(addr, zero)); 6006 } 6007 6008 static boolean_t 6009 ixl_bcast_mac(const uint8_t *addr) 6010 { 6011 6012 return (cmp_etheraddr(addr, ixl_bcast_addr)); 6013 } 6014 6015 static int 6016 ixl_vf_mac_valid(struct ixl_vf *vf, const uint8_t *addr) 6017 { 6018 6019 if (ixl_zero_mac(addr) || ixl_bcast_mac(addr)) 6020 return (EINVAL); 6021 6022 /* 6023 * If the VF is not allowed to change its MAC address, don't let it 6024 * set a MAC filter for an address that is not a multicast address and 6025 * is not its assigned MAC. 6026 */ 6027 if (!(vf->vf_flags & VF_FLAG_SET_MAC_CAP) && 6028 !(ETHER_IS_MULTICAST(addr) || cmp_etheraddr(addr, vf->mac))) 6029 return (EPERM); 6030 6031 return (0); 6032 } 6033 6034 static void 6035 ixl_vf_add_mac_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg, 6036 uint16_t msg_size) 6037 { 6038 struct i40e_virtchnl_ether_addr_list *addr_list; 6039 struct i40e_virtchnl_ether_addr *addr; 6040 struct ixl_vsi *vsi; 6041 int i; 6042 size_t expected_size; 6043 6044 vsi = &vf->vsi; 6045 6046 if (msg_size < sizeof(*addr_list)) { 6047 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS, 6048 I40E_ERR_PARAM); 6049 return; 6050 } 6051 6052 addr_list = msg; 6053 expected_size = sizeof(*addr_list) + 6054 addr_list->num_elements * sizeof(*addr); 6055 6056 if (addr_list->num_elements == 0 || 6057 addr_list->vsi_id != vsi->vsi_num || 6058 msg_size != expected_size) { 6059 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS, 6060 I40E_ERR_PARAM); 6061 return; 6062 } 6063 6064 for (i = 0; i < addr_list->num_elements; i++) { 6065 if (ixl_vf_mac_valid(vf, addr_list->list[i].addr) != 0) { 6066 i40e_send_vf_nack(pf, vf, 6067 I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS, I40E_ERR_PARAM); 6068 return; 6069 } 6070 } 6071 6072 for (i = 0; i < addr_list->num_elements; i++) { 6073 addr = &addr_list->list[i]; 6074 ixl_add_filter(vsi, addr->addr, IXL_VLAN_ANY); 6075 } 6076 6077 ixl_send_vf_ack(pf, vf, I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS); 6078 } 6079 6080 static void 6081 ixl_vf_del_mac_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg, 6082 uint16_t msg_size) 6083 { 6084 struct i40e_virtchnl_ether_addr_list *addr_list; 6085 struct i40e_virtchnl_ether_addr *addr; 6086 size_t expected_size; 6087 int i; 6088 6089 if (msg_size < sizeof(*addr_list)) { 6090 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS, 6091 I40E_ERR_PARAM); 6092 return; 6093 } 6094 6095 addr_list = msg; 6096 expected_size = sizeof(*addr_list) + 6097 addr_list->num_elements * sizeof(*addr); 6098 6099 if (addr_list->num_elements == 0 || 6100 addr_list->vsi_id != vf->vsi.vsi_num || 6101 msg_size != expected_size) { 6102 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS, 6103 I40E_ERR_PARAM); 6104 return; 6105 } 6106 6107 for (i = 0; i < addr_list->num_elements; i++) { 6108 addr = &addr_list->list[i]; 6109 if (ixl_zero_mac(addr->addr) || ixl_bcast_mac(addr->addr)) { 6110 i40e_send_vf_nack(pf, vf, 6111 I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS, I40E_ERR_PARAM); 6112 return; 6113 } 6114 } 6115 6116 for (i = 0; i < addr_list->num_elements; i++) { 6117 addr = &addr_list->list[i]; 6118 ixl_del_filter(&vf->vsi, addr->addr, IXL_VLAN_ANY); 6119 } 6120 6121 ixl_send_vf_ack(pf, vf, I40E_VIRTCHNL_OP_DEL_ETHER_ADDRESS); 6122 } 6123 6124 static enum i40e_status_code 6125 ixl_vf_enable_vlan_strip(struct ixl_pf *pf, struct ixl_vf *vf) 6126 { 6127 struct i40e_vsi_context vsi_ctx; 6128 6129 vsi_ctx.seid = vf->vsi.seid; 6130 6131 bzero(&vsi_ctx.info, sizeof(vsi_ctx.info)); 6132 vsi_ctx.info.valid_sections = htole16(I40E_AQ_VSI_PROP_VLAN_VALID); 6133 vsi_ctx.info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_ALL | 6134 I40E_AQ_VSI_PVLAN_EMOD_STR_BOTH; 6135 return (i40e_aq_update_vsi_params(&pf->hw, &vsi_ctx, NULL)); 6136 } 6137 6138 static void 6139 ixl_vf_add_vlan_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg, 6140 uint16_t msg_size) 6141 { 6142 struct i40e_virtchnl_vlan_filter_list *filter_list; 6143 enum i40e_status_code code; 6144 size_t expected_size; 6145 int i; 6146 6147 if (msg_size < sizeof(*filter_list)) { 6148 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_VLAN, 6149 I40E_ERR_PARAM); 6150 return; 6151 } 6152 6153 filter_list = msg; 6154 expected_size = sizeof(*filter_list) + 6155 filter_list->num_elements * sizeof(uint16_t); 6156 if (filter_list->num_elements == 0 || 6157 filter_list->vsi_id != vf->vsi.vsi_num || 6158 msg_size != expected_size) { 6159 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_VLAN, 6160 I40E_ERR_PARAM); 6161 return; 6162 } 6163 6164 if (!(vf->vf_flags & VF_FLAG_VLAN_CAP)) { 6165 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_VLAN, 6166 I40E_ERR_PARAM); 6167 return; 6168 } 6169 6170 for (i = 0; i < filter_list->num_elements; i++) { 6171 if (filter_list->vlan_id[i] > EVL_VLID_MASK) { 6172 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_VLAN, 6173 I40E_ERR_PARAM); 6174 return; 6175 } 6176 } 6177 6178 code = ixl_vf_enable_vlan_strip(pf, vf); 6179 if (code != I40E_SUCCESS) { 6180 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_VLAN, 6181 I40E_ERR_PARAM); 6182 } 6183 6184 for (i = 0; i < filter_list->num_elements; i++) 6185 ixl_add_filter(&vf->vsi, vf->mac, filter_list->vlan_id[i]); 6186 6187 ixl_send_vf_ack(pf, vf, I40E_VIRTCHNL_OP_ADD_VLAN); 6188 } 6189 6190 static void 6191 ixl_vf_del_vlan_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg, 6192 uint16_t msg_size) 6193 { 6194 struct i40e_virtchnl_vlan_filter_list *filter_list; 6195 int i; 6196 size_t expected_size; 6197 6198 if (msg_size < sizeof(*filter_list)) { 6199 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_DEL_VLAN, 6200 I40E_ERR_PARAM); 6201 return; 6202 } 6203 6204 filter_list = msg; 6205 expected_size = sizeof(*filter_list) + 6206 filter_list->num_elements * sizeof(uint16_t); 6207 if (filter_list->num_elements == 0 || 6208 filter_list->vsi_id != vf->vsi.vsi_num || 6209 msg_size != expected_size) { 6210 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_DEL_VLAN, 6211 I40E_ERR_PARAM); 6212 return; 6213 } 6214 6215 for (i = 0; i < filter_list->num_elements; i++) { 6216 if (filter_list->vlan_id[i] > EVL_VLID_MASK) { 6217 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_VLAN, 6218 I40E_ERR_PARAM); 6219 return; 6220 } 6221 } 6222 6223 if (!(vf->vf_flags & VF_FLAG_VLAN_CAP)) { 6224 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_VLAN, 6225 I40E_ERR_PARAM); 6226 return; 6227 } 6228 6229 for (i = 0; i < filter_list->num_elements; i++) 6230 ixl_del_filter(&vf->vsi, vf->mac, filter_list->vlan_id[i]); 6231 6232 ixl_send_vf_ack(pf, vf, I40E_VIRTCHNL_OP_DEL_VLAN); 6233 } 6234 6235 static void 6236 ixl_vf_config_promisc_msg(struct ixl_pf *pf, struct ixl_vf *vf, 6237 void *msg, uint16_t msg_size) 6238 { 6239 struct i40e_virtchnl_promisc_info *info; 6240 enum i40e_status_code code; 6241 6242 if (msg_size != sizeof(*info)) { 6243 i40e_send_vf_nack(pf, vf, 6244 I40E_VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE, I40E_ERR_PARAM); 6245 return; 6246 } 6247 6248 if (!vf->vf_flags & VF_FLAG_PROMISC_CAP) { 6249 i40e_send_vf_nack(pf, vf, 6250 I40E_VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE, I40E_ERR_PARAM); 6251 return; 6252 } 6253 6254 info = msg; 6255 if (info->vsi_id != vf->vsi.vsi_num) { 6256 i40e_send_vf_nack(pf, vf, 6257 I40E_VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE, I40E_ERR_PARAM); 6258 return; 6259 } 6260 6261 code = i40e_aq_set_vsi_unicast_promiscuous(&pf->hw, info->vsi_id, 6262 info->flags & I40E_FLAG_VF_UNICAST_PROMISC, NULL); 6263 if (code != I40E_SUCCESS) { 6264 i40e_send_vf_nack(pf, vf, 6265 I40E_VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE, code); 6266 return; 6267 } 6268 6269 code = i40e_aq_set_vsi_multicast_promiscuous(&pf->hw, info->vsi_id, 6270 info->flags & I40E_FLAG_VF_MULTICAST_PROMISC, NULL); 6271 if (code != I40E_SUCCESS) { 6272 i40e_send_vf_nack(pf, vf, 6273 I40E_VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE, code); 6274 return; 6275 } 6276 6277 ixl_send_vf_ack(pf, vf, I40E_VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE); 6278 } 6279 6280 static void 6281 ixl_vf_get_stats_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg, 6282 uint16_t msg_size) 6283 { 6284 struct i40e_virtchnl_queue_select *queue; 6285 6286 if (msg_size != sizeof(*queue)) { 6287 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_GET_STATS, 6288 I40E_ERR_PARAM); 6289 return; 6290 } 6291 6292 queue = msg; 6293 if (queue->vsi_id != vf->vsi.vsi_num) { 6294 i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_GET_STATS, 6295 I40E_ERR_PARAM); 6296 return; 6297 } 6298 6299 ixl_update_eth_stats(&vf->vsi); 6300 6301 ixl_send_vf_msg(pf, vf, I40E_VIRTCHNL_OP_GET_STATS, 6302 I40E_SUCCESS, &vf->vsi.eth_stats, sizeof(vf->vsi.eth_stats)); 6303 } 6304 6305 static void 6306 ixl_handle_vf_msg(struct ixl_pf *pf, struct i40e_arq_event_info *event) 6307 { 6308 struct ixl_vf *vf; 6309 void *msg; 6310 uint16_t vf_num, msg_size; 6311 uint32_t opcode; 6312 6313 vf_num = le16toh(event->desc.retval) - pf->hw.func_caps.vf_base_id; 6314 opcode = le32toh(event->desc.cookie_high); 6315 6316 if (vf_num >= pf->num_vfs) { 6317 device_printf(pf->dev, "Got msg from illegal VF: %d\n", vf_num); 6318 return; 6319 } 6320 6321 vf = &pf->vfs[vf_num]; 6322 msg = event->msg_buf; 6323 msg_size = event->msg_len; 6324 6325 I40E_VC_DEBUG(pf, ixl_vc_opcode_level(opcode), 6326 "Got msg %s(%d) from VF-%d of size %d\n", 6327 ixl_vc_opcode_str(opcode), opcode, vf_num, msg_size); 6328 6329 switch (opcode) { 6330 case I40E_VIRTCHNL_OP_VERSION: 6331 ixl_vf_version_msg(pf, vf, msg, msg_size); 6332 break; 6333 case I40E_VIRTCHNL_OP_RESET_VF: 6334 ixl_vf_reset_msg(pf, vf, msg, msg_size); 6335 break; 6336 case I40E_VIRTCHNL_OP_GET_VF_RESOURCES: 6337 ixl_vf_get_resources_msg(pf, vf, msg, msg_size); 6338 break; 6339 case I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES: 6340 ixl_vf_config_vsi_msg(pf, vf, msg, msg_size); 6341 break; 6342 case I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP: 6343 ixl_vf_config_irq_msg(pf, vf, msg, msg_size); 6344 break; 6345 case I40E_VIRTCHNL_OP_ENABLE_QUEUES: 6346 ixl_vf_enable_queues_msg(pf, vf, msg, msg_size); 6347 break; 6348 case I40E_VIRTCHNL_OP_DISABLE_QUEUES: 6349 ixl_vf_disable_queues_msg(pf, vf, msg, msg_size); 6350 break; 6351 case I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS: 6352 ixl_vf_add_mac_msg(pf, vf, msg, msg_size); 6353 break; 6354 case I40E_VIRTCHNL_OP_DEL_ETHER_ADDRESS: 6355 ixl_vf_del_mac_msg(pf, vf, msg, msg_size); 6356 break; 6357 case I40E_VIRTCHNL_OP_ADD_VLAN: 6358 ixl_vf_add_vlan_msg(pf, vf, msg, msg_size); 6359 break; 6360 case I40E_VIRTCHNL_OP_DEL_VLAN: 6361 ixl_vf_del_vlan_msg(pf, vf, msg, msg_size); 6362 break; 6363 case I40E_VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE: 6364 ixl_vf_config_promisc_msg(pf, vf, msg, msg_size); 6365 break; 6366 case I40E_VIRTCHNL_OP_GET_STATS: 6367 ixl_vf_get_stats_msg(pf, vf, msg, msg_size); 6368 break; 6369 6370 /* These two opcodes have been superseded by CONFIG_VSI_QUEUES. */ 6371 case I40E_VIRTCHNL_OP_CONFIG_TX_QUEUE: 6372 case I40E_VIRTCHNL_OP_CONFIG_RX_QUEUE: 6373 default: 6374 i40e_send_vf_nack(pf, vf, opcode, I40E_ERR_NOT_IMPLEMENTED); 6375 break; 6376 } 6377 } 6378 6379 /* Handle any VFs that have reset themselves via a Function Level Reset(FLR). */ 6380 static void 6381 ixl_handle_vflr(void *arg, int pending) 6382 { 6383 struct ixl_pf *pf; 6384 struct i40e_hw *hw; 6385 uint16_t global_vf_num; 6386 uint32_t vflrstat_index, vflrstat_mask, vflrstat, icr0; 6387 int i; 6388 6389 pf = arg; 6390 hw = &pf->hw; 6391 6392 IXL_PF_LOCK(pf); 6393 for (i = 0; i < pf->num_vfs; i++) { 6394 global_vf_num = hw->func_caps.vf_base_id + i; 6395 6396 vflrstat_index = IXL_GLGEN_VFLRSTAT_INDEX(global_vf_num); 6397 vflrstat_mask = IXL_GLGEN_VFLRSTAT_MASK(global_vf_num); 6398 vflrstat = rd32(hw, I40E_GLGEN_VFLRSTAT(vflrstat_index)); 6399 if (vflrstat & vflrstat_mask) { 6400 wr32(hw, I40E_GLGEN_VFLRSTAT(vflrstat_index), 6401 vflrstat_mask); 6402 6403 ixl_reinit_vf(pf, &pf->vfs[i]); 6404 } 6405 } 6406 6407 icr0 = rd32(hw, I40E_PFINT_ICR0_ENA); 6408 icr0 |= I40E_PFINT_ICR0_ENA_VFLR_MASK; 6409 wr32(hw, I40E_PFINT_ICR0_ENA, icr0); 6410 ixl_flush(hw); 6411 6412 IXL_PF_UNLOCK(pf); 6413 } 6414 6415 static int 6416 ixl_adminq_err_to_errno(enum i40e_admin_queue_err err) 6417 { 6418 6419 switch (err) { 6420 case I40E_AQ_RC_EPERM: 6421 return (EPERM); 6422 case I40E_AQ_RC_ENOENT: 6423 return (ENOENT); 6424 case I40E_AQ_RC_ESRCH: 6425 return (ESRCH); 6426 case I40E_AQ_RC_EINTR: 6427 return (EINTR); 6428 case I40E_AQ_RC_EIO: 6429 return (EIO); 6430 case I40E_AQ_RC_ENXIO: 6431 return (ENXIO); 6432 case I40E_AQ_RC_E2BIG: 6433 return (E2BIG); 6434 case I40E_AQ_RC_EAGAIN: 6435 return (EAGAIN); 6436 case I40E_AQ_RC_ENOMEM: 6437 return (ENOMEM); 6438 case I40E_AQ_RC_EACCES: 6439 return (EACCES); 6440 case I40E_AQ_RC_EFAULT: 6441 return (EFAULT); 6442 case I40E_AQ_RC_EBUSY: 6443 return (EBUSY); 6444 case I40E_AQ_RC_EEXIST: 6445 return (EEXIST); 6446 case I40E_AQ_RC_EINVAL: 6447 return (EINVAL); 6448 case I40E_AQ_RC_ENOTTY: 6449 return (ENOTTY); 6450 case I40E_AQ_RC_ENOSPC: 6451 return (ENOSPC); 6452 case I40E_AQ_RC_ENOSYS: 6453 return (ENOSYS); 6454 case I40E_AQ_RC_ERANGE: 6455 return (ERANGE); 6456 case I40E_AQ_RC_EFLUSHED: 6457 return (EINVAL); /* No exact equivalent in errno.h */ 6458 case I40E_AQ_RC_BAD_ADDR: 6459 return (EFAULT); 6460 case I40E_AQ_RC_EMODE: 6461 return (EPERM); 6462 case I40E_AQ_RC_EFBIG: 6463 return (EFBIG); 6464 default: 6465 return (EINVAL); 6466 } 6467 } 6468 6469 static int 6470 ixl_init_iov(device_t dev, uint16_t num_vfs, const nvlist_t *params) 6471 { 6472 struct ixl_pf *pf; 6473 struct i40e_hw *hw; 6474 struct ixl_vsi *pf_vsi; 6475 enum i40e_status_code ret; 6476 int i, error; 6477 6478 pf = device_get_softc(dev); 6479 hw = &pf->hw; 6480 pf_vsi = &pf->vsi; 6481 6482 IXL_PF_LOCK(pf); 6483 pf->vfs = malloc(sizeof(struct ixl_vf) * num_vfs, M_IXL, M_NOWAIT | 6484 M_ZERO); 6485 6486 if (pf->vfs == NULL) { 6487 error = ENOMEM; 6488 goto fail; 6489 } 6490 6491 for (i = 0; i < num_vfs; i++) 6492 sysctl_ctx_init(&pf->vfs[i].ctx); 6493 6494 ret = i40e_aq_add_veb(hw, pf_vsi->uplink_seid, pf_vsi->seid, 6495 1, FALSE, FALSE, &pf->veb_seid, NULL); 6496 if (ret != I40E_SUCCESS) { 6497 error = ixl_adminq_err_to_errno(hw->aq.asq_last_status); 6498 device_printf(dev, "add_veb failed; code=%d error=%d", ret, 6499 error); 6500 goto fail; 6501 } 6502 6503 ixl_configure_msix(pf); 6504 ixl_enable_adminq(hw); 6505 6506 pf->num_vfs = num_vfs; 6507 IXL_PF_UNLOCK(pf); 6508 return (0); 6509 6510 fail: 6511 free(pf->vfs, M_IXL); 6512 pf->vfs = NULL; 6513 IXL_PF_UNLOCK(pf); 6514 return (error); 6515 } 6516 6517 static void 6518 ixl_uninit_iov(device_t dev) 6519 { 6520 struct ixl_pf *pf; 6521 struct i40e_hw *hw; 6522 struct ixl_vsi *vsi; 6523 struct ifnet *ifp; 6524 struct ixl_vf *vfs; 6525 int i, num_vfs; 6526 6527 pf = device_get_softc(dev); 6528 hw = &pf->hw; 6529 vsi = &pf->vsi; 6530 ifp = vsi->ifp; 6531 6532 IXL_PF_LOCK(pf); 6533 for (i = 0; i < pf->num_vfs; i++) { 6534 if (pf->vfs[i].vsi.seid != 0) 6535 i40e_aq_delete_element(hw, pf->vfs[i].vsi.seid, NULL); 6536 } 6537 6538 if (pf->veb_seid != 0) { 6539 i40e_aq_delete_element(hw, pf->veb_seid, NULL); 6540 pf->veb_seid = 0; 6541 } 6542 6543 if ((if_getdrvflags(ifp) & IFF_DRV_RUNNING) == 0) 6544 ixl_disable_intr(vsi); 6545 6546 vfs = pf->vfs; 6547 num_vfs = pf->num_vfs; 6548 6549 pf->vfs = NULL; 6550 pf->num_vfs = 0; 6551 IXL_PF_UNLOCK(pf); 6552 6553 /* Do this after the unlock as sysctl_ctx_free might sleep. */ 6554 for (i = 0; i < num_vfs; i++) 6555 sysctl_ctx_free(&vfs[i].ctx); 6556 free(vfs, M_IXL); 6557 } 6558 6559 static int 6560 ixl_add_vf(device_t dev, uint16_t vfnum, const nvlist_t *params) 6561 { 6562 char sysctl_name[QUEUE_NAME_LEN]; 6563 struct ixl_pf *pf; 6564 struct ixl_vf *vf; 6565 const void *mac; 6566 size_t size; 6567 int error; 6568 6569 pf = device_get_softc(dev); 6570 vf = &pf->vfs[vfnum]; 6571 6572 IXL_PF_LOCK(pf); 6573 vf->vf_num = vfnum; 6574 6575 vf->vsi.back = pf; 6576 vf->vf_flags = VF_FLAG_ENABLED; 6577 SLIST_INIT(&vf->vsi.ftl); 6578 6579 error = ixl_vf_setup_vsi(pf, vf); 6580 if (error != 0) 6581 goto out; 6582 6583 if (nvlist_exists_binary(params, "mac-addr")) { 6584 mac = nvlist_get_binary(params, "mac-addr", &size); 6585 bcopy(mac, vf->mac, ETHER_ADDR_LEN); 6586 6587 if (nvlist_get_bool(params, "allow-set-mac")) 6588 vf->vf_flags |= VF_FLAG_SET_MAC_CAP; 6589 } else 6590 /* 6591 * If the administrator has not specified a MAC address then 6592 * we must allow the VF to choose one. 6593 */ 6594 vf->vf_flags |= VF_FLAG_SET_MAC_CAP; 6595 6596 if (nvlist_get_bool(params, "mac-anti-spoof")) 6597 vf->vf_flags |= VF_FLAG_MAC_ANTI_SPOOF; 6598 6599 if (nvlist_get_bool(params, "allow-promisc")) 6600 vf->vf_flags |= VF_FLAG_PROMISC_CAP; 6601 6602 vf->vf_flags |= VF_FLAG_VLAN_CAP; 6603 6604 ixl_reset_vf(pf, vf); 6605 out: 6606 IXL_PF_UNLOCK(pf); 6607 if (error == 0) { 6608 snprintf(sysctl_name, sizeof(sysctl_name), "vf%d", vfnum); 6609 ixl_add_vsi_sysctls(pf, &vf->vsi, &vf->ctx, sysctl_name); 6610 } 6611 6612 return (error); 6613 } 6614 #endif /* PCI_IOV */ 6615