1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright(c) 2013 - 2021 Intel Corporation. */ 3 4 #include <generated/utsrelease.h> 5 #include <linux/crash_dump.h> 6 #include <linux/if_bridge.h> 7 #include <linux/if_macvlan.h> 8 #include <linux/module.h> 9 #include <net/pkt_cls.h> 10 #include <net/xdp_sock_drv.h> 11 12 /* Local includes */ 13 #include "i40e.h" 14 #include "i40e_devids.h" 15 #include "i40e_diag.h" 16 #include "i40e_lan_hmc.h" 17 #include "i40e_virtchnl_pf.h" 18 #include "i40e_xsk.h" 19 20 /* All i40e tracepoints are defined by the include below, which 21 * must be included exactly once across the whole kernel with 22 * CREATE_TRACE_POINTS defined 23 */ 24 #define CREATE_TRACE_POINTS 25 #include "i40e_trace.h" 26 27 const char i40e_driver_name[] = "i40e"; 28 static const char i40e_driver_string[] = 29 "Intel(R) Ethernet Connection XL710 Network Driver"; 30 31 static const char i40e_copyright[] = "Copyright (c) 2013 - 2019 Intel Corporation."; 32 33 /* a bit of forward declarations */ 34 static void i40e_vsi_reinit_locked(struct i40e_vsi *vsi); 35 static void i40e_handle_reset_warning(struct i40e_pf *pf, bool lock_acquired); 36 static int i40e_add_vsi(struct i40e_vsi *vsi); 37 static int i40e_add_veb(struct i40e_veb *veb, struct i40e_vsi *vsi); 38 static int i40e_setup_pf_switch(struct i40e_pf *pf, bool reinit, bool lock_acquired); 39 static int i40e_setup_misc_vector(struct i40e_pf *pf); 40 static void i40e_determine_queue_usage(struct i40e_pf *pf); 41 static int i40e_setup_pf_filter_control(struct i40e_pf *pf); 42 static void i40e_prep_for_reset(struct i40e_pf *pf); 43 static void i40e_reset_and_rebuild(struct i40e_pf *pf, bool reinit, 44 bool lock_acquired); 45 static int i40e_reset(struct i40e_pf *pf); 46 static void i40e_rebuild(struct i40e_pf *pf, bool reinit, bool lock_acquired); 47 static int i40e_setup_misc_vector_for_recovery_mode(struct i40e_pf *pf); 48 static int i40e_restore_interrupt_scheme(struct i40e_pf *pf); 49 static bool i40e_check_recovery_mode(struct i40e_pf *pf); 50 static int i40e_init_recovery_mode(struct i40e_pf *pf, struct i40e_hw *hw); 51 static void i40e_fdir_sb_setup(struct i40e_pf *pf); 52 static int i40e_veb_get_bw_info(struct i40e_veb *veb); 53 static int i40e_get_capabilities(struct i40e_pf *pf, 54 enum i40e_admin_queue_opc list_type); 55 static bool i40e_is_total_port_shutdown_enabled(struct i40e_pf *pf); 56 57 /* i40e_pci_tbl - PCI Device ID Table 58 * 59 * Last entry must be all 0s 60 * 61 * { Vendor ID, Device ID, SubVendor ID, SubDevice ID, 62 * Class, Class Mask, private data (not used) } 63 */ 64 static const struct pci_device_id i40e_pci_tbl[] = { 65 {PCI_VDEVICE(INTEL, I40E_DEV_ID_SFP_XL710), 0}, 66 {PCI_VDEVICE(INTEL, I40E_DEV_ID_QEMU), 0}, 67 {PCI_VDEVICE(INTEL, I40E_DEV_ID_KX_B), 0}, 68 {PCI_VDEVICE(INTEL, I40E_DEV_ID_KX_C), 0}, 69 {PCI_VDEVICE(INTEL, I40E_DEV_ID_QSFP_A), 0}, 70 {PCI_VDEVICE(INTEL, I40E_DEV_ID_QSFP_B), 0}, 71 {PCI_VDEVICE(INTEL, I40E_DEV_ID_QSFP_C), 0}, 72 {PCI_VDEVICE(INTEL, I40E_DEV_ID_1G_BASE_T_BC), 0}, 73 {PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_BASE_T), 0}, 74 {PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_BASE_T4), 0}, 75 {PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_BASE_T_BC), 0}, 76 {PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_SFP), 0}, 77 {PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_B), 0}, 78 {PCI_VDEVICE(INTEL, I40E_DEV_ID_KX_X722), 0}, 79 {PCI_VDEVICE(INTEL, I40E_DEV_ID_QSFP_X722), 0}, 80 {PCI_VDEVICE(INTEL, I40E_DEV_ID_SFP_X722), 0}, 81 {PCI_VDEVICE(INTEL, I40E_DEV_ID_1G_BASE_T_X722), 0}, 82 {PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_BASE_T_X722), 0}, 83 {PCI_VDEVICE(INTEL, I40E_DEV_ID_SFP_I_X722), 0}, 84 {PCI_VDEVICE(INTEL, I40E_DEV_ID_SFP_X722_A), 0}, 85 {PCI_VDEVICE(INTEL, I40E_DEV_ID_20G_KR2), 0}, 86 {PCI_VDEVICE(INTEL, I40E_DEV_ID_20G_KR2_A), 0}, 87 {PCI_VDEVICE(INTEL, I40E_DEV_ID_X710_N3000), 0}, 88 {PCI_VDEVICE(INTEL, I40E_DEV_ID_XXV710_N3000), 0}, 89 {PCI_VDEVICE(INTEL, I40E_DEV_ID_25G_B), 0}, 90 {PCI_VDEVICE(INTEL, I40E_DEV_ID_25G_SFP28), 0}, 91 /* required last entry */ 92 {0, } 93 }; 94 MODULE_DEVICE_TABLE(pci, i40e_pci_tbl); 95 96 #define I40E_MAX_VF_COUNT 128 97 static int debug = -1; 98 module_param(debug, uint, 0); 99 MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all), Debug mask (0x8XXXXXXX)"); 100 101 MODULE_AUTHOR("Intel Corporation, <e1000-devel@lists.sourceforge.net>"); 102 MODULE_DESCRIPTION("Intel(R) Ethernet Connection XL710 Network Driver"); 103 MODULE_IMPORT_NS(LIBIE); 104 MODULE_LICENSE("GPL v2"); 105 106 static struct workqueue_struct *i40e_wq; 107 108 static void netdev_hw_addr_refcnt(struct i40e_mac_filter *f, 109 struct net_device *netdev, int delta) 110 { 111 struct netdev_hw_addr_list *ha_list; 112 struct netdev_hw_addr *ha; 113 114 if (!f || !netdev) 115 return; 116 117 if (is_unicast_ether_addr(f->macaddr) || is_link_local_ether_addr(f->macaddr)) 118 ha_list = &netdev->uc; 119 else 120 ha_list = &netdev->mc; 121 122 netdev_hw_addr_list_for_each(ha, ha_list) { 123 if (ether_addr_equal(ha->addr, f->macaddr)) { 124 ha->refcount += delta; 125 if (ha->refcount <= 0) 126 ha->refcount = 1; 127 break; 128 } 129 } 130 } 131 132 /** 133 * i40e_hw_to_dev - get device pointer from the hardware structure 134 * @hw: pointer to the device HW structure 135 **/ 136 struct device *i40e_hw_to_dev(struct i40e_hw *hw) 137 { 138 struct i40e_pf *pf = i40e_hw_to_pf(hw); 139 140 return &pf->pdev->dev; 141 } 142 143 /** 144 * i40e_allocate_dma_mem - OS specific memory alloc for shared code 145 * @hw: pointer to the HW structure 146 * @mem: ptr to mem struct to fill out 147 * @size: size of memory requested 148 * @alignment: what to align the allocation to 149 **/ 150 int i40e_allocate_dma_mem(struct i40e_hw *hw, struct i40e_dma_mem *mem, 151 u64 size, u32 alignment) 152 { 153 struct i40e_pf *pf = i40e_hw_to_pf(hw); 154 155 mem->size = ALIGN(size, alignment); 156 mem->va = dma_alloc_coherent(&pf->pdev->dev, mem->size, &mem->pa, 157 GFP_KERNEL); 158 if (!mem->va) 159 return -ENOMEM; 160 161 return 0; 162 } 163 164 /** 165 * i40e_free_dma_mem - OS specific memory free for shared code 166 * @hw: pointer to the HW structure 167 * @mem: ptr to mem struct to free 168 **/ 169 int i40e_free_dma_mem(struct i40e_hw *hw, struct i40e_dma_mem *mem) 170 { 171 struct i40e_pf *pf = i40e_hw_to_pf(hw); 172 173 dma_free_coherent(&pf->pdev->dev, mem->size, mem->va, mem->pa); 174 mem->va = NULL; 175 mem->pa = 0; 176 mem->size = 0; 177 178 return 0; 179 } 180 181 /** 182 * i40e_allocate_virt_mem - OS specific memory alloc for shared code 183 * @hw: pointer to the HW structure 184 * @mem: ptr to mem struct to fill out 185 * @size: size of memory requested 186 **/ 187 int i40e_allocate_virt_mem(struct i40e_hw *hw, struct i40e_virt_mem *mem, 188 u32 size) 189 { 190 mem->size = size; 191 mem->va = kzalloc(size, GFP_KERNEL); 192 193 if (!mem->va) 194 return -ENOMEM; 195 196 return 0; 197 } 198 199 /** 200 * i40e_free_virt_mem - OS specific memory free for shared code 201 * @hw: pointer to the HW structure 202 * @mem: ptr to mem struct to free 203 **/ 204 int i40e_free_virt_mem(struct i40e_hw *hw, struct i40e_virt_mem *mem) 205 { 206 /* it's ok to kfree a NULL pointer */ 207 kfree(mem->va); 208 mem->va = NULL; 209 mem->size = 0; 210 211 return 0; 212 } 213 214 /** 215 * i40e_get_lump - find a lump of free generic resource 216 * @pf: board private structure 217 * @pile: the pile of resource to search 218 * @needed: the number of items needed 219 * @id: an owner id to stick on the items assigned 220 * 221 * Returns the base item index of the lump, or negative for error 222 **/ 223 static int i40e_get_lump(struct i40e_pf *pf, struct i40e_lump_tracking *pile, 224 u16 needed, u16 id) 225 { 226 int ret = -ENOMEM; 227 int i, j; 228 229 if (!pile || needed == 0 || id >= I40E_PILE_VALID_BIT) { 230 dev_info(&pf->pdev->dev, 231 "param err: pile=%s needed=%d id=0x%04x\n", 232 pile ? "<valid>" : "<null>", needed, id); 233 return -EINVAL; 234 } 235 236 /* Allocate last queue in the pile for FDIR VSI queue 237 * so it doesn't fragment the qp_pile 238 */ 239 if (pile == pf->qp_pile && pf->vsi[id]->type == I40E_VSI_FDIR) { 240 if (pile->list[pile->num_entries - 1] & I40E_PILE_VALID_BIT) { 241 dev_err(&pf->pdev->dev, 242 "Cannot allocate queue %d for I40E_VSI_FDIR\n", 243 pile->num_entries - 1); 244 return -ENOMEM; 245 } 246 pile->list[pile->num_entries - 1] = id | I40E_PILE_VALID_BIT; 247 return pile->num_entries - 1; 248 } 249 250 i = 0; 251 while (i < pile->num_entries) { 252 /* skip already allocated entries */ 253 if (pile->list[i] & I40E_PILE_VALID_BIT) { 254 i++; 255 continue; 256 } 257 258 /* do we have enough in this lump? */ 259 for (j = 0; (j < needed) && ((i+j) < pile->num_entries); j++) { 260 if (pile->list[i+j] & I40E_PILE_VALID_BIT) 261 break; 262 } 263 264 if (j == needed) { 265 /* there was enough, so assign it to the requestor */ 266 for (j = 0; j < needed; j++) 267 pile->list[i+j] = id | I40E_PILE_VALID_BIT; 268 ret = i; 269 break; 270 } 271 272 /* not enough, so skip over it and continue looking */ 273 i += j; 274 } 275 276 return ret; 277 } 278 279 /** 280 * i40e_put_lump - return a lump of generic resource 281 * @pile: the pile of resource to search 282 * @index: the base item index 283 * @id: the owner id of the items assigned 284 * 285 * Returns the count of items in the lump 286 **/ 287 static int i40e_put_lump(struct i40e_lump_tracking *pile, u16 index, u16 id) 288 { 289 int valid_id = (id | I40E_PILE_VALID_BIT); 290 int count = 0; 291 u16 i; 292 293 if (!pile || index >= pile->num_entries) 294 return -EINVAL; 295 296 for (i = index; 297 i < pile->num_entries && pile->list[i] == valid_id; 298 i++) { 299 pile->list[i] = 0; 300 count++; 301 } 302 303 304 return count; 305 } 306 307 /** 308 * i40e_find_vsi_from_id - searches for the vsi with the given id 309 * @pf: the pf structure to search for the vsi 310 * @id: id of the vsi it is searching for 311 **/ 312 struct i40e_vsi *i40e_find_vsi_from_id(struct i40e_pf *pf, u16 id) 313 { 314 struct i40e_vsi *vsi; 315 int i; 316 317 i40e_pf_for_each_vsi(pf, i, vsi) 318 if (vsi->id == id) 319 return vsi; 320 321 return NULL; 322 } 323 324 /** 325 * i40e_service_event_schedule - Schedule the service task to wake up 326 * @pf: board private structure 327 * 328 * If not already scheduled, this puts the task into the work queue 329 **/ 330 void i40e_service_event_schedule(struct i40e_pf *pf) 331 { 332 if ((!test_bit(__I40E_DOWN, pf->state) && 333 !test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state)) || 334 test_bit(__I40E_RECOVERY_MODE, pf->state)) 335 queue_work(i40e_wq, &pf->service_task); 336 } 337 338 /** 339 * i40e_tx_timeout - Respond to a Tx Hang 340 * @netdev: network interface device structure 341 * @txqueue: queue number timing out 342 * 343 * If any port has noticed a Tx timeout, it is likely that the whole 344 * device is munged, not just the one netdev port, so go for the full 345 * reset. 346 **/ 347 static void i40e_tx_timeout(struct net_device *netdev, unsigned int txqueue) 348 { 349 struct i40e_netdev_priv *np = netdev_priv(netdev); 350 struct i40e_vsi *vsi = np->vsi; 351 struct i40e_pf *pf = vsi->back; 352 struct i40e_ring *tx_ring = NULL; 353 unsigned int i; 354 u32 head, val; 355 356 pf->tx_timeout_count++; 357 358 /* with txqueue index, find the tx_ring struct */ 359 for (i = 0; i < vsi->num_queue_pairs; i++) { 360 if (vsi->tx_rings[i] && vsi->tx_rings[i]->desc) { 361 if (txqueue == 362 vsi->tx_rings[i]->queue_index) { 363 tx_ring = vsi->tx_rings[i]; 364 break; 365 } 366 } 367 } 368 369 if (time_after(jiffies, (pf->tx_timeout_last_recovery + HZ*20))) 370 pf->tx_timeout_recovery_level = 1; /* reset after some time */ 371 else if (time_before(jiffies, 372 (pf->tx_timeout_last_recovery + netdev->watchdog_timeo))) 373 return; /* don't do any new action before the next timeout */ 374 375 /* don't kick off another recovery if one is already pending */ 376 if (test_and_set_bit(__I40E_TIMEOUT_RECOVERY_PENDING, pf->state)) 377 return; 378 379 if (tx_ring) { 380 head = i40e_get_head(tx_ring); 381 /* Read interrupt register */ 382 if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags)) 383 val = rd32(&pf->hw, 384 I40E_PFINT_DYN_CTLN(tx_ring->q_vector->v_idx + 385 tx_ring->vsi->base_vector - 1)); 386 else 387 val = rd32(&pf->hw, I40E_PFINT_DYN_CTL0); 388 389 netdev_info(netdev, "tx_timeout: VSI_seid: %d, Q %d, NTC: 0x%x, HWB: 0x%x, NTU: 0x%x, TAIL: 0x%x, INT: 0x%x\n", 390 vsi->seid, txqueue, tx_ring->next_to_clean, 391 head, tx_ring->next_to_use, 392 readl(tx_ring->tail), val); 393 } 394 395 pf->tx_timeout_last_recovery = jiffies; 396 netdev_info(netdev, "tx_timeout recovery level %d, txqueue %d\n", 397 pf->tx_timeout_recovery_level, txqueue); 398 399 switch (pf->tx_timeout_recovery_level) { 400 case 1: 401 set_bit(__I40E_PF_RESET_REQUESTED, pf->state); 402 break; 403 case 2: 404 set_bit(__I40E_CORE_RESET_REQUESTED, pf->state); 405 break; 406 case 3: 407 set_bit(__I40E_GLOBAL_RESET_REQUESTED, pf->state); 408 break; 409 default: 410 netdev_err(netdev, "tx_timeout recovery unsuccessful, device is in non-recoverable state.\n"); 411 set_bit(__I40E_DOWN_REQUESTED, pf->state); 412 set_bit(__I40E_VSI_DOWN_REQUESTED, vsi->state); 413 break; 414 } 415 416 i40e_service_event_schedule(pf); 417 pf->tx_timeout_recovery_level++; 418 } 419 420 /** 421 * i40e_get_vsi_stats_struct - Get System Network Statistics 422 * @vsi: the VSI we care about 423 * 424 * Returns the address of the device statistics structure. 425 * The statistics are actually updated from the service task. 426 **/ 427 struct rtnl_link_stats64 *i40e_get_vsi_stats_struct(struct i40e_vsi *vsi) 428 { 429 return &vsi->net_stats; 430 } 431 432 /** 433 * i40e_get_netdev_stats_struct_tx - populate stats from a Tx ring 434 * @ring: Tx ring to get statistics from 435 * @stats: statistics entry to be updated 436 **/ 437 static void i40e_get_netdev_stats_struct_tx(struct i40e_ring *ring, 438 struct rtnl_link_stats64 *stats) 439 { 440 u64 bytes, packets; 441 unsigned int start; 442 443 do { 444 start = u64_stats_fetch_begin(&ring->syncp); 445 packets = ring->stats.packets; 446 bytes = ring->stats.bytes; 447 } while (u64_stats_fetch_retry(&ring->syncp, start)); 448 449 stats->tx_packets += packets; 450 stats->tx_bytes += bytes; 451 } 452 453 /** 454 * i40e_get_netdev_stats_struct - Get statistics for netdev interface 455 * @netdev: network interface device structure 456 * @stats: data structure to store statistics 457 * 458 * Returns the address of the device statistics structure. 459 * The statistics are actually updated from the service task. 460 **/ 461 static void i40e_get_netdev_stats_struct(struct net_device *netdev, 462 struct rtnl_link_stats64 *stats) 463 { 464 struct i40e_netdev_priv *np = netdev_priv(netdev); 465 struct i40e_vsi *vsi = np->vsi; 466 struct rtnl_link_stats64 *vsi_stats = i40e_get_vsi_stats_struct(vsi); 467 struct i40e_ring *ring; 468 int i; 469 470 if (test_bit(__I40E_VSI_DOWN, vsi->state)) 471 return; 472 473 if (!vsi->tx_rings) 474 return; 475 476 rcu_read_lock(); 477 for (i = 0; i < vsi->num_queue_pairs; i++) { 478 u64 bytes, packets; 479 unsigned int start; 480 481 ring = READ_ONCE(vsi->tx_rings[i]); 482 if (!ring) 483 continue; 484 i40e_get_netdev_stats_struct_tx(ring, stats); 485 486 if (i40e_enabled_xdp_vsi(vsi)) { 487 ring = READ_ONCE(vsi->xdp_rings[i]); 488 if (!ring) 489 continue; 490 i40e_get_netdev_stats_struct_tx(ring, stats); 491 } 492 493 ring = READ_ONCE(vsi->rx_rings[i]); 494 if (!ring) 495 continue; 496 do { 497 start = u64_stats_fetch_begin(&ring->syncp); 498 packets = ring->stats.packets; 499 bytes = ring->stats.bytes; 500 } while (u64_stats_fetch_retry(&ring->syncp, start)); 501 502 stats->rx_packets += packets; 503 stats->rx_bytes += bytes; 504 505 } 506 rcu_read_unlock(); 507 508 /* following stats updated by i40e_watchdog_subtask() */ 509 stats->multicast = vsi_stats->multicast; 510 stats->tx_errors = vsi_stats->tx_errors; 511 stats->tx_dropped = vsi_stats->tx_dropped; 512 stats->rx_errors = vsi_stats->rx_errors; 513 stats->rx_dropped = vsi_stats->rx_dropped; 514 stats->rx_missed_errors = vsi_stats->rx_missed_errors; 515 stats->rx_crc_errors = vsi_stats->rx_crc_errors; 516 stats->rx_length_errors = vsi_stats->rx_length_errors; 517 } 518 519 /** 520 * i40e_vsi_reset_stats - Resets all stats of the given vsi 521 * @vsi: the VSI to have its stats reset 522 **/ 523 void i40e_vsi_reset_stats(struct i40e_vsi *vsi) 524 { 525 struct rtnl_link_stats64 *ns; 526 int i; 527 528 if (!vsi) 529 return; 530 531 ns = i40e_get_vsi_stats_struct(vsi); 532 memset(ns, 0, sizeof(*ns)); 533 memset(&vsi->net_stats_offsets, 0, sizeof(vsi->net_stats_offsets)); 534 memset(&vsi->eth_stats, 0, sizeof(vsi->eth_stats)); 535 memset(&vsi->eth_stats_offsets, 0, sizeof(vsi->eth_stats_offsets)); 536 if (vsi->rx_rings && vsi->rx_rings[0]) { 537 for (i = 0; i < vsi->num_queue_pairs; i++) { 538 memset(&vsi->rx_rings[i]->stats, 0, 539 sizeof(vsi->rx_rings[i]->stats)); 540 memset(&vsi->rx_rings[i]->rx_stats, 0, 541 sizeof(vsi->rx_rings[i]->rx_stats)); 542 memset(&vsi->tx_rings[i]->stats, 0, 543 sizeof(vsi->tx_rings[i]->stats)); 544 memset(&vsi->tx_rings[i]->tx_stats, 0, 545 sizeof(vsi->tx_rings[i]->tx_stats)); 546 } 547 } 548 vsi->stat_offsets_loaded = false; 549 } 550 551 /** 552 * i40e_pf_reset_stats - Reset all of the stats for the given PF 553 * @pf: the PF to be reset 554 **/ 555 void i40e_pf_reset_stats(struct i40e_pf *pf) 556 { 557 struct i40e_veb *veb; 558 int i; 559 560 memset(&pf->stats, 0, sizeof(pf->stats)); 561 memset(&pf->stats_offsets, 0, sizeof(pf->stats_offsets)); 562 pf->stat_offsets_loaded = false; 563 564 i40e_pf_for_each_veb(pf, i, veb) { 565 memset(&veb->stats, 0, sizeof(veb->stats)); 566 memset(&veb->stats_offsets, 0, sizeof(veb->stats_offsets)); 567 memset(&veb->tc_stats, 0, sizeof(veb->tc_stats)); 568 memset(&veb->tc_stats_offsets, 0, sizeof(veb->tc_stats_offsets)); 569 veb->stat_offsets_loaded = false; 570 } 571 pf->hw_csum_rx_error = 0; 572 } 573 574 /** 575 * i40e_compute_pci_to_hw_id - compute index form PCI function. 576 * @vsi: ptr to the VSI to read from. 577 * @hw: ptr to the hardware info. 578 **/ 579 static u32 i40e_compute_pci_to_hw_id(struct i40e_vsi *vsi, struct i40e_hw *hw) 580 { 581 int pf_count = i40e_get_pf_count(hw); 582 583 if (vsi->type == I40E_VSI_SRIOV) 584 return (hw->port * BIT(7)) / pf_count + vsi->vf_id; 585 586 return hw->port + BIT(7); 587 } 588 589 /** 590 * i40e_stat_update64 - read and update a 64 bit stat from the chip. 591 * @hw: ptr to the hardware info. 592 * @hireg: the high 32 bit reg to read. 593 * @loreg: the low 32 bit reg to read. 594 * @offset_loaded: has the initial offset been loaded yet. 595 * @offset: ptr to current offset value. 596 * @stat: ptr to the stat. 597 * 598 * Since the device stats are not reset at PFReset, they will not 599 * be zeroed when the driver starts. We'll save the first values read 600 * and use them as offsets to be subtracted from the raw values in order 601 * to report stats that count from zero. 602 **/ 603 static void i40e_stat_update64(struct i40e_hw *hw, u32 hireg, u32 loreg, 604 bool offset_loaded, u64 *offset, u64 *stat) 605 { 606 u64 new_data; 607 608 new_data = rd64(hw, loreg); 609 610 if (!offset_loaded || new_data < *offset) 611 *offset = new_data; 612 *stat = new_data - *offset; 613 } 614 615 /** 616 * i40e_stat_update48 - read and update a 48 bit stat from the chip 617 * @hw: ptr to the hardware info 618 * @hireg: the high 32 bit reg to read 619 * @loreg: the low 32 bit reg to read 620 * @offset_loaded: has the initial offset been loaded yet 621 * @offset: ptr to current offset value 622 * @stat: ptr to the stat 623 * 624 * Since the device stats are not reset at PFReset, they likely will not 625 * be zeroed when the driver starts. We'll save the first values read 626 * and use them as offsets to be subtracted from the raw values in order 627 * to report stats that count from zero. In the process, we also manage 628 * the potential roll-over. 629 **/ 630 static void i40e_stat_update48(struct i40e_hw *hw, u32 hireg, u32 loreg, 631 bool offset_loaded, u64 *offset, u64 *stat) 632 { 633 u64 new_data; 634 635 if (hw->device_id == I40E_DEV_ID_QEMU) { 636 new_data = rd32(hw, loreg); 637 new_data |= ((u64)(rd32(hw, hireg) & 0xFFFF)) << 32; 638 } else { 639 new_data = rd64(hw, loreg); 640 } 641 if (!offset_loaded) 642 *offset = new_data; 643 if (likely(new_data >= *offset)) 644 *stat = new_data - *offset; 645 else 646 *stat = (new_data + BIT_ULL(48)) - *offset; 647 *stat &= 0xFFFFFFFFFFFFULL; 648 } 649 650 /** 651 * i40e_stat_update32 - read and update a 32 bit stat from the chip 652 * @hw: ptr to the hardware info 653 * @reg: the hw reg to read 654 * @offset_loaded: has the initial offset been loaded yet 655 * @offset: ptr to current offset value 656 * @stat: ptr to the stat 657 **/ 658 static void i40e_stat_update32(struct i40e_hw *hw, u32 reg, 659 bool offset_loaded, u64 *offset, u64 *stat) 660 { 661 u32 new_data; 662 663 new_data = rd32(hw, reg); 664 if (!offset_loaded) 665 *offset = new_data; 666 if (likely(new_data >= *offset)) 667 *stat = (u32)(new_data - *offset); 668 else 669 *stat = (u32)((new_data + BIT_ULL(32)) - *offset); 670 } 671 672 /** 673 * i40e_stat_update_and_clear32 - read and clear hw reg, update a 32 bit stat 674 * @hw: ptr to the hardware info 675 * @reg: the hw reg to read and clear 676 * @stat: ptr to the stat 677 **/ 678 static void i40e_stat_update_and_clear32(struct i40e_hw *hw, u32 reg, u64 *stat) 679 { 680 u32 new_data = rd32(hw, reg); 681 682 wr32(hw, reg, 1); /* must write a nonzero value to clear register */ 683 *stat += new_data; 684 } 685 686 /** 687 * i40e_stats_update_rx_discards - update rx_discards. 688 * @vsi: ptr to the VSI to be updated. 689 * @hw: ptr to the hardware info. 690 * @stat_idx: VSI's stat_counter_idx. 691 * @offset_loaded: ptr to the VSI's stat_offsets_loaded. 692 * @stat_offset: ptr to stat_offset to store first read of specific register. 693 * @stat: ptr to VSI's stat to be updated. 694 **/ 695 static void 696 i40e_stats_update_rx_discards(struct i40e_vsi *vsi, struct i40e_hw *hw, 697 int stat_idx, bool offset_loaded, 698 struct i40e_eth_stats *stat_offset, 699 struct i40e_eth_stats *stat) 700 { 701 i40e_stat_update32(hw, I40E_GLV_RDPC(stat_idx), offset_loaded, 702 &stat_offset->rx_discards, &stat->rx_discards); 703 i40e_stat_update64(hw, 704 I40E_GL_RXERR1H(i40e_compute_pci_to_hw_id(vsi, hw)), 705 I40E_GL_RXERR1L(i40e_compute_pci_to_hw_id(vsi, hw)), 706 offset_loaded, &stat_offset->rx_discards_other, 707 &stat->rx_discards_other); 708 } 709 710 /** 711 * i40e_update_eth_stats - Update VSI-specific ethernet statistics counters. 712 * @vsi: the VSI to be updated 713 **/ 714 void i40e_update_eth_stats(struct i40e_vsi *vsi) 715 { 716 int stat_idx = le16_to_cpu(vsi->info.stat_counter_idx); 717 struct i40e_pf *pf = vsi->back; 718 struct i40e_hw *hw = &pf->hw; 719 struct i40e_eth_stats *oes; 720 struct i40e_eth_stats *es; /* device's eth stats */ 721 722 es = &vsi->eth_stats; 723 oes = &vsi->eth_stats_offsets; 724 725 /* Gather up the stats that the hw collects */ 726 i40e_stat_update32(hw, I40E_GLV_TEPC(stat_idx), 727 vsi->stat_offsets_loaded, 728 &oes->tx_errors, &es->tx_errors); 729 i40e_stat_update32(hw, I40E_GLV_RUPP(stat_idx), 730 vsi->stat_offsets_loaded, 731 &oes->rx_unknown_protocol, &es->rx_unknown_protocol); 732 733 i40e_stat_update48(hw, I40E_GLV_GORCH(stat_idx), 734 I40E_GLV_GORCL(stat_idx), 735 vsi->stat_offsets_loaded, 736 &oes->rx_bytes, &es->rx_bytes); 737 i40e_stat_update48(hw, I40E_GLV_UPRCH(stat_idx), 738 I40E_GLV_UPRCL(stat_idx), 739 vsi->stat_offsets_loaded, 740 &oes->rx_unicast, &es->rx_unicast); 741 i40e_stat_update48(hw, I40E_GLV_MPRCH(stat_idx), 742 I40E_GLV_MPRCL(stat_idx), 743 vsi->stat_offsets_loaded, 744 &oes->rx_multicast, &es->rx_multicast); 745 i40e_stat_update48(hw, I40E_GLV_BPRCH(stat_idx), 746 I40E_GLV_BPRCL(stat_idx), 747 vsi->stat_offsets_loaded, 748 &oes->rx_broadcast, &es->rx_broadcast); 749 750 i40e_stat_update48(hw, I40E_GLV_GOTCH(stat_idx), 751 I40E_GLV_GOTCL(stat_idx), 752 vsi->stat_offsets_loaded, 753 &oes->tx_bytes, &es->tx_bytes); 754 i40e_stat_update48(hw, I40E_GLV_UPTCH(stat_idx), 755 I40E_GLV_UPTCL(stat_idx), 756 vsi->stat_offsets_loaded, 757 &oes->tx_unicast, &es->tx_unicast); 758 i40e_stat_update48(hw, I40E_GLV_MPTCH(stat_idx), 759 I40E_GLV_MPTCL(stat_idx), 760 vsi->stat_offsets_loaded, 761 &oes->tx_multicast, &es->tx_multicast); 762 i40e_stat_update48(hw, I40E_GLV_BPTCH(stat_idx), 763 I40E_GLV_BPTCL(stat_idx), 764 vsi->stat_offsets_loaded, 765 &oes->tx_broadcast, &es->tx_broadcast); 766 767 i40e_stats_update_rx_discards(vsi, hw, stat_idx, 768 vsi->stat_offsets_loaded, oes, es); 769 770 vsi->stat_offsets_loaded = true; 771 } 772 773 /** 774 * i40e_update_veb_stats - Update Switch component statistics 775 * @veb: the VEB being updated 776 **/ 777 void i40e_update_veb_stats(struct i40e_veb *veb) 778 { 779 struct i40e_pf *pf = veb->pf; 780 struct i40e_hw *hw = &pf->hw; 781 struct i40e_eth_stats *oes; 782 struct i40e_eth_stats *es; /* device's eth stats */ 783 struct i40e_veb_tc_stats *veb_oes; 784 struct i40e_veb_tc_stats *veb_es; 785 int i, idx = 0; 786 787 idx = veb->stats_idx; 788 es = &veb->stats; 789 oes = &veb->stats_offsets; 790 veb_es = &veb->tc_stats; 791 veb_oes = &veb->tc_stats_offsets; 792 793 /* Gather up the stats that the hw collects */ 794 i40e_stat_update32(hw, I40E_GLSW_TDPC(idx), 795 veb->stat_offsets_loaded, 796 &oes->tx_discards, &es->tx_discards); 797 if (hw->revision_id > 0) 798 i40e_stat_update32(hw, I40E_GLSW_RUPP(idx), 799 veb->stat_offsets_loaded, 800 &oes->rx_unknown_protocol, 801 &es->rx_unknown_protocol); 802 i40e_stat_update48(hw, I40E_GLSW_GORCH(idx), I40E_GLSW_GORCL(idx), 803 veb->stat_offsets_loaded, 804 &oes->rx_bytes, &es->rx_bytes); 805 i40e_stat_update48(hw, I40E_GLSW_UPRCH(idx), I40E_GLSW_UPRCL(idx), 806 veb->stat_offsets_loaded, 807 &oes->rx_unicast, &es->rx_unicast); 808 i40e_stat_update48(hw, I40E_GLSW_MPRCH(idx), I40E_GLSW_MPRCL(idx), 809 veb->stat_offsets_loaded, 810 &oes->rx_multicast, &es->rx_multicast); 811 i40e_stat_update48(hw, I40E_GLSW_BPRCH(idx), I40E_GLSW_BPRCL(idx), 812 veb->stat_offsets_loaded, 813 &oes->rx_broadcast, &es->rx_broadcast); 814 815 i40e_stat_update48(hw, I40E_GLSW_GOTCH(idx), I40E_GLSW_GOTCL(idx), 816 veb->stat_offsets_loaded, 817 &oes->tx_bytes, &es->tx_bytes); 818 i40e_stat_update48(hw, I40E_GLSW_UPTCH(idx), I40E_GLSW_UPTCL(idx), 819 veb->stat_offsets_loaded, 820 &oes->tx_unicast, &es->tx_unicast); 821 i40e_stat_update48(hw, I40E_GLSW_MPTCH(idx), I40E_GLSW_MPTCL(idx), 822 veb->stat_offsets_loaded, 823 &oes->tx_multicast, &es->tx_multicast); 824 i40e_stat_update48(hw, I40E_GLSW_BPTCH(idx), I40E_GLSW_BPTCL(idx), 825 veb->stat_offsets_loaded, 826 &oes->tx_broadcast, &es->tx_broadcast); 827 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 828 i40e_stat_update48(hw, I40E_GLVEBTC_RPCH(i, idx), 829 I40E_GLVEBTC_RPCL(i, idx), 830 veb->stat_offsets_loaded, 831 &veb_oes->tc_rx_packets[i], 832 &veb_es->tc_rx_packets[i]); 833 i40e_stat_update48(hw, I40E_GLVEBTC_RBCH(i, idx), 834 I40E_GLVEBTC_RBCL(i, idx), 835 veb->stat_offsets_loaded, 836 &veb_oes->tc_rx_bytes[i], 837 &veb_es->tc_rx_bytes[i]); 838 i40e_stat_update48(hw, I40E_GLVEBTC_TPCH(i, idx), 839 I40E_GLVEBTC_TPCL(i, idx), 840 veb->stat_offsets_loaded, 841 &veb_oes->tc_tx_packets[i], 842 &veb_es->tc_tx_packets[i]); 843 i40e_stat_update48(hw, I40E_GLVEBTC_TBCH(i, idx), 844 I40E_GLVEBTC_TBCL(i, idx), 845 veb->stat_offsets_loaded, 846 &veb_oes->tc_tx_bytes[i], 847 &veb_es->tc_tx_bytes[i]); 848 } 849 veb->stat_offsets_loaded = true; 850 } 851 852 /** 853 * i40e_update_vsi_stats - Update the vsi statistics counters. 854 * @vsi: the VSI to be updated 855 * 856 * There are a few instances where we store the same stat in a 857 * couple of different structs. This is partly because we have 858 * the netdev stats that need to be filled out, which is slightly 859 * different from the "eth_stats" defined by the chip and used in 860 * VF communications. We sort it out here. 861 **/ 862 static void i40e_update_vsi_stats(struct i40e_vsi *vsi) 863 { 864 u64 rx_page, rx_buf, rx_reuse, rx_alloc, rx_waive, rx_busy; 865 struct i40e_pf *pf = vsi->back; 866 struct rtnl_link_stats64 *ons; 867 struct rtnl_link_stats64 *ns; /* netdev stats */ 868 struct i40e_eth_stats *oes; 869 struct i40e_eth_stats *es; /* device's eth stats */ 870 u64 tx_restart, tx_busy; 871 struct i40e_ring *p; 872 u64 bytes, packets; 873 unsigned int start; 874 u64 tx_linearize; 875 u64 tx_force_wb; 876 u64 tx_stopped; 877 u64 rx_p, rx_b; 878 u64 tx_p, tx_b; 879 u16 q; 880 881 if (test_bit(__I40E_VSI_DOWN, vsi->state) || 882 test_bit(__I40E_CONFIG_BUSY, pf->state)) 883 return; 884 885 ns = i40e_get_vsi_stats_struct(vsi); 886 ons = &vsi->net_stats_offsets; 887 es = &vsi->eth_stats; 888 oes = &vsi->eth_stats_offsets; 889 890 /* Gather up the netdev and vsi stats that the driver collects 891 * on the fly during packet processing 892 */ 893 rx_b = rx_p = 0; 894 tx_b = tx_p = 0; 895 tx_restart = tx_busy = tx_linearize = tx_force_wb = 0; 896 tx_stopped = 0; 897 rx_page = 0; 898 rx_buf = 0; 899 rx_reuse = 0; 900 rx_alloc = 0; 901 rx_waive = 0; 902 rx_busy = 0; 903 rcu_read_lock(); 904 for (q = 0; q < vsi->num_queue_pairs; q++) { 905 /* locate Tx ring */ 906 p = READ_ONCE(vsi->tx_rings[q]); 907 if (!p) 908 continue; 909 910 do { 911 start = u64_stats_fetch_begin(&p->syncp); 912 packets = p->stats.packets; 913 bytes = p->stats.bytes; 914 } while (u64_stats_fetch_retry(&p->syncp, start)); 915 tx_b += bytes; 916 tx_p += packets; 917 tx_restart += p->tx_stats.restart_queue; 918 tx_busy += p->tx_stats.tx_busy; 919 tx_linearize += p->tx_stats.tx_linearize; 920 tx_force_wb += p->tx_stats.tx_force_wb; 921 tx_stopped += p->tx_stats.tx_stopped; 922 923 /* locate Rx ring */ 924 p = READ_ONCE(vsi->rx_rings[q]); 925 if (!p) 926 continue; 927 928 do { 929 start = u64_stats_fetch_begin(&p->syncp); 930 packets = p->stats.packets; 931 bytes = p->stats.bytes; 932 } while (u64_stats_fetch_retry(&p->syncp, start)); 933 rx_b += bytes; 934 rx_p += packets; 935 rx_buf += p->rx_stats.alloc_buff_failed; 936 rx_page += p->rx_stats.alloc_page_failed; 937 rx_reuse += p->rx_stats.page_reuse_count; 938 rx_alloc += p->rx_stats.page_alloc_count; 939 rx_waive += p->rx_stats.page_waive_count; 940 rx_busy += p->rx_stats.page_busy_count; 941 942 if (i40e_enabled_xdp_vsi(vsi)) { 943 /* locate XDP ring */ 944 p = READ_ONCE(vsi->xdp_rings[q]); 945 if (!p) 946 continue; 947 948 do { 949 start = u64_stats_fetch_begin(&p->syncp); 950 packets = p->stats.packets; 951 bytes = p->stats.bytes; 952 } while (u64_stats_fetch_retry(&p->syncp, start)); 953 tx_b += bytes; 954 tx_p += packets; 955 tx_restart += p->tx_stats.restart_queue; 956 tx_busy += p->tx_stats.tx_busy; 957 tx_linearize += p->tx_stats.tx_linearize; 958 tx_force_wb += p->tx_stats.tx_force_wb; 959 } 960 } 961 rcu_read_unlock(); 962 vsi->tx_restart = tx_restart; 963 vsi->tx_busy = tx_busy; 964 vsi->tx_linearize = tx_linearize; 965 vsi->tx_force_wb = tx_force_wb; 966 vsi->tx_stopped = tx_stopped; 967 vsi->rx_page_failed = rx_page; 968 vsi->rx_buf_failed = rx_buf; 969 vsi->rx_page_reuse = rx_reuse; 970 vsi->rx_page_alloc = rx_alloc; 971 vsi->rx_page_waive = rx_waive; 972 vsi->rx_page_busy = rx_busy; 973 974 ns->rx_packets = rx_p; 975 ns->rx_bytes = rx_b; 976 ns->tx_packets = tx_p; 977 ns->tx_bytes = tx_b; 978 979 /* update netdev stats from eth stats */ 980 i40e_update_eth_stats(vsi); 981 ons->tx_errors = oes->tx_errors; 982 ns->tx_errors = es->tx_errors; 983 ons->multicast = oes->rx_multicast; 984 ns->multicast = es->rx_multicast; 985 ons->rx_dropped = oes->rx_discards_other; 986 ns->rx_dropped = es->rx_discards_other; 987 ons->rx_missed_errors = oes->rx_discards; 988 ns->rx_missed_errors = es->rx_discards; 989 ons->tx_dropped = oes->tx_discards; 990 ns->tx_dropped = es->tx_discards; 991 992 /* pull in a couple PF stats if this is the main vsi */ 993 if (vsi->type == I40E_VSI_MAIN) { 994 ns->rx_crc_errors = pf->stats.crc_errors; 995 ns->rx_errors = pf->stats.crc_errors + pf->stats.illegal_bytes; 996 ns->rx_length_errors = pf->stats.rx_length_errors; 997 } 998 } 999 1000 /** 1001 * i40e_update_pf_stats - Update the PF statistics counters. 1002 * @pf: the PF to be updated 1003 **/ 1004 static void i40e_update_pf_stats(struct i40e_pf *pf) 1005 { 1006 struct i40e_hw_port_stats *osd = &pf->stats_offsets; 1007 struct i40e_hw_port_stats *nsd = &pf->stats; 1008 struct i40e_hw *hw = &pf->hw; 1009 u32 val; 1010 int i; 1011 1012 i40e_stat_update48(hw, I40E_GLPRT_GORCH(hw->port), 1013 I40E_GLPRT_GORCL(hw->port), 1014 pf->stat_offsets_loaded, 1015 &osd->eth.rx_bytes, &nsd->eth.rx_bytes); 1016 i40e_stat_update48(hw, I40E_GLPRT_GOTCH(hw->port), 1017 I40E_GLPRT_GOTCL(hw->port), 1018 pf->stat_offsets_loaded, 1019 &osd->eth.tx_bytes, &nsd->eth.tx_bytes); 1020 i40e_stat_update32(hw, I40E_GLPRT_RDPC(hw->port), 1021 pf->stat_offsets_loaded, 1022 &osd->eth.rx_discards, 1023 &nsd->eth.rx_discards); 1024 i40e_stat_update48(hw, I40E_GLPRT_UPRCH(hw->port), 1025 I40E_GLPRT_UPRCL(hw->port), 1026 pf->stat_offsets_loaded, 1027 &osd->eth.rx_unicast, 1028 &nsd->eth.rx_unicast); 1029 i40e_stat_update48(hw, I40E_GLPRT_MPRCH(hw->port), 1030 I40E_GLPRT_MPRCL(hw->port), 1031 pf->stat_offsets_loaded, 1032 &osd->eth.rx_multicast, 1033 &nsd->eth.rx_multicast); 1034 i40e_stat_update48(hw, I40E_GLPRT_BPRCH(hw->port), 1035 I40E_GLPRT_BPRCL(hw->port), 1036 pf->stat_offsets_loaded, 1037 &osd->eth.rx_broadcast, 1038 &nsd->eth.rx_broadcast); 1039 i40e_stat_update48(hw, I40E_GLPRT_UPTCH(hw->port), 1040 I40E_GLPRT_UPTCL(hw->port), 1041 pf->stat_offsets_loaded, 1042 &osd->eth.tx_unicast, 1043 &nsd->eth.tx_unicast); 1044 i40e_stat_update48(hw, I40E_GLPRT_MPTCH(hw->port), 1045 I40E_GLPRT_MPTCL(hw->port), 1046 pf->stat_offsets_loaded, 1047 &osd->eth.tx_multicast, 1048 &nsd->eth.tx_multicast); 1049 i40e_stat_update48(hw, I40E_GLPRT_BPTCH(hw->port), 1050 I40E_GLPRT_BPTCL(hw->port), 1051 pf->stat_offsets_loaded, 1052 &osd->eth.tx_broadcast, 1053 &nsd->eth.tx_broadcast); 1054 1055 i40e_stat_update32(hw, I40E_GLPRT_TDOLD(hw->port), 1056 pf->stat_offsets_loaded, 1057 &osd->tx_dropped_link_down, 1058 &nsd->tx_dropped_link_down); 1059 1060 i40e_stat_update32(hw, I40E_GLPRT_CRCERRS(hw->port), 1061 pf->stat_offsets_loaded, 1062 &osd->crc_errors, &nsd->crc_errors); 1063 1064 i40e_stat_update32(hw, I40E_GLPRT_ILLERRC(hw->port), 1065 pf->stat_offsets_loaded, 1066 &osd->illegal_bytes, &nsd->illegal_bytes); 1067 1068 i40e_stat_update32(hw, I40E_GLPRT_MLFC(hw->port), 1069 pf->stat_offsets_loaded, 1070 &osd->mac_local_faults, 1071 &nsd->mac_local_faults); 1072 i40e_stat_update32(hw, I40E_GLPRT_MRFC(hw->port), 1073 pf->stat_offsets_loaded, 1074 &osd->mac_remote_faults, 1075 &nsd->mac_remote_faults); 1076 1077 i40e_stat_update32(hw, I40E_GLPRT_RLEC(hw->port), 1078 pf->stat_offsets_loaded, 1079 &osd->rx_length_errors, 1080 &nsd->rx_length_errors); 1081 1082 i40e_stat_update32(hw, I40E_GLPRT_LXONRXC(hw->port), 1083 pf->stat_offsets_loaded, 1084 &osd->link_xon_rx, &nsd->link_xon_rx); 1085 i40e_stat_update32(hw, I40E_GLPRT_LXONTXC(hw->port), 1086 pf->stat_offsets_loaded, 1087 &osd->link_xon_tx, &nsd->link_xon_tx); 1088 i40e_stat_update32(hw, I40E_GLPRT_LXOFFRXC(hw->port), 1089 pf->stat_offsets_loaded, 1090 &osd->link_xoff_rx, &nsd->link_xoff_rx); 1091 i40e_stat_update32(hw, I40E_GLPRT_LXOFFTXC(hw->port), 1092 pf->stat_offsets_loaded, 1093 &osd->link_xoff_tx, &nsd->link_xoff_tx); 1094 1095 for (i = 0; i < 8; i++) { 1096 i40e_stat_update32(hw, I40E_GLPRT_PXOFFRXC(hw->port, i), 1097 pf->stat_offsets_loaded, 1098 &osd->priority_xoff_rx[i], 1099 &nsd->priority_xoff_rx[i]); 1100 i40e_stat_update32(hw, I40E_GLPRT_PXONRXC(hw->port, i), 1101 pf->stat_offsets_loaded, 1102 &osd->priority_xon_rx[i], 1103 &nsd->priority_xon_rx[i]); 1104 i40e_stat_update32(hw, I40E_GLPRT_PXONTXC(hw->port, i), 1105 pf->stat_offsets_loaded, 1106 &osd->priority_xon_tx[i], 1107 &nsd->priority_xon_tx[i]); 1108 i40e_stat_update32(hw, I40E_GLPRT_PXOFFTXC(hw->port, i), 1109 pf->stat_offsets_loaded, 1110 &osd->priority_xoff_tx[i], 1111 &nsd->priority_xoff_tx[i]); 1112 i40e_stat_update32(hw, 1113 I40E_GLPRT_RXON2OFFCNT(hw->port, i), 1114 pf->stat_offsets_loaded, 1115 &osd->priority_xon_2_xoff[i], 1116 &nsd->priority_xon_2_xoff[i]); 1117 } 1118 1119 i40e_stat_update48(hw, I40E_GLPRT_PRC64H(hw->port), 1120 I40E_GLPRT_PRC64L(hw->port), 1121 pf->stat_offsets_loaded, 1122 &osd->rx_size_64, &nsd->rx_size_64); 1123 i40e_stat_update48(hw, I40E_GLPRT_PRC127H(hw->port), 1124 I40E_GLPRT_PRC127L(hw->port), 1125 pf->stat_offsets_loaded, 1126 &osd->rx_size_127, &nsd->rx_size_127); 1127 i40e_stat_update48(hw, I40E_GLPRT_PRC255H(hw->port), 1128 I40E_GLPRT_PRC255L(hw->port), 1129 pf->stat_offsets_loaded, 1130 &osd->rx_size_255, &nsd->rx_size_255); 1131 i40e_stat_update48(hw, I40E_GLPRT_PRC511H(hw->port), 1132 I40E_GLPRT_PRC511L(hw->port), 1133 pf->stat_offsets_loaded, 1134 &osd->rx_size_511, &nsd->rx_size_511); 1135 i40e_stat_update48(hw, I40E_GLPRT_PRC1023H(hw->port), 1136 I40E_GLPRT_PRC1023L(hw->port), 1137 pf->stat_offsets_loaded, 1138 &osd->rx_size_1023, &nsd->rx_size_1023); 1139 i40e_stat_update48(hw, I40E_GLPRT_PRC1522H(hw->port), 1140 I40E_GLPRT_PRC1522L(hw->port), 1141 pf->stat_offsets_loaded, 1142 &osd->rx_size_1522, &nsd->rx_size_1522); 1143 i40e_stat_update48(hw, I40E_GLPRT_PRC9522H(hw->port), 1144 I40E_GLPRT_PRC9522L(hw->port), 1145 pf->stat_offsets_loaded, 1146 &osd->rx_size_big, &nsd->rx_size_big); 1147 1148 i40e_stat_update48(hw, I40E_GLPRT_PTC64H(hw->port), 1149 I40E_GLPRT_PTC64L(hw->port), 1150 pf->stat_offsets_loaded, 1151 &osd->tx_size_64, &nsd->tx_size_64); 1152 i40e_stat_update48(hw, I40E_GLPRT_PTC127H(hw->port), 1153 I40E_GLPRT_PTC127L(hw->port), 1154 pf->stat_offsets_loaded, 1155 &osd->tx_size_127, &nsd->tx_size_127); 1156 i40e_stat_update48(hw, I40E_GLPRT_PTC255H(hw->port), 1157 I40E_GLPRT_PTC255L(hw->port), 1158 pf->stat_offsets_loaded, 1159 &osd->tx_size_255, &nsd->tx_size_255); 1160 i40e_stat_update48(hw, I40E_GLPRT_PTC511H(hw->port), 1161 I40E_GLPRT_PTC511L(hw->port), 1162 pf->stat_offsets_loaded, 1163 &osd->tx_size_511, &nsd->tx_size_511); 1164 i40e_stat_update48(hw, I40E_GLPRT_PTC1023H(hw->port), 1165 I40E_GLPRT_PTC1023L(hw->port), 1166 pf->stat_offsets_loaded, 1167 &osd->tx_size_1023, &nsd->tx_size_1023); 1168 i40e_stat_update48(hw, I40E_GLPRT_PTC1522H(hw->port), 1169 I40E_GLPRT_PTC1522L(hw->port), 1170 pf->stat_offsets_loaded, 1171 &osd->tx_size_1522, &nsd->tx_size_1522); 1172 i40e_stat_update48(hw, I40E_GLPRT_PTC9522H(hw->port), 1173 I40E_GLPRT_PTC9522L(hw->port), 1174 pf->stat_offsets_loaded, 1175 &osd->tx_size_big, &nsd->tx_size_big); 1176 1177 i40e_stat_update32(hw, I40E_GLPRT_RUC(hw->port), 1178 pf->stat_offsets_loaded, 1179 &osd->rx_undersize, &nsd->rx_undersize); 1180 i40e_stat_update32(hw, I40E_GLPRT_RFC(hw->port), 1181 pf->stat_offsets_loaded, 1182 &osd->rx_fragments, &nsd->rx_fragments); 1183 i40e_stat_update32(hw, I40E_GLPRT_ROC(hw->port), 1184 pf->stat_offsets_loaded, 1185 &osd->rx_oversize, &nsd->rx_oversize); 1186 i40e_stat_update32(hw, I40E_GLPRT_RJC(hw->port), 1187 pf->stat_offsets_loaded, 1188 &osd->rx_jabber, &nsd->rx_jabber); 1189 1190 /* FDIR stats */ 1191 i40e_stat_update_and_clear32(hw, 1192 I40E_GLQF_PCNT(I40E_FD_ATR_STAT_IDX(hw->pf_id)), 1193 &nsd->fd_atr_match); 1194 i40e_stat_update_and_clear32(hw, 1195 I40E_GLQF_PCNT(I40E_FD_SB_STAT_IDX(hw->pf_id)), 1196 &nsd->fd_sb_match); 1197 i40e_stat_update_and_clear32(hw, 1198 I40E_GLQF_PCNT(I40E_FD_ATR_TUNNEL_STAT_IDX(hw->pf_id)), 1199 &nsd->fd_atr_tunnel_match); 1200 1201 val = rd32(hw, I40E_PRTPM_EEE_STAT); 1202 nsd->tx_lpi_status = 1203 FIELD_GET(I40E_PRTPM_EEE_STAT_TX_LPI_STATUS_MASK, val); 1204 nsd->rx_lpi_status = 1205 FIELD_GET(I40E_PRTPM_EEE_STAT_RX_LPI_STATUS_MASK, val); 1206 i40e_stat_update32(hw, I40E_PRTPM_TLPIC, 1207 pf->stat_offsets_loaded, 1208 &osd->tx_lpi_count, &nsd->tx_lpi_count); 1209 i40e_stat_update32(hw, I40E_PRTPM_RLPIC, 1210 pf->stat_offsets_loaded, 1211 &osd->rx_lpi_count, &nsd->rx_lpi_count); 1212 1213 if (test_bit(I40E_FLAG_FD_SB_ENA, pf->flags) && 1214 !test_bit(__I40E_FD_SB_AUTO_DISABLED, pf->state)) 1215 nsd->fd_sb_status = true; 1216 else 1217 nsd->fd_sb_status = false; 1218 1219 if (test_bit(I40E_FLAG_FD_ATR_ENA, pf->flags) && 1220 !test_bit(__I40E_FD_ATR_AUTO_DISABLED, pf->state)) 1221 nsd->fd_atr_status = true; 1222 else 1223 nsd->fd_atr_status = false; 1224 1225 pf->stat_offsets_loaded = true; 1226 } 1227 1228 /** 1229 * i40e_update_stats - Update the various statistics counters. 1230 * @vsi: the VSI to be updated 1231 * 1232 * Update the various stats for this VSI and its related entities. 1233 **/ 1234 void i40e_update_stats(struct i40e_vsi *vsi) 1235 { 1236 struct i40e_pf *pf = vsi->back; 1237 1238 if (vsi->type == I40E_VSI_MAIN) 1239 i40e_update_pf_stats(pf); 1240 1241 i40e_update_vsi_stats(vsi); 1242 } 1243 1244 /** 1245 * i40e_count_filters - counts VSI mac filters 1246 * @vsi: the VSI to be searched 1247 * 1248 * Returns count of mac filters 1249 **/ 1250 int i40e_count_filters(struct i40e_vsi *vsi) 1251 { 1252 struct i40e_mac_filter *f; 1253 struct hlist_node *h; 1254 int bkt; 1255 int cnt = 0; 1256 1257 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) { 1258 if (f->state == I40E_FILTER_NEW || 1259 f->state == I40E_FILTER_ACTIVE) 1260 ++cnt; 1261 } 1262 1263 return cnt; 1264 } 1265 1266 /** 1267 * i40e_find_filter - Search VSI filter list for specific mac/vlan filter 1268 * @vsi: the VSI to be searched 1269 * @macaddr: the MAC address 1270 * @vlan: the vlan 1271 * 1272 * Returns ptr to the filter object or NULL 1273 **/ 1274 static struct i40e_mac_filter *i40e_find_filter(struct i40e_vsi *vsi, 1275 const u8 *macaddr, s16 vlan) 1276 { 1277 struct i40e_mac_filter *f; 1278 u64 key; 1279 1280 if (!vsi || !macaddr) 1281 return NULL; 1282 1283 key = i40e_addr_to_hkey(macaddr); 1284 hash_for_each_possible(vsi->mac_filter_hash, f, hlist, key) { 1285 if ((ether_addr_equal(macaddr, f->macaddr)) && 1286 (vlan == f->vlan)) 1287 return f; 1288 } 1289 return NULL; 1290 } 1291 1292 /** 1293 * i40e_find_mac - Find a mac addr in the macvlan filters list 1294 * @vsi: the VSI to be searched 1295 * @macaddr: the MAC address we are searching for 1296 * 1297 * Returns the first filter with the provided MAC address or NULL if 1298 * MAC address was not found 1299 **/ 1300 struct i40e_mac_filter *i40e_find_mac(struct i40e_vsi *vsi, const u8 *macaddr) 1301 { 1302 struct i40e_mac_filter *f; 1303 u64 key; 1304 1305 if (!vsi || !macaddr) 1306 return NULL; 1307 1308 key = i40e_addr_to_hkey(macaddr); 1309 hash_for_each_possible(vsi->mac_filter_hash, f, hlist, key) { 1310 if ((ether_addr_equal(macaddr, f->macaddr))) 1311 return f; 1312 } 1313 return NULL; 1314 } 1315 1316 /** 1317 * i40e_is_vsi_in_vlan - Check if VSI is in vlan mode 1318 * @vsi: the VSI to be searched 1319 * 1320 * Returns true if VSI is in vlan mode or false otherwise 1321 **/ 1322 bool i40e_is_vsi_in_vlan(struct i40e_vsi *vsi) 1323 { 1324 /* If we have a PVID, always operate in VLAN mode */ 1325 if (vsi->info.pvid) 1326 return true; 1327 1328 /* We need to operate in VLAN mode whenever we have any filters with 1329 * a VLAN other than I40E_VLAN_ALL. We could check the table each 1330 * time, incurring search cost repeatedly. However, we can notice two 1331 * things: 1332 * 1333 * 1) the only place where we can gain a VLAN filter is in 1334 * i40e_add_filter. 1335 * 1336 * 2) the only place where filters are actually removed is in 1337 * i40e_sync_filters_subtask. 1338 * 1339 * Thus, we can simply use a boolean value, has_vlan_filters which we 1340 * will set to true when we add a VLAN filter in i40e_add_filter. Then 1341 * we have to perform the full search after deleting filters in 1342 * i40e_sync_filters_subtask, but we already have to search 1343 * filters here and can perform the check at the same time. This 1344 * results in avoiding embedding a loop for VLAN mode inside another 1345 * loop over all the filters, and should maintain correctness as noted 1346 * above. 1347 */ 1348 return vsi->has_vlan_filter; 1349 } 1350 1351 /** 1352 * i40e_correct_mac_vlan_filters - Correct non-VLAN filters if necessary 1353 * @vsi: the VSI to configure 1354 * @tmp_add_list: list of filters ready to be added 1355 * @tmp_del_list: list of filters ready to be deleted 1356 * @vlan_filters: the number of active VLAN filters 1357 * 1358 * Update VLAN=0 and VLAN=-1 (I40E_VLAN_ANY) filters properly so that they 1359 * behave as expected. If we have any active VLAN filters remaining or about 1360 * to be added then we need to update non-VLAN filters to be marked as VLAN=0 1361 * so that they only match against untagged traffic. If we no longer have any 1362 * active VLAN filters, we need to make all non-VLAN filters marked as VLAN=-1 1363 * so that they match against both tagged and untagged traffic. In this way, 1364 * we ensure that we correctly receive the desired traffic. This ensures that 1365 * when we have an active VLAN we will receive only untagged traffic and 1366 * traffic matching active VLANs. If we have no active VLANs then we will 1367 * operate in non-VLAN mode and receive all traffic, tagged or untagged. 1368 * 1369 * Finally, in a similar fashion, this function also corrects filters when 1370 * there is an active PVID assigned to this VSI. 1371 * 1372 * In case of memory allocation failure return -ENOMEM. Otherwise, return 0. 1373 * 1374 * This function is only expected to be called from within 1375 * i40e_sync_vsi_filters. 1376 * 1377 * NOTE: This function expects to be called while under the 1378 * mac_filter_hash_lock 1379 */ 1380 static int i40e_correct_mac_vlan_filters(struct i40e_vsi *vsi, 1381 struct hlist_head *tmp_add_list, 1382 struct hlist_head *tmp_del_list, 1383 int vlan_filters) 1384 { 1385 s16 pvid = le16_to_cpu(vsi->info.pvid); 1386 struct i40e_mac_filter *f, *add_head; 1387 struct i40e_new_mac_filter *new; 1388 struct hlist_node *h; 1389 int bkt, new_vlan; 1390 1391 /* To determine if a particular filter needs to be replaced we 1392 * have the three following conditions: 1393 * 1394 * a) if we have a PVID assigned, then all filters which are 1395 * not marked as VLAN=PVID must be replaced with filters that 1396 * are. 1397 * b) otherwise, if we have any active VLANS, all filters 1398 * which are marked as VLAN=-1 must be replaced with 1399 * filters marked as VLAN=0 1400 * c) finally, if we do not have any active VLANS, all filters 1401 * which are marked as VLAN=0 must be replaced with filters 1402 * marked as VLAN=-1 1403 */ 1404 1405 /* Update the filters about to be added in place */ 1406 hlist_for_each_entry(new, tmp_add_list, hlist) { 1407 if (pvid && new->f->vlan != pvid) 1408 new->f->vlan = pvid; 1409 else if (vlan_filters && new->f->vlan == I40E_VLAN_ANY) 1410 new->f->vlan = 0; 1411 else if (!vlan_filters && new->f->vlan == 0) 1412 new->f->vlan = I40E_VLAN_ANY; 1413 } 1414 1415 /* Update the remaining active filters */ 1416 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) { 1417 /* Combine the checks for whether a filter needs to be changed 1418 * and then determine the new VLAN inside the if block, in 1419 * order to avoid duplicating code for adding the new filter 1420 * then deleting the old filter. 1421 */ 1422 if ((pvid && f->vlan != pvid) || 1423 (vlan_filters && f->vlan == I40E_VLAN_ANY) || 1424 (!vlan_filters && f->vlan == 0)) { 1425 /* Determine the new vlan we will be adding */ 1426 if (pvid) 1427 new_vlan = pvid; 1428 else if (vlan_filters) 1429 new_vlan = 0; 1430 else 1431 new_vlan = I40E_VLAN_ANY; 1432 1433 /* Create the new filter */ 1434 add_head = i40e_add_filter(vsi, f->macaddr, new_vlan); 1435 if (!add_head) 1436 return -ENOMEM; 1437 1438 /* Create a temporary i40e_new_mac_filter */ 1439 new = kzalloc(sizeof(*new), GFP_ATOMIC); 1440 if (!new) 1441 return -ENOMEM; 1442 1443 new->f = add_head; 1444 new->state = add_head->state; 1445 1446 /* Add the new filter to the tmp list */ 1447 hlist_add_head(&new->hlist, tmp_add_list); 1448 1449 /* Put the original filter into the delete list */ 1450 f->state = I40E_FILTER_REMOVE; 1451 hash_del(&f->hlist); 1452 hlist_add_head(&f->hlist, tmp_del_list); 1453 } 1454 } 1455 1456 vsi->has_vlan_filter = !!vlan_filters; 1457 1458 return 0; 1459 } 1460 1461 /** 1462 * i40e_get_vf_new_vlan - Get new vlan id on a vf 1463 * @vsi: the vsi to configure 1464 * @new_mac: new mac filter to be added 1465 * @f: existing mac filter, replaced with new_mac->f if new_mac is not NULL 1466 * @vlan_filters: the number of active VLAN filters 1467 * @trusted: flag if the VF is trusted 1468 * 1469 * Get new VLAN id based on current VLAN filters, trust, PVID 1470 * and vf-vlan-prune-disable flag. 1471 * 1472 * Returns the value of the new vlan filter or 1473 * the old value if no new filter is needed. 1474 */ 1475 static s16 i40e_get_vf_new_vlan(struct i40e_vsi *vsi, 1476 struct i40e_new_mac_filter *new_mac, 1477 struct i40e_mac_filter *f, 1478 int vlan_filters, 1479 bool trusted) 1480 { 1481 s16 pvid = le16_to_cpu(vsi->info.pvid); 1482 struct i40e_pf *pf = vsi->back; 1483 bool is_any; 1484 1485 if (new_mac) 1486 f = new_mac->f; 1487 1488 if (pvid && f->vlan != pvid) 1489 return pvid; 1490 1491 is_any = (trusted || 1492 !test_bit(I40E_FLAG_VF_VLAN_PRUNING_ENA, pf->flags)); 1493 1494 if ((vlan_filters && f->vlan == I40E_VLAN_ANY) || 1495 (!is_any && !vlan_filters && f->vlan == I40E_VLAN_ANY) || 1496 (is_any && !vlan_filters && f->vlan == 0)) { 1497 if (is_any) 1498 return I40E_VLAN_ANY; 1499 else 1500 return 0; 1501 } 1502 1503 return f->vlan; 1504 } 1505 1506 /** 1507 * i40e_correct_vf_mac_vlan_filters - Correct non-VLAN VF filters if necessary 1508 * @vsi: the vsi to configure 1509 * @tmp_add_list: list of filters ready to be added 1510 * @tmp_del_list: list of filters ready to be deleted 1511 * @vlan_filters: the number of active VLAN filters 1512 * @trusted: flag if the VF is trusted 1513 * 1514 * Correct VF VLAN filters based on current VLAN filters, trust, PVID 1515 * and vf-vlan-prune-disable flag. 1516 * 1517 * In case of memory allocation failure return -ENOMEM. Otherwise, return 0. 1518 * 1519 * This function is only expected to be called from within 1520 * i40e_sync_vsi_filters. 1521 * 1522 * NOTE: This function expects to be called while under the 1523 * mac_filter_hash_lock 1524 */ 1525 static int i40e_correct_vf_mac_vlan_filters(struct i40e_vsi *vsi, 1526 struct hlist_head *tmp_add_list, 1527 struct hlist_head *tmp_del_list, 1528 int vlan_filters, 1529 bool trusted) 1530 { 1531 struct i40e_mac_filter *f, *add_head; 1532 struct i40e_new_mac_filter *new_mac; 1533 struct hlist_node *h; 1534 int bkt, new_vlan; 1535 1536 hlist_for_each_entry(new_mac, tmp_add_list, hlist) { 1537 new_mac->f->vlan = i40e_get_vf_new_vlan(vsi, new_mac, NULL, 1538 vlan_filters, trusted); 1539 } 1540 1541 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) { 1542 new_vlan = i40e_get_vf_new_vlan(vsi, NULL, f, vlan_filters, 1543 trusted); 1544 if (new_vlan != f->vlan) { 1545 add_head = i40e_add_filter(vsi, f->macaddr, new_vlan); 1546 if (!add_head) 1547 return -ENOMEM; 1548 /* Create a temporary i40e_new_mac_filter */ 1549 new_mac = kzalloc(sizeof(*new_mac), GFP_ATOMIC); 1550 if (!new_mac) 1551 return -ENOMEM; 1552 new_mac->f = add_head; 1553 new_mac->state = add_head->state; 1554 1555 /* Add the new filter to the tmp list */ 1556 hlist_add_head(&new_mac->hlist, tmp_add_list); 1557 1558 /* Put the original filter into the delete list */ 1559 f->state = I40E_FILTER_REMOVE; 1560 hash_del(&f->hlist); 1561 hlist_add_head(&f->hlist, tmp_del_list); 1562 } 1563 } 1564 1565 vsi->has_vlan_filter = !!vlan_filters; 1566 return 0; 1567 } 1568 1569 /** 1570 * i40e_rm_default_mac_filter - Remove the default MAC filter set by NVM 1571 * @vsi: the PF Main VSI - inappropriate for any other VSI 1572 * @macaddr: the MAC address 1573 * 1574 * Remove whatever filter the firmware set up so the driver can manage 1575 * its own filtering intelligently. 1576 **/ 1577 static void i40e_rm_default_mac_filter(struct i40e_vsi *vsi, u8 *macaddr) 1578 { 1579 struct i40e_aqc_remove_macvlan_element_data element; 1580 struct i40e_pf *pf = vsi->back; 1581 1582 /* Only appropriate for the PF main VSI */ 1583 if (vsi->type != I40E_VSI_MAIN) 1584 return; 1585 1586 memset(&element, 0, sizeof(element)); 1587 ether_addr_copy(element.mac_addr, macaddr); 1588 element.vlan_tag = 0; 1589 /* Ignore error returns, some firmware does it this way... */ 1590 element.flags = I40E_AQC_MACVLAN_DEL_PERFECT_MATCH; 1591 i40e_aq_remove_macvlan(&pf->hw, vsi->seid, &element, 1, NULL); 1592 1593 memset(&element, 0, sizeof(element)); 1594 ether_addr_copy(element.mac_addr, macaddr); 1595 element.vlan_tag = 0; 1596 /* ...and some firmware does it this way. */ 1597 element.flags = I40E_AQC_MACVLAN_DEL_PERFECT_MATCH | 1598 I40E_AQC_MACVLAN_DEL_IGNORE_VLAN; 1599 i40e_aq_remove_macvlan(&pf->hw, vsi->seid, &element, 1, NULL); 1600 } 1601 1602 /** 1603 * i40e_add_filter - Add a mac/vlan filter to the VSI 1604 * @vsi: the VSI to be searched 1605 * @macaddr: the MAC address 1606 * @vlan: the vlan 1607 * 1608 * Returns ptr to the filter object or NULL when no memory available. 1609 * 1610 * NOTE: This function is expected to be called with mac_filter_hash_lock 1611 * being held. 1612 **/ 1613 struct i40e_mac_filter *i40e_add_filter(struct i40e_vsi *vsi, 1614 const u8 *macaddr, s16 vlan) 1615 { 1616 struct i40e_mac_filter *f; 1617 u64 key; 1618 1619 if (!vsi || !macaddr) 1620 return NULL; 1621 1622 f = i40e_find_filter(vsi, macaddr, vlan); 1623 if (!f) { 1624 f = kzalloc(sizeof(*f), GFP_ATOMIC); 1625 if (!f) 1626 return NULL; 1627 1628 /* Update the boolean indicating if we need to function in 1629 * VLAN mode. 1630 */ 1631 if (vlan >= 0) 1632 vsi->has_vlan_filter = true; 1633 1634 ether_addr_copy(f->macaddr, macaddr); 1635 f->vlan = vlan; 1636 f->state = I40E_FILTER_NEW; 1637 INIT_HLIST_NODE(&f->hlist); 1638 1639 key = i40e_addr_to_hkey(macaddr); 1640 hash_add(vsi->mac_filter_hash, &f->hlist, key); 1641 1642 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED; 1643 set_bit(__I40E_MACVLAN_SYNC_PENDING, vsi->back->state); 1644 } 1645 1646 /* If we're asked to add a filter that has been marked for removal, it 1647 * is safe to simply restore it to active state. __i40e_del_filter 1648 * will have simply deleted any filters which were previously marked 1649 * NEW or FAILED, so if it is currently marked REMOVE it must have 1650 * previously been ACTIVE. Since we haven't yet run the sync filters 1651 * task, just restore this filter to the ACTIVE state so that the 1652 * sync task leaves it in place 1653 */ 1654 if (f->state == I40E_FILTER_REMOVE) 1655 f->state = I40E_FILTER_ACTIVE; 1656 1657 return f; 1658 } 1659 1660 /** 1661 * __i40e_del_filter - Remove a specific filter from the VSI 1662 * @vsi: VSI to remove from 1663 * @f: the filter to remove from the list 1664 * 1665 * This function should be called instead of i40e_del_filter only if you know 1666 * the exact filter you will remove already, such as via i40e_find_filter or 1667 * i40e_find_mac. 1668 * 1669 * NOTE: This function is expected to be called with mac_filter_hash_lock 1670 * being held. 1671 * ANOTHER NOTE: This function MUST be called from within the context of 1672 * the "safe" variants of any list iterators, e.g. list_for_each_entry_safe() 1673 * instead of list_for_each_entry(). 1674 **/ 1675 void __i40e_del_filter(struct i40e_vsi *vsi, struct i40e_mac_filter *f) 1676 { 1677 if (!f) 1678 return; 1679 1680 /* If the filter was never added to firmware then we can just delete it 1681 * directly and we don't want to set the status to remove or else an 1682 * admin queue command will unnecessarily fire. 1683 */ 1684 if ((f->state == I40E_FILTER_FAILED) || 1685 (f->state == I40E_FILTER_NEW)) { 1686 hash_del(&f->hlist); 1687 kfree(f); 1688 } else { 1689 f->state = I40E_FILTER_REMOVE; 1690 } 1691 1692 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED; 1693 set_bit(__I40E_MACVLAN_SYNC_PENDING, vsi->back->state); 1694 } 1695 1696 /** 1697 * i40e_del_filter - Remove a MAC/VLAN filter from the VSI 1698 * @vsi: the VSI to be searched 1699 * @macaddr: the MAC address 1700 * @vlan: the VLAN 1701 * 1702 * NOTE: This function is expected to be called with mac_filter_hash_lock 1703 * being held. 1704 * ANOTHER NOTE: This function MUST be called from within the context of 1705 * the "safe" variants of any list iterators, e.g. list_for_each_entry_safe() 1706 * instead of list_for_each_entry(). 1707 **/ 1708 void i40e_del_filter(struct i40e_vsi *vsi, const u8 *macaddr, s16 vlan) 1709 { 1710 struct i40e_mac_filter *f; 1711 1712 if (!vsi || !macaddr) 1713 return; 1714 1715 f = i40e_find_filter(vsi, macaddr, vlan); 1716 __i40e_del_filter(vsi, f); 1717 } 1718 1719 /** 1720 * i40e_add_mac_filter - Add a MAC filter for all active VLANs 1721 * @vsi: the VSI to be searched 1722 * @macaddr: the mac address to be filtered 1723 * 1724 * If we're not in VLAN mode, just add the filter to I40E_VLAN_ANY. Otherwise, 1725 * go through all the macvlan filters and add a macvlan filter for each 1726 * unique vlan that already exists. If a PVID has been assigned, instead only 1727 * add the macaddr to that VLAN. 1728 * 1729 * Returns last filter added on success, else NULL 1730 **/ 1731 struct i40e_mac_filter *i40e_add_mac_filter(struct i40e_vsi *vsi, 1732 const u8 *macaddr) 1733 { 1734 struct i40e_mac_filter *f, *add = NULL; 1735 struct hlist_node *h; 1736 int bkt; 1737 1738 if (vsi->info.pvid) 1739 return i40e_add_filter(vsi, macaddr, 1740 le16_to_cpu(vsi->info.pvid)); 1741 1742 if (!i40e_is_vsi_in_vlan(vsi)) 1743 return i40e_add_filter(vsi, macaddr, I40E_VLAN_ANY); 1744 1745 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) { 1746 if (f->state == I40E_FILTER_REMOVE) 1747 continue; 1748 add = i40e_add_filter(vsi, macaddr, f->vlan); 1749 if (!add) 1750 return NULL; 1751 } 1752 1753 return add; 1754 } 1755 1756 /** 1757 * i40e_del_mac_filter - Remove a MAC filter from all VLANs 1758 * @vsi: the VSI to be searched 1759 * @macaddr: the mac address to be removed 1760 * 1761 * Removes a given MAC address from a VSI regardless of what VLAN it has been 1762 * associated with. 1763 * 1764 * Returns 0 for success, or error 1765 **/ 1766 int i40e_del_mac_filter(struct i40e_vsi *vsi, const u8 *macaddr) 1767 { 1768 struct i40e_mac_filter *f; 1769 struct hlist_node *h; 1770 bool found = false; 1771 int bkt; 1772 1773 lockdep_assert_held(&vsi->mac_filter_hash_lock); 1774 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) { 1775 if (ether_addr_equal(macaddr, f->macaddr)) { 1776 __i40e_del_filter(vsi, f); 1777 found = true; 1778 } 1779 } 1780 1781 if (found) 1782 return 0; 1783 else 1784 return -ENOENT; 1785 } 1786 1787 /** 1788 * i40e_set_mac - NDO callback to set mac address 1789 * @netdev: network interface device structure 1790 * @p: pointer to an address structure 1791 * 1792 * Returns 0 on success, negative on failure 1793 **/ 1794 static int i40e_set_mac(struct net_device *netdev, void *p) 1795 { 1796 struct i40e_netdev_priv *np = netdev_priv(netdev); 1797 struct i40e_vsi *vsi = np->vsi; 1798 struct i40e_pf *pf = vsi->back; 1799 struct i40e_hw *hw = &pf->hw; 1800 struct sockaddr *addr = p; 1801 1802 if (!is_valid_ether_addr(addr->sa_data)) 1803 return -EADDRNOTAVAIL; 1804 1805 if (test_bit(__I40E_DOWN, pf->state) || 1806 test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state)) 1807 return -EADDRNOTAVAIL; 1808 1809 if (ether_addr_equal(hw->mac.addr, addr->sa_data)) 1810 netdev_info(netdev, "returning to hw mac address %pM\n", 1811 hw->mac.addr); 1812 else 1813 netdev_info(netdev, "set new mac address %pM\n", addr->sa_data); 1814 1815 /* Copy the address first, so that we avoid a possible race with 1816 * .set_rx_mode(). 1817 * - Remove old address from MAC filter 1818 * - Copy new address 1819 * - Add new address to MAC filter 1820 */ 1821 spin_lock_bh(&vsi->mac_filter_hash_lock); 1822 i40e_del_mac_filter(vsi, netdev->dev_addr); 1823 eth_hw_addr_set(netdev, addr->sa_data); 1824 i40e_add_mac_filter(vsi, netdev->dev_addr); 1825 spin_unlock_bh(&vsi->mac_filter_hash_lock); 1826 1827 if (vsi->type == I40E_VSI_MAIN) { 1828 int ret; 1829 1830 ret = i40e_aq_mac_address_write(hw, I40E_AQC_WRITE_TYPE_LAA_WOL, 1831 addr->sa_data, NULL); 1832 if (ret) 1833 netdev_info(netdev, "Ignoring error from firmware on LAA update, status %pe, AQ ret %s\n", 1834 ERR_PTR(ret), 1835 i40e_aq_str(hw, hw->aq.asq_last_status)); 1836 } 1837 1838 /* schedule our worker thread which will take care of 1839 * applying the new filter changes 1840 */ 1841 i40e_service_event_schedule(pf); 1842 return 0; 1843 } 1844 1845 /** 1846 * i40e_config_rss_aq - Prepare for RSS using AQ commands 1847 * @vsi: vsi structure 1848 * @seed: RSS hash seed 1849 * @lut: pointer to lookup table of lut_size 1850 * @lut_size: size of the lookup table 1851 **/ 1852 static int i40e_config_rss_aq(struct i40e_vsi *vsi, const u8 *seed, 1853 u8 *lut, u16 lut_size) 1854 { 1855 struct i40e_pf *pf = vsi->back; 1856 struct i40e_hw *hw = &pf->hw; 1857 int ret = 0; 1858 1859 if (seed) { 1860 struct i40e_aqc_get_set_rss_key_data *seed_dw = 1861 (struct i40e_aqc_get_set_rss_key_data *)seed; 1862 ret = i40e_aq_set_rss_key(hw, vsi->id, seed_dw); 1863 if (ret) { 1864 dev_info(&pf->pdev->dev, 1865 "Cannot set RSS key, err %pe aq_err %s\n", 1866 ERR_PTR(ret), 1867 i40e_aq_str(hw, hw->aq.asq_last_status)); 1868 return ret; 1869 } 1870 } 1871 if (lut) { 1872 bool pf_lut = vsi->type == I40E_VSI_MAIN; 1873 1874 ret = i40e_aq_set_rss_lut(hw, vsi->id, pf_lut, lut, lut_size); 1875 if (ret) { 1876 dev_info(&pf->pdev->dev, 1877 "Cannot set RSS lut, err %pe aq_err %s\n", 1878 ERR_PTR(ret), 1879 i40e_aq_str(hw, hw->aq.asq_last_status)); 1880 return ret; 1881 } 1882 } 1883 return ret; 1884 } 1885 1886 /** 1887 * i40e_vsi_config_rss - Prepare for VSI(VMDq) RSS if used 1888 * @vsi: VSI structure 1889 **/ 1890 static int i40e_vsi_config_rss(struct i40e_vsi *vsi) 1891 { 1892 struct i40e_pf *pf = vsi->back; 1893 u8 seed[I40E_HKEY_ARRAY_SIZE]; 1894 u8 *lut; 1895 int ret; 1896 1897 if (!test_bit(I40E_HW_CAP_RSS_AQ, pf->hw.caps)) 1898 return 0; 1899 if (!vsi->rss_size) 1900 vsi->rss_size = min_t(int, pf->alloc_rss_size, 1901 vsi->num_queue_pairs); 1902 if (!vsi->rss_size) 1903 return -EINVAL; 1904 lut = kzalloc(vsi->rss_table_size, GFP_KERNEL); 1905 if (!lut) 1906 return -ENOMEM; 1907 1908 /* Use the user configured hash keys and lookup table if there is one, 1909 * otherwise use default 1910 */ 1911 if (vsi->rss_lut_user) 1912 memcpy(lut, vsi->rss_lut_user, vsi->rss_table_size); 1913 else 1914 i40e_fill_rss_lut(pf, lut, vsi->rss_table_size, vsi->rss_size); 1915 if (vsi->rss_hkey_user) 1916 memcpy(seed, vsi->rss_hkey_user, I40E_HKEY_ARRAY_SIZE); 1917 else 1918 netdev_rss_key_fill((void *)seed, I40E_HKEY_ARRAY_SIZE); 1919 ret = i40e_config_rss_aq(vsi, seed, lut, vsi->rss_table_size); 1920 kfree(lut); 1921 return ret; 1922 } 1923 1924 /** 1925 * i40e_vsi_setup_queue_map_mqprio - Prepares mqprio based tc_config 1926 * @vsi: the VSI being configured, 1927 * @ctxt: VSI context structure 1928 * @enabled_tc: number of traffic classes to enable 1929 * 1930 * Prepares VSI tc_config to have queue configurations based on MQPRIO options. 1931 **/ 1932 static int i40e_vsi_setup_queue_map_mqprio(struct i40e_vsi *vsi, 1933 struct i40e_vsi_context *ctxt, 1934 u8 enabled_tc) 1935 { 1936 u16 qcount = 0, max_qcount, qmap, sections = 0; 1937 int i, override_q, pow, num_qps, ret; 1938 u8 netdev_tc = 0, offset = 0; 1939 1940 if (vsi->type != I40E_VSI_MAIN) 1941 return -EINVAL; 1942 sections = I40E_AQ_VSI_PROP_QUEUE_MAP_VALID; 1943 sections |= I40E_AQ_VSI_PROP_SCHED_VALID; 1944 vsi->tc_config.numtc = vsi->mqprio_qopt.qopt.num_tc; 1945 vsi->tc_config.enabled_tc = enabled_tc ? enabled_tc : 1; 1946 num_qps = vsi->mqprio_qopt.qopt.count[0]; 1947 1948 /* find the next higher power-of-2 of num queue pairs */ 1949 pow = ilog2(num_qps); 1950 if (!is_power_of_2(num_qps)) 1951 pow++; 1952 qmap = (offset << I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT) | 1953 (pow << I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT); 1954 1955 /* Setup queue offset/count for all TCs for given VSI */ 1956 max_qcount = vsi->mqprio_qopt.qopt.count[0]; 1957 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 1958 /* See if the given TC is enabled for the given VSI */ 1959 if (vsi->tc_config.enabled_tc & BIT(i)) { 1960 offset = vsi->mqprio_qopt.qopt.offset[i]; 1961 qcount = vsi->mqprio_qopt.qopt.count[i]; 1962 if (qcount > max_qcount) 1963 max_qcount = qcount; 1964 vsi->tc_config.tc_info[i].qoffset = offset; 1965 vsi->tc_config.tc_info[i].qcount = qcount; 1966 vsi->tc_config.tc_info[i].netdev_tc = netdev_tc++; 1967 } else { 1968 /* TC is not enabled so set the offset to 1969 * default queue and allocate one queue 1970 * for the given TC. 1971 */ 1972 vsi->tc_config.tc_info[i].qoffset = 0; 1973 vsi->tc_config.tc_info[i].qcount = 1; 1974 vsi->tc_config.tc_info[i].netdev_tc = 0; 1975 } 1976 } 1977 1978 /* Set actual Tx/Rx queue pairs */ 1979 vsi->num_queue_pairs = offset + qcount; 1980 1981 /* Setup queue TC[0].qmap for given VSI context */ 1982 ctxt->info.tc_mapping[0] = cpu_to_le16(qmap); 1983 ctxt->info.mapping_flags |= cpu_to_le16(I40E_AQ_VSI_QUE_MAP_CONTIG); 1984 ctxt->info.queue_mapping[0] = cpu_to_le16(vsi->base_queue); 1985 ctxt->info.valid_sections |= cpu_to_le16(sections); 1986 1987 /* Reconfigure RSS for main VSI with max queue count */ 1988 vsi->rss_size = max_qcount; 1989 ret = i40e_vsi_config_rss(vsi); 1990 if (ret) { 1991 dev_info(&vsi->back->pdev->dev, 1992 "Failed to reconfig rss for num_queues (%u)\n", 1993 max_qcount); 1994 return ret; 1995 } 1996 vsi->reconfig_rss = true; 1997 dev_dbg(&vsi->back->pdev->dev, 1998 "Reconfigured rss with num_queues (%u)\n", max_qcount); 1999 2000 /* Find queue count available for channel VSIs and starting offset 2001 * for channel VSIs 2002 */ 2003 override_q = vsi->mqprio_qopt.qopt.count[0]; 2004 if (override_q && override_q < vsi->num_queue_pairs) { 2005 vsi->cnt_q_avail = vsi->num_queue_pairs - override_q; 2006 vsi->next_base_queue = override_q; 2007 } 2008 return 0; 2009 } 2010 2011 /** 2012 * i40e_vsi_setup_queue_map - Setup a VSI queue map based on enabled_tc 2013 * @vsi: the VSI being setup 2014 * @ctxt: VSI context structure 2015 * @enabled_tc: Enabled TCs bitmap 2016 * @is_add: True if called before Add VSI 2017 * 2018 * Setup VSI queue mapping for enabled traffic classes. 2019 **/ 2020 static void i40e_vsi_setup_queue_map(struct i40e_vsi *vsi, 2021 struct i40e_vsi_context *ctxt, 2022 u8 enabled_tc, 2023 bool is_add) 2024 { 2025 struct i40e_pf *pf = vsi->back; 2026 u16 num_tc_qps = 0; 2027 u16 sections = 0; 2028 u8 netdev_tc = 0; 2029 u16 numtc = 1; 2030 u16 qcount; 2031 u8 offset; 2032 u16 qmap; 2033 int i; 2034 2035 sections = I40E_AQ_VSI_PROP_QUEUE_MAP_VALID; 2036 offset = 0; 2037 /* zero out queue mapping, it will get updated on the end of the function */ 2038 memset(ctxt->info.queue_mapping, 0, sizeof(ctxt->info.queue_mapping)); 2039 2040 if (vsi->type == I40E_VSI_MAIN) { 2041 /* This code helps add more queue to the VSI if we have 2042 * more cores than RSS can support, the higher cores will 2043 * be served by ATR or other filters. Furthermore, the 2044 * non-zero req_queue_pairs says that user requested a new 2045 * queue count via ethtool's set_channels, so use this 2046 * value for queues distribution across traffic classes 2047 * We need at least one queue pair for the interface 2048 * to be usable as we see in else statement. 2049 */ 2050 if (vsi->req_queue_pairs > 0) 2051 vsi->num_queue_pairs = vsi->req_queue_pairs; 2052 else if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags)) 2053 vsi->num_queue_pairs = pf->num_lan_msix; 2054 else 2055 vsi->num_queue_pairs = 1; 2056 } 2057 2058 /* Number of queues per enabled TC */ 2059 if (vsi->type == I40E_VSI_MAIN || 2060 (vsi->type == I40E_VSI_SRIOV && vsi->num_queue_pairs != 0)) 2061 num_tc_qps = vsi->num_queue_pairs; 2062 else 2063 num_tc_qps = vsi->alloc_queue_pairs; 2064 2065 if (enabled_tc && test_bit(I40E_FLAG_DCB_ENA, vsi->back->flags)) { 2066 /* Find numtc from enabled TC bitmap */ 2067 for (i = 0, numtc = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 2068 if (enabled_tc & BIT(i)) /* TC is enabled */ 2069 numtc++; 2070 } 2071 if (!numtc) { 2072 dev_warn(&pf->pdev->dev, "DCB is enabled but no TC enabled, forcing TC0\n"); 2073 numtc = 1; 2074 } 2075 num_tc_qps = num_tc_qps / numtc; 2076 num_tc_qps = min_t(int, num_tc_qps, 2077 i40e_pf_get_max_q_per_tc(pf)); 2078 } 2079 2080 vsi->tc_config.numtc = numtc; 2081 vsi->tc_config.enabled_tc = enabled_tc ? enabled_tc : 1; 2082 2083 /* Do not allow use more TC queue pairs than MSI-X vectors exist */ 2084 if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags)) 2085 num_tc_qps = min_t(int, num_tc_qps, pf->num_lan_msix); 2086 2087 /* Setup queue offset/count for all TCs for given VSI */ 2088 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 2089 /* See if the given TC is enabled for the given VSI */ 2090 if (vsi->tc_config.enabled_tc & BIT(i)) { 2091 /* TC is enabled */ 2092 int pow, num_qps; 2093 2094 switch (vsi->type) { 2095 case I40E_VSI_MAIN: 2096 if ((!test_bit(I40E_FLAG_FD_SB_ENA, 2097 pf->flags) && 2098 !test_bit(I40E_FLAG_FD_ATR_ENA, 2099 pf->flags)) || 2100 vsi->tc_config.enabled_tc != 1) { 2101 qcount = min_t(int, pf->alloc_rss_size, 2102 num_tc_qps); 2103 break; 2104 } 2105 fallthrough; 2106 case I40E_VSI_FDIR: 2107 case I40E_VSI_SRIOV: 2108 case I40E_VSI_VMDQ2: 2109 default: 2110 qcount = num_tc_qps; 2111 WARN_ON(i != 0); 2112 break; 2113 } 2114 vsi->tc_config.tc_info[i].qoffset = offset; 2115 vsi->tc_config.tc_info[i].qcount = qcount; 2116 2117 /* find the next higher power-of-2 of num queue pairs */ 2118 num_qps = qcount; 2119 pow = 0; 2120 while (num_qps && (BIT_ULL(pow) < qcount)) { 2121 pow++; 2122 num_qps >>= 1; 2123 } 2124 2125 vsi->tc_config.tc_info[i].netdev_tc = netdev_tc++; 2126 qmap = 2127 (offset << I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT) | 2128 (pow << I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT); 2129 2130 offset += qcount; 2131 } else { 2132 /* TC is not enabled so set the offset to 2133 * default queue and allocate one queue 2134 * for the given TC. 2135 */ 2136 vsi->tc_config.tc_info[i].qoffset = 0; 2137 vsi->tc_config.tc_info[i].qcount = 1; 2138 vsi->tc_config.tc_info[i].netdev_tc = 0; 2139 2140 qmap = 0; 2141 } 2142 ctxt->info.tc_mapping[i] = cpu_to_le16(qmap); 2143 } 2144 /* Do not change previously set num_queue_pairs for PFs and VFs*/ 2145 if ((vsi->type == I40E_VSI_MAIN && numtc != 1) || 2146 (vsi->type == I40E_VSI_SRIOV && vsi->num_queue_pairs == 0) || 2147 (vsi->type != I40E_VSI_MAIN && vsi->type != I40E_VSI_SRIOV)) 2148 vsi->num_queue_pairs = offset; 2149 2150 /* Scheduler section valid can only be set for ADD VSI */ 2151 if (is_add) { 2152 sections |= I40E_AQ_VSI_PROP_SCHED_VALID; 2153 2154 ctxt->info.up_enable_bits = enabled_tc; 2155 } 2156 if (vsi->type == I40E_VSI_SRIOV) { 2157 ctxt->info.mapping_flags |= 2158 cpu_to_le16(I40E_AQ_VSI_QUE_MAP_NONCONTIG); 2159 for (i = 0; i < vsi->num_queue_pairs; i++) 2160 ctxt->info.queue_mapping[i] = 2161 cpu_to_le16(vsi->base_queue + i); 2162 } else { 2163 ctxt->info.mapping_flags |= 2164 cpu_to_le16(I40E_AQ_VSI_QUE_MAP_CONTIG); 2165 ctxt->info.queue_mapping[0] = cpu_to_le16(vsi->base_queue); 2166 } 2167 ctxt->info.valid_sections |= cpu_to_le16(sections); 2168 } 2169 2170 /** 2171 * i40e_addr_sync - Callback for dev_(mc|uc)_sync to add address 2172 * @netdev: the netdevice 2173 * @addr: address to add 2174 * 2175 * Called by __dev_(mc|uc)_sync when an address needs to be added. We call 2176 * __dev_(uc|mc)_sync from .set_rx_mode and guarantee to hold the hash lock. 2177 */ 2178 static int i40e_addr_sync(struct net_device *netdev, const u8 *addr) 2179 { 2180 struct i40e_netdev_priv *np = netdev_priv(netdev); 2181 struct i40e_vsi *vsi = np->vsi; 2182 2183 if (i40e_add_mac_filter(vsi, addr)) 2184 return 0; 2185 else 2186 return -ENOMEM; 2187 } 2188 2189 /** 2190 * i40e_addr_unsync - Callback for dev_(mc|uc)_sync to remove address 2191 * @netdev: the netdevice 2192 * @addr: address to add 2193 * 2194 * Called by __dev_(mc|uc)_sync when an address needs to be removed. We call 2195 * __dev_(uc|mc)_sync from .set_rx_mode and guarantee to hold the hash lock. 2196 */ 2197 static int i40e_addr_unsync(struct net_device *netdev, const u8 *addr) 2198 { 2199 struct i40e_netdev_priv *np = netdev_priv(netdev); 2200 struct i40e_vsi *vsi = np->vsi; 2201 2202 /* Under some circumstances, we might receive a request to delete 2203 * our own device address from our uc list. Because we store the 2204 * device address in the VSI's MAC/VLAN filter list, we need to ignore 2205 * such requests and not delete our device address from this list. 2206 */ 2207 if (ether_addr_equal(addr, netdev->dev_addr)) 2208 return 0; 2209 2210 i40e_del_mac_filter(vsi, addr); 2211 2212 return 0; 2213 } 2214 2215 /** 2216 * i40e_set_rx_mode - NDO callback to set the netdev filters 2217 * @netdev: network interface device structure 2218 **/ 2219 static void i40e_set_rx_mode(struct net_device *netdev) 2220 { 2221 struct i40e_netdev_priv *np = netdev_priv(netdev); 2222 struct i40e_vsi *vsi = np->vsi; 2223 2224 spin_lock_bh(&vsi->mac_filter_hash_lock); 2225 2226 __dev_uc_sync(netdev, i40e_addr_sync, i40e_addr_unsync); 2227 __dev_mc_sync(netdev, i40e_addr_sync, i40e_addr_unsync); 2228 2229 spin_unlock_bh(&vsi->mac_filter_hash_lock); 2230 2231 /* check for other flag changes */ 2232 if (vsi->current_netdev_flags != vsi->netdev->flags) { 2233 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED; 2234 set_bit(__I40E_MACVLAN_SYNC_PENDING, vsi->back->state); 2235 } 2236 } 2237 2238 /** 2239 * i40e_undo_del_filter_entries - Undo the changes made to MAC filter entries 2240 * @vsi: Pointer to VSI struct 2241 * @from: Pointer to list which contains MAC filter entries - changes to 2242 * those entries needs to be undone. 2243 * 2244 * MAC filter entries from this list were slated for deletion. 2245 **/ 2246 static void i40e_undo_del_filter_entries(struct i40e_vsi *vsi, 2247 struct hlist_head *from) 2248 { 2249 struct i40e_mac_filter *f; 2250 struct hlist_node *h; 2251 2252 hlist_for_each_entry_safe(f, h, from, hlist) { 2253 u64 key = i40e_addr_to_hkey(f->macaddr); 2254 2255 /* Move the element back into MAC filter list*/ 2256 hlist_del(&f->hlist); 2257 hash_add(vsi->mac_filter_hash, &f->hlist, key); 2258 } 2259 } 2260 2261 /** 2262 * i40e_undo_add_filter_entries - Undo the changes made to MAC filter entries 2263 * @vsi: Pointer to vsi struct 2264 * @from: Pointer to list which contains MAC filter entries - changes to 2265 * those entries needs to be undone. 2266 * 2267 * MAC filter entries from this list were slated for addition. 2268 **/ 2269 static void i40e_undo_add_filter_entries(struct i40e_vsi *vsi, 2270 struct hlist_head *from) 2271 { 2272 struct i40e_new_mac_filter *new; 2273 struct hlist_node *h; 2274 2275 hlist_for_each_entry_safe(new, h, from, hlist) { 2276 /* We can simply free the wrapper structure */ 2277 hlist_del(&new->hlist); 2278 netdev_hw_addr_refcnt(new->f, vsi->netdev, -1); 2279 kfree(new); 2280 } 2281 } 2282 2283 /** 2284 * i40e_next_filter - Get the next non-broadcast filter from a list 2285 * @next: pointer to filter in list 2286 * 2287 * Returns the next non-broadcast filter in the list. Required so that we 2288 * ignore broadcast filters within the list, since these are not handled via 2289 * the normal firmware update path. 2290 */ 2291 static 2292 struct i40e_new_mac_filter *i40e_next_filter(struct i40e_new_mac_filter *next) 2293 { 2294 hlist_for_each_entry_continue(next, hlist) { 2295 if (!is_broadcast_ether_addr(next->f->macaddr)) 2296 return next; 2297 } 2298 2299 return NULL; 2300 } 2301 2302 /** 2303 * i40e_update_filter_state - Update filter state based on return data 2304 * from firmware 2305 * @count: Number of filters added 2306 * @add_list: return data from fw 2307 * @add_head: pointer to first filter in current batch 2308 * 2309 * MAC filter entries from list were slated to be added to device. Returns 2310 * number of successful filters. Note that 0 does NOT mean success! 2311 **/ 2312 static int 2313 i40e_update_filter_state(int count, 2314 struct i40e_aqc_add_macvlan_element_data *add_list, 2315 struct i40e_new_mac_filter *add_head) 2316 { 2317 int retval = 0; 2318 int i; 2319 2320 for (i = 0; i < count; i++) { 2321 /* Always check status of each filter. We don't need to check 2322 * the firmware return status because we pre-set the filter 2323 * status to I40E_AQC_MM_ERR_NO_RES when sending the filter 2324 * request to the adminq. Thus, if it no longer matches then 2325 * we know the filter is active. 2326 */ 2327 if (add_list[i].match_method == I40E_AQC_MM_ERR_NO_RES) { 2328 add_head->state = I40E_FILTER_FAILED; 2329 } else { 2330 add_head->state = I40E_FILTER_ACTIVE; 2331 retval++; 2332 } 2333 2334 add_head = i40e_next_filter(add_head); 2335 if (!add_head) 2336 break; 2337 } 2338 2339 return retval; 2340 } 2341 2342 /** 2343 * i40e_aqc_del_filters - Request firmware to delete a set of filters 2344 * @vsi: ptr to the VSI 2345 * @vsi_name: name to display in messages 2346 * @list: the list of filters to send to firmware 2347 * @num_del: the number of filters to delete 2348 * @retval: Set to -EIO on failure to delete 2349 * 2350 * Send a request to firmware via AdminQ to delete a set of filters. Uses 2351 * *retval instead of a return value so that success does not force ret_val to 2352 * be set to 0. This ensures that a sequence of calls to this function 2353 * preserve the previous value of *retval on successful delete. 2354 */ 2355 static 2356 void i40e_aqc_del_filters(struct i40e_vsi *vsi, const char *vsi_name, 2357 struct i40e_aqc_remove_macvlan_element_data *list, 2358 int num_del, int *retval) 2359 { 2360 struct i40e_hw *hw = &vsi->back->hw; 2361 enum i40e_admin_queue_err aq_status; 2362 int aq_ret; 2363 2364 aq_ret = i40e_aq_remove_macvlan_v2(hw, vsi->seid, list, num_del, NULL, 2365 &aq_status); 2366 2367 /* Explicitly ignore and do not report when firmware returns ENOENT */ 2368 if (aq_ret && !(aq_status == I40E_AQ_RC_ENOENT)) { 2369 *retval = -EIO; 2370 dev_info(&vsi->back->pdev->dev, 2371 "ignoring delete macvlan error on %s, err %pe, aq_err %s\n", 2372 vsi_name, ERR_PTR(aq_ret), 2373 i40e_aq_str(hw, aq_status)); 2374 } 2375 } 2376 2377 /** 2378 * i40e_aqc_add_filters - Request firmware to add a set of filters 2379 * @vsi: ptr to the VSI 2380 * @vsi_name: name to display in messages 2381 * @list: the list of filters to send to firmware 2382 * @add_head: Position in the add hlist 2383 * @num_add: the number of filters to add 2384 * 2385 * Send a request to firmware via AdminQ to add a chunk of filters. Will set 2386 * __I40E_VSI_OVERFLOW_PROMISC bit in vsi->state if the firmware has run out of 2387 * space for more filters. 2388 */ 2389 static 2390 void i40e_aqc_add_filters(struct i40e_vsi *vsi, const char *vsi_name, 2391 struct i40e_aqc_add_macvlan_element_data *list, 2392 struct i40e_new_mac_filter *add_head, 2393 int num_add) 2394 { 2395 struct i40e_hw *hw = &vsi->back->hw; 2396 enum i40e_admin_queue_err aq_status; 2397 int fcnt; 2398 2399 i40e_aq_add_macvlan_v2(hw, vsi->seid, list, num_add, NULL, &aq_status); 2400 fcnt = i40e_update_filter_state(num_add, list, add_head); 2401 2402 if (fcnt != num_add) { 2403 if (vsi->type == I40E_VSI_MAIN) { 2404 set_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state); 2405 dev_warn(&vsi->back->pdev->dev, 2406 "Error %s adding RX filters on %s, promiscuous mode forced on\n", 2407 i40e_aq_str(hw, aq_status), vsi_name); 2408 } else if (vsi->type == I40E_VSI_SRIOV || 2409 vsi->type == I40E_VSI_VMDQ1 || 2410 vsi->type == I40E_VSI_VMDQ2) { 2411 dev_warn(&vsi->back->pdev->dev, 2412 "Error %s adding RX filters on %s, please set promiscuous on manually for %s\n", 2413 i40e_aq_str(hw, aq_status), vsi_name, 2414 vsi_name); 2415 } else { 2416 dev_warn(&vsi->back->pdev->dev, 2417 "Error %s adding RX filters on %s, incorrect VSI type: %i.\n", 2418 i40e_aq_str(hw, aq_status), vsi_name, 2419 vsi->type); 2420 } 2421 } 2422 } 2423 2424 /** 2425 * i40e_aqc_broadcast_filter - Set promiscuous broadcast flags 2426 * @vsi: pointer to the VSI 2427 * @vsi_name: the VSI name 2428 * @f: filter data 2429 * 2430 * This function sets or clears the promiscuous broadcast flags for VLAN 2431 * filters in order to properly receive broadcast frames. Assumes that only 2432 * broadcast filters are passed. 2433 * 2434 * Returns status indicating success or failure; 2435 **/ 2436 static int 2437 i40e_aqc_broadcast_filter(struct i40e_vsi *vsi, const char *vsi_name, 2438 struct i40e_mac_filter *f) 2439 { 2440 bool enable = f->state == I40E_FILTER_NEW; 2441 struct i40e_hw *hw = &vsi->back->hw; 2442 int aq_ret; 2443 2444 if (f->vlan == I40E_VLAN_ANY) { 2445 aq_ret = i40e_aq_set_vsi_broadcast(hw, 2446 vsi->seid, 2447 enable, 2448 NULL); 2449 } else { 2450 aq_ret = i40e_aq_set_vsi_bc_promisc_on_vlan(hw, 2451 vsi->seid, 2452 enable, 2453 f->vlan, 2454 NULL); 2455 } 2456 2457 if (aq_ret) { 2458 set_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state); 2459 dev_warn(&vsi->back->pdev->dev, 2460 "Error %s, forcing overflow promiscuous on %s\n", 2461 i40e_aq_str(hw, hw->aq.asq_last_status), 2462 vsi_name); 2463 } 2464 2465 return aq_ret; 2466 } 2467 2468 /** 2469 * i40e_set_promiscuous - set promiscuous mode 2470 * @pf: board private structure 2471 * @promisc: promisc on or off 2472 * 2473 * There are different ways of setting promiscuous mode on a PF depending on 2474 * what state/environment we're in. This identifies and sets it appropriately. 2475 * Returns 0 on success. 2476 **/ 2477 static int i40e_set_promiscuous(struct i40e_pf *pf, bool promisc) 2478 { 2479 struct i40e_vsi *vsi = i40e_pf_get_main_vsi(pf); 2480 struct i40e_hw *hw = &pf->hw; 2481 int aq_ret; 2482 2483 if (vsi->type == I40E_VSI_MAIN && 2484 i40e_pf_get_main_veb(pf) && 2485 !test_bit(I40E_FLAG_MFP_ENA, pf->flags)) { 2486 /* set defport ON for Main VSI instead of true promisc 2487 * this way we will get all unicast/multicast and VLAN 2488 * promisc behavior but will not get VF or VMDq traffic 2489 * replicated on the Main VSI. 2490 */ 2491 if (promisc) 2492 aq_ret = i40e_aq_set_default_vsi(hw, 2493 vsi->seid, 2494 NULL); 2495 else 2496 aq_ret = i40e_aq_clear_default_vsi(hw, 2497 vsi->seid, 2498 NULL); 2499 if (aq_ret) { 2500 dev_info(&pf->pdev->dev, 2501 "Set default VSI failed, err %pe, aq_err %s\n", 2502 ERR_PTR(aq_ret), 2503 i40e_aq_str(hw, hw->aq.asq_last_status)); 2504 } 2505 } else { 2506 aq_ret = i40e_aq_set_vsi_unicast_promiscuous( 2507 hw, 2508 vsi->seid, 2509 promisc, NULL, 2510 true); 2511 if (aq_ret) { 2512 dev_info(&pf->pdev->dev, 2513 "set unicast promisc failed, err %pe, aq_err %s\n", 2514 ERR_PTR(aq_ret), 2515 i40e_aq_str(hw, hw->aq.asq_last_status)); 2516 } 2517 aq_ret = i40e_aq_set_vsi_multicast_promiscuous( 2518 hw, 2519 vsi->seid, 2520 promisc, NULL); 2521 if (aq_ret) { 2522 dev_info(&pf->pdev->dev, 2523 "set multicast promisc failed, err %pe, aq_err %s\n", 2524 ERR_PTR(aq_ret), 2525 i40e_aq_str(hw, hw->aq.asq_last_status)); 2526 } 2527 } 2528 2529 if (!aq_ret) 2530 pf->cur_promisc = promisc; 2531 2532 return aq_ret; 2533 } 2534 2535 /** 2536 * i40e_sync_vsi_filters - Update the VSI filter list to the HW 2537 * @vsi: ptr to the VSI 2538 * 2539 * Push any outstanding VSI filter changes through the AdminQ. 2540 * 2541 * Returns 0 or error value 2542 **/ 2543 int i40e_sync_vsi_filters(struct i40e_vsi *vsi) 2544 { 2545 struct hlist_head tmp_add_list, tmp_del_list; 2546 struct i40e_mac_filter *f; 2547 struct i40e_new_mac_filter *new, *add_head = NULL; 2548 struct i40e_hw *hw = &vsi->back->hw; 2549 bool old_overflow, new_overflow; 2550 unsigned int failed_filters = 0; 2551 unsigned int vlan_filters = 0; 2552 char vsi_name[16] = "PF"; 2553 int filter_list_len = 0; 2554 u32 changed_flags = 0; 2555 struct hlist_node *h; 2556 struct i40e_pf *pf; 2557 int num_add = 0; 2558 int num_del = 0; 2559 int aq_ret = 0; 2560 int retval = 0; 2561 u16 cmd_flags; 2562 int list_size; 2563 int bkt; 2564 2565 /* empty array typed pointers, kcalloc later */ 2566 struct i40e_aqc_add_macvlan_element_data *add_list; 2567 struct i40e_aqc_remove_macvlan_element_data *del_list; 2568 2569 while (test_and_set_bit(__I40E_VSI_SYNCING_FILTERS, vsi->state)) 2570 usleep_range(1000, 2000); 2571 pf = vsi->back; 2572 2573 old_overflow = test_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state); 2574 2575 if (vsi->netdev) { 2576 changed_flags = vsi->current_netdev_flags ^ vsi->netdev->flags; 2577 vsi->current_netdev_flags = vsi->netdev->flags; 2578 } 2579 2580 INIT_HLIST_HEAD(&tmp_add_list); 2581 INIT_HLIST_HEAD(&tmp_del_list); 2582 2583 if (vsi->type == I40E_VSI_SRIOV) 2584 snprintf(vsi_name, sizeof(vsi_name) - 1, "VF %d", vsi->vf_id); 2585 else if (vsi->type != I40E_VSI_MAIN) 2586 snprintf(vsi_name, sizeof(vsi_name) - 1, "vsi %d", vsi->seid); 2587 2588 if (vsi->flags & I40E_VSI_FLAG_FILTER_CHANGED) { 2589 vsi->flags &= ~I40E_VSI_FLAG_FILTER_CHANGED; 2590 2591 spin_lock_bh(&vsi->mac_filter_hash_lock); 2592 /* Create a list of filters to delete. */ 2593 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) { 2594 if (f->state == I40E_FILTER_REMOVE) { 2595 /* Move the element into temporary del_list */ 2596 hash_del(&f->hlist); 2597 hlist_add_head(&f->hlist, &tmp_del_list); 2598 2599 /* Avoid counting removed filters */ 2600 continue; 2601 } 2602 if (f->state == I40E_FILTER_NEW) { 2603 /* Create a temporary i40e_new_mac_filter */ 2604 new = kzalloc(sizeof(*new), GFP_ATOMIC); 2605 if (!new) 2606 goto err_no_memory_locked; 2607 2608 /* Store pointer to the real filter */ 2609 new->f = f; 2610 new->state = f->state; 2611 2612 /* Add it to the hash list */ 2613 hlist_add_head(&new->hlist, &tmp_add_list); 2614 } 2615 2616 /* Count the number of active (current and new) VLAN 2617 * filters we have now. Does not count filters which 2618 * are marked for deletion. 2619 */ 2620 if (f->vlan > 0) 2621 vlan_filters++; 2622 } 2623 2624 if (vsi->type != I40E_VSI_SRIOV) 2625 retval = i40e_correct_mac_vlan_filters 2626 (vsi, &tmp_add_list, &tmp_del_list, 2627 vlan_filters); 2628 else if (pf->vf) 2629 retval = i40e_correct_vf_mac_vlan_filters 2630 (vsi, &tmp_add_list, &tmp_del_list, 2631 vlan_filters, pf->vf[vsi->vf_id].trusted); 2632 2633 hlist_for_each_entry(new, &tmp_add_list, hlist) 2634 netdev_hw_addr_refcnt(new->f, vsi->netdev, 1); 2635 2636 if (retval) 2637 goto err_no_memory_locked; 2638 2639 spin_unlock_bh(&vsi->mac_filter_hash_lock); 2640 } 2641 2642 /* Now process 'del_list' outside the lock */ 2643 if (!hlist_empty(&tmp_del_list)) { 2644 filter_list_len = hw->aq.asq_buf_size / 2645 sizeof(struct i40e_aqc_remove_macvlan_element_data); 2646 list_size = filter_list_len * 2647 sizeof(struct i40e_aqc_remove_macvlan_element_data); 2648 del_list = kzalloc(list_size, GFP_ATOMIC); 2649 if (!del_list) 2650 goto err_no_memory; 2651 2652 hlist_for_each_entry_safe(f, h, &tmp_del_list, hlist) { 2653 cmd_flags = 0; 2654 2655 /* handle broadcast filters by updating the broadcast 2656 * promiscuous flag and release filter list. 2657 */ 2658 if (is_broadcast_ether_addr(f->macaddr)) { 2659 i40e_aqc_broadcast_filter(vsi, vsi_name, f); 2660 2661 hlist_del(&f->hlist); 2662 kfree(f); 2663 continue; 2664 } 2665 2666 /* add to delete list */ 2667 ether_addr_copy(del_list[num_del].mac_addr, f->macaddr); 2668 if (f->vlan == I40E_VLAN_ANY) { 2669 del_list[num_del].vlan_tag = 0; 2670 cmd_flags |= I40E_AQC_MACVLAN_DEL_IGNORE_VLAN; 2671 } else { 2672 del_list[num_del].vlan_tag = 2673 cpu_to_le16((u16)(f->vlan)); 2674 } 2675 2676 cmd_flags |= I40E_AQC_MACVLAN_DEL_PERFECT_MATCH; 2677 del_list[num_del].flags = cmd_flags; 2678 num_del++; 2679 2680 /* flush a full buffer */ 2681 if (num_del == filter_list_len) { 2682 i40e_aqc_del_filters(vsi, vsi_name, del_list, 2683 num_del, &retval); 2684 memset(del_list, 0, list_size); 2685 num_del = 0; 2686 } 2687 /* Release memory for MAC filter entries which were 2688 * synced up with HW. 2689 */ 2690 hlist_del(&f->hlist); 2691 kfree(f); 2692 } 2693 2694 if (num_del) { 2695 i40e_aqc_del_filters(vsi, vsi_name, del_list, 2696 num_del, &retval); 2697 } 2698 2699 kfree(del_list); 2700 del_list = NULL; 2701 } 2702 2703 if (!hlist_empty(&tmp_add_list)) { 2704 /* Do all the adds now. */ 2705 filter_list_len = hw->aq.asq_buf_size / 2706 sizeof(struct i40e_aqc_add_macvlan_element_data); 2707 list_size = filter_list_len * 2708 sizeof(struct i40e_aqc_add_macvlan_element_data); 2709 add_list = kzalloc(list_size, GFP_ATOMIC); 2710 if (!add_list) 2711 goto err_no_memory; 2712 2713 num_add = 0; 2714 hlist_for_each_entry_safe(new, h, &tmp_add_list, hlist) { 2715 /* handle broadcast filters by updating the broadcast 2716 * promiscuous flag instead of adding a MAC filter. 2717 */ 2718 if (is_broadcast_ether_addr(new->f->macaddr)) { 2719 if (i40e_aqc_broadcast_filter(vsi, vsi_name, 2720 new->f)) 2721 new->state = I40E_FILTER_FAILED; 2722 else 2723 new->state = I40E_FILTER_ACTIVE; 2724 continue; 2725 } 2726 2727 /* add to add array */ 2728 if (num_add == 0) 2729 add_head = new; 2730 cmd_flags = 0; 2731 ether_addr_copy(add_list[num_add].mac_addr, 2732 new->f->macaddr); 2733 if (new->f->vlan == I40E_VLAN_ANY) { 2734 add_list[num_add].vlan_tag = 0; 2735 cmd_flags |= I40E_AQC_MACVLAN_ADD_IGNORE_VLAN; 2736 } else { 2737 add_list[num_add].vlan_tag = 2738 cpu_to_le16((u16)(new->f->vlan)); 2739 } 2740 add_list[num_add].queue_number = 0; 2741 /* set invalid match method for later detection */ 2742 add_list[num_add].match_method = I40E_AQC_MM_ERR_NO_RES; 2743 cmd_flags |= I40E_AQC_MACVLAN_ADD_PERFECT_MATCH; 2744 add_list[num_add].flags = cpu_to_le16(cmd_flags); 2745 num_add++; 2746 2747 /* flush a full buffer */ 2748 if (num_add == filter_list_len) { 2749 i40e_aqc_add_filters(vsi, vsi_name, add_list, 2750 add_head, num_add); 2751 memset(add_list, 0, list_size); 2752 num_add = 0; 2753 } 2754 } 2755 if (num_add) { 2756 i40e_aqc_add_filters(vsi, vsi_name, add_list, add_head, 2757 num_add); 2758 } 2759 /* Now move all of the filters from the temp add list back to 2760 * the VSI's list. 2761 */ 2762 spin_lock_bh(&vsi->mac_filter_hash_lock); 2763 hlist_for_each_entry_safe(new, h, &tmp_add_list, hlist) { 2764 /* Only update the state if we're still NEW */ 2765 if (new->f->state == I40E_FILTER_NEW) 2766 new->f->state = new->state; 2767 hlist_del(&new->hlist); 2768 netdev_hw_addr_refcnt(new->f, vsi->netdev, -1); 2769 kfree(new); 2770 } 2771 spin_unlock_bh(&vsi->mac_filter_hash_lock); 2772 kfree(add_list); 2773 add_list = NULL; 2774 } 2775 2776 /* Determine the number of active and failed filters. */ 2777 spin_lock_bh(&vsi->mac_filter_hash_lock); 2778 vsi->active_filters = 0; 2779 hash_for_each(vsi->mac_filter_hash, bkt, f, hlist) { 2780 if (f->state == I40E_FILTER_ACTIVE) 2781 vsi->active_filters++; 2782 else if (f->state == I40E_FILTER_FAILED) 2783 failed_filters++; 2784 } 2785 spin_unlock_bh(&vsi->mac_filter_hash_lock); 2786 2787 /* Check if we are able to exit overflow promiscuous mode. We can 2788 * safely exit if we didn't just enter, we no longer have any failed 2789 * filters, and we have reduced filters below the threshold value. 2790 */ 2791 if (old_overflow && !failed_filters && 2792 vsi->active_filters < vsi->promisc_threshold) { 2793 dev_info(&pf->pdev->dev, 2794 "filter logjam cleared on %s, leaving overflow promiscuous mode\n", 2795 vsi_name); 2796 clear_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state); 2797 vsi->promisc_threshold = 0; 2798 } 2799 2800 /* if the VF is not trusted do not do promisc */ 2801 if (vsi->type == I40E_VSI_SRIOV && pf->vf && 2802 !pf->vf[vsi->vf_id].trusted) { 2803 clear_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state); 2804 goto out; 2805 } 2806 2807 new_overflow = test_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state); 2808 2809 /* If we are entering overflow promiscuous, we need to calculate a new 2810 * threshold for when we are safe to exit 2811 */ 2812 if (!old_overflow && new_overflow) 2813 vsi->promisc_threshold = (vsi->active_filters * 3) / 4; 2814 2815 /* check for changes in promiscuous modes */ 2816 if (changed_flags & IFF_ALLMULTI) { 2817 bool cur_multipromisc; 2818 2819 cur_multipromisc = !!(vsi->current_netdev_flags & IFF_ALLMULTI); 2820 aq_ret = i40e_aq_set_vsi_multicast_promiscuous(&vsi->back->hw, 2821 vsi->seid, 2822 cur_multipromisc, 2823 NULL); 2824 if (aq_ret) { 2825 retval = i40e_aq_rc_to_posix(aq_ret, 2826 hw->aq.asq_last_status); 2827 dev_info(&pf->pdev->dev, 2828 "set multi promisc failed on %s, err %pe aq_err %s\n", 2829 vsi_name, 2830 ERR_PTR(aq_ret), 2831 i40e_aq_str(hw, hw->aq.asq_last_status)); 2832 } else { 2833 dev_info(&pf->pdev->dev, "%s allmulti mode.\n", 2834 cur_multipromisc ? "entering" : "leaving"); 2835 } 2836 } 2837 2838 if ((changed_flags & IFF_PROMISC) || old_overflow != new_overflow) { 2839 bool cur_promisc; 2840 2841 cur_promisc = (!!(vsi->current_netdev_flags & IFF_PROMISC) || 2842 new_overflow); 2843 aq_ret = i40e_set_promiscuous(pf, cur_promisc); 2844 if (aq_ret) { 2845 retval = i40e_aq_rc_to_posix(aq_ret, 2846 hw->aq.asq_last_status); 2847 dev_info(&pf->pdev->dev, 2848 "Setting promiscuous %s failed on %s, err %pe aq_err %s\n", 2849 cur_promisc ? "on" : "off", 2850 vsi_name, 2851 ERR_PTR(aq_ret), 2852 i40e_aq_str(hw, hw->aq.asq_last_status)); 2853 } 2854 } 2855 out: 2856 /* if something went wrong then set the changed flag so we try again */ 2857 if (retval) 2858 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED; 2859 2860 clear_bit(__I40E_VSI_SYNCING_FILTERS, vsi->state); 2861 return retval; 2862 2863 err_no_memory: 2864 /* Restore elements on the temporary add and delete lists */ 2865 spin_lock_bh(&vsi->mac_filter_hash_lock); 2866 err_no_memory_locked: 2867 i40e_undo_del_filter_entries(vsi, &tmp_del_list); 2868 i40e_undo_add_filter_entries(vsi, &tmp_add_list); 2869 spin_unlock_bh(&vsi->mac_filter_hash_lock); 2870 2871 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED; 2872 clear_bit(__I40E_VSI_SYNCING_FILTERS, vsi->state); 2873 return -ENOMEM; 2874 } 2875 2876 /** 2877 * i40e_sync_filters_subtask - Sync the VSI filter list with HW 2878 * @pf: board private structure 2879 **/ 2880 static void i40e_sync_filters_subtask(struct i40e_pf *pf) 2881 { 2882 struct i40e_vsi *vsi; 2883 int v; 2884 2885 if (!pf) 2886 return; 2887 if (!test_and_clear_bit(__I40E_MACVLAN_SYNC_PENDING, pf->state)) 2888 return; 2889 if (test_bit(__I40E_VF_DISABLE, pf->state)) { 2890 set_bit(__I40E_MACVLAN_SYNC_PENDING, pf->state); 2891 return; 2892 } 2893 2894 i40e_pf_for_each_vsi(pf, v, vsi) { 2895 if ((vsi->flags & I40E_VSI_FLAG_FILTER_CHANGED) && 2896 !test_bit(__I40E_VSI_RELEASING, vsi->state)) { 2897 int ret = i40e_sync_vsi_filters(vsi); 2898 2899 if (ret) { 2900 /* come back and try again later */ 2901 set_bit(__I40E_MACVLAN_SYNC_PENDING, 2902 pf->state); 2903 break; 2904 } 2905 } 2906 } 2907 } 2908 2909 /** 2910 * i40e_calculate_vsi_rx_buf_len - Calculates buffer length 2911 * 2912 * @vsi: VSI to calculate rx_buf_len from 2913 */ 2914 static u16 i40e_calculate_vsi_rx_buf_len(struct i40e_vsi *vsi) 2915 { 2916 if (!vsi->netdev || test_bit(I40E_FLAG_LEGACY_RX_ENA, vsi->back->flags)) 2917 return SKB_WITH_OVERHEAD(I40E_RXBUFFER_2048); 2918 2919 return PAGE_SIZE < 8192 ? I40E_RXBUFFER_3072 : I40E_RXBUFFER_2048; 2920 } 2921 2922 /** 2923 * i40e_max_vsi_frame_size - returns the maximum allowed frame size for VSI 2924 * @vsi: the vsi 2925 * @xdp_prog: XDP program 2926 **/ 2927 static int i40e_max_vsi_frame_size(struct i40e_vsi *vsi, 2928 struct bpf_prog *xdp_prog) 2929 { 2930 u16 rx_buf_len = i40e_calculate_vsi_rx_buf_len(vsi); 2931 u16 chain_len; 2932 2933 if (xdp_prog && !xdp_prog->aux->xdp_has_frags) 2934 chain_len = 1; 2935 else 2936 chain_len = I40E_MAX_CHAINED_RX_BUFFERS; 2937 2938 return min_t(u16, rx_buf_len * chain_len, I40E_MAX_RXBUFFER); 2939 } 2940 2941 /** 2942 * i40e_change_mtu - NDO callback to change the Maximum Transfer Unit 2943 * @netdev: network interface device structure 2944 * @new_mtu: new value for maximum frame size 2945 * 2946 * Returns 0 on success, negative on failure 2947 **/ 2948 static int i40e_change_mtu(struct net_device *netdev, int new_mtu) 2949 { 2950 struct i40e_netdev_priv *np = netdev_priv(netdev); 2951 struct i40e_vsi *vsi = np->vsi; 2952 struct i40e_pf *pf = vsi->back; 2953 int frame_size; 2954 2955 frame_size = i40e_max_vsi_frame_size(vsi, vsi->xdp_prog); 2956 if (new_mtu > frame_size - I40E_PACKET_HDR_PAD) { 2957 netdev_err(netdev, "Error changing mtu to %d, Max is %d\n", 2958 new_mtu, frame_size - I40E_PACKET_HDR_PAD); 2959 return -EINVAL; 2960 } 2961 2962 netdev_dbg(netdev, "changing MTU from %d to %d\n", 2963 netdev->mtu, new_mtu); 2964 WRITE_ONCE(netdev->mtu, new_mtu); 2965 if (netif_running(netdev)) 2966 i40e_vsi_reinit_locked(vsi); 2967 set_bit(__I40E_CLIENT_SERVICE_REQUESTED, pf->state); 2968 set_bit(__I40E_CLIENT_L2_CHANGE, pf->state); 2969 return 0; 2970 } 2971 2972 /** 2973 * i40e_ioctl - Access the hwtstamp interface 2974 * @netdev: network interface device structure 2975 * @ifr: interface request data 2976 * @cmd: ioctl command 2977 **/ 2978 int i40e_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd) 2979 { 2980 struct i40e_netdev_priv *np = netdev_priv(netdev); 2981 struct i40e_pf *pf = np->vsi->back; 2982 2983 switch (cmd) { 2984 case SIOCGHWTSTAMP: 2985 return i40e_ptp_get_ts_config(pf, ifr); 2986 case SIOCSHWTSTAMP: 2987 return i40e_ptp_set_ts_config(pf, ifr); 2988 default: 2989 return -EOPNOTSUPP; 2990 } 2991 } 2992 2993 /** 2994 * i40e_vlan_stripping_enable - Turn on vlan stripping for the VSI 2995 * @vsi: the vsi being adjusted 2996 **/ 2997 void i40e_vlan_stripping_enable(struct i40e_vsi *vsi) 2998 { 2999 struct i40e_vsi_context ctxt; 3000 int ret; 3001 3002 /* Don't modify stripping options if a port VLAN is active */ 3003 if (vsi->info.pvid) 3004 return; 3005 3006 if ((vsi->info.valid_sections & 3007 cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID)) && 3008 ((vsi->info.port_vlan_flags & I40E_AQ_VSI_PVLAN_MODE_MASK) == 0)) 3009 return; /* already enabled */ 3010 3011 vsi->info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID); 3012 vsi->info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_ALL | 3013 I40E_AQ_VSI_PVLAN_EMOD_STR_BOTH; 3014 3015 ctxt.seid = vsi->seid; 3016 ctxt.info = vsi->info; 3017 ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL); 3018 if (ret) { 3019 dev_info(&vsi->back->pdev->dev, 3020 "update vlan stripping failed, err %pe aq_err %s\n", 3021 ERR_PTR(ret), 3022 i40e_aq_str(&vsi->back->hw, 3023 vsi->back->hw.aq.asq_last_status)); 3024 } 3025 } 3026 3027 /** 3028 * i40e_vlan_stripping_disable - Turn off vlan stripping for the VSI 3029 * @vsi: the vsi being adjusted 3030 **/ 3031 void i40e_vlan_stripping_disable(struct i40e_vsi *vsi) 3032 { 3033 struct i40e_vsi_context ctxt; 3034 int ret; 3035 3036 /* Don't modify stripping options if a port VLAN is active */ 3037 if (vsi->info.pvid) 3038 return; 3039 3040 if ((vsi->info.valid_sections & 3041 cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID)) && 3042 ((vsi->info.port_vlan_flags & I40E_AQ_VSI_PVLAN_EMOD_MASK) == 3043 I40E_AQ_VSI_PVLAN_EMOD_MASK)) 3044 return; /* already disabled */ 3045 3046 vsi->info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID); 3047 vsi->info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_ALL | 3048 I40E_AQ_VSI_PVLAN_EMOD_NOTHING; 3049 3050 ctxt.seid = vsi->seid; 3051 ctxt.info = vsi->info; 3052 ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL); 3053 if (ret) { 3054 dev_info(&vsi->back->pdev->dev, 3055 "update vlan stripping failed, err %pe aq_err %s\n", 3056 ERR_PTR(ret), 3057 i40e_aq_str(&vsi->back->hw, 3058 vsi->back->hw.aq.asq_last_status)); 3059 } 3060 } 3061 3062 /** 3063 * i40e_add_vlan_all_mac - Add a MAC/VLAN filter for each existing MAC address 3064 * @vsi: the vsi being configured 3065 * @vid: vlan id to be added (0 = untagged only , -1 = any) 3066 * 3067 * This is a helper function for adding a new MAC/VLAN filter with the 3068 * specified VLAN for each existing MAC address already in the hash table. 3069 * This function does *not* perform any accounting to update filters based on 3070 * VLAN mode. 3071 * 3072 * NOTE: this function expects to be called while under the 3073 * mac_filter_hash_lock 3074 **/ 3075 int i40e_add_vlan_all_mac(struct i40e_vsi *vsi, s16 vid) 3076 { 3077 struct i40e_mac_filter *f, *add_f; 3078 struct hlist_node *h; 3079 int bkt; 3080 3081 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) { 3082 /* If we're asked to add a filter that has been marked for 3083 * removal, it is safe to simply restore it to active state. 3084 * __i40e_del_filter will have simply deleted any filters which 3085 * were previously marked NEW or FAILED, so if it is currently 3086 * marked REMOVE it must have previously been ACTIVE. Since we 3087 * haven't yet run the sync filters task, just restore this 3088 * filter to the ACTIVE state so that the sync task leaves it 3089 * in place. 3090 */ 3091 if (f->state == I40E_FILTER_REMOVE && f->vlan == vid) { 3092 f->state = I40E_FILTER_ACTIVE; 3093 continue; 3094 } else if (f->state == I40E_FILTER_REMOVE) { 3095 continue; 3096 } 3097 add_f = i40e_add_filter(vsi, f->macaddr, vid); 3098 if (!add_f) { 3099 dev_info(&vsi->back->pdev->dev, 3100 "Could not add vlan filter %d for %pM\n", 3101 vid, f->macaddr); 3102 return -ENOMEM; 3103 } 3104 } 3105 3106 return 0; 3107 } 3108 3109 /** 3110 * i40e_vsi_add_vlan - Add VSI membership for given VLAN 3111 * @vsi: the VSI being configured 3112 * @vid: VLAN id to be added 3113 **/ 3114 int i40e_vsi_add_vlan(struct i40e_vsi *vsi, u16 vid) 3115 { 3116 int err; 3117 3118 if (vsi->info.pvid) 3119 return -EINVAL; 3120 3121 /* The network stack will attempt to add VID=0, with the intention to 3122 * receive priority tagged packets with a VLAN of 0. Our HW receives 3123 * these packets by default when configured to receive untagged 3124 * packets, so we don't need to add a filter for this case. 3125 * Additionally, HW interprets adding a VID=0 filter as meaning to 3126 * receive *only* tagged traffic and stops receiving untagged traffic. 3127 * Thus, we do not want to actually add a filter for VID=0 3128 */ 3129 if (!vid) 3130 return 0; 3131 3132 /* Locked once because all functions invoked below iterates list*/ 3133 spin_lock_bh(&vsi->mac_filter_hash_lock); 3134 err = i40e_add_vlan_all_mac(vsi, vid); 3135 spin_unlock_bh(&vsi->mac_filter_hash_lock); 3136 if (err) 3137 return err; 3138 3139 /* schedule our worker thread which will take care of 3140 * applying the new filter changes 3141 */ 3142 i40e_service_event_schedule(vsi->back); 3143 return 0; 3144 } 3145 3146 /** 3147 * i40e_rm_vlan_all_mac - Remove MAC/VLAN pair for all MAC with the given VLAN 3148 * @vsi: the vsi being configured 3149 * @vid: vlan id to be removed (0 = untagged only , -1 = any) 3150 * 3151 * This function should be used to remove all VLAN filters which match the 3152 * given VID. It does not schedule the service event and does not take the 3153 * mac_filter_hash_lock so it may be combined with other operations under 3154 * a single invocation of the mac_filter_hash_lock. 3155 * 3156 * NOTE: this function expects to be called while under the 3157 * mac_filter_hash_lock 3158 */ 3159 void i40e_rm_vlan_all_mac(struct i40e_vsi *vsi, s16 vid) 3160 { 3161 struct i40e_mac_filter *f; 3162 struct hlist_node *h; 3163 int bkt; 3164 3165 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) { 3166 if (f->vlan == vid) 3167 __i40e_del_filter(vsi, f); 3168 } 3169 } 3170 3171 /** 3172 * i40e_vsi_kill_vlan - Remove VSI membership for given VLAN 3173 * @vsi: the VSI being configured 3174 * @vid: VLAN id to be removed 3175 **/ 3176 void i40e_vsi_kill_vlan(struct i40e_vsi *vsi, u16 vid) 3177 { 3178 if (!vid || vsi->info.pvid) 3179 return; 3180 3181 spin_lock_bh(&vsi->mac_filter_hash_lock); 3182 i40e_rm_vlan_all_mac(vsi, vid); 3183 spin_unlock_bh(&vsi->mac_filter_hash_lock); 3184 3185 /* schedule our worker thread which will take care of 3186 * applying the new filter changes 3187 */ 3188 i40e_service_event_schedule(vsi->back); 3189 } 3190 3191 /** 3192 * i40e_vlan_rx_add_vid - Add a vlan id filter to HW offload 3193 * @netdev: network interface to be adjusted 3194 * @proto: unused protocol value 3195 * @vid: vlan id to be added 3196 * 3197 * net_device_ops implementation for adding vlan ids 3198 **/ 3199 static int i40e_vlan_rx_add_vid(struct net_device *netdev, 3200 __always_unused __be16 proto, u16 vid) 3201 { 3202 struct i40e_netdev_priv *np = netdev_priv(netdev); 3203 struct i40e_vsi *vsi = np->vsi; 3204 int ret = 0; 3205 3206 if (vid >= VLAN_N_VID) 3207 return -EINVAL; 3208 3209 ret = i40e_vsi_add_vlan(vsi, vid); 3210 if (!ret) 3211 set_bit(vid, vsi->active_vlans); 3212 3213 return ret; 3214 } 3215 3216 /** 3217 * i40e_vlan_rx_add_vid_up - Add a vlan id filter to HW offload in UP path 3218 * @netdev: network interface to be adjusted 3219 * @proto: unused protocol value 3220 * @vid: vlan id to be added 3221 **/ 3222 static void i40e_vlan_rx_add_vid_up(struct net_device *netdev, 3223 __always_unused __be16 proto, u16 vid) 3224 { 3225 struct i40e_netdev_priv *np = netdev_priv(netdev); 3226 struct i40e_vsi *vsi = np->vsi; 3227 3228 if (vid >= VLAN_N_VID) 3229 return; 3230 set_bit(vid, vsi->active_vlans); 3231 } 3232 3233 /** 3234 * i40e_vlan_rx_kill_vid - Remove a vlan id filter from HW offload 3235 * @netdev: network interface to be adjusted 3236 * @proto: unused protocol value 3237 * @vid: vlan id to be removed 3238 * 3239 * net_device_ops implementation for removing vlan ids 3240 **/ 3241 static int i40e_vlan_rx_kill_vid(struct net_device *netdev, 3242 __always_unused __be16 proto, u16 vid) 3243 { 3244 struct i40e_netdev_priv *np = netdev_priv(netdev); 3245 struct i40e_vsi *vsi = np->vsi; 3246 3247 /* return code is ignored as there is nothing a user 3248 * can do about failure to remove and a log message was 3249 * already printed from the other function 3250 */ 3251 i40e_vsi_kill_vlan(vsi, vid); 3252 3253 clear_bit(vid, vsi->active_vlans); 3254 3255 return 0; 3256 } 3257 3258 /** 3259 * i40e_restore_vlan - Reinstate vlans when vsi/netdev comes back up 3260 * @vsi: the vsi being brought back up 3261 **/ 3262 static void i40e_restore_vlan(struct i40e_vsi *vsi) 3263 { 3264 u16 vid; 3265 3266 if (!vsi->netdev) 3267 return; 3268 3269 if (vsi->netdev->features & NETIF_F_HW_VLAN_CTAG_RX) 3270 i40e_vlan_stripping_enable(vsi); 3271 else 3272 i40e_vlan_stripping_disable(vsi); 3273 3274 for_each_set_bit(vid, vsi->active_vlans, VLAN_N_VID) 3275 i40e_vlan_rx_add_vid_up(vsi->netdev, htons(ETH_P_8021Q), 3276 vid); 3277 } 3278 3279 /** 3280 * i40e_vsi_add_pvid - Add pvid for the VSI 3281 * @vsi: the vsi being adjusted 3282 * @vid: the vlan id to set as a PVID 3283 **/ 3284 int i40e_vsi_add_pvid(struct i40e_vsi *vsi, u16 vid) 3285 { 3286 struct i40e_vsi_context ctxt; 3287 int ret; 3288 3289 vsi->info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID); 3290 vsi->info.pvid = cpu_to_le16(vid); 3291 vsi->info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_TAGGED | 3292 I40E_AQ_VSI_PVLAN_INSERT_PVID | 3293 I40E_AQ_VSI_PVLAN_EMOD_STR; 3294 3295 ctxt.seid = vsi->seid; 3296 ctxt.info = vsi->info; 3297 ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL); 3298 if (ret) { 3299 dev_info(&vsi->back->pdev->dev, 3300 "add pvid failed, err %pe aq_err %s\n", 3301 ERR_PTR(ret), 3302 i40e_aq_str(&vsi->back->hw, 3303 vsi->back->hw.aq.asq_last_status)); 3304 return -ENOENT; 3305 } 3306 3307 return 0; 3308 } 3309 3310 /** 3311 * i40e_vsi_remove_pvid - Remove the pvid from the VSI 3312 * @vsi: the vsi being adjusted 3313 * 3314 * Just use the vlan_rx_register() service to put it back to normal 3315 **/ 3316 void i40e_vsi_remove_pvid(struct i40e_vsi *vsi) 3317 { 3318 vsi->info.pvid = 0; 3319 3320 i40e_vlan_stripping_disable(vsi); 3321 } 3322 3323 /** 3324 * i40e_vsi_setup_tx_resources - Allocate VSI Tx queue resources 3325 * @vsi: ptr to the VSI 3326 * 3327 * If this function returns with an error, then it's possible one or 3328 * more of the rings is populated (while the rest are not). It is the 3329 * callers duty to clean those orphaned rings. 3330 * 3331 * Return 0 on success, negative on failure 3332 **/ 3333 static int i40e_vsi_setup_tx_resources(struct i40e_vsi *vsi) 3334 { 3335 int i, err = 0; 3336 3337 for (i = 0; i < vsi->num_queue_pairs && !err; i++) 3338 err = i40e_setup_tx_descriptors(vsi->tx_rings[i]); 3339 3340 if (!i40e_enabled_xdp_vsi(vsi)) 3341 return err; 3342 3343 for (i = 0; i < vsi->num_queue_pairs && !err; i++) 3344 err = i40e_setup_tx_descriptors(vsi->xdp_rings[i]); 3345 3346 return err; 3347 } 3348 3349 /** 3350 * i40e_vsi_free_tx_resources - Free Tx resources for VSI queues 3351 * @vsi: ptr to the VSI 3352 * 3353 * Free VSI's transmit software resources 3354 **/ 3355 static void i40e_vsi_free_tx_resources(struct i40e_vsi *vsi) 3356 { 3357 int i; 3358 3359 if (vsi->tx_rings) { 3360 for (i = 0; i < vsi->num_queue_pairs; i++) 3361 if (vsi->tx_rings[i] && vsi->tx_rings[i]->desc) 3362 i40e_free_tx_resources(vsi->tx_rings[i]); 3363 } 3364 3365 if (vsi->xdp_rings) { 3366 for (i = 0; i < vsi->num_queue_pairs; i++) 3367 if (vsi->xdp_rings[i] && vsi->xdp_rings[i]->desc) 3368 i40e_free_tx_resources(vsi->xdp_rings[i]); 3369 } 3370 } 3371 3372 /** 3373 * i40e_vsi_setup_rx_resources - Allocate VSI queues Rx resources 3374 * @vsi: ptr to the VSI 3375 * 3376 * If this function returns with an error, then it's possible one or 3377 * more of the rings is populated (while the rest are not). It is the 3378 * callers duty to clean those orphaned rings. 3379 * 3380 * Return 0 on success, negative on failure 3381 **/ 3382 static int i40e_vsi_setup_rx_resources(struct i40e_vsi *vsi) 3383 { 3384 int i, err = 0; 3385 3386 for (i = 0; i < vsi->num_queue_pairs && !err; i++) 3387 err = i40e_setup_rx_descriptors(vsi->rx_rings[i]); 3388 return err; 3389 } 3390 3391 /** 3392 * i40e_vsi_free_rx_resources - Free Rx Resources for VSI queues 3393 * @vsi: ptr to the VSI 3394 * 3395 * Free all receive software resources 3396 **/ 3397 static void i40e_vsi_free_rx_resources(struct i40e_vsi *vsi) 3398 { 3399 int i; 3400 3401 if (!vsi->rx_rings) 3402 return; 3403 3404 for (i = 0; i < vsi->num_queue_pairs; i++) 3405 if (vsi->rx_rings[i] && vsi->rx_rings[i]->desc) 3406 i40e_free_rx_resources(vsi->rx_rings[i]); 3407 } 3408 3409 /** 3410 * i40e_config_xps_tx_ring - Configure XPS for a Tx ring 3411 * @ring: The Tx ring to configure 3412 * 3413 * This enables/disables XPS for a given Tx descriptor ring 3414 * based on the TCs enabled for the VSI that ring belongs to. 3415 **/ 3416 static void i40e_config_xps_tx_ring(struct i40e_ring *ring) 3417 { 3418 int cpu; 3419 3420 if (!ring->q_vector || !ring->netdev || ring->ch) 3421 return; 3422 3423 /* We only initialize XPS once, so as not to overwrite user settings */ 3424 if (test_and_set_bit(__I40E_TX_XPS_INIT_DONE, ring->state)) 3425 return; 3426 3427 cpu = cpumask_local_spread(ring->q_vector->v_idx, -1); 3428 netif_set_xps_queue(ring->netdev, get_cpu_mask(cpu), 3429 ring->queue_index); 3430 } 3431 3432 /** 3433 * i40e_xsk_pool - Retrieve the AF_XDP buffer pool if XDP and ZC is enabled 3434 * @ring: The Tx or Rx ring 3435 * 3436 * Returns the AF_XDP buffer pool or NULL. 3437 **/ 3438 static struct xsk_buff_pool *i40e_xsk_pool(struct i40e_ring *ring) 3439 { 3440 bool xdp_on = i40e_enabled_xdp_vsi(ring->vsi); 3441 int qid = ring->queue_index; 3442 3443 if (ring_is_xdp(ring)) 3444 qid -= ring->vsi->alloc_queue_pairs; 3445 3446 if (!xdp_on || !test_bit(qid, ring->vsi->af_xdp_zc_qps)) 3447 return NULL; 3448 3449 return xsk_get_pool_from_qid(ring->vsi->netdev, qid); 3450 } 3451 3452 /** 3453 * i40e_configure_tx_ring - Configure a transmit ring context and rest 3454 * @ring: The Tx ring to configure 3455 * 3456 * Configure the Tx descriptor ring in the HMC context. 3457 **/ 3458 static int i40e_configure_tx_ring(struct i40e_ring *ring) 3459 { 3460 struct i40e_vsi *vsi = ring->vsi; 3461 u16 pf_q = vsi->base_queue + ring->queue_index; 3462 struct i40e_hw *hw = &vsi->back->hw; 3463 struct i40e_hmc_obj_txq tx_ctx; 3464 u32 qtx_ctl = 0; 3465 int err = 0; 3466 3467 if (ring_is_xdp(ring)) 3468 ring->xsk_pool = i40e_xsk_pool(ring); 3469 3470 /* some ATR related tx ring init */ 3471 if (test_bit(I40E_FLAG_FD_ATR_ENA, vsi->back->flags)) { 3472 ring->atr_sample_rate = I40E_DEFAULT_ATR_SAMPLE_RATE; 3473 ring->atr_count = 0; 3474 } else { 3475 ring->atr_sample_rate = 0; 3476 } 3477 3478 /* configure XPS */ 3479 i40e_config_xps_tx_ring(ring); 3480 3481 /* clear the context structure first */ 3482 memset(&tx_ctx, 0, sizeof(tx_ctx)); 3483 3484 tx_ctx.new_context = 1; 3485 tx_ctx.base = (ring->dma / 128); 3486 tx_ctx.qlen = ring->count; 3487 if (test_bit(I40E_FLAG_FD_SB_ENA, vsi->back->flags) || 3488 test_bit(I40E_FLAG_FD_ATR_ENA, vsi->back->flags)) 3489 tx_ctx.fd_ena = 1; 3490 if (test_bit(I40E_FLAG_PTP_ENA, vsi->back->flags)) 3491 tx_ctx.timesync_ena = 1; 3492 /* FDIR VSI tx ring can still use RS bit and writebacks */ 3493 if (vsi->type != I40E_VSI_FDIR) 3494 tx_ctx.head_wb_ena = 1; 3495 tx_ctx.head_wb_addr = ring->dma + 3496 (ring->count * sizeof(struct i40e_tx_desc)); 3497 3498 /* As part of VSI creation/update, FW allocates certain 3499 * Tx arbitration queue sets for each TC enabled for 3500 * the VSI. The FW returns the handles to these queue 3501 * sets as part of the response buffer to Add VSI, 3502 * Update VSI, etc. AQ commands. It is expected that 3503 * these queue set handles be associated with the Tx 3504 * queues by the driver as part of the TX queue context 3505 * initialization. This has to be done regardless of 3506 * DCB as by default everything is mapped to TC0. 3507 */ 3508 3509 if (ring->ch) 3510 tx_ctx.rdylist = 3511 le16_to_cpu(ring->ch->info.qs_handle[ring->dcb_tc]); 3512 3513 else 3514 tx_ctx.rdylist = le16_to_cpu(vsi->info.qs_handle[ring->dcb_tc]); 3515 3516 tx_ctx.rdylist_act = 0; 3517 3518 /* clear the context in the HMC */ 3519 err = i40e_clear_lan_tx_queue_context(hw, pf_q); 3520 if (err) { 3521 dev_info(&vsi->back->pdev->dev, 3522 "Failed to clear LAN Tx queue context on Tx ring %d (pf_q %d), error: %d\n", 3523 ring->queue_index, pf_q, err); 3524 return -ENOMEM; 3525 } 3526 3527 /* set the context in the HMC */ 3528 err = i40e_set_lan_tx_queue_context(hw, pf_q, &tx_ctx); 3529 if (err) { 3530 dev_info(&vsi->back->pdev->dev, 3531 "Failed to set LAN Tx queue context on Tx ring %d (pf_q %d, error: %d\n", 3532 ring->queue_index, pf_q, err); 3533 return -ENOMEM; 3534 } 3535 3536 /* Now associate this queue with this PCI function */ 3537 if (ring->ch) { 3538 if (ring->ch->type == I40E_VSI_VMDQ2) 3539 qtx_ctl = I40E_QTX_CTL_VM_QUEUE; 3540 else 3541 return -EINVAL; 3542 3543 qtx_ctl |= FIELD_PREP(I40E_QTX_CTL_VFVM_INDX_MASK, 3544 ring->ch->vsi_number); 3545 } else { 3546 if (vsi->type == I40E_VSI_VMDQ2) { 3547 qtx_ctl = I40E_QTX_CTL_VM_QUEUE; 3548 qtx_ctl |= FIELD_PREP(I40E_QTX_CTL_VFVM_INDX_MASK, 3549 vsi->id); 3550 } else { 3551 qtx_ctl = I40E_QTX_CTL_PF_QUEUE; 3552 } 3553 } 3554 3555 qtx_ctl |= FIELD_PREP(I40E_QTX_CTL_PF_INDX_MASK, hw->pf_id); 3556 wr32(hw, I40E_QTX_CTL(pf_q), qtx_ctl); 3557 i40e_flush(hw); 3558 3559 /* cache tail off for easier writes later */ 3560 ring->tail = hw->hw_addr + I40E_QTX_TAIL(pf_q); 3561 3562 return 0; 3563 } 3564 3565 /** 3566 * i40e_rx_offset - Return expected offset into page to access data 3567 * @rx_ring: Ring we are requesting offset of 3568 * 3569 * Returns the offset value for ring into the data buffer. 3570 */ 3571 static unsigned int i40e_rx_offset(struct i40e_ring *rx_ring) 3572 { 3573 return ring_uses_build_skb(rx_ring) ? I40E_SKB_PAD : 0; 3574 } 3575 3576 /** 3577 * i40e_configure_rx_ring - Configure a receive ring context 3578 * @ring: The Rx ring to configure 3579 * 3580 * Configure the Rx descriptor ring in the HMC context. 3581 **/ 3582 static int i40e_configure_rx_ring(struct i40e_ring *ring) 3583 { 3584 struct i40e_vsi *vsi = ring->vsi; 3585 u32 chain_len = vsi->back->hw.func_caps.rx_buf_chain_len; 3586 u16 pf_q = vsi->base_queue + ring->queue_index; 3587 struct i40e_hw *hw = &vsi->back->hw; 3588 struct i40e_hmc_obj_rxq rx_ctx; 3589 int err = 0; 3590 bool ok; 3591 3592 bitmap_zero(ring->state, __I40E_RING_STATE_NBITS); 3593 3594 /* clear the context structure first */ 3595 memset(&rx_ctx, 0, sizeof(rx_ctx)); 3596 3597 ring->rx_buf_len = vsi->rx_buf_len; 3598 3599 /* XDP RX-queue info only needed for RX rings exposed to XDP */ 3600 if (ring->vsi->type != I40E_VSI_MAIN) 3601 goto skip; 3602 3603 if (!xdp_rxq_info_is_reg(&ring->xdp_rxq)) { 3604 err = __xdp_rxq_info_reg(&ring->xdp_rxq, ring->netdev, 3605 ring->queue_index, 3606 ring->q_vector->napi.napi_id, 3607 ring->rx_buf_len); 3608 if (err) 3609 return err; 3610 } 3611 3612 ring->xsk_pool = i40e_xsk_pool(ring); 3613 if (ring->xsk_pool) { 3614 xdp_rxq_info_unreg(&ring->xdp_rxq); 3615 ring->rx_buf_len = xsk_pool_get_rx_frame_size(ring->xsk_pool); 3616 err = __xdp_rxq_info_reg(&ring->xdp_rxq, ring->netdev, 3617 ring->queue_index, 3618 ring->q_vector->napi.napi_id, 3619 ring->rx_buf_len); 3620 if (err) 3621 return err; 3622 err = xdp_rxq_info_reg_mem_model(&ring->xdp_rxq, 3623 MEM_TYPE_XSK_BUFF_POOL, 3624 NULL); 3625 if (err) 3626 return err; 3627 dev_info(&vsi->back->pdev->dev, 3628 "Registered XDP mem model MEM_TYPE_XSK_BUFF_POOL on Rx ring %d\n", 3629 ring->queue_index); 3630 3631 } else { 3632 err = xdp_rxq_info_reg_mem_model(&ring->xdp_rxq, 3633 MEM_TYPE_PAGE_SHARED, 3634 NULL); 3635 if (err) 3636 return err; 3637 } 3638 3639 skip: 3640 xdp_init_buff(&ring->xdp, i40e_rx_pg_size(ring) / 2, &ring->xdp_rxq); 3641 3642 rx_ctx.dbuff = DIV_ROUND_UP(ring->rx_buf_len, 3643 BIT_ULL(I40E_RXQ_CTX_DBUFF_SHIFT)); 3644 3645 rx_ctx.base = (ring->dma / 128); 3646 rx_ctx.qlen = ring->count; 3647 3648 /* use 16 byte descriptors */ 3649 rx_ctx.dsize = 0; 3650 3651 /* descriptor type is always zero 3652 * rx_ctx.dtype = 0; 3653 */ 3654 rx_ctx.hsplit_0 = 0; 3655 3656 rx_ctx.rxmax = min_t(u16, vsi->max_frame, chain_len * ring->rx_buf_len); 3657 if (hw->revision_id == 0) 3658 rx_ctx.lrxqthresh = 0; 3659 else 3660 rx_ctx.lrxqthresh = 1; 3661 rx_ctx.crcstrip = 1; 3662 rx_ctx.l2tsel = 1; 3663 /* this controls whether VLAN is stripped from inner headers */ 3664 rx_ctx.showiv = 0; 3665 /* set the prefena field to 1 because the manual says to */ 3666 rx_ctx.prefena = 1; 3667 3668 /* clear the context in the HMC */ 3669 err = i40e_clear_lan_rx_queue_context(hw, pf_q); 3670 if (err) { 3671 dev_info(&vsi->back->pdev->dev, 3672 "Failed to clear LAN Rx queue context on Rx ring %d (pf_q %d), error: %d\n", 3673 ring->queue_index, pf_q, err); 3674 return -ENOMEM; 3675 } 3676 3677 /* set the context in the HMC */ 3678 err = i40e_set_lan_rx_queue_context(hw, pf_q, &rx_ctx); 3679 if (err) { 3680 dev_info(&vsi->back->pdev->dev, 3681 "Failed to set LAN Rx queue context on Rx ring %d (pf_q %d), error: %d\n", 3682 ring->queue_index, pf_q, err); 3683 return -ENOMEM; 3684 } 3685 3686 /* configure Rx buffer alignment */ 3687 if (!vsi->netdev || test_bit(I40E_FLAG_LEGACY_RX_ENA, vsi->back->flags)) { 3688 if (I40E_2K_TOO_SMALL_WITH_PADDING) { 3689 dev_info(&vsi->back->pdev->dev, 3690 "2k Rx buffer is too small to fit standard MTU and skb_shared_info\n"); 3691 return -EOPNOTSUPP; 3692 } 3693 clear_ring_build_skb_enabled(ring); 3694 } else { 3695 set_ring_build_skb_enabled(ring); 3696 } 3697 3698 ring->rx_offset = i40e_rx_offset(ring); 3699 3700 /* cache tail for quicker writes, and clear the reg before use */ 3701 ring->tail = hw->hw_addr + I40E_QRX_TAIL(pf_q); 3702 writel(0, ring->tail); 3703 3704 if (ring->xsk_pool) { 3705 xsk_pool_set_rxq_info(ring->xsk_pool, &ring->xdp_rxq); 3706 ok = i40e_alloc_rx_buffers_zc(ring, I40E_DESC_UNUSED(ring)); 3707 } else { 3708 ok = !i40e_alloc_rx_buffers(ring, I40E_DESC_UNUSED(ring)); 3709 } 3710 if (!ok) { 3711 /* Log this in case the user has forgotten to give the kernel 3712 * any buffers, even later in the application. 3713 */ 3714 dev_info(&vsi->back->pdev->dev, 3715 "Failed to allocate some buffers on %sRx ring %d (pf_q %d)\n", 3716 ring->xsk_pool ? "AF_XDP ZC enabled " : "", 3717 ring->queue_index, pf_q); 3718 } 3719 3720 return 0; 3721 } 3722 3723 /** 3724 * i40e_vsi_configure_tx - Configure the VSI for Tx 3725 * @vsi: VSI structure describing this set of rings and resources 3726 * 3727 * Configure the Tx VSI for operation. 3728 **/ 3729 static int i40e_vsi_configure_tx(struct i40e_vsi *vsi) 3730 { 3731 int err = 0; 3732 u16 i; 3733 3734 for (i = 0; (i < vsi->num_queue_pairs) && !err; i++) 3735 err = i40e_configure_tx_ring(vsi->tx_rings[i]); 3736 3737 if (err || !i40e_enabled_xdp_vsi(vsi)) 3738 return err; 3739 3740 for (i = 0; (i < vsi->num_queue_pairs) && !err; i++) 3741 err = i40e_configure_tx_ring(vsi->xdp_rings[i]); 3742 3743 return err; 3744 } 3745 3746 /** 3747 * i40e_vsi_configure_rx - Configure the VSI for Rx 3748 * @vsi: the VSI being configured 3749 * 3750 * Configure the Rx VSI for operation. 3751 **/ 3752 static int i40e_vsi_configure_rx(struct i40e_vsi *vsi) 3753 { 3754 int err = 0; 3755 u16 i; 3756 3757 vsi->max_frame = i40e_max_vsi_frame_size(vsi, vsi->xdp_prog); 3758 vsi->rx_buf_len = i40e_calculate_vsi_rx_buf_len(vsi); 3759 3760 #if (PAGE_SIZE < 8192) 3761 if (vsi->netdev && !I40E_2K_TOO_SMALL_WITH_PADDING && 3762 vsi->netdev->mtu <= ETH_DATA_LEN) { 3763 vsi->rx_buf_len = I40E_RXBUFFER_1536 - NET_IP_ALIGN; 3764 vsi->max_frame = vsi->rx_buf_len; 3765 } 3766 #endif 3767 3768 /* set up individual rings */ 3769 for (i = 0; i < vsi->num_queue_pairs && !err; i++) 3770 err = i40e_configure_rx_ring(vsi->rx_rings[i]); 3771 3772 return err; 3773 } 3774 3775 /** 3776 * i40e_vsi_config_dcb_rings - Update rings to reflect DCB TC 3777 * @vsi: ptr to the VSI 3778 **/ 3779 static void i40e_vsi_config_dcb_rings(struct i40e_vsi *vsi) 3780 { 3781 struct i40e_ring *tx_ring, *rx_ring; 3782 u16 qoffset, qcount; 3783 int i, n; 3784 3785 if (!test_bit(I40E_FLAG_DCB_ENA, vsi->back->flags)) { 3786 /* Reset the TC information */ 3787 for (i = 0; i < vsi->num_queue_pairs; i++) { 3788 rx_ring = vsi->rx_rings[i]; 3789 tx_ring = vsi->tx_rings[i]; 3790 rx_ring->dcb_tc = 0; 3791 tx_ring->dcb_tc = 0; 3792 } 3793 return; 3794 } 3795 3796 for (n = 0; n < I40E_MAX_TRAFFIC_CLASS; n++) { 3797 if (!(vsi->tc_config.enabled_tc & BIT_ULL(n))) 3798 continue; 3799 3800 qoffset = vsi->tc_config.tc_info[n].qoffset; 3801 qcount = vsi->tc_config.tc_info[n].qcount; 3802 for (i = qoffset; i < (qoffset + qcount); i++) { 3803 rx_ring = vsi->rx_rings[i]; 3804 tx_ring = vsi->tx_rings[i]; 3805 rx_ring->dcb_tc = n; 3806 tx_ring->dcb_tc = n; 3807 } 3808 } 3809 } 3810 3811 /** 3812 * i40e_set_vsi_rx_mode - Call set_rx_mode on a VSI 3813 * @vsi: ptr to the VSI 3814 **/ 3815 static void i40e_set_vsi_rx_mode(struct i40e_vsi *vsi) 3816 { 3817 if (vsi->netdev) 3818 i40e_set_rx_mode(vsi->netdev); 3819 } 3820 3821 /** 3822 * i40e_reset_fdir_filter_cnt - Reset flow director filter counters 3823 * @pf: Pointer to the targeted PF 3824 * 3825 * Set all flow director counters to 0. 3826 */ 3827 static void i40e_reset_fdir_filter_cnt(struct i40e_pf *pf) 3828 { 3829 pf->fd_tcp4_filter_cnt = 0; 3830 pf->fd_udp4_filter_cnt = 0; 3831 pf->fd_sctp4_filter_cnt = 0; 3832 pf->fd_ip4_filter_cnt = 0; 3833 pf->fd_tcp6_filter_cnt = 0; 3834 pf->fd_udp6_filter_cnt = 0; 3835 pf->fd_sctp6_filter_cnt = 0; 3836 pf->fd_ip6_filter_cnt = 0; 3837 } 3838 3839 /** 3840 * i40e_fdir_filter_restore - Restore the Sideband Flow Director filters 3841 * @vsi: Pointer to the targeted VSI 3842 * 3843 * This function replays the hlist on the hw where all the SB Flow Director 3844 * filters were saved. 3845 **/ 3846 static void i40e_fdir_filter_restore(struct i40e_vsi *vsi) 3847 { 3848 struct i40e_fdir_filter *filter; 3849 struct i40e_pf *pf = vsi->back; 3850 struct hlist_node *node; 3851 3852 if (!test_bit(I40E_FLAG_FD_SB_ENA, pf->flags)) 3853 return; 3854 3855 /* Reset FDir counters as we're replaying all existing filters */ 3856 i40e_reset_fdir_filter_cnt(pf); 3857 3858 hlist_for_each_entry_safe(filter, node, 3859 &pf->fdir_filter_list, fdir_node) { 3860 i40e_add_del_fdir(vsi, filter, true); 3861 } 3862 } 3863 3864 /** 3865 * i40e_vsi_configure - Set up the VSI for action 3866 * @vsi: the VSI being configured 3867 **/ 3868 static int i40e_vsi_configure(struct i40e_vsi *vsi) 3869 { 3870 int err; 3871 3872 i40e_set_vsi_rx_mode(vsi); 3873 i40e_restore_vlan(vsi); 3874 i40e_vsi_config_dcb_rings(vsi); 3875 err = i40e_vsi_configure_tx(vsi); 3876 if (!err) 3877 err = i40e_vsi_configure_rx(vsi); 3878 3879 return err; 3880 } 3881 3882 /** 3883 * i40e_vsi_configure_msix - MSIX mode Interrupt Config in the HW 3884 * @vsi: the VSI being configured 3885 **/ 3886 static void i40e_vsi_configure_msix(struct i40e_vsi *vsi) 3887 { 3888 bool has_xdp = i40e_enabled_xdp_vsi(vsi); 3889 struct i40e_pf *pf = vsi->back; 3890 struct i40e_hw *hw = &pf->hw; 3891 u16 vector; 3892 int i, q; 3893 u32 qp; 3894 3895 /* The interrupt indexing is offset by 1 in the PFINT_ITRn 3896 * and PFINT_LNKLSTn registers, e.g.: 3897 * PFINT_ITRn[0..n-1] gets msix-1..msix-n (qpair interrupts) 3898 */ 3899 qp = vsi->base_queue; 3900 vector = vsi->base_vector; 3901 for (i = 0; i < vsi->num_q_vectors; i++, vector++) { 3902 struct i40e_q_vector *q_vector = vsi->q_vectors[i]; 3903 3904 q_vector->rx.next_update = jiffies + 1; 3905 q_vector->rx.target_itr = 3906 ITR_TO_REG(vsi->rx_rings[i]->itr_setting); 3907 wr32(hw, I40E_PFINT_ITRN(I40E_RX_ITR, vector - 1), 3908 q_vector->rx.target_itr >> 1); 3909 q_vector->rx.current_itr = q_vector->rx.target_itr; 3910 3911 q_vector->tx.next_update = jiffies + 1; 3912 q_vector->tx.target_itr = 3913 ITR_TO_REG(vsi->tx_rings[i]->itr_setting); 3914 wr32(hw, I40E_PFINT_ITRN(I40E_TX_ITR, vector - 1), 3915 q_vector->tx.target_itr >> 1); 3916 q_vector->tx.current_itr = q_vector->tx.target_itr; 3917 3918 /* Set ITR for software interrupts triggered after exiting 3919 * busy-loop polling. 3920 */ 3921 wr32(hw, I40E_PFINT_ITRN(I40E_SW_ITR, vector - 1), 3922 I40E_ITR_20K); 3923 3924 wr32(hw, I40E_PFINT_RATEN(vector - 1), 3925 i40e_intrl_usec_to_reg(vsi->int_rate_limit)); 3926 3927 /* begin of linked list for RX queue assigned to this vector */ 3928 wr32(hw, I40E_PFINT_LNKLSTN(vector - 1), qp); 3929 for (q = 0; q < q_vector->num_ringpairs; q++) { 3930 u32 nextqp = has_xdp ? qp + vsi->alloc_queue_pairs : qp; 3931 u32 val; 3932 3933 val = I40E_QINT_RQCTL_CAUSE_ENA_MASK | 3934 (I40E_RX_ITR << I40E_QINT_RQCTL_ITR_INDX_SHIFT) | 3935 (vector << I40E_QINT_RQCTL_MSIX_INDX_SHIFT) | 3936 (nextqp << I40E_QINT_RQCTL_NEXTQ_INDX_SHIFT) | 3937 (I40E_QUEUE_TYPE_TX << 3938 I40E_QINT_RQCTL_NEXTQ_TYPE_SHIFT); 3939 3940 wr32(hw, I40E_QINT_RQCTL(qp), val); 3941 3942 if (has_xdp) { 3943 /* TX queue with next queue set to TX */ 3944 val = I40E_QINT_TQCTL_CAUSE_ENA_MASK | 3945 (I40E_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT) | 3946 (vector << I40E_QINT_TQCTL_MSIX_INDX_SHIFT) | 3947 (qp << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT) | 3948 (I40E_QUEUE_TYPE_TX << 3949 I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT); 3950 3951 wr32(hw, I40E_QINT_TQCTL(nextqp), val); 3952 } 3953 /* TX queue with next RX or end of linked list */ 3954 val = I40E_QINT_TQCTL_CAUSE_ENA_MASK | 3955 (I40E_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT) | 3956 (vector << I40E_QINT_TQCTL_MSIX_INDX_SHIFT) | 3957 ((qp + 1) << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT) | 3958 (I40E_QUEUE_TYPE_RX << 3959 I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT); 3960 3961 /* Terminate the linked list */ 3962 if (q == (q_vector->num_ringpairs - 1)) 3963 val |= (I40E_QUEUE_END_OF_LIST << 3964 I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT); 3965 3966 wr32(hw, I40E_QINT_TQCTL(qp), val); 3967 qp++; 3968 } 3969 } 3970 3971 i40e_flush(hw); 3972 } 3973 3974 /** 3975 * i40e_enable_misc_int_causes - enable the non-queue interrupts 3976 * @pf: pointer to private device data structure 3977 **/ 3978 static void i40e_enable_misc_int_causes(struct i40e_pf *pf) 3979 { 3980 struct i40e_hw *hw = &pf->hw; 3981 u32 val; 3982 3983 /* clear things first */ 3984 wr32(hw, I40E_PFINT_ICR0_ENA, 0); /* disable all */ 3985 rd32(hw, I40E_PFINT_ICR0); /* read to clear */ 3986 3987 val = I40E_PFINT_ICR0_ENA_ECC_ERR_MASK | 3988 I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK | 3989 I40E_PFINT_ICR0_ENA_GRST_MASK | 3990 I40E_PFINT_ICR0_ENA_PCI_EXCEPTION_MASK | 3991 I40E_PFINT_ICR0_ENA_GPIO_MASK | 3992 I40E_PFINT_ICR0_ENA_HMC_ERR_MASK | 3993 I40E_PFINT_ICR0_ENA_VFLR_MASK | 3994 I40E_PFINT_ICR0_ENA_ADMINQ_MASK; 3995 3996 if (test_bit(I40E_FLAG_IWARP_ENA, pf->flags)) 3997 val |= I40E_PFINT_ICR0_ENA_PE_CRITERR_MASK; 3998 3999 if (test_bit(I40E_FLAG_PTP_ENA, pf->flags)) 4000 val |= I40E_PFINT_ICR0_ENA_TIMESYNC_MASK; 4001 4002 wr32(hw, I40E_PFINT_ICR0_ENA, val); 4003 4004 /* SW_ITR_IDX = 0, but don't change INTENA */ 4005 wr32(hw, I40E_PFINT_DYN_CTL0, I40E_PFINT_DYN_CTL0_SW_ITR_INDX_MASK | 4006 I40E_PFINT_DYN_CTL0_INTENA_MSK_MASK); 4007 4008 /* OTHER_ITR_IDX = 0 */ 4009 wr32(hw, I40E_PFINT_STAT_CTL0, 0); 4010 } 4011 4012 /** 4013 * i40e_configure_msi_and_legacy - Legacy mode interrupt config in the HW 4014 * @vsi: the VSI being configured 4015 **/ 4016 static void i40e_configure_msi_and_legacy(struct i40e_vsi *vsi) 4017 { 4018 u32 nextqp = i40e_enabled_xdp_vsi(vsi) ? vsi->alloc_queue_pairs : 0; 4019 struct i40e_q_vector *q_vector = vsi->q_vectors[0]; 4020 struct i40e_pf *pf = vsi->back; 4021 struct i40e_hw *hw = &pf->hw; 4022 4023 /* set the ITR configuration */ 4024 q_vector->rx.next_update = jiffies + 1; 4025 q_vector->rx.target_itr = ITR_TO_REG(vsi->rx_rings[0]->itr_setting); 4026 wr32(hw, I40E_PFINT_ITR0(I40E_RX_ITR), q_vector->rx.target_itr >> 1); 4027 q_vector->rx.current_itr = q_vector->rx.target_itr; 4028 q_vector->tx.next_update = jiffies + 1; 4029 q_vector->tx.target_itr = ITR_TO_REG(vsi->tx_rings[0]->itr_setting); 4030 wr32(hw, I40E_PFINT_ITR0(I40E_TX_ITR), q_vector->tx.target_itr >> 1); 4031 q_vector->tx.current_itr = q_vector->tx.target_itr; 4032 4033 i40e_enable_misc_int_causes(pf); 4034 4035 /* FIRSTQ_INDX = 0, FIRSTQ_TYPE = 0 (rx) */ 4036 wr32(hw, I40E_PFINT_LNKLST0, 0); 4037 4038 /* Associate the queue pair to the vector and enable the queue 4039 * interrupt RX queue in linked list with next queue set to TX 4040 */ 4041 wr32(hw, I40E_QINT_RQCTL(0), I40E_QINT_RQCTL_VAL(nextqp, 0, TX)); 4042 4043 if (i40e_enabled_xdp_vsi(vsi)) { 4044 /* TX queue in linked list with next queue set to TX */ 4045 wr32(hw, I40E_QINT_TQCTL(nextqp), 4046 I40E_QINT_TQCTL_VAL(nextqp, 0, TX)); 4047 } 4048 4049 /* last TX queue so the next RX queue doesn't matter */ 4050 wr32(hw, I40E_QINT_TQCTL(0), 4051 I40E_QINT_TQCTL_VAL(I40E_QUEUE_END_OF_LIST, 0, RX)); 4052 i40e_flush(hw); 4053 } 4054 4055 /** 4056 * i40e_irq_dynamic_disable_icr0 - Disable default interrupt generation for icr0 4057 * @pf: board private structure 4058 **/ 4059 void i40e_irq_dynamic_disable_icr0(struct i40e_pf *pf) 4060 { 4061 struct i40e_hw *hw = &pf->hw; 4062 4063 wr32(hw, I40E_PFINT_DYN_CTL0, 4064 I40E_ITR_NONE << I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT); 4065 i40e_flush(hw); 4066 } 4067 4068 /** 4069 * i40e_irq_dynamic_enable_icr0 - Enable default interrupt generation for icr0 4070 * @pf: board private structure 4071 **/ 4072 void i40e_irq_dynamic_enable_icr0(struct i40e_pf *pf) 4073 { 4074 struct i40e_hw *hw = &pf->hw; 4075 u32 val; 4076 4077 val = I40E_PFINT_DYN_CTL0_INTENA_MASK | 4078 I40E_PFINT_DYN_CTL0_CLEARPBA_MASK | 4079 (I40E_ITR_NONE << I40E_PFINT_DYN_CTL0_ITR_INDX_SHIFT); 4080 4081 wr32(hw, I40E_PFINT_DYN_CTL0, val); 4082 i40e_flush(hw); 4083 } 4084 4085 /** 4086 * i40e_msix_clean_rings - MSIX mode Interrupt Handler 4087 * @irq: interrupt number 4088 * @data: pointer to a q_vector 4089 **/ 4090 static irqreturn_t i40e_msix_clean_rings(int irq, void *data) 4091 { 4092 struct i40e_q_vector *q_vector = data; 4093 4094 if (!q_vector->tx.ring && !q_vector->rx.ring) 4095 return IRQ_HANDLED; 4096 4097 napi_schedule_irqoff(&q_vector->napi); 4098 4099 return IRQ_HANDLED; 4100 } 4101 4102 /** 4103 * i40e_irq_affinity_notify - Callback for affinity changes 4104 * @notify: context as to what irq was changed 4105 * @mask: the new affinity mask 4106 * 4107 * This is a callback function used by the irq_set_affinity_notifier function 4108 * so that we may register to receive changes to the irq affinity masks. 4109 **/ 4110 static void i40e_irq_affinity_notify(struct irq_affinity_notify *notify, 4111 const cpumask_t *mask) 4112 { 4113 struct i40e_q_vector *q_vector = 4114 container_of(notify, struct i40e_q_vector, affinity_notify); 4115 4116 cpumask_copy(&q_vector->affinity_mask, mask); 4117 } 4118 4119 /** 4120 * i40e_irq_affinity_release - Callback for affinity notifier release 4121 * @ref: internal core kernel usage 4122 * 4123 * This is a callback function used by the irq_set_affinity_notifier function 4124 * to inform the current notification subscriber that they will no longer 4125 * receive notifications. 4126 **/ 4127 static void i40e_irq_affinity_release(struct kref *ref) {} 4128 4129 /** 4130 * i40e_vsi_request_irq_msix - Initialize MSI-X interrupts 4131 * @vsi: the VSI being configured 4132 * @basename: name for the vector 4133 * 4134 * Allocates MSI-X vectors and requests interrupts from the kernel. 4135 **/ 4136 static int i40e_vsi_request_irq_msix(struct i40e_vsi *vsi, char *basename) 4137 { 4138 int q_vectors = vsi->num_q_vectors; 4139 struct i40e_pf *pf = vsi->back; 4140 int base = vsi->base_vector; 4141 int rx_int_idx = 0; 4142 int tx_int_idx = 0; 4143 int vector, err; 4144 int irq_num; 4145 int cpu; 4146 4147 for (vector = 0; vector < q_vectors; vector++) { 4148 struct i40e_q_vector *q_vector = vsi->q_vectors[vector]; 4149 4150 irq_num = pf->msix_entries[base + vector].vector; 4151 4152 if (q_vector->tx.ring && q_vector->rx.ring) { 4153 snprintf(q_vector->name, sizeof(q_vector->name) - 1, 4154 "%s-%s-%d", basename, "TxRx", rx_int_idx++); 4155 tx_int_idx++; 4156 } else if (q_vector->rx.ring) { 4157 snprintf(q_vector->name, sizeof(q_vector->name) - 1, 4158 "%s-%s-%d", basename, "rx", rx_int_idx++); 4159 } else if (q_vector->tx.ring) { 4160 snprintf(q_vector->name, sizeof(q_vector->name) - 1, 4161 "%s-%s-%d", basename, "tx", tx_int_idx++); 4162 } else { 4163 /* skip this unused q_vector */ 4164 continue; 4165 } 4166 err = request_irq(irq_num, 4167 vsi->irq_handler, 4168 0, 4169 q_vector->name, 4170 q_vector); 4171 if (err) { 4172 dev_info(&pf->pdev->dev, 4173 "MSIX request_irq failed, error: %d\n", err); 4174 goto free_queue_irqs; 4175 } 4176 4177 /* register for affinity change notifications */ 4178 q_vector->irq_num = irq_num; 4179 q_vector->affinity_notify.notify = i40e_irq_affinity_notify; 4180 q_vector->affinity_notify.release = i40e_irq_affinity_release; 4181 irq_set_affinity_notifier(irq_num, &q_vector->affinity_notify); 4182 /* Spread affinity hints out across online CPUs. 4183 * 4184 * get_cpu_mask returns a static constant mask with 4185 * a permanent lifetime so it's ok to pass to 4186 * irq_update_affinity_hint without making a copy. 4187 */ 4188 cpu = cpumask_local_spread(q_vector->v_idx, -1); 4189 irq_update_affinity_hint(irq_num, get_cpu_mask(cpu)); 4190 } 4191 4192 vsi->irqs_ready = true; 4193 return 0; 4194 4195 free_queue_irqs: 4196 while (vector) { 4197 vector--; 4198 irq_num = pf->msix_entries[base + vector].vector; 4199 irq_set_affinity_notifier(irq_num, NULL); 4200 irq_update_affinity_hint(irq_num, NULL); 4201 free_irq(irq_num, &vsi->q_vectors[vector]); 4202 } 4203 return err; 4204 } 4205 4206 /** 4207 * i40e_vsi_disable_irq - Mask off queue interrupt generation on the VSI 4208 * @vsi: the VSI being un-configured 4209 **/ 4210 static void i40e_vsi_disable_irq(struct i40e_vsi *vsi) 4211 { 4212 struct i40e_pf *pf = vsi->back; 4213 struct i40e_hw *hw = &pf->hw; 4214 int base = vsi->base_vector; 4215 int i; 4216 4217 /* disable interrupt causation from each queue */ 4218 for (i = 0; i < vsi->num_queue_pairs; i++) { 4219 u32 val; 4220 4221 val = rd32(hw, I40E_QINT_TQCTL(vsi->tx_rings[i]->reg_idx)); 4222 val &= ~I40E_QINT_TQCTL_CAUSE_ENA_MASK; 4223 wr32(hw, I40E_QINT_TQCTL(vsi->tx_rings[i]->reg_idx), val); 4224 4225 val = rd32(hw, I40E_QINT_RQCTL(vsi->rx_rings[i]->reg_idx)); 4226 val &= ~I40E_QINT_RQCTL_CAUSE_ENA_MASK; 4227 wr32(hw, I40E_QINT_RQCTL(vsi->rx_rings[i]->reg_idx), val); 4228 4229 if (!i40e_enabled_xdp_vsi(vsi)) 4230 continue; 4231 wr32(hw, I40E_QINT_TQCTL(vsi->xdp_rings[i]->reg_idx), 0); 4232 } 4233 4234 /* disable each interrupt */ 4235 if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags)) { 4236 for (i = vsi->base_vector; 4237 i < (vsi->num_q_vectors + vsi->base_vector); i++) 4238 wr32(hw, I40E_PFINT_DYN_CTLN(i - 1), 0); 4239 4240 i40e_flush(hw); 4241 for (i = 0; i < vsi->num_q_vectors; i++) 4242 synchronize_irq(pf->msix_entries[i + base].vector); 4243 } else { 4244 /* Legacy and MSI mode - this stops all interrupt handling */ 4245 wr32(hw, I40E_PFINT_ICR0_ENA, 0); 4246 wr32(hw, I40E_PFINT_DYN_CTL0, 0); 4247 i40e_flush(hw); 4248 synchronize_irq(pf->pdev->irq); 4249 } 4250 } 4251 4252 /** 4253 * i40e_vsi_enable_irq - Enable IRQ for the given VSI 4254 * @vsi: the VSI being configured 4255 **/ 4256 static int i40e_vsi_enable_irq(struct i40e_vsi *vsi) 4257 { 4258 struct i40e_pf *pf = vsi->back; 4259 int i; 4260 4261 if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags)) { 4262 for (i = 0; i < vsi->num_q_vectors; i++) 4263 i40e_irq_dynamic_enable(vsi, i); 4264 } else { 4265 i40e_irq_dynamic_enable_icr0(pf); 4266 } 4267 4268 i40e_flush(&pf->hw); 4269 return 0; 4270 } 4271 4272 /** 4273 * i40e_free_misc_vector - Free the vector that handles non-queue events 4274 * @pf: board private structure 4275 **/ 4276 static void i40e_free_misc_vector(struct i40e_pf *pf) 4277 { 4278 /* Disable ICR 0 */ 4279 wr32(&pf->hw, I40E_PFINT_ICR0_ENA, 0); 4280 i40e_flush(&pf->hw); 4281 4282 if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags) && pf->msix_entries) { 4283 free_irq(pf->msix_entries[0].vector, pf); 4284 clear_bit(__I40E_MISC_IRQ_REQUESTED, pf->state); 4285 } 4286 } 4287 4288 /** 4289 * i40e_intr - MSI/Legacy and non-queue interrupt handler 4290 * @irq: interrupt number 4291 * @data: pointer to a q_vector 4292 * 4293 * This is the handler used for all MSI/Legacy interrupts, and deals 4294 * with both queue and non-queue interrupts. This is also used in 4295 * MSIX mode to handle the non-queue interrupts. 4296 **/ 4297 static irqreturn_t i40e_intr(int irq, void *data) 4298 { 4299 struct i40e_pf *pf = (struct i40e_pf *)data; 4300 struct i40e_hw *hw = &pf->hw; 4301 irqreturn_t ret = IRQ_NONE; 4302 u32 icr0, icr0_remaining; 4303 u32 val, ena_mask; 4304 4305 icr0 = rd32(hw, I40E_PFINT_ICR0); 4306 ena_mask = rd32(hw, I40E_PFINT_ICR0_ENA); 4307 4308 /* if sharing a legacy IRQ, we might get called w/o an intr pending */ 4309 if ((icr0 & I40E_PFINT_ICR0_INTEVENT_MASK) == 0) 4310 goto enable_intr; 4311 4312 /* if interrupt but no bits showing, must be SWINT */ 4313 if (((icr0 & ~I40E_PFINT_ICR0_INTEVENT_MASK) == 0) || 4314 (icr0 & I40E_PFINT_ICR0_SWINT_MASK)) 4315 pf->sw_int_count++; 4316 4317 if (test_bit(I40E_FLAG_IWARP_ENA, pf->flags) && 4318 (icr0 & I40E_PFINT_ICR0_ENA_PE_CRITERR_MASK)) { 4319 ena_mask &= ~I40E_PFINT_ICR0_ENA_PE_CRITERR_MASK; 4320 dev_dbg(&pf->pdev->dev, "cleared PE_CRITERR\n"); 4321 set_bit(__I40E_CORE_RESET_REQUESTED, pf->state); 4322 } 4323 4324 /* only q0 is used in MSI/Legacy mode, and none are used in MSIX */ 4325 if (icr0 & I40E_PFINT_ICR0_QUEUE_0_MASK) { 4326 struct i40e_vsi *vsi = i40e_pf_get_main_vsi(pf); 4327 struct i40e_q_vector *q_vector = vsi->q_vectors[0]; 4328 4329 /* We do not have a way to disarm Queue causes while leaving 4330 * interrupt enabled for all other causes, ideally 4331 * interrupt should be disabled while we are in NAPI but 4332 * this is not a performance path and napi_schedule() 4333 * can deal with rescheduling. 4334 */ 4335 if (!test_bit(__I40E_DOWN, pf->state)) 4336 napi_schedule_irqoff(&q_vector->napi); 4337 } 4338 4339 if (icr0 & I40E_PFINT_ICR0_ADMINQ_MASK) { 4340 ena_mask &= ~I40E_PFINT_ICR0_ENA_ADMINQ_MASK; 4341 set_bit(__I40E_ADMINQ_EVENT_PENDING, pf->state); 4342 i40e_debug(&pf->hw, I40E_DEBUG_NVM, "AdminQ event\n"); 4343 } 4344 4345 if (icr0 & I40E_PFINT_ICR0_MAL_DETECT_MASK) { 4346 ena_mask &= ~I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK; 4347 set_bit(__I40E_MDD_EVENT_PENDING, pf->state); 4348 } 4349 4350 if (icr0 & I40E_PFINT_ICR0_VFLR_MASK) { 4351 /* disable any further VFLR event notifications */ 4352 if (test_bit(__I40E_VF_RESETS_DISABLED, pf->state)) { 4353 u32 reg = rd32(hw, I40E_PFINT_ICR0_ENA); 4354 4355 reg &= ~I40E_PFINT_ICR0_VFLR_MASK; 4356 wr32(hw, I40E_PFINT_ICR0_ENA, reg); 4357 } else { 4358 ena_mask &= ~I40E_PFINT_ICR0_ENA_VFLR_MASK; 4359 set_bit(__I40E_VFLR_EVENT_PENDING, pf->state); 4360 } 4361 } 4362 4363 if (icr0 & I40E_PFINT_ICR0_GRST_MASK) { 4364 if (!test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state)) 4365 set_bit(__I40E_RESET_INTR_RECEIVED, pf->state); 4366 ena_mask &= ~I40E_PFINT_ICR0_ENA_GRST_MASK; 4367 val = rd32(hw, I40E_GLGEN_RSTAT); 4368 val = FIELD_GET(I40E_GLGEN_RSTAT_RESET_TYPE_MASK, val); 4369 if (val == I40E_RESET_CORER) { 4370 pf->corer_count++; 4371 } else if (val == I40E_RESET_GLOBR) { 4372 pf->globr_count++; 4373 } else if (val == I40E_RESET_EMPR) { 4374 pf->empr_count++; 4375 set_bit(__I40E_EMP_RESET_INTR_RECEIVED, pf->state); 4376 } 4377 } 4378 4379 if (icr0 & I40E_PFINT_ICR0_HMC_ERR_MASK) { 4380 icr0 &= ~I40E_PFINT_ICR0_HMC_ERR_MASK; 4381 dev_info(&pf->pdev->dev, "HMC error interrupt\n"); 4382 dev_info(&pf->pdev->dev, "HMC error info 0x%x, HMC error data 0x%x\n", 4383 rd32(hw, I40E_PFHMC_ERRORINFO), 4384 rd32(hw, I40E_PFHMC_ERRORDATA)); 4385 } 4386 4387 if (icr0 & I40E_PFINT_ICR0_TIMESYNC_MASK) { 4388 u32 prttsyn_stat = rd32(hw, I40E_PRTTSYN_STAT_0); 4389 4390 if (prttsyn_stat & I40E_PRTTSYN_STAT_0_EVENT0_MASK) 4391 schedule_work(&pf->ptp_extts0_work); 4392 4393 if (prttsyn_stat & I40E_PRTTSYN_STAT_0_TXTIME_MASK) 4394 i40e_ptp_tx_hwtstamp(pf); 4395 4396 icr0 &= ~I40E_PFINT_ICR0_ENA_TIMESYNC_MASK; 4397 } 4398 4399 /* If a critical error is pending we have no choice but to reset the 4400 * device. 4401 * Report and mask out any remaining unexpected interrupts. 4402 */ 4403 icr0_remaining = icr0 & ena_mask; 4404 if (icr0_remaining) { 4405 dev_info(&pf->pdev->dev, "unhandled interrupt icr0=0x%08x\n", 4406 icr0_remaining); 4407 if ((icr0_remaining & I40E_PFINT_ICR0_PE_CRITERR_MASK) || 4408 (icr0_remaining & I40E_PFINT_ICR0_PCI_EXCEPTION_MASK) || 4409 (icr0_remaining & I40E_PFINT_ICR0_ECC_ERR_MASK)) { 4410 dev_info(&pf->pdev->dev, "device will be reset\n"); 4411 set_bit(__I40E_PF_RESET_REQUESTED, pf->state); 4412 i40e_service_event_schedule(pf); 4413 } 4414 ena_mask &= ~icr0_remaining; 4415 } 4416 ret = IRQ_HANDLED; 4417 4418 enable_intr: 4419 /* re-enable interrupt causes */ 4420 wr32(hw, I40E_PFINT_ICR0_ENA, ena_mask); 4421 if (!test_bit(__I40E_DOWN, pf->state) || 4422 test_bit(__I40E_RECOVERY_MODE, pf->state)) { 4423 i40e_service_event_schedule(pf); 4424 i40e_irq_dynamic_enable_icr0(pf); 4425 } 4426 4427 return ret; 4428 } 4429 4430 /** 4431 * i40e_clean_fdir_tx_irq - Reclaim resources after transmit completes 4432 * @tx_ring: tx ring to clean 4433 * @budget: how many cleans we're allowed 4434 * 4435 * Returns true if there's any budget left (e.g. the clean is finished) 4436 **/ 4437 static bool i40e_clean_fdir_tx_irq(struct i40e_ring *tx_ring, int budget) 4438 { 4439 struct i40e_vsi *vsi = tx_ring->vsi; 4440 u16 i = tx_ring->next_to_clean; 4441 struct i40e_tx_buffer *tx_buf; 4442 struct i40e_tx_desc *tx_desc; 4443 4444 tx_buf = &tx_ring->tx_bi[i]; 4445 tx_desc = I40E_TX_DESC(tx_ring, i); 4446 i -= tx_ring->count; 4447 4448 do { 4449 struct i40e_tx_desc *eop_desc = tx_buf->next_to_watch; 4450 4451 /* if next_to_watch is not set then there is no work pending */ 4452 if (!eop_desc) 4453 break; 4454 4455 /* prevent any other reads prior to eop_desc */ 4456 smp_rmb(); 4457 4458 /* if the descriptor isn't done, no work yet to do */ 4459 if (!(eop_desc->cmd_type_offset_bsz & 4460 cpu_to_le64(I40E_TX_DESC_DTYPE_DESC_DONE))) 4461 break; 4462 4463 /* clear next_to_watch to prevent false hangs */ 4464 tx_buf->next_to_watch = NULL; 4465 4466 tx_desc->buffer_addr = 0; 4467 tx_desc->cmd_type_offset_bsz = 0; 4468 /* move past filter desc */ 4469 tx_buf++; 4470 tx_desc++; 4471 i++; 4472 if (unlikely(!i)) { 4473 i -= tx_ring->count; 4474 tx_buf = tx_ring->tx_bi; 4475 tx_desc = I40E_TX_DESC(tx_ring, 0); 4476 } 4477 /* unmap skb header data */ 4478 dma_unmap_single(tx_ring->dev, 4479 dma_unmap_addr(tx_buf, dma), 4480 dma_unmap_len(tx_buf, len), 4481 DMA_TO_DEVICE); 4482 if (tx_buf->tx_flags & I40E_TX_FLAGS_FD_SB) 4483 kfree(tx_buf->raw_buf); 4484 4485 tx_buf->raw_buf = NULL; 4486 tx_buf->tx_flags = 0; 4487 tx_buf->next_to_watch = NULL; 4488 dma_unmap_len_set(tx_buf, len, 0); 4489 tx_desc->buffer_addr = 0; 4490 tx_desc->cmd_type_offset_bsz = 0; 4491 4492 /* move us past the eop_desc for start of next FD desc */ 4493 tx_buf++; 4494 tx_desc++; 4495 i++; 4496 if (unlikely(!i)) { 4497 i -= tx_ring->count; 4498 tx_buf = tx_ring->tx_bi; 4499 tx_desc = I40E_TX_DESC(tx_ring, 0); 4500 } 4501 4502 /* update budget accounting */ 4503 budget--; 4504 } while (likely(budget)); 4505 4506 i += tx_ring->count; 4507 tx_ring->next_to_clean = i; 4508 4509 if (test_bit(I40E_FLAG_MSIX_ENA, vsi->back->flags)) 4510 i40e_irq_dynamic_enable(vsi, tx_ring->q_vector->v_idx); 4511 4512 return budget > 0; 4513 } 4514 4515 /** 4516 * i40e_fdir_clean_ring - Interrupt Handler for FDIR SB ring 4517 * @irq: interrupt number 4518 * @data: pointer to a q_vector 4519 **/ 4520 static irqreturn_t i40e_fdir_clean_ring(int irq, void *data) 4521 { 4522 struct i40e_q_vector *q_vector = data; 4523 struct i40e_vsi *vsi; 4524 4525 if (!q_vector->tx.ring) 4526 return IRQ_HANDLED; 4527 4528 vsi = q_vector->tx.ring->vsi; 4529 i40e_clean_fdir_tx_irq(q_vector->tx.ring, vsi->work_limit); 4530 4531 return IRQ_HANDLED; 4532 } 4533 4534 /** 4535 * i40e_map_vector_to_qp - Assigns the queue pair to the vector 4536 * @vsi: the VSI being configured 4537 * @v_idx: vector index 4538 * @qp_idx: queue pair index 4539 **/ 4540 static void i40e_map_vector_to_qp(struct i40e_vsi *vsi, int v_idx, int qp_idx) 4541 { 4542 struct i40e_q_vector *q_vector = vsi->q_vectors[v_idx]; 4543 struct i40e_ring *tx_ring = vsi->tx_rings[qp_idx]; 4544 struct i40e_ring *rx_ring = vsi->rx_rings[qp_idx]; 4545 4546 tx_ring->q_vector = q_vector; 4547 tx_ring->next = q_vector->tx.ring; 4548 q_vector->tx.ring = tx_ring; 4549 q_vector->tx.count++; 4550 4551 /* Place XDP Tx ring in the same q_vector ring list as regular Tx */ 4552 if (i40e_enabled_xdp_vsi(vsi)) { 4553 struct i40e_ring *xdp_ring = vsi->xdp_rings[qp_idx]; 4554 4555 xdp_ring->q_vector = q_vector; 4556 xdp_ring->next = q_vector->tx.ring; 4557 q_vector->tx.ring = xdp_ring; 4558 q_vector->tx.count++; 4559 } 4560 4561 rx_ring->q_vector = q_vector; 4562 rx_ring->next = q_vector->rx.ring; 4563 q_vector->rx.ring = rx_ring; 4564 q_vector->rx.count++; 4565 } 4566 4567 /** 4568 * i40e_vsi_map_rings_to_vectors - Maps descriptor rings to vectors 4569 * @vsi: the VSI being configured 4570 * 4571 * This function maps descriptor rings to the queue-specific vectors 4572 * we were allotted through the MSI-X enabling code. Ideally, we'd have 4573 * one vector per queue pair, but on a constrained vector budget, we 4574 * group the queue pairs as "efficiently" as possible. 4575 **/ 4576 static void i40e_vsi_map_rings_to_vectors(struct i40e_vsi *vsi) 4577 { 4578 int qp_remaining = vsi->num_queue_pairs; 4579 int q_vectors = vsi->num_q_vectors; 4580 int num_ringpairs; 4581 int v_start = 0; 4582 int qp_idx = 0; 4583 4584 /* If we don't have enough vectors for a 1-to-1 mapping, we'll have to 4585 * group them so there are multiple queues per vector. 4586 * It is also important to go through all the vectors available to be 4587 * sure that if we don't use all the vectors, that the remaining vectors 4588 * are cleared. This is especially important when decreasing the 4589 * number of queues in use. 4590 */ 4591 for (; v_start < q_vectors; v_start++) { 4592 struct i40e_q_vector *q_vector = vsi->q_vectors[v_start]; 4593 4594 num_ringpairs = DIV_ROUND_UP(qp_remaining, q_vectors - v_start); 4595 4596 q_vector->num_ringpairs = num_ringpairs; 4597 q_vector->reg_idx = q_vector->v_idx + vsi->base_vector - 1; 4598 4599 q_vector->rx.count = 0; 4600 q_vector->tx.count = 0; 4601 q_vector->rx.ring = NULL; 4602 q_vector->tx.ring = NULL; 4603 4604 while (num_ringpairs--) { 4605 i40e_map_vector_to_qp(vsi, v_start, qp_idx); 4606 qp_idx++; 4607 qp_remaining--; 4608 } 4609 } 4610 } 4611 4612 /** 4613 * i40e_vsi_request_irq - Request IRQ from the OS 4614 * @vsi: the VSI being configured 4615 * @basename: name for the vector 4616 **/ 4617 static int i40e_vsi_request_irq(struct i40e_vsi *vsi, char *basename) 4618 { 4619 struct i40e_pf *pf = vsi->back; 4620 int err; 4621 4622 if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags)) 4623 err = i40e_vsi_request_irq_msix(vsi, basename); 4624 else if (test_bit(I40E_FLAG_MSI_ENA, pf->flags)) 4625 err = request_irq(pf->pdev->irq, i40e_intr, 0, 4626 pf->int_name, pf); 4627 else 4628 err = request_irq(pf->pdev->irq, i40e_intr, IRQF_SHARED, 4629 pf->int_name, pf); 4630 4631 if (err) 4632 dev_info(&pf->pdev->dev, "request_irq failed, Error %d\n", err); 4633 4634 return err; 4635 } 4636 4637 #ifdef CONFIG_NET_POLL_CONTROLLER 4638 /** 4639 * i40e_netpoll - A Polling 'interrupt' handler 4640 * @netdev: network interface device structure 4641 * 4642 * This is used by netconsole to send skbs without having to re-enable 4643 * interrupts. It's not called while the normal interrupt routine is executing. 4644 **/ 4645 static void i40e_netpoll(struct net_device *netdev) 4646 { 4647 struct i40e_netdev_priv *np = netdev_priv(netdev); 4648 struct i40e_vsi *vsi = np->vsi; 4649 struct i40e_pf *pf = vsi->back; 4650 int i; 4651 4652 /* if interface is down do nothing */ 4653 if (test_bit(__I40E_VSI_DOWN, vsi->state)) 4654 return; 4655 4656 if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags)) { 4657 for (i = 0; i < vsi->num_q_vectors; i++) 4658 i40e_msix_clean_rings(0, vsi->q_vectors[i]); 4659 } else { 4660 i40e_intr(pf->pdev->irq, netdev); 4661 } 4662 } 4663 #endif 4664 4665 #define I40E_QTX_ENA_WAIT_COUNT 50 4666 4667 /** 4668 * i40e_pf_txq_wait - Wait for a PF's Tx queue to be enabled or disabled 4669 * @pf: the PF being configured 4670 * @pf_q: the PF queue 4671 * @enable: enable or disable state of the queue 4672 * 4673 * This routine will wait for the given Tx queue of the PF to reach the 4674 * enabled or disabled state. 4675 * Returns -ETIMEDOUT in case of failing to reach the requested state after 4676 * multiple retries; else will return 0 in case of success. 4677 **/ 4678 static int i40e_pf_txq_wait(struct i40e_pf *pf, int pf_q, bool enable) 4679 { 4680 int i; 4681 u32 tx_reg; 4682 4683 for (i = 0; i < I40E_QUEUE_WAIT_RETRY_LIMIT; i++) { 4684 tx_reg = rd32(&pf->hw, I40E_QTX_ENA(pf_q)); 4685 if (enable == !!(tx_reg & I40E_QTX_ENA_QENA_STAT_MASK)) 4686 break; 4687 4688 usleep_range(10, 20); 4689 } 4690 if (i >= I40E_QUEUE_WAIT_RETRY_LIMIT) 4691 return -ETIMEDOUT; 4692 4693 return 0; 4694 } 4695 4696 /** 4697 * i40e_control_tx_q - Start or stop a particular Tx queue 4698 * @pf: the PF structure 4699 * @pf_q: the PF queue to configure 4700 * @enable: start or stop the queue 4701 * 4702 * This function enables or disables a single queue. Note that any delay 4703 * required after the operation is expected to be handled by the caller of 4704 * this function. 4705 **/ 4706 static void i40e_control_tx_q(struct i40e_pf *pf, int pf_q, bool enable) 4707 { 4708 struct i40e_hw *hw = &pf->hw; 4709 u32 tx_reg; 4710 int i; 4711 4712 /* warn the TX unit of coming changes */ 4713 i40e_pre_tx_queue_cfg(&pf->hw, pf_q, enable); 4714 if (!enable) 4715 usleep_range(10, 20); 4716 4717 for (i = 0; i < I40E_QTX_ENA_WAIT_COUNT; i++) { 4718 tx_reg = rd32(hw, I40E_QTX_ENA(pf_q)); 4719 if (((tx_reg >> I40E_QTX_ENA_QENA_REQ_SHIFT) & 1) == 4720 ((tx_reg >> I40E_QTX_ENA_QENA_STAT_SHIFT) & 1)) 4721 break; 4722 usleep_range(1000, 2000); 4723 } 4724 4725 /* Skip if the queue is already in the requested state */ 4726 if (enable == !!(tx_reg & I40E_QTX_ENA_QENA_STAT_MASK)) 4727 return; 4728 4729 /* turn on/off the queue */ 4730 if (enable) { 4731 wr32(hw, I40E_QTX_HEAD(pf_q), 0); 4732 tx_reg |= I40E_QTX_ENA_QENA_REQ_MASK; 4733 } else { 4734 tx_reg &= ~I40E_QTX_ENA_QENA_REQ_MASK; 4735 } 4736 4737 wr32(hw, I40E_QTX_ENA(pf_q), tx_reg); 4738 } 4739 4740 /** 4741 * i40e_control_wait_tx_q - Start/stop Tx queue and wait for completion 4742 * @seid: VSI SEID 4743 * @pf: the PF structure 4744 * @pf_q: the PF queue to configure 4745 * @is_xdp: true if the queue is used for XDP 4746 * @enable: start or stop the queue 4747 **/ 4748 int i40e_control_wait_tx_q(int seid, struct i40e_pf *pf, int pf_q, 4749 bool is_xdp, bool enable) 4750 { 4751 int ret; 4752 4753 i40e_control_tx_q(pf, pf_q, enable); 4754 4755 /* wait for the change to finish */ 4756 ret = i40e_pf_txq_wait(pf, pf_q, enable); 4757 if (ret) { 4758 dev_info(&pf->pdev->dev, 4759 "VSI seid %d %sTx ring %d %sable timeout\n", 4760 seid, (is_xdp ? "XDP " : ""), pf_q, 4761 (enable ? "en" : "dis")); 4762 } 4763 4764 return ret; 4765 } 4766 4767 /** 4768 * i40e_vsi_enable_tx - Start a VSI's rings 4769 * @vsi: the VSI being configured 4770 **/ 4771 static int i40e_vsi_enable_tx(struct i40e_vsi *vsi) 4772 { 4773 struct i40e_pf *pf = vsi->back; 4774 int i, pf_q, ret = 0; 4775 4776 pf_q = vsi->base_queue; 4777 for (i = 0; i < vsi->num_queue_pairs; i++, pf_q++) { 4778 ret = i40e_control_wait_tx_q(vsi->seid, pf, 4779 pf_q, 4780 false /*is xdp*/, true); 4781 if (ret) 4782 break; 4783 4784 if (!i40e_enabled_xdp_vsi(vsi)) 4785 continue; 4786 4787 ret = i40e_control_wait_tx_q(vsi->seid, pf, 4788 pf_q + vsi->alloc_queue_pairs, 4789 true /*is xdp*/, true); 4790 if (ret) 4791 break; 4792 } 4793 return ret; 4794 } 4795 4796 /** 4797 * i40e_pf_rxq_wait - Wait for a PF's Rx queue to be enabled or disabled 4798 * @pf: the PF being configured 4799 * @pf_q: the PF queue 4800 * @enable: enable or disable state of the queue 4801 * 4802 * This routine will wait for the given Rx queue of the PF to reach the 4803 * enabled or disabled state. 4804 * Returns -ETIMEDOUT in case of failing to reach the requested state after 4805 * multiple retries; else will return 0 in case of success. 4806 **/ 4807 static int i40e_pf_rxq_wait(struct i40e_pf *pf, int pf_q, bool enable) 4808 { 4809 int i; 4810 u32 rx_reg; 4811 4812 for (i = 0; i < I40E_QUEUE_WAIT_RETRY_LIMIT; i++) { 4813 rx_reg = rd32(&pf->hw, I40E_QRX_ENA(pf_q)); 4814 if (enable == !!(rx_reg & I40E_QRX_ENA_QENA_STAT_MASK)) 4815 break; 4816 4817 usleep_range(10, 20); 4818 } 4819 if (i >= I40E_QUEUE_WAIT_RETRY_LIMIT) 4820 return -ETIMEDOUT; 4821 4822 return 0; 4823 } 4824 4825 /** 4826 * i40e_control_rx_q - Start or stop a particular Rx queue 4827 * @pf: the PF structure 4828 * @pf_q: the PF queue to configure 4829 * @enable: start or stop the queue 4830 * 4831 * This function enables or disables a single queue. Note that 4832 * any delay required after the operation is expected to be 4833 * handled by the caller of this function. 4834 **/ 4835 static void i40e_control_rx_q(struct i40e_pf *pf, int pf_q, bool enable) 4836 { 4837 struct i40e_hw *hw = &pf->hw; 4838 u32 rx_reg; 4839 int i; 4840 4841 for (i = 0; i < I40E_QTX_ENA_WAIT_COUNT; i++) { 4842 rx_reg = rd32(hw, I40E_QRX_ENA(pf_q)); 4843 if (((rx_reg >> I40E_QRX_ENA_QENA_REQ_SHIFT) & 1) == 4844 ((rx_reg >> I40E_QRX_ENA_QENA_STAT_SHIFT) & 1)) 4845 break; 4846 usleep_range(1000, 2000); 4847 } 4848 4849 /* Skip if the queue is already in the requested state */ 4850 if (enable == !!(rx_reg & I40E_QRX_ENA_QENA_STAT_MASK)) 4851 return; 4852 4853 /* turn on/off the queue */ 4854 if (enable) 4855 rx_reg |= I40E_QRX_ENA_QENA_REQ_MASK; 4856 else 4857 rx_reg &= ~I40E_QRX_ENA_QENA_REQ_MASK; 4858 4859 wr32(hw, I40E_QRX_ENA(pf_q), rx_reg); 4860 } 4861 4862 /** 4863 * i40e_control_wait_rx_q 4864 * @pf: the PF structure 4865 * @pf_q: queue being configured 4866 * @enable: start or stop the rings 4867 * 4868 * This function enables or disables a single queue along with waiting 4869 * for the change to finish. The caller of this function should handle 4870 * the delays needed in the case of disabling queues. 4871 **/ 4872 int i40e_control_wait_rx_q(struct i40e_pf *pf, int pf_q, bool enable) 4873 { 4874 int ret = 0; 4875 4876 i40e_control_rx_q(pf, pf_q, enable); 4877 4878 /* wait for the change to finish */ 4879 ret = i40e_pf_rxq_wait(pf, pf_q, enable); 4880 if (ret) 4881 return ret; 4882 4883 return ret; 4884 } 4885 4886 /** 4887 * i40e_vsi_enable_rx - Start a VSI's rings 4888 * @vsi: the VSI being configured 4889 **/ 4890 static int i40e_vsi_enable_rx(struct i40e_vsi *vsi) 4891 { 4892 struct i40e_pf *pf = vsi->back; 4893 int i, pf_q, ret = 0; 4894 4895 pf_q = vsi->base_queue; 4896 for (i = 0; i < vsi->num_queue_pairs; i++, pf_q++) { 4897 ret = i40e_control_wait_rx_q(pf, pf_q, true); 4898 if (ret) { 4899 dev_info(&pf->pdev->dev, 4900 "VSI seid %d Rx ring %d enable timeout\n", 4901 vsi->seid, pf_q); 4902 break; 4903 } 4904 } 4905 4906 return ret; 4907 } 4908 4909 /** 4910 * i40e_vsi_start_rings - Start a VSI's rings 4911 * @vsi: the VSI being configured 4912 **/ 4913 int i40e_vsi_start_rings(struct i40e_vsi *vsi) 4914 { 4915 int ret = 0; 4916 4917 /* do rx first for enable and last for disable */ 4918 ret = i40e_vsi_enable_rx(vsi); 4919 if (ret) 4920 return ret; 4921 ret = i40e_vsi_enable_tx(vsi); 4922 4923 return ret; 4924 } 4925 4926 #define I40E_DISABLE_TX_GAP_MSEC 50 4927 4928 /** 4929 * i40e_vsi_stop_rings - Stop a VSI's rings 4930 * @vsi: the VSI being configured 4931 **/ 4932 void i40e_vsi_stop_rings(struct i40e_vsi *vsi) 4933 { 4934 struct i40e_pf *pf = vsi->back; 4935 u32 pf_q, tx_q_end, rx_q_end; 4936 4937 /* When port TX is suspended, don't wait */ 4938 if (test_bit(__I40E_PORT_SUSPENDED, vsi->back->state)) 4939 return i40e_vsi_stop_rings_no_wait(vsi); 4940 4941 tx_q_end = vsi->base_queue + 4942 vsi->alloc_queue_pairs * (i40e_enabled_xdp_vsi(vsi) ? 2 : 1); 4943 for (pf_q = vsi->base_queue; pf_q < tx_q_end; pf_q++) 4944 i40e_pre_tx_queue_cfg(&pf->hw, pf_q, false); 4945 4946 rx_q_end = vsi->base_queue + vsi->num_queue_pairs; 4947 for (pf_q = vsi->base_queue; pf_q < rx_q_end; pf_q++) 4948 i40e_control_rx_q(pf, pf_q, false); 4949 4950 msleep(I40E_DISABLE_TX_GAP_MSEC); 4951 for (pf_q = vsi->base_queue; pf_q < tx_q_end; pf_q++) 4952 wr32(&pf->hw, I40E_QTX_ENA(pf_q), 0); 4953 4954 i40e_vsi_wait_queues_disabled(vsi); 4955 } 4956 4957 /** 4958 * i40e_vsi_stop_rings_no_wait - Stop a VSI's rings and do not delay 4959 * @vsi: the VSI being shutdown 4960 * 4961 * This function stops all the rings for a VSI but does not delay to verify 4962 * that rings have been disabled. It is expected that the caller is shutting 4963 * down multiple VSIs at once and will delay together for all the VSIs after 4964 * initiating the shutdown. This is particularly useful for shutting down lots 4965 * of VFs together. Otherwise, a large delay can be incurred while configuring 4966 * each VSI in serial. 4967 **/ 4968 void i40e_vsi_stop_rings_no_wait(struct i40e_vsi *vsi) 4969 { 4970 struct i40e_pf *pf = vsi->back; 4971 int i, pf_q; 4972 4973 pf_q = vsi->base_queue; 4974 for (i = 0; i < vsi->num_queue_pairs; i++, pf_q++) { 4975 i40e_control_tx_q(pf, pf_q, false); 4976 i40e_control_rx_q(pf, pf_q, false); 4977 } 4978 } 4979 4980 /** 4981 * i40e_vsi_free_irq - Free the irq association with the OS 4982 * @vsi: the VSI being configured 4983 **/ 4984 static void i40e_vsi_free_irq(struct i40e_vsi *vsi) 4985 { 4986 struct i40e_pf *pf = vsi->back; 4987 struct i40e_hw *hw = &pf->hw; 4988 int base = vsi->base_vector; 4989 u32 val, qp; 4990 int i; 4991 4992 if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags)) { 4993 if (!vsi->q_vectors) 4994 return; 4995 4996 if (!vsi->irqs_ready) 4997 return; 4998 4999 vsi->irqs_ready = false; 5000 for (i = 0; i < vsi->num_q_vectors; i++) { 5001 int irq_num; 5002 u16 vector; 5003 5004 vector = i + base; 5005 irq_num = pf->msix_entries[vector].vector; 5006 5007 /* free only the irqs that were actually requested */ 5008 if (!vsi->q_vectors[i] || 5009 !vsi->q_vectors[i]->num_ringpairs) 5010 continue; 5011 5012 /* clear the affinity notifier in the IRQ descriptor */ 5013 irq_set_affinity_notifier(irq_num, NULL); 5014 /* remove our suggested affinity mask for this IRQ */ 5015 irq_update_affinity_hint(irq_num, NULL); 5016 free_irq(irq_num, vsi->q_vectors[i]); 5017 5018 /* Tear down the interrupt queue link list 5019 * 5020 * We know that they come in pairs and always 5021 * the Rx first, then the Tx. To clear the 5022 * link list, stick the EOL value into the 5023 * next_q field of the registers. 5024 */ 5025 val = rd32(hw, I40E_PFINT_LNKLSTN(vector - 1)); 5026 qp = FIELD_GET(I40E_PFINT_LNKLSTN_FIRSTQ_INDX_MASK, 5027 val); 5028 val |= I40E_QUEUE_END_OF_LIST 5029 << I40E_PFINT_LNKLSTN_FIRSTQ_INDX_SHIFT; 5030 wr32(hw, I40E_PFINT_LNKLSTN(vector - 1), val); 5031 5032 while (qp != I40E_QUEUE_END_OF_LIST) { 5033 u32 next; 5034 5035 val = rd32(hw, I40E_QINT_RQCTL(qp)); 5036 5037 val &= ~(I40E_QINT_RQCTL_MSIX_INDX_MASK | 5038 I40E_QINT_RQCTL_MSIX0_INDX_MASK | 5039 I40E_QINT_RQCTL_CAUSE_ENA_MASK | 5040 I40E_QINT_RQCTL_INTEVENT_MASK); 5041 5042 val |= (I40E_QINT_RQCTL_ITR_INDX_MASK | 5043 I40E_QINT_RQCTL_NEXTQ_INDX_MASK); 5044 5045 wr32(hw, I40E_QINT_RQCTL(qp), val); 5046 5047 val = rd32(hw, I40E_QINT_TQCTL(qp)); 5048 5049 next = FIELD_GET(I40E_QINT_TQCTL_NEXTQ_INDX_MASK, 5050 val); 5051 5052 val &= ~(I40E_QINT_TQCTL_MSIX_INDX_MASK | 5053 I40E_QINT_TQCTL_MSIX0_INDX_MASK | 5054 I40E_QINT_TQCTL_CAUSE_ENA_MASK | 5055 I40E_QINT_TQCTL_INTEVENT_MASK); 5056 5057 val |= (I40E_QINT_TQCTL_ITR_INDX_MASK | 5058 I40E_QINT_TQCTL_NEXTQ_INDX_MASK); 5059 5060 wr32(hw, I40E_QINT_TQCTL(qp), val); 5061 qp = next; 5062 } 5063 } 5064 } else { 5065 free_irq(pf->pdev->irq, pf); 5066 5067 val = rd32(hw, I40E_PFINT_LNKLST0); 5068 qp = FIELD_GET(I40E_PFINT_LNKLSTN_FIRSTQ_INDX_MASK, val); 5069 val |= I40E_QUEUE_END_OF_LIST 5070 << I40E_PFINT_LNKLST0_FIRSTQ_INDX_SHIFT; 5071 wr32(hw, I40E_PFINT_LNKLST0, val); 5072 5073 val = rd32(hw, I40E_QINT_RQCTL(qp)); 5074 val &= ~(I40E_QINT_RQCTL_MSIX_INDX_MASK | 5075 I40E_QINT_RQCTL_MSIX0_INDX_MASK | 5076 I40E_QINT_RQCTL_CAUSE_ENA_MASK | 5077 I40E_QINT_RQCTL_INTEVENT_MASK); 5078 5079 val |= (I40E_QINT_RQCTL_ITR_INDX_MASK | 5080 I40E_QINT_RQCTL_NEXTQ_INDX_MASK); 5081 5082 wr32(hw, I40E_QINT_RQCTL(qp), val); 5083 5084 val = rd32(hw, I40E_QINT_TQCTL(qp)); 5085 5086 val &= ~(I40E_QINT_TQCTL_MSIX_INDX_MASK | 5087 I40E_QINT_TQCTL_MSIX0_INDX_MASK | 5088 I40E_QINT_TQCTL_CAUSE_ENA_MASK | 5089 I40E_QINT_TQCTL_INTEVENT_MASK); 5090 5091 val |= (I40E_QINT_TQCTL_ITR_INDX_MASK | 5092 I40E_QINT_TQCTL_NEXTQ_INDX_MASK); 5093 5094 wr32(hw, I40E_QINT_TQCTL(qp), val); 5095 } 5096 } 5097 5098 /** 5099 * i40e_free_q_vector - Free memory allocated for specific interrupt vector 5100 * @vsi: the VSI being configured 5101 * @v_idx: Index of vector to be freed 5102 * 5103 * This function frees the memory allocated to the q_vector. In addition if 5104 * NAPI is enabled it will delete any references to the NAPI struct prior 5105 * to freeing the q_vector. 5106 **/ 5107 static void i40e_free_q_vector(struct i40e_vsi *vsi, int v_idx) 5108 { 5109 struct i40e_q_vector *q_vector = vsi->q_vectors[v_idx]; 5110 struct i40e_ring *ring; 5111 5112 if (!q_vector) 5113 return; 5114 5115 /* disassociate q_vector from rings */ 5116 i40e_for_each_ring(ring, q_vector->tx) 5117 ring->q_vector = NULL; 5118 5119 i40e_for_each_ring(ring, q_vector->rx) 5120 ring->q_vector = NULL; 5121 5122 /* only VSI w/ an associated netdev is set up w/ NAPI */ 5123 if (vsi->netdev) 5124 netif_napi_del(&q_vector->napi); 5125 5126 vsi->q_vectors[v_idx] = NULL; 5127 5128 kfree_rcu(q_vector, rcu); 5129 } 5130 5131 /** 5132 * i40e_vsi_free_q_vectors - Free memory allocated for interrupt vectors 5133 * @vsi: the VSI being un-configured 5134 * 5135 * This frees the memory allocated to the q_vectors and 5136 * deletes references to the NAPI struct. 5137 **/ 5138 static void i40e_vsi_free_q_vectors(struct i40e_vsi *vsi) 5139 { 5140 int v_idx; 5141 5142 for (v_idx = 0; v_idx < vsi->num_q_vectors; v_idx++) 5143 i40e_free_q_vector(vsi, v_idx); 5144 } 5145 5146 /** 5147 * i40e_reset_interrupt_capability - Disable interrupt setup in OS 5148 * @pf: board private structure 5149 **/ 5150 static void i40e_reset_interrupt_capability(struct i40e_pf *pf) 5151 { 5152 /* If we're in Legacy mode, the interrupt was cleaned in vsi_close */ 5153 if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags)) { 5154 pci_disable_msix(pf->pdev); 5155 kfree(pf->msix_entries); 5156 pf->msix_entries = NULL; 5157 kfree(pf->irq_pile); 5158 pf->irq_pile = NULL; 5159 } else if (test_bit(I40E_FLAG_MSI_ENA, pf->flags)) { 5160 pci_disable_msi(pf->pdev); 5161 } 5162 clear_bit(I40E_FLAG_MSI_ENA, pf->flags); 5163 clear_bit(I40E_FLAG_MSIX_ENA, pf->flags); 5164 } 5165 5166 /** 5167 * i40e_clear_interrupt_scheme - Clear the current interrupt scheme settings 5168 * @pf: board private structure 5169 * 5170 * We go through and clear interrupt specific resources and reset the structure 5171 * to pre-load conditions 5172 **/ 5173 static void i40e_clear_interrupt_scheme(struct i40e_pf *pf) 5174 { 5175 struct i40e_vsi *vsi; 5176 int i; 5177 5178 if (test_bit(__I40E_MISC_IRQ_REQUESTED, pf->state)) 5179 i40e_free_misc_vector(pf); 5180 5181 i40e_put_lump(pf->irq_pile, pf->iwarp_base_vector, 5182 I40E_IWARP_IRQ_PILE_ID); 5183 5184 i40e_put_lump(pf->irq_pile, 0, I40E_PILE_VALID_BIT-1); 5185 5186 i40e_pf_for_each_vsi(pf, i, vsi) 5187 i40e_vsi_free_q_vectors(vsi); 5188 5189 i40e_reset_interrupt_capability(pf); 5190 } 5191 5192 /** 5193 * i40e_napi_enable_all - Enable NAPI for all q_vectors in the VSI 5194 * @vsi: the VSI being configured 5195 **/ 5196 static void i40e_napi_enable_all(struct i40e_vsi *vsi) 5197 { 5198 int q_idx; 5199 5200 if (!vsi->netdev) 5201 return; 5202 5203 for (q_idx = 0; q_idx < vsi->num_q_vectors; q_idx++) { 5204 struct i40e_q_vector *q_vector = vsi->q_vectors[q_idx]; 5205 5206 if (q_vector->rx.ring || q_vector->tx.ring) 5207 napi_enable(&q_vector->napi); 5208 } 5209 } 5210 5211 /** 5212 * i40e_napi_disable_all - Disable NAPI for all q_vectors in the VSI 5213 * @vsi: the VSI being configured 5214 **/ 5215 static void i40e_napi_disable_all(struct i40e_vsi *vsi) 5216 { 5217 int q_idx; 5218 5219 if (!vsi->netdev) 5220 return; 5221 5222 for (q_idx = 0; q_idx < vsi->num_q_vectors; q_idx++) { 5223 struct i40e_q_vector *q_vector = vsi->q_vectors[q_idx]; 5224 5225 if (q_vector->rx.ring || q_vector->tx.ring) 5226 napi_disable(&q_vector->napi); 5227 } 5228 } 5229 5230 /** 5231 * i40e_vsi_close - Shut down a VSI 5232 * @vsi: the vsi to be quelled 5233 **/ 5234 static void i40e_vsi_close(struct i40e_vsi *vsi) 5235 { 5236 struct i40e_pf *pf = vsi->back; 5237 if (!test_and_set_bit(__I40E_VSI_DOWN, vsi->state)) 5238 i40e_down(vsi); 5239 i40e_vsi_free_irq(vsi); 5240 i40e_vsi_free_tx_resources(vsi); 5241 i40e_vsi_free_rx_resources(vsi); 5242 vsi->current_netdev_flags = 0; 5243 set_bit(__I40E_CLIENT_SERVICE_REQUESTED, pf->state); 5244 if (test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state)) 5245 set_bit(__I40E_CLIENT_RESET, pf->state); 5246 } 5247 5248 /** 5249 * i40e_quiesce_vsi - Pause a given VSI 5250 * @vsi: the VSI being paused 5251 **/ 5252 static void i40e_quiesce_vsi(struct i40e_vsi *vsi) 5253 { 5254 if (test_bit(__I40E_VSI_DOWN, vsi->state)) 5255 return; 5256 5257 set_bit(__I40E_VSI_NEEDS_RESTART, vsi->state); 5258 if (vsi->netdev && netif_running(vsi->netdev)) 5259 vsi->netdev->netdev_ops->ndo_stop(vsi->netdev); 5260 else 5261 i40e_vsi_close(vsi); 5262 } 5263 5264 /** 5265 * i40e_unquiesce_vsi - Resume a given VSI 5266 * @vsi: the VSI being resumed 5267 **/ 5268 static void i40e_unquiesce_vsi(struct i40e_vsi *vsi) 5269 { 5270 if (!test_and_clear_bit(__I40E_VSI_NEEDS_RESTART, vsi->state)) 5271 return; 5272 5273 if (vsi->netdev && netif_running(vsi->netdev)) 5274 vsi->netdev->netdev_ops->ndo_open(vsi->netdev); 5275 else 5276 i40e_vsi_open(vsi); /* this clears the DOWN bit */ 5277 } 5278 5279 /** 5280 * i40e_pf_quiesce_all_vsi - Pause all VSIs on a PF 5281 * @pf: the PF 5282 **/ 5283 static void i40e_pf_quiesce_all_vsi(struct i40e_pf *pf) 5284 { 5285 struct i40e_vsi *vsi; 5286 int v; 5287 5288 i40e_pf_for_each_vsi(pf, v, vsi) 5289 i40e_quiesce_vsi(vsi); 5290 } 5291 5292 /** 5293 * i40e_pf_unquiesce_all_vsi - Resume all VSIs on a PF 5294 * @pf: the PF 5295 **/ 5296 static void i40e_pf_unquiesce_all_vsi(struct i40e_pf *pf) 5297 { 5298 struct i40e_vsi *vsi; 5299 int v; 5300 5301 i40e_pf_for_each_vsi(pf, v, vsi) 5302 i40e_unquiesce_vsi(vsi); 5303 } 5304 5305 /** 5306 * i40e_vsi_wait_queues_disabled - Wait for VSI's queues to be disabled 5307 * @vsi: the VSI being configured 5308 * 5309 * Wait until all queues on a given VSI have been disabled. 5310 **/ 5311 int i40e_vsi_wait_queues_disabled(struct i40e_vsi *vsi) 5312 { 5313 struct i40e_pf *pf = vsi->back; 5314 int i, pf_q, ret; 5315 5316 pf_q = vsi->base_queue; 5317 for (i = 0; i < vsi->num_queue_pairs; i++, pf_q++) { 5318 /* Check and wait for the Tx queue */ 5319 ret = i40e_pf_txq_wait(pf, pf_q, false); 5320 if (ret) { 5321 dev_info(&pf->pdev->dev, 5322 "VSI seid %d Tx ring %d disable timeout\n", 5323 vsi->seid, pf_q); 5324 return ret; 5325 } 5326 5327 if (!i40e_enabled_xdp_vsi(vsi)) 5328 goto wait_rx; 5329 5330 /* Check and wait for the XDP Tx queue */ 5331 ret = i40e_pf_txq_wait(pf, pf_q + vsi->alloc_queue_pairs, 5332 false); 5333 if (ret) { 5334 dev_info(&pf->pdev->dev, 5335 "VSI seid %d XDP Tx ring %d disable timeout\n", 5336 vsi->seid, pf_q); 5337 return ret; 5338 } 5339 wait_rx: 5340 /* Check and wait for the Rx queue */ 5341 ret = i40e_pf_rxq_wait(pf, pf_q, false); 5342 if (ret) { 5343 dev_info(&pf->pdev->dev, 5344 "VSI seid %d Rx ring %d disable timeout\n", 5345 vsi->seid, pf_q); 5346 return ret; 5347 } 5348 } 5349 5350 return 0; 5351 } 5352 5353 #ifdef CONFIG_I40E_DCB 5354 /** 5355 * i40e_pf_wait_queues_disabled - Wait for all queues of PF VSIs to be disabled 5356 * @pf: the PF 5357 * 5358 * This function waits for the queues to be in disabled state for all the 5359 * VSIs that are managed by this PF. 5360 **/ 5361 static int i40e_pf_wait_queues_disabled(struct i40e_pf *pf) 5362 { 5363 struct i40e_vsi *vsi; 5364 int v, ret = 0; 5365 5366 i40e_pf_for_each_vsi(pf, v, vsi) { 5367 ret = i40e_vsi_wait_queues_disabled(vsi); 5368 if (ret) 5369 break; 5370 } 5371 5372 return ret; 5373 } 5374 5375 #endif 5376 5377 /** 5378 * i40e_get_iscsi_tc_map - Return TC map for iSCSI APP 5379 * @pf: pointer to PF 5380 * 5381 * Get TC map for ISCSI PF type that will include iSCSI TC 5382 * and LAN TC. 5383 **/ 5384 static u8 i40e_get_iscsi_tc_map(struct i40e_pf *pf) 5385 { 5386 struct i40e_dcb_app_priority_table app; 5387 struct i40e_hw *hw = &pf->hw; 5388 u8 enabled_tc = 1; /* TC0 is always enabled */ 5389 u8 tc, i; 5390 /* Get the iSCSI APP TLV */ 5391 struct i40e_dcbx_config *dcbcfg = &hw->local_dcbx_config; 5392 5393 for (i = 0; i < dcbcfg->numapps; i++) { 5394 app = dcbcfg->app[i]; 5395 if (app.selector == I40E_APP_SEL_TCPIP && 5396 app.protocolid == I40E_APP_PROTOID_ISCSI) { 5397 tc = dcbcfg->etscfg.prioritytable[app.priority]; 5398 enabled_tc |= BIT(tc); 5399 break; 5400 } 5401 } 5402 5403 return enabled_tc; 5404 } 5405 5406 /** 5407 * i40e_dcb_get_num_tc - Get the number of TCs from DCBx config 5408 * @dcbcfg: the corresponding DCBx configuration structure 5409 * 5410 * Return the number of TCs from given DCBx configuration 5411 **/ 5412 static u8 i40e_dcb_get_num_tc(struct i40e_dcbx_config *dcbcfg) 5413 { 5414 int i, tc_unused = 0; 5415 u8 num_tc = 0; 5416 u8 ret = 0; 5417 5418 /* Scan the ETS Config Priority Table to find 5419 * traffic class enabled for a given priority 5420 * and create a bitmask of enabled TCs 5421 */ 5422 for (i = 0; i < I40E_MAX_USER_PRIORITY; i++) 5423 num_tc |= BIT(dcbcfg->etscfg.prioritytable[i]); 5424 5425 /* Now scan the bitmask to check for 5426 * contiguous TCs starting with TC0 5427 */ 5428 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 5429 if (num_tc & BIT(i)) { 5430 if (!tc_unused) { 5431 ret++; 5432 } else { 5433 pr_err("Non-contiguous TC - Disabling DCB\n"); 5434 return 1; 5435 } 5436 } else { 5437 tc_unused = 1; 5438 } 5439 } 5440 5441 /* There is always at least TC0 */ 5442 if (!ret) 5443 ret = 1; 5444 5445 return ret; 5446 } 5447 5448 /** 5449 * i40e_dcb_get_enabled_tc - Get enabled traffic classes 5450 * @dcbcfg: the corresponding DCBx configuration structure 5451 * 5452 * Query the current DCB configuration and return the number of 5453 * traffic classes enabled from the given DCBX config 5454 **/ 5455 static u8 i40e_dcb_get_enabled_tc(struct i40e_dcbx_config *dcbcfg) 5456 { 5457 u8 num_tc = i40e_dcb_get_num_tc(dcbcfg); 5458 u8 enabled_tc = 1; 5459 u8 i; 5460 5461 for (i = 0; i < num_tc; i++) 5462 enabled_tc |= BIT(i); 5463 5464 return enabled_tc; 5465 } 5466 5467 /** 5468 * i40e_mqprio_get_enabled_tc - Get enabled traffic classes 5469 * @pf: PF being queried 5470 * 5471 * Query the current MQPRIO configuration and return the number of 5472 * traffic classes enabled. 5473 **/ 5474 static u8 i40e_mqprio_get_enabled_tc(struct i40e_pf *pf) 5475 { 5476 struct i40e_vsi *vsi = i40e_pf_get_main_vsi(pf); 5477 u8 num_tc = vsi->mqprio_qopt.qopt.num_tc; 5478 u8 enabled_tc = 1, i; 5479 5480 for (i = 1; i < num_tc; i++) 5481 enabled_tc |= BIT(i); 5482 return enabled_tc; 5483 } 5484 5485 /** 5486 * i40e_pf_get_num_tc - Get enabled traffic classes for PF 5487 * @pf: PF being queried 5488 * 5489 * Return number of traffic classes enabled for the given PF 5490 **/ 5491 static u8 i40e_pf_get_num_tc(struct i40e_pf *pf) 5492 { 5493 u8 i, enabled_tc = 1; 5494 u8 num_tc = 0; 5495 5496 if (i40e_is_tc_mqprio_enabled(pf)) { 5497 struct i40e_vsi *vsi = i40e_pf_get_main_vsi(pf); 5498 5499 return vsi->mqprio_qopt.qopt.num_tc; 5500 } 5501 5502 /* If neither MQPRIO nor DCB is enabled, then always use single TC */ 5503 if (!test_bit(I40E_FLAG_DCB_ENA, pf->flags)) 5504 return 1; 5505 5506 /* SFP mode will be enabled for all TCs on port */ 5507 if (!test_bit(I40E_FLAG_MFP_ENA, pf->flags)) 5508 return i40e_dcb_get_num_tc(&pf->hw.local_dcbx_config); 5509 5510 /* MFP mode return count of enabled TCs for this PF */ 5511 if (pf->hw.func_caps.iscsi) 5512 enabled_tc = i40e_get_iscsi_tc_map(pf); 5513 else 5514 return 1; /* Only TC0 */ 5515 5516 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 5517 if (enabled_tc & BIT(i)) 5518 num_tc++; 5519 } 5520 return num_tc; 5521 } 5522 5523 /** 5524 * i40e_pf_get_tc_map - Get bitmap for enabled traffic classes 5525 * @pf: PF being queried 5526 * 5527 * Return a bitmap for enabled traffic classes for this PF. 5528 **/ 5529 static u8 i40e_pf_get_tc_map(struct i40e_pf *pf) 5530 { 5531 if (i40e_is_tc_mqprio_enabled(pf)) 5532 return i40e_mqprio_get_enabled_tc(pf); 5533 5534 /* If neither MQPRIO nor DCB is enabled for this PF then just return 5535 * default TC 5536 */ 5537 if (!test_bit(I40E_FLAG_DCB_ENA, pf->flags)) 5538 return I40E_DEFAULT_TRAFFIC_CLASS; 5539 5540 /* SFP mode we want PF to be enabled for all TCs */ 5541 if (!test_bit(I40E_FLAG_MFP_ENA, pf->flags)) 5542 return i40e_dcb_get_enabled_tc(&pf->hw.local_dcbx_config); 5543 5544 /* MFP enabled and iSCSI PF type */ 5545 if (pf->hw.func_caps.iscsi) 5546 return i40e_get_iscsi_tc_map(pf); 5547 else 5548 return I40E_DEFAULT_TRAFFIC_CLASS; 5549 } 5550 5551 /** 5552 * i40e_vsi_get_bw_info - Query VSI BW Information 5553 * @vsi: the VSI being queried 5554 * 5555 * Returns 0 on success, negative value on failure 5556 **/ 5557 static int i40e_vsi_get_bw_info(struct i40e_vsi *vsi) 5558 { 5559 struct i40e_aqc_query_vsi_ets_sla_config_resp bw_ets_config = {0}; 5560 struct i40e_aqc_query_vsi_bw_config_resp bw_config = {0}; 5561 struct i40e_pf *pf = vsi->back; 5562 struct i40e_hw *hw = &pf->hw; 5563 u32 tc_bw_max; 5564 int ret; 5565 int i; 5566 5567 /* Get the VSI level BW configuration */ 5568 ret = i40e_aq_query_vsi_bw_config(hw, vsi->seid, &bw_config, NULL); 5569 if (ret) { 5570 dev_info(&pf->pdev->dev, 5571 "couldn't get PF vsi bw config, err %pe aq_err %s\n", 5572 ERR_PTR(ret), 5573 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 5574 return -EINVAL; 5575 } 5576 5577 /* Get the VSI level BW configuration per TC */ 5578 ret = i40e_aq_query_vsi_ets_sla_config(hw, vsi->seid, &bw_ets_config, 5579 NULL); 5580 if (ret) { 5581 dev_info(&pf->pdev->dev, 5582 "couldn't get PF vsi ets bw config, err %pe aq_err %s\n", 5583 ERR_PTR(ret), 5584 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 5585 return -EINVAL; 5586 } 5587 5588 if (bw_config.tc_valid_bits != bw_ets_config.tc_valid_bits) { 5589 dev_info(&pf->pdev->dev, 5590 "Enabled TCs mismatch from querying VSI BW info 0x%08x 0x%08x\n", 5591 bw_config.tc_valid_bits, 5592 bw_ets_config.tc_valid_bits); 5593 /* Still continuing */ 5594 } 5595 5596 vsi->bw_limit = le16_to_cpu(bw_config.port_bw_limit); 5597 vsi->bw_max_quanta = bw_config.max_bw; 5598 tc_bw_max = le16_to_cpu(bw_ets_config.tc_bw_max[0]) | 5599 (le16_to_cpu(bw_ets_config.tc_bw_max[1]) << 16); 5600 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 5601 vsi->bw_ets_share_credits[i] = bw_ets_config.share_credits[i]; 5602 vsi->bw_ets_limit_credits[i] = 5603 le16_to_cpu(bw_ets_config.credits[i]); 5604 /* 3 bits out of 4 for each TC */ 5605 vsi->bw_ets_max_quanta[i] = (u8)((tc_bw_max >> (i*4)) & 0x7); 5606 } 5607 5608 return 0; 5609 } 5610 5611 /** 5612 * i40e_vsi_configure_bw_alloc - Configure VSI BW allocation per TC 5613 * @vsi: the VSI being configured 5614 * @enabled_tc: TC bitmap 5615 * @bw_share: BW shared credits per TC 5616 * 5617 * Returns 0 on success, negative value on failure 5618 **/ 5619 static int i40e_vsi_configure_bw_alloc(struct i40e_vsi *vsi, u8 enabled_tc, 5620 u8 *bw_share) 5621 { 5622 struct i40e_aqc_configure_vsi_tc_bw_data bw_data; 5623 struct i40e_pf *pf = vsi->back; 5624 int ret; 5625 int i; 5626 5627 /* There is no need to reset BW when mqprio mode is on. */ 5628 if (i40e_is_tc_mqprio_enabled(pf)) 5629 return 0; 5630 if (!vsi->mqprio_qopt.qopt.hw && !test_bit(I40E_FLAG_DCB_ENA, pf->flags)) { 5631 ret = i40e_set_bw_limit(vsi, vsi->seid, 0); 5632 if (ret) 5633 dev_info(&pf->pdev->dev, 5634 "Failed to reset tx rate for vsi->seid %u\n", 5635 vsi->seid); 5636 return ret; 5637 } 5638 memset(&bw_data, 0, sizeof(bw_data)); 5639 bw_data.tc_valid_bits = enabled_tc; 5640 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) 5641 bw_data.tc_bw_credits[i] = bw_share[i]; 5642 5643 ret = i40e_aq_config_vsi_tc_bw(&pf->hw, vsi->seid, &bw_data, NULL); 5644 if (ret) { 5645 dev_info(&pf->pdev->dev, 5646 "AQ command Config VSI BW allocation per TC failed = %d\n", 5647 pf->hw.aq.asq_last_status); 5648 return -EINVAL; 5649 } 5650 5651 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) 5652 vsi->info.qs_handle[i] = bw_data.qs_handles[i]; 5653 5654 return 0; 5655 } 5656 5657 /** 5658 * i40e_vsi_config_netdev_tc - Setup the netdev TC configuration 5659 * @vsi: the VSI being configured 5660 * @enabled_tc: TC map to be enabled 5661 * 5662 **/ 5663 static void i40e_vsi_config_netdev_tc(struct i40e_vsi *vsi, u8 enabled_tc) 5664 { 5665 struct net_device *netdev = vsi->netdev; 5666 struct i40e_pf *pf = vsi->back; 5667 struct i40e_hw *hw = &pf->hw; 5668 u8 netdev_tc = 0; 5669 int i; 5670 struct i40e_dcbx_config *dcbcfg = &hw->local_dcbx_config; 5671 5672 if (!netdev) 5673 return; 5674 5675 if (!enabled_tc) { 5676 netdev_reset_tc(netdev); 5677 return; 5678 } 5679 5680 /* Set up actual enabled TCs on the VSI */ 5681 if (netdev_set_num_tc(netdev, vsi->tc_config.numtc)) 5682 return; 5683 5684 /* set per TC queues for the VSI */ 5685 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 5686 /* Only set TC queues for enabled tcs 5687 * 5688 * e.g. For a VSI that has TC0 and TC3 enabled the 5689 * enabled_tc bitmap would be 0x00001001; the driver 5690 * will set the numtc for netdev as 2 that will be 5691 * referenced by the netdev layer as TC 0 and 1. 5692 */ 5693 if (vsi->tc_config.enabled_tc & BIT(i)) 5694 netdev_set_tc_queue(netdev, 5695 vsi->tc_config.tc_info[i].netdev_tc, 5696 vsi->tc_config.tc_info[i].qcount, 5697 vsi->tc_config.tc_info[i].qoffset); 5698 } 5699 5700 if (i40e_is_tc_mqprio_enabled(pf)) 5701 return; 5702 5703 /* Assign UP2TC map for the VSI */ 5704 for (i = 0; i < I40E_MAX_USER_PRIORITY; i++) { 5705 /* Get the actual TC# for the UP */ 5706 u8 ets_tc = dcbcfg->etscfg.prioritytable[i]; 5707 /* Get the mapped netdev TC# for the UP */ 5708 netdev_tc = vsi->tc_config.tc_info[ets_tc].netdev_tc; 5709 netdev_set_prio_tc_map(netdev, i, netdev_tc); 5710 } 5711 } 5712 5713 /** 5714 * i40e_vsi_update_queue_map - Update our copy of VSi info with new queue map 5715 * @vsi: the VSI being configured 5716 * @ctxt: the ctxt buffer returned from AQ VSI update param command 5717 **/ 5718 static void i40e_vsi_update_queue_map(struct i40e_vsi *vsi, 5719 struct i40e_vsi_context *ctxt) 5720 { 5721 /* copy just the sections touched not the entire info 5722 * since not all sections are valid as returned by 5723 * update vsi params 5724 */ 5725 vsi->info.mapping_flags = ctxt->info.mapping_flags; 5726 memcpy(&vsi->info.queue_mapping, 5727 &ctxt->info.queue_mapping, sizeof(vsi->info.queue_mapping)); 5728 memcpy(&vsi->info.tc_mapping, ctxt->info.tc_mapping, 5729 sizeof(vsi->info.tc_mapping)); 5730 } 5731 5732 /** 5733 * i40e_update_adq_vsi_queues - update queue mapping for ADq VSI 5734 * @vsi: the VSI being reconfigured 5735 * @vsi_offset: offset from main VF VSI 5736 */ 5737 int i40e_update_adq_vsi_queues(struct i40e_vsi *vsi, int vsi_offset) 5738 { 5739 struct i40e_vsi_context ctxt = {}; 5740 struct i40e_pf *pf; 5741 struct i40e_hw *hw; 5742 int ret; 5743 5744 if (!vsi) 5745 return -EINVAL; 5746 pf = vsi->back; 5747 hw = &pf->hw; 5748 5749 ctxt.seid = vsi->seid; 5750 ctxt.pf_num = hw->pf_id; 5751 ctxt.vf_num = vsi->vf_id + hw->func_caps.vf_base_id + vsi_offset; 5752 ctxt.uplink_seid = vsi->uplink_seid; 5753 ctxt.connection_type = I40E_AQ_VSI_CONN_TYPE_NORMAL; 5754 ctxt.flags = I40E_AQ_VSI_TYPE_VF; 5755 ctxt.info = vsi->info; 5756 5757 i40e_vsi_setup_queue_map(vsi, &ctxt, vsi->tc_config.enabled_tc, 5758 false); 5759 if (vsi->reconfig_rss) { 5760 vsi->rss_size = min_t(int, pf->alloc_rss_size, 5761 vsi->num_queue_pairs); 5762 ret = i40e_vsi_config_rss(vsi); 5763 if (ret) { 5764 dev_info(&pf->pdev->dev, "Failed to reconfig rss for num_queues\n"); 5765 return ret; 5766 } 5767 vsi->reconfig_rss = false; 5768 } 5769 5770 ret = i40e_aq_update_vsi_params(hw, &ctxt, NULL); 5771 if (ret) { 5772 dev_info(&pf->pdev->dev, "Update vsi config failed, err %pe aq_err %s\n", 5773 ERR_PTR(ret), 5774 i40e_aq_str(hw, hw->aq.asq_last_status)); 5775 return ret; 5776 } 5777 /* update the local VSI info with updated queue map */ 5778 i40e_vsi_update_queue_map(vsi, &ctxt); 5779 vsi->info.valid_sections = 0; 5780 5781 return ret; 5782 } 5783 5784 /** 5785 * i40e_vsi_config_tc - Configure VSI Tx Scheduler for given TC map 5786 * @vsi: VSI to be configured 5787 * @enabled_tc: TC bitmap 5788 * 5789 * This configures a particular VSI for TCs that are mapped to the 5790 * given TC bitmap. It uses default bandwidth share for TCs across 5791 * VSIs to configure TC for a particular VSI. 5792 * 5793 * NOTE: 5794 * It is expected that the VSI queues have been quisced before calling 5795 * this function. 5796 **/ 5797 static int i40e_vsi_config_tc(struct i40e_vsi *vsi, u8 enabled_tc) 5798 { 5799 u8 bw_share[I40E_MAX_TRAFFIC_CLASS] = {0}; 5800 struct i40e_pf *pf = vsi->back; 5801 struct i40e_hw *hw = &pf->hw; 5802 struct i40e_vsi_context ctxt; 5803 int ret = 0; 5804 int i; 5805 5806 /* Check if enabled_tc is same as existing or new TCs */ 5807 if (vsi->tc_config.enabled_tc == enabled_tc && 5808 vsi->mqprio_qopt.mode != TC_MQPRIO_MODE_CHANNEL) 5809 return ret; 5810 5811 /* Enable ETS TCs with equal BW Share for now across all VSIs */ 5812 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 5813 if (enabled_tc & BIT(i)) 5814 bw_share[i] = 1; 5815 } 5816 5817 ret = i40e_vsi_configure_bw_alloc(vsi, enabled_tc, bw_share); 5818 if (ret) { 5819 struct i40e_aqc_query_vsi_bw_config_resp bw_config = {0}; 5820 5821 dev_info(&pf->pdev->dev, 5822 "Failed configuring TC map %d for VSI %d\n", 5823 enabled_tc, vsi->seid); 5824 ret = i40e_aq_query_vsi_bw_config(hw, vsi->seid, 5825 &bw_config, NULL); 5826 if (ret) { 5827 dev_info(&pf->pdev->dev, 5828 "Failed querying vsi bw info, err %pe aq_err %s\n", 5829 ERR_PTR(ret), 5830 i40e_aq_str(hw, hw->aq.asq_last_status)); 5831 goto out; 5832 } 5833 if ((bw_config.tc_valid_bits & enabled_tc) != enabled_tc) { 5834 u8 valid_tc = bw_config.tc_valid_bits & enabled_tc; 5835 5836 if (!valid_tc) 5837 valid_tc = bw_config.tc_valid_bits; 5838 /* Always enable TC0, no matter what */ 5839 valid_tc |= 1; 5840 dev_info(&pf->pdev->dev, 5841 "Requested tc 0x%x, but FW reports 0x%x as valid. Attempting to use 0x%x.\n", 5842 enabled_tc, bw_config.tc_valid_bits, valid_tc); 5843 enabled_tc = valid_tc; 5844 } 5845 5846 ret = i40e_vsi_configure_bw_alloc(vsi, enabled_tc, bw_share); 5847 if (ret) { 5848 dev_err(&pf->pdev->dev, 5849 "Unable to configure TC map %d for VSI %d\n", 5850 enabled_tc, vsi->seid); 5851 goto out; 5852 } 5853 } 5854 5855 /* Update Queue Pairs Mapping for currently enabled UPs */ 5856 ctxt.seid = vsi->seid; 5857 ctxt.pf_num = vsi->back->hw.pf_id; 5858 ctxt.vf_num = 0; 5859 ctxt.uplink_seid = vsi->uplink_seid; 5860 ctxt.info = vsi->info; 5861 if (i40e_is_tc_mqprio_enabled(pf)) { 5862 ret = i40e_vsi_setup_queue_map_mqprio(vsi, &ctxt, enabled_tc); 5863 if (ret) 5864 goto out; 5865 } else { 5866 i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, false); 5867 } 5868 5869 /* On destroying the qdisc, reset vsi->rss_size, as number of enabled 5870 * queues changed. 5871 */ 5872 if (!vsi->mqprio_qopt.qopt.hw && vsi->reconfig_rss) { 5873 vsi->rss_size = min_t(int, vsi->back->alloc_rss_size, 5874 vsi->num_queue_pairs); 5875 ret = i40e_vsi_config_rss(vsi); 5876 if (ret) { 5877 dev_info(&vsi->back->pdev->dev, 5878 "Failed to reconfig rss for num_queues\n"); 5879 return ret; 5880 } 5881 vsi->reconfig_rss = false; 5882 } 5883 if (test_bit(I40E_FLAG_IWARP_ENA, vsi->back->flags)) { 5884 ctxt.info.valid_sections |= 5885 cpu_to_le16(I40E_AQ_VSI_PROP_QUEUE_OPT_VALID); 5886 ctxt.info.queueing_opt_flags |= I40E_AQ_VSI_QUE_OPT_TCP_ENA; 5887 } 5888 5889 /* Update the VSI after updating the VSI queue-mapping 5890 * information 5891 */ 5892 ret = i40e_aq_update_vsi_params(hw, &ctxt, NULL); 5893 if (ret) { 5894 dev_info(&pf->pdev->dev, 5895 "Update vsi tc config failed, err %pe aq_err %s\n", 5896 ERR_PTR(ret), 5897 i40e_aq_str(hw, hw->aq.asq_last_status)); 5898 goto out; 5899 } 5900 /* update the local VSI info with updated queue map */ 5901 i40e_vsi_update_queue_map(vsi, &ctxt); 5902 vsi->info.valid_sections = 0; 5903 5904 /* Update current VSI BW information */ 5905 ret = i40e_vsi_get_bw_info(vsi); 5906 if (ret) { 5907 dev_info(&pf->pdev->dev, 5908 "Failed updating vsi bw info, err %pe aq_err %s\n", 5909 ERR_PTR(ret), 5910 i40e_aq_str(hw, hw->aq.asq_last_status)); 5911 goto out; 5912 } 5913 5914 /* Update the netdev TC setup */ 5915 i40e_vsi_config_netdev_tc(vsi, enabled_tc); 5916 out: 5917 return ret; 5918 } 5919 5920 /** 5921 * i40e_vsi_reconfig_tc - Reconfigure VSI Tx Scheduler for stored TC map 5922 * @vsi: VSI to be reconfigured 5923 * 5924 * This reconfigures a particular VSI for TCs that are mapped to the 5925 * TC bitmap stored previously for the VSI. 5926 * 5927 * Context: It is expected that the VSI queues have been quisced before 5928 * calling this function. 5929 * 5930 * Return: 0 on success, negative value on failure 5931 **/ 5932 static int i40e_vsi_reconfig_tc(struct i40e_vsi *vsi) 5933 { 5934 u8 enabled_tc; 5935 5936 enabled_tc = vsi->tc_config.enabled_tc; 5937 vsi->tc_config.enabled_tc = 0; 5938 5939 return i40e_vsi_config_tc(vsi, enabled_tc); 5940 } 5941 5942 /** 5943 * i40e_get_link_speed - Returns link speed for the interface 5944 * @vsi: VSI to be configured 5945 * 5946 **/ 5947 static int i40e_get_link_speed(struct i40e_vsi *vsi) 5948 { 5949 struct i40e_pf *pf = vsi->back; 5950 5951 switch (pf->hw.phy.link_info.link_speed) { 5952 case I40E_LINK_SPEED_40GB: 5953 return 40000; 5954 case I40E_LINK_SPEED_25GB: 5955 return 25000; 5956 case I40E_LINK_SPEED_20GB: 5957 return 20000; 5958 case I40E_LINK_SPEED_10GB: 5959 return 10000; 5960 case I40E_LINK_SPEED_1GB: 5961 return 1000; 5962 default: 5963 return -EINVAL; 5964 } 5965 } 5966 5967 /** 5968 * i40e_bw_bytes_to_mbits - Convert max_tx_rate from bytes to mbits 5969 * @vsi: Pointer to vsi structure 5970 * @max_tx_rate: max TX rate in bytes to be converted into Mbits 5971 * 5972 * Helper function to convert units before send to set BW limit 5973 **/ 5974 static u64 i40e_bw_bytes_to_mbits(struct i40e_vsi *vsi, u64 max_tx_rate) 5975 { 5976 if (max_tx_rate < I40E_BW_MBPS_DIVISOR) { 5977 dev_warn(&vsi->back->pdev->dev, 5978 "Setting max tx rate to minimum usable value of 50Mbps.\n"); 5979 max_tx_rate = I40E_BW_CREDIT_DIVISOR; 5980 } else { 5981 do_div(max_tx_rate, I40E_BW_MBPS_DIVISOR); 5982 } 5983 5984 return max_tx_rate; 5985 } 5986 5987 /** 5988 * i40e_set_bw_limit - setup BW limit for Tx traffic based on max_tx_rate 5989 * @vsi: VSI to be configured 5990 * @seid: seid of the channel/VSI 5991 * @max_tx_rate: max TX rate to be configured as BW limit 5992 * 5993 * Helper function to set BW limit for a given VSI 5994 **/ 5995 int i40e_set_bw_limit(struct i40e_vsi *vsi, u16 seid, u64 max_tx_rate) 5996 { 5997 struct i40e_pf *pf = vsi->back; 5998 u64 credits = 0; 5999 int speed = 0; 6000 int ret = 0; 6001 6002 speed = i40e_get_link_speed(vsi); 6003 if (max_tx_rate > speed) { 6004 dev_err(&pf->pdev->dev, 6005 "Invalid max tx rate %llu specified for VSI seid %d.", 6006 max_tx_rate, seid); 6007 return -EINVAL; 6008 } 6009 if (max_tx_rate && max_tx_rate < I40E_BW_CREDIT_DIVISOR) { 6010 dev_warn(&pf->pdev->dev, 6011 "Setting max tx rate to minimum usable value of 50Mbps.\n"); 6012 max_tx_rate = I40E_BW_CREDIT_DIVISOR; 6013 } 6014 6015 /* Tx rate credits are in values of 50Mbps, 0 is disabled */ 6016 credits = max_tx_rate; 6017 do_div(credits, I40E_BW_CREDIT_DIVISOR); 6018 ret = i40e_aq_config_vsi_bw_limit(&pf->hw, seid, credits, 6019 I40E_MAX_BW_INACTIVE_ACCUM, NULL); 6020 if (ret) 6021 dev_err(&pf->pdev->dev, 6022 "Failed set tx rate (%llu Mbps) for vsi->seid %u, err %pe aq_err %s\n", 6023 max_tx_rate, seid, ERR_PTR(ret), 6024 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 6025 return ret; 6026 } 6027 6028 /** 6029 * i40e_remove_queue_channels - Remove queue channels for the TCs 6030 * @vsi: VSI to be configured 6031 * 6032 * Remove queue channels for the TCs 6033 **/ 6034 static void i40e_remove_queue_channels(struct i40e_vsi *vsi) 6035 { 6036 enum i40e_admin_queue_err last_aq_status; 6037 struct i40e_cloud_filter *cfilter; 6038 struct i40e_channel *ch, *ch_tmp; 6039 struct i40e_pf *pf = vsi->back; 6040 struct hlist_node *node; 6041 int ret, i; 6042 6043 /* Reset rss size that was stored when reconfiguring rss for 6044 * channel VSIs with non-power-of-2 queue count. 6045 */ 6046 vsi->current_rss_size = 0; 6047 6048 /* perform cleanup for channels if they exist */ 6049 if (list_empty(&vsi->ch_list)) 6050 return; 6051 6052 list_for_each_entry_safe(ch, ch_tmp, &vsi->ch_list, list) { 6053 struct i40e_vsi *p_vsi; 6054 6055 list_del(&ch->list); 6056 p_vsi = ch->parent_vsi; 6057 if (!p_vsi || !ch->initialized) { 6058 kfree(ch); 6059 continue; 6060 } 6061 /* Reset queue contexts */ 6062 for (i = 0; i < ch->num_queue_pairs; i++) { 6063 struct i40e_ring *tx_ring, *rx_ring; 6064 u16 pf_q; 6065 6066 pf_q = ch->base_queue + i; 6067 tx_ring = vsi->tx_rings[pf_q]; 6068 tx_ring->ch = NULL; 6069 6070 rx_ring = vsi->rx_rings[pf_q]; 6071 rx_ring->ch = NULL; 6072 } 6073 6074 /* Reset BW configured for this VSI via mqprio */ 6075 ret = i40e_set_bw_limit(vsi, ch->seid, 0); 6076 if (ret) 6077 dev_info(&vsi->back->pdev->dev, 6078 "Failed to reset tx rate for ch->seid %u\n", 6079 ch->seid); 6080 6081 /* delete cloud filters associated with this channel */ 6082 hlist_for_each_entry_safe(cfilter, node, 6083 &pf->cloud_filter_list, cloud_node) { 6084 if (cfilter->seid != ch->seid) 6085 continue; 6086 6087 hash_del(&cfilter->cloud_node); 6088 if (cfilter->dst_port) 6089 ret = i40e_add_del_cloud_filter_big_buf(vsi, 6090 cfilter, 6091 false); 6092 else 6093 ret = i40e_add_del_cloud_filter(vsi, cfilter, 6094 false); 6095 last_aq_status = pf->hw.aq.asq_last_status; 6096 if (ret) 6097 dev_info(&pf->pdev->dev, 6098 "Failed to delete cloud filter, err %pe aq_err %s\n", 6099 ERR_PTR(ret), 6100 i40e_aq_str(&pf->hw, last_aq_status)); 6101 kfree(cfilter); 6102 } 6103 6104 /* delete VSI from FW */ 6105 ret = i40e_aq_delete_element(&vsi->back->hw, ch->seid, 6106 NULL); 6107 if (ret) 6108 dev_err(&vsi->back->pdev->dev, 6109 "unable to remove channel (%d) for parent VSI(%d)\n", 6110 ch->seid, p_vsi->seid); 6111 kfree(ch); 6112 } 6113 INIT_LIST_HEAD(&vsi->ch_list); 6114 } 6115 6116 /** 6117 * i40e_get_max_queues_for_channel 6118 * @vsi: ptr to VSI to which channels are associated with 6119 * 6120 * Helper function which returns max value among the queue counts set on the 6121 * channels/TCs created. 6122 **/ 6123 static int i40e_get_max_queues_for_channel(struct i40e_vsi *vsi) 6124 { 6125 struct i40e_channel *ch, *ch_tmp; 6126 int max = 0; 6127 6128 list_for_each_entry_safe(ch, ch_tmp, &vsi->ch_list, list) { 6129 if (!ch->initialized) 6130 continue; 6131 if (ch->num_queue_pairs > max) 6132 max = ch->num_queue_pairs; 6133 } 6134 6135 return max; 6136 } 6137 6138 /** 6139 * i40e_validate_num_queues - validate num_queues w.r.t channel 6140 * @pf: ptr to PF device 6141 * @num_queues: number of queues 6142 * @vsi: the parent VSI 6143 * @reconfig_rss: indicates should the RSS be reconfigured or not 6144 * 6145 * This function validates number of queues in the context of new channel 6146 * which is being established and determines if RSS should be reconfigured 6147 * or not for parent VSI. 6148 **/ 6149 static int i40e_validate_num_queues(struct i40e_pf *pf, int num_queues, 6150 struct i40e_vsi *vsi, bool *reconfig_rss) 6151 { 6152 int max_ch_queues; 6153 6154 if (!reconfig_rss) 6155 return -EINVAL; 6156 6157 *reconfig_rss = false; 6158 if (vsi->current_rss_size) { 6159 if (num_queues > vsi->current_rss_size) { 6160 dev_dbg(&pf->pdev->dev, 6161 "Error: num_queues (%d) > vsi's current_size(%d)\n", 6162 num_queues, vsi->current_rss_size); 6163 return -EINVAL; 6164 } else if ((num_queues < vsi->current_rss_size) && 6165 (!is_power_of_2(num_queues))) { 6166 dev_dbg(&pf->pdev->dev, 6167 "Error: num_queues (%d) < vsi's current_size(%d), but not power of 2\n", 6168 num_queues, vsi->current_rss_size); 6169 return -EINVAL; 6170 } 6171 } 6172 6173 if (!is_power_of_2(num_queues)) { 6174 /* Find the max num_queues configured for channel if channel 6175 * exist. 6176 * if channel exist, then enforce 'num_queues' to be more than 6177 * max ever queues configured for channel. 6178 */ 6179 max_ch_queues = i40e_get_max_queues_for_channel(vsi); 6180 if (num_queues < max_ch_queues) { 6181 dev_dbg(&pf->pdev->dev, 6182 "Error: num_queues (%d) < max queues configured for channel(%d)\n", 6183 num_queues, max_ch_queues); 6184 return -EINVAL; 6185 } 6186 *reconfig_rss = true; 6187 } 6188 6189 return 0; 6190 } 6191 6192 /** 6193 * i40e_vsi_reconfig_rss - reconfig RSS based on specified rss_size 6194 * @vsi: the VSI being setup 6195 * @rss_size: size of RSS, accordingly LUT gets reprogrammed 6196 * 6197 * This function reconfigures RSS by reprogramming LUTs using 'rss_size' 6198 **/ 6199 static int i40e_vsi_reconfig_rss(struct i40e_vsi *vsi, u16 rss_size) 6200 { 6201 struct i40e_pf *pf = vsi->back; 6202 u8 seed[I40E_HKEY_ARRAY_SIZE]; 6203 struct i40e_hw *hw = &pf->hw; 6204 int local_rss_size; 6205 u8 *lut; 6206 int ret; 6207 6208 if (!vsi->rss_size) 6209 return -EINVAL; 6210 6211 if (rss_size > vsi->rss_size) 6212 return -EINVAL; 6213 6214 local_rss_size = min_t(int, vsi->rss_size, rss_size); 6215 lut = kzalloc(vsi->rss_table_size, GFP_KERNEL); 6216 if (!lut) 6217 return -ENOMEM; 6218 6219 /* Ignoring user configured lut if there is one */ 6220 i40e_fill_rss_lut(pf, lut, vsi->rss_table_size, local_rss_size); 6221 6222 /* Use user configured hash key if there is one, otherwise 6223 * use default. 6224 */ 6225 if (vsi->rss_hkey_user) 6226 memcpy(seed, vsi->rss_hkey_user, I40E_HKEY_ARRAY_SIZE); 6227 else 6228 netdev_rss_key_fill((void *)seed, I40E_HKEY_ARRAY_SIZE); 6229 6230 ret = i40e_config_rss(vsi, seed, lut, vsi->rss_table_size); 6231 if (ret) { 6232 dev_info(&pf->pdev->dev, 6233 "Cannot set RSS lut, err %pe aq_err %s\n", 6234 ERR_PTR(ret), 6235 i40e_aq_str(hw, hw->aq.asq_last_status)); 6236 kfree(lut); 6237 return ret; 6238 } 6239 kfree(lut); 6240 6241 /* Do the update w.r.t. storing rss_size */ 6242 if (!vsi->orig_rss_size) 6243 vsi->orig_rss_size = vsi->rss_size; 6244 vsi->current_rss_size = local_rss_size; 6245 6246 return ret; 6247 } 6248 6249 /** 6250 * i40e_channel_setup_queue_map - Setup a channel queue map 6251 * @pf: ptr to PF device 6252 * @ctxt: VSI context structure 6253 * @ch: ptr to channel structure 6254 * 6255 * Setup queue map for a specific channel 6256 **/ 6257 static void i40e_channel_setup_queue_map(struct i40e_pf *pf, 6258 struct i40e_vsi_context *ctxt, 6259 struct i40e_channel *ch) 6260 { 6261 u16 qcount, qmap, sections = 0; 6262 u8 offset = 0; 6263 int pow; 6264 6265 sections = I40E_AQ_VSI_PROP_QUEUE_MAP_VALID; 6266 sections |= I40E_AQ_VSI_PROP_SCHED_VALID; 6267 6268 qcount = min_t(int, ch->num_queue_pairs, pf->num_lan_msix); 6269 ch->num_queue_pairs = qcount; 6270 6271 /* find the next higher power-of-2 of num queue pairs */ 6272 pow = ilog2(qcount); 6273 if (!is_power_of_2(qcount)) 6274 pow++; 6275 6276 qmap = (offset << I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT) | 6277 (pow << I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT); 6278 6279 /* Setup queue TC[0].qmap for given VSI context */ 6280 ctxt->info.tc_mapping[0] = cpu_to_le16(qmap); 6281 6282 ctxt->info.up_enable_bits = 0x1; /* TC0 enabled */ 6283 ctxt->info.mapping_flags |= cpu_to_le16(I40E_AQ_VSI_QUE_MAP_CONTIG); 6284 ctxt->info.queue_mapping[0] = cpu_to_le16(ch->base_queue); 6285 ctxt->info.valid_sections |= cpu_to_le16(sections); 6286 } 6287 6288 /** 6289 * i40e_add_channel - add a channel by adding VSI 6290 * @pf: ptr to PF device 6291 * @uplink_seid: underlying HW switching element (VEB) ID 6292 * @ch: ptr to channel structure 6293 * 6294 * Add a channel (VSI) using add_vsi and queue_map 6295 **/ 6296 static int i40e_add_channel(struct i40e_pf *pf, u16 uplink_seid, 6297 struct i40e_channel *ch) 6298 { 6299 struct i40e_hw *hw = &pf->hw; 6300 struct i40e_vsi_context ctxt; 6301 u8 enabled_tc = 0x1; /* TC0 enabled */ 6302 int ret; 6303 6304 if (ch->type != I40E_VSI_VMDQ2) { 6305 dev_info(&pf->pdev->dev, 6306 "add new vsi failed, ch->type %d\n", ch->type); 6307 return -EINVAL; 6308 } 6309 6310 memset(&ctxt, 0, sizeof(ctxt)); 6311 ctxt.pf_num = hw->pf_id; 6312 ctxt.vf_num = 0; 6313 ctxt.uplink_seid = uplink_seid; 6314 ctxt.connection_type = I40E_AQ_VSI_CONN_TYPE_NORMAL; 6315 if (ch->type == I40E_VSI_VMDQ2) 6316 ctxt.flags = I40E_AQ_VSI_TYPE_VMDQ2; 6317 6318 if (test_bit(I40E_FLAG_VEB_MODE_ENA, pf->flags)) { 6319 ctxt.info.valid_sections |= 6320 cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID); 6321 ctxt.info.switch_id = 6322 cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB); 6323 } 6324 6325 /* Set queue map for a given VSI context */ 6326 i40e_channel_setup_queue_map(pf, &ctxt, ch); 6327 6328 /* Now time to create VSI */ 6329 ret = i40e_aq_add_vsi(hw, &ctxt, NULL); 6330 if (ret) { 6331 dev_info(&pf->pdev->dev, 6332 "add new vsi failed, err %pe aq_err %s\n", 6333 ERR_PTR(ret), 6334 i40e_aq_str(&pf->hw, 6335 pf->hw.aq.asq_last_status)); 6336 return -ENOENT; 6337 } 6338 6339 /* Success, update channel, set enabled_tc only if the channel 6340 * is not a macvlan 6341 */ 6342 ch->enabled_tc = !i40e_is_channel_macvlan(ch) && enabled_tc; 6343 ch->seid = ctxt.seid; 6344 ch->vsi_number = ctxt.vsi_number; 6345 ch->stat_counter_idx = le16_to_cpu(ctxt.info.stat_counter_idx); 6346 6347 /* copy just the sections touched not the entire info 6348 * since not all sections are valid as returned by 6349 * update vsi params 6350 */ 6351 ch->info.mapping_flags = ctxt.info.mapping_flags; 6352 memcpy(&ch->info.queue_mapping, 6353 &ctxt.info.queue_mapping, sizeof(ctxt.info.queue_mapping)); 6354 memcpy(&ch->info.tc_mapping, ctxt.info.tc_mapping, 6355 sizeof(ctxt.info.tc_mapping)); 6356 6357 return 0; 6358 } 6359 6360 static int i40e_channel_config_bw(struct i40e_vsi *vsi, struct i40e_channel *ch, 6361 u8 *bw_share) 6362 { 6363 struct i40e_aqc_configure_vsi_tc_bw_data bw_data; 6364 int ret; 6365 int i; 6366 6367 memset(&bw_data, 0, sizeof(bw_data)); 6368 bw_data.tc_valid_bits = ch->enabled_tc; 6369 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) 6370 bw_data.tc_bw_credits[i] = bw_share[i]; 6371 6372 ret = i40e_aq_config_vsi_tc_bw(&vsi->back->hw, ch->seid, 6373 &bw_data, NULL); 6374 if (ret) { 6375 dev_info(&vsi->back->pdev->dev, 6376 "Config VSI BW allocation per TC failed, aq_err: %d for new_vsi->seid %u\n", 6377 vsi->back->hw.aq.asq_last_status, ch->seid); 6378 return -EINVAL; 6379 } 6380 6381 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) 6382 ch->info.qs_handle[i] = bw_data.qs_handles[i]; 6383 6384 return 0; 6385 } 6386 6387 /** 6388 * i40e_channel_config_tx_ring - config TX ring associated with new channel 6389 * @pf: ptr to PF device 6390 * @vsi: the VSI being setup 6391 * @ch: ptr to channel structure 6392 * 6393 * Configure TX rings associated with channel (VSI) since queues are being 6394 * from parent VSI. 6395 **/ 6396 static int i40e_channel_config_tx_ring(struct i40e_pf *pf, 6397 struct i40e_vsi *vsi, 6398 struct i40e_channel *ch) 6399 { 6400 u8 bw_share[I40E_MAX_TRAFFIC_CLASS] = {0}; 6401 int ret; 6402 int i; 6403 6404 /* Enable ETS TCs with equal BW Share for now across all VSIs */ 6405 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 6406 if (ch->enabled_tc & BIT(i)) 6407 bw_share[i] = 1; 6408 } 6409 6410 /* configure BW for new VSI */ 6411 ret = i40e_channel_config_bw(vsi, ch, bw_share); 6412 if (ret) { 6413 dev_info(&vsi->back->pdev->dev, 6414 "Failed configuring TC map %d for channel (seid %u)\n", 6415 ch->enabled_tc, ch->seid); 6416 return ret; 6417 } 6418 6419 for (i = 0; i < ch->num_queue_pairs; i++) { 6420 struct i40e_ring *tx_ring, *rx_ring; 6421 u16 pf_q; 6422 6423 pf_q = ch->base_queue + i; 6424 6425 /* Get to TX ring ptr of main VSI, for re-setup TX queue 6426 * context 6427 */ 6428 tx_ring = vsi->tx_rings[pf_q]; 6429 tx_ring->ch = ch; 6430 6431 /* Get the RX ring ptr */ 6432 rx_ring = vsi->rx_rings[pf_q]; 6433 rx_ring->ch = ch; 6434 } 6435 6436 return 0; 6437 } 6438 6439 /** 6440 * i40e_setup_hw_channel - setup new channel 6441 * @pf: ptr to PF device 6442 * @vsi: the VSI being setup 6443 * @ch: ptr to channel structure 6444 * @uplink_seid: underlying HW switching element (VEB) ID 6445 * @type: type of channel to be created (VMDq2/VF) 6446 * 6447 * Setup new channel (VSI) based on specified type (VMDq2/VF) 6448 * and configures TX rings accordingly 6449 **/ 6450 static inline int i40e_setup_hw_channel(struct i40e_pf *pf, 6451 struct i40e_vsi *vsi, 6452 struct i40e_channel *ch, 6453 u16 uplink_seid, u8 type) 6454 { 6455 int ret; 6456 6457 ch->initialized = false; 6458 ch->base_queue = vsi->next_base_queue; 6459 ch->type = type; 6460 6461 /* Proceed with creation of channel (VMDq2) VSI */ 6462 ret = i40e_add_channel(pf, uplink_seid, ch); 6463 if (ret) { 6464 dev_info(&pf->pdev->dev, 6465 "failed to add_channel using uplink_seid %u\n", 6466 uplink_seid); 6467 return ret; 6468 } 6469 6470 /* Mark the successful creation of channel */ 6471 ch->initialized = true; 6472 6473 /* Reconfigure TX queues using QTX_CTL register */ 6474 ret = i40e_channel_config_tx_ring(pf, vsi, ch); 6475 if (ret) { 6476 dev_info(&pf->pdev->dev, 6477 "failed to configure TX rings for channel %u\n", 6478 ch->seid); 6479 return ret; 6480 } 6481 6482 /* update 'next_base_queue' */ 6483 vsi->next_base_queue = vsi->next_base_queue + ch->num_queue_pairs; 6484 dev_dbg(&pf->pdev->dev, 6485 "Added channel: vsi_seid %u, vsi_number %u, stat_counter_idx %u, num_queue_pairs %u, pf->next_base_queue %d\n", 6486 ch->seid, ch->vsi_number, ch->stat_counter_idx, 6487 ch->num_queue_pairs, 6488 vsi->next_base_queue); 6489 return ret; 6490 } 6491 6492 /** 6493 * i40e_setup_channel - setup new channel using uplink element 6494 * @pf: ptr to PF device 6495 * @vsi: pointer to the VSI to set up the channel within 6496 * @ch: ptr to channel structure 6497 * 6498 * Setup new channel (VSI) based on specified type (VMDq2/VF) 6499 * and uplink switching element (uplink_seid) 6500 **/ 6501 static bool i40e_setup_channel(struct i40e_pf *pf, struct i40e_vsi *vsi, 6502 struct i40e_channel *ch) 6503 { 6504 struct i40e_vsi *main_vsi; 6505 u8 vsi_type; 6506 u16 seid; 6507 int ret; 6508 6509 if (vsi->type == I40E_VSI_MAIN) { 6510 vsi_type = I40E_VSI_VMDQ2; 6511 } else { 6512 dev_err(&pf->pdev->dev, "unsupported parent vsi type(%d)\n", 6513 vsi->type); 6514 return false; 6515 } 6516 6517 /* underlying switching element */ 6518 main_vsi = i40e_pf_get_main_vsi(pf); 6519 seid = main_vsi->uplink_seid; 6520 6521 /* create channel (VSI), configure TX rings */ 6522 ret = i40e_setup_hw_channel(pf, vsi, ch, seid, vsi_type); 6523 if (ret) { 6524 dev_err(&pf->pdev->dev, "failed to setup hw_channel\n"); 6525 return false; 6526 } 6527 6528 return ch->initialized ? true : false; 6529 } 6530 6531 /** 6532 * i40e_validate_and_set_switch_mode - sets up switch mode correctly 6533 * @vsi: ptr to VSI which has PF backing 6534 * 6535 * Sets up switch mode correctly if it needs to be changed and perform 6536 * what are allowed modes. 6537 **/ 6538 static int i40e_validate_and_set_switch_mode(struct i40e_vsi *vsi) 6539 { 6540 u8 mode; 6541 struct i40e_pf *pf = vsi->back; 6542 struct i40e_hw *hw = &pf->hw; 6543 int ret; 6544 6545 ret = i40e_get_capabilities(pf, i40e_aqc_opc_list_dev_capabilities); 6546 if (ret) 6547 return -EINVAL; 6548 6549 if (hw->dev_caps.switch_mode) { 6550 /* if switch mode is set, support mode2 (non-tunneled for 6551 * cloud filter) for now 6552 */ 6553 u32 switch_mode = hw->dev_caps.switch_mode & 6554 I40E_SWITCH_MODE_MASK; 6555 if (switch_mode >= I40E_CLOUD_FILTER_MODE1) { 6556 if (switch_mode == I40E_CLOUD_FILTER_MODE2) 6557 return 0; 6558 dev_err(&pf->pdev->dev, 6559 "Invalid switch_mode (%d), only non-tunneled mode for cloud filter is supported\n", 6560 hw->dev_caps.switch_mode); 6561 return -EINVAL; 6562 } 6563 } 6564 6565 /* Set Bit 7 to be valid */ 6566 mode = I40E_AQ_SET_SWITCH_BIT7_VALID; 6567 6568 /* Set L4type for TCP support */ 6569 mode |= I40E_AQ_SET_SWITCH_L4_TYPE_TCP; 6570 6571 /* Set cloud filter mode */ 6572 mode |= I40E_AQ_SET_SWITCH_MODE_NON_TUNNEL; 6573 6574 /* Prep mode field for set_switch_config */ 6575 ret = i40e_aq_set_switch_config(hw, pf->last_sw_conf_flags, 6576 pf->last_sw_conf_valid_flags, 6577 mode, NULL); 6578 if (ret && hw->aq.asq_last_status != I40E_AQ_RC_ESRCH) 6579 dev_err(&pf->pdev->dev, 6580 "couldn't set switch config bits, err %pe aq_err %s\n", 6581 ERR_PTR(ret), 6582 i40e_aq_str(hw, 6583 hw->aq.asq_last_status)); 6584 6585 return ret; 6586 } 6587 6588 /** 6589 * i40e_create_queue_channel - function to create channel 6590 * @vsi: VSI to be configured 6591 * @ch: ptr to channel (it contains channel specific params) 6592 * 6593 * This function creates channel (VSI) using num_queues specified by user, 6594 * reconfigs RSS if needed. 6595 **/ 6596 int i40e_create_queue_channel(struct i40e_vsi *vsi, 6597 struct i40e_channel *ch) 6598 { 6599 struct i40e_pf *pf = vsi->back; 6600 bool reconfig_rss; 6601 int err; 6602 6603 if (!ch) 6604 return -EINVAL; 6605 6606 if (!ch->num_queue_pairs) { 6607 dev_err(&pf->pdev->dev, "Invalid num_queues requested: %d\n", 6608 ch->num_queue_pairs); 6609 return -EINVAL; 6610 } 6611 6612 /* validate user requested num_queues for channel */ 6613 err = i40e_validate_num_queues(pf, ch->num_queue_pairs, vsi, 6614 &reconfig_rss); 6615 if (err) { 6616 dev_info(&pf->pdev->dev, "Failed to validate num_queues (%d)\n", 6617 ch->num_queue_pairs); 6618 return -EINVAL; 6619 } 6620 6621 /* By default we are in VEPA mode, if this is the first VF/VMDq 6622 * VSI to be added switch to VEB mode. 6623 */ 6624 6625 if (!test_bit(I40E_FLAG_VEB_MODE_ENA, pf->flags)) { 6626 set_bit(I40E_FLAG_VEB_MODE_ENA, pf->flags); 6627 6628 if (vsi->type == I40E_VSI_MAIN) { 6629 if (i40e_is_tc_mqprio_enabled(pf)) 6630 i40e_do_reset(pf, I40E_PF_RESET_FLAG, true); 6631 else 6632 i40e_do_reset_safe(pf, I40E_PF_RESET_FLAG); 6633 } 6634 /* now onwards for main VSI, number of queues will be value 6635 * of TC0's queue count 6636 */ 6637 } 6638 6639 /* By this time, vsi->cnt_q_avail shall be set to non-zero and 6640 * it should be more than num_queues 6641 */ 6642 if (!vsi->cnt_q_avail || vsi->cnt_q_avail < ch->num_queue_pairs) { 6643 dev_dbg(&pf->pdev->dev, 6644 "Error: cnt_q_avail (%u) less than num_queues %d\n", 6645 vsi->cnt_q_avail, ch->num_queue_pairs); 6646 return -EINVAL; 6647 } 6648 6649 /* reconfig_rss only if vsi type is MAIN_VSI */ 6650 if (reconfig_rss && (vsi->type == I40E_VSI_MAIN)) { 6651 err = i40e_vsi_reconfig_rss(vsi, ch->num_queue_pairs); 6652 if (err) { 6653 dev_info(&pf->pdev->dev, 6654 "Error: unable to reconfig rss for num_queues (%u)\n", 6655 ch->num_queue_pairs); 6656 return -EINVAL; 6657 } 6658 } 6659 6660 if (!i40e_setup_channel(pf, vsi, ch)) { 6661 dev_info(&pf->pdev->dev, "Failed to setup channel\n"); 6662 return -EINVAL; 6663 } 6664 6665 dev_info(&pf->pdev->dev, 6666 "Setup channel (id:%u) utilizing num_queues %d\n", 6667 ch->seid, ch->num_queue_pairs); 6668 6669 /* configure VSI for BW limit */ 6670 if (ch->max_tx_rate) { 6671 u64 credits = ch->max_tx_rate; 6672 6673 if (i40e_set_bw_limit(vsi, ch->seid, ch->max_tx_rate)) 6674 return -EINVAL; 6675 6676 do_div(credits, I40E_BW_CREDIT_DIVISOR); 6677 dev_dbg(&pf->pdev->dev, 6678 "Set tx rate of %llu Mbps (count of 50Mbps %llu) for vsi->seid %u\n", 6679 ch->max_tx_rate, 6680 credits, 6681 ch->seid); 6682 } 6683 6684 /* in case of VF, this will be main SRIOV VSI */ 6685 ch->parent_vsi = vsi; 6686 6687 /* and update main_vsi's count for queue_available to use */ 6688 vsi->cnt_q_avail -= ch->num_queue_pairs; 6689 6690 return 0; 6691 } 6692 6693 /** 6694 * i40e_configure_queue_channels - Add queue channel for the given TCs 6695 * @vsi: VSI to be configured 6696 * 6697 * Configures queue channel mapping to the given TCs 6698 **/ 6699 static int i40e_configure_queue_channels(struct i40e_vsi *vsi) 6700 { 6701 struct i40e_channel *ch; 6702 u64 max_rate = 0; 6703 int ret = 0, i; 6704 6705 /* Create app vsi with the TCs. Main VSI with TC0 is already set up */ 6706 vsi->tc_seid_map[0] = vsi->seid; 6707 for (i = 1; i < I40E_MAX_TRAFFIC_CLASS; i++) { 6708 if (vsi->tc_config.enabled_tc & BIT(i)) { 6709 ch = kzalloc(sizeof(*ch), GFP_KERNEL); 6710 if (!ch) { 6711 ret = -ENOMEM; 6712 goto err_free; 6713 } 6714 6715 INIT_LIST_HEAD(&ch->list); 6716 ch->num_queue_pairs = 6717 vsi->tc_config.tc_info[i].qcount; 6718 ch->base_queue = 6719 vsi->tc_config.tc_info[i].qoffset; 6720 6721 /* Bandwidth limit through tc interface is in bytes/s, 6722 * change to Mbit/s 6723 */ 6724 max_rate = vsi->mqprio_qopt.max_rate[i]; 6725 do_div(max_rate, I40E_BW_MBPS_DIVISOR); 6726 ch->max_tx_rate = max_rate; 6727 6728 list_add_tail(&ch->list, &vsi->ch_list); 6729 6730 ret = i40e_create_queue_channel(vsi, ch); 6731 if (ret) { 6732 dev_err(&vsi->back->pdev->dev, 6733 "Failed creating queue channel with TC%d: queues %d\n", 6734 i, ch->num_queue_pairs); 6735 goto err_free; 6736 } 6737 vsi->tc_seid_map[i] = ch->seid; 6738 } 6739 } 6740 6741 /* reset to reconfigure TX queue contexts */ 6742 i40e_do_reset(vsi->back, I40E_PF_RESET_FLAG, true); 6743 return ret; 6744 6745 err_free: 6746 i40e_remove_queue_channels(vsi); 6747 return ret; 6748 } 6749 6750 /** 6751 * i40e_veb_config_tc - Configure TCs for given VEB 6752 * @veb: given VEB 6753 * @enabled_tc: TC bitmap 6754 * 6755 * Configures given TC bitmap for VEB (switching) element 6756 **/ 6757 int i40e_veb_config_tc(struct i40e_veb *veb, u8 enabled_tc) 6758 { 6759 struct i40e_aqc_configure_switching_comp_bw_config_data bw_data = {0}; 6760 struct i40e_pf *pf = veb->pf; 6761 int ret = 0; 6762 int i; 6763 6764 /* No TCs or already enabled TCs just return */ 6765 if (!enabled_tc || veb->enabled_tc == enabled_tc) 6766 return ret; 6767 6768 bw_data.tc_valid_bits = enabled_tc; 6769 /* bw_data.absolute_credits is not set (relative) */ 6770 6771 /* Enable ETS TCs with equal BW Share for now */ 6772 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 6773 if (enabled_tc & BIT(i)) 6774 bw_data.tc_bw_share_credits[i] = 1; 6775 } 6776 6777 ret = i40e_aq_config_switch_comp_bw_config(&pf->hw, veb->seid, 6778 &bw_data, NULL); 6779 if (ret) { 6780 dev_info(&pf->pdev->dev, 6781 "VEB bw config failed, err %pe aq_err %s\n", 6782 ERR_PTR(ret), 6783 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 6784 goto out; 6785 } 6786 6787 /* Update the BW information */ 6788 ret = i40e_veb_get_bw_info(veb); 6789 if (ret) { 6790 dev_info(&pf->pdev->dev, 6791 "Failed getting veb bw config, err %pe aq_err %s\n", 6792 ERR_PTR(ret), 6793 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 6794 } 6795 6796 out: 6797 return ret; 6798 } 6799 6800 #ifdef CONFIG_I40E_DCB 6801 /** 6802 * i40e_dcb_reconfigure - Reconfigure all VEBs and VSIs 6803 * @pf: PF struct 6804 * 6805 * Reconfigure VEB/VSIs on a given PF; it is assumed that 6806 * the caller would've quiesce all the VSIs before calling 6807 * this function 6808 **/ 6809 static void i40e_dcb_reconfigure(struct i40e_pf *pf) 6810 { 6811 struct i40e_vsi *vsi; 6812 struct i40e_veb *veb; 6813 u8 tc_map = 0; 6814 int ret; 6815 int v; 6816 6817 /* Enable the TCs available on PF to all VEBs */ 6818 tc_map = i40e_pf_get_tc_map(pf); 6819 if (tc_map == I40E_DEFAULT_TRAFFIC_CLASS) 6820 return; 6821 6822 i40e_pf_for_each_veb(pf, v, veb) { 6823 ret = i40e_veb_config_tc(veb, tc_map); 6824 if (ret) { 6825 dev_info(&pf->pdev->dev, 6826 "Failed configuring TC for VEB seid=%d\n", 6827 veb->seid); 6828 /* Will try to configure as many components */ 6829 } 6830 } 6831 6832 /* Update each VSI */ 6833 i40e_pf_for_each_vsi(pf, v, vsi) { 6834 /* - Enable all TCs for the LAN VSI 6835 * - For all others keep them at TC0 for now 6836 */ 6837 if (vsi->type == I40E_VSI_MAIN) 6838 tc_map = i40e_pf_get_tc_map(pf); 6839 else 6840 tc_map = I40E_DEFAULT_TRAFFIC_CLASS; 6841 6842 ret = i40e_vsi_config_tc(vsi, tc_map); 6843 if (ret) { 6844 dev_info(&pf->pdev->dev, 6845 "Failed configuring TC for VSI seid=%d\n", 6846 vsi->seid); 6847 /* Will try to configure as many components */ 6848 } else { 6849 /* Re-configure VSI vectors based on updated TC map */ 6850 i40e_vsi_map_rings_to_vectors(vsi); 6851 if (vsi->netdev) 6852 i40e_dcbnl_set_all(vsi); 6853 } 6854 } 6855 } 6856 6857 /** 6858 * i40e_resume_port_tx - Resume port Tx 6859 * @pf: PF struct 6860 * 6861 * Resume a port's Tx and issue a PF reset in case of failure to 6862 * resume. 6863 **/ 6864 static int i40e_resume_port_tx(struct i40e_pf *pf) 6865 { 6866 struct i40e_hw *hw = &pf->hw; 6867 int ret; 6868 6869 ret = i40e_aq_resume_port_tx(hw, NULL); 6870 if (ret) { 6871 dev_info(&pf->pdev->dev, 6872 "Resume Port Tx failed, err %pe aq_err %s\n", 6873 ERR_PTR(ret), 6874 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 6875 /* Schedule PF reset to recover */ 6876 set_bit(__I40E_PF_RESET_REQUESTED, pf->state); 6877 i40e_service_event_schedule(pf); 6878 } 6879 6880 return ret; 6881 } 6882 6883 /** 6884 * i40e_suspend_port_tx - Suspend port Tx 6885 * @pf: PF struct 6886 * 6887 * Suspend a port's Tx and issue a PF reset in case of failure. 6888 **/ 6889 static int i40e_suspend_port_tx(struct i40e_pf *pf) 6890 { 6891 struct i40e_hw *hw = &pf->hw; 6892 int ret; 6893 6894 ret = i40e_aq_suspend_port_tx(hw, pf->mac_seid, NULL); 6895 if (ret) { 6896 dev_info(&pf->pdev->dev, 6897 "Suspend Port Tx failed, err %pe aq_err %s\n", 6898 ERR_PTR(ret), 6899 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 6900 /* Schedule PF reset to recover */ 6901 set_bit(__I40E_PF_RESET_REQUESTED, pf->state); 6902 i40e_service_event_schedule(pf); 6903 } 6904 6905 return ret; 6906 } 6907 6908 /** 6909 * i40e_hw_set_dcb_config - Program new DCBX settings into HW 6910 * @pf: PF being configured 6911 * @new_cfg: New DCBX configuration 6912 * 6913 * Program DCB settings into HW and reconfigure VEB/VSIs on 6914 * given PF. Uses "Set LLDP MIB" AQC to program the hardware. 6915 **/ 6916 static int i40e_hw_set_dcb_config(struct i40e_pf *pf, 6917 struct i40e_dcbx_config *new_cfg) 6918 { 6919 struct i40e_dcbx_config *old_cfg = &pf->hw.local_dcbx_config; 6920 int ret; 6921 6922 /* Check if need reconfiguration */ 6923 if (!memcmp(&new_cfg, &old_cfg, sizeof(new_cfg))) { 6924 dev_dbg(&pf->pdev->dev, "No Change in DCB Config required.\n"); 6925 return 0; 6926 } 6927 6928 /* Config change disable all VSIs */ 6929 i40e_pf_quiesce_all_vsi(pf); 6930 6931 /* Copy the new config to the current config */ 6932 *old_cfg = *new_cfg; 6933 old_cfg->etsrec = old_cfg->etscfg; 6934 ret = i40e_set_dcb_config(&pf->hw); 6935 if (ret) { 6936 dev_info(&pf->pdev->dev, 6937 "Set DCB Config failed, err %pe aq_err %s\n", 6938 ERR_PTR(ret), 6939 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 6940 goto out; 6941 } 6942 6943 /* Changes in configuration update VEB/VSI */ 6944 i40e_dcb_reconfigure(pf); 6945 out: 6946 /* In case of reset do not try to resume anything */ 6947 if (!test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state)) { 6948 /* Re-start the VSIs if disabled */ 6949 ret = i40e_resume_port_tx(pf); 6950 /* In case of error no point in resuming VSIs */ 6951 if (ret) 6952 goto err; 6953 i40e_pf_unquiesce_all_vsi(pf); 6954 } 6955 err: 6956 return ret; 6957 } 6958 6959 /** 6960 * i40e_hw_dcb_config - Program new DCBX settings into HW 6961 * @pf: PF being configured 6962 * @new_cfg: New DCBX configuration 6963 * 6964 * Program DCB settings into HW and reconfigure VEB/VSIs on 6965 * given PF 6966 **/ 6967 int i40e_hw_dcb_config(struct i40e_pf *pf, struct i40e_dcbx_config *new_cfg) 6968 { 6969 struct i40e_aqc_configure_switching_comp_ets_data ets_data; 6970 u8 prio_type[I40E_MAX_TRAFFIC_CLASS] = {0}; 6971 u32 mfs_tc[I40E_MAX_TRAFFIC_CLASS]; 6972 struct i40e_dcbx_config *old_cfg; 6973 u8 mode[I40E_MAX_TRAFFIC_CLASS]; 6974 struct i40e_rx_pb_config pb_cfg; 6975 struct i40e_hw *hw = &pf->hw; 6976 u8 num_ports = hw->num_ports; 6977 bool need_reconfig; 6978 int ret = -EINVAL; 6979 u8 lltc_map = 0; 6980 u8 tc_map = 0; 6981 u8 new_numtc; 6982 u8 i; 6983 6984 dev_dbg(&pf->pdev->dev, "Configuring DCB registers directly\n"); 6985 /* Un-pack information to Program ETS HW via shared API 6986 * numtc, tcmap 6987 * LLTC map 6988 * ETS/NON-ETS arbiter mode 6989 * max exponent (credit refills) 6990 * Total number of ports 6991 * PFC priority bit-map 6992 * Priority Table 6993 * BW % per TC 6994 * Arbiter mode between UPs sharing same TC 6995 * TSA table (ETS or non-ETS) 6996 * EEE enabled or not 6997 * MFS TC table 6998 */ 6999 7000 new_numtc = i40e_dcb_get_num_tc(new_cfg); 7001 7002 memset(&ets_data, 0, sizeof(ets_data)); 7003 for (i = 0; i < new_numtc; i++) { 7004 tc_map |= BIT(i); 7005 switch (new_cfg->etscfg.tsatable[i]) { 7006 case I40E_IEEE_TSA_ETS: 7007 prio_type[i] = I40E_DCB_PRIO_TYPE_ETS; 7008 ets_data.tc_bw_share_credits[i] = 7009 new_cfg->etscfg.tcbwtable[i]; 7010 break; 7011 case I40E_IEEE_TSA_STRICT: 7012 prio_type[i] = I40E_DCB_PRIO_TYPE_STRICT; 7013 lltc_map |= BIT(i); 7014 ets_data.tc_bw_share_credits[i] = 7015 I40E_DCB_STRICT_PRIO_CREDITS; 7016 break; 7017 default: 7018 /* Invalid TSA type */ 7019 need_reconfig = false; 7020 goto out; 7021 } 7022 } 7023 7024 old_cfg = &hw->local_dcbx_config; 7025 /* Check if need reconfiguration */ 7026 need_reconfig = i40e_dcb_need_reconfig(pf, old_cfg, new_cfg); 7027 7028 /* If needed, enable/disable frame tagging, disable all VSIs 7029 * and suspend port tx 7030 */ 7031 if (need_reconfig) { 7032 /* Enable DCB tagging only when more than one TC */ 7033 if (new_numtc > 1) 7034 set_bit(I40E_FLAG_DCB_ENA, pf->flags); 7035 else 7036 clear_bit(I40E_FLAG_DCB_ENA, pf->flags); 7037 7038 set_bit(__I40E_PORT_SUSPENDED, pf->state); 7039 /* Reconfiguration needed quiesce all VSIs */ 7040 i40e_pf_quiesce_all_vsi(pf); 7041 ret = i40e_suspend_port_tx(pf); 7042 if (ret) 7043 goto err; 7044 } 7045 7046 /* Configure Port ETS Tx Scheduler */ 7047 ets_data.tc_valid_bits = tc_map; 7048 ets_data.tc_strict_priority_flags = lltc_map; 7049 ret = i40e_aq_config_switch_comp_ets 7050 (hw, pf->mac_seid, &ets_data, 7051 i40e_aqc_opc_modify_switching_comp_ets, NULL); 7052 if (ret) { 7053 dev_info(&pf->pdev->dev, 7054 "Modify Port ETS failed, err %pe aq_err %s\n", 7055 ERR_PTR(ret), 7056 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 7057 goto out; 7058 } 7059 7060 /* Configure Rx ETS HW */ 7061 memset(&mode, I40E_DCB_ARB_MODE_ROUND_ROBIN, sizeof(mode)); 7062 i40e_dcb_hw_set_num_tc(hw, new_numtc); 7063 i40e_dcb_hw_rx_fifo_config(hw, I40E_DCB_ARB_MODE_ROUND_ROBIN, 7064 I40E_DCB_ARB_MODE_STRICT_PRIORITY, 7065 I40E_DCB_DEFAULT_MAX_EXPONENT, 7066 lltc_map); 7067 i40e_dcb_hw_rx_cmd_monitor_config(hw, new_numtc, num_ports); 7068 i40e_dcb_hw_rx_ets_bw_config(hw, new_cfg->etscfg.tcbwtable, mode, 7069 prio_type); 7070 i40e_dcb_hw_pfc_config(hw, new_cfg->pfc.pfcenable, 7071 new_cfg->etscfg.prioritytable); 7072 i40e_dcb_hw_rx_up2tc_config(hw, new_cfg->etscfg.prioritytable); 7073 7074 /* Configure Rx Packet Buffers in HW */ 7075 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 7076 struct i40e_vsi *main_vsi = i40e_pf_get_main_vsi(pf); 7077 7078 mfs_tc[i] = main_vsi->netdev->mtu; 7079 mfs_tc[i] += I40E_PACKET_HDR_PAD; 7080 } 7081 7082 i40e_dcb_hw_calculate_pool_sizes(hw, num_ports, 7083 false, new_cfg->pfc.pfcenable, 7084 mfs_tc, &pb_cfg); 7085 i40e_dcb_hw_rx_pb_config(hw, &pf->pb_cfg, &pb_cfg); 7086 7087 /* Update the local Rx Packet buffer config */ 7088 pf->pb_cfg = pb_cfg; 7089 7090 /* Inform the FW about changes to DCB configuration */ 7091 ret = i40e_aq_dcb_updated(&pf->hw, NULL); 7092 if (ret) { 7093 dev_info(&pf->pdev->dev, 7094 "DCB Updated failed, err %pe aq_err %s\n", 7095 ERR_PTR(ret), 7096 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 7097 goto out; 7098 } 7099 7100 /* Update the port DCBx configuration */ 7101 *old_cfg = *new_cfg; 7102 7103 /* Changes in configuration update VEB/VSI */ 7104 i40e_dcb_reconfigure(pf); 7105 out: 7106 /* Re-start the VSIs if disabled */ 7107 if (need_reconfig) { 7108 ret = i40e_resume_port_tx(pf); 7109 7110 clear_bit(__I40E_PORT_SUSPENDED, pf->state); 7111 /* In case of error no point in resuming VSIs */ 7112 if (ret) 7113 goto err; 7114 7115 /* Wait for the PF's queues to be disabled */ 7116 ret = i40e_pf_wait_queues_disabled(pf); 7117 if (ret) { 7118 /* Schedule PF reset to recover */ 7119 set_bit(__I40E_PF_RESET_REQUESTED, pf->state); 7120 i40e_service_event_schedule(pf); 7121 goto err; 7122 } else { 7123 i40e_pf_unquiesce_all_vsi(pf); 7124 set_bit(__I40E_CLIENT_SERVICE_REQUESTED, pf->state); 7125 set_bit(__I40E_CLIENT_L2_CHANGE, pf->state); 7126 } 7127 /* registers are set, lets apply */ 7128 if (test_bit(I40E_HW_CAP_USE_SET_LLDP_MIB, pf->hw.caps)) 7129 ret = i40e_hw_set_dcb_config(pf, new_cfg); 7130 } 7131 7132 err: 7133 return ret; 7134 } 7135 7136 /** 7137 * i40e_dcb_sw_default_config - Set default DCB configuration when DCB in SW 7138 * @pf: PF being queried 7139 * 7140 * Set default DCB configuration in case DCB is to be done in SW. 7141 **/ 7142 int i40e_dcb_sw_default_config(struct i40e_pf *pf) 7143 { 7144 struct i40e_dcbx_config *dcb_cfg = &pf->hw.local_dcbx_config; 7145 struct i40e_aqc_configure_switching_comp_ets_data ets_data; 7146 struct i40e_hw *hw = &pf->hw; 7147 int err; 7148 7149 if (test_bit(I40E_HW_CAP_USE_SET_LLDP_MIB, pf->hw.caps)) { 7150 /* Update the local cached instance with TC0 ETS */ 7151 memset(&pf->tmp_cfg, 0, sizeof(struct i40e_dcbx_config)); 7152 pf->tmp_cfg.etscfg.willing = I40E_IEEE_DEFAULT_ETS_WILLING; 7153 pf->tmp_cfg.etscfg.maxtcs = 0; 7154 pf->tmp_cfg.etscfg.tcbwtable[0] = I40E_IEEE_DEFAULT_ETS_TCBW; 7155 pf->tmp_cfg.etscfg.tsatable[0] = I40E_IEEE_TSA_ETS; 7156 pf->tmp_cfg.pfc.willing = I40E_IEEE_DEFAULT_PFC_WILLING; 7157 pf->tmp_cfg.pfc.pfccap = I40E_MAX_TRAFFIC_CLASS; 7158 /* FW needs one App to configure HW */ 7159 pf->tmp_cfg.numapps = I40E_IEEE_DEFAULT_NUM_APPS; 7160 pf->tmp_cfg.app[0].selector = I40E_APP_SEL_ETHTYPE; 7161 pf->tmp_cfg.app[0].priority = I40E_IEEE_DEFAULT_APP_PRIO; 7162 pf->tmp_cfg.app[0].protocolid = I40E_APP_PROTOID_FCOE; 7163 7164 return i40e_hw_set_dcb_config(pf, &pf->tmp_cfg); 7165 } 7166 7167 memset(&ets_data, 0, sizeof(ets_data)); 7168 ets_data.tc_valid_bits = I40E_DEFAULT_TRAFFIC_CLASS; /* TC0 only */ 7169 ets_data.tc_strict_priority_flags = 0; /* ETS */ 7170 ets_data.tc_bw_share_credits[0] = I40E_IEEE_DEFAULT_ETS_TCBW; /* 100% to TC0 */ 7171 7172 /* Enable ETS on the Physical port */ 7173 err = i40e_aq_config_switch_comp_ets 7174 (hw, pf->mac_seid, &ets_data, 7175 i40e_aqc_opc_enable_switching_comp_ets, NULL); 7176 if (err) { 7177 dev_info(&pf->pdev->dev, 7178 "Enable Port ETS failed, err %pe aq_err %s\n", 7179 ERR_PTR(err), 7180 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 7181 err = -ENOENT; 7182 goto out; 7183 } 7184 7185 /* Update the local cached instance with TC0 ETS */ 7186 dcb_cfg->etscfg.willing = I40E_IEEE_DEFAULT_ETS_WILLING; 7187 dcb_cfg->etscfg.cbs = 0; 7188 dcb_cfg->etscfg.maxtcs = I40E_MAX_TRAFFIC_CLASS; 7189 dcb_cfg->etscfg.tcbwtable[0] = I40E_IEEE_DEFAULT_ETS_TCBW; 7190 7191 out: 7192 return err; 7193 } 7194 7195 /** 7196 * i40e_init_pf_dcb - Initialize DCB configuration 7197 * @pf: PF being configured 7198 * 7199 * Query the current DCB configuration and cache it 7200 * in the hardware structure 7201 **/ 7202 static int i40e_init_pf_dcb(struct i40e_pf *pf) 7203 { 7204 struct i40e_hw *hw = &pf->hw; 7205 int err; 7206 7207 /* Do not enable DCB for SW1 and SW2 images even if the FW is capable 7208 * Also do not enable DCBx if FW LLDP agent is disabled 7209 */ 7210 if (test_bit(I40E_HW_CAP_NO_DCB_SUPPORT, pf->hw.caps)) { 7211 dev_info(&pf->pdev->dev, "DCB is not supported.\n"); 7212 err = -EOPNOTSUPP; 7213 goto out; 7214 } 7215 if (test_bit(I40E_FLAG_FW_LLDP_DIS, pf->flags)) { 7216 dev_info(&pf->pdev->dev, "FW LLDP is disabled, attempting SW DCB\n"); 7217 err = i40e_dcb_sw_default_config(pf); 7218 if (err) { 7219 dev_info(&pf->pdev->dev, "Could not initialize SW DCB\n"); 7220 goto out; 7221 } 7222 dev_info(&pf->pdev->dev, "SW DCB initialization succeeded.\n"); 7223 pf->dcbx_cap = DCB_CAP_DCBX_HOST | 7224 DCB_CAP_DCBX_VER_IEEE; 7225 /* at init capable but disabled */ 7226 set_bit(I40E_FLAG_DCB_CAPABLE, pf->flags); 7227 clear_bit(I40E_FLAG_DCB_ENA, pf->flags); 7228 goto out; 7229 } 7230 err = i40e_init_dcb(hw, true); 7231 if (!err) { 7232 /* Device/Function is not DCBX capable */ 7233 if ((!hw->func_caps.dcb) || 7234 (hw->dcbx_status == I40E_DCBX_STATUS_DISABLED)) { 7235 dev_info(&pf->pdev->dev, 7236 "DCBX offload is not supported or is disabled for this PF.\n"); 7237 } else { 7238 /* When status is not DISABLED then DCBX in FW */ 7239 pf->dcbx_cap = DCB_CAP_DCBX_LLD_MANAGED | 7240 DCB_CAP_DCBX_VER_IEEE; 7241 7242 set_bit(I40E_FLAG_DCB_CAPABLE, pf->flags); 7243 /* Enable DCB tagging only when more than one TC 7244 * or explicitly disable if only one TC 7245 */ 7246 if (i40e_dcb_get_num_tc(&hw->local_dcbx_config) > 1) 7247 set_bit(I40E_FLAG_DCB_ENA, pf->flags); 7248 else 7249 clear_bit(I40E_FLAG_DCB_ENA, pf->flags); 7250 dev_dbg(&pf->pdev->dev, 7251 "DCBX offload is supported for this PF.\n"); 7252 } 7253 } else if (pf->hw.aq.asq_last_status == I40E_AQ_RC_EPERM) { 7254 dev_info(&pf->pdev->dev, "FW LLDP disabled for this PF.\n"); 7255 set_bit(I40E_FLAG_FW_LLDP_DIS, pf->flags); 7256 } else { 7257 dev_info(&pf->pdev->dev, 7258 "Query for DCB configuration failed, err %pe aq_err %s\n", 7259 ERR_PTR(err), 7260 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 7261 } 7262 7263 out: 7264 return err; 7265 } 7266 #endif /* CONFIG_I40E_DCB */ 7267 7268 /** 7269 * i40e_print_link_message - print link up or down 7270 * @vsi: the VSI for which link needs a message 7271 * @isup: true of link is up, false otherwise 7272 */ 7273 void i40e_print_link_message(struct i40e_vsi *vsi, bool isup) 7274 { 7275 enum i40e_aq_link_speed new_speed; 7276 struct i40e_pf *pf = vsi->back; 7277 char *speed = "Unknown"; 7278 char *fc = "Unknown"; 7279 char *fec = ""; 7280 char *req_fec = ""; 7281 char *an = ""; 7282 7283 if (isup) 7284 new_speed = pf->hw.phy.link_info.link_speed; 7285 else 7286 new_speed = I40E_LINK_SPEED_UNKNOWN; 7287 7288 if ((vsi->current_isup == isup) && (vsi->current_speed == new_speed)) 7289 return; 7290 vsi->current_isup = isup; 7291 vsi->current_speed = new_speed; 7292 if (!isup) { 7293 netdev_info(vsi->netdev, "NIC Link is Down\n"); 7294 return; 7295 } 7296 7297 /* Warn user if link speed on NPAR enabled partition is not at 7298 * least 10GB 7299 */ 7300 if (pf->hw.func_caps.npar_enable && 7301 (pf->hw.phy.link_info.link_speed == I40E_LINK_SPEED_1GB || 7302 pf->hw.phy.link_info.link_speed == I40E_LINK_SPEED_100MB)) 7303 netdev_warn(vsi->netdev, 7304 "The partition detected link speed that is less than 10Gbps\n"); 7305 7306 switch (pf->hw.phy.link_info.link_speed) { 7307 case I40E_LINK_SPEED_40GB: 7308 speed = "40 G"; 7309 break; 7310 case I40E_LINK_SPEED_20GB: 7311 speed = "20 G"; 7312 break; 7313 case I40E_LINK_SPEED_25GB: 7314 speed = "25 G"; 7315 break; 7316 case I40E_LINK_SPEED_10GB: 7317 speed = "10 G"; 7318 break; 7319 case I40E_LINK_SPEED_5GB: 7320 speed = "5 G"; 7321 break; 7322 case I40E_LINK_SPEED_2_5GB: 7323 speed = "2.5 G"; 7324 break; 7325 case I40E_LINK_SPEED_1GB: 7326 speed = "1000 M"; 7327 break; 7328 case I40E_LINK_SPEED_100MB: 7329 speed = "100 M"; 7330 break; 7331 default: 7332 break; 7333 } 7334 7335 switch (pf->hw.fc.current_mode) { 7336 case I40E_FC_FULL: 7337 fc = "RX/TX"; 7338 break; 7339 case I40E_FC_TX_PAUSE: 7340 fc = "TX"; 7341 break; 7342 case I40E_FC_RX_PAUSE: 7343 fc = "RX"; 7344 break; 7345 default: 7346 fc = "None"; 7347 break; 7348 } 7349 7350 if (pf->hw.phy.link_info.link_speed == I40E_LINK_SPEED_25GB) { 7351 req_fec = "None"; 7352 fec = "None"; 7353 an = "False"; 7354 7355 if (pf->hw.phy.link_info.an_info & I40E_AQ_AN_COMPLETED) 7356 an = "True"; 7357 7358 if (pf->hw.phy.link_info.fec_info & 7359 I40E_AQ_CONFIG_FEC_KR_ENA) 7360 fec = "CL74 FC-FEC/BASE-R"; 7361 else if (pf->hw.phy.link_info.fec_info & 7362 I40E_AQ_CONFIG_FEC_RS_ENA) 7363 fec = "CL108 RS-FEC"; 7364 7365 /* 'CL108 RS-FEC' should be displayed when RS is requested, or 7366 * both RS and FC are requested 7367 */ 7368 if (vsi->back->hw.phy.link_info.req_fec_info & 7369 (I40E_AQ_REQUEST_FEC_KR | I40E_AQ_REQUEST_FEC_RS)) { 7370 if (vsi->back->hw.phy.link_info.req_fec_info & 7371 I40E_AQ_REQUEST_FEC_RS) 7372 req_fec = "CL108 RS-FEC"; 7373 else 7374 req_fec = "CL74 FC-FEC/BASE-R"; 7375 } 7376 netdev_info(vsi->netdev, 7377 "NIC Link is Up, %sbps Full Duplex, Requested FEC: %s, Negotiated FEC: %s, Autoneg: %s, Flow Control: %s\n", 7378 speed, req_fec, fec, an, fc); 7379 } else if (pf->hw.device_id == I40E_DEV_ID_KX_X722) { 7380 req_fec = "None"; 7381 fec = "None"; 7382 an = "False"; 7383 7384 if (pf->hw.phy.link_info.an_info & I40E_AQ_AN_COMPLETED) 7385 an = "True"; 7386 7387 if (pf->hw.phy.link_info.fec_info & 7388 I40E_AQ_CONFIG_FEC_KR_ENA) 7389 fec = "CL74 FC-FEC/BASE-R"; 7390 7391 if (pf->hw.phy.link_info.req_fec_info & 7392 I40E_AQ_REQUEST_FEC_KR) 7393 req_fec = "CL74 FC-FEC/BASE-R"; 7394 7395 netdev_info(vsi->netdev, 7396 "NIC Link is Up, %sbps Full Duplex, Requested FEC: %s, Negotiated FEC: %s, Autoneg: %s, Flow Control: %s\n", 7397 speed, req_fec, fec, an, fc); 7398 } else { 7399 netdev_info(vsi->netdev, 7400 "NIC Link is Up, %sbps Full Duplex, Flow Control: %s\n", 7401 speed, fc); 7402 } 7403 7404 } 7405 7406 /** 7407 * i40e_up_complete - Finish the last steps of bringing up a connection 7408 * @vsi: the VSI being configured 7409 **/ 7410 static int i40e_up_complete(struct i40e_vsi *vsi) 7411 { 7412 struct i40e_pf *pf = vsi->back; 7413 int err; 7414 7415 if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags)) 7416 i40e_vsi_configure_msix(vsi); 7417 else 7418 i40e_configure_msi_and_legacy(vsi); 7419 7420 /* start rings */ 7421 err = i40e_vsi_start_rings(vsi); 7422 if (err) 7423 return err; 7424 7425 clear_bit(__I40E_VSI_DOWN, vsi->state); 7426 i40e_napi_enable_all(vsi); 7427 i40e_vsi_enable_irq(vsi); 7428 7429 if ((pf->hw.phy.link_info.link_info & I40E_AQ_LINK_UP) && 7430 (vsi->netdev)) { 7431 i40e_print_link_message(vsi, true); 7432 netif_tx_start_all_queues(vsi->netdev); 7433 netif_carrier_on(vsi->netdev); 7434 } 7435 7436 /* replay FDIR SB filters */ 7437 if (vsi->type == I40E_VSI_FDIR) { 7438 /* reset fd counters */ 7439 pf->fd_add_err = 0; 7440 pf->fd_atr_cnt = 0; 7441 i40e_fdir_filter_restore(vsi); 7442 } 7443 7444 /* On the next run of the service_task, notify any clients of the new 7445 * opened netdev 7446 */ 7447 set_bit(__I40E_CLIENT_SERVICE_REQUESTED, pf->state); 7448 i40e_service_event_schedule(pf); 7449 7450 return 0; 7451 } 7452 7453 /** 7454 * i40e_vsi_reinit_locked - Reset the VSI 7455 * @vsi: the VSI being configured 7456 * 7457 * Rebuild the ring structs after some configuration 7458 * has changed, e.g. MTU size. 7459 **/ 7460 static void i40e_vsi_reinit_locked(struct i40e_vsi *vsi) 7461 { 7462 struct i40e_pf *pf = vsi->back; 7463 7464 while (test_and_set_bit(__I40E_CONFIG_BUSY, pf->state)) 7465 usleep_range(1000, 2000); 7466 i40e_down(vsi); 7467 7468 i40e_up(vsi); 7469 clear_bit(__I40E_CONFIG_BUSY, pf->state); 7470 } 7471 7472 /** 7473 * i40e_force_link_state - Force the link status 7474 * @pf: board private structure 7475 * @is_up: whether the link state should be forced up or down 7476 **/ 7477 static int i40e_force_link_state(struct i40e_pf *pf, bool is_up) 7478 { 7479 struct i40e_aq_get_phy_abilities_resp abilities; 7480 struct i40e_aq_set_phy_config config = {0}; 7481 bool non_zero_phy_type = is_up; 7482 struct i40e_hw *hw = &pf->hw; 7483 u64 mask; 7484 u8 speed; 7485 int err; 7486 7487 /* Card might've been put in an unstable state by other drivers 7488 * and applications, which causes incorrect speed values being 7489 * set on startup. In order to clear speed registers, we call 7490 * get_phy_capabilities twice, once to get initial state of 7491 * available speeds, and once to get current PHY config. 7492 */ 7493 err = i40e_aq_get_phy_capabilities(hw, false, true, &abilities, 7494 NULL); 7495 if (err) { 7496 dev_err(&pf->pdev->dev, 7497 "failed to get phy cap., ret = %pe last_status = %s\n", 7498 ERR_PTR(err), 7499 i40e_aq_str(hw, hw->aq.asq_last_status)); 7500 return err; 7501 } 7502 speed = abilities.link_speed; 7503 7504 /* Get the current phy config */ 7505 err = i40e_aq_get_phy_capabilities(hw, false, false, &abilities, 7506 NULL); 7507 if (err) { 7508 dev_err(&pf->pdev->dev, 7509 "failed to get phy cap., ret = %pe last_status = %s\n", 7510 ERR_PTR(err), 7511 i40e_aq_str(hw, hw->aq.asq_last_status)); 7512 return err; 7513 } 7514 7515 /* If link needs to go up, but was not forced to go down, 7516 * and its speed values are OK, no need for a flap 7517 * if non_zero_phy_type was set, still need to force up 7518 */ 7519 if (test_bit(I40E_FLAG_TOTAL_PORT_SHUTDOWN_ENA, pf->flags)) 7520 non_zero_phy_type = true; 7521 else if (is_up && abilities.phy_type != 0 && abilities.link_speed != 0) 7522 return 0; 7523 7524 /* To force link we need to set bits for all supported PHY types, 7525 * but there are now more than 32, so we need to split the bitmap 7526 * across two fields. 7527 */ 7528 mask = I40E_PHY_TYPES_BITMASK; 7529 config.phy_type = 7530 non_zero_phy_type ? cpu_to_le32((u32)(mask & 0xffffffff)) : 0; 7531 config.phy_type_ext = 7532 non_zero_phy_type ? (u8)((mask >> 32) & 0xff) : 0; 7533 /* Copy the old settings, except of phy_type */ 7534 config.abilities = abilities.abilities; 7535 if (test_bit(I40E_FLAG_TOTAL_PORT_SHUTDOWN_ENA, pf->flags)) { 7536 if (is_up) 7537 config.abilities |= I40E_AQ_PHY_ENABLE_LINK; 7538 else 7539 config.abilities &= ~(I40E_AQ_PHY_ENABLE_LINK); 7540 } 7541 if (abilities.link_speed != 0) 7542 config.link_speed = abilities.link_speed; 7543 else 7544 config.link_speed = speed; 7545 config.eee_capability = abilities.eee_capability; 7546 config.eeer = abilities.eeer_val; 7547 config.low_power_ctrl = abilities.d3_lpan; 7548 config.fec_config = abilities.fec_cfg_curr_mod_ext_info & 7549 I40E_AQ_PHY_FEC_CONFIG_MASK; 7550 err = i40e_aq_set_phy_config(hw, &config, NULL); 7551 7552 if (err) { 7553 dev_err(&pf->pdev->dev, 7554 "set phy config ret = %pe last_status = %s\n", 7555 ERR_PTR(err), 7556 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 7557 return err; 7558 } 7559 7560 /* Update the link info */ 7561 err = i40e_update_link_info(hw); 7562 if (err) { 7563 /* Wait a little bit (on 40G cards it sometimes takes a really 7564 * long time for link to come back from the atomic reset) 7565 * and try once more 7566 */ 7567 msleep(1000); 7568 i40e_update_link_info(hw); 7569 } 7570 7571 i40e_aq_set_link_restart_an(hw, is_up, NULL); 7572 7573 return 0; 7574 } 7575 7576 /** 7577 * i40e_up - Bring the connection back up after being down 7578 * @vsi: the VSI being configured 7579 **/ 7580 int i40e_up(struct i40e_vsi *vsi) 7581 { 7582 int err; 7583 7584 if (vsi->type == I40E_VSI_MAIN && 7585 (test_bit(I40E_FLAG_LINK_DOWN_ON_CLOSE_ENA, vsi->back->flags) || 7586 test_bit(I40E_FLAG_TOTAL_PORT_SHUTDOWN_ENA, vsi->back->flags))) 7587 i40e_force_link_state(vsi->back, true); 7588 7589 err = i40e_vsi_configure(vsi); 7590 if (!err) 7591 err = i40e_up_complete(vsi); 7592 7593 return err; 7594 } 7595 7596 /** 7597 * i40e_down - Shutdown the connection processing 7598 * @vsi: the VSI being stopped 7599 **/ 7600 void i40e_down(struct i40e_vsi *vsi) 7601 { 7602 int i; 7603 7604 /* It is assumed that the caller of this function 7605 * sets the vsi->state __I40E_VSI_DOWN bit. 7606 */ 7607 if (vsi->netdev) { 7608 netif_carrier_off(vsi->netdev); 7609 netif_tx_disable(vsi->netdev); 7610 } 7611 i40e_vsi_disable_irq(vsi); 7612 i40e_vsi_stop_rings(vsi); 7613 if (vsi->type == I40E_VSI_MAIN && 7614 (test_bit(I40E_FLAG_LINK_DOWN_ON_CLOSE_ENA, vsi->back->flags) || 7615 test_bit(I40E_FLAG_TOTAL_PORT_SHUTDOWN_ENA, vsi->back->flags))) 7616 i40e_force_link_state(vsi->back, false); 7617 i40e_napi_disable_all(vsi); 7618 7619 for (i = 0; i < vsi->num_queue_pairs; i++) { 7620 i40e_clean_tx_ring(vsi->tx_rings[i]); 7621 if (i40e_enabled_xdp_vsi(vsi)) { 7622 /* Make sure that in-progress ndo_xdp_xmit and 7623 * ndo_xsk_wakeup calls are completed. 7624 */ 7625 synchronize_rcu(); 7626 i40e_clean_tx_ring(vsi->xdp_rings[i]); 7627 } 7628 i40e_clean_rx_ring(vsi->rx_rings[i]); 7629 } 7630 7631 } 7632 7633 /** 7634 * i40e_validate_mqprio_qopt- validate queue mapping info 7635 * @vsi: the VSI being configured 7636 * @mqprio_qopt: queue parametrs 7637 **/ 7638 static int i40e_validate_mqprio_qopt(struct i40e_vsi *vsi, 7639 struct tc_mqprio_qopt_offload *mqprio_qopt) 7640 { 7641 u64 sum_max_rate = 0; 7642 u64 max_rate = 0; 7643 int i; 7644 7645 if (mqprio_qopt->qopt.offset[0] != 0 || 7646 mqprio_qopt->qopt.num_tc < 1 || 7647 mqprio_qopt->qopt.num_tc > I40E_MAX_TRAFFIC_CLASS) 7648 return -EINVAL; 7649 for (i = 0; ; i++) { 7650 if (!mqprio_qopt->qopt.count[i]) 7651 return -EINVAL; 7652 if (mqprio_qopt->min_rate[i]) { 7653 dev_err(&vsi->back->pdev->dev, 7654 "Invalid min tx rate (greater than 0) specified\n"); 7655 return -EINVAL; 7656 } 7657 max_rate = mqprio_qopt->max_rate[i]; 7658 do_div(max_rate, I40E_BW_MBPS_DIVISOR); 7659 sum_max_rate += max_rate; 7660 7661 if (i >= mqprio_qopt->qopt.num_tc - 1) 7662 break; 7663 if (mqprio_qopt->qopt.offset[i + 1] != 7664 (mqprio_qopt->qopt.offset[i] + mqprio_qopt->qopt.count[i])) 7665 return -EINVAL; 7666 } 7667 if (vsi->num_queue_pairs < 7668 (mqprio_qopt->qopt.offset[i] + mqprio_qopt->qopt.count[i])) { 7669 dev_err(&vsi->back->pdev->dev, 7670 "Failed to create traffic channel, insufficient number of queues.\n"); 7671 return -EINVAL; 7672 } 7673 if (sum_max_rate > i40e_get_link_speed(vsi)) { 7674 dev_err(&vsi->back->pdev->dev, 7675 "Invalid max tx rate specified\n"); 7676 return -EINVAL; 7677 } 7678 return 0; 7679 } 7680 7681 /** 7682 * i40e_vsi_set_default_tc_config - set default values for tc configuration 7683 * @vsi: the VSI being configured 7684 **/ 7685 static void i40e_vsi_set_default_tc_config(struct i40e_vsi *vsi) 7686 { 7687 u16 qcount; 7688 int i; 7689 7690 /* Only TC0 is enabled */ 7691 vsi->tc_config.numtc = 1; 7692 vsi->tc_config.enabled_tc = 1; 7693 qcount = min_t(int, vsi->alloc_queue_pairs, 7694 i40e_pf_get_max_q_per_tc(vsi->back)); 7695 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 7696 /* For the TC that is not enabled set the offset to default 7697 * queue and allocate one queue for the given TC. 7698 */ 7699 vsi->tc_config.tc_info[i].qoffset = 0; 7700 if (i == 0) 7701 vsi->tc_config.tc_info[i].qcount = qcount; 7702 else 7703 vsi->tc_config.tc_info[i].qcount = 1; 7704 vsi->tc_config.tc_info[i].netdev_tc = 0; 7705 } 7706 } 7707 7708 /** 7709 * i40e_del_macvlan_filter 7710 * @hw: pointer to the HW structure 7711 * @seid: seid of the channel VSI 7712 * @macaddr: the mac address to apply as a filter 7713 * @aq_err: store the admin Q error 7714 * 7715 * This function deletes a mac filter on the channel VSI which serves as the 7716 * macvlan. Returns 0 on success. 7717 **/ 7718 static int i40e_del_macvlan_filter(struct i40e_hw *hw, u16 seid, 7719 const u8 *macaddr, int *aq_err) 7720 { 7721 struct i40e_aqc_remove_macvlan_element_data element; 7722 int status; 7723 7724 memset(&element, 0, sizeof(element)); 7725 ether_addr_copy(element.mac_addr, macaddr); 7726 element.vlan_tag = 0; 7727 element.flags = I40E_AQC_MACVLAN_DEL_PERFECT_MATCH; 7728 status = i40e_aq_remove_macvlan(hw, seid, &element, 1, NULL); 7729 *aq_err = hw->aq.asq_last_status; 7730 7731 return status; 7732 } 7733 7734 /** 7735 * i40e_add_macvlan_filter 7736 * @hw: pointer to the HW structure 7737 * @seid: seid of the channel VSI 7738 * @macaddr: the mac address to apply as a filter 7739 * @aq_err: store the admin Q error 7740 * 7741 * This function adds a mac filter on the channel VSI which serves as the 7742 * macvlan. Returns 0 on success. 7743 **/ 7744 static int i40e_add_macvlan_filter(struct i40e_hw *hw, u16 seid, 7745 const u8 *macaddr, int *aq_err) 7746 { 7747 struct i40e_aqc_add_macvlan_element_data element; 7748 u16 cmd_flags = 0; 7749 int status; 7750 7751 ether_addr_copy(element.mac_addr, macaddr); 7752 element.vlan_tag = 0; 7753 element.queue_number = 0; 7754 element.match_method = I40E_AQC_MM_ERR_NO_RES; 7755 cmd_flags |= I40E_AQC_MACVLAN_ADD_PERFECT_MATCH; 7756 element.flags = cpu_to_le16(cmd_flags); 7757 status = i40e_aq_add_macvlan(hw, seid, &element, 1, NULL); 7758 *aq_err = hw->aq.asq_last_status; 7759 7760 return status; 7761 } 7762 7763 /** 7764 * i40e_reset_ch_rings - Reset the queue contexts in a channel 7765 * @vsi: the VSI we want to access 7766 * @ch: the channel we want to access 7767 */ 7768 static void i40e_reset_ch_rings(struct i40e_vsi *vsi, struct i40e_channel *ch) 7769 { 7770 struct i40e_ring *tx_ring, *rx_ring; 7771 u16 pf_q; 7772 int i; 7773 7774 for (i = 0; i < ch->num_queue_pairs; i++) { 7775 pf_q = ch->base_queue + i; 7776 tx_ring = vsi->tx_rings[pf_q]; 7777 tx_ring->ch = NULL; 7778 rx_ring = vsi->rx_rings[pf_q]; 7779 rx_ring->ch = NULL; 7780 } 7781 } 7782 7783 /** 7784 * i40e_free_macvlan_channels 7785 * @vsi: the VSI we want to access 7786 * 7787 * This function frees the Qs of the channel VSI from 7788 * the stack and also deletes the channel VSIs which 7789 * serve as macvlans. 7790 */ 7791 static void i40e_free_macvlan_channels(struct i40e_vsi *vsi) 7792 { 7793 struct i40e_channel *ch, *ch_tmp; 7794 int ret; 7795 7796 if (list_empty(&vsi->macvlan_list)) 7797 return; 7798 7799 list_for_each_entry_safe(ch, ch_tmp, &vsi->macvlan_list, list) { 7800 struct i40e_vsi *parent_vsi; 7801 7802 if (i40e_is_channel_macvlan(ch)) { 7803 i40e_reset_ch_rings(vsi, ch); 7804 clear_bit(ch->fwd->bit_no, vsi->fwd_bitmask); 7805 netdev_unbind_sb_channel(vsi->netdev, ch->fwd->netdev); 7806 netdev_set_sb_channel(ch->fwd->netdev, 0); 7807 kfree(ch->fwd); 7808 ch->fwd = NULL; 7809 } 7810 7811 list_del(&ch->list); 7812 parent_vsi = ch->parent_vsi; 7813 if (!parent_vsi || !ch->initialized) { 7814 kfree(ch); 7815 continue; 7816 } 7817 7818 /* remove the VSI */ 7819 ret = i40e_aq_delete_element(&vsi->back->hw, ch->seid, 7820 NULL); 7821 if (ret) 7822 dev_err(&vsi->back->pdev->dev, 7823 "unable to remove channel (%d) for parent VSI(%d)\n", 7824 ch->seid, parent_vsi->seid); 7825 kfree(ch); 7826 } 7827 vsi->macvlan_cnt = 0; 7828 } 7829 7830 /** 7831 * i40e_fwd_ring_up - bring the macvlan device up 7832 * @vsi: the VSI we want to access 7833 * @vdev: macvlan netdevice 7834 * @fwd: the private fwd structure 7835 */ 7836 static int i40e_fwd_ring_up(struct i40e_vsi *vsi, struct net_device *vdev, 7837 struct i40e_fwd_adapter *fwd) 7838 { 7839 struct i40e_channel *ch = NULL, *ch_tmp, *iter; 7840 int ret = 0, num_tc = 1, i, aq_err; 7841 struct i40e_pf *pf = vsi->back; 7842 struct i40e_hw *hw = &pf->hw; 7843 7844 /* Go through the list and find an available channel */ 7845 list_for_each_entry_safe(iter, ch_tmp, &vsi->macvlan_list, list) { 7846 if (!i40e_is_channel_macvlan(iter)) { 7847 iter->fwd = fwd; 7848 /* record configuration for macvlan interface in vdev */ 7849 for (i = 0; i < num_tc; i++) 7850 netdev_bind_sb_channel_queue(vsi->netdev, vdev, 7851 i, 7852 iter->num_queue_pairs, 7853 iter->base_queue); 7854 for (i = 0; i < iter->num_queue_pairs; i++) { 7855 struct i40e_ring *tx_ring, *rx_ring; 7856 u16 pf_q; 7857 7858 pf_q = iter->base_queue + i; 7859 7860 /* Get to TX ring ptr */ 7861 tx_ring = vsi->tx_rings[pf_q]; 7862 tx_ring->ch = iter; 7863 7864 /* Get the RX ring ptr */ 7865 rx_ring = vsi->rx_rings[pf_q]; 7866 rx_ring->ch = iter; 7867 } 7868 ch = iter; 7869 break; 7870 } 7871 } 7872 7873 if (!ch) 7874 return -EINVAL; 7875 7876 /* Guarantee all rings are updated before we update the 7877 * MAC address filter. 7878 */ 7879 wmb(); 7880 7881 /* Add a mac filter */ 7882 ret = i40e_add_macvlan_filter(hw, ch->seid, vdev->dev_addr, &aq_err); 7883 if (ret) { 7884 /* if we cannot add the MAC rule then disable the offload */ 7885 macvlan_release_l2fw_offload(vdev); 7886 for (i = 0; i < ch->num_queue_pairs; i++) { 7887 struct i40e_ring *rx_ring; 7888 u16 pf_q; 7889 7890 pf_q = ch->base_queue + i; 7891 rx_ring = vsi->rx_rings[pf_q]; 7892 rx_ring->netdev = NULL; 7893 } 7894 dev_info(&pf->pdev->dev, 7895 "Error adding mac filter on macvlan err %pe, aq_err %s\n", 7896 ERR_PTR(ret), 7897 i40e_aq_str(hw, aq_err)); 7898 netdev_err(vdev, "L2fwd offload disabled to L2 filter error\n"); 7899 } 7900 7901 return ret; 7902 } 7903 7904 /** 7905 * i40e_setup_macvlans - create the channels which will be macvlans 7906 * @vsi: the VSI we want to access 7907 * @macvlan_cnt: no. of macvlans to be setup 7908 * @qcnt: no. of Qs per macvlan 7909 * @vdev: macvlan netdevice 7910 */ 7911 static int i40e_setup_macvlans(struct i40e_vsi *vsi, u16 macvlan_cnt, u16 qcnt, 7912 struct net_device *vdev) 7913 { 7914 struct i40e_pf *pf = vsi->back; 7915 struct i40e_hw *hw = &pf->hw; 7916 struct i40e_vsi_context ctxt; 7917 u16 sections, qmap, num_qps; 7918 struct i40e_channel *ch; 7919 int i, pow, ret = 0; 7920 u8 offset = 0; 7921 7922 if (vsi->type != I40E_VSI_MAIN || !macvlan_cnt) 7923 return -EINVAL; 7924 7925 num_qps = vsi->num_queue_pairs - (macvlan_cnt * qcnt); 7926 7927 /* find the next higher power-of-2 of num queue pairs */ 7928 pow = fls(roundup_pow_of_two(num_qps) - 1); 7929 7930 qmap = (offset << I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT) | 7931 (pow << I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT); 7932 7933 /* Setup context bits for the main VSI */ 7934 sections = I40E_AQ_VSI_PROP_QUEUE_MAP_VALID; 7935 sections |= I40E_AQ_VSI_PROP_SCHED_VALID; 7936 memset(&ctxt, 0, sizeof(ctxt)); 7937 ctxt.seid = vsi->seid; 7938 ctxt.pf_num = vsi->back->hw.pf_id; 7939 ctxt.vf_num = 0; 7940 ctxt.uplink_seid = vsi->uplink_seid; 7941 ctxt.info = vsi->info; 7942 ctxt.info.tc_mapping[0] = cpu_to_le16(qmap); 7943 ctxt.info.mapping_flags |= cpu_to_le16(I40E_AQ_VSI_QUE_MAP_CONTIG); 7944 ctxt.info.queue_mapping[0] = cpu_to_le16(vsi->base_queue); 7945 ctxt.info.valid_sections |= cpu_to_le16(sections); 7946 7947 /* Reconfigure RSS for main VSI with new max queue count */ 7948 vsi->rss_size = max_t(u16, num_qps, qcnt); 7949 ret = i40e_vsi_config_rss(vsi); 7950 if (ret) { 7951 dev_info(&pf->pdev->dev, 7952 "Failed to reconfig RSS for num_queues (%u)\n", 7953 vsi->rss_size); 7954 return ret; 7955 } 7956 vsi->reconfig_rss = true; 7957 dev_dbg(&vsi->back->pdev->dev, 7958 "Reconfigured RSS with num_queues (%u)\n", vsi->rss_size); 7959 vsi->next_base_queue = num_qps; 7960 vsi->cnt_q_avail = vsi->num_queue_pairs - num_qps; 7961 7962 /* Update the VSI after updating the VSI queue-mapping 7963 * information 7964 */ 7965 ret = i40e_aq_update_vsi_params(hw, &ctxt, NULL); 7966 if (ret) { 7967 dev_info(&pf->pdev->dev, 7968 "Update vsi tc config failed, err %pe aq_err %s\n", 7969 ERR_PTR(ret), 7970 i40e_aq_str(hw, hw->aq.asq_last_status)); 7971 return ret; 7972 } 7973 /* update the local VSI info with updated queue map */ 7974 i40e_vsi_update_queue_map(vsi, &ctxt); 7975 vsi->info.valid_sections = 0; 7976 7977 /* Create channels for macvlans */ 7978 INIT_LIST_HEAD(&vsi->macvlan_list); 7979 for (i = 0; i < macvlan_cnt; i++) { 7980 ch = kzalloc(sizeof(*ch), GFP_KERNEL); 7981 if (!ch) { 7982 ret = -ENOMEM; 7983 goto err_free; 7984 } 7985 INIT_LIST_HEAD(&ch->list); 7986 ch->num_queue_pairs = qcnt; 7987 if (!i40e_setup_channel(pf, vsi, ch)) { 7988 ret = -EINVAL; 7989 kfree(ch); 7990 goto err_free; 7991 } 7992 ch->parent_vsi = vsi; 7993 vsi->cnt_q_avail -= ch->num_queue_pairs; 7994 vsi->macvlan_cnt++; 7995 list_add_tail(&ch->list, &vsi->macvlan_list); 7996 } 7997 7998 return ret; 7999 8000 err_free: 8001 dev_info(&pf->pdev->dev, "Failed to setup macvlans\n"); 8002 i40e_free_macvlan_channels(vsi); 8003 8004 return ret; 8005 } 8006 8007 /** 8008 * i40e_fwd_add - configure macvlans 8009 * @netdev: net device to configure 8010 * @vdev: macvlan netdevice 8011 **/ 8012 static void *i40e_fwd_add(struct net_device *netdev, struct net_device *vdev) 8013 { 8014 struct i40e_netdev_priv *np = netdev_priv(netdev); 8015 u16 q_per_macvlan = 0, macvlan_cnt = 0, vectors; 8016 struct i40e_vsi *vsi = np->vsi; 8017 struct i40e_pf *pf = vsi->back; 8018 struct i40e_fwd_adapter *fwd; 8019 int avail_macvlan, ret; 8020 8021 if (test_bit(I40E_FLAG_DCB_ENA, pf->flags)) { 8022 netdev_info(netdev, "Macvlans are not supported when DCB is enabled\n"); 8023 return ERR_PTR(-EINVAL); 8024 } 8025 if (i40e_is_tc_mqprio_enabled(pf)) { 8026 netdev_info(netdev, "Macvlans are not supported when HW TC offload is on\n"); 8027 return ERR_PTR(-EINVAL); 8028 } 8029 if (pf->num_lan_msix < I40E_MIN_MACVLAN_VECTORS) { 8030 netdev_info(netdev, "Not enough vectors available to support macvlans\n"); 8031 return ERR_PTR(-EINVAL); 8032 } 8033 8034 /* The macvlan device has to be a single Q device so that the 8035 * tc_to_txq field can be reused to pick the tx queue. 8036 */ 8037 if (netif_is_multiqueue(vdev)) 8038 return ERR_PTR(-ERANGE); 8039 8040 if (!vsi->macvlan_cnt) { 8041 /* reserve bit 0 for the pf device */ 8042 set_bit(0, vsi->fwd_bitmask); 8043 8044 /* Try to reserve as many queues as possible for macvlans. First 8045 * reserve 3/4th of max vectors, then half, then quarter and 8046 * calculate Qs per macvlan as you go 8047 */ 8048 vectors = pf->num_lan_msix; 8049 if (vectors <= I40E_MAX_MACVLANS && vectors > 64) { 8050 /* allocate 4 Qs per macvlan and 32 Qs to the PF*/ 8051 q_per_macvlan = 4; 8052 macvlan_cnt = (vectors - 32) / 4; 8053 } else if (vectors <= 64 && vectors > 32) { 8054 /* allocate 2 Qs per macvlan and 16 Qs to the PF*/ 8055 q_per_macvlan = 2; 8056 macvlan_cnt = (vectors - 16) / 2; 8057 } else if (vectors <= 32 && vectors > 16) { 8058 /* allocate 1 Q per macvlan and 16 Qs to the PF*/ 8059 q_per_macvlan = 1; 8060 macvlan_cnt = vectors - 16; 8061 } else if (vectors <= 16 && vectors > 8) { 8062 /* allocate 1 Q per macvlan and 8 Qs to the PF */ 8063 q_per_macvlan = 1; 8064 macvlan_cnt = vectors - 8; 8065 } else { 8066 /* allocate 1 Q per macvlan and 1 Q to the PF */ 8067 q_per_macvlan = 1; 8068 macvlan_cnt = vectors - 1; 8069 } 8070 8071 if (macvlan_cnt == 0) 8072 return ERR_PTR(-EBUSY); 8073 8074 /* Quiesce VSI queues */ 8075 i40e_quiesce_vsi(vsi); 8076 8077 /* sets up the macvlans but does not "enable" them */ 8078 ret = i40e_setup_macvlans(vsi, macvlan_cnt, q_per_macvlan, 8079 vdev); 8080 if (ret) 8081 return ERR_PTR(ret); 8082 8083 /* Unquiesce VSI */ 8084 i40e_unquiesce_vsi(vsi); 8085 } 8086 avail_macvlan = find_first_zero_bit(vsi->fwd_bitmask, 8087 vsi->macvlan_cnt); 8088 if (avail_macvlan >= I40E_MAX_MACVLANS) 8089 return ERR_PTR(-EBUSY); 8090 8091 /* create the fwd struct */ 8092 fwd = kzalloc(sizeof(*fwd), GFP_KERNEL); 8093 if (!fwd) 8094 return ERR_PTR(-ENOMEM); 8095 8096 set_bit(avail_macvlan, vsi->fwd_bitmask); 8097 fwd->bit_no = avail_macvlan; 8098 netdev_set_sb_channel(vdev, avail_macvlan); 8099 fwd->netdev = vdev; 8100 8101 if (!netif_running(netdev)) 8102 return fwd; 8103 8104 /* Set fwd ring up */ 8105 ret = i40e_fwd_ring_up(vsi, vdev, fwd); 8106 if (ret) { 8107 /* unbind the queues and drop the subordinate channel config */ 8108 netdev_unbind_sb_channel(netdev, vdev); 8109 netdev_set_sb_channel(vdev, 0); 8110 8111 kfree(fwd); 8112 return ERR_PTR(-EINVAL); 8113 } 8114 8115 return fwd; 8116 } 8117 8118 /** 8119 * i40e_del_all_macvlans - Delete all the mac filters on the channels 8120 * @vsi: the VSI we want to access 8121 */ 8122 static void i40e_del_all_macvlans(struct i40e_vsi *vsi) 8123 { 8124 struct i40e_channel *ch, *ch_tmp; 8125 struct i40e_pf *pf = vsi->back; 8126 struct i40e_hw *hw = &pf->hw; 8127 int aq_err, ret = 0; 8128 8129 if (list_empty(&vsi->macvlan_list)) 8130 return; 8131 8132 list_for_each_entry_safe(ch, ch_tmp, &vsi->macvlan_list, list) { 8133 if (i40e_is_channel_macvlan(ch)) { 8134 ret = i40e_del_macvlan_filter(hw, ch->seid, 8135 i40e_channel_mac(ch), 8136 &aq_err); 8137 if (!ret) { 8138 /* Reset queue contexts */ 8139 i40e_reset_ch_rings(vsi, ch); 8140 clear_bit(ch->fwd->bit_no, vsi->fwd_bitmask); 8141 netdev_unbind_sb_channel(vsi->netdev, 8142 ch->fwd->netdev); 8143 netdev_set_sb_channel(ch->fwd->netdev, 0); 8144 kfree(ch->fwd); 8145 ch->fwd = NULL; 8146 } 8147 } 8148 } 8149 } 8150 8151 /** 8152 * i40e_fwd_del - delete macvlan interfaces 8153 * @netdev: net device to configure 8154 * @vdev: macvlan netdevice 8155 */ 8156 static void i40e_fwd_del(struct net_device *netdev, void *vdev) 8157 { 8158 struct i40e_netdev_priv *np = netdev_priv(netdev); 8159 struct i40e_fwd_adapter *fwd = vdev; 8160 struct i40e_channel *ch, *ch_tmp; 8161 struct i40e_vsi *vsi = np->vsi; 8162 struct i40e_pf *pf = vsi->back; 8163 struct i40e_hw *hw = &pf->hw; 8164 int aq_err, ret = 0; 8165 8166 /* Find the channel associated with the macvlan and del mac filter */ 8167 list_for_each_entry_safe(ch, ch_tmp, &vsi->macvlan_list, list) { 8168 if (i40e_is_channel_macvlan(ch) && 8169 ether_addr_equal(i40e_channel_mac(ch), 8170 fwd->netdev->dev_addr)) { 8171 ret = i40e_del_macvlan_filter(hw, ch->seid, 8172 i40e_channel_mac(ch), 8173 &aq_err); 8174 if (!ret) { 8175 /* Reset queue contexts */ 8176 i40e_reset_ch_rings(vsi, ch); 8177 clear_bit(ch->fwd->bit_no, vsi->fwd_bitmask); 8178 netdev_unbind_sb_channel(netdev, fwd->netdev); 8179 netdev_set_sb_channel(fwd->netdev, 0); 8180 kfree(ch->fwd); 8181 ch->fwd = NULL; 8182 } else { 8183 dev_info(&pf->pdev->dev, 8184 "Error deleting mac filter on macvlan err %pe, aq_err %s\n", 8185 ERR_PTR(ret), 8186 i40e_aq_str(hw, aq_err)); 8187 } 8188 break; 8189 } 8190 } 8191 } 8192 8193 /** 8194 * i40e_setup_tc - configure multiple traffic classes 8195 * @netdev: net device to configure 8196 * @type_data: tc offload data 8197 **/ 8198 static int i40e_setup_tc(struct net_device *netdev, void *type_data) 8199 { 8200 struct tc_mqprio_qopt_offload *mqprio_qopt = type_data; 8201 struct i40e_netdev_priv *np = netdev_priv(netdev); 8202 struct i40e_vsi *vsi = np->vsi; 8203 struct i40e_pf *pf = vsi->back; 8204 u8 enabled_tc = 0, num_tc, hw; 8205 bool need_reset = false; 8206 int old_queue_pairs; 8207 int ret = -EINVAL; 8208 u16 mode; 8209 int i; 8210 8211 old_queue_pairs = vsi->num_queue_pairs; 8212 num_tc = mqprio_qopt->qopt.num_tc; 8213 hw = mqprio_qopt->qopt.hw; 8214 mode = mqprio_qopt->mode; 8215 if (!hw) { 8216 clear_bit(I40E_FLAG_TC_MQPRIO_ENA, pf->flags); 8217 memcpy(&vsi->mqprio_qopt, mqprio_qopt, sizeof(*mqprio_qopt)); 8218 goto config_tc; 8219 } 8220 8221 /* Check if MFP enabled */ 8222 if (test_bit(I40E_FLAG_MFP_ENA, pf->flags)) { 8223 netdev_info(netdev, 8224 "Configuring TC not supported in MFP mode\n"); 8225 return ret; 8226 } 8227 switch (mode) { 8228 case TC_MQPRIO_MODE_DCB: 8229 clear_bit(I40E_FLAG_TC_MQPRIO_ENA, pf->flags); 8230 8231 /* Check if DCB enabled to continue */ 8232 if (!test_bit(I40E_FLAG_DCB_ENA, pf->flags)) { 8233 netdev_info(netdev, 8234 "DCB is not enabled for adapter\n"); 8235 return ret; 8236 } 8237 8238 /* Check whether tc count is within enabled limit */ 8239 if (num_tc > i40e_pf_get_num_tc(pf)) { 8240 netdev_info(netdev, 8241 "TC count greater than enabled on link for adapter\n"); 8242 return ret; 8243 } 8244 break; 8245 case TC_MQPRIO_MODE_CHANNEL: 8246 if (test_bit(I40E_FLAG_DCB_ENA, pf->flags)) { 8247 netdev_info(netdev, 8248 "Full offload of TC Mqprio options is not supported when DCB is enabled\n"); 8249 return ret; 8250 } 8251 if (!test_bit(I40E_FLAG_MSIX_ENA, pf->flags)) 8252 return ret; 8253 ret = i40e_validate_mqprio_qopt(vsi, mqprio_qopt); 8254 if (ret) 8255 return ret; 8256 memcpy(&vsi->mqprio_qopt, mqprio_qopt, 8257 sizeof(*mqprio_qopt)); 8258 set_bit(I40E_FLAG_TC_MQPRIO_ENA, pf->flags); 8259 clear_bit(I40E_FLAG_DCB_ENA, pf->flags); 8260 break; 8261 default: 8262 return -EINVAL; 8263 } 8264 8265 config_tc: 8266 /* Generate TC map for number of tc requested */ 8267 for (i = 0; i < num_tc; i++) 8268 enabled_tc |= BIT(i); 8269 8270 /* Requesting same TC configuration as already enabled */ 8271 if (enabled_tc == vsi->tc_config.enabled_tc && 8272 mode != TC_MQPRIO_MODE_CHANNEL) 8273 return 0; 8274 8275 /* Quiesce VSI queues */ 8276 i40e_quiesce_vsi(vsi); 8277 8278 if (!hw && !i40e_is_tc_mqprio_enabled(pf)) 8279 i40e_remove_queue_channels(vsi); 8280 8281 /* Configure VSI for enabled TCs */ 8282 ret = i40e_vsi_config_tc(vsi, enabled_tc); 8283 if (ret) { 8284 netdev_info(netdev, "Failed configuring TC for VSI seid=%d\n", 8285 vsi->seid); 8286 need_reset = true; 8287 goto exit; 8288 } else if (enabled_tc && 8289 (!is_power_of_2(vsi->tc_config.tc_info[0].qcount))) { 8290 netdev_info(netdev, 8291 "Failed to create channel. Override queues (%u) not power of 2\n", 8292 vsi->tc_config.tc_info[0].qcount); 8293 ret = -EINVAL; 8294 need_reset = true; 8295 goto exit; 8296 } 8297 8298 dev_info(&vsi->back->pdev->dev, 8299 "Setup channel (id:%u) utilizing num_queues %d\n", 8300 vsi->seid, vsi->tc_config.tc_info[0].qcount); 8301 8302 if (i40e_is_tc_mqprio_enabled(pf)) { 8303 if (vsi->mqprio_qopt.max_rate[0]) { 8304 u64 max_tx_rate = i40e_bw_bytes_to_mbits(vsi, 8305 vsi->mqprio_qopt.max_rate[0]); 8306 8307 ret = i40e_set_bw_limit(vsi, vsi->seid, max_tx_rate); 8308 if (!ret) { 8309 u64 credits = max_tx_rate; 8310 8311 do_div(credits, I40E_BW_CREDIT_DIVISOR); 8312 dev_dbg(&vsi->back->pdev->dev, 8313 "Set tx rate of %llu Mbps (count of 50Mbps %llu) for vsi->seid %u\n", 8314 max_tx_rate, 8315 credits, 8316 vsi->seid); 8317 } else { 8318 need_reset = true; 8319 goto exit; 8320 } 8321 } 8322 ret = i40e_configure_queue_channels(vsi); 8323 if (ret) { 8324 vsi->num_queue_pairs = old_queue_pairs; 8325 netdev_info(netdev, 8326 "Failed configuring queue channels\n"); 8327 need_reset = true; 8328 goto exit; 8329 } 8330 } 8331 8332 exit: 8333 /* Reset the configuration data to defaults, only TC0 is enabled */ 8334 if (need_reset) { 8335 i40e_vsi_set_default_tc_config(vsi); 8336 need_reset = false; 8337 } 8338 8339 /* Unquiesce VSI */ 8340 i40e_unquiesce_vsi(vsi); 8341 return ret; 8342 } 8343 8344 /** 8345 * i40e_set_cld_element - sets cloud filter element data 8346 * @filter: cloud filter rule 8347 * @cld: ptr to cloud filter element data 8348 * 8349 * This is helper function to copy data into cloud filter element 8350 **/ 8351 static inline void 8352 i40e_set_cld_element(struct i40e_cloud_filter *filter, 8353 struct i40e_aqc_cloud_filters_element_data *cld) 8354 { 8355 u32 ipa; 8356 int i; 8357 8358 memset(cld, 0, sizeof(*cld)); 8359 ether_addr_copy(cld->outer_mac, filter->dst_mac); 8360 ether_addr_copy(cld->inner_mac, filter->src_mac); 8361 8362 if (filter->n_proto != ETH_P_IP && filter->n_proto != ETH_P_IPV6) 8363 return; 8364 8365 if (filter->n_proto == ETH_P_IPV6) { 8366 #define IPV6_MAX_INDEX (ARRAY_SIZE(filter->dst_ipv6) - 1) 8367 for (i = 0; i < ARRAY_SIZE(filter->dst_ipv6); i++) { 8368 ipa = be32_to_cpu(filter->dst_ipv6[IPV6_MAX_INDEX - i]); 8369 8370 *(__le32 *)&cld->ipaddr.raw_v6.data[i * 2] = cpu_to_le32(ipa); 8371 } 8372 } else { 8373 ipa = be32_to_cpu(filter->dst_ipv4); 8374 8375 memcpy(&cld->ipaddr.v4.data, &ipa, sizeof(ipa)); 8376 } 8377 8378 cld->inner_vlan = cpu_to_le16(ntohs(filter->vlan_id)); 8379 8380 /* tenant_id is not supported by FW now, once the support is enabled 8381 * fill the cld->tenant_id with cpu_to_le32(filter->tenant_id) 8382 */ 8383 if (filter->tenant_id) 8384 return; 8385 } 8386 8387 /** 8388 * i40e_add_del_cloud_filter - Add/del cloud filter 8389 * @vsi: pointer to VSI 8390 * @filter: cloud filter rule 8391 * @add: if true, add, if false, delete 8392 * 8393 * Add or delete a cloud filter for a specific flow spec. 8394 * Returns 0 if the filter were successfully added. 8395 **/ 8396 int i40e_add_del_cloud_filter(struct i40e_vsi *vsi, 8397 struct i40e_cloud_filter *filter, bool add) 8398 { 8399 struct i40e_aqc_cloud_filters_element_data cld_filter; 8400 struct i40e_pf *pf = vsi->back; 8401 int ret; 8402 static const u16 flag_table[128] = { 8403 [I40E_CLOUD_FILTER_FLAGS_OMAC] = 8404 I40E_AQC_ADD_CLOUD_FILTER_OMAC, 8405 [I40E_CLOUD_FILTER_FLAGS_IMAC] = 8406 I40E_AQC_ADD_CLOUD_FILTER_IMAC, 8407 [I40E_CLOUD_FILTER_FLAGS_IMAC_IVLAN] = 8408 I40E_AQC_ADD_CLOUD_FILTER_IMAC_IVLAN, 8409 [I40E_CLOUD_FILTER_FLAGS_IMAC_TEN_ID] = 8410 I40E_AQC_ADD_CLOUD_FILTER_IMAC_TEN_ID, 8411 [I40E_CLOUD_FILTER_FLAGS_OMAC_TEN_ID_IMAC] = 8412 I40E_AQC_ADD_CLOUD_FILTER_OMAC_TEN_ID_IMAC, 8413 [I40E_CLOUD_FILTER_FLAGS_IMAC_IVLAN_TEN_ID] = 8414 I40E_AQC_ADD_CLOUD_FILTER_IMAC_IVLAN_TEN_ID, 8415 [I40E_CLOUD_FILTER_FLAGS_IIP] = 8416 I40E_AQC_ADD_CLOUD_FILTER_IIP, 8417 }; 8418 8419 if (filter->flags >= ARRAY_SIZE(flag_table)) 8420 return -EIO; 8421 8422 memset(&cld_filter, 0, sizeof(cld_filter)); 8423 8424 /* copy element needed to add cloud filter from filter */ 8425 i40e_set_cld_element(filter, &cld_filter); 8426 8427 if (filter->tunnel_type != I40E_CLOUD_TNL_TYPE_NONE) 8428 cld_filter.flags = cpu_to_le16(filter->tunnel_type << 8429 I40E_AQC_ADD_CLOUD_TNL_TYPE_SHIFT); 8430 8431 if (filter->n_proto == ETH_P_IPV6) 8432 cld_filter.flags |= cpu_to_le16(flag_table[filter->flags] | 8433 I40E_AQC_ADD_CLOUD_FLAGS_IPV6); 8434 else 8435 cld_filter.flags |= cpu_to_le16(flag_table[filter->flags] | 8436 I40E_AQC_ADD_CLOUD_FLAGS_IPV4); 8437 8438 if (add) 8439 ret = i40e_aq_add_cloud_filters(&pf->hw, filter->seid, 8440 &cld_filter, 1); 8441 else 8442 ret = i40e_aq_rem_cloud_filters(&pf->hw, filter->seid, 8443 &cld_filter, 1); 8444 if (ret) 8445 dev_dbg(&pf->pdev->dev, 8446 "Failed to %s cloud filter using l4 port %u, err %d aq_err %d\n", 8447 add ? "add" : "delete", filter->dst_port, ret, 8448 pf->hw.aq.asq_last_status); 8449 else 8450 dev_info(&pf->pdev->dev, 8451 "%s cloud filter for VSI: %d\n", 8452 add ? "Added" : "Deleted", filter->seid); 8453 return ret; 8454 } 8455 8456 /** 8457 * i40e_add_del_cloud_filter_big_buf - Add/del cloud filter using big_buf 8458 * @vsi: pointer to VSI 8459 * @filter: cloud filter rule 8460 * @add: if true, add, if false, delete 8461 * 8462 * Add or delete a cloud filter for a specific flow spec using big buffer. 8463 * Returns 0 if the filter were successfully added. 8464 **/ 8465 int i40e_add_del_cloud_filter_big_buf(struct i40e_vsi *vsi, 8466 struct i40e_cloud_filter *filter, 8467 bool add) 8468 { 8469 struct i40e_aqc_cloud_filters_element_bb cld_filter; 8470 struct i40e_pf *pf = vsi->back; 8471 int ret; 8472 8473 /* Both (src/dst) valid mac_addr are not supported */ 8474 if ((is_valid_ether_addr(filter->dst_mac) && 8475 is_valid_ether_addr(filter->src_mac)) || 8476 (is_multicast_ether_addr(filter->dst_mac) && 8477 is_multicast_ether_addr(filter->src_mac))) 8478 return -EOPNOTSUPP; 8479 8480 /* Big buffer cloud filter needs 'L4 port' to be non-zero. Also, UDP 8481 * ports are not supported via big buffer now. 8482 */ 8483 if (!filter->dst_port || filter->ip_proto == IPPROTO_UDP) 8484 return -EOPNOTSUPP; 8485 8486 /* adding filter using src_port/src_ip is not supported at this stage */ 8487 if (filter->src_port || 8488 (filter->src_ipv4 && filter->n_proto != ETH_P_IPV6) || 8489 !ipv6_addr_any(&filter->ip.v6.src_ip6)) 8490 return -EOPNOTSUPP; 8491 8492 memset(&cld_filter, 0, sizeof(cld_filter)); 8493 8494 /* copy element needed to add cloud filter from filter */ 8495 i40e_set_cld_element(filter, &cld_filter.element); 8496 8497 if (is_valid_ether_addr(filter->dst_mac) || 8498 is_valid_ether_addr(filter->src_mac) || 8499 is_multicast_ether_addr(filter->dst_mac) || 8500 is_multicast_ether_addr(filter->src_mac)) { 8501 /* MAC + IP : unsupported mode */ 8502 if (filter->dst_ipv4) 8503 return -EOPNOTSUPP; 8504 8505 /* since we validated that L4 port must be valid before 8506 * we get here, start with respective "flags" value 8507 * and update if vlan is present or not 8508 */ 8509 cld_filter.element.flags = 8510 cpu_to_le16(I40E_AQC_ADD_CLOUD_FILTER_MAC_PORT); 8511 8512 if (filter->vlan_id) { 8513 cld_filter.element.flags = 8514 cpu_to_le16(I40E_AQC_ADD_CLOUD_FILTER_MAC_VLAN_PORT); 8515 } 8516 8517 } else if ((filter->dst_ipv4 && filter->n_proto != ETH_P_IPV6) || 8518 !ipv6_addr_any(&filter->ip.v6.dst_ip6)) { 8519 cld_filter.element.flags = 8520 cpu_to_le16(I40E_AQC_ADD_CLOUD_FILTER_IP_PORT); 8521 if (filter->n_proto == ETH_P_IPV6) 8522 cld_filter.element.flags |= 8523 cpu_to_le16(I40E_AQC_ADD_CLOUD_FLAGS_IPV6); 8524 else 8525 cld_filter.element.flags |= 8526 cpu_to_le16(I40E_AQC_ADD_CLOUD_FLAGS_IPV4); 8527 } else { 8528 dev_err(&pf->pdev->dev, 8529 "either mac or ip has to be valid for cloud filter\n"); 8530 return -EINVAL; 8531 } 8532 8533 /* Now copy L4 port in Byte 6..7 in general fields */ 8534 cld_filter.general_fields[I40E_AQC_ADD_CLOUD_FV_FLU_0X16_WORD0] = 8535 be16_to_cpu(filter->dst_port); 8536 8537 if (add) { 8538 /* Validate current device switch mode, change if necessary */ 8539 ret = i40e_validate_and_set_switch_mode(vsi); 8540 if (ret) { 8541 dev_err(&pf->pdev->dev, 8542 "failed to set switch mode, ret %d\n", 8543 ret); 8544 return ret; 8545 } 8546 8547 ret = i40e_aq_add_cloud_filters_bb(&pf->hw, filter->seid, 8548 &cld_filter, 1); 8549 } else { 8550 ret = i40e_aq_rem_cloud_filters_bb(&pf->hw, filter->seid, 8551 &cld_filter, 1); 8552 } 8553 8554 if (ret) 8555 dev_dbg(&pf->pdev->dev, 8556 "Failed to %s cloud filter(big buffer) err %d aq_err %d\n", 8557 add ? "add" : "delete", ret, pf->hw.aq.asq_last_status); 8558 else 8559 dev_info(&pf->pdev->dev, 8560 "%s cloud filter for VSI: %d, L4 port: %d\n", 8561 add ? "add" : "delete", filter->seid, 8562 ntohs(filter->dst_port)); 8563 return ret; 8564 } 8565 8566 /** 8567 * i40e_parse_cls_flower - Parse tc flower filters provided by kernel 8568 * @vsi: Pointer to VSI 8569 * @f: Pointer to struct flow_cls_offload 8570 * @filter: Pointer to cloud filter structure 8571 * 8572 **/ 8573 static int i40e_parse_cls_flower(struct i40e_vsi *vsi, 8574 struct flow_cls_offload *f, 8575 struct i40e_cloud_filter *filter) 8576 { 8577 struct flow_rule *rule = flow_cls_offload_flow_rule(f); 8578 struct flow_dissector *dissector = rule->match.dissector; 8579 u16 n_proto_mask = 0, n_proto_key = 0, addr_type = 0; 8580 struct i40e_pf *pf = vsi->back; 8581 u8 field_flags = 0; 8582 8583 if (dissector->used_keys & 8584 ~(BIT_ULL(FLOW_DISSECTOR_KEY_CONTROL) | 8585 BIT_ULL(FLOW_DISSECTOR_KEY_BASIC) | 8586 BIT_ULL(FLOW_DISSECTOR_KEY_ETH_ADDRS) | 8587 BIT_ULL(FLOW_DISSECTOR_KEY_VLAN) | 8588 BIT_ULL(FLOW_DISSECTOR_KEY_IPV4_ADDRS) | 8589 BIT_ULL(FLOW_DISSECTOR_KEY_IPV6_ADDRS) | 8590 BIT_ULL(FLOW_DISSECTOR_KEY_PORTS) | 8591 BIT_ULL(FLOW_DISSECTOR_KEY_ENC_KEYID))) { 8592 dev_err(&pf->pdev->dev, "Unsupported key used: 0x%llx\n", 8593 dissector->used_keys); 8594 return -EOPNOTSUPP; 8595 } 8596 8597 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_ENC_KEYID)) { 8598 struct flow_match_enc_keyid match; 8599 8600 flow_rule_match_enc_keyid(rule, &match); 8601 if (match.mask->keyid != 0) 8602 field_flags |= I40E_CLOUD_FIELD_TEN_ID; 8603 8604 filter->tenant_id = be32_to_cpu(match.key->keyid); 8605 } 8606 8607 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_BASIC)) { 8608 struct flow_match_basic match; 8609 8610 flow_rule_match_basic(rule, &match); 8611 n_proto_key = ntohs(match.key->n_proto); 8612 n_proto_mask = ntohs(match.mask->n_proto); 8613 8614 if (n_proto_key == ETH_P_ALL) { 8615 n_proto_key = 0; 8616 n_proto_mask = 0; 8617 } 8618 filter->n_proto = n_proto_key & n_proto_mask; 8619 filter->ip_proto = match.key->ip_proto; 8620 } 8621 8622 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_ETH_ADDRS)) { 8623 struct flow_match_eth_addrs match; 8624 8625 flow_rule_match_eth_addrs(rule, &match); 8626 8627 /* use is_broadcast and is_zero to check for all 0xf or 0 */ 8628 if (!is_zero_ether_addr(match.mask->dst)) { 8629 if (is_broadcast_ether_addr(match.mask->dst)) { 8630 field_flags |= I40E_CLOUD_FIELD_OMAC; 8631 } else { 8632 dev_err(&pf->pdev->dev, "Bad ether dest mask %pM\n", 8633 match.mask->dst); 8634 return -EIO; 8635 } 8636 } 8637 8638 if (!is_zero_ether_addr(match.mask->src)) { 8639 if (is_broadcast_ether_addr(match.mask->src)) { 8640 field_flags |= I40E_CLOUD_FIELD_IMAC; 8641 } else { 8642 dev_err(&pf->pdev->dev, "Bad ether src mask %pM\n", 8643 match.mask->src); 8644 return -EIO; 8645 } 8646 } 8647 ether_addr_copy(filter->dst_mac, match.key->dst); 8648 ether_addr_copy(filter->src_mac, match.key->src); 8649 } 8650 8651 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_VLAN)) { 8652 struct flow_match_vlan match; 8653 8654 flow_rule_match_vlan(rule, &match); 8655 if (match.mask->vlan_id) { 8656 if (match.mask->vlan_id == VLAN_VID_MASK) { 8657 field_flags |= I40E_CLOUD_FIELD_IVLAN; 8658 8659 } else { 8660 dev_err(&pf->pdev->dev, "Bad vlan mask 0x%04x\n", 8661 match.mask->vlan_id); 8662 return -EIO; 8663 } 8664 } 8665 8666 filter->vlan_id = cpu_to_be16(match.key->vlan_id); 8667 } 8668 8669 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_CONTROL)) { 8670 struct flow_match_control match; 8671 8672 flow_rule_match_control(rule, &match); 8673 addr_type = match.key->addr_type; 8674 8675 if (flow_rule_has_control_flags(match.mask->flags, 8676 f->common.extack)) 8677 return -EOPNOTSUPP; 8678 } 8679 8680 if (addr_type == FLOW_DISSECTOR_KEY_IPV4_ADDRS) { 8681 struct flow_match_ipv4_addrs match; 8682 8683 flow_rule_match_ipv4_addrs(rule, &match); 8684 if (match.mask->dst) { 8685 if (match.mask->dst == cpu_to_be32(0xffffffff)) { 8686 field_flags |= I40E_CLOUD_FIELD_IIP; 8687 } else { 8688 dev_err(&pf->pdev->dev, "Bad ip dst mask %pI4b\n", 8689 &match.mask->dst); 8690 return -EIO; 8691 } 8692 } 8693 8694 if (match.mask->src) { 8695 if (match.mask->src == cpu_to_be32(0xffffffff)) { 8696 field_flags |= I40E_CLOUD_FIELD_IIP; 8697 } else { 8698 dev_err(&pf->pdev->dev, "Bad ip src mask %pI4b\n", 8699 &match.mask->src); 8700 return -EIO; 8701 } 8702 } 8703 8704 if (field_flags & I40E_CLOUD_FIELD_TEN_ID) { 8705 dev_err(&pf->pdev->dev, "Tenant id not allowed for ip filter\n"); 8706 return -EIO; 8707 } 8708 filter->dst_ipv4 = match.key->dst; 8709 filter->src_ipv4 = match.key->src; 8710 } 8711 8712 if (addr_type == FLOW_DISSECTOR_KEY_IPV6_ADDRS) { 8713 struct flow_match_ipv6_addrs match; 8714 8715 flow_rule_match_ipv6_addrs(rule, &match); 8716 8717 /* src and dest IPV6 address should not be LOOPBACK 8718 * (0:0:0:0:0:0:0:1), which can be represented as ::1 8719 */ 8720 if (ipv6_addr_loopback(&match.key->dst) || 8721 ipv6_addr_loopback(&match.key->src)) { 8722 dev_err(&pf->pdev->dev, 8723 "Bad ipv6, addr is LOOPBACK\n"); 8724 return -EIO; 8725 } 8726 if (!ipv6_addr_any(&match.mask->dst) || 8727 !ipv6_addr_any(&match.mask->src)) 8728 field_flags |= I40E_CLOUD_FIELD_IIP; 8729 8730 memcpy(&filter->src_ipv6, &match.key->src.s6_addr32, 8731 sizeof(filter->src_ipv6)); 8732 memcpy(&filter->dst_ipv6, &match.key->dst.s6_addr32, 8733 sizeof(filter->dst_ipv6)); 8734 } 8735 8736 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_PORTS)) { 8737 struct flow_match_ports match; 8738 8739 flow_rule_match_ports(rule, &match); 8740 if (match.mask->src) { 8741 if (match.mask->src == cpu_to_be16(0xffff)) { 8742 field_flags |= I40E_CLOUD_FIELD_IIP; 8743 } else { 8744 dev_err(&pf->pdev->dev, "Bad src port mask 0x%04x\n", 8745 be16_to_cpu(match.mask->src)); 8746 return -EIO; 8747 } 8748 } 8749 8750 if (match.mask->dst) { 8751 if (match.mask->dst == cpu_to_be16(0xffff)) { 8752 field_flags |= I40E_CLOUD_FIELD_IIP; 8753 } else { 8754 dev_err(&pf->pdev->dev, "Bad dst port mask 0x%04x\n", 8755 be16_to_cpu(match.mask->dst)); 8756 return -EIO; 8757 } 8758 } 8759 8760 filter->dst_port = match.key->dst; 8761 filter->src_port = match.key->src; 8762 8763 switch (filter->ip_proto) { 8764 case IPPROTO_TCP: 8765 case IPPROTO_UDP: 8766 break; 8767 default: 8768 dev_err(&pf->pdev->dev, 8769 "Only UDP and TCP transport are supported\n"); 8770 return -EINVAL; 8771 } 8772 } 8773 filter->flags = field_flags; 8774 return 0; 8775 } 8776 8777 /** 8778 * i40e_handle_tclass: Forward to a traffic class on the device 8779 * @vsi: Pointer to VSI 8780 * @tc: traffic class index on the device 8781 * @filter: Pointer to cloud filter structure 8782 * 8783 **/ 8784 static int i40e_handle_tclass(struct i40e_vsi *vsi, u32 tc, 8785 struct i40e_cloud_filter *filter) 8786 { 8787 struct i40e_channel *ch, *ch_tmp; 8788 8789 /* direct to a traffic class on the same device */ 8790 if (tc == 0) { 8791 filter->seid = vsi->seid; 8792 return 0; 8793 } else if (vsi->tc_config.enabled_tc & BIT(tc)) { 8794 if (!filter->dst_port) { 8795 dev_err(&vsi->back->pdev->dev, 8796 "Specify destination port to direct to traffic class that is not default\n"); 8797 return -EINVAL; 8798 } 8799 if (list_empty(&vsi->ch_list)) 8800 return -EINVAL; 8801 list_for_each_entry_safe(ch, ch_tmp, &vsi->ch_list, 8802 list) { 8803 if (ch->seid == vsi->tc_seid_map[tc]) 8804 filter->seid = ch->seid; 8805 } 8806 return 0; 8807 } 8808 dev_err(&vsi->back->pdev->dev, "TC is not enabled\n"); 8809 return -EINVAL; 8810 } 8811 8812 /** 8813 * i40e_configure_clsflower - Configure tc flower filters 8814 * @vsi: Pointer to VSI 8815 * @cls_flower: Pointer to struct flow_cls_offload 8816 * 8817 **/ 8818 static int i40e_configure_clsflower(struct i40e_vsi *vsi, 8819 struct flow_cls_offload *cls_flower) 8820 { 8821 int tc = tc_classid_to_hwtc(vsi->netdev, cls_flower->classid); 8822 struct i40e_cloud_filter *filter = NULL; 8823 struct i40e_pf *pf = vsi->back; 8824 int err = 0; 8825 8826 if (tc < 0) { 8827 dev_err(&vsi->back->pdev->dev, "Invalid traffic class\n"); 8828 return -EOPNOTSUPP; 8829 } 8830 8831 if (!tc) { 8832 dev_err(&pf->pdev->dev, "Unable to add filter because of invalid destination"); 8833 return -EINVAL; 8834 } 8835 8836 if (test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state) || 8837 test_bit(__I40E_RESET_INTR_RECEIVED, pf->state)) 8838 return -EBUSY; 8839 8840 if (pf->fdir_pf_active_filters || 8841 (!hlist_empty(&pf->fdir_filter_list))) { 8842 dev_err(&vsi->back->pdev->dev, 8843 "Flow Director Sideband filters exists, turn ntuple off to configure cloud filters\n"); 8844 return -EINVAL; 8845 } 8846 8847 if (test_bit(I40E_FLAG_FD_SB_ENA, vsi->back->flags)) { 8848 dev_err(&vsi->back->pdev->dev, 8849 "Disable Flow Director Sideband, configuring Cloud filters via tc-flower\n"); 8850 clear_bit(I40E_FLAG_FD_SB_ENA, vsi->back->flags); 8851 clear_bit(I40E_FLAG_FD_SB_TO_CLOUD_FILTER, vsi->back->flags); 8852 } 8853 8854 filter = kzalloc(sizeof(*filter), GFP_KERNEL); 8855 if (!filter) 8856 return -ENOMEM; 8857 8858 filter->cookie = cls_flower->cookie; 8859 8860 err = i40e_parse_cls_flower(vsi, cls_flower, filter); 8861 if (err < 0) 8862 goto err; 8863 8864 err = i40e_handle_tclass(vsi, tc, filter); 8865 if (err < 0) 8866 goto err; 8867 8868 /* Add cloud filter */ 8869 if (filter->dst_port) 8870 err = i40e_add_del_cloud_filter_big_buf(vsi, filter, true); 8871 else 8872 err = i40e_add_del_cloud_filter(vsi, filter, true); 8873 8874 if (err) { 8875 dev_err(&pf->pdev->dev, "Failed to add cloud filter, err %d\n", 8876 err); 8877 goto err; 8878 } 8879 8880 /* add filter to the ordered list */ 8881 INIT_HLIST_NODE(&filter->cloud_node); 8882 8883 hlist_add_head(&filter->cloud_node, &pf->cloud_filter_list); 8884 8885 pf->num_cloud_filters++; 8886 8887 return err; 8888 err: 8889 kfree(filter); 8890 return err; 8891 } 8892 8893 /** 8894 * i40e_find_cloud_filter - Find the could filter in the list 8895 * @vsi: Pointer to VSI 8896 * @cookie: filter specific cookie 8897 * 8898 **/ 8899 static struct i40e_cloud_filter *i40e_find_cloud_filter(struct i40e_vsi *vsi, 8900 unsigned long *cookie) 8901 { 8902 struct i40e_cloud_filter *filter = NULL; 8903 struct hlist_node *node2; 8904 8905 hlist_for_each_entry_safe(filter, node2, 8906 &vsi->back->cloud_filter_list, cloud_node) 8907 if (!memcmp(cookie, &filter->cookie, sizeof(filter->cookie))) 8908 return filter; 8909 return NULL; 8910 } 8911 8912 /** 8913 * i40e_delete_clsflower - Remove tc flower filters 8914 * @vsi: Pointer to VSI 8915 * @cls_flower: Pointer to struct flow_cls_offload 8916 * 8917 **/ 8918 static int i40e_delete_clsflower(struct i40e_vsi *vsi, 8919 struct flow_cls_offload *cls_flower) 8920 { 8921 struct i40e_cloud_filter *filter = NULL; 8922 struct i40e_pf *pf = vsi->back; 8923 int err = 0; 8924 8925 filter = i40e_find_cloud_filter(vsi, &cls_flower->cookie); 8926 8927 if (!filter) 8928 return -EINVAL; 8929 8930 hash_del(&filter->cloud_node); 8931 8932 if (filter->dst_port) 8933 err = i40e_add_del_cloud_filter_big_buf(vsi, filter, false); 8934 else 8935 err = i40e_add_del_cloud_filter(vsi, filter, false); 8936 8937 kfree(filter); 8938 if (err) { 8939 dev_err(&pf->pdev->dev, 8940 "Failed to delete cloud filter, err %pe\n", 8941 ERR_PTR(err)); 8942 return i40e_aq_rc_to_posix(err, pf->hw.aq.asq_last_status); 8943 } 8944 8945 pf->num_cloud_filters--; 8946 if (!pf->num_cloud_filters) 8947 if (test_bit(I40E_FLAG_FD_SB_TO_CLOUD_FILTER, pf->flags) && 8948 !test_bit(I40E_FLAG_FD_SB_INACTIVE, pf->flags)) { 8949 set_bit(I40E_FLAG_FD_SB_ENA, pf->flags); 8950 clear_bit(I40E_FLAG_FD_SB_TO_CLOUD_FILTER, pf->flags); 8951 clear_bit(I40E_FLAG_FD_SB_INACTIVE, pf->flags); 8952 } 8953 return 0; 8954 } 8955 8956 /** 8957 * i40e_setup_tc_cls_flower - flower classifier offloads 8958 * @np: net device to configure 8959 * @cls_flower: offload data 8960 **/ 8961 static int i40e_setup_tc_cls_flower(struct i40e_netdev_priv *np, 8962 struct flow_cls_offload *cls_flower) 8963 { 8964 struct i40e_vsi *vsi = np->vsi; 8965 8966 switch (cls_flower->command) { 8967 case FLOW_CLS_REPLACE: 8968 return i40e_configure_clsflower(vsi, cls_flower); 8969 case FLOW_CLS_DESTROY: 8970 return i40e_delete_clsflower(vsi, cls_flower); 8971 case FLOW_CLS_STATS: 8972 return -EOPNOTSUPP; 8973 default: 8974 return -EOPNOTSUPP; 8975 } 8976 } 8977 8978 static int i40e_setup_tc_block_cb(enum tc_setup_type type, void *type_data, 8979 void *cb_priv) 8980 { 8981 struct i40e_netdev_priv *np = cb_priv; 8982 8983 if (!tc_cls_can_offload_and_chain0(np->vsi->netdev, type_data)) 8984 return -EOPNOTSUPP; 8985 8986 switch (type) { 8987 case TC_SETUP_CLSFLOWER: 8988 return i40e_setup_tc_cls_flower(np, type_data); 8989 8990 default: 8991 return -EOPNOTSUPP; 8992 } 8993 } 8994 8995 static LIST_HEAD(i40e_block_cb_list); 8996 8997 static int __i40e_setup_tc(struct net_device *netdev, enum tc_setup_type type, 8998 void *type_data) 8999 { 9000 struct i40e_netdev_priv *np = netdev_priv(netdev); 9001 9002 switch (type) { 9003 case TC_SETUP_QDISC_MQPRIO: 9004 return i40e_setup_tc(netdev, type_data); 9005 case TC_SETUP_BLOCK: 9006 return flow_block_cb_setup_simple(type_data, 9007 &i40e_block_cb_list, 9008 i40e_setup_tc_block_cb, 9009 np, np, true); 9010 default: 9011 return -EOPNOTSUPP; 9012 } 9013 } 9014 9015 /** 9016 * i40e_open - Called when a network interface is made active 9017 * @netdev: network interface device structure 9018 * 9019 * The open entry point is called when a network interface is made 9020 * active by the system (IFF_UP). At this point all resources needed 9021 * for transmit and receive operations are allocated, the interrupt 9022 * handler is registered with the OS, the netdev watchdog subtask is 9023 * enabled, and the stack is notified that the interface is ready. 9024 * 9025 * Returns 0 on success, negative value on failure 9026 **/ 9027 int i40e_open(struct net_device *netdev) 9028 { 9029 struct i40e_netdev_priv *np = netdev_priv(netdev); 9030 struct i40e_vsi *vsi = np->vsi; 9031 struct i40e_pf *pf = vsi->back; 9032 int err; 9033 9034 /* disallow open during test or if eeprom is broken */ 9035 if (test_bit(__I40E_TESTING, pf->state) || 9036 test_bit(__I40E_BAD_EEPROM, pf->state)) 9037 return -EBUSY; 9038 9039 netif_carrier_off(netdev); 9040 9041 if (i40e_force_link_state(pf, true)) 9042 return -EAGAIN; 9043 9044 err = i40e_vsi_open(vsi); 9045 if (err) 9046 return err; 9047 9048 /* configure global TSO hardware offload settings */ 9049 wr32(&pf->hw, I40E_GLLAN_TSOMSK_F, be32_to_cpu(TCP_FLAG_PSH | 9050 TCP_FLAG_FIN) >> 16); 9051 wr32(&pf->hw, I40E_GLLAN_TSOMSK_M, be32_to_cpu(TCP_FLAG_PSH | 9052 TCP_FLAG_FIN | 9053 TCP_FLAG_CWR) >> 16); 9054 wr32(&pf->hw, I40E_GLLAN_TSOMSK_L, be32_to_cpu(TCP_FLAG_CWR) >> 16); 9055 udp_tunnel_get_rx_info(netdev); 9056 9057 return 0; 9058 } 9059 9060 /** 9061 * i40e_netif_set_realnum_tx_rx_queues - Update number of tx/rx queues 9062 * @vsi: vsi structure 9063 * 9064 * This updates netdev's number of tx/rx queues 9065 * 9066 * Returns status of setting tx/rx queues 9067 **/ 9068 static int i40e_netif_set_realnum_tx_rx_queues(struct i40e_vsi *vsi) 9069 { 9070 int ret; 9071 9072 ret = netif_set_real_num_rx_queues(vsi->netdev, 9073 vsi->num_queue_pairs); 9074 if (ret) 9075 return ret; 9076 9077 return netif_set_real_num_tx_queues(vsi->netdev, 9078 vsi->num_queue_pairs); 9079 } 9080 9081 /** 9082 * i40e_vsi_open - 9083 * @vsi: the VSI to open 9084 * 9085 * Finish initialization of the VSI. 9086 * 9087 * Returns 0 on success, negative value on failure 9088 * 9089 * Note: expects to be called while under rtnl_lock() 9090 **/ 9091 int i40e_vsi_open(struct i40e_vsi *vsi) 9092 { 9093 struct i40e_pf *pf = vsi->back; 9094 char int_name[I40E_INT_NAME_STR_LEN]; 9095 int err; 9096 9097 /* allocate descriptors */ 9098 err = i40e_vsi_setup_tx_resources(vsi); 9099 if (err) 9100 goto err_setup_tx; 9101 err = i40e_vsi_setup_rx_resources(vsi); 9102 if (err) 9103 goto err_setup_rx; 9104 9105 err = i40e_vsi_configure(vsi); 9106 if (err) 9107 goto err_setup_rx; 9108 9109 if (vsi->netdev) { 9110 snprintf(int_name, sizeof(int_name) - 1, "%s-%s", 9111 dev_driver_string(&pf->pdev->dev), vsi->netdev->name); 9112 err = i40e_vsi_request_irq(vsi, int_name); 9113 if (err) 9114 goto err_setup_rx; 9115 9116 /* Notify the stack of the actual queue counts. */ 9117 err = i40e_netif_set_realnum_tx_rx_queues(vsi); 9118 if (err) 9119 goto err_set_queues; 9120 9121 } else if (vsi->type == I40E_VSI_FDIR) { 9122 snprintf(int_name, sizeof(int_name) - 1, "%s-%s:fdir", 9123 dev_driver_string(&pf->pdev->dev), 9124 dev_name(&pf->pdev->dev)); 9125 err = i40e_vsi_request_irq(vsi, int_name); 9126 if (err) 9127 goto err_setup_rx; 9128 9129 } else { 9130 err = -EINVAL; 9131 goto err_setup_rx; 9132 } 9133 9134 err = i40e_up_complete(vsi); 9135 if (err) 9136 goto err_up_complete; 9137 9138 return 0; 9139 9140 err_up_complete: 9141 i40e_down(vsi); 9142 err_set_queues: 9143 i40e_vsi_free_irq(vsi); 9144 err_setup_rx: 9145 i40e_vsi_free_rx_resources(vsi); 9146 err_setup_tx: 9147 i40e_vsi_free_tx_resources(vsi); 9148 if (vsi->type == I40E_VSI_MAIN) 9149 i40e_do_reset(pf, I40E_PF_RESET_FLAG, true); 9150 9151 return err; 9152 } 9153 9154 /** 9155 * i40e_fdir_filter_exit - Cleans up the Flow Director accounting 9156 * @pf: Pointer to PF 9157 * 9158 * This function destroys the hlist where all the Flow Director 9159 * filters were saved. 9160 **/ 9161 static void i40e_fdir_filter_exit(struct i40e_pf *pf) 9162 { 9163 struct i40e_fdir_filter *filter; 9164 struct i40e_flex_pit *pit_entry, *tmp; 9165 struct hlist_node *node2; 9166 9167 hlist_for_each_entry_safe(filter, node2, 9168 &pf->fdir_filter_list, fdir_node) { 9169 hlist_del(&filter->fdir_node); 9170 kfree(filter); 9171 } 9172 9173 list_for_each_entry_safe(pit_entry, tmp, &pf->l3_flex_pit_list, list) { 9174 list_del(&pit_entry->list); 9175 kfree(pit_entry); 9176 } 9177 INIT_LIST_HEAD(&pf->l3_flex_pit_list); 9178 9179 list_for_each_entry_safe(pit_entry, tmp, &pf->l4_flex_pit_list, list) { 9180 list_del(&pit_entry->list); 9181 kfree(pit_entry); 9182 } 9183 INIT_LIST_HEAD(&pf->l4_flex_pit_list); 9184 9185 pf->fdir_pf_active_filters = 0; 9186 i40e_reset_fdir_filter_cnt(pf); 9187 9188 /* Reprogram the default input set for TCP/IPv4 */ 9189 i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV4_TCP, 9190 I40E_L3_SRC_MASK | I40E_L3_DST_MASK | 9191 I40E_L4_SRC_MASK | I40E_L4_DST_MASK); 9192 9193 /* Reprogram the default input set for TCP/IPv6 */ 9194 i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV6_TCP, 9195 I40E_L3_V6_SRC_MASK | I40E_L3_V6_DST_MASK | 9196 I40E_L4_SRC_MASK | I40E_L4_DST_MASK); 9197 9198 /* Reprogram the default input set for UDP/IPv4 */ 9199 i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV4_UDP, 9200 I40E_L3_SRC_MASK | I40E_L3_DST_MASK | 9201 I40E_L4_SRC_MASK | I40E_L4_DST_MASK); 9202 9203 /* Reprogram the default input set for UDP/IPv6 */ 9204 i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV6_UDP, 9205 I40E_L3_V6_SRC_MASK | I40E_L3_V6_DST_MASK | 9206 I40E_L4_SRC_MASK | I40E_L4_DST_MASK); 9207 9208 /* Reprogram the default input set for SCTP/IPv4 */ 9209 i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV4_SCTP, 9210 I40E_L3_SRC_MASK | I40E_L3_DST_MASK | 9211 I40E_L4_SRC_MASK | I40E_L4_DST_MASK); 9212 9213 /* Reprogram the default input set for SCTP/IPv6 */ 9214 i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV6_SCTP, 9215 I40E_L3_V6_SRC_MASK | I40E_L3_V6_DST_MASK | 9216 I40E_L4_SRC_MASK | I40E_L4_DST_MASK); 9217 9218 /* Reprogram the default input set for Other/IPv4 */ 9219 i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV4_OTHER, 9220 I40E_L3_SRC_MASK | I40E_L3_DST_MASK); 9221 9222 i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_FRAG_IPV4, 9223 I40E_L3_SRC_MASK | I40E_L3_DST_MASK); 9224 9225 /* Reprogram the default input set for Other/IPv6 */ 9226 i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV6_OTHER, 9227 I40E_L3_SRC_MASK | I40E_L3_DST_MASK); 9228 9229 i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_FRAG_IPV6, 9230 I40E_L3_SRC_MASK | I40E_L3_DST_MASK); 9231 } 9232 9233 /** 9234 * i40e_cloud_filter_exit - Cleans up the cloud filters 9235 * @pf: Pointer to PF 9236 * 9237 * This function destroys the hlist where all the cloud filters 9238 * were saved. 9239 **/ 9240 static void i40e_cloud_filter_exit(struct i40e_pf *pf) 9241 { 9242 struct i40e_cloud_filter *cfilter; 9243 struct hlist_node *node; 9244 9245 hlist_for_each_entry_safe(cfilter, node, 9246 &pf->cloud_filter_list, cloud_node) { 9247 hlist_del(&cfilter->cloud_node); 9248 kfree(cfilter); 9249 } 9250 pf->num_cloud_filters = 0; 9251 9252 if (test_bit(I40E_FLAG_FD_SB_TO_CLOUD_FILTER, pf->flags) && 9253 !test_bit(I40E_FLAG_FD_SB_INACTIVE, pf->flags)) { 9254 set_bit(I40E_FLAG_FD_SB_ENA, pf->flags); 9255 clear_bit(I40E_FLAG_FD_SB_TO_CLOUD_FILTER, pf->flags); 9256 clear_bit(I40E_FLAG_FD_SB_INACTIVE, pf->flags); 9257 } 9258 } 9259 9260 /** 9261 * i40e_close - Disables a network interface 9262 * @netdev: network interface device structure 9263 * 9264 * The close entry point is called when an interface is de-activated 9265 * by the OS. The hardware is still under the driver's control, but 9266 * this netdev interface is disabled. 9267 * 9268 * Returns 0, this is not allowed to fail 9269 **/ 9270 int i40e_close(struct net_device *netdev) 9271 { 9272 struct i40e_netdev_priv *np = netdev_priv(netdev); 9273 struct i40e_vsi *vsi = np->vsi; 9274 9275 i40e_vsi_close(vsi); 9276 9277 return 0; 9278 } 9279 9280 /** 9281 * i40e_do_reset - Start a PF or Core Reset sequence 9282 * @pf: board private structure 9283 * @reset_flags: which reset is requested 9284 * @lock_acquired: indicates whether or not the lock has been acquired 9285 * before this function was called. 9286 * 9287 * The essential difference in resets is that the PF Reset 9288 * doesn't clear the packet buffers, doesn't reset the PE 9289 * firmware, and doesn't bother the other PFs on the chip. 9290 **/ 9291 void i40e_do_reset(struct i40e_pf *pf, u32 reset_flags, bool lock_acquired) 9292 { 9293 struct i40e_vsi *vsi; 9294 u32 val; 9295 int i; 9296 9297 /* do the biggest reset indicated */ 9298 if (reset_flags & BIT_ULL(__I40E_GLOBAL_RESET_REQUESTED)) { 9299 9300 /* Request a Global Reset 9301 * 9302 * This will start the chip's countdown to the actual full 9303 * chip reset event, and a warning interrupt to be sent 9304 * to all PFs, including the requestor. Our handler 9305 * for the warning interrupt will deal with the shutdown 9306 * and recovery of the switch setup. 9307 */ 9308 dev_dbg(&pf->pdev->dev, "GlobalR requested\n"); 9309 val = rd32(&pf->hw, I40E_GLGEN_RTRIG); 9310 val |= I40E_GLGEN_RTRIG_GLOBR_MASK; 9311 wr32(&pf->hw, I40E_GLGEN_RTRIG, val); 9312 9313 } else if (reset_flags & BIT_ULL(__I40E_CORE_RESET_REQUESTED)) { 9314 9315 /* Request a Core Reset 9316 * 9317 * Same as Global Reset, except does *not* include the MAC/PHY 9318 */ 9319 dev_dbg(&pf->pdev->dev, "CoreR requested\n"); 9320 val = rd32(&pf->hw, I40E_GLGEN_RTRIG); 9321 val |= I40E_GLGEN_RTRIG_CORER_MASK; 9322 wr32(&pf->hw, I40E_GLGEN_RTRIG, val); 9323 i40e_flush(&pf->hw); 9324 9325 } else if (reset_flags & I40E_PF_RESET_FLAG) { 9326 9327 /* Request a PF Reset 9328 * 9329 * Resets only the PF-specific registers 9330 * 9331 * This goes directly to the tear-down and rebuild of 9332 * the switch, since we need to do all the recovery as 9333 * for the Core Reset. 9334 */ 9335 dev_dbg(&pf->pdev->dev, "PFR requested\n"); 9336 i40e_handle_reset_warning(pf, lock_acquired); 9337 9338 } else if (reset_flags & I40E_PF_RESET_AND_REBUILD_FLAG) { 9339 /* Request a PF Reset 9340 * 9341 * Resets PF and reinitializes PFs VSI. 9342 */ 9343 i40e_prep_for_reset(pf); 9344 i40e_reset_and_rebuild(pf, true, lock_acquired); 9345 dev_info(&pf->pdev->dev, 9346 test_bit(I40E_FLAG_FW_LLDP_DIS, pf->flags) ? 9347 "FW LLDP is disabled\n" : 9348 "FW LLDP is enabled\n"); 9349 9350 } else if (reset_flags & BIT_ULL(__I40E_REINIT_REQUESTED)) { 9351 /* Find the VSI(s) that requested a re-init */ 9352 dev_info(&pf->pdev->dev, "VSI reinit requested\n"); 9353 9354 i40e_pf_for_each_vsi(pf, i, vsi) { 9355 if (test_and_clear_bit(__I40E_VSI_REINIT_REQUESTED, 9356 vsi->state)) 9357 i40e_vsi_reinit_locked(vsi); 9358 } 9359 } else if (reset_flags & BIT_ULL(__I40E_DOWN_REQUESTED)) { 9360 /* Find the VSI(s) that needs to be brought down */ 9361 dev_info(&pf->pdev->dev, "VSI down requested\n"); 9362 9363 i40e_pf_for_each_vsi(pf, i, vsi) { 9364 if (test_and_clear_bit(__I40E_VSI_DOWN_REQUESTED, 9365 vsi->state)) { 9366 set_bit(__I40E_VSI_DOWN, vsi->state); 9367 i40e_down(vsi); 9368 } 9369 } 9370 } else { 9371 dev_info(&pf->pdev->dev, 9372 "bad reset request 0x%08x\n", reset_flags); 9373 } 9374 } 9375 9376 #ifdef CONFIG_I40E_DCB 9377 /** 9378 * i40e_dcb_need_reconfig - Check if DCB needs reconfig 9379 * @pf: board private structure 9380 * @old_cfg: current DCB config 9381 * @new_cfg: new DCB config 9382 **/ 9383 bool i40e_dcb_need_reconfig(struct i40e_pf *pf, 9384 struct i40e_dcbx_config *old_cfg, 9385 struct i40e_dcbx_config *new_cfg) 9386 { 9387 bool need_reconfig = false; 9388 9389 /* Check if ETS configuration has changed */ 9390 if (memcmp(&new_cfg->etscfg, 9391 &old_cfg->etscfg, 9392 sizeof(new_cfg->etscfg))) { 9393 /* If Priority Table has changed reconfig is needed */ 9394 if (memcmp(&new_cfg->etscfg.prioritytable, 9395 &old_cfg->etscfg.prioritytable, 9396 sizeof(new_cfg->etscfg.prioritytable))) { 9397 need_reconfig = true; 9398 dev_dbg(&pf->pdev->dev, "ETS UP2TC changed.\n"); 9399 } 9400 9401 if (memcmp(&new_cfg->etscfg.tcbwtable, 9402 &old_cfg->etscfg.tcbwtable, 9403 sizeof(new_cfg->etscfg.tcbwtable))) 9404 dev_dbg(&pf->pdev->dev, "ETS TC BW Table changed.\n"); 9405 9406 if (memcmp(&new_cfg->etscfg.tsatable, 9407 &old_cfg->etscfg.tsatable, 9408 sizeof(new_cfg->etscfg.tsatable))) 9409 dev_dbg(&pf->pdev->dev, "ETS TSA Table changed.\n"); 9410 } 9411 9412 /* Check if PFC configuration has changed */ 9413 if (memcmp(&new_cfg->pfc, 9414 &old_cfg->pfc, 9415 sizeof(new_cfg->pfc))) { 9416 need_reconfig = true; 9417 dev_dbg(&pf->pdev->dev, "PFC config change detected.\n"); 9418 } 9419 9420 /* Check if APP Table has changed */ 9421 if (memcmp(&new_cfg->app, 9422 &old_cfg->app, 9423 sizeof(new_cfg->app))) { 9424 need_reconfig = true; 9425 dev_dbg(&pf->pdev->dev, "APP Table change detected.\n"); 9426 } 9427 9428 dev_dbg(&pf->pdev->dev, "dcb need_reconfig=%d\n", need_reconfig); 9429 return need_reconfig; 9430 } 9431 9432 /** 9433 * i40e_handle_lldp_event - Handle LLDP Change MIB event 9434 * @pf: board private structure 9435 * @e: event info posted on ARQ 9436 **/ 9437 static int i40e_handle_lldp_event(struct i40e_pf *pf, 9438 struct i40e_arq_event_info *e) 9439 { 9440 struct i40e_aqc_lldp_get_mib *mib = 9441 (struct i40e_aqc_lldp_get_mib *)&e->desc.params.raw; 9442 struct i40e_hw *hw = &pf->hw; 9443 struct i40e_dcbx_config tmp_dcbx_cfg; 9444 bool need_reconfig = false; 9445 int ret = 0; 9446 u8 type; 9447 9448 /* X710-T*L 2.5G and 5G speeds don't support DCB */ 9449 if (I40E_IS_X710TL_DEVICE(hw->device_id) && 9450 (hw->phy.link_info.link_speed & 9451 ~(I40E_LINK_SPEED_2_5GB | I40E_LINK_SPEED_5GB)) && 9452 !test_bit(I40E_FLAG_DCB_CAPABLE, pf->flags)) 9453 /* let firmware decide if the DCB should be disabled */ 9454 set_bit(I40E_FLAG_DCB_CAPABLE, pf->flags); 9455 9456 /* Not DCB capable or capability disabled */ 9457 if (!test_bit(I40E_FLAG_DCB_CAPABLE, pf->flags)) 9458 return ret; 9459 9460 /* Ignore if event is not for Nearest Bridge */ 9461 type = ((mib->type >> I40E_AQ_LLDP_BRIDGE_TYPE_SHIFT) 9462 & I40E_AQ_LLDP_BRIDGE_TYPE_MASK); 9463 dev_dbg(&pf->pdev->dev, "LLDP event mib bridge type 0x%x\n", type); 9464 if (type != I40E_AQ_LLDP_BRIDGE_TYPE_NEAREST_BRIDGE) 9465 return ret; 9466 9467 /* Check MIB Type and return if event for Remote MIB update */ 9468 type = mib->type & I40E_AQ_LLDP_MIB_TYPE_MASK; 9469 dev_dbg(&pf->pdev->dev, 9470 "LLDP event mib type %s\n", type ? "remote" : "local"); 9471 if (type == I40E_AQ_LLDP_MIB_REMOTE) { 9472 /* Update the remote cached instance and return */ 9473 ret = i40e_aq_get_dcb_config(hw, I40E_AQ_LLDP_MIB_REMOTE, 9474 I40E_AQ_LLDP_BRIDGE_TYPE_NEAREST_BRIDGE, 9475 &hw->remote_dcbx_config); 9476 goto exit; 9477 } 9478 9479 /* Store the old configuration */ 9480 tmp_dcbx_cfg = hw->local_dcbx_config; 9481 9482 /* Reset the old DCBx configuration data */ 9483 memset(&hw->local_dcbx_config, 0, sizeof(hw->local_dcbx_config)); 9484 /* Get updated DCBX data from firmware */ 9485 ret = i40e_get_dcb_config(&pf->hw); 9486 if (ret) { 9487 /* X710-T*L 2.5G and 5G speeds don't support DCB */ 9488 if (I40E_IS_X710TL_DEVICE(hw->device_id) && 9489 (hw->phy.link_info.link_speed & 9490 (I40E_LINK_SPEED_2_5GB | I40E_LINK_SPEED_5GB))) { 9491 dev_warn(&pf->pdev->dev, 9492 "DCB is not supported for X710-T*L 2.5/5G speeds\n"); 9493 clear_bit(I40E_FLAG_DCB_CAPABLE, pf->flags); 9494 } else { 9495 dev_info(&pf->pdev->dev, 9496 "Failed querying DCB configuration data from firmware, err %pe aq_err %s\n", 9497 ERR_PTR(ret), 9498 i40e_aq_str(&pf->hw, 9499 pf->hw.aq.asq_last_status)); 9500 } 9501 goto exit; 9502 } 9503 9504 /* No change detected in DCBX configs */ 9505 if (!memcmp(&tmp_dcbx_cfg, &hw->local_dcbx_config, 9506 sizeof(tmp_dcbx_cfg))) { 9507 dev_dbg(&pf->pdev->dev, "No change detected in DCBX configuration.\n"); 9508 goto exit; 9509 } 9510 9511 need_reconfig = i40e_dcb_need_reconfig(pf, &tmp_dcbx_cfg, 9512 &hw->local_dcbx_config); 9513 9514 i40e_dcbnl_flush_apps(pf, &tmp_dcbx_cfg, &hw->local_dcbx_config); 9515 9516 if (!need_reconfig) 9517 goto exit; 9518 9519 /* Enable DCB tagging only when more than one TC */ 9520 if (i40e_dcb_get_num_tc(&hw->local_dcbx_config) > 1) 9521 set_bit(I40E_FLAG_DCB_ENA, pf->flags); 9522 else 9523 clear_bit(I40E_FLAG_DCB_ENA, pf->flags); 9524 9525 set_bit(__I40E_PORT_SUSPENDED, pf->state); 9526 /* Reconfiguration needed quiesce all VSIs */ 9527 i40e_pf_quiesce_all_vsi(pf); 9528 9529 /* Changes in configuration update VEB/VSI */ 9530 i40e_dcb_reconfigure(pf); 9531 9532 ret = i40e_resume_port_tx(pf); 9533 9534 clear_bit(__I40E_PORT_SUSPENDED, pf->state); 9535 /* In case of error no point in resuming VSIs */ 9536 if (ret) 9537 goto exit; 9538 9539 /* Wait for the PF's queues to be disabled */ 9540 ret = i40e_pf_wait_queues_disabled(pf); 9541 if (ret) { 9542 /* Schedule PF reset to recover */ 9543 set_bit(__I40E_PF_RESET_REQUESTED, pf->state); 9544 i40e_service_event_schedule(pf); 9545 } else { 9546 i40e_pf_unquiesce_all_vsi(pf); 9547 set_bit(__I40E_CLIENT_SERVICE_REQUESTED, pf->state); 9548 set_bit(__I40E_CLIENT_L2_CHANGE, pf->state); 9549 } 9550 9551 exit: 9552 return ret; 9553 } 9554 #endif /* CONFIG_I40E_DCB */ 9555 9556 /** 9557 * i40e_do_reset_safe - Protected reset path for userland calls. 9558 * @pf: board private structure 9559 * @reset_flags: which reset is requested 9560 * 9561 **/ 9562 void i40e_do_reset_safe(struct i40e_pf *pf, u32 reset_flags) 9563 { 9564 rtnl_lock(); 9565 i40e_do_reset(pf, reset_flags, true); 9566 rtnl_unlock(); 9567 } 9568 9569 /** 9570 * i40e_handle_lan_overflow_event - Handler for LAN queue overflow event 9571 * @pf: board private structure 9572 * @e: event info posted on ARQ 9573 * 9574 * Handler for LAN Queue Overflow Event generated by the firmware for PF 9575 * and VF queues 9576 **/ 9577 static void i40e_handle_lan_overflow_event(struct i40e_pf *pf, 9578 struct i40e_arq_event_info *e) 9579 { 9580 struct i40e_aqc_lan_overflow *data = 9581 (struct i40e_aqc_lan_overflow *)&e->desc.params.raw; 9582 u32 queue = le32_to_cpu(data->prtdcb_rupto); 9583 u32 qtx_ctl = le32_to_cpu(data->otx_ctl); 9584 struct i40e_hw *hw = &pf->hw; 9585 struct i40e_vf *vf; 9586 u16 vf_id; 9587 9588 dev_dbg(&pf->pdev->dev, "overflow Rx Queue Number = %d QTX_CTL=0x%08x\n", 9589 queue, qtx_ctl); 9590 9591 if (FIELD_GET(I40E_QTX_CTL_PFVF_Q_MASK, qtx_ctl) != 9592 I40E_QTX_CTL_VF_QUEUE) 9593 return; 9594 9595 /* Queue belongs to VF, find the VF and issue VF reset */ 9596 vf_id = FIELD_GET(I40E_QTX_CTL_VFVM_INDX_MASK, qtx_ctl); 9597 vf_id -= hw->func_caps.vf_base_id; 9598 vf = &pf->vf[vf_id]; 9599 i40e_vc_notify_vf_reset(vf); 9600 /* Allow VF to process pending reset notification */ 9601 msleep(20); 9602 i40e_reset_vf(vf, false); 9603 } 9604 9605 /** 9606 * i40e_get_cur_guaranteed_fd_count - Get the consumed guaranteed FD filters 9607 * @pf: board private structure 9608 **/ 9609 u32 i40e_get_cur_guaranteed_fd_count(struct i40e_pf *pf) 9610 { 9611 u32 val, fcnt_prog; 9612 9613 val = rd32(&pf->hw, I40E_PFQF_FDSTAT); 9614 fcnt_prog = (val & I40E_PFQF_FDSTAT_GUARANT_CNT_MASK); 9615 return fcnt_prog; 9616 } 9617 9618 /** 9619 * i40e_get_current_fd_count - Get total FD filters programmed for this PF 9620 * @pf: board private structure 9621 **/ 9622 u32 i40e_get_current_fd_count(struct i40e_pf *pf) 9623 { 9624 u32 val, fcnt_prog; 9625 9626 val = rd32(&pf->hw, I40E_PFQF_FDSTAT); 9627 fcnt_prog = (val & I40E_PFQF_FDSTAT_GUARANT_CNT_MASK) + 9628 FIELD_GET(I40E_PFQF_FDSTAT_BEST_CNT_MASK, val); 9629 return fcnt_prog; 9630 } 9631 9632 /** 9633 * i40e_get_global_fd_count - Get total FD filters programmed on device 9634 * @pf: board private structure 9635 **/ 9636 u32 i40e_get_global_fd_count(struct i40e_pf *pf) 9637 { 9638 u32 val, fcnt_prog; 9639 9640 val = rd32(&pf->hw, I40E_GLQF_FDCNT_0); 9641 fcnt_prog = (val & I40E_GLQF_FDCNT_0_GUARANT_CNT_MASK) + 9642 FIELD_GET(I40E_GLQF_FDCNT_0_BESTCNT_MASK, val); 9643 return fcnt_prog; 9644 } 9645 9646 /** 9647 * i40e_reenable_fdir_sb - Restore FDir SB capability 9648 * @pf: board private structure 9649 **/ 9650 static void i40e_reenable_fdir_sb(struct i40e_pf *pf) 9651 { 9652 if (test_and_clear_bit(__I40E_FD_SB_AUTO_DISABLED, pf->state)) 9653 if (test_bit(I40E_FLAG_FD_SB_ENA, pf->flags) && 9654 (I40E_DEBUG_FD & pf->hw.debug_mask)) 9655 dev_info(&pf->pdev->dev, "FD Sideband/ntuple is being enabled since we have space in the table now\n"); 9656 } 9657 9658 /** 9659 * i40e_reenable_fdir_atr - Restore FDir ATR capability 9660 * @pf: board private structure 9661 **/ 9662 static void i40e_reenable_fdir_atr(struct i40e_pf *pf) 9663 { 9664 if (test_and_clear_bit(__I40E_FD_ATR_AUTO_DISABLED, pf->state)) { 9665 /* ATR uses the same filtering logic as SB rules. It only 9666 * functions properly if the input set mask is at the default 9667 * settings. It is safe to restore the default input set 9668 * because there are no active TCPv4 filter rules. 9669 */ 9670 i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV4_TCP, 9671 I40E_L3_SRC_MASK | I40E_L3_DST_MASK | 9672 I40E_L4_SRC_MASK | I40E_L4_DST_MASK); 9673 9674 if (test_bit(I40E_FLAG_FD_ATR_ENA, pf->flags) && 9675 (I40E_DEBUG_FD & pf->hw.debug_mask)) 9676 dev_info(&pf->pdev->dev, "ATR is being enabled since we have space in the table and there are no conflicting ntuple rules\n"); 9677 } 9678 } 9679 9680 /** 9681 * i40e_delete_invalid_filter - Delete an invalid FDIR filter 9682 * @pf: board private structure 9683 * @filter: FDir filter to remove 9684 */ 9685 static void i40e_delete_invalid_filter(struct i40e_pf *pf, 9686 struct i40e_fdir_filter *filter) 9687 { 9688 /* Update counters */ 9689 pf->fdir_pf_active_filters--; 9690 pf->fd_inv = 0; 9691 9692 switch (filter->flow_type) { 9693 case TCP_V4_FLOW: 9694 pf->fd_tcp4_filter_cnt--; 9695 break; 9696 case UDP_V4_FLOW: 9697 pf->fd_udp4_filter_cnt--; 9698 break; 9699 case SCTP_V4_FLOW: 9700 pf->fd_sctp4_filter_cnt--; 9701 break; 9702 case TCP_V6_FLOW: 9703 pf->fd_tcp6_filter_cnt--; 9704 break; 9705 case UDP_V6_FLOW: 9706 pf->fd_udp6_filter_cnt--; 9707 break; 9708 case SCTP_V6_FLOW: 9709 pf->fd_udp6_filter_cnt--; 9710 break; 9711 case IP_USER_FLOW: 9712 switch (filter->ipl4_proto) { 9713 case IPPROTO_TCP: 9714 pf->fd_tcp4_filter_cnt--; 9715 break; 9716 case IPPROTO_UDP: 9717 pf->fd_udp4_filter_cnt--; 9718 break; 9719 case IPPROTO_SCTP: 9720 pf->fd_sctp4_filter_cnt--; 9721 break; 9722 case IPPROTO_IP: 9723 pf->fd_ip4_filter_cnt--; 9724 break; 9725 } 9726 break; 9727 case IPV6_USER_FLOW: 9728 switch (filter->ipl4_proto) { 9729 case IPPROTO_TCP: 9730 pf->fd_tcp6_filter_cnt--; 9731 break; 9732 case IPPROTO_UDP: 9733 pf->fd_udp6_filter_cnt--; 9734 break; 9735 case IPPROTO_SCTP: 9736 pf->fd_sctp6_filter_cnt--; 9737 break; 9738 case IPPROTO_IP: 9739 pf->fd_ip6_filter_cnt--; 9740 break; 9741 } 9742 break; 9743 } 9744 9745 /* Remove the filter from the list and free memory */ 9746 hlist_del(&filter->fdir_node); 9747 kfree(filter); 9748 } 9749 9750 /** 9751 * i40e_fdir_check_and_reenable - Function to reenabe FD ATR or SB if disabled 9752 * @pf: board private structure 9753 **/ 9754 void i40e_fdir_check_and_reenable(struct i40e_pf *pf) 9755 { 9756 struct i40e_fdir_filter *filter; 9757 u32 fcnt_prog, fcnt_avail; 9758 struct hlist_node *node; 9759 9760 if (test_bit(__I40E_FD_FLUSH_REQUESTED, pf->state)) 9761 return; 9762 9763 /* Check if we have enough room to re-enable FDir SB capability. */ 9764 fcnt_prog = i40e_get_global_fd_count(pf); 9765 fcnt_avail = pf->fdir_pf_filter_count; 9766 if ((fcnt_prog < (fcnt_avail - I40E_FDIR_BUFFER_HEAD_ROOM)) || 9767 (pf->fd_add_err == 0) || 9768 (i40e_get_current_atr_cnt(pf) < pf->fd_atr_cnt)) 9769 i40e_reenable_fdir_sb(pf); 9770 9771 /* We should wait for even more space before re-enabling ATR. 9772 * Additionally, we cannot enable ATR as long as we still have TCP SB 9773 * rules active. 9774 */ 9775 if ((fcnt_prog < (fcnt_avail - I40E_FDIR_BUFFER_HEAD_ROOM_FOR_ATR)) && 9776 pf->fd_tcp4_filter_cnt == 0 && pf->fd_tcp6_filter_cnt == 0) 9777 i40e_reenable_fdir_atr(pf); 9778 9779 /* if hw had a problem adding a filter, delete it */ 9780 if (pf->fd_inv > 0) { 9781 hlist_for_each_entry_safe(filter, node, 9782 &pf->fdir_filter_list, fdir_node) 9783 if (filter->fd_id == pf->fd_inv) 9784 i40e_delete_invalid_filter(pf, filter); 9785 } 9786 } 9787 9788 #define I40E_MIN_FD_FLUSH_INTERVAL 10 9789 #define I40E_MIN_FD_FLUSH_SB_ATR_UNSTABLE 30 9790 /** 9791 * i40e_fdir_flush_and_replay - Function to flush all FD filters and replay SB 9792 * @pf: board private structure 9793 **/ 9794 static void i40e_fdir_flush_and_replay(struct i40e_pf *pf) 9795 { 9796 unsigned long min_flush_time; 9797 int flush_wait_retry = 50; 9798 bool disable_atr = false; 9799 int fd_room; 9800 int reg; 9801 9802 if (!time_after(jiffies, pf->fd_flush_timestamp + 9803 (I40E_MIN_FD_FLUSH_INTERVAL * HZ))) 9804 return; 9805 9806 /* If the flush is happening too quick and we have mostly SB rules we 9807 * should not re-enable ATR for some time. 9808 */ 9809 min_flush_time = pf->fd_flush_timestamp + 9810 (I40E_MIN_FD_FLUSH_SB_ATR_UNSTABLE * HZ); 9811 fd_room = pf->fdir_pf_filter_count - pf->fdir_pf_active_filters; 9812 9813 if (!(time_after(jiffies, min_flush_time)) && 9814 (fd_room < I40E_FDIR_BUFFER_HEAD_ROOM_FOR_ATR)) { 9815 if (I40E_DEBUG_FD & pf->hw.debug_mask) 9816 dev_info(&pf->pdev->dev, "ATR disabled, not enough FD filter space.\n"); 9817 disable_atr = true; 9818 } 9819 9820 pf->fd_flush_timestamp = jiffies; 9821 set_bit(__I40E_FD_ATR_AUTO_DISABLED, pf->state); 9822 /* flush all filters */ 9823 wr32(&pf->hw, I40E_PFQF_CTL_1, 9824 I40E_PFQF_CTL_1_CLEARFDTABLE_MASK); 9825 i40e_flush(&pf->hw); 9826 pf->fd_flush_cnt++; 9827 pf->fd_add_err = 0; 9828 do { 9829 /* Check FD flush status every 5-6msec */ 9830 usleep_range(5000, 6000); 9831 reg = rd32(&pf->hw, I40E_PFQF_CTL_1); 9832 if (!(reg & I40E_PFQF_CTL_1_CLEARFDTABLE_MASK)) 9833 break; 9834 } while (flush_wait_retry--); 9835 if (reg & I40E_PFQF_CTL_1_CLEARFDTABLE_MASK) { 9836 dev_warn(&pf->pdev->dev, "FD table did not flush, needs more time\n"); 9837 } else { 9838 /* replay sideband filters */ 9839 i40e_fdir_filter_restore(i40e_pf_get_main_vsi(pf)); 9840 if (!disable_atr && !pf->fd_tcp4_filter_cnt) 9841 clear_bit(__I40E_FD_ATR_AUTO_DISABLED, pf->state); 9842 clear_bit(__I40E_FD_FLUSH_REQUESTED, pf->state); 9843 if (I40E_DEBUG_FD & pf->hw.debug_mask) 9844 dev_info(&pf->pdev->dev, "FD Filter table flushed and FD-SB replayed.\n"); 9845 } 9846 } 9847 9848 /** 9849 * i40e_get_current_atr_cnt - Get the count of total FD ATR filters programmed 9850 * @pf: board private structure 9851 **/ 9852 u32 i40e_get_current_atr_cnt(struct i40e_pf *pf) 9853 { 9854 return i40e_get_current_fd_count(pf) - pf->fdir_pf_active_filters; 9855 } 9856 9857 /** 9858 * i40e_fdir_reinit_subtask - Worker thread to reinit FDIR filter table 9859 * @pf: board private structure 9860 **/ 9861 static void i40e_fdir_reinit_subtask(struct i40e_pf *pf) 9862 { 9863 9864 /* if interface is down do nothing */ 9865 if (test_bit(__I40E_DOWN, pf->state)) 9866 return; 9867 9868 if (test_bit(__I40E_FD_FLUSH_REQUESTED, pf->state)) 9869 i40e_fdir_flush_and_replay(pf); 9870 9871 i40e_fdir_check_and_reenable(pf); 9872 9873 } 9874 9875 /** 9876 * i40e_vsi_link_event - notify VSI of a link event 9877 * @vsi: vsi to be notified 9878 * @link_up: link up or down 9879 **/ 9880 static void i40e_vsi_link_event(struct i40e_vsi *vsi, bool link_up) 9881 { 9882 if (!vsi || test_bit(__I40E_VSI_DOWN, vsi->state)) 9883 return; 9884 9885 switch (vsi->type) { 9886 case I40E_VSI_MAIN: 9887 if (!vsi->netdev || !vsi->netdev_registered) 9888 break; 9889 9890 if (link_up) { 9891 netif_carrier_on(vsi->netdev); 9892 netif_tx_wake_all_queues(vsi->netdev); 9893 } else { 9894 netif_carrier_off(vsi->netdev); 9895 netif_tx_stop_all_queues(vsi->netdev); 9896 } 9897 break; 9898 9899 case I40E_VSI_SRIOV: 9900 case I40E_VSI_VMDQ2: 9901 case I40E_VSI_CTRL: 9902 case I40E_VSI_IWARP: 9903 case I40E_VSI_MIRROR: 9904 default: 9905 /* there is no notification for other VSIs */ 9906 break; 9907 } 9908 } 9909 9910 /** 9911 * i40e_veb_link_event - notify elements on the veb of a link event 9912 * @veb: veb to be notified 9913 * @link_up: link up or down 9914 **/ 9915 static void i40e_veb_link_event(struct i40e_veb *veb, bool link_up) 9916 { 9917 struct i40e_vsi *vsi; 9918 struct i40e_pf *pf; 9919 int i; 9920 9921 if (!veb || !veb->pf) 9922 return; 9923 pf = veb->pf; 9924 9925 /* Send link event to contained VSIs */ 9926 i40e_pf_for_each_vsi(pf, i, vsi) 9927 if (vsi->uplink_seid == veb->seid) 9928 i40e_vsi_link_event(vsi, link_up); 9929 } 9930 9931 /** 9932 * i40e_link_event - Update netif_carrier status 9933 * @pf: board private structure 9934 **/ 9935 static void i40e_link_event(struct i40e_pf *pf) 9936 { 9937 struct i40e_vsi *vsi = i40e_pf_get_main_vsi(pf); 9938 struct i40e_veb *veb = i40e_pf_get_main_veb(pf); 9939 u8 new_link_speed, old_link_speed; 9940 bool new_link, old_link; 9941 int status; 9942 #ifdef CONFIG_I40E_DCB 9943 int err; 9944 #endif /* CONFIG_I40E_DCB */ 9945 9946 /* set this to force the get_link_status call to refresh state */ 9947 pf->hw.phy.get_link_info = true; 9948 old_link = (pf->hw.phy.link_info_old.link_info & I40E_AQ_LINK_UP); 9949 status = i40e_get_link_status(&pf->hw, &new_link); 9950 9951 /* On success, disable temp link polling */ 9952 if (status == 0) { 9953 clear_bit(__I40E_TEMP_LINK_POLLING, pf->state); 9954 } else { 9955 /* Enable link polling temporarily until i40e_get_link_status 9956 * returns 0 9957 */ 9958 set_bit(__I40E_TEMP_LINK_POLLING, pf->state); 9959 dev_dbg(&pf->pdev->dev, "couldn't get link state, status: %d\n", 9960 status); 9961 return; 9962 } 9963 9964 old_link_speed = pf->hw.phy.link_info_old.link_speed; 9965 new_link_speed = pf->hw.phy.link_info.link_speed; 9966 9967 if (new_link == old_link && 9968 new_link_speed == old_link_speed && 9969 (test_bit(__I40E_VSI_DOWN, vsi->state) || 9970 new_link == netif_carrier_ok(vsi->netdev))) 9971 return; 9972 9973 i40e_print_link_message(vsi, new_link); 9974 9975 /* Notify the base of the switch tree connected to 9976 * the link. Floating VEBs are not notified. 9977 */ 9978 if (veb) 9979 i40e_veb_link_event(veb, new_link); 9980 else 9981 i40e_vsi_link_event(vsi, new_link); 9982 9983 if (pf->vf) 9984 i40e_vc_notify_link_state(pf); 9985 9986 if (test_bit(I40E_FLAG_PTP_ENA, pf->flags)) 9987 i40e_ptp_set_increment(pf); 9988 #ifdef CONFIG_I40E_DCB 9989 if (new_link == old_link) 9990 return; 9991 /* Not SW DCB so firmware will take care of default settings */ 9992 if (pf->dcbx_cap & DCB_CAP_DCBX_LLD_MANAGED) 9993 return; 9994 9995 /* We cover here only link down, as after link up in case of SW DCB 9996 * SW LLDP agent will take care of setting it up 9997 */ 9998 if (!new_link) { 9999 dev_dbg(&pf->pdev->dev, "Reconfig DCB to single TC as result of Link Down\n"); 10000 memset(&pf->tmp_cfg, 0, sizeof(pf->tmp_cfg)); 10001 err = i40e_dcb_sw_default_config(pf); 10002 if (err) { 10003 clear_bit(I40E_FLAG_DCB_CAPABLE, pf->flags); 10004 clear_bit(I40E_FLAG_DCB_ENA, pf->flags); 10005 } else { 10006 pf->dcbx_cap = DCB_CAP_DCBX_HOST | 10007 DCB_CAP_DCBX_VER_IEEE; 10008 set_bit(I40E_FLAG_DCB_CAPABLE, pf->flags); 10009 clear_bit(I40E_FLAG_DCB_ENA, pf->flags); 10010 } 10011 } 10012 #endif /* CONFIG_I40E_DCB */ 10013 } 10014 10015 /** 10016 * i40e_watchdog_subtask - periodic checks not using event driven response 10017 * @pf: board private structure 10018 **/ 10019 static void i40e_watchdog_subtask(struct i40e_pf *pf) 10020 { 10021 struct i40e_vsi *vsi; 10022 struct i40e_veb *veb; 10023 int i; 10024 10025 /* if interface is down do nothing */ 10026 if (test_bit(__I40E_DOWN, pf->state) || 10027 test_bit(__I40E_CONFIG_BUSY, pf->state)) 10028 return; 10029 10030 /* make sure we don't do these things too often */ 10031 if (time_before(jiffies, (pf->service_timer_previous + 10032 pf->service_timer_period))) 10033 return; 10034 pf->service_timer_previous = jiffies; 10035 10036 if (test_bit(I40E_FLAG_LINK_POLLING_ENA, pf->flags) || 10037 test_bit(__I40E_TEMP_LINK_POLLING, pf->state)) 10038 i40e_link_event(pf); 10039 10040 /* Update the stats for active netdevs so the network stack 10041 * can look at updated numbers whenever it cares to 10042 */ 10043 i40e_pf_for_each_vsi(pf, i, vsi) 10044 if (vsi->netdev) 10045 i40e_update_stats(vsi); 10046 10047 if (test_bit(I40E_FLAG_VEB_STATS_ENA, pf->flags)) { 10048 /* Update the stats for the active switching components */ 10049 i40e_pf_for_each_veb(pf, i, veb) 10050 i40e_update_veb_stats(veb); 10051 } 10052 10053 i40e_ptp_rx_hang(pf); 10054 i40e_ptp_tx_hang(pf); 10055 } 10056 10057 /** 10058 * i40e_reset_subtask - Set up for resetting the device and driver 10059 * @pf: board private structure 10060 **/ 10061 static void i40e_reset_subtask(struct i40e_pf *pf) 10062 { 10063 u32 reset_flags = 0; 10064 10065 if (test_bit(__I40E_REINIT_REQUESTED, pf->state)) { 10066 reset_flags |= BIT(__I40E_REINIT_REQUESTED); 10067 clear_bit(__I40E_REINIT_REQUESTED, pf->state); 10068 } 10069 if (test_bit(__I40E_PF_RESET_REQUESTED, pf->state)) { 10070 reset_flags |= BIT(__I40E_PF_RESET_REQUESTED); 10071 clear_bit(__I40E_PF_RESET_REQUESTED, pf->state); 10072 } 10073 if (test_bit(__I40E_CORE_RESET_REQUESTED, pf->state)) { 10074 reset_flags |= BIT(__I40E_CORE_RESET_REQUESTED); 10075 clear_bit(__I40E_CORE_RESET_REQUESTED, pf->state); 10076 } 10077 if (test_bit(__I40E_GLOBAL_RESET_REQUESTED, pf->state)) { 10078 reset_flags |= BIT(__I40E_GLOBAL_RESET_REQUESTED); 10079 clear_bit(__I40E_GLOBAL_RESET_REQUESTED, pf->state); 10080 } 10081 if (test_bit(__I40E_DOWN_REQUESTED, pf->state)) { 10082 reset_flags |= BIT(__I40E_DOWN_REQUESTED); 10083 clear_bit(__I40E_DOWN_REQUESTED, pf->state); 10084 } 10085 10086 /* If there's a recovery already waiting, it takes 10087 * precedence before starting a new reset sequence. 10088 */ 10089 if (test_bit(__I40E_RESET_INTR_RECEIVED, pf->state)) { 10090 i40e_prep_for_reset(pf); 10091 i40e_reset(pf); 10092 i40e_rebuild(pf, false, false); 10093 } 10094 10095 /* If we're already down or resetting, just bail */ 10096 if (reset_flags && 10097 !test_bit(__I40E_DOWN, pf->state) && 10098 !test_bit(__I40E_CONFIG_BUSY, pf->state)) { 10099 i40e_do_reset(pf, reset_flags, false); 10100 } 10101 } 10102 10103 /** 10104 * i40e_handle_link_event - Handle link event 10105 * @pf: board private structure 10106 * @e: event info posted on ARQ 10107 **/ 10108 static void i40e_handle_link_event(struct i40e_pf *pf, 10109 struct i40e_arq_event_info *e) 10110 { 10111 struct i40e_aqc_get_link_status *status = 10112 (struct i40e_aqc_get_link_status *)&e->desc.params.raw; 10113 10114 /* Do a new status request to re-enable LSE reporting 10115 * and load new status information into the hw struct 10116 * This completely ignores any state information 10117 * in the ARQ event info, instead choosing to always 10118 * issue the AQ update link status command. 10119 */ 10120 i40e_link_event(pf); 10121 10122 /* Check if module meets thermal requirements */ 10123 if (status->phy_type == I40E_PHY_TYPE_NOT_SUPPORTED_HIGH_TEMP) { 10124 dev_err(&pf->pdev->dev, 10125 "Rx/Tx is disabled on this device because the module does not meet thermal requirements.\n"); 10126 dev_err(&pf->pdev->dev, 10127 "Refer to the Intel(R) Ethernet Adapters and Devices User Guide for a list of supported modules.\n"); 10128 } else { 10129 /* check for unqualified module, if link is down, suppress 10130 * the message if link was forced to be down. 10131 */ 10132 if ((status->link_info & I40E_AQ_MEDIA_AVAILABLE) && 10133 (!(status->an_info & I40E_AQ_QUALIFIED_MODULE)) && 10134 (!(status->link_info & I40E_AQ_LINK_UP)) && 10135 (!test_bit(I40E_FLAG_LINK_DOWN_ON_CLOSE_ENA, pf->flags))) { 10136 dev_err(&pf->pdev->dev, 10137 "Rx/Tx is disabled on this device because an unsupported SFP module type was detected.\n"); 10138 dev_err(&pf->pdev->dev, 10139 "Refer to the Intel(R) Ethernet Adapters and Devices User Guide for a list of supported modules.\n"); 10140 } 10141 } 10142 } 10143 10144 /** 10145 * i40e_clean_adminq_subtask - Clean the AdminQ rings 10146 * @pf: board private structure 10147 **/ 10148 static void i40e_clean_adminq_subtask(struct i40e_pf *pf) 10149 { 10150 struct i40e_arq_event_info event; 10151 struct i40e_hw *hw = &pf->hw; 10152 u16 pending, i = 0; 10153 u16 opcode; 10154 u32 oldval; 10155 int ret; 10156 u32 val; 10157 10158 /* Do not run clean AQ when PF reset fails */ 10159 if (test_bit(__I40E_RESET_FAILED, pf->state)) 10160 return; 10161 10162 /* check for error indications */ 10163 val = rd32(&pf->hw, I40E_PF_ARQLEN); 10164 oldval = val; 10165 if (val & I40E_PF_ARQLEN_ARQVFE_MASK) { 10166 if (hw->debug_mask & I40E_DEBUG_AQ) 10167 dev_info(&pf->pdev->dev, "ARQ VF Error detected\n"); 10168 val &= ~I40E_PF_ARQLEN_ARQVFE_MASK; 10169 } 10170 if (val & I40E_PF_ARQLEN_ARQOVFL_MASK) { 10171 if (hw->debug_mask & I40E_DEBUG_AQ) 10172 dev_info(&pf->pdev->dev, "ARQ Overflow Error detected\n"); 10173 val &= ~I40E_PF_ARQLEN_ARQOVFL_MASK; 10174 pf->arq_overflows++; 10175 } 10176 if (val & I40E_PF_ARQLEN_ARQCRIT_MASK) { 10177 if (hw->debug_mask & I40E_DEBUG_AQ) 10178 dev_info(&pf->pdev->dev, "ARQ Critical Error detected\n"); 10179 val &= ~I40E_PF_ARQLEN_ARQCRIT_MASK; 10180 } 10181 if (oldval != val) 10182 wr32(&pf->hw, I40E_PF_ARQLEN, val); 10183 10184 val = rd32(&pf->hw, I40E_PF_ATQLEN); 10185 oldval = val; 10186 if (val & I40E_PF_ATQLEN_ATQVFE_MASK) { 10187 if (pf->hw.debug_mask & I40E_DEBUG_AQ) 10188 dev_info(&pf->pdev->dev, "ASQ VF Error detected\n"); 10189 val &= ~I40E_PF_ATQLEN_ATQVFE_MASK; 10190 } 10191 if (val & I40E_PF_ATQLEN_ATQOVFL_MASK) { 10192 if (pf->hw.debug_mask & I40E_DEBUG_AQ) 10193 dev_info(&pf->pdev->dev, "ASQ Overflow Error detected\n"); 10194 val &= ~I40E_PF_ATQLEN_ATQOVFL_MASK; 10195 } 10196 if (val & I40E_PF_ATQLEN_ATQCRIT_MASK) { 10197 if (pf->hw.debug_mask & I40E_DEBUG_AQ) 10198 dev_info(&pf->pdev->dev, "ASQ Critical Error detected\n"); 10199 val &= ~I40E_PF_ATQLEN_ATQCRIT_MASK; 10200 } 10201 if (oldval != val) 10202 wr32(&pf->hw, I40E_PF_ATQLEN, val); 10203 10204 event.buf_len = I40E_MAX_AQ_BUF_SIZE; 10205 event.msg_buf = kzalloc(event.buf_len, GFP_KERNEL); 10206 if (!event.msg_buf) 10207 return; 10208 10209 do { 10210 ret = i40e_clean_arq_element(hw, &event, &pending); 10211 if (ret == -EALREADY) 10212 break; 10213 else if (ret) { 10214 dev_info(&pf->pdev->dev, "ARQ event error %d\n", ret); 10215 break; 10216 } 10217 10218 opcode = le16_to_cpu(event.desc.opcode); 10219 switch (opcode) { 10220 10221 case i40e_aqc_opc_get_link_status: 10222 rtnl_lock(); 10223 i40e_handle_link_event(pf, &event); 10224 rtnl_unlock(); 10225 break; 10226 case i40e_aqc_opc_send_msg_to_pf: 10227 ret = i40e_vc_process_vf_msg(pf, 10228 le16_to_cpu(event.desc.retval), 10229 le32_to_cpu(event.desc.cookie_high), 10230 le32_to_cpu(event.desc.cookie_low), 10231 event.msg_buf, 10232 event.msg_len); 10233 break; 10234 case i40e_aqc_opc_lldp_update_mib: 10235 dev_dbg(&pf->pdev->dev, "ARQ: Update LLDP MIB event received\n"); 10236 #ifdef CONFIG_I40E_DCB 10237 rtnl_lock(); 10238 i40e_handle_lldp_event(pf, &event); 10239 rtnl_unlock(); 10240 #endif /* CONFIG_I40E_DCB */ 10241 break; 10242 case i40e_aqc_opc_event_lan_overflow: 10243 dev_dbg(&pf->pdev->dev, "ARQ LAN queue overflow event received\n"); 10244 i40e_handle_lan_overflow_event(pf, &event); 10245 break; 10246 case i40e_aqc_opc_send_msg_to_peer: 10247 dev_info(&pf->pdev->dev, "ARQ: Msg from other pf\n"); 10248 break; 10249 case i40e_aqc_opc_nvm_erase: 10250 case i40e_aqc_opc_nvm_update: 10251 case i40e_aqc_opc_oem_post_update: 10252 i40e_debug(&pf->hw, I40E_DEBUG_NVM, 10253 "ARQ NVM operation 0x%04x completed\n", 10254 opcode); 10255 break; 10256 default: 10257 dev_info(&pf->pdev->dev, 10258 "ARQ: Unknown event 0x%04x ignored\n", 10259 opcode); 10260 break; 10261 } 10262 } while (i++ < I40E_AQ_WORK_LIMIT); 10263 10264 if (i < I40E_AQ_WORK_LIMIT) 10265 clear_bit(__I40E_ADMINQ_EVENT_PENDING, pf->state); 10266 10267 /* re-enable Admin queue interrupt cause */ 10268 val = rd32(hw, I40E_PFINT_ICR0_ENA); 10269 val |= I40E_PFINT_ICR0_ENA_ADMINQ_MASK; 10270 wr32(hw, I40E_PFINT_ICR0_ENA, val); 10271 i40e_flush(hw); 10272 10273 kfree(event.msg_buf); 10274 } 10275 10276 /** 10277 * i40e_verify_eeprom - make sure eeprom is good to use 10278 * @pf: board private structure 10279 **/ 10280 static void i40e_verify_eeprom(struct i40e_pf *pf) 10281 { 10282 int err; 10283 10284 err = i40e_diag_eeprom_test(&pf->hw); 10285 if (err) { 10286 /* retry in case of garbage read */ 10287 err = i40e_diag_eeprom_test(&pf->hw); 10288 if (err) { 10289 dev_info(&pf->pdev->dev, "eeprom check failed (%d), Tx/Rx traffic disabled\n", 10290 err); 10291 set_bit(__I40E_BAD_EEPROM, pf->state); 10292 } 10293 } 10294 10295 if (!err && test_bit(__I40E_BAD_EEPROM, pf->state)) { 10296 dev_info(&pf->pdev->dev, "eeprom check passed, Tx/Rx traffic enabled\n"); 10297 clear_bit(__I40E_BAD_EEPROM, pf->state); 10298 } 10299 } 10300 10301 /** 10302 * i40e_enable_pf_switch_lb 10303 * @pf: pointer to the PF structure 10304 * 10305 * enable switch loop back or die - no point in a return value 10306 **/ 10307 static void i40e_enable_pf_switch_lb(struct i40e_pf *pf) 10308 { 10309 struct i40e_vsi *vsi = i40e_pf_get_main_vsi(pf); 10310 struct i40e_vsi_context ctxt; 10311 int ret; 10312 10313 ctxt.seid = pf->main_vsi_seid; 10314 ctxt.pf_num = pf->hw.pf_id; 10315 ctxt.vf_num = 0; 10316 ret = i40e_aq_get_vsi_params(&pf->hw, &ctxt, NULL); 10317 if (ret) { 10318 dev_info(&pf->pdev->dev, 10319 "couldn't get PF vsi config, err %pe aq_err %s\n", 10320 ERR_PTR(ret), 10321 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 10322 return; 10323 } 10324 ctxt.flags = I40E_AQ_VSI_TYPE_PF; 10325 ctxt.info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID); 10326 ctxt.info.switch_id |= cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB); 10327 10328 ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL); 10329 if (ret) { 10330 dev_info(&pf->pdev->dev, 10331 "update vsi switch failed, err %pe aq_err %s\n", 10332 ERR_PTR(ret), 10333 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 10334 } 10335 } 10336 10337 /** 10338 * i40e_disable_pf_switch_lb 10339 * @pf: pointer to the PF structure 10340 * 10341 * disable switch loop back or die - no point in a return value 10342 **/ 10343 static void i40e_disable_pf_switch_lb(struct i40e_pf *pf) 10344 { 10345 struct i40e_vsi *vsi = i40e_pf_get_main_vsi(pf); 10346 struct i40e_vsi_context ctxt; 10347 int ret; 10348 10349 ctxt.seid = pf->main_vsi_seid; 10350 ctxt.pf_num = pf->hw.pf_id; 10351 ctxt.vf_num = 0; 10352 ret = i40e_aq_get_vsi_params(&pf->hw, &ctxt, NULL); 10353 if (ret) { 10354 dev_info(&pf->pdev->dev, 10355 "couldn't get PF vsi config, err %pe aq_err %s\n", 10356 ERR_PTR(ret), 10357 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 10358 return; 10359 } 10360 ctxt.flags = I40E_AQ_VSI_TYPE_PF; 10361 ctxt.info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID); 10362 ctxt.info.switch_id &= ~cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB); 10363 10364 ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL); 10365 if (ret) { 10366 dev_info(&pf->pdev->dev, 10367 "update vsi switch failed, err %pe aq_err %s\n", 10368 ERR_PTR(ret), 10369 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 10370 } 10371 } 10372 10373 /** 10374 * i40e_config_bridge_mode - Configure the HW bridge mode 10375 * @veb: pointer to the bridge instance 10376 * 10377 * Configure the loop back mode for the LAN VSI that is downlink to the 10378 * specified HW bridge instance. It is expected this function is called 10379 * when a new HW bridge is instantiated. 10380 **/ 10381 static void i40e_config_bridge_mode(struct i40e_veb *veb) 10382 { 10383 struct i40e_pf *pf = veb->pf; 10384 10385 if (pf->hw.debug_mask & I40E_DEBUG_LAN) 10386 dev_info(&pf->pdev->dev, "enabling bridge mode: %s\n", 10387 veb->bridge_mode == BRIDGE_MODE_VEPA ? "VEPA" : "VEB"); 10388 if (veb->bridge_mode & BRIDGE_MODE_VEPA) 10389 i40e_disable_pf_switch_lb(pf); 10390 else 10391 i40e_enable_pf_switch_lb(pf); 10392 } 10393 10394 /** 10395 * i40e_reconstitute_veb - rebuild the VEB and VSIs connected to it 10396 * @veb: pointer to the VEB instance 10397 * 10398 * This is a function that builds the attached VSIs. We track the connections 10399 * through our own index numbers because the seid's from the HW could change 10400 * across the reset. 10401 **/ 10402 static int i40e_reconstitute_veb(struct i40e_veb *veb) 10403 { 10404 struct i40e_vsi *ctl_vsi = NULL; 10405 struct i40e_pf *pf = veb->pf; 10406 struct i40e_vsi *vsi; 10407 int v, ret; 10408 10409 /* As we do not maintain PV (port virtualizer) switch element then 10410 * there can be only one non-floating VEB that have uplink to MAC SEID 10411 * and its control VSI is the main one. 10412 */ 10413 if (WARN_ON(veb->uplink_seid && veb->uplink_seid != pf->mac_seid)) { 10414 dev_err(&pf->pdev->dev, 10415 "Invalid uplink SEID for VEB %d\n", veb->idx); 10416 return -ENOENT; 10417 } 10418 10419 if (veb->uplink_seid == pf->mac_seid) { 10420 /* Check that the LAN VSI has VEB owning flag set */ 10421 ctl_vsi = i40e_pf_get_main_vsi(pf); 10422 10423 if (WARN_ON(ctl_vsi->veb_idx != veb->idx || 10424 !(ctl_vsi->flags & I40E_VSI_FLAG_VEB_OWNER))) { 10425 dev_err(&pf->pdev->dev, 10426 "Invalid control VSI for VEB %d\n", veb->idx); 10427 return -ENOENT; 10428 } 10429 10430 /* Add the control VSI to switch */ 10431 ret = i40e_add_vsi(ctl_vsi); 10432 if (ret) { 10433 dev_err(&pf->pdev->dev, 10434 "Rebuild of owner VSI for VEB %d failed: %d\n", 10435 veb->idx, ret); 10436 return ret; 10437 } 10438 10439 i40e_vsi_reset_stats(ctl_vsi); 10440 } 10441 10442 /* create the VEB in the switch and move the VSI onto the VEB */ 10443 ret = i40e_add_veb(veb, ctl_vsi); 10444 if (ret) 10445 return ret; 10446 10447 if (veb->uplink_seid) { 10448 if (test_bit(I40E_FLAG_VEB_MODE_ENA, pf->flags)) 10449 veb->bridge_mode = BRIDGE_MODE_VEB; 10450 else 10451 veb->bridge_mode = BRIDGE_MODE_VEPA; 10452 i40e_config_bridge_mode(veb); 10453 } 10454 10455 /* create the remaining VSIs attached to this VEB */ 10456 i40e_pf_for_each_vsi(pf, v, vsi) { 10457 if (vsi == ctl_vsi) 10458 continue; 10459 10460 if (vsi->veb_idx == veb->idx) { 10461 vsi->uplink_seid = veb->seid; 10462 ret = i40e_add_vsi(vsi); 10463 if (ret) { 10464 dev_info(&pf->pdev->dev, 10465 "rebuild of vsi_idx %d failed: %d\n", 10466 v, ret); 10467 return ret; 10468 } 10469 i40e_vsi_reset_stats(vsi); 10470 } 10471 } 10472 10473 return ret; 10474 } 10475 10476 /** 10477 * i40e_get_capabilities - get info about the HW 10478 * @pf: the PF struct 10479 * @list_type: AQ capability to be queried 10480 **/ 10481 static int i40e_get_capabilities(struct i40e_pf *pf, 10482 enum i40e_admin_queue_opc list_type) 10483 { 10484 struct i40e_aqc_list_capabilities_element_resp *cap_buf; 10485 u16 data_size; 10486 int buf_len; 10487 int err; 10488 10489 buf_len = 40 * sizeof(struct i40e_aqc_list_capabilities_element_resp); 10490 do { 10491 cap_buf = kzalloc(buf_len, GFP_KERNEL); 10492 if (!cap_buf) 10493 return -ENOMEM; 10494 10495 /* this loads the data into the hw struct for us */ 10496 err = i40e_aq_discover_capabilities(&pf->hw, cap_buf, buf_len, 10497 &data_size, list_type, 10498 NULL); 10499 /* data loaded, buffer no longer needed */ 10500 kfree(cap_buf); 10501 10502 if (pf->hw.aq.asq_last_status == I40E_AQ_RC_ENOMEM) { 10503 /* retry with a larger buffer */ 10504 buf_len = data_size; 10505 } else if (pf->hw.aq.asq_last_status != I40E_AQ_RC_OK || err) { 10506 dev_info(&pf->pdev->dev, 10507 "capability discovery failed, err %pe aq_err %s\n", 10508 ERR_PTR(err), 10509 i40e_aq_str(&pf->hw, 10510 pf->hw.aq.asq_last_status)); 10511 return -ENODEV; 10512 } 10513 } while (err); 10514 10515 if (pf->hw.debug_mask & I40E_DEBUG_USER) { 10516 if (list_type == i40e_aqc_opc_list_func_capabilities) { 10517 dev_info(&pf->pdev->dev, 10518 "pf=%d, num_vfs=%d, msix_pf=%d, msix_vf=%d, fd_g=%d, fd_b=%d, pf_max_q=%d num_vsi=%d\n", 10519 pf->hw.pf_id, pf->hw.func_caps.num_vfs, 10520 pf->hw.func_caps.num_msix_vectors, 10521 pf->hw.func_caps.num_msix_vectors_vf, 10522 pf->hw.func_caps.fd_filters_guaranteed, 10523 pf->hw.func_caps.fd_filters_best_effort, 10524 pf->hw.func_caps.num_tx_qp, 10525 pf->hw.func_caps.num_vsis); 10526 } else if (list_type == i40e_aqc_opc_list_dev_capabilities) { 10527 dev_info(&pf->pdev->dev, 10528 "switch_mode=0x%04x, function_valid=0x%08x\n", 10529 pf->hw.dev_caps.switch_mode, 10530 pf->hw.dev_caps.valid_functions); 10531 dev_info(&pf->pdev->dev, 10532 "SR-IOV=%d, num_vfs for all function=%u\n", 10533 pf->hw.dev_caps.sr_iov_1_1, 10534 pf->hw.dev_caps.num_vfs); 10535 dev_info(&pf->pdev->dev, 10536 "num_vsis=%u, num_rx:%u, num_tx=%u\n", 10537 pf->hw.dev_caps.num_vsis, 10538 pf->hw.dev_caps.num_rx_qp, 10539 pf->hw.dev_caps.num_tx_qp); 10540 } 10541 } 10542 if (list_type == i40e_aqc_opc_list_func_capabilities) { 10543 #define DEF_NUM_VSI (1 + (pf->hw.func_caps.fcoe ? 1 : 0) \ 10544 + pf->hw.func_caps.num_vfs) 10545 if (pf->hw.revision_id == 0 && 10546 pf->hw.func_caps.num_vsis < DEF_NUM_VSI) { 10547 dev_info(&pf->pdev->dev, 10548 "got num_vsis %d, setting num_vsis to %d\n", 10549 pf->hw.func_caps.num_vsis, DEF_NUM_VSI); 10550 pf->hw.func_caps.num_vsis = DEF_NUM_VSI; 10551 } 10552 } 10553 return 0; 10554 } 10555 10556 static int i40e_vsi_clear(struct i40e_vsi *vsi); 10557 10558 /** 10559 * i40e_fdir_sb_setup - initialize the Flow Director resources for Sideband 10560 * @pf: board private structure 10561 **/ 10562 static void i40e_fdir_sb_setup(struct i40e_pf *pf) 10563 { 10564 struct i40e_vsi *main_vsi, *vsi; 10565 10566 /* quick workaround for an NVM issue that leaves a critical register 10567 * uninitialized 10568 */ 10569 if (!rd32(&pf->hw, I40E_GLQF_HKEY(0))) { 10570 static const u32 hkey[] = { 10571 0xe640d33f, 0xcdfe98ab, 0x73fa7161, 0x0d7a7d36, 10572 0xeacb7d61, 0xaa4f05b6, 0x9c5c89ed, 0xfc425ddb, 10573 0xa4654832, 0xfc7461d4, 0x8f827619, 0xf5c63c21, 10574 0x95b3a76d}; 10575 int i; 10576 10577 for (i = 0; i <= I40E_GLQF_HKEY_MAX_INDEX; i++) 10578 wr32(&pf->hw, I40E_GLQF_HKEY(i), hkey[i]); 10579 } 10580 10581 if (!test_bit(I40E_FLAG_FD_SB_ENA, pf->flags)) 10582 return; 10583 10584 /* find existing VSI and see if it needs configuring */ 10585 vsi = i40e_find_vsi_by_type(pf, I40E_VSI_FDIR); 10586 10587 /* create a new VSI if none exists */ 10588 if (!vsi) { 10589 main_vsi = i40e_pf_get_main_vsi(pf); 10590 vsi = i40e_vsi_setup(pf, I40E_VSI_FDIR, main_vsi->seid, 0); 10591 if (!vsi) { 10592 dev_info(&pf->pdev->dev, "Couldn't create FDir VSI\n"); 10593 clear_bit(I40E_FLAG_FD_SB_ENA, pf->flags); 10594 set_bit(I40E_FLAG_FD_SB_INACTIVE, pf->flags); 10595 return; 10596 } 10597 } 10598 10599 i40e_vsi_setup_irqhandler(vsi, i40e_fdir_clean_ring); 10600 } 10601 10602 /** 10603 * i40e_fdir_teardown - release the Flow Director resources 10604 * @pf: board private structure 10605 **/ 10606 static void i40e_fdir_teardown(struct i40e_pf *pf) 10607 { 10608 struct i40e_vsi *vsi; 10609 10610 i40e_fdir_filter_exit(pf); 10611 vsi = i40e_find_vsi_by_type(pf, I40E_VSI_FDIR); 10612 if (vsi) 10613 i40e_vsi_release(vsi); 10614 } 10615 10616 /** 10617 * i40e_rebuild_cloud_filters - Rebuilds cloud filters for VSIs 10618 * @vsi: PF main vsi 10619 * @seid: seid of main or channel VSIs 10620 * 10621 * Rebuilds cloud filters associated with main VSI and channel VSIs if they 10622 * existed before reset 10623 **/ 10624 static int i40e_rebuild_cloud_filters(struct i40e_vsi *vsi, u16 seid) 10625 { 10626 struct i40e_cloud_filter *cfilter; 10627 struct i40e_pf *pf = vsi->back; 10628 struct hlist_node *node; 10629 int ret; 10630 10631 /* Add cloud filters back if they exist */ 10632 hlist_for_each_entry_safe(cfilter, node, &pf->cloud_filter_list, 10633 cloud_node) { 10634 if (cfilter->seid != seid) 10635 continue; 10636 10637 if (cfilter->dst_port) 10638 ret = i40e_add_del_cloud_filter_big_buf(vsi, cfilter, 10639 true); 10640 else 10641 ret = i40e_add_del_cloud_filter(vsi, cfilter, true); 10642 10643 if (ret) { 10644 dev_dbg(&pf->pdev->dev, 10645 "Failed to rebuild cloud filter, err %pe aq_err %s\n", 10646 ERR_PTR(ret), 10647 i40e_aq_str(&pf->hw, 10648 pf->hw.aq.asq_last_status)); 10649 return ret; 10650 } 10651 } 10652 return 0; 10653 } 10654 10655 /** 10656 * i40e_rebuild_channels - Rebuilds channel VSIs if they existed before reset 10657 * @vsi: PF main vsi 10658 * 10659 * Rebuilds channel VSIs if they existed before reset 10660 **/ 10661 static int i40e_rebuild_channels(struct i40e_vsi *vsi) 10662 { 10663 struct i40e_channel *ch, *ch_tmp; 10664 int ret; 10665 10666 if (list_empty(&vsi->ch_list)) 10667 return 0; 10668 10669 list_for_each_entry_safe(ch, ch_tmp, &vsi->ch_list, list) { 10670 if (!ch->initialized) 10671 break; 10672 /* Proceed with creation of channel (VMDq2) VSI */ 10673 ret = i40e_add_channel(vsi->back, vsi->uplink_seid, ch); 10674 if (ret) { 10675 dev_info(&vsi->back->pdev->dev, 10676 "failed to rebuild channels using uplink_seid %u\n", 10677 vsi->uplink_seid); 10678 return ret; 10679 } 10680 /* Reconfigure TX queues using QTX_CTL register */ 10681 ret = i40e_channel_config_tx_ring(vsi->back, vsi, ch); 10682 if (ret) { 10683 dev_info(&vsi->back->pdev->dev, 10684 "failed to configure TX rings for channel %u\n", 10685 ch->seid); 10686 return ret; 10687 } 10688 /* update 'next_base_queue' */ 10689 vsi->next_base_queue = vsi->next_base_queue + 10690 ch->num_queue_pairs; 10691 if (ch->max_tx_rate) { 10692 u64 credits = ch->max_tx_rate; 10693 10694 if (i40e_set_bw_limit(vsi, ch->seid, 10695 ch->max_tx_rate)) 10696 return -EINVAL; 10697 10698 do_div(credits, I40E_BW_CREDIT_DIVISOR); 10699 dev_dbg(&vsi->back->pdev->dev, 10700 "Set tx rate of %llu Mbps (count of 50Mbps %llu) for vsi->seid %u\n", 10701 ch->max_tx_rate, 10702 credits, 10703 ch->seid); 10704 } 10705 ret = i40e_rebuild_cloud_filters(vsi, ch->seid); 10706 if (ret) { 10707 dev_dbg(&vsi->back->pdev->dev, 10708 "Failed to rebuild cloud filters for channel VSI %u\n", 10709 ch->seid); 10710 return ret; 10711 } 10712 } 10713 return 0; 10714 } 10715 10716 /** 10717 * i40e_clean_xps_state - clean xps state for every tx_ring 10718 * @vsi: ptr to the VSI 10719 **/ 10720 static void i40e_clean_xps_state(struct i40e_vsi *vsi) 10721 { 10722 int i; 10723 10724 if (vsi->tx_rings) 10725 for (i = 0; i < vsi->num_queue_pairs; i++) 10726 if (vsi->tx_rings[i]) 10727 clear_bit(__I40E_TX_XPS_INIT_DONE, 10728 vsi->tx_rings[i]->state); 10729 } 10730 10731 /** 10732 * i40e_prep_for_reset - prep for the core to reset 10733 * @pf: board private structure 10734 * 10735 * Close up the VFs and other things in prep for PF Reset. 10736 **/ 10737 static void i40e_prep_for_reset(struct i40e_pf *pf) 10738 { 10739 struct i40e_hw *hw = &pf->hw; 10740 struct i40e_vsi *vsi; 10741 int ret = 0; 10742 u32 v; 10743 10744 clear_bit(__I40E_RESET_INTR_RECEIVED, pf->state); 10745 if (test_and_set_bit(__I40E_RESET_RECOVERY_PENDING, pf->state)) 10746 return; 10747 if (i40e_check_asq_alive(&pf->hw)) 10748 i40e_vc_notify_reset(pf); 10749 10750 dev_dbg(&pf->pdev->dev, "Tearing down internal switch for reset\n"); 10751 10752 /* quiesce the VSIs and their queues that are not already DOWN */ 10753 i40e_pf_quiesce_all_vsi(pf); 10754 10755 i40e_pf_for_each_vsi(pf, v, vsi) { 10756 i40e_clean_xps_state(vsi); 10757 vsi->seid = 0; 10758 } 10759 10760 i40e_shutdown_adminq(&pf->hw); 10761 10762 /* call shutdown HMC */ 10763 if (hw->hmc.hmc_obj) { 10764 ret = i40e_shutdown_lan_hmc(hw); 10765 if (ret) 10766 dev_warn(&pf->pdev->dev, 10767 "shutdown_lan_hmc failed: %d\n", ret); 10768 } 10769 10770 /* Save the current PTP time so that we can restore the time after the 10771 * reset completes. 10772 */ 10773 i40e_ptp_save_hw_time(pf); 10774 } 10775 10776 /** 10777 * i40e_send_version - update firmware with driver version 10778 * @pf: PF struct 10779 */ 10780 static void i40e_send_version(struct i40e_pf *pf) 10781 { 10782 struct i40e_driver_version dv; 10783 10784 dv.major_version = 0xff; 10785 dv.minor_version = 0xff; 10786 dv.build_version = 0xff; 10787 dv.subbuild_version = 0; 10788 strscpy(dv.driver_string, UTS_RELEASE, sizeof(dv.driver_string)); 10789 i40e_aq_send_driver_version(&pf->hw, &dv, NULL); 10790 } 10791 10792 /** 10793 * i40e_get_oem_version - get OEM specific version information 10794 * @hw: pointer to the hardware structure 10795 **/ 10796 static void i40e_get_oem_version(struct i40e_hw *hw) 10797 { 10798 u16 block_offset = 0xffff; 10799 u16 block_length = 0; 10800 u16 capabilities = 0; 10801 u16 gen_snap = 0; 10802 u16 release = 0; 10803 10804 #define I40E_SR_NVM_OEM_VERSION_PTR 0x1B 10805 #define I40E_NVM_OEM_LENGTH_OFFSET 0x00 10806 #define I40E_NVM_OEM_CAPABILITIES_OFFSET 0x01 10807 #define I40E_NVM_OEM_GEN_OFFSET 0x02 10808 #define I40E_NVM_OEM_RELEASE_OFFSET 0x03 10809 #define I40E_NVM_OEM_CAPABILITIES_MASK 0x000F 10810 #define I40E_NVM_OEM_LENGTH 3 10811 10812 /* Check if pointer to OEM version block is valid. */ 10813 i40e_read_nvm_word(hw, I40E_SR_NVM_OEM_VERSION_PTR, &block_offset); 10814 if (block_offset == 0xffff) 10815 return; 10816 10817 /* Check if OEM version block has correct length. */ 10818 i40e_read_nvm_word(hw, block_offset + I40E_NVM_OEM_LENGTH_OFFSET, 10819 &block_length); 10820 if (block_length < I40E_NVM_OEM_LENGTH) 10821 return; 10822 10823 /* Check if OEM version format is as expected. */ 10824 i40e_read_nvm_word(hw, block_offset + I40E_NVM_OEM_CAPABILITIES_OFFSET, 10825 &capabilities); 10826 if ((capabilities & I40E_NVM_OEM_CAPABILITIES_MASK) != 0) 10827 return; 10828 10829 i40e_read_nvm_word(hw, block_offset + I40E_NVM_OEM_GEN_OFFSET, 10830 &gen_snap); 10831 i40e_read_nvm_word(hw, block_offset + I40E_NVM_OEM_RELEASE_OFFSET, 10832 &release); 10833 hw->nvm.oem_ver = 10834 FIELD_PREP(I40E_OEM_GEN_MASK | I40E_OEM_SNAP_MASK, gen_snap) | 10835 FIELD_PREP(I40E_OEM_RELEASE_MASK, release); 10836 hw->nvm.eetrack = I40E_OEM_EETRACK_ID; 10837 } 10838 10839 /** 10840 * i40e_reset - wait for core reset to finish reset, reset pf if corer not seen 10841 * @pf: board private structure 10842 **/ 10843 static int i40e_reset(struct i40e_pf *pf) 10844 { 10845 struct i40e_hw *hw = &pf->hw; 10846 int ret; 10847 10848 ret = i40e_pf_reset(hw); 10849 if (ret) { 10850 dev_info(&pf->pdev->dev, "PF reset failed, %d\n", ret); 10851 set_bit(__I40E_RESET_FAILED, pf->state); 10852 clear_bit(__I40E_RESET_RECOVERY_PENDING, pf->state); 10853 } else { 10854 pf->pfr_count++; 10855 } 10856 return ret; 10857 } 10858 10859 /** 10860 * i40e_rebuild - rebuild using a saved config 10861 * @pf: board private structure 10862 * @reinit: if the Main VSI needs to re-initialized. 10863 * @lock_acquired: indicates whether or not the lock has been acquired 10864 * before this function was called. 10865 **/ 10866 static void i40e_rebuild(struct i40e_pf *pf, bool reinit, bool lock_acquired) 10867 { 10868 const bool is_recovery_mode_reported = i40e_check_recovery_mode(pf); 10869 struct i40e_vsi *vsi = i40e_pf_get_main_vsi(pf); 10870 struct i40e_hw *hw = &pf->hw; 10871 struct i40e_veb *veb; 10872 int ret; 10873 u32 val; 10874 int v; 10875 10876 if (test_bit(__I40E_EMP_RESET_INTR_RECEIVED, pf->state) && 10877 is_recovery_mode_reported) 10878 i40e_set_ethtool_ops(vsi->netdev); 10879 10880 if (test_bit(__I40E_DOWN, pf->state) && 10881 !test_bit(__I40E_RECOVERY_MODE, pf->state)) 10882 goto clear_recovery; 10883 dev_dbg(&pf->pdev->dev, "Rebuilding internal switch\n"); 10884 10885 /* rebuild the basics for the AdminQ, HMC, and initial HW switch */ 10886 ret = i40e_init_adminq(&pf->hw); 10887 if (ret) { 10888 dev_info(&pf->pdev->dev, "Rebuild AdminQ failed, err %pe aq_err %s\n", 10889 ERR_PTR(ret), 10890 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 10891 goto clear_recovery; 10892 } 10893 i40e_get_oem_version(&pf->hw); 10894 10895 if (test_and_clear_bit(__I40E_EMP_RESET_INTR_RECEIVED, pf->state)) { 10896 /* The following delay is necessary for firmware update. */ 10897 mdelay(1000); 10898 } 10899 10900 /* re-verify the eeprom if we just had an EMP reset */ 10901 if (test_and_clear_bit(__I40E_EMP_RESET_INTR_RECEIVED, pf->state)) 10902 i40e_verify_eeprom(pf); 10903 10904 /* if we are going out of or into recovery mode we have to act 10905 * accordingly with regard to resources initialization 10906 * and deinitialization 10907 */ 10908 if (test_bit(__I40E_RECOVERY_MODE, pf->state)) { 10909 if (i40e_get_capabilities(pf, 10910 i40e_aqc_opc_list_func_capabilities)) 10911 goto end_unlock; 10912 10913 if (is_recovery_mode_reported) { 10914 /* we're staying in recovery mode so we'll reinitialize 10915 * misc vector here 10916 */ 10917 if (i40e_setup_misc_vector_for_recovery_mode(pf)) 10918 goto end_unlock; 10919 } else { 10920 if (!lock_acquired) 10921 rtnl_lock(); 10922 /* we're going out of recovery mode so we'll free 10923 * the IRQ allocated specifically for recovery mode 10924 * and restore the interrupt scheme 10925 */ 10926 free_irq(pf->pdev->irq, pf); 10927 i40e_clear_interrupt_scheme(pf); 10928 if (i40e_restore_interrupt_scheme(pf)) 10929 goto end_unlock; 10930 } 10931 10932 /* tell the firmware that we're starting */ 10933 i40e_send_version(pf); 10934 10935 /* bail out in case recovery mode was detected, as there is 10936 * no need for further configuration. 10937 */ 10938 goto end_unlock; 10939 } 10940 10941 i40e_clear_pxe_mode(hw); 10942 ret = i40e_get_capabilities(pf, i40e_aqc_opc_list_func_capabilities); 10943 if (ret) 10944 goto end_core_reset; 10945 10946 ret = i40e_init_lan_hmc(hw, hw->func_caps.num_tx_qp, 10947 hw->func_caps.num_rx_qp, 0, 0); 10948 if (ret) { 10949 dev_info(&pf->pdev->dev, "init_lan_hmc failed: %d\n", ret); 10950 goto end_core_reset; 10951 } 10952 ret = i40e_configure_lan_hmc(hw, I40E_HMC_MODEL_DIRECT_ONLY); 10953 if (ret) { 10954 dev_info(&pf->pdev->dev, "configure_lan_hmc failed: %d\n", ret); 10955 goto end_core_reset; 10956 } 10957 10958 #ifdef CONFIG_I40E_DCB 10959 /* Enable FW to write a default DCB config on link-up 10960 * unless I40E_FLAG_TC_MQPRIO was enabled or DCB 10961 * is not supported with new link speed 10962 */ 10963 if (i40e_is_tc_mqprio_enabled(pf)) { 10964 i40e_aq_set_dcb_parameters(hw, false, NULL); 10965 } else { 10966 if (I40E_IS_X710TL_DEVICE(hw->device_id) && 10967 (hw->phy.link_info.link_speed & 10968 (I40E_LINK_SPEED_2_5GB | I40E_LINK_SPEED_5GB))) { 10969 i40e_aq_set_dcb_parameters(hw, false, NULL); 10970 dev_warn(&pf->pdev->dev, 10971 "DCB is not supported for X710-T*L 2.5/5G speeds\n"); 10972 clear_bit(I40E_FLAG_DCB_CAPABLE, pf->flags); 10973 } else { 10974 i40e_aq_set_dcb_parameters(hw, true, NULL); 10975 ret = i40e_init_pf_dcb(pf); 10976 if (ret) { 10977 dev_info(&pf->pdev->dev, "DCB init failed %d, disabled\n", 10978 ret); 10979 clear_bit(I40E_FLAG_DCB_CAPABLE, pf->flags); 10980 /* Continue without DCB enabled */ 10981 } 10982 } 10983 } 10984 10985 #endif /* CONFIG_I40E_DCB */ 10986 if (!lock_acquired) 10987 rtnl_lock(); 10988 ret = i40e_setup_pf_switch(pf, reinit, true); 10989 if (ret) 10990 goto end_unlock; 10991 10992 /* The driver only wants link up/down and module qualification 10993 * reports from firmware. Note the negative logic. 10994 */ 10995 ret = i40e_aq_set_phy_int_mask(&pf->hw, 10996 ~(I40E_AQ_EVENT_LINK_UPDOWN | 10997 I40E_AQ_EVENT_MEDIA_NA | 10998 I40E_AQ_EVENT_MODULE_QUAL_FAIL), NULL); 10999 if (ret) 11000 dev_info(&pf->pdev->dev, "set phy mask fail, err %pe aq_err %s\n", 11001 ERR_PTR(ret), 11002 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 11003 11004 /* Rebuild the VSIs and VEBs that existed before reset. 11005 * They are still in our local switch element arrays, so only 11006 * need to rebuild the switch model in the HW. 11007 * 11008 * If there were VEBs but the reconstitution failed, we'll try 11009 * to recover minimal use by getting the basic PF VSI working. 11010 */ 11011 if (vsi->uplink_seid != pf->mac_seid) { 11012 dev_dbg(&pf->pdev->dev, "attempting to rebuild switch\n"); 11013 11014 /* Rebuild VEBs */ 11015 i40e_pf_for_each_veb(pf, v, veb) { 11016 ret = i40e_reconstitute_veb(veb); 11017 if (!ret) 11018 continue; 11019 11020 /* If Main VEB failed, we're in deep doodoo, 11021 * so give up rebuilding the switch and set up 11022 * for minimal rebuild of PF VSI. 11023 * If orphan failed, we'll report the error 11024 * but try to keep going. 11025 */ 11026 if (veb->uplink_seid == pf->mac_seid) { 11027 dev_info(&pf->pdev->dev, 11028 "rebuild of switch failed: %d, will try to set up simple PF connection\n", 11029 ret); 11030 vsi->uplink_seid = pf->mac_seid; 11031 break; 11032 } else if (veb->uplink_seid == 0) { 11033 dev_info(&pf->pdev->dev, 11034 "rebuild of orphan VEB failed: %d\n", 11035 ret); 11036 } 11037 } 11038 } 11039 11040 if (vsi->uplink_seid == pf->mac_seid) { 11041 dev_dbg(&pf->pdev->dev, "attempting to rebuild PF VSI\n"); 11042 /* no VEB, so rebuild only the Main VSI */ 11043 ret = i40e_add_vsi(vsi); 11044 if (ret) { 11045 dev_info(&pf->pdev->dev, 11046 "rebuild of Main VSI failed: %d\n", ret); 11047 goto end_unlock; 11048 } 11049 } 11050 11051 if (vsi->mqprio_qopt.max_rate[0]) { 11052 u64 max_tx_rate = i40e_bw_bytes_to_mbits(vsi, 11053 vsi->mqprio_qopt.max_rate[0]); 11054 u64 credits = 0; 11055 11056 ret = i40e_set_bw_limit(vsi, vsi->seid, max_tx_rate); 11057 if (ret) 11058 goto end_unlock; 11059 11060 credits = max_tx_rate; 11061 do_div(credits, I40E_BW_CREDIT_DIVISOR); 11062 dev_dbg(&vsi->back->pdev->dev, 11063 "Set tx rate of %llu Mbps (count of 50Mbps %llu) for vsi->seid %u\n", 11064 max_tx_rate, 11065 credits, 11066 vsi->seid); 11067 } 11068 11069 ret = i40e_rebuild_cloud_filters(vsi, vsi->seid); 11070 if (ret) 11071 goto end_unlock; 11072 11073 /* PF Main VSI is rebuild by now, go ahead and rebuild channel VSIs 11074 * for this main VSI if they exist 11075 */ 11076 ret = i40e_rebuild_channels(vsi); 11077 if (ret) 11078 goto end_unlock; 11079 11080 /* Reconfigure hardware for allowing smaller MSS in the case 11081 * of TSO, so that we avoid the MDD being fired and causing 11082 * a reset in the case of small MSS+TSO. 11083 */ 11084 #define I40E_REG_MSS 0x000E64DC 11085 #define I40E_REG_MSS_MIN_MASK 0x3FF0000 11086 #define I40E_64BYTE_MSS 0x400000 11087 val = rd32(hw, I40E_REG_MSS); 11088 if ((val & I40E_REG_MSS_MIN_MASK) > I40E_64BYTE_MSS) { 11089 val &= ~I40E_REG_MSS_MIN_MASK; 11090 val |= I40E_64BYTE_MSS; 11091 wr32(hw, I40E_REG_MSS, val); 11092 } 11093 11094 if (test_bit(I40E_HW_CAP_RESTART_AUTONEG, pf->hw.caps)) { 11095 msleep(75); 11096 ret = i40e_aq_set_link_restart_an(&pf->hw, true, NULL); 11097 if (ret) 11098 dev_info(&pf->pdev->dev, "link restart failed, err %pe aq_err %s\n", 11099 ERR_PTR(ret), 11100 i40e_aq_str(&pf->hw, 11101 pf->hw.aq.asq_last_status)); 11102 } 11103 /* reinit the misc interrupt */ 11104 if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags)) { 11105 ret = i40e_setup_misc_vector(pf); 11106 if (ret) 11107 goto end_unlock; 11108 } 11109 11110 /* Add a filter to drop all Flow control frames from any VSI from being 11111 * transmitted. By doing so we stop a malicious VF from sending out 11112 * PAUSE or PFC frames and potentially controlling traffic for other 11113 * PF/VF VSIs. 11114 * The FW can still send Flow control frames if enabled. 11115 */ 11116 i40e_add_filter_to_drop_tx_flow_control_frames(&pf->hw, 11117 pf->main_vsi_seid); 11118 11119 /* restart the VSIs that were rebuilt and running before the reset */ 11120 i40e_pf_unquiesce_all_vsi(pf); 11121 11122 /* Release the RTNL lock before we start resetting VFs */ 11123 if (!lock_acquired) 11124 rtnl_unlock(); 11125 11126 /* Restore promiscuous settings */ 11127 ret = i40e_set_promiscuous(pf, pf->cur_promisc); 11128 if (ret) 11129 dev_warn(&pf->pdev->dev, 11130 "Failed to restore promiscuous setting: %s, err %pe aq_err %s\n", 11131 pf->cur_promisc ? "on" : "off", 11132 ERR_PTR(ret), 11133 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 11134 11135 i40e_reset_all_vfs(pf, true); 11136 11137 /* tell the firmware that we're starting */ 11138 i40e_send_version(pf); 11139 11140 /* We've already released the lock, so don't do it again */ 11141 goto end_core_reset; 11142 11143 end_unlock: 11144 if (!lock_acquired) 11145 rtnl_unlock(); 11146 end_core_reset: 11147 clear_bit(__I40E_RESET_FAILED, pf->state); 11148 clear_recovery: 11149 clear_bit(__I40E_RESET_RECOVERY_PENDING, pf->state); 11150 clear_bit(__I40E_TIMEOUT_RECOVERY_PENDING, pf->state); 11151 } 11152 11153 /** 11154 * i40e_reset_and_rebuild - reset and rebuild using a saved config 11155 * @pf: board private structure 11156 * @reinit: if the Main VSI needs to re-initialized. 11157 * @lock_acquired: indicates whether or not the lock has been acquired 11158 * before this function was called. 11159 **/ 11160 static void i40e_reset_and_rebuild(struct i40e_pf *pf, bool reinit, 11161 bool lock_acquired) 11162 { 11163 int ret; 11164 11165 if (test_bit(__I40E_IN_REMOVE, pf->state)) 11166 return; 11167 /* Now we wait for GRST to settle out. 11168 * We don't have to delete the VEBs or VSIs from the hw switch 11169 * because the reset will make them disappear. 11170 */ 11171 ret = i40e_reset(pf); 11172 if (!ret) 11173 i40e_rebuild(pf, reinit, lock_acquired); 11174 else 11175 dev_err(&pf->pdev->dev, "%s: i40e_reset() FAILED", __func__); 11176 } 11177 11178 /** 11179 * i40e_handle_reset_warning - prep for the PF to reset, reset and rebuild 11180 * @pf: board private structure 11181 * 11182 * Close up the VFs and other things in prep for a Core Reset, 11183 * then get ready to rebuild the world. 11184 * @lock_acquired: indicates whether or not the lock has been acquired 11185 * before this function was called. 11186 **/ 11187 static void i40e_handle_reset_warning(struct i40e_pf *pf, bool lock_acquired) 11188 { 11189 i40e_prep_for_reset(pf); 11190 i40e_reset_and_rebuild(pf, false, lock_acquired); 11191 } 11192 11193 /** 11194 * i40e_handle_mdd_event 11195 * @pf: pointer to the PF structure 11196 * 11197 * Called from the MDD irq handler to identify possibly malicious vfs 11198 **/ 11199 static void i40e_handle_mdd_event(struct i40e_pf *pf) 11200 { 11201 struct i40e_hw *hw = &pf->hw; 11202 bool mdd_detected = false; 11203 struct i40e_vf *vf; 11204 u32 reg; 11205 int i; 11206 11207 if (!test_bit(__I40E_MDD_EVENT_PENDING, pf->state)) 11208 return; 11209 11210 /* find what triggered the MDD event */ 11211 reg = rd32(hw, I40E_GL_MDET_TX); 11212 if (reg & I40E_GL_MDET_TX_VALID_MASK) { 11213 u8 pf_num = FIELD_GET(I40E_GL_MDET_TX_PF_NUM_MASK, reg); 11214 u16 vf_num = FIELD_GET(I40E_GL_MDET_TX_VF_NUM_MASK, reg); 11215 u8 event = FIELD_GET(I40E_GL_MDET_TX_EVENT_MASK, reg); 11216 u16 queue = FIELD_GET(I40E_GL_MDET_TX_QUEUE_MASK, reg) - 11217 pf->hw.func_caps.base_queue; 11218 if (netif_msg_tx_err(pf)) 11219 dev_info(&pf->pdev->dev, "Malicious Driver Detection event 0x%02x on TX queue %d PF number 0x%02x VF number 0x%02x\n", 11220 event, queue, pf_num, vf_num); 11221 wr32(hw, I40E_GL_MDET_TX, 0xffffffff); 11222 mdd_detected = true; 11223 } 11224 reg = rd32(hw, I40E_GL_MDET_RX); 11225 if (reg & I40E_GL_MDET_RX_VALID_MASK) { 11226 u8 func = FIELD_GET(I40E_GL_MDET_RX_FUNCTION_MASK, reg); 11227 u8 event = FIELD_GET(I40E_GL_MDET_RX_EVENT_MASK, reg); 11228 u16 queue = FIELD_GET(I40E_GL_MDET_RX_QUEUE_MASK, reg) - 11229 pf->hw.func_caps.base_queue; 11230 if (netif_msg_rx_err(pf)) 11231 dev_info(&pf->pdev->dev, "Malicious Driver Detection event 0x%02x on RX queue %d of function 0x%02x\n", 11232 event, queue, func); 11233 wr32(hw, I40E_GL_MDET_RX, 0xffffffff); 11234 mdd_detected = true; 11235 } 11236 11237 if (mdd_detected) { 11238 reg = rd32(hw, I40E_PF_MDET_TX); 11239 if (reg & I40E_PF_MDET_TX_VALID_MASK) { 11240 wr32(hw, I40E_PF_MDET_TX, 0xFFFF); 11241 dev_dbg(&pf->pdev->dev, "TX driver issue detected on PF\n"); 11242 } 11243 reg = rd32(hw, I40E_PF_MDET_RX); 11244 if (reg & I40E_PF_MDET_RX_VALID_MASK) { 11245 wr32(hw, I40E_PF_MDET_RX, 0xFFFF); 11246 dev_dbg(&pf->pdev->dev, "RX driver issue detected on PF\n"); 11247 } 11248 } 11249 11250 /* see if one of the VFs needs its hand slapped */ 11251 for (i = 0; i < pf->num_alloc_vfs && mdd_detected; i++) { 11252 vf = &(pf->vf[i]); 11253 reg = rd32(hw, I40E_VP_MDET_TX(i)); 11254 if (reg & I40E_VP_MDET_TX_VALID_MASK) { 11255 wr32(hw, I40E_VP_MDET_TX(i), 0xFFFF); 11256 vf->num_mdd_events++; 11257 dev_info(&pf->pdev->dev, "TX driver issue detected on VF %d\n", 11258 i); 11259 dev_info(&pf->pdev->dev, 11260 "Use PF Control I/F to re-enable the VF\n"); 11261 set_bit(I40E_VF_STATE_DISABLED, &vf->vf_states); 11262 } 11263 11264 reg = rd32(hw, I40E_VP_MDET_RX(i)); 11265 if (reg & I40E_VP_MDET_RX_VALID_MASK) { 11266 wr32(hw, I40E_VP_MDET_RX(i), 0xFFFF); 11267 vf->num_mdd_events++; 11268 dev_info(&pf->pdev->dev, "RX driver issue detected on VF %d\n", 11269 i); 11270 dev_info(&pf->pdev->dev, 11271 "Use PF Control I/F to re-enable the VF\n"); 11272 set_bit(I40E_VF_STATE_DISABLED, &vf->vf_states); 11273 } 11274 } 11275 11276 /* re-enable mdd interrupt cause */ 11277 clear_bit(__I40E_MDD_EVENT_PENDING, pf->state); 11278 reg = rd32(hw, I40E_PFINT_ICR0_ENA); 11279 reg |= I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK; 11280 wr32(hw, I40E_PFINT_ICR0_ENA, reg); 11281 i40e_flush(hw); 11282 } 11283 11284 /** 11285 * i40e_service_task - Run the driver's async subtasks 11286 * @work: pointer to work_struct containing our data 11287 **/ 11288 static void i40e_service_task(struct work_struct *work) 11289 { 11290 struct i40e_pf *pf = container_of(work, 11291 struct i40e_pf, 11292 service_task); 11293 unsigned long start_time = jiffies; 11294 11295 /* don't bother with service tasks if a reset is in progress */ 11296 if (test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state) || 11297 test_bit(__I40E_SUSPENDED, pf->state)) 11298 return; 11299 11300 if (test_and_set_bit(__I40E_SERVICE_SCHED, pf->state)) 11301 return; 11302 11303 if (!test_bit(__I40E_RECOVERY_MODE, pf->state)) { 11304 i40e_detect_recover_hung(pf); 11305 i40e_sync_filters_subtask(pf); 11306 i40e_reset_subtask(pf); 11307 i40e_handle_mdd_event(pf); 11308 i40e_vc_process_vflr_event(pf); 11309 i40e_watchdog_subtask(pf); 11310 i40e_fdir_reinit_subtask(pf); 11311 if (test_and_clear_bit(__I40E_CLIENT_RESET, pf->state)) { 11312 /* Client subtask will reopen next time through. */ 11313 i40e_notify_client_of_netdev_close(pf, true); 11314 } else { 11315 i40e_client_subtask(pf); 11316 if (test_and_clear_bit(__I40E_CLIENT_L2_CHANGE, 11317 pf->state)) 11318 i40e_notify_client_of_l2_param_changes(pf); 11319 } 11320 i40e_sync_filters_subtask(pf); 11321 } else { 11322 i40e_reset_subtask(pf); 11323 } 11324 11325 i40e_clean_adminq_subtask(pf); 11326 11327 /* flush memory to make sure state is correct before next watchdog */ 11328 smp_mb__before_atomic(); 11329 clear_bit(__I40E_SERVICE_SCHED, pf->state); 11330 11331 /* If the tasks have taken longer than one timer cycle or there 11332 * is more work to be done, reschedule the service task now 11333 * rather than wait for the timer to tick again. 11334 */ 11335 if (time_after(jiffies, (start_time + pf->service_timer_period)) || 11336 test_bit(__I40E_ADMINQ_EVENT_PENDING, pf->state) || 11337 test_bit(__I40E_MDD_EVENT_PENDING, pf->state) || 11338 test_bit(__I40E_VFLR_EVENT_PENDING, pf->state)) 11339 i40e_service_event_schedule(pf); 11340 } 11341 11342 /** 11343 * i40e_service_timer - timer callback 11344 * @t: timer list pointer 11345 **/ 11346 static void i40e_service_timer(struct timer_list *t) 11347 { 11348 struct i40e_pf *pf = from_timer(pf, t, service_timer); 11349 11350 mod_timer(&pf->service_timer, 11351 round_jiffies(jiffies + pf->service_timer_period)); 11352 i40e_service_event_schedule(pf); 11353 } 11354 11355 /** 11356 * i40e_set_num_rings_in_vsi - Determine number of rings in the VSI 11357 * @vsi: the VSI being configured 11358 **/ 11359 static int i40e_set_num_rings_in_vsi(struct i40e_vsi *vsi) 11360 { 11361 struct i40e_pf *pf = vsi->back; 11362 11363 switch (vsi->type) { 11364 case I40E_VSI_MAIN: 11365 vsi->alloc_queue_pairs = pf->num_lan_qps; 11366 if (!vsi->num_tx_desc) 11367 vsi->num_tx_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS, 11368 I40E_REQ_DESCRIPTOR_MULTIPLE); 11369 if (!vsi->num_rx_desc) 11370 vsi->num_rx_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS, 11371 I40E_REQ_DESCRIPTOR_MULTIPLE); 11372 if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags)) 11373 vsi->num_q_vectors = pf->num_lan_msix; 11374 else 11375 vsi->num_q_vectors = 1; 11376 11377 break; 11378 11379 case I40E_VSI_FDIR: 11380 vsi->alloc_queue_pairs = 1; 11381 vsi->num_tx_desc = ALIGN(I40E_FDIR_RING_COUNT, 11382 I40E_REQ_DESCRIPTOR_MULTIPLE); 11383 vsi->num_rx_desc = ALIGN(I40E_FDIR_RING_COUNT, 11384 I40E_REQ_DESCRIPTOR_MULTIPLE); 11385 vsi->num_q_vectors = pf->num_fdsb_msix; 11386 break; 11387 11388 case I40E_VSI_VMDQ2: 11389 vsi->alloc_queue_pairs = pf->num_vmdq_qps; 11390 if (!vsi->num_tx_desc) 11391 vsi->num_tx_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS, 11392 I40E_REQ_DESCRIPTOR_MULTIPLE); 11393 if (!vsi->num_rx_desc) 11394 vsi->num_rx_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS, 11395 I40E_REQ_DESCRIPTOR_MULTIPLE); 11396 vsi->num_q_vectors = pf->num_vmdq_msix; 11397 break; 11398 11399 case I40E_VSI_SRIOV: 11400 vsi->alloc_queue_pairs = pf->num_vf_qps; 11401 if (!vsi->num_tx_desc) 11402 vsi->num_tx_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS, 11403 I40E_REQ_DESCRIPTOR_MULTIPLE); 11404 if (!vsi->num_rx_desc) 11405 vsi->num_rx_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS, 11406 I40E_REQ_DESCRIPTOR_MULTIPLE); 11407 break; 11408 11409 default: 11410 WARN_ON(1); 11411 return -ENODATA; 11412 } 11413 11414 if (is_kdump_kernel()) { 11415 vsi->num_tx_desc = I40E_MIN_NUM_DESCRIPTORS; 11416 vsi->num_rx_desc = I40E_MIN_NUM_DESCRIPTORS; 11417 } 11418 11419 return 0; 11420 } 11421 11422 /** 11423 * i40e_vsi_alloc_arrays - Allocate queue and vector pointer arrays for the vsi 11424 * @vsi: VSI pointer 11425 * @alloc_qvectors: a bool to specify if q_vectors need to be allocated. 11426 * 11427 * On error: returns error code (negative) 11428 * On success: returns 0 11429 **/ 11430 static int i40e_vsi_alloc_arrays(struct i40e_vsi *vsi, bool alloc_qvectors) 11431 { 11432 struct i40e_ring **next_rings; 11433 int size; 11434 int ret = 0; 11435 11436 /* allocate memory for both Tx, XDP Tx and Rx ring pointers */ 11437 size = sizeof(struct i40e_ring *) * vsi->alloc_queue_pairs * 11438 (i40e_enabled_xdp_vsi(vsi) ? 3 : 2); 11439 vsi->tx_rings = kzalloc(size, GFP_KERNEL); 11440 if (!vsi->tx_rings) 11441 return -ENOMEM; 11442 next_rings = vsi->tx_rings + vsi->alloc_queue_pairs; 11443 if (i40e_enabled_xdp_vsi(vsi)) { 11444 vsi->xdp_rings = next_rings; 11445 next_rings += vsi->alloc_queue_pairs; 11446 } 11447 vsi->rx_rings = next_rings; 11448 11449 if (alloc_qvectors) { 11450 /* allocate memory for q_vector pointers */ 11451 size = sizeof(struct i40e_q_vector *) * vsi->num_q_vectors; 11452 vsi->q_vectors = kzalloc(size, GFP_KERNEL); 11453 if (!vsi->q_vectors) { 11454 ret = -ENOMEM; 11455 goto err_vectors; 11456 } 11457 } 11458 return ret; 11459 11460 err_vectors: 11461 kfree(vsi->tx_rings); 11462 return ret; 11463 } 11464 11465 /** 11466 * i40e_vsi_mem_alloc - Allocates the next available struct vsi in the PF 11467 * @pf: board private structure 11468 * @type: type of VSI 11469 * 11470 * On error: returns error code (negative) 11471 * On success: returns vsi index in PF (positive) 11472 **/ 11473 static int i40e_vsi_mem_alloc(struct i40e_pf *pf, enum i40e_vsi_type type) 11474 { 11475 int ret = -ENODEV; 11476 struct i40e_vsi *vsi; 11477 int vsi_idx; 11478 int i; 11479 11480 /* Need to protect the allocation of the VSIs at the PF level */ 11481 mutex_lock(&pf->switch_mutex); 11482 11483 /* VSI list may be fragmented if VSI creation/destruction has 11484 * been happening. We can afford to do a quick scan to look 11485 * for any free VSIs in the list. 11486 * 11487 * find next empty vsi slot, looping back around if necessary 11488 */ 11489 i = pf->next_vsi; 11490 while (i < pf->num_alloc_vsi && pf->vsi[i]) 11491 i++; 11492 if (i >= pf->num_alloc_vsi) { 11493 i = 0; 11494 while (i < pf->next_vsi && pf->vsi[i]) 11495 i++; 11496 } 11497 11498 if (i < pf->num_alloc_vsi && !pf->vsi[i]) { 11499 vsi_idx = i; /* Found one! */ 11500 } else { 11501 ret = -ENODEV; 11502 goto unlock_pf; /* out of VSI slots! */ 11503 } 11504 pf->next_vsi = ++i; 11505 11506 vsi = kzalloc(sizeof(*vsi), GFP_KERNEL); 11507 if (!vsi) { 11508 ret = -ENOMEM; 11509 goto unlock_pf; 11510 } 11511 vsi->type = type; 11512 vsi->back = pf; 11513 set_bit(__I40E_VSI_DOWN, vsi->state); 11514 vsi->flags = 0; 11515 vsi->idx = vsi_idx; 11516 vsi->int_rate_limit = 0; 11517 vsi->rss_table_size = (vsi->type == I40E_VSI_MAIN) ? 11518 pf->rss_table_size : 64; 11519 vsi->netdev_registered = false; 11520 vsi->work_limit = I40E_DEFAULT_IRQ_WORK; 11521 hash_init(vsi->mac_filter_hash); 11522 vsi->irqs_ready = false; 11523 11524 if (type == I40E_VSI_MAIN) { 11525 vsi->af_xdp_zc_qps = bitmap_zalloc(pf->num_lan_qps, GFP_KERNEL); 11526 if (!vsi->af_xdp_zc_qps) 11527 goto err_rings; 11528 } 11529 11530 ret = i40e_set_num_rings_in_vsi(vsi); 11531 if (ret) 11532 goto err_rings; 11533 11534 ret = i40e_vsi_alloc_arrays(vsi, true); 11535 if (ret) 11536 goto err_rings; 11537 11538 /* Setup default MSIX irq handler for VSI */ 11539 i40e_vsi_setup_irqhandler(vsi, i40e_msix_clean_rings); 11540 11541 /* Initialize VSI lock */ 11542 spin_lock_init(&vsi->mac_filter_hash_lock); 11543 pf->vsi[vsi_idx] = vsi; 11544 ret = vsi_idx; 11545 goto unlock_pf; 11546 11547 err_rings: 11548 bitmap_free(vsi->af_xdp_zc_qps); 11549 pf->next_vsi = i - 1; 11550 kfree(vsi); 11551 unlock_pf: 11552 mutex_unlock(&pf->switch_mutex); 11553 return ret; 11554 } 11555 11556 /** 11557 * i40e_vsi_free_arrays - Free queue and vector pointer arrays for the VSI 11558 * @vsi: VSI pointer 11559 * @free_qvectors: a bool to specify if q_vectors need to be freed. 11560 * 11561 * On error: returns error code (negative) 11562 * On success: returns 0 11563 **/ 11564 static void i40e_vsi_free_arrays(struct i40e_vsi *vsi, bool free_qvectors) 11565 { 11566 /* free the ring and vector containers */ 11567 if (free_qvectors) { 11568 kfree(vsi->q_vectors); 11569 vsi->q_vectors = NULL; 11570 } 11571 kfree(vsi->tx_rings); 11572 vsi->tx_rings = NULL; 11573 vsi->rx_rings = NULL; 11574 vsi->xdp_rings = NULL; 11575 } 11576 11577 /** 11578 * i40e_clear_rss_config_user - clear the user configured RSS hash keys 11579 * and lookup table 11580 * @vsi: Pointer to VSI structure 11581 */ 11582 static void i40e_clear_rss_config_user(struct i40e_vsi *vsi) 11583 { 11584 if (!vsi) 11585 return; 11586 11587 kfree(vsi->rss_hkey_user); 11588 vsi->rss_hkey_user = NULL; 11589 11590 kfree(vsi->rss_lut_user); 11591 vsi->rss_lut_user = NULL; 11592 } 11593 11594 /** 11595 * i40e_vsi_clear - Deallocate the VSI provided 11596 * @vsi: the VSI being un-configured 11597 **/ 11598 static int i40e_vsi_clear(struct i40e_vsi *vsi) 11599 { 11600 struct i40e_pf *pf; 11601 11602 if (!vsi) 11603 return 0; 11604 11605 if (!vsi->back) 11606 goto free_vsi; 11607 pf = vsi->back; 11608 11609 mutex_lock(&pf->switch_mutex); 11610 if (!pf->vsi[vsi->idx]) { 11611 dev_err(&pf->pdev->dev, "pf->vsi[%d] is NULL, just free vsi[%d](type %d)\n", 11612 vsi->idx, vsi->idx, vsi->type); 11613 goto unlock_vsi; 11614 } 11615 11616 if (pf->vsi[vsi->idx] != vsi) { 11617 dev_err(&pf->pdev->dev, 11618 "pf->vsi[%d](type %d) != vsi[%d](type %d): no free!\n", 11619 pf->vsi[vsi->idx]->idx, 11620 pf->vsi[vsi->idx]->type, 11621 vsi->idx, vsi->type); 11622 goto unlock_vsi; 11623 } 11624 11625 /* updates the PF for this cleared vsi */ 11626 i40e_put_lump(pf->qp_pile, vsi->base_queue, vsi->idx); 11627 i40e_put_lump(pf->irq_pile, vsi->base_vector, vsi->idx); 11628 11629 bitmap_free(vsi->af_xdp_zc_qps); 11630 i40e_vsi_free_arrays(vsi, true); 11631 i40e_clear_rss_config_user(vsi); 11632 11633 pf->vsi[vsi->idx] = NULL; 11634 if (vsi->idx < pf->next_vsi) 11635 pf->next_vsi = vsi->idx; 11636 11637 unlock_vsi: 11638 mutex_unlock(&pf->switch_mutex); 11639 free_vsi: 11640 kfree(vsi); 11641 11642 return 0; 11643 } 11644 11645 /** 11646 * i40e_vsi_clear_rings - Deallocates the Rx and Tx rings for the provided VSI 11647 * @vsi: the VSI being cleaned 11648 **/ 11649 static void i40e_vsi_clear_rings(struct i40e_vsi *vsi) 11650 { 11651 int i; 11652 11653 if (vsi->tx_rings && vsi->tx_rings[0]) { 11654 for (i = 0; i < vsi->alloc_queue_pairs; i++) { 11655 kfree_rcu(vsi->tx_rings[i], rcu); 11656 WRITE_ONCE(vsi->tx_rings[i], NULL); 11657 WRITE_ONCE(vsi->rx_rings[i], NULL); 11658 if (vsi->xdp_rings) 11659 WRITE_ONCE(vsi->xdp_rings[i], NULL); 11660 } 11661 } 11662 } 11663 11664 /** 11665 * i40e_alloc_rings - Allocates the Rx and Tx rings for the provided VSI 11666 * @vsi: the VSI being configured 11667 **/ 11668 static int i40e_alloc_rings(struct i40e_vsi *vsi) 11669 { 11670 int i, qpv = i40e_enabled_xdp_vsi(vsi) ? 3 : 2; 11671 struct i40e_pf *pf = vsi->back; 11672 struct i40e_ring *ring; 11673 11674 /* Set basic values in the rings to be used later during open() */ 11675 for (i = 0; i < vsi->alloc_queue_pairs; i++) { 11676 /* allocate space for both Tx and Rx in one shot */ 11677 ring = kcalloc(qpv, sizeof(struct i40e_ring), GFP_KERNEL); 11678 if (!ring) 11679 goto err_out; 11680 11681 ring->queue_index = i; 11682 ring->reg_idx = vsi->base_queue + i; 11683 ring->ring_active = false; 11684 ring->vsi = vsi; 11685 ring->netdev = vsi->netdev; 11686 ring->dev = &pf->pdev->dev; 11687 ring->count = vsi->num_tx_desc; 11688 ring->size = 0; 11689 ring->dcb_tc = 0; 11690 if (test_bit(I40E_HW_CAP_WB_ON_ITR, vsi->back->hw.caps)) 11691 ring->flags = I40E_TXR_FLAGS_WB_ON_ITR; 11692 ring->itr_setting = pf->tx_itr_default; 11693 WRITE_ONCE(vsi->tx_rings[i], ring++); 11694 11695 if (!i40e_enabled_xdp_vsi(vsi)) 11696 goto setup_rx; 11697 11698 ring->queue_index = vsi->alloc_queue_pairs + i; 11699 ring->reg_idx = vsi->base_queue + ring->queue_index; 11700 ring->ring_active = false; 11701 ring->vsi = vsi; 11702 ring->netdev = NULL; 11703 ring->dev = &pf->pdev->dev; 11704 ring->count = vsi->num_tx_desc; 11705 ring->size = 0; 11706 ring->dcb_tc = 0; 11707 if (test_bit(I40E_HW_CAP_WB_ON_ITR, vsi->back->hw.caps)) 11708 ring->flags = I40E_TXR_FLAGS_WB_ON_ITR; 11709 set_ring_xdp(ring); 11710 ring->itr_setting = pf->tx_itr_default; 11711 WRITE_ONCE(vsi->xdp_rings[i], ring++); 11712 11713 setup_rx: 11714 ring->queue_index = i; 11715 ring->reg_idx = vsi->base_queue + i; 11716 ring->ring_active = false; 11717 ring->vsi = vsi; 11718 ring->netdev = vsi->netdev; 11719 ring->dev = &pf->pdev->dev; 11720 ring->count = vsi->num_rx_desc; 11721 ring->size = 0; 11722 ring->dcb_tc = 0; 11723 ring->itr_setting = pf->rx_itr_default; 11724 WRITE_ONCE(vsi->rx_rings[i], ring); 11725 } 11726 11727 return 0; 11728 11729 err_out: 11730 i40e_vsi_clear_rings(vsi); 11731 return -ENOMEM; 11732 } 11733 11734 /** 11735 * i40e_reserve_msix_vectors - Reserve MSI-X vectors in the kernel 11736 * @pf: board private structure 11737 * @vectors: the number of MSI-X vectors to request 11738 * 11739 * Returns the number of vectors reserved, or error 11740 **/ 11741 static int i40e_reserve_msix_vectors(struct i40e_pf *pf, int vectors) 11742 { 11743 vectors = pci_enable_msix_range(pf->pdev, pf->msix_entries, 11744 I40E_MIN_MSIX, vectors); 11745 if (vectors < 0) { 11746 dev_info(&pf->pdev->dev, 11747 "MSI-X vector reservation failed: %d\n", vectors); 11748 vectors = 0; 11749 } 11750 11751 return vectors; 11752 } 11753 11754 /** 11755 * i40e_init_msix - Setup the MSIX capability 11756 * @pf: board private structure 11757 * 11758 * Work with the OS to set up the MSIX vectors needed. 11759 * 11760 * Returns the number of vectors reserved or negative on failure 11761 **/ 11762 static int i40e_init_msix(struct i40e_pf *pf) 11763 { 11764 struct i40e_hw *hw = &pf->hw; 11765 int cpus, extra_vectors; 11766 int vectors_left; 11767 int v_budget, i; 11768 int v_actual; 11769 int iwarp_requested = 0; 11770 11771 if (!test_bit(I40E_FLAG_MSIX_ENA, pf->flags)) 11772 return -ENODEV; 11773 11774 /* The number of vectors we'll request will be comprised of: 11775 * - Add 1 for "other" cause for Admin Queue events, etc. 11776 * - The number of LAN queue pairs 11777 * - Queues being used for RSS. 11778 * We don't need as many as max_rss_size vectors. 11779 * use rss_size instead in the calculation since that 11780 * is governed by number of cpus in the system. 11781 * - assumes symmetric Tx/Rx pairing 11782 * - The number of VMDq pairs 11783 * - The CPU count within the NUMA node if iWARP is enabled 11784 * Once we count this up, try the request. 11785 * 11786 * If we can't get what we want, we'll simplify to nearly nothing 11787 * and try again. If that still fails, we punt. 11788 */ 11789 vectors_left = hw->func_caps.num_msix_vectors; 11790 v_budget = 0; 11791 11792 /* reserve one vector for miscellaneous handler */ 11793 if (vectors_left) { 11794 v_budget++; 11795 vectors_left--; 11796 } 11797 11798 /* reserve some vectors for the main PF traffic queues. Initially we 11799 * only reserve at most 50% of the available vectors, in the case that 11800 * the number of online CPUs is large. This ensures that we can enable 11801 * extra features as well. Once we've enabled the other features, we 11802 * will use any remaining vectors to reach as close as we can to the 11803 * number of online CPUs. 11804 */ 11805 cpus = num_online_cpus(); 11806 pf->num_lan_msix = min_t(int, cpus, vectors_left / 2); 11807 vectors_left -= pf->num_lan_msix; 11808 11809 /* reserve one vector for sideband flow director */ 11810 if (test_bit(I40E_FLAG_FD_SB_ENA, pf->flags)) { 11811 if (vectors_left) { 11812 pf->num_fdsb_msix = 1; 11813 v_budget++; 11814 vectors_left--; 11815 } else { 11816 pf->num_fdsb_msix = 0; 11817 } 11818 } 11819 11820 /* can we reserve enough for iWARP? */ 11821 if (test_bit(I40E_FLAG_IWARP_ENA, pf->flags)) { 11822 iwarp_requested = pf->num_iwarp_msix; 11823 11824 if (!vectors_left) 11825 pf->num_iwarp_msix = 0; 11826 else if (vectors_left < pf->num_iwarp_msix) 11827 pf->num_iwarp_msix = 1; 11828 v_budget += pf->num_iwarp_msix; 11829 vectors_left -= pf->num_iwarp_msix; 11830 } 11831 11832 /* any vectors left over go for VMDq support */ 11833 if (test_bit(I40E_FLAG_VMDQ_ENA, pf->flags)) { 11834 if (!vectors_left) { 11835 pf->num_vmdq_msix = 0; 11836 pf->num_vmdq_qps = 0; 11837 } else { 11838 int vmdq_vecs_wanted = 11839 pf->num_vmdq_vsis * pf->num_vmdq_qps; 11840 int vmdq_vecs = 11841 min_t(int, vectors_left, vmdq_vecs_wanted); 11842 11843 /* if we're short on vectors for what's desired, we limit 11844 * the queues per vmdq. If this is still more than are 11845 * available, the user will need to change the number of 11846 * queues/vectors used by the PF later with the ethtool 11847 * channels command 11848 */ 11849 if (vectors_left < vmdq_vecs_wanted) { 11850 pf->num_vmdq_qps = 1; 11851 vmdq_vecs_wanted = pf->num_vmdq_vsis; 11852 vmdq_vecs = min_t(int, 11853 vectors_left, 11854 vmdq_vecs_wanted); 11855 } 11856 pf->num_vmdq_msix = pf->num_vmdq_qps; 11857 11858 v_budget += vmdq_vecs; 11859 vectors_left -= vmdq_vecs; 11860 } 11861 } 11862 11863 /* On systems with a large number of SMP cores, we previously limited 11864 * the number of vectors for num_lan_msix to be at most 50% of the 11865 * available vectors, to allow for other features. Now, we add back 11866 * the remaining vectors. However, we ensure that the total 11867 * num_lan_msix will not exceed num_online_cpus(). To do this, we 11868 * calculate the number of vectors we can add without going over the 11869 * cap of CPUs. For systems with a small number of CPUs this will be 11870 * zero. 11871 */ 11872 extra_vectors = min_t(int, cpus - pf->num_lan_msix, vectors_left); 11873 pf->num_lan_msix += extra_vectors; 11874 vectors_left -= extra_vectors; 11875 11876 WARN(vectors_left < 0, 11877 "Calculation of remaining vectors underflowed. This is an accounting bug when determining total MSI-X vectors.\n"); 11878 11879 v_budget += pf->num_lan_msix; 11880 pf->msix_entries = kcalloc(v_budget, sizeof(struct msix_entry), 11881 GFP_KERNEL); 11882 if (!pf->msix_entries) 11883 return -ENOMEM; 11884 11885 for (i = 0; i < v_budget; i++) 11886 pf->msix_entries[i].entry = i; 11887 v_actual = i40e_reserve_msix_vectors(pf, v_budget); 11888 11889 if (v_actual < I40E_MIN_MSIX) { 11890 clear_bit(I40E_FLAG_MSIX_ENA, pf->flags); 11891 kfree(pf->msix_entries); 11892 pf->msix_entries = NULL; 11893 pci_disable_msix(pf->pdev); 11894 return -ENODEV; 11895 11896 } else if (v_actual == I40E_MIN_MSIX) { 11897 /* Adjust for minimal MSIX use */ 11898 pf->num_vmdq_vsis = 0; 11899 pf->num_vmdq_qps = 0; 11900 pf->num_lan_qps = 1; 11901 pf->num_lan_msix = 1; 11902 11903 } else if (v_actual != v_budget) { 11904 /* If we have limited resources, we will start with no vectors 11905 * for the special features and then allocate vectors to some 11906 * of these features based on the policy and at the end disable 11907 * the features that did not get any vectors. 11908 */ 11909 int vec; 11910 11911 dev_info(&pf->pdev->dev, 11912 "MSI-X vector limit reached with %d, wanted %d, attempting to redistribute vectors\n", 11913 v_actual, v_budget); 11914 /* reserve the misc vector */ 11915 vec = v_actual - 1; 11916 11917 /* Scale vector usage down */ 11918 pf->num_vmdq_msix = 1; /* force VMDqs to only one vector */ 11919 pf->num_vmdq_vsis = 1; 11920 pf->num_vmdq_qps = 1; 11921 11922 /* partition out the remaining vectors */ 11923 switch (vec) { 11924 case 2: 11925 pf->num_lan_msix = 1; 11926 break; 11927 case 3: 11928 if (test_bit(I40E_FLAG_IWARP_ENA, pf->flags)) { 11929 pf->num_lan_msix = 1; 11930 pf->num_iwarp_msix = 1; 11931 } else { 11932 pf->num_lan_msix = 2; 11933 } 11934 break; 11935 default: 11936 if (test_bit(I40E_FLAG_IWARP_ENA, pf->flags)) { 11937 pf->num_iwarp_msix = min_t(int, (vec / 3), 11938 iwarp_requested); 11939 pf->num_vmdq_vsis = min_t(int, (vec / 3), 11940 I40E_DEFAULT_NUM_VMDQ_VSI); 11941 } else { 11942 pf->num_vmdq_vsis = min_t(int, (vec / 2), 11943 I40E_DEFAULT_NUM_VMDQ_VSI); 11944 } 11945 if (test_bit(I40E_FLAG_FD_SB_ENA, pf->flags)) { 11946 pf->num_fdsb_msix = 1; 11947 vec--; 11948 } 11949 pf->num_lan_msix = min_t(int, 11950 (vec - (pf->num_iwarp_msix + pf->num_vmdq_vsis)), 11951 pf->num_lan_msix); 11952 pf->num_lan_qps = pf->num_lan_msix; 11953 break; 11954 } 11955 } 11956 11957 if (test_bit(I40E_FLAG_FD_SB_ENA, pf->flags) && pf->num_fdsb_msix == 0) { 11958 dev_info(&pf->pdev->dev, "Sideband Flowdir disabled, not enough MSI-X vectors\n"); 11959 clear_bit(I40E_FLAG_FD_SB_ENA, pf->flags); 11960 set_bit(I40E_FLAG_FD_SB_INACTIVE, pf->flags); 11961 } 11962 if (test_bit(I40E_FLAG_VMDQ_ENA, pf->flags) && pf->num_vmdq_msix == 0) { 11963 dev_info(&pf->pdev->dev, "VMDq disabled, not enough MSI-X vectors\n"); 11964 clear_bit(I40E_FLAG_VMDQ_ENA, pf->flags); 11965 } 11966 11967 if (test_bit(I40E_FLAG_IWARP_ENA, pf->flags) && 11968 pf->num_iwarp_msix == 0) { 11969 dev_info(&pf->pdev->dev, "IWARP disabled, not enough MSI-X vectors\n"); 11970 clear_bit(I40E_FLAG_IWARP_ENA, pf->flags); 11971 } 11972 i40e_debug(&pf->hw, I40E_DEBUG_INIT, 11973 "MSI-X vector distribution: PF %d, VMDq %d, FDSB %d, iWARP %d\n", 11974 pf->num_lan_msix, 11975 pf->num_vmdq_msix * pf->num_vmdq_vsis, 11976 pf->num_fdsb_msix, 11977 pf->num_iwarp_msix); 11978 11979 return v_actual; 11980 } 11981 11982 /** 11983 * i40e_vsi_alloc_q_vector - Allocate memory for a single interrupt vector 11984 * @vsi: the VSI being configured 11985 * @v_idx: index of the vector in the vsi struct 11986 * 11987 * We allocate one q_vector. If allocation fails we return -ENOMEM. 11988 **/ 11989 static int i40e_vsi_alloc_q_vector(struct i40e_vsi *vsi, int v_idx) 11990 { 11991 struct i40e_q_vector *q_vector; 11992 11993 /* allocate q_vector */ 11994 q_vector = kzalloc(sizeof(struct i40e_q_vector), GFP_KERNEL); 11995 if (!q_vector) 11996 return -ENOMEM; 11997 11998 q_vector->vsi = vsi; 11999 q_vector->v_idx = v_idx; 12000 cpumask_copy(&q_vector->affinity_mask, cpu_possible_mask); 12001 12002 if (vsi->netdev) 12003 netif_napi_add(vsi->netdev, &q_vector->napi, i40e_napi_poll); 12004 12005 /* tie q_vector and vsi together */ 12006 vsi->q_vectors[v_idx] = q_vector; 12007 12008 return 0; 12009 } 12010 12011 /** 12012 * i40e_vsi_alloc_q_vectors - Allocate memory for interrupt vectors 12013 * @vsi: the VSI being configured 12014 * 12015 * We allocate one q_vector per queue interrupt. If allocation fails we 12016 * return -ENOMEM. 12017 **/ 12018 static int i40e_vsi_alloc_q_vectors(struct i40e_vsi *vsi) 12019 { 12020 struct i40e_pf *pf = vsi->back; 12021 int err, v_idx, num_q_vectors; 12022 12023 /* if not MSIX, give the one vector only to the LAN VSI */ 12024 if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags)) 12025 num_q_vectors = vsi->num_q_vectors; 12026 else if (vsi->type == I40E_VSI_MAIN) 12027 num_q_vectors = 1; 12028 else 12029 return -EINVAL; 12030 12031 for (v_idx = 0; v_idx < num_q_vectors; v_idx++) { 12032 err = i40e_vsi_alloc_q_vector(vsi, v_idx); 12033 if (err) 12034 goto err_out; 12035 } 12036 12037 return 0; 12038 12039 err_out: 12040 while (v_idx--) 12041 i40e_free_q_vector(vsi, v_idx); 12042 12043 return err; 12044 } 12045 12046 /** 12047 * i40e_init_interrupt_scheme - Determine proper interrupt scheme 12048 * @pf: board private structure to initialize 12049 **/ 12050 static int i40e_init_interrupt_scheme(struct i40e_pf *pf) 12051 { 12052 int vectors = 0; 12053 ssize_t size; 12054 12055 if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags)) { 12056 vectors = i40e_init_msix(pf); 12057 if (vectors < 0) { 12058 clear_bit(I40E_FLAG_MSIX_ENA, pf->flags); 12059 clear_bit(I40E_FLAG_IWARP_ENA, pf->flags); 12060 clear_bit(I40E_FLAG_RSS_ENA, pf->flags); 12061 clear_bit(I40E_FLAG_DCB_CAPABLE, pf->flags); 12062 clear_bit(I40E_FLAG_DCB_ENA, pf->flags); 12063 clear_bit(I40E_FLAG_SRIOV_ENA, pf->flags); 12064 clear_bit(I40E_FLAG_FD_SB_ENA, pf->flags); 12065 clear_bit(I40E_FLAG_FD_ATR_ENA, pf->flags); 12066 clear_bit(I40E_FLAG_VMDQ_ENA, pf->flags); 12067 set_bit(I40E_FLAG_FD_SB_INACTIVE, pf->flags); 12068 12069 /* rework the queue expectations without MSIX */ 12070 i40e_determine_queue_usage(pf); 12071 } 12072 } 12073 12074 if (!test_bit(I40E_FLAG_MSIX_ENA, pf->flags) && 12075 test_bit(I40E_FLAG_MSI_ENA, pf->flags)) { 12076 dev_info(&pf->pdev->dev, "MSI-X not available, trying MSI\n"); 12077 vectors = pci_enable_msi(pf->pdev); 12078 if (vectors < 0) { 12079 dev_info(&pf->pdev->dev, "MSI init failed - %d\n", 12080 vectors); 12081 clear_bit(I40E_FLAG_MSI_ENA, pf->flags); 12082 } 12083 vectors = 1; /* one MSI or Legacy vector */ 12084 } 12085 12086 if (!test_bit(I40E_FLAG_MSI_ENA, pf->flags) && 12087 !test_bit(I40E_FLAG_MSIX_ENA, pf->flags)) 12088 dev_info(&pf->pdev->dev, "MSI-X and MSI not available, falling back to Legacy IRQ\n"); 12089 12090 /* set up vector assignment tracking */ 12091 size = sizeof(struct i40e_lump_tracking) + (sizeof(u16) * vectors); 12092 pf->irq_pile = kzalloc(size, GFP_KERNEL); 12093 if (!pf->irq_pile) 12094 return -ENOMEM; 12095 12096 pf->irq_pile->num_entries = vectors; 12097 12098 /* track first vector for misc interrupts, ignore return */ 12099 (void)i40e_get_lump(pf, pf->irq_pile, 1, I40E_PILE_VALID_BIT - 1); 12100 12101 return 0; 12102 } 12103 12104 /** 12105 * i40e_restore_interrupt_scheme - Restore the interrupt scheme 12106 * @pf: private board data structure 12107 * 12108 * Restore the interrupt scheme that was cleared when we suspended the 12109 * device. This should be called during resume to re-allocate the q_vectors 12110 * and reacquire IRQs. 12111 */ 12112 static int i40e_restore_interrupt_scheme(struct i40e_pf *pf) 12113 { 12114 struct i40e_vsi *vsi; 12115 int err, i; 12116 12117 /* We cleared the MSI and MSI-X flags when disabling the old interrupt 12118 * scheme. We need to re-enabled them here in order to attempt to 12119 * re-acquire the MSI or MSI-X vectors 12120 */ 12121 set_bit(I40E_FLAG_MSI_ENA, pf->flags); 12122 set_bit(I40E_FLAG_MSIX_ENA, pf->flags); 12123 12124 err = i40e_init_interrupt_scheme(pf); 12125 if (err) 12126 return err; 12127 12128 /* Now that we've re-acquired IRQs, we need to remap the vectors and 12129 * rings together again. 12130 */ 12131 i40e_pf_for_each_vsi(pf, i, vsi) { 12132 err = i40e_vsi_alloc_q_vectors(vsi); 12133 if (err) 12134 goto err_unwind; 12135 12136 i40e_vsi_map_rings_to_vectors(vsi); 12137 } 12138 12139 err = i40e_setup_misc_vector(pf); 12140 if (err) 12141 goto err_unwind; 12142 12143 if (test_bit(I40E_FLAG_IWARP_ENA, pf->flags)) 12144 i40e_client_update_msix_info(pf); 12145 12146 return 0; 12147 12148 err_unwind: 12149 while (i--) { 12150 if (pf->vsi[i]) 12151 i40e_vsi_free_q_vectors(pf->vsi[i]); 12152 } 12153 12154 return err; 12155 } 12156 12157 /** 12158 * i40e_setup_misc_vector_for_recovery_mode - Setup the misc vector to handle 12159 * non queue events in recovery mode 12160 * @pf: board private structure 12161 * 12162 * This sets up the handler for MSIX 0 or MSI/legacy, which is used to manage 12163 * the non-queue interrupts, e.g. AdminQ and errors in recovery mode. 12164 * This is handled differently than in recovery mode since no Tx/Rx resources 12165 * are being allocated. 12166 **/ 12167 static int i40e_setup_misc_vector_for_recovery_mode(struct i40e_pf *pf) 12168 { 12169 int err; 12170 12171 if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags)) { 12172 err = i40e_setup_misc_vector(pf); 12173 12174 if (err) { 12175 dev_info(&pf->pdev->dev, 12176 "MSI-X misc vector request failed, error %d\n", 12177 err); 12178 return err; 12179 } 12180 } else { 12181 u32 flags = test_bit(I40E_FLAG_MSI_ENA, pf->flags) ? 0 : IRQF_SHARED; 12182 12183 err = request_irq(pf->pdev->irq, i40e_intr, flags, 12184 pf->int_name, pf); 12185 12186 if (err) { 12187 dev_info(&pf->pdev->dev, 12188 "MSI/legacy misc vector request failed, error %d\n", 12189 err); 12190 return err; 12191 } 12192 i40e_enable_misc_int_causes(pf); 12193 i40e_irq_dynamic_enable_icr0(pf); 12194 } 12195 12196 return 0; 12197 } 12198 12199 /** 12200 * i40e_setup_misc_vector - Setup the misc vector to handle non queue events 12201 * @pf: board private structure 12202 * 12203 * This sets up the handler for MSIX 0, which is used to manage the 12204 * non-queue interrupts, e.g. AdminQ and errors. This is not used 12205 * when in MSI or Legacy interrupt mode. 12206 **/ 12207 static int i40e_setup_misc_vector(struct i40e_pf *pf) 12208 { 12209 struct i40e_hw *hw = &pf->hw; 12210 int err = 0; 12211 12212 /* Only request the IRQ once, the first time through. */ 12213 if (!test_and_set_bit(__I40E_MISC_IRQ_REQUESTED, pf->state)) { 12214 err = request_irq(pf->msix_entries[0].vector, 12215 i40e_intr, 0, pf->int_name, pf); 12216 if (err) { 12217 clear_bit(__I40E_MISC_IRQ_REQUESTED, pf->state); 12218 dev_info(&pf->pdev->dev, 12219 "request_irq for %s failed: %d\n", 12220 pf->int_name, err); 12221 return -EFAULT; 12222 } 12223 } 12224 12225 i40e_enable_misc_int_causes(pf); 12226 12227 /* associate no queues to the misc vector */ 12228 wr32(hw, I40E_PFINT_LNKLST0, I40E_QUEUE_END_OF_LIST); 12229 wr32(hw, I40E_PFINT_ITR0(I40E_RX_ITR), I40E_ITR_8K >> 1); 12230 12231 i40e_flush(hw); 12232 12233 i40e_irq_dynamic_enable_icr0(pf); 12234 12235 return err; 12236 } 12237 12238 /** 12239 * i40e_get_rss_aq - Get RSS keys and lut by using AQ commands 12240 * @vsi: Pointer to vsi structure 12241 * @seed: Buffter to store the hash keys 12242 * @lut: Buffer to store the lookup table entries 12243 * @lut_size: Size of buffer to store the lookup table entries 12244 * 12245 * Return 0 on success, negative on failure 12246 */ 12247 static int i40e_get_rss_aq(struct i40e_vsi *vsi, const u8 *seed, 12248 u8 *lut, u16 lut_size) 12249 { 12250 struct i40e_pf *pf = vsi->back; 12251 struct i40e_hw *hw = &pf->hw; 12252 int ret = 0; 12253 12254 if (seed) { 12255 ret = i40e_aq_get_rss_key(hw, vsi->id, 12256 (struct i40e_aqc_get_set_rss_key_data *)seed); 12257 if (ret) { 12258 dev_info(&pf->pdev->dev, 12259 "Cannot get RSS key, err %pe aq_err %s\n", 12260 ERR_PTR(ret), 12261 i40e_aq_str(&pf->hw, 12262 pf->hw.aq.asq_last_status)); 12263 return ret; 12264 } 12265 } 12266 12267 if (lut) { 12268 bool pf_lut = vsi->type == I40E_VSI_MAIN; 12269 12270 ret = i40e_aq_get_rss_lut(hw, vsi->id, pf_lut, lut, lut_size); 12271 if (ret) { 12272 dev_info(&pf->pdev->dev, 12273 "Cannot get RSS lut, err %pe aq_err %s\n", 12274 ERR_PTR(ret), 12275 i40e_aq_str(&pf->hw, 12276 pf->hw.aq.asq_last_status)); 12277 return ret; 12278 } 12279 } 12280 12281 return ret; 12282 } 12283 12284 /** 12285 * i40e_config_rss_reg - Configure RSS keys and lut by writing registers 12286 * @vsi: Pointer to vsi structure 12287 * @seed: RSS hash seed 12288 * @lut: Lookup table 12289 * @lut_size: Lookup table size 12290 * 12291 * Returns 0 on success, negative on failure 12292 **/ 12293 static int i40e_config_rss_reg(struct i40e_vsi *vsi, const u8 *seed, 12294 const u8 *lut, u16 lut_size) 12295 { 12296 struct i40e_pf *pf = vsi->back; 12297 struct i40e_hw *hw = &pf->hw; 12298 u16 vf_id = vsi->vf_id; 12299 u8 i; 12300 12301 /* Fill out hash function seed */ 12302 if (seed) { 12303 u32 *seed_dw = (u32 *)seed; 12304 12305 if (vsi->type == I40E_VSI_MAIN) { 12306 for (i = 0; i <= I40E_PFQF_HKEY_MAX_INDEX; i++) 12307 wr32(hw, I40E_PFQF_HKEY(i), seed_dw[i]); 12308 } else if (vsi->type == I40E_VSI_SRIOV) { 12309 for (i = 0; i <= I40E_VFQF_HKEY1_MAX_INDEX; i++) 12310 wr32(hw, I40E_VFQF_HKEY1(i, vf_id), seed_dw[i]); 12311 } else { 12312 dev_err(&pf->pdev->dev, "Cannot set RSS seed - invalid VSI type\n"); 12313 } 12314 } 12315 12316 if (lut) { 12317 u32 *lut_dw = (u32 *)lut; 12318 12319 if (vsi->type == I40E_VSI_MAIN) { 12320 if (lut_size != I40E_HLUT_ARRAY_SIZE) 12321 return -EINVAL; 12322 for (i = 0; i <= I40E_PFQF_HLUT_MAX_INDEX; i++) 12323 wr32(hw, I40E_PFQF_HLUT(i), lut_dw[i]); 12324 } else if (vsi->type == I40E_VSI_SRIOV) { 12325 if (lut_size != I40E_VF_HLUT_ARRAY_SIZE) 12326 return -EINVAL; 12327 for (i = 0; i <= I40E_VFQF_HLUT_MAX_INDEX; i++) 12328 wr32(hw, I40E_VFQF_HLUT1(i, vf_id), lut_dw[i]); 12329 } else { 12330 dev_err(&pf->pdev->dev, "Cannot set RSS LUT - invalid VSI type\n"); 12331 } 12332 } 12333 i40e_flush(hw); 12334 12335 return 0; 12336 } 12337 12338 /** 12339 * i40e_get_rss_reg - Get the RSS keys and lut by reading registers 12340 * @vsi: Pointer to VSI structure 12341 * @seed: Buffer to store the keys 12342 * @lut: Buffer to store the lookup table entries 12343 * @lut_size: Size of buffer to store the lookup table entries 12344 * 12345 * Returns 0 on success, negative on failure 12346 */ 12347 static int i40e_get_rss_reg(struct i40e_vsi *vsi, u8 *seed, 12348 u8 *lut, u16 lut_size) 12349 { 12350 struct i40e_pf *pf = vsi->back; 12351 struct i40e_hw *hw = &pf->hw; 12352 u16 i; 12353 12354 if (seed) { 12355 u32 *seed_dw = (u32 *)seed; 12356 12357 for (i = 0; i <= I40E_PFQF_HKEY_MAX_INDEX; i++) 12358 seed_dw[i] = i40e_read_rx_ctl(hw, I40E_PFQF_HKEY(i)); 12359 } 12360 if (lut) { 12361 u32 *lut_dw = (u32 *)lut; 12362 12363 if (lut_size != I40E_HLUT_ARRAY_SIZE) 12364 return -EINVAL; 12365 for (i = 0; i <= I40E_PFQF_HLUT_MAX_INDEX; i++) 12366 lut_dw[i] = rd32(hw, I40E_PFQF_HLUT(i)); 12367 } 12368 12369 return 0; 12370 } 12371 12372 /** 12373 * i40e_config_rss - Configure RSS keys and lut 12374 * @vsi: Pointer to VSI structure 12375 * @seed: RSS hash seed 12376 * @lut: Lookup table 12377 * @lut_size: Lookup table size 12378 * 12379 * Returns 0 on success, negative on failure 12380 */ 12381 int i40e_config_rss(struct i40e_vsi *vsi, u8 *seed, u8 *lut, u16 lut_size) 12382 { 12383 struct i40e_pf *pf = vsi->back; 12384 12385 if (test_bit(I40E_HW_CAP_RSS_AQ, pf->hw.caps)) 12386 return i40e_config_rss_aq(vsi, seed, lut, lut_size); 12387 else 12388 return i40e_config_rss_reg(vsi, seed, lut, lut_size); 12389 } 12390 12391 /** 12392 * i40e_get_rss - Get RSS keys and lut 12393 * @vsi: Pointer to VSI structure 12394 * @seed: Buffer to store the keys 12395 * @lut: Buffer to store the lookup table entries 12396 * @lut_size: Size of buffer to store the lookup table entries 12397 * 12398 * Returns 0 on success, negative on failure 12399 */ 12400 int i40e_get_rss(struct i40e_vsi *vsi, u8 *seed, u8 *lut, u16 lut_size) 12401 { 12402 struct i40e_pf *pf = vsi->back; 12403 12404 if (test_bit(I40E_HW_CAP_RSS_AQ, pf->hw.caps)) 12405 return i40e_get_rss_aq(vsi, seed, lut, lut_size); 12406 else 12407 return i40e_get_rss_reg(vsi, seed, lut, lut_size); 12408 } 12409 12410 /** 12411 * i40e_fill_rss_lut - Fill the RSS lookup table with default values 12412 * @pf: Pointer to board private structure 12413 * @lut: Lookup table 12414 * @rss_table_size: Lookup table size 12415 * @rss_size: Range of queue number for hashing 12416 */ 12417 void i40e_fill_rss_lut(struct i40e_pf *pf, u8 *lut, 12418 u16 rss_table_size, u16 rss_size) 12419 { 12420 u16 i; 12421 12422 for (i = 0; i < rss_table_size; i++) 12423 lut[i] = i % rss_size; 12424 } 12425 12426 /** 12427 * i40e_pf_config_rss - Prepare for RSS if used 12428 * @pf: board private structure 12429 **/ 12430 static int i40e_pf_config_rss(struct i40e_pf *pf) 12431 { 12432 struct i40e_vsi *vsi = i40e_pf_get_main_vsi(pf); 12433 u8 seed[I40E_HKEY_ARRAY_SIZE]; 12434 u8 *lut; 12435 struct i40e_hw *hw = &pf->hw; 12436 u32 reg_val; 12437 u64 hena; 12438 int ret; 12439 12440 /* By default we enable TCP/UDP with IPv4/IPv6 ptypes */ 12441 hena = (u64)i40e_read_rx_ctl(hw, I40E_PFQF_HENA(0)) | 12442 ((u64)i40e_read_rx_ctl(hw, I40E_PFQF_HENA(1)) << 32); 12443 hena |= i40e_pf_get_default_rss_hena(pf); 12444 12445 i40e_write_rx_ctl(hw, I40E_PFQF_HENA(0), (u32)hena); 12446 i40e_write_rx_ctl(hw, I40E_PFQF_HENA(1), (u32)(hena >> 32)); 12447 12448 /* Determine the RSS table size based on the hardware capabilities */ 12449 reg_val = i40e_read_rx_ctl(hw, I40E_PFQF_CTL_0); 12450 reg_val = (pf->rss_table_size == 512) ? 12451 (reg_val | I40E_PFQF_CTL_0_HASHLUTSIZE_512) : 12452 (reg_val & ~I40E_PFQF_CTL_0_HASHLUTSIZE_512); 12453 i40e_write_rx_ctl(hw, I40E_PFQF_CTL_0, reg_val); 12454 12455 /* Determine the RSS size of the VSI */ 12456 if (!vsi->rss_size) { 12457 u16 qcount; 12458 /* If the firmware does something weird during VSI init, we 12459 * could end up with zero TCs. Check for that to avoid 12460 * divide-by-zero. It probably won't pass traffic, but it also 12461 * won't panic. 12462 */ 12463 qcount = vsi->num_queue_pairs / 12464 (vsi->tc_config.numtc ? vsi->tc_config.numtc : 1); 12465 vsi->rss_size = min_t(int, pf->alloc_rss_size, qcount); 12466 } 12467 if (!vsi->rss_size) 12468 return -EINVAL; 12469 12470 lut = kzalloc(vsi->rss_table_size, GFP_KERNEL); 12471 if (!lut) 12472 return -ENOMEM; 12473 12474 /* Use user configured lut if there is one, otherwise use default */ 12475 if (vsi->rss_lut_user) 12476 memcpy(lut, vsi->rss_lut_user, vsi->rss_table_size); 12477 else 12478 i40e_fill_rss_lut(pf, lut, vsi->rss_table_size, vsi->rss_size); 12479 12480 /* Use user configured hash key if there is one, otherwise 12481 * use default. 12482 */ 12483 if (vsi->rss_hkey_user) 12484 memcpy(seed, vsi->rss_hkey_user, I40E_HKEY_ARRAY_SIZE); 12485 else 12486 netdev_rss_key_fill((void *)seed, I40E_HKEY_ARRAY_SIZE); 12487 ret = i40e_config_rss(vsi, seed, lut, vsi->rss_table_size); 12488 kfree(lut); 12489 12490 return ret; 12491 } 12492 12493 /** 12494 * i40e_reconfig_rss_queues - change number of queues for rss and rebuild 12495 * @pf: board private structure 12496 * @queue_count: the requested queue count for rss. 12497 * 12498 * returns 0 if rss is not enabled, if enabled returns the final rss queue 12499 * count which may be different from the requested queue count. 12500 * Note: expects to be called while under rtnl_lock() 12501 **/ 12502 int i40e_reconfig_rss_queues(struct i40e_pf *pf, int queue_count) 12503 { 12504 struct i40e_vsi *vsi = i40e_pf_get_main_vsi(pf); 12505 int new_rss_size; 12506 12507 if (!test_bit(I40E_FLAG_RSS_ENA, pf->flags)) 12508 return 0; 12509 12510 queue_count = min_t(int, queue_count, num_online_cpus()); 12511 new_rss_size = min_t(int, queue_count, pf->rss_size_max); 12512 12513 if (queue_count != vsi->num_queue_pairs) { 12514 u16 qcount; 12515 12516 vsi->req_queue_pairs = queue_count; 12517 i40e_prep_for_reset(pf); 12518 if (test_bit(__I40E_IN_REMOVE, pf->state)) 12519 return pf->alloc_rss_size; 12520 12521 pf->alloc_rss_size = new_rss_size; 12522 12523 i40e_reset_and_rebuild(pf, true, true); 12524 12525 /* Discard the user configured hash keys and lut, if less 12526 * queues are enabled. 12527 */ 12528 if (queue_count < vsi->rss_size) { 12529 i40e_clear_rss_config_user(vsi); 12530 dev_dbg(&pf->pdev->dev, 12531 "discard user configured hash keys and lut\n"); 12532 } 12533 12534 /* Reset vsi->rss_size, as number of enabled queues changed */ 12535 qcount = vsi->num_queue_pairs / vsi->tc_config.numtc; 12536 vsi->rss_size = min_t(int, pf->alloc_rss_size, qcount); 12537 12538 i40e_pf_config_rss(pf); 12539 } 12540 dev_info(&pf->pdev->dev, "User requested queue count/HW max RSS count: %d/%d\n", 12541 vsi->req_queue_pairs, pf->rss_size_max); 12542 return pf->alloc_rss_size; 12543 } 12544 12545 /** 12546 * i40e_get_partition_bw_setting - Retrieve BW settings for this PF partition 12547 * @pf: board private structure 12548 **/ 12549 int i40e_get_partition_bw_setting(struct i40e_pf *pf) 12550 { 12551 bool min_valid, max_valid; 12552 u32 max_bw, min_bw; 12553 int status; 12554 12555 status = i40e_read_bw_from_alt_ram(&pf->hw, &max_bw, &min_bw, 12556 &min_valid, &max_valid); 12557 12558 if (!status) { 12559 if (min_valid) 12560 pf->min_bw = min_bw; 12561 if (max_valid) 12562 pf->max_bw = max_bw; 12563 } 12564 12565 return status; 12566 } 12567 12568 /** 12569 * i40e_set_partition_bw_setting - Set BW settings for this PF partition 12570 * @pf: board private structure 12571 **/ 12572 int i40e_set_partition_bw_setting(struct i40e_pf *pf) 12573 { 12574 struct i40e_aqc_configure_partition_bw_data bw_data; 12575 int status; 12576 12577 memset(&bw_data, 0, sizeof(bw_data)); 12578 12579 /* Set the valid bit for this PF */ 12580 bw_data.pf_valid_bits = cpu_to_le16(BIT(pf->hw.pf_id)); 12581 bw_data.max_bw[pf->hw.pf_id] = pf->max_bw & I40E_ALT_BW_VALUE_MASK; 12582 bw_data.min_bw[pf->hw.pf_id] = pf->min_bw & I40E_ALT_BW_VALUE_MASK; 12583 12584 /* Set the new bandwidths */ 12585 status = i40e_aq_configure_partition_bw(&pf->hw, &bw_data, NULL); 12586 12587 return status; 12588 } 12589 12590 /** 12591 * i40e_commit_partition_bw_setting - Commit BW settings for this PF partition 12592 * @pf: board private structure 12593 **/ 12594 int i40e_commit_partition_bw_setting(struct i40e_pf *pf) 12595 { 12596 /* Commit temporary BW setting to permanent NVM image */ 12597 enum i40e_admin_queue_err last_aq_status; 12598 u16 nvm_word; 12599 int ret; 12600 12601 if (pf->hw.partition_id != 1) { 12602 dev_info(&pf->pdev->dev, 12603 "Commit BW only works on partition 1! This is partition %d", 12604 pf->hw.partition_id); 12605 ret = -EOPNOTSUPP; 12606 goto bw_commit_out; 12607 } 12608 12609 /* Acquire NVM for read access */ 12610 ret = i40e_acquire_nvm(&pf->hw, I40E_RESOURCE_READ); 12611 last_aq_status = pf->hw.aq.asq_last_status; 12612 if (ret) { 12613 dev_info(&pf->pdev->dev, 12614 "Cannot acquire NVM for read access, err %pe aq_err %s\n", 12615 ERR_PTR(ret), 12616 i40e_aq_str(&pf->hw, last_aq_status)); 12617 goto bw_commit_out; 12618 } 12619 12620 /* Read word 0x10 of NVM - SW compatibility word 1 */ 12621 ret = i40e_aq_read_nvm(&pf->hw, 12622 I40E_SR_NVM_CONTROL_WORD, 12623 0x10, sizeof(nvm_word), &nvm_word, 12624 false, NULL); 12625 /* Save off last admin queue command status before releasing 12626 * the NVM 12627 */ 12628 last_aq_status = pf->hw.aq.asq_last_status; 12629 i40e_release_nvm(&pf->hw); 12630 if (ret) { 12631 dev_info(&pf->pdev->dev, "NVM read error, err %pe aq_err %s\n", 12632 ERR_PTR(ret), 12633 i40e_aq_str(&pf->hw, last_aq_status)); 12634 goto bw_commit_out; 12635 } 12636 12637 /* Wait a bit for NVM release to complete */ 12638 msleep(50); 12639 12640 /* Acquire NVM for write access */ 12641 ret = i40e_acquire_nvm(&pf->hw, I40E_RESOURCE_WRITE); 12642 last_aq_status = pf->hw.aq.asq_last_status; 12643 if (ret) { 12644 dev_info(&pf->pdev->dev, 12645 "Cannot acquire NVM for write access, err %pe aq_err %s\n", 12646 ERR_PTR(ret), 12647 i40e_aq_str(&pf->hw, last_aq_status)); 12648 goto bw_commit_out; 12649 } 12650 /* Write it back out unchanged to initiate update NVM, 12651 * which will force a write of the shadow (alt) RAM to 12652 * the NVM - thus storing the bandwidth values permanently. 12653 */ 12654 ret = i40e_aq_update_nvm(&pf->hw, 12655 I40E_SR_NVM_CONTROL_WORD, 12656 0x10, sizeof(nvm_word), 12657 &nvm_word, true, 0, NULL); 12658 /* Save off last admin queue command status before releasing 12659 * the NVM 12660 */ 12661 last_aq_status = pf->hw.aq.asq_last_status; 12662 i40e_release_nvm(&pf->hw); 12663 if (ret) 12664 dev_info(&pf->pdev->dev, 12665 "BW settings NOT SAVED, err %pe aq_err %s\n", 12666 ERR_PTR(ret), 12667 i40e_aq_str(&pf->hw, last_aq_status)); 12668 bw_commit_out: 12669 12670 return ret; 12671 } 12672 12673 /** 12674 * i40e_is_total_port_shutdown_enabled - read NVM and return value 12675 * if total port shutdown feature is enabled for this PF 12676 * @pf: board private structure 12677 **/ 12678 static bool i40e_is_total_port_shutdown_enabled(struct i40e_pf *pf) 12679 { 12680 #define I40E_TOTAL_PORT_SHUTDOWN_ENABLED BIT(4) 12681 #define I40E_FEATURES_ENABLE_PTR 0x2A 12682 #define I40E_CURRENT_SETTING_PTR 0x2B 12683 #define I40E_LINK_BEHAVIOR_WORD_OFFSET 0x2D 12684 #define I40E_LINK_BEHAVIOR_WORD_LENGTH 0x1 12685 #define I40E_LINK_BEHAVIOR_OS_FORCED_ENABLED BIT(0) 12686 #define I40E_LINK_BEHAVIOR_PORT_BIT_LENGTH 4 12687 u16 sr_emp_sr_settings_ptr = 0; 12688 u16 features_enable = 0; 12689 u16 link_behavior = 0; 12690 int read_status = 0; 12691 bool ret = false; 12692 12693 read_status = i40e_read_nvm_word(&pf->hw, 12694 I40E_SR_EMP_SR_SETTINGS_PTR, 12695 &sr_emp_sr_settings_ptr); 12696 if (read_status) 12697 goto err_nvm; 12698 read_status = i40e_read_nvm_word(&pf->hw, 12699 sr_emp_sr_settings_ptr + 12700 I40E_FEATURES_ENABLE_PTR, 12701 &features_enable); 12702 if (read_status) 12703 goto err_nvm; 12704 if (I40E_TOTAL_PORT_SHUTDOWN_ENABLED & features_enable) { 12705 read_status = i40e_read_nvm_module_data(&pf->hw, 12706 I40E_SR_EMP_SR_SETTINGS_PTR, 12707 I40E_CURRENT_SETTING_PTR, 12708 I40E_LINK_BEHAVIOR_WORD_OFFSET, 12709 I40E_LINK_BEHAVIOR_WORD_LENGTH, 12710 &link_behavior); 12711 if (read_status) 12712 goto err_nvm; 12713 link_behavior >>= (pf->hw.port * I40E_LINK_BEHAVIOR_PORT_BIT_LENGTH); 12714 ret = I40E_LINK_BEHAVIOR_OS_FORCED_ENABLED & link_behavior; 12715 } 12716 return ret; 12717 12718 err_nvm: 12719 dev_warn(&pf->pdev->dev, 12720 "total-port-shutdown feature is off due to read nvm error: %pe\n", 12721 ERR_PTR(read_status)); 12722 return ret; 12723 } 12724 12725 /** 12726 * i40e_sw_init - Initialize general software structures (struct i40e_pf) 12727 * @pf: board private structure to initialize 12728 * 12729 * i40e_sw_init initializes the Adapter private data structure. 12730 * Fields are initialized based on PCI device information and 12731 * OS network device settings (MTU size). 12732 **/ 12733 static int i40e_sw_init(struct i40e_pf *pf) 12734 { 12735 int err = 0; 12736 int size; 12737 u16 pow; 12738 12739 /* Set default capability flags */ 12740 bitmap_zero(pf->flags, I40E_PF_FLAGS_NBITS); 12741 set_bit(I40E_FLAG_MSI_ENA, pf->flags); 12742 set_bit(I40E_FLAG_MSIX_ENA, pf->flags); 12743 12744 /* Set default ITR */ 12745 pf->rx_itr_default = I40E_ITR_RX_DEF; 12746 pf->tx_itr_default = I40E_ITR_TX_DEF; 12747 12748 /* Depending on PF configurations, it is possible that the RSS 12749 * maximum might end up larger than the available queues 12750 */ 12751 pf->rss_size_max = BIT(pf->hw.func_caps.rss_table_entry_width); 12752 pf->alloc_rss_size = 1; 12753 pf->rss_table_size = pf->hw.func_caps.rss_table_size; 12754 pf->rss_size_max = min_t(int, pf->rss_size_max, 12755 pf->hw.func_caps.num_tx_qp); 12756 12757 /* find the next higher power-of-2 of num cpus */ 12758 pow = roundup_pow_of_two(num_online_cpus()); 12759 pf->rss_size_max = min_t(int, pf->rss_size_max, pow); 12760 12761 if (pf->hw.func_caps.rss) { 12762 set_bit(I40E_FLAG_RSS_ENA, pf->flags); 12763 pf->alloc_rss_size = min_t(int, pf->rss_size_max, 12764 num_online_cpus()); 12765 } 12766 12767 /* MFP mode enabled */ 12768 if (pf->hw.func_caps.npar_enable || pf->hw.func_caps.flex10_enable) { 12769 set_bit(I40E_FLAG_MFP_ENA, pf->flags); 12770 dev_info(&pf->pdev->dev, "MFP mode Enabled\n"); 12771 if (i40e_get_partition_bw_setting(pf)) { 12772 dev_warn(&pf->pdev->dev, 12773 "Could not get partition bw settings\n"); 12774 } else { 12775 dev_info(&pf->pdev->dev, 12776 "Partition BW Min = %8.8x, Max = %8.8x\n", 12777 pf->min_bw, pf->max_bw); 12778 12779 /* nudge the Tx scheduler */ 12780 i40e_set_partition_bw_setting(pf); 12781 } 12782 } 12783 12784 if ((pf->hw.func_caps.fd_filters_guaranteed > 0) || 12785 (pf->hw.func_caps.fd_filters_best_effort > 0)) { 12786 set_bit(I40E_FLAG_FD_ATR_ENA, pf->flags); 12787 if (test_bit(I40E_FLAG_MFP_ENA, pf->flags) && 12788 pf->hw.num_partitions > 1) 12789 dev_info(&pf->pdev->dev, 12790 "Flow Director Sideband mode Disabled in MFP mode\n"); 12791 else 12792 set_bit(I40E_FLAG_FD_SB_ENA, pf->flags); 12793 pf->fdir_pf_filter_count = 12794 pf->hw.func_caps.fd_filters_guaranteed; 12795 pf->hw.fdir_shared_filter_count = 12796 pf->hw.func_caps.fd_filters_best_effort; 12797 } 12798 12799 /* Enable HW ATR eviction if possible */ 12800 if (test_bit(I40E_HW_CAP_ATR_EVICT, pf->hw.caps)) 12801 set_bit(I40E_FLAG_HW_ATR_EVICT_ENA, pf->flags); 12802 12803 if (pf->hw.func_caps.vmdq && num_online_cpus() != 1) { 12804 pf->num_vmdq_vsis = I40E_DEFAULT_NUM_VMDQ_VSI; 12805 set_bit(I40E_FLAG_VMDQ_ENA, pf->flags); 12806 pf->num_vmdq_qps = i40e_default_queues_per_vmdq(pf); 12807 } 12808 12809 if (pf->hw.func_caps.iwarp && num_online_cpus() != 1) { 12810 set_bit(I40E_FLAG_IWARP_ENA, pf->flags); 12811 /* IWARP needs one extra vector for CQP just like MISC.*/ 12812 pf->num_iwarp_msix = (int)num_online_cpus() + 1; 12813 } 12814 /* Stopping FW LLDP engine is supported on XL710 and X722 12815 * starting from FW versions determined in i40e_init_adminq. 12816 * Stopping the FW LLDP engine is not supported on XL710 12817 * if NPAR is functioning so unset this hw flag in this case. 12818 */ 12819 if (pf->hw.mac.type == I40E_MAC_XL710 && 12820 pf->hw.func_caps.npar_enable) 12821 clear_bit(I40E_HW_CAP_FW_LLDP_STOPPABLE, pf->hw.caps); 12822 12823 #ifdef CONFIG_PCI_IOV 12824 if (pf->hw.func_caps.num_vfs && pf->hw.partition_id == 1) { 12825 pf->num_vf_qps = I40E_DEFAULT_QUEUES_PER_VF; 12826 set_bit(I40E_FLAG_SRIOV_ENA, pf->flags); 12827 pf->num_req_vfs = min_t(int, 12828 pf->hw.func_caps.num_vfs, 12829 I40E_MAX_VF_COUNT); 12830 } 12831 #endif /* CONFIG_PCI_IOV */ 12832 pf->lan_veb = I40E_NO_VEB; 12833 pf->lan_vsi = I40E_NO_VSI; 12834 12835 /* By default FW has this off for performance reasons */ 12836 clear_bit(I40E_FLAG_VEB_STATS_ENA, pf->flags); 12837 12838 /* set up queue assignment tracking */ 12839 size = sizeof(struct i40e_lump_tracking) 12840 + (sizeof(u16) * pf->hw.func_caps.num_tx_qp); 12841 pf->qp_pile = kzalloc(size, GFP_KERNEL); 12842 if (!pf->qp_pile) { 12843 err = -ENOMEM; 12844 goto sw_init_done; 12845 } 12846 pf->qp_pile->num_entries = pf->hw.func_caps.num_tx_qp; 12847 12848 pf->tx_timeout_recovery_level = 1; 12849 12850 if (pf->hw.mac.type != I40E_MAC_X722 && 12851 i40e_is_total_port_shutdown_enabled(pf)) { 12852 /* Link down on close must be on when total port shutdown 12853 * is enabled for a given port 12854 */ 12855 set_bit(I40E_FLAG_TOTAL_PORT_SHUTDOWN_ENA, pf->flags); 12856 set_bit(I40E_FLAG_LINK_DOWN_ON_CLOSE_ENA, pf->flags); 12857 dev_info(&pf->pdev->dev, 12858 "total-port-shutdown was enabled, link-down-on-close is forced on\n"); 12859 } 12860 mutex_init(&pf->switch_mutex); 12861 12862 sw_init_done: 12863 return err; 12864 } 12865 12866 /** 12867 * i40e_set_ntuple - set the ntuple feature flag and take action 12868 * @pf: board private structure to initialize 12869 * @features: the feature set that the stack is suggesting 12870 * 12871 * returns a bool to indicate if reset needs to happen 12872 **/ 12873 bool i40e_set_ntuple(struct i40e_pf *pf, netdev_features_t features) 12874 { 12875 bool need_reset = false; 12876 12877 /* Check if Flow Director n-tuple support was enabled or disabled. If 12878 * the state changed, we need to reset. 12879 */ 12880 if (features & NETIF_F_NTUPLE) { 12881 /* Enable filters and mark for reset */ 12882 if (!test_bit(I40E_FLAG_FD_SB_ENA, pf->flags)) 12883 need_reset = true; 12884 /* enable FD_SB only if there is MSI-X vector and no cloud 12885 * filters exist 12886 */ 12887 if (pf->num_fdsb_msix > 0 && !pf->num_cloud_filters) { 12888 set_bit(I40E_FLAG_FD_SB_ENA, pf->flags); 12889 clear_bit(I40E_FLAG_FD_SB_INACTIVE, pf->flags); 12890 } 12891 } else { 12892 /* turn off filters, mark for reset and clear SW filter list */ 12893 if (test_bit(I40E_FLAG_FD_SB_ENA, pf->flags)) { 12894 need_reset = true; 12895 i40e_fdir_filter_exit(pf); 12896 } 12897 clear_bit(I40E_FLAG_FD_SB_ENA, pf->flags); 12898 clear_bit(__I40E_FD_SB_AUTO_DISABLED, pf->state); 12899 set_bit(I40E_FLAG_FD_SB_INACTIVE, pf->flags); 12900 12901 /* reset fd counters */ 12902 pf->fd_add_err = 0; 12903 pf->fd_atr_cnt = 0; 12904 /* if ATR was auto disabled it can be re-enabled. */ 12905 if (test_and_clear_bit(__I40E_FD_ATR_AUTO_DISABLED, pf->state)) 12906 if (test_bit(I40E_FLAG_FD_ATR_ENA, pf->flags) && 12907 (I40E_DEBUG_FD & pf->hw.debug_mask)) 12908 dev_info(&pf->pdev->dev, "ATR re-enabled.\n"); 12909 } 12910 return need_reset; 12911 } 12912 12913 /** 12914 * i40e_clear_rss_lut - clear the rx hash lookup table 12915 * @vsi: the VSI being configured 12916 **/ 12917 static void i40e_clear_rss_lut(struct i40e_vsi *vsi) 12918 { 12919 struct i40e_pf *pf = vsi->back; 12920 struct i40e_hw *hw = &pf->hw; 12921 u16 vf_id = vsi->vf_id; 12922 u8 i; 12923 12924 if (vsi->type == I40E_VSI_MAIN) { 12925 for (i = 0; i <= I40E_PFQF_HLUT_MAX_INDEX; i++) 12926 wr32(hw, I40E_PFQF_HLUT(i), 0); 12927 } else if (vsi->type == I40E_VSI_SRIOV) { 12928 for (i = 0; i <= I40E_VFQF_HLUT_MAX_INDEX; i++) 12929 i40e_write_rx_ctl(hw, I40E_VFQF_HLUT1(i, vf_id), 0); 12930 } else { 12931 dev_err(&pf->pdev->dev, "Cannot set RSS LUT - invalid VSI type\n"); 12932 } 12933 } 12934 12935 /** 12936 * i40e_set_loopback - turn on/off loopback mode on underlying PF 12937 * @vsi: ptr to VSI 12938 * @ena: flag to indicate the on/off setting 12939 */ 12940 static int i40e_set_loopback(struct i40e_vsi *vsi, bool ena) 12941 { 12942 bool if_running = netif_running(vsi->netdev) && 12943 !test_and_set_bit(__I40E_VSI_DOWN, vsi->state); 12944 int ret; 12945 12946 if (if_running) 12947 i40e_down(vsi); 12948 12949 ret = i40e_aq_set_mac_loopback(&vsi->back->hw, ena, NULL); 12950 if (ret) 12951 netdev_err(vsi->netdev, "Failed to toggle loopback state\n"); 12952 if (if_running) 12953 i40e_up(vsi); 12954 12955 return ret; 12956 } 12957 12958 /** 12959 * i40e_set_features - set the netdev feature flags 12960 * @netdev: ptr to the netdev being adjusted 12961 * @features: the feature set that the stack is suggesting 12962 * Note: expects to be called while under rtnl_lock() 12963 **/ 12964 static int i40e_set_features(struct net_device *netdev, 12965 netdev_features_t features) 12966 { 12967 struct i40e_netdev_priv *np = netdev_priv(netdev); 12968 struct i40e_vsi *vsi = np->vsi; 12969 struct i40e_pf *pf = vsi->back; 12970 bool need_reset; 12971 12972 if (features & NETIF_F_RXHASH && !(netdev->features & NETIF_F_RXHASH)) 12973 i40e_pf_config_rss(pf); 12974 else if (!(features & NETIF_F_RXHASH) && 12975 netdev->features & NETIF_F_RXHASH) 12976 i40e_clear_rss_lut(vsi); 12977 12978 if (features & NETIF_F_HW_VLAN_CTAG_RX) 12979 i40e_vlan_stripping_enable(vsi); 12980 else 12981 i40e_vlan_stripping_disable(vsi); 12982 12983 if (!(features & NETIF_F_HW_TC) && 12984 (netdev->features & NETIF_F_HW_TC) && pf->num_cloud_filters) { 12985 dev_err(&pf->pdev->dev, 12986 "Offloaded tc filters active, can't turn hw_tc_offload off"); 12987 return -EINVAL; 12988 } 12989 12990 if (!(features & NETIF_F_HW_L2FW_DOFFLOAD) && vsi->macvlan_cnt) 12991 i40e_del_all_macvlans(vsi); 12992 12993 need_reset = i40e_set_ntuple(pf, features); 12994 12995 if (need_reset) 12996 i40e_do_reset(pf, I40E_PF_RESET_FLAG, true); 12997 12998 if ((features ^ netdev->features) & NETIF_F_LOOPBACK) 12999 return i40e_set_loopback(vsi, !!(features & NETIF_F_LOOPBACK)); 13000 13001 return 0; 13002 } 13003 13004 static int i40e_udp_tunnel_set_port(struct net_device *netdev, 13005 unsigned int table, unsigned int idx, 13006 struct udp_tunnel_info *ti) 13007 { 13008 struct i40e_netdev_priv *np = netdev_priv(netdev); 13009 struct i40e_hw *hw = &np->vsi->back->hw; 13010 u8 type, filter_index; 13011 int ret; 13012 13013 type = ti->type == UDP_TUNNEL_TYPE_VXLAN ? I40E_AQC_TUNNEL_TYPE_VXLAN : 13014 I40E_AQC_TUNNEL_TYPE_NGE; 13015 13016 ret = i40e_aq_add_udp_tunnel(hw, ntohs(ti->port), type, &filter_index, 13017 NULL); 13018 if (ret) { 13019 netdev_info(netdev, "add UDP port failed, err %pe aq_err %s\n", 13020 ERR_PTR(ret), 13021 i40e_aq_str(hw, hw->aq.asq_last_status)); 13022 return -EIO; 13023 } 13024 13025 udp_tunnel_nic_set_port_priv(netdev, table, idx, filter_index); 13026 return 0; 13027 } 13028 13029 static int i40e_udp_tunnel_unset_port(struct net_device *netdev, 13030 unsigned int table, unsigned int idx, 13031 struct udp_tunnel_info *ti) 13032 { 13033 struct i40e_netdev_priv *np = netdev_priv(netdev); 13034 struct i40e_hw *hw = &np->vsi->back->hw; 13035 int ret; 13036 13037 ret = i40e_aq_del_udp_tunnel(hw, ti->hw_priv, NULL); 13038 if (ret) { 13039 netdev_info(netdev, "delete UDP port failed, err %pe aq_err %s\n", 13040 ERR_PTR(ret), 13041 i40e_aq_str(hw, hw->aq.asq_last_status)); 13042 return -EIO; 13043 } 13044 13045 return 0; 13046 } 13047 13048 static int i40e_get_phys_port_id(struct net_device *netdev, 13049 struct netdev_phys_item_id *ppid) 13050 { 13051 struct i40e_netdev_priv *np = netdev_priv(netdev); 13052 struct i40e_pf *pf = np->vsi->back; 13053 struct i40e_hw *hw = &pf->hw; 13054 13055 if (!test_bit(I40E_HW_CAP_PORT_ID_VALID, pf->hw.caps)) 13056 return -EOPNOTSUPP; 13057 13058 ppid->id_len = min_t(int, sizeof(hw->mac.port_addr), sizeof(ppid->id)); 13059 memcpy(ppid->id, hw->mac.port_addr, ppid->id_len); 13060 13061 return 0; 13062 } 13063 13064 /** 13065 * i40e_ndo_fdb_add - add an entry to the hardware database 13066 * @ndm: the input from the stack 13067 * @tb: pointer to array of nladdr (unused) 13068 * @dev: the net device pointer 13069 * @addr: the MAC address entry being added 13070 * @vid: VLAN ID 13071 * @flags: instructions from stack about fdb operation 13072 * @extack: netlink extended ack, unused currently 13073 */ 13074 static int i40e_ndo_fdb_add(struct ndmsg *ndm, struct nlattr *tb[], 13075 struct net_device *dev, 13076 const unsigned char *addr, u16 vid, 13077 u16 flags, 13078 struct netlink_ext_ack *extack) 13079 { 13080 struct i40e_netdev_priv *np = netdev_priv(dev); 13081 struct i40e_pf *pf = np->vsi->back; 13082 int err = 0; 13083 13084 if (!test_bit(I40E_FLAG_SRIOV_ENA, pf->flags)) 13085 return -EOPNOTSUPP; 13086 13087 if (vid) { 13088 pr_info("%s: vlans aren't supported yet for dev_uc|mc_add()\n", dev->name); 13089 return -EINVAL; 13090 } 13091 13092 /* Hardware does not support aging addresses so if a 13093 * ndm_state is given only allow permanent addresses 13094 */ 13095 if (ndm->ndm_state && !(ndm->ndm_state & NUD_PERMANENT)) { 13096 netdev_info(dev, "FDB only supports static addresses\n"); 13097 return -EINVAL; 13098 } 13099 13100 if (is_unicast_ether_addr(addr) || is_link_local_ether_addr(addr)) 13101 err = dev_uc_add_excl(dev, addr); 13102 else if (is_multicast_ether_addr(addr)) 13103 err = dev_mc_add_excl(dev, addr); 13104 else 13105 err = -EINVAL; 13106 13107 /* Only return duplicate errors if NLM_F_EXCL is set */ 13108 if (err == -EEXIST && !(flags & NLM_F_EXCL)) 13109 err = 0; 13110 13111 return err; 13112 } 13113 13114 /** 13115 * i40e_ndo_bridge_setlink - Set the hardware bridge mode 13116 * @dev: the netdev being configured 13117 * @nlh: RTNL message 13118 * @flags: bridge flags 13119 * @extack: netlink extended ack 13120 * 13121 * Inserts a new hardware bridge if not already created and 13122 * enables the bridging mode requested (VEB or VEPA). If the 13123 * hardware bridge has already been inserted and the request 13124 * is to change the mode then that requires a PF reset to 13125 * allow rebuild of the components with required hardware 13126 * bridge mode enabled. 13127 * 13128 * Note: expects to be called while under rtnl_lock() 13129 **/ 13130 static int i40e_ndo_bridge_setlink(struct net_device *dev, 13131 struct nlmsghdr *nlh, 13132 u16 flags, 13133 struct netlink_ext_ack *extack) 13134 { 13135 struct i40e_netdev_priv *np = netdev_priv(dev); 13136 struct i40e_vsi *vsi = np->vsi; 13137 struct i40e_pf *pf = vsi->back; 13138 struct nlattr *attr, *br_spec; 13139 struct i40e_veb *veb; 13140 int rem; 13141 13142 /* Only for PF VSI for now */ 13143 if (vsi->type != I40E_VSI_MAIN) 13144 return -EOPNOTSUPP; 13145 13146 /* Find the HW bridge for PF VSI */ 13147 veb = i40e_pf_get_veb_by_seid(pf, vsi->uplink_seid); 13148 13149 br_spec = nlmsg_find_attr(nlh, sizeof(struct ifinfomsg), IFLA_AF_SPEC); 13150 if (!br_spec) 13151 return -EINVAL; 13152 13153 nla_for_each_nested_type(attr, IFLA_BRIDGE_MODE, br_spec, rem) { 13154 __u16 mode = nla_get_u16(attr); 13155 13156 if ((mode != BRIDGE_MODE_VEPA) && 13157 (mode != BRIDGE_MODE_VEB)) 13158 return -EINVAL; 13159 13160 /* Insert a new HW bridge */ 13161 if (!veb) { 13162 veb = i40e_veb_setup(pf, vsi->uplink_seid, vsi->seid, 13163 vsi->tc_config.enabled_tc); 13164 if (veb) { 13165 veb->bridge_mode = mode; 13166 i40e_config_bridge_mode(veb); 13167 } else { 13168 /* No Bridge HW offload available */ 13169 return -ENOENT; 13170 } 13171 break; 13172 } else if (mode != veb->bridge_mode) { 13173 /* Existing HW bridge but different mode needs reset */ 13174 veb->bridge_mode = mode; 13175 /* TODO: If no VFs or VMDq VSIs, disallow VEB mode */ 13176 if (mode == BRIDGE_MODE_VEB) 13177 set_bit(I40E_FLAG_VEB_MODE_ENA, pf->flags); 13178 else 13179 clear_bit(I40E_FLAG_VEB_MODE_ENA, pf->flags); 13180 i40e_do_reset(pf, I40E_PF_RESET_FLAG, true); 13181 break; 13182 } 13183 } 13184 13185 return 0; 13186 } 13187 13188 /** 13189 * i40e_ndo_bridge_getlink - Get the hardware bridge mode 13190 * @skb: skb buff 13191 * @pid: process id 13192 * @seq: RTNL message seq # 13193 * @dev: the netdev being configured 13194 * @filter_mask: unused 13195 * @nlflags: netlink flags passed in 13196 * 13197 * Return the mode in which the hardware bridge is operating in 13198 * i.e VEB or VEPA. 13199 **/ 13200 static int i40e_ndo_bridge_getlink(struct sk_buff *skb, u32 pid, u32 seq, 13201 struct net_device *dev, 13202 u32 __always_unused filter_mask, 13203 int nlflags) 13204 { 13205 struct i40e_netdev_priv *np = netdev_priv(dev); 13206 struct i40e_vsi *vsi = np->vsi; 13207 struct i40e_pf *pf = vsi->back; 13208 struct i40e_veb *veb; 13209 13210 /* Only for PF VSI for now */ 13211 if (vsi->type != I40E_VSI_MAIN) 13212 return -EOPNOTSUPP; 13213 13214 /* Find the HW bridge for the PF VSI */ 13215 veb = i40e_pf_get_veb_by_seid(pf, vsi->uplink_seid); 13216 if (!veb) 13217 return 0; 13218 13219 return ndo_dflt_bridge_getlink(skb, pid, seq, dev, veb->bridge_mode, 13220 0, 0, nlflags, filter_mask, NULL); 13221 } 13222 13223 /** 13224 * i40e_features_check - Validate encapsulated packet conforms to limits 13225 * @skb: skb buff 13226 * @dev: This physical port's netdev 13227 * @features: Offload features that the stack believes apply 13228 **/ 13229 static netdev_features_t i40e_features_check(struct sk_buff *skb, 13230 struct net_device *dev, 13231 netdev_features_t features) 13232 { 13233 size_t len; 13234 13235 /* No point in doing any of this if neither checksum nor GSO are 13236 * being requested for this frame. We can rule out both by just 13237 * checking for CHECKSUM_PARTIAL 13238 */ 13239 if (skb->ip_summed != CHECKSUM_PARTIAL) 13240 return features; 13241 13242 /* We cannot support GSO if the MSS is going to be less than 13243 * 64 bytes. If it is then we need to drop support for GSO. 13244 */ 13245 if (skb_is_gso(skb) && (skb_shinfo(skb)->gso_size < 64)) 13246 features &= ~NETIF_F_GSO_MASK; 13247 13248 /* MACLEN can support at most 63 words */ 13249 len = skb_network_offset(skb); 13250 if (len & ~(63 * 2)) 13251 goto out_err; 13252 13253 /* IPLEN and EIPLEN can support at most 127 dwords */ 13254 len = skb_network_header_len(skb); 13255 if (len & ~(127 * 4)) 13256 goto out_err; 13257 13258 if (skb->encapsulation) { 13259 /* L4TUNLEN can support 127 words */ 13260 len = skb_inner_network_header(skb) - skb_transport_header(skb); 13261 if (len & ~(127 * 2)) 13262 goto out_err; 13263 13264 /* IPLEN can support at most 127 dwords */ 13265 len = skb_inner_transport_header(skb) - 13266 skb_inner_network_header(skb); 13267 if (len & ~(127 * 4)) 13268 goto out_err; 13269 } 13270 13271 /* No need to validate L4LEN as TCP is the only protocol with a 13272 * flexible value and we support all possible values supported 13273 * by TCP, which is at most 15 dwords 13274 */ 13275 13276 return features; 13277 out_err: 13278 return features & ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK); 13279 } 13280 13281 /** 13282 * i40e_xdp_setup - add/remove an XDP program 13283 * @vsi: VSI to changed 13284 * @prog: XDP program 13285 * @extack: netlink extended ack 13286 **/ 13287 static int i40e_xdp_setup(struct i40e_vsi *vsi, struct bpf_prog *prog, 13288 struct netlink_ext_ack *extack) 13289 { 13290 int frame_size = i40e_max_vsi_frame_size(vsi, prog); 13291 struct i40e_pf *pf = vsi->back; 13292 struct bpf_prog *old_prog; 13293 bool need_reset; 13294 int i; 13295 13296 /* VSI shall be deleted in a moment, block loading new programs */ 13297 if (prog && test_bit(__I40E_IN_REMOVE, pf->state)) 13298 return -EINVAL; 13299 13300 /* Don't allow frames that span over multiple buffers */ 13301 if (vsi->netdev->mtu > frame_size - I40E_PACKET_HDR_PAD) { 13302 NL_SET_ERR_MSG_MOD(extack, "MTU too large for linear frames and XDP prog does not support frags"); 13303 return -EINVAL; 13304 } 13305 13306 /* When turning XDP on->off/off->on we reset and rebuild the rings. */ 13307 need_reset = (i40e_enabled_xdp_vsi(vsi) != !!prog); 13308 if (need_reset) 13309 i40e_prep_for_reset(pf); 13310 13311 old_prog = xchg(&vsi->xdp_prog, prog); 13312 13313 if (need_reset) { 13314 if (!prog) { 13315 xdp_features_clear_redirect_target(vsi->netdev); 13316 /* Wait until ndo_xsk_wakeup completes. */ 13317 synchronize_rcu(); 13318 } 13319 i40e_reset_and_rebuild(pf, true, true); 13320 } 13321 13322 if (!i40e_enabled_xdp_vsi(vsi) && prog) { 13323 if (i40e_realloc_rx_bi_zc(vsi, true)) 13324 return -ENOMEM; 13325 } else if (i40e_enabled_xdp_vsi(vsi) && !prog) { 13326 if (i40e_realloc_rx_bi_zc(vsi, false)) 13327 return -ENOMEM; 13328 } 13329 13330 for (i = 0; i < vsi->num_queue_pairs; i++) 13331 WRITE_ONCE(vsi->rx_rings[i]->xdp_prog, vsi->xdp_prog); 13332 13333 if (old_prog) 13334 bpf_prog_put(old_prog); 13335 13336 /* Kick start the NAPI context if there is an AF_XDP socket open 13337 * on that queue id. This so that receiving will start. 13338 */ 13339 if (need_reset && prog) { 13340 for (i = 0; i < vsi->num_queue_pairs; i++) 13341 if (vsi->xdp_rings[i]->xsk_pool) 13342 (void)i40e_xsk_wakeup(vsi->netdev, i, 13343 XDP_WAKEUP_RX); 13344 xdp_features_set_redirect_target(vsi->netdev, true); 13345 } 13346 13347 return 0; 13348 } 13349 13350 /** 13351 * i40e_enter_busy_conf - Enters busy config state 13352 * @vsi: vsi 13353 * 13354 * Returns 0 on success, <0 for failure. 13355 **/ 13356 static int i40e_enter_busy_conf(struct i40e_vsi *vsi) 13357 { 13358 struct i40e_pf *pf = vsi->back; 13359 int timeout = 50; 13360 13361 while (test_and_set_bit(__I40E_CONFIG_BUSY, pf->state)) { 13362 timeout--; 13363 if (!timeout) 13364 return -EBUSY; 13365 usleep_range(1000, 2000); 13366 } 13367 13368 return 0; 13369 } 13370 13371 /** 13372 * i40e_exit_busy_conf - Exits busy config state 13373 * @vsi: vsi 13374 **/ 13375 static void i40e_exit_busy_conf(struct i40e_vsi *vsi) 13376 { 13377 struct i40e_pf *pf = vsi->back; 13378 13379 clear_bit(__I40E_CONFIG_BUSY, pf->state); 13380 } 13381 13382 /** 13383 * i40e_queue_pair_reset_stats - Resets all statistics for a queue pair 13384 * @vsi: vsi 13385 * @queue_pair: queue pair 13386 **/ 13387 static void i40e_queue_pair_reset_stats(struct i40e_vsi *vsi, int queue_pair) 13388 { 13389 memset(&vsi->rx_rings[queue_pair]->rx_stats, 0, 13390 sizeof(vsi->rx_rings[queue_pair]->rx_stats)); 13391 memset(&vsi->tx_rings[queue_pair]->stats, 0, 13392 sizeof(vsi->tx_rings[queue_pair]->stats)); 13393 if (i40e_enabled_xdp_vsi(vsi)) { 13394 memset(&vsi->xdp_rings[queue_pair]->stats, 0, 13395 sizeof(vsi->xdp_rings[queue_pair]->stats)); 13396 } 13397 } 13398 13399 /** 13400 * i40e_queue_pair_clean_rings - Cleans all the rings of a queue pair 13401 * @vsi: vsi 13402 * @queue_pair: queue pair 13403 **/ 13404 static void i40e_queue_pair_clean_rings(struct i40e_vsi *vsi, int queue_pair) 13405 { 13406 i40e_clean_tx_ring(vsi->tx_rings[queue_pair]); 13407 if (i40e_enabled_xdp_vsi(vsi)) { 13408 /* Make sure that in-progress ndo_xdp_xmit calls are 13409 * completed. 13410 */ 13411 synchronize_rcu(); 13412 i40e_clean_tx_ring(vsi->xdp_rings[queue_pair]); 13413 } 13414 i40e_clean_rx_ring(vsi->rx_rings[queue_pair]); 13415 } 13416 13417 /** 13418 * i40e_queue_pair_toggle_napi - Enables/disables NAPI for a queue pair 13419 * @vsi: vsi 13420 * @queue_pair: queue pair 13421 * @enable: true for enable, false for disable 13422 **/ 13423 static void i40e_queue_pair_toggle_napi(struct i40e_vsi *vsi, int queue_pair, 13424 bool enable) 13425 { 13426 struct i40e_ring *rxr = vsi->rx_rings[queue_pair]; 13427 struct i40e_q_vector *q_vector = rxr->q_vector; 13428 13429 if (!vsi->netdev) 13430 return; 13431 13432 /* All rings in a qp belong to the same qvector. */ 13433 if (q_vector->rx.ring || q_vector->tx.ring) { 13434 if (enable) 13435 napi_enable(&q_vector->napi); 13436 else 13437 napi_disable(&q_vector->napi); 13438 } 13439 } 13440 13441 /** 13442 * i40e_queue_pair_toggle_rings - Enables/disables all rings for a queue pair 13443 * @vsi: vsi 13444 * @queue_pair: queue pair 13445 * @enable: true for enable, false for disable 13446 * 13447 * Returns 0 on success, <0 on failure. 13448 **/ 13449 static int i40e_queue_pair_toggle_rings(struct i40e_vsi *vsi, int queue_pair, 13450 bool enable) 13451 { 13452 struct i40e_pf *pf = vsi->back; 13453 int pf_q, ret = 0; 13454 13455 pf_q = vsi->base_queue + queue_pair; 13456 ret = i40e_control_wait_tx_q(vsi->seid, pf, pf_q, 13457 false /*is xdp*/, enable); 13458 if (ret) { 13459 dev_info(&pf->pdev->dev, 13460 "VSI seid %d Tx ring %d %sable timeout\n", 13461 vsi->seid, pf_q, (enable ? "en" : "dis")); 13462 return ret; 13463 } 13464 13465 i40e_control_rx_q(pf, pf_q, enable); 13466 ret = i40e_pf_rxq_wait(pf, pf_q, enable); 13467 if (ret) { 13468 dev_info(&pf->pdev->dev, 13469 "VSI seid %d Rx ring %d %sable timeout\n", 13470 vsi->seid, pf_q, (enable ? "en" : "dis")); 13471 return ret; 13472 } 13473 13474 /* Due to HW errata, on Rx disable only, the register can 13475 * indicate done before it really is. Needs 50ms to be sure 13476 */ 13477 if (!enable) 13478 mdelay(50); 13479 13480 if (!i40e_enabled_xdp_vsi(vsi)) 13481 return ret; 13482 13483 ret = i40e_control_wait_tx_q(vsi->seid, pf, 13484 pf_q + vsi->alloc_queue_pairs, 13485 true /*is xdp*/, enable); 13486 if (ret) { 13487 dev_info(&pf->pdev->dev, 13488 "VSI seid %d XDP Tx ring %d %sable timeout\n", 13489 vsi->seid, pf_q, (enable ? "en" : "dis")); 13490 } 13491 13492 return ret; 13493 } 13494 13495 /** 13496 * i40e_queue_pair_enable_irq - Enables interrupts for a queue pair 13497 * @vsi: vsi 13498 * @queue_pair: queue_pair 13499 **/ 13500 static void i40e_queue_pair_enable_irq(struct i40e_vsi *vsi, int queue_pair) 13501 { 13502 struct i40e_ring *rxr = vsi->rx_rings[queue_pair]; 13503 struct i40e_pf *pf = vsi->back; 13504 struct i40e_hw *hw = &pf->hw; 13505 13506 /* All rings in a qp belong to the same qvector. */ 13507 if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags)) 13508 i40e_irq_dynamic_enable(vsi, rxr->q_vector->v_idx); 13509 else 13510 i40e_irq_dynamic_enable_icr0(pf); 13511 13512 i40e_flush(hw); 13513 } 13514 13515 /** 13516 * i40e_queue_pair_disable_irq - Disables interrupts for a queue pair 13517 * @vsi: vsi 13518 * @queue_pair: queue_pair 13519 **/ 13520 static void i40e_queue_pair_disable_irq(struct i40e_vsi *vsi, int queue_pair) 13521 { 13522 struct i40e_ring *rxr = vsi->rx_rings[queue_pair]; 13523 struct i40e_pf *pf = vsi->back; 13524 struct i40e_hw *hw = &pf->hw; 13525 13526 /* For simplicity, instead of removing the qp interrupt causes 13527 * from the interrupt linked list, we simply disable the interrupt, and 13528 * leave the list intact. 13529 * 13530 * All rings in a qp belong to the same qvector. 13531 */ 13532 if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags)) { 13533 u32 intpf = vsi->base_vector + rxr->q_vector->v_idx; 13534 13535 wr32(hw, I40E_PFINT_DYN_CTLN(intpf - 1), 0); 13536 i40e_flush(hw); 13537 synchronize_irq(pf->msix_entries[intpf].vector); 13538 } else { 13539 /* Legacy and MSI mode - this stops all interrupt handling */ 13540 wr32(hw, I40E_PFINT_ICR0_ENA, 0); 13541 wr32(hw, I40E_PFINT_DYN_CTL0, 0); 13542 i40e_flush(hw); 13543 synchronize_irq(pf->pdev->irq); 13544 } 13545 } 13546 13547 /** 13548 * i40e_queue_pair_disable - Disables a queue pair 13549 * @vsi: vsi 13550 * @queue_pair: queue pair 13551 * 13552 * Returns 0 on success, <0 on failure. 13553 **/ 13554 int i40e_queue_pair_disable(struct i40e_vsi *vsi, int queue_pair) 13555 { 13556 int err; 13557 13558 err = i40e_enter_busy_conf(vsi); 13559 if (err) 13560 return err; 13561 13562 i40e_queue_pair_disable_irq(vsi, queue_pair); 13563 i40e_queue_pair_toggle_napi(vsi, queue_pair, false /* off */); 13564 err = i40e_queue_pair_toggle_rings(vsi, queue_pair, false /* off */); 13565 i40e_clean_rx_ring(vsi->rx_rings[queue_pair]); 13566 i40e_queue_pair_clean_rings(vsi, queue_pair); 13567 i40e_queue_pair_reset_stats(vsi, queue_pair); 13568 13569 return err; 13570 } 13571 13572 /** 13573 * i40e_queue_pair_enable - Enables a queue pair 13574 * @vsi: vsi 13575 * @queue_pair: queue pair 13576 * 13577 * Returns 0 on success, <0 on failure. 13578 **/ 13579 int i40e_queue_pair_enable(struct i40e_vsi *vsi, int queue_pair) 13580 { 13581 int err; 13582 13583 err = i40e_configure_tx_ring(vsi->tx_rings[queue_pair]); 13584 if (err) 13585 return err; 13586 13587 if (i40e_enabled_xdp_vsi(vsi)) { 13588 err = i40e_configure_tx_ring(vsi->xdp_rings[queue_pair]); 13589 if (err) 13590 return err; 13591 } 13592 13593 err = i40e_configure_rx_ring(vsi->rx_rings[queue_pair]); 13594 if (err) 13595 return err; 13596 13597 err = i40e_queue_pair_toggle_rings(vsi, queue_pair, true /* on */); 13598 i40e_queue_pair_toggle_napi(vsi, queue_pair, true /* on */); 13599 i40e_queue_pair_enable_irq(vsi, queue_pair); 13600 13601 i40e_exit_busy_conf(vsi); 13602 13603 return err; 13604 } 13605 13606 /** 13607 * i40e_xdp - implements ndo_bpf for i40e 13608 * @dev: netdevice 13609 * @xdp: XDP command 13610 **/ 13611 static int i40e_xdp(struct net_device *dev, 13612 struct netdev_bpf *xdp) 13613 { 13614 struct i40e_netdev_priv *np = netdev_priv(dev); 13615 struct i40e_vsi *vsi = np->vsi; 13616 13617 if (vsi->type != I40E_VSI_MAIN) 13618 return -EINVAL; 13619 13620 switch (xdp->command) { 13621 case XDP_SETUP_PROG: 13622 return i40e_xdp_setup(vsi, xdp->prog, xdp->extack); 13623 case XDP_SETUP_XSK_POOL: 13624 return i40e_xsk_pool_setup(vsi, xdp->xsk.pool, 13625 xdp->xsk.queue_id); 13626 default: 13627 return -EINVAL; 13628 } 13629 } 13630 13631 static const struct net_device_ops i40e_netdev_ops = { 13632 .ndo_open = i40e_open, 13633 .ndo_stop = i40e_close, 13634 .ndo_start_xmit = i40e_lan_xmit_frame, 13635 .ndo_get_stats64 = i40e_get_netdev_stats_struct, 13636 .ndo_set_rx_mode = i40e_set_rx_mode, 13637 .ndo_validate_addr = eth_validate_addr, 13638 .ndo_set_mac_address = i40e_set_mac, 13639 .ndo_change_mtu = i40e_change_mtu, 13640 .ndo_eth_ioctl = i40e_ioctl, 13641 .ndo_tx_timeout = i40e_tx_timeout, 13642 .ndo_vlan_rx_add_vid = i40e_vlan_rx_add_vid, 13643 .ndo_vlan_rx_kill_vid = i40e_vlan_rx_kill_vid, 13644 #ifdef CONFIG_NET_POLL_CONTROLLER 13645 .ndo_poll_controller = i40e_netpoll, 13646 #endif 13647 .ndo_setup_tc = __i40e_setup_tc, 13648 .ndo_select_queue = i40e_lan_select_queue, 13649 .ndo_set_features = i40e_set_features, 13650 .ndo_set_vf_mac = i40e_ndo_set_vf_mac, 13651 .ndo_set_vf_vlan = i40e_ndo_set_vf_port_vlan, 13652 .ndo_get_vf_stats = i40e_get_vf_stats, 13653 .ndo_set_vf_rate = i40e_ndo_set_vf_bw, 13654 .ndo_get_vf_config = i40e_ndo_get_vf_config, 13655 .ndo_set_vf_link_state = i40e_ndo_set_vf_link_state, 13656 .ndo_set_vf_spoofchk = i40e_ndo_set_vf_spoofchk, 13657 .ndo_set_vf_trust = i40e_ndo_set_vf_trust, 13658 .ndo_get_phys_port_id = i40e_get_phys_port_id, 13659 .ndo_fdb_add = i40e_ndo_fdb_add, 13660 .ndo_features_check = i40e_features_check, 13661 .ndo_bridge_getlink = i40e_ndo_bridge_getlink, 13662 .ndo_bridge_setlink = i40e_ndo_bridge_setlink, 13663 .ndo_bpf = i40e_xdp, 13664 .ndo_xdp_xmit = i40e_xdp_xmit, 13665 .ndo_xsk_wakeup = i40e_xsk_wakeup, 13666 .ndo_dfwd_add_station = i40e_fwd_add, 13667 .ndo_dfwd_del_station = i40e_fwd_del, 13668 }; 13669 13670 /** 13671 * i40e_config_netdev - Setup the netdev flags 13672 * @vsi: the VSI being configured 13673 * 13674 * Returns 0 on success, negative value on failure 13675 **/ 13676 static int i40e_config_netdev(struct i40e_vsi *vsi) 13677 { 13678 struct i40e_pf *pf = vsi->back; 13679 struct i40e_hw *hw = &pf->hw; 13680 struct i40e_netdev_priv *np; 13681 struct net_device *netdev; 13682 u8 broadcast[ETH_ALEN]; 13683 u8 mac_addr[ETH_ALEN]; 13684 int etherdev_size; 13685 netdev_features_t hw_enc_features; 13686 netdev_features_t hw_features; 13687 13688 etherdev_size = sizeof(struct i40e_netdev_priv); 13689 netdev = alloc_etherdev_mq(etherdev_size, vsi->alloc_queue_pairs); 13690 if (!netdev) 13691 return -ENOMEM; 13692 13693 vsi->netdev = netdev; 13694 np = netdev_priv(netdev); 13695 np->vsi = vsi; 13696 13697 hw_enc_features = NETIF_F_SG | 13698 NETIF_F_HW_CSUM | 13699 NETIF_F_HIGHDMA | 13700 NETIF_F_SOFT_FEATURES | 13701 NETIF_F_TSO | 13702 NETIF_F_TSO_ECN | 13703 NETIF_F_TSO6 | 13704 NETIF_F_GSO_GRE | 13705 NETIF_F_GSO_GRE_CSUM | 13706 NETIF_F_GSO_PARTIAL | 13707 NETIF_F_GSO_IPXIP4 | 13708 NETIF_F_GSO_IPXIP6 | 13709 NETIF_F_GSO_UDP_TUNNEL | 13710 NETIF_F_GSO_UDP_TUNNEL_CSUM | 13711 NETIF_F_GSO_UDP_L4 | 13712 NETIF_F_SCTP_CRC | 13713 NETIF_F_RXHASH | 13714 NETIF_F_RXCSUM | 13715 0; 13716 13717 if (!test_bit(I40E_HW_CAP_OUTER_UDP_CSUM, pf->hw.caps)) 13718 netdev->gso_partial_features |= NETIF_F_GSO_UDP_TUNNEL_CSUM; 13719 13720 netdev->udp_tunnel_nic_info = &pf->udp_tunnel_nic; 13721 13722 netdev->gso_partial_features |= NETIF_F_GSO_GRE_CSUM; 13723 13724 netdev->hw_enc_features |= hw_enc_features; 13725 13726 /* record features VLANs can make use of */ 13727 netdev->vlan_features |= hw_enc_features | NETIF_F_TSO_MANGLEID; 13728 13729 #define I40E_GSO_PARTIAL_FEATURES (NETIF_F_GSO_GRE | \ 13730 NETIF_F_GSO_GRE_CSUM | \ 13731 NETIF_F_GSO_IPXIP4 | \ 13732 NETIF_F_GSO_IPXIP6 | \ 13733 NETIF_F_GSO_UDP_TUNNEL | \ 13734 NETIF_F_GSO_UDP_TUNNEL_CSUM) 13735 13736 netdev->gso_partial_features = I40E_GSO_PARTIAL_FEATURES; 13737 netdev->features |= NETIF_F_GSO_PARTIAL | 13738 I40E_GSO_PARTIAL_FEATURES; 13739 13740 netdev->mpls_features |= NETIF_F_SG; 13741 netdev->mpls_features |= NETIF_F_HW_CSUM; 13742 netdev->mpls_features |= NETIF_F_TSO; 13743 netdev->mpls_features |= NETIF_F_TSO6; 13744 netdev->mpls_features |= I40E_GSO_PARTIAL_FEATURES; 13745 13746 /* enable macvlan offloads */ 13747 netdev->hw_features |= NETIF_F_HW_L2FW_DOFFLOAD; 13748 13749 hw_features = hw_enc_features | 13750 NETIF_F_HW_VLAN_CTAG_TX | 13751 NETIF_F_HW_VLAN_CTAG_RX; 13752 13753 if (!test_bit(I40E_FLAG_MFP_ENA, pf->flags)) 13754 hw_features |= NETIF_F_NTUPLE | NETIF_F_HW_TC; 13755 13756 netdev->hw_features |= hw_features | NETIF_F_LOOPBACK; 13757 13758 netdev->features |= hw_features | NETIF_F_HW_VLAN_CTAG_FILTER; 13759 netdev->hw_enc_features |= NETIF_F_TSO_MANGLEID; 13760 13761 netdev->features &= ~NETIF_F_HW_TC; 13762 13763 if (vsi->type == I40E_VSI_MAIN) { 13764 SET_NETDEV_DEV(netdev, &pf->pdev->dev); 13765 ether_addr_copy(mac_addr, hw->mac.perm_addr); 13766 /* The following steps are necessary for two reasons. First, 13767 * some older NVM configurations load a default MAC-VLAN 13768 * filter that will accept any tagged packet, and we want to 13769 * replace this with a normal filter. Additionally, it is 13770 * possible our MAC address was provided by the platform using 13771 * Open Firmware or similar. 13772 * 13773 * Thus, we need to remove the default filter and install one 13774 * specific to the MAC address. 13775 */ 13776 i40e_rm_default_mac_filter(vsi, mac_addr); 13777 spin_lock_bh(&vsi->mac_filter_hash_lock); 13778 i40e_add_mac_filter(vsi, mac_addr); 13779 spin_unlock_bh(&vsi->mac_filter_hash_lock); 13780 13781 netdev->xdp_features = NETDEV_XDP_ACT_BASIC | 13782 NETDEV_XDP_ACT_REDIRECT | 13783 NETDEV_XDP_ACT_XSK_ZEROCOPY | 13784 NETDEV_XDP_ACT_RX_SG; 13785 netdev->xdp_zc_max_segs = I40E_MAX_BUFFER_TXD; 13786 } else { 13787 /* Relate the VSI_VMDQ name to the VSI_MAIN name. Note that we 13788 * are still limited by IFNAMSIZ, but we're adding 'v%d\0' to 13789 * the end, which is 4 bytes long, so force truncation of the 13790 * original name by IFNAMSIZ - 4 13791 */ 13792 struct i40e_vsi *main_vsi = i40e_pf_get_main_vsi(pf); 13793 13794 snprintf(netdev->name, IFNAMSIZ, "%.*sv%%d", IFNAMSIZ - 4, 13795 main_vsi->netdev->name); 13796 eth_random_addr(mac_addr); 13797 13798 spin_lock_bh(&vsi->mac_filter_hash_lock); 13799 i40e_add_mac_filter(vsi, mac_addr); 13800 spin_unlock_bh(&vsi->mac_filter_hash_lock); 13801 } 13802 13803 /* Add the broadcast filter so that we initially will receive 13804 * broadcast packets. Note that when a new VLAN is first added the 13805 * driver will convert all filters marked I40E_VLAN_ANY into VLAN 13806 * specific filters as part of transitioning into "vlan" operation. 13807 * When more VLANs are added, the driver will copy each existing MAC 13808 * filter and add it for the new VLAN. 13809 * 13810 * Broadcast filters are handled specially by 13811 * i40e_sync_filters_subtask, as the driver must to set the broadcast 13812 * promiscuous bit instead of adding this directly as a MAC/VLAN 13813 * filter. The subtask will update the correct broadcast promiscuous 13814 * bits as VLANs become active or inactive. 13815 */ 13816 eth_broadcast_addr(broadcast); 13817 spin_lock_bh(&vsi->mac_filter_hash_lock); 13818 i40e_add_mac_filter(vsi, broadcast); 13819 spin_unlock_bh(&vsi->mac_filter_hash_lock); 13820 13821 eth_hw_addr_set(netdev, mac_addr); 13822 ether_addr_copy(netdev->perm_addr, mac_addr); 13823 13824 /* i40iw_net_event() reads 16 bytes from neigh->primary_key */ 13825 netdev->neigh_priv_len = sizeof(u32) * 4; 13826 13827 netdev->priv_flags |= IFF_UNICAST_FLT; 13828 netdev->priv_flags |= IFF_SUPP_NOFCS; 13829 /* Setup netdev TC information */ 13830 i40e_vsi_config_netdev_tc(vsi, vsi->tc_config.enabled_tc); 13831 13832 netdev->netdev_ops = &i40e_netdev_ops; 13833 netdev->watchdog_timeo = 5 * HZ; 13834 i40e_set_ethtool_ops(netdev); 13835 13836 /* MTU range: 68 - 9706 */ 13837 netdev->min_mtu = ETH_MIN_MTU; 13838 netdev->max_mtu = I40E_MAX_RXBUFFER - I40E_PACKET_HDR_PAD; 13839 13840 return 0; 13841 } 13842 13843 /** 13844 * i40e_vsi_delete - Delete a VSI from the switch 13845 * @vsi: the VSI being removed 13846 * 13847 * Returns 0 on success, negative value on failure 13848 **/ 13849 static void i40e_vsi_delete(struct i40e_vsi *vsi) 13850 { 13851 /* remove default VSI is not allowed */ 13852 if (vsi == vsi->back->vsi[vsi->back->lan_vsi]) 13853 return; 13854 13855 i40e_aq_delete_element(&vsi->back->hw, vsi->seid, NULL); 13856 } 13857 13858 /** 13859 * i40e_is_vsi_uplink_mode_veb - Check if the VSI's uplink bridge mode is VEB 13860 * @vsi: the VSI being queried 13861 * 13862 * Returns 1 if HW bridge mode is VEB and return 0 in case of VEPA mode 13863 **/ 13864 int i40e_is_vsi_uplink_mode_veb(struct i40e_vsi *vsi) 13865 { 13866 struct i40e_veb *veb; 13867 struct i40e_pf *pf = vsi->back; 13868 13869 /* Uplink is not a bridge so default to VEB */ 13870 if (vsi->veb_idx >= I40E_MAX_VEB) 13871 return 1; 13872 13873 veb = pf->veb[vsi->veb_idx]; 13874 if (!veb) { 13875 dev_info(&pf->pdev->dev, 13876 "There is no veb associated with the bridge\n"); 13877 return -ENOENT; 13878 } 13879 13880 /* Uplink is a bridge in VEPA mode */ 13881 if (veb->bridge_mode & BRIDGE_MODE_VEPA) { 13882 return 0; 13883 } else { 13884 /* Uplink is a bridge in VEB mode */ 13885 return 1; 13886 } 13887 13888 /* VEPA is now default bridge, so return 0 */ 13889 return 0; 13890 } 13891 13892 /** 13893 * i40e_add_vsi - Add a VSI to the switch 13894 * @vsi: the VSI being configured 13895 * 13896 * This initializes a VSI context depending on the VSI type to be added and 13897 * passes it down to the add_vsi aq command. 13898 **/ 13899 static int i40e_add_vsi(struct i40e_vsi *vsi) 13900 { 13901 int ret = -ENODEV; 13902 struct i40e_pf *pf = vsi->back; 13903 struct i40e_hw *hw = &pf->hw; 13904 struct i40e_vsi_context ctxt; 13905 struct i40e_mac_filter *f; 13906 struct hlist_node *h; 13907 int bkt; 13908 13909 u8 enabled_tc = 0x1; /* TC0 enabled */ 13910 int f_count = 0; 13911 13912 memset(&ctxt, 0, sizeof(ctxt)); 13913 switch (vsi->type) { 13914 case I40E_VSI_MAIN: 13915 /* The PF's main VSI is already setup as part of the 13916 * device initialization, so we'll not bother with 13917 * the add_vsi call, but we will retrieve the current 13918 * VSI context. 13919 */ 13920 ctxt.seid = pf->main_vsi_seid; 13921 ctxt.pf_num = pf->hw.pf_id; 13922 ctxt.vf_num = 0; 13923 ret = i40e_aq_get_vsi_params(&pf->hw, &ctxt, NULL); 13924 ctxt.flags = I40E_AQ_VSI_TYPE_PF; 13925 if (ret) { 13926 dev_info(&pf->pdev->dev, 13927 "couldn't get PF vsi config, err %pe aq_err %s\n", 13928 ERR_PTR(ret), 13929 i40e_aq_str(&pf->hw, 13930 pf->hw.aq.asq_last_status)); 13931 return -ENOENT; 13932 } 13933 vsi->info = ctxt.info; 13934 vsi->info.valid_sections = 0; 13935 13936 vsi->seid = ctxt.seid; 13937 vsi->id = ctxt.vsi_number; 13938 13939 enabled_tc = i40e_pf_get_tc_map(pf); 13940 13941 /* Source pruning is enabled by default, so the flag is 13942 * negative logic - if it's set, we need to fiddle with 13943 * the VSI to disable source pruning. 13944 */ 13945 if (test_bit(I40E_FLAG_SOURCE_PRUNING_DIS, pf->flags)) { 13946 memset(&ctxt, 0, sizeof(ctxt)); 13947 ctxt.seid = pf->main_vsi_seid; 13948 ctxt.pf_num = pf->hw.pf_id; 13949 ctxt.vf_num = 0; 13950 ctxt.info.valid_sections |= 13951 cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID); 13952 ctxt.info.switch_id = 13953 cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_LOCAL_LB); 13954 ret = i40e_aq_update_vsi_params(hw, &ctxt, NULL); 13955 if (ret) { 13956 dev_info(&pf->pdev->dev, 13957 "update vsi failed, err %d aq_err %s\n", 13958 ret, 13959 i40e_aq_str(&pf->hw, 13960 pf->hw.aq.asq_last_status)); 13961 ret = -ENOENT; 13962 goto err; 13963 } 13964 } 13965 13966 /* MFP mode setup queue map and update VSI */ 13967 if (test_bit(I40E_FLAG_MFP_ENA, pf->flags) && 13968 !(pf->hw.func_caps.iscsi)) { /* NIC type PF */ 13969 memset(&ctxt, 0, sizeof(ctxt)); 13970 ctxt.seid = pf->main_vsi_seid; 13971 ctxt.pf_num = pf->hw.pf_id; 13972 ctxt.vf_num = 0; 13973 i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, false); 13974 ret = i40e_aq_update_vsi_params(hw, &ctxt, NULL); 13975 if (ret) { 13976 dev_info(&pf->pdev->dev, 13977 "update vsi failed, err %pe aq_err %s\n", 13978 ERR_PTR(ret), 13979 i40e_aq_str(&pf->hw, 13980 pf->hw.aq.asq_last_status)); 13981 ret = -ENOENT; 13982 goto err; 13983 } 13984 /* update the local VSI info queue map */ 13985 i40e_vsi_update_queue_map(vsi, &ctxt); 13986 vsi->info.valid_sections = 0; 13987 } else { 13988 /* Default/Main VSI is only enabled for TC0 13989 * reconfigure it to enable all TCs that are 13990 * available on the port in SFP mode. 13991 * For MFP case the iSCSI PF would use this 13992 * flow to enable LAN+iSCSI TC. 13993 */ 13994 ret = i40e_vsi_config_tc(vsi, enabled_tc); 13995 if (ret) { 13996 /* Single TC condition is not fatal, 13997 * message and continue 13998 */ 13999 dev_info(&pf->pdev->dev, 14000 "failed to configure TCs for main VSI tc_map 0x%08x, err %pe aq_err %s\n", 14001 enabled_tc, 14002 ERR_PTR(ret), 14003 i40e_aq_str(&pf->hw, 14004 pf->hw.aq.asq_last_status)); 14005 } 14006 } 14007 break; 14008 14009 case I40E_VSI_FDIR: 14010 ctxt.pf_num = hw->pf_id; 14011 ctxt.vf_num = 0; 14012 ctxt.uplink_seid = vsi->uplink_seid; 14013 ctxt.connection_type = I40E_AQ_VSI_CONN_TYPE_NORMAL; 14014 ctxt.flags = I40E_AQ_VSI_TYPE_PF; 14015 if (test_bit(I40E_FLAG_VEB_MODE_ENA, pf->flags) && 14016 (i40e_is_vsi_uplink_mode_veb(vsi))) { 14017 ctxt.info.valid_sections |= 14018 cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID); 14019 ctxt.info.switch_id = 14020 cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB); 14021 } 14022 i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, true); 14023 break; 14024 14025 case I40E_VSI_VMDQ2: 14026 ctxt.pf_num = hw->pf_id; 14027 ctxt.vf_num = 0; 14028 ctxt.uplink_seid = vsi->uplink_seid; 14029 ctxt.connection_type = I40E_AQ_VSI_CONN_TYPE_NORMAL; 14030 ctxt.flags = I40E_AQ_VSI_TYPE_VMDQ2; 14031 14032 /* This VSI is connected to VEB so the switch_id 14033 * should be set to zero by default. 14034 */ 14035 if (i40e_is_vsi_uplink_mode_veb(vsi)) { 14036 ctxt.info.valid_sections |= 14037 cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID); 14038 ctxt.info.switch_id = 14039 cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB); 14040 } 14041 14042 /* Setup the VSI tx/rx queue map for TC0 only for now */ 14043 i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, true); 14044 break; 14045 14046 case I40E_VSI_SRIOV: 14047 ctxt.pf_num = hw->pf_id; 14048 ctxt.vf_num = vsi->vf_id + hw->func_caps.vf_base_id; 14049 ctxt.uplink_seid = vsi->uplink_seid; 14050 ctxt.connection_type = I40E_AQ_VSI_CONN_TYPE_NORMAL; 14051 ctxt.flags = I40E_AQ_VSI_TYPE_VF; 14052 14053 /* This VSI is connected to VEB so the switch_id 14054 * should be set to zero by default. 14055 */ 14056 if (i40e_is_vsi_uplink_mode_veb(vsi)) { 14057 ctxt.info.valid_sections |= 14058 cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID); 14059 ctxt.info.switch_id = 14060 cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB); 14061 } 14062 14063 if (test_bit(I40E_FLAG_IWARP_ENA, vsi->back->flags)) { 14064 ctxt.info.valid_sections |= 14065 cpu_to_le16(I40E_AQ_VSI_PROP_QUEUE_OPT_VALID); 14066 ctxt.info.queueing_opt_flags |= 14067 (I40E_AQ_VSI_QUE_OPT_TCP_ENA | 14068 I40E_AQ_VSI_QUE_OPT_RSS_LUT_VSI); 14069 } 14070 14071 ctxt.info.valid_sections |= cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID); 14072 ctxt.info.port_vlan_flags |= I40E_AQ_VSI_PVLAN_MODE_ALL; 14073 if (pf->vf[vsi->vf_id].spoofchk) { 14074 ctxt.info.valid_sections |= 14075 cpu_to_le16(I40E_AQ_VSI_PROP_SECURITY_VALID); 14076 ctxt.info.sec_flags |= 14077 (I40E_AQ_VSI_SEC_FLAG_ENABLE_VLAN_CHK | 14078 I40E_AQ_VSI_SEC_FLAG_ENABLE_MAC_CHK); 14079 } 14080 /* Setup the VSI tx/rx queue map for TC0 only for now */ 14081 i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, true); 14082 break; 14083 14084 case I40E_VSI_IWARP: 14085 /* send down message to iWARP */ 14086 break; 14087 14088 default: 14089 return -ENODEV; 14090 } 14091 14092 if (vsi->type != I40E_VSI_MAIN) { 14093 ret = i40e_aq_add_vsi(hw, &ctxt, NULL); 14094 if (ret) { 14095 dev_info(&vsi->back->pdev->dev, 14096 "add vsi failed, err %pe aq_err %s\n", 14097 ERR_PTR(ret), 14098 i40e_aq_str(&pf->hw, 14099 pf->hw.aq.asq_last_status)); 14100 ret = -ENOENT; 14101 goto err; 14102 } 14103 vsi->info = ctxt.info; 14104 vsi->info.valid_sections = 0; 14105 vsi->seid = ctxt.seid; 14106 vsi->id = ctxt.vsi_number; 14107 } 14108 14109 spin_lock_bh(&vsi->mac_filter_hash_lock); 14110 vsi->active_filters = 0; 14111 /* If macvlan filters already exist, force them to get loaded */ 14112 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) { 14113 f->state = I40E_FILTER_NEW; 14114 f_count++; 14115 } 14116 spin_unlock_bh(&vsi->mac_filter_hash_lock); 14117 clear_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state); 14118 14119 if (f_count) { 14120 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED; 14121 set_bit(__I40E_MACVLAN_SYNC_PENDING, pf->state); 14122 } 14123 14124 /* Update VSI BW information */ 14125 ret = i40e_vsi_get_bw_info(vsi); 14126 if (ret) { 14127 dev_info(&pf->pdev->dev, 14128 "couldn't get vsi bw info, err %pe aq_err %s\n", 14129 ERR_PTR(ret), 14130 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 14131 /* VSI is already added so not tearing that up */ 14132 ret = 0; 14133 } 14134 14135 err: 14136 return ret; 14137 } 14138 14139 /** 14140 * i40e_vsi_release - Delete a VSI and free its resources 14141 * @vsi: the VSI being removed 14142 * 14143 * Returns 0 on success or < 0 on error 14144 **/ 14145 int i40e_vsi_release(struct i40e_vsi *vsi) 14146 { 14147 struct i40e_mac_filter *f; 14148 struct hlist_node *h; 14149 struct i40e_veb *veb; 14150 struct i40e_pf *pf; 14151 u16 uplink_seid; 14152 int i, n, bkt; 14153 14154 pf = vsi->back; 14155 14156 /* release of a VEB-owner or last VSI is not allowed */ 14157 if (vsi->flags & I40E_VSI_FLAG_VEB_OWNER) { 14158 dev_info(&pf->pdev->dev, "VSI %d has existing VEB %d\n", 14159 vsi->seid, vsi->uplink_seid); 14160 return -ENODEV; 14161 } 14162 if (vsi->type == I40E_VSI_MAIN && !test_bit(__I40E_DOWN, pf->state)) { 14163 dev_info(&pf->pdev->dev, "Can't remove PF VSI\n"); 14164 return -ENODEV; 14165 } 14166 set_bit(__I40E_VSI_RELEASING, vsi->state); 14167 uplink_seid = vsi->uplink_seid; 14168 14169 if (vsi->type != I40E_VSI_SRIOV) { 14170 if (vsi->netdev_registered) { 14171 vsi->netdev_registered = false; 14172 if (vsi->netdev) { 14173 /* results in a call to i40e_close() */ 14174 unregister_netdev(vsi->netdev); 14175 } 14176 } else { 14177 i40e_vsi_close(vsi); 14178 } 14179 i40e_vsi_disable_irq(vsi); 14180 } 14181 14182 if (vsi->type == I40E_VSI_MAIN) 14183 i40e_devlink_destroy_port(pf); 14184 14185 spin_lock_bh(&vsi->mac_filter_hash_lock); 14186 14187 /* clear the sync flag on all filters */ 14188 if (vsi->netdev) { 14189 __dev_uc_unsync(vsi->netdev, NULL); 14190 __dev_mc_unsync(vsi->netdev, NULL); 14191 } 14192 14193 /* make sure any remaining filters are marked for deletion */ 14194 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) 14195 __i40e_del_filter(vsi, f); 14196 14197 spin_unlock_bh(&vsi->mac_filter_hash_lock); 14198 14199 i40e_sync_vsi_filters(vsi); 14200 14201 i40e_vsi_delete(vsi); 14202 i40e_vsi_free_q_vectors(vsi); 14203 if (vsi->netdev) { 14204 free_netdev(vsi->netdev); 14205 vsi->netdev = NULL; 14206 } 14207 i40e_vsi_clear_rings(vsi); 14208 i40e_vsi_clear(vsi); 14209 14210 /* If this was the last thing on the VEB, except for the 14211 * controlling VSI, remove the VEB, which puts the controlling 14212 * VSI onto the uplink port. 14213 * 14214 * Well, okay, there's one more exception here: don't remove 14215 * the floating VEBs yet. We'll wait for an explicit remove request 14216 * from up the network stack. 14217 */ 14218 veb = i40e_pf_get_veb_by_seid(pf, uplink_seid); 14219 if (veb && veb->uplink_seid) { 14220 n = 0; 14221 14222 /* Count non-controlling VSIs present on the VEB */ 14223 i40e_pf_for_each_vsi(pf, i, vsi) 14224 if (vsi->uplink_seid == uplink_seid && 14225 (vsi->flags & I40E_VSI_FLAG_VEB_OWNER) == 0) 14226 n++; 14227 14228 /* If there is no VSI except the control one then release 14229 * the VEB and put the control VSI onto VEB uplink. 14230 */ 14231 if (!n) 14232 i40e_veb_release(veb); 14233 } 14234 14235 return 0; 14236 } 14237 14238 /** 14239 * i40e_vsi_setup_vectors - Set up the q_vectors for the given VSI 14240 * @vsi: ptr to the VSI 14241 * 14242 * This should only be called after i40e_vsi_mem_alloc() which allocates the 14243 * corresponding SW VSI structure and initializes num_queue_pairs for the 14244 * newly allocated VSI. 14245 * 14246 * Returns 0 on success or negative on failure 14247 **/ 14248 static int i40e_vsi_setup_vectors(struct i40e_vsi *vsi) 14249 { 14250 int ret = -ENOENT; 14251 struct i40e_pf *pf = vsi->back; 14252 14253 if (vsi->q_vectors[0]) { 14254 dev_info(&pf->pdev->dev, "VSI %d has existing q_vectors\n", 14255 vsi->seid); 14256 return -EEXIST; 14257 } 14258 14259 if (vsi->base_vector) { 14260 dev_info(&pf->pdev->dev, "VSI %d has non-zero base vector %d\n", 14261 vsi->seid, vsi->base_vector); 14262 return -EEXIST; 14263 } 14264 14265 ret = i40e_vsi_alloc_q_vectors(vsi); 14266 if (ret) { 14267 dev_info(&pf->pdev->dev, 14268 "failed to allocate %d q_vector for VSI %d, ret=%d\n", 14269 vsi->num_q_vectors, vsi->seid, ret); 14270 vsi->num_q_vectors = 0; 14271 goto vector_setup_out; 14272 } 14273 14274 /* In Legacy mode, we do not have to get any other vector since we 14275 * piggyback on the misc/ICR0 for queue interrupts. 14276 */ 14277 if (!test_bit(I40E_FLAG_MSIX_ENA, pf->flags)) 14278 return ret; 14279 if (vsi->num_q_vectors) 14280 vsi->base_vector = i40e_get_lump(pf, pf->irq_pile, 14281 vsi->num_q_vectors, vsi->idx); 14282 if (vsi->base_vector < 0) { 14283 dev_info(&pf->pdev->dev, 14284 "failed to get tracking for %d vectors for VSI %d, err=%d\n", 14285 vsi->num_q_vectors, vsi->seid, vsi->base_vector); 14286 i40e_vsi_free_q_vectors(vsi); 14287 ret = -ENOENT; 14288 goto vector_setup_out; 14289 } 14290 14291 vector_setup_out: 14292 return ret; 14293 } 14294 14295 /** 14296 * i40e_vsi_reinit_setup - return and reallocate resources for a VSI 14297 * @vsi: pointer to the vsi. 14298 * 14299 * This re-allocates a vsi's queue resources. 14300 * 14301 * Returns pointer to the successfully allocated and configured VSI sw struct 14302 * on success, otherwise returns NULL on failure. 14303 **/ 14304 static struct i40e_vsi *i40e_vsi_reinit_setup(struct i40e_vsi *vsi) 14305 { 14306 struct i40e_vsi *main_vsi; 14307 u16 alloc_queue_pairs; 14308 struct i40e_pf *pf; 14309 int ret; 14310 14311 if (!vsi) 14312 return NULL; 14313 14314 pf = vsi->back; 14315 14316 i40e_put_lump(pf->qp_pile, vsi->base_queue, vsi->idx); 14317 i40e_vsi_clear_rings(vsi); 14318 14319 i40e_vsi_free_arrays(vsi, false); 14320 i40e_set_num_rings_in_vsi(vsi); 14321 ret = i40e_vsi_alloc_arrays(vsi, false); 14322 if (ret) 14323 goto err_vsi; 14324 14325 alloc_queue_pairs = vsi->alloc_queue_pairs * 14326 (i40e_enabled_xdp_vsi(vsi) ? 2 : 1); 14327 14328 ret = i40e_get_lump(pf, pf->qp_pile, alloc_queue_pairs, vsi->idx); 14329 if (ret < 0) { 14330 dev_info(&pf->pdev->dev, 14331 "failed to get tracking for %d queues for VSI %d err %d\n", 14332 alloc_queue_pairs, vsi->seid, ret); 14333 goto err_vsi; 14334 } 14335 vsi->base_queue = ret; 14336 14337 /* Update the FW view of the VSI. Force a reset of TC and queue 14338 * layout configurations. 14339 */ 14340 main_vsi = i40e_pf_get_main_vsi(pf); 14341 main_vsi->seid = pf->main_vsi_seid; 14342 i40e_vsi_reconfig_tc(main_vsi); 14343 14344 if (vsi->type == I40E_VSI_MAIN) 14345 i40e_rm_default_mac_filter(vsi, pf->hw.mac.perm_addr); 14346 14347 /* assign it some queues */ 14348 ret = i40e_alloc_rings(vsi); 14349 if (ret) 14350 goto err_rings; 14351 14352 /* map all of the rings to the q_vectors */ 14353 i40e_vsi_map_rings_to_vectors(vsi); 14354 return vsi; 14355 14356 err_rings: 14357 i40e_vsi_free_q_vectors(vsi); 14358 if (vsi->netdev_registered) { 14359 vsi->netdev_registered = false; 14360 unregister_netdev(vsi->netdev); 14361 free_netdev(vsi->netdev); 14362 vsi->netdev = NULL; 14363 } 14364 if (vsi->type == I40E_VSI_MAIN) 14365 i40e_devlink_destroy_port(pf); 14366 i40e_aq_delete_element(&pf->hw, vsi->seid, NULL); 14367 err_vsi: 14368 i40e_vsi_clear(vsi); 14369 return NULL; 14370 } 14371 14372 /** 14373 * i40e_vsi_setup - Set up a VSI by a given type 14374 * @pf: board private structure 14375 * @type: VSI type 14376 * @uplink_seid: the switch element to link to 14377 * @param1: usage depends upon VSI type. For VF types, indicates VF id 14378 * 14379 * This allocates the sw VSI structure and its queue resources, then add a VSI 14380 * to the identified VEB. 14381 * 14382 * Returns pointer to the successfully allocated and configure VSI sw struct on 14383 * success, otherwise returns NULL on failure. 14384 **/ 14385 struct i40e_vsi *i40e_vsi_setup(struct i40e_pf *pf, u8 type, 14386 u16 uplink_seid, u32 param1) 14387 { 14388 struct i40e_vsi *vsi = NULL; 14389 struct i40e_veb *veb = NULL; 14390 u16 alloc_queue_pairs; 14391 int v_idx; 14392 int ret; 14393 14394 /* The requested uplink_seid must be either 14395 * - the PF's port seid 14396 * no VEB is needed because this is the PF 14397 * or this is a Flow Director special case VSI 14398 * - seid of an existing VEB 14399 * - seid of a VSI that owns an existing VEB 14400 * - seid of a VSI that doesn't own a VEB 14401 * a new VEB is created and the VSI becomes the owner 14402 * - seid of the PF VSI, which is what creates the first VEB 14403 * this is a special case of the previous 14404 * 14405 * Find which uplink_seid we were given and create a new VEB if needed 14406 */ 14407 veb = i40e_pf_get_veb_by_seid(pf, uplink_seid); 14408 if (!veb && uplink_seid != pf->mac_seid) { 14409 vsi = i40e_pf_get_vsi_by_seid(pf, uplink_seid); 14410 if (!vsi) { 14411 dev_info(&pf->pdev->dev, "no such uplink_seid %d\n", 14412 uplink_seid); 14413 return NULL; 14414 } 14415 14416 if (vsi->uplink_seid == pf->mac_seid) 14417 veb = i40e_veb_setup(pf, pf->mac_seid, vsi->seid, 14418 vsi->tc_config.enabled_tc); 14419 else if ((vsi->flags & I40E_VSI_FLAG_VEB_OWNER) == 0) 14420 veb = i40e_veb_setup(pf, vsi->uplink_seid, vsi->seid, 14421 vsi->tc_config.enabled_tc); 14422 if (veb) { 14423 if (vsi->type != I40E_VSI_MAIN) { 14424 dev_info(&vsi->back->pdev->dev, 14425 "New VSI creation error, uplink seid of LAN VSI expected.\n"); 14426 return NULL; 14427 } 14428 /* We come up by default in VEPA mode if SRIOV is not 14429 * already enabled, in which case we can't force VEPA 14430 * mode. 14431 */ 14432 if (!test_bit(I40E_FLAG_VEB_MODE_ENA, pf->flags)) { 14433 veb->bridge_mode = BRIDGE_MODE_VEPA; 14434 clear_bit(I40E_FLAG_VEB_MODE_ENA, pf->flags); 14435 } 14436 i40e_config_bridge_mode(veb); 14437 } 14438 veb = i40e_pf_get_veb_by_seid(pf, vsi->uplink_seid); 14439 if (!veb) { 14440 dev_info(&pf->pdev->dev, "couldn't add VEB\n"); 14441 return NULL; 14442 } 14443 14444 vsi->flags |= I40E_VSI_FLAG_VEB_OWNER; 14445 uplink_seid = veb->seid; 14446 } 14447 14448 /* get vsi sw struct */ 14449 v_idx = i40e_vsi_mem_alloc(pf, type); 14450 if (v_idx < 0) 14451 goto err_alloc; 14452 vsi = pf->vsi[v_idx]; 14453 if (!vsi) 14454 goto err_alloc; 14455 vsi->type = type; 14456 vsi->veb_idx = (veb ? veb->idx : I40E_NO_VEB); 14457 14458 if (type == I40E_VSI_MAIN) 14459 pf->lan_vsi = v_idx; 14460 else if (type == I40E_VSI_SRIOV) 14461 vsi->vf_id = param1; 14462 /* assign it some queues */ 14463 alloc_queue_pairs = vsi->alloc_queue_pairs * 14464 (i40e_enabled_xdp_vsi(vsi) ? 2 : 1); 14465 14466 ret = i40e_get_lump(pf, pf->qp_pile, alloc_queue_pairs, vsi->idx); 14467 if (ret < 0) { 14468 dev_info(&pf->pdev->dev, 14469 "failed to get tracking for %d queues for VSI %d err=%d\n", 14470 alloc_queue_pairs, vsi->seid, ret); 14471 goto err_vsi; 14472 } 14473 vsi->base_queue = ret; 14474 14475 /* get a VSI from the hardware */ 14476 vsi->uplink_seid = uplink_seid; 14477 ret = i40e_add_vsi(vsi); 14478 if (ret) 14479 goto err_vsi; 14480 14481 switch (vsi->type) { 14482 /* setup the netdev if needed */ 14483 case I40E_VSI_MAIN: 14484 case I40E_VSI_VMDQ2: 14485 ret = i40e_config_netdev(vsi); 14486 if (ret) 14487 goto err_netdev; 14488 ret = i40e_netif_set_realnum_tx_rx_queues(vsi); 14489 if (ret) 14490 goto err_netdev; 14491 if (vsi->type == I40E_VSI_MAIN) { 14492 ret = i40e_devlink_create_port(pf); 14493 if (ret) 14494 goto err_netdev; 14495 SET_NETDEV_DEVLINK_PORT(vsi->netdev, &pf->devlink_port); 14496 } 14497 ret = register_netdev(vsi->netdev); 14498 if (ret) 14499 goto err_dl_port; 14500 vsi->netdev_registered = true; 14501 netif_carrier_off(vsi->netdev); 14502 #ifdef CONFIG_I40E_DCB 14503 /* Setup DCB netlink interface */ 14504 i40e_dcbnl_setup(vsi); 14505 #endif /* CONFIG_I40E_DCB */ 14506 fallthrough; 14507 case I40E_VSI_FDIR: 14508 /* set up vectors and rings if needed */ 14509 ret = i40e_vsi_setup_vectors(vsi); 14510 if (ret) 14511 goto err_msix; 14512 14513 ret = i40e_alloc_rings(vsi); 14514 if (ret) 14515 goto err_rings; 14516 14517 /* map all of the rings to the q_vectors */ 14518 i40e_vsi_map_rings_to_vectors(vsi); 14519 14520 i40e_vsi_reset_stats(vsi); 14521 break; 14522 default: 14523 /* no netdev or rings for the other VSI types */ 14524 break; 14525 } 14526 14527 if (test_bit(I40E_HW_CAP_RSS_AQ, pf->hw.caps) && 14528 vsi->type == I40E_VSI_VMDQ2) { 14529 ret = i40e_vsi_config_rss(vsi); 14530 if (ret) 14531 goto err_config; 14532 } 14533 return vsi; 14534 14535 err_config: 14536 i40e_vsi_clear_rings(vsi); 14537 err_rings: 14538 i40e_vsi_free_q_vectors(vsi); 14539 err_msix: 14540 if (vsi->netdev_registered) { 14541 vsi->netdev_registered = false; 14542 unregister_netdev(vsi->netdev); 14543 free_netdev(vsi->netdev); 14544 vsi->netdev = NULL; 14545 } 14546 err_dl_port: 14547 if (vsi->type == I40E_VSI_MAIN) 14548 i40e_devlink_destroy_port(pf); 14549 err_netdev: 14550 i40e_aq_delete_element(&pf->hw, vsi->seid, NULL); 14551 err_vsi: 14552 i40e_vsi_clear(vsi); 14553 err_alloc: 14554 return NULL; 14555 } 14556 14557 /** 14558 * i40e_veb_get_bw_info - Query VEB BW information 14559 * @veb: the veb to query 14560 * 14561 * Query the Tx scheduler BW configuration data for given VEB 14562 **/ 14563 static int i40e_veb_get_bw_info(struct i40e_veb *veb) 14564 { 14565 struct i40e_aqc_query_switching_comp_ets_config_resp ets_data; 14566 struct i40e_aqc_query_switching_comp_bw_config_resp bw_data; 14567 struct i40e_pf *pf = veb->pf; 14568 struct i40e_hw *hw = &pf->hw; 14569 u32 tc_bw_max; 14570 int ret = 0; 14571 int i; 14572 14573 ret = i40e_aq_query_switch_comp_bw_config(hw, veb->seid, 14574 &bw_data, NULL); 14575 if (ret) { 14576 dev_info(&pf->pdev->dev, 14577 "query veb bw config failed, err %pe aq_err %s\n", 14578 ERR_PTR(ret), 14579 i40e_aq_str(&pf->hw, hw->aq.asq_last_status)); 14580 goto out; 14581 } 14582 14583 ret = i40e_aq_query_switch_comp_ets_config(hw, veb->seid, 14584 &ets_data, NULL); 14585 if (ret) { 14586 dev_info(&pf->pdev->dev, 14587 "query veb bw ets config failed, err %pe aq_err %s\n", 14588 ERR_PTR(ret), 14589 i40e_aq_str(&pf->hw, hw->aq.asq_last_status)); 14590 goto out; 14591 } 14592 14593 veb->bw_limit = le16_to_cpu(ets_data.port_bw_limit); 14594 veb->bw_max_quanta = ets_data.tc_bw_max; 14595 veb->is_abs_credits = bw_data.absolute_credits_enable; 14596 veb->enabled_tc = ets_data.tc_valid_bits; 14597 tc_bw_max = le16_to_cpu(bw_data.tc_bw_max[0]) | 14598 (le16_to_cpu(bw_data.tc_bw_max[1]) << 16); 14599 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 14600 veb->bw_tc_share_credits[i] = bw_data.tc_bw_share_credits[i]; 14601 veb->bw_tc_limit_credits[i] = 14602 le16_to_cpu(bw_data.tc_bw_limits[i]); 14603 veb->bw_tc_max_quanta[i] = ((tc_bw_max >> (i*4)) & 0x7); 14604 } 14605 14606 out: 14607 return ret; 14608 } 14609 14610 /** 14611 * i40e_veb_mem_alloc - Allocates the next available struct veb in the PF 14612 * @pf: board private structure 14613 * 14614 * On error: returns error code (negative) 14615 * On success: returns vsi index in PF (positive) 14616 **/ 14617 static int i40e_veb_mem_alloc(struct i40e_pf *pf) 14618 { 14619 int ret = -ENOENT; 14620 struct i40e_veb *veb; 14621 int i; 14622 14623 /* Need to protect the allocation of switch elements at the PF level */ 14624 mutex_lock(&pf->switch_mutex); 14625 14626 /* VEB list may be fragmented if VEB creation/destruction has 14627 * been happening. We can afford to do a quick scan to look 14628 * for any free slots in the list. 14629 * 14630 * find next empty veb slot, looping back around if necessary 14631 */ 14632 i = 0; 14633 while ((i < I40E_MAX_VEB) && (pf->veb[i] != NULL)) 14634 i++; 14635 if (i >= I40E_MAX_VEB) { 14636 ret = -ENOMEM; 14637 goto err_alloc_veb; /* out of VEB slots! */ 14638 } 14639 14640 veb = kzalloc(sizeof(*veb), GFP_KERNEL); 14641 if (!veb) { 14642 ret = -ENOMEM; 14643 goto err_alloc_veb; 14644 } 14645 veb->pf = pf; 14646 veb->idx = i; 14647 veb->enabled_tc = 1; 14648 14649 pf->veb[i] = veb; 14650 ret = i; 14651 err_alloc_veb: 14652 mutex_unlock(&pf->switch_mutex); 14653 return ret; 14654 } 14655 14656 /** 14657 * i40e_switch_branch_release - Delete a branch of the switch tree 14658 * @branch: where to start deleting 14659 * 14660 * This uses recursion to find the tips of the branch to be 14661 * removed, deleting until we get back to and can delete this VEB. 14662 **/ 14663 static void i40e_switch_branch_release(struct i40e_veb *branch) 14664 { 14665 struct i40e_pf *pf = branch->pf; 14666 u16 branch_seid = branch->seid; 14667 u16 veb_idx = branch->idx; 14668 struct i40e_vsi *vsi; 14669 struct i40e_veb *veb; 14670 int i; 14671 14672 /* release any VEBs on this VEB - RECURSION */ 14673 i40e_pf_for_each_veb(pf, i, veb) 14674 if (veb->uplink_seid == branch->seid) 14675 i40e_switch_branch_release(veb); 14676 14677 /* Release the VSIs on this VEB, but not the owner VSI. 14678 * 14679 * NOTE: Removing the last VSI on a VEB has the SIDE EFFECT of removing 14680 * the VEB itself, so don't use (*branch) after this loop. 14681 */ 14682 i40e_pf_for_each_vsi(pf, i, vsi) 14683 if (vsi->uplink_seid == branch_seid && 14684 (vsi->flags & I40E_VSI_FLAG_VEB_OWNER) == 0) 14685 i40e_vsi_release(vsi); 14686 14687 /* There's one corner case where the VEB might not have been 14688 * removed, so double check it here and remove it if needed. 14689 * This case happens if the veb was created from the debugfs 14690 * commands and no VSIs were added to it. 14691 */ 14692 if (pf->veb[veb_idx]) 14693 i40e_veb_release(pf->veb[veb_idx]); 14694 } 14695 14696 /** 14697 * i40e_veb_clear - remove veb struct 14698 * @veb: the veb to remove 14699 **/ 14700 static void i40e_veb_clear(struct i40e_veb *veb) 14701 { 14702 if (!veb) 14703 return; 14704 14705 if (veb->pf) { 14706 struct i40e_pf *pf = veb->pf; 14707 14708 mutex_lock(&pf->switch_mutex); 14709 if (pf->veb[veb->idx] == veb) 14710 pf->veb[veb->idx] = NULL; 14711 mutex_unlock(&pf->switch_mutex); 14712 } 14713 14714 kfree(veb); 14715 } 14716 14717 /** 14718 * i40e_veb_release - Delete a VEB and free its resources 14719 * @veb: the VEB being removed 14720 **/ 14721 void i40e_veb_release(struct i40e_veb *veb) 14722 { 14723 struct i40e_vsi *vsi, *vsi_it; 14724 struct i40e_pf *pf; 14725 int i, n = 0; 14726 14727 pf = veb->pf; 14728 14729 /* find the remaining VSI and check for extras */ 14730 i40e_pf_for_each_vsi(pf, i, vsi_it) 14731 if (vsi_it->uplink_seid == veb->seid) { 14732 if (vsi_it->flags & I40E_VSI_FLAG_VEB_OWNER) 14733 vsi = vsi_it; 14734 n++; 14735 } 14736 14737 /* Floating VEB has to be empty and regular one must have 14738 * single owner VSI. 14739 */ 14740 if ((veb->uplink_seid && n != 1) || (!veb->uplink_seid && n != 0)) { 14741 dev_info(&pf->pdev->dev, 14742 "can't remove VEB %d with %d VSIs left\n", 14743 veb->seid, n); 14744 return; 14745 } 14746 14747 /* For regular VEB move the owner VSI to uplink port */ 14748 if (veb->uplink_seid) { 14749 vsi->flags &= ~I40E_VSI_FLAG_VEB_OWNER; 14750 vsi->uplink_seid = veb->uplink_seid; 14751 vsi->veb_idx = I40E_NO_VEB; 14752 } 14753 14754 i40e_aq_delete_element(&pf->hw, veb->seid, NULL); 14755 i40e_veb_clear(veb); 14756 } 14757 14758 /** 14759 * i40e_add_veb - create the VEB in the switch 14760 * @veb: the VEB to be instantiated 14761 * @vsi: the controlling VSI 14762 **/ 14763 static int i40e_add_veb(struct i40e_veb *veb, struct i40e_vsi *vsi) 14764 { 14765 struct i40e_pf *pf = veb->pf; 14766 bool enable_stats = !!test_bit(I40E_FLAG_VEB_STATS_ENA, pf->flags); 14767 int ret; 14768 14769 ret = i40e_aq_add_veb(&pf->hw, veb->uplink_seid, vsi ? vsi->seid : 0, 14770 veb->enabled_tc, vsi ? false : true, 14771 &veb->seid, enable_stats, NULL); 14772 14773 /* get a VEB from the hardware */ 14774 if (ret) { 14775 dev_info(&pf->pdev->dev, 14776 "couldn't add VEB, err %pe aq_err %s\n", 14777 ERR_PTR(ret), 14778 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 14779 return -EPERM; 14780 } 14781 14782 /* get statistics counter */ 14783 ret = i40e_aq_get_veb_parameters(&pf->hw, veb->seid, NULL, NULL, 14784 &veb->stats_idx, NULL, NULL, NULL); 14785 if (ret) { 14786 dev_info(&pf->pdev->dev, 14787 "couldn't get VEB statistics idx, err %pe aq_err %s\n", 14788 ERR_PTR(ret), 14789 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 14790 return -EPERM; 14791 } 14792 ret = i40e_veb_get_bw_info(veb); 14793 if (ret) { 14794 dev_info(&pf->pdev->dev, 14795 "couldn't get VEB bw info, err %pe aq_err %s\n", 14796 ERR_PTR(ret), 14797 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 14798 i40e_aq_delete_element(&pf->hw, veb->seid, NULL); 14799 return -ENOENT; 14800 } 14801 14802 if (vsi) { 14803 vsi->uplink_seid = veb->seid; 14804 vsi->veb_idx = veb->idx; 14805 vsi->flags |= I40E_VSI_FLAG_VEB_OWNER; 14806 } 14807 14808 return 0; 14809 } 14810 14811 /** 14812 * i40e_veb_setup - Set up a VEB 14813 * @pf: board private structure 14814 * @uplink_seid: the switch element to link to 14815 * @vsi_seid: the initial VSI seid 14816 * @enabled_tc: Enabled TC bit-map 14817 * 14818 * This allocates the sw VEB structure and links it into the switch 14819 * It is possible and legal for this to be a duplicate of an already 14820 * existing VEB. It is also possible for both uplink and vsi seids 14821 * to be zero, in order to create a floating VEB. 14822 * 14823 * Returns pointer to the successfully allocated VEB sw struct on 14824 * success, otherwise returns NULL on failure. 14825 **/ 14826 struct i40e_veb *i40e_veb_setup(struct i40e_pf *pf, u16 uplink_seid, 14827 u16 vsi_seid, u8 enabled_tc) 14828 { 14829 struct i40e_vsi *vsi = NULL; 14830 struct i40e_veb *veb; 14831 int veb_idx; 14832 int ret; 14833 14834 /* if one seid is 0, the other must be 0 to create a floating relay */ 14835 if ((uplink_seid == 0 || vsi_seid == 0) && 14836 (uplink_seid + vsi_seid != 0)) { 14837 dev_info(&pf->pdev->dev, 14838 "one, not both seid's are 0: uplink=%d vsi=%d\n", 14839 uplink_seid, vsi_seid); 14840 return NULL; 14841 } 14842 14843 /* make sure there is such a vsi and uplink */ 14844 if (vsi_seid) { 14845 vsi = i40e_pf_get_vsi_by_seid(pf, vsi_seid); 14846 if (!vsi) { 14847 dev_err(&pf->pdev->dev, "vsi seid %d not found\n", 14848 vsi_seid); 14849 return NULL; 14850 } 14851 } 14852 14853 /* get veb sw struct */ 14854 veb_idx = i40e_veb_mem_alloc(pf); 14855 if (veb_idx < 0) 14856 goto err_alloc; 14857 veb = pf->veb[veb_idx]; 14858 veb->uplink_seid = uplink_seid; 14859 veb->enabled_tc = (enabled_tc ? enabled_tc : 0x1); 14860 14861 /* create the VEB in the switch */ 14862 ret = i40e_add_veb(veb, vsi); 14863 if (ret) 14864 goto err_veb; 14865 14866 if (vsi && vsi->idx == pf->lan_vsi) 14867 pf->lan_veb = veb->idx; 14868 14869 return veb; 14870 14871 err_veb: 14872 i40e_veb_clear(veb); 14873 err_alloc: 14874 return NULL; 14875 } 14876 14877 /** 14878 * i40e_setup_pf_switch_element - set PF vars based on switch type 14879 * @pf: board private structure 14880 * @ele: element we are building info from 14881 * @num_reported: total number of elements 14882 * @printconfig: should we print the contents 14883 * 14884 * helper function to assist in extracting a few useful SEID values. 14885 **/ 14886 static void i40e_setup_pf_switch_element(struct i40e_pf *pf, 14887 struct i40e_aqc_switch_config_element_resp *ele, 14888 u16 num_reported, bool printconfig) 14889 { 14890 u16 downlink_seid = le16_to_cpu(ele->downlink_seid); 14891 u16 uplink_seid = le16_to_cpu(ele->uplink_seid); 14892 u8 element_type = ele->element_type; 14893 u16 seid = le16_to_cpu(ele->seid); 14894 struct i40e_veb *veb; 14895 14896 if (printconfig) 14897 dev_info(&pf->pdev->dev, 14898 "type=%d seid=%d uplink=%d downlink=%d\n", 14899 element_type, seid, uplink_seid, downlink_seid); 14900 14901 switch (element_type) { 14902 case I40E_SWITCH_ELEMENT_TYPE_MAC: 14903 pf->mac_seid = seid; 14904 break; 14905 case I40E_SWITCH_ELEMENT_TYPE_VEB: 14906 /* Main VEB? */ 14907 if (uplink_seid != pf->mac_seid) 14908 break; 14909 veb = i40e_pf_get_main_veb(pf); 14910 if (!veb) { 14911 int v; 14912 14913 /* find existing or else empty VEB */ 14914 veb = i40e_pf_get_veb_by_seid(pf, seid); 14915 if (veb) { 14916 pf->lan_veb = veb->idx; 14917 } else { 14918 v = i40e_veb_mem_alloc(pf); 14919 if (v < 0) 14920 break; 14921 pf->lan_veb = v; 14922 } 14923 } 14924 14925 /* Try to get again main VEB as pf->lan_veb may have changed */ 14926 veb = i40e_pf_get_main_veb(pf); 14927 if (!veb) 14928 break; 14929 14930 veb->seid = seid; 14931 veb->uplink_seid = pf->mac_seid; 14932 veb->pf = pf; 14933 break; 14934 case I40E_SWITCH_ELEMENT_TYPE_VSI: 14935 if (num_reported != 1) 14936 break; 14937 /* This is immediately after a reset so we can assume this is 14938 * the PF's VSI 14939 */ 14940 pf->mac_seid = uplink_seid; 14941 pf->main_vsi_seid = seid; 14942 if (printconfig) 14943 dev_info(&pf->pdev->dev, 14944 "pf_seid=%d main_vsi_seid=%d\n", 14945 downlink_seid, pf->main_vsi_seid); 14946 break; 14947 case I40E_SWITCH_ELEMENT_TYPE_PF: 14948 case I40E_SWITCH_ELEMENT_TYPE_VF: 14949 case I40E_SWITCH_ELEMENT_TYPE_EMP: 14950 case I40E_SWITCH_ELEMENT_TYPE_BMC: 14951 case I40E_SWITCH_ELEMENT_TYPE_PE: 14952 case I40E_SWITCH_ELEMENT_TYPE_PA: 14953 /* ignore these for now */ 14954 break; 14955 default: 14956 dev_info(&pf->pdev->dev, "unknown element type=%d seid=%d\n", 14957 element_type, seid); 14958 break; 14959 } 14960 } 14961 14962 /** 14963 * i40e_fetch_switch_configuration - Get switch config from firmware 14964 * @pf: board private structure 14965 * @printconfig: should we print the contents 14966 * 14967 * Get the current switch configuration from the device and 14968 * extract a few useful SEID values. 14969 **/ 14970 int i40e_fetch_switch_configuration(struct i40e_pf *pf, bool printconfig) 14971 { 14972 struct i40e_aqc_get_switch_config_resp *sw_config; 14973 u16 next_seid = 0; 14974 int ret = 0; 14975 u8 *aq_buf; 14976 int i; 14977 14978 aq_buf = kzalloc(I40E_AQ_LARGE_BUF, GFP_KERNEL); 14979 if (!aq_buf) 14980 return -ENOMEM; 14981 14982 sw_config = (struct i40e_aqc_get_switch_config_resp *)aq_buf; 14983 do { 14984 u16 num_reported, num_total; 14985 14986 ret = i40e_aq_get_switch_config(&pf->hw, sw_config, 14987 I40E_AQ_LARGE_BUF, 14988 &next_seid, NULL); 14989 if (ret) { 14990 dev_info(&pf->pdev->dev, 14991 "get switch config failed err %d aq_err %s\n", 14992 ret, 14993 i40e_aq_str(&pf->hw, 14994 pf->hw.aq.asq_last_status)); 14995 kfree(aq_buf); 14996 return -ENOENT; 14997 } 14998 14999 num_reported = le16_to_cpu(sw_config->header.num_reported); 15000 num_total = le16_to_cpu(sw_config->header.num_total); 15001 15002 if (printconfig) 15003 dev_info(&pf->pdev->dev, 15004 "header: %d reported %d total\n", 15005 num_reported, num_total); 15006 15007 for (i = 0; i < num_reported; i++) { 15008 struct i40e_aqc_switch_config_element_resp *ele = 15009 &sw_config->element[i]; 15010 15011 i40e_setup_pf_switch_element(pf, ele, num_reported, 15012 printconfig); 15013 } 15014 } while (next_seid != 0); 15015 15016 kfree(aq_buf); 15017 return ret; 15018 } 15019 15020 /** 15021 * i40e_setup_pf_switch - Setup the HW switch on startup or after reset 15022 * @pf: board private structure 15023 * @reinit: if the Main VSI needs to re-initialized. 15024 * @lock_acquired: indicates whether or not the lock has been acquired 15025 * 15026 * Returns 0 on success, negative value on failure 15027 **/ 15028 static int i40e_setup_pf_switch(struct i40e_pf *pf, bool reinit, bool lock_acquired) 15029 { 15030 struct i40e_vsi *main_vsi; 15031 u16 flags = 0; 15032 int ret; 15033 15034 /* find out what's out there already */ 15035 ret = i40e_fetch_switch_configuration(pf, false); 15036 if (ret) { 15037 dev_info(&pf->pdev->dev, 15038 "couldn't fetch switch config, err %pe aq_err %s\n", 15039 ERR_PTR(ret), 15040 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 15041 return ret; 15042 } 15043 i40e_pf_reset_stats(pf); 15044 15045 /* set the switch config bit for the whole device to 15046 * support limited promisc or true promisc 15047 * when user requests promisc. The default is limited 15048 * promisc. 15049 */ 15050 15051 if ((pf->hw.pf_id == 0) && 15052 !test_bit(I40E_FLAG_TRUE_PROMISC_ENA, pf->flags)) { 15053 flags = I40E_AQ_SET_SWITCH_CFG_PROMISC; 15054 pf->last_sw_conf_flags = flags; 15055 } 15056 15057 if (pf->hw.pf_id == 0) { 15058 u16 valid_flags; 15059 15060 valid_flags = I40E_AQ_SET_SWITCH_CFG_PROMISC; 15061 ret = i40e_aq_set_switch_config(&pf->hw, flags, valid_flags, 0, 15062 NULL); 15063 if (ret && pf->hw.aq.asq_last_status != I40E_AQ_RC_ESRCH) { 15064 dev_info(&pf->pdev->dev, 15065 "couldn't set switch config bits, err %pe aq_err %s\n", 15066 ERR_PTR(ret), 15067 i40e_aq_str(&pf->hw, 15068 pf->hw.aq.asq_last_status)); 15069 /* not a fatal problem, just keep going */ 15070 } 15071 pf->last_sw_conf_valid_flags = valid_flags; 15072 } 15073 15074 /* first time setup */ 15075 main_vsi = i40e_pf_get_main_vsi(pf); 15076 if (!main_vsi || reinit) { 15077 struct i40e_veb *veb; 15078 u16 uplink_seid; 15079 15080 /* Set up the PF VSI associated with the PF's main VSI 15081 * that is already in the HW switch 15082 */ 15083 veb = i40e_pf_get_main_veb(pf); 15084 if (veb) 15085 uplink_seid = veb->seid; 15086 else 15087 uplink_seid = pf->mac_seid; 15088 if (!main_vsi) 15089 main_vsi = i40e_vsi_setup(pf, I40E_VSI_MAIN, 15090 uplink_seid, 0); 15091 else if (reinit) 15092 main_vsi = i40e_vsi_reinit_setup(main_vsi); 15093 if (!main_vsi) { 15094 dev_info(&pf->pdev->dev, "setup of MAIN VSI failed\n"); 15095 i40e_cloud_filter_exit(pf); 15096 i40e_fdir_teardown(pf); 15097 return -EAGAIN; 15098 } 15099 } else { 15100 /* force a reset of TC and queue layout configurations */ 15101 main_vsi->seid = pf->main_vsi_seid; 15102 i40e_vsi_reconfig_tc(main_vsi); 15103 } 15104 i40e_vlan_stripping_disable(main_vsi); 15105 15106 i40e_fdir_sb_setup(pf); 15107 15108 /* Setup static PF queue filter control settings */ 15109 ret = i40e_setup_pf_filter_control(pf); 15110 if (ret) { 15111 dev_info(&pf->pdev->dev, "setup_pf_filter_control failed: %d\n", 15112 ret); 15113 /* Failure here should not stop continuing other steps */ 15114 } 15115 15116 /* enable RSS in the HW, even for only one queue, as the stack can use 15117 * the hash 15118 */ 15119 if (test_bit(I40E_FLAG_RSS_ENA, pf->flags)) 15120 i40e_pf_config_rss(pf); 15121 15122 /* fill in link information and enable LSE reporting */ 15123 i40e_link_event(pf); 15124 15125 i40e_ptp_init(pf); 15126 15127 if (!lock_acquired) 15128 rtnl_lock(); 15129 15130 /* repopulate tunnel port filters */ 15131 udp_tunnel_nic_reset_ntf(main_vsi->netdev); 15132 15133 if (!lock_acquired) 15134 rtnl_unlock(); 15135 15136 return ret; 15137 } 15138 15139 /** 15140 * i40e_determine_queue_usage - Work out queue distribution 15141 * @pf: board private structure 15142 **/ 15143 static void i40e_determine_queue_usage(struct i40e_pf *pf) 15144 { 15145 int queues_left; 15146 int q_max; 15147 15148 pf->num_lan_qps = 0; 15149 15150 /* Find the max queues to be put into basic use. We'll always be 15151 * using TC0, whether or not DCB is running, and TC0 will get the 15152 * big RSS set. 15153 */ 15154 queues_left = pf->hw.func_caps.num_tx_qp; 15155 15156 if ((queues_left == 1) || 15157 !test_bit(I40E_FLAG_MSIX_ENA, pf->flags)) { 15158 /* one qp for PF, no queues for anything else */ 15159 queues_left = 0; 15160 pf->alloc_rss_size = pf->num_lan_qps = 1; 15161 15162 /* make sure all the fancies are disabled */ 15163 clear_bit(I40E_FLAG_RSS_ENA, pf->flags); 15164 clear_bit(I40E_FLAG_IWARP_ENA, pf->flags); 15165 clear_bit(I40E_FLAG_FD_SB_ENA, pf->flags); 15166 clear_bit(I40E_FLAG_FD_ATR_ENA, pf->flags); 15167 clear_bit(I40E_FLAG_DCB_CAPABLE, pf->flags); 15168 clear_bit(I40E_FLAG_DCB_ENA, pf->flags); 15169 clear_bit(I40E_FLAG_SRIOV_ENA, pf->flags); 15170 clear_bit(I40E_FLAG_VMDQ_ENA, pf->flags); 15171 set_bit(I40E_FLAG_FD_SB_INACTIVE, pf->flags); 15172 } else if (!test_bit(I40E_FLAG_RSS_ENA, pf->flags) && 15173 !test_bit(I40E_FLAG_FD_SB_ENA, pf->flags) && 15174 !test_bit(I40E_FLAG_FD_ATR_ENA, pf->flags) && 15175 !test_bit(I40E_FLAG_DCB_CAPABLE, pf->flags)) { 15176 /* one qp for PF */ 15177 pf->alloc_rss_size = pf->num_lan_qps = 1; 15178 queues_left -= pf->num_lan_qps; 15179 15180 clear_bit(I40E_FLAG_RSS_ENA, pf->flags); 15181 clear_bit(I40E_FLAG_IWARP_ENA, pf->flags); 15182 clear_bit(I40E_FLAG_FD_SB_ENA, pf->flags); 15183 clear_bit(I40E_FLAG_FD_ATR_ENA, pf->flags); 15184 clear_bit(I40E_FLAG_DCB_ENA, pf->flags); 15185 clear_bit(I40E_FLAG_VMDQ_ENA, pf->flags); 15186 set_bit(I40E_FLAG_FD_SB_INACTIVE, pf->flags); 15187 } else { 15188 /* Not enough queues for all TCs */ 15189 if (test_bit(I40E_FLAG_DCB_CAPABLE, pf->flags) && 15190 queues_left < I40E_MAX_TRAFFIC_CLASS) { 15191 clear_bit(I40E_FLAG_DCB_CAPABLE, pf->flags); 15192 clear_bit(I40E_FLAG_DCB_ENA, pf->flags); 15193 dev_info(&pf->pdev->dev, "not enough queues for DCB. DCB is disabled.\n"); 15194 } 15195 15196 /* limit lan qps to the smaller of qps, cpus or msix */ 15197 q_max = max_t(int, pf->rss_size_max, num_online_cpus()); 15198 q_max = min_t(int, q_max, pf->hw.func_caps.num_tx_qp); 15199 q_max = min_t(int, q_max, pf->hw.func_caps.num_msix_vectors); 15200 pf->num_lan_qps = q_max; 15201 15202 queues_left -= pf->num_lan_qps; 15203 } 15204 15205 if (test_bit(I40E_FLAG_FD_SB_ENA, pf->flags)) { 15206 if (queues_left > 1) { 15207 queues_left -= 1; /* save 1 queue for FD */ 15208 } else { 15209 clear_bit(I40E_FLAG_FD_SB_ENA, pf->flags); 15210 set_bit(I40E_FLAG_FD_SB_INACTIVE, pf->flags); 15211 dev_info(&pf->pdev->dev, "not enough queues for Flow Director. Flow Director feature is disabled\n"); 15212 } 15213 } 15214 15215 if (test_bit(I40E_FLAG_SRIOV_ENA, pf->flags) && 15216 pf->num_vf_qps && pf->num_req_vfs && queues_left) { 15217 pf->num_req_vfs = min_t(int, pf->num_req_vfs, 15218 (queues_left / pf->num_vf_qps)); 15219 queues_left -= (pf->num_req_vfs * pf->num_vf_qps); 15220 } 15221 15222 if (test_bit(I40E_FLAG_VMDQ_ENA, pf->flags) && 15223 pf->num_vmdq_vsis && pf->num_vmdq_qps && queues_left) { 15224 pf->num_vmdq_vsis = min_t(int, pf->num_vmdq_vsis, 15225 (queues_left / pf->num_vmdq_qps)); 15226 queues_left -= (pf->num_vmdq_vsis * pf->num_vmdq_qps); 15227 } 15228 15229 pf->queues_left = queues_left; 15230 dev_dbg(&pf->pdev->dev, 15231 "qs_avail=%d FD SB=%d lan_qs=%d lan_tc0=%d vf=%d*%d vmdq=%d*%d, remaining=%d\n", 15232 pf->hw.func_caps.num_tx_qp, 15233 !!test_bit(I40E_FLAG_FD_SB_ENA, pf->flags), 15234 pf->num_lan_qps, pf->alloc_rss_size, pf->num_req_vfs, 15235 pf->num_vf_qps, pf->num_vmdq_vsis, pf->num_vmdq_qps, 15236 queues_left); 15237 } 15238 15239 /** 15240 * i40e_setup_pf_filter_control - Setup PF static filter control 15241 * @pf: PF to be setup 15242 * 15243 * i40e_setup_pf_filter_control sets up a PF's initial filter control 15244 * settings. If PE/FCoE are enabled then it will also set the per PF 15245 * based filter sizes required for them. It also enables Flow director, 15246 * ethertype and macvlan type filter settings for the pf. 15247 * 15248 * Returns 0 on success, negative on failure 15249 **/ 15250 static int i40e_setup_pf_filter_control(struct i40e_pf *pf) 15251 { 15252 struct i40e_filter_control_settings *settings = &pf->filter_settings; 15253 15254 settings->hash_lut_size = I40E_HASH_LUT_SIZE_128; 15255 15256 /* Flow Director is enabled */ 15257 if (test_bit(I40E_FLAG_FD_SB_ENA, pf->flags) || 15258 test_bit(I40E_FLAG_FD_ATR_ENA, pf->flags)) 15259 settings->enable_fdir = true; 15260 15261 /* Ethtype and MACVLAN filters enabled for PF */ 15262 settings->enable_ethtype = true; 15263 settings->enable_macvlan = true; 15264 15265 if (i40e_set_filter_control(&pf->hw, settings)) 15266 return -ENOENT; 15267 15268 return 0; 15269 } 15270 15271 #define INFO_STRING_LEN 255 15272 #define REMAIN(__x) (INFO_STRING_LEN - (__x)) 15273 static void i40e_print_features(struct i40e_pf *pf) 15274 { 15275 struct i40e_vsi *main_vsi = i40e_pf_get_main_vsi(pf); 15276 struct i40e_hw *hw = &pf->hw; 15277 char *buf; 15278 int i; 15279 15280 buf = kmalloc(INFO_STRING_LEN, GFP_KERNEL); 15281 if (!buf) 15282 return; 15283 15284 i = snprintf(buf, INFO_STRING_LEN, "Features: PF-id[%d]", hw->pf_id); 15285 #ifdef CONFIG_PCI_IOV 15286 i += scnprintf(&buf[i], REMAIN(i), " VFs: %d", pf->num_req_vfs); 15287 #endif 15288 i += scnprintf(&buf[i], REMAIN(i), " VSIs: %d QP: %d", 15289 pf->hw.func_caps.num_vsis, main_vsi->num_queue_pairs); 15290 if (test_bit(I40E_FLAG_RSS_ENA, pf->flags)) 15291 i += scnprintf(&buf[i], REMAIN(i), " RSS"); 15292 if (test_bit(I40E_FLAG_FD_ATR_ENA, pf->flags)) 15293 i += scnprintf(&buf[i], REMAIN(i), " FD_ATR"); 15294 if (test_bit(I40E_FLAG_FD_SB_ENA, pf->flags)) { 15295 i += scnprintf(&buf[i], REMAIN(i), " FD_SB"); 15296 i += scnprintf(&buf[i], REMAIN(i), " NTUPLE"); 15297 } 15298 if (test_bit(I40E_FLAG_DCB_CAPABLE, pf->flags)) 15299 i += scnprintf(&buf[i], REMAIN(i), " DCB"); 15300 i += scnprintf(&buf[i], REMAIN(i), " VxLAN"); 15301 i += scnprintf(&buf[i], REMAIN(i), " Geneve"); 15302 if (test_bit(I40E_FLAG_PTP_ENA, pf->flags)) 15303 i += scnprintf(&buf[i], REMAIN(i), " PTP"); 15304 if (test_bit(I40E_FLAG_VEB_MODE_ENA, pf->flags)) 15305 i += scnprintf(&buf[i], REMAIN(i), " VEB"); 15306 else 15307 i += scnprintf(&buf[i], REMAIN(i), " VEPA"); 15308 15309 dev_info(&pf->pdev->dev, "%s\n", buf); 15310 kfree(buf); 15311 WARN_ON(i > INFO_STRING_LEN); 15312 } 15313 15314 /** 15315 * i40e_get_platform_mac_addr - get platform-specific MAC address 15316 * @pdev: PCI device information struct 15317 * @pf: board private structure 15318 * 15319 * Look up the MAC address for the device. First we'll try 15320 * eth_platform_get_mac_address, which will check Open Firmware, or arch 15321 * specific fallback. Otherwise, we'll default to the stored value in 15322 * firmware. 15323 **/ 15324 static void i40e_get_platform_mac_addr(struct pci_dev *pdev, struct i40e_pf *pf) 15325 { 15326 if (eth_platform_get_mac_address(&pdev->dev, pf->hw.mac.addr)) 15327 i40e_get_mac_addr(&pf->hw, pf->hw.mac.addr); 15328 } 15329 15330 /** 15331 * i40e_set_fec_in_flags - helper function for setting FEC options in flags 15332 * @fec_cfg: FEC option to set in flags 15333 * @flags: ptr to flags in which we set FEC option 15334 **/ 15335 void i40e_set_fec_in_flags(u8 fec_cfg, unsigned long *flags) 15336 { 15337 if (fec_cfg & I40E_AQ_SET_FEC_AUTO) { 15338 set_bit(I40E_FLAG_RS_FEC, flags); 15339 set_bit(I40E_FLAG_BASE_R_FEC, flags); 15340 } 15341 if ((fec_cfg & I40E_AQ_SET_FEC_REQUEST_RS) || 15342 (fec_cfg & I40E_AQ_SET_FEC_ABILITY_RS)) { 15343 set_bit(I40E_FLAG_RS_FEC, flags); 15344 clear_bit(I40E_FLAG_BASE_R_FEC, flags); 15345 } 15346 if ((fec_cfg & I40E_AQ_SET_FEC_REQUEST_KR) || 15347 (fec_cfg & I40E_AQ_SET_FEC_ABILITY_KR)) { 15348 set_bit(I40E_FLAG_BASE_R_FEC, flags); 15349 clear_bit(I40E_FLAG_RS_FEC, flags); 15350 } 15351 if (fec_cfg == 0) { 15352 clear_bit(I40E_FLAG_RS_FEC, flags); 15353 clear_bit(I40E_FLAG_BASE_R_FEC, flags); 15354 } 15355 } 15356 15357 /** 15358 * i40e_check_recovery_mode - check if we are running transition firmware 15359 * @pf: board private structure 15360 * 15361 * Check registers indicating the firmware runs in recovery mode. Sets the 15362 * appropriate driver state. 15363 * 15364 * Returns true if the recovery mode was detected, false otherwise 15365 **/ 15366 static bool i40e_check_recovery_mode(struct i40e_pf *pf) 15367 { 15368 u32 val = rd32(&pf->hw, I40E_GL_FWSTS); 15369 15370 if (val & I40E_GL_FWSTS_FWS1B_MASK) { 15371 dev_crit(&pf->pdev->dev, "Firmware recovery mode detected. Limiting functionality.\n"); 15372 dev_crit(&pf->pdev->dev, "Refer to the Intel(R) Ethernet Adapters and Devices User Guide for details on firmware recovery mode.\n"); 15373 set_bit(__I40E_RECOVERY_MODE, pf->state); 15374 15375 return true; 15376 } 15377 if (test_bit(__I40E_RECOVERY_MODE, pf->state)) 15378 dev_info(&pf->pdev->dev, "Please do Power-On Reset to initialize adapter in normal mode with full functionality.\n"); 15379 15380 return false; 15381 } 15382 15383 /** 15384 * i40e_pf_loop_reset - perform reset in a loop. 15385 * @pf: board private structure 15386 * 15387 * This function is useful when a NIC is about to enter recovery mode. 15388 * When a NIC's internal data structures are corrupted the NIC's 15389 * firmware is going to enter recovery mode. 15390 * Right after a POR it takes about 7 minutes for firmware to enter 15391 * recovery mode. Until that time a NIC is in some kind of intermediate 15392 * state. After that time period the NIC almost surely enters 15393 * recovery mode. The only way for a driver to detect intermediate 15394 * state is to issue a series of pf-resets and check a return value. 15395 * If a PF reset returns success then the firmware could be in recovery 15396 * mode so the caller of this code needs to check for recovery mode 15397 * if this function returns success. There is a little chance that 15398 * firmware will hang in intermediate state forever. 15399 * Since waiting 7 minutes is quite a lot of time this function waits 15400 * 10 seconds and then gives up by returning an error. 15401 * 15402 * Return 0 on success, negative on failure. 15403 **/ 15404 static int i40e_pf_loop_reset(struct i40e_pf *pf) 15405 { 15406 /* wait max 10 seconds for PF reset to succeed */ 15407 const unsigned long time_end = jiffies + 10 * HZ; 15408 struct i40e_hw *hw = &pf->hw; 15409 int ret; 15410 15411 ret = i40e_pf_reset(hw); 15412 while (ret != 0 && time_before(jiffies, time_end)) { 15413 usleep_range(10000, 20000); 15414 ret = i40e_pf_reset(hw); 15415 } 15416 15417 if (ret == 0) 15418 pf->pfr_count++; 15419 else 15420 dev_info(&pf->pdev->dev, "PF reset failed: %d\n", ret); 15421 15422 return ret; 15423 } 15424 15425 /** 15426 * i40e_check_fw_empr - check if FW issued unexpected EMP Reset 15427 * @pf: board private structure 15428 * 15429 * Check FW registers to determine if FW issued unexpected EMP Reset. 15430 * Every time when unexpected EMP Reset occurs the FW increments 15431 * a counter of unexpected EMP Resets. When the counter reaches 10 15432 * the FW should enter the Recovery mode 15433 * 15434 * Returns true if FW issued unexpected EMP Reset 15435 **/ 15436 static bool i40e_check_fw_empr(struct i40e_pf *pf) 15437 { 15438 const u32 fw_sts = rd32(&pf->hw, I40E_GL_FWSTS) & 15439 I40E_GL_FWSTS_FWS1B_MASK; 15440 return (fw_sts > I40E_GL_FWSTS_FWS1B_EMPR_0) && 15441 (fw_sts <= I40E_GL_FWSTS_FWS1B_EMPR_10); 15442 } 15443 15444 /** 15445 * i40e_handle_resets - handle EMP resets and PF resets 15446 * @pf: board private structure 15447 * 15448 * Handle both EMP resets and PF resets and conclude whether there are 15449 * any issues regarding these resets. If there are any issues then 15450 * generate log entry. 15451 * 15452 * Return 0 if NIC is healthy or negative value when there are issues 15453 * with resets 15454 **/ 15455 static int i40e_handle_resets(struct i40e_pf *pf) 15456 { 15457 const int pfr = i40e_pf_loop_reset(pf); 15458 const bool is_empr = i40e_check_fw_empr(pf); 15459 15460 if (is_empr || pfr != 0) 15461 dev_crit(&pf->pdev->dev, "Entering recovery mode due to repeated FW resets. This may take several minutes. Refer to the Intel(R) Ethernet Adapters and Devices User Guide.\n"); 15462 15463 return is_empr ? -EIO : pfr; 15464 } 15465 15466 /** 15467 * i40e_init_recovery_mode - initialize subsystems needed in recovery mode 15468 * @pf: board private structure 15469 * @hw: ptr to the hardware info 15470 * 15471 * This function does a minimal setup of all subsystems needed for running 15472 * recovery mode. 15473 * 15474 * Returns 0 on success, negative on failure 15475 **/ 15476 static int i40e_init_recovery_mode(struct i40e_pf *pf, struct i40e_hw *hw) 15477 { 15478 struct i40e_vsi *vsi; 15479 int err; 15480 int v_idx; 15481 15482 pci_set_drvdata(pf->pdev, pf); 15483 pci_save_state(pf->pdev); 15484 15485 /* set up periodic task facility */ 15486 timer_setup(&pf->service_timer, i40e_service_timer, 0); 15487 pf->service_timer_period = HZ; 15488 15489 INIT_WORK(&pf->service_task, i40e_service_task); 15490 clear_bit(__I40E_SERVICE_SCHED, pf->state); 15491 15492 err = i40e_init_interrupt_scheme(pf); 15493 if (err) 15494 goto err_switch_setup; 15495 15496 /* The number of VSIs reported by the FW is the minimum guaranteed 15497 * to us; HW supports far more and we share the remaining pool with 15498 * the other PFs. We allocate space for more than the guarantee with 15499 * the understanding that we might not get them all later. 15500 */ 15501 if (pf->hw.func_caps.num_vsis < I40E_MIN_VSI_ALLOC) 15502 pf->num_alloc_vsi = I40E_MIN_VSI_ALLOC; 15503 else 15504 pf->num_alloc_vsi = pf->hw.func_caps.num_vsis; 15505 15506 /* Set up the vsi struct and our local tracking of the MAIN PF vsi. */ 15507 pf->vsi = kcalloc(pf->num_alloc_vsi, sizeof(struct i40e_vsi *), 15508 GFP_KERNEL); 15509 if (!pf->vsi) { 15510 err = -ENOMEM; 15511 goto err_switch_setup; 15512 } 15513 15514 /* We allocate one VSI which is needed as absolute minimum 15515 * in order to register the netdev 15516 */ 15517 v_idx = i40e_vsi_mem_alloc(pf, I40E_VSI_MAIN); 15518 if (v_idx < 0) { 15519 err = v_idx; 15520 goto err_switch_setup; 15521 } 15522 pf->lan_vsi = v_idx; 15523 vsi = pf->vsi[v_idx]; 15524 if (!vsi) { 15525 err = -EFAULT; 15526 goto err_switch_setup; 15527 } 15528 vsi->alloc_queue_pairs = 1; 15529 err = i40e_config_netdev(vsi); 15530 if (err) 15531 goto err_switch_setup; 15532 err = register_netdev(vsi->netdev); 15533 if (err) 15534 goto err_switch_setup; 15535 vsi->netdev_registered = true; 15536 i40e_dbg_pf_init(pf); 15537 15538 err = i40e_setup_misc_vector_for_recovery_mode(pf); 15539 if (err) 15540 goto err_switch_setup; 15541 15542 /* tell the firmware that we're starting */ 15543 i40e_send_version(pf); 15544 15545 /* since everything's happy, start the service_task timer */ 15546 mod_timer(&pf->service_timer, 15547 round_jiffies(jiffies + pf->service_timer_period)); 15548 15549 return 0; 15550 15551 err_switch_setup: 15552 i40e_reset_interrupt_capability(pf); 15553 timer_shutdown_sync(&pf->service_timer); 15554 i40e_shutdown_adminq(hw); 15555 iounmap(hw->hw_addr); 15556 pci_release_mem_regions(pf->pdev); 15557 pci_disable_device(pf->pdev); 15558 i40e_free_pf(pf); 15559 15560 return err; 15561 } 15562 15563 /** 15564 * i40e_set_subsystem_device_id - set subsystem device id 15565 * @hw: pointer to the hardware info 15566 * 15567 * Set PCI subsystem device id either from a pci_dev structure or 15568 * a specific FW register. 15569 **/ 15570 static inline void i40e_set_subsystem_device_id(struct i40e_hw *hw) 15571 { 15572 struct i40e_pf *pf = i40e_hw_to_pf(hw); 15573 15574 hw->subsystem_device_id = pf->pdev->subsystem_device ? 15575 pf->pdev->subsystem_device : 15576 (ushort)(rd32(hw, I40E_PFPCI_SUBSYSID) & USHRT_MAX); 15577 } 15578 15579 /** 15580 * i40e_probe - Device initialization routine 15581 * @pdev: PCI device information struct 15582 * @ent: entry in i40e_pci_tbl 15583 * 15584 * i40e_probe initializes a PF identified by a pci_dev structure. 15585 * The OS initialization, configuring of the PF private structure, 15586 * and a hardware reset occur. 15587 * 15588 * Returns 0 on success, negative on failure 15589 **/ 15590 static int i40e_probe(struct pci_dev *pdev, const struct pci_device_id *ent) 15591 { 15592 struct i40e_aq_get_phy_abilities_resp abilities; 15593 #ifdef CONFIG_I40E_DCB 15594 enum i40e_get_fw_lldp_status_resp lldp_status; 15595 #endif /* CONFIG_I40E_DCB */ 15596 struct i40e_vsi *vsi; 15597 struct i40e_pf *pf; 15598 struct i40e_hw *hw; 15599 u16 wol_nvm_bits; 15600 char nvm_ver[32]; 15601 u16 link_status; 15602 #ifdef CONFIG_I40E_DCB 15603 int status; 15604 #endif /* CONFIG_I40E_DCB */ 15605 int err; 15606 u32 val; 15607 15608 err = pci_enable_device_mem(pdev); 15609 if (err) 15610 return err; 15611 15612 /* set up for high or low dma */ 15613 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); 15614 if (err) { 15615 dev_err(&pdev->dev, 15616 "DMA configuration failed: 0x%x\n", err); 15617 goto err_dma; 15618 } 15619 15620 /* set up pci connections */ 15621 err = pci_request_mem_regions(pdev, i40e_driver_name); 15622 if (err) { 15623 dev_info(&pdev->dev, 15624 "pci_request_selected_regions failed %d\n", err); 15625 goto err_pci_reg; 15626 } 15627 15628 pci_set_master(pdev); 15629 15630 /* Now that we have a PCI connection, we need to do the 15631 * low level device setup. This is primarily setting up 15632 * the Admin Queue structures and then querying for the 15633 * device's current profile information. 15634 */ 15635 pf = i40e_alloc_pf(&pdev->dev); 15636 if (!pf) { 15637 err = -ENOMEM; 15638 goto err_pf_alloc; 15639 } 15640 pf->next_vsi = 0; 15641 pf->pdev = pdev; 15642 set_bit(__I40E_DOWN, pf->state); 15643 15644 hw = &pf->hw; 15645 15646 pf->ioremap_len = min_t(int, pci_resource_len(pdev, 0), 15647 I40E_MAX_CSR_SPACE); 15648 /* We believe that the highest register to read is 15649 * I40E_GLGEN_STAT_CLEAR, so we check if the BAR size 15650 * is not less than that before mapping to prevent a 15651 * kernel panic. 15652 */ 15653 if (pf->ioremap_len < I40E_GLGEN_STAT_CLEAR) { 15654 dev_err(&pdev->dev, "Cannot map registers, bar size 0x%X too small, aborting\n", 15655 pf->ioremap_len); 15656 err = -ENOMEM; 15657 goto err_ioremap; 15658 } 15659 hw->hw_addr = ioremap(pci_resource_start(pdev, 0), pf->ioremap_len); 15660 if (!hw->hw_addr) { 15661 err = -EIO; 15662 dev_info(&pdev->dev, "ioremap(0x%04x, 0x%04x) failed: 0x%x\n", 15663 (unsigned int)pci_resource_start(pdev, 0), 15664 pf->ioremap_len, err); 15665 goto err_ioremap; 15666 } 15667 hw->vendor_id = pdev->vendor; 15668 hw->device_id = pdev->device; 15669 pci_read_config_byte(pdev, PCI_REVISION_ID, &hw->revision_id); 15670 hw->subsystem_vendor_id = pdev->subsystem_vendor; 15671 i40e_set_subsystem_device_id(hw); 15672 hw->bus.device = PCI_SLOT(pdev->devfn); 15673 hw->bus.func = PCI_FUNC(pdev->devfn); 15674 hw->bus.bus_id = pdev->bus->number; 15675 15676 /* Select something other than the 802.1ad ethertype for the 15677 * switch to use internally and drop on ingress. 15678 */ 15679 hw->switch_tag = 0xffff; 15680 hw->first_tag = ETH_P_8021AD; 15681 hw->second_tag = ETH_P_8021Q; 15682 15683 INIT_LIST_HEAD(&pf->l3_flex_pit_list); 15684 INIT_LIST_HEAD(&pf->l4_flex_pit_list); 15685 INIT_LIST_HEAD(&pf->ddp_old_prof); 15686 15687 /* set up the locks for the AQ, do this only once in probe 15688 * and destroy them only once in remove 15689 */ 15690 mutex_init(&hw->aq.asq_mutex); 15691 mutex_init(&hw->aq.arq_mutex); 15692 15693 pf->msg_enable = netif_msg_init(debug, 15694 NETIF_MSG_DRV | 15695 NETIF_MSG_PROBE | 15696 NETIF_MSG_LINK); 15697 if (debug < -1) 15698 pf->hw.debug_mask = debug; 15699 15700 /* do a special CORER for clearing PXE mode once at init */ 15701 if (hw->revision_id == 0 && 15702 (rd32(hw, I40E_GLLAN_RCTL_0) & I40E_GLLAN_RCTL_0_PXE_MODE_MASK)) { 15703 wr32(hw, I40E_GLGEN_RTRIG, I40E_GLGEN_RTRIG_CORER_MASK); 15704 i40e_flush(hw); 15705 msleep(200); 15706 pf->corer_count++; 15707 15708 i40e_clear_pxe_mode(hw); 15709 } 15710 15711 /* Reset here to make sure all is clean and to define PF 'n' */ 15712 i40e_clear_hw(hw); 15713 15714 err = i40e_set_mac_type(hw); 15715 if (err) { 15716 dev_warn(&pdev->dev, "unidentified MAC or BLANK NVM: %d\n", 15717 err); 15718 goto err_pf_reset; 15719 } 15720 15721 err = i40e_handle_resets(pf); 15722 if (err) 15723 goto err_pf_reset; 15724 15725 i40e_check_recovery_mode(pf); 15726 15727 if (is_kdump_kernel()) { 15728 hw->aq.num_arq_entries = I40E_MIN_ARQ_LEN; 15729 hw->aq.num_asq_entries = I40E_MIN_ASQ_LEN; 15730 } else { 15731 hw->aq.num_arq_entries = I40E_AQ_LEN; 15732 hw->aq.num_asq_entries = I40E_AQ_LEN; 15733 } 15734 hw->aq.arq_buf_size = I40E_MAX_AQ_BUF_SIZE; 15735 hw->aq.asq_buf_size = I40E_MAX_AQ_BUF_SIZE; 15736 15737 snprintf(pf->int_name, sizeof(pf->int_name) - 1, 15738 "%s-%s:misc", 15739 dev_driver_string(&pf->pdev->dev), dev_name(&pdev->dev)); 15740 15741 err = i40e_init_shared_code(hw); 15742 if (err) { 15743 dev_warn(&pdev->dev, "unidentified MAC or BLANK NVM: %d\n", 15744 err); 15745 goto err_pf_reset; 15746 } 15747 15748 /* set up a default setting for link flow control */ 15749 pf->hw.fc.requested_mode = I40E_FC_NONE; 15750 15751 err = i40e_init_adminq(hw); 15752 if (err) { 15753 if (err == -EIO) 15754 dev_info(&pdev->dev, 15755 "The driver for the device stopped because the NVM image v%u.%u is newer than expected v%u.%u. You must install the most recent version of the network driver.\n", 15756 hw->aq.api_maj_ver, 15757 hw->aq.api_min_ver, 15758 I40E_FW_API_VERSION_MAJOR, 15759 I40E_FW_MINOR_VERSION(hw)); 15760 else 15761 dev_info(&pdev->dev, 15762 "The driver for the device stopped because the device firmware failed to init. Try updating your NVM image.\n"); 15763 15764 goto err_pf_reset; 15765 } 15766 i40e_get_oem_version(hw); 15767 i40e_get_pba_string(hw); 15768 15769 /* provide nvm, fw, api versions, vendor:device id, subsys vendor:device id */ 15770 i40e_nvm_version_str(hw, nvm_ver, sizeof(nvm_ver)); 15771 dev_info(&pdev->dev, "fw %d.%d.%05d api %d.%d nvm %s [%04x:%04x] [%04x:%04x]\n", 15772 hw->aq.fw_maj_ver, hw->aq.fw_min_ver, hw->aq.fw_build, 15773 hw->aq.api_maj_ver, hw->aq.api_min_ver, nvm_ver, 15774 hw->vendor_id, hw->device_id, hw->subsystem_vendor_id, 15775 hw->subsystem_device_id); 15776 15777 if (i40e_is_aq_api_ver_ge(hw, I40E_FW_API_VERSION_MAJOR, 15778 I40E_FW_MINOR_VERSION(hw) + 1)) 15779 dev_dbg(&pdev->dev, 15780 "The driver for the device detected a newer version of the NVM image v%u.%u than v%u.%u.\n", 15781 hw->aq.api_maj_ver, 15782 hw->aq.api_min_ver, 15783 I40E_FW_API_VERSION_MAJOR, 15784 I40E_FW_MINOR_VERSION(hw)); 15785 else if (i40e_is_aq_api_ver_lt(hw, 1, 4)) 15786 dev_info(&pdev->dev, 15787 "The driver for the device detected an older version of the NVM image v%u.%u than expected v%u.%u. Please update the NVM image.\n", 15788 hw->aq.api_maj_ver, 15789 hw->aq.api_min_ver, 15790 I40E_FW_API_VERSION_MAJOR, 15791 I40E_FW_MINOR_VERSION(hw)); 15792 15793 i40e_verify_eeprom(pf); 15794 15795 /* Rev 0 hardware was never productized */ 15796 if (hw->revision_id < 1) 15797 dev_warn(&pdev->dev, "This device is a pre-production adapter/LOM. Please be aware there may be issues with your hardware. If you are experiencing problems please contact your Intel or hardware representative who provided you with this hardware.\n"); 15798 15799 i40e_clear_pxe_mode(hw); 15800 15801 err = i40e_get_capabilities(pf, i40e_aqc_opc_list_func_capabilities); 15802 if (err) 15803 goto err_adminq_setup; 15804 15805 err = i40e_sw_init(pf); 15806 if (err) { 15807 dev_info(&pdev->dev, "sw_init failed: %d\n", err); 15808 goto err_sw_init; 15809 } 15810 15811 if (test_bit(__I40E_RECOVERY_MODE, pf->state)) 15812 return i40e_init_recovery_mode(pf, hw); 15813 15814 err = i40e_init_lan_hmc(hw, hw->func_caps.num_tx_qp, 15815 hw->func_caps.num_rx_qp, 0, 0); 15816 if (err) { 15817 dev_info(&pdev->dev, "init_lan_hmc failed: %d\n", err); 15818 goto err_init_lan_hmc; 15819 } 15820 15821 err = i40e_configure_lan_hmc(hw, I40E_HMC_MODEL_DIRECT_ONLY); 15822 if (err) { 15823 dev_info(&pdev->dev, "configure_lan_hmc failed: %d\n", err); 15824 err = -ENOENT; 15825 goto err_configure_lan_hmc; 15826 } 15827 15828 /* Disable LLDP for NICs that have firmware versions lower than v4.3. 15829 * Ignore error return codes because if it was already disabled via 15830 * hardware settings this will fail 15831 */ 15832 if (test_bit(I40E_HW_CAP_STOP_FW_LLDP, pf->hw.caps)) { 15833 dev_info(&pdev->dev, "Stopping firmware LLDP agent.\n"); 15834 i40e_aq_stop_lldp(hw, true, false, NULL); 15835 } 15836 15837 /* allow a platform config to override the HW addr */ 15838 i40e_get_platform_mac_addr(pdev, pf); 15839 15840 if (!is_valid_ether_addr(hw->mac.addr)) { 15841 dev_info(&pdev->dev, "invalid MAC address %pM\n", hw->mac.addr); 15842 err = -EIO; 15843 goto err_mac_addr; 15844 } 15845 dev_info(&pdev->dev, "MAC address: %pM\n", hw->mac.addr); 15846 ether_addr_copy(hw->mac.perm_addr, hw->mac.addr); 15847 i40e_get_port_mac_addr(hw, hw->mac.port_addr); 15848 if (is_valid_ether_addr(hw->mac.port_addr)) 15849 set_bit(I40E_HW_CAP_PORT_ID_VALID, pf->hw.caps); 15850 15851 i40e_ptp_alloc_pins(pf); 15852 pci_set_drvdata(pdev, pf); 15853 pci_save_state(pdev); 15854 15855 #ifdef CONFIG_I40E_DCB 15856 status = i40e_get_fw_lldp_status(&pf->hw, &lldp_status); 15857 (!status && 15858 lldp_status == I40E_GET_FW_LLDP_STATUS_ENABLED) ? 15859 (clear_bit(I40E_FLAG_FW_LLDP_DIS, pf->flags)) : 15860 (set_bit(I40E_FLAG_FW_LLDP_DIS, pf->flags)); 15861 dev_info(&pdev->dev, 15862 test_bit(I40E_FLAG_FW_LLDP_DIS, pf->flags) ? 15863 "FW LLDP is disabled\n" : 15864 "FW LLDP is enabled\n"); 15865 15866 /* Enable FW to write default DCB config on link-up */ 15867 i40e_aq_set_dcb_parameters(hw, true, NULL); 15868 15869 err = i40e_init_pf_dcb(pf); 15870 if (err) { 15871 dev_info(&pdev->dev, "DCB init failed %d, disabled\n", err); 15872 clear_bit(I40E_FLAG_DCB_CAPABLE, pf->flags); 15873 clear_bit(I40E_FLAG_DCB_ENA, pf->flags); 15874 /* Continue without DCB enabled */ 15875 } 15876 #endif /* CONFIG_I40E_DCB */ 15877 15878 /* set up periodic task facility */ 15879 timer_setup(&pf->service_timer, i40e_service_timer, 0); 15880 pf->service_timer_period = HZ; 15881 15882 INIT_WORK(&pf->service_task, i40e_service_task); 15883 clear_bit(__I40E_SERVICE_SCHED, pf->state); 15884 15885 /* NVM bit on means WoL disabled for the port */ 15886 i40e_read_nvm_word(hw, I40E_SR_NVM_WAKE_ON_LAN, &wol_nvm_bits); 15887 if (BIT (hw->port) & wol_nvm_bits || hw->partition_id != 1) 15888 pf->wol_en = false; 15889 else 15890 pf->wol_en = true; 15891 device_set_wakeup_enable(&pf->pdev->dev, pf->wol_en); 15892 15893 /* set up the main switch operations */ 15894 i40e_determine_queue_usage(pf); 15895 err = i40e_init_interrupt_scheme(pf); 15896 if (err) 15897 goto err_switch_setup; 15898 15899 /* Reduce Tx and Rx pairs for kdump 15900 * When MSI-X is enabled, it's not allowed to use more TC queue 15901 * pairs than MSI-X vectors (pf->num_lan_msix) exist. Thus 15902 * vsi->num_queue_pairs will be equal to pf->num_lan_msix, i.e., 1. 15903 */ 15904 if (is_kdump_kernel()) 15905 pf->num_lan_msix = 1; 15906 15907 pf->udp_tunnel_nic.set_port = i40e_udp_tunnel_set_port; 15908 pf->udp_tunnel_nic.unset_port = i40e_udp_tunnel_unset_port; 15909 pf->udp_tunnel_nic.flags = UDP_TUNNEL_NIC_INFO_MAY_SLEEP; 15910 pf->udp_tunnel_nic.shared = &pf->udp_tunnel_shared; 15911 pf->udp_tunnel_nic.tables[0].n_entries = I40E_MAX_PF_UDP_OFFLOAD_PORTS; 15912 pf->udp_tunnel_nic.tables[0].tunnel_types = UDP_TUNNEL_TYPE_VXLAN | 15913 UDP_TUNNEL_TYPE_GENEVE; 15914 15915 /* The number of VSIs reported by the FW is the minimum guaranteed 15916 * to us; HW supports far more and we share the remaining pool with 15917 * the other PFs. We allocate space for more than the guarantee with 15918 * the understanding that we might not get them all later. 15919 */ 15920 if (pf->hw.func_caps.num_vsis < I40E_MIN_VSI_ALLOC) 15921 pf->num_alloc_vsi = I40E_MIN_VSI_ALLOC; 15922 else 15923 pf->num_alloc_vsi = pf->hw.func_caps.num_vsis; 15924 if (pf->num_alloc_vsi > UDP_TUNNEL_NIC_MAX_SHARING_DEVICES) { 15925 dev_warn(&pf->pdev->dev, 15926 "limiting the VSI count due to UDP tunnel limitation %d > %d\n", 15927 pf->num_alloc_vsi, UDP_TUNNEL_NIC_MAX_SHARING_DEVICES); 15928 pf->num_alloc_vsi = UDP_TUNNEL_NIC_MAX_SHARING_DEVICES; 15929 } 15930 15931 /* Set up the *vsi struct and our local tracking of the MAIN PF vsi. */ 15932 pf->vsi = kcalloc(pf->num_alloc_vsi, sizeof(struct i40e_vsi *), 15933 GFP_KERNEL); 15934 if (!pf->vsi) { 15935 err = -ENOMEM; 15936 goto err_switch_setup; 15937 } 15938 15939 #ifdef CONFIG_PCI_IOV 15940 /* prep for VF support */ 15941 if (test_bit(I40E_FLAG_SRIOV_ENA, pf->flags) && 15942 test_bit(I40E_FLAG_MSIX_ENA, pf->flags) && 15943 !test_bit(__I40E_BAD_EEPROM, pf->state)) { 15944 if (pci_num_vf(pdev)) 15945 set_bit(I40E_FLAG_VEB_MODE_ENA, pf->flags); 15946 } 15947 #endif 15948 err = i40e_setup_pf_switch(pf, false, false); 15949 if (err) { 15950 dev_info(&pdev->dev, "setup_pf_switch failed: %d\n", err); 15951 goto err_vsis; 15952 } 15953 15954 vsi = i40e_pf_get_main_vsi(pf); 15955 INIT_LIST_HEAD(&vsi->ch_list); 15956 15957 /* if FDIR VSI was set up, start it now */ 15958 vsi = i40e_find_vsi_by_type(pf, I40E_VSI_FDIR); 15959 if (vsi) 15960 i40e_vsi_open(vsi); 15961 15962 /* The driver only wants link up/down and module qualification 15963 * reports from firmware. Note the negative logic. 15964 */ 15965 err = i40e_aq_set_phy_int_mask(&pf->hw, 15966 ~(I40E_AQ_EVENT_LINK_UPDOWN | 15967 I40E_AQ_EVENT_MEDIA_NA | 15968 I40E_AQ_EVENT_MODULE_QUAL_FAIL), NULL); 15969 if (err) 15970 dev_info(&pf->pdev->dev, "set phy mask fail, err %pe aq_err %s\n", 15971 ERR_PTR(err), 15972 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 15973 15974 /* Reconfigure hardware for allowing smaller MSS in the case 15975 * of TSO, so that we avoid the MDD being fired and causing 15976 * a reset in the case of small MSS+TSO. 15977 */ 15978 val = rd32(hw, I40E_REG_MSS); 15979 if ((val & I40E_REG_MSS_MIN_MASK) > I40E_64BYTE_MSS) { 15980 val &= ~I40E_REG_MSS_MIN_MASK; 15981 val |= I40E_64BYTE_MSS; 15982 wr32(hw, I40E_REG_MSS, val); 15983 } 15984 15985 if (test_bit(I40E_HW_CAP_RESTART_AUTONEG, pf->hw.caps)) { 15986 msleep(75); 15987 err = i40e_aq_set_link_restart_an(&pf->hw, true, NULL); 15988 if (err) 15989 dev_info(&pf->pdev->dev, "link restart failed, err %pe aq_err %s\n", 15990 ERR_PTR(err), 15991 i40e_aq_str(&pf->hw, 15992 pf->hw.aq.asq_last_status)); 15993 } 15994 /* The main driver is (mostly) up and happy. We need to set this state 15995 * before setting up the misc vector or we get a race and the vector 15996 * ends up disabled forever. 15997 */ 15998 clear_bit(__I40E_DOWN, pf->state); 15999 16000 /* In case of MSIX we are going to setup the misc vector right here 16001 * to handle admin queue events etc. In case of legacy and MSI 16002 * the misc functionality and queue processing is combined in 16003 * the same vector and that gets setup at open. 16004 */ 16005 if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags)) { 16006 err = i40e_setup_misc_vector(pf); 16007 if (err) { 16008 dev_info(&pdev->dev, 16009 "setup of misc vector failed: %d\n", err); 16010 i40e_cloud_filter_exit(pf); 16011 i40e_fdir_teardown(pf); 16012 goto err_vsis; 16013 } 16014 } 16015 16016 #ifdef CONFIG_PCI_IOV 16017 /* prep for VF support */ 16018 if (test_bit(I40E_FLAG_SRIOV_ENA, pf->flags) && 16019 test_bit(I40E_FLAG_MSIX_ENA, pf->flags) && 16020 !test_bit(__I40E_BAD_EEPROM, pf->state)) { 16021 /* disable link interrupts for VFs */ 16022 val = rd32(hw, I40E_PFGEN_PORTMDIO_NUM); 16023 val &= ~I40E_PFGEN_PORTMDIO_NUM_VFLINK_STAT_ENA_MASK; 16024 wr32(hw, I40E_PFGEN_PORTMDIO_NUM, val); 16025 i40e_flush(hw); 16026 16027 if (pci_num_vf(pdev)) { 16028 dev_info(&pdev->dev, 16029 "Active VFs found, allocating resources.\n"); 16030 err = i40e_alloc_vfs(pf, pci_num_vf(pdev)); 16031 if (err) 16032 dev_info(&pdev->dev, 16033 "Error %d allocating resources for existing VFs\n", 16034 err); 16035 } 16036 } 16037 #endif /* CONFIG_PCI_IOV */ 16038 16039 if (test_bit(I40E_FLAG_IWARP_ENA, pf->flags)) { 16040 pf->iwarp_base_vector = i40e_get_lump(pf, pf->irq_pile, 16041 pf->num_iwarp_msix, 16042 I40E_IWARP_IRQ_PILE_ID); 16043 if (pf->iwarp_base_vector < 0) { 16044 dev_info(&pdev->dev, 16045 "failed to get tracking for %d vectors for IWARP err=%d\n", 16046 pf->num_iwarp_msix, pf->iwarp_base_vector); 16047 clear_bit(I40E_FLAG_IWARP_ENA, pf->flags); 16048 } 16049 } 16050 16051 i40e_dbg_pf_init(pf); 16052 16053 /* tell the firmware that we're starting */ 16054 i40e_send_version(pf); 16055 16056 /* since everything's happy, start the service_task timer */ 16057 mod_timer(&pf->service_timer, 16058 round_jiffies(jiffies + pf->service_timer_period)); 16059 16060 /* add this PF to client device list and launch a client service task */ 16061 if (test_bit(I40E_FLAG_IWARP_ENA, pf->flags)) { 16062 err = i40e_lan_add_device(pf); 16063 if (err) 16064 dev_info(&pdev->dev, "Failed to add PF to client API service list: %d\n", 16065 err); 16066 } 16067 16068 #define PCI_SPEED_SIZE 8 16069 #define PCI_WIDTH_SIZE 8 16070 /* Devices on the IOSF bus do not have this information 16071 * and will report PCI Gen 1 x 1 by default so don't bother 16072 * checking them. 16073 */ 16074 if (!test_bit(I40E_HW_CAP_NO_PCI_LINK_CHECK, pf->hw.caps)) { 16075 char speed[PCI_SPEED_SIZE] = "Unknown"; 16076 char width[PCI_WIDTH_SIZE] = "Unknown"; 16077 16078 /* Get the negotiated link width and speed from PCI config 16079 * space 16080 */ 16081 pcie_capability_read_word(pf->pdev, PCI_EXP_LNKSTA, 16082 &link_status); 16083 16084 i40e_set_pci_config_data(hw, link_status); 16085 16086 switch (hw->bus.speed) { 16087 case i40e_bus_speed_8000: 16088 strscpy(speed, "8.0", PCI_SPEED_SIZE); break; 16089 case i40e_bus_speed_5000: 16090 strscpy(speed, "5.0", PCI_SPEED_SIZE); break; 16091 case i40e_bus_speed_2500: 16092 strscpy(speed, "2.5", PCI_SPEED_SIZE); break; 16093 default: 16094 break; 16095 } 16096 switch (hw->bus.width) { 16097 case i40e_bus_width_pcie_x8: 16098 strscpy(width, "8", PCI_WIDTH_SIZE); break; 16099 case i40e_bus_width_pcie_x4: 16100 strscpy(width, "4", PCI_WIDTH_SIZE); break; 16101 case i40e_bus_width_pcie_x2: 16102 strscpy(width, "2", PCI_WIDTH_SIZE); break; 16103 case i40e_bus_width_pcie_x1: 16104 strscpy(width, "1", PCI_WIDTH_SIZE); break; 16105 default: 16106 break; 16107 } 16108 16109 dev_info(&pdev->dev, "PCI-Express: Speed %sGT/s Width x%s\n", 16110 speed, width); 16111 16112 if (hw->bus.width < i40e_bus_width_pcie_x8 || 16113 hw->bus.speed < i40e_bus_speed_8000) { 16114 dev_warn(&pdev->dev, "PCI-Express bandwidth available for this device may be insufficient for optimal performance.\n"); 16115 dev_warn(&pdev->dev, "Please move the device to a different PCI-e link with more lanes and/or higher transfer rate.\n"); 16116 } 16117 } 16118 16119 /* get the requested speeds from the fw */ 16120 err = i40e_aq_get_phy_capabilities(hw, false, false, &abilities, NULL); 16121 if (err) 16122 dev_dbg(&pf->pdev->dev, "get requested speeds ret = %pe last_status = %s\n", 16123 ERR_PTR(err), 16124 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 16125 pf->hw.phy.link_info.requested_speeds = abilities.link_speed; 16126 16127 /* set the FEC config due to the board capabilities */ 16128 i40e_set_fec_in_flags(abilities.fec_cfg_curr_mod_ext_info, pf->flags); 16129 16130 /* get the supported phy types from the fw */ 16131 err = i40e_aq_get_phy_capabilities(hw, false, true, &abilities, NULL); 16132 if (err) 16133 dev_dbg(&pf->pdev->dev, "get supported phy types ret = %pe last_status = %s\n", 16134 ERR_PTR(err), 16135 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 16136 16137 /* make sure the MFS hasn't been set lower than the default */ 16138 #define MAX_FRAME_SIZE_DEFAULT 0x2600 16139 val = FIELD_GET(I40E_PRTGL_SAH_MFS_MASK, 16140 rd32(&pf->hw, I40E_PRTGL_SAH)); 16141 if (val < MAX_FRAME_SIZE_DEFAULT) 16142 dev_warn(&pdev->dev, "MFS for port %x (%d) has been set below the default (%d)\n", 16143 pf->hw.port, val, MAX_FRAME_SIZE_DEFAULT); 16144 16145 /* Add a filter to drop all Flow control frames from any VSI from being 16146 * transmitted. By doing so we stop a malicious VF from sending out 16147 * PAUSE or PFC frames and potentially controlling traffic for other 16148 * PF/VF VSIs. 16149 * The FW can still send Flow control frames if enabled. 16150 */ 16151 i40e_add_filter_to_drop_tx_flow_control_frames(&pf->hw, 16152 pf->main_vsi_seid); 16153 16154 if ((pf->hw.device_id == I40E_DEV_ID_10G_BASE_T) || 16155 (pf->hw.device_id == I40E_DEV_ID_10G_BASE_T4)) 16156 set_bit(I40E_HW_CAP_PHY_CONTROLS_LEDS, pf->hw.caps); 16157 if (pf->hw.device_id == I40E_DEV_ID_SFP_I_X722) 16158 set_bit(I40E_HW_CAP_CRT_RETIMER, pf->hw.caps); 16159 /* print a string summarizing features */ 16160 i40e_print_features(pf); 16161 16162 i40e_devlink_register(pf); 16163 16164 return 0; 16165 16166 /* Unwind what we've done if something failed in the setup */ 16167 err_vsis: 16168 set_bit(__I40E_DOWN, pf->state); 16169 i40e_clear_interrupt_scheme(pf); 16170 kfree(pf->vsi); 16171 err_switch_setup: 16172 i40e_reset_interrupt_capability(pf); 16173 timer_shutdown_sync(&pf->service_timer); 16174 err_mac_addr: 16175 err_configure_lan_hmc: 16176 (void)i40e_shutdown_lan_hmc(hw); 16177 err_init_lan_hmc: 16178 kfree(pf->qp_pile); 16179 err_sw_init: 16180 err_adminq_setup: 16181 err_pf_reset: 16182 iounmap(hw->hw_addr); 16183 err_ioremap: 16184 i40e_free_pf(pf); 16185 err_pf_alloc: 16186 pci_release_mem_regions(pdev); 16187 err_pci_reg: 16188 err_dma: 16189 pci_disable_device(pdev); 16190 return err; 16191 } 16192 16193 /** 16194 * i40e_remove - Device removal routine 16195 * @pdev: PCI device information struct 16196 * 16197 * i40e_remove is called by the PCI subsystem to alert the driver 16198 * that is should release a PCI device. This could be caused by a 16199 * Hot-Plug event, or because the driver is going to be removed from 16200 * memory. 16201 **/ 16202 static void i40e_remove(struct pci_dev *pdev) 16203 { 16204 struct i40e_pf *pf = pci_get_drvdata(pdev); 16205 struct i40e_hw *hw = &pf->hw; 16206 struct i40e_vsi *vsi; 16207 struct i40e_veb *veb; 16208 int ret_code; 16209 int i; 16210 16211 i40e_devlink_unregister(pf); 16212 16213 i40e_dbg_pf_exit(pf); 16214 16215 i40e_ptp_stop(pf); 16216 16217 /* Disable RSS in hw */ 16218 i40e_write_rx_ctl(hw, I40E_PFQF_HENA(0), 0); 16219 i40e_write_rx_ctl(hw, I40E_PFQF_HENA(1), 0); 16220 16221 /* Grab __I40E_RESET_RECOVERY_PENDING and set __I40E_IN_REMOVE 16222 * flags, once they are set, i40e_rebuild should not be called as 16223 * i40e_prep_for_reset always returns early. 16224 */ 16225 while (test_and_set_bit(__I40E_RESET_RECOVERY_PENDING, pf->state)) 16226 usleep_range(1000, 2000); 16227 set_bit(__I40E_IN_REMOVE, pf->state); 16228 16229 if (test_bit(I40E_FLAG_SRIOV_ENA, pf->flags)) { 16230 set_bit(__I40E_VF_RESETS_DISABLED, pf->state); 16231 i40e_free_vfs(pf); 16232 clear_bit(I40E_FLAG_SRIOV_ENA, pf->flags); 16233 } 16234 /* no more scheduling of any task */ 16235 set_bit(__I40E_SUSPENDED, pf->state); 16236 set_bit(__I40E_DOWN, pf->state); 16237 if (pf->service_timer.function) 16238 timer_shutdown_sync(&pf->service_timer); 16239 if (pf->service_task.func) 16240 cancel_work_sync(&pf->service_task); 16241 16242 if (test_bit(__I40E_RECOVERY_MODE, pf->state)) { 16243 struct i40e_vsi *vsi = pf->vsi[0]; 16244 16245 /* We know that we have allocated only one vsi for this PF, 16246 * it was just for registering netdevice, so the interface 16247 * could be visible in the 'ifconfig' output 16248 */ 16249 unregister_netdev(vsi->netdev); 16250 free_netdev(vsi->netdev); 16251 16252 goto unmap; 16253 } 16254 16255 /* Client close must be called explicitly here because the timer 16256 * has been stopped. 16257 */ 16258 i40e_notify_client_of_netdev_close(pf, false); 16259 16260 i40e_fdir_teardown(pf); 16261 16262 /* If there is a switch structure or any orphans, remove them. 16263 * This will leave only the PF's VSI remaining. 16264 */ 16265 i40e_pf_for_each_veb(pf, i, veb) 16266 if (veb->uplink_seid == pf->mac_seid || 16267 veb->uplink_seid == 0) 16268 i40e_switch_branch_release(veb); 16269 16270 /* Now we can shutdown the PF's VSIs, just before we kill 16271 * adminq and hmc. 16272 */ 16273 i40e_pf_for_each_vsi(pf, i, vsi) { 16274 i40e_vsi_close(vsi); 16275 i40e_vsi_release(vsi); 16276 pf->vsi[i] = NULL; 16277 } 16278 16279 i40e_cloud_filter_exit(pf); 16280 16281 /* remove attached clients */ 16282 if (test_bit(I40E_FLAG_IWARP_ENA, pf->flags)) { 16283 ret_code = i40e_lan_del_device(pf); 16284 if (ret_code) 16285 dev_warn(&pdev->dev, "Failed to delete client device: %d\n", 16286 ret_code); 16287 } 16288 16289 /* shutdown and destroy the HMC */ 16290 if (hw->hmc.hmc_obj) { 16291 ret_code = i40e_shutdown_lan_hmc(hw); 16292 if (ret_code) 16293 dev_warn(&pdev->dev, 16294 "Failed to destroy the HMC resources: %d\n", 16295 ret_code); 16296 } 16297 16298 unmap: 16299 /* Free MSI/legacy interrupt 0 when in recovery mode. */ 16300 if (test_bit(__I40E_RECOVERY_MODE, pf->state) && 16301 !test_bit(I40E_FLAG_MSIX_ENA, pf->flags)) 16302 free_irq(pf->pdev->irq, pf); 16303 16304 /* shutdown the adminq */ 16305 i40e_shutdown_adminq(hw); 16306 16307 /* destroy the locks only once, here */ 16308 mutex_destroy(&hw->aq.arq_mutex); 16309 mutex_destroy(&hw->aq.asq_mutex); 16310 16311 /* Clear all dynamic memory lists of rings, q_vectors, and VSIs */ 16312 rtnl_lock(); 16313 i40e_clear_interrupt_scheme(pf); 16314 i40e_pf_for_each_vsi(pf, i, vsi) { 16315 if (!test_bit(__I40E_RECOVERY_MODE, pf->state)) 16316 i40e_vsi_clear_rings(vsi); 16317 16318 i40e_vsi_clear(vsi); 16319 pf->vsi[i] = NULL; 16320 } 16321 rtnl_unlock(); 16322 16323 i40e_pf_for_each_veb(pf, i, veb) { 16324 kfree(veb); 16325 pf->veb[i] = NULL; 16326 } 16327 16328 kfree(pf->qp_pile); 16329 kfree(pf->vsi); 16330 16331 iounmap(hw->hw_addr); 16332 i40e_free_pf(pf); 16333 pci_release_mem_regions(pdev); 16334 16335 pci_disable_device(pdev); 16336 } 16337 16338 /** 16339 * i40e_enable_mc_magic_wake - enable multicast magic packet wake up 16340 * using the mac_address_write admin q function 16341 * @pf: pointer to i40e_pf struct 16342 **/ 16343 static void i40e_enable_mc_magic_wake(struct i40e_pf *pf) 16344 { 16345 struct i40e_vsi *main_vsi = i40e_pf_get_main_vsi(pf); 16346 struct i40e_hw *hw = &pf->hw; 16347 u8 mac_addr[6]; 16348 u16 flags = 0; 16349 int ret; 16350 16351 /* Get current MAC address in case it's an LAA */ 16352 if (main_vsi && main_vsi->netdev) { 16353 ether_addr_copy(mac_addr, main_vsi->netdev->dev_addr); 16354 } else { 16355 dev_err(&pf->pdev->dev, 16356 "Failed to retrieve MAC address; using default\n"); 16357 ether_addr_copy(mac_addr, hw->mac.addr); 16358 } 16359 16360 /* The FW expects the mac address write cmd to first be called with 16361 * one of these flags before calling it again with the multicast 16362 * enable flags. 16363 */ 16364 flags = I40E_AQC_WRITE_TYPE_LAA_WOL; 16365 16366 if (hw->func_caps.flex10_enable && hw->partition_id != 1) 16367 flags = I40E_AQC_WRITE_TYPE_LAA_ONLY; 16368 16369 ret = i40e_aq_mac_address_write(hw, flags, mac_addr, NULL); 16370 if (ret) { 16371 dev_err(&pf->pdev->dev, 16372 "Failed to update MAC address registers; cannot enable Multicast Magic packet wake up"); 16373 return; 16374 } 16375 16376 flags = I40E_AQC_MC_MAG_EN 16377 | I40E_AQC_WOL_PRESERVE_ON_PFR 16378 | I40E_AQC_WRITE_TYPE_UPDATE_MC_MAG; 16379 ret = i40e_aq_mac_address_write(hw, flags, mac_addr, NULL); 16380 if (ret) 16381 dev_err(&pf->pdev->dev, 16382 "Failed to enable Multicast Magic Packet wake up\n"); 16383 } 16384 16385 /** 16386 * i40e_io_suspend - suspend all IO operations 16387 * @pf: pointer to i40e_pf struct 16388 * 16389 **/ 16390 static int i40e_io_suspend(struct i40e_pf *pf) 16391 { 16392 struct i40e_hw *hw = &pf->hw; 16393 16394 set_bit(__I40E_DOWN, pf->state); 16395 16396 /* Ensure service task will not be running */ 16397 del_timer_sync(&pf->service_timer); 16398 cancel_work_sync(&pf->service_task); 16399 16400 /* Client close must be called explicitly here because the timer 16401 * has been stopped. 16402 */ 16403 i40e_notify_client_of_netdev_close(pf, false); 16404 16405 if (test_bit(I40E_HW_CAP_WOL_MC_MAGIC_PKT_WAKE, pf->hw.caps) && 16406 pf->wol_en) 16407 i40e_enable_mc_magic_wake(pf); 16408 16409 /* Since we're going to destroy queues during the 16410 * i40e_clear_interrupt_scheme() we should hold the RTNL lock for this 16411 * whole section 16412 */ 16413 rtnl_lock(); 16414 16415 i40e_prep_for_reset(pf); 16416 16417 wr32(hw, I40E_PFPM_APM, (pf->wol_en ? I40E_PFPM_APM_APME_MASK : 0)); 16418 wr32(hw, I40E_PFPM_WUFC, (pf->wol_en ? I40E_PFPM_WUFC_MAG_MASK : 0)); 16419 16420 /* Clear the interrupt scheme and release our IRQs so that the system 16421 * can safely hibernate even when there are a large number of CPUs. 16422 * Otherwise hibernation might fail when mapping all the vectors back 16423 * to CPU0. 16424 */ 16425 i40e_clear_interrupt_scheme(pf); 16426 16427 rtnl_unlock(); 16428 16429 return 0; 16430 } 16431 16432 /** 16433 * i40e_io_resume - resume IO operations 16434 * @pf: pointer to i40e_pf struct 16435 * 16436 **/ 16437 static int i40e_io_resume(struct i40e_pf *pf) 16438 { 16439 struct device *dev = &pf->pdev->dev; 16440 int err; 16441 16442 /* We need to hold the RTNL lock prior to restoring interrupt schemes, 16443 * since we're going to be restoring queues 16444 */ 16445 rtnl_lock(); 16446 16447 /* We cleared the interrupt scheme when we suspended, so we need to 16448 * restore it now to resume device functionality. 16449 */ 16450 err = i40e_restore_interrupt_scheme(pf); 16451 if (err) { 16452 dev_err(dev, "Cannot restore interrupt scheme: %d\n", 16453 err); 16454 } 16455 16456 clear_bit(__I40E_DOWN, pf->state); 16457 i40e_reset_and_rebuild(pf, false, true); 16458 16459 rtnl_unlock(); 16460 16461 /* Clear suspended state last after everything is recovered */ 16462 clear_bit(__I40E_SUSPENDED, pf->state); 16463 16464 /* Restart the service task */ 16465 mod_timer(&pf->service_timer, 16466 round_jiffies(jiffies + pf->service_timer_period)); 16467 16468 return 0; 16469 } 16470 16471 /** 16472 * i40e_pci_error_detected - warning that something funky happened in PCI land 16473 * @pdev: PCI device information struct 16474 * @error: the type of PCI error 16475 * 16476 * Called to warn that something happened and the error handling steps 16477 * are in progress. Allows the driver to quiesce things, be ready for 16478 * remediation. 16479 **/ 16480 static pci_ers_result_t i40e_pci_error_detected(struct pci_dev *pdev, 16481 pci_channel_state_t error) 16482 { 16483 struct i40e_pf *pf = pci_get_drvdata(pdev); 16484 16485 dev_info(&pdev->dev, "%s: error %d\n", __func__, error); 16486 16487 if (!pf) { 16488 dev_info(&pdev->dev, 16489 "Cannot recover - error happened during device probe\n"); 16490 return PCI_ERS_RESULT_DISCONNECT; 16491 } 16492 16493 /* shutdown all operations */ 16494 if (!test_bit(__I40E_SUSPENDED, pf->state)) 16495 i40e_io_suspend(pf); 16496 16497 /* Request a slot reset */ 16498 return PCI_ERS_RESULT_NEED_RESET; 16499 } 16500 16501 /** 16502 * i40e_pci_error_slot_reset - a PCI slot reset just happened 16503 * @pdev: PCI device information struct 16504 * 16505 * Called to find if the driver can work with the device now that 16506 * the pci slot has been reset. If a basic connection seems good 16507 * (registers are readable and have sane content) then return a 16508 * happy little PCI_ERS_RESULT_xxx. 16509 **/ 16510 static pci_ers_result_t i40e_pci_error_slot_reset(struct pci_dev *pdev) 16511 { 16512 struct i40e_pf *pf = pci_get_drvdata(pdev); 16513 pci_ers_result_t result; 16514 u32 reg; 16515 16516 dev_dbg(&pdev->dev, "%s\n", __func__); 16517 /* enable I/O and memory of the device */ 16518 if (pci_enable_device(pdev)) { 16519 dev_info(&pdev->dev, 16520 "Cannot re-enable PCI device after reset.\n"); 16521 result = PCI_ERS_RESULT_DISCONNECT; 16522 } else { 16523 pci_set_master(pdev); 16524 pci_restore_state(pdev); 16525 pci_save_state(pdev); 16526 pci_wake_from_d3(pdev, false); 16527 16528 reg = rd32(&pf->hw, I40E_GLGEN_RTRIG); 16529 if (reg == 0) 16530 result = PCI_ERS_RESULT_RECOVERED; 16531 else 16532 result = PCI_ERS_RESULT_DISCONNECT; 16533 } 16534 16535 return result; 16536 } 16537 16538 /** 16539 * i40e_pci_error_reset_prepare - prepare device driver for pci reset 16540 * @pdev: PCI device information struct 16541 */ 16542 static void i40e_pci_error_reset_prepare(struct pci_dev *pdev) 16543 { 16544 struct i40e_pf *pf = pci_get_drvdata(pdev); 16545 16546 i40e_prep_for_reset(pf); 16547 } 16548 16549 /** 16550 * i40e_pci_error_reset_done - pci reset done, device driver reset can begin 16551 * @pdev: PCI device information struct 16552 */ 16553 static void i40e_pci_error_reset_done(struct pci_dev *pdev) 16554 { 16555 struct i40e_pf *pf = pci_get_drvdata(pdev); 16556 16557 if (test_bit(__I40E_IN_REMOVE, pf->state)) 16558 return; 16559 16560 i40e_reset_and_rebuild(pf, false, false); 16561 #ifdef CONFIG_PCI_IOV 16562 i40e_restore_all_vfs_msi_state(pdev); 16563 #endif /* CONFIG_PCI_IOV */ 16564 } 16565 16566 /** 16567 * i40e_pci_error_resume - restart operations after PCI error recovery 16568 * @pdev: PCI device information struct 16569 * 16570 * Called to allow the driver to bring things back up after PCI error 16571 * and/or reset recovery has finished. 16572 **/ 16573 static void i40e_pci_error_resume(struct pci_dev *pdev) 16574 { 16575 struct i40e_pf *pf = pci_get_drvdata(pdev); 16576 16577 dev_dbg(&pdev->dev, "%s\n", __func__); 16578 if (test_bit(__I40E_SUSPENDED, pf->state)) 16579 return; 16580 16581 i40e_io_resume(pf); 16582 } 16583 16584 /** 16585 * i40e_shutdown - PCI callback for shutting down 16586 * @pdev: PCI device information struct 16587 **/ 16588 static void i40e_shutdown(struct pci_dev *pdev) 16589 { 16590 struct i40e_pf *pf = pci_get_drvdata(pdev); 16591 struct i40e_hw *hw = &pf->hw; 16592 16593 set_bit(__I40E_SUSPENDED, pf->state); 16594 set_bit(__I40E_DOWN, pf->state); 16595 16596 del_timer_sync(&pf->service_timer); 16597 cancel_work_sync(&pf->service_task); 16598 i40e_cloud_filter_exit(pf); 16599 i40e_fdir_teardown(pf); 16600 16601 /* Client close must be called explicitly here because the timer 16602 * has been stopped. 16603 */ 16604 i40e_notify_client_of_netdev_close(pf, false); 16605 16606 if (test_bit(I40E_HW_CAP_WOL_MC_MAGIC_PKT_WAKE, pf->hw.caps) && 16607 pf->wol_en) 16608 i40e_enable_mc_magic_wake(pf); 16609 16610 i40e_prep_for_reset(pf); 16611 16612 wr32(hw, I40E_PFPM_APM, 16613 (pf->wol_en ? I40E_PFPM_APM_APME_MASK : 0)); 16614 wr32(hw, I40E_PFPM_WUFC, 16615 (pf->wol_en ? I40E_PFPM_WUFC_MAG_MASK : 0)); 16616 16617 /* Free MSI/legacy interrupt 0 when in recovery mode. */ 16618 if (test_bit(__I40E_RECOVERY_MODE, pf->state) && 16619 !test_bit(I40E_FLAG_MSIX_ENA, pf->flags)) 16620 free_irq(pf->pdev->irq, pf); 16621 16622 /* Since we're going to destroy queues during the 16623 * i40e_clear_interrupt_scheme() we should hold the RTNL lock for this 16624 * whole section 16625 */ 16626 rtnl_lock(); 16627 i40e_clear_interrupt_scheme(pf); 16628 rtnl_unlock(); 16629 16630 if (system_state == SYSTEM_POWER_OFF) { 16631 pci_wake_from_d3(pdev, pf->wol_en); 16632 pci_set_power_state(pdev, PCI_D3hot); 16633 } 16634 } 16635 16636 /** 16637 * i40e_suspend - PM callback for moving to D3 16638 * @dev: generic device information structure 16639 **/ 16640 static int i40e_suspend(struct device *dev) 16641 { 16642 struct i40e_pf *pf = dev_get_drvdata(dev); 16643 16644 /* If we're already suspended, then there is nothing to do */ 16645 if (test_and_set_bit(__I40E_SUSPENDED, pf->state)) 16646 return 0; 16647 return i40e_io_suspend(pf); 16648 } 16649 16650 /** 16651 * i40e_resume - PM callback for waking up from D3 16652 * @dev: generic device information structure 16653 **/ 16654 static int i40e_resume(struct device *dev) 16655 { 16656 struct i40e_pf *pf = dev_get_drvdata(dev); 16657 16658 /* If we're not suspended, then there is nothing to do */ 16659 if (!test_bit(__I40E_SUSPENDED, pf->state)) 16660 return 0; 16661 return i40e_io_resume(pf); 16662 } 16663 16664 static const struct pci_error_handlers i40e_err_handler = { 16665 .error_detected = i40e_pci_error_detected, 16666 .slot_reset = i40e_pci_error_slot_reset, 16667 .reset_prepare = i40e_pci_error_reset_prepare, 16668 .reset_done = i40e_pci_error_reset_done, 16669 .resume = i40e_pci_error_resume, 16670 }; 16671 16672 static DEFINE_SIMPLE_DEV_PM_OPS(i40e_pm_ops, i40e_suspend, i40e_resume); 16673 16674 static struct pci_driver i40e_driver = { 16675 .name = i40e_driver_name, 16676 .id_table = i40e_pci_tbl, 16677 .probe = i40e_probe, 16678 .remove = i40e_remove, 16679 .driver.pm = pm_sleep_ptr(&i40e_pm_ops), 16680 .shutdown = i40e_shutdown, 16681 .err_handler = &i40e_err_handler, 16682 .sriov_configure = i40e_pci_sriov_configure, 16683 }; 16684 16685 /** 16686 * i40e_init_module - Driver registration routine 16687 * 16688 * i40e_init_module is the first routine called when the driver is 16689 * loaded. All it does is register with the PCI subsystem. 16690 **/ 16691 static int __init i40e_init_module(void) 16692 { 16693 int err; 16694 16695 pr_info("%s: %s\n", i40e_driver_name, i40e_driver_string); 16696 pr_info("%s: %s\n", i40e_driver_name, i40e_copyright); 16697 16698 /* There is no need to throttle the number of active tasks because 16699 * each device limits its own task using a state bit for scheduling 16700 * the service task, and the device tasks do not interfere with each 16701 * other, so we don't set a max task limit. We must set WQ_MEM_RECLAIM 16702 * since we need to be able to guarantee forward progress even under 16703 * memory pressure. 16704 */ 16705 i40e_wq = alloc_workqueue("%s", 0, 0, i40e_driver_name); 16706 if (!i40e_wq) { 16707 pr_err("%s: Failed to create workqueue\n", i40e_driver_name); 16708 return -ENOMEM; 16709 } 16710 16711 i40e_dbg_init(); 16712 err = pci_register_driver(&i40e_driver); 16713 if (err) { 16714 destroy_workqueue(i40e_wq); 16715 i40e_dbg_exit(); 16716 return err; 16717 } 16718 16719 return 0; 16720 } 16721 module_init(i40e_init_module); 16722 16723 /** 16724 * i40e_exit_module - Driver exit cleanup routine 16725 * 16726 * i40e_exit_module is called just before the driver is removed 16727 * from memory. 16728 **/ 16729 static void __exit i40e_exit_module(void) 16730 { 16731 pci_unregister_driver(&i40e_driver); 16732 destroy_workqueue(i40e_wq); 16733 ida_destroy(&i40e_client_ida); 16734 i40e_dbg_exit(); 16735 } 16736 module_exit(i40e_exit_module); 16737