1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright(c) 2013 - 2021 Intel Corporation. */ 3 4 #include <generated/utsrelease.h> 5 #include <linux/crash_dump.h> 6 #include <linux/net/intel/libie/pctype.h> 7 #include <linux/if_bridge.h> 8 #include <linux/if_macvlan.h> 9 #include <linux/module.h> 10 #include <net/pkt_cls.h> 11 #include <net/xdp_sock_drv.h> 12 13 /* Local includes */ 14 #include "i40e.h" 15 #include "i40e_devids.h" 16 #include "i40e_diag.h" 17 #include "i40e_lan_hmc.h" 18 #include "i40e_virtchnl_pf.h" 19 #include "i40e_xsk.h" 20 21 /* All i40e tracepoints are defined by the include below, which 22 * must be included exactly once across the whole kernel with 23 * CREATE_TRACE_POINTS defined 24 */ 25 #define CREATE_TRACE_POINTS 26 #include "i40e_trace.h" 27 28 const char i40e_driver_name[] = "i40e"; 29 static const char i40e_driver_string[] = 30 "Intel(R) Ethernet Connection XL710 Network Driver"; 31 32 static const char i40e_copyright[] = "Copyright (c) 2013 - 2019 Intel Corporation."; 33 34 /* a bit of forward declarations */ 35 static void i40e_vsi_reinit_locked(struct i40e_vsi *vsi); 36 static void i40e_handle_reset_warning(struct i40e_pf *pf, bool lock_acquired); 37 static int i40e_add_vsi(struct i40e_vsi *vsi); 38 static int i40e_add_veb(struct i40e_veb *veb, struct i40e_vsi *vsi); 39 static int i40e_setup_pf_switch(struct i40e_pf *pf, bool reinit, bool lock_acquired); 40 static int i40e_setup_misc_vector(struct i40e_pf *pf); 41 static void i40e_determine_queue_usage(struct i40e_pf *pf); 42 static int i40e_setup_pf_filter_control(struct i40e_pf *pf); 43 static void i40e_prep_for_reset(struct i40e_pf *pf); 44 static void i40e_reset_and_rebuild(struct i40e_pf *pf, bool reinit, 45 bool lock_acquired); 46 static int i40e_reset(struct i40e_pf *pf); 47 static void i40e_rebuild(struct i40e_pf *pf, bool reinit, bool lock_acquired); 48 static int i40e_setup_misc_vector_for_recovery_mode(struct i40e_pf *pf); 49 static int i40e_restore_interrupt_scheme(struct i40e_pf *pf); 50 static bool i40e_check_recovery_mode(struct i40e_pf *pf); 51 static int i40e_init_recovery_mode(struct i40e_pf *pf, struct i40e_hw *hw); 52 static void i40e_fdir_sb_setup(struct i40e_pf *pf); 53 static int i40e_veb_get_bw_info(struct i40e_veb *veb); 54 static int i40e_get_capabilities(struct i40e_pf *pf, 55 enum i40e_admin_queue_opc list_type); 56 static bool i40e_is_total_port_shutdown_enabled(struct i40e_pf *pf); 57 58 /* i40e_pci_tbl - PCI Device ID Table 59 * 60 * Last entry must be all 0s 61 * 62 * { Vendor ID, Device ID, SubVendor ID, SubDevice ID, 63 * Class, Class Mask, private data (not used) } 64 */ 65 static const struct pci_device_id i40e_pci_tbl[] = { 66 { PCI_VDEVICE(INTEL, I40E_DEV_ID_SFP_XL710) }, 67 { PCI_VDEVICE(INTEL, I40E_DEV_ID_QEMU) }, 68 { PCI_VDEVICE(INTEL, I40E_DEV_ID_KX_B) }, 69 { PCI_VDEVICE(INTEL, I40E_DEV_ID_KX_C) }, 70 { PCI_VDEVICE(INTEL, I40E_DEV_ID_QSFP_A) }, 71 { PCI_VDEVICE(INTEL, I40E_DEV_ID_QSFP_B) }, 72 { PCI_VDEVICE(INTEL, I40E_DEV_ID_QSFP_C) }, 73 { PCI_VDEVICE(INTEL, I40E_DEV_ID_1G_BASE_T_BC) }, 74 { PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_BASE_T) }, 75 { PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_BASE_T4) }, 76 { PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_BASE_T_BC) }, 77 { PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_SFP) }, 78 /* 79 * This ID conflicts with ipw2200, but the devices can be differentiated 80 * because i40e devices use PCI_CLASS_NETWORK_ETHERNET and ipw2200 81 * devices use PCI_CLASS_NETWORK_OTHER. 82 */ 83 { 84 PCI_DEVICE(PCI_VENDOR_ID_INTEL, I40E_DEV_ID_10G_B), 85 .class = PCI_CLASS_NETWORK_ETHERNET << 8, 86 .class_mask = 0xffff00, 87 }, 88 { PCI_VDEVICE(INTEL, I40E_DEV_ID_KX_X722) }, 89 { PCI_VDEVICE(INTEL, I40E_DEV_ID_QSFP_X722) }, 90 { PCI_VDEVICE(INTEL, I40E_DEV_ID_SFP_X722) }, 91 { PCI_VDEVICE(INTEL, I40E_DEV_ID_1G_BASE_T_X722) }, 92 { PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_BASE_T_X722) }, 93 { PCI_VDEVICE(INTEL, I40E_DEV_ID_SFP_I_X722) }, 94 { PCI_VDEVICE(INTEL, I40E_DEV_ID_SFP_X722_A) }, 95 { PCI_VDEVICE(INTEL, I40E_DEV_ID_20G_KR2) }, 96 { PCI_VDEVICE(INTEL, I40E_DEV_ID_20G_KR2_A) }, 97 { PCI_VDEVICE(INTEL, I40E_DEV_ID_X710_N3000) }, 98 { PCI_VDEVICE(INTEL, I40E_DEV_ID_XXV710_N3000) }, 99 { PCI_VDEVICE(INTEL, I40E_DEV_ID_25G_B) }, 100 { PCI_VDEVICE(INTEL, I40E_DEV_ID_25G_SFP28) }, 101 /* required last entry */ 102 { } 103 }; 104 MODULE_DEVICE_TABLE(pci, i40e_pci_tbl); 105 106 #define I40E_MAX_VF_COUNT 128 107 static int debug = -1; 108 module_param(debug, uint, 0); 109 MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all), Debug mask (0x8XXXXXXX)"); 110 111 MODULE_DESCRIPTION("Intel(R) Ethernet Connection XL710 Network Driver"); 112 MODULE_IMPORT_NS("LIBIE"); 113 MODULE_IMPORT_NS("LIBIE_ADMINQ"); 114 MODULE_LICENSE("GPL v2"); 115 116 static struct workqueue_struct *i40e_wq; 117 118 static void netdev_hw_addr_refcnt(struct i40e_mac_filter *f, 119 struct net_device *netdev, int delta) 120 { 121 struct netdev_hw_addr_list *ha_list; 122 struct netdev_hw_addr *ha; 123 124 if (!f || !netdev) 125 return; 126 127 if (is_unicast_ether_addr(f->macaddr) || is_link_local_ether_addr(f->macaddr)) 128 ha_list = &netdev->uc; 129 else 130 ha_list = &netdev->mc; 131 132 netdev_hw_addr_list_for_each(ha, ha_list) { 133 if (ether_addr_equal(ha->addr, f->macaddr)) { 134 ha->refcount += delta; 135 if (ha->refcount <= 0) 136 ha->refcount = 1; 137 break; 138 } 139 } 140 } 141 142 /** 143 * i40e_hw_to_dev - get device pointer from the hardware structure 144 * @hw: pointer to the device HW structure 145 **/ 146 struct device *i40e_hw_to_dev(struct i40e_hw *hw) 147 { 148 struct i40e_pf *pf = i40e_hw_to_pf(hw); 149 150 return &pf->pdev->dev; 151 } 152 153 /** 154 * i40e_allocate_dma_mem - OS specific memory alloc for shared code 155 * @hw: pointer to the HW structure 156 * @mem: ptr to mem struct to fill out 157 * @size: size of memory requested 158 * @alignment: what to align the allocation to 159 **/ 160 int i40e_allocate_dma_mem(struct i40e_hw *hw, struct i40e_dma_mem *mem, 161 u64 size, u32 alignment) 162 { 163 struct i40e_pf *pf = i40e_hw_to_pf(hw); 164 165 mem->size = ALIGN(size, alignment); 166 mem->va = dma_alloc_coherent(&pf->pdev->dev, mem->size, &mem->pa, 167 GFP_KERNEL); 168 if (!mem->va) 169 return -ENOMEM; 170 171 return 0; 172 } 173 174 /** 175 * i40e_free_dma_mem - OS specific memory free for shared code 176 * @hw: pointer to the HW structure 177 * @mem: ptr to mem struct to free 178 **/ 179 int i40e_free_dma_mem(struct i40e_hw *hw, struct i40e_dma_mem *mem) 180 { 181 struct i40e_pf *pf = i40e_hw_to_pf(hw); 182 183 dma_free_coherent(&pf->pdev->dev, mem->size, mem->va, mem->pa); 184 mem->va = NULL; 185 mem->pa = 0; 186 mem->size = 0; 187 188 return 0; 189 } 190 191 /** 192 * i40e_allocate_virt_mem - OS specific memory alloc for shared code 193 * @hw: pointer to the HW structure 194 * @mem: ptr to mem struct to fill out 195 * @size: size of memory requested 196 **/ 197 int i40e_allocate_virt_mem(struct i40e_hw *hw, struct i40e_virt_mem *mem, 198 u32 size) 199 { 200 mem->size = size; 201 mem->va = kzalloc(size, GFP_KERNEL); 202 203 if (!mem->va) 204 return -ENOMEM; 205 206 return 0; 207 } 208 209 /** 210 * i40e_free_virt_mem - OS specific memory free for shared code 211 * @hw: pointer to the HW structure 212 * @mem: ptr to mem struct to free 213 **/ 214 int i40e_free_virt_mem(struct i40e_hw *hw, struct i40e_virt_mem *mem) 215 { 216 /* it's ok to kfree a NULL pointer */ 217 kfree(mem->va); 218 mem->va = NULL; 219 mem->size = 0; 220 221 return 0; 222 } 223 224 /** 225 * i40e_get_lump - find a lump of free generic resource 226 * @pf: board private structure 227 * @pile: the pile of resource to search 228 * @needed: the number of items needed 229 * @id: an owner id to stick on the items assigned 230 * 231 * Returns the base item index of the lump, or negative for error 232 **/ 233 static int i40e_get_lump(struct i40e_pf *pf, struct i40e_lump_tracking *pile, 234 u16 needed, u16 id) 235 { 236 int ret = -ENOMEM; 237 int i, j; 238 239 if (!pile || needed == 0 || id >= I40E_PILE_VALID_BIT) { 240 dev_info(&pf->pdev->dev, 241 "param err: pile=%s needed=%d id=0x%04x\n", 242 pile ? "<valid>" : "<null>", needed, id); 243 return -EINVAL; 244 } 245 246 /* Allocate last queue in the pile for FDIR VSI queue 247 * so it doesn't fragment the qp_pile 248 */ 249 if (pile == pf->qp_pile && pf->vsi[id]->type == I40E_VSI_FDIR) { 250 if (pile->list[pile->num_entries - 1] & I40E_PILE_VALID_BIT) { 251 dev_err(&pf->pdev->dev, 252 "Cannot allocate queue %d for I40E_VSI_FDIR\n", 253 pile->num_entries - 1); 254 return -ENOMEM; 255 } 256 pile->list[pile->num_entries - 1] = id | I40E_PILE_VALID_BIT; 257 return pile->num_entries - 1; 258 } 259 260 i = 0; 261 while (i < pile->num_entries) { 262 /* skip already allocated entries */ 263 if (pile->list[i] & I40E_PILE_VALID_BIT) { 264 i++; 265 continue; 266 } 267 268 /* do we have enough in this lump? */ 269 for (j = 0; (j < needed) && ((i+j) < pile->num_entries); j++) { 270 if (pile->list[i+j] & I40E_PILE_VALID_BIT) 271 break; 272 } 273 274 if (j == needed) { 275 /* there was enough, so assign it to the requestor */ 276 for (j = 0; j < needed; j++) 277 pile->list[i+j] = id | I40E_PILE_VALID_BIT; 278 ret = i; 279 break; 280 } 281 282 /* not enough, so skip over it and continue looking */ 283 i += j; 284 } 285 286 return ret; 287 } 288 289 /** 290 * i40e_put_lump - return a lump of generic resource 291 * @pile: the pile of resource to search 292 * @index: the base item index 293 * @id: the owner id of the items assigned 294 * 295 * Returns the count of items in the lump 296 **/ 297 static int i40e_put_lump(struct i40e_lump_tracking *pile, u16 index, u16 id) 298 { 299 int valid_id = (id | I40E_PILE_VALID_BIT); 300 int count = 0; 301 u16 i; 302 303 if (!pile || index >= pile->num_entries) 304 return -EINVAL; 305 306 for (i = index; 307 i < pile->num_entries && pile->list[i] == valid_id; 308 i++) { 309 pile->list[i] = 0; 310 count++; 311 } 312 313 314 return count; 315 } 316 317 /** 318 * i40e_find_vsi_from_id - searches for the vsi with the given id 319 * @pf: the pf structure to search for the vsi 320 * @id: id of the vsi it is searching for 321 **/ 322 struct i40e_vsi *i40e_find_vsi_from_id(struct i40e_pf *pf, u16 id) 323 { 324 struct i40e_vsi *vsi; 325 int i; 326 327 i40e_pf_for_each_vsi(pf, i, vsi) 328 if (vsi->id == id) 329 return vsi; 330 331 return NULL; 332 } 333 334 /** 335 * i40e_service_event_schedule - Schedule the service task to wake up 336 * @pf: board private structure 337 * 338 * If not already scheduled, this puts the task into the work queue 339 **/ 340 void i40e_service_event_schedule(struct i40e_pf *pf) 341 { 342 if ((!test_bit(__I40E_DOWN, pf->state) && 343 !test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state)) || 344 test_bit(__I40E_RECOVERY_MODE, pf->state)) 345 queue_work(i40e_wq, &pf->service_task); 346 } 347 348 /** 349 * i40e_tx_timeout - Respond to a Tx Hang 350 * @netdev: network interface device structure 351 * @txqueue: queue number timing out 352 * 353 * If any port has noticed a Tx timeout, it is likely that the whole 354 * device is munged, not just the one netdev port, so go for the full 355 * reset. 356 **/ 357 static void i40e_tx_timeout(struct net_device *netdev, unsigned int txqueue) 358 { 359 struct i40e_netdev_priv *np = netdev_priv(netdev); 360 struct i40e_vsi *vsi = np->vsi; 361 struct i40e_pf *pf = vsi->back; 362 struct i40e_ring *tx_ring = NULL; 363 unsigned int i; 364 u32 head, val; 365 366 pf->tx_timeout_count++; 367 368 /* with txqueue index, find the tx_ring struct */ 369 for (i = 0; i < vsi->num_queue_pairs; i++) { 370 if (vsi->tx_rings[i] && vsi->tx_rings[i]->desc) { 371 if (txqueue == 372 vsi->tx_rings[i]->queue_index) { 373 tx_ring = vsi->tx_rings[i]; 374 break; 375 } 376 } 377 } 378 379 if (time_after(jiffies, (pf->tx_timeout_last_recovery + HZ*20))) 380 pf->tx_timeout_recovery_level = 1; /* reset after some time */ 381 else if (time_before(jiffies, 382 (pf->tx_timeout_last_recovery + netdev->watchdog_timeo))) 383 return; /* don't do any new action before the next timeout */ 384 385 /* don't kick off another recovery if one is already pending */ 386 if (test_and_set_bit(__I40E_TIMEOUT_RECOVERY_PENDING, pf->state)) 387 return; 388 389 if (tx_ring) { 390 head = i40e_get_head(tx_ring); 391 /* Read interrupt register */ 392 if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags)) 393 val = rd32(&pf->hw, 394 I40E_PFINT_DYN_CTLN(tx_ring->q_vector->v_idx + 395 tx_ring->vsi->base_vector - 1)); 396 else 397 val = rd32(&pf->hw, I40E_PFINT_DYN_CTL0); 398 399 netdev_info(netdev, "tx_timeout: VSI_seid: %d, Q %d, NTC: 0x%x, HWB: 0x%x, NTU: 0x%x, TAIL: 0x%x, INT: 0x%x\n", 400 vsi->seid, txqueue, tx_ring->next_to_clean, 401 head, tx_ring->next_to_use, 402 readl(tx_ring->tail), val); 403 } 404 405 pf->tx_timeout_last_recovery = jiffies; 406 netdev_info(netdev, "tx_timeout recovery level %d, txqueue %d\n", 407 pf->tx_timeout_recovery_level, txqueue); 408 409 switch (pf->tx_timeout_recovery_level) { 410 case 1: 411 set_bit(__I40E_PF_RESET_REQUESTED, pf->state); 412 break; 413 case 2: 414 set_bit(__I40E_CORE_RESET_REQUESTED, pf->state); 415 break; 416 case 3: 417 set_bit(__I40E_GLOBAL_RESET_REQUESTED, pf->state); 418 break; 419 default: 420 netdev_err(netdev, "tx_timeout recovery unsuccessful, device is in non-recoverable state.\n"); 421 set_bit(__I40E_DOWN_REQUESTED, pf->state); 422 set_bit(__I40E_VSI_DOWN_REQUESTED, vsi->state); 423 break; 424 } 425 426 i40e_service_event_schedule(pf); 427 pf->tx_timeout_recovery_level++; 428 } 429 430 /** 431 * i40e_get_vsi_stats_struct - Get System Network Statistics 432 * @vsi: the VSI we care about 433 * 434 * Returns the address of the device statistics structure. 435 * The statistics are actually updated from the service task. 436 **/ 437 struct rtnl_link_stats64 *i40e_get_vsi_stats_struct(struct i40e_vsi *vsi) 438 { 439 return &vsi->net_stats; 440 } 441 442 /** 443 * i40e_get_netdev_stats_struct_tx - populate stats from a Tx ring 444 * @ring: Tx ring to get statistics from 445 * @stats: statistics entry to be updated 446 **/ 447 static void i40e_get_netdev_stats_struct_tx(struct i40e_ring *ring, 448 struct rtnl_link_stats64 *stats) 449 { 450 u64 bytes, packets; 451 unsigned int start; 452 453 do { 454 start = u64_stats_fetch_begin(&ring->syncp); 455 packets = ring->stats.packets; 456 bytes = ring->stats.bytes; 457 } while (u64_stats_fetch_retry(&ring->syncp, start)); 458 459 stats->tx_packets += packets; 460 stats->tx_bytes += bytes; 461 } 462 463 /** 464 * i40e_get_netdev_stats_struct - Get statistics for netdev interface 465 * @netdev: network interface device structure 466 * @stats: data structure to store statistics 467 * 468 * Returns the address of the device statistics structure. 469 * The statistics are actually updated from the service task. 470 **/ 471 static void i40e_get_netdev_stats_struct(struct net_device *netdev, 472 struct rtnl_link_stats64 *stats) 473 { 474 struct i40e_netdev_priv *np = netdev_priv(netdev); 475 struct i40e_vsi *vsi = np->vsi; 476 struct rtnl_link_stats64 *vsi_stats = i40e_get_vsi_stats_struct(vsi); 477 struct i40e_ring *ring; 478 int i; 479 480 if (test_bit(__I40E_VSI_DOWN, vsi->state)) 481 return; 482 483 if (!vsi->tx_rings) 484 return; 485 486 rcu_read_lock(); 487 for (i = 0; i < vsi->num_queue_pairs; i++) { 488 u64 bytes, packets; 489 unsigned int start; 490 491 ring = READ_ONCE(vsi->tx_rings[i]); 492 if (!ring) 493 continue; 494 i40e_get_netdev_stats_struct_tx(ring, stats); 495 496 if (i40e_enabled_xdp_vsi(vsi)) { 497 ring = READ_ONCE(vsi->xdp_rings[i]); 498 if (!ring) 499 continue; 500 i40e_get_netdev_stats_struct_tx(ring, stats); 501 } 502 503 ring = READ_ONCE(vsi->rx_rings[i]); 504 if (!ring) 505 continue; 506 do { 507 start = u64_stats_fetch_begin(&ring->syncp); 508 packets = ring->stats.packets; 509 bytes = ring->stats.bytes; 510 } while (u64_stats_fetch_retry(&ring->syncp, start)); 511 512 stats->rx_packets += packets; 513 stats->rx_bytes += bytes; 514 515 } 516 rcu_read_unlock(); 517 518 /* following stats updated by i40e_watchdog_subtask() */ 519 stats->multicast = vsi_stats->multicast; 520 stats->tx_errors = vsi_stats->tx_errors; 521 stats->tx_dropped = vsi_stats->tx_dropped; 522 stats->rx_errors = vsi_stats->rx_errors; 523 stats->rx_dropped = vsi_stats->rx_dropped; 524 stats->rx_missed_errors = vsi_stats->rx_missed_errors; 525 stats->rx_crc_errors = vsi_stats->rx_crc_errors; 526 stats->rx_length_errors = vsi_stats->rx_length_errors; 527 } 528 529 /** 530 * i40e_vsi_reset_stats - Resets all stats of the given vsi 531 * @vsi: the VSI to have its stats reset 532 **/ 533 void i40e_vsi_reset_stats(struct i40e_vsi *vsi) 534 { 535 struct rtnl_link_stats64 *ns; 536 int i; 537 538 if (!vsi) 539 return; 540 541 ns = i40e_get_vsi_stats_struct(vsi); 542 memset(ns, 0, sizeof(*ns)); 543 memset(&vsi->net_stats_offsets, 0, sizeof(vsi->net_stats_offsets)); 544 memset(&vsi->eth_stats, 0, sizeof(vsi->eth_stats)); 545 memset(&vsi->eth_stats_offsets, 0, sizeof(vsi->eth_stats_offsets)); 546 if (vsi->rx_rings && vsi->rx_rings[0]) { 547 for (i = 0; i < vsi->num_queue_pairs; i++) { 548 memset(&vsi->rx_rings[i]->stats, 0, 549 sizeof(vsi->rx_rings[i]->stats)); 550 memset(&vsi->rx_rings[i]->rx_stats, 0, 551 sizeof(vsi->rx_rings[i]->rx_stats)); 552 memset(&vsi->tx_rings[i]->stats, 0, 553 sizeof(vsi->tx_rings[i]->stats)); 554 memset(&vsi->tx_rings[i]->tx_stats, 0, 555 sizeof(vsi->tx_rings[i]->tx_stats)); 556 } 557 } 558 vsi->stat_offsets_loaded = false; 559 } 560 561 /** 562 * i40e_pf_reset_stats - Reset all of the stats for the given PF 563 * @pf: the PF to be reset 564 **/ 565 void i40e_pf_reset_stats(struct i40e_pf *pf) 566 { 567 struct i40e_veb *veb; 568 int i; 569 570 memset(&pf->stats, 0, sizeof(pf->stats)); 571 memset(&pf->stats_offsets, 0, sizeof(pf->stats_offsets)); 572 pf->stat_offsets_loaded = false; 573 574 i40e_pf_for_each_veb(pf, i, veb) { 575 memset(&veb->stats, 0, sizeof(veb->stats)); 576 memset(&veb->stats_offsets, 0, sizeof(veb->stats_offsets)); 577 memset(&veb->tc_stats, 0, sizeof(veb->tc_stats)); 578 memset(&veb->tc_stats_offsets, 0, sizeof(veb->tc_stats_offsets)); 579 veb->stat_offsets_loaded = false; 580 } 581 pf->hw_csum_rx_error = 0; 582 } 583 584 /** 585 * i40e_compute_pci_to_hw_id - compute index form PCI function. 586 * @vsi: ptr to the VSI to read from. 587 * @hw: ptr to the hardware info. 588 **/ 589 static u32 i40e_compute_pci_to_hw_id(struct i40e_vsi *vsi, struct i40e_hw *hw) 590 { 591 int pf_count = i40e_get_pf_count(hw); 592 593 if (vsi->type == I40E_VSI_SRIOV) 594 return (hw->port * BIT(7)) / pf_count + vsi->vf_id; 595 596 return hw->port + BIT(7); 597 } 598 599 /** 600 * i40e_stat_update64 - read and update a 64 bit stat from the chip. 601 * @hw: ptr to the hardware info. 602 * @hireg: the high 32 bit reg to read. 603 * @loreg: the low 32 bit reg to read. 604 * @offset_loaded: has the initial offset been loaded yet. 605 * @offset: ptr to current offset value. 606 * @stat: ptr to the stat. 607 * 608 * Since the device stats are not reset at PFReset, they will not 609 * be zeroed when the driver starts. We'll save the first values read 610 * and use them as offsets to be subtracted from the raw values in order 611 * to report stats that count from zero. 612 **/ 613 static void i40e_stat_update64(struct i40e_hw *hw, u32 hireg, u32 loreg, 614 bool offset_loaded, u64 *offset, u64 *stat) 615 { 616 u64 new_data; 617 618 new_data = rd64(hw, loreg); 619 620 if (!offset_loaded || new_data < *offset) 621 *offset = new_data; 622 *stat = new_data - *offset; 623 } 624 625 /** 626 * i40e_stat_update48 - read and update a 48 bit stat from the chip 627 * @hw: ptr to the hardware info 628 * @hireg: the high 32 bit reg to read 629 * @loreg: the low 32 bit reg to read 630 * @offset_loaded: has the initial offset been loaded yet 631 * @offset: ptr to current offset value 632 * @stat: ptr to the stat 633 * 634 * Since the device stats are not reset at PFReset, they likely will not 635 * be zeroed when the driver starts. We'll save the first values read 636 * and use them as offsets to be subtracted from the raw values in order 637 * to report stats that count from zero. In the process, we also manage 638 * the potential roll-over. 639 **/ 640 static void i40e_stat_update48(struct i40e_hw *hw, u32 hireg, u32 loreg, 641 bool offset_loaded, u64 *offset, u64 *stat) 642 { 643 u64 new_data; 644 645 if (hw->device_id == I40E_DEV_ID_QEMU) { 646 new_data = rd32(hw, loreg); 647 new_data |= ((u64)(rd32(hw, hireg) & 0xFFFF)) << 32; 648 } else { 649 new_data = rd64(hw, loreg); 650 } 651 if (!offset_loaded) 652 *offset = new_data; 653 if (likely(new_data >= *offset)) 654 *stat = new_data - *offset; 655 else 656 *stat = (new_data + BIT_ULL(48)) - *offset; 657 *stat &= 0xFFFFFFFFFFFFULL; 658 } 659 660 /** 661 * i40e_stat_update32 - read and update a 32 bit stat from the chip 662 * @hw: ptr to the hardware info 663 * @reg: the hw reg to read 664 * @offset_loaded: has the initial offset been loaded yet 665 * @offset: ptr to current offset value 666 * @stat: ptr to the stat 667 **/ 668 static void i40e_stat_update32(struct i40e_hw *hw, u32 reg, 669 bool offset_loaded, u64 *offset, u64 *stat) 670 { 671 u32 new_data; 672 673 new_data = rd32(hw, reg); 674 if (!offset_loaded) 675 *offset = new_data; 676 if (likely(new_data >= *offset)) 677 *stat = (u32)(new_data - *offset); 678 else 679 *stat = (u32)((new_data + BIT_ULL(32)) - *offset); 680 } 681 682 /** 683 * i40e_stat_update_and_clear32 - read and clear hw reg, update a 32 bit stat 684 * @hw: ptr to the hardware info 685 * @reg: the hw reg to read and clear 686 * @stat: ptr to the stat 687 **/ 688 static void i40e_stat_update_and_clear32(struct i40e_hw *hw, u32 reg, u64 *stat) 689 { 690 u32 new_data = rd32(hw, reg); 691 692 wr32(hw, reg, 1); /* must write a nonzero value to clear register */ 693 *stat += new_data; 694 } 695 696 /** 697 * i40e_stats_update_rx_discards - update rx_discards. 698 * @vsi: ptr to the VSI to be updated. 699 * @hw: ptr to the hardware info. 700 * @stat_idx: VSI's stat_counter_idx. 701 * @offset_loaded: ptr to the VSI's stat_offsets_loaded. 702 * @stat_offset: ptr to stat_offset to store first read of specific register. 703 * @stat: ptr to VSI's stat to be updated. 704 **/ 705 static void 706 i40e_stats_update_rx_discards(struct i40e_vsi *vsi, struct i40e_hw *hw, 707 int stat_idx, bool offset_loaded, 708 struct i40e_eth_stats *stat_offset, 709 struct i40e_eth_stats *stat) 710 { 711 i40e_stat_update32(hw, I40E_GLV_RDPC(stat_idx), offset_loaded, 712 &stat_offset->rx_discards, &stat->rx_discards); 713 i40e_stat_update64(hw, 714 I40E_GL_RXERR1H(i40e_compute_pci_to_hw_id(vsi, hw)), 715 I40E_GL_RXERR1L(i40e_compute_pci_to_hw_id(vsi, hw)), 716 offset_loaded, &stat_offset->rx_discards_other, 717 &stat->rx_discards_other); 718 } 719 720 /** 721 * i40e_update_eth_stats - Update VSI-specific ethernet statistics counters. 722 * @vsi: the VSI to be updated 723 **/ 724 void i40e_update_eth_stats(struct i40e_vsi *vsi) 725 { 726 int stat_idx = le16_to_cpu(vsi->info.stat_counter_idx); 727 struct i40e_pf *pf = vsi->back; 728 struct i40e_hw *hw = &pf->hw; 729 struct i40e_eth_stats *oes; 730 struct i40e_eth_stats *es; /* device's eth stats */ 731 732 es = &vsi->eth_stats; 733 oes = &vsi->eth_stats_offsets; 734 735 /* Gather up the stats that the hw collects */ 736 i40e_stat_update32(hw, I40E_GLV_TEPC(stat_idx), 737 vsi->stat_offsets_loaded, 738 &oes->tx_errors, &es->tx_errors); 739 i40e_stat_update32(hw, I40E_GLV_RUPP(stat_idx), 740 vsi->stat_offsets_loaded, 741 &oes->rx_unknown_protocol, &es->rx_unknown_protocol); 742 743 i40e_stat_update48(hw, I40E_GLV_GORCH(stat_idx), 744 I40E_GLV_GORCL(stat_idx), 745 vsi->stat_offsets_loaded, 746 &oes->rx_bytes, &es->rx_bytes); 747 i40e_stat_update48(hw, I40E_GLV_UPRCH(stat_idx), 748 I40E_GLV_UPRCL(stat_idx), 749 vsi->stat_offsets_loaded, 750 &oes->rx_unicast, &es->rx_unicast); 751 i40e_stat_update48(hw, I40E_GLV_MPRCH(stat_idx), 752 I40E_GLV_MPRCL(stat_idx), 753 vsi->stat_offsets_loaded, 754 &oes->rx_multicast, &es->rx_multicast); 755 i40e_stat_update48(hw, I40E_GLV_BPRCH(stat_idx), 756 I40E_GLV_BPRCL(stat_idx), 757 vsi->stat_offsets_loaded, 758 &oes->rx_broadcast, &es->rx_broadcast); 759 760 i40e_stat_update48(hw, I40E_GLV_GOTCH(stat_idx), 761 I40E_GLV_GOTCL(stat_idx), 762 vsi->stat_offsets_loaded, 763 &oes->tx_bytes, &es->tx_bytes); 764 i40e_stat_update48(hw, I40E_GLV_UPTCH(stat_idx), 765 I40E_GLV_UPTCL(stat_idx), 766 vsi->stat_offsets_loaded, 767 &oes->tx_unicast, &es->tx_unicast); 768 i40e_stat_update48(hw, I40E_GLV_MPTCH(stat_idx), 769 I40E_GLV_MPTCL(stat_idx), 770 vsi->stat_offsets_loaded, 771 &oes->tx_multicast, &es->tx_multicast); 772 i40e_stat_update48(hw, I40E_GLV_BPTCH(stat_idx), 773 I40E_GLV_BPTCL(stat_idx), 774 vsi->stat_offsets_loaded, 775 &oes->tx_broadcast, &es->tx_broadcast); 776 777 i40e_stats_update_rx_discards(vsi, hw, stat_idx, 778 vsi->stat_offsets_loaded, oes, es); 779 780 vsi->stat_offsets_loaded = true; 781 } 782 783 /** 784 * i40e_update_veb_stats - Update Switch component statistics 785 * @veb: the VEB being updated 786 **/ 787 void i40e_update_veb_stats(struct i40e_veb *veb) 788 { 789 struct i40e_pf *pf = veb->pf; 790 struct i40e_hw *hw = &pf->hw; 791 struct i40e_eth_stats *oes; 792 struct i40e_eth_stats *es; /* device's eth stats */ 793 struct i40e_veb_tc_stats *veb_oes; 794 struct i40e_veb_tc_stats *veb_es; 795 int i, idx = 0; 796 797 idx = veb->stats_idx; 798 es = &veb->stats; 799 oes = &veb->stats_offsets; 800 veb_es = &veb->tc_stats; 801 veb_oes = &veb->tc_stats_offsets; 802 803 /* Gather up the stats that the hw collects */ 804 i40e_stat_update32(hw, I40E_GLSW_TDPC(idx), 805 veb->stat_offsets_loaded, 806 &oes->tx_discards, &es->tx_discards); 807 if (hw->revision_id > 0) 808 i40e_stat_update32(hw, I40E_GLSW_RUPP(idx), 809 veb->stat_offsets_loaded, 810 &oes->rx_unknown_protocol, 811 &es->rx_unknown_protocol); 812 i40e_stat_update48(hw, I40E_GLSW_GORCH(idx), I40E_GLSW_GORCL(idx), 813 veb->stat_offsets_loaded, 814 &oes->rx_bytes, &es->rx_bytes); 815 i40e_stat_update48(hw, I40E_GLSW_UPRCH(idx), I40E_GLSW_UPRCL(idx), 816 veb->stat_offsets_loaded, 817 &oes->rx_unicast, &es->rx_unicast); 818 i40e_stat_update48(hw, I40E_GLSW_MPRCH(idx), I40E_GLSW_MPRCL(idx), 819 veb->stat_offsets_loaded, 820 &oes->rx_multicast, &es->rx_multicast); 821 i40e_stat_update48(hw, I40E_GLSW_BPRCH(idx), I40E_GLSW_BPRCL(idx), 822 veb->stat_offsets_loaded, 823 &oes->rx_broadcast, &es->rx_broadcast); 824 825 i40e_stat_update48(hw, I40E_GLSW_GOTCH(idx), I40E_GLSW_GOTCL(idx), 826 veb->stat_offsets_loaded, 827 &oes->tx_bytes, &es->tx_bytes); 828 i40e_stat_update48(hw, I40E_GLSW_UPTCH(idx), I40E_GLSW_UPTCL(idx), 829 veb->stat_offsets_loaded, 830 &oes->tx_unicast, &es->tx_unicast); 831 i40e_stat_update48(hw, I40E_GLSW_MPTCH(idx), I40E_GLSW_MPTCL(idx), 832 veb->stat_offsets_loaded, 833 &oes->tx_multicast, &es->tx_multicast); 834 i40e_stat_update48(hw, I40E_GLSW_BPTCH(idx), I40E_GLSW_BPTCL(idx), 835 veb->stat_offsets_loaded, 836 &oes->tx_broadcast, &es->tx_broadcast); 837 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 838 i40e_stat_update48(hw, I40E_GLVEBTC_RPCH(i, idx), 839 I40E_GLVEBTC_RPCL(i, idx), 840 veb->stat_offsets_loaded, 841 &veb_oes->tc_rx_packets[i], 842 &veb_es->tc_rx_packets[i]); 843 i40e_stat_update48(hw, I40E_GLVEBTC_RBCH(i, idx), 844 I40E_GLVEBTC_RBCL(i, idx), 845 veb->stat_offsets_loaded, 846 &veb_oes->tc_rx_bytes[i], 847 &veb_es->tc_rx_bytes[i]); 848 i40e_stat_update48(hw, I40E_GLVEBTC_TPCH(i, idx), 849 I40E_GLVEBTC_TPCL(i, idx), 850 veb->stat_offsets_loaded, 851 &veb_oes->tc_tx_packets[i], 852 &veb_es->tc_tx_packets[i]); 853 i40e_stat_update48(hw, I40E_GLVEBTC_TBCH(i, idx), 854 I40E_GLVEBTC_TBCL(i, idx), 855 veb->stat_offsets_loaded, 856 &veb_oes->tc_tx_bytes[i], 857 &veb_es->tc_tx_bytes[i]); 858 } 859 veb->stat_offsets_loaded = true; 860 } 861 862 /** 863 * i40e_update_vsi_stats - Update the vsi statistics counters. 864 * @vsi: the VSI to be updated 865 * 866 * There are a few instances where we store the same stat in a 867 * couple of different structs. This is partly because we have 868 * the netdev stats that need to be filled out, which is slightly 869 * different from the "eth_stats" defined by the chip and used in 870 * VF communications. We sort it out here. 871 **/ 872 static void i40e_update_vsi_stats(struct i40e_vsi *vsi) 873 { 874 u64 rx_page, rx_buf, rx_reuse, rx_alloc, rx_waive, rx_busy; 875 struct i40e_pf *pf = vsi->back; 876 struct rtnl_link_stats64 *ons; 877 struct rtnl_link_stats64 *ns; /* netdev stats */ 878 struct i40e_eth_stats *oes; 879 struct i40e_eth_stats *es; /* device's eth stats */ 880 u64 tx_restart, tx_busy; 881 struct i40e_ring *p; 882 u64 bytes, packets; 883 unsigned int start; 884 u64 tx_linearize; 885 u64 tx_force_wb; 886 u64 tx_stopped; 887 u64 rx_p, rx_b; 888 u64 tx_p, tx_b; 889 u16 q; 890 891 if (test_bit(__I40E_VSI_DOWN, vsi->state) || 892 test_bit(__I40E_CONFIG_BUSY, pf->state)) 893 return; 894 895 ns = i40e_get_vsi_stats_struct(vsi); 896 ons = &vsi->net_stats_offsets; 897 es = &vsi->eth_stats; 898 oes = &vsi->eth_stats_offsets; 899 900 /* Gather up the netdev and vsi stats that the driver collects 901 * on the fly during packet processing 902 */ 903 rx_b = rx_p = 0; 904 tx_b = tx_p = 0; 905 tx_restart = tx_busy = tx_linearize = tx_force_wb = 0; 906 tx_stopped = 0; 907 rx_page = 0; 908 rx_buf = 0; 909 rx_reuse = 0; 910 rx_alloc = 0; 911 rx_waive = 0; 912 rx_busy = 0; 913 rcu_read_lock(); 914 for (q = 0; q < vsi->num_queue_pairs; q++) { 915 /* locate Tx ring */ 916 p = READ_ONCE(vsi->tx_rings[q]); 917 if (!p) 918 continue; 919 920 do { 921 start = u64_stats_fetch_begin(&p->syncp); 922 packets = p->stats.packets; 923 bytes = p->stats.bytes; 924 } while (u64_stats_fetch_retry(&p->syncp, start)); 925 tx_b += bytes; 926 tx_p += packets; 927 tx_restart += p->tx_stats.restart_queue; 928 tx_busy += p->tx_stats.tx_busy; 929 tx_linearize += p->tx_stats.tx_linearize; 930 tx_force_wb += p->tx_stats.tx_force_wb; 931 tx_stopped += p->tx_stats.tx_stopped; 932 933 /* locate Rx ring */ 934 p = READ_ONCE(vsi->rx_rings[q]); 935 if (!p) 936 continue; 937 938 do { 939 start = u64_stats_fetch_begin(&p->syncp); 940 packets = p->stats.packets; 941 bytes = p->stats.bytes; 942 } while (u64_stats_fetch_retry(&p->syncp, start)); 943 rx_b += bytes; 944 rx_p += packets; 945 rx_buf += p->rx_stats.alloc_buff_failed; 946 rx_page += p->rx_stats.alloc_page_failed; 947 rx_reuse += p->rx_stats.page_reuse_count; 948 rx_alloc += p->rx_stats.page_alloc_count; 949 rx_waive += p->rx_stats.page_waive_count; 950 rx_busy += p->rx_stats.page_busy_count; 951 952 if (i40e_enabled_xdp_vsi(vsi)) { 953 /* locate XDP ring */ 954 p = READ_ONCE(vsi->xdp_rings[q]); 955 if (!p) 956 continue; 957 958 do { 959 start = u64_stats_fetch_begin(&p->syncp); 960 packets = p->stats.packets; 961 bytes = p->stats.bytes; 962 } while (u64_stats_fetch_retry(&p->syncp, start)); 963 tx_b += bytes; 964 tx_p += packets; 965 tx_restart += p->tx_stats.restart_queue; 966 tx_busy += p->tx_stats.tx_busy; 967 tx_linearize += p->tx_stats.tx_linearize; 968 tx_force_wb += p->tx_stats.tx_force_wb; 969 } 970 } 971 rcu_read_unlock(); 972 vsi->tx_restart = tx_restart; 973 vsi->tx_busy = tx_busy; 974 vsi->tx_linearize = tx_linearize; 975 vsi->tx_force_wb = tx_force_wb; 976 vsi->tx_stopped = tx_stopped; 977 vsi->rx_page_failed = rx_page; 978 vsi->rx_buf_failed = rx_buf; 979 vsi->rx_page_reuse = rx_reuse; 980 vsi->rx_page_alloc = rx_alloc; 981 vsi->rx_page_waive = rx_waive; 982 vsi->rx_page_busy = rx_busy; 983 984 ns->rx_packets = rx_p; 985 ns->rx_bytes = rx_b; 986 ns->tx_packets = tx_p; 987 ns->tx_bytes = tx_b; 988 989 /* update netdev stats from eth stats */ 990 i40e_update_eth_stats(vsi); 991 ons->tx_errors = oes->tx_errors; 992 ns->tx_errors = es->tx_errors; 993 ons->multicast = oes->rx_multicast; 994 ns->multicast = es->rx_multicast; 995 ons->rx_dropped = oes->rx_discards_other; 996 ns->rx_dropped = es->rx_discards_other; 997 ons->rx_missed_errors = oes->rx_discards; 998 ns->rx_missed_errors = es->rx_discards; 999 ons->tx_dropped = oes->tx_discards; 1000 ns->tx_dropped = es->tx_discards; 1001 1002 /* pull in a couple PF stats if this is the main vsi */ 1003 if (vsi->type == I40E_VSI_MAIN) { 1004 ns->rx_crc_errors = pf->stats.crc_errors; 1005 ns->rx_errors = pf->stats.crc_errors + pf->stats.illegal_bytes; 1006 ns->rx_length_errors = pf->stats.rx_length_errors; 1007 } 1008 } 1009 1010 /** 1011 * i40e_update_pf_stats - Update the PF statistics counters. 1012 * @pf: the PF to be updated 1013 **/ 1014 static void i40e_update_pf_stats(struct i40e_pf *pf) 1015 { 1016 struct i40e_hw_port_stats *osd = &pf->stats_offsets; 1017 struct i40e_hw_port_stats *nsd = &pf->stats; 1018 struct i40e_hw *hw = &pf->hw; 1019 u32 val; 1020 int i; 1021 1022 i40e_stat_update48(hw, I40E_GLPRT_GORCH(hw->port), 1023 I40E_GLPRT_GORCL(hw->port), 1024 pf->stat_offsets_loaded, 1025 &osd->eth.rx_bytes, &nsd->eth.rx_bytes); 1026 i40e_stat_update48(hw, I40E_GLPRT_GOTCH(hw->port), 1027 I40E_GLPRT_GOTCL(hw->port), 1028 pf->stat_offsets_loaded, 1029 &osd->eth.tx_bytes, &nsd->eth.tx_bytes); 1030 i40e_stat_update32(hw, I40E_GLPRT_RDPC(hw->port), 1031 pf->stat_offsets_loaded, 1032 &osd->eth.rx_discards, 1033 &nsd->eth.rx_discards); 1034 i40e_stat_update48(hw, I40E_GLPRT_UPRCH(hw->port), 1035 I40E_GLPRT_UPRCL(hw->port), 1036 pf->stat_offsets_loaded, 1037 &osd->eth.rx_unicast, 1038 &nsd->eth.rx_unicast); 1039 i40e_stat_update48(hw, I40E_GLPRT_MPRCH(hw->port), 1040 I40E_GLPRT_MPRCL(hw->port), 1041 pf->stat_offsets_loaded, 1042 &osd->eth.rx_multicast, 1043 &nsd->eth.rx_multicast); 1044 i40e_stat_update48(hw, I40E_GLPRT_BPRCH(hw->port), 1045 I40E_GLPRT_BPRCL(hw->port), 1046 pf->stat_offsets_loaded, 1047 &osd->eth.rx_broadcast, 1048 &nsd->eth.rx_broadcast); 1049 i40e_stat_update48(hw, I40E_GLPRT_UPTCH(hw->port), 1050 I40E_GLPRT_UPTCL(hw->port), 1051 pf->stat_offsets_loaded, 1052 &osd->eth.tx_unicast, 1053 &nsd->eth.tx_unicast); 1054 i40e_stat_update48(hw, I40E_GLPRT_MPTCH(hw->port), 1055 I40E_GLPRT_MPTCL(hw->port), 1056 pf->stat_offsets_loaded, 1057 &osd->eth.tx_multicast, 1058 &nsd->eth.tx_multicast); 1059 i40e_stat_update48(hw, I40E_GLPRT_BPTCH(hw->port), 1060 I40E_GLPRT_BPTCL(hw->port), 1061 pf->stat_offsets_loaded, 1062 &osd->eth.tx_broadcast, 1063 &nsd->eth.tx_broadcast); 1064 1065 i40e_stat_update32(hw, I40E_GLPRT_TDOLD(hw->port), 1066 pf->stat_offsets_loaded, 1067 &osd->tx_dropped_link_down, 1068 &nsd->tx_dropped_link_down); 1069 1070 i40e_stat_update32(hw, I40E_GLPRT_CRCERRS(hw->port), 1071 pf->stat_offsets_loaded, 1072 &osd->crc_errors, &nsd->crc_errors); 1073 1074 i40e_stat_update32(hw, I40E_GLPRT_ILLERRC(hw->port), 1075 pf->stat_offsets_loaded, 1076 &osd->illegal_bytes, &nsd->illegal_bytes); 1077 1078 i40e_stat_update32(hw, I40E_GLPRT_MLFC(hw->port), 1079 pf->stat_offsets_loaded, 1080 &osd->mac_local_faults, 1081 &nsd->mac_local_faults); 1082 i40e_stat_update32(hw, I40E_GLPRT_MRFC(hw->port), 1083 pf->stat_offsets_loaded, 1084 &osd->mac_remote_faults, 1085 &nsd->mac_remote_faults); 1086 1087 i40e_stat_update32(hw, I40E_GLPRT_RLEC(hw->port), 1088 pf->stat_offsets_loaded, 1089 &osd->rx_length_errors, 1090 &nsd->rx_length_errors); 1091 1092 i40e_stat_update32(hw, I40E_GLPRT_LXONRXC(hw->port), 1093 pf->stat_offsets_loaded, 1094 &osd->link_xon_rx, &nsd->link_xon_rx); 1095 i40e_stat_update32(hw, I40E_GLPRT_LXONTXC(hw->port), 1096 pf->stat_offsets_loaded, 1097 &osd->link_xon_tx, &nsd->link_xon_tx); 1098 i40e_stat_update32(hw, I40E_GLPRT_LXOFFRXC(hw->port), 1099 pf->stat_offsets_loaded, 1100 &osd->link_xoff_rx, &nsd->link_xoff_rx); 1101 i40e_stat_update32(hw, I40E_GLPRT_LXOFFTXC(hw->port), 1102 pf->stat_offsets_loaded, 1103 &osd->link_xoff_tx, &nsd->link_xoff_tx); 1104 1105 for (i = 0; i < 8; i++) { 1106 i40e_stat_update32(hw, I40E_GLPRT_PXOFFRXC(hw->port, i), 1107 pf->stat_offsets_loaded, 1108 &osd->priority_xoff_rx[i], 1109 &nsd->priority_xoff_rx[i]); 1110 i40e_stat_update32(hw, I40E_GLPRT_PXONRXC(hw->port, i), 1111 pf->stat_offsets_loaded, 1112 &osd->priority_xon_rx[i], 1113 &nsd->priority_xon_rx[i]); 1114 i40e_stat_update32(hw, I40E_GLPRT_PXONTXC(hw->port, i), 1115 pf->stat_offsets_loaded, 1116 &osd->priority_xon_tx[i], 1117 &nsd->priority_xon_tx[i]); 1118 i40e_stat_update32(hw, I40E_GLPRT_PXOFFTXC(hw->port, i), 1119 pf->stat_offsets_loaded, 1120 &osd->priority_xoff_tx[i], 1121 &nsd->priority_xoff_tx[i]); 1122 i40e_stat_update32(hw, 1123 I40E_GLPRT_RXON2OFFCNT(hw->port, i), 1124 pf->stat_offsets_loaded, 1125 &osd->priority_xon_2_xoff[i], 1126 &nsd->priority_xon_2_xoff[i]); 1127 } 1128 1129 i40e_stat_update48(hw, I40E_GLPRT_PRC64H(hw->port), 1130 I40E_GLPRT_PRC64L(hw->port), 1131 pf->stat_offsets_loaded, 1132 &osd->rx_size_64, &nsd->rx_size_64); 1133 i40e_stat_update48(hw, I40E_GLPRT_PRC127H(hw->port), 1134 I40E_GLPRT_PRC127L(hw->port), 1135 pf->stat_offsets_loaded, 1136 &osd->rx_size_127, &nsd->rx_size_127); 1137 i40e_stat_update48(hw, I40E_GLPRT_PRC255H(hw->port), 1138 I40E_GLPRT_PRC255L(hw->port), 1139 pf->stat_offsets_loaded, 1140 &osd->rx_size_255, &nsd->rx_size_255); 1141 i40e_stat_update48(hw, I40E_GLPRT_PRC511H(hw->port), 1142 I40E_GLPRT_PRC511L(hw->port), 1143 pf->stat_offsets_loaded, 1144 &osd->rx_size_511, &nsd->rx_size_511); 1145 i40e_stat_update48(hw, I40E_GLPRT_PRC1023H(hw->port), 1146 I40E_GLPRT_PRC1023L(hw->port), 1147 pf->stat_offsets_loaded, 1148 &osd->rx_size_1023, &nsd->rx_size_1023); 1149 i40e_stat_update48(hw, I40E_GLPRT_PRC1522H(hw->port), 1150 I40E_GLPRT_PRC1522L(hw->port), 1151 pf->stat_offsets_loaded, 1152 &osd->rx_size_1522, &nsd->rx_size_1522); 1153 i40e_stat_update48(hw, I40E_GLPRT_PRC9522H(hw->port), 1154 I40E_GLPRT_PRC9522L(hw->port), 1155 pf->stat_offsets_loaded, 1156 &osd->rx_size_big, &nsd->rx_size_big); 1157 1158 i40e_stat_update48(hw, I40E_GLPRT_PTC64H(hw->port), 1159 I40E_GLPRT_PTC64L(hw->port), 1160 pf->stat_offsets_loaded, 1161 &osd->tx_size_64, &nsd->tx_size_64); 1162 i40e_stat_update48(hw, I40E_GLPRT_PTC127H(hw->port), 1163 I40E_GLPRT_PTC127L(hw->port), 1164 pf->stat_offsets_loaded, 1165 &osd->tx_size_127, &nsd->tx_size_127); 1166 i40e_stat_update48(hw, I40E_GLPRT_PTC255H(hw->port), 1167 I40E_GLPRT_PTC255L(hw->port), 1168 pf->stat_offsets_loaded, 1169 &osd->tx_size_255, &nsd->tx_size_255); 1170 i40e_stat_update48(hw, I40E_GLPRT_PTC511H(hw->port), 1171 I40E_GLPRT_PTC511L(hw->port), 1172 pf->stat_offsets_loaded, 1173 &osd->tx_size_511, &nsd->tx_size_511); 1174 i40e_stat_update48(hw, I40E_GLPRT_PTC1023H(hw->port), 1175 I40E_GLPRT_PTC1023L(hw->port), 1176 pf->stat_offsets_loaded, 1177 &osd->tx_size_1023, &nsd->tx_size_1023); 1178 i40e_stat_update48(hw, I40E_GLPRT_PTC1522H(hw->port), 1179 I40E_GLPRT_PTC1522L(hw->port), 1180 pf->stat_offsets_loaded, 1181 &osd->tx_size_1522, &nsd->tx_size_1522); 1182 i40e_stat_update48(hw, I40E_GLPRT_PTC9522H(hw->port), 1183 I40E_GLPRT_PTC9522L(hw->port), 1184 pf->stat_offsets_loaded, 1185 &osd->tx_size_big, &nsd->tx_size_big); 1186 1187 i40e_stat_update32(hw, I40E_GLPRT_RUC(hw->port), 1188 pf->stat_offsets_loaded, 1189 &osd->rx_undersize, &nsd->rx_undersize); 1190 i40e_stat_update32(hw, I40E_GLPRT_RFC(hw->port), 1191 pf->stat_offsets_loaded, 1192 &osd->rx_fragments, &nsd->rx_fragments); 1193 i40e_stat_update32(hw, I40E_GLPRT_ROC(hw->port), 1194 pf->stat_offsets_loaded, 1195 &osd->rx_oversize, &nsd->rx_oversize); 1196 i40e_stat_update32(hw, I40E_GLPRT_RJC(hw->port), 1197 pf->stat_offsets_loaded, 1198 &osd->rx_jabber, &nsd->rx_jabber); 1199 1200 /* FDIR stats */ 1201 i40e_stat_update_and_clear32(hw, 1202 I40E_GLQF_PCNT(I40E_FD_ATR_STAT_IDX(hw->pf_id)), 1203 &nsd->fd_atr_match); 1204 i40e_stat_update_and_clear32(hw, 1205 I40E_GLQF_PCNT(I40E_FD_SB_STAT_IDX(hw->pf_id)), 1206 &nsd->fd_sb_match); 1207 i40e_stat_update_and_clear32(hw, 1208 I40E_GLQF_PCNT(I40E_FD_ATR_TUNNEL_STAT_IDX(hw->pf_id)), 1209 &nsd->fd_atr_tunnel_match); 1210 1211 val = rd32(hw, I40E_PRTPM_EEE_STAT); 1212 nsd->tx_lpi_status = 1213 FIELD_GET(I40E_PRTPM_EEE_STAT_TX_LPI_STATUS_MASK, val); 1214 nsd->rx_lpi_status = 1215 FIELD_GET(I40E_PRTPM_EEE_STAT_RX_LPI_STATUS_MASK, val); 1216 i40e_stat_update32(hw, I40E_PRTPM_TLPIC, 1217 pf->stat_offsets_loaded, 1218 &osd->tx_lpi_count, &nsd->tx_lpi_count); 1219 i40e_stat_update32(hw, I40E_PRTPM_RLPIC, 1220 pf->stat_offsets_loaded, 1221 &osd->rx_lpi_count, &nsd->rx_lpi_count); 1222 1223 if (test_bit(I40E_FLAG_FD_SB_ENA, pf->flags) && 1224 !test_bit(__I40E_FD_SB_AUTO_DISABLED, pf->state)) 1225 nsd->fd_sb_status = true; 1226 else 1227 nsd->fd_sb_status = false; 1228 1229 if (test_bit(I40E_FLAG_FD_ATR_ENA, pf->flags) && 1230 !test_bit(__I40E_FD_ATR_AUTO_DISABLED, pf->state)) 1231 nsd->fd_atr_status = true; 1232 else 1233 nsd->fd_atr_status = false; 1234 1235 pf->stat_offsets_loaded = true; 1236 } 1237 1238 /** 1239 * i40e_update_stats - Update the various statistics counters. 1240 * @vsi: the VSI to be updated 1241 * 1242 * Update the various stats for this VSI and its related entities. 1243 **/ 1244 void i40e_update_stats(struct i40e_vsi *vsi) 1245 { 1246 struct i40e_pf *pf = vsi->back; 1247 1248 if (vsi->type == I40E_VSI_MAIN) 1249 i40e_update_pf_stats(pf); 1250 1251 i40e_update_vsi_stats(vsi); 1252 } 1253 1254 /** 1255 * i40e_count_all_filters - counts VSI MAC filters 1256 * @vsi: the VSI to be searched 1257 * 1258 * Return: count of MAC filters in any state. 1259 */ 1260 int i40e_count_all_filters(struct i40e_vsi *vsi) 1261 { 1262 struct i40e_mac_filter *f; 1263 struct hlist_node *h; 1264 int bkt, cnt = 0; 1265 1266 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) 1267 cnt++; 1268 1269 return cnt; 1270 } 1271 1272 /** 1273 * i40e_count_active_filters - counts VSI MAC filters 1274 * @vsi: the VSI to be searched 1275 * 1276 * Return: count of active MAC filters. 1277 */ 1278 int i40e_count_active_filters(struct i40e_vsi *vsi) 1279 { 1280 struct i40e_mac_filter *f; 1281 struct hlist_node *h; 1282 int bkt; 1283 int cnt = 0; 1284 1285 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) { 1286 if (f->state == I40E_FILTER_NEW || 1287 f->state == I40E_FILTER_NEW_SYNC || 1288 f->state == I40E_FILTER_ACTIVE) 1289 ++cnt; 1290 } 1291 1292 return cnt; 1293 } 1294 1295 /** 1296 * i40e_find_filter - Search VSI filter list for specific mac/vlan filter 1297 * @vsi: the VSI to be searched 1298 * @macaddr: the MAC address 1299 * @vlan: the vlan 1300 * 1301 * Returns ptr to the filter object or NULL 1302 **/ 1303 static struct i40e_mac_filter *i40e_find_filter(struct i40e_vsi *vsi, 1304 const u8 *macaddr, s16 vlan) 1305 { 1306 struct i40e_mac_filter *f; 1307 u64 key; 1308 1309 if (!vsi || !macaddr) 1310 return NULL; 1311 1312 key = i40e_addr_to_hkey(macaddr); 1313 hash_for_each_possible(vsi->mac_filter_hash, f, hlist, key) { 1314 if ((ether_addr_equal(macaddr, f->macaddr)) && 1315 (vlan == f->vlan)) 1316 return f; 1317 } 1318 return NULL; 1319 } 1320 1321 /** 1322 * i40e_find_mac - Find a mac addr in the macvlan filters list 1323 * @vsi: the VSI to be searched 1324 * @macaddr: the MAC address we are searching for 1325 * 1326 * Returns the first filter with the provided MAC address or NULL if 1327 * MAC address was not found 1328 **/ 1329 struct i40e_mac_filter *i40e_find_mac(struct i40e_vsi *vsi, const u8 *macaddr) 1330 { 1331 struct i40e_mac_filter *f; 1332 u64 key; 1333 1334 if (!vsi || !macaddr) 1335 return NULL; 1336 1337 key = i40e_addr_to_hkey(macaddr); 1338 hash_for_each_possible(vsi->mac_filter_hash, f, hlist, key) { 1339 if ((ether_addr_equal(macaddr, f->macaddr))) 1340 return f; 1341 } 1342 return NULL; 1343 } 1344 1345 /** 1346 * i40e_is_vsi_in_vlan - Check if VSI is in vlan mode 1347 * @vsi: the VSI to be searched 1348 * 1349 * Returns true if VSI is in vlan mode or false otherwise 1350 **/ 1351 bool i40e_is_vsi_in_vlan(struct i40e_vsi *vsi) 1352 { 1353 /* If we have a PVID, always operate in VLAN mode */ 1354 if (vsi->info.pvid) 1355 return true; 1356 1357 /* We need to operate in VLAN mode whenever we have any filters with 1358 * a VLAN other than I40E_VLAN_ALL. We could check the table each 1359 * time, incurring search cost repeatedly. However, we can notice two 1360 * things: 1361 * 1362 * 1) the only place where we can gain a VLAN filter is in 1363 * i40e_add_filter. 1364 * 1365 * 2) the only place where filters are actually removed is in 1366 * i40e_sync_filters_subtask. 1367 * 1368 * Thus, we can simply use a boolean value, has_vlan_filters which we 1369 * will set to true when we add a VLAN filter in i40e_add_filter. Then 1370 * we have to perform the full search after deleting filters in 1371 * i40e_sync_filters_subtask, but we already have to search 1372 * filters here and can perform the check at the same time. This 1373 * results in avoiding embedding a loop for VLAN mode inside another 1374 * loop over all the filters, and should maintain correctness as noted 1375 * above. 1376 */ 1377 return vsi->has_vlan_filter; 1378 } 1379 1380 /** 1381 * i40e_correct_mac_vlan_filters - Correct non-VLAN filters if necessary 1382 * @vsi: the VSI to configure 1383 * @tmp_add_list: list of filters ready to be added 1384 * @tmp_del_list: list of filters ready to be deleted 1385 * @vlan_filters: the number of active VLAN filters 1386 * 1387 * Update VLAN=0 and VLAN=-1 (I40E_VLAN_ANY) filters properly so that they 1388 * behave as expected. If we have any active VLAN filters remaining or about 1389 * to be added then we need to update non-VLAN filters to be marked as VLAN=0 1390 * so that they only match against untagged traffic. If we no longer have any 1391 * active VLAN filters, we need to make all non-VLAN filters marked as VLAN=-1 1392 * so that they match against both tagged and untagged traffic. In this way, 1393 * we ensure that we correctly receive the desired traffic. This ensures that 1394 * when we have an active VLAN we will receive only untagged traffic and 1395 * traffic matching active VLANs. If we have no active VLANs then we will 1396 * operate in non-VLAN mode and receive all traffic, tagged or untagged. 1397 * 1398 * Finally, in a similar fashion, this function also corrects filters when 1399 * there is an active PVID assigned to this VSI. 1400 * 1401 * In case of memory allocation failure return -ENOMEM. Otherwise, return 0. 1402 * 1403 * This function is only expected to be called from within 1404 * i40e_sync_vsi_filters. 1405 * 1406 * NOTE: This function expects to be called while under the 1407 * mac_filter_hash_lock 1408 */ 1409 static int i40e_correct_mac_vlan_filters(struct i40e_vsi *vsi, 1410 struct hlist_head *tmp_add_list, 1411 struct hlist_head *tmp_del_list, 1412 int vlan_filters) 1413 { 1414 s16 pvid = le16_to_cpu(vsi->info.pvid); 1415 struct i40e_mac_filter *f, *add_head; 1416 struct i40e_new_mac_filter *new; 1417 struct hlist_node *h; 1418 int bkt, new_vlan; 1419 1420 /* To determine if a particular filter needs to be replaced we 1421 * have the three following conditions: 1422 * 1423 * a) if we have a PVID assigned, then all filters which are 1424 * not marked as VLAN=PVID must be replaced with filters that 1425 * are. 1426 * b) otherwise, if we have any active VLANS, all filters 1427 * which are marked as VLAN=-1 must be replaced with 1428 * filters marked as VLAN=0 1429 * c) finally, if we do not have any active VLANS, all filters 1430 * which are marked as VLAN=0 must be replaced with filters 1431 * marked as VLAN=-1 1432 */ 1433 1434 /* Update the filters about to be added in place */ 1435 hlist_for_each_entry(new, tmp_add_list, hlist) { 1436 if (pvid && new->f->vlan != pvid) 1437 new->f->vlan = pvid; 1438 else if (vlan_filters && new->f->vlan == I40E_VLAN_ANY) 1439 new->f->vlan = 0; 1440 else if (!vlan_filters && new->f->vlan == 0) 1441 new->f->vlan = I40E_VLAN_ANY; 1442 } 1443 1444 /* Update the remaining active filters */ 1445 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) { 1446 /* Combine the checks for whether a filter needs to be changed 1447 * and then determine the new VLAN inside the if block, in 1448 * order to avoid duplicating code for adding the new filter 1449 * then deleting the old filter. 1450 */ 1451 if ((pvid && f->vlan != pvid) || 1452 (vlan_filters && f->vlan == I40E_VLAN_ANY) || 1453 (!vlan_filters && f->vlan == 0)) { 1454 /* Determine the new vlan we will be adding */ 1455 if (pvid) 1456 new_vlan = pvid; 1457 else if (vlan_filters) 1458 new_vlan = 0; 1459 else 1460 new_vlan = I40E_VLAN_ANY; 1461 1462 /* Create the new filter */ 1463 add_head = i40e_add_filter(vsi, f->macaddr, new_vlan); 1464 if (!add_head) 1465 return -ENOMEM; 1466 1467 /* Create a temporary i40e_new_mac_filter */ 1468 new = kzalloc_obj(*new, GFP_ATOMIC); 1469 if (!new) 1470 return -ENOMEM; 1471 1472 new->f = add_head; 1473 new->state = add_head->state; 1474 if (add_head->state == I40E_FILTER_NEW) 1475 add_head->state = I40E_FILTER_NEW_SYNC; 1476 1477 /* Add the new filter to the tmp list */ 1478 hlist_add_head(&new->hlist, tmp_add_list); 1479 1480 /* Put the original filter into the delete list */ 1481 f->state = I40E_FILTER_REMOVE; 1482 hash_del(&f->hlist); 1483 hlist_add_head(&f->hlist, tmp_del_list); 1484 } 1485 } 1486 1487 vsi->has_vlan_filter = !!vlan_filters; 1488 1489 return 0; 1490 } 1491 1492 /** 1493 * i40e_get_vf_new_vlan - Get new vlan id on a vf 1494 * @vsi: the vsi to configure 1495 * @new_mac: new mac filter to be added 1496 * @f: existing mac filter, replaced with new_mac->f if new_mac is not NULL 1497 * @vlan_filters: the number of active VLAN filters 1498 * @trusted: flag if the VF is trusted 1499 * 1500 * Get new VLAN id based on current VLAN filters, trust, PVID 1501 * and vf-vlan-prune-disable flag. 1502 * 1503 * Returns the value of the new vlan filter or 1504 * the old value if no new filter is needed. 1505 */ 1506 static s16 i40e_get_vf_new_vlan(struct i40e_vsi *vsi, 1507 struct i40e_new_mac_filter *new_mac, 1508 struct i40e_mac_filter *f, 1509 int vlan_filters, 1510 bool trusted) 1511 { 1512 s16 pvid = le16_to_cpu(vsi->info.pvid); 1513 struct i40e_pf *pf = vsi->back; 1514 bool is_any; 1515 1516 if (new_mac) 1517 f = new_mac->f; 1518 1519 if (pvid && f->vlan != pvid) 1520 return pvid; 1521 1522 is_any = (trusted || 1523 !test_bit(I40E_FLAG_VF_VLAN_PRUNING_ENA, pf->flags)); 1524 1525 if ((vlan_filters && f->vlan == I40E_VLAN_ANY) || 1526 (!is_any && !vlan_filters && f->vlan == I40E_VLAN_ANY) || 1527 (is_any && !vlan_filters && f->vlan == 0)) { 1528 if (is_any) 1529 return I40E_VLAN_ANY; 1530 else 1531 return 0; 1532 } 1533 1534 return f->vlan; 1535 } 1536 1537 /** 1538 * i40e_correct_vf_mac_vlan_filters - Correct non-VLAN VF filters if necessary 1539 * @vsi: the vsi to configure 1540 * @tmp_add_list: list of filters ready to be added 1541 * @tmp_del_list: list of filters ready to be deleted 1542 * @vlan_filters: the number of active VLAN filters 1543 * @trusted: flag if the VF is trusted 1544 * 1545 * Correct VF VLAN filters based on current VLAN filters, trust, PVID 1546 * and vf-vlan-prune-disable flag. 1547 * 1548 * In case of memory allocation failure return -ENOMEM. Otherwise, return 0. 1549 * 1550 * This function is only expected to be called from within 1551 * i40e_sync_vsi_filters. 1552 * 1553 * NOTE: This function expects to be called while under the 1554 * mac_filter_hash_lock 1555 */ 1556 static int i40e_correct_vf_mac_vlan_filters(struct i40e_vsi *vsi, 1557 struct hlist_head *tmp_add_list, 1558 struct hlist_head *tmp_del_list, 1559 int vlan_filters, 1560 bool trusted) 1561 { 1562 struct i40e_mac_filter *f, *add_head; 1563 struct i40e_new_mac_filter *new_mac; 1564 struct hlist_node *h; 1565 int bkt, new_vlan; 1566 1567 hlist_for_each_entry(new_mac, tmp_add_list, hlist) { 1568 new_mac->f->vlan = i40e_get_vf_new_vlan(vsi, new_mac, NULL, 1569 vlan_filters, trusted); 1570 } 1571 1572 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) { 1573 new_vlan = i40e_get_vf_new_vlan(vsi, NULL, f, vlan_filters, 1574 trusted); 1575 if (new_vlan != f->vlan) { 1576 add_head = i40e_add_filter(vsi, f->macaddr, new_vlan); 1577 if (!add_head) 1578 return -ENOMEM; 1579 /* Create a temporary i40e_new_mac_filter */ 1580 new_mac = kzalloc_obj(*new_mac, GFP_ATOMIC); 1581 if (!new_mac) 1582 return -ENOMEM; 1583 new_mac->f = add_head; 1584 new_mac->state = add_head->state; 1585 if (add_head->state == I40E_FILTER_NEW) 1586 add_head->state = I40E_FILTER_NEW_SYNC; 1587 1588 /* Add the new filter to the tmp list */ 1589 hlist_add_head(&new_mac->hlist, tmp_add_list); 1590 1591 /* Put the original filter into the delete list */ 1592 f->state = I40E_FILTER_REMOVE; 1593 hash_del(&f->hlist); 1594 hlist_add_head(&f->hlist, tmp_del_list); 1595 } 1596 } 1597 1598 vsi->has_vlan_filter = !!vlan_filters; 1599 return 0; 1600 } 1601 1602 /** 1603 * i40e_rm_default_mac_filter - Remove the default MAC filter set by NVM 1604 * @vsi: the PF Main VSI - inappropriate for any other VSI 1605 * @macaddr: the MAC address 1606 * 1607 * Remove whatever filter the firmware set up so the driver can manage 1608 * its own filtering intelligently. 1609 **/ 1610 static void i40e_rm_default_mac_filter(struct i40e_vsi *vsi, u8 *macaddr) 1611 { 1612 struct i40e_aqc_remove_macvlan_element_data element; 1613 struct i40e_pf *pf = vsi->back; 1614 1615 /* Only appropriate for the PF main VSI */ 1616 if (vsi->type != I40E_VSI_MAIN) 1617 return; 1618 1619 memset(&element, 0, sizeof(element)); 1620 ether_addr_copy(element.mac_addr, macaddr); 1621 element.vlan_tag = 0; 1622 /* Ignore error returns, some firmware does it this way... */ 1623 element.flags = I40E_AQC_MACVLAN_DEL_PERFECT_MATCH; 1624 i40e_aq_remove_macvlan(&pf->hw, vsi->seid, &element, 1, NULL); 1625 1626 memset(&element, 0, sizeof(element)); 1627 ether_addr_copy(element.mac_addr, macaddr); 1628 element.vlan_tag = 0; 1629 /* ...and some firmware does it this way. */ 1630 element.flags = I40E_AQC_MACVLAN_DEL_PERFECT_MATCH | 1631 I40E_AQC_MACVLAN_DEL_IGNORE_VLAN; 1632 i40e_aq_remove_macvlan(&pf->hw, vsi->seid, &element, 1, NULL); 1633 } 1634 1635 /** 1636 * i40e_add_filter - Add a mac/vlan filter to the VSI 1637 * @vsi: the VSI to be searched 1638 * @macaddr: the MAC address 1639 * @vlan: the vlan 1640 * 1641 * Returns ptr to the filter object or NULL when no memory available. 1642 * 1643 * NOTE: This function is expected to be called with mac_filter_hash_lock 1644 * being held. 1645 **/ 1646 struct i40e_mac_filter *i40e_add_filter(struct i40e_vsi *vsi, 1647 const u8 *macaddr, s16 vlan) 1648 { 1649 struct i40e_mac_filter *f; 1650 u64 key; 1651 1652 if (!vsi || !macaddr) 1653 return NULL; 1654 1655 f = i40e_find_filter(vsi, macaddr, vlan); 1656 if (!f) { 1657 f = kzalloc_obj(*f, GFP_ATOMIC); 1658 if (!f) 1659 return NULL; 1660 1661 /* Update the boolean indicating if we need to function in 1662 * VLAN mode. 1663 */ 1664 if (vlan >= 0) 1665 vsi->has_vlan_filter = true; 1666 1667 ether_addr_copy(f->macaddr, macaddr); 1668 f->vlan = vlan; 1669 f->state = I40E_FILTER_NEW; 1670 INIT_HLIST_NODE(&f->hlist); 1671 1672 key = i40e_addr_to_hkey(macaddr); 1673 hash_add(vsi->mac_filter_hash, &f->hlist, key); 1674 1675 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED; 1676 set_bit(__I40E_MACVLAN_SYNC_PENDING, vsi->back->state); 1677 } 1678 1679 /* If we're asked to add a filter that has been marked for removal, it 1680 * is safe to simply restore it to active state. __i40e_del_filter 1681 * will have simply deleted any filters which were previously marked 1682 * NEW or FAILED, so if it is currently marked REMOVE it must have 1683 * previously been ACTIVE. Since we haven't yet run the sync filters 1684 * task, just restore this filter to the ACTIVE state so that the 1685 * sync task leaves it in place 1686 */ 1687 if (f->state == I40E_FILTER_REMOVE) 1688 f->state = I40E_FILTER_ACTIVE; 1689 1690 return f; 1691 } 1692 1693 /** 1694 * __i40e_del_filter - Remove a specific filter from the VSI 1695 * @vsi: VSI to remove from 1696 * @f: the filter to remove from the list 1697 * 1698 * This function requires you've found * the exact filter you will remove 1699 * already, such as via i40e_find_filter or i40e_find_mac. 1700 * 1701 * NOTE: This function is expected to be called with mac_filter_hash_lock 1702 * being held. 1703 * ANOTHER NOTE: This function MUST be called from within the context of 1704 * the "safe" variants of any list iterators, e.g. list_for_each_entry_safe() 1705 * instead of list_for_each_entry(). 1706 **/ 1707 void __i40e_del_filter(struct i40e_vsi *vsi, struct i40e_mac_filter *f) 1708 { 1709 if (!f) 1710 return; 1711 1712 /* If the filter was never added to firmware then we can just delete it 1713 * directly and we don't want to set the status to remove or else an 1714 * admin queue command will unnecessarily fire. 1715 */ 1716 if ((f->state == I40E_FILTER_FAILED) || 1717 (f->state == I40E_FILTER_NEW)) { 1718 hash_del(&f->hlist); 1719 kfree(f); 1720 } else { 1721 f->state = I40E_FILTER_REMOVE; 1722 } 1723 1724 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED; 1725 set_bit(__I40E_MACVLAN_SYNC_PENDING, vsi->back->state); 1726 } 1727 1728 /** 1729 * i40e_add_mac_filter - Add a MAC filter for all active VLANs 1730 * @vsi: the VSI to be searched 1731 * @macaddr: the mac address to be filtered 1732 * 1733 * If we're not in VLAN mode, just add the filter to I40E_VLAN_ANY. Otherwise, 1734 * go through all the macvlan filters and add a macvlan filter for each 1735 * unique vlan that already exists. If a PVID has been assigned, instead only 1736 * add the macaddr to that VLAN. 1737 * 1738 * Returns last filter added on success, else NULL 1739 **/ 1740 struct i40e_mac_filter *i40e_add_mac_filter(struct i40e_vsi *vsi, 1741 const u8 *macaddr) 1742 { 1743 struct i40e_mac_filter *f, *add = NULL; 1744 struct hlist_node *h; 1745 int bkt; 1746 1747 lockdep_assert_held(&vsi->mac_filter_hash_lock); 1748 if (vsi->info.pvid) 1749 return i40e_add_filter(vsi, macaddr, 1750 le16_to_cpu(vsi->info.pvid)); 1751 1752 if (!i40e_is_vsi_in_vlan(vsi)) 1753 return i40e_add_filter(vsi, macaddr, I40E_VLAN_ANY); 1754 1755 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) { 1756 if (f->state == I40E_FILTER_REMOVE) 1757 continue; 1758 add = i40e_add_filter(vsi, macaddr, f->vlan); 1759 if (!add) 1760 return NULL; 1761 } 1762 1763 return add; 1764 } 1765 1766 /** 1767 * i40e_del_mac_filter - Remove a MAC filter from all VLANs 1768 * @vsi: the VSI to be searched 1769 * @macaddr: the mac address to be removed 1770 * 1771 * Removes a given MAC address from a VSI regardless of what VLAN it has been 1772 * associated with. 1773 * 1774 * Returns 0 for success, or error 1775 **/ 1776 int i40e_del_mac_filter(struct i40e_vsi *vsi, const u8 *macaddr) 1777 { 1778 struct i40e_mac_filter *f; 1779 struct hlist_node *h; 1780 bool found = false; 1781 int bkt; 1782 1783 lockdep_assert_held(&vsi->mac_filter_hash_lock); 1784 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) { 1785 if (ether_addr_equal(macaddr, f->macaddr)) { 1786 __i40e_del_filter(vsi, f); 1787 found = true; 1788 } 1789 } 1790 1791 if (found) 1792 return 0; 1793 else 1794 return -ENOENT; 1795 } 1796 1797 /** 1798 * i40e_set_mac - NDO callback to set mac address 1799 * @netdev: network interface device structure 1800 * @p: pointer to an address structure 1801 * 1802 * Returns 0 on success, negative on failure 1803 **/ 1804 static int i40e_set_mac(struct net_device *netdev, void *p) 1805 { 1806 struct i40e_netdev_priv *np = netdev_priv(netdev); 1807 struct i40e_vsi *vsi = np->vsi; 1808 struct i40e_pf *pf = vsi->back; 1809 struct i40e_hw *hw = &pf->hw; 1810 struct sockaddr *addr = p; 1811 1812 if (!is_valid_ether_addr(addr->sa_data)) 1813 return -EADDRNOTAVAIL; 1814 1815 if (test_bit(__I40E_DOWN, pf->state) || 1816 test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state)) 1817 return -EADDRNOTAVAIL; 1818 1819 if (ether_addr_equal(hw->mac.addr, addr->sa_data)) 1820 netdev_info(netdev, "returning to hw mac address %pM\n", 1821 hw->mac.addr); 1822 else 1823 netdev_info(netdev, "set new mac address %pM\n", addr->sa_data); 1824 1825 /* Copy the address first, so that we avoid a possible race with 1826 * .set_rx_mode(). 1827 * - Remove old address from MAC filter 1828 * - Copy new address 1829 * - Add new address to MAC filter 1830 */ 1831 spin_lock_bh(&vsi->mac_filter_hash_lock); 1832 i40e_del_mac_filter(vsi, netdev->dev_addr); 1833 eth_hw_addr_set(netdev, addr->sa_data); 1834 i40e_add_mac_filter(vsi, netdev->dev_addr); 1835 spin_unlock_bh(&vsi->mac_filter_hash_lock); 1836 1837 if (vsi->type == I40E_VSI_MAIN) { 1838 int ret; 1839 1840 ret = i40e_aq_mac_address_write(hw, I40E_AQC_WRITE_TYPE_LAA_WOL, 1841 addr->sa_data, NULL); 1842 if (ret) 1843 netdev_info(netdev, "Ignoring error from firmware on LAA update, status %pe, AQ ret %s\n", 1844 ERR_PTR(ret), 1845 libie_aq_str(hw->aq.asq_last_status)); 1846 } 1847 1848 /* schedule our worker thread which will take care of 1849 * applying the new filter changes 1850 */ 1851 i40e_service_event_schedule(pf); 1852 return 0; 1853 } 1854 1855 /** 1856 * i40e_config_rss_aq - Prepare for RSS using AQ commands 1857 * @vsi: vsi structure 1858 * @seed: RSS hash seed 1859 * @lut: pointer to lookup table of lut_size 1860 * @lut_size: size of the lookup table 1861 **/ 1862 static int i40e_config_rss_aq(struct i40e_vsi *vsi, const u8 *seed, 1863 u8 *lut, u16 lut_size) 1864 { 1865 struct i40e_pf *pf = vsi->back; 1866 struct i40e_hw *hw = &pf->hw; 1867 int ret = 0; 1868 1869 if (seed) { 1870 struct i40e_aqc_get_set_rss_key_data *seed_dw = 1871 (struct i40e_aqc_get_set_rss_key_data *)seed; 1872 ret = i40e_aq_set_rss_key(hw, vsi->id, seed_dw); 1873 if (ret) { 1874 dev_info(&pf->pdev->dev, 1875 "Cannot set RSS key, err %pe aq_err %s\n", 1876 ERR_PTR(ret), 1877 libie_aq_str(hw->aq.asq_last_status)); 1878 return ret; 1879 } 1880 } 1881 if (lut) { 1882 bool pf_lut = vsi->type == I40E_VSI_MAIN; 1883 1884 ret = i40e_aq_set_rss_lut(hw, vsi->id, pf_lut, lut, lut_size); 1885 if (ret) { 1886 dev_info(&pf->pdev->dev, 1887 "Cannot set RSS lut, err %pe aq_err %s\n", 1888 ERR_PTR(ret), 1889 libie_aq_str(hw->aq.asq_last_status)); 1890 return ret; 1891 } 1892 } 1893 return ret; 1894 } 1895 1896 /** 1897 * i40e_vsi_config_rss - Prepare for VSI(VMDq) RSS if used 1898 * @vsi: VSI structure 1899 **/ 1900 static int i40e_vsi_config_rss(struct i40e_vsi *vsi) 1901 { 1902 struct i40e_pf *pf = vsi->back; 1903 u8 seed[I40E_HKEY_ARRAY_SIZE]; 1904 u8 *lut; 1905 int ret; 1906 1907 if (!test_bit(I40E_HW_CAP_RSS_AQ, pf->hw.caps)) 1908 return 0; 1909 if (!vsi->rss_size) 1910 vsi->rss_size = min_t(int, pf->alloc_rss_size, 1911 vsi->num_queue_pairs); 1912 if (!vsi->rss_size) 1913 return -EINVAL; 1914 lut = kzalloc(vsi->rss_table_size, GFP_KERNEL); 1915 if (!lut) 1916 return -ENOMEM; 1917 1918 /* Use the user configured hash keys and lookup table if there is one, 1919 * otherwise use default 1920 */ 1921 if (vsi->rss_lut_user) 1922 memcpy(lut, vsi->rss_lut_user, vsi->rss_table_size); 1923 else 1924 i40e_fill_rss_lut(pf, lut, vsi->rss_table_size, vsi->rss_size); 1925 if (vsi->rss_hkey_user) 1926 memcpy(seed, vsi->rss_hkey_user, I40E_HKEY_ARRAY_SIZE); 1927 else 1928 netdev_rss_key_fill((void *)seed, I40E_HKEY_ARRAY_SIZE); 1929 ret = i40e_config_rss_aq(vsi, seed, lut, vsi->rss_table_size); 1930 kfree(lut); 1931 return ret; 1932 } 1933 1934 /** 1935 * i40e_vsi_setup_queue_map_mqprio - Prepares mqprio based tc_config 1936 * @vsi: the VSI being configured, 1937 * @ctxt: VSI context structure 1938 * @enabled_tc: number of traffic classes to enable 1939 * 1940 * Prepares VSI tc_config to have queue configurations based on MQPRIO options. 1941 **/ 1942 static int i40e_vsi_setup_queue_map_mqprio(struct i40e_vsi *vsi, 1943 struct i40e_vsi_context *ctxt, 1944 u8 enabled_tc) 1945 { 1946 u16 qcount = 0, max_qcount, qmap, sections = 0; 1947 int i, override_q, pow, num_qps, ret; 1948 u8 netdev_tc = 0, offset = 0; 1949 1950 if (vsi->type != I40E_VSI_MAIN) 1951 return -EINVAL; 1952 sections = I40E_AQ_VSI_PROP_QUEUE_MAP_VALID; 1953 sections |= I40E_AQ_VSI_PROP_SCHED_VALID; 1954 vsi->tc_config.numtc = vsi->mqprio_qopt.qopt.num_tc; 1955 vsi->tc_config.enabled_tc = enabled_tc ? enabled_tc : 1; 1956 num_qps = vsi->mqprio_qopt.qopt.count[0]; 1957 1958 /* find the next higher power-of-2 of num queue pairs */ 1959 pow = ilog2(num_qps); 1960 if (!is_power_of_2(num_qps)) 1961 pow++; 1962 qmap = (offset << I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT) | 1963 (pow << I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT); 1964 1965 /* Setup queue offset/count for all TCs for given VSI */ 1966 max_qcount = vsi->mqprio_qopt.qopt.count[0]; 1967 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 1968 /* See if the given TC is enabled for the given VSI */ 1969 if (vsi->tc_config.enabled_tc & BIT(i)) { 1970 offset = vsi->mqprio_qopt.qopt.offset[i]; 1971 qcount = vsi->mqprio_qopt.qopt.count[i]; 1972 if (qcount > max_qcount) 1973 max_qcount = qcount; 1974 vsi->tc_config.tc_info[i].qoffset = offset; 1975 vsi->tc_config.tc_info[i].qcount = qcount; 1976 vsi->tc_config.tc_info[i].netdev_tc = netdev_tc++; 1977 } else { 1978 /* TC is not enabled so set the offset to 1979 * default queue and allocate one queue 1980 * for the given TC. 1981 */ 1982 vsi->tc_config.tc_info[i].qoffset = 0; 1983 vsi->tc_config.tc_info[i].qcount = 1; 1984 vsi->tc_config.tc_info[i].netdev_tc = 0; 1985 } 1986 } 1987 1988 /* Set actual Tx/Rx queue pairs */ 1989 vsi->num_queue_pairs = offset + qcount; 1990 1991 /* Setup queue TC[0].qmap for given VSI context */ 1992 ctxt->info.tc_mapping[0] = cpu_to_le16(qmap); 1993 ctxt->info.mapping_flags |= cpu_to_le16(I40E_AQ_VSI_QUE_MAP_CONTIG); 1994 ctxt->info.queue_mapping[0] = cpu_to_le16(vsi->base_queue); 1995 ctxt->info.valid_sections |= cpu_to_le16(sections); 1996 1997 /* Reconfigure RSS for main VSI with max queue count */ 1998 vsi->rss_size = max_qcount; 1999 ret = i40e_vsi_config_rss(vsi); 2000 if (ret) { 2001 dev_info(&vsi->back->pdev->dev, 2002 "Failed to reconfig rss for num_queues (%u)\n", 2003 max_qcount); 2004 return ret; 2005 } 2006 vsi->reconfig_rss = true; 2007 dev_dbg(&vsi->back->pdev->dev, 2008 "Reconfigured rss with num_queues (%u)\n", max_qcount); 2009 2010 /* Find queue count available for channel VSIs and starting offset 2011 * for channel VSIs 2012 */ 2013 override_q = vsi->mqprio_qopt.qopt.count[0]; 2014 if (override_q && override_q < vsi->num_queue_pairs) { 2015 vsi->cnt_q_avail = vsi->num_queue_pairs - override_q; 2016 vsi->next_base_queue = override_q; 2017 } 2018 return 0; 2019 } 2020 2021 /** 2022 * i40e_vsi_setup_queue_map - Setup a VSI queue map based on enabled_tc 2023 * @vsi: the VSI being setup 2024 * @ctxt: VSI context structure 2025 * @enabled_tc: Enabled TCs bitmap 2026 * @is_add: True if called before Add VSI 2027 * 2028 * Setup VSI queue mapping for enabled traffic classes. 2029 **/ 2030 static void i40e_vsi_setup_queue_map(struct i40e_vsi *vsi, 2031 struct i40e_vsi_context *ctxt, 2032 u8 enabled_tc, 2033 bool is_add) 2034 { 2035 struct i40e_pf *pf = vsi->back; 2036 u16 num_tc_qps = 0; 2037 u16 sections = 0; 2038 u8 netdev_tc = 0; 2039 u16 numtc = 1; 2040 u16 qcount; 2041 u8 offset; 2042 u16 qmap; 2043 int i; 2044 2045 sections = I40E_AQ_VSI_PROP_QUEUE_MAP_VALID; 2046 offset = 0; 2047 /* zero out queue mapping, it will get updated on the end of the function */ 2048 memset(ctxt->info.queue_mapping, 0, sizeof(ctxt->info.queue_mapping)); 2049 2050 if (vsi->type == I40E_VSI_MAIN) { 2051 /* This code helps add more queue to the VSI if we have 2052 * more cores than RSS can support, the higher cores will 2053 * be served by ATR or other filters. Furthermore, the 2054 * non-zero req_queue_pairs says that user requested a new 2055 * queue count via ethtool's set_channels, so use this 2056 * value for queues distribution across traffic classes 2057 * We need at least one queue pair for the interface 2058 * to be usable as we see in else statement. 2059 */ 2060 if (vsi->req_queue_pairs > 0) 2061 vsi->num_queue_pairs = vsi->req_queue_pairs; 2062 else if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags)) 2063 vsi->num_queue_pairs = pf->num_lan_msix; 2064 else 2065 vsi->num_queue_pairs = 1; 2066 } 2067 2068 /* Number of queues per enabled TC */ 2069 if (vsi->type == I40E_VSI_MAIN || 2070 (vsi->type == I40E_VSI_SRIOV && vsi->num_queue_pairs != 0)) 2071 num_tc_qps = vsi->num_queue_pairs; 2072 else 2073 num_tc_qps = vsi->alloc_queue_pairs; 2074 2075 if (enabled_tc && test_bit(I40E_FLAG_DCB_ENA, vsi->back->flags)) { 2076 /* Find numtc from enabled TC bitmap */ 2077 for (i = 0, numtc = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 2078 if (enabled_tc & BIT(i)) /* TC is enabled */ 2079 numtc++; 2080 } 2081 if (!numtc) { 2082 dev_warn(&pf->pdev->dev, "DCB is enabled but no TC enabled, forcing TC0\n"); 2083 numtc = 1; 2084 } 2085 num_tc_qps = num_tc_qps / numtc; 2086 num_tc_qps = min_t(int, num_tc_qps, 2087 i40e_pf_get_max_q_per_tc(pf)); 2088 } 2089 2090 vsi->tc_config.numtc = numtc; 2091 vsi->tc_config.enabled_tc = enabled_tc ? enabled_tc : 1; 2092 2093 /* Do not allow use more TC queue pairs than MSI-X vectors exist */ 2094 if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags)) 2095 num_tc_qps = min_t(int, num_tc_qps, pf->num_lan_msix); 2096 2097 /* Setup queue offset/count for all TCs for given VSI */ 2098 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 2099 /* See if the given TC is enabled for the given VSI */ 2100 if (vsi->tc_config.enabled_tc & BIT(i)) { 2101 /* TC is enabled */ 2102 int pow, num_qps; 2103 2104 switch (vsi->type) { 2105 case I40E_VSI_MAIN: 2106 if ((!test_bit(I40E_FLAG_FD_SB_ENA, 2107 pf->flags) && 2108 !test_bit(I40E_FLAG_FD_ATR_ENA, 2109 pf->flags)) || 2110 vsi->tc_config.enabled_tc != 1) { 2111 qcount = min_t(int, pf->alloc_rss_size, 2112 num_tc_qps); 2113 break; 2114 } 2115 fallthrough; 2116 case I40E_VSI_FDIR: 2117 case I40E_VSI_SRIOV: 2118 case I40E_VSI_VMDQ2: 2119 default: 2120 qcount = num_tc_qps; 2121 WARN_ON(i != 0); 2122 break; 2123 } 2124 vsi->tc_config.tc_info[i].qoffset = offset; 2125 vsi->tc_config.tc_info[i].qcount = qcount; 2126 2127 /* find the next higher power-of-2 of num queue pairs */ 2128 num_qps = qcount; 2129 pow = 0; 2130 while (num_qps && (BIT_ULL(pow) < qcount)) { 2131 pow++; 2132 num_qps >>= 1; 2133 } 2134 2135 vsi->tc_config.tc_info[i].netdev_tc = netdev_tc++; 2136 qmap = 2137 (offset << I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT) | 2138 (pow << I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT); 2139 2140 offset += qcount; 2141 } else { 2142 /* TC is not enabled so set the offset to 2143 * default queue and allocate one queue 2144 * for the given TC. 2145 */ 2146 vsi->tc_config.tc_info[i].qoffset = 0; 2147 vsi->tc_config.tc_info[i].qcount = 1; 2148 vsi->tc_config.tc_info[i].netdev_tc = 0; 2149 2150 qmap = 0; 2151 } 2152 ctxt->info.tc_mapping[i] = cpu_to_le16(qmap); 2153 } 2154 /* Do not change previously set num_queue_pairs for PFs and VFs*/ 2155 if ((vsi->type == I40E_VSI_MAIN && numtc != 1) || 2156 (vsi->type == I40E_VSI_SRIOV && vsi->num_queue_pairs == 0) || 2157 (vsi->type != I40E_VSI_MAIN && vsi->type != I40E_VSI_SRIOV)) 2158 vsi->num_queue_pairs = offset; 2159 2160 /* Scheduler section valid can only be set for ADD VSI */ 2161 if (is_add) { 2162 sections |= I40E_AQ_VSI_PROP_SCHED_VALID; 2163 2164 ctxt->info.up_enable_bits = enabled_tc; 2165 } 2166 if (vsi->type == I40E_VSI_SRIOV) { 2167 ctxt->info.mapping_flags |= 2168 cpu_to_le16(I40E_AQ_VSI_QUE_MAP_NONCONTIG); 2169 for (i = 0; i < vsi->num_queue_pairs; i++) 2170 ctxt->info.queue_mapping[i] = 2171 cpu_to_le16(vsi->base_queue + i); 2172 } else { 2173 ctxt->info.mapping_flags |= 2174 cpu_to_le16(I40E_AQ_VSI_QUE_MAP_CONTIG); 2175 ctxt->info.queue_mapping[0] = cpu_to_le16(vsi->base_queue); 2176 } 2177 ctxt->info.valid_sections |= cpu_to_le16(sections); 2178 } 2179 2180 /** 2181 * i40e_addr_sync - Callback for dev_(mc|uc)_sync to add address 2182 * @netdev: the netdevice 2183 * @addr: address to add 2184 * 2185 * Called by __dev_(mc|uc)_sync when an address needs to be added. We call 2186 * __dev_(uc|mc)_sync from .set_rx_mode and guarantee to hold the hash lock. 2187 */ 2188 static int i40e_addr_sync(struct net_device *netdev, const u8 *addr) 2189 { 2190 struct i40e_netdev_priv *np = netdev_priv(netdev); 2191 struct i40e_vsi *vsi = np->vsi; 2192 2193 if (i40e_add_mac_filter(vsi, addr)) 2194 return 0; 2195 else 2196 return -ENOMEM; 2197 } 2198 2199 /** 2200 * i40e_addr_unsync - Callback for dev_(mc|uc)_sync to remove address 2201 * @netdev: the netdevice 2202 * @addr: address to add 2203 * 2204 * Called by __dev_(mc|uc)_sync when an address needs to be removed. We call 2205 * __dev_(uc|mc)_sync from .set_rx_mode and guarantee to hold the hash lock. 2206 */ 2207 static int i40e_addr_unsync(struct net_device *netdev, const u8 *addr) 2208 { 2209 struct i40e_netdev_priv *np = netdev_priv(netdev); 2210 struct i40e_vsi *vsi = np->vsi; 2211 2212 /* Under some circumstances, we might receive a request to delete 2213 * our own device address from our uc list. Because we store the 2214 * device address in the VSI's MAC/VLAN filter list, we need to ignore 2215 * such requests and not delete our device address from this list. 2216 */ 2217 if (ether_addr_equal(addr, netdev->dev_addr)) 2218 return 0; 2219 2220 i40e_del_mac_filter(vsi, addr); 2221 2222 return 0; 2223 } 2224 2225 /** 2226 * i40e_set_rx_mode - NDO callback to set the netdev filters 2227 * @netdev: network interface device structure 2228 **/ 2229 static void i40e_set_rx_mode(struct net_device *netdev) 2230 { 2231 struct i40e_netdev_priv *np = netdev_priv(netdev); 2232 struct i40e_vsi *vsi = np->vsi; 2233 2234 spin_lock_bh(&vsi->mac_filter_hash_lock); 2235 2236 __dev_uc_sync(netdev, i40e_addr_sync, i40e_addr_unsync); 2237 __dev_mc_sync(netdev, i40e_addr_sync, i40e_addr_unsync); 2238 2239 spin_unlock_bh(&vsi->mac_filter_hash_lock); 2240 2241 /* check for other flag changes */ 2242 if (vsi->current_netdev_flags != vsi->netdev->flags) { 2243 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED; 2244 set_bit(__I40E_MACVLAN_SYNC_PENDING, vsi->back->state); 2245 } 2246 i40e_service_event_schedule(vsi->back); 2247 } 2248 2249 /** 2250 * i40e_undo_del_filter_entries - Undo the changes made to MAC filter entries 2251 * @vsi: Pointer to VSI struct 2252 * @from: Pointer to list which contains MAC filter entries - changes to 2253 * those entries needs to be undone. 2254 * 2255 * MAC filter entries from this list were slated for deletion. 2256 **/ 2257 static void i40e_undo_del_filter_entries(struct i40e_vsi *vsi, 2258 struct hlist_head *from) 2259 { 2260 struct i40e_mac_filter *f; 2261 struct hlist_node *h; 2262 2263 hlist_for_each_entry_safe(f, h, from, hlist) { 2264 u64 key = i40e_addr_to_hkey(f->macaddr); 2265 2266 /* Move the element back into MAC filter list*/ 2267 hlist_del(&f->hlist); 2268 hash_add(vsi->mac_filter_hash, &f->hlist, key); 2269 } 2270 } 2271 2272 /** 2273 * i40e_undo_add_filter_entries - Undo the changes made to MAC filter entries 2274 * @vsi: Pointer to vsi struct 2275 * @from: Pointer to list which contains MAC filter entries - changes to 2276 * those entries needs to be undone. 2277 * 2278 * MAC filter entries from this list were slated for addition. 2279 **/ 2280 static void i40e_undo_add_filter_entries(struct i40e_vsi *vsi, 2281 struct hlist_head *from) 2282 { 2283 struct i40e_new_mac_filter *new; 2284 struct hlist_node *h; 2285 2286 hlist_for_each_entry_safe(new, h, from, hlist) { 2287 /* We can simply free the wrapper structure */ 2288 hlist_del(&new->hlist); 2289 netdev_hw_addr_refcnt(new->f, vsi->netdev, -1); 2290 kfree(new); 2291 } 2292 } 2293 2294 /** 2295 * i40e_next_filter - Get the next non-broadcast filter from a list 2296 * @next: pointer to filter in list 2297 * 2298 * Returns the next non-broadcast filter in the list. Required so that we 2299 * ignore broadcast filters within the list, since these are not handled via 2300 * the normal firmware update path. 2301 */ 2302 static 2303 struct i40e_new_mac_filter *i40e_next_filter(struct i40e_new_mac_filter *next) 2304 { 2305 hlist_for_each_entry_continue(next, hlist) { 2306 if (!is_broadcast_ether_addr(next->f->macaddr)) 2307 return next; 2308 } 2309 2310 return NULL; 2311 } 2312 2313 /** 2314 * i40e_update_filter_state - Update filter state based on return data 2315 * from firmware 2316 * @count: Number of filters added 2317 * @add_list: return data from fw 2318 * @add_head: pointer to first filter in current batch 2319 * 2320 * MAC filter entries from list were slated to be added to device. Returns 2321 * number of successful filters. Note that 0 does NOT mean success! 2322 **/ 2323 static int 2324 i40e_update_filter_state(int count, 2325 struct i40e_aqc_add_macvlan_element_data *add_list, 2326 struct i40e_new_mac_filter *add_head) 2327 { 2328 int retval = 0; 2329 int i; 2330 2331 for (i = 0; i < count; i++) { 2332 /* Always check status of each filter. We don't need to check 2333 * the firmware return status because we pre-set the filter 2334 * status to I40E_AQC_MM_ERR_NO_RES when sending the filter 2335 * request to the adminq. Thus, if it no longer matches then 2336 * we know the filter is active. 2337 */ 2338 if (add_list[i].match_method == I40E_AQC_MM_ERR_NO_RES) { 2339 add_head->state = I40E_FILTER_FAILED; 2340 } else { 2341 add_head->state = I40E_FILTER_ACTIVE; 2342 retval++; 2343 } 2344 2345 add_head = i40e_next_filter(add_head); 2346 if (!add_head) 2347 break; 2348 } 2349 2350 return retval; 2351 } 2352 2353 /** 2354 * i40e_aqc_del_filters - Request firmware to delete a set of filters 2355 * @vsi: ptr to the VSI 2356 * @vsi_name: name to display in messages 2357 * @list: the list of filters to send to firmware 2358 * @num_del: the number of filters to delete 2359 * @retval: Set to -EIO on failure to delete 2360 * 2361 * Send a request to firmware via AdminQ to delete a set of filters. Uses 2362 * *retval instead of a return value so that success does not force ret_val to 2363 * be set to 0. This ensures that a sequence of calls to this function 2364 * preserve the previous value of *retval on successful delete. 2365 */ 2366 static 2367 void i40e_aqc_del_filters(struct i40e_vsi *vsi, const char *vsi_name, 2368 struct i40e_aqc_remove_macvlan_element_data *list, 2369 int num_del, int *retval) 2370 { 2371 struct i40e_hw *hw = &vsi->back->hw; 2372 enum libie_aq_err aq_status; 2373 int aq_ret; 2374 2375 aq_ret = i40e_aq_remove_macvlan_v2(hw, vsi->seid, list, num_del, NULL, 2376 &aq_status); 2377 2378 /* Explicitly ignore and do not report when firmware returns ENOENT */ 2379 if (aq_ret && !(aq_status == LIBIE_AQ_RC_ENOENT)) { 2380 *retval = -EIO; 2381 dev_info(&vsi->back->pdev->dev, 2382 "ignoring delete macvlan error on %s, err %pe, aq_err %s\n", 2383 vsi_name, ERR_PTR(aq_ret), libie_aq_str(aq_status)); 2384 } 2385 } 2386 2387 /** 2388 * i40e_aqc_add_filters - Request firmware to add a set of filters 2389 * @vsi: ptr to the VSI 2390 * @vsi_name: name to display in messages 2391 * @list: the list of filters to send to firmware 2392 * @add_head: Position in the add hlist 2393 * @num_add: the number of filters to add 2394 * 2395 * Send a request to firmware via AdminQ to add a chunk of filters. Will set 2396 * __I40E_VSI_OVERFLOW_PROMISC bit in vsi->state if the firmware has run out of 2397 * space for more filters. 2398 */ 2399 static 2400 void i40e_aqc_add_filters(struct i40e_vsi *vsi, const char *vsi_name, 2401 struct i40e_aqc_add_macvlan_element_data *list, 2402 struct i40e_new_mac_filter *add_head, 2403 int num_add) 2404 { 2405 struct i40e_hw *hw = &vsi->back->hw; 2406 enum libie_aq_err aq_status; 2407 int fcnt; 2408 2409 i40e_aq_add_macvlan_v2(hw, vsi->seid, list, num_add, NULL, &aq_status); 2410 fcnt = i40e_update_filter_state(num_add, list, add_head); 2411 2412 if (fcnt != num_add) { 2413 if (vsi->type == I40E_VSI_MAIN) { 2414 set_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state); 2415 dev_warn(&vsi->back->pdev->dev, 2416 "Error %s adding RX filters on %s, promiscuous mode forced on\n", 2417 libie_aq_str(aq_status), vsi_name); 2418 } else if (vsi->type == I40E_VSI_SRIOV || 2419 vsi->type == I40E_VSI_VMDQ1 || 2420 vsi->type == I40E_VSI_VMDQ2) { 2421 dev_warn(&vsi->back->pdev->dev, 2422 "Error %s adding RX filters on %s, please set promiscuous on manually for %s\n", 2423 libie_aq_str(aq_status), vsi_name, vsi_name); 2424 } else { 2425 dev_warn(&vsi->back->pdev->dev, 2426 "Error %s adding RX filters on %s, incorrect VSI type: %i.\n", 2427 libie_aq_str(aq_status), vsi_name, vsi->type); 2428 } 2429 } 2430 } 2431 2432 /** 2433 * i40e_aqc_broadcast_filter - Set promiscuous broadcast flags 2434 * @vsi: pointer to the VSI 2435 * @vsi_name: the VSI name 2436 * @f: filter data 2437 * 2438 * This function sets or clears the promiscuous broadcast flags for VLAN 2439 * filters in order to properly receive broadcast frames. Assumes that only 2440 * broadcast filters are passed. 2441 * 2442 * Returns status indicating success or failure; 2443 **/ 2444 static int 2445 i40e_aqc_broadcast_filter(struct i40e_vsi *vsi, const char *vsi_name, 2446 struct i40e_mac_filter *f) 2447 { 2448 bool enable = f->state == I40E_FILTER_NEW || 2449 f->state == I40E_FILTER_NEW_SYNC; 2450 struct i40e_hw *hw = &vsi->back->hw; 2451 int aq_ret; 2452 2453 if (f->vlan == I40E_VLAN_ANY) { 2454 aq_ret = i40e_aq_set_vsi_broadcast(hw, 2455 vsi->seid, 2456 enable, 2457 NULL); 2458 } else { 2459 aq_ret = i40e_aq_set_vsi_bc_promisc_on_vlan(hw, 2460 vsi->seid, 2461 enable, 2462 f->vlan, 2463 NULL); 2464 } 2465 2466 if (aq_ret) { 2467 set_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state); 2468 dev_warn(&vsi->back->pdev->dev, 2469 "Error %s, forcing overflow promiscuous on %s\n", 2470 libie_aq_str(hw->aq.asq_last_status), vsi_name); 2471 } 2472 2473 return aq_ret; 2474 } 2475 2476 /** 2477 * i40e_set_promiscuous - set promiscuous mode 2478 * @pf: board private structure 2479 * @promisc: promisc on or off 2480 * 2481 * There are different ways of setting promiscuous mode on a PF depending on 2482 * what state/environment we're in. This identifies and sets it appropriately. 2483 * Returns 0 on success. 2484 **/ 2485 static int i40e_set_promiscuous(struct i40e_pf *pf, bool promisc) 2486 { 2487 struct i40e_vsi *vsi = i40e_pf_get_main_vsi(pf); 2488 struct i40e_hw *hw = &pf->hw; 2489 int aq_ret; 2490 2491 if (vsi->type == I40E_VSI_MAIN && 2492 i40e_pf_get_main_veb(pf) && 2493 !test_bit(I40E_FLAG_MFP_ENA, pf->flags)) { 2494 /* set defport ON for Main VSI instead of true promisc 2495 * this way we will get all unicast/multicast and VLAN 2496 * promisc behavior but will not get VF or VMDq traffic 2497 * replicated on the Main VSI. 2498 */ 2499 if (promisc) 2500 aq_ret = i40e_aq_set_default_vsi(hw, 2501 vsi->seid, 2502 NULL); 2503 else 2504 aq_ret = i40e_aq_clear_default_vsi(hw, 2505 vsi->seid, 2506 NULL); 2507 if (aq_ret) { 2508 dev_info(&pf->pdev->dev, 2509 "Set default VSI failed, err %pe, aq_err %s\n", 2510 ERR_PTR(aq_ret), 2511 libie_aq_str(hw->aq.asq_last_status)); 2512 } 2513 } else { 2514 aq_ret = i40e_aq_set_vsi_unicast_promiscuous( 2515 hw, 2516 vsi->seid, 2517 promisc, NULL, 2518 true); 2519 if (aq_ret) { 2520 dev_info(&pf->pdev->dev, 2521 "set unicast promisc failed, err %pe, aq_err %s\n", 2522 ERR_PTR(aq_ret), 2523 libie_aq_str(hw->aq.asq_last_status)); 2524 } 2525 aq_ret = i40e_aq_set_vsi_multicast_promiscuous( 2526 hw, 2527 vsi->seid, 2528 promisc, NULL); 2529 if (aq_ret) { 2530 dev_info(&pf->pdev->dev, 2531 "set multicast promisc failed, err %pe, aq_err %s\n", 2532 ERR_PTR(aq_ret), 2533 libie_aq_str(hw->aq.asq_last_status)); 2534 } 2535 } 2536 2537 if (!aq_ret) 2538 pf->cur_promisc = promisc; 2539 2540 return aq_ret; 2541 } 2542 2543 /** 2544 * i40e_sync_vsi_filters - Update the VSI filter list to the HW 2545 * @vsi: ptr to the VSI 2546 * 2547 * Push any outstanding VSI filter changes through the AdminQ. 2548 * 2549 * Returns 0 or error value 2550 **/ 2551 int i40e_sync_vsi_filters(struct i40e_vsi *vsi) 2552 { 2553 struct hlist_head tmp_add_list, tmp_del_list; 2554 struct i40e_mac_filter *f; 2555 struct i40e_new_mac_filter *new, *add_head = NULL; 2556 struct i40e_hw *hw = &vsi->back->hw; 2557 bool old_overflow, new_overflow; 2558 unsigned int failed_filters = 0; 2559 unsigned int vlan_filters = 0; 2560 char vsi_name[16] = "PF"; 2561 int filter_list_len = 0; 2562 u32 changed_flags = 0; 2563 struct hlist_node *h; 2564 struct i40e_pf *pf; 2565 int num_add = 0; 2566 int num_del = 0; 2567 int aq_ret = 0; 2568 int retval = 0; 2569 u16 cmd_flags; 2570 int list_size; 2571 int bkt; 2572 2573 /* empty array typed pointers, kcalloc later */ 2574 struct i40e_aqc_add_macvlan_element_data *add_list; 2575 struct i40e_aqc_remove_macvlan_element_data *del_list; 2576 2577 while (test_and_set_bit(__I40E_VSI_SYNCING_FILTERS, vsi->state)) 2578 usleep_range(1000, 2000); 2579 pf = vsi->back; 2580 2581 old_overflow = test_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state); 2582 2583 if (vsi->netdev) { 2584 changed_flags = vsi->current_netdev_flags ^ vsi->netdev->flags; 2585 vsi->current_netdev_flags = vsi->netdev->flags; 2586 } 2587 2588 INIT_HLIST_HEAD(&tmp_add_list); 2589 INIT_HLIST_HEAD(&tmp_del_list); 2590 2591 if (vsi->type == I40E_VSI_SRIOV) 2592 snprintf(vsi_name, sizeof(vsi_name) - 1, "VF %d", vsi->vf_id); 2593 else if (vsi->type != I40E_VSI_MAIN) 2594 snprintf(vsi_name, sizeof(vsi_name) - 1, "vsi %d", vsi->seid); 2595 2596 if (vsi->flags & I40E_VSI_FLAG_FILTER_CHANGED) { 2597 vsi->flags &= ~I40E_VSI_FLAG_FILTER_CHANGED; 2598 2599 spin_lock_bh(&vsi->mac_filter_hash_lock); 2600 /* Create a list of filters to delete. */ 2601 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) { 2602 if (f->state == I40E_FILTER_REMOVE) { 2603 /* Move the element into temporary del_list */ 2604 hash_del(&f->hlist); 2605 hlist_add_head(&f->hlist, &tmp_del_list); 2606 2607 /* Avoid counting removed filters */ 2608 continue; 2609 } 2610 if (f->state == I40E_FILTER_NEW) { 2611 /* Create a temporary i40e_new_mac_filter */ 2612 new = kzalloc_obj(*new, GFP_ATOMIC); 2613 if (!new) 2614 goto err_no_memory_locked; 2615 2616 /* Store pointer to the real filter */ 2617 new->f = f; 2618 new->state = f->state; 2619 2620 /* Add it to the hash list */ 2621 hlist_add_head(&new->hlist, &tmp_add_list); 2622 f->state = I40E_FILTER_NEW_SYNC; 2623 } 2624 2625 /* Count the number of active (current and new) VLAN 2626 * filters we have now. Does not count filters which 2627 * are marked for deletion. 2628 */ 2629 if (f->vlan > 0) 2630 vlan_filters++; 2631 } 2632 2633 if (vsi->type != I40E_VSI_SRIOV) 2634 retval = i40e_correct_mac_vlan_filters 2635 (vsi, &tmp_add_list, &tmp_del_list, 2636 vlan_filters); 2637 else if (pf->vf) 2638 retval = i40e_correct_vf_mac_vlan_filters 2639 (vsi, &tmp_add_list, &tmp_del_list, 2640 vlan_filters, pf->vf[vsi->vf_id].trusted); 2641 2642 hlist_for_each_entry(new, &tmp_add_list, hlist) 2643 netdev_hw_addr_refcnt(new->f, vsi->netdev, 1); 2644 2645 if (retval) 2646 goto err_no_memory_locked; 2647 2648 spin_unlock_bh(&vsi->mac_filter_hash_lock); 2649 } 2650 2651 /* Now process 'del_list' outside the lock */ 2652 if (!hlist_empty(&tmp_del_list)) { 2653 filter_list_len = hw->aq.asq_buf_size / 2654 sizeof(struct i40e_aqc_remove_macvlan_element_data); 2655 list_size = filter_list_len * 2656 sizeof(struct i40e_aqc_remove_macvlan_element_data); 2657 del_list = kzalloc(list_size, GFP_ATOMIC); 2658 if (!del_list) 2659 goto err_no_memory; 2660 2661 hlist_for_each_entry_safe(f, h, &tmp_del_list, hlist) { 2662 cmd_flags = 0; 2663 2664 /* handle broadcast filters by updating the broadcast 2665 * promiscuous flag and release filter list. 2666 */ 2667 if (is_broadcast_ether_addr(f->macaddr)) { 2668 i40e_aqc_broadcast_filter(vsi, vsi_name, f); 2669 2670 hlist_del(&f->hlist); 2671 kfree(f); 2672 continue; 2673 } 2674 2675 /* add to delete list */ 2676 ether_addr_copy(del_list[num_del].mac_addr, f->macaddr); 2677 if (f->vlan == I40E_VLAN_ANY) { 2678 del_list[num_del].vlan_tag = 0; 2679 cmd_flags |= I40E_AQC_MACVLAN_DEL_IGNORE_VLAN; 2680 } else { 2681 del_list[num_del].vlan_tag = 2682 cpu_to_le16((u16)(f->vlan)); 2683 } 2684 2685 cmd_flags |= I40E_AQC_MACVLAN_DEL_PERFECT_MATCH; 2686 del_list[num_del].flags = cmd_flags; 2687 num_del++; 2688 2689 /* flush a full buffer */ 2690 if (num_del == filter_list_len) { 2691 i40e_aqc_del_filters(vsi, vsi_name, del_list, 2692 num_del, &retval); 2693 memset(del_list, 0, list_size); 2694 num_del = 0; 2695 } 2696 /* Release memory for MAC filter entries which were 2697 * synced up with HW. 2698 */ 2699 hlist_del(&f->hlist); 2700 kfree(f); 2701 } 2702 2703 if (num_del) { 2704 i40e_aqc_del_filters(vsi, vsi_name, del_list, 2705 num_del, &retval); 2706 } 2707 2708 kfree(del_list); 2709 del_list = NULL; 2710 } 2711 2712 if (!hlist_empty(&tmp_add_list)) { 2713 /* Do all the adds now. */ 2714 filter_list_len = hw->aq.asq_buf_size / 2715 sizeof(struct i40e_aqc_add_macvlan_element_data); 2716 list_size = filter_list_len * 2717 sizeof(struct i40e_aqc_add_macvlan_element_data); 2718 add_list = kzalloc(list_size, GFP_ATOMIC); 2719 if (!add_list) 2720 goto err_no_memory; 2721 2722 num_add = 0; 2723 hlist_for_each_entry_safe(new, h, &tmp_add_list, hlist) { 2724 /* handle broadcast filters by updating the broadcast 2725 * promiscuous flag instead of adding a MAC filter. 2726 */ 2727 if (is_broadcast_ether_addr(new->f->macaddr)) { 2728 if (i40e_aqc_broadcast_filter(vsi, vsi_name, 2729 new->f)) 2730 new->state = I40E_FILTER_FAILED; 2731 else 2732 new->state = I40E_FILTER_ACTIVE; 2733 continue; 2734 } 2735 2736 /* add to add array */ 2737 if (num_add == 0) 2738 add_head = new; 2739 cmd_flags = 0; 2740 ether_addr_copy(add_list[num_add].mac_addr, 2741 new->f->macaddr); 2742 if (new->f->vlan == I40E_VLAN_ANY) { 2743 add_list[num_add].vlan_tag = 0; 2744 cmd_flags |= I40E_AQC_MACVLAN_ADD_IGNORE_VLAN; 2745 } else { 2746 add_list[num_add].vlan_tag = 2747 cpu_to_le16((u16)(new->f->vlan)); 2748 } 2749 add_list[num_add].queue_number = 0; 2750 /* set invalid match method for later detection */ 2751 add_list[num_add].match_method = I40E_AQC_MM_ERR_NO_RES; 2752 cmd_flags |= I40E_AQC_MACVLAN_ADD_PERFECT_MATCH; 2753 add_list[num_add].flags = cpu_to_le16(cmd_flags); 2754 num_add++; 2755 2756 /* flush a full buffer */ 2757 if (num_add == filter_list_len) { 2758 i40e_aqc_add_filters(vsi, vsi_name, add_list, 2759 add_head, num_add); 2760 memset(add_list, 0, list_size); 2761 num_add = 0; 2762 } 2763 } 2764 if (num_add) { 2765 i40e_aqc_add_filters(vsi, vsi_name, add_list, add_head, 2766 num_add); 2767 } 2768 /* Now move all of the filters from the temp add list back to 2769 * the VSI's list. 2770 */ 2771 spin_lock_bh(&vsi->mac_filter_hash_lock); 2772 hlist_for_each_entry_safe(new, h, &tmp_add_list, hlist) { 2773 /* Only update the state if we're still NEW */ 2774 if (new->f->state == I40E_FILTER_NEW || 2775 new->f->state == I40E_FILTER_NEW_SYNC) 2776 new->f->state = new->state; 2777 hlist_del(&new->hlist); 2778 netdev_hw_addr_refcnt(new->f, vsi->netdev, -1); 2779 kfree(new); 2780 } 2781 spin_unlock_bh(&vsi->mac_filter_hash_lock); 2782 kfree(add_list); 2783 add_list = NULL; 2784 } 2785 2786 /* Determine the number of active and failed filters. */ 2787 spin_lock_bh(&vsi->mac_filter_hash_lock); 2788 vsi->active_filters = 0; 2789 hash_for_each(vsi->mac_filter_hash, bkt, f, hlist) { 2790 if (f->state == I40E_FILTER_ACTIVE) 2791 vsi->active_filters++; 2792 else if (f->state == I40E_FILTER_FAILED) 2793 failed_filters++; 2794 } 2795 spin_unlock_bh(&vsi->mac_filter_hash_lock); 2796 2797 /* Check if we are able to exit overflow promiscuous mode. We can 2798 * safely exit if we didn't just enter, we no longer have any failed 2799 * filters, and we have reduced filters below the threshold value. 2800 */ 2801 if (old_overflow && !failed_filters && 2802 vsi->active_filters < vsi->promisc_threshold) { 2803 dev_info(&pf->pdev->dev, 2804 "filter logjam cleared on %s, leaving overflow promiscuous mode\n", 2805 vsi_name); 2806 clear_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state); 2807 vsi->promisc_threshold = 0; 2808 } 2809 2810 /* if the VF is not trusted do not do promisc */ 2811 if (vsi->type == I40E_VSI_SRIOV && pf->vf && 2812 !pf->vf[vsi->vf_id].trusted) { 2813 clear_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state); 2814 goto out; 2815 } 2816 2817 new_overflow = test_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state); 2818 2819 /* If we are entering overflow promiscuous, we need to calculate a new 2820 * threshold for when we are safe to exit 2821 */ 2822 if (!old_overflow && new_overflow) 2823 vsi->promisc_threshold = (vsi->active_filters * 3) / 4; 2824 2825 /* check for changes in promiscuous modes */ 2826 if (changed_flags & IFF_ALLMULTI) { 2827 bool cur_multipromisc; 2828 2829 cur_multipromisc = !!(vsi->current_netdev_flags & IFF_ALLMULTI); 2830 aq_ret = i40e_aq_set_vsi_multicast_promiscuous(&vsi->back->hw, 2831 vsi->seid, 2832 cur_multipromisc, 2833 NULL); 2834 if (aq_ret) { 2835 retval = i40e_aq_rc_to_posix(aq_ret, 2836 hw->aq.asq_last_status); 2837 dev_info(&pf->pdev->dev, 2838 "set multi promisc failed on %s, err %pe aq_err %s\n", 2839 vsi_name, 2840 ERR_PTR(aq_ret), 2841 libie_aq_str(hw->aq.asq_last_status)); 2842 } else { 2843 dev_info(&pf->pdev->dev, "%s allmulti mode.\n", 2844 cur_multipromisc ? "entering" : "leaving"); 2845 } 2846 } 2847 2848 if ((changed_flags & IFF_PROMISC) || old_overflow != new_overflow) { 2849 bool cur_promisc; 2850 2851 cur_promisc = (!!(vsi->current_netdev_flags & IFF_PROMISC) || 2852 new_overflow); 2853 aq_ret = i40e_set_promiscuous(pf, cur_promisc); 2854 if (aq_ret) { 2855 retval = i40e_aq_rc_to_posix(aq_ret, 2856 hw->aq.asq_last_status); 2857 dev_info(&pf->pdev->dev, 2858 "Setting promiscuous %s failed on %s, err %pe aq_err %s\n", 2859 cur_promisc ? "on" : "off", 2860 vsi_name, 2861 ERR_PTR(aq_ret), 2862 libie_aq_str(hw->aq.asq_last_status)); 2863 } 2864 } 2865 out: 2866 /* if something went wrong then set the changed flag so we try again */ 2867 if (retval) 2868 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED; 2869 2870 clear_bit(__I40E_VSI_SYNCING_FILTERS, vsi->state); 2871 return retval; 2872 2873 err_no_memory: 2874 /* Restore elements on the temporary add and delete lists */ 2875 spin_lock_bh(&vsi->mac_filter_hash_lock); 2876 err_no_memory_locked: 2877 i40e_undo_del_filter_entries(vsi, &tmp_del_list); 2878 i40e_undo_add_filter_entries(vsi, &tmp_add_list); 2879 spin_unlock_bh(&vsi->mac_filter_hash_lock); 2880 2881 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED; 2882 clear_bit(__I40E_VSI_SYNCING_FILTERS, vsi->state); 2883 return -ENOMEM; 2884 } 2885 2886 /** 2887 * i40e_sync_filters_subtask - Sync the VSI filter list with HW 2888 * @pf: board private structure 2889 **/ 2890 static void i40e_sync_filters_subtask(struct i40e_pf *pf) 2891 { 2892 struct i40e_vsi *vsi; 2893 int v; 2894 2895 if (!pf) 2896 return; 2897 if (!test_and_clear_bit(__I40E_MACVLAN_SYNC_PENDING, pf->state)) 2898 return; 2899 if (test_bit(__I40E_VF_DISABLE, pf->state)) { 2900 set_bit(__I40E_MACVLAN_SYNC_PENDING, pf->state); 2901 return; 2902 } 2903 2904 i40e_pf_for_each_vsi(pf, v, vsi) { 2905 if ((vsi->flags & I40E_VSI_FLAG_FILTER_CHANGED) && 2906 !test_bit(__I40E_VSI_RELEASING, vsi->state)) { 2907 int ret = i40e_sync_vsi_filters(vsi); 2908 2909 if (ret) { 2910 /* come back and try again later */ 2911 set_bit(__I40E_MACVLAN_SYNC_PENDING, 2912 pf->state); 2913 break; 2914 } 2915 } 2916 } 2917 } 2918 2919 /** 2920 * i40e_calculate_vsi_rx_buf_len - Calculates buffer length 2921 * 2922 * @vsi: VSI to calculate rx_buf_len from 2923 */ 2924 static u16 i40e_calculate_vsi_rx_buf_len(struct i40e_vsi *vsi) 2925 { 2926 if (!vsi->netdev || test_bit(I40E_FLAG_LEGACY_RX_ENA, vsi->back->flags)) 2927 return SKB_WITH_OVERHEAD(I40E_RXBUFFER_2048); 2928 2929 return PAGE_SIZE < 8192 ? I40E_RXBUFFER_3072 : I40E_RXBUFFER_2048; 2930 } 2931 2932 /** 2933 * i40e_max_vsi_frame_size - returns the maximum allowed frame size for VSI 2934 * @vsi: the vsi 2935 * @xdp_prog: XDP program 2936 **/ 2937 static int i40e_max_vsi_frame_size(struct i40e_vsi *vsi, 2938 struct bpf_prog *xdp_prog) 2939 { 2940 u16 rx_buf_len = i40e_calculate_vsi_rx_buf_len(vsi); 2941 u16 chain_len; 2942 2943 if (xdp_prog && !xdp_prog->aux->xdp_has_frags) 2944 chain_len = 1; 2945 else 2946 chain_len = I40E_MAX_CHAINED_RX_BUFFERS; 2947 2948 return min_t(u16, rx_buf_len * chain_len, I40E_MAX_RXBUFFER); 2949 } 2950 2951 /** 2952 * i40e_change_mtu - NDO callback to change the Maximum Transfer Unit 2953 * @netdev: network interface device structure 2954 * @new_mtu: new value for maximum frame size 2955 * 2956 * Returns 0 on success, negative on failure 2957 **/ 2958 static int i40e_change_mtu(struct net_device *netdev, int new_mtu) 2959 { 2960 struct i40e_netdev_priv *np = netdev_priv(netdev); 2961 struct i40e_vsi *vsi = np->vsi; 2962 struct i40e_pf *pf = vsi->back; 2963 int frame_size; 2964 2965 frame_size = i40e_max_vsi_frame_size(vsi, vsi->xdp_prog); 2966 if (new_mtu > frame_size - I40E_PACKET_HDR_PAD) { 2967 netdev_err(netdev, "Error changing mtu to %d, Max is %d\n", 2968 new_mtu, frame_size - I40E_PACKET_HDR_PAD); 2969 return -EINVAL; 2970 } 2971 2972 netdev_dbg(netdev, "changing MTU from %d to %d\n", 2973 netdev->mtu, new_mtu); 2974 WRITE_ONCE(netdev->mtu, new_mtu); 2975 if (netif_running(netdev)) 2976 i40e_vsi_reinit_locked(vsi); 2977 set_bit(__I40E_CLIENT_SERVICE_REQUESTED, pf->state); 2978 set_bit(__I40E_CLIENT_L2_CHANGE, pf->state); 2979 return 0; 2980 } 2981 2982 /** 2983 * i40e_vlan_stripping_enable - Turn on vlan stripping for the VSI 2984 * @vsi: the vsi being adjusted 2985 **/ 2986 void i40e_vlan_stripping_enable(struct i40e_vsi *vsi) 2987 { 2988 struct i40e_vsi_context ctxt; 2989 int ret; 2990 2991 /* Don't modify stripping options if a port VLAN is active */ 2992 if (vsi->info.pvid) 2993 return; 2994 2995 if ((vsi->info.valid_sections & 2996 cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID)) && 2997 ((vsi->info.port_vlan_flags & I40E_AQ_VSI_PVLAN_MODE_MASK) == 0)) 2998 return; /* already enabled */ 2999 3000 vsi->info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID); 3001 vsi->info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_ALL | 3002 I40E_AQ_VSI_PVLAN_EMOD_STR_BOTH; 3003 3004 ctxt.seid = vsi->seid; 3005 ctxt.info = vsi->info; 3006 ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL); 3007 if (ret) { 3008 dev_info(&vsi->back->pdev->dev, 3009 "update vlan stripping failed, err %pe aq_err %s\n", 3010 ERR_PTR(ret), 3011 libie_aq_str(vsi->back->hw.aq.asq_last_status)); 3012 } 3013 } 3014 3015 /** 3016 * i40e_vlan_stripping_disable - Turn off vlan stripping for the VSI 3017 * @vsi: the vsi being adjusted 3018 **/ 3019 void i40e_vlan_stripping_disable(struct i40e_vsi *vsi) 3020 { 3021 struct i40e_vsi_context ctxt; 3022 int ret; 3023 3024 /* Don't modify stripping options if a port VLAN is active */ 3025 if (vsi->info.pvid) 3026 return; 3027 3028 if ((vsi->info.valid_sections & 3029 cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID)) && 3030 ((vsi->info.port_vlan_flags & I40E_AQ_VSI_PVLAN_EMOD_MASK) == 3031 I40E_AQ_VSI_PVLAN_EMOD_MASK)) 3032 return; /* already disabled */ 3033 3034 vsi->info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID); 3035 vsi->info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_ALL | 3036 I40E_AQ_VSI_PVLAN_EMOD_NOTHING; 3037 3038 ctxt.seid = vsi->seid; 3039 ctxt.info = vsi->info; 3040 ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL); 3041 if (ret) { 3042 dev_info(&vsi->back->pdev->dev, 3043 "update vlan stripping failed, err %pe aq_err %s\n", 3044 ERR_PTR(ret), 3045 libie_aq_str(vsi->back->hw.aq.asq_last_status)); 3046 } 3047 } 3048 3049 /** 3050 * i40e_add_vlan_all_mac - Add a MAC/VLAN filter for each existing MAC address 3051 * @vsi: the vsi being configured 3052 * @vid: vlan id to be added (0 = untagged only , -1 = any) 3053 * 3054 * This is a helper function for adding a new MAC/VLAN filter with the 3055 * specified VLAN for each existing MAC address already in the hash table. 3056 * This function does *not* perform any accounting to update filters based on 3057 * VLAN mode. 3058 * 3059 * NOTE: this function expects to be called while under the 3060 * mac_filter_hash_lock 3061 **/ 3062 int i40e_add_vlan_all_mac(struct i40e_vsi *vsi, s16 vid) 3063 { 3064 struct i40e_mac_filter *f, *add_f; 3065 struct hlist_node *h; 3066 int bkt; 3067 3068 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) { 3069 /* If we're asked to add a filter that has been marked for 3070 * removal, it is safe to simply restore it to active state. 3071 * __i40e_del_filter will have simply deleted any filters which 3072 * were previously marked NEW or FAILED, so if it is currently 3073 * marked REMOVE it must have previously been ACTIVE. Since we 3074 * haven't yet run the sync filters task, just restore this 3075 * filter to the ACTIVE state so that the sync task leaves it 3076 * in place. 3077 */ 3078 if (f->state == I40E_FILTER_REMOVE && f->vlan == vid) { 3079 f->state = I40E_FILTER_ACTIVE; 3080 continue; 3081 } else if (f->state == I40E_FILTER_REMOVE) { 3082 continue; 3083 } 3084 add_f = i40e_add_filter(vsi, f->macaddr, vid); 3085 if (!add_f) { 3086 dev_info(&vsi->back->pdev->dev, 3087 "Could not add vlan filter %d for %pM\n", 3088 vid, f->macaddr); 3089 return -ENOMEM; 3090 } 3091 } 3092 3093 return 0; 3094 } 3095 3096 /** 3097 * i40e_vsi_add_vlan - Add VSI membership for given VLAN 3098 * @vsi: the VSI being configured 3099 * @vid: VLAN id to be added 3100 **/ 3101 int i40e_vsi_add_vlan(struct i40e_vsi *vsi, u16 vid) 3102 { 3103 int err; 3104 3105 if (vsi->info.pvid) 3106 return -EINVAL; 3107 3108 /* The network stack will attempt to add VID=0, with the intention to 3109 * receive priority tagged packets with a VLAN of 0. Our HW receives 3110 * these packets by default when configured to receive untagged 3111 * packets, so we don't need to add a filter for this case. 3112 * Additionally, HW interprets adding a VID=0 filter as meaning to 3113 * receive *only* tagged traffic and stops receiving untagged traffic. 3114 * Thus, we do not want to actually add a filter for VID=0 3115 */ 3116 if (!vid) 3117 return 0; 3118 3119 /* Locked once because all functions invoked below iterates list*/ 3120 spin_lock_bh(&vsi->mac_filter_hash_lock); 3121 err = i40e_add_vlan_all_mac(vsi, vid); 3122 spin_unlock_bh(&vsi->mac_filter_hash_lock); 3123 if (err) 3124 return err; 3125 3126 /* schedule our worker thread which will take care of 3127 * applying the new filter changes 3128 */ 3129 i40e_service_event_schedule(vsi->back); 3130 return 0; 3131 } 3132 3133 /** 3134 * i40e_rm_vlan_all_mac - Remove MAC/VLAN pair for all MAC with the given VLAN 3135 * @vsi: the vsi being configured 3136 * @vid: vlan id to be removed (0 = untagged only , -1 = any) 3137 * 3138 * This function should be used to remove all VLAN filters which match the 3139 * given VID. It does not schedule the service event and does not take the 3140 * mac_filter_hash_lock so it may be combined with other operations under 3141 * a single invocation of the mac_filter_hash_lock. 3142 * 3143 * NOTE: this function expects to be called while under the 3144 * mac_filter_hash_lock 3145 */ 3146 void i40e_rm_vlan_all_mac(struct i40e_vsi *vsi, s16 vid) 3147 { 3148 struct i40e_mac_filter *f; 3149 struct hlist_node *h; 3150 int bkt; 3151 3152 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) { 3153 if (f->vlan == vid) 3154 __i40e_del_filter(vsi, f); 3155 } 3156 } 3157 3158 /** 3159 * i40e_vsi_kill_vlan - Remove VSI membership for given VLAN 3160 * @vsi: the VSI being configured 3161 * @vid: VLAN id to be removed 3162 **/ 3163 void i40e_vsi_kill_vlan(struct i40e_vsi *vsi, u16 vid) 3164 { 3165 if (!vid || vsi->info.pvid) 3166 return; 3167 3168 spin_lock_bh(&vsi->mac_filter_hash_lock); 3169 i40e_rm_vlan_all_mac(vsi, vid); 3170 spin_unlock_bh(&vsi->mac_filter_hash_lock); 3171 3172 /* schedule our worker thread which will take care of 3173 * applying the new filter changes 3174 */ 3175 i40e_service_event_schedule(vsi->back); 3176 } 3177 3178 /** 3179 * i40e_vlan_rx_add_vid - Add a vlan id filter to HW offload 3180 * @netdev: network interface to be adjusted 3181 * @proto: unused protocol value 3182 * @vid: vlan id to be added 3183 * 3184 * net_device_ops implementation for adding vlan ids 3185 **/ 3186 static int i40e_vlan_rx_add_vid(struct net_device *netdev, 3187 __always_unused __be16 proto, u16 vid) 3188 { 3189 struct i40e_netdev_priv *np = netdev_priv(netdev); 3190 struct i40e_vsi *vsi = np->vsi; 3191 int ret = 0; 3192 3193 if (vid >= VLAN_N_VID) 3194 return -EINVAL; 3195 3196 ret = i40e_vsi_add_vlan(vsi, vid); 3197 if (!ret) 3198 set_bit(vid, vsi->active_vlans); 3199 3200 return ret; 3201 } 3202 3203 /** 3204 * i40e_vlan_rx_add_vid_up - Add a vlan id filter to HW offload in UP path 3205 * @netdev: network interface to be adjusted 3206 * @proto: unused protocol value 3207 * @vid: vlan id to be added 3208 **/ 3209 static void i40e_vlan_rx_add_vid_up(struct net_device *netdev, 3210 __always_unused __be16 proto, u16 vid) 3211 { 3212 struct i40e_netdev_priv *np = netdev_priv(netdev); 3213 struct i40e_vsi *vsi = np->vsi; 3214 3215 if (vid >= VLAN_N_VID) 3216 return; 3217 set_bit(vid, vsi->active_vlans); 3218 } 3219 3220 /** 3221 * i40e_vlan_rx_kill_vid - Remove a vlan id filter from HW offload 3222 * @netdev: network interface to be adjusted 3223 * @proto: unused protocol value 3224 * @vid: vlan id to be removed 3225 * 3226 * net_device_ops implementation for removing vlan ids 3227 **/ 3228 static int i40e_vlan_rx_kill_vid(struct net_device *netdev, 3229 __always_unused __be16 proto, u16 vid) 3230 { 3231 struct i40e_netdev_priv *np = netdev_priv(netdev); 3232 struct i40e_vsi *vsi = np->vsi; 3233 3234 /* return code is ignored as there is nothing a user 3235 * can do about failure to remove and a log message was 3236 * already printed from the other function 3237 */ 3238 i40e_vsi_kill_vlan(vsi, vid); 3239 3240 clear_bit(vid, vsi->active_vlans); 3241 3242 return 0; 3243 } 3244 3245 /** 3246 * i40e_restore_vlan - Reinstate vlans when vsi/netdev comes back up 3247 * @vsi: the vsi being brought back up 3248 **/ 3249 static void i40e_restore_vlan(struct i40e_vsi *vsi) 3250 { 3251 u16 vid; 3252 3253 if (!vsi->netdev) 3254 return; 3255 3256 if (vsi->netdev->features & NETIF_F_HW_VLAN_CTAG_RX) 3257 i40e_vlan_stripping_enable(vsi); 3258 else 3259 i40e_vlan_stripping_disable(vsi); 3260 3261 for_each_set_bit(vid, vsi->active_vlans, VLAN_N_VID) 3262 i40e_vlan_rx_add_vid_up(vsi->netdev, htons(ETH_P_8021Q), 3263 vid); 3264 } 3265 3266 /** 3267 * i40e_vsi_add_pvid - Add pvid for the VSI 3268 * @vsi: the vsi being adjusted 3269 * @vid: the vlan id to set as a PVID 3270 **/ 3271 int i40e_vsi_add_pvid(struct i40e_vsi *vsi, u16 vid) 3272 { 3273 struct i40e_vsi_context ctxt; 3274 int ret; 3275 3276 vsi->info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID); 3277 vsi->info.pvid = cpu_to_le16(vid); 3278 vsi->info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_TAGGED | 3279 I40E_AQ_VSI_PVLAN_INSERT_PVID | 3280 I40E_AQ_VSI_PVLAN_EMOD_STR; 3281 3282 ctxt.seid = vsi->seid; 3283 ctxt.info = vsi->info; 3284 ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL); 3285 if (ret) { 3286 dev_info(&vsi->back->pdev->dev, 3287 "add pvid failed, err %pe aq_err %s\n", 3288 ERR_PTR(ret), 3289 libie_aq_str(vsi->back->hw.aq.asq_last_status)); 3290 return -ENOENT; 3291 } 3292 3293 return 0; 3294 } 3295 3296 /** 3297 * i40e_vsi_remove_pvid - Remove the pvid from the VSI 3298 * @vsi: the vsi being adjusted 3299 * 3300 * Just use the vlan_rx_register() service to put it back to normal 3301 **/ 3302 void i40e_vsi_remove_pvid(struct i40e_vsi *vsi) 3303 { 3304 vsi->info.pvid = 0; 3305 3306 i40e_vlan_stripping_disable(vsi); 3307 } 3308 3309 /** 3310 * i40e_vsi_setup_tx_resources - Allocate VSI Tx queue resources 3311 * @vsi: ptr to the VSI 3312 * 3313 * If this function returns with an error, then it's possible one or 3314 * more of the rings is populated (while the rest are not). It is the 3315 * callers duty to clean those orphaned rings. 3316 * 3317 * Return 0 on success, negative on failure 3318 **/ 3319 static int i40e_vsi_setup_tx_resources(struct i40e_vsi *vsi) 3320 { 3321 int i, err = 0; 3322 3323 for (i = 0; i < vsi->num_queue_pairs && !err; i++) 3324 err = i40e_setup_tx_descriptors(vsi->tx_rings[i]); 3325 3326 if (!i40e_enabled_xdp_vsi(vsi)) 3327 return err; 3328 3329 for (i = 0; i < vsi->num_queue_pairs && !err; i++) 3330 err = i40e_setup_tx_descriptors(vsi->xdp_rings[i]); 3331 3332 return err; 3333 } 3334 3335 /** 3336 * i40e_vsi_free_tx_resources - Free Tx resources for VSI queues 3337 * @vsi: ptr to the VSI 3338 * 3339 * Free VSI's transmit software resources 3340 **/ 3341 static void i40e_vsi_free_tx_resources(struct i40e_vsi *vsi) 3342 { 3343 int i; 3344 3345 if (vsi->tx_rings) { 3346 for (i = 0; i < vsi->num_queue_pairs; i++) 3347 if (vsi->tx_rings[i] && vsi->tx_rings[i]->desc) 3348 i40e_free_tx_resources(vsi->tx_rings[i]); 3349 } 3350 3351 if (vsi->xdp_rings) { 3352 for (i = 0; i < vsi->num_queue_pairs; i++) 3353 if (vsi->xdp_rings[i] && vsi->xdp_rings[i]->desc) 3354 i40e_free_tx_resources(vsi->xdp_rings[i]); 3355 } 3356 } 3357 3358 /** 3359 * i40e_vsi_setup_rx_resources - Allocate VSI queues Rx resources 3360 * @vsi: ptr to the VSI 3361 * 3362 * If this function returns with an error, then it's possible one or 3363 * more of the rings is populated (while the rest are not). It is the 3364 * callers duty to clean those orphaned rings. 3365 * 3366 * Return 0 on success, negative on failure 3367 **/ 3368 static int i40e_vsi_setup_rx_resources(struct i40e_vsi *vsi) 3369 { 3370 int i, err = 0; 3371 3372 for (i = 0; i < vsi->num_queue_pairs && !err; i++) 3373 err = i40e_setup_rx_descriptors(vsi->rx_rings[i]); 3374 return err; 3375 } 3376 3377 /** 3378 * i40e_vsi_free_rx_resources - Free Rx Resources for VSI queues 3379 * @vsi: ptr to the VSI 3380 * 3381 * Free all receive software resources 3382 **/ 3383 static void i40e_vsi_free_rx_resources(struct i40e_vsi *vsi) 3384 { 3385 int i; 3386 3387 if (!vsi->rx_rings) 3388 return; 3389 3390 for (i = 0; i < vsi->num_queue_pairs; i++) 3391 if (vsi->rx_rings[i] && vsi->rx_rings[i]->desc) 3392 i40e_free_rx_resources(vsi->rx_rings[i]); 3393 } 3394 3395 /** 3396 * i40e_config_xps_tx_ring - Configure XPS for a Tx ring 3397 * @ring: The Tx ring to configure 3398 * 3399 * This enables/disables XPS for a given Tx descriptor ring 3400 * based on the TCs enabled for the VSI that ring belongs to. 3401 **/ 3402 static void i40e_config_xps_tx_ring(struct i40e_ring *ring) 3403 { 3404 int cpu; 3405 3406 if (!ring->q_vector || !ring->netdev || ring->ch) 3407 return; 3408 3409 /* We only initialize XPS once, so as not to overwrite user settings */ 3410 if (test_and_set_bit(__I40E_TX_XPS_INIT_DONE, ring->state)) 3411 return; 3412 3413 cpu = cpumask_local_spread(ring->q_vector->v_idx, -1); 3414 netif_set_xps_queue(ring->netdev, get_cpu_mask(cpu), 3415 ring->queue_index); 3416 } 3417 3418 /** 3419 * i40e_xsk_pool - Retrieve the AF_XDP buffer pool if XDP and ZC is enabled 3420 * @ring: The Tx or Rx ring 3421 * 3422 * Returns the AF_XDP buffer pool or NULL. 3423 **/ 3424 static struct xsk_buff_pool *i40e_xsk_pool(struct i40e_ring *ring) 3425 { 3426 bool xdp_on = i40e_enabled_xdp_vsi(ring->vsi); 3427 int qid = ring->queue_index; 3428 3429 if (ring_is_xdp(ring)) 3430 qid -= ring->vsi->alloc_queue_pairs; 3431 3432 if (!xdp_on || !test_bit(qid, ring->vsi->af_xdp_zc_qps)) 3433 return NULL; 3434 3435 return xsk_get_pool_from_qid(ring->vsi->netdev, qid); 3436 } 3437 3438 /** 3439 * i40e_configure_tx_ring - Configure a transmit ring context and rest 3440 * @ring: The Tx ring to configure 3441 * 3442 * Configure the Tx descriptor ring in the HMC context. 3443 **/ 3444 static int i40e_configure_tx_ring(struct i40e_ring *ring) 3445 { 3446 struct i40e_vsi *vsi = ring->vsi; 3447 u16 pf_q = vsi->base_queue + ring->queue_index; 3448 struct i40e_hw *hw = &vsi->back->hw; 3449 struct i40e_hmc_obj_txq tx_ctx; 3450 u32 qtx_ctl = 0; 3451 int err = 0; 3452 3453 if (ring_is_xdp(ring)) 3454 ring->xsk_pool = i40e_xsk_pool(ring); 3455 3456 /* some ATR related tx ring init */ 3457 if (test_bit(I40E_FLAG_FD_ATR_ENA, vsi->back->flags)) { 3458 ring->atr_sample_rate = I40E_DEFAULT_ATR_SAMPLE_RATE; 3459 ring->atr_count = 0; 3460 } else { 3461 ring->atr_sample_rate = 0; 3462 } 3463 3464 /* configure XPS */ 3465 i40e_config_xps_tx_ring(ring); 3466 3467 /* clear the context structure first */ 3468 memset(&tx_ctx, 0, sizeof(tx_ctx)); 3469 3470 tx_ctx.new_context = 1; 3471 tx_ctx.base = (ring->dma / 128); 3472 tx_ctx.qlen = ring->count; 3473 if (test_bit(I40E_FLAG_FD_SB_ENA, vsi->back->flags) || 3474 test_bit(I40E_FLAG_FD_ATR_ENA, vsi->back->flags)) 3475 tx_ctx.fd_ena = 1; 3476 if (test_bit(I40E_FLAG_PTP_ENA, vsi->back->flags)) 3477 tx_ctx.timesync_ena = 1; 3478 /* FDIR VSI tx ring can still use RS bit and writebacks */ 3479 if (vsi->type != I40E_VSI_FDIR) 3480 tx_ctx.head_wb_ena = 1; 3481 tx_ctx.head_wb_addr = ring->dma + 3482 (ring->count * sizeof(struct i40e_tx_desc)); 3483 3484 /* As part of VSI creation/update, FW allocates certain 3485 * Tx arbitration queue sets for each TC enabled for 3486 * the VSI. The FW returns the handles to these queue 3487 * sets as part of the response buffer to Add VSI, 3488 * Update VSI, etc. AQ commands. It is expected that 3489 * these queue set handles be associated with the Tx 3490 * queues by the driver as part of the TX queue context 3491 * initialization. This has to be done regardless of 3492 * DCB as by default everything is mapped to TC0. 3493 */ 3494 3495 if (ring->ch) 3496 tx_ctx.rdylist = 3497 le16_to_cpu(ring->ch->info.qs_handle[ring->dcb_tc]); 3498 3499 else 3500 tx_ctx.rdylist = le16_to_cpu(vsi->info.qs_handle[ring->dcb_tc]); 3501 3502 tx_ctx.rdylist_act = 0; 3503 3504 /* clear the context in the HMC */ 3505 err = i40e_clear_lan_tx_queue_context(hw, pf_q); 3506 if (err) { 3507 dev_info(&vsi->back->pdev->dev, 3508 "Failed to clear LAN Tx queue context on Tx ring %d (pf_q %d), error: %d\n", 3509 ring->queue_index, pf_q, err); 3510 return -ENOMEM; 3511 } 3512 3513 /* set the context in the HMC */ 3514 err = i40e_set_lan_tx_queue_context(hw, pf_q, &tx_ctx); 3515 if (err) { 3516 dev_info(&vsi->back->pdev->dev, 3517 "Failed to set LAN Tx queue context on Tx ring %d (pf_q %d, error: %d\n", 3518 ring->queue_index, pf_q, err); 3519 return -ENOMEM; 3520 } 3521 3522 /* Now associate this queue with this PCI function */ 3523 if (ring->ch) { 3524 if (ring->ch->type == I40E_VSI_VMDQ2) 3525 qtx_ctl = I40E_QTX_CTL_VM_QUEUE; 3526 else 3527 return -EINVAL; 3528 3529 qtx_ctl |= FIELD_PREP(I40E_QTX_CTL_VFVM_INDX_MASK, 3530 ring->ch->vsi_number); 3531 } else { 3532 if (vsi->type == I40E_VSI_VMDQ2) { 3533 qtx_ctl = I40E_QTX_CTL_VM_QUEUE; 3534 qtx_ctl |= FIELD_PREP(I40E_QTX_CTL_VFVM_INDX_MASK, 3535 vsi->id); 3536 } else { 3537 qtx_ctl = I40E_QTX_CTL_PF_QUEUE; 3538 } 3539 } 3540 3541 qtx_ctl |= FIELD_PREP(I40E_QTX_CTL_PF_INDX_MASK, hw->pf_id); 3542 wr32(hw, I40E_QTX_CTL(pf_q), qtx_ctl); 3543 i40e_flush(hw); 3544 3545 /* cache tail off for easier writes later */ 3546 ring->tail = hw->hw_addr + I40E_QTX_TAIL(pf_q); 3547 3548 return 0; 3549 } 3550 3551 /** 3552 * i40e_rx_offset - Return expected offset into page to access data 3553 * @rx_ring: Ring we are requesting offset of 3554 * 3555 * Returns the offset value for ring into the data buffer. 3556 */ 3557 static unsigned int i40e_rx_offset(struct i40e_ring *rx_ring) 3558 { 3559 return ring_uses_build_skb(rx_ring) ? I40E_SKB_PAD : 0; 3560 } 3561 3562 /** 3563 * i40e_configure_rx_ring - Configure a receive ring context 3564 * @ring: The Rx ring to configure 3565 * 3566 * Configure the Rx descriptor ring in the HMC context. 3567 **/ 3568 static int i40e_configure_rx_ring(struct i40e_ring *ring) 3569 { 3570 struct i40e_vsi *vsi = ring->vsi; 3571 u32 chain_len = vsi->back->hw.func_caps.rx_buf_chain_len; 3572 u16 pf_q = vsi->base_queue + ring->queue_index; 3573 struct i40e_hw *hw = &vsi->back->hw; 3574 struct i40e_hmc_obj_rxq rx_ctx; 3575 u32 xdp_frame_sz; 3576 int err = 0; 3577 bool ok; 3578 3579 bitmap_zero(ring->state, __I40E_RING_STATE_NBITS); 3580 3581 /* clear the context structure first */ 3582 memset(&rx_ctx, 0, sizeof(rx_ctx)); 3583 3584 ring->rx_buf_len = vsi->rx_buf_len; 3585 xdp_frame_sz = i40e_rx_pg_size(ring) / 2; 3586 3587 /* XDP RX-queue info only needed for RX rings exposed to XDP */ 3588 if (ring->vsi->type != I40E_VSI_MAIN) 3589 goto skip; 3590 3591 ring->xsk_pool = i40e_xsk_pool(ring); 3592 if (ring->xsk_pool) { 3593 xdp_frame_sz = xsk_pool_get_rx_frag_step(ring->xsk_pool); 3594 ring->rx_buf_len = xsk_pool_get_rx_frame_size(ring->xsk_pool); 3595 err = __xdp_rxq_info_reg(&ring->xdp_rxq, ring->netdev, 3596 ring->queue_index, 3597 ring->q_vector->napi.napi_id, 3598 xdp_frame_sz); 3599 if (err) 3600 return err; 3601 err = xdp_rxq_info_reg_mem_model(&ring->xdp_rxq, 3602 MEM_TYPE_XSK_BUFF_POOL, 3603 NULL); 3604 if (err) 3605 goto unreg_xdp; 3606 dev_info(&vsi->back->pdev->dev, 3607 "Registered XDP mem model MEM_TYPE_XSK_BUFF_POOL on Rx ring %d\n", 3608 ring->queue_index); 3609 3610 } else { 3611 err = __xdp_rxq_info_reg(&ring->xdp_rxq, ring->netdev, 3612 ring->queue_index, 3613 ring->q_vector->napi.napi_id, 3614 xdp_frame_sz); 3615 if (err) 3616 return err; 3617 err = xdp_rxq_info_reg_mem_model(&ring->xdp_rxq, 3618 MEM_TYPE_PAGE_SHARED, 3619 NULL); 3620 if (err) 3621 goto unreg_xdp; 3622 } 3623 3624 skip: 3625 xdp_init_buff(&ring->xdp, xdp_frame_sz, &ring->xdp_rxq); 3626 3627 rx_ctx.dbuff = DIV_ROUND_UP(ring->rx_buf_len, 3628 BIT_ULL(I40E_RXQ_CTX_DBUFF_SHIFT)); 3629 3630 rx_ctx.base = (ring->dma / 128); 3631 rx_ctx.qlen = ring->count; 3632 3633 /* use 16 byte descriptors */ 3634 rx_ctx.dsize = 0; 3635 3636 /* descriptor type is always zero 3637 * rx_ctx.dtype = 0; 3638 */ 3639 rx_ctx.hsplit_0 = 0; 3640 3641 rx_ctx.rxmax = min_t(u16, vsi->max_frame, chain_len * ring->rx_buf_len); 3642 if (hw->revision_id == 0) 3643 rx_ctx.lrxqthresh = 0; 3644 else 3645 rx_ctx.lrxqthresh = 1; 3646 rx_ctx.crcstrip = 1; 3647 rx_ctx.l2tsel = 1; 3648 /* this controls whether VLAN is stripped from inner headers */ 3649 rx_ctx.showiv = 0; 3650 /* set the prefena field to 1 because the manual says to */ 3651 rx_ctx.prefena = 1; 3652 3653 /* clear the context in the HMC */ 3654 err = i40e_clear_lan_rx_queue_context(hw, pf_q); 3655 if (err) { 3656 dev_info(&vsi->back->pdev->dev, 3657 "Failed to clear LAN Rx queue context on Rx ring %d (pf_q %d), error: %d\n", 3658 ring->queue_index, pf_q, err); 3659 err = -ENOMEM; 3660 goto unreg_xdp; 3661 } 3662 3663 /* set the context in the HMC */ 3664 err = i40e_set_lan_rx_queue_context(hw, pf_q, &rx_ctx); 3665 if (err) { 3666 dev_info(&vsi->back->pdev->dev, 3667 "Failed to set LAN Rx queue context on Rx ring %d (pf_q %d), error: %d\n", 3668 ring->queue_index, pf_q, err); 3669 err = -ENOMEM; 3670 goto unreg_xdp; 3671 } 3672 3673 /* configure Rx buffer alignment */ 3674 if (!vsi->netdev || test_bit(I40E_FLAG_LEGACY_RX_ENA, vsi->back->flags)) { 3675 if (I40E_2K_TOO_SMALL_WITH_PADDING) { 3676 dev_info(&vsi->back->pdev->dev, 3677 "2k Rx buffer is too small to fit standard MTU and skb_shared_info\n"); 3678 err = -EOPNOTSUPP; 3679 goto unreg_xdp; 3680 } 3681 clear_ring_build_skb_enabled(ring); 3682 } else { 3683 set_ring_build_skb_enabled(ring); 3684 } 3685 3686 ring->rx_offset = i40e_rx_offset(ring); 3687 3688 /* cache tail for quicker writes, and clear the reg before use */ 3689 ring->tail = hw->hw_addr + I40E_QRX_TAIL(pf_q); 3690 writel(0, ring->tail); 3691 3692 if (ring->xsk_pool) { 3693 xsk_pool_set_rxq_info(ring->xsk_pool, &ring->xdp_rxq); 3694 ok = i40e_alloc_rx_buffers_zc(ring, I40E_DESC_UNUSED(ring)); 3695 } else { 3696 ok = !i40e_alloc_rx_buffers(ring, I40E_DESC_UNUSED(ring)); 3697 } 3698 if (!ok) { 3699 /* Log this in case the user has forgotten to give the kernel 3700 * any buffers, even later in the application. 3701 */ 3702 dev_info(&vsi->back->pdev->dev, 3703 "Failed to allocate some buffers on %sRx ring %d (pf_q %d)\n", 3704 ring->xsk_pool ? "AF_XDP ZC enabled " : "", 3705 ring->queue_index, pf_q); 3706 } 3707 3708 return 0; 3709 unreg_xdp: 3710 if (ring->vsi->type == I40E_VSI_MAIN) 3711 xdp_rxq_info_unreg(&ring->xdp_rxq); 3712 3713 return err; 3714 } 3715 3716 /** 3717 * i40e_vsi_configure_tx - Configure the VSI for Tx 3718 * @vsi: VSI structure describing this set of rings and resources 3719 * 3720 * Configure the Tx VSI for operation. 3721 **/ 3722 static int i40e_vsi_configure_tx(struct i40e_vsi *vsi) 3723 { 3724 int err = 0; 3725 u16 i; 3726 3727 for (i = 0; (i < vsi->num_queue_pairs) && !err; i++) 3728 err = i40e_configure_tx_ring(vsi->tx_rings[i]); 3729 3730 if (err || !i40e_enabled_xdp_vsi(vsi)) 3731 return err; 3732 3733 for (i = 0; (i < vsi->num_queue_pairs) && !err; i++) 3734 err = i40e_configure_tx_ring(vsi->xdp_rings[i]); 3735 3736 return err; 3737 } 3738 3739 /** 3740 * i40e_vsi_configure_rx - Configure the VSI for Rx 3741 * @vsi: the VSI being configured 3742 * 3743 * Configure the Rx VSI for operation. 3744 **/ 3745 static int i40e_vsi_configure_rx(struct i40e_vsi *vsi) 3746 { 3747 int err = 0; 3748 u16 i; 3749 3750 vsi->max_frame = i40e_max_vsi_frame_size(vsi, vsi->xdp_prog); 3751 vsi->rx_buf_len = i40e_calculate_vsi_rx_buf_len(vsi); 3752 3753 #if (PAGE_SIZE < 8192) 3754 if (vsi->netdev && !I40E_2K_TOO_SMALL_WITH_PADDING && 3755 vsi->netdev->mtu <= ETH_DATA_LEN) { 3756 vsi->rx_buf_len = I40E_RXBUFFER_1536 - NET_IP_ALIGN; 3757 vsi->max_frame = vsi->rx_buf_len; 3758 } 3759 #endif 3760 3761 /* set up individual rings */ 3762 for (i = 0; i < vsi->num_queue_pairs && !err; i++) 3763 err = i40e_configure_rx_ring(vsi->rx_rings[i]); 3764 3765 return err; 3766 } 3767 3768 /** 3769 * i40e_vsi_config_dcb_rings - Update rings to reflect DCB TC 3770 * @vsi: ptr to the VSI 3771 **/ 3772 static void i40e_vsi_config_dcb_rings(struct i40e_vsi *vsi) 3773 { 3774 struct i40e_ring *tx_ring, *rx_ring; 3775 u16 qoffset, qcount; 3776 int i, n; 3777 3778 if (!test_bit(I40E_FLAG_DCB_ENA, vsi->back->flags)) { 3779 /* Reset the TC information */ 3780 for (i = 0; i < vsi->num_queue_pairs; i++) { 3781 rx_ring = vsi->rx_rings[i]; 3782 tx_ring = vsi->tx_rings[i]; 3783 rx_ring->dcb_tc = 0; 3784 tx_ring->dcb_tc = 0; 3785 } 3786 return; 3787 } 3788 3789 for (n = 0; n < I40E_MAX_TRAFFIC_CLASS; n++) { 3790 if (!(vsi->tc_config.enabled_tc & BIT_ULL(n))) 3791 continue; 3792 3793 qoffset = vsi->tc_config.tc_info[n].qoffset; 3794 qcount = vsi->tc_config.tc_info[n].qcount; 3795 for (i = qoffset; i < (qoffset + qcount); i++) { 3796 rx_ring = vsi->rx_rings[i]; 3797 tx_ring = vsi->tx_rings[i]; 3798 rx_ring->dcb_tc = n; 3799 tx_ring->dcb_tc = n; 3800 } 3801 } 3802 } 3803 3804 /** 3805 * i40e_set_vsi_rx_mode - Call set_rx_mode on a VSI 3806 * @vsi: ptr to the VSI 3807 **/ 3808 static void i40e_set_vsi_rx_mode(struct i40e_vsi *vsi) 3809 { 3810 if (vsi->netdev) 3811 i40e_set_rx_mode(vsi->netdev); 3812 } 3813 3814 /** 3815 * i40e_reset_fdir_filter_cnt - Reset flow director filter counters 3816 * @pf: Pointer to the targeted PF 3817 * 3818 * Set all flow director counters to 0. 3819 */ 3820 static void i40e_reset_fdir_filter_cnt(struct i40e_pf *pf) 3821 { 3822 pf->fd_tcp4_filter_cnt = 0; 3823 pf->fd_udp4_filter_cnt = 0; 3824 pf->fd_sctp4_filter_cnt = 0; 3825 pf->fd_ip4_filter_cnt = 0; 3826 pf->fd_tcp6_filter_cnt = 0; 3827 pf->fd_udp6_filter_cnt = 0; 3828 pf->fd_sctp6_filter_cnt = 0; 3829 pf->fd_ip6_filter_cnt = 0; 3830 } 3831 3832 /** 3833 * i40e_fdir_filter_restore - Restore the Sideband Flow Director filters 3834 * @vsi: Pointer to the targeted VSI 3835 * 3836 * This function replays the hlist on the hw where all the SB Flow Director 3837 * filters were saved. 3838 **/ 3839 static void i40e_fdir_filter_restore(struct i40e_vsi *vsi) 3840 { 3841 struct i40e_fdir_filter *filter; 3842 struct i40e_pf *pf = vsi->back; 3843 struct hlist_node *node; 3844 3845 if (!test_bit(I40E_FLAG_FD_SB_ENA, pf->flags)) 3846 return; 3847 3848 /* Reset FDir counters as we're replaying all existing filters */ 3849 i40e_reset_fdir_filter_cnt(pf); 3850 3851 hlist_for_each_entry_safe(filter, node, 3852 &pf->fdir_filter_list, fdir_node) { 3853 i40e_add_del_fdir(vsi, filter, true); 3854 } 3855 } 3856 3857 /** 3858 * i40e_vsi_configure - Set up the VSI for action 3859 * @vsi: the VSI being configured 3860 **/ 3861 static int i40e_vsi_configure(struct i40e_vsi *vsi) 3862 { 3863 int err; 3864 3865 i40e_set_vsi_rx_mode(vsi); 3866 i40e_restore_vlan(vsi); 3867 i40e_vsi_config_dcb_rings(vsi); 3868 err = i40e_vsi_configure_tx(vsi); 3869 if (!err) 3870 err = i40e_vsi_configure_rx(vsi); 3871 3872 return err; 3873 } 3874 3875 /** 3876 * i40e_vsi_configure_msix - MSIX mode Interrupt Config in the HW 3877 * @vsi: the VSI being configured 3878 **/ 3879 static void i40e_vsi_configure_msix(struct i40e_vsi *vsi) 3880 { 3881 bool has_xdp = i40e_enabled_xdp_vsi(vsi); 3882 struct i40e_pf *pf = vsi->back; 3883 struct i40e_hw *hw = &pf->hw; 3884 u16 vector; 3885 int i, q; 3886 u32 qp; 3887 3888 /* The interrupt indexing is offset by 1 in the PFINT_ITRn 3889 * and PFINT_LNKLSTn registers, e.g.: 3890 * PFINT_ITRn[0..n-1] gets msix-1..msix-n (qpair interrupts) 3891 */ 3892 qp = vsi->base_queue; 3893 vector = vsi->base_vector; 3894 for (i = 0; i < vsi->num_q_vectors; i++, vector++) { 3895 struct i40e_q_vector *q_vector = vsi->q_vectors[i]; 3896 3897 q_vector->rx.next_update = jiffies + 1; 3898 q_vector->rx.target_itr = 3899 ITR_TO_REG(vsi->rx_rings[i]->itr_setting); 3900 wr32(hw, I40E_PFINT_ITRN(I40E_RX_ITR, vector - 1), 3901 q_vector->rx.target_itr >> 1); 3902 q_vector->rx.current_itr = q_vector->rx.target_itr; 3903 3904 q_vector->tx.next_update = jiffies + 1; 3905 q_vector->tx.target_itr = 3906 ITR_TO_REG(vsi->tx_rings[i]->itr_setting); 3907 wr32(hw, I40E_PFINT_ITRN(I40E_TX_ITR, vector - 1), 3908 q_vector->tx.target_itr >> 1); 3909 q_vector->tx.current_itr = q_vector->tx.target_itr; 3910 3911 /* Set ITR for software interrupts triggered after exiting 3912 * busy-loop polling. 3913 */ 3914 wr32(hw, I40E_PFINT_ITRN(I40E_SW_ITR, vector - 1), 3915 I40E_ITR_20K); 3916 3917 wr32(hw, I40E_PFINT_RATEN(vector - 1), 3918 i40e_intrl_usec_to_reg(vsi->int_rate_limit)); 3919 3920 /* begin of linked list for RX queue assigned to this vector */ 3921 wr32(hw, I40E_PFINT_LNKLSTN(vector - 1), qp); 3922 for (q = 0; q < q_vector->num_ringpairs; q++) { 3923 u32 nextqp = has_xdp ? qp + vsi->alloc_queue_pairs : qp; 3924 u32 val; 3925 3926 val = I40E_QINT_RQCTL_CAUSE_ENA_MASK | 3927 (I40E_RX_ITR << I40E_QINT_RQCTL_ITR_INDX_SHIFT) | 3928 (vector << I40E_QINT_RQCTL_MSIX_INDX_SHIFT) | 3929 (nextqp << I40E_QINT_RQCTL_NEXTQ_INDX_SHIFT) | 3930 (I40E_QUEUE_TYPE_TX << 3931 I40E_QINT_RQCTL_NEXTQ_TYPE_SHIFT); 3932 3933 wr32(hw, I40E_QINT_RQCTL(qp), val); 3934 3935 if (has_xdp) { 3936 /* TX queue with next queue set to TX */ 3937 val = I40E_QINT_TQCTL_CAUSE_ENA_MASK | 3938 (I40E_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT) | 3939 (vector << I40E_QINT_TQCTL_MSIX_INDX_SHIFT) | 3940 (qp << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT) | 3941 (I40E_QUEUE_TYPE_TX << 3942 I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT); 3943 3944 wr32(hw, I40E_QINT_TQCTL(nextqp), val); 3945 } 3946 /* TX queue with next RX or end of linked list */ 3947 val = I40E_QINT_TQCTL_CAUSE_ENA_MASK | 3948 (I40E_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT) | 3949 (vector << I40E_QINT_TQCTL_MSIX_INDX_SHIFT) | 3950 ((qp + 1) << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT) | 3951 (I40E_QUEUE_TYPE_RX << 3952 I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT); 3953 3954 /* Terminate the linked list */ 3955 if (q == (q_vector->num_ringpairs - 1)) 3956 val |= (I40E_QUEUE_END_OF_LIST << 3957 I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT); 3958 3959 wr32(hw, I40E_QINT_TQCTL(qp), val); 3960 qp++; 3961 } 3962 } 3963 3964 i40e_flush(hw); 3965 } 3966 3967 /** 3968 * i40e_enable_misc_int_causes - enable the non-queue interrupts 3969 * @pf: pointer to private device data structure 3970 **/ 3971 static void i40e_enable_misc_int_causes(struct i40e_pf *pf) 3972 { 3973 struct i40e_hw *hw = &pf->hw; 3974 u32 val; 3975 3976 /* clear things first */ 3977 wr32(hw, I40E_PFINT_ICR0_ENA, 0); /* disable all */ 3978 rd32(hw, I40E_PFINT_ICR0); /* read to clear */ 3979 3980 val = I40E_PFINT_ICR0_ENA_ECC_ERR_MASK | 3981 I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK | 3982 I40E_PFINT_ICR0_ENA_GRST_MASK | 3983 I40E_PFINT_ICR0_ENA_PCI_EXCEPTION_MASK | 3984 I40E_PFINT_ICR0_ENA_GPIO_MASK | 3985 I40E_PFINT_ICR0_ENA_HMC_ERR_MASK | 3986 I40E_PFINT_ICR0_ENA_VFLR_MASK | 3987 I40E_PFINT_ICR0_ENA_ADMINQ_MASK; 3988 3989 if (test_bit(I40E_FLAG_IWARP_ENA, pf->flags)) 3990 val |= I40E_PFINT_ICR0_ENA_PE_CRITERR_MASK; 3991 3992 if (test_bit(I40E_FLAG_PTP_ENA, pf->flags)) 3993 val |= I40E_PFINT_ICR0_ENA_TIMESYNC_MASK; 3994 3995 wr32(hw, I40E_PFINT_ICR0_ENA, val); 3996 3997 /* SW_ITR_IDX = 0, but don't change INTENA */ 3998 wr32(hw, I40E_PFINT_DYN_CTL0, I40E_PFINT_DYN_CTL0_SW_ITR_INDX_MASK | 3999 I40E_PFINT_DYN_CTL0_INTENA_MSK_MASK); 4000 4001 /* OTHER_ITR_IDX = 0 */ 4002 wr32(hw, I40E_PFINT_STAT_CTL0, 0); 4003 } 4004 4005 /** 4006 * i40e_configure_msi_and_legacy - Legacy mode interrupt config in the HW 4007 * @vsi: the VSI being configured 4008 **/ 4009 static void i40e_configure_msi_and_legacy(struct i40e_vsi *vsi) 4010 { 4011 u32 nextqp = i40e_enabled_xdp_vsi(vsi) ? vsi->alloc_queue_pairs : 0; 4012 struct i40e_q_vector *q_vector = vsi->q_vectors[0]; 4013 struct i40e_pf *pf = vsi->back; 4014 struct i40e_hw *hw = &pf->hw; 4015 4016 /* set the ITR configuration */ 4017 q_vector->rx.next_update = jiffies + 1; 4018 q_vector->rx.target_itr = ITR_TO_REG(vsi->rx_rings[0]->itr_setting); 4019 wr32(hw, I40E_PFINT_ITR0(I40E_RX_ITR), q_vector->rx.target_itr >> 1); 4020 q_vector->rx.current_itr = q_vector->rx.target_itr; 4021 q_vector->tx.next_update = jiffies + 1; 4022 q_vector->tx.target_itr = ITR_TO_REG(vsi->tx_rings[0]->itr_setting); 4023 wr32(hw, I40E_PFINT_ITR0(I40E_TX_ITR), q_vector->tx.target_itr >> 1); 4024 q_vector->tx.current_itr = q_vector->tx.target_itr; 4025 4026 i40e_enable_misc_int_causes(pf); 4027 4028 /* FIRSTQ_INDX = 0, FIRSTQ_TYPE = 0 (rx) */ 4029 wr32(hw, I40E_PFINT_LNKLST0, 0); 4030 4031 /* Associate the queue pair to the vector and enable the queue 4032 * interrupt RX queue in linked list with next queue set to TX 4033 */ 4034 wr32(hw, I40E_QINT_RQCTL(0), I40E_QINT_RQCTL_VAL(nextqp, 0, TX)); 4035 4036 if (i40e_enabled_xdp_vsi(vsi)) { 4037 /* TX queue in linked list with next queue set to TX */ 4038 wr32(hw, I40E_QINT_TQCTL(nextqp), 4039 I40E_QINT_TQCTL_VAL(nextqp, 0, TX)); 4040 } 4041 4042 /* last TX queue so the next RX queue doesn't matter */ 4043 wr32(hw, I40E_QINT_TQCTL(0), 4044 I40E_QINT_TQCTL_VAL(I40E_QUEUE_END_OF_LIST, 0, RX)); 4045 i40e_flush(hw); 4046 } 4047 4048 /** 4049 * i40e_irq_dynamic_disable_icr0 - Disable default interrupt generation for icr0 4050 * @pf: board private structure 4051 **/ 4052 void i40e_irq_dynamic_disable_icr0(struct i40e_pf *pf) 4053 { 4054 struct i40e_hw *hw = &pf->hw; 4055 4056 wr32(hw, I40E_PFINT_DYN_CTL0, 4057 I40E_ITR_NONE << I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT); 4058 i40e_flush(hw); 4059 } 4060 4061 /** 4062 * i40e_irq_dynamic_enable_icr0 - Enable default interrupt generation for icr0 4063 * @pf: board private structure 4064 **/ 4065 void i40e_irq_dynamic_enable_icr0(struct i40e_pf *pf) 4066 { 4067 struct i40e_hw *hw = &pf->hw; 4068 u32 val; 4069 4070 val = I40E_PFINT_DYN_CTL0_INTENA_MASK | 4071 I40E_PFINT_DYN_CTL0_CLEARPBA_MASK | 4072 (I40E_ITR_NONE << I40E_PFINT_DYN_CTL0_ITR_INDX_SHIFT); 4073 4074 wr32(hw, I40E_PFINT_DYN_CTL0, val); 4075 i40e_flush(hw); 4076 } 4077 4078 /** 4079 * i40e_msix_clean_rings - MSIX mode Interrupt Handler 4080 * @irq: interrupt number 4081 * @data: pointer to a q_vector 4082 **/ 4083 static irqreturn_t i40e_msix_clean_rings(int irq, void *data) 4084 { 4085 struct i40e_q_vector *q_vector = data; 4086 4087 if (!q_vector->tx.ring && !q_vector->rx.ring) 4088 return IRQ_HANDLED; 4089 4090 napi_schedule_irqoff(&q_vector->napi); 4091 4092 return IRQ_HANDLED; 4093 } 4094 4095 /** 4096 * i40e_irq_affinity_notify - Callback for affinity changes 4097 * @notify: context as to what irq was changed 4098 * @mask: the new affinity mask 4099 * 4100 * This is a callback function used by the irq_set_affinity_notifier function 4101 * so that we may register to receive changes to the irq affinity masks. 4102 **/ 4103 static void i40e_irq_affinity_notify(struct irq_affinity_notify *notify, 4104 const cpumask_t *mask) 4105 { 4106 struct i40e_q_vector *q_vector = 4107 container_of(notify, struct i40e_q_vector, affinity_notify); 4108 4109 cpumask_copy(&q_vector->affinity_mask, mask); 4110 } 4111 4112 /** 4113 * i40e_irq_affinity_release - Callback for affinity notifier release 4114 * @ref: internal core kernel usage 4115 * 4116 * This is a callback function used by the irq_set_affinity_notifier function 4117 * to inform the current notification subscriber that they will no longer 4118 * receive notifications. 4119 **/ 4120 static void i40e_irq_affinity_release(struct kref *ref) {} 4121 4122 /** 4123 * i40e_vsi_request_irq_msix - Initialize MSI-X interrupts 4124 * @vsi: the VSI being configured 4125 * @basename: name for the vector 4126 * 4127 * Allocates MSI-X vectors and requests interrupts from the kernel. 4128 **/ 4129 static int i40e_vsi_request_irq_msix(struct i40e_vsi *vsi, char *basename) 4130 { 4131 int q_vectors = vsi->num_q_vectors; 4132 struct i40e_pf *pf = vsi->back; 4133 int base = vsi->base_vector; 4134 int rx_int_idx = 0; 4135 int tx_int_idx = 0; 4136 int vector, err; 4137 int irq_num; 4138 int cpu; 4139 4140 for (vector = 0; vector < q_vectors; vector++) { 4141 struct i40e_q_vector *q_vector = vsi->q_vectors[vector]; 4142 4143 irq_num = pf->msix_entries[base + vector].vector; 4144 4145 if (q_vector->tx.ring && q_vector->rx.ring) { 4146 snprintf(q_vector->name, sizeof(q_vector->name) - 1, 4147 "%s-%s-%d", basename, "TxRx", rx_int_idx++); 4148 tx_int_idx++; 4149 } else if (q_vector->rx.ring) { 4150 snprintf(q_vector->name, sizeof(q_vector->name) - 1, 4151 "%s-%s-%d", basename, "rx", rx_int_idx++); 4152 } else if (q_vector->tx.ring) { 4153 snprintf(q_vector->name, sizeof(q_vector->name) - 1, 4154 "%s-%s-%d", basename, "tx", tx_int_idx++); 4155 } else { 4156 /* skip this unused q_vector */ 4157 continue; 4158 } 4159 err = request_irq(irq_num, 4160 vsi->irq_handler, 4161 0, 4162 q_vector->name, 4163 q_vector); 4164 if (err) { 4165 dev_info(&pf->pdev->dev, 4166 "MSIX request_irq failed, error: %d\n", err); 4167 goto free_queue_irqs; 4168 } 4169 4170 /* register for affinity change notifications */ 4171 q_vector->irq_num = irq_num; 4172 q_vector->affinity_notify.notify = i40e_irq_affinity_notify; 4173 q_vector->affinity_notify.release = i40e_irq_affinity_release; 4174 irq_set_affinity_notifier(irq_num, &q_vector->affinity_notify); 4175 /* Spread affinity hints out across online CPUs. 4176 * 4177 * get_cpu_mask returns a static constant mask with 4178 * a permanent lifetime so it's ok to pass to 4179 * irq_update_affinity_hint without making a copy. 4180 */ 4181 cpu = cpumask_local_spread(q_vector->v_idx, -1); 4182 irq_update_affinity_hint(irq_num, get_cpu_mask(cpu)); 4183 } 4184 4185 vsi->irqs_ready = true; 4186 return 0; 4187 4188 free_queue_irqs: 4189 while (vector) { 4190 vector--; 4191 irq_num = pf->msix_entries[base + vector].vector; 4192 irq_set_affinity_notifier(irq_num, NULL); 4193 irq_update_affinity_hint(irq_num, NULL); 4194 free_irq(irq_num, vsi->q_vectors[vector]); 4195 } 4196 return err; 4197 } 4198 4199 /** 4200 * i40e_vsi_disable_irq - Mask off queue interrupt generation on the VSI 4201 * @vsi: the VSI being un-configured 4202 **/ 4203 static void i40e_vsi_disable_irq(struct i40e_vsi *vsi) 4204 { 4205 struct i40e_pf *pf = vsi->back; 4206 struct i40e_hw *hw = &pf->hw; 4207 int base = vsi->base_vector; 4208 int i; 4209 4210 /* disable interrupt causation from each queue */ 4211 for (i = 0; i < vsi->num_queue_pairs; i++) { 4212 u32 val; 4213 4214 val = rd32(hw, I40E_QINT_TQCTL(vsi->tx_rings[i]->reg_idx)); 4215 val &= ~I40E_QINT_TQCTL_CAUSE_ENA_MASK; 4216 wr32(hw, I40E_QINT_TQCTL(vsi->tx_rings[i]->reg_idx), val); 4217 4218 val = rd32(hw, I40E_QINT_RQCTL(vsi->rx_rings[i]->reg_idx)); 4219 val &= ~I40E_QINT_RQCTL_CAUSE_ENA_MASK; 4220 wr32(hw, I40E_QINT_RQCTL(vsi->rx_rings[i]->reg_idx), val); 4221 4222 if (!i40e_enabled_xdp_vsi(vsi)) 4223 continue; 4224 wr32(hw, I40E_QINT_TQCTL(vsi->xdp_rings[i]->reg_idx), 0); 4225 } 4226 4227 /* disable each interrupt */ 4228 if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags)) { 4229 for (i = vsi->base_vector; 4230 i < (vsi->num_q_vectors + vsi->base_vector); i++) 4231 wr32(hw, I40E_PFINT_DYN_CTLN(i - 1), 0); 4232 4233 i40e_flush(hw); 4234 for (i = 0; i < vsi->num_q_vectors; i++) 4235 synchronize_irq(pf->msix_entries[i + base].vector); 4236 } else { 4237 /* Legacy and MSI mode - this stops all interrupt handling */ 4238 wr32(hw, I40E_PFINT_ICR0_ENA, 0); 4239 wr32(hw, I40E_PFINT_DYN_CTL0, 0); 4240 i40e_flush(hw); 4241 synchronize_irq(pf->pdev->irq); 4242 } 4243 } 4244 4245 /** 4246 * i40e_vsi_enable_irq - Enable IRQ for the given VSI 4247 * @vsi: the VSI being configured 4248 **/ 4249 static int i40e_vsi_enable_irq(struct i40e_vsi *vsi) 4250 { 4251 struct i40e_pf *pf = vsi->back; 4252 int i; 4253 4254 if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags)) { 4255 for (i = 0; i < vsi->num_q_vectors; i++) 4256 i40e_irq_dynamic_enable(vsi, i); 4257 } else { 4258 i40e_irq_dynamic_enable_icr0(pf); 4259 } 4260 4261 i40e_flush(&pf->hw); 4262 return 0; 4263 } 4264 4265 /** 4266 * i40e_free_misc_vector - Free the vector that handles non-queue events 4267 * @pf: board private structure 4268 **/ 4269 static void i40e_free_misc_vector(struct i40e_pf *pf) 4270 { 4271 /* Disable ICR 0 */ 4272 wr32(&pf->hw, I40E_PFINT_ICR0_ENA, 0); 4273 i40e_flush(&pf->hw); 4274 4275 if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags) && pf->msix_entries) { 4276 free_irq(pf->msix_entries[0].vector, pf); 4277 clear_bit(__I40E_MISC_IRQ_REQUESTED, pf->state); 4278 } 4279 } 4280 4281 /** 4282 * i40e_intr - MSI/Legacy and non-queue interrupt handler 4283 * @irq: interrupt number 4284 * @data: pointer to a q_vector 4285 * 4286 * This is the handler used for all MSI/Legacy interrupts, and deals 4287 * with both queue and non-queue interrupts. This is also used in 4288 * MSIX mode to handle the non-queue interrupts. 4289 **/ 4290 static irqreturn_t i40e_intr(int irq, void *data) 4291 { 4292 struct i40e_pf *pf = (struct i40e_pf *)data; 4293 struct i40e_hw *hw = &pf->hw; 4294 irqreturn_t ret = IRQ_NONE; 4295 u32 icr0, icr0_remaining; 4296 u32 val, ena_mask; 4297 4298 icr0 = rd32(hw, I40E_PFINT_ICR0); 4299 ena_mask = rd32(hw, I40E_PFINT_ICR0_ENA); 4300 4301 /* if sharing a legacy IRQ, we might get called w/o an intr pending */ 4302 if ((icr0 & I40E_PFINT_ICR0_INTEVENT_MASK) == 0) 4303 goto enable_intr; 4304 4305 /* if interrupt but no bits showing, must be SWINT */ 4306 if (((icr0 & ~I40E_PFINT_ICR0_INTEVENT_MASK) == 0) || 4307 (icr0 & I40E_PFINT_ICR0_SWINT_MASK)) 4308 pf->sw_int_count++; 4309 4310 if (test_bit(I40E_FLAG_IWARP_ENA, pf->flags) && 4311 (icr0 & I40E_PFINT_ICR0_ENA_PE_CRITERR_MASK)) { 4312 ena_mask &= ~I40E_PFINT_ICR0_ENA_PE_CRITERR_MASK; 4313 dev_dbg(&pf->pdev->dev, "cleared PE_CRITERR\n"); 4314 set_bit(__I40E_CORE_RESET_REQUESTED, pf->state); 4315 } 4316 4317 /* only q0 is used in MSI/Legacy mode, and none are used in MSIX */ 4318 if (icr0 & I40E_PFINT_ICR0_QUEUE_0_MASK) { 4319 struct i40e_vsi *vsi = i40e_pf_get_main_vsi(pf); 4320 struct i40e_q_vector *q_vector = vsi->q_vectors[0]; 4321 4322 /* We do not have a way to disarm Queue causes while leaving 4323 * interrupt enabled for all other causes, ideally 4324 * interrupt should be disabled while we are in NAPI but 4325 * this is not a performance path and napi_schedule() 4326 * can deal with rescheduling. 4327 */ 4328 if (!test_bit(__I40E_DOWN, pf->state)) 4329 napi_schedule_irqoff(&q_vector->napi); 4330 } 4331 4332 if (icr0 & I40E_PFINT_ICR0_ADMINQ_MASK) { 4333 ena_mask &= ~I40E_PFINT_ICR0_ENA_ADMINQ_MASK; 4334 set_bit(__I40E_ADMINQ_EVENT_PENDING, pf->state); 4335 i40e_debug(&pf->hw, I40E_DEBUG_NVM, "AdminQ event\n"); 4336 } 4337 4338 if (icr0 & I40E_PFINT_ICR0_MAL_DETECT_MASK) { 4339 ena_mask &= ~I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK; 4340 set_bit(__I40E_MDD_EVENT_PENDING, pf->state); 4341 } 4342 4343 if (icr0 & I40E_PFINT_ICR0_VFLR_MASK) { 4344 /* disable any further VFLR event notifications */ 4345 if (test_bit(__I40E_VF_RESETS_DISABLED, pf->state)) { 4346 u32 reg = rd32(hw, I40E_PFINT_ICR0_ENA); 4347 4348 reg &= ~I40E_PFINT_ICR0_VFLR_MASK; 4349 wr32(hw, I40E_PFINT_ICR0_ENA, reg); 4350 } else { 4351 ena_mask &= ~I40E_PFINT_ICR0_ENA_VFLR_MASK; 4352 set_bit(__I40E_VFLR_EVENT_PENDING, pf->state); 4353 } 4354 } 4355 4356 if (icr0 & I40E_PFINT_ICR0_GRST_MASK) { 4357 if (!test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state)) 4358 set_bit(__I40E_RESET_INTR_RECEIVED, pf->state); 4359 ena_mask &= ~I40E_PFINT_ICR0_ENA_GRST_MASK; 4360 val = rd32(hw, I40E_GLGEN_RSTAT); 4361 val = FIELD_GET(I40E_GLGEN_RSTAT_RESET_TYPE_MASK, val); 4362 if (val == I40E_RESET_CORER) { 4363 pf->corer_count++; 4364 } else if (val == I40E_RESET_GLOBR) { 4365 pf->globr_count++; 4366 } else if (val == I40E_RESET_EMPR) { 4367 pf->empr_count++; 4368 set_bit(__I40E_EMP_RESET_INTR_RECEIVED, pf->state); 4369 } 4370 } 4371 4372 if (icr0 & I40E_PFINT_ICR0_HMC_ERR_MASK) { 4373 icr0 &= ~I40E_PFINT_ICR0_HMC_ERR_MASK; 4374 dev_info(&pf->pdev->dev, "HMC error interrupt\n"); 4375 dev_info(&pf->pdev->dev, "HMC error info 0x%x, HMC error data 0x%x\n", 4376 rd32(hw, I40E_PFHMC_ERRORINFO), 4377 rd32(hw, I40E_PFHMC_ERRORDATA)); 4378 } 4379 4380 if (icr0 & I40E_PFINT_ICR0_TIMESYNC_MASK) { 4381 u32 prttsyn_stat = rd32(hw, I40E_PRTTSYN_STAT_0); 4382 4383 if (prttsyn_stat & I40E_PRTTSYN_STAT_0_EVENT0_MASK) 4384 schedule_work(&pf->ptp_extts0_work); 4385 4386 if (prttsyn_stat & I40E_PRTTSYN_STAT_0_TXTIME_MASK) 4387 i40e_ptp_tx_hwtstamp(pf); 4388 4389 icr0 &= ~I40E_PFINT_ICR0_ENA_TIMESYNC_MASK; 4390 } 4391 4392 /* If a critical error is pending we have no choice but to reset the 4393 * device. 4394 * Report and mask out any remaining unexpected interrupts. 4395 */ 4396 icr0_remaining = icr0 & ena_mask; 4397 if (icr0_remaining) { 4398 dev_info(&pf->pdev->dev, "unhandled interrupt icr0=0x%08x\n", 4399 icr0_remaining); 4400 if ((icr0_remaining & I40E_PFINT_ICR0_PE_CRITERR_MASK) || 4401 (icr0_remaining & I40E_PFINT_ICR0_PCI_EXCEPTION_MASK) || 4402 (icr0_remaining & I40E_PFINT_ICR0_ECC_ERR_MASK)) { 4403 dev_info(&pf->pdev->dev, "device will be reset\n"); 4404 set_bit(__I40E_PF_RESET_REQUESTED, pf->state); 4405 i40e_service_event_schedule(pf); 4406 } 4407 ena_mask &= ~icr0_remaining; 4408 } 4409 ret = IRQ_HANDLED; 4410 4411 enable_intr: 4412 /* re-enable interrupt causes */ 4413 wr32(hw, I40E_PFINT_ICR0_ENA, ena_mask); 4414 if (!test_bit(__I40E_DOWN, pf->state) || 4415 test_bit(__I40E_RECOVERY_MODE, pf->state)) { 4416 i40e_service_event_schedule(pf); 4417 i40e_irq_dynamic_enable_icr0(pf); 4418 } 4419 4420 return ret; 4421 } 4422 4423 /** 4424 * i40e_clean_fdir_tx_irq - Reclaim resources after transmit completes 4425 * @tx_ring: tx ring to clean 4426 * @budget: how many cleans we're allowed 4427 * 4428 * Returns true if there's any budget left (e.g. the clean is finished) 4429 **/ 4430 static bool i40e_clean_fdir_tx_irq(struct i40e_ring *tx_ring, int budget) 4431 { 4432 struct i40e_vsi *vsi = tx_ring->vsi; 4433 u16 i = tx_ring->next_to_clean; 4434 struct i40e_tx_buffer *tx_buf; 4435 struct i40e_tx_desc *tx_desc; 4436 4437 tx_buf = &tx_ring->tx_bi[i]; 4438 tx_desc = I40E_TX_DESC(tx_ring, i); 4439 i -= tx_ring->count; 4440 4441 do { 4442 struct i40e_tx_desc *eop_desc = tx_buf->next_to_watch; 4443 4444 /* if next_to_watch is not set then there is no work pending */ 4445 if (!eop_desc) 4446 break; 4447 4448 /* prevent any other reads prior to eop_desc */ 4449 smp_rmb(); 4450 4451 /* if the descriptor isn't done, no work yet to do */ 4452 if (!(eop_desc->cmd_type_offset_bsz & 4453 cpu_to_le64(I40E_TX_DESC_DTYPE_DESC_DONE))) 4454 break; 4455 4456 /* clear next_to_watch to prevent false hangs */ 4457 tx_buf->next_to_watch = NULL; 4458 4459 tx_desc->buffer_addr = 0; 4460 tx_desc->cmd_type_offset_bsz = 0; 4461 /* move past filter desc */ 4462 tx_buf++; 4463 tx_desc++; 4464 i++; 4465 if (unlikely(!i)) { 4466 i -= tx_ring->count; 4467 tx_buf = tx_ring->tx_bi; 4468 tx_desc = I40E_TX_DESC(tx_ring, 0); 4469 } 4470 /* unmap skb header data */ 4471 dma_unmap_single(tx_ring->dev, 4472 dma_unmap_addr(tx_buf, dma), 4473 dma_unmap_len(tx_buf, len), 4474 DMA_TO_DEVICE); 4475 if (tx_buf->tx_flags & I40E_TX_FLAGS_FD_SB) 4476 kfree(tx_buf->raw_buf); 4477 4478 tx_buf->raw_buf = NULL; 4479 tx_buf->tx_flags = 0; 4480 tx_buf->next_to_watch = NULL; 4481 dma_unmap_len_set(tx_buf, len, 0); 4482 tx_desc->buffer_addr = 0; 4483 tx_desc->cmd_type_offset_bsz = 0; 4484 4485 /* move us past the eop_desc for start of next FD desc */ 4486 tx_buf++; 4487 tx_desc++; 4488 i++; 4489 if (unlikely(!i)) { 4490 i -= tx_ring->count; 4491 tx_buf = tx_ring->tx_bi; 4492 tx_desc = I40E_TX_DESC(tx_ring, 0); 4493 } 4494 4495 /* update budget accounting */ 4496 budget--; 4497 } while (likely(budget)); 4498 4499 i += tx_ring->count; 4500 tx_ring->next_to_clean = i; 4501 4502 if (test_bit(I40E_FLAG_MSIX_ENA, vsi->back->flags)) 4503 i40e_irq_dynamic_enable(vsi, tx_ring->q_vector->v_idx); 4504 4505 return budget > 0; 4506 } 4507 4508 /** 4509 * i40e_fdir_clean_ring - Interrupt Handler for FDIR SB ring 4510 * @irq: interrupt number 4511 * @data: pointer to a q_vector 4512 **/ 4513 static irqreturn_t i40e_fdir_clean_ring(int irq, void *data) 4514 { 4515 struct i40e_q_vector *q_vector = data; 4516 struct i40e_vsi *vsi; 4517 4518 if (!q_vector->tx.ring) 4519 return IRQ_HANDLED; 4520 4521 vsi = q_vector->tx.ring->vsi; 4522 i40e_clean_fdir_tx_irq(q_vector->tx.ring, vsi->work_limit); 4523 4524 return IRQ_HANDLED; 4525 } 4526 4527 /** 4528 * i40e_map_vector_to_qp - Assigns the queue pair to the vector 4529 * @vsi: the VSI being configured 4530 * @v_idx: vector index 4531 * @qp_idx: queue pair index 4532 **/ 4533 static void i40e_map_vector_to_qp(struct i40e_vsi *vsi, int v_idx, int qp_idx) 4534 { 4535 struct i40e_q_vector *q_vector = vsi->q_vectors[v_idx]; 4536 struct i40e_ring *tx_ring = vsi->tx_rings[qp_idx]; 4537 struct i40e_ring *rx_ring = vsi->rx_rings[qp_idx]; 4538 4539 tx_ring->q_vector = q_vector; 4540 tx_ring->next = q_vector->tx.ring; 4541 q_vector->tx.ring = tx_ring; 4542 q_vector->tx.count++; 4543 4544 /* Place XDP Tx ring in the same q_vector ring list as regular Tx */ 4545 if (i40e_enabled_xdp_vsi(vsi)) { 4546 struct i40e_ring *xdp_ring = vsi->xdp_rings[qp_idx]; 4547 4548 xdp_ring->q_vector = q_vector; 4549 xdp_ring->next = q_vector->tx.ring; 4550 q_vector->tx.ring = xdp_ring; 4551 q_vector->tx.count++; 4552 } 4553 4554 rx_ring->q_vector = q_vector; 4555 rx_ring->next = q_vector->rx.ring; 4556 q_vector->rx.ring = rx_ring; 4557 q_vector->rx.count++; 4558 } 4559 4560 /** 4561 * i40e_vsi_map_rings_to_vectors - Maps descriptor rings to vectors 4562 * @vsi: the VSI being configured 4563 * 4564 * This function maps descriptor rings to the queue-specific vectors 4565 * we were allotted through the MSI-X enabling code. Ideally, we'd have 4566 * one vector per queue pair, but on a constrained vector budget, we 4567 * group the queue pairs as "efficiently" as possible. 4568 **/ 4569 static void i40e_vsi_map_rings_to_vectors(struct i40e_vsi *vsi) 4570 { 4571 int qp_remaining = vsi->num_queue_pairs; 4572 int q_vectors = vsi->num_q_vectors; 4573 int num_ringpairs; 4574 int v_start = 0; 4575 int qp_idx = 0; 4576 4577 /* If we don't have enough vectors for a 1-to-1 mapping, we'll have to 4578 * group them so there are multiple queues per vector. 4579 * It is also important to go through all the vectors available to be 4580 * sure that if we don't use all the vectors, that the remaining vectors 4581 * are cleared. This is especially important when decreasing the 4582 * number of queues in use. 4583 */ 4584 for (; v_start < q_vectors; v_start++) { 4585 struct i40e_q_vector *q_vector = vsi->q_vectors[v_start]; 4586 4587 num_ringpairs = DIV_ROUND_UP(qp_remaining, q_vectors - v_start); 4588 4589 q_vector->num_ringpairs = num_ringpairs; 4590 q_vector->reg_idx = q_vector->v_idx + vsi->base_vector - 1; 4591 4592 q_vector->rx.count = 0; 4593 q_vector->tx.count = 0; 4594 q_vector->rx.ring = NULL; 4595 q_vector->tx.ring = NULL; 4596 4597 while (num_ringpairs--) { 4598 i40e_map_vector_to_qp(vsi, v_start, qp_idx); 4599 qp_idx++; 4600 qp_remaining--; 4601 } 4602 } 4603 } 4604 4605 /** 4606 * i40e_vsi_request_irq - Request IRQ from the OS 4607 * @vsi: the VSI being configured 4608 * @basename: name for the vector 4609 **/ 4610 static int i40e_vsi_request_irq(struct i40e_vsi *vsi, char *basename) 4611 { 4612 struct i40e_pf *pf = vsi->back; 4613 int err; 4614 4615 if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags)) 4616 err = i40e_vsi_request_irq_msix(vsi, basename); 4617 else if (test_bit(I40E_FLAG_MSI_ENA, pf->flags)) 4618 err = request_irq(pf->pdev->irq, i40e_intr, 0, 4619 pf->int_name, pf); 4620 else 4621 err = request_irq(pf->pdev->irq, i40e_intr, IRQF_SHARED, 4622 pf->int_name, pf); 4623 4624 if (err) 4625 dev_info(&pf->pdev->dev, "request_irq failed, Error %d\n", err); 4626 4627 return err; 4628 } 4629 4630 #ifdef CONFIG_NET_POLL_CONTROLLER 4631 /** 4632 * i40e_netpoll - A Polling 'interrupt' handler 4633 * @netdev: network interface device structure 4634 * 4635 * This is used by netconsole to send skbs without having to re-enable 4636 * interrupts. It's not called while the normal interrupt routine is executing. 4637 **/ 4638 static void i40e_netpoll(struct net_device *netdev) 4639 { 4640 struct i40e_netdev_priv *np = netdev_priv(netdev); 4641 struct i40e_vsi *vsi = np->vsi; 4642 struct i40e_pf *pf = vsi->back; 4643 int i; 4644 4645 /* if interface is down do nothing */ 4646 if (test_bit(__I40E_VSI_DOWN, vsi->state)) 4647 return; 4648 4649 if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags)) { 4650 for (i = 0; i < vsi->num_q_vectors; i++) 4651 i40e_msix_clean_rings(0, vsi->q_vectors[i]); 4652 } else { 4653 i40e_intr(pf->pdev->irq, netdev); 4654 } 4655 } 4656 #endif 4657 4658 #define I40E_QTX_ENA_WAIT_COUNT 50 4659 4660 /** 4661 * i40e_pf_txq_wait - Wait for a PF's Tx queue to be enabled or disabled 4662 * @pf: the PF being configured 4663 * @pf_q: the PF queue 4664 * @enable: enable or disable state of the queue 4665 * 4666 * This routine will wait for the given Tx queue of the PF to reach the 4667 * enabled or disabled state. 4668 * Returns -ETIMEDOUT in case of failing to reach the requested state after 4669 * multiple retries; else will return 0 in case of success. 4670 **/ 4671 static int i40e_pf_txq_wait(struct i40e_pf *pf, int pf_q, bool enable) 4672 { 4673 int i; 4674 u32 tx_reg; 4675 4676 for (i = 0; i < I40E_QUEUE_WAIT_RETRY_LIMIT; i++) { 4677 tx_reg = rd32(&pf->hw, I40E_QTX_ENA(pf_q)); 4678 if (enable == !!(tx_reg & I40E_QTX_ENA_QENA_STAT_MASK)) 4679 break; 4680 4681 usleep_range(10, 20); 4682 } 4683 if (i >= I40E_QUEUE_WAIT_RETRY_LIMIT) 4684 return -ETIMEDOUT; 4685 4686 return 0; 4687 } 4688 4689 /** 4690 * i40e_control_tx_q - Start or stop a particular Tx queue 4691 * @pf: the PF structure 4692 * @pf_q: the PF queue to configure 4693 * @enable: start or stop the queue 4694 * 4695 * This function enables or disables a single queue. Note that any delay 4696 * required after the operation is expected to be handled by the caller of 4697 * this function. 4698 **/ 4699 static void i40e_control_tx_q(struct i40e_pf *pf, int pf_q, bool enable) 4700 { 4701 struct i40e_hw *hw = &pf->hw; 4702 u32 tx_reg; 4703 int i; 4704 4705 /* warn the TX unit of coming changes */ 4706 i40e_pre_tx_queue_cfg(&pf->hw, pf_q, enable); 4707 if (!enable) 4708 usleep_range(10, 20); 4709 4710 for (i = 0; i < I40E_QTX_ENA_WAIT_COUNT; i++) { 4711 tx_reg = rd32(hw, I40E_QTX_ENA(pf_q)); 4712 if (((tx_reg >> I40E_QTX_ENA_QENA_REQ_SHIFT) & 1) == 4713 ((tx_reg >> I40E_QTX_ENA_QENA_STAT_SHIFT) & 1)) 4714 break; 4715 usleep_range(1000, 2000); 4716 } 4717 4718 /* Skip if the queue is already in the requested state */ 4719 if (enable == !!(tx_reg & I40E_QTX_ENA_QENA_STAT_MASK)) 4720 return; 4721 4722 /* turn on/off the queue */ 4723 if (enable) { 4724 wr32(hw, I40E_QTX_HEAD(pf_q), 0); 4725 tx_reg |= I40E_QTX_ENA_QENA_REQ_MASK; 4726 } else { 4727 tx_reg &= ~I40E_QTX_ENA_QENA_REQ_MASK; 4728 } 4729 4730 wr32(hw, I40E_QTX_ENA(pf_q), tx_reg); 4731 } 4732 4733 /** 4734 * i40e_control_wait_tx_q - Start/stop Tx queue and wait for completion 4735 * @seid: VSI SEID 4736 * @pf: the PF structure 4737 * @pf_q: the PF queue to configure 4738 * @is_xdp: true if the queue is used for XDP 4739 * @enable: start or stop the queue 4740 **/ 4741 int i40e_control_wait_tx_q(int seid, struct i40e_pf *pf, int pf_q, 4742 bool is_xdp, bool enable) 4743 { 4744 int ret; 4745 4746 i40e_control_tx_q(pf, pf_q, enable); 4747 4748 /* wait for the change to finish */ 4749 ret = i40e_pf_txq_wait(pf, pf_q, enable); 4750 if (ret) { 4751 dev_info(&pf->pdev->dev, 4752 "VSI seid %d %sTx ring %d %sable timeout\n", 4753 seid, (is_xdp ? "XDP " : ""), pf_q, 4754 (enable ? "en" : "dis")); 4755 } 4756 4757 return ret; 4758 } 4759 4760 /** 4761 * i40e_vsi_enable_tx - Start a VSI's rings 4762 * @vsi: the VSI being configured 4763 **/ 4764 static int i40e_vsi_enable_tx(struct i40e_vsi *vsi) 4765 { 4766 struct i40e_pf *pf = vsi->back; 4767 int i, pf_q, ret = 0; 4768 4769 pf_q = vsi->base_queue; 4770 for (i = 0; i < vsi->num_queue_pairs; i++, pf_q++) { 4771 ret = i40e_control_wait_tx_q(vsi->seid, pf, 4772 pf_q, 4773 false /*is xdp*/, true); 4774 if (ret) 4775 break; 4776 4777 if (!i40e_enabled_xdp_vsi(vsi)) 4778 continue; 4779 4780 ret = i40e_control_wait_tx_q(vsi->seid, pf, 4781 pf_q + vsi->alloc_queue_pairs, 4782 true /*is xdp*/, true); 4783 if (ret) 4784 break; 4785 } 4786 return ret; 4787 } 4788 4789 /** 4790 * i40e_pf_rxq_wait - Wait for a PF's Rx queue to be enabled or disabled 4791 * @pf: the PF being configured 4792 * @pf_q: the PF queue 4793 * @enable: enable or disable state of the queue 4794 * 4795 * This routine will wait for the given Rx queue of the PF to reach the 4796 * enabled or disabled state. 4797 * Returns -ETIMEDOUT in case of failing to reach the requested state after 4798 * multiple retries; else will return 0 in case of success. 4799 **/ 4800 static int i40e_pf_rxq_wait(struct i40e_pf *pf, int pf_q, bool enable) 4801 { 4802 int i; 4803 u32 rx_reg; 4804 4805 for (i = 0; i < I40E_QUEUE_WAIT_RETRY_LIMIT; i++) { 4806 rx_reg = rd32(&pf->hw, I40E_QRX_ENA(pf_q)); 4807 if (enable == !!(rx_reg & I40E_QRX_ENA_QENA_STAT_MASK)) 4808 break; 4809 4810 usleep_range(10, 20); 4811 } 4812 if (i >= I40E_QUEUE_WAIT_RETRY_LIMIT) 4813 return -ETIMEDOUT; 4814 4815 return 0; 4816 } 4817 4818 /** 4819 * i40e_control_rx_q - Start or stop a particular Rx queue 4820 * @pf: the PF structure 4821 * @pf_q: the PF queue to configure 4822 * @enable: start or stop the queue 4823 * 4824 * This function enables or disables a single queue. Note that 4825 * any delay required after the operation is expected to be 4826 * handled by the caller of this function. 4827 **/ 4828 static void i40e_control_rx_q(struct i40e_pf *pf, int pf_q, bool enable) 4829 { 4830 struct i40e_hw *hw = &pf->hw; 4831 u32 rx_reg; 4832 int i; 4833 4834 for (i = 0; i < I40E_QTX_ENA_WAIT_COUNT; i++) { 4835 rx_reg = rd32(hw, I40E_QRX_ENA(pf_q)); 4836 if (((rx_reg >> I40E_QRX_ENA_QENA_REQ_SHIFT) & 1) == 4837 ((rx_reg >> I40E_QRX_ENA_QENA_STAT_SHIFT) & 1)) 4838 break; 4839 usleep_range(1000, 2000); 4840 } 4841 4842 /* Skip if the queue is already in the requested state */ 4843 if (enable == !!(rx_reg & I40E_QRX_ENA_QENA_STAT_MASK)) 4844 return; 4845 4846 /* turn on/off the queue */ 4847 if (enable) 4848 rx_reg |= I40E_QRX_ENA_QENA_REQ_MASK; 4849 else 4850 rx_reg &= ~I40E_QRX_ENA_QENA_REQ_MASK; 4851 4852 wr32(hw, I40E_QRX_ENA(pf_q), rx_reg); 4853 } 4854 4855 /** 4856 * i40e_control_wait_rx_q 4857 * @pf: the PF structure 4858 * @pf_q: queue being configured 4859 * @enable: start or stop the rings 4860 * 4861 * This function enables or disables a single queue along with waiting 4862 * for the change to finish. The caller of this function should handle 4863 * the delays needed in the case of disabling queues. 4864 **/ 4865 int i40e_control_wait_rx_q(struct i40e_pf *pf, int pf_q, bool enable) 4866 { 4867 i40e_control_rx_q(pf, pf_q, enable); 4868 4869 /* wait for the change to finish */ 4870 return i40e_pf_rxq_wait(pf, pf_q, enable); 4871 } 4872 4873 /** 4874 * i40e_vsi_enable_rx - Start a VSI's rings 4875 * @vsi: the VSI being configured 4876 **/ 4877 static int i40e_vsi_enable_rx(struct i40e_vsi *vsi) 4878 { 4879 struct i40e_pf *pf = vsi->back; 4880 int i, pf_q, ret = 0; 4881 4882 pf_q = vsi->base_queue; 4883 for (i = 0; i < vsi->num_queue_pairs; i++, pf_q++) { 4884 ret = i40e_control_wait_rx_q(pf, pf_q, true); 4885 if (ret) { 4886 dev_info(&pf->pdev->dev, 4887 "VSI seid %d Rx ring %d enable timeout\n", 4888 vsi->seid, pf_q); 4889 break; 4890 } 4891 } 4892 4893 return ret; 4894 } 4895 4896 /** 4897 * i40e_vsi_start_rings - Start a VSI's rings 4898 * @vsi: the VSI being configured 4899 **/ 4900 int i40e_vsi_start_rings(struct i40e_vsi *vsi) 4901 { 4902 int ret = 0; 4903 4904 /* do rx first for enable and last for disable */ 4905 ret = i40e_vsi_enable_rx(vsi); 4906 if (ret) 4907 return ret; 4908 ret = i40e_vsi_enable_tx(vsi); 4909 4910 return ret; 4911 } 4912 4913 #define I40E_DISABLE_TX_GAP_MSEC 50 4914 4915 /** 4916 * i40e_vsi_stop_rings - Stop a VSI's rings 4917 * @vsi: the VSI being configured 4918 **/ 4919 void i40e_vsi_stop_rings(struct i40e_vsi *vsi) 4920 { 4921 struct i40e_pf *pf = vsi->back; 4922 u32 pf_q, tx_q_end, rx_q_end; 4923 4924 /* When port TX is suspended, don't wait */ 4925 if (test_bit(__I40E_PORT_SUSPENDED, vsi->back->state)) 4926 return i40e_vsi_stop_rings_no_wait(vsi); 4927 4928 tx_q_end = vsi->base_queue + 4929 vsi->alloc_queue_pairs * (i40e_enabled_xdp_vsi(vsi) ? 2 : 1); 4930 for (pf_q = vsi->base_queue; pf_q < tx_q_end; pf_q++) 4931 i40e_pre_tx_queue_cfg(&pf->hw, pf_q, false); 4932 4933 rx_q_end = vsi->base_queue + vsi->num_queue_pairs; 4934 for (pf_q = vsi->base_queue; pf_q < rx_q_end; pf_q++) 4935 i40e_control_rx_q(pf, pf_q, false); 4936 4937 msleep(I40E_DISABLE_TX_GAP_MSEC); 4938 for (pf_q = vsi->base_queue; pf_q < tx_q_end; pf_q++) 4939 wr32(&pf->hw, I40E_QTX_ENA(pf_q), 0); 4940 4941 i40e_vsi_wait_queues_disabled(vsi); 4942 } 4943 4944 /** 4945 * i40e_vsi_stop_rings_no_wait - Stop a VSI's rings and do not delay 4946 * @vsi: the VSI being shutdown 4947 * 4948 * This function stops all the rings for a VSI but does not delay to verify 4949 * that rings have been disabled. It is expected that the caller is shutting 4950 * down multiple VSIs at once and will delay together for all the VSIs after 4951 * initiating the shutdown. This is particularly useful for shutting down lots 4952 * of VFs together. Otherwise, a large delay can be incurred while configuring 4953 * each VSI in serial. 4954 **/ 4955 void i40e_vsi_stop_rings_no_wait(struct i40e_vsi *vsi) 4956 { 4957 struct i40e_pf *pf = vsi->back; 4958 int i, pf_q; 4959 4960 pf_q = vsi->base_queue; 4961 for (i = 0; i < vsi->num_queue_pairs; i++, pf_q++) { 4962 i40e_control_tx_q(pf, pf_q, false); 4963 i40e_control_rx_q(pf, pf_q, false); 4964 } 4965 } 4966 4967 /** 4968 * i40e_vsi_free_irq - Free the irq association with the OS 4969 * @vsi: the VSI being configured 4970 **/ 4971 static void i40e_vsi_free_irq(struct i40e_vsi *vsi) 4972 { 4973 struct i40e_pf *pf = vsi->back; 4974 struct i40e_hw *hw = &pf->hw; 4975 int base = vsi->base_vector; 4976 u32 val, qp; 4977 int i; 4978 4979 if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags)) { 4980 if (!vsi->q_vectors) 4981 return; 4982 4983 if (!vsi->irqs_ready) 4984 return; 4985 4986 vsi->irqs_ready = false; 4987 for (i = 0; i < vsi->num_q_vectors; i++) { 4988 int irq_num; 4989 u16 vector; 4990 4991 vector = i + base; 4992 irq_num = pf->msix_entries[vector].vector; 4993 4994 /* free only the irqs that were actually requested */ 4995 if (!vsi->q_vectors[i] || 4996 !vsi->q_vectors[i]->num_ringpairs) 4997 continue; 4998 4999 /* clear the affinity notifier in the IRQ descriptor */ 5000 irq_set_affinity_notifier(irq_num, NULL); 5001 /* remove our suggested affinity mask for this IRQ */ 5002 irq_update_affinity_hint(irq_num, NULL); 5003 free_irq(irq_num, vsi->q_vectors[i]); 5004 5005 /* Tear down the interrupt queue link list 5006 * 5007 * We know that they come in pairs and always 5008 * the Rx first, then the Tx. To clear the 5009 * link list, stick the EOL value into the 5010 * next_q field of the registers. 5011 */ 5012 val = rd32(hw, I40E_PFINT_LNKLSTN(vector - 1)); 5013 qp = FIELD_GET(I40E_PFINT_LNKLSTN_FIRSTQ_INDX_MASK, 5014 val); 5015 val |= I40E_QUEUE_END_OF_LIST 5016 << I40E_PFINT_LNKLSTN_FIRSTQ_INDX_SHIFT; 5017 wr32(hw, I40E_PFINT_LNKLSTN(vector - 1), val); 5018 5019 while (qp != I40E_QUEUE_END_OF_LIST) { 5020 u32 next; 5021 5022 val = rd32(hw, I40E_QINT_RQCTL(qp)); 5023 5024 val &= ~(I40E_QINT_RQCTL_MSIX_INDX_MASK | 5025 I40E_QINT_RQCTL_MSIX0_INDX_MASK | 5026 I40E_QINT_RQCTL_CAUSE_ENA_MASK | 5027 I40E_QINT_RQCTL_INTEVENT_MASK); 5028 5029 val |= (I40E_QINT_RQCTL_ITR_INDX_MASK | 5030 I40E_QINT_RQCTL_NEXTQ_INDX_MASK); 5031 5032 wr32(hw, I40E_QINT_RQCTL(qp), val); 5033 5034 val = rd32(hw, I40E_QINT_TQCTL(qp)); 5035 5036 next = FIELD_GET(I40E_QINT_TQCTL_NEXTQ_INDX_MASK, 5037 val); 5038 5039 val &= ~(I40E_QINT_TQCTL_MSIX_INDX_MASK | 5040 I40E_QINT_TQCTL_MSIX0_INDX_MASK | 5041 I40E_QINT_TQCTL_CAUSE_ENA_MASK | 5042 I40E_QINT_TQCTL_INTEVENT_MASK); 5043 5044 val |= (I40E_QINT_TQCTL_ITR_INDX_MASK | 5045 I40E_QINT_TQCTL_NEXTQ_INDX_MASK); 5046 5047 wr32(hw, I40E_QINT_TQCTL(qp), val); 5048 qp = next; 5049 } 5050 } 5051 } else { 5052 free_irq(pf->pdev->irq, pf); 5053 5054 val = rd32(hw, I40E_PFINT_LNKLST0); 5055 qp = FIELD_GET(I40E_PFINT_LNKLSTN_FIRSTQ_INDX_MASK, val); 5056 val |= I40E_QUEUE_END_OF_LIST 5057 << I40E_PFINT_LNKLST0_FIRSTQ_INDX_SHIFT; 5058 wr32(hw, I40E_PFINT_LNKLST0, val); 5059 5060 val = rd32(hw, I40E_QINT_RQCTL(qp)); 5061 val &= ~(I40E_QINT_RQCTL_MSIX_INDX_MASK | 5062 I40E_QINT_RQCTL_MSIX0_INDX_MASK | 5063 I40E_QINT_RQCTL_CAUSE_ENA_MASK | 5064 I40E_QINT_RQCTL_INTEVENT_MASK); 5065 5066 val |= (I40E_QINT_RQCTL_ITR_INDX_MASK | 5067 I40E_QINT_RQCTL_NEXTQ_INDX_MASK); 5068 5069 wr32(hw, I40E_QINT_RQCTL(qp), val); 5070 5071 val = rd32(hw, I40E_QINT_TQCTL(qp)); 5072 5073 val &= ~(I40E_QINT_TQCTL_MSIX_INDX_MASK | 5074 I40E_QINT_TQCTL_MSIX0_INDX_MASK | 5075 I40E_QINT_TQCTL_CAUSE_ENA_MASK | 5076 I40E_QINT_TQCTL_INTEVENT_MASK); 5077 5078 val |= (I40E_QINT_TQCTL_ITR_INDX_MASK | 5079 I40E_QINT_TQCTL_NEXTQ_INDX_MASK); 5080 5081 wr32(hw, I40E_QINT_TQCTL(qp), val); 5082 } 5083 } 5084 5085 /** 5086 * i40e_free_q_vector - Free memory allocated for specific interrupt vector 5087 * @vsi: the VSI being configured 5088 * @v_idx: Index of vector to be freed 5089 * 5090 * This function frees the memory allocated to the q_vector. In addition if 5091 * NAPI is enabled it will delete any references to the NAPI struct prior 5092 * to freeing the q_vector. 5093 **/ 5094 static void i40e_free_q_vector(struct i40e_vsi *vsi, int v_idx) 5095 { 5096 struct i40e_q_vector *q_vector = vsi->q_vectors[v_idx]; 5097 struct i40e_ring *ring; 5098 5099 if (!q_vector) 5100 return; 5101 5102 /* disassociate q_vector from rings */ 5103 i40e_for_each_ring(ring, q_vector->tx) 5104 ring->q_vector = NULL; 5105 5106 i40e_for_each_ring(ring, q_vector->rx) 5107 ring->q_vector = NULL; 5108 5109 /* only VSI w/ an associated netdev is set up w/ NAPI */ 5110 if (vsi->netdev) 5111 netif_napi_del(&q_vector->napi); 5112 5113 vsi->q_vectors[v_idx] = NULL; 5114 5115 kfree_rcu(q_vector, rcu); 5116 } 5117 5118 /** 5119 * i40e_vsi_free_q_vectors - Free memory allocated for interrupt vectors 5120 * @vsi: the VSI being un-configured 5121 * 5122 * This frees the memory allocated to the q_vectors and 5123 * deletes references to the NAPI struct. 5124 **/ 5125 static void i40e_vsi_free_q_vectors(struct i40e_vsi *vsi) 5126 { 5127 int v_idx; 5128 5129 for (v_idx = 0; v_idx < vsi->num_q_vectors; v_idx++) 5130 i40e_free_q_vector(vsi, v_idx); 5131 } 5132 5133 /** 5134 * i40e_reset_interrupt_capability - Disable interrupt setup in OS 5135 * @pf: board private structure 5136 **/ 5137 static void i40e_reset_interrupt_capability(struct i40e_pf *pf) 5138 { 5139 /* If we're in Legacy mode, the interrupt was cleaned in vsi_close */ 5140 if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags)) { 5141 pci_disable_msix(pf->pdev); 5142 kfree(pf->msix_entries); 5143 pf->msix_entries = NULL; 5144 kfree(pf->irq_pile); 5145 pf->irq_pile = NULL; 5146 } else if (test_bit(I40E_FLAG_MSI_ENA, pf->flags)) { 5147 pci_disable_msi(pf->pdev); 5148 } 5149 clear_bit(I40E_FLAG_MSI_ENA, pf->flags); 5150 clear_bit(I40E_FLAG_MSIX_ENA, pf->flags); 5151 } 5152 5153 /** 5154 * i40e_clear_interrupt_scheme - Clear the current interrupt scheme settings 5155 * @pf: board private structure 5156 * 5157 * We go through and clear interrupt specific resources and reset the structure 5158 * to pre-load conditions 5159 **/ 5160 static void i40e_clear_interrupt_scheme(struct i40e_pf *pf) 5161 { 5162 struct i40e_vsi *vsi; 5163 int i; 5164 5165 if (test_bit(__I40E_MISC_IRQ_REQUESTED, pf->state)) 5166 i40e_free_misc_vector(pf); 5167 5168 i40e_put_lump(pf->irq_pile, pf->iwarp_base_vector, 5169 I40E_IWARP_IRQ_PILE_ID); 5170 5171 i40e_put_lump(pf->irq_pile, 0, I40E_PILE_VALID_BIT-1); 5172 5173 i40e_pf_for_each_vsi(pf, i, vsi) 5174 i40e_vsi_free_q_vectors(vsi); 5175 5176 i40e_reset_interrupt_capability(pf); 5177 } 5178 5179 /** 5180 * i40e_napi_enable_all - Enable NAPI for all q_vectors in the VSI 5181 * @vsi: the VSI being configured 5182 **/ 5183 static void i40e_napi_enable_all(struct i40e_vsi *vsi) 5184 { 5185 int q_idx; 5186 5187 if (!vsi->netdev) 5188 return; 5189 5190 for (q_idx = 0; q_idx < vsi->num_q_vectors; q_idx++) { 5191 struct i40e_q_vector *q_vector = vsi->q_vectors[q_idx]; 5192 5193 if (q_vector->rx.ring || q_vector->tx.ring) 5194 napi_enable(&q_vector->napi); 5195 } 5196 } 5197 5198 /** 5199 * i40e_napi_disable_all - Disable NAPI for all q_vectors in the VSI 5200 * @vsi: the VSI being configured 5201 **/ 5202 static void i40e_napi_disable_all(struct i40e_vsi *vsi) 5203 { 5204 int q_idx; 5205 5206 if (!vsi->netdev) 5207 return; 5208 5209 for (q_idx = 0; q_idx < vsi->num_q_vectors; q_idx++) { 5210 struct i40e_q_vector *q_vector = vsi->q_vectors[q_idx]; 5211 5212 if (q_vector->rx.ring || q_vector->tx.ring) 5213 napi_disable(&q_vector->napi); 5214 } 5215 } 5216 5217 /** 5218 * i40e_vsi_close - Shut down a VSI 5219 * @vsi: the vsi to be quelled 5220 **/ 5221 static void i40e_vsi_close(struct i40e_vsi *vsi) 5222 { 5223 struct i40e_pf *pf = vsi->back; 5224 if (!test_and_set_bit(__I40E_VSI_DOWN, vsi->state)) 5225 i40e_down(vsi); 5226 i40e_vsi_free_irq(vsi); 5227 i40e_vsi_free_tx_resources(vsi); 5228 i40e_vsi_free_rx_resources(vsi); 5229 vsi->current_netdev_flags = 0; 5230 set_bit(__I40E_CLIENT_SERVICE_REQUESTED, pf->state); 5231 if (test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state)) 5232 set_bit(__I40E_CLIENT_RESET, pf->state); 5233 } 5234 5235 /** 5236 * i40e_quiesce_vsi - Pause a given VSI 5237 * @vsi: the VSI being paused 5238 **/ 5239 static void i40e_quiesce_vsi(struct i40e_vsi *vsi) 5240 { 5241 if (test_bit(__I40E_VSI_DOWN, vsi->state)) 5242 return; 5243 5244 set_bit(__I40E_VSI_NEEDS_RESTART, vsi->state); 5245 if (vsi->netdev && netif_running(vsi->netdev)) 5246 vsi->netdev->netdev_ops->ndo_stop(vsi->netdev); 5247 else 5248 i40e_vsi_close(vsi); 5249 } 5250 5251 /** 5252 * i40e_unquiesce_vsi - Resume a given VSI 5253 * @vsi: the VSI being resumed 5254 **/ 5255 static void i40e_unquiesce_vsi(struct i40e_vsi *vsi) 5256 { 5257 if (!test_and_clear_bit(__I40E_VSI_NEEDS_RESTART, vsi->state)) 5258 return; 5259 5260 if (vsi->netdev && netif_running(vsi->netdev)) 5261 vsi->netdev->netdev_ops->ndo_open(vsi->netdev); 5262 else 5263 i40e_vsi_open(vsi); /* this clears the DOWN bit */ 5264 } 5265 5266 /** 5267 * i40e_pf_quiesce_all_vsi - Pause all VSIs on a PF 5268 * @pf: the PF 5269 **/ 5270 static void i40e_pf_quiesce_all_vsi(struct i40e_pf *pf) 5271 { 5272 struct i40e_vsi *vsi; 5273 int v; 5274 5275 i40e_pf_for_each_vsi(pf, v, vsi) 5276 i40e_quiesce_vsi(vsi); 5277 } 5278 5279 /** 5280 * i40e_pf_unquiesce_all_vsi - Resume all VSIs on a PF 5281 * @pf: the PF 5282 **/ 5283 static void i40e_pf_unquiesce_all_vsi(struct i40e_pf *pf) 5284 { 5285 struct i40e_vsi *vsi; 5286 int v; 5287 5288 i40e_pf_for_each_vsi(pf, v, vsi) 5289 i40e_unquiesce_vsi(vsi); 5290 } 5291 5292 /** 5293 * i40e_vsi_wait_queues_disabled - Wait for VSI's queues to be disabled 5294 * @vsi: the VSI being configured 5295 * 5296 * Wait until all queues on a given VSI have been disabled. 5297 **/ 5298 int i40e_vsi_wait_queues_disabled(struct i40e_vsi *vsi) 5299 { 5300 struct i40e_pf *pf = vsi->back; 5301 int i, pf_q, ret; 5302 5303 pf_q = vsi->base_queue; 5304 for (i = 0; i < vsi->num_queue_pairs; i++, pf_q++) { 5305 /* Check and wait for the Tx queue */ 5306 ret = i40e_pf_txq_wait(pf, pf_q, false); 5307 if (ret) { 5308 dev_info(&pf->pdev->dev, 5309 "VSI seid %d Tx ring %d disable timeout\n", 5310 vsi->seid, pf_q); 5311 return ret; 5312 } 5313 5314 if (!i40e_enabled_xdp_vsi(vsi)) 5315 goto wait_rx; 5316 5317 /* Check and wait for the XDP Tx queue */ 5318 ret = i40e_pf_txq_wait(pf, pf_q + vsi->alloc_queue_pairs, 5319 false); 5320 if (ret) { 5321 dev_info(&pf->pdev->dev, 5322 "VSI seid %d XDP Tx ring %d disable timeout\n", 5323 vsi->seid, pf_q); 5324 return ret; 5325 } 5326 wait_rx: 5327 /* Check and wait for the Rx queue */ 5328 ret = i40e_pf_rxq_wait(pf, pf_q, false); 5329 if (ret) { 5330 dev_info(&pf->pdev->dev, 5331 "VSI seid %d Rx ring %d disable timeout\n", 5332 vsi->seid, pf_q); 5333 return ret; 5334 } 5335 } 5336 5337 return 0; 5338 } 5339 5340 #ifdef CONFIG_I40E_DCB 5341 /** 5342 * i40e_pf_wait_queues_disabled - Wait for all queues of PF VSIs to be disabled 5343 * @pf: the PF 5344 * 5345 * This function waits for the queues to be in disabled state for all the 5346 * VSIs that are managed by this PF. 5347 **/ 5348 static int i40e_pf_wait_queues_disabled(struct i40e_pf *pf) 5349 { 5350 struct i40e_vsi *vsi; 5351 int v, ret = 0; 5352 5353 i40e_pf_for_each_vsi(pf, v, vsi) { 5354 ret = i40e_vsi_wait_queues_disabled(vsi); 5355 if (ret) 5356 break; 5357 } 5358 5359 return ret; 5360 } 5361 5362 #endif 5363 5364 /** 5365 * i40e_get_iscsi_tc_map - Return TC map for iSCSI APP 5366 * @pf: pointer to PF 5367 * 5368 * Get TC map for ISCSI PF type that will include iSCSI TC 5369 * and LAN TC. 5370 **/ 5371 static u8 i40e_get_iscsi_tc_map(struct i40e_pf *pf) 5372 { 5373 struct i40e_dcb_app_priority_table app; 5374 struct i40e_hw *hw = &pf->hw; 5375 u8 enabled_tc = 1; /* TC0 is always enabled */ 5376 u8 tc, i; 5377 /* Get the iSCSI APP TLV */ 5378 struct i40e_dcbx_config *dcbcfg = &hw->local_dcbx_config; 5379 5380 for (i = 0; i < dcbcfg->numapps; i++) { 5381 app = dcbcfg->app[i]; 5382 if (app.selector == I40E_APP_SEL_TCPIP && 5383 app.protocolid == I40E_APP_PROTOID_ISCSI) { 5384 tc = dcbcfg->etscfg.prioritytable[app.priority]; 5385 enabled_tc |= BIT(tc); 5386 break; 5387 } 5388 } 5389 5390 return enabled_tc; 5391 } 5392 5393 /** 5394 * i40e_dcb_get_num_tc - Get the number of TCs from DCBx config 5395 * @dcbcfg: the corresponding DCBx configuration structure 5396 * 5397 * Return the number of TCs from given DCBx configuration 5398 **/ 5399 static u8 i40e_dcb_get_num_tc(struct i40e_dcbx_config *dcbcfg) 5400 { 5401 int i, tc_unused = 0; 5402 u8 num_tc = 0; 5403 u8 ret = 0; 5404 5405 /* Scan the ETS Config Priority Table to find 5406 * traffic class enabled for a given priority 5407 * and create a bitmask of enabled TCs 5408 */ 5409 for (i = 0; i < I40E_MAX_USER_PRIORITY; i++) 5410 num_tc |= BIT(dcbcfg->etscfg.prioritytable[i]); 5411 5412 /* Now scan the bitmask to check for 5413 * contiguous TCs starting with TC0 5414 */ 5415 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 5416 if (num_tc & BIT(i)) { 5417 if (!tc_unused) { 5418 ret++; 5419 } else { 5420 pr_err("Non-contiguous TC - Disabling DCB\n"); 5421 return 1; 5422 } 5423 } else { 5424 tc_unused = 1; 5425 } 5426 } 5427 5428 /* There is always at least TC0 */ 5429 if (!ret) 5430 ret = 1; 5431 5432 return ret; 5433 } 5434 5435 /** 5436 * i40e_dcb_get_enabled_tc - Get enabled traffic classes 5437 * @dcbcfg: the corresponding DCBx configuration structure 5438 * 5439 * Query the current DCB configuration and return the number of 5440 * traffic classes enabled from the given DCBX config 5441 **/ 5442 static u8 i40e_dcb_get_enabled_tc(struct i40e_dcbx_config *dcbcfg) 5443 { 5444 u8 num_tc = i40e_dcb_get_num_tc(dcbcfg); 5445 u8 enabled_tc = 1; 5446 u8 i; 5447 5448 for (i = 0; i < num_tc; i++) 5449 enabled_tc |= BIT(i); 5450 5451 return enabled_tc; 5452 } 5453 5454 /** 5455 * i40e_mqprio_get_enabled_tc - Get enabled traffic classes 5456 * @pf: PF being queried 5457 * 5458 * Query the current MQPRIO configuration and return the number of 5459 * traffic classes enabled. 5460 **/ 5461 static u8 i40e_mqprio_get_enabled_tc(struct i40e_pf *pf) 5462 { 5463 struct i40e_vsi *vsi = i40e_pf_get_main_vsi(pf); 5464 u8 num_tc = vsi->mqprio_qopt.qopt.num_tc; 5465 u8 enabled_tc = 1, i; 5466 5467 for (i = 1; i < num_tc; i++) 5468 enabled_tc |= BIT(i); 5469 return enabled_tc; 5470 } 5471 5472 /** 5473 * i40e_pf_get_num_tc - Get enabled traffic classes for PF 5474 * @pf: PF being queried 5475 * 5476 * Return number of traffic classes enabled for the given PF 5477 **/ 5478 static u8 i40e_pf_get_num_tc(struct i40e_pf *pf) 5479 { 5480 u8 i, enabled_tc = 1; 5481 u8 num_tc = 0; 5482 5483 if (i40e_is_tc_mqprio_enabled(pf)) { 5484 struct i40e_vsi *vsi = i40e_pf_get_main_vsi(pf); 5485 5486 return vsi->mqprio_qopt.qopt.num_tc; 5487 } 5488 5489 /* If neither MQPRIO nor DCB is enabled, then always use single TC */ 5490 if (!test_bit(I40E_FLAG_DCB_ENA, pf->flags)) 5491 return 1; 5492 5493 /* SFP mode will be enabled for all TCs on port */ 5494 if (!test_bit(I40E_FLAG_MFP_ENA, pf->flags)) 5495 return i40e_dcb_get_num_tc(&pf->hw.local_dcbx_config); 5496 5497 /* MFP mode return count of enabled TCs for this PF */ 5498 if (pf->hw.func_caps.iscsi) 5499 enabled_tc = i40e_get_iscsi_tc_map(pf); 5500 else 5501 return 1; /* Only TC0 */ 5502 5503 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 5504 if (enabled_tc & BIT(i)) 5505 num_tc++; 5506 } 5507 return num_tc; 5508 } 5509 5510 /** 5511 * i40e_pf_get_tc_map - Get bitmap for enabled traffic classes 5512 * @pf: PF being queried 5513 * 5514 * Return a bitmap for enabled traffic classes for this PF. 5515 **/ 5516 static u8 i40e_pf_get_tc_map(struct i40e_pf *pf) 5517 { 5518 if (i40e_is_tc_mqprio_enabled(pf)) 5519 return i40e_mqprio_get_enabled_tc(pf); 5520 5521 /* If neither MQPRIO nor DCB is enabled for this PF then just return 5522 * default TC 5523 */ 5524 if (!test_bit(I40E_FLAG_DCB_ENA, pf->flags)) 5525 return I40E_DEFAULT_TRAFFIC_CLASS; 5526 5527 /* SFP mode we want PF to be enabled for all TCs */ 5528 if (!test_bit(I40E_FLAG_MFP_ENA, pf->flags)) 5529 return i40e_dcb_get_enabled_tc(&pf->hw.local_dcbx_config); 5530 5531 /* MFP enabled and iSCSI PF type */ 5532 if (pf->hw.func_caps.iscsi) 5533 return i40e_get_iscsi_tc_map(pf); 5534 else 5535 return I40E_DEFAULT_TRAFFIC_CLASS; 5536 } 5537 5538 /** 5539 * i40e_vsi_get_bw_info - Query VSI BW Information 5540 * @vsi: the VSI being queried 5541 * 5542 * Returns 0 on success, negative value on failure 5543 **/ 5544 static int i40e_vsi_get_bw_info(struct i40e_vsi *vsi) 5545 { 5546 struct i40e_aqc_query_vsi_ets_sla_config_resp bw_ets_config = {0}; 5547 struct i40e_aqc_query_vsi_bw_config_resp bw_config = {0}; 5548 struct i40e_pf *pf = vsi->back; 5549 struct i40e_hw *hw = &pf->hw; 5550 u32 tc_bw_max; 5551 int ret; 5552 int i; 5553 5554 /* Get the VSI level BW configuration */ 5555 ret = i40e_aq_query_vsi_bw_config(hw, vsi->seid, &bw_config, NULL); 5556 if (ret) { 5557 dev_info(&pf->pdev->dev, 5558 "couldn't get PF vsi bw config, err %pe aq_err %s\n", 5559 ERR_PTR(ret), 5560 libie_aq_str(pf->hw.aq.asq_last_status)); 5561 return -EINVAL; 5562 } 5563 5564 /* Get the VSI level BW configuration per TC */ 5565 ret = i40e_aq_query_vsi_ets_sla_config(hw, vsi->seid, &bw_ets_config, 5566 NULL); 5567 if (ret) { 5568 dev_info(&pf->pdev->dev, 5569 "couldn't get PF vsi ets bw config, err %pe aq_err %s\n", 5570 ERR_PTR(ret), 5571 libie_aq_str(pf->hw.aq.asq_last_status)); 5572 return -EINVAL; 5573 } 5574 5575 if (bw_config.tc_valid_bits != bw_ets_config.tc_valid_bits) { 5576 dev_info(&pf->pdev->dev, 5577 "Enabled TCs mismatch from querying VSI BW info 0x%08x 0x%08x\n", 5578 bw_config.tc_valid_bits, 5579 bw_ets_config.tc_valid_bits); 5580 /* Still continuing */ 5581 } 5582 5583 vsi->bw_limit = le16_to_cpu(bw_config.port_bw_limit); 5584 vsi->bw_max_quanta = bw_config.max_bw; 5585 tc_bw_max = le16_to_cpu(bw_ets_config.tc_bw_max[0]) | 5586 (le16_to_cpu(bw_ets_config.tc_bw_max[1]) << 16); 5587 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 5588 vsi->bw_ets_share_credits[i] = bw_ets_config.share_credits[i]; 5589 vsi->bw_ets_limit_credits[i] = 5590 le16_to_cpu(bw_ets_config.credits[i]); 5591 /* 3 bits out of 4 for each TC */ 5592 vsi->bw_ets_max_quanta[i] = (u8)((tc_bw_max >> (i*4)) & 0x7); 5593 } 5594 5595 return 0; 5596 } 5597 5598 /** 5599 * i40e_vsi_configure_bw_alloc - Configure VSI BW allocation per TC 5600 * @vsi: the VSI being configured 5601 * @enabled_tc: TC bitmap 5602 * @bw_share: BW shared credits per TC 5603 * 5604 * Returns 0 on success, negative value on failure 5605 **/ 5606 static int i40e_vsi_configure_bw_alloc(struct i40e_vsi *vsi, u8 enabled_tc, 5607 u8 *bw_share) 5608 { 5609 struct i40e_aqc_configure_vsi_tc_bw_data bw_data; 5610 struct i40e_pf *pf = vsi->back; 5611 int ret; 5612 int i; 5613 5614 /* There is no need to reset BW when mqprio mode is on. */ 5615 if (i40e_is_tc_mqprio_enabled(pf)) 5616 return 0; 5617 if (!vsi->mqprio_qopt.qopt.hw && !test_bit(I40E_FLAG_DCB_ENA, pf->flags)) { 5618 ret = i40e_set_bw_limit(vsi, vsi->seid, 0); 5619 if (ret) 5620 dev_info(&pf->pdev->dev, 5621 "Failed to reset tx rate for vsi->seid %u\n", 5622 vsi->seid); 5623 return ret; 5624 } 5625 memset(&bw_data, 0, sizeof(bw_data)); 5626 bw_data.tc_valid_bits = enabled_tc; 5627 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) 5628 bw_data.tc_bw_credits[i] = bw_share[i]; 5629 5630 ret = i40e_aq_config_vsi_tc_bw(&pf->hw, vsi->seid, &bw_data, NULL); 5631 if (ret) { 5632 dev_info(&pf->pdev->dev, 5633 "AQ command Config VSI BW allocation per TC failed = %d\n", 5634 pf->hw.aq.asq_last_status); 5635 return -EINVAL; 5636 } 5637 5638 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) 5639 vsi->info.qs_handle[i] = bw_data.qs_handles[i]; 5640 5641 return 0; 5642 } 5643 5644 /** 5645 * i40e_vsi_config_netdev_tc - Setup the netdev TC configuration 5646 * @vsi: the VSI being configured 5647 * @enabled_tc: TC map to be enabled 5648 * 5649 **/ 5650 static void i40e_vsi_config_netdev_tc(struct i40e_vsi *vsi, u8 enabled_tc) 5651 { 5652 struct net_device *netdev = vsi->netdev; 5653 struct i40e_pf *pf = vsi->back; 5654 struct i40e_hw *hw = &pf->hw; 5655 u8 netdev_tc = 0; 5656 int i; 5657 struct i40e_dcbx_config *dcbcfg = &hw->local_dcbx_config; 5658 5659 if (!netdev) 5660 return; 5661 5662 if (!enabled_tc) { 5663 netdev_reset_tc(netdev); 5664 return; 5665 } 5666 5667 /* Set up actual enabled TCs on the VSI */ 5668 if (netdev_set_num_tc(netdev, vsi->tc_config.numtc)) 5669 return; 5670 5671 /* set per TC queues for the VSI */ 5672 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 5673 /* Only set TC queues for enabled tcs 5674 * 5675 * e.g. For a VSI that has TC0 and TC3 enabled the 5676 * enabled_tc bitmap would be 0x00001001; the driver 5677 * will set the numtc for netdev as 2 that will be 5678 * referenced by the netdev layer as TC 0 and 1. 5679 */ 5680 if (vsi->tc_config.enabled_tc & BIT(i)) 5681 netdev_set_tc_queue(netdev, 5682 vsi->tc_config.tc_info[i].netdev_tc, 5683 vsi->tc_config.tc_info[i].qcount, 5684 vsi->tc_config.tc_info[i].qoffset); 5685 } 5686 5687 if (i40e_is_tc_mqprio_enabled(pf)) 5688 return; 5689 5690 /* Assign UP2TC map for the VSI */ 5691 for (i = 0; i < I40E_MAX_USER_PRIORITY; i++) { 5692 /* Get the actual TC# for the UP */ 5693 u8 ets_tc = dcbcfg->etscfg.prioritytable[i]; 5694 /* Get the mapped netdev TC# for the UP */ 5695 netdev_tc = vsi->tc_config.tc_info[ets_tc].netdev_tc; 5696 netdev_set_prio_tc_map(netdev, i, netdev_tc); 5697 } 5698 } 5699 5700 /** 5701 * i40e_vsi_update_queue_map - Update our copy of VSi info with new queue map 5702 * @vsi: the VSI being configured 5703 * @ctxt: the ctxt buffer returned from AQ VSI update param command 5704 **/ 5705 static void i40e_vsi_update_queue_map(struct i40e_vsi *vsi, 5706 struct i40e_vsi_context *ctxt) 5707 { 5708 /* copy just the sections touched not the entire info 5709 * since not all sections are valid as returned by 5710 * update vsi params 5711 */ 5712 vsi->info.mapping_flags = ctxt->info.mapping_flags; 5713 memcpy(&vsi->info.queue_mapping, 5714 &ctxt->info.queue_mapping, sizeof(vsi->info.queue_mapping)); 5715 memcpy(&vsi->info.tc_mapping, ctxt->info.tc_mapping, 5716 sizeof(vsi->info.tc_mapping)); 5717 } 5718 5719 /** 5720 * i40e_update_adq_vsi_queues - update queue mapping for ADq VSI 5721 * @vsi: the VSI being reconfigured 5722 * @vsi_offset: offset from main VF VSI 5723 */ 5724 int i40e_update_adq_vsi_queues(struct i40e_vsi *vsi, int vsi_offset) 5725 { 5726 struct i40e_vsi_context ctxt = {}; 5727 struct i40e_pf *pf; 5728 struct i40e_hw *hw; 5729 int ret; 5730 5731 if (!vsi) 5732 return -EINVAL; 5733 pf = vsi->back; 5734 hw = &pf->hw; 5735 5736 ctxt.seid = vsi->seid; 5737 ctxt.pf_num = hw->pf_id; 5738 ctxt.vf_num = vsi->vf_id + hw->func_caps.vf_base_id + vsi_offset; 5739 ctxt.uplink_seid = vsi->uplink_seid; 5740 ctxt.connection_type = I40E_AQ_VSI_CONN_TYPE_NORMAL; 5741 ctxt.flags = I40E_AQ_VSI_TYPE_VF; 5742 ctxt.info = vsi->info; 5743 5744 i40e_vsi_setup_queue_map(vsi, &ctxt, vsi->tc_config.enabled_tc, 5745 false); 5746 if (vsi->reconfig_rss) { 5747 vsi->rss_size = min_t(int, pf->alloc_rss_size, 5748 vsi->num_queue_pairs); 5749 ret = i40e_vsi_config_rss(vsi); 5750 if (ret) { 5751 dev_info(&pf->pdev->dev, "Failed to reconfig rss for num_queues\n"); 5752 return ret; 5753 } 5754 vsi->reconfig_rss = false; 5755 } 5756 5757 ret = i40e_aq_update_vsi_params(hw, &ctxt, NULL); 5758 if (ret) { 5759 dev_info(&pf->pdev->dev, "Update vsi config failed, err %pe aq_err %s\n", 5760 ERR_PTR(ret), 5761 libie_aq_str(hw->aq.asq_last_status)); 5762 return ret; 5763 } 5764 /* update the local VSI info with updated queue map */ 5765 i40e_vsi_update_queue_map(vsi, &ctxt); 5766 vsi->info.valid_sections = 0; 5767 5768 return ret; 5769 } 5770 5771 /** 5772 * i40e_vsi_config_tc - Configure VSI Tx Scheduler for given TC map 5773 * @vsi: VSI to be configured 5774 * @enabled_tc: TC bitmap 5775 * 5776 * This configures a particular VSI for TCs that are mapped to the 5777 * given TC bitmap. It uses default bandwidth share for TCs across 5778 * VSIs to configure TC for a particular VSI. 5779 * 5780 * NOTE: 5781 * It is expected that the VSI queues have been quisced before calling 5782 * this function. 5783 **/ 5784 static int i40e_vsi_config_tc(struct i40e_vsi *vsi, u8 enabled_tc) 5785 { 5786 u8 bw_share[I40E_MAX_TRAFFIC_CLASS] = {0}; 5787 struct i40e_pf *pf = vsi->back; 5788 struct i40e_hw *hw = &pf->hw; 5789 struct i40e_vsi_context ctxt; 5790 int ret = 0; 5791 int i; 5792 5793 /* Check if enabled_tc is same as existing or new TCs */ 5794 if (vsi->tc_config.enabled_tc == enabled_tc && 5795 vsi->mqprio_qopt.mode != TC_MQPRIO_MODE_CHANNEL) 5796 return ret; 5797 5798 /* Enable ETS TCs with equal BW Share for now across all VSIs */ 5799 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 5800 if (enabled_tc & BIT(i)) 5801 bw_share[i] = 1; 5802 } 5803 5804 ret = i40e_vsi_configure_bw_alloc(vsi, enabled_tc, bw_share); 5805 if (ret) { 5806 struct i40e_aqc_query_vsi_bw_config_resp bw_config = {0}; 5807 5808 dev_info(&pf->pdev->dev, 5809 "Failed configuring TC map %d for VSI %d\n", 5810 enabled_tc, vsi->seid); 5811 ret = i40e_aq_query_vsi_bw_config(hw, vsi->seid, 5812 &bw_config, NULL); 5813 if (ret) { 5814 dev_info(&pf->pdev->dev, 5815 "Failed querying vsi bw info, err %pe aq_err %s\n", 5816 ERR_PTR(ret), 5817 libie_aq_str(hw->aq.asq_last_status)); 5818 goto out; 5819 } 5820 if ((bw_config.tc_valid_bits & enabled_tc) != enabled_tc) { 5821 u8 valid_tc = bw_config.tc_valid_bits & enabled_tc; 5822 5823 if (!valid_tc) 5824 valid_tc = bw_config.tc_valid_bits; 5825 /* Always enable TC0, no matter what */ 5826 valid_tc |= 1; 5827 dev_info(&pf->pdev->dev, 5828 "Requested tc 0x%x, but FW reports 0x%x as valid. Attempting to use 0x%x.\n", 5829 enabled_tc, bw_config.tc_valid_bits, valid_tc); 5830 enabled_tc = valid_tc; 5831 } 5832 5833 ret = i40e_vsi_configure_bw_alloc(vsi, enabled_tc, bw_share); 5834 if (ret) { 5835 dev_err(&pf->pdev->dev, 5836 "Unable to configure TC map %d for VSI %d\n", 5837 enabled_tc, vsi->seid); 5838 goto out; 5839 } 5840 } 5841 5842 /* Update Queue Pairs Mapping for currently enabled UPs */ 5843 ctxt.seid = vsi->seid; 5844 ctxt.pf_num = vsi->back->hw.pf_id; 5845 ctxt.vf_num = 0; 5846 ctxt.uplink_seid = vsi->uplink_seid; 5847 ctxt.info = vsi->info; 5848 if (i40e_is_tc_mqprio_enabled(pf)) { 5849 ret = i40e_vsi_setup_queue_map_mqprio(vsi, &ctxt, enabled_tc); 5850 if (ret) 5851 goto out; 5852 } else { 5853 i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, false); 5854 } 5855 5856 /* On destroying the qdisc, reset vsi->rss_size, as number of enabled 5857 * queues changed. 5858 */ 5859 if (!vsi->mqprio_qopt.qopt.hw && vsi->reconfig_rss) { 5860 vsi->rss_size = min_t(int, vsi->back->alloc_rss_size, 5861 vsi->num_queue_pairs); 5862 ret = i40e_vsi_config_rss(vsi); 5863 if (ret) { 5864 dev_info(&vsi->back->pdev->dev, 5865 "Failed to reconfig rss for num_queues\n"); 5866 return ret; 5867 } 5868 vsi->reconfig_rss = false; 5869 } 5870 if (test_bit(I40E_FLAG_IWARP_ENA, vsi->back->flags)) { 5871 ctxt.info.valid_sections |= 5872 cpu_to_le16(I40E_AQ_VSI_PROP_QUEUE_OPT_VALID); 5873 ctxt.info.queueing_opt_flags |= I40E_AQ_VSI_QUE_OPT_TCP_ENA; 5874 } 5875 5876 /* Update the VSI after updating the VSI queue-mapping 5877 * information 5878 */ 5879 ret = i40e_aq_update_vsi_params(hw, &ctxt, NULL); 5880 if (ret) { 5881 dev_info(&pf->pdev->dev, 5882 "Update vsi tc config failed, err %pe aq_err %s\n", 5883 ERR_PTR(ret), 5884 libie_aq_str(hw->aq.asq_last_status)); 5885 goto out; 5886 } 5887 /* update the local VSI info with updated queue map */ 5888 i40e_vsi_update_queue_map(vsi, &ctxt); 5889 vsi->info.valid_sections = 0; 5890 5891 /* Update current VSI BW information */ 5892 ret = i40e_vsi_get_bw_info(vsi); 5893 if (ret) { 5894 dev_info(&pf->pdev->dev, 5895 "Failed updating vsi bw info, err %pe aq_err %s\n", 5896 ERR_PTR(ret), 5897 libie_aq_str(hw->aq.asq_last_status)); 5898 goto out; 5899 } 5900 5901 /* Update the netdev TC setup */ 5902 i40e_vsi_config_netdev_tc(vsi, enabled_tc); 5903 out: 5904 return ret; 5905 } 5906 5907 /** 5908 * i40e_vsi_reconfig_tc - Reconfigure VSI Tx Scheduler for stored TC map 5909 * @vsi: VSI to be reconfigured 5910 * 5911 * This reconfigures a particular VSI for TCs that are mapped to the 5912 * TC bitmap stored previously for the VSI. 5913 * 5914 * Context: It is expected that the VSI queues have been quisced before 5915 * calling this function. 5916 * 5917 * Return: 0 on success, negative value on failure 5918 **/ 5919 static int i40e_vsi_reconfig_tc(struct i40e_vsi *vsi) 5920 { 5921 u8 enabled_tc; 5922 5923 enabled_tc = vsi->tc_config.enabled_tc; 5924 vsi->tc_config.enabled_tc = 0; 5925 5926 return i40e_vsi_config_tc(vsi, enabled_tc); 5927 } 5928 5929 /** 5930 * i40e_get_link_speed - Returns link speed for the interface 5931 * @vsi: VSI to be configured 5932 * 5933 **/ 5934 static int i40e_get_link_speed(struct i40e_vsi *vsi) 5935 { 5936 struct i40e_pf *pf = vsi->back; 5937 5938 switch (pf->hw.phy.link_info.link_speed) { 5939 case I40E_LINK_SPEED_40GB: 5940 return 40000; 5941 case I40E_LINK_SPEED_25GB: 5942 return 25000; 5943 case I40E_LINK_SPEED_20GB: 5944 return 20000; 5945 case I40E_LINK_SPEED_10GB: 5946 return 10000; 5947 case I40E_LINK_SPEED_1GB: 5948 return 1000; 5949 default: 5950 return -EINVAL; 5951 } 5952 } 5953 5954 /** 5955 * i40e_bw_bytes_to_mbits - Convert max_tx_rate from bytes to mbits 5956 * @vsi: Pointer to vsi structure 5957 * @max_tx_rate: max TX rate in bytes to be converted into Mbits 5958 * 5959 * Helper function to convert units before send to set BW limit 5960 **/ 5961 static u64 i40e_bw_bytes_to_mbits(struct i40e_vsi *vsi, u64 max_tx_rate) 5962 { 5963 if (max_tx_rate < I40E_BW_MBPS_DIVISOR) { 5964 dev_warn(&vsi->back->pdev->dev, 5965 "Setting max tx rate to minimum usable value of 50Mbps.\n"); 5966 max_tx_rate = I40E_BW_CREDIT_DIVISOR; 5967 } else { 5968 do_div(max_tx_rate, I40E_BW_MBPS_DIVISOR); 5969 } 5970 5971 return max_tx_rate; 5972 } 5973 5974 /** 5975 * i40e_set_bw_limit - setup BW limit for Tx traffic based on max_tx_rate 5976 * @vsi: VSI to be configured 5977 * @seid: seid of the channel/VSI 5978 * @max_tx_rate: max TX rate to be configured as BW limit 5979 * 5980 * Helper function to set BW limit for a given VSI 5981 **/ 5982 int i40e_set_bw_limit(struct i40e_vsi *vsi, u16 seid, u64 max_tx_rate) 5983 { 5984 struct i40e_pf *pf = vsi->back; 5985 u64 credits = 0; 5986 int speed = 0; 5987 int ret = 0; 5988 5989 speed = i40e_get_link_speed(vsi); 5990 if (max_tx_rate > speed) { 5991 dev_err(&pf->pdev->dev, 5992 "Invalid max tx rate %llu specified for VSI seid %d.", 5993 max_tx_rate, seid); 5994 return -EINVAL; 5995 } 5996 if (max_tx_rate && max_tx_rate < I40E_BW_CREDIT_DIVISOR) { 5997 dev_warn(&pf->pdev->dev, 5998 "Setting max tx rate to minimum usable value of 50Mbps.\n"); 5999 max_tx_rate = I40E_BW_CREDIT_DIVISOR; 6000 } 6001 6002 /* Tx rate credits are in values of 50Mbps, 0 is disabled */ 6003 credits = max_tx_rate; 6004 do_div(credits, I40E_BW_CREDIT_DIVISOR); 6005 ret = i40e_aq_config_vsi_bw_limit(&pf->hw, seid, credits, 6006 I40E_MAX_BW_INACTIVE_ACCUM, NULL); 6007 if (ret) 6008 dev_err(&pf->pdev->dev, 6009 "Failed set tx rate (%llu Mbps) for vsi->seid %u, err %pe aq_err %s\n", 6010 max_tx_rate, seid, ERR_PTR(ret), 6011 libie_aq_str(pf->hw.aq.asq_last_status)); 6012 return ret; 6013 } 6014 6015 /** 6016 * i40e_remove_queue_channels - Remove queue channels for the TCs 6017 * @vsi: VSI to be configured 6018 * 6019 * Remove queue channels for the TCs 6020 **/ 6021 static void i40e_remove_queue_channels(struct i40e_vsi *vsi) 6022 { 6023 struct i40e_cloud_filter *cfilter; 6024 enum libie_aq_err last_aq_status; 6025 struct i40e_channel *ch, *ch_tmp; 6026 struct i40e_pf *pf = vsi->back; 6027 struct hlist_node *node; 6028 int ret, i; 6029 6030 /* Reset rss size that was stored when reconfiguring rss for 6031 * channel VSIs with non-power-of-2 queue count. 6032 */ 6033 vsi->current_rss_size = 0; 6034 6035 /* perform cleanup for channels if they exist */ 6036 if (list_empty(&vsi->ch_list)) 6037 return; 6038 6039 list_for_each_entry_safe(ch, ch_tmp, &vsi->ch_list, list) { 6040 struct i40e_vsi *p_vsi; 6041 6042 list_del(&ch->list); 6043 p_vsi = ch->parent_vsi; 6044 if (!p_vsi || !ch->initialized) { 6045 kfree(ch); 6046 continue; 6047 } 6048 /* Reset queue contexts */ 6049 for (i = 0; i < ch->num_queue_pairs; i++) { 6050 struct i40e_ring *tx_ring, *rx_ring; 6051 u16 pf_q; 6052 6053 pf_q = ch->base_queue + i; 6054 tx_ring = vsi->tx_rings[pf_q]; 6055 tx_ring->ch = NULL; 6056 6057 rx_ring = vsi->rx_rings[pf_q]; 6058 rx_ring->ch = NULL; 6059 } 6060 6061 /* Reset BW configured for this VSI via mqprio */ 6062 ret = i40e_set_bw_limit(vsi, ch->seid, 0); 6063 if (ret) 6064 dev_info(&vsi->back->pdev->dev, 6065 "Failed to reset tx rate for ch->seid %u\n", 6066 ch->seid); 6067 6068 /* delete cloud filters associated with this channel */ 6069 hlist_for_each_entry_safe(cfilter, node, 6070 &pf->cloud_filter_list, cloud_node) { 6071 if (cfilter->seid != ch->seid) 6072 continue; 6073 6074 hash_del(&cfilter->cloud_node); 6075 if (cfilter->dst_port) 6076 ret = i40e_add_del_cloud_filter_big_buf(vsi, 6077 cfilter, 6078 false); 6079 else 6080 ret = i40e_add_del_cloud_filter(vsi, cfilter, 6081 false); 6082 last_aq_status = pf->hw.aq.asq_last_status; 6083 if (ret) 6084 dev_info(&pf->pdev->dev, 6085 "Failed to delete cloud filter, err %pe aq_err %s\n", 6086 ERR_PTR(ret), 6087 libie_aq_str(last_aq_status)); 6088 kfree(cfilter); 6089 } 6090 6091 /* delete VSI from FW */ 6092 ret = i40e_aq_delete_element(&vsi->back->hw, ch->seid, 6093 NULL); 6094 if (ret) 6095 dev_err(&vsi->back->pdev->dev, 6096 "unable to remove channel (%d) for parent VSI(%d)\n", 6097 ch->seid, p_vsi->seid); 6098 kfree(ch); 6099 } 6100 INIT_LIST_HEAD(&vsi->ch_list); 6101 } 6102 6103 /** 6104 * i40e_get_max_queues_for_channel 6105 * @vsi: ptr to VSI to which channels are associated with 6106 * 6107 * Helper function which returns max value among the queue counts set on the 6108 * channels/TCs created. 6109 **/ 6110 static int i40e_get_max_queues_for_channel(struct i40e_vsi *vsi) 6111 { 6112 struct i40e_channel *ch, *ch_tmp; 6113 int max = 0; 6114 6115 list_for_each_entry_safe(ch, ch_tmp, &vsi->ch_list, list) { 6116 if (!ch->initialized) 6117 continue; 6118 if (ch->num_queue_pairs > max) 6119 max = ch->num_queue_pairs; 6120 } 6121 6122 return max; 6123 } 6124 6125 /** 6126 * i40e_validate_num_queues - validate num_queues w.r.t channel 6127 * @pf: ptr to PF device 6128 * @num_queues: number of queues 6129 * @vsi: the parent VSI 6130 * @reconfig_rss: indicates should the RSS be reconfigured or not 6131 * 6132 * This function validates number of queues in the context of new channel 6133 * which is being established and determines if RSS should be reconfigured 6134 * or not for parent VSI. 6135 **/ 6136 static int i40e_validate_num_queues(struct i40e_pf *pf, int num_queues, 6137 struct i40e_vsi *vsi, bool *reconfig_rss) 6138 { 6139 int max_ch_queues; 6140 6141 if (!reconfig_rss) 6142 return -EINVAL; 6143 6144 *reconfig_rss = false; 6145 if (vsi->current_rss_size) { 6146 if (num_queues > vsi->current_rss_size) { 6147 dev_dbg(&pf->pdev->dev, 6148 "Error: num_queues (%d) > vsi's current_size(%d)\n", 6149 num_queues, vsi->current_rss_size); 6150 return -EINVAL; 6151 } else if ((num_queues < vsi->current_rss_size) && 6152 (!is_power_of_2(num_queues))) { 6153 dev_dbg(&pf->pdev->dev, 6154 "Error: num_queues (%d) < vsi's current_size(%d), but not power of 2\n", 6155 num_queues, vsi->current_rss_size); 6156 return -EINVAL; 6157 } 6158 } 6159 6160 if (!is_power_of_2(num_queues)) { 6161 /* Find the max num_queues configured for channel if channel 6162 * exist. 6163 * if channel exist, then enforce 'num_queues' to be more than 6164 * max ever queues configured for channel. 6165 */ 6166 max_ch_queues = i40e_get_max_queues_for_channel(vsi); 6167 if (num_queues < max_ch_queues) { 6168 dev_dbg(&pf->pdev->dev, 6169 "Error: num_queues (%d) < max queues configured for channel(%d)\n", 6170 num_queues, max_ch_queues); 6171 return -EINVAL; 6172 } 6173 *reconfig_rss = true; 6174 } 6175 6176 return 0; 6177 } 6178 6179 /** 6180 * i40e_vsi_reconfig_rss - reconfig RSS based on specified rss_size 6181 * @vsi: the VSI being setup 6182 * @rss_size: size of RSS, accordingly LUT gets reprogrammed 6183 * 6184 * This function reconfigures RSS by reprogramming LUTs using 'rss_size' 6185 **/ 6186 static int i40e_vsi_reconfig_rss(struct i40e_vsi *vsi, u16 rss_size) 6187 { 6188 struct i40e_pf *pf = vsi->back; 6189 u8 seed[I40E_HKEY_ARRAY_SIZE]; 6190 struct i40e_hw *hw = &pf->hw; 6191 int local_rss_size; 6192 u8 *lut; 6193 int ret; 6194 6195 if (!vsi->rss_size) 6196 return -EINVAL; 6197 6198 if (rss_size > vsi->rss_size) 6199 return -EINVAL; 6200 6201 local_rss_size = min_t(int, vsi->rss_size, rss_size); 6202 lut = kzalloc(vsi->rss_table_size, GFP_KERNEL); 6203 if (!lut) 6204 return -ENOMEM; 6205 6206 /* Ignoring user configured lut if there is one */ 6207 i40e_fill_rss_lut(pf, lut, vsi->rss_table_size, local_rss_size); 6208 6209 /* Use user configured hash key if there is one, otherwise 6210 * use default. 6211 */ 6212 if (vsi->rss_hkey_user) 6213 memcpy(seed, vsi->rss_hkey_user, I40E_HKEY_ARRAY_SIZE); 6214 else 6215 netdev_rss_key_fill((void *)seed, I40E_HKEY_ARRAY_SIZE); 6216 6217 ret = i40e_config_rss(vsi, seed, lut, vsi->rss_table_size); 6218 if (ret) { 6219 dev_info(&pf->pdev->dev, 6220 "Cannot set RSS lut, err %pe aq_err %s\n", 6221 ERR_PTR(ret), 6222 libie_aq_str(hw->aq.asq_last_status)); 6223 kfree(lut); 6224 return ret; 6225 } 6226 kfree(lut); 6227 6228 /* Do the update w.r.t. storing rss_size */ 6229 if (!vsi->orig_rss_size) 6230 vsi->orig_rss_size = vsi->rss_size; 6231 vsi->current_rss_size = local_rss_size; 6232 6233 return ret; 6234 } 6235 6236 /** 6237 * i40e_channel_setup_queue_map - Setup a channel queue map 6238 * @pf: ptr to PF device 6239 * @ctxt: VSI context structure 6240 * @ch: ptr to channel structure 6241 * 6242 * Setup queue map for a specific channel 6243 **/ 6244 static void i40e_channel_setup_queue_map(struct i40e_pf *pf, 6245 struct i40e_vsi_context *ctxt, 6246 struct i40e_channel *ch) 6247 { 6248 u16 qcount, qmap, sections = 0; 6249 u8 offset = 0; 6250 int pow; 6251 6252 sections = I40E_AQ_VSI_PROP_QUEUE_MAP_VALID; 6253 sections |= I40E_AQ_VSI_PROP_SCHED_VALID; 6254 6255 qcount = min_t(int, ch->num_queue_pairs, pf->num_lan_msix); 6256 ch->num_queue_pairs = qcount; 6257 6258 /* find the next higher power-of-2 of num queue pairs */ 6259 pow = ilog2(qcount); 6260 if (!is_power_of_2(qcount)) 6261 pow++; 6262 6263 qmap = (offset << I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT) | 6264 (pow << I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT); 6265 6266 /* Setup queue TC[0].qmap for given VSI context */ 6267 ctxt->info.tc_mapping[0] = cpu_to_le16(qmap); 6268 6269 ctxt->info.up_enable_bits = 0x1; /* TC0 enabled */ 6270 ctxt->info.mapping_flags |= cpu_to_le16(I40E_AQ_VSI_QUE_MAP_CONTIG); 6271 ctxt->info.queue_mapping[0] = cpu_to_le16(ch->base_queue); 6272 ctxt->info.valid_sections |= cpu_to_le16(sections); 6273 } 6274 6275 /** 6276 * i40e_add_channel - add a channel by adding VSI 6277 * @pf: ptr to PF device 6278 * @uplink_seid: underlying HW switching element (VEB) ID 6279 * @ch: ptr to channel structure 6280 * 6281 * Add a channel (VSI) using add_vsi and queue_map 6282 **/ 6283 static int i40e_add_channel(struct i40e_pf *pf, u16 uplink_seid, 6284 struct i40e_channel *ch) 6285 { 6286 struct i40e_hw *hw = &pf->hw; 6287 struct i40e_vsi_context ctxt; 6288 u8 enabled_tc = 0x1; /* TC0 enabled */ 6289 int ret; 6290 6291 if (ch->type != I40E_VSI_VMDQ2) { 6292 dev_info(&pf->pdev->dev, 6293 "add new vsi failed, ch->type %d\n", ch->type); 6294 return -EINVAL; 6295 } 6296 6297 memset(&ctxt, 0, sizeof(ctxt)); 6298 ctxt.pf_num = hw->pf_id; 6299 ctxt.vf_num = 0; 6300 ctxt.uplink_seid = uplink_seid; 6301 ctxt.connection_type = I40E_AQ_VSI_CONN_TYPE_NORMAL; 6302 if (ch->type == I40E_VSI_VMDQ2) 6303 ctxt.flags = I40E_AQ_VSI_TYPE_VMDQ2; 6304 6305 if (test_bit(I40E_FLAG_VEB_MODE_ENA, pf->flags)) { 6306 ctxt.info.valid_sections |= 6307 cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID); 6308 ctxt.info.switch_id = 6309 cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB); 6310 } 6311 6312 /* Set queue map for a given VSI context */ 6313 i40e_channel_setup_queue_map(pf, &ctxt, ch); 6314 6315 /* Now time to create VSI */ 6316 ret = i40e_aq_add_vsi(hw, &ctxt, NULL); 6317 if (ret) { 6318 dev_info(&pf->pdev->dev, 6319 "add new vsi failed, err %pe aq_err %s\n", 6320 ERR_PTR(ret), 6321 libie_aq_str(pf->hw.aq.asq_last_status)); 6322 return -ENOENT; 6323 } 6324 6325 /* Success, update channel, set enabled_tc only if the channel 6326 * is not a macvlan 6327 */ 6328 ch->enabled_tc = !i40e_is_channel_macvlan(ch) && enabled_tc; 6329 ch->seid = ctxt.seid; 6330 ch->vsi_number = ctxt.vsi_number; 6331 ch->stat_counter_idx = le16_to_cpu(ctxt.info.stat_counter_idx); 6332 6333 /* copy just the sections touched not the entire info 6334 * since not all sections are valid as returned by 6335 * update vsi params 6336 */ 6337 ch->info.mapping_flags = ctxt.info.mapping_flags; 6338 memcpy(&ch->info.queue_mapping, 6339 &ctxt.info.queue_mapping, sizeof(ctxt.info.queue_mapping)); 6340 memcpy(&ch->info.tc_mapping, ctxt.info.tc_mapping, 6341 sizeof(ctxt.info.tc_mapping)); 6342 6343 return 0; 6344 } 6345 6346 static int i40e_channel_config_bw(struct i40e_vsi *vsi, struct i40e_channel *ch, 6347 u8 *bw_share) 6348 { 6349 struct i40e_aqc_configure_vsi_tc_bw_data bw_data; 6350 int ret; 6351 int i; 6352 6353 memset(&bw_data, 0, sizeof(bw_data)); 6354 bw_data.tc_valid_bits = ch->enabled_tc; 6355 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) 6356 bw_data.tc_bw_credits[i] = bw_share[i]; 6357 6358 ret = i40e_aq_config_vsi_tc_bw(&vsi->back->hw, ch->seid, 6359 &bw_data, NULL); 6360 if (ret) { 6361 dev_info(&vsi->back->pdev->dev, 6362 "Config VSI BW allocation per TC failed, aq_err: %d for new_vsi->seid %u\n", 6363 vsi->back->hw.aq.asq_last_status, ch->seid); 6364 return -EINVAL; 6365 } 6366 6367 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) 6368 ch->info.qs_handle[i] = bw_data.qs_handles[i]; 6369 6370 return 0; 6371 } 6372 6373 /** 6374 * i40e_channel_config_tx_ring - config TX ring associated with new channel 6375 * @pf: ptr to PF device 6376 * @vsi: the VSI being setup 6377 * @ch: ptr to channel structure 6378 * 6379 * Configure TX rings associated with channel (VSI) since queues are being 6380 * from parent VSI. 6381 **/ 6382 static int i40e_channel_config_tx_ring(struct i40e_pf *pf, 6383 struct i40e_vsi *vsi, 6384 struct i40e_channel *ch) 6385 { 6386 u8 bw_share[I40E_MAX_TRAFFIC_CLASS] = {0}; 6387 int ret; 6388 int i; 6389 6390 /* Enable ETS TCs with equal BW Share for now across all VSIs */ 6391 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 6392 if (ch->enabled_tc & BIT(i)) 6393 bw_share[i] = 1; 6394 } 6395 6396 /* configure BW for new VSI */ 6397 ret = i40e_channel_config_bw(vsi, ch, bw_share); 6398 if (ret) { 6399 dev_info(&vsi->back->pdev->dev, 6400 "Failed configuring TC map %d for channel (seid %u)\n", 6401 ch->enabled_tc, ch->seid); 6402 return ret; 6403 } 6404 6405 for (i = 0; i < ch->num_queue_pairs; i++) { 6406 struct i40e_ring *tx_ring, *rx_ring; 6407 u16 pf_q; 6408 6409 pf_q = ch->base_queue + i; 6410 6411 /* Get to TX ring ptr of main VSI, for re-setup TX queue 6412 * context 6413 */ 6414 tx_ring = vsi->tx_rings[pf_q]; 6415 tx_ring->ch = ch; 6416 6417 /* Get the RX ring ptr */ 6418 rx_ring = vsi->rx_rings[pf_q]; 6419 rx_ring->ch = ch; 6420 } 6421 6422 return 0; 6423 } 6424 6425 /** 6426 * i40e_setup_hw_channel - setup new channel 6427 * @pf: ptr to PF device 6428 * @vsi: the VSI being setup 6429 * @ch: ptr to channel structure 6430 * @uplink_seid: underlying HW switching element (VEB) ID 6431 * @type: type of channel to be created (VMDq2/VF) 6432 * 6433 * Setup new channel (VSI) based on specified type (VMDq2/VF) 6434 * and configures TX rings accordingly 6435 **/ 6436 static inline int i40e_setup_hw_channel(struct i40e_pf *pf, 6437 struct i40e_vsi *vsi, 6438 struct i40e_channel *ch, 6439 u16 uplink_seid, u8 type) 6440 { 6441 int ret; 6442 6443 ch->initialized = false; 6444 ch->base_queue = vsi->next_base_queue; 6445 ch->type = type; 6446 6447 /* Proceed with creation of channel (VMDq2) VSI */ 6448 ret = i40e_add_channel(pf, uplink_seid, ch); 6449 if (ret) { 6450 dev_info(&pf->pdev->dev, 6451 "failed to add_channel using uplink_seid %u\n", 6452 uplink_seid); 6453 return ret; 6454 } 6455 6456 /* Mark the successful creation of channel */ 6457 ch->initialized = true; 6458 6459 /* Reconfigure TX queues using QTX_CTL register */ 6460 ret = i40e_channel_config_tx_ring(pf, vsi, ch); 6461 if (ret) { 6462 dev_info(&pf->pdev->dev, 6463 "failed to configure TX rings for channel %u\n", 6464 ch->seid); 6465 return ret; 6466 } 6467 6468 /* update 'next_base_queue' */ 6469 vsi->next_base_queue = vsi->next_base_queue + ch->num_queue_pairs; 6470 dev_dbg(&pf->pdev->dev, 6471 "Added channel: vsi_seid %u, vsi_number %u, stat_counter_idx %u, num_queue_pairs %u, pf->next_base_queue %d\n", 6472 ch->seid, ch->vsi_number, ch->stat_counter_idx, 6473 ch->num_queue_pairs, 6474 vsi->next_base_queue); 6475 return ret; 6476 } 6477 6478 /** 6479 * i40e_setup_channel - setup new channel using uplink element 6480 * @pf: ptr to PF device 6481 * @vsi: pointer to the VSI to set up the channel within 6482 * @ch: ptr to channel structure 6483 * 6484 * Setup new channel (VSI) based on specified type (VMDq2/VF) 6485 * and uplink switching element (uplink_seid) 6486 **/ 6487 static bool i40e_setup_channel(struct i40e_pf *pf, struct i40e_vsi *vsi, 6488 struct i40e_channel *ch) 6489 { 6490 struct i40e_vsi *main_vsi; 6491 u8 vsi_type; 6492 u16 seid; 6493 int ret; 6494 6495 if (vsi->type == I40E_VSI_MAIN) { 6496 vsi_type = I40E_VSI_VMDQ2; 6497 } else { 6498 dev_err(&pf->pdev->dev, "unsupported parent vsi type(%d)\n", 6499 vsi->type); 6500 return false; 6501 } 6502 6503 /* underlying switching element */ 6504 main_vsi = i40e_pf_get_main_vsi(pf); 6505 seid = main_vsi->uplink_seid; 6506 6507 /* create channel (VSI), configure TX rings */ 6508 ret = i40e_setup_hw_channel(pf, vsi, ch, seid, vsi_type); 6509 if (ret) { 6510 dev_err(&pf->pdev->dev, "failed to setup hw_channel\n"); 6511 return false; 6512 } 6513 6514 return ch->initialized ? true : false; 6515 } 6516 6517 /** 6518 * i40e_validate_and_set_switch_mode - sets up switch mode correctly 6519 * @vsi: ptr to VSI which has PF backing 6520 * 6521 * Sets up switch mode correctly if it needs to be changed and perform 6522 * what are allowed modes. 6523 **/ 6524 static int i40e_validate_and_set_switch_mode(struct i40e_vsi *vsi) 6525 { 6526 u8 mode; 6527 struct i40e_pf *pf = vsi->back; 6528 struct i40e_hw *hw = &pf->hw; 6529 int ret; 6530 6531 ret = i40e_get_capabilities(pf, i40e_aqc_opc_list_dev_capabilities); 6532 if (ret) 6533 return -EINVAL; 6534 6535 if (hw->dev_caps.switch_mode) { 6536 /* if switch mode is set, support mode2 (non-tunneled for 6537 * cloud filter) for now 6538 */ 6539 u32 switch_mode = hw->dev_caps.switch_mode & 6540 I40E_SWITCH_MODE_MASK; 6541 if (switch_mode >= I40E_CLOUD_FILTER_MODE1) { 6542 if (switch_mode == I40E_CLOUD_FILTER_MODE2) 6543 return 0; 6544 dev_err(&pf->pdev->dev, 6545 "Invalid switch_mode (%d), only non-tunneled mode for cloud filter is supported\n", 6546 hw->dev_caps.switch_mode); 6547 return -EINVAL; 6548 } 6549 } 6550 6551 /* Set Bit 7 to be valid */ 6552 mode = I40E_AQ_SET_SWITCH_BIT7_VALID; 6553 6554 /* Set L4type for TCP support */ 6555 mode |= I40E_AQ_SET_SWITCH_L4_TYPE_TCP; 6556 6557 /* Set cloud filter mode */ 6558 mode |= I40E_AQ_SET_SWITCH_MODE_NON_TUNNEL; 6559 6560 /* Prep mode field for set_switch_config */ 6561 ret = i40e_aq_set_switch_config(hw, pf->last_sw_conf_flags, 6562 pf->last_sw_conf_valid_flags, 6563 mode, NULL); 6564 if (ret && hw->aq.asq_last_status != LIBIE_AQ_RC_ESRCH) 6565 dev_err(&pf->pdev->dev, 6566 "couldn't set switch config bits, err %pe aq_err %s\n", 6567 ERR_PTR(ret), libie_aq_str(hw->aq.asq_last_status)); 6568 6569 return ret; 6570 } 6571 6572 /** 6573 * i40e_create_queue_channel - function to create channel 6574 * @vsi: VSI to be configured 6575 * @ch: ptr to channel (it contains channel specific params) 6576 * 6577 * This function creates channel (VSI) using num_queues specified by user, 6578 * reconfigs RSS if needed. 6579 **/ 6580 int i40e_create_queue_channel(struct i40e_vsi *vsi, 6581 struct i40e_channel *ch) 6582 { 6583 struct i40e_pf *pf = vsi->back; 6584 bool reconfig_rss; 6585 int err; 6586 6587 if (!ch) 6588 return -EINVAL; 6589 6590 if (!ch->num_queue_pairs) { 6591 dev_err(&pf->pdev->dev, "Invalid num_queues requested: %d\n", 6592 ch->num_queue_pairs); 6593 return -EINVAL; 6594 } 6595 6596 /* validate user requested num_queues for channel */ 6597 err = i40e_validate_num_queues(pf, ch->num_queue_pairs, vsi, 6598 &reconfig_rss); 6599 if (err) { 6600 dev_info(&pf->pdev->dev, "Failed to validate num_queues (%d)\n", 6601 ch->num_queue_pairs); 6602 return -EINVAL; 6603 } 6604 6605 /* By default we are in VEPA mode, if this is the first VF/VMDq 6606 * VSI to be added switch to VEB mode. 6607 */ 6608 6609 if (!test_bit(I40E_FLAG_VEB_MODE_ENA, pf->flags)) { 6610 set_bit(I40E_FLAG_VEB_MODE_ENA, pf->flags); 6611 6612 if (vsi->type == I40E_VSI_MAIN) { 6613 if (i40e_is_tc_mqprio_enabled(pf)) 6614 i40e_do_reset(pf, I40E_PF_RESET_FLAG, true); 6615 else 6616 i40e_do_reset_safe(pf, I40E_PF_RESET_FLAG); 6617 } 6618 /* now onwards for main VSI, number of queues will be value 6619 * of TC0's queue count 6620 */ 6621 } 6622 6623 /* By this time, vsi->cnt_q_avail shall be set to non-zero and 6624 * it should be more than num_queues 6625 */ 6626 if (!vsi->cnt_q_avail || vsi->cnt_q_avail < ch->num_queue_pairs) { 6627 dev_dbg(&pf->pdev->dev, 6628 "Error: cnt_q_avail (%u) less than num_queues %d\n", 6629 vsi->cnt_q_avail, ch->num_queue_pairs); 6630 return -EINVAL; 6631 } 6632 6633 /* reconfig_rss only if vsi type is MAIN_VSI */ 6634 if (reconfig_rss && (vsi->type == I40E_VSI_MAIN)) { 6635 err = i40e_vsi_reconfig_rss(vsi, ch->num_queue_pairs); 6636 if (err) { 6637 dev_info(&pf->pdev->dev, 6638 "Error: unable to reconfig rss for num_queues (%u)\n", 6639 ch->num_queue_pairs); 6640 return -EINVAL; 6641 } 6642 } 6643 6644 if (!i40e_setup_channel(pf, vsi, ch)) { 6645 dev_info(&pf->pdev->dev, "Failed to setup channel\n"); 6646 return -EINVAL; 6647 } 6648 6649 dev_info(&pf->pdev->dev, 6650 "Setup channel (id:%u) utilizing num_queues %d\n", 6651 ch->seid, ch->num_queue_pairs); 6652 6653 /* configure VSI for BW limit */ 6654 if (ch->max_tx_rate) { 6655 u64 credits = ch->max_tx_rate; 6656 6657 if (i40e_set_bw_limit(vsi, ch->seid, ch->max_tx_rate)) 6658 return -EINVAL; 6659 6660 do_div(credits, I40E_BW_CREDIT_DIVISOR); 6661 dev_dbg(&pf->pdev->dev, 6662 "Set tx rate of %llu Mbps (count of 50Mbps %llu) for vsi->seid %u\n", 6663 ch->max_tx_rate, 6664 credits, 6665 ch->seid); 6666 } 6667 6668 /* in case of VF, this will be main SRIOV VSI */ 6669 ch->parent_vsi = vsi; 6670 6671 /* and update main_vsi's count for queue_available to use */ 6672 vsi->cnt_q_avail -= ch->num_queue_pairs; 6673 6674 return 0; 6675 } 6676 6677 /** 6678 * i40e_configure_queue_channels - Add queue channel for the given TCs 6679 * @vsi: VSI to be configured 6680 * 6681 * Configures queue channel mapping to the given TCs 6682 **/ 6683 static int i40e_configure_queue_channels(struct i40e_vsi *vsi) 6684 { 6685 struct i40e_channel *ch; 6686 u64 max_rate = 0; 6687 int ret = 0, i; 6688 6689 /* Create app vsi with the TCs. Main VSI with TC0 is already set up */ 6690 vsi->tc_seid_map[0] = vsi->seid; 6691 for (i = 1; i < I40E_MAX_TRAFFIC_CLASS; i++) { 6692 if (vsi->tc_config.enabled_tc & BIT(i)) { 6693 ch = kzalloc_obj(*ch); 6694 if (!ch) { 6695 ret = -ENOMEM; 6696 goto err_free; 6697 } 6698 6699 INIT_LIST_HEAD(&ch->list); 6700 ch->num_queue_pairs = 6701 vsi->tc_config.tc_info[i].qcount; 6702 ch->base_queue = 6703 vsi->tc_config.tc_info[i].qoffset; 6704 6705 /* Bandwidth limit through tc interface is in bytes/s, 6706 * change to Mbit/s 6707 */ 6708 max_rate = vsi->mqprio_qopt.max_rate[i]; 6709 do_div(max_rate, I40E_BW_MBPS_DIVISOR); 6710 ch->max_tx_rate = max_rate; 6711 6712 list_add_tail(&ch->list, &vsi->ch_list); 6713 6714 ret = i40e_create_queue_channel(vsi, ch); 6715 if (ret) { 6716 dev_err(&vsi->back->pdev->dev, 6717 "Failed creating queue channel with TC%d: queues %d\n", 6718 i, ch->num_queue_pairs); 6719 goto err_free; 6720 } 6721 vsi->tc_seid_map[i] = ch->seid; 6722 } 6723 } 6724 6725 /* reset to reconfigure TX queue contexts */ 6726 i40e_do_reset(vsi->back, I40E_PF_RESET_FLAG, true); 6727 return ret; 6728 6729 err_free: 6730 i40e_remove_queue_channels(vsi); 6731 return ret; 6732 } 6733 6734 /** 6735 * i40e_veb_config_tc - Configure TCs for given VEB 6736 * @veb: given VEB 6737 * @enabled_tc: TC bitmap 6738 * 6739 * Configures given TC bitmap for VEB (switching) element 6740 **/ 6741 int i40e_veb_config_tc(struct i40e_veb *veb, u8 enabled_tc) 6742 { 6743 struct i40e_aqc_configure_switching_comp_bw_config_data bw_data = {0}; 6744 struct i40e_pf *pf = veb->pf; 6745 int ret = 0; 6746 int i; 6747 6748 /* No TCs or already enabled TCs just return */ 6749 if (!enabled_tc || veb->enabled_tc == enabled_tc) 6750 return ret; 6751 6752 bw_data.tc_valid_bits = enabled_tc; 6753 /* bw_data.absolute_credits is not set (relative) */ 6754 6755 /* Enable ETS TCs with equal BW Share for now */ 6756 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 6757 if (enabled_tc & BIT(i)) 6758 bw_data.tc_bw_share_credits[i] = 1; 6759 } 6760 6761 ret = i40e_aq_config_switch_comp_bw_config(&pf->hw, veb->seid, 6762 &bw_data, NULL); 6763 if (ret) { 6764 dev_info(&pf->pdev->dev, 6765 "VEB bw config failed, err %pe aq_err %s\n", 6766 ERR_PTR(ret), libie_aq_str(pf->hw.aq.asq_last_status)); 6767 goto out; 6768 } 6769 6770 /* Update the BW information */ 6771 ret = i40e_veb_get_bw_info(veb); 6772 if (ret) { 6773 dev_info(&pf->pdev->dev, 6774 "Failed getting veb bw config, err %pe aq_err %s\n", 6775 ERR_PTR(ret), libie_aq_str(pf->hw.aq.asq_last_status)); 6776 } 6777 6778 out: 6779 return ret; 6780 } 6781 6782 #ifdef CONFIG_I40E_DCB 6783 /** 6784 * i40e_dcb_reconfigure - Reconfigure all VEBs and VSIs 6785 * @pf: PF struct 6786 * 6787 * Reconfigure VEB/VSIs on a given PF; it is assumed that 6788 * the caller would've quiesce all the VSIs before calling 6789 * this function 6790 **/ 6791 static void i40e_dcb_reconfigure(struct i40e_pf *pf) 6792 { 6793 struct i40e_vsi *vsi; 6794 struct i40e_veb *veb; 6795 u8 tc_map = 0; 6796 int ret; 6797 int v; 6798 6799 /* Enable the TCs available on PF to all VEBs */ 6800 tc_map = i40e_pf_get_tc_map(pf); 6801 if (tc_map == I40E_DEFAULT_TRAFFIC_CLASS) 6802 return; 6803 6804 i40e_pf_for_each_veb(pf, v, veb) { 6805 ret = i40e_veb_config_tc(veb, tc_map); 6806 if (ret) { 6807 dev_info(&pf->pdev->dev, 6808 "Failed configuring TC for VEB seid=%d\n", 6809 veb->seid); 6810 /* Will try to configure as many components */ 6811 } 6812 } 6813 6814 /* Update each VSI */ 6815 i40e_pf_for_each_vsi(pf, v, vsi) { 6816 /* - Enable all TCs for the LAN VSI 6817 * - For all others keep them at TC0 for now 6818 */ 6819 if (vsi->type == I40E_VSI_MAIN) 6820 tc_map = i40e_pf_get_tc_map(pf); 6821 else 6822 tc_map = I40E_DEFAULT_TRAFFIC_CLASS; 6823 6824 ret = i40e_vsi_config_tc(vsi, tc_map); 6825 if (ret) { 6826 dev_info(&pf->pdev->dev, 6827 "Failed configuring TC for VSI seid=%d\n", 6828 vsi->seid); 6829 /* Will try to configure as many components */ 6830 } else { 6831 /* Re-configure VSI vectors based on updated TC map */ 6832 i40e_vsi_map_rings_to_vectors(vsi); 6833 if (vsi->netdev) 6834 i40e_dcbnl_set_all(vsi); 6835 } 6836 } 6837 } 6838 6839 /** 6840 * i40e_resume_port_tx - Resume port Tx 6841 * @pf: PF struct 6842 * 6843 * Resume a port's Tx and issue a PF reset in case of failure to 6844 * resume. 6845 **/ 6846 static int i40e_resume_port_tx(struct i40e_pf *pf) 6847 { 6848 struct i40e_hw *hw = &pf->hw; 6849 int ret; 6850 6851 ret = i40e_aq_resume_port_tx(hw, NULL); 6852 if (ret) { 6853 dev_info(&pf->pdev->dev, 6854 "Resume Port Tx failed, err %pe aq_err %s\n", 6855 ERR_PTR(ret), 6856 libie_aq_str(pf->hw.aq.asq_last_status)); 6857 /* Schedule PF reset to recover */ 6858 set_bit(__I40E_PF_RESET_REQUESTED, pf->state); 6859 i40e_service_event_schedule(pf); 6860 } 6861 6862 return ret; 6863 } 6864 6865 /** 6866 * i40e_suspend_port_tx - Suspend port Tx 6867 * @pf: PF struct 6868 * 6869 * Suspend a port's Tx and issue a PF reset in case of failure. 6870 **/ 6871 static int i40e_suspend_port_tx(struct i40e_pf *pf) 6872 { 6873 struct i40e_hw *hw = &pf->hw; 6874 int ret; 6875 6876 ret = i40e_aq_suspend_port_tx(hw, pf->mac_seid, NULL); 6877 if (ret) { 6878 dev_info(&pf->pdev->dev, 6879 "Suspend Port Tx failed, err %pe aq_err %s\n", 6880 ERR_PTR(ret), libie_aq_str(pf->hw.aq.asq_last_status)); 6881 /* Schedule PF reset to recover */ 6882 set_bit(__I40E_PF_RESET_REQUESTED, pf->state); 6883 i40e_service_event_schedule(pf); 6884 } 6885 6886 return ret; 6887 } 6888 6889 /** 6890 * i40e_hw_set_dcb_config - Program new DCBX settings into HW 6891 * @pf: PF being configured 6892 * @new_cfg: New DCBX configuration 6893 * 6894 * Program DCB settings into HW and reconfigure VEB/VSIs on 6895 * given PF. Uses "Set LLDP MIB" AQC to program the hardware. 6896 **/ 6897 static int i40e_hw_set_dcb_config(struct i40e_pf *pf, 6898 struct i40e_dcbx_config *new_cfg) 6899 { 6900 struct i40e_dcbx_config *old_cfg = &pf->hw.local_dcbx_config; 6901 int ret; 6902 6903 /* Check if need reconfiguration */ 6904 if (!memcmp(&new_cfg, &old_cfg, sizeof(new_cfg))) { 6905 dev_dbg(&pf->pdev->dev, "No Change in DCB Config required.\n"); 6906 return 0; 6907 } 6908 6909 /* Config change disable all VSIs */ 6910 i40e_pf_quiesce_all_vsi(pf); 6911 6912 /* Copy the new config to the current config */ 6913 *old_cfg = *new_cfg; 6914 old_cfg->etsrec = old_cfg->etscfg; 6915 ret = i40e_set_dcb_config(&pf->hw); 6916 if (ret) { 6917 dev_info(&pf->pdev->dev, 6918 "Set DCB Config failed, err %pe aq_err %s\n", 6919 ERR_PTR(ret), libie_aq_str(pf->hw.aq.asq_last_status)); 6920 goto out; 6921 } 6922 6923 /* Changes in configuration update VEB/VSI */ 6924 i40e_dcb_reconfigure(pf); 6925 out: 6926 /* In case of reset do not try to resume anything */ 6927 if (!test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state)) { 6928 /* Re-start the VSIs if disabled */ 6929 ret = i40e_resume_port_tx(pf); 6930 /* In case of error no point in resuming VSIs */ 6931 if (ret) 6932 goto err; 6933 i40e_pf_unquiesce_all_vsi(pf); 6934 } 6935 err: 6936 return ret; 6937 } 6938 6939 /** 6940 * i40e_hw_dcb_config - Program new DCBX settings into HW 6941 * @pf: PF being configured 6942 * @new_cfg: New DCBX configuration 6943 * 6944 * Program DCB settings into HW and reconfigure VEB/VSIs on 6945 * given PF 6946 **/ 6947 int i40e_hw_dcb_config(struct i40e_pf *pf, struct i40e_dcbx_config *new_cfg) 6948 { 6949 struct i40e_aqc_configure_switching_comp_ets_data ets_data; 6950 u8 prio_type[I40E_MAX_TRAFFIC_CLASS] = {0}; 6951 u32 mfs_tc[I40E_MAX_TRAFFIC_CLASS]; 6952 struct i40e_dcbx_config *old_cfg; 6953 u8 mode[I40E_MAX_TRAFFIC_CLASS]; 6954 struct i40e_rx_pb_config pb_cfg; 6955 struct i40e_hw *hw = &pf->hw; 6956 u8 num_ports = hw->num_ports; 6957 bool need_reconfig; 6958 int ret = -EINVAL; 6959 u8 lltc_map = 0; 6960 u8 tc_map = 0; 6961 u8 new_numtc; 6962 u8 i; 6963 6964 dev_dbg(&pf->pdev->dev, "Configuring DCB registers directly\n"); 6965 /* Un-pack information to Program ETS HW via shared API 6966 * numtc, tcmap 6967 * LLTC map 6968 * ETS/NON-ETS arbiter mode 6969 * max exponent (credit refills) 6970 * Total number of ports 6971 * PFC priority bit-map 6972 * Priority Table 6973 * BW % per TC 6974 * Arbiter mode between UPs sharing same TC 6975 * TSA table (ETS or non-ETS) 6976 * EEE enabled or not 6977 * MFS TC table 6978 */ 6979 6980 new_numtc = i40e_dcb_get_num_tc(new_cfg); 6981 6982 memset(&ets_data, 0, sizeof(ets_data)); 6983 for (i = 0; i < new_numtc; i++) { 6984 tc_map |= BIT(i); 6985 switch (new_cfg->etscfg.tsatable[i]) { 6986 case I40E_IEEE_TSA_ETS: 6987 prio_type[i] = I40E_DCB_PRIO_TYPE_ETS; 6988 ets_data.tc_bw_share_credits[i] = 6989 new_cfg->etscfg.tcbwtable[i]; 6990 break; 6991 case I40E_IEEE_TSA_STRICT: 6992 prio_type[i] = I40E_DCB_PRIO_TYPE_STRICT; 6993 lltc_map |= BIT(i); 6994 ets_data.tc_bw_share_credits[i] = 6995 I40E_DCB_STRICT_PRIO_CREDITS; 6996 break; 6997 default: 6998 /* Invalid TSA type */ 6999 need_reconfig = false; 7000 goto out; 7001 } 7002 } 7003 7004 old_cfg = &hw->local_dcbx_config; 7005 /* Check if need reconfiguration */ 7006 need_reconfig = i40e_dcb_need_reconfig(pf, old_cfg, new_cfg); 7007 7008 /* If needed, enable/disable frame tagging, disable all VSIs 7009 * and suspend port tx 7010 */ 7011 if (need_reconfig) { 7012 /* Enable DCB tagging only when more than one TC */ 7013 if (new_numtc > 1) 7014 set_bit(I40E_FLAG_DCB_ENA, pf->flags); 7015 else 7016 clear_bit(I40E_FLAG_DCB_ENA, pf->flags); 7017 7018 set_bit(__I40E_PORT_SUSPENDED, pf->state); 7019 /* Reconfiguration needed quiesce all VSIs */ 7020 i40e_pf_quiesce_all_vsi(pf); 7021 ret = i40e_suspend_port_tx(pf); 7022 if (ret) 7023 goto err; 7024 } 7025 7026 /* Configure Port ETS Tx Scheduler */ 7027 ets_data.tc_valid_bits = tc_map; 7028 ets_data.tc_strict_priority_flags = lltc_map; 7029 ret = i40e_aq_config_switch_comp_ets 7030 (hw, pf->mac_seid, &ets_data, 7031 i40e_aqc_opc_modify_switching_comp_ets, NULL); 7032 if (ret) { 7033 dev_info(&pf->pdev->dev, 7034 "Modify Port ETS failed, err %pe aq_err %s\n", 7035 ERR_PTR(ret), libie_aq_str(pf->hw.aq.asq_last_status)); 7036 goto out; 7037 } 7038 7039 /* Configure Rx ETS HW */ 7040 memset(&mode, I40E_DCB_ARB_MODE_ROUND_ROBIN, sizeof(mode)); 7041 i40e_dcb_hw_set_num_tc(hw, new_numtc); 7042 i40e_dcb_hw_rx_fifo_config(hw, I40E_DCB_ARB_MODE_ROUND_ROBIN, 7043 I40E_DCB_ARB_MODE_STRICT_PRIORITY, 7044 I40E_DCB_DEFAULT_MAX_EXPONENT, 7045 lltc_map); 7046 i40e_dcb_hw_rx_cmd_monitor_config(hw, new_numtc, num_ports); 7047 i40e_dcb_hw_rx_ets_bw_config(hw, new_cfg->etscfg.tcbwtable, mode, 7048 prio_type); 7049 i40e_dcb_hw_pfc_config(hw, new_cfg->pfc.pfcenable, 7050 new_cfg->etscfg.prioritytable); 7051 i40e_dcb_hw_rx_up2tc_config(hw, new_cfg->etscfg.prioritytable); 7052 7053 /* Configure Rx Packet Buffers in HW */ 7054 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 7055 struct i40e_vsi *main_vsi = i40e_pf_get_main_vsi(pf); 7056 7057 mfs_tc[i] = main_vsi->netdev->mtu; 7058 mfs_tc[i] += I40E_PACKET_HDR_PAD; 7059 } 7060 7061 i40e_dcb_hw_calculate_pool_sizes(hw, num_ports, 7062 false, new_cfg->pfc.pfcenable, 7063 mfs_tc, &pb_cfg); 7064 i40e_dcb_hw_rx_pb_config(hw, &pf->pb_cfg, &pb_cfg); 7065 7066 /* Update the local Rx Packet buffer config */ 7067 pf->pb_cfg = pb_cfg; 7068 7069 /* Inform the FW about changes to DCB configuration */ 7070 ret = i40e_aq_dcb_updated(&pf->hw, NULL); 7071 if (ret) { 7072 dev_info(&pf->pdev->dev, 7073 "DCB Updated failed, err %pe aq_err %s\n", 7074 ERR_PTR(ret), libie_aq_str(pf->hw.aq.asq_last_status)); 7075 goto out; 7076 } 7077 7078 /* Update the port DCBx configuration */ 7079 *old_cfg = *new_cfg; 7080 7081 /* Changes in configuration update VEB/VSI */ 7082 i40e_dcb_reconfigure(pf); 7083 out: 7084 /* Re-start the VSIs if disabled */ 7085 if (need_reconfig) { 7086 ret = i40e_resume_port_tx(pf); 7087 7088 clear_bit(__I40E_PORT_SUSPENDED, pf->state); 7089 /* In case of error no point in resuming VSIs */ 7090 if (ret) 7091 goto err; 7092 7093 /* Wait for the PF's queues to be disabled */ 7094 ret = i40e_pf_wait_queues_disabled(pf); 7095 if (ret) { 7096 /* Schedule PF reset to recover */ 7097 set_bit(__I40E_PF_RESET_REQUESTED, pf->state); 7098 i40e_service_event_schedule(pf); 7099 goto err; 7100 } else { 7101 i40e_pf_unquiesce_all_vsi(pf); 7102 set_bit(__I40E_CLIENT_SERVICE_REQUESTED, pf->state); 7103 set_bit(__I40E_CLIENT_L2_CHANGE, pf->state); 7104 } 7105 /* registers are set, lets apply */ 7106 if (test_bit(I40E_HW_CAP_USE_SET_LLDP_MIB, pf->hw.caps)) 7107 ret = i40e_hw_set_dcb_config(pf, new_cfg); 7108 } 7109 7110 err: 7111 return ret; 7112 } 7113 7114 /** 7115 * i40e_dcb_sw_default_config - Set default DCB configuration when DCB in SW 7116 * @pf: PF being queried 7117 * 7118 * Set default DCB configuration in case DCB is to be done in SW. 7119 **/ 7120 int i40e_dcb_sw_default_config(struct i40e_pf *pf) 7121 { 7122 struct i40e_dcbx_config *dcb_cfg = &pf->hw.local_dcbx_config; 7123 struct i40e_aqc_configure_switching_comp_ets_data ets_data; 7124 struct i40e_hw *hw = &pf->hw; 7125 int err; 7126 7127 if (test_bit(I40E_HW_CAP_USE_SET_LLDP_MIB, pf->hw.caps)) { 7128 /* Update the local cached instance with TC0 ETS */ 7129 memset(&pf->tmp_cfg, 0, sizeof(struct i40e_dcbx_config)); 7130 pf->tmp_cfg.etscfg.willing = I40E_IEEE_DEFAULT_ETS_WILLING; 7131 pf->tmp_cfg.etscfg.maxtcs = 0; 7132 pf->tmp_cfg.etscfg.tcbwtable[0] = I40E_IEEE_DEFAULT_ETS_TCBW; 7133 pf->tmp_cfg.etscfg.tsatable[0] = I40E_IEEE_TSA_ETS; 7134 pf->tmp_cfg.pfc.willing = I40E_IEEE_DEFAULT_PFC_WILLING; 7135 pf->tmp_cfg.pfc.pfccap = I40E_MAX_TRAFFIC_CLASS; 7136 /* FW needs one App to configure HW */ 7137 pf->tmp_cfg.numapps = I40E_IEEE_DEFAULT_NUM_APPS; 7138 pf->tmp_cfg.app[0].selector = I40E_APP_SEL_ETHTYPE; 7139 pf->tmp_cfg.app[0].priority = I40E_IEEE_DEFAULT_APP_PRIO; 7140 pf->tmp_cfg.app[0].protocolid = I40E_APP_PROTOID_FCOE; 7141 7142 return i40e_hw_set_dcb_config(pf, &pf->tmp_cfg); 7143 } 7144 7145 memset(&ets_data, 0, sizeof(ets_data)); 7146 ets_data.tc_valid_bits = I40E_DEFAULT_TRAFFIC_CLASS; /* TC0 only */ 7147 ets_data.tc_strict_priority_flags = 0; /* ETS */ 7148 ets_data.tc_bw_share_credits[0] = I40E_IEEE_DEFAULT_ETS_TCBW; /* 100% to TC0 */ 7149 7150 /* Enable ETS on the Physical port */ 7151 err = i40e_aq_config_switch_comp_ets 7152 (hw, pf->mac_seid, &ets_data, 7153 i40e_aqc_opc_enable_switching_comp_ets, NULL); 7154 if (err) { 7155 dev_info(&pf->pdev->dev, 7156 "Enable Port ETS failed, err %pe aq_err %s\n", 7157 ERR_PTR(err), libie_aq_str(pf->hw.aq.asq_last_status)); 7158 err = -ENOENT; 7159 goto out; 7160 } 7161 7162 /* Update the local cached instance with TC0 ETS */ 7163 dcb_cfg->etscfg.willing = I40E_IEEE_DEFAULT_ETS_WILLING; 7164 dcb_cfg->etscfg.cbs = 0; 7165 dcb_cfg->etscfg.maxtcs = I40E_MAX_TRAFFIC_CLASS; 7166 dcb_cfg->etscfg.tcbwtable[0] = I40E_IEEE_DEFAULT_ETS_TCBW; 7167 7168 out: 7169 return err; 7170 } 7171 7172 /** 7173 * i40e_init_pf_dcb - Initialize DCB configuration 7174 * @pf: PF being configured 7175 * 7176 * Query the current DCB configuration and cache it 7177 * in the hardware structure 7178 **/ 7179 static int i40e_init_pf_dcb(struct i40e_pf *pf) 7180 { 7181 struct i40e_hw *hw = &pf->hw; 7182 int err; 7183 7184 /* Do not enable DCB for SW1 and SW2 images even if the FW is capable 7185 * Also do not enable DCBx if FW LLDP agent is disabled 7186 */ 7187 if (test_bit(I40E_HW_CAP_NO_DCB_SUPPORT, pf->hw.caps)) { 7188 dev_info(&pf->pdev->dev, "DCB is not supported.\n"); 7189 err = -EOPNOTSUPP; 7190 goto out; 7191 } 7192 if (test_bit(I40E_FLAG_FW_LLDP_DIS, pf->flags)) { 7193 dev_info(&pf->pdev->dev, "FW LLDP is disabled, attempting SW DCB\n"); 7194 err = i40e_dcb_sw_default_config(pf); 7195 if (err) { 7196 dev_info(&pf->pdev->dev, "Could not initialize SW DCB\n"); 7197 goto out; 7198 } 7199 dev_info(&pf->pdev->dev, "SW DCB initialization succeeded.\n"); 7200 pf->dcbx_cap = DCB_CAP_DCBX_HOST | 7201 DCB_CAP_DCBX_VER_IEEE; 7202 /* at init capable but disabled */ 7203 set_bit(I40E_FLAG_DCB_CAPABLE, pf->flags); 7204 clear_bit(I40E_FLAG_DCB_ENA, pf->flags); 7205 goto out; 7206 } 7207 err = i40e_init_dcb(hw, true); 7208 if (!err) { 7209 /* Device/Function is not DCBX capable */ 7210 if ((!hw->func_caps.dcb) || 7211 (hw->dcbx_status == I40E_DCBX_STATUS_DISABLED)) { 7212 dev_info(&pf->pdev->dev, 7213 "DCBX offload is not supported or is disabled for this PF.\n"); 7214 } else { 7215 /* When status is not DISABLED then DCBX in FW */ 7216 pf->dcbx_cap = DCB_CAP_DCBX_LLD_MANAGED | 7217 DCB_CAP_DCBX_VER_IEEE; 7218 7219 set_bit(I40E_FLAG_DCB_CAPABLE, pf->flags); 7220 /* Enable DCB tagging only when more than one TC 7221 * or explicitly disable if only one TC 7222 */ 7223 if (i40e_dcb_get_num_tc(&hw->local_dcbx_config) > 1) 7224 set_bit(I40E_FLAG_DCB_ENA, pf->flags); 7225 else 7226 clear_bit(I40E_FLAG_DCB_ENA, pf->flags); 7227 dev_dbg(&pf->pdev->dev, 7228 "DCBX offload is supported for this PF.\n"); 7229 } 7230 } else if (pf->hw.aq.asq_last_status == LIBIE_AQ_RC_EPERM) { 7231 dev_info(&pf->pdev->dev, "FW LLDP disabled for this PF.\n"); 7232 set_bit(I40E_FLAG_FW_LLDP_DIS, pf->flags); 7233 } else { 7234 dev_info(&pf->pdev->dev, 7235 "Query for DCB configuration failed, err %pe aq_err %s\n", 7236 ERR_PTR(err), libie_aq_str(pf->hw.aq.asq_last_status)); 7237 } 7238 7239 out: 7240 return err; 7241 } 7242 #endif /* CONFIG_I40E_DCB */ 7243 7244 static void i40e_print_link_message_eee(struct i40e_vsi *vsi, 7245 const char *speed, const char *fc) 7246 { 7247 struct ethtool_keee kedata; 7248 7249 memzero_explicit(&kedata, sizeof(kedata)); 7250 if (vsi->netdev->ethtool_ops->get_eee) 7251 vsi->netdev->ethtool_ops->get_eee(vsi->netdev, &kedata); 7252 7253 if (!linkmode_empty(kedata.supported)) 7254 netdev_info(vsi->netdev, 7255 "NIC Link is Up, %sbps Full Duplex, Flow Control: %s, EEE: %s\n", 7256 speed, fc, 7257 kedata.eee_enabled ? "Enabled" : "Disabled"); 7258 else 7259 netdev_info(vsi->netdev, 7260 "NIC Link is Up, %sbps Full Duplex, Flow Control: %s\n", 7261 speed, fc); 7262 } 7263 7264 /** 7265 * i40e_print_link_message - print link up or down 7266 * @vsi: the VSI for which link needs a message 7267 * @isup: true of link is up, false otherwise 7268 */ 7269 void i40e_print_link_message(struct i40e_vsi *vsi, bool isup) 7270 { 7271 enum i40e_aq_link_speed new_speed; 7272 struct i40e_pf *pf = vsi->back; 7273 char *speed = "Unknown"; 7274 char *fc = "Unknown"; 7275 char *fec = ""; 7276 char *req_fec = ""; 7277 char *an = ""; 7278 7279 if (isup) 7280 new_speed = pf->hw.phy.link_info.link_speed; 7281 else 7282 new_speed = I40E_LINK_SPEED_UNKNOWN; 7283 7284 if ((vsi->current_isup == isup) && (vsi->current_speed == new_speed)) 7285 return; 7286 vsi->current_isup = isup; 7287 vsi->current_speed = new_speed; 7288 if (!isup) { 7289 netdev_info(vsi->netdev, "NIC Link is Down\n"); 7290 return; 7291 } 7292 7293 /* Warn user if link speed on NPAR enabled partition is not at 7294 * least 10GB 7295 */ 7296 if (pf->hw.func_caps.npar_enable && 7297 (pf->hw.phy.link_info.link_speed == I40E_LINK_SPEED_1GB || 7298 pf->hw.phy.link_info.link_speed == I40E_LINK_SPEED_100MB)) 7299 netdev_warn(vsi->netdev, 7300 "The partition detected link speed that is less than 10Gbps\n"); 7301 7302 switch (pf->hw.phy.link_info.link_speed) { 7303 case I40E_LINK_SPEED_40GB: 7304 speed = "40 G"; 7305 break; 7306 case I40E_LINK_SPEED_20GB: 7307 speed = "20 G"; 7308 break; 7309 case I40E_LINK_SPEED_25GB: 7310 speed = "25 G"; 7311 break; 7312 case I40E_LINK_SPEED_10GB: 7313 speed = "10 G"; 7314 break; 7315 case I40E_LINK_SPEED_5GB: 7316 speed = "5 G"; 7317 break; 7318 case I40E_LINK_SPEED_2_5GB: 7319 speed = "2.5 G"; 7320 break; 7321 case I40E_LINK_SPEED_1GB: 7322 speed = "1000 M"; 7323 break; 7324 case I40E_LINK_SPEED_100MB: 7325 speed = "100 M"; 7326 break; 7327 default: 7328 break; 7329 } 7330 7331 switch (pf->hw.fc.current_mode) { 7332 case I40E_FC_FULL: 7333 fc = "RX/TX"; 7334 break; 7335 case I40E_FC_TX_PAUSE: 7336 fc = "TX"; 7337 break; 7338 case I40E_FC_RX_PAUSE: 7339 fc = "RX"; 7340 break; 7341 default: 7342 fc = "None"; 7343 break; 7344 } 7345 7346 if (pf->hw.phy.link_info.link_speed == I40E_LINK_SPEED_25GB) { 7347 req_fec = "None"; 7348 fec = "None"; 7349 an = "False"; 7350 7351 if (pf->hw.phy.link_info.an_info & I40E_AQ_AN_COMPLETED) 7352 an = "True"; 7353 7354 if (pf->hw.phy.link_info.fec_info & 7355 I40E_AQ_CONFIG_FEC_KR_ENA) 7356 fec = "CL74 FC-FEC/BASE-R"; 7357 else if (pf->hw.phy.link_info.fec_info & 7358 I40E_AQ_CONFIG_FEC_RS_ENA) 7359 fec = "CL108 RS-FEC"; 7360 7361 /* 'CL108 RS-FEC' should be displayed when RS is requested, or 7362 * both RS and FC are requested 7363 */ 7364 if (vsi->back->hw.phy.link_info.req_fec_info & 7365 (I40E_AQ_REQUEST_FEC_KR | I40E_AQ_REQUEST_FEC_RS)) { 7366 if (vsi->back->hw.phy.link_info.req_fec_info & 7367 I40E_AQ_REQUEST_FEC_RS) 7368 req_fec = "CL108 RS-FEC"; 7369 else 7370 req_fec = "CL74 FC-FEC/BASE-R"; 7371 } 7372 netdev_info(vsi->netdev, 7373 "NIC Link is Up, %sbps Full Duplex, Requested FEC: %s, Negotiated FEC: %s, Autoneg: %s, Flow Control: %s\n", 7374 speed, req_fec, fec, an, fc); 7375 } else if (pf->hw.device_id == I40E_DEV_ID_KX_X722) { 7376 req_fec = "None"; 7377 fec = "None"; 7378 an = "False"; 7379 7380 if (pf->hw.phy.link_info.an_info & I40E_AQ_AN_COMPLETED) 7381 an = "True"; 7382 7383 if (pf->hw.phy.link_info.fec_info & 7384 I40E_AQ_CONFIG_FEC_KR_ENA) 7385 fec = "CL74 FC-FEC/BASE-R"; 7386 7387 if (pf->hw.phy.link_info.req_fec_info & 7388 I40E_AQ_REQUEST_FEC_KR) 7389 req_fec = "CL74 FC-FEC/BASE-R"; 7390 7391 netdev_info(vsi->netdev, 7392 "NIC Link is Up, %sbps Full Duplex, Requested FEC: %s, Negotiated FEC: %s, Autoneg: %s, Flow Control: %s\n", 7393 speed, req_fec, fec, an, fc); 7394 } else { 7395 i40e_print_link_message_eee(vsi, speed, fc); 7396 } 7397 7398 } 7399 7400 /** 7401 * i40e_up_complete - Finish the last steps of bringing up a connection 7402 * @vsi: the VSI being configured 7403 **/ 7404 static int i40e_up_complete(struct i40e_vsi *vsi) 7405 { 7406 struct i40e_pf *pf = vsi->back; 7407 int err; 7408 7409 if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags)) 7410 i40e_vsi_configure_msix(vsi); 7411 else 7412 i40e_configure_msi_and_legacy(vsi); 7413 7414 /* start rings */ 7415 err = i40e_vsi_start_rings(vsi); 7416 if (err) 7417 return err; 7418 7419 clear_bit(__I40E_VSI_DOWN, vsi->state); 7420 i40e_napi_enable_all(vsi); 7421 i40e_vsi_enable_irq(vsi); 7422 7423 if ((pf->hw.phy.link_info.link_info & I40E_AQ_LINK_UP) && 7424 (vsi->netdev)) { 7425 i40e_print_link_message(vsi, true); 7426 netif_tx_start_all_queues(vsi->netdev); 7427 netif_carrier_on(vsi->netdev); 7428 } 7429 7430 /* replay FDIR SB filters */ 7431 if (vsi->type == I40E_VSI_FDIR) { 7432 /* reset fd counters */ 7433 pf->fd_add_err = 0; 7434 pf->fd_atr_cnt = 0; 7435 i40e_fdir_filter_restore(vsi); 7436 } 7437 7438 /* On the next run of the service_task, notify any clients of the new 7439 * opened netdev 7440 */ 7441 set_bit(__I40E_CLIENT_SERVICE_REQUESTED, pf->state); 7442 i40e_service_event_schedule(pf); 7443 7444 return 0; 7445 } 7446 7447 /** 7448 * i40e_vsi_reinit_locked - Reset the VSI 7449 * @vsi: the VSI being configured 7450 * 7451 * Rebuild the ring structs after some configuration 7452 * has changed, e.g. MTU size. 7453 **/ 7454 static void i40e_vsi_reinit_locked(struct i40e_vsi *vsi) 7455 { 7456 struct i40e_pf *pf = vsi->back; 7457 7458 while (test_and_set_bit(__I40E_CONFIG_BUSY, pf->state)) 7459 usleep_range(1000, 2000); 7460 i40e_down(vsi); 7461 7462 i40e_up(vsi); 7463 clear_bit(__I40E_CONFIG_BUSY, pf->state); 7464 } 7465 7466 /** 7467 * i40e_force_link_state - Force the link status 7468 * @pf: board private structure 7469 * @is_up: whether the link state should be forced up or down 7470 **/ 7471 static int i40e_force_link_state(struct i40e_pf *pf, bool is_up) 7472 { 7473 struct i40e_aq_get_phy_abilities_resp abilities; 7474 struct i40e_aq_set_phy_config config = {0}; 7475 bool non_zero_phy_type = is_up; 7476 struct i40e_hw *hw = &pf->hw; 7477 u64 mask; 7478 u8 speed; 7479 int err; 7480 7481 /* Card might've been put in an unstable state by other drivers 7482 * and applications, which causes incorrect speed values being 7483 * set on startup. In order to clear speed registers, we call 7484 * get_phy_capabilities twice, once to get initial state of 7485 * available speeds, and once to get current PHY config. 7486 */ 7487 err = i40e_aq_get_phy_capabilities(hw, false, true, &abilities, 7488 NULL); 7489 if (err) { 7490 dev_err(&pf->pdev->dev, 7491 "failed to get phy cap., ret = %pe last_status = %s\n", 7492 ERR_PTR(err), libie_aq_str(hw->aq.asq_last_status)); 7493 return err; 7494 } 7495 speed = abilities.link_speed; 7496 7497 /* Get the current phy config */ 7498 err = i40e_aq_get_phy_capabilities(hw, false, false, &abilities, 7499 NULL); 7500 if (err) { 7501 dev_err(&pf->pdev->dev, 7502 "failed to get phy cap., ret = %pe last_status = %s\n", 7503 ERR_PTR(err), libie_aq_str(hw->aq.asq_last_status)); 7504 return err; 7505 } 7506 7507 /* If link needs to go up, but was not forced to go down, 7508 * and its speed values are OK, no need for a flap 7509 * if non_zero_phy_type was set, still need to force up 7510 */ 7511 if (test_bit(I40E_FLAG_TOTAL_PORT_SHUTDOWN_ENA, pf->flags)) 7512 non_zero_phy_type = true; 7513 else if (is_up && abilities.phy_type != 0 && abilities.link_speed != 0) 7514 return 0; 7515 7516 /* To force link we need to set bits for all supported PHY types, 7517 * but there are now more than 32, so we need to split the bitmap 7518 * across two fields. 7519 */ 7520 mask = I40E_PHY_TYPES_BITMASK; 7521 config.phy_type = 7522 non_zero_phy_type ? cpu_to_le32((u32)(mask & 0xffffffff)) : 0; 7523 config.phy_type_ext = 7524 non_zero_phy_type ? (u8)((mask >> 32) & 0xff) : 0; 7525 /* Copy the old settings, except of phy_type */ 7526 config.abilities = abilities.abilities; 7527 if (test_bit(I40E_FLAG_TOTAL_PORT_SHUTDOWN_ENA, pf->flags)) { 7528 if (is_up) 7529 config.abilities |= I40E_AQ_PHY_ENABLE_LINK; 7530 else 7531 config.abilities &= ~(I40E_AQ_PHY_ENABLE_LINK); 7532 } 7533 if (abilities.link_speed != 0) 7534 config.link_speed = abilities.link_speed; 7535 else 7536 config.link_speed = speed; 7537 config.eee_capability = abilities.eee_capability; 7538 config.eeer = abilities.eeer_val; 7539 config.low_power_ctrl = abilities.d3_lpan; 7540 config.fec_config = abilities.fec_cfg_curr_mod_ext_info & 7541 I40E_AQ_PHY_FEC_CONFIG_MASK; 7542 err = i40e_aq_set_phy_config(hw, &config, NULL); 7543 7544 if (err) { 7545 dev_err(&pf->pdev->dev, 7546 "set phy config ret = %pe last_status = %s\n", 7547 ERR_PTR(err), libie_aq_str(pf->hw.aq.asq_last_status)); 7548 return err; 7549 } 7550 7551 /* Update the link info */ 7552 err = i40e_update_link_info(hw); 7553 if (err) { 7554 /* Wait a little bit (on 40G cards it sometimes takes a really 7555 * long time for link to come back from the atomic reset) 7556 * and try once more 7557 */ 7558 msleep(1000); 7559 i40e_update_link_info(hw); 7560 } 7561 7562 i40e_aq_set_link_restart_an(hw, is_up, NULL); 7563 7564 return 0; 7565 } 7566 7567 /** 7568 * i40e_up - Bring the connection back up after being down 7569 * @vsi: the VSI being configured 7570 **/ 7571 int i40e_up(struct i40e_vsi *vsi) 7572 { 7573 int err; 7574 7575 if (vsi->type == I40E_VSI_MAIN && 7576 (test_bit(I40E_FLAG_LINK_DOWN_ON_CLOSE_ENA, vsi->back->flags) || 7577 test_bit(I40E_FLAG_TOTAL_PORT_SHUTDOWN_ENA, vsi->back->flags))) 7578 i40e_force_link_state(vsi->back, true); 7579 7580 err = i40e_vsi_configure(vsi); 7581 if (!err) 7582 err = i40e_up_complete(vsi); 7583 7584 return err; 7585 } 7586 7587 /** 7588 * i40e_down - Shutdown the connection processing 7589 * @vsi: the VSI being stopped 7590 **/ 7591 void i40e_down(struct i40e_vsi *vsi) 7592 { 7593 int i; 7594 7595 /* It is assumed that the caller of this function 7596 * sets the vsi->state __I40E_VSI_DOWN bit. 7597 */ 7598 if (vsi->netdev) { 7599 netif_carrier_off(vsi->netdev); 7600 netif_tx_disable(vsi->netdev); 7601 } 7602 i40e_vsi_disable_irq(vsi); 7603 i40e_vsi_stop_rings(vsi); 7604 if (vsi->type == I40E_VSI_MAIN && 7605 (test_bit(I40E_FLAG_LINK_DOWN_ON_CLOSE_ENA, vsi->back->flags) || 7606 test_bit(I40E_FLAG_TOTAL_PORT_SHUTDOWN_ENA, vsi->back->flags))) 7607 i40e_force_link_state(vsi->back, false); 7608 i40e_napi_disable_all(vsi); 7609 7610 for (i = 0; i < vsi->num_queue_pairs; i++) { 7611 i40e_clean_tx_ring(vsi->tx_rings[i]); 7612 if (i40e_enabled_xdp_vsi(vsi)) { 7613 /* Make sure that in-progress ndo_xdp_xmit and 7614 * ndo_xsk_wakeup calls are completed. 7615 */ 7616 synchronize_rcu(); 7617 i40e_clean_tx_ring(vsi->xdp_rings[i]); 7618 } 7619 i40e_clean_rx_ring(vsi->rx_rings[i]); 7620 } 7621 7622 } 7623 7624 /** 7625 * i40e_validate_mqprio_qopt- validate queue mapping info 7626 * @vsi: the VSI being configured 7627 * @mqprio_qopt: queue parametrs 7628 **/ 7629 static int i40e_validate_mqprio_qopt(struct i40e_vsi *vsi, 7630 struct tc_mqprio_qopt_offload *mqprio_qopt) 7631 { 7632 u64 sum_max_rate = 0; 7633 u64 max_rate = 0; 7634 int i; 7635 7636 if (mqprio_qopt->qopt.offset[0] != 0 || 7637 mqprio_qopt->qopt.num_tc < 1 || 7638 mqprio_qopt->qopt.num_tc > I40E_MAX_TRAFFIC_CLASS) 7639 return -EINVAL; 7640 for (i = 0; ; i++) { 7641 if (!mqprio_qopt->qopt.count[i]) 7642 return -EINVAL; 7643 if (mqprio_qopt->min_rate[i]) { 7644 dev_err(&vsi->back->pdev->dev, 7645 "Invalid min tx rate (greater than 0) specified\n"); 7646 return -EINVAL; 7647 } 7648 max_rate = mqprio_qopt->max_rate[i]; 7649 do_div(max_rate, I40E_BW_MBPS_DIVISOR); 7650 sum_max_rate += max_rate; 7651 7652 if (i >= mqprio_qopt->qopt.num_tc - 1) 7653 break; 7654 if (mqprio_qopt->qopt.offset[i + 1] != 7655 (mqprio_qopt->qopt.offset[i] + mqprio_qopt->qopt.count[i])) 7656 return -EINVAL; 7657 } 7658 if (vsi->num_queue_pairs < 7659 (mqprio_qopt->qopt.offset[i] + mqprio_qopt->qopt.count[i])) { 7660 dev_err(&vsi->back->pdev->dev, 7661 "Failed to create traffic channel, insufficient number of queues.\n"); 7662 return -EINVAL; 7663 } 7664 if (sum_max_rate > i40e_get_link_speed(vsi)) { 7665 dev_err(&vsi->back->pdev->dev, 7666 "Invalid max tx rate specified\n"); 7667 return -EINVAL; 7668 } 7669 return 0; 7670 } 7671 7672 /** 7673 * i40e_vsi_set_default_tc_config - set default values for tc configuration 7674 * @vsi: the VSI being configured 7675 **/ 7676 static void i40e_vsi_set_default_tc_config(struct i40e_vsi *vsi) 7677 { 7678 u16 qcount; 7679 int i; 7680 7681 /* Only TC0 is enabled */ 7682 vsi->tc_config.numtc = 1; 7683 vsi->tc_config.enabled_tc = 1; 7684 qcount = min_t(int, vsi->alloc_queue_pairs, 7685 i40e_pf_get_max_q_per_tc(vsi->back)); 7686 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 7687 /* For the TC that is not enabled set the offset to default 7688 * queue and allocate one queue for the given TC. 7689 */ 7690 vsi->tc_config.tc_info[i].qoffset = 0; 7691 if (i == 0) 7692 vsi->tc_config.tc_info[i].qcount = qcount; 7693 else 7694 vsi->tc_config.tc_info[i].qcount = 1; 7695 vsi->tc_config.tc_info[i].netdev_tc = 0; 7696 } 7697 } 7698 7699 /** 7700 * i40e_del_macvlan_filter 7701 * @hw: pointer to the HW structure 7702 * @seid: seid of the channel VSI 7703 * @macaddr: the mac address to apply as a filter 7704 * @aq_err: store the admin Q error 7705 * 7706 * This function deletes a mac filter on the channel VSI which serves as the 7707 * macvlan. Returns 0 on success. 7708 **/ 7709 static int i40e_del_macvlan_filter(struct i40e_hw *hw, u16 seid, 7710 const u8 *macaddr, int *aq_err) 7711 { 7712 struct i40e_aqc_remove_macvlan_element_data element; 7713 int status; 7714 7715 memset(&element, 0, sizeof(element)); 7716 ether_addr_copy(element.mac_addr, macaddr); 7717 element.vlan_tag = 0; 7718 element.flags = I40E_AQC_MACVLAN_DEL_PERFECT_MATCH; 7719 status = i40e_aq_remove_macvlan(hw, seid, &element, 1, NULL); 7720 *aq_err = hw->aq.asq_last_status; 7721 7722 return status; 7723 } 7724 7725 /** 7726 * i40e_add_macvlan_filter 7727 * @hw: pointer to the HW structure 7728 * @seid: seid of the channel VSI 7729 * @macaddr: the mac address to apply as a filter 7730 * @aq_err: store the admin Q error 7731 * 7732 * This function adds a mac filter on the channel VSI which serves as the 7733 * macvlan. Returns 0 on success. 7734 **/ 7735 static int i40e_add_macvlan_filter(struct i40e_hw *hw, u16 seid, 7736 const u8 *macaddr, int *aq_err) 7737 { 7738 struct i40e_aqc_add_macvlan_element_data element; 7739 u16 cmd_flags = 0; 7740 int status; 7741 7742 ether_addr_copy(element.mac_addr, macaddr); 7743 element.vlan_tag = 0; 7744 element.queue_number = 0; 7745 element.match_method = I40E_AQC_MM_ERR_NO_RES; 7746 cmd_flags |= I40E_AQC_MACVLAN_ADD_PERFECT_MATCH; 7747 element.flags = cpu_to_le16(cmd_flags); 7748 status = i40e_aq_add_macvlan(hw, seid, &element, 1, NULL); 7749 *aq_err = hw->aq.asq_last_status; 7750 7751 return status; 7752 } 7753 7754 /** 7755 * i40e_reset_ch_rings - Reset the queue contexts in a channel 7756 * @vsi: the VSI we want to access 7757 * @ch: the channel we want to access 7758 */ 7759 static void i40e_reset_ch_rings(struct i40e_vsi *vsi, struct i40e_channel *ch) 7760 { 7761 struct i40e_ring *tx_ring, *rx_ring; 7762 u16 pf_q; 7763 int i; 7764 7765 for (i = 0; i < ch->num_queue_pairs; i++) { 7766 pf_q = ch->base_queue + i; 7767 tx_ring = vsi->tx_rings[pf_q]; 7768 tx_ring->ch = NULL; 7769 rx_ring = vsi->rx_rings[pf_q]; 7770 rx_ring->ch = NULL; 7771 } 7772 } 7773 7774 /** 7775 * i40e_free_macvlan_channels 7776 * @vsi: the VSI we want to access 7777 * 7778 * This function frees the Qs of the channel VSI from 7779 * the stack and also deletes the channel VSIs which 7780 * serve as macvlans. 7781 */ 7782 static void i40e_free_macvlan_channels(struct i40e_vsi *vsi) 7783 { 7784 struct i40e_channel *ch, *ch_tmp; 7785 int ret; 7786 7787 if (list_empty(&vsi->macvlan_list)) 7788 return; 7789 7790 list_for_each_entry_safe(ch, ch_tmp, &vsi->macvlan_list, list) { 7791 struct i40e_vsi *parent_vsi; 7792 7793 if (i40e_is_channel_macvlan(ch)) { 7794 i40e_reset_ch_rings(vsi, ch); 7795 clear_bit(ch->fwd->bit_no, vsi->fwd_bitmask); 7796 netdev_unbind_sb_channel(vsi->netdev, ch->fwd->netdev); 7797 netdev_set_sb_channel(ch->fwd->netdev, 0); 7798 kfree(ch->fwd); 7799 ch->fwd = NULL; 7800 } 7801 7802 list_del(&ch->list); 7803 parent_vsi = ch->parent_vsi; 7804 if (!parent_vsi || !ch->initialized) { 7805 kfree(ch); 7806 continue; 7807 } 7808 7809 /* remove the VSI */ 7810 ret = i40e_aq_delete_element(&vsi->back->hw, ch->seid, 7811 NULL); 7812 if (ret) 7813 dev_err(&vsi->back->pdev->dev, 7814 "unable to remove channel (%d) for parent VSI(%d)\n", 7815 ch->seid, parent_vsi->seid); 7816 kfree(ch); 7817 } 7818 vsi->macvlan_cnt = 0; 7819 } 7820 7821 /** 7822 * i40e_fwd_ring_up - bring the macvlan device up 7823 * @vsi: the VSI we want to access 7824 * @vdev: macvlan netdevice 7825 * @fwd: the private fwd structure 7826 */ 7827 static int i40e_fwd_ring_up(struct i40e_vsi *vsi, struct net_device *vdev, 7828 struct i40e_fwd_adapter *fwd) 7829 { 7830 struct i40e_channel *ch = NULL, *ch_tmp, *iter; 7831 int ret = 0, num_tc = 1, i, aq_err; 7832 struct i40e_pf *pf = vsi->back; 7833 struct i40e_hw *hw = &pf->hw; 7834 7835 /* Go through the list and find an available channel */ 7836 list_for_each_entry_safe(iter, ch_tmp, &vsi->macvlan_list, list) { 7837 if (!i40e_is_channel_macvlan(iter)) { 7838 iter->fwd = fwd; 7839 /* record configuration for macvlan interface in vdev */ 7840 for (i = 0; i < num_tc; i++) 7841 netdev_bind_sb_channel_queue(vsi->netdev, vdev, 7842 i, 7843 iter->num_queue_pairs, 7844 iter->base_queue); 7845 for (i = 0; i < iter->num_queue_pairs; i++) { 7846 struct i40e_ring *tx_ring, *rx_ring; 7847 u16 pf_q; 7848 7849 pf_q = iter->base_queue + i; 7850 7851 /* Get to TX ring ptr */ 7852 tx_ring = vsi->tx_rings[pf_q]; 7853 tx_ring->ch = iter; 7854 7855 /* Get the RX ring ptr */ 7856 rx_ring = vsi->rx_rings[pf_q]; 7857 rx_ring->ch = iter; 7858 } 7859 ch = iter; 7860 break; 7861 } 7862 } 7863 7864 if (!ch) 7865 return -EINVAL; 7866 7867 /* Guarantee all rings are updated before we update the 7868 * MAC address filter. 7869 */ 7870 wmb(); 7871 7872 /* Add a mac filter */ 7873 ret = i40e_add_macvlan_filter(hw, ch->seid, vdev->dev_addr, &aq_err); 7874 if (ret) { 7875 /* if we cannot add the MAC rule then disable the offload */ 7876 macvlan_release_l2fw_offload(vdev); 7877 for (i = 0; i < ch->num_queue_pairs; i++) { 7878 struct i40e_ring *rx_ring; 7879 u16 pf_q; 7880 7881 pf_q = ch->base_queue + i; 7882 rx_ring = vsi->rx_rings[pf_q]; 7883 rx_ring->netdev = NULL; 7884 } 7885 dev_info(&pf->pdev->dev, 7886 "Error adding mac filter on macvlan err %pe, aq_err %s\n", 7887 ERR_PTR(ret), libie_aq_str(aq_err)); 7888 netdev_err(vdev, "L2fwd offload disabled to L2 filter error\n"); 7889 } 7890 7891 return ret; 7892 } 7893 7894 /** 7895 * i40e_setup_macvlans - create the channels which will be macvlans 7896 * @vsi: the VSI we want to access 7897 * @macvlan_cnt: no. of macvlans to be setup 7898 * @qcnt: no. of Qs per macvlan 7899 * @vdev: macvlan netdevice 7900 */ 7901 static int i40e_setup_macvlans(struct i40e_vsi *vsi, u16 macvlan_cnt, u16 qcnt, 7902 struct net_device *vdev) 7903 { 7904 struct i40e_pf *pf = vsi->back; 7905 struct i40e_hw *hw = &pf->hw; 7906 struct i40e_vsi_context ctxt; 7907 u16 sections, qmap, num_qps; 7908 struct i40e_channel *ch; 7909 int i, pow, ret = 0; 7910 u8 offset = 0; 7911 7912 if (vsi->type != I40E_VSI_MAIN || !macvlan_cnt) 7913 return -EINVAL; 7914 7915 num_qps = vsi->num_queue_pairs - (macvlan_cnt * qcnt); 7916 7917 /* find the next higher power-of-2 of num queue pairs */ 7918 pow = fls(roundup_pow_of_two(num_qps) - 1); 7919 7920 qmap = (offset << I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT) | 7921 (pow << I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT); 7922 7923 /* Setup context bits for the main VSI */ 7924 sections = I40E_AQ_VSI_PROP_QUEUE_MAP_VALID; 7925 sections |= I40E_AQ_VSI_PROP_SCHED_VALID; 7926 memset(&ctxt, 0, sizeof(ctxt)); 7927 ctxt.seid = vsi->seid; 7928 ctxt.pf_num = vsi->back->hw.pf_id; 7929 ctxt.vf_num = 0; 7930 ctxt.uplink_seid = vsi->uplink_seid; 7931 ctxt.info = vsi->info; 7932 ctxt.info.tc_mapping[0] = cpu_to_le16(qmap); 7933 ctxt.info.mapping_flags |= cpu_to_le16(I40E_AQ_VSI_QUE_MAP_CONTIG); 7934 ctxt.info.queue_mapping[0] = cpu_to_le16(vsi->base_queue); 7935 ctxt.info.valid_sections |= cpu_to_le16(sections); 7936 7937 /* Reconfigure RSS for main VSI with new max queue count */ 7938 vsi->rss_size = max_t(u16, num_qps, qcnt); 7939 ret = i40e_vsi_config_rss(vsi); 7940 if (ret) { 7941 dev_info(&pf->pdev->dev, 7942 "Failed to reconfig RSS for num_queues (%u)\n", 7943 vsi->rss_size); 7944 return ret; 7945 } 7946 vsi->reconfig_rss = true; 7947 dev_dbg(&vsi->back->pdev->dev, 7948 "Reconfigured RSS with num_queues (%u)\n", vsi->rss_size); 7949 vsi->next_base_queue = num_qps; 7950 vsi->cnt_q_avail = vsi->num_queue_pairs - num_qps; 7951 7952 /* Update the VSI after updating the VSI queue-mapping 7953 * information 7954 */ 7955 ret = i40e_aq_update_vsi_params(hw, &ctxt, NULL); 7956 if (ret) { 7957 dev_info(&pf->pdev->dev, 7958 "Update vsi tc config failed, err %pe aq_err %s\n", 7959 ERR_PTR(ret), libie_aq_str(hw->aq.asq_last_status)); 7960 return ret; 7961 } 7962 /* update the local VSI info with updated queue map */ 7963 i40e_vsi_update_queue_map(vsi, &ctxt); 7964 vsi->info.valid_sections = 0; 7965 7966 /* Create channels for macvlans */ 7967 INIT_LIST_HEAD(&vsi->macvlan_list); 7968 for (i = 0; i < macvlan_cnt; i++) { 7969 ch = kzalloc_obj(*ch); 7970 if (!ch) { 7971 ret = -ENOMEM; 7972 goto err_free; 7973 } 7974 INIT_LIST_HEAD(&ch->list); 7975 ch->num_queue_pairs = qcnt; 7976 if (!i40e_setup_channel(pf, vsi, ch)) { 7977 ret = -EINVAL; 7978 kfree(ch); 7979 goto err_free; 7980 } 7981 ch->parent_vsi = vsi; 7982 vsi->cnt_q_avail -= ch->num_queue_pairs; 7983 vsi->macvlan_cnt++; 7984 list_add_tail(&ch->list, &vsi->macvlan_list); 7985 } 7986 7987 return ret; 7988 7989 err_free: 7990 dev_info(&pf->pdev->dev, "Failed to setup macvlans\n"); 7991 i40e_free_macvlan_channels(vsi); 7992 7993 return ret; 7994 } 7995 7996 /** 7997 * i40e_fwd_add - configure macvlans 7998 * @netdev: net device to configure 7999 * @vdev: macvlan netdevice 8000 **/ 8001 static void *i40e_fwd_add(struct net_device *netdev, struct net_device *vdev) 8002 { 8003 struct i40e_netdev_priv *np = netdev_priv(netdev); 8004 u16 q_per_macvlan = 0, macvlan_cnt = 0, vectors; 8005 struct i40e_vsi *vsi = np->vsi; 8006 struct i40e_pf *pf = vsi->back; 8007 struct i40e_fwd_adapter *fwd; 8008 int avail_macvlan, ret; 8009 8010 if (test_bit(I40E_FLAG_DCB_ENA, pf->flags)) { 8011 netdev_info(netdev, "Macvlans are not supported when DCB is enabled\n"); 8012 return ERR_PTR(-EINVAL); 8013 } 8014 if (i40e_is_tc_mqprio_enabled(pf)) { 8015 netdev_info(netdev, "Macvlans are not supported when HW TC offload is on\n"); 8016 return ERR_PTR(-EINVAL); 8017 } 8018 if (pf->num_lan_msix < I40E_MIN_MACVLAN_VECTORS) { 8019 netdev_info(netdev, "Not enough vectors available to support macvlans\n"); 8020 return ERR_PTR(-EINVAL); 8021 } 8022 8023 /* The macvlan device has to be a single Q device so that the 8024 * tc_to_txq field can be reused to pick the tx queue. 8025 */ 8026 if (netif_is_multiqueue(vdev)) 8027 return ERR_PTR(-ERANGE); 8028 8029 if (!vsi->macvlan_cnt) { 8030 /* reserve bit 0 for the pf device */ 8031 set_bit(0, vsi->fwd_bitmask); 8032 8033 /* Try to reserve as many queues as possible for macvlans. First 8034 * reserve 3/4th of max vectors, then half, then quarter and 8035 * calculate Qs per macvlan as you go 8036 */ 8037 vectors = pf->num_lan_msix; 8038 if (vectors <= I40E_MAX_MACVLANS && vectors > 64) { 8039 /* allocate 4 Qs per macvlan and 32 Qs to the PF*/ 8040 q_per_macvlan = 4; 8041 macvlan_cnt = (vectors - 32) / 4; 8042 } else if (vectors <= 64 && vectors > 32) { 8043 /* allocate 2 Qs per macvlan and 16 Qs to the PF*/ 8044 q_per_macvlan = 2; 8045 macvlan_cnt = (vectors - 16) / 2; 8046 } else if (vectors <= 32 && vectors > 16) { 8047 /* allocate 1 Q per macvlan and 16 Qs to the PF*/ 8048 q_per_macvlan = 1; 8049 macvlan_cnt = vectors - 16; 8050 } else if (vectors <= 16 && vectors > 8) { 8051 /* allocate 1 Q per macvlan and 8 Qs to the PF */ 8052 q_per_macvlan = 1; 8053 macvlan_cnt = vectors - 8; 8054 } else { 8055 /* allocate 1 Q per macvlan and 1 Q to the PF */ 8056 q_per_macvlan = 1; 8057 macvlan_cnt = vectors - 1; 8058 } 8059 8060 if (macvlan_cnt == 0) 8061 return ERR_PTR(-EBUSY); 8062 8063 /* Quiesce VSI queues */ 8064 i40e_quiesce_vsi(vsi); 8065 8066 /* sets up the macvlans but does not "enable" them */ 8067 ret = i40e_setup_macvlans(vsi, macvlan_cnt, q_per_macvlan, 8068 vdev); 8069 if (ret) 8070 return ERR_PTR(ret); 8071 8072 /* Unquiesce VSI */ 8073 i40e_unquiesce_vsi(vsi); 8074 } 8075 avail_macvlan = find_first_zero_bit(vsi->fwd_bitmask, 8076 vsi->macvlan_cnt); 8077 if (avail_macvlan >= I40E_MAX_MACVLANS) 8078 return ERR_PTR(-EBUSY); 8079 8080 /* create the fwd struct */ 8081 fwd = kzalloc_obj(*fwd); 8082 if (!fwd) 8083 return ERR_PTR(-ENOMEM); 8084 8085 set_bit(avail_macvlan, vsi->fwd_bitmask); 8086 fwd->bit_no = avail_macvlan; 8087 netdev_set_sb_channel(vdev, avail_macvlan); 8088 fwd->netdev = vdev; 8089 8090 if (!netif_running(netdev)) 8091 return fwd; 8092 8093 /* Set fwd ring up */ 8094 ret = i40e_fwd_ring_up(vsi, vdev, fwd); 8095 if (ret) { 8096 /* unbind the queues and drop the subordinate channel config */ 8097 netdev_unbind_sb_channel(netdev, vdev); 8098 netdev_set_sb_channel(vdev, 0); 8099 8100 kfree(fwd); 8101 return ERR_PTR(-EINVAL); 8102 } 8103 8104 return fwd; 8105 } 8106 8107 /** 8108 * i40e_del_all_macvlans - Delete all the mac filters on the channels 8109 * @vsi: the VSI we want to access 8110 */ 8111 static void i40e_del_all_macvlans(struct i40e_vsi *vsi) 8112 { 8113 struct i40e_channel *ch, *ch_tmp; 8114 struct i40e_pf *pf = vsi->back; 8115 struct i40e_hw *hw = &pf->hw; 8116 int aq_err, ret = 0; 8117 8118 if (list_empty(&vsi->macvlan_list)) 8119 return; 8120 8121 list_for_each_entry_safe(ch, ch_tmp, &vsi->macvlan_list, list) { 8122 if (i40e_is_channel_macvlan(ch)) { 8123 ret = i40e_del_macvlan_filter(hw, ch->seid, 8124 i40e_channel_mac(ch), 8125 &aq_err); 8126 if (!ret) { 8127 /* Reset queue contexts */ 8128 i40e_reset_ch_rings(vsi, ch); 8129 clear_bit(ch->fwd->bit_no, vsi->fwd_bitmask); 8130 netdev_unbind_sb_channel(vsi->netdev, 8131 ch->fwd->netdev); 8132 netdev_set_sb_channel(ch->fwd->netdev, 0); 8133 kfree(ch->fwd); 8134 ch->fwd = NULL; 8135 } 8136 } 8137 } 8138 } 8139 8140 /** 8141 * i40e_fwd_del - delete macvlan interfaces 8142 * @netdev: net device to configure 8143 * @vdev: macvlan netdevice 8144 */ 8145 static void i40e_fwd_del(struct net_device *netdev, void *vdev) 8146 { 8147 struct i40e_netdev_priv *np = netdev_priv(netdev); 8148 struct i40e_fwd_adapter *fwd = vdev; 8149 struct i40e_channel *ch, *ch_tmp; 8150 struct i40e_vsi *vsi = np->vsi; 8151 struct i40e_pf *pf = vsi->back; 8152 struct i40e_hw *hw = &pf->hw; 8153 int aq_err, ret = 0; 8154 8155 /* Find the channel associated with the macvlan and del mac filter */ 8156 list_for_each_entry_safe(ch, ch_tmp, &vsi->macvlan_list, list) { 8157 if (i40e_is_channel_macvlan(ch) && 8158 ether_addr_equal(i40e_channel_mac(ch), 8159 fwd->netdev->dev_addr)) { 8160 ret = i40e_del_macvlan_filter(hw, ch->seid, 8161 i40e_channel_mac(ch), 8162 &aq_err); 8163 if (!ret) { 8164 /* Reset queue contexts */ 8165 i40e_reset_ch_rings(vsi, ch); 8166 clear_bit(ch->fwd->bit_no, vsi->fwd_bitmask); 8167 netdev_unbind_sb_channel(netdev, fwd->netdev); 8168 netdev_set_sb_channel(fwd->netdev, 0); 8169 kfree(ch->fwd); 8170 ch->fwd = NULL; 8171 } else { 8172 dev_info(&pf->pdev->dev, 8173 "Error deleting mac filter on macvlan err %pe, aq_err %s\n", 8174 ERR_PTR(ret), libie_aq_str(aq_err)); 8175 } 8176 break; 8177 } 8178 } 8179 } 8180 8181 /** 8182 * i40e_setup_tc - configure multiple traffic classes 8183 * @netdev: net device to configure 8184 * @type_data: tc offload data 8185 **/ 8186 static int i40e_setup_tc(struct net_device *netdev, void *type_data) 8187 { 8188 struct tc_mqprio_qopt_offload *mqprio_qopt = type_data; 8189 struct i40e_netdev_priv *np = netdev_priv(netdev); 8190 struct i40e_vsi *vsi = np->vsi; 8191 struct i40e_pf *pf = vsi->back; 8192 u8 enabled_tc = 0, num_tc, hw; 8193 bool need_reset = false; 8194 int old_queue_pairs; 8195 int ret = -EINVAL; 8196 u16 mode; 8197 int i; 8198 8199 old_queue_pairs = vsi->num_queue_pairs; 8200 num_tc = mqprio_qopt->qopt.num_tc; 8201 hw = mqprio_qopt->qopt.hw; 8202 mode = mqprio_qopt->mode; 8203 if (!hw) { 8204 clear_bit(I40E_FLAG_TC_MQPRIO_ENA, pf->flags); 8205 memcpy(&vsi->mqprio_qopt, mqprio_qopt, sizeof(*mqprio_qopt)); 8206 goto config_tc; 8207 } 8208 8209 /* Check if MFP enabled */ 8210 if (test_bit(I40E_FLAG_MFP_ENA, pf->flags)) { 8211 netdev_info(netdev, 8212 "Configuring TC not supported in MFP mode\n"); 8213 return ret; 8214 } 8215 switch (mode) { 8216 case TC_MQPRIO_MODE_DCB: 8217 clear_bit(I40E_FLAG_TC_MQPRIO_ENA, pf->flags); 8218 8219 /* Check if DCB enabled to continue */ 8220 if (!test_bit(I40E_FLAG_DCB_ENA, pf->flags)) { 8221 netdev_info(netdev, 8222 "DCB is not enabled for adapter\n"); 8223 return ret; 8224 } 8225 8226 /* Check whether tc count is within enabled limit */ 8227 if (num_tc > i40e_pf_get_num_tc(pf)) { 8228 netdev_info(netdev, 8229 "TC count greater than enabled on link for adapter\n"); 8230 return ret; 8231 } 8232 break; 8233 case TC_MQPRIO_MODE_CHANNEL: 8234 if (test_bit(I40E_FLAG_DCB_ENA, pf->flags)) { 8235 netdev_info(netdev, 8236 "Full offload of TC Mqprio options is not supported when DCB is enabled\n"); 8237 return ret; 8238 } 8239 if (!test_bit(I40E_FLAG_MSIX_ENA, pf->flags)) 8240 return ret; 8241 ret = i40e_validate_mqprio_qopt(vsi, mqprio_qopt); 8242 if (ret) 8243 return ret; 8244 memcpy(&vsi->mqprio_qopt, mqprio_qopt, 8245 sizeof(*mqprio_qopt)); 8246 set_bit(I40E_FLAG_TC_MQPRIO_ENA, pf->flags); 8247 clear_bit(I40E_FLAG_DCB_ENA, pf->flags); 8248 break; 8249 default: 8250 return -EINVAL; 8251 } 8252 8253 config_tc: 8254 /* Generate TC map for number of tc requested */ 8255 for (i = 0; i < num_tc; i++) 8256 enabled_tc |= BIT(i); 8257 8258 /* Requesting same TC configuration as already enabled */ 8259 if (enabled_tc == vsi->tc_config.enabled_tc && 8260 mode != TC_MQPRIO_MODE_CHANNEL) 8261 return 0; 8262 8263 /* Quiesce VSI queues */ 8264 i40e_quiesce_vsi(vsi); 8265 8266 if (!hw && !i40e_is_tc_mqprio_enabled(pf)) 8267 i40e_remove_queue_channels(vsi); 8268 8269 /* Configure VSI for enabled TCs */ 8270 ret = i40e_vsi_config_tc(vsi, enabled_tc); 8271 if (ret) { 8272 netdev_info(netdev, "Failed configuring TC for VSI seid=%d\n", 8273 vsi->seid); 8274 need_reset = true; 8275 goto exit; 8276 } else if (enabled_tc && 8277 (!is_power_of_2(vsi->tc_config.tc_info[0].qcount))) { 8278 netdev_info(netdev, 8279 "Failed to create channel. Override queues (%u) not power of 2\n", 8280 vsi->tc_config.tc_info[0].qcount); 8281 ret = -EINVAL; 8282 need_reset = true; 8283 goto exit; 8284 } 8285 8286 dev_info(&vsi->back->pdev->dev, 8287 "Setup channel (id:%u) utilizing num_queues %d\n", 8288 vsi->seid, vsi->tc_config.tc_info[0].qcount); 8289 8290 if (i40e_is_tc_mqprio_enabled(pf)) { 8291 if (vsi->mqprio_qopt.max_rate[0]) { 8292 u64 max_tx_rate = i40e_bw_bytes_to_mbits(vsi, 8293 vsi->mqprio_qopt.max_rate[0]); 8294 8295 ret = i40e_set_bw_limit(vsi, vsi->seid, max_tx_rate); 8296 if (!ret) { 8297 u64 credits = max_tx_rate; 8298 8299 do_div(credits, I40E_BW_CREDIT_DIVISOR); 8300 dev_dbg(&vsi->back->pdev->dev, 8301 "Set tx rate of %llu Mbps (count of 50Mbps %llu) for vsi->seid %u\n", 8302 max_tx_rate, 8303 credits, 8304 vsi->seid); 8305 } else { 8306 need_reset = true; 8307 goto exit; 8308 } 8309 } 8310 ret = i40e_configure_queue_channels(vsi); 8311 if (ret) { 8312 vsi->num_queue_pairs = old_queue_pairs; 8313 netdev_info(netdev, 8314 "Failed configuring queue channels\n"); 8315 need_reset = true; 8316 goto exit; 8317 } 8318 } 8319 8320 exit: 8321 /* Reset the configuration data to defaults, only TC0 is enabled */ 8322 if (need_reset) { 8323 i40e_vsi_set_default_tc_config(vsi); 8324 need_reset = false; 8325 } 8326 8327 /* Unquiesce VSI */ 8328 i40e_unquiesce_vsi(vsi); 8329 return ret; 8330 } 8331 8332 /** 8333 * i40e_set_cld_element - sets cloud filter element data 8334 * @filter: cloud filter rule 8335 * @cld: ptr to cloud filter element data 8336 * 8337 * This is helper function to copy data into cloud filter element 8338 **/ 8339 static inline void 8340 i40e_set_cld_element(struct i40e_cloud_filter *filter, 8341 struct i40e_aqc_cloud_filters_element_data *cld) 8342 { 8343 u32 ipa; 8344 int i; 8345 8346 memset(cld, 0, sizeof(*cld)); 8347 ether_addr_copy(cld->outer_mac, filter->dst_mac); 8348 ether_addr_copy(cld->inner_mac, filter->src_mac); 8349 8350 if (filter->n_proto != ETH_P_IP && filter->n_proto != ETH_P_IPV6) 8351 return; 8352 8353 if (filter->n_proto == ETH_P_IPV6) { 8354 #define IPV6_MAX_INDEX (ARRAY_SIZE(filter->dst_ipv6) - 1) 8355 for (i = 0; i < ARRAY_SIZE(filter->dst_ipv6); i++) { 8356 ipa = be32_to_cpu(filter->dst_ipv6[IPV6_MAX_INDEX - i]); 8357 8358 *(__le32 *)&cld->ipaddr.raw_v6.data[i * 2] = cpu_to_le32(ipa); 8359 } 8360 } else { 8361 ipa = be32_to_cpu(filter->dst_ipv4); 8362 8363 memcpy(&cld->ipaddr.v4.data, &ipa, sizeof(ipa)); 8364 } 8365 8366 cld->inner_vlan = cpu_to_le16(ntohs(filter->vlan_id)); 8367 8368 /* tenant_id is not supported by FW now, once the support is enabled 8369 * fill the cld->tenant_id with cpu_to_le32(filter->tenant_id) 8370 */ 8371 if (filter->tenant_id) 8372 return; 8373 } 8374 8375 /** 8376 * i40e_add_del_cloud_filter - Add/del cloud filter 8377 * @vsi: pointer to VSI 8378 * @filter: cloud filter rule 8379 * @add: if true, add, if false, delete 8380 * 8381 * Add or delete a cloud filter for a specific flow spec. 8382 * Returns 0 if the filter were successfully added. 8383 **/ 8384 int i40e_add_del_cloud_filter(struct i40e_vsi *vsi, 8385 struct i40e_cloud_filter *filter, bool add) 8386 { 8387 struct i40e_aqc_cloud_filters_element_data cld_filter; 8388 struct i40e_pf *pf = vsi->back; 8389 int ret; 8390 static const u16 flag_table[128] = { 8391 [I40E_CLOUD_FILTER_FLAGS_OMAC] = 8392 I40E_AQC_ADD_CLOUD_FILTER_OMAC, 8393 [I40E_CLOUD_FILTER_FLAGS_IMAC] = 8394 I40E_AQC_ADD_CLOUD_FILTER_IMAC, 8395 [I40E_CLOUD_FILTER_FLAGS_IMAC_IVLAN] = 8396 I40E_AQC_ADD_CLOUD_FILTER_IMAC_IVLAN, 8397 [I40E_CLOUD_FILTER_FLAGS_IMAC_TEN_ID] = 8398 I40E_AQC_ADD_CLOUD_FILTER_IMAC_TEN_ID, 8399 [I40E_CLOUD_FILTER_FLAGS_OMAC_TEN_ID_IMAC] = 8400 I40E_AQC_ADD_CLOUD_FILTER_OMAC_TEN_ID_IMAC, 8401 [I40E_CLOUD_FILTER_FLAGS_IMAC_IVLAN_TEN_ID] = 8402 I40E_AQC_ADD_CLOUD_FILTER_IMAC_IVLAN_TEN_ID, 8403 [I40E_CLOUD_FILTER_FLAGS_IIP] = 8404 I40E_AQC_ADD_CLOUD_FILTER_IIP, 8405 }; 8406 8407 if (filter->flags >= ARRAY_SIZE(flag_table)) 8408 return -EIO; 8409 8410 memset(&cld_filter, 0, sizeof(cld_filter)); 8411 8412 /* copy element needed to add cloud filter from filter */ 8413 i40e_set_cld_element(filter, &cld_filter); 8414 8415 if (filter->tunnel_type != I40E_CLOUD_TNL_TYPE_NONE) 8416 cld_filter.flags = cpu_to_le16(filter->tunnel_type << 8417 I40E_AQC_ADD_CLOUD_TNL_TYPE_SHIFT); 8418 8419 if (filter->n_proto == ETH_P_IPV6) 8420 cld_filter.flags |= cpu_to_le16(flag_table[filter->flags] | 8421 I40E_AQC_ADD_CLOUD_FLAGS_IPV6); 8422 else 8423 cld_filter.flags |= cpu_to_le16(flag_table[filter->flags] | 8424 I40E_AQC_ADD_CLOUD_FLAGS_IPV4); 8425 8426 if (add) 8427 ret = i40e_aq_add_cloud_filters(&pf->hw, filter->seid, 8428 &cld_filter, 1); 8429 else 8430 ret = i40e_aq_rem_cloud_filters(&pf->hw, filter->seid, 8431 &cld_filter, 1); 8432 if (ret) 8433 dev_dbg(&pf->pdev->dev, 8434 "Failed to %s cloud filter using l4 port %u, err %d aq_err %d\n", 8435 add ? "add" : "delete", filter->dst_port, ret, 8436 pf->hw.aq.asq_last_status); 8437 else 8438 dev_info(&pf->pdev->dev, 8439 "%s cloud filter for VSI: %d\n", 8440 add ? "Added" : "Deleted", filter->seid); 8441 return ret; 8442 } 8443 8444 /** 8445 * i40e_add_del_cloud_filter_big_buf - Add/del cloud filter using big_buf 8446 * @vsi: pointer to VSI 8447 * @filter: cloud filter rule 8448 * @add: if true, add, if false, delete 8449 * 8450 * Add or delete a cloud filter for a specific flow spec using big buffer. 8451 * Returns 0 if the filter were successfully added. 8452 **/ 8453 int i40e_add_del_cloud_filter_big_buf(struct i40e_vsi *vsi, 8454 struct i40e_cloud_filter *filter, 8455 bool add) 8456 { 8457 struct i40e_aqc_cloud_filters_element_bb cld_filter; 8458 struct i40e_pf *pf = vsi->back; 8459 int ret; 8460 8461 /* Both (src/dst) valid mac_addr are not supported */ 8462 if ((is_valid_ether_addr(filter->dst_mac) && 8463 is_valid_ether_addr(filter->src_mac)) || 8464 (is_multicast_ether_addr(filter->dst_mac) && 8465 is_multicast_ether_addr(filter->src_mac))) 8466 return -EOPNOTSUPP; 8467 8468 /* Big buffer cloud filter needs 'L4 port' to be non-zero. Also, UDP 8469 * ports are not supported via big buffer now. 8470 */ 8471 if (!filter->dst_port || filter->ip_proto == IPPROTO_UDP) 8472 return -EOPNOTSUPP; 8473 8474 /* adding filter using src_port/src_ip is not supported at this stage */ 8475 if (filter->src_port || 8476 (filter->src_ipv4 && filter->n_proto != ETH_P_IPV6) || 8477 !ipv6_addr_any(&filter->ip.v6.src_ip6)) 8478 return -EOPNOTSUPP; 8479 8480 memset(&cld_filter, 0, sizeof(cld_filter)); 8481 8482 /* copy element needed to add cloud filter from filter */ 8483 i40e_set_cld_element(filter, &cld_filter.element); 8484 8485 if (is_valid_ether_addr(filter->dst_mac) || 8486 is_valid_ether_addr(filter->src_mac) || 8487 is_multicast_ether_addr(filter->dst_mac) || 8488 is_multicast_ether_addr(filter->src_mac)) { 8489 /* MAC + IP : unsupported mode */ 8490 if (filter->dst_ipv4) 8491 return -EOPNOTSUPP; 8492 8493 /* since we validated that L4 port must be valid before 8494 * we get here, start with respective "flags" value 8495 * and update if vlan is present or not 8496 */ 8497 cld_filter.element.flags = 8498 cpu_to_le16(I40E_AQC_ADD_CLOUD_FILTER_MAC_PORT); 8499 8500 if (filter->vlan_id) { 8501 cld_filter.element.flags = 8502 cpu_to_le16(I40E_AQC_ADD_CLOUD_FILTER_MAC_VLAN_PORT); 8503 } 8504 8505 } else if ((filter->dst_ipv4 && filter->n_proto != ETH_P_IPV6) || 8506 !ipv6_addr_any(&filter->ip.v6.dst_ip6)) { 8507 cld_filter.element.flags = 8508 cpu_to_le16(I40E_AQC_ADD_CLOUD_FILTER_IP_PORT); 8509 if (filter->n_proto == ETH_P_IPV6) 8510 cld_filter.element.flags |= 8511 cpu_to_le16(I40E_AQC_ADD_CLOUD_FLAGS_IPV6); 8512 else 8513 cld_filter.element.flags |= 8514 cpu_to_le16(I40E_AQC_ADD_CLOUD_FLAGS_IPV4); 8515 } else { 8516 dev_err(&pf->pdev->dev, 8517 "either mac or ip has to be valid for cloud filter\n"); 8518 return -EINVAL; 8519 } 8520 8521 /* Now copy L4 port in Byte 6..7 in general fields */ 8522 cld_filter.general_fields[I40E_AQC_ADD_CLOUD_FV_FLU_0X16_WORD0] = 8523 be16_to_cpu(filter->dst_port); 8524 8525 if (add) { 8526 /* Validate current device switch mode, change if necessary */ 8527 ret = i40e_validate_and_set_switch_mode(vsi); 8528 if (ret) { 8529 dev_err(&pf->pdev->dev, 8530 "failed to set switch mode, ret %d\n", 8531 ret); 8532 return ret; 8533 } 8534 8535 ret = i40e_aq_add_cloud_filters_bb(&pf->hw, filter->seid, 8536 &cld_filter, 1); 8537 } else { 8538 ret = i40e_aq_rem_cloud_filters_bb(&pf->hw, filter->seid, 8539 &cld_filter, 1); 8540 } 8541 8542 if (ret) 8543 dev_dbg(&pf->pdev->dev, 8544 "Failed to %s cloud filter(big buffer) err %d aq_err %d\n", 8545 add ? "add" : "delete", ret, pf->hw.aq.asq_last_status); 8546 else 8547 dev_info(&pf->pdev->dev, 8548 "%s cloud filter for VSI: %d, L4 port: %d\n", 8549 add ? "add" : "delete", filter->seid, 8550 ntohs(filter->dst_port)); 8551 return ret; 8552 } 8553 8554 /** 8555 * i40e_parse_cls_flower - Parse tc flower filters provided by kernel 8556 * @vsi: Pointer to VSI 8557 * @f: Pointer to struct flow_cls_offload 8558 * @filter: Pointer to cloud filter structure 8559 * 8560 **/ 8561 static int i40e_parse_cls_flower(struct i40e_vsi *vsi, 8562 struct flow_cls_offload *f, 8563 struct i40e_cloud_filter *filter) 8564 { 8565 struct flow_rule *rule = flow_cls_offload_flow_rule(f); 8566 struct flow_dissector *dissector = rule->match.dissector; 8567 u16 n_proto_mask = 0, n_proto_key = 0, addr_type = 0; 8568 struct i40e_pf *pf = vsi->back; 8569 u8 field_flags = 0; 8570 8571 if (dissector->used_keys & 8572 ~(BIT_ULL(FLOW_DISSECTOR_KEY_CONTROL) | 8573 BIT_ULL(FLOW_DISSECTOR_KEY_BASIC) | 8574 BIT_ULL(FLOW_DISSECTOR_KEY_ETH_ADDRS) | 8575 BIT_ULL(FLOW_DISSECTOR_KEY_VLAN) | 8576 BIT_ULL(FLOW_DISSECTOR_KEY_IPV4_ADDRS) | 8577 BIT_ULL(FLOW_DISSECTOR_KEY_IPV6_ADDRS) | 8578 BIT_ULL(FLOW_DISSECTOR_KEY_PORTS) | 8579 BIT_ULL(FLOW_DISSECTOR_KEY_ENC_KEYID))) { 8580 dev_err(&pf->pdev->dev, "Unsupported key used: 0x%llx\n", 8581 dissector->used_keys); 8582 return -EOPNOTSUPP; 8583 } 8584 8585 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_ENC_KEYID)) { 8586 struct flow_match_enc_keyid match; 8587 8588 flow_rule_match_enc_keyid(rule, &match); 8589 if (match.mask->keyid != 0) 8590 field_flags |= I40E_CLOUD_FIELD_TEN_ID; 8591 8592 filter->tenant_id = be32_to_cpu(match.key->keyid); 8593 } 8594 8595 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_BASIC)) { 8596 struct flow_match_basic match; 8597 8598 flow_rule_match_basic(rule, &match); 8599 n_proto_key = ntohs(match.key->n_proto); 8600 n_proto_mask = ntohs(match.mask->n_proto); 8601 8602 if (n_proto_key == ETH_P_ALL) { 8603 n_proto_key = 0; 8604 n_proto_mask = 0; 8605 } 8606 filter->n_proto = n_proto_key & n_proto_mask; 8607 filter->ip_proto = match.key->ip_proto; 8608 } 8609 8610 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_ETH_ADDRS)) { 8611 struct flow_match_eth_addrs match; 8612 8613 flow_rule_match_eth_addrs(rule, &match); 8614 8615 /* use is_broadcast and is_zero to check for all 0xf or 0 */ 8616 if (!is_zero_ether_addr(match.mask->dst)) { 8617 if (is_broadcast_ether_addr(match.mask->dst)) { 8618 field_flags |= I40E_CLOUD_FIELD_OMAC; 8619 } else { 8620 dev_err(&pf->pdev->dev, "Bad ether dest mask %pM\n", 8621 match.mask->dst); 8622 return -EIO; 8623 } 8624 } 8625 8626 if (!is_zero_ether_addr(match.mask->src)) { 8627 if (is_broadcast_ether_addr(match.mask->src)) { 8628 field_flags |= I40E_CLOUD_FIELD_IMAC; 8629 } else { 8630 dev_err(&pf->pdev->dev, "Bad ether src mask %pM\n", 8631 match.mask->src); 8632 return -EIO; 8633 } 8634 } 8635 ether_addr_copy(filter->dst_mac, match.key->dst); 8636 ether_addr_copy(filter->src_mac, match.key->src); 8637 } 8638 8639 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_VLAN)) { 8640 struct flow_match_vlan match; 8641 8642 flow_rule_match_vlan(rule, &match); 8643 if (match.mask->vlan_id) { 8644 if (match.mask->vlan_id == VLAN_VID_MASK) { 8645 field_flags |= I40E_CLOUD_FIELD_IVLAN; 8646 8647 } else { 8648 dev_err(&pf->pdev->dev, "Bad vlan mask 0x%04x\n", 8649 match.mask->vlan_id); 8650 return -EIO; 8651 } 8652 } 8653 8654 filter->vlan_id = cpu_to_be16(match.key->vlan_id); 8655 } 8656 8657 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_CONTROL)) { 8658 struct flow_match_control match; 8659 8660 flow_rule_match_control(rule, &match); 8661 addr_type = match.key->addr_type; 8662 8663 if (flow_rule_has_control_flags(match.mask->flags, 8664 f->common.extack)) 8665 return -EOPNOTSUPP; 8666 } 8667 8668 if (addr_type == FLOW_DISSECTOR_KEY_IPV4_ADDRS) { 8669 struct flow_match_ipv4_addrs match; 8670 8671 flow_rule_match_ipv4_addrs(rule, &match); 8672 if (match.mask->dst) { 8673 if (match.mask->dst == cpu_to_be32(0xffffffff)) { 8674 field_flags |= I40E_CLOUD_FIELD_IIP; 8675 } else { 8676 dev_err(&pf->pdev->dev, "Bad ip dst mask %pI4b\n", 8677 &match.mask->dst); 8678 return -EIO; 8679 } 8680 } 8681 8682 if (match.mask->src) { 8683 if (match.mask->src == cpu_to_be32(0xffffffff)) { 8684 field_flags |= I40E_CLOUD_FIELD_IIP; 8685 } else { 8686 dev_err(&pf->pdev->dev, "Bad ip src mask %pI4b\n", 8687 &match.mask->src); 8688 return -EIO; 8689 } 8690 } 8691 8692 if (field_flags & I40E_CLOUD_FIELD_TEN_ID) { 8693 dev_err(&pf->pdev->dev, "Tenant id not allowed for ip filter\n"); 8694 return -EIO; 8695 } 8696 filter->dst_ipv4 = match.key->dst; 8697 filter->src_ipv4 = match.key->src; 8698 } 8699 8700 if (addr_type == FLOW_DISSECTOR_KEY_IPV6_ADDRS) { 8701 struct flow_match_ipv6_addrs match; 8702 8703 flow_rule_match_ipv6_addrs(rule, &match); 8704 8705 /* src and dest IPV6 address should not be LOOPBACK 8706 * (0:0:0:0:0:0:0:1), which can be represented as ::1 8707 */ 8708 if (ipv6_addr_loopback(&match.key->dst) || 8709 ipv6_addr_loopback(&match.key->src)) { 8710 dev_err(&pf->pdev->dev, 8711 "Bad ipv6, addr is LOOPBACK\n"); 8712 return -EIO; 8713 } 8714 if (!ipv6_addr_any(&match.mask->dst) || 8715 !ipv6_addr_any(&match.mask->src)) 8716 field_flags |= I40E_CLOUD_FIELD_IIP; 8717 8718 memcpy(&filter->src_ipv6, &match.key->src.s6_addr32, 8719 sizeof(filter->src_ipv6)); 8720 memcpy(&filter->dst_ipv6, &match.key->dst.s6_addr32, 8721 sizeof(filter->dst_ipv6)); 8722 } 8723 8724 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_PORTS)) { 8725 struct flow_match_ports match; 8726 8727 flow_rule_match_ports(rule, &match); 8728 if (match.mask->src) { 8729 if (match.mask->src == cpu_to_be16(0xffff)) { 8730 field_flags |= I40E_CLOUD_FIELD_IIP; 8731 } else { 8732 dev_err(&pf->pdev->dev, "Bad src port mask 0x%04x\n", 8733 be16_to_cpu(match.mask->src)); 8734 return -EIO; 8735 } 8736 } 8737 8738 if (match.mask->dst) { 8739 if (match.mask->dst == cpu_to_be16(0xffff)) { 8740 field_flags |= I40E_CLOUD_FIELD_IIP; 8741 } else { 8742 dev_err(&pf->pdev->dev, "Bad dst port mask 0x%04x\n", 8743 be16_to_cpu(match.mask->dst)); 8744 return -EIO; 8745 } 8746 } 8747 8748 filter->dst_port = match.key->dst; 8749 filter->src_port = match.key->src; 8750 8751 switch (filter->ip_proto) { 8752 case IPPROTO_TCP: 8753 case IPPROTO_UDP: 8754 break; 8755 default: 8756 dev_err(&pf->pdev->dev, 8757 "Only UDP and TCP transport are supported\n"); 8758 return -EINVAL; 8759 } 8760 } 8761 filter->flags = field_flags; 8762 return 0; 8763 } 8764 8765 /** 8766 * i40e_handle_tclass: Forward to a traffic class on the device 8767 * @vsi: Pointer to VSI 8768 * @tc: traffic class index on the device 8769 * @filter: Pointer to cloud filter structure 8770 * 8771 **/ 8772 static int i40e_handle_tclass(struct i40e_vsi *vsi, u32 tc, 8773 struct i40e_cloud_filter *filter) 8774 { 8775 struct i40e_channel *ch, *ch_tmp; 8776 8777 /* direct to a traffic class on the same device */ 8778 if (tc == 0) { 8779 filter->seid = vsi->seid; 8780 return 0; 8781 } else if (vsi->tc_config.enabled_tc & BIT(tc)) { 8782 if (!filter->dst_port) { 8783 dev_err(&vsi->back->pdev->dev, 8784 "Specify destination port to direct to traffic class that is not default\n"); 8785 return -EINVAL; 8786 } 8787 if (list_empty(&vsi->ch_list)) 8788 return -EINVAL; 8789 list_for_each_entry_safe(ch, ch_tmp, &vsi->ch_list, 8790 list) { 8791 if (ch->seid == vsi->tc_seid_map[tc]) 8792 filter->seid = ch->seid; 8793 } 8794 return 0; 8795 } 8796 dev_err(&vsi->back->pdev->dev, "TC is not enabled\n"); 8797 return -EINVAL; 8798 } 8799 8800 /** 8801 * i40e_configure_clsflower - Configure tc flower filters 8802 * @vsi: Pointer to VSI 8803 * @cls_flower: Pointer to struct flow_cls_offload 8804 * 8805 **/ 8806 static int i40e_configure_clsflower(struct i40e_vsi *vsi, 8807 struct flow_cls_offload *cls_flower) 8808 { 8809 int tc = tc_classid_to_hwtc(vsi->netdev, cls_flower->classid); 8810 struct i40e_cloud_filter *filter = NULL; 8811 struct i40e_pf *pf = vsi->back; 8812 int err = 0; 8813 8814 if (tc < 0) { 8815 dev_err(&vsi->back->pdev->dev, "Invalid traffic class\n"); 8816 return -EOPNOTSUPP; 8817 } 8818 8819 if (!tc) { 8820 dev_err(&pf->pdev->dev, "Unable to add filter because of invalid destination"); 8821 return -EINVAL; 8822 } 8823 8824 if (test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state) || 8825 test_bit(__I40E_RESET_INTR_RECEIVED, pf->state)) 8826 return -EBUSY; 8827 8828 if (pf->fdir_pf_active_filters || 8829 (!hlist_empty(&pf->fdir_filter_list))) { 8830 dev_err(&vsi->back->pdev->dev, 8831 "Flow Director Sideband filters exists, turn ntuple off to configure cloud filters\n"); 8832 return -EINVAL; 8833 } 8834 8835 if (test_bit(I40E_FLAG_FD_SB_ENA, vsi->back->flags)) { 8836 dev_err(&vsi->back->pdev->dev, 8837 "Disable Flow Director Sideband, configuring Cloud filters via tc-flower\n"); 8838 clear_bit(I40E_FLAG_FD_SB_ENA, vsi->back->flags); 8839 clear_bit(I40E_FLAG_FD_SB_TO_CLOUD_FILTER, vsi->back->flags); 8840 } 8841 8842 filter = kzalloc_obj(*filter); 8843 if (!filter) 8844 return -ENOMEM; 8845 8846 filter->cookie = cls_flower->cookie; 8847 8848 err = i40e_parse_cls_flower(vsi, cls_flower, filter); 8849 if (err < 0) 8850 goto err; 8851 8852 err = i40e_handle_tclass(vsi, tc, filter); 8853 if (err < 0) 8854 goto err; 8855 8856 /* Add cloud filter */ 8857 if (filter->dst_port) 8858 err = i40e_add_del_cloud_filter_big_buf(vsi, filter, true); 8859 else 8860 err = i40e_add_del_cloud_filter(vsi, filter, true); 8861 8862 if (err) { 8863 dev_err(&pf->pdev->dev, "Failed to add cloud filter, err %d\n", 8864 err); 8865 goto err; 8866 } 8867 8868 /* add filter to the ordered list */ 8869 INIT_HLIST_NODE(&filter->cloud_node); 8870 8871 hlist_add_head(&filter->cloud_node, &pf->cloud_filter_list); 8872 8873 pf->num_cloud_filters++; 8874 8875 return err; 8876 err: 8877 kfree(filter); 8878 return err; 8879 } 8880 8881 /** 8882 * i40e_find_cloud_filter - Find the could filter in the list 8883 * @vsi: Pointer to VSI 8884 * @cookie: filter specific cookie 8885 * 8886 **/ 8887 static struct i40e_cloud_filter *i40e_find_cloud_filter(struct i40e_vsi *vsi, 8888 unsigned long *cookie) 8889 { 8890 struct i40e_cloud_filter *filter = NULL; 8891 struct hlist_node *node2; 8892 8893 hlist_for_each_entry_safe(filter, node2, 8894 &vsi->back->cloud_filter_list, cloud_node) 8895 if (!memcmp(cookie, &filter->cookie, sizeof(filter->cookie))) 8896 return filter; 8897 return NULL; 8898 } 8899 8900 /** 8901 * i40e_delete_clsflower - Remove tc flower filters 8902 * @vsi: Pointer to VSI 8903 * @cls_flower: Pointer to struct flow_cls_offload 8904 * 8905 **/ 8906 static int i40e_delete_clsflower(struct i40e_vsi *vsi, 8907 struct flow_cls_offload *cls_flower) 8908 { 8909 struct i40e_cloud_filter *filter = NULL; 8910 struct i40e_pf *pf = vsi->back; 8911 int err = 0; 8912 8913 filter = i40e_find_cloud_filter(vsi, &cls_flower->cookie); 8914 8915 if (!filter) 8916 return -EINVAL; 8917 8918 hash_del(&filter->cloud_node); 8919 8920 if (filter->dst_port) 8921 err = i40e_add_del_cloud_filter_big_buf(vsi, filter, false); 8922 else 8923 err = i40e_add_del_cloud_filter(vsi, filter, false); 8924 8925 kfree(filter); 8926 if (err) { 8927 dev_err(&pf->pdev->dev, 8928 "Failed to delete cloud filter, err %pe\n", 8929 ERR_PTR(err)); 8930 return i40e_aq_rc_to_posix(err, pf->hw.aq.asq_last_status); 8931 } 8932 8933 pf->num_cloud_filters--; 8934 if (!pf->num_cloud_filters) 8935 if (test_bit(I40E_FLAG_FD_SB_TO_CLOUD_FILTER, pf->flags) && 8936 !test_bit(I40E_FLAG_FD_SB_INACTIVE, pf->flags)) { 8937 set_bit(I40E_FLAG_FD_SB_ENA, pf->flags); 8938 clear_bit(I40E_FLAG_FD_SB_TO_CLOUD_FILTER, pf->flags); 8939 clear_bit(I40E_FLAG_FD_SB_INACTIVE, pf->flags); 8940 } 8941 return 0; 8942 } 8943 8944 /** 8945 * i40e_setup_tc_cls_flower - flower classifier offloads 8946 * @np: net device to configure 8947 * @cls_flower: offload data 8948 **/ 8949 static int i40e_setup_tc_cls_flower(struct i40e_netdev_priv *np, 8950 struct flow_cls_offload *cls_flower) 8951 { 8952 struct i40e_vsi *vsi = np->vsi; 8953 8954 switch (cls_flower->command) { 8955 case FLOW_CLS_REPLACE: 8956 return i40e_configure_clsflower(vsi, cls_flower); 8957 case FLOW_CLS_DESTROY: 8958 return i40e_delete_clsflower(vsi, cls_flower); 8959 case FLOW_CLS_STATS: 8960 return -EOPNOTSUPP; 8961 default: 8962 return -EOPNOTSUPP; 8963 } 8964 } 8965 8966 static int i40e_setup_tc_block_cb(enum tc_setup_type type, void *type_data, 8967 void *cb_priv) 8968 { 8969 struct i40e_netdev_priv *np = cb_priv; 8970 8971 if (!tc_cls_can_offload_and_chain0(np->vsi->netdev, type_data)) 8972 return -EOPNOTSUPP; 8973 8974 switch (type) { 8975 case TC_SETUP_CLSFLOWER: 8976 return i40e_setup_tc_cls_flower(np, type_data); 8977 8978 default: 8979 return -EOPNOTSUPP; 8980 } 8981 } 8982 8983 static LIST_HEAD(i40e_block_cb_list); 8984 8985 static int __i40e_setup_tc(struct net_device *netdev, enum tc_setup_type type, 8986 void *type_data) 8987 { 8988 struct i40e_netdev_priv *np = netdev_priv(netdev); 8989 8990 switch (type) { 8991 case TC_SETUP_QDISC_MQPRIO: 8992 return i40e_setup_tc(netdev, type_data); 8993 case TC_SETUP_BLOCK: 8994 return flow_block_cb_setup_simple(type_data, 8995 &i40e_block_cb_list, 8996 i40e_setup_tc_block_cb, 8997 np, np, true); 8998 default: 8999 return -EOPNOTSUPP; 9000 } 9001 } 9002 9003 /** 9004 * i40e_open - Called when a network interface is made active 9005 * @netdev: network interface device structure 9006 * 9007 * The open entry point is called when a network interface is made 9008 * active by the system (IFF_UP). At this point all resources needed 9009 * for transmit and receive operations are allocated, the interrupt 9010 * handler is registered with the OS, the netdev watchdog subtask is 9011 * enabled, and the stack is notified that the interface is ready. 9012 * 9013 * Returns 0 on success, negative value on failure 9014 **/ 9015 int i40e_open(struct net_device *netdev) 9016 { 9017 struct i40e_netdev_priv *np = netdev_priv(netdev); 9018 struct i40e_vsi *vsi = np->vsi; 9019 struct i40e_pf *pf = vsi->back; 9020 int err; 9021 9022 /* disallow open during test or if eeprom is broken */ 9023 if (test_bit(__I40E_TESTING, pf->state) || 9024 test_bit(__I40E_BAD_EEPROM, pf->state)) 9025 return -EBUSY; 9026 9027 netif_carrier_off(netdev); 9028 9029 if (i40e_force_link_state(pf, true)) 9030 return -EAGAIN; 9031 9032 err = i40e_vsi_open(vsi); 9033 if (err) 9034 return err; 9035 9036 /* configure global TSO hardware offload settings */ 9037 wr32(&pf->hw, I40E_GLLAN_TSOMSK_F, be32_to_cpu(TCP_FLAG_PSH | 9038 TCP_FLAG_FIN) >> 16); 9039 wr32(&pf->hw, I40E_GLLAN_TSOMSK_M, be32_to_cpu(TCP_FLAG_PSH | 9040 TCP_FLAG_FIN | 9041 TCP_FLAG_CWR) >> 16); 9042 wr32(&pf->hw, I40E_GLLAN_TSOMSK_L, be32_to_cpu(TCP_FLAG_CWR) >> 16); 9043 9044 return 0; 9045 } 9046 9047 /** 9048 * i40e_netif_set_realnum_tx_rx_queues - Update number of tx/rx queues 9049 * @vsi: vsi structure 9050 * 9051 * This updates netdev's number of tx/rx queues 9052 * 9053 * Returns status of setting tx/rx queues 9054 **/ 9055 static int i40e_netif_set_realnum_tx_rx_queues(struct i40e_vsi *vsi) 9056 { 9057 int ret; 9058 9059 ret = netif_set_real_num_rx_queues(vsi->netdev, 9060 vsi->num_queue_pairs); 9061 if (ret) 9062 return ret; 9063 9064 return netif_set_real_num_tx_queues(vsi->netdev, 9065 vsi->num_queue_pairs); 9066 } 9067 9068 /** 9069 * i40e_vsi_open - 9070 * @vsi: the VSI to open 9071 * 9072 * Finish initialization of the VSI. 9073 * 9074 * Returns 0 on success, negative value on failure 9075 * 9076 * Note: expects to be called while under rtnl_lock() 9077 **/ 9078 int i40e_vsi_open(struct i40e_vsi *vsi) 9079 { 9080 struct i40e_pf *pf = vsi->back; 9081 char int_name[I40E_INT_NAME_STR_LEN]; 9082 int err; 9083 9084 /* allocate descriptors */ 9085 err = i40e_vsi_setup_tx_resources(vsi); 9086 if (err) 9087 goto err_setup_tx; 9088 err = i40e_vsi_setup_rx_resources(vsi); 9089 if (err) 9090 goto err_setup_rx; 9091 9092 err = i40e_vsi_configure(vsi); 9093 if (err) 9094 goto err_setup_rx; 9095 9096 if (vsi->netdev) { 9097 snprintf(int_name, sizeof(int_name) - 1, "%s-%s", 9098 dev_driver_string(&pf->pdev->dev), vsi->netdev->name); 9099 err = i40e_vsi_request_irq(vsi, int_name); 9100 if (err) 9101 goto err_setup_rx; 9102 9103 /* Notify the stack of the actual queue counts. */ 9104 err = i40e_netif_set_realnum_tx_rx_queues(vsi); 9105 if (err) 9106 goto err_set_queues; 9107 9108 } else if (vsi->type == I40E_VSI_FDIR) { 9109 snprintf(int_name, sizeof(int_name) - 1, "%s-%s:fdir", 9110 dev_driver_string(&pf->pdev->dev), 9111 dev_name(&pf->pdev->dev)); 9112 err = i40e_vsi_request_irq(vsi, int_name); 9113 if (err) 9114 goto err_setup_rx; 9115 9116 } else { 9117 err = -EINVAL; 9118 goto err_setup_rx; 9119 } 9120 9121 err = i40e_up_complete(vsi); 9122 if (err) 9123 goto err_up_complete; 9124 9125 return 0; 9126 9127 err_up_complete: 9128 i40e_down(vsi); 9129 err_set_queues: 9130 i40e_vsi_free_irq(vsi); 9131 err_setup_rx: 9132 i40e_vsi_free_rx_resources(vsi); 9133 err_setup_tx: 9134 i40e_vsi_free_tx_resources(vsi); 9135 if (vsi->type == I40E_VSI_MAIN) 9136 i40e_do_reset(pf, I40E_PF_RESET_FLAG, true); 9137 9138 return err; 9139 } 9140 9141 /** 9142 * i40e_fdir_filter_exit - Cleans up the Flow Director accounting 9143 * @pf: Pointer to PF 9144 * 9145 * This function destroys the hlist where all the Flow Director 9146 * filters were saved. 9147 **/ 9148 static void i40e_fdir_filter_exit(struct i40e_pf *pf) 9149 { 9150 struct i40e_fdir_filter *filter; 9151 struct i40e_flex_pit *pit_entry, *tmp; 9152 struct hlist_node *node2; 9153 9154 hlist_for_each_entry_safe(filter, node2, 9155 &pf->fdir_filter_list, fdir_node) { 9156 hlist_del(&filter->fdir_node); 9157 kfree(filter); 9158 } 9159 9160 list_for_each_entry_safe(pit_entry, tmp, &pf->l3_flex_pit_list, list) { 9161 list_del(&pit_entry->list); 9162 kfree(pit_entry); 9163 } 9164 INIT_LIST_HEAD(&pf->l3_flex_pit_list); 9165 9166 list_for_each_entry_safe(pit_entry, tmp, &pf->l4_flex_pit_list, list) { 9167 list_del(&pit_entry->list); 9168 kfree(pit_entry); 9169 } 9170 INIT_LIST_HEAD(&pf->l4_flex_pit_list); 9171 9172 pf->fdir_pf_active_filters = 0; 9173 i40e_reset_fdir_filter_cnt(pf); 9174 9175 /* Reprogram the default input set for TCP/IPv4 */ 9176 i40e_write_fd_input_set(pf, LIBIE_FILTER_PCTYPE_NONF_IPV4_TCP, 9177 I40E_L3_SRC_MASK | I40E_L3_DST_MASK | 9178 I40E_L4_SRC_MASK | I40E_L4_DST_MASK); 9179 9180 /* Reprogram the default input set for TCP/IPv6 */ 9181 i40e_write_fd_input_set(pf, LIBIE_FILTER_PCTYPE_NONF_IPV6_TCP, 9182 I40E_L3_V6_SRC_MASK | I40E_L3_V6_DST_MASK | 9183 I40E_L4_SRC_MASK | I40E_L4_DST_MASK); 9184 9185 /* Reprogram the default input set for UDP/IPv4 */ 9186 i40e_write_fd_input_set(pf, LIBIE_FILTER_PCTYPE_NONF_IPV4_UDP, 9187 I40E_L3_SRC_MASK | I40E_L3_DST_MASK | 9188 I40E_L4_SRC_MASK | I40E_L4_DST_MASK); 9189 9190 /* Reprogram the default input set for UDP/IPv6 */ 9191 i40e_write_fd_input_set(pf, LIBIE_FILTER_PCTYPE_NONF_IPV6_UDP, 9192 I40E_L3_V6_SRC_MASK | I40E_L3_V6_DST_MASK | 9193 I40E_L4_SRC_MASK | I40E_L4_DST_MASK); 9194 9195 /* Reprogram the default input set for SCTP/IPv4 */ 9196 i40e_write_fd_input_set(pf, LIBIE_FILTER_PCTYPE_NONF_IPV4_SCTP, 9197 I40E_L3_SRC_MASK | I40E_L3_DST_MASK | 9198 I40E_L4_SRC_MASK | I40E_L4_DST_MASK); 9199 9200 /* Reprogram the default input set for SCTP/IPv6 */ 9201 i40e_write_fd_input_set(pf, LIBIE_FILTER_PCTYPE_NONF_IPV6_SCTP, 9202 I40E_L3_V6_SRC_MASK | I40E_L3_V6_DST_MASK | 9203 I40E_L4_SRC_MASK | I40E_L4_DST_MASK); 9204 9205 /* Reprogram the default input set for Other/IPv4 */ 9206 i40e_write_fd_input_set(pf, LIBIE_FILTER_PCTYPE_NONF_IPV4_OTHER, 9207 I40E_L3_SRC_MASK | I40E_L3_DST_MASK); 9208 9209 i40e_write_fd_input_set(pf, LIBIE_FILTER_PCTYPE_FRAG_IPV4, 9210 I40E_L3_SRC_MASK | I40E_L3_DST_MASK); 9211 9212 /* Reprogram the default input set for Other/IPv6 */ 9213 i40e_write_fd_input_set(pf, LIBIE_FILTER_PCTYPE_NONF_IPV6_OTHER, 9214 I40E_L3_SRC_MASK | I40E_L3_DST_MASK); 9215 9216 i40e_write_fd_input_set(pf, LIBIE_FILTER_PCTYPE_FRAG_IPV6, 9217 I40E_L3_SRC_MASK | I40E_L3_DST_MASK); 9218 } 9219 9220 /** 9221 * i40e_cloud_filter_exit - Cleans up the cloud filters 9222 * @pf: Pointer to PF 9223 * 9224 * This function destroys the hlist where all the cloud filters 9225 * were saved. 9226 **/ 9227 static void i40e_cloud_filter_exit(struct i40e_pf *pf) 9228 { 9229 struct i40e_cloud_filter *cfilter; 9230 struct hlist_node *node; 9231 9232 hlist_for_each_entry_safe(cfilter, node, 9233 &pf->cloud_filter_list, cloud_node) { 9234 hlist_del(&cfilter->cloud_node); 9235 kfree(cfilter); 9236 } 9237 pf->num_cloud_filters = 0; 9238 9239 if (test_bit(I40E_FLAG_FD_SB_TO_CLOUD_FILTER, pf->flags) && 9240 !test_bit(I40E_FLAG_FD_SB_INACTIVE, pf->flags)) { 9241 set_bit(I40E_FLAG_FD_SB_ENA, pf->flags); 9242 clear_bit(I40E_FLAG_FD_SB_TO_CLOUD_FILTER, pf->flags); 9243 clear_bit(I40E_FLAG_FD_SB_INACTIVE, pf->flags); 9244 } 9245 } 9246 9247 /** 9248 * i40e_close - Disables a network interface 9249 * @netdev: network interface device structure 9250 * 9251 * The close entry point is called when an interface is de-activated 9252 * by the OS. The hardware is still under the driver's control, but 9253 * this netdev interface is disabled. 9254 * 9255 * Returns 0, this is not allowed to fail 9256 **/ 9257 int i40e_close(struct net_device *netdev) 9258 { 9259 struct i40e_netdev_priv *np = netdev_priv(netdev); 9260 struct i40e_vsi *vsi = np->vsi; 9261 9262 i40e_vsi_close(vsi); 9263 9264 return 0; 9265 } 9266 9267 /** 9268 * i40e_do_reset - Start a PF or Core Reset sequence 9269 * @pf: board private structure 9270 * @reset_flags: which reset is requested 9271 * @lock_acquired: indicates whether or not the lock has been acquired 9272 * before this function was called. 9273 * 9274 * The essential difference in resets is that the PF Reset 9275 * doesn't clear the packet buffers, doesn't reset the PE 9276 * firmware, and doesn't bother the other PFs on the chip. 9277 **/ 9278 void i40e_do_reset(struct i40e_pf *pf, u32 reset_flags, bool lock_acquired) 9279 { 9280 struct i40e_vsi *vsi; 9281 u32 val; 9282 int i; 9283 9284 /* do the biggest reset indicated */ 9285 if (reset_flags & BIT_ULL(__I40E_GLOBAL_RESET_REQUESTED)) { 9286 9287 /* Request a Global Reset 9288 * 9289 * This will start the chip's countdown to the actual full 9290 * chip reset event, and a warning interrupt to be sent 9291 * to all PFs, including the requestor. Our handler 9292 * for the warning interrupt will deal with the shutdown 9293 * and recovery of the switch setup. 9294 */ 9295 dev_dbg(&pf->pdev->dev, "GlobalR requested\n"); 9296 val = rd32(&pf->hw, I40E_GLGEN_RTRIG); 9297 val |= I40E_GLGEN_RTRIG_GLOBR_MASK; 9298 wr32(&pf->hw, I40E_GLGEN_RTRIG, val); 9299 9300 } else if (reset_flags & BIT_ULL(__I40E_CORE_RESET_REQUESTED)) { 9301 9302 /* Request a Core Reset 9303 * 9304 * Same as Global Reset, except does *not* include the MAC/PHY 9305 */ 9306 dev_dbg(&pf->pdev->dev, "CoreR requested\n"); 9307 val = rd32(&pf->hw, I40E_GLGEN_RTRIG); 9308 val |= I40E_GLGEN_RTRIG_CORER_MASK; 9309 wr32(&pf->hw, I40E_GLGEN_RTRIG, val); 9310 i40e_flush(&pf->hw); 9311 9312 } else if (reset_flags & I40E_PF_RESET_FLAG) { 9313 9314 /* Request a PF Reset 9315 * 9316 * Resets only the PF-specific registers 9317 * 9318 * This goes directly to the tear-down and rebuild of 9319 * the switch, since we need to do all the recovery as 9320 * for the Core Reset. 9321 */ 9322 dev_dbg(&pf->pdev->dev, "PFR requested\n"); 9323 i40e_handle_reset_warning(pf, lock_acquired); 9324 9325 } else if (reset_flags & I40E_PF_RESET_AND_REBUILD_FLAG) { 9326 /* Request a PF Reset 9327 * 9328 * Resets PF and reinitializes PFs VSI. 9329 */ 9330 i40e_prep_for_reset(pf); 9331 i40e_reset_and_rebuild(pf, true, lock_acquired); 9332 dev_info(&pf->pdev->dev, 9333 test_bit(I40E_FLAG_FW_LLDP_DIS, pf->flags) ? 9334 "FW LLDP is disabled\n" : 9335 "FW LLDP is enabled\n"); 9336 9337 } else if (reset_flags & BIT_ULL(__I40E_REINIT_REQUESTED)) { 9338 /* Find the VSI(s) that requested a re-init */ 9339 dev_info(&pf->pdev->dev, "VSI reinit requested\n"); 9340 9341 i40e_pf_for_each_vsi(pf, i, vsi) { 9342 if (test_and_clear_bit(__I40E_VSI_REINIT_REQUESTED, 9343 vsi->state)) 9344 i40e_vsi_reinit_locked(vsi); 9345 } 9346 } else if (reset_flags & BIT_ULL(__I40E_DOWN_REQUESTED)) { 9347 /* Find the VSI(s) that needs to be brought down */ 9348 dev_info(&pf->pdev->dev, "VSI down requested\n"); 9349 9350 i40e_pf_for_each_vsi(pf, i, vsi) { 9351 if (test_and_clear_bit(__I40E_VSI_DOWN_REQUESTED, 9352 vsi->state)) { 9353 set_bit(__I40E_VSI_DOWN, vsi->state); 9354 i40e_down(vsi); 9355 } 9356 } 9357 } else { 9358 dev_info(&pf->pdev->dev, 9359 "bad reset request 0x%08x\n", reset_flags); 9360 } 9361 } 9362 9363 #ifdef CONFIG_I40E_DCB 9364 /** 9365 * i40e_dcb_need_reconfig - Check if DCB needs reconfig 9366 * @pf: board private structure 9367 * @old_cfg: current DCB config 9368 * @new_cfg: new DCB config 9369 **/ 9370 bool i40e_dcb_need_reconfig(struct i40e_pf *pf, 9371 struct i40e_dcbx_config *old_cfg, 9372 struct i40e_dcbx_config *new_cfg) 9373 { 9374 bool need_reconfig = false; 9375 9376 /* Check if ETS configuration has changed */ 9377 if (memcmp(&new_cfg->etscfg, 9378 &old_cfg->etscfg, 9379 sizeof(new_cfg->etscfg))) { 9380 /* If Priority Table has changed reconfig is needed */ 9381 if (memcmp(&new_cfg->etscfg.prioritytable, 9382 &old_cfg->etscfg.prioritytable, 9383 sizeof(new_cfg->etscfg.prioritytable))) { 9384 need_reconfig = true; 9385 dev_dbg(&pf->pdev->dev, "ETS UP2TC changed.\n"); 9386 } 9387 9388 if (memcmp(&new_cfg->etscfg.tcbwtable, 9389 &old_cfg->etscfg.tcbwtable, 9390 sizeof(new_cfg->etscfg.tcbwtable))) 9391 dev_dbg(&pf->pdev->dev, "ETS TC BW Table changed.\n"); 9392 9393 if (memcmp(&new_cfg->etscfg.tsatable, 9394 &old_cfg->etscfg.tsatable, 9395 sizeof(new_cfg->etscfg.tsatable))) 9396 dev_dbg(&pf->pdev->dev, "ETS TSA Table changed.\n"); 9397 } 9398 9399 /* Check if PFC configuration has changed */ 9400 if (memcmp(&new_cfg->pfc, 9401 &old_cfg->pfc, 9402 sizeof(new_cfg->pfc))) { 9403 need_reconfig = true; 9404 dev_dbg(&pf->pdev->dev, "PFC config change detected.\n"); 9405 } 9406 9407 /* Check if APP Table has changed */ 9408 if (memcmp(&new_cfg->app, 9409 &old_cfg->app, 9410 sizeof(new_cfg->app))) { 9411 need_reconfig = true; 9412 dev_dbg(&pf->pdev->dev, "APP Table change detected.\n"); 9413 } 9414 9415 dev_dbg(&pf->pdev->dev, "dcb need_reconfig=%d\n", need_reconfig); 9416 return need_reconfig; 9417 } 9418 9419 /** 9420 * i40e_handle_lldp_event - Handle LLDP Change MIB event 9421 * @pf: board private structure 9422 * @e: event info posted on ARQ 9423 **/ 9424 static int i40e_handle_lldp_event(struct i40e_pf *pf, 9425 struct i40e_arq_event_info *e) 9426 { 9427 struct i40e_aqc_lldp_get_mib *mib = libie_aq_raw(&e->desc); 9428 struct i40e_hw *hw = &pf->hw; 9429 struct i40e_dcbx_config tmp_dcbx_cfg; 9430 bool need_reconfig = false; 9431 int ret = 0; 9432 u8 type; 9433 9434 /* X710-T*L 2.5G and 5G speeds don't support DCB */ 9435 if (I40E_IS_X710TL_DEVICE(hw->device_id) && 9436 (hw->phy.link_info.link_speed & 9437 ~(I40E_LINK_SPEED_2_5GB | I40E_LINK_SPEED_5GB)) && 9438 !test_bit(I40E_FLAG_DCB_CAPABLE, pf->flags)) 9439 /* let firmware decide if the DCB should be disabled */ 9440 set_bit(I40E_FLAG_DCB_CAPABLE, pf->flags); 9441 9442 /* Not DCB capable or capability disabled */ 9443 if (!test_bit(I40E_FLAG_DCB_CAPABLE, pf->flags)) 9444 return ret; 9445 9446 /* Ignore if event is not for Nearest Bridge */ 9447 type = ((mib->type >> I40E_AQ_LLDP_BRIDGE_TYPE_SHIFT) 9448 & I40E_AQ_LLDP_BRIDGE_TYPE_MASK); 9449 dev_dbg(&pf->pdev->dev, "LLDP event mib bridge type 0x%x\n", type); 9450 if (type != I40E_AQ_LLDP_BRIDGE_TYPE_NEAREST_BRIDGE) 9451 return ret; 9452 9453 /* Check MIB Type and return if event for Remote MIB update */ 9454 type = mib->type & I40E_AQ_LLDP_MIB_TYPE_MASK; 9455 dev_dbg(&pf->pdev->dev, 9456 "LLDP event mib type %s\n", type ? "remote" : "local"); 9457 if (type == I40E_AQ_LLDP_MIB_REMOTE) { 9458 /* Update the remote cached instance and return */ 9459 ret = i40e_aq_get_dcb_config(hw, I40E_AQ_LLDP_MIB_REMOTE, 9460 I40E_AQ_LLDP_BRIDGE_TYPE_NEAREST_BRIDGE, 9461 &hw->remote_dcbx_config); 9462 goto exit; 9463 } 9464 9465 /* Store the old configuration */ 9466 tmp_dcbx_cfg = hw->local_dcbx_config; 9467 9468 /* Reset the old DCBx configuration data */ 9469 memset(&hw->local_dcbx_config, 0, sizeof(hw->local_dcbx_config)); 9470 /* Get updated DCBX data from firmware */ 9471 ret = i40e_get_dcb_config(&pf->hw); 9472 if (ret) { 9473 /* X710-T*L 2.5G and 5G speeds don't support DCB */ 9474 if (I40E_IS_X710TL_DEVICE(hw->device_id) && 9475 (hw->phy.link_info.link_speed & 9476 (I40E_LINK_SPEED_2_5GB | I40E_LINK_SPEED_5GB))) { 9477 dev_warn(&pf->pdev->dev, 9478 "DCB is not supported for X710-T*L 2.5/5G speeds\n"); 9479 clear_bit(I40E_FLAG_DCB_CAPABLE, pf->flags); 9480 } else { 9481 dev_info(&pf->pdev->dev, 9482 "Failed querying DCB configuration data from firmware, err %pe aq_err %s\n", 9483 ERR_PTR(ret), 9484 libie_aq_str(pf->hw.aq.asq_last_status)); 9485 } 9486 goto exit; 9487 } 9488 9489 /* No change detected in DCBX configs */ 9490 if (!memcmp(&tmp_dcbx_cfg, &hw->local_dcbx_config, 9491 sizeof(tmp_dcbx_cfg))) { 9492 dev_dbg(&pf->pdev->dev, "No change detected in DCBX configuration.\n"); 9493 goto exit; 9494 } 9495 9496 need_reconfig = i40e_dcb_need_reconfig(pf, &tmp_dcbx_cfg, 9497 &hw->local_dcbx_config); 9498 9499 i40e_dcbnl_flush_apps(pf, &tmp_dcbx_cfg, &hw->local_dcbx_config); 9500 9501 if (!need_reconfig) 9502 goto exit; 9503 9504 /* Enable DCB tagging only when more than one TC */ 9505 if (i40e_dcb_get_num_tc(&hw->local_dcbx_config) > 1) 9506 set_bit(I40E_FLAG_DCB_ENA, pf->flags); 9507 else 9508 clear_bit(I40E_FLAG_DCB_ENA, pf->flags); 9509 9510 set_bit(__I40E_PORT_SUSPENDED, pf->state); 9511 /* Reconfiguration needed quiesce all VSIs */ 9512 i40e_pf_quiesce_all_vsi(pf); 9513 9514 /* Changes in configuration update VEB/VSI */ 9515 i40e_dcb_reconfigure(pf); 9516 9517 ret = i40e_resume_port_tx(pf); 9518 9519 clear_bit(__I40E_PORT_SUSPENDED, pf->state); 9520 /* In case of error no point in resuming VSIs */ 9521 if (ret) 9522 goto exit; 9523 9524 /* Wait for the PF's queues to be disabled */ 9525 ret = i40e_pf_wait_queues_disabled(pf); 9526 if (ret) { 9527 /* Schedule PF reset to recover */ 9528 set_bit(__I40E_PF_RESET_REQUESTED, pf->state); 9529 i40e_service_event_schedule(pf); 9530 } else { 9531 i40e_pf_unquiesce_all_vsi(pf); 9532 set_bit(__I40E_CLIENT_SERVICE_REQUESTED, pf->state); 9533 set_bit(__I40E_CLIENT_L2_CHANGE, pf->state); 9534 } 9535 9536 exit: 9537 return ret; 9538 } 9539 #endif /* CONFIG_I40E_DCB */ 9540 9541 /** 9542 * i40e_do_reset_safe - Protected reset path for userland calls. 9543 * @pf: board private structure 9544 * @reset_flags: which reset is requested 9545 * 9546 **/ 9547 void i40e_do_reset_safe(struct i40e_pf *pf, u32 reset_flags) 9548 { 9549 rtnl_lock(); 9550 i40e_do_reset(pf, reset_flags, true); 9551 rtnl_unlock(); 9552 } 9553 9554 /** 9555 * i40e_handle_lan_overflow_event - Handler for LAN queue overflow event 9556 * @pf: board private structure 9557 * @e: event info posted on ARQ 9558 * 9559 * Handler for LAN Queue Overflow Event generated by the firmware for PF 9560 * and VF queues 9561 **/ 9562 static void i40e_handle_lan_overflow_event(struct i40e_pf *pf, 9563 struct i40e_arq_event_info *e) 9564 { 9565 struct i40e_aqc_lan_overflow *data = libie_aq_raw(&e->desc); 9566 u32 queue = le32_to_cpu(data->prtdcb_rupto); 9567 u32 qtx_ctl = le32_to_cpu(data->otx_ctl); 9568 struct i40e_hw *hw = &pf->hw; 9569 struct i40e_vf *vf; 9570 u16 vf_id; 9571 9572 dev_dbg(&pf->pdev->dev, "overflow Rx Queue Number = %d QTX_CTL=0x%08x\n", 9573 queue, qtx_ctl); 9574 9575 if (FIELD_GET(I40E_QTX_CTL_PFVF_Q_MASK, qtx_ctl) != 9576 I40E_QTX_CTL_VF_QUEUE) 9577 return; 9578 9579 /* Queue belongs to VF, find the VF and issue VF reset */ 9580 vf_id = FIELD_GET(I40E_QTX_CTL_VFVM_INDX_MASK, qtx_ctl); 9581 vf_id -= hw->func_caps.vf_base_id; 9582 vf = &pf->vf[vf_id]; 9583 i40e_vc_notify_vf_reset(vf); 9584 /* Allow VF to process pending reset notification */ 9585 msleep(20); 9586 i40e_reset_vf(vf, false); 9587 } 9588 9589 /** 9590 * i40e_get_current_fd_count - Get total FD filters programmed for this PF 9591 * @pf: board private structure 9592 **/ 9593 u32 i40e_get_current_fd_count(struct i40e_pf *pf) 9594 { 9595 u32 val, fcnt_prog; 9596 9597 val = rd32(&pf->hw, I40E_PFQF_FDSTAT); 9598 fcnt_prog = (val & I40E_PFQF_FDSTAT_GUARANT_CNT_MASK) + 9599 FIELD_GET(I40E_PFQF_FDSTAT_BEST_CNT_MASK, val); 9600 return fcnt_prog; 9601 } 9602 9603 /** 9604 * i40e_get_global_fd_count - Get total FD filters programmed on device 9605 * @pf: board private structure 9606 **/ 9607 u32 i40e_get_global_fd_count(struct i40e_pf *pf) 9608 { 9609 u32 val, fcnt_prog; 9610 9611 val = rd32(&pf->hw, I40E_GLQF_FDCNT_0); 9612 fcnt_prog = (val & I40E_GLQF_FDCNT_0_GUARANT_CNT_MASK) + 9613 FIELD_GET(I40E_GLQF_FDCNT_0_BESTCNT_MASK, val); 9614 return fcnt_prog; 9615 } 9616 9617 /** 9618 * i40e_reenable_fdir_sb - Restore FDir SB capability 9619 * @pf: board private structure 9620 **/ 9621 static void i40e_reenable_fdir_sb(struct i40e_pf *pf) 9622 { 9623 if (test_and_clear_bit(__I40E_FD_SB_AUTO_DISABLED, pf->state)) 9624 if (test_bit(I40E_FLAG_FD_SB_ENA, pf->flags) && 9625 (I40E_DEBUG_FD & pf->hw.debug_mask)) 9626 dev_info(&pf->pdev->dev, "FD Sideband/ntuple is being enabled since we have space in the table now\n"); 9627 } 9628 9629 /** 9630 * i40e_reenable_fdir_atr - Restore FDir ATR capability 9631 * @pf: board private structure 9632 **/ 9633 static void i40e_reenable_fdir_atr(struct i40e_pf *pf) 9634 { 9635 if (test_and_clear_bit(__I40E_FD_ATR_AUTO_DISABLED, pf->state)) { 9636 /* ATR uses the same filtering logic as SB rules. It only 9637 * functions properly if the input set mask is at the default 9638 * settings. It is safe to restore the default input set 9639 * because there are no active TCPv4 filter rules. 9640 */ 9641 i40e_write_fd_input_set(pf, LIBIE_FILTER_PCTYPE_NONF_IPV4_TCP, 9642 I40E_L3_SRC_MASK | I40E_L3_DST_MASK | 9643 I40E_L4_SRC_MASK | I40E_L4_DST_MASK); 9644 9645 if (test_bit(I40E_FLAG_FD_ATR_ENA, pf->flags) && 9646 (I40E_DEBUG_FD & pf->hw.debug_mask)) 9647 dev_info(&pf->pdev->dev, "ATR is being enabled since we have space in the table and there are no conflicting ntuple rules\n"); 9648 } 9649 } 9650 9651 /** 9652 * i40e_delete_invalid_filter - Delete an invalid FDIR filter 9653 * @pf: board private structure 9654 * @filter: FDir filter to remove 9655 */ 9656 static void i40e_delete_invalid_filter(struct i40e_pf *pf, 9657 struct i40e_fdir_filter *filter) 9658 { 9659 /* Update counters */ 9660 pf->fdir_pf_active_filters--; 9661 pf->fd_inv = 0; 9662 9663 switch (filter->flow_type) { 9664 case TCP_V4_FLOW: 9665 pf->fd_tcp4_filter_cnt--; 9666 break; 9667 case UDP_V4_FLOW: 9668 pf->fd_udp4_filter_cnt--; 9669 break; 9670 case SCTP_V4_FLOW: 9671 pf->fd_sctp4_filter_cnt--; 9672 break; 9673 case TCP_V6_FLOW: 9674 pf->fd_tcp6_filter_cnt--; 9675 break; 9676 case UDP_V6_FLOW: 9677 pf->fd_udp6_filter_cnt--; 9678 break; 9679 case SCTP_V6_FLOW: 9680 pf->fd_udp6_filter_cnt--; 9681 break; 9682 case IP_USER_FLOW: 9683 switch (filter->ipl4_proto) { 9684 case IPPROTO_TCP: 9685 pf->fd_tcp4_filter_cnt--; 9686 break; 9687 case IPPROTO_UDP: 9688 pf->fd_udp4_filter_cnt--; 9689 break; 9690 case IPPROTO_SCTP: 9691 pf->fd_sctp4_filter_cnt--; 9692 break; 9693 case IPPROTO_IP: 9694 pf->fd_ip4_filter_cnt--; 9695 break; 9696 } 9697 break; 9698 case IPV6_USER_FLOW: 9699 switch (filter->ipl4_proto) { 9700 case IPPROTO_TCP: 9701 pf->fd_tcp6_filter_cnt--; 9702 break; 9703 case IPPROTO_UDP: 9704 pf->fd_udp6_filter_cnt--; 9705 break; 9706 case IPPROTO_SCTP: 9707 pf->fd_sctp6_filter_cnt--; 9708 break; 9709 case IPPROTO_IP: 9710 pf->fd_ip6_filter_cnt--; 9711 break; 9712 } 9713 break; 9714 } 9715 9716 /* Remove the filter from the list and free memory */ 9717 hlist_del(&filter->fdir_node); 9718 kfree(filter); 9719 } 9720 9721 /** 9722 * i40e_fdir_check_and_reenable - Function to reenabe FD ATR or SB if disabled 9723 * @pf: board private structure 9724 **/ 9725 void i40e_fdir_check_and_reenable(struct i40e_pf *pf) 9726 { 9727 struct i40e_fdir_filter *filter; 9728 u32 fcnt_prog, fcnt_avail; 9729 struct hlist_node *node; 9730 9731 if (test_bit(__I40E_FD_FLUSH_REQUESTED, pf->state)) 9732 return; 9733 9734 /* Check if we have enough room to re-enable FDir SB capability. */ 9735 fcnt_prog = i40e_get_global_fd_count(pf); 9736 fcnt_avail = pf->fdir_pf_filter_count; 9737 if ((fcnt_prog < (fcnt_avail - I40E_FDIR_BUFFER_HEAD_ROOM)) || 9738 (pf->fd_add_err == 0) || 9739 (i40e_get_current_atr_cnt(pf) < pf->fd_atr_cnt)) 9740 i40e_reenable_fdir_sb(pf); 9741 9742 /* We should wait for even more space before re-enabling ATR. 9743 * Additionally, we cannot enable ATR as long as we still have TCP SB 9744 * rules active. 9745 */ 9746 if ((fcnt_prog < (fcnt_avail - I40E_FDIR_BUFFER_HEAD_ROOM_FOR_ATR)) && 9747 pf->fd_tcp4_filter_cnt == 0 && pf->fd_tcp6_filter_cnt == 0) 9748 i40e_reenable_fdir_atr(pf); 9749 9750 /* if hw had a problem adding a filter, delete it */ 9751 if (pf->fd_inv > 0) { 9752 hlist_for_each_entry_safe(filter, node, 9753 &pf->fdir_filter_list, fdir_node) 9754 if (filter->fd_id == pf->fd_inv) 9755 i40e_delete_invalid_filter(pf, filter); 9756 } 9757 } 9758 9759 #define I40E_MIN_FD_FLUSH_INTERVAL 10 9760 #define I40E_MIN_FD_FLUSH_SB_ATR_UNSTABLE 30 9761 /** 9762 * i40e_fdir_flush_and_replay - Function to flush all FD filters and replay SB 9763 * @pf: board private structure 9764 **/ 9765 static void i40e_fdir_flush_and_replay(struct i40e_pf *pf) 9766 { 9767 unsigned long min_flush_time; 9768 int flush_wait_retry = 50; 9769 bool disable_atr = false; 9770 int fd_room; 9771 int reg; 9772 9773 if (!time_after(jiffies, pf->fd_flush_timestamp + 9774 (I40E_MIN_FD_FLUSH_INTERVAL * HZ))) 9775 return; 9776 9777 /* If the flush is happening too quick and we have mostly SB rules we 9778 * should not re-enable ATR for some time. 9779 */ 9780 min_flush_time = pf->fd_flush_timestamp + 9781 (I40E_MIN_FD_FLUSH_SB_ATR_UNSTABLE * HZ); 9782 fd_room = pf->fdir_pf_filter_count - pf->fdir_pf_active_filters; 9783 9784 if (!(time_after(jiffies, min_flush_time)) && 9785 (fd_room < I40E_FDIR_BUFFER_HEAD_ROOM_FOR_ATR)) { 9786 if (I40E_DEBUG_FD & pf->hw.debug_mask) 9787 dev_info(&pf->pdev->dev, "ATR disabled, not enough FD filter space.\n"); 9788 disable_atr = true; 9789 } 9790 9791 pf->fd_flush_timestamp = jiffies; 9792 set_bit(__I40E_FD_ATR_AUTO_DISABLED, pf->state); 9793 /* flush all filters */ 9794 wr32(&pf->hw, I40E_PFQF_CTL_1, 9795 I40E_PFQF_CTL_1_CLEARFDTABLE_MASK); 9796 i40e_flush(&pf->hw); 9797 pf->fd_flush_cnt++; 9798 pf->fd_add_err = 0; 9799 do { 9800 /* Check FD flush status every 5-6msec */ 9801 usleep_range(5000, 6000); 9802 reg = rd32(&pf->hw, I40E_PFQF_CTL_1); 9803 if (!(reg & I40E_PFQF_CTL_1_CLEARFDTABLE_MASK)) 9804 break; 9805 } while (flush_wait_retry--); 9806 if (reg & I40E_PFQF_CTL_1_CLEARFDTABLE_MASK) { 9807 dev_warn(&pf->pdev->dev, "FD table did not flush, needs more time\n"); 9808 } else { 9809 /* replay sideband filters */ 9810 i40e_fdir_filter_restore(i40e_pf_get_main_vsi(pf)); 9811 if (!disable_atr && !pf->fd_tcp4_filter_cnt) 9812 clear_bit(__I40E_FD_ATR_AUTO_DISABLED, pf->state); 9813 clear_bit(__I40E_FD_FLUSH_REQUESTED, pf->state); 9814 if (I40E_DEBUG_FD & pf->hw.debug_mask) 9815 dev_info(&pf->pdev->dev, "FD Filter table flushed and FD-SB replayed.\n"); 9816 } 9817 } 9818 9819 /** 9820 * i40e_get_current_atr_cnt - Get the count of total FD ATR filters programmed 9821 * @pf: board private structure 9822 **/ 9823 u32 i40e_get_current_atr_cnt(struct i40e_pf *pf) 9824 { 9825 return i40e_get_current_fd_count(pf) - pf->fdir_pf_active_filters; 9826 } 9827 9828 /** 9829 * i40e_fdir_reinit_subtask - Worker thread to reinit FDIR filter table 9830 * @pf: board private structure 9831 **/ 9832 static void i40e_fdir_reinit_subtask(struct i40e_pf *pf) 9833 { 9834 9835 /* if interface is down do nothing */ 9836 if (test_bit(__I40E_DOWN, pf->state)) 9837 return; 9838 9839 if (test_bit(__I40E_FD_FLUSH_REQUESTED, pf->state)) 9840 i40e_fdir_flush_and_replay(pf); 9841 9842 i40e_fdir_check_and_reenable(pf); 9843 9844 } 9845 9846 /** 9847 * i40e_vsi_link_event - notify VSI of a link event 9848 * @vsi: vsi to be notified 9849 * @link_up: link up or down 9850 **/ 9851 static void i40e_vsi_link_event(struct i40e_vsi *vsi, bool link_up) 9852 { 9853 if (!vsi || test_bit(__I40E_VSI_DOWN, vsi->state)) 9854 return; 9855 9856 switch (vsi->type) { 9857 case I40E_VSI_MAIN: 9858 if (!vsi->netdev || !vsi->netdev_registered) 9859 break; 9860 9861 if (link_up) { 9862 netif_carrier_on(vsi->netdev); 9863 netif_tx_wake_all_queues(vsi->netdev); 9864 } else { 9865 netif_carrier_off(vsi->netdev); 9866 netif_tx_stop_all_queues(vsi->netdev); 9867 } 9868 break; 9869 9870 case I40E_VSI_SRIOV: 9871 case I40E_VSI_VMDQ2: 9872 case I40E_VSI_CTRL: 9873 case I40E_VSI_IWARP: 9874 case I40E_VSI_MIRROR: 9875 default: 9876 /* there is no notification for other VSIs */ 9877 break; 9878 } 9879 } 9880 9881 /** 9882 * i40e_veb_link_event - notify elements on the veb of a link event 9883 * @veb: veb to be notified 9884 * @link_up: link up or down 9885 **/ 9886 static void i40e_veb_link_event(struct i40e_veb *veb, bool link_up) 9887 { 9888 struct i40e_vsi *vsi; 9889 struct i40e_pf *pf; 9890 int i; 9891 9892 if (!veb || !veb->pf) 9893 return; 9894 pf = veb->pf; 9895 9896 /* Send link event to contained VSIs */ 9897 i40e_pf_for_each_vsi(pf, i, vsi) 9898 if (vsi->uplink_seid == veb->seid) 9899 i40e_vsi_link_event(vsi, link_up); 9900 } 9901 9902 /** 9903 * i40e_link_event - Update netif_carrier status 9904 * @pf: board private structure 9905 **/ 9906 static void i40e_link_event(struct i40e_pf *pf) 9907 { 9908 struct i40e_vsi *vsi = i40e_pf_get_main_vsi(pf); 9909 struct i40e_veb *veb = i40e_pf_get_main_veb(pf); 9910 u8 new_link_speed, old_link_speed; 9911 bool new_link, old_link; 9912 int status; 9913 #ifdef CONFIG_I40E_DCB 9914 int err; 9915 #endif /* CONFIG_I40E_DCB */ 9916 9917 /* set this to force the get_link_status call to refresh state */ 9918 pf->hw.phy.get_link_info = true; 9919 old_link = (pf->hw.phy.link_info_old.link_info & I40E_AQ_LINK_UP); 9920 status = i40e_get_link_status(&pf->hw, &new_link); 9921 9922 /* On success, disable temp link polling */ 9923 if (status == 0) { 9924 clear_bit(__I40E_TEMP_LINK_POLLING, pf->state); 9925 } else { 9926 /* Enable link polling temporarily until i40e_get_link_status 9927 * returns 0 9928 */ 9929 set_bit(__I40E_TEMP_LINK_POLLING, pf->state); 9930 dev_dbg(&pf->pdev->dev, "couldn't get link state, status: %d\n", 9931 status); 9932 return; 9933 } 9934 9935 old_link_speed = pf->hw.phy.link_info_old.link_speed; 9936 new_link_speed = pf->hw.phy.link_info.link_speed; 9937 9938 if (new_link == old_link && 9939 new_link_speed == old_link_speed && 9940 (test_bit(__I40E_VSI_DOWN, vsi->state) || 9941 new_link == netif_carrier_ok(vsi->netdev))) 9942 return; 9943 9944 if (!new_link && old_link) 9945 pf->link_down_events++; 9946 9947 i40e_print_link_message(vsi, new_link); 9948 9949 /* Notify the base of the switch tree connected to 9950 * the link. Floating VEBs are not notified. 9951 */ 9952 if (veb) 9953 i40e_veb_link_event(veb, new_link); 9954 else 9955 i40e_vsi_link_event(vsi, new_link); 9956 9957 if (pf->vf) 9958 i40e_vc_notify_link_state(pf); 9959 9960 if (test_bit(I40E_FLAG_PTP_ENA, pf->flags)) 9961 i40e_ptp_set_increment(pf); 9962 #ifdef CONFIG_I40E_DCB 9963 if (new_link == old_link) 9964 return; 9965 /* Not SW DCB so firmware will take care of default settings */ 9966 if (pf->dcbx_cap & DCB_CAP_DCBX_LLD_MANAGED) 9967 return; 9968 9969 /* We cover here only link down, as after link up in case of SW DCB 9970 * SW LLDP agent will take care of setting it up 9971 */ 9972 if (!new_link) { 9973 dev_dbg(&pf->pdev->dev, "Reconfig DCB to single TC as result of Link Down\n"); 9974 memset(&pf->tmp_cfg, 0, sizeof(pf->tmp_cfg)); 9975 err = i40e_dcb_sw_default_config(pf); 9976 if (err) { 9977 clear_bit(I40E_FLAG_DCB_CAPABLE, pf->flags); 9978 clear_bit(I40E_FLAG_DCB_ENA, pf->flags); 9979 } else { 9980 pf->dcbx_cap = DCB_CAP_DCBX_HOST | 9981 DCB_CAP_DCBX_VER_IEEE; 9982 set_bit(I40E_FLAG_DCB_CAPABLE, pf->flags); 9983 clear_bit(I40E_FLAG_DCB_ENA, pf->flags); 9984 } 9985 } 9986 #endif /* CONFIG_I40E_DCB */ 9987 } 9988 9989 /** 9990 * i40e_watchdog_subtask - periodic checks not using event driven response 9991 * @pf: board private structure 9992 **/ 9993 static void i40e_watchdog_subtask(struct i40e_pf *pf) 9994 { 9995 struct i40e_vsi *vsi; 9996 struct i40e_veb *veb; 9997 int i; 9998 9999 /* if interface is down do nothing */ 10000 if (test_bit(__I40E_DOWN, pf->state) || 10001 test_bit(__I40E_CONFIG_BUSY, pf->state)) 10002 return; 10003 10004 /* make sure we don't do these things too often */ 10005 if (time_before(jiffies, (pf->service_timer_previous + 10006 pf->service_timer_period))) 10007 return; 10008 pf->service_timer_previous = jiffies; 10009 10010 if (test_bit(I40E_FLAG_LINK_POLLING_ENA, pf->flags) || 10011 test_bit(__I40E_TEMP_LINK_POLLING, pf->state)) 10012 i40e_link_event(pf); 10013 10014 /* Update the stats for active netdevs so the network stack 10015 * can look at updated numbers whenever it cares to 10016 */ 10017 i40e_pf_for_each_vsi(pf, i, vsi) 10018 if (vsi->netdev) 10019 i40e_update_stats(vsi); 10020 10021 if (test_bit(I40E_FLAG_VEB_STATS_ENA, pf->flags)) { 10022 /* Update the stats for the active switching components */ 10023 i40e_pf_for_each_veb(pf, i, veb) 10024 i40e_update_veb_stats(veb); 10025 } 10026 10027 i40e_ptp_rx_hang(pf); 10028 i40e_ptp_tx_hang(pf); 10029 } 10030 10031 /** 10032 * i40e_reset_subtask - Set up for resetting the device and driver 10033 * @pf: board private structure 10034 **/ 10035 static void i40e_reset_subtask(struct i40e_pf *pf) 10036 { 10037 u32 reset_flags = 0; 10038 10039 if (test_bit(__I40E_REINIT_REQUESTED, pf->state)) { 10040 reset_flags |= BIT(__I40E_REINIT_REQUESTED); 10041 clear_bit(__I40E_REINIT_REQUESTED, pf->state); 10042 } 10043 if (test_bit(__I40E_PF_RESET_REQUESTED, pf->state)) { 10044 reset_flags |= BIT(__I40E_PF_RESET_REQUESTED); 10045 clear_bit(__I40E_PF_RESET_REQUESTED, pf->state); 10046 } 10047 if (test_bit(__I40E_CORE_RESET_REQUESTED, pf->state)) { 10048 reset_flags |= BIT(__I40E_CORE_RESET_REQUESTED); 10049 clear_bit(__I40E_CORE_RESET_REQUESTED, pf->state); 10050 } 10051 if (test_bit(__I40E_GLOBAL_RESET_REQUESTED, pf->state)) { 10052 reset_flags |= BIT(__I40E_GLOBAL_RESET_REQUESTED); 10053 clear_bit(__I40E_GLOBAL_RESET_REQUESTED, pf->state); 10054 } 10055 if (test_bit(__I40E_DOWN_REQUESTED, pf->state)) { 10056 reset_flags |= BIT(__I40E_DOWN_REQUESTED); 10057 clear_bit(__I40E_DOWN_REQUESTED, pf->state); 10058 } 10059 10060 /* If there's a recovery already waiting, it takes 10061 * precedence before starting a new reset sequence. 10062 */ 10063 if (test_bit(__I40E_RESET_INTR_RECEIVED, pf->state)) { 10064 i40e_prep_for_reset(pf); 10065 i40e_reset(pf); 10066 i40e_rebuild(pf, false, false); 10067 } 10068 10069 /* If we're already down or resetting, just bail */ 10070 if (reset_flags && 10071 !test_bit(__I40E_DOWN, pf->state) && 10072 !test_bit(__I40E_CONFIG_BUSY, pf->state)) { 10073 i40e_do_reset(pf, reset_flags, false); 10074 } 10075 } 10076 10077 /** 10078 * i40e_handle_link_event - Handle link event 10079 * @pf: board private structure 10080 * @e: event info posted on ARQ 10081 **/ 10082 static void i40e_handle_link_event(struct i40e_pf *pf, 10083 struct i40e_arq_event_info *e) 10084 { 10085 struct i40e_aqc_get_link_status *status = libie_aq_raw(&e->desc); 10086 10087 /* Do a new status request to re-enable LSE reporting 10088 * and load new status information into the hw struct 10089 * This completely ignores any state information 10090 * in the ARQ event info, instead choosing to always 10091 * issue the AQ update link status command. 10092 */ 10093 i40e_link_event(pf); 10094 10095 /* Check if module meets thermal requirements */ 10096 if (status->phy_type == I40E_PHY_TYPE_NOT_SUPPORTED_HIGH_TEMP) { 10097 dev_err(&pf->pdev->dev, 10098 "Rx/Tx is disabled on this device because the module does not meet thermal requirements.\n"); 10099 dev_err(&pf->pdev->dev, 10100 "Refer to the Intel(R) Ethernet Adapters and Devices User Guide for a list of supported modules.\n"); 10101 } else { 10102 /* check for unqualified module, if link is down, suppress 10103 * the message if link was forced to be down. 10104 */ 10105 if ((status->link_info & I40E_AQ_MEDIA_AVAILABLE) && 10106 (!(status->an_info & I40E_AQ_QUALIFIED_MODULE)) && 10107 (!(status->link_info & I40E_AQ_LINK_UP)) && 10108 (!test_bit(I40E_FLAG_LINK_DOWN_ON_CLOSE_ENA, pf->flags))) { 10109 dev_err(&pf->pdev->dev, 10110 "Rx/Tx is disabled on this device because an unsupported SFP module type was detected.\n"); 10111 dev_err(&pf->pdev->dev, 10112 "Refer to the Intel(R) Ethernet Adapters and Devices User Guide for a list of supported modules.\n"); 10113 } 10114 } 10115 } 10116 10117 /** 10118 * i40e_clean_adminq_subtask - Clean the AdminQ rings 10119 * @pf: board private structure 10120 **/ 10121 static void i40e_clean_adminq_subtask(struct i40e_pf *pf) 10122 { 10123 struct i40e_arq_event_info event; 10124 struct i40e_hw *hw = &pf->hw; 10125 u16 pending, i = 0; 10126 u16 opcode; 10127 u32 oldval; 10128 int ret; 10129 u32 val; 10130 10131 /* Do not run clean AQ when PF reset fails */ 10132 if (test_bit(__I40E_RESET_FAILED, pf->state)) 10133 return; 10134 10135 /* check for error indications */ 10136 val = rd32(&pf->hw, I40E_PF_ARQLEN); 10137 oldval = val; 10138 if (val & I40E_PF_ARQLEN_ARQVFE_MASK) { 10139 if (hw->debug_mask & I40E_DEBUG_AQ) 10140 dev_info(&pf->pdev->dev, "ARQ VF Error detected\n"); 10141 val &= ~I40E_PF_ARQLEN_ARQVFE_MASK; 10142 } 10143 if (val & I40E_PF_ARQLEN_ARQOVFL_MASK) { 10144 if (hw->debug_mask & I40E_DEBUG_AQ) 10145 dev_info(&pf->pdev->dev, "ARQ Overflow Error detected\n"); 10146 val &= ~I40E_PF_ARQLEN_ARQOVFL_MASK; 10147 pf->arq_overflows++; 10148 } 10149 if (val & I40E_PF_ARQLEN_ARQCRIT_MASK) { 10150 if (hw->debug_mask & I40E_DEBUG_AQ) 10151 dev_info(&pf->pdev->dev, "ARQ Critical Error detected\n"); 10152 val &= ~I40E_PF_ARQLEN_ARQCRIT_MASK; 10153 } 10154 if (oldval != val) 10155 wr32(&pf->hw, I40E_PF_ARQLEN, val); 10156 10157 val = rd32(&pf->hw, I40E_PF_ATQLEN); 10158 oldval = val; 10159 if (val & I40E_PF_ATQLEN_ATQVFE_MASK) { 10160 if (pf->hw.debug_mask & I40E_DEBUG_AQ) 10161 dev_info(&pf->pdev->dev, "ASQ VF Error detected\n"); 10162 val &= ~I40E_PF_ATQLEN_ATQVFE_MASK; 10163 } 10164 if (val & I40E_PF_ATQLEN_ATQOVFL_MASK) { 10165 if (pf->hw.debug_mask & I40E_DEBUG_AQ) 10166 dev_info(&pf->pdev->dev, "ASQ Overflow Error detected\n"); 10167 val &= ~I40E_PF_ATQLEN_ATQOVFL_MASK; 10168 } 10169 if (val & I40E_PF_ATQLEN_ATQCRIT_MASK) { 10170 if (pf->hw.debug_mask & I40E_DEBUG_AQ) 10171 dev_info(&pf->pdev->dev, "ASQ Critical Error detected\n"); 10172 val &= ~I40E_PF_ATQLEN_ATQCRIT_MASK; 10173 } 10174 if (oldval != val) 10175 wr32(&pf->hw, I40E_PF_ATQLEN, val); 10176 10177 event.buf_len = I40E_MAX_AQ_BUF_SIZE; 10178 event.msg_buf = kzalloc(event.buf_len, GFP_KERNEL); 10179 if (!event.msg_buf) 10180 return; 10181 10182 do { 10183 ret = i40e_clean_arq_element(hw, &event, &pending); 10184 if (ret == -EALREADY) 10185 break; 10186 else if (ret) { 10187 dev_info(&pf->pdev->dev, "ARQ event error %d\n", ret); 10188 break; 10189 } 10190 10191 opcode = le16_to_cpu(event.desc.opcode); 10192 switch (opcode) { 10193 10194 case i40e_aqc_opc_get_link_status: 10195 rtnl_lock(); 10196 i40e_handle_link_event(pf, &event); 10197 rtnl_unlock(); 10198 break; 10199 case i40e_aqc_opc_send_msg_to_pf: 10200 ret = i40e_vc_process_vf_msg(pf, 10201 le16_to_cpu(event.desc.retval), 10202 le32_to_cpu(event.desc.cookie_high), 10203 le32_to_cpu(event.desc.cookie_low), 10204 event.msg_buf, 10205 event.msg_len); 10206 break; 10207 case i40e_aqc_opc_lldp_update_mib: 10208 dev_dbg(&pf->pdev->dev, "ARQ: Update LLDP MIB event received\n"); 10209 #ifdef CONFIG_I40E_DCB 10210 rtnl_lock(); 10211 i40e_handle_lldp_event(pf, &event); 10212 rtnl_unlock(); 10213 #endif /* CONFIG_I40E_DCB */ 10214 break; 10215 case i40e_aqc_opc_event_lan_overflow: 10216 dev_dbg(&pf->pdev->dev, "ARQ LAN queue overflow event received\n"); 10217 i40e_handle_lan_overflow_event(pf, &event); 10218 break; 10219 case i40e_aqc_opc_send_msg_to_peer: 10220 dev_info(&pf->pdev->dev, "ARQ: Msg from other pf\n"); 10221 break; 10222 case i40e_aqc_opc_nvm_erase: 10223 case i40e_aqc_opc_nvm_update: 10224 case i40e_aqc_opc_oem_post_update: 10225 i40e_debug(&pf->hw, I40E_DEBUG_NVM, 10226 "ARQ NVM operation 0x%04x completed\n", 10227 opcode); 10228 break; 10229 default: 10230 dev_info(&pf->pdev->dev, 10231 "ARQ: Unknown event 0x%04x ignored\n", 10232 opcode); 10233 break; 10234 } 10235 } while (i++ < I40E_AQ_WORK_LIMIT); 10236 10237 if (i < I40E_AQ_WORK_LIMIT) 10238 clear_bit(__I40E_ADMINQ_EVENT_PENDING, pf->state); 10239 10240 /* re-enable Admin queue interrupt cause */ 10241 val = rd32(hw, I40E_PFINT_ICR0_ENA); 10242 val |= I40E_PFINT_ICR0_ENA_ADMINQ_MASK; 10243 wr32(hw, I40E_PFINT_ICR0_ENA, val); 10244 i40e_flush(hw); 10245 10246 kfree(event.msg_buf); 10247 } 10248 10249 /** 10250 * i40e_verify_eeprom - make sure eeprom is good to use 10251 * @pf: board private structure 10252 **/ 10253 static void i40e_verify_eeprom(struct i40e_pf *pf) 10254 { 10255 int err; 10256 10257 err = i40e_diag_eeprom_test(&pf->hw); 10258 if (err) { 10259 /* retry in case of garbage read */ 10260 err = i40e_diag_eeprom_test(&pf->hw); 10261 if (err) { 10262 dev_info(&pf->pdev->dev, "eeprom check failed (%d), Tx/Rx traffic disabled\n", 10263 err); 10264 set_bit(__I40E_BAD_EEPROM, pf->state); 10265 } 10266 } 10267 10268 if (!err && test_bit(__I40E_BAD_EEPROM, pf->state)) { 10269 dev_info(&pf->pdev->dev, "eeprom check passed, Tx/Rx traffic enabled\n"); 10270 clear_bit(__I40E_BAD_EEPROM, pf->state); 10271 } 10272 } 10273 10274 /** 10275 * i40e_enable_pf_switch_lb 10276 * @pf: pointer to the PF structure 10277 * 10278 * enable switch loop back or die - no point in a return value 10279 **/ 10280 static void i40e_enable_pf_switch_lb(struct i40e_pf *pf) 10281 { 10282 struct i40e_vsi *vsi = i40e_pf_get_main_vsi(pf); 10283 struct i40e_vsi_context ctxt; 10284 int ret; 10285 10286 ctxt.seid = pf->main_vsi_seid; 10287 ctxt.pf_num = pf->hw.pf_id; 10288 ctxt.vf_num = 0; 10289 ret = i40e_aq_get_vsi_params(&pf->hw, &ctxt, NULL); 10290 if (ret) { 10291 dev_info(&pf->pdev->dev, 10292 "couldn't get PF vsi config, err %pe aq_err %s\n", 10293 ERR_PTR(ret), libie_aq_str(pf->hw.aq.asq_last_status)); 10294 return; 10295 } 10296 ctxt.flags = I40E_AQ_VSI_TYPE_PF; 10297 ctxt.info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID); 10298 ctxt.info.switch_id |= cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB); 10299 10300 ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL); 10301 if (ret) { 10302 dev_info(&pf->pdev->dev, 10303 "update vsi switch failed, err %pe aq_err %s\n", 10304 ERR_PTR(ret), libie_aq_str(pf->hw.aq.asq_last_status)); 10305 } 10306 } 10307 10308 /** 10309 * i40e_disable_pf_switch_lb 10310 * @pf: pointer to the PF structure 10311 * 10312 * disable switch loop back or die - no point in a return value 10313 **/ 10314 static void i40e_disable_pf_switch_lb(struct i40e_pf *pf) 10315 { 10316 struct i40e_vsi *vsi = i40e_pf_get_main_vsi(pf); 10317 struct i40e_vsi_context ctxt; 10318 int ret; 10319 10320 ctxt.seid = pf->main_vsi_seid; 10321 ctxt.pf_num = pf->hw.pf_id; 10322 ctxt.vf_num = 0; 10323 ret = i40e_aq_get_vsi_params(&pf->hw, &ctxt, NULL); 10324 if (ret) { 10325 dev_info(&pf->pdev->dev, 10326 "couldn't get PF vsi config, err %pe aq_err %s\n", 10327 ERR_PTR(ret), libie_aq_str(pf->hw.aq.asq_last_status)); 10328 return; 10329 } 10330 ctxt.flags = I40E_AQ_VSI_TYPE_PF; 10331 ctxt.info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID); 10332 ctxt.info.switch_id &= ~cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB); 10333 10334 ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL); 10335 if (ret) { 10336 dev_info(&pf->pdev->dev, 10337 "update vsi switch failed, err %pe aq_err %s\n", 10338 ERR_PTR(ret), libie_aq_str(pf->hw.aq.asq_last_status)); 10339 } 10340 } 10341 10342 /** 10343 * i40e_config_bridge_mode - Configure the HW bridge mode 10344 * @veb: pointer to the bridge instance 10345 * 10346 * Configure the loop back mode for the LAN VSI that is downlink to the 10347 * specified HW bridge instance. It is expected this function is called 10348 * when a new HW bridge is instantiated. 10349 **/ 10350 static void i40e_config_bridge_mode(struct i40e_veb *veb) 10351 { 10352 struct i40e_pf *pf = veb->pf; 10353 10354 if (pf->hw.debug_mask & I40E_DEBUG_LAN) 10355 dev_info(&pf->pdev->dev, "enabling bridge mode: %s\n", 10356 veb->bridge_mode == BRIDGE_MODE_VEPA ? "VEPA" : "VEB"); 10357 if (veb->bridge_mode & BRIDGE_MODE_VEPA) 10358 i40e_disable_pf_switch_lb(pf); 10359 else 10360 i40e_enable_pf_switch_lb(pf); 10361 } 10362 10363 /** 10364 * i40e_reconstitute_veb - rebuild the VEB and VSIs connected to it 10365 * @veb: pointer to the VEB instance 10366 * 10367 * This is a function that builds the attached VSIs. We track the connections 10368 * through our own index numbers because the seid's from the HW could change 10369 * across the reset. 10370 **/ 10371 static int i40e_reconstitute_veb(struct i40e_veb *veb) 10372 { 10373 struct i40e_vsi *ctl_vsi = NULL; 10374 struct i40e_pf *pf = veb->pf; 10375 struct i40e_vsi *vsi; 10376 int v, ret; 10377 10378 /* As we do not maintain PV (port virtualizer) switch element then 10379 * there can be only one non-floating VEB that have uplink to MAC SEID 10380 * and its control VSI is the main one. 10381 */ 10382 if (WARN_ON(veb->uplink_seid && veb->uplink_seid != pf->mac_seid)) { 10383 dev_err(&pf->pdev->dev, 10384 "Invalid uplink SEID for VEB %d\n", veb->idx); 10385 return -ENOENT; 10386 } 10387 10388 if (veb->uplink_seid == pf->mac_seid) { 10389 /* Check that the LAN VSI has VEB owning flag set */ 10390 ctl_vsi = i40e_pf_get_main_vsi(pf); 10391 10392 if (WARN_ON(ctl_vsi->veb_idx != veb->idx || 10393 !(ctl_vsi->flags & I40E_VSI_FLAG_VEB_OWNER))) { 10394 dev_err(&pf->pdev->dev, 10395 "Invalid control VSI for VEB %d\n", veb->idx); 10396 return -ENOENT; 10397 } 10398 10399 /* Add the control VSI to switch */ 10400 ret = i40e_add_vsi(ctl_vsi); 10401 if (ret) { 10402 dev_err(&pf->pdev->dev, 10403 "Rebuild of owner VSI for VEB %d failed: %d\n", 10404 veb->idx, ret); 10405 return ret; 10406 } 10407 10408 i40e_vsi_reset_stats(ctl_vsi); 10409 } 10410 10411 /* create the VEB in the switch and move the VSI onto the VEB */ 10412 ret = i40e_add_veb(veb, ctl_vsi); 10413 if (ret) 10414 return ret; 10415 10416 if (veb->uplink_seid) { 10417 if (test_bit(I40E_FLAG_VEB_MODE_ENA, pf->flags)) 10418 veb->bridge_mode = BRIDGE_MODE_VEB; 10419 else 10420 veb->bridge_mode = BRIDGE_MODE_VEPA; 10421 i40e_config_bridge_mode(veb); 10422 } 10423 10424 /* create the remaining VSIs attached to this VEB */ 10425 i40e_pf_for_each_vsi(pf, v, vsi) { 10426 if (vsi == ctl_vsi) 10427 continue; 10428 10429 if (vsi->veb_idx == veb->idx) { 10430 vsi->uplink_seid = veb->seid; 10431 ret = i40e_add_vsi(vsi); 10432 if (ret) { 10433 dev_info(&pf->pdev->dev, 10434 "rebuild of vsi_idx %d failed: %d\n", 10435 v, ret); 10436 return ret; 10437 } 10438 i40e_vsi_reset_stats(vsi); 10439 } 10440 } 10441 10442 return ret; 10443 } 10444 10445 /** 10446 * i40e_get_capabilities - get info about the HW 10447 * @pf: the PF struct 10448 * @list_type: AQ capability to be queried 10449 **/ 10450 static int i40e_get_capabilities(struct i40e_pf *pf, 10451 enum i40e_admin_queue_opc list_type) 10452 { 10453 struct libie_aqc_list_caps_elem *cap_buf; 10454 u16 data_size; 10455 int buf_len; 10456 int err; 10457 10458 buf_len = 40 * sizeof(struct libie_aqc_list_caps_elem); 10459 do { 10460 cap_buf = kzalloc(buf_len, GFP_KERNEL); 10461 if (!cap_buf) 10462 return -ENOMEM; 10463 10464 /* this loads the data into the hw struct for us */ 10465 err = i40e_aq_discover_capabilities(&pf->hw, cap_buf, buf_len, 10466 &data_size, list_type, 10467 NULL); 10468 /* data loaded, buffer no longer needed */ 10469 kfree(cap_buf); 10470 10471 if (pf->hw.aq.asq_last_status == LIBIE_AQ_RC_ENOMEM) { 10472 /* retry with a larger buffer */ 10473 buf_len = data_size; 10474 } else if (pf->hw.aq.asq_last_status != LIBIE_AQ_RC_OK || err) { 10475 dev_info(&pf->pdev->dev, 10476 "capability discovery failed, err %pe aq_err %s\n", 10477 ERR_PTR(err), 10478 libie_aq_str(pf->hw.aq.asq_last_status)); 10479 return -ENODEV; 10480 } 10481 } while (err); 10482 10483 if (pf->hw.debug_mask & I40E_DEBUG_USER) { 10484 if (list_type == i40e_aqc_opc_list_func_capabilities) { 10485 dev_info(&pf->pdev->dev, 10486 "pf=%d, num_vfs=%d, msix_pf=%d, msix_vf=%d, fd_g=%d, fd_b=%d, pf_max_q=%d num_vsi=%d\n", 10487 pf->hw.pf_id, pf->hw.func_caps.num_vfs, 10488 pf->hw.func_caps.num_msix_vectors, 10489 pf->hw.func_caps.num_msix_vectors_vf, 10490 pf->hw.func_caps.fd_filters_guaranteed, 10491 pf->hw.func_caps.fd_filters_best_effort, 10492 pf->hw.func_caps.num_tx_qp, 10493 pf->hw.func_caps.num_vsis); 10494 } else if (list_type == i40e_aqc_opc_list_dev_capabilities) { 10495 dev_info(&pf->pdev->dev, 10496 "switch_mode=0x%04x, function_valid=0x%08x\n", 10497 pf->hw.dev_caps.switch_mode, 10498 pf->hw.dev_caps.valid_functions); 10499 dev_info(&pf->pdev->dev, 10500 "SR-IOV=%d, num_vfs for all function=%u\n", 10501 pf->hw.dev_caps.sr_iov_1_1, 10502 pf->hw.dev_caps.num_vfs); 10503 dev_info(&pf->pdev->dev, 10504 "num_vsis=%u, num_rx:%u, num_tx=%u\n", 10505 pf->hw.dev_caps.num_vsis, 10506 pf->hw.dev_caps.num_rx_qp, 10507 pf->hw.dev_caps.num_tx_qp); 10508 } 10509 } 10510 if (list_type == i40e_aqc_opc_list_func_capabilities) { 10511 #define DEF_NUM_VSI (1 + (pf->hw.func_caps.fcoe ? 1 : 0) \ 10512 + pf->hw.func_caps.num_vfs) 10513 if (pf->hw.revision_id == 0 && 10514 pf->hw.func_caps.num_vsis < DEF_NUM_VSI) { 10515 dev_info(&pf->pdev->dev, 10516 "got num_vsis %d, setting num_vsis to %d\n", 10517 pf->hw.func_caps.num_vsis, DEF_NUM_VSI); 10518 pf->hw.func_caps.num_vsis = DEF_NUM_VSI; 10519 } 10520 } 10521 return 0; 10522 } 10523 10524 static int i40e_vsi_clear(struct i40e_vsi *vsi); 10525 10526 /** 10527 * i40e_fdir_sb_setup - initialize the Flow Director resources for Sideband 10528 * @pf: board private structure 10529 **/ 10530 static void i40e_fdir_sb_setup(struct i40e_pf *pf) 10531 { 10532 struct i40e_vsi *main_vsi, *vsi; 10533 10534 /* quick workaround for an NVM issue that leaves a critical register 10535 * uninitialized 10536 */ 10537 if (!rd32(&pf->hw, I40E_GLQF_HKEY(0))) { 10538 static const u32 hkey[] = { 10539 0xe640d33f, 0xcdfe98ab, 0x73fa7161, 0x0d7a7d36, 10540 0xeacb7d61, 0xaa4f05b6, 0x9c5c89ed, 0xfc425ddb, 10541 0xa4654832, 0xfc7461d4, 0x8f827619, 0xf5c63c21, 10542 0x95b3a76d}; 10543 int i; 10544 10545 for (i = 0; i <= I40E_GLQF_HKEY_MAX_INDEX; i++) 10546 wr32(&pf->hw, I40E_GLQF_HKEY(i), hkey[i]); 10547 } 10548 10549 if (!test_bit(I40E_FLAG_FD_SB_ENA, pf->flags)) 10550 return; 10551 10552 /* find existing VSI and see if it needs configuring */ 10553 vsi = i40e_find_vsi_by_type(pf, I40E_VSI_FDIR); 10554 10555 /* create a new VSI if none exists */ 10556 if (!vsi) { 10557 main_vsi = i40e_pf_get_main_vsi(pf); 10558 vsi = i40e_vsi_setup(pf, I40E_VSI_FDIR, main_vsi->seid, 0); 10559 if (!vsi) { 10560 dev_info(&pf->pdev->dev, "Couldn't create FDir VSI\n"); 10561 clear_bit(I40E_FLAG_FD_SB_ENA, pf->flags); 10562 set_bit(I40E_FLAG_FD_SB_INACTIVE, pf->flags); 10563 return; 10564 } 10565 } 10566 10567 i40e_vsi_setup_irqhandler(vsi, i40e_fdir_clean_ring); 10568 } 10569 10570 /** 10571 * i40e_fdir_teardown - release the Flow Director resources 10572 * @pf: board private structure 10573 **/ 10574 static void i40e_fdir_teardown(struct i40e_pf *pf) 10575 { 10576 struct i40e_vsi *vsi; 10577 10578 i40e_fdir_filter_exit(pf); 10579 vsi = i40e_find_vsi_by_type(pf, I40E_VSI_FDIR); 10580 if (vsi) 10581 i40e_vsi_release(vsi); 10582 } 10583 10584 /** 10585 * i40e_rebuild_cloud_filters - Rebuilds cloud filters for VSIs 10586 * @vsi: PF main vsi 10587 * @seid: seid of main or channel VSIs 10588 * 10589 * Rebuilds cloud filters associated with main VSI and channel VSIs if they 10590 * existed before reset 10591 **/ 10592 static int i40e_rebuild_cloud_filters(struct i40e_vsi *vsi, u16 seid) 10593 { 10594 struct i40e_cloud_filter *cfilter; 10595 struct i40e_pf *pf = vsi->back; 10596 struct hlist_node *node; 10597 int ret; 10598 10599 /* Add cloud filters back if they exist */ 10600 hlist_for_each_entry_safe(cfilter, node, &pf->cloud_filter_list, 10601 cloud_node) { 10602 if (cfilter->seid != seid) 10603 continue; 10604 10605 if (cfilter->dst_port) 10606 ret = i40e_add_del_cloud_filter_big_buf(vsi, cfilter, 10607 true); 10608 else 10609 ret = i40e_add_del_cloud_filter(vsi, cfilter, true); 10610 10611 if (ret) { 10612 dev_dbg(&pf->pdev->dev, 10613 "Failed to rebuild cloud filter, err %pe aq_err %s\n", 10614 ERR_PTR(ret), 10615 libie_aq_str(pf->hw.aq.asq_last_status)); 10616 return ret; 10617 } 10618 } 10619 return 0; 10620 } 10621 10622 /** 10623 * i40e_rebuild_channels - Rebuilds channel VSIs if they existed before reset 10624 * @vsi: PF main vsi 10625 * 10626 * Rebuilds channel VSIs if they existed before reset 10627 **/ 10628 static int i40e_rebuild_channels(struct i40e_vsi *vsi) 10629 { 10630 struct i40e_channel *ch, *ch_tmp; 10631 int ret; 10632 10633 if (list_empty(&vsi->ch_list)) 10634 return 0; 10635 10636 list_for_each_entry_safe(ch, ch_tmp, &vsi->ch_list, list) { 10637 if (!ch->initialized) 10638 break; 10639 /* Proceed with creation of channel (VMDq2) VSI */ 10640 ret = i40e_add_channel(vsi->back, vsi->uplink_seid, ch); 10641 if (ret) { 10642 dev_info(&vsi->back->pdev->dev, 10643 "failed to rebuild channels using uplink_seid %u\n", 10644 vsi->uplink_seid); 10645 return ret; 10646 } 10647 /* Reconfigure TX queues using QTX_CTL register */ 10648 ret = i40e_channel_config_tx_ring(vsi->back, vsi, ch); 10649 if (ret) { 10650 dev_info(&vsi->back->pdev->dev, 10651 "failed to configure TX rings for channel %u\n", 10652 ch->seid); 10653 return ret; 10654 } 10655 /* update 'next_base_queue' */ 10656 vsi->next_base_queue = vsi->next_base_queue + 10657 ch->num_queue_pairs; 10658 if (ch->max_tx_rate) { 10659 u64 credits = ch->max_tx_rate; 10660 10661 if (i40e_set_bw_limit(vsi, ch->seid, 10662 ch->max_tx_rate)) 10663 return -EINVAL; 10664 10665 do_div(credits, I40E_BW_CREDIT_DIVISOR); 10666 dev_dbg(&vsi->back->pdev->dev, 10667 "Set tx rate of %llu Mbps (count of 50Mbps %llu) for vsi->seid %u\n", 10668 ch->max_tx_rate, 10669 credits, 10670 ch->seid); 10671 } 10672 ret = i40e_rebuild_cloud_filters(vsi, ch->seid); 10673 if (ret) { 10674 dev_dbg(&vsi->back->pdev->dev, 10675 "Failed to rebuild cloud filters for channel VSI %u\n", 10676 ch->seid); 10677 return ret; 10678 } 10679 } 10680 return 0; 10681 } 10682 10683 /** 10684 * i40e_clean_xps_state - clean xps state for every tx_ring 10685 * @vsi: ptr to the VSI 10686 **/ 10687 static void i40e_clean_xps_state(struct i40e_vsi *vsi) 10688 { 10689 int i; 10690 10691 if (vsi->tx_rings) 10692 for (i = 0; i < vsi->num_queue_pairs; i++) 10693 if (vsi->tx_rings[i]) 10694 clear_bit(__I40E_TX_XPS_INIT_DONE, 10695 vsi->tx_rings[i]->state); 10696 } 10697 10698 /** 10699 * i40e_prep_for_reset - prep for the core to reset 10700 * @pf: board private structure 10701 * 10702 * Close up the VFs and other things in prep for PF Reset. 10703 **/ 10704 static void i40e_prep_for_reset(struct i40e_pf *pf) 10705 { 10706 struct i40e_hw *hw = &pf->hw; 10707 struct i40e_vsi *vsi; 10708 int ret = 0; 10709 u32 v; 10710 10711 clear_bit(__I40E_RESET_INTR_RECEIVED, pf->state); 10712 if (test_and_set_bit(__I40E_RESET_RECOVERY_PENDING, pf->state)) 10713 return; 10714 if (i40e_check_asq_alive(&pf->hw)) 10715 i40e_vc_notify_reset(pf); 10716 10717 dev_dbg(&pf->pdev->dev, "Tearing down internal switch for reset\n"); 10718 10719 /* quiesce the VSIs and their queues that are not already DOWN */ 10720 i40e_pf_quiesce_all_vsi(pf); 10721 10722 i40e_pf_for_each_vsi(pf, v, vsi) { 10723 i40e_clean_xps_state(vsi); 10724 vsi->seid = 0; 10725 } 10726 10727 i40e_shutdown_adminq(&pf->hw); 10728 10729 /* call shutdown HMC */ 10730 if (hw->hmc.hmc_obj) { 10731 ret = i40e_shutdown_lan_hmc(hw); 10732 if (ret) 10733 dev_warn(&pf->pdev->dev, 10734 "shutdown_lan_hmc failed: %d\n", ret); 10735 } 10736 10737 /* Save the current PTP time so that we can restore the time after the 10738 * reset completes. 10739 */ 10740 i40e_ptp_save_hw_time(pf); 10741 } 10742 10743 /** 10744 * i40e_send_version - update firmware with driver version 10745 * @pf: PF struct 10746 */ 10747 static void i40e_send_version(struct i40e_pf *pf) 10748 { 10749 struct i40e_driver_version dv; 10750 10751 dv.major_version = 0xff; 10752 dv.minor_version = 0xff; 10753 dv.build_version = 0xff; 10754 dv.subbuild_version = 0; 10755 strscpy(dv.driver_string, UTS_RELEASE, sizeof(dv.driver_string)); 10756 i40e_aq_send_driver_version(&pf->hw, &dv, NULL); 10757 } 10758 10759 /** 10760 * i40e_get_oem_version - get OEM specific version information 10761 * @hw: pointer to the hardware structure 10762 **/ 10763 static void i40e_get_oem_version(struct i40e_hw *hw) 10764 { 10765 u16 block_offset = 0xffff; 10766 u16 block_length = 0; 10767 u16 capabilities = 0; 10768 u16 gen_snap = 0; 10769 u16 release = 0; 10770 10771 #define I40E_SR_NVM_OEM_VERSION_PTR 0x1B 10772 #define I40E_NVM_OEM_LENGTH_OFFSET 0x00 10773 #define I40E_NVM_OEM_CAPABILITIES_OFFSET 0x01 10774 #define I40E_NVM_OEM_GEN_OFFSET 0x02 10775 #define I40E_NVM_OEM_RELEASE_OFFSET 0x03 10776 #define I40E_NVM_OEM_CAPABILITIES_MASK 0x000F 10777 #define I40E_NVM_OEM_LENGTH 3 10778 10779 /* Check if pointer to OEM version block is valid. */ 10780 i40e_read_nvm_word(hw, I40E_SR_NVM_OEM_VERSION_PTR, &block_offset); 10781 if (block_offset == 0xffff) 10782 return; 10783 10784 /* Check if OEM version block has correct length. */ 10785 i40e_read_nvm_word(hw, block_offset + I40E_NVM_OEM_LENGTH_OFFSET, 10786 &block_length); 10787 if (block_length < I40E_NVM_OEM_LENGTH) 10788 return; 10789 10790 /* Check if OEM version format is as expected. */ 10791 i40e_read_nvm_word(hw, block_offset + I40E_NVM_OEM_CAPABILITIES_OFFSET, 10792 &capabilities); 10793 if ((capabilities & I40E_NVM_OEM_CAPABILITIES_MASK) != 0) 10794 return; 10795 10796 i40e_read_nvm_word(hw, block_offset + I40E_NVM_OEM_GEN_OFFSET, 10797 &gen_snap); 10798 i40e_read_nvm_word(hw, block_offset + I40E_NVM_OEM_RELEASE_OFFSET, 10799 &release); 10800 hw->nvm.oem_ver = 10801 FIELD_PREP(I40E_OEM_GEN_MASK | I40E_OEM_SNAP_MASK, gen_snap) | 10802 FIELD_PREP(I40E_OEM_RELEASE_MASK, release); 10803 hw->nvm.eetrack = I40E_OEM_EETRACK_ID; 10804 } 10805 10806 /** 10807 * i40e_reset - wait for core reset to finish reset, reset pf if corer not seen 10808 * @pf: board private structure 10809 **/ 10810 static int i40e_reset(struct i40e_pf *pf) 10811 { 10812 struct i40e_hw *hw = &pf->hw; 10813 int ret; 10814 10815 ret = i40e_pf_reset(hw); 10816 if (ret) { 10817 dev_info(&pf->pdev->dev, "PF reset failed, %d\n", ret); 10818 set_bit(__I40E_RESET_FAILED, pf->state); 10819 clear_bit(__I40E_RESET_RECOVERY_PENDING, pf->state); 10820 } else { 10821 pf->pfr_count++; 10822 } 10823 return ret; 10824 } 10825 10826 /** 10827 * i40e_rebuild - rebuild using a saved config 10828 * @pf: board private structure 10829 * @reinit: if the Main VSI needs to re-initialized. 10830 * @lock_acquired: indicates whether or not the lock has been acquired 10831 * before this function was called. 10832 **/ 10833 static void i40e_rebuild(struct i40e_pf *pf, bool reinit, bool lock_acquired) 10834 { 10835 const bool is_recovery_mode_reported = i40e_check_recovery_mode(pf); 10836 struct i40e_vsi *vsi = i40e_pf_get_main_vsi(pf); 10837 struct i40e_hw *hw = &pf->hw; 10838 struct i40e_veb *veb; 10839 int ret; 10840 u32 val; 10841 int v; 10842 10843 if (test_bit(__I40E_EMP_RESET_INTR_RECEIVED, pf->state) && 10844 is_recovery_mode_reported) 10845 i40e_set_ethtool_ops(vsi->netdev); 10846 10847 if (test_bit(__I40E_DOWN, pf->state) && 10848 !test_bit(__I40E_RECOVERY_MODE, pf->state)) 10849 goto clear_recovery; 10850 dev_dbg(&pf->pdev->dev, "Rebuilding internal switch\n"); 10851 10852 /* rebuild the basics for the AdminQ, HMC, and initial HW switch */ 10853 ret = i40e_init_adminq(&pf->hw); 10854 if (ret) { 10855 dev_info(&pf->pdev->dev, "Rebuild AdminQ failed, err %pe aq_err %s\n", 10856 ERR_PTR(ret), libie_aq_str(pf->hw.aq.asq_last_status)); 10857 goto clear_recovery; 10858 } 10859 i40e_get_oem_version(&pf->hw); 10860 10861 if (test_and_clear_bit(__I40E_EMP_RESET_INTR_RECEIVED, pf->state)) { 10862 /* The following delay is necessary for firmware update. */ 10863 mdelay(1000); 10864 } 10865 10866 /* re-verify the eeprom if we just had an EMP reset */ 10867 if (test_and_clear_bit(__I40E_EMP_RESET_INTR_RECEIVED, pf->state)) 10868 i40e_verify_eeprom(pf); 10869 10870 /* if we are going out of or into recovery mode we have to act 10871 * accordingly with regard to resources initialization 10872 * and deinitialization 10873 */ 10874 if (test_bit(__I40E_RECOVERY_MODE, pf->state)) { 10875 if (i40e_get_capabilities(pf, 10876 i40e_aqc_opc_list_func_capabilities)) 10877 goto end_unlock; 10878 10879 if (is_recovery_mode_reported) { 10880 /* we're staying in recovery mode so we'll reinitialize 10881 * misc vector here 10882 */ 10883 if (i40e_setup_misc_vector_for_recovery_mode(pf)) 10884 goto end_unlock; 10885 } else { 10886 if (!lock_acquired) 10887 rtnl_lock(); 10888 /* we're going out of recovery mode so we'll free 10889 * the IRQ allocated specifically for recovery mode 10890 * and restore the interrupt scheme 10891 */ 10892 free_irq(pf->pdev->irq, pf); 10893 i40e_clear_interrupt_scheme(pf); 10894 if (i40e_restore_interrupt_scheme(pf)) 10895 goto end_unlock; 10896 } 10897 10898 /* tell the firmware that we're starting */ 10899 i40e_send_version(pf); 10900 10901 /* bail out in case recovery mode was detected, as there is 10902 * no need for further configuration. 10903 */ 10904 goto end_unlock; 10905 } 10906 10907 i40e_clear_pxe_mode(hw); 10908 ret = i40e_get_capabilities(pf, i40e_aqc_opc_list_func_capabilities); 10909 if (ret) 10910 goto end_core_reset; 10911 10912 ret = i40e_init_lan_hmc(hw, hw->func_caps.num_tx_qp, 10913 hw->func_caps.num_rx_qp, 0, 0); 10914 if (ret) { 10915 dev_info(&pf->pdev->dev, "init_lan_hmc failed: %d\n", ret); 10916 goto end_core_reset; 10917 } 10918 ret = i40e_configure_lan_hmc(hw, I40E_HMC_MODEL_DIRECT_ONLY); 10919 if (ret) { 10920 dev_info(&pf->pdev->dev, "configure_lan_hmc failed: %d\n", ret); 10921 goto end_core_reset; 10922 } 10923 10924 #ifdef CONFIG_I40E_DCB 10925 /* Enable FW to write a default DCB config on link-up 10926 * unless I40E_FLAG_TC_MQPRIO was enabled or DCB 10927 * is not supported with new link speed 10928 */ 10929 if (i40e_is_tc_mqprio_enabled(pf)) { 10930 i40e_aq_set_dcb_parameters(hw, false, NULL); 10931 } else { 10932 if (I40E_IS_X710TL_DEVICE(hw->device_id) && 10933 (hw->phy.link_info.link_speed & 10934 (I40E_LINK_SPEED_2_5GB | I40E_LINK_SPEED_5GB))) { 10935 i40e_aq_set_dcb_parameters(hw, false, NULL); 10936 dev_warn(&pf->pdev->dev, 10937 "DCB is not supported for X710-T*L 2.5/5G speeds\n"); 10938 clear_bit(I40E_FLAG_DCB_CAPABLE, pf->flags); 10939 } else { 10940 i40e_aq_set_dcb_parameters(hw, true, NULL); 10941 ret = i40e_init_pf_dcb(pf); 10942 if (ret) { 10943 dev_info(&pf->pdev->dev, "DCB init failed %d, disabled\n", 10944 ret); 10945 clear_bit(I40E_FLAG_DCB_CAPABLE, pf->flags); 10946 /* Continue without DCB enabled */ 10947 } 10948 } 10949 } 10950 10951 #endif /* CONFIG_I40E_DCB */ 10952 if (!lock_acquired) 10953 rtnl_lock(); 10954 ret = i40e_setup_pf_switch(pf, reinit, true); 10955 if (ret) 10956 goto end_unlock; 10957 10958 /* The driver only wants link up/down and module qualification 10959 * reports from firmware. Note the negative logic. 10960 */ 10961 ret = i40e_aq_set_phy_int_mask(&pf->hw, 10962 ~(I40E_AQ_EVENT_LINK_UPDOWN | 10963 I40E_AQ_EVENT_MEDIA_NA | 10964 I40E_AQ_EVENT_MODULE_QUAL_FAIL), NULL); 10965 if (ret) 10966 dev_info(&pf->pdev->dev, "set phy mask fail, err %pe aq_err %s\n", 10967 ERR_PTR(ret), libie_aq_str(pf->hw.aq.asq_last_status)); 10968 10969 /* Rebuild the VSIs and VEBs that existed before reset. 10970 * They are still in our local switch element arrays, so only 10971 * need to rebuild the switch model in the HW. 10972 * 10973 * If there were VEBs but the reconstitution failed, we'll try 10974 * to recover minimal use by getting the basic PF VSI working. 10975 */ 10976 if (vsi->uplink_seid != pf->mac_seid) { 10977 dev_dbg(&pf->pdev->dev, "attempting to rebuild switch\n"); 10978 10979 /* Rebuild VEBs */ 10980 i40e_pf_for_each_veb(pf, v, veb) { 10981 ret = i40e_reconstitute_veb(veb); 10982 if (!ret) 10983 continue; 10984 10985 /* If Main VEB failed, we're in deep doodoo, 10986 * so give up rebuilding the switch and set up 10987 * for minimal rebuild of PF VSI. 10988 * If orphan failed, we'll report the error 10989 * but try to keep going. 10990 */ 10991 if (veb->uplink_seid == pf->mac_seid) { 10992 dev_info(&pf->pdev->dev, 10993 "rebuild of switch failed: %d, will try to set up simple PF connection\n", 10994 ret); 10995 vsi->uplink_seid = pf->mac_seid; 10996 break; 10997 } else if (veb->uplink_seid == 0) { 10998 dev_info(&pf->pdev->dev, 10999 "rebuild of orphan VEB failed: %d\n", 11000 ret); 11001 } 11002 } 11003 } 11004 11005 if (vsi->uplink_seid == pf->mac_seid) { 11006 dev_dbg(&pf->pdev->dev, "attempting to rebuild PF VSI\n"); 11007 /* no VEB, so rebuild only the Main VSI */ 11008 ret = i40e_add_vsi(vsi); 11009 if (ret) { 11010 dev_info(&pf->pdev->dev, 11011 "rebuild of Main VSI failed: %d\n", ret); 11012 goto end_unlock; 11013 } 11014 } 11015 11016 if (vsi->mqprio_qopt.max_rate[0]) { 11017 u64 max_tx_rate = i40e_bw_bytes_to_mbits(vsi, 11018 vsi->mqprio_qopt.max_rate[0]); 11019 u64 credits = 0; 11020 11021 ret = i40e_set_bw_limit(vsi, vsi->seid, max_tx_rate); 11022 if (ret) 11023 goto end_unlock; 11024 11025 credits = max_tx_rate; 11026 do_div(credits, I40E_BW_CREDIT_DIVISOR); 11027 dev_dbg(&vsi->back->pdev->dev, 11028 "Set tx rate of %llu Mbps (count of 50Mbps %llu) for vsi->seid %u\n", 11029 max_tx_rate, 11030 credits, 11031 vsi->seid); 11032 } 11033 11034 ret = i40e_rebuild_cloud_filters(vsi, vsi->seid); 11035 if (ret) 11036 goto end_unlock; 11037 11038 /* PF Main VSI is rebuild by now, go ahead and rebuild channel VSIs 11039 * for this main VSI if they exist 11040 */ 11041 ret = i40e_rebuild_channels(vsi); 11042 if (ret) 11043 goto end_unlock; 11044 11045 /* Reconfigure hardware for allowing smaller MSS in the case 11046 * of TSO, so that we avoid the MDD being fired and causing 11047 * a reset in the case of small MSS+TSO. 11048 */ 11049 #define I40E_REG_MSS 0x000E64DC 11050 #define I40E_REG_MSS_MIN_MASK 0x3FF0000 11051 #define I40E_64BYTE_MSS 0x400000 11052 val = rd32(hw, I40E_REG_MSS); 11053 if ((val & I40E_REG_MSS_MIN_MASK) > I40E_64BYTE_MSS) { 11054 val &= ~I40E_REG_MSS_MIN_MASK; 11055 val |= I40E_64BYTE_MSS; 11056 wr32(hw, I40E_REG_MSS, val); 11057 } 11058 11059 if (test_bit(I40E_HW_CAP_RESTART_AUTONEG, pf->hw.caps)) { 11060 msleep(75); 11061 ret = i40e_aq_set_link_restart_an(&pf->hw, true, NULL); 11062 if (ret) 11063 dev_info(&pf->pdev->dev, "link restart failed, err %pe aq_err %s\n", 11064 ERR_PTR(ret), 11065 libie_aq_str(pf->hw.aq.asq_last_status)); 11066 } 11067 /* reinit the misc interrupt */ 11068 if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags)) { 11069 ret = i40e_setup_misc_vector(pf); 11070 if (ret) 11071 goto end_unlock; 11072 } 11073 11074 /* Add a filter to drop all Flow control frames from any VSI from being 11075 * transmitted. By doing so we stop a malicious VF from sending out 11076 * PAUSE or PFC frames and potentially controlling traffic for other 11077 * PF/VF VSIs. 11078 * The FW can still send Flow control frames if enabled. 11079 */ 11080 i40e_add_filter_to_drop_tx_flow_control_frames(&pf->hw, 11081 pf->main_vsi_seid); 11082 11083 /* restart the VSIs that were rebuilt and running before the reset */ 11084 i40e_pf_unquiesce_all_vsi(pf); 11085 11086 /* Release the RTNL lock before we start resetting VFs */ 11087 if (!lock_acquired) 11088 rtnl_unlock(); 11089 11090 /* Restore promiscuous settings */ 11091 ret = i40e_set_promiscuous(pf, pf->cur_promisc); 11092 if (ret) 11093 dev_warn(&pf->pdev->dev, 11094 "Failed to restore promiscuous setting: %s, err %pe aq_err %s\n", 11095 pf->cur_promisc ? "on" : "off", 11096 ERR_PTR(ret), libie_aq_str(pf->hw.aq.asq_last_status)); 11097 11098 i40e_reset_all_vfs(pf, true); 11099 11100 /* tell the firmware that we're starting */ 11101 i40e_send_version(pf); 11102 11103 /* We've already released the lock, so don't do it again */ 11104 goto end_core_reset; 11105 11106 end_unlock: 11107 if (!lock_acquired) 11108 rtnl_unlock(); 11109 end_core_reset: 11110 clear_bit(__I40E_RESET_FAILED, pf->state); 11111 clear_recovery: 11112 clear_bit(__I40E_RESET_RECOVERY_PENDING, pf->state); 11113 clear_bit(__I40E_TIMEOUT_RECOVERY_PENDING, pf->state); 11114 } 11115 11116 /** 11117 * i40e_reset_and_rebuild - reset and rebuild using a saved config 11118 * @pf: board private structure 11119 * @reinit: if the Main VSI needs to re-initialized. 11120 * @lock_acquired: indicates whether or not the lock has been acquired 11121 * before this function was called. 11122 **/ 11123 static void i40e_reset_and_rebuild(struct i40e_pf *pf, bool reinit, 11124 bool lock_acquired) 11125 { 11126 int ret; 11127 11128 if (test_bit(__I40E_IN_REMOVE, pf->state)) 11129 return; 11130 /* Now we wait for GRST to settle out. 11131 * We don't have to delete the VEBs or VSIs from the hw switch 11132 * because the reset will make them disappear. 11133 */ 11134 ret = i40e_reset(pf); 11135 if (!ret) 11136 i40e_rebuild(pf, reinit, lock_acquired); 11137 else 11138 dev_err(&pf->pdev->dev, "%s: i40e_reset() FAILED", __func__); 11139 } 11140 11141 /** 11142 * i40e_handle_reset_warning - prep for the PF to reset, reset and rebuild 11143 * @pf: board private structure 11144 * 11145 * Close up the VFs and other things in prep for a Core Reset, 11146 * then get ready to rebuild the world. 11147 * @lock_acquired: indicates whether or not the lock has been acquired 11148 * before this function was called. 11149 **/ 11150 static void i40e_handle_reset_warning(struct i40e_pf *pf, bool lock_acquired) 11151 { 11152 i40e_prep_for_reset(pf); 11153 i40e_reset_and_rebuild(pf, false, lock_acquired); 11154 } 11155 11156 /** 11157 * i40e_print_vf_mdd_event - print VF Tx/Rx malicious driver detect event 11158 * @pf: board private structure 11159 * @vf: pointer to the VF structure 11160 * @is_tx: true - for Tx event, false - for Rx 11161 */ 11162 static void i40e_print_vf_mdd_event(struct i40e_pf *pf, struct i40e_vf *vf, 11163 bool is_tx) 11164 { 11165 dev_err(&pf->pdev->dev, is_tx ? 11166 "%lld Tx Malicious Driver Detection events detected on PF %d VF %d MAC %pm. mdd-auto-reset-vfs=%s\n" : 11167 "%lld Rx Malicious Driver Detection events detected on PF %d VF %d MAC %pm. mdd-auto-reset-vfs=%s\n", 11168 is_tx ? vf->mdd_tx_events.count : vf->mdd_rx_events.count, 11169 pf->hw.pf_id, 11170 vf->vf_id, 11171 vf->default_lan_addr.addr, 11172 str_on_off(test_bit(I40E_FLAG_MDD_AUTO_RESET_VF, pf->flags))); 11173 } 11174 11175 /** 11176 * i40e_print_vfs_mdd_events - print VFs malicious driver detect event 11177 * @pf: pointer to the PF structure 11178 * 11179 * Called from i40e_handle_mdd_event to rate limit and print VFs MDD events. 11180 */ 11181 static void i40e_print_vfs_mdd_events(struct i40e_pf *pf) 11182 { 11183 unsigned int i; 11184 11185 /* check that there are pending MDD events to print */ 11186 if (!test_and_clear_bit(__I40E_MDD_VF_PRINT_PENDING, pf->state)) 11187 return; 11188 11189 if (!__ratelimit(&pf->mdd_message_rate_limit)) 11190 return; 11191 11192 for (i = 0; i < pf->num_alloc_vfs; i++) { 11193 struct i40e_vf *vf = &pf->vf[i]; 11194 bool is_printed = false; 11195 11196 /* only print Rx MDD event message if there are new events */ 11197 if (vf->mdd_rx_events.count != vf->mdd_rx_events.last_printed) { 11198 vf->mdd_rx_events.last_printed = vf->mdd_rx_events.count; 11199 i40e_print_vf_mdd_event(pf, vf, false); 11200 is_printed = true; 11201 } 11202 11203 /* only print Tx MDD event message if there are new events */ 11204 if (vf->mdd_tx_events.count != vf->mdd_tx_events.last_printed) { 11205 vf->mdd_tx_events.last_printed = vf->mdd_tx_events.count; 11206 i40e_print_vf_mdd_event(pf, vf, true); 11207 is_printed = true; 11208 } 11209 11210 if (is_printed && !test_bit(I40E_FLAG_MDD_AUTO_RESET_VF, pf->flags)) 11211 dev_info(&pf->pdev->dev, 11212 "Use PF Control I/F to re-enable the VF #%d\n", 11213 i); 11214 } 11215 } 11216 11217 /** 11218 * i40e_handle_mdd_event 11219 * @pf: pointer to the PF structure 11220 * 11221 * Called from the MDD irq handler to identify possibly malicious vfs 11222 **/ 11223 static void i40e_handle_mdd_event(struct i40e_pf *pf) 11224 { 11225 struct i40e_hw *hw = &pf->hw; 11226 bool mdd_detected = false; 11227 struct i40e_vf *vf; 11228 u32 reg; 11229 int i; 11230 11231 if (!test_and_clear_bit(__I40E_MDD_EVENT_PENDING, pf->state)) { 11232 /* Since the VF MDD event logging is rate limited, check if 11233 * there are pending MDD events. 11234 */ 11235 i40e_print_vfs_mdd_events(pf); 11236 return; 11237 } 11238 11239 /* find what triggered the MDD event */ 11240 reg = rd32(hw, I40E_GL_MDET_TX); 11241 if (reg & I40E_GL_MDET_TX_VALID_MASK) { 11242 u8 pf_num = FIELD_GET(I40E_GL_MDET_TX_PF_NUM_MASK, reg); 11243 u16 vf_num = FIELD_GET(I40E_GL_MDET_TX_VF_NUM_MASK, reg); 11244 u8 event = FIELD_GET(I40E_GL_MDET_TX_EVENT_MASK, reg); 11245 u16 queue = FIELD_GET(I40E_GL_MDET_TX_QUEUE_MASK, reg) - 11246 pf->hw.func_caps.base_queue; 11247 if (netif_msg_tx_err(pf)) 11248 dev_info(&pf->pdev->dev, "Malicious Driver Detection event 0x%02x on TX queue %d PF number 0x%02x VF number 0x%02x\n", 11249 event, queue, pf_num, vf_num); 11250 wr32(hw, I40E_GL_MDET_TX, 0xffffffff); 11251 mdd_detected = true; 11252 } 11253 reg = rd32(hw, I40E_GL_MDET_RX); 11254 if (reg & I40E_GL_MDET_RX_VALID_MASK) { 11255 u8 func = FIELD_GET(I40E_GL_MDET_RX_FUNCTION_MASK, reg); 11256 u8 event = FIELD_GET(I40E_GL_MDET_RX_EVENT_MASK, reg); 11257 u16 queue = FIELD_GET(I40E_GL_MDET_RX_QUEUE_MASK, reg) - 11258 pf->hw.func_caps.base_queue; 11259 if (netif_msg_rx_err(pf)) 11260 dev_info(&pf->pdev->dev, "Malicious Driver Detection event 0x%02x on RX queue %d of function 0x%02x\n", 11261 event, queue, func); 11262 wr32(hw, I40E_GL_MDET_RX, 0xffffffff); 11263 mdd_detected = true; 11264 } 11265 11266 if (mdd_detected) { 11267 reg = rd32(hw, I40E_PF_MDET_TX); 11268 if (reg & I40E_PF_MDET_TX_VALID_MASK) { 11269 wr32(hw, I40E_PF_MDET_TX, 0xFFFF); 11270 dev_dbg(&pf->pdev->dev, "TX driver issue detected on PF\n"); 11271 } 11272 reg = rd32(hw, I40E_PF_MDET_RX); 11273 if (reg & I40E_PF_MDET_RX_VALID_MASK) { 11274 wr32(hw, I40E_PF_MDET_RX, 0xFFFF); 11275 dev_dbg(&pf->pdev->dev, "RX driver issue detected on PF\n"); 11276 } 11277 } 11278 11279 /* see if one of the VFs needs its hand slapped */ 11280 for (i = 0; i < pf->num_alloc_vfs && mdd_detected; i++) { 11281 bool is_mdd_on_tx = false; 11282 bool is_mdd_on_rx = false; 11283 11284 vf = &(pf->vf[i]); 11285 reg = rd32(hw, I40E_VP_MDET_TX(i)); 11286 if (reg & I40E_VP_MDET_TX_VALID_MASK) { 11287 set_bit(__I40E_MDD_VF_PRINT_PENDING, pf->state); 11288 wr32(hw, I40E_VP_MDET_TX(i), 0xFFFF); 11289 vf->mdd_tx_events.count++; 11290 set_bit(I40E_VF_STATE_DISABLED, &vf->vf_states); 11291 is_mdd_on_tx = true; 11292 } 11293 11294 reg = rd32(hw, I40E_VP_MDET_RX(i)); 11295 if (reg & I40E_VP_MDET_RX_VALID_MASK) { 11296 set_bit(__I40E_MDD_VF_PRINT_PENDING, pf->state); 11297 wr32(hw, I40E_VP_MDET_RX(i), 0xFFFF); 11298 vf->mdd_rx_events.count++; 11299 set_bit(I40E_VF_STATE_DISABLED, &vf->vf_states); 11300 is_mdd_on_rx = true; 11301 } 11302 11303 if ((is_mdd_on_tx || is_mdd_on_rx) && 11304 test_bit(I40E_FLAG_MDD_AUTO_RESET_VF, pf->flags)) { 11305 /* VF MDD event counters will be cleared by 11306 * reset, so print the event prior to reset. 11307 */ 11308 if (is_mdd_on_rx) 11309 i40e_print_vf_mdd_event(pf, vf, false); 11310 if (is_mdd_on_tx) 11311 i40e_print_vf_mdd_event(pf, vf, true); 11312 11313 i40e_vc_reset_vf(vf, true); 11314 } 11315 } 11316 11317 reg = rd32(hw, I40E_PFINT_ICR0_ENA); 11318 reg |= I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK; 11319 wr32(hw, I40E_PFINT_ICR0_ENA, reg); 11320 i40e_flush(hw); 11321 11322 i40e_print_vfs_mdd_events(pf); 11323 } 11324 11325 /** 11326 * i40e_service_task - Run the driver's async subtasks 11327 * @work: pointer to work_struct containing our data 11328 **/ 11329 static void i40e_service_task(struct work_struct *work) 11330 { 11331 struct i40e_pf *pf = container_of(work, 11332 struct i40e_pf, 11333 service_task); 11334 unsigned long start_time = jiffies; 11335 11336 /* don't bother with service tasks if a reset is in progress */ 11337 if (test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state) || 11338 test_bit(__I40E_SUSPENDED, pf->state)) 11339 return; 11340 11341 if (test_and_set_bit(__I40E_SERVICE_SCHED, pf->state)) 11342 return; 11343 11344 if (!test_bit(__I40E_RECOVERY_MODE, pf->state)) { 11345 i40e_detect_recover_hung(pf); 11346 i40e_sync_filters_subtask(pf); 11347 i40e_reset_subtask(pf); 11348 i40e_handle_mdd_event(pf); 11349 i40e_vc_process_vflr_event(pf); 11350 i40e_watchdog_subtask(pf); 11351 i40e_fdir_reinit_subtask(pf); 11352 if (test_and_clear_bit(__I40E_CLIENT_RESET, pf->state)) { 11353 /* Client subtask will reopen next time through. */ 11354 i40e_notify_client_of_netdev_close(pf, true); 11355 } else { 11356 i40e_client_subtask(pf); 11357 if (test_and_clear_bit(__I40E_CLIENT_L2_CHANGE, 11358 pf->state)) 11359 i40e_notify_client_of_l2_param_changes(pf); 11360 } 11361 i40e_sync_filters_subtask(pf); 11362 } else { 11363 i40e_reset_subtask(pf); 11364 } 11365 11366 i40e_clean_adminq_subtask(pf); 11367 11368 /* flush memory to make sure state is correct before next watchdog */ 11369 smp_mb__before_atomic(); 11370 clear_bit(__I40E_SERVICE_SCHED, pf->state); 11371 11372 /* If the tasks have taken longer than one timer cycle or there 11373 * is more work to be done, reschedule the service task now 11374 * rather than wait for the timer to tick again. 11375 */ 11376 if (time_after(jiffies, (start_time + pf->service_timer_period)) || 11377 test_bit(__I40E_ADMINQ_EVENT_PENDING, pf->state) || 11378 test_bit(__I40E_MDD_EVENT_PENDING, pf->state) || 11379 test_bit(__I40E_VFLR_EVENT_PENDING, pf->state)) 11380 i40e_service_event_schedule(pf); 11381 } 11382 11383 /** 11384 * i40e_service_timer - timer callback 11385 * @t: timer list pointer 11386 **/ 11387 static void i40e_service_timer(struct timer_list *t) 11388 { 11389 struct i40e_pf *pf = timer_container_of(pf, t, service_timer); 11390 11391 mod_timer(&pf->service_timer, 11392 round_jiffies(jiffies + pf->service_timer_period)); 11393 i40e_service_event_schedule(pf); 11394 } 11395 11396 /** 11397 * i40e_set_num_rings_in_vsi - Determine number of rings in the VSI 11398 * @vsi: the VSI being configured 11399 **/ 11400 static int i40e_set_num_rings_in_vsi(struct i40e_vsi *vsi) 11401 { 11402 struct i40e_pf *pf = vsi->back; 11403 11404 switch (vsi->type) { 11405 case I40E_VSI_MAIN: 11406 vsi->alloc_queue_pairs = pf->num_lan_qps; 11407 if (!vsi->num_tx_desc) 11408 vsi->num_tx_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS, 11409 I40E_REQ_DESCRIPTOR_MULTIPLE); 11410 if (!vsi->num_rx_desc) 11411 vsi->num_rx_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS, 11412 I40E_REQ_DESCRIPTOR_MULTIPLE); 11413 if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags)) 11414 vsi->num_q_vectors = pf->num_lan_msix; 11415 else 11416 vsi->num_q_vectors = 1; 11417 11418 break; 11419 11420 case I40E_VSI_FDIR: 11421 vsi->alloc_queue_pairs = 1; 11422 vsi->num_tx_desc = ALIGN(I40E_FDIR_RING_COUNT, 11423 I40E_REQ_DESCRIPTOR_MULTIPLE); 11424 vsi->num_rx_desc = ALIGN(I40E_FDIR_RING_COUNT, 11425 I40E_REQ_DESCRIPTOR_MULTIPLE); 11426 vsi->num_q_vectors = pf->num_fdsb_msix; 11427 break; 11428 11429 case I40E_VSI_VMDQ2: 11430 vsi->alloc_queue_pairs = pf->num_vmdq_qps; 11431 if (!vsi->num_tx_desc) 11432 vsi->num_tx_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS, 11433 I40E_REQ_DESCRIPTOR_MULTIPLE); 11434 if (!vsi->num_rx_desc) 11435 vsi->num_rx_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS, 11436 I40E_REQ_DESCRIPTOR_MULTIPLE); 11437 vsi->num_q_vectors = pf->num_vmdq_msix; 11438 break; 11439 11440 case I40E_VSI_SRIOV: 11441 vsi->alloc_queue_pairs = pf->num_vf_qps; 11442 if (!vsi->num_tx_desc) 11443 vsi->num_tx_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS, 11444 I40E_REQ_DESCRIPTOR_MULTIPLE); 11445 if (!vsi->num_rx_desc) 11446 vsi->num_rx_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS, 11447 I40E_REQ_DESCRIPTOR_MULTIPLE); 11448 break; 11449 11450 default: 11451 WARN_ON(1); 11452 return -ENODATA; 11453 } 11454 11455 if (is_kdump_kernel()) { 11456 vsi->num_tx_desc = I40E_MIN_NUM_DESCRIPTORS; 11457 vsi->num_rx_desc = I40E_MIN_NUM_DESCRIPTORS; 11458 } 11459 11460 return 0; 11461 } 11462 11463 /** 11464 * i40e_vsi_alloc_arrays - Allocate queue and vector pointer arrays for the vsi 11465 * @vsi: VSI pointer 11466 * @alloc_qvectors: a bool to specify if q_vectors need to be allocated. 11467 * 11468 * On error: returns error code (negative) 11469 * On success: returns 0 11470 **/ 11471 static int i40e_vsi_alloc_arrays(struct i40e_vsi *vsi, bool alloc_qvectors) 11472 { 11473 struct i40e_ring **next_rings; 11474 int size; 11475 int ret = 0; 11476 11477 /* allocate memory for both Tx, XDP Tx and Rx ring pointers */ 11478 size = sizeof(struct i40e_ring *) * vsi->alloc_queue_pairs * 11479 (i40e_enabled_xdp_vsi(vsi) ? 3 : 2); 11480 vsi->tx_rings = kzalloc(size, GFP_KERNEL); 11481 if (!vsi->tx_rings) 11482 return -ENOMEM; 11483 next_rings = vsi->tx_rings + vsi->alloc_queue_pairs; 11484 if (i40e_enabled_xdp_vsi(vsi)) { 11485 vsi->xdp_rings = next_rings; 11486 next_rings += vsi->alloc_queue_pairs; 11487 } 11488 vsi->rx_rings = next_rings; 11489 11490 if (alloc_qvectors) { 11491 /* allocate memory for q_vector pointers */ 11492 size = sizeof(struct i40e_q_vector *) * vsi->num_q_vectors; 11493 vsi->q_vectors = kzalloc(size, GFP_KERNEL); 11494 if (!vsi->q_vectors) { 11495 ret = -ENOMEM; 11496 goto err_vectors; 11497 } 11498 } 11499 return ret; 11500 11501 err_vectors: 11502 kfree(vsi->tx_rings); 11503 return ret; 11504 } 11505 11506 /** 11507 * i40e_vsi_mem_alloc - Allocates the next available struct vsi in the PF 11508 * @pf: board private structure 11509 * @type: type of VSI 11510 * 11511 * On error: returns error code (negative) 11512 * On success: returns vsi index in PF (positive) 11513 **/ 11514 static int i40e_vsi_mem_alloc(struct i40e_pf *pf, enum i40e_vsi_type type) 11515 { 11516 int ret = -ENODEV; 11517 struct i40e_vsi *vsi; 11518 int vsi_idx; 11519 int i; 11520 11521 /* Need to protect the allocation of the VSIs at the PF level */ 11522 mutex_lock(&pf->switch_mutex); 11523 11524 /* VSI list may be fragmented if VSI creation/destruction has 11525 * been happening. We can afford to do a quick scan to look 11526 * for any free VSIs in the list. 11527 * 11528 * find next empty vsi slot, looping back around if necessary 11529 */ 11530 i = pf->next_vsi; 11531 while (i < pf->num_alloc_vsi && pf->vsi[i]) 11532 i++; 11533 if (i >= pf->num_alloc_vsi) { 11534 i = 0; 11535 while (i < pf->next_vsi && pf->vsi[i]) 11536 i++; 11537 } 11538 11539 if (i < pf->num_alloc_vsi && !pf->vsi[i]) { 11540 vsi_idx = i; /* Found one! */ 11541 } else { 11542 ret = -ENODEV; 11543 goto unlock_pf; /* out of VSI slots! */ 11544 } 11545 pf->next_vsi = ++i; 11546 11547 vsi = kzalloc_obj(*vsi); 11548 if (!vsi) { 11549 ret = -ENOMEM; 11550 goto unlock_pf; 11551 } 11552 vsi->type = type; 11553 vsi->back = pf; 11554 set_bit(__I40E_VSI_DOWN, vsi->state); 11555 vsi->flags = 0; 11556 vsi->idx = vsi_idx; 11557 vsi->int_rate_limit = 0; 11558 vsi->rss_table_size = (vsi->type == I40E_VSI_MAIN) ? 11559 pf->rss_table_size : 64; 11560 vsi->netdev_registered = false; 11561 vsi->work_limit = I40E_DEFAULT_IRQ_WORK; 11562 hash_init(vsi->mac_filter_hash); 11563 vsi->irqs_ready = false; 11564 11565 if (type == I40E_VSI_MAIN) { 11566 vsi->af_xdp_zc_qps = bitmap_zalloc(pf->num_lan_qps, GFP_KERNEL); 11567 if (!vsi->af_xdp_zc_qps) 11568 goto err_rings; 11569 } 11570 11571 ret = i40e_set_num_rings_in_vsi(vsi); 11572 if (ret) 11573 goto err_rings; 11574 11575 ret = i40e_vsi_alloc_arrays(vsi, true); 11576 if (ret) 11577 goto err_rings; 11578 11579 /* Setup default MSIX irq handler for VSI */ 11580 i40e_vsi_setup_irqhandler(vsi, i40e_msix_clean_rings); 11581 11582 /* Initialize VSI lock */ 11583 spin_lock_init(&vsi->mac_filter_hash_lock); 11584 pf->vsi[vsi_idx] = vsi; 11585 ret = vsi_idx; 11586 goto unlock_pf; 11587 11588 err_rings: 11589 bitmap_free(vsi->af_xdp_zc_qps); 11590 pf->next_vsi = i - 1; 11591 kfree(vsi); 11592 unlock_pf: 11593 mutex_unlock(&pf->switch_mutex); 11594 return ret; 11595 } 11596 11597 /** 11598 * i40e_vsi_free_arrays - Free queue and vector pointer arrays for the VSI 11599 * @vsi: VSI pointer 11600 * @free_qvectors: a bool to specify if q_vectors need to be freed. 11601 * 11602 * On error: returns error code (negative) 11603 * On success: returns 0 11604 **/ 11605 static void i40e_vsi_free_arrays(struct i40e_vsi *vsi, bool free_qvectors) 11606 { 11607 /* free the ring and vector containers */ 11608 if (free_qvectors) { 11609 kfree(vsi->q_vectors); 11610 vsi->q_vectors = NULL; 11611 } 11612 kfree(vsi->tx_rings); 11613 vsi->tx_rings = NULL; 11614 vsi->rx_rings = NULL; 11615 vsi->xdp_rings = NULL; 11616 } 11617 11618 /** 11619 * i40e_clear_rss_config_user - clear the user configured RSS hash keys 11620 * and lookup table 11621 * @vsi: Pointer to VSI structure 11622 */ 11623 static void i40e_clear_rss_config_user(struct i40e_vsi *vsi) 11624 { 11625 if (!vsi) 11626 return; 11627 11628 kfree(vsi->rss_hkey_user); 11629 vsi->rss_hkey_user = NULL; 11630 11631 kfree(vsi->rss_lut_user); 11632 vsi->rss_lut_user = NULL; 11633 } 11634 11635 /** 11636 * i40e_vsi_clear - Deallocate the VSI provided 11637 * @vsi: the VSI being un-configured 11638 **/ 11639 static int i40e_vsi_clear(struct i40e_vsi *vsi) 11640 { 11641 struct i40e_pf *pf; 11642 11643 if (!vsi) 11644 return 0; 11645 11646 if (!vsi->back) 11647 goto free_vsi; 11648 pf = vsi->back; 11649 11650 mutex_lock(&pf->switch_mutex); 11651 if (!pf->vsi[vsi->idx]) { 11652 dev_err(&pf->pdev->dev, "pf->vsi[%d] is NULL, just free vsi[%d](type %d)\n", 11653 vsi->idx, vsi->idx, vsi->type); 11654 goto unlock_vsi; 11655 } 11656 11657 if (pf->vsi[vsi->idx] != vsi) { 11658 dev_err(&pf->pdev->dev, 11659 "pf->vsi[%d](type %d) != vsi[%d](type %d): no free!\n", 11660 pf->vsi[vsi->idx]->idx, 11661 pf->vsi[vsi->idx]->type, 11662 vsi->idx, vsi->type); 11663 goto unlock_vsi; 11664 } 11665 11666 /* updates the PF for this cleared vsi */ 11667 i40e_put_lump(pf->qp_pile, vsi->base_queue, vsi->idx); 11668 i40e_put_lump(pf->irq_pile, vsi->base_vector, vsi->idx); 11669 11670 bitmap_free(vsi->af_xdp_zc_qps); 11671 i40e_vsi_free_arrays(vsi, true); 11672 i40e_clear_rss_config_user(vsi); 11673 11674 pf->vsi[vsi->idx] = NULL; 11675 if (vsi->idx < pf->next_vsi) 11676 pf->next_vsi = vsi->idx; 11677 11678 unlock_vsi: 11679 mutex_unlock(&pf->switch_mutex); 11680 free_vsi: 11681 kfree(vsi); 11682 11683 return 0; 11684 } 11685 11686 /** 11687 * i40e_vsi_clear_rings - Deallocates the Rx and Tx rings for the provided VSI 11688 * @vsi: the VSI being cleaned 11689 **/ 11690 static void i40e_vsi_clear_rings(struct i40e_vsi *vsi) 11691 { 11692 int i; 11693 11694 if (vsi->tx_rings && vsi->tx_rings[0]) { 11695 for (i = 0; i < vsi->alloc_queue_pairs; i++) { 11696 kfree_rcu(vsi->tx_rings[i], rcu); 11697 WRITE_ONCE(vsi->tx_rings[i], NULL); 11698 WRITE_ONCE(vsi->rx_rings[i], NULL); 11699 if (vsi->xdp_rings) 11700 WRITE_ONCE(vsi->xdp_rings[i], NULL); 11701 } 11702 } 11703 } 11704 11705 /** 11706 * i40e_alloc_rings - Allocates the Rx and Tx rings for the provided VSI 11707 * @vsi: the VSI being configured 11708 **/ 11709 static int i40e_alloc_rings(struct i40e_vsi *vsi) 11710 { 11711 int i, qpv = i40e_enabled_xdp_vsi(vsi) ? 3 : 2; 11712 struct i40e_pf *pf = vsi->back; 11713 struct i40e_ring *ring; 11714 11715 /* Set basic values in the rings to be used later during open() */ 11716 for (i = 0; i < vsi->alloc_queue_pairs; i++) { 11717 /* allocate space for both Tx and Rx in one shot */ 11718 ring = kzalloc_objs(struct i40e_ring, qpv); 11719 if (!ring) 11720 goto err_out; 11721 11722 ring->queue_index = i; 11723 ring->reg_idx = vsi->base_queue + i; 11724 ring->ring_active = false; 11725 ring->vsi = vsi; 11726 ring->netdev = vsi->netdev; 11727 ring->dev = &pf->pdev->dev; 11728 ring->count = vsi->num_tx_desc; 11729 ring->size = 0; 11730 ring->dcb_tc = 0; 11731 if (test_bit(I40E_HW_CAP_WB_ON_ITR, vsi->back->hw.caps)) 11732 ring->flags = I40E_TXR_FLAGS_WB_ON_ITR; 11733 ring->itr_setting = pf->tx_itr_default; 11734 WRITE_ONCE(vsi->tx_rings[i], ring++); 11735 11736 if (!i40e_enabled_xdp_vsi(vsi)) 11737 goto setup_rx; 11738 11739 ring->queue_index = vsi->alloc_queue_pairs + i; 11740 ring->reg_idx = vsi->base_queue + ring->queue_index; 11741 ring->ring_active = false; 11742 ring->vsi = vsi; 11743 ring->netdev = NULL; 11744 ring->dev = &pf->pdev->dev; 11745 ring->count = vsi->num_tx_desc; 11746 ring->size = 0; 11747 ring->dcb_tc = 0; 11748 if (test_bit(I40E_HW_CAP_WB_ON_ITR, vsi->back->hw.caps)) 11749 ring->flags = I40E_TXR_FLAGS_WB_ON_ITR; 11750 set_ring_xdp(ring); 11751 ring->itr_setting = pf->tx_itr_default; 11752 WRITE_ONCE(vsi->xdp_rings[i], ring++); 11753 11754 setup_rx: 11755 ring->queue_index = i; 11756 ring->reg_idx = vsi->base_queue + i; 11757 ring->ring_active = false; 11758 ring->vsi = vsi; 11759 ring->netdev = vsi->netdev; 11760 ring->dev = &pf->pdev->dev; 11761 ring->count = vsi->num_rx_desc; 11762 ring->size = 0; 11763 ring->dcb_tc = 0; 11764 ring->itr_setting = pf->rx_itr_default; 11765 WRITE_ONCE(vsi->rx_rings[i], ring); 11766 } 11767 11768 return 0; 11769 11770 err_out: 11771 i40e_vsi_clear_rings(vsi); 11772 return -ENOMEM; 11773 } 11774 11775 /** 11776 * i40e_reserve_msix_vectors - Reserve MSI-X vectors in the kernel 11777 * @pf: board private structure 11778 * @vectors: the number of MSI-X vectors to request 11779 * 11780 * Returns the number of vectors reserved, or error 11781 **/ 11782 static int i40e_reserve_msix_vectors(struct i40e_pf *pf, int vectors) 11783 { 11784 vectors = pci_enable_msix_range(pf->pdev, pf->msix_entries, 11785 I40E_MIN_MSIX, vectors); 11786 if (vectors < 0) { 11787 dev_info(&pf->pdev->dev, 11788 "MSI-X vector reservation failed: %d\n", vectors); 11789 vectors = 0; 11790 } 11791 11792 return vectors; 11793 } 11794 11795 /** 11796 * i40e_init_msix - Setup the MSIX capability 11797 * @pf: board private structure 11798 * 11799 * Work with the OS to set up the MSIX vectors needed. 11800 * 11801 * Returns the number of vectors reserved or negative on failure 11802 **/ 11803 static int i40e_init_msix(struct i40e_pf *pf) 11804 { 11805 struct i40e_hw *hw = &pf->hw; 11806 int cpus, extra_vectors; 11807 int vectors_left; 11808 int v_budget, i; 11809 int v_actual; 11810 int iwarp_requested = 0; 11811 11812 if (!test_bit(I40E_FLAG_MSIX_ENA, pf->flags)) 11813 return -ENODEV; 11814 11815 /* The number of vectors we'll request will be comprised of: 11816 * - Add 1 for "other" cause for Admin Queue events, etc. 11817 * - The number of LAN queue pairs 11818 * - Queues being used for RSS. 11819 * We don't need as many as max_rss_size vectors. 11820 * use rss_size instead in the calculation since that 11821 * is governed by number of cpus in the system. 11822 * - assumes symmetric Tx/Rx pairing 11823 * - The number of VMDq pairs 11824 * - The CPU count within the NUMA node if iWARP is enabled 11825 * Once we count this up, try the request. 11826 * 11827 * If we can't get what we want, we'll simplify to nearly nothing 11828 * and try again. If that still fails, we punt. 11829 */ 11830 vectors_left = hw->func_caps.num_msix_vectors; 11831 v_budget = 0; 11832 11833 /* reserve one vector for miscellaneous handler */ 11834 if (vectors_left) { 11835 v_budget++; 11836 vectors_left--; 11837 } 11838 11839 /* reserve some vectors for the main PF traffic queues. Initially we 11840 * only reserve at most 50% of the available vectors, in the case that 11841 * the number of online CPUs is large. This ensures that we can enable 11842 * extra features as well. Once we've enabled the other features, we 11843 * will use any remaining vectors to reach as close as we can to the 11844 * number of online CPUs. 11845 */ 11846 cpus = num_online_cpus(); 11847 pf->num_lan_msix = min_t(int, cpus, vectors_left / 2); 11848 vectors_left -= pf->num_lan_msix; 11849 11850 /* reserve one vector for sideband flow director */ 11851 if (test_bit(I40E_FLAG_FD_SB_ENA, pf->flags)) { 11852 if (vectors_left) { 11853 pf->num_fdsb_msix = 1; 11854 v_budget++; 11855 vectors_left--; 11856 } else { 11857 pf->num_fdsb_msix = 0; 11858 } 11859 } 11860 11861 /* can we reserve enough for iWARP? */ 11862 if (test_bit(I40E_FLAG_IWARP_ENA, pf->flags)) { 11863 iwarp_requested = pf->num_iwarp_msix; 11864 11865 if (!vectors_left) 11866 pf->num_iwarp_msix = 0; 11867 else if (vectors_left < pf->num_iwarp_msix) 11868 pf->num_iwarp_msix = 1; 11869 v_budget += pf->num_iwarp_msix; 11870 vectors_left -= pf->num_iwarp_msix; 11871 } 11872 11873 /* any vectors left over go for VMDq support */ 11874 if (test_bit(I40E_FLAG_VMDQ_ENA, pf->flags)) { 11875 if (!vectors_left) { 11876 pf->num_vmdq_msix = 0; 11877 pf->num_vmdq_qps = 0; 11878 } else { 11879 int vmdq_vecs_wanted = 11880 pf->num_vmdq_vsis * pf->num_vmdq_qps; 11881 int vmdq_vecs = 11882 min_t(int, vectors_left, vmdq_vecs_wanted); 11883 11884 /* if we're short on vectors for what's desired, we limit 11885 * the queues per vmdq. If this is still more than are 11886 * available, the user will need to change the number of 11887 * queues/vectors used by the PF later with the ethtool 11888 * channels command 11889 */ 11890 if (vectors_left < vmdq_vecs_wanted) { 11891 pf->num_vmdq_qps = 1; 11892 vmdq_vecs_wanted = pf->num_vmdq_vsis; 11893 vmdq_vecs = min_t(int, 11894 vectors_left, 11895 vmdq_vecs_wanted); 11896 } 11897 pf->num_vmdq_msix = pf->num_vmdq_qps; 11898 11899 v_budget += vmdq_vecs; 11900 vectors_left -= vmdq_vecs; 11901 } 11902 } 11903 11904 /* On systems with a large number of SMP cores, we previously limited 11905 * the number of vectors for num_lan_msix to be at most 50% of the 11906 * available vectors, to allow for other features. Now, we add back 11907 * the remaining vectors. However, we ensure that the total 11908 * num_lan_msix will not exceed num_online_cpus(). To do this, we 11909 * calculate the number of vectors we can add without going over the 11910 * cap of CPUs. For systems with a small number of CPUs this will be 11911 * zero. 11912 */ 11913 extra_vectors = min_t(int, cpus - pf->num_lan_msix, vectors_left); 11914 pf->num_lan_msix += extra_vectors; 11915 vectors_left -= extra_vectors; 11916 11917 WARN(vectors_left < 0, 11918 "Calculation of remaining vectors underflowed. This is an accounting bug when determining total MSI-X vectors.\n"); 11919 11920 v_budget += pf->num_lan_msix; 11921 pf->msix_entries = kzalloc_objs(struct msix_entry, v_budget); 11922 if (!pf->msix_entries) 11923 return -ENOMEM; 11924 11925 for (i = 0; i < v_budget; i++) 11926 pf->msix_entries[i].entry = i; 11927 v_actual = i40e_reserve_msix_vectors(pf, v_budget); 11928 11929 if (v_actual < I40E_MIN_MSIX) { 11930 clear_bit(I40E_FLAG_MSIX_ENA, pf->flags); 11931 kfree(pf->msix_entries); 11932 pf->msix_entries = NULL; 11933 pci_disable_msix(pf->pdev); 11934 return -ENODEV; 11935 11936 } else if (v_actual == I40E_MIN_MSIX) { 11937 /* Adjust for minimal MSIX use */ 11938 pf->num_vmdq_vsis = 0; 11939 pf->num_vmdq_qps = 0; 11940 pf->num_lan_qps = 1; 11941 pf->num_lan_msix = 1; 11942 11943 } else if (v_actual != v_budget) { 11944 /* If we have limited resources, we will start with no vectors 11945 * for the special features and then allocate vectors to some 11946 * of these features based on the policy and at the end disable 11947 * the features that did not get any vectors. 11948 */ 11949 int vec; 11950 11951 dev_info(&pf->pdev->dev, 11952 "MSI-X vector limit reached with %d, wanted %d, attempting to redistribute vectors\n", 11953 v_actual, v_budget); 11954 /* reserve the misc vector */ 11955 vec = v_actual - 1; 11956 11957 /* Scale vector usage down */ 11958 pf->num_vmdq_msix = 1; /* force VMDqs to only one vector */ 11959 pf->num_vmdq_vsis = 1; 11960 pf->num_vmdq_qps = 1; 11961 11962 /* partition out the remaining vectors */ 11963 switch (vec) { 11964 case 2: 11965 pf->num_lan_msix = 1; 11966 break; 11967 case 3: 11968 if (test_bit(I40E_FLAG_IWARP_ENA, pf->flags)) { 11969 pf->num_lan_msix = 1; 11970 pf->num_iwarp_msix = 1; 11971 } else { 11972 pf->num_lan_msix = 2; 11973 } 11974 break; 11975 default: 11976 if (test_bit(I40E_FLAG_IWARP_ENA, pf->flags)) { 11977 pf->num_iwarp_msix = min_t(int, (vec / 3), 11978 iwarp_requested); 11979 pf->num_vmdq_vsis = min_t(int, (vec / 3), 11980 I40E_DEFAULT_NUM_VMDQ_VSI); 11981 } else { 11982 pf->num_vmdq_vsis = min_t(int, (vec / 2), 11983 I40E_DEFAULT_NUM_VMDQ_VSI); 11984 } 11985 if (test_bit(I40E_FLAG_FD_SB_ENA, pf->flags)) { 11986 pf->num_fdsb_msix = 1; 11987 vec--; 11988 } 11989 pf->num_lan_msix = min_t(int, 11990 (vec - (pf->num_iwarp_msix + pf->num_vmdq_vsis)), 11991 pf->num_lan_msix); 11992 pf->num_lan_qps = pf->num_lan_msix; 11993 break; 11994 } 11995 } 11996 11997 if (test_bit(I40E_FLAG_FD_SB_ENA, pf->flags) && pf->num_fdsb_msix == 0) { 11998 dev_info(&pf->pdev->dev, "Sideband Flowdir disabled, not enough MSI-X vectors\n"); 11999 clear_bit(I40E_FLAG_FD_SB_ENA, pf->flags); 12000 set_bit(I40E_FLAG_FD_SB_INACTIVE, pf->flags); 12001 } 12002 if (test_bit(I40E_FLAG_VMDQ_ENA, pf->flags) && pf->num_vmdq_msix == 0) { 12003 dev_info(&pf->pdev->dev, "VMDq disabled, not enough MSI-X vectors\n"); 12004 clear_bit(I40E_FLAG_VMDQ_ENA, pf->flags); 12005 } 12006 12007 if (test_bit(I40E_FLAG_IWARP_ENA, pf->flags) && 12008 pf->num_iwarp_msix == 0) { 12009 dev_info(&pf->pdev->dev, "IWARP disabled, not enough MSI-X vectors\n"); 12010 clear_bit(I40E_FLAG_IWARP_ENA, pf->flags); 12011 } 12012 i40e_debug(&pf->hw, I40E_DEBUG_INIT, 12013 "MSI-X vector distribution: PF %d, VMDq %d, FDSB %d, iWARP %d\n", 12014 pf->num_lan_msix, 12015 pf->num_vmdq_msix * pf->num_vmdq_vsis, 12016 pf->num_fdsb_msix, 12017 pf->num_iwarp_msix); 12018 12019 return v_actual; 12020 } 12021 12022 /** 12023 * i40e_vsi_alloc_q_vector - Allocate memory for a single interrupt vector 12024 * @vsi: the VSI being configured 12025 * @v_idx: index of the vector in the vsi struct 12026 * 12027 * We allocate one q_vector. If allocation fails we return -ENOMEM. 12028 **/ 12029 static int i40e_vsi_alloc_q_vector(struct i40e_vsi *vsi, int v_idx) 12030 { 12031 struct i40e_q_vector *q_vector; 12032 12033 /* allocate q_vector */ 12034 q_vector = kzalloc_obj(struct i40e_q_vector); 12035 if (!q_vector) 12036 return -ENOMEM; 12037 12038 q_vector->vsi = vsi; 12039 q_vector->v_idx = v_idx; 12040 cpumask_copy(&q_vector->affinity_mask, cpu_possible_mask); 12041 12042 if (vsi->netdev) 12043 netif_napi_add(vsi->netdev, &q_vector->napi, i40e_napi_poll); 12044 12045 /* tie q_vector and vsi together */ 12046 vsi->q_vectors[v_idx] = q_vector; 12047 12048 return 0; 12049 } 12050 12051 /** 12052 * i40e_vsi_alloc_q_vectors - Allocate memory for interrupt vectors 12053 * @vsi: the VSI being configured 12054 * 12055 * We allocate one q_vector per queue interrupt. If allocation fails we 12056 * return -ENOMEM. 12057 **/ 12058 static int i40e_vsi_alloc_q_vectors(struct i40e_vsi *vsi) 12059 { 12060 struct i40e_pf *pf = vsi->back; 12061 int err, v_idx, num_q_vectors; 12062 12063 /* if not MSIX, give the one vector only to the LAN VSI */ 12064 if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags)) 12065 num_q_vectors = vsi->num_q_vectors; 12066 else if (vsi->type == I40E_VSI_MAIN) 12067 num_q_vectors = 1; 12068 else 12069 return -EINVAL; 12070 12071 for (v_idx = 0; v_idx < num_q_vectors; v_idx++) { 12072 err = i40e_vsi_alloc_q_vector(vsi, v_idx); 12073 if (err) 12074 goto err_out; 12075 } 12076 12077 return 0; 12078 12079 err_out: 12080 while (v_idx--) 12081 i40e_free_q_vector(vsi, v_idx); 12082 12083 return err; 12084 } 12085 12086 /** 12087 * i40e_init_interrupt_scheme - Determine proper interrupt scheme 12088 * @pf: board private structure to initialize 12089 **/ 12090 static int i40e_init_interrupt_scheme(struct i40e_pf *pf) 12091 { 12092 int vectors = 0; 12093 ssize_t size; 12094 12095 if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags)) { 12096 vectors = i40e_init_msix(pf); 12097 if (vectors < 0) { 12098 clear_bit(I40E_FLAG_MSIX_ENA, pf->flags); 12099 clear_bit(I40E_FLAG_IWARP_ENA, pf->flags); 12100 clear_bit(I40E_FLAG_RSS_ENA, pf->flags); 12101 clear_bit(I40E_FLAG_DCB_CAPABLE, pf->flags); 12102 clear_bit(I40E_FLAG_DCB_ENA, pf->flags); 12103 clear_bit(I40E_FLAG_SRIOV_ENA, pf->flags); 12104 clear_bit(I40E_FLAG_FD_SB_ENA, pf->flags); 12105 clear_bit(I40E_FLAG_FD_ATR_ENA, pf->flags); 12106 clear_bit(I40E_FLAG_VMDQ_ENA, pf->flags); 12107 set_bit(I40E_FLAG_FD_SB_INACTIVE, pf->flags); 12108 12109 /* rework the queue expectations without MSIX */ 12110 i40e_determine_queue_usage(pf); 12111 } 12112 } 12113 12114 if (!test_bit(I40E_FLAG_MSIX_ENA, pf->flags) && 12115 test_bit(I40E_FLAG_MSI_ENA, pf->flags)) { 12116 dev_info(&pf->pdev->dev, "MSI-X not available, trying MSI\n"); 12117 vectors = pci_enable_msi(pf->pdev); 12118 if (vectors < 0) { 12119 dev_info(&pf->pdev->dev, "MSI init failed - %d\n", 12120 vectors); 12121 clear_bit(I40E_FLAG_MSI_ENA, pf->flags); 12122 } 12123 vectors = 1; /* one MSI or Legacy vector */ 12124 } 12125 12126 if (!test_bit(I40E_FLAG_MSI_ENA, pf->flags) && 12127 !test_bit(I40E_FLAG_MSIX_ENA, pf->flags)) 12128 dev_info(&pf->pdev->dev, "MSI-X and MSI not available, falling back to Legacy IRQ\n"); 12129 12130 /* set up vector assignment tracking */ 12131 size = sizeof(struct i40e_lump_tracking) + (sizeof(u16) * vectors); 12132 pf->irq_pile = kzalloc(size, GFP_KERNEL); 12133 if (!pf->irq_pile) 12134 return -ENOMEM; 12135 12136 pf->irq_pile->num_entries = vectors; 12137 12138 /* track first vector for misc interrupts, ignore return */ 12139 (void)i40e_get_lump(pf, pf->irq_pile, 1, I40E_PILE_VALID_BIT - 1); 12140 12141 return 0; 12142 } 12143 12144 /** 12145 * i40e_restore_interrupt_scheme - Restore the interrupt scheme 12146 * @pf: private board data structure 12147 * 12148 * Restore the interrupt scheme that was cleared when we suspended the 12149 * device. This should be called during resume to re-allocate the q_vectors 12150 * and reacquire IRQs. 12151 */ 12152 static int i40e_restore_interrupt_scheme(struct i40e_pf *pf) 12153 { 12154 struct i40e_vsi *vsi; 12155 int err, i; 12156 12157 /* We cleared the MSI and MSI-X flags when disabling the old interrupt 12158 * scheme. We need to re-enabled them here in order to attempt to 12159 * re-acquire the MSI or MSI-X vectors 12160 */ 12161 set_bit(I40E_FLAG_MSI_ENA, pf->flags); 12162 set_bit(I40E_FLAG_MSIX_ENA, pf->flags); 12163 12164 err = i40e_init_interrupt_scheme(pf); 12165 if (err) 12166 return err; 12167 12168 /* Now that we've re-acquired IRQs, we need to remap the vectors and 12169 * rings together again. 12170 */ 12171 i40e_pf_for_each_vsi(pf, i, vsi) { 12172 err = i40e_vsi_alloc_q_vectors(vsi); 12173 if (err) 12174 goto err_unwind; 12175 12176 i40e_vsi_map_rings_to_vectors(vsi); 12177 } 12178 12179 err = i40e_setup_misc_vector(pf); 12180 if (err) 12181 goto err_unwind; 12182 12183 if (test_bit(I40E_FLAG_IWARP_ENA, pf->flags)) 12184 i40e_client_update_msix_info(pf); 12185 12186 return 0; 12187 12188 err_unwind: 12189 while (i--) { 12190 if (pf->vsi[i]) 12191 i40e_vsi_free_q_vectors(pf->vsi[i]); 12192 } 12193 12194 return err; 12195 } 12196 12197 /** 12198 * i40e_setup_misc_vector_for_recovery_mode - Setup the misc vector to handle 12199 * non queue events in recovery mode 12200 * @pf: board private structure 12201 * 12202 * This sets up the handler for MSIX 0 or MSI/legacy, which is used to manage 12203 * the non-queue interrupts, e.g. AdminQ and errors in recovery mode. 12204 * This is handled differently than in recovery mode since no Tx/Rx resources 12205 * are being allocated. 12206 **/ 12207 static int i40e_setup_misc_vector_for_recovery_mode(struct i40e_pf *pf) 12208 { 12209 int err; 12210 12211 if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags)) { 12212 err = i40e_setup_misc_vector(pf); 12213 12214 if (err) { 12215 dev_info(&pf->pdev->dev, 12216 "MSI-X misc vector request failed, error %d\n", 12217 err); 12218 return err; 12219 } 12220 } else { 12221 u32 flags = test_bit(I40E_FLAG_MSI_ENA, pf->flags) ? 0 : IRQF_SHARED; 12222 12223 err = request_irq(pf->pdev->irq, i40e_intr, flags, 12224 pf->int_name, pf); 12225 12226 if (err) { 12227 dev_info(&pf->pdev->dev, 12228 "MSI/legacy misc vector request failed, error %d\n", 12229 err); 12230 return err; 12231 } 12232 i40e_enable_misc_int_causes(pf); 12233 i40e_irq_dynamic_enable_icr0(pf); 12234 } 12235 12236 return 0; 12237 } 12238 12239 /** 12240 * i40e_setup_misc_vector - Setup the misc vector to handle non queue events 12241 * @pf: board private structure 12242 * 12243 * This sets up the handler for MSIX 0, which is used to manage the 12244 * non-queue interrupts, e.g. AdminQ and errors. This is not used 12245 * when in MSI or Legacy interrupt mode. 12246 **/ 12247 static int i40e_setup_misc_vector(struct i40e_pf *pf) 12248 { 12249 struct i40e_hw *hw = &pf->hw; 12250 int err = 0; 12251 12252 /* Only request the IRQ once, the first time through. */ 12253 if (!test_and_set_bit(__I40E_MISC_IRQ_REQUESTED, pf->state)) { 12254 err = request_irq(pf->msix_entries[0].vector, 12255 i40e_intr, 0, pf->int_name, pf); 12256 if (err) { 12257 clear_bit(__I40E_MISC_IRQ_REQUESTED, pf->state); 12258 dev_info(&pf->pdev->dev, 12259 "request_irq for %s failed: %d\n", 12260 pf->int_name, err); 12261 return -EFAULT; 12262 } 12263 } 12264 12265 i40e_enable_misc_int_causes(pf); 12266 12267 /* associate no queues to the misc vector */ 12268 wr32(hw, I40E_PFINT_LNKLST0, I40E_QUEUE_END_OF_LIST); 12269 wr32(hw, I40E_PFINT_ITR0(I40E_RX_ITR), I40E_ITR_8K >> 1); 12270 12271 i40e_flush(hw); 12272 12273 i40e_irq_dynamic_enable_icr0(pf); 12274 12275 return err; 12276 } 12277 12278 /** 12279 * i40e_get_rss_aq - Get RSS keys and lut by using AQ commands 12280 * @vsi: Pointer to vsi structure 12281 * @seed: Buffter to store the hash keys 12282 * @lut: Buffer to store the lookup table entries 12283 * @lut_size: Size of buffer to store the lookup table entries 12284 * 12285 * Return 0 on success, negative on failure 12286 */ 12287 static int i40e_get_rss_aq(struct i40e_vsi *vsi, const u8 *seed, 12288 u8 *lut, u16 lut_size) 12289 { 12290 struct i40e_pf *pf = vsi->back; 12291 struct i40e_hw *hw = &pf->hw; 12292 int ret = 0; 12293 12294 if (seed) { 12295 ret = i40e_aq_get_rss_key(hw, vsi->id, 12296 (struct i40e_aqc_get_set_rss_key_data *)seed); 12297 if (ret) { 12298 dev_info(&pf->pdev->dev, 12299 "Cannot get RSS key, err %pe aq_err %s\n", 12300 ERR_PTR(ret), 12301 libie_aq_str(pf->hw.aq.asq_last_status)); 12302 return ret; 12303 } 12304 } 12305 12306 if (lut) { 12307 bool pf_lut = vsi->type == I40E_VSI_MAIN; 12308 12309 ret = i40e_aq_get_rss_lut(hw, vsi->id, pf_lut, lut, lut_size); 12310 if (ret) { 12311 dev_info(&pf->pdev->dev, 12312 "Cannot get RSS lut, err %pe aq_err %s\n", 12313 ERR_PTR(ret), 12314 libie_aq_str(pf->hw.aq.asq_last_status)); 12315 return ret; 12316 } 12317 } 12318 12319 return ret; 12320 } 12321 12322 /** 12323 * i40e_config_rss_reg - Configure RSS keys and lut by writing registers 12324 * @vsi: Pointer to vsi structure 12325 * @seed: RSS hash seed 12326 * @lut: Lookup table 12327 * @lut_size: Lookup table size 12328 * 12329 * Returns 0 on success, negative on failure 12330 **/ 12331 static int i40e_config_rss_reg(struct i40e_vsi *vsi, const u8 *seed, 12332 const u8 *lut, u16 lut_size) 12333 { 12334 struct i40e_pf *pf = vsi->back; 12335 struct i40e_hw *hw = &pf->hw; 12336 u16 vf_id = vsi->vf_id; 12337 u8 i; 12338 12339 /* Fill out hash function seed */ 12340 if (seed) { 12341 u32 *seed_dw = (u32 *)seed; 12342 12343 if (vsi->type == I40E_VSI_MAIN) { 12344 for (i = 0; i <= I40E_PFQF_HKEY_MAX_INDEX; i++) 12345 wr32(hw, I40E_PFQF_HKEY(i), seed_dw[i]); 12346 } else if (vsi->type == I40E_VSI_SRIOV) { 12347 for (i = 0; i <= I40E_VFQF_HKEY1_MAX_INDEX; i++) 12348 wr32(hw, I40E_VFQF_HKEY1(i, vf_id), seed_dw[i]); 12349 } else { 12350 dev_err(&pf->pdev->dev, "Cannot set RSS seed - invalid VSI type\n"); 12351 } 12352 } 12353 12354 if (lut) { 12355 u32 *lut_dw = (u32 *)lut; 12356 12357 if (vsi->type == I40E_VSI_MAIN) { 12358 if (lut_size != I40E_HLUT_ARRAY_SIZE) 12359 return -EINVAL; 12360 for (i = 0; i <= I40E_PFQF_HLUT_MAX_INDEX; i++) 12361 wr32(hw, I40E_PFQF_HLUT(i), lut_dw[i]); 12362 } else if (vsi->type == I40E_VSI_SRIOV) { 12363 if (lut_size != I40E_VF_HLUT_ARRAY_SIZE) 12364 return -EINVAL; 12365 for (i = 0; i <= I40E_VFQF_HLUT_MAX_INDEX; i++) 12366 wr32(hw, I40E_VFQF_HLUT1(i, vf_id), lut_dw[i]); 12367 } else { 12368 dev_err(&pf->pdev->dev, "Cannot set RSS LUT - invalid VSI type\n"); 12369 } 12370 } 12371 i40e_flush(hw); 12372 12373 return 0; 12374 } 12375 12376 /** 12377 * i40e_get_rss_reg - Get the RSS keys and lut by reading registers 12378 * @vsi: Pointer to VSI structure 12379 * @seed: Buffer to store the keys 12380 * @lut: Buffer to store the lookup table entries 12381 * @lut_size: Size of buffer to store the lookup table entries 12382 * 12383 * Returns 0 on success, negative on failure 12384 */ 12385 static int i40e_get_rss_reg(struct i40e_vsi *vsi, u8 *seed, 12386 u8 *lut, u16 lut_size) 12387 { 12388 struct i40e_pf *pf = vsi->back; 12389 struct i40e_hw *hw = &pf->hw; 12390 u16 i; 12391 12392 if (seed) { 12393 u32 *seed_dw = (u32 *)seed; 12394 12395 for (i = 0; i <= I40E_PFQF_HKEY_MAX_INDEX; i++) 12396 seed_dw[i] = i40e_read_rx_ctl(hw, I40E_PFQF_HKEY(i)); 12397 } 12398 if (lut) { 12399 u32 *lut_dw = (u32 *)lut; 12400 12401 if (lut_size != I40E_HLUT_ARRAY_SIZE) 12402 return -EINVAL; 12403 for (i = 0; i <= I40E_PFQF_HLUT_MAX_INDEX; i++) 12404 lut_dw[i] = rd32(hw, I40E_PFQF_HLUT(i)); 12405 } 12406 12407 return 0; 12408 } 12409 12410 /** 12411 * i40e_config_rss - Configure RSS keys and lut 12412 * @vsi: Pointer to VSI structure 12413 * @seed: RSS hash seed 12414 * @lut: Lookup table 12415 * @lut_size: Lookup table size 12416 * 12417 * Returns 0 on success, negative on failure 12418 */ 12419 int i40e_config_rss(struct i40e_vsi *vsi, u8 *seed, u8 *lut, u16 lut_size) 12420 { 12421 struct i40e_pf *pf = vsi->back; 12422 12423 if (test_bit(I40E_HW_CAP_RSS_AQ, pf->hw.caps)) 12424 return i40e_config_rss_aq(vsi, seed, lut, lut_size); 12425 else 12426 return i40e_config_rss_reg(vsi, seed, lut, lut_size); 12427 } 12428 12429 /** 12430 * i40e_get_rss - Get RSS keys and lut 12431 * @vsi: Pointer to VSI structure 12432 * @seed: Buffer to store the keys 12433 * @lut: Buffer to store the lookup table entries 12434 * @lut_size: Size of buffer to store the lookup table entries 12435 * 12436 * Returns 0 on success, negative on failure 12437 */ 12438 int i40e_get_rss(struct i40e_vsi *vsi, u8 *seed, u8 *lut, u16 lut_size) 12439 { 12440 struct i40e_pf *pf = vsi->back; 12441 12442 if (test_bit(I40E_HW_CAP_RSS_AQ, pf->hw.caps)) 12443 return i40e_get_rss_aq(vsi, seed, lut, lut_size); 12444 else 12445 return i40e_get_rss_reg(vsi, seed, lut, lut_size); 12446 } 12447 12448 /** 12449 * i40e_fill_rss_lut - Fill the RSS lookup table with default values 12450 * @pf: Pointer to board private structure 12451 * @lut: Lookup table 12452 * @rss_table_size: Lookup table size 12453 * @rss_size: Range of queue number for hashing 12454 */ 12455 void i40e_fill_rss_lut(struct i40e_pf *pf, u8 *lut, 12456 u16 rss_table_size, u16 rss_size) 12457 { 12458 u16 i; 12459 12460 for (i = 0; i < rss_table_size; i++) 12461 lut[i] = i % rss_size; 12462 } 12463 12464 /** 12465 * i40e_pf_config_rss - Prepare for RSS if used 12466 * @pf: board private structure 12467 **/ 12468 static int i40e_pf_config_rss(struct i40e_pf *pf) 12469 { 12470 struct i40e_vsi *vsi = i40e_pf_get_main_vsi(pf); 12471 u8 seed[I40E_HKEY_ARRAY_SIZE]; 12472 u8 *lut; 12473 struct i40e_hw *hw = &pf->hw; 12474 u32 reg_val; 12475 u64 hena; 12476 int ret; 12477 12478 /* By default we enable TCP/UDP with IPv4/IPv6 ptypes */ 12479 hena = (u64)i40e_read_rx_ctl(hw, I40E_PFQF_HENA(0)) | 12480 ((u64)i40e_read_rx_ctl(hw, I40E_PFQF_HENA(1)) << 32); 12481 hena |= i40e_pf_get_default_rss_hashcfg(pf); 12482 12483 i40e_write_rx_ctl(hw, I40E_PFQF_HENA(0), (u32)hena); 12484 i40e_write_rx_ctl(hw, I40E_PFQF_HENA(1), (u32)(hena >> 32)); 12485 12486 /* Determine the RSS table size based on the hardware capabilities */ 12487 reg_val = i40e_read_rx_ctl(hw, I40E_PFQF_CTL_0); 12488 reg_val = (pf->rss_table_size == 512) ? 12489 (reg_val | I40E_PFQF_CTL_0_HASHLUTSIZE_512) : 12490 (reg_val & ~I40E_PFQF_CTL_0_HASHLUTSIZE_512); 12491 i40e_write_rx_ctl(hw, I40E_PFQF_CTL_0, reg_val); 12492 12493 /* Determine the RSS size of the VSI */ 12494 if (!vsi->rss_size) { 12495 u16 qcount; 12496 /* If the firmware does something weird during VSI init, we 12497 * could end up with zero TCs. Check for that to avoid 12498 * divide-by-zero. It probably won't pass traffic, but it also 12499 * won't panic. 12500 */ 12501 qcount = vsi->num_queue_pairs / 12502 (vsi->tc_config.numtc ? vsi->tc_config.numtc : 1); 12503 vsi->rss_size = min_t(int, pf->alloc_rss_size, qcount); 12504 } 12505 if (!vsi->rss_size) 12506 return -EINVAL; 12507 12508 lut = kzalloc(vsi->rss_table_size, GFP_KERNEL); 12509 if (!lut) 12510 return -ENOMEM; 12511 12512 /* Use user configured lut if there is one, otherwise use default */ 12513 if (vsi->rss_lut_user) 12514 memcpy(lut, vsi->rss_lut_user, vsi->rss_table_size); 12515 else 12516 i40e_fill_rss_lut(pf, lut, vsi->rss_table_size, vsi->rss_size); 12517 12518 /* Use user configured hash key if there is one, otherwise 12519 * use default. 12520 */ 12521 if (vsi->rss_hkey_user) 12522 memcpy(seed, vsi->rss_hkey_user, I40E_HKEY_ARRAY_SIZE); 12523 else 12524 netdev_rss_key_fill((void *)seed, I40E_HKEY_ARRAY_SIZE); 12525 ret = i40e_config_rss(vsi, seed, lut, vsi->rss_table_size); 12526 kfree(lut); 12527 12528 return ret; 12529 } 12530 12531 /** 12532 * i40e_reconfig_rss_queues - change number of queues for rss and rebuild 12533 * @pf: board private structure 12534 * @queue_count: the requested queue count for rss. 12535 * 12536 * returns 0 if rss is not enabled, if enabled returns the final rss queue 12537 * count which may be different from the requested queue count. 12538 * Note: expects to be called while under rtnl_lock() 12539 **/ 12540 int i40e_reconfig_rss_queues(struct i40e_pf *pf, int queue_count) 12541 { 12542 struct i40e_vsi *vsi = i40e_pf_get_main_vsi(pf); 12543 int new_rss_size; 12544 12545 if (!test_bit(I40E_FLAG_RSS_ENA, pf->flags)) 12546 return 0; 12547 12548 queue_count = min_t(int, queue_count, num_online_cpus()); 12549 new_rss_size = min_t(int, queue_count, pf->rss_size_max); 12550 12551 if (queue_count != vsi->num_queue_pairs) { 12552 u16 qcount; 12553 12554 vsi->req_queue_pairs = queue_count; 12555 i40e_prep_for_reset(pf); 12556 if (test_bit(__I40E_IN_REMOVE, pf->state)) 12557 return pf->alloc_rss_size; 12558 12559 pf->alloc_rss_size = new_rss_size; 12560 12561 i40e_reset_and_rebuild(pf, true, true); 12562 12563 /* Discard the user configured hash keys and lut, if less 12564 * queues are enabled. 12565 */ 12566 if (queue_count < vsi->rss_size) { 12567 i40e_clear_rss_config_user(vsi); 12568 dev_dbg(&pf->pdev->dev, 12569 "discard user configured hash keys and lut\n"); 12570 } 12571 12572 /* Reset vsi->rss_size, as number of enabled queues changed */ 12573 qcount = vsi->num_queue_pairs / vsi->tc_config.numtc; 12574 vsi->rss_size = min_t(int, pf->alloc_rss_size, qcount); 12575 12576 i40e_pf_config_rss(pf); 12577 } 12578 dev_info(&pf->pdev->dev, "User requested queue count/HW max RSS count: %d/%d\n", 12579 vsi->req_queue_pairs, pf->rss_size_max); 12580 return pf->alloc_rss_size; 12581 } 12582 12583 /** 12584 * i40e_get_partition_bw_setting - Retrieve BW settings for this PF partition 12585 * @pf: board private structure 12586 **/ 12587 int i40e_get_partition_bw_setting(struct i40e_pf *pf) 12588 { 12589 bool min_valid, max_valid; 12590 u32 max_bw, min_bw; 12591 int status; 12592 12593 status = i40e_read_bw_from_alt_ram(&pf->hw, &max_bw, &min_bw, 12594 &min_valid, &max_valid); 12595 12596 if (!status) { 12597 if (min_valid) 12598 pf->min_bw = min_bw; 12599 if (max_valid) 12600 pf->max_bw = max_bw; 12601 } 12602 12603 return status; 12604 } 12605 12606 /** 12607 * i40e_set_partition_bw_setting - Set BW settings for this PF partition 12608 * @pf: board private structure 12609 **/ 12610 int i40e_set_partition_bw_setting(struct i40e_pf *pf) 12611 { 12612 struct i40e_aqc_configure_partition_bw_data bw_data; 12613 int status; 12614 12615 memset(&bw_data, 0, sizeof(bw_data)); 12616 12617 /* Set the valid bit for this PF */ 12618 bw_data.pf_valid_bits = cpu_to_le16(BIT(pf->hw.pf_id)); 12619 bw_data.max_bw[pf->hw.pf_id] = pf->max_bw & I40E_ALT_BW_VALUE_MASK; 12620 bw_data.min_bw[pf->hw.pf_id] = pf->min_bw & I40E_ALT_BW_VALUE_MASK; 12621 12622 /* Set the new bandwidths */ 12623 status = i40e_aq_configure_partition_bw(&pf->hw, &bw_data, NULL); 12624 12625 return status; 12626 } 12627 12628 /** 12629 * i40e_is_total_port_shutdown_enabled - read NVM and return value 12630 * if total port shutdown feature is enabled for this PF 12631 * @pf: board private structure 12632 **/ 12633 static bool i40e_is_total_port_shutdown_enabled(struct i40e_pf *pf) 12634 { 12635 #define I40E_TOTAL_PORT_SHUTDOWN_ENABLED BIT(4) 12636 #define I40E_FEATURES_ENABLE_PTR 0x2A 12637 #define I40E_CURRENT_SETTING_PTR 0x2B 12638 #define I40E_LINK_BEHAVIOR_WORD_OFFSET 0x2D 12639 #define I40E_LINK_BEHAVIOR_WORD_LENGTH 0x1 12640 #define I40E_LINK_BEHAVIOR_OS_FORCED_ENABLED BIT(0) 12641 #define I40E_LINK_BEHAVIOR_PORT_BIT_LENGTH 4 12642 u16 sr_emp_sr_settings_ptr = 0; 12643 u16 features_enable = 0; 12644 u16 link_behavior = 0; 12645 int read_status = 0; 12646 bool ret = false; 12647 12648 read_status = i40e_read_nvm_word(&pf->hw, 12649 I40E_SR_EMP_SR_SETTINGS_PTR, 12650 &sr_emp_sr_settings_ptr); 12651 if (read_status) 12652 goto err_nvm; 12653 read_status = i40e_read_nvm_word(&pf->hw, 12654 sr_emp_sr_settings_ptr + 12655 I40E_FEATURES_ENABLE_PTR, 12656 &features_enable); 12657 if (read_status) 12658 goto err_nvm; 12659 if (I40E_TOTAL_PORT_SHUTDOWN_ENABLED & features_enable) { 12660 read_status = i40e_read_nvm_module_data(&pf->hw, 12661 I40E_SR_EMP_SR_SETTINGS_PTR, 12662 I40E_CURRENT_SETTING_PTR, 12663 I40E_LINK_BEHAVIOR_WORD_OFFSET, 12664 I40E_LINK_BEHAVIOR_WORD_LENGTH, 12665 &link_behavior); 12666 if (read_status) 12667 goto err_nvm; 12668 link_behavior >>= (pf->hw.port * I40E_LINK_BEHAVIOR_PORT_BIT_LENGTH); 12669 ret = I40E_LINK_BEHAVIOR_OS_FORCED_ENABLED & link_behavior; 12670 } 12671 return ret; 12672 12673 err_nvm: 12674 dev_warn(&pf->pdev->dev, 12675 "total-port-shutdown feature is off due to read nvm error: %pe\n", 12676 ERR_PTR(read_status)); 12677 return ret; 12678 } 12679 12680 /** 12681 * i40e_sw_init - Initialize general software structures (struct i40e_pf) 12682 * @pf: board private structure to initialize 12683 * 12684 * i40e_sw_init initializes the Adapter private data structure. 12685 * Fields are initialized based on PCI device information and 12686 * OS network device settings (MTU size). 12687 **/ 12688 static int i40e_sw_init(struct i40e_pf *pf) 12689 { 12690 int err = 0; 12691 int size; 12692 u16 pow; 12693 12694 /* Set default capability flags */ 12695 bitmap_zero(pf->flags, I40E_PF_FLAGS_NBITS); 12696 set_bit(I40E_FLAG_MSI_ENA, pf->flags); 12697 set_bit(I40E_FLAG_MSIX_ENA, pf->flags); 12698 12699 /* Set default ITR */ 12700 pf->rx_itr_default = I40E_ITR_RX_DEF; 12701 pf->tx_itr_default = I40E_ITR_TX_DEF; 12702 12703 /* Depending on PF configurations, it is possible that the RSS 12704 * maximum might end up larger than the available queues 12705 */ 12706 pf->rss_size_max = BIT(pf->hw.func_caps.rss_table_entry_width); 12707 pf->alloc_rss_size = 1; 12708 pf->rss_table_size = pf->hw.func_caps.rss_table_size; 12709 pf->rss_size_max = min_t(int, pf->rss_size_max, 12710 pf->hw.func_caps.num_tx_qp); 12711 12712 /* find the next higher power-of-2 of num cpus */ 12713 pow = roundup_pow_of_two(num_online_cpus()); 12714 pf->rss_size_max = min_t(int, pf->rss_size_max, pow); 12715 12716 if (pf->hw.func_caps.rss) { 12717 set_bit(I40E_FLAG_RSS_ENA, pf->flags); 12718 pf->alloc_rss_size = min_t(int, pf->rss_size_max, 12719 num_online_cpus()); 12720 } 12721 12722 /* MFP mode enabled */ 12723 if (pf->hw.func_caps.npar_enable || pf->hw.func_caps.flex10_enable) { 12724 set_bit(I40E_FLAG_MFP_ENA, pf->flags); 12725 dev_info(&pf->pdev->dev, "MFP mode Enabled\n"); 12726 if (i40e_get_partition_bw_setting(pf)) { 12727 dev_warn(&pf->pdev->dev, 12728 "Could not get partition bw settings\n"); 12729 } else { 12730 dev_info(&pf->pdev->dev, 12731 "Partition BW Min = %8.8x, Max = %8.8x\n", 12732 pf->min_bw, pf->max_bw); 12733 12734 /* nudge the Tx scheduler */ 12735 i40e_set_partition_bw_setting(pf); 12736 } 12737 } 12738 12739 if ((pf->hw.func_caps.fd_filters_guaranteed > 0) || 12740 (pf->hw.func_caps.fd_filters_best_effort > 0)) { 12741 set_bit(I40E_FLAG_FD_ATR_ENA, pf->flags); 12742 if (test_bit(I40E_FLAG_MFP_ENA, pf->flags) && 12743 pf->hw.num_partitions > 1) 12744 dev_info(&pf->pdev->dev, 12745 "Flow Director Sideband mode Disabled in MFP mode\n"); 12746 else 12747 set_bit(I40E_FLAG_FD_SB_ENA, pf->flags); 12748 pf->fdir_pf_filter_count = 12749 pf->hw.func_caps.fd_filters_guaranteed; 12750 pf->hw.fdir_shared_filter_count = 12751 pf->hw.func_caps.fd_filters_best_effort; 12752 } 12753 12754 /* Enable HW ATR eviction if possible */ 12755 if (test_bit(I40E_HW_CAP_ATR_EVICT, pf->hw.caps)) 12756 set_bit(I40E_FLAG_HW_ATR_EVICT_ENA, pf->flags); 12757 12758 if (pf->hw.func_caps.vmdq && num_online_cpus() != 1) { 12759 pf->num_vmdq_vsis = I40E_DEFAULT_NUM_VMDQ_VSI; 12760 set_bit(I40E_FLAG_VMDQ_ENA, pf->flags); 12761 pf->num_vmdq_qps = i40e_default_queues_per_vmdq(pf); 12762 } 12763 12764 if (pf->hw.func_caps.iwarp && num_online_cpus() != 1) { 12765 set_bit(I40E_FLAG_IWARP_ENA, pf->flags); 12766 /* IWARP needs one extra vector for CQP just like MISC.*/ 12767 pf->num_iwarp_msix = (int)num_online_cpus() + 1; 12768 } 12769 /* Stopping FW LLDP engine is supported on XL710 and X722 12770 * starting from FW versions determined in i40e_init_adminq. 12771 * Stopping the FW LLDP engine is not supported on XL710 12772 * if NPAR is functioning so unset this hw flag in this case. 12773 */ 12774 if (pf->hw.mac.type == I40E_MAC_XL710 && 12775 pf->hw.func_caps.npar_enable) 12776 clear_bit(I40E_HW_CAP_FW_LLDP_STOPPABLE, pf->hw.caps); 12777 12778 #ifdef CONFIG_PCI_IOV 12779 if (pf->hw.func_caps.num_vfs && pf->hw.partition_id == 1) { 12780 pf->num_vf_qps = I40E_DEFAULT_QUEUES_PER_VF; 12781 set_bit(I40E_FLAG_SRIOV_ENA, pf->flags); 12782 pf->num_req_vfs = min_t(int, 12783 pf->hw.func_caps.num_vfs, 12784 I40E_MAX_VF_COUNT); 12785 } 12786 #endif /* CONFIG_PCI_IOV */ 12787 pf->lan_veb = I40E_NO_VEB; 12788 pf->lan_vsi = I40E_NO_VSI; 12789 12790 /* By default FW has this off for performance reasons */ 12791 clear_bit(I40E_FLAG_VEB_STATS_ENA, pf->flags); 12792 12793 /* set up queue assignment tracking */ 12794 size = sizeof(struct i40e_lump_tracking) 12795 + (sizeof(u16) * pf->hw.func_caps.num_tx_qp); 12796 pf->qp_pile = kzalloc(size, GFP_KERNEL); 12797 if (!pf->qp_pile) { 12798 err = -ENOMEM; 12799 goto sw_init_done; 12800 } 12801 pf->qp_pile->num_entries = pf->hw.func_caps.num_tx_qp; 12802 12803 pf->tx_timeout_recovery_level = 1; 12804 12805 if (pf->hw.mac.type != I40E_MAC_X722 && 12806 i40e_is_total_port_shutdown_enabled(pf)) { 12807 /* Link down on close must be on when total port shutdown 12808 * is enabled for a given port 12809 */ 12810 set_bit(I40E_FLAG_TOTAL_PORT_SHUTDOWN_ENA, pf->flags); 12811 set_bit(I40E_FLAG_LINK_DOWN_ON_CLOSE_ENA, pf->flags); 12812 dev_info(&pf->pdev->dev, 12813 "total-port-shutdown was enabled, link-down-on-close is forced on\n"); 12814 } 12815 mutex_init(&pf->switch_mutex); 12816 12817 sw_init_done: 12818 return err; 12819 } 12820 12821 /** 12822 * i40e_set_ntuple - set the ntuple feature flag and take action 12823 * @pf: board private structure to initialize 12824 * @features: the feature set that the stack is suggesting 12825 * 12826 * returns a bool to indicate if reset needs to happen 12827 **/ 12828 bool i40e_set_ntuple(struct i40e_pf *pf, netdev_features_t features) 12829 { 12830 bool need_reset = false; 12831 12832 /* Check if Flow Director n-tuple support was enabled or disabled. If 12833 * the state changed, we need to reset. 12834 */ 12835 if (features & NETIF_F_NTUPLE) { 12836 /* Enable filters and mark for reset */ 12837 if (!test_bit(I40E_FLAG_FD_SB_ENA, pf->flags)) 12838 need_reset = true; 12839 /* enable FD_SB only if there is MSI-X vector and no cloud 12840 * filters exist 12841 */ 12842 if (pf->num_fdsb_msix > 0 && !pf->num_cloud_filters) { 12843 set_bit(I40E_FLAG_FD_SB_ENA, pf->flags); 12844 clear_bit(I40E_FLAG_FD_SB_INACTIVE, pf->flags); 12845 } 12846 } else { 12847 /* turn off filters, mark for reset and clear SW filter list */ 12848 if (test_bit(I40E_FLAG_FD_SB_ENA, pf->flags)) { 12849 need_reset = true; 12850 i40e_fdir_filter_exit(pf); 12851 } 12852 clear_bit(I40E_FLAG_FD_SB_ENA, pf->flags); 12853 clear_bit(__I40E_FD_SB_AUTO_DISABLED, pf->state); 12854 set_bit(I40E_FLAG_FD_SB_INACTIVE, pf->flags); 12855 12856 /* reset fd counters */ 12857 pf->fd_add_err = 0; 12858 pf->fd_atr_cnt = 0; 12859 /* if ATR was auto disabled it can be re-enabled. */ 12860 if (test_and_clear_bit(__I40E_FD_ATR_AUTO_DISABLED, pf->state)) 12861 if (test_bit(I40E_FLAG_FD_ATR_ENA, pf->flags) && 12862 (I40E_DEBUG_FD & pf->hw.debug_mask)) 12863 dev_info(&pf->pdev->dev, "ATR re-enabled.\n"); 12864 } 12865 return need_reset; 12866 } 12867 12868 /** 12869 * i40e_clear_rss_lut - clear the rx hash lookup table 12870 * @vsi: the VSI being configured 12871 **/ 12872 static void i40e_clear_rss_lut(struct i40e_vsi *vsi) 12873 { 12874 struct i40e_pf *pf = vsi->back; 12875 struct i40e_hw *hw = &pf->hw; 12876 u16 vf_id = vsi->vf_id; 12877 u8 i; 12878 12879 if (vsi->type == I40E_VSI_MAIN) { 12880 for (i = 0; i <= I40E_PFQF_HLUT_MAX_INDEX; i++) 12881 wr32(hw, I40E_PFQF_HLUT(i), 0); 12882 } else if (vsi->type == I40E_VSI_SRIOV) { 12883 for (i = 0; i <= I40E_VFQF_HLUT_MAX_INDEX; i++) 12884 i40e_write_rx_ctl(hw, I40E_VFQF_HLUT1(i, vf_id), 0); 12885 } else { 12886 dev_err(&pf->pdev->dev, "Cannot set RSS LUT - invalid VSI type\n"); 12887 } 12888 } 12889 12890 /** 12891 * i40e_set_loopback - turn on/off loopback mode on underlying PF 12892 * @vsi: ptr to VSI 12893 * @ena: flag to indicate the on/off setting 12894 */ 12895 static int i40e_set_loopback(struct i40e_vsi *vsi, bool ena) 12896 { 12897 bool if_running = netif_running(vsi->netdev) && 12898 !test_and_set_bit(__I40E_VSI_DOWN, vsi->state); 12899 int ret; 12900 12901 if (if_running) 12902 i40e_down(vsi); 12903 12904 ret = i40e_aq_set_mac_loopback(&vsi->back->hw, ena, NULL); 12905 if (ret) 12906 netdev_err(vsi->netdev, "Failed to toggle loopback state\n"); 12907 if (if_running) 12908 i40e_up(vsi); 12909 12910 return ret; 12911 } 12912 12913 /** 12914 * i40e_set_features - set the netdev feature flags 12915 * @netdev: ptr to the netdev being adjusted 12916 * @features: the feature set that the stack is suggesting 12917 * Note: expects to be called while under rtnl_lock() 12918 **/ 12919 static int i40e_set_features(struct net_device *netdev, 12920 netdev_features_t features) 12921 { 12922 struct i40e_netdev_priv *np = netdev_priv(netdev); 12923 struct i40e_vsi *vsi = np->vsi; 12924 struct i40e_pf *pf = vsi->back; 12925 bool need_reset; 12926 12927 if (features & NETIF_F_RXHASH && !(netdev->features & NETIF_F_RXHASH)) 12928 i40e_pf_config_rss(pf); 12929 else if (!(features & NETIF_F_RXHASH) && 12930 netdev->features & NETIF_F_RXHASH) 12931 i40e_clear_rss_lut(vsi); 12932 12933 if (features & NETIF_F_HW_VLAN_CTAG_RX) 12934 i40e_vlan_stripping_enable(vsi); 12935 else 12936 i40e_vlan_stripping_disable(vsi); 12937 12938 if (!(features & NETIF_F_HW_TC) && 12939 (netdev->features & NETIF_F_HW_TC) && pf->num_cloud_filters) { 12940 dev_err(&pf->pdev->dev, 12941 "Offloaded tc filters active, can't turn hw_tc_offload off"); 12942 return -EINVAL; 12943 } 12944 12945 if (!(features & NETIF_F_HW_L2FW_DOFFLOAD) && vsi->macvlan_cnt) 12946 i40e_del_all_macvlans(vsi); 12947 12948 need_reset = i40e_set_ntuple(pf, features); 12949 12950 if (need_reset) 12951 i40e_do_reset(pf, I40E_PF_RESET_FLAG, true); 12952 12953 if ((features ^ netdev->features) & NETIF_F_LOOPBACK) 12954 return i40e_set_loopback(vsi, !!(features & NETIF_F_LOOPBACK)); 12955 12956 return 0; 12957 } 12958 12959 static int i40e_udp_tunnel_set_port(struct net_device *netdev, 12960 unsigned int table, unsigned int idx, 12961 struct udp_tunnel_info *ti) 12962 { 12963 struct i40e_netdev_priv *np = netdev_priv(netdev); 12964 struct i40e_hw *hw = &np->vsi->back->hw; 12965 u8 type, filter_index; 12966 int ret; 12967 12968 type = ti->type == UDP_TUNNEL_TYPE_VXLAN ? I40E_AQC_TUNNEL_TYPE_VXLAN : 12969 I40E_AQC_TUNNEL_TYPE_NGE; 12970 12971 ret = i40e_aq_add_udp_tunnel(hw, ntohs(ti->port), type, &filter_index, 12972 NULL); 12973 if (ret) { 12974 netdev_info(netdev, "add UDP port failed, err %pe aq_err %s\n", 12975 ERR_PTR(ret), libie_aq_str(hw->aq.asq_last_status)); 12976 return -EIO; 12977 } 12978 12979 udp_tunnel_nic_set_port_priv(netdev, table, idx, filter_index); 12980 return 0; 12981 } 12982 12983 static int i40e_udp_tunnel_unset_port(struct net_device *netdev, 12984 unsigned int table, unsigned int idx, 12985 struct udp_tunnel_info *ti) 12986 { 12987 struct i40e_netdev_priv *np = netdev_priv(netdev); 12988 struct i40e_hw *hw = &np->vsi->back->hw; 12989 int ret; 12990 12991 ret = i40e_aq_del_udp_tunnel(hw, ti->hw_priv, NULL); 12992 if (ret) { 12993 netdev_info(netdev, "delete UDP port failed, err %pe aq_err %s\n", 12994 ERR_PTR(ret), libie_aq_str(hw->aq.asq_last_status)); 12995 return -EIO; 12996 } 12997 12998 return 0; 12999 } 13000 13001 static int i40e_get_phys_port_id(struct net_device *netdev, 13002 struct netdev_phys_item_id *ppid) 13003 { 13004 struct i40e_netdev_priv *np = netdev_priv(netdev); 13005 struct i40e_pf *pf = np->vsi->back; 13006 struct i40e_hw *hw = &pf->hw; 13007 13008 if (!test_bit(I40E_HW_CAP_PORT_ID_VALID, pf->hw.caps)) 13009 return -EOPNOTSUPP; 13010 13011 ppid->id_len = min_t(int, sizeof(hw->mac.port_addr), sizeof(ppid->id)); 13012 memcpy(ppid->id, hw->mac.port_addr, ppid->id_len); 13013 13014 return 0; 13015 } 13016 13017 /** 13018 * i40e_ndo_fdb_add - add an entry to the hardware database 13019 * @ndm: the input from the stack 13020 * @tb: pointer to array of nladdr (unused) 13021 * @dev: the net device pointer 13022 * @addr: the MAC address entry being added 13023 * @vid: VLAN ID 13024 * @flags: instructions from stack about fdb operation 13025 * @notified: whether notification was emitted 13026 * @extack: netlink extended ack, unused currently 13027 */ 13028 static int i40e_ndo_fdb_add(struct ndmsg *ndm, struct nlattr *tb[], 13029 struct net_device *dev, 13030 const unsigned char *addr, u16 vid, 13031 u16 flags, bool *notified, 13032 struct netlink_ext_ack *extack) 13033 { 13034 struct i40e_netdev_priv *np = netdev_priv(dev); 13035 struct i40e_pf *pf = np->vsi->back; 13036 int err = 0; 13037 13038 if (!test_bit(I40E_FLAG_SRIOV_ENA, pf->flags)) 13039 return -EOPNOTSUPP; 13040 13041 if (vid) { 13042 pr_info("%s: vlans aren't supported yet for dev_uc|mc_add()\n", dev->name); 13043 return -EINVAL; 13044 } 13045 13046 /* Hardware does not support aging addresses so if a 13047 * ndm_state is given only allow permanent addresses 13048 */ 13049 if (ndm->ndm_state && !(ndm->ndm_state & NUD_PERMANENT)) { 13050 netdev_info(dev, "FDB only supports static addresses\n"); 13051 return -EINVAL; 13052 } 13053 13054 if (is_unicast_ether_addr(addr) || is_link_local_ether_addr(addr)) 13055 err = dev_uc_add_excl(dev, addr); 13056 else if (is_multicast_ether_addr(addr)) 13057 err = dev_mc_add_excl(dev, addr); 13058 else 13059 err = -EINVAL; 13060 13061 /* Only return duplicate errors if NLM_F_EXCL is set */ 13062 if (err == -EEXIST && !(flags & NLM_F_EXCL)) 13063 err = 0; 13064 13065 return err; 13066 } 13067 13068 /** 13069 * i40e_ndo_bridge_setlink - Set the hardware bridge mode 13070 * @dev: the netdev being configured 13071 * @nlh: RTNL message 13072 * @flags: bridge flags 13073 * @extack: netlink extended ack 13074 * 13075 * Inserts a new hardware bridge if not already created and 13076 * enables the bridging mode requested (VEB or VEPA). If the 13077 * hardware bridge has already been inserted and the request 13078 * is to change the mode then that requires a PF reset to 13079 * allow rebuild of the components with required hardware 13080 * bridge mode enabled. 13081 * 13082 * Note: expects to be called while under rtnl_lock() 13083 **/ 13084 static int i40e_ndo_bridge_setlink(struct net_device *dev, 13085 struct nlmsghdr *nlh, 13086 u16 flags, 13087 struct netlink_ext_ack *extack) 13088 { 13089 struct i40e_netdev_priv *np = netdev_priv(dev); 13090 struct i40e_vsi *vsi = np->vsi; 13091 struct i40e_pf *pf = vsi->back; 13092 struct nlattr *attr, *br_spec; 13093 struct i40e_veb *veb; 13094 int rem; 13095 13096 /* Only for PF VSI for now */ 13097 if (vsi->type != I40E_VSI_MAIN) 13098 return -EOPNOTSUPP; 13099 13100 /* Find the HW bridge for PF VSI */ 13101 veb = i40e_pf_get_veb_by_seid(pf, vsi->uplink_seid); 13102 13103 br_spec = nlmsg_find_attr(nlh, sizeof(struct ifinfomsg), IFLA_AF_SPEC); 13104 if (!br_spec) 13105 return -EINVAL; 13106 13107 nla_for_each_nested_type(attr, IFLA_BRIDGE_MODE, br_spec, rem) { 13108 __u16 mode = nla_get_u16(attr); 13109 13110 if ((mode != BRIDGE_MODE_VEPA) && 13111 (mode != BRIDGE_MODE_VEB)) 13112 return -EINVAL; 13113 13114 /* Insert a new HW bridge */ 13115 if (!veb) { 13116 veb = i40e_veb_setup(pf, vsi->uplink_seid, vsi->seid, 13117 vsi->tc_config.enabled_tc); 13118 if (veb) { 13119 veb->bridge_mode = mode; 13120 i40e_config_bridge_mode(veb); 13121 } else { 13122 /* No Bridge HW offload available */ 13123 return -ENOENT; 13124 } 13125 break; 13126 } else if (mode != veb->bridge_mode) { 13127 /* Existing HW bridge but different mode needs reset */ 13128 veb->bridge_mode = mode; 13129 /* TODO: If no VFs or VMDq VSIs, disallow VEB mode */ 13130 if (mode == BRIDGE_MODE_VEB) 13131 set_bit(I40E_FLAG_VEB_MODE_ENA, pf->flags); 13132 else 13133 clear_bit(I40E_FLAG_VEB_MODE_ENA, pf->flags); 13134 i40e_do_reset(pf, I40E_PF_RESET_FLAG, true); 13135 break; 13136 } 13137 } 13138 13139 return 0; 13140 } 13141 13142 /** 13143 * i40e_ndo_bridge_getlink - Get the hardware bridge mode 13144 * @skb: skb buff 13145 * @pid: process id 13146 * @seq: RTNL message seq # 13147 * @dev: the netdev being configured 13148 * @filter_mask: unused 13149 * @nlflags: netlink flags passed in 13150 * 13151 * Return the mode in which the hardware bridge is operating in 13152 * i.e VEB or VEPA. 13153 **/ 13154 static int i40e_ndo_bridge_getlink(struct sk_buff *skb, u32 pid, u32 seq, 13155 struct net_device *dev, 13156 u32 __always_unused filter_mask, 13157 int nlflags) 13158 { 13159 struct i40e_netdev_priv *np = netdev_priv(dev); 13160 struct i40e_vsi *vsi = np->vsi; 13161 struct i40e_pf *pf = vsi->back; 13162 struct i40e_veb *veb; 13163 13164 /* Only for PF VSI for now */ 13165 if (vsi->type != I40E_VSI_MAIN) 13166 return -EOPNOTSUPP; 13167 13168 /* Find the HW bridge for the PF VSI */ 13169 veb = i40e_pf_get_veb_by_seid(pf, vsi->uplink_seid); 13170 if (!veb) 13171 return 0; 13172 13173 return ndo_dflt_bridge_getlink(skb, pid, seq, dev, veb->bridge_mode, 13174 0, 0, nlflags, filter_mask, NULL); 13175 } 13176 13177 /** 13178 * i40e_features_check - Validate encapsulated packet conforms to limits 13179 * @skb: skb buff 13180 * @dev: This physical port's netdev 13181 * @features: Offload features that the stack believes apply 13182 **/ 13183 static netdev_features_t i40e_features_check(struct sk_buff *skb, 13184 struct net_device *dev, 13185 netdev_features_t features) 13186 { 13187 size_t len; 13188 13189 /* No point in doing any of this if neither checksum nor GSO are 13190 * being requested for this frame. We can rule out both by just 13191 * checking for CHECKSUM_PARTIAL 13192 */ 13193 if (skb->ip_summed != CHECKSUM_PARTIAL) 13194 return features; 13195 13196 /* We cannot support GSO if the MSS is going to be less than 13197 * 64 bytes. If it is then we need to drop support for GSO. 13198 */ 13199 if (skb_is_gso(skb) && (skb_shinfo(skb)->gso_size < 64)) 13200 features &= ~NETIF_F_GSO_MASK; 13201 13202 /* MACLEN can support at most 63 words */ 13203 len = skb_network_offset(skb); 13204 if (len & ~(63 * 2)) 13205 goto out_err; 13206 13207 /* IPLEN and EIPLEN can support at most 127 dwords */ 13208 len = skb_network_header_len(skb); 13209 if (len & ~(127 * 4)) 13210 goto out_err; 13211 13212 if (skb->encapsulation) { 13213 /* L4TUNLEN can support 127 words */ 13214 len = skb_inner_network_header(skb) - skb_transport_header(skb); 13215 if (len & ~(127 * 2)) 13216 goto out_err; 13217 13218 /* IPLEN can support at most 127 dwords */ 13219 len = skb_inner_transport_header(skb) - 13220 skb_inner_network_header(skb); 13221 if (len & ~(127 * 4)) 13222 goto out_err; 13223 } 13224 13225 /* No need to validate L4LEN as TCP is the only protocol with a 13226 * flexible value and we support all possible values supported 13227 * by TCP, which is at most 15 dwords 13228 */ 13229 13230 return features; 13231 out_err: 13232 return features & ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK); 13233 } 13234 13235 /** 13236 * i40e_xdp_setup - add/remove an XDP program 13237 * @vsi: VSI to changed 13238 * @prog: XDP program 13239 * @extack: netlink extended ack 13240 **/ 13241 static int i40e_xdp_setup(struct i40e_vsi *vsi, struct bpf_prog *prog, 13242 struct netlink_ext_ack *extack) 13243 { 13244 int frame_size = i40e_max_vsi_frame_size(vsi, prog); 13245 struct i40e_pf *pf = vsi->back; 13246 struct bpf_prog *old_prog; 13247 bool need_reset; 13248 int i; 13249 13250 /* VSI shall be deleted in a moment, block loading new programs */ 13251 if (prog && test_bit(__I40E_IN_REMOVE, pf->state)) 13252 return -EINVAL; 13253 13254 /* Don't allow frames that span over multiple buffers */ 13255 if (vsi->netdev->mtu > frame_size - I40E_PACKET_HDR_PAD) { 13256 NL_SET_ERR_MSG_MOD(extack, "MTU too large for linear frames and XDP prog does not support frags"); 13257 return -EINVAL; 13258 } 13259 13260 /* When turning XDP on->off/off->on we reset and rebuild the rings. */ 13261 need_reset = (i40e_enabled_xdp_vsi(vsi) != !!prog); 13262 if (need_reset) 13263 i40e_prep_for_reset(pf); 13264 13265 old_prog = xchg(&vsi->xdp_prog, prog); 13266 13267 if (need_reset) { 13268 if (!prog) { 13269 xdp_features_clear_redirect_target(vsi->netdev); 13270 /* Wait until ndo_xsk_wakeup completes. */ 13271 synchronize_rcu(); 13272 } 13273 i40e_reset_and_rebuild(pf, true, true); 13274 } 13275 13276 if (!i40e_enabled_xdp_vsi(vsi) && prog) { 13277 if (i40e_realloc_rx_bi_zc(vsi, true)) 13278 return -ENOMEM; 13279 } else if (i40e_enabled_xdp_vsi(vsi) && !prog) { 13280 if (i40e_realloc_rx_bi_zc(vsi, false)) 13281 return -ENOMEM; 13282 } 13283 13284 for (i = 0; i < vsi->num_queue_pairs; i++) 13285 WRITE_ONCE(vsi->rx_rings[i]->xdp_prog, vsi->xdp_prog); 13286 13287 if (old_prog) 13288 bpf_prog_put(old_prog); 13289 13290 /* Kick start the NAPI context if there is an AF_XDP socket open 13291 * on that queue id. This so that receiving will start. 13292 */ 13293 if (need_reset && prog) { 13294 for (i = 0; i < vsi->num_queue_pairs; i++) 13295 if (vsi->xdp_rings[i]->xsk_pool) 13296 (void)i40e_xsk_wakeup(vsi->netdev, i, 13297 XDP_WAKEUP_RX); 13298 xdp_features_set_redirect_target(vsi->netdev, true); 13299 } 13300 13301 return 0; 13302 } 13303 13304 /** 13305 * i40e_enter_busy_conf - Enters busy config state 13306 * @vsi: vsi 13307 * 13308 * Returns 0 on success, <0 for failure. 13309 **/ 13310 static int i40e_enter_busy_conf(struct i40e_vsi *vsi) 13311 { 13312 struct i40e_pf *pf = vsi->back; 13313 int timeout = 50; 13314 13315 while (test_and_set_bit(__I40E_CONFIG_BUSY, pf->state)) { 13316 timeout--; 13317 if (!timeout) 13318 return -EBUSY; 13319 usleep_range(1000, 2000); 13320 } 13321 13322 return 0; 13323 } 13324 13325 /** 13326 * i40e_exit_busy_conf - Exits busy config state 13327 * @vsi: vsi 13328 **/ 13329 static void i40e_exit_busy_conf(struct i40e_vsi *vsi) 13330 { 13331 struct i40e_pf *pf = vsi->back; 13332 13333 clear_bit(__I40E_CONFIG_BUSY, pf->state); 13334 } 13335 13336 /** 13337 * i40e_queue_pair_reset_stats - Resets all statistics for a queue pair 13338 * @vsi: vsi 13339 * @queue_pair: queue pair 13340 **/ 13341 static void i40e_queue_pair_reset_stats(struct i40e_vsi *vsi, int queue_pair) 13342 { 13343 memset(&vsi->rx_rings[queue_pair]->rx_stats, 0, 13344 sizeof(vsi->rx_rings[queue_pair]->rx_stats)); 13345 memset(&vsi->tx_rings[queue_pair]->stats, 0, 13346 sizeof(vsi->tx_rings[queue_pair]->stats)); 13347 if (i40e_enabled_xdp_vsi(vsi)) { 13348 memset(&vsi->xdp_rings[queue_pair]->stats, 0, 13349 sizeof(vsi->xdp_rings[queue_pair]->stats)); 13350 } 13351 } 13352 13353 /** 13354 * i40e_queue_pair_clean_rings - Cleans all the rings of a queue pair 13355 * @vsi: vsi 13356 * @queue_pair: queue pair 13357 **/ 13358 static void i40e_queue_pair_clean_rings(struct i40e_vsi *vsi, int queue_pair) 13359 { 13360 i40e_clean_tx_ring(vsi->tx_rings[queue_pair]); 13361 if (i40e_enabled_xdp_vsi(vsi)) { 13362 /* Make sure that in-progress ndo_xdp_xmit calls are 13363 * completed. 13364 */ 13365 synchronize_rcu(); 13366 i40e_clean_tx_ring(vsi->xdp_rings[queue_pair]); 13367 } 13368 i40e_clean_rx_ring(vsi->rx_rings[queue_pair]); 13369 } 13370 13371 /** 13372 * i40e_queue_pair_toggle_napi - Enables/disables NAPI for a queue pair 13373 * @vsi: vsi 13374 * @queue_pair: queue pair 13375 * @enable: true for enable, false for disable 13376 **/ 13377 static void i40e_queue_pair_toggle_napi(struct i40e_vsi *vsi, int queue_pair, 13378 bool enable) 13379 { 13380 struct i40e_ring *rxr = vsi->rx_rings[queue_pair]; 13381 struct i40e_q_vector *q_vector = rxr->q_vector; 13382 13383 if (!vsi->netdev) 13384 return; 13385 13386 /* All rings in a qp belong to the same qvector. */ 13387 if (q_vector->rx.ring || q_vector->tx.ring) { 13388 if (enable) 13389 napi_enable(&q_vector->napi); 13390 else 13391 napi_disable(&q_vector->napi); 13392 } 13393 } 13394 13395 /** 13396 * i40e_queue_pair_toggle_rings - Enables/disables all rings for a queue pair 13397 * @vsi: vsi 13398 * @queue_pair: queue pair 13399 * @enable: true for enable, false for disable 13400 * 13401 * Returns 0 on success, <0 on failure. 13402 **/ 13403 static int i40e_queue_pair_toggle_rings(struct i40e_vsi *vsi, int queue_pair, 13404 bool enable) 13405 { 13406 struct i40e_pf *pf = vsi->back; 13407 int pf_q, ret = 0; 13408 13409 pf_q = vsi->base_queue + queue_pair; 13410 ret = i40e_control_wait_tx_q(vsi->seid, pf, pf_q, 13411 false /*is xdp*/, enable); 13412 if (ret) { 13413 dev_info(&pf->pdev->dev, 13414 "VSI seid %d Tx ring %d %sable timeout\n", 13415 vsi->seid, pf_q, (enable ? "en" : "dis")); 13416 return ret; 13417 } 13418 13419 i40e_control_rx_q(pf, pf_q, enable); 13420 ret = i40e_pf_rxq_wait(pf, pf_q, enable); 13421 if (ret) { 13422 dev_info(&pf->pdev->dev, 13423 "VSI seid %d Rx ring %d %sable timeout\n", 13424 vsi->seid, pf_q, (enable ? "en" : "dis")); 13425 return ret; 13426 } 13427 13428 /* Due to HW errata, on Rx disable only, the register can 13429 * indicate done before it really is. Needs 50ms to be sure 13430 */ 13431 if (!enable) 13432 mdelay(50); 13433 13434 if (!i40e_enabled_xdp_vsi(vsi)) 13435 return ret; 13436 13437 ret = i40e_control_wait_tx_q(vsi->seid, pf, 13438 pf_q + vsi->alloc_queue_pairs, 13439 true /*is xdp*/, enable); 13440 if (ret) { 13441 dev_info(&pf->pdev->dev, 13442 "VSI seid %d XDP Tx ring %d %sable timeout\n", 13443 vsi->seid, pf_q, (enable ? "en" : "dis")); 13444 } 13445 13446 return ret; 13447 } 13448 13449 /** 13450 * i40e_queue_pair_enable_irq - Enables interrupts for a queue pair 13451 * @vsi: vsi 13452 * @queue_pair: queue_pair 13453 **/ 13454 static void i40e_queue_pair_enable_irq(struct i40e_vsi *vsi, int queue_pair) 13455 { 13456 struct i40e_ring *rxr = vsi->rx_rings[queue_pair]; 13457 struct i40e_pf *pf = vsi->back; 13458 struct i40e_hw *hw = &pf->hw; 13459 13460 /* All rings in a qp belong to the same qvector. */ 13461 if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags)) 13462 i40e_irq_dynamic_enable(vsi, rxr->q_vector->v_idx); 13463 else 13464 i40e_irq_dynamic_enable_icr0(pf); 13465 13466 i40e_flush(hw); 13467 } 13468 13469 /** 13470 * i40e_queue_pair_disable_irq - Disables interrupts for a queue pair 13471 * @vsi: vsi 13472 * @queue_pair: queue_pair 13473 **/ 13474 static void i40e_queue_pair_disable_irq(struct i40e_vsi *vsi, int queue_pair) 13475 { 13476 struct i40e_ring *rxr = vsi->rx_rings[queue_pair]; 13477 struct i40e_pf *pf = vsi->back; 13478 struct i40e_hw *hw = &pf->hw; 13479 13480 /* For simplicity, instead of removing the qp interrupt causes 13481 * from the interrupt linked list, we simply disable the interrupt, and 13482 * leave the list intact. 13483 * 13484 * All rings in a qp belong to the same qvector. 13485 */ 13486 if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags)) { 13487 u32 intpf = vsi->base_vector + rxr->q_vector->v_idx; 13488 13489 wr32(hw, I40E_PFINT_DYN_CTLN(intpf - 1), 0); 13490 i40e_flush(hw); 13491 synchronize_irq(pf->msix_entries[intpf].vector); 13492 } else { 13493 /* Legacy and MSI mode - this stops all interrupt handling */ 13494 wr32(hw, I40E_PFINT_ICR0_ENA, 0); 13495 wr32(hw, I40E_PFINT_DYN_CTL0, 0); 13496 i40e_flush(hw); 13497 synchronize_irq(pf->pdev->irq); 13498 } 13499 } 13500 13501 /** 13502 * i40e_queue_pair_disable - Disables a queue pair 13503 * @vsi: vsi 13504 * @queue_pair: queue pair 13505 * 13506 * Returns 0 on success, <0 on failure. 13507 **/ 13508 int i40e_queue_pair_disable(struct i40e_vsi *vsi, int queue_pair) 13509 { 13510 int err; 13511 13512 err = i40e_enter_busy_conf(vsi); 13513 if (err) 13514 return err; 13515 13516 i40e_queue_pair_disable_irq(vsi, queue_pair); 13517 i40e_queue_pair_toggle_napi(vsi, queue_pair, false /* off */); 13518 err = i40e_queue_pair_toggle_rings(vsi, queue_pair, false /* off */); 13519 i40e_clean_rx_ring(vsi->rx_rings[queue_pair]); 13520 i40e_queue_pair_clean_rings(vsi, queue_pair); 13521 i40e_queue_pair_reset_stats(vsi, queue_pair); 13522 13523 return err; 13524 } 13525 13526 /** 13527 * i40e_queue_pair_enable - Enables a queue pair 13528 * @vsi: vsi 13529 * @queue_pair: queue pair 13530 * 13531 * Returns 0 on success, <0 on failure. 13532 **/ 13533 int i40e_queue_pair_enable(struct i40e_vsi *vsi, int queue_pair) 13534 { 13535 int err; 13536 13537 err = i40e_configure_tx_ring(vsi->tx_rings[queue_pair]); 13538 if (err) 13539 return err; 13540 13541 if (i40e_enabled_xdp_vsi(vsi)) { 13542 err = i40e_configure_tx_ring(vsi->xdp_rings[queue_pair]); 13543 if (err) 13544 return err; 13545 } 13546 13547 err = i40e_configure_rx_ring(vsi->rx_rings[queue_pair]); 13548 if (err) 13549 return err; 13550 13551 err = i40e_queue_pair_toggle_rings(vsi, queue_pair, true /* on */); 13552 i40e_queue_pair_toggle_napi(vsi, queue_pair, true /* on */); 13553 i40e_queue_pair_enable_irq(vsi, queue_pair); 13554 13555 i40e_exit_busy_conf(vsi); 13556 13557 return err; 13558 } 13559 13560 /** 13561 * i40e_xdp - implements ndo_bpf for i40e 13562 * @dev: netdevice 13563 * @xdp: XDP command 13564 **/ 13565 static int i40e_xdp(struct net_device *dev, 13566 struct netdev_bpf *xdp) 13567 { 13568 struct i40e_netdev_priv *np = netdev_priv(dev); 13569 struct i40e_vsi *vsi = np->vsi; 13570 13571 if (vsi->type != I40E_VSI_MAIN) 13572 return -EINVAL; 13573 13574 switch (xdp->command) { 13575 case XDP_SETUP_PROG: 13576 return i40e_xdp_setup(vsi, xdp->prog, xdp->extack); 13577 case XDP_SETUP_XSK_POOL: 13578 return i40e_xsk_pool_setup(vsi, xdp->xsk.pool, 13579 xdp->xsk.queue_id); 13580 default: 13581 return -EINVAL; 13582 } 13583 } 13584 13585 static const struct net_device_ops i40e_netdev_ops = { 13586 .ndo_open = i40e_open, 13587 .ndo_stop = i40e_close, 13588 .ndo_start_xmit = i40e_lan_xmit_frame, 13589 .ndo_get_stats64 = i40e_get_netdev_stats_struct, 13590 .ndo_set_rx_mode = i40e_set_rx_mode, 13591 .ndo_validate_addr = eth_validate_addr, 13592 .ndo_set_mac_address = i40e_set_mac, 13593 .ndo_change_mtu = i40e_change_mtu, 13594 .ndo_tx_timeout = i40e_tx_timeout, 13595 .ndo_vlan_rx_add_vid = i40e_vlan_rx_add_vid, 13596 .ndo_vlan_rx_kill_vid = i40e_vlan_rx_kill_vid, 13597 #ifdef CONFIG_NET_POLL_CONTROLLER 13598 .ndo_poll_controller = i40e_netpoll, 13599 #endif 13600 .ndo_setup_tc = __i40e_setup_tc, 13601 .ndo_select_queue = i40e_lan_select_queue, 13602 .ndo_set_features = i40e_set_features, 13603 .ndo_set_vf_mac = i40e_ndo_set_vf_mac, 13604 .ndo_set_vf_vlan = i40e_ndo_set_vf_port_vlan, 13605 .ndo_get_vf_stats = i40e_get_vf_stats, 13606 .ndo_set_vf_rate = i40e_ndo_set_vf_bw, 13607 .ndo_get_vf_config = i40e_ndo_get_vf_config, 13608 .ndo_set_vf_link_state = i40e_ndo_set_vf_link_state, 13609 .ndo_set_vf_spoofchk = i40e_ndo_set_vf_spoofchk, 13610 .ndo_set_vf_trust = i40e_ndo_set_vf_trust, 13611 .ndo_get_phys_port_id = i40e_get_phys_port_id, 13612 .ndo_fdb_add = i40e_ndo_fdb_add, 13613 .ndo_features_check = i40e_features_check, 13614 .ndo_bridge_getlink = i40e_ndo_bridge_getlink, 13615 .ndo_bridge_setlink = i40e_ndo_bridge_setlink, 13616 .ndo_bpf = i40e_xdp, 13617 .ndo_xdp_xmit = i40e_xdp_xmit, 13618 .ndo_xsk_wakeup = i40e_xsk_wakeup, 13619 .ndo_dfwd_add_station = i40e_fwd_add, 13620 .ndo_dfwd_del_station = i40e_fwd_del, 13621 .ndo_hwtstamp_get = i40e_ptp_hwtstamp_get, 13622 .ndo_hwtstamp_set = i40e_ptp_hwtstamp_set, 13623 }; 13624 13625 /** 13626 * i40e_config_netdev - Setup the netdev flags 13627 * @vsi: the VSI being configured 13628 * 13629 * Returns 0 on success, negative value on failure 13630 **/ 13631 static int i40e_config_netdev(struct i40e_vsi *vsi) 13632 { 13633 struct i40e_pf *pf = vsi->back; 13634 struct i40e_hw *hw = &pf->hw; 13635 struct i40e_netdev_priv *np; 13636 struct net_device *netdev; 13637 u8 broadcast[ETH_ALEN]; 13638 u8 mac_addr[ETH_ALEN]; 13639 int etherdev_size; 13640 netdev_features_t hw_enc_features; 13641 netdev_features_t hw_features; 13642 13643 etherdev_size = sizeof(struct i40e_netdev_priv); 13644 netdev = alloc_etherdev_mq(etherdev_size, vsi->alloc_queue_pairs); 13645 if (!netdev) 13646 return -ENOMEM; 13647 13648 vsi->netdev = netdev; 13649 np = netdev_priv(netdev); 13650 np->vsi = vsi; 13651 13652 hw_enc_features = NETIF_F_SG | 13653 NETIF_F_HW_CSUM | 13654 NETIF_F_HIGHDMA | 13655 NETIF_F_SOFT_FEATURES | 13656 NETIF_F_TSO | 13657 NETIF_F_TSO_ECN | 13658 NETIF_F_TSO6 | 13659 NETIF_F_GSO_GRE | 13660 NETIF_F_GSO_GRE_CSUM | 13661 NETIF_F_GSO_PARTIAL | 13662 NETIF_F_GSO_IPXIP4 | 13663 NETIF_F_GSO_IPXIP6 | 13664 NETIF_F_GSO_UDP_TUNNEL | 13665 NETIF_F_GSO_UDP_TUNNEL_CSUM | 13666 NETIF_F_GSO_UDP_L4 | 13667 NETIF_F_SCTP_CRC | 13668 NETIF_F_RXHASH | 13669 NETIF_F_RXCSUM | 13670 0; 13671 13672 if (!test_bit(I40E_HW_CAP_OUTER_UDP_CSUM, pf->hw.caps)) 13673 netdev->gso_partial_features |= NETIF_F_GSO_UDP_TUNNEL_CSUM; 13674 13675 netdev->udp_tunnel_nic_info = &pf->udp_tunnel_nic; 13676 13677 netdev->gso_partial_features |= NETIF_F_GSO_GRE_CSUM; 13678 13679 netdev->hw_enc_features |= hw_enc_features; 13680 13681 /* record features VLANs can make use of */ 13682 netdev->vlan_features |= hw_enc_features | NETIF_F_TSO_MANGLEID; 13683 13684 #define I40E_GSO_PARTIAL_FEATURES (NETIF_F_GSO_GRE | \ 13685 NETIF_F_GSO_GRE_CSUM | \ 13686 NETIF_F_GSO_IPXIP4 | \ 13687 NETIF_F_GSO_IPXIP6 | \ 13688 NETIF_F_GSO_UDP_TUNNEL | \ 13689 NETIF_F_GSO_UDP_TUNNEL_CSUM) 13690 13691 netdev->gso_partial_features = I40E_GSO_PARTIAL_FEATURES; 13692 netdev->features |= NETIF_F_GSO_PARTIAL | 13693 I40E_GSO_PARTIAL_FEATURES; 13694 13695 netdev->mpls_features |= NETIF_F_SG; 13696 netdev->mpls_features |= NETIF_F_HW_CSUM; 13697 netdev->mpls_features |= NETIF_F_TSO; 13698 netdev->mpls_features |= NETIF_F_TSO6; 13699 netdev->mpls_features |= I40E_GSO_PARTIAL_FEATURES; 13700 13701 /* enable macvlan offloads */ 13702 netdev->hw_features |= NETIF_F_HW_L2FW_DOFFLOAD; 13703 13704 hw_features = hw_enc_features | 13705 NETIF_F_HW_VLAN_CTAG_TX | 13706 NETIF_F_HW_VLAN_CTAG_RX; 13707 13708 if (!test_bit(I40E_FLAG_MFP_ENA, pf->flags)) 13709 hw_features |= NETIF_F_NTUPLE | NETIF_F_HW_TC; 13710 13711 netdev->hw_features |= hw_features | NETIF_F_LOOPBACK; 13712 13713 netdev->features |= hw_features | NETIF_F_HW_VLAN_CTAG_FILTER; 13714 netdev->hw_enc_features |= NETIF_F_TSO_MANGLEID; 13715 13716 netdev->features &= ~NETIF_F_HW_TC; 13717 13718 if (vsi->type == I40E_VSI_MAIN) { 13719 SET_NETDEV_DEV(netdev, &pf->pdev->dev); 13720 ether_addr_copy(mac_addr, hw->mac.perm_addr); 13721 /* The following steps are necessary for two reasons. First, 13722 * some older NVM configurations load a default MAC-VLAN 13723 * filter that will accept any tagged packet, and we want to 13724 * replace this with a normal filter. Additionally, it is 13725 * possible our MAC address was provided by the platform using 13726 * Open Firmware or similar. 13727 * 13728 * Thus, we need to remove the default filter and install one 13729 * specific to the MAC address. 13730 */ 13731 i40e_rm_default_mac_filter(vsi, mac_addr); 13732 spin_lock_bh(&vsi->mac_filter_hash_lock); 13733 i40e_add_mac_filter(vsi, mac_addr); 13734 spin_unlock_bh(&vsi->mac_filter_hash_lock); 13735 13736 netdev->xdp_features = NETDEV_XDP_ACT_BASIC | 13737 NETDEV_XDP_ACT_REDIRECT | 13738 NETDEV_XDP_ACT_XSK_ZEROCOPY | 13739 NETDEV_XDP_ACT_RX_SG; 13740 netdev->xdp_zc_max_segs = I40E_MAX_BUFFER_TXD; 13741 } else { 13742 /* Relate the VSI_VMDQ name to the VSI_MAIN name. Note that we 13743 * are still limited by IFNAMSIZ, but we're adding 'v%d\0' to 13744 * the end, which is 4 bytes long, so force truncation of the 13745 * original name by IFNAMSIZ - 4 13746 */ 13747 struct i40e_vsi *main_vsi = i40e_pf_get_main_vsi(pf); 13748 13749 snprintf(netdev->name, IFNAMSIZ, "%.*sv%%d", IFNAMSIZ - 4, 13750 main_vsi->netdev->name); 13751 eth_random_addr(mac_addr); 13752 13753 spin_lock_bh(&vsi->mac_filter_hash_lock); 13754 i40e_add_mac_filter(vsi, mac_addr); 13755 spin_unlock_bh(&vsi->mac_filter_hash_lock); 13756 } 13757 13758 /* Add the broadcast filter so that we initially will receive 13759 * broadcast packets. Note that when a new VLAN is first added the 13760 * driver will convert all filters marked I40E_VLAN_ANY into VLAN 13761 * specific filters as part of transitioning into "vlan" operation. 13762 * When more VLANs are added, the driver will copy each existing MAC 13763 * filter and add it for the new VLAN. 13764 * 13765 * Broadcast filters are handled specially by 13766 * i40e_sync_filters_subtask, as the driver must to set the broadcast 13767 * promiscuous bit instead of adding this directly as a MAC/VLAN 13768 * filter. The subtask will update the correct broadcast promiscuous 13769 * bits as VLANs become active or inactive. 13770 */ 13771 eth_broadcast_addr(broadcast); 13772 spin_lock_bh(&vsi->mac_filter_hash_lock); 13773 i40e_add_mac_filter(vsi, broadcast); 13774 spin_unlock_bh(&vsi->mac_filter_hash_lock); 13775 13776 eth_hw_addr_set(netdev, mac_addr); 13777 ether_addr_copy(netdev->perm_addr, mac_addr); 13778 13779 /* i40iw_net_event() reads 16 bytes from neigh->primary_key */ 13780 netdev->neigh_priv_len = sizeof(u32) * 4; 13781 13782 netdev->priv_flags |= IFF_UNICAST_FLT; 13783 /* Setup netdev TC information */ 13784 i40e_vsi_config_netdev_tc(vsi, vsi->tc_config.enabled_tc); 13785 13786 netdev->netdev_ops = &i40e_netdev_ops; 13787 netdev->watchdog_timeo = 5 * HZ; 13788 i40e_set_ethtool_ops(netdev); 13789 13790 /* MTU range: 68 - 9706 */ 13791 netdev->min_mtu = ETH_MIN_MTU; 13792 netdev->max_mtu = I40E_MAX_RXBUFFER - I40E_PACKET_HDR_PAD; 13793 13794 return 0; 13795 } 13796 13797 /** 13798 * i40e_vsi_delete - Delete a VSI from the switch 13799 * @vsi: the VSI being removed 13800 * 13801 * Returns 0 on success, negative value on failure 13802 **/ 13803 static void i40e_vsi_delete(struct i40e_vsi *vsi) 13804 { 13805 /* remove default VSI is not allowed */ 13806 if (vsi == vsi->back->vsi[vsi->back->lan_vsi]) 13807 return; 13808 13809 i40e_aq_delete_element(&vsi->back->hw, vsi->seid, NULL); 13810 } 13811 13812 /** 13813 * i40e_is_vsi_uplink_mode_veb - Check if the VSI's uplink bridge mode is VEB 13814 * @vsi: the VSI being queried 13815 * 13816 * Returns 1 if HW bridge mode is VEB and return 0 in case of VEPA mode 13817 **/ 13818 int i40e_is_vsi_uplink_mode_veb(struct i40e_vsi *vsi) 13819 { 13820 struct i40e_veb *veb; 13821 struct i40e_pf *pf = vsi->back; 13822 13823 /* Uplink is not a bridge so default to VEB */ 13824 if (vsi->veb_idx >= I40E_MAX_VEB) 13825 return 1; 13826 13827 veb = pf->veb[vsi->veb_idx]; 13828 if (!veb) { 13829 dev_info(&pf->pdev->dev, 13830 "There is no veb associated with the bridge\n"); 13831 return -ENOENT; 13832 } 13833 13834 /* Uplink is a bridge in VEPA mode */ 13835 if (veb->bridge_mode & BRIDGE_MODE_VEPA) { 13836 return 0; 13837 } else { 13838 /* Uplink is a bridge in VEB mode */ 13839 return 1; 13840 } 13841 13842 /* VEPA is now default bridge, so return 0 */ 13843 return 0; 13844 } 13845 13846 /** 13847 * i40e_add_vsi - Add a VSI to the switch 13848 * @vsi: the VSI being configured 13849 * 13850 * This initializes a VSI context depending on the VSI type to be added and 13851 * passes it down to the add_vsi aq command. 13852 **/ 13853 static int i40e_add_vsi(struct i40e_vsi *vsi) 13854 { 13855 int ret = -ENODEV; 13856 struct i40e_pf *pf = vsi->back; 13857 struct i40e_hw *hw = &pf->hw; 13858 struct i40e_vsi_context ctxt; 13859 struct i40e_mac_filter *f; 13860 struct hlist_node *h; 13861 int bkt; 13862 13863 u8 enabled_tc = 0x1; /* TC0 enabled */ 13864 int f_count = 0; 13865 13866 memset(&ctxt, 0, sizeof(ctxt)); 13867 switch (vsi->type) { 13868 case I40E_VSI_MAIN: 13869 /* The PF's main VSI is already setup as part of the 13870 * device initialization, so we'll not bother with 13871 * the add_vsi call, but we will retrieve the current 13872 * VSI context. 13873 */ 13874 ctxt.seid = pf->main_vsi_seid; 13875 ctxt.pf_num = pf->hw.pf_id; 13876 ctxt.vf_num = 0; 13877 ret = i40e_aq_get_vsi_params(&pf->hw, &ctxt, NULL); 13878 ctxt.flags = I40E_AQ_VSI_TYPE_PF; 13879 if (ret) { 13880 dev_info(&pf->pdev->dev, 13881 "couldn't get PF vsi config, err %pe aq_err %s\n", 13882 ERR_PTR(ret), 13883 libie_aq_str(pf->hw.aq.asq_last_status)); 13884 return -ENOENT; 13885 } 13886 vsi->info = ctxt.info; 13887 vsi->info.valid_sections = 0; 13888 13889 vsi->seid = ctxt.seid; 13890 vsi->id = ctxt.vsi_number; 13891 13892 enabled_tc = i40e_pf_get_tc_map(pf); 13893 13894 /* Source pruning is enabled by default, so the flag is 13895 * negative logic - if it's set, we need to fiddle with 13896 * the VSI to disable source pruning. 13897 */ 13898 if (test_bit(I40E_FLAG_SOURCE_PRUNING_DIS, pf->flags)) { 13899 memset(&ctxt, 0, sizeof(ctxt)); 13900 ctxt.seid = pf->main_vsi_seid; 13901 ctxt.pf_num = pf->hw.pf_id; 13902 ctxt.vf_num = 0; 13903 ctxt.info.valid_sections |= 13904 cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID); 13905 ctxt.info.switch_id = 13906 cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_LOCAL_LB); 13907 ret = i40e_aq_update_vsi_params(hw, &ctxt, NULL); 13908 if (ret) { 13909 dev_info(&pf->pdev->dev, 13910 "update vsi failed, err %d aq_err %s\n", 13911 ret, 13912 libie_aq_str(pf->hw.aq.asq_last_status)); 13913 ret = -ENOENT; 13914 goto err; 13915 } 13916 } 13917 13918 /* MFP mode setup queue map and update VSI */ 13919 if (test_bit(I40E_FLAG_MFP_ENA, pf->flags) && 13920 !(pf->hw.func_caps.iscsi)) { /* NIC type PF */ 13921 memset(&ctxt, 0, sizeof(ctxt)); 13922 ctxt.seid = pf->main_vsi_seid; 13923 ctxt.pf_num = pf->hw.pf_id; 13924 ctxt.vf_num = 0; 13925 i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, false); 13926 ret = i40e_aq_update_vsi_params(hw, &ctxt, NULL); 13927 if (ret) { 13928 dev_info(&pf->pdev->dev, 13929 "update vsi failed, err %pe aq_err %s\n", 13930 ERR_PTR(ret), 13931 libie_aq_str(pf->hw.aq.asq_last_status)); 13932 ret = -ENOENT; 13933 goto err; 13934 } 13935 /* update the local VSI info queue map */ 13936 i40e_vsi_update_queue_map(vsi, &ctxt); 13937 vsi->info.valid_sections = 0; 13938 } else { 13939 /* Default/Main VSI is only enabled for TC0 13940 * reconfigure it to enable all TCs that are 13941 * available on the port in SFP mode. 13942 * For MFP case the iSCSI PF would use this 13943 * flow to enable LAN+iSCSI TC. 13944 */ 13945 ret = i40e_vsi_config_tc(vsi, enabled_tc); 13946 if (ret) { 13947 /* Single TC condition is not fatal, 13948 * message and continue 13949 */ 13950 dev_info(&pf->pdev->dev, 13951 "failed to configure TCs for main VSI tc_map 0x%08x, err %pe aq_err %s\n", 13952 enabled_tc, 13953 ERR_PTR(ret), 13954 libie_aq_str(pf->hw.aq.asq_last_status)); 13955 } 13956 } 13957 break; 13958 13959 case I40E_VSI_FDIR: 13960 ctxt.pf_num = hw->pf_id; 13961 ctxt.vf_num = 0; 13962 ctxt.uplink_seid = vsi->uplink_seid; 13963 ctxt.connection_type = I40E_AQ_VSI_CONN_TYPE_NORMAL; 13964 ctxt.flags = I40E_AQ_VSI_TYPE_PF; 13965 if (test_bit(I40E_FLAG_VEB_MODE_ENA, pf->flags) && 13966 (i40e_is_vsi_uplink_mode_veb(vsi))) { 13967 ctxt.info.valid_sections |= 13968 cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID); 13969 ctxt.info.switch_id = 13970 cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB); 13971 } 13972 i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, true); 13973 break; 13974 13975 case I40E_VSI_VMDQ2: 13976 ctxt.pf_num = hw->pf_id; 13977 ctxt.vf_num = 0; 13978 ctxt.uplink_seid = vsi->uplink_seid; 13979 ctxt.connection_type = I40E_AQ_VSI_CONN_TYPE_NORMAL; 13980 ctxt.flags = I40E_AQ_VSI_TYPE_VMDQ2; 13981 13982 /* This VSI is connected to VEB so the switch_id 13983 * should be set to zero by default. 13984 */ 13985 if (i40e_is_vsi_uplink_mode_veb(vsi)) { 13986 ctxt.info.valid_sections |= 13987 cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID); 13988 ctxt.info.switch_id = 13989 cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB); 13990 } 13991 13992 /* Setup the VSI tx/rx queue map for TC0 only for now */ 13993 i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, true); 13994 break; 13995 13996 case I40E_VSI_SRIOV: 13997 ctxt.pf_num = hw->pf_id; 13998 ctxt.vf_num = vsi->vf_id + hw->func_caps.vf_base_id; 13999 ctxt.uplink_seid = vsi->uplink_seid; 14000 ctxt.connection_type = I40E_AQ_VSI_CONN_TYPE_NORMAL; 14001 ctxt.flags = I40E_AQ_VSI_TYPE_VF; 14002 14003 /* This VSI is connected to VEB so the switch_id 14004 * should be set to zero by default. 14005 */ 14006 if (i40e_is_vsi_uplink_mode_veb(vsi)) { 14007 ctxt.info.valid_sections |= 14008 cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID); 14009 ctxt.info.switch_id = 14010 cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB); 14011 } 14012 14013 if (test_bit(I40E_FLAG_IWARP_ENA, vsi->back->flags)) { 14014 ctxt.info.valid_sections |= 14015 cpu_to_le16(I40E_AQ_VSI_PROP_QUEUE_OPT_VALID); 14016 ctxt.info.queueing_opt_flags |= 14017 (I40E_AQ_VSI_QUE_OPT_TCP_ENA | 14018 I40E_AQ_VSI_QUE_OPT_RSS_LUT_VSI); 14019 } 14020 14021 ctxt.info.valid_sections |= cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID); 14022 ctxt.info.port_vlan_flags |= I40E_AQ_VSI_PVLAN_MODE_ALL; 14023 if (pf->vf[vsi->vf_id].spoofchk) { 14024 ctxt.info.valid_sections |= 14025 cpu_to_le16(I40E_AQ_VSI_PROP_SECURITY_VALID); 14026 ctxt.info.sec_flags |= 14027 (I40E_AQ_VSI_SEC_FLAG_ENABLE_VLAN_CHK | 14028 I40E_AQ_VSI_SEC_FLAG_ENABLE_MAC_CHK); 14029 } 14030 /* Setup the VSI tx/rx queue map for TC0 only for now */ 14031 i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, true); 14032 break; 14033 14034 case I40E_VSI_IWARP: 14035 /* send down message to iWARP */ 14036 break; 14037 14038 default: 14039 return -ENODEV; 14040 } 14041 14042 if (vsi->type != I40E_VSI_MAIN) { 14043 ret = i40e_aq_add_vsi(hw, &ctxt, NULL); 14044 if (ret) { 14045 dev_info(&vsi->back->pdev->dev, 14046 "add vsi failed, err %pe aq_err %s\n", 14047 ERR_PTR(ret), 14048 libie_aq_str(pf->hw.aq.asq_last_status)); 14049 ret = -ENOENT; 14050 goto err; 14051 } 14052 vsi->info = ctxt.info; 14053 vsi->info.valid_sections = 0; 14054 vsi->seid = ctxt.seid; 14055 vsi->id = ctxt.vsi_number; 14056 } 14057 14058 spin_lock_bh(&vsi->mac_filter_hash_lock); 14059 vsi->active_filters = 0; 14060 /* If macvlan filters already exist, force them to get loaded */ 14061 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) { 14062 f->state = I40E_FILTER_NEW; 14063 f_count++; 14064 } 14065 spin_unlock_bh(&vsi->mac_filter_hash_lock); 14066 clear_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state); 14067 14068 if (f_count) { 14069 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED; 14070 set_bit(__I40E_MACVLAN_SYNC_PENDING, pf->state); 14071 } 14072 14073 /* Update VSI BW information */ 14074 ret = i40e_vsi_get_bw_info(vsi); 14075 if (ret) { 14076 dev_info(&pf->pdev->dev, 14077 "couldn't get vsi bw info, err %pe aq_err %s\n", 14078 ERR_PTR(ret), libie_aq_str(pf->hw.aq.asq_last_status)); 14079 /* VSI is already added so not tearing that up */ 14080 ret = 0; 14081 } 14082 14083 err: 14084 return ret; 14085 } 14086 14087 /** 14088 * i40e_vsi_release - Delete a VSI and free its resources 14089 * @vsi: the VSI being removed 14090 * 14091 * Returns 0 on success or < 0 on error 14092 **/ 14093 int i40e_vsi_release(struct i40e_vsi *vsi) 14094 { 14095 struct i40e_mac_filter *f; 14096 struct hlist_node *h; 14097 struct i40e_veb *veb; 14098 struct i40e_pf *pf; 14099 u16 uplink_seid; 14100 int i, n, bkt; 14101 14102 pf = vsi->back; 14103 14104 /* release of a VEB-owner or last VSI is not allowed */ 14105 if (vsi->flags & I40E_VSI_FLAG_VEB_OWNER) { 14106 dev_info(&pf->pdev->dev, "VSI %d has existing VEB %d\n", 14107 vsi->seid, vsi->uplink_seid); 14108 return -ENODEV; 14109 } 14110 if (vsi->type == I40E_VSI_MAIN && !test_bit(__I40E_DOWN, pf->state)) { 14111 dev_info(&pf->pdev->dev, "Can't remove PF VSI\n"); 14112 return -ENODEV; 14113 } 14114 set_bit(__I40E_VSI_RELEASING, vsi->state); 14115 uplink_seid = vsi->uplink_seid; 14116 14117 if (vsi->type != I40E_VSI_SRIOV) { 14118 if (vsi->netdev_registered) { 14119 vsi->netdev_registered = false; 14120 if (vsi->netdev) { 14121 /* results in a call to i40e_close() */ 14122 unregister_netdev(vsi->netdev); 14123 } 14124 } else { 14125 i40e_vsi_close(vsi); 14126 } 14127 i40e_vsi_disable_irq(vsi); 14128 } 14129 14130 if (vsi->type == I40E_VSI_MAIN) 14131 i40e_devlink_destroy_port(pf); 14132 14133 spin_lock_bh(&vsi->mac_filter_hash_lock); 14134 14135 /* clear the sync flag on all filters */ 14136 if (vsi->netdev) { 14137 __dev_uc_unsync(vsi->netdev, NULL); 14138 __dev_mc_unsync(vsi->netdev, NULL); 14139 } 14140 14141 /* make sure any remaining filters are marked for deletion */ 14142 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) 14143 __i40e_del_filter(vsi, f); 14144 14145 spin_unlock_bh(&vsi->mac_filter_hash_lock); 14146 14147 i40e_sync_vsi_filters(vsi); 14148 14149 i40e_vsi_delete(vsi); 14150 i40e_vsi_free_q_vectors(vsi); 14151 if (vsi->netdev) { 14152 free_netdev(vsi->netdev); 14153 vsi->netdev = NULL; 14154 } 14155 i40e_vsi_clear_rings(vsi); 14156 i40e_vsi_clear(vsi); 14157 14158 /* If this was the last thing on the VEB, except for the 14159 * controlling VSI, remove the VEB, which puts the controlling 14160 * VSI onto the uplink port. 14161 * 14162 * Well, okay, there's one more exception here: don't remove 14163 * the floating VEBs yet. We'll wait for an explicit remove request 14164 * from up the network stack. 14165 */ 14166 veb = i40e_pf_get_veb_by_seid(pf, uplink_seid); 14167 if (veb && veb->uplink_seid) { 14168 n = 0; 14169 14170 /* Count non-controlling VSIs present on the VEB */ 14171 i40e_pf_for_each_vsi(pf, i, vsi) 14172 if (vsi->uplink_seid == uplink_seid && 14173 (vsi->flags & I40E_VSI_FLAG_VEB_OWNER) == 0) 14174 n++; 14175 14176 /* If there is no VSI except the control one then release 14177 * the VEB and put the control VSI onto VEB uplink. 14178 */ 14179 if (!n) 14180 i40e_veb_release(veb); 14181 } 14182 14183 return 0; 14184 } 14185 14186 /** 14187 * i40e_vsi_setup_vectors - Set up the q_vectors for the given VSI 14188 * @vsi: ptr to the VSI 14189 * 14190 * This should only be called after i40e_vsi_mem_alloc() which allocates the 14191 * corresponding SW VSI structure and initializes num_queue_pairs for the 14192 * newly allocated VSI. 14193 * 14194 * Returns 0 on success or negative on failure 14195 **/ 14196 static int i40e_vsi_setup_vectors(struct i40e_vsi *vsi) 14197 { 14198 int ret = -ENOENT; 14199 struct i40e_pf *pf = vsi->back; 14200 14201 if (vsi->q_vectors[0]) { 14202 dev_info(&pf->pdev->dev, "VSI %d has existing q_vectors\n", 14203 vsi->seid); 14204 return -EEXIST; 14205 } 14206 14207 if (vsi->base_vector) { 14208 dev_info(&pf->pdev->dev, "VSI %d has non-zero base vector %d\n", 14209 vsi->seid, vsi->base_vector); 14210 return -EEXIST; 14211 } 14212 14213 ret = i40e_vsi_alloc_q_vectors(vsi); 14214 if (ret) { 14215 dev_info(&pf->pdev->dev, 14216 "failed to allocate %d q_vector for VSI %d, ret=%d\n", 14217 vsi->num_q_vectors, vsi->seid, ret); 14218 vsi->num_q_vectors = 0; 14219 goto vector_setup_out; 14220 } 14221 14222 /* In Legacy mode, we do not have to get any other vector since we 14223 * piggyback on the misc/ICR0 for queue interrupts. 14224 */ 14225 if (!test_bit(I40E_FLAG_MSIX_ENA, pf->flags)) 14226 return ret; 14227 if (vsi->num_q_vectors) 14228 vsi->base_vector = i40e_get_lump(pf, pf->irq_pile, 14229 vsi->num_q_vectors, vsi->idx); 14230 if (vsi->base_vector < 0) { 14231 dev_info(&pf->pdev->dev, 14232 "failed to get tracking for %d vectors for VSI %d, err=%d\n", 14233 vsi->num_q_vectors, vsi->seid, vsi->base_vector); 14234 i40e_vsi_free_q_vectors(vsi); 14235 ret = -ENOENT; 14236 goto vector_setup_out; 14237 } 14238 14239 vector_setup_out: 14240 return ret; 14241 } 14242 14243 /** 14244 * i40e_vsi_reinit_setup - return and reallocate resources for a VSI 14245 * @vsi: pointer to the vsi. 14246 * 14247 * This re-allocates a vsi's queue resources. 14248 * 14249 * Returns pointer to the successfully allocated and configured VSI sw struct 14250 * on success, otherwise returns NULL on failure. 14251 **/ 14252 static struct i40e_vsi *i40e_vsi_reinit_setup(struct i40e_vsi *vsi) 14253 { 14254 struct i40e_vsi *main_vsi; 14255 u16 alloc_queue_pairs; 14256 struct i40e_pf *pf; 14257 int ret; 14258 14259 if (!vsi) 14260 return NULL; 14261 14262 pf = vsi->back; 14263 14264 i40e_put_lump(pf->qp_pile, vsi->base_queue, vsi->idx); 14265 i40e_vsi_clear_rings(vsi); 14266 14267 i40e_vsi_free_arrays(vsi, false); 14268 i40e_set_num_rings_in_vsi(vsi); 14269 ret = i40e_vsi_alloc_arrays(vsi, false); 14270 if (ret) 14271 goto err_vsi; 14272 14273 alloc_queue_pairs = vsi->alloc_queue_pairs * 14274 (i40e_enabled_xdp_vsi(vsi) ? 2 : 1); 14275 14276 ret = i40e_get_lump(pf, pf->qp_pile, alloc_queue_pairs, vsi->idx); 14277 if (ret < 0) { 14278 dev_info(&pf->pdev->dev, 14279 "failed to get tracking for %d queues for VSI %d err %d\n", 14280 alloc_queue_pairs, vsi->seid, ret); 14281 goto err_vsi; 14282 } 14283 vsi->base_queue = ret; 14284 14285 /* Update the FW view of the VSI. Force a reset of TC and queue 14286 * layout configurations. 14287 */ 14288 main_vsi = i40e_pf_get_main_vsi(pf); 14289 main_vsi->seid = pf->main_vsi_seid; 14290 i40e_vsi_reconfig_tc(main_vsi); 14291 14292 if (vsi->type == I40E_VSI_MAIN) 14293 i40e_rm_default_mac_filter(vsi, pf->hw.mac.perm_addr); 14294 14295 /* assign it some queues */ 14296 ret = i40e_alloc_rings(vsi); 14297 if (ret) 14298 goto err_rings; 14299 14300 /* map all of the rings to the q_vectors */ 14301 i40e_vsi_map_rings_to_vectors(vsi); 14302 return vsi; 14303 14304 err_rings: 14305 i40e_vsi_free_q_vectors(vsi); 14306 if (vsi->netdev_registered) { 14307 vsi->netdev_registered = false; 14308 unregister_netdev(vsi->netdev); 14309 free_netdev(vsi->netdev); 14310 vsi->netdev = NULL; 14311 } 14312 if (vsi->type == I40E_VSI_MAIN) 14313 i40e_devlink_destroy_port(pf); 14314 i40e_aq_delete_element(&pf->hw, vsi->seid, NULL); 14315 err_vsi: 14316 i40e_vsi_clear(vsi); 14317 return NULL; 14318 } 14319 14320 /** 14321 * i40e_vsi_setup - Set up a VSI by a given type 14322 * @pf: board private structure 14323 * @type: VSI type 14324 * @uplink_seid: the switch element to link to 14325 * @param1: usage depends upon VSI type. For VF types, indicates VF id 14326 * 14327 * This allocates the sw VSI structure and its queue resources, then add a VSI 14328 * to the identified VEB. 14329 * 14330 * Returns pointer to the successfully allocated and configure VSI sw struct on 14331 * success, otherwise returns NULL on failure. 14332 **/ 14333 struct i40e_vsi *i40e_vsi_setup(struct i40e_pf *pf, u8 type, 14334 u16 uplink_seid, u32 param1) 14335 { 14336 struct i40e_vsi *vsi = NULL; 14337 struct i40e_veb *veb = NULL; 14338 u16 alloc_queue_pairs; 14339 int v_idx; 14340 int ret; 14341 14342 /* The requested uplink_seid must be either 14343 * - the PF's port seid 14344 * no VEB is needed because this is the PF 14345 * or this is a Flow Director special case VSI 14346 * - seid of an existing VEB 14347 * - seid of a VSI that owns an existing VEB 14348 * - seid of a VSI that doesn't own a VEB 14349 * a new VEB is created and the VSI becomes the owner 14350 * - seid of the PF VSI, which is what creates the first VEB 14351 * this is a special case of the previous 14352 * 14353 * Find which uplink_seid we were given and create a new VEB if needed 14354 */ 14355 veb = i40e_pf_get_veb_by_seid(pf, uplink_seid); 14356 if (!veb && uplink_seid != pf->mac_seid) { 14357 vsi = i40e_pf_get_vsi_by_seid(pf, uplink_seid); 14358 if (!vsi) { 14359 dev_info(&pf->pdev->dev, "no such uplink_seid %d\n", 14360 uplink_seid); 14361 return NULL; 14362 } 14363 14364 if (vsi->uplink_seid == pf->mac_seid) 14365 veb = i40e_veb_setup(pf, pf->mac_seid, vsi->seid, 14366 vsi->tc_config.enabled_tc); 14367 else if ((vsi->flags & I40E_VSI_FLAG_VEB_OWNER) == 0) 14368 veb = i40e_veb_setup(pf, vsi->uplink_seid, vsi->seid, 14369 vsi->tc_config.enabled_tc); 14370 if (veb) { 14371 if (vsi->type != I40E_VSI_MAIN) { 14372 dev_info(&vsi->back->pdev->dev, 14373 "New VSI creation error, uplink seid of LAN VSI expected.\n"); 14374 return NULL; 14375 } 14376 /* We come up by default in VEPA mode if SRIOV is not 14377 * already enabled, in which case we can't force VEPA 14378 * mode. 14379 */ 14380 if (!test_bit(I40E_FLAG_VEB_MODE_ENA, pf->flags)) { 14381 veb->bridge_mode = BRIDGE_MODE_VEPA; 14382 clear_bit(I40E_FLAG_VEB_MODE_ENA, pf->flags); 14383 } 14384 i40e_config_bridge_mode(veb); 14385 } 14386 veb = i40e_pf_get_veb_by_seid(pf, vsi->uplink_seid); 14387 if (!veb) { 14388 dev_info(&pf->pdev->dev, "couldn't add VEB\n"); 14389 return NULL; 14390 } 14391 14392 vsi->flags |= I40E_VSI_FLAG_VEB_OWNER; 14393 uplink_seid = veb->seid; 14394 } 14395 14396 /* get vsi sw struct */ 14397 v_idx = i40e_vsi_mem_alloc(pf, type); 14398 if (v_idx < 0) 14399 goto err_alloc; 14400 vsi = pf->vsi[v_idx]; 14401 if (!vsi) 14402 goto err_alloc; 14403 vsi->type = type; 14404 vsi->veb_idx = (veb ? veb->idx : I40E_NO_VEB); 14405 14406 if (type == I40E_VSI_MAIN) 14407 pf->lan_vsi = v_idx; 14408 else if (type == I40E_VSI_SRIOV) 14409 vsi->vf_id = param1; 14410 /* assign it some queues */ 14411 alloc_queue_pairs = vsi->alloc_queue_pairs * 14412 (i40e_enabled_xdp_vsi(vsi) ? 2 : 1); 14413 14414 ret = i40e_get_lump(pf, pf->qp_pile, alloc_queue_pairs, vsi->idx); 14415 if (ret < 0) { 14416 dev_info(&pf->pdev->dev, 14417 "failed to get tracking for %d queues for VSI %d err=%d\n", 14418 alloc_queue_pairs, vsi->seid, ret); 14419 goto err_vsi; 14420 } 14421 vsi->base_queue = ret; 14422 14423 /* get a VSI from the hardware */ 14424 vsi->uplink_seid = uplink_seid; 14425 ret = i40e_add_vsi(vsi); 14426 if (ret) 14427 goto err_vsi; 14428 14429 switch (vsi->type) { 14430 /* setup the netdev if needed */ 14431 case I40E_VSI_MAIN: 14432 case I40E_VSI_VMDQ2: 14433 ret = i40e_config_netdev(vsi); 14434 if (ret) 14435 goto err_netdev; 14436 ret = i40e_netif_set_realnum_tx_rx_queues(vsi); 14437 if (ret) 14438 goto err_netdev; 14439 if (vsi->type == I40E_VSI_MAIN) { 14440 ret = i40e_devlink_create_port(pf); 14441 if (ret) 14442 goto err_netdev; 14443 SET_NETDEV_DEVLINK_PORT(vsi->netdev, &pf->devlink_port); 14444 } 14445 ret = register_netdev(vsi->netdev); 14446 if (ret) 14447 goto err_dl_port; 14448 vsi->netdev_registered = true; 14449 netif_carrier_off(vsi->netdev); 14450 #ifdef CONFIG_I40E_DCB 14451 /* Setup DCB netlink interface */ 14452 i40e_dcbnl_setup(vsi); 14453 #endif /* CONFIG_I40E_DCB */ 14454 fallthrough; 14455 case I40E_VSI_FDIR: 14456 /* set up vectors and rings if needed */ 14457 ret = i40e_vsi_setup_vectors(vsi); 14458 if (ret) 14459 goto err_msix; 14460 14461 ret = i40e_alloc_rings(vsi); 14462 if (ret) 14463 goto err_rings; 14464 14465 /* map all of the rings to the q_vectors */ 14466 i40e_vsi_map_rings_to_vectors(vsi); 14467 14468 i40e_vsi_reset_stats(vsi); 14469 break; 14470 default: 14471 /* no netdev or rings for the other VSI types */ 14472 break; 14473 } 14474 14475 if (test_bit(I40E_HW_CAP_RSS_AQ, pf->hw.caps) && 14476 vsi->type == I40E_VSI_VMDQ2) { 14477 ret = i40e_vsi_config_rss(vsi); 14478 if (ret) 14479 goto err_config; 14480 } 14481 return vsi; 14482 14483 err_config: 14484 i40e_vsi_clear_rings(vsi); 14485 err_rings: 14486 i40e_vsi_free_q_vectors(vsi); 14487 err_msix: 14488 if (vsi->netdev_registered) { 14489 vsi->netdev_registered = false; 14490 unregister_netdev(vsi->netdev); 14491 free_netdev(vsi->netdev); 14492 vsi->netdev = NULL; 14493 } 14494 err_dl_port: 14495 if (vsi->type == I40E_VSI_MAIN) 14496 i40e_devlink_destroy_port(pf); 14497 err_netdev: 14498 i40e_aq_delete_element(&pf->hw, vsi->seid, NULL); 14499 err_vsi: 14500 i40e_vsi_clear(vsi); 14501 err_alloc: 14502 return NULL; 14503 } 14504 14505 /** 14506 * i40e_veb_get_bw_info - Query VEB BW information 14507 * @veb: the veb to query 14508 * 14509 * Query the Tx scheduler BW configuration data for given VEB 14510 **/ 14511 static int i40e_veb_get_bw_info(struct i40e_veb *veb) 14512 { 14513 struct i40e_aqc_query_switching_comp_ets_config_resp ets_data; 14514 struct i40e_aqc_query_switching_comp_bw_config_resp bw_data; 14515 struct i40e_pf *pf = veb->pf; 14516 struct i40e_hw *hw = &pf->hw; 14517 u32 tc_bw_max; 14518 int ret = 0; 14519 int i; 14520 14521 ret = i40e_aq_query_switch_comp_bw_config(hw, veb->seid, 14522 &bw_data, NULL); 14523 if (ret) { 14524 dev_info(&pf->pdev->dev, 14525 "query veb bw config failed, err %pe aq_err %s\n", 14526 ERR_PTR(ret), libie_aq_str(hw->aq.asq_last_status)); 14527 goto out; 14528 } 14529 14530 ret = i40e_aq_query_switch_comp_ets_config(hw, veb->seid, 14531 &ets_data, NULL); 14532 if (ret) { 14533 dev_info(&pf->pdev->dev, 14534 "query veb bw ets config failed, err %pe aq_err %s\n", 14535 ERR_PTR(ret), libie_aq_str(hw->aq.asq_last_status)); 14536 goto out; 14537 } 14538 14539 veb->bw_limit = le16_to_cpu(ets_data.port_bw_limit); 14540 veb->bw_max_quanta = ets_data.tc_bw_max; 14541 veb->is_abs_credits = bw_data.absolute_credits_enable; 14542 veb->enabled_tc = ets_data.tc_valid_bits; 14543 tc_bw_max = le16_to_cpu(bw_data.tc_bw_max[0]) | 14544 (le16_to_cpu(bw_data.tc_bw_max[1]) << 16); 14545 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 14546 veb->bw_tc_share_credits[i] = bw_data.tc_bw_share_credits[i]; 14547 veb->bw_tc_limit_credits[i] = 14548 le16_to_cpu(bw_data.tc_bw_limits[i]); 14549 veb->bw_tc_max_quanta[i] = ((tc_bw_max >> (i*4)) & 0x7); 14550 } 14551 14552 out: 14553 return ret; 14554 } 14555 14556 /** 14557 * i40e_veb_mem_alloc - Allocates the next available struct veb in the PF 14558 * @pf: board private structure 14559 * 14560 * On error: returns error code (negative) 14561 * On success: returns vsi index in PF (positive) 14562 **/ 14563 static int i40e_veb_mem_alloc(struct i40e_pf *pf) 14564 { 14565 int ret = -ENOENT; 14566 struct i40e_veb *veb; 14567 int i; 14568 14569 /* Need to protect the allocation of switch elements at the PF level */ 14570 mutex_lock(&pf->switch_mutex); 14571 14572 /* VEB list may be fragmented if VEB creation/destruction has 14573 * been happening. We can afford to do a quick scan to look 14574 * for any free slots in the list. 14575 * 14576 * find next empty veb slot, looping back around if necessary 14577 */ 14578 i = 0; 14579 while ((i < I40E_MAX_VEB) && (pf->veb[i] != NULL)) 14580 i++; 14581 if (i >= I40E_MAX_VEB) { 14582 ret = -ENOMEM; 14583 goto err_alloc_veb; /* out of VEB slots! */ 14584 } 14585 14586 veb = kzalloc_obj(*veb); 14587 if (!veb) { 14588 ret = -ENOMEM; 14589 goto err_alloc_veb; 14590 } 14591 veb->pf = pf; 14592 veb->idx = i; 14593 veb->enabled_tc = 1; 14594 14595 pf->veb[i] = veb; 14596 ret = i; 14597 err_alloc_veb: 14598 mutex_unlock(&pf->switch_mutex); 14599 return ret; 14600 } 14601 14602 /** 14603 * i40e_switch_branch_release - Delete a branch of the switch tree 14604 * @branch: where to start deleting 14605 * 14606 * This uses recursion to find the tips of the branch to be 14607 * removed, deleting until we get back to and can delete this VEB. 14608 **/ 14609 static void i40e_switch_branch_release(struct i40e_veb *branch) 14610 { 14611 struct i40e_pf *pf = branch->pf; 14612 u16 branch_seid = branch->seid; 14613 u16 veb_idx = branch->idx; 14614 struct i40e_vsi *vsi; 14615 struct i40e_veb *veb; 14616 int i; 14617 14618 /* release any VEBs on this VEB - RECURSION */ 14619 i40e_pf_for_each_veb(pf, i, veb) 14620 if (veb->uplink_seid == branch->seid) 14621 i40e_switch_branch_release(veb); 14622 14623 /* Release the VSIs on this VEB, but not the owner VSI. 14624 * 14625 * NOTE: Removing the last VSI on a VEB has the SIDE EFFECT of removing 14626 * the VEB itself, so don't use (*branch) after this loop. 14627 */ 14628 i40e_pf_for_each_vsi(pf, i, vsi) 14629 if (vsi->uplink_seid == branch_seid && 14630 (vsi->flags & I40E_VSI_FLAG_VEB_OWNER) == 0) 14631 i40e_vsi_release(vsi); 14632 14633 /* There's one corner case where the VEB might not have been 14634 * removed, so double check it here and remove it if needed. 14635 * This case happens if the veb was created from the debugfs 14636 * commands and no VSIs were added to it. 14637 */ 14638 if (pf->veb[veb_idx]) 14639 i40e_veb_release(pf->veb[veb_idx]); 14640 } 14641 14642 /** 14643 * i40e_veb_clear - remove veb struct 14644 * @veb: the veb to remove 14645 **/ 14646 static void i40e_veb_clear(struct i40e_veb *veb) 14647 { 14648 if (!veb) 14649 return; 14650 14651 if (veb->pf) { 14652 struct i40e_pf *pf = veb->pf; 14653 14654 mutex_lock(&pf->switch_mutex); 14655 if (pf->veb[veb->idx] == veb) 14656 pf->veb[veb->idx] = NULL; 14657 mutex_unlock(&pf->switch_mutex); 14658 } 14659 14660 kfree(veb); 14661 } 14662 14663 /** 14664 * i40e_veb_release - Delete a VEB and free its resources 14665 * @veb: the VEB being removed 14666 **/ 14667 void i40e_veb_release(struct i40e_veb *veb) 14668 { 14669 struct i40e_vsi *vsi, *vsi_it; 14670 struct i40e_pf *pf; 14671 int i, n = 0; 14672 14673 pf = veb->pf; 14674 14675 /* find the remaining VSI and check for extras */ 14676 i40e_pf_for_each_vsi(pf, i, vsi_it) 14677 if (vsi_it->uplink_seid == veb->seid) { 14678 if (vsi_it->flags & I40E_VSI_FLAG_VEB_OWNER) 14679 vsi = vsi_it; 14680 n++; 14681 } 14682 14683 /* Floating VEB has to be empty and regular one must have 14684 * single owner VSI. 14685 */ 14686 if ((veb->uplink_seid && n != 1) || (!veb->uplink_seid && n != 0)) { 14687 dev_info(&pf->pdev->dev, 14688 "can't remove VEB %d with %d VSIs left\n", 14689 veb->seid, n); 14690 return; 14691 } 14692 14693 /* For regular VEB move the owner VSI to uplink port */ 14694 if (veb->uplink_seid) { 14695 vsi->flags &= ~I40E_VSI_FLAG_VEB_OWNER; 14696 vsi->uplink_seid = veb->uplink_seid; 14697 vsi->veb_idx = I40E_NO_VEB; 14698 } 14699 14700 i40e_aq_delete_element(&pf->hw, veb->seid, NULL); 14701 i40e_veb_clear(veb); 14702 } 14703 14704 /** 14705 * i40e_add_veb - create the VEB in the switch 14706 * @veb: the VEB to be instantiated 14707 * @vsi: the controlling VSI 14708 **/ 14709 static int i40e_add_veb(struct i40e_veb *veb, struct i40e_vsi *vsi) 14710 { 14711 struct i40e_pf *pf = veb->pf; 14712 bool enable_stats = !!test_bit(I40E_FLAG_VEB_STATS_ENA, pf->flags); 14713 int ret; 14714 14715 ret = i40e_aq_add_veb(&pf->hw, veb->uplink_seid, vsi ? vsi->seid : 0, 14716 veb->enabled_tc, vsi ? false : true, 14717 &veb->seid, enable_stats, NULL); 14718 14719 /* get a VEB from the hardware */ 14720 if (ret) { 14721 dev_info(&pf->pdev->dev, 14722 "couldn't add VEB, err %pe aq_err %s\n", 14723 ERR_PTR(ret), libie_aq_str(pf->hw.aq.asq_last_status)); 14724 return -EPERM; 14725 } 14726 14727 /* get statistics counter */ 14728 ret = i40e_aq_get_veb_parameters(&pf->hw, veb->seid, NULL, NULL, 14729 &veb->stats_idx, NULL, NULL, NULL); 14730 if (ret) { 14731 dev_info(&pf->pdev->dev, 14732 "couldn't get VEB statistics idx, err %pe aq_err %s\n", 14733 ERR_PTR(ret), libie_aq_str(pf->hw.aq.asq_last_status)); 14734 return -EPERM; 14735 } 14736 ret = i40e_veb_get_bw_info(veb); 14737 if (ret) { 14738 dev_info(&pf->pdev->dev, 14739 "couldn't get VEB bw info, err %pe aq_err %s\n", 14740 ERR_PTR(ret), libie_aq_str(pf->hw.aq.asq_last_status)); 14741 i40e_aq_delete_element(&pf->hw, veb->seid, NULL); 14742 return -ENOENT; 14743 } 14744 14745 if (vsi) { 14746 vsi->uplink_seid = veb->seid; 14747 vsi->veb_idx = veb->idx; 14748 vsi->flags |= I40E_VSI_FLAG_VEB_OWNER; 14749 } 14750 14751 return 0; 14752 } 14753 14754 /** 14755 * i40e_veb_setup - Set up a VEB 14756 * @pf: board private structure 14757 * @uplink_seid: the switch element to link to 14758 * @vsi_seid: the initial VSI seid 14759 * @enabled_tc: Enabled TC bit-map 14760 * 14761 * This allocates the sw VEB structure and links it into the switch 14762 * It is possible and legal for this to be a duplicate of an already 14763 * existing VEB. It is also possible for both uplink and vsi seids 14764 * to be zero, in order to create a floating VEB. 14765 * 14766 * Returns pointer to the successfully allocated VEB sw struct on 14767 * success, otherwise returns NULL on failure. 14768 **/ 14769 struct i40e_veb *i40e_veb_setup(struct i40e_pf *pf, u16 uplink_seid, 14770 u16 vsi_seid, u8 enabled_tc) 14771 { 14772 struct i40e_vsi *vsi = NULL; 14773 struct i40e_veb *veb; 14774 int veb_idx; 14775 int ret; 14776 14777 /* if one seid is 0, the other must be 0 to create a floating relay */ 14778 if ((uplink_seid == 0 || vsi_seid == 0) && 14779 (uplink_seid + vsi_seid != 0)) { 14780 dev_info(&pf->pdev->dev, 14781 "one, not both seid's are 0: uplink=%d vsi=%d\n", 14782 uplink_seid, vsi_seid); 14783 return NULL; 14784 } 14785 14786 /* make sure there is such a vsi and uplink */ 14787 if (vsi_seid) { 14788 vsi = i40e_pf_get_vsi_by_seid(pf, vsi_seid); 14789 if (!vsi) { 14790 dev_err(&pf->pdev->dev, "vsi seid %d not found\n", 14791 vsi_seid); 14792 return NULL; 14793 } 14794 } 14795 14796 /* get veb sw struct */ 14797 veb_idx = i40e_veb_mem_alloc(pf); 14798 if (veb_idx < 0) 14799 goto err_alloc; 14800 veb = pf->veb[veb_idx]; 14801 veb->uplink_seid = uplink_seid; 14802 veb->enabled_tc = (enabled_tc ? enabled_tc : 0x1); 14803 14804 /* create the VEB in the switch */ 14805 ret = i40e_add_veb(veb, vsi); 14806 if (ret) 14807 goto err_veb; 14808 14809 if (vsi && vsi->idx == pf->lan_vsi) 14810 pf->lan_veb = veb->idx; 14811 14812 return veb; 14813 14814 err_veb: 14815 i40e_veb_clear(veb); 14816 err_alloc: 14817 return NULL; 14818 } 14819 14820 /** 14821 * i40e_setup_pf_switch_element - set PF vars based on switch type 14822 * @pf: board private structure 14823 * @ele: element we are building info from 14824 * @num_reported: total number of elements 14825 * @printconfig: should we print the contents 14826 * 14827 * helper function to assist in extracting a few useful SEID values. 14828 **/ 14829 static void i40e_setup_pf_switch_element(struct i40e_pf *pf, 14830 struct i40e_aqc_switch_config_element_resp *ele, 14831 u16 num_reported, bool printconfig) 14832 { 14833 u16 downlink_seid = le16_to_cpu(ele->downlink_seid); 14834 u16 uplink_seid = le16_to_cpu(ele->uplink_seid); 14835 u8 element_type = ele->element_type; 14836 u16 seid = le16_to_cpu(ele->seid); 14837 struct i40e_veb *veb; 14838 14839 if (printconfig) 14840 dev_info(&pf->pdev->dev, 14841 "type=%d seid=%d uplink=%d downlink=%d\n", 14842 element_type, seid, uplink_seid, downlink_seid); 14843 14844 switch (element_type) { 14845 case I40E_SWITCH_ELEMENT_TYPE_MAC: 14846 pf->mac_seid = seid; 14847 break; 14848 case I40E_SWITCH_ELEMENT_TYPE_VEB: 14849 /* Main VEB? */ 14850 if (uplink_seid != pf->mac_seid) 14851 break; 14852 veb = i40e_pf_get_main_veb(pf); 14853 if (!veb) { 14854 int v; 14855 14856 /* find existing or else empty VEB */ 14857 veb = i40e_pf_get_veb_by_seid(pf, seid); 14858 if (veb) { 14859 pf->lan_veb = veb->idx; 14860 } else { 14861 v = i40e_veb_mem_alloc(pf); 14862 if (v < 0) 14863 break; 14864 pf->lan_veb = v; 14865 } 14866 } 14867 14868 /* Try to get again main VEB as pf->lan_veb may have changed */ 14869 veb = i40e_pf_get_main_veb(pf); 14870 if (!veb) 14871 break; 14872 14873 veb->seid = seid; 14874 veb->uplink_seid = pf->mac_seid; 14875 veb->pf = pf; 14876 break; 14877 case I40E_SWITCH_ELEMENT_TYPE_VSI: 14878 if (num_reported != 1) 14879 break; 14880 /* This is immediately after a reset so we can assume this is 14881 * the PF's VSI 14882 */ 14883 pf->mac_seid = uplink_seid; 14884 pf->main_vsi_seid = seid; 14885 if (printconfig) 14886 dev_info(&pf->pdev->dev, 14887 "pf_seid=%d main_vsi_seid=%d\n", 14888 downlink_seid, pf->main_vsi_seid); 14889 break; 14890 case I40E_SWITCH_ELEMENT_TYPE_PF: 14891 case I40E_SWITCH_ELEMENT_TYPE_VF: 14892 case I40E_SWITCH_ELEMENT_TYPE_EMP: 14893 case I40E_SWITCH_ELEMENT_TYPE_BMC: 14894 case I40E_SWITCH_ELEMENT_TYPE_PE: 14895 case I40E_SWITCH_ELEMENT_TYPE_PA: 14896 /* ignore these for now */ 14897 break; 14898 default: 14899 dev_info(&pf->pdev->dev, "unknown element type=%d seid=%d\n", 14900 element_type, seid); 14901 break; 14902 } 14903 } 14904 14905 /** 14906 * i40e_fetch_switch_configuration - Get switch config from firmware 14907 * @pf: board private structure 14908 * @printconfig: should we print the contents 14909 * 14910 * Get the current switch configuration from the device and 14911 * extract a few useful SEID values. 14912 **/ 14913 int i40e_fetch_switch_configuration(struct i40e_pf *pf, bool printconfig) 14914 { 14915 struct i40e_aqc_get_switch_config_resp *sw_config; 14916 u16 next_seid = 0; 14917 int ret = 0; 14918 u8 *aq_buf; 14919 int i; 14920 14921 aq_buf = kzalloc(I40E_AQ_LARGE_BUF, GFP_KERNEL); 14922 if (!aq_buf) 14923 return -ENOMEM; 14924 14925 sw_config = (struct i40e_aqc_get_switch_config_resp *)aq_buf; 14926 do { 14927 u16 num_reported, num_total; 14928 14929 ret = i40e_aq_get_switch_config(&pf->hw, sw_config, 14930 I40E_AQ_LARGE_BUF, 14931 &next_seid, NULL); 14932 if (ret) { 14933 dev_info(&pf->pdev->dev, 14934 "get switch config failed err %d aq_err %s\n", 14935 ret, libie_aq_str(pf->hw.aq.asq_last_status)); 14936 kfree(aq_buf); 14937 return -ENOENT; 14938 } 14939 14940 num_reported = le16_to_cpu(sw_config->header.num_reported); 14941 num_total = le16_to_cpu(sw_config->header.num_total); 14942 14943 if (printconfig) 14944 dev_info(&pf->pdev->dev, 14945 "header: %d reported %d total\n", 14946 num_reported, num_total); 14947 14948 for (i = 0; i < num_reported; i++) { 14949 struct i40e_aqc_switch_config_element_resp *ele = 14950 &sw_config->element[i]; 14951 14952 i40e_setup_pf_switch_element(pf, ele, num_reported, 14953 printconfig); 14954 } 14955 } while (next_seid != 0); 14956 14957 kfree(aq_buf); 14958 return ret; 14959 } 14960 14961 /** 14962 * i40e_setup_pf_switch - Setup the HW switch on startup or after reset 14963 * @pf: board private structure 14964 * @reinit: if the Main VSI needs to re-initialized. 14965 * @lock_acquired: indicates whether or not the lock has been acquired 14966 * 14967 * Returns 0 on success, negative value on failure 14968 **/ 14969 static int i40e_setup_pf_switch(struct i40e_pf *pf, bool reinit, bool lock_acquired) 14970 { 14971 struct i40e_vsi *main_vsi; 14972 u16 flags = 0; 14973 int ret; 14974 14975 /* find out what's out there already */ 14976 ret = i40e_fetch_switch_configuration(pf, false); 14977 if (ret) { 14978 dev_info(&pf->pdev->dev, 14979 "couldn't fetch switch config, err %pe aq_err %s\n", 14980 ERR_PTR(ret), libie_aq_str(pf->hw.aq.asq_last_status)); 14981 return ret; 14982 } 14983 i40e_pf_reset_stats(pf); 14984 14985 /* set the switch config bit for the whole device to 14986 * support limited promisc or true promisc 14987 * when user requests promisc. The default is limited 14988 * promisc. 14989 */ 14990 14991 if ((pf->hw.pf_id == 0) && 14992 !test_bit(I40E_FLAG_TRUE_PROMISC_ENA, pf->flags)) { 14993 flags = I40E_AQ_SET_SWITCH_CFG_PROMISC; 14994 pf->last_sw_conf_flags = flags; 14995 } 14996 14997 if (pf->hw.pf_id == 0) { 14998 u16 valid_flags; 14999 15000 valid_flags = I40E_AQ_SET_SWITCH_CFG_PROMISC; 15001 ret = i40e_aq_set_switch_config(&pf->hw, flags, valid_flags, 0, 15002 NULL); 15003 if (ret && pf->hw.aq.asq_last_status != LIBIE_AQ_RC_ESRCH) { 15004 dev_info(&pf->pdev->dev, 15005 "couldn't set switch config bits, err %pe aq_err %s\n", 15006 ERR_PTR(ret), 15007 libie_aq_str(pf->hw.aq.asq_last_status)); 15008 /* not a fatal problem, just keep going */ 15009 } 15010 pf->last_sw_conf_valid_flags = valid_flags; 15011 } 15012 15013 /* first time setup */ 15014 main_vsi = i40e_pf_get_main_vsi(pf); 15015 if (!main_vsi || reinit) { 15016 struct i40e_veb *veb; 15017 u16 uplink_seid; 15018 15019 /* Set up the PF VSI associated with the PF's main VSI 15020 * that is already in the HW switch 15021 */ 15022 veb = i40e_pf_get_main_veb(pf); 15023 if (veb) 15024 uplink_seid = veb->seid; 15025 else 15026 uplink_seid = pf->mac_seid; 15027 if (!main_vsi) 15028 main_vsi = i40e_vsi_setup(pf, I40E_VSI_MAIN, 15029 uplink_seid, 0); 15030 else if (reinit) 15031 main_vsi = i40e_vsi_reinit_setup(main_vsi); 15032 if (!main_vsi) { 15033 dev_info(&pf->pdev->dev, "setup of MAIN VSI failed\n"); 15034 i40e_cloud_filter_exit(pf); 15035 i40e_fdir_teardown(pf); 15036 return -EAGAIN; 15037 } 15038 } else { 15039 /* force a reset of TC and queue layout configurations */ 15040 main_vsi->seid = pf->main_vsi_seid; 15041 i40e_vsi_reconfig_tc(main_vsi); 15042 } 15043 i40e_vlan_stripping_disable(main_vsi); 15044 15045 i40e_fdir_sb_setup(pf); 15046 15047 /* Setup static PF queue filter control settings */ 15048 ret = i40e_setup_pf_filter_control(pf); 15049 if (ret) { 15050 dev_info(&pf->pdev->dev, "setup_pf_filter_control failed: %d\n", 15051 ret); 15052 /* Failure here should not stop continuing other steps */ 15053 } 15054 15055 /* enable RSS in the HW, even for only one queue, as the stack can use 15056 * the hash 15057 */ 15058 if (test_bit(I40E_FLAG_RSS_ENA, pf->flags)) 15059 i40e_pf_config_rss(pf); 15060 15061 /* fill in link information and enable LSE reporting */ 15062 i40e_link_event(pf); 15063 15064 i40e_ptp_init(pf); 15065 15066 if (!lock_acquired) 15067 rtnl_lock(); 15068 15069 /* repopulate tunnel port filters */ 15070 udp_tunnel_nic_reset_ntf(main_vsi->netdev); 15071 15072 if (!lock_acquired) 15073 rtnl_unlock(); 15074 15075 return ret; 15076 } 15077 15078 /** 15079 * i40e_determine_queue_usage - Work out queue distribution 15080 * @pf: board private structure 15081 **/ 15082 static void i40e_determine_queue_usage(struct i40e_pf *pf) 15083 { 15084 int queues_left; 15085 int q_max; 15086 15087 pf->num_lan_qps = 0; 15088 15089 /* Find the max queues to be put into basic use. We'll always be 15090 * using TC0, whether or not DCB is running, and TC0 will get the 15091 * big RSS set. 15092 */ 15093 queues_left = pf->hw.func_caps.num_tx_qp; 15094 15095 if ((queues_left == 1) || 15096 !test_bit(I40E_FLAG_MSIX_ENA, pf->flags)) { 15097 /* one qp for PF, no queues for anything else */ 15098 queues_left = 0; 15099 pf->alloc_rss_size = pf->num_lan_qps = 1; 15100 15101 /* make sure all the fancies are disabled */ 15102 clear_bit(I40E_FLAG_RSS_ENA, pf->flags); 15103 clear_bit(I40E_FLAG_IWARP_ENA, pf->flags); 15104 clear_bit(I40E_FLAG_FD_SB_ENA, pf->flags); 15105 clear_bit(I40E_FLAG_FD_ATR_ENA, pf->flags); 15106 clear_bit(I40E_FLAG_DCB_CAPABLE, pf->flags); 15107 clear_bit(I40E_FLAG_DCB_ENA, pf->flags); 15108 clear_bit(I40E_FLAG_SRIOV_ENA, pf->flags); 15109 clear_bit(I40E_FLAG_VMDQ_ENA, pf->flags); 15110 set_bit(I40E_FLAG_FD_SB_INACTIVE, pf->flags); 15111 } else if (!test_bit(I40E_FLAG_RSS_ENA, pf->flags) && 15112 !test_bit(I40E_FLAG_FD_SB_ENA, pf->flags) && 15113 !test_bit(I40E_FLAG_FD_ATR_ENA, pf->flags) && 15114 !test_bit(I40E_FLAG_DCB_CAPABLE, pf->flags)) { 15115 /* one qp for PF */ 15116 pf->alloc_rss_size = pf->num_lan_qps = 1; 15117 queues_left -= pf->num_lan_qps; 15118 15119 clear_bit(I40E_FLAG_RSS_ENA, pf->flags); 15120 clear_bit(I40E_FLAG_IWARP_ENA, pf->flags); 15121 clear_bit(I40E_FLAG_FD_SB_ENA, pf->flags); 15122 clear_bit(I40E_FLAG_FD_ATR_ENA, pf->flags); 15123 clear_bit(I40E_FLAG_DCB_ENA, pf->flags); 15124 clear_bit(I40E_FLAG_VMDQ_ENA, pf->flags); 15125 set_bit(I40E_FLAG_FD_SB_INACTIVE, pf->flags); 15126 } else { 15127 /* Not enough queues for all TCs */ 15128 if (test_bit(I40E_FLAG_DCB_CAPABLE, pf->flags) && 15129 queues_left < I40E_MAX_TRAFFIC_CLASS) { 15130 clear_bit(I40E_FLAG_DCB_CAPABLE, pf->flags); 15131 clear_bit(I40E_FLAG_DCB_ENA, pf->flags); 15132 dev_info(&pf->pdev->dev, "not enough queues for DCB. DCB is disabled.\n"); 15133 } 15134 15135 /* limit lan qps to the smaller of qps, cpus or msix */ 15136 q_max = max_t(int, pf->rss_size_max, num_online_cpus()); 15137 q_max = min_t(int, q_max, pf->hw.func_caps.num_tx_qp); 15138 q_max = min_t(int, q_max, pf->hw.func_caps.num_msix_vectors); 15139 pf->num_lan_qps = q_max; 15140 15141 queues_left -= pf->num_lan_qps; 15142 } 15143 15144 if (test_bit(I40E_FLAG_FD_SB_ENA, pf->flags)) { 15145 if (queues_left > 1) { 15146 queues_left -= 1; /* save 1 queue for FD */ 15147 } else { 15148 clear_bit(I40E_FLAG_FD_SB_ENA, pf->flags); 15149 set_bit(I40E_FLAG_FD_SB_INACTIVE, pf->flags); 15150 dev_info(&pf->pdev->dev, "not enough queues for Flow Director. Flow Director feature is disabled\n"); 15151 } 15152 } 15153 15154 if (test_bit(I40E_FLAG_SRIOV_ENA, pf->flags) && 15155 pf->num_vf_qps && pf->num_req_vfs && queues_left) { 15156 pf->num_req_vfs = min_t(int, pf->num_req_vfs, 15157 (queues_left / pf->num_vf_qps)); 15158 queues_left -= (pf->num_req_vfs * pf->num_vf_qps); 15159 } 15160 15161 if (test_bit(I40E_FLAG_VMDQ_ENA, pf->flags) && 15162 pf->num_vmdq_vsis && pf->num_vmdq_qps && queues_left) { 15163 pf->num_vmdq_vsis = min_t(int, pf->num_vmdq_vsis, 15164 (queues_left / pf->num_vmdq_qps)); 15165 queues_left -= (pf->num_vmdq_vsis * pf->num_vmdq_qps); 15166 } 15167 15168 pf->queues_left = queues_left; 15169 dev_dbg(&pf->pdev->dev, 15170 "qs_avail=%d FD SB=%d lan_qs=%d lan_tc0=%d vf=%d*%d vmdq=%d*%d, remaining=%d\n", 15171 pf->hw.func_caps.num_tx_qp, 15172 !!test_bit(I40E_FLAG_FD_SB_ENA, pf->flags), 15173 pf->num_lan_qps, pf->alloc_rss_size, pf->num_req_vfs, 15174 pf->num_vf_qps, pf->num_vmdq_vsis, pf->num_vmdq_qps, 15175 queues_left); 15176 } 15177 15178 /** 15179 * i40e_setup_pf_filter_control - Setup PF static filter control 15180 * @pf: PF to be setup 15181 * 15182 * i40e_setup_pf_filter_control sets up a PF's initial filter control 15183 * settings. If PE/FCoE are enabled then it will also set the per PF 15184 * based filter sizes required for them. It also enables Flow director, 15185 * ethertype and macvlan type filter settings for the pf. 15186 * 15187 * Returns 0 on success, negative on failure 15188 **/ 15189 static int i40e_setup_pf_filter_control(struct i40e_pf *pf) 15190 { 15191 struct i40e_filter_control_settings *settings = &pf->filter_settings; 15192 15193 settings->hash_lut_size = I40E_HASH_LUT_SIZE_128; 15194 15195 /* Flow Director is enabled */ 15196 if (test_bit(I40E_FLAG_FD_SB_ENA, pf->flags) || 15197 test_bit(I40E_FLAG_FD_ATR_ENA, pf->flags)) 15198 settings->enable_fdir = true; 15199 15200 /* Ethtype and MACVLAN filters enabled for PF */ 15201 settings->enable_ethtype = true; 15202 settings->enable_macvlan = true; 15203 15204 if (i40e_set_filter_control(&pf->hw, settings)) 15205 return -ENOENT; 15206 15207 return 0; 15208 } 15209 15210 #define INFO_STRING_LEN 255 15211 #define REMAIN(__x) (INFO_STRING_LEN - (__x)) 15212 static void i40e_print_features(struct i40e_pf *pf) 15213 { 15214 struct i40e_vsi *main_vsi = i40e_pf_get_main_vsi(pf); 15215 struct i40e_hw *hw = &pf->hw; 15216 char *buf; 15217 int i; 15218 15219 buf = kmalloc(INFO_STRING_LEN, GFP_KERNEL); 15220 if (!buf) 15221 return; 15222 15223 i = snprintf(buf, INFO_STRING_LEN, "Features: PF-id[%d]", hw->pf_id); 15224 #ifdef CONFIG_PCI_IOV 15225 i += scnprintf(&buf[i], REMAIN(i), " VFs: %d", pf->num_req_vfs); 15226 #endif 15227 i += scnprintf(&buf[i], REMAIN(i), " VSIs: %d QP: %d", 15228 pf->hw.func_caps.num_vsis, main_vsi->num_queue_pairs); 15229 if (test_bit(I40E_FLAG_RSS_ENA, pf->flags)) 15230 i += scnprintf(&buf[i], REMAIN(i), " RSS"); 15231 if (test_bit(I40E_FLAG_FD_ATR_ENA, pf->flags)) 15232 i += scnprintf(&buf[i], REMAIN(i), " FD_ATR"); 15233 if (test_bit(I40E_FLAG_FD_SB_ENA, pf->flags)) { 15234 i += scnprintf(&buf[i], REMAIN(i), " FD_SB"); 15235 i += scnprintf(&buf[i], REMAIN(i), " NTUPLE"); 15236 } 15237 if (test_bit(I40E_FLAG_DCB_CAPABLE, pf->flags)) 15238 i += scnprintf(&buf[i], REMAIN(i), " DCB"); 15239 i += scnprintf(&buf[i], REMAIN(i), " VxLAN"); 15240 i += scnprintf(&buf[i], REMAIN(i), " Geneve"); 15241 if (test_bit(I40E_FLAG_PTP_ENA, pf->flags)) 15242 i += scnprintf(&buf[i], REMAIN(i), " PTP"); 15243 if (test_bit(I40E_FLAG_VEB_MODE_ENA, pf->flags)) 15244 i += scnprintf(&buf[i], REMAIN(i), " VEB"); 15245 else 15246 i += scnprintf(&buf[i], REMAIN(i), " VEPA"); 15247 15248 dev_info(&pf->pdev->dev, "%s\n", buf); 15249 kfree(buf); 15250 WARN_ON(i > INFO_STRING_LEN); 15251 } 15252 15253 /** 15254 * i40e_get_platform_mac_addr - get platform-specific MAC address 15255 * @pdev: PCI device information struct 15256 * @pf: board private structure 15257 * 15258 * Look up the MAC address for the device. First we'll try 15259 * eth_platform_get_mac_address, which will check Open Firmware, or arch 15260 * specific fallback. Otherwise, we'll default to the stored value in 15261 * firmware. 15262 **/ 15263 static void i40e_get_platform_mac_addr(struct pci_dev *pdev, struct i40e_pf *pf) 15264 { 15265 if (eth_platform_get_mac_address(&pdev->dev, pf->hw.mac.addr)) 15266 i40e_get_mac_addr(&pf->hw, pf->hw.mac.addr); 15267 } 15268 15269 /** 15270 * i40e_set_fec_in_flags - helper function for setting FEC options in flags 15271 * @fec_cfg: FEC option to set in flags 15272 * @flags: ptr to flags in which we set FEC option 15273 **/ 15274 void i40e_set_fec_in_flags(u8 fec_cfg, unsigned long *flags) 15275 { 15276 if (fec_cfg & I40E_AQ_SET_FEC_AUTO) { 15277 set_bit(I40E_FLAG_RS_FEC, flags); 15278 set_bit(I40E_FLAG_BASE_R_FEC, flags); 15279 } 15280 if ((fec_cfg & I40E_AQ_SET_FEC_REQUEST_RS) || 15281 (fec_cfg & I40E_AQ_SET_FEC_ABILITY_RS)) { 15282 set_bit(I40E_FLAG_RS_FEC, flags); 15283 clear_bit(I40E_FLAG_BASE_R_FEC, flags); 15284 } 15285 if ((fec_cfg & I40E_AQ_SET_FEC_REQUEST_KR) || 15286 (fec_cfg & I40E_AQ_SET_FEC_ABILITY_KR)) { 15287 set_bit(I40E_FLAG_BASE_R_FEC, flags); 15288 clear_bit(I40E_FLAG_RS_FEC, flags); 15289 } 15290 if (fec_cfg == 0) { 15291 clear_bit(I40E_FLAG_RS_FEC, flags); 15292 clear_bit(I40E_FLAG_BASE_R_FEC, flags); 15293 } 15294 } 15295 15296 /** 15297 * i40e_check_recovery_mode - check if we are running transition firmware 15298 * @pf: board private structure 15299 * 15300 * Check registers indicating the firmware runs in recovery mode. Sets the 15301 * appropriate driver state. 15302 * 15303 * Returns true if the recovery mode was detected, false otherwise 15304 **/ 15305 static bool i40e_check_recovery_mode(struct i40e_pf *pf) 15306 { 15307 u32 val = rd32(&pf->hw, I40E_GL_FWSTS); 15308 15309 if (val & I40E_GL_FWSTS_FWS1B_MASK) { 15310 dev_crit(&pf->pdev->dev, "Firmware recovery mode detected. Limiting functionality.\n"); 15311 dev_crit(&pf->pdev->dev, "Refer to the Intel(R) Ethernet Adapters and Devices User Guide for details on firmware recovery mode.\n"); 15312 set_bit(__I40E_RECOVERY_MODE, pf->state); 15313 15314 return true; 15315 } 15316 if (test_bit(__I40E_RECOVERY_MODE, pf->state)) 15317 dev_info(&pf->pdev->dev, "Please do Power-On Reset to initialize adapter in normal mode with full functionality.\n"); 15318 15319 return false; 15320 } 15321 15322 /** 15323 * i40e_pf_loop_reset - perform reset in a loop. 15324 * @pf: board private structure 15325 * 15326 * This function is useful when a NIC is about to enter recovery mode. 15327 * When a NIC's internal data structures are corrupted the NIC's 15328 * firmware is going to enter recovery mode. 15329 * Right after a POR it takes about 7 minutes for firmware to enter 15330 * recovery mode. Until that time a NIC is in some kind of intermediate 15331 * state. After that time period the NIC almost surely enters 15332 * recovery mode. The only way for a driver to detect intermediate 15333 * state is to issue a series of pf-resets and check a return value. 15334 * If a PF reset returns success then the firmware could be in recovery 15335 * mode so the caller of this code needs to check for recovery mode 15336 * if this function returns success. There is a little chance that 15337 * firmware will hang in intermediate state forever. 15338 * Since waiting 7 minutes is quite a lot of time this function waits 15339 * 10 seconds and then gives up by returning an error. 15340 * 15341 * Return 0 on success, negative on failure. 15342 **/ 15343 static int i40e_pf_loop_reset(struct i40e_pf *pf) 15344 { 15345 /* wait max 10 seconds for PF reset to succeed */ 15346 const unsigned long time_end = jiffies + 10 * HZ; 15347 struct i40e_hw *hw = &pf->hw; 15348 int ret; 15349 15350 ret = i40e_pf_reset(hw); 15351 while (ret != 0 && time_before(jiffies, time_end)) { 15352 usleep_range(10000, 20000); 15353 ret = i40e_pf_reset(hw); 15354 } 15355 15356 if (ret == 0) 15357 pf->pfr_count++; 15358 else 15359 dev_info(&pf->pdev->dev, "PF reset failed: %d\n", ret); 15360 15361 return ret; 15362 } 15363 15364 /** 15365 * i40e_check_fw_empr - check if FW issued unexpected EMP Reset 15366 * @pf: board private structure 15367 * 15368 * Check FW registers to determine if FW issued unexpected EMP Reset. 15369 * Every time when unexpected EMP Reset occurs the FW increments 15370 * a counter of unexpected EMP Resets. When the counter reaches 10 15371 * the FW should enter the Recovery mode 15372 * 15373 * Returns true if FW issued unexpected EMP Reset 15374 **/ 15375 static bool i40e_check_fw_empr(struct i40e_pf *pf) 15376 { 15377 const u32 fw_sts = rd32(&pf->hw, I40E_GL_FWSTS) & 15378 I40E_GL_FWSTS_FWS1B_MASK; 15379 return (fw_sts > I40E_GL_FWSTS_FWS1B_EMPR_0) && 15380 (fw_sts <= I40E_GL_FWSTS_FWS1B_EMPR_10); 15381 } 15382 15383 /** 15384 * i40e_handle_resets - handle EMP resets and PF resets 15385 * @pf: board private structure 15386 * 15387 * Handle both EMP resets and PF resets and conclude whether there are 15388 * any issues regarding these resets. If there are any issues then 15389 * generate log entry. 15390 * 15391 * Return 0 if NIC is healthy or negative value when there are issues 15392 * with resets 15393 **/ 15394 static int i40e_handle_resets(struct i40e_pf *pf) 15395 { 15396 const int pfr = i40e_pf_loop_reset(pf); 15397 const bool is_empr = i40e_check_fw_empr(pf); 15398 15399 if (is_empr || pfr != 0) 15400 dev_crit(&pf->pdev->dev, "Entering recovery mode due to repeated FW resets. This may take several minutes. Refer to the Intel(R) Ethernet Adapters and Devices User Guide.\n"); 15401 15402 return is_empr ? -EIO : pfr; 15403 } 15404 15405 /** 15406 * i40e_init_recovery_mode - initialize subsystems needed in recovery mode 15407 * @pf: board private structure 15408 * @hw: ptr to the hardware info 15409 * 15410 * This function does a minimal setup of all subsystems needed for running 15411 * recovery mode. 15412 * 15413 * Returns 0 on success, negative on failure 15414 **/ 15415 static int i40e_init_recovery_mode(struct i40e_pf *pf, struct i40e_hw *hw) 15416 { 15417 struct i40e_vsi *vsi; 15418 int err; 15419 int v_idx; 15420 15421 pci_set_drvdata(pf->pdev, pf); 15422 pci_save_state(pf->pdev); 15423 15424 /* set up periodic task facility */ 15425 timer_setup(&pf->service_timer, i40e_service_timer, 0); 15426 pf->service_timer_period = HZ; 15427 15428 INIT_WORK(&pf->service_task, i40e_service_task); 15429 clear_bit(__I40E_SERVICE_SCHED, pf->state); 15430 15431 err = i40e_init_interrupt_scheme(pf); 15432 if (err) 15433 goto err_switch_setup; 15434 15435 /* The number of VSIs reported by the FW is the minimum guaranteed 15436 * to us; HW supports far more and we share the remaining pool with 15437 * the other PFs. We allocate space for more than the guarantee with 15438 * the understanding that we might not get them all later. 15439 */ 15440 if (pf->hw.func_caps.num_vsis < I40E_MIN_VSI_ALLOC) 15441 pf->num_alloc_vsi = I40E_MIN_VSI_ALLOC; 15442 else 15443 pf->num_alloc_vsi = pf->hw.func_caps.num_vsis; 15444 15445 /* Set up the vsi struct and our local tracking of the MAIN PF vsi. */ 15446 pf->vsi = kzalloc_objs(struct i40e_vsi *, pf->num_alloc_vsi); 15447 if (!pf->vsi) { 15448 err = -ENOMEM; 15449 goto err_switch_setup; 15450 } 15451 15452 /* We allocate one VSI which is needed as absolute minimum 15453 * in order to register the netdev 15454 */ 15455 v_idx = i40e_vsi_mem_alloc(pf, I40E_VSI_MAIN); 15456 if (v_idx < 0) { 15457 err = v_idx; 15458 goto err_switch_setup; 15459 } 15460 pf->lan_vsi = v_idx; 15461 vsi = pf->vsi[v_idx]; 15462 if (!vsi) { 15463 err = -EFAULT; 15464 goto err_switch_setup; 15465 } 15466 vsi->alloc_queue_pairs = 1; 15467 err = i40e_config_netdev(vsi); 15468 if (err) 15469 goto err_switch_setup; 15470 err = register_netdev(vsi->netdev); 15471 if (err) 15472 goto err_switch_setup; 15473 vsi->netdev_registered = true; 15474 i40e_dbg_pf_init(pf); 15475 15476 err = i40e_setup_misc_vector_for_recovery_mode(pf); 15477 if (err) 15478 goto err_switch_setup; 15479 15480 /* tell the firmware that we're starting */ 15481 i40e_send_version(pf); 15482 15483 /* since everything's happy, start the service_task timer */ 15484 mod_timer(&pf->service_timer, 15485 round_jiffies(jiffies + pf->service_timer_period)); 15486 15487 return 0; 15488 15489 err_switch_setup: 15490 i40e_reset_interrupt_capability(pf); 15491 timer_shutdown_sync(&pf->service_timer); 15492 i40e_shutdown_adminq(hw); 15493 iounmap(hw->hw_addr); 15494 pci_release_mem_regions(pf->pdev); 15495 pci_disable_device(pf->pdev); 15496 i40e_free_pf(pf); 15497 15498 return err; 15499 } 15500 15501 /** 15502 * i40e_set_subsystem_device_id - set subsystem device id 15503 * @hw: pointer to the hardware info 15504 * 15505 * Set PCI subsystem device id either from a pci_dev structure or 15506 * a specific FW register. 15507 **/ 15508 static inline void i40e_set_subsystem_device_id(struct i40e_hw *hw) 15509 { 15510 struct i40e_pf *pf = i40e_hw_to_pf(hw); 15511 15512 hw->subsystem_device_id = pf->pdev->subsystem_device ? 15513 pf->pdev->subsystem_device : 15514 (ushort)(rd32(hw, I40E_PFPCI_SUBSYSID) & USHRT_MAX); 15515 } 15516 15517 /** 15518 * i40e_probe - Device initialization routine 15519 * @pdev: PCI device information struct 15520 * @ent: entry in i40e_pci_tbl 15521 * 15522 * i40e_probe initializes a PF identified by a pci_dev structure. 15523 * The OS initialization, configuring of the PF private structure, 15524 * and a hardware reset occur. 15525 * 15526 * Returns 0 on success, negative on failure 15527 **/ 15528 static int i40e_probe(struct pci_dev *pdev, const struct pci_device_id *ent) 15529 { 15530 struct i40e_aq_get_phy_abilities_resp abilities; 15531 #ifdef CONFIG_I40E_DCB 15532 enum i40e_get_fw_lldp_status_resp lldp_status; 15533 #endif /* CONFIG_I40E_DCB */ 15534 struct i40e_vsi *vsi; 15535 struct i40e_pf *pf; 15536 struct i40e_hw *hw; 15537 u16 wol_nvm_bits; 15538 char nvm_ver[32]; 15539 u16 link_status; 15540 #ifdef CONFIG_I40E_DCB 15541 int status; 15542 #endif /* CONFIG_I40E_DCB */ 15543 int err; 15544 u32 val; 15545 15546 err = pci_enable_device_mem(pdev); 15547 if (err) 15548 return err; 15549 15550 /* set up for high or low dma */ 15551 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); 15552 if (err) { 15553 dev_err(&pdev->dev, 15554 "DMA configuration failed: 0x%x\n", err); 15555 goto err_dma; 15556 } 15557 15558 /* set up pci connections */ 15559 err = pci_request_mem_regions(pdev, i40e_driver_name); 15560 if (err) { 15561 dev_info(&pdev->dev, 15562 "pci_request_selected_regions failed %d\n", err); 15563 goto err_pci_reg; 15564 } 15565 15566 pci_set_master(pdev); 15567 15568 /* Now that we have a PCI connection, we need to do the 15569 * low level device setup. This is primarily setting up 15570 * the Admin Queue structures and then querying for the 15571 * device's current profile information. 15572 */ 15573 pf = i40e_alloc_pf(&pdev->dev); 15574 if (!pf) { 15575 err = -ENOMEM; 15576 goto err_pf_alloc; 15577 } 15578 pf->next_vsi = 0; 15579 pf->pdev = pdev; 15580 set_bit(__I40E_DOWN, pf->state); 15581 15582 hw = &pf->hw; 15583 15584 pf->ioremap_len = min_t(int, pci_resource_len(pdev, 0), 15585 I40E_MAX_CSR_SPACE); 15586 /* We believe that the highest register to read is 15587 * I40E_GLGEN_STAT_CLEAR, so we check if the BAR size 15588 * is not less than that before mapping to prevent a 15589 * kernel panic. 15590 */ 15591 if (pf->ioremap_len < I40E_GLGEN_STAT_CLEAR) { 15592 dev_err(&pdev->dev, "Cannot map registers, bar size 0x%X too small, aborting\n", 15593 pf->ioremap_len); 15594 err = -ENOMEM; 15595 goto err_ioremap; 15596 } 15597 hw->hw_addr = ioremap(pci_resource_start(pdev, 0), pf->ioremap_len); 15598 if (!hw->hw_addr) { 15599 err = -EIO; 15600 dev_info(&pdev->dev, "ioremap(0x%04x, 0x%04x) failed: 0x%x\n", 15601 (unsigned int)pci_resource_start(pdev, 0), 15602 pf->ioremap_len, err); 15603 goto err_ioremap; 15604 } 15605 hw->vendor_id = pdev->vendor; 15606 hw->device_id = pdev->device; 15607 pci_read_config_byte(pdev, PCI_REVISION_ID, &hw->revision_id); 15608 hw->subsystem_vendor_id = pdev->subsystem_vendor; 15609 i40e_set_subsystem_device_id(hw); 15610 hw->bus.device = PCI_SLOT(pdev->devfn); 15611 hw->bus.func = PCI_FUNC(pdev->devfn); 15612 hw->bus.bus_id = pdev->bus->number; 15613 15614 /* Select something other than the 802.1ad ethertype for the 15615 * switch to use internally and drop on ingress. 15616 */ 15617 hw->switch_tag = 0xffff; 15618 hw->first_tag = ETH_P_8021AD; 15619 hw->second_tag = ETH_P_8021Q; 15620 15621 INIT_LIST_HEAD(&pf->l3_flex_pit_list); 15622 INIT_LIST_HEAD(&pf->l4_flex_pit_list); 15623 INIT_LIST_HEAD(&pf->ddp_old_prof); 15624 15625 /* set up the locks for the AQ, do this only once in probe 15626 * and destroy them only once in remove 15627 */ 15628 mutex_init(&hw->aq.asq_mutex); 15629 mutex_init(&hw->aq.arq_mutex); 15630 15631 pf->msg_enable = netif_msg_init(debug, 15632 NETIF_MSG_DRV | 15633 NETIF_MSG_PROBE | 15634 NETIF_MSG_LINK); 15635 if (debug < -1) 15636 pf->hw.debug_mask = debug; 15637 15638 /* do a special CORER for clearing PXE mode once at init */ 15639 if (hw->revision_id == 0 && 15640 (rd32(hw, I40E_GLLAN_RCTL_0) & I40E_GLLAN_RCTL_0_PXE_MODE_MASK)) { 15641 wr32(hw, I40E_GLGEN_RTRIG, I40E_GLGEN_RTRIG_CORER_MASK); 15642 i40e_flush(hw); 15643 msleep(200); 15644 pf->corer_count++; 15645 15646 i40e_clear_pxe_mode(hw); 15647 } 15648 15649 /* Reset here to make sure all is clean and to define PF 'n' */ 15650 i40e_clear_hw(hw); 15651 15652 err = i40e_set_mac_type(hw); 15653 if (err) { 15654 dev_warn(&pdev->dev, "unidentified MAC or BLANK NVM: %d\n", 15655 err); 15656 goto err_pf_reset; 15657 } 15658 15659 err = i40e_handle_resets(pf); 15660 if (err) 15661 goto err_pf_reset; 15662 15663 i40e_check_recovery_mode(pf); 15664 15665 if (is_kdump_kernel()) { 15666 hw->aq.num_arq_entries = I40E_MIN_ARQ_LEN; 15667 hw->aq.num_asq_entries = I40E_MIN_ASQ_LEN; 15668 } else { 15669 hw->aq.num_arq_entries = I40E_AQ_LEN; 15670 hw->aq.num_asq_entries = I40E_AQ_LEN; 15671 } 15672 hw->aq.arq_buf_size = I40E_MAX_AQ_BUF_SIZE; 15673 hw->aq.asq_buf_size = I40E_MAX_AQ_BUF_SIZE; 15674 15675 snprintf(pf->int_name, sizeof(pf->int_name) - 1, 15676 "%s-%s:misc", 15677 dev_driver_string(&pf->pdev->dev), dev_name(&pdev->dev)); 15678 15679 err = i40e_init_shared_code(hw); 15680 if (err) { 15681 dev_warn(&pdev->dev, "unidentified MAC or BLANK NVM: %d\n", 15682 err); 15683 goto err_pf_reset; 15684 } 15685 15686 /* set up a default setting for link flow control */ 15687 pf->hw.fc.requested_mode = I40E_FC_NONE; 15688 15689 err = i40e_init_adminq(hw); 15690 if (err) { 15691 if (err == -EIO) 15692 dev_info(&pdev->dev, 15693 "The driver for the device stopped because the NVM image v%u.%u is newer than expected v%u.%u. You must install the most recent version of the network driver.\n", 15694 hw->aq.api_maj_ver, 15695 hw->aq.api_min_ver, 15696 I40E_FW_API_VERSION_MAJOR, 15697 I40E_FW_MINOR_VERSION(hw)); 15698 else 15699 dev_info(&pdev->dev, 15700 "The driver for the device stopped because the device firmware failed to init. Try updating your NVM image.\n"); 15701 15702 goto err_pf_reset; 15703 } 15704 i40e_get_oem_version(hw); 15705 i40e_get_pba_string(hw); 15706 15707 /* provide nvm, fw, api versions, vendor:device id, subsys vendor:device id */ 15708 i40e_nvm_version_str(hw, nvm_ver, sizeof(nvm_ver)); 15709 dev_info(&pdev->dev, "fw %d.%d.%05d api %d.%d nvm %s [%04x:%04x] [%04x:%04x]\n", 15710 hw->aq.fw_maj_ver, hw->aq.fw_min_ver, hw->aq.fw_build, 15711 hw->aq.api_maj_ver, hw->aq.api_min_ver, nvm_ver, 15712 hw->vendor_id, hw->device_id, hw->subsystem_vendor_id, 15713 hw->subsystem_device_id); 15714 15715 if (i40e_is_aq_api_ver_ge(hw, I40E_FW_API_VERSION_MAJOR, 15716 I40E_FW_MINOR_VERSION(hw) + 1)) 15717 dev_dbg(&pdev->dev, 15718 "The driver for the device detected a newer version of the NVM image v%u.%u than v%u.%u.\n", 15719 hw->aq.api_maj_ver, 15720 hw->aq.api_min_ver, 15721 I40E_FW_API_VERSION_MAJOR, 15722 I40E_FW_MINOR_VERSION(hw)); 15723 else if (i40e_is_aq_api_ver_lt(hw, 1, 4)) 15724 dev_info(&pdev->dev, 15725 "The driver for the device detected an older version of the NVM image v%u.%u than expected v%u.%u. Please update the NVM image.\n", 15726 hw->aq.api_maj_ver, 15727 hw->aq.api_min_ver, 15728 I40E_FW_API_VERSION_MAJOR, 15729 I40E_FW_MINOR_VERSION(hw)); 15730 15731 i40e_verify_eeprom(pf); 15732 15733 /* Rev 0 hardware was never productized */ 15734 if (hw->revision_id < 1) 15735 dev_warn(&pdev->dev, "This device is a pre-production adapter/LOM. Please be aware there may be issues with your hardware. If you are experiencing problems please contact your Intel or hardware representative who provided you with this hardware.\n"); 15736 15737 i40e_clear_pxe_mode(hw); 15738 15739 err = i40e_get_capabilities(pf, i40e_aqc_opc_list_func_capabilities); 15740 if (err) 15741 goto err_adminq_setup; 15742 15743 err = i40e_sw_init(pf); 15744 if (err) { 15745 dev_info(&pdev->dev, "sw_init failed: %d\n", err); 15746 goto err_sw_init; 15747 } 15748 15749 if (test_bit(__I40E_RECOVERY_MODE, pf->state)) 15750 return i40e_init_recovery_mode(pf, hw); 15751 15752 err = i40e_init_lan_hmc(hw, hw->func_caps.num_tx_qp, 15753 hw->func_caps.num_rx_qp, 0, 0); 15754 if (err) { 15755 dev_info(&pdev->dev, "init_lan_hmc failed: %d\n", err); 15756 goto err_init_lan_hmc; 15757 } 15758 15759 err = i40e_configure_lan_hmc(hw, I40E_HMC_MODEL_DIRECT_ONLY); 15760 if (err) { 15761 dev_info(&pdev->dev, "configure_lan_hmc failed: %d\n", err); 15762 err = -ENOENT; 15763 goto err_configure_lan_hmc; 15764 } 15765 15766 /* Disable LLDP for NICs that have firmware versions lower than v4.3. 15767 * Ignore error return codes because if it was already disabled via 15768 * hardware settings this will fail 15769 */ 15770 if (test_bit(I40E_HW_CAP_STOP_FW_LLDP, pf->hw.caps)) { 15771 dev_info(&pdev->dev, "Stopping firmware LLDP agent.\n"); 15772 i40e_aq_stop_lldp(hw, true, false, NULL); 15773 } 15774 15775 /* allow a platform config to override the HW addr */ 15776 i40e_get_platform_mac_addr(pdev, pf); 15777 15778 if (!is_valid_ether_addr(hw->mac.addr)) { 15779 dev_info(&pdev->dev, "invalid MAC address %pM\n", hw->mac.addr); 15780 err = -EIO; 15781 goto err_mac_addr; 15782 } 15783 dev_info(&pdev->dev, "MAC address: %pM\n", hw->mac.addr); 15784 ether_addr_copy(hw->mac.perm_addr, hw->mac.addr); 15785 i40e_get_port_mac_addr(hw, hw->mac.port_addr); 15786 if (is_valid_ether_addr(hw->mac.port_addr)) 15787 set_bit(I40E_HW_CAP_PORT_ID_VALID, pf->hw.caps); 15788 15789 i40e_ptp_alloc_pins(pf); 15790 pci_set_drvdata(pdev, pf); 15791 pci_save_state(pdev); 15792 15793 #ifdef CONFIG_I40E_DCB 15794 status = i40e_get_fw_lldp_status(&pf->hw, &lldp_status); 15795 (!status && 15796 lldp_status == I40E_GET_FW_LLDP_STATUS_ENABLED) ? 15797 (clear_bit(I40E_FLAG_FW_LLDP_DIS, pf->flags)) : 15798 (set_bit(I40E_FLAG_FW_LLDP_DIS, pf->flags)); 15799 dev_info(&pdev->dev, 15800 test_bit(I40E_FLAG_FW_LLDP_DIS, pf->flags) ? 15801 "FW LLDP is disabled\n" : 15802 "FW LLDP is enabled\n"); 15803 15804 /* Enable FW to write default DCB config on link-up */ 15805 i40e_aq_set_dcb_parameters(hw, true, NULL); 15806 15807 err = i40e_init_pf_dcb(pf); 15808 if (err) { 15809 dev_info(&pdev->dev, "DCB init failed %d, disabled\n", err); 15810 clear_bit(I40E_FLAG_DCB_CAPABLE, pf->flags); 15811 clear_bit(I40E_FLAG_DCB_ENA, pf->flags); 15812 /* Continue without DCB enabled */ 15813 } 15814 #endif /* CONFIG_I40E_DCB */ 15815 15816 /* set up periodic task facility */ 15817 timer_setup(&pf->service_timer, i40e_service_timer, 0); 15818 pf->service_timer_period = HZ; 15819 15820 INIT_WORK(&pf->service_task, i40e_service_task); 15821 clear_bit(__I40E_SERVICE_SCHED, pf->state); 15822 15823 /* NVM bit on means WoL disabled for the port */ 15824 i40e_read_nvm_word(hw, I40E_SR_NVM_WAKE_ON_LAN, &wol_nvm_bits); 15825 if (BIT (hw->port) & wol_nvm_bits || hw->partition_id != 1) 15826 pf->wol_en = false; 15827 else 15828 pf->wol_en = true; 15829 device_set_wakeup_enable(&pf->pdev->dev, pf->wol_en); 15830 15831 /* set up the main switch operations */ 15832 i40e_determine_queue_usage(pf); 15833 err = i40e_init_interrupt_scheme(pf); 15834 if (err) 15835 goto err_switch_setup; 15836 15837 /* Reduce Tx and Rx pairs for kdump 15838 * When MSI-X is enabled, it's not allowed to use more TC queue 15839 * pairs than MSI-X vectors (pf->num_lan_msix) exist. Thus 15840 * vsi->num_queue_pairs will be equal to pf->num_lan_msix, i.e., 1. 15841 */ 15842 if (is_kdump_kernel()) 15843 pf->num_lan_msix = 1; 15844 15845 pf->udp_tunnel_nic.set_port = i40e_udp_tunnel_set_port; 15846 pf->udp_tunnel_nic.unset_port = i40e_udp_tunnel_unset_port; 15847 pf->udp_tunnel_nic.shared = &pf->udp_tunnel_shared; 15848 pf->udp_tunnel_nic.tables[0].n_entries = I40E_MAX_PF_UDP_OFFLOAD_PORTS; 15849 pf->udp_tunnel_nic.tables[0].tunnel_types = UDP_TUNNEL_TYPE_VXLAN | 15850 UDP_TUNNEL_TYPE_GENEVE; 15851 15852 /* The number of VSIs reported by the FW is the minimum guaranteed 15853 * to us; HW supports far more and we share the remaining pool with 15854 * the other PFs. We allocate space for more than the guarantee with 15855 * the understanding that we might not get them all later. 15856 */ 15857 if (pf->hw.func_caps.num_vsis < I40E_MIN_VSI_ALLOC) 15858 pf->num_alloc_vsi = I40E_MIN_VSI_ALLOC; 15859 else 15860 pf->num_alloc_vsi = pf->hw.func_caps.num_vsis; 15861 if (pf->num_alloc_vsi > UDP_TUNNEL_NIC_MAX_SHARING_DEVICES) { 15862 dev_warn(&pf->pdev->dev, 15863 "limiting the VSI count due to UDP tunnel limitation %d > %d\n", 15864 pf->num_alloc_vsi, UDP_TUNNEL_NIC_MAX_SHARING_DEVICES); 15865 pf->num_alloc_vsi = UDP_TUNNEL_NIC_MAX_SHARING_DEVICES; 15866 } 15867 15868 /* Set up the *vsi struct and our local tracking of the MAIN PF vsi. */ 15869 pf->vsi = kzalloc_objs(struct i40e_vsi *, pf->num_alloc_vsi); 15870 if (!pf->vsi) { 15871 err = -ENOMEM; 15872 goto err_switch_setup; 15873 } 15874 15875 #ifdef CONFIG_PCI_IOV 15876 /* prep for VF support */ 15877 if (test_bit(I40E_FLAG_SRIOV_ENA, pf->flags) && 15878 test_bit(I40E_FLAG_MSIX_ENA, pf->flags) && 15879 !test_bit(__I40E_BAD_EEPROM, pf->state)) { 15880 if (pci_num_vf(pdev)) 15881 set_bit(I40E_FLAG_VEB_MODE_ENA, pf->flags); 15882 } 15883 #endif 15884 err = i40e_setup_pf_switch(pf, false, false); 15885 if (err) { 15886 dev_info(&pdev->dev, "setup_pf_switch failed: %d\n", err); 15887 goto err_vsis; 15888 } 15889 15890 vsi = i40e_pf_get_main_vsi(pf); 15891 INIT_LIST_HEAD(&vsi->ch_list); 15892 15893 /* if FDIR VSI was set up, start it now */ 15894 vsi = i40e_find_vsi_by_type(pf, I40E_VSI_FDIR); 15895 if (vsi) 15896 i40e_vsi_open(vsi); 15897 15898 /* The driver only wants link up/down and module qualification 15899 * reports from firmware. Note the negative logic. 15900 */ 15901 err = i40e_aq_set_phy_int_mask(&pf->hw, 15902 ~(I40E_AQ_EVENT_LINK_UPDOWN | 15903 I40E_AQ_EVENT_MEDIA_NA | 15904 I40E_AQ_EVENT_MODULE_QUAL_FAIL), NULL); 15905 if (err) 15906 dev_info(&pf->pdev->dev, "set phy mask fail, err %pe aq_err %s\n", 15907 ERR_PTR(err), libie_aq_str(pf->hw.aq.asq_last_status)); 15908 15909 /* VF MDD event logs are rate limited to one second intervals */ 15910 ratelimit_state_init(&pf->mdd_message_rate_limit, 1 * HZ, 1); 15911 15912 /* Reconfigure hardware for allowing smaller MSS in the case 15913 * of TSO, so that we avoid the MDD being fired and causing 15914 * a reset in the case of small MSS+TSO. 15915 */ 15916 val = rd32(hw, I40E_REG_MSS); 15917 if ((val & I40E_REG_MSS_MIN_MASK) > I40E_64BYTE_MSS) { 15918 val &= ~I40E_REG_MSS_MIN_MASK; 15919 val |= I40E_64BYTE_MSS; 15920 wr32(hw, I40E_REG_MSS, val); 15921 } 15922 15923 if (test_bit(I40E_HW_CAP_RESTART_AUTONEG, pf->hw.caps)) { 15924 msleep(75); 15925 err = i40e_aq_set_link_restart_an(&pf->hw, true, NULL); 15926 if (err) 15927 dev_info(&pf->pdev->dev, "link restart failed, err %pe aq_err %s\n", 15928 ERR_PTR(err), 15929 libie_aq_str(pf->hw.aq.asq_last_status)); 15930 } 15931 /* The main driver is (mostly) up and happy. We need to set this state 15932 * before setting up the misc vector or we get a race and the vector 15933 * ends up disabled forever. 15934 */ 15935 clear_bit(__I40E_DOWN, pf->state); 15936 15937 /* In case of MSIX we are going to setup the misc vector right here 15938 * to handle admin queue events etc. In case of legacy and MSI 15939 * the misc functionality and queue processing is combined in 15940 * the same vector and that gets setup at open. 15941 */ 15942 if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags)) { 15943 err = i40e_setup_misc_vector(pf); 15944 if (err) { 15945 dev_info(&pdev->dev, 15946 "setup of misc vector failed: %d\n", err); 15947 i40e_cloud_filter_exit(pf); 15948 i40e_fdir_teardown(pf); 15949 goto err_vsis; 15950 } 15951 } 15952 15953 #ifdef CONFIG_PCI_IOV 15954 /* prep for VF support */ 15955 if (test_bit(I40E_FLAG_SRIOV_ENA, pf->flags) && 15956 test_bit(I40E_FLAG_MSIX_ENA, pf->flags) && 15957 !test_bit(__I40E_BAD_EEPROM, pf->state)) { 15958 /* disable link interrupts for VFs */ 15959 val = rd32(hw, I40E_PFGEN_PORTMDIO_NUM); 15960 val &= ~I40E_PFGEN_PORTMDIO_NUM_VFLINK_STAT_ENA_MASK; 15961 wr32(hw, I40E_PFGEN_PORTMDIO_NUM, val); 15962 i40e_flush(hw); 15963 15964 if (pci_num_vf(pdev)) { 15965 dev_info(&pdev->dev, 15966 "Active VFs found, allocating resources.\n"); 15967 err = i40e_alloc_vfs(pf, pci_num_vf(pdev)); 15968 if (err) 15969 dev_info(&pdev->dev, 15970 "Error %d allocating resources for existing VFs\n", 15971 err); 15972 } 15973 } 15974 #endif /* CONFIG_PCI_IOV */ 15975 15976 if (test_bit(I40E_FLAG_IWARP_ENA, pf->flags)) { 15977 pf->iwarp_base_vector = i40e_get_lump(pf, pf->irq_pile, 15978 pf->num_iwarp_msix, 15979 I40E_IWARP_IRQ_PILE_ID); 15980 if (pf->iwarp_base_vector < 0) { 15981 dev_info(&pdev->dev, 15982 "failed to get tracking for %d vectors for IWARP err=%d\n", 15983 pf->num_iwarp_msix, pf->iwarp_base_vector); 15984 clear_bit(I40E_FLAG_IWARP_ENA, pf->flags); 15985 } 15986 } 15987 15988 i40e_dbg_pf_init(pf); 15989 15990 /* tell the firmware that we're starting */ 15991 i40e_send_version(pf); 15992 15993 /* since everything's happy, start the service_task timer */ 15994 mod_timer(&pf->service_timer, 15995 round_jiffies(jiffies + pf->service_timer_period)); 15996 15997 /* add this PF to client device list and launch a client service task */ 15998 if (test_bit(I40E_FLAG_IWARP_ENA, pf->flags)) { 15999 err = i40e_lan_add_device(pf); 16000 if (err) 16001 dev_info(&pdev->dev, "Failed to add PF to client API service list: %d\n", 16002 err); 16003 } 16004 16005 #define PCI_SPEED_SIZE 8 16006 #define PCI_WIDTH_SIZE 8 16007 /* Devices on the IOSF bus do not have this information 16008 * and will report PCI Gen 1 x 1 by default so don't bother 16009 * checking them. 16010 */ 16011 if (!test_bit(I40E_HW_CAP_NO_PCI_LINK_CHECK, pf->hw.caps)) { 16012 char speed[PCI_SPEED_SIZE] = "Unknown"; 16013 char width[PCI_WIDTH_SIZE] = "Unknown"; 16014 16015 /* Get the negotiated link width and speed from PCI config 16016 * space 16017 */ 16018 pcie_capability_read_word(pf->pdev, PCI_EXP_LNKSTA, 16019 &link_status); 16020 16021 i40e_set_pci_config_data(hw, link_status); 16022 16023 switch (hw->bus.speed) { 16024 case i40e_bus_speed_8000: 16025 strscpy(speed, "8.0", PCI_SPEED_SIZE); break; 16026 case i40e_bus_speed_5000: 16027 strscpy(speed, "5.0", PCI_SPEED_SIZE); break; 16028 case i40e_bus_speed_2500: 16029 strscpy(speed, "2.5", PCI_SPEED_SIZE); break; 16030 default: 16031 break; 16032 } 16033 switch (hw->bus.width) { 16034 case i40e_bus_width_pcie_x8: 16035 strscpy(width, "8", PCI_WIDTH_SIZE); break; 16036 case i40e_bus_width_pcie_x4: 16037 strscpy(width, "4", PCI_WIDTH_SIZE); break; 16038 case i40e_bus_width_pcie_x2: 16039 strscpy(width, "2", PCI_WIDTH_SIZE); break; 16040 case i40e_bus_width_pcie_x1: 16041 strscpy(width, "1", PCI_WIDTH_SIZE); break; 16042 default: 16043 break; 16044 } 16045 16046 dev_info(&pdev->dev, "PCI-Express: Speed %sGT/s Width x%s\n", 16047 speed, width); 16048 16049 if (hw->bus.width < i40e_bus_width_pcie_x8 || 16050 hw->bus.speed < i40e_bus_speed_8000) { 16051 dev_warn(&pdev->dev, "PCI-Express bandwidth available for this device may be insufficient for optimal performance.\n"); 16052 dev_warn(&pdev->dev, "Please move the device to a different PCI-e link with more lanes and/or higher transfer rate.\n"); 16053 } 16054 } 16055 16056 /* get the requested speeds from the fw */ 16057 err = i40e_aq_get_phy_capabilities(hw, false, false, &abilities, NULL); 16058 if (err) 16059 dev_dbg(&pf->pdev->dev, "get requested speeds ret = %pe last_status = %s\n", 16060 ERR_PTR(err), libie_aq_str(pf->hw.aq.asq_last_status)); 16061 pf->hw.phy.link_info.requested_speeds = abilities.link_speed; 16062 16063 /* set the FEC config due to the board capabilities */ 16064 i40e_set_fec_in_flags(abilities.fec_cfg_curr_mod_ext_info, pf->flags); 16065 16066 /* get the supported phy types from the fw */ 16067 err = i40e_aq_get_phy_capabilities(hw, false, true, &abilities, NULL); 16068 if (err) 16069 dev_dbg(&pf->pdev->dev, "get supported phy types ret = %pe last_status = %s\n", 16070 ERR_PTR(err), libie_aq_str(pf->hw.aq.asq_last_status)); 16071 16072 #define MAX_FRAME_SIZE_DEFAULT 0x2600 16073 16074 err = i40e_aq_set_mac_config(hw, MAX_FRAME_SIZE_DEFAULT, NULL); 16075 if (err) 16076 dev_warn(&pdev->dev, "set mac config ret = %pe last_status = %s\n", 16077 ERR_PTR(err), libie_aq_str(pf->hw.aq.asq_last_status)); 16078 16079 /* Make sure the MFS is set to the expected value */ 16080 val = rd32(hw, I40E_PRTGL_SAH); 16081 FIELD_MODIFY(I40E_PRTGL_SAH_MFS_MASK, &val, MAX_FRAME_SIZE_DEFAULT); 16082 wr32(hw, I40E_PRTGL_SAH, val); 16083 16084 /* Add a filter to drop all Flow control frames from any VSI from being 16085 * transmitted. By doing so we stop a malicious VF from sending out 16086 * PAUSE or PFC frames and potentially controlling traffic for other 16087 * PF/VF VSIs. 16088 * The FW can still send Flow control frames if enabled. 16089 */ 16090 i40e_add_filter_to_drop_tx_flow_control_frames(&pf->hw, 16091 pf->main_vsi_seid); 16092 16093 if ((pf->hw.device_id == I40E_DEV_ID_10G_BASE_T) || 16094 (pf->hw.device_id == I40E_DEV_ID_10G_BASE_T4)) 16095 set_bit(I40E_HW_CAP_PHY_CONTROLS_LEDS, pf->hw.caps); 16096 if (pf->hw.device_id == I40E_DEV_ID_SFP_I_X722) 16097 set_bit(I40E_HW_CAP_CRT_RETIMER, pf->hw.caps); 16098 /* print a string summarizing features */ 16099 i40e_print_features(pf); 16100 16101 i40e_devlink_register(pf); 16102 16103 return 0; 16104 16105 /* Unwind what we've done if something failed in the setup */ 16106 err_vsis: 16107 set_bit(__I40E_DOWN, pf->state); 16108 i40e_ptp_stop(pf); 16109 i40e_clear_interrupt_scheme(pf); 16110 kfree(pf->vsi); 16111 err_switch_setup: 16112 i40e_ptp_free_pins(pf); 16113 i40e_reset_interrupt_capability(pf); 16114 timer_shutdown_sync(&pf->service_timer); 16115 err_mac_addr: 16116 err_configure_lan_hmc: 16117 (void)i40e_shutdown_lan_hmc(hw); 16118 err_init_lan_hmc: 16119 kfree(pf->qp_pile); 16120 err_sw_init: 16121 err_adminq_setup: 16122 err_pf_reset: 16123 iounmap(hw->hw_addr); 16124 err_ioremap: 16125 i40e_free_pf(pf); 16126 err_pf_alloc: 16127 pci_release_mem_regions(pdev); 16128 err_pci_reg: 16129 err_dma: 16130 pci_disable_device(pdev); 16131 return err; 16132 } 16133 16134 /** 16135 * i40e_remove - Device removal routine 16136 * @pdev: PCI device information struct 16137 * 16138 * i40e_remove is called by the PCI subsystem to alert the driver 16139 * that is should release a PCI device. This could be caused by a 16140 * Hot-Plug event, or because the driver is going to be removed from 16141 * memory. 16142 **/ 16143 static void i40e_remove(struct pci_dev *pdev) 16144 { 16145 struct i40e_pf *pf = pci_get_drvdata(pdev); 16146 struct i40e_hw *hw = &pf->hw; 16147 struct i40e_vsi *vsi; 16148 struct i40e_veb *veb; 16149 int ret_code; 16150 int i; 16151 16152 i40e_devlink_unregister(pf); 16153 16154 i40e_dbg_pf_exit(pf); 16155 16156 i40e_ptp_stop(pf); 16157 16158 /* Disable RSS in hw */ 16159 i40e_write_rx_ctl(hw, I40E_PFQF_HENA(0), 0); 16160 i40e_write_rx_ctl(hw, I40E_PFQF_HENA(1), 0); 16161 16162 /* Grab __I40E_RESET_RECOVERY_PENDING and set __I40E_IN_REMOVE 16163 * flags, once they are set, i40e_rebuild should not be called as 16164 * i40e_prep_for_reset always returns early. 16165 */ 16166 while (test_and_set_bit(__I40E_RESET_RECOVERY_PENDING, pf->state)) 16167 usleep_range(1000, 2000); 16168 set_bit(__I40E_IN_REMOVE, pf->state); 16169 16170 if (test_bit(I40E_FLAG_SRIOV_ENA, pf->flags)) { 16171 set_bit(__I40E_VF_RESETS_DISABLED, pf->state); 16172 i40e_free_vfs(pf); 16173 clear_bit(I40E_FLAG_SRIOV_ENA, pf->flags); 16174 } 16175 /* no more scheduling of any task */ 16176 set_bit(__I40E_SUSPENDED, pf->state); 16177 set_bit(__I40E_DOWN, pf->state); 16178 if (pf->service_timer.function) 16179 timer_shutdown_sync(&pf->service_timer); 16180 if (pf->service_task.func) 16181 cancel_work_sync(&pf->service_task); 16182 16183 if (test_bit(__I40E_RECOVERY_MODE, pf->state)) { 16184 struct i40e_vsi *vsi = pf->vsi[0]; 16185 16186 /* We know that we have allocated only one vsi for this PF, 16187 * it was just for registering netdevice, so the interface 16188 * could be visible in the 'ifconfig' output 16189 */ 16190 unregister_netdev(vsi->netdev); 16191 free_netdev(vsi->netdev); 16192 16193 goto unmap; 16194 } 16195 16196 /* Client close must be called explicitly here because the timer 16197 * has been stopped. 16198 */ 16199 i40e_notify_client_of_netdev_close(pf, false); 16200 16201 i40e_fdir_teardown(pf); 16202 16203 /* If there is a switch structure or any orphans, remove them. 16204 * This will leave only the PF's VSI remaining. 16205 */ 16206 i40e_pf_for_each_veb(pf, i, veb) 16207 if (veb->uplink_seid == pf->mac_seid || 16208 veb->uplink_seid == 0) 16209 i40e_switch_branch_release(veb); 16210 16211 /* Now we can shutdown the PF's VSIs, just before we kill 16212 * adminq and hmc. 16213 */ 16214 i40e_pf_for_each_vsi(pf, i, vsi) { 16215 i40e_vsi_close(vsi); 16216 i40e_vsi_release(vsi); 16217 pf->vsi[i] = NULL; 16218 } 16219 16220 i40e_cloud_filter_exit(pf); 16221 16222 /* remove attached clients */ 16223 if (test_bit(I40E_FLAG_IWARP_ENA, pf->flags)) { 16224 ret_code = i40e_lan_del_device(pf); 16225 if (ret_code) 16226 dev_warn(&pdev->dev, "Failed to delete client device: %d\n", 16227 ret_code); 16228 } 16229 16230 /* shutdown and destroy the HMC */ 16231 if (hw->hmc.hmc_obj) { 16232 ret_code = i40e_shutdown_lan_hmc(hw); 16233 if (ret_code) 16234 dev_warn(&pdev->dev, 16235 "Failed to destroy the HMC resources: %d\n", 16236 ret_code); 16237 } 16238 16239 unmap: 16240 /* Free MSI/legacy interrupt 0 when in recovery mode. */ 16241 if (test_bit(__I40E_RECOVERY_MODE, pf->state) && 16242 !test_bit(I40E_FLAG_MSIX_ENA, pf->flags)) 16243 free_irq(pf->pdev->irq, pf); 16244 16245 /* shutdown the adminq */ 16246 i40e_shutdown_adminq(hw); 16247 16248 /* destroy the locks only once, here */ 16249 mutex_destroy(&hw->aq.arq_mutex); 16250 mutex_destroy(&hw->aq.asq_mutex); 16251 16252 /* Clear all dynamic memory lists of rings, q_vectors, and VSIs */ 16253 rtnl_lock(); 16254 i40e_clear_interrupt_scheme(pf); 16255 i40e_pf_for_each_vsi(pf, i, vsi) { 16256 if (!test_bit(__I40E_RECOVERY_MODE, pf->state)) 16257 i40e_vsi_clear_rings(vsi); 16258 16259 i40e_vsi_clear(vsi); 16260 pf->vsi[i] = NULL; 16261 } 16262 rtnl_unlock(); 16263 16264 i40e_pf_for_each_veb(pf, i, veb) { 16265 kfree(veb); 16266 pf->veb[i] = NULL; 16267 } 16268 16269 kfree(pf->qp_pile); 16270 kfree(pf->vsi); 16271 16272 iounmap(hw->hw_addr); 16273 i40e_free_pf(pf); 16274 pci_release_mem_regions(pdev); 16275 16276 pci_disable_device(pdev); 16277 } 16278 16279 /** 16280 * i40e_enable_mc_magic_wake - enable multicast magic packet wake up 16281 * using the mac_address_write admin q function 16282 * @pf: pointer to i40e_pf struct 16283 **/ 16284 static void i40e_enable_mc_magic_wake(struct i40e_pf *pf) 16285 { 16286 struct i40e_vsi *main_vsi = i40e_pf_get_main_vsi(pf); 16287 struct i40e_hw *hw = &pf->hw; 16288 u8 mac_addr[6]; 16289 u16 flags = 0; 16290 int ret; 16291 16292 /* Get current MAC address in case it's an LAA */ 16293 if (main_vsi && main_vsi->netdev) { 16294 ether_addr_copy(mac_addr, main_vsi->netdev->dev_addr); 16295 } else { 16296 dev_err(&pf->pdev->dev, 16297 "Failed to retrieve MAC address; using default\n"); 16298 ether_addr_copy(mac_addr, hw->mac.addr); 16299 } 16300 16301 /* The FW expects the mac address write cmd to first be called with 16302 * one of these flags before calling it again with the multicast 16303 * enable flags. 16304 */ 16305 flags = I40E_AQC_WRITE_TYPE_LAA_WOL; 16306 16307 if (hw->func_caps.flex10_enable && hw->partition_id != 1) 16308 flags = I40E_AQC_WRITE_TYPE_LAA_ONLY; 16309 16310 ret = i40e_aq_mac_address_write(hw, flags, mac_addr, NULL); 16311 if (ret) { 16312 dev_err(&pf->pdev->dev, 16313 "Failed to update MAC address registers; cannot enable Multicast Magic packet wake up"); 16314 return; 16315 } 16316 16317 flags = I40E_AQC_MC_MAG_EN 16318 | I40E_AQC_WOL_PRESERVE_ON_PFR 16319 | I40E_AQC_WRITE_TYPE_UPDATE_MC_MAG; 16320 ret = i40e_aq_mac_address_write(hw, flags, mac_addr, NULL); 16321 if (ret) 16322 dev_err(&pf->pdev->dev, 16323 "Failed to enable Multicast Magic Packet wake up\n"); 16324 } 16325 16326 /** 16327 * i40e_io_suspend - suspend all IO operations 16328 * @pf: pointer to i40e_pf struct 16329 * 16330 **/ 16331 static int i40e_io_suspend(struct i40e_pf *pf) 16332 { 16333 struct i40e_hw *hw = &pf->hw; 16334 16335 set_bit(__I40E_DOWN, pf->state); 16336 16337 /* Ensure service task will not be running */ 16338 timer_delete_sync(&pf->service_timer); 16339 cancel_work_sync(&pf->service_task); 16340 16341 /* Client close must be called explicitly here because the timer 16342 * has been stopped. 16343 */ 16344 i40e_notify_client_of_netdev_close(pf, false); 16345 16346 if (test_bit(I40E_HW_CAP_WOL_MC_MAGIC_PKT_WAKE, pf->hw.caps) && 16347 pf->wol_en) 16348 i40e_enable_mc_magic_wake(pf); 16349 16350 /* Since we're going to destroy queues during the 16351 * i40e_clear_interrupt_scheme() we should hold the RTNL lock for this 16352 * whole section 16353 */ 16354 rtnl_lock(); 16355 16356 i40e_prep_for_reset(pf); 16357 16358 wr32(hw, I40E_PFPM_APM, (pf->wol_en ? I40E_PFPM_APM_APME_MASK : 0)); 16359 wr32(hw, I40E_PFPM_WUFC, (pf->wol_en ? I40E_PFPM_WUFC_MAG_MASK : 0)); 16360 16361 /* Clear the interrupt scheme and release our IRQs so that the system 16362 * can safely hibernate even when there are a large number of CPUs. 16363 * Otherwise hibernation might fail when mapping all the vectors back 16364 * to CPU0. 16365 */ 16366 i40e_clear_interrupt_scheme(pf); 16367 16368 rtnl_unlock(); 16369 16370 return 0; 16371 } 16372 16373 /** 16374 * i40e_io_resume - resume IO operations 16375 * @pf: pointer to i40e_pf struct 16376 * 16377 **/ 16378 static int i40e_io_resume(struct i40e_pf *pf) 16379 { 16380 struct device *dev = &pf->pdev->dev; 16381 int err; 16382 16383 /* We need to hold the RTNL lock prior to restoring interrupt schemes, 16384 * since we're going to be restoring queues 16385 */ 16386 rtnl_lock(); 16387 16388 /* We cleared the interrupt scheme when we suspended, so we need to 16389 * restore it now to resume device functionality. 16390 */ 16391 err = i40e_restore_interrupt_scheme(pf); 16392 if (err) { 16393 dev_err(dev, "Cannot restore interrupt scheme: %d\n", 16394 err); 16395 } 16396 16397 clear_bit(__I40E_DOWN, pf->state); 16398 i40e_reset_and_rebuild(pf, false, true); 16399 16400 rtnl_unlock(); 16401 16402 /* Clear suspended state last after everything is recovered */ 16403 clear_bit(__I40E_SUSPENDED, pf->state); 16404 16405 /* Restart the service task */ 16406 mod_timer(&pf->service_timer, 16407 round_jiffies(jiffies + pf->service_timer_period)); 16408 16409 return 0; 16410 } 16411 16412 /** 16413 * i40e_pci_error_detected - warning that something funky happened in PCI land 16414 * @pdev: PCI device information struct 16415 * @error: the type of PCI error 16416 * 16417 * Called to warn that something happened and the error handling steps 16418 * are in progress. Allows the driver to quiesce things, be ready for 16419 * remediation. 16420 **/ 16421 static pci_ers_result_t i40e_pci_error_detected(struct pci_dev *pdev, 16422 pci_channel_state_t error) 16423 { 16424 struct i40e_pf *pf = pci_get_drvdata(pdev); 16425 16426 dev_info(&pdev->dev, "%s: error %d\n", __func__, error); 16427 16428 if (!pf) { 16429 dev_info(&pdev->dev, 16430 "Cannot recover - error happened during device probe\n"); 16431 return PCI_ERS_RESULT_DISCONNECT; 16432 } 16433 16434 /* shutdown all operations */ 16435 if (!test_bit(__I40E_SUSPENDED, pf->state)) 16436 i40e_io_suspend(pf); 16437 16438 /* Request a slot reset */ 16439 return PCI_ERS_RESULT_NEED_RESET; 16440 } 16441 16442 /** 16443 * i40e_pci_error_slot_reset - a PCI slot reset just happened 16444 * @pdev: PCI device information struct 16445 * 16446 * Called to find if the driver can work with the device now that 16447 * the pci slot has been reset. If a basic connection seems good 16448 * (registers are readable and have sane content) then return a 16449 * happy little PCI_ERS_RESULT_xxx. 16450 **/ 16451 static pci_ers_result_t i40e_pci_error_slot_reset(struct pci_dev *pdev) 16452 { 16453 struct i40e_pf *pf = pci_get_drvdata(pdev); 16454 pci_ers_result_t result; 16455 u32 reg; 16456 16457 dev_dbg(&pdev->dev, "%s\n", __func__); 16458 /* enable I/O and memory of the device */ 16459 if (pci_enable_device(pdev)) { 16460 dev_info(&pdev->dev, 16461 "Cannot re-enable PCI device after reset.\n"); 16462 result = PCI_ERS_RESULT_DISCONNECT; 16463 } else { 16464 pci_set_master(pdev); 16465 pci_restore_state(pdev); 16466 pci_wake_from_d3(pdev, false); 16467 16468 reg = rd32(&pf->hw, I40E_GLGEN_RTRIG); 16469 if (reg == 0) 16470 result = PCI_ERS_RESULT_RECOVERED; 16471 else 16472 result = PCI_ERS_RESULT_DISCONNECT; 16473 } 16474 16475 return result; 16476 } 16477 16478 /** 16479 * i40e_pci_error_reset_prepare - prepare device driver for pci reset 16480 * @pdev: PCI device information struct 16481 */ 16482 static void i40e_pci_error_reset_prepare(struct pci_dev *pdev) 16483 { 16484 struct i40e_pf *pf = pci_get_drvdata(pdev); 16485 16486 i40e_prep_for_reset(pf); 16487 } 16488 16489 /** 16490 * i40e_pci_error_reset_done - pci reset done, device driver reset can begin 16491 * @pdev: PCI device information struct 16492 */ 16493 static void i40e_pci_error_reset_done(struct pci_dev *pdev) 16494 { 16495 struct i40e_pf *pf = pci_get_drvdata(pdev); 16496 16497 if (test_bit(__I40E_IN_REMOVE, pf->state)) 16498 return; 16499 16500 i40e_reset_and_rebuild(pf, false, false); 16501 #ifdef CONFIG_PCI_IOV 16502 i40e_restore_all_vfs_msi_state(pdev); 16503 #endif /* CONFIG_PCI_IOV */ 16504 } 16505 16506 /** 16507 * i40e_pci_error_resume - restart operations after PCI error recovery 16508 * @pdev: PCI device information struct 16509 * 16510 * Called to allow the driver to bring things back up after PCI error 16511 * and/or reset recovery has finished. 16512 **/ 16513 static void i40e_pci_error_resume(struct pci_dev *pdev) 16514 { 16515 struct i40e_pf *pf = pci_get_drvdata(pdev); 16516 16517 dev_dbg(&pdev->dev, "%s\n", __func__); 16518 if (test_bit(__I40E_SUSPENDED, pf->state)) 16519 return; 16520 16521 i40e_io_resume(pf); 16522 } 16523 16524 /** 16525 * i40e_shutdown - PCI callback for shutting down 16526 * @pdev: PCI device information struct 16527 **/ 16528 static void i40e_shutdown(struct pci_dev *pdev) 16529 { 16530 struct i40e_pf *pf = pci_get_drvdata(pdev); 16531 struct i40e_hw *hw = &pf->hw; 16532 16533 set_bit(__I40E_SUSPENDED, pf->state); 16534 set_bit(__I40E_DOWN, pf->state); 16535 16536 timer_delete_sync(&pf->service_timer); 16537 cancel_work_sync(&pf->service_task); 16538 i40e_cloud_filter_exit(pf); 16539 i40e_fdir_teardown(pf); 16540 16541 /* Client close must be called explicitly here because the timer 16542 * has been stopped. 16543 */ 16544 i40e_notify_client_of_netdev_close(pf, false); 16545 16546 if (test_bit(I40E_HW_CAP_WOL_MC_MAGIC_PKT_WAKE, pf->hw.caps) && 16547 pf->wol_en) 16548 i40e_enable_mc_magic_wake(pf); 16549 16550 i40e_prep_for_reset(pf); 16551 16552 wr32(hw, I40E_PFPM_APM, 16553 (pf->wol_en ? I40E_PFPM_APM_APME_MASK : 0)); 16554 wr32(hw, I40E_PFPM_WUFC, 16555 (pf->wol_en ? I40E_PFPM_WUFC_MAG_MASK : 0)); 16556 16557 /* Free MSI/legacy interrupt 0 when in recovery mode. */ 16558 if (test_bit(__I40E_RECOVERY_MODE, pf->state) && 16559 !test_bit(I40E_FLAG_MSIX_ENA, pf->flags)) 16560 free_irq(pf->pdev->irq, pf); 16561 16562 /* Since we're going to destroy queues during the 16563 * i40e_clear_interrupt_scheme() we should hold the RTNL lock for this 16564 * whole section 16565 */ 16566 rtnl_lock(); 16567 i40e_clear_interrupt_scheme(pf); 16568 rtnl_unlock(); 16569 16570 if (system_state == SYSTEM_POWER_OFF) { 16571 pci_wake_from_d3(pdev, pf->wol_en); 16572 pci_set_power_state(pdev, PCI_D3hot); 16573 } 16574 } 16575 16576 /** 16577 * i40e_suspend - PM callback for moving to D3 16578 * @dev: generic device information structure 16579 **/ 16580 static int i40e_suspend(struct device *dev) 16581 { 16582 struct i40e_pf *pf = dev_get_drvdata(dev); 16583 16584 /* If we're already suspended, then there is nothing to do */ 16585 if (test_and_set_bit(__I40E_SUSPENDED, pf->state)) 16586 return 0; 16587 return i40e_io_suspend(pf); 16588 } 16589 16590 /** 16591 * i40e_resume - PM callback for waking up from D3 16592 * @dev: generic device information structure 16593 **/ 16594 static int i40e_resume(struct device *dev) 16595 { 16596 struct i40e_pf *pf = dev_get_drvdata(dev); 16597 16598 /* If we're not suspended, then there is nothing to do */ 16599 if (!test_bit(__I40E_SUSPENDED, pf->state)) 16600 return 0; 16601 return i40e_io_resume(pf); 16602 } 16603 16604 static const struct pci_error_handlers i40e_err_handler = { 16605 .error_detected = i40e_pci_error_detected, 16606 .slot_reset = i40e_pci_error_slot_reset, 16607 .reset_prepare = i40e_pci_error_reset_prepare, 16608 .reset_done = i40e_pci_error_reset_done, 16609 .resume = i40e_pci_error_resume, 16610 }; 16611 16612 static DEFINE_SIMPLE_DEV_PM_OPS(i40e_pm_ops, i40e_suspend, i40e_resume); 16613 16614 static struct pci_driver i40e_driver = { 16615 .name = i40e_driver_name, 16616 .id_table = i40e_pci_tbl, 16617 .probe = i40e_probe, 16618 .remove = i40e_remove, 16619 .driver.pm = pm_sleep_ptr(&i40e_pm_ops), 16620 .shutdown = i40e_shutdown, 16621 .err_handler = &i40e_err_handler, 16622 .sriov_configure = i40e_pci_sriov_configure, 16623 }; 16624 16625 /** 16626 * i40e_init_module - Driver registration routine 16627 * 16628 * i40e_init_module is the first routine called when the driver is 16629 * loaded. All it does is register with the PCI subsystem. 16630 **/ 16631 static int __init i40e_init_module(void) 16632 { 16633 int err; 16634 16635 pr_info("%s: %s\n", i40e_driver_name, i40e_driver_string); 16636 pr_info("%s: %s\n", i40e_driver_name, i40e_copyright); 16637 16638 /* There is no need to throttle the number of active tasks because 16639 * each device limits its own task using a state bit for scheduling 16640 * the service task, and the device tasks do not interfere with each 16641 * other, so we don't set a max task limit. We must set WQ_MEM_RECLAIM 16642 * since we need to be able to guarantee forward progress even under 16643 * memory pressure. 16644 */ 16645 i40e_wq = alloc_workqueue("%s", WQ_PERCPU, 0, i40e_driver_name); 16646 if (!i40e_wq) { 16647 pr_err("%s: Failed to create workqueue\n", i40e_driver_name); 16648 return -ENOMEM; 16649 } 16650 16651 i40e_dbg_init(); 16652 err = pci_register_driver(&i40e_driver); 16653 if (err) { 16654 destroy_workqueue(i40e_wq); 16655 i40e_dbg_exit(); 16656 return err; 16657 } 16658 16659 return 0; 16660 } 16661 module_init(i40e_init_module); 16662 16663 /** 16664 * i40e_exit_module - Driver exit cleanup routine 16665 * 16666 * i40e_exit_module is called just before the driver is removed 16667 * from memory. 16668 **/ 16669 static void __exit i40e_exit_module(void) 16670 { 16671 pci_unregister_driver(&i40e_driver); 16672 destroy_workqueue(i40e_wq); 16673 ida_destroy(&i40e_client_ida); 16674 i40e_dbg_exit(); 16675 } 16676 module_exit(i40e_exit_module); 16677