1 /******************************************************************************* 2 * 3 * Intel Ethernet Controller XL710 Family Linux Driver 4 * Copyright(c) 2013 - 2017 Intel Corporation. 5 * 6 * This program is free software; you can redistribute it and/or modify it 7 * under the terms and conditions of the GNU General Public License, 8 * version 2, as published by the Free Software Foundation. 9 * 10 * This program is distributed in the hope it will be useful, but WITHOUT 11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 13 * more details. 14 * 15 * You should have received a copy of the GNU General Public License along 16 * with this program. If not, see <http://www.gnu.org/licenses/>. 17 * 18 * The full GNU General Public License is included in this distribution in 19 * the file called "COPYING". 20 * 21 * Contact Information: 22 * e1000-devel Mailing List <e1000-devel@lists.sourceforge.net> 23 * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497 24 * 25 ******************************************************************************/ 26 27 #include <linux/etherdevice.h> 28 #include <linux/of_net.h> 29 #include <linux/pci.h> 30 #include <linux/bpf.h> 31 32 /* Local includes */ 33 #include "i40e.h" 34 #include "i40e_diag.h" 35 #include <net/udp_tunnel.h> 36 /* All i40e tracepoints are defined by the include below, which 37 * must be included exactly once across the whole kernel with 38 * CREATE_TRACE_POINTS defined 39 */ 40 #define CREATE_TRACE_POINTS 41 #include "i40e_trace.h" 42 43 const char i40e_driver_name[] = "i40e"; 44 static const char i40e_driver_string[] = 45 "Intel(R) Ethernet Connection XL710 Network Driver"; 46 47 #define DRV_KERN "-k" 48 49 #define DRV_VERSION_MAJOR 2 50 #define DRV_VERSION_MINOR 3 51 #define DRV_VERSION_BUILD 2 52 #define DRV_VERSION __stringify(DRV_VERSION_MAJOR) "." \ 53 __stringify(DRV_VERSION_MINOR) "." \ 54 __stringify(DRV_VERSION_BUILD) DRV_KERN 55 const char i40e_driver_version_str[] = DRV_VERSION; 56 static const char i40e_copyright[] = "Copyright (c) 2013 - 2014 Intel Corporation."; 57 58 /* a bit of forward declarations */ 59 static void i40e_vsi_reinit_locked(struct i40e_vsi *vsi); 60 static void i40e_handle_reset_warning(struct i40e_pf *pf, bool lock_acquired); 61 static int i40e_add_vsi(struct i40e_vsi *vsi); 62 static int i40e_add_veb(struct i40e_veb *veb, struct i40e_vsi *vsi); 63 static int i40e_setup_pf_switch(struct i40e_pf *pf, bool reinit); 64 static int i40e_setup_misc_vector(struct i40e_pf *pf); 65 static void i40e_determine_queue_usage(struct i40e_pf *pf); 66 static int i40e_setup_pf_filter_control(struct i40e_pf *pf); 67 static void i40e_prep_for_reset(struct i40e_pf *pf, bool lock_acquired); 68 static int i40e_reset(struct i40e_pf *pf); 69 static void i40e_rebuild(struct i40e_pf *pf, bool reinit, bool lock_acquired); 70 static void i40e_fdir_sb_setup(struct i40e_pf *pf); 71 static int i40e_veb_get_bw_info(struct i40e_veb *veb); 72 static int i40e_add_del_cloud_filter(struct i40e_vsi *vsi, 73 struct i40e_cloud_filter *filter, 74 bool add); 75 static int i40e_add_del_cloud_filter_big_buf(struct i40e_vsi *vsi, 76 struct i40e_cloud_filter *filter, 77 bool add); 78 static int i40e_get_capabilities(struct i40e_pf *pf, 79 enum i40e_admin_queue_opc list_type); 80 81 82 /* i40e_pci_tbl - PCI Device ID Table 83 * 84 * Last entry must be all 0s 85 * 86 * { Vendor ID, Device ID, SubVendor ID, SubDevice ID, 87 * Class, Class Mask, private data (not used) } 88 */ 89 static const struct pci_device_id i40e_pci_tbl[] = { 90 {PCI_VDEVICE(INTEL, I40E_DEV_ID_SFP_XL710), 0}, 91 {PCI_VDEVICE(INTEL, I40E_DEV_ID_QEMU), 0}, 92 {PCI_VDEVICE(INTEL, I40E_DEV_ID_KX_B), 0}, 93 {PCI_VDEVICE(INTEL, I40E_DEV_ID_KX_C), 0}, 94 {PCI_VDEVICE(INTEL, I40E_DEV_ID_QSFP_A), 0}, 95 {PCI_VDEVICE(INTEL, I40E_DEV_ID_QSFP_B), 0}, 96 {PCI_VDEVICE(INTEL, I40E_DEV_ID_QSFP_C), 0}, 97 {PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_BASE_T), 0}, 98 {PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_BASE_T4), 0}, 99 {PCI_VDEVICE(INTEL, I40E_DEV_ID_KX_X722), 0}, 100 {PCI_VDEVICE(INTEL, I40E_DEV_ID_QSFP_X722), 0}, 101 {PCI_VDEVICE(INTEL, I40E_DEV_ID_SFP_X722), 0}, 102 {PCI_VDEVICE(INTEL, I40E_DEV_ID_1G_BASE_T_X722), 0}, 103 {PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_BASE_T_X722), 0}, 104 {PCI_VDEVICE(INTEL, I40E_DEV_ID_SFP_I_X722), 0}, 105 {PCI_VDEVICE(INTEL, I40E_DEV_ID_20G_KR2), 0}, 106 {PCI_VDEVICE(INTEL, I40E_DEV_ID_20G_KR2_A), 0}, 107 {PCI_VDEVICE(INTEL, I40E_DEV_ID_25G_B), 0}, 108 {PCI_VDEVICE(INTEL, I40E_DEV_ID_25G_SFP28), 0}, 109 /* required last entry */ 110 {0, } 111 }; 112 MODULE_DEVICE_TABLE(pci, i40e_pci_tbl); 113 114 #define I40E_MAX_VF_COUNT 128 115 static int debug = -1; 116 module_param(debug, uint, 0); 117 MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all), Debug mask (0x8XXXXXXX)"); 118 119 MODULE_AUTHOR("Intel Corporation, <e1000-devel@lists.sourceforge.net>"); 120 MODULE_DESCRIPTION("Intel(R) Ethernet Connection XL710 Network Driver"); 121 MODULE_LICENSE("GPL"); 122 MODULE_VERSION(DRV_VERSION); 123 124 static struct workqueue_struct *i40e_wq; 125 126 /** 127 * i40e_allocate_dma_mem_d - OS specific memory alloc for shared code 128 * @hw: pointer to the HW structure 129 * @mem: ptr to mem struct to fill out 130 * @size: size of memory requested 131 * @alignment: what to align the allocation to 132 **/ 133 int i40e_allocate_dma_mem_d(struct i40e_hw *hw, struct i40e_dma_mem *mem, 134 u64 size, u32 alignment) 135 { 136 struct i40e_pf *pf = (struct i40e_pf *)hw->back; 137 138 mem->size = ALIGN(size, alignment); 139 mem->va = dma_zalloc_coherent(&pf->pdev->dev, mem->size, 140 &mem->pa, GFP_KERNEL); 141 if (!mem->va) 142 return -ENOMEM; 143 144 return 0; 145 } 146 147 /** 148 * i40e_free_dma_mem_d - OS specific memory free for shared code 149 * @hw: pointer to the HW structure 150 * @mem: ptr to mem struct to free 151 **/ 152 int i40e_free_dma_mem_d(struct i40e_hw *hw, struct i40e_dma_mem *mem) 153 { 154 struct i40e_pf *pf = (struct i40e_pf *)hw->back; 155 156 dma_free_coherent(&pf->pdev->dev, mem->size, mem->va, mem->pa); 157 mem->va = NULL; 158 mem->pa = 0; 159 mem->size = 0; 160 161 return 0; 162 } 163 164 /** 165 * i40e_allocate_virt_mem_d - OS specific memory alloc for shared code 166 * @hw: pointer to the HW structure 167 * @mem: ptr to mem struct to fill out 168 * @size: size of memory requested 169 **/ 170 int i40e_allocate_virt_mem_d(struct i40e_hw *hw, struct i40e_virt_mem *mem, 171 u32 size) 172 { 173 mem->size = size; 174 mem->va = kzalloc(size, GFP_KERNEL); 175 176 if (!mem->va) 177 return -ENOMEM; 178 179 return 0; 180 } 181 182 /** 183 * i40e_free_virt_mem_d - OS specific memory free for shared code 184 * @hw: pointer to the HW structure 185 * @mem: ptr to mem struct to free 186 **/ 187 int i40e_free_virt_mem_d(struct i40e_hw *hw, struct i40e_virt_mem *mem) 188 { 189 /* it's ok to kfree a NULL pointer */ 190 kfree(mem->va); 191 mem->va = NULL; 192 mem->size = 0; 193 194 return 0; 195 } 196 197 /** 198 * i40e_get_lump - find a lump of free generic resource 199 * @pf: board private structure 200 * @pile: the pile of resource to search 201 * @needed: the number of items needed 202 * @id: an owner id to stick on the items assigned 203 * 204 * Returns the base item index of the lump, or negative for error 205 * 206 * The search_hint trick and lack of advanced fit-finding only work 207 * because we're highly likely to have all the same size lump requests. 208 * Linear search time and any fragmentation should be minimal. 209 **/ 210 static int i40e_get_lump(struct i40e_pf *pf, struct i40e_lump_tracking *pile, 211 u16 needed, u16 id) 212 { 213 int ret = -ENOMEM; 214 int i, j; 215 216 if (!pile || needed == 0 || id >= I40E_PILE_VALID_BIT) { 217 dev_info(&pf->pdev->dev, 218 "param err: pile=%p needed=%d id=0x%04x\n", 219 pile, needed, id); 220 return -EINVAL; 221 } 222 223 /* start the linear search with an imperfect hint */ 224 i = pile->search_hint; 225 while (i < pile->num_entries) { 226 /* skip already allocated entries */ 227 if (pile->list[i] & I40E_PILE_VALID_BIT) { 228 i++; 229 continue; 230 } 231 232 /* do we have enough in this lump? */ 233 for (j = 0; (j < needed) && ((i+j) < pile->num_entries); j++) { 234 if (pile->list[i+j] & I40E_PILE_VALID_BIT) 235 break; 236 } 237 238 if (j == needed) { 239 /* there was enough, so assign it to the requestor */ 240 for (j = 0; j < needed; j++) 241 pile->list[i+j] = id | I40E_PILE_VALID_BIT; 242 ret = i; 243 pile->search_hint = i + j; 244 break; 245 } 246 247 /* not enough, so skip over it and continue looking */ 248 i += j; 249 } 250 251 return ret; 252 } 253 254 /** 255 * i40e_put_lump - return a lump of generic resource 256 * @pile: the pile of resource to search 257 * @index: the base item index 258 * @id: the owner id of the items assigned 259 * 260 * Returns the count of items in the lump 261 **/ 262 static int i40e_put_lump(struct i40e_lump_tracking *pile, u16 index, u16 id) 263 { 264 int valid_id = (id | I40E_PILE_VALID_BIT); 265 int count = 0; 266 int i; 267 268 if (!pile || index >= pile->num_entries) 269 return -EINVAL; 270 271 for (i = index; 272 i < pile->num_entries && pile->list[i] == valid_id; 273 i++) { 274 pile->list[i] = 0; 275 count++; 276 } 277 278 if (count && index < pile->search_hint) 279 pile->search_hint = index; 280 281 return count; 282 } 283 284 /** 285 * i40e_find_vsi_from_id - searches for the vsi with the given id 286 * @pf - the pf structure to search for the vsi 287 * @id - id of the vsi it is searching for 288 **/ 289 struct i40e_vsi *i40e_find_vsi_from_id(struct i40e_pf *pf, u16 id) 290 { 291 int i; 292 293 for (i = 0; i < pf->num_alloc_vsi; i++) 294 if (pf->vsi[i] && (pf->vsi[i]->id == id)) 295 return pf->vsi[i]; 296 297 return NULL; 298 } 299 300 /** 301 * i40e_service_event_schedule - Schedule the service task to wake up 302 * @pf: board private structure 303 * 304 * If not already scheduled, this puts the task into the work queue 305 **/ 306 void i40e_service_event_schedule(struct i40e_pf *pf) 307 { 308 if (!test_bit(__I40E_DOWN, pf->state) && 309 !test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state)) 310 queue_work(i40e_wq, &pf->service_task); 311 } 312 313 /** 314 * i40e_tx_timeout - Respond to a Tx Hang 315 * @netdev: network interface device structure 316 * 317 * If any port has noticed a Tx timeout, it is likely that the whole 318 * device is munged, not just the one netdev port, so go for the full 319 * reset. 320 **/ 321 static void i40e_tx_timeout(struct net_device *netdev) 322 { 323 struct i40e_netdev_priv *np = netdev_priv(netdev); 324 struct i40e_vsi *vsi = np->vsi; 325 struct i40e_pf *pf = vsi->back; 326 struct i40e_ring *tx_ring = NULL; 327 unsigned int i, hung_queue = 0; 328 u32 head, val; 329 330 pf->tx_timeout_count++; 331 332 /* find the stopped queue the same way the stack does */ 333 for (i = 0; i < netdev->num_tx_queues; i++) { 334 struct netdev_queue *q; 335 unsigned long trans_start; 336 337 q = netdev_get_tx_queue(netdev, i); 338 trans_start = q->trans_start; 339 if (netif_xmit_stopped(q) && 340 time_after(jiffies, 341 (trans_start + netdev->watchdog_timeo))) { 342 hung_queue = i; 343 break; 344 } 345 } 346 347 if (i == netdev->num_tx_queues) { 348 netdev_info(netdev, "tx_timeout: no netdev hung queue found\n"); 349 } else { 350 /* now that we have an index, find the tx_ring struct */ 351 for (i = 0; i < vsi->num_queue_pairs; i++) { 352 if (vsi->tx_rings[i] && vsi->tx_rings[i]->desc) { 353 if (hung_queue == 354 vsi->tx_rings[i]->queue_index) { 355 tx_ring = vsi->tx_rings[i]; 356 break; 357 } 358 } 359 } 360 } 361 362 if (time_after(jiffies, (pf->tx_timeout_last_recovery + HZ*20))) 363 pf->tx_timeout_recovery_level = 1; /* reset after some time */ 364 else if (time_before(jiffies, 365 (pf->tx_timeout_last_recovery + netdev->watchdog_timeo))) 366 return; /* don't do any new action before the next timeout */ 367 368 if (tx_ring) { 369 head = i40e_get_head(tx_ring); 370 /* Read interrupt register */ 371 if (pf->flags & I40E_FLAG_MSIX_ENABLED) 372 val = rd32(&pf->hw, 373 I40E_PFINT_DYN_CTLN(tx_ring->q_vector->v_idx + 374 tx_ring->vsi->base_vector - 1)); 375 else 376 val = rd32(&pf->hw, I40E_PFINT_DYN_CTL0); 377 378 netdev_info(netdev, "tx_timeout: VSI_seid: %d, Q %d, NTC: 0x%x, HWB: 0x%x, NTU: 0x%x, TAIL: 0x%x, INT: 0x%x\n", 379 vsi->seid, hung_queue, tx_ring->next_to_clean, 380 head, tx_ring->next_to_use, 381 readl(tx_ring->tail), val); 382 } 383 384 pf->tx_timeout_last_recovery = jiffies; 385 netdev_info(netdev, "tx_timeout recovery level %d, hung_queue %d\n", 386 pf->tx_timeout_recovery_level, hung_queue); 387 388 switch (pf->tx_timeout_recovery_level) { 389 case 1: 390 set_bit(__I40E_PF_RESET_REQUESTED, pf->state); 391 break; 392 case 2: 393 set_bit(__I40E_CORE_RESET_REQUESTED, pf->state); 394 break; 395 case 3: 396 set_bit(__I40E_GLOBAL_RESET_REQUESTED, pf->state); 397 break; 398 default: 399 netdev_err(netdev, "tx_timeout recovery unsuccessful\n"); 400 break; 401 } 402 403 i40e_service_event_schedule(pf); 404 pf->tx_timeout_recovery_level++; 405 } 406 407 /** 408 * i40e_get_vsi_stats_struct - Get System Network Statistics 409 * @vsi: the VSI we care about 410 * 411 * Returns the address of the device statistics structure. 412 * The statistics are actually updated from the service task. 413 **/ 414 struct rtnl_link_stats64 *i40e_get_vsi_stats_struct(struct i40e_vsi *vsi) 415 { 416 return &vsi->net_stats; 417 } 418 419 /** 420 * i40e_get_netdev_stats_struct_tx - populate stats from a Tx ring 421 * @ring: Tx ring to get statistics from 422 * @stats: statistics entry to be updated 423 **/ 424 static void i40e_get_netdev_stats_struct_tx(struct i40e_ring *ring, 425 struct rtnl_link_stats64 *stats) 426 { 427 u64 bytes, packets; 428 unsigned int start; 429 430 do { 431 start = u64_stats_fetch_begin_irq(&ring->syncp); 432 packets = ring->stats.packets; 433 bytes = ring->stats.bytes; 434 } while (u64_stats_fetch_retry_irq(&ring->syncp, start)); 435 436 stats->tx_packets += packets; 437 stats->tx_bytes += bytes; 438 } 439 440 /** 441 * i40e_get_netdev_stats_struct - Get statistics for netdev interface 442 * @netdev: network interface device structure 443 * 444 * Returns the address of the device statistics structure. 445 * The statistics are actually updated from the service task. 446 **/ 447 static void i40e_get_netdev_stats_struct(struct net_device *netdev, 448 struct rtnl_link_stats64 *stats) 449 { 450 struct i40e_netdev_priv *np = netdev_priv(netdev); 451 struct i40e_ring *tx_ring, *rx_ring; 452 struct i40e_vsi *vsi = np->vsi; 453 struct rtnl_link_stats64 *vsi_stats = i40e_get_vsi_stats_struct(vsi); 454 int i; 455 456 if (test_bit(__I40E_VSI_DOWN, vsi->state)) 457 return; 458 459 if (!vsi->tx_rings) 460 return; 461 462 rcu_read_lock(); 463 for (i = 0; i < vsi->num_queue_pairs; i++) { 464 u64 bytes, packets; 465 unsigned int start; 466 467 tx_ring = READ_ONCE(vsi->tx_rings[i]); 468 if (!tx_ring) 469 continue; 470 i40e_get_netdev_stats_struct_tx(tx_ring, stats); 471 472 rx_ring = &tx_ring[1]; 473 474 do { 475 start = u64_stats_fetch_begin_irq(&rx_ring->syncp); 476 packets = rx_ring->stats.packets; 477 bytes = rx_ring->stats.bytes; 478 } while (u64_stats_fetch_retry_irq(&rx_ring->syncp, start)); 479 480 stats->rx_packets += packets; 481 stats->rx_bytes += bytes; 482 483 if (i40e_enabled_xdp_vsi(vsi)) 484 i40e_get_netdev_stats_struct_tx(&rx_ring[1], stats); 485 } 486 rcu_read_unlock(); 487 488 /* following stats updated by i40e_watchdog_subtask() */ 489 stats->multicast = vsi_stats->multicast; 490 stats->tx_errors = vsi_stats->tx_errors; 491 stats->tx_dropped = vsi_stats->tx_dropped; 492 stats->rx_errors = vsi_stats->rx_errors; 493 stats->rx_dropped = vsi_stats->rx_dropped; 494 stats->rx_crc_errors = vsi_stats->rx_crc_errors; 495 stats->rx_length_errors = vsi_stats->rx_length_errors; 496 } 497 498 /** 499 * i40e_vsi_reset_stats - Resets all stats of the given vsi 500 * @vsi: the VSI to have its stats reset 501 **/ 502 void i40e_vsi_reset_stats(struct i40e_vsi *vsi) 503 { 504 struct rtnl_link_stats64 *ns; 505 int i; 506 507 if (!vsi) 508 return; 509 510 ns = i40e_get_vsi_stats_struct(vsi); 511 memset(ns, 0, sizeof(*ns)); 512 memset(&vsi->net_stats_offsets, 0, sizeof(vsi->net_stats_offsets)); 513 memset(&vsi->eth_stats, 0, sizeof(vsi->eth_stats)); 514 memset(&vsi->eth_stats_offsets, 0, sizeof(vsi->eth_stats_offsets)); 515 if (vsi->rx_rings && vsi->rx_rings[0]) { 516 for (i = 0; i < vsi->num_queue_pairs; i++) { 517 memset(&vsi->rx_rings[i]->stats, 0, 518 sizeof(vsi->rx_rings[i]->stats)); 519 memset(&vsi->rx_rings[i]->rx_stats, 0, 520 sizeof(vsi->rx_rings[i]->rx_stats)); 521 memset(&vsi->tx_rings[i]->stats, 0, 522 sizeof(vsi->tx_rings[i]->stats)); 523 memset(&vsi->tx_rings[i]->tx_stats, 0, 524 sizeof(vsi->tx_rings[i]->tx_stats)); 525 } 526 } 527 vsi->stat_offsets_loaded = false; 528 } 529 530 /** 531 * i40e_pf_reset_stats - Reset all of the stats for the given PF 532 * @pf: the PF to be reset 533 **/ 534 void i40e_pf_reset_stats(struct i40e_pf *pf) 535 { 536 int i; 537 538 memset(&pf->stats, 0, sizeof(pf->stats)); 539 memset(&pf->stats_offsets, 0, sizeof(pf->stats_offsets)); 540 pf->stat_offsets_loaded = false; 541 542 for (i = 0; i < I40E_MAX_VEB; i++) { 543 if (pf->veb[i]) { 544 memset(&pf->veb[i]->stats, 0, 545 sizeof(pf->veb[i]->stats)); 546 memset(&pf->veb[i]->stats_offsets, 0, 547 sizeof(pf->veb[i]->stats_offsets)); 548 pf->veb[i]->stat_offsets_loaded = false; 549 } 550 } 551 pf->hw_csum_rx_error = 0; 552 } 553 554 /** 555 * i40e_stat_update48 - read and update a 48 bit stat from the chip 556 * @hw: ptr to the hardware info 557 * @hireg: the high 32 bit reg to read 558 * @loreg: the low 32 bit reg to read 559 * @offset_loaded: has the initial offset been loaded yet 560 * @offset: ptr to current offset value 561 * @stat: ptr to the stat 562 * 563 * Since the device stats are not reset at PFReset, they likely will not 564 * be zeroed when the driver starts. We'll save the first values read 565 * and use them as offsets to be subtracted from the raw values in order 566 * to report stats that count from zero. In the process, we also manage 567 * the potential roll-over. 568 **/ 569 static void i40e_stat_update48(struct i40e_hw *hw, u32 hireg, u32 loreg, 570 bool offset_loaded, u64 *offset, u64 *stat) 571 { 572 u64 new_data; 573 574 if (hw->device_id == I40E_DEV_ID_QEMU) { 575 new_data = rd32(hw, loreg); 576 new_data |= ((u64)(rd32(hw, hireg) & 0xFFFF)) << 32; 577 } else { 578 new_data = rd64(hw, loreg); 579 } 580 if (!offset_loaded) 581 *offset = new_data; 582 if (likely(new_data >= *offset)) 583 *stat = new_data - *offset; 584 else 585 *stat = (new_data + BIT_ULL(48)) - *offset; 586 *stat &= 0xFFFFFFFFFFFFULL; 587 } 588 589 /** 590 * i40e_stat_update32 - read and update a 32 bit stat from the chip 591 * @hw: ptr to the hardware info 592 * @reg: the hw reg to read 593 * @offset_loaded: has the initial offset been loaded yet 594 * @offset: ptr to current offset value 595 * @stat: ptr to the stat 596 **/ 597 static void i40e_stat_update32(struct i40e_hw *hw, u32 reg, 598 bool offset_loaded, u64 *offset, u64 *stat) 599 { 600 u32 new_data; 601 602 new_data = rd32(hw, reg); 603 if (!offset_loaded) 604 *offset = new_data; 605 if (likely(new_data >= *offset)) 606 *stat = (u32)(new_data - *offset); 607 else 608 *stat = (u32)((new_data + BIT_ULL(32)) - *offset); 609 } 610 611 /** 612 * i40e_stat_update_and_clear32 - read and clear hw reg, update a 32 bit stat 613 * @hw: ptr to the hardware info 614 * @reg: the hw reg to read and clear 615 * @stat: ptr to the stat 616 **/ 617 static void i40e_stat_update_and_clear32(struct i40e_hw *hw, u32 reg, u64 *stat) 618 { 619 u32 new_data = rd32(hw, reg); 620 621 wr32(hw, reg, 1); /* must write a nonzero value to clear register */ 622 *stat += new_data; 623 } 624 625 /** 626 * i40e_update_eth_stats - Update VSI-specific ethernet statistics counters. 627 * @vsi: the VSI to be updated 628 **/ 629 void i40e_update_eth_stats(struct i40e_vsi *vsi) 630 { 631 int stat_idx = le16_to_cpu(vsi->info.stat_counter_idx); 632 struct i40e_pf *pf = vsi->back; 633 struct i40e_hw *hw = &pf->hw; 634 struct i40e_eth_stats *oes; 635 struct i40e_eth_stats *es; /* device's eth stats */ 636 637 es = &vsi->eth_stats; 638 oes = &vsi->eth_stats_offsets; 639 640 /* Gather up the stats that the hw collects */ 641 i40e_stat_update32(hw, I40E_GLV_TEPC(stat_idx), 642 vsi->stat_offsets_loaded, 643 &oes->tx_errors, &es->tx_errors); 644 i40e_stat_update32(hw, I40E_GLV_RDPC(stat_idx), 645 vsi->stat_offsets_loaded, 646 &oes->rx_discards, &es->rx_discards); 647 i40e_stat_update32(hw, I40E_GLV_RUPP(stat_idx), 648 vsi->stat_offsets_loaded, 649 &oes->rx_unknown_protocol, &es->rx_unknown_protocol); 650 i40e_stat_update32(hw, I40E_GLV_TEPC(stat_idx), 651 vsi->stat_offsets_loaded, 652 &oes->tx_errors, &es->tx_errors); 653 654 i40e_stat_update48(hw, I40E_GLV_GORCH(stat_idx), 655 I40E_GLV_GORCL(stat_idx), 656 vsi->stat_offsets_loaded, 657 &oes->rx_bytes, &es->rx_bytes); 658 i40e_stat_update48(hw, I40E_GLV_UPRCH(stat_idx), 659 I40E_GLV_UPRCL(stat_idx), 660 vsi->stat_offsets_loaded, 661 &oes->rx_unicast, &es->rx_unicast); 662 i40e_stat_update48(hw, I40E_GLV_MPRCH(stat_idx), 663 I40E_GLV_MPRCL(stat_idx), 664 vsi->stat_offsets_loaded, 665 &oes->rx_multicast, &es->rx_multicast); 666 i40e_stat_update48(hw, I40E_GLV_BPRCH(stat_idx), 667 I40E_GLV_BPRCL(stat_idx), 668 vsi->stat_offsets_loaded, 669 &oes->rx_broadcast, &es->rx_broadcast); 670 671 i40e_stat_update48(hw, I40E_GLV_GOTCH(stat_idx), 672 I40E_GLV_GOTCL(stat_idx), 673 vsi->stat_offsets_loaded, 674 &oes->tx_bytes, &es->tx_bytes); 675 i40e_stat_update48(hw, I40E_GLV_UPTCH(stat_idx), 676 I40E_GLV_UPTCL(stat_idx), 677 vsi->stat_offsets_loaded, 678 &oes->tx_unicast, &es->tx_unicast); 679 i40e_stat_update48(hw, I40E_GLV_MPTCH(stat_idx), 680 I40E_GLV_MPTCL(stat_idx), 681 vsi->stat_offsets_loaded, 682 &oes->tx_multicast, &es->tx_multicast); 683 i40e_stat_update48(hw, I40E_GLV_BPTCH(stat_idx), 684 I40E_GLV_BPTCL(stat_idx), 685 vsi->stat_offsets_loaded, 686 &oes->tx_broadcast, &es->tx_broadcast); 687 vsi->stat_offsets_loaded = true; 688 } 689 690 /** 691 * i40e_update_veb_stats - Update Switch component statistics 692 * @veb: the VEB being updated 693 **/ 694 static void i40e_update_veb_stats(struct i40e_veb *veb) 695 { 696 struct i40e_pf *pf = veb->pf; 697 struct i40e_hw *hw = &pf->hw; 698 struct i40e_eth_stats *oes; 699 struct i40e_eth_stats *es; /* device's eth stats */ 700 struct i40e_veb_tc_stats *veb_oes; 701 struct i40e_veb_tc_stats *veb_es; 702 int i, idx = 0; 703 704 idx = veb->stats_idx; 705 es = &veb->stats; 706 oes = &veb->stats_offsets; 707 veb_es = &veb->tc_stats; 708 veb_oes = &veb->tc_stats_offsets; 709 710 /* Gather up the stats that the hw collects */ 711 i40e_stat_update32(hw, I40E_GLSW_TDPC(idx), 712 veb->stat_offsets_loaded, 713 &oes->tx_discards, &es->tx_discards); 714 if (hw->revision_id > 0) 715 i40e_stat_update32(hw, I40E_GLSW_RUPP(idx), 716 veb->stat_offsets_loaded, 717 &oes->rx_unknown_protocol, 718 &es->rx_unknown_protocol); 719 i40e_stat_update48(hw, I40E_GLSW_GORCH(idx), I40E_GLSW_GORCL(idx), 720 veb->stat_offsets_loaded, 721 &oes->rx_bytes, &es->rx_bytes); 722 i40e_stat_update48(hw, I40E_GLSW_UPRCH(idx), I40E_GLSW_UPRCL(idx), 723 veb->stat_offsets_loaded, 724 &oes->rx_unicast, &es->rx_unicast); 725 i40e_stat_update48(hw, I40E_GLSW_MPRCH(idx), I40E_GLSW_MPRCL(idx), 726 veb->stat_offsets_loaded, 727 &oes->rx_multicast, &es->rx_multicast); 728 i40e_stat_update48(hw, I40E_GLSW_BPRCH(idx), I40E_GLSW_BPRCL(idx), 729 veb->stat_offsets_loaded, 730 &oes->rx_broadcast, &es->rx_broadcast); 731 732 i40e_stat_update48(hw, I40E_GLSW_GOTCH(idx), I40E_GLSW_GOTCL(idx), 733 veb->stat_offsets_loaded, 734 &oes->tx_bytes, &es->tx_bytes); 735 i40e_stat_update48(hw, I40E_GLSW_UPTCH(idx), I40E_GLSW_UPTCL(idx), 736 veb->stat_offsets_loaded, 737 &oes->tx_unicast, &es->tx_unicast); 738 i40e_stat_update48(hw, I40E_GLSW_MPTCH(idx), I40E_GLSW_MPTCL(idx), 739 veb->stat_offsets_loaded, 740 &oes->tx_multicast, &es->tx_multicast); 741 i40e_stat_update48(hw, I40E_GLSW_BPTCH(idx), I40E_GLSW_BPTCL(idx), 742 veb->stat_offsets_loaded, 743 &oes->tx_broadcast, &es->tx_broadcast); 744 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 745 i40e_stat_update48(hw, I40E_GLVEBTC_RPCH(i, idx), 746 I40E_GLVEBTC_RPCL(i, idx), 747 veb->stat_offsets_loaded, 748 &veb_oes->tc_rx_packets[i], 749 &veb_es->tc_rx_packets[i]); 750 i40e_stat_update48(hw, I40E_GLVEBTC_RBCH(i, idx), 751 I40E_GLVEBTC_RBCL(i, idx), 752 veb->stat_offsets_loaded, 753 &veb_oes->tc_rx_bytes[i], 754 &veb_es->tc_rx_bytes[i]); 755 i40e_stat_update48(hw, I40E_GLVEBTC_TPCH(i, idx), 756 I40E_GLVEBTC_TPCL(i, idx), 757 veb->stat_offsets_loaded, 758 &veb_oes->tc_tx_packets[i], 759 &veb_es->tc_tx_packets[i]); 760 i40e_stat_update48(hw, I40E_GLVEBTC_TBCH(i, idx), 761 I40E_GLVEBTC_TBCL(i, idx), 762 veb->stat_offsets_loaded, 763 &veb_oes->tc_tx_bytes[i], 764 &veb_es->tc_tx_bytes[i]); 765 } 766 veb->stat_offsets_loaded = true; 767 } 768 769 /** 770 * i40e_update_vsi_stats - Update the vsi statistics counters. 771 * @vsi: the VSI to be updated 772 * 773 * There are a few instances where we store the same stat in a 774 * couple of different structs. This is partly because we have 775 * the netdev stats that need to be filled out, which is slightly 776 * different from the "eth_stats" defined by the chip and used in 777 * VF communications. We sort it out here. 778 **/ 779 static void i40e_update_vsi_stats(struct i40e_vsi *vsi) 780 { 781 struct i40e_pf *pf = vsi->back; 782 struct rtnl_link_stats64 *ons; 783 struct rtnl_link_stats64 *ns; /* netdev stats */ 784 struct i40e_eth_stats *oes; 785 struct i40e_eth_stats *es; /* device's eth stats */ 786 u32 tx_restart, tx_busy; 787 struct i40e_ring *p; 788 u32 rx_page, rx_buf; 789 u64 bytes, packets; 790 unsigned int start; 791 u64 tx_linearize; 792 u64 tx_force_wb; 793 u64 rx_p, rx_b; 794 u64 tx_p, tx_b; 795 u16 q; 796 797 if (test_bit(__I40E_VSI_DOWN, vsi->state) || 798 test_bit(__I40E_CONFIG_BUSY, pf->state)) 799 return; 800 801 ns = i40e_get_vsi_stats_struct(vsi); 802 ons = &vsi->net_stats_offsets; 803 es = &vsi->eth_stats; 804 oes = &vsi->eth_stats_offsets; 805 806 /* Gather up the netdev and vsi stats that the driver collects 807 * on the fly during packet processing 808 */ 809 rx_b = rx_p = 0; 810 tx_b = tx_p = 0; 811 tx_restart = tx_busy = tx_linearize = tx_force_wb = 0; 812 rx_page = 0; 813 rx_buf = 0; 814 rcu_read_lock(); 815 for (q = 0; q < vsi->num_queue_pairs; q++) { 816 /* locate Tx ring */ 817 p = READ_ONCE(vsi->tx_rings[q]); 818 819 do { 820 start = u64_stats_fetch_begin_irq(&p->syncp); 821 packets = p->stats.packets; 822 bytes = p->stats.bytes; 823 } while (u64_stats_fetch_retry_irq(&p->syncp, start)); 824 tx_b += bytes; 825 tx_p += packets; 826 tx_restart += p->tx_stats.restart_queue; 827 tx_busy += p->tx_stats.tx_busy; 828 tx_linearize += p->tx_stats.tx_linearize; 829 tx_force_wb += p->tx_stats.tx_force_wb; 830 831 /* Rx queue is part of the same block as Tx queue */ 832 p = &p[1]; 833 do { 834 start = u64_stats_fetch_begin_irq(&p->syncp); 835 packets = p->stats.packets; 836 bytes = p->stats.bytes; 837 } while (u64_stats_fetch_retry_irq(&p->syncp, start)); 838 rx_b += bytes; 839 rx_p += packets; 840 rx_buf += p->rx_stats.alloc_buff_failed; 841 rx_page += p->rx_stats.alloc_page_failed; 842 } 843 rcu_read_unlock(); 844 vsi->tx_restart = tx_restart; 845 vsi->tx_busy = tx_busy; 846 vsi->tx_linearize = tx_linearize; 847 vsi->tx_force_wb = tx_force_wb; 848 vsi->rx_page_failed = rx_page; 849 vsi->rx_buf_failed = rx_buf; 850 851 ns->rx_packets = rx_p; 852 ns->rx_bytes = rx_b; 853 ns->tx_packets = tx_p; 854 ns->tx_bytes = tx_b; 855 856 /* update netdev stats from eth stats */ 857 i40e_update_eth_stats(vsi); 858 ons->tx_errors = oes->tx_errors; 859 ns->tx_errors = es->tx_errors; 860 ons->multicast = oes->rx_multicast; 861 ns->multicast = es->rx_multicast; 862 ons->rx_dropped = oes->rx_discards; 863 ns->rx_dropped = es->rx_discards; 864 ons->tx_dropped = oes->tx_discards; 865 ns->tx_dropped = es->tx_discards; 866 867 /* pull in a couple PF stats if this is the main vsi */ 868 if (vsi == pf->vsi[pf->lan_vsi]) { 869 ns->rx_crc_errors = pf->stats.crc_errors; 870 ns->rx_errors = pf->stats.crc_errors + pf->stats.illegal_bytes; 871 ns->rx_length_errors = pf->stats.rx_length_errors; 872 } 873 } 874 875 /** 876 * i40e_update_pf_stats - Update the PF statistics counters. 877 * @pf: the PF to be updated 878 **/ 879 static void i40e_update_pf_stats(struct i40e_pf *pf) 880 { 881 struct i40e_hw_port_stats *osd = &pf->stats_offsets; 882 struct i40e_hw_port_stats *nsd = &pf->stats; 883 struct i40e_hw *hw = &pf->hw; 884 u32 val; 885 int i; 886 887 i40e_stat_update48(hw, I40E_GLPRT_GORCH(hw->port), 888 I40E_GLPRT_GORCL(hw->port), 889 pf->stat_offsets_loaded, 890 &osd->eth.rx_bytes, &nsd->eth.rx_bytes); 891 i40e_stat_update48(hw, I40E_GLPRT_GOTCH(hw->port), 892 I40E_GLPRT_GOTCL(hw->port), 893 pf->stat_offsets_loaded, 894 &osd->eth.tx_bytes, &nsd->eth.tx_bytes); 895 i40e_stat_update32(hw, I40E_GLPRT_RDPC(hw->port), 896 pf->stat_offsets_loaded, 897 &osd->eth.rx_discards, 898 &nsd->eth.rx_discards); 899 i40e_stat_update48(hw, I40E_GLPRT_UPRCH(hw->port), 900 I40E_GLPRT_UPRCL(hw->port), 901 pf->stat_offsets_loaded, 902 &osd->eth.rx_unicast, 903 &nsd->eth.rx_unicast); 904 i40e_stat_update48(hw, I40E_GLPRT_MPRCH(hw->port), 905 I40E_GLPRT_MPRCL(hw->port), 906 pf->stat_offsets_loaded, 907 &osd->eth.rx_multicast, 908 &nsd->eth.rx_multicast); 909 i40e_stat_update48(hw, I40E_GLPRT_BPRCH(hw->port), 910 I40E_GLPRT_BPRCL(hw->port), 911 pf->stat_offsets_loaded, 912 &osd->eth.rx_broadcast, 913 &nsd->eth.rx_broadcast); 914 i40e_stat_update48(hw, I40E_GLPRT_UPTCH(hw->port), 915 I40E_GLPRT_UPTCL(hw->port), 916 pf->stat_offsets_loaded, 917 &osd->eth.tx_unicast, 918 &nsd->eth.tx_unicast); 919 i40e_stat_update48(hw, I40E_GLPRT_MPTCH(hw->port), 920 I40E_GLPRT_MPTCL(hw->port), 921 pf->stat_offsets_loaded, 922 &osd->eth.tx_multicast, 923 &nsd->eth.tx_multicast); 924 i40e_stat_update48(hw, I40E_GLPRT_BPTCH(hw->port), 925 I40E_GLPRT_BPTCL(hw->port), 926 pf->stat_offsets_loaded, 927 &osd->eth.tx_broadcast, 928 &nsd->eth.tx_broadcast); 929 930 i40e_stat_update32(hw, I40E_GLPRT_TDOLD(hw->port), 931 pf->stat_offsets_loaded, 932 &osd->tx_dropped_link_down, 933 &nsd->tx_dropped_link_down); 934 935 i40e_stat_update32(hw, I40E_GLPRT_CRCERRS(hw->port), 936 pf->stat_offsets_loaded, 937 &osd->crc_errors, &nsd->crc_errors); 938 939 i40e_stat_update32(hw, I40E_GLPRT_ILLERRC(hw->port), 940 pf->stat_offsets_loaded, 941 &osd->illegal_bytes, &nsd->illegal_bytes); 942 943 i40e_stat_update32(hw, I40E_GLPRT_MLFC(hw->port), 944 pf->stat_offsets_loaded, 945 &osd->mac_local_faults, 946 &nsd->mac_local_faults); 947 i40e_stat_update32(hw, I40E_GLPRT_MRFC(hw->port), 948 pf->stat_offsets_loaded, 949 &osd->mac_remote_faults, 950 &nsd->mac_remote_faults); 951 952 i40e_stat_update32(hw, I40E_GLPRT_RLEC(hw->port), 953 pf->stat_offsets_loaded, 954 &osd->rx_length_errors, 955 &nsd->rx_length_errors); 956 957 i40e_stat_update32(hw, I40E_GLPRT_LXONRXC(hw->port), 958 pf->stat_offsets_loaded, 959 &osd->link_xon_rx, &nsd->link_xon_rx); 960 i40e_stat_update32(hw, I40E_GLPRT_LXONTXC(hw->port), 961 pf->stat_offsets_loaded, 962 &osd->link_xon_tx, &nsd->link_xon_tx); 963 i40e_stat_update32(hw, I40E_GLPRT_LXOFFRXC(hw->port), 964 pf->stat_offsets_loaded, 965 &osd->link_xoff_rx, &nsd->link_xoff_rx); 966 i40e_stat_update32(hw, I40E_GLPRT_LXOFFTXC(hw->port), 967 pf->stat_offsets_loaded, 968 &osd->link_xoff_tx, &nsd->link_xoff_tx); 969 970 for (i = 0; i < 8; i++) { 971 i40e_stat_update32(hw, I40E_GLPRT_PXOFFRXC(hw->port, i), 972 pf->stat_offsets_loaded, 973 &osd->priority_xoff_rx[i], 974 &nsd->priority_xoff_rx[i]); 975 i40e_stat_update32(hw, I40E_GLPRT_PXONRXC(hw->port, i), 976 pf->stat_offsets_loaded, 977 &osd->priority_xon_rx[i], 978 &nsd->priority_xon_rx[i]); 979 i40e_stat_update32(hw, I40E_GLPRT_PXONTXC(hw->port, i), 980 pf->stat_offsets_loaded, 981 &osd->priority_xon_tx[i], 982 &nsd->priority_xon_tx[i]); 983 i40e_stat_update32(hw, I40E_GLPRT_PXOFFTXC(hw->port, i), 984 pf->stat_offsets_loaded, 985 &osd->priority_xoff_tx[i], 986 &nsd->priority_xoff_tx[i]); 987 i40e_stat_update32(hw, 988 I40E_GLPRT_RXON2OFFCNT(hw->port, i), 989 pf->stat_offsets_loaded, 990 &osd->priority_xon_2_xoff[i], 991 &nsd->priority_xon_2_xoff[i]); 992 } 993 994 i40e_stat_update48(hw, I40E_GLPRT_PRC64H(hw->port), 995 I40E_GLPRT_PRC64L(hw->port), 996 pf->stat_offsets_loaded, 997 &osd->rx_size_64, &nsd->rx_size_64); 998 i40e_stat_update48(hw, I40E_GLPRT_PRC127H(hw->port), 999 I40E_GLPRT_PRC127L(hw->port), 1000 pf->stat_offsets_loaded, 1001 &osd->rx_size_127, &nsd->rx_size_127); 1002 i40e_stat_update48(hw, I40E_GLPRT_PRC255H(hw->port), 1003 I40E_GLPRT_PRC255L(hw->port), 1004 pf->stat_offsets_loaded, 1005 &osd->rx_size_255, &nsd->rx_size_255); 1006 i40e_stat_update48(hw, I40E_GLPRT_PRC511H(hw->port), 1007 I40E_GLPRT_PRC511L(hw->port), 1008 pf->stat_offsets_loaded, 1009 &osd->rx_size_511, &nsd->rx_size_511); 1010 i40e_stat_update48(hw, I40E_GLPRT_PRC1023H(hw->port), 1011 I40E_GLPRT_PRC1023L(hw->port), 1012 pf->stat_offsets_loaded, 1013 &osd->rx_size_1023, &nsd->rx_size_1023); 1014 i40e_stat_update48(hw, I40E_GLPRT_PRC1522H(hw->port), 1015 I40E_GLPRT_PRC1522L(hw->port), 1016 pf->stat_offsets_loaded, 1017 &osd->rx_size_1522, &nsd->rx_size_1522); 1018 i40e_stat_update48(hw, I40E_GLPRT_PRC9522H(hw->port), 1019 I40E_GLPRT_PRC9522L(hw->port), 1020 pf->stat_offsets_loaded, 1021 &osd->rx_size_big, &nsd->rx_size_big); 1022 1023 i40e_stat_update48(hw, I40E_GLPRT_PTC64H(hw->port), 1024 I40E_GLPRT_PTC64L(hw->port), 1025 pf->stat_offsets_loaded, 1026 &osd->tx_size_64, &nsd->tx_size_64); 1027 i40e_stat_update48(hw, I40E_GLPRT_PTC127H(hw->port), 1028 I40E_GLPRT_PTC127L(hw->port), 1029 pf->stat_offsets_loaded, 1030 &osd->tx_size_127, &nsd->tx_size_127); 1031 i40e_stat_update48(hw, I40E_GLPRT_PTC255H(hw->port), 1032 I40E_GLPRT_PTC255L(hw->port), 1033 pf->stat_offsets_loaded, 1034 &osd->tx_size_255, &nsd->tx_size_255); 1035 i40e_stat_update48(hw, I40E_GLPRT_PTC511H(hw->port), 1036 I40E_GLPRT_PTC511L(hw->port), 1037 pf->stat_offsets_loaded, 1038 &osd->tx_size_511, &nsd->tx_size_511); 1039 i40e_stat_update48(hw, I40E_GLPRT_PTC1023H(hw->port), 1040 I40E_GLPRT_PTC1023L(hw->port), 1041 pf->stat_offsets_loaded, 1042 &osd->tx_size_1023, &nsd->tx_size_1023); 1043 i40e_stat_update48(hw, I40E_GLPRT_PTC1522H(hw->port), 1044 I40E_GLPRT_PTC1522L(hw->port), 1045 pf->stat_offsets_loaded, 1046 &osd->tx_size_1522, &nsd->tx_size_1522); 1047 i40e_stat_update48(hw, I40E_GLPRT_PTC9522H(hw->port), 1048 I40E_GLPRT_PTC9522L(hw->port), 1049 pf->stat_offsets_loaded, 1050 &osd->tx_size_big, &nsd->tx_size_big); 1051 1052 i40e_stat_update32(hw, I40E_GLPRT_RUC(hw->port), 1053 pf->stat_offsets_loaded, 1054 &osd->rx_undersize, &nsd->rx_undersize); 1055 i40e_stat_update32(hw, I40E_GLPRT_RFC(hw->port), 1056 pf->stat_offsets_loaded, 1057 &osd->rx_fragments, &nsd->rx_fragments); 1058 i40e_stat_update32(hw, I40E_GLPRT_ROC(hw->port), 1059 pf->stat_offsets_loaded, 1060 &osd->rx_oversize, &nsd->rx_oversize); 1061 i40e_stat_update32(hw, I40E_GLPRT_RJC(hw->port), 1062 pf->stat_offsets_loaded, 1063 &osd->rx_jabber, &nsd->rx_jabber); 1064 1065 /* FDIR stats */ 1066 i40e_stat_update_and_clear32(hw, 1067 I40E_GLQF_PCNT(I40E_FD_ATR_STAT_IDX(hw->pf_id)), 1068 &nsd->fd_atr_match); 1069 i40e_stat_update_and_clear32(hw, 1070 I40E_GLQF_PCNT(I40E_FD_SB_STAT_IDX(hw->pf_id)), 1071 &nsd->fd_sb_match); 1072 i40e_stat_update_and_clear32(hw, 1073 I40E_GLQF_PCNT(I40E_FD_ATR_TUNNEL_STAT_IDX(hw->pf_id)), 1074 &nsd->fd_atr_tunnel_match); 1075 1076 val = rd32(hw, I40E_PRTPM_EEE_STAT); 1077 nsd->tx_lpi_status = 1078 (val & I40E_PRTPM_EEE_STAT_TX_LPI_STATUS_MASK) >> 1079 I40E_PRTPM_EEE_STAT_TX_LPI_STATUS_SHIFT; 1080 nsd->rx_lpi_status = 1081 (val & I40E_PRTPM_EEE_STAT_RX_LPI_STATUS_MASK) >> 1082 I40E_PRTPM_EEE_STAT_RX_LPI_STATUS_SHIFT; 1083 i40e_stat_update32(hw, I40E_PRTPM_TLPIC, 1084 pf->stat_offsets_loaded, 1085 &osd->tx_lpi_count, &nsd->tx_lpi_count); 1086 i40e_stat_update32(hw, I40E_PRTPM_RLPIC, 1087 pf->stat_offsets_loaded, 1088 &osd->rx_lpi_count, &nsd->rx_lpi_count); 1089 1090 if (pf->flags & I40E_FLAG_FD_SB_ENABLED && 1091 !(pf->flags & I40E_FLAG_FD_SB_AUTO_DISABLED)) 1092 nsd->fd_sb_status = true; 1093 else 1094 nsd->fd_sb_status = false; 1095 1096 if (pf->flags & I40E_FLAG_FD_ATR_ENABLED && 1097 !(pf->flags & I40E_FLAG_FD_ATR_AUTO_DISABLED)) 1098 nsd->fd_atr_status = true; 1099 else 1100 nsd->fd_atr_status = false; 1101 1102 pf->stat_offsets_loaded = true; 1103 } 1104 1105 /** 1106 * i40e_update_stats - Update the various statistics counters. 1107 * @vsi: the VSI to be updated 1108 * 1109 * Update the various stats for this VSI and its related entities. 1110 **/ 1111 void i40e_update_stats(struct i40e_vsi *vsi) 1112 { 1113 struct i40e_pf *pf = vsi->back; 1114 1115 if (vsi == pf->vsi[pf->lan_vsi]) 1116 i40e_update_pf_stats(pf); 1117 1118 i40e_update_vsi_stats(vsi); 1119 } 1120 1121 /** 1122 * i40e_find_filter - Search VSI filter list for specific mac/vlan filter 1123 * @vsi: the VSI to be searched 1124 * @macaddr: the MAC address 1125 * @vlan: the vlan 1126 * 1127 * Returns ptr to the filter object or NULL 1128 **/ 1129 static struct i40e_mac_filter *i40e_find_filter(struct i40e_vsi *vsi, 1130 const u8 *macaddr, s16 vlan) 1131 { 1132 struct i40e_mac_filter *f; 1133 u64 key; 1134 1135 if (!vsi || !macaddr) 1136 return NULL; 1137 1138 key = i40e_addr_to_hkey(macaddr); 1139 hash_for_each_possible(vsi->mac_filter_hash, f, hlist, key) { 1140 if ((ether_addr_equal(macaddr, f->macaddr)) && 1141 (vlan == f->vlan)) 1142 return f; 1143 } 1144 return NULL; 1145 } 1146 1147 /** 1148 * i40e_find_mac - Find a mac addr in the macvlan filters list 1149 * @vsi: the VSI to be searched 1150 * @macaddr: the MAC address we are searching for 1151 * 1152 * Returns the first filter with the provided MAC address or NULL if 1153 * MAC address was not found 1154 **/ 1155 struct i40e_mac_filter *i40e_find_mac(struct i40e_vsi *vsi, const u8 *macaddr) 1156 { 1157 struct i40e_mac_filter *f; 1158 u64 key; 1159 1160 if (!vsi || !macaddr) 1161 return NULL; 1162 1163 key = i40e_addr_to_hkey(macaddr); 1164 hash_for_each_possible(vsi->mac_filter_hash, f, hlist, key) { 1165 if ((ether_addr_equal(macaddr, f->macaddr))) 1166 return f; 1167 } 1168 return NULL; 1169 } 1170 1171 /** 1172 * i40e_is_vsi_in_vlan - Check if VSI is in vlan mode 1173 * @vsi: the VSI to be searched 1174 * 1175 * Returns true if VSI is in vlan mode or false otherwise 1176 **/ 1177 bool i40e_is_vsi_in_vlan(struct i40e_vsi *vsi) 1178 { 1179 /* If we have a PVID, always operate in VLAN mode */ 1180 if (vsi->info.pvid) 1181 return true; 1182 1183 /* We need to operate in VLAN mode whenever we have any filters with 1184 * a VLAN other than I40E_VLAN_ALL. We could check the table each 1185 * time, incurring search cost repeatedly. However, we can notice two 1186 * things: 1187 * 1188 * 1) the only place where we can gain a VLAN filter is in 1189 * i40e_add_filter. 1190 * 1191 * 2) the only place where filters are actually removed is in 1192 * i40e_sync_filters_subtask. 1193 * 1194 * Thus, we can simply use a boolean value, has_vlan_filters which we 1195 * will set to true when we add a VLAN filter in i40e_add_filter. Then 1196 * we have to perform the full search after deleting filters in 1197 * i40e_sync_filters_subtask, but we already have to search 1198 * filters here and can perform the check at the same time. This 1199 * results in avoiding embedding a loop for VLAN mode inside another 1200 * loop over all the filters, and should maintain correctness as noted 1201 * above. 1202 */ 1203 return vsi->has_vlan_filter; 1204 } 1205 1206 /** 1207 * i40e_correct_mac_vlan_filters - Correct non-VLAN filters if necessary 1208 * @vsi: the VSI to configure 1209 * @tmp_add_list: list of filters ready to be added 1210 * @tmp_del_list: list of filters ready to be deleted 1211 * @vlan_filters: the number of active VLAN filters 1212 * 1213 * Update VLAN=0 and VLAN=-1 (I40E_VLAN_ANY) filters properly so that they 1214 * behave as expected. If we have any active VLAN filters remaining or about 1215 * to be added then we need to update non-VLAN filters to be marked as VLAN=0 1216 * so that they only match against untagged traffic. If we no longer have any 1217 * active VLAN filters, we need to make all non-VLAN filters marked as VLAN=-1 1218 * so that they match against both tagged and untagged traffic. In this way, 1219 * we ensure that we correctly receive the desired traffic. This ensures that 1220 * when we have an active VLAN we will receive only untagged traffic and 1221 * traffic matching active VLANs. If we have no active VLANs then we will 1222 * operate in non-VLAN mode and receive all traffic, tagged or untagged. 1223 * 1224 * Finally, in a similar fashion, this function also corrects filters when 1225 * there is an active PVID assigned to this VSI. 1226 * 1227 * In case of memory allocation failure return -ENOMEM. Otherwise, return 0. 1228 * 1229 * This function is only expected to be called from within 1230 * i40e_sync_vsi_filters. 1231 * 1232 * NOTE: This function expects to be called while under the 1233 * mac_filter_hash_lock 1234 */ 1235 static int i40e_correct_mac_vlan_filters(struct i40e_vsi *vsi, 1236 struct hlist_head *tmp_add_list, 1237 struct hlist_head *tmp_del_list, 1238 int vlan_filters) 1239 { 1240 s16 pvid = le16_to_cpu(vsi->info.pvid); 1241 struct i40e_mac_filter *f, *add_head; 1242 struct i40e_new_mac_filter *new; 1243 struct hlist_node *h; 1244 int bkt, new_vlan; 1245 1246 /* To determine if a particular filter needs to be replaced we 1247 * have the three following conditions: 1248 * 1249 * a) if we have a PVID assigned, then all filters which are 1250 * not marked as VLAN=PVID must be replaced with filters that 1251 * are. 1252 * b) otherwise, if we have any active VLANS, all filters 1253 * which are marked as VLAN=-1 must be replaced with 1254 * filters marked as VLAN=0 1255 * c) finally, if we do not have any active VLANS, all filters 1256 * which are marked as VLAN=0 must be replaced with filters 1257 * marked as VLAN=-1 1258 */ 1259 1260 /* Update the filters about to be added in place */ 1261 hlist_for_each_entry(new, tmp_add_list, hlist) { 1262 if (pvid && new->f->vlan != pvid) 1263 new->f->vlan = pvid; 1264 else if (vlan_filters && new->f->vlan == I40E_VLAN_ANY) 1265 new->f->vlan = 0; 1266 else if (!vlan_filters && new->f->vlan == 0) 1267 new->f->vlan = I40E_VLAN_ANY; 1268 } 1269 1270 /* Update the remaining active filters */ 1271 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) { 1272 /* Combine the checks for whether a filter needs to be changed 1273 * and then determine the new VLAN inside the if block, in 1274 * order to avoid duplicating code for adding the new filter 1275 * then deleting the old filter. 1276 */ 1277 if ((pvid && f->vlan != pvid) || 1278 (vlan_filters && f->vlan == I40E_VLAN_ANY) || 1279 (!vlan_filters && f->vlan == 0)) { 1280 /* Determine the new vlan we will be adding */ 1281 if (pvid) 1282 new_vlan = pvid; 1283 else if (vlan_filters) 1284 new_vlan = 0; 1285 else 1286 new_vlan = I40E_VLAN_ANY; 1287 1288 /* Create the new filter */ 1289 add_head = i40e_add_filter(vsi, f->macaddr, new_vlan); 1290 if (!add_head) 1291 return -ENOMEM; 1292 1293 /* Create a temporary i40e_new_mac_filter */ 1294 new = kzalloc(sizeof(*new), GFP_ATOMIC); 1295 if (!new) 1296 return -ENOMEM; 1297 1298 new->f = add_head; 1299 new->state = add_head->state; 1300 1301 /* Add the new filter to the tmp list */ 1302 hlist_add_head(&new->hlist, tmp_add_list); 1303 1304 /* Put the original filter into the delete list */ 1305 f->state = I40E_FILTER_REMOVE; 1306 hash_del(&f->hlist); 1307 hlist_add_head(&f->hlist, tmp_del_list); 1308 } 1309 } 1310 1311 vsi->has_vlan_filter = !!vlan_filters; 1312 1313 return 0; 1314 } 1315 1316 /** 1317 * i40e_rm_default_mac_filter - Remove the default MAC filter set by NVM 1318 * @vsi: the PF Main VSI - inappropriate for any other VSI 1319 * @macaddr: the MAC address 1320 * 1321 * Remove whatever filter the firmware set up so the driver can manage 1322 * its own filtering intelligently. 1323 **/ 1324 static void i40e_rm_default_mac_filter(struct i40e_vsi *vsi, u8 *macaddr) 1325 { 1326 struct i40e_aqc_remove_macvlan_element_data element; 1327 struct i40e_pf *pf = vsi->back; 1328 1329 /* Only appropriate for the PF main VSI */ 1330 if (vsi->type != I40E_VSI_MAIN) 1331 return; 1332 1333 memset(&element, 0, sizeof(element)); 1334 ether_addr_copy(element.mac_addr, macaddr); 1335 element.vlan_tag = 0; 1336 /* Ignore error returns, some firmware does it this way... */ 1337 element.flags = I40E_AQC_MACVLAN_DEL_PERFECT_MATCH; 1338 i40e_aq_remove_macvlan(&pf->hw, vsi->seid, &element, 1, NULL); 1339 1340 memset(&element, 0, sizeof(element)); 1341 ether_addr_copy(element.mac_addr, macaddr); 1342 element.vlan_tag = 0; 1343 /* ...and some firmware does it this way. */ 1344 element.flags = I40E_AQC_MACVLAN_DEL_PERFECT_MATCH | 1345 I40E_AQC_MACVLAN_DEL_IGNORE_VLAN; 1346 i40e_aq_remove_macvlan(&pf->hw, vsi->seid, &element, 1, NULL); 1347 } 1348 1349 /** 1350 * i40e_add_filter - Add a mac/vlan filter to the VSI 1351 * @vsi: the VSI to be searched 1352 * @macaddr: the MAC address 1353 * @vlan: the vlan 1354 * 1355 * Returns ptr to the filter object or NULL when no memory available. 1356 * 1357 * NOTE: This function is expected to be called with mac_filter_hash_lock 1358 * being held. 1359 **/ 1360 struct i40e_mac_filter *i40e_add_filter(struct i40e_vsi *vsi, 1361 const u8 *macaddr, s16 vlan) 1362 { 1363 struct i40e_mac_filter *f; 1364 u64 key; 1365 1366 if (!vsi || !macaddr) 1367 return NULL; 1368 1369 f = i40e_find_filter(vsi, macaddr, vlan); 1370 if (!f) { 1371 f = kzalloc(sizeof(*f), GFP_ATOMIC); 1372 if (!f) 1373 return NULL; 1374 1375 /* Update the boolean indicating if we need to function in 1376 * VLAN mode. 1377 */ 1378 if (vlan >= 0) 1379 vsi->has_vlan_filter = true; 1380 1381 ether_addr_copy(f->macaddr, macaddr); 1382 f->vlan = vlan; 1383 /* If we're in overflow promisc mode, set the state directly 1384 * to failed, so we don't bother to try sending the filter 1385 * to the hardware. 1386 */ 1387 if (test_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state)) 1388 f->state = I40E_FILTER_FAILED; 1389 else 1390 f->state = I40E_FILTER_NEW; 1391 INIT_HLIST_NODE(&f->hlist); 1392 1393 key = i40e_addr_to_hkey(macaddr); 1394 hash_add(vsi->mac_filter_hash, &f->hlist, key); 1395 1396 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED; 1397 vsi->back->flags |= I40E_FLAG_FILTER_SYNC; 1398 } 1399 1400 /* If we're asked to add a filter that has been marked for removal, it 1401 * is safe to simply restore it to active state. __i40e_del_filter 1402 * will have simply deleted any filters which were previously marked 1403 * NEW or FAILED, so if it is currently marked REMOVE it must have 1404 * previously been ACTIVE. Since we haven't yet run the sync filters 1405 * task, just restore this filter to the ACTIVE state so that the 1406 * sync task leaves it in place 1407 */ 1408 if (f->state == I40E_FILTER_REMOVE) 1409 f->state = I40E_FILTER_ACTIVE; 1410 1411 return f; 1412 } 1413 1414 /** 1415 * __i40e_del_filter - Remove a specific filter from the VSI 1416 * @vsi: VSI to remove from 1417 * @f: the filter to remove from the list 1418 * 1419 * This function should be called instead of i40e_del_filter only if you know 1420 * the exact filter you will remove already, such as via i40e_find_filter or 1421 * i40e_find_mac. 1422 * 1423 * NOTE: This function is expected to be called with mac_filter_hash_lock 1424 * being held. 1425 * ANOTHER NOTE: This function MUST be called from within the context of 1426 * the "safe" variants of any list iterators, e.g. list_for_each_entry_safe() 1427 * instead of list_for_each_entry(). 1428 **/ 1429 void __i40e_del_filter(struct i40e_vsi *vsi, struct i40e_mac_filter *f) 1430 { 1431 if (!f) 1432 return; 1433 1434 /* If the filter was never added to firmware then we can just delete it 1435 * directly and we don't want to set the status to remove or else an 1436 * admin queue command will unnecessarily fire. 1437 */ 1438 if ((f->state == I40E_FILTER_FAILED) || 1439 (f->state == I40E_FILTER_NEW)) { 1440 hash_del(&f->hlist); 1441 kfree(f); 1442 } else { 1443 f->state = I40E_FILTER_REMOVE; 1444 } 1445 1446 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED; 1447 vsi->back->flags |= I40E_FLAG_FILTER_SYNC; 1448 } 1449 1450 /** 1451 * i40e_del_filter - Remove a MAC/VLAN filter from the VSI 1452 * @vsi: the VSI to be searched 1453 * @macaddr: the MAC address 1454 * @vlan: the VLAN 1455 * 1456 * NOTE: This function is expected to be called with mac_filter_hash_lock 1457 * being held. 1458 * ANOTHER NOTE: This function MUST be called from within the context of 1459 * the "safe" variants of any list iterators, e.g. list_for_each_entry_safe() 1460 * instead of list_for_each_entry(). 1461 **/ 1462 void i40e_del_filter(struct i40e_vsi *vsi, const u8 *macaddr, s16 vlan) 1463 { 1464 struct i40e_mac_filter *f; 1465 1466 if (!vsi || !macaddr) 1467 return; 1468 1469 f = i40e_find_filter(vsi, macaddr, vlan); 1470 __i40e_del_filter(vsi, f); 1471 } 1472 1473 /** 1474 * i40e_add_mac_filter - Add a MAC filter for all active VLANs 1475 * @vsi: the VSI to be searched 1476 * @macaddr: the mac address to be filtered 1477 * 1478 * If we're not in VLAN mode, just add the filter to I40E_VLAN_ANY. Otherwise, 1479 * go through all the macvlan filters and add a macvlan filter for each 1480 * unique vlan that already exists. If a PVID has been assigned, instead only 1481 * add the macaddr to that VLAN. 1482 * 1483 * Returns last filter added on success, else NULL 1484 **/ 1485 struct i40e_mac_filter *i40e_add_mac_filter(struct i40e_vsi *vsi, 1486 const u8 *macaddr) 1487 { 1488 struct i40e_mac_filter *f, *add = NULL; 1489 struct hlist_node *h; 1490 int bkt; 1491 1492 if (vsi->info.pvid) 1493 return i40e_add_filter(vsi, macaddr, 1494 le16_to_cpu(vsi->info.pvid)); 1495 1496 if (!i40e_is_vsi_in_vlan(vsi)) 1497 return i40e_add_filter(vsi, macaddr, I40E_VLAN_ANY); 1498 1499 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) { 1500 if (f->state == I40E_FILTER_REMOVE) 1501 continue; 1502 add = i40e_add_filter(vsi, macaddr, f->vlan); 1503 if (!add) 1504 return NULL; 1505 } 1506 1507 return add; 1508 } 1509 1510 /** 1511 * i40e_del_mac_filter - Remove a MAC filter from all VLANs 1512 * @vsi: the VSI to be searched 1513 * @macaddr: the mac address to be removed 1514 * 1515 * Removes a given MAC address from a VSI regardless of what VLAN it has been 1516 * associated with. 1517 * 1518 * Returns 0 for success, or error 1519 **/ 1520 int i40e_del_mac_filter(struct i40e_vsi *vsi, const u8 *macaddr) 1521 { 1522 struct i40e_mac_filter *f; 1523 struct hlist_node *h; 1524 bool found = false; 1525 int bkt; 1526 1527 WARN(!spin_is_locked(&vsi->mac_filter_hash_lock), 1528 "Missing mac_filter_hash_lock\n"); 1529 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) { 1530 if (ether_addr_equal(macaddr, f->macaddr)) { 1531 __i40e_del_filter(vsi, f); 1532 found = true; 1533 } 1534 } 1535 1536 if (found) 1537 return 0; 1538 else 1539 return -ENOENT; 1540 } 1541 1542 /** 1543 * i40e_set_mac - NDO callback to set mac address 1544 * @netdev: network interface device structure 1545 * @p: pointer to an address structure 1546 * 1547 * Returns 0 on success, negative on failure 1548 **/ 1549 static int i40e_set_mac(struct net_device *netdev, void *p) 1550 { 1551 struct i40e_netdev_priv *np = netdev_priv(netdev); 1552 struct i40e_vsi *vsi = np->vsi; 1553 struct i40e_pf *pf = vsi->back; 1554 struct i40e_hw *hw = &pf->hw; 1555 struct sockaddr *addr = p; 1556 1557 if (!is_valid_ether_addr(addr->sa_data)) 1558 return -EADDRNOTAVAIL; 1559 1560 if (ether_addr_equal(netdev->dev_addr, addr->sa_data)) { 1561 netdev_info(netdev, "already using mac address %pM\n", 1562 addr->sa_data); 1563 return 0; 1564 } 1565 1566 if (test_bit(__I40E_VSI_DOWN, vsi->back->state) || 1567 test_bit(__I40E_RESET_RECOVERY_PENDING, vsi->back->state)) 1568 return -EADDRNOTAVAIL; 1569 1570 if (ether_addr_equal(hw->mac.addr, addr->sa_data)) 1571 netdev_info(netdev, "returning to hw mac address %pM\n", 1572 hw->mac.addr); 1573 else 1574 netdev_info(netdev, "set new mac address %pM\n", addr->sa_data); 1575 1576 /* Copy the address first, so that we avoid a possible race with 1577 * .set_rx_mode(). If we copy after changing the address in the filter 1578 * list, we might open ourselves to a narrow race window where 1579 * .set_rx_mode could delete our dev_addr filter and prevent traffic 1580 * from passing. 1581 */ 1582 ether_addr_copy(netdev->dev_addr, addr->sa_data); 1583 1584 spin_lock_bh(&vsi->mac_filter_hash_lock); 1585 i40e_del_mac_filter(vsi, netdev->dev_addr); 1586 i40e_add_mac_filter(vsi, addr->sa_data); 1587 spin_unlock_bh(&vsi->mac_filter_hash_lock); 1588 if (vsi->type == I40E_VSI_MAIN) { 1589 i40e_status ret; 1590 1591 ret = i40e_aq_mac_address_write(&vsi->back->hw, 1592 I40E_AQC_WRITE_TYPE_LAA_WOL, 1593 addr->sa_data, NULL); 1594 if (ret) 1595 netdev_info(netdev, "Ignoring error from firmware on LAA update, status %s, AQ ret %s\n", 1596 i40e_stat_str(hw, ret), 1597 i40e_aq_str(hw, hw->aq.asq_last_status)); 1598 } 1599 1600 /* schedule our worker thread which will take care of 1601 * applying the new filter changes 1602 */ 1603 i40e_service_event_schedule(vsi->back); 1604 return 0; 1605 } 1606 1607 /** 1608 * i40e_config_rss_aq - Prepare for RSS using AQ commands 1609 * @vsi: vsi structure 1610 * @seed: RSS hash seed 1611 **/ 1612 static int i40e_config_rss_aq(struct i40e_vsi *vsi, const u8 *seed, 1613 u8 *lut, u16 lut_size) 1614 { 1615 struct i40e_pf *pf = vsi->back; 1616 struct i40e_hw *hw = &pf->hw; 1617 int ret = 0; 1618 1619 if (seed) { 1620 struct i40e_aqc_get_set_rss_key_data *seed_dw = 1621 (struct i40e_aqc_get_set_rss_key_data *)seed; 1622 ret = i40e_aq_set_rss_key(hw, vsi->id, seed_dw); 1623 if (ret) { 1624 dev_info(&pf->pdev->dev, 1625 "Cannot set RSS key, err %s aq_err %s\n", 1626 i40e_stat_str(hw, ret), 1627 i40e_aq_str(hw, hw->aq.asq_last_status)); 1628 return ret; 1629 } 1630 } 1631 if (lut) { 1632 bool pf_lut = vsi->type == I40E_VSI_MAIN ? true : false; 1633 1634 ret = i40e_aq_set_rss_lut(hw, vsi->id, pf_lut, lut, lut_size); 1635 if (ret) { 1636 dev_info(&pf->pdev->dev, 1637 "Cannot set RSS lut, err %s aq_err %s\n", 1638 i40e_stat_str(hw, ret), 1639 i40e_aq_str(hw, hw->aq.asq_last_status)); 1640 return ret; 1641 } 1642 } 1643 return ret; 1644 } 1645 1646 /** 1647 * i40e_vsi_config_rss - Prepare for VSI(VMDq) RSS if used 1648 * @vsi: VSI structure 1649 **/ 1650 static int i40e_vsi_config_rss(struct i40e_vsi *vsi) 1651 { 1652 struct i40e_pf *pf = vsi->back; 1653 u8 seed[I40E_HKEY_ARRAY_SIZE]; 1654 u8 *lut; 1655 int ret; 1656 1657 if (!(pf->hw_features & I40E_HW_RSS_AQ_CAPABLE)) 1658 return 0; 1659 if (!vsi->rss_size) 1660 vsi->rss_size = min_t(int, pf->alloc_rss_size, 1661 vsi->num_queue_pairs); 1662 if (!vsi->rss_size) 1663 return -EINVAL; 1664 lut = kzalloc(vsi->rss_table_size, GFP_KERNEL); 1665 if (!lut) 1666 return -ENOMEM; 1667 1668 /* Use the user configured hash keys and lookup table if there is one, 1669 * otherwise use default 1670 */ 1671 if (vsi->rss_lut_user) 1672 memcpy(lut, vsi->rss_lut_user, vsi->rss_table_size); 1673 else 1674 i40e_fill_rss_lut(pf, lut, vsi->rss_table_size, vsi->rss_size); 1675 if (vsi->rss_hkey_user) 1676 memcpy(seed, vsi->rss_hkey_user, I40E_HKEY_ARRAY_SIZE); 1677 else 1678 netdev_rss_key_fill((void *)seed, I40E_HKEY_ARRAY_SIZE); 1679 ret = i40e_config_rss_aq(vsi, seed, lut, vsi->rss_table_size); 1680 kfree(lut); 1681 return ret; 1682 } 1683 1684 /** 1685 * i40e_vsi_setup_queue_map_mqprio - Prepares mqprio based tc_config 1686 * @vsi: the VSI being configured, 1687 * @ctxt: VSI context structure 1688 * @enabled_tc: number of traffic classes to enable 1689 * 1690 * Prepares VSI tc_config to have queue configurations based on MQPRIO options. 1691 **/ 1692 static int i40e_vsi_setup_queue_map_mqprio(struct i40e_vsi *vsi, 1693 struct i40e_vsi_context *ctxt, 1694 u8 enabled_tc) 1695 { 1696 u16 qcount = 0, max_qcount, qmap, sections = 0; 1697 int i, override_q, pow, num_qps, ret; 1698 u8 netdev_tc = 0, offset = 0; 1699 1700 if (vsi->type != I40E_VSI_MAIN) 1701 return -EINVAL; 1702 sections = I40E_AQ_VSI_PROP_QUEUE_MAP_VALID; 1703 sections |= I40E_AQ_VSI_PROP_SCHED_VALID; 1704 vsi->tc_config.numtc = vsi->mqprio_qopt.qopt.num_tc; 1705 vsi->tc_config.enabled_tc = enabled_tc ? enabled_tc : 1; 1706 num_qps = vsi->mqprio_qopt.qopt.count[0]; 1707 1708 /* find the next higher power-of-2 of num queue pairs */ 1709 pow = ilog2(num_qps); 1710 if (!is_power_of_2(num_qps)) 1711 pow++; 1712 qmap = (offset << I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT) | 1713 (pow << I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT); 1714 1715 /* Setup queue offset/count for all TCs for given VSI */ 1716 max_qcount = vsi->mqprio_qopt.qopt.count[0]; 1717 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 1718 /* See if the given TC is enabled for the given VSI */ 1719 if (vsi->tc_config.enabled_tc & BIT(i)) { 1720 offset = vsi->mqprio_qopt.qopt.offset[i]; 1721 qcount = vsi->mqprio_qopt.qopt.count[i]; 1722 if (qcount > max_qcount) 1723 max_qcount = qcount; 1724 vsi->tc_config.tc_info[i].qoffset = offset; 1725 vsi->tc_config.tc_info[i].qcount = qcount; 1726 vsi->tc_config.tc_info[i].netdev_tc = netdev_tc++; 1727 } else { 1728 /* TC is not enabled so set the offset to 1729 * default queue and allocate one queue 1730 * for the given TC. 1731 */ 1732 vsi->tc_config.tc_info[i].qoffset = 0; 1733 vsi->tc_config.tc_info[i].qcount = 1; 1734 vsi->tc_config.tc_info[i].netdev_tc = 0; 1735 } 1736 } 1737 1738 /* Set actual Tx/Rx queue pairs */ 1739 vsi->num_queue_pairs = offset + qcount; 1740 1741 /* Setup queue TC[0].qmap for given VSI context */ 1742 ctxt->info.tc_mapping[0] = cpu_to_le16(qmap); 1743 ctxt->info.mapping_flags |= cpu_to_le16(I40E_AQ_VSI_QUE_MAP_CONTIG); 1744 ctxt->info.queue_mapping[0] = cpu_to_le16(vsi->base_queue); 1745 ctxt->info.valid_sections |= cpu_to_le16(sections); 1746 1747 /* Reconfigure RSS for main VSI with max queue count */ 1748 vsi->rss_size = max_qcount; 1749 ret = i40e_vsi_config_rss(vsi); 1750 if (ret) { 1751 dev_info(&vsi->back->pdev->dev, 1752 "Failed to reconfig rss for num_queues (%u)\n", 1753 max_qcount); 1754 return ret; 1755 } 1756 vsi->reconfig_rss = true; 1757 dev_dbg(&vsi->back->pdev->dev, 1758 "Reconfigured rss with num_queues (%u)\n", max_qcount); 1759 1760 /* Find queue count available for channel VSIs and starting offset 1761 * for channel VSIs 1762 */ 1763 override_q = vsi->mqprio_qopt.qopt.count[0]; 1764 if (override_q && override_q < vsi->num_queue_pairs) { 1765 vsi->cnt_q_avail = vsi->num_queue_pairs - override_q; 1766 vsi->next_base_queue = override_q; 1767 } 1768 return 0; 1769 } 1770 1771 /** 1772 * i40e_vsi_setup_queue_map - Setup a VSI queue map based on enabled_tc 1773 * @vsi: the VSI being setup 1774 * @ctxt: VSI context structure 1775 * @enabled_tc: Enabled TCs bitmap 1776 * @is_add: True if called before Add VSI 1777 * 1778 * Setup VSI queue mapping for enabled traffic classes. 1779 **/ 1780 static void i40e_vsi_setup_queue_map(struct i40e_vsi *vsi, 1781 struct i40e_vsi_context *ctxt, 1782 u8 enabled_tc, 1783 bool is_add) 1784 { 1785 struct i40e_pf *pf = vsi->back; 1786 u16 sections = 0; 1787 u8 netdev_tc = 0; 1788 u16 numtc = 0; 1789 u16 qcount; 1790 u8 offset; 1791 u16 qmap; 1792 int i; 1793 u16 num_tc_qps = 0; 1794 1795 sections = I40E_AQ_VSI_PROP_QUEUE_MAP_VALID; 1796 offset = 0; 1797 1798 if (enabled_tc && (vsi->back->flags & I40E_FLAG_DCB_ENABLED)) { 1799 /* Find numtc from enabled TC bitmap */ 1800 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 1801 if (enabled_tc & BIT(i)) /* TC is enabled */ 1802 numtc++; 1803 } 1804 if (!numtc) { 1805 dev_warn(&pf->pdev->dev, "DCB is enabled but no TC enabled, forcing TC0\n"); 1806 numtc = 1; 1807 } 1808 } else { 1809 /* At least TC0 is enabled in non-DCB, non-MQPRIO case */ 1810 numtc = 1; 1811 } 1812 1813 vsi->tc_config.numtc = numtc; 1814 vsi->tc_config.enabled_tc = enabled_tc ? enabled_tc : 1; 1815 /* Number of queues per enabled TC */ 1816 qcount = vsi->alloc_queue_pairs; 1817 1818 num_tc_qps = qcount / numtc; 1819 num_tc_qps = min_t(int, num_tc_qps, i40e_pf_get_max_q_per_tc(pf)); 1820 1821 /* Do not allow use more TC queue pairs than MSI-X vectors exist */ 1822 if (pf->flags & I40E_FLAG_MSIX_ENABLED) 1823 num_tc_qps = min_t(int, num_tc_qps, pf->num_lan_msix); 1824 1825 /* Setup queue offset/count for all TCs for given VSI */ 1826 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 1827 /* See if the given TC is enabled for the given VSI */ 1828 if (vsi->tc_config.enabled_tc & BIT(i)) { 1829 /* TC is enabled */ 1830 int pow, num_qps; 1831 1832 switch (vsi->type) { 1833 case I40E_VSI_MAIN: 1834 qcount = min_t(int, pf->alloc_rss_size, 1835 num_tc_qps); 1836 break; 1837 case I40E_VSI_FDIR: 1838 case I40E_VSI_SRIOV: 1839 case I40E_VSI_VMDQ2: 1840 default: 1841 qcount = num_tc_qps; 1842 WARN_ON(i != 0); 1843 break; 1844 } 1845 vsi->tc_config.tc_info[i].qoffset = offset; 1846 vsi->tc_config.tc_info[i].qcount = qcount; 1847 1848 /* find the next higher power-of-2 of num queue pairs */ 1849 num_qps = qcount; 1850 pow = 0; 1851 while (num_qps && (BIT_ULL(pow) < qcount)) { 1852 pow++; 1853 num_qps >>= 1; 1854 } 1855 1856 vsi->tc_config.tc_info[i].netdev_tc = netdev_tc++; 1857 qmap = 1858 (offset << I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT) | 1859 (pow << I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT); 1860 1861 offset += qcount; 1862 } else { 1863 /* TC is not enabled so set the offset to 1864 * default queue and allocate one queue 1865 * for the given TC. 1866 */ 1867 vsi->tc_config.tc_info[i].qoffset = 0; 1868 vsi->tc_config.tc_info[i].qcount = 1; 1869 vsi->tc_config.tc_info[i].netdev_tc = 0; 1870 1871 qmap = 0; 1872 } 1873 ctxt->info.tc_mapping[i] = cpu_to_le16(qmap); 1874 } 1875 1876 /* Set actual Tx/Rx queue pairs */ 1877 vsi->num_queue_pairs = offset; 1878 if ((vsi->type == I40E_VSI_MAIN) && (numtc == 1)) { 1879 if (vsi->req_queue_pairs > 0) 1880 vsi->num_queue_pairs = vsi->req_queue_pairs; 1881 else if (pf->flags & I40E_FLAG_MSIX_ENABLED) 1882 vsi->num_queue_pairs = pf->num_lan_msix; 1883 } 1884 1885 /* Scheduler section valid can only be set for ADD VSI */ 1886 if (is_add) { 1887 sections |= I40E_AQ_VSI_PROP_SCHED_VALID; 1888 1889 ctxt->info.up_enable_bits = enabled_tc; 1890 } 1891 if (vsi->type == I40E_VSI_SRIOV) { 1892 ctxt->info.mapping_flags |= 1893 cpu_to_le16(I40E_AQ_VSI_QUE_MAP_NONCONTIG); 1894 for (i = 0; i < vsi->num_queue_pairs; i++) 1895 ctxt->info.queue_mapping[i] = 1896 cpu_to_le16(vsi->base_queue + i); 1897 } else { 1898 ctxt->info.mapping_flags |= 1899 cpu_to_le16(I40E_AQ_VSI_QUE_MAP_CONTIG); 1900 ctxt->info.queue_mapping[0] = cpu_to_le16(vsi->base_queue); 1901 } 1902 ctxt->info.valid_sections |= cpu_to_le16(sections); 1903 } 1904 1905 /** 1906 * i40e_addr_sync - Callback for dev_(mc|uc)_sync to add address 1907 * @netdev: the netdevice 1908 * @addr: address to add 1909 * 1910 * Called by __dev_(mc|uc)_sync when an address needs to be added. We call 1911 * __dev_(uc|mc)_sync from .set_rx_mode and guarantee to hold the hash lock. 1912 */ 1913 static int i40e_addr_sync(struct net_device *netdev, const u8 *addr) 1914 { 1915 struct i40e_netdev_priv *np = netdev_priv(netdev); 1916 struct i40e_vsi *vsi = np->vsi; 1917 1918 if (i40e_add_mac_filter(vsi, addr)) 1919 return 0; 1920 else 1921 return -ENOMEM; 1922 } 1923 1924 /** 1925 * i40e_addr_unsync - Callback for dev_(mc|uc)_sync to remove address 1926 * @netdev: the netdevice 1927 * @addr: address to add 1928 * 1929 * Called by __dev_(mc|uc)_sync when an address needs to be removed. We call 1930 * __dev_(uc|mc)_sync from .set_rx_mode and guarantee to hold the hash lock. 1931 */ 1932 static int i40e_addr_unsync(struct net_device *netdev, const u8 *addr) 1933 { 1934 struct i40e_netdev_priv *np = netdev_priv(netdev); 1935 struct i40e_vsi *vsi = np->vsi; 1936 1937 /* Under some circumstances, we might receive a request to delete 1938 * our own device address from our uc list. Because we store the 1939 * device address in the VSI's MAC/VLAN filter list, we need to ignore 1940 * such requests and not delete our device address from this list. 1941 */ 1942 if (ether_addr_equal(addr, netdev->dev_addr)) 1943 return 0; 1944 1945 i40e_del_mac_filter(vsi, addr); 1946 1947 return 0; 1948 } 1949 1950 /** 1951 * i40e_set_rx_mode - NDO callback to set the netdev filters 1952 * @netdev: network interface device structure 1953 **/ 1954 static void i40e_set_rx_mode(struct net_device *netdev) 1955 { 1956 struct i40e_netdev_priv *np = netdev_priv(netdev); 1957 struct i40e_vsi *vsi = np->vsi; 1958 1959 spin_lock_bh(&vsi->mac_filter_hash_lock); 1960 1961 __dev_uc_sync(netdev, i40e_addr_sync, i40e_addr_unsync); 1962 __dev_mc_sync(netdev, i40e_addr_sync, i40e_addr_unsync); 1963 1964 spin_unlock_bh(&vsi->mac_filter_hash_lock); 1965 1966 /* check for other flag changes */ 1967 if (vsi->current_netdev_flags != vsi->netdev->flags) { 1968 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED; 1969 vsi->back->flags |= I40E_FLAG_FILTER_SYNC; 1970 } 1971 } 1972 1973 /** 1974 * i40e_undo_del_filter_entries - Undo the changes made to MAC filter entries 1975 * @vsi: Pointer to VSI struct 1976 * @from: Pointer to list which contains MAC filter entries - changes to 1977 * those entries needs to be undone. 1978 * 1979 * MAC filter entries from this list were slated for deletion. 1980 **/ 1981 static void i40e_undo_del_filter_entries(struct i40e_vsi *vsi, 1982 struct hlist_head *from) 1983 { 1984 struct i40e_mac_filter *f; 1985 struct hlist_node *h; 1986 1987 hlist_for_each_entry_safe(f, h, from, hlist) { 1988 u64 key = i40e_addr_to_hkey(f->macaddr); 1989 1990 /* Move the element back into MAC filter list*/ 1991 hlist_del(&f->hlist); 1992 hash_add(vsi->mac_filter_hash, &f->hlist, key); 1993 } 1994 } 1995 1996 /** 1997 * i40e_undo_add_filter_entries - Undo the changes made to MAC filter entries 1998 * @vsi: Pointer to vsi struct 1999 * @from: Pointer to list which contains MAC filter entries - changes to 2000 * those entries needs to be undone. 2001 * 2002 * MAC filter entries from this list were slated for addition. 2003 **/ 2004 static void i40e_undo_add_filter_entries(struct i40e_vsi *vsi, 2005 struct hlist_head *from) 2006 { 2007 struct i40e_new_mac_filter *new; 2008 struct hlist_node *h; 2009 2010 hlist_for_each_entry_safe(new, h, from, hlist) { 2011 /* We can simply free the wrapper structure */ 2012 hlist_del(&new->hlist); 2013 kfree(new); 2014 } 2015 } 2016 2017 /** 2018 * i40e_next_entry - Get the next non-broadcast filter from a list 2019 * @next: pointer to filter in list 2020 * 2021 * Returns the next non-broadcast filter in the list. Required so that we 2022 * ignore broadcast filters within the list, since these are not handled via 2023 * the normal firmware update path. 2024 */ 2025 static 2026 struct i40e_new_mac_filter *i40e_next_filter(struct i40e_new_mac_filter *next) 2027 { 2028 hlist_for_each_entry_continue(next, hlist) { 2029 if (!is_broadcast_ether_addr(next->f->macaddr)) 2030 return next; 2031 } 2032 2033 return NULL; 2034 } 2035 2036 /** 2037 * i40e_update_filter_state - Update filter state based on return data 2038 * from firmware 2039 * @count: Number of filters added 2040 * @add_list: return data from fw 2041 * @head: pointer to first filter in current batch 2042 * 2043 * MAC filter entries from list were slated to be added to device. Returns 2044 * number of successful filters. Note that 0 does NOT mean success! 2045 **/ 2046 static int 2047 i40e_update_filter_state(int count, 2048 struct i40e_aqc_add_macvlan_element_data *add_list, 2049 struct i40e_new_mac_filter *add_head) 2050 { 2051 int retval = 0; 2052 int i; 2053 2054 for (i = 0; i < count; i++) { 2055 /* Always check status of each filter. We don't need to check 2056 * the firmware return status because we pre-set the filter 2057 * status to I40E_AQC_MM_ERR_NO_RES when sending the filter 2058 * request to the adminq. Thus, if it no longer matches then 2059 * we know the filter is active. 2060 */ 2061 if (add_list[i].match_method == I40E_AQC_MM_ERR_NO_RES) { 2062 add_head->state = I40E_FILTER_FAILED; 2063 } else { 2064 add_head->state = I40E_FILTER_ACTIVE; 2065 retval++; 2066 } 2067 2068 add_head = i40e_next_filter(add_head); 2069 if (!add_head) 2070 break; 2071 } 2072 2073 return retval; 2074 } 2075 2076 /** 2077 * i40e_aqc_del_filters - Request firmware to delete a set of filters 2078 * @vsi: ptr to the VSI 2079 * @vsi_name: name to display in messages 2080 * @list: the list of filters to send to firmware 2081 * @num_del: the number of filters to delete 2082 * @retval: Set to -EIO on failure to delete 2083 * 2084 * Send a request to firmware via AdminQ to delete a set of filters. Uses 2085 * *retval instead of a return value so that success does not force ret_val to 2086 * be set to 0. This ensures that a sequence of calls to this function 2087 * preserve the previous value of *retval on successful delete. 2088 */ 2089 static 2090 void i40e_aqc_del_filters(struct i40e_vsi *vsi, const char *vsi_name, 2091 struct i40e_aqc_remove_macvlan_element_data *list, 2092 int num_del, int *retval) 2093 { 2094 struct i40e_hw *hw = &vsi->back->hw; 2095 i40e_status aq_ret; 2096 int aq_err; 2097 2098 aq_ret = i40e_aq_remove_macvlan(hw, vsi->seid, list, num_del, NULL); 2099 aq_err = hw->aq.asq_last_status; 2100 2101 /* Explicitly ignore and do not report when firmware returns ENOENT */ 2102 if (aq_ret && !(aq_err == I40E_AQ_RC_ENOENT)) { 2103 *retval = -EIO; 2104 dev_info(&vsi->back->pdev->dev, 2105 "ignoring delete macvlan error on %s, err %s, aq_err %s\n", 2106 vsi_name, i40e_stat_str(hw, aq_ret), 2107 i40e_aq_str(hw, aq_err)); 2108 } 2109 } 2110 2111 /** 2112 * i40e_aqc_add_filters - Request firmware to add a set of filters 2113 * @vsi: ptr to the VSI 2114 * @vsi_name: name to display in messages 2115 * @list: the list of filters to send to firmware 2116 * @add_head: Position in the add hlist 2117 * @num_add: the number of filters to add 2118 * @promisc_change: set to true on exit if promiscuous mode was forced on 2119 * 2120 * Send a request to firmware via AdminQ to add a chunk of filters. Will set 2121 * promisc_changed to true if the firmware has run out of space for more 2122 * filters. 2123 */ 2124 static 2125 void i40e_aqc_add_filters(struct i40e_vsi *vsi, const char *vsi_name, 2126 struct i40e_aqc_add_macvlan_element_data *list, 2127 struct i40e_new_mac_filter *add_head, 2128 int num_add, bool *promisc_changed) 2129 { 2130 struct i40e_hw *hw = &vsi->back->hw; 2131 int aq_err, fcnt; 2132 2133 i40e_aq_add_macvlan(hw, vsi->seid, list, num_add, NULL); 2134 aq_err = hw->aq.asq_last_status; 2135 fcnt = i40e_update_filter_state(num_add, list, add_head); 2136 2137 if (fcnt != num_add) { 2138 *promisc_changed = true; 2139 set_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state); 2140 dev_warn(&vsi->back->pdev->dev, 2141 "Error %s adding RX filters on %s, promiscuous mode forced on\n", 2142 i40e_aq_str(hw, aq_err), 2143 vsi_name); 2144 } 2145 } 2146 2147 /** 2148 * i40e_aqc_broadcast_filter - Set promiscuous broadcast flags 2149 * @vsi: pointer to the VSI 2150 * @f: filter data 2151 * 2152 * This function sets or clears the promiscuous broadcast flags for VLAN 2153 * filters in order to properly receive broadcast frames. Assumes that only 2154 * broadcast filters are passed. 2155 * 2156 * Returns status indicating success or failure; 2157 **/ 2158 static i40e_status 2159 i40e_aqc_broadcast_filter(struct i40e_vsi *vsi, const char *vsi_name, 2160 struct i40e_mac_filter *f) 2161 { 2162 bool enable = f->state == I40E_FILTER_NEW; 2163 struct i40e_hw *hw = &vsi->back->hw; 2164 i40e_status aq_ret; 2165 2166 if (f->vlan == I40E_VLAN_ANY) { 2167 aq_ret = i40e_aq_set_vsi_broadcast(hw, 2168 vsi->seid, 2169 enable, 2170 NULL); 2171 } else { 2172 aq_ret = i40e_aq_set_vsi_bc_promisc_on_vlan(hw, 2173 vsi->seid, 2174 enable, 2175 f->vlan, 2176 NULL); 2177 } 2178 2179 if (aq_ret) 2180 dev_warn(&vsi->back->pdev->dev, 2181 "Error %s setting broadcast promiscuous mode on %s\n", 2182 i40e_aq_str(hw, hw->aq.asq_last_status), 2183 vsi_name); 2184 2185 return aq_ret; 2186 } 2187 2188 /** 2189 * i40e_set_promiscuous - set promiscuous mode 2190 * @pf: board private structure 2191 * @promisc: promisc on or off 2192 * 2193 * There are different ways of setting promiscuous mode on a PF depending on 2194 * what state/environment we're in. This identifies and sets it appropriately. 2195 * Returns 0 on success. 2196 **/ 2197 static int i40e_set_promiscuous(struct i40e_pf *pf, bool promisc) 2198 { 2199 struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi]; 2200 struct i40e_hw *hw = &pf->hw; 2201 i40e_status aq_ret; 2202 2203 if (vsi->type == I40E_VSI_MAIN && 2204 pf->lan_veb != I40E_NO_VEB && 2205 !(pf->flags & I40E_FLAG_MFP_ENABLED)) { 2206 /* set defport ON for Main VSI instead of true promisc 2207 * this way we will get all unicast/multicast and VLAN 2208 * promisc behavior but will not get VF or VMDq traffic 2209 * replicated on the Main VSI. 2210 */ 2211 if (promisc) 2212 aq_ret = i40e_aq_set_default_vsi(hw, 2213 vsi->seid, 2214 NULL); 2215 else 2216 aq_ret = i40e_aq_clear_default_vsi(hw, 2217 vsi->seid, 2218 NULL); 2219 if (aq_ret) { 2220 dev_info(&pf->pdev->dev, 2221 "Set default VSI failed, err %s, aq_err %s\n", 2222 i40e_stat_str(hw, aq_ret), 2223 i40e_aq_str(hw, hw->aq.asq_last_status)); 2224 } 2225 } else { 2226 aq_ret = i40e_aq_set_vsi_unicast_promiscuous( 2227 hw, 2228 vsi->seid, 2229 promisc, NULL, 2230 true); 2231 if (aq_ret) { 2232 dev_info(&pf->pdev->dev, 2233 "set unicast promisc failed, err %s, aq_err %s\n", 2234 i40e_stat_str(hw, aq_ret), 2235 i40e_aq_str(hw, hw->aq.asq_last_status)); 2236 } 2237 aq_ret = i40e_aq_set_vsi_multicast_promiscuous( 2238 hw, 2239 vsi->seid, 2240 promisc, NULL); 2241 if (aq_ret) { 2242 dev_info(&pf->pdev->dev, 2243 "set multicast promisc failed, err %s, aq_err %s\n", 2244 i40e_stat_str(hw, aq_ret), 2245 i40e_aq_str(hw, hw->aq.asq_last_status)); 2246 } 2247 } 2248 2249 if (!aq_ret) 2250 pf->cur_promisc = promisc; 2251 2252 return aq_ret; 2253 } 2254 2255 /** 2256 * i40e_sync_vsi_filters - Update the VSI filter list to the HW 2257 * @vsi: ptr to the VSI 2258 * 2259 * Push any outstanding VSI filter changes through the AdminQ. 2260 * 2261 * Returns 0 or error value 2262 **/ 2263 int i40e_sync_vsi_filters(struct i40e_vsi *vsi) 2264 { 2265 struct hlist_head tmp_add_list, tmp_del_list; 2266 struct i40e_mac_filter *f; 2267 struct i40e_new_mac_filter *new, *add_head = NULL; 2268 struct i40e_hw *hw = &vsi->back->hw; 2269 unsigned int failed_filters = 0; 2270 unsigned int vlan_filters = 0; 2271 bool promisc_changed = false; 2272 char vsi_name[16] = "PF"; 2273 int filter_list_len = 0; 2274 i40e_status aq_ret = 0; 2275 u32 changed_flags = 0; 2276 struct hlist_node *h; 2277 struct i40e_pf *pf; 2278 int num_add = 0; 2279 int num_del = 0; 2280 int retval = 0; 2281 u16 cmd_flags; 2282 int list_size; 2283 int bkt; 2284 2285 /* empty array typed pointers, kcalloc later */ 2286 struct i40e_aqc_add_macvlan_element_data *add_list; 2287 struct i40e_aqc_remove_macvlan_element_data *del_list; 2288 2289 while (test_and_set_bit(__I40E_VSI_SYNCING_FILTERS, vsi->state)) 2290 usleep_range(1000, 2000); 2291 pf = vsi->back; 2292 2293 if (vsi->netdev) { 2294 changed_flags = vsi->current_netdev_flags ^ vsi->netdev->flags; 2295 vsi->current_netdev_flags = vsi->netdev->flags; 2296 } 2297 2298 INIT_HLIST_HEAD(&tmp_add_list); 2299 INIT_HLIST_HEAD(&tmp_del_list); 2300 2301 if (vsi->type == I40E_VSI_SRIOV) 2302 snprintf(vsi_name, sizeof(vsi_name) - 1, "VF %d", vsi->vf_id); 2303 else if (vsi->type != I40E_VSI_MAIN) 2304 snprintf(vsi_name, sizeof(vsi_name) - 1, "vsi %d", vsi->seid); 2305 2306 if (vsi->flags & I40E_VSI_FLAG_FILTER_CHANGED) { 2307 vsi->flags &= ~I40E_VSI_FLAG_FILTER_CHANGED; 2308 2309 spin_lock_bh(&vsi->mac_filter_hash_lock); 2310 /* Create a list of filters to delete. */ 2311 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) { 2312 if (f->state == I40E_FILTER_REMOVE) { 2313 /* Move the element into temporary del_list */ 2314 hash_del(&f->hlist); 2315 hlist_add_head(&f->hlist, &tmp_del_list); 2316 2317 /* Avoid counting removed filters */ 2318 continue; 2319 } 2320 if (f->state == I40E_FILTER_NEW) { 2321 /* Create a temporary i40e_new_mac_filter */ 2322 new = kzalloc(sizeof(*new), GFP_ATOMIC); 2323 if (!new) 2324 goto err_no_memory_locked; 2325 2326 /* Store pointer to the real filter */ 2327 new->f = f; 2328 new->state = f->state; 2329 2330 /* Add it to the hash list */ 2331 hlist_add_head(&new->hlist, &tmp_add_list); 2332 } 2333 2334 /* Count the number of active (current and new) VLAN 2335 * filters we have now. Does not count filters which 2336 * are marked for deletion. 2337 */ 2338 if (f->vlan > 0) 2339 vlan_filters++; 2340 } 2341 2342 retval = i40e_correct_mac_vlan_filters(vsi, 2343 &tmp_add_list, 2344 &tmp_del_list, 2345 vlan_filters); 2346 if (retval) 2347 goto err_no_memory_locked; 2348 2349 spin_unlock_bh(&vsi->mac_filter_hash_lock); 2350 } 2351 2352 /* Now process 'del_list' outside the lock */ 2353 if (!hlist_empty(&tmp_del_list)) { 2354 filter_list_len = hw->aq.asq_buf_size / 2355 sizeof(struct i40e_aqc_remove_macvlan_element_data); 2356 list_size = filter_list_len * 2357 sizeof(struct i40e_aqc_remove_macvlan_element_data); 2358 del_list = kzalloc(list_size, GFP_ATOMIC); 2359 if (!del_list) 2360 goto err_no_memory; 2361 2362 hlist_for_each_entry_safe(f, h, &tmp_del_list, hlist) { 2363 cmd_flags = 0; 2364 2365 /* handle broadcast filters by updating the broadcast 2366 * promiscuous flag and release filter list. 2367 */ 2368 if (is_broadcast_ether_addr(f->macaddr)) { 2369 i40e_aqc_broadcast_filter(vsi, vsi_name, f); 2370 2371 hlist_del(&f->hlist); 2372 kfree(f); 2373 continue; 2374 } 2375 2376 /* add to delete list */ 2377 ether_addr_copy(del_list[num_del].mac_addr, f->macaddr); 2378 if (f->vlan == I40E_VLAN_ANY) { 2379 del_list[num_del].vlan_tag = 0; 2380 cmd_flags |= I40E_AQC_MACVLAN_DEL_IGNORE_VLAN; 2381 } else { 2382 del_list[num_del].vlan_tag = 2383 cpu_to_le16((u16)(f->vlan)); 2384 } 2385 2386 cmd_flags |= I40E_AQC_MACVLAN_DEL_PERFECT_MATCH; 2387 del_list[num_del].flags = cmd_flags; 2388 num_del++; 2389 2390 /* flush a full buffer */ 2391 if (num_del == filter_list_len) { 2392 i40e_aqc_del_filters(vsi, vsi_name, del_list, 2393 num_del, &retval); 2394 memset(del_list, 0, list_size); 2395 num_del = 0; 2396 } 2397 /* Release memory for MAC filter entries which were 2398 * synced up with HW. 2399 */ 2400 hlist_del(&f->hlist); 2401 kfree(f); 2402 } 2403 2404 if (num_del) { 2405 i40e_aqc_del_filters(vsi, vsi_name, del_list, 2406 num_del, &retval); 2407 } 2408 2409 kfree(del_list); 2410 del_list = NULL; 2411 } 2412 2413 if (!hlist_empty(&tmp_add_list)) { 2414 /* Do all the adds now. */ 2415 filter_list_len = hw->aq.asq_buf_size / 2416 sizeof(struct i40e_aqc_add_macvlan_element_data); 2417 list_size = filter_list_len * 2418 sizeof(struct i40e_aqc_add_macvlan_element_data); 2419 add_list = kzalloc(list_size, GFP_ATOMIC); 2420 if (!add_list) 2421 goto err_no_memory; 2422 2423 num_add = 0; 2424 hlist_for_each_entry_safe(new, h, &tmp_add_list, hlist) { 2425 if (test_bit(__I40E_VSI_OVERFLOW_PROMISC, 2426 vsi->state)) { 2427 new->state = I40E_FILTER_FAILED; 2428 continue; 2429 } 2430 2431 /* handle broadcast filters by updating the broadcast 2432 * promiscuous flag instead of adding a MAC filter. 2433 */ 2434 if (is_broadcast_ether_addr(new->f->macaddr)) { 2435 if (i40e_aqc_broadcast_filter(vsi, vsi_name, 2436 new->f)) 2437 new->state = I40E_FILTER_FAILED; 2438 else 2439 new->state = I40E_FILTER_ACTIVE; 2440 continue; 2441 } 2442 2443 /* add to add array */ 2444 if (num_add == 0) 2445 add_head = new; 2446 cmd_flags = 0; 2447 ether_addr_copy(add_list[num_add].mac_addr, 2448 new->f->macaddr); 2449 if (new->f->vlan == I40E_VLAN_ANY) { 2450 add_list[num_add].vlan_tag = 0; 2451 cmd_flags |= I40E_AQC_MACVLAN_ADD_IGNORE_VLAN; 2452 } else { 2453 add_list[num_add].vlan_tag = 2454 cpu_to_le16((u16)(new->f->vlan)); 2455 } 2456 add_list[num_add].queue_number = 0; 2457 /* set invalid match method for later detection */ 2458 add_list[num_add].match_method = I40E_AQC_MM_ERR_NO_RES; 2459 cmd_flags |= I40E_AQC_MACVLAN_ADD_PERFECT_MATCH; 2460 add_list[num_add].flags = cpu_to_le16(cmd_flags); 2461 num_add++; 2462 2463 /* flush a full buffer */ 2464 if (num_add == filter_list_len) { 2465 i40e_aqc_add_filters(vsi, vsi_name, add_list, 2466 add_head, num_add, 2467 &promisc_changed); 2468 memset(add_list, 0, list_size); 2469 num_add = 0; 2470 } 2471 } 2472 if (num_add) { 2473 i40e_aqc_add_filters(vsi, vsi_name, add_list, add_head, 2474 num_add, &promisc_changed); 2475 } 2476 /* Now move all of the filters from the temp add list back to 2477 * the VSI's list. 2478 */ 2479 spin_lock_bh(&vsi->mac_filter_hash_lock); 2480 hlist_for_each_entry_safe(new, h, &tmp_add_list, hlist) { 2481 /* Only update the state if we're still NEW */ 2482 if (new->f->state == I40E_FILTER_NEW) 2483 new->f->state = new->state; 2484 hlist_del(&new->hlist); 2485 kfree(new); 2486 } 2487 spin_unlock_bh(&vsi->mac_filter_hash_lock); 2488 kfree(add_list); 2489 add_list = NULL; 2490 } 2491 2492 /* Determine the number of active and failed filters. */ 2493 spin_lock_bh(&vsi->mac_filter_hash_lock); 2494 vsi->active_filters = 0; 2495 hash_for_each(vsi->mac_filter_hash, bkt, f, hlist) { 2496 if (f->state == I40E_FILTER_ACTIVE) 2497 vsi->active_filters++; 2498 else if (f->state == I40E_FILTER_FAILED) 2499 failed_filters++; 2500 } 2501 spin_unlock_bh(&vsi->mac_filter_hash_lock); 2502 2503 /* If promiscuous mode has changed, we need to calculate a new 2504 * threshold for when we are safe to exit 2505 */ 2506 if (promisc_changed) 2507 vsi->promisc_threshold = (vsi->active_filters * 3) / 4; 2508 2509 /* Check if we are able to exit overflow promiscuous mode. We can 2510 * safely exit if we didn't just enter, we no longer have any failed 2511 * filters, and we have reduced filters below the threshold value. 2512 */ 2513 if (test_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state) && 2514 !promisc_changed && !failed_filters && 2515 (vsi->active_filters < vsi->promisc_threshold)) { 2516 dev_info(&pf->pdev->dev, 2517 "filter logjam cleared on %s, leaving overflow promiscuous mode\n", 2518 vsi_name); 2519 clear_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state); 2520 promisc_changed = true; 2521 vsi->promisc_threshold = 0; 2522 } 2523 2524 /* if the VF is not trusted do not do promisc */ 2525 if ((vsi->type == I40E_VSI_SRIOV) && !pf->vf[vsi->vf_id].trusted) { 2526 clear_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state); 2527 goto out; 2528 } 2529 2530 /* check for changes in promiscuous modes */ 2531 if (changed_flags & IFF_ALLMULTI) { 2532 bool cur_multipromisc; 2533 2534 cur_multipromisc = !!(vsi->current_netdev_flags & IFF_ALLMULTI); 2535 aq_ret = i40e_aq_set_vsi_multicast_promiscuous(&vsi->back->hw, 2536 vsi->seid, 2537 cur_multipromisc, 2538 NULL); 2539 if (aq_ret) { 2540 retval = i40e_aq_rc_to_posix(aq_ret, 2541 hw->aq.asq_last_status); 2542 dev_info(&pf->pdev->dev, 2543 "set multi promisc failed on %s, err %s aq_err %s\n", 2544 vsi_name, 2545 i40e_stat_str(hw, aq_ret), 2546 i40e_aq_str(hw, hw->aq.asq_last_status)); 2547 } 2548 } 2549 2550 if ((changed_flags & IFF_PROMISC) || promisc_changed) { 2551 bool cur_promisc; 2552 2553 cur_promisc = (!!(vsi->current_netdev_flags & IFF_PROMISC) || 2554 test_bit(__I40E_VSI_OVERFLOW_PROMISC, 2555 vsi->state)); 2556 aq_ret = i40e_set_promiscuous(pf, cur_promisc); 2557 if (aq_ret) { 2558 retval = i40e_aq_rc_to_posix(aq_ret, 2559 hw->aq.asq_last_status); 2560 dev_info(&pf->pdev->dev, 2561 "Setting promiscuous %s failed on %s, err %s aq_err %s\n", 2562 cur_promisc ? "on" : "off", 2563 vsi_name, 2564 i40e_stat_str(hw, aq_ret), 2565 i40e_aq_str(hw, hw->aq.asq_last_status)); 2566 } 2567 } 2568 out: 2569 /* if something went wrong then set the changed flag so we try again */ 2570 if (retval) 2571 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED; 2572 2573 clear_bit(__I40E_VSI_SYNCING_FILTERS, vsi->state); 2574 return retval; 2575 2576 err_no_memory: 2577 /* Restore elements on the temporary add and delete lists */ 2578 spin_lock_bh(&vsi->mac_filter_hash_lock); 2579 err_no_memory_locked: 2580 i40e_undo_del_filter_entries(vsi, &tmp_del_list); 2581 i40e_undo_add_filter_entries(vsi, &tmp_add_list); 2582 spin_unlock_bh(&vsi->mac_filter_hash_lock); 2583 2584 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED; 2585 clear_bit(__I40E_VSI_SYNCING_FILTERS, vsi->state); 2586 return -ENOMEM; 2587 } 2588 2589 /** 2590 * i40e_sync_filters_subtask - Sync the VSI filter list with HW 2591 * @pf: board private structure 2592 **/ 2593 static void i40e_sync_filters_subtask(struct i40e_pf *pf) 2594 { 2595 int v; 2596 2597 if (!pf || !(pf->flags & I40E_FLAG_FILTER_SYNC)) 2598 return; 2599 pf->flags &= ~I40E_FLAG_FILTER_SYNC; 2600 2601 for (v = 0; v < pf->num_alloc_vsi; v++) { 2602 if (pf->vsi[v] && 2603 (pf->vsi[v]->flags & I40E_VSI_FLAG_FILTER_CHANGED)) { 2604 int ret = i40e_sync_vsi_filters(pf->vsi[v]); 2605 2606 if (ret) { 2607 /* come back and try again later */ 2608 pf->flags |= I40E_FLAG_FILTER_SYNC; 2609 break; 2610 } 2611 } 2612 } 2613 } 2614 2615 /** 2616 * i40e_max_xdp_frame_size - returns the maximum allowed frame size for XDP 2617 * @vsi: the vsi 2618 **/ 2619 static int i40e_max_xdp_frame_size(struct i40e_vsi *vsi) 2620 { 2621 if (PAGE_SIZE >= 8192 || (vsi->back->flags & I40E_FLAG_LEGACY_RX)) 2622 return I40E_RXBUFFER_2048; 2623 else 2624 return I40E_RXBUFFER_3072; 2625 } 2626 2627 /** 2628 * i40e_change_mtu - NDO callback to change the Maximum Transfer Unit 2629 * @netdev: network interface device structure 2630 * @new_mtu: new value for maximum frame size 2631 * 2632 * Returns 0 on success, negative on failure 2633 **/ 2634 static int i40e_change_mtu(struct net_device *netdev, int new_mtu) 2635 { 2636 struct i40e_netdev_priv *np = netdev_priv(netdev); 2637 struct i40e_vsi *vsi = np->vsi; 2638 struct i40e_pf *pf = vsi->back; 2639 2640 if (i40e_enabled_xdp_vsi(vsi)) { 2641 int frame_size = new_mtu + ETH_HLEN + ETH_FCS_LEN + VLAN_HLEN; 2642 2643 if (frame_size > i40e_max_xdp_frame_size(vsi)) 2644 return -EINVAL; 2645 } 2646 2647 netdev_info(netdev, "changing MTU from %d to %d\n", 2648 netdev->mtu, new_mtu); 2649 netdev->mtu = new_mtu; 2650 if (netif_running(netdev)) 2651 i40e_vsi_reinit_locked(vsi); 2652 pf->flags |= (I40E_FLAG_SERVICE_CLIENT_REQUESTED | 2653 I40E_FLAG_CLIENT_L2_CHANGE); 2654 return 0; 2655 } 2656 2657 /** 2658 * i40e_ioctl - Access the hwtstamp interface 2659 * @netdev: network interface device structure 2660 * @ifr: interface request data 2661 * @cmd: ioctl command 2662 **/ 2663 int i40e_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd) 2664 { 2665 struct i40e_netdev_priv *np = netdev_priv(netdev); 2666 struct i40e_pf *pf = np->vsi->back; 2667 2668 switch (cmd) { 2669 case SIOCGHWTSTAMP: 2670 return i40e_ptp_get_ts_config(pf, ifr); 2671 case SIOCSHWTSTAMP: 2672 return i40e_ptp_set_ts_config(pf, ifr); 2673 default: 2674 return -EOPNOTSUPP; 2675 } 2676 } 2677 2678 /** 2679 * i40e_vlan_stripping_enable - Turn on vlan stripping for the VSI 2680 * @vsi: the vsi being adjusted 2681 **/ 2682 void i40e_vlan_stripping_enable(struct i40e_vsi *vsi) 2683 { 2684 struct i40e_vsi_context ctxt; 2685 i40e_status ret; 2686 2687 if ((vsi->info.valid_sections & 2688 cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID)) && 2689 ((vsi->info.port_vlan_flags & I40E_AQ_VSI_PVLAN_MODE_MASK) == 0)) 2690 return; /* already enabled */ 2691 2692 vsi->info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID); 2693 vsi->info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_ALL | 2694 I40E_AQ_VSI_PVLAN_EMOD_STR_BOTH; 2695 2696 ctxt.seid = vsi->seid; 2697 ctxt.info = vsi->info; 2698 ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL); 2699 if (ret) { 2700 dev_info(&vsi->back->pdev->dev, 2701 "update vlan stripping failed, err %s aq_err %s\n", 2702 i40e_stat_str(&vsi->back->hw, ret), 2703 i40e_aq_str(&vsi->back->hw, 2704 vsi->back->hw.aq.asq_last_status)); 2705 } 2706 } 2707 2708 /** 2709 * i40e_vlan_stripping_disable - Turn off vlan stripping for the VSI 2710 * @vsi: the vsi being adjusted 2711 **/ 2712 void i40e_vlan_stripping_disable(struct i40e_vsi *vsi) 2713 { 2714 struct i40e_vsi_context ctxt; 2715 i40e_status ret; 2716 2717 if ((vsi->info.valid_sections & 2718 cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID)) && 2719 ((vsi->info.port_vlan_flags & I40E_AQ_VSI_PVLAN_EMOD_MASK) == 2720 I40E_AQ_VSI_PVLAN_EMOD_MASK)) 2721 return; /* already disabled */ 2722 2723 vsi->info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID); 2724 vsi->info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_ALL | 2725 I40E_AQ_VSI_PVLAN_EMOD_NOTHING; 2726 2727 ctxt.seid = vsi->seid; 2728 ctxt.info = vsi->info; 2729 ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL); 2730 if (ret) { 2731 dev_info(&vsi->back->pdev->dev, 2732 "update vlan stripping failed, err %s aq_err %s\n", 2733 i40e_stat_str(&vsi->back->hw, ret), 2734 i40e_aq_str(&vsi->back->hw, 2735 vsi->back->hw.aq.asq_last_status)); 2736 } 2737 } 2738 2739 /** 2740 * i40e_vlan_rx_register - Setup or shutdown vlan offload 2741 * @netdev: network interface to be adjusted 2742 * @features: netdev features to test if VLAN offload is enabled or not 2743 **/ 2744 static void i40e_vlan_rx_register(struct net_device *netdev, u32 features) 2745 { 2746 struct i40e_netdev_priv *np = netdev_priv(netdev); 2747 struct i40e_vsi *vsi = np->vsi; 2748 2749 if (features & NETIF_F_HW_VLAN_CTAG_RX) 2750 i40e_vlan_stripping_enable(vsi); 2751 else 2752 i40e_vlan_stripping_disable(vsi); 2753 } 2754 2755 /** 2756 * i40e_add_vlan_all_mac - Add a MAC/VLAN filter for each existing MAC address 2757 * @vsi: the vsi being configured 2758 * @vid: vlan id to be added (0 = untagged only , -1 = any) 2759 * 2760 * This is a helper function for adding a new MAC/VLAN filter with the 2761 * specified VLAN for each existing MAC address already in the hash table. 2762 * This function does *not* perform any accounting to update filters based on 2763 * VLAN mode. 2764 * 2765 * NOTE: this function expects to be called while under the 2766 * mac_filter_hash_lock 2767 **/ 2768 int i40e_add_vlan_all_mac(struct i40e_vsi *vsi, s16 vid) 2769 { 2770 struct i40e_mac_filter *f, *add_f; 2771 struct hlist_node *h; 2772 int bkt; 2773 2774 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) { 2775 if (f->state == I40E_FILTER_REMOVE) 2776 continue; 2777 add_f = i40e_add_filter(vsi, f->macaddr, vid); 2778 if (!add_f) { 2779 dev_info(&vsi->back->pdev->dev, 2780 "Could not add vlan filter %d for %pM\n", 2781 vid, f->macaddr); 2782 return -ENOMEM; 2783 } 2784 } 2785 2786 return 0; 2787 } 2788 2789 /** 2790 * i40e_vsi_add_vlan - Add VSI membership for given VLAN 2791 * @vsi: the VSI being configured 2792 * @vid: VLAN id to be added 2793 **/ 2794 int i40e_vsi_add_vlan(struct i40e_vsi *vsi, u16 vid) 2795 { 2796 int err; 2797 2798 if (vsi->info.pvid) 2799 return -EINVAL; 2800 2801 /* The network stack will attempt to add VID=0, with the intention to 2802 * receive priority tagged packets with a VLAN of 0. Our HW receives 2803 * these packets by default when configured to receive untagged 2804 * packets, so we don't need to add a filter for this case. 2805 * Additionally, HW interprets adding a VID=0 filter as meaning to 2806 * receive *only* tagged traffic and stops receiving untagged traffic. 2807 * Thus, we do not want to actually add a filter for VID=0 2808 */ 2809 if (!vid) 2810 return 0; 2811 2812 /* Locked once because all functions invoked below iterates list*/ 2813 spin_lock_bh(&vsi->mac_filter_hash_lock); 2814 err = i40e_add_vlan_all_mac(vsi, vid); 2815 spin_unlock_bh(&vsi->mac_filter_hash_lock); 2816 if (err) 2817 return err; 2818 2819 /* schedule our worker thread which will take care of 2820 * applying the new filter changes 2821 */ 2822 i40e_service_event_schedule(vsi->back); 2823 return 0; 2824 } 2825 2826 /** 2827 * i40e_rm_vlan_all_mac - Remove MAC/VLAN pair for all MAC with the given VLAN 2828 * @vsi: the vsi being configured 2829 * @vid: vlan id to be removed (0 = untagged only , -1 = any) 2830 * 2831 * This function should be used to remove all VLAN filters which match the 2832 * given VID. It does not schedule the service event and does not take the 2833 * mac_filter_hash_lock so it may be combined with other operations under 2834 * a single invocation of the mac_filter_hash_lock. 2835 * 2836 * NOTE: this function expects to be called while under the 2837 * mac_filter_hash_lock 2838 */ 2839 void i40e_rm_vlan_all_mac(struct i40e_vsi *vsi, s16 vid) 2840 { 2841 struct i40e_mac_filter *f; 2842 struct hlist_node *h; 2843 int bkt; 2844 2845 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) { 2846 if (f->vlan == vid) 2847 __i40e_del_filter(vsi, f); 2848 } 2849 } 2850 2851 /** 2852 * i40e_vsi_kill_vlan - Remove VSI membership for given VLAN 2853 * @vsi: the VSI being configured 2854 * @vid: VLAN id to be removed 2855 **/ 2856 void i40e_vsi_kill_vlan(struct i40e_vsi *vsi, u16 vid) 2857 { 2858 if (!vid || vsi->info.pvid) 2859 return; 2860 2861 spin_lock_bh(&vsi->mac_filter_hash_lock); 2862 i40e_rm_vlan_all_mac(vsi, vid); 2863 spin_unlock_bh(&vsi->mac_filter_hash_lock); 2864 2865 /* schedule our worker thread which will take care of 2866 * applying the new filter changes 2867 */ 2868 i40e_service_event_schedule(vsi->back); 2869 } 2870 2871 /** 2872 * i40e_vlan_rx_add_vid - Add a vlan id filter to HW offload 2873 * @netdev: network interface to be adjusted 2874 * @vid: vlan id to be added 2875 * 2876 * net_device_ops implementation for adding vlan ids 2877 **/ 2878 static int i40e_vlan_rx_add_vid(struct net_device *netdev, 2879 __always_unused __be16 proto, u16 vid) 2880 { 2881 struct i40e_netdev_priv *np = netdev_priv(netdev); 2882 struct i40e_vsi *vsi = np->vsi; 2883 int ret = 0; 2884 2885 if (vid >= VLAN_N_VID) 2886 return -EINVAL; 2887 2888 ret = i40e_vsi_add_vlan(vsi, vid); 2889 if (!ret) 2890 set_bit(vid, vsi->active_vlans); 2891 2892 return ret; 2893 } 2894 2895 /** 2896 * i40e_vlan_rx_kill_vid - Remove a vlan id filter from HW offload 2897 * @netdev: network interface to be adjusted 2898 * @vid: vlan id to be removed 2899 * 2900 * net_device_ops implementation for removing vlan ids 2901 **/ 2902 static int i40e_vlan_rx_kill_vid(struct net_device *netdev, 2903 __always_unused __be16 proto, u16 vid) 2904 { 2905 struct i40e_netdev_priv *np = netdev_priv(netdev); 2906 struct i40e_vsi *vsi = np->vsi; 2907 2908 /* return code is ignored as there is nothing a user 2909 * can do about failure to remove and a log message was 2910 * already printed from the other function 2911 */ 2912 i40e_vsi_kill_vlan(vsi, vid); 2913 2914 clear_bit(vid, vsi->active_vlans); 2915 2916 return 0; 2917 } 2918 2919 /** 2920 * i40e_restore_vlan - Reinstate vlans when vsi/netdev comes back up 2921 * @vsi: the vsi being brought back up 2922 **/ 2923 static void i40e_restore_vlan(struct i40e_vsi *vsi) 2924 { 2925 u16 vid; 2926 2927 if (!vsi->netdev) 2928 return; 2929 2930 i40e_vlan_rx_register(vsi->netdev, vsi->netdev->features); 2931 2932 for_each_set_bit(vid, vsi->active_vlans, VLAN_N_VID) 2933 i40e_vlan_rx_add_vid(vsi->netdev, htons(ETH_P_8021Q), 2934 vid); 2935 } 2936 2937 /** 2938 * i40e_vsi_add_pvid - Add pvid for the VSI 2939 * @vsi: the vsi being adjusted 2940 * @vid: the vlan id to set as a PVID 2941 **/ 2942 int i40e_vsi_add_pvid(struct i40e_vsi *vsi, u16 vid) 2943 { 2944 struct i40e_vsi_context ctxt; 2945 i40e_status ret; 2946 2947 vsi->info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID); 2948 vsi->info.pvid = cpu_to_le16(vid); 2949 vsi->info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_TAGGED | 2950 I40E_AQ_VSI_PVLAN_INSERT_PVID | 2951 I40E_AQ_VSI_PVLAN_EMOD_STR; 2952 2953 ctxt.seid = vsi->seid; 2954 ctxt.info = vsi->info; 2955 ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL); 2956 if (ret) { 2957 dev_info(&vsi->back->pdev->dev, 2958 "add pvid failed, err %s aq_err %s\n", 2959 i40e_stat_str(&vsi->back->hw, ret), 2960 i40e_aq_str(&vsi->back->hw, 2961 vsi->back->hw.aq.asq_last_status)); 2962 return -ENOENT; 2963 } 2964 2965 return 0; 2966 } 2967 2968 /** 2969 * i40e_vsi_remove_pvid - Remove the pvid from the VSI 2970 * @vsi: the vsi being adjusted 2971 * 2972 * Just use the vlan_rx_register() service to put it back to normal 2973 **/ 2974 void i40e_vsi_remove_pvid(struct i40e_vsi *vsi) 2975 { 2976 i40e_vlan_stripping_disable(vsi); 2977 2978 vsi->info.pvid = 0; 2979 } 2980 2981 /** 2982 * i40e_vsi_setup_tx_resources - Allocate VSI Tx queue resources 2983 * @vsi: ptr to the VSI 2984 * 2985 * If this function returns with an error, then it's possible one or 2986 * more of the rings is populated (while the rest are not). It is the 2987 * callers duty to clean those orphaned rings. 2988 * 2989 * Return 0 on success, negative on failure 2990 **/ 2991 static int i40e_vsi_setup_tx_resources(struct i40e_vsi *vsi) 2992 { 2993 int i, err = 0; 2994 2995 for (i = 0; i < vsi->num_queue_pairs && !err; i++) 2996 err = i40e_setup_tx_descriptors(vsi->tx_rings[i]); 2997 2998 if (!i40e_enabled_xdp_vsi(vsi)) 2999 return err; 3000 3001 for (i = 0; i < vsi->num_queue_pairs && !err; i++) 3002 err = i40e_setup_tx_descriptors(vsi->xdp_rings[i]); 3003 3004 return err; 3005 } 3006 3007 /** 3008 * i40e_vsi_free_tx_resources - Free Tx resources for VSI queues 3009 * @vsi: ptr to the VSI 3010 * 3011 * Free VSI's transmit software resources 3012 **/ 3013 static void i40e_vsi_free_tx_resources(struct i40e_vsi *vsi) 3014 { 3015 int i; 3016 3017 if (vsi->tx_rings) { 3018 for (i = 0; i < vsi->num_queue_pairs; i++) 3019 if (vsi->tx_rings[i] && vsi->tx_rings[i]->desc) 3020 i40e_free_tx_resources(vsi->tx_rings[i]); 3021 } 3022 3023 if (vsi->xdp_rings) { 3024 for (i = 0; i < vsi->num_queue_pairs; i++) 3025 if (vsi->xdp_rings[i] && vsi->xdp_rings[i]->desc) 3026 i40e_free_tx_resources(vsi->xdp_rings[i]); 3027 } 3028 } 3029 3030 /** 3031 * i40e_vsi_setup_rx_resources - Allocate VSI queues Rx resources 3032 * @vsi: ptr to the VSI 3033 * 3034 * If this function returns with an error, then it's possible one or 3035 * more of the rings is populated (while the rest are not). It is the 3036 * callers duty to clean those orphaned rings. 3037 * 3038 * Return 0 on success, negative on failure 3039 **/ 3040 static int i40e_vsi_setup_rx_resources(struct i40e_vsi *vsi) 3041 { 3042 int i, err = 0; 3043 3044 for (i = 0; i < vsi->num_queue_pairs && !err; i++) 3045 err = i40e_setup_rx_descriptors(vsi->rx_rings[i]); 3046 return err; 3047 } 3048 3049 /** 3050 * i40e_vsi_free_rx_resources - Free Rx Resources for VSI queues 3051 * @vsi: ptr to the VSI 3052 * 3053 * Free all receive software resources 3054 **/ 3055 static void i40e_vsi_free_rx_resources(struct i40e_vsi *vsi) 3056 { 3057 int i; 3058 3059 if (!vsi->rx_rings) 3060 return; 3061 3062 for (i = 0; i < vsi->num_queue_pairs; i++) 3063 if (vsi->rx_rings[i] && vsi->rx_rings[i]->desc) 3064 i40e_free_rx_resources(vsi->rx_rings[i]); 3065 } 3066 3067 /** 3068 * i40e_config_xps_tx_ring - Configure XPS for a Tx ring 3069 * @ring: The Tx ring to configure 3070 * 3071 * This enables/disables XPS for a given Tx descriptor ring 3072 * based on the TCs enabled for the VSI that ring belongs to. 3073 **/ 3074 static void i40e_config_xps_tx_ring(struct i40e_ring *ring) 3075 { 3076 int cpu; 3077 3078 if (!ring->q_vector || !ring->netdev || ring->ch) 3079 return; 3080 3081 /* We only initialize XPS once, so as not to overwrite user settings */ 3082 if (test_and_set_bit(__I40E_TX_XPS_INIT_DONE, ring->state)) 3083 return; 3084 3085 cpu = cpumask_local_spread(ring->q_vector->v_idx, -1); 3086 netif_set_xps_queue(ring->netdev, get_cpu_mask(cpu), 3087 ring->queue_index); 3088 } 3089 3090 /** 3091 * i40e_configure_tx_ring - Configure a transmit ring context and rest 3092 * @ring: The Tx ring to configure 3093 * 3094 * Configure the Tx descriptor ring in the HMC context. 3095 **/ 3096 static int i40e_configure_tx_ring(struct i40e_ring *ring) 3097 { 3098 struct i40e_vsi *vsi = ring->vsi; 3099 u16 pf_q = vsi->base_queue + ring->queue_index; 3100 struct i40e_hw *hw = &vsi->back->hw; 3101 struct i40e_hmc_obj_txq tx_ctx; 3102 i40e_status err = 0; 3103 u32 qtx_ctl = 0; 3104 3105 /* some ATR related tx ring init */ 3106 if (vsi->back->flags & I40E_FLAG_FD_ATR_ENABLED) { 3107 ring->atr_sample_rate = vsi->back->atr_sample_rate; 3108 ring->atr_count = 0; 3109 } else { 3110 ring->atr_sample_rate = 0; 3111 } 3112 3113 /* configure XPS */ 3114 i40e_config_xps_tx_ring(ring); 3115 3116 /* clear the context structure first */ 3117 memset(&tx_ctx, 0, sizeof(tx_ctx)); 3118 3119 tx_ctx.new_context = 1; 3120 tx_ctx.base = (ring->dma / 128); 3121 tx_ctx.qlen = ring->count; 3122 tx_ctx.fd_ena = !!(vsi->back->flags & (I40E_FLAG_FD_SB_ENABLED | 3123 I40E_FLAG_FD_ATR_ENABLED)); 3124 tx_ctx.timesync_ena = !!(vsi->back->flags & I40E_FLAG_PTP); 3125 /* FDIR VSI tx ring can still use RS bit and writebacks */ 3126 if (vsi->type != I40E_VSI_FDIR) 3127 tx_ctx.head_wb_ena = 1; 3128 tx_ctx.head_wb_addr = ring->dma + 3129 (ring->count * sizeof(struct i40e_tx_desc)); 3130 3131 /* As part of VSI creation/update, FW allocates certain 3132 * Tx arbitration queue sets for each TC enabled for 3133 * the VSI. The FW returns the handles to these queue 3134 * sets as part of the response buffer to Add VSI, 3135 * Update VSI, etc. AQ commands. It is expected that 3136 * these queue set handles be associated with the Tx 3137 * queues by the driver as part of the TX queue context 3138 * initialization. This has to be done regardless of 3139 * DCB as by default everything is mapped to TC0. 3140 */ 3141 3142 if (ring->ch) 3143 tx_ctx.rdylist = 3144 le16_to_cpu(ring->ch->info.qs_handle[ring->dcb_tc]); 3145 3146 else 3147 tx_ctx.rdylist = le16_to_cpu(vsi->info.qs_handle[ring->dcb_tc]); 3148 3149 tx_ctx.rdylist_act = 0; 3150 3151 /* clear the context in the HMC */ 3152 err = i40e_clear_lan_tx_queue_context(hw, pf_q); 3153 if (err) { 3154 dev_info(&vsi->back->pdev->dev, 3155 "Failed to clear LAN Tx queue context on Tx ring %d (pf_q %d), error: %d\n", 3156 ring->queue_index, pf_q, err); 3157 return -ENOMEM; 3158 } 3159 3160 /* set the context in the HMC */ 3161 err = i40e_set_lan_tx_queue_context(hw, pf_q, &tx_ctx); 3162 if (err) { 3163 dev_info(&vsi->back->pdev->dev, 3164 "Failed to set LAN Tx queue context on Tx ring %d (pf_q %d, error: %d\n", 3165 ring->queue_index, pf_q, err); 3166 return -ENOMEM; 3167 } 3168 3169 /* Now associate this queue with this PCI function */ 3170 if (ring->ch) { 3171 if (ring->ch->type == I40E_VSI_VMDQ2) 3172 qtx_ctl = I40E_QTX_CTL_VM_QUEUE; 3173 else 3174 return -EINVAL; 3175 3176 qtx_ctl |= (ring->ch->vsi_number << 3177 I40E_QTX_CTL_VFVM_INDX_SHIFT) & 3178 I40E_QTX_CTL_VFVM_INDX_MASK; 3179 } else { 3180 if (vsi->type == I40E_VSI_VMDQ2) { 3181 qtx_ctl = I40E_QTX_CTL_VM_QUEUE; 3182 qtx_ctl |= ((vsi->id) << I40E_QTX_CTL_VFVM_INDX_SHIFT) & 3183 I40E_QTX_CTL_VFVM_INDX_MASK; 3184 } else { 3185 qtx_ctl = I40E_QTX_CTL_PF_QUEUE; 3186 } 3187 } 3188 3189 qtx_ctl |= ((hw->pf_id << I40E_QTX_CTL_PF_INDX_SHIFT) & 3190 I40E_QTX_CTL_PF_INDX_MASK); 3191 wr32(hw, I40E_QTX_CTL(pf_q), qtx_ctl); 3192 i40e_flush(hw); 3193 3194 /* cache tail off for easier writes later */ 3195 ring->tail = hw->hw_addr + I40E_QTX_TAIL(pf_q); 3196 3197 return 0; 3198 } 3199 3200 /** 3201 * i40e_configure_rx_ring - Configure a receive ring context 3202 * @ring: The Rx ring to configure 3203 * 3204 * Configure the Rx descriptor ring in the HMC context. 3205 **/ 3206 static int i40e_configure_rx_ring(struct i40e_ring *ring) 3207 { 3208 struct i40e_vsi *vsi = ring->vsi; 3209 u32 chain_len = vsi->back->hw.func_caps.rx_buf_chain_len; 3210 u16 pf_q = vsi->base_queue + ring->queue_index; 3211 struct i40e_hw *hw = &vsi->back->hw; 3212 struct i40e_hmc_obj_rxq rx_ctx; 3213 i40e_status err = 0; 3214 3215 bitmap_zero(ring->state, __I40E_RING_STATE_NBITS); 3216 3217 /* clear the context structure first */ 3218 memset(&rx_ctx, 0, sizeof(rx_ctx)); 3219 3220 ring->rx_buf_len = vsi->rx_buf_len; 3221 3222 rx_ctx.dbuff = DIV_ROUND_UP(ring->rx_buf_len, 3223 BIT_ULL(I40E_RXQ_CTX_DBUFF_SHIFT)); 3224 3225 rx_ctx.base = (ring->dma / 128); 3226 rx_ctx.qlen = ring->count; 3227 3228 /* use 32 byte descriptors */ 3229 rx_ctx.dsize = 1; 3230 3231 /* descriptor type is always zero 3232 * rx_ctx.dtype = 0; 3233 */ 3234 rx_ctx.hsplit_0 = 0; 3235 3236 rx_ctx.rxmax = min_t(u16, vsi->max_frame, chain_len * ring->rx_buf_len); 3237 if (hw->revision_id == 0) 3238 rx_ctx.lrxqthresh = 0; 3239 else 3240 rx_ctx.lrxqthresh = 1; 3241 rx_ctx.crcstrip = 1; 3242 rx_ctx.l2tsel = 1; 3243 /* this controls whether VLAN is stripped from inner headers */ 3244 rx_ctx.showiv = 0; 3245 /* set the prefena field to 1 because the manual says to */ 3246 rx_ctx.prefena = 1; 3247 3248 /* clear the context in the HMC */ 3249 err = i40e_clear_lan_rx_queue_context(hw, pf_q); 3250 if (err) { 3251 dev_info(&vsi->back->pdev->dev, 3252 "Failed to clear LAN Rx queue context on Rx ring %d (pf_q %d), error: %d\n", 3253 ring->queue_index, pf_q, err); 3254 return -ENOMEM; 3255 } 3256 3257 /* set the context in the HMC */ 3258 err = i40e_set_lan_rx_queue_context(hw, pf_q, &rx_ctx); 3259 if (err) { 3260 dev_info(&vsi->back->pdev->dev, 3261 "Failed to set LAN Rx queue context on Rx ring %d (pf_q %d), error: %d\n", 3262 ring->queue_index, pf_q, err); 3263 return -ENOMEM; 3264 } 3265 3266 /* configure Rx buffer alignment */ 3267 if (!vsi->netdev || (vsi->back->flags & I40E_FLAG_LEGACY_RX)) 3268 clear_ring_build_skb_enabled(ring); 3269 else 3270 set_ring_build_skb_enabled(ring); 3271 3272 /* cache tail for quicker writes, and clear the reg before use */ 3273 ring->tail = hw->hw_addr + I40E_QRX_TAIL(pf_q); 3274 writel(0, ring->tail); 3275 3276 i40e_alloc_rx_buffers(ring, I40E_DESC_UNUSED(ring)); 3277 3278 return 0; 3279 } 3280 3281 /** 3282 * i40e_vsi_configure_tx - Configure the VSI for Tx 3283 * @vsi: VSI structure describing this set of rings and resources 3284 * 3285 * Configure the Tx VSI for operation. 3286 **/ 3287 static int i40e_vsi_configure_tx(struct i40e_vsi *vsi) 3288 { 3289 int err = 0; 3290 u16 i; 3291 3292 for (i = 0; (i < vsi->num_queue_pairs) && !err; i++) 3293 err = i40e_configure_tx_ring(vsi->tx_rings[i]); 3294 3295 if (!i40e_enabled_xdp_vsi(vsi)) 3296 return err; 3297 3298 for (i = 0; (i < vsi->num_queue_pairs) && !err; i++) 3299 err = i40e_configure_tx_ring(vsi->xdp_rings[i]); 3300 3301 return err; 3302 } 3303 3304 /** 3305 * i40e_vsi_configure_rx - Configure the VSI for Rx 3306 * @vsi: the VSI being configured 3307 * 3308 * Configure the Rx VSI for operation. 3309 **/ 3310 static int i40e_vsi_configure_rx(struct i40e_vsi *vsi) 3311 { 3312 int err = 0; 3313 u16 i; 3314 3315 if (!vsi->netdev || (vsi->back->flags & I40E_FLAG_LEGACY_RX)) { 3316 vsi->max_frame = I40E_MAX_RXBUFFER; 3317 vsi->rx_buf_len = I40E_RXBUFFER_2048; 3318 #if (PAGE_SIZE < 8192) 3319 } else if (!I40E_2K_TOO_SMALL_WITH_PADDING && 3320 (vsi->netdev->mtu <= ETH_DATA_LEN)) { 3321 vsi->max_frame = I40E_RXBUFFER_1536 - NET_IP_ALIGN; 3322 vsi->rx_buf_len = I40E_RXBUFFER_1536 - NET_IP_ALIGN; 3323 #endif 3324 } else { 3325 vsi->max_frame = I40E_MAX_RXBUFFER; 3326 vsi->rx_buf_len = (PAGE_SIZE < 8192) ? I40E_RXBUFFER_3072 : 3327 I40E_RXBUFFER_2048; 3328 } 3329 3330 /* set up individual rings */ 3331 for (i = 0; i < vsi->num_queue_pairs && !err; i++) 3332 err = i40e_configure_rx_ring(vsi->rx_rings[i]); 3333 3334 return err; 3335 } 3336 3337 /** 3338 * i40e_vsi_config_dcb_rings - Update rings to reflect DCB TC 3339 * @vsi: ptr to the VSI 3340 **/ 3341 static void i40e_vsi_config_dcb_rings(struct i40e_vsi *vsi) 3342 { 3343 struct i40e_ring *tx_ring, *rx_ring; 3344 u16 qoffset, qcount; 3345 int i, n; 3346 3347 if (!(vsi->back->flags & I40E_FLAG_DCB_ENABLED)) { 3348 /* Reset the TC information */ 3349 for (i = 0; i < vsi->num_queue_pairs; i++) { 3350 rx_ring = vsi->rx_rings[i]; 3351 tx_ring = vsi->tx_rings[i]; 3352 rx_ring->dcb_tc = 0; 3353 tx_ring->dcb_tc = 0; 3354 } 3355 return; 3356 } 3357 3358 for (n = 0; n < I40E_MAX_TRAFFIC_CLASS; n++) { 3359 if (!(vsi->tc_config.enabled_tc & BIT_ULL(n))) 3360 continue; 3361 3362 qoffset = vsi->tc_config.tc_info[n].qoffset; 3363 qcount = vsi->tc_config.tc_info[n].qcount; 3364 for (i = qoffset; i < (qoffset + qcount); i++) { 3365 rx_ring = vsi->rx_rings[i]; 3366 tx_ring = vsi->tx_rings[i]; 3367 rx_ring->dcb_tc = n; 3368 tx_ring->dcb_tc = n; 3369 } 3370 } 3371 } 3372 3373 /** 3374 * i40e_set_vsi_rx_mode - Call set_rx_mode on a VSI 3375 * @vsi: ptr to the VSI 3376 **/ 3377 static void i40e_set_vsi_rx_mode(struct i40e_vsi *vsi) 3378 { 3379 if (vsi->netdev) 3380 i40e_set_rx_mode(vsi->netdev); 3381 } 3382 3383 /** 3384 * i40e_fdir_filter_restore - Restore the Sideband Flow Director filters 3385 * @vsi: Pointer to the targeted VSI 3386 * 3387 * This function replays the hlist on the hw where all the SB Flow Director 3388 * filters were saved. 3389 **/ 3390 static void i40e_fdir_filter_restore(struct i40e_vsi *vsi) 3391 { 3392 struct i40e_fdir_filter *filter; 3393 struct i40e_pf *pf = vsi->back; 3394 struct hlist_node *node; 3395 3396 if (!(pf->flags & I40E_FLAG_FD_SB_ENABLED)) 3397 return; 3398 3399 /* Reset FDir counters as we're replaying all existing filters */ 3400 pf->fd_tcp4_filter_cnt = 0; 3401 pf->fd_udp4_filter_cnt = 0; 3402 pf->fd_sctp4_filter_cnt = 0; 3403 pf->fd_ip4_filter_cnt = 0; 3404 3405 hlist_for_each_entry_safe(filter, node, 3406 &pf->fdir_filter_list, fdir_node) { 3407 i40e_add_del_fdir(vsi, filter, true); 3408 } 3409 } 3410 3411 /** 3412 * i40e_vsi_configure - Set up the VSI for action 3413 * @vsi: the VSI being configured 3414 **/ 3415 static int i40e_vsi_configure(struct i40e_vsi *vsi) 3416 { 3417 int err; 3418 3419 i40e_set_vsi_rx_mode(vsi); 3420 i40e_restore_vlan(vsi); 3421 i40e_vsi_config_dcb_rings(vsi); 3422 err = i40e_vsi_configure_tx(vsi); 3423 if (!err) 3424 err = i40e_vsi_configure_rx(vsi); 3425 3426 return err; 3427 } 3428 3429 /** 3430 * i40e_vsi_configure_msix - MSIX mode Interrupt Config in the HW 3431 * @vsi: the VSI being configured 3432 **/ 3433 static void i40e_vsi_configure_msix(struct i40e_vsi *vsi) 3434 { 3435 bool has_xdp = i40e_enabled_xdp_vsi(vsi); 3436 struct i40e_pf *pf = vsi->back; 3437 struct i40e_hw *hw = &pf->hw; 3438 u16 vector; 3439 int i, q; 3440 u32 qp; 3441 3442 /* The interrupt indexing is offset by 1 in the PFINT_ITRn 3443 * and PFINT_LNKLSTn registers, e.g.: 3444 * PFINT_ITRn[0..n-1] gets msix-1..msix-n (qpair interrupts) 3445 */ 3446 qp = vsi->base_queue; 3447 vector = vsi->base_vector; 3448 for (i = 0; i < vsi->num_q_vectors; i++, vector++) { 3449 struct i40e_q_vector *q_vector = vsi->q_vectors[i]; 3450 3451 q_vector->itr_countdown = ITR_COUNTDOWN_START; 3452 q_vector->rx.itr = ITR_TO_REG(vsi->rx_rings[i]->rx_itr_setting); 3453 q_vector->rx.latency_range = I40E_LOW_LATENCY; 3454 wr32(hw, I40E_PFINT_ITRN(I40E_RX_ITR, vector - 1), 3455 q_vector->rx.itr); 3456 q_vector->tx.itr = ITR_TO_REG(vsi->tx_rings[i]->tx_itr_setting); 3457 q_vector->tx.latency_range = I40E_LOW_LATENCY; 3458 wr32(hw, I40E_PFINT_ITRN(I40E_TX_ITR, vector - 1), 3459 q_vector->tx.itr); 3460 wr32(hw, I40E_PFINT_RATEN(vector - 1), 3461 i40e_intrl_usec_to_reg(vsi->int_rate_limit)); 3462 3463 /* Linked list for the queuepairs assigned to this vector */ 3464 wr32(hw, I40E_PFINT_LNKLSTN(vector - 1), qp); 3465 for (q = 0; q < q_vector->num_ringpairs; q++) { 3466 u32 nextqp = has_xdp ? qp + vsi->alloc_queue_pairs : qp; 3467 u32 val; 3468 3469 val = I40E_QINT_RQCTL_CAUSE_ENA_MASK | 3470 (I40E_RX_ITR << I40E_QINT_RQCTL_ITR_INDX_SHIFT) | 3471 (vector << I40E_QINT_RQCTL_MSIX_INDX_SHIFT) | 3472 (nextqp << I40E_QINT_RQCTL_NEXTQ_INDX_SHIFT) | 3473 (I40E_QUEUE_TYPE_TX << 3474 I40E_QINT_RQCTL_NEXTQ_TYPE_SHIFT); 3475 3476 wr32(hw, I40E_QINT_RQCTL(qp), val); 3477 3478 if (has_xdp) { 3479 val = I40E_QINT_TQCTL_CAUSE_ENA_MASK | 3480 (I40E_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT) | 3481 (vector << I40E_QINT_TQCTL_MSIX_INDX_SHIFT) | 3482 (qp << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT) | 3483 (I40E_QUEUE_TYPE_TX << 3484 I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT); 3485 3486 wr32(hw, I40E_QINT_TQCTL(nextqp), val); 3487 } 3488 3489 val = I40E_QINT_TQCTL_CAUSE_ENA_MASK | 3490 (I40E_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT) | 3491 (vector << I40E_QINT_TQCTL_MSIX_INDX_SHIFT) | 3492 ((qp + 1) << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT) | 3493 (I40E_QUEUE_TYPE_RX << 3494 I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT); 3495 3496 /* Terminate the linked list */ 3497 if (q == (q_vector->num_ringpairs - 1)) 3498 val |= (I40E_QUEUE_END_OF_LIST << 3499 I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT); 3500 3501 wr32(hw, I40E_QINT_TQCTL(qp), val); 3502 qp++; 3503 } 3504 } 3505 3506 i40e_flush(hw); 3507 } 3508 3509 /** 3510 * i40e_enable_misc_int_causes - enable the non-queue interrupts 3511 * @hw: ptr to the hardware info 3512 **/ 3513 static void i40e_enable_misc_int_causes(struct i40e_pf *pf) 3514 { 3515 struct i40e_hw *hw = &pf->hw; 3516 u32 val; 3517 3518 /* clear things first */ 3519 wr32(hw, I40E_PFINT_ICR0_ENA, 0); /* disable all */ 3520 rd32(hw, I40E_PFINT_ICR0); /* read to clear */ 3521 3522 val = I40E_PFINT_ICR0_ENA_ECC_ERR_MASK | 3523 I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK | 3524 I40E_PFINT_ICR0_ENA_GRST_MASK | 3525 I40E_PFINT_ICR0_ENA_PCI_EXCEPTION_MASK | 3526 I40E_PFINT_ICR0_ENA_GPIO_MASK | 3527 I40E_PFINT_ICR0_ENA_HMC_ERR_MASK | 3528 I40E_PFINT_ICR0_ENA_VFLR_MASK | 3529 I40E_PFINT_ICR0_ENA_ADMINQ_MASK; 3530 3531 if (pf->flags & I40E_FLAG_IWARP_ENABLED) 3532 val |= I40E_PFINT_ICR0_ENA_PE_CRITERR_MASK; 3533 3534 if (pf->flags & I40E_FLAG_PTP) 3535 val |= I40E_PFINT_ICR0_ENA_TIMESYNC_MASK; 3536 3537 wr32(hw, I40E_PFINT_ICR0_ENA, val); 3538 3539 /* SW_ITR_IDX = 0, but don't change INTENA */ 3540 wr32(hw, I40E_PFINT_DYN_CTL0, I40E_PFINT_DYN_CTL0_SW_ITR_INDX_MASK | 3541 I40E_PFINT_DYN_CTL0_INTENA_MSK_MASK); 3542 3543 /* OTHER_ITR_IDX = 0 */ 3544 wr32(hw, I40E_PFINT_STAT_CTL0, 0); 3545 } 3546 3547 /** 3548 * i40e_configure_msi_and_legacy - Legacy mode interrupt config in the HW 3549 * @vsi: the VSI being configured 3550 **/ 3551 static void i40e_configure_msi_and_legacy(struct i40e_vsi *vsi) 3552 { 3553 u32 nextqp = i40e_enabled_xdp_vsi(vsi) ? vsi->alloc_queue_pairs : 0; 3554 struct i40e_q_vector *q_vector = vsi->q_vectors[0]; 3555 struct i40e_pf *pf = vsi->back; 3556 struct i40e_hw *hw = &pf->hw; 3557 u32 val; 3558 3559 /* set the ITR configuration */ 3560 q_vector->itr_countdown = ITR_COUNTDOWN_START; 3561 q_vector->rx.itr = ITR_TO_REG(vsi->rx_rings[0]->rx_itr_setting); 3562 q_vector->rx.latency_range = I40E_LOW_LATENCY; 3563 wr32(hw, I40E_PFINT_ITR0(I40E_RX_ITR), q_vector->rx.itr); 3564 q_vector->tx.itr = ITR_TO_REG(vsi->tx_rings[0]->tx_itr_setting); 3565 q_vector->tx.latency_range = I40E_LOW_LATENCY; 3566 wr32(hw, I40E_PFINT_ITR0(I40E_TX_ITR), q_vector->tx.itr); 3567 3568 i40e_enable_misc_int_causes(pf); 3569 3570 /* FIRSTQ_INDX = 0, FIRSTQ_TYPE = 0 (rx) */ 3571 wr32(hw, I40E_PFINT_LNKLST0, 0); 3572 3573 /* Associate the queue pair to the vector and enable the queue int */ 3574 val = I40E_QINT_RQCTL_CAUSE_ENA_MASK | 3575 (I40E_RX_ITR << I40E_QINT_RQCTL_ITR_INDX_SHIFT) | 3576 (nextqp << I40E_QINT_RQCTL_NEXTQ_INDX_SHIFT)| 3577 (I40E_QUEUE_TYPE_TX << I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT); 3578 3579 wr32(hw, I40E_QINT_RQCTL(0), val); 3580 3581 if (i40e_enabled_xdp_vsi(vsi)) { 3582 val = I40E_QINT_TQCTL_CAUSE_ENA_MASK | 3583 (I40E_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT)| 3584 (I40E_QUEUE_TYPE_TX 3585 << I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT); 3586 3587 wr32(hw, I40E_QINT_TQCTL(nextqp), val); 3588 } 3589 3590 val = I40E_QINT_TQCTL_CAUSE_ENA_MASK | 3591 (I40E_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT) | 3592 (I40E_QUEUE_END_OF_LIST << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT); 3593 3594 wr32(hw, I40E_QINT_TQCTL(0), val); 3595 i40e_flush(hw); 3596 } 3597 3598 /** 3599 * i40e_irq_dynamic_disable_icr0 - Disable default interrupt generation for icr0 3600 * @pf: board private structure 3601 **/ 3602 void i40e_irq_dynamic_disable_icr0(struct i40e_pf *pf) 3603 { 3604 struct i40e_hw *hw = &pf->hw; 3605 3606 wr32(hw, I40E_PFINT_DYN_CTL0, 3607 I40E_ITR_NONE << I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT); 3608 i40e_flush(hw); 3609 } 3610 3611 /** 3612 * i40e_irq_dynamic_enable_icr0 - Enable default interrupt generation for icr0 3613 * @pf: board private structure 3614 **/ 3615 void i40e_irq_dynamic_enable_icr0(struct i40e_pf *pf) 3616 { 3617 struct i40e_hw *hw = &pf->hw; 3618 u32 val; 3619 3620 val = I40E_PFINT_DYN_CTL0_INTENA_MASK | 3621 I40E_PFINT_DYN_CTL0_CLEARPBA_MASK | 3622 (I40E_ITR_NONE << I40E_PFINT_DYN_CTL0_ITR_INDX_SHIFT); 3623 3624 wr32(hw, I40E_PFINT_DYN_CTL0, val); 3625 i40e_flush(hw); 3626 } 3627 3628 /** 3629 * i40e_msix_clean_rings - MSIX mode Interrupt Handler 3630 * @irq: interrupt number 3631 * @data: pointer to a q_vector 3632 **/ 3633 static irqreturn_t i40e_msix_clean_rings(int irq, void *data) 3634 { 3635 struct i40e_q_vector *q_vector = data; 3636 3637 if (!q_vector->tx.ring && !q_vector->rx.ring) 3638 return IRQ_HANDLED; 3639 3640 napi_schedule_irqoff(&q_vector->napi); 3641 3642 return IRQ_HANDLED; 3643 } 3644 3645 /** 3646 * i40e_irq_affinity_notify - Callback for affinity changes 3647 * @notify: context as to what irq was changed 3648 * @mask: the new affinity mask 3649 * 3650 * This is a callback function used by the irq_set_affinity_notifier function 3651 * so that we may register to receive changes to the irq affinity masks. 3652 **/ 3653 static void i40e_irq_affinity_notify(struct irq_affinity_notify *notify, 3654 const cpumask_t *mask) 3655 { 3656 struct i40e_q_vector *q_vector = 3657 container_of(notify, struct i40e_q_vector, affinity_notify); 3658 3659 cpumask_copy(&q_vector->affinity_mask, mask); 3660 } 3661 3662 /** 3663 * i40e_irq_affinity_release - Callback for affinity notifier release 3664 * @ref: internal core kernel usage 3665 * 3666 * This is a callback function used by the irq_set_affinity_notifier function 3667 * to inform the current notification subscriber that they will no longer 3668 * receive notifications. 3669 **/ 3670 static void i40e_irq_affinity_release(struct kref *ref) {} 3671 3672 /** 3673 * i40e_vsi_request_irq_msix - Initialize MSI-X interrupts 3674 * @vsi: the VSI being configured 3675 * @basename: name for the vector 3676 * 3677 * Allocates MSI-X vectors and requests interrupts from the kernel. 3678 **/ 3679 static int i40e_vsi_request_irq_msix(struct i40e_vsi *vsi, char *basename) 3680 { 3681 int q_vectors = vsi->num_q_vectors; 3682 struct i40e_pf *pf = vsi->back; 3683 int base = vsi->base_vector; 3684 int rx_int_idx = 0; 3685 int tx_int_idx = 0; 3686 int vector, err; 3687 int irq_num; 3688 int cpu; 3689 3690 for (vector = 0; vector < q_vectors; vector++) { 3691 struct i40e_q_vector *q_vector = vsi->q_vectors[vector]; 3692 3693 irq_num = pf->msix_entries[base + vector].vector; 3694 3695 if (q_vector->tx.ring && q_vector->rx.ring) { 3696 snprintf(q_vector->name, sizeof(q_vector->name) - 1, 3697 "%s-%s-%d", basename, "TxRx", rx_int_idx++); 3698 tx_int_idx++; 3699 } else if (q_vector->rx.ring) { 3700 snprintf(q_vector->name, sizeof(q_vector->name) - 1, 3701 "%s-%s-%d", basename, "rx", rx_int_idx++); 3702 } else if (q_vector->tx.ring) { 3703 snprintf(q_vector->name, sizeof(q_vector->name) - 1, 3704 "%s-%s-%d", basename, "tx", tx_int_idx++); 3705 } else { 3706 /* skip this unused q_vector */ 3707 continue; 3708 } 3709 err = request_irq(irq_num, 3710 vsi->irq_handler, 3711 0, 3712 q_vector->name, 3713 q_vector); 3714 if (err) { 3715 dev_info(&pf->pdev->dev, 3716 "MSIX request_irq failed, error: %d\n", err); 3717 goto free_queue_irqs; 3718 } 3719 3720 /* register for affinity change notifications */ 3721 q_vector->affinity_notify.notify = i40e_irq_affinity_notify; 3722 q_vector->affinity_notify.release = i40e_irq_affinity_release; 3723 irq_set_affinity_notifier(irq_num, &q_vector->affinity_notify); 3724 /* Spread affinity hints out across online CPUs. 3725 * 3726 * get_cpu_mask returns a static constant mask with 3727 * a permanent lifetime so it's ok to pass to 3728 * irq_set_affinity_hint without making a copy. 3729 */ 3730 cpu = cpumask_local_spread(q_vector->v_idx, -1); 3731 irq_set_affinity_hint(irq_num, get_cpu_mask(cpu)); 3732 } 3733 3734 vsi->irqs_ready = true; 3735 return 0; 3736 3737 free_queue_irqs: 3738 while (vector) { 3739 vector--; 3740 irq_num = pf->msix_entries[base + vector].vector; 3741 irq_set_affinity_notifier(irq_num, NULL); 3742 irq_set_affinity_hint(irq_num, NULL); 3743 free_irq(irq_num, &vsi->q_vectors[vector]); 3744 } 3745 return err; 3746 } 3747 3748 /** 3749 * i40e_vsi_disable_irq - Mask off queue interrupt generation on the VSI 3750 * @vsi: the VSI being un-configured 3751 **/ 3752 static void i40e_vsi_disable_irq(struct i40e_vsi *vsi) 3753 { 3754 struct i40e_pf *pf = vsi->back; 3755 struct i40e_hw *hw = &pf->hw; 3756 int base = vsi->base_vector; 3757 int i; 3758 3759 /* disable interrupt causation from each queue */ 3760 for (i = 0; i < vsi->num_queue_pairs; i++) { 3761 u32 val; 3762 3763 val = rd32(hw, I40E_QINT_TQCTL(vsi->tx_rings[i]->reg_idx)); 3764 val &= ~I40E_QINT_TQCTL_CAUSE_ENA_MASK; 3765 wr32(hw, I40E_QINT_TQCTL(vsi->tx_rings[i]->reg_idx), val); 3766 3767 val = rd32(hw, I40E_QINT_RQCTL(vsi->rx_rings[i]->reg_idx)); 3768 val &= ~I40E_QINT_RQCTL_CAUSE_ENA_MASK; 3769 wr32(hw, I40E_QINT_RQCTL(vsi->rx_rings[i]->reg_idx), val); 3770 3771 if (!i40e_enabled_xdp_vsi(vsi)) 3772 continue; 3773 wr32(hw, I40E_QINT_TQCTL(vsi->xdp_rings[i]->reg_idx), 0); 3774 } 3775 3776 /* disable each interrupt */ 3777 if (pf->flags & I40E_FLAG_MSIX_ENABLED) { 3778 for (i = vsi->base_vector; 3779 i < (vsi->num_q_vectors + vsi->base_vector); i++) 3780 wr32(hw, I40E_PFINT_DYN_CTLN(i - 1), 0); 3781 3782 i40e_flush(hw); 3783 for (i = 0; i < vsi->num_q_vectors; i++) 3784 synchronize_irq(pf->msix_entries[i + base].vector); 3785 } else { 3786 /* Legacy and MSI mode - this stops all interrupt handling */ 3787 wr32(hw, I40E_PFINT_ICR0_ENA, 0); 3788 wr32(hw, I40E_PFINT_DYN_CTL0, 0); 3789 i40e_flush(hw); 3790 synchronize_irq(pf->pdev->irq); 3791 } 3792 } 3793 3794 /** 3795 * i40e_vsi_enable_irq - Enable IRQ for the given VSI 3796 * @vsi: the VSI being configured 3797 **/ 3798 static int i40e_vsi_enable_irq(struct i40e_vsi *vsi) 3799 { 3800 struct i40e_pf *pf = vsi->back; 3801 int i; 3802 3803 if (pf->flags & I40E_FLAG_MSIX_ENABLED) { 3804 for (i = 0; i < vsi->num_q_vectors; i++) 3805 i40e_irq_dynamic_enable(vsi, i); 3806 } else { 3807 i40e_irq_dynamic_enable_icr0(pf); 3808 } 3809 3810 i40e_flush(&pf->hw); 3811 return 0; 3812 } 3813 3814 /** 3815 * i40e_free_misc_vector - Free the vector that handles non-queue events 3816 * @pf: board private structure 3817 **/ 3818 static void i40e_free_misc_vector(struct i40e_pf *pf) 3819 { 3820 /* Disable ICR 0 */ 3821 wr32(&pf->hw, I40E_PFINT_ICR0_ENA, 0); 3822 i40e_flush(&pf->hw); 3823 3824 if (pf->flags & I40E_FLAG_MSIX_ENABLED && pf->msix_entries) { 3825 synchronize_irq(pf->msix_entries[0].vector); 3826 free_irq(pf->msix_entries[0].vector, pf); 3827 clear_bit(__I40E_MISC_IRQ_REQUESTED, pf->state); 3828 } 3829 } 3830 3831 /** 3832 * i40e_intr - MSI/Legacy and non-queue interrupt handler 3833 * @irq: interrupt number 3834 * @data: pointer to a q_vector 3835 * 3836 * This is the handler used for all MSI/Legacy interrupts, and deals 3837 * with both queue and non-queue interrupts. This is also used in 3838 * MSIX mode to handle the non-queue interrupts. 3839 **/ 3840 static irqreturn_t i40e_intr(int irq, void *data) 3841 { 3842 struct i40e_pf *pf = (struct i40e_pf *)data; 3843 struct i40e_hw *hw = &pf->hw; 3844 irqreturn_t ret = IRQ_NONE; 3845 u32 icr0, icr0_remaining; 3846 u32 val, ena_mask; 3847 3848 icr0 = rd32(hw, I40E_PFINT_ICR0); 3849 ena_mask = rd32(hw, I40E_PFINT_ICR0_ENA); 3850 3851 /* if sharing a legacy IRQ, we might get called w/o an intr pending */ 3852 if ((icr0 & I40E_PFINT_ICR0_INTEVENT_MASK) == 0) 3853 goto enable_intr; 3854 3855 /* if interrupt but no bits showing, must be SWINT */ 3856 if (((icr0 & ~I40E_PFINT_ICR0_INTEVENT_MASK) == 0) || 3857 (icr0 & I40E_PFINT_ICR0_SWINT_MASK)) 3858 pf->sw_int_count++; 3859 3860 if ((pf->flags & I40E_FLAG_IWARP_ENABLED) && 3861 (icr0 & I40E_PFINT_ICR0_ENA_PE_CRITERR_MASK)) { 3862 ena_mask &= ~I40E_PFINT_ICR0_ENA_PE_CRITERR_MASK; 3863 dev_dbg(&pf->pdev->dev, "cleared PE_CRITERR\n"); 3864 set_bit(__I40E_CORE_RESET_REQUESTED, pf->state); 3865 } 3866 3867 /* only q0 is used in MSI/Legacy mode, and none are used in MSIX */ 3868 if (icr0 & I40E_PFINT_ICR0_QUEUE_0_MASK) { 3869 struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi]; 3870 struct i40e_q_vector *q_vector = vsi->q_vectors[0]; 3871 3872 /* We do not have a way to disarm Queue causes while leaving 3873 * interrupt enabled for all other causes, ideally 3874 * interrupt should be disabled while we are in NAPI but 3875 * this is not a performance path and napi_schedule() 3876 * can deal with rescheduling. 3877 */ 3878 if (!test_bit(__I40E_DOWN, pf->state)) 3879 napi_schedule_irqoff(&q_vector->napi); 3880 } 3881 3882 if (icr0 & I40E_PFINT_ICR0_ADMINQ_MASK) { 3883 ena_mask &= ~I40E_PFINT_ICR0_ENA_ADMINQ_MASK; 3884 set_bit(__I40E_ADMINQ_EVENT_PENDING, pf->state); 3885 i40e_debug(&pf->hw, I40E_DEBUG_NVM, "AdminQ event\n"); 3886 } 3887 3888 if (icr0 & I40E_PFINT_ICR0_MAL_DETECT_MASK) { 3889 ena_mask &= ~I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK; 3890 set_bit(__I40E_MDD_EVENT_PENDING, pf->state); 3891 } 3892 3893 if (icr0 & I40E_PFINT_ICR0_VFLR_MASK) { 3894 ena_mask &= ~I40E_PFINT_ICR0_ENA_VFLR_MASK; 3895 set_bit(__I40E_VFLR_EVENT_PENDING, pf->state); 3896 } 3897 3898 if (icr0 & I40E_PFINT_ICR0_GRST_MASK) { 3899 if (!test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state)) 3900 set_bit(__I40E_RESET_INTR_RECEIVED, pf->state); 3901 ena_mask &= ~I40E_PFINT_ICR0_ENA_GRST_MASK; 3902 val = rd32(hw, I40E_GLGEN_RSTAT); 3903 val = (val & I40E_GLGEN_RSTAT_RESET_TYPE_MASK) 3904 >> I40E_GLGEN_RSTAT_RESET_TYPE_SHIFT; 3905 if (val == I40E_RESET_CORER) { 3906 pf->corer_count++; 3907 } else if (val == I40E_RESET_GLOBR) { 3908 pf->globr_count++; 3909 } else if (val == I40E_RESET_EMPR) { 3910 pf->empr_count++; 3911 set_bit(__I40E_EMP_RESET_INTR_RECEIVED, pf->state); 3912 } 3913 } 3914 3915 if (icr0 & I40E_PFINT_ICR0_HMC_ERR_MASK) { 3916 icr0 &= ~I40E_PFINT_ICR0_HMC_ERR_MASK; 3917 dev_info(&pf->pdev->dev, "HMC error interrupt\n"); 3918 dev_info(&pf->pdev->dev, "HMC error info 0x%x, HMC error data 0x%x\n", 3919 rd32(hw, I40E_PFHMC_ERRORINFO), 3920 rd32(hw, I40E_PFHMC_ERRORDATA)); 3921 } 3922 3923 if (icr0 & I40E_PFINT_ICR0_TIMESYNC_MASK) { 3924 u32 prttsyn_stat = rd32(hw, I40E_PRTTSYN_STAT_0); 3925 3926 if (prttsyn_stat & I40E_PRTTSYN_STAT_0_TXTIME_MASK) { 3927 icr0 &= ~I40E_PFINT_ICR0_ENA_TIMESYNC_MASK; 3928 i40e_ptp_tx_hwtstamp(pf); 3929 } 3930 } 3931 3932 /* If a critical error is pending we have no choice but to reset the 3933 * device. 3934 * Report and mask out any remaining unexpected interrupts. 3935 */ 3936 icr0_remaining = icr0 & ena_mask; 3937 if (icr0_remaining) { 3938 dev_info(&pf->pdev->dev, "unhandled interrupt icr0=0x%08x\n", 3939 icr0_remaining); 3940 if ((icr0_remaining & I40E_PFINT_ICR0_PE_CRITERR_MASK) || 3941 (icr0_remaining & I40E_PFINT_ICR0_PCI_EXCEPTION_MASK) || 3942 (icr0_remaining & I40E_PFINT_ICR0_ECC_ERR_MASK)) { 3943 dev_info(&pf->pdev->dev, "device will be reset\n"); 3944 set_bit(__I40E_PF_RESET_REQUESTED, pf->state); 3945 i40e_service_event_schedule(pf); 3946 } 3947 ena_mask &= ~icr0_remaining; 3948 } 3949 ret = IRQ_HANDLED; 3950 3951 enable_intr: 3952 /* re-enable interrupt causes */ 3953 wr32(hw, I40E_PFINT_ICR0_ENA, ena_mask); 3954 if (!test_bit(__I40E_DOWN, pf->state)) { 3955 i40e_service_event_schedule(pf); 3956 i40e_irq_dynamic_enable_icr0(pf); 3957 } 3958 3959 return ret; 3960 } 3961 3962 /** 3963 * i40e_clean_fdir_tx_irq - Reclaim resources after transmit completes 3964 * @tx_ring: tx ring to clean 3965 * @budget: how many cleans we're allowed 3966 * 3967 * Returns true if there's any budget left (e.g. the clean is finished) 3968 **/ 3969 static bool i40e_clean_fdir_tx_irq(struct i40e_ring *tx_ring, int budget) 3970 { 3971 struct i40e_vsi *vsi = tx_ring->vsi; 3972 u16 i = tx_ring->next_to_clean; 3973 struct i40e_tx_buffer *tx_buf; 3974 struct i40e_tx_desc *tx_desc; 3975 3976 tx_buf = &tx_ring->tx_bi[i]; 3977 tx_desc = I40E_TX_DESC(tx_ring, i); 3978 i -= tx_ring->count; 3979 3980 do { 3981 struct i40e_tx_desc *eop_desc = tx_buf->next_to_watch; 3982 3983 /* if next_to_watch is not set then there is no work pending */ 3984 if (!eop_desc) 3985 break; 3986 3987 /* prevent any other reads prior to eop_desc */ 3988 smp_rmb(); 3989 3990 /* if the descriptor isn't done, no work yet to do */ 3991 if (!(eop_desc->cmd_type_offset_bsz & 3992 cpu_to_le64(I40E_TX_DESC_DTYPE_DESC_DONE))) 3993 break; 3994 3995 /* clear next_to_watch to prevent false hangs */ 3996 tx_buf->next_to_watch = NULL; 3997 3998 tx_desc->buffer_addr = 0; 3999 tx_desc->cmd_type_offset_bsz = 0; 4000 /* move past filter desc */ 4001 tx_buf++; 4002 tx_desc++; 4003 i++; 4004 if (unlikely(!i)) { 4005 i -= tx_ring->count; 4006 tx_buf = tx_ring->tx_bi; 4007 tx_desc = I40E_TX_DESC(tx_ring, 0); 4008 } 4009 /* unmap skb header data */ 4010 dma_unmap_single(tx_ring->dev, 4011 dma_unmap_addr(tx_buf, dma), 4012 dma_unmap_len(tx_buf, len), 4013 DMA_TO_DEVICE); 4014 if (tx_buf->tx_flags & I40E_TX_FLAGS_FD_SB) 4015 kfree(tx_buf->raw_buf); 4016 4017 tx_buf->raw_buf = NULL; 4018 tx_buf->tx_flags = 0; 4019 tx_buf->next_to_watch = NULL; 4020 dma_unmap_len_set(tx_buf, len, 0); 4021 tx_desc->buffer_addr = 0; 4022 tx_desc->cmd_type_offset_bsz = 0; 4023 4024 /* move us past the eop_desc for start of next FD desc */ 4025 tx_buf++; 4026 tx_desc++; 4027 i++; 4028 if (unlikely(!i)) { 4029 i -= tx_ring->count; 4030 tx_buf = tx_ring->tx_bi; 4031 tx_desc = I40E_TX_DESC(tx_ring, 0); 4032 } 4033 4034 /* update budget accounting */ 4035 budget--; 4036 } while (likely(budget)); 4037 4038 i += tx_ring->count; 4039 tx_ring->next_to_clean = i; 4040 4041 if (vsi->back->flags & I40E_FLAG_MSIX_ENABLED) 4042 i40e_irq_dynamic_enable(vsi, tx_ring->q_vector->v_idx); 4043 4044 return budget > 0; 4045 } 4046 4047 /** 4048 * i40e_fdir_clean_ring - Interrupt Handler for FDIR SB ring 4049 * @irq: interrupt number 4050 * @data: pointer to a q_vector 4051 **/ 4052 static irqreturn_t i40e_fdir_clean_ring(int irq, void *data) 4053 { 4054 struct i40e_q_vector *q_vector = data; 4055 struct i40e_vsi *vsi; 4056 4057 if (!q_vector->tx.ring) 4058 return IRQ_HANDLED; 4059 4060 vsi = q_vector->tx.ring->vsi; 4061 i40e_clean_fdir_tx_irq(q_vector->tx.ring, vsi->work_limit); 4062 4063 return IRQ_HANDLED; 4064 } 4065 4066 /** 4067 * i40e_map_vector_to_qp - Assigns the queue pair to the vector 4068 * @vsi: the VSI being configured 4069 * @v_idx: vector index 4070 * @qp_idx: queue pair index 4071 **/ 4072 static void i40e_map_vector_to_qp(struct i40e_vsi *vsi, int v_idx, int qp_idx) 4073 { 4074 struct i40e_q_vector *q_vector = vsi->q_vectors[v_idx]; 4075 struct i40e_ring *tx_ring = vsi->tx_rings[qp_idx]; 4076 struct i40e_ring *rx_ring = vsi->rx_rings[qp_idx]; 4077 4078 tx_ring->q_vector = q_vector; 4079 tx_ring->next = q_vector->tx.ring; 4080 q_vector->tx.ring = tx_ring; 4081 q_vector->tx.count++; 4082 4083 /* Place XDP Tx ring in the same q_vector ring list as regular Tx */ 4084 if (i40e_enabled_xdp_vsi(vsi)) { 4085 struct i40e_ring *xdp_ring = vsi->xdp_rings[qp_idx]; 4086 4087 xdp_ring->q_vector = q_vector; 4088 xdp_ring->next = q_vector->tx.ring; 4089 q_vector->tx.ring = xdp_ring; 4090 q_vector->tx.count++; 4091 } 4092 4093 rx_ring->q_vector = q_vector; 4094 rx_ring->next = q_vector->rx.ring; 4095 q_vector->rx.ring = rx_ring; 4096 q_vector->rx.count++; 4097 } 4098 4099 /** 4100 * i40e_vsi_map_rings_to_vectors - Maps descriptor rings to vectors 4101 * @vsi: the VSI being configured 4102 * 4103 * This function maps descriptor rings to the queue-specific vectors 4104 * we were allotted through the MSI-X enabling code. Ideally, we'd have 4105 * one vector per queue pair, but on a constrained vector budget, we 4106 * group the queue pairs as "efficiently" as possible. 4107 **/ 4108 static void i40e_vsi_map_rings_to_vectors(struct i40e_vsi *vsi) 4109 { 4110 int qp_remaining = vsi->num_queue_pairs; 4111 int q_vectors = vsi->num_q_vectors; 4112 int num_ringpairs; 4113 int v_start = 0; 4114 int qp_idx = 0; 4115 4116 /* If we don't have enough vectors for a 1-to-1 mapping, we'll have to 4117 * group them so there are multiple queues per vector. 4118 * It is also important to go through all the vectors available to be 4119 * sure that if we don't use all the vectors, that the remaining vectors 4120 * are cleared. This is especially important when decreasing the 4121 * number of queues in use. 4122 */ 4123 for (; v_start < q_vectors; v_start++) { 4124 struct i40e_q_vector *q_vector = vsi->q_vectors[v_start]; 4125 4126 num_ringpairs = DIV_ROUND_UP(qp_remaining, q_vectors - v_start); 4127 4128 q_vector->num_ringpairs = num_ringpairs; 4129 q_vector->reg_idx = q_vector->v_idx + vsi->base_vector - 1; 4130 4131 q_vector->rx.count = 0; 4132 q_vector->tx.count = 0; 4133 q_vector->rx.ring = NULL; 4134 q_vector->tx.ring = NULL; 4135 4136 while (num_ringpairs--) { 4137 i40e_map_vector_to_qp(vsi, v_start, qp_idx); 4138 qp_idx++; 4139 qp_remaining--; 4140 } 4141 } 4142 } 4143 4144 /** 4145 * i40e_vsi_request_irq - Request IRQ from the OS 4146 * @vsi: the VSI being configured 4147 * @basename: name for the vector 4148 **/ 4149 static int i40e_vsi_request_irq(struct i40e_vsi *vsi, char *basename) 4150 { 4151 struct i40e_pf *pf = vsi->back; 4152 int err; 4153 4154 if (pf->flags & I40E_FLAG_MSIX_ENABLED) 4155 err = i40e_vsi_request_irq_msix(vsi, basename); 4156 else if (pf->flags & I40E_FLAG_MSI_ENABLED) 4157 err = request_irq(pf->pdev->irq, i40e_intr, 0, 4158 pf->int_name, pf); 4159 else 4160 err = request_irq(pf->pdev->irq, i40e_intr, IRQF_SHARED, 4161 pf->int_name, pf); 4162 4163 if (err) 4164 dev_info(&pf->pdev->dev, "request_irq failed, Error %d\n", err); 4165 4166 return err; 4167 } 4168 4169 #ifdef CONFIG_NET_POLL_CONTROLLER 4170 /** 4171 * i40e_netpoll - A Polling 'interrupt' handler 4172 * @netdev: network interface device structure 4173 * 4174 * This is used by netconsole to send skbs without having to re-enable 4175 * interrupts. It's not called while the normal interrupt routine is executing. 4176 **/ 4177 static void i40e_netpoll(struct net_device *netdev) 4178 { 4179 struct i40e_netdev_priv *np = netdev_priv(netdev); 4180 struct i40e_vsi *vsi = np->vsi; 4181 struct i40e_pf *pf = vsi->back; 4182 int i; 4183 4184 /* if interface is down do nothing */ 4185 if (test_bit(__I40E_VSI_DOWN, vsi->state)) 4186 return; 4187 4188 if (pf->flags & I40E_FLAG_MSIX_ENABLED) { 4189 for (i = 0; i < vsi->num_q_vectors; i++) 4190 i40e_msix_clean_rings(0, vsi->q_vectors[i]); 4191 } else { 4192 i40e_intr(pf->pdev->irq, netdev); 4193 } 4194 } 4195 #endif 4196 4197 #define I40E_QTX_ENA_WAIT_COUNT 50 4198 4199 /** 4200 * i40e_pf_txq_wait - Wait for a PF's Tx queue to be enabled or disabled 4201 * @pf: the PF being configured 4202 * @pf_q: the PF queue 4203 * @enable: enable or disable state of the queue 4204 * 4205 * This routine will wait for the given Tx queue of the PF to reach the 4206 * enabled or disabled state. 4207 * Returns -ETIMEDOUT in case of failing to reach the requested state after 4208 * multiple retries; else will return 0 in case of success. 4209 **/ 4210 static int i40e_pf_txq_wait(struct i40e_pf *pf, int pf_q, bool enable) 4211 { 4212 int i; 4213 u32 tx_reg; 4214 4215 for (i = 0; i < I40E_QUEUE_WAIT_RETRY_LIMIT; i++) { 4216 tx_reg = rd32(&pf->hw, I40E_QTX_ENA(pf_q)); 4217 if (enable == !!(tx_reg & I40E_QTX_ENA_QENA_STAT_MASK)) 4218 break; 4219 4220 usleep_range(10, 20); 4221 } 4222 if (i >= I40E_QUEUE_WAIT_RETRY_LIMIT) 4223 return -ETIMEDOUT; 4224 4225 return 0; 4226 } 4227 4228 /** 4229 * i40e_control_tx_q - Start or stop a particular Tx queue 4230 * @pf: the PF structure 4231 * @pf_q: the PF queue to configure 4232 * @enable: start or stop the queue 4233 * 4234 * This function enables or disables a single queue. Note that any delay 4235 * required after the operation is expected to be handled by the caller of 4236 * this function. 4237 **/ 4238 static void i40e_control_tx_q(struct i40e_pf *pf, int pf_q, bool enable) 4239 { 4240 struct i40e_hw *hw = &pf->hw; 4241 u32 tx_reg; 4242 int i; 4243 4244 /* warn the TX unit of coming changes */ 4245 i40e_pre_tx_queue_cfg(&pf->hw, pf_q, enable); 4246 if (!enable) 4247 usleep_range(10, 20); 4248 4249 for (i = 0; i < I40E_QTX_ENA_WAIT_COUNT; i++) { 4250 tx_reg = rd32(hw, I40E_QTX_ENA(pf_q)); 4251 if (((tx_reg >> I40E_QTX_ENA_QENA_REQ_SHIFT) & 1) == 4252 ((tx_reg >> I40E_QTX_ENA_QENA_STAT_SHIFT) & 1)) 4253 break; 4254 usleep_range(1000, 2000); 4255 } 4256 4257 /* Skip if the queue is already in the requested state */ 4258 if (enable == !!(tx_reg & I40E_QTX_ENA_QENA_STAT_MASK)) 4259 return; 4260 4261 /* turn on/off the queue */ 4262 if (enable) { 4263 wr32(hw, I40E_QTX_HEAD(pf_q), 0); 4264 tx_reg |= I40E_QTX_ENA_QENA_REQ_MASK; 4265 } else { 4266 tx_reg &= ~I40E_QTX_ENA_QENA_REQ_MASK; 4267 } 4268 4269 wr32(hw, I40E_QTX_ENA(pf_q), tx_reg); 4270 } 4271 4272 /** 4273 * i40e_control_wait_tx_q - Start/stop Tx queue and wait for completion 4274 * @seid: VSI SEID 4275 * @pf: the PF structure 4276 * @pf_q: the PF queue to configure 4277 * @is_xdp: true if the queue is used for XDP 4278 * @enable: start or stop the queue 4279 **/ 4280 static int i40e_control_wait_tx_q(int seid, struct i40e_pf *pf, int pf_q, 4281 bool is_xdp, bool enable) 4282 { 4283 int ret; 4284 4285 i40e_control_tx_q(pf, pf_q, enable); 4286 4287 /* wait for the change to finish */ 4288 ret = i40e_pf_txq_wait(pf, pf_q, enable); 4289 if (ret) { 4290 dev_info(&pf->pdev->dev, 4291 "VSI seid %d %sTx ring %d %sable timeout\n", 4292 seid, (is_xdp ? "XDP " : ""), pf_q, 4293 (enable ? "en" : "dis")); 4294 } 4295 4296 return ret; 4297 } 4298 4299 /** 4300 * i40e_vsi_control_tx - Start or stop a VSI's rings 4301 * @vsi: the VSI being configured 4302 * @enable: start or stop the rings 4303 **/ 4304 static int i40e_vsi_control_tx(struct i40e_vsi *vsi, bool enable) 4305 { 4306 struct i40e_pf *pf = vsi->back; 4307 int i, pf_q, ret = 0; 4308 4309 pf_q = vsi->base_queue; 4310 for (i = 0; i < vsi->num_queue_pairs; i++, pf_q++) { 4311 ret = i40e_control_wait_tx_q(vsi->seid, pf, 4312 pf_q, 4313 false /*is xdp*/, enable); 4314 if (ret) 4315 break; 4316 4317 if (!i40e_enabled_xdp_vsi(vsi)) 4318 continue; 4319 4320 ret = i40e_control_wait_tx_q(vsi->seid, pf, 4321 pf_q + vsi->alloc_queue_pairs, 4322 true /*is xdp*/, enable); 4323 if (ret) 4324 break; 4325 } 4326 4327 return ret; 4328 } 4329 4330 /** 4331 * i40e_pf_rxq_wait - Wait for a PF's Rx queue to be enabled or disabled 4332 * @pf: the PF being configured 4333 * @pf_q: the PF queue 4334 * @enable: enable or disable state of the queue 4335 * 4336 * This routine will wait for the given Rx queue of the PF to reach the 4337 * enabled or disabled state. 4338 * Returns -ETIMEDOUT in case of failing to reach the requested state after 4339 * multiple retries; else will return 0 in case of success. 4340 **/ 4341 static int i40e_pf_rxq_wait(struct i40e_pf *pf, int pf_q, bool enable) 4342 { 4343 int i; 4344 u32 rx_reg; 4345 4346 for (i = 0; i < I40E_QUEUE_WAIT_RETRY_LIMIT; i++) { 4347 rx_reg = rd32(&pf->hw, I40E_QRX_ENA(pf_q)); 4348 if (enable == !!(rx_reg & I40E_QRX_ENA_QENA_STAT_MASK)) 4349 break; 4350 4351 usleep_range(10, 20); 4352 } 4353 if (i >= I40E_QUEUE_WAIT_RETRY_LIMIT) 4354 return -ETIMEDOUT; 4355 4356 return 0; 4357 } 4358 4359 /** 4360 * i40e_control_rx_q - Start or stop a particular Rx queue 4361 * @pf: the PF structure 4362 * @pf_q: the PF queue to configure 4363 * @enable: start or stop the queue 4364 * 4365 * This function enables or disables a single queue. Note that any delay 4366 * required after the operation is expected to be handled by the caller of 4367 * this function. 4368 **/ 4369 static void i40e_control_rx_q(struct i40e_pf *pf, int pf_q, bool enable) 4370 { 4371 struct i40e_hw *hw = &pf->hw; 4372 u32 rx_reg; 4373 int i; 4374 4375 for (i = 0; i < I40E_QTX_ENA_WAIT_COUNT; i++) { 4376 rx_reg = rd32(hw, I40E_QRX_ENA(pf_q)); 4377 if (((rx_reg >> I40E_QRX_ENA_QENA_REQ_SHIFT) & 1) == 4378 ((rx_reg >> I40E_QRX_ENA_QENA_STAT_SHIFT) & 1)) 4379 break; 4380 usleep_range(1000, 2000); 4381 } 4382 4383 /* Skip if the queue is already in the requested state */ 4384 if (enable == !!(rx_reg & I40E_QRX_ENA_QENA_STAT_MASK)) 4385 return; 4386 4387 /* turn on/off the queue */ 4388 if (enable) 4389 rx_reg |= I40E_QRX_ENA_QENA_REQ_MASK; 4390 else 4391 rx_reg &= ~I40E_QRX_ENA_QENA_REQ_MASK; 4392 4393 wr32(hw, I40E_QRX_ENA(pf_q), rx_reg); 4394 } 4395 4396 /** 4397 * i40e_vsi_control_rx - Start or stop a VSI's rings 4398 * @vsi: the VSI being configured 4399 * @enable: start or stop the rings 4400 **/ 4401 static int i40e_vsi_control_rx(struct i40e_vsi *vsi, bool enable) 4402 { 4403 struct i40e_pf *pf = vsi->back; 4404 int i, pf_q, ret = 0; 4405 4406 pf_q = vsi->base_queue; 4407 for (i = 0; i < vsi->num_queue_pairs; i++, pf_q++) { 4408 i40e_control_rx_q(pf, pf_q, enable); 4409 4410 /* wait for the change to finish */ 4411 ret = i40e_pf_rxq_wait(pf, pf_q, enable); 4412 if (ret) { 4413 dev_info(&pf->pdev->dev, 4414 "VSI seid %d Rx ring %d %sable timeout\n", 4415 vsi->seid, pf_q, (enable ? "en" : "dis")); 4416 break; 4417 } 4418 } 4419 4420 /* Due to HW errata, on Rx disable only, the register can indicate done 4421 * before it really is. Needs 50ms to be sure 4422 */ 4423 if (!enable) 4424 mdelay(50); 4425 4426 return ret; 4427 } 4428 4429 /** 4430 * i40e_vsi_start_rings - Start a VSI's rings 4431 * @vsi: the VSI being configured 4432 **/ 4433 int i40e_vsi_start_rings(struct i40e_vsi *vsi) 4434 { 4435 int ret = 0; 4436 4437 /* do rx first for enable and last for disable */ 4438 ret = i40e_vsi_control_rx(vsi, true); 4439 if (ret) 4440 return ret; 4441 ret = i40e_vsi_control_tx(vsi, true); 4442 4443 return ret; 4444 } 4445 4446 /** 4447 * i40e_vsi_stop_rings - Stop a VSI's rings 4448 * @vsi: the VSI being configured 4449 **/ 4450 void i40e_vsi_stop_rings(struct i40e_vsi *vsi) 4451 { 4452 /* When port TX is suspended, don't wait */ 4453 if (test_bit(__I40E_PORT_SUSPENDED, vsi->back->state)) 4454 return i40e_vsi_stop_rings_no_wait(vsi); 4455 4456 /* do rx first for enable and last for disable 4457 * Ignore return value, we need to shutdown whatever we can 4458 */ 4459 i40e_vsi_control_tx(vsi, false); 4460 i40e_vsi_control_rx(vsi, false); 4461 } 4462 4463 /** 4464 * i40e_vsi_stop_rings_no_wait - Stop a VSI's rings and do not delay 4465 * @vsi: the VSI being shutdown 4466 * 4467 * This function stops all the rings for a VSI but does not delay to verify 4468 * that rings have been disabled. It is expected that the caller is shutting 4469 * down multiple VSIs at once and will delay together for all the VSIs after 4470 * initiating the shutdown. This is particularly useful for shutting down lots 4471 * of VFs together. Otherwise, a large delay can be incurred while configuring 4472 * each VSI in serial. 4473 **/ 4474 void i40e_vsi_stop_rings_no_wait(struct i40e_vsi *vsi) 4475 { 4476 struct i40e_pf *pf = vsi->back; 4477 int i, pf_q; 4478 4479 pf_q = vsi->base_queue; 4480 for (i = 0; i < vsi->num_queue_pairs; i++, pf_q++) { 4481 i40e_control_tx_q(pf, pf_q, false); 4482 i40e_control_rx_q(pf, pf_q, false); 4483 } 4484 } 4485 4486 /** 4487 * i40e_vsi_free_irq - Free the irq association with the OS 4488 * @vsi: the VSI being configured 4489 **/ 4490 static void i40e_vsi_free_irq(struct i40e_vsi *vsi) 4491 { 4492 struct i40e_pf *pf = vsi->back; 4493 struct i40e_hw *hw = &pf->hw; 4494 int base = vsi->base_vector; 4495 u32 val, qp; 4496 int i; 4497 4498 if (pf->flags & I40E_FLAG_MSIX_ENABLED) { 4499 if (!vsi->q_vectors) 4500 return; 4501 4502 if (!vsi->irqs_ready) 4503 return; 4504 4505 vsi->irqs_ready = false; 4506 for (i = 0; i < vsi->num_q_vectors; i++) { 4507 int irq_num; 4508 u16 vector; 4509 4510 vector = i + base; 4511 irq_num = pf->msix_entries[vector].vector; 4512 4513 /* free only the irqs that were actually requested */ 4514 if (!vsi->q_vectors[i] || 4515 !vsi->q_vectors[i]->num_ringpairs) 4516 continue; 4517 4518 /* clear the affinity notifier in the IRQ descriptor */ 4519 irq_set_affinity_notifier(irq_num, NULL); 4520 /* remove our suggested affinity mask for this IRQ */ 4521 irq_set_affinity_hint(irq_num, NULL); 4522 synchronize_irq(irq_num); 4523 free_irq(irq_num, vsi->q_vectors[i]); 4524 4525 /* Tear down the interrupt queue link list 4526 * 4527 * We know that they come in pairs and always 4528 * the Rx first, then the Tx. To clear the 4529 * link list, stick the EOL value into the 4530 * next_q field of the registers. 4531 */ 4532 val = rd32(hw, I40E_PFINT_LNKLSTN(vector - 1)); 4533 qp = (val & I40E_PFINT_LNKLSTN_FIRSTQ_INDX_MASK) 4534 >> I40E_PFINT_LNKLSTN_FIRSTQ_INDX_SHIFT; 4535 val |= I40E_QUEUE_END_OF_LIST 4536 << I40E_PFINT_LNKLSTN_FIRSTQ_INDX_SHIFT; 4537 wr32(hw, I40E_PFINT_LNKLSTN(vector - 1), val); 4538 4539 while (qp != I40E_QUEUE_END_OF_LIST) { 4540 u32 next; 4541 4542 val = rd32(hw, I40E_QINT_RQCTL(qp)); 4543 4544 val &= ~(I40E_QINT_RQCTL_MSIX_INDX_MASK | 4545 I40E_QINT_RQCTL_MSIX0_INDX_MASK | 4546 I40E_QINT_RQCTL_CAUSE_ENA_MASK | 4547 I40E_QINT_RQCTL_INTEVENT_MASK); 4548 4549 val |= (I40E_QINT_RQCTL_ITR_INDX_MASK | 4550 I40E_QINT_RQCTL_NEXTQ_INDX_MASK); 4551 4552 wr32(hw, I40E_QINT_RQCTL(qp), val); 4553 4554 val = rd32(hw, I40E_QINT_TQCTL(qp)); 4555 4556 next = (val & I40E_QINT_TQCTL_NEXTQ_INDX_MASK) 4557 >> I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT; 4558 4559 val &= ~(I40E_QINT_TQCTL_MSIX_INDX_MASK | 4560 I40E_QINT_TQCTL_MSIX0_INDX_MASK | 4561 I40E_QINT_TQCTL_CAUSE_ENA_MASK | 4562 I40E_QINT_TQCTL_INTEVENT_MASK); 4563 4564 val |= (I40E_QINT_TQCTL_ITR_INDX_MASK | 4565 I40E_QINT_TQCTL_NEXTQ_INDX_MASK); 4566 4567 wr32(hw, I40E_QINT_TQCTL(qp), val); 4568 qp = next; 4569 } 4570 } 4571 } else { 4572 free_irq(pf->pdev->irq, pf); 4573 4574 val = rd32(hw, I40E_PFINT_LNKLST0); 4575 qp = (val & I40E_PFINT_LNKLSTN_FIRSTQ_INDX_MASK) 4576 >> I40E_PFINT_LNKLSTN_FIRSTQ_INDX_SHIFT; 4577 val |= I40E_QUEUE_END_OF_LIST 4578 << I40E_PFINT_LNKLST0_FIRSTQ_INDX_SHIFT; 4579 wr32(hw, I40E_PFINT_LNKLST0, val); 4580 4581 val = rd32(hw, I40E_QINT_RQCTL(qp)); 4582 val &= ~(I40E_QINT_RQCTL_MSIX_INDX_MASK | 4583 I40E_QINT_RQCTL_MSIX0_INDX_MASK | 4584 I40E_QINT_RQCTL_CAUSE_ENA_MASK | 4585 I40E_QINT_RQCTL_INTEVENT_MASK); 4586 4587 val |= (I40E_QINT_RQCTL_ITR_INDX_MASK | 4588 I40E_QINT_RQCTL_NEXTQ_INDX_MASK); 4589 4590 wr32(hw, I40E_QINT_RQCTL(qp), val); 4591 4592 val = rd32(hw, I40E_QINT_TQCTL(qp)); 4593 4594 val &= ~(I40E_QINT_TQCTL_MSIX_INDX_MASK | 4595 I40E_QINT_TQCTL_MSIX0_INDX_MASK | 4596 I40E_QINT_TQCTL_CAUSE_ENA_MASK | 4597 I40E_QINT_TQCTL_INTEVENT_MASK); 4598 4599 val |= (I40E_QINT_TQCTL_ITR_INDX_MASK | 4600 I40E_QINT_TQCTL_NEXTQ_INDX_MASK); 4601 4602 wr32(hw, I40E_QINT_TQCTL(qp), val); 4603 } 4604 } 4605 4606 /** 4607 * i40e_free_q_vector - Free memory allocated for specific interrupt vector 4608 * @vsi: the VSI being configured 4609 * @v_idx: Index of vector to be freed 4610 * 4611 * This function frees the memory allocated to the q_vector. In addition if 4612 * NAPI is enabled it will delete any references to the NAPI struct prior 4613 * to freeing the q_vector. 4614 **/ 4615 static void i40e_free_q_vector(struct i40e_vsi *vsi, int v_idx) 4616 { 4617 struct i40e_q_vector *q_vector = vsi->q_vectors[v_idx]; 4618 struct i40e_ring *ring; 4619 4620 if (!q_vector) 4621 return; 4622 4623 /* disassociate q_vector from rings */ 4624 i40e_for_each_ring(ring, q_vector->tx) 4625 ring->q_vector = NULL; 4626 4627 i40e_for_each_ring(ring, q_vector->rx) 4628 ring->q_vector = NULL; 4629 4630 /* only VSI w/ an associated netdev is set up w/ NAPI */ 4631 if (vsi->netdev) 4632 netif_napi_del(&q_vector->napi); 4633 4634 vsi->q_vectors[v_idx] = NULL; 4635 4636 kfree_rcu(q_vector, rcu); 4637 } 4638 4639 /** 4640 * i40e_vsi_free_q_vectors - Free memory allocated for interrupt vectors 4641 * @vsi: the VSI being un-configured 4642 * 4643 * This frees the memory allocated to the q_vectors and 4644 * deletes references to the NAPI struct. 4645 **/ 4646 static void i40e_vsi_free_q_vectors(struct i40e_vsi *vsi) 4647 { 4648 int v_idx; 4649 4650 for (v_idx = 0; v_idx < vsi->num_q_vectors; v_idx++) 4651 i40e_free_q_vector(vsi, v_idx); 4652 } 4653 4654 /** 4655 * i40e_reset_interrupt_capability - Disable interrupt setup in OS 4656 * @pf: board private structure 4657 **/ 4658 static void i40e_reset_interrupt_capability(struct i40e_pf *pf) 4659 { 4660 /* If we're in Legacy mode, the interrupt was cleaned in vsi_close */ 4661 if (pf->flags & I40E_FLAG_MSIX_ENABLED) { 4662 pci_disable_msix(pf->pdev); 4663 kfree(pf->msix_entries); 4664 pf->msix_entries = NULL; 4665 kfree(pf->irq_pile); 4666 pf->irq_pile = NULL; 4667 } else if (pf->flags & I40E_FLAG_MSI_ENABLED) { 4668 pci_disable_msi(pf->pdev); 4669 } 4670 pf->flags &= ~(I40E_FLAG_MSIX_ENABLED | I40E_FLAG_MSI_ENABLED); 4671 } 4672 4673 /** 4674 * i40e_clear_interrupt_scheme - Clear the current interrupt scheme settings 4675 * @pf: board private structure 4676 * 4677 * We go through and clear interrupt specific resources and reset the structure 4678 * to pre-load conditions 4679 **/ 4680 static void i40e_clear_interrupt_scheme(struct i40e_pf *pf) 4681 { 4682 int i; 4683 4684 i40e_free_misc_vector(pf); 4685 4686 i40e_put_lump(pf->irq_pile, pf->iwarp_base_vector, 4687 I40E_IWARP_IRQ_PILE_ID); 4688 4689 i40e_put_lump(pf->irq_pile, 0, I40E_PILE_VALID_BIT-1); 4690 for (i = 0; i < pf->num_alloc_vsi; i++) 4691 if (pf->vsi[i]) 4692 i40e_vsi_free_q_vectors(pf->vsi[i]); 4693 i40e_reset_interrupt_capability(pf); 4694 } 4695 4696 /** 4697 * i40e_napi_enable_all - Enable NAPI for all q_vectors in the VSI 4698 * @vsi: the VSI being configured 4699 **/ 4700 static void i40e_napi_enable_all(struct i40e_vsi *vsi) 4701 { 4702 int q_idx; 4703 4704 if (!vsi->netdev) 4705 return; 4706 4707 for (q_idx = 0; q_idx < vsi->num_q_vectors; q_idx++) { 4708 struct i40e_q_vector *q_vector = vsi->q_vectors[q_idx]; 4709 4710 if (q_vector->rx.ring || q_vector->tx.ring) 4711 napi_enable(&q_vector->napi); 4712 } 4713 } 4714 4715 /** 4716 * i40e_napi_disable_all - Disable NAPI for all q_vectors in the VSI 4717 * @vsi: the VSI being configured 4718 **/ 4719 static void i40e_napi_disable_all(struct i40e_vsi *vsi) 4720 { 4721 int q_idx; 4722 4723 if (!vsi->netdev) 4724 return; 4725 4726 for (q_idx = 0; q_idx < vsi->num_q_vectors; q_idx++) { 4727 struct i40e_q_vector *q_vector = vsi->q_vectors[q_idx]; 4728 4729 if (q_vector->rx.ring || q_vector->tx.ring) 4730 napi_disable(&q_vector->napi); 4731 } 4732 } 4733 4734 /** 4735 * i40e_vsi_close - Shut down a VSI 4736 * @vsi: the vsi to be quelled 4737 **/ 4738 static void i40e_vsi_close(struct i40e_vsi *vsi) 4739 { 4740 struct i40e_pf *pf = vsi->back; 4741 if (!test_and_set_bit(__I40E_VSI_DOWN, vsi->state)) 4742 i40e_down(vsi); 4743 i40e_vsi_free_irq(vsi); 4744 i40e_vsi_free_tx_resources(vsi); 4745 i40e_vsi_free_rx_resources(vsi); 4746 vsi->current_netdev_flags = 0; 4747 pf->flags |= I40E_FLAG_SERVICE_CLIENT_REQUESTED; 4748 if (test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state)) 4749 pf->flags |= I40E_FLAG_CLIENT_RESET; 4750 } 4751 4752 /** 4753 * i40e_quiesce_vsi - Pause a given VSI 4754 * @vsi: the VSI being paused 4755 **/ 4756 static void i40e_quiesce_vsi(struct i40e_vsi *vsi) 4757 { 4758 if (test_bit(__I40E_VSI_DOWN, vsi->state)) 4759 return; 4760 4761 set_bit(__I40E_VSI_NEEDS_RESTART, vsi->state); 4762 if (vsi->netdev && netif_running(vsi->netdev)) 4763 vsi->netdev->netdev_ops->ndo_stop(vsi->netdev); 4764 else 4765 i40e_vsi_close(vsi); 4766 } 4767 4768 /** 4769 * i40e_unquiesce_vsi - Resume a given VSI 4770 * @vsi: the VSI being resumed 4771 **/ 4772 static void i40e_unquiesce_vsi(struct i40e_vsi *vsi) 4773 { 4774 if (!test_and_clear_bit(__I40E_VSI_NEEDS_RESTART, vsi->state)) 4775 return; 4776 4777 if (vsi->netdev && netif_running(vsi->netdev)) 4778 vsi->netdev->netdev_ops->ndo_open(vsi->netdev); 4779 else 4780 i40e_vsi_open(vsi); /* this clears the DOWN bit */ 4781 } 4782 4783 /** 4784 * i40e_pf_quiesce_all_vsi - Pause all VSIs on a PF 4785 * @pf: the PF 4786 **/ 4787 static void i40e_pf_quiesce_all_vsi(struct i40e_pf *pf) 4788 { 4789 int v; 4790 4791 for (v = 0; v < pf->num_alloc_vsi; v++) { 4792 if (pf->vsi[v]) 4793 i40e_quiesce_vsi(pf->vsi[v]); 4794 } 4795 } 4796 4797 /** 4798 * i40e_pf_unquiesce_all_vsi - Resume all VSIs on a PF 4799 * @pf: the PF 4800 **/ 4801 static void i40e_pf_unquiesce_all_vsi(struct i40e_pf *pf) 4802 { 4803 int v; 4804 4805 for (v = 0; v < pf->num_alloc_vsi; v++) { 4806 if (pf->vsi[v]) 4807 i40e_unquiesce_vsi(pf->vsi[v]); 4808 } 4809 } 4810 4811 /** 4812 * i40e_vsi_wait_queues_disabled - Wait for VSI's queues to be disabled 4813 * @vsi: the VSI being configured 4814 * 4815 * Wait until all queues on a given VSI have been disabled. 4816 **/ 4817 int i40e_vsi_wait_queues_disabled(struct i40e_vsi *vsi) 4818 { 4819 struct i40e_pf *pf = vsi->back; 4820 int i, pf_q, ret; 4821 4822 pf_q = vsi->base_queue; 4823 for (i = 0; i < vsi->num_queue_pairs; i++, pf_q++) { 4824 /* Check and wait for the Tx queue */ 4825 ret = i40e_pf_txq_wait(pf, pf_q, false); 4826 if (ret) { 4827 dev_info(&pf->pdev->dev, 4828 "VSI seid %d Tx ring %d disable timeout\n", 4829 vsi->seid, pf_q); 4830 return ret; 4831 } 4832 4833 if (!i40e_enabled_xdp_vsi(vsi)) 4834 goto wait_rx; 4835 4836 /* Check and wait for the XDP Tx queue */ 4837 ret = i40e_pf_txq_wait(pf, pf_q + vsi->alloc_queue_pairs, 4838 false); 4839 if (ret) { 4840 dev_info(&pf->pdev->dev, 4841 "VSI seid %d XDP Tx ring %d disable timeout\n", 4842 vsi->seid, pf_q); 4843 return ret; 4844 } 4845 wait_rx: 4846 /* Check and wait for the Rx queue */ 4847 ret = i40e_pf_rxq_wait(pf, pf_q, false); 4848 if (ret) { 4849 dev_info(&pf->pdev->dev, 4850 "VSI seid %d Rx ring %d disable timeout\n", 4851 vsi->seid, pf_q); 4852 return ret; 4853 } 4854 } 4855 4856 return 0; 4857 } 4858 4859 #ifdef CONFIG_I40E_DCB 4860 /** 4861 * i40e_pf_wait_queues_disabled - Wait for all queues of PF VSIs to be disabled 4862 * @pf: the PF 4863 * 4864 * This function waits for the queues to be in disabled state for all the 4865 * VSIs that are managed by this PF. 4866 **/ 4867 static int i40e_pf_wait_queues_disabled(struct i40e_pf *pf) 4868 { 4869 int v, ret = 0; 4870 4871 for (v = 0; v < pf->hw.func_caps.num_vsis; v++) { 4872 if (pf->vsi[v]) { 4873 ret = i40e_vsi_wait_queues_disabled(pf->vsi[v]); 4874 if (ret) 4875 break; 4876 } 4877 } 4878 4879 return ret; 4880 } 4881 4882 #endif 4883 4884 /** 4885 * i40e_get_iscsi_tc_map - Return TC map for iSCSI APP 4886 * @pf: pointer to PF 4887 * 4888 * Get TC map for ISCSI PF type that will include iSCSI TC 4889 * and LAN TC. 4890 **/ 4891 static u8 i40e_get_iscsi_tc_map(struct i40e_pf *pf) 4892 { 4893 struct i40e_dcb_app_priority_table app; 4894 struct i40e_hw *hw = &pf->hw; 4895 u8 enabled_tc = 1; /* TC0 is always enabled */ 4896 u8 tc, i; 4897 /* Get the iSCSI APP TLV */ 4898 struct i40e_dcbx_config *dcbcfg = &hw->local_dcbx_config; 4899 4900 for (i = 0; i < dcbcfg->numapps; i++) { 4901 app = dcbcfg->app[i]; 4902 if (app.selector == I40E_APP_SEL_TCPIP && 4903 app.protocolid == I40E_APP_PROTOID_ISCSI) { 4904 tc = dcbcfg->etscfg.prioritytable[app.priority]; 4905 enabled_tc |= BIT(tc); 4906 break; 4907 } 4908 } 4909 4910 return enabled_tc; 4911 } 4912 4913 /** 4914 * i40e_dcb_get_num_tc - Get the number of TCs from DCBx config 4915 * @dcbcfg: the corresponding DCBx configuration structure 4916 * 4917 * Return the number of TCs from given DCBx configuration 4918 **/ 4919 static u8 i40e_dcb_get_num_tc(struct i40e_dcbx_config *dcbcfg) 4920 { 4921 int i, tc_unused = 0; 4922 u8 num_tc = 0; 4923 u8 ret = 0; 4924 4925 /* Scan the ETS Config Priority Table to find 4926 * traffic class enabled for a given priority 4927 * and create a bitmask of enabled TCs 4928 */ 4929 for (i = 0; i < I40E_MAX_USER_PRIORITY; i++) 4930 num_tc |= BIT(dcbcfg->etscfg.prioritytable[i]); 4931 4932 /* Now scan the bitmask to check for 4933 * contiguous TCs starting with TC0 4934 */ 4935 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 4936 if (num_tc & BIT(i)) { 4937 if (!tc_unused) { 4938 ret++; 4939 } else { 4940 pr_err("Non-contiguous TC - Disabling DCB\n"); 4941 return 1; 4942 } 4943 } else { 4944 tc_unused = 1; 4945 } 4946 } 4947 4948 /* There is always at least TC0 */ 4949 if (!ret) 4950 ret = 1; 4951 4952 return ret; 4953 } 4954 4955 /** 4956 * i40e_dcb_get_enabled_tc - Get enabled traffic classes 4957 * @dcbcfg: the corresponding DCBx configuration structure 4958 * 4959 * Query the current DCB configuration and return the number of 4960 * traffic classes enabled from the given DCBX config 4961 **/ 4962 static u8 i40e_dcb_get_enabled_tc(struct i40e_dcbx_config *dcbcfg) 4963 { 4964 u8 num_tc = i40e_dcb_get_num_tc(dcbcfg); 4965 u8 enabled_tc = 1; 4966 u8 i; 4967 4968 for (i = 0; i < num_tc; i++) 4969 enabled_tc |= BIT(i); 4970 4971 return enabled_tc; 4972 } 4973 4974 /** 4975 * i40e_mqprio_get_enabled_tc - Get enabled traffic classes 4976 * @pf: PF being queried 4977 * 4978 * Query the current MQPRIO configuration and return the number of 4979 * traffic classes enabled. 4980 **/ 4981 static u8 i40e_mqprio_get_enabled_tc(struct i40e_pf *pf) 4982 { 4983 struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi]; 4984 u8 num_tc = vsi->mqprio_qopt.qopt.num_tc; 4985 u8 enabled_tc = 1, i; 4986 4987 for (i = 1; i < num_tc; i++) 4988 enabled_tc |= BIT(i); 4989 return enabled_tc; 4990 } 4991 4992 /** 4993 * i40e_pf_get_num_tc - Get enabled traffic classes for PF 4994 * @pf: PF being queried 4995 * 4996 * Return number of traffic classes enabled for the given PF 4997 **/ 4998 static u8 i40e_pf_get_num_tc(struct i40e_pf *pf) 4999 { 5000 struct i40e_hw *hw = &pf->hw; 5001 u8 i, enabled_tc = 1; 5002 u8 num_tc = 0; 5003 struct i40e_dcbx_config *dcbcfg = &hw->local_dcbx_config; 5004 5005 if (pf->flags & I40E_FLAG_TC_MQPRIO) 5006 return pf->vsi[pf->lan_vsi]->mqprio_qopt.qopt.num_tc; 5007 5008 /* If neither MQPRIO nor DCB is enabled, then always use single TC */ 5009 if (!(pf->flags & I40E_FLAG_DCB_ENABLED)) 5010 return 1; 5011 5012 /* SFP mode will be enabled for all TCs on port */ 5013 if (!(pf->flags & I40E_FLAG_MFP_ENABLED)) 5014 return i40e_dcb_get_num_tc(dcbcfg); 5015 5016 /* MFP mode return count of enabled TCs for this PF */ 5017 if (pf->hw.func_caps.iscsi) 5018 enabled_tc = i40e_get_iscsi_tc_map(pf); 5019 else 5020 return 1; /* Only TC0 */ 5021 5022 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 5023 if (enabled_tc & BIT(i)) 5024 num_tc++; 5025 } 5026 return num_tc; 5027 } 5028 5029 /** 5030 * i40e_pf_get_pf_tc_map - Get bitmap for enabled traffic classes 5031 * @pf: PF being queried 5032 * 5033 * Return a bitmap for enabled traffic classes for this PF. 5034 **/ 5035 static u8 i40e_pf_get_tc_map(struct i40e_pf *pf) 5036 { 5037 if (pf->flags & I40E_FLAG_TC_MQPRIO) 5038 return i40e_mqprio_get_enabled_tc(pf); 5039 5040 /* If neither MQPRIO nor DCB is enabled for this PF then just return 5041 * default TC 5042 */ 5043 if (!(pf->flags & I40E_FLAG_DCB_ENABLED)) 5044 return I40E_DEFAULT_TRAFFIC_CLASS; 5045 5046 /* SFP mode we want PF to be enabled for all TCs */ 5047 if (!(pf->flags & I40E_FLAG_MFP_ENABLED)) 5048 return i40e_dcb_get_enabled_tc(&pf->hw.local_dcbx_config); 5049 5050 /* MFP enabled and iSCSI PF type */ 5051 if (pf->hw.func_caps.iscsi) 5052 return i40e_get_iscsi_tc_map(pf); 5053 else 5054 return I40E_DEFAULT_TRAFFIC_CLASS; 5055 } 5056 5057 /** 5058 * i40e_vsi_get_bw_info - Query VSI BW Information 5059 * @vsi: the VSI being queried 5060 * 5061 * Returns 0 on success, negative value on failure 5062 **/ 5063 static int i40e_vsi_get_bw_info(struct i40e_vsi *vsi) 5064 { 5065 struct i40e_aqc_query_vsi_ets_sla_config_resp bw_ets_config = {0}; 5066 struct i40e_aqc_query_vsi_bw_config_resp bw_config = {0}; 5067 struct i40e_pf *pf = vsi->back; 5068 struct i40e_hw *hw = &pf->hw; 5069 i40e_status ret; 5070 u32 tc_bw_max; 5071 int i; 5072 5073 /* Get the VSI level BW configuration */ 5074 ret = i40e_aq_query_vsi_bw_config(hw, vsi->seid, &bw_config, NULL); 5075 if (ret) { 5076 dev_info(&pf->pdev->dev, 5077 "couldn't get PF vsi bw config, err %s aq_err %s\n", 5078 i40e_stat_str(&pf->hw, ret), 5079 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 5080 return -EINVAL; 5081 } 5082 5083 /* Get the VSI level BW configuration per TC */ 5084 ret = i40e_aq_query_vsi_ets_sla_config(hw, vsi->seid, &bw_ets_config, 5085 NULL); 5086 if (ret) { 5087 dev_info(&pf->pdev->dev, 5088 "couldn't get PF vsi ets bw config, err %s aq_err %s\n", 5089 i40e_stat_str(&pf->hw, ret), 5090 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 5091 return -EINVAL; 5092 } 5093 5094 if (bw_config.tc_valid_bits != bw_ets_config.tc_valid_bits) { 5095 dev_info(&pf->pdev->dev, 5096 "Enabled TCs mismatch from querying VSI BW info 0x%08x 0x%08x\n", 5097 bw_config.tc_valid_bits, 5098 bw_ets_config.tc_valid_bits); 5099 /* Still continuing */ 5100 } 5101 5102 vsi->bw_limit = le16_to_cpu(bw_config.port_bw_limit); 5103 vsi->bw_max_quanta = bw_config.max_bw; 5104 tc_bw_max = le16_to_cpu(bw_ets_config.tc_bw_max[0]) | 5105 (le16_to_cpu(bw_ets_config.tc_bw_max[1]) << 16); 5106 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 5107 vsi->bw_ets_share_credits[i] = bw_ets_config.share_credits[i]; 5108 vsi->bw_ets_limit_credits[i] = 5109 le16_to_cpu(bw_ets_config.credits[i]); 5110 /* 3 bits out of 4 for each TC */ 5111 vsi->bw_ets_max_quanta[i] = (u8)((tc_bw_max >> (i*4)) & 0x7); 5112 } 5113 5114 return 0; 5115 } 5116 5117 /** 5118 * i40e_vsi_configure_bw_alloc - Configure VSI BW allocation per TC 5119 * @vsi: the VSI being configured 5120 * @enabled_tc: TC bitmap 5121 * @bw_credits: BW shared credits per TC 5122 * 5123 * Returns 0 on success, negative value on failure 5124 **/ 5125 static int i40e_vsi_configure_bw_alloc(struct i40e_vsi *vsi, u8 enabled_tc, 5126 u8 *bw_share) 5127 { 5128 struct i40e_aqc_configure_vsi_tc_bw_data bw_data; 5129 i40e_status ret; 5130 int i; 5131 5132 if (vsi->back->flags & I40E_FLAG_TC_MQPRIO) 5133 return 0; 5134 if (!vsi->mqprio_qopt.qopt.hw) { 5135 ret = i40e_set_bw_limit(vsi, vsi->seid, 0); 5136 if (ret) 5137 dev_info(&vsi->back->pdev->dev, 5138 "Failed to reset tx rate for vsi->seid %u\n", 5139 vsi->seid); 5140 return ret; 5141 } 5142 bw_data.tc_valid_bits = enabled_tc; 5143 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) 5144 bw_data.tc_bw_credits[i] = bw_share[i]; 5145 5146 ret = i40e_aq_config_vsi_tc_bw(&vsi->back->hw, vsi->seid, &bw_data, 5147 NULL); 5148 if (ret) { 5149 dev_info(&vsi->back->pdev->dev, 5150 "AQ command Config VSI BW allocation per TC failed = %d\n", 5151 vsi->back->hw.aq.asq_last_status); 5152 return -EINVAL; 5153 } 5154 5155 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) 5156 vsi->info.qs_handle[i] = bw_data.qs_handles[i]; 5157 5158 return 0; 5159 } 5160 5161 /** 5162 * i40e_vsi_config_netdev_tc - Setup the netdev TC configuration 5163 * @vsi: the VSI being configured 5164 * @enabled_tc: TC map to be enabled 5165 * 5166 **/ 5167 static void i40e_vsi_config_netdev_tc(struct i40e_vsi *vsi, u8 enabled_tc) 5168 { 5169 struct net_device *netdev = vsi->netdev; 5170 struct i40e_pf *pf = vsi->back; 5171 struct i40e_hw *hw = &pf->hw; 5172 u8 netdev_tc = 0; 5173 int i; 5174 struct i40e_dcbx_config *dcbcfg = &hw->local_dcbx_config; 5175 5176 if (!netdev) 5177 return; 5178 5179 if (!enabled_tc) { 5180 netdev_reset_tc(netdev); 5181 return; 5182 } 5183 5184 /* Set up actual enabled TCs on the VSI */ 5185 if (netdev_set_num_tc(netdev, vsi->tc_config.numtc)) 5186 return; 5187 5188 /* set per TC queues for the VSI */ 5189 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 5190 /* Only set TC queues for enabled tcs 5191 * 5192 * e.g. For a VSI that has TC0 and TC3 enabled the 5193 * enabled_tc bitmap would be 0x00001001; the driver 5194 * will set the numtc for netdev as 2 that will be 5195 * referenced by the netdev layer as TC 0 and 1. 5196 */ 5197 if (vsi->tc_config.enabled_tc & BIT(i)) 5198 netdev_set_tc_queue(netdev, 5199 vsi->tc_config.tc_info[i].netdev_tc, 5200 vsi->tc_config.tc_info[i].qcount, 5201 vsi->tc_config.tc_info[i].qoffset); 5202 } 5203 5204 if (pf->flags & I40E_FLAG_TC_MQPRIO) 5205 return; 5206 5207 /* Assign UP2TC map for the VSI */ 5208 for (i = 0; i < I40E_MAX_USER_PRIORITY; i++) { 5209 /* Get the actual TC# for the UP */ 5210 u8 ets_tc = dcbcfg->etscfg.prioritytable[i]; 5211 /* Get the mapped netdev TC# for the UP */ 5212 netdev_tc = vsi->tc_config.tc_info[ets_tc].netdev_tc; 5213 netdev_set_prio_tc_map(netdev, i, netdev_tc); 5214 } 5215 } 5216 5217 /** 5218 * i40e_vsi_update_queue_map - Update our copy of VSi info with new queue map 5219 * @vsi: the VSI being configured 5220 * @ctxt: the ctxt buffer returned from AQ VSI update param command 5221 **/ 5222 static void i40e_vsi_update_queue_map(struct i40e_vsi *vsi, 5223 struct i40e_vsi_context *ctxt) 5224 { 5225 /* copy just the sections touched not the entire info 5226 * since not all sections are valid as returned by 5227 * update vsi params 5228 */ 5229 vsi->info.mapping_flags = ctxt->info.mapping_flags; 5230 memcpy(&vsi->info.queue_mapping, 5231 &ctxt->info.queue_mapping, sizeof(vsi->info.queue_mapping)); 5232 memcpy(&vsi->info.tc_mapping, ctxt->info.tc_mapping, 5233 sizeof(vsi->info.tc_mapping)); 5234 } 5235 5236 /** 5237 * i40e_vsi_config_tc - Configure VSI Tx Scheduler for given TC map 5238 * @vsi: VSI to be configured 5239 * @enabled_tc: TC bitmap 5240 * 5241 * This configures a particular VSI for TCs that are mapped to the 5242 * given TC bitmap. It uses default bandwidth share for TCs across 5243 * VSIs to configure TC for a particular VSI. 5244 * 5245 * NOTE: 5246 * It is expected that the VSI queues have been quisced before calling 5247 * this function. 5248 **/ 5249 static int i40e_vsi_config_tc(struct i40e_vsi *vsi, u8 enabled_tc) 5250 { 5251 u8 bw_share[I40E_MAX_TRAFFIC_CLASS] = {0}; 5252 struct i40e_pf *pf = vsi->back; 5253 struct i40e_hw *hw = &pf->hw; 5254 struct i40e_vsi_context ctxt; 5255 int ret = 0; 5256 int i; 5257 5258 /* Check if enabled_tc is same as existing or new TCs */ 5259 if (vsi->tc_config.enabled_tc == enabled_tc && 5260 vsi->mqprio_qopt.mode != TC_MQPRIO_MODE_CHANNEL) 5261 return ret; 5262 5263 /* Enable ETS TCs with equal BW Share for now across all VSIs */ 5264 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 5265 if (enabled_tc & BIT(i)) 5266 bw_share[i] = 1; 5267 } 5268 5269 ret = i40e_vsi_configure_bw_alloc(vsi, enabled_tc, bw_share); 5270 if (ret) { 5271 struct i40e_aqc_query_vsi_bw_config_resp bw_config = {0}; 5272 5273 dev_info(&pf->pdev->dev, 5274 "Failed configuring TC map %d for VSI %d\n", 5275 enabled_tc, vsi->seid); 5276 ret = i40e_aq_query_vsi_bw_config(hw, vsi->seid, 5277 &bw_config, NULL); 5278 if (ret) { 5279 dev_info(&pf->pdev->dev, 5280 "Failed querying vsi bw info, err %s aq_err %s\n", 5281 i40e_stat_str(hw, ret), 5282 i40e_aq_str(hw, hw->aq.asq_last_status)); 5283 goto out; 5284 } 5285 if ((bw_config.tc_valid_bits & enabled_tc) != enabled_tc) { 5286 u8 valid_tc = bw_config.tc_valid_bits & enabled_tc; 5287 5288 if (!valid_tc) 5289 valid_tc = bw_config.tc_valid_bits; 5290 /* Always enable TC0, no matter what */ 5291 valid_tc |= 1; 5292 dev_info(&pf->pdev->dev, 5293 "Requested tc 0x%x, but FW reports 0x%x as valid. Attempting to use 0x%x.\n", 5294 enabled_tc, bw_config.tc_valid_bits, valid_tc); 5295 enabled_tc = valid_tc; 5296 } 5297 5298 ret = i40e_vsi_configure_bw_alloc(vsi, enabled_tc, bw_share); 5299 if (ret) { 5300 dev_err(&pf->pdev->dev, 5301 "Unable to configure TC map %d for VSI %d\n", 5302 enabled_tc, vsi->seid); 5303 goto out; 5304 } 5305 } 5306 5307 /* Update Queue Pairs Mapping for currently enabled UPs */ 5308 ctxt.seid = vsi->seid; 5309 ctxt.pf_num = vsi->back->hw.pf_id; 5310 ctxt.vf_num = 0; 5311 ctxt.uplink_seid = vsi->uplink_seid; 5312 ctxt.info = vsi->info; 5313 if (vsi->back->flags & I40E_FLAG_TC_MQPRIO) { 5314 ret = i40e_vsi_setup_queue_map_mqprio(vsi, &ctxt, enabled_tc); 5315 if (ret) 5316 goto out; 5317 } else { 5318 i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, false); 5319 } 5320 5321 /* On destroying the qdisc, reset vsi->rss_size, as number of enabled 5322 * queues changed. 5323 */ 5324 if (!vsi->mqprio_qopt.qopt.hw && vsi->reconfig_rss) { 5325 vsi->rss_size = min_t(int, vsi->back->alloc_rss_size, 5326 vsi->num_queue_pairs); 5327 ret = i40e_vsi_config_rss(vsi); 5328 if (ret) { 5329 dev_info(&vsi->back->pdev->dev, 5330 "Failed to reconfig rss for num_queues\n"); 5331 return ret; 5332 } 5333 vsi->reconfig_rss = false; 5334 } 5335 if (vsi->back->flags & I40E_FLAG_IWARP_ENABLED) { 5336 ctxt.info.valid_sections |= 5337 cpu_to_le16(I40E_AQ_VSI_PROP_QUEUE_OPT_VALID); 5338 ctxt.info.queueing_opt_flags |= I40E_AQ_VSI_QUE_OPT_TCP_ENA; 5339 } 5340 5341 /* Update the VSI after updating the VSI queue-mapping 5342 * information 5343 */ 5344 ret = i40e_aq_update_vsi_params(hw, &ctxt, NULL); 5345 if (ret) { 5346 dev_info(&pf->pdev->dev, 5347 "Update vsi tc config failed, err %s aq_err %s\n", 5348 i40e_stat_str(hw, ret), 5349 i40e_aq_str(hw, hw->aq.asq_last_status)); 5350 goto out; 5351 } 5352 /* update the local VSI info with updated queue map */ 5353 i40e_vsi_update_queue_map(vsi, &ctxt); 5354 vsi->info.valid_sections = 0; 5355 5356 /* Update current VSI BW information */ 5357 ret = i40e_vsi_get_bw_info(vsi); 5358 if (ret) { 5359 dev_info(&pf->pdev->dev, 5360 "Failed updating vsi bw info, err %s aq_err %s\n", 5361 i40e_stat_str(hw, ret), 5362 i40e_aq_str(hw, hw->aq.asq_last_status)); 5363 goto out; 5364 } 5365 5366 /* Update the netdev TC setup */ 5367 i40e_vsi_config_netdev_tc(vsi, enabled_tc); 5368 out: 5369 return ret; 5370 } 5371 5372 /** 5373 * i40e_get_link_speed - Returns link speed for the interface 5374 * @vsi: VSI to be configured 5375 * 5376 **/ 5377 int i40e_get_link_speed(struct i40e_vsi *vsi) 5378 { 5379 struct i40e_pf *pf = vsi->back; 5380 5381 switch (pf->hw.phy.link_info.link_speed) { 5382 case I40E_LINK_SPEED_40GB: 5383 return 40000; 5384 case I40E_LINK_SPEED_25GB: 5385 return 25000; 5386 case I40E_LINK_SPEED_20GB: 5387 return 20000; 5388 case I40E_LINK_SPEED_10GB: 5389 return 10000; 5390 case I40E_LINK_SPEED_1GB: 5391 return 1000; 5392 default: 5393 return -EINVAL; 5394 } 5395 } 5396 5397 /** 5398 * i40e_set_bw_limit - setup BW limit for Tx traffic based on max_tx_rate 5399 * @vsi: VSI to be configured 5400 * @seid: seid of the channel/VSI 5401 * @max_tx_rate: max TX rate to be configured as BW limit 5402 * 5403 * Helper function to set BW limit for a given VSI 5404 **/ 5405 int i40e_set_bw_limit(struct i40e_vsi *vsi, u16 seid, u64 max_tx_rate) 5406 { 5407 struct i40e_pf *pf = vsi->back; 5408 u64 credits = 0; 5409 int speed = 0; 5410 int ret = 0; 5411 5412 speed = i40e_get_link_speed(vsi); 5413 if (max_tx_rate > speed) { 5414 dev_err(&pf->pdev->dev, 5415 "Invalid max tx rate %llu specified for VSI seid %d.", 5416 max_tx_rate, seid); 5417 return -EINVAL; 5418 } 5419 if (max_tx_rate && max_tx_rate < 50) { 5420 dev_warn(&pf->pdev->dev, 5421 "Setting max tx rate to minimum usable value of 50Mbps.\n"); 5422 max_tx_rate = 50; 5423 } 5424 5425 /* Tx rate credits are in values of 50Mbps, 0 is disabled */ 5426 credits = max_tx_rate; 5427 do_div(credits, I40E_BW_CREDIT_DIVISOR); 5428 ret = i40e_aq_config_vsi_bw_limit(&pf->hw, seid, credits, 5429 I40E_MAX_BW_INACTIVE_ACCUM, NULL); 5430 if (ret) 5431 dev_err(&pf->pdev->dev, 5432 "Failed set tx rate (%llu Mbps) for vsi->seid %u, err %s aq_err %s\n", 5433 max_tx_rate, seid, i40e_stat_str(&pf->hw, ret), 5434 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 5435 return ret; 5436 } 5437 5438 /** 5439 * i40e_remove_queue_channels - Remove queue channels for the TCs 5440 * @vsi: VSI to be configured 5441 * 5442 * Remove queue channels for the TCs 5443 **/ 5444 static void i40e_remove_queue_channels(struct i40e_vsi *vsi) 5445 { 5446 enum i40e_admin_queue_err last_aq_status; 5447 struct i40e_cloud_filter *cfilter; 5448 struct i40e_channel *ch, *ch_tmp; 5449 struct i40e_pf *pf = vsi->back; 5450 struct hlist_node *node; 5451 int ret, i; 5452 5453 /* Reset rss size that was stored when reconfiguring rss for 5454 * channel VSIs with non-power-of-2 queue count. 5455 */ 5456 vsi->current_rss_size = 0; 5457 5458 /* perform cleanup for channels if they exist */ 5459 if (list_empty(&vsi->ch_list)) 5460 return; 5461 5462 list_for_each_entry_safe(ch, ch_tmp, &vsi->ch_list, list) { 5463 struct i40e_vsi *p_vsi; 5464 5465 list_del(&ch->list); 5466 p_vsi = ch->parent_vsi; 5467 if (!p_vsi || !ch->initialized) { 5468 kfree(ch); 5469 continue; 5470 } 5471 /* Reset queue contexts */ 5472 for (i = 0; i < ch->num_queue_pairs; i++) { 5473 struct i40e_ring *tx_ring, *rx_ring; 5474 u16 pf_q; 5475 5476 pf_q = ch->base_queue + i; 5477 tx_ring = vsi->tx_rings[pf_q]; 5478 tx_ring->ch = NULL; 5479 5480 rx_ring = vsi->rx_rings[pf_q]; 5481 rx_ring->ch = NULL; 5482 } 5483 5484 /* Reset BW configured for this VSI via mqprio */ 5485 ret = i40e_set_bw_limit(vsi, ch->seid, 0); 5486 if (ret) 5487 dev_info(&vsi->back->pdev->dev, 5488 "Failed to reset tx rate for ch->seid %u\n", 5489 ch->seid); 5490 5491 /* delete cloud filters associated with this channel */ 5492 hlist_for_each_entry_safe(cfilter, node, 5493 &pf->cloud_filter_list, cloud_node) { 5494 if (cfilter->seid != ch->seid) 5495 continue; 5496 5497 hash_del(&cfilter->cloud_node); 5498 if (cfilter->dst_port) 5499 ret = i40e_add_del_cloud_filter_big_buf(vsi, 5500 cfilter, 5501 false); 5502 else 5503 ret = i40e_add_del_cloud_filter(vsi, cfilter, 5504 false); 5505 last_aq_status = pf->hw.aq.asq_last_status; 5506 if (ret) 5507 dev_info(&pf->pdev->dev, 5508 "Failed to delete cloud filter, err %s aq_err %s\n", 5509 i40e_stat_str(&pf->hw, ret), 5510 i40e_aq_str(&pf->hw, last_aq_status)); 5511 kfree(cfilter); 5512 } 5513 5514 /* delete VSI from FW */ 5515 ret = i40e_aq_delete_element(&vsi->back->hw, ch->seid, 5516 NULL); 5517 if (ret) 5518 dev_err(&vsi->back->pdev->dev, 5519 "unable to remove channel (%d) for parent VSI(%d)\n", 5520 ch->seid, p_vsi->seid); 5521 kfree(ch); 5522 } 5523 INIT_LIST_HEAD(&vsi->ch_list); 5524 } 5525 5526 /** 5527 * i40e_is_any_channel - channel exist or not 5528 * @vsi: ptr to VSI to which channels are associated with 5529 * 5530 * Returns true or false if channel(s) exist for associated VSI or not 5531 **/ 5532 static bool i40e_is_any_channel(struct i40e_vsi *vsi) 5533 { 5534 struct i40e_channel *ch, *ch_tmp; 5535 5536 list_for_each_entry_safe(ch, ch_tmp, &vsi->ch_list, list) { 5537 if (ch->initialized) 5538 return true; 5539 } 5540 5541 return false; 5542 } 5543 5544 /** 5545 * i40e_get_max_queues_for_channel 5546 * @vsi: ptr to VSI to which channels are associated with 5547 * 5548 * Helper function which returns max value among the queue counts set on the 5549 * channels/TCs created. 5550 **/ 5551 static int i40e_get_max_queues_for_channel(struct i40e_vsi *vsi) 5552 { 5553 struct i40e_channel *ch, *ch_tmp; 5554 int max = 0; 5555 5556 list_for_each_entry_safe(ch, ch_tmp, &vsi->ch_list, list) { 5557 if (!ch->initialized) 5558 continue; 5559 if (ch->num_queue_pairs > max) 5560 max = ch->num_queue_pairs; 5561 } 5562 5563 return max; 5564 } 5565 5566 /** 5567 * i40e_validate_num_queues - validate num_queues w.r.t channel 5568 * @pf: ptr to PF device 5569 * @num_queues: number of queues 5570 * @vsi: the parent VSI 5571 * @reconfig_rss: indicates should the RSS be reconfigured or not 5572 * 5573 * This function validates number of queues in the context of new channel 5574 * which is being established and determines if RSS should be reconfigured 5575 * or not for parent VSI. 5576 **/ 5577 static int i40e_validate_num_queues(struct i40e_pf *pf, int num_queues, 5578 struct i40e_vsi *vsi, bool *reconfig_rss) 5579 { 5580 int max_ch_queues; 5581 5582 if (!reconfig_rss) 5583 return -EINVAL; 5584 5585 *reconfig_rss = false; 5586 if (vsi->current_rss_size) { 5587 if (num_queues > vsi->current_rss_size) { 5588 dev_dbg(&pf->pdev->dev, 5589 "Error: num_queues (%d) > vsi's current_size(%d)\n", 5590 num_queues, vsi->current_rss_size); 5591 return -EINVAL; 5592 } else if ((num_queues < vsi->current_rss_size) && 5593 (!is_power_of_2(num_queues))) { 5594 dev_dbg(&pf->pdev->dev, 5595 "Error: num_queues (%d) < vsi's current_size(%d), but not power of 2\n", 5596 num_queues, vsi->current_rss_size); 5597 return -EINVAL; 5598 } 5599 } 5600 5601 if (!is_power_of_2(num_queues)) { 5602 /* Find the max num_queues configured for channel if channel 5603 * exist. 5604 * if channel exist, then enforce 'num_queues' to be more than 5605 * max ever queues configured for channel. 5606 */ 5607 max_ch_queues = i40e_get_max_queues_for_channel(vsi); 5608 if (num_queues < max_ch_queues) { 5609 dev_dbg(&pf->pdev->dev, 5610 "Error: num_queues (%d) < max queues configured for channel(%d)\n", 5611 num_queues, max_ch_queues); 5612 return -EINVAL; 5613 } 5614 *reconfig_rss = true; 5615 } 5616 5617 return 0; 5618 } 5619 5620 /** 5621 * i40e_vsi_reconfig_rss - reconfig RSS based on specified rss_size 5622 * @vsi: the VSI being setup 5623 * @rss_size: size of RSS, accordingly LUT gets reprogrammed 5624 * 5625 * This function reconfigures RSS by reprogramming LUTs using 'rss_size' 5626 **/ 5627 static int i40e_vsi_reconfig_rss(struct i40e_vsi *vsi, u16 rss_size) 5628 { 5629 struct i40e_pf *pf = vsi->back; 5630 u8 seed[I40E_HKEY_ARRAY_SIZE]; 5631 struct i40e_hw *hw = &pf->hw; 5632 int local_rss_size; 5633 u8 *lut; 5634 int ret; 5635 5636 if (!vsi->rss_size) 5637 return -EINVAL; 5638 5639 if (rss_size > vsi->rss_size) 5640 return -EINVAL; 5641 5642 local_rss_size = min_t(int, vsi->rss_size, rss_size); 5643 lut = kzalloc(vsi->rss_table_size, GFP_KERNEL); 5644 if (!lut) 5645 return -ENOMEM; 5646 5647 /* Ignoring user configured lut if there is one */ 5648 i40e_fill_rss_lut(pf, lut, vsi->rss_table_size, local_rss_size); 5649 5650 /* Use user configured hash key if there is one, otherwise 5651 * use default. 5652 */ 5653 if (vsi->rss_hkey_user) 5654 memcpy(seed, vsi->rss_hkey_user, I40E_HKEY_ARRAY_SIZE); 5655 else 5656 netdev_rss_key_fill((void *)seed, I40E_HKEY_ARRAY_SIZE); 5657 5658 ret = i40e_config_rss(vsi, seed, lut, vsi->rss_table_size); 5659 if (ret) { 5660 dev_info(&pf->pdev->dev, 5661 "Cannot set RSS lut, err %s aq_err %s\n", 5662 i40e_stat_str(hw, ret), 5663 i40e_aq_str(hw, hw->aq.asq_last_status)); 5664 kfree(lut); 5665 return ret; 5666 } 5667 kfree(lut); 5668 5669 /* Do the update w.r.t. storing rss_size */ 5670 if (!vsi->orig_rss_size) 5671 vsi->orig_rss_size = vsi->rss_size; 5672 vsi->current_rss_size = local_rss_size; 5673 5674 return ret; 5675 } 5676 5677 /** 5678 * i40e_channel_setup_queue_map - Setup a channel queue map 5679 * @pf: ptr to PF device 5680 * @vsi: the VSI being setup 5681 * @ctxt: VSI context structure 5682 * @ch: ptr to channel structure 5683 * 5684 * Setup queue map for a specific channel 5685 **/ 5686 static void i40e_channel_setup_queue_map(struct i40e_pf *pf, 5687 struct i40e_vsi_context *ctxt, 5688 struct i40e_channel *ch) 5689 { 5690 u16 qcount, qmap, sections = 0; 5691 u8 offset = 0; 5692 int pow; 5693 5694 sections = I40E_AQ_VSI_PROP_QUEUE_MAP_VALID; 5695 sections |= I40E_AQ_VSI_PROP_SCHED_VALID; 5696 5697 qcount = min_t(int, ch->num_queue_pairs, pf->num_lan_msix); 5698 ch->num_queue_pairs = qcount; 5699 5700 /* find the next higher power-of-2 of num queue pairs */ 5701 pow = ilog2(qcount); 5702 if (!is_power_of_2(qcount)) 5703 pow++; 5704 5705 qmap = (offset << I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT) | 5706 (pow << I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT); 5707 5708 /* Setup queue TC[0].qmap for given VSI context */ 5709 ctxt->info.tc_mapping[0] = cpu_to_le16(qmap); 5710 5711 ctxt->info.up_enable_bits = 0x1; /* TC0 enabled */ 5712 ctxt->info.mapping_flags |= cpu_to_le16(I40E_AQ_VSI_QUE_MAP_CONTIG); 5713 ctxt->info.queue_mapping[0] = cpu_to_le16(ch->base_queue); 5714 ctxt->info.valid_sections |= cpu_to_le16(sections); 5715 } 5716 5717 /** 5718 * i40e_add_channel - add a channel by adding VSI 5719 * @pf: ptr to PF device 5720 * @uplink_seid: underlying HW switching element (VEB) ID 5721 * @ch: ptr to channel structure 5722 * 5723 * Add a channel (VSI) using add_vsi and queue_map 5724 **/ 5725 static int i40e_add_channel(struct i40e_pf *pf, u16 uplink_seid, 5726 struct i40e_channel *ch) 5727 { 5728 struct i40e_hw *hw = &pf->hw; 5729 struct i40e_vsi_context ctxt; 5730 u8 enabled_tc = 0x1; /* TC0 enabled */ 5731 int ret; 5732 5733 if (ch->type != I40E_VSI_VMDQ2) { 5734 dev_info(&pf->pdev->dev, 5735 "add new vsi failed, ch->type %d\n", ch->type); 5736 return -EINVAL; 5737 } 5738 5739 memset(&ctxt, 0, sizeof(ctxt)); 5740 ctxt.pf_num = hw->pf_id; 5741 ctxt.vf_num = 0; 5742 ctxt.uplink_seid = uplink_seid; 5743 ctxt.connection_type = I40E_AQ_VSI_CONN_TYPE_NORMAL; 5744 if (ch->type == I40E_VSI_VMDQ2) 5745 ctxt.flags = I40E_AQ_VSI_TYPE_VMDQ2; 5746 5747 if (pf->flags & I40E_FLAG_VEB_MODE_ENABLED) { 5748 ctxt.info.valid_sections |= 5749 cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID); 5750 ctxt.info.switch_id = 5751 cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB); 5752 } 5753 5754 /* Set queue map for a given VSI context */ 5755 i40e_channel_setup_queue_map(pf, &ctxt, ch); 5756 5757 /* Now time to create VSI */ 5758 ret = i40e_aq_add_vsi(hw, &ctxt, NULL); 5759 if (ret) { 5760 dev_info(&pf->pdev->dev, 5761 "add new vsi failed, err %s aq_err %s\n", 5762 i40e_stat_str(&pf->hw, ret), 5763 i40e_aq_str(&pf->hw, 5764 pf->hw.aq.asq_last_status)); 5765 return -ENOENT; 5766 } 5767 5768 /* Success, update channel */ 5769 ch->enabled_tc = enabled_tc; 5770 ch->seid = ctxt.seid; 5771 ch->vsi_number = ctxt.vsi_number; 5772 ch->stat_counter_idx = cpu_to_le16(ctxt.info.stat_counter_idx); 5773 5774 /* copy just the sections touched not the entire info 5775 * since not all sections are valid as returned by 5776 * update vsi params 5777 */ 5778 ch->info.mapping_flags = ctxt.info.mapping_flags; 5779 memcpy(&ch->info.queue_mapping, 5780 &ctxt.info.queue_mapping, sizeof(ctxt.info.queue_mapping)); 5781 memcpy(&ch->info.tc_mapping, ctxt.info.tc_mapping, 5782 sizeof(ctxt.info.tc_mapping)); 5783 5784 return 0; 5785 } 5786 5787 static int i40e_channel_config_bw(struct i40e_vsi *vsi, struct i40e_channel *ch, 5788 u8 *bw_share) 5789 { 5790 struct i40e_aqc_configure_vsi_tc_bw_data bw_data; 5791 i40e_status ret; 5792 int i; 5793 5794 bw_data.tc_valid_bits = ch->enabled_tc; 5795 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) 5796 bw_data.tc_bw_credits[i] = bw_share[i]; 5797 5798 ret = i40e_aq_config_vsi_tc_bw(&vsi->back->hw, ch->seid, 5799 &bw_data, NULL); 5800 if (ret) { 5801 dev_info(&vsi->back->pdev->dev, 5802 "Config VSI BW allocation per TC failed, aq_err: %d for new_vsi->seid %u\n", 5803 vsi->back->hw.aq.asq_last_status, ch->seid); 5804 return -EINVAL; 5805 } 5806 5807 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) 5808 ch->info.qs_handle[i] = bw_data.qs_handles[i]; 5809 5810 return 0; 5811 } 5812 5813 /** 5814 * i40e_channel_config_tx_ring - config TX ring associated with new channel 5815 * @pf: ptr to PF device 5816 * @vsi: the VSI being setup 5817 * @ch: ptr to channel structure 5818 * 5819 * Configure TX rings associated with channel (VSI) since queues are being 5820 * from parent VSI. 5821 **/ 5822 static int i40e_channel_config_tx_ring(struct i40e_pf *pf, 5823 struct i40e_vsi *vsi, 5824 struct i40e_channel *ch) 5825 { 5826 i40e_status ret; 5827 int i; 5828 u8 bw_share[I40E_MAX_TRAFFIC_CLASS] = {0}; 5829 5830 /* Enable ETS TCs with equal BW Share for now across all VSIs */ 5831 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 5832 if (ch->enabled_tc & BIT(i)) 5833 bw_share[i] = 1; 5834 } 5835 5836 /* configure BW for new VSI */ 5837 ret = i40e_channel_config_bw(vsi, ch, bw_share); 5838 if (ret) { 5839 dev_info(&vsi->back->pdev->dev, 5840 "Failed configuring TC map %d for channel (seid %u)\n", 5841 ch->enabled_tc, ch->seid); 5842 return ret; 5843 } 5844 5845 for (i = 0; i < ch->num_queue_pairs; i++) { 5846 struct i40e_ring *tx_ring, *rx_ring; 5847 u16 pf_q; 5848 5849 pf_q = ch->base_queue + i; 5850 5851 /* Get to TX ring ptr of main VSI, for re-setup TX queue 5852 * context 5853 */ 5854 tx_ring = vsi->tx_rings[pf_q]; 5855 tx_ring->ch = ch; 5856 5857 /* Get the RX ring ptr */ 5858 rx_ring = vsi->rx_rings[pf_q]; 5859 rx_ring->ch = ch; 5860 } 5861 5862 return 0; 5863 } 5864 5865 /** 5866 * i40e_setup_hw_channel - setup new channel 5867 * @pf: ptr to PF device 5868 * @vsi: the VSI being setup 5869 * @ch: ptr to channel structure 5870 * @uplink_seid: underlying HW switching element (VEB) ID 5871 * @type: type of channel to be created (VMDq2/VF) 5872 * 5873 * Setup new channel (VSI) based on specified type (VMDq2/VF) 5874 * and configures TX rings accordingly 5875 **/ 5876 static inline int i40e_setup_hw_channel(struct i40e_pf *pf, 5877 struct i40e_vsi *vsi, 5878 struct i40e_channel *ch, 5879 u16 uplink_seid, u8 type) 5880 { 5881 int ret; 5882 5883 ch->initialized = false; 5884 ch->base_queue = vsi->next_base_queue; 5885 ch->type = type; 5886 5887 /* Proceed with creation of channel (VMDq2) VSI */ 5888 ret = i40e_add_channel(pf, uplink_seid, ch); 5889 if (ret) { 5890 dev_info(&pf->pdev->dev, 5891 "failed to add_channel using uplink_seid %u\n", 5892 uplink_seid); 5893 return ret; 5894 } 5895 5896 /* Mark the successful creation of channel */ 5897 ch->initialized = true; 5898 5899 /* Reconfigure TX queues using QTX_CTL register */ 5900 ret = i40e_channel_config_tx_ring(pf, vsi, ch); 5901 if (ret) { 5902 dev_info(&pf->pdev->dev, 5903 "failed to configure TX rings for channel %u\n", 5904 ch->seid); 5905 return ret; 5906 } 5907 5908 /* update 'next_base_queue' */ 5909 vsi->next_base_queue = vsi->next_base_queue + ch->num_queue_pairs; 5910 dev_dbg(&pf->pdev->dev, 5911 "Added channel: vsi_seid %u, vsi_number %u, stat_counter_idx %u, num_queue_pairs %u, pf->next_base_queue %d\n", 5912 ch->seid, ch->vsi_number, ch->stat_counter_idx, 5913 ch->num_queue_pairs, 5914 vsi->next_base_queue); 5915 return ret; 5916 } 5917 5918 /** 5919 * i40e_setup_channel - setup new channel using uplink element 5920 * @pf: ptr to PF device 5921 * @type: type of channel to be created (VMDq2/VF) 5922 * @uplink_seid: underlying HW switching element (VEB) ID 5923 * @ch: ptr to channel structure 5924 * 5925 * Setup new channel (VSI) based on specified type (VMDq2/VF) 5926 * and uplink switching element (uplink_seid) 5927 **/ 5928 static bool i40e_setup_channel(struct i40e_pf *pf, struct i40e_vsi *vsi, 5929 struct i40e_channel *ch) 5930 { 5931 u8 vsi_type; 5932 u16 seid; 5933 int ret; 5934 5935 if (vsi->type == I40E_VSI_MAIN) { 5936 vsi_type = I40E_VSI_VMDQ2; 5937 } else { 5938 dev_err(&pf->pdev->dev, "unsupported parent vsi type(%d)\n", 5939 vsi->type); 5940 return false; 5941 } 5942 5943 /* underlying switching element */ 5944 seid = pf->vsi[pf->lan_vsi]->uplink_seid; 5945 5946 /* create channel (VSI), configure TX rings */ 5947 ret = i40e_setup_hw_channel(pf, vsi, ch, seid, vsi_type); 5948 if (ret) { 5949 dev_err(&pf->pdev->dev, "failed to setup hw_channel\n"); 5950 return false; 5951 } 5952 5953 return ch->initialized ? true : false; 5954 } 5955 5956 /** 5957 * i40e_validate_and_set_switch_mode - sets up switch mode correctly 5958 * @vsi: ptr to VSI which has PF backing 5959 * 5960 * Sets up switch mode correctly if it needs to be changed and perform 5961 * what are allowed modes. 5962 **/ 5963 static int i40e_validate_and_set_switch_mode(struct i40e_vsi *vsi) 5964 { 5965 u8 mode; 5966 struct i40e_pf *pf = vsi->back; 5967 struct i40e_hw *hw = &pf->hw; 5968 int ret; 5969 5970 ret = i40e_get_capabilities(pf, i40e_aqc_opc_list_dev_capabilities); 5971 if (ret) 5972 return -EINVAL; 5973 5974 if (hw->dev_caps.switch_mode) { 5975 /* if switch mode is set, support mode2 (non-tunneled for 5976 * cloud filter) for now 5977 */ 5978 u32 switch_mode = hw->dev_caps.switch_mode & 5979 I40E_SWITCH_MODE_MASK; 5980 if (switch_mode >= I40E_CLOUD_FILTER_MODE1) { 5981 if (switch_mode == I40E_CLOUD_FILTER_MODE2) 5982 return 0; 5983 dev_err(&pf->pdev->dev, 5984 "Invalid switch_mode (%d), only non-tunneled mode for cloud filter is supported\n", 5985 hw->dev_caps.switch_mode); 5986 return -EINVAL; 5987 } 5988 } 5989 5990 /* Set Bit 7 to be valid */ 5991 mode = I40E_AQ_SET_SWITCH_BIT7_VALID; 5992 5993 /* Set L4type for TCP support */ 5994 mode |= I40E_AQ_SET_SWITCH_L4_TYPE_TCP; 5995 5996 /* Set cloud filter mode */ 5997 mode |= I40E_AQ_SET_SWITCH_MODE_NON_TUNNEL; 5998 5999 /* Prep mode field for set_switch_config */ 6000 ret = i40e_aq_set_switch_config(hw, pf->last_sw_conf_flags, 6001 pf->last_sw_conf_valid_flags, 6002 mode, NULL); 6003 if (ret && hw->aq.asq_last_status != I40E_AQ_RC_ESRCH) 6004 dev_err(&pf->pdev->dev, 6005 "couldn't set switch config bits, err %s aq_err %s\n", 6006 i40e_stat_str(hw, ret), 6007 i40e_aq_str(hw, 6008 hw->aq.asq_last_status)); 6009 6010 return ret; 6011 } 6012 6013 /** 6014 * i40e_create_queue_channel - function to create channel 6015 * @vsi: VSI to be configured 6016 * @ch: ptr to channel (it contains channel specific params) 6017 * 6018 * This function creates channel (VSI) using num_queues specified by user, 6019 * reconfigs RSS if needed. 6020 **/ 6021 int i40e_create_queue_channel(struct i40e_vsi *vsi, 6022 struct i40e_channel *ch) 6023 { 6024 struct i40e_pf *pf = vsi->back; 6025 bool reconfig_rss; 6026 int err; 6027 6028 if (!ch) 6029 return -EINVAL; 6030 6031 if (!ch->num_queue_pairs) { 6032 dev_err(&pf->pdev->dev, "Invalid num_queues requested: %d\n", 6033 ch->num_queue_pairs); 6034 return -EINVAL; 6035 } 6036 6037 /* validate user requested num_queues for channel */ 6038 err = i40e_validate_num_queues(pf, ch->num_queue_pairs, vsi, 6039 &reconfig_rss); 6040 if (err) { 6041 dev_info(&pf->pdev->dev, "Failed to validate num_queues (%d)\n", 6042 ch->num_queue_pairs); 6043 return -EINVAL; 6044 } 6045 6046 /* By default we are in VEPA mode, if this is the first VF/VMDq 6047 * VSI to be added switch to VEB mode. 6048 */ 6049 if ((!(pf->flags & I40E_FLAG_VEB_MODE_ENABLED)) || 6050 (!i40e_is_any_channel(vsi))) { 6051 if (!is_power_of_2(vsi->tc_config.tc_info[0].qcount)) { 6052 dev_dbg(&pf->pdev->dev, 6053 "Failed to create channel. Override queues (%u) not power of 2\n", 6054 vsi->tc_config.tc_info[0].qcount); 6055 return -EINVAL; 6056 } 6057 6058 if (!(pf->flags & I40E_FLAG_VEB_MODE_ENABLED)) { 6059 pf->flags |= I40E_FLAG_VEB_MODE_ENABLED; 6060 6061 if (vsi->type == I40E_VSI_MAIN) { 6062 if (pf->flags & I40E_FLAG_TC_MQPRIO) 6063 i40e_do_reset(pf, I40E_PF_RESET_FLAG, 6064 true); 6065 else 6066 i40e_do_reset_safe(pf, 6067 I40E_PF_RESET_FLAG); 6068 } 6069 } 6070 /* now onwards for main VSI, number of queues will be value 6071 * of TC0's queue count 6072 */ 6073 } 6074 6075 /* By this time, vsi->cnt_q_avail shall be set to non-zero and 6076 * it should be more than num_queues 6077 */ 6078 if (!vsi->cnt_q_avail || vsi->cnt_q_avail < ch->num_queue_pairs) { 6079 dev_dbg(&pf->pdev->dev, 6080 "Error: cnt_q_avail (%u) less than num_queues %d\n", 6081 vsi->cnt_q_avail, ch->num_queue_pairs); 6082 return -EINVAL; 6083 } 6084 6085 /* reconfig_rss only if vsi type is MAIN_VSI */ 6086 if (reconfig_rss && (vsi->type == I40E_VSI_MAIN)) { 6087 err = i40e_vsi_reconfig_rss(vsi, ch->num_queue_pairs); 6088 if (err) { 6089 dev_info(&pf->pdev->dev, 6090 "Error: unable to reconfig rss for num_queues (%u)\n", 6091 ch->num_queue_pairs); 6092 return -EINVAL; 6093 } 6094 } 6095 6096 if (!i40e_setup_channel(pf, vsi, ch)) { 6097 dev_info(&pf->pdev->dev, "Failed to setup channel\n"); 6098 return -EINVAL; 6099 } 6100 6101 dev_info(&pf->pdev->dev, 6102 "Setup channel (id:%u) utilizing num_queues %d\n", 6103 ch->seid, ch->num_queue_pairs); 6104 6105 /* configure VSI for BW limit */ 6106 if (ch->max_tx_rate) { 6107 u64 credits = ch->max_tx_rate; 6108 6109 if (i40e_set_bw_limit(vsi, ch->seid, ch->max_tx_rate)) 6110 return -EINVAL; 6111 6112 do_div(credits, I40E_BW_CREDIT_DIVISOR); 6113 dev_dbg(&pf->pdev->dev, 6114 "Set tx rate of %llu Mbps (count of 50Mbps %llu) for vsi->seid %u\n", 6115 ch->max_tx_rate, 6116 credits, 6117 ch->seid); 6118 } 6119 6120 /* in case of VF, this will be main SRIOV VSI */ 6121 ch->parent_vsi = vsi; 6122 6123 /* and update main_vsi's count for queue_available to use */ 6124 vsi->cnt_q_avail -= ch->num_queue_pairs; 6125 6126 return 0; 6127 } 6128 6129 /** 6130 * i40e_configure_queue_channels - Add queue channel for the given TCs 6131 * @vsi: VSI to be configured 6132 * 6133 * Configures queue channel mapping to the given TCs 6134 **/ 6135 static int i40e_configure_queue_channels(struct i40e_vsi *vsi) 6136 { 6137 struct i40e_channel *ch; 6138 u64 max_rate = 0; 6139 int ret = 0, i; 6140 6141 /* Create app vsi with the TCs. Main VSI with TC0 is already set up */ 6142 vsi->tc_seid_map[0] = vsi->seid; 6143 for (i = 1; i < I40E_MAX_TRAFFIC_CLASS; i++) { 6144 if (vsi->tc_config.enabled_tc & BIT(i)) { 6145 ch = kzalloc(sizeof(*ch), GFP_KERNEL); 6146 if (!ch) { 6147 ret = -ENOMEM; 6148 goto err_free; 6149 } 6150 6151 INIT_LIST_HEAD(&ch->list); 6152 ch->num_queue_pairs = 6153 vsi->tc_config.tc_info[i].qcount; 6154 ch->base_queue = 6155 vsi->tc_config.tc_info[i].qoffset; 6156 6157 /* Bandwidth limit through tc interface is in bytes/s, 6158 * change to Mbit/s 6159 */ 6160 max_rate = vsi->mqprio_qopt.max_rate[i]; 6161 do_div(max_rate, I40E_BW_MBPS_DIVISOR); 6162 ch->max_tx_rate = max_rate; 6163 6164 list_add_tail(&ch->list, &vsi->ch_list); 6165 6166 ret = i40e_create_queue_channel(vsi, ch); 6167 if (ret) { 6168 dev_err(&vsi->back->pdev->dev, 6169 "Failed creating queue channel with TC%d: queues %d\n", 6170 i, ch->num_queue_pairs); 6171 goto err_free; 6172 } 6173 vsi->tc_seid_map[i] = ch->seid; 6174 } 6175 } 6176 return ret; 6177 6178 err_free: 6179 i40e_remove_queue_channels(vsi); 6180 return ret; 6181 } 6182 6183 /** 6184 * i40e_veb_config_tc - Configure TCs for given VEB 6185 * @veb: given VEB 6186 * @enabled_tc: TC bitmap 6187 * 6188 * Configures given TC bitmap for VEB (switching) element 6189 **/ 6190 int i40e_veb_config_tc(struct i40e_veb *veb, u8 enabled_tc) 6191 { 6192 struct i40e_aqc_configure_switching_comp_bw_config_data bw_data = {0}; 6193 struct i40e_pf *pf = veb->pf; 6194 int ret = 0; 6195 int i; 6196 6197 /* No TCs or already enabled TCs just return */ 6198 if (!enabled_tc || veb->enabled_tc == enabled_tc) 6199 return ret; 6200 6201 bw_data.tc_valid_bits = enabled_tc; 6202 /* bw_data.absolute_credits is not set (relative) */ 6203 6204 /* Enable ETS TCs with equal BW Share for now */ 6205 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 6206 if (enabled_tc & BIT(i)) 6207 bw_data.tc_bw_share_credits[i] = 1; 6208 } 6209 6210 ret = i40e_aq_config_switch_comp_bw_config(&pf->hw, veb->seid, 6211 &bw_data, NULL); 6212 if (ret) { 6213 dev_info(&pf->pdev->dev, 6214 "VEB bw config failed, err %s aq_err %s\n", 6215 i40e_stat_str(&pf->hw, ret), 6216 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 6217 goto out; 6218 } 6219 6220 /* Update the BW information */ 6221 ret = i40e_veb_get_bw_info(veb); 6222 if (ret) { 6223 dev_info(&pf->pdev->dev, 6224 "Failed getting veb bw config, err %s aq_err %s\n", 6225 i40e_stat_str(&pf->hw, ret), 6226 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 6227 } 6228 6229 out: 6230 return ret; 6231 } 6232 6233 #ifdef CONFIG_I40E_DCB 6234 /** 6235 * i40e_dcb_reconfigure - Reconfigure all VEBs and VSIs 6236 * @pf: PF struct 6237 * 6238 * Reconfigure VEB/VSIs on a given PF; it is assumed that 6239 * the caller would've quiesce all the VSIs before calling 6240 * this function 6241 **/ 6242 static void i40e_dcb_reconfigure(struct i40e_pf *pf) 6243 { 6244 u8 tc_map = 0; 6245 int ret; 6246 u8 v; 6247 6248 /* Enable the TCs available on PF to all VEBs */ 6249 tc_map = i40e_pf_get_tc_map(pf); 6250 for (v = 0; v < I40E_MAX_VEB; v++) { 6251 if (!pf->veb[v]) 6252 continue; 6253 ret = i40e_veb_config_tc(pf->veb[v], tc_map); 6254 if (ret) { 6255 dev_info(&pf->pdev->dev, 6256 "Failed configuring TC for VEB seid=%d\n", 6257 pf->veb[v]->seid); 6258 /* Will try to configure as many components */ 6259 } 6260 } 6261 6262 /* Update each VSI */ 6263 for (v = 0; v < pf->num_alloc_vsi; v++) { 6264 if (!pf->vsi[v]) 6265 continue; 6266 6267 /* - Enable all TCs for the LAN VSI 6268 * - For all others keep them at TC0 for now 6269 */ 6270 if (v == pf->lan_vsi) 6271 tc_map = i40e_pf_get_tc_map(pf); 6272 else 6273 tc_map = I40E_DEFAULT_TRAFFIC_CLASS; 6274 6275 ret = i40e_vsi_config_tc(pf->vsi[v], tc_map); 6276 if (ret) { 6277 dev_info(&pf->pdev->dev, 6278 "Failed configuring TC for VSI seid=%d\n", 6279 pf->vsi[v]->seid); 6280 /* Will try to configure as many components */ 6281 } else { 6282 /* Re-configure VSI vectors based on updated TC map */ 6283 i40e_vsi_map_rings_to_vectors(pf->vsi[v]); 6284 if (pf->vsi[v]->netdev) 6285 i40e_dcbnl_set_all(pf->vsi[v]); 6286 } 6287 } 6288 } 6289 6290 /** 6291 * i40e_resume_port_tx - Resume port Tx 6292 * @pf: PF struct 6293 * 6294 * Resume a port's Tx and issue a PF reset in case of failure to 6295 * resume. 6296 **/ 6297 static int i40e_resume_port_tx(struct i40e_pf *pf) 6298 { 6299 struct i40e_hw *hw = &pf->hw; 6300 int ret; 6301 6302 ret = i40e_aq_resume_port_tx(hw, NULL); 6303 if (ret) { 6304 dev_info(&pf->pdev->dev, 6305 "Resume Port Tx failed, err %s aq_err %s\n", 6306 i40e_stat_str(&pf->hw, ret), 6307 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 6308 /* Schedule PF reset to recover */ 6309 set_bit(__I40E_PF_RESET_REQUESTED, pf->state); 6310 i40e_service_event_schedule(pf); 6311 } 6312 6313 return ret; 6314 } 6315 6316 /** 6317 * i40e_init_pf_dcb - Initialize DCB configuration 6318 * @pf: PF being configured 6319 * 6320 * Query the current DCB configuration and cache it 6321 * in the hardware structure 6322 **/ 6323 static int i40e_init_pf_dcb(struct i40e_pf *pf) 6324 { 6325 struct i40e_hw *hw = &pf->hw; 6326 int err = 0; 6327 6328 /* Do not enable DCB for SW1 and SW2 images even if the FW is capable 6329 * Also do not enable DCBx if FW LLDP agent is disabled 6330 */ 6331 if ((pf->hw_features & I40E_HW_NO_DCB_SUPPORT) || 6332 (pf->flags & I40E_FLAG_DISABLE_FW_LLDP)) 6333 goto out; 6334 6335 /* Get the initial DCB configuration */ 6336 err = i40e_init_dcb(hw); 6337 if (!err) { 6338 /* Device/Function is not DCBX capable */ 6339 if ((!hw->func_caps.dcb) || 6340 (hw->dcbx_status == I40E_DCBX_STATUS_DISABLED)) { 6341 dev_info(&pf->pdev->dev, 6342 "DCBX offload is not supported or is disabled for this PF.\n"); 6343 } else { 6344 /* When status is not DISABLED then DCBX in FW */ 6345 pf->dcbx_cap = DCB_CAP_DCBX_LLD_MANAGED | 6346 DCB_CAP_DCBX_VER_IEEE; 6347 6348 pf->flags |= I40E_FLAG_DCB_CAPABLE; 6349 /* Enable DCB tagging only when more than one TC 6350 * or explicitly disable if only one TC 6351 */ 6352 if (i40e_dcb_get_num_tc(&hw->local_dcbx_config) > 1) 6353 pf->flags |= I40E_FLAG_DCB_ENABLED; 6354 else 6355 pf->flags &= ~I40E_FLAG_DCB_ENABLED; 6356 dev_dbg(&pf->pdev->dev, 6357 "DCBX offload is supported for this PF.\n"); 6358 } 6359 } else if (pf->hw.aq.asq_last_status == I40E_AQ_RC_EPERM) { 6360 dev_info(&pf->pdev->dev, "FW LLDP disabled for this PF.\n"); 6361 pf->flags |= I40E_FLAG_DISABLE_FW_LLDP; 6362 } else { 6363 dev_info(&pf->pdev->dev, 6364 "Query for DCB configuration failed, err %s aq_err %s\n", 6365 i40e_stat_str(&pf->hw, err), 6366 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 6367 } 6368 6369 out: 6370 return err; 6371 } 6372 #endif /* CONFIG_I40E_DCB */ 6373 #define SPEED_SIZE 14 6374 #define FC_SIZE 8 6375 /** 6376 * i40e_print_link_message - print link up or down 6377 * @vsi: the VSI for which link needs a message 6378 */ 6379 void i40e_print_link_message(struct i40e_vsi *vsi, bool isup) 6380 { 6381 enum i40e_aq_link_speed new_speed; 6382 struct i40e_pf *pf = vsi->back; 6383 char *speed = "Unknown"; 6384 char *fc = "Unknown"; 6385 char *fec = ""; 6386 char *req_fec = ""; 6387 char *an = ""; 6388 6389 new_speed = pf->hw.phy.link_info.link_speed; 6390 6391 if ((vsi->current_isup == isup) && (vsi->current_speed == new_speed)) 6392 return; 6393 vsi->current_isup = isup; 6394 vsi->current_speed = new_speed; 6395 if (!isup) { 6396 netdev_info(vsi->netdev, "NIC Link is Down\n"); 6397 return; 6398 } 6399 6400 /* Warn user if link speed on NPAR enabled partition is not at 6401 * least 10GB 6402 */ 6403 if (pf->hw.func_caps.npar_enable && 6404 (pf->hw.phy.link_info.link_speed == I40E_LINK_SPEED_1GB || 6405 pf->hw.phy.link_info.link_speed == I40E_LINK_SPEED_100MB)) 6406 netdev_warn(vsi->netdev, 6407 "The partition detected link speed that is less than 10Gbps\n"); 6408 6409 switch (pf->hw.phy.link_info.link_speed) { 6410 case I40E_LINK_SPEED_40GB: 6411 speed = "40 G"; 6412 break; 6413 case I40E_LINK_SPEED_20GB: 6414 speed = "20 G"; 6415 break; 6416 case I40E_LINK_SPEED_25GB: 6417 speed = "25 G"; 6418 break; 6419 case I40E_LINK_SPEED_10GB: 6420 speed = "10 G"; 6421 break; 6422 case I40E_LINK_SPEED_1GB: 6423 speed = "1000 M"; 6424 break; 6425 case I40E_LINK_SPEED_100MB: 6426 speed = "100 M"; 6427 break; 6428 default: 6429 break; 6430 } 6431 6432 switch (pf->hw.fc.current_mode) { 6433 case I40E_FC_FULL: 6434 fc = "RX/TX"; 6435 break; 6436 case I40E_FC_TX_PAUSE: 6437 fc = "TX"; 6438 break; 6439 case I40E_FC_RX_PAUSE: 6440 fc = "RX"; 6441 break; 6442 default: 6443 fc = "None"; 6444 break; 6445 } 6446 6447 if (pf->hw.phy.link_info.link_speed == I40E_LINK_SPEED_25GB) { 6448 req_fec = ", Requested FEC: None"; 6449 fec = ", FEC: None"; 6450 an = ", Autoneg: False"; 6451 6452 if (pf->hw.phy.link_info.an_info & I40E_AQ_AN_COMPLETED) 6453 an = ", Autoneg: True"; 6454 6455 if (pf->hw.phy.link_info.fec_info & 6456 I40E_AQ_CONFIG_FEC_KR_ENA) 6457 fec = ", FEC: CL74 FC-FEC/BASE-R"; 6458 else if (pf->hw.phy.link_info.fec_info & 6459 I40E_AQ_CONFIG_FEC_RS_ENA) 6460 fec = ", FEC: CL108 RS-FEC"; 6461 6462 /* 'CL108 RS-FEC' should be displayed when RS is requested, or 6463 * both RS and FC are requested 6464 */ 6465 if (vsi->back->hw.phy.link_info.req_fec_info & 6466 (I40E_AQ_REQUEST_FEC_KR | I40E_AQ_REQUEST_FEC_RS)) { 6467 if (vsi->back->hw.phy.link_info.req_fec_info & 6468 I40E_AQ_REQUEST_FEC_RS) 6469 req_fec = ", Requested FEC: CL108 RS-FEC"; 6470 else 6471 req_fec = ", Requested FEC: CL74 FC-FEC/BASE-R"; 6472 } 6473 } 6474 6475 netdev_info(vsi->netdev, "NIC Link is Up, %sbps Full Duplex%s%s%s, Flow Control: %s\n", 6476 speed, req_fec, fec, an, fc); 6477 } 6478 6479 /** 6480 * i40e_up_complete - Finish the last steps of bringing up a connection 6481 * @vsi: the VSI being configured 6482 **/ 6483 static int i40e_up_complete(struct i40e_vsi *vsi) 6484 { 6485 struct i40e_pf *pf = vsi->back; 6486 int err; 6487 6488 if (pf->flags & I40E_FLAG_MSIX_ENABLED) 6489 i40e_vsi_configure_msix(vsi); 6490 else 6491 i40e_configure_msi_and_legacy(vsi); 6492 6493 /* start rings */ 6494 err = i40e_vsi_start_rings(vsi); 6495 if (err) 6496 return err; 6497 6498 clear_bit(__I40E_VSI_DOWN, vsi->state); 6499 i40e_napi_enable_all(vsi); 6500 i40e_vsi_enable_irq(vsi); 6501 6502 if ((pf->hw.phy.link_info.link_info & I40E_AQ_LINK_UP) && 6503 (vsi->netdev)) { 6504 i40e_print_link_message(vsi, true); 6505 netif_tx_start_all_queues(vsi->netdev); 6506 netif_carrier_on(vsi->netdev); 6507 } 6508 6509 /* replay FDIR SB filters */ 6510 if (vsi->type == I40E_VSI_FDIR) { 6511 /* reset fd counters */ 6512 pf->fd_add_err = 0; 6513 pf->fd_atr_cnt = 0; 6514 i40e_fdir_filter_restore(vsi); 6515 } 6516 6517 /* On the next run of the service_task, notify any clients of the new 6518 * opened netdev 6519 */ 6520 pf->flags |= I40E_FLAG_SERVICE_CLIENT_REQUESTED; 6521 i40e_service_event_schedule(pf); 6522 6523 return 0; 6524 } 6525 6526 /** 6527 * i40e_vsi_reinit_locked - Reset the VSI 6528 * @vsi: the VSI being configured 6529 * 6530 * Rebuild the ring structs after some configuration 6531 * has changed, e.g. MTU size. 6532 **/ 6533 static void i40e_vsi_reinit_locked(struct i40e_vsi *vsi) 6534 { 6535 struct i40e_pf *pf = vsi->back; 6536 6537 WARN_ON(in_interrupt()); 6538 while (test_and_set_bit(__I40E_CONFIG_BUSY, pf->state)) 6539 usleep_range(1000, 2000); 6540 i40e_down(vsi); 6541 6542 i40e_up(vsi); 6543 clear_bit(__I40E_CONFIG_BUSY, pf->state); 6544 } 6545 6546 /** 6547 * i40e_up - Bring the connection back up after being down 6548 * @vsi: the VSI being configured 6549 **/ 6550 int i40e_up(struct i40e_vsi *vsi) 6551 { 6552 int err; 6553 6554 err = i40e_vsi_configure(vsi); 6555 if (!err) 6556 err = i40e_up_complete(vsi); 6557 6558 return err; 6559 } 6560 6561 /** 6562 * i40e_down - Shutdown the connection processing 6563 * @vsi: the VSI being stopped 6564 **/ 6565 void i40e_down(struct i40e_vsi *vsi) 6566 { 6567 int i; 6568 6569 /* It is assumed that the caller of this function 6570 * sets the vsi->state __I40E_VSI_DOWN bit. 6571 */ 6572 if (vsi->netdev) { 6573 netif_carrier_off(vsi->netdev); 6574 netif_tx_disable(vsi->netdev); 6575 } 6576 i40e_vsi_disable_irq(vsi); 6577 i40e_vsi_stop_rings(vsi); 6578 i40e_napi_disable_all(vsi); 6579 6580 for (i = 0; i < vsi->num_queue_pairs; i++) { 6581 i40e_clean_tx_ring(vsi->tx_rings[i]); 6582 if (i40e_enabled_xdp_vsi(vsi)) 6583 i40e_clean_tx_ring(vsi->xdp_rings[i]); 6584 i40e_clean_rx_ring(vsi->rx_rings[i]); 6585 } 6586 6587 } 6588 6589 /** 6590 * i40e_validate_mqprio_qopt- validate queue mapping info 6591 * @vsi: the VSI being configured 6592 * @mqprio_qopt: queue parametrs 6593 **/ 6594 static int i40e_validate_mqprio_qopt(struct i40e_vsi *vsi, 6595 struct tc_mqprio_qopt_offload *mqprio_qopt) 6596 { 6597 u64 sum_max_rate = 0; 6598 u64 max_rate = 0; 6599 int i; 6600 6601 if (mqprio_qopt->qopt.offset[0] != 0 || 6602 mqprio_qopt->qopt.num_tc < 1 || 6603 mqprio_qopt->qopt.num_tc > I40E_MAX_TRAFFIC_CLASS) 6604 return -EINVAL; 6605 for (i = 0; ; i++) { 6606 if (!mqprio_qopt->qopt.count[i]) 6607 return -EINVAL; 6608 if (mqprio_qopt->min_rate[i]) { 6609 dev_err(&vsi->back->pdev->dev, 6610 "Invalid min tx rate (greater than 0) specified\n"); 6611 return -EINVAL; 6612 } 6613 max_rate = mqprio_qopt->max_rate[i]; 6614 do_div(max_rate, I40E_BW_MBPS_DIVISOR); 6615 sum_max_rate += max_rate; 6616 6617 if (i >= mqprio_qopt->qopt.num_tc - 1) 6618 break; 6619 if (mqprio_qopt->qopt.offset[i + 1] != 6620 (mqprio_qopt->qopt.offset[i] + mqprio_qopt->qopt.count[i])) 6621 return -EINVAL; 6622 } 6623 if (vsi->num_queue_pairs < 6624 (mqprio_qopt->qopt.offset[i] + mqprio_qopt->qopt.count[i])) { 6625 return -EINVAL; 6626 } 6627 if (sum_max_rate > i40e_get_link_speed(vsi)) { 6628 dev_err(&vsi->back->pdev->dev, 6629 "Invalid max tx rate specified\n"); 6630 return -EINVAL; 6631 } 6632 return 0; 6633 } 6634 6635 /** 6636 * i40e_vsi_set_default_tc_config - set default values for tc configuration 6637 * @vsi: the VSI being configured 6638 **/ 6639 static void i40e_vsi_set_default_tc_config(struct i40e_vsi *vsi) 6640 { 6641 u16 qcount; 6642 int i; 6643 6644 /* Only TC0 is enabled */ 6645 vsi->tc_config.numtc = 1; 6646 vsi->tc_config.enabled_tc = 1; 6647 qcount = min_t(int, vsi->alloc_queue_pairs, 6648 i40e_pf_get_max_q_per_tc(vsi->back)); 6649 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 6650 /* For the TC that is not enabled set the offset to to default 6651 * queue and allocate one queue for the given TC. 6652 */ 6653 vsi->tc_config.tc_info[i].qoffset = 0; 6654 if (i == 0) 6655 vsi->tc_config.tc_info[i].qcount = qcount; 6656 else 6657 vsi->tc_config.tc_info[i].qcount = 1; 6658 vsi->tc_config.tc_info[i].netdev_tc = 0; 6659 } 6660 } 6661 6662 /** 6663 * i40e_setup_tc - configure multiple traffic classes 6664 * @netdev: net device to configure 6665 * @type_data: tc offload data 6666 **/ 6667 static int i40e_setup_tc(struct net_device *netdev, void *type_data) 6668 { 6669 struct tc_mqprio_qopt_offload *mqprio_qopt = type_data; 6670 struct i40e_netdev_priv *np = netdev_priv(netdev); 6671 struct i40e_vsi *vsi = np->vsi; 6672 struct i40e_pf *pf = vsi->back; 6673 u8 enabled_tc = 0, num_tc, hw; 6674 bool need_reset = false; 6675 int ret = -EINVAL; 6676 u16 mode; 6677 int i; 6678 6679 num_tc = mqprio_qopt->qopt.num_tc; 6680 hw = mqprio_qopt->qopt.hw; 6681 mode = mqprio_qopt->mode; 6682 if (!hw) { 6683 pf->flags &= ~I40E_FLAG_TC_MQPRIO; 6684 memcpy(&vsi->mqprio_qopt, mqprio_qopt, sizeof(*mqprio_qopt)); 6685 goto config_tc; 6686 } 6687 6688 /* Check if MFP enabled */ 6689 if (pf->flags & I40E_FLAG_MFP_ENABLED) { 6690 netdev_info(netdev, 6691 "Configuring TC not supported in MFP mode\n"); 6692 return ret; 6693 } 6694 switch (mode) { 6695 case TC_MQPRIO_MODE_DCB: 6696 pf->flags &= ~I40E_FLAG_TC_MQPRIO; 6697 6698 /* Check if DCB enabled to continue */ 6699 if (!(pf->flags & I40E_FLAG_DCB_ENABLED)) { 6700 netdev_info(netdev, 6701 "DCB is not enabled for adapter\n"); 6702 return ret; 6703 } 6704 6705 /* Check whether tc count is within enabled limit */ 6706 if (num_tc > i40e_pf_get_num_tc(pf)) { 6707 netdev_info(netdev, 6708 "TC count greater than enabled on link for adapter\n"); 6709 return ret; 6710 } 6711 break; 6712 case TC_MQPRIO_MODE_CHANNEL: 6713 if (pf->flags & I40E_FLAG_DCB_ENABLED) { 6714 netdev_info(netdev, 6715 "Full offload of TC Mqprio options is not supported when DCB is enabled\n"); 6716 return ret; 6717 } 6718 if (!(pf->flags & I40E_FLAG_MSIX_ENABLED)) 6719 return ret; 6720 ret = i40e_validate_mqprio_qopt(vsi, mqprio_qopt); 6721 if (ret) 6722 return ret; 6723 memcpy(&vsi->mqprio_qopt, mqprio_qopt, 6724 sizeof(*mqprio_qopt)); 6725 pf->flags |= I40E_FLAG_TC_MQPRIO; 6726 pf->flags &= ~I40E_FLAG_DCB_ENABLED; 6727 break; 6728 default: 6729 return -EINVAL; 6730 } 6731 6732 config_tc: 6733 /* Generate TC map for number of tc requested */ 6734 for (i = 0; i < num_tc; i++) 6735 enabled_tc |= BIT(i); 6736 6737 /* Requesting same TC configuration as already enabled */ 6738 if (enabled_tc == vsi->tc_config.enabled_tc && 6739 mode != TC_MQPRIO_MODE_CHANNEL) 6740 return 0; 6741 6742 /* Quiesce VSI queues */ 6743 i40e_quiesce_vsi(vsi); 6744 6745 if (!hw && !(pf->flags & I40E_FLAG_TC_MQPRIO)) 6746 i40e_remove_queue_channels(vsi); 6747 6748 /* Configure VSI for enabled TCs */ 6749 ret = i40e_vsi_config_tc(vsi, enabled_tc); 6750 if (ret) { 6751 netdev_info(netdev, "Failed configuring TC for VSI seid=%d\n", 6752 vsi->seid); 6753 need_reset = true; 6754 goto exit; 6755 } 6756 6757 if (pf->flags & I40E_FLAG_TC_MQPRIO) { 6758 if (vsi->mqprio_qopt.max_rate[0]) { 6759 u64 max_tx_rate = vsi->mqprio_qopt.max_rate[0]; 6760 6761 do_div(max_tx_rate, I40E_BW_MBPS_DIVISOR); 6762 ret = i40e_set_bw_limit(vsi, vsi->seid, max_tx_rate); 6763 if (!ret) { 6764 u64 credits = max_tx_rate; 6765 6766 do_div(credits, I40E_BW_CREDIT_DIVISOR); 6767 dev_dbg(&vsi->back->pdev->dev, 6768 "Set tx rate of %llu Mbps (count of 50Mbps %llu) for vsi->seid %u\n", 6769 max_tx_rate, 6770 credits, 6771 vsi->seid); 6772 } else { 6773 need_reset = true; 6774 goto exit; 6775 } 6776 } 6777 ret = i40e_configure_queue_channels(vsi); 6778 if (ret) { 6779 netdev_info(netdev, 6780 "Failed configuring queue channels\n"); 6781 need_reset = true; 6782 goto exit; 6783 } 6784 } 6785 6786 exit: 6787 /* Reset the configuration data to defaults, only TC0 is enabled */ 6788 if (need_reset) { 6789 i40e_vsi_set_default_tc_config(vsi); 6790 need_reset = false; 6791 } 6792 6793 /* Unquiesce VSI */ 6794 i40e_unquiesce_vsi(vsi); 6795 return ret; 6796 } 6797 6798 /** 6799 * i40e_set_cld_element - sets cloud filter element data 6800 * @filter: cloud filter rule 6801 * @cld: ptr to cloud filter element data 6802 * 6803 * This is helper function to copy data into cloud filter element 6804 **/ 6805 static inline void 6806 i40e_set_cld_element(struct i40e_cloud_filter *filter, 6807 struct i40e_aqc_cloud_filters_element_data *cld) 6808 { 6809 int i, j; 6810 u32 ipa; 6811 6812 memset(cld, 0, sizeof(*cld)); 6813 ether_addr_copy(cld->outer_mac, filter->dst_mac); 6814 ether_addr_copy(cld->inner_mac, filter->src_mac); 6815 6816 if (filter->n_proto != ETH_P_IP && filter->n_proto != ETH_P_IPV6) 6817 return; 6818 6819 if (filter->n_proto == ETH_P_IPV6) { 6820 #define IPV6_MAX_INDEX (ARRAY_SIZE(filter->dst_ipv6) - 1) 6821 for (i = 0, j = 0; i < ARRAY_SIZE(filter->dst_ipv6); 6822 i++, j += 2) { 6823 ipa = be32_to_cpu(filter->dst_ipv6[IPV6_MAX_INDEX - i]); 6824 ipa = cpu_to_le32(ipa); 6825 memcpy(&cld->ipaddr.raw_v6.data[j], &ipa, sizeof(ipa)); 6826 } 6827 } else { 6828 ipa = be32_to_cpu(filter->dst_ipv4); 6829 memcpy(&cld->ipaddr.v4.data, &ipa, sizeof(ipa)); 6830 } 6831 6832 cld->inner_vlan = cpu_to_le16(ntohs(filter->vlan_id)); 6833 6834 /* tenant_id is not supported by FW now, once the support is enabled 6835 * fill the cld->tenant_id with cpu_to_le32(filter->tenant_id) 6836 */ 6837 if (filter->tenant_id) 6838 return; 6839 } 6840 6841 /** 6842 * i40e_add_del_cloud_filter - Add/del cloud filter 6843 * @vsi: pointer to VSI 6844 * @filter: cloud filter rule 6845 * @add: if true, add, if false, delete 6846 * 6847 * Add or delete a cloud filter for a specific flow spec. 6848 * Returns 0 if the filter were successfully added. 6849 **/ 6850 static int i40e_add_del_cloud_filter(struct i40e_vsi *vsi, 6851 struct i40e_cloud_filter *filter, bool add) 6852 { 6853 struct i40e_aqc_cloud_filters_element_data cld_filter; 6854 struct i40e_pf *pf = vsi->back; 6855 int ret; 6856 static const u16 flag_table[128] = { 6857 [I40E_CLOUD_FILTER_FLAGS_OMAC] = 6858 I40E_AQC_ADD_CLOUD_FILTER_OMAC, 6859 [I40E_CLOUD_FILTER_FLAGS_IMAC] = 6860 I40E_AQC_ADD_CLOUD_FILTER_IMAC, 6861 [I40E_CLOUD_FILTER_FLAGS_IMAC_IVLAN] = 6862 I40E_AQC_ADD_CLOUD_FILTER_IMAC_IVLAN, 6863 [I40E_CLOUD_FILTER_FLAGS_IMAC_TEN_ID] = 6864 I40E_AQC_ADD_CLOUD_FILTER_IMAC_TEN_ID, 6865 [I40E_CLOUD_FILTER_FLAGS_OMAC_TEN_ID_IMAC] = 6866 I40E_AQC_ADD_CLOUD_FILTER_OMAC_TEN_ID_IMAC, 6867 [I40E_CLOUD_FILTER_FLAGS_IMAC_IVLAN_TEN_ID] = 6868 I40E_AQC_ADD_CLOUD_FILTER_IMAC_IVLAN_TEN_ID, 6869 [I40E_CLOUD_FILTER_FLAGS_IIP] = 6870 I40E_AQC_ADD_CLOUD_FILTER_IIP, 6871 }; 6872 6873 if (filter->flags >= ARRAY_SIZE(flag_table)) 6874 return I40E_ERR_CONFIG; 6875 6876 /* copy element needed to add cloud filter from filter */ 6877 i40e_set_cld_element(filter, &cld_filter); 6878 6879 if (filter->tunnel_type != I40E_CLOUD_TNL_TYPE_NONE) 6880 cld_filter.flags = cpu_to_le16(filter->tunnel_type << 6881 I40E_AQC_ADD_CLOUD_TNL_TYPE_SHIFT); 6882 6883 if (filter->n_proto == ETH_P_IPV6) 6884 cld_filter.flags |= cpu_to_le16(flag_table[filter->flags] | 6885 I40E_AQC_ADD_CLOUD_FLAGS_IPV6); 6886 else 6887 cld_filter.flags |= cpu_to_le16(flag_table[filter->flags] | 6888 I40E_AQC_ADD_CLOUD_FLAGS_IPV4); 6889 6890 if (add) 6891 ret = i40e_aq_add_cloud_filters(&pf->hw, filter->seid, 6892 &cld_filter, 1); 6893 else 6894 ret = i40e_aq_rem_cloud_filters(&pf->hw, filter->seid, 6895 &cld_filter, 1); 6896 if (ret) 6897 dev_dbg(&pf->pdev->dev, 6898 "Failed to %s cloud filter using l4 port %u, err %d aq_err %d\n", 6899 add ? "add" : "delete", filter->dst_port, ret, 6900 pf->hw.aq.asq_last_status); 6901 else 6902 dev_info(&pf->pdev->dev, 6903 "%s cloud filter for VSI: %d\n", 6904 add ? "Added" : "Deleted", filter->seid); 6905 return ret; 6906 } 6907 6908 /** 6909 * i40e_add_del_cloud_filter_big_buf - Add/del cloud filter using big_buf 6910 * @vsi: pointer to VSI 6911 * @filter: cloud filter rule 6912 * @add: if true, add, if false, delete 6913 * 6914 * Add or delete a cloud filter for a specific flow spec using big buffer. 6915 * Returns 0 if the filter were successfully added. 6916 **/ 6917 static int i40e_add_del_cloud_filter_big_buf(struct i40e_vsi *vsi, 6918 struct i40e_cloud_filter *filter, 6919 bool add) 6920 { 6921 struct i40e_aqc_cloud_filters_element_bb cld_filter; 6922 struct i40e_pf *pf = vsi->back; 6923 int ret; 6924 6925 /* Both (src/dst) valid mac_addr are not supported */ 6926 if ((is_valid_ether_addr(filter->dst_mac) && 6927 is_valid_ether_addr(filter->src_mac)) || 6928 (is_multicast_ether_addr(filter->dst_mac) && 6929 is_multicast_ether_addr(filter->src_mac))) 6930 return -EOPNOTSUPP; 6931 6932 /* Big buffer cloud filter needs 'L4 port' to be non-zero. Also, UDP 6933 * ports are not supported via big buffer now. 6934 */ 6935 if (!filter->dst_port || filter->ip_proto == IPPROTO_UDP) 6936 return -EOPNOTSUPP; 6937 6938 /* adding filter using src_port/src_ip is not supported at this stage */ 6939 if (filter->src_port || filter->src_ipv4 || 6940 !ipv6_addr_any(&filter->ip.v6.src_ip6)) 6941 return -EOPNOTSUPP; 6942 6943 /* copy element needed to add cloud filter from filter */ 6944 i40e_set_cld_element(filter, &cld_filter.element); 6945 6946 if (is_valid_ether_addr(filter->dst_mac) || 6947 is_valid_ether_addr(filter->src_mac) || 6948 is_multicast_ether_addr(filter->dst_mac) || 6949 is_multicast_ether_addr(filter->src_mac)) { 6950 /* MAC + IP : unsupported mode */ 6951 if (filter->dst_ipv4) 6952 return -EOPNOTSUPP; 6953 6954 /* since we validated that L4 port must be valid before 6955 * we get here, start with respective "flags" value 6956 * and update if vlan is present or not 6957 */ 6958 cld_filter.element.flags = 6959 cpu_to_le16(I40E_AQC_ADD_CLOUD_FILTER_MAC_PORT); 6960 6961 if (filter->vlan_id) { 6962 cld_filter.element.flags = 6963 cpu_to_le16(I40E_AQC_ADD_CLOUD_FILTER_MAC_VLAN_PORT); 6964 } 6965 6966 } else if (filter->dst_ipv4 || 6967 !ipv6_addr_any(&filter->ip.v6.dst_ip6)) { 6968 cld_filter.element.flags = 6969 cpu_to_le16(I40E_AQC_ADD_CLOUD_FILTER_IP_PORT); 6970 if (filter->n_proto == ETH_P_IPV6) 6971 cld_filter.element.flags |= 6972 cpu_to_le16(I40E_AQC_ADD_CLOUD_FLAGS_IPV6); 6973 else 6974 cld_filter.element.flags |= 6975 cpu_to_le16(I40E_AQC_ADD_CLOUD_FLAGS_IPV4); 6976 } else { 6977 dev_err(&pf->pdev->dev, 6978 "either mac or ip has to be valid for cloud filter\n"); 6979 return -EINVAL; 6980 } 6981 6982 /* Now copy L4 port in Byte 6..7 in general fields */ 6983 cld_filter.general_fields[I40E_AQC_ADD_CLOUD_FV_FLU_0X16_WORD0] = 6984 be16_to_cpu(filter->dst_port); 6985 6986 if (add) { 6987 /* Validate current device switch mode, change if necessary */ 6988 ret = i40e_validate_and_set_switch_mode(vsi); 6989 if (ret) { 6990 dev_err(&pf->pdev->dev, 6991 "failed to set switch mode, ret %d\n", 6992 ret); 6993 return ret; 6994 } 6995 6996 ret = i40e_aq_add_cloud_filters_bb(&pf->hw, filter->seid, 6997 &cld_filter, 1); 6998 } else { 6999 ret = i40e_aq_rem_cloud_filters_bb(&pf->hw, filter->seid, 7000 &cld_filter, 1); 7001 } 7002 7003 if (ret) 7004 dev_dbg(&pf->pdev->dev, 7005 "Failed to %s cloud filter(big buffer) err %d aq_err %d\n", 7006 add ? "add" : "delete", ret, pf->hw.aq.asq_last_status); 7007 else 7008 dev_info(&pf->pdev->dev, 7009 "%s cloud filter for VSI: %d, L4 port: %d\n", 7010 add ? "add" : "delete", filter->seid, 7011 ntohs(filter->dst_port)); 7012 return ret; 7013 } 7014 7015 /** 7016 * i40e_parse_cls_flower - Parse tc flower filters provided by kernel 7017 * @vsi: Pointer to VSI 7018 * @cls_flower: Pointer to struct tc_cls_flower_offload 7019 * @filter: Pointer to cloud filter structure 7020 * 7021 **/ 7022 static int i40e_parse_cls_flower(struct i40e_vsi *vsi, 7023 struct tc_cls_flower_offload *f, 7024 struct i40e_cloud_filter *filter) 7025 { 7026 u16 n_proto_mask = 0, n_proto_key = 0, addr_type = 0; 7027 struct i40e_pf *pf = vsi->back; 7028 u8 field_flags = 0; 7029 7030 if (f->dissector->used_keys & 7031 ~(BIT(FLOW_DISSECTOR_KEY_CONTROL) | 7032 BIT(FLOW_DISSECTOR_KEY_BASIC) | 7033 BIT(FLOW_DISSECTOR_KEY_ETH_ADDRS) | 7034 BIT(FLOW_DISSECTOR_KEY_VLAN) | 7035 BIT(FLOW_DISSECTOR_KEY_IPV4_ADDRS) | 7036 BIT(FLOW_DISSECTOR_KEY_IPV6_ADDRS) | 7037 BIT(FLOW_DISSECTOR_KEY_PORTS) | 7038 BIT(FLOW_DISSECTOR_KEY_ENC_KEYID))) { 7039 dev_err(&pf->pdev->dev, "Unsupported key used: 0x%x\n", 7040 f->dissector->used_keys); 7041 return -EOPNOTSUPP; 7042 } 7043 7044 if (dissector_uses_key(f->dissector, FLOW_DISSECTOR_KEY_ENC_KEYID)) { 7045 struct flow_dissector_key_keyid *key = 7046 skb_flow_dissector_target(f->dissector, 7047 FLOW_DISSECTOR_KEY_ENC_KEYID, 7048 f->key); 7049 7050 struct flow_dissector_key_keyid *mask = 7051 skb_flow_dissector_target(f->dissector, 7052 FLOW_DISSECTOR_KEY_ENC_KEYID, 7053 f->mask); 7054 7055 if (mask->keyid != 0) 7056 field_flags |= I40E_CLOUD_FIELD_TEN_ID; 7057 7058 filter->tenant_id = be32_to_cpu(key->keyid); 7059 } 7060 7061 if (dissector_uses_key(f->dissector, FLOW_DISSECTOR_KEY_BASIC)) { 7062 struct flow_dissector_key_basic *key = 7063 skb_flow_dissector_target(f->dissector, 7064 FLOW_DISSECTOR_KEY_BASIC, 7065 f->key); 7066 7067 struct flow_dissector_key_basic *mask = 7068 skb_flow_dissector_target(f->dissector, 7069 FLOW_DISSECTOR_KEY_BASIC, 7070 f->mask); 7071 7072 n_proto_key = ntohs(key->n_proto); 7073 n_proto_mask = ntohs(mask->n_proto); 7074 7075 if (n_proto_key == ETH_P_ALL) { 7076 n_proto_key = 0; 7077 n_proto_mask = 0; 7078 } 7079 filter->n_proto = n_proto_key & n_proto_mask; 7080 filter->ip_proto = key->ip_proto; 7081 } 7082 7083 if (dissector_uses_key(f->dissector, FLOW_DISSECTOR_KEY_ETH_ADDRS)) { 7084 struct flow_dissector_key_eth_addrs *key = 7085 skb_flow_dissector_target(f->dissector, 7086 FLOW_DISSECTOR_KEY_ETH_ADDRS, 7087 f->key); 7088 7089 struct flow_dissector_key_eth_addrs *mask = 7090 skb_flow_dissector_target(f->dissector, 7091 FLOW_DISSECTOR_KEY_ETH_ADDRS, 7092 f->mask); 7093 7094 /* use is_broadcast and is_zero to check for all 0xf or 0 */ 7095 if (!is_zero_ether_addr(mask->dst)) { 7096 if (is_broadcast_ether_addr(mask->dst)) { 7097 field_flags |= I40E_CLOUD_FIELD_OMAC; 7098 } else { 7099 dev_err(&pf->pdev->dev, "Bad ether dest mask %pM\n", 7100 mask->dst); 7101 return I40E_ERR_CONFIG; 7102 } 7103 } 7104 7105 if (!is_zero_ether_addr(mask->src)) { 7106 if (is_broadcast_ether_addr(mask->src)) { 7107 field_flags |= I40E_CLOUD_FIELD_IMAC; 7108 } else { 7109 dev_err(&pf->pdev->dev, "Bad ether src mask %pM\n", 7110 mask->src); 7111 return I40E_ERR_CONFIG; 7112 } 7113 } 7114 ether_addr_copy(filter->dst_mac, key->dst); 7115 ether_addr_copy(filter->src_mac, key->src); 7116 } 7117 7118 if (dissector_uses_key(f->dissector, FLOW_DISSECTOR_KEY_VLAN)) { 7119 struct flow_dissector_key_vlan *key = 7120 skb_flow_dissector_target(f->dissector, 7121 FLOW_DISSECTOR_KEY_VLAN, 7122 f->key); 7123 struct flow_dissector_key_vlan *mask = 7124 skb_flow_dissector_target(f->dissector, 7125 FLOW_DISSECTOR_KEY_VLAN, 7126 f->mask); 7127 7128 if (mask->vlan_id) { 7129 if (mask->vlan_id == VLAN_VID_MASK) { 7130 field_flags |= I40E_CLOUD_FIELD_IVLAN; 7131 7132 } else { 7133 dev_err(&pf->pdev->dev, "Bad vlan mask 0x%04x\n", 7134 mask->vlan_id); 7135 return I40E_ERR_CONFIG; 7136 } 7137 } 7138 7139 filter->vlan_id = cpu_to_be16(key->vlan_id); 7140 } 7141 7142 if (dissector_uses_key(f->dissector, FLOW_DISSECTOR_KEY_CONTROL)) { 7143 struct flow_dissector_key_control *key = 7144 skb_flow_dissector_target(f->dissector, 7145 FLOW_DISSECTOR_KEY_CONTROL, 7146 f->key); 7147 7148 addr_type = key->addr_type; 7149 } 7150 7151 if (addr_type == FLOW_DISSECTOR_KEY_IPV4_ADDRS) { 7152 struct flow_dissector_key_ipv4_addrs *key = 7153 skb_flow_dissector_target(f->dissector, 7154 FLOW_DISSECTOR_KEY_IPV4_ADDRS, 7155 f->key); 7156 struct flow_dissector_key_ipv4_addrs *mask = 7157 skb_flow_dissector_target(f->dissector, 7158 FLOW_DISSECTOR_KEY_IPV4_ADDRS, 7159 f->mask); 7160 7161 if (mask->dst) { 7162 if (mask->dst == cpu_to_be32(0xffffffff)) { 7163 field_flags |= I40E_CLOUD_FIELD_IIP; 7164 } else { 7165 mask->dst = be32_to_cpu(mask->dst); 7166 dev_err(&pf->pdev->dev, "Bad ip dst mask %pI4\n", 7167 &mask->dst); 7168 return I40E_ERR_CONFIG; 7169 } 7170 } 7171 7172 if (mask->src) { 7173 if (mask->src == cpu_to_be32(0xffffffff)) { 7174 field_flags |= I40E_CLOUD_FIELD_IIP; 7175 } else { 7176 mask->src = be32_to_cpu(mask->src); 7177 dev_err(&pf->pdev->dev, "Bad ip src mask %pI4\n", 7178 &mask->src); 7179 return I40E_ERR_CONFIG; 7180 } 7181 } 7182 7183 if (field_flags & I40E_CLOUD_FIELD_TEN_ID) { 7184 dev_err(&pf->pdev->dev, "Tenant id not allowed for ip filter\n"); 7185 return I40E_ERR_CONFIG; 7186 } 7187 filter->dst_ipv4 = key->dst; 7188 filter->src_ipv4 = key->src; 7189 } 7190 7191 if (addr_type == FLOW_DISSECTOR_KEY_IPV6_ADDRS) { 7192 struct flow_dissector_key_ipv6_addrs *key = 7193 skb_flow_dissector_target(f->dissector, 7194 FLOW_DISSECTOR_KEY_IPV6_ADDRS, 7195 f->key); 7196 struct flow_dissector_key_ipv6_addrs *mask = 7197 skb_flow_dissector_target(f->dissector, 7198 FLOW_DISSECTOR_KEY_IPV6_ADDRS, 7199 f->mask); 7200 7201 /* src and dest IPV6 address should not be LOOPBACK 7202 * (0:0:0:0:0:0:0:1), which can be represented as ::1 7203 */ 7204 if (ipv6_addr_loopback(&key->dst) || 7205 ipv6_addr_loopback(&key->src)) { 7206 dev_err(&pf->pdev->dev, 7207 "Bad ipv6, addr is LOOPBACK\n"); 7208 return I40E_ERR_CONFIG; 7209 } 7210 if (!ipv6_addr_any(&mask->dst) || !ipv6_addr_any(&mask->src)) 7211 field_flags |= I40E_CLOUD_FIELD_IIP; 7212 7213 memcpy(&filter->src_ipv6, &key->src.s6_addr32, 7214 sizeof(filter->src_ipv6)); 7215 memcpy(&filter->dst_ipv6, &key->dst.s6_addr32, 7216 sizeof(filter->dst_ipv6)); 7217 } 7218 7219 if (dissector_uses_key(f->dissector, FLOW_DISSECTOR_KEY_PORTS)) { 7220 struct flow_dissector_key_ports *key = 7221 skb_flow_dissector_target(f->dissector, 7222 FLOW_DISSECTOR_KEY_PORTS, 7223 f->key); 7224 struct flow_dissector_key_ports *mask = 7225 skb_flow_dissector_target(f->dissector, 7226 FLOW_DISSECTOR_KEY_PORTS, 7227 f->mask); 7228 7229 if (mask->src) { 7230 if (mask->src == cpu_to_be16(0xffff)) { 7231 field_flags |= I40E_CLOUD_FIELD_IIP; 7232 } else { 7233 dev_err(&pf->pdev->dev, "Bad src port mask 0x%04x\n", 7234 be16_to_cpu(mask->src)); 7235 return I40E_ERR_CONFIG; 7236 } 7237 } 7238 7239 if (mask->dst) { 7240 if (mask->dst == cpu_to_be16(0xffff)) { 7241 field_flags |= I40E_CLOUD_FIELD_IIP; 7242 } else { 7243 dev_err(&pf->pdev->dev, "Bad dst port mask 0x%04x\n", 7244 be16_to_cpu(mask->dst)); 7245 return I40E_ERR_CONFIG; 7246 } 7247 } 7248 7249 filter->dst_port = key->dst; 7250 filter->src_port = key->src; 7251 7252 switch (filter->ip_proto) { 7253 case IPPROTO_TCP: 7254 case IPPROTO_UDP: 7255 break; 7256 default: 7257 dev_err(&pf->pdev->dev, 7258 "Only UDP and TCP transport are supported\n"); 7259 return -EINVAL; 7260 } 7261 } 7262 filter->flags = field_flags; 7263 return 0; 7264 } 7265 7266 /** 7267 * i40e_handle_tclass: Forward to a traffic class on the device 7268 * @vsi: Pointer to VSI 7269 * @tc: traffic class index on the device 7270 * @filter: Pointer to cloud filter structure 7271 * 7272 **/ 7273 static int i40e_handle_tclass(struct i40e_vsi *vsi, u32 tc, 7274 struct i40e_cloud_filter *filter) 7275 { 7276 struct i40e_channel *ch, *ch_tmp; 7277 7278 /* direct to a traffic class on the same device */ 7279 if (tc == 0) { 7280 filter->seid = vsi->seid; 7281 return 0; 7282 } else if (vsi->tc_config.enabled_tc & BIT(tc)) { 7283 if (!filter->dst_port) { 7284 dev_err(&vsi->back->pdev->dev, 7285 "Specify destination port to direct to traffic class that is not default\n"); 7286 return -EINVAL; 7287 } 7288 if (list_empty(&vsi->ch_list)) 7289 return -EINVAL; 7290 list_for_each_entry_safe(ch, ch_tmp, &vsi->ch_list, 7291 list) { 7292 if (ch->seid == vsi->tc_seid_map[tc]) 7293 filter->seid = ch->seid; 7294 } 7295 return 0; 7296 } 7297 dev_err(&vsi->back->pdev->dev, "TC is not enabled\n"); 7298 return -EINVAL; 7299 } 7300 7301 /** 7302 * i40e_configure_clsflower - Configure tc flower filters 7303 * @vsi: Pointer to VSI 7304 * @cls_flower: Pointer to struct tc_cls_flower_offload 7305 * 7306 **/ 7307 static int i40e_configure_clsflower(struct i40e_vsi *vsi, 7308 struct tc_cls_flower_offload *cls_flower) 7309 { 7310 int tc = tc_classid_to_hwtc(vsi->netdev, cls_flower->classid); 7311 struct i40e_cloud_filter *filter = NULL; 7312 struct i40e_pf *pf = vsi->back; 7313 int err = 0; 7314 7315 if (tc < 0) { 7316 dev_err(&vsi->back->pdev->dev, "Invalid traffic class\n"); 7317 return -EOPNOTSUPP; 7318 } 7319 7320 if (test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state) || 7321 test_bit(__I40E_RESET_INTR_RECEIVED, pf->state)) 7322 return -EBUSY; 7323 7324 if (pf->fdir_pf_active_filters || 7325 (!hlist_empty(&pf->fdir_filter_list))) { 7326 dev_err(&vsi->back->pdev->dev, 7327 "Flow Director Sideband filters exists, turn ntuple off to configure cloud filters\n"); 7328 return -EINVAL; 7329 } 7330 7331 if (vsi->back->flags & I40E_FLAG_FD_SB_ENABLED) { 7332 dev_err(&vsi->back->pdev->dev, 7333 "Disable Flow Director Sideband, configuring Cloud filters via tc-flower\n"); 7334 vsi->back->flags &= ~I40E_FLAG_FD_SB_ENABLED; 7335 vsi->back->flags |= I40E_FLAG_FD_SB_TO_CLOUD_FILTER; 7336 } 7337 7338 filter = kzalloc(sizeof(*filter), GFP_KERNEL); 7339 if (!filter) 7340 return -ENOMEM; 7341 7342 filter->cookie = cls_flower->cookie; 7343 7344 err = i40e_parse_cls_flower(vsi, cls_flower, filter); 7345 if (err < 0) 7346 goto err; 7347 7348 err = i40e_handle_tclass(vsi, tc, filter); 7349 if (err < 0) 7350 goto err; 7351 7352 /* Add cloud filter */ 7353 if (filter->dst_port) 7354 err = i40e_add_del_cloud_filter_big_buf(vsi, filter, true); 7355 else 7356 err = i40e_add_del_cloud_filter(vsi, filter, true); 7357 7358 if (err) { 7359 dev_err(&pf->pdev->dev, 7360 "Failed to add cloud filter, err %s\n", 7361 i40e_stat_str(&pf->hw, err)); 7362 goto err; 7363 } 7364 7365 /* add filter to the ordered list */ 7366 INIT_HLIST_NODE(&filter->cloud_node); 7367 7368 hlist_add_head(&filter->cloud_node, &pf->cloud_filter_list); 7369 7370 pf->num_cloud_filters++; 7371 7372 return err; 7373 err: 7374 kfree(filter); 7375 return err; 7376 } 7377 7378 /** 7379 * i40e_find_cloud_filter - Find the could filter in the list 7380 * @vsi: Pointer to VSI 7381 * @cookie: filter specific cookie 7382 * 7383 **/ 7384 static struct i40e_cloud_filter *i40e_find_cloud_filter(struct i40e_vsi *vsi, 7385 unsigned long *cookie) 7386 { 7387 struct i40e_cloud_filter *filter = NULL; 7388 struct hlist_node *node2; 7389 7390 hlist_for_each_entry_safe(filter, node2, 7391 &vsi->back->cloud_filter_list, cloud_node) 7392 if (!memcmp(cookie, &filter->cookie, sizeof(filter->cookie))) 7393 return filter; 7394 return NULL; 7395 } 7396 7397 /** 7398 * i40e_delete_clsflower - Remove tc flower filters 7399 * @vsi: Pointer to VSI 7400 * @cls_flower: Pointer to struct tc_cls_flower_offload 7401 * 7402 **/ 7403 static int i40e_delete_clsflower(struct i40e_vsi *vsi, 7404 struct tc_cls_flower_offload *cls_flower) 7405 { 7406 struct i40e_cloud_filter *filter = NULL; 7407 struct i40e_pf *pf = vsi->back; 7408 int err = 0; 7409 7410 filter = i40e_find_cloud_filter(vsi, &cls_flower->cookie); 7411 7412 if (!filter) 7413 return -EINVAL; 7414 7415 hash_del(&filter->cloud_node); 7416 7417 if (filter->dst_port) 7418 err = i40e_add_del_cloud_filter_big_buf(vsi, filter, false); 7419 else 7420 err = i40e_add_del_cloud_filter(vsi, filter, false); 7421 7422 kfree(filter); 7423 if (err) { 7424 dev_err(&pf->pdev->dev, 7425 "Failed to delete cloud filter, err %s\n", 7426 i40e_stat_str(&pf->hw, err)); 7427 return i40e_aq_rc_to_posix(err, pf->hw.aq.asq_last_status); 7428 } 7429 7430 pf->num_cloud_filters--; 7431 if (!pf->num_cloud_filters) 7432 if ((pf->flags & I40E_FLAG_FD_SB_TO_CLOUD_FILTER) && 7433 !(pf->flags & I40E_FLAG_FD_SB_INACTIVE)) { 7434 pf->flags |= I40E_FLAG_FD_SB_ENABLED; 7435 pf->flags &= ~I40E_FLAG_FD_SB_TO_CLOUD_FILTER; 7436 pf->flags &= ~I40E_FLAG_FD_SB_INACTIVE; 7437 } 7438 return 0; 7439 } 7440 7441 /** 7442 * i40e_setup_tc_cls_flower - flower classifier offloads 7443 * @netdev: net device to configure 7444 * @type_data: offload data 7445 **/ 7446 static int i40e_setup_tc_cls_flower(struct i40e_netdev_priv *np, 7447 struct tc_cls_flower_offload *cls_flower) 7448 { 7449 struct i40e_vsi *vsi = np->vsi; 7450 7451 switch (cls_flower->command) { 7452 case TC_CLSFLOWER_REPLACE: 7453 return i40e_configure_clsflower(vsi, cls_flower); 7454 case TC_CLSFLOWER_DESTROY: 7455 return i40e_delete_clsflower(vsi, cls_flower); 7456 case TC_CLSFLOWER_STATS: 7457 return -EOPNOTSUPP; 7458 default: 7459 return -EINVAL; 7460 } 7461 } 7462 7463 static int i40e_setup_tc_block_cb(enum tc_setup_type type, void *type_data, 7464 void *cb_priv) 7465 { 7466 struct i40e_netdev_priv *np = cb_priv; 7467 7468 if (!tc_cls_can_offload_and_chain0(np->vsi->netdev, type_data)) 7469 return -EOPNOTSUPP; 7470 7471 switch (type) { 7472 case TC_SETUP_CLSFLOWER: 7473 return i40e_setup_tc_cls_flower(np, type_data); 7474 7475 default: 7476 return -EOPNOTSUPP; 7477 } 7478 } 7479 7480 static int i40e_setup_tc_block(struct net_device *dev, 7481 struct tc_block_offload *f) 7482 { 7483 struct i40e_netdev_priv *np = netdev_priv(dev); 7484 7485 if (f->binder_type != TCF_BLOCK_BINDER_TYPE_CLSACT_INGRESS) 7486 return -EOPNOTSUPP; 7487 7488 switch (f->command) { 7489 case TC_BLOCK_BIND: 7490 return tcf_block_cb_register(f->block, i40e_setup_tc_block_cb, 7491 np, np); 7492 case TC_BLOCK_UNBIND: 7493 tcf_block_cb_unregister(f->block, i40e_setup_tc_block_cb, np); 7494 return 0; 7495 default: 7496 return -EOPNOTSUPP; 7497 } 7498 } 7499 7500 static int __i40e_setup_tc(struct net_device *netdev, enum tc_setup_type type, 7501 void *type_data) 7502 { 7503 switch (type) { 7504 case TC_SETUP_QDISC_MQPRIO: 7505 return i40e_setup_tc(netdev, type_data); 7506 case TC_SETUP_BLOCK: 7507 return i40e_setup_tc_block(netdev, type_data); 7508 default: 7509 return -EOPNOTSUPP; 7510 } 7511 } 7512 7513 /** 7514 * i40e_open - Called when a network interface is made active 7515 * @netdev: network interface device structure 7516 * 7517 * The open entry point is called when a network interface is made 7518 * active by the system (IFF_UP). At this point all resources needed 7519 * for transmit and receive operations are allocated, the interrupt 7520 * handler is registered with the OS, the netdev watchdog subtask is 7521 * enabled, and the stack is notified that the interface is ready. 7522 * 7523 * Returns 0 on success, negative value on failure 7524 **/ 7525 int i40e_open(struct net_device *netdev) 7526 { 7527 struct i40e_netdev_priv *np = netdev_priv(netdev); 7528 struct i40e_vsi *vsi = np->vsi; 7529 struct i40e_pf *pf = vsi->back; 7530 int err; 7531 7532 /* disallow open during test or if eeprom is broken */ 7533 if (test_bit(__I40E_TESTING, pf->state) || 7534 test_bit(__I40E_BAD_EEPROM, pf->state)) 7535 return -EBUSY; 7536 7537 netif_carrier_off(netdev); 7538 7539 err = i40e_vsi_open(vsi); 7540 if (err) 7541 return err; 7542 7543 /* configure global TSO hardware offload settings */ 7544 wr32(&pf->hw, I40E_GLLAN_TSOMSK_F, be32_to_cpu(TCP_FLAG_PSH | 7545 TCP_FLAG_FIN) >> 16); 7546 wr32(&pf->hw, I40E_GLLAN_TSOMSK_M, be32_to_cpu(TCP_FLAG_PSH | 7547 TCP_FLAG_FIN | 7548 TCP_FLAG_CWR) >> 16); 7549 wr32(&pf->hw, I40E_GLLAN_TSOMSK_L, be32_to_cpu(TCP_FLAG_CWR) >> 16); 7550 7551 udp_tunnel_get_rx_info(netdev); 7552 7553 return 0; 7554 } 7555 7556 /** 7557 * i40e_vsi_open - 7558 * @vsi: the VSI to open 7559 * 7560 * Finish initialization of the VSI. 7561 * 7562 * Returns 0 on success, negative value on failure 7563 * 7564 * Note: expects to be called while under rtnl_lock() 7565 **/ 7566 int i40e_vsi_open(struct i40e_vsi *vsi) 7567 { 7568 struct i40e_pf *pf = vsi->back; 7569 char int_name[I40E_INT_NAME_STR_LEN]; 7570 int err; 7571 7572 /* allocate descriptors */ 7573 err = i40e_vsi_setup_tx_resources(vsi); 7574 if (err) 7575 goto err_setup_tx; 7576 err = i40e_vsi_setup_rx_resources(vsi); 7577 if (err) 7578 goto err_setup_rx; 7579 7580 err = i40e_vsi_configure(vsi); 7581 if (err) 7582 goto err_setup_rx; 7583 7584 if (vsi->netdev) { 7585 snprintf(int_name, sizeof(int_name) - 1, "%s-%s", 7586 dev_driver_string(&pf->pdev->dev), vsi->netdev->name); 7587 err = i40e_vsi_request_irq(vsi, int_name); 7588 if (err) 7589 goto err_setup_rx; 7590 7591 /* Notify the stack of the actual queue counts. */ 7592 err = netif_set_real_num_tx_queues(vsi->netdev, 7593 vsi->num_queue_pairs); 7594 if (err) 7595 goto err_set_queues; 7596 7597 err = netif_set_real_num_rx_queues(vsi->netdev, 7598 vsi->num_queue_pairs); 7599 if (err) 7600 goto err_set_queues; 7601 7602 } else if (vsi->type == I40E_VSI_FDIR) { 7603 snprintf(int_name, sizeof(int_name) - 1, "%s-%s:fdir", 7604 dev_driver_string(&pf->pdev->dev), 7605 dev_name(&pf->pdev->dev)); 7606 err = i40e_vsi_request_irq(vsi, int_name); 7607 7608 } else { 7609 err = -EINVAL; 7610 goto err_setup_rx; 7611 } 7612 7613 err = i40e_up_complete(vsi); 7614 if (err) 7615 goto err_up_complete; 7616 7617 return 0; 7618 7619 err_up_complete: 7620 i40e_down(vsi); 7621 err_set_queues: 7622 i40e_vsi_free_irq(vsi); 7623 err_setup_rx: 7624 i40e_vsi_free_rx_resources(vsi); 7625 err_setup_tx: 7626 i40e_vsi_free_tx_resources(vsi); 7627 if (vsi == pf->vsi[pf->lan_vsi]) 7628 i40e_do_reset(pf, I40E_PF_RESET_FLAG, true); 7629 7630 return err; 7631 } 7632 7633 /** 7634 * i40e_fdir_filter_exit - Cleans up the Flow Director accounting 7635 * @pf: Pointer to PF 7636 * 7637 * This function destroys the hlist where all the Flow Director 7638 * filters were saved. 7639 **/ 7640 static void i40e_fdir_filter_exit(struct i40e_pf *pf) 7641 { 7642 struct i40e_fdir_filter *filter; 7643 struct i40e_flex_pit *pit_entry, *tmp; 7644 struct hlist_node *node2; 7645 7646 hlist_for_each_entry_safe(filter, node2, 7647 &pf->fdir_filter_list, fdir_node) { 7648 hlist_del(&filter->fdir_node); 7649 kfree(filter); 7650 } 7651 7652 list_for_each_entry_safe(pit_entry, tmp, &pf->l3_flex_pit_list, list) { 7653 list_del(&pit_entry->list); 7654 kfree(pit_entry); 7655 } 7656 INIT_LIST_HEAD(&pf->l3_flex_pit_list); 7657 7658 list_for_each_entry_safe(pit_entry, tmp, &pf->l4_flex_pit_list, list) { 7659 list_del(&pit_entry->list); 7660 kfree(pit_entry); 7661 } 7662 INIT_LIST_HEAD(&pf->l4_flex_pit_list); 7663 7664 pf->fdir_pf_active_filters = 0; 7665 pf->fd_tcp4_filter_cnt = 0; 7666 pf->fd_udp4_filter_cnt = 0; 7667 pf->fd_sctp4_filter_cnt = 0; 7668 pf->fd_ip4_filter_cnt = 0; 7669 7670 /* Reprogram the default input set for TCP/IPv4 */ 7671 i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV4_TCP, 7672 I40E_L3_SRC_MASK | I40E_L3_DST_MASK | 7673 I40E_L4_SRC_MASK | I40E_L4_DST_MASK); 7674 7675 /* Reprogram the default input set for UDP/IPv4 */ 7676 i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV4_UDP, 7677 I40E_L3_SRC_MASK | I40E_L3_DST_MASK | 7678 I40E_L4_SRC_MASK | I40E_L4_DST_MASK); 7679 7680 /* Reprogram the default input set for SCTP/IPv4 */ 7681 i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV4_SCTP, 7682 I40E_L3_SRC_MASK | I40E_L3_DST_MASK | 7683 I40E_L4_SRC_MASK | I40E_L4_DST_MASK); 7684 7685 /* Reprogram the default input set for Other/IPv4 */ 7686 i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV4_OTHER, 7687 I40E_L3_SRC_MASK | I40E_L3_DST_MASK); 7688 7689 i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_FRAG_IPV4, 7690 I40E_L3_SRC_MASK | I40E_L3_DST_MASK); 7691 } 7692 7693 /** 7694 * i40e_cloud_filter_exit - Cleans up the cloud filters 7695 * @pf: Pointer to PF 7696 * 7697 * This function destroys the hlist where all the cloud filters 7698 * were saved. 7699 **/ 7700 static void i40e_cloud_filter_exit(struct i40e_pf *pf) 7701 { 7702 struct i40e_cloud_filter *cfilter; 7703 struct hlist_node *node; 7704 7705 hlist_for_each_entry_safe(cfilter, node, 7706 &pf->cloud_filter_list, cloud_node) { 7707 hlist_del(&cfilter->cloud_node); 7708 kfree(cfilter); 7709 } 7710 pf->num_cloud_filters = 0; 7711 7712 if ((pf->flags & I40E_FLAG_FD_SB_TO_CLOUD_FILTER) && 7713 !(pf->flags & I40E_FLAG_FD_SB_INACTIVE)) { 7714 pf->flags |= I40E_FLAG_FD_SB_ENABLED; 7715 pf->flags &= ~I40E_FLAG_FD_SB_TO_CLOUD_FILTER; 7716 pf->flags &= ~I40E_FLAG_FD_SB_INACTIVE; 7717 } 7718 } 7719 7720 /** 7721 * i40e_close - Disables a network interface 7722 * @netdev: network interface device structure 7723 * 7724 * The close entry point is called when an interface is de-activated 7725 * by the OS. The hardware is still under the driver's control, but 7726 * this netdev interface is disabled. 7727 * 7728 * Returns 0, this is not allowed to fail 7729 **/ 7730 int i40e_close(struct net_device *netdev) 7731 { 7732 struct i40e_netdev_priv *np = netdev_priv(netdev); 7733 struct i40e_vsi *vsi = np->vsi; 7734 7735 i40e_vsi_close(vsi); 7736 7737 return 0; 7738 } 7739 7740 /** 7741 * i40e_do_reset - Start a PF or Core Reset sequence 7742 * @pf: board private structure 7743 * @reset_flags: which reset is requested 7744 * @lock_acquired: indicates whether or not the lock has been acquired 7745 * before this function was called. 7746 * 7747 * The essential difference in resets is that the PF Reset 7748 * doesn't clear the packet buffers, doesn't reset the PE 7749 * firmware, and doesn't bother the other PFs on the chip. 7750 **/ 7751 void i40e_do_reset(struct i40e_pf *pf, u32 reset_flags, bool lock_acquired) 7752 { 7753 u32 val; 7754 7755 WARN_ON(in_interrupt()); 7756 7757 7758 /* do the biggest reset indicated */ 7759 if (reset_flags & BIT_ULL(__I40E_GLOBAL_RESET_REQUESTED)) { 7760 7761 /* Request a Global Reset 7762 * 7763 * This will start the chip's countdown to the actual full 7764 * chip reset event, and a warning interrupt to be sent 7765 * to all PFs, including the requestor. Our handler 7766 * for the warning interrupt will deal with the shutdown 7767 * and recovery of the switch setup. 7768 */ 7769 dev_dbg(&pf->pdev->dev, "GlobalR requested\n"); 7770 val = rd32(&pf->hw, I40E_GLGEN_RTRIG); 7771 val |= I40E_GLGEN_RTRIG_GLOBR_MASK; 7772 wr32(&pf->hw, I40E_GLGEN_RTRIG, val); 7773 7774 } else if (reset_flags & BIT_ULL(__I40E_CORE_RESET_REQUESTED)) { 7775 7776 /* Request a Core Reset 7777 * 7778 * Same as Global Reset, except does *not* include the MAC/PHY 7779 */ 7780 dev_dbg(&pf->pdev->dev, "CoreR requested\n"); 7781 val = rd32(&pf->hw, I40E_GLGEN_RTRIG); 7782 val |= I40E_GLGEN_RTRIG_CORER_MASK; 7783 wr32(&pf->hw, I40E_GLGEN_RTRIG, val); 7784 i40e_flush(&pf->hw); 7785 7786 } else if (reset_flags & I40E_PF_RESET_FLAG) { 7787 7788 /* Request a PF Reset 7789 * 7790 * Resets only the PF-specific registers 7791 * 7792 * This goes directly to the tear-down and rebuild of 7793 * the switch, since we need to do all the recovery as 7794 * for the Core Reset. 7795 */ 7796 dev_dbg(&pf->pdev->dev, "PFR requested\n"); 7797 i40e_handle_reset_warning(pf, lock_acquired); 7798 7799 } else if (reset_flags & BIT_ULL(__I40E_REINIT_REQUESTED)) { 7800 int v; 7801 7802 /* Find the VSI(s) that requested a re-init */ 7803 dev_info(&pf->pdev->dev, 7804 "VSI reinit requested\n"); 7805 for (v = 0; v < pf->num_alloc_vsi; v++) { 7806 struct i40e_vsi *vsi = pf->vsi[v]; 7807 7808 if (vsi != NULL && 7809 test_and_clear_bit(__I40E_VSI_REINIT_REQUESTED, 7810 vsi->state)) 7811 i40e_vsi_reinit_locked(pf->vsi[v]); 7812 } 7813 } else if (reset_flags & BIT_ULL(__I40E_DOWN_REQUESTED)) { 7814 int v; 7815 7816 /* Find the VSI(s) that needs to be brought down */ 7817 dev_info(&pf->pdev->dev, "VSI down requested\n"); 7818 for (v = 0; v < pf->num_alloc_vsi; v++) { 7819 struct i40e_vsi *vsi = pf->vsi[v]; 7820 7821 if (vsi != NULL && 7822 test_and_clear_bit(__I40E_VSI_DOWN_REQUESTED, 7823 vsi->state)) { 7824 set_bit(__I40E_VSI_DOWN, vsi->state); 7825 i40e_down(vsi); 7826 } 7827 } 7828 } else { 7829 dev_info(&pf->pdev->dev, 7830 "bad reset request 0x%08x\n", reset_flags); 7831 } 7832 } 7833 7834 #ifdef CONFIG_I40E_DCB 7835 /** 7836 * i40e_dcb_need_reconfig - Check if DCB needs reconfig 7837 * @pf: board private structure 7838 * @old_cfg: current DCB config 7839 * @new_cfg: new DCB config 7840 **/ 7841 bool i40e_dcb_need_reconfig(struct i40e_pf *pf, 7842 struct i40e_dcbx_config *old_cfg, 7843 struct i40e_dcbx_config *new_cfg) 7844 { 7845 bool need_reconfig = false; 7846 7847 /* Check if ETS configuration has changed */ 7848 if (memcmp(&new_cfg->etscfg, 7849 &old_cfg->etscfg, 7850 sizeof(new_cfg->etscfg))) { 7851 /* If Priority Table has changed reconfig is needed */ 7852 if (memcmp(&new_cfg->etscfg.prioritytable, 7853 &old_cfg->etscfg.prioritytable, 7854 sizeof(new_cfg->etscfg.prioritytable))) { 7855 need_reconfig = true; 7856 dev_dbg(&pf->pdev->dev, "ETS UP2TC changed.\n"); 7857 } 7858 7859 if (memcmp(&new_cfg->etscfg.tcbwtable, 7860 &old_cfg->etscfg.tcbwtable, 7861 sizeof(new_cfg->etscfg.tcbwtable))) 7862 dev_dbg(&pf->pdev->dev, "ETS TC BW Table changed.\n"); 7863 7864 if (memcmp(&new_cfg->etscfg.tsatable, 7865 &old_cfg->etscfg.tsatable, 7866 sizeof(new_cfg->etscfg.tsatable))) 7867 dev_dbg(&pf->pdev->dev, "ETS TSA Table changed.\n"); 7868 } 7869 7870 /* Check if PFC configuration has changed */ 7871 if (memcmp(&new_cfg->pfc, 7872 &old_cfg->pfc, 7873 sizeof(new_cfg->pfc))) { 7874 need_reconfig = true; 7875 dev_dbg(&pf->pdev->dev, "PFC config change detected.\n"); 7876 } 7877 7878 /* Check if APP Table has changed */ 7879 if (memcmp(&new_cfg->app, 7880 &old_cfg->app, 7881 sizeof(new_cfg->app))) { 7882 need_reconfig = true; 7883 dev_dbg(&pf->pdev->dev, "APP Table change detected.\n"); 7884 } 7885 7886 dev_dbg(&pf->pdev->dev, "dcb need_reconfig=%d\n", need_reconfig); 7887 return need_reconfig; 7888 } 7889 7890 /** 7891 * i40e_handle_lldp_event - Handle LLDP Change MIB event 7892 * @pf: board private structure 7893 * @e: event info posted on ARQ 7894 **/ 7895 static int i40e_handle_lldp_event(struct i40e_pf *pf, 7896 struct i40e_arq_event_info *e) 7897 { 7898 struct i40e_aqc_lldp_get_mib *mib = 7899 (struct i40e_aqc_lldp_get_mib *)&e->desc.params.raw; 7900 struct i40e_hw *hw = &pf->hw; 7901 struct i40e_dcbx_config tmp_dcbx_cfg; 7902 bool need_reconfig = false; 7903 int ret = 0; 7904 u8 type; 7905 7906 /* Not DCB capable or capability disabled */ 7907 if (!(pf->flags & I40E_FLAG_DCB_CAPABLE)) 7908 return ret; 7909 7910 /* Ignore if event is not for Nearest Bridge */ 7911 type = ((mib->type >> I40E_AQ_LLDP_BRIDGE_TYPE_SHIFT) 7912 & I40E_AQ_LLDP_BRIDGE_TYPE_MASK); 7913 dev_dbg(&pf->pdev->dev, "LLDP event mib bridge type 0x%x\n", type); 7914 if (type != I40E_AQ_LLDP_BRIDGE_TYPE_NEAREST_BRIDGE) 7915 return ret; 7916 7917 /* Check MIB Type and return if event for Remote MIB update */ 7918 type = mib->type & I40E_AQ_LLDP_MIB_TYPE_MASK; 7919 dev_dbg(&pf->pdev->dev, 7920 "LLDP event mib type %s\n", type ? "remote" : "local"); 7921 if (type == I40E_AQ_LLDP_MIB_REMOTE) { 7922 /* Update the remote cached instance and return */ 7923 ret = i40e_aq_get_dcb_config(hw, I40E_AQ_LLDP_MIB_REMOTE, 7924 I40E_AQ_LLDP_BRIDGE_TYPE_NEAREST_BRIDGE, 7925 &hw->remote_dcbx_config); 7926 goto exit; 7927 } 7928 7929 /* Store the old configuration */ 7930 tmp_dcbx_cfg = hw->local_dcbx_config; 7931 7932 /* Reset the old DCBx configuration data */ 7933 memset(&hw->local_dcbx_config, 0, sizeof(hw->local_dcbx_config)); 7934 /* Get updated DCBX data from firmware */ 7935 ret = i40e_get_dcb_config(&pf->hw); 7936 if (ret) { 7937 dev_info(&pf->pdev->dev, 7938 "Failed querying DCB configuration data from firmware, err %s aq_err %s\n", 7939 i40e_stat_str(&pf->hw, ret), 7940 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 7941 goto exit; 7942 } 7943 7944 /* No change detected in DCBX configs */ 7945 if (!memcmp(&tmp_dcbx_cfg, &hw->local_dcbx_config, 7946 sizeof(tmp_dcbx_cfg))) { 7947 dev_dbg(&pf->pdev->dev, "No change detected in DCBX configuration.\n"); 7948 goto exit; 7949 } 7950 7951 need_reconfig = i40e_dcb_need_reconfig(pf, &tmp_dcbx_cfg, 7952 &hw->local_dcbx_config); 7953 7954 i40e_dcbnl_flush_apps(pf, &tmp_dcbx_cfg, &hw->local_dcbx_config); 7955 7956 if (!need_reconfig) 7957 goto exit; 7958 7959 /* Enable DCB tagging only when more than one TC */ 7960 if (i40e_dcb_get_num_tc(&hw->local_dcbx_config) > 1) 7961 pf->flags |= I40E_FLAG_DCB_ENABLED; 7962 else 7963 pf->flags &= ~I40E_FLAG_DCB_ENABLED; 7964 7965 set_bit(__I40E_PORT_SUSPENDED, pf->state); 7966 /* Reconfiguration needed quiesce all VSIs */ 7967 i40e_pf_quiesce_all_vsi(pf); 7968 7969 /* Changes in configuration update VEB/VSI */ 7970 i40e_dcb_reconfigure(pf); 7971 7972 ret = i40e_resume_port_tx(pf); 7973 7974 clear_bit(__I40E_PORT_SUSPENDED, pf->state); 7975 /* In case of error no point in resuming VSIs */ 7976 if (ret) 7977 goto exit; 7978 7979 /* Wait for the PF's queues to be disabled */ 7980 ret = i40e_pf_wait_queues_disabled(pf); 7981 if (ret) { 7982 /* Schedule PF reset to recover */ 7983 set_bit(__I40E_PF_RESET_REQUESTED, pf->state); 7984 i40e_service_event_schedule(pf); 7985 } else { 7986 i40e_pf_unquiesce_all_vsi(pf); 7987 pf->flags |= (I40E_FLAG_SERVICE_CLIENT_REQUESTED | 7988 I40E_FLAG_CLIENT_L2_CHANGE); 7989 } 7990 7991 exit: 7992 return ret; 7993 } 7994 #endif /* CONFIG_I40E_DCB */ 7995 7996 /** 7997 * i40e_do_reset_safe - Protected reset path for userland calls. 7998 * @pf: board private structure 7999 * @reset_flags: which reset is requested 8000 * 8001 **/ 8002 void i40e_do_reset_safe(struct i40e_pf *pf, u32 reset_flags) 8003 { 8004 rtnl_lock(); 8005 i40e_do_reset(pf, reset_flags, true); 8006 rtnl_unlock(); 8007 } 8008 8009 /** 8010 * i40e_handle_lan_overflow_event - Handler for LAN queue overflow event 8011 * @pf: board private structure 8012 * @e: event info posted on ARQ 8013 * 8014 * Handler for LAN Queue Overflow Event generated by the firmware for PF 8015 * and VF queues 8016 **/ 8017 static void i40e_handle_lan_overflow_event(struct i40e_pf *pf, 8018 struct i40e_arq_event_info *e) 8019 { 8020 struct i40e_aqc_lan_overflow *data = 8021 (struct i40e_aqc_lan_overflow *)&e->desc.params.raw; 8022 u32 queue = le32_to_cpu(data->prtdcb_rupto); 8023 u32 qtx_ctl = le32_to_cpu(data->otx_ctl); 8024 struct i40e_hw *hw = &pf->hw; 8025 struct i40e_vf *vf; 8026 u16 vf_id; 8027 8028 dev_dbg(&pf->pdev->dev, "overflow Rx Queue Number = %d QTX_CTL=0x%08x\n", 8029 queue, qtx_ctl); 8030 8031 /* Queue belongs to VF, find the VF and issue VF reset */ 8032 if (((qtx_ctl & I40E_QTX_CTL_PFVF_Q_MASK) 8033 >> I40E_QTX_CTL_PFVF_Q_SHIFT) == I40E_QTX_CTL_VF_QUEUE) { 8034 vf_id = (u16)((qtx_ctl & I40E_QTX_CTL_VFVM_INDX_MASK) 8035 >> I40E_QTX_CTL_VFVM_INDX_SHIFT); 8036 vf_id -= hw->func_caps.vf_base_id; 8037 vf = &pf->vf[vf_id]; 8038 i40e_vc_notify_vf_reset(vf); 8039 /* Allow VF to process pending reset notification */ 8040 msleep(20); 8041 i40e_reset_vf(vf, false); 8042 } 8043 } 8044 8045 /** 8046 * i40e_get_cur_guaranteed_fd_count - Get the consumed guaranteed FD filters 8047 * @pf: board private structure 8048 **/ 8049 u32 i40e_get_cur_guaranteed_fd_count(struct i40e_pf *pf) 8050 { 8051 u32 val, fcnt_prog; 8052 8053 val = rd32(&pf->hw, I40E_PFQF_FDSTAT); 8054 fcnt_prog = (val & I40E_PFQF_FDSTAT_GUARANT_CNT_MASK); 8055 return fcnt_prog; 8056 } 8057 8058 /** 8059 * i40e_get_current_fd_count - Get total FD filters programmed for this PF 8060 * @pf: board private structure 8061 **/ 8062 u32 i40e_get_current_fd_count(struct i40e_pf *pf) 8063 { 8064 u32 val, fcnt_prog; 8065 8066 val = rd32(&pf->hw, I40E_PFQF_FDSTAT); 8067 fcnt_prog = (val & I40E_PFQF_FDSTAT_GUARANT_CNT_MASK) + 8068 ((val & I40E_PFQF_FDSTAT_BEST_CNT_MASK) >> 8069 I40E_PFQF_FDSTAT_BEST_CNT_SHIFT); 8070 return fcnt_prog; 8071 } 8072 8073 /** 8074 * i40e_get_global_fd_count - Get total FD filters programmed on device 8075 * @pf: board private structure 8076 **/ 8077 u32 i40e_get_global_fd_count(struct i40e_pf *pf) 8078 { 8079 u32 val, fcnt_prog; 8080 8081 val = rd32(&pf->hw, I40E_GLQF_FDCNT_0); 8082 fcnt_prog = (val & I40E_GLQF_FDCNT_0_GUARANT_CNT_MASK) + 8083 ((val & I40E_GLQF_FDCNT_0_BESTCNT_MASK) >> 8084 I40E_GLQF_FDCNT_0_BESTCNT_SHIFT); 8085 return fcnt_prog; 8086 } 8087 8088 /** 8089 * i40e_fdir_check_and_reenable - Function to reenabe FD ATR or SB if disabled 8090 * @pf: board private structure 8091 **/ 8092 void i40e_fdir_check_and_reenable(struct i40e_pf *pf) 8093 { 8094 struct i40e_fdir_filter *filter; 8095 u32 fcnt_prog, fcnt_avail; 8096 struct hlist_node *node; 8097 8098 if (test_bit(__I40E_FD_FLUSH_REQUESTED, pf->state)) 8099 return; 8100 8101 /* Check if we have enough room to re-enable FDir SB capability. */ 8102 fcnt_prog = i40e_get_global_fd_count(pf); 8103 fcnt_avail = pf->fdir_pf_filter_count; 8104 if ((fcnt_prog < (fcnt_avail - I40E_FDIR_BUFFER_HEAD_ROOM)) || 8105 (pf->fd_add_err == 0) || 8106 (i40e_get_current_atr_cnt(pf) < pf->fd_atr_cnt)) { 8107 if (pf->flags & I40E_FLAG_FD_SB_AUTO_DISABLED) { 8108 pf->flags &= ~I40E_FLAG_FD_SB_AUTO_DISABLED; 8109 if ((pf->flags & I40E_FLAG_FD_SB_ENABLED) && 8110 (I40E_DEBUG_FD & pf->hw.debug_mask)) 8111 dev_info(&pf->pdev->dev, "FD Sideband/ntuple is being enabled since we have space in the table now\n"); 8112 } 8113 } 8114 8115 /* We should wait for even more space before re-enabling ATR. 8116 * Additionally, we cannot enable ATR as long as we still have TCP SB 8117 * rules active. 8118 */ 8119 if ((fcnt_prog < (fcnt_avail - I40E_FDIR_BUFFER_HEAD_ROOM_FOR_ATR)) && 8120 (pf->fd_tcp4_filter_cnt == 0)) { 8121 if (pf->flags & I40E_FLAG_FD_ATR_AUTO_DISABLED) { 8122 pf->flags &= ~I40E_FLAG_FD_ATR_AUTO_DISABLED; 8123 if ((pf->flags & I40E_FLAG_FD_ATR_ENABLED) && 8124 (I40E_DEBUG_FD & pf->hw.debug_mask)) 8125 dev_info(&pf->pdev->dev, "ATR is being enabled since we have space in the table and there are no conflicting ntuple rules\n"); 8126 } 8127 } 8128 8129 /* if hw had a problem adding a filter, delete it */ 8130 if (pf->fd_inv > 0) { 8131 hlist_for_each_entry_safe(filter, node, 8132 &pf->fdir_filter_list, fdir_node) { 8133 if (filter->fd_id == pf->fd_inv) { 8134 hlist_del(&filter->fdir_node); 8135 kfree(filter); 8136 pf->fdir_pf_active_filters--; 8137 pf->fd_inv = 0; 8138 } 8139 } 8140 } 8141 } 8142 8143 #define I40E_MIN_FD_FLUSH_INTERVAL 10 8144 #define I40E_MIN_FD_FLUSH_SB_ATR_UNSTABLE 30 8145 /** 8146 * i40e_fdir_flush_and_replay - Function to flush all FD filters and replay SB 8147 * @pf: board private structure 8148 **/ 8149 static void i40e_fdir_flush_and_replay(struct i40e_pf *pf) 8150 { 8151 unsigned long min_flush_time; 8152 int flush_wait_retry = 50; 8153 bool disable_atr = false; 8154 int fd_room; 8155 int reg; 8156 8157 if (!time_after(jiffies, pf->fd_flush_timestamp + 8158 (I40E_MIN_FD_FLUSH_INTERVAL * HZ))) 8159 return; 8160 8161 /* If the flush is happening too quick and we have mostly SB rules we 8162 * should not re-enable ATR for some time. 8163 */ 8164 min_flush_time = pf->fd_flush_timestamp + 8165 (I40E_MIN_FD_FLUSH_SB_ATR_UNSTABLE * HZ); 8166 fd_room = pf->fdir_pf_filter_count - pf->fdir_pf_active_filters; 8167 8168 if (!(time_after(jiffies, min_flush_time)) && 8169 (fd_room < I40E_FDIR_BUFFER_HEAD_ROOM_FOR_ATR)) { 8170 if (I40E_DEBUG_FD & pf->hw.debug_mask) 8171 dev_info(&pf->pdev->dev, "ATR disabled, not enough FD filter space.\n"); 8172 disable_atr = true; 8173 } 8174 8175 pf->fd_flush_timestamp = jiffies; 8176 pf->flags |= I40E_FLAG_FD_ATR_AUTO_DISABLED; 8177 /* flush all filters */ 8178 wr32(&pf->hw, I40E_PFQF_CTL_1, 8179 I40E_PFQF_CTL_1_CLEARFDTABLE_MASK); 8180 i40e_flush(&pf->hw); 8181 pf->fd_flush_cnt++; 8182 pf->fd_add_err = 0; 8183 do { 8184 /* Check FD flush status every 5-6msec */ 8185 usleep_range(5000, 6000); 8186 reg = rd32(&pf->hw, I40E_PFQF_CTL_1); 8187 if (!(reg & I40E_PFQF_CTL_1_CLEARFDTABLE_MASK)) 8188 break; 8189 } while (flush_wait_retry--); 8190 if (reg & I40E_PFQF_CTL_1_CLEARFDTABLE_MASK) { 8191 dev_warn(&pf->pdev->dev, "FD table did not flush, needs more time\n"); 8192 } else { 8193 /* replay sideband filters */ 8194 i40e_fdir_filter_restore(pf->vsi[pf->lan_vsi]); 8195 if (!disable_atr && !pf->fd_tcp4_filter_cnt) 8196 pf->flags &= ~I40E_FLAG_FD_ATR_AUTO_DISABLED; 8197 clear_bit(__I40E_FD_FLUSH_REQUESTED, pf->state); 8198 if (I40E_DEBUG_FD & pf->hw.debug_mask) 8199 dev_info(&pf->pdev->dev, "FD Filter table flushed and FD-SB replayed.\n"); 8200 } 8201 } 8202 8203 /** 8204 * i40e_get_current_atr_count - Get the count of total FD ATR filters programmed 8205 * @pf: board private structure 8206 **/ 8207 u32 i40e_get_current_atr_cnt(struct i40e_pf *pf) 8208 { 8209 return i40e_get_current_fd_count(pf) - pf->fdir_pf_active_filters; 8210 } 8211 8212 /* We can see up to 256 filter programming desc in transit if the filters are 8213 * being applied really fast; before we see the first 8214 * filter miss error on Rx queue 0. Accumulating enough error messages before 8215 * reacting will make sure we don't cause flush too often. 8216 */ 8217 #define I40E_MAX_FD_PROGRAM_ERROR 256 8218 8219 /** 8220 * i40e_fdir_reinit_subtask - Worker thread to reinit FDIR filter table 8221 * @pf: board private structure 8222 **/ 8223 static void i40e_fdir_reinit_subtask(struct i40e_pf *pf) 8224 { 8225 8226 /* if interface is down do nothing */ 8227 if (test_bit(__I40E_DOWN, pf->state)) 8228 return; 8229 8230 if (test_bit(__I40E_FD_FLUSH_REQUESTED, pf->state)) 8231 i40e_fdir_flush_and_replay(pf); 8232 8233 i40e_fdir_check_and_reenable(pf); 8234 8235 } 8236 8237 /** 8238 * i40e_vsi_link_event - notify VSI of a link event 8239 * @vsi: vsi to be notified 8240 * @link_up: link up or down 8241 **/ 8242 static void i40e_vsi_link_event(struct i40e_vsi *vsi, bool link_up) 8243 { 8244 if (!vsi || test_bit(__I40E_VSI_DOWN, vsi->state)) 8245 return; 8246 8247 switch (vsi->type) { 8248 case I40E_VSI_MAIN: 8249 if (!vsi->netdev || !vsi->netdev_registered) 8250 break; 8251 8252 if (link_up) { 8253 netif_carrier_on(vsi->netdev); 8254 netif_tx_wake_all_queues(vsi->netdev); 8255 } else { 8256 netif_carrier_off(vsi->netdev); 8257 netif_tx_stop_all_queues(vsi->netdev); 8258 } 8259 break; 8260 8261 case I40E_VSI_SRIOV: 8262 case I40E_VSI_VMDQ2: 8263 case I40E_VSI_CTRL: 8264 case I40E_VSI_IWARP: 8265 case I40E_VSI_MIRROR: 8266 default: 8267 /* there is no notification for other VSIs */ 8268 break; 8269 } 8270 } 8271 8272 /** 8273 * i40e_veb_link_event - notify elements on the veb of a link event 8274 * @veb: veb to be notified 8275 * @link_up: link up or down 8276 **/ 8277 static void i40e_veb_link_event(struct i40e_veb *veb, bool link_up) 8278 { 8279 struct i40e_pf *pf; 8280 int i; 8281 8282 if (!veb || !veb->pf) 8283 return; 8284 pf = veb->pf; 8285 8286 /* depth first... */ 8287 for (i = 0; i < I40E_MAX_VEB; i++) 8288 if (pf->veb[i] && (pf->veb[i]->uplink_seid == veb->seid)) 8289 i40e_veb_link_event(pf->veb[i], link_up); 8290 8291 /* ... now the local VSIs */ 8292 for (i = 0; i < pf->num_alloc_vsi; i++) 8293 if (pf->vsi[i] && (pf->vsi[i]->uplink_seid == veb->seid)) 8294 i40e_vsi_link_event(pf->vsi[i], link_up); 8295 } 8296 8297 /** 8298 * i40e_link_event - Update netif_carrier status 8299 * @pf: board private structure 8300 **/ 8301 static void i40e_link_event(struct i40e_pf *pf) 8302 { 8303 struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi]; 8304 u8 new_link_speed, old_link_speed; 8305 i40e_status status; 8306 bool new_link, old_link; 8307 8308 /* save off old link status information */ 8309 pf->hw.phy.link_info_old = pf->hw.phy.link_info; 8310 8311 /* set this to force the get_link_status call to refresh state */ 8312 pf->hw.phy.get_link_info = true; 8313 8314 old_link = (pf->hw.phy.link_info_old.link_info & I40E_AQ_LINK_UP); 8315 8316 status = i40e_get_link_status(&pf->hw, &new_link); 8317 8318 /* On success, disable temp link polling */ 8319 if (status == I40E_SUCCESS) { 8320 if (pf->flags & I40E_FLAG_TEMP_LINK_POLLING) 8321 pf->flags &= ~I40E_FLAG_TEMP_LINK_POLLING; 8322 } else { 8323 /* Enable link polling temporarily until i40e_get_link_status 8324 * returns I40E_SUCCESS 8325 */ 8326 pf->flags |= I40E_FLAG_TEMP_LINK_POLLING; 8327 dev_dbg(&pf->pdev->dev, "couldn't get link state, status: %d\n", 8328 status); 8329 return; 8330 } 8331 8332 old_link_speed = pf->hw.phy.link_info_old.link_speed; 8333 new_link_speed = pf->hw.phy.link_info.link_speed; 8334 8335 if (new_link == old_link && 8336 new_link_speed == old_link_speed && 8337 (test_bit(__I40E_VSI_DOWN, vsi->state) || 8338 new_link == netif_carrier_ok(vsi->netdev))) 8339 return; 8340 8341 i40e_print_link_message(vsi, new_link); 8342 8343 /* Notify the base of the switch tree connected to 8344 * the link. Floating VEBs are not notified. 8345 */ 8346 if (pf->lan_veb != I40E_NO_VEB && pf->veb[pf->lan_veb]) 8347 i40e_veb_link_event(pf->veb[pf->lan_veb], new_link); 8348 else 8349 i40e_vsi_link_event(vsi, new_link); 8350 8351 if (pf->vf) 8352 i40e_vc_notify_link_state(pf); 8353 8354 if (pf->flags & I40E_FLAG_PTP) 8355 i40e_ptp_set_increment(pf); 8356 } 8357 8358 /** 8359 * i40e_watchdog_subtask - periodic checks not using event driven response 8360 * @pf: board private structure 8361 **/ 8362 static void i40e_watchdog_subtask(struct i40e_pf *pf) 8363 { 8364 int i; 8365 8366 /* if interface is down do nothing */ 8367 if (test_bit(__I40E_DOWN, pf->state) || 8368 test_bit(__I40E_CONFIG_BUSY, pf->state)) 8369 return; 8370 8371 /* make sure we don't do these things too often */ 8372 if (time_before(jiffies, (pf->service_timer_previous + 8373 pf->service_timer_period))) 8374 return; 8375 pf->service_timer_previous = jiffies; 8376 8377 if ((pf->flags & I40E_FLAG_LINK_POLLING_ENABLED) || 8378 (pf->flags & I40E_FLAG_TEMP_LINK_POLLING)) 8379 i40e_link_event(pf); 8380 8381 /* Update the stats for active netdevs so the network stack 8382 * can look at updated numbers whenever it cares to 8383 */ 8384 for (i = 0; i < pf->num_alloc_vsi; i++) 8385 if (pf->vsi[i] && pf->vsi[i]->netdev) 8386 i40e_update_stats(pf->vsi[i]); 8387 8388 if (pf->flags & I40E_FLAG_VEB_STATS_ENABLED) { 8389 /* Update the stats for the active switching components */ 8390 for (i = 0; i < I40E_MAX_VEB; i++) 8391 if (pf->veb[i]) 8392 i40e_update_veb_stats(pf->veb[i]); 8393 } 8394 8395 i40e_ptp_rx_hang(pf); 8396 i40e_ptp_tx_hang(pf); 8397 } 8398 8399 /** 8400 * i40e_reset_subtask - Set up for resetting the device and driver 8401 * @pf: board private structure 8402 **/ 8403 static void i40e_reset_subtask(struct i40e_pf *pf) 8404 { 8405 u32 reset_flags = 0; 8406 8407 if (test_bit(__I40E_REINIT_REQUESTED, pf->state)) { 8408 reset_flags |= BIT(__I40E_REINIT_REQUESTED); 8409 clear_bit(__I40E_REINIT_REQUESTED, pf->state); 8410 } 8411 if (test_bit(__I40E_PF_RESET_REQUESTED, pf->state)) { 8412 reset_flags |= BIT(__I40E_PF_RESET_REQUESTED); 8413 clear_bit(__I40E_PF_RESET_REQUESTED, pf->state); 8414 } 8415 if (test_bit(__I40E_CORE_RESET_REQUESTED, pf->state)) { 8416 reset_flags |= BIT(__I40E_CORE_RESET_REQUESTED); 8417 clear_bit(__I40E_CORE_RESET_REQUESTED, pf->state); 8418 } 8419 if (test_bit(__I40E_GLOBAL_RESET_REQUESTED, pf->state)) { 8420 reset_flags |= BIT(__I40E_GLOBAL_RESET_REQUESTED); 8421 clear_bit(__I40E_GLOBAL_RESET_REQUESTED, pf->state); 8422 } 8423 if (test_bit(__I40E_DOWN_REQUESTED, pf->state)) { 8424 reset_flags |= BIT(__I40E_DOWN_REQUESTED); 8425 clear_bit(__I40E_DOWN_REQUESTED, pf->state); 8426 } 8427 8428 /* If there's a recovery already waiting, it takes 8429 * precedence before starting a new reset sequence. 8430 */ 8431 if (test_bit(__I40E_RESET_INTR_RECEIVED, pf->state)) { 8432 i40e_prep_for_reset(pf, false); 8433 i40e_reset(pf); 8434 i40e_rebuild(pf, false, false); 8435 } 8436 8437 /* If we're already down or resetting, just bail */ 8438 if (reset_flags && 8439 !test_bit(__I40E_DOWN, pf->state) && 8440 !test_bit(__I40E_CONFIG_BUSY, pf->state)) { 8441 i40e_do_reset(pf, reset_flags, false); 8442 } 8443 } 8444 8445 /** 8446 * i40e_handle_link_event - Handle link event 8447 * @pf: board private structure 8448 * @e: event info posted on ARQ 8449 **/ 8450 static void i40e_handle_link_event(struct i40e_pf *pf, 8451 struct i40e_arq_event_info *e) 8452 { 8453 struct i40e_aqc_get_link_status *status = 8454 (struct i40e_aqc_get_link_status *)&e->desc.params.raw; 8455 8456 /* Do a new status request to re-enable LSE reporting 8457 * and load new status information into the hw struct 8458 * This completely ignores any state information 8459 * in the ARQ event info, instead choosing to always 8460 * issue the AQ update link status command. 8461 */ 8462 i40e_link_event(pf); 8463 8464 /* Check if module meets thermal requirements */ 8465 if (status->phy_type == I40E_PHY_TYPE_NOT_SUPPORTED_HIGH_TEMP) { 8466 dev_err(&pf->pdev->dev, 8467 "Rx/Tx is disabled on this device because the module does not meet thermal requirements.\n"); 8468 dev_err(&pf->pdev->dev, 8469 "Refer to the Intel(R) Ethernet Adapters and Devices User Guide for a list of supported modules.\n"); 8470 } else { 8471 /* check for unqualified module, if link is down, suppress 8472 * the message if link was forced to be down. 8473 */ 8474 if ((status->link_info & I40E_AQ_MEDIA_AVAILABLE) && 8475 (!(status->an_info & I40E_AQ_QUALIFIED_MODULE)) && 8476 (!(status->link_info & I40E_AQ_LINK_UP)) && 8477 (!(pf->flags & I40E_FLAG_LINK_DOWN_ON_CLOSE_ENABLED))) { 8478 dev_err(&pf->pdev->dev, 8479 "Rx/Tx is disabled on this device because an unsupported SFP module type was detected.\n"); 8480 dev_err(&pf->pdev->dev, 8481 "Refer to the Intel(R) Ethernet Adapters and Devices User Guide for a list of supported modules.\n"); 8482 } 8483 } 8484 } 8485 8486 /** 8487 * i40e_clean_adminq_subtask - Clean the AdminQ rings 8488 * @pf: board private structure 8489 **/ 8490 static void i40e_clean_adminq_subtask(struct i40e_pf *pf) 8491 { 8492 struct i40e_arq_event_info event; 8493 struct i40e_hw *hw = &pf->hw; 8494 u16 pending, i = 0; 8495 i40e_status ret; 8496 u16 opcode; 8497 u32 oldval; 8498 u32 val; 8499 8500 /* Do not run clean AQ when PF reset fails */ 8501 if (test_bit(__I40E_RESET_FAILED, pf->state)) 8502 return; 8503 8504 /* check for error indications */ 8505 val = rd32(&pf->hw, pf->hw.aq.arq.len); 8506 oldval = val; 8507 if (val & I40E_PF_ARQLEN_ARQVFE_MASK) { 8508 if (hw->debug_mask & I40E_DEBUG_AQ) 8509 dev_info(&pf->pdev->dev, "ARQ VF Error detected\n"); 8510 val &= ~I40E_PF_ARQLEN_ARQVFE_MASK; 8511 } 8512 if (val & I40E_PF_ARQLEN_ARQOVFL_MASK) { 8513 if (hw->debug_mask & I40E_DEBUG_AQ) 8514 dev_info(&pf->pdev->dev, "ARQ Overflow Error detected\n"); 8515 val &= ~I40E_PF_ARQLEN_ARQOVFL_MASK; 8516 pf->arq_overflows++; 8517 } 8518 if (val & I40E_PF_ARQLEN_ARQCRIT_MASK) { 8519 if (hw->debug_mask & I40E_DEBUG_AQ) 8520 dev_info(&pf->pdev->dev, "ARQ Critical Error detected\n"); 8521 val &= ~I40E_PF_ARQLEN_ARQCRIT_MASK; 8522 } 8523 if (oldval != val) 8524 wr32(&pf->hw, pf->hw.aq.arq.len, val); 8525 8526 val = rd32(&pf->hw, pf->hw.aq.asq.len); 8527 oldval = val; 8528 if (val & I40E_PF_ATQLEN_ATQVFE_MASK) { 8529 if (pf->hw.debug_mask & I40E_DEBUG_AQ) 8530 dev_info(&pf->pdev->dev, "ASQ VF Error detected\n"); 8531 val &= ~I40E_PF_ATQLEN_ATQVFE_MASK; 8532 } 8533 if (val & I40E_PF_ATQLEN_ATQOVFL_MASK) { 8534 if (pf->hw.debug_mask & I40E_DEBUG_AQ) 8535 dev_info(&pf->pdev->dev, "ASQ Overflow Error detected\n"); 8536 val &= ~I40E_PF_ATQLEN_ATQOVFL_MASK; 8537 } 8538 if (val & I40E_PF_ATQLEN_ATQCRIT_MASK) { 8539 if (pf->hw.debug_mask & I40E_DEBUG_AQ) 8540 dev_info(&pf->pdev->dev, "ASQ Critical Error detected\n"); 8541 val &= ~I40E_PF_ATQLEN_ATQCRIT_MASK; 8542 } 8543 if (oldval != val) 8544 wr32(&pf->hw, pf->hw.aq.asq.len, val); 8545 8546 event.buf_len = I40E_MAX_AQ_BUF_SIZE; 8547 event.msg_buf = kzalloc(event.buf_len, GFP_KERNEL); 8548 if (!event.msg_buf) 8549 return; 8550 8551 do { 8552 ret = i40e_clean_arq_element(hw, &event, &pending); 8553 if (ret == I40E_ERR_ADMIN_QUEUE_NO_WORK) 8554 break; 8555 else if (ret) { 8556 dev_info(&pf->pdev->dev, "ARQ event error %d\n", ret); 8557 break; 8558 } 8559 8560 opcode = le16_to_cpu(event.desc.opcode); 8561 switch (opcode) { 8562 8563 case i40e_aqc_opc_get_link_status: 8564 i40e_handle_link_event(pf, &event); 8565 break; 8566 case i40e_aqc_opc_send_msg_to_pf: 8567 ret = i40e_vc_process_vf_msg(pf, 8568 le16_to_cpu(event.desc.retval), 8569 le32_to_cpu(event.desc.cookie_high), 8570 le32_to_cpu(event.desc.cookie_low), 8571 event.msg_buf, 8572 event.msg_len); 8573 break; 8574 case i40e_aqc_opc_lldp_update_mib: 8575 dev_dbg(&pf->pdev->dev, "ARQ: Update LLDP MIB event received\n"); 8576 #ifdef CONFIG_I40E_DCB 8577 rtnl_lock(); 8578 ret = i40e_handle_lldp_event(pf, &event); 8579 rtnl_unlock(); 8580 #endif /* CONFIG_I40E_DCB */ 8581 break; 8582 case i40e_aqc_opc_event_lan_overflow: 8583 dev_dbg(&pf->pdev->dev, "ARQ LAN queue overflow event received\n"); 8584 i40e_handle_lan_overflow_event(pf, &event); 8585 break; 8586 case i40e_aqc_opc_send_msg_to_peer: 8587 dev_info(&pf->pdev->dev, "ARQ: Msg from other pf\n"); 8588 break; 8589 case i40e_aqc_opc_nvm_erase: 8590 case i40e_aqc_opc_nvm_update: 8591 case i40e_aqc_opc_oem_post_update: 8592 i40e_debug(&pf->hw, I40E_DEBUG_NVM, 8593 "ARQ NVM operation 0x%04x completed\n", 8594 opcode); 8595 break; 8596 default: 8597 dev_info(&pf->pdev->dev, 8598 "ARQ: Unknown event 0x%04x ignored\n", 8599 opcode); 8600 break; 8601 } 8602 } while (i++ < pf->adminq_work_limit); 8603 8604 if (i < pf->adminq_work_limit) 8605 clear_bit(__I40E_ADMINQ_EVENT_PENDING, pf->state); 8606 8607 /* re-enable Admin queue interrupt cause */ 8608 val = rd32(hw, I40E_PFINT_ICR0_ENA); 8609 val |= I40E_PFINT_ICR0_ENA_ADMINQ_MASK; 8610 wr32(hw, I40E_PFINT_ICR0_ENA, val); 8611 i40e_flush(hw); 8612 8613 kfree(event.msg_buf); 8614 } 8615 8616 /** 8617 * i40e_verify_eeprom - make sure eeprom is good to use 8618 * @pf: board private structure 8619 **/ 8620 static void i40e_verify_eeprom(struct i40e_pf *pf) 8621 { 8622 int err; 8623 8624 err = i40e_diag_eeprom_test(&pf->hw); 8625 if (err) { 8626 /* retry in case of garbage read */ 8627 err = i40e_diag_eeprom_test(&pf->hw); 8628 if (err) { 8629 dev_info(&pf->pdev->dev, "eeprom check failed (%d), Tx/Rx traffic disabled\n", 8630 err); 8631 set_bit(__I40E_BAD_EEPROM, pf->state); 8632 } 8633 } 8634 8635 if (!err && test_bit(__I40E_BAD_EEPROM, pf->state)) { 8636 dev_info(&pf->pdev->dev, "eeprom check passed, Tx/Rx traffic enabled\n"); 8637 clear_bit(__I40E_BAD_EEPROM, pf->state); 8638 } 8639 } 8640 8641 /** 8642 * i40e_enable_pf_switch_lb 8643 * @pf: pointer to the PF structure 8644 * 8645 * enable switch loop back or die - no point in a return value 8646 **/ 8647 static void i40e_enable_pf_switch_lb(struct i40e_pf *pf) 8648 { 8649 struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi]; 8650 struct i40e_vsi_context ctxt; 8651 int ret; 8652 8653 ctxt.seid = pf->main_vsi_seid; 8654 ctxt.pf_num = pf->hw.pf_id; 8655 ctxt.vf_num = 0; 8656 ret = i40e_aq_get_vsi_params(&pf->hw, &ctxt, NULL); 8657 if (ret) { 8658 dev_info(&pf->pdev->dev, 8659 "couldn't get PF vsi config, err %s aq_err %s\n", 8660 i40e_stat_str(&pf->hw, ret), 8661 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 8662 return; 8663 } 8664 ctxt.flags = I40E_AQ_VSI_TYPE_PF; 8665 ctxt.info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID); 8666 ctxt.info.switch_id |= cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB); 8667 8668 ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL); 8669 if (ret) { 8670 dev_info(&pf->pdev->dev, 8671 "update vsi switch failed, err %s aq_err %s\n", 8672 i40e_stat_str(&pf->hw, ret), 8673 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 8674 } 8675 } 8676 8677 /** 8678 * i40e_disable_pf_switch_lb 8679 * @pf: pointer to the PF structure 8680 * 8681 * disable switch loop back or die - no point in a return value 8682 **/ 8683 static void i40e_disable_pf_switch_lb(struct i40e_pf *pf) 8684 { 8685 struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi]; 8686 struct i40e_vsi_context ctxt; 8687 int ret; 8688 8689 ctxt.seid = pf->main_vsi_seid; 8690 ctxt.pf_num = pf->hw.pf_id; 8691 ctxt.vf_num = 0; 8692 ret = i40e_aq_get_vsi_params(&pf->hw, &ctxt, NULL); 8693 if (ret) { 8694 dev_info(&pf->pdev->dev, 8695 "couldn't get PF vsi config, err %s aq_err %s\n", 8696 i40e_stat_str(&pf->hw, ret), 8697 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 8698 return; 8699 } 8700 ctxt.flags = I40E_AQ_VSI_TYPE_PF; 8701 ctxt.info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID); 8702 ctxt.info.switch_id &= ~cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB); 8703 8704 ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL); 8705 if (ret) { 8706 dev_info(&pf->pdev->dev, 8707 "update vsi switch failed, err %s aq_err %s\n", 8708 i40e_stat_str(&pf->hw, ret), 8709 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 8710 } 8711 } 8712 8713 /** 8714 * i40e_config_bridge_mode - Configure the HW bridge mode 8715 * @veb: pointer to the bridge instance 8716 * 8717 * Configure the loop back mode for the LAN VSI that is downlink to the 8718 * specified HW bridge instance. It is expected this function is called 8719 * when a new HW bridge is instantiated. 8720 **/ 8721 static void i40e_config_bridge_mode(struct i40e_veb *veb) 8722 { 8723 struct i40e_pf *pf = veb->pf; 8724 8725 if (pf->hw.debug_mask & I40E_DEBUG_LAN) 8726 dev_info(&pf->pdev->dev, "enabling bridge mode: %s\n", 8727 veb->bridge_mode == BRIDGE_MODE_VEPA ? "VEPA" : "VEB"); 8728 if (veb->bridge_mode & BRIDGE_MODE_VEPA) 8729 i40e_disable_pf_switch_lb(pf); 8730 else 8731 i40e_enable_pf_switch_lb(pf); 8732 } 8733 8734 /** 8735 * i40e_reconstitute_veb - rebuild the VEB and anything connected to it 8736 * @veb: pointer to the VEB instance 8737 * 8738 * This is a recursive function that first builds the attached VSIs then 8739 * recurses in to build the next layer of VEB. We track the connections 8740 * through our own index numbers because the seid's from the HW could 8741 * change across the reset. 8742 **/ 8743 static int i40e_reconstitute_veb(struct i40e_veb *veb) 8744 { 8745 struct i40e_vsi *ctl_vsi = NULL; 8746 struct i40e_pf *pf = veb->pf; 8747 int v, veb_idx; 8748 int ret; 8749 8750 /* build VSI that owns this VEB, temporarily attached to base VEB */ 8751 for (v = 0; v < pf->num_alloc_vsi && !ctl_vsi; v++) { 8752 if (pf->vsi[v] && 8753 pf->vsi[v]->veb_idx == veb->idx && 8754 pf->vsi[v]->flags & I40E_VSI_FLAG_VEB_OWNER) { 8755 ctl_vsi = pf->vsi[v]; 8756 break; 8757 } 8758 } 8759 if (!ctl_vsi) { 8760 dev_info(&pf->pdev->dev, 8761 "missing owner VSI for veb_idx %d\n", veb->idx); 8762 ret = -ENOENT; 8763 goto end_reconstitute; 8764 } 8765 if (ctl_vsi != pf->vsi[pf->lan_vsi]) 8766 ctl_vsi->uplink_seid = pf->vsi[pf->lan_vsi]->uplink_seid; 8767 ret = i40e_add_vsi(ctl_vsi); 8768 if (ret) { 8769 dev_info(&pf->pdev->dev, 8770 "rebuild of veb_idx %d owner VSI failed: %d\n", 8771 veb->idx, ret); 8772 goto end_reconstitute; 8773 } 8774 i40e_vsi_reset_stats(ctl_vsi); 8775 8776 /* create the VEB in the switch and move the VSI onto the VEB */ 8777 ret = i40e_add_veb(veb, ctl_vsi); 8778 if (ret) 8779 goto end_reconstitute; 8780 8781 if (pf->flags & I40E_FLAG_VEB_MODE_ENABLED) 8782 veb->bridge_mode = BRIDGE_MODE_VEB; 8783 else 8784 veb->bridge_mode = BRIDGE_MODE_VEPA; 8785 i40e_config_bridge_mode(veb); 8786 8787 /* create the remaining VSIs attached to this VEB */ 8788 for (v = 0; v < pf->num_alloc_vsi; v++) { 8789 if (!pf->vsi[v] || pf->vsi[v] == ctl_vsi) 8790 continue; 8791 8792 if (pf->vsi[v]->veb_idx == veb->idx) { 8793 struct i40e_vsi *vsi = pf->vsi[v]; 8794 8795 vsi->uplink_seid = veb->seid; 8796 ret = i40e_add_vsi(vsi); 8797 if (ret) { 8798 dev_info(&pf->pdev->dev, 8799 "rebuild of vsi_idx %d failed: %d\n", 8800 v, ret); 8801 goto end_reconstitute; 8802 } 8803 i40e_vsi_reset_stats(vsi); 8804 } 8805 } 8806 8807 /* create any VEBs attached to this VEB - RECURSION */ 8808 for (veb_idx = 0; veb_idx < I40E_MAX_VEB; veb_idx++) { 8809 if (pf->veb[veb_idx] && pf->veb[veb_idx]->veb_idx == veb->idx) { 8810 pf->veb[veb_idx]->uplink_seid = veb->seid; 8811 ret = i40e_reconstitute_veb(pf->veb[veb_idx]); 8812 if (ret) 8813 break; 8814 } 8815 } 8816 8817 end_reconstitute: 8818 return ret; 8819 } 8820 8821 /** 8822 * i40e_get_capabilities - get info about the HW 8823 * @pf: the PF struct 8824 **/ 8825 static int i40e_get_capabilities(struct i40e_pf *pf, 8826 enum i40e_admin_queue_opc list_type) 8827 { 8828 struct i40e_aqc_list_capabilities_element_resp *cap_buf; 8829 u16 data_size; 8830 int buf_len; 8831 int err; 8832 8833 buf_len = 40 * sizeof(struct i40e_aqc_list_capabilities_element_resp); 8834 do { 8835 cap_buf = kzalloc(buf_len, GFP_KERNEL); 8836 if (!cap_buf) 8837 return -ENOMEM; 8838 8839 /* this loads the data into the hw struct for us */ 8840 err = i40e_aq_discover_capabilities(&pf->hw, cap_buf, buf_len, 8841 &data_size, list_type, 8842 NULL); 8843 /* data loaded, buffer no longer needed */ 8844 kfree(cap_buf); 8845 8846 if (pf->hw.aq.asq_last_status == I40E_AQ_RC_ENOMEM) { 8847 /* retry with a larger buffer */ 8848 buf_len = data_size; 8849 } else if (pf->hw.aq.asq_last_status != I40E_AQ_RC_OK) { 8850 dev_info(&pf->pdev->dev, 8851 "capability discovery failed, err %s aq_err %s\n", 8852 i40e_stat_str(&pf->hw, err), 8853 i40e_aq_str(&pf->hw, 8854 pf->hw.aq.asq_last_status)); 8855 return -ENODEV; 8856 } 8857 } while (err); 8858 8859 if (pf->hw.debug_mask & I40E_DEBUG_USER) { 8860 if (list_type == i40e_aqc_opc_list_func_capabilities) { 8861 dev_info(&pf->pdev->dev, 8862 "pf=%d, num_vfs=%d, msix_pf=%d, msix_vf=%d, fd_g=%d, fd_b=%d, pf_max_q=%d num_vsi=%d\n", 8863 pf->hw.pf_id, pf->hw.func_caps.num_vfs, 8864 pf->hw.func_caps.num_msix_vectors, 8865 pf->hw.func_caps.num_msix_vectors_vf, 8866 pf->hw.func_caps.fd_filters_guaranteed, 8867 pf->hw.func_caps.fd_filters_best_effort, 8868 pf->hw.func_caps.num_tx_qp, 8869 pf->hw.func_caps.num_vsis); 8870 } else if (list_type == i40e_aqc_opc_list_dev_capabilities) { 8871 dev_info(&pf->pdev->dev, 8872 "switch_mode=0x%04x, function_valid=0x%08x\n", 8873 pf->hw.dev_caps.switch_mode, 8874 pf->hw.dev_caps.valid_functions); 8875 dev_info(&pf->pdev->dev, 8876 "SR-IOV=%d, num_vfs for all function=%u\n", 8877 pf->hw.dev_caps.sr_iov_1_1, 8878 pf->hw.dev_caps.num_vfs); 8879 dev_info(&pf->pdev->dev, 8880 "num_vsis=%u, num_rx:%u, num_tx=%u\n", 8881 pf->hw.dev_caps.num_vsis, 8882 pf->hw.dev_caps.num_rx_qp, 8883 pf->hw.dev_caps.num_tx_qp); 8884 } 8885 } 8886 if (list_type == i40e_aqc_opc_list_func_capabilities) { 8887 #define DEF_NUM_VSI (1 + (pf->hw.func_caps.fcoe ? 1 : 0) \ 8888 + pf->hw.func_caps.num_vfs) 8889 if (pf->hw.revision_id == 0 && 8890 pf->hw.func_caps.num_vsis < DEF_NUM_VSI) { 8891 dev_info(&pf->pdev->dev, 8892 "got num_vsis %d, setting num_vsis to %d\n", 8893 pf->hw.func_caps.num_vsis, DEF_NUM_VSI); 8894 pf->hw.func_caps.num_vsis = DEF_NUM_VSI; 8895 } 8896 } 8897 return 0; 8898 } 8899 8900 static int i40e_vsi_clear(struct i40e_vsi *vsi); 8901 8902 /** 8903 * i40e_fdir_sb_setup - initialize the Flow Director resources for Sideband 8904 * @pf: board private structure 8905 **/ 8906 static void i40e_fdir_sb_setup(struct i40e_pf *pf) 8907 { 8908 struct i40e_vsi *vsi; 8909 8910 /* quick workaround for an NVM issue that leaves a critical register 8911 * uninitialized 8912 */ 8913 if (!rd32(&pf->hw, I40E_GLQF_HKEY(0))) { 8914 static const u32 hkey[] = { 8915 0xe640d33f, 0xcdfe98ab, 0x73fa7161, 0x0d7a7d36, 8916 0xeacb7d61, 0xaa4f05b6, 0x9c5c89ed, 0xfc425ddb, 8917 0xa4654832, 0xfc7461d4, 0x8f827619, 0xf5c63c21, 8918 0x95b3a76d}; 8919 int i; 8920 8921 for (i = 0; i <= I40E_GLQF_HKEY_MAX_INDEX; i++) 8922 wr32(&pf->hw, I40E_GLQF_HKEY(i), hkey[i]); 8923 } 8924 8925 if (!(pf->flags & I40E_FLAG_FD_SB_ENABLED)) 8926 return; 8927 8928 /* find existing VSI and see if it needs configuring */ 8929 vsi = i40e_find_vsi_by_type(pf, I40E_VSI_FDIR); 8930 8931 /* create a new VSI if none exists */ 8932 if (!vsi) { 8933 vsi = i40e_vsi_setup(pf, I40E_VSI_FDIR, 8934 pf->vsi[pf->lan_vsi]->seid, 0); 8935 if (!vsi) { 8936 dev_info(&pf->pdev->dev, "Couldn't create FDir VSI\n"); 8937 pf->flags &= ~I40E_FLAG_FD_SB_ENABLED; 8938 pf->flags |= I40E_FLAG_FD_SB_INACTIVE; 8939 return; 8940 } 8941 } 8942 8943 i40e_vsi_setup_irqhandler(vsi, i40e_fdir_clean_ring); 8944 } 8945 8946 /** 8947 * i40e_fdir_teardown - release the Flow Director resources 8948 * @pf: board private structure 8949 **/ 8950 static void i40e_fdir_teardown(struct i40e_pf *pf) 8951 { 8952 struct i40e_vsi *vsi; 8953 8954 i40e_fdir_filter_exit(pf); 8955 vsi = i40e_find_vsi_by_type(pf, I40E_VSI_FDIR); 8956 if (vsi) 8957 i40e_vsi_release(vsi); 8958 } 8959 8960 /** 8961 * i40e_rebuild_cloud_filters - Rebuilds cloud filters for VSIs 8962 * @vsi: PF main vsi 8963 * @seid: seid of main or channel VSIs 8964 * 8965 * Rebuilds cloud filters associated with main VSI and channel VSIs if they 8966 * existed before reset 8967 **/ 8968 static int i40e_rebuild_cloud_filters(struct i40e_vsi *vsi, u16 seid) 8969 { 8970 struct i40e_cloud_filter *cfilter; 8971 struct i40e_pf *pf = vsi->back; 8972 struct hlist_node *node; 8973 i40e_status ret; 8974 8975 /* Add cloud filters back if they exist */ 8976 hlist_for_each_entry_safe(cfilter, node, &pf->cloud_filter_list, 8977 cloud_node) { 8978 if (cfilter->seid != seid) 8979 continue; 8980 8981 if (cfilter->dst_port) 8982 ret = i40e_add_del_cloud_filter_big_buf(vsi, cfilter, 8983 true); 8984 else 8985 ret = i40e_add_del_cloud_filter(vsi, cfilter, true); 8986 8987 if (ret) { 8988 dev_dbg(&pf->pdev->dev, 8989 "Failed to rebuild cloud filter, err %s aq_err %s\n", 8990 i40e_stat_str(&pf->hw, ret), 8991 i40e_aq_str(&pf->hw, 8992 pf->hw.aq.asq_last_status)); 8993 return ret; 8994 } 8995 } 8996 return 0; 8997 } 8998 8999 /** 9000 * i40e_rebuild_channels - Rebuilds channel VSIs if they existed before reset 9001 * @vsi: PF main vsi 9002 * 9003 * Rebuilds channel VSIs if they existed before reset 9004 **/ 9005 static int i40e_rebuild_channels(struct i40e_vsi *vsi) 9006 { 9007 struct i40e_channel *ch, *ch_tmp; 9008 i40e_status ret; 9009 9010 if (list_empty(&vsi->ch_list)) 9011 return 0; 9012 9013 list_for_each_entry_safe(ch, ch_tmp, &vsi->ch_list, list) { 9014 if (!ch->initialized) 9015 break; 9016 /* Proceed with creation of channel (VMDq2) VSI */ 9017 ret = i40e_add_channel(vsi->back, vsi->uplink_seid, ch); 9018 if (ret) { 9019 dev_info(&vsi->back->pdev->dev, 9020 "failed to rebuild channels using uplink_seid %u\n", 9021 vsi->uplink_seid); 9022 return ret; 9023 } 9024 /* Reconfigure TX queues using QTX_CTL register */ 9025 ret = i40e_channel_config_tx_ring(vsi->back, vsi, ch); 9026 if (ret) { 9027 dev_info(&vsi->back->pdev->dev, 9028 "failed to configure TX rings for channel %u\n", 9029 ch->seid); 9030 return ret; 9031 } 9032 /* update 'next_base_queue' */ 9033 vsi->next_base_queue = vsi->next_base_queue + 9034 ch->num_queue_pairs; 9035 if (ch->max_tx_rate) { 9036 u64 credits = ch->max_tx_rate; 9037 9038 if (i40e_set_bw_limit(vsi, ch->seid, 9039 ch->max_tx_rate)) 9040 return -EINVAL; 9041 9042 do_div(credits, I40E_BW_CREDIT_DIVISOR); 9043 dev_dbg(&vsi->back->pdev->dev, 9044 "Set tx rate of %llu Mbps (count of 50Mbps %llu) for vsi->seid %u\n", 9045 ch->max_tx_rate, 9046 credits, 9047 ch->seid); 9048 } 9049 ret = i40e_rebuild_cloud_filters(vsi, ch->seid); 9050 if (ret) { 9051 dev_dbg(&vsi->back->pdev->dev, 9052 "Failed to rebuild cloud filters for channel VSI %u\n", 9053 ch->seid); 9054 return ret; 9055 } 9056 } 9057 return 0; 9058 } 9059 9060 /** 9061 * i40e_prep_for_reset - prep for the core to reset 9062 * @pf: board private structure 9063 * @lock_acquired: indicates whether or not the lock has been acquired 9064 * before this function was called. 9065 * 9066 * Close up the VFs and other things in prep for PF Reset. 9067 **/ 9068 static void i40e_prep_for_reset(struct i40e_pf *pf, bool lock_acquired) 9069 { 9070 struct i40e_hw *hw = &pf->hw; 9071 i40e_status ret = 0; 9072 u32 v; 9073 9074 clear_bit(__I40E_RESET_INTR_RECEIVED, pf->state); 9075 if (test_and_set_bit(__I40E_RESET_RECOVERY_PENDING, pf->state)) 9076 return; 9077 if (i40e_check_asq_alive(&pf->hw)) 9078 i40e_vc_notify_reset(pf); 9079 9080 dev_dbg(&pf->pdev->dev, "Tearing down internal switch for reset\n"); 9081 9082 /* quiesce the VSIs and their queues that are not already DOWN */ 9083 /* pf_quiesce_all_vsi modifies netdev structures -rtnl_lock needed */ 9084 if (!lock_acquired) 9085 rtnl_lock(); 9086 i40e_pf_quiesce_all_vsi(pf); 9087 if (!lock_acquired) 9088 rtnl_unlock(); 9089 9090 for (v = 0; v < pf->num_alloc_vsi; v++) { 9091 if (pf->vsi[v]) 9092 pf->vsi[v]->seid = 0; 9093 } 9094 9095 i40e_shutdown_adminq(&pf->hw); 9096 9097 /* call shutdown HMC */ 9098 if (hw->hmc.hmc_obj) { 9099 ret = i40e_shutdown_lan_hmc(hw); 9100 if (ret) 9101 dev_warn(&pf->pdev->dev, 9102 "shutdown_lan_hmc failed: %d\n", ret); 9103 } 9104 } 9105 9106 /** 9107 * i40e_send_version - update firmware with driver version 9108 * @pf: PF struct 9109 */ 9110 static void i40e_send_version(struct i40e_pf *pf) 9111 { 9112 struct i40e_driver_version dv; 9113 9114 dv.major_version = DRV_VERSION_MAJOR; 9115 dv.minor_version = DRV_VERSION_MINOR; 9116 dv.build_version = DRV_VERSION_BUILD; 9117 dv.subbuild_version = 0; 9118 strlcpy(dv.driver_string, DRV_VERSION, sizeof(dv.driver_string)); 9119 i40e_aq_send_driver_version(&pf->hw, &dv, NULL); 9120 } 9121 9122 /** 9123 * i40e_get_oem_version - get OEM specific version information 9124 * @hw: pointer to the hardware structure 9125 **/ 9126 static void i40e_get_oem_version(struct i40e_hw *hw) 9127 { 9128 u16 block_offset = 0xffff; 9129 u16 block_length = 0; 9130 u16 capabilities = 0; 9131 u16 gen_snap = 0; 9132 u16 release = 0; 9133 9134 #define I40E_SR_NVM_OEM_VERSION_PTR 0x1B 9135 #define I40E_NVM_OEM_LENGTH_OFFSET 0x00 9136 #define I40E_NVM_OEM_CAPABILITIES_OFFSET 0x01 9137 #define I40E_NVM_OEM_GEN_OFFSET 0x02 9138 #define I40E_NVM_OEM_RELEASE_OFFSET 0x03 9139 #define I40E_NVM_OEM_CAPABILITIES_MASK 0x000F 9140 #define I40E_NVM_OEM_LENGTH 3 9141 9142 /* Check if pointer to OEM version block is valid. */ 9143 i40e_read_nvm_word(hw, I40E_SR_NVM_OEM_VERSION_PTR, &block_offset); 9144 if (block_offset == 0xffff) 9145 return; 9146 9147 /* Check if OEM version block has correct length. */ 9148 i40e_read_nvm_word(hw, block_offset + I40E_NVM_OEM_LENGTH_OFFSET, 9149 &block_length); 9150 if (block_length < I40E_NVM_OEM_LENGTH) 9151 return; 9152 9153 /* Check if OEM version format is as expected. */ 9154 i40e_read_nvm_word(hw, block_offset + I40E_NVM_OEM_CAPABILITIES_OFFSET, 9155 &capabilities); 9156 if ((capabilities & I40E_NVM_OEM_CAPABILITIES_MASK) != 0) 9157 return; 9158 9159 i40e_read_nvm_word(hw, block_offset + I40E_NVM_OEM_GEN_OFFSET, 9160 &gen_snap); 9161 i40e_read_nvm_word(hw, block_offset + I40E_NVM_OEM_RELEASE_OFFSET, 9162 &release); 9163 hw->nvm.oem_ver = (gen_snap << I40E_OEM_SNAP_SHIFT) | release; 9164 hw->nvm.eetrack = I40E_OEM_EETRACK_ID; 9165 } 9166 9167 /** 9168 * i40e_reset - wait for core reset to finish reset, reset pf if corer not seen 9169 * @pf: board private structure 9170 **/ 9171 static int i40e_reset(struct i40e_pf *pf) 9172 { 9173 struct i40e_hw *hw = &pf->hw; 9174 i40e_status ret; 9175 9176 ret = i40e_pf_reset(hw); 9177 if (ret) { 9178 dev_info(&pf->pdev->dev, "PF reset failed, %d\n", ret); 9179 set_bit(__I40E_RESET_FAILED, pf->state); 9180 clear_bit(__I40E_RESET_RECOVERY_PENDING, pf->state); 9181 } else { 9182 pf->pfr_count++; 9183 } 9184 return ret; 9185 } 9186 9187 /** 9188 * i40e_rebuild - rebuild using a saved config 9189 * @pf: board private structure 9190 * @reinit: if the Main VSI needs to re-initialized. 9191 * @lock_acquired: indicates whether or not the lock has been acquired 9192 * before this function was called. 9193 **/ 9194 static void i40e_rebuild(struct i40e_pf *pf, bool reinit, bool lock_acquired) 9195 { 9196 struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi]; 9197 struct i40e_hw *hw = &pf->hw; 9198 u8 set_fc_aq_fail = 0; 9199 i40e_status ret; 9200 u32 val; 9201 int v; 9202 9203 if (test_bit(__I40E_DOWN, pf->state)) 9204 goto clear_recovery; 9205 dev_dbg(&pf->pdev->dev, "Rebuilding internal switch\n"); 9206 9207 /* rebuild the basics for the AdminQ, HMC, and initial HW switch */ 9208 ret = i40e_init_adminq(&pf->hw); 9209 if (ret) { 9210 dev_info(&pf->pdev->dev, "Rebuild AdminQ failed, err %s aq_err %s\n", 9211 i40e_stat_str(&pf->hw, ret), 9212 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 9213 goto clear_recovery; 9214 } 9215 i40e_get_oem_version(&pf->hw); 9216 9217 /* re-verify the eeprom if we just had an EMP reset */ 9218 if (test_and_clear_bit(__I40E_EMP_RESET_INTR_RECEIVED, pf->state)) 9219 i40e_verify_eeprom(pf); 9220 9221 i40e_clear_pxe_mode(hw); 9222 ret = i40e_get_capabilities(pf, i40e_aqc_opc_list_func_capabilities); 9223 if (ret) 9224 goto end_core_reset; 9225 9226 ret = i40e_init_lan_hmc(hw, hw->func_caps.num_tx_qp, 9227 hw->func_caps.num_rx_qp, 0, 0); 9228 if (ret) { 9229 dev_info(&pf->pdev->dev, "init_lan_hmc failed: %d\n", ret); 9230 goto end_core_reset; 9231 } 9232 ret = i40e_configure_lan_hmc(hw, I40E_HMC_MODEL_DIRECT_ONLY); 9233 if (ret) { 9234 dev_info(&pf->pdev->dev, "configure_lan_hmc failed: %d\n", ret); 9235 goto end_core_reset; 9236 } 9237 9238 /* Enable FW to write a default DCB config on link-up */ 9239 i40e_aq_set_dcb_parameters(hw, true, NULL); 9240 9241 #ifdef CONFIG_I40E_DCB 9242 ret = i40e_init_pf_dcb(pf); 9243 if (ret) { 9244 dev_info(&pf->pdev->dev, "DCB init failed %d, disabled\n", ret); 9245 pf->flags &= ~I40E_FLAG_DCB_CAPABLE; 9246 /* Continue without DCB enabled */ 9247 } 9248 #endif /* CONFIG_I40E_DCB */ 9249 /* do basic switch setup */ 9250 if (!lock_acquired) 9251 rtnl_lock(); 9252 ret = i40e_setup_pf_switch(pf, reinit); 9253 if (ret) 9254 goto end_unlock; 9255 9256 /* The driver only wants link up/down and module qualification 9257 * reports from firmware. Note the negative logic. 9258 */ 9259 ret = i40e_aq_set_phy_int_mask(&pf->hw, 9260 ~(I40E_AQ_EVENT_LINK_UPDOWN | 9261 I40E_AQ_EVENT_MEDIA_NA | 9262 I40E_AQ_EVENT_MODULE_QUAL_FAIL), NULL); 9263 if (ret) 9264 dev_info(&pf->pdev->dev, "set phy mask fail, err %s aq_err %s\n", 9265 i40e_stat_str(&pf->hw, ret), 9266 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 9267 9268 /* make sure our flow control settings are restored */ 9269 ret = i40e_set_fc(&pf->hw, &set_fc_aq_fail, true); 9270 if (ret) 9271 dev_dbg(&pf->pdev->dev, "setting flow control: ret = %s last_status = %s\n", 9272 i40e_stat_str(&pf->hw, ret), 9273 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 9274 9275 /* Rebuild the VSIs and VEBs that existed before reset. 9276 * They are still in our local switch element arrays, so only 9277 * need to rebuild the switch model in the HW. 9278 * 9279 * If there were VEBs but the reconstitution failed, we'll try 9280 * try to recover minimal use by getting the basic PF VSI working. 9281 */ 9282 if (vsi->uplink_seid != pf->mac_seid) { 9283 dev_dbg(&pf->pdev->dev, "attempting to rebuild switch\n"); 9284 /* find the one VEB connected to the MAC, and find orphans */ 9285 for (v = 0; v < I40E_MAX_VEB; v++) { 9286 if (!pf->veb[v]) 9287 continue; 9288 9289 if (pf->veb[v]->uplink_seid == pf->mac_seid || 9290 pf->veb[v]->uplink_seid == 0) { 9291 ret = i40e_reconstitute_veb(pf->veb[v]); 9292 9293 if (!ret) 9294 continue; 9295 9296 /* If Main VEB failed, we're in deep doodoo, 9297 * so give up rebuilding the switch and set up 9298 * for minimal rebuild of PF VSI. 9299 * If orphan failed, we'll report the error 9300 * but try to keep going. 9301 */ 9302 if (pf->veb[v]->uplink_seid == pf->mac_seid) { 9303 dev_info(&pf->pdev->dev, 9304 "rebuild of switch failed: %d, will try to set up simple PF connection\n", 9305 ret); 9306 vsi->uplink_seid = pf->mac_seid; 9307 break; 9308 } else if (pf->veb[v]->uplink_seid == 0) { 9309 dev_info(&pf->pdev->dev, 9310 "rebuild of orphan VEB failed: %d\n", 9311 ret); 9312 } 9313 } 9314 } 9315 } 9316 9317 if (vsi->uplink_seid == pf->mac_seid) { 9318 dev_dbg(&pf->pdev->dev, "attempting to rebuild PF VSI\n"); 9319 /* no VEB, so rebuild only the Main VSI */ 9320 ret = i40e_add_vsi(vsi); 9321 if (ret) { 9322 dev_info(&pf->pdev->dev, 9323 "rebuild of Main VSI failed: %d\n", ret); 9324 goto end_unlock; 9325 } 9326 } 9327 9328 if (vsi->mqprio_qopt.max_rate[0]) { 9329 u64 max_tx_rate = vsi->mqprio_qopt.max_rate[0]; 9330 u64 credits = 0; 9331 9332 do_div(max_tx_rate, I40E_BW_MBPS_DIVISOR); 9333 ret = i40e_set_bw_limit(vsi, vsi->seid, max_tx_rate); 9334 if (ret) 9335 goto end_unlock; 9336 9337 credits = max_tx_rate; 9338 do_div(credits, I40E_BW_CREDIT_DIVISOR); 9339 dev_dbg(&vsi->back->pdev->dev, 9340 "Set tx rate of %llu Mbps (count of 50Mbps %llu) for vsi->seid %u\n", 9341 max_tx_rate, 9342 credits, 9343 vsi->seid); 9344 } 9345 9346 ret = i40e_rebuild_cloud_filters(vsi, vsi->seid); 9347 if (ret) 9348 goto end_unlock; 9349 9350 /* PF Main VSI is rebuild by now, go ahead and rebuild channel VSIs 9351 * for this main VSI if they exist 9352 */ 9353 ret = i40e_rebuild_channels(vsi); 9354 if (ret) 9355 goto end_unlock; 9356 9357 /* Reconfigure hardware for allowing smaller MSS in the case 9358 * of TSO, so that we avoid the MDD being fired and causing 9359 * a reset in the case of small MSS+TSO. 9360 */ 9361 #define I40E_REG_MSS 0x000E64DC 9362 #define I40E_REG_MSS_MIN_MASK 0x3FF0000 9363 #define I40E_64BYTE_MSS 0x400000 9364 val = rd32(hw, I40E_REG_MSS); 9365 if ((val & I40E_REG_MSS_MIN_MASK) > I40E_64BYTE_MSS) { 9366 val &= ~I40E_REG_MSS_MIN_MASK; 9367 val |= I40E_64BYTE_MSS; 9368 wr32(hw, I40E_REG_MSS, val); 9369 } 9370 9371 if (pf->hw_features & I40E_HW_RESTART_AUTONEG) { 9372 msleep(75); 9373 ret = i40e_aq_set_link_restart_an(&pf->hw, true, NULL); 9374 if (ret) 9375 dev_info(&pf->pdev->dev, "link restart failed, err %s aq_err %s\n", 9376 i40e_stat_str(&pf->hw, ret), 9377 i40e_aq_str(&pf->hw, 9378 pf->hw.aq.asq_last_status)); 9379 } 9380 /* reinit the misc interrupt */ 9381 if (pf->flags & I40E_FLAG_MSIX_ENABLED) 9382 ret = i40e_setup_misc_vector(pf); 9383 9384 /* Add a filter to drop all Flow control frames from any VSI from being 9385 * transmitted. By doing so we stop a malicious VF from sending out 9386 * PAUSE or PFC frames and potentially controlling traffic for other 9387 * PF/VF VSIs. 9388 * The FW can still send Flow control frames if enabled. 9389 */ 9390 i40e_add_filter_to_drop_tx_flow_control_frames(&pf->hw, 9391 pf->main_vsi_seid); 9392 9393 /* restart the VSIs that were rebuilt and running before the reset */ 9394 i40e_pf_unquiesce_all_vsi(pf); 9395 9396 /* Release the RTNL lock before we start resetting VFs */ 9397 if (!lock_acquired) 9398 rtnl_unlock(); 9399 9400 /* Restore promiscuous settings */ 9401 ret = i40e_set_promiscuous(pf, pf->cur_promisc); 9402 if (ret) 9403 dev_warn(&pf->pdev->dev, 9404 "Failed to restore promiscuous setting: %s, err %s aq_err %s\n", 9405 pf->cur_promisc ? "on" : "off", 9406 i40e_stat_str(&pf->hw, ret), 9407 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 9408 9409 i40e_reset_all_vfs(pf, true); 9410 9411 /* tell the firmware that we're starting */ 9412 i40e_send_version(pf); 9413 9414 /* We've already released the lock, so don't do it again */ 9415 goto end_core_reset; 9416 9417 end_unlock: 9418 if (!lock_acquired) 9419 rtnl_unlock(); 9420 end_core_reset: 9421 clear_bit(__I40E_RESET_FAILED, pf->state); 9422 clear_recovery: 9423 clear_bit(__I40E_RESET_RECOVERY_PENDING, pf->state); 9424 } 9425 9426 /** 9427 * i40e_reset_and_rebuild - reset and rebuild using a saved config 9428 * @pf: board private structure 9429 * @reinit: if the Main VSI needs to re-initialized. 9430 * @lock_acquired: indicates whether or not the lock has been acquired 9431 * before this function was called. 9432 **/ 9433 static void i40e_reset_and_rebuild(struct i40e_pf *pf, bool reinit, 9434 bool lock_acquired) 9435 { 9436 int ret; 9437 /* Now we wait for GRST to settle out. 9438 * We don't have to delete the VEBs or VSIs from the hw switch 9439 * because the reset will make them disappear. 9440 */ 9441 ret = i40e_reset(pf); 9442 if (!ret) 9443 i40e_rebuild(pf, reinit, lock_acquired); 9444 } 9445 9446 /** 9447 * i40e_handle_reset_warning - prep for the PF to reset, reset and rebuild 9448 * @pf: board private structure 9449 * 9450 * Close up the VFs and other things in prep for a Core Reset, 9451 * then get ready to rebuild the world. 9452 * @lock_acquired: indicates whether or not the lock has been acquired 9453 * before this function was called. 9454 **/ 9455 static void i40e_handle_reset_warning(struct i40e_pf *pf, bool lock_acquired) 9456 { 9457 i40e_prep_for_reset(pf, lock_acquired); 9458 i40e_reset_and_rebuild(pf, false, lock_acquired); 9459 } 9460 9461 /** 9462 * i40e_handle_mdd_event 9463 * @pf: pointer to the PF structure 9464 * 9465 * Called from the MDD irq handler to identify possibly malicious vfs 9466 **/ 9467 static void i40e_handle_mdd_event(struct i40e_pf *pf) 9468 { 9469 struct i40e_hw *hw = &pf->hw; 9470 bool mdd_detected = false; 9471 bool pf_mdd_detected = false; 9472 struct i40e_vf *vf; 9473 u32 reg; 9474 int i; 9475 9476 if (!test_bit(__I40E_MDD_EVENT_PENDING, pf->state)) 9477 return; 9478 9479 /* find what triggered the MDD event */ 9480 reg = rd32(hw, I40E_GL_MDET_TX); 9481 if (reg & I40E_GL_MDET_TX_VALID_MASK) { 9482 u8 pf_num = (reg & I40E_GL_MDET_TX_PF_NUM_MASK) >> 9483 I40E_GL_MDET_TX_PF_NUM_SHIFT; 9484 u16 vf_num = (reg & I40E_GL_MDET_TX_VF_NUM_MASK) >> 9485 I40E_GL_MDET_TX_VF_NUM_SHIFT; 9486 u8 event = (reg & I40E_GL_MDET_TX_EVENT_MASK) >> 9487 I40E_GL_MDET_TX_EVENT_SHIFT; 9488 u16 queue = ((reg & I40E_GL_MDET_TX_QUEUE_MASK) >> 9489 I40E_GL_MDET_TX_QUEUE_SHIFT) - 9490 pf->hw.func_caps.base_queue; 9491 if (netif_msg_tx_err(pf)) 9492 dev_info(&pf->pdev->dev, "Malicious Driver Detection event 0x%02x on TX queue %d PF number 0x%02x VF number 0x%02x\n", 9493 event, queue, pf_num, vf_num); 9494 wr32(hw, I40E_GL_MDET_TX, 0xffffffff); 9495 mdd_detected = true; 9496 } 9497 reg = rd32(hw, I40E_GL_MDET_RX); 9498 if (reg & I40E_GL_MDET_RX_VALID_MASK) { 9499 u8 func = (reg & I40E_GL_MDET_RX_FUNCTION_MASK) >> 9500 I40E_GL_MDET_RX_FUNCTION_SHIFT; 9501 u8 event = (reg & I40E_GL_MDET_RX_EVENT_MASK) >> 9502 I40E_GL_MDET_RX_EVENT_SHIFT; 9503 u16 queue = ((reg & I40E_GL_MDET_RX_QUEUE_MASK) >> 9504 I40E_GL_MDET_RX_QUEUE_SHIFT) - 9505 pf->hw.func_caps.base_queue; 9506 if (netif_msg_rx_err(pf)) 9507 dev_info(&pf->pdev->dev, "Malicious Driver Detection event 0x%02x on RX queue %d of function 0x%02x\n", 9508 event, queue, func); 9509 wr32(hw, I40E_GL_MDET_RX, 0xffffffff); 9510 mdd_detected = true; 9511 } 9512 9513 if (mdd_detected) { 9514 reg = rd32(hw, I40E_PF_MDET_TX); 9515 if (reg & I40E_PF_MDET_TX_VALID_MASK) { 9516 wr32(hw, I40E_PF_MDET_TX, 0xFFFF); 9517 dev_info(&pf->pdev->dev, "TX driver issue detected, PF reset issued\n"); 9518 pf_mdd_detected = true; 9519 } 9520 reg = rd32(hw, I40E_PF_MDET_RX); 9521 if (reg & I40E_PF_MDET_RX_VALID_MASK) { 9522 wr32(hw, I40E_PF_MDET_RX, 0xFFFF); 9523 dev_info(&pf->pdev->dev, "RX driver issue detected, PF reset issued\n"); 9524 pf_mdd_detected = true; 9525 } 9526 /* Queue belongs to the PF, initiate a reset */ 9527 if (pf_mdd_detected) { 9528 set_bit(__I40E_PF_RESET_REQUESTED, pf->state); 9529 i40e_service_event_schedule(pf); 9530 } 9531 } 9532 9533 /* see if one of the VFs needs its hand slapped */ 9534 for (i = 0; i < pf->num_alloc_vfs && mdd_detected; i++) { 9535 vf = &(pf->vf[i]); 9536 reg = rd32(hw, I40E_VP_MDET_TX(i)); 9537 if (reg & I40E_VP_MDET_TX_VALID_MASK) { 9538 wr32(hw, I40E_VP_MDET_TX(i), 0xFFFF); 9539 vf->num_mdd_events++; 9540 dev_info(&pf->pdev->dev, "TX driver issue detected on VF %d\n", 9541 i); 9542 } 9543 9544 reg = rd32(hw, I40E_VP_MDET_RX(i)); 9545 if (reg & I40E_VP_MDET_RX_VALID_MASK) { 9546 wr32(hw, I40E_VP_MDET_RX(i), 0xFFFF); 9547 vf->num_mdd_events++; 9548 dev_info(&pf->pdev->dev, "RX driver issue detected on VF %d\n", 9549 i); 9550 } 9551 9552 if (vf->num_mdd_events > I40E_DEFAULT_NUM_MDD_EVENTS_ALLOWED) { 9553 dev_info(&pf->pdev->dev, 9554 "Too many MDD events on VF %d, disabled\n", i); 9555 dev_info(&pf->pdev->dev, 9556 "Use PF Control I/F to re-enable the VF\n"); 9557 set_bit(I40E_VF_STATE_DISABLED, &vf->vf_states); 9558 } 9559 } 9560 9561 /* re-enable mdd interrupt cause */ 9562 clear_bit(__I40E_MDD_EVENT_PENDING, pf->state); 9563 reg = rd32(hw, I40E_PFINT_ICR0_ENA); 9564 reg |= I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK; 9565 wr32(hw, I40E_PFINT_ICR0_ENA, reg); 9566 i40e_flush(hw); 9567 } 9568 9569 static const char *i40e_tunnel_name(struct i40e_udp_port_config *port) 9570 { 9571 switch (port->type) { 9572 case UDP_TUNNEL_TYPE_VXLAN: 9573 return "vxlan"; 9574 case UDP_TUNNEL_TYPE_GENEVE: 9575 return "geneve"; 9576 default: 9577 return "unknown"; 9578 } 9579 } 9580 9581 /** 9582 * i40e_sync_udp_filters - Trigger a sync event for existing UDP filters 9583 * @pf: board private structure 9584 **/ 9585 static void i40e_sync_udp_filters(struct i40e_pf *pf) 9586 { 9587 int i; 9588 9589 /* loop through and set pending bit for all active UDP filters */ 9590 for (i = 0; i < I40E_MAX_PF_UDP_OFFLOAD_PORTS; i++) { 9591 if (pf->udp_ports[i].port) 9592 pf->pending_udp_bitmap |= BIT_ULL(i); 9593 } 9594 9595 pf->flags |= I40E_FLAG_UDP_FILTER_SYNC; 9596 } 9597 9598 /** 9599 * i40e_sync_udp_filters_subtask - Sync the VSI filter list with HW 9600 * @pf: board private structure 9601 **/ 9602 static void i40e_sync_udp_filters_subtask(struct i40e_pf *pf) 9603 { 9604 struct i40e_hw *hw = &pf->hw; 9605 i40e_status ret; 9606 u16 port; 9607 int i; 9608 9609 if (!(pf->flags & I40E_FLAG_UDP_FILTER_SYNC)) 9610 return; 9611 9612 pf->flags &= ~I40E_FLAG_UDP_FILTER_SYNC; 9613 9614 for (i = 0; i < I40E_MAX_PF_UDP_OFFLOAD_PORTS; i++) { 9615 if (pf->pending_udp_bitmap & BIT_ULL(i)) { 9616 pf->pending_udp_bitmap &= ~BIT_ULL(i); 9617 port = pf->udp_ports[i].port; 9618 if (port) 9619 ret = i40e_aq_add_udp_tunnel(hw, port, 9620 pf->udp_ports[i].type, 9621 NULL, NULL); 9622 else 9623 ret = i40e_aq_del_udp_tunnel(hw, i, NULL); 9624 9625 if (ret) { 9626 dev_info(&pf->pdev->dev, 9627 "%s %s port %d, index %d failed, err %s aq_err %s\n", 9628 i40e_tunnel_name(&pf->udp_ports[i]), 9629 port ? "add" : "delete", 9630 port, i, 9631 i40e_stat_str(&pf->hw, ret), 9632 i40e_aq_str(&pf->hw, 9633 pf->hw.aq.asq_last_status)); 9634 pf->udp_ports[i].port = 0; 9635 } 9636 } 9637 } 9638 } 9639 9640 /** 9641 * i40e_service_task - Run the driver's async subtasks 9642 * @work: pointer to work_struct containing our data 9643 **/ 9644 static void i40e_service_task(struct work_struct *work) 9645 { 9646 struct i40e_pf *pf = container_of(work, 9647 struct i40e_pf, 9648 service_task); 9649 unsigned long start_time = jiffies; 9650 9651 /* don't bother with service tasks if a reset is in progress */ 9652 if (test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state)) 9653 return; 9654 9655 if (test_and_set_bit(__I40E_SERVICE_SCHED, pf->state)) 9656 return; 9657 9658 i40e_detect_recover_hung(pf->vsi[pf->lan_vsi]); 9659 i40e_sync_filters_subtask(pf); 9660 i40e_reset_subtask(pf); 9661 i40e_handle_mdd_event(pf); 9662 i40e_vc_process_vflr_event(pf); 9663 i40e_watchdog_subtask(pf); 9664 i40e_fdir_reinit_subtask(pf); 9665 if (pf->flags & I40E_FLAG_CLIENT_RESET) { 9666 /* Client subtask will reopen next time through. */ 9667 i40e_notify_client_of_netdev_close(pf->vsi[pf->lan_vsi], true); 9668 pf->flags &= ~I40E_FLAG_CLIENT_RESET; 9669 } else { 9670 i40e_client_subtask(pf); 9671 if (pf->flags & I40E_FLAG_CLIENT_L2_CHANGE) { 9672 i40e_notify_client_of_l2_param_changes( 9673 pf->vsi[pf->lan_vsi]); 9674 pf->flags &= ~I40E_FLAG_CLIENT_L2_CHANGE; 9675 } 9676 } 9677 i40e_sync_filters_subtask(pf); 9678 i40e_sync_udp_filters_subtask(pf); 9679 i40e_clean_adminq_subtask(pf); 9680 9681 /* flush memory to make sure state is correct before next watchdog */ 9682 smp_mb__before_atomic(); 9683 clear_bit(__I40E_SERVICE_SCHED, pf->state); 9684 9685 /* If the tasks have taken longer than one timer cycle or there 9686 * is more work to be done, reschedule the service task now 9687 * rather than wait for the timer to tick again. 9688 */ 9689 if (time_after(jiffies, (start_time + pf->service_timer_period)) || 9690 test_bit(__I40E_ADMINQ_EVENT_PENDING, pf->state) || 9691 test_bit(__I40E_MDD_EVENT_PENDING, pf->state) || 9692 test_bit(__I40E_VFLR_EVENT_PENDING, pf->state)) 9693 i40e_service_event_schedule(pf); 9694 } 9695 9696 /** 9697 * i40e_service_timer - timer callback 9698 * @data: pointer to PF struct 9699 **/ 9700 static void i40e_service_timer(struct timer_list *t) 9701 { 9702 struct i40e_pf *pf = from_timer(pf, t, service_timer); 9703 9704 mod_timer(&pf->service_timer, 9705 round_jiffies(jiffies + pf->service_timer_period)); 9706 i40e_service_event_schedule(pf); 9707 } 9708 9709 /** 9710 * i40e_set_num_rings_in_vsi - Determine number of rings in the VSI 9711 * @vsi: the VSI being configured 9712 **/ 9713 static int i40e_set_num_rings_in_vsi(struct i40e_vsi *vsi) 9714 { 9715 struct i40e_pf *pf = vsi->back; 9716 9717 switch (vsi->type) { 9718 case I40E_VSI_MAIN: 9719 vsi->alloc_queue_pairs = pf->num_lan_qps; 9720 vsi->num_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS, 9721 I40E_REQ_DESCRIPTOR_MULTIPLE); 9722 if (pf->flags & I40E_FLAG_MSIX_ENABLED) 9723 vsi->num_q_vectors = pf->num_lan_msix; 9724 else 9725 vsi->num_q_vectors = 1; 9726 9727 break; 9728 9729 case I40E_VSI_FDIR: 9730 vsi->alloc_queue_pairs = 1; 9731 vsi->num_desc = ALIGN(I40E_FDIR_RING_COUNT, 9732 I40E_REQ_DESCRIPTOR_MULTIPLE); 9733 vsi->num_q_vectors = pf->num_fdsb_msix; 9734 break; 9735 9736 case I40E_VSI_VMDQ2: 9737 vsi->alloc_queue_pairs = pf->num_vmdq_qps; 9738 vsi->num_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS, 9739 I40E_REQ_DESCRIPTOR_MULTIPLE); 9740 vsi->num_q_vectors = pf->num_vmdq_msix; 9741 break; 9742 9743 case I40E_VSI_SRIOV: 9744 vsi->alloc_queue_pairs = pf->num_vf_qps; 9745 vsi->num_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS, 9746 I40E_REQ_DESCRIPTOR_MULTIPLE); 9747 break; 9748 9749 default: 9750 WARN_ON(1); 9751 return -ENODATA; 9752 } 9753 9754 return 0; 9755 } 9756 9757 /** 9758 * i40e_vsi_alloc_arrays - Allocate queue and vector pointer arrays for the vsi 9759 * @vsi: VSI pointer 9760 * @alloc_qvectors: a bool to specify if q_vectors need to be allocated. 9761 * 9762 * On error: returns error code (negative) 9763 * On success: returns 0 9764 **/ 9765 static int i40e_vsi_alloc_arrays(struct i40e_vsi *vsi, bool alloc_qvectors) 9766 { 9767 struct i40e_ring **next_rings; 9768 int size; 9769 int ret = 0; 9770 9771 /* allocate memory for both Tx, XDP Tx and Rx ring pointers */ 9772 size = sizeof(struct i40e_ring *) * vsi->alloc_queue_pairs * 9773 (i40e_enabled_xdp_vsi(vsi) ? 3 : 2); 9774 vsi->tx_rings = kzalloc(size, GFP_KERNEL); 9775 if (!vsi->tx_rings) 9776 return -ENOMEM; 9777 next_rings = vsi->tx_rings + vsi->alloc_queue_pairs; 9778 if (i40e_enabled_xdp_vsi(vsi)) { 9779 vsi->xdp_rings = next_rings; 9780 next_rings += vsi->alloc_queue_pairs; 9781 } 9782 vsi->rx_rings = next_rings; 9783 9784 if (alloc_qvectors) { 9785 /* allocate memory for q_vector pointers */ 9786 size = sizeof(struct i40e_q_vector *) * vsi->num_q_vectors; 9787 vsi->q_vectors = kzalloc(size, GFP_KERNEL); 9788 if (!vsi->q_vectors) { 9789 ret = -ENOMEM; 9790 goto err_vectors; 9791 } 9792 } 9793 return ret; 9794 9795 err_vectors: 9796 kfree(vsi->tx_rings); 9797 return ret; 9798 } 9799 9800 /** 9801 * i40e_vsi_mem_alloc - Allocates the next available struct vsi in the PF 9802 * @pf: board private structure 9803 * @type: type of VSI 9804 * 9805 * On error: returns error code (negative) 9806 * On success: returns vsi index in PF (positive) 9807 **/ 9808 static int i40e_vsi_mem_alloc(struct i40e_pf *pf, enum i40e_vsi_type type) 9809 { 9810 int ret = -ENODEV; 9811 struct i40e_vsi *vsi; 9812 int vsi_idx; 9813 int i; 9814 9815 /* Need to protect the allocation of the VSIs at the PF level */ 9816 mutex_lock(&pf->switch_mutex); 9817 9818 /* VSI list may be fragmented if VSI creation/destruction has 9819 * been happening. We can afford to do a quick scan to look 9820 * for any free VSIs in the list. 9821 * 9822 * find next empty vsi slot, looping back around if necessary 9823 */ 9824 i = pf->next_vsi; 9825 while (i < pf->num_alloc_vsi && pf->vsi[i]) 9826 i++; 9827 if (i >= pf->num_alloc_vsi) { 9828 i = 0; 9829 while (i < pf->next_vsi && pf->vsi[i]) 9830 i++; 9831 } 9832 9833 if (i < pf->num_alloc_vsi && !pf->vsi[i]) { 9834 vsi_idx = i; /* Found one! */ 9835 } else { 9836 ret = -ENODEV; 9837 goto unlock_pf; /* out of VSI slots! */ 9838 } 9839 pf->next_vsi = ++i; 9840 9841 vsi = kzalloc(sizeof(*vsi), GFP_KERNEL); 9842 if (!vsi) { 9843 ret = -ENOMEM; 9844 goto unlock_pf; 9845 } 9846 vsi->type = type; 9847 vsi->back = pf; 9848 set_bit(__I40E_VSI_DOWN, vsi->state); 9849 vsi->flags = 0; 9850 vsi->idx = vsi_idx; 9851 vsi->int_rate_limit = 0; 9852 vsi->rss_table_size = (vsi->type == I40E_VSI_MAIN) ? 9853 pf->rss_table_size : 64; 9854 vsi->netdev_registered = false; 9855 vsi->work_limit = I40E_DEFAULT_IRQ_WORK; 9856 hash_init(vsi->mac_filter_hash); 9857 vsi->irqs_ready = false; 9858 9859 ret = i40e_set_num_rings_in_vsi(vsi); 9860 if (ret) 9861 goto err_rings; 9862 9863 ret = i40e_vsi_alloc_arrays(vsi, true); 9864 if (ret) 9865 goto err_rings; 9866 9867 /* Setup default MSIX irq handler for VSI */ 9868 i40e_vsi_setup_irqhandler(vsi, i40e_msix_clean_rings); 9869 9870 /* Initialize VSI lock */ 9871 spin_lock_init(&vsi->mac_filter_hash_lock); 9872 pf->vsi[vsi_idx] = vsi; 9873 ret = vsi_idx; 9874 goto unlock_pf; 9875 9876 err_rings: 9877 pf->next_vsi = i - 1; 9878 kfree(vsi); 9879 unlock_pf: 9880 mutex_unlock(&pf->switch_mutex); 9881 return ret; 9882 } 9883 9884 /** 9885 * i40e_vsi_free_arrays - Free queue and vector pointer arrays for the VSI 9886 * @type: VSI pointer 9887 * @free_qvectors: a bool to specify if q_vectors need to be freed. 9888 * 9889 * On error: returns error code (negative) 9890 * On success: returns 0 9891 **/ 9892 static void i40e_vsi_free_arrays(struct i40e_vsi *vsi, bool free_qvectors) 9893 { 9894 /* free the ring and vector containers */ 9895 if (free_qvectors) { 9896 kfree(vsi->q_vectors); 9897 vsi->q_vectors = NULL; 9898 } 9899 kfree(vsi->tx_rings); 9900 vsi->tx_rings = NULL; 9901 vsi->rx_rings = NULL; 9902 vsi->xdp_rings = NULL; 9903 } 9904 9905 /** 9906 * i40e_clear_rss_config_user - clear the user configured RSS hash keys 9907 * and lookup table 9908 * @vsi: Pointer to VSI structure 9909 */ 9910 static void i40e_clear_rss_config_user(struct i40e_vsi *vsi) 9911 { 9912 if (!vsi) 9913 return; 9914 9915 kfree(vsi->rss_hkey_user); 9916 vsi->rss_hkey_user = NULL; 9917 9918 kfree(vsi->rss_lut_user); 9919 vsi->rss_lut_user = NULL; 9920 } 9921 9922 /** 9923 * i40e_vsi_clear - Deallocate the VSI provided 9924 * @vsi: the VSI being un-configured 9925 **/ 9926 static int i40e_vsi_clear(struct i40e_vsi *vsi) 9927 { 9928 struct i40e_pf *pf; 9929 9930 if (!vsi) 9931 return 0; 9932 9933 if (!vsi->back) 9934 goto free_vsi; 9935 pf = vsi->back; 9936 9937 mutex_lock(&pf->switch_mutex); 9938 if (!pf->vsi[vsi->idx]) { 9939 dev_err(&pf->pdev->dev, "pf->vsi[%d] is NULL, just free vsi[%d](%p,type %d)\n", 9940 vsi->idx, vsi->idx, vsi, vsi->type); 9941 goto unlock_vsi; 9942 } 9943 9944 if (pf->vsi[vsi->idx] != vsi) { 9945 dev_err(&pf->pdev->dev, 9946 "pf->vsi[%d](%p, type %d) != vsi[%d](%p,type %d): no free!\n", 9947 pf->vsi[vsi->idx]->idx, 9948 pf->vsi[vsi->idx], 9949 pf->vsi[vsi->idx]->type, 9950 vsi->idx, vsi, vsi->type); 9951 goto unlock_vsi; 9952 } 9953 9954 /* updates the PF for this cleared vsi */ 9955 i40e_put_lump(pf->qp_pile, vsi->base_queue, vsi->idx); 9956 i40e_put_lump(pf->irq_pile, vsi->base_vector, vsi->idx); 9957 9958 i40e_vsi_free_arrays(vsi, true); 9959 i40e_clear_rss_config_user(vsi); 9960 9961 pf->vsi[vsi->idx] = NULL; 9962 if (vsi->idx < pf->next_vsi) 9963 pf->next_vsi = vsi->idx; 9964 9965 unlock_vsi: 9966 mutex_unlock(&pf->switch_mutex); 9967 free_vsi: 9968 kfree(vsi); 9969 9970 return 0; 9971 } 9972 9973 /** 9974 * i40e_vsi_clear_rings - Deallocates the Rx and Tx rings for the provided VSI 9975 * @vsi: the VSI being cleaned 9976 **/ 9977 static void i40e_vsi_clear_rings(struct i40e_vsi *vsi) 9978 { 9979 int i; 9980 9981 if (vsi->tx_rings && vsi->tx_rings[0]) { 9982 for (i = 0; i < vsi->alloc_queue_pairs; i++) { 9983 kfree_rcu(vsi->tx_rings[i], rcu); 9984 vsi->tx_rings[i] = NULL; 9985 vsi->rx_rings[i] = NULL; 9986 if (vsi->xdp_rings) 9987 vsi->xdp_rings[i] = NULL; 9988 } 9989 } 9990 } 9991 9992 /** 9993 * i40e_alloc_rings - Allocates the Rx and Tx rings for the provided VSI 9994 * @vsi: the VSI being configured 9995 **/ 9996 static int i40e_alloc_rings(struct i40e_vsi *vsi) 9997 { 9998 int i, qpv = i40e_enabled_xdp_vsi(vsi) ? 3 : 2; 9999 struct i40e_pf *pf = vsi->back; 10000 struct i40e_ring *ring; 10001 10002 /* Set basic values in the rings to be used later during open() */ 10003 for (i = 0; i < vsi->alloc_queue_pairs; i++) { 10004 /* allocate space for both Tx and Rx in one shot */ 10005 ring = kcalloc(qpv, sizeof(struct i40e_ring), GFP_KERNEL); 10006 if (!ring) 10007 goto err_out; 10008 10009 ring->queue_index = i; 10010 ring->reg_idx = vsi->base_queue + i; 10011 ring->ring_active = false; 10012 ring->vsi = vsi; 10013 ring->netdev = vsi->netdev; 10014 ring->dev = &pf->pdev->dev; 10015 ring->count = vsi->num_desc; 10016 ring->size = 0; 10017 ring->dcb_tc = 0; 10018 if (vsi->back->hw_features & I40E_HW_WB_ON_ITR_CAPABLE) 10019 ring->flags = I40E_TXR_FLAGS_WB_ON_ITR; 10020 ring->tx_itr_setting = pf->tx_itr_default; 10021 vsi->tx_rings[i] = ring++; 10022 10023 if (!i40e_enabled_xdp_vsi(vsi)) 10024 goto setup_rx; 10025 10026 ring->queue_index = vsi->alloc_queue_pairs + i; 10027 ring->reg_idx = vsi->base_queue + ring->queue_index; 10028 ring->ring_active = false; 10029 ring->vsi = vsi; 10030 ring->netdev = NULL; 10031 ring->dev = &pf->pdev->dev; 10032 ring->count = vsi->num_desc; 10033 ring->size = 0; 10034 ring->dcb_tc = 0; 10035 if (vsi->back->hw_features & I40E_HW_WB_ON_ITR_CAPABLE) 10036 ring->flags = I40E_TXR_FLAGS_WB_ON_ITR; 10037 set_ring_xdp(ring); 10038 ring->tx_itr_setting = pf->tx_itr_default; 10039 vsi->xdp_rings[i] = ring++; 10040 10041 setup_rx: 10042 ring->queue_index = i; 10043 ring->reg_idx = vsi->base_queue + i; 10044 ring->ring_active = false; 10045 ring->vsi = vsi; 10046 ring->netdev = vsi->netdev; 10047 ring->dev = &pf->pdev->dev; 10048 ring->count = vsi->num_desc; 10049 ring->size = 0; 10050 ring->dcb_tc = 0; 10051 ring->rx_itr_setting = pf->rx_itr_default; 10052 vsi->rx_rings[i] = ring; 10053 } 10054 10055 return 0; 10056 10057 err_out: 10058 i40e_vsi_clear_rings(vsi); 10059 return -ENOMEM; 10060 } 10061 10062 /** 10063 * i40e_reserve_msix_vectors - Reserve MSI-X vectors in the kernel 10064 * @pf: board private structure 10065 * @vectors: the number of MSI-X vectors to request 10066 * 10067 * Returns the number of vectors reserved, or error 10068 **/ 10069 static int i40e_reserve_msix_vectors(struct i40e_pf *pf, int vectors) 10070 { 10071 vectors = pci_enable_msix_range(pf->pdev, pf->msix_entries, 10072 I40E_MIN_MSIX, vectors); 10073 if (vectors < 0) { 10074 dev_info(&pf->pdev->dev, 10075 "MSI-X vector reservation failed: %d\n", vectors); 10076 vectors = 0; 10077 } 10078 10079 return vectors; 10080 } 10081 10082 /** 10083 * i40e_init_msix - Setup the MSIX capability 10084 * @pf: board private structure 10085 * 10086 * Work with the OS to set up the MSIX vectors needed. 10087 * 10088 * Returns the number of vectors reserved or negative on failure 10089 **/ 10090 static int i40e_init_msix(struct i40e_pf *pf) 10091 { 10092 struct i40e_hw *hw = &pf->hw; 10093 int cpus, extra_vectors; 10094 int vectors_left; 10095 int v_budget, i; 10096 int v_actual; 10097 int iwarp_requested = 0; 10098 10099 if (!(pf->flags & I40E_FLAG_MSIX_ENABLED)) 10100 return -ENODEV; 10101 10102 /* The number of vectors we'll request will be comprised of: 10103 * - Add 1 for "other" cause for Admin Queue events, etc. 10104 * - The number of LAN queue pairs 10105 * - Queues being used for RSS. 10106 * We don't need as many as max_rss_size vectors. 10107 * use rss_size instead in the calculation since that 10108 * is governed by number of cpus in the system. 10109 * - assumes symmetric Tx/Rx pairing 10110 * - The number of VMDq pairs 10111 * - The CPU count within the NUMA node if iWARP is enabled 10112 * Once we count this up, try the request. 10113 * 10114 * If we can't get what we want, we'll simplify to nearly nothing 10115 * and try again. If that still fails, we punt. 10116 */ 10117 vectors_left = hw->func_caps.num_msix_vectors; 10118 v_budget = 0; 10119 10120 /* reserve one vector for miscellaneous handler */ 10121 if (vectors_left) { 10122 v_budget++; 10123 vectors_left--; 10124 } 10125 10126 /* reserve some vectors for the main PF traffic queues. Initially we 10127 * only reserve at most 50% of the available vectors, in the case that 10128 * the number of online CPUs is large. This ensures that we can enable 10129 * extra features as well. Once we've enabled the other features, we 10130 * will use any remaining vectors to reach as close as we can to the 10131 * number of online CPUs. 10132 */ 10133 cpus = num_online_cpus(); 10134 pf->num_lan_msix = min_t(int, cpus, vectors_left / 2); 10135 vectors_left -= pf->num_lan_msix; 10136 10137 /* reserve one vector for sideband flow director */ 10138 if (pf->flags & I40E_FLAG_FD_SB_ENABLED) { 10139 if (vectors_left) { 10140 pf->num_fdsb_msix = 1; 10141 v_budget++; 10142 vectors_left--; 10143 } else { 10144 pf->num_fdsb_msix = 0; 10145 } 10146 } 10147 10148 /* can we reserve enough for iWARP? */ 10149 if (pf->flags & I40E_FLAG_IWARP_ENABLED) { 10150 iwarp_requested = pf->num_iwarp_msix; 10151 10152 if (!vectors_left) 10153 pf->num_iwarp_msix = 0; 10154 else if (vectors_left < pf->num_iwarp_msix) 10155 pf->num_iwarp_msix = 1; 10156 v_budget += pf->num_iwarp_msix; 10157 vectors_left -= pf->num_iwarp_msix; 10158 } 10159 10160 /* any vectors left over go for VMDq support */ 10161 if (pf->flags & I40E_FLAG_VMDQ_ENABLED) { 10162 int vmdq_vecs_wanted = pf->num_vmdq_vsis * pf->num_vmdq_qps; 10163 int vmdq_vecs = min_t(int, vectors_left, vmdq_vecs_wanted); 10164 10165 if (!vectors_left) { 10166 pf->num_vmdq_msix = 0; 10167 pf->num_vmdq_qps = 0; 10168 } else { 10169 /* if we're short on vectors for what's desired, we limit 10170 * the queues per vmdq. If this is still more than are 10171 * available, the user will need to change the number of 10172 * queues/vectors used by the PF later with the ethtool 10173 * channels command 10174 */ 10175 if (vmdq_vecs < vmdq_vecs_wanted) 10176 pf->num_vmdq_qps = 1; 10177 pf->num_vmdq_msix = pf->num_vmdq_qps; 10178 10179 v_budget += vmdq_vecs; 10180 vectors_left -= vmdq_vecs; 10181 } 10182 } 10183 10184 /* On systems with a large number of SMP cores, we previously limited 10185 * the number of vectors for num_lan_msix to be at most 50% of the 10186 * available vectors, to allow for other features. Now, we add back 10187 * the remaining vectors. However, we ensure that the total 10188 * num_lan_msix will not exceed num_online_cpus(). To do this, we 10189 * calculate the number of vectors we can add without going over the 10190 * cap of CPUs. For systems with a small number of CPUs this will be 10191 * zero. 10192 */ 10193 extra_vectors = min_t(int, cpus - pf->num_lan_msix, vectors_left); 10194 pf->num_lan_msix += extra_vectors; 10195 vectors_left -= extra_vectors; 10196 10197 WARN(vectors_left < 0, 10198 "Calculation of remaining vectors underflowed. This is an accounting bug when determining total MSI-X vectors.\n"); 10199 10200 v_budget += pf->num_lan_msix; 10201 pf->msix_entries = kcalloc(v_budget, sizeof(struct msix_entry), 10202 GFP_KERNEL); 10203 if (!pf->msix_entries) 10204 return -ENOMEM; 10205 10206 for (i = 0; i < v_budget; i++) 10207 pf->msix_entries[i].entry = i; 10208 v_actual = i40e_reserve_msix_vectors(pf, v_budget); 10209 10210 if (v_actual < I40E_MIN_MSIX) { 10211 pf->flags &= ~I40E_FLAG_MSIX_ENABLED; 10212 kfree(pf->msix_entries); 10213 pf->msix_entries = NULL; 10214 pci_disable_msix(pf->pdev); 10215 return -ENODEV; 10216 10217 } else if (v_actual == I40E_MIN_MSIX) { 10218 /* Adjust for minimal MSIX use */ 10219 pf->num_vmdq_vsis = 0; 10220 pf->num_vmdq_qps = 0; 10221 pf->num_lan_qps = 1; 10222 pf->num_lan_msix = 1; 10223 10224 } else if (v_actual != v_budget) { 10225 /* If we have limited resources, we will start with no vectors 10226 * for the special features and then allocate vectors to some 10227 * of these features based on the policy and at the end disable 10228 * the features that did not get any vectors. 10229 */ 10230 int vec; 10231 10232 dev_info(&pf->pdev->dev, 10233 "MSI-X vector limit reached with %d, wanted %d, attempting to redistribute vectors\n", 10234 v_actual, v_budget); 10235 /* reserve the misc vector */ 10236 vec = v_actual - 1; 10237 10238 /* Scale vector usage down */ 10239 pf->num_vmdq_msix = 1; /* force VMDqs to only one vector */ 10240 pf->num_vmdq_vsis = 1; 10241 pf->num_vmdq_qps = 1; 10242 10243 /* partition out the remaining vectors */ 10244 switch (vec) { 10245 case 2: 10246 pf->num_lan_msix = 1; 10247 break; 10248 case 3: 10249 if (pf->flags & I40E_FLAG_IWARP_ENABLED) { 10250 pf->num_lan_msix = 1; 10251 pf->num_iwarp_msix = 1; 10252 } else { 10253 pf->num_lan_msix = 2; 10254 } 10255 break; 10256 default: 10257 if (pf->flags & I40E_FLAG_IWARP_ENABLED) { 10258 pf->num_iwarp_msix = min_t(int, (vec / 3), 10259 iwarp_requested); 10260 pf->num_vmdq_vsis = min_t(int, (vec / 3), 10261 I40E_DEFAULT_NUM_VMDQ_VSI); 10262 } else { 10263 pf->num_vmdq_vsis = min_t(int, (vec / 2), 10264 I40E_DEFAULT_NUM_VMDQ_VSI); 10265 } 10266 if (pf->flags & I40E_FLAG_FD_SB_ENABLED) { 10267 pf->num_fdsb_msix = 1; 10268 vec--; 10269 } 10270 pf->num_lan_msix = min_t(int, 10271 (vec - (pf->num_iwarp_msix + pf->num_vmdq_vsis)), 10272 pf->num_lan_msix); 10273 pf->num_lan_qps = pf->num_lan_msix; 10274 break; 10275 } 10276 } 10277 10278 if ((pf->flags & I40E_FLAG_FD_SB_ENABLED) && 10279 (pf->num_fdsb_msix == 0)) { 10280 dev_info(&pf->pdev->dev, "Sideband Flowdir disabled, not enough MSI-X vectors\n"); 10281 pf->flags &= ~I40E_FLAG_FD_SB_ENABLED; 10282 pf->flags |= I40E_FLAG_FD_SB_INACTIVE; 10283 } 10284 if ((pf->flags & I40E_FLAG_VMDQ_ENABLED) && 10285 (pf->num_vmdq_msix == 0)) { 10286 dev_info(&pf->pdev->dev, "VMDq disabled, not enough MSI-X vectors\n"); 10287 pf->flags &= ~I40E_FLAG_VMDQ_ENABLED; 10288 } 10289 10290 if ((pf->flags & I40E_FLAG_IWARP_ENABLED) && 10291 (pf->num_iwarp_msix == 0)) { 10292 dev_info(&pf->pdev->dev, "IWARP disabled, not enough MSI-X vectors\n"); 10293 pf->flags &= ~I40E_FLAG_IWARP_ENABLED; 10294 } 10295 i40e_debug(&pf->hw, I40E_DEBUG_INIT, 10296 "MSI-X vector distribution: PF %d, VMDq %d, FDSB %d, iWARP %d\n", 10297 pf->num_lan_msix, 10298 pf->num_vmdq_msix * pf->num_vmdq_vsis, 10299 pf->num_fdsb_msix, 10300 pf->num_iwarp_msix); 10301 10302 return v_actual; 10303 } 10304 10305 /** 10306 * i40e_vsi_alloc_q_vector - Allocate memory for a single interrupt vector 10307 * @vsi: the VSI being configured 10308 * @v_idx: index of the vector in the vsi struct 10309 * @cpu: cpu to be used on affinity_mask 10310 * 10311 * We allocate one q_vector. If allocation fails we return -ENOMEM. 10312 **/ 10313 static int i40e_vsi_alloc_q_vector(struct i40e_vsi *vsi, int v_idx, int cpu) 10314 { 10315 struct i40e_q_vector *q_vector; 10316 10317 /* allocate q_vector */ 10318 q_vector = kzalloc(sizeof(struct i40e_q_vector), GFP_KERNEL); 10319 if (!q_vector) 10320 return -ENOMEM; 10321 10322 q_vector->vsi = vsi; 10323 q_vector->v_idx = v_idx; 10324 cpumask_copy(&q_vector->affinity_mask, cpu_possible_mask); 10325 10326 if (vsi->netdev) 10327 netif_napi_add(vsi->netdev, &q_vector->napi, 10328 i40e_napi_poll, NAPI_POLL_WEIGHT); 10329 10330 q_vector->rx.latency_range = I40E_LOW_LATENCY; 10331 q_vector->tx.latency_range = I40E_LOW_LATENCY; 10332 10333 /* tie q_vector and vsi together */ 10334 vsi->q_vectors[v_idx] = q_vector; 10335 10336 return 0; 10337 } 10338 10339 /** 10340 * i40e_vsi_alloc_q_vectors - Allocate memory for interrupt vectors 10341 * @vsi: the VSI being configured 10342 * 10343 * We allocate one q_vector per queue interrupt. If allocation fails we 10344 * return -ENOMEM. 10345 **/ 10346 static int i40e_vsi_alloc_q_vectors(struct i40e_vsi *vsi) 10347 { 10348 struct i40e_pf *pf = vsi->back; 10349 int err, v_idx, num_q_vectors, current_cpu; 10350 10351 /* if not MSIX, give the one vector only to the LAN VSI */ 10352 if (pf->flags & I40E_FLAG_MSIX_ENABLED) 10353 num_q_vectors = vsi->num_q_vectors; 10354 else if (vsi == pf->vsi[pf->lan_vsi]) 10355 num_q_vectors = 1; 10356 else 10357 return -EINVAL; 10358 10359 current_cpu = cpumask_first(cpu_online_mask); 10360 10361 for (v_idx = 0; v_idx < num_q_vectors; v_idx++) { 10362 err = i40e_vsi_alloc_q_vector(vsi, v_idx, current_cpu); 10363 if (err) 10364 goto err_out; 10365 current_cpu = cpumask_next(current_cpu, cpu_online_mask); 10366 if (unlikely(current_cpu >= nr_cpu_ids)) 10367 current_cpu = cpumask_first(cpu_online_mask); 10368 } 10369 10370 return 0; 10371 10372 err_out: 10373 while (v_idx--) 10374 i40e_free_q_vector(vsi, v_idx); 10375 10376 return err; 10377 } 10378 10379 /** 10380 * i40e_init_interrupt_scheme - Determine proper interrupt scheme 10381 * @pf: board private structure to initialize 10382 **/ 10383 static int i40e_init_interrupt_scheme(struct i40e_pf *pf) 10384 { 10385 int vectors = 0; 10386 ssize_t size; 10387 10388 if (pf->flags & I40E_FLAG_MSIX_ENABLED) { 10389 vectors = i40e_init_msix(pf); 10390 if (vectors < 0) { 10391 pf->flags &= ~(I40E_FLAG_MSIX_ENABLED | 10392 I40E_FLAG_IWARP_ENABLED | 10393 I40E_FLAG_RSS_ENABLED | 10394 I40E_FLAG_DCB_CAPABLE | 10395 I40E_FLAG_DCB_ENABLED | 10396 I40E_FLAG_SRIOV_ENABLED | 10397 I40E_FLAG_FD_SB_ENABLED | 10398 I40E_FLAG_FD_ATR_ENABLED | 10399 I40E_FLAG_VMDQ_ENABLED); 10400 pf->flags |= I40E_FLAG_FD_SB_INACTIVE; 10401 10402 /* rework the queue expectations without MSIX */ 10403 i40e_determine_queue_usage(pf); 10404 } 10405 } 10406 10407 if (!(pf->flags & I40E_FLAG_MSIX_ENABLED) && 10408 (pf->flags & I40E_FLAG_MSI_ENABLED)) { 10409 dev_info(&pf->pdev->dev, "MSI-X not available, trying MSI\n"); 10410 vectors = pci_enable_msi(pf->pdev); 10411 if (vectors < 0) { 10412 dev_info(&pf->pdev->dev, "MSI init failed - %d\n", 10413 vectors); 10414 pf->flags &= ~I40E_FLAG_MSI_ENABLED; 10415 } 10416 vectors = 1; /* one MSI or Legacy vector */ 10417 } 10418 10419 if (!(pf->flags & (I40E_FLAG_MSIX_ENABLED | I40E_FLAG_MSI_ENABLED))) 10420 dev_info(&pf->pdev->dev, "MSI-X and MSI not available, falling back to Legacy IRQ\n"); 10421 10422 /* set up vector assignment tracking */ 10423 size = sizeof(struct i40e_lump_tracking) + (sizeof(u16) * vectors); 10424 pf->irq_pile = kzalloc(size, GFP_KERNEL); 10425 if (!pf->irq_pile) 10426 return -ENOMEM; 10427 10428 pf->irq_pile->num_entries = vectors; 10429 pf->irq_pile->search_hint = 0; 10430 10431 /* track first vector for misc interrupts, ignore return */ 10432 (void)i40e_get_lump(pf, pf->irq_pile, 1, I40E_PILE_VALID_BIT - 1); 10433 10434 return 0; 10435 } 10436 10437 /** 10438 * i40e_restore_interrupt_scheme - Restore the interrupt scheme 10439 * @pf: private board data structure 10440 * 10441 * Restore the interrupt scheme that was cleared when we suspended the 10442 * device. This should be called during resume to re-allocate the q_vectors 10443 * and reacquire IRQs. 10444 */ 10445 static int i40e_restore_interrupt_scheme(struct i40e_pf *pf) 10446 { 10447 int err, i; 10448 10449 /* We cleared the MSI and MSI-X flags when disabling the old interrupt 10450 * scheme. We need to re-enabled them here in order to attempt to 10451 * re-acquire the MSI or MSI-X vectors 10452 */ 10453 pf->flags |= (I40E_FLAG_MSIX_ENABLED | I40E_FLAG_MSI_ENABLED); 10454 10455 err = i40e_init_interrupt_scheme(pf); 10456 if (err) 10457 return err; 10458 10459 /* Now that we've re-acquired IRQs, we need to remap the vectors and 10460 * rings together again. 10461 */ 10462 for (i = 0; i < pf->num_alloc_vsi; i++) { 10463 if (pf->vsi[i]) { 10464 err = i40e_vsi_alloc_q_vectors(pf->vsi[i]); 10465 if (err) 10466 goto err_unwind; 10467 i40e_vsi_map_rings_to_vectors(pf->vsi[i]); 10468 } 10469 } 10470 10471 err = i40e_setup_misc_vector(pf); 10472 if (err) 10473 goto err_unwind; 10474 10475 return 0; 10476 10477 err_unwind: 10478 while (i--) { 10479 if (pf->vsi[i]) 10480 i40e_vsi_free_q_vectors(pf->vsi[i]); 10481 } 10482 10483 return err; 10484 } 10485 10486 /** 10487 * i40e_setup_misc_vector - Setup the misc vector to handle non queue events 10488 * @pf: board private structure 10489 * 10490 * This sets up the handler for MSIX 0, which is used to manage the 10491 * non-queue interrupts, e.g. AdminQ and errors. This is not used 10492 * when in MSI or Legacy interrupt mode. 10493 **/ 10494 static int i40e_setup_misc_vector(struct i40e_pf *pf) 10495 { 10496 struct i40e_hw *hw = &pf->hw; 10497 int err = 0; 10498 10499 /* Only request the IRQ once, the first time through. */ 10500 if (!test_and_set_bit(__I40E_MISC_IRQ_REQUESTED, pf->state)) { 10501 err = request_irq(pf->msix_entries[0].vector, 10502 i40e_intr, 0, pf->int_name, pf); 10503 if (err) { 10504 clear_bit(__I40E_MISC_IRQ_REQUESTED, pf->state); 10505 dev_info(&pf->pdev->dev, 10506 "request_irq for %s failed: %d\n", 10507 pf->int_name, err); 10508 return -EFAULT; 10509 } 10510 } 10511 10512 i40e_enable_misc_int_causes(pf); 10513 10514 /* associate no queues to the misc vector */ 10515 wr32(hw, I40E_PFINT_LNKLST0, I40E_QUEUE_END_OF_LIST); 10516 wr32(hw, I40E_PFINT_ITR0(I40E_RX_ITR), I40E_ITR_8K); 10517 10518 i40e_flush(hw); 10519 10520 i40e_irq_dynamic_enable_icr0(pf); 10521 10522 return err; 10523 } 10524 10525 /** 10526 * i40e_get_rss_aq - Get RSS keys and lut by using AQ commands 10527 * @vsi: Pointer to vsi structure 10528 * @seed: Buffter to store the hash keys 10529 * @lut: Buffer to store the lookup table entries 10530 * @lut_size: Size of buffer to store the lookup table entries 10531 * 10532 * Return 0 on success, negative on failure 10533 */ 10534 static int i40e_get_rss_aq(struct i40e_vsi *vsi, const u8 *seed, 10535 u8 *lut, u16 lut_size) 10536 { 10537 struct i40e_pf *pf = vsi->back; 10538 struct i40e_hw *hw = &pf->hw; 10539 int ret = 0; 10540 10541 if (seed) { 10542 ret = i40e_aq_get_rss_key(hw, vsi->id, 10543 (struct i40e_aqc_get_set_rss_key_data *)seed); 10544 if (ret) { 10545 dev_info(&pf->pdev->dev, 10546 "Cannot get RSS key, err %s aq_err %s\n", 10547 i40e_stat_str(&pf->hw, ret), 10548 i40e_aq_str(&pf->hw, 10549 pf->hw.aq.asq_last_status)); 10550 return ret; 10551 } 10552 } 10553 10554 if (lut) { 10555 bool pf_lut = vsi->type == I40E_VSI_MAIN ? true : false; 10556 10557 ret = i40e_aq_get_rss_lut(hw, vsi->id, pf_lut, lut, lut_size); 10558 if (ret) { 10559 dev_info(&pf->pdev->dev, 10560 "Cannot get RSS lut, err %s aq_err %s\n", 10561 i40e_stat_str(&pf->hw, ret), 10562 i40e_aq_str(&pf->hw, 10563 pf->hw.aq.asq_last_status)); 10564 return ret; 10565 } 10566 } 10567 10568 return ret; 10569 } 10570 10571 /** 10572 * i40e_config_rss_reg - Configure RSS keys and lut by writing registers 10573 * @vsi: Pointer to vsi structure 10574 * @seed: RSS hash seed 10575 * @lut: Lookup table 10576 * @lut_size: Lookup table size 10577 * 10578 * Returns 0 on success, negative on failure 10579 **/ 10580 static int i40e_config_rss_reg(struct i40e_vsi *vsi, const u8 *seed, 10581 const u8 *lut, u16 lut_size) 10582 { 10583 struct i40e_pf *pf = vsi->back; 10584 struct i40e_hw *hw = &pf->hw; 10585 u16 vf_id = vsi->vf_id; 10586 u8 i; 10587 10588 /* Fill out hash function seed */ 10589 if (seed) { 10590 u32 *seed_dw = (u32 *)seed; 10591 10592 if (vsi->type == I40E_VSI_MAIN) { 10593 for (i = 0; i <= I40E_PFQF_HKEY_MAX_INDEX; i++) 10594 wr32(hw, I40E_PFQF_HKEY(i), seed_dw[i]); 10595 } else if (vsi->type == I40E_VSI_SRIOV) { 10596 for (i = 0; i <= I40E_VFQF_HKEY1_MAX_INDEX; i++) 10597 wr32(hw, I40E_VFQF_HKEY1(i, vf_id), seed_dw[i]); 10598 } else { 10599 dev_err(&pf->pdev->dev, "Cannot set RSS seed - invalid VSI type\n"); 10600 } 10601 } 10602 10603 if (lut) { 10604 u32 *lut_dw = (u32 *)lut; 10605 10606 if (vsi->type == I40E_VSI_MAIN) { 10607 if (lut_size != I40E_HLUT_ARRAY_SIZE) 10608 return -EINVAL; 10609 for (i = 0; i <= I40E_PFQF_HLUT_MAX_INDEX; i++) 10610 wr32(hw, I40E_PFQF_HLUT(i), lut_dw[i]); 10611 } else if (vsi->type == I40E_VSI_SRIOV) { 10612 if (lut_size != I40E_VF_HLUT_ARRAY_SIZE) 10613 return -EINVAL; 10614 for (i = 0; i <= I40E_VFQF_HLUT_MAX_INDEX; i++) 10615 wr32(hw, I40E_VFQF_HLUT1(i, vf_id), lut_dw[i]); 10616 } else { 10617 dev_err(&pf->pdev->dev, "Cannot set RSS LUT - invalid VSI type\n"); 10618 } 10619 } 10620 i40e_flush(hw); 10621 10622 return 0; 10623 } 10624 10625 /** 10626 * i40e_get_rss_reg - Get the RSS keys and lut by reading registers 10627 * @vsi: Pointer to VSI structure 10628 * @seed: Buffer to store the keys 10629 * @lut: Buffer to store the lookup table entries 10630 * @lut_size: Size of buffer to store the lookup table entries 10631 * 10632 * Returns 0 on success, negative on failure 10633 */ 10634 static int i40e_get_rss_reg(struct i40e_vsi *vsi, u8 *seed, 10635 u8 *lut, u16 lut_size) 10636 { 10637 struct i40e_pf *pf = vsi->back; 10638 struct i40e_hw *hw = &pf->hw; 10639 u16 i; 10640 10641 if (seed) { 10642 u32 *seed_dw = (u32 *)seed; 10643 10644 for (i = 0; i <= I40E_PFQF_HKEY_MAX_INDEX; i++) 10645 seed_dw[i] = i40e_read_rx_ctl(hw, I40E_PFQF_HKEY(i)); 10646 } 10647 if (lut) { 10648 u32 *lut_dw = (u32 *)lut; 10649 10650 if (lut_size != I40E_HLUT_ARRAY_SIZE) 10651 return -EINVAL; 10652 for (i = 0; i <= I40E_PFQF_HLUT_MAX_INDEX; i++) 10653 lut_dw[i] = rd32(hw, I40E_PFQF_HLUT(i)); 10654 } 10655 10656 return 0; 10657 } 10658 10659 /** 10660 * i40e_config_rss - Configure RSS keys and lut 10661 * @vsi: Pointer to VSI structure 10662 * @seed: RSS hash seed 10663 * @lut: Lookup table 10664 * @lut_size: Lookup table size 10665 * 10666 * Returns 0 on success, negative on failure 10667 */ 10668 int i40e_config_rss(struct i40e_vsi *vsi, u8 *seed, u8 *lut, u16 lut_size) 10669 { 10670 struct i40e_pf *pf = vsi->back; 10671 10672 if (pf->hw_features & I40E_HW_RSS_AQ_CAPABLE) 10673 return i40e_config_rss_aq(vsi, seed, lut, lut_size); 10674 else 10675 return i40e_config_rss_reg(vsi, seed, lut, lut_size); 10676 } 10677 10678 /** 10679 * i40e_get_rss - Get RSS keys and lut 10680 * @vsi: Pointer to VSI structure 10681 * @seed: Buffer to store the keys 10682 * @lut: Buffer to store the lookup table entries 10683 * lut_size: Size of buffer to store the lookup table entries 10684 * 10685 * Returns 0 on success, negative on failure 10686 */ 10687 int i40e_get_rss(struct i40e_vsi *vsi, u8 *seed, u8 *lut, u16 lut_size) 10688 { 10689 struct i40e_pf *pf = vsi->back; 10690 10691 if (pf->hw_features & I40E_HW_RSS_AQ_CAPABLE) 10692 return i40e_get_rss_aq(vsi, seed, lut, lut_size); 10693 else 10694 return i40e_get_rss_reg(vsi, seed, lut, lut_size); 10695 } 10696 10697 /** 10698 * i40e_fill_rss_lut - Fill the RSS lookup table with default values 10699 * @pf: Pointer to board private structure 10700 * @lut: Lookup table 10701 * @rss_table_size: Lookup table size 10702 * @rss_size: Range of queue number for hashing 10703 */ 10704 void i40e_fill_rss_lut(struct i40e_pf *pf, u8 *lut, 10705 u16 rss_table_size, u16 rss_size) 10706 { 10707 u16 i; 10708 10709 for (i = 0; i < rss_table_size; i++) 10710 lut[i] = i % rss_size; 10711 } 10712 10713 /** 10714 * i40e_pf_config_rss - Prepare for RSS if used 10715 * @pf: board private structure 10716 **/ 10717 static int i40e_pf_config_rss(struct i40e_pf *pf) 10718 { 10719 struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi]; 10720 u8 seed[I40E_HKEY_ARRAY_SIZE]; 10721 u8 *lut; 10722 struct i40e_hw *hw = &pf->hw; 10723 u32 reg_val; 10724 u64 hena; 10725 int ret; 10726 10727 /* By default we enable TCP/UDP with IPv4/IPv6 ptypes */ 10728 hena = (u64)i40e_read_rx_ctl(hw, I40E_PFQF_HENA(0)) | 10729 ((u64)i40e_read_rx_ctl(hw, I40E_PFQF_HENA(1)) << 32); 10730 hena |= i40e_pf_get_default_rss_hena(pf); 10731 10732 i40e_write_rx_ctl(hw, I40E_PFQF_HENA(0), (u32)hena); 10733 i40e_write_rx_ctl(hw, I40E_PFQF_HENA(1), (u32)(hena >> 32)); 10734 10735 /* Determine the RSS table size based on the hardware capabilities */ 10736 reg_val = i40e_read_rx_ctl(hw, I40E_PFQF_CTL_0); 10737 reg_val = (pf->rss_table_size == 512) ? 10738 (reg_val | I40E_PFQF_CTL_0_HASHLUTSIZE_512) : 10739 (reg_val & ~I40E_PFQF_CTL_0_HASHLUTSIZE_512); 10740 i40e_write_rx_ctl(hw, I40E_PFQF_CTL_0, reg_val); 10741 10742 /* Determine the RSS size of the VSI */ 10743 if (!vsi->rss_size) { 10744 u16 qcount; 10745 /* If the firmware does something weird during VSI init, we 10746 * could end up with zero TCs. Check for that to avoid 10747 * divide-by-zero. It probably won't pass traffic, but it also 10748 * won't panic. 10749 */ 10750 qcount = vsi->num_queue_pairs / 10751 (vsi->tc_config.numtc ? vsi->tc_config.numtc : 1); 10752 vsi->rss_size = min_t(int, pf->alloc_rss_size, qcount); 10753 } 10754 if (!vsi->rss_size) 10755 return -EINVAL; 10756 10757 lut = kzalloc(vsi->rss_table_size, GFP_KERNEL); 10758 if (!lut) 10759 return -ENOMEM; 10760 10761 /* Use user configured lut if there is one, otherwise use default */ 10762 if (vsi->rss_lut_user) 10763 memcpy(lut, vsi->rss_lut_user, vsi->rss_table_size); 10764 else 10765 i40e_fill_rss_lut(pf, lut, vsi->rss_table_size, vsi->rss_size); 10766 10767 /* Use user configured hash key if there is one, otherwise 10768 * use default. 10769 */ 10770 if (vsi->rss_hkey_user) 10771 memcpy(seed, vsi->rss_hkey_user, I40E_HKEY_ARRAY_SIZE); 10772 else 10773 netdev_rss_key_fill((void *)seed, I40E_HKEY_ARRAY_SIZE); 10774 ret = i40e_config_rss(vsi, seed, lut, vsi->rss_table_size); 10775 kfree(lut); 10776 10777 return ret; 10778 } 10779 10780 /** 10781 * i40e_reconfig_rss_queues - change number of queues for rss and rebuild 10782 * @pf: board private structure 10783 * @queue_count: the requested queue count for rss. 10784 * 10785 * returns 0 if rss is not enabled, if enabled returns the final rss queue 10786 * count which may be different from the requested queue count. 10787 * Note: expects to be called while under rtnl_lock() 10788 **/ 10789 int i40e_reconfig_rss_queues(struct i40e_pf *pf, int queue_count) 10790 { 10791 struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi]; 10792 int new_rss_size; 10793 10794 if (!(pf->flags & I40E_FLAG_RSS_ENABLED)) 10795 return 0; 10796 10797 new_rss_size = min_t(int, queue_count, pf->rss_size_max); 10798 10799 if (queue_count != vsi->num_queue_pairs) { 10800 u16 qcount; 10801 10802 vsi->req_queue_pairs = queue_count; 10803 i40e_prep_for_reset(pf, true); 10804 10805 pf->alloc_rss_size = new_rss_size; 10806 10807 i40e_reset_and_rebuild(pf, true, true); 10808 10809 /* Discard the user configured hash keys and lut, if less 10810 * queues are enabled. 10811 */ 10812 if (queue_count < vsi->rss_size) { 10813 i40e_clear_rss_config_user(vsi); 10814 dev_dbg(&pf->pdev->dev, 10815 "discard user configured hash keys and lut\n"); 10816 } 10817 10818 /* Reset vsi->rss_size, as number of enabled queues changed */ 10819 qcount = vsi->num_queue_pairs / vsi->tc_config.numtc; 10820 vsi->rss_size = min_t(int, pf->alloc_rss_size, qcount); 10821 10822 i40e_pf_config_rss(pf); 10823 } 10824 dev_info(&pf->pdev->dev, "User requested queue count/HW max RSS count: %d/%d\n", 10825 vsi->req_queue_pairs, pf->rss_size_max); 10826 return pf->alloc_rss_size; 10827 } 10828 10829 /** 10830 * i40e_get_partition_bw_setting - Retrieve BW settings for this PF partition 10831 * @pf: board private structure 10832 **/ 10833 i40e_status i40e_get_partition_bw_setting(struct i40e_pf *pf) 10834 { 10835 i40e_status status; 10836 bool min_valid, max_valid; 10837 u32 max_bw, min_bw; 10838 10839 status = i40e_read_bw_from_alt_ram(&pf->hw, &max_bw, &min_bw, 10840 &min_valid, &max_valid); 10841 10842 if (!status) { 10843 if (min_valid) 10844 pf->min_bw = min_bw; 10845 if (max_valid) 10846 pf->max_bw = max_bw; 10847 } 10848 10849 return status; 10850 } 10851 10852 /** 10853 * i40e_set_partition_bw_setting - Set BW settings for this PF partition 10854 * @pf: board private structure 10855 **/ 10856 i40e_status i40e_set_partition_bw_setting(struct i40e_pf *pf) 10857 { 10858 struct i40e_aqc_configure_partition_bw_data bw_data; 10859 i40e_status status; 10860 10861 /* Set the valid bit for this PF */ 10862 bw_data.pf_valid_bits = cpu_to_le16(BIT(pf->hw.pf_id)); 10863 bw_data.max_bw[pf->hw.pf_id] = pf->max_bw & I40E_ALT_BW_VALUE_MASK; 10864 bw_data.min_bw[pf->hw.pf_id] = pf->min_bw & I40E_ALT_BW_VALUE_MASK; 10865 10866 /* Set the new bandwidths */ 10867 status = i40e_aq_configure_partition_bw(&pf->hw, &bw_data, NULL); 10868 10869 return status; 10870 } 10871 10872 /** 10873 * i40e_commit_partition_bw_setting - Commit BW settings for this PF partition 10874 * @pf: board private structure 10875 **/ 10876 i40e_status i40e_commit_partition_bw_setting(struct i40e_pf *pf) 10877 { 10878 /* Commit temporary BW setting to permanent NVM image */ 10879 enum i40e_admin_queue_err last_aq_status; 10880 i40e_status ret; 10881 u16 nvm_word; 10882 10883 if (pf->hw.partition_id != 1) { 10884 dev_info(&pf->pdev->dev, 10885 "Commit BW only works on partition 1! This is partition %d", 10886 pf->hw.partition_id); 10887 ret = I40E_NOT_SUPPORTED; 10888 goto bw_commit_out; 10889 } 10890 10891 /* Acquire NVM for read access */ 10892 ret = i40e_acquire_nvm(&pf->hw, I40E_RESOURCE_READ); 10893 last_aq_status = pf->hw.aq.asq_last_status; 10894 if (ret) { 10895 dev_info(&pf->pdev->dev, 10896 "Cannot acquire NVM for read access, err %s aq_err %s\n", 10897 i40e_stat_str(&pf->hw, ret), 10898 i40e_aq_str(&pf->hw, last_aq_status)); 10899 goto bw_commit_out; 10900 } 10901 10902 /* Read word 0x10 of NVM - SW compatibility word 1 */ 10903 ret = i40e_aq_read_nvm(&pf->hw, 10904 I40E_SR_NVM_CONTROL_WORD, 10905 0x10, sizeof(nvm_word), &nvm_word, 10906 false, NULL); 10907 /* Save off last admin queue command status before releasing 10908 * the NVM 10909 */ 10910 last_aq_status = pf->hw.aq.asq_last_status; 10911 i40e_release_nvm(&pf->hw); 10912 if (ret) { 10913 dev_info(&pf->pdev->dev, "NVM read error, err %s aq_err %s\n", 10914 i40e_stat_str(&pf->hw, ret), 10915 i40e_aq_str(&pf->hw, last_aq_status)); 10916 goto bw_commit_out; 10917 } 10918 10919 /* Wait a bit for NVM release to complete */ 10920 msleep(50); 10921 10922 /* Acquire NVM for write access */ 10923 ret = i40e_acquire_nvm(&pf->hw, I40E_RESOURCE_WRITE); 10924 last_aq_status = pf->hw.aq.asq_last_status; 10925 if (ret) { 10926 dev_info(&pf->pdev->dev, 10927 "Cannot acquire NVM for write access, err %s aq_err %s\n", 10928 i40e_stat_str(&pf->hw, ret), 10929 i40e_aq_str(&pf->hw, last_aq_status)); 10930 goto bw_commit_out; 10931 } 10932 /* Write it back out unchanged to initiate update NVM, 10933 * which will force a write of the shadow (alt) RAM to 10934 * the NVM - thus storing the bandwidth values permanently. 10935 */ 10936 ret = i40e_aq_update_nvm(&pf->hw, 10937 I40E_SR_NVM_CONTROL_WORD, 10938 0x10, sizeof(nvm_word), 10939 &nvm_word, true, 0, NULL); 10940 /* Save off last admin queue command status before releasing 10941 * the NVM 10942 */ 10943 last_aq_status = pf->hw.aq.asq_last_status; 10944 i40e_release_nvm(&pf->hw); 10945 if (ret) 10946 dev_info(&pf->pdev->dev, 10947 "BW settings NOT SAVED, err %s aq_err %s\n", 10948 i40e_stat_str(&pf->hw, ret), 10949 i40e_aq_str(&pf->hw, last_aq_status)); 10950 bw_commit_out: 10951 10952 return ret; 10953 } 10954 10955 /** 10956 * i40e_sw_init - Initialize general software structures (struct i40e_pf) 10957 * @pf: board private structure to initialize 10958 * 10959 * i40e_sw_init initializes the Adapter private data structure. 10960 * Fields are initialized based on PCI device information and 10961 * OS network device settings (MTU size). 10962 **/ 10963 static int i40e_sw_init(struct i40e_pf *pf) 10964 { 10965 int err = 0; 10966 int size; 10967 10968 /* Set default capability flags */ 10969 pf->flags = I40E_FLAG_RX_CSUM_ENABLED | 10970 I40E_FLAG_MSI_ENABLED | 10971 I40E_FLAG_MSIX_ENABLED; 10972 10973 /* Set default ITR */ 10974 pf->rx_itr_default = I40E_ITR_RX_DEF; 10975 pf->tx_itr_default = I40E_ITR_TX_DEF; 10976 10977 /* Depending on PF configurations, it is possible that the RSS 10978 * maximum might end up larger than the available queues 10979 */ 10980 pf->rss_size_max = BIT(pf->hw.func_caps.rss_table_entry_width); 10981 pf->alloc_rss_size = 1; 10982 pf->rss_table_size = pf->hw.func_caps.rss_table_size; 10983 pf->rss_size_max = min_t(int, pf->rss_size_max, 10984 pf->hw.func_caps.num_tx_qp); 10985 if (pf->hw.func_caps.rss) { 10986 pf->flags |= I40E_FLAG_RSS_ENABLED; 10987 pf->alloc_rss_size = min_t(int, pf->rss_size_max, 10988 num_online_cpus()); 10989 } 10990 10991 /* MFP mode enabled */ 10992 if (pf->hw.func_caps.npar_enable || pf->hw.func_caps.flex10_enable) { 10993 pf->flags |= I40E_FLAG_MFP_ENABLED; 10994 dev_info(&pf->pdev->dev, "MFP mode Enabled\n"); 10995 if (i40e_get_partition_bw_setting(pf)) { 10996 dev_warn(&pf->pdev->dev, 10997 "Could not get partition bw settings\n"); 10998 } else { 10999 dev_info(&pf->pdev->dev, 11000 "Partition BW Min = %8.8x, Max = %8.8x\n", 11001 pf->min_bw, pf->max_bw); 11002 11003 /* nudge the Tx scheduler */ 11004 i40e_set_partition_bw_setting(pf); 11005 } 11006 } 11007 11008 if ((pf->hw.func_caps.fd_filters_guaranteed > 0) || 11009 (pf->hw.func_caps.fd_filters_best_effort > 0)) { 11010 pf->flags |= I40E_FLAG_FD_ATR_ENABLED; 11011 pf->atr_sample_rate = I40E_DEFAULT_ATR_SAMPLE_RATE; 11012 if (pf->flags & I40E_FLAG_MFP_ENABLED && 11013 pf->hw.num_partitions > 1) 11014 dev_info(&pf->pdev->dev, 11015 "Flow Director Sideband mode Disabled in MFP mode\n"); 11016 else 11017 pf->flags |= I40E_FLAG_FD_SB_ENABLED; 11018 pf->fdir_pf_filter_count = 11019 pf->hw.func_caps.fd_filters_guaranteed; 11020 pf->hw.fdir_shared_filter_count = 11021 pf->hw.func_caps.fd_filters_best_effort; 11022 } 11023 11024 if (pf->hw.mac.type == I40E_MAC_X722) { 11025 pf->hw_features |= (I40E_HW_RSS_AQ_CAPABLE | 11026 I40E_HW_128_QP_RSS_CAPABLE | 11027 I40E_HW_ATR_EVICT_CAPABLE | 11028 I40E_HW_WB_ON_ITR_CAPABLE | 11029 I40E_HW_MULTIPLE_TCP_UDP_RSS_PCTYPE | 11030 I40E_HW_NO_PCI_LINK_CHECK | 11031 I40E_HW_USE_SET_LLDP_MIB | 11032 I40E_HW_GENEVE_OFFLOAD_CAPABLE | 11033 I40E_HW_PTP_L4_CAPABLE | 11034 I40E_HW_WOL_MC_MAGIC_PKT_WAKE | 11035 I40E_HW_OUTER_UDP_CSUM_CAPABLE); 11036 11037 #define I40E_FDEVICT_PCTYPE_DEFAULT 0xc03 11038 if (rd32(&pf->hw, I40E_GLQF_FDEVICTENA(1)) != 11039 I40E_FDEVICT_PCTYPE_DEFAULT) { 11040 dev_warn(&pf->pdev->dev, 11041 "FD EVICT PCTYPES are not right, disable FD HW EVICT\n"); 11042 pf->hw_features &= ~I40E_HW_ATR_EVICT_CAPABLE; 11043 } 11044 } else if ((pf->hw.aq.api_maj_ver > 1) || 11045 ((pf->hw.aq.api_maj_ver == 1) && 11046 (pf->hw.aq.api_min_ver > 4))) { 11047 /* Supported in FW API version higher than 1.4 */ 11048 pf->hw_features |= I40E_HW_GENEVE_OFFLOAD_CAPABLE; 11049 } 11050 11051 /* Enable HW ATR eviction if possible */ 11052 if (pf->hw_features & I40E_HW_ATR_EVICT_CAPABLE) 11053 pf->flags |= I40E_FLAG_HW_ATR_EVICT_ENABLED; 11054 11055 if ((pf->hw.mac.type == I40E_MAC_XL710) && 11056 (((pf->hw.aq.fw_maj_ver == 4) && (pf->hw.aq.fw_min_ver < 33)) || 11057 (pf->hw.aq.fw_maj_ver < 4))) { 11058 pf->hw_features |= I40E_HW_RESTART_AUTONEG; 11059 /* No DCB support for FW < v4.33 */ 11060 pf->hw_features |= I40E_HW_NO_DCB_SUPPORT; 11061 } 11062 11063 /* Disable FW LLDP if FW < v4.3 */ 11064 if ((pf->hw.mac.type == I40E_MAC_XL710) && 11065 (((pf->hw.aq.fw_maj_ver == 4) && (pf->hw.aq.fw_min_ver < 3)) || 11066 (pf->hw.aq.fw_maj_ver < 4))) 11067 pf->hw_features |= I40E_HW_STOP_FW_LLDP; 11068 11069 /* Use the FW Set LLDP MIB API if FW > v4.40 */ 11070 if ((pf->hw.mac.type == I40E_MAC_XL710) && 11071 (((pf->hw.aq.fw_maj_ver == 4) && (pf->hw.aq.fw_min_ver >= 40)) || 11072 (pf->hw.aq.fw_maj_ver >= 5))) 11073 pf->hw_features |= I40E_HW_USE_SET_LLDP_MIB; 11074 11075 /* Enable PTP L4 if FW > v6.0 */ 11076 if (pf->hw.mac.type == I40E_MAC_XL710 && 11077 pf->hw.aq.fw_maj_ver >= 6) 11078 pf->hw_features |= I40E_HW_PTP_L4_CAPABLE; 11079 11080 if (pf->hw.func_caps.vmdq && num_online_cpus() != 1) { 11081 pf->num_vmdq_vsis = I40E_DEFAULT_NUM_VMDQ_VSI; 11082 pf->flags |= I40E_FLAG_VMDQ_ENABLED; 11083 pf->num_vmdq_qps = i40e_default_queues_per_vmdq(pf); 11084 } 11085 11086 if (pf->hw.func_caps.iwarp && num_online_cpus() != 1) { 11087 pf->flags |= I40E_FLAG_IWARP_ENABLED; 11088 /* IWARP needs one extra vector for CQP just like MISC.*/ 11089 pf->num_iwarp_msix = (int)num_online_cpus() + 1; 11090 } 11091 11092 #ifdef CONFIG_PCI_IOV 11093 if (pf->hw.func_caps.num_vfs && pf->hw.partition_id == 1) { 11094 pf->num_vf_qps = I40E_DEFAULT_QUEUES_PER_VF; 11095 pf->flags |= I40E_FLAG_SRIOV_ENABLED; 11096 pf->num_req_vfs = min_t(int, 11097 pf->hw.func_caps.num_vfs, 11098 I40E_MAX_VF_COUNT); 11099 } 11100 #endif /* CONFIG_PCI_IOV */ 11101 pf->eeprom_version = 0xDEAD; 11102 pf->lan_veb = I40E_NO_VEB; 11103 pf->lan_vsi = I40E_NO_VSI; 11104 11105 /* By default FW has this off for performance reasons */ 11106 pf->flags &= ~I40E_FLAG_VEB_STATS_ENABLED; 11107 11108 /* set up queue assignment tracking */ 11109 size = sizeof(struct i40e_lump_tracking) 11110 + (sizeof(u16) * pf->hw.func_caps.num_tx_qp); 11111 pf->qp_pile = kzalloc(size, GFP_KERNEL); 11112 if (!pf->qp_pile) { 11113 err = -ENOMEM; 11114 goto sw_init_done; 11115 } 11116 pf->qp_pile->num_entries = pf->hw.func_caps.num_tx_qp; 11117 pf->qp_pile->search_hint = 0; 11118 11119 pf->tx_timeout_recovery_level = 1; 11120 11121 mutex_init(&pf->switch_mutex); 11122 11123 sw_init_done: 11124 return err; 11125 } 11126 11127 /** 11128 * i40e_set_ntuple - set the ntuple feature flag and take action 11129 * @pf: board private structure to initialize 11130 * @features: the feature set that the stack is suggesting 11131 * 11132 * returns a bool to indicate if reset needs to happen 11133 **/ 11134 bool i40e_set_ntuple(struct i40e_pf *pf, netdev_features_t features) 11135 { 11136 bool need_reset = false; 11137 11138 /* Check if Flow Director n-tuple support was enabled or disabled. If 11139 * the state changed, we need to reset. 11140 */ 11141 if (features & NETIF_F_NTUPLE) { 11142 /* Enable filters and mark for reset */ 11143 if (!(pf->flags & I40E_FLAG_FD_SB_ENABLED)) 11144 need_reset = true; 11145 /* enable FD_SB only if there is MSI-X vector and no cloud 11146 * filters exist 11147 */ 11148 if (pf->num_fdsb_msix > 0 && !pf->num_cloud_filters) { 11149 pf->flags |= I40E_FLAG_FD_SB_ENABLED; 11150 pf->flags &= ~I40E_FLAG_FD_SB_INACTIVE; 11151 } 11152 } else { 11153 /* turn off filters, mark for reset and clear SW filter list */ 11154 if (pf->flags & I40E_FLAG_FD_SB_ENABLED) { 11155 need_reset = true; 11156 i40e_fdir_filter_exit(pf); 11157 } 11158 pf->flags &= ~(I40E_FLAG_FD_SB_ENABLED | 11159 I40E_FLAG_FD_SB_AUTO_DISABLED); 11160 pf->flags |= I40E_FLAG_FD_SB_INACTIVE; 11161 11162 /* reset fd counters */ 11163 pf->fd_add_err = 0; 11164 pf->fd_atr_cnt = 0; 11165 /* if ATR was auto disabled it can be re-enabled. */ 11166 if (pf->flags & I40E_FLAG_FD_ATR_AUTO_DISABLED) { 11167 pf->flags &= ~I40E_FLAG_FD_ATR_AUTO_DISABLED; 11168 if ((pf->flags & I40E_FLAG_FD_ATR_ENABLED) && 11169 (I40E_DEBUG_FD & pf->hw.debug_mask)) 11170 dev_info(&pf->pdev->dev, "ATR re-enabled.\n"); 11171 } 11172 } 11173 return need_reset; 11174 } 11175 11176 /** 11177 * i40e_clear_rss_lut - clear the rx hash lookup table 11178 * @vsi: the VSI being configured 11179 **/ 11180 static void i40e_clear_rss_lut(struct i40e_vsi *vsi) 11181 { 11182 struct i40e_pf *pf = vsi->back; 11183 struct i40e_hw *hw = &pf->hw; 11184 u16 vf_id = vsi->vf_id; 11185 u8 i; 11186 11187 if (vsi->type == I40E_VSI_MAIN) { 11188 for (i = 0; i <= I40E_PFQF_HLUT_MAX_INDEX; i++) 11189 wr32(hw, I40E_PFQF_HLUT(i), 0); 11190 } else if (vsi->type == I40E_VSI_SRIOV) { 11191 for (i = 0; i <= I40E_VFQF_HLUT_MAX_INDEX; i++) 11192 i40e_write_rx_ctl(hw, I40E_VFQF_HLUT1(i, vf_id), 0); 11193 } else { 11194 dev_err(&pf->pdev->dev, "Cannot set RSS LUT - invalid VSI type\n"); 11195 } 11196 } 11197 11198 /** 11199 * i40e_set_features - set the netdev feature flags 11200 * @netdev: ptr to the netdev being adjusted 11201 * @features: the feature set that the stack is suggesting 11202 * Note: expects to be called while under rtnl_lock() 11203 **/ 11204 static int i40e_set_features(struct net_device *netdev, 11205 netdev_features_t features) 11206 { 11207 struct i40e_netdev_priv *np = netdev_priv(netdev); 11208 struct i40e_vsi *vsi = np->vsi; 11209 struct i40e_pf *pf = vsi->back; 11210 bool need_reset; 11211 11212 if (features & NETIF_F_RXHASH && !(netdev->features & NETIF_F_RXHASH)) 11213 i40e_pf_config_rss(pf); 11214 else if (!(features & NETIF_F_RXHASH) && 11215 netdev->features & NETIF_F_RXHASH) 11216 i40e_clear_rss_lut(vsi); 11217 11218 if (features & NETIF_F_HW_VLAN_CTAG_RX) 11219 i40e_vlan_stripping_enable(vsi); 11220 else 11221 i40e_vlan_stripping_disable(vsi); 11222 11223 if (!(features & NETIF_F_HW_TC) && pf->num_cloud_filters) { 11224 dev_err(&pf->pdev->dev, 11225 "Offloaded tc filters active, can't turn hw_tc_offload off"); 11226 return -EINVAL; 11227 } 11228 11229 need_reset = i40e_set_ntuple(pf, features); 11230 11231 if (need_reset) 11232 i40e_do_reset(pf, I40E_PF_RESET_FLAG, true); 11233 11234 return 0; 11235 } 11236 11237 /** 11238 * i40e_get_udp_port_idx - Lookup a possibly offloaded for Rx UDP port 11239 * @pf: board private structure 11240 * @port: The UDP port to look up 11241 * 11242 * Returns the index number or I40E_MAX_PF_UDP_OFFLOAD_PORTS if port not found 11243 **/ 11244 static u8 i40e_get_udp_port_idx(struct i40e_pf *pf, u16 port) 11245 { 11246 u8 i; 11247 11248 for (i = 0; i < I40E_MAX_PF_UDP_OFFLOAD_PORTS; i++) { 11249 if (pf->udp_ports[i].port == port) 11250 return i; 11251 } 11252 11253 return i; 11254 } 11255 11256 /** 11257 * i40e_udp_tunnel_add - Get notifications about UDP tunnel ports that come up 11258 * @netdev: This physical port's netdev 11259 * @ti: Tunnel endpoint information 11260 **/ 11261 static void i40e_udp_tunnel_add(struct net_device *netdev, 11262 struct udp_tunnel_info *ti) 11263 { 11264 struct i40e_netdev_priv *np = netdev_priv(netdev); 11265 struct i40e_vsi *vsi = np->vsi; 11266 struct i40e_pf *pf = vsi->back; 11267 u16 port = ntohs(ti->port); 11268 u8 next_idx; 11269 u8 idx; 11270 11271 idx = i40e_get_udp_port_idx(pf, port); 11272 11273 /* Check if port already exists */ 11274 if (idx < I40E_MAX_PF_UDP_OFFLOAD_PORTS) { 11275 netdev_info(netdev, "port %d already offloaded\n", port); 11276 return; 11277 } 11278 11279 /* Now check if there is space to add the new port */ 11280 next_idx = i40e_get_udp_port_idx(pf, 0); 11281 11282 if (next_idx == I40E_MAX_PF_UDP_OFFLOAD_PORTS) { 11283 netdev_info(netdev, "maximum number of offloaded UDP ports reached, not adding port %d\n", 11284 port); 11285 return; 11286 } 11287 11288 switch (ti->type) { 11289 case UDP_TUNNEL_TYPE_VXLAN: 11290 pf->udp_ports[next_idx].type = I40E_AQC_TUNNEL_TYPE_VXLAN; 11291 break; 11292 case UDP_TUNNEL_TYPE_GENEVE: 11293 if (!(pf->hw_features & I40E_HW_GENEVE_OFFLOAD_CAPABLE)) 11294 return; 11295 pf->udp_ports[next_idx].type = I40E_AQC_TUNNEL_TYPE_NGE; 11296 break; 11297 default: 11298 return; 11299 } 11300 11301 /* New port: add it and mark its index in the bitmap */ 11302 pf->udp_ports[next_idx].port = port; 11303 pf->pending_udp_bitmap |= BIT_ULL(next_idx); 11304 pf->flags |= I40E_FLAG_UDP_FILTER_SYNC; 11305 } 11306 11307 /** 11308 * i40e_udp_tunnel_del - Get notifications about UDP tunnel ports that go away 11309 * @netdev: This physical port's netdev 11310 * @ti: Tunnel endpoint information 11311 **/ 11312 static void i40e_udp_tunnel_del(struct net_device *netdev, 11313 struct udp_tunnel_info *ti) 11314 { 11315 struct i40e_netdev_priv *np = netdev_priv(netdev); 11316 struct i40e_vsi *vsi = np->vsi; 11317 struct i40e_pf *pf = vsi->back; 11318 u16 port = ntohs(ti->port); 11319 u8 idx; 11320 11321 idx = i40e_get_udp_port_idx(pf, port); 11322 11323 /* Check if port already exists */ 11324 if (idx >= I40E_MAX_PF_UDP_OFFLOAD_PORTS) 11325 goto not_found; 11326 11327 switch (ti->type) { 11328 case UDP_TUNNEL_TYPE_VXLAN: 11329 if (pf->udp_ports[idx].type != I40E_AQC_TUNNEL_TYPE_VXLAN) 11330 goto not_found; 11331 break; 11332 case UDP_TUNNEL_TYPE_GENEVE: 11333 if (pf->udp_ports[idx].type != I40E_AQC_TUNNEL_TYPE_NGE) 11334 goto not_found; 11335 break; 11336 default: 11337 goto not_found; 11338 } 11339 11340 /* if port exists, set it to 0 (mark for deletion) 11341 * and make it pending 11342 */ 11343 pf->udp_ports[idx].port = 0; 11344 pf->pending_udp_bitmap |= BIT_ULL(idx); 11345 pf->flags |= I40E_FLAG_UDP_FILTER_SYNC; 11346 11347 return; 11348 not_found: 11349 netdev_warn(netdev, "UDP port %d was not found, not deleting\n", 11350 port); 11351 } 11352 11353 static int i40e_get_phys_port_id(struct net_device *netdev, 11354 struct netdev_phys_item_id *ppid) 11355 { 11356 struct i40e_netdev_priv *np = netdev_priv(netdev); 11357 struct i40e_pf *pf = np->vsi->back; 11358 struct i40e_hw *hw = &pf->hw; 11359 11360 if (!(pf->hw_features & I40E_HW_PORT_ID_VALID)) 11361 return -EOPNOTSUPP; 11362 11363 ppid->id_len = min_t(int, sizeof(hw->mac.port_addr), sizeof(ppid->id)); 11364 memcpy(ppid->id, hw->mac.port_addr, ppid->id_len); 11365 11366 return 0; 11367 } 11368 11369 /** 11370 * i40e_ndo_fdb_add - add an entry to the hardware database 11371 * @ndm: the input from the stack 11372 * @tb: pointer to array of nladdr (unused) 11373 * @dev: the net device pointer 11374 * @addr: the MAC address entry being added 11375 * @flags: instructions from stack about fdb operation 11376 */ 11377 static int i40e_ndo_fdb_add(struct ndmsg *ndm, struct nlattr *tb[], 11378 struct net_device *dev, 11379 const unsigned char *addr, u16 vid, 11380 u16 flags) 11381 { 11382 struct i40e_netdev_priv *np = netdev_priv(dev); 11383 struct i40e_pf *pf = np->vsi->back; 11384 int err = 0; 11385 11386 if (!(pf->flags & I40E_FLAG_SRIOV_ENABLED)) 11387 return -EOPNOTSUPP; 11388 11389 if (vid) { 11390 pr_info("%s: vlans aren't supported yet for dev_uc|mc_add()\n", dev->name); 11391 return -EINVAL; 11392 } 11393 11394 /* Hardware does not support aging addresses so if a 11395 * ndm_state is given only allow permanent addresses 11396 */ 11397 if (ndm->ndm_state && !(ndm->ndm_state & NUD_PERMANENT)) { 11398 netdev_info(dev, "FDB only supports static addresses\n"); 11399 return -EINVAL; 11400 } 11401 11402 if (is_unicast_ether_addr(addr) || is_link_local_ether_addr(addr)) 11403 err = dev_uc_add_excl(dev, addr); 11404 else if (is_multicast_ether_addr(addr)) 11405 err = dev_mc_add_excl(dev, addr); 11406 else 11407 err = -EINVAL; 11408 11409 /* Only return duplicate errors if NLM_F_EXCL is set */ 11410 if (err == -EEXIST && !(flags & NLM_F_EXCL)) 11411 err = 0; 11412 11413 return err; 11414 } 11415 11416 /** 11417 * i40e_ndo_bridge_setlink - Set the hardware bridge mode 11418 * @dev: the netdev being configured 11419 * @nlh: RTNL message 11420 * 11421 * Inserts a new hardware bridge if not already created and 11422 * enables the bridging mode requested (VEB or VEPA). If the 11423 * hardware bridge has already been inserted and the request 11424 * is to change the mode then that requires a PF reset to 11425 * allow rebuild of the components with required hardware 11426 * bridge mode enabled. 11427 * 11428 * Note: expects to be called while under rtnl_lock() 11429 **/ 11430 static int i40e_ndo_bridge_setlink(struct net_device *dev, 11431 struct nlmsghdr *nlh, 11432 u16 flags) 11433 { 11434 struct i40e_netdev_priv *np = netdev_priv(dev); 11435 struct i40e_vsi *vsi = np->vsi; 11436 struct i40e_pf *pf = vsi->back; 11437 struct i40e_veb *veb = NULL; 11438 struct nlattr *attr, *br_spec; 11439 int i, rem; 11440 11441 /* Only for PF VSI for now */ 11442 if (vsi->seid != pf->vsi[pf->lan_vsi]->seid) 11443 return -EOPNOTSUPP; 11444 11445 /* Find the HW bridge for PF VSI */ 11446 for (i = 0; i < I40E_MAX_VEB && !veb; i++) { 11447 if (pf->veb[i] && pf->veb[i]->seid == vsi->uplink_seid) 11448 veb = pf->veb[i]; 11449 } 11450 11451 br_spec = nlmsg_find_attr(nlh, sizeof(struct ifinfomsg), IFLA_AF_SPEC); 11452 11453 nla_for_each_nested(attr, br_spec, rem) { 11454 __u16 mode; 11455 11456 if (nla_type(attr) != IFLA_BRIDGE_MODE) 11457 continue; 11458 11459 mode = nla_get_u16(attr); 11460 if ((mode != BRIDGE_MODE_VEPA) && 11461 (mode != BRIDGE_MODE_VEB)) 11462 return -EINVAL; 11463 11464 /* Insert a new HW bridge */ 11465 if (!veb) { 11466 veb = i40e_veb_setup(pf, 0, vsi->uplink_seid, vsi->seid, 11467 vsi->tc_config.enabled_tc); 11468 if (veb) { 11469 veb->bridge_mode = mode; 11470 i40e_config_bridge_mode(veb); 11471 } else { 11472 /* No Bridge HW offload available */ 11473 return -ENOENT; 11474 } 11475 break; 11476 } else if (mode != veb->bridge_mode) { 11477 /* Existing HW bridge but different mode needs reset */ 11478 veb->bridge_mode = mode; 11479 /* TODO: If no VFs or VMDq VSIs, disallow VEB mode */ 11480 if (mode == BRIDGE_MODE_VEB) 11481 pf->flags |= I40E_FLAG_VEB_MODE_ENABLED; 11482 else 11483 pf->flags &= ~I40E_FLAG_VEB_MODE_ENABLED; 11484 i40e_do_reset(pf, I40E_PF_RESET_FLAG, true); 11485 break; 11486 } 11487 } 11488 11489 return 0; 11490 } 11491 11492 /** 11493 * i40e_ndo_bridge_getlink - Get the hardware bridge mode 11494 * @skb: skb buff 11495 * @pid: process id 11496 * @seq: RTNL message seq # 11497 * @dev: the netdev being configured 11498 * @filter_mask: unused 11499 * @nlflags: netlink flags passed in 11500 * 11501 * Return the mode in which the hardware bridge is operating in 11502 * i.e VEB or VEPA. 11503 **/ 11504 static int i40e_ndo_bridge_getlink(struct sk_buff *skb, u32 pid, u32 seq, 11505 struct net_device *dev, 11506 u32 __always_unused filter_mask, 11507 int nlflags) 11508 { 11509 struct i40e_netdev_priv *np = netdev_priv(dev); 11510 struct i40e_vsi *vsi = np->vsi; 11511 struct i40e_pf *pf = vsi->back; 11512 struct i40e_veb *veb = NULL; 11513 int i; 11514 11515 /* Only for PF VSI for now */ 11516 if (vsi->seid != pf->vsi[pf->lan_vsi]->seid) 11517 return -EOPNOTSUPP; 11518 11519 /* Find the HW bridge for the PF VSI */ 11520 for (i = 0; i < I40E_MAX_VEB && !veb; i++) { 11521 if (pf->veb[i] && pf->veb[i]->seid == vsi->uplink_seid) 11522 veb = pf->veb[i]; 11523 } 11524 11525 if (!veb) 11526 return 0; 11527 11528 return ndo_dflt_bridge_getlink(skb, pid, seq, dev, veb->bridge_mode, 11529 0, 0, nlflags, filter_mask, NULL); 11530 } 11531 11532 /** 11533 * i40e_features_check - Validate encapsulated packet conforms to limits 11534 * @skb: skb buff 11535 * @dev: This physical port's netdev 11536 * @features: Offload features that the stack believes apply 11537 **/ 11538 static netdev_features_t i40e_features_check(struct sk_buff *skb, 11539 struct net_device *dev, 11540 netdev_features_t features) 11541 { 11542 size_t len; 11543 11544 /* No point in doing any of this if neither checksum nor GSO are 11545 * being requested for this frame. We can rule out both by just 11546 * checking for CHECKSUM_PARTIAL 11547 */ 11548 if (skb->ip_summed != CHECKSUM_PARTIAL) 11549 return features; 11550 11551 /* We cannot support GSO if the MSS is going to be less than 11552 * 64 bytes. If it is then we need to drop support for GSO. 11553 */ 11554 if (skb_is_gso(skb) && (skb_shinfo(skb)->gso_size < 64)) 11555 features &= ~NETIF_F_GSO_MASK; 11556 11557 /* MACLEN can support at most 63 words */ 11558 len = skb_network_header(skb) - skb->data; 11559 if (len & ~(63 * 2)) 11560 goto out_err; 11561 11562 /* IPLEN and EIPLEN can support at most 127 dwords */ 11563 len = skb_transport_header(skb) - skb_network_header(skb); 11564 if (len & ~(127 * 4)) 11565 goto out_err; 11566 11567 if (skb->encapsulation) { 11568 /* L4TUNLEN can support 127 words */ 11569 len = skb_inner_network_header(skb) - skb_transport_header(skb); 11570 if (len & ~(127 * 2)) 11571 goto out_err; 11572 11573 /* IPLEN can support at most 127 dwords */ 11574 len = skb_inner_transport_header(skb) - 11575 skb_inner_network_header(skb); 11576 if (len & ~(127 * 4)) 11577 goto out_err; 11578 } 11579 11580 /* No need to validate L4LEN as TCP is the only protocol with a 11581 * a flexible value and we support all possible values supported 11582 * by TCP, which is at most 15 dwords 11583 */ 11584 11585 return features; 11586 out_err: 11587 return features & ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK); 11588 } 11589 11590 /** 11591 * i40e_xdp_setup - add/remove an XDP program 11592 * @vsi: VSI to changed 11593 * @prog: XDP program 11594 **/ 11595 static int i40e_xdp_setup(struct i40e_vsi *vsi, 11596 struct bpf_prog *prog) 11597 { 11598 int frame_size = vsi->netdev->mtu + ETH_HLEN + ETH_FCS_LEN + VLAN_HLEN; 11599 struct i40e_pf *pf = vsi->back; 11600 struct bpf_prog *old_prog; 11601 bool need_reset; 11602 int i; 11603 11604 /* Don't allow frames that span over multiple buffers */ 11605 if (frame_size > vsi->rx_buf_len) 11606 return -EINVAL; 11607 11608 if (!i40e_enabled_xdp_vsi(vsi) && !prog) 11609 return 0; 11610 11611 /* When turning XDP on->off/off->on we reset and rebuild the rings. */ 11612 need_reset = (i40e_enabled_xdp_vsi(vsi) != !!prog); 11613 11614 if (need_reset) 11615 i40e_prep_for_reset(pf, true); 11616 11617 old_prog = xchg(&vsi->xdp_prog, prog); 11618 11619 if (need_reset) 11620 i40e_reset_and_rebuild(pf, true, true); 11621 11622 for (i = 0; i < vsi->num_queue_pairs; i++) 11623 WRITE_ONCE(vsi->rx_rings[i]->xdp_prog, vsi->xdp_prog); 11624 11625 if (old_prog) 11626 bpf_prog_put(old_prog); 11627 11628 return 0; 11629 } 11630 11631 /** 11632 * i40e_xdp - implements ndo_bpf for i40e 11633 * @dev: netdevice 11634 * @xdp: XDP command 11635 **/ 11636 static int i40e_xdp(struct net_device *dev, 11637 struct netdev_bpf *xdp) 11638 { 11639 struct i40e_netdev_priv *np = netdev_priv(dev); 11640 struct i40e_vsi *vsi = np->vsi; 11641 11642 if (vsi->type != I40E_VSI_MAIN) 11643 return -EINVAL; 11644 11645 switch (xdp->command) { 11646 case XDP_SETUP_PROG: 11647 return i40e_xdp_setup(vsi, xdp->prog); 11648 case XDP_QUERY_PROG: 11649 xdp->prog_attached = i40e_enabled_xdp_vsi(vsi); 11650 xdp->prog_id = vsi->xdp_prog ? vsi->xdp_prog->aux->id : 0; 11651 return 0; 11652 default: 11653 return -EINVAL; 11654 } 11655 } 11656 11657 static const struct net_device_ops i40e_netdev_ops = { 11658 .ndo_open = i40e_open, 11659 .ndo_stop = i40e_close, 11660 .ndo_start_xmit = i40e_lan_xmit_frame, 11661 .ndo_get_stats64 = i40e_get_netdev_stats_struct, 11662 .ndo_set_rx_mode = i40e_set_rx_mode, 11663 .ndo_validate_addr = eth_validate_addr, 11664 .ndo_set_mac_address = i40e_set_mac, 11665 .ndo_change_mtu = i40e_change_mtu, 11666 .ndo_do_ioctl = i40e_ioctl, 11667 .ndo_tx_timeout = i40e_tx_timeout, 11668 .ndo_vlan_rx_add_vid = i40e_vlan_rx_add_vid, 11669 .ndo_vlan_rx_kill_vid = i40e_vlan_rx_kill_vid, 11670 #ifdef CONFIG_NET_POLL_CONTROLLER 11671 .ndo_poll_controller = i40e_netpoll, 11672 #endif 11673 .ndo_setup_tc = __i40e_setup_tc, 11674 .ndo_set_features = i40e_set_features, 11675 .ndo_set_vf_mac = i40e_ndo_set_vf_mac, 11676 .ndo_set_vf_vlan = i40e_ndo_set_vf_port_vlan, 11677 .ndo_set_vf_rate = i40e_ndo_set_vf_bw, 11678 .ndo_get_vf_config = i40e_ndo_get_vf_config, 11679 .ndo_set_vf_link_state = i40e_ndo_set_vf_link_state, 11680 .ndo_set_vf_spoofchk = i40e_ndo_set_vf_spoofchk, 11681 .ndo_set_vf_trust = i40e_ndo_set_vf_trust, 11682 .ndo_udp_tunnel_add = i40e_udp_tunnel_add, 11683 .ndo_udp_tunnel_del = i40e_udp_tunnel_del, 11684 .ndo_get_phys_port_id = i40e_get_phys_port_id, 11685 .ndo_fdb_add = i40e_ndo_fdb_add, 11686 .ndo_features_check = i40e_features_check, 11687 .ndo_bridge_getlink = i40e_ndo_bridge_getlink, 11688 .ndo_bridge_setlink = i40e_ndo_bridge_setlink, 11689 .ndo_bpf = i40e_xdp, 11690 }; 11691 11692 /** 11693 * i40e_config_netdev - Setup the netdev flags 11694 * @vsi: the VSI being configured 11695 * 11696 * Returns 0 on success, negative value on failure 11697 **/ 11698 static int i40e_config_netdev(struct i40e_vsi *vsi) 11699 { 11700 struct i40e_pf *pf = vsi->back; 11701 struct i40e_hw *hw = &pf->hw; 11702 struct i40e_netdev_priv *np; 11703 struct net_device *netdev; 11704 u8 broadcast[ETH_ALEN]; 11705 u8 mac_addr[ETH_ALEN]; 11706 int etherdev_size; 11707 netdev_features_t hw_enc_features; 11708 netdev_features_t hw_features; 11709 11710 etherdev_size = sizeof(struct i40e_netdev_priv); 11711 netdev = alloc_etherdev_mq(etherdev_size, vsi->alloc_queue_pairs); 11712 if (!netdev) 11713 return -ENOMEM; 11714 11715 vsi->netdev = netdev; 11716 np = netdev_priv(netdev); 11717 np->vsi = vsi; 11718 11719 hw_enc_features = NETIF_F_SG | 11720 NETIF_F_IP_CSUM | 11721 NETIF_F_IPV6_CSUM | 11722 NETIF_F_HIGHDMA | 11723 NETIF_F_SOFT_FEATURES | 11724 NETIF_F_TSO | 11725 NETIF_F_TSO_ECN | 11726 NETIF_F_TSO6 | 11727 NETIF_F_GSO_GRE | 11728 NETIF_F_GSO_GRE_CSUM | 11729 NETIF_F_GSO_PARTIAL | 11730 NETIF_F_GSO_UDP_TUNNEL | 11731 NETIF_F_GSO_UDP_TUNNEL_CSUM | 11732 NETIF_F_SCTP_CRC | 11733 NETIF_F_RXHASH | 11734 NETIF_F_RXCSUM | 11735 0; 11736 11737 if (!(pf->hw_features & I40E_HW_OUTER_UDP_CSUM_CAPABLE)) 11738 netdev->gso_partial_features |= NETIF_F_GSO_UDP_TUNNEL_CSUM; 11739 11740 netdev->gso_partial_features |= NETIF_F_GSO_GRE_CSUM; 11741 11742 netdev->hw_enc_features |= hw_enc_features; 11743 11744 /* record features VLANs can make use of */ 11745 netdev->vlan_features |= hw_enc_features | NETIF_F_TSO_MANGLEID; 11746 11747 if (!(pf->flags & I40E_FLAG_MFP_ENABLED)) 11748 netdev->hw_features |= NETIF_F_NTUPLE | NETIF_F_HW_TC; 11749 11750 hw_features = hw_enc_features | 11751 NETIF_F_HW_VLAN_CTAG_TX | 11752 NETIF_F_HW_VLAN_CTAG_RX; 11753 11754 netdev->hw_features |= hw_features; 11755 11756 netdev->features |= hw_features | NETIF_F_HW_VLAN_CTAG_FILTER; 11757 netdev->hw_enc_features |= NETIF_F_TSO_MANGLEID; 11758 11759 if (vsi->type == I40E_VSI_MAIN) { 11760 SET_NETDEV_DEV(netdev, &pf->pdev->dev); 11761 ether_addr_copy(mac_addr, hw->mac.perm_addr); 11762 /* The following steps are necessary for two reasons. First, 11763 * some older NVM configurations load a default MAC-VLAN 11764 * filter that will accept any tagged packet, and we want to 11765 * replace this with a normal filter. Additionally, it is 11766 * possible our MAC address was provided by the platform using 11767 * Open Firmware or similar. 11768 * 11769 * Thus, we need to remove the default filter and install one 11770 * specific to the MAC address. 11771 */ 11772 i40e_rm_default_mac_filter(vsi, mac_addr); 11773 spin_lock_bh(&vsi->mac_filter_hash_lock); 11774 i40e_add_mac_filter(vsi, mac_addr); 11775 spin_unlock_bh(&vsi->mac_filter_hash_lock); 11776 } else { 11777 /* Relate the VSI_VMDQ name to the VSI_MAIN name. Note that we 11778 * are still limited by IFNAMSIZ, but we're adding 'v%d\0' to 11779 * the end, which is 4 bytes long, so force truncation of the 11780 * original name by IFNAMSIZ - 4 11781 */ 11782 snprintf(netdev->name, IFNAMSIZ, "%.*sv%%d", 11783 IFNAMSIZ - 4, 11784 pf->vsi[pf->lan_vsi]->netdev->name); 11785 random_ether_addr(mac_addr); 11786 11787 spin_lock_bh(&vsi->mac_filter_hash_lock); 11788 i40e_add_mac_filter(vsi, mac_addr); 11789 spin_unlock_bh(&vsi->mac_filter_hash_lock); 11790 } 11791 11792 /* Add the broadcast filter so that we initially will receive 11793 * broadcast packets. Note that when a new VLAN is first added the 11794 * driver will convert all filters marked I40E_VLAN_ANY into VLAN 11795 * specific filters as part of transitioning into "vlan" operation. 11796 * When more VLANs are added, the driver will copy each existing MAC 11797 * filter and add it for the new VLAN. 11798 * 11799 * Broadcast filters are handled specially by 11800 * i40e_sync_filters_subtask, as the driver must to set the broadcast 11801 * promiscuous bit instead of adding this directly as a MAC/VLAN 11802 * filter. The subtask will update the correct broadcast promiscuous 11803 * bits as VLANs become active or inactive. 11804 */ 11805 eth_broadcast_addr(broadcast); 11806 spin_lock_bh(&vsi->mac_filter_hash_lock); 11807 i40e_add_mac_filter(vsi, broadcast); 11808 spin_unlock_bh(&vsi->mac_filter_hash_lock); 11809 11810 ether_addr_copy(netdev->dev_addr, mac_addr); 11811 ether_addr_copy(netdev->perm_addr, mac_addr); 11812 11813 netdev->priv_flags |= IFF_UNICAST_FLT; 11814 netdev->priv_flags |= IFF_SUPP_NOFCS; 11815 /* Setup netdev TC information */ 11816 i40e_vsi_config_netdev_tc(vsi, vsi->tc_config.enabled_tc); 11817 11818 netdev->netdev_ops = &i40e_netdev_ops; 11819 netdev->watchdog_timeo = 5 * HZ; 11820 i40e_set_ethtool_ops(netdev); 11821 11822 /* MTU range: 68 - 9706 */ 11823 netdev->min_mtu = ETH_MIN_MTU; 11824 netdev->max_mtu = I40E_MAX_RXBUFFER - I40E_PACKET_HDR_PAD; 11825 11826 return 0; 11827 } 11828 11829 /** 11830 * i40e_vsi_delete - Delete a VSI from the switch 11831 * @vsi: the VSI being removed 11832 * 11833 * Returns 0 on success, negative value on failure 11834 **/ 11835 static void i40e_vsi_delete(struct i40e_vsi *vsi) 11836 { 11837 /* remove default VSI is not allowed */ 11838 if (vsi == vsi->back->vsi[vsi->back->lan_vsi]) 11839 return; 11840 11841 i40e_aq_delete_element(&vsi->back->hw, vsi->seid, NULL); 11842 } 11843 11844 /** 11845 * i40e_is_vsi_uplink_mode_veb - Check if the VSI's uplink bridge mode is VEB 11846 * @vsi: the VSI being queried 11847 * 11848 * Returns 1 if HW bridge mode is VEB and return 0 in case of VEPA mode 11849 **/ 11850 int i40e_is_vsi_uplink_mode_veb(struct i40e_vsi *vsi) 11851 { 11852 struct i40e_veb *veb; 11853 struct i40e_pf *pf = vsi->back; 11854 11855 /* Uplink is not a bridge so default to VEB */ 11856 if (vsi->veb_idx == I40E_NO_VEB) 11857 return 1; 11858 11859 veb = pf->veb[vsi->veb_idx]; 11860 if (!veb) { 11861 dev_info(&pf->pdev->dev, 11862 "There is no veb associated with the bridge\n"); 11863 return -ENOENT; 11864 } 11865 11866 /* Uplink is a bridge in VEPA mode */ 11867 if (veb->bridge_mode & BRIDGE_MODE_VEPA) { 11868 return 0; 11869 } else { 11870 /* Uplink is a bridge in VEB mode */ 11871 return 1; 11872 } 11873 11874 /* VEPA is now default bridge, so return 0 */ 11875 return 0; 11876 } 11877 11878 /** 11879 * i40e_add_vsi - Add a VSI to the switch 11880 * @vsi: the VSI being configured 11881 * 11882 * This initializes a VSI context depending on the VSI type to be added and 11883 * passes it down to the add_vsi aq command. 11884 **/ 11885 static int i40e_add_vsi(struct i40e_vsi *vsi) 11886 { 11887 int ret = -ENODEV; 11888 struct i40e_pf *pf = vsi->back; 11889 struct i40e_hw *hw = &pf->hw; 11890 struct i40e_vsi_context ctxt; 11891 struct i40e_mac_filter *f; 11892 struct hlist_node *h; 11893 int bkt; 11894 11895 u8 enabled_tc = 0x1; /* TC0 enabled */ 11896 int f_count = 0; 11897 11898 memset(&ctxt, 0, sizeof(ctxt)); 11899 switch (vsi->type) { 11900 case I40E_VSI_MAIN: 11901 /* The PF's main VSI is already setup as part of the 11902 * device initialization, so we'll not bother with 11903 * the add_vsi call, but we will retrieve the current 11904 * VSI context. 11905 */ 11906 ctxt.seid = pf->main_vsi_seid; 11907 ctxt.pf_num = pf->hw.pf_id; 11908 ctxt.vf_num = 0; 11909 ret = i40e_aq_get_vsi_params(&pf->hw, &ctxt, NULL); 11910 ctxt.flags = I40E_AQ_VSI_TYPE_PF; 11911 if (ret) { 11912 dev_info(&pf->pdev->dev, 11913 "couldn't get PF vsi config, err %s aq_err %s\n", 11914 i40e_stat_str(&pf->hw, ret), 11915 i40e_aq_str(&pf->hw, 11916 pf->hw.aq.asq_last_status)); 11917 return -ENOENT; 11918 } 11919 vsi->info = ctxt.info; 11920 vsi->info.valid_sections = 0; 11921 11922 vsi->seid = ctxt.seid; 11923 vsi->id = ctxt.vsi_number; 11924 11925 enabled_tc = i40e_pf_get_tc_map(pf); 11926 11927 /* Source pruning is enabled by default, so the flag is 11928 * negative logic - if it's set, we need to fiddle with 11929 * the VSI to disable source pruning. 11930 */ 11931 if (pf->flags & I40E_FLAG_SOURCE_PRUNING_DISABLED) { 11932 memset(&ctxt, 0, sizeof(ctxt)); 11933 ctxt.seid = pf->main_vsi_seid; 11934 ctxt.pf_num = pf->hw.pf_id; 11935 ctxt.vf_num = 0; 11936 ctxt.info.valid_sections |= 11937 cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID); 11938 ctxt.info.switch_id = 11939 cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_LOCAL_LB); 11940 ret = i40e_aq_update_vsi_params(hw, &ctxt, NULL); 11941 if (ret) { 11942 dev_info(&pf->pdev->dev, 11943 "update vsi failed, err %s aq_err %s\n", 11944 i40e_stat_str(&pf->hw, ret), 11945 i40e_aq_str(&pf->hw, 11946 pf->hw.aq.asq_last_status)); 11947 ret = -ENOENT; 11948 goto err; 11949 } 11950 } 11951 11952 /* MFP mode setup queue map and update VSI */ 11953 if ((pf->flags & I40E_FLAG_MFP_ENABLED) && 11954 !(pf->hw.func_caps.iscsi)) { /* NIC type PF */ 11955 memset(&ctxt, 0, sizeof(ctxt)); 11956 ctxt.seid = pf->main_vsi_seid; 11957 ctxt.pf_num = pf->hw.pf_id; 11958 ctxt.vf_num = 0; 11959 i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, false); 11960 ret = i40e_aq_update_vsi_params(hw, &ctxt, NULL); 11961 if (ret) { 11962 dev_info(&pf->pdev->dev, 11963 "update vsi failed, err %s aq_err %s\n", 11964 i40e_stat_str(&pf->hw, ret), 11965 i40e_aq_str(&pf->hw, 11966 pf->hw.aq.asq_last_status)); 11967 ret = -ENOENT; 11968 goto err; 11969 } 11970 /* update the local VSI info queue map */ 11971 i40e_vsi_update_queue_map(vsi, &ctxt); 11972 vsi->info.valid_sections = 0; 11973 } else { 11974 /* Default/Main VSI is only enabled for TC0 11975 * reconfigure it to enable all TCs that are 11976 * available on the port in SFP mode. 11977 * For MFP case the iSCSI PF would use this 11978 * flow to enable LAN+iSCSI TC. 11979 */ 11980 ret = i40e_vsi_config_tc(vsi, enabled_tc); 11981 if (ret) { 11982 /* Single TC condition is not fatal, 11983 * message and continue 11984 */ 11985 dev_info(&pf->pdev->dev, 11986 "failed to configure TCs for main VSI tc_map 0x%08x, err %s aq_err %s\n", 11987 enabled_tc, 11988 i40e_stat_str(&pf->hw, ret), 11989 i40e_aq_str(&pf->hw, 11990 pf->hw.aq.asq_last_status)); 11991 } 11992 } 11993 break; 11994 11995 case I40E_VSI_FDIR: 11996 ctxt.pf_num = hw->pf_id; 11997 ctxt.vf_num = 0; 11998 ctxt.uplink_seid = vsi->uplink_seid; 11999 ctxt.connection_type = I40E_AQ_VSI_CONN_TYPE_NORMAL; 12000 ctxt.flags = I40E_AQ_VSI_TYPE_PF; 12001 if ((pf->flags & I40E_FLAG_VEB_MODE_ENABLED) && 12002 (i40e_is_vsi_uplink_mode_veb(vsi))) { 12003 ctxt.info.valid_sections |= 12004 cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID); 12005 ctxt.info.switch_id = 12006 cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB); 12007 } 12008 i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, true); 12009 break; 12010 12011 case I40E_VSI_VMDQ2: 12012 ctxt.pf_num = hw->pf_id; 12013 ctxt.vf_num = 0; 12014 ctxt.uplink_seid = vsi->uplink_seid; 12015 ctxt.connection_type = I40E_AQ_VSI_CONN_TYPE_NORMAL; 12016 ctxt.flags = I40E_AQ_VSI_TYPE_VMDQ2; 12017 12018 /* This VSI is connected to VEB so the switch_id 12019 * should be set to zero by default. 12020 */ 12021 if (i40e_is_vsi_uplink_mode_veb(vsi)) { 12022 ctxt.info.valid_sections |= 12023 cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID); 12024 ctxt.info.switch_id = 12025 cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB); 12026 } 12027 12028 /* Setup the VSI tx/rx queue map for TC0 only for now */ 12029 i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, true); 12030 break; 12031 12032 case I40E_VSI_SRIOV: 12033 ctxt.pf_num = hw->pf_id; 12034 ctxt.vf_num = vsi->vf_id + hw->func_caps.vf_base_id; 12035 ctxt.uplink_seid = vsi->uplink_seid; 12036 ctxt.connection_type = I40E_AQ_VSI_CONN_TYPE_NORMAL; 12037 ctxt.flags = I40E_AQ_VSI_TYPE_VF; 12038 12039 /* This VSI is connected to VEB so the switch_id 12040 * should be set to zero by default. 12041 */ 12042 if (i40e_is_vsi_uplink_mode_veb(vsi)) { 12043 ctxt.info.valid_sections |= 12044 cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID); 12045 ctxt.info.switch_id = 12046 cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB); 12047 } 12048 12049 if (vsi->back->flags & I40E_FLAG_IWARP_ENABLED) { 12050 ctxt.info.valid_sections |= 12051 cpu_to_le16(I40E_AQ_VSI_PROP_QUEUE_OPT_VALID); 12052 ctxt.info.queueing_opt_flags |= 12053 (I40E_AQ_VSI_QUE_OPT_TCP_ENA | 12054 I40E_AQ_VSI_QUE_OPT_RSS_LUT_VSI); 12055 } 12056 12057 ctxt.info.valid_sections |= cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID); 12058 ctxt.info.port_vlan_flags |= I40E_AQ_VSI_PVLAN_MODE_ALL; 12059 if (pf->vf[vsi->vf_id].spoofchk) { 12060 ctxt.info.valid_sections |= 12061 cpu_to_le16(I40E_AQ_VSI_PROP_SECURITY_VALID); 12062 ctxt.info.sec_flags |= 12063 (I40E_AQ_VSI_SEC_FLAG_ENABLE_VLAN_CHK | 12064 I40E_AQ_VSI_SEC_FLAG_ENABLE_MAC_CHK); 12065 } 12066 /* Setup the VSI tx/rx queue map for TC0 only for now */ 12067 i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, true); 12068 break; 12069 12070 case I40E_VSI_IWARP: 12071 /* send down message to iWARP */ 12072 break; 12073 12074 default: 12075 return -ENODEV; 12076 } 12077 12078 if (vsi->type != I40E_VSI_MAIN) { 12079 ret = i40e_aq_add_vsi(hw, &ctxt, NULL); 12080 if (ret) { 12081 dev_info(&vsi->back->pdev->dev, 12082 "add vsi failed, err %s aq_err %s\n", 12083 i40e_stat_str(&pf->hw, ret), 12084 i40e_aq_str(&pf->hw, 12085 pf->hw.aq.asq_last_status)); 12086 ret = -ENOENT; 12087 goto err; 12088 } 12089 vsi->info = ctxt.info; 12090 vsi->info.valid_sections = 0; 12091 vsi->seid = ctxt.seid; 12092 vsi->id = ctxt.vsi_number; 12093 } 12094 12095 vsi->active_filters = 0; 12096 clear_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state); 12097 spin_lock_bh(&vsi->mac_filter_hash_lock); 12098 /* If macvlan filters already exist, force them to get loaded */ 12099 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) { 12100 f->state = I40E_FILTER_NEW; 12101 f_count++; 12102 } 12103 spin_unlock_bh(&vsi->mac_filter_hash_lock); 12104 12105 if (f_count) { 12106 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED; 12107 pf->flags |= I40E_FLAG_FILTER_SYNC; 12108 } 12109 12110 /* Update VSI BW information */ 12111 ret = i40e_vsi_get_bw_info(vsi); 12112 if (ret) { 12113 dev_info(&pf->pdev->dev, 12114 "couldn't get vsi bw info, err %s aq_err %s\n", 12115 i40e_stat_str(&pf->hw, ret), 12116 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 12117 /* VSI is already added so not tearing that up */ 12118 ret = 0; 12119 } 12120 12121 err: 12122 return ret; 12123 } 12124 12125 /** 12126 * i40e_vsi_release - Delete a VSI and free its resources 12127 * @vsi: the VSI being removed 12128 * 12129 * Returns 0 on success or < 0 on error 12130 **/ 12131 int i40e_vsi_release(struct i40e_vsi *vsi) 12132 { 12133 struct i40e_mac_filter *f; 12134 struct hlist_node *h; 12135 struct i40e_veb *veb = NULL; 12136 struct i40e_pf *pf; 12137 u16 uplink_seid; 12138 int i, n, bkt; 12139 12140 pf = vsi->back; 12141 12142 /* release of a VEB-owner or last VSI is not allowed */ 12143 if (vsi->flags & I40E_VSI_FLAG_VEB_OWNER) { 12144 dev_info(&pf->pdev->dev, "VSI %d has existing VEB %d\n", 12145 vsi->seid, vsi->uplink_seid); 12146 return -ENODEV; 12147 } 12148 if (vsi == pf->vsi[pf->lan_vsi] && 12149 !test_bit(__I40E_DOWN, pf->state)) { 12150 dev_info(&pf->pdev->dev, "Can't remove PF VSI\n"); 12151 return -ENODEV; 12152 } 12153 12154 uplink_seid = vsi->uplink_seid; 12155 if (vsi->type != I40E_VSI_SRIOV) { 12156 if (vsi->netdev_registered) { 12157 vsi->netdev_registered = false; 12158 if (vsi->netdev) { 12159 /* results in a call to i40e_close() */ 12160 unregister_netdev(vsi->netdev); 12161 } 12162 } else { 12163 i40e_vsi_close(vsi); 12164 } 12165 i40e_vsi_disable_irq(vsi); 12166 } 12167 12168 spin_lock_bh(&vsi->mac_filter_hash_lock); 12169 12170 /* clear the sync flag on all filters */ 12171 if (vsi->netdev) { 12172 __dev_uc_unsync(vsi->netdev, NULL); 12173 __dev_mc_unsync(vsi->netdev, NULL); 12174 } 12175 12176 /* make sure any remaining filters are marked for deletion */ 12177 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) 12178 __i40e_del_filter(vsi, f); 12179 12180 spin_unlock_bh(&vsi->mac_filter_hash_lock); 12181 12182 i40e_sync_vsi_filters(vsi); 12183 12184 i40e_vsi_delete(vsi); 12185 i40e_vsi_free_q_vectors(vsi); 12186 if (vsi->netdev) { 12187 free_netdev(vsi->netdev); 12188 vsi->netdev = NULL; 12189 } 12190 i40e_vsi_clear_rings(vsi); 12191 i40e_vsi_clear(vsi); 12192 12193 /* If this was the last thing on the VEB, except for the 12194 * controlling VSI, remove the VEB, which puts the controlling 12195 * VSI onto the next level down in the switch. 12196 * 12197 * Well, okay, there's one more exception here: don't remove 12198 * the orphan VEBs yet. We'll wait for an explicit remove request 12199 * from up the network stack. 12200 */ 12201 for (n = 0, i = 0; i < pf->num_alloc_vsi; i++) { 12202 if (pf->vsi[i] && 12203 pf->vsi[i]->uplink_seid == uplink_seid && 12204 (pf->vsi[i]->flags & I40E_VSI_FLAG_VEB_OWNER) == 0) { 12205 n++; /* count the VSIs */ 12206 } 12207 } 12208 for (i = 0; i < I40E_MAX_VEB; i++) { 12209 if (!pf->veb[i]) 12210 continue; 12211 if (pf->veb[i]->uplink_seid == uplink_seid) 12212 n++; /* count the VEBs */ 12213 if (pf->veb[i]->seid == uplink_seid) 12214 veb = pf->veb[i]; 12215 } 12216 if (n == 0 && veb && veb->uplink_seid != 0) 12217 i40e_veb_release(veb); 12218 12219 return 0; 12220 } 12221 12222 /** 12223 * i40e_vsi_setup_vectors - Set up the q_vectors for the given VSI 12224 * @vsi: ptr to the VSI 12225 * 12226 * This should only be called after i40e_vsi_mem_alloc() which allocates the 12227 * corresponding SW VSI structure and initializes num_queue_pairs for the 12228 * newly allocated VSI. 12229 * 12230 * Returns 0 on success or negative on failure 12231 **/ 12232 static int i40e_vsi_setup_vectors(struct i40e_vsi *vsi) 12233 { 12234 int ret = -ENOENT; 12235 struct i40e_pf *pf = vsi->back; 12236 12237 if (vsi->q_vectors[0]) { 12238 dev_info(&pf->pdev->dev, "VSI %d has existing q_vectors\n", 12239 vsi->seid); 12240 return -EEXIST; 12241 } 12242 12243 if (vsi->base_vector) { 12244 dev_info(&pf->pdev->dev, "VSI %d has non-zero base vector %d\n", 12245 vsi->seid, vsi->base_vector); 12246 return -EEXIST; 12247 } 12248 12249 ret = i40e_vsi_alloc_q_vectors(vsi); 12250 if (ret) { 12251 dev_info(&pf->pdev->dev, 12252 "failed to allocate %d q_vector for VSI %d, ret=%d\n", 12253 vsi->num_q_vectors, vsi->seid, ret); 12254 vsi->num_q_vectors = 0; 12255 goto vector_setup_out; 12256 } 12257 12258 /* In Legacy mode, we do not have to get any other vector since we 12259 * piggyback on the misc/ICR0 for queue interrupts. 12260 */ 12261 if (!(pf->flags & I40E_FLAG_MSIX_ENABLED)) 12262 return ret; 12263 if (vsi->num_q_vectors) 12264 vsi->base_vector = i40e_get_lump(pf, pf->irq_pile, 12265 vsi->num_q_vectors, vsi->idx); 12266 if (vsi->base_vector < 0) { 12267 dev_info(&pf->pdev->dev, 12268 "failed to get tracking for %d vectors for VSI %d, err=%d\n", 12269 vsi->num_q_vectors, vsi->seid, vsi->base_vector); 12270 i40e_vsi_free_q_vectors(vsi); 12271 ret = -ENOENT; 12272 goto vector_setup_out; 12273 } 12274 12275 vector_setup_out: 12276 return ret; 12277 } 12278 12279 /** 12280 * i40e_vsi_reinit_setup - return and reallocate resources for a VSI 12281 * @vsi: pointer to the vsi. 12282 * 12283 * This re-allocates a vsi's queue resources. 12284 * 12285 * Returns pointer to the successfully allocated and configured VSI sw struct 12286 * on success, otherwise returns NULL on failure. 12287 **/ 12288 static struct i40e_vsi *i40e_vsi_reinit_setup(struct i40e_vsi *vsi) 12289 { 12290 u16 alloc_queue_pairs; 12291 struct i40e_pf *pf; 12292 u8 enabled_tc; 12293 int ret; 12294 12295 if (!vsi) 12296 return NULL; 12297 12298 pf = vsi->back; 12299 12300 i40e_put_lump(pf->qp_pile, vsi->base_queue, vsi->idx); 12301 i40e_vsi_clear_rings(vsi); 12302 12303 i40e_vsi_free_arrays(vsi, false); 12304 i40e_set_num_rings_in_vsi(vsi); 12305 ret = i40e_vsi_alloc_arrays(vsi, false); 12306 if (ret) 12307 goto err_vsi; 12308 12309 alloc_queue_pairs = vsi->alloc_queue_pairs * 12310 (i40e_enabled_xdp_vsi(vsi) ? 2 : 1); 12311 12312 ret = i40e_get_lump(pf, pf->qp_pile, alloc_queue_pairs, vsi->idx); 12313 if (ret < 0) { 12314 dev_info(&pf->pdev->dev, 12315 "failed to get tracking for %d queues for VSI %d err %d\n", 12316 alloc_queue_pairs, vsi->seid, ret); 12317 goto err_vsi; 12318 } 12319 vsi->base_queue = ret; 12320 12321 /* Update the FW view of the VSI. Force a reset of TC and queue 12322 * layout configurations. 12323 */ 12324 enabled_tc = pf->vsi[pf->lan_vsi]->tc_config.enabled_tc; 12325 pf->vsi[pf->lan_vsi]->tc_config.enabled_tc = 0; 12326 pf->vsi[pf->lan_vsi]->seid = pf->main_vsi_seid; 12327 i40e_vsi_config_tc(pf->vsi[pf->lan_vsi], enabled_tc); 12328 if (vsi->type == I40E_VSI_MAIN) 12329 i40e_rm_default_mac_filter(vsi, pf->hw.mac.perm_addr); 12330 12331 /* assign it some queues */ 12332 ret = i40e_alloc_rings(vsi); 12333 if (ret) 12334 goto err_rings; 12335 12336 /* map all of the rings to the q_vectors */ 12337 i40e_vsi_map_rings_to_vectors(vsi); 12338 return vsi; 12339 12340 err_rings: 12341 i40e_vsi_free_q_vectors(vsi); 12342 if (vsi->netdev_registered) { 12343 vsi->netdev_registered = false; 12344 unregister_netdev(vsi->netdev); 12345 free_netdev(vsi->netdev); 12346 vsi->netdev = NULL; 12347 } 12348 i40e_aq_delete_element(&pf->hw, vsi->seid, NULL); 12349 err_vsi: 12350 i40e_vsi_clear(vsi); 12351 return NULL; 12352 } 12353 12354 /** 12355 * i40e_vsi_setup - Set up a VSI by a given type 12356 * @pf: board private structure 12357 * @type: VSI type 12358 * @uplink_seid: the switch element to link to 12359 * @param1: usage depends upon VSI type. For VF types, indicates VF id 12360 * 12361 * This allocates the sw VSI structure and its queue resources, then add a VSI 12362 * to the identified VEB. 12363 * 12364 * Returns pointer to the successfully allocated and configure VSI sw struct on 12365 * success, otherwise returns NULL on failure. 12366 **/ 12367 struct i40e_vsi *i40e_vsi_setup(struct i40e_pf *pf, u8 type, 12368 u16 uplink_seid, u32 param1) 12369 { 12370 struct i40e_vsi *vsi = NULL; 12371 struct i40e_veb *veb = NULL; 12372 u16 alloc_queue_pairs; 12373 int ret, i; 12374 int v_idx; 12375 12376 /* The requested uplink_seid must be either 12377 * - the PF's port seid 12378 * no VEB is needed because this is the PF 12379 * or this is a Flow Director special case VSI 12380 * - seid of an existing VEB 12381 * - seid of a VSI that owns an existing VEB 12382 * - seid of a VSI that doesn't own a VEB 12383 * a new VEB is created and the VSI becomes the owner 12384 * - seid of the PF VSI, which is what creates the first VEB 12385 * this is a special case of the previous 12386 * 12387 * Find which uplink_seid we were given and create a new VEB if needed 12388 */ 12389 for (i = 0; i < I40E_MAX_VEB; i++) { 12390 if (pf->veb[i] && pf->veb[i]->seid == uplink_seid) { 12391 veb = pf->veb[i]; 12392 break; 12393 } 12394 } 12395 12396 if (!veb && uplink_seid != pf->mac_seid) { 12397 12398 for (i = 0; i < pf->num_alloc_vsi; i++) { 12399 if (pf->vsi[i] && pf->vsi[i]->seid == uplink_seid) { 12400 vsi = pf->vsi[i]; 12401 break; 12402 } 12403 } 12404 if (!vsi) { 12405 dev_info(&pf->pdev->dev, "no such uplink_seid %d\n", 12406 uplink_seid); 12407 return NULL; 12408 } 12409 12410 if (vsi->uplink_seid == pf->mac_seid) 12411 veb = i40e_veb_setup(pf, 0, pf->mac_seid, vsi->seid, 12412 vsi->tc_config.enabled_tc); 12413 else if ((vsi->flags & I40E_VSI_FLAG_VEB_OWNER) == 0) 12414 veb = i40e_veb_setup(pf, 0, vsi->uplink_seid, vsi->seid, 12415 vsi->tc_config.enabled_tc); 12416 if (veb) { 12417 if (vsi->seid != pf->vsi[pf->lan_vsi]->seid) { 12418 dev_info(&vsi->back->pdev->dev, 12419 "New VSI creation error, uplink seid of LAN VSI expected.\n"); 12420 return NULL; 12421 } 12422 /* We come up by default in VEPA mode if SRIOV is not 12423 * already enabled, in which case we can't force VEPA 12424 * mode. 12425 */ 12426 if (!(pf->flags & I40E_FLAG_VEB_MODE_ENABLED)) { 12427 veb->bridge_mode = BRIDGE_MODE_VEPA; 12428 pf->flags &= ~I40E_FLAG_VEB_MODE_ENABLED; 12429 } 12430 i40e_config_bridge_mode(veb); 12431 } 12432 for (i = 0; i < I40E_MAX_VEB && !veb; i++) { 12433 if (pf->veb[i] && pf->veb[i]->seid == vsi->uplink_seid) 12434 veb = pf->veb[i]; 12435 } 12436 if (!veb) { 12437 dev_info(&pf->pdev->dev, "couldn't add VEB\n"); 12438 return NULL; 12439 } 12440 12441 vsi->flags |= I40E_VSI_FLAG_VEB_OWNER; 12442 uplink_seid = veb->seid; 12443 } 12444 12445 /* get vsi sw struct */ 12446 v_idx = i40e_vsi_mem_alloc(pf, type); 12447 if (v_idx < 0) 12448 goto err_alloc; 12449 vsi = pf->vsi[v_idx]; 12450 if (!vsi) 12451 goto err_alloc; 12452 vsi->type = type; 12453 vsi->veb_idx = (veb ? veb->idx : I40E_NO_VEB); 12454 12455 if (type == I40E_VSI_MAIN) 12456 pf->lan_vsi = v_idx; 12457 else if (type == I40E_VSI_SRIOV) 12458 vsi->vf_id = param1; 12459 /* assign it some queues */ 12460 alloc_queue_pairs = vsi->alloc_queue_pairs * 12461 (i40e_enabled_xdp_vsi(vsi) ? 2 : 1); 12462 12463 ret = i40e_get_lump(pf, pf->qp_pile, alloc_queue_pairs, vsi->idx); 12464 if (ret < 0) { 12465 dev_info(&pf->pdev->dev, 12466 "failed to get tracking for %d queues for VSI %d err=%d\n", 12467 alloc_queue_pairs, vsi->seid, ret); 12468 goto err_vsi; 12469 } 12470 vsi->base_queue = ret; 12471 12472 /* get a VSI from the hardware */ 12473 vsi->uplink_seid = uplink_seid; 12474 ret = i40e_add_vsi(vsi); 12475 if (ret) 12476 goto err_vsi; 12477 12478 switch (vsi->type) { 12479 /* setup the netdev if needed */ 12480 case I40E_VSI_MAIN: 12481 case I40E_VSI_VMDQ2: 12482 ret = i40e_config_netdev(vsi); 12483 if (ret) 12484 goto err_netdev; 12485 ret = register_netdev(vsi->netdev); 12486 if (ret) 12487 goto err_netdev; 12488 vsi->netdev_registered = true; 12489 netif_carrier_off(vsi->netdev); 12490 #ifdef CONFIG_I40E_DCB 12491 /* Setup DCB netlink interface */ 12492 i40e_dcbnl_setup(vsi); 12493 #endif /* CONFIG_I40E_DCB */ 12494 /* fall through */ 12495 12496 case I40E_VSI_FDIR: 12497 /* set up vectors and rings if needed */ 12498 ret = i40e_vsi_setup_vectors(vsi); 12499 if (ret) 12500 goto err_msix; 12501 12502 ret = i40e_alloc_rings(vsi); 12503 if (ret) 12504 goto err_rings; 12505 12506 /* map all of the rings to the q_vectors */ 12507 i40e_vsi_map_rings_to_vectors(vsi); 12508 12509 i40e_vsi_reset_stats(vsi); 12510 break; 12511 12512 default: 12513 /* no netdev or rings for the other VSI types */ 12514 break; 12515 } 12516 12517 if ((pf->hw_features & I40E_HW_RSS_AQ_CAPABLE) && 12518 (vsi->type == I40E_VSI_VMDQ2)) { 12519 ret = i40e_vsi_config_rss(vsi); 12520 } 12521 return vsi; 12522 12523 err_rings: 12524 i40e_vsi_free_q_vectors(vsi); 12525 err_msix: 12526 if (vsi->netdev_registered) { 12527 vsi->netdev_registered = false; 12528 unregister_netdev(vsi->netdev); 12529 free_netdev(vsi->netdev); 12530 vsi->netdev = NULL; 12531 } 12532 err_netdev: 12533 i40e_aq_delete_element(&pf->hw, vsi->seid, NULL); 12534 err_vsi: 12535 i40e_vsi_clear(vsi); 12536 err_alloc: 12537 return NULL; 12538 } 12539 12540 /** 12541 * i40e_veb_get_bw_info - Query VEB BW information 12542 * @veb: the veb to query 12543 * 12544 * Query the Tx scheduler BW configuration data for given VEB 12545 **/ 12546 static int i40e_veb_get_bw_info(struct i40e_veb *veb) 12547 { 12548 struct i40e_aqc_query_switching_comp_ets_config_resp ets_data; 12549 struct i40e_aqc_query_switching_comp_bw_config_resp bw_data; 12550 struct i40e_pf *pf = veb->pf; 12551 struct i40e_hw *hw = &pf->hw; 12552 u32 tc_bw_max; 12553 int ret = 0; 12554 int i; 12555 12556 ret = i40e_aq_query_switch_comp_bw_config(hw, veb->seid, 12557 &bw_data, NULL); 12558 if (ret) { 12559 dev_info(&pf->pdev->dev, 12560 "query veb bw config failed, err %s aq_err %s\n", 12561 i40e_stat_str(&pf->hw, ret), 12562 i40e_aq_str(&pf->hw, hw->aq.asq_last_status)); 12563 goto out; 12564 } 12565 12566 ret = i40e_aq_query_switch_comp_ets_config(hw, veb->seid, 12567 &ets_data, NULL); 12568 if (ret) { 12569 dev_info(&pf->pdev->dev, 12570 "query veb bw ets config failed, err %s aq_err %s\n", 12571 i40e_stat_str(&pf->hw, ret), 12572 i40e_aq_str(&pf->hw, hw->aq.asq_last_status)); 12573 goto out; 12574 } 12575 12576 veb->bw_limit = le16_to_cpu(ets_data.port_bw_limit); 12577 veb->bw_max_quanta = ets_data.tc_bw_max; 12578 veb->is_abs_credits = bw_data.absolute_credits_enable; 12579 veb->enabled_tc = ets_data.tc_valid_bits; 12580 tc_bw_max = le16_to_cpu(bw_data.tc_bw_max[0]) | 12581 (le16_to_cpu(bw_data.tc_bw_max[1]) << 16); 12582 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 12583 veb->bw_tc_share_credits[i] = bw_data.tc_bw_share_credits[i]; 12584 veb->bw_tc_limit_credits[i] = 12585 le16_to_cpu(bw_data.tc_bw_limits[i]); 12586 veb->bw_tc_max_quanta[i] = ((tc_bw_max >> (i*4)) & 0x7); 12587 } 12588 12589 out: 12590 return ret; 12591 } 12592 12593 /** 12594 * i40e_veb_mem_alloc - Allocates the next available struct veb in the PF 12595 * @pf: board private structure 12596 * 12597 * On error: returns error code (negative) 12598 * On success: returns vsi index in PF (positive) 12599 **/ 12600 static int i40e_veb_mem_alloc(struct i40e_pf *pf) 12601 { 12602 int ret = -ENOENT; 12603 struct i40e_veb *veb; 12604 int i; 12605 12606 /* Need to protect the allocation of switch elements at the PF level */ 12607 mutex_lock(&pf->switch_mutex); 12608 12609 /* VEB list may be fragmented if VEB creation/destruction has 12610 * been happening. We can afford to do a quick scan to look 12611 * for any free slots in the list. 12612 * 12613 * find next empty veb slot, looping back around if necessary 12614 */ 12615 i = 0; 12616 while ((i < I40E_MAX_VEB) && (pf->veb[i] != NULL)) 12617 i++; 12618 if (i >= I40E_MAX_VEB) { 12619 ret = -ENOMEM; 12620 goto err_alloc_veb; /* out of VEB slots! */ 12621 } 12622 12623 veb = kzalloc(sizeof(*veb), GFP_KERNEL); 12624 if (!veb) { 12625 ret = -ENOMEM; 12626 goto err_alloc_veb; 12627 } 12628 veb->pf = pf; 12629 veb->idx = i; 12630 veb->enabled_tc = 1; 12631 12632 pf->veb[i] = veb; 12633 ret = i; 12634 err_alloc_veb: 12635 mutex_unlock(&pf->switch_mutex); 12636 return ret; 12637 } 12638 12639 /** 12640 * i40e_switch_branch_release - Delete a branch of the switch tree 12641 * @branch: where to start deleting 12642 * 12643 * This uses recursion to find the tips of the branch to be 12644 * removed, deleting until we get back to and can delete this VEB. 12645 **/ 12646 static void i40e_switch_branch_release(struct i40e_veb *branch) 12647 { 12648 struct i40e_pf *pf = branch->pf; 12649 u16 branch_seid = branch->seid; 12650 u16 veb_idx = branch->idx; 12651 int i; 12652 12653 /* release any VEBs on this VEB - RECURSION */ 12654 for (i = 0; i < I40E_MAX_VEB; i++) { 12655 if (!pf->veb[i]) 12656 continue; 12657 if (pf->veb[i]->uplink_seid == branch->seid) 12658 i40e_switch_branch_release(pf->veb[i]); 12659 } 12660 12661 /* Release the VSIs on this VEB, but not the owner VSI. 12662 * 12663 * NOTE: Removing the last VSI on a VEB has the SIDE EFFECT of removing 12664 * the VEB itself, so don't use (*branch) after this loop. 12665 */ 12666 for (i = 0; i < pf->num_alloc_vsi; i++) { 12667 if (!pf->vsi[i]) 12668 continue; 12669 if (pf->vsi[i]->uplink_seid == branch_seid && 12670 (pf->vsi[i]->flags & I40E_VSI_FLAG_VEB_OWNER) == 0) { 12671 i40e_vsi_release(pf->vsi[i]); 12672 } 12673 } 12674 12675 /* There's one corner case where the VEB might not have been 12676 * removed, so double check it here and remove it if needed. 12677 * This case happens if the veb was created from the debugfs 12678 * commands and no VSIs were added to it. 12679 */ 12680 if (pf->veb[veb_idx]) 12681 i40e_veb_release(pf->veb[veb_idx]); 12682 } 12683 12684 /** 12685 * i40e_veb_clear - remove veb struct 12686 * @veb: the veb to remove 12687 **/ 12688 static void i40e_veb_clear(struct i40e_veb *veb) 12689 { 12690 if (!veb) 12691 return; 12692 12693 if (veb->pf) { 12694 struct i40e_pf *pf = veb->pf; 12695 12696 mutex_lock(&pf->switch_mutex); 12697 if (pf->veb[veb->idx] == veb) 12698 pf->veb[veb->idx] = NULL; 12699 mutex_unlock(&pf->switch_mutex); 12700 } 12701 12702 kfree(veb); 12703 } 12704 12705 /** 12706 * i40e_veb_release - Delete a VEB and free its resources 12707 * @veb: the VEB being removed 12708 **/ 12709 void i40e_veb_release(struct i40e_veb *veb) 12710 { 12711 struct i40e_vsi *vsi = NULL; 12712 struct i40e_pf *pf; 12713 int i, n = 0; 12714 12715 pf = veb->pf; 12716 12717 /* find the remaining VSI and check for extras */ 12718 for (i = 0; i < pf->num_alloc_vsi; i++) { 12719 if (pf->vsi[i] && pf->vsi[i]->uplink_seid == veb->seid) { 12720 n++; 12721 vsi = pf->vsi[i]; 12722 } 12723 } 12724 if (n != 1) { 12725 dev_info(&pf->pdev->dev, 12726 "can't remove VEB %d with %d VSIs left\n", 12727 veb->seid, n); 12728 return; 12729 } 12730 12731 /* move the remaining VSI to uplink veb */ 12732 vsi->flags &= ~I40E_VSI_FLAG_VEB_OWNER; 12733 if (veb->uplink_seid) { 12734 vsi->uplink_seid = veb->uplink_seid; 12735 if (veb->uplink_seid == pf->mac_seid) 12736 vsi->veb_idx = I40E_NO_VEB; 12737 else 12738 vsi->veb_idx = veb->veb_idx; 12739 } else { 12740 /* floating VEB */ 12741 vsi->uplink_seid = pf->vsi[pf->lan_vsi]->uplink_seid; 12742 vsi->veb_idx = pf->vsi[pf->lan_vsi]->veb_idx; 12743 } 12744 12745 i40e_aq_delete_element(&pf->hw, veb->seid, NULL); 12746 i40e_veb_clear(veb); 12747 } 12748 12749 /** 12750 * i40e_add_veb - create the VEB in the switch 12751 * @veb: the VEB to be instantiated 12752 * @vsi: the controlling VSI 12753 **/ 12754 static int i40e_add_veb(struct i40e_veb *veb, struct i40e_vsi *vsi) 12755 { 12756 struct i40e_pf *pf = veb->pf; 12757 bool enable_stats = !!(pf->flags & I40E_FLAG_VEB_STATS_ENABLED); 12758 int ret; 12759 12760 ret = i40e_aq_add_veb(&pf->hw, veb->uplink_seid, vsi->seid, 12761 veb->enabled_tc, false, 12762 &veb->seid, enable_stats, NULL); 12763 12764 /* get a VEB from the hardware */ 12765 if (ret) { 12766 dev_info(&pf->pdev->dev, 12767 "couldn't add VEB, err %s aq_err %s\n", 12768 i40e_stat_str(&pf->hw, ret), 12769 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 12770 return -EPERM; 12771 } 12772 12773 /* get statistics counter */ 12774 ret = i40e_aq_get_veb_parameters(&pf->hw, veb->seid, NULL, NULL, 12775 &veb->stats_idx, NULL, NULL, NULL); 12776 if (ret) { 12777 dev_info(&pf->pdev->dev, 12778 "couldn't get VEB statistics idx, err %s aq_err %s\n", 12779 i40e_stat_str(&pf->hw, ret), 12780 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 12781 return -EPERM; 12782 } 12783 ret = i40e_veb_get_bw_info(veb); 12784 if (ret) { 12785 dev_info(&pf->pdev->dev, 12786 "couldn't get VEB bw info, err %s aq_err %s\n", 12787 i40e_stat_str(&pf->hw, ret), 12788 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 12789 i40e_aq_delete_element(&pf->hw, veb->seid, NULL); 12790 return -ENOENT; 12791 } 12792 12793 vsi->uplink_seid = veb->seid; 12794 vsi->veb_idx = veb->idx; 12795 vsi->flags |= I40E_VSI_FLAG_VEB_OWNER; 12796 12797 return 0; 12798 } 12799 12800 /** 12801 * i40e_veb_setup - Set up a VEB 12802 * @pf: board private structure 12803 * @flags: VEB setup flags 12804 * @uplink_seid: the switch element to link to 12805 * @vsi_seid: the initial VSI seid 12806 * @enabled_tc: Enabled TC bit-map 12807 * 12808 * This allocates the sw VEB structure and links it into the switch 12809 * It is possible and legal for this to be a duplicate of an already 12810 * existing VEB. It is also possible for both uplink and vsi seids 12811 * to be zero, in order to create a floating VEB. 12812 * 12813 * Returns pointer to the successfully allocated VEB sw struct on 12814 * success, otherwise returns NULL on failure. 12815 **/ 12816 struct i40e_veb *i40e_veb_setup(struct i40e_pf *pf, u16 flags, 12817 u16 uplink_seid, u16 vsi_seid, 12818 u8 enabled_tc) 12819 { 12820 struct i40e_veb *veb, *uplink_veb = NULL; 12821 int vsi_idx, veb_idx; 12822 int ret; 12823 12824 /* if one seid is 0, the other must be 0 to create a floating relay */ 12825 if ((uplink_seid == 0 || vsi_seid == 0) && 12826 (uplink_seid + vsi_seid != 0)) { 12827 dev_info(&pf->pdev->dev, 12828 "one, not both seid's are 0: uplink=%d vsi=%d\n", 12829 uplink_seid, vsi_seid); 12830 return NULL; 12831 } 12832 12833 /* make sure there is such a vsi and uplink */ 12834 for (vsi_idx = 0; vsi_idx < pf->num_alloc_vsi; vsi_idx++) 12835 if (pf->vsi[vsi_idx] && pf->vsi[vsi_idx]->seid == vsi_seid) 12836 break; 12837 if (vsi_idx >= pf->num_alloc_vsi && vsi_seid != 0) { 12838 dev_info(&pf->pdev->dev, "vsi seid %d not found\n", 12839 vsi_seid); 12840 return NULL; 12841 } 12842 12843 if (uplink_seid && uplink_seid != pf->mac_seid) { 12844 for (veb_idx = 0; veb_idx < I40E_MAX_VEB; veb_idx++) { 12845 if (pf->veb[veb_idx] && 12846 pf->veb[veb_idx]->seid == uplink_seid) { 12847 uplink_veb = pf->veb[veb_idx]; 12848 break; 12849 } 12850 } 12851 if (!uplink_veb) { 12852 dev_info(&pf->pdev->dev, 12853 "uplink seid %d not found\n", uplink_seid); 12854 return NULL; 12855 } 12856 } 12857 12858 /* get veb sw struct */ 12859 veb_idx = i40e_veb_mem_alloc(pf); 12860 if (veb_idx < 0) 12861 goto err_alloc; 12862 veb = pf->veb[veb_idx]; 12863 veb->flags = flags; 12864 veb->uplink_seid = uplink_seid; 12865 veb->veb_idx = (uplink_veb ? uplink_veb->idx : I40E_NO_VEB); 12866 veb->enabled_tc = (enabled_tc ? enabled_tc : 0x1); 12867 12868 /* create the VEB in the switch */ 12869 ret = i40e_add_veb(veb, pf->vsi[vsi_idx]); 12870 if (ret) 12871 goto err_veb; 12872 if (vsi_idx == pf->lan_vsi) 12873 pf->lan_veb = veb->idx; 12874 12875 return veb; 12876 12877 err_veb: 12878 i40e_veb_clear(veb); 12879 err_alloc: 12880 return NULL; 12881 } 12882 12883 /** 12884 * i40e_setup_pf_switch_element - set PF vars based on switch type 12885 * @pf: board private structure 12886 * @ele: element we are building info from 12887 * @num_reported: total number of elements 12888 * @printconfig: should we print the contents 12889 * 12890 * helper function to assist in extracting a few useful SEID values. 12891 **/ 12892 static void i40e_setup_pf_switch_element(struct i40e_pf *pf, 12893 struct i40e_aqc_switch_config_element_resp *ele, 12894 u16 num_reported, bool printconfig) 12895 { 12896 u16 downlink_seid = le16_to_cpu(ele->downlink_seid); 12897 u16 uplink_seid = le16_to_cpu(ele->uplink_seid); 12898 u8 element_type = ele->element_type; 12899 u16 seid = le16_to_cpu(ele->seid); 12900 12901 if (printconfig) 12902 dev_info(&pf->pdev->dev, 12903 "type=%d seid=%d uplink=%d downlink=%d\n", 12904 element_type, seid, uplink_seid, downlink_seid); 12905 12906 switch (element_type) { 12907 case I40E_SWITCH_ELEMENT_TYPE_MAC: 12908 pf->mac_seid = seid; 12909 break; 12910 case I40E_SWITCH_ELEMENT_TYPE_VEB: 12911 /* Main VEB? */ 12912 if (uplink_seid != pf->mac_seid) 12913 break; 12914 if (pf->lan_veb == I40E_NO_VEB) { 12915 int v; 12916 12917 /* find existing or else empty VEB */ 12918 for (v = 0; v < I40E_MAX_VEB; v++) { 12919 if (pf->veb[v] && (pf->veb[v]->seid == seid)) { 12920 pf->lan_veb = v; 12921 break; 12922 } 12923 } 12924 if (pf->lan_veb == I40E_NO_VEB) { 12925 v = i40e_veb_mem_alloc(pf); 12926 if (v < 0) 12927 break; 12928 pf->lan_veb = v; 12929 } 12930 } 12931 12932 pf->veb[pf->lan_veb]->seid = seid; 12933 pf->veb[pf->lan_veb]->uplink_seid = pf->mac_seid; 12934 pf->veb[pf->lan_veb]->pf = pf; 12935 pf->veb[pf->lan_veb]->veb_idx = I40E_NO_VEB; 12936 break; 12937 case I40E_SWITCH_ELEMENT_TYPE_VSI: 12938 if (num_reported != 1) 12939 break; 12940 /* This is immediately after a reset so we can assume this is 12941 * the PF's VSI 12942 */ 12943 pf->mac_seid = uplink_seid; 12944 pf->pf_seid = downlink_seid; 12945 pf->main_vsi_seid = seid; 12946 if (printconfig) 12947 dev_info(&pf->pdev->dev, 12948 "pf_seid=%d main_vsi_seid=%d\n", 12949 pf->pf_seid, pf->main_vsi_seid); 12950 break; 12951 case I40E_SWITCH_ELEMENT_TYPE_PF: 12952 case I40E_SWITCH_ELEMENT_TYPE_VF: 12953 case I40E_SWITCH_ELEMENT_TYPE_EMP: 12954 case I40E_SWITCH_ELEMENT_TYPE_BMC: 12955 case I40E_SWITCH_ELEMENT_TYPE_PE: 12956 case I40E_SWITCH_ELEMENT_TYPE_PA: 12957 /* ignore these for now */ 12958 break; 12959 default: 12960 dev_info(&pf->pdev->dev, "unknown element type=%d seid=%d\n", 12961 element_type, seid); 12962 break; 12963 } 12964 } 12965 12966 /** 12967 * i40e_fetch_switch_configuration - Get switch config from firmware 12968 * @pf: board private structure 12969 * @printconfig: should we print the contents 12970 * 12971 * Get the current switch configuration from the device and 12972 * extract a few useful SEID values. 12973 **/ 12974 int i40e_fetch_switch_configuration(struct i40e_pf *pf, bool printconfig) 12975 { 12976 struct i40e_aqc_get_switch_config_resp *sw_config; 12977 u16 next_seid = 0; 12978 int ret = 0; 12979 u8 *aq_buf; 12980 int i; 12981 12982 aq_buf = kzalloc(I40E_AQ_LARGE_BUF, GFP_KERNEL); 12983 if (!aq_buf) 12984 return -ENOMEM; 12985 12986 sw_config = (struct i40e_aqc_get_switch_config_resp *)aq_buf; 12987 do { 12988 u16 num_reported, num_total; 12989 12990 ret = i40e_aq_get_switch_config(&pf->hw, sw_config, 12991 I40E_AQ_LARGE_BUF, 12992 &next_seid, NULL); 12993 if (ret) { 12994 dev_info(&pf->pdev->dev, 12995 "get switch config failed err %s aq_err %s\n", 12996 i40e_stat_str(&pf->hw, ret), 12997 i40e_aq_str(&pf->hw, 12998 pf->hw.aq.asq_last_status)); 12999 kfree(aq_buf); 13000 return -ENOENT; 13001 } 13002 13003 num_reported = le16_to_cpu(sw_config->header.num_reported); 13004 num_total = le16_to_cpu(sw_config->header.num_total); 13005 13006 if (printconfig) 13007 dev_info(&pf->pdev->dev, 13008 "header: %d reported %d total\n", 13009 num_reported, num_total); 13010 13011 for (i = 0; i < num_reported; i++) { 13012 struct i40e_aqc_switch_config_element_resp *ele = 13013 &sw_config->element[i]; 13014 13015 i40e_setup_pf_switch_element(pf, ele, num_reported, 13016 printconfig); 13017 } 13018 } while (next_seid != 0); 13019 13020 kfree(aq_buf); 13021 return ret; 13022 } 13023 13024 /** 13025 * i40e_setup_pf_switch - Setup the HW switch on startup or after reset 13026 * @pf: board private structure 13027 * @reinit: if the Main VSI needs to re-initialized. 13028 * 13029 * Returns 0 on success, negative value on failure 13030 **/ 13031 static int i40e_setup_pf_switch(struct i40e_pf *pf, bool reinit) 13032 { 13033 u16 flags = 0; 13034 int ret; 13035 13036 /* find out what's out there already */ 13037 ret = i40e_fetch_switch_configuration(pf, false); 13038 if (ret) { 13039 dev_info(&pf->pdev->dev, 13040 "couldn't fetch switch config, err %s aq_err %s\n", 13041 i40e_stat_str(&pf->hw, ret), 13042 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 13043 return ret; 13044 } 13045 i40e_pf_reset_stats(pf); 13046 13047 /* set the switch config bit for the whole device to 13048 * support limited promisc or true promisc 13049 * when user requests promisc. The default is limited 13050 * promisc. 13051 */ 13052 13053 if ((pf->hw.pf_id == 0) && 13054 !(pf->flags & I40E_FLAG_TRUE_PROMISC_SUPPORT)) { 13055 flags = I40E_AQ_SET_SWITCH_CFG_PROMISC; 13056 pf->last_sw_conf_flags = flags; 13057 } 13058 13059 if (pf->hw.pf_id == 0) { 13060 u16 valid_flags; 13061 13062 valid_flags = I40E_AQ_SET_SWITCH_CFG_PROMISC; 13063 ret = i40e_aq_set_switch_config(&pf->hw, flags, valid_flags, 0, 13064 NULL); 13065 if (ret && pf->hw.aq.asq_last_status != I40E_AQ_RC_ESRCH) { 13066 dev_info(&pf->pdev->dev, 13067 "couldn't set switch config bits, err %s aq_err %s\n", 13068 i40e_stat_str(&pf->hw, ret), 13069 i40e_aq_str(&pf->hw, 13070 pf->hw.aq.asq_last_status)); 13071 /* not a fatal problem, just keep going */ 13072 } 13073 pf->last_sw_conf_valid_flags = valid_flags; 13074 } 13075 13076 /* first time setup */ 13077 if (pf->lan_vsi == I40E_NO_VSI || reinit) { 13078 struct i40e_vsi *vsi = NULL; 13079 u16 uplink_seid; 13080 13081 /* Set up the PF VSI associated with the PF's main VSI 13082 * that is already in the HW switch 13083 */ 13084 if (pf->lan_veb != I40E_NO_VEB && pf->veb[pf->lan_veb]) 13085 uplink_seid = pf->veb[pf->lan_veb]->seid; 13086 else 13087 uplink_seid = pf->mac_seid; 13088 if (pf->lan_vsi == I40E_NO_VSI) 13089 vsi = i40e_vsi_setup(pf, I40E_VSI_MAIN, uplink_seid, 0); 13090 else if (reinit) 13091 vsi = i40e_vsi_reinit_setup(pf->vsi[pf->lan_vsi]); 13092 if (!vsi) { 13093 dev_info(&pf->pdev->dev, "setup of MAIN VSI failed\n"); 13094 i40e_cloud_filter_exit(pf); 13095 i40e_fdir_teardown(pf); 13096 return -EAGAIN; 13097 } 13098 } else { 13099 /* force a reset of TC and queue layout configurations */ 13100 u8 enabled_tc = pf->vsi[pf->lan_vsi]->tc_config.enabled_tc; 13101 13102 pf->vsi[pf->lan_vsi]->tc_config.enabled_tc = 0; 13103 pf->vsi[pf->lan_vsi]->seid = pf->main_vsi_seid; 13104 i40e_vsi_config_tc(pf->vsi[pf->lan_vsi], enabled_tc); 13105 } 13106 i40e_vlan_stripping_disable(pf->vsi[pf->lan_vsi]); 13107 13108 i40e_fdir_sb_setup(pf); 13109 13110 /* Setup static PF queue filter control settings */ 13111 ret = i40e_setup_pf_filter_control(pf); 13112 if (ret) { 13113 dev_info(&pf->pdev->dev, "setup_pf_filter_control failed: %d\n", 13114 ret); 13115 /* Failure here should not stop continuing other steps */ 13116 } 13117 13118 /* enable RSS in the HW, even for only one queue, as the stack can use 13119 * the hash 13120 */ 13121 if ((pf->flags & I40E_FLAG_RSS_ENABLED)) 13122 i40e_pf_config_rss(pf); 13123 13124 /* fill in link information and enable LSE reporting */ 13125 i40e_link_event(pf); 13126 13127 /* Initialize user-specific link properties */ 13128 pf->fc_autoneg_status = ((pf->hw.phy.link_info.an_info & 13129 I40E_AQ_AN_COMPLETED) ? true : false); 13130 13131 i40e_ptp_init(pf); 13132 13133 /* repopulate tunnel port filters */ 13134 i40e_sync_udp_filters(pf); 13135 13136 return ret; 13137 } 13138 13139 /** 13140 * i40e_determine_queue_usage - Work out queue distribution 13141 * @pf: board private structure 13142 **/ 13143 static void i40e_determine_queue_usage(struct i40e_pf *pf) 13144 { 13145 int queues_left; 13146 int q_max; 13147 13148 pf->num_lan_qps = 0; 13149 13150 /* Find the max queues to be put into basic use. We'll always be 13151 * using TC0, whether or not DCB is running, and TC0 will get the 13152 * big RSS set. 13153 */ 13154 queues_left = pf->hw.func_caps.num_tx_qp; 13155 13156 if ((queues_left == 1) || 13157 !(pf->flags & I40E_FLAG_MSIX_ENABLED)) { 13158 /* one qp for PF, no queues for anything else */ 13159 queues_left = 0; 13160 pf->alloc_rss_size = pf->num_lan_qps = 1; 13161 13162 /* make sure all the fancies are disabled */ 13163 pf->flags &= ~(I40E_FLAG_RSS_ENABLED | 13164 I40E_FLAG_IWARP_ENABLED | 13165 I40E_FLAG_FD_SB_ENABLED | 13166 I40E_FLAG_FD_ATR_ENABLED | 13167 I40E_FLAG_DCB_CAPABLE | 13168 I40E_FLAG_DCB_ENABLED | 13169 I40E_FLAG_SRIOV_ENABLED | 13170 I40E_FLAG_VMDQ_ENABLED); 13171 pf->flags |= I40E_FLAG_FD_SB_INACTIVE; 13172 } else if (!(pf->flags & (I40E_FLAG_RSS_ENABLED | 13173 I40E_FLAG_FD_SB_ENABLED | 13174 I40E_FLAG_FD_ATR_ENABLED | 13175 I40E_FLAG_DCB_CAPABLE))) { 13176 /* one qp for PF */ 13177 pf->alloc_rss_size = pf->num_lan_qps = 1; 13178 queues_left -= pf->num_lan_qps; 13179 13180 pf->flags &= ~(I40E_FLAG_RSS_ENABLED | 13181 I40E_FLAG_IWARP_ENABLED | 13182 I40E_FLAG_FD_SB_ENABLED | 13183 I40E_FLAG_FD_ATR_ENABLED | 13184 I40E_FLAG_DCB_ENABLED | 13185 I40E_FLAG_VMDQ_ENABLED); 13186 pf->flags |= I40E_FLAG_FD_SB_INACTIVE; 13187 } else { 13188 /* Not enough queues for all TCs */ 13189 if ((pf->flags & I40E_FLAG_DCB_CAPABLE) && 13190 (queues_left < I40E_MAX_TRAFFIC_CLASS)) { 13191 pf->flags &= ~(I40E_FLAG_DCB_CAPABLE | 13192 I40E_FLAG_DCB_ENABLED); 13193 dev_info(&pf->pdev->dev, "not enough queues for DCB. DCB is disabled.\n"); 13194 } 13195 13196 /* limit lan qps to the smaller of qps, cpus or msix */ 13197 q_max = max_t(int, pf->rss_size_max, num_online_cpus()); 13198 q_max = min_t(int, q_max, pf->hw.func_caps.num_tx_qp); 13199 q_max = min_t(int, q_max, pf->hw.func_caps.num_msix_vectors); 13200 pf->num_lan_qps = q_max; 13201 13202 queues_left -= pf->num_lan_qps; 13203 } 13204 13205 if (pf->flags & I40E_FLAG_FD_SB_ENABLED) { 13206 if (queues_left > 1) { 13207 queues_left -= 1; /* save 1 queue for FD */ 13208 } else { 13209 pf->flags &= ~I40E_FLAG_FD_SB_ENABLED; 13210 pf->flags |= I40E_FLAG_FD_SB_INACTIVE; 13211 dev_info(&pf->pdev->dev, "not enough queues for Flow Director. Flow Director feature is disabled\n"); 13212 } 13213 } 13214 13215 if ((pf->flags & I40E_FLAG_SRIOV_ENABLED) && 13216 pf->num_vf_qps && pf->num_req_vfs && queues_left) { 13217 pf->num_req_vfs = min_t(int, pf->num_req_vfs, 13218 (queues_left / pf->num_vf_qps)); 13219 queues_left -= (pf->num_req_vfs * pf->num_vf_qps); 13220 } 13221 13222 if ((pf->flags & I40E_FLAG_VMDQ_ENABLED) && 13223 pf->num_vmdq_vsis && pf->num_vmdq_qps && queues_left) { 13224 pf->num_vmdq_vsis = min_t(int, pf->num_vmdq_vsis, 13225 (queues_left / pf->num_vmdq_qps)); 13226 queues_left -= (pf->num_vmdq_vsis * pf->num_vmdq_qps); 13227 } 13228 13229 pf->queues_left = queues_left; 13230 dev_dbg(&pf->pdev->dev, 13231 "qs_avail=%d FD SB=%d lan_qs=%d lan_tc0=%d vf=%d*%d vmdq=%d*%d, remaining=%d\n", 13232 pf->hw.func_caps.num_tx_qp, 13233 !!(pf->flags & I40E_FLAG_FD_SB_ENABLED), 13234 pf->num_lan_qps, pf->alloc_rss_size, pf->num_req_vfs, 13235 pf->num_vf_qps, pf->num_vmdq_vsis, pf->num_vmdq_qps, 13236 queues_left); 13237 } 13238 13239 /** 13240 * i40e_setup_pf_filter_control - Setup PF static filter control 13241 * @pf: PF to be setup 13242 * 13243 * i40e_setup_pf_filter_control sets up a PF's initial filter control 13244 * settings. If PE/FCoE are enabled then it will also set the per PF 13245 * based filter sizes required for them. It also enables Flow director, 13246 * ethertype and macvlan type filter settings for the pf. 13247 * 13248 * Returns 0 on success, negative on failure 13249 **/ 13250 static int i40e_setup_pf_filter_control(struct i40e_pf *pf) 13251 { 13252 struct i40e_filter_control_settings *settings = &pf->filter_settings; 13253 13254 settings->hash_lut_size = I40E_HASH_LUT_SIZE_128; 13255 13256 /* Flow Director is enabled */ 13257 if (pf->flags & (I40E_FLAG_FD_SB_ENABLED | I40E_FLAG_FD_ATR_ENABLED)) 13258 settings->enable_fdir = true; 13259 13260 /* Ethtype and MACVLAN filters enabled for PF */ 13261 settings->enable_ethtype = true; 13262 settings->enable_macvlan = true; 13263 13264 if (i40e_set_filter_control(&pf->hw, settings)) 13265 return -ENOENT; 13266 13267 return 0; 13268 } 13269 13270 #define INFO_STRING_LEN 255 13271 #define REMAIN(__x) (INFO_STRING_LEN - (__x)) 13272 static void i40e_print_features(struct i40e_pf *pf) 13273 { 13274 struct i40e_hw *hw = &pf->hw; 13275 char *buf; 13276 int i; 13277 13278 buf = kmalloc(INFO_STRING_LEN, GFP_KERNEL); 13279 if (!buf) 13280 return; 13281 13282 i = snprintf(buf, INFO_STRING_LEN, "Features: PF-id[%d]", hw->pf_id); 13283 #ifdef CONFIG_PCI_IOV 13284 i += snprintf(&buf[i], REMAIN(i), " VFs: %d", pf->num_req_vfs); 13285 #endif 13286 i += snprintf(&buf[i], REMAIN(i), " VSIs: %d QP: %d", 13287 pf->hw.func_caps.num_vsis, 13288 pf->vsi[pf->lan_vsi]->num_queue_pairs); 13289 if (pf->flags & I40E_FLAG_RSS_ENABLED) 13290 i += snprintf(&buf[i], REMAIN(i), " RSS"); 13291 if (pf->flags & I40E_FLAG_FD_ATR_ENABLED) 13292 i += snprintf(&buf[i], REMAIN(i), " FD_ATR"); 13293 if (pf->flags & I40E_FLAG_FD_SB_ENABLED) { 13294 i += snprintf(&buf[i], REMAIN(i), " FD_SB"); 13295 i += snprintf(&buf[i], REMAIN(i), " NTUPLE"); 13296 } 13297 if (pf->flags & I40E_FLAG_DCB_CAPABLE) 13298 i += snprintf(&buf[i], REMAIN(i), " DCB"); 13299 i += snprintf(&buf[i], REMAIN(i), " VxLAN"); 13300 i += snprintf(&buf[i], REMAIN(i), " Geneve"); 13301 if (pf->flags & I40E_FLAG_PTP) 13302 i += snprintf(&buf[i], REMAIN(i), " PTP"); 13303 if (pf->flags & I40E_FLAG_VEB_MODE_ENABLED) 13304 i += snprintf(&buf[i], REMAIN(i), " VEB"); 13305 else 13306 i += snprintf(&buf[i], REMAIN(i), " VEPA"); 13307 13308 dev_info(&pf->pdev->dev, "%s\n", buf); 13309 kfree(buf); 13310 WARN_ON(i > INFO_STRING_LEN); 13311 } 13312 13313 /** 13314 * i40e_get_platform_mac_addr - get platform-specific MAC address 13315 * @pdev: PCI device information struct 13316 * @pf: board private structure 13317 * 13318 * Look up the MAC address for the device. First we'll try 13319 * eth_platform_get_mac_address, which will check Open Firmware, or arch 13320 * specific fallback. Otherwise, we'll default to the stored value in 13321 * firmware. 13322 **/ 13323 static void i40e_get_platform_mac_addr(struct pci_dev *pdev, struct i40e_pf *pf) 13324 { 13325 if (eth_platform_get_mac_address(&pdev->dev, pf->hw.mac.addr)) 13326 i40e_get_mac_addr(&pf->hw, pf->hw.mac.addr); 13327 } 13328 13329 /** 13330 * i40e_probe - Device initialization routine 13331 * @pdev: PCI device information struct 13332 * @ent: entry in i40e_pci_tbl 13333 * 13334 * i40e_probe initializes a PF identified by a pci_dev structure. 13335 * The OS initialization, configuring of the PF private structure, 13336 * and a hardware reset occur. 13337 * 13338 * Returns 0 on success, negative on failure 13339 **/ 13340 static int i40e_probe(struct pci_dev *pdev, const struct pci_device_id *ent) 13341 { 13342 struct i40e_aq_get_phy_abilities_resp abilities; 13343 struct i40e_pf *pf; 13344 struct i40e_hw *hw; 13345 static u16 pfs_found; 13346 u16 wol_nvm_bits; 13347 u16 link_status; 13348 int err; 13349 u32 val; 13350 u32 i; 13351 u8 set_fc_aq_fail; 13352 13353 err = pci_enable_device_mem(pdev); 13354 if (err) 13355 return err; 13356 13357 /* set up for high or low dma */ 13358 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); 13359 if (err) { 13360 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)); 13361 if (err) { 13362 dev_err(&pdev->dev, 13363 "DMA configuration failed: 0x%x\n", err); 13364 goto err_dma; 13365 } 13366 } 13367 13368 /* set up pci connections */ 13369 err = pci_request_mem_regions(pdev, i40e_driver_name); 13370 if (err) { 13371 dev_info(&pdev->dev, 13372 "pci_request_selected_regions failed %d\n", err); 13373 goto err_pci_reg; 13374 } 13375 13376 pci_enable_pcie_error_reporting(pdev); 13377 pci_set_master(pdev); 13378 13379 /* Now that we have a PCI connection, we need to do the 13380 * low level device setup. This is primarily setting up 13381 * the Admin Queue structures and then querying for the 13382 * device's current profile information. 13383 */ 13384 pf = kzalloc(sizeof(*pf), GFP_KERNEL); 13385 if (!pf) { 13386 err = -ENOMEM; 13387 goto err_pf_alloc; 13388 } 13389 pf->next_vsi = 0; 13390 pf->pdev = pdev; 13391 set_bit(__I40E_DOWN, pf->state); 13392 13393 hw = &pf->hw; 13394 hw->back = pf; 13395 13396 pf->ioremap_len = min_t(int, pci_resource_len(pdev, 0), 13397 I40E_MAX_CSR_SPACE); 13398 13399 hw->hw_addr = ioremap(pci_resource_start(pdev, 0), pf->ioremap_len); 13400 if (!hw->hw_addr) { 13401 err = -EIO; 13402 dev_info(&pdev->dev, "ioremap(0x%04x, 0x%04x) failed: 0x%x\n", 13403 (unsigned int)pci_resource_start(pdev, 0), 13404 pf->ioremap_len, err); 13405 goto err_ioremap; 13406 } 13407 hw->vendor_id = pdev->vendor; 13408 hw->device_id = pdev->device; 13409 pci_read_config_byte(pdev, PCI_REVISION_ID, &hw->revision_id); 13410 hw->subsystem_vendor_id = pdev->subsystem_vendor; 13411 hw->subsystem_device_id = pdev->subsystem_device; 13412 hw->bus.device = PCI_SLOT(pdev->devfn); 13413 hw->bus.func = PCI_FUNC(pdev->devfn); 13414 hw->bus.bus_id = pdev->bus->number; 13415 pf->instance = pfs_found; 13416 13417 /* Select something other than the 802.1ad ethertype for the 13418 * switch to use internally and drop on ingress. 13419 */ 13420 hw->switch_tag = 0xffff; 13421 hw->first_tag = ETH_P_8021AD; 13422 hw->second_tag = ETH_P_8021Q; 13423 13424 INIT_LIST_HEAD(&pf->l3_flex_pit_list); 13425 INIT_LIST_HEAD(&pf->l4_flex_pit_list); 13426 13427 /* set up the locks for the AQ, do this only once in probe 13428 * and destroy them only once in remove 13429 */ 13430 mutex_init(&hw->aq.asq_mutex); 13431 mutex_init(&hw->aq.arq_mutex); 13432 13433 pf->msg_enable = netif_msg_init(debug, 13434 NETIF_MSG_DRV | 13435 NETIF_MSG_PROBE | 13436 NETIF_MSG_LINK); 13437 if (debug < -1) 13438 pf->hw.debug_mask = debug; 13439 13440 /* do a special CORER for clearing PXE mode once at init */ 13441 if (hw->revision_id == 0 && 13442 (rd32(hw, I40E_GLLAN_RCTL_0) & I40E_GLLAN_RCTL_0_PXE_MODE_MASK)) { 13443 wr32(hw, I40E_GLGEN_RTRIG, I40E_GLGEN_RTRIG_CORER_MASK); 13444 i40e_flush(hw); 13445 msleep(200); 13446 pf->corer_count++; 13447 13448 i40e_clear_pxe_mode(hw); 13449 } 13450 13451 /* Reset here to make sure all is clean and to define PF 'n' */ 13452 i40e_clear_hw(hw); 13453 err = i40e_pf_reset(hw); 13454 if (err) { 13455 dev_info(&pdev->dev, "Initial pf_reset failed: %d\n", err); 13456 goto err_pf_reset; 13457 } 13458 pf->pfr_count++; 13459 13460 hw->aq.num_arq_entries = I40E_AQ_LEN; 13461 hw->aq.num_asq_entries = I40E_AQ_LEN; 13462 hw->aq.arq_buf_size = I40E_MAX_AQ_BUF_SIZE; 13463 hw->aq.asq_buf_size = I40E_MAX_AQ_BUF_SIZE; 13464 pf->adminq_work_limit = I40E_AQ_WORK_LIMIT; 13465 13466 snprintf(pf->int_name, sizeof(pf->int_name) - 1, 13467 "%s-%s:misc", 13468 dev_driver_string(&pf->pdev->dev), dev_name(&pdev->dev)); 13469 13470 err = i40e_init_shared_code(hw); 13471 if (err) { 13472 dev_warn(&pdev->dev, "unidentified MAC or BLANK NVM: %d\n", 13473 err); 13474 goto err_pf_reset; 13475 } 13476 13477 /* set up a default setting for link flow control */ 13478 pf->hw.fc.requested_mode = I40E_FC_NONE; 13479 13480 err = i40e_init_adminq(hw); 13481 if (err) { 13482 if (err == I40E_ERR_FIRMWARE_API_VERSION) 13483 dev_info(&pdev->dev, 13484 "The driver for the device stopped because the NVM image is newer than expected. You must install the most recent version of the network driver.\n"); 13485 else 13486 dev_info(&pdev->dev, 13487 "The driver for the device stopped because the device firmware failed to init. Try updating your NVM image.\n"); 13488 13489 goto err_pf_reset; 13490 } 13491 i40e_get_oem_version(hw); 13492 13493 /* provide nvm, fw, api versions */ 13494 dev_info(&pdev->dev, "fw %d.%d.%05d api %d.%d nvm %s\n", 13495 hw->aq.fw_maj_ver, hw->aq.fw_min_ver, hw->aq.fw_build, 13496 hw->aq.api_maj_ver, hw->aq.api_min_ver, 13497 i40e_nvm_version_str(hw)); 13498 13499 if (hw->aq.api_maj_ver == I40E_FW_API_VERSION_MAJOR && 13500 hw->aq.api_min_ver > I40E_FW_MINOR_VERSION(hw)) 13501 dev_info(&pdev->dev, 13502 "The driver for the device detected a newer version of the NVM image than expected. Please install the most recent version of the network driver.\n"); 13503 else if (hw->aq.api_maj_ver == 1 && hw->aq.api_min_ver < 4) 13504 dev_info(&pdev->dev, 13505 "The driver for the device detected an older version of the NVM image than expected. Please update the NVM image.\n"); 13506 13507 i40e_verify_eeprom(pf); 13508 13509 /* Rev 0 hardware was never productized */ 13510 if (hw->revision_id < 1) 13511 dev_warn(&pdev->dev, "This device is a pre-production adapter/LOM. Please be aware there may be issues with your hardware. If you are experiencing problems please contact your Intel or hardware representative who provided you with this hardware.\n"); 13512 13513 i40e_clear_pxe_mode(hw); 13514 err = i40e_get_capabilities(pf, i40e_aqc_opc_list_func_capabilities); 13515 if (err) 13516 goto err_adminq_setup; 13517 13518 err = i40e_sw_init(pf); 13519 if (err) { 13520 dev_info(&pdev->dev, "sw_init failed: %d\n", err); 13521 goto err_sw_init; 13522 } 13523 13524 err = i40e_init_lan_hmc(hw, hw->func_caps.num_tx_qp, 13525 hw->func_caps.num_rx_qp, 0, 0); 13526 if (err) { 13527 dev_info(&pdev->dev, "init_lan_hmc failed: %d\n", err); 13528 goto err_init_lan_hmc; 13529 } 13530 13531 err = i40e_configure_lan_hmc(hw, I40E_HMC_MODEL_DIRECT_ONLY); 13532 if (err) { 13533 dev_info(&pdev->dev, "configure_lan_hmc failed: %d\n", err); 13534 err = -ENOENT; 13535 goto err_configure_lan_hmc; 13536 } 13537 13538 /* Disable LLDP for NICs that have firmware versions lower than v4.3. 13539 * Ignore error return codes because if it was already disabled via 13540 * hardware settings this will fail 13541 */ 13542 if (pf->hw_features & I40E_HW_STOP_FW_LLDP) { 13543 dev_info(&pdev->dev, "Stopping firmware LLDP agent.\n"); 13544 i40e_aq_stop_lldp(hw, true, NULL); 13545 } 13546 13547 /* allow a platform config to override the HW addr */ 13548 i40e_get_platform_mac_addr(pdev, pf); 13549 13550 if (!is_valid_ether_addr(hw->mac.addr)) { 13551 dev_info(&pdev->dev, "invalid MAC address %pM\n", hw->mac.addr); 13552 err = -EIO; 13553 goto err_mac_addr; 13554 } 13555 dev_info(&pdev->dev, "MAC address: %pM\n", hw->mac.addr); 13556 ether_addr_copy(hw->mac.perm_addr, hw->mac.addr); 13557 i40e_get_port_mac_addr(hw, hw->mac.port_addr); 13558 if (is_valid_ether_addr(hw->mac.port_addr)) 13559 pf->hw_features |= I40E_HW_PORT_ID_VALID; 13560 13561 pci_set_drvdata(pdev, pf); 13562 pci_save_state(pdev); 13563 13564 /* Enable FW to write default DCB config on link-up */ 13565 i40e_aq_set_dcb_parameters(hw, true, NULL); 13566 13567 #ifdef CONFIG_I40E_DCB 13568 err = i40e_init_pf_dcb(pf); 13569 if (err) { 13570 dev_info(&pdev->dev, "DCB init failed %d, disabled\n", err); 13571 pf->flags &= ~(I40E_FLAG_DCB_CAPABLE | I40E_FLAG_DCB_ENABLED); 13572 /* Continue without DCB enabled */ 13573 } 13574 #endif /* CONFIG_I40E_DCB */ 13575 13576 /* set up periodic task facility */ 13577 timer_setup(&pf->service_timer, i40e_service_timer, 0); 13578 pf->service_timer_period = HZ; 13579 13580 INIT_WORK(&pf->service_task, i40e_service_task); 13581 clear_bit(__I40E_SERVICE_SCHED, pf->state); 13582 13583 /* NVM bit on means WoL disabled for the port */ 13584 i40e_read_nvm_word(hw, I40E_SR_NVM_WAKE_ON_LAN, &wol_nvm_bits); 13585 if (BIT (hw->port) & wol_nvm_bits || hw->partition_id != 1) 13586 pf->wol_en = false; 13587 else 13588 pf->wol_en = true; 13589 device_set_wakeup_enable(&pf->pdev->dev, pf->wol_en); 13590 13591 /* set up the main switch operations */ 13592 i40e_determine_queue_usage(pf); 13593 err = i40e_init_interrupt_scheme(pf); 13594 if (err) 13595 goto err_switch_setup; 13596 13597 /* The number of VSIs reported by the FW is the minimum guaranteed 13598 * to us; HW supports far more and we share the remaining pool with 13599 * the other PFs. We allocate space for more than the guarantee with 13600 * the understanding that we might not get them all later. 13601 */ 13602 if (pf->hw.func_caps.num_vsis < I40E_MIN_VSI_ALLOC) 13603 pf->num_alloc_vsi = I40E_MIN_VSI_ALLOC; 13604 else 13605 pf->num_alloc_vsi = pf->hw.func_caps.num_vsis; 13606 13607 /* Set up the *vsi struct and our local tracking of the MAIN PF vsi. */ 13608 pf->vsi = kcalloc(pf->num_alloc_vsi, sizeof(struct i40e_vsi *), 13609 GFP_KERNEL); 13610 if (!pf->vsi) { 13611 err = -ENOMEM; 13612 goto err_switch_setup; 13613 } 13614 13615 #ifdef CONFIG_PCI_IOV 13616 /* prep for VF support */ 13617 if ((pf->flags & I40E_FLAG_SRIOV_ENABLED) && 13618 (pf->flags & I40E_FLAG_MSIX_ENABLED) && 13619 !test_bit(__I40E_BAD_EEPROM, pf->state)) { 13620 if (pci_num_vf(pdev)) 13621 pf->flags |= I40E_FLAG_VEB_MODE_ENABLED; 13622 } 13623 #endif 13624 err = i40e_setup_pf_switch(pf, false); 13625 if (err) { 13626 dev_info(&pdev->dev, "setup_pf_switch failed: %d\n", err); 13627 goto err_vsis; 13628 } 13629 INIT_LIST_HEAD(&pf->vsi[pf->lan_vsi]->ch_list); 13630 13631 /* Make sure flow control is set according to current settings */ 13632 err = i40e_set_fc(hw, &set_fc_aq_fail, true); 13633 if (set_fc_aq_fail & I40E_SET_FC_AQ_FAIL_GET) 13634 dev_dbg(&pf->pdev->dev, 13635 "Set fc with err %s aq_err %s on get_phy_cap\n", 13636 i40e_stat_str(hw, err), 13637 i40e_aq_str(hw, hw->aq.asq_last_status)); 13638 if (set_fc_aq_fail & I40E_SET_FC_AQ_FAIL_SET) 13639 dev_dbg(&pf->pdev->dev, 13640 "Set fc with err %s aq_err %s on set_phy_config\n", 13641 i40e_stat_str(hw, err), 13642 i40e_aq_str(hw, hw->aq.asq_last_status)); 13643 if (set_fc_aq_fail & I40E_SET_FC_AQ_FAIL_UPDATE) 13644 dev_dbg(&pf->pdev->dev, 13645 "Set fc with err %s aq_err %s on get_link_info\n", 13646 i40e_stat_str(hw, err), 13647 i40e_aq_str(hw, hw->aq.asq_last_status)); 13648 13649 /* if FDIR VSI was set up, start it now */ 13650 for (i = 0; i < pf->num_alloc_vsi; i++) { 13651 if (pf->vsi[i] && pf->vsi[i]->type == I40E_VSI_FDIR) { 13652 i40e_vsi_open(pf->vsi[i]); 13653 break; 13654 } 13655 } 13656 13657 /* The driver only wants link up/down and module qualification 13658 * reports from firmware. Note the negative logic. 13659 */ 13660 err = i40e_aq_set_phy_int_mask(&pf->hw, 13661 ~(I40E_AQ_EVENT_LINK_UPDOWN | 13662 I40E_AQ_EVENT_MEDIA_NA | 13663 I40E_AQ_EVENT_MODULE_QUAL_FAIL), NULL); 13664 if (err) 13665 dev_info(&pf->pdev->dev, "set phy mask fail, err %s aq_err %s\n", 13666 i40e_stat_str(&pf->hw, err), 13667 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 13668 13669 /* Reconfigure hardware for allowing smaller MSS in the case 13670 * of TSO, so that we avoid the MDD being fired and causing 13671 * a reset in the case of small MSS+TSO. 13672 */ 13673 val = rd32(hw, I40E_REG_MSS); 13674 if ((val & I40E_REG_MSS_MIN_MASK) > I40E_64BYTE_MSS) { 13675 val &= ~I40E_REG_MSS_MIN_MASK; 13676 val |= I40E_64BYTE_MSS; 13677 wr32(hw, I40E_REG_MSS, val); 13678 } 13679 13680 if (pf->hw_features & I40E_HW_RESTART_AUTONEG) { 13681 msleep(75); 13682 err = i40e_aq_set_link_restart_an(&pf->hw, true, NULL); 13683 if (err) 13684 dev_info(&pf->pdev->dev, "link restart failed, err %s aq_err %s\n", 13685 i40e_stat_str(&pf->hw, err), 13686 i40e_aq_str(&pf->hw, 13687 pf->hw.aq.asq_last_status)); 13688 } 13689 /* The main driver is (mostly) up and happy. We need to set this state 13690 * before setting up the misc vector or we get a race and the vector 13691 * ends up disabled forever. 13692 */ 13693 clear_bit(__I40E_DOWN, pf->state); 13694 13695 /* In case of MSIX we are going to setup the misc vector right here 13696 * to handle admin queue events etc. In case of legacy and MSI 13697 * the misc functionality and queue processing is combined in 13698 * the same vector and that gets setup at open. 13699 */ 13700 if (pf->flags & I40E_FLAG_MSIX_ENABLED) { 13701 err = i40e_setup_misc_vector(pf); 13702 if (err) { 13703 dev_info(&pdev->dev, 13704 "setup of misc vector failed: %d\n", err); 13705 goto err_vsis; 13706 } 13707 } 13708 13709 #ifdef CONFIG_PCI_IOV 13710 /* prep for VF support */ 13711 if ((pf->flags & I40E_FLAG_SRIOV_ENABLED) && 13712 (pf->flags & I40E_FLAG_MSIX_ENABLED) && 13713 !test_bit(__I40E_BAD_EEPROM, pf->state)) { 13714 /* disable link interrupts for VFs */ 13715 val = rd32(hw, I40E_PFGEN_PORTMDIO_NUM); 13716 val &= ~I40E_PFGEN_PORTMDIO_NUM_VFLINK_STAT_ENA_MASK; 13717 wr32(hw, I40E_PFGEN_PORTMDIO_NUM, val); 13718 i40e_flush(hw); 13719 13720 if (pci_num_vf(pdev)) { 13721 dev_info(&pdev->dev, 13722 "Active VFs found, allocating resources.\n"); 13723 err = i40e_alloc_vfs(pf, pci_num_vf(pdev)); 13724 if (err) 13725 dev_info(&pdev->dev, 13726 "Error %d allocating resources for existing VFs\n", 13727 err); 13728 } 13729 } 13730 #endif /* CONFIG_PCI_IOV */ 13731 13732 if (pf->flags & I40E_FLAG_IWARP_ENABLED) { 13733 pf->iwarp_base_vector = i40e_get_lump(pf, pf->irq_pile, 13734 pf->num_iwarp_msix, 13735 I40E_IWARP_IRQ_PILE_ID); 13736 if (pf->iwarp_base_vector < 0) { 13737 dev_info(&pdev->dev, 13738 "failed to get tracking for %d vectors for IWARP err=%d\n", 13739 pf->num_iwarp_msix, pf->iwarp_base_vector); 13740 pf->flags &= ~I40E_FLAG_IWARP_ENABLED; 13741 } 13742 } 13743 13744 i40e_dbg_pf_init(pf); 13745 13746 /* tell the firmware that we're starting */ 13747 i40e_send_version(pf); 13748 13749 /* since everything's happy, start the service_task timer */ 13750 mod_timer(&pf->service_timer, 13751 round_jiffies(jiffies + pf->service_timer_period)); 13752 13753 /* add this PF to client device list and launch a client service task */ 13754 if (pf->flags & I40E_FLAG_IWARP_ENABLED) { 13755 err = i40e_lan_add_device(pf); 13756 if (err) 13757 dev_info(&pdev->dev, "Failed to add PF to client API service list: %d\n", 13758 err); 13759 } 13760 13761 #define PCI_SPEED_SIZE 8 13762 #define PCI_WIDTH_SIZE 8 13763 /* Devices on the IOSF bus do not have this information 13764 * and will report PCI Gen 1 x 1 by default so don't bother 13765 * checking them. 13766 */ 13767 if (!(pf->hw_features & I40E_HW_NO_PCI_LINK_CHECK)) { 13768 char speed[PCI_SPEED_SIZE] = "Unknown"; 13769 char width[PCI_WIDTH_SIZE] = "Unknown"; 13770 13771 /* Get the negotiated link width and speed from PCI config 13772 * space 13773 */ 13774 pcie_capability_read_word(pf->pdev, PCI_EXP_LNKSTA, 13775 &link_status); 13776 13777 i40e_set_pci_config_data(hw, link_status); 13778 13779 switch (hw->bus.speed) { 13780 case i40e_bus_speed_8000: 13781 strncpy(speed, "8.0", PCI_SPEED_SIZE); break; 13782 case i40e_bus_speed_5000: 13783 strncpy(speed, "5.0", PCI_SPEED_SIZE); break; 13784 case i40e_bus_speed_2500: 13785 strncpy(speed, "2.5", PCI_SPEED_SIZE); break; 13786 default: 13787 break; 13788 } 13789 switch (hw->bus.width) { 13790 case i40e_bus_width_pcie_x8: 13791 strncpy(width, "8", PCI_WIDTH_SIZE); break; 13792 case i40e_bus_width_pcie_x4: 13793 strncpy(width, "4", PCI_WIDTH_SIZE); break; 13794 case i40e_bus_width_pcie_x2: 13795 strncpy(width, "2", PCI_WIDTH_SIZE); break; 13796 case i40e_bus_width_pcie_x1: 13797 strncpy(width, "1", PCI_WIDTH_SIZE); break; 13798 default: 13799 break; 13800 } 13801 13802 dev_info(&pdev->dev, "PCI-Express: Speed %sGT/s Width x%s\n", 13803 speed, width); 13804 13805 if (hw->bus.width < i40e_bus_width_pcie_x8 || 13806 hw->bus.speed < i40e_bus_speed_8000) { 13807 dev_warn(&pdev->dev, "PCI-Express bandwidth available for this device may be insufficient for optimal performance.\n"); 13808 dev_warn(&pdev->dev, "Please move the device to a different PCI-e link with more lanes and/or higher transfer rate.\n"); 13809 } 13810 } 13811 13812 /* get the requested speeds from the fw */ 13813 err = i40e_aq_get_phy_capabilities(hw, false, false, &abilities, NULL); 13814 if (err) 13815 dev_dbg(&pf->pdev->dev, "get requested speeds ret = %s last_status = %s\n", 13816 i40e_stat_str(&pf->hw, err), 13817 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 13818 pf->hw.phy.link_info.requested_speeds = abilities.link_speed; 13819 13820 /* get the supported phy types from the fw */ 13821 err = i40e_aq_get_phy_capabilities(hw, false, true, &abilities, NULL); 13822 if (err) 13823 dev_dbg(&pf->pdev->dev, "get supported phy types ret = %s last_status = %s\n", 13824 i40e_stat_str(&pf->hw, err), 13825 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 13826 13827 /* Add a filter to drop all Flow control frames from any VSI from being 13828 * transmitted. By doing so we stop a malicious VF from sending out 13829 * PAUSE or PFC frames and potentially controlling traffic for other 13830 * PF/VF VSIs. 13831 * The FW can still send Flow control frames if enabled. 13832 */ 13833 i40e_add_filter_to_drop_tx_flow_control_frames(&pf->hw, 13834 pf->main_vsi_seid); 13835 13836 if ((pf->hw.device_id == I40E_DEV_ID_10G_BASE_T) || 13837 (pf->hw.device_id == I40E_DEV_ID_10G_BASE_T4)) 13838 pf->hw_features |= I40E_HW_PHY_CONTROLS_LEDS; 13839 if (pf->hw.device_id == I40E_DEV_ID_SFP_I_X722) 13840 pf->hw_features |= I40E_HW_HAVE_CRT_RETIMER; 13841 /* print a string summarizing features */ 13842 i40e_print_features(pf); 13843 13844 return 0; 13845 13846 /* Unwind what we've done if something failed in the setup */ 13847 err_vsis: 13848 set_bit(__I40E_DOWN, pf->state); 13849 i40e_clear_interrupt_scheme(pf); 13850 kfree(pf->vsi); 13851 err_switch_setup: 13852 i40e_reset_interrupt_capability(pf); 13853 del_timer_sync(&pf->service_timer); 13854 err_mac_addr: 13855 err_configure_lan_hmc: 13856 (void)i40e_shutdown_lan_hmc(hw); 13857 err_init_lan_hmc: 13858 kfree(pf->qp_pile); 13859 err_sw_init: 13860 err_adminq_setup: 13861 err_pf_reset: 13862 iounmap(hw->hw_addr); 13863 err_ioremap: 13864 kfree(pf); 13865 err_pf_alloc: 13866 pci_disable_pcie_error_reporting(pdev); 13867 pci_release_mem_regions(pdev); 13868 err_pci_reg: 13869 err_dma: 13870 pci_disable_device(pdev); 13871 return err; 13872 } 13873 13874 /** 13875 * i40e_remove - Device removal routine 13876 * @pdev: PCI device information struct 13877 * 13878 * i40e_remove is called by the PCI subsystem to alert the driver 13879 * that is should release a PCI device. This could be caused by a 13880 * Hot-Plug event, or because the driver is going to be removed from 13881 * memory. 13882 **/ 13883 static void i40e_remove(struct pci_dev *pdev) 13884 { 13885 struct i40e_pf *pf = pci_get_drvdata(pdev); 13886 struct i40e_hw *hw = &pf->hw; 13887 i40e_status ret_code; 13888 int i; 13889 13890 i40e_dbg_pf_exit(pf); 13891 13892 i40e_ptp_stop(pf); 13893 13894 /* Disable RSS in hw */ 13895 i40e_write_rx_ctl(hw, I40E_PFQF_HENA(0), 0); 13896 i40e_write_rx_ctl(hw, I40E_PFQF_HENA(1), 0); 13897 13898 /* no more scheduling of any task */ 13899 set_bit(__I40E_SUSPENDED, pf->state); 13900 set_bit(__I40E_DOWN, pf->state); 13901 if (pf->service_timer.function) 13902 del_timer_sync(&pf->service_timer); 13903 if (pf->service_task.func) 13904 cancel_work_sync(&pf->service_task); 13905 13906 /* Client close must be called explicitly here because the timer 13907 * has been stopped. 13908 */ 13909 i40e_notify_client_of_netdev_close(pf->vsi[pf->lan_vsi], false); 13910 13911 if (pf->flags & I40E_FLAG_SRIOV_ENABLED) { 13912 i40e_free_vfs(pf); 13913 pf->flags &= ~I40E_FLAG_SRIOV_ENABLED; 13914 } 13915 13916 i40e_fdir_teardown(pf); 13917 13918 /* If there is a switch structure or any orphans, remove them. 13919 * This will leave only the PF's VSI remaining. 13920 */ 13921 for (i = 0; i < I40E_MAX_VEB; i++) { 13922 if (!pf->veb[i]) 13923 continue; 13924 13925 if (pf->veb[i]->uplink_seid == pf->mac_seid || 13926 pf->veb[i]->uplink_seid == 0) 13927 i40e_switch_branch_release(pf->veb[i]); 13928 } 13929 13930 /* Now we can shutdown the PF's VSI, just before we kill 13931 * adminq and hmc. 13932 */ 13933 if (pf->vsi[pf->lan_vsi]) 13934 i40e_vsi_release(pf->vsi[pf->lan_vsi]); 13935 13936 i40e_cloud_filter_exit(pf); 13937 13938 /* remove attached clients */ 13939 if (pf->flags & I40E_FLAG_IWARP_ENABLED) { 13940 ret_code = i40e_lan_del_device(pf); 13941 if (ret_code) 13942 dev_warn(&pdev->dev, "Failed to delete client device: %d\n", 13943 ret_code); 13944 } 13945 13946 /* shutdown and destroy the HMC */ 13947 if (hw->hmc.hmc_obj) { 13948 ret_code = i40e_shutdown_lan_hmc(hw); 13949 if (ret_code) 13950 dev_warn(&pdev->dev, 13951 "Failed to destroy the HMC resources: %d\n", 13952 ret_code); 13953 } 13954 13955 /* shutdown the adminq */ 13956 i40e_shutdown_adminq(hw); 13957 13958 /* destroy the locks only once, here */ 13959 mutex_destroy(&hw->aq.arq_mutex); 13960 mutex_destroy(&hw->aq.asq_mutex); 13961 13962 /* Clear all dynamic memory lists of rings, q_vectors, and VSIs */ 13963 i40e_clear_interrupt_scheme(pf); 13964 for (i = 0; i < pf->num_alloc_vsi; i++) { 13965 if (pf->vsi[i]) { 13966 i40e_vsi_clear_rings(pf->vsi[i]); 13967 i40e_vsi_clear(pf->vsi[i]); 13968 pf->vsi[i] = NULL; 13969 } 13970 } 13971 13972 for (i = 0; i < I40E_MAX_VEB; i++) { 13973 kfree(pf->veb[i]); 13974 pf->veb[i] = NULL; 13975 } 13976 13977 kfree(pf->qp_pile); 13978 kfree(pf->vsi); 13979 13980 iounmap(hw->hw_addr); 13981 kfree(pf); 13982 pci_release_mem_regions(pdev); 13983 13984 pci_disable_pcie_error_reporting(pdev); 13985 pci_disable_device(pdev); 13986 } 13987 13988 /** 13989 * i40e_pci_error_detected - warning that something funky happened in PCI land 13990 * @pdev: PCI device information struct 13991 * 13992 * Called to warn that something happened and the error handling steps 13993 * are in progress. Allows the driver to quiesce things, be ready for 13994 * remediation. 13995 **/ 13996 static pci_ers_result_t i40e_pci_error_detected(struct pci_dev *pdev, 13997 enum pci_channel_state error) 13998 { 13999 struct i40e_pf *pf = pci_get_drvdata(pdev); 14000 14001 dev_info(&pdev->dev, "%s: error %d\n", __func__, error); 14002 14003 if (!pf) { 14004 dev_info(&pdev->dev, 14005 "Cannot recover - error happened during device probe\n"); 14006 return PCI_ERS_RESULT_DISCONNECT; 14007 } 14008 14009 /* shutdown all operations */ 14010 if (!test_bit(__I40E_SUSPENDED, pf->state)) 14011 i40e_prep_for_reset(pf, false); 14012 14013 /* Request a slot reset */ 14014 return PCI_ERS_RESULT_NEED_RESET; 14015 } 14016 14017 /** 14018 * i40e_pci_error_slot_reset - a PCI slot reset just happened 14019 * @pdev: PCI device information struct 14020 * 14021 * Called to find if the driver can work with the device now that 14022 * the pci slot has been reset. If a basic connection seems good 14023 * (registers are readable and have sane content) then return a 14024 * happy little PCI_ERS_RESULT_xxx. 14025 **/ 14026 static pci_ers_result_t i40e_pci_error_slot_reset(struct pci_dev *pdev) 14027 { 14028 struct i40e_pf *pf = pci_get_drvdata(pdev); 14029 pci_ers_result_t result; 14030 int err; 14031 u32 reg; 14032 14033 dev_dbg(&pdev->dev, "%s\n", __func__); 14034 if (pci_enable_device_mem(pdev)) { 14035 dev_info(&pdev->dev, 14036 "Cannot re-enable PCI device after reset.\n"); 14037 result = PCI_ERS_RESULT_DISCONNECT; 14038 } else { 14039 pci_set_master(pdev); 14040 pci_restore_state(pdev); 14041 pci_save_state(pdev); 14042 pci_wake_from_d3(pdev, false); 14043 14044 reg = rd32(&pf->hw, I40E_GLGEN_RTRIG); 14045 if (reg == 0) 14046 result = PCI_ERS_RESULT_RECOVERED; 14047 else 14048 result = PCI_ERS_RESULT_DISCONNECT; 14049 } 14050 14051 err = pci_cleanup_aer_uncorrect_error_status(pdev); 14052 if (err) { 14053 dev_info(&pdev->dev, 14054 "pci_cleanup_aer_uncorrect_error_status failed 0x%0x\n", 14055 err); 14056 /* non-fatal, continue */ 14057 } 14058 14059 return result; 14060 } 14061 14062 /** 14063 * i40e_pci_error_reset_prepare - prepare device driver for pci reset 14064 * @pdev: PCI device information struct 14065 */ 14066 static void i40e_pci_error_reset_prepare(struct pci_dev *pdev) 14067 { 14068 struct i40e_pf *pf = pci_get_drvdata(pdev); 14069 14070 i40e_prep_for_reset(pf, false); 14071 } 14072 14073 /** 14074 * i40e_pci_error_reset_done - pci reset done, device driver reset can begin 14075 * @pdev: PCI device information struct 14076 */ 14077 static void i40e_pci_error_reset_done(struct pci_dev *pdev) 14078 { 14079 struct i40e_pf *pf = pci_get_drvdata(pdev); 14080 14081 i40e_reset_and_rebuild(pf, false, false); 14082 } 14083 14084 /** 14085 * i40e_pci_error_resume - restart operations after PCI error recovery 14086 * @pdev: PCI device information struct 14087 * 14088 * Called to allow the driver to bring things back up after PCI error 14089 * and/or reset recovery has finished. 14090 **/ 14091 static void i40e_pci_error_resume(struct pci_dev *pdev) 14092 { 14093 struct i40e_pf *pf = pci_get_drvdata(pdev); 14094 14095 dev_dbg(&pdev->dev, "%s\n", __func__); 14096 if (test_bit(__I40E_SUSPENDED, pf->state)) 14097 return; 14098 14099 i40e_handle_reset_warning(pf, false); 14100 } 14101 14102 /** 14103 * i40e_enable_mc_magic_wake - enable multicast magic packet wake up 14104 * using the mac_address_write admin q function 14105 * @pf: pointer to i40e_pf struct 14106 **/ 14107 static void i40e_enable_mc_magic_wake(struct i40e_pf *pf) 14108 { 14109 struct i40e_hw *hw = &pf->hw; 14110 i40e_status ret; 14111 u8 mac_addr[6]; 14112 u16 flags = 0; 14113 14114 /* Get current MAC address in case it's an LAA */ 14115 if (pf->vsi[pf->lan_vsi] && pf->vsi[pf->lan_vsi]->netdev) { 14116 ether_addr_copy(mac_addr, 14117 pf->vsi[pf->lan_vsi]->netdev->dev_addr); 14118 } else { 14119 dev_err(&pf->pdev->dev, 14120 "Failed to retrieve MAC address; using default\n"); 14121 ether_addr_copy(mac_addr, hw->mac.addr); 14122 } 14123 14124 /* The FW expects the mac address write cmd to first be called with 14125 * one of these flags before calling it again with the multicast 14126 * enable flags. 14127 */ 14128 flags = I40E_AQC_WRITE_TYPE_LAA_WOL; 14129 14130 if (hw->func_caps.flex10_enable && hw->partition_id != 1) 14131 flags = I40E_AQC_WRITE_TYPE_LAA_ONLY; 14132 14133 ret = i40e_aq_mac_address_write(hw, flags, mac_addr, NULL); 14134 if (ret) { 14135 dev_err(&pf->pdev->dev, 14136 "Failed to update MAC address registers; cannot enable Multicast Magic packet wake up"); 14137 return; 14138 } 14139 14140 flags = I40E_AQC_MC_MAG_EN 14141 | I40E_AQC_WOL_PRESERVE_ON_PFR 14142 | I40E_AQC_WRITE_TYPE_UPDATE_MC_MAG; 14143 ret = i40e_aq_mac_address_write(hw, flags, mac_addr, NULL); 14144 if (ret) 14145 dev_err(&pf->pdev->dev, 14146 "Failed to enable Multicast Magic Packet wake up\n"); 14147 } 14148 14149 /** 14150 * i40e_shutdown - PCI callback for shutting down 14151 * @pdev: PCI device information struct 14152 **/ 14153 static void i40e_shutdown(struct pci_dev *pdev) 14154 { 14155 struct i40e_pf *pf = pci_get_drvdata(pdev); 14156 struct i40e_hw *hw = &pf->hw; 14157 14158 set_bit(__I40E_SUSPENDED, pf->state); 14159 set_bit(__I40E_DOWN, pf->state); 14160 rtnl_lock(); 14161 i40e_prep_for_reset(pf, true); 14162 rtnl_unlock(); 14163 14164 wr32(hw, I40E_PFPM_APM, (pf->wol_en ? I40E_PFPM_APM_APME_MASK : 0)); 14165 wr32(hw, I40E_PFPM_WUFC, (pf->wol_en ? I40E_PFPM_WUFC_MAG_MASK : 0)); 14166 14167 del_timer_sync(&pf->service_timer); 14168 cancel_work_sync(&pf->service_task); 14169 i40e_cloud_filter_exit(pf); 14170 i40e_fdir_teardown(pf); 14171 14172 /* Client close must be called explicitly here because the timer 14173 * has been stopped. 14174 */ 14175 i40e_notify_client_of_netdev_close(pf->vsi[pf->lan_vsi], false); 14176 14177 if (pf->wol_en && (pf->hw_features & I40E_HW_WOL_MC_MAGIC_PKT_WAKE)) 14178 i40e_enable_mc_magic_wake(pf); 14179 14180 i40e_prep_for_reset(pf, false); 14181 14182 wr32(hw, I40E_PFPM_APM, 14183 (pf->wol_en ? I40E_PFPM_APM_APME_MASK : 0)); 14184 wr32(hw, I40E_PFPM_WUFC, 14185 (pf->wol_en ? I40E_PFPM_WUFC_MAG_MASK : 0)); 14186 14187 i40e_clear_interrupt_scheme(pf); 14188 14189 if (system_state == SYSTEM_POWER_OFF) { 14190 pci_wake_from_d3(pdev, pf->wol_en); 14191 pci_set_power_state(pdev, PCI_D3hot); 14192 } 14193 } 14194 14195 /** 14196 * i40e_suspend - PM callback for moving to D3 14197 * @dev: generic device information structure 14198 **/ 14199 static int __maybe_unused i40e_suspend(struct device *dev) 14200 { 14201 struct pci_dev *pdev = to_pci_dev(dev); 14202 struct i40e_pf *pf = pci_get_drvdata(pdev); 14203 struct i40e_hw *hw = &pf->hw; 14204 14205 /* If we're already suspended, then there is nothing to do */ 14206 if (test_and_set_bit(__I40E_SUSPENDED, pf->state)) 14207 return 0; 14208 14209 set_bit(__I40E_DOWN, pf->state); 14210 14211 /* Ensure service task will not be running */ 14212 del_timer_sync(&pf->service_timer); 14213 cancel_work_sync(&pf->service_task); 14214 14215 if (pf->wol_en && (pf->hw_features & I40E_HW_WOL_MC_MAGIC_PKT_WAKE)) 14216 i40e_enable_mc_magic_wake(pf); 14217 14218 i40e_prep_for_reset(pf, false); 14219 14220 wr32(hw, I40E_PFPM_APM, (pf->wol_en ? I40E_PFPM_APM_APME_MASK : 0)); 14221 wr32(hw, I40E_PFPM_WUFC, (pf->wol_en ? I40E_PFPM_WUFC_MAG_MASK : 0)); 14222 14223 /* Clear the interrupt scheme and release our IRQs so that the system 14224 * can safely hibernate even when there are a large number of CPUs. 14225 * Otherwise hibernation might fail when mapping all the vectors back 14226 * to CPU0. 14227 */ 14228 i40e_clear_interrupt_scheme(pf); 14229 14230 return 0; 14231 } 14232 14233 /** 14234 * i40e_resume - PM callback for waking up from D3 14235 * @dev: generic device information structure 14236 **/ 14237 static int __maybe_unused i40e_resume(struct device *dev) 14238 { 14239 struct pci_dev *pdev = to_pci_dev(dev); 14240 struct i40e_pf *pf = pci_get_drvdata(pdev); 14241 int err; 14242 14243 /* If we're not suspended, then there is nothing to do */ 14244 if (!test_bit(__I40E_SUSPENDED, pf->state)) 14245 return 0; 14246 14247 /* We cleared the interrupt scheme when we suspended, so we need to 14248 * restore it now to resume device functionality. 14249 */ 14250 err = i40e_restore_interrupt_scheme(pf); 14251 if (err) { 14252 dev_err(&pdev->dev, "Cannot restore interrupt scheme: %d\n", 14253 err); 14254 } 14255 14256 clear_bit(__I40E_DOWN, pf->state); 14257 i40e_reset_and_rebuild(pf, false, false); 14258 14259 /* Clear suspended state last after everything is recovered */ 14260 clear_bit(__I40E_SUSPENDED, pf->state); 14261 14262 /* Restart the service task */ 14263 mod_timer(&pf->service_timer, 14264 round_jiffies(jiffies + pf->service_timer_period)); 14265 14266 return 0; 14267 } 14268 14269 static const struct pci_error_handlers i40e_err_handler = { 14270 .error_detected = i40e_pci_error_detected, 14271 .slot_reset = i40e_pci_error_slot_reset, 14272 .reset_prepare = i40e_pci_error_reset_prepare, 14273 .reset_done = i40e_pci_error_reset_done, 14274 .resume = i40e_pci_error_resume, 14275 }; 14276 14277 static SIMPLE_DEV_PM_OPS(i40e_pm_ops, i40e_suspend, i40e_resume); 14278 14279 static struct pci_driver i40e_driver = { 14280 .name = i40e_driver_name, 14281 .id_table = i40e_pci_tbl, 14282 .probe = i40e_probe, 14283 .remove = i40e_remove, 14284 .driver = { 14285 .pm = &i40e_pm_ops, 14286 }, 14287 .shutdown = i40e_shutdown, 14288 .err_handler = &i40e_err_handler, 14289 .sriov_configure = i40e_pci_sriov_configure, 14290 }; 14291 14292 /** 14293 * i40e_init_module - Driver registration routine 14294 * 14295 * i40e_init_module is the first routine called when the driver is 14296 * loaded. All it does is register with the PCI subsystem. 14297 **/ 14298 static int __init i40e_init_module(void) 14299 { 14300 pr_info("%s: %s - version %s\n", i40e_driver_name, 14301 i40e_driver_string, i40e_driver_version_str); 14302 pr_info("%s: %s\n", i40e_driver_name, i40e_copyright); 14303 14304 /* There is no need to throttle the number of active tasks because 14305 * each device limits its own task using a state bit for scheduling 14306 * the service task, and the device tasks do not interfere with each 14307 * other, so we don't set a max task limit. We must set WQ_MEM_RECLAIM 14308 * since we need to be able to guarantee forward progress even under 14309 * memory pressure. 14310 */ 14311 i40e_wq = alloc_workqueue("%s", WQ_MEM_RECLAIM, 0, i40e_driver_name); 14312 if (!i40e_wq) { 14313 pr_err("%s: Failed to create workqueue\n", i40e_driver_name); 14314 return -ENOMEM; 14315 } 14316 14317 i40e_dbg_init(); 14318 return pci_register_driver(&i40e_driver); 14319 } 14320 module_init(i40e_init_module); 14321 14322 /** 14323 * i40e_exit_module - Driver exit cleanup routine 14324 * 14325 * i40e_exit_module is called just before the driver is removed 14326 * from memory. 14327 **/ 14328 static void __exit i40e_exit_module(void) 14329 { 14330 pci_unregister_driver(&i40e_driver); 14331 destroy_workqueue(i40e_wq); 14332 i40e_dbg_exit(); 14333 } 14334 module_exit(i40e_exit_module); 14335