1 /******************************************************************************* 2 * 3 * Intel Ethernet Controller XL710 Family Linux Driver 4 * Copyright(c) 2013 - 2017 Intel Corporation. 5 * 6 * This program is free software; you can redistribute it and/or modify it 7 * under the terms and conditions of the GNU General Public License, 8 * version 2, as published by the Free Software Foundation. 9 * 10 * This program is distributed in the hope it will be useful, but WITHOUT 11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 13 * more details. 14 * 15 * You should have received a copy of the GNU General Public License along 16 * with this program. If not, see <http://www.gnu.org/licenses/>. 17 * 18 * The full GNU General Public License is included in this distribution in 19 * the file called "COPYING". 20 * 21 * Contact Information: 22 * e1000-devel Mailing List <e1000-devel@lists.sourceforge.net> 23 * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497 24 * 25 ******************************************************************************/ 26 27 #include <linux/etherdevice.h> 28 #include <linux/of_net.h> 29 #include <linux/pci.h> 30 #include <linux/bpf.h> 31 32 /* Local includes */ 33 #include "i40e.h" 34 #include "i40e_diag.h" 35 #include <net/udp_tunnel.h> 36 /* All i40e tracepoints are defined by the include below, which 37 * must be included exactly once across the whole kernel with 38 * CREATE_TRACE_POINTS defined 39 */ 40 #define CREATE_TRACE_POINTS 41 #include "i40e_trace.h" 42 43 const char i40e_driver_name[] = "i40e"; 44 static const char i40e_driver_string[] = 45 "Intel(R) Ethernet Connection XL710 Network Driver"; 46 47 #define DRV_KERN "-k" 48 49 #define DRV_VERSION_MAJOR 2 50 #define DRV_VERSION_MINOR 1 51 #define DRV_VERSION_BUILD 14 52 #define DRV_VERSION __stringify(DRV_VERSION_MAJOR) "." \ 53 __stringify(DRV_VERSION_MINOR) "." \ 54 __stringify(DRV_VERSION_BUILD) DRV_KERN 55 const char i40e_driver_version_str[] = DRV_VERSION; 56 static const char i40e_copyright[] = "Copyright (c) 2013 - 2014 Intel Corporation."; 57 58 /* a bit of forward declarations */ 59 static void i40e_vsi_reinit_locked(struct i40e_vsi *vsi); 60 static void i40e_handle_reset_warning(struct i40e_pf *pf, bool lock_acquired); 61 static int i40e_add_vsi(struct i40e_vsi *vsi); 62 static int i40e_add_veb(struct i40e_veb *veb, struct i40e_vsi *vsi); 63 static int i40e_setup_pf_switch(struct i40e_pf *pf, bool reinit); 64 static int i40e_setup_misc_vector(struct i40e_pf *pf); 65 static void i40e_determine_queue_usage(struct i40e_pf *pf); 66 static int i40e_setup_pf_filter_control(struct i40e_pf *pf); 67 static void i40e_prep_for_reset(struct i40e_pf *pf, bool lock_acquired); 68 static int i40e_reset(struct i40e_pf *pf); 69 static void i40e_rebuild(struct i40e_pf *pf, bool reinit, bool lock_acquired); 70 static void i40e_fdir_sb_setup(struct i40e_pf *pf); 71 static int i40e_veb_get_bw_info(struct i40e_veb *veb); 72 static int i40e_add_del_cloud_filter(struct i40e_vsi *vsi, 73 struct i40e_cloud_filter *filter, 74 bool add); 75 static int i40e_add_del_cloud_filter_big_buf(struct i40e_vsi *vsi, 76 struct i40e_cloud_filter *filter, 77 bool add); 78 static int i40e_get_capabilities(struct i40e_pf *pf, 79 enum i40e_admin_queue_opc list_type); 80 81 82 /* i40e_pci_tbl - PCI Device ID Table 83 * 84 * Last entry must be all 0s 85 * 86 * { Vendor ID, Device ID, SubVendor ID, SubDevice ID, 87 * Class, Class Mask, private data (not used) } 88 */ 89 static const struct pci_device_id i40e_pci_tbl[] = { 90 {PCI_VDEVICE(INTEL, I40E_DEV_ID_SFP_XL710), 0}, 91 {PCI_VDEVICE(INTEL, I40E_DEV_ID_QEMU), 0}, 92 {PCI_VDEVICE(INTEL, I40E_DEV_ID_KX_B), 0}, 93 {PCI_VDEVICE(INTEL, I40E_DEV_ID_KX_C), 0}, 94 {PCI_VDEVICE(INTEL, I40E_DEV_ID_QSFP_A), 0}, 95 {PCI_VDEVICE(INTEL, I40E_DEV_ID_QSFP_B), 0}, 96 {PCI_VDEVICE(INTEL, I40E_DEV_ID_QSFP_C), 0}, 97 {PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_BASE_T), 0}, 98 {PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_BASE_T4), 0}, 99 {PCI_VDEVICE(INTEL, I40E_DEV_ID_KX_X722), 0}, 100 {PCI_VDEVICE(INTEL, I40E_DEV_ID_QSFP_X722), 0}, 101 {PCI_VDEVICE(INTEL, I40E_DEV_ID_SFP_X722), 0}, 102 {PCI_VDEVICE(INTEL, I40E_DEV_ID_1G_BASE_T_X722), 0}, 103 {PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_BASE_T_X722), 0}, 104 {PCI_VDEVICE(INTEL, I40E_DEV_ID_SFP_I_X722), 0}, 105 {PCI_VDEVICE(INTEL, I40E_DEV_ID_20G_KR2), 0}, 106 {PCI_VDEVICE(INTEL, I40E_DEV_ID_20G_KR2_A), 0}, 107 {PCI_VDEVICE(INTEL, I40E_DEV_ID_25G_B), 0}, 108 {PCI_VDEVICE(INTEL, I40E_DEV_ID_25G_SFP28), 0}, 109 /* required last entry */ 110 {0, } 111 }; 112 MODULE_DEVICE_TABLE(pci, i40e_pci_tbl); 113 114 #define I40E_MAX_VF_COUNT 128 115 static int debug = -1; 116 module_param(debug, uint, 0); 117 MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all), Debug mask (0x8XXXXXXX)"); 118 119 MODULE_AUTHOR("Intel Corporation, <e1000-devel@lists.sourceforge.net>"); 120 MODULE_DESCRIPTION("Intel(R) Ethernet Connection XL710 Network Driver"); 121 MODULE_LICENSE("GPL"); 122 MODULE_VERSION(DRV_VERSION); 123 124 static struct workqueue_struct *i40e_wq; 125 126 /** 127 * i40e_allocate_dma_mem_d - OS specific memory alloc for shared code 128 * @hw: pointer to the HW structure 129 * @mem: ptr to mem struct to fill out 130 * @size: size of memory requested 131 * @alignment: what to align the allocation to 132 **/ 133 int i40e_allocate_dma_mem_d(struct i40e_hw *hw, struct i40e_dma_mem *mem, 134 u64 size, u32 alignment) 135 { 136 struct i40e_pf *pf = (struct i40e_pf *)hw->back; 137 138 mem->size = ALIGN(size, alignment); 139 mem->va = dma_zalloc_coherent(&pf->pdev->dev, mem->size, 140 &mem->pa, GFP_KERNEL); 141 if (!mem->va) 142 return -ENOMEM; 143 144 return 0; 145 } 146 147 /** 148 * i40e_free_dma_mem_d - OS specific memory free for shared code 149 * @hw: pointer to the HW structure 150 * @mem: ptr to mem struct to free 151 **/ 152 int i40e_free_dma_mem_d(struct i40e_hw *hw, struct i40e_dma_mem *mem) 153 { 154 struct i40e_pf *pf = (struct i40e_pf *)hw->back; 155 156 dma_free_coherent(&pf->pdev->dev, mem->size, mem->va, mem->pa); 157 mem->va = NULL; 158 mem->pa = 0; 159 mem->size = 0; 160 161 return 0; 162 } 163 164 /** 165 * i40e_allocate_virt_mem_d - OS specific memory alloc for shared code 166 * @hw: pointer to the HW structure 167 * @mem: ptr to mem struct to fill out 168 * @size: size of memory requested 169 **/ 170 int i40e_allocate_virt_mem_d(struct i40e_hw *hw, struct i40e_virt_mem *mem, 171 u32 size) 172 { 173 mem->size = size; 174 mem->va = kzalloc(size, GFP_KERNEL); 175 176 if (!mem->va) 177 return -ENOMEM; 178 179 return 0; 180 } 181 182 /** 183 * i40e_free_virt_mem_d - OS specific memory free for shared code 184 * @hw: pointer to the HW structure 185 * @mem: ptr to mem struct to free 186 **/ 187 int i40e_free_virt_mem_d(struct i40e_hw *hw, struct i40e_virt_mem *mem) 188 { 189 /* it's ok to kfree a NULL pointer */ 190 kfree(mem->va); 191 mem->va = NULL; 192 mem->size = 0; 193 194 return 0; 195 } 196 197 /** 198 * i40e_get_lump - find a lump of free generic resource 199 * @pf: board private structure 200 * @pile: the pile of resource to search 201 * @needed: the number of items needed 202 * @id: an owner id to stick on the items assigned 203 * 204 * Returns the base item index of the lump, or negative for error 205 * 206 * The search_hint trick and lack of advanced fit-finding only work 207 * because we're highly likely to have all the same size lump requests. 208 * Linear search time and any fragmentation should be minimal. 209 **/ 210 static int i40e_get_lump(struct i40e_pf *pf, struct i40e_lump_tracking *pile, 211 u16 needed, u16 id) 212 { 213 int ret = -ENOMEM; 214 int i, j; 215 216 if (!pile || needed == 0 || id >= I40E_PILE_VALID_BIT) { 217 dev_info(&pf->pdev->dev, 218 "param err: pile=%p needed=%d id=0x%04x\n", 219 pile, needed, id); 220 return -EINVAL; 221 } 222 223 /* start the linear search with an imperfect hint */ 224 i = pile->search_hint; 225 while (i < pile->num_entries) { 226 /* skip already allocated entries */ 227 if (pile->list[i] & I40E_PILE_VALID_BIT) { 228 i++; 229 continue; 230 } 231 232 /* do we have enough in this lump? */ 233 for (j = 0; (j < needed) && ((i+j) < pile->num_entries); j++) { 234 if (pile->list[i+j] & I40E_PILE_VALID_BIT) 235 break; 236 } 237 238 if (j == needed) { 239 /* there was enough, so assign it to the requestor */ 240 for (j = 0; j < needed; j++) 241 pile->list[i+j] = id | I40E_PILE_VALID_BIT; 242 ret = i; 243 pile->search_hint = i + j; 244 break; 245 } 246 247 /* not enough, so skip over it and continue looking */ 248 i += j; 249 } 250 251 return ret; 252 } 253 254 /** 255 * i40e_put_lump - return a lump of generic resource 256 * @pile: the pile of resource to search 257 * @index: the base item index 258 * @id: the owner id of the items assigned 259 * 260 * Returns the count of items in the lump 261 **/ 262 static int i40e_put_lump(struct i40e_lump_tracking *pile, u16 index, u16 id) 263 { 264 int valid_id = (id | I40E_PILE_VALID_BIT); 265 int count = 0; 266 int i; 267 268 if (!pile || index >= pile->num_entries) 269 return -EINVAL; 270 271 for (i = index; 272 i < pile->num_entries && pile->list[i] == valid_id; 273 i++) { 274 pile->list[i] = 0; 275 count++; 276 } 277 278 if (count && index < pile->search_hint) 279 pile->search_hint = index; 280 281 return count; 282 } 283 284 /** 285 * i40e_find_vsi_from_id - searches for the vsi with the given id 286 * @pf - the pf structure to search for the vsi 287 * @id - id of the vsi it is searching for 288 **/ 289 struct i40e_vsi *i40e_find_vsi_from_id(struct i40e_pf *pf, u16 id) 290 { 291 int i; 292 293 for (i = 0; i < pf->num_alloc_vsi; i++) 294 if (pf->vsi[i] && (pf->vsi[i]->id == id)) 295 return pf->vsi[i]; 296 297 return NULL; 298 } 299 300 /** 301 * i40e_service_event_schedule - Schedule the service task to wake up 302 * @pf: board private structure 303 * 304 * If not already scheduled, this puts the task into the work queue 305 **/ 306 void i40e_service_event_schedule(struct i40e_pf *pf) 307 { 308 if (!test_bit(__I40E_DOWN, pf->state) && 309 !test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state)) 310 queue_work(i40e_wq, &pf->service_task); 311 } 312 313 /** 314 * i40e_tx_timeout - Respond to a Tx Hang 315 * @netdev: network interface device structure 316 * 317 * If any port has noticed a Tx timeout, it is likely that the whole 318 * device is munged, not just the one netdev port, so go for the full 319 * reset. 320 **/ 321 static void i40e_tx_timeout(struct net_device *netdev) 322 { 323 struct i40e_netdev_priv *np = netdev_priv(netdev); 324 struct i40e_vsi *vsi = np->vsi; 325 struct i40e_pf *pf = vsi->back; 326 struct i40e_ring *tx_ring = NULL; 327 unsigned int i, hung_queue = 0; 328 u32 head, val; 329 330 pf->tx_timeout_count++; 331 332 /* find the stopped queue the same way the stack does */ 333 for (i = 0; i < netdev->num_tx_queues; i++) { 334 struct netdev_queue *q; 335 unsigned long trans_start; 336 337 q = netdev_get_tx_queue(netdev, i); 338 trans_start = q->trans_start; 339 if (netif_xmit_stopped(q) && 340 time_after(jiffies, 341 (trans_start + netdev->watchdog_timeo))) { 342 hung_queue = i; 343 break; 344 } 345 } 346 347 if (i == netdev->num_tx_queues) { 348 netdev_info(netdev, "tx_timeout: no netdev hung queue found\n"); 349 } else { 350 /* now that we have an index, find the tx_ring struct */ 351 for (i = 0; i < vsi->num_queue_pairs; i++) { 352 if (vsi->tx_rings[i] && vsi->tx_rings[i]->desc) { 353 if (hung_queue == 354 vsi->tx_rings[i]->queue_index) { 355 tx_ring = vsi->tx_rings[i]; 356 break; 357 } 358 } 359 } 360 } 361 362 if (time_after(jiffies, (pf->tx_timeout_last_recovery + HZ*20))) 363 pf->tx_timeout_recovery_level = 1; /* reset after some time */ 364 else if (time_before(jiffies, 365 (pf->tx_timeout_last_recovery + netdev->watchdog_timeo))) 366 return; /* don't do any new action before the next timeout */ 367 368 if (tx_ring) { 369 head = i40e_get_head(tx_ring); 370 /* Read interrupt register */ 371 if (pf->flags & I40E_FLAG_MSIX_ENABLED) 372 val = rd32(&pf->hw, 373 I40E_PFINT_DYN_CTLN(tx_ring->q_vector->v_idx + 374 tx_ring->vsi->base_vector - 1)); 375 else 376 val = rd32(&pf->hw, I40E_PFINT_DYN_CTL0); 377 378 netdev_info(netdev, "tx_timeout: VSI_seid: %d, Q %d, NTC: 0x%x, HWB: 0x%x, NTU: 0x%x, TAIL: 0x%x, INT: 0x%x\n", 379 vsi->seid, hung_queue, tx_ring->next_to_clean, 380 head, tx_ring->next_to_use, 381 readl(tx_ring->tail), val); 382 } 383 384 pf->tx_timeout_last_recovery = jiffies; 385 netdev_info(netdev, "tx_timeout recovery level %d, hung_queue %d\n", 386 pf->tx_timeout_recovery_level, hung_queue); 387 388 switch (pf->tx_timeout_recovery_level) { 389 case 1: 390 set_bit(__I40E_PF_RESET_REQUESTED, pf->state); 391 break; 392 case 2: 393 set_bit(__I40E_CORE_RESET_REQUESTED, pf->state); 394 break; 395 case 3: 396 set_bit(__I40E_GLOBAL_RESET_REQUESTED, pf->state); 397 break; 398 default: 399 netdev_err(netdev, "tx_timeout recovery unsuccessful\n"); 400 break; 401 } 402 403 i40e_service_event_schedule(pf); 404 pf->tx_timeout_recovery_level++; 405 } 406 407 /** 408 * i40e_get_vsi_stats_struct - Get System Network Statistics 409 * @vsi: the VSI we care about 410 * 411 * Returns the address of the device statistics structure. 412 * The statistics are actually updated from the service task. 413 **/ 414 struct rtnl_link_stats64 *i40e_get_vsi_stats_struct(struct i40e_vsi *vsi) 415 { 416 return &vsi->net_stats; 417 } 418 419 /** 420 * i40e_get_netdev_stats_struct_tx - populate stats from a Tx ring 421 * @ring: Tx ring to get statistics from 422 * @stats: statistics entry to be updated 423 **/ 424 static void i40e_get_netdev_stats_struct_tx(struct i40e_ring *ring, 425 struct rtnl_link_stats64 *stats) 426 { 427 u64 bytes, packets; 428 unsigned int start; 429 430 do { 431 start = u64_stats_fetch_begin_irq(&ring->syncp); 432 packets = ring->stats.packets; 433 bytes = ring->stats.bytes; 434 } while (u64_stats_fetch_retry_irq(&ring->syncp, start)); 435 436 stats->tx_packets += packets; 437 stats->tx_bytes += bytes; 438 } 439 440 /** 441 * i40e_get_netdev_stats_struct - Get statistics for netdev interface 442 * @netdev: network interface device structure 443 * 444 * Returns the address of the device statistics structure. 445 * The statistics are actually updated from the service task. 446 **/ 447 static void i40e_get_netdev_stats_struct(struct net_device *netdev, 448 struct rtnl_link_stats64 *stats) 449 { 450 struct i40e_netdev_priv *np = netdev_priv(netdev); 451 struct i40e_ring *tx_ring, *rx_ring; 452 struct i40e_vsi *vsi = np->vsi; 453 struct rtnl_link_stats64 *vsi_stats = i40e_get_vsi_stats_struct(vsi); 454 int i; 455 456 if (test_bit(__I40E_VSI_DOWN, vsi->state)) 457 return; 458 459 if (!vsi->tx_rings) 460 return; 461 462 rcu_read_lock(); 463 for (i = 0; i < vsi->num_queue_pairs; i++) { 464 u64 bytes, packets; 465 unsigned int start; 466 467 tx_ring = READ_ONCE(vsi->tx_rings[i]); 468 if (!tx_ring) 469 continue; 470 i40e_get_netdev_stats_struct_tx(tx_ring, stats); 471 472 rx_ring = &tx_ring[1]; 473 474 do { 475 start = u64_stats_fetch_begin_irq(&rx_ring->syncp); 476 packets = rx_ring->stats.packets; 477 bytes = rx_ring->stats.bytes; 478 } while (u64_stats_fetch_retry_irq(&rx_ring->syncp, start)); 479 480 stats->rx_packets += packets; 481 stats->rx_bytes += bytes; 482 483 if (i40e_enabled_xdp_vsi(vsi)) 484 i40e_get_netdev_stats_struct_tx(&rx_ring[1], stats); 485 } 486 rcu_read_unlock(); 487 488 /* following stats updated by i40e_watchdog_subtask() */ 489 stats->multicast = vsi_stats->multicast; 490 stats->tx_errors = vsi_stats->tx_errors; 491 stats->tx_dropped = vsi_stats->tx_dropped; 492 stats->rx_errors = vsi_stats->rx_errors; 493 stats->rx_dropped = vsi_stats->rx_dropped; 494 stats->rx_crc_errors = vsi_stats->rx_crc_errors; 495 stats->rx_length_errors = vsi_stats->rx_length_errors; 496 } 497 498 /** 499 * i40e_vsi_reset_stats - Resets all stats of the given vsi 500 * @vsi: the VSI to have its stats reset 501 **/ 502 void i40e_vsi_reset_stats(struct i40e_vsi *vsi) 503 { 504 struct rtnl_link_stats64 *ns; 505 int i; 506 507 if (!vsi) 508 return; 509 510 ns = i40e_get_vsi_stats_struct(vsi); 511 memset(ns, 0, sizeof(*ns)); 512 memset(&vsi->net_stats_offsets, 0, sizeof(vsi->net_stats_offsets)); 513 memset(&vsi->eth_stats, 0, sizeof(vsi->eth_stats)); 514 memset(&vsi->eth_stats_offsets, 0, sizeof(vsi->eth_stats_offsets)); 515 if (vsi->rx_rings && vsi->rx_rings[0]) { 516 for (i = 0; i < vsi->num_queue_pairs; i++) { 517 memset(&vsi->rx_rings[i]->stats, 0, 518 sizeof(vsi->rx_rings[i]->stats)); 519 memset(&vsi->rx_rings[i]->rx_stats, 0, 520 sizeof(vsi->rx_rings[i]->rx_stats)); 521 memset(&vsi->tx_rings[i]->stats, 0, 522 sizeof(vsi->tx_rings[i]->stats)); 523 memset(&vsi->tx_rings[i]->tx_stats, 0, 524 sizeof(vsi->tx_rings[i]->tx_stats)); 525 } 526 } 527 vsi->stat_offsets_loaded = false; 528 } 529 530 /** 531 * i40e_pf_reset_stats - Reset all of the stats for the given PF 532 * @pf: the PF to be reset 533 **/ 534 void i40e_pf_reset_stats(struct i40e_pf *pf) 535 { 536 int i; 537 538 memset(&pf->stats, 0, sizeof(pf->stats)); 539 memset(&pf->stats_offsets, 0, sizeof(pf->stats_offsets)); 540 pf->stat_offsets_loaded = false; 541 542 for (i = 0; i < I40E_MAX_VEB; i++) { 543 if (pf->veb[i]) { 544 memset(&pf->veb[i]->stats, 0, 545 sizeof(pf->veb[i]->stats)); 546 memset(&pf->veb[i]->stats_offsets, 0, 547 sizeof(pf->veb[i]->stats_offsets)); 548 pf->veb[i]->stat_offsets_loaded = false; 549 } 550 } 551 pf->hw_csum_rx_error = 0; 552 } 553 554 /** 555 * i40e_stat_update48 - read and update a 48 bit stat from the chip 556 * @hw: ptr to the hardware info 557 * @hireg: the high 32 bit reg to read 558 * @loreg: the low 32 bit reg to read 559 * @offset_loaded: has the initial offset been loaded yet 560 * @offset: ptr to current offset value 561 * @stat: ptr to the stat 562 * 563 * Since the device stats are not reset at PFReset, they likely will not 564 * be zeroed when the driver starts. We'll save the first values read 565 * and use them as offsets to be subtracted from the raw values in order 566 * to report stats that count from zero. In the process, we also manage 567 * the potential roll-over. 568 **/ 569 static void i40e_stat_update48(struct i40e_hw *hw, u32 hireg, u32 loreg, 570 bool offset_loaded, u64 *offset, u64 *stat) 571 { 572 u64 new_data; 573 574 if (hw->device_id == I40E_DEV_ID_QEMU) { 575 new_data = rd32(hw, loreg); 576 new_data |= ((u64)(rd32(hw, hireg) & 0xFFFF)) << 32; 577 } else { 578 new_data = rd64(hw, loreg); 579 } 580 if (!offset_loaded) 581 *offset = new_data; 582 if (likely(new_data >= *offset)) 583 *stat = new_data - *offset; 584 else 585 *stat = (new_data + BIT_ULL(48)) - *offset; 586 *stat &= 0xFFFFFFFFFFFFULL; 587 } 588 589 /** 590 * i40e_stat_update32 - read and update a 32 bit stat from the chip 591 * @hw: ptr to the hardware info 592 * @reg: the hw reg to read 593 * @offset_loaded: has the initial offset been loaded yet 594 * @offset: ptr to current offset value 595 * @stat: ptr to the stat 596 **/ 597 static void i40e_stat_update32(struct i40e_hw *hw, u32 reg, 598 bool offset_loaded, u64 *offset, u64 *stat) 599 { 600 u32 new_data; 601 602 new_data = rd32(hw, reg); 603 if (!offset_loaded) 604 *offset = new_data; 605 if (likely(new_data >= *offset)) 606 *stat = (u32)(new_data - *offset); 607 else 608 *stat = (u32)((new_data + BIT_ULL(32)) - *offset); 609 } 610 611 /** 612 * i40e_stat_update_and_clear32 - read and clear hw reg, update a 32 bit stat 613 * @hw: ptr to the hardware info 614 * @reg: the hw reg to read and clear 615 * @stat: ptr to the stat 616 **/ 617 static void i40e_stat_update_and_clear32(struct i40e_hw *hw, u32 reg, u64 *stat) 618 { 619 u32 new_data = rd32(hw, reg); 620 621 wr32(hw, reg, 1); /* must write a nonzero value to clear register */ 622 *stat += new_data; 623 } 624 625 /** 626 * i40e_update_eth_stats - Update VSI-specific ethernet statistics counters. 627 * @vsi: the VSI to be updated 628 **/ 629 void i40e_update_eth_stats(struct i40e_vsi *vsi) 630 { 631 int stat_idx = le16_to_cpu(vsi->info.stat_counter_idx); 632 struct i40e_pf *pf = vsi->back; 633 struct i40e_hw *hw = &pf->hw; 634 struct i40e_eth_stats *oes; 635 struct i40e_eth_stats *es; /* device's eth stats */ 636 637 es = &vsi->eth_stats; 638 oes = &vsi->eth_stats_offsets; 639 640 /* Gather up the stats that the hw collects */ 641 i40e_stat_update32(hw, I40E_GLV_TEPC(stat_idx), 642 vsi->stat_offsets_loaded, 643 &oes->tx_errors, &es->tx_errors); 644 i40e_stat_update32(hw, I40E_GLV_RDPC(stat_idx), 645 vsi->stat_offsets_loaded, 646 &oes->rx_discards, &es->rx_discards); 647 i40e_stat_update32(hw, I40E_GLV_RUPP(stat_idx), 648 vsi->stat_offsets_loaded, 649 &oes->rx_unknown_protocol, &es->rx_unknown_protocol); 650 i40e_stat_update32(hw, I40E_GLV_TEPC(stat_idx), 651 vsi->stat_offsets_loaded, 652 &oes->tx_errors, &es->tx_errors); 653 654 i40e_stat_update48(hw, I40E_GLV_GORCH(stat_idx), 655 I40E_GLV_GORCL(stat_idx), 656 vsi->stat_offsets_loaded, 657 &oes->rx_bytes, &es->rx_bytes); 658 i40e_stat_update48(hw, I40E_GLV_UPRCH(stat_idx), 659 I40E_GLV_UPRCL(stat_idx), 660 vsi->stat_offsets_loaded, 661 &oes->rx_unicast, &es->rx_unicast); 662 i40e_stat_update48(hw, I40E_GLV_MPRCH(stat_idx), 663 I40E_GLV_MPRCL(stat_idx), 664 vsi->stat_offsets_loaded, 665 &oes->rx_multicast, &es->rx_multicast); 666 i40e_stat_update48(hw, I40E_GLV_BPRCH(stat_idx), 667 I40E_GLV_BPRCL(stat_idx), 668 vsi->stat_offsets_loaded, 669 &oes->rx_broadcast, &es->rx_broadcast); 670 671 i40e_stat_update48(hw, I40E_GLV_GOTCH(stat_idx), 672 I40E_GLV_GOTCL(stat_idx), 673 vsi->stat_offsets_loaded, 674 &oes->tx_bytes, &es->tx_bytes); 675 i40e_stat_update48(hw, I40E_GLV_UPTCH(stat_idx), 676 I40E_GLV_UPTCL(stat_idx), 677 vsi->stat_offsets_loaded, 678 &oes->tx_unicast, &es->tx_unicast); 679 i40e_stat_update48(hw, I40E_GLV_MPTCH(stat_idx), 680 I40E_GLV_MPTCL(stat_idx), 681 vsi->stat_offsets_loaded, 682 &oes->tx_multicast, &es->tx_multicast); 683 i40e_stat_update48(hw, I40E_GLV_BPTCH(stat_idx), 684 I40E_GLV_BPTCL(stat_idx), 685 vsi->stat_offsets_loaded, 686 &oes->tx_broadcast, &es->tx_broadcast); 687 vsi->stat_offsets_loaded = true; 688 } 689 690 /** 691 * i40e_update_veb_stats - Update Switch component statistics 692 * @veb: the VEB being updated 693 **/ 694 static void i40e_update_veb_stats(struct i40e_veb *veb) 695 { 696 struct i40e_pf *pf = veb->pf; 697 struct i40e_hw *hw = &pf->hw; 698 struct i40e_eth_stats *oes; 699 struct i40e_eth_stats *es; /* device's eth stats */ 700 struct i40e_veb_tc_stats *veb_oes; 701 struct i40e_veb_tc_stats *veb_es; 702 int i, idx = 0; 703 704 idx = veb->stats_idx; 705 es = &veb->stats; 706 oes = &veb->stats_offsets; 707 veb_es = &veb->tc_stats; 708 veb_oes = &veb->tc_stats_offsets; 709 710 /* Gather up the stats that the hw collects */ 711 i40e_stat_update32(hw, I40E_GLSW_TDPC(idx), 712 veb->stat_offsets_loaded, 713 &oes->tx_discards, &es->tx_discards); 714 if (hw->revision_id > 0) 715 i40e_stat_update32(hw, I40E_GLSW_RUPP(idx), 716 veb->stat_offsets_loaded, 717 &oes->rx_unknown_protocol, 718 &es->rx_unknown_protocol); 719 i40e_stat_update48(hw, I40E_GLSW_GORCH(idx), I40E_GLSW_GORCL(idx), 720 veb->stat_offsets_loaded, 721 &oes->rx_bytes, &es->rx_bytes); 722 i40e_stat_update48(hw, I40E_GLSW_UPRCH(idx), I40E_GLSW_UPRCL(idx), 723 veb->stat_offsets_loaded, 724 &oes->rx_unicast, &es->rx_unicast); 725 i40e_stat_update48(hw, I40E_GLSW_MPRCH(idx), I40E_GLSW_MPRCL(idx), 726 veb->stat_offsets_loaded, 727 &oes->rx_multicast, &es->rx_multicast); 728 i40e_stat_update48(hw, I40E_GLSW_BPRCH(idx), I40E_GLSW_BPRCL(idx), 729 veb->stat_offsets_loaded, 730 &oes->rx_broadcast, &es->rx_broadcast); 731 732 i40e_stat_update48(hw, I40E_GLSW_GOTCH(idx), I40E_GLSW_GOTCL(idx), 733 veb->stat_offsets_loaded, 734 &oes->tx_bytes, &es->tx_bytes); 735 i40e_stat_update48(hw, I40E_GLSW_UPTCH(idx), I40E_GLSW_UPTCL(idx), 736 veb->stat_offsets_loaded, 737 &oes->tx_unicast, &es->tx_unicast); 738 i40e_stat_update48(hw, I40E_GLSW_MPTCH(idx), I40E_GLSW_MPTCL(idx), 739 veb->stat_offsets_loaded, 740 &oes->tx_multicast, &es->tx_multicast); 741 i40e_stat_update48(hw, I40E_GLSW_BPTCH(idx), I40E_GLSW_BPTCL(idx), 742 veb->stat_offsets_loaded, 743 &oes->tx_broadcast, &es->tx_broadcast); 744 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 745 i40e_stat_update48(hw, I40E_GLVEBTC_RPCH(i, idx), 746 I40E_GLVEBTC_RPCL(i, idx), 747 veb->stat_offsets_loaded, 748 &veb_oes->tc_rx_packets[i], 749 &veb_es->tc_rx_packets[i]); 750 i40e_stat_update48(hw, I40E_GLVEBTC_RBCH(i, idx), 751 I40E_GLVEBTC_RBCL(i, idx), 752 veb->stat_offsets_loaded, 753 &veb_oes->tc_rx_bytes[i], 754 &veb_es->tc_rx_bytes[i]); 755 i40e_stat_update48(hw, I40E_GLVEBTC_TPCH(i, idx), 756 I40E_GLVEBTC_TPCL(i, idx), 757 veb->stat_offsets_loaded, 758 &veb_oes->tc_tx_packets[i], 759 &veb_es->tc_tx_packets[i]); 760 i40e_stat_update48(hw, I40E_GLVEBTC_TBCH(i, idx), 761 I40E_GLVEBTC_TBCL(i, idx), 762 veb->stat_offsets_loaded, 763 &veb_oes->tc_tx_bytes[i], 764 &veb_es->tc_tx_bytes[i]); 765 } 766 veb->stat_offsets_loaded = true; 767 } 768 769 /** 770 * i40e_update_vsi_stats - Update the vsi statistics counters. 771 * @vsi: the VSI to be updated 772 * 773 * There are a few instances where we store the same stat in a 774 * couple of different structs. This is partly because we have 775 * the netdev stats that need to be filled out, which is slightly 776 * different from the "eth_stats" defined by the chip and used in 777 * VF communications. We sort it out here. 778 **/ 779 static void i40e_update_vsi_stats(struct i40e_vsi *vsi) 780 { 781 struct i40e_pf *pf = vsi->back; 782 struct rtnl_link_stats64 *ons; 783 struct rtnl_link_stats64 *ns; /* netdev stats */ 784 struct i40e_eth_stats *oes; 785 struct i40e_eth_stats *es; /* device's eth stats */ 786 u32 tx_restart, tx_busy; 787 struct i40e_ring *p; 788 u32 rx_page, rx_buf; 789 u64 bytes, packets; 790 unsigned int start; 791 u64 tx_linearize; 792 u64 tx_force_wb; 793 u64 rx_p, rx_b; 794 u64 tx_p, tx_b; 795 u16 q; 796 797 if (test_bit(__I40E_VSI_DOWN, vsi->state) || 798 test_bit(__I40E_CONFIG_BUSY, pf->state)) 799 return; 800 801 ns = i40e_get_vsi_stats_struct(vsi); 802 ons = &vsi->net_stats_offsets; 803 es = &vsi->eth_stats; 804 oes = &vsi->eth_stats_offsets; 805 806 /* Gather up the netdev and vsi stats that the driver collects 807 * on the fly during packet processing 808 */ 809 rx_b = rx_p = 0; 810 tx_b = tx_p = 0; 811 tx_restart = tx_busy = tx_linearize = tx_force_wb = 0; 812 rx_page = 0; 813 rx_buf = 0; 814 rcu_read_lock(); 815 for (q = 0; q < vsi->num_queue_pairs; q++) { 816 /* locate Tx ring */ 817 p = READ_ONCE(vsi->tx_rings[q]); 818 819 do { 820 start = u64_stats_fetch_begin_irq(&p->syncp); 821 packets = p->stats.packets; 822 bytes = p->stats.bytes; 823 } while (u64_stats_fetch_retry_irq(&p->syncp, start)); 824 tx_b += bytes; 825 tx_p += packets; 826 tx_restart += p->tx_stats.restart_queue; 827 tx_busy += p->tx_stats.tx_busy; 828 tx_linearize += p->tx_stats.tx_linearize; 829 tx_force_wb += p->tx_stats.tx_force_wb; 830 831 /* Rx queue is part of the same block as Tx queue */ 832 p = &p[1]; 833 do { 834 start = u64_stats_fetch_begin_irq(&p->syncp); 835 packets = p->stats.packets; 836 bytes = p->stats.bytes; 837 } while (u64_stats_fetch_retry_irq(&p->syncp, start)); 838 rx_b += bytes; 839 rx_p += packets; 840 rx_buf += p->rx_stats.alloc_buff_failed; 841 rx_page += p->rx_stats.alloc_page_failed; 842 } 843 rcu_read_unlock(); 844 vsi->tx_restart = tx_restart; 845 vsi->tx_busy = tx_busy; 846 vsi->tx_linearize = tx_linearize; 847 vsi->tx_force_wb = tx_force_wb; 848 vsi->rx_page_failed = rx_page; 849 vsi->rx_buf_failed = rx_buf; 850 851 ns->rx_packets = rx_p; 852 ns->rx_bytes = rx_b; 853 ns->tx_packets = tx_p; 854 ns->tx_bytes = tx_b; 855 856 /* update netdev stats from eth stats */ 857 i40e_update_eth_stats(vsi); 858 ons->tx_errors = oes->tx_errors; 859 ns->tx_errors = es->tx_errors; 860 ons->multicast = oes->rx_multicast; 861 ns->multicast = es->rx_multicast; 862 ons->rx_dropped = oes->rx_discards; 863 ns->rx_dropped = es->rx_discards; 864 ons->tx_dropped = oes->tx_discards; 865 ns->tx_dropped = es->tx_discards; 866 867 /* pull in a couple PF stats if this is the main vsi */ 868 if (vsi == pf->vsi[pf->lan_vsi]) { 869 ns->rx_crc_errors = pf->stats.crc_errors; 870 ns->rx_errors = pf->stats.crc_errors + pf->stats.illegal_bytes; 871 ns->rx_length_errors = pf->stats.rx_length_errors; 872 } 873 } 874 875 /** 876 * i40e_update_pf_stats - Update the PF statistics counters. 877 * @pf: the PF to be updated 878 **/ 879 static void i40e_update_pf_stats(struct i40e_pf *pf) 880 { 881 struct i40e_hw_port_stats *osd = &pf->stats_offsets; 882 struct i40e_hw_port_stats *nsd = &pf->stats; 883 struct i40e_hw *hw = &pf->hw; 884 u32 val; 885 int i; 886 887 i40e_stat_update48(hw, I40E_GLPRT_GORCH(hw->port), 888 I40E_GLPRT_GORCL(hw->port), 889 pf->stat_offsets_loaded, 890 &osd->eth.rx_bytes, &nsd->eth.rx_bytes); 891 i40e_stat_update48(hw, I40E_GLPRT_GOTCH(hw->port), 892 I40E_GLPRT_GOTCL(hw->port), 893 pf->stat_offsets_loaded, 894 &osd->eth.tx_bytes, &nsd->eth.tx_bytes); 895 i40e_stat_update32(hw, I40E_GLPRT_RDPC(hw->port), 896 pf->stat_offsets_loaded, 897 &osd->eth.rx_discards, 898 &nsd->eth.rx_discards); 899 i40e_stat_update48(hw, I40E_GLPRT_UPRCH(hw->port), 900 I40E_GLPRT_UPRCL(hw->port), 901 pf->stat_offsets_loaded, 902 &osd->eth.rx_unicast, 903 &nsd->eth.rx_unicast); 904 i40e_stat_update48(hw, I40E_GLPRT_MPRCH(hw->port), 905 I40E_GLPRT_MPRCL(hw->port), 906 pf->stat_offsets_loaded, 907 &osd->eth.rx_multicast, 908 &nsd->eth.rx_multicast); 909 i40e_stat_update48(hw, I40E_GLPRT_BPRCH(hw->port), 910 I40E_GLPRT_BPRCL(hw->port), 911 pf->stat_offsets_loaded, 912 &osd->eth.rx_broadcast, 913 &nsd->eth.rx_broadcast); 914 i40e_stat_update48(hw, I40E_GLPRT_UPTCH(hw->port), 915 I40E_GLPRT_UPTCL(hw->port), 916 pf->stat_offsets_loaded, 917 &osd->eth.tx_unicast, 918 &nsd->eth.tx_unicast); 919 i40e_stat_update48(hw, I40E_GLPRT_MPTCH(hw->port), 920 I40E_GLPRT_MPTCL(hw->port), 921 pf->stat_offsets_loaded, 922 &osd->eth.tx_multicast, 923 &nsd->eth.tx_multicast); 924 i40e_stat_update48(hw, I40E_GLPRT_BPTCH(hw->port), 925 I40E_GLPRT_BPTCL(hw->port), 926 pf->stat_offsets_loaded, 927 &osd->eth.tx_broadcast, 928 &nsd->eth.tx_broadcast); 929 930 i40e_stat_update32(hw, I40E_GLPRT_TDOLD(hw->port), 931 pf->stat_offsets_loaded, 932 &osd->tx_dropped_link_down, 933 &nsd->tx_dropped_link_down); 934 935 i40e_stat_update32(hw, I40E_GLPRT_CRCERRS(hw->port), 936 pf->stat_offsets_loaded, 937 &osd->crc_errors, &nsd->crc_errors); 938 939 i40e_stat_update32(hw, I40E_GLPRT_ILLERRC(hw->port), 940 pf->stat_offsets_loaded, 941 &osd->illegal_bytes, &nsd->illegal_bytes); 942 943 i40e_stat_update32(hw, I40E_GLPRT_MLFC(hw->port), 944 pf->stat_offsets_loaded, 945 &osd->mac_local_faults, 946 &nsd->mac_local_faults); 947 i40e_stat_update32(hw, I40E_GLPRT_MRFC(hw->port), 948 pf->stat_offsets_loaded, 949 &osd->mac_remote_faults, 950 &nsd->mac_remote_faults); 951 952 i40e_stat_update32(hw, I40E_GLPRT_RLEC(hw->port), 953 pf->stat_offsets_loaded, 954 &osd->rx_length_errors, 955 &nsd->rx_length_errors); 956 957 i40e_stat_update32(hw, I40E_GLPRT_LXONRXC(hw->port), 958 pf->stat_offsets_loaded, 959 &osd->link_xon_rx, &nsd->link_xon_rx); 960 i40e_stat_update32(hw, I40E_GLPRT_LXONTXC(hw->port), 961 pf->stat_offsets_loaded, 962 &osd->link_xon_tx, &nsd->link_xon_tx); 963 i40e_stat_update32(hw, I40E_GLPRT_LXOFFRXC(hw->port), 964 pf->stat_offsets_loaded, 965 &osd->link_xoff_rx, &nsd->link_xoff_rx); 966 i40e_stat_update32(hw, I40E_GLPRT_LXOFFTXC(hw->port), 967 pf->stat_offsets_loaded, 968 &osd->link_xoff_tx, &nsd->link_xoff_tx); 969 970 for (i = 0; i < 8; i++) { 971 i40e_stat_update32(hw, I40E_GLPRT_PXOFFRXC(hw->port, i), 972 pf->stat_offsets_loaded, 973 &osd->priority_xoff_rx[i], 974 &nsd->priority_xoff_rx[i]); 975 i40e_stat_update32(hw, I40E_GLPRT_PXONRXC(hw->port, i), 976 pf->stat_offsets_loaded, 977 &osd->priority_xon_rx[i], 978 &nsd->priority_xon_rx[i]); 979 i40e_stat_update32(hw, I40E_GLPRT_PXONTXC(hw->port, i), 980 pf->stat_offsets_loaded, 981 &osd->priority_xon_tx[i], 982 &nsd->priority_xon_tx[i]); 983 i40e_stat_update32(hw, I40E_GLPRT_PXOFFTXC(hw->port, i), 984 pf->stat_offsets_loaded, 985 &osd->priority_xoff_tx[i], 986 &nsd->priority_xoff_tx[i]); 987 i40e_stat_update32(hw, 988 I40E_GLPRT_RXON2OFFCNT(hw->port, i), 989 pf->stat_offsets_loaded, 990 &osd->priority_xon_2_xoff[i], 991 &nsd->priority_xon_2_xoff[i]); 992 } 993 994 i40e_stat_update48(hw, I40E_GLPRT_PRC64H(hw->port), 995 I40E_GLPRT_PRC64L(hw->port), 996 pf->stat_offsets_loaded, 997 &osd->rx_size_64, &nsd->rx_size_64); 998 i40e_stat_update48(hw, I40E_GLPRT_PRC127H(hw->port), 999 I40E_GLPRT_PRC127L(hw->port), 1000 pf->stat_offsets_loaded, 1001 &osd->rx_size_127, &nsd->rx_size_127); 1002 i40e_stat_update48(hw, I40E_GLPRT_PRC255H(hw->port), 1003 I40E_GLPRT_PRC255L(hw->port), 1004 pf->stat_offsets_loaded, 1005 &osd->rx_size_255, &nsd->rx_size_255); 1006 i40e_stat_update48(hw, I40E_GLPRT_PRC511H(hw->port), 1007 I40E_GLPRT_PRC511L(hw->port), 1008 pf->stat_offsets_loaded, 1009 &osd->rx_size_511, &nsd->rx_size_511); 1010 i40e_stat_update48(hw, I40E_GLPRT_PRC1023H(hw->port), 1011 I40E_GLPRT_PRC1023L(hw->port), 1012 pf->stat_offsets_loaded, 1013 &osd->rx_size_1023, &nsd->rx_size_1023); 1014 i40e_stat_update48(hw, I40E_GLPRT_PRC1522H(hw->port), 1015 I40E_GLPRT_PRC1522L(hw->port), 1016 pf->stat_offsets_loaded, 1017 &osd->rx_size_1522, &nsd->rx_size_1522); 1018 i40e_stat_update48(hw, I40E_GLPRT_PRC9522H(hw->port), 1019 I40E_GLPRT_PRC9522L(hw->port), 1020 pf->stat_offsets_loaded, 1021 &osd->rx_size_big, &nsd->rx_size_big); 1022 1023 i40e_stat_update48(hw, I40E_GLPRT_PTC64H(hw->port), 1024 I40E_GLPRT_PTC64L(hw->port), 1025 pf->stat_offsets_loaded, 1026 &osd->tx_size_64, &nsd->tx_size_64); 1027 i40e_stat_update48(hw, I40E_GLPRT_PTC127H(hw->port), 1028 I40E_GLPRT_PTC127L(hw->port), 1029 pf->stat_offsets_loaded, 1030 &osd->tx_size_127, &nsd->tx_size_127); 1031 i40e_stat_update48(hw, I40E_GLPRT_PTC255H(hw->port), 1032 I40E_GLPRT_PTC255L(hw->port), 1033 pf->stat_offsets_loaded, 1034 &osd->tx_size_255, &nsd->tx_size_255); 1035 i40e_stat_update48(hw, I40E_GLPRT_PTC511H(hw->port), 1036 I40E_GLPRT_PTC511L(hw->port), 1037 pf->stat_offsets_loaded, 1038 &osd->tx_size_511, &nsd->tx_size_511); 1039 i40e_stat_update48(hw, I40E_GLPRT_PTC1023H(hw->port), 1040 I40E_GLPRT_PTC1023L(hw->port), 1041 pf->stat_offsets_loaded, 1042 &osd->tx_size_1023, &nsd->tx_size_1023); 1043 i40e_stat_update48(hw, I40E_GLPRT_PTC1522H(hw->port), 1044 I40E_GLPRT_PTC1522L(hw->port), 1045 pf->stat_offsets_loaded, 1046 &osd->tx_size_1522, &nsd->tx_size_1522); 1047 i40e_stat_update48(hw, I40E_GLPRT_PTC9522H(hw->port), 1048 I40E_GLPRT_PTC9522L(hw->port), 1049 pf->stat_offsets_loaded, 1050 &osd->tx_size_big, &nsd->tx_size_big); 1051 1052 i40e_stat_update32(hw, I40E_GLPRT_RUC(hw->port), 1053 pf->stat_offsets_loaded, 1054 &osd->rx_undersize, &nsd->rx_undersize); 1055 i40e_stat_update32(hw, I40E_GLPRT_RFC(hw->port), 1056 pf->stat_offsets_loaded, 1057 &osd->rx_fragments, &nsd->rx_fragments); 1058 i40e_stat_update32(hw, I40E_GLPRT_ROC(hw->port), 1059 pf->stat_offsets_loaded, 1060 &osd->rx_oversize, &nsd->rx_oversize); 1061 i40e_stat_update32(hw, I40E_GLPRT_RJC(hw->port), 1062 pf->stat_offsets_loaded, 1063 &osd->rx_jabber, &nsd->rx_jabber); 1064 1065 /* FDIR stats */ 1066 i40e_stat_update_and_clear32(hw, 1067 I40E_GLQF_PCNT(I40E_FD_ATR_STAT_IDX(hw->pf_id)), 1068 &nsd->fd_atr_match); 1069 i40e_stat_update_and_clear32(hw, 1070 I40E_GLQF_PCNT(I40E_FD_SB_STAT_IDX(hw->pf_id)), 1071 &nsd->fd_sb_match); 1072 i40e_stat_update_and_clear32(hw, 1073 I40E_GLQF_PCNT(I40E_FD_ATR_TUNNEL_STAT_IDX(hw->pf_id)), 1074 &nsd->fd_atr_tunnel_match); 1075 1076 val = rd32(hw, I40E_PRTPM_EEE_STAT); 1077 nsd->tx_lpi_status = 1078 (val & I40E_PRTPM_EEE_STAT_TX_LPI_STATUS_MASK) >> 1079 I40E_PRTPM_EEE_STAT_TX_LPI_STATUS_SHIFT; 1080 nsd->rx_lpi_status = 1081 (val & I40E_PRTPM_EEE_STAT_RX_LPI_STATUS_MASK) >> 1082 I40E_PRTPM_EEE_STAT_RX_LPI_STATUS_SHIFT; 1083 i40e_stat_update32(hw, I40E_PRTPM_TLPIC, 1084 pf->stat_offsets_loaded, 1085 &osd->tx_lpi_count, &nsd->tx_lpi_count); 1086 i40e_stat_update32(hw, I40E_PRTPM_RLPIC, 1087 pf->stat_offsets_loaded, 1088 &osd->rx_lpi_count, &nsd->rx_lpi_count); 1089 1090 if (pf->flags & I40E_FLAG_FD_SB_ENABLED && 1091 !(pf->flags & I40E_FLAG_FD_SB_AUTO_DISABLED)) 1092 nsd->fd_sb_status = true; 1093 else 1094 nsd->fd_sb_status = false; 1095 1096 if (pf->flags & I40E_FLAG_FD_ATR_ENABLED && 1097 !(pf->flags & I40E_FLAG_FD_ATR_AUTO_DISABLED)) 1098 nsd->fd_atr_status = true; 1099 else 1100 nsd->fd_atr_status = false; 1101 1102 pf->stat_offsets_loaded = true; 1103 } 1104 1105 /** 1106 * i40e_update_stats - Update the various statistics counters. 1107 * @vsi: the VSI to be updated 1108 * 1109 * Update the various stats for this VSI and its related entities. 1110 **/ 1111 void i40e_update_stats(struct i40e_vsi *vsi) 1112 { 1113 struct i40e_pf *pf = vsi->back; 1114 1115 if (vsi == pf->vsi[pf->lan_vsi]) 1116 i40e_update_pf_stats(pf); 1117 1118 i40e_update_vsi_stats(vsi); 1119 } 1120 1121 /** 1122 * i40e_find_filter - Search VSI filter list for specific mac/vlan filter 1123 * @vsi: the VSI to be searched 1124 * @macaddr: the MAC address 1125 * @vlan: the vlan 1126 * 1127 * Returns ptr to the filter object or NULL 1128 **/ 1129 static struct i40e_mac_filter *i40e_find_filter(struct i40e_vsi *vsi, 1130 const u8 *macaddr, s16 vlan) 1131 { 1132 struct i40e_mac_filter *f; 1133 u64 key; 1134 1135 if (!vsi || !macaddr) 1136 return NULL; 1137 1138 key = i40e_addr_to_hkey(macaddr); 1139 hash_for_each_possible(vsi->mac_filter_hash, f, hlist, key) { 1140 if ((ether_addr_equal(macaddr, f->macaddr)) && 1141 (vlan == f->vlan)) 1142 return f; 1143 } 1144 return NULL; 1145 } 1146 1147 /** 1148 * i40e_find_mac - Find a mac addr in the macvlan filters list 1149 * @vsi: the VSI to be searched 1150 * @macaddr: the MAC address we are searching for 1151 * 1152 * Returns the first filter with the provided MAC address or NULL if 1153 * MAC address was not found 1154 **/ 1155 struct i40e_mac_filter *i40e_find_mac(struct i40e_vsi *vsi, const u8 *macaddr) 1156 { 1157 struct i40e_mac_filter *f; 1158 u64 key; 1159 1160 if (!vsi || !macaddr) 1161 return NULL; 1162 1163 key = i40e_addr_to_hkey(macaddr); 1164 hash_for_each_possible(vsi->mac_filter_hash, f, hlist, key) { 1165 if ((ether_addr_equal(macaddr, f->macaddr))) 1166 return f; 1167 } 1168 return NULL; 1169 } 1170 1171 /** 1172 * i40e_is_vsi_in_vlan - Check if VSI is in vlan mode 1173 * @vsi: the VSI to be searched 1174 * 1175 * Returns true if VSI is in vlan mode or false otherwise 1176 **/ 1177 bool i40e_is_vsi_in_vlan(struct i40e_vsi *vsi) 1178 { 1179 /* If we have a PVID, always operate in VLAN mode */ 1180 if (vsi->info.pvid) 1181 return true; 1182 1183 /* We need to operate in VLAN mode whenever we have any filters with 1184 * a VLAN other than I40E_VLAN_ALL. We could check the table each 1185 * time, incurring search cost repeatedly. However, we can notice two 1186 * things: 1187 * 1188 * 1) the only place where we can gain a VLAN filter is in 1189 * i40e_add_filter. 1190 * 1191 * 2) the only place where filters are actually removed is in 1192 * i40e_sync_filters_subtask. 1193 * 1194 * Thus, we can simply use a boolean value, has_vlan_filters which we 1195 * will set to true when we add a VLAN filter in i40e_add_filter. Then 1196 * we have to perform the full search after deleting filters in 1197 * i40e_sync_filters_subtask, but we already have to search 1198 * filters here and can perform the check at the same time. This 1199 * results in avoiding embedding a loop for VLAN mode inside another 1200 * loop over all the filters, and should maintain correctness as noted 1201 * above. 1202 */ 1203 return vsi->has_vlan_filter; 1204 } 1205 1206 /** 1207 * i40e_correct_mac_vlan_filters - Correct non-VLAN filters if necessary 1208 * @vsi: the VSI to configure 1209 * @tmp_add_list: list of filters ready to be added 1210 * @tmp_del_list: list of filters ready to be deleted 1211 * @vlan_filters: the number of active VLAN filters 1212 * 1213 * Update VLAN=0 and VLAN=-1 (I40E_VLAN_ANY) filters properly so that they 1214 * behave as expected. If we have any active VLAN filters remaining or about 1215 * to be added then we need to update non-VLAN filters to be marked as VLAN=0 1216 * so that they only match against untagged traffic. If we no longer have any 1217 * active VLAN filters, we need to make all non-VLAN filters marked as VLAN=-1 1218 * so that they match against both tagged and untagged traffic. In this way, 1219 * we ensure that we correctly receive the desired traffic. This ensures that 1220 * when we have an active VLAN we will receive only untagged traffic and 1221 * traffic matching active VLANs. If we have no active VLANs then we will 1222 * operate in non-VLAN mode and receive all traffic, tagged or untagged. 1223 * 1224 * Finally, in a similar fashion, this function also corrects filters when 1225 * there is an active PVID assigned to this VSI. 1226 * 1227 * In case of memory allocation failure return -ENOMEM. Otherwise, return 0. 1228 * 1229 * This function is only expected to be called from within 1230 * i40e_sync_vsi_filters. 1231 * 1232 * NOTE: This function expects to be called while under the 1233 * mac_filter_hash_lock 1234 */ 1235 static int i40e_correct_mac_vlan_filters(struct i40e_vsi *vsi, 1236 struct hlist_head *tmp_add_list, 1237 struct hlist_head *tmp_del_list, 1238 int vlan_filters) 1239 { 1240 s16 pvid = le16_to_cpu(vsi->info.pvid); 1241 struct i40e_mac_filter *f, *add_head; 1242 struct i40e_new_mac_filter *new; 1243 struct hlist_node *h; 1244 int bkt, new_vlan; 1245 1246 /* To determine if a particular filter needs to be replaced we 1247 * have the three following conditions: 1248 * 1249 * a) if we have a PVID assigned, then all filters which are 1250 * not marked as VLAN=PVID must be replaced with filters that 1251 * are. 1252 * b) otherwise, if we have any active VLANS, all filters 1253 * which are marked as VLAN=-1 must be replaced with 1254 * filters marked as VLAN=0 1255 * c) finally, if we do not have any active VLANS, all filters 1256 * which are marked as VLAN=0 must be replaced with filters 1257 * marked as VLAN=-1 1258 */ 1259 1260 /* Update the filters about to be added in place */ 1261 hlist_for_each_entry(new, tmp_add_list, hlist) { 1262 if (pvid && new->f->vlan != pvid) 1263 new->f->vlan = pvid; 1264 else if (vlan_filters && new->f->vlan == I40E_VLAN_ANY) 1265 new->f->vlan = 0; 1266 else if (!vlan_filters && new->f->vlan == 0) 1267 new->f->vlan = I40E_VLAN_ANY; 1268 } 1269 1270 /* Update the remaining active filters */ 1271 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) { 1272 /* Combine the checks for whether a filter needs to be changed 1273 * and then determine the new VLAN inside the if block, in 1274 * order to avoid duplicating code for adding the new filter 1275 * then deleting the old filter. 1276 */ 1277 if ((pvid && f->vlan != pvid) || 1278 (vlan_filters && f->vlan == I40E_VLAN_ANY) || 1279 (!vlan_filters && f->vlan == 0)) { 1280 /* Determine the new vlan we will be adding */ 1281 if (pvid) 1282 new_vlan = pvid; 1283 else if (vlan_filters) 1284 new_vlan = 0; 1285 else 1286 new_vlan = I40E_VLAN_ANY; 1287 1288 /* Create the new filter */ 1289 add_head = i40e_add_filter(vsi, f->macaddr, new_vlan); 1290 if (!add_head) 1291 return -ENOMEM; 1292 1293 /* Create a temporary i40e_new_mac_filter */ 1294 new = kzalloc(sizeof(*new), GFP_ATOMIC); 1295 if (!new) 1296 return -ENOMEM; 1297 1298 new->f = add_head; 1299 new->state = add_head->state; 1300 1301 /* Add the new filter to the tmp list */ 1302 hlist_add_head(&new->hlist, tmp_add_list); 1303 1304 /* Put the original filter into the delete list */ 1305 f->state = I40E_FILTER_REMOVE; 1306 hash_del(&f->hlist); 1307 hlist_add_head(&f->hlist, tmp_del_list); 1308 } 1309 } 1310 1311 vsi->has_vlan_filter = !!vlan_filters; 1312 1313 return 0; 1314 } 1315 1316 /** 1317 * i40e_rm_default_mac_filter - Remove the default MAC filter set by NVM 1318 * @vsi: the PF Main VSI - inappropriate for any other VSI 1319 * @macaddr: the MAC address 1320 * 1321 * Remove whatever filter the firmware set up so the driver can manage 1322 * its own filtering intelligently. 1323 **/ 1324 static void i40e_rm_default_mac_filter(struct i40e_vsi *vsi, u8 *macaddr) 1325 { 1326 struct i40e_aqc_remove_macvlan_element_data element; 1327 struct i40e_pf *pf = vsi->back; 1328 1329 /* Only appropriate for the PF main VSI */ 1330 if (vsi->type != I40E_VSI_MAIN) 1331 return; 1332 1333 memset(&element, 0, sizeof(element)); 1334 ether_addr_copy(element.mac_addr, macaddr); 1335 element.vlan_tag = 0; 1336 /* Ignore error returns, some firmware does it this way... */ 1337 element.flags = I40E_AQC_MACVLAN_DEL_PERFECT_MATCH; 1338 i40e_aq_remove_macvlan(&pf->hw, vsi->seid, &element, 1, NULL); 1339 1340 memset(&element, 0, sizeof(element)); 1341 ether_addr_copy(element.mac_addr, macaddr); 1342 element.vlan_tag = 0; 1343 /* ...and some firmware does it this way. */ 1344 element.flags = I40E_AQC_MACVLAN_DEL_PERFECT_MATCH | 1345 I40E_AQC_MACVLAN_DEL_IGNORE_VLAN; 1346 i40e_aq_remove_macvlan(&pf->hw, vsi->seid, &element, 1, NULL); 1347 } 1348 1349 /** 1350 * i40e_add_filter - Add a mac/vlan filter to the VSI 1351 * @vsi: the VSI to be searched 1352 * @macaddr: the MAC address 1353 * @vlan: the vlan 1354 * 1355 * Returns ptr to the filter object or NULL when no memory available. 1356 * 1357 * NOTE: This function is expected to be called with mac_filter_hash_lock 1358 * being held. 1359 **/ 1360 struct i40e_mac_filter *i40e_add_filter(struct i40e_vsi *vsi, 1361 const u8 *macaddr, s16 vlan) 1362 { 1363 struct i40e_mac_filter *f; 1364 u64 key; 1365 1366 if (!vsi || !macaddr) 1367 return NULL; 1368 1369 f = i40e_find_filter(vsi, macaddr, vlan); 1370 if (!f) { 1371 f = kzalloc(sizeof(*f), GFP_ATOMIC); 1372 if (!f) 1373 return NULL; 1374 1375 /* Update the boolean indicating if we need to function in 1376 * VLAN mode. 1377 */ 1378 if (vlan >= 0) 1379 vsi->has_vlan_filter = true; 1380 1381 ether_addr_copy(f->macaddr, macaddr); 1382 f->vlan = vlan; 1383 /* If we're in overflow promisc mode, set the state directly 1384 * to failed, so we don't bother to try sending the filter 1385 * to the hardware. 1386 */ 1387 if (test_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state)) 1388 f->state = I40E_FILTER_FAILED; 1389 else 1390 f->state = I40E_FILTER_NEW; 1391 INIT_HLIST_NODE(&f->hlist); 1392 1393 key = i40e_addr_to_hkey(macaddr); 1394 hash_add(vsi->mac_filter_hash, &f->hlist, key); 1395 1396 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED; 1397 vsi->back->flags |= I40E_FLAG_FILTER_SYNC; 1398 } 1399 1400 /* If we're asked to add a filter that has been marked for removal, it 1401 * is safe to simply restore it to active state. __i40e_del_filter 1402 * will have simply deleted any filters which were previously marked 1403 * NEW or FAILED, so if it is currently marked REMOVE it must have 1404 * previously been ACTIVE. Since we haven't yet run the sync filters 1405 * task, just restore this filter to the ACTIVE state so that the 1406 * sync task leaves it in place 1407 */ 1408 if (f->state == I40E_FILTER_REMOVE) 1409 f->state = I40E_FILTER_ACTIVE; 1410 1411 return f; 1412 } 1413 1414 /** 1415 * __i40e_del_filter - Remove a specific filter from the VSI 1416 * @vsi: VSI to remove from 1417 * @f: the filter to remove from the list 1418 * 1419 * This function should be called instead of i40e_del_filter only if you know 1420 * the exact filter you will remove already, such as via i40e_find_filter or 1421 * i40e_find_mac. 1422 * 1423 * NOTE: This function is expected to be called with mac_filter_hash_lock 1424 * being held. 1425 * ANOTHER NOTE: This function MUST be called from within the context of 1426 * the "safe" variants of any list iterators, e.g. list_for_each_entry_safe() 1427 * instead of list_for_each_entry(). 1428 **/ 1429 void __i40e_del_filter(struct i40e_vsi *vsi, struct i40e_mac_filter *f) 1430 { 1431 if (!f) 1432 return; 1433 1434 /* If the filter was never added to firmware then we can just delete it 1435 * directly and we don't want to set the status to remove or else an 1436 * admin queue command will unnecessarily fire. 1437 */ 1438 if ((f->state == I40E_FILTER_FAILED) || 1439 (f->state == I40E_FILTER_NEW)) { 1440 hash_del(&f->hlist); 1441 kfree(f); 1442 } else { 1443 f->state = I40E_FILTER_REMOVE; 1444 } 1445 1446 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED; 1447 vsi->back->flags |= I40E_FLAG_FILTER_SYNC; 1448 } 1449 1450 /** 1451 * i40e_del_filter - Remove a MAC/VLAN filter from the VSI 1452 * @vsi: the VSI to be searched 1453 * @macaddr: the MAC address 1454 * @vlan: the VLAN 1455 * 1456 * NOTE: This function is expected to be called with mac_filter_hash_lock 1457 * being held. 1458 * ANOTHER NOTE: This function MUST be called from within the context of 1459 * the "safe" variants of any list iterators, e.g. list_for_each_entry_safe() 1460 * instead of list_for_each_entry(). 1461 **/ 1462 void i40e_del_filter(struct i40e_vsi *vsi, const u8 *macaddr, s16 vlan) 1463 { 1464 struct i40e_mac_filter *f; 1465 1466 if (!vsi || !macaddr) 1467 return; 1468 1469 f = i40e_find_filter(vsi, macaddr, vlan); 1470 __i40e_del_filter(vsi, f); 1471 } 1472 1473 /** 1474 * i40e_add_mac_filter - Add a MAC filter for all active VLANs 1475 * @vsi: the VSI to be searched 1476 * @macaddr: the mac address to be filtered 1477 * 1478 * If we're not in VLAN mode, just add the filter to I40E_VLAN_ANY. Otherwise, 1479 * go through all the macvlan filters and add a macvlan filter for each 1480 * unique vlan that already exists. If a PVID has been assigned, instead only 1481 * add the macaddr to that VLAN. 1482 * 1483 * Returns last filter added on success, else NULL 1484 **/ 1485 struct i40e_mac_filter *i40e_add_mac_filter(struct i40e_vsi *vsi, 1486 const u8 *macaddr) 1487 { 1488 struct i40e_mac_filter *f, *add = NULL; 1489 struct hlist_node *h; 1490 int bkt; 1491 1492 if (vsi->info.pvid) 1493 return i40e_add_filter(vsi, macaddr, 1494 le16_to_cpu(vsi->info.pvid)); 1495 1496 if (!i40e_is_vsi_in_vlan(vsi)) 1497 return i40e_add_filter(vsi, macaddr, I40E_VLAN_ANY); 1498 1499 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) { 1500 if (f->state == I40E_FILTER_REMOVE) 1501 continue; 1502 add = i40e_add_filter(vsi, macaddr, f->vlan); 1503 if (!add) 1504 return NULL; 1505 } 1506 1507 return add; 1508 } 1509 1510 /** 1511 * i40e_del_mac_filter - Remove a MAC filter from all VLANs 1512 * @vsi: the VSI to be searched 1513 * @macaddr: the mac address to be removed 1514 * 1515 * Removes a given MAC address from a VSI regardless of what VLAN it has been 1516 * associated with. 1517 * 1518 * Returns 0 for success, or error 1519 **/ 1520 int i40e_del_mac_filter(struct i40e_vsi *vsi, const u8 *macaddr) 1521 { 1522 struct i40e_mac_filter *f; 1523 struct hlist_node *h; 1524 bool found = false; 1525 int bkt; 1526 1527 WARN(!spin_is_locked(&vsi->mac_filter_hash_lock), 1528 "Missing mac_filter_hash_lock\n"); 1529 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) { 1530 if (ether_addr_equal(macaddr, f->macaddr)) { 1531 __i40e_del_filter(vsi, f); 1532 found = true; 1533 } 1534 } 1535 1536 if (found) 1537 return 0; 1538 else 1539 return -ENOENT; 1540 } 1541 1542 /** 1543 * i40e_set_mac - NDO callback to set mac address 1544 * @netdev: network interface device structure 1545 * @p: pointer to an address structure 1546 * 1547 * Returns 0 on success, negative on failure 1548 **/ 1549 static int i40e_set_mac(struct net_device *netdev, void *p) 1550 { 1551 struct i40e_netdev_priv *np = netdev_priv(netdev); 1552 struct i40e_vsi *vsi = np->vsi; 1553 struct i40e_pf *pf = vsi->back; 1554 struct i40e_hw *hw = &pf->hw; 1555 struct sockaddr *addr = p; 1556 1557 if (!is_valid_ether_addr(addr->sa_data)) 1558 return -EADDRNOTAVAIL; 1559 1560 if (ether_addr_equal(netdev->dev_addr, addr->sa_data)) { 1561 netdev_info(netdev, "already using mac address %pM\n", 1562 addr->sa_data); 1563 return 0; 1564 } 1565 1566 if (test_bit(__I40E_VSI_DOWN, vsi->back->state) || 1567 test_bit(__I40E_RESET_RECOVERY_PENDING, vsi->back->state)) 1568 return -EADDRNOTAVAIL; 1569 1570 if (ether_addr_equal(hw->mac.addr, addr->sa_data)) 1571 netdev_info(netdev, "returning to hw mac address %pM\n", 1572 hw->mac.addr); 1573 else 1574 netdev_info(netdev, "set new mac address %pM\n", addr->sa_data); 1575 1576 /* Copy the address first, so that we avoid a possible race with 1577 * .set_rx_mode(). If we copy after changing the address in the filter 1578 * list, we might open ourselves to a narrow race window where 1579 * .set_rx_mode could delete our dev_addr filter and prevent traffic 1580 * from passing. 1581 */ 1582 ether_addr_copy(netdev->dev_addr, addr->sa_data); 1583 1584 spin_lock_bh(&vsi->mac_filter_hash_lock); 1585 i40e_del_mac_filter(vsi, netdev->dev_addr); 1586 i40e_add_mac_filter(vsi, addr->sa_data); 1587 spin_unlock_bh(&vsi->mac_filter_hash_lock); 1588 if (vsi->type == I40E_VSI_MAIN) { 1589 i40e_status ret; 1590 1591 ret = i40e_aq_mac_address_write(&vsi->back->hw, 1592 I40E_AQC_WRITE_TYPE_LAA_WOL, 1593 addr->sa_data, NULL); 1594 if (ret) 1595 netdev_info(netdev, "Ignoring error from firmware on LAA update, status %s, AQ ret %s\n", 1596 i40e_stat_str(hw, ret), 1597 i40e_aq_str(hw, hw->aq.asq_last_status)); 1598 } 1599 1600 /* schedule our worker thread which will take care of 1601 * applying the new filter changes 1602 */ 1603 i40e_service_event_schedule(vsi->back); 1604 return 0; 1605 } 1606 1607 /** 1608 * i40e_config_rss_aq - Prepare for RSS using AQ commands 1609 * @vsi: vsi structure 1610 * @seed: RSS hash seed 1611 **/ 1612 static int i40e_config_rss_aq(struct i40e_vsi *vsi, const u8 *seed, 1613 u8 *lut, u16 lut_size) 1614 { 1615 struct i40e_pf *pf = vsi->back; 1616 struct i40e_hw *hw = &pf->hw; 1617 int ret = 0; 1618 1619 if (seed) { 1620 struct i40e_aqc_get_set_rss_key_data *seed_dw = 1621 (struct i40e_aqc_get_set_rss_key_data *)seed; 1622 ret = i40e_aq_set_rss_key(hw, vsi->id, seed_dw); 1623 if (ret) { 1624 dev_info(&pf->pdev->dev, 1625 "Cannot set RSS key, err %s aq_err %s\n", 1626 i40e_stat_str(hw, ret), 1627 i40e_aq_str(hw, hw->aq.asq_last_status)); 1628 return ret; 1629 } 1630 } 1631 if (lut) { 1632 bool pf_lut = vsi->type == I40E_VSI_MAIN ? true : false; 1633 1634 ret = i40e_aq_set_rss_lut(hw, vsi->id, pf_lut, lut, lut_size); 1635 if (ret) { 1636 dev_info(&pf->pdev->dev, 1637 "Cannot set RSS lut, err %s aq_err %s\n", 1638 i40e_stat_str(hw, ret), 1639 i40e_aq_str(hw, hw->aq.asq_last_status)); 1640 return ret; 1641 } 1642 } 1643 return ret; 1644 } 1645 1646 /** 1647 * i40e_vsi_config_rss - Prepare for VSI(VMDq) RSS if used 1648 * @vsi: VSI structure 1649 **/ 1650 static int i40e_vsi_config_rss(struct i40e_vsi *vsi) 1651 { 1652 struct i40e_pf *pf = vsi->back; 1653 u8 seed[I40E_HKEY_ARRAY_SIZE]; 1654 u8 *lut; 1655 int ret; 1656 1657 if (!(pf->hw_features & I40E_HW_RSS_AQ_CAPABLE)) 1658 return 0; 1659 if (!vsi->rss_size) 1660 vsi->rss_size = min_t(int, pf->alloc_rss_size, 1661 vsi->num_queue_pairs); 1662 if (!vsi->rss_size) 1663 return -EINVAL; 1664 lut = kzalloc(vsi->rss_table_size, GFP_KERNEL); 1665 if (!lut) 1666 return -ENOMEM; 1667 1668 /* Use the user configured hash keys and lookup table if there is one, 1669 * otherwise use default 1670 */ 1671 if (vsi->rss_lut_user) 1672 memcpy(lut, vsi->rss_lut_user, vsi->rss_table_size); 1673 else 1674 i40e_fill_rss_lut(pf, lut, vsi->rss_table_size, vsi->rss_size); 1675 if (vsi->rss_hkey_user) 1676 memcpy(seed, vsi->rss_hkey_user, I40E_HKEY_ARRAY_SIZE); 1677 else 1678 netdev_rss_key_fill((void *)seed, I40E_HKEY_ARRAY_SIZE); 1679 ret = i40e_config_rss_aq(vsi, seed, lut, vsi->rss_table_size); 1680 kfree(lut); 1681 return ret; 1682 } 1683 1684 /** 1685 * i40e_vsi_setup_queue_map_mqprio - Prepares mqprio based tc_config 1686 * @vsi: the VSI being configured, 1687 * @ctxt: VSI context structure 1688 * @enabled_tc: number of traffic classes to enable 1689 * 1690 * Prepares VSI tc_config to have queue configurations based on MQPRIO options. 1691 **/ 1692 static int i40e_vsi_setup_queue_map_mqprio(struct i40e_vsi *vsi, 1693 struct i40e_vsi_context *ctxt, 1694 u8 enabled_tc) 1695 { 1696 u16 qcount = 0, max_qcount, qmap, sections = 0; 1697 int i, override_q, pow, num_qps, ret; 1698 u8 netdev_tc = 0, offset = 0; 1699 1700 if (vsi->type != I40E_VSI_MAIN) 1701 return -EINVAL; 1702 sections = I40E_AQ_VSI_PROP_QUEUE_MAP_VALID; 1703 sections |= I40E_AQ_VSI_PROP_SCHED_VALID; 1704 vsi->tc_config.numtc = vsi->mqprio_qopt.qopt.num_tc; 1705 vsi->tc_config.enabled_tc = enabled_tc ? enabled_tc : 1; 1706 num_qps = vsi->mqprio_qopt.qopt.count[0]; 1707 1708 /* find the next higher power-of-2 of num queue pairs */ 1709 pow = ilog2(num_qps); 1710 if (!is_power_of_2(num_qps)) 1711 pow++; 1712 qmap = (offset << I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT) | 1713 (pow << I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT); 1714 1715 /* Setup queue offset/count for all TCs for given VSI */ 1716 max_qcount = vsi->mqprio_qopt.qopt.count[0]; 1717 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 1718 /* See if the given TC is enabled for the given VSI */ 1719 if (vsi->tc_config.enabled_tc & BIT(i)) { 1720 offset = vsi->mqprio_qopt.qopt.offset[i]; 1721 qcount = vsi->mqprio_qopt.qopt.count[i]; 1722 if (qcount > max_qcount) 1723 max_qcount = qcount; 1724 vsi->tc_config.tc_info[i].qoffset = offset; 1725 vsi->tc_config.tc_info[i].qcount = qcount; 1726 vsi->tc_config.tc_info[i].netdev_tc = netdev_tc++; 1727 } else { 1728 /* TC is not enabled so set the offset to 1729 * default queue and allocate one queue 1730 * for the given TC. 1731 */ 1732 vsi->tc_config.tc_info[i].qoffset = 0; 1733 vsi->tc_config.tc_info[i].qcount = 1; 1734 vsi->tc_config.tc_info[i].netdev_tc = 0; 1735 } 1736 } 1737 1738 /* Set actual Tx/Rx queue pairs */ 1739 vsi->num_queue_pairs = offset + qcount; 1740 1741 /* Setup queue TC[0].qmap for given VSI context */ 1742 ctxt->info.tc_mapping[0] = cpu_to_le16(qmap); 1743 ctxt->info.mapping_flags |= cpu_to_le16(I40E_AQ_VSI_QUE_MAP_CONTIG); 1744 ctxt->info.queue_mapping[0] = cpu_to_le16(vsi->base_queue); 1745 ctxt->info.valid_sections |= cpu_to_le16(sections); 1746 1747 /* Reconfigure RSS for main VSI with max queue count */ 1748 vsi->rss_size = max_qcount; 1749 ret = i40e_vsi_config_rss(vsi); 1750 if (ret) { 1751 dev_info(&vsi->back->pdev->dev, 1752 "Failed to reconfig rss for num_queues (%u)\n", 1753 max_qcount); 1754 return ret; 1755 } 1756 vsi->reconfig_rss = true; 1757 dev_dbg(&vsi->back->pdev->dev, 1758 "Reconfigured rss with num_queues (%u)\n", max_qcount); 1759 1760 /* Find queue count available for channel VSIs and starting offset 1761 * for channel VSIs 1762 */ 1763 override_q = vsi->mqprio_qopt.qopt.count[0]; 1764 if (override_q && override_q < vsi->num_queue_pairs) { 1765 vsi->cnt_q_avail = vsi->num_queue_pairs - override_q; 1766 vsi->next_base_queue = override_q; 1767 } 1768 return 0; 1769 } 1770 1771 /** 1772 * i40e_vsi_setup_queue_map - Setup a VSI queue map based on enabled_tc 1773 * @vsi: the VSI being setup 1774 * @ctxt: VSI context structure 1775 * @enabled_tc: Enabled TCs bitmap 1776 * @is_add: True if called before Add VSI 1777 * 1778 * Setup VSI queue mapping for enabled traffic classes. 1779 **/ 1780 static void i40e_vsi_setup_queue_map(struct i40e_vsi *vsi, 1781 struct i40e_vsi_context *ctxt, 1782 u8 enabled_tc, 1783 bool is_add) 1784 { 1785 struct i40e_pf *pf = vsi->back; 1786 u16 sections = 0; 1787 u8 netdev_tc = 0; 1788 u16 numtc = 0; 1789 u16 qcount; 1790 u8 offset; 1791 u16 qmap; 1792 int i; 1793 u16 num_tc_qps = 0; 1794 1795 sections = I40E_AQ_VSI_PROP_QUEUE_MAP_VALID; 1796 offset = 0; 1797 1798 if (enabled_tc && (vsi->back->flags & I40E_FLAG_DCB_ENABLED)) { 1799 /* Find numtc from enabled TC bitmap */ 1800 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 1801 if (enabled_tc & BIT(i)) /* TC is enabled */ 1802 numtc++; 1803 } 1804 if (!numtc) { 1805 dev_warn(&pf->pdev->dev, "DCB is enabled but no TC enabled, forcing TC0\n"); 1806 numtc = 1; 1807 } 1808 } else { 1809 /* At least TC0 is enabled in non-DCB, non-MQPRIO case */ 1810 numtc = 1; 1811 } 1812 1813 vsi->tc_config.numtc = numtc; 1814 vsi->tc_config.enabled_tc = enabled_tc ? enabled_tc : 1; 1815 /* Number of queues per enabled TC */ 1816 qcount = vsi->alloc_queue_pairs; 1817 1818 num_tc_qps = qcount / numtc; 1819 num_tc_qps = min_t(int, num_tc_qps, i40e_pf_get_max_q_per_tc(pf)); 1820 1821 /* Setup queue offset/count for all TCs for given VSI */ 1822 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 1823 /* See if the given TC is enabled for the given VSI */ 1824 if (vsi->tc_config.enabled_tc & BIT(i)) { 1825 /* TC is enabled */ 1826 int pow, num_qps; 1827 1828 switch (vsi->type) { 1829 case I40E_VSI_MAIN: 1830 qcount = min_t(int, pf->alloc_rss_size, 1831 num_tc_qps); 1832 break; 1833 case I40E_VSI_FDIR: 1834 case I40E_VSI_SRIOV: 1835 case I40E_VSI_VMDQ2: 1836 default: 1837 qcount = num_tc_qps; 1838 WARN_ON(i != 0); 1839 break; 1840 } 1841 vsi->tc_config.tc_info[i].qoffset = offset; 1842 vsi->tc_config.tc_info[i].qcount = qcount; 1843 1844 /* find the next higher power-of-2 of num queue pairs */ 1845 num_qps = qcount; 1846 pow = 0; 1847 while (num_qps && (BIT_ULL(pow) < qcount)) { 1848 pow++; 1849 num_qps >>= 1; 1850 } 1851 1852 vsi->tc_config.tc_info[i].netdev_tc = netdev_tc++; 1853 qmap = 1854 (offset << I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT) | 1855 (pow << I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT); 1856 1857 offset += qcount; 1858 } else { 1859 /* TC is not enabled so set the offset to 1860 * default queue and allocate one queue 1861 * for the given TC. 1862 */ 1863 vsi->tc_config.tc_info[i].qoffset = 0; 1864 vsi->tc_config.tc_info[i].qcount = 1; 1865 vsi->tc_config.tc_info[i].netdev_tc = 0; 1866 1867 qmap = 0; 1868 } 1869 ctxt->info.tc_mapping[i] = cpu_to_le16(qmap); 1870 } 1871 1872 /* Set actual Tx/Rx queue pairs */ 1873 vsi->num_queue_pairs = offset; 1874 if ((vsi->type == I40E_VSI_MAIN) && (numtc == 1)) { 1875 if (vsi->req_queue_pairs > 0) 1876 vsi->num_queue_pairs = vsi->req_queue_pairs; 1877 else if (pf->flags & I40E_FLAG_MSIX_ENABLED) 1878 vsi->num_queue_pairs = pf->num_lan_msix; 1879 } 1880 1881 /* Scheduler section valid can only be set for ADD VSI */ 1882 if (is_add) { 1883 sections |= I40E_AQ_VSI_PROP_SCHED_VALID; 1884 1885 ctxt->info.up_enable_bits = enabled_tc; 1886 } 1887 if (vsi->type == I40E_VSI_SRIOV) { 1888 ctxt->info.mapping_flags |= 1889 cpu_to_le16(I40E_AQ_VSI_QUE_MAP_NONCONTIG); 1890 for (i = 0; i < vsi->num_queue_pairs; i++) 1891 ctxt->info.queue_mapping[i] = 1892 cpu_to_le16(vsi->base_queue + i); 1893 } else { 1894 ctxt->info.mapping_flags |= 1895 cpu_to_le16(I40E_AQ_VSI_QUE_MAP_CONTIG); 1896 ctxt->info.queue_mapping[0] = cpu_to_le16(vsi->base_queue); 1897 } 1898 ctxt->info.valid_sections |= cpu_to_le16(sections); 1899 } 1900 1901 /** 1902 * i40e_addr_sync - Callback for dev_(mc|uc)_sync to add address 1903 * @netdev: the netdevice 1904 * @addr: address to add 1905 * 1906 * Called by __dev_(mc|uc)_sync when an address needs to be added. We call 1907 * __dev_(uc|mc)_sync from .set_rx_mode and guarantee to hold the hash lock. 1908 */ 1909 static int i40e_addr_sync(struct net_device *netdev, const u8 *addr) 1910 { 1911 struct i40e_netdev_priv *np = netdev_priv(netdev); 1912 struct i40e_vsi *vsi = np->vsi; 1913 1914 if (i40e_add_mac_filter(vsi, addr)) 1915 return 0; 1916 else 1917 return -ENOMEM; 1918 } 1919 1920 /** 1921 * i40e_addr_unsync - Callback for dev_(mc|uc)_sync to remove address 1922 * @netdev: the netdevice 1923 * @addr: address to add 1924 * 1925 * Called by __dev_(mc|uc)_sync when an address needs to be removed. We call 1926 * __dev_(uc|mc)_sync from .set_rx_mode and guarantee to hold the hash lock. 1927 */ 1928 static int i40e_addr_unsync(struct net_device *netdev, const u8 *addr) 1929 { 1930 struct i40e_netdev_priv *np = netdev_priv(netdev); 1931 struct i40e_vsi *vsi = np->vsi; 1932 1933 /* Under some circumstances, we might receive a request to delete 1934 * our own device address from our uc list. Because we store the 1935 * device address in the VSI's MAC/VLAN filter list, we need to ignore 1936 * such requests and not delete our device address from this list. 1937 */ 1938 if (ether_addr_equal(addr, netdev->dev_addr)) 1939 return 0; 1940 1941 i40e_del_mac_filter(vsi, addr); 1942 1943 return 0; 1944 } 1945 1946 /** 1947 * i40e_set_rx_mode - NDO callback to set the netdev filters 1948 * @netdev: network interface device structure 1949 **/ 1950 static void i40e_set_rx_mode(struct net_device *netdev) 1951 { 1952 struct i40e_netdev_priv *np = netdev_priv(netdev); 1953 struct i40e_vsi *vsi = np->vsi; 1954 1955 spin_lock_bh(&vsi->mac_filter_hash_lock); 1956 1957 __dev_uc_sync(netdev, i40e_addr_sync, i40e_addr_unsync); 1958 __dev_mc_sync(netdev, i40e_addr_sync, i40e_addr_unsync); 1959 1960 spin_unlock_bh(&vsi->mac_filter_hash_lock); 1961 1962 /* check for other flag changes */ 1963 if (vsi->current_netdev_flags != vsi->netdev->flags) { 1964 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED; 1965 vsi->back->flags |= I40E_FLAG_FILTER_SYNC; 1966 } 1967 } 1968 1969 /** 1970 * i40e_undo_del_filter_entries - Undo the changes made to MAC filter entries 1971 * @vsi: Pointer to VSI struct 1972 * @from: Pointer to list which contains MAC filter entries - changes to 1973 * those entries needs to be undone. 1974 * 1975 * MAC filter entries from this list were slated for deletion. 1976 **/ 1977 static void i40e_undo_del_filter_entries(struct i40e_vsi *vsi, 1978 struct hlist_head *from) 1979 { 1980 struct i40e_mac_filter *f; 1981 struct hlist_node *h; 1982 1983 hlist_for_each_entry_safe(f, h, from, hlist) { 1984 u64 key = i40e_addr_to_hkey(f->macaddr); 1985 1986 /* Move the element back into MAC filter list*/ 1987 hlist_del(&f->hlist); 1988 hash_add(vsi->mac_filter_hash, &f->hlist, key); 1989 } 1990 } 1991 1992 /** 1993 * i40e_undo_add_filter_entries - Undo the changes made to MAC filter entries 1994 * @vsi: Pointer to vsi struct 1995 * @from: Pointer to list which contains MAC filter entries - changes to 1996 * those entries needs to be undone. 1997 * 1998 * MAC filter entries from this list were slated for addition. 1999 **/ 2000 static void i40e_undo_add_filter_entries(struct i40e_vsi *vsi, 2001 struct hlist_head *from) 2002 { 2003 struct i40e_new_mac_filter *new; 2004 struct hlist_node *h; 2005 2006 hlist_for_each_entry_safe(new, h, from, hlist) { 2007 /* We can simply free the wrapper structure */ 2008 hlist_del(&new->hlist); 2009 kfree(new); 2010 } 2011 } 2012 2013 /** 2014 * i40e_next_entry - Get the next non-broadcast filter from a list 2015 * @next: pointer to filter in list 2016 * 2017 * Returns the next non-broadcast filter in the list. Required so that we 2018 * ignore broadcast filters within the list, since these are not handled via 2019 * the normal firmware update path. 2020 */ 2021 static 2022 struct i40e_new_mac_filter *i40e_next_filter(struct i40e_new_mac_filter *next) 2023 { 2024 hlist_for_each_entry_continue(next, hlist) { 2025 if (!is_broadcast_ether_addr(next->f->macaddr)) 2026 return next; 2027 } 2028 2029 return NULL; 2030 } 2031 2032 /** 2033 * i40e_update_filter_state - Update filter state based on return data 2034 * from firmware 2035 * @count: Number of filters added 2036 * @add_list: return data from fw 2037 * @head: pointer to first filter in current batch 2038 * 2039 * MAC filter entries from list were slated to be added to device. Returns 2040 * number of successful filters. Note that 0 does NOT mean success! 2041 **/ 2042 static int 2043 i40e_update_filter_state(int count, 2044 struct i40e_aqc_add_macvlan_element_data *add_list, 2045 struct i40e_new_mac_filter *add_head) 2046 { 2047 int retval = 0; 2048 int i; 2049 2050 for (i = 0; i < count; i++) { 2051 /* Always check status of each filter. We don't need to check 2052 * the firmware return status because we pre-set the filter 2053 * status to I40E_AQC_MM_ERR_NO_RES when sending the filter 2054 * request to the adminq. Thus, if it no longer matches then 2055 * we know the filter is active. 2056 */ 2057 if (add_list[i].match_method == I40E_AQC_MM_ERR_NO_RES) { 2058 add_head->state = I40E_FILTER_FAILED; 2059 } else { 2060 add_head->state = I40E_FILTER_ACTIVE; 2061 retval++; 2062 } 2063 2064 add_head = i40e_next_filter(add_head); 2065 if (!add_head) 2066 break; 2067 } 2068 2069 return retval; 2070 } 2071 2072 /** 2073 * i40e_aqc_del_filters - Request firmware to delete a set of filters 2074 * @vsi: ptr to the VSI 2075 * @vsi_name: name to display in messages 2076 * @list: the list of filters to send to firmware 2077 * @num_del: the number of filters to delete 2078 * @retval: Set to -EIO on failure to delete 2079 * 2080 * Send a request to firmware via AdminQ to delete a set of filters. Uses 2081 * *retval instead of a return value so that success does not force ret_val to 2082 * be set to 0. This ensures that a sequence of calls to this function 2083 * preserve the previous value of *retval on successful delete. 2084 */ 2085 static 2086 void i40e_aqc_del_filters(struct i40e_vsi *vsi, const char *vsi_name, 2087 struct i40e_aqc_remove_macvlan_element_data *list, 2088 int num_del, int *retval) 2089 { 2090 struct i40e_hw *hw = &vsi->back->hw; 2091 i40e_status aq_ret; 2092 int aq_err; 2093 2094 aq_ret = i40e_aq_remove_macvlan(hw, vsi->seid, list, num_del, NULL); 2095 aq_err = hw->aq.asq_last_status; 2096 2097 /* Explicitly ignore and do not report when firmware returns ENOENT */ 2098 if (aq_ret && !(aq_err == I40E_AQ_RC_ENOENT)) { 2099 *retval = -EIO; 2100 dev_info(&vsi->back->pdev->dev, 2101 "ignoring delete macvlan error on %s, err %s, aq_err %s\n", 2102 vsi_name, i40e_stat_str(hw, aq_ret), 2103 i40e_aq_str(hw, aq_err)); 2104 } 2105 } 2106 2107 /** 2108 * i40e_aqc_add_filters - Request firmware to add a set of filters 2109 * @vsi: ptr to the VSI 2110 * @vsi_name: name to display in messages 2111 * @list: the list of filters to send to firmware 2112 * @add_head: Position in the add hlist 2113 * @num_add: the number of filters to add 2114 * @promisc_change: set to true on exit if promiscuous mode was forced on 2115 * 2116 * Send a request to firmware via AdminQ to add a chunk of filters. Will set 2117 * promisc_changed to true if the firmware has run out of space for more 2118 * filters. 2119 */ 2120 static 2121 void i40e_aqc_add_filters(struct i40e_vsi *vsi, const char *vsi_name, 2122 struct i40e_aqc_add_macvlan_element_data *list, 2123 struct i40e_new_mac_filter *add_head, 2124 int num_add, bool *promisc_changed) 2125 { 2126 struct i40e_hw *hw = &vsi->back->hw; 2127 int aq_err, fcnt; 2128 2129 i40e_aq_add_macvlan(hw, vsi->seid, list, num_add, NULL); 2130 aq_err = hw->aq.asq_last_status; 2131 fcnt = i40e_update_filter_state(num_add, list, add_head); 2132 2133 if (fcnt != num_add) { 2134 *promisc_changed = true; 2135 set_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state); 2136 dev_warn(&vsi->back->pdev->dev, 2137 "Error %s adding RX filters on %s, promiscuous mode forced on\n", 2138 i40e_aq_str(hw, aq_err), 2139 vsi_name); 2140 } 2141 } 2142 2143 /** 2144 * i40e_aqc_broadcast_filter - Set promiscuous broadcast flags 2145 * @vsi: pointer to the VSI 2146 * @f: filter data 2147 * 2148 * This function sets or clears the promiscuous broadcast flags for VLAN 2149 * filters in order to properly receive broadcast frames. Assumes that only 2150 * broadcast filters are passed. 2151 * 2152 * Returns status indicating success or failure; 2153 **/ 2154 static i40e_status 2155 i40e_aqc_broadcast_filter(struct i40e_vsi *vsi, const char *vsi_name, 2156 struct i40e_mac_filter *f) 2157 { 2158 bool enable = f->state == I40E_FILTER_NEW; 2159 struct i40e_hw *hw = &vsi->back->hw; 2160 i40e_status aq_ret; 2161 2162 if (f->vlan == I40E_VLAN_ANY) { 2163 aq_ret = i40e_aq_set_vsi_broadcast(hw, 2164 vsi->seid, 2165 enable, 2166 NULL); 2167 } else { 2168 aq_ret = i40e_aq_set_vsi_bc_promisc_on_vlan(hw, 2169 vsi->seid, 2170 enable, 2171 f->vlan, 2172 NULL); 2173 } 2174 2175 if (aq_ret) 2176 dev_warn(&vsi->back->pdev->dev, 2177 "Error %s setting broadcast promiscuous mode on %s\n", 2178 i40e_aq_str(hw, hw->aq.asq_last_status), 2179 vsi_name); 2180 2181 return aq_ret; 2182 } 2183 2184 /** 2185 * i40e_set_promiscuous - set promiscuous mode 2186 * @pf: board private structure 2187 * @promisc: promisc on or off 2188 * 2189 * There are different ways of setting promiscuous mode on a PF depending on 2190 * what state/environment we're in. This identifies and sets it appropriately. 2191 * Returns 0 on success. 2192 **/ 2193 static int i40e_set_promiscuous(struct i40e_pf *pf, bool promisc) 2194 { 2195 struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi]; 2196 struct i40e_hw *hw = &pf->hw; 2197 i40e_status aq_ret; 2198 2199 if (vsi->type == I40E_VSI_MAIN && 2200 pf->lan_veb != I40E_NO_VEB && 2201 !(pf->flags & I40E_FLAG_MFP_ENABLED)) { 2202 /* set defport ON for Main VSI instead of true promisc 2203 * this way we will get all unicast/multicast and VLAN 2204 * promisc behavior but will not get VF or VMDq traffic 2205 * replicated on the Main VSI. 2206 */ 2207 if (promisc) 2208 aq_ret = i40e_aq_set_default_vsi(hw, 2209 vsi->seid, 2210 NULL); 2211 else 2212 aq_ret = i40e_aq_clear_default_vsi(hw, 2213 vsi->seid, 2214 NULL); 2215 if (aq_ret) { 2216 dev_info(&pf->pdev->dev, 2217 "Set default VSI failed, err %s, aq_err %s\n", 2218 i40e_stat_str(hw, aq_ret), 2219 i40e_aq_str(hw, hw->aq.asq_last_status)); 2220 } 2221 } else { 2222 aq_ret = i40e_aq_set_vsi_unicast_promiscuous( 2223 hw, 2224 vsi->seid, 2225 promisc, NULL, 2226 true); 2227 if (aq_ret) { 2228 dev_info(&pf->pdev->dev, 2229 "set unicast promisc failed, err %s, aq_err %s\n", 2230 i40e_stat_str(hw, aq_ret), 2231 i40e_aq_str(hw, hw->aq.asq_last_status)); 2232 } 2233 aq_ret = i40e_aq_set_vsi_multicast_promiscuous( 2234 hw, 2235 vsi->seid, 2236 promisc, NULL); 2237 if (aq_ret) { 2238 dev_info(&pf->pdev->dev, 2239 "set multicast promisc failed, err %s, aq_err %s\n", 2240 i40e_stat_str(hw, aq_ret), 2241 i40e_aq_str(hw, hw->aq.asq_last_status)); 2242 } 2243 } 2244 2245 if (!aq_ret) 2246 pf->cur_promisc = promisc; 2247 2248 return aq_ret; 2249 } 2250 2251 /** 2252 * i40e_sync_vsi_filters - Update the VSI filter list to the HW 2253 * @vsi: ptr to the VSI 2254 * 2255 * Push any outstanding VSI filter changes through the AdminQ. 2256 * 2257 * Returns 0 or error value 2258 **/ 2259 int i40e_sync_vsi_filters(struct i40e_vsi *vsi) 2260 { 2261 struct hlist_head tmp_add_list, tmp_del_list; 2262 struct i40e_mac_filter *f; 2263 struct i40e_new_mac_filter *new, *add_head = NULL; 2264 struct i40e_hw *hw = &vsi->back->hw; 2265 unsigned int failed_filters = 0; 2266 unsigned int vlan_filters = 0; 2267 bool promisc_changed = false; 2268 char vsi_name[16] = "PF"; 2269 int filter_list_len = 0; 2270 i40e_status aq_ret = 0; 2271 u32 changed_flags = 0; 2272 struct hlist_node *h; 2273 struct i40e_pf *pf; 2274 int num_add = 0; 2275 int num_del = 0; 2276 int retval = 0; 2277 u16 cmd_flags; 2278 int list_size; 2279 int bkt; 2280 2281 /* empty array typed pointers, kcalloc later */ 2282 struct i40e_aqc_add_macvlan_element_data *add_list; 2283 struct i40e_aqc_remove_macvlan_element_data *del_list; 2284 2285 while (test_and_set_bit(__I40E_VSI_SYNCING_FILTERS, vsi->state)) 2286 usleep_range(1000, 2000); 2287 pf = vsi->back; 2288 2289 if (vsi->netdev) { 2290 changed_flags = vsi->current_netdev_flags ^ vsi->netdev->flags; 2291 vsi->current_netdev_flags = vsi->netdev->flags; 2292 } 2293 2294 INIT_HLIST_HEAD(&tmp_add_list); 2295 INIT_HLIST_HEAD(&tmp_del_list); 2296 2297 if (vsi->type == I40E_VSI_SRIOV) 2298 snprintf(vsi_name, sizeof(vsi_name) - 1, "VF %d", vsi->vf_id); 2299 else if (vsi->type != I40E_VSI_MAIN) 2300 snprintf(vsi_name, sizeof(vsi_name) - 1, "vsi %d", vsi->seid); 2301 2302 if (vsi->flags & I40E_VSI_FLAG_FILTER_CHANGED) { 2303 vsi->flags &= ~I40E_VSI_FLAG_FILTER_CHANGED; 2304 2305 spin_lock_bh(&vsi->mac_filter_hash_lock); 2306 /* Create a list of filters to delete. */ 2307 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) { 2308 if (f->state == I40E_FILTER_REMOVE) { 2309 /* Move the element into temporary del_list */ 2310 hash_del(&f->hlist); 2311 hlist_add_head(&f->hlist, &tmp_del_list); 2312 2313 /* Avoid counting removed filters */ 2314 continue; 2315 } 2316 if (f->state == I40E_FILTER_NEW) { 2317 /* Create a temporary i40e_new_mac_filter */ 2318 new = kzalloc(sizeof(*new), GFP_ATOMIC); 2319 if (!new) 2320 goto err_no_memory_locked; 2321 2322 /* Store pointer to the real filter */ 2323 new->f = f; 2324 new->state = f->state; 2325 2326 /* Add it to the hash list */ 2327 hlist_add_head(&new->hlist, &tmp_add_list); 2328 } 2329 2330 /* Count the number of active (current and new) VLAN 2331 * filters we have now. Does not count filters which 2332 * are marked for deletion. 2333 */ 2334 if (f->vlan > 0) 2335 vlan_filters++; 2336 } 2337 2338 retval = i40e_correct_mac_vlan_filters(vsi, 2339 &tmp_add_list, 2340 &tmp_del_list, 2341 vlan_filters); 2342 if (retval) 2343 goto err_no_memory_locked; 2344 2345 spin_unlock_bh(&vsi->mac_filter_hash_lock); 2346 } 2347 2348 /* Now process 'del_list' outside the lock */ 2349 if (!hlist_empty(&tmp_del_list)) { 2350 filter_list_len = hw->aq.asq_buf_size / 2351 sizeof(struct i40e_aqc_remove_macvlan_element_data); 2352 list_size = filter_list_len * 2353 sizeof(struct i40e_aqc_remove_macvlan_element_data); 2354 del_list = kzalloc(list_size, GFP_ATOMIC); 2355 if (!del_list) 2356 goto err_no_memory; 2357 2358 hlist_for_each_entry_safe(f, h, &tmp_del_list, hlist) { 2359 cmd_flags = 0; 2360 2361 /* handle broadcast filters by updating the broadcast 2362 * promiscuous flag and release filter list. 2363 */ 2364 if (is_broadcast_ether_addr(f->macaddr)) { 2365 i40e_aqc_broadcast_filter(vsi, vsi_name, f); 2366 2367 hlist_del(&f->hlist); 2368 kfree(f); 2369 continue; 2370 } 2371 2372 /* add to delete list */ 2373 ether_addr_copy(del_list[num_del].mac_addr, f->macaddr); 2374 if (f->vlan == I40E_VLAN_ANY) { 2375 del_list[num_del].vlan_tag = 0; 2376 cmd_flags |= I40E_AQC_MACVLAN_DEL_IGNORE_VLAN; 2377 } else { 2378 del_list[num_del].vlan_tag = 2379 cpu_to_le16((u16)(f->vlan)); 2380 } 2381 2382 cmd_flags |= I40E_AQC_MACVLAN_DEL_PERFECT_MATCH; 2383 del_list[num_del].flags = cmd_flags; 2384 num_del++; 2385 2386 /* flush a full buffer */ 2387 if (num_del == filter_list_len) { 2388 i40e_aqc_del_filters(vsi, vsi_name, del_list, 2389 num_del, &retval); 2390 memset(del_list, 0, list_size); 2391 num_del = 0; 2392 } 2393 /* Release memory for MAC filter entries which were 2394 * synced up with HW. 2395 */ 2396 hlist_del(&f->hlist); 2397 kfree(f); 2398 } 2399 2400 if (num_del) { 2401 i40e_aqc_del_filters(vsi, vsi_name, del_list, 2402 num_del, &retval); 2403 } 2404 2405 kfree(del_list); 2406 del_list = NULL; 2407 } 2408 2409 if (!hlist_empty(&tmp_add_list)) { 2410 /* Do all the adds now. */ 2411 filter_list_len = hw->aq.asq_buf_size / 2412 sizeof(struct i40e_aqc_add_macvlan_element_data); 2413 list_size = filter_list_len * 2414 sizeof(struct i40e_aqc_add_macvlan_element_data); 2415 add_list = kzalloc(list_size, GFP_ATOMIC); 2416 if (!add_list) 2417 goto err_no_memory; 2418 2419 num_add = 0; 2420 hlist_for_each_entry_safe(new, h, &tmp_add_list, hlist) { 2421 if (test_bit(__I40E_VSI_OVERFLOW_PROMISC, 2422 vsi->state)) { 2423 new->state = I40E_FILTER_FAILED; 2424 continue; 2425 } 2426 2427 /* handle broadcast filters by updating the broadcast 2428 * promiscuous flag instead of adding a MAC filter. 2429 */ 2430 if (is_broadcast_ether_addr(new->f->macaddr)) { 2431 if (i40e_aqc_broadcast_filter(vsi, vsi_name, 2432 new->f)) 2433 new->state = I40E_FILTER_FAILED; 2434 else 2435 new->state = I40E_FILTER_ACTIVE; 2436 continue; 2437 } 2438 2439 /* add to add array */ 2440 if (num_add == 0) 2441 add_head = new; 2442 cmd_flags = 0; 2443 ether_addr_copy(add_list[num_add].mac_addr, 2444 new->f->macaddr); 2445 if (new->f->vlan == I40E_VLAN_ANY) { 2446 add_list[num_add].vlan_tag = 0; 2447 cmd_flags |= I40E_AQC_MACVLAN_ADD_IGNORE_VLAN; 2448 } else { 2449 add_list[num_add].vlan_tag = 2450 cpu_to_le16((u16)(new->f->vlan)); 2451 } 2452 add_list[num_add].queue_number = 0; 2453 /* set invalid match method for later detection */ 2454 add_list[num_add].match_method = I40E_AQC_MM_ERR_NO_RES; 2455 cmd_flags |= I40E_AQC_MACVLAN_ADD_PERFECT_MATCH; 2456 add_list[num_add].flags = cpu_to_le16(cmd_flags); 2457 num_add++; 2458 2459 /* flush a full buffer */ 2460 if (num_add == filter_list_len) { 2461 i40e_aqc_add_filters(vsi, vsi_name, add_list, 2462 add_head, num_add, 2463 &promisc_changed); 2464 memset(add_list, 0, list_size); 2465 num_add = 0; 2466 } 2467 } 2468 if (num_add) { 2469 i40e_aqc_add_filters(vsi, vsi_name, add_list, add_head, 2470 num_add, &promisc_changed); 2471 } 2472 /* Now move all of the filters from the temp add list back to 2473 * the VSI's list. 2474 */ 2475 spin_lock_bh(&vsi->mac_filter_hash_lock); 2476 hlist_for_each_entry_safe(new, h, &tmp_add_list, hlist) { 2477 /* Only update the state if we're still NEW */ 2478 if (new->f->state == I40E_FILTER_NEW) 2479 new->f->state = new->state; 2480 hlist_del(&new->hlist); 2481 kfree(new); 2482 } 2483 spin_unlock_bh(&vsi->mac_filter_hash_lock); 2484 kfree(add_list); 2485 add_list = NULL; 2486 } 2487 2488 /* Determine the number of active and failed filters. */ 2489 spin_lock_bh(&vsi->mac_filter_hash_lock); 2490 vsi->active_filters = 0; 2491 hash_for_each(vsi->mac_filter_hash, bkt, f, hlist) { 2492 if (f->state == I40E_FILTER_ACTIVE) 2493 vsi->active_filters++; 2494 else if (f->state == I40E_FILTER_FAILED) 2495 failed_filters++; 2496 } 2497 spin_unlock_bh(&vsi->mac_filter_hash_lock); 2498 2499 /* If promiscuous mode has changed, we need to calculate a new 2500 * threshold for when we are safe to exit 2501 */ 2502 if (promisc_changed) 2503 vsi->promisc_threshold = (vsi->active_filters * 3) / 4; 2504 2505 /* Check if we are able to exit overflow promiscuous mode. We can 2506 * safely exit if we didn't just enter, we no longer have any failed 2507 * filters, and we have reduced filters below the threshold value. 2508 */ 2509 if (test_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state) && 2510 !promisc_changed && !failed_filters && 2511 (vsi->active_filters < vsi->promisc_threshold)) { 2512 dev_info(&pf->pdev->dev, 2513 "filter logjam cleared on %s, leaving overflow promiscuous mode\n", 2514 vsi_name); 2515 clear_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state); 2516 promisc_changed = true; 2517 vsi->promisc_threshold = 0; 2518 } 2519 2520 /* if the VF is not trusted do not do promisc */ 2521 if ((vsi->type == I40E_VSI_SRIOV) && !pf->vf[vsi->vf_id].trusted) { 2522 clear_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state); 2523 goto out; 2524 } 2525 2526 /* check for changes in promiscuous modes */ 2527 if (changed_flags & IFF_ALLMULTI) { 2528 bool cur_multipromisc; 2529 2530 cur_multipromisc = !!(vsi->current_netdev_flags & IFF_ALLMULTI); 2531 aq_ret = i40e_aq_set_vsi_multicast_promiscuous(&vsi->back->hw, 2532 vsi->seid, 2533 cur_multipromisc, 2534 NULL); 2535 if (aq_ret) { 2536 retval = i40e_aq_rc_to_posix(aq_ret, 2537 hw->aq.asq_last_status); 2538 dev_info(&pf->pdev->dev, 2539 "set multi promisc failed on %s, err %s aq_err %s\n", 2540 vsi_name, 2541 i40e_stat_str(hw, aq_ret), 2542 i40e_aq_str(hw, hw->aq.asq_last_status)); 2543 } 2544 } 2545 2546 if ((changed_flags & IFF_PROMISC) || promisc_changed) { 2547 bool cur_promisc; 2548 2549 cur_promisc = (!!(vsi->current_netdev_flags & IFF_PROMISC) || 2550 test_bit(__I40E_VSI_OVERFLOW_PROMISC, 2551 vsi->state)); 2552 aq_ret = i40e_set_promiscuous(pf, cur_promisc); 2553 if (aq_ret) { 2554 retval = i40e_aq_rc_to_posix(aq_ret, 2555 hw->aq.asq_last_status); 2556 dev_info(&pf->pdev->dev, 2557 "Setting promiscuous %s failed on %s, err %s aq_err %s\n", 2558 cur_promisc ? "on" : "off", 2559 vsi_name, 2560 i40e_stat_str(hw, aq_ret), 2561 i40e_aq_str(hw, hw->aq.asq_last_status)); 2562 } 2563 } 2564 out: 2565 /* if something went wrong then set the changed flag so we try again */ 2566 if (retval) 2567 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED; 2568 2569 clear_bit(__I40E_VSI_SYNCING_FILTERS, vsi->state); 2570 return retval; 2571 2572 err_no_memory: 2573 /* Restore elements on the temporary add and delete lists */ 2574 spin_lock_bh(&vsi->mac_filter_hash_lock); 2575 err_no_memory_locked: 2576 i40e_undo_del_filter_entries(vsi, &tmp_del_list); 2577 i40e_undo_add_filter_entries(vsi, &tmp_add_list); 2578 spin_unlock_bh(&vsi->mac_filter_hash_lock); 2579 2580 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED; 2581 clear_bit(__I40E_VSI_SYNCING_FILTERS, vsi->state); 2582 return -ENOMEM; 2583 } 2584 2585 /** 2586 * i40e_sync_filters_subtask - Sync the VSI filter list with HW 2587 * @pf: board private structure 2588 **/ 2589 static void i40e_sync_filters_subtask(struct i40e_pf *pf) 2590 { 2591 int v; 2592 2593 if (!pf || !(pf->flags & I40E_FLAG_FILTER_SYNC)) 2594 return; 2595 pf->flags &= ~I40E_FLAG_FILTER_SYNC; 2596 2597 for (v = 0; v < pf->num_alloc_vsi; v++) { 2598 if (pf->vsi[v] && 2599 (pf->vsi[v]->flags & I40E_VSI_FLAG_FILTER_CHANGED)) { 2600 int ret = i40e_sync_vsi_filters(pf->vsi[v]); 2601 2602 if (ret) { 2603 /* come back and try again later */ 2604 pf->flags |= I40E_FLAG_FILTER_SYNC; 2605 break; 2606 } 2607 } 2608 } 2609 } 2610 2611 /** 2612 * i40e_max_xdp_frame_size - returns the maximum allowed frame size for XDP 2613 * @vsi: the vsi 2614 **/ 2615 static int i40e_max_xdp_frame_size(struct i40e_vsi *vsi) 2616 { 2617 if (PAGE_SIZE >= 8192 || (vsi->back->flags & I40E_FLAG_LEGACY_RX)) 2618 return I40E_RXBUFFER_2048; 2619 else 2620 return I40E_RXBUFFER_3072; 2621 } 2622 2623 /** 2624 * i40e_change_mtu - NDO callback to change the Maximum Transfer Unit 2625 * @netdev: network interface device structure 2626 * @new_mtu: new value for maximum frame size 2627 * 2628 * Returns 0 on success, negative on failure 2629 **/ 2630 static int i40e_change_mtu(struct net_device *netdev, int new_mtu) 2631 { 2632 struct i40e_netdev_priv *np = netdev_priv(netdev); 2633 struct i40e_vsi *vsi = np->vsi; 2634 struct i40e_pf *pf = vsi->back; 2635 2636 if (i40e_enabled_xdp_vsi(vsi)) { 2637 int frame_size = new_mtu + ETH_HLEN + ETH_FCS_LEN + VLAN_HLEN; 2638 2639 if (frame_size > i40e_max_xdp_frame_size(vsi)) 2640 return -EINVAL; 2641 } 2642 2643 netdev_info(netdev, "changing MTU from %d to %d\n", 2644 netdev->mtu, new_mtu); 2645 netdev->mtu = new_mtu; 2646 if (netif_running(netdev)) 2647 i40e_vsi_reinit_locked(vsi); 2648 pf->flags |= (I40E_FLAG_SERVICE_CLIENT_REQUESTED | 2649 I40E_FLAG_CLIENT_L2_CHANGE); 2650 return 0; 2651 } 2652 2653 /** 2654 * i40e_ioctl - Access the hwtstamp interface 2655 * @netdev: network interface device structure 2656 * @ifr: interface request data 2657 * @cmd: ioctl command 2658 **/ 2659 int i40e_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd) 2660 { 2661 struct i40e_netdev_priv *np = netdev_priv(netdev); 2662 struct i40e_pf *pf = np->vsi->back; 2663 2664 switch (cmd) { 2665 case SIOCGHWTSTAMP: 2666 return i40e_ptp_get_ts_config(pf, ifr); 2667 case SIOCSHWTSTAMP: 2668 return i40e_ptp_set_ts_config(pf, ifr); 2669 default: 2670 return -EOPNOTSUPP; 2671 } 2672 } 2673 2674 /** 2675 * i40e_vlan_stripping_enable - Turn on vlan stripping for the VSI 2676 * @vsi: the vsi being adjusted 2677 **/ 2678 void i40e_vlan_stripping_enable(struct i40e_vsi *vsi) 2679 { 2680 struct i40e_vsi_context ctxt; 2681 i40e_status ret; 2682 2683 if ((vsi->info.valid_sections & 2684 cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID)) && 2685 ((vsi->info.port_vlan_flags & I40E_AQ_VSI_PVLAN_MODE_MASK) == 0)) 2686 return; /* already enabled */ 2687 2688 vsi->info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID); 2689 vsi->info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_ALL | 2690 I40E_AQ_VSI_PVLAN_EMOD_STR_BOTH; 2691 2692 ctxt.seid = vsi->seid; 2693 ctxt.info = vsi->info; 2694 ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL); 2695 if (ret) { 2696 dev_info(&vsi->back->pdev->dev, 2697 "update vlan stripping failed, err %s aq_err %s\n", 2698 i40e_stat_str(&vsi->back->hw, ret), 2699 i40e_aq_str(&vsi->back->hw, 2700 vsi->back->hw.aq.asq_last_status)); 2701 } 2702 } 2703 2704 /** 2705 * i40e_vlan_stripping_disable - Turn off vlan stripping for the VSI 2706 * @vsi: the vsi being adjusted 2707 **/ 2708 void i40e_vlan_stripping_disable(struct i40e_vsi *vsi) 2709 { 2710 struct i40e_vsi_context ctxt; 2711 i40e_status ret; 2712 2713 if ((vsi->info.valid_sections & 2714 cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID)) && 2715 ((vsi->info.port_vlan_flags & I40E_AQ_VSI_PVLAN_EMOD_MASK) == 2716 I40E_AQ_VSI_PVLAN_EMOD_MASK)) 2717 return; /* already disabled */ 2718 2719 vsi->info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID); 2720 vsi->info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_ALL | 2721 I40E_AQ_VSI_PVLAN_EMOD_NOTHING; 2722 2723 ctxt.seid = vsi->seid; 2724 ctxt.info = vsi->info; 2725 ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL); 2726 if (ret) { 2727 dev_info(&vsi->back->pdev->dev, 2728 "update vlan stripping failed, err %s aq_err %s\n", 2729 i40e_stat_str(&vsi->back->hw, ret), 2730 i40e_aq_str(&vsi->back->hw, 2731 vsi->back->hw.aq.asq_last_status)); 2732 } 2733 } 2734 2735 /** 2736 * i40e_vlan_rx_register - Setup or shutdown vlan offload 2737 * @netdev: network interface to be adjusted 2738 * @features: netdev features to test if VLAN offload is enabled or not 2739 **/ 2740 static void i40e_vlan_rx_register(struct net_device *netdev, u32 features) 2741 { 2742 struct i40e_netdev_priv *np = netdev_priv(netdev); 2743 struct i40e_vsi *vsi = np->vsi; 2744 2745 if (features & NETIF_F_HW_VLAN_CTAG_RX) 2746 i40e_vlan_stripping_enable(vsi); 2747 else 2748 i40e_vlan_stripping_disable(vsi); 2749 } 2750 2751 /** 2752 * i40e_add_vlan_all_mac - Add a MAC/VLAN filter for each existing MAC address 2753 * @vsi: the vsi being configured 2754 * @vid: vlan id to be added (0 = untagged only , -1 = any) 2755 * 2756 * This is a helper function for adding a new MAC/VLAN filter with the 2757 * specified VLAN for each existing MAC address already in the hash table. 2758 * This function does *not* perform any accounting to update filters based on 2759 * VLAN mode. 2760 * 2761 * NOTE: this function expects to be called while under the 2762 * mac_filter_hash_lock 2763 **/ 2764 int i40e_add_vlan_all_mac(struct i40e_vsi *vsi, s16 vid) 2765 { 2766 struct i40e_mac_filter *f, *add_f; 2767 struct hlist_node *h; 2768 int bkt; 2769 2770 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) { 2771 if (f->state == I40E_FILTER_REMOVE) 2772 continue; 2773 add_f = i40e_add_filter(vsi, f->macaddr, vid); 2774 if (!add_f) { 2775 dev_info(&vsi->back->pdev->dev, 2776 "Could not add vlan filter %d for %pM\n", 2777 vid, f->macaddr); 2778 return -ENOMEM; 2779 } 2780 } 2781 2782 return 0; 2783 } 2784 2785 /** 2786 * i40e_vsi_add_vlan - Add VSI membership for given VLAN 2787 * @vsi: the VSI being configured 2788 * @vid: VLAN id to be added 2789 **/ 2790 int i40e_vsi_add_vlan(struct i40e_vsi *vsi, u16 vid) 2791 { 2792 int err; 2793 2794 if (vsi->info.pvid) 2795 return -EINVAL; 2796 2797 /* The network stack will attempt to add VID=0, with the intention to 2798 * receive priority tagged packets with a VLAN of 0. Our HW receives 2799 * these packets by default when configured to receive untagged 2800 * packets, so we don't need to add a filter for this case. 2801 * Additionally, HW interprets adding a VID=0 filter as meaning to 2802 * receive *only* tagged traffic and stops receiving untagged traffic. 2803 * Thus, we do not want to actually add a filter for VID=0 2804 */ 2805 if (!vid) 2806 return 0; 2807 2808 /* Locked once because all functions invoked below iterates list*/ 2809 spin_lock_bh(&vsi->mac_filter_hash_lock); 2810 err = i40e_add_vlan_all_mac(vsi, vid); 2811 spin_unlock_bh(&vsi->mac_filter_hash_lock); 2812 if (err) 2813 return err; 2814 2815 /* schedule our worker thread which will take care of 2816 * applying the new filter changes 2817 */ 2818 i40e_service_event_schedule(vsi->back); 2819 return 0; 2820 } 2821 2822 /** 2823 * i40e_rm_vlan_all_mac - Remove MAC/VLAN pair for all MAC with the given VLAN 2824 * @vsi: the vsi being configured 2825 * @vid: vlan id to be removed (0 = untagged only , -1 = any) 2826 * 2827 * This function should be used to remove all VLAN filters which match the 2828 * given VID. It does not schedule the service event and does not take the 2829 * mac_filter_hash_lock so it may be combined with other operations under 2830 * a single invocation of the mac_filter_hash_lock. 2831 * 2832 * NOTE: this function expects to be called while under the 2833 * mac_filter_hash_lock 2834 */ 2835 void i40e_rm_vlan_all_mac(struct i40e_vsi *vsi, s16 vid) 2836 { 2837 struct i40e_mac_filter *f; 2838 struct hlist_node *h; 2839 int bkt; 2840 2841 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) { 2842 if (f->vlan == vid) 2843 __i40e_del_filter(vsi, f); 2844 } 2845 } 2846 2847 /** 2848 * i40e_vsi_kill_vlan - Remove VSI membership for given VLAN 2849 * @vsi: the VSI being configured 2850 * @vid: VLAN id to be removed 2851 **/ 2852 void i40e_vsi_kill_vlan(struct i40e_vsi *vsi, u16 vid) 2853 { 2854 if (!vid || vsi->info.pvid) 2855 return; 2856 2857 spin_lock_bh(&vsi->mac_filter_hash_lock); 2858 i40e_rm_vlan_all_mac(vsi, vid); 2859 spin_unlock_bh(&vsi->mac_filter_hash_lock); 2860 2861 /* schedule our worker thread which will take care of 2862 * applying the new filter changes 2863 */ 2864 i40e_service_event_schedule(vsi->back); 2865 } 2866 2867 /** 2868 * i40e_vlan_rx_add_vid - Add a vlan id filter to HW offload 2869 * @netdev: network interface to be adjusted 2870 * @vid: vlan id to be added 2871 * 2872 * net_device_ops implementation for adding vlan ids 2873 **/ 2874 static int i40e_vlan_rx_add_vid(struct net_device *netdev, 2875 __always_unused __be16 proto, u16 vid) 2876 { 2877 struct i40e_netdev_priv *np = netdev_priv(netdev); 2878 struct i40e_vsi *vsi = np->vsi; 2879 int ret = 0; 2880 2881 if (vid >= VLAN_N_VID) 2882 return -EINVAL; 2883 2884 ret = i40e_vsi_add_vlan(vsi, vid); 2885 if (!ret) 2886 set_bit(vid, vsi->active_vlans); 2887 2888 return ret; 2889 } 2890 2891 /** 2892 * i40e_vlan_rx_kill_vid - Remove a vlan id filter from HW offload 2893 * @netdev: network interface to be adjusted 2894 * @vid: vlan id to be removed 2895 * 2896 * net_device_ops implementation for removing vlan ids 2897 **/ 2898 static int i40e_vlan_rx_kill_vid(struct net_device *netdev, 2899 __always_unused __be16 proto, u16 vid) 2900 { 2901 struct i40e_netdev_priv *np = netdev_priv(netdev); 2902 struct i40e_vsi *vsi = np->vsi; 2903 2904 /* return code is ignored as there is nothing a user 2905 * can do about failure to remove and a log message was 2906 * already printed from the other function 2907 */ 2908 i40e_vsi_kill_vlan(vsi, vid); 2909 2910 clear_bit(vid, vsi->active_vlans); 2911 2912 return 0; 2913 } 2914 2915 /** 2916 * i40e_restore_vlan - Reinstate vlans when vsi/netdev comes back up 2917 * @vsi: the vsi being brought back up 2918 **/ 2919 static void i40e_restore_vlan(struct i40e_vsi *vsi) 2920 { 2921 u16 vid; 2922 2923 if (!vsi->netdev) 2924 return; 2925 2926 i40e_vlan_rx_register(vsi->netdev, vsi->netdev->features); 2927 2928 for_each_set_bit(vid, vsi->active_vlans, VLAN_N_VID) 2929 i40e_vlan_rx_add_vid(vsi->netdev, htons(ETH_P_8021Q), 2930 vid); 2931 } 2932 2933 /** 2934 * i40e_vsi_add_pvid - Add pvid for the VSI 2935 * @vsi: the vsi being adjusted 2936 * @vid: the vlan id to set as a PVID 2937 **/ 2938 int i40e_vsi_add_pvid(struct i40e_vsi *vsi, u16 vid) 2939 { 2940 struct i40e_vsi_context ctxt; 2941 i40e_status ret; 2942 2943 vsi->info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID); 2944 vsi->info.pvid = cpu_to_le16(vid); 2945 vsi->info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_TAGGED | 2946 I40E_AQ_VSI_PVLAN_INSERT_PVID | 2947 I40E_AQ_VSI_PVLAN_EMOD_STR; 2948 2949 ctxt.seid = vsi->seid; 2950 ctxt.info = vsi->info; 2951 ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL); 2952 if (ret) { 2953 dev_info(&vsi->back->pdev->dev, 2954 "add pvid failed, err %s aq_err %s\n", 2955 i40e_stat_str(&vsi->back->hw, ret), 2956 i40e_aq_str(&vsi->back->hw, 2957 vsi->back->hw.aq.asq_last_status)); 2958 return -ENOENT; 2959 } 2960 2961 return 0; 2962 } 2963 2964 /** 2965 * i40e_vsi_remove_pvid - Remove the pvid from the VSI 2966 * @vsi: the vsi being adjusted 2967 * 2968 * Just use the vlan_rx_register() service to put it back to normal 2969 **/ 2970 void i40e_vsi_remove_pvid(struct i40e_vsi *vsi) 2971 { 2972 i40e_vlan_stripping_disable(vsi); 2973 2974 vsi->info.pvid = 0; 2975 } 2976 2977 /** 2978 * i40e_vsi_setup_tx_resources - Allocate VSI Tx queue resources 2979 * @vsi: ptr to the VSI 2980 * 2981 * If this function returns with an error, then it's possible one or 2982 * more of the rings is populated (while the rest are not). It is the 2983 * callers duty to clean those orphaned rings. 2984 * 2985 * Return 0 on success, negative on failure 2986 **/ 2987 static int i40e_vsi_setup_tx_resources(struct i40e_vsi *vsi) 2988 { 2989 int i, err = 0; 2990 2991 for (i = 0; i < vsi->num_queue_pairs && !err; i++) 2992 err = i40e_setup_tx_descriptors(vsi->tx_rings[i]); 2993 2994 if (!i40e_enabled_xdp_vsi(vsi)) 2995 return err; 2996 2997 for (i = 0; i < vsi->num_queue_pairs && !err; i++) 2998 err = i40e_setup_tx_descriptors(vsi->xdp_rings[i]); 2999 3000 return err; 3001 } 3002 3003 /** 3004 * i40e_vsi_free_tx_resources - Free Tx resources for VSI queues 3005 * @vsi: ptr to the VSI 3006 * 3007 * Free VSI's transmit software resources 3008 **/ 3009 static void i40e_vsi_free_tx_resources(struct i40e_vsi *vsi) 3010 { 3011 int i; 3012 3013 if (vsi->tx_rings) { 3014 for (i = 0; i < vsi->num_queue_pairs; i++) 3015 if (vsi->tx_rings[i] && vsi->tx_rings[i]->desc) 3016 i40e_free_tx_resources(vsi->tx_rings[i]); 3017 } 3018 3019 if (vsi->xdp_rings) { 3020 for (i = 0; i < vsi->num_queue_pairs; i++) 3021 if (vsi->xdp_rings[i] && vsi->xdp_rings[i]->desc) 3022 i40e_free_tx_resources(vsi->xdp_rings[i]); 3023 } 3024 } 3025 3026 /** 3027 * i40e_vsi_setup_rx_resources - Allocate VSI queues Rx resources 3028 * @vsi: ptr to the VSI 3029 * 3030 * If this function returns with an error, then it's possible one or 3031 * more of the rings is populated (while the rest are not). It is the 3032 * callers duty to clean those orphaned rings. 3033 * 3034 * Return 0 on success, negative on failure 3035 **/ 3036 static int i40e_vsi_setup_rx_resources(struct i40e_vsi *vsi) 3037 { 3038 int i, err = 0; 3039 3040 for (i = 0; i < vsi->num_queue_pairs && !err; i++) 3041 err = i40e_setup_rx_descriptors(vsi->rx_rings[i]); 3042 return err; 3043 } 3044 3045 /** 3046 * i40e_vsi_free_rx_resources - Free Rx Resources for VSI queues 3047 * @vsi: ptr to the VSI 3048 * 3049 * Free all receive software resources 3050 **/ 3051 static void i40e_vsi_free_rx_resources(struct i40e_vsi *vsi) 3052 { 3053 int i; 3054 3055 if (!vsi->rx_rings) 3056 return; 3057 3058 for (i = 0; i < vsi->num_queue_pairs; i++) 3059 if (vsi->rx_rings[i] && vsi->rx_rings[i]->desc) 3060 i40e_free_rx_resources(vsi->rx_rings[i]); 3061 } 3062 3063 /** 3064 * i40e_config_xps_tx_ring - Configure XPS for a Tx ring 3065 * @ring: The Tx ring to configure 3066 * 3067 * This enables/disables XPS for a given Tx descriptor ring 3068 * based on the TCs enabled for the VSI that ring belongs to. 3069 **/ 3070 static void i40e_config_xps_tx_ring(struct i40e_ring *ring) 3071 { 3072 int cpu; 3073 3074 if (!ring->q_vector || !ring->netdev || ring->ch) 3075 return; 3076 3077 /* We only initialize XPS once, so as not to overwrite user settings */ 3078 if (test_and_set_bit(__I40E_TX_XPS_INIT_DONE, ring->state)) 3079 return; 3080 3081 cpu = cpumask_local_spread(ring->q_vector->v_idx, -1); 3082 netif_set_xps_queue(ring->netdev, get_cpu_mask(cpu), 3083 ring->queue_index); 3084 } 3085 3086 /** 3087 * i40e_configure_tx_ring - Configure a transmit ring context and rest 3088 * @ring: The Tx ring to configure 3089 * 3090 * Configure the Tx descriptor ring in the HMC context. 3091 **/ 3092 static int i40e_configure_tx_ring(struct i40e_ring *ring) 3093 { 3094 struct i40e_vsi *vsi = ring->vsi; 3095 u16 pf_q = vsi->base_queue + ring->queue_index; 3096 struct i40e_hw *hw = &vsi->back->hw; 3097 struct i40e_hmc_obj_txq tx_ctx; 3098 i40e_status err = 0; 3099 u32 qtx_ctl = 0; 3100 3101 /* some ATR related tx ring init */ 3102 if (vsi->back->flags & I40E_FLAG_FD_ATR_ENABLED) { 3103 ring->atr_sample_rate = vsi->back->atr_sample_rate; 3104 ring->atr_count = 0; 3105 } else { 3106 ring->atr_sample_rate = 0; 3107 } 3108 3109 /* configure XPS */ 3110 i40e_config_xps_tx_ring(ring); 3111 3112 /* clear the context structure first */ 3113 memset(&tx_ctx, 0, sizeof(tx_ctx)); 3114 3115 tx_ctx.new_context = 1; 3116 tx_ctx.base = (ring->dma / 128); 3117 tx_ctx.qlen = ring->count; 3118 tx_ctx.fd_ena = !!(vsi->back->flags & (I40E_FLAG_FD_SB_ENABLED | 3119 I40E_FLAG_FD_ATR_ENABLED)); 3120 tx_ctx.timesync_ena = !!(vsi->back->flags & I40E_FLAG_PTP); 3121 /* FDIR VSI tx ring can still use RS bit and writebacks */ 3122 if (vsi->type != I40E_VSI_FDIR) 3123 tx_ctx.head_wb_ena = 1; 3124 tx_ctx.head_wb_addr = ring->dma + 3125 (ring->count * sizeof(struct i40e_tx_desc)); 3126 3127 /* As part of VSI creation/update, FW allocates certain 3128 * Tx arbitration queue sets for each TC enabled for 3129 * the VSI. The FW returns the handles to these queue 3130 * sets as part of the response buffer to Add VSI, 3131 * Update VSI, etc. AQ commands. It is expected that 3132 * these queue set handles be associated with the Tx 3133 * queues by the driver as part of the TX queue context 3134 * initialization. This has to be done regardless of 3135 * DCB as by default everything is mapped to TC0. 3136 */ 3137 3138 if (ring->ch) 3139 tx_ctx.rdylist = 3140 le16_to_cpu(ring->ch->info.qs_handle[ring->dcb_tc]); 3141 3142 else 3143 tx_ctx.rdylist = le16_to_cpu(vsi->info.qs_handle[ring->dcb_tc]); 3144 3145 tx_ctx.rdylist_act = 0; 3146 3147 /* clear the context in the HMC */ 3148 err = i40e_clear_lan_tx_queue_context(hw, pf_q); 3149 if (err) { 3150 dev_info(&vsi->back->pdev->dev, 3151 "Failed to clear LAN Tx queue context on Tx ring %d (pf_q %d), error: %d\n", 3152 ring->queue_index, pf_q, err); 3153 return -ENOMEM; 3154 } 3155 3156 /* set the context in the HMC */ 3157 err = i40e_set_lan_tx_queue_context(hw, pf_q, &tx_ctx); 3158 if (err) { 3159 dev_info(&vsi->back->pdev->dev, 3160 "Failed to set LAN Tx queue context on Tx ring %d (pf_q %d, error: %d\n", 3161 ring->queue_index, pf_q, err); 3162 return -ENOMEM; 3163 } 3164 3165 /* Now associate this queue with this PCI function */ 3166 if (ring->ch) { 3167 if (ring->ch->type == I40E_VSI_VMDQ2) 3168 qtx_ctl = I40E_QTX_CTL_VM_QUEUE; 3169 else 3170 return -EINVAL; 3171 3172 qtx_ctl |= (ring->ch->vsi_number << 3173 I40E_QTX_CTL_VFVM_INDX_SHIFT) & 3174 I40E_QTX_CTL_VFVM_INDX_MASK; 3175 } else { 3176 if (vsi->type == I40E_VSI_VMDQ2) { 3177 qtx_ctl = I40E_QTX_CTL_VM_QUEUE; 3178 qtx_ctl |= ((vsi->id) << I40E_QTX_CTL_VFVM_INDX_SHIFT) & 3179 I40E_QTX_CTL_VFVM_INDX_MASK; 3180 } else { 3181 qtx_ctl = I40E_QTX_CTL_PF_QUEUE; 3182 } 3183 } 3184 3185 qtx_ctl |= ((hw->pf_id << I40E_QTX_CTL_PF_INDX_SHIFT) & 3186 I40E_QTX_CTL_PF_INDX_MASK); 3187 wr32(hw, I40E_QTX_CTL(pf_q), qtx_ctl); 3188 i40e_flush(hw); 3189 3190 /* cache tail off for easier writes later */ 3191 ring->tail = hw->hw_addr + I40E_QTX_TAIL(pf_q); 3192 3193 return 0; 3194 } 3195 3196 /** 3197 * i40e_configure_rx_ring - Configure a receive ring context 3198 * @ring: The Rx ring to configure 3199 * 3200 * Configure the Rx descriptor ring in the HMC context. 3201 **/ 3202 static int i40e_configure_rx_ring(struct i40e_ring *ring) 3203 { 3204 struct i40e_vsi *vsi = ring->vsi; 3205 u32 chain_len = vsi->back->hw.func_caps.rx_buf_chain_len; 3206 u16 pf_q = vsi->base_queue + ring->queue_index; 3207 struct i40e_hw *hw = &vsi->back->hw; 3208 struct i40e_hmc_obj_rxq rx_ctx; 3209 i40e_status err = 0; 3210 3211 bitmap_zero(ring->state, __I40E_RING_STATE_NBITS); 3212 3213 /* clear the context structure first */ 3214 memset(&rx_ctx, 0, sizeof(rx_ctx)); 3215 3216 ring->rx_buf_len = vsi->rx_buf_len; 3217 3218 rx_ctx.dbuff = DIV_ROUND_UP(ring->rx_buf_len, 3219 BIT_ULL(I40E_RXQ_CTX_DBUFF_SHIFT)); 3220 3221 rx_ctx.base = (ring->dma / 128); 3222 rx_ctx.qlen = ring->count; 3223 3224 /* use 32 byte descriptors */ 3225 rx_ctx.dsize = 1; 3226 3227 /* descriptor type is always zero 3228 * rx_ctx.dtype = 0; 3229 */ 3230 rx_ctx.hsplit_0 = 0; 3231 3232 rx_ctx.rxmax = min_t(u16, vsi->max_frame, chain_len * ring->rx_buf_len); 3233 if (hw->revision_id == 0) 3234 rx_ctx.lrxqthresh = 0; 3235 else 3236 rx_ctx.lrxqthresh = 1; 3237 rx_ctx.crcstrip = 1; 3238 rx_ctx.l2tsel = 1; 3239 /* this controls whether VLAN is stripped from inner headers */ 3240 rx_ctx.showiv = 0; 3241 /* set the prefena field to 1 because the manual says to */ 3242 rx_ctx.prefena = 1; 3243 3244 /* clear the context in the HMC */ 3245 err = i40e_clear_lan_rx_queue_context(hw, pf_q); 3246 if (err) { 3247 dev_info(&vsi->back->pdev->dev, 3248 "Failed to clear LAN Rx queue context on Rx ring %d (pf_q %d), error: %d\n", 3249 ring->queue_index, pf_q, err); 3250 return -ENOMEM; 3251 } 3252 3253 /* set the context in the HMC */ 3254 err = i40e_set_lan_rx_queue_context(hw, pf_q, &rx_ctx); 3255 if (err) { 3256 dev_info(&vsi->back->pdev->dev, 3257 "Failed to set LAN Rx queue context on Rx ring %d (pf_q %d), error: %d\n", 3258 ring->queue_index, pf_q, err); 3259 return -ENOMEM; 3260 } 3261 3262 /* configure Rx buffer alignment */ 3263 if (!vsi->netdev || (vsi->back->flags & I40E_FLAG_LEGACY_RX)) 3264 clear_ring_build_skb_enabled(ring); 3265 else 3266 set_ring_build_skb_enabled(ring); 3267 3268 /* cache tail for quicker writes, and clear the reg before use */ 3269 ring->tail = hw->hw_addr + I40E_QRX_TAIL(pf_q); 3270 writel(0, ring->tail); 3271 3272 i40e_alloc_rx_buffers(ring, I40E_DESC_UNUSED(ring)); 3273 3274 return 0; 3275 } 3276 3277 /** 3278 * i40e_vsi_configure_tx - Configure the VSI for Tx 3279 * @vsi: VSI structure describing this set of rings and resources 3280 * 3281 * Configure the Tx VSI for operation. 3282 **/ 3283 static int i40e_vsi_configure_tx(struct i40e_vsi *vsi) 3284 { 3285 int err = 0; 3286 u16 i; 3287 3288 for (i = 0; (i < vsi->num_queue_pairs) && !err; i++) 3289 err = i40e_configure_tx_ring(vsi->tx_rings[i]); 3290 3291 if (!i40e_enabled_xdp_vsi(vsi)) 3292 return err; 3293 3294 for (i = 0; (i < vsi->num_queue_pairs) && !err; i++) 3295 err = i40e_configure_tx_ring(vsi->xdp_rings[i]); 3296 3297 return err; 3298 } 3299 3300 /** 3301 * i40e_vsi_configure_rx - Configure the VSI for Rx 3302 * @vsi: the VSI being configured 3303 * 3304 * Configure the Rx VSI for operation. 3305 **/ 3306 static int i40e_vsi_configure_rx(struct i40e_vsi *vsi) 3307 { 3308 int err = 0; 3309 u16 i; 3310 3311 if (!vsi->netdev || (vsi->back->flags & I40E_FLAG_LEGACY_RX)) { 3312 vsi->max_frame = I40E_MAX_RXBUFFER; 3313 vsi->rx_buf_len = I40E_RXBUFFER_2048; 3314 #if (PAGE_SIZE < 8192) 3315 } else if (!I40E_2K_TOO_SMALL_WITH_PADDING && 3316 (vsi->netdev->mtu <= ETH_DATA_LEN)) { 3317 vsi->max_frame = I40E_RXBUFFER_1536 - NET_IP_ALIGN; 3318 vsi->rx_buf_len = I40E_RXBUFFER_1536 - NET_IP_ALIGN; 3319 #endif 3320 } else { 3321 vsi->max_frame = I40E_MAX_RXBUFFER; 3322 vsi->rx_buf_len = (PAGE_SIZE < 8192) ? I40E_RXBUFFER_3072 : 3323 I40E_RXBUFFER_2048; 3324 } 3325 3326 /* set up individual rings */ 3327 for (i = 0; i < vsi->num_queue_pairs && !err; i++) 3328 err = i40e_configure_rx_ring(vsi->rx_rings[i]); 3329 3330 return err; 3331 } 3332 3333 /** 3334 * i40e_vsi_config_dcb_rings - Update rings to reflect DCB TC 3335 * @vsi: ptr to the VSI 3336 **/ 3337 static void i40e_vsi_config_dcb_rings(struct i40e_vsi *vsi) 3338 { 3339 struct i40e_ring *tx_ring, *rx_ring; 3340 u16 qoffset, qcount; 3341 int i, n; 3342 3343 if (!(vsi->back->flags & I40E_FLAG_DCB_ENABLED)) { 3344 /* Reset the TC information */ 3345 for (i = 0; i < vsi->num_queue_pairs; i++) { 3346 rx_ring = vsi->rx_rings[i]; 3347 tx_ring = vsi->tx_rings[i]; 3348 rx_ring->dcb_tc = 0; 3349 tx_ring->dcb_tc = 0; 3350 } 3351 return; 3352 } 3353 3354 for (n = 0; n < I40E_MAX_TRAFFIC_CLASS; n++) { 3355 if (!(vsi->tc_config.enabled_tc & BIT_ULL(n))) 3356 continue; 3357 3358 qoffset = vsi->tc_config.tc_info[n].qoffset; 3359 qcount = vsi->tc_config.tc_info[n].qcount; 3360 for (i = qoffset; i < (qoffset + qcount); i++) { 3361 rx_ring = vsi->rx_rings[i]; 3362 tx_ring = vsi->tx_rings[i]; 3363 rx_ring->dcb_tc = n; 3364 tx_ring->dcb_tc = n; 3365 } 3366 } 3367 } 3368 3369 /** 3370 * i40e_set_vsi_rx_mode - Call set_rx_mode on a VSI 3371 * @vsi: ptr to the VSI 3372 **/ 3373 static void i40e_set_vsi_rx_mode(struct i40e_vsi *vsi) 3374 { 3375 if (vsi->netdev) 3376 i40e_set_rx_mode(vsi->netdev); 3377 } 3378 3379 /** 3380 * i40e_fdir_filter_restore - Restore the Sideband Flow Director filters 3381 * @vsi: Pointer to the targeted VSI 3382 * 3383 * This function replays the hlist on the hw where all the SB Flow Director 3384 * filters were saved. 3385 **/ 3386 static void i40e_fdir_filter_restore(struct i40e_vsi *vsi) 3387 { 3388 struct i40e_fdir_filter *filter; 3389 struct i40e_pf *pf = vsi->back; 3390 struct hlist_node *node; 3391 3392 if (!(pf->flags & I40E_FLAG_FD_SB_ENABLED)) 3393 return; 3394 3395 /* Reset FDir counters as we're replaying all existing filters */ 3396 pf->fd_tcp4_filter_cnt = 0; 3397 pf->fd_udp4_filter_cnt = 0; 3398 pf->fd_sctp4_filter_cnt = 0; 3399 pf->fd_ip4_filter_cnt = 0; 3400 3401 hlist_for_each_entry_safe(filter, node, 3402 &pf->fdir_filter_list, fdir_node) { 3403 i40e_add_del_fdir(vsi, filter, true); 3404 } 3405 } 3406 3407 /** 3408 * i40e_vsi_configure - Set up the VSI for action 3409 * @vsi: the VSI being configured 3410 **/ 3411 static int i40e_vsi_configure(struct i40e_vsi *vsi) 3412 { 3413 int err; 3414 3415 i40e_set_vsi_rx_mode(vsi); 3416 i40e_restore_vlan(vsi); 3417 i40e_vsi_config_dcb_rings(vsi); 3418 err = i40e_vsi_configure_tx(vsi); 3419 if (!err) 3420 err = i40e_vsi_configure_rx(vsi); 3421 3422 return err; 3423 } 3424 3425 /** 3426 * i40e_vsi_configure_msix - MSIX mode Interrupt Config in the HW 3427 * @vsi: the VSI being configured 3428 **/ 3429 static void i40e_vsi_configure_msix(struct i40e_vsi *vsi) 3430 { 3431 bool has_xdp = i40e_enabled_xdp_vsi(vsi); 3432 struct i40e_pf *pf = vsi->back; 3433 struct i40e_hw *hw = &pf->hw; 3434 u16 vector; 3435 int i, q; 3436 u32 qp; 3437 3438 /* The interrupt indexing is offset by 1 in the PFINT_ITRn 3439 * and PFINT_LNKLSTn registers, e.g.: 3440 * PFINT_ITRn[0..n-1] gets msix-1..msix-n (qpair interrupts) 3441 */ 3442 qp = vsi->base_queue; 3443 vector = vsi->base_vector; 3444 for (i = 0; i < vsi->num_q_vectors; i++, vector++) { 3445 struct i40e_q_vector *q_vector = vsi->q_vectors[i]; 3446 3447 q_vector->itr_countdown = ITR_COUNTDOWN_START; 3448 q_vector->rx.itr = ITR_TO_REG(vsi->rx_rings[i]->rx_itr_setting); 3449 q_vector->rx.latency_range = I40E_LOW_LATENCY; 3450 wr32(hw, I40E_PFINT_ITRN(I40E_RX_ITR, vector - 1), 3451 q_vector->rx.itr); 3452 q_vector->tx.itr = ITR_TO_REG(vsi->tx_rings[i]->tx_itr_setting); 3453 q_vector->tx.latency_range = I40E_LOW_LATENCY; 3454 wr32(hw, I40E_PFINT_ITRN(I40E_TX_ITR, vector - 1), 3455 q_vector->tx.itr); 3456 wr32(hw, I40E_PFINT_RATEN(vector - 1), 3457 i40e_intrl_usec_to_reg(vsi->int_rate_limit)); 3458 3459 /* Linked list for the queuepairs assigned to this vector */ 3460 wr32(hw, I40E_PFINT_LNKLSTN(vector - 1), qp); 3461 for (q = 0; q < q_vector->num_ringpairs; q++) { 3462 u32 nextqp = has_xdp ? qp + vsi->alloc_queue_pairs : qp; 3463 u32 val; 3464 3465 val = I40E_QINT_RQCTL_CAUSE_ENA_MASK | 3466 (I40E_RX_ITR << I40E_QINT_RQCTL_ITR_INDX_SHIFT) | 3467 (vector << I40E_QINT_RQCTL_MSIX_INDX_SHIFT) | 3468 (nextqp << I40E_QINT_RQCTL_NEXTQ_INDX_SHIFT) | 3469 (I40E_QUEUE_TYPE_TX << 3470 I40E_QINT_RQCTL_NEXTQ_TYPE_SHIFT); 3471 3472 wr32(hw, I40E_QINT_RQCTL(qp), val); 3473 3474 if (has_xdp) { 3475 val = I40E_QINT_TQCTL_CAUSE_ENA_MASK | 3476 (I40E_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT) | 3477 (vector << I40E_QINT_TQCTL_MSIX_INDX_SHIFT) | 3478 (qp << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT) | 3479 (I40E_QUEUE_TYPE_TX << 3480 I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT); 3481 3482 wr32(hw, I40E_QINT_TQCTL(nextqp), val); 3483 } 3484 3485 val = I40E_QINT_TQCTL_CAUSE_ENA_MASK | 3486 (I40E_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT) | 3487 (vector << I40E_QINT_TQCTL_MSIX_INDX_SHIFT) | 3488 ((qp + 1) << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT) | 3489 (I40E_QUEUE_TYPE_RX << 3490 I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT); 3491 3492 /* Terminate the linked list */ 3493 if (q == (q_vector->num_ringpairs - 1)) 3494 val |= (I40E_QUEUE_END_OF_LIST << 3495 I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT); 3496 3497 wr32(hw, I40E_QINT_TQCTL(qp), val); 3498 qp++; 3499 } 3500 } 3501 3502 i40e_flush(hw); 3503 } 3504 3505 /** 3506 * i40e_enable_misc_int_causes - enable the non-queue interrupts 3507 * @hw: ptr to the hardware info 3508 **/ 3509 static void i40e_enable_misc_int_causes(struct i40e_pf *pf) 3510 { 3511 struct i40e_hw *hw = &pf->hw; 3512 u32 val; 3513 3514 /* clear things first */ 3515 wr32(hw, I40E_PFINT_ICR0_ENA, 0); /* disable all */ 3516 rd32(hw, I40E_PFINT_ICR0); /* read to clear */ 3517 3518 val = I40E_PFINT_ICR0_ENA_ECC_ERR_MASK | 3519 I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK | 3520 I40E_PFINT_ICR0_ENA_GRST_MASK | 3521 I40E_PFINT_ICR0_ENA_PCI_EXCEPTION_MASK | 3522 I40E_PFINT_ICR0_ENA_GPIO_MASK | 3523 I40E_PFINT_ICR0_ENA_HMC_ERR_MASK | 3524 I40E_PFINT_ICR0_ENA_VFLR_MASK | 3525 I40E_PFINT_ICR0_ENA_ADMINQ_MASK; 3526 3527 if (pf->flags & I40E_FLAG_IWARP_ENABLED) 3528 val |= I40E_PFINT_ICR0_ENA_PE_CRITERR_MASK; 3529 3530 if (pf->flags & I40E_FLAG_PTP) 3531 val |= I40E_PFINT_ICR0_ENA_TIMESYNC_MASK; 3532 3533 wr32(hw, I40E_PFINT_ICR0_ENA, val); 3534 3535 /* SW_ITR_IDX = 0, but don't change INTENA */ 3536 wr32(hw, I40E_PFINT_DYN_CTL0, I40E_PFINT_DYN_CTL0_SW_ITR_INDX_MASK | 3537 I40E_PFINT_DYN_CTL0_INTENA_MSK_MASK); 3538 3539 /* OTHER_ITR_IDX = 0 */ 3540 wr32(hw, I40E_PFINT_STAT_CTL0, 0); 3541 } 3542 3543 /** 3544 * i40e_configure_msi_and_legacy - Legacy mode interrupt config in the HW 3545 * @vsi: the VSI being configured 3546 **/ 3547 static void i40e_configure_msi_and_legacy(struct i40e_vsi *vsi) 3548 { 3549 u32 nextqp = i40e_enabled_xdp_vsi(vsi) ? vsi->alloc_queue_pairs : 0; 3550 struct i40e_q_vector *q_vector = vsi->q_vectors[0]; 3551 struct i40e_pf *pf = vsi->back; 3552 struct i40e_hw *hw = &pf->hw; 3553 u32 val; 3554 3555 /* set the ITR configuration */ 3556 q_vector->itr_countdown = ITR_COUNTDOWN_START; 3557 q_vector->rx.itr = ITR_TO_REG(vsi->rx_rings[0]->rx_itr_setting); 3558 q_vector->rx.latency_range = I40E_LOW_LATENCY; 3559 wr32(hw, I40E_PFINT_ITR0(I40E_RX_ITR), q_vector->rx.itr); 3560 q_vector->tx.itr = ITR_TO_REG(vsi->tx_rings[0]->tx_itr_setting); 3561 q_vector->tx.latency_range = I40E_LOW_LATENCY; 3562 wr32(hw, I40E_PFINT_ITR0(I40E_TX_ITR), q_vector->tx.itr); 3563 3564 i40e_enable_misc_int_causes(pf); 3565 3566 /* FIRSTQ_INDX = 0, FIRSTQ_TYPE = 0 (rx) */ 3567 wr32(hw, I40E_PFINT_LNKLST0, 0); 3568 3569 /* Associate the queue pair to the vector and enable the queue int */ 3570 val = I40E_QINT_RQCTL_CAUSE_ENA_MASK | 3571 (I40E_RX_ITR << I40E_QINT_RQCTL_ITR_INDX_SHIFT) | 3572 (nextqp << I40E_QINT_RQCTL_NEXTQ_INDX_SHIFT)| 3573 (I40E_QUEUE_TYPE_TX << I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT); 3574 3575 wr32(hw, I40E_QINT_RQCTL(0), val); 3576 3577 if (i40e_enabled_xdp_vsi(vsi)) { 3578 val = I40E_QINT_TQCTL_CAUSE_ENA_MASK | 3579 (I40E_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT)| 3580 (I40E_QUEUE_TYPE_TX 3581 << I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT); 3582 3583 wr32(hw, I40E_QINT_TQCTL(nextqp), val); 3584 } 3585 3586 val = I40E_QINT_TQCTL_CAUSE_ENA_MASK | 3587 (I40E_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT) | 3588 (I40E_QUEUE_END_OF_LIST << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT); 3589 3590 wr32(hw, I40E_QINT_TQCTL(0), val); 3591 i40e_flush(hw); 3592 } 3593 3594 /** 3595 * i40e_irq_dynamic_disable_icr0 - Disable default interrupt generation for icr0 3596 * @pf: board private structure 3597 **/ 3598 void i40e_irq_dynamic_disable_icr0(struct i40e_pf *pf) 3599 { 3600 struct i40e_hw *hw = &pf->hw; 3601 3602 wr32(hw, I40E_PFINT_DYN_CTL0, 3603 I40E_ITR_NONE << I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT); 3604 i40e_flush(hw); 3605 } 3606 3607 /** 3608 * i40e_irq_dynamic_enable_icr0 - Enable default interrupt generation for icr0 3609 * @pf: board private structure 3610 **/ 3611 void i40e_irq_dynamic_enable_icr0(struct i40e_pf *pf) 3612 { 3613 struct i40e_hw *hw = &pf->hw; 3614 u32 val; 3615 3616 val = I40E_PFINT_DYN_CTL0_INTENA_MASK | 3617 I40E_PFINT_DYN_CTL0_CLEARPBA_MASK | 3618 (I40E_ITR_NONE << I40E_PFINT_DYN_CTL0_ITR_INDX_SHIFT); 3619 3620 wr32(hw, I40E_PFINT_DYN_CTL0, val); 3621 i40e_flush(hw); 3622 } 3623 3624 /** 3625 * i40e_msix_clean_rings - MSIX mode Interrupt Handler 3626 * @irq: interrupt number 3627 * @data: pointer to a q_vector 3628 **/ 3629 static irqreturn_t i40e_msix_clean_rings(int irq, void *data) 3630 { 3631 struct i40e_q_vector *q_vector = data; 3632 3633 if (!q_vector->tx.ring && !q_vector->rx.ring) 3634 return IRQ_HANDLED; 3635 3636 napi_schedule_irqoff(&q_vector->napi); 3637 3638 return IRQ_HANDLED; 3639 } 3640 3641 /** 3642 * i40e_irq_affinity_notify - Callback for affinity changes 3643 * @notify: context as to what irq was changed 3644 * @mask: the new affinity mask 3645 * 3646 * This is a callback function used by the irq_set_affinity_notifier function 3647 * so that we may register to receive changes to the irq affinity masks. 3648 **/ 3649 static void i40e_irq_affinity_notify(struct irq_affinity_notify *notify, 3650 const cpumask_t *mask) 3651 { 3652 struct i40e_q_vector *q_vector = 3653 container_of(notify, struct i40e_q_vector, affinity_notify); 3654 3655 cpumask_copy(&q_vector->affinity_mask, mask); 3656 } 3657 3658 /** 3659 * i40e_irq_affinity_release - Callback for affinity notifier release 3660 * @ref: internal core kernel usage 3661 * 3662 * This is a callback function used by the irq_set_affinity_notifier function 3663 * to inform the current notification subscriber that they will no longer 3664 * receive notifications. 3665 **/ 3666 static void i40e_irq_affinity_release(struct kref *ref) {} 3667 3668 /** 3669 * i40e_vsi_request_irq_msix - Initialize MSI-X interrupts 3670 * @vsi: the VSI being configured 3671 * @basename: name for the vector 3672 * 3673 * Allocates MSI-X vectors and requests interrupts from the kernel. 3674 **/ 3675 static int i40e_vsi_request_irq_msix(struct i40e_vsi *vsi, char *basename) 3676 { 3677 int q_vectors = vsi->num_q_vectors; 3678 struct i40e_pf *pf = vsi->back; 3679 int base = vsi->base_vector; 3680 int rx_int_idx = 0; 3681 int tx_int_idx = 0; 3682 int vector, err; 3683 int irq_num; 3684 int cpu; 3685 3686 for (vector = 0; vector < q_vectors; vector++) { 3687 struct i40e_q_vector *q_vector = vsi->q_vectors[vector]; 3688 3689 irq_num = pf->msix_entries[base + vector].vector; 3690 3691 if (q_vector->tx.ring && q_vector->rx.ring) { 3692 snprintf(q_vector->name, sizeof(q_vector->name) - 1, 3693 "%s-%s-%d", basename, "TxRx", rx_int_idx++); 3694 tx_int_idx++; 3695 } else if (q_vector->rx.ring) { 3696 snprintf(q_vector->name, sizeof(q_vector->name) - 1, 3697 "%s-%s-%d", basename, "rx", rx_int_idx++); 3698 } else if (q_vector->tx.ring) { 3699 snprintf(q_vector->name, sizeof(q_vector->name) - 1, 3700 "%s-%s-%d", basename, "tx", tx_int_idx++); 3701 } else { 3702 /* skip this unused q_vector */ 3703 continue; 3704 } 3705 err = request_irq(irq_num, 3706 vsi->irq_handler, 3707 0, 3708 q_vector->name, 3709 q_vector); 3710 if (err) { 3711 dev_info(&pf->pdev->dev, 3712 "MSIX request_irq failed, error: %d\n", err); 3713 goto free_queue_irqs; 3714 } 3715 3716 /* register for affinity change notifications */ 3717 q_vector->affinity_notify.notify = i40e_irq_affinity_notify; 3718 q_vector->affinity_notify.release = i40e_irq_affinity_release; 3719 irq_set_affinity_notifier(irq_num, &q_vector->affinity_notify); 3720 /* Spread affinity hints out across online CPUs. 3721 * 3722 * get_cpu_mask returns a static constant mask with 3723 * a permanent lifetime so it's ok to pass to 3724 * irq_set_affinity_hint without making a copy. 3725 */ 3726 cpu = cpumask_local_spread(q_vector->v_idx, -1); 3727 irq_set_affinity_hint(irq_num, get_cpu_mask(cpu)); 3728 } 3729 3730 vsi->irqs_ready = true; 3731 return 0; 3732 3733 free_queue_irqs: 3734 while (vector) { 3735 vector--; 3736 irq_num = pf->msix_entries[base + vector].vector; 3737 irq_set_affinity_notifier(irq_num, NULL); 3738 irq_set_affinity_hint(irq_num, NULL); 3739 free_irq(irq_num, &vsi->q_vectors[vector]); 3740 } 3741 return err; 3742 } 3743 3744 /** 3745 * i40e_vsi_disable_irq - Mask off queue interrupt generation on the VSI 3746 * @vsi: the VSI being un-configured 3747 **/ 3748 static void i40e_vsi_disable_irq(struct i40e_vsi *vsi) 3749 { 3750 struct i40e_pf *pf = vsi->back; 3751 struct i40e_hw *hw = &pf->hw; 3752 int base = vsi->base_vector; 3753 int i; 3754 3755 /* disable interrupt causation from each queue */ 3756 for (i = 0; i < vsi->num_queue_pairs; i++) { 3757 u32 val; 3758 3759 val = rd32(hw, I40E_QINT_TQCTL(vsi->tx_rings[i]->reg_idx)); 3760 val &= ~I40E_QINT_TQCTL_CAUSE_ENA_MASK; 3761 wr32(hw, I40E_QINT_TQCTL(vsi->tx_rings[i]->reg_idx), val); 3762 3763 val = rd32(hw, I40E_QINT_RQCTL(vsi->rx_rings[i]->reg_idx)); 3764 val &= ~I40E_QINT_RQCTL_CAUSE_ENA_MASK; 3765 wr32(hw, I40E_QINT_RQCTL(vsi->rx_rings[i]->reg_idx), val); 3766 3767 if (!i40e_enabled_xdp_vsi(vsi)) 3768 continue; 3769 wr32(hw, I40E_QINT_TQCTL(vsi->xdp_rings[i]->reg_idx), 0); 3770 } 3771 3772 /* disable each interrupt */ 3773 if (pf->flags & I40E_FLAG_MSIX_ENABLED) { 3774 for (i = vsi->base_vector; 3775 i < (vsi->num_q_vectors + vsi->base_vector); i++) 3776 wr32(hw, I40E_PFINT_DYN_CTLN(i - 1), 0); 3777 3778 i40e_flush(hw); 3779 for (i = 0; i < vsi->num_q_vectors; i++) 3780 synchronize_irq(pf->msix_entries[i + base].vector); 3781 } else { 3782 /* Legacy and MSI mode - this stops all interrupt handling */ 3783 wr32(hw, I40E_PFINT_ICR0_ENA, 0); 3784 wr32(hw, I40E_PFINT_DYN_CTL0, 0); 3785 i40e_flush(hw); 3786 synchronize_irq(pf->pdev->irq); 3787 } 3788 } 3789 3790 /** 3791 * i40e_vsi_enable_irq - Enable IRQ for the given VSI 3792 * @vsi: the VSI being configured 3793 **/ 3794 static int i40e_vsi_enable_irq(struct i40e_vsi *vsi) 3795 { 3796 struct i40e_pf *pf = vsi->back; 3797 int i; 3798 3799 if (pf->flags & I40E_FLAG_MSIX_ENABLED) { 3800 for (i = 0; i < vsi->num_q_vectors; i++) 3801 i40e_irq_dynamic_enable(vsi, i); 3802 } else { 3803 i40e_irq_dynamic_enable_icr0(pf); 3804 } 3805 3806 i40e_flush(&pf->hw); 3807 return 0; 3808 } 3809 3810 /** 3811 * i40e_free_misc_vector - Free the vector that handles non-queue events 3812 * @pf: board private structure 3813 **/ 3814 static void i40e_free_misc_vector(struct i40e_pf *pf) 3815 { 3816 /* Disable ICR 0 */ 3817 wr32(&pf->hw, I40E_PFINT_ICR0_ENA, 0); 3818 i40e_flush(&pf->hw); 3819 3820 if (pf->flags & I40E_FLAG_MSIX_ENABLED && pf->msix_entries) { 3821 synchronize_irq(pf->msix_entries[0].vector); 3822 free_irq(pf->msix_entries[0].vector, pf); 3823 clear_bit(__I40E_MISC_IRQ_REQUESTED, pf->state); 3824 } 3825 } 3826 3827 /** 3828 * i40e_intr - MSI/Legacy and non-queue interrupt handler 3829 * @irq: interrupt number 3830 * @data: pointer to a q_vector 3831 * 3832 * This is the handler used for all MSI/Legacy interrupts, and deals 3833 * with both queue and non-queue interrupts. This is also used in 3834 * MSIX mode to handle the non-queue interrupts. 3835 **/ 3836 static irqreturn_t i40e_intr(int irq, void *data) 3837 { 3838 struct i40e_pf *pf = (struct i40e_pf *)data; 3839 struct i40e_hw *hw = &pf->hw; 3840 irqreturn_t ret = IRQ_NONE; 3841 u32 icr0, icr0_remaining; 3842 u32 val, ena_mask; 3843 3844 icr0 = rd32(hw, I40E_PFINT_ICR0); 3845 ena_mask = rd32(hw, I40E_PFINT_ICR0_ENA); 3846 3847 /* if sharing a legacy IRQ, we might get called w/o an intr pending */ 3848 if ((icr0 & I40E_PFINT_ICR0_INTEVENT_MASK) == 0) 3849 goto enable_intr; 3850 3851 /* if interrupt but no bits showing, must be SWINT */ 3852 if (((icr0 & ~I40E_PFINT_ICR0_INTEVENT_MASK) == 0) || 3853 (icr0 & I40E_PFINT_ICR0_SWINT_MASK)) 3854 pf->sw_int_count++; 3855 3856 if ((pf->flags & I40E_FLAG_IWARP_ENABLED) && 3857 (icr0 & I40E_PFINT_ICR0_ENA_PE_CRITERR_MASK)) { 3858 ena_mask &= ~I40E_PFINT_ICR0_ENA_PE_CRITERR_MASK; 3859 dev_dbg(&pf->pdev->dev, "cleared PE_CRITERR\n"); 3860 set_bit(__I40E_CORE_RESET_REQUESTED, pf->state); 3861 } 3862 3863 /* only q0 is used in MSI/Legacy mode, and none are used in MSIX */ 3864 if (icr0 & I40E_PFINT_ICR0_QUEUE_0_MASK) { 3865 struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi]; 3866 struct i40e_q_vector *q_vector = vsi->q_vectors[0]; 3867 3868 /* We do not have a way to disarm Queue causes while leaving 3869 * interrupt enabled for all other causes, ideally 3870 * interrupt should be disabled while we are in NAPI but 3871 * this is not a performance path and napi_schedule() 3872 * can deal with rescheduling. 3873 */ 3874 if (!test_bit(__I40E_DOWN, pf->state)) 3875 napi_schedule_irqoff(&q_vector->napi); 3876 } 3877 3878 if (icr0 & I40E_PFINT_ICR0_ADMINQ_MASK) { 3879 ena_mask &= ~I40E_PFINT_ICR0_ENA_ADMINQ_MASK; 3880 set_bit(__I40E_ADMINQ_EVENT_PENDING, pf->state); 3881 i40e_debug(&pf->hw, I40E_DEBUG_NVM, "AdminQ event\n"); 3882 } 3883 3884 if (icr0 & I40E_PFINT_ICR0_MAL_DETECT_MASK) { 3885 ena_mask &= ~I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK; 3886 set_bit(__I40E_MDD_EVENT_PENDING, pf->state); 3887 } 3888 3889 if (icr0 & I40E_PFINT_ICR0_VFLR_MASK) { 3890 ena_mask &= ~I40E_PFINT_ICR0_ENA_VFLR_MASK; 3891 set_bit(__I40E_VFLR_EVENT_PENDING, pf->state); 3892 } 3893 3894 if (icr0 & I40E_PFINT_ICR0_GRST_MASK) { 3895 if (!test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state)) 3896 set_bit(__I40E_RESET_INTR_RECEIVED, pf->state); 3897 ena_mask &= ~I40E_PFINT_ICR0_ENA_GRST_MASK; 3898 val = rd32(hw, I40E_GLGEN_RSTAT); 3899 val = (val & I40E_GLGEN_RSTAT_RESET_TYPE_MASK) 3900 >> I40E_GLGEN_RSTAT_RESET_TYPE_SHIFT; 3901 if (val == I40E_RESET_CORER) { 3902 pf->corer_count++; 3903 } else if (val == I40E_RESET_GLOBR) { 3904 pf->globr_count++; 3905 } else if (val == I40E_RESET_EMPR) { 3906 pf->empr_count++; 3907 set_bit(__I40E_EMP_RESET_INTR_RECEIVED, pf->state); 3908 } 3909 } 3910 3911 if (icr0 & I40E_PFINT_ICR0_HMC_ERR_MASK) { 3912 icr0 &= ~I40E_PFINT_ICR0_HMC_ERR_MASK; 3913 dev_info(&pf->pdev->dev, "HMC error interrupt\n"); 3914 dev_info(&pf->pdev->dev, "HMC error info 0x%x, HMC error data 0x%x\n", 3915 rd32(hw, I40E_PFHMC_ERRORINFO), 3916 rd32(hw, I40E_PFHMC_ERRORDATA)); 3917 } 3918 3919 if (icr0 & I40E_PFINT_ICR0_TIMESYNC_MASK) { 3920 u32 prttsyn_stat = rd32(hw, I40E_PRTTSYN_STAT_0); 3921 3922 if (prttsyn_stat & I40E_PRTTSYN_STAT_0_TXTIME_MASK) { 3923 icr0 &= ~I40E_PFINT_ICR0_ENA_TIMESYNC_MASK; 3924 i40e_ptp_tx_hwtstamp(pf); 3925 } 3926 } 3927 3928 /* If a critical error is pending we have no choice but to reset the 3929 * device. 3930 * Report and mask out any remaining unexpected interrupts. 3931 */ 3932 icr0_remaining = icr0 & ena_mask; 3933 if (icr0_remaining) { 3934 dev_info(&pf->pdev->dev, "unhandled interrupt icr0=0x%08x\n", 3935 icr0_remaining); 3936 if ((icr0_remaining & I40E_PFINT_ICR0_PE_CRITERR_MASK) || 3937 (icr0_remaining & I40E_PFINT_ICR0_PCI_EXCEPTION_MASK) || 3938 (icr0_remaining & I40E_PFINT_ICR0_ECC_ERR_MASK)) { 3939 dev_info(&pf->pdev->dev, "device will be reset\n"); 3940 set_bit(__I40E_PF_RESET_REQUESTED, pf->state); 3941 i40e_service_event_schedule(pf); 3942 } 3943 ena_mask &= ~icr0_remaining; 3944 } 3945 ret = IRQ_HANDLED; 3946 3947 enable_intr: 3948 /* re-enable interrupt causes */ 3949 wr32(hw, I40E_PFINT_ICR0_ENA, ena_mask); 3950 if (!test_bit(__I40E_DOWN, pf->state)) { 3951 i40e_service_event_schedule(pf); 3952 i40e_irq_dynamic_enable_icr0(pf); 3953 } 3954 3955 return ret; 3956 } 3957 3958 /** 3959 * i40e_clean_fdir_tx_irq - Reclaim resources after transmit completes 3960 * @tx_ring: tx ring to clean 3961 * @budget: how many cleans we're allowed 3962 * 3963 * Returns true if there's any budget left (e.g. the clean is finished) 3964 **/ 3965 static bool i40e_clean_fdir_tx_irq(struct i40e_ring *tx_ring, int budget) 3966 { 3967 struct i40e_vsi *vsi = tx_ring->vsi; 3968 u16 i = tx_ring->next_to_clean; 3969 struct i40e_tx_buffer *tx_buf; 3970 struct i40e_tx_desc *tx_desc; 3971 3972 tx_buf = &tx_ring->tx_bi[i]; 3973 tx_desc = I40E_TX_DESC(tx_ring, i); 3974 i -= tx_ring->count; 3975 3976 do { 3977 struct i40e_tx_desc *eop_desc = tx_buf->next_to_watch; 3978 3979 /* if next_to_watch is not set then there is no work pending */ 3980 if (!eop_desc) 3981 break; 3982 3983 /* prevent any other reads prior to eop_desc */ 3984 smp_rmb(); 3985 3986 /* if the descriptor isn't done, no work yet to do */ 3987 if (!(eop_desc->cmd_type_offset_bsz & 3988 cpu_to_le64(I40E_TX_DESC_DTYPE_DESC_DONE))) 3989 break; 3990 3991 /* clear next_to_watch to prevent false hangs */ 3992 tx_buf->next_to_watch = NULL; 3993 3994 tx_desc->buffer_addr = 0; 3995 tx_desc->cmd_type_offset_bsz = 0; 3996 /* move past filter desc */ 3997 tx_buf++; 3998 tx_desc++; 3999 i++; 4000 if (unlikely(!i)) { 4001 i -= tx_ring->count; 4002 tx_buf = tx_ring->tx_bi; 4003 tx_desc = I40E_TX_DESC(tx_ring, 0); 4004 } 4005 /* unmap skb header data */ 4006 dma_unmap_single(tx_ring->dev, 4007 dma_unmap_addr(tx_buf, dma), 4008 dma_unmap_len(tx_buf, len), 4009 DMA_TO_DEVICE); 4010 if (tx_buf->tx_flags & I40E_TX_FLAGS_FD_SB) 4011 kfree(tx_buf->raw_buf); 4012 4013 tx_buf->raw_buf = NULL; 4014 tx_buf->tx_flags = 0; 4015 tx_buf->next_to_watch = NULL; 4016 dma_unmap_len_set(tx_buf, len, 0); 4017 tx_desc->buffer_addr = 0; 4018 tx_desc->cmd_type_offset_bsz = 0; 4019 4020 /* move us past the eop_desc for start of next FD desc */ 4021 tx_buf++; 4022 tx_desc++; 4023 i++; 4024 if (unlikely(!i)) { 4025 i -= tx_ring->count; 4026 tx_buf = tx_ring->tx_bi; 4027 tx_desc = I40E_TX_DESC(tx_ring, 0); 4028 } 4029 4030 /* update budget accounting */ 4031 budget--; 4032 } while (likely(budget)); 4033 4034 i += tx_ring->count; 4035 tx_ring->next_to_clean = i; 4036 4037 if (vsi->back->flags & I40E_FLAG_MSIX_ENABLED) 4038 i40e_irq_dynamic_enable(vsi, tx_ring->q_vector->v_idx); 4039 4040 return budget > 0; 4041 } 4042 4043 /** 4044 * i40e_fdir_clean_ring - Interrupt Handler for FDIR SB ring 4045 * @irq: interrupt number 4046 * @data: pointer to a q_vector 4047 **/ 4048 static irqreturn_t i40e_fdir_clean_ring(int irq, void *data) 4049 { 4050 struct i40e_q_vector *q_vector = data; 4051 struct i40e_vsi *vsi; 4052 4053 if (!q_vector->tx.ring) 4054 return IRQ_HANDLED; 4055 4056 vsi = q_vector->tx.ring->vsi; 4057 i40e_clean_fdir_tx_irq(q_vector->tx.ring, vsi->work_limit); 4058 4059 return IRQ_HANDLED; 4060 } 4061 4062 /** 4063 * i40e_map_vector_to_qp - Assigns the queue pair to the vector 4064 * @vsi: the VSI being configured 4065 * @v_idx: vector index 4066 * @qp_idx: queue pair index 4067 **/ 4068 static void i40e_map_vector_to_qp(struct i40e_vsi *vsi, int v_idx, int qp_idx) 4069 { 4070 struct i40e_q_vector *q_vector = vsi->q_vectors[v_idx]; 4071 struct i40e_ring *tx_ring = vsi->tx_rings[qp_idx]; 4072 struct i40e_ring *rx_ring = vsi->rx_rings[qp_idx]; 4073 4074 tx_ring->q_vector = q_vector; 4075 tx_ring->next = q_vector->tx.ring; 4076 q_vector->tx.ring = tx_ring; 4077 q_vector->tx.count++; 4078 4079 /* Place XDP Tx ring in the same q_vector ring list as regular Tx */ 4080 if (i40e_enabled_xdp_vsi(vsi)) { 4081 struct i40e_ring *xdp_ring = vsi->xdp_rings[qp_idx]; 4082 4083 xdp_ring->q_vector = q_vector; 4084 xdp_ring->next = q_vector->tx.ring; 4085 q_vector->tx.ring = xdp_ring; 4086 q_vector->tx.count++; 4087 } 4088 4089 rx_ring->q_vector = q_vector; 4090 rx_ring->next = q_vector->rx.ring; 4091 q_vector->rx.ring = rx_ring; 4092 q_vector->rx.count++; 4093 } 4094 4095 /** 4096 * i40e_vsi_map_rings_to_vectors - Maps descriptor rings to vectors 4097 * @vsi: the VSI being configured 4098 * 4099 * This function maps descriptor rings to the queue-specific vectors 4100 * we were allotted through the MSI-X enabling code. Ideally, we'd have 4101 * one vector per queue pair, but on a constrained vector budget, we 4102 * group the queue pairs as "efficiently" as possible. 4103 **/ 4104 static void i40e_vsi_map_rings_to_vectors(struct i40e_vsi *vsi) 4105 { 4106 int qp_remaining = vsi->num_queue_pairs; 4107 int q_vectors = vsi->num_q_vectors; 4108 int num_ringpairs; 4109 int v_start = 0; 4110 int qp_idx = 0; 4111 4112 /* If we don't have enough vectors for a 1-to-1 mapping, we'll have to 4113 * group them so there are multiple queues per vector. 4114 * It is also important to go through all the vectors available to be 4115 * sure that if we don't use all the vectors, that the remaining vectors 4116 * are cleared. This is especially important when decreasing the 4117 * number of queues in use. 4118 */ 4119 for (; v_start < q_vectors; v_start++) { 4120 struct i40e_q_vector *q_vector = vsi->q_vectors[v_start]; 4121 4122 num_ringpairs = DIV_ROUND_UP(qp_remaining, q_vectors - v_start); 4123 4124 q_vector->num_ringpairs = num_ringpairs; 4125 4126 q_vector->rx.count = 0; 4127 q_vector->tx.count = 0; 4128 q_vector->rx.ring = NULL; 4129 q_vector->tx.ring = NULL; 4130 4131 while (num_ringpairs--) { 4132 i40e_map_vector_to_qp(vsi, v_start, qp_idx); 4133 qp_idx++; 4134 qp_remaining--; 4135 } 4136 } 4137 } 4138 4139 /** 4140 * i40e_vsi_request_irq - Request IRQ from the OS 4141 * @vsi: the VSI being configured 4142 * @basename: name for the vector 4143 **/ 4144 static int i40e_vsi_request_irq(struct i40e_vsi *vsi, char *basename) 4145 { 4146 struct i40e_pf *pf = vsi->back; 4147 int err; 4148 4149 if (pf->flags & I40E_FLAG_MSIX_ENABLED) 4150 err = i40e_vsi_request_irq_msix(vsi, basename); 4151 else if (pf->flags & I40E_FLAG_MSI_ENABLED) 4152 err = request_irq(pf->pdev->irq, i40e_intr, 0, 4153 pf->int_name, pf); 4154 else 4155 err = request_irq(pf->pdev->irq, i40e_intr, IRQF_SHARED, 4156 pf->int_name, pf); 4157 4158 if (err) 4159 dev_info(&pf->pdev->dev, "request_irq failed, Error %d\n", err); 4160 4161 return err; 4162 } 4163 4164 #ifdef CONFIG_NET_POLL_CONTROLLER 4165 /** 4166 * i40e_netpoll - A Polling 'interrupt' handler 4167 * @netdev: network interface device structure 4168 * 4169 * This is used by netconsole to send skbs without having to re-enable 4170 * interrupts. It's not called while the normal interrupt routine is executing. 4171 **/ 4172 static void i40e_netpoll(struct net_device *netdev) 4173 { 4174 struct i40e_netdev_priv *np = netdev_priv(netdev); 4175 struct i40e_vsi *vsi = np->vsi; 4176 struct i40e_pf *pf = vsi->back; 4177 int i; 4178 4179 /* if interface is down do nothing */ 4180 if (test_bit(__I40E_VSI_DOWN, vsi->state)) 4181 return; 4182 4183 if (pf->flags & I40E_FLAG_MSIX_ENABLED) { 4184 for (i = 0; i < vsi->num_q_vectors; i++) 4185 i40e_msix_clean_rings(0, vsi->q_vectors[i]); 4186 } else { 4187 i40e_intr(pf->pdev->irq, netdev); 4188 } 4189 } 4190 #endif 4191 4192 #define I40E_QTX_ENA_WAIT_COUNT 50 4193 4194 /** 4195 * i40e_pf_txq_wait - Wait for a PF's Tx queue to be enabled or disabled 4196 * @pf: the PF being configured 4197 * @pf_q: the PF queue 4198 * @enable: enable or disable state of the queue 4199 * 4200 * This routine will wait for the given Tx queue of the PF to reach the 4201 * enabled or disabled state. 4202 * Returns -ETIMEDOUT in case of failing to reach the requested state after 4203 * multiple retries; else will return 0 in case of success. 4204 **/ 4205 static int i40e_pf_txq_wait(struct i40e_pf *pf, int pf_q, bool enable) 4206 { 4207 int i; 4208 u32 tx_reg; 4209 4210 for (i = 0; i < I40E_QUEUE_WAIT_RETRY_LIMIT; i++) { 4211 tx_reg = rd32(&pf->hw, I40E_QTX_ENA(pf_q)); 4212 if (enable == !!(tx_reg & I40E_QTX_ENA_QENA_STAT_MASK)) 4213 break; 4214 4215 usleep_range(10, 20); 4216 } 4217 if (i >= I40E_QUEUE_WAIT_RETRY_LIMIT) 4218 return -ETIMEDOUT; 4219 4220 return 0; 4221 } 4222 4223 /** 4224 * i40e_control_tx_q - Start or stop a particular Tx queue 4225 * @pf: the PF structure 4226 * @pf_q: the PF queue to configure 4227 * @enable: start or stop the queue 4228 * 4229 * This function enables or disables a single queue. Note that any delay 4230 * required after the operation is expected to be handled by the caller of 4231 * this function. 4232 **/ 4233 static void i40e_control_tx_q(struct i40e_pf *pf, int pf_q, bool enable) 4234 { 4235 struct i40e_hw *hw = &pf->hw; 4236 u32 tx_reg; 4237 int i; 4238 4239 /* warn the TX unit of coming changes */ 4240 i40e_pre_tx_queue_cfg(&pf->hw, pf_q, enable); 4241 if (!enable) 4242 usleep_range(10, 20); 4243 4244 for (i = 0; i < I40E_QTX_ENA_WAIT_COUNT; i++) { 4245 tx_reg = rd32(hw, I40E_QTX_ENA(pf_q)); 4246 if (((tx_reg >> I40E_QTX_ENA_QENA_REQ_SHIFT) & 1) == 4247 ((tx_reg >> I40E_QTX_ENA_QENA_STAT_SHIFT) & 1)) 4248 break; 4249 usleep_range(1000, 2000); 4250 } 4251 4252 /* Skip if the queue is already in the requested state */ 4253 if (enable == !!(tx_reg & I40E_QTX_ENA_QENA_STAT_MASK)) 4254 return; 4255 4256 /* turn on/off the queue */ 4257 if (enable) { 4258 wr32(hw, I40E_QTX_HEAD(pf_q), 0); 4259 tx_reg |= I40E_QTX_ENA_QENA_REQ_MASK; 4260 } else { 4261 tx_reg &= ~I40E_QTX_ENA_QENA_REQ_MASK; 4262 } 4263 4264 wr32(hw, I40E_QTX_ENA(pf_q), tx_reg); 4265 } 4266 4267 /** 4268 * i40e_control_wait_tx_q - Start/stop Tx queue and wait for completion 4269 * @seid: VSI SEID 4270 * @pf: the PF structure 4271 * @pf_q: the PF queue to configure 4272 * @is_xdp: true if the queue is used for XDP 4273 * @enable: start or stop the queue 4274 **/ 4275 static int i40e_control_wait_tx_q(int seid, struct i40e_pf *pf, int pf_q, 4276 bool is_xdp, bool enable) 4277 { 4278 int ret; 4279 4280 i40e_control_tx_q(pf, pf_q, enable); 4281 4282 /* wait for the change to finish */ 4283 ret = i40e_pf_txq_wait(pf, pf_q, enable); 4284 if (ret) { 4285 dev_info(&pf->pdev->dev, 4286 "VSI seid %d %sTx ring %d %sable timeout\n", 4287 seid, (is_xdp ? "XDP " : ""), pf_q, 4288 (enable ? "en" : "dis")); 4289 } 4290 4291 return ret; 4292 } 4293 4294 /** 4295 * i40e_vsi_control_tx - Start or stop a VSI's rings 4296 * @vsi: the VSI being configured 4297 * @enable: start or stop the rings 4298 **/ 4299 static int i40e_vsi_control_tx(struct i40e_vsi *vsi, bool enable) 4300 { 4301 struct i40e_pf *pf = vsi->back; 4302 int i, pf_q, ret = 0; 4303 4304 pf_q = vsi->base_queue; 4305 for (i = 0; i < vsi->num_queue_pairs; i++, pf_q++) { 4306 ret = i40e_control_wait_tx_q(vsi->seid, pf, 4307 pf_q, 4308 false /*is xdp*/, enable); 4309 if (ret) 4310 break; 4311 4312 if (!i40e_enabled_xdp_vsi(vsi)) 4313 continue; 4314 4315 ret = i40e_control_wait_tx_q(vsi->seid, pf, 4316 pf_q + vsi->alloc_queue_pairs, 4317 true /*is xdp*/, enable); 4318 if (ret) 4319 break; 4320 } 4321 4322 return ret; 4323 } 4324 4325 /** 4326 * i40e_pf_rxq_wait - Wait for a PF's Rx queue to be enabled or disabled 4327 * @pf: the PF being configured 4328 * @pf_q: the PF queue 4329 * @enable: enable or disable state of the queue 4330 * 4331 * This routine will wait for the given Rx queue of the PF to reach the 4332 * enabled or disabled state. 4333 * Returns -ETIMEDOUT in case of failing to reach the requested state after 4334 * multiple retries; else will return 0 in case of success. 4335 **/ 4336 static int i40e_pf_rxq_wait(struct i40e_pf *pf, int pf_q, bool enable) 4337 { 4338 int i; 4339 u32 rx_reg; 4340 4341 for (i = 0; i < I40E_QUEUE_WAIT_RETRY_LIMIT; i++) { 4342 rx_reg = rd32(&pf->hw, I40E_QRX_ENA(pf_q)); 4343 if (enable == !!(rx_reg & I40E_QRX_ENA_QENA_STAT_MASK)) 4344 break; 4345 4346 usleep_range(10, 20); 4347 } 4348 if (i >= I40E_QUEUE_WAIT_RETRY_LIMIT) 4349 return -ETIMEDOUT; 4350 4351 return 0; 4352 } 4353 4354 /** 4355 * i40e_control_rx_q - Start or stop a particular Rx queue 4356 * @pf: the PF structure 4357 * @pf_q: the PF queue to configure 4358 * @enable: start or stop the queue 4359 * 4360 * This function enables or disables a single queue. Note that any delay 4361 * required after the operation is expected to be handled by the caller of 4362 * this function. 4363 **/ 4364 static void i40e_control_rx_q(struct i40e_pf *pf, int pf_q, bool enable) 4365 { 4366 struct i40e_hw *hw = &pf->hw; 4367 u32 rx_reg; 4368 int i; 4369 4370 for (i = 0; i < I40E_QTX_ENA_WAIT_COUNT; i++) { 4371 rx_reg = rd32(hw, I40E_QRX_ENA(pf_q)); 4372 if (((rx_reg >> I40E_QRX_ENA_QENA_REQ_SHIFT) & 1) == 4373 ((rx_reg >> I40E_QRX_ENA_QENA_STAT_SHIFT) & 1)) 4374 break; 4375 usleep_range(1000, 2000); 4376 } 4377 4378 /* Skip if the queue is already in the requested state */ 4379 if (enable == !!(rx_reg & I40E_QRX_ENA_QENA_STAT_MASK)) 4380 return; 4381 4382 /* turn on/off the queue */ 4383 if (enable) 4384 rx_reg |= I40E_QRX_ENA_QENA_REQ_MASK; 4385 else 4386 rx_reg &= ~I40E_QRX_ENA_QENA_REQ_MASK; 4387 4388 wr32(hw, I40E_QRX_ENA(pf_q), rx_reg); 4389 } 4390 4391 /** 4392 * i40e_vsi_control_rx - Start or stop a VSI's rings 4393 * @vsi: the VSI being configured 4394 * @enable: start or stop the rings 4395 **/ 4396 static int i40e_vsi_control_rx(struct i40e_vsi *vsi, bool enable) 4397 { 4398 struct i40e_pf *pf = vsi->back; 4399 int i, pf_q, ret = 0; 4400 4401 pf_q = vsi->base_queue; 4402 for (i = 0; i < vsi->num_queue_pairs; i++, pf_q++) { 4403 i40e_control_rx_q(pf, pf_q, enable); 4404 4405 /* wait for the change to finish */ 4406 ret = i40e_pf_rxq_wait(pf, pf_q, enable); 4407 if (ret) { 4408 dev_info(&pf->pdev->dev, 4409 "VSI seid %d Rx ring %d %sable timeout\n", 4410 vsi->seid, pf_q, (enable ? "en" : "dis")); 4411 break; 4412 } 4413 } 4414 4415 /* Due to HW errata, on Rx disable only, the register can indicate done 4416 * before it really is. Needs 50ms to be sure 4417 */ 4418 if (!enable) 4419 mdelay(50); 4420 4421 return ret; 4422 } 4423 4424 /** 4425 * i40e_vsi_start_rings - Start a VSI's rings 4426 * @vsi: the VSI being configured 4427 **/ 4428 int i40e_vsi_start_rings(struct i40e_vsi *vsi) 4429 { 4430 int ret = 0; 4431 4432 /* do rx first for enable and last for disable */ 4433 ret = i40e_vsi_control_rx(vsi, true); 4434 if (ret) 4435 return ret; 4436 ret = i40e_vsi_control_tx(vsi, true); 4437 4438 return ret; 4439 } 4440 4441 /** 4442 * i40e_vsi_stop_rings - Stop a VSI's rings 4443 * @vsi: the VSI being configured 4444 **/ 4445 void i40e_vsi_stop_rings(struct i40e_vsi *vsi) 4446 { 4447 /* When port TX is suspended, don't wait */ 4448 if (test_bit(__I40E_PORT_SUSPENDED, vsi->back->state)) 4449 return i40e_vsi_stop_rings_no_wait(vsi); 4450 4451 /* do rx first for enable and last for disable 4452 * Ignore return value, we need to shutdown whatever we can 4453 */ 4454 i40e_vsi_control_tx(vsi, false); 4455 i40e_vsi_control_rx(vsi, false); 4456 } 4457 4458 /** 4459 * i40e_vsi_stop_rings_no_wait - Stop a VSI's rings and do not delay 4460 * @vsi: the VSI being shutdown 4461 * 4462 * This function stops all the rings for a VSI but does not delay to verify 4463 * that rings have been disabled. It is expected that the caller is shutting 4464 * down multiple VSIs at once and will delay together for all the VSIs after 4465 * initiating the shutdown. This is particularly useful for shutting down lots 4466 * of VFs together. Otherwise, a large delay can be incurred while configuring 4467 * each VSI in serial. 4468 **/ 4469 void i40e_vsi_stop_rings_no_wait(struct i40e_vsi *vsi) 4470 { 4471 struct i40e_pf *pf = vsi->back; 4472 int i, pf_q; 4473 4474 pf_q = vsi->base_queue; 4475 for (i = 0; i < vsi->num_queue_pairs; i++, pf_q++) { 4476 i40e_control_tx_q(pf, pf_q, false); 4477 i40e_control_rx_q(pf, pf_q, false); 4478 } 4479 } 4480 4481 /** 4482 * i40e_vsi_free_irq - Free the irq association with the OS 4483 * @vsi: the VSI being configured 4484 **/ 4485 static void i40e_vsi_free_irq(struct i40e_vsi *vsi) 4486 { 4487 struct i40e_pf *pf = vsi->back; 4488 struct i40e_hw *hw = &pf->hw; 4489 int base = vsi->base_vector; 4490 u32 val, qp; 4491 int i; 4492 4493 if (pf->flags & I40E_FLAG_MSIX_ENABLED) { 4494 if (!vsi->q_vectors) 4495 return; 4496 4497 if (!vsi->irqs_ready) 4498 return; 4499 4500 vsi->irqs_ready = false; 4501 for (i = 0; i < vsi->num_q_vectors; i++) { 4502 int irq_num; 4503 u16 vector; 4504 4505 vector = i + base; 4506 irq_num = pf->msix_entries[vector].vector; 4507 4508 /* free only the irqs that were actually requested */ 4509 if (!vsi->q_vectors[i] || 4510 !vsi->q_vectors[i]->num_ringpairs) 4511 continue; 4512 4513 /* clear the affinity notifier in the IRQ descriptor */ 4514 irq_set_affinity_notifier(irq_num, NULL); 4515 /* remove our suggested affinity mask for this IRQ */ 4516 irq_set_affinity_hint(irq_num, NULL); 4517 synchronize_irq(irq_num); 4518 free_irq(irq_num, vsi->q_vectors[i]); 4519 4520 /* Tear down the interrupt queue link list 4521 * 4522 * We know that they come in pairs and always 4523 * the Rx first, then the Tx. To clear the 4524 * link list, stick the EOL value into the 4525 * next_q field of the registers. 4526 */ 4527 val = rd32(hw, I40E_PFINT_LNKLSTN(vector - 1)); 4528 qp = (val & I40E_PFINT_LNKLSTN_FIRSTQ_INDX_MASK) 4529 >> I40E_PFINT_LNKLSTN_FIRSTQ_INDX_SHIFT; 4530 val |= I40E_QUEUE_END_OF_LIST 4531 << I40E_PFINT_LNKLSTN_FIRSTQ_INDX_SHIFT; 4532 wr32(hw, I40E_PFINT_LNKLSTN(vector - 1), val); 4533 4534 while (qp != I40E_QUEUE_END_OF_LIST) { 4535 u32 next; 4536 4537 val = rd32(hw, I40E_QINT_RQCTL(qp)); 4538 4539 val &= ~(I40E_QINT_RQCTL_MSIX_INDX_MASK | 4540 I40E_QINT_RQCTL_MSIX0_INDX_MASK | 4541 I40E_QINT_RQCTL_CAUSE_ENA_MASK | 4542 I40E_QINT_RQCTL_INTEVENT_MASK); 4543 4544 val |= (I40E_QINT_RQCTL_ITR_INDX_MASK | 4545 I40E_QINT_RQCTL_NEXTQ_INDX_MASK); 4546 4547 wr32(hw, I40E_QINT_RQCTL(qp), val); 4548 4549 val = rd32(hw, I40E_QINT_TQCTL(qp)); 4550 4551 next = (val & I40E_QINT_TQCTL_NEXTQ_INDX_MASK) 4552 >> I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT; 4553 4554 val &= ~(I40E_QINT_TQCTL_MSIX_INDX_MASK | 4555 I40E_QINT_TQCTL_MSIX0_INDX_MASK | 4556 I40E_QINT_TQCTL_CAUSE_ENA_MASK | 4557 I40E_QINT_TQCTL_INTEVENT_MASK); 4558 4559 val |= (I40E_QINT_TQCTL_ITR_INDX_MASK | 4560 I40E_QINT_TQCTL_NEXTQ_INDX_MASK); 4561 4562 wr32(hw, I40E_QINT_TQCTL(qp), val); 4563 qp = next; 4564 } 4565 } 4566 } else { 4567 free_irq(pf->pdev->irq, pf); 4568 4569 val = rd32(hw, I40E_PFINT_LNKLST0); 4570 qp = (val & I40E_PFINT_LNKLSTN_FIRSTQ_INDX_MASK) 4571 >> I40E_PFINT_LNKLSTN_FIRSTQ_INDX_SHIFT; 4572 val |= I40E_QUEUE_END_OF_LIST 4573 << I40E_PFINT_LNKLST0_FIRSTQ_INDX_SHIFT; 4574 wr32(hw, I40E_PFINT_LNKLST0, val); 4575 4576 val = rd32(hw, I40E_QINT_RQCTL(qp)); 4577 val &= ~(I40E_QINT_RQCTL_MSIX_INDX_MASK | 4578 I40E_QINT_RQCTL_MSIX0_INDX_MASK | 4579 I40E_QINT_RQCTL_CAUSE_ENA_MASK | 4580 I40E_QINT_RQCTL_INTEVENT_MASK); 4581 4582 val |= (I40E_QINT_RQCTL_ITR_INDX_MASK | 4583 I40E_QINT_RQCTL_NEXTQ_INDX_MASK); 4584 4585 wr32(hw, I40E_QINT_RQCTL(qp), val); 4586 4587 val = rd32(hw, I40E_QINT_TQCTL(qp)); 4588 4589 val &= ~(I40E_QINT_TQCTL_MSIX_INDX_MASK | 4590 I40E_QINT_TQCTL_MSIX0_INDX_MASK | 4591 I40E_QINT_TQCTL_CAUSE_ENA_MASK | 4592 I40E_QINT_TQCTL_INTEVENT_MASK); 4593 4594 val |= (I40E_QINT_TQCTL_ITR_INDX_MASK | 4595 I40E_QINT_TQCTL_NEXTQ_INDX_MASK); 4596 4597 wr32(hw, I40E_QINT_TQCTL(qp), val); 4598 } 4599 } 4600 4601 /** 4602 * i40e_free_q_vector - Free memory allocated for specific interrupt vector 4603 * @vsi: the VSI being configured 4604 * @v_idx: Index of vector to be freed 4605 * 4606 * This function frees the memory allocated to the q_vector. In addition if 4607 * NAPI is enabled it will delete any references to the NAPI struct prior 4608 * to freeing the q_vector. 4609 **/ 4610 static void i40e_free_q_vector(struct i40e_vsi *vsi, int v_idx) 4611 { 4612 struct i40e_q_vector *q_vector = vsi->q_vectors[v_idx]; 4613 struct i40e_ring *ring; 4614 4615 if (!q_vector) 4616 return; 4617 4618 /* disassociate q_vector from rings */ 4619 i40e_for_each_ring(ring, q_vector->tx) 4620 ring->q_vector = NULL; 4621 4622 i40e_for_each_ring(ring, q_vector->rx) 4623 ring->q_vector = NULL; 4624 4625 /* only VSI w/ an associated netdev is set up w/ NAPI */ 4626 if (vsi->netdev) 4627 netif_napi_del(&q_vector->napi); 4628 4629 vsi->q_vectors[v_idx] = NULL; 4630 4631 kfree_rcu(q_vector, rcu); 4632 } 4633 4634 /** 4635 * i40e_vsi_free_q_vectors - Free memory allocated for interrupt vectors 4636 * @vsi: the VSI being un-configured 4637 * 4638 * This frees the memory allocated to the q_vectors and 4639 * deletes references to the NAPI struct. 4640 **/ 4641 static void i40e_vsi_free_q_vectors(struct i40e_vsi *vsi) 4642 { 4643 int v_idx; 4644 4645 for (v_idx = 0; v_idx < vsi->num_q_vectors; v_idx++) 4646 i40e_free_q_vector(vsi, v_idx); 4647 } 4648 4649 /** 4650 * i40e_reset_interrupt_capability - Disable interrupt setup in OS 4651 * @pf: board private structure 4652 **/ 4653 static void i40e_reset_interrupt_capability(struct i40e_pf *pf) 4654 { 4655 /* If we're in Legacy mode, the interrupt was cleaned in vsi_close */ 4656 if (pf->flags & I40E_FLAG_MSIX_ENABLED) { 4657 pci_disable_msix(pf->pdev); 4658 kfree(pf->msix_entries); 4659 pf->msix_entries = NULL; 4660 kfree(pf->irq_pile); 4661 pf->irq_pile = NULL; 4662 } else if (pf->flags & I40E_FLAG_MSI_ENABLED) { 4663 pci_disable_msi(pf->pdev); 4664 } 4665 pf->flags &= ~(I40E_FLAG_MSIX_ENABLED | I40E_FLAG_MSI_ENABLED); 4666 } 4667 4668 /** 4669 * i40e_clear_interrupt_scheme - Clear the current interrupt scheme settings 4670 * @pf: board private structure 4671 * 4672 * We go through and clear interrupt specific resources and reset the structure 4673 * to pre-load conditions 4674 **/ 4675 static void i40e_clear_interrupt_scheme(struct i40e_pf *pf) 4676 { 4677 int i; 4678 4679 i40e_free_misc_vector(pf); 4680 4681 i40e_put_lump(pf->irq_pile, pf->iwarp_base_vector, 4682 I40E_IWARP_IRQ_PILE_ID); 4683 4684 i40e_put_lump(pf->irq_pile, 0, I40E_PILE_VALID_BIT-1); 4685 for (i = 0; i < pf->num_alloc_vsi; i++) 4686 if (pf->vsi[i]) 4687 i40e_vsi_free_q_vectors(pf->vsi[i]); 4688 i40e_reset_interrupt_capability(pf); 4689 } 4690 4691 /** 4692 * i40e_napi_enable_all - Enable NAPI for all q_vectors in the VSI 4693 * @vsi: the VSI being configured 4694 **/ 4695 static void i40e_napi_enable_all(struct i40e_vsi *vsi) 4696 { 4697 int q_idx; 4698 4699 if (!vsi->netdev) 4700 return; 4701 4702 for (q_idx = 0; q_idx < vsi->num_q_vectors; q_idx++) { 4703 struct i40e_q_vector *q_vector = vsi->q_vectors[q_idx]; 4704 4705 if (q_vector->rx.ring || q_vector->tx.ring) 4706 napi_enable(&q_vector->napi); 4707 } 4708 } 4709 4710 /** 4711 * i40e_napi_disable_all - Disable NAPI for all q_vectors in the VSI 4712 * @vsi: the VSI being configured 4713 **/ 4714 static void i40e_napi_disable_all(struct i40e_vsi *vsi) 4715 { 4716 int q_idx; 4717 4718 if (!vsi->netdev) 4719 return; 4720 4721 for (q_idx = 0; q_idx < vsi->num_q_vectors; q_idx++) { 4722 struct i40e_q_vector *q_vector = vsi->q_vectors[q_idx]; 4723 4724 if (q_vector->rx.ring || q_vector->tx.ring) 4725 napi_disable(&q_vector->napi); 4726 } 4727 } 4728 4729 /** 4730 * i40e_vsi_close - Shut down a VSI 4731 * @vsi: the vsi to be quelled 4732 **/ 4733 static void i40e_vsi_close(struct i40e_vsi *vsi) 4734 { 4735 struct i40e_pf *pf = vsi->back; 4736 if (!test_and_set_bit(__I40E_VSI_DOWN, vsi->state)) 4737 i40e_down(vsi); 4738 i40e_vsi_free_irq(vsi); 4739 i40e_vsi_free_tx_resources(vsi); 4740 i40e_vsi_free_rx_resources(vsi); 4741 vsi->current_netdev_flags = 0; 4742 pf->flags |= I40E_FLAG_SERVICE_CLIENT_REQUESTED; 4743 if (test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state)) 4744 pf->flags |= I40E_FLAG_CLIENT_RESET; 4745 } 4746 4747 /** 4748 * i40e_quiesce_vsi - Pause a given VSI 4749 * @vsi: the VSI being paused 4750 **/ 4751 static void i40e_quiesce_vsi(struct i40e_vsi *vsi) 4752 { 4753 if (test_bit(__I40E_VSI_DOWN, vsi->state)) 4754 return; 4755 4756 set_bit(__I40E_VSI_NEEDS_RESTART, vsi->state); 4757 if (vsi->netdev && netif_running(vsi->netdev)) 4758 vsi->netdev->netdev_ops->ndo_stop(vsi->netdev); 4759 else 4760 i40e_vsi_close(vsi); 4761 } 4762 4763 /** 4764 * i40e_unquiesce_vsi - Resume a given VSI 4765 * @vsi: the VSI being resumed 4766 **/ 4767 static void i40e_unquiesce_vsi(struct i40e_vsi *vsi) 4768 { 4769 if (!test_and_clear_bit(__I40E_VSI_NEEDS_RESTART, vsi->state)) 4770 return; 4771 4772 if (vsi->netdev && netif_running(vsi->netdev)) 4773 vsi->netdev->netdev_ops->ndo_open(vsi->netdev); 4774 else 4775 i40e_vsi_open(vsi); /* this clears the DOWN bit */ 4776 } 4777 4778 /** 4779 * i40e_pf_quiesce_all_vsi - Pause all VSIs on a PF 4780 * @pf: the PF 4781 **/ 4782 static void i40e_pf_quiesce_all_vsi(struct i40e_pf *pf) 4783 { 4784 int v; 4785 4786 for (v = 0; v < pf->num_alloc_vsi; v++) { 4787 if (pf->vsi[v]) 4788 i40e_quiesce_vsi(pf->vsi[v]); 4789 } 4790 } 4791 4792 /** 4793 * i40e_pf_unquiesce_all_vsi - Resume all VSIs on a PF 4794 * @pf: the PF 4795 **/ 4796 static void i40e_pf_unquiesce_all_vsi(struct i40e_pf *pf) 4797 { 4798 int v; 4799 4800 for (v = 0; v < pf->num_alloc_vsi; v++) { 4801 if (pf->vsi[v]) 4802 i40e_unquiesce_vsi(pf->vsi[v]); 4803 } 4804 } 4805 4806 /** 4807 * i40e_vsi_wait_queues_disabled - Wait for VSI's queues to be disabled 4808 * @vsi: the VSI being configured 4809 * 4810 * Wait until all queues on a given VSI have been disabled. 4811 **/ 4812 int i40e_vsi_wait_queues_disabled(struct i40e_vsi *vsi) 4813 { 4814 struct i40e_pf *pf = vsi->back; 4815 int i, pf_q, ret; 4816 4817 pf_q = vsi->base_queue; 4818 for (i = 0; i < vsi->num_queue_pairs; i++, pf_q++) { 4819 /* Check and wait for the Tx queue */ 4820 ret = i40e_pf_txq_wait(pf, pf_q, false); 4821 if (ret) { 4822 dev_info(&pf->pdev->dev, 4823 "VSI seid %d Tx ring %d disable timeout\n", 4824 vsi->seid, pf_q); 4825 return ret; 4826 } 4827 4828 if (!i40e_enabled_xdp_vsi(vsi)) 4829 goto wait_rx; 4830 4831 /* Check and wait for the XDP Tx queue */ 4832 ret = i40e_pf_txq_wait(pf, pf_q + vsi->alloc_queue_pairs, 4833 false); 4834 if (ret) { 4835 dev_info(&pf->pdev->dev, 4836 "VSI seid %d XDP Tx ring %d disable timeout\n", 4837 vsi->seid, pf_q); 4838 return ret; 4839 } 4840 wait_rx: 4841 /* Check and wait for the Rx queue */ 4842 ret = i40e_pf_rxq_wait(pf, pf_q, false); 4843 if (ret) { 4844 dev_info(&pf->pdev->dev, 4845 "VSI seid %d Rx ring %d disable timeout\n", 4846 vsi->seid, pf_q); 4847 return ret; 4848 } 4849 } 4850 4851 return 0; 4852 } 4853 4854 #ifdef CONFIG_I40E_DCB 4855 /** 4856 * i40e_pf_wait_queues_disabled - Wait for all queues of PF VSIs to be disabled 4857 * @pf: the PF 4858 * 4859 * This function waits for the queues to be in disabled state for all the 4860 * VSIs that are managed by this PF. 4861 **/ 4862 static int i40e_pf_wait_queues_disabled(struct i40e_pf *pf) 4863 { 4864 int v, ret = 0; 4865 4866 for (v = 0; v < pf->hw.func_caps.num_vsis; v++) { 4867 if (pf->vsi[v]) { 4868 ret = i40e_vsi_wait_queues_disabled(pf->vsi[v]); 4869 if (ret) 4870 break; 4871 } 4872 } 4873 4874 return ret; 4875 } 4876 4877 #endif 4878 4879 /** 4880 * i40e_detect_recover_hung_queue - Function to detect and recover hung_queue 4881 * @q_idx: TX queue number 4882 * @vsi: Pointer to VSI struct 4883 * 4884 * This function checks specified queue for given VSI. Detects hung condition. 4885 * We proactively detect hung TX queues by checking if interrupts are disabled 4886 * but there are pending descriptors. If it appears hung, attempt to recover 4887 * by triggering a SW interrupt. 4888 **/ 4889 static void i40e_detect_recover_hung_queue(int q_idx, struct i40e_vsi *vsi) 4890 { 4891 struct i40e_ring *tx_ring = NULL; 4892 struct i40e_pf *pf; 4893 u32 val, tx_pending; 4894 int i; 4895 4896 pf = vsi->back; 4897 4898 /* now that we have an index, find the tx_ring struct */ 4899 for (i = 0; i < vsi->num_queue_pairs; i++) { 4900 if (vsi->tx_rings[i] && vsi->tx_rings[i]->desc) { 4901 if (q_idx == vsi->tx_rings[i]->queue_index) { 4902 tx_ring = vsi->tx_rings[i]; 4903 break; 4904 } 4905 } 4906 } 4907 4908 if (!tx_ring) 4909 return; 4910 4911 /* Read interrupt register */ 4912 if (pf->flags & I40E_FLAG_MSIX_ENABLED) 4913 val = rd32(&pf->hw, 4914 I40E_PFINT_DYN_CTLN(tx_ring->q_vector->v_idx + 4915 tx_ring->vsi->base_vector - 1)); 4916 else 4917 val = rd32(&pf->hw, I40E_PFINT_DYN_CTL0); 4918 4919 tx_pending = i40e_get_tx_pending(tx_ring); 4920 4921 /* Interrupts are disabled and TX pending is non-zero, 4922 * trigger the SW interrupt (don't wait). Worst case 4923 * there will be one extra interrupt which may result 4924 * into not cleaning any queues because queues are cleaned. 4925 */ 4926 if (tx_pending && (!(val & I40E_PFINT_DYN_CTLN_INTENA_MASK))) 4927 i40e_force_wb(vsi, tx_ring->q_vector); 4928 } 4929 4930 /** 4931 * i40e_detect_recover_hung - Function to detect and recover hung_queues 4932 * @pf: pointer to PF struct 4933 * 4934 * LAN VSI has netdev and netdev has TX queues. This function is to check 4935 * each of those TX queues if they are hung, trigger recovery by issuing 4936 * SW interrupt. 4937 **/ 4938 static void i40e_detect_recover_hung(struct i40e_pf *pf) 4939 { 4940 struct net_device *netdev; 4941 struct i40e_vsi *vsi; 4942 unsigned int i; 4943 4944 /* Only for LAN VSI */ 4945 vsi = pf->vsi[pf->lan_vsi]; 4946 4947 if (!vsi) 4948 return; 4949 4950 /* Make sure, VSI state is not DOWN/RECOVERY_PENDING */ 4951 if (test_bit(__I40E_VSI_DOWN, vsi->back->state) || 4952 test_bit(__I40E_RESET_RECOVERY_PENDING, vsi->back->state)) 4953 return; 4954 4955 /* Make sure type is MAIN VSI */ 4956 if (vsi->type != I40E_VSI_MAIN) 4957 return; 4958 4959 netdev = vsi->netdev; 4960 if (!netdev) 4961 return; 4962 4963 /* Bail out if netif_carrier is not OK */ 4964 if (!netif_carrier_ok(netdev)) 4965 return; 4966 4967 /* Go thru' TX queues for netdev */ 4968 for (i = 0; i < netdev->num_tx_queues; i++) { 4969 struct netdev_queue *q; 4970 4971 q = netdev_get_tx_queue(netdev, i); 4972 if (q) 4973 i40e_detect_recover_hung_queue(i, vsi); 4974 } 4975 } 4976 4977 /** 4978 * i40e_get_iscsi_tc_map - Return TC map for iSCSI APP 4979 * @pf: pointer to PF 4980 * 4981 * Get TC map for ISCSI PF type that will include iSCSI TC 4982 * and LAN TC. 4983 **/ 4984 static u8 i40e_get_iscsi_tc_map(struct i40e_pf *pf) 4985 { 4986 struct i40e_dcb_app_priority_table app; 4987 struct i40e_hw *hw = &pf->hw; 4988 u8 enabled_tc = 1; /* TC0 is always enabled */ 4989 u8 tc, i; 4990 /* Get the iSCSI APP TLV */ 4991 struct i40e_dcbx_config *dcbcfg = &hw->local_dcbx_config; 4992 4993 for (i = 0; i < dcbcfg->numapps; i++) { 4994 app = dcbcfg->app[i]; 4995 if (app.selector == I40E_APP_SEL_TCPIP && 4996 app.protocolid == I40E_APP_PROTOID_ISCSI) { 4997 tc = dcbcfg->etscfg.prioritytable[app.priority]; 4998 enabled_tc |= BIT(tc); 4999 break; 5000 } 5001 } 5002 5003 return enabled_tc; 5004 } 5005 5006 /** 5007 * i40e_dcb_get_num_tc - Get the number of TCs from DCBx config 5008 * @dcbcfg: the corresponding DCBx configuration structure 5009 * 5010 * Return the number of TCs from given DCBx configuration 5011 **/ 5012 static u8 i40e_dcb_get_num_tc(struct i40e_dcbx_config *dcbcfg) 5013 { 5014 int i, tc_unused = 0; 5015 u8 num_tc = 0; 5016 u8 ret = 0; 5017 5018 /* Scan the ETS Config Priority Table to find 5019 * traffic class enabled for a given priority 5020 * and create a bitmask of enabled TCs 5021 */ 5022 for (i = 0; i < I40E_MAX_USER_PRIORITY; i++) 5023 num_tc |= BIT(dcbcfg->etscfg.prioritytable[i]); 5024 5025 /* Now scan the bitmask to check for 5026 * contiguous TCs starting with TC0 5027 */ 5028 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 5029 if (num_tc & BIT(i)) { 5030 if (!tc_unused) { 5031 ret++; 5032 } else { 5033 pr_err("Non-contiguous TC - Disabling DCB\n"); 5034 return 1; 5035 } 5036 } else { 5037 tc_unused = 1; 5038 } 5039 } 5040 5041 /* There is always at least TC0 */ 5042 if (!ret) 5043 ret = 1; 5044 5045 return ret; 5046 } 5047 5048 /** 5049 * i40e_dcb_get_enabled_tc - Get enabled traffic classes 5050 * @dcbcfg: the corresponding DCBx configuration structure 5051 * 5052 * Query the current DCB configuration and return the number of 5053 * traffic classes enabled from the given DCBX config 5054 **/ 5055 static u8 i40e_dcb_get_enabled_tc(struct i40e_dcbx_config *dcbcfg) 5056 { 5057 u8 num_tc = i40e_dcb_get_num_tc(dcbcfg); 5058 u8 enabled_tc = 1; 5059 u8 i; 5060 5061 for (i = 0; i < num_tc; i++) 5062 enabled_tc |= BIT(i); 5063 5064 return enabled_tc; 5065 } 5066 5067 /** 5068 * i40e_mqprio_get_enabled_tc - Get enabled traffic classes 5069 * @pf: PF being queried 5070 * 5071 * Query the current MQPRIO configuration and return the number of 5072 * traffic classes enabled. 5073 **/ 5074 static u8 i40e_mqprio_get_enabled_tc(struct i40e_pf *pf) 5075 { 5076 struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi]; 5077 u8 num_tc = vsi->mqprio_qopt.qopt.num_tc; 5078 u8 enabled_tc = 1, i; 5079 5080 for (i = 1; i < num_tc; i++) 5081 enabled_tc |= BIT(i); 5082 return enabled_tc; 5083 } 5084 5085 /** 5086 * i40e_pf_get_num_tc - Get enabled traffic classes for PF 5087 * @pf: PF being queried 5088 * 5089 * Return number of traffic classes enabled for the given PF 5090 **/ 5091 static u8 i40e_pf_get_num_tc(struct i40e_pf *pf) 5092 { 5093 struct i40e_hw *hw = &pf->hw; 5094 u8 i, enabled_tc = 1; 5095 u8 num_tc = 0; 5096 struct i40e_dcbx_config *dcbcfg = &hw->local_dcbx_config; 5097 5098 if (pf->flags & I40E_FLAG_TC_MQPRIO) 5099 return pf->vsi[pf->lan_vsi]->mqprio_qopt.qopt.num_tc; 5100 5101 /* If neither MQPRIO nor DCB is enabled, then always use single TC */ 5102 if (!(pf->flags & I40E_FLAG_DCB_ENABLED)) 5103 return 1; 5104 5105 /* SFP mode will be enabled for all TCs on port */ 5106 if (!(pf->flags & I40E_FLAG_MFP_ENABLED)) 5107 return i40e_dcb_get_num_tc(dcbcfg); 5108 5109 /* MFP mode return count of enabled TCs for this PF */ 5110 if (pf->hw.func_caps.iscsi) 5111 enabled_tc = i40e_get_iscsi_tc_map(pf); 5112 else 5113 return 1; /* Only TC0 */ 5114 5115 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 5116 if (enabled_tc & BIT(i)) 5117 num_tc++; 5118 } 5119 return num_tc; 5120 } 5121 5122 /** 5123 * i40e_pf_get_pf_tc_map - Get bitmap for enabled traffic classes 5124 * @pf: PF being queried 5125 * 5126 * Return a bitmap for enabled traffic classes for this PF. 5127 **/ 5128 static u8 i40e_pf_get_tc_map(struct i40e_pf *pf) 5129 { 5130 if (pf->flags & I40E_FLAG_TC_MQPRIO) 5131 return i40e_mqprio_get_enabled_tc(pf); 5132 5133 /* If neither MQPRIO nor DCB is enabled for this PF then just return 5134 * default TC 5135 */ 5136 if (!(pf->flags & I40E_FLAG_DCB_ENABLED)) 5137 return I40E_DEFAULT_TRAFFIC_CLASS; 5138 5139 /* SFP mode we want PF to be enabled for all TCs */ 5140 if (!(pf->flags & I40E_FLAG_MFP_ENABLED)) 5141 return i40e_dcb_get_enabled_tc(&pf->hw.local_dcbx_config); 5142 5143 /* MFP enabled and iSCSI PF type */ 5144 if (pf->hw.func_caps.iscsi) 5145 return i40e_get_iscsi_tc_map(pf); 5146 else 5147 return I40E_DEFAULT_TRAFFIC_CLASS; 5148 } 5149 5150 /** 5151 * i40e_vsi_get_bw_info - Query VSI BW Information 5152 * @vsi: the VSI being queried 5153 * 5154 * Returns 0 on success, negative value on failure 5155 **/ 5156 static int i40e_vsi_get_bw_info(struct i40e_vsi *vsi) 5157 { 5158 struct i40e_aqc_query_vsi_ets_sla_config_resp bw_ets_config = {0}; 5159 struct i40e_aqc_query_vsi_bw_config_resp bw_config = {0}; 5160 struct i40e_pf *pf = vsi->back; 5161 struct i40e_hw *hw = &pf->hw; 5162 i40e_status ret; 5163 u32 tc_bw_max; 5164 int i; 5165 5166 /* Get the VSI level BW configuration */ 5167 ret = i40e_aq_query_vsi_bw_config(hw, vsi->seid, &bw_config, NULL); 5168 if (ret) { 5169 dev_info(&pf->pdev->dev, 5170 "couldn't get PF vsi bw config, err %s aq_err %s\n", 5171 i40e_stat_str(&pf->hw, ret), 5172 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 5173 return -EINVAL; 5174 } 5175 5176 /* Get the VSI level BW configuration per TC */ 5177 ret = i40e_aq_query_vsi_ets_sla_config(hw, vsi->seid, &bw_ets_config, 5178 NULL); 5179 if (ret) { 5180 dev_info(&pf->pdev->dev, 5181 "couldn't get PF vsi ets bw config, err %s aq_err %s\n", 5182 i40e_stat_str(&pf->hw, ret), 5183 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 5184 return -EINVAL; 5185 } 5186 5187 if (bw_config.tc_valid_bits != bw_ets_config.tc_valid_bits) { 5188 dev_info(&pf->pdev->dev, 5189 "Enabled TCs mismatch from querying VSI BW info 0x%08x 0x%08x\n", 5190 bw_config.tc_valid_bits, 5191 bw_ets_config.tc_valid_bits); 5192 /* Still continuing */ 5193 } 5194 5195 vsi->bw_limit = le16_to_cpu(bw_config.port_bw_limit); 5196 vsi->bw_max_quanta = bw_config.max_bw; 5197 tc_bw_max = le16_to_cpu(bw_ets_config.tc_bw_max[0]) | 5198 (le16_to_cpu(bw_ets_config.tc_bw_max[1]) << 16); 5199 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 5200 vsi->bw_ets_share_credits[i] = bw_ets_config.share_credits[i]; 5201 vsi->bw_ets_limit_credits[i] = 5202 le16_to_cpu(bw_ets_config.credits[i]); 5203 /* 3 bits out of 4 for each TC */ 5204 vsi->bw_ets_max_quanta[i] = (u8)((tc_bw_max >> (i*4)) & 0x7); 5205 } 5206 5207 return 0; 5208 } 5209 5210 /** 5211 * i40e_vsi_configure_bw_alloc - Configure VSI BW allocation per TC 5212 * @vsi: the VSI being configured 5213 * @enabled_tc: TC bitmap 5214 * @bw_credits: BW shared credits per TC 5215 * 5216 * Returns 0 on success, negative value on failure 5217 **/ 5218 static int i40e_vsi_configure_bw_alloc(struct i40e_vsi *vsi, u8 enabled_tc, 5219 u8 *bw_share) 5220 { 5221 struct i40e_aqc_configure_vsi_tc_bw_data bw_data; 5222 i40e_status ret; 5223 int i; 5224 5225 if (vsi->back->flags & I40E_FLAG_TC_MQPRIO) 5226 return 0; 5227 if (!vsi->mqprio_qopt.qopt.hw) { 5228 ret = i40e_set_bw_limit(vsi, vsi->seid, 0); 5229 if (ret) 5230 dev_info(&vsi->back->pdev->dev, 5231 "Failed to reset tx rate for vsi->seid %u\n", 5232 vsi->seid); 5233 return ret; 5234 } 5235 bw_data.tc_valid_bits = enabled_tc; 5236 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) 5237 bw_data.tc_bw_credits[i] = bw_share[i]; 5238 5239 ret = i40e_aq_config_vsi_tc_bw(&vsi->back->hw, vsi->seid, &bw_data, 5240 NULL); 5241 if (ret) { 5242 dev_info(&vsi->back->pdev->dev, 5243 "AQ command Config VSI BW allocation per TC failed = %d\n", 5244 vsi->back->hw.aq.asq_last_status); 5245 return -EINVAL; 5246 } 5247 5248 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) 5249 vsi->info.qs_handle[i] = bw_data.qs_handles[i]; 5250 5251 return 0; 5252 } 5253 5254 /** 5255 * i40e_vsi_config_netdev_tc - Setup the netdev TC configuration 5256 * @vsi: the VSI being configured 5257 * @enabled_tc: TC map to be enabled 5258 * 5259 **/ 5260 static void i40e_vsi_config_netdev_tc(struct i40e_vsi *vsi, u8 enabled_tc) 5261 { 5262 struct net_device *netdev = vsi->netdev; 5263 struct i40e_pf *pf = vsi->back; 5264 struct i40e_hw *hw = &pf->hw; 5265 u8 netdev_tc = 0; 5266 int i; 5267 struct i40e_dcbx_config *dcbcfg = &hw->local_dcbx_config; 5268 5269 if (!netdev) 5270 return; 5271 5272 if (!enabled_tc) { 5273 netdev_reset_tc(netdev); 5274 return; 5275 } 5276 5277 /* Set up actual enabled TCs on the VSI */ 5278 if (netdev_set_num_tc(netdev, vsi->tc_config.numtc)) 5279 return; 5280 5281 /* set per TC queues for the VSI */ 5282 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 5283 /* Only set TC queues for enabled tcs 5284 * 5285 * e.g. For a VSI that has TC0 and TC3 enabled the 5286 * enabled_tc bitmap would be 0x00001001; the driver 5287 * will set the numtc for netdev as 2 that will be 5288 * referenced by the netdev layer as TC 0 and 1. 5289 */ 5290 if (vsi->tc_config.enabled_tc & BIT(i)) 5291 netdev_set_tc_queue(netdev, 5292 vsi->tc_config.tc_info[i].netdev_tc, 5293 vsi->tc_config.tc_info[i].qcount, 5294 vsi->tc_config.tc_info[i].qoffset); 5295 } 5296 5297 if (pf->flags & I40E_FLAG_TC_MQPRIO) 5298 return; 5299 5300 /* Assign UP2TC map for the VSI */ 5301 for (i = 0; i < I40E_MAX_USER_PRIORITY; i++) { 5302 /* Get the actual TC# for the UP */ 5303 u8 ets_tc = dcbcfg->etscfg.prioritytable[i]; 5304 /* Get the mapped netdev TC# for the UP */ 5305 netdev_tc = vsi->tc_config.tc_info[ets_tc].netdev_tc; 5306 netdev_set_prio_tc_map(netdev, i, netdev_tc); 5307 } 5308 } 5309 5310 /** 5311 * i40e_vsi_update_queue_map - Update our copy of VSi info with new queue map 5312 * @vsi: the VSI being configured 5313 * @ctxt: the ctxt buffer returned from AQ VSI update param command 5314 **/ 5315 static void i40e_vsi_update_queue_map(struct i40e_vsi *vsi, 5316 struct i40e_vsi_context *ctxt) 5317 { 5318 /* copy just the sections touched not the entire info 5319 * since not all sections are valid as returned by 5320 * update vsi params 5321 */ 5322 vsi->info.mapping_flags = ctxt->info.mapping_flags; 5323 memcpy(&vsi->info.queue_mapping, 5324 &ctxt->info.queue_mapping, sizeof(vsi->info.queue_mapping)); 5325 memcpy(&vsi->info.tc_mapping, ctxt->info.tc_mapping, 5326 sizeof(vsi->info.tc_mapping)); 5327 } 5328 5329 /** 5330 * i40e_vsi_config_tc - Configure VSI Tx Scheduler for given TC map 5331 * @vsi: VSI to be configured 5332 * @enabled_tc: TC bitmap 5333 * 5334 * This configures a particular VSI for TCs that are mapped to the 5335 * given TC bitmap. It uses default bandwidth share for TCs across 5336 * VSIs to configure TC for a particular VSI. 5337 * 5338 * NOTE: 5339 * It is expected that the VSI queues have been quisced before calling 5340 * this function. 5341 **/ 5342 static int i40e_vsi_config_tc(struct i40e_vsi *vsi, u8 enabled_tc) 5343 { 5344 u8 bw_share[I40E_MAX_TRAFFIC_CLASS] = {0}; 5345 struct i40e_vsi_context ctxt; 5346 int ret = 0; 5347 int i; 5348 5349 /* Check if enabled_tc is same as existing or new TCs */ 5350 if (vsi->tc_config.enabled_tc == enabled_tc && 5351 vsi->mqprio_qopt.mode != TC_MQPRIO_MODE_CHANNEL) 5352 return ret; 5353 5354 /* Enable ETS TCs with equal BW Share for now across all VSIs */ 5355 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 5356 if (enabled_tc & BIT(i)) 5357 bw_share[i] = 1; 5358 } 5359 5360 ret = i40e_vsi_configure_bw_alloc(vsi, enabled_tc, bw_share); 5361 if (ret) { 5362 dev_info(&vsi->back->pdev->dev, 5363 "Failed configuring TC map %d for VSI %d\n", 5364 enabled_tc, vsi->seid); 5365 goto out; 5366 } 5367 5368 /* Update Queue Pairs Mapping for currently enabled UPs */ 5369 ctxt.seid = vsi->seid; 5370 ctxt.pf_num = vsi->back->hw.pf_id; 5371 ctxt.vf_num = 0; 5372 ctxt.uplink_seid = vsi->uplink_seid; 5373 ctxt.info = vsi->info; 5374 if (vsi->back->flags & I40E_FLAG_TC_MQPRIO) { 5375 ret = i40e_vsi_setup_queue_map_mqprio(vsi, &ctxt, enabled_tc); 5376 if (ret) 5377 goto out; 5378 } else { 5379 i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, false); 5380 } 5381 5382 /* On destroying the qdisc, reset vsi->rss_size, as number of enabled 5383 * queues changed. 5384 */ 5385 if (!vsi->mqprio_qopt.qopt.hw && vsi->reconfig_rss) { 5386 vsi->rss_size = min_t(int, vsi->back->alloc_rss_size, 5387 vsi->num_queue_pairs); 5388 ret = i40e_vsi_config_rss(vsi); 5389 if (ret) { 5390 dev_info(&vsi->back->pdev->dev, 5391 "Failed to reconfig rss for num_queues\n"); 5392 return ret; 5393 } 5394 vsi->reconfig_rss = false; 5395 } 5396 if (vsi->back->flags & I40E_FLAG_IWARP_ENABLED) { 5397 ctxt.info.valid_sections |= 5398 cpu_to_le16(I40E_AQ_VSI_PROP_QUEUE_OPT_VALID); 5399 ctxt.info.queueing_opt_flags |= I40E_AQ_VSI_QUE_OPT_TCP_ENA; 5400 } 5401 5402 /* Update the VSI after updating the VSI queue-mapping 5403 * information 5404 */ 5405 ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL); 5406 if (ret) { 5407 dev_info(&vsi->back->pdev->dev, 5408 "Update vsi tc config failed, err %s aq_err %s\n", 5409 i40e_stat_str(&vsi->back->hw, ret), 5410 i40e_aq_str(&vsi->back->hw, 5411 vsi->back->hw.aq.asq_last_status)); 5412 goto out; 5413 } 5414 /* update the local VSI info with updated queue map */ 5415 i40e_vsi_update_queue_map(vsi, &ctxt); 5416 vsi->info.valid_sections = 0; 5417 5418 /* Update current VSI BW information */ 5419 ret = i40e_vsi_get_bw_info(vsi); 5420 if (ret) { 5421 dev_info(&vsi->back->pdev->dev, 5422 "Failed updating vsi bw info, err %s aq_err %s\n", 5423 i40e_stat_str(&vsi->back->hw, ret), 5424 i40e_aq_str(&vsi->back->hw, 5425 vsi->back->hw.aq.asq_last_status)); 5426 goto out; 5427 } 5428 5429 /* Update the netdev TC setup */ 5430 i40e_vsi_config_netdev_tc(vsi, enabled_tc); 5431 out: 5432 return ret; 5433 } 5434 5435 /** 5436 * i40e_get_link_speed - Returns link speed for the interface 5437 * @vsi: VSI to be configured 5438 * 5439 **/ 5440 int i40e_get_link_speed(struct i40e_vsi *vsi) 5441 { 5442 struct i40e_pf *pf = vsi->back; 5443 5444 switch (pf->hw.phy.link_info.link_speed) { 5445 case I40E_LINK_SPEED_40GB: 5446 return 40000; 5447 case I40E_LINK_SPEED_25GB: 5448 return 25000; 5449 case I40E_LINK_SPEED_20GB: 5450 return 20000; 5451 case I40E_LINK_SPEED_10GB: 5452 return 10000; 5453 case I40E_LINK_SPEED_1GB: 5454 return 1000; 5455 default: 5456 return -EINVAL; 5457 } 5458 } 5459 5460 /** 5461 * i40e_set_bw_limit - setup BW limit for Tx traffic based on max_tx_rate 5462 * @vsi: VSI to be configured 5463 * @seid: seid of the channel/VSI 5464 * @max_tx_rate: max TX rate to be configured as BW limit 5465 * 5466 * Helper function to set BW limit for a given VSI 5467 **/ 5468 int i40e_set_bw_limit(struct i40e_vsi *vsi, u16 seid, u64 max_tx_rate) 5469 { 5470 struct i40e_pf *pf = vsi->back; 5471 u64 credits = 0; 5472 int speed = 0; 5473 int ret = 0; 5474 5475 speed = i40e_get_link_speed(vsi); 5476 if (max_tx_rate > speed) { 5477 dev_err(&pf->pdev->dev, 5478 "Invalid max tx rate %llu specified for VSI seid %d.", 5479 max_tx_rate, seid); 5480 return -EINVAL; 5481 } 5482 if (max_tx_rate && max_tx_rate < 50) { 5483 dev_warn(&pf->pdev->dev, 5484 "Setting max tx rate to minimum usable value of 50Mbps.\n"); 5485 max_tx_rate = 50; 5486 } 5487 5488 /* Tx rate credits are in values of 50Mbps, 0 is disabled */ 5489 credits = max_tx_rate; 5490 do_div(credits, I40E_BW_CREDIT_DIVISOR); 5491 ret = i40e_aq_config_vsi_bw_limit(&pf->hw, seid, credits, 5492 I40E_MAX_BW_INACTIVE_ACCUM, NULL); 5493 if (ret) 5494 dev_err(&pf->pdev->dev, 5495 "Failed set tx rate (%llu Mbps) for vsi->seid %u, err %s aq_err %s\n", 5496 max_tx_rate, seid, i40e_stat_str(&pf->hw, ret), 5497 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 5498 return ret; 5499 } 5500 5501 /** 5502 * i40e_remove_queue_channels - Remove queue channels for the TCs 5503 * @vsi: VSI to be configured 5504 * 5505 * Remove queue channels for the TCs 5506 **/ 5507 static void i40e_remove_queue_channels(struct i40e_vsi *vsi) 5508 { 5509 enum i40e_admin_queue_err last_aq_status; 5510 struct i40e_cloud_filter *cfilter; 5511 struct i40e_channel *ch, *ch_tmp; 5512 struct i40e_pf *pf = vsi->back; 5513 struct hlist_node *node; 5514 int ret, i; 5515 5516 /* Reset rss size that was stored when reconfiguring rss for 5517 * channel VSIs with non-power-of-2 queue count. 5518 */ 5519 vsi->current_rss_size = 0; 5520 5521 /* perform cleanup for channels if they exist */ 5522 if (list_empty(&vsi->ch_list)) 5523 return; 5524 5525 list_for_each_entry_safe(ch, ch_tmp, &vsi->ch_list, list) { 5526 struct i40e_vsi *p_vsi; 5527 5528 list_del(&ch->list); 5529 p_vsi = ch->parent_vsi; 5530 if (!p_vsi || !ch->initialized) { 5531 kfree(ch); 5532 continue; 5533 } 5534 /* Reset queue contexts */ 5535 for (i = 0; i < ch->num_queue_pairs; i++) { 5536 struct i40e_ring *tx_ring, *rx_ring; 5537 u16 pf_q; 5538 5539 pf_q = ch->base_queue + i; 5540 tx_ring = vsi->tx_rings[pf_q]; 5541 tx_ring->ch = NULL; 5542 5543 rx_ring = vsi->rx_rings[pf_q]; 5544 rx_ring->ch = NULL; 5545 } 5546 5547 /* Reset BW configured for this VSI via mqprio */ 5548 ret = i40e_set_bw_limit(vsi, ch->seid, 0); 5549 if (ret) 5550 dev_info(&vsi->back->pdev->dev, 5551 "Failed to reset tx rate for ch->seid %u\n", 5552 ch->seid); 5553 5554 /* delete cloud filters associated with this channel */ 5555 hlist_for_each_entry_safe(cfilter, node, 5556 &pf->cloud_filter_list, cloud_node) { 5557 if (cfilter->seid != ch->seid) 5558 continue; 5559 5560 hash_del(&cfilter->cloud_node); 5561 if (cfilter->dst_port) 5562 ret = i40e_add_del_cloud_filter_big_buf(vsi, 5563 cfilter, 5564 false); 5565 else 5566 ret = i40e_add_del_cloud_filter(vsi, cfilter, 5567 false); 5568 last_aq_status = pf->hw.aq.asq_last_status; 5569 if (ret) 5570 dev_info(&pf->pdev->dev, 5571 "Failed to delete cloud filter, err %s aq_err %s\n", 5572 i40e_stat_str(&pf->hw, ret), 5573 i40e_aq_str(&pf->hw, last_aq_status)); 5574 kfree(cfilter); 5575 } 5576 5577 /* delete VSI from FW */ 5578 ret = i40e_aq_delete_element(&vsi->back->hw, ch->seid, 5579 NULL); 5580 if (ret) 5581 dev_err(&vsi->back->pdev->dev, 5582 "unable to remove channel (%d) for parent VSI(%d)\n", 5583 ch->seid, p_vsi->seid); 5584 kfree(ch); 5585 } 5586 INIT_LIST_HEAD(&vsi->ch_list); 5587 } 5588 5589 /** 5590 * i40e_is_any_channel - channel exist or not 5591 * @vsi: ptr to VSI to which channels are associated with 5592 * 5593 * Returns true or false if channel(s) exist for associated VSI or not 5594 **/ 5595 static bool i40e_is_any_channel(struct i40e_vsi *vsi) 5596 { 5597 struct i40e_channel *ch, *ch_tmp; 5598 5599 list_for_each_entry_safe(ch, ch_tmp, &vsi->ch_list, list) { 5600 if (ch->initialized) 5601 return true; 5602 } 5603 5604 return false; 5605 } 5606 5607 /** 5608 * i40e_get_max_queues_for_channel 5609 * @vsi: ptr to VSI to which channels are associated with 5610 * 5611 * Helper function which returns max value among the queue counts set on the 5612 * channels/TCs created. 5613 **/ 5614 static int i40e_get_max_queues_for_channel(struct i40e_vsi *vsi) 5615 { 5616 struct i40e_channel *ch, *ch_tmp; 5617 int max = 0; 5618 5619 list_for_each_entry_safe(ch, ch_tmp, &vsi->ch_list, list) { 5620 if (!ch->initialized) 5621 continue; 5622 if (ch->num_queue_pairs > max) 5623 max = ch->num_queue_pairs; 5624 } 5625 5626 return max; 5627 } 5628 5629 /** 5630 * i40e_validate_num_queues - validate num_queues w.r.t channel 5631 * @pf: ptr to PF device 5632 * @num_queues: number of queues 5633 * @vsi: the parent VSI 5634 * @reconfig_rss: indicates should the RSS be reconfigured or not 5635 * 5636 * This function validates number of queues in the context of new channel 5637 * which is being established and determines if RSS should be reconfigured 5638 * or not for parent VSI. 5639 **/ 5640 static int i40e_validate_num_queues(struct i40e_pf *pf, int num_queues, 5641 struct i40e_vsi *vsi, bool *reconfig_rss) 5642 { 5643 int max_ch_queues; 5644 5645 if (!reconfig_rss) 5646 return -EINVAL; 5647 5648 *reconfig_rss = false; 5649 if (vsi->current_rss_size) { 5650 if (num_queues > vsi->current_rss_size) { 5651 dev_dbg(&pf->pdev->dev, 5652 "Error: num_queues (%d) > vsi's current_size(%d)\n", 5653 num_queues, vsi->current_rss_size); 5654 return -EINVAL; 5655 } else if ((num_queues < vsi->current_rss_size) && 5656 (!is_power_of_2(num_queues))) { 5657 dev_dbg(&pf->pdev->dev, 5658 "Error: num_queues (%d) < vsi's current_size(%d), but not power of 2\n", 5659 num_queues, vsi->current_rss_size); 5660 return -EINVAL; 5661 } 5662 } 5663 5664 if (!is_power_of_2(num_queues)) { 5665 /* Find the max num_queues configured for channel if channel 5666 * exist. 5667 * if channel exist, then enforce 'num_queues' to be more than 5668 * max ever queues configured for channel. 5669 */ 5670 max_ch_queues = i40e_get_max_queues_for_channel(vsi); 5671 if (num_queues < max_ch_queues) { 5672 dev_dbg(&pf->pdev->dev, 5673 "Error: num_queues (%d) < max queues configured for channel(%d)\n", 5674 num_queues, max_ch_queues); 5675 return -EINVAL; 5676 } 5677 *reconfig_rss = true; 5678 } 5679 5680 return 0; 5681 } 5682 5683 /** 5684 * i40e_vsi_reconfig_rss - reconfig RSS based on specified rss_size 5685 * @vsi: the VSI being setup 5686 * @rss_size: size of RSS, accordingly LUT gets reprogrammed 5687 * 5688 * This function reconfigures RSS by reprogramming LUTs using 'rss_size' 5689 **/ 5690 static int i40e_vsi_reconfig_rss(struct i40e_vsi *vsi, u16 rss_size) 5691 { 5692 struct i40e_pf *pf = vsi->back; 5693 u8 seed[I40E_HKEY_ARRAY_SIZE]; 5694 struct i40e_hw *hw = &pf->hw; 5695 int local_rss_size; 5696 u8 *lut; 5697 int ret; 5698 5699 if (!vsi->rss_size) 5700 return -EINVAL; 5701 5702 if (rss_size > vsi->rss_size) 5703 return -EINVAL; 5704 5705 local_rss_size = min_t(int, vsi->rss_size, rss_size); 5706 lut = kzalloc(vsi->rss_table_size, GFP_KERNEL); 5707 if (!lut) 5708 return -ENOMEM; 5709 5710 /* Ignoring user configured lut if there is one */ 5711 i40e_fill_rss_lut(pf, lut, vsi->rss_table_size, local_rss_size); 5712 5713 /* Use user configured hash key if there is one, otherwise 5714 * use default. 5715 */ 5716 if (vsi->rss_hkey_user) 5717 memcpy(seed, vsi->rss_hkey_user, I40E_HKEY_ARRAY_SIZE); 5718 else 5719 netdev_rss_key_fill((void *)seed, I40E_HKEY_ARRAY_SIZE); 5720 5721 ret = i40e_config_rss(vsi, seed, lut, vsi->rss_table_size); 5722 if (ret) { 5723 dev_info(&pf->pdev->dev, 5724 "Cannot set RSS lut, err %s aq_err %s\n", 5725 i40e_stat_str(hw, ret), 5726 i40e_aq_str(hw, hw->aq.asq_last_status)); 5727 kfree(lut); 5728 return ret; 5729 } 5730 kfree(lut); 5731 5732 /* Do the update w.r.t. storing rss_size */ 5733 if (!vsi->orig_rss_size) 5734 vsi->orig_rss_size = vsi->rss_size; 5735 vsi->current_rss_size = local_rss_size; 5736 5737 return ret; 5738 } 5739 5740 /** 5741 * i40e_channel_setup_queue_map - Setup a channel queue map 5742 * @pf: ptr to PF device 5743 * @vsi: the VSI being setup 5744 * @ctxt: VSI context structure 5745 * @ch: ptr to channel structure 5746 * 5747 * Setup queue map for a specific channel 5748 **/ 5749 static void i40e_channel_setup_queue_map(struct i40e_pf *pf, 5750 struct i40e_vsi_context *ctxt, 5751 struct i40e_channel *ch) 5752 { 5753 u16 qcount, qmap, sections = 0; 5754 u8 offset = 0; 5755 int pow; 5756 5757 sections = I40E_AQ_VSI_PROP_QUEUE_MAP_VALID; 5758 sections |= I40E_AQ_VSI_PROP_SCHED_VALID; 5759 5760 qcount = min_t(int, ch->num_queue_pairs, pf->num_lan_msix); 5761 ch->num_queue_pairs = qcount; 5762 5763 /* find the next higher power-of-2 of num queue pairs */ 5764 pow = ilog2(qcount); 5765 if (!is_power_of_2(qcount)) 5766 pow++; 5767 5768 qmap = (offset << I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT) | 5769 (pow << I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT); 5770 5771 /* Setup queue TC[0].qmap for given VSI context */ 5772 ctxt->info.tc_mapping[0] = cpu_to_le16(qmap); 5773 5774 ctxt->info.up_enable_bits = 0x1; /* TC0 enabled */ 5775 ctxt->info.mapping_flags |= cpu_to_le16(I40E_AQ_VSI_QUE_MAP_CONTIG); 5776 ctxt->info.queue_mapping[0] = cpu_to_le16(ch->base_queue); 5777 ctxt->info.valid_sections |= cpu_to_le16(sections); 5778 } 5779 5780 /** 5781 * i40e_add_channel - add a channel by adding VSI 5782 * @pf: ptr to PF device 5783 * @uplink_seid: underlying HW switching element (VEB) ID 5784 * @ch: ptr to channel structure 5785 * 5786 * Add a channel (VSI) using add_vsi and queue_map 5787 **/ 5788 static int i40e_add_channel(struct i40e_pf *pf, u16 uplink_seid, 5789 struct i40e_channel *ch) 5790 { 5791 struct i40e_hw *hw = &pf->hw; 5792 struct i40e_vsi_context ctxt; 5793 u8 enabled_tc = 0x1; /* TC0 enabled */ 5794 int ret; 5795 5796 if (ch->type != I40E_VSI_VMDQ2) { 5797 dev_info(&pf->pdev->dev, 5798 "add new vsi failed, ch->type %d\n", ch->type); 5799 return -EINVAL; 5800 } 5801 5802 memset(&ctxt, 0, sizeof(ctxt)); 5803 ctxt.pf_num = hw->pf_id; 5804 ctxt.vf_num = 0; 5805 ctxt.uplink_seid = uplink_seid; 5806 ctxt.connection_type = I40E_AQ_VSI_CONN_TYPE_NORMAL; 5807 if (ch->type == I40E_VSI_VMDQ2) 5808 ctxt.flags = I40E_AQ_VSI_TYPE_VMDQ2; 5809 5810 if (pf->flags & I40E_FLAG_VEB_MODE_ENABLED) { 5811 ctxt.info.valid_sections |= 5812 cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID); 5813 ctxt.info.switch_id = 5814 cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB); 5815 } 5816 5817 /* Set queue map for a given VSI context */ 5818 i40e_channel_setup_queue_map(pf, &ctxt, ch); 5819 5820 /* Now time to create VSI */ 5821 ret = i40e_aq_add_vsi(hw, &ctxt, NULL); 5822 if (ret) { 5823 dev_info(&pf->pdev->dev, 5824 "add new vsi failed, err %s aq_err %s\n", 5825 i40e_stat_str(&pf->hw, ret), 5826 i40e_aq_str(&pf->hw, 5827 pf->hw.aq.asq_last_status)); 5828 return -ENOENT; 5829 } 5830 5831 /* Success, update channel */ 5832 ch->enabled_tc = enabled_tc; 5833 ch->seid = ctxt.seid; 5834 ch->vsi_number = ctxt.vsi_number; 5835 ch->stat_counter_idx = cpu_to_le16(ctxt.info.stat_counter_idx); 5836 5837 /* copy just the sections touched not the entire info 5838 * since not all sections are valid as returned by 5839 * update vsi params 5840 */ 5841 ch->info.mapping_flags = ctxt.info.mapping_flags; 5842 memcpy(&ch->info.queue_mapping, 5843 &ctxt.info.queue_mapping, sizeof(ctxt.info.queue_mapping)); 5844 memcpy(&ch->info.tc_mapping, ctxt.info.tc_mapping, 5845 sizeof(ctxt.info.tc_mapping)); 5846 5847 return 0; 5848 } 5849 5850 static int i40e_channel_config_bw(struct i40e_vsi *vsi, struct i40e_channel *ch, 5851 u8 *bw_share) 5852 { 5853 struct i40e_aqc_configure_vsi_tc_bw_data bw_data; 5854 i40e_status ret; 5855 int i; 5856 5857 bw_data.tc_valid_bits = ch->enabled_tc; 5858 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) 5859 bw_data.tc_bw_credits[i] = bw_share[i]; 5860 5861 ret = i40e_aq_config_vsi_tc_bw(&vsi->back->hw, ch->seid, 5862 &bw_data, NULL); 5863 if (ret) { 5864 dev_info(&vsi->back->pdev->dev, 5865 "Config VSI BW allocation per TC failed, aq_err: %d for new_vsi->seid %u\n", 5866 vsi->back->hw.aq.asq_last_status, ch->seid); 5867 return -EINVAL; 5868 } 5869 5870 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) 5871 ch->info.qs_handle[i] = bw_data.qs_handles[i]; 5872 5873 return 0; 5874 } 5875 5876 /** 5877 * i40e_channel_config_tx_ring - config TX ring associated with new channel 5878 * @pf: ptr to PF device 5879 * @vsi: the VSI being setup 5880 * @ch: ptr to channel structure 5881 * 5882 * Configure TX rings associated with channel (VSI) since queues are being 5883 * from parent VSI. 5884 **/ 5885 static int i40e_channel_config_tx_ring(struct i40e_pf *pf, 5886 struct i40e_vsi *vsi, 5887 struct i40e_channel *ch) 5888 { 5889 i40e_status ret; 5890 int i; 5891 u8 bw_share[I40E_MAX_TRAFFIC_CLASS] = {0}; 5892 5893 /* Enable ETS TCs with equal BW Share for now across all VSIs */ 5894 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 5895 if (ch->enabled_tc & BIT(i)) 5896 bw_share[i] = 1; 5897 } 5898 5899 /* configure BW for new VSI */ 5900 ret = i40e_channel_config_bw(vsi, ch, bw_share); 5901 if (ret) { 5902 dev_info(&vsi->back->pdev->dev, 5903 "Failed configuring TC map %d for channel (seid %u)\n", 5904 ch->enabled_tc, ch->seid); 5905 return ret; 5906 } 5907 5908 for (i = 0; i < ch->num_queue_pairs; i++) { 5909 struct i40e_ring *tx_ring, *rx_ring; 5910 u16 pf_q; 5911 5912 pf_q = ch->base_queue + i; 5913 5914 /* Get to TX ring ptr of main VSI, for re-setup TX queue 5915 * context 5916 */ 5917 tx_ring = vsi->tx_rings[pf_q]; 5918 tx_ring->ch = ch; 5919 5920 /* Get the RX ring ptr */ 5921 rx_ring = vsi->rx_rings[pf_q]; 5922 rx_ring->ch = ch; 5923 } 5924 5925 return 0; 5926 } 5927 5928 /** 5929 * i40e_setup_hw_channel - setup new channel 5930 * @pf: ptr to PF device 5931 * @vsi: the VSI being setup 5932 * @ch: ptr to channel structure 5933 * @uplink_seid: underlying HW switching element (VEB) ID 5934 * @type: type of channel to be created (VMDq2/VF) 5935 * 5936 * Setup new channel (VSI) based on specified type (VMDq2/VF) 5937 * and configures TX rings accordingly 5938 **/ 5939 static inline int i40e_setup_hw_channel(struct i40e_pf *pf, 5940 struct i40e_vsi *vsi, 5941 struct i40e_channel *ch, 5942 u16 uplink_seid, u8 type) 5943 { 5944 int ret; 5945 5946 ch->initialized = false; 5947 ch->base_queue = vsi->next_base_queue; 5948 ch->type = type; 5949 5950 /* Proceed with creation of channel (VMDq2) VSI */ 5951 ret = i40e_add_channel(pf, uplink_seid, ch); 5952 if (ret) { 5953 dev_info(&pf->pdev->dev, 5954 "failed to add_channel using uplink_seid %u\n", 5955 uplink_seid); 5956 return ret; 5957 } 5958 5959 /* Mark the successful creation of channel */ 5960 ch->initialized = true; 5961 5962 /* Reconfigure TX queues using QTX_CTL register */ 5963 ret = i40e_channel_config_tx_ring(pf, vsi, ch); 5964 if (ret) { 5965 dev_info(&pf->pdev->dev, 5966 "failed to configure TX rings for channel %u\n", 5967 ch->seid); 5968 return ret; 5969 } 5970 5971 /* update 'next_base_queue' */ 5972 vsi->next_base_queue = vsi->next_base_queue + ch->num_queue_pairs; 5973 dev_dbg(&pf->pdev->dev, 5974 "Added channel: vsi_seid %u, vsi_number %u, stat_counter_idx %u, num_queue_pairs %u, pf->next_base_queue %d\n", 5975 ch->seid, ch->vsi_number, ch->stat_counter_idx, 5976 ch->num_queue_pairs, 5977 vsi->next_base_queue); 5978 return ret; 5979 } 5980 5981 /** 5982 * i40e_setup_channel - setup new channel using uplink element 5983 * @pf: ptr to PF device 5984 * @type: type of channel to be created (VMDq2/VF) 5985 * @uplink_seid: underlying HW switching element (VEB) ID 5986 * @ch: ptr to channel structure 5987 * 5988 * Setup new channel (VSI) based on specified type (VMDq2/VF) 5989 * and uplink switching element (uplink_seid) 5990 **/ 5991 static bool i40e_setup_channel(struct i40e_pf *pf, struct i40e_vsi *vsi, 5992 struct i40e_channel *ch) 5993 { 5994 u8 vsi_type; 5995 u16 seid; 5996 int ret; 5997 5998 if (vsi->type == I40E_VSI_MAIN) { 5999 vsi_type = I40E_VSI_VMDQ2; 6000 } else { 6001 dev_err(&pf->pdev->dev, "unsupported parent vsi type(%d)\n", 6002 vsi->type); 6003 return false; 6004 } 6005 6006 /* underlying switching element */ 6007 seid = pf->vsi[pf->lan_vsi]->uplink_seid; 6008 6009 /* create channel (VSI), configure TX rings */ 6010 ret = i40e_setup_hw_channel(pf, vsi, ch, seid, vsi_type); 6011 if (ret) { 6012 dev_err(&pf->pdev->dev, "failed to setup hw_channel\n"); 6013 return false; 6014 } 6015 6016 return ch->initialized ? true : false; 6017 } 6018 6019 /** 6020 * i40e_validate_and_set_switch_mode - sets up switch mode correctly 6021 * @vsi: ptr to VSI which has PF backing 6022 * 6023 * Sets up switch mode correctly if it needs to be changed and perform 6024 * what are allowed modes. 6025 **/ 6026 static int i40e_validate_and_set_switch_mode(struct i40e_vsi *vsi) 6027 { 6028 u8 mode; 6029 struct i40e_pf *pf = vsi->back; 6030 struct i40e_hw *hw = &pf->hw; 6031 int ret; 6032 6033 ret = i40e_get_capabilities(pf, i40e_aqc_opc_list_dev_capabilities); 6034 if (ret) 6035 return -EINVAL; 6036 6037 if (hw->dev_caps.switch_mode) { 6038 /* if switch mode is set, support mode2 (non-tunneled for 6039 * cloud filter) for now 6040 */ 6041 u32 switch_mode = hw->dev_caps.switch_mode & 6042 I40E_SWITCH_MODE_MASK; 6043 if (switch_mode >= I40E_CLOUD_FILTER_MODE1) { 6044 if (switch_mode == I40E_CLOUD_FILTER_MODE2) 6045 return 0; 6046 dev_err(&pf->pdev->dev, 6047 "Invalid switch_mode (%d), only non-tunneled mode for cloud filter is supported\n", 6048 hw->dev_caps.switch_mode); 6049 return -EINVAL; 6050 } 6051 } 6052 6053 /* Set Bit 7 to be valid */ 6054 mode = I40E_AQ_SET_SWITCH_BIT7_VALID; 6055 6056 /* Set L4type for TCP support */ 6057 mode |= I40E_AQ_SET_SWITCH_L4_TYPE_TCP; 6058 6059 /* Set cloud filter mode */ 6060 mode |= I40E_AQ_SET_SWITCH_MODE_NON_TUNNEL; 6061 6062 /* Prep mode field for set_switch_config */ 6063 ret = i40e_aq_set_switch_config(hw, pf->last_sw_conf_flags, 6064 pf->last_sw_conf_valid_flags, 6065 mode, NULL); 6066 if (ret && hw->aq.asq_last_status != I40E_AQ_RC_ESRCH) 6067 dev_err(&pf->pdev->dev, 6068 "couldn't set switch config bits, err %s aq_err %s\n", 6069 i40e_stat_str(hw, ret), 6070 i40e_aq_str(hw, 6071 hw->aq.asq_last_status)); 6072 6073 return ret; 6074 } 6075 6076 /** 6077 * i40e_create_queue_channel - function to create channel 6078 * @vsi: VSI to be configured 6079 * @ch: ptr to channel (it contains channel specific params) 6080 * 6081 * This function creates channel (VSI) using num_queues specified by user, 6082 * reconfigs RSS if needed. 6083 **/ 6084 int i40e_create_queue_channel(struct i40e_vsi *vsi, 6085 struct i40e_channel *ch) 6086 { 6087 struct i40e_pf *pf = vsi->back; 6088 bool reconfig_rss; 6089 int err; 6090 6091 if (!ch) 6092 return -EINVAL; 6093 6094 if (!ch->num_queue_pairs) { 6095 dev_err(&pf->pdev->dev, "Invalid num_queues requested: %d\n", 6096 ch->num_queue_pairs); 6097 return -EINVAL; 6098 } 6099 6100 /* validate user requested num_queues for channel */ 6101 err = i40e_validate_num_queues(pf, ch->num_queue_pairs, vsi, 6102 &reconfig_rss); 6103 if (err) { 6104 dev_info(&pf->pdev->dev, "Failed to validate num_queues (%d)\n", 6105 ch->num_queue_pairs); 6106 return -EINVAL; 6107 } 6108 6109 /* By default we are in VEPA mode, if this is the first VF/VMDq 6110 * VSI to be added switch to VEB mode. 6111 */ 6112 if ((!(pf->flags & I40E_FLAG_VEB_MODE_ENABLED)) || 6113 (!i40e_is_any_channel(vsi))) { 6114 if (!is_power_of_2(vsi->tc_config.tc_info[0].qcount)) { 6115 dev_dbg(&pf->pdev->dev, 6116 "Failed to create channel. Override queues (%u) not power of 2\n", 6117 vsi->tc_config.tc_info[0].qcount); 6118 return -EINVAL; 6119 } 6120 6121 if (!(pf->flags & I40E_FLAG_VEB_MODE_ENABLED)) { 6122 pf->flags |= I40E_FLAG_VEB_MODE_ENABLED; 6123 6124 if (vsi->type == I40E_VSI_MAIN) { 6125 if (pf->flags & I40E_FLAG_TC_MQPRIO) 6126 i40e_do_reset(pf, I40E_PF_RESET_FLAG, 6127 true); 6128 else 6129 i40e_do_reset_safe(pf, 6130 I40E_PF_RESET_FLAG); 6131 } 6132 } 6133 /* now onwards for main VSI, number of queues will be value 6134 * of TC0's queue count 6135 */ 6136 } 6137 6138 /* By this time, vsi->cnt_q_avail shall be set to non-zero and 6139 * it should be more than num_queues 6140 */ 6141 if (!vsi->cnt_q_avail || vsi->cnt_q_avail < ch->num_queue_pairs) { 6142 dev_dbg(&pf->pdev->dev, 6143 "Error: cnt_q_avail (%u) less than num_queues %d\n", 6144 vsi->cnt_q_avail, ch->num_queue_pairs); 6145 return -EINVAL; 6146 } 6147 6148 /* reconfig_rss only if vsi type is MAIN_VSI */ 6149 if (reconfig_rss && (vsi->type == I40E_VSI_MAIN)) { 6150 err = i40e_vsi_reconfig_rss(vsi, ch->num_queue_pairs); 6151 if (err) { 6152 dev_info(&pf->pdev->dev, 6153 "Error: unable to reconfig rss for num_queues (%u)\n", 6154 ch->num_queue_pairs); 6155 return -EINVAL; 6156 } 6157 } 6158 6159 if (!i40e_setup_channel(pf, vsi, ch)) { 6160 dev_info(&pf->pdev->dev, "Failed to setup channel\n"); 6161 return -EINVAL; 6162 } 6163 6164 dev_info(&pf->pdev->dev, 6165 "Setup channel (id:%u) utilizing num_queues %d\n", 6166 ch->seid, ch->num_queue_pairs); 6167 6168 /* configure VSI for BW limit */ 6169 if (ch->max_tx_rate) { 6170 u64 credits = ch->max_tx_rate; 6171 6172 if (i40e_set_bw_limit(vsi, ch->seid, ch->max_tx_rate)) 6173 return -EINVAL; 6174 6175 do_div(credits, I40E_BW_CREDIT_DIVISOR); 6176 dev_dbg(&pf->pdev->dev, 6177 "Set tx rate of %llu Mbps (count of 50Mbps %llu) for vsi->seid %u\n", 6178 ch->max_tx_rate, 6179 credits, 6180 ch->seid); 6181 } 6182 6183 /* in case of VF, this will be main SRIOV VSI */ 6184 ch->parent_vsi = vsi; 6185 6186 /* and update main_vsi's count for queue_available to use */ 6187 vsi->cnt_q_avail -= ch->num_queue_pairs; 6188 6189 return 0; 6190 } 6191 6192 /** 6193 * i40e_configure_queue_channels - Add queue channel for the given TCs 6194 * @vsi: VSI to be configured 6195 * 6196 * Configures queue channel mapping to the given TCs 6197 **/ 6198 static int i40e_configure_queue_channels(struct i40e_vsi *vsi) 6199 { 6200 struct i40e_channel *ch; 6201 u64 max_rate = 0; 6202 int ret = 0, i; 6203 6204 /* Create app vsi with the TCs. Main VSI with TC0 is already set up */ 6205 vsi->tc_seid_map[0] = vsi->seid; 6206 for (i = 1; i < I40E_MAX_TRAFFIC_CLASS; i++) { 6207 if (vsi->tc_config.enabled_tc & BIT(i)) { 6208 ch = kzalloc(sizeof(*ch), GFP_KERNEL); 6209 if (!ch) { 6210 ret = -ENOMEM; 6211 goto err_free; 6212 } 6213 6214 INIT_LIST_HEAD(&ch->list); 6215 ch->num_queue_pairs = 6216 vsi->tc_config.tc_info[i].qcount; 6217 ch->base_queue = 6218 vsi->tc_config.tc_info[i].qoffset; 6219 6220 /* Bandwidth limit through tc interface is in bytes/s, 6221 * change to Mbit/s 6222 */ 6223 max_rate = vsi->mqprio_qopt.max_rate[i]; 6224 do_div(max_rate, I40E_BW_MBPS_DIVISOR); 6225 ch->max_tx_rate = max_rate; 6226 6227 list_add_tail(&ch->list, &vsi->ch_list); 6228 6229 ret = i40e_create_queue_channel(vsi, ch); 6230 if (ret) { 6231 dev_err(&vsi->back->pdev->dev, 6232 "Failed creating queue channel with TC%d: queues %d\n", 6233 i, ch->num_queue_pairs); 6234 goto err_free; 6235 } 6236 vsi->tc_seid_map[i] = ch->seid; 6237 } 6238 } 6239 return ret; 6240 6241 err_free: 6242 i40e_remove_queue_channels(vsi); 6243 return ret; 6244 } 6245 6246 /** 6247 * i40e_veb_config_tc - Configure TCs for given VEB 6248 * @veb: given VEB 6249 * @enabled_tc: TC bitmap 6250 * 6251 * Configures given TC bitmap for VEB (switching) element 6252 **/ 6253 int i40e_veb_config_tc(struct i40e_veb *veb, u8 enabled_tc) 6254 { 6255 struct i40e_aqc_configure_switching_comp_bw_config_data bw_data = {0}; 6256 struct i40e_pf *pf = veb->pf; 6257 int ret = 0; 6258 int i; 6259 6260 /* No TCs or already enabled TCs just return */ 6261 if (!enabled_tc || veb->enabled_tc == enabled_tc) 6262 return ret; 6263 6264 bw_data.tc_valid_bits = enabled_tc; 6265 /* bw_data.absolute_credits is not set (relative) */ 6266 6267 /* Enable ETS TCs with equal BW Share for now */ 6268 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 6269 if (enabled_tc & BIT(i)) 6270 bw_data.tc_bw_share_credits[i] = 1; 6271 } 6272 6273 ret = i40e_aq_config_switch_comp_bw_config(&pf->hw, veb->seid, 6274 &bw_data, NULL); 6275 if (ret) { 6276 dev_info(&pf->pdev->dev, 6277 "VEB bw config failed, err %s aq_err %s\n", 6278 i40e_stat_str(&pf->hw, ret), 6279 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 6280 goto out; 6281 } 6282 6283 /* Update the BW information */ 6284 ret = i40e_veb_get_bw_info(veb); 6285 if (ret) { 6286 dev_info(&pf->pdev->dev, 6287 "Failed getting veb bw config, err %s aq_err %s\n", 6288 i40e_stat_str(&pf->hw, ret), 6289 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 6290 } 6291 6292 out: 6293 return ret; 6294 } 6295 6296 #ifdef CONFIG_I40E_DCB 6297 /** 6298 * i40e_dcb_reconfigure - Reconfigure all VEBs and VSIs 6299 * @pf: PF struct 6300 * 6301 * Reconfigure VEB/VSIs on a given PF; it is assumed that 6302 * the caller would've quiesce all the VSIs before calling 6303 * this function 6304 **/ 6305 static void i40e_dcb_reconfigure(struct i40e_pf *pf) 6306 { 6307 u8 tc_map = 0; 6308 int ret; 6309 u8 v; 6310 6311 /* Enable the TCs available on PF to all VEBs */ 6312 tc_map = i40e_pf_get_tc_map(pf); 6313 for (v = 0; v < I40E_MAX_VEB; v++) { 6314 if (!pf->veb[v]) 6315 continue; 6316 ret = i40e_veb_config_tc(pf->veb[v], tc_map); 6317 if (ret) { 6318 dev_info(&pf->pdev->dev, 6319 "Failed configuring TC for VEB seid=%d\n", 6320 pf->veb[v]->seid); 6321 /* Will try to configure as many components */ 6322 } 6323 } 6324 6325 /* Update each VSI */ 6326 for (v = 0; v < pf->num_alloc_vsi; v++) { 6327 if (!pf->vsi[v]) 6328 continue; 6329 6330 /* - Enable all TCs for the LAN VSI 6331 * - For all others keep them at TC0 for now 6332 */ 6333 if (v == pf->lan_vsi) 6334 tc_map = i40e_pf_get_tc_map(pf); 6335 else 6336 tc_map = I40E_DEFAULT_TRAFFIC_CLASS; 6337 6338 ret = i40e_vsi_config_tc(pf->vsi[v], tc_map); 6339 if (ret) { 6340 dev_info(&pf->pdev->dev, 6341 "Failed configuring TC for VSI seid=%d\n", 6342 pf->vsi[v]->seid); 6343 /* Will try to configure as many components */ 6344 } else { 6345 /* Re-configure VSI vectors based on updated TC map */ 6346 i40e_vsi_map_rings_to_vectors(pf->vsi[v]); 6347 if (pf->vsi[v]->netdev) 6348 i40e_dcbnl_set_all(pf->vsi[v]); 6349 } 6350 } 6351 } 6352 6353 /** 6354 * i40e_resume_port_tx - Resume port Tx 6355 * @pf: PF struct 6356 * 6357 * Resume a port's Tx and issue a PF reset in case of failure to 6358 * resume. 6359 **/ 6360 static int i40e_resume_port_tx(struct i40e_pf *pf) 6361 { 6362 struct i40e_hw *hw = &pf->hw; 6363 int ret; 6364 6365 ret = i40e_aq_resume_port_tx(hw, NULL); 6366 if (ret) { 6367 dev_info(&pf->pdev->dev, 6368 "Resume Port Tx failed, err %s aq_err %s\n", 6369 i40e_stat_str(&pf->hw, ret), 6370 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 6371 /* Schedule PF reset to recover */ 6372 set_bit(__I40E_PF_RESET_REQUESTED, pf->state); 6373 i40e_service_event_schedule(pf); 6374 } 6375 6376 return ret; 6377 } 6378 6379 /** 6380 * i40e_init_pf_dcb - Initialize DCB configuration 6381 * @pf: PF being configured 6382 * 6383 * Query the current DCB configuration and cache it 6384 * in the hardware structure 6385 **/ 6386 static int i40e_init_pf_dcb(struct i40e_pf *pf) 6387 { 6388 struct i40e_hw *hw = &pf->hw; 6389 int err = 0; 6390 6391 /* Do not enable DCB for SW1 and SW2 images even if the FW is capable */ 6392 if (pf->hw_features & I40E_HW_NO_DCB_SUPPORT) 6393 goto out; 6394 6395 /* Get the initial DCB configuration */ 6396 err = i40e_init_dcb(hw); 6397 if (!err) { 6398 /* Device/Function is not DCBX capable */ 6399 if ((!hw->func_caps.dcb) || 6400 (hw->dcbx_status == I40E_DCBX_STATUS_DISABLED)) { 6401 dev_info(&pf->pdev->dev, 6402 "DCBX offload is not supported or is disabled for this PF.\n"); 6403 } else { 6404 /* When status is not DISABLED then DCBX in FW */ 6405 pf->dcbx_cap = DCB_CAP_DCBX_LLD_MANAGED | 6406 DCB_CAP_DCBX_VER_IEEE; 6407 6408 pf->flags |= I40E_FLAG_DCB_CAPABLE; 6409 /* Enable DCB tagging only when more than one TC 6410 * or explicitly disable if only one TC 6411 */ 6412 if (i40e_dcb_get_num_tc(&hw->local_dcbx_config) > 1) 6413 pf->flags |= I40E_FLAG_DCB_ENABLED; 6414 else 6415 pf->flags &= ~I40E_FLAG_DCB_ENABLED; 6416 dev_dbg(&pf->pdev->dev, 6417 "DCBX offload is supported for this PF.\n"); 6418 } 6419 } else { 6420 dev_info(&pf->pdev->dev, 6421 "Query for DCB configuration failed, err %s aq_err %s\n", 6422 i40e_stat_str(&pf->hw, err), 6423 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 6424 } 6425 6426 out: 6427 return err; 6428 } 6429 #endif /* CONFIG_I40E_DCB */ 6430 #define SPEED_SIZE 14 6431 #define FC_SIZE 8 6432 /** 6433 * i40e_print_link_message - print link up or down 6434 * @vsi: the VSI for which link needs a message 6435 */ 6436 void i40e_print_link_message(struct i40e_vsi *vsi, bool isup) 6437 { 6438 enum i40e_aq_link_speed new_speed; 6439 struct i40e_pf *pf = vsi->back; 6440 char *speed = "Unknown"; 6441 char *fc = "Unknown"; 6442 char *fec = ""; 6443 char *req_fec = ""; 6444 char *an = ""; 6445 6446 new_speed = pf->hw.phy.link_info.link_speed; 6447 6448 if ((vsi->current_isup == isup) && (vsi->current_speed == new_speed)) 6449 return; 6450 vsi->current_isup = isup; 6451 vsi->current_speed = new_speed; 6452 if (!isup) { 6453 netdev_info(vsi->netdev, "NIC Link is Down\n"); 6454 return; 6455 } 6456 6457 /* Warn user if link speed on NPAR enabled partition is not at 6458 * least 10GB 6459 */ 6460 if (pf->hw.func_caps.npar_enable && 6461 (pf->hw.phy.link_info.link_speed == I40E_LINK_SPEED_1GB || 6462 pf->hw.phy.link_info.link_speed == I40E_LINK_SPEED_100MB)) 6463 netdev_warn(vsi->netdev, 6464 "The partition detected link speed that is less than 10Gbps\n"); 6465 6466 switch (pf->hw.phy.link_info.link_speed) { 6467 case I40E_LINK_SPEED_40GB: 6468 speed = "40 G"; 6469 break; 6470 case I40E_LINK_SPEED_20GB: 6471 speed = "20 G"; 6472 break; 6473 case I40E_LINK_SPEED_25GB: 6474 speed = "25 G"; 6475 break; 6476 case I40E_LINK_SPEED_10GB: 6477 speed = "10 G"; 6478 break; 6479 case I40E_LINK_SPEED_1GB: 6480 speed = "1000 M"; 6481 break; 6482 case I40E_LINK_SPEED_100MB: 6483 speed = "100 M"; 6484 break; 6485 default: 6486 break; 6487 } 6488 6489 switch (pf->hw.fc.current_mode) { 6490 case I40E_FC_FULL: 6491 fc = "RX/TX"; 6492 break; 6493 case I40E_FC_TX_PAUSE: 6494 fc = "TX"; 6495 break; 6496 case I40E_FC_RX_PAUSE: 6497 fc = "RX"; 6498 break; 6499 default: 6500 fc = "None"; 6501 break; 6502 } 6503 6504 if (pf->hw.phy.link_info.link_speed == I40E_LINK_SPEED_25GB) { 6505 req_fec = ", Requested FEC: None"; 6506 fec = ", FEC: None"; 6507 an = ", Autoneg: False"; 6508 6509 if (pf->hw.phy.link_info.an_info & I40E_AQ_AN_COMPLETED) 6510 an = ", Autoneg: True"; 6511 6512 if (pf->hw.phy.link_info.fec_info & 6513 I40E_AQ_CONFIG_FEC_KR_ENA) 6514 fec = ", FEC: CL74 FC-FEC/BASE-R"; 6515 else if (pf->hw.phy.link_info.fec_info & 6516 I40E_AQ_CONFIG_FEC_RS_ENA) 6517 fec = ", FEC: CL108 RS-FEC"; 6518 6519 /* 'CL108 RS-FEC' should be displayed when RS is requested, or 6520 * both RS and FC are requested 6521 */ 6522 if (vsi->back->hw.phy.link_info.req_fec_info & 6523 (I40E_AQ_REQUEST_FEC_KR | I40E_AQ_REQUEST_FEC_RS)) { 6524 if (vsi->back->hw.phy.link_info.req_fec_info & 6525 I40E_AQ_REQUEST_FEC_RS) 6526 req_fec = ", Requested FEC: CL108 RS-FEC"; 6527 else 6528 req_fec = ", Requested FEC: CL74 FC-FEC/BASE-R"; 6529 } 6530 } 6531 6532 netdev_info(vsi->netdev, "NIC Link is Up, %sbps Full Duplex%s%s%s, Flow Control: %s\n", 6533 speed, req_fec, fec, an, fc); 6534 } 6535 6536 /** 6537 * i40e_up_complete - Finish the last steps of bringing up a connection 6538 * @vsi: the VSI being configured 6539 **/ 6540 static int i40e_up_complete(struct i40e_vsi *vsi) 6541 { 6542 struct i40e_pf *pf = vsi->back; 6543 int err; 6544 6545 if (pf->flags & I40E_FLAG_MSIX_ENABLED) 6546 i40e_vsi_configure_msix(vsi); 6547 else 6548 i40e_configure_msi_and_legacy(vsi); 6549 6550 /* start rings */ 6551 err = i40e_vsi_start_rings(vsi); 6552 if (err) 6553 return err; 6554 6555 clear_bit(__I40E_VSI_DOWN, vsi->state); 6556 i40e_napi_enable_all(vsi); 6557 i40e_vsi_enable_irq(vsi); 6558 6559 if ((pf->hw.phy.link_info.link_info & I40E_AQ_LINK_UP) && 6560 (vsi->netdev)) { 6561 i40e_print_link_message(vsi, true); 6562 netif_tx_start_all_queues(vsi->netdev); 6563 netif_carrier_on(vsi->netdev); 6564 } 6565 6566 /* replay FDIR SB filters */ 6567 if (vsi->type == I40E_VSI_FDIR) { 6568 /* reset fd counters */ 6569 pf->fd_add_err = 0; 6570 pf->fd_atr_cnt = 0; 6571 i40e_fdir_filter_restore(vsi); 6572 } 6573 6574 /* On the next run of the service_task, notify any clients of the new 6575 * opened netdev 6576 */ 6577 pf->flags |= I40E_FLAG_SERVICE_CLIENT_REQUESTED; 6578 i40e_service_event_schedule(pf); 6579 6580 return 0; 6581 } 6582 6583 /** 6584 * i40e_vsi_reinit_locked - Reset the VSI 6585 * @vsi: the VSI being configured 6586 * 6587 * Rebuild the ring structs after some configuration 6588 * has changed, e.g. MTU size. 6589 **/ 6590 static void i40e_vsi_reinit_locked(struct i40e_vsi *vsi) 6591 { 6592 struct i40e_pf *pf = vsi->back; 6593 6594 WARN_ON(in_interrupt()); 6595 while (test_and_set_bit(__I40E_CONFIG_BUSY, pf->state)) 6596 usleep_range(1000, 2000); 6597 i40e_down(vsi); 6598 6599 i40e_up(vsi); 6600 clear_bit(__I40E_CONFIG_BUSY, pf->state); 6601 } 6602 6603 /** 6604 * i40e_up - Bring the connection back up after being down 6605 * @vsi: the VSI being configured 6606 **/ 6607 int i40e_up(struct i40e_vsi *vsi) 6608 { 6609 int err; 6610 6611 err = i40e_vsi_configure(vsi); 6612 if (!err) 6613 err = i40e_up_complete(vsi); 6614 6615 return err; 6616 } 6617 6618 /** 6619 * i40e_down - Shutdown the connection processing 6620 * @vsi: the VSI being stopped 6621 **/ 6622 void i40e_down(struct i40e_vsi *vsi) 6623 { 6624 int i; 6625 6626 /* It is assumed that the caller of this function 6627 * sets the vsi->state __I40E_VSI_DOWN bit. 6628 */ 6629 if (vsi->netdev) { 6630 netif_carrier_off(vsi->netdev); 6631 netif_tx_disable(vsi->netdev); 6632 } 6633 i40e_vsi_disable_irq(vsi); 6634 i40e_vsi_stop_rings(vsi); 6635 i40e_napi_disable_all(vsi); 6636 6637 for (i = 0; i < vsi->num_queue_pairs; i++) { 6638 i40e_clean_tx_ring(vsi->tx_rings[i]); 6639 if (i40e_enabled_xdp_vsi(vsi)) 6640 i40e_clean_tx_ring(vsi->xdp_rings[i]); 6641 i40e_clean_rx_ring(vsi->rx_rings[i]); 6642 } 6643 6644 } 6645 6646 /** 6647 * i40e_validate_mqprio_qopt- validate queue mapping info 6648 * @vsi: the VSI being configured 6649 * @mqprio_qopt: queue parametrs 6650 **/ 6651 static int i40e_validate_mqprio_qopt(struct i40e_vsi *vsi, 6652 struct tc_mqprio_qopt_offload *mqprio_qopt) 6653 { 6654 u64 sum_max_rate = 0; 6655 u64 max_rate = 0; 6656 int i; 6657 6658 if (mqprio_qopt->qopt.offset[0] != 0 || 6659 mqprio_qopt->qopt.num_tc < 1 || 6660 mqprio_qopt->qopt.num_tc > I40E_MAX_TRAFFIC_CLASS) 6661 return -EINVAL; 6662 for (i = 0; ; i++) { 6663 if (!mqprio_qopt->qopt.count[i]) 6664 return -EINVAL; 6665 if (mqprio_qopt->min_rate[i]) { 6666 dev_err(&vsi->back->pdev->dev, 6667 "Invalid min tx rate (greater than 0) specified\n"); 6668 return -EINVAL; 6669 } 6670 max_rate = mqprio_qopt->max_rate[i]; 6671 do_div(max_rate, I40E_BW_MBPS_DIVISOR); 6672 sum_max_rate += max_rate; 6673 6674 if (i >= mqprio_qopt->qopt.num_tc - 1) 6675 break; 6676 if (mqprio_qopt->qopt.offset[i + 1] != 6677 (mqprio_qopt->qopt.offset[i] + mqprio_qopt->qopt.count[i])) 6678 return -EINVAL; 6679 } 6680 if (vsi->num_queue_pairs < 6681 (mqprio_qopt->qopt.offset[i] + mqprio_qopt->qopt.count[i])) { 6682 return -EINVAL; 6683 } 6684 if (sum_max_rate > i40e_get_link_speed(vsi)) { 6685 dev_err(&vsi->back->pdev->dev, 6686 "Invalid max tx rate specified\n"); 6687 return -EINVAL; 6688 } 6689 return 0; 6690 } 6691 6692 /** 6693 * i40e_vsi_set_default_tc_config - set default values for tc configuration 6694 * @vsi: the VSI being configured 6695 **/ 6696 static void i40e_vsi_set_default_tc_config(struct i40e_vsi *vsi) 6697 { 6698 u16 qcount; 6699 int i; 6700 6701 /* Only TC0 is enabled */ 6702 vsi->tc_config.numtc = 1; 6703 vsi->tc_config.enabled_tc = 1; 6704 qcount = min_t(int, vsi->alloc_queue_pairs, 6705 i40e_pf_get_max_q_per_tc(vsi->back)); 6706 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 6707 /* For the TC that is not enabled set the offset to to default 6708 * queue and allocate one queue for the given TC. 6709 */ 6710 vsi->tc_config.tc_info[i].qoffset = 0; 6711 if (i == 0) 6712 vsi->tc_config.tc_info[i].qcount = qcount; 6713 else 6714 vsi->tc_config.tc_info[i].qcount = 1; 6715 vsi->tc_config.tc_info[i].netdev_tc = 0; 6716 } 6717 } 6718 6719 /** 6720 * i40e_setup_tc - configure multiple traffic classes 6721 * @netdev: net device to configure 6722 * @type_data: tc offload data 6723 **/ 6724 static int i40e_setup_tc(struct net_device *netdev, void *type_data) 6725 { 6726 struct tc_mqprio_qopt_offload *mqprio_qopt = type_data; 6727 struct i40e_netdev_priv *np = netdev_priv(netdev); 6728 struct i40e_vsi *vsi = np->vsi; 6729 struct i40e_pf *pf = vsi->back; 6730 u8 enabled_tc = 0, num_tc, hw; 6731 bool need_reset = false; 6732 int ret = -EINVAL; 6733 u16 mode; 6734 int i; 6735 6736 num_tc = mqprio_qopt->qopt.num_tc; 6737 hw = mqprio_qopt->qopt.hw; 6738 mode = mqprio_qopt->mode; 6739 if (!hw) { 6740 pf->flags &= ~I40E_FLAG_TC_MQPRIO; 6741 memcpy(&vsi->mqprio_qopt, mqprio_qopt, sizeof(*mqprio_qopt)); 6742 goto config_tc; 6743 } 6744 6745 /* Check if MFP enabled */ 6746 if (pf->flags & I40E_FLAG_MFP_ENABLED) { 6747 netdev_info(netdev, 6748 "Configuring TC not supported in MFP mode\n"); 6749 return ret; 6750 } 6751 switch (mode) { 6752 case TC_MQPRIO_MODE_DCB: 6753 pf->flags &= ~I40E_FLAG_TC_MQPRIO; 6754 6755 /* Check if DCB enabled to continue */ 6756 if (!(pf->flags & I40E_FLAG_DCB_ENABLED)) { 6757 netdev_info(netdev, 6758 "DCB is not enabled for adapter\n"); 6759 return ret; 6760 } 6761 6762 /* Check whether tc count is within enabled limit */ 6763 if (num_tc > i40e_pf_get_num_tc(pf)) { 6764 netdev_info(netdev, 6765 "TC count greater than enabled on link for adapter\n"); 6766 return ret; 6767 } 6768 break; 6769 case TC_MQPRIO_MODE_CHANNEL: 6770 if (pf->flags & I40E_FLAG_DCB_ENABLED) { 6771 netdev_info(netdev, 6772 "Full offload of TC Mqprio options is not supported when DCB is enabled\n"); 6773 return ret; 6774 } 6775 if (!(pf->flags & I40E_FLAG_MSIX_ENABLED)) 6776 return ret; 6777 ret = i40e_validate_mqprio_qopt(vsi, mqprio_qopt); 6778 if (ret) 6779 return ret; 6780 memcpy(&vsi->mqprio_qopt, mqprio_qopt, 6781 sizeof(*mqprio_qopt)); 6782 pf->flags |= I40E_FLAG_TC_MQPRIO; 6783 pf->flags &= ~I40E_FLAG_DCB_ENABLED; 6784 break; 6785 default: 6786 return -EINVAL; 6787 } 6788 6789 config_tc: 6790 /* Generate TC map for number of tc requested */ 6791 for (i = 0; i < num_tc; i++) 6792 enabled_tc |= BIT(i); 6793 6794 /* Requesting same TC configuration as already enabled */ 6795 if (enabled_tc == vsi->tc_config.enabled_tc && 6796 mode != TC_MQPRIO_MODE_CHANNEL) 6797 return 0; 6798 6799 /* Quiesce VSI queues */ 6800 i40e_quiesce_vsi(vsi); 6801 6802 if (!hw && !(pf->flags & I40E_FLAG_TC_MQPRIO)) 6803 i40e_remove_queue_channels(vsi); 6804 6805 /* Configure VSI for enabled TCs */ 6806 ret = i40e_vsi_config_tc(vsi, enabled_tc); 6807 if (ret) { 6808 netdev_info(netdev, "Failed configuring TC for VSI seid=%d\n", 6809 vsi->seid); 6810 need_reset = true; 6811 goto exit; 6812 } 6813 6814 if (pf->flags & I40E_FLAG_TC_MQPRIO) { 6815 if (vsi->mqprio_qopt.max_rate[0]) { 6816 u64 max_tx_rate = vsi->mqprio_qopt.max_rate[0]; 6817 6818 do_div(max_tx_rate, I40E_BW_MBPS_DIVISOR); 6819 ret = i40e_set_bw_limit(vsi, vsi->seid, max_tx_rate); 6820 if (!ret) { 6821 u64 credits = max_tx_rate; 6822 6823 do_div(credits, I40E_BW_CREDIT_DIVISOR); 6824 dev_dbg(&vsi->back->pdev->dev, 6825 "Set tx rate of %llu Mbps (count of 50Mbps %llu) for vsi->seid %u\n", 6826 max_tx_rate, 6827 credits, 6828 vsi->seid); 6829 } else { 6830 need_reset = true; 6831 goto exit; 6832 } 6833 } 6834 ret = i40e_configure_queue_channels(vsi); 6835 if (ret) { 6836 netdev_info(netdev, 6837 "Failed configuring queue channels\n"); 6838 need_reset = true; 6839 goto exit; 6840 } 6841 } 6842 6843 exit: 6844 /* Reset the configuration data to defaults, only TC0 is enabled */ 6845 if (need_reset) { 6846 i40e_vsi_set_default_tc_config(vsi); 6847 need_reset = false; 6848 } 6849 6850 /* Unquiesce VSI */ 6851 i40e_unquiesce_vsi(vsi); 6852 return ret; 6853 } 6854 6855 /** 6856 * i40e_set_cld_element - sets cloud filter element data 6857 * @filter: cloud filter rule 6858 * @cld: ptr to cloud filter element data 6859 * 6860 * This is helper function to copy data into cloud filter element 6861 **/ 6862 static inline void 6863 i40e_set_cld_element(struct i40e_cloud_filter *filter, 6864 struct i40e_aqc_cloud_filters_element_data *cld) 6865 { 6866 int i, j; 6867 u32 ipa; 6868 6869 memset(cld, 0, sizeof(*cld)); 6870 ether_addr_copy(cld->outer_mac, filter->dst_mac); 6871 ether_addr_copy(cld->inner_mac, filter->src_mac); 6872 6873 if (filter->n_proto != ETH_P_IP && filter->n_proto != ETH_P_IPV6) 6874 return; 6875 6876 if (filter->n_proto == ETH_P_IPV6) { 6877 #define IPV6_MAX_INDEX (ARRAY_SIZE(filter->dst_ipv6) - 1) 6878 for (i = 0, j = 0; i < ARRAY_SIZE(filter->dst_ipv6); 6879 i++, j += 2) { 6880 ipa = be32_to_cpu(filter->dst_ipv6[IPV6_MAX_INDEX - i]); 6881 ipa = cpu_to_le32(ipa); 6882 memcpy(&cld->ipaddr.raw_v6.data[j], &ipa, sizeof(ipa)); 6883 } 6884 } else { 6885 ipa = be32_to_cpu(filter->dst_ipv4); 6886 memcpy(&cld->ipaddr.v4.data, &ipa, sizeof(ipa)); 6887 } 6888 6889 cld->inner_vlan = cpu_to_le16(ntohs(filter->vlan_id)); 6890 6891 /* tenant_id is not supported by FW now, once the support is enabled 6892 * fill the cld->tenant_id with cpu_to_le32(filter->tenant_id) 6893 */ 6894 if (filter->tenant_id) 6895 return; 6896 } 6897 6898 /** 6899 * i40e_add_del_cloud_filter - Add/del cloud filter 6900 * @vsi: pointer to VSI 6901 * @filter: cloud filter rule 6902 * @add: if true, add, if false, delete 6903 * 6904 * Add or delete a cloud filter for a specific flow spec. 6905 * Returns 0 if the filter were successfully added. 6906 **/ 6907 static int i40e_add_del_cloud_filter(struct i40e_vsi *vsi, 6908 struct i40e_cloud_filter *filter, bool add) 6909 { 6910 struct i40e_aqc_cloud_filters_element_data cld_filter; 6911 struct i40e_pf *pf = vsi->back; 6912 int ret; 6913 static const u16 flag_table[128] = { 6914 [I40E_CLOUD_FILTER_FLAGS_OMAC] = 6915 I40E_AQC_ADD_CLOUD_FILTER_OMAC, 6916 [I40E_CLOUD_FILTER_FLAGS_IMAC] = 6917 I40E_AQC_ADD_CLOUD_FILTER_IMAC, 6918 [I40E_CLOUD_FILTER_FLAGS_IMAC_IVLAN] = 6919 I40E_AQC_ADD_CLOUD_FILTER_IMAC_IVLAN, 6920 [I40E_CLOUD_FILTER_FLAGS_IMAC_TEN_ID] = 6921 I40E_AQC_ADD_CLOUD_FILTER_IMAC_TEN_ID, 6922 [I40E_CLOUD_FILTER_FLAGS_OMAC_TEN_ID_IMAC] = 6923 I40E_AQC_ADD_CLOUD_FILTER_OMAC_TEN_ID_IMAC, 6924 [I40E_CLOUD_FILTER_FLAGS_IMAC_IVLAN_TEN_ID] = 6925 I40E_AQC_ADD_CLOUD_FILTER_IMAC_IVLAN_TEN_ID, 6926 [I40E_CLOUD_FILTER_FLAGS_IIP] = 6927 I40E_AQC_ADD_CLOUD_FILTER_IIP, 6928 }; 6929 6930 if (filter->flags >= ARRAY_SIZE(flag_table)) 6931 return I40E_ERR_CONFIG; 6932 6933 /* copy element needed to add cloud filter from filter */ 6934 i40e_set_cld_element(filter, &cld_filter); 6935 6936 if (filter->tunnel_type != I40E_CLOUD_TNL_TYPE_NONE) 6937 cld_filter.flags = cpu_to_le16(filter->tunnel_type << 6938 I40E_AQC_ADD_CLOUD_TNL_TYPE_SHIFT); 6939 6940 if (filter->n_proto == ETH_P_IPV6) 6941 cld_filter.flags |= cpu_to_le16(flag_table[filter->flags] | 6942 I40E_AQC_ADD_CLOUD_FLAGS_IPV6); 6943 else 6944 cld_filter.flags |= cpu_to_le16(flag_table[filter->flags] | 6945 I40E_AQC_ADD_CLOUD_FLAGS_IPV4); 6946 6947 if (add) 6948 ret = i40e_aq_add_cloud_filters(&pf->hw, filter->seid, 6949 &cld_filter, 1); 6950 else 6951 ret = i40e_aq_rem_cloud_filters(&pf->hw, filter->seid, 6952 &cld_filter, 1); 6953 if (ret) 6954 dev_dbg(&pf->pdev->dev, 6955 "Failed to %s cloud filter using l4 port %u, err %d aq_err %d\n", 6956 add ? "add" : "delete", filter->dst_port, ret, 6957 pf->hw.aq.asq_last_status); 6958 else 6959 dev_info(&pf->pdev->dev, 6960 "%s cloud filter for VSI: %d\n", 6961 add ? "Added" : "Deleted", filter->seid); 6962 return ret; 6963 } 6964 6965 /** 6966 * i40e_add_del_cloud_filter_big_buf - Add/del cloud filter using big_buf 6967 * @vsi: pointer to VSI 6968 * @filter: cloud filter rule 6969 * @add: if true, add, if false, delete 6970 * 6971 * Add or delete a cloud filter for a specific flow spec using big buffer. 6972 * Returns 0 if the filter were successfully added. 6973 **/ 6974 static int i40e_add_del_cloud_filter_big_buf(struct i40e_vsi *vsi, 6975 struct i40e_cloud_filter *filter, 6976 bool add) 6977 { 6978 struct i40e_aqc_cloud_filters_element_bb cld_filter; 6979 struct i40e_pf *pf = vsi->back; 6980 int ret; 6981 6982 /* Both (src/dst) valid mac_addr are not supported */ 6983 if ((is_valid_ether_addr(filter->dst_mac) && 6984 is_valid_ether_addr(filter->src_mac)) || 6985 (is_multicast_ether_addr(filter->dst_mac) && 6986 is_multicast_ether_addr(filter->src_mac))) 6987 return -EOPNOTSUPP; 6988 6989 /* Big buffer cloud filter needs 'L4 port' to be non-zero. Also, UDP 6990 * ports are not supported via big buffer now. 6991 */ 6992 if (!filter->dst_port || filter->ip_proto == IPPROTO_UDP) 6993 return -EOPNOTSUPP; 6994 6995 /* adding filter using src_port/src_ip is not supported at this stage */ 6996 if (filter->src_port || filter->src_ipv4 || 6997 !ipv6_addr_any(&filter->ip.v6.src_ip6)) 6998 return -EOPNOTSUPP; 6999 7000 /* copy element needed to add cloud filter from filter */ 7001 i40e_set_cld_element(filter, &cld_filter.element); 7002 7003 if (is_valid_ether_addr(filter->dst_mac) || 7004 is_valid_ether_addr(filter->src_mac) || 7005 is_multicast_ether_addr(filter->dst_mac) || 7006 is_multicast_ether_addr(filter->src_mac)) { 7007 /* MAC + IP : unsupported mode */ 7008 if (filter->dst_ipv4) 7009 return -EOPNOTSUPP; 7010 7011 /* since we validated that L4 port must be valid before 7012 * we get here, start with respective "flags" value 7013 * and update if vlan is present or not 7014 */ 7015 cld_filter.element.flags = 7016 cpu_to_le16(I40E_AQC_ADD_CLOUD_FILTER_MAC_PORT); 7017 7018 if (filter->vlan_id) { 7019 cld_filter.element.flags = 7020 cpu_to_le16(I40E_AQC_ADD_CLOUD_FILTER_MAC_VLAN_PORT); 7021 } 7022 7023 } else if (filter->dst_ipv4 || 7024 !ipv6_addr_any(&filter->ip.v6.dst_ip6)) { 7025 cld_filter.element.flags = 7026 cpu_to_le16(I40E_AQC_ADD_CLOUD_FILTER_IP_PORT); 7027 if (filter->n_proto == ETH_P_IPV6) 7028 cld_filter.element.flags |= 7029 cpu_to_le16(I40E_AQC_ADD_CLOUD_FLAGS_IPV6); 7030 else 7031 cld_filter.element.flags |= 7032 cpu_to_le16(I40E_AQC_ADD_CLOUD_FLAGS_IPV4); 7033 } else { 7034 dev_err(&pf->pdev->dev, 7035 "either mac or ip has to be valid for cloud filter\n"); 7036 return -EINVAL; 7037 } 7038 7039 /* Now copy L4 port in Byte 6..7 in general fields */ 7040 cld_filter.general_fields[I40E_AQC_ADD_CLOUD_FV_FLU_0X16_WORD0] = 7041 be16_to_cpu(filter->dst_port); 7042 7043 if (add) { 7044 /* Validate current device switch mode, change if necessary */ 7045 ret = i40e_validate_and_set_switch_mode(vsi); 7046 if (ret) { 7047 dev_err(&pf->pdev->dev, 7048 "failed to set switch mode, ret %d\n", 7049 ret); 7050 return ret; 7051 } 7052 7053 ret = i40e_aq_add_cloud_filters_bb(&pf->hw, filter->seid, 7054 &cld_filter, 1); 7055 } else { 7056 ret = i40e_aq_rem_cloud_filters_bb(&pf->hw, filter->seid, 7057 &cld_filter, 1); 7058 } 7059 7060 if (ret) 7061 dev_dbg(&pf->pdev->dev, 7062 "Failed to %s cloud filter(big buffer) err %d aq_err %d\n", 7063 add ? "add" : "delete", ret, pf->hw.aq.asq_last_status); 7064 else 7065 dev_info(&pf->pdev->dev, 7066 "%s cloud filter for VSI: %d, L4 port: %d\n", 7067 add ? "add" : "delete", filter->seid, 7068 ntohs(filter->dst_port)); 7069 return ret; 7070 } 7071 7072 /** 7073 * i40e_parse_cls_flower - Parse tc flower filters provided by kernel 7074 * @vsi: Pointer to VSI 7075 * @cls_flower: Pointer to struct tc_cls_flower_offload 7076 * @filter: Pointer to cloud filter structure 7077 * 7078 **/ 7079 static int i40e_parse_cls_flower(struct i40e_vsi *vsi, 7080 struct tc_cls_flower_offload *f, 7081 struct i40e_cloud_filter *filter) 7082 { 7083 u16 n_proto_mask = 0, n_proto_key = 0, addr_type = 0; 7084 struct i40e_pf *pf = vsi->back; 7085 u8 field_flags = 0; 7086 7087 if (f->dissector->used_keys & 7088 ~(BIT(FLOW_DISSECTOR_KEY_CONTROL) | 7089 BIT(FLOW_DISSECTOR_KEY_BASIC) | 7090 BIT(FLOW_DISSECTOR_KEY_ETH_ADDRS) | 7091 BIT(FLOW_DISSECTOR_KEY_VLAN) | 7092 BIT(FLOW_DISSECTOR_KEY_IPV4_ADDRS) | 7093 BIT(FLOW_DISSECTOR_KEY_IPV6_ADDRS) | 7094 BIT(FLOW_DISSECTOR_KEY_PORTS) | 7095 BIT(FLOW_DISSECTOR_KEY_ENC_KEYID))) { 7096 dev_err(&pf->pdev->dev, "Unsupported key used: 0x%x\n", 7097 f->dissector->used_keys); 7098 return -EOPNOTSUPP; 7099 } 7100 7101 if (dissector_uses_key(f->dissector, FLOW_DISSECTOR_KEY_ENC_KEYID)) { 7102 struct flow_dissector_key_keyid *key = 7103 skb_flow_dissector_target(f->dissector, 7104 FLOW_DISSECTOR_KEY_ENC_KEYID, 7105 f->key); 7106 7107 struct flow_dissector_key_keyid *mask = 7108 skb_flow_dissector_target(f->dissector, 7109 FLOW_DISSECTOR_KEY_ENC_KEYID, 7110 f->mask); 7111 7112 if (mask->keyid != 0) 7113 field_flags |= I40E_CLOUD_FIELD_TEN_ID; 7114 7115 filter->tenant_id = be32_to_cpu(key->keyid); 7116 } 7117 7118 if (dissector_uses_key(f->dissector, FLOW_DISSECTOR_KEY_BASIC)) { 7119 struct flow_dissector_key_basic *key = 7120 skb_flow_dissector_target(f->dissector, 7121 FLOW_DISSECTOR_KEY_BASIC, 7122 f->key); 7123 7124 struct flow_dissector_key_basic *mask = 7125 skb_flow_dissector_target(f->dissector, 7126 FLOW_DISSECTOR_KEY_BASIC, 7127 f->mask); 7128 7129 n_proto_key = ntohs(key->n_proto); 7130 n_proto_mask = ntohs(mask->n_proto); 7131 7132 if (n_proto_key == ETH_P_ALL) { 7133 n_proto_key = 0; 7134 n_proto_mask = 0; 7135 } 7136 filter->n_proto = n_proto_key & n_proto_mask; 7137 filter->ip_proto = key->ip_proto; 7138 } 7139 7140 if (dissector_uses_key(f->dissector, FLOW_DISSECTOR_KEY_ETH_ADDRS)) { 7141 struct flow_dissector_key_eth_addrs *key = 7142 skb_flow_dissector_target(f->dissector, 7143 FLOW_DISSECTOR_KEY_ETH_ADDRS, 7144 f->key); 7145 7146 struct flow_dissector_key_eth_addrs *mask = 7147 skb_flow_dissector_target(f->dissector, 7148 FLOW_DISSECTOR_KEY_ETH_ADDRS, 7149 f->mask); 7150 7151 /* use is_broadcast and is_zero to check for all 0xf or 0 */ 7152 if (!is_zero_ether_addr(mask->dst)) { 7153 if (is_broadcast_ether_addr(mask->dst)) { 7154 field_flags |= I40E_CLOUD_FIELD_OMAC; 7155 } else { 7156 dev_err(&pf->pdev->dev, "Bad ether dest mask %pM\n", 7157 mask->dst); 7158 return I40E_ERR_CONFIG; 7159 } 7160 } 7161 7162 if (!is_zero_ether_addr(mask->src)) { 7163 if (is_broadcast_ether_addr(mask->src)) { 7164 field_flags |= I40E_CLOUD_FIELD_IMAC; 7165 } else { 7166 dev_err(&pf->pdev->dev, "Bad ether src mask %pM\n", 7167 mask->src); 7168 return I40E_ERR_CONFIG; 7169 } 7170 } 7171 ether_addr_copy(filter->dst_mac, key->dst); 7172 ether_addr_copy(filter->src_mac, key->src); 7173 } 7174 7175 if (dissector_uses_key(f->dissector, FLOW_DISSECTOR_KEY_VLAN)) { 7176 struct flow_dissector_key_vlan *key = 7177 skb_flow_dissector_target(f->dissector, 7178 FLOW_DISSECTOR_KEY_VLAN, 7179 f->key); 7180 struct flow_dissector_key_vlan *mask = 7181 skb_flow_dissector_target(f->dissector, 7182 FLOW_DISSECTOR_KEY_VLAN, 7183 f->mask); 7184 7185 if (mask->vlan_id) { 7186 if (mask->vlan_id == VLAN_VID_MASK) { 7187 field_flags |= I40E_CLOUD_FIELD_IVLAN; 7188 7189 } else { 7190 dev_err(&pf->pdev->dev, "Bad vlan mask 0x%04x\n", 7191 mask->vlan_id); 7192 return I40E_ERR_CONFIG; 7193 } 7194 } 7195 7196 filter->vlan_id = cpu_to_be16(key->vlan_id); 7197 } 7198 7199 if (dissector_uses_key(f->dissector, FLOW_DISSECTOR_KEY_CONTROL)) { 7200 struct flow_dissector_key_control *key = 7201 skb_flow_dissector_target(f->dissector, 7202 FLOW_DISSECTOR_KEY_CONTROL, 7203 f->key); 7204 7205 addr_type = key->addr_type; 7206 } 7207 7208 if (addr_type == FLOW_DISSECTOR_KEY_IPV4_ADDRS) { 7209 struct flow_dissector_key_ipv4_addrs *key = 7210 skb_flow_dissector_target(f->dissector, 7211 FLOW_DISSECTOR_KEY_IPV4_ADDRS, 7212 f->key); 7213 struct flow_dissector_key_ipv4_addrs *mask = 7214 skb_flow_dissector_target(f->dissector, 7215 FLOW_DISSECTOR_KEY_IPV4_ADDRS, 7216 f->mask); 7217 7218 if (mask->dst) { 7219 if (mask->dst == cpu_to_be32(0xffffffff)) { 7220 field_flags |= I40E_CLOUD_FIELD_IIP; 7221 } else { 7222 mask->dst = be32_to_cpu(mask->dst); 7223 dev_err(&pf->pdev->dev, "Bad ip dst mask %pI4\n", 7224 &mask->dst); 7225 return I40E_ERR_CONFIG; 7226 } 7227 } 7228 7229 if (mask->src) { 7230 if (mask->src == cpu_to_be32(0xffffffff)) { 7231 field_flags |= I40E_CLOUD_FIELD_IIP; 7232 } else { 7233 mask->src = be32_to_cpu(mask->src); 7234 dev_err(&pf->pdev->dev, "Bad ip src mask %pI4\n", 7235 &mask->src); 7236 return I40E_ERR_CONFIG; 7237 } 7238 } 7239 7240 if (field_flags & I40E_CLOUD_FIELD_TEN_ID) { 7241 dev_err(&pf->pdev->dev, "Tenant id not allowed for ip filter\n"); 7242 return I40E_ERR_CONFIG; 7243 } 7244 filter->dst_ipv4 = key->dst; 7245 filter->src_ipv4 = key->src; 7246 } 7247 7248 if (addr_type == FLOW_DISSECTOR_KEY_IPV6_ADDRS) { 7249 struct flow_dissector_key_ipv6_addrs *key = 7250 skb_flow_dissector_target(f->dissector, 7251 FLOW_DISSECTOR_KEY_IPV6_ADDRS, 7252 f->key); 7253 struct flow_dissector_key_ipv6_addrs *mask = 7254 skb_flow_dissector_target(f->dissector, 7255 FLOW_DISSECTOR_KEY_IPV6_ADDRS, 7256 f->mask); 7257 7258 /* src and dest IPV6 address should not be LOOPBACK 7259 * (0:0:0:0:0:0:0:1), which can be represented as ::1 7260 */ 7261 if (ipv6_addr_loopback(&key->dst) || 7262 ipv6_addr_loopback(&key->src)) { 7263 dev_err(&pf->pdev->dev, 7264 "Bad ipv6, addr is LOOPBACK\n"); 7265 return I40E_ERR_CONFIG; 7266 } 7267 if (!ipv6_addr_any(&mask->dst) || !ipv6_addr_any(&mask->src)) 7268 field_flags |= I40E_CLOUD_FIELD_IIP; 7269 7270 memcpy(&filter->src_ipv6, &key->src.s6_addr32, 7271 sizeof(filter->src_ipv6)); 7272 memcpy(&filter->dst_ipv6, &key->dst.s6_addr32, 7273 sizeof(filter->dst_ipv6)); 7274 } 7275 7276 if (dissector_uses_key(f->dissector, FLOW_DISSECTOR_KEY_PORTS)) { 7277 struct flow_dissector_key_ports *key = 7278 skb_flow_dissector_target(f->dissector, 7279 FLOW_DISSECTOR_KEY_PORTS, 7280 f->key); 7281 struct flow_dissector_key_ports *mask = 7282 skb_flow_dissector_target(f->dissector, 7283 FLOW_DISSECTOR_KEY_PORTS, 7284 f->mask); 7285 7286 if (mask->src) { 7287 if (mask->src == cpu_to_be16(0xffff)) { 7288 field_flags |= I40E_CLOUD_FIELD_IIP; 7289 } else { 7290 dev_err(&pf->pdev->dev, "Bad src port mask 0x%04x\n", 7291 be16_to_cpu(mask->src)); 7292 return I40E_ERR_CONFIG; 7293 } 7294 } 7295 7296 if (mask->dst) { 7297 if (mask->dst == cpu_to_be16(0xffff)) { 7298 field_flags |= I40E_CLOUD_FIELD_IIP; 7299 } else { 7300 dev_err(&pf->pdev->dev, "Bad dst port mask 0x%04x\n", 7301 be16_to_cpu(mask->dst)); 7302 return I40E_ERR_CONFIG; 7303 } 7304 } 7305 7306 filter->dst_port = key->dst; 7307 filter->src_port = key->src; 7308 7309 switch (filter->ip_proto) { 7310 case IPPROTO_TCP: 7311 case IPPROTO_UDP: 7312 break; 7313 default: 7314 dev_err(&pf->pdev->dev, 7315 "Only UDP and TCP transport are supported\n"); 7316 return -EINVAL; 7317 } 7318 } 7319 filter->flags = field_flags; 7320 return 0; 7321 } 7322 7323 /** 7324 * i40e_handle_tclass: Forward to a traffic class on the device 7325 * @vsi: Pointer to VSI 7326 * @tc: traffic class index on the device 7327 * @filter: Pointer to cloud filter structure 7328 * 7329 **/ 7330 static int i40e_handle_tclass(struct i40e_vsi *vsi, u32 tc, 7331 struct i40e_cloud_filter *filter) 7332 { 7333 struct i40e_channel *ch, *ch_tmp; 7334 7335 /* direct to a traffic class on the same device */ 7336 if (tc == 0) { 7337 filter->seid = vsi->seid; 7338 return 0; 7339 } else if (vsi->tc_config.enabled_tc & BIT(tc)) { 7340 if (!filter->dst_port) { 7341 dev_err(&vsi->back->pdev->dev, 7342 "Specify destination port to direct to traffic class that is not default\n"); 7343 return -EINVAL; 7344 } 7345 if (list_empty(&vsi->ch_list)) 7346 return -EINVAL; 7347 list_for_each_entry_safe(ch, ch_tmp, &vsi->ch_list, 7348 list) { 7349 if (ch->seid == vsi->tc_seid_map[tc]) 7350 filter->seid = ch->seid; 7351 } 7352 return 0; 7353 } 7354 dev_err(&vsi->back->pdev->dev, "TC is not enabled\n"); 7355 return -EINVAL; 7356 } 7357 7358 /** 7359 * i40e_configure_clsflower - Configure tc flower filters 7360 * @vsi: Pointer to VSI 7361 * @cls_flower: Pointer to struct tc_cls_flower_offload 7362 * 7363 **/ 7364 static int i40e_configure_clsflower(struct i40e_vsi *vsi, 7365 struct tc_cls_flower_offload *cls_flower) 7366 { 7367 int tc = tc_classid_to_hwtc(vsi->netdev, cls_flower->classid); 7368 struct i40e_cloud_filter *filter = NULL; 7369 struct i40e_pf *pf = vsi->back; 7370 int err = 0; 7371 7372 if (tc < 0) { 7373 dev_err(&vsi->back->pdev->dev, "Invalid traffic class\n"); 7374 return -EOPNOTSUPP; 7375 } 7376 7377 if (test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state) || 7378 test_bit(__I40E_RESET_INTR_RECEIVED, pf->state)) 7379 return -EBUSY; 7380 7381 if (pf->fdir_pf_active_filters || 7382 (!hlist_empty(&pf->fdir_filter_list))) { 7383 dev_err(&vsi->back->pdev->dev, 7384 "Flow Director Sideband filters exists, turn ntuple off to configure cloud filters\n"); 7385 return -EINVAL; 7386 } 7387 7388 if (vsi->back->flags & I40E_FLAG_FD_SB_ENABLED) { 7389 dev_err(&vsi->back->pdev->dev, 7390 "Disable Flow Director Sideband, configuring Cloud filters via tc-flower\n"); 7391 vsi->back->flags &= ~I40E_FLAG_FD_SB_ENABLED; 7392 vsi->back->flags |= I40E_FLAG_FD_SB_TO_CLOUD_FILTER; 7393 } 7394 7395 filter = kzalloc(sizeof(*filter), GFP_KERNEL); 7396 if (!filter) 7397 return -ENOMEM; 7398 7399 filter->cookie = cls_flower->cookie; 7400 7401 err = i40e_parse_cls_flower(vsi, cls_flower, filter); 7402 if (err < 0) 7403 goto err; 7404 7405 err = i40e_handle_tclass(vsi, tc, filter); 7406 if (err < 0) 7407 goto err; 7408 7409 /* Add cloud filter */ 7410 if (filter->dst_port) 7411 err = i40e_add_del_cloud_filter_big_buf(vsi, filter, true); 7412 else 7413 err = i40e_add_del_cloud_filter(vsi, filter, true); 7414 7415 if (err) { 7416 dev_err(&pf->pdev->dev, 7417 "Failed to add cloud filter, err %s\n", 7418 i40e_stat_str(&pf->hw, err)); 7419 goto err; 7420 } 7421 7422 /* add filter to the ordered list */ 7423 INIT_HLIST_NODE(&filter->cloud_node); 7424 7425 hlist_add_head(&filter->cloud_node, &pf->cloud_filter_list); 7426 7427 pf->num_cloud_filters++; 7428 7429 return err; 7430 err: 7431 kfree(filter); 7432 return err; 7433 } 7434 7435 /** 7436 * i40e_find_cloud_filter - Find the could filter in the list 7437 * @vsi: Pointer to VSI 7438 * @cookie: filter specific cookie 7439 * 7440 **/ 7441 static struct i40e_cloud_filter *i40e_find_cloud_filter(struct i40e_vsi *vsi, 7442 unsigned long *cookie) 7443 { 7444 struct i40e_cloud_filter *filter = NULL; 7445 struct hlist_node *node2; 7446 7447 hlist_for_each_entry_safe(filter, node2, 7448 &vsi->back->cloud_filter_list, cloud_node) 7449 if (!memcmp(cookie, &filter->cookie, sizeof(filter->cookie))) 7450 return filter; 7451 return NULL; 7452 } 7453 7454 /** 7455 * i40e_delete_clsflower - Remove tc flower filters 7456 * @vsi: Pointer to VSI 7457 * @cls_flower: Pointer to struct tc_cls_flower_offload 7458 * 7459 **/ 7460 static int i40e_delete_clsflower(struct i40e_vsi *vsi, 7461 struct tc_cls_flower_offload *cls_flower) 7462 { 7463 struct i40e_cloud_filter *filter = NULL; 7464 struct i40e_pf *pf = vsi->back; 7465 int err = 0; 7466 7467 filter = i40e_find_cloud_filter(vsi, &cls_flower->cookie); 7468 7469 if (!filter) 7470 return -EINVAL; 7471 7472 hash_del(&filter->cloud_node); 7473 7474 if (filter->dst_port) 7475 err = i40e_add_del_cloud_filter_big_buf(vsi, filter, false); 7476 else 7477 err = i40e_add_del_cloud_filter(vsi, filter, false); 7478 7479 kfree(filter); 7480 if (err) { 7481 dev_err(&pf->pdev->dev, 7482 "Failed to delete cloud filter, err %s\n", 7483 i40e_stat_str(&pf->hw, err)); 7484 return i40e_aq_rc_to_posix(err, pf->hw.aq.asq_last_status); 7485 } 7486 7487 pf->num_cloud_filters--; 7488 if (!pf->num_cloud_filters) 7489 if ((pf->flags & I40E_FLAG_FD_SB_TO_CLOUD_FILTER) && 7490 !(pf->flags & I40E_FLAG_FD_SB_INACTIVE)) { 7491 pf->flags |= I40E_FLAG_FD_SB_ENABLED; 7492 pf->flags &= ~I40E_FLAG_FD_SB_TO_CLOUD_FILTER; 7493 pf->flags &= ~I40E_FLAG_FD_SB_INACTIVE; 7494 } 7495 return 0; 7496 } 7497 7498 /** 7499 * i40e_setup_tc_cls_flower - flower classifier offloads 7500 * @netdev: net device to configure 7501 * @type_data: offload data 7502 **/ 7503 static int i40e_setup_tc_cls_flower(struct i40e_netdev_priv *np, 7504 struct tc_cls_flower_offload *cls_flower) 7505 { 7506 struct i40e_vsi *vsi = np->vsi; 7507 7508 if (!tc_can_offload(vsi->netdev)) 7509 return -EOPNOTSUPP; 7510 if (cls_flower->common.chain_index) 7511 return -EOPNOTSUPP; 7512 7513 switch (cls_flower->command) { 7514 case TC_CLSFLOWER_REPLACE: 7515 return i40e_configure_clsflower(vsi, cls_flower); 7516 case TC_CLSFLOWER_DESTROY: 7517 return i40e_delete_clsflower(vsi, cls_flower); 7518 case TC_CLSFLOWER_STATS: 7519 return -EOPNOTSUPP; 7520 default: 7521 return -EINVAL; 7522 } 7523 } 7524 7525 static int i40e_setup_tc_block_cb(enum tc_setup_type type, void *type_data, 7526 void *cb_priv) 7527 { 7528 struct i40e_netdev_priv *np = cb_priv; 7529 7530 switch (type) { 7531 case TC_SETUP_CLSFLOWER: 7532 return i40e_setup_tc_cls_flower(np, type_data); 7533 7534 default: 7535 return -EOPNOTSUPP; 7536 } 7537 } 7538 7539 static int i40e_setup_tc_block(struct net_device *dev, 7540 struct tc_block_offload *f) 7541 { 7542 struct i40e_netdev_priv *np = netdev_priv(dev); 7543 7544 if (f->binder_type != TCF_BLOCK_BINDER_TYPE_CLSACT_INGRESS) 7545 return -EOPNOTSUPP; 7546 7547 switch (f->command) { 7548 case TC_BLOCK_BIND: 7549 return tcf_block_cb_register(f->block, i40e_setup_tc_block_cb, 7550 np, np); 7551 case TC_BLOCK_UNBIND: 7552 tcf_block_cb_unregister(f->block, i40e_setup_tc_block_cb, np); 7553 return 0; 7554 default: 7555 return -EOPNOTSUPP; 7556 } 7557 } 7558 7559 static int __i40e_setup_tc(struct net_device *netdev, enum tc_setup_type type, 7560 void *type_data) 7561 { 7562 switch (type) { 7563 case TC_SETUP_QDISC_MQPRIO: 7564 return i40e_setup_tc(netdev, type_data); 7565 case TC_SETUP_BLOCK: 7566 return i40e_setup_tc_block(netdev, type_data); 7567 default: 7568 return -EOPNOTSUPP; 7569 } 7570 } 7571 7572 /** 7573 * i40e_open - Called when a network interface is made active 7574 * @netdev: network interface device structure 7575 * 7576 * The open entry point is called when a network interface is made 7577 * active by the system (IFF_UP). At this point all resources needed 7578 * for transmit and receive operations are allocated, the interrupt 7579 * handler is registered with the OS, the netdev watchdog subtask is 7580 * enabled, and the stack is notified that the interface is ready. 7581 * 7582 * Returns 0 on success, negative value on failure 7583 **/ 7584 int i40e_open(struct net_device *netdev) 7585 { 7586 struct i40e_netdev_priv *np = netdev_priv(netdev); 7587 struct i40e_vsi *vsi = np->vsi; 7588 struct i40e_pf *pf = vsi->back; 7589 int err; 7590 7591 /* disallow open during test or if eeprom is broken */ 7592 if (test_bit(__I40E_TESTING, pf->state) || 7593 test_bit(__I40E_BAD_EEPROM, pf->state)) 7594 return -EBUSY; 7595 7596 netif_carrier_off(netdev); 7597 7598 err = i40e_vsi_open(vsi); 7599 if (err) 7600 return err; 7601 7602 /* configure global TSO hardware offload settings */ 7603 wr32(&pf->hw, I40E_GLLAN_TSOMSK_F, be32_to_cpu(TCP_FLAG_PSH | 7604 TCP_FLAG_FIN) >> 16); 7605 wr32(&pf->hw, I40E_GLLAN_TSOMSK_M, be32_to_cpu(TCP_FLAG_PSH | 7606 TCP_FLAG_FIN | 7607 TCP_FLAG_CWR) >> 16); 7608 wr32(&pf->hw, I40E_GLLAN_TSOMSK_L, be32_to_cpu(TCP_FLAG_CWR) >> 16); 7609 7610 udp_tunnel_get_rx_info(netdev); 7611 7612 return 0; 7613 } 7614 7615 /** 7616 * i40e_vsi_open - 7617 * @vsi: the VSI to open 7618 * 7619 * Finish initialization of the VSI. 7620 * 7621 * Returns 0 on success, negative value on failure 7622 * 7623 * Note: expects to be called while under rtnl_lock() 7624 **/ 7625 int i40e_vsi_open(struct i40e_vsi *vsi) 7626 { 7627 struct i40e_pf *pf = vsi->back; 7628 char int_name[I40E_INT_NAME_STR_LEN]; 7629 int err; 7630 7631 /* allocate descriptors */ 7632 err = i40e_vsi_setup_tx_resources(vsi); 7633 if (err) 7634 goto err_setup_tx; 7635 err = i40e_vsi_setup_rx_resources(vsi); 7636 if (err) 7637 goto err_setup_rx; 7638 7639 err = i40e_vsi_configure(vsi); 7640 if (err) 7641 goto err_setup_rx; 7642 7643 if (vsi->netdev) { 7644 snprintf(int_name, sizeof(int_name) - 1, "%s-%s", 7645 dev_driver_string(&pf->pdev->dev), vsi->netdev->name); 7646 err = i40e_vsi_request_irq(vsi, int_name); 7647 if (err) 7648 goto err_setup_rx; 7649 7650 /* Notify the stack of the actual queue counts. */ 7651 err = netif_set_real_num_tx_queues(vsi->netdev, 7652 vsi->num_queue_pairs); 7653 if (err) 7654 goto err_set_queues; 7655 7656 err = netif_set_real_num_rx_queues(vsi->netdev, 7657 vsi->num_queue_pairs); 7658 if (err) 7659 goto err_set_queues; 7660 7661 } else if (vsi->type == I40E_VSI_FDIR) { 7662 snprintf(int_name, sizeof(int_name) - 1, "%s-%s:fdir", 7663 dev_driver_string(&pf->pdev->dev), 7664 dev_name(&pf->pdev->dev)); 7665 err = i40e_vsi_request_irq(vsi, int_name); 7666 7667 } else { 7668 err = -EINVAL; 7669 goto err_setup_rx; 7670 } 7671 7672 err = i40e_up_complete(vsi); 7673 if (err) 7674 goto err_up_complete; 7675 7676 return 0; 7677 7678 err_up_complete: 7679 i40e_down(vsi); 7680 err_set_queues: 7681 i40e_vsi_free_irq(vsi); 7682 err_setup_rx: 7683 i40e_vsi_free_rx_resources(vsi); 7684 err_setup_tx: 7685 i40e_vsi_free_tx_resources(vsi); 7686 if (vsi == pf->vsi[pf->lan_vsi]) 7687 i40e_do_reset(pf, I40E_PF_RESET_FLAG, true); 7688 7689 return err; 7690 } 7691 7692 /** 7693 * i40e_fdir_filter_exit - Cleans up the Flow Director accounting 7694 * @pf: Pointer to PF 7695 * 7696 * This function destroys the hlist where all the Flow Director 7697 * filters were saved. 7698 **/ 7699 static void i40e_fdir_filter_exit(struct i40e_pf *pf) 7700 { 7701 struct i40e_fdir_filter *filter; 7702 struct i40e_flex_pit *pit_entry, *tmp; 7703 struct hlist_node *node2; 7704 7705 hlist_for_each_entry_safe(filter, node2, 7706 &pf->fdir_filter_list, fdir_node) { 7707 hlist_del(&filter->fdir_node); 7708 kfree(filter); 7709 } 7710 7711 list_for_each_entry_safe(pit_entry, tmp, &pf->l3_flex_pit_list, list) { 7712 list_del(&pit_entry->list); 7713 kfree(pit_entry); 7714 } 7715 INIT_LIST_HEAD(&pf->l3_flex_pit_list); 7716 7717 list_for_each_entry_safe(pit_entry, tmp, &pf->l4_flex_pit_list, list) { 7718 list_del(&pit_entry->list); 7719 kfree(pit_entry); 7720 } 7721 INIT_LIST_HEAD(&pf->l4_flex_pit_list); 7722 7723 pf->fdir_pf_active_filters = 0; 7724 pf->fd_tcp4_filter_cnt = 0; 7725 pf->fd_udp4_filter_cnt = 0; 7726 pf->fd_sctp4_filter_cnt = 0; 7727 pf->fd_ip4_filter_cnt = 0; 7728 7729 /* Reprogram the default input set for TCP/IPv4 */ 7730 i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV4_TCP, 7731 I40E_L3_SRC_MASK | I40E_L3_DST_MASK | 7732 I40E_L4_SRC_MASK | I40E_L4_DST_MASK); 7733 7734 /* Reprogram the default input set for UDP/IPv4 */ 7735 i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV4_UDP, 7736 I40E_L3_SRC_MASK | I40E_L3_DST_MASK | 7737 I40E_L4_SRC_MASK | I40E_L4_DST_MASK); 7738 7739 /* Reprogram the default input set for SCTP/IPv4 */ 7740 i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV4_SCTP, 7741 I40E_L3_SRC_MASK | I40E_L3_DST_MASK | 7742 I40E_L4_SRC_MASK | I40E_L4_DST_MASK); 7743 7744 /* Reprogram the default input set for Other/IPv4 */ 7745 i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV4_OTHER, 7746 I40E_L3_SRC_MASK | I40E_L3_DST_MASK); 7747 } 7748 7749 /** 7750 * i40e_cloud_filter_exit - Cleans up the cloud filters 7751 * @pf: Pointer to PF 7752 * 7753 * This function destroys the hlist where all the cloud filters 7754 * were saved. 7755 **/ 7756 static void i40e_cloud_filter_exit(struct i40e_pf *pf) 7757 { 7758 struct i40e_cloud_filter *cfilter; 7759 struct hlist_node *node; 7760 7761 hlist_for_each_entry_safe(cfilter, node, 7762 &pf->cloud_filter_list, cloud_node) { 7763 hlist_del(&cfilter->cloud_node); 7764 kfree(cfilter); 7765 } 7766 pf->num_cloud_filters = 0; 7767 7768 if ((pf->flags & I40E_FLAG_FD_SB_TO_CLOUD_FILTER) && 7769 !(pf->flags & I40E_FLAG_FD_SB_INACTIVE)) { 7770 pf->flags |= I40E_FLAG_FD_SB_ENABLED; 7771 pf->flags &= ~I40E_FLAG_FD_SB_TO_CLOUD_FILTER; 7772 pf->flags &= ~I40E_FLAG_FD_SB_INACTIVE; 7773 } 7774 } 7775 7776 /** 7777 * i40e_close - Disables a network interface 7778 * @netdev: network interface device structure 7779 * 7780 * The close entry point is called when an interface is de-activated 7781 * by the OS. The hardware is still under the driver's control, but 7782 * this netdev interface is disabled. 7783 * 7784 * Returns 0, this is not allowed to fail 7785 **/ 7786 int i40e_close(struct net_device *netdev) 7787 { 7788 struct i40e_netdev_priv *np = netdev_priv(netdev); 7789 struct i40e_vsi *vsi = np->vsi; 7790 7791 i40e_vsi_close(vsi); 7792 7793 return 0; 7794 } 7795 7796 /** 7797 * i40e_do_reset - Start a PF or Core Reset sequence 7798 * @pf: board private structure 7799 * @reset_flags: which reset is requested 7800 * @lock_acquired: indicates whether or not the lock has been acquired 7801 * before this function was called. 7802 * 7803 * The essential difference in resets is that the PF Reset 7804 * doesn't clear the packet buffers, doesn't reset the PE 7805 * firmware, and doesn't bother the other PFs on the chip. 7806 **/ 7807 void i40e_do_reset(struct i40e_pf *pf, u32 reset_flags, bool lock_acquired) 7808 { 7809 u32 val; 7810 7811 WARN_ON(in_interrupt()); 7812 7813 7814 /* do the biggest reset indicated */ 7815 if (reset_flags & BIT_ULL(__I40E_GLOBAL_RESET_REQUESTED)) { 7816 7817 /* Request a Global Reset 7818 * 7819 * This will start the chip's countdown to the actual full 7820 * chip reset event, and a warning interrupt to be sent 7821 * to all PFs, including the requestor. Our handler 7822 * for the warning interrupt will deal with the shutdown 7823 * and recovery of the switch setup. 7824 */ 7825 dev_dbg(&pf->pdev->dev, "GlobalR requested\n"); 7826 val = rd32(&pf->hw, I40E_GLGEN_RTRIG); 7827 val |= I40E_GLGEN_RTRIG_GLOBR_MASK; 7828 wr32(&pf->hw, I40E_GLGEN_RTRIG, val); 7829 7830 } else if (reset_flags & BIT_ULL(__I40E_CORE_RESET_REQUESTED)) { 7831 7832 /* Request a Core Reset 7833 * 7834 * Same as Global Reset, except does *not* include the MAC/PHY 7835 */ 7836 dev_dbg(&pf->pdev->dev, "CoreR requested\n"); 7837 val = rd32(&pf->hw, I40E_GLGEN_RTRIG); 7838 val |= I40E_GLGEN_RTRIG_CORER_MASK; 7839 wr32(&pf->hw, I40E_GLGEN_RTRIG, val); 7840 i40e_flush(&pf->hw); 7841 7842 } else if (reset_flags & I40E_PF_RESET_FLAG) { 7843 7844 /* Request a PF Reset 7845 * 7846 * Resets only the PF-specific registers 7847 * 7848 * This goes directly to the tear-down and rebuild of 7849 * the switch, since we need to do all the recovery as 7850 * for the Core Reset. 7851 */ 7852 dev_dbg(&pf->pdev->dev, "PFR requested\n"); 7853 i40e_handle_reset_warning(pf, lock_acquired); 7854 7855 } else if (reset_flags & BIT_ULL(__I40E_REINIT_REQUESTED)) { 7856 int v; 7857 7858 /* Find the VSI(s) that requested a re-init */ 7859 dev_info(&pf->pdev->dev, 7860 "VSI reinit requested\n"); 7861 for (v = 0; v < pf->num_alloc_vsi; v++) { 7862 struct i40e_vsi *vsi = pf->vsi[v]; 7863 7864 if (vsi != NULL && 7865 test_and_clear_bit(__I40E_VSI_REINIT_REQUESTED, 7866 vsi->state)) 7867 i40e_vsi_reinit_locked(pf->vsi[v]); 7868 } 7869 } else if (reset_flags & BIT_ULL(__I40E_DOWN_REQUESTED)) { 7870 int v; 7871 7872 /* Find the VSI(s) that needs to be brought down */ 7873 dev_info(&pf->pdev->dev, "VSI down requested\n"); 7874 for (v = 0; v < pf->num_alloc_vsi; v++) { 7875 struct i40e_vsi *vsi = pf->vsi[v]; 7876 7877 if (vsi != NULL && 7878 test_and_clear_bit(__I40E_VSI_DOWN_REQUESTED, 7879 vsi->state)) { 7880 set_bit(__I40E_VSI_DOWN, vsi->state); 7881 i40e_down(vsi); 7882 } 7883 } 7884 } else { 7885 dev_info(&pf->pdev->dev, 7886 "bad reset request 0x%08x\n", reset_flags); 7887 } 7888 } 7889 7890 #ifdef CONFIG_I40E_DCB 7891 /** 7892 * i40e_dcb_need_reconfig - Check if DCB needs reconfig 7893 * @pf: board private structure 7894 * @old_cfg: current DCB config 7895 * @new_cfg: new DCB config 7896 **/ 7897 bool i40e_dcb_need_reconfig(struct i40e_pf *pf, 7898 struct i40e_dcbx_config *old_cfg, 7899 struct i40e_dcbx_config *new_cfg) 7900 { 7901 bool need_reconfig = false; 7902 7903 /* Check if ETS configuration has changed */ 7904 if (memcmp(&new_cfg->etscfg, 7905 &old_cfg->etscfg, 7906 sizeof(new_cfg->etscfg))) { 7907 /* If Priority Table has changed reconfig is needed */ 7908 if (memcmp(&new_cfg->etscfg.prioritytable, 7909 &old_cfg->etscfg.prioritytable, 7910 sizeof(new_cfg->etscfg.prioritytable))) { 7911 need_reconfig = true; 7912 dev_dbg(&pf->pdev->dev, "ETS UP2TC changed.\n"); 7913 } 7914 7915 if (memcmp(&new_cfg->etscfg.tcbwtable, 7916 &old_cfg->etscfg.tcbwtable, 7917 sizeof(new_cfg->etscfg.tcbwtable))) 7918 dev_dbg(&pf->pdev->dev, "ETS TC BW Table changed.\n"); 7919 7920 if (memcmp(&new_cfg->etscfg.tsatable, 7921 &old_cfg->etscfg.tsatable, 7922 sizeof(new_cfg->etscfg.tsatable))) 7923 dev_dbg(&pf->pdev->dev, "ETS TSA Table changed.\n"); 7924 } 7925 7926 /* Check if PFC configuration has changed */ 7927 if (memcmp(&new_cfg->pfc, 7928 &old_cfg->pfc, 7929 sizeof(new_cfg->pfc))) { 7930 need_reconfig = true; 7931 dev_dbg(&pf->pdev->dev, "PFC config change detected.\n"); 7932 } 7933 7934 /* Check if APP Table has changed */ 7935 if (memcmp(&new_cfg->app, 7936 &old_cfg->app, 7937 sizeof(new_cfg->app))) { 7938 need_reconfig = true; 7939 dev_dbg(&pf->pdev->dev, "APP Table change detected.\n"); 7940 } 7941 7942 dev_dbg(&pf->pdev->dev, "dcb need_reconfig=%d\n", need_reconfig); 7943 return need_reconfig; 7944 } 7945 7946 /** 7947 * i40e_handle_lldp_event - Handle LLDP Change MIB event 7948 * @pf: board private structure 7949 * @e: event info posted on ARQ 7950 **/ 7951 static int i40e_handle_lldp_event(struct i40e_pf *pf, 7952 struct i40e_arq_event_info *e) 7953 { 7954 struct i40e_aqc_lldp_get_mib *mib = 7955 (struct i40e_aqc_lldp_get_mib *)&e->desc.params.raw; 7956 struct i40e_hw *hw = &pf->hw; 7957 struct i40e_dcbx_config tmp_dcbx_cfg; 7958 bool need_reconfig = false; 7959 int ret = 0; 7960 u8 type; 7961 7962 /* Not DCB capable or capability disabled */ 7963 if (!(pf->flags & I40E_FLAG_DCB_CAPABLE)) 7964 return ret; 7965 7966 /* Ignore if event is not for Nearest Bridge */ 7967 type = ((mib->type >> I40E_AQ_LLDP_BRIDGE_TYPE_SHIFT) 7968 & I40E_AQ_LLDP_BRIDGE_TYPE_MASK); 7969 dev_dbg(&pf->pdev->dev, "LLDP event mib bridge type 0x%x\n", type); 7970 if (type != I40E_AQ_LLDP_BRIDGE_TYPE_NEAREST_BRIDGE) 7971 return ret; 7972 7973 /* Check MIB Type and return if event for Remote MIB update */ 7974 type = mib->type & I40E_AQ_LLDP_MIB_TYPE_MASK; 7975 dev_dbg(&pf->pdev->dev, 7976 "LLDP event mib type %s\n", type ? "remote" : "local"); 7977 if (type == I40E_AQ_LLDP_MIB_REMOTE) { 7978 /* Update the remote cached instance and return */ 7979 ret = i40e_aq_get_dcb_config(hw, I40E_AQ_LLDP_MIB_REMOTE, 7980 I40E_AQ_LLDP_BRIDGE_TYPE_NEAREST_BRIDGE, 7981 &hw->remote_dcbx_config); 7982 goto exit; 7983 } 7984 7985 /* Store the old configuration */ 7986 tmp_dcbx_cfg = hw->local_dcbx_config; 7987 7988 /* Reset the old DCBx configuration data */ 7989 memset(&hw->local_dcbx_config, 0, sizeof(hw->local_dcbx_config)); 7990 /* Get updated DCBX data from firmware */ 7991 ret = i40e_get_dcb_config(&pf->hw); 7992 if (ret) { 7993 dev_info(&pf->pdev->dev, 7994 "Failed querying DCB configuration data from firmware, err %s aq_err %s\n", 7995 i40e_stat_str(&pf->hw, ret), 7996 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 7997 goto exit; 7998 } 7999 8000 /* No change detected in DCBX configs */ 8001 if (!memcmp(&tmp_dcbx_cfg, &hw->local_dcbx_config, 8002 sizeof(tmp_dcbx_cfg))) { 8003 dev_dbg(&pf->pdev->dev, "No change detected in DCBX configuration.\n"); 8004 goto exit; 8005 } 8006 8007 need_reconfig = i40e_dcb_need_reconfig(pf, &tmp_dcbx_cfg, 8008 &hw->local_dcbx_config); 8009 8010 i40e_dcbnl_flush_apps(pf, &tmp_dcbx_cfg, &hw->local_dcbx_config); 8011 8012 if (!need_reconfig) 8013 goto exit; 8014 8015 /* Enable DCB tagging only when more than one TC */ 8016 if (i40e_dcb_get_num_tc(&hw->local_dcbx_config) > 1) 8017 pf->flags |= I40E_FLAG_DCB_ENABLED; 8018 else 8019 pf->flags &= ~I40E_FLAG_DCB_ENABLED; 8020 8021 set_bit(__I40E_PORT_SUSPENDED, pf->state); 8022 /* Reconfiguration needed quiesce all VSIs */ 8023 i40e_pf_quiesce_all_vsi(pf); 8024 8025 /* Changes in configuration update VEB/VSI */ 8026 i40e_dcb_reconfigure(pf); 8027 8028 ret = i40e_resume_port_tx(pf); 8029 8030 clear_bit(__I40E_PORT_SUSPENDED, pf->state); 8031 /* In case of error no point in resuming VSIs */ 8032 if (ret) 8033 goto exit; 8034 8035 /* Wait for the PF's queues to be disabled */ 8036 ret = i40e_pf_wait_queues_disabled(pf); 8037 if (ret) { 8038 /* Schedule PF reset to recover */ 8039 set_bit(__I40E_PF_RESET_REQUESTED, pf->state); 8040 i40e_service_event_schedule(pf); 8041 } else { 8042 i40e_pf_unquiesce_all_vsi(pf); 8043 pf->flags |= (I40E_FLAG_SERVICE_CLIENT_REQUESTED | 8044 I40E_FLAG_CLIENT_L2_CHANGE); 8045 } 8046 8047 exit: 8048 return ret; 8049 } 8050 #endif /* CONFIG_I40E_DCB */ 8051 8052 /** 8053 * i40e_do_reset_safe - Protected reset path for userland calls. 8054 * @pf: board private structure 8055 * @reset_flags: which reset is requested 8056 * 8057 **/ 8058 void i40e_do_reset_safe(struct i40e_pf *pf, u32 reset_flags) 8059 { 8060 rtnl_lock(); 8061 i40e_do_reset(pf, reset_flags, true); 8062 rtnl_unlock(); 8063 } 8064 8065 /** 8066 * i40e_handle_lan_overflow_event - Handler for LAN queue overflow event 8067 * @pf: board private structure 8068 * @e: event info posted on ARQ 8069 * 8070 * Handler for LAN Queue Overflow Event generated by the firmware for PF 8071 * and VF queues 8072 **/ 8073 static void i40e_handle_lan_overflow_event(struct i40e_pf *pf, 8074 struct i40e_arq_event_info *e) 8075 { 8076 struct i40e_aqc_lan_overflow *data = 8077 (struct i40e_aqc_lan_overflow *)&e->desc.params.raw; 8078 u32 queue = le32_to_cpu(data->prtdcb_rupto); 8079 u32 qtx_ctl = le32_to_cpu(data->otx_ctl); 8080 struct i40e_hw *hw = &pf->hw; 8081 struct i40e_vf *vf; 8082 u16 vf_id; 8083 8084 dev_dbg(&pf->pdev->dev, "overflow Rx Queue Number = %d QTX_CTL=0x%08x\n", 8085 queue, qtx_ctl); 8086 8087 /* Queue belongs to VF, find the VF and issue VF reset */ 8088 if (((qtx_ctl & I40E_QTX_CTL_PFVF_Q_MASK) 8089 >> I40E_QTX_CTL_PFVF_Q_SHIFT) == I40E_QTX_CTL_VF_QUEUE) { 8090 vf_id = (u16)((qtx_ctl & I40E_QTX_CTL_VFVM_INDX_MASK) 8091 >> I40E_QTX_CTL_VFVM_INDX_SHIFT); 8092 vf_id -= hw->func_caps.vf_base_id; 8093 vf = &pf->vf[vf_id]; 8094 i40e_vc_notify_vf_reset(vf); 8095 /* Allow VF to process pending reset notification */ 8096 msleep(20); 8097 i40e_reset_vf(vf, false); 8098 } 8099 } 8100 8101 /** 8102 * i40e_get_cur_guaranteed_fd_count - Get the consumed guaranteed FD filters 8103 * @pf: board private structure 8104 **/ 8105 u32 i40e_get_cur_guaranteed_fd_count(struct i40e_pf *pf) 8106 { 8107 u32 val, fcnt_prog; 8108 8109 val = rd32(&pf->hw, I40E_PFQF_FDSTAT); 8110 fcnt_prog = (val & I40E_PFQF_FDSTAT_GUARANT_CNT_MASK); 8111 return fcnt_prog; 8112 } 8113 8114 /** 8115 * i40e_get_current_fd_count - Get total FD filters programmed for this PF 8116 * @pf: board private structure 8117 **/ 8118 u32 i40e_get_current_fd_count(struct i40e_pf *pf) 8119 { 8120 u32 val, fcnt_prog; 8121 8122 val = rd32(&pf->hw, I40E_PFQF_FDSTAT); 8123 fcnt_prog = (val & I40E_PFQF_FDSTAT_GUARANT_CNT_MASK) + 8124 ((val & I40E_PFQF_FDSTAT_BEST_CNT_MASK) >> 8125 I40E_PFQF_FDSTAT_BEST_CNT_SHIFT); 8126 return fcnt_prog; 8127 } 8128 8129 /** 8130 * i40e_get_global_fd_count - Get total FD filters programmed on device 8131 * @pf: board private structure 8132 **/ 8133 u32 i40e_get_global_fd_count(struct i40e_pf *pf) 8134 { 8135 u32 val, fcnt_prog; 8136 8137 val = rd32(&pf->hw, I40E_GLQF_FDCNT_0); 8138 fcnt_prog = (val & I40E_GLQF_FDCNT_0_GUARANT_CNT_MASK) + 8139 ((val & I40E_GLQF_FDCNT_0_BESTCNT_MASK) >> 8140 I40E_GLQF_FDCNT_0_BESTCNT_SHIFT); 8141 return fcnt_prog; 8142 } 8143 8144 /** 8145 * i40e_fdir_check_and_reenable - Function to reenabe FD ATR or SB if disabled 8146 * @pf: board private structure 8147 **/ 8148 void i40e_fdir_check_and_reenable(struct i40e_pf *pf) 8149 { 8150 struct i40e_fdir_filter *filter; 8151 u32 fcnt_prog, fcnt_avail; 8152 struct hlist_node *node; 8153 8154 if (test_bit(__I40E_FD_FLUSH_REQUESTED, pf->state)) 8155 return; 8156 8157 /* Check if we have enough room to re-enable FDir SB capability. */ 8158 fcnt_prog = i40e_get_global_fd_count(pf); 8159 fcnt_avail = pf->fdir_pf_filter_count; 8160 if ((fcnt_prog < (fcnt_avail - I40E_FDIR_BUFFER_HEAD_ROOM)) || 8161 (pf->fd_add_err == 0) || 8162 (i40e_get_current_atr_cnt(pf) < pf->fd_atr_cnt)) { 8163 if (pf->flags & I40E_FLAG_FD_SB_AUTO_DISABLED) { 8164 pf->flags &= ~I40E_FLAG_FD_SB_AUTO_DISABLED; 8165 if ((pf->flags & I40E_FLAG_FD_SB_ENABLED) && 8166 (I40E_DEBUG_FD & pf->hw.debug_mask)) 8167 dev_info(&pf->pdev->dev, "FD Sideband/ntuple is being enabled since we have space in the table now\n"); 8168 } 8169 } 8170 8171 /* We should wait for even more space before re-enabling ATR. 8172 * Additionally, we cannot enable ATR as long as we still have TCP SB 8173 * rules active. 8174 */ 8175 if ((fcnt_prog < (fcnt_avail - I40E_FDIR_BUFFER_HEAD_ROOM_FOR_ATR)) && 8176 (pf->fd_tcp4_filter_cnt == 0)) { 8177 if (pf->flags & I40E_FLAG_FD_ATR_AUTO_DISABLED) { 8178 pf->flags &= ~I40E_FLAG_FD_ATR_AUTO_DISABLED; 8179 if ((pf->flags & I40E_FLAG_FD_ATR_ENABLED) && 8180 (I40E_DEBUG_FD & pf->hw.debug_mask)) 8181 dev_info(&pf->pdev->dev, "ATR is being enabled since we have space in the table and there are no conflicting ntuple rules\n"); 8182 } 8183 } 8184 8185 /* if hw had a problem adding a filter, delete it */ 8186 if (pf->fd_inv > 0) { 8187 hlist_for_each_entry_safe(filter, node, 8188 &pf->fdir_filter_list, fdir_node) { 8189 if (filter->fd_id == pf->fd_inv) { 8190 hlist_del(&filter->fdir_node); 8191 kfree(filter); 8192 pf->fdir_pf_active_filters--; 8193 pf->fd_inv = 0; 8194 } 8195 } 8196 } 8197 } 8198 8199 #define I40E_MIN_FD_FLUSH_INTERVAL 10 8200 #define I40E_MIN_FD_FLUSH_SB_ATR_UNSTABLE 30 8201 /** 8202 * i40e_fdir_flush_and_replay - Function to flush all FD filters and replay SB 8203 * @pf: board private structure 8204 **/ 8205 static void i40e_fdir_flush_and_replay(struct i40e_pf *pf) 8206 { 8207 unsigned long min_flush_time; 8208 int flush_wait_retry = 50; 8209 bool disable_atr = false; 8210 int fd_room; 8211 int reg; 8212 8213 if (!time_after(jiffies, pf->fd_flush_timestamp + 8214 (I40E_MIN_FD_FLUSH_INTERVAL * HZ))) 8215 return; 8216 8217 /* If the flush is happening too quick and we have mostly SB rules we 8218 * should not re-enable ATR for some time. 8219 */ 8220 min_flush_time = pf->fd_flush_timestamp + 8221 (I40E_MIN_FD_FLUSH_SB_ATR_UNSTABLE * HZ); 8222 fd_room = pf->fdir_pf_filter_count - pf->fdir_pf_active_filters; 8223 8224 if (!(time_after(jiffies, min_flush_time)) && 8225 (fd_room < I40E_FDIR_BUFFER_HEAD_ROOM_FOR_ATR)) { 8226 if (I40E_DEBUG_FD & pf->hw.debug_mask) 8227 dev_info(&pf->pdev->dev, "ATR disabled, not enough FD filter space.\n"); 8228 disable_atr = true; 8229 } 8230 8231 pf->fd_flush_timestamp = jiffies; 8232 pf->flags |= I40E_FLAG_FD_ATR_AUTO_DISABLED; 8233 /* flush all filters */ 8234 wr32(&pf->hw, I40E_PFQF_CTL_1, 8235 I40E_PFQF_CTL_1_CLEARFDTABLE_MASK); 8236 i40e_flush(&pf->hw); 8237 pf->fd_flush_cnt++; 8238 pf->fd_add_err = 0; 8239 do { 8240 /* Check FD flush status every 5-6msec */ 8241 usleep_range(5000, 6000); 8242 reg = rd32(&pf->hw, I40E_PFQF_CTL_1); 8243 if (!(reg & I40E_PFQF_CTL_1_CLEARFDTABLE_MASK)) 8244 break; 8245 } while (flush_wait_retry--); 8246 if (reg & I40E_PFQF_CTL_1_CLEARFDTABLE_MASK) { 8247 dev_warn(&pf->pdev->dev, "FD table did not flush, needs more time\n"); 8248 } else { 8249 /* replay sideband filters */ 8250 i40e_fdir_filter_restore(pf->vsi[pf->lan_vsi]); 8251 if (!disable_atr && !pf->fd_tcp4_filter_cnt) 8252 pf->flags &= ~I40E_FLAG_FD_ATR_AUTO_DISABLED; 8253 clear_bit(__I40E_FD_FLUSH_REQUESTED, pf->state); 8254 if (I40E_DEBUG_FD & pf->hw.debug_mask) 8255 dev_info(&pf->pdev->dev, "FD Filter table flushed and FD-SB replayed.\n"); 8256 } 8257 } 8258 8259 /** 8260 * i40e_get_current_atr_count - Get the count of total FD ATR filters programmed 8261 * @pf: board private structure 8262 **/ 8263 u32 i40e_get_current_atr_cnt(struct i40e_pf *pf) 8264 { 8265 return i40e_get_current_fd_count(pf) - pf->fdir_pf_active_filters; 8266 } 8267 8268 /* We can see up to 256 filter programming desc in transit if the filters are 8269 * being applied really fast; before we see the first 8270 * filter miss error on Rx queue 0. Accumulating enough error messages before 8271 * reacting will make sure we don't cause flush too often. 8272 */ 8273 #define I40E_MAX_FD_PROGRAM_ERROR 256 8274 8275 /** 8276 * i40e_fdir_reinit_subtask - Worker thread to reinit FDIR filter table 8277 * @pf: board private structure 8278 **/ 8279 static void i40e_fdir_reinit_subtask(struct i40e_pf *pf) 8280 { 8281 8282 /* if interface is down do nothing */ 8283 if (test_bit(__I40E_DOWN, pf->state)) 8284 return; 8285 8286 if (test_bit(__I40E_FD_FLUSH_REQUESTED, pf->state)) 8287 i40e_fdir_flush_and_replay(pf); 8288 8289 i40e_fdir_check_and_reenable(pf); 8290 8291 } 8292 8293 /** 8294 * i40e_vsi_link_event - notify VSI of a link event 8295 * @vsi: vsi to be notified 8296 * @link_up: link up or down 8297 **/ 8298 static void i40e_vsi_link_event(struct i40e_vsi *vsi, bool link_up) 8299 { 8300 if (!vsi || test_bit(__I40E_VSI_DOWN, vsi->state)) 8301 return; 8302 8303 switch (vsi->type) { 8304 case I40E_VSI_MAIN: 8305 if (!vsi->netdev || !vsi->netdev_registered) 8306 break; 8307 8308 if (link_up) { 8309 netif_carrier_on(vsi->netdev); 8310 netif_tx_wake_all_queues(vsi->netdev); 8311 } else { 8312 netif_carrier_off(vsi->netdev); 8313 netif_tx_stop_all_queues(vsi->netdev); 8314 } 8315 break; 8316 8317 case I40E_VSI_SRIOV: 8318 case I40E_VSI_VMDQ2: 8319 case I40E_VSI_CTRL: 8320 case I40E_VSI_IWARP: 8321 case I40E_VSI_MIRROR: 8322 default: 8323 /* there is no notification for other VSIs */ 8324 break; 8325 } 8326 } 8327 8328 /** 8329 * i40e_veb_link_event - notify elements on the veb of a link event 8330 * @veb: veb to be notified 8331 * @link_up: link up or down 8332 **/ 8333 static void i40e_veb_link_event(struct i40e_veb *veb, bool link_up) 8334 { 8335 struct i40e_pf *pf; 8336 int i; 8337 8338 if (!veb || !veb->pf) 8339 return; 8340 pf = veb->pf; 8341 8342 /* depth first... */ 8343 for (i = 0; i < I40E_MAX_VEB; i++) 8344 if (pf->veb[i] && (pf->veb[i]->uplink_seid == veb->seid)) 8345 i40e_veb_link_event(pf->veb[i], link_up); 8346 8347 /* ... now the local VSIs */ 8348 for (i = 0; i < pf->num_alloc_vsi; i++) 8349 if (pf->vsi[i] && (pf->vsi[i]->uplink_seid == veb->seid)) 8350 i40e_vsi_link_event(pf->vsi[i], link_up); 8351 } 8352 8353 /** 8354 * i40e_link_event - Update netif_carrier status 8355 * @pf: board private structure 8356 **/ 8357 static void i40e_link_event(struct i40e_pf *pf) 8358 { 8359 struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi]; 8360 u8 new_link_speed, old_link_speed; 8361 i40e_status status; 8362 bool new_link, old_link; 8363 8364 /* save off old link status information */ 8365 pf->hw.phy.link_info_old = pf->hw.phy.link_info; 8366 8367 /* set this to force the get_link_status call to refresh state */ 8368 pf->hw.phy.get_link_info = true; 8369 8370 old_link = (pf->hw.phy.link_info_old.link_info & I40E_AQ_LINK_UP); 8371 8372 status = i40e_get_link_status(&pf->hw, &new_link); 8373 8374 /* On success, disable temp link polling */ 8375 if (status == I40E_SUCCESS) { 8376 if (pf->flags & I40E_FLAG_TEMP_LINK_POLLING) 8377 pf->flags &= ~I40E_FLAG_TEMP_LINK_POLLING; 8378 } else { 8379 /* Enable link polling temporarily until i40e_get_link_status 8380 * returns I40E_SUCCESS 8381 */ 8382 pf->flags |= I40E_FLAG_TEMP_LINK_POLLING; 8383 dev_dbg(&pf->pdev->dev, "couldn't get link state, status: %d\n", 8384 status); 8385 return; 8386 } 8387 8388 old_link_speed = pf->hw.phy.link_info_old.link_speed; 8389 new_link_speed = pf->hw.phy.link_info.link_speed; 8390 8391 if (new_link == old_link && 8392 new_link_speed == old_link_speed && 8393 (test_bit(__I40E_VSI_DOWN, vsi->state) || 8394 new_link == netif_carrier_ok(vsi->netdev))) 8395 return; 8396 8397 i40e_print_link_message(vsi, new_link); 8398 8399 /* Notify the base of the switch tree connected to 8400 * the link. Floating VEBs are not notified. 8401 */ 8402 if (pf->lan_veb != I40E_NO_VEB && pf->veb[pf->lan_veb]) 8403 i40e_veb_link_event(pf->veb[pf->lan_veb], new_link); 8404 else 8405 i40e_vsi_link_event(vsi, new_link); 8406 8407 if (pf->vf) 8408 i40e_vc_notify_link_state(pf); 8409 8410 if (pf->flags & I40E_FLAG_PTP) 8411 i40e_ptp_set_increment(pf); 8412 } 8413 8414 /** 8415 * i40e_watchdog_subtask - periodic checks not using event driven response 8416 * @pf: board private structure 8417 **/ 8418 static void i40e_watchdog_subtask(struct i40e_pf *pf) 8419 { 8420 int i; 8421 8422 /* if interface is down do nothing */ 8423 if (test_bit(__I40E_DOWN, pf->state) || 8424 test_bit(__I40E_CONFIG_BUSY, pf->state)) 8425 return; 8426 8427 /* make sure we don't do these things too often */ 8428 if (time_before(jiffies, (pf->service_timer_previous + 8429 pf->service_timer_period))) 8430 return; 8431 pf->service_timer_previous = jiffies; 8432 8433 if ((pf->flags & I40E_FLAG_LINK_POLLING_ENABLED) || 8434 (pf->flags & I40E_FLAG_TEMP_LINK_POLLING)) 8435 i40e_link_event(pf); 8436 8437 /* Update the stats for active netdevs so the network stack 8438 * can look at updated numbers whenever it cares to 8439 */ 8440 for (i = 0; i < pf->num_alloc_vsi; i++) 8441 if (pf->vsi[i] && pf->vsi[i]->netdev) 8442 i40e_update_stats(pf->vsi[i]); 8443 8444 if (pf->flags & I40E_FLAG_VEB_STATS_ENABLED) { 8445 /* Update the stats for the active switching components */ 8446 for (i = 0; i < I40E_MAX_VEB; i++) 8447 if (pf->veb[i]) 8448 i40e_update_veb_stats(pf->veb[i]); 8449 } 8450 8451 i40e_ptp_rx_hang(pf); 8452 i40e_ptp_tx_hang(pf); 8453 } 8454 8455 /** 8456 * i40e_reset_subtask - Set up for resetting the device and driver 8457 * @pf: board private structure 8458 **/ 8459 static void i40e_reset_subtask(struct i40e_pf *pf) 8460 { 8461 u32 reset_flags = 0; 8462 8463 if (test_bit(__I40E_REINIT_REQUESTED, pf->state)) { 8464 reset_flags |= BIT(__I40E_REINIT_REQUESTED); 8465 clear_bit(__I40E_REINIT_REQUESTED, pf->state); 8466 } 8467 if (test_bit(__I40E_PF_RESET_REQUESTED, pf->state)) { 8468 reset_flags |= BIT(__I40E_PF_RESET_REQUESTED); 8469 clear_bit(__I40E_PF_RESET_REQUESTED, pf->state); 8470 } 8471 if (test_bit(__I40E_CORE_RESET_REQUESTED, pf->state)) { 8472 reset_flags |= BIT(__I40E_CORE_RESET_REQUESTED); 8473 clear_bit(__I40E_CORE_RESET_REQUESTED, pf->state); 8474 } 8475 if (test_bit(__I40E_GLOBAL_RESET_REQUESTED, pf->state)) { 8476 reset_flags |= BIT(__I40E_GLOBAL_RESET_REQUESTED); 8477 clear_bit(__I40E_GLOBAL_RESET_REQUESTED, pf->state); 8478 } 8479 if (test_bit(__I40E_DOWN_REQUESTED, pf->state)) { 8480 reset_flags |= BIT(__I40E_DOWN_REQUESTED); 8481 clear_bit(__I40E_DOWN_REQUESTED, pf->state); 8482 } 8483 8484 /* If there's a recovery already waiting, it takes 8485 * precedence before starting a new reset sequence. 8486 */ 8487 if (test_bit(__I40E_RESET_INTR_RECEIVED, pf->state)) { 8488 i40e_prep_for_reset(pf, false); 8489 i40e_reset(pf); 8490 i40e_rebuild(pf, false, false); 8491 } 8492 8493 /* If we're already down or resetting, just bail */ 8494 if (reset_flags && 8495 !test_bit(__I40E_DOWN, pf->state) && 8496 !test_bit(__I40E_CONFIG_BUSY, pf->state)) { 8497 i40e_do_reset(pf, reset_flags, false); 8498 } 8499 } 8500 8501 /** 8502 * i40e_handle_link_event - Handle link event 8503 * @pf: board private structure 8504 * @e: event info posted on ARQ 8505 **/ 8506 static void i40e_handle_link_event(struct i40e_pf *pf, 8507 struct i40e_arq_event_info *e) 8508 { 8509 struct i40e_aqc_get_link_status *status = 8510 (struct i40e_aqc_get_link_status *)&e->desc.params.raw; 8511 8512 /* Do a new status request to re-enable LSE reporting 8513 * and load new status information into the hw struct 8514 * This completely ignores any state information 8515 * in the ARQ event info, instead choosing to always 8516 * issue the AQ update link status command. 8517 */ 8518 i40e_link_event(pf); 8519 8520 /* Check if module meets thermal requirements */ 8521 if (status->phy_type == I40E_PHY_TYPE_NOT_SUPPORTED_HIGH_TEMP) { 8522 dev_err(&pf->pdev->dev, 8523 "Rx/Tx is disabled on this device because the module does not meet thermal requirements.\n"); 8524 dev_err(&pf->pdev->dev, 8525 "Refer to the Intel(R) Ethernet Adapters and Devices User Guide for a list of supported modules.\n"); 8526 } else { 8527 /* check for unqualified module, if link is down, suppress 8528 * the message if link was forced to be down. 8529 */ 8530 if ((status->link_info & I40E_AQ_MEDIA_AVAILABLE) && 8531 (!(status->an_info & I40E_AQ_QUALIFIED_MODULE)) && 8532 (!(status->link_info & I40E_AQ_LINK_UP)) && 8533 (!(pf->flags & I40E_FLAG_LINK_DOWN_ON_CLOSE_ENABLED))) { 8534 dev_err(&pf->pdev->dev, 8535 "Rx/Tx is disabled on this device because an unsupported SFP module type was detected.\n"); 8536 dev_err(&pf->pdev->dev, 8537 "Refer to the Intel(R) Ethernet Adapters and Devices User Guide for a list of supported modules.\n"); 8538 } 8539 } 8540 } 8541 8542 /** 8543 * i40e_clean_adminq_subtask - Clean the AdminQ rings 8544 * @pf: board private structure 8545 **/ 8546 static void i40e_clean_adminq_subtask(struct i40e_pf *pf) 8547 { 8548 struct i40e_arq_event_info event; 8549 struct i40e_hw *hw = &pf->hw; 8550 u16 pending, i = 0; 8551 i40e_status ret; 8552 u16 opcode; 8553 u32 oldval; 8554 u32 val; 8555 8556 /* Do not run clean AQ when PF reset fails */ 8557 if (test_bit(__I40E_RESET_FAILED, pf->state)) 8558 return; 8559 8560 /* check for error indications */ 8561 val = rd32(&pf->hw, pf->hw.aq.arq.len); 8562 oldval = val; 8563 if (val & I40E_PF_ARQLEN_ARQVFE_MASK) { 8564 if (hw->debug_mask & I40E_DEBUG_AQ) 8565 dev_info(&pf->pdev->dev, "ARQ VF Error detected\n"); 8566 val &= ~I40E_PF_ARQLEN_ARQVFE_MASK; 8567 } 8568 if (val & I40E_PF_ARQLEN_ARQOVFL_MASK) { 8569 if (hw->debug_mask & I40E_DEBUG_AQ) 8570 dev_info(&pf->pdev->dev, "ARQ Overflow Error detected\n"); 8571 val &= ~I40E_PF_ARQLEN_ARQOVFL_MASK; 8572 pf->arq_overflows++; 8573 } 8574 if (val & I40E_PF_ARQLEN_ARQCRIT_MASK) { 8575 if (hw->debug_mask & I40E_DEBUG_AQ) 8576 dev_info(&pf->pdev->dev, "ARQ Critical Error detected\n"); 8577 val &= ~I40E_PF_ARQLEN_ARQCRIT_MASK; 8578 } 8579 if (oldval != val) 8580 wr32(&pf->hw, pf->hw.aq.arq.len, val); 8581 8582 val = rd32(&pf->hw, pf->hw.aq.asq.len); 8583 oldval = val; 8584 if (val & I40E_PF_ATQLEN_ATQVFE_MASK) { 8585 if (pf->hw.debug_mask & I40E_DEBUG_AQ) 8586 dev_info(&pf->pdev->dev, "ASQ VF Error detected\n"); 8587 val &= ~I40E_PF_ATQLEN_ATQVFE_MASK; 8588 } 8589 if (val & I40E_PF_ATQLEN_ATQOVFL_MASK) { 8590 if (pf->hw.debug_mask & I40E_DEBUG_AQ) 8591 dev_info(&pf->pdev->dev, "ASQ Overflow Error detected\n"); 8592 val &= ~I40E_PF_ATQLEN_ATQOVFL_MASK; 8593 } 8594 if (val & I40E_PF_ATQLEN_ATQCRIT_MASK) { 8595 if (pf->hw.debug_mask & I40E_DEBUG_AQ) 8596 dev_info(&pf->pdev->dev, "ASQ Critical Error detected\n"); 8597 val &= ~I40E_PF_ATQLEN_ATQCRIT_MASK; 8598 } 8599 if (oldval != val) 8600 wr32(&pf->hw, pf->hw.aq.asq.len, val); 8601 8602 event.buf_len = I40E_MAX_AQ_BUF_SIZE; 8603 event.msg_buf = kzalloc(event.buf_len, GFP_KERNEL); 8604 if (!event.msg_buf) 8605 return; 8606 8607 do { 8608 ret = i40e_clean_arq_element(hw, &event, &pending); 8609 if (ret == I40E_ERR_ADMIN_QUEUE_NO_WORK) 8610 break; 8611 else if (ret) { 8612 dev_info(&pf->pdev->dev, "ARQ event error %d\n", ret); 8613 break; 8614 } 8615 8616 opcode = le16_to_cpu(event.desc.opcode); 8617 switch (opcode) { 8618 8619 case i40e_aqc_opc_get_link_status: 8620 i40e_handle_link_event(pf, &event); 8621 break; 8622 case i40e_aqc_opc_send_msg_to_pf: 8623 ret = i40e_vc_process_vf_msg(pf, 8624 le16_to_cpu(event.desc.retval), 8625 le32_to_cpu(event.desc.cookie_high), 8626 le32_to_cpu(event.desc.cookie_low), 8627 event.msg_buf, 8628 event.msg_len); 8629 break; 8630 case i40e_aqc_opc_lldp_update_mib: 8631 dev_dbg(&pf->pdev->dev, "ARQ: Update LLDP MIB event received\n"); 8632 #ifdef CONFIG_I40E_DCB 8633 rtnl_lock(); 8634 ret = i40e_handle_lldp_event(pf, &event); 8635 rtnl_unlock(); 8636 #endif /* CONFIG_I40E_DCB */ 8637 break; 8638 case i40e_aqc_opc_event_lan_overflow: 8639 dev_dbg(&pf->pdev->dev, "ARQ LAN queue overflow event received\n"); 8640 i40e_handle_lan_overflow_event(pf, &event); 8641 break; 8642 case i40e_aqc_opc_send_msg_to_peer: 8643 dev_info(&pf->pdev->dev, "ARQ: Msg from other pf\n"); 8644 break; 8645 case i40e_aqc_opc_nvm_erase: 8646 case i40e_aqc_opc_nvm_update: 8647 case i40e_aqc_opc_oem_post_update: 8648 i40e_debug(&pf->hw, I40E_DEBUG_NVM, 8649 "ARQ NVM operation 0x%04x completed\n", 8650 opcode); 8651 break; 8652 default: 8653 dev_info(&pf->pdev->dev, 8654 "ARQ: Unknown event 0x%04x ignored\n", 8655 opcode); 8656 break; 8657 } 8658 } while (i++ < pf->adminq_work_limit); 8659 8660 if (i < pf->adminq_work_limit) 8661 clear_bit(__I40E_ADMINQ_EVENT_PENDING, pf->state); 8662 8663 /* re-enable Admin queue interrupt cause */ 8664 val = rd32(hw, I40E_PFINT_ICR0_ENA); 8665 val |= I40E_PFINT_ICR0_ENA_ADMINQ_MASK; 8666 wr32(hw, I40E_PFINT_ICR0_ENA, val); 8667 i40e_flush(hw); 8668 8669 kfree(event.msg_buf); 8670 } 8671 8672 /** 8673 * i40e_verify_eeprom - make sure eeprom is good to use 8674 * @pf: board private structure 8675 **/ 8676 static void i40e_verify_eeprom(struct i40e_pf *pf) 8677 { 8678 int err; 8679 8680 err = i40e_diag_eeprom_test(&pf->hw); 8681 if (err) { 8682 /* retry in case of garbage read */ 8683 err = i40e_diag_eeprom_test(&pf->hw); 8684 if (err) { 8685 dev_info(&pf->pdev->dev, "eeprom check failed (%d), Tx/Rx traffic disabled\n", 8686 err); 8687 set_bit(__I40E_BAD_EEPROM, pf->state); 8688 } 8689 } 8690 8691 if (!err && test_bit(__I40E_BAD_EEPROM, pf->state)) { 8692 dev_info(&pf->pdev->dev, "eeprom check passed, Tx/Rx traffic enabled\n"); 8693 clear_bit(__I40E_BAD_EEPROM, pf->state); 8694 } 8695 } 8696 8697 /** 8698 * i40e_enable_pf_switch_lb 8699 * @pf: pointer to the PF structure 8700 * 8701 * enable switch loop back or die - no point in a return value 8702 **/ 8703 static void i40e_enable_pf_switch_lb(struct i40e_pf *pf) 8704 { 8705 struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi]; 8706 struct i40e_vsi_context ctxt; 8707 int ret; 8708 8709 ctxt.seid = pf->main_vsi_seid; 8710 ctxt.pf_num = pf->hw.pf_id; 8711 ctxt.vf_num = 0; 8712 ret = i40e_aq_get_vsi_params(&pf->hw, &ctxt, NULL); 8713 if (ret) { 8714 dev_info(&pf->pdev->dev, 8715 "couldn't get PF vsi config, err %s aq_err %s\n", 8716 i40e_stat_str(&pf->hw, ret), 8717 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 8718 return; 8719 } 8720 ctxt.flags = I40E_AQ_VSI_TYPE_PF; 8721 ctxt.info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID); 8722 ctxt.info.switch_id |= cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB); 8723 8724 ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL); 8725 if (ret) { 8726 dev_info(&pf->pdev->dev, 8727 "update vsi switch failed, err %s aq_err %s\n", 8728 i40e_stat_str(&pf->hw, ret), 8729 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 8730 } 8731 } 8732 8733 /** 8734 * i40e_disable_pf_switch_lb 8735 * @pf: pointer to the PF structure 8736 * 8737 * disable switch loop back or die - no point in a return value 8738 **/ 8739 static void i40e_disable_pf_switch_lb(struct i40e_pf *pf) 8740 { 8741 struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi]; 8742 struct i40e_vsi_context ctxt; 8743 int ret; 8744 8745 ctxt.seid = pf->main_vsi_seid; 8746 ctxt.pf_num = pf->hw.pf_id; 8747 ctxt.vf_num = 0; 8748 ret = i40e_aq_get_vsi_params(&pf->hw, &ctxt, NULL); 8749 if (ret) { 8750 dev_info(&pf->pdev->dev, 8751 "couldn't get PF vsi config, err %s aq_err %s\n", 8752 i40e_stat_str(&pf->hw, ret), 8753 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 8754 return; 8755 } 8756 ctxt.flags = I40E_AQ_VSI_TYPE_PF; 8757 ctxt.info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID); 8758 ctxt.info.switch_id &= ~cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB); 8759 8760 ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL); 8761 if (ret) { 8762 dev_info(&pf->pdev->dev, 8763 "update vsi switch failed, err %s aq_err %s\n", 8764 i40e_stat_str(&pf->hw, ret), 8765 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 8766 } 8767 } 8768 8769 /** 8770 * i40e_config_bridge_mode - Configure the HW bridge mode 8771 * @veb: pointer to the bridge instance 8772 * 8773 * Configure the loop back mode for the LAN VSI that is downlink to the 8774 * specified HW bridge instance. It is expected this function is called 8775 * when a new HW bridge is instantiated. 8776 **/ 8777 static void i40e_config_bridge_mode(struct i40e_veb *veb) 8778 { 8779 struct i40e_pf *pf = veb->pf; 8780 8781 if (pf->hw.debug_mask & I40E_DEBUG_LAN) 8782 dev_info(&pf->pdev->dev, "enabling bridge mode: %s\n", 8783 veb->bridge_mode == BRIDGE_MODE_VEPA ? "VEPA" : "VEB"); 8784 if (veb->bridge_mode & BRIDGE_MODE_VEPA) 8785 i40e_disable_pf_switch_lb(pf); 8786 else 8787 i40e_enable_pf_switch_lb(pf); 8788 } 8789 8790 /** 8791 * i40e_reconstitute_veb - rebuild the VEB and anything connected to it 8792 * @veb: pointer to the VEB instance 8793 * 8794 * This is a recursive function that first builds the attached VSIs then 8795 * recurses in to build the next layer of VEB. We track the connections 8796 * through our own index numbers because the seid's from the HW could 8797 * change across the reset. 8798 **/ 8799 static int i40e_reconstitute_veb(struct i40e_veb *veb) 8800 { 8801 struct i40e_vsi *ctl_vsi = NULL; 8802 struct i40e_pf *pf = veb->pf; 8803 int v, veb_idx; 8804 int ret; 8805 8806 /* build VSI that owns this VEB, temporarily attached to base VEB */ 8807 for (v = 0; v < pf->num_alloc_vsi && !ctl_vsi; v++) { 8808 if (pf->vsi[v] && 8809 pf->vsi[v]->veb_idx == veb->idx && 8810 pf->vsi[v]->flags & I40E_VSI_FLAG_VEB_OWNER) { 8811 ctl_vsi = pf->vsi[v]; 8812 break; 8813 } 8814 } 8815 if (!ctl_vsi) { 8816 dev_info(&pf->pdev->dev, 8817 "missing owner VSI for veb_idx %d\n", veb->idx); 8818 ret = -ENOENT; 8819 goto end_reconstitute; 8820 } 8821 if (ctl_vsi != pf->vsi[pf->lan_vsi]) 8822 ctl_vsi->uplink_seid = pf->vsi[pf->lan_vsi]->uplink_seid; 8823 ret = i40e_add_vsi(ctl_vsi); 8824 if (ret) { 8825 dev_info(&pf->pdev->dev, 8826 "rebuild of veb_idx %d owner VSI failed: %d\n", 8827 veb->idx, ret); 8828 goto end_reconstitute; 8829 } 8830 i40e_vsi_reset_stats(ctl_vsi); 8831 8832 /* create the VEB in the switch and move the VSI onto the VEB */ 8833 ret = i40e_add_veb(veb, ctl_vsi); 8834 if (ret) 8835 goto end_reconstitute; 8836 8837 if (pf->flags & I40E_FLAG_VEB_MODE_ENABLED) 8838 veb->bridge_mode = BRIDGE_MODE_VEB; 8839 else 8840 veb->bridge_mode = BRIDGE_MODE_VEPA; 8841 i40e_config_bridge_mode(veb); 8842 8843 /* create the remaining VSIs attached to this VEB */ 8844 for (v = 0; v < pf->num_alloc_vsi; v++) { 8845 if (!pf->vsi[v] || pf->vsi[v] == ctl_vsi) 8846 continue; 8847 8848 if (pf->vsi[v]->veb_idx == veb->idx) { 8849 struct i40e_vsi *vsi = pf->vsi[v]; 8850 8851 vsi->uplink_seid = veb->seid; 8852 ret = i40e_add_vsi(vsi); 8853 if (ret) { 8854 dev_info(&pf->pdev->dev, 8855 "rebuild of vsi_idx %d failed: %d\n", 8856 v, ret); 8857 goto end_reconstitute; 8858 } 8859 i40e_vsi_reset_stats(vsi); 8860 } 8861 } 8862 8863 /* create any VEBs attached to this VEB - RECURSION */ 8864 for (veb_idx = 0; veb_idx < I40E_MAX_VEB; veb_idx++) { 8865 if (pf->veb[veb_idx] && pf->veb[veb_idx]->veb_idx == veb->idx) { 8866 pf->veb[veb_idx]->uplink_seid = veb->seid; 8867 ret = i40e_reconstitute_veb(pf->veb[veb_idx]); 8868 if (ret) 8869 break; 8870 } 8871 } 8872 8873 end_reconstitute: 8874 return ret; 8875 } 8876 8877 /** 8878 * i40e_get_capabilities - get info about the HW 8879 * @pf: the PF struct 8880 **/ 8881 static int i40e_get_capabilities(struct i40e_pf *pf, 8882 enum i40e_admin_queue_opc list_type) 8883 { 8884 struct i40e_aqc_list_capabilities_element_resp *cap_buf; 8885 u16 data_size; 8886 int buf_len; 8887 int err; 8888 8889 buf_len = 40 * sizeof(struct i40e_aqc_list_capabilities_element_resp); 8890 do { 8891 cap_buf = kzalloc(buf_len, GFP_KERNEL); 8892 if (!cap_buf) 8893 return -ENOMEM; 8894 8895 /* this loads the data into the hw struct for us */ 8896 err = i40e_aq_discover_capabilities(&pf->hw, cap_buf, buf_len, 8897 &data_size, list_type, 8898 NULL); 8899 /* data loaded, buffer no longer needed */ 8900 kfree(cap_buf); 8901 8902 if (pf->hw.aq.asq_last_status == I40E_AQ_RC_ENOMEM) { 8903 /* retry with a larger buffer */ 8904 buf_len = data_size; 8905 } else if (pf->hw.aq.asq_last_status != I40E_AQ_RC_OK) { 8906 dev_info(&pf->pdev->dev, 8907 "capability discovery failed, err %s aq_err %s\n", 8908 i40e_stat_str(&pf->hw, err), 8909 i40e_aq_str(&pf->hw, 8910 pf->hw.aq.asq_last_status)); 8911 return -ENODEV; 8912 } 8913 } while (err); 8914 8915 if (pf->hw.debug_mask & I40E_DEBUG_USER) { 8916 if (list_type == i40e_aqc_opc_list_func_capabilities) { 8917 dev_info(&pf->pdev->dev, 8918 "pf=%d, num_vfs=%d, msix_pf=%d, msix_vf=%d, fd_g=%d, fd_b=%d, pf_max_q=%d num_vsi=%d\n", 8919 pf->hw.pf_id, pf->hw.func_caps.num_vfs, 8920 pf->hw.func_caps.num_msix_vectors, 8921 pf->hw.func_caps.num_msix_vectors_vf, 8922 pf->hw.func_caps.fd_filters_guaranteed, 8923 pf->hw.func_caps.fd_filters_best_effort, 8924 pf->hw.func_caps.num_tx_qp, 8925 pf->hw.func_caps.num_vsis); 8926 } else if (list_type == i40e_aqc_opc_list_dev_capabilities) { 8927 dev_info(&pf->pdev->dev, 8928 "switch_mode=0x%04x, function_valid=0x%08x\n", 8929 pf->hw.dev_caps.switch_mode, 8930 pf->hw.dev_caps.valid_functions); 8931 dev_info(&pf->pdev->dev, 8932 "SR-IOV=%d, num_vfs for all function=%u\n", 8933 pf->hw.dev_caps.sr_iov_1_1, 8934 pf->hw.dev_caps.num_vfs); 8935 dev_info(&pf->pdev->dev, 8936 "num_vsis=%u, num_rx:%u, num_tx=%u\n", 8937 pf->hw.dev_caps.num_vsis, 8938 pf->hw.dev_caps.num_rx_qp, 8939 pf->hw.dev_caps.num_tx_qp); 8940 } 8941 } 8942 if (list_type == i40e_aqc_opc_list_func_capabilities) { 8943 #define DEF_NUM_VSI (1 + (pf->hw.func_caps.fcoe ? 1 : 0) \ 8944 + pf->hw.func_caps.num_vfs) 8945 if (pf->hw.revision_id == 0 && 8946 pf->hw.func_caps.num_vsis < DEF_NUM_VSI) { 8947 dev_info(&pf->pdev->dev, 8948 "got num_vsis %d, setting num_vsis to %d\n", 8949 pf->hw.func_caps.num_vsis, DEF_NUM_VSI); 8950 pf->hw.func_caps.num_vsis = DEF_NUM_VSI; 8951 } 8952 } 8953 return 0; 8954 } 8955 8956 static int i40e_vsi_clear(struct i40e_vsi *vsi); 8957 8958 /** 8959 * i40e_fdir_sb_setup - initialize the Flow Director resources for Sideband 8960 * @pf: board private structure 8961 **/ 8962 static void i40e_fdir_sb_setup(struct i40e_pf *pf) 8963 { 8964 struct i40e_vsi *vsi; 8965 8966 /* quick workaround for an NVM issue that leaves a critical register 8967 * uninitialized 8968 */ 8969 if (!rd32(&pf->hw, I40E_GLQF_HKEY(0))) { 8970 static const u32 hkey[] = { 8971 0xe640d33f, 0xcdfe98ab, 0x73fa7161, 0x0d7a7d36, 8972 0xeacb7d61, 0xaa4f05b6, 0x9c5c89ed, 0xfc425ddb, 8973 0xa4654832, 0xfc7461d4, 0x8f827619, 0xf5c63c21, 8974 0x95b3a76d}; 8975 int i; 8976 8977 for (i = 0; i <= I40E_GLQF_HKEY_MAX_INDEX; i++) 8978 wr32(&pf->hw, I40E_GLQF_HKEY(i), hkey[i]); 8979 } 8980 8981 if (!(pf->flags & I40E_FLAG_FD_SB_ENABLED)) 8982 return; 8983 8984 /* find existing VSI and see if it needs configuring */ 8985 vsi = i40e_find_vsi_by_type(pf, I40E_VSI_FDIR); 8986 8987 /* create a new VSI if none exists */ 8988 if (!vsi) { 8989 vsi = i40e_vsi_setup(pf, I40E_VSI_FDIR, 8990 pf->vsi[pf->lan_vsi]->seid, 0); 8991 if (!vsi) { 8992 dev_info(&pf->pdev->dev, "Couldn't create FDir VSI\n"); 8993 pf->flags &= ~I40E_FLAG_FD_SB_ENABLED; 8994 pf->flags |= I40E_FLAG_FD_SB_INACTIVE; 8995 return; 8996 } 8997 } 8998 8999 i40e_vsi_setup_irqhandler(vsi, i40e_fdir_clean_ring); 9000 } 9001 9002 /** 9003 * i40e_fdir_teardown - release the Flow Director resources 9004 * @pf: board private structure 9005 **/ 9006 static void i40e_fdir_teardown(struct i40e_pf *pf) 9007 { 9008 struct i40e_vsi *vsi; 9009 9010 i40e_fdir_filter_exit(pf); 9011 vsi = i40e_find_vsi_by_type(pf, I40E_VSI_FDIR); 9012 if (vsi) 9013 i40e_vsi_release(vsi); 9014 } 9015 9016 /** 9017 * i40e_rebuild_cloud_filters - Rebuilds cloud filters for VSIs 9018 * @vsi: PF main vsi 9019 * @seid: seid of main or channel VSIs 9020 * 9021 * Rebuilds cloud filters associated with main VSI and channel VSIs if they 9022 * existed before reset 9023 **/ 9024 static int i40e_rebuild_cloud_filters(struct i40e_vsi *vsi, u16 seid) 9025 { 9026 struct i40e_cloud_filter *cfilter; 9027 struct i40e_pf *pf = vsi->back; 9028 struct hlist_node *node; 9029 i40e_status ret; 9030 9031 /* Add cloud filters back if they exist */ 9032 hlist_for_each_entry_safe(cfilter, node, &pf->cloud_filter_list, 9033 cloud_node) { 9034 if (cfilter->seid != seid) 9035 continue; 9036 9037 if (cfilter->dst_port) 9038 ret = i40e_add_del_cloud_filter_big_buf(vsi, cfilter, 9039 true); 9040 else 9041 ret = i40e_add_del_cloud_filter(vsi, cfilter, true); 9042 9043 if (ret) { 9044 dev_dbg(&pf->pdev->dev, 9045 "Failed to rebuild cloud filter, err %s aq_err %s\n", 9046 i40e_stat_str(&pf->hw, ret), 9047 i40e_aq_str(&pf->hw, 9048 pf->hw.aq.asq_last_status)); 9049 return ret; 9050 } 9051 } 9052 return 0; 9053 } 9054 9055 /** 9056 * i40e_rebuild_channels - Rebuilds channel VSIs if they existed before reset 9057 * @vsi: PF main vsi 9058 * 9059 * Rebuilds channel VSIs if they existed before reset 9060 **/ 9061 static int i40e_rebuild_channels(struct i40e_vsi *vsi) 9062 { 9063 struct i40e_channel *ch, *ch_tmp; 9064 i40e_status ret; 9065 9066 if (list_empty(&vsi->ch_list)) 9067 return 0; 9068 9069 list_for_each_entry_safe(ch, ch_tmp, &vsi->ch_list, list) { 9070 if (!ch->initialized) 9071 break; 9072 /* Proceed with creation of channel (VMDq2) VSI */ 9073 ret = i40e_add_channel(vsi->back, vsi->uplink_seid, ch); 9074 if (ret) { 9075 dev_info(&vsi->back->pdev->dev, 9076 "failed to rebuild channels using uplink_seid %u\n", 9077 vsi->uplink_seid); 9078 return ret; 9079 } 9080 if (ch->max_tx_rate) { 9081 u64 credits = ch->max_tx_rate; 9082 9083 if (i40e_set_bw_limit(vsi, ch->seid, 9084 ch->max_tx_rate)) 9085 return -EINVAL; 9086 9087 do_div(credits, I40E_BW_CREDIT_DIVISOR); 9088 dev_dbg(&vsi->back->pdev->dev, 9089 "Set tx rate of %llu Mbps (count of 50Mbps %llu) for vsi->seid %u\n", 9090 ch->max_tx_rate, 9091 credits, 9092 ch->seid); 9093 } 9094 ret = i40e_rebuild_cloud_filters(vsi, ch->seid); 9095 if (ret) { 9096 dev_dbg(&vsi->back->pdev->dev, 9097 "Failed to rebuild cloud filters for channel VSI %u\n", 9098 ch->seid); 9099 return ret; 9100 } 9101 } 9102 return 0; 9103 } 9104 9105 /** 9106 * i40e_prep_for_reset - prep for the core to reset 9107 * @pf: board private structure 9108 * @lock_acquired: indicates whether or not the lock has been acquired 9109 * before this function was called. 9110 * 9111 * Close up the VFs and other things in prep for PF Reset. 9112 **/ 9113 static void i40e_prep_for_reset(struct i40e_pf *pf, bool lock_acquired) 9114 { 9115 struct i40e_hw *hw = &pf->hw; 9116 i40e_status ret = 0; 9117 u32 v; 9118 9119 clear_bit(__I40E_RESET_INTR_RECEIVED, pf->state); 9120 if (test_and_set_bit(__I40E_RESET_RECOVERY_PENDING, pf->state)) 9121 return; 9122 if (i40e_check_asq_alive(&pf->hw)) 9123 i40e_vc_notify_reset(pf); 9124 9125 dev_dbg(&pf->pdev->dev, "Tearing down internal switch for reset\n"); 9126 9127 /* quiesce the VSIs and their queues that are not already DOWN */ 9128 /* pf_quiesce_all_vsi modifies netdev structures -rtnl_lock needed */ 9129 if (!lock_acquired) 9130 rtnl_lock(); 9131 i40e_pf_quiesce_all_vsi(pf); 9132 if (!lock_acquired) 9133 rtnl_unlock(); 9134 9135 for (v = 0; v < pf->num_alloc_vsi; v++) { 9136 if (pf->vsi[v]) 9137 pf->vsi[v]->seid = 0; 9138 } 9139 9140 i40e_shutdown_adminq(&pf->hw); 9141 9142 /* call shutdown HMC */ 9143 if (hw->hmc.hmc_obj) { 9144 ret = i40e_shutdown_lan_hmc(hw); 9145 if (ret) 9146 dev_warn(&pf->pdev->dev, 9147 "shutdown_lan_hmc failed: %d\n", ret); 9148 } 9149 } 9150 9151 /** 9152 * i40e_send_version - update firmware with driver version 9153 * @pf: PF struct 9154 */ 9155 static void i40e_send_version(struct i40e_pf *pf) 9156 { 9157 struct i40e_driver_version dv; 9158 9159 dv.major_version = DRV_VERSION_MAJOR; 9160 dv.minor_version = DRV_VERSION_MINOR; 9161 dv.build_version = DRV_VERSION_BUILD; 9162 dv.subbuild_version = 0; 9163 strlcpy(dv.driver_string, DRV_VERSION, sizeof(dv.driver_string)); 9164 i40e_aq_send_driver_version(&pf->hw, &dv, NULL); 9165 } 9166 9167 /** 9168 * i40e_get_oem_version - get OEM specific version information 9169 * @hw: pointer to the hardware structure 9170 **/ 9171 static void i40e_get_oem_version(struct i40e_hw *hw) 9172 { 9173 u16 block_offset = 0xffff; 9174 u16 block_length = 0; 9175 u16 capabilities = 0; 9176 u16 gen_snap = 0; 9177 u16 release = 0; 9178 9179 #define I40E_SR_NVM_OEM_VERSION_PTR 0x1B 9180 #define I40E_NVM_OEM_LENGTH_OFFSET 0x00 9181 #define I40E_NVM_OEM_CAPABILITIES_OFFSET 0x01 9182 #define I40E_NVM_OEM_GEN_OFFSET 0x02 9183 #define I40E_NVM_OEM_RELEASE_OFFSET 0x03 9184 #define I40E_NVM_OEM_CAPABILITIES_MASK 0x000F 9185 #define I40E_NVM_OEM_LENGTH 3 9186 9187 /* Check if pointer to OEM version block is valid. */ 9188 i40e_read_nvm_word(hw, I40E_SR_NVM_OEM_VERSION_PTR, &block_offset); 9189 if (block_offset == 0xffff) 9190 return; 9191 9192 /* Check if OEM version block has correct length. */ 9193 i40e_read_nvm_word(hw, block_offset + I40E_NVM_OEM_LENGTH_OFFSET, 9194 &block_length); 9195 if (block_length < I40E_NVM_OEM_LENGTH) 9196 return; 9197 9198 /* Check if OEM version format is as expected. */ 9199 i40e_read_nvm_word(hw, block_offset + I40E_NVM_OEM_CAPABILITIES_OFFSET, 9200 &capabilities); 9201 if ((capabilities & I40E_NVM_OEM_CAPABILITIES_MASK) != 0) 9202 return; 9203 9204 i40e_read_nvm_word(hw, block_offset + I40E_NVM_OEM_GEN_OFFSET, 9205 &gen_snap); 9206 i40e_read_nvm_word(hw, block_offset + I40E_NVM_OEM_RELEASE_OFFSET, 9207 &release); 9208 hw->nvm.oem_ver = (gen_snap << I40E_OEM_SNAP_SHIFT) | release; 9209 hw->nvm.eetrack = I40E_OEM_EETRACK_ID; 9210 } 9211 9212 /** 9213 * i40e_reset - wait for core reset to finish reset, reset pf if corer not seen 9214 * @pf: board private structure 9215 **/ 9216 static int i40e_reset(struct i40e_pf *pf) 9217 { 9218 struct i40e_hw *hw = &pf->hw; 9219 i40e_status ret; 9220 9221 ret = i40e_pf_reset(hw); 9222 if (ret) { 9223 dev_info(&pf->pdev->dev, "PF reset failed, %d\n", ret); 9224 set_bit(__I40E_RESET_FAILED, pf->state); 9225 clear_bit(__I40E_RESET_RECOVERY_PENDING, pf->state); 9226 } else { 9227 pf->pfr_count++; 9228 } 9229 return ret; 9230 } 9231 9232 /** 9233 * i40e_rebuild - rebuild using a saved config 9234 * @pf: board private structure 9235 * @reinit: if the Main VSI needs to re-initialized. 9236 * @lock_acquired: indicates whether or not the lock has been acquired 9237 * before this function was called. 9238 **/ 9239 static void i40e_rebuild(struct i40e_pf *pf, bool reinit, bool lock_acquired) 9240 { 9241 struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi]; 9242 struct i40e_hw *hw = &pf->hw; 9243 u8 set_fc_aq_fail = 0; 9244 i40e_status ret; 9245 u32 val; 9246 int v; 9247 9248 if (test_bit(__I40E_DOWN, pf->state)) 9249 goto clear_recovery; 9250 dev_dbg(&pf->pdev->dev, "Rebuilding internal switch\n"); 9251 9252 /* rebuild the basics for the AdminQ, HMC, and initial HW switch */ 9253 ret = i40e_init_adminq(&pf->hw); 9254 if (ret) { 9255 dev_info(&pf->pdev->dev, "Rebuild AdminQ failed, err %s aq_err %s\n", 9256 i40e_stat_str(&pf->hw, ret), 9257 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 9258 goto clear_recovery; 9259 } 9260 i40e_get_oem_version(&pf->hw); 9261 9262 /* re-verify the eeprom if we just had an EMP reset */ 9263 if (test_and_clear_bit(__I40E_EMP_RESET_INTR_RECEIVED, pf->state)) 9264 i40e_verify_eeprom(pf); 9265 9266 i40e_clear_pxe_mode(hw); 9267 ret = i40e_get_capabilities(pf, i40e_aqc_opc_list_func_capabilities); 9268 if (ret) 9269 goto end_core_reset; 9270 9271 ret = i40e_init_lan_hmc(hw, hw->func_caps.num_tx_qp, 9272 hw->func_caps.num_rx_qp, 0, 0); 9273 if (ret) { 9274 dev_info(&pf->pdev->dev, "init_lan_hmc failed: %d\n", ret); 9275 goto end_core_reset; 9276 } 9277 ret = i40e_configure_lan_hmc(hw, I40E_HMC_MODEL_DIRECT_ONLY); 9278 if (ret) { 9279 dev_info(&pf->pdev->dev, "configure_lan_hmc failed: %d\n", ret); 9280 goto end_core_reset; 9281 } 9282 9283 #ifdef CONFIG_I40E_DCB 9284 ret = i40e_init_pf_dcb(pf); 9285 if (ret) { 9286 dev_info(&pf->pdev->dev, "DCB init failed %d, disabled\n", ret); 9287 pf->flags &= ~I40E_FLAG_DCB_CAPABLE; 9288 /* Continue without DCB enabled */ 9289 } 9290 #endif /* CONFIG_I40E_DCB */ 9291 /* do basic switch setup */ 9292 if (!lock_acquired) 9293 rtnl_lock(); 9294 ret = i40e_setup_pf_switch(pf, reinit); 9295 if (ret) 9296 goto end_unlock; 9297 9298 /* The driver only wants link up/down and module qualification 9299 * reports from firmware. Note the negative logic. 9300 */ 9301 ret = i40e_aq_set_phy_int_mask(&pf->hw, 9302 ~(I40E_AQ_EVENT_LINK_UPDOWN | 9303 I40E_AQ_EVENT_MEDIA_NA | 9304 I40E_AQ_EVENT_MODULE_QUAL_FAIL), NULL); 9305 if (ret) 9306 dev_info(&pf->pdev->dev, "set phy mask fail, err %s aq_err %s\n", 9307 i40e_stat_str(&pf->hw, ret), 9308 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 9309 9310 /* make sure our flow control settings are restored */ 9311 ret = i40e_set_fc(&pf->hw, &set_fc_aq_fail, true); 9312 if (ret) 9313 dev_dbg(&pf->pdev->dev, "setting flow control: ret = %s last_status = %s\n", 9314 i40e_stat_str(&pf->hw, ret), 9315 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 9316 9317 /* Rebuild the VSIs and VEBs that existed before reset. 9318 * They are still in our local switch element arrays, so only 9319 * need to rebuild the switch model in the HW. 9320 * 9321 * If there were VEBs but the reconstitution failed, we'll try 9322 * try to recover minimal use by getting the basic PF VSI working. 9323 */ 9324 if (vsi->uplink_seid != pf->mac_seid) { 9325 dev_dbg(&pf->pdev->dev, "attempting to rebuild switch\n"); 9326 /* find the one VEB connected to the MAC, and find orphans */ 9327 for (v = 0; v < I40E_MAX_VEB; v++) { 9328 if (!pf->veb[v]) 9329 continue; 9330 9331 if (pf->veb[v]->uplink_seid == pf->mac_seid || 9332 pf->veb[v]->uplink_seid == 0) { 9333 ret = i40e_reconstitute_veb(pf->veb[v]); 9334 9335 if (!ret) 9336 continue; 9337 9338 /* If Main VEB failed, we're in deep doodoo, 9339 * so give up rebuilding the switch and set up 9340 * for minimal rebuild of PF VSI. 9341 * If orphan failed, we'll report the error 9342 * but try to keep going. 9343 */ 9344 if (pf->veb[v]->uplink_seid == pf->mac_seid) { 9345 dev_info(&pf->pdev->dev, 9346 "rebuild of switch failed: %d, will try to set up simple PF connection\n", 9347 ret); 9348 vsi->uplink_seid = pf->mac_seid; 9349 break; 9350 } else if (pf->veb[v]->uplink_seid == 0) { 9351 dev_info(&pf->pdev->dev, 9352 "rebuild of orphan VEB failed: %d\n", 9353 ret); 9354 } 9355 } 9356 } 9357 } 9358 9359 if (vsi->uplink_seid == pf->mac_seid) { 9360 dev_dbg(&pf->pdev->dev, "attempting to rebuild PF VSI\n"); 9361 /* no VEB, so rebuild only the Main VSI */ 9362 ret = i40e_add_vsi(vsi); 9363 if (ret) { 9364 dev_info(&pf->pdev->dev, 9365 "rebuild of Main VSI failed: %d\n", ret); 9366 goto end_unlock; 9367 } 9368 } 9369 9370 if (vsi->mqprio_qopt.max_rate[0]) { 9371 u64 max_tx_rate = vsi->mqprio_qopt.max_rate[0]; 9372 u64 credits = 0; 9373 9374 do_div(max_tx_rate, I40E_BW_MBPS_DIVISOR); 9375 ret = i40e_set_bw_limit(vsi, vsi->seid, max_tx_rate); 9376 if (ret) 9377 goto end_unlock; 9378 9379 credits = max_tx_rate; 9380 do_div(credits, I40E_BW_CREDIT_DIVISOR); 9381 dev_dbg(&vsi->back->pdev->dev, 9382 "Set tx rate of %llu Mbps (count of 50Mbps %llu) for vsi->seid %u\n", 9383 max_tx_rate, 9384 credits, 9385 vsi->seid); 9386 } 9387 9388 ret = i40e_rebuild_cloud_filters(vsi, vsi->seid); 9389 if (ret) 9390 goto end_unlock; 9391 9392 /* PF Main VSI is rebuild by now, go ahead and rebuild channel VSIs 9393 * for this main VSI if they exist 9394 */ 9395 ret = i40e_rebuild_channels(vsi); 9396 if (ret) 9397 goto end_unlock; 9398 9399 /* Reconfigure hardware for allowing smaller MSS in the case 9400 * of TSO, so that we avoid the MDD being fired and causing 9401 * a reset in the case of small MSS+TSO. 9402 */ 9403 #define I40E_REG_MSS 0x000E64DC 9404 #define I40E_REG_MSS_MIN_MASK 0x3FF0000 9405 #define I40E_64BYTE_MSS 0x400000 9406 val = rd32(hw, I40E_REG_MSS); 9407 if ((val & I40E_REG_MSS_MIN_MASK) > I40E_64BYTE_MSS) { 9408 val &= ~I40E_REG_MSS_MIN_MASK; 9409 val |= I40E_64BYTE_MSS; 9410 wr32(hw, I40E_REG_MSS, val); 9411 } 9412 9413 if (pf->hw_features & I40E_HW_RESTART_AUTONEG) { 9414 msleep(75); 9415 ret = i40e_aq_set_link_restart_an(&pf->hw, true, NULL); 9416 if (ret) 9417 dev_info(&pf->pdev->dev, "link restart failed, err %s aq_err %s\n", 9418 i40e_stat_str(&pf->hw, ret), 9419 i40e_aq_str(&pf->hw, 9420 pf->hw.aq.asq_last_status)); 9421 } 9422 /* reinit the misc interrupt */ 9423 if (pf->flags & I40E_FLAG_MSIX_ENABLED) 9424 ret = i40e_setup_misc_vector(pf); 9425 9426 /* Add a filter to drop all Flow control frames from any VSI from being 9427 * transmitted. By doing so we stop a malicious VF from sending out 9428 * PAUSE or PFC frames and potentially controlling traffic for other 9429 * PF/VF VSIs. 9430 * The FW can still send Flow control frames if enabled. 9431 */ 9432 i40e_add_filter_to_drop_tx_flow_control_frames(&pf->hw, 9433 pf->main_vsi_seid); 9434 9435 /* restart the VSIs that were rebuilt and running before the reset */ 9436 i40e_pf_unquiesce_all_vsi(pf); 9437 9438 /* Release the RTNL lock before we start resetting VFs */ 9439 if (!lock_acquired) 9440 rtnl_unlock(); 9441 9442 /* Restore promiscuous settings */ 9443 ret = i40e_set_promiscuous(pf, pf->cur_promisc); 9444 if (ret) 9445 dev_warn(&pf->pdev->dev, 9446 "Failed to restore promiscuous setting: %s, err %s aq_err %s\n", 9447 pf->cur_promisc ? "on" : "off", 9448 i40e_stat_str(&pf->hw, ret), 9449 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 9450 9451 i40e_reset_all_vfs(pf, true); 9452 9453 /* tell the firmware that we're starting */ 9454 i40e_send_version(pf); 9455 9456 /* We've already released the lock, so don't do it again */ 9457 goto end_core_reset; 9458 9459 end_unlock: 9460 if (!lock_acquired) 9461 rtnl_unlock(); 9462 end_core_reset: 9463 clear_bit(__I40E_RESET_FAILED, pf->state); 9464 clear_recovery: 9465 clear_bit(__I40E_RESET_RECOVERY_PENDING, pf->state); 9466 } 9467 9468 /** 9469 * i40e_reset_and_rebuild - reset and rebuild using a saved config 9470 * @pf: board private structure 9471 * @reinit: if the Main VSI needs to re-initialized. 9472 * @lock_acquired: indicates whether or not the lock has been acquired 9473 * before this function was called. 9474 **/ 9475 static void i40e_reset_and_rebuild(struct i40e_pf *pf, bool reinit, 9476 bool lock_acquired) 9477 { 9478 int ret; 9479 /* Now we wait for GRST to settle out. 9480 * We don't have to delete the VEBs or VSIs from the hw switch 9481 * because the reset will make them disappear. 9482 */ 9483 ret = i40e_reset(pf); 9484 if (!ret) 9485 i40e_rebuild(pf, reinit, lock_acquired); 9486 } 9487 9488 /** 9489 * i40e_handle_reset_warning - prep for the PF to reset, reset and rebuild 9490 * @pf: board private structure 9491 * 9492 * Close up the VFs and other things in prep for a Core Reset, 9493 * then get ready to rebuild the world. 9494 * @lock_acquired: indicates whether or not the lock has been acquired 9495 * before this function was called. 9496 **/ 9497 static void i40e_handle_reset_warning(struct i40e_pf *pf, bool lock_acquired) 9498 { 9499 i40e_prep_for_reset(pf, lock_acquired); 9500 i40e_reset_and_rebuild(pf, false, lock_acquired); 9501 } 9502 9503 /** 9504 * i40e_handle_mdd_event 9505 * @pf: pointer to the PF structure 9506 * 9507 * Called from the MDD irq handler to identify possibly malicious vfs 9508 **/ 9509 static void i40e_handle_mdd_event(struct i40e_pf *pf) 9510 { 9511 struct i40e_hw *hw = &pf->hw; 9512 bool mdd_detected = false; 9513 bool pf_mdd_detected = false; 9514 struct i40e_vf *vf; 9515 u32 reg; 9516 int i; 9517 9518 if (!test_bit(__I40E_MDD_EVENT_PENDING, pf->state)) 9519 return; 9520 9521 /* find what triggered the MDD event */ 9522 reg = rd32(hw, I40E_GL_MDET_TX); 9523 if (reg & I40E_GL_MDET_TX_VALID_MASK) { 9524 u8 pf_num = (reg & I40E_GL_MDET_TX_PF_NUM_MASK) >> 9525 I40E_GL_MDET_TX_PF_NUM_SHIFT; 9526 u16 vf_num = (reg & I40E_GL_MDET_TX_VF_NUM_MASK) >> 9527 I40E_GL_MDET_TX_VF_NUM_SHIFT; 9528 u8 event = (reg & I40E_GL_MDET_TX_EVENT_MASK) >> 9529 I40E_GL_MDET_TX_EVENT_SHIFT; 9530 u16 queue = ((reg & I40E_GL_MDET_TX_QUEUE_MASK) >> 9531 I40E_GL_MDET_TX_QUEUE_SHIFT) - 9532 pf->hw.func_caps.base_queue; 9533 if (netif_msg_tx_err(pf)) 9534 dev_info(&pf->pdev->dev, "Malicious Driver Detection event 0x%02x on TX queue %d PF number 0x%02x VF number 0x%02x\n", 9535 event, queue, pf_num, vf_num); 9536 wr32(hw, I40E_GL_MDET_TX, 0xffffffff); 9537 mdd_detected = true; 9538 } 9539 reg = rd32(hw, I40E_GL_MDET_RX); 9540 if (reg & I40E_GL_MDET_RX_VALID_MASK) { 9541 u8 func = (reg & I40E_GL_MDET_RX_FUNCTION_MASK) >> 9542 I40E_GL_MDET_RX_FUNCTION_SHIFT; 9543 u8 event = (reg & I40E_GL_MDET_RX_EVENT_MASK) >> 9544 I40E_GL_MDET_RX_EVENT_SHIFT; 9545 u16 queue = ((reg & I40E_GL_MDET_RX_QUEUE_MASK) >> 9546 I40E_GL_MDET_RX_QUEUE_SHIFT) - 9547 pf->hw.func_caps.base_queue; 9548 if (netif_msg_rx_err(pf)) 9549 dev_info(&pf->pdev->dev, "Malicious Driver Detection event 0x%02x on RX queue %d of function 0x%02x\n", 9550 event, queue, func); 9551 wr32(hw, I40E_GL_MDET_RX, 0xffffffff); 9552 mdd_detected = true; 9553 } 9554 9555 if (mdd_detected) { 9556 reg = rd32(hw, I40E_PF_MDET_TX); 9557 if (reg & I40E_PF_MDET_TX_VALID_MASK) { 9558 wr32(hw, I40E_PF_MDET_TX, 0xFFFF); 9559 dev_info(&pf->pdev->dev, "TX driver issue detected, PF reset issued\n"); 9560 pf_mdd_detected = true; 9561 } 9562 reg = rd32(hw, I40E_PF_MDET_RX); 9563 if (reg & I40E_PF_MDET_RX_VALID_MASK) { 9564 wr32(hw, I40E_PF_MDET_RX, 0xFFFF); 9565 dev_info(&pf->pdev->dev, "RX driver issue detected, PF reset issued\n"); 9566 pf_mdd_detected = true; 9567 } 9568 /* Queue belongs to the PF, initiate a reset */ 9569 if (pf_mdd_detected) { 9570 set_bit(__I40E_PF_RESET_REQUESTED, pf->state); 9571 i40e_service_event_schedule(pf); 9572 } 9573 } 9574 9575 /* see if one of the VFs needs its hand slapped */ 9576 for (i = 0; i < pf->num_alloc_vfs && mdd_detected; i++) { 9577 vf = &(pf->vf[i]); 9578 reg = rd32(hw, I40E_VP_MDET_TX(i)); 9579 if (reg & I40E_VP_MDET_TX_VALID_MASK) { 9580 wr32(hw, I40E_VP_MDET_TX(i), 0xFFFF); 9581 vf->num_mdd_events++; 9582 dev_info(&pf->pdev->dev, "TX driver issue detected on VF %d\n", 9583 i); 9584 } 9585 9586 reg = rd32(hw, I40E_VP_MDET_RX(i)); 9587 if (reg & I40E_VP_MDET_RX_VALID_MASK) { 9588 wr32(hw, I40E_VP_MDET_RX(i), 0xFFFF); 9589 vf->num_mdd_events++; 9590 dev_info(&pf->pdev->dev, "RX driver issue detected on VF %d\n", 9591 i); 9592 } 9593 9594 if (vf->num_mdd_events > I40E_DEFAULT_NUM_MDD_EVENTS_ALLOWED) { 9595 dev_info(&pf->pdev->dev, 9596 "Too many MDD events on VF %d, disabled\n", i); 9597 dev_info(&pf->pdev->dev, 9598 "Use PF Control I/F to re-enable the VF\n"); 9599 set_bit(I40E_VF_STATE_DISABLED, &vf->vf_states); 9600 } 9601 } 9602 9603 /* re-enable mdd interrupt cause */ 9604 clear_bit(__I40E_MDD_EVENT_PENDING, pf->state); 9605 reg = rd32(hw, I40E_PFINT_ICR0_ENA); 9606 reg |= I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK; 9607 wr32(hw, I40E_PFINT_ICR0_ENA, reg); 9608 i40e_flush(hw); 9609 } 9610 9611 static const char *i40e_tunnel_name(struct i40e_udp_port_config *port) 9612 { 9613 switch (port->type) { 9614 case UDP_TUNNEL_TYPE_VXLAN: 9615 return "vxlan"; 9616 case UDP_TUNNEL_TYPE_GENEVE: 9617 return "geneve"; 9618 default: 9619 return "unknown"; 9620 } 9621 } 9622 9623 /** 9624 * i40e_sync_udp_filters - Trigger a sync event for existing UDP filters 9625 * @pf: board private structure 9626 **/ 9627 static void i40e_sync_udp_filters(struct i40e_pf *pf) 9628 { 9629 int i; 9630 9631 /* loop through and set pending bit for all active UDP filters */ 9632 for (i = 0; i < I40E_MAX_PF_UDP_OFFLOAD_PORTS; i++) { 9633 if (pf->udp_ports[i].port) 9634 pf->pending_udp_bitmap |= BIT_ULL(i); 9635 } 9636 9637 pf->flags |= I40E_FLAG_UDP_FILTER_SYNC; 9638 } 9639 9640 /** 9641 * i40e_sync_udp_filters_subtask - Sync the VSI filter list with HW 9642 * @pf: board private structure 9643 **/ 9644 static void i40e_sync_udp_filters_subtask(struct i40e_pf *pf) 9645 { 9646 struct i40e_hw *hw = &pf->hw; 9647 i40e_status ret; 9648 u16 port; 9649 int i; 9650 9651 if (!(pf->flags & I40E_FLAG_UDP_FILTER_SYNC)) 9652 return; 9653 9654 pf->flags &= ~I40E_FLAG_UDP_FILTER_SYNC; 9655 9656 for (i = 0; i < I40E_MAX_PF_UDP_OFFLOAD_PORTS; i++) { 9657 if (pf->pending_udp_bitmap & BIT_ULL(i)) { 9658 pf->pending_udp_bitmap &= ~BIT_ULL(i); 9659 port = pf->udp_ports[i].port; 9660 if (port) 9661 ret = i40e_aq_add_udp_tunnel(hw, port, 9662 pf->udp_ports[i].type, 9663 NULL, NULL); 9664 else 9665 ret = i40e_aq_del_udp_tunnel(hw, i, NULL); 9666 9667 if (ret) { 9668 dev_info(&pf->pdev->dev, 9669 "%s %s port %d, index %d failed, err %s aq_err %s\n", 9670 i40e_tunnel_name(&pf->udp_ports[i]), 9671 port ? "add" : "delete", 9672 port, i, 9673 i40e_stat_str(&pf->hw, ret), 9674 i40e_aq_str(&pf->hw, 9675 pf->hw.aq.asq_last_status)); 9676 pf->udp_ports[i].port = 0; 9677 } 9678 } 9679 } 9680 } 9681 9682 /** 9683 * i40e_service_task - Run the driver's async subtasks 9684 * @work: pointer to work_struct containing our data 9685 **/ 9686 static void i40e_service_task(struct work_struct *work) 9687 { 9688 struct i40e_pf *pf = container_of(work, 9689 struct i40e_pf, 9690 service_task); 9691 unsigned long start_time = jiffies; 9692 9693 /* don't bother with service tasks if a reset is in progress */ 9694 if (test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state)) 9695 return; 9696 9697 if (test_and_set_bit(__I40E_SERVICE_SCHED, pf->state)) 9698 return; 9699 9700 i40e_detect_recover_hung(pf); 9701 i40e_sync_filters_subtask(pf); 9702 i40e_reset_subtask(pf); 9703 i40e_handle_mdd_event(pf); 9704 i40e_vc_process_vflr_event(pf); 9705 i40e_watchdog_subtask(pf); 9706 i40e_fdir_reinit_subtask(pf); 9707 if (pf->flags & I40E_FLAG_CLIENT_RESET) { 9708 /* Client subtask will reopen next time through. */ 9709 i40e_notify_client_of_netdev_close(pf->vsi[pf->lan_vsi], true); 9710 pf->flags &= ~I40E_FLAG_CLIENT_RESET; 9711 } else { 9712 i40e_client_subtask(pf); 9713 if (pf->flags & I40E_FLAG_CLIENT_L2_CHANGE) { 9714 i40e_notify_client_of_l2_param_changes( 9715 pf->vsi[pf->lan_vsi]); 9716 pf->flags &= ~I40E_FLAG_CLIENT_L2_CHANGE; 9717 } 9718 } 9719 i40e_sync_filters_subtask(pf); 9720 i40e_sync_udp_filters_subtask(pf); 9721 i40e_clean_adminq_subtask(pf); 9722 9723 /* flush memory to make sure state is correct before next watchdog */ 9724 smp_mb__before_atomic(); 9725 clear_bit(__I40E_SERVICE_SCHED, pf->state); 9726 9727 /* If the tasks have taken longer than one timer cycle or there 9728 * is more work to be done, reschedule the service task now 9729 * rather than wait for the timer to tick again. 9730 */ 9731 if (time_after(jiffies, (start_time + pf->service_timer_period)) || 9732 test_bit(__I40E_ADMINQ_EVENT_PENDING, pf->state) || 9733 test_bit(__I40E_MDD_EVENT_PENDING, pf->state) || 9734 test_bit(__I40E_VFLR_EVENT_PENDING, pf->state)) 9735 i40e_service_event_schedule(pf); 9736 } 9737 9738 /** 9739 * i40e_service_timer - timer callback 9740 * @data: pointer to PF struct 9741 **/ 9742 static void i40e_service_timer(struct timer_list *t) 9743 { 9744 struct i40e_pf *pf = from_timer(pf, t, service_timer); 9745 9746 mod_timer(&pf->service_timer, 9747 round_jiffies(jiffies + pf->service_timer_period)); 9748 i40e_service_event_schedule(pf); 9749 } 9750 9751 /** 9752 * i40e_set_num_rings_in_vsi - Determine number of rings in the VSI 9753 * @vsi: the VSI being configured 9754 **/ 9755 static int i40e_set_num_rings_in_vsi(struct i40e_vsi *vsi) 9756 { 9757 struct i40e_pf *pf = vsi->back; 9758 9759 switch (vsi->type) { 9760 case I40E_VSI_MAIN: 9761 vsi->alloc_queue_pairs = pf->num_lan_qps; 9762 vsi->num_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS, 9763 I40E_REQ_DESCRIPTOR_MULTIPLE); 9764 if (pf->flags & I40E_FLAG_MSIX_ENABLED) 9765 vsi->num_q_vectors = pf->num_lan_msix; 9766 else 9767 vsi->num_q_vectors = 1; 9768 9769 break; 9770 9771 case I40E_VSI_FDIR: 9772 vsi->alloc_queue_pairs = 1; 9773 vsi->num_desc = ALIGN(I40E_FDIR_RING_COUNT, 9774 I40E_REQ_DESCRIPTOR_MULTIPLE); 9775 vsi->num_q_vectors = pf->num_fdsb_msix; 9776 break; 9777 9778 case I40E_VSI_VMDQ2: 9779 vsi->alloc_queue_pairs = pf->num_vmdq_qps; 9780 vsi->num_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS, 9781 I40E_REQ_DESCRIPTOR_MULTIPLE); 9782 vsi->num_q_vectors = pf->num_vmdq_msix; 9783 break; 9784 9785 case I40E_VSI_SRIOV: 9786 vsi->alloc_queue_pairs = pf->num_vf_qps; 9787 vsi->num_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS, 9788 I40E_REQ_DESCRIPTOR_MULTIPLE); 9789 break; 9790 9791 default: 9792 WARN_ON(1); 9793 return -ENODATA; 9794 } 9795 9796 return 0; 9797 } 9798 9799 /** 9800 * i40e_vsi_alloc_arrays - Allocate queue and vector pointer arrays for the vsi 9801 * @vsi: VSI pointer 9802 * @alloc_qvectors: a bool to specify if q_vectors need to be allocated. 9803 * 9804 * On error: returns error code (negative) 9805 * On success: returns 0 9806 **/ 9807 static int i40e_vsi_alloc_arrays(struct i40e_vsi *vsi, bool alloc_qvectors) 9808 { 9809 struct i40e_ring **next_rings; 9810 int size; 9811 int ret = 0; 9812 9813 /* allocate memory for both Tx, XDP Tx and Rx ring pointers */ 9814 size = sizeof(struct i40e_ring *) * vsi->alloc_queue_pairs * 9815 (i40e_enabled_xdp_vsi(vsi) ? 3 : 2); 9816 vsi->tx_rings = kzalloc(size, GFP_KERNEL); 9817 if (!vsi->tx_rings) 9818 return -ENOMEM; 9819 next_rings = vsi->tx_rings + vsi->alloc_queue_pairs; 9820 if (i40e_enabled_xdp_vsi(vsi)) { 9821 vsi->xdp_rings = next_rings; 9822 next_rings += vsi->alloc_queue_pairs; 9823 } 9824 vsi->rx_rings = next_rings; 9825 9826 if (alloc_qvectors) { 9827 /* allocate memory for q_vector pointers */ 9828 size = sizeof(struct i40e_q_vector *) * vsi->num_q_vectors; 9829 vsi->q_vectors = kzalloc(size, GFP_KERNEL); 9830 if (!vsi->q_vectors) { 9831 ret = -ENOMEM; 9832 goto err_vectors; 9833 } 9834 } 9835 return ret; 9836 9837 err_vectors: 9838 kfree(vsi->tx_rings); 9839 return ret; 9840 } 9841 9842 /** 9843 * i40e_vsi_mem_alloc - Allocates the next available struct vsi in the PF 9844 * @pf: board private structure 9845 * @type: type of VSI 9846 * 9847 * On error: returns error code (negative) 9848 * On success: returns vsi index in PF (positive) 9849 **/ 9850 static int i40e_vsi_mem_alloc(struct i40e_pf *pf, enum i40e_vsi_type type) 9851 { 9852 int ret = -ENODEV; 9853 struct i40e_vsi *vsi; 9854 int vsi_idx; 9855 int i; 9856 9857 /* Need to protect the allocation of the VSIs at the PF level */ 9858 mutex_lock(&pf->switch_mutex); 9859 9860 /* VSI list may be fragmented if VSI creation/destruction has 9861 * been happening. We can afford to do a quick scan to look 9862 * for any free VSIs in the list. 9863 * 9864 * find next empty vsi slot, looping back around if necessary 9865 */ 9866 i = pf->next_vsi; 9867 while (i < pf->num_alloc_vsi && pf->vsi[i]) 9868 i++; 9869 if (i >= pf->num_alloc_vsi) { 9870 i = 0; 9871 while (i < pf->next_vsi && pf->vsi[i]) 9872 i++; 9873 } 9874 9875 if (i < pf->num_alloc_vsi && !pf->vsi[i]) { 9876 vsi_idx = i; /* Found one! */ 9877 } else { 9878 ret = -ENODEV; 9879 goto unlock_pf; /* out of VSI slots! */ 9880 } 9881 pf->next_vsi = ++i; 9882 9883 vsi = kzalloc(sizeof(*vsi), GFP_KERNEL); 9884 if (!vsi) { 9885 ret = -ENOMEM; 9886 goto unlock_pf; 9887 } 9888 vsi->type = type; 9889 vsi->back = pf; 9890 set_bit(__I40E_VSI_DOWN, vsi->state); 9891 vsi->flags = 0; 9892 vsi->idx = vsi_idx; 9893 vsi->int_rate_limit = 0; 9894 vsi->rss_table_size = (vsi->type == I40E_VSI_MAIN) ? 9895 pf->rss_table_size : 64; 9896 vsi->netdev_registered = false; 9897 vsi->work_limit = I40E_DEFAULT_IRQ_WORK; 9898 hash_init(vsi->mac_filter_hash); 9899 vsi->irqs_ready = false; 9900 9901 ret = i40e_set_num_rings_in_vsi(vsi); 9902 if (ret) 9903 goto err_rings; 9904 9905 ret = i40e_vsi_alloc_arrays(vsi, true); 9906 if (ret) 9907 goto err_rings; 9908 9909 /* Setup default MSIX irq handler for VSI */ 9910 i40e_vsi_setup_irqhandler(vsi, i40e_msix_clean_rings); 9911 9912 /* Initialize VSI lock */ 9913 spin_lock_init(&vsi->mac_filter_hash_lock); 9914 pf->vsi[vsi_idx] = vsi; 9915 ret = vsi_idx; 9916 goto unlock_pf; 9917 9918 err_rings: 9919 pf->next_vsi = i - 1; 9920 kfree(vsi); 9921 unlock_pf: 9922 mutex_unlock(&pf->switch_mutex); 9923 return ret; 9924 } 9925 9926 /** 9927 * i40e_vsi_free_arrays - Free queue and vector pointer arrays for the VSI 9928 * @type: VSI pointer 9929 * @free_qvectors: a bool to specify if q_vectors need to be freed. 9930 * 9931 * On error: returns error code (negative) 9932 * On success: returns 0 9933 **/ 9934 static void i40e_vsi_free_arrays(struct i40e_vsi *vsi, bool free_qvectors) 9935 { 9936 /* free the ring and vector containers */ 9937 if (free_qvectors) { 9938 kfree(vsi->q_vectors); 9939 vsi->q_vectors = NULL; 9940 } 9941 kfree(vsi->tx_rings); 9942 vsi->tx_rings = NULL; 9943 vsi->rx_rings = NULL; 9944 vsi->xdp_rings = NULL; 9945 } 9946 9947 /** 9948 * i40e_clear_rss_config_user - clear the user configured RSS hash keys 9949 * and lookup table 9950 * @vsi: Pointer to VSI structure 9951 */ 9952 static void i40e_clear_rss_config_user(struct i40e_vsi *vsi) 9953 { 9954 if (!vsi) 9955 return; 9956 9957 kfree(vsi->rss_hkey_user); 9958 vsi->rss_hkey_user = NULL; 9959 9960 kfree(vsi->rss_lut_user); 9961 vsi->rss_lut_user = NULL; 9962 } 9963 9964 /** 9965 * i40e_vsi_clear - Deallocate the VSI provided 9966 * @vsi: the VSI being un-configured 9967 **/ 9968 static int i40e_vsi_clear(struct i40e_vsi *vsi) 9969 { 9970 struct i40e_pf *pf; 9971 9972 if (!vsi) 9973 return 0; 9974 9975 if (!vsi->back) 9976 goto free_vsi; 9977 pf = vsi->back; 9978 9979 mutex_lock(&pf->switch_mutex); 9980 if (!pf->vsi[vsi->idx]) { 9981 dev_err(&pf->pdev->dev, "pf->vsi[%d] is NULL, just free vsi[%d](%p,type %d)\n", 9982 vsi->idx, vsi->idx, vsi, vsi->type); 9983 goto unlock_vsi; 9984 } 9985 9986 if (pf->vsi[vsi->idx] != vsi) { 9987 dev_err(&pf->pdev->dev, 9988 "pf->vsi[%d](%p, type %d) != vsi[%d](%p,type %d): no free!\n", 9989 pf->vsi[vsi->idx]->idx, 9990 pf->vsi[vsi->idx], 9991 pf->vsi[vsi->idx]->type, 9992 vsi->idx, vsi, vsi->type); 9993 goto unlock_vsi; 9994 } 9995 9996 /* updates the PF for this cleared vsi */ 9997 i40e_put_lump(pf->qp_pile, vsi->base_queue, vsi->idx); 9998 i40e_put_lump(pf->irq_pile, vsi->base_vector, vsi->idx); 9999 10000 i40e_vsi_free_arrays(vsi, true); 10001 i40e_clear_rss_config_user(vsi); 10002 10003 pf->vsi[vsi->idx] = NULL; 10004 if (vsi->idx < pf->next_vsi) 10005 pf->next_vsi = vsi->idx; 10006 10007 unlock_vsi: 10008 mutex_unlock(&pf->switch_mutex); 10009 free_vsi: 10010 kfree(vsi); 10011 10012 return 0; 10013 } 10014 10015 /** 10016 * i40e_vsi_clear_rings - Deallocates the Rx and Tx rings for the provided VSI 10017 * @vsi: the VSI being cleaned 10018 **/ 10019 static void i40e_vsi_clear_rings(struct i40e_vsi *vsi) 10020 { 10021 int i; 10022 10023 if (vsi->tx_rings && vsi->tx_rings[0]) { 10024 for (i = 0; i < vsi->alloc_queue_pairs; i++) { 10025 kfree_rcu(vsi->tx_rings[i], rcu); 10026 vsi->tx_rings[i] = NULL; 10027 vsi->rx_rings[i] = NULL; 10028 if (vsi->xdp_rings) 10029 vsi->xdp_rings[i] = NULL; 10030 } 10031 } 10032 } 10033 10034 /** 10035 * i40e_alloc_rings - Allocates the Rx and Tx rings for the provided VSI 10036 * @vsi: the VSI being configured 10037 **/ 10038 static int i40e_alloc_rings(struct i40e_vsi *vsi) 10039 { 10040 int i, qpv = i40e_enabled_xdp_vsi(vsi) ? 3 : 2; 10041 struct i40e_pf *pf = vsi->back; 10042 struct i40e_ring *ring; 10043 10044 /* Set basic values in the rings to be used later during open() */ 10045 for (i = 0; i < vsi->alloc_queue_pairs; i++) { 10046 /* allocate space for both Tx and Rx in one shot */ 10047 ring = kcalloc(qpv, sizeof(struct i40e_ring), GFP_KERNEL); 10048 if (!ring) 10049 goto err_out; 10050 10051 ring->queue_index = i; 10052 ring->reg_idx = vsi->base_queue + i; 10053 ring->ring_active = false; 10054 ring->vsi = vsi; 10055 ring->netdev = vsi->netdev; 10056 ring->dev = &pf->pdev->dev; 10057 ring->count = vsi->num_desc; 10058 ring->size = 0; 10059 ring->dcb_tc = 0; 10060 if (vsi->back->hw_features & I40E_HW_WB_ON_ITR_CAPABLE) 10061 ring->flags = I40E_TXR_FLAGS_WB_ON_ITR; 10062 ring->tx_itr_setting = pf->tx_itr_default; 10063 vsi->tx_rings[i] = ring++; 10064 10065 if (!i40e_enabled_xdp_vsi(vsi)) 10066 goto setup_rx; 10067 10068 ring->queue_index = vsi->alloc_queue_pairs + i; 10069 ring->reg_idx = vsi->base_queue + ring->queue_index; 10070 ring->ring_active = false; 10071 ring->vsi = vsi; 10072 ring->netdev = NULL; 10073 ring->dev = &pf->pdev->dev; 10074 ring->count = vsi->num_desc; 10075 ring->size = 0; 10076 ring->dcb_tc = 0; 10077 if (vsi->back->hw_features & I40E_HW_WB_ON_ITR_CAPABLE) 10078 ring->flags = I40E_TXR_FLAGS_WB_ON_ITR; 10079 set_ring_xdp(ring); 10080 ring->tx_itr_setting = pf->tx_itr_default; 10081 vsi->xdp_rings[i] = ring++; 10082 10083 setup_rx: 10084 ring->queue_index = i; 10085 ring->reg_idx = vsi->base_queue + i; 10086 ring->ring_active = false; 10087 ring->vsi = vsi; 10088 ring->netdev = vsi->netdev; 10089 ring->dev = &pf->pdev->dev; 10090 ring->count = vsi->num_desc; 10091 ring->size = 0; 10092 ring->dcb_tc = 0; 10093 ring->rx_itr_setting = pf->rx_itr_default; 10094 vsi->rx_rings[i] = ring; 10095 } 10096 10097 return 0; 10098 10099 err_out: 10100 i40e_vsi_clear_rings(vsi); 10101 return -ENOMEM; 10102 } 10103 10104 /** 10105 * i40e_reserve_msix_vectors - Reserve MSI-X vectors in the kernel 10106 * @pf: board private structure 10107 * @vectors: the number of MSI-X vectors to request 10108 * 10109 * Returns the number of vectors reserved, or error 10110 **/ 10111 static int i40e_reserve_msix_vectors(struct i40e_pf *pf, int vectors) 10112 { 10113 vectors = pci_enable_msix_range(pf->pdev, pf->msix_entries, 10114 I40E_MIN_MSIX, vectors); 10115 if (vectors < 0) { 10116 dev_info(&pf->pdev->dev, 10117 "MSI-X vector reservation failed: %d\n", vectors); 10118 vectors = 0; 10119 } 10120 10121 return vectors; 10122 } 10123 10124 /** 10125 * i40e_init_msix - Setup the MSIX capability 10126 * @pf: board private structure 10127 * 10128 * Work with the OS to set up the MSIX vectors needed. 10129 * 10130 * Returns the number of vectors reserved or negative on failure 10131 **/ 10132 static int i40e_init_msix(struct i40e_pf *pf) 10133 { 10134 struct i40e_hw *hw = &pf->hw; 10135 int cpus, extra_vectors; 10136 int vectors_left; 10137 int v_budget, i; 10138 int v_actual; 10139 int iwarp_requested = 0; 10140 10141 if (!(pf->flags & I40E_FLAG_MSIX_ENABLED)) 10142 return -ENODEV; 10143 10144 /* The number of vectors we'll request will be comprised of: 10145 * - Add 1 for "other" cause for Admin Queue events, etc. 10146 * - The number of LAN queue pairs 10147 * - Queues being used for RSS. 10148 * We don't need as many as max_rss_size vectors. 10149 * use rss_size instead in the calculation since that 10150 * is governed by number of cpus in the system. 10151 * - assumes symmetric Tx/Rx pairing 10152 * - The number of VMDq pairs 10153 * - The CPU count within the NUMA node if iWARP is enabled 10154 * Once we count this up, try the request. 10155 * 10156 * If we can't get what we want, we'll simplify to nearly nothing 10157 * and try again. If that still fails, we punt. 10158 */ 10159 vectors_left = hw->func_caps.num_msix_vectors; 10160 v_budget = 0; 10161 10162 /* reserve one vector for miscellaneous handler */ 10163 if (vectors_left) { 10164 v_budget++; 10165 vectors_left--; 10166 } 10167 10168 /* reserve some vectors for the main PF traffic queues. Initially we 10169 * only reserve at most 50% of the available vectors, in the case that 10170 * the number of online CPUs is large. This ensures that we can enable 10171 * extra features as well. Once we've enabled the other features, we 10172 * will use any remaining vectors to reach as close as we can to the 10173 * number of online CPUs. 10174 */ 10175 cpus = num_online_cpus(); 10176 pf->num_lan_msix = min_t(int, cpus, vectors_left / 2); 10177 vectors_left -= pf->num_lan_msix; 10178 10179 /* reserve one vector for sideband flow director */ 10180 if (pf->flags & I40E_FLAG_FD_SB_ENABLED) { 10181 if (vectors_left) { 10182 pf->num_fdsb_msix = 1; 10183 v_budget++; 10184 vectors_left--; 10185 } else { 10186 pf->num_fdsb_msix = 0; 10187 } 10188 } 10189 10190 /* can we reserve enough for iWARP? */ 10191 if (pf->flags & I40E_FLAG_IWARP_ENABLED) { 10192 iwarp_requested = pf->num_iwarp_msix; 10193 10194 if (!vectors_left) 10195 pf->num_iwarp_msix = 0; 10196 else if (vectors_left < pf->num_iwarp_msix) 10197 pf->num_iwarp_msix = 1; 10198 v_budget += pf->num_iwarp_msix; 10199 vectors_left -= pf->num_iwarp_msix; 10200 } 10201 10202 /* any vectors left over go for VMDq support */ 10203 if (pf->flags & I40E_FLAG_VMDQ_ENABLED) { 10204 int vmdq_vecs_wanted = pf->num_vmdq_vsis * pf->num_vmdq_qps; 10205 int vmdq_vecs = min_t(int, vectors_left, vmdq_vecs_wanted); 10206 10207 if (!vectors_left) { 10208 pf->num_vmdq_msix = 0; 10209 pf->num_vmdq_qps = 0; 10210 } else { 10211 /* if we're short on vectors for what's desired, we limit 10212 * the queues per vmdq. If this is still more than are 10213 * available, the user will need to change the number of 10214 * queues/vectors used by the PF later with the ethtool 10215 * channels command 10216 */ 10217 if (vmdq_vecs < vmdq_vecs_wanted) 10218 pf->num_vmdq_qps = 1; 10219 pf->num_vmdq_msix = pf->num_vmdq_qps; 10220 10221 v_budget += vmdq_vecs; 10222 vectors_left -= vmdq_vecs; 10223 } 10224 } 10225 10226 /* On systems with a large number of SMP cores, we previously limited 10227 * the number of vectors for num_lan_msix to be at most 50% of the 10228 * available vectors, to allow for other features. Now, we add back 10229 * the remaining vectors. However, we ensure that the total 10230 * num_lan_msix will not exceed num_online_cpus(). To do this, we 10231 * calculate the number of vectors we can add without going over the 10232 * cap of CPUs. For systems with a small number of CPUs this will be 10233 * zero. 10234 */ 10235 extra_vectors = min_t(int, cpus - pf->num_lan_msix, vectors_left); 10236 pf->num_lan_msix += extra_vectors; 10237 vectors_left -= extra_vectors; 10238 10239 WARN(vectors_left < 0, 10240 "Calculation of remaining vectors underflowed. This is an accounting bug when determining total MSI-X vectors.\n"); 10241 10242 v_budget += pf->num_lan_msix; 10243 pf->msix_entries = kcalloc(v_budget, sizeof(struct msix_entry), 10244 GFP_KERNEL); 10245 if (!pf->msix_entries) 10246 return -ENOMEM; 10247 10248 for (i = 0; i < v_budget; i++) 10249 pf->msix_entries[i].entry = i; 10250 v_actual = i40e_reserve_msix_vectors(pf, v_budget); 10251 10252 if (v_actual < I40E_MIN_MSIX) { 10253 pf->flags &= ~I40E_FLAG_MSIX_ENABLED; 10254 kfree(pf->msix_entries); 10255 pf->msix_entries = NULL; 10256 pci_disable_msix(pf->pdev); 10257 return -ENODEV; 10258 10259 } else if (v_actual == I40E_MIN_MSIX) { 10260 /* Adjust for minimal MSIX use */ 10261 pf->num_vmdq_vsis = 0; 10262 pf->num_vmdq_qps = 0; 10263 pf->num_lan_qps = 1; 10264 pf->num_lan_msix = 1; 10265 10266 } else if (v_actual != v_budget) { 10267 /* If we have limited resources, we will start with no vectors 10268 * for the special features and then allocate vectors to some 10269 * of these features based on the policy and at the end disable 10270 * the features that did not get any vectors. 10271 */ 10272 int vec; 10273 10274 dev_info(&pf->pdev->dev, 10275 "MSI-X vector limit reached with %d, wanted %d, attempting to redistribute vectors\n", 10276 v_actual, v_budget); 10277 /* reserve the misc vector */ 10278 vec = v_actual - 1; 10279 10280 /* Scale vector usage down */ 10281 pf->num_vmdq_msix = 1; /* force VMDqs to only one vector */ 10282 pf->num_vmdq_vsis = 1; 10283 pf->num_vmdq_qps = 1; 10284 10285 /* partition out the remaining vectors */ 10286 switch (vec) { 10287 case 2: 10288 pf->num_lan_msix = 1; 10289 break; 10290 case 3: 10291 if (pf->flags & I40E_FLAG_IWARP_ENABLED) { 10292 pf->num_lan_msix = 1; 10293 pf->num_iwarp_msix = 1; 10294 } else { 10295 pf->num_lan_msix = 2; 10296 } 10297 break; 10298 default: 10299 if (pf->flags & I40E_FLAG_IWARP_ENABLED) { 10300 pf->num_iwarp_msix = min_t(int, (vec / 3), 10301 iwarp_requested); 10302 pf->num_vmdq_vsis = min_t(int, (vec / 3), 10303 I40E_DEFAULT_NUM_VMDQ_VSI); 10304 } else { 10305 pf->num_vmdq_vsis = min_t(int, (vec / 2), 10306 I40E_DEFAULT_NUM_VMDQ_VSI); 10307 } 10308 if (pf->flags & I40E_FLAG_FD_SB_ENABLED) { 10309 pf->num_fdsb_msix = 1; 10310 vec--; 10311 } 10312 pf->num_lan_msix = min_t(int, 10313 (vec - (pf->num_iwarp_msix + pf->num_vmdq_vsis)), 10314 pf->num_lan_msix); 10315 pf->num_lan_qps = pf->num_lan_msix; 10316 break; 10317 } 10318 } 10319 10320 if ((pf->flags & I40E_FLAG_FD_SB_ENABLED) && 10321 (pf->num_fdsb_msix == 0)) { 10322 dev_info(&pf->pdev->dev, "Sideband Flowdir disabled, not enough MSI-X vectors\n"); 10323 pf->flags &= ~I40E_FLAG_FD_SB_ENABLED; 10324 pf->flags |= I40E_FLAG_FD_SB_INACTIVE; 10325 } 10326 if ((pf->flags & I40E_FLAG_VMDQ_ENABLED) && 10327 (pf->num_vmdq_msix == 0)) { 10328 dev_info(&pf->pdev->dev, "VMDq disabled, not enough MSI-X vectors\n"); 10329 pf->flags &= ~I40E_FLAG_VMDQ_ENABLED; 10330 } 10331 10332 if ((pf->flags & I40E_FLAG_IWARP_ENABLED) && 10333 (pf->num_iwarp_msix == 0)) { 10334 dev_info(&pf->pdev->dev, "IWARP disabled, not enough MSI-X vectors\n"); 10335 pf->flags &= ~I40E_FLAG_IWARP_ENABLED; 10336 } 10337 i40e_debug(&pf->hw, I40E_DEBUG_INIT, 10338 "MSI-X vector distribution: PF %d, VMDq %d, FDSB %d, iWARP %d\n", 10339 pf->num_lan_msix, 10340 pf->num_vmdq_msix * pf->num_vmdq_vsis, 10341 pf->num_fdsb_msix, 10342 pf->num_iwarp_msix); 10343 10344 return v_actual; 10345 } 10346 10347 /** 10348 * i40e_vsi_alloc_q_vector - Allocate memory for a single interrupt vector 10349 * @vsi: the VSI being configured 10350 * @v_idx: index of the vector in the vsi struct 10351 * @cpu: cpu to be used on affinity_mask 10352 * 10353 * We allocate one q_vector. If allocation fails we return -ENOMEM. 10354 **/ 10355 static int i40e_vsi_alloc_q_vector(struct i40e_vsi *vsi, int v_idx, int cpu) 10356 { 10357 struct i40e_q_vector *q_vector; 10358 10359 /* allocate q_vector */ 10360 q_vector = kzalloc(sizeof(struct i40e_q_vector), GFP_KERNEL); 10361 if (!q_vector) 10362 return -ENOMEM; 10363 10364 q_vector->vsi = vsi; 10365 q_vector->v_idx = v_idx; 10366 cpumask_copy(&q_vector->affinity_mask, cpu_possible_mask); 10367 10368 if (vsi->netdev) 10369 netif_napi_add(vsi->netdev, &q_vector->napi, 10370 i40e_napi_poll, NAPI_POLL_WEIGHT); 10371 10372 q_vector->rx.latency_range = I40E_LOW_LATENCY; 10373 q_vector->tx.latency_range = I40E_LOW_LATENCY; 10374 10375 /* tie q_vector and vsi together */ 10376 vsi->q_vectors[v_idx] = q_vector; 10377 10378 return 0; 10379 } 10380 10381 /** 10382 * i40e_vsi_alloc_q_vectors - Allocate memory for interrupt vectors 10383 * @vsi: the VSI being configured 10384 * 10385 * We allocate one q_vector per queue interrupt. If allocation fails we 10386 * return -ENOMEM. 10387 **/ 10388 static int i40e_vsi_alloc_q_vectors(struct i40e_vsi *vsi) 10389 { 10390 struct i40e_pf *pf = vsi->back; 10391 int err, v_idx, num_q_vectors, current_cpu; 10392 10393 /* if not MSIX, give the one vector only to the LAN VSI */ 10394 if (pf->flags & I40E_FLAG_MSIX_ENABLED) 10395 num_q_vectors = vsi->num_q_vectors; 10396 else if (vsi == pf->vsi[pf->lan_vsi]) 10397 num_q_vectors = 1; 10398 else 10399 return -EINVAL; 10400 10401 current_cpu = cpumask_first(cpu_online_mask); 10402 10403 for (v_idx = 0; v_idx < num_q_vectors; v_idx++) { 10404 err = i40e_vsi_alloc_q_vector(vsi, v_idx, current_cpu); 10405 if (err) 10406 goto err_out; 10407 current_cpu = cpumask_next(current_cpu, cpu_online_mask); 10408 if (unlikely(current_cpu >= nr_cpu_ids)) 10409 current_cpu = cpumask_first(cpu_online_mask); 10410 } 10411 10412 return 0; 10413 10414 err_out: 10415 while (v_idx--) 10416 i40e_free_q_vector(vsi, v_idx); 10417 10418 return err; 10419 } 10420 10421 /** 10422 * i40e_init_interrupt_scheme - Determine proper interrupt scheme 10423 * @pf: board private structure to initialize 10424 **/ 10425 static int i40e_init_interrupt_scheme(struct i40e_pf *pf) 10426 { 10427 int vectors = 0; 10428 ssize_t size; 10429 10430 if (pf->flags & I40E_FLAG_MSIX_ENABLED) { 10431 vectors = i40e_init_msix(pf); 10432 if (vectors < 0) { 10433 pf->flags &= ~(I40E_FLAG_MSIX_ENABLED | 10434 I40E_FLAG_IWARP_ENABLED | 10435 I40E_FLAG_RSS_ENABLED | 10436 I40E_FLAG_DCB_CAPABLE | 10437 I40E_FLAG_DCB_ENABLED | 10438 I40E_FLAG_SRIOV_ENABLED | 10439 I40E_FLAG_FD_SB_ENABLED | 10440 I40E_FLAG_FD_ATR_ENABLED | 10441 I40E_FLAG_VMDQ_ENABLED); 10442 pf->flags |= I40E_FLAG_FD_SB_INACTIVE; 10443 10444 /* rework the queue expectations without MSIX */ 10445 i40e_determine_queue_usage(pf); 10446 } 10447 } 10448 10449 if (!(pf->flags & I40E_FLAG_MSIX_ENABLED) && 10450 (pf->flags & I40E_FLAG_MSI_ENABLED)) { 10451 dev_info(&pf->pdev->dev, "MSI-X not available, trying MSI\n"); 10452 vectors = pci_enable_msi(pf->pdev); 10453 if (vectors < 0) { 10454 dev_info(&pf->pdev->dev, "MSI init failed - %d\n", 10455 vectors); 10456 pf->flags &= ~I40E_FLAG_MSI_ENABLED; 10457 } 10458 vectors = 1; /* one MSI or Legacy vector */ 10459 } 10460 10461 if (!(pf->flags & (I40E_FLAG_MSIX_ENABLED | I40E_FLAG_MSI_ENABLED))) 10462 dev_info(&pf->pdev->dev, "MSI-X and MSI not available, falling back to Legacy IRQ\n"); 10463 10464 /* set up vector assignment tracking */ 10465 size = sizeof(struct i40e_lump_tracking) + (sizeof(u16) * vectors); 10466 pf->irq_pile = kzalloc(size, GFP_KERNEL); 10467 if (!pf->irq_pile) { 10468 dev_err(&pf->pdev->dev, "error allocating irq_pile memory\n"); 10469 return -ENOMEM; 10470 } 10471 pf->irq_pile->num_entries = vectors; 10472 pf->irq_pile->search_hint = 0; 10473 10474 /* track first vector for misc interrupts, ignore return */ 10475 (void)i40e_get_lump(pf, pf->irq_pile, 1, I40E_PILE_VALID_BIT - 1); 10476 10477 return 0; 10478 } 10479 10480 /** 10481 * i40e_restore_interrupt_scheme - Restore the interrupt scheme 10482 * @pf: private board data structure 10483 * 10484 * Restore the interrupt scheme that was cleared when we suspended the 10485 * device. This should be called during resume to re-allocate the q_vectors 10486 * and reacquire IRQs. 10487 */ 10488 static int i40e_restore_interrupt_scheme(struct i40e_pf *pf) 10489 { 10490 int err, i; 10491 10492 /* We cleared the MSI and MSI-X flags when disabling the old interrupt 10493 * scheme. We need to re-enabled them here in order to attempt to 10494 * re-acquire the MSI or MSI-X vectors 10495 */ 10496 pf->flags |= (I40E_FLAG_MSIX_ENABLED | I40E_FLAG_MSI_ENABLED); 10497 10498 err = i40e_init_interrupt_scheme(pf); 10499 if (err) 10500 return err; 10501 10502 /* Now that we've re-acquired IRQs, we need to remap the vectors and 10503 * rings together again. 10504 */ 10505 for (i = 0; i < pf->num_alloc_vsi; i++) { 10506 if (pf->vsi[i]) { 10507 err = i40e_vsi_alloc_q_vectors(pf->vsi[i]); 10508 if (err) 10509 goto err_unwind; 10510 i40e_vsi_map_rings_to_vectors(pf->vsi[i]); 10511 } 10512 } 10513 10514 err = i40e_setup_misc_vector(pf); 10515 if (err) 10516 goto err_unwind; 10517 10518 return 0; 10519 10520 err_unwind: 10521 while (i--) { 10522 if (pf->vsi[i]) 10523 i40e_vsi_free_q_vectors(pf->vsi[i]); 10524 } 10525 10526 return err; 10527 } 10528 10529 /** 10530 * i40e_setup_misc_vector - Setup the misc vector to handle non queue events 10531 * @pf: board private structure 10532 * 10533 * This sets up the handler for MSIX 0, which is used to manage the 10534 * non-queue interrupts, e.g. AdminQ and errors. This is not used 10535 * when in MSI or Legacy interrupt mode. 10536 **/ 10537 static int i40e_setup_misc_vector(struct i40e_pf *pf) 10538 { 10539 struct i40e_hw *hw = &pf->hw; 10540 int err = 0; 10541 10542 /* Only request the IRQ once, the first time through. */ 10543 if (!test_and_set_bit(__I40E_MISC_IRQ_REQUESTED, pf->state)) { 10544 err = request_irq(pf->msix_entries[0].vector, 10545 i40e_intr, 0, pf->int_name, pf); 10546 if (err) { 10547 clear_bit(__I40E_MISC_IRQ_REQUESTED, pf->state); 10548 dev_info(&pf->pdev->dev, 10549 "request_irq for %s failed: %d\n", 10550 pf->int_name, err); 10551 return -EFAULT; 10552 } 10553 } 10554 10555 i40e_enable_misc_int_causes(pf); 10556 10557 /* associate no queues to the misc vector */ 10558 wr32(hw, I40E_PFINT_LNKLST0, I40E_QUEUE_END_OF_LIST); 10559 wr32(hw, I40E_PFINT_ITR0(I40E_RX_ITR), I40E_ITR_8K); 10560 10561 i40e_flush(hw); 10562 10563 i40e_irq_dynamic_enable_icr0(pf); 10564 10565 return err; 10566 } 10567 10568 /** 10569 * i40e_get_rss_aq - Get RSS keys and lut by using AQ commands 10570 * @vsi: Pointer to vsi structure 10571 * @seed: Buffter to store the hash keys 10572 * @lut: Buffer to store the lookup table entries 10573 * @lut_size: Size of buffer to store the lookup table entries 10574 * 10575 * Return 0 on success, negative on failure 10576 */ 10577 static int i40e_get_rss_aq(struct i40e_vsi *vsi, const u8 *seed, 10578 u8 *lut, u16 lut_size) 10579 { 10580 struct i40e_pf *pf = vsi->back; 10581 struct i40e_hw *hw = &pf->hw; 10582 int ret = 0; 10583 10584 if (seed) { 10585 ret = i40e_aq_get_rss_key(hw, vsi->id, 10586 (struct i40e_aqc_get_set_rss_key_data *)seed); 10587 if (ret) { 10588 dev_info(&pf->pdev->dev, 10589 "Cannot get RSS key, err %s aq_err %s\n", 10590 i40e_stat_str(&pf->hw, ret), 10591 i40e_aq_str(&pf->hw, 10592 pf->hw.aq.asq_last_status)); 10593 return ret; 10594 } 10595 } 10596 10597 if (lut) { 10598 bool pf_lut = vsi->type == I40E_VSI_MAIN ? true : false; 10599 10600 ret = i40e_aq_get_rss_lut(hw, vsi->id, pf_lut, lut, lut_size); 10601 if (ret) { 10602 dev_info(&pf->pdev->dev, 10603 "Cannot get RSS lut, err %s aq_err %s\n", 10604 i40e_stat_str(&pf->hw, ret), 10605 i40e_aq_str(&pf->hw, 10606 pf->hw.aq.asq_last_status)); 10607 return ret; 10608 } 10609 } 10610 10611 return ret; 10612 } 10613 10614 /** 10615 * i40e_config_rss_reg - Configure RSS keys and lut by writing registers 10616 * @vsi: Pointer to vsi structure 10617 * @seed: RSS hash seed 10618 * @lut: Lookup table 10619 * @lut_size: Lookup table size 10620 * 10621 * Returns 0 on success, negative on failure 10622 **/ 10623 static int i40e_config_rss_reg(struct i40e_vsi *vsi, const u8 *seed, 10624 const u8 *lut, u16 lut_size) 10625 { 10626 struct i40e_pf *pf = vsi->back; 10627 struct i40e_hw *hw = &pf->hw; 10628 u16 vf_id = vsi->vf_id; 10629 u8 i; 10630 10631 /* Fill out hash function seed */ 10632 if (seed) { 10633 u32 *seed_dw = (u32 *)seed; 10634 10635 if (vsi->type == I40E_VSI_MAIN) { 10636 for (i = 0; i <= I40E_PFQF_HKEY_MAX_INDEX; i++) 10637 wr32(hw, I40E_PFQF_HKEY(i), seed_dw[i]); 10638 } else if (vsi->type == I40E_VSI_SRIOV) { 10639 for (i = 0; i <= I40E_VFQF_HKEY1_MAX_INDEX; i++) 10640 wr32(hw, I40E_VFQF_HKEY1(i, vf_id), seed_dw[i]); 10641 } else { 10642 dev_err(&pf->pdev->dev, "Cannot set RSS seed - invalid VSI type\n"); 10643 } 10644 } 10645 10646 if (lut) { 10647 u32 *lut_dw = (u32 *)lut; 10648 10649 if (vsi->type == I40E_VSI_MAIN) { 10650 if (lut_size != I40E_HLUT_ARRAY_SIZE) 10651 return -EINVAL; 10652 for (i = 0; i <= I40E_PFQF_HLUT_MAX_INDEX; i++) 10653 wr32(hw, I40E_PFQF_HLUT(i), lut_dw[i]); 10654 } else if (vsi->type == I40E_VSI_SRIOV) { 10655 if (lut_size != I40E_VF_HLUT_ARRAY_SIZE) 10656 return -EINVAL; 10657 for (i = 0; i <= I40E_VFQF_HLUT_MAX_INDEX; i++) 10658 wr32(hw, I40E_VFQF_HLUT1(i, vf_id), lut_dw[i]); 10659 } else { 10660 dev_err(&pf->pdev->dev, "Cannot set RSS LUT - invalid VSI type\n"); 10661 } 10662 } 10663 i40e_flush(hw); 10664 10665 return 0; 10666 } 10667 10668 /** 10669 * i40e_get_rss_reg - Get the RSS keys and lut by reading registers 10670 * @vsi: Pointer to VSI structure 10671 * @seed: Buffer to store the keys 10672 * @lut: Buffer to store the lookup table entries 10673 * @lut_size: Size of buffer to store the lookup table entries 10674 * 10675 * Returns 0 on success, negative on failure 10676 */ 10677 static int i40e_get_rss_reg(struct i40e_vsi *vsi, u8 *seed, 10678 u8 *lut, u16 lut_size) 10679 { 10680 struct i40e_pf *pf = vsi->back; 10681 struct i40e_hw *hw = &pf->hw; 10682 u16 i; 10683 10684 if (seed) { 10685 u32 *seed_dw = (u32 *)seed; 10686 10687 for (i = 0; i <= I40E_PFQF_HKEY_MAX_INDEX; i++) 10688 seed_dw[i] = i40e_read_rx_ctl(hw, I40E_PFQF_HKEY(i)); 10689 } 10690 if (lut) { 10691 u32 *lut_dw = (u32 *)lut; 10692 10693 if (lut_size != I40E_HLUT_ARRAY_SIZE) 10694 return -EINVAL; 10695 for (i = 0; i <= I40E_PFQF_HLUT_MAX_INDEX; i++) 10696 lut_dw[i] = rd32(hw, I40E_PFQF_HLUT(i)); 10697 } 10698 10699 return 0; 10700 } 10701 10702 /** 10703 * i40e_config_rss - Configure RSS keys and lut 10704 * @vsi: Pointer to VSI structure 10705 * @seed: RSS hash seed 10706 * @lut: Lookup table 10707 * @lut_size: Lookup table size 10708 * 10709 * Returns 0 on success, negative on failure 10710 */ 10711 int i40e_config_rss(struct i40e_vsi *vsi, u8 *seed, u8 *lut, u16 lut_size) 10712 { 10713 struct i40e_pf *pf = vsi->back; 10714 10715 if (pf->hw_features & I40E_HW_RSS_AQ_CAPABLE) 10716 return i40e_config_rss_aq(vsi, seed, lut, lut_size); 10717 else 10718 return i40e_config_rss_reg(vsi, seed, lut, lut_size); 10719 } 10720 10721 /** 10722 * i40e_get_rss - Get RSS keys and lut 10723 * @vsi: Pointer to VSI structure 10724 * @seed: Buffer to store the keys 10725 * @lut: Buffer to store the lookup table entries 10726 * lut_size: Size of buffer to store the lookup table entries 10727 * 10728 * Returns 0 on success, negative on failure 10729 */ 10730 int i40e_get_rss(struct i40e_vsi *vsi, u8 *seed, u8 *lut, u16 lut_size) 10731 { 10732 struct i40e_pf *pf = vsi->back; 10733 10734 if (pf->hw_features & I40E_HW_RSS_AQ_CAPABLE) 10735 return i40e_get_rss_aq(vsi, seed, lut, lut_size); 10736 else 10737 return i40e_get_rss_reg(vsi, seed, lut, lut_size); 10738 } 10739 10740 /** 10741 * i40e_fill_rss_lut - Fill the RSS lookup table with default values 10742 * @pf: Pointer to board private structure 10743 * @lut: Lookup table 10744 * @rss_table_size: Lookup table size 10745 * @rss_size: Range of queue number for hashing 10746 */ 10747 void i40e_fill_rss_lut(struct i40e_pf *pf, u8 *lut, 10748 u16 rss_table_size, u16 rss_size) 10749 { 10750 u16 i; 10751 10752 for (i = 0; i < rss_table_size; i++) 10753 lut[i] = i % rss_size; 10754 } 10755 10756 /** 10757 * i40e_pf_config_rss - Prepare for RSS if used 10758 * @pf: board private structure 10759 **/ 10760 static int i40e_pf_config_rss(struct i40e_pf *pf) 10761 { 10762 struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi]; 10763 u8 seed[I40E_HKEY_ARRAY_SIZE]; 10764 u8 *lut; 10765 struct i40e_hw *hw = &pf->hw; 10766 u32 reg_val; 10767 u64 hena; 10768 int ret; 10769 10770 /* By default we enable TCP/UDP with IPv4/IPv6 ptypes */ 10771 hena = (u64)i40e_read_rx_ctl(hw, I40E_PFQF_HENA(0)) | 10772 ((u64)i40e_read_rx_ctl(hw, I40E_PFQF_HENA(1)) << 32); 10773 hena |= i40e_pf_get_default_rss_hena(pf); 10774 10775 i40e_write_rx_ctl(hw, I40E_PFQF_HENA(0), (u32)hena); 10776 i40e_write_rx_ctl(hw, I40E_PFQF_HENA(1), (u32)(hena >> 32)); 10777 10778 /* Determine the RSS table size based on the hardware capabilities */ 10779 reg_val = i40e_read_rx_ctl(hw, I40E_PFQF_CTL_0); 10780 reg_val = (pf->rss_table_size == 512) ? 10781 (reg_val | I40E_PFQF_CTL_0_HASHLUTSIZE_512) : 10782 (reg_val & ~I40E_PFQF_CTL_0_HASHLUTSIZE_512); 10783 i40e_write_rx_ctl(hw, I40E_PFQF_CTL_0, reg_val); 10784 10785 /* Determine the RSS size of the VSI */ 10786 if (!vsi->rss_size) { 10787 u16 qcount; 10788 10789 qcount = vsi->num_queue_pairs / vsi->tc_config.numtc; 10790 vsi->rss_size = min_t(int, pf->alloc_rss_size, qcount); 10791 } 10792 if (!vsi->rss_size) 10793 return -EINVAL; 10794 10795 lut = kzalloc(vsi->rss_table_size, GFP_KERNEL); 10796 if (!lut) 10797 return -ENOMEM; 10798 10799 /* Use user configured lut if there is one, otherwise use default */ 10800 if (vsi->rss_lut_user) 10801 memcpy(lut, vsi->rss_lut_user, vsi->rss_table_size); 10802 else 10803 i40e_fill_rss_lut(pf, lut, vsi->rss_table_size, vsi->rss_size); 10804 10805 /* Use user configured hash key if there is one, otherwise 10806 * use default. 10807 */ 10808 if (vsi->rss_hkey_user) 10809 memcpy(seed, vsi->rss_hkey_user, I40E_HKEY_ARRAY_SIZE); 10810 else 10811 netdev_rss_key_fill((void *)seed, I40E_HKEY_ARRAY_SIZE); 10812 ret = i40e_config_rss(vsi, seed, lut, vsi->rss_table_size); 10813 kfree(lut); 10814 10815 return ret; 10816 } 10817 10818 /** 10819 * i40e_reconfig_rss_queues - change number of queues for rss and rebuild 10820 * @pf: board private structure 10821 * @queue_count: the requested queue count for rss. 10822 * 10823 * returns 0 if rss is not enabled, if enabled returns the final rss queue 10824 * count which may be different from the requested queue count. 10825 * Note: expects to be called while under rtnl_lock() 10826 **/ 10827 int i40e_reconfig_rss_queues(struct i40e_pf *pf, int queue_count) 10828 { 10829 struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi]; 10830 int new_rss_size; 10831 10832 if (!(pf->flags & I40E_FLAG_RSS_ENABLED)) 10833 return 0; 10834 10835 new_rss_size = min_t(int, queue_count, pf->rss_size_max); 10836 10837 if (queue_count != vsi->num_queue_pairs) { 10838 u16 qcount; 10839 10840 vsi->req_queue_pairs = queue_count; 10841 i40e_prep_for_reset(pf, true); 10842 10843 pf->alloc_rss_size = new_rss_size; 10844 10845 i40e_reset_and_rebuild(pf, true, true); 10846 10847 /* Discard the user configured hash keys and lut, if less 10848 * queues are enabled. 10849 */ 10850 if (queue_count < vsi->rss_size) { 10851 i40e_clear_rss_config_user(vsi); 10852 dev_dbg(&pf->pdev->dev, 10853 "discard user configured hash keys and lut\n"); 10854 } 10855 10856 /* Reset vsi->rss_size, as number of enabled queues changed */ 10857 qcount = vsi->num_queue_pairs / vsi->tc_config.numtc; 10858 vsi->rss_size = min_t(int, pf->alloc_rss_size, qcount); 10859 10860 i40e_pf_config_rss(pf); 10861 } 10862 dev_info(&pf->pdev->dev, "User requested queue count/HW max RSS count: %d/%d\n", 10863 vsi->req_queue_pairs, pf->rss_size_max); 10864 return pf->alloc_rss_size; 10865 } 10866 10867 /** 10868 * i40e_get_partition_bw_setting - Retrieve BW settings for this PF partition 10869 * @pf: board private structure 10870 **/ 10871 i40e_status i40e_get_partition_bw_setting(struct i40e_pf *pf) 10872 { 10873 i40e_status status; 10874 bool min_valid, max_valid; 10875 u32 max_bw, min_bw; 10876 10877 status = i40e_read_bw_from_alt_ram(&pf->hw, &max_bw, &min_bw, 10878 &min_valid, &max_valid); 10879 10880 if (!status) { 10881 if (min_valid) 10882 pf->min_bw = min_bw; 10883 if (max_valid) 10884 pf->max_bw = max_bw; 10885 } 10886 10887 return status; 10888 } 10889 10890 /** 10891 * i40e_set_partition_bw_setting - Set BW settings for this PF partition 10892 * @pf: board private structure 10893 **/ 10894 i40e_status i40e_set_partition_bw_setting(struct i40e_pf *pf) 10895 { 10896 struct i40e_aqc_configure_partition_bw_data bw_data; 10897 i40e_status status; 10898 10899 /* Set the valid bit for this PF */ 10900 bw_data.pf_valid_bits = cpu_to_le16(BIT(pf->hw.pf_id)); 10901 bw_data.max_bw[pf->hw.pf_id] = pf->max_bw & I40E_ALT_BW_VALUE_MASK; 10902 bw_data.min_bw[pf->hw.pf_id] = pf->min_bw & I40E_ALT_BW_VALUE_MASK; 10903 10904 /* Set the new bandwidths */ 10905 status = i40e_aq_configure_partition_bw(&pf->hw, &bw_data, NULL); 10906 10907 return status; 10908 } 10909 10910 /** 10911 * i40e_commit_partition_bw_setting - Commit BW settings for this PF partition 10912 * @pf: board private structure 10913 **/ 10914 i40e_status i40e_commit_partition_bw_setting(struct i40e_pf *pf) 10915 { 10916 /* Commit temporary BW setting to permanent NVM image */ 10917 enum i40e_admin_queue_err last_aq_status; 10918 i40e_status ret; 10919 u16 nvm_word; 10920 10921 if (pf->hw.partition_id != 1) { 10922 dev_info(&pf->pdev->dev, 10923 "Commit BW only works on partition 1! This is partition %d", 10924 pf->hw.partition_id); 10925 ret = I40E_NOT_SUPPORTED; 10926 goto bw_commit_out; 10927 } 10928 10929 /* Acquire NVM for read access */ 10930 ret = i40e_acquire_nvm(&pf->hw, I40E_RESOURCE_READ); 10931 last_aq_status = pf->hw.aq.asq_last_status; 10932 if (ret) { 10933 dev_info(&pf->pdev->dev, 10934 "Cannot acquire NVM for read access, err %s aq_err %s\n", 10935 i40e_stat_str(&pf->hw, ret), 10936 i40e_aq_str(&pf->hw, last_aq_status)); 10937 goto bw_commit_out; 10938 } 10939 10940 /* Read word 0x10 of NVM - SW compatibility word 1 */ 10941 ret = i40e_aq_read_nvm(&pf->hw, 10942 I40E_SR_NVM_CONTROL_WORD, 10943 0x10, sizeof(nvm_word), &nvm_word, 10944 false, NULL); 10945 /* Save off last admin queue command status before releasing 10946 * the NVM 10947 */ 10948 last_aq_status = pf->hw.aq.asq_last_status; 10949 i40e_release_nvm(&pf->hw); 10950 if (ret) { 10951 dev_info(&pf->pdev->dev, "NVM read error, err %s aq_err %s\n", 10952 i40e_stat_str(&pf->hw, ret), 10953 i40e_aq_str(&pf->hw, last_aq_status)); 10954 goto bw_commit_out; 10955 } 10956 10957 /* Wait a bit for NVM release to complete */ 10958 msleep(50); 10959 10960 /* Acquire NVM for write access */ 10961 ret = i40e_acquire_nvm(&pf->hw, I40E_RESOURCE_WRITE); 10962 last_aq_status = pf->hw.aq.asq_last_status; 10963 if (ret) { 10964 dev_info(&pf->pdev->dev, 10965 "Cannot acquire NVM for write access, err %s aq_err %s\n", 10966 i40e_stat_str(&pf->hw, ret), 10967 i40e_aq_str(&pf->hw, last_aq_status)); 10968 goto bw_commit_out; 10969 } 10970 /* Write it back out unchanged to initiate update NVM, 10971 * which will force a write of the shadow (alt) RAM to 10972 * the NVM - thus storing the bandwidth values permanently. 10973 */ 10974 ret = i40e_aq_update_nvm(&pf->hw, 10975 I40E_SR_NVM_CONTROL_WORD, 10976 0x10, sizeof(nvm_word), 10977 &nvm_word, true, NULL); 10978 /* Save off last admin queue command status before releasing 10979 * the NVM 10980 */ 10981 last_aq_status = pf->hw.aq.asq_last_status; 10982 i40e_release_nvm(&pf->hw); 10983 if (ret) 10984 dev_info(&pf->pdev->dev, 10985 "BW settings NOT SAVED, err %s aq_err %s\n", 10986 i40e_stat_str(&pf->hw, ret), 10987 i40e_aq_str(&pf->hw, last_aq_status)); 10988 bw_commit_out: 10989 10990 return ret; 10991 } 10992 10993 /** 10994 * i40e_sw_init - Initialize general software structures (struct i40e_pf) 10995 * @pf: board private structure to initialize 10996 * 10997 * i40e_sw_init initializes the Adapter private data structure. 10998 * Fields are initialized based on PCI device information and 10999 * OS network device settings (MTU size). 11000 **/ 11001 static int i40e_sw_init(struct i40e_pf *pf) 11002 { 11003 int err = 0; 11004 int size; 11005 11006 /* Set default capability flags */ 11007 pf->flags = I40E_FLAG_RX_CSUM_ENABLED | 11008 I40E_FLAG_MSI_ENABLED | 11009 I40E_FLAG_MSIX_ENABLED; 11010 11011 /* Set default ITR */ 11012 pf->rx_itr_default = I40E_ITR_RX_DEF; 11013 pf->tx_itr_default = I40E_ITR_TX_DEF; 11014 11015 /* Depending on PF configurations, it is possible that the RSS 11016 * maximum might end up larger than the available queues 11017 */ 11018 pf->rss_size_max = BIT(pf->hw.func_caps.rss_table_entry_width); 11019 pf->alloc_rss_size = 1; 11020 pf->rss_table_size = pf->hw.func_caps.rss_table_size; 11021 pf->rss_size_max = min_t(int, pf->rss_size_max, 11022 pf->hw.func_caps.num_tx_qp); 11023 if (pf->hw.func_caps.rss) { 11024 pf->flags |= I40E_FLAG_RSS_ENABLED; 11025 pf->alloc_rss_size = min_t(int, pf->rss_size_max, 11026 num_online_cpus()); 11027 } 11028 11029 /* MFP mode enabled */ 11030 if (pf->hw.func_caps.npar_enable || pf->hw.func_caps.flex10_enable) { 11031 pf->flags |= I40E_FLAG_MFP_ENABLED; 11032 dev_info(&pf->pdev->dev, "MFP mode Enabled\n"); 11033 if (i40e_get_partition_bw_setting(pf)) { 11034 dev_warn(&pf->pdev->dev, 11035 "Could not get partition bw settings\n"); 11036 } else { 11037 dev_info(&pf->pdev->dev, 11038 "Partition BW Min = %8.8x, Max = %8.8x\n", 11039 pf->min_bw, pf->max_bw); 11040 11041 /* nudge the Tx scheduler */ 11042 i40e_set_partition_bw_setting(pf); 11043 } 11044 } 11045 11046 if ((pf->hw.func_caps.fd_filters_guaranteed > 0) || 11047 (pf->hw.func_caps.fd_filters_best_effort > 0)) { 11048 pf->flags |= I40E_FLAG_FD_ATR_ENABLED; 11049 pf->atr_sample_rate = I40E_DEFAULT_ATR_SAMPLE_RATE; 11050 if (pf->flags & I40E_FLAG_MFP_ENABLED && 11051 pf->hw.num_partitions > 1) 11052 dev_info(&pf->pdev->dev, 11053 "Flow Director Sideband mode Disabled in MFP mode\n"); 11054 else 11055 pf->flags |= I40E_FLAG_FD_SB_ENABLED; 11056 pf->fdir_pf_filter_count = 11057 pf->hw.func_caps.fd_filters_guaranteed; 11058 pf->hw.fdir_shared_filter_count = 11059 pf->hw.func_caps.fd_filters_best_effort; 11060 } 11061 11062 if (pf->hw.mac.type == I40E_MAC_X722) { 11063 pf->hw_features |= (I40E_HW_RSS_AQ_CAPABLE | 11064 I40E_HW_128_QP_RSS_CAPABLE | 11065 I40E_HW_ATR_EVICT_CAPABLE | 11066 I40E_HW_WB_ON_ITR_CAPABLE | 11067 I40E_HW_MULTIPLE_TCP_UDP_RSS_PCTYPE | 11068 I40E_HW_NO_PCI_LINK_CHECK | 11069 I40E_HW_USE_SET_LLDP_MIB | 11070 I40E_HW_GENEVE_OFFLOAD_CAPABLE | 11071 I40E_HW_PTP_L4_CAPABLE | 11072 I40E_HW_WOL_MC_MAGIC_PKT_WAKE | 11073 I40E_HW_OUTER_UDP_CSUM_CAPABLE); 11074 11075 #define I40E_FDEVICT_PCTYPE_DEFAULT 0xc03 11076 if (rd32(&pf->hw, I40E_GLQF_FDEVICTENA(1)) != 11077 I40E_FDEVICT_PCTYPE_DEFAULT) { 11078 dev_warn(&pf->pdev->dev, 11079 "FD EVICT PCTYPES are not right, disable FD HW EVICT\n"); 11080 pf->hw_features &= ~I40E_HW_ATR_EVICT_CAPABLE; 11081 } 11082 } else if ((pf->hw.aq.api_maj_ver > 1) || 11083 ((pf->hw.aq.api_maj_ver == 1) && 11084 (pf->hw.aq.api_min_ver > 4))) { 11085 /* Supported in FW API version higher than 1.4 */ 11086 pf->hw_features |= I40E_HW_GENEVE_OFFLOAD_CAPABLE; 11087 } 11088 11089 /* Enable HW ATR eviction if possible */ 11090 if (pf->hw_features & I40E_HW_ATR_EVICT_CAPABLE) 11091 pf->flags |= I40E_FLAG_HW_ATR_EVICT_ENABLED; 11092 11093 if ((pf->hw.mac.type == I40E_MAC_XL710) && 11094 (((pf->hw.aq.fw_maj_ver == 4) && (pf->hw.aq.fw_min_ver < 33)) || 11095 (pf->hw.aq.fw_maj_ver < 4))) { 11096 pf->hw_features |= I40E_HW_RESTART_AUTONEG; 11097 /* No DCB support for FW < v4.33 */ 11098 pf->hw_features |= I40E_HW_NO_DCB_SUPPORT; 11099 } 11100 11101 /* Disable FW LLDP if FW < v4.3 */ 11102 if ((pf->hw.mac.type == I40E_MAC_XL710) && 11103 (((pf->hw.aq.fw_maj_ver == 4) && (pf->hw.aq.fw_min_ver < 3)) || 11104 (pf->hw.aq.fw_maj_ver < 4))) 11105 pf->hw_features |= I40E_HW_STOP_FW_LLDP; 11106 11107 /* Use the FW Set LLDP MIB API if FW > v4.40 */ 11108 if ((pf->hw.mac.type == I40E_MAC_XL710) && 11109 (((pf->hw.aq.fw_maj_ver == 4) && (pf->hw.aq.fw_min_ver >= 40)) || 11110 (pf->hw.aq.fw_maj_ver >= 5))) 11111 pf->hw_features |= I40E_HW_USE_SET_LLDP_MIB; 11112 11113 /* Enable PTP L4 if FW > v6.0 */ 11114 if (pf->hw.mac.type == I40E_MAC_XL710 && 11115 pf->hw.aq.fw_maj_ver >= 6) 11116 pf->hw_features |= I40E_HW_PTP_L4_CAPABLE; 11117 11118 if (pf->hw.func_caps.vmdq) { 11119 pf->num_vmdq_vsis = I40E_DEFAULT_NUM_VMDQ_VSI; 11120 pf->flags |= I40E_FLAG_VMDQ_ENABLED; 11121 pf->num_vmdq_qps = i40e_default_queues_per_vmdq(pf); 11122 } 11123 11124 if (pf->hw.func_caps.iwarp) { 11125 pf->flags |= I40E_FLAG_IWARP_ENABLED; 11126 /* IWARP needs one extra vector for CQP just like MISC.*/ 11127 pf->num_iwarp_msix = (int)num_online_cpus() + 1; 11128 } 11129 11130 #ifdef CONFIG_PCI_IOV 11131 if (pf->hw.func_caps.num_vfs && pf->hw.partition_id == 1) { 11132 pf->num_vf_qps = I40E_DEFAULT_QUEUES_PER_VF; 11133 pf->flags |= I40E_FLAG_SRIOV_ENABLED; 11134 pf->num_req_vfs = min_t(int, 11135 pf->hw.func_caps.num_vfs, 11136 I40E_MAX_VF_COUNT); 11137 } 11138 #endif /* CONFIG_PCI_IOV */ 11139 pf->eeprom_version = 0xDEAD; 11140 pf->lan_veb = I40E_NO_VEB; 11141 pf->lan_vsi = I40E_NO_VSI; 11142 11143 /* By default FW has this off for performance reasons */ 11144 pf->flags &= ~I40E_FLAG_VEB_STATS_ENABLED; 11145 11146 /* set up queue assignment tracking */ 11147 size = sizeof(struct i40e_lump_tracking) 11148 + (sizeof(u16) * pf->hw.func_caps.num_tx_qp); 11149 pf->qp_pile = kzalloc(size, GFP_KERNEL); 11150 if (!pf->qp_pile) { 11151 err = -ENOMEM; 11152 goto sw_init_done; 11153 } 11154 pf->qp_pile->num_entries = pf->hw.func_caps.num_tx_qp; 11155 pf->qp_pile->search_hint = 0; 11156 11157 pf->tx_timeout_recovery_level = 1; 11158 11159 mutex_init(&pf->switch_mutex); 11160 11161 sw_init_done: 11162 return err; 11163 } 11164 11165 /** 11166 * i40e_set_ntuple - set the ntuple feature flag and take action 11167 * @pf: board private structure to initialize 11168 * @features: the feature set that the stack is suggesting 11169 * 11170 * returns a bool to indicate if reset needs to happen 11171 **/ 11172 bool i40e_set_ntuple(struct i40e_pf *pf, netdev_features_t features) 11173 { 11174 bool need_reset = false; 11175 11176 /* Check if Flow Director n-tuple support was enabled or disabled. If 11177 * the state changed, we need to reset. 11178 */ 11179 if (features & NETIF_F_NTUPLE) { 11180 /* Enable filters and mark for reset */ 11181 if (!(pf->flags & I40E_FLAG_FD_SB_ENABLED)) 11182 need_reset = true; 11183 /* enable FD_SB only if there is MSI-X vector and no cloud 11184 * filters exist 11185 */ 11186 if (pf->num_fdsb_msix > 0 && !pf->num_cloud_filters) { 11187 pf->flags |= I40E_FLAG_FD_SB_ENABLED; 11188 pf->flags &= ~I40E_FLAG_FD_SB_INACTIVE; 11189 } 11190 } else { 11191 /* turn off filters, mark for reset and clear SW filter list */ 11192 if (pf->flags & I40E_FLAG_FD_SB_ENABLED) { 11193 need_reset = true; 11194 i40e_fdir_filter_exit(pf); 11195 } 11196 pf->flags &= ~(I40E_FLAG_FD_SB_ENABLED | 11197 I40E_FLAG_FD_SB_AUTO_DISABLED); 11198 pf->flags |= I40E_FLAG_FD_SB_INACTIVE; 11199 11200 /* reset fd counters */ 11201 pf->fd_add_err = 0; 11202 pf->fd_atr_cnt = 0; 11203 /* if ATR was auto disabled it can be re-enabled. */ 11204 if (pf->flags & I40E_FLAG_FD_ATR_AUTO_DISABLED) { 11205 pf->flags &= ~I40E_FLAG_FD_ATR_AUTO_DISABLED; 11206 if ((pf->flags & I40E_FLAG_FD_ATR_ENABLED) && 11207 (I40E_DEBUG_FD & pf->hw.debug_mask)) 11208 dev_info(&pf->pdev->dev, "ATR re-enabled.\n"); 11209 } 11210 } 11211 return need_reset; 11212 } 11213 11214 /** 11215 * i40e_clear_rss_lut - clear the rx hash lookup table 11216 * @vsi: the VSI being configured 11217 **/ 11218 static void i40e_clear_rss_lut(struct i40e_vsi *vsi) 11219 { 11220 struct i40e_pf *pf = vsi->back; 11221 struct i40e_hw *hw = &pf->hw; 11222 u16 vf_id = vsi->vf_id; 11223 u8 i; 11224 11225 if (vsi->type == I40E_VSI_MAIN) { 11226 for (i = 0; i <= I40E_PFQF_HLUT_MAX_INDEX; i++) 11227 wr32(hw, I40E_PFQF_HLUT(i), 0); 11228 } else if (vsi->type == I40E_VSI_SRIOV) { 11229 for (i = 0; i <= I40E_VFQF_HLUT_MAX_INDEX; i++) 11230 i40e_write_rx_ctl(hw, I40E_VFQF_HLUT1(i, vf_id), 0); 11231 } else { 11232 dev_err(&pf->pdev->dev, "Cannot set RSS LUT - invalid VSI type\n"); 11233 } 11234 } 11235 11236 /** 11237 * i40e_set_features - set the netdev feature flags 11238 * @netdev: ptr to the netdev being adjusted 11239 * @features: the feature set that the stack is suggesting 11240 * Note: expects to be called while under rtnl_lock() 11241 **/ 11242 static int i40e_set_features(struct net_device *netdev, 11243 netdev_features_t features) 11244 { 11245 struct i40e_netdev_priv *np = netdev_priv(netdev); 11246 struct i40e_vsi *vsi = np->vsi; 11247 struct i40e_pf *pf = vsi->back; 11248 bool need_reset; 11249 11250 if (features & NETIF_F_RXHASH && !(netdev->features & NETIF_F_RXHASH)) 11251 i40e_pf_config_rss(pf); 11252 else if (!(features & NETIF_F_RXHASH) && 11253 netdev->features & NETIF_F_RXHASH) 11254 i40e_clear_rss_lut(vsi); 11255 11256 if (features & NETIF_F_HW_VLAN_CTAG_RX) 11257 i40e_vlan_stripping_enable(vsi); 11258 else 11259 i40e_vlan_stripping_disable(vsi); 11260 11261 if (!(features & NETIF_F_HW_TC) && pf->num_cloud_filters) { 11262 dev_err(&pf->pdev->dev, 11263 "Offloaded tc filters active, can't turn hw_tc_offload off"); 11264 return -EINVAL; 11265 } 11266 11267 need_reset = i40e_set_ntuple(pf, features); 11268 11269 if (need_reset) 11270 i40e_do_reset(pf, I40E_PF_RESET_FLAG, true); 11271 11272 return 0; 11273 } 11274 11275 /** 11276 * i40e_get_udp_port_idx - Lookup a possibly offloaded for Rx UDP port 11277 * @pf: board private structure 11278 * @port: The UDP port to look up 11279 * 11280 * Returns the index number or I40E_MAX_PF_UDP_OFFLOAD_PORTS if port not found 11281 **/ 11282 static u8 i40e_get_udp_port_idx(struct i40e_pf *pf, u16 port) 11283 { 11284 u8 i; 11285 11286 for (i = 0; i < I40E_MAX_PF_UDP_OFFLOAD_PORTS; i++) { 11287 if (pf->udp_ports[i].port == port) 11288 return i; 11289 } 11290 11291 return i; 11292 } 11293 11294 /** 11295 * i40e_udp_tunnel_add - Get notifications about UDP tunnel ports that come up 11296 * @netdev: This physical port's netdev 11297 * @ti: Tunnel endpoint information 11298 **/ 11299 static void i40e_udp_tunnel_add(struct net_device *netdev, 11300 struct udp_tunnel_info *ti) 11301 { 11302 struct i40e_netdev_priv *np = netdev_priv(netdev); 11303 struct i40e_vsi *vsi = np->vsi; 11304 struct i40e_pf *pf = vsi->back; 11305 u16 port = ntohs(ti->port); 11306 u8 next_idx; 11307 u8 idx; 11308 11309 idx = i40e_get_udp_port_idx(pf, port); 11310 11311 /* Check if port already exists */ 11312 if (idx < I40E_MAX_PF_UDP_OFFLOAD_PORTS) { 11313 netdev_info(netdev, "port %d already offloaded\n", port); 11314 return; 11315 } 11316 11317 /* Now check if there is space to add the new port */ 11318 next_idx = i40e_get_udp_port_idx(pf, 0); 11319 11320 if (next_idx == I40E_MAX_PF_UDP_OFFLOAD_PORTS) { 11321 netdev_info(netdev, "maximum number of offloaded UDP ports reached, not adding port %d\n", 11322 port); 11323 return; 11324 } 11325 11326 switch (ti->type) { 11327 case UDP_TUNNEL_TYPE_VXLAN: 11328 pf->udp_ports[next_idx].type = I40E_AQC_TUNNEL_TYPE_VXLAN; 11329 break; 11330 case UDP_TUNNEL_TYPE_GENEVE: 11331 if (!(pf->hw_features & I40E_HW_GENEVE_OFFLOAD_CAPABLE)) 11332 return; 11333 pf->udp_ports[next_idx].type = I40E_AQC_TUNNEL_TYPE_NGE; 11334 break; 11335 default: 11336 return; 11337 } 11338 11339 /* New port: add it and mark its index in the bitmap */ 11340 pf->udp_ports[next_idx].port = port; 11341 pf->pending_udp_bitmap |= BIT_ULL(next_idx); 11342 pf->flags |= I40E_FLAG_UDP_FILTER_SYNC; 11343 } 11344 11345 /** 11346 * i40e_udp_tunnel_del - Get notifications about UDP tunnel ports that go away 11347 * @netdev: This physical port's netdev 11348 * @ti: Tunnel endpoint information 11349 **/ 11350 static void i40e_udp_tunnel_del(struct net_device *netdev, 11351 struct udp_tunnel_info *ti) 11352 { 11353 struct i40e_netdev_priv *np = netdev_priv(netdev); 11354 struct i40e_vsi *vsi = np->vsi; 11355 struct i40e_pf *pf = vsi->back; 11356 u16 port = ntohs(ti->port); 11357 u8 idx; 11358 11359 idx = i40e_get_udp_port_idx(pf, port); 11360 11361 /* Check if port already exists */ 11362 if (idx >= I40E_MAX_PF_UDP_OFFLOAD_PORTS) 11363 goto not_found; 11364 11365 switch (ti->type) { 11366 case UDP_TUNNEL_TYPE_VXLAN: 11367 if (pf->udp_ports[idx].type != I40E_AQC_TUNNEL_TYPE_VXLAN) 11368 goto not_found; 11369 break; 11370 case UDP_TUNNEL_TYPE_GENEVE: 11371 if (pf->udp_ports[idx].type != I40E_AQC_TUNNEL_TYPE_NGE) 11372 goto not_found; 11373 break; 11374 default: 11375 goto not_found; 11376 } 11377 11378 /* if port exists, set it to 0 (mark for deletion) 11379 * and make it pending 11380 */ 11381 pf->udp_ports[idx].port = 0; 11382 pf->pending_udp_bitmap |= BIT_ULL(idx); 11383 pf->flags |= I40E_FLAG_UDP_FILTER_SYNC; 11384 11385 return; 11386 not_found: 11387 netdev_warn(netdev, "UDP port %d was not found, not deleting\n", 11388 port); 11389 } 11390 11391 static int i40e_get_phys_port_id(struct net_device *netdev, 11392 struct netdev_phys_item_id *ppid) 11393 { 11394 struct i40e_netdev_priv *np = netdev_priv(netdev); 11395 struct i40e_pf *pf = np->vsi->back; 11396 struct i40e_hw *hw = &pf->hw; 11397 11398 if (!(pf->hw_features & I40E_HW_PORT_ID_VALID)) 11399 return -EOPNOTSUPP; 11400 11401 ppid->id_len = min_t(int, sizeof(hw->mac.port_addr), sizeof(ppid->id)); 11402 memcpy(ppid->id, hw->mac.port_addr, ppid->id_len); 11403 11404 return 0; 11405 } 11406 11407 /** 11408 * i40e_ndo_fdb_add - add an entry to the hardware database 11409 * @ndm: the input from the stack 11410 * @tb: pointer to array of nladdr (unused) 11411 * @dev: the net device pointer 11412 * @addr: the MAC address entry being added 11413 * @flags: instructions from stack about fdb operation 11414 */ 11415 static int i40e_ndo_fdb_add(struct ndmsg *ndm, struct nlattr *tb[], 11416 struct net_device *dev, 11417 const unsigned char *addr, u16 vid, 11418 u16 flags) 11419 { 11420 struct i40e_netdev_priv *np = netdev_priv(dev); 11421 struct i40e_pf *pf = np->vsi->back; 11422 int err = 0; 11423 11424 if (!(pf->flags & I40E_FLAG_SRIOV_ENABLED)) 11425 return -EOPNOTSUPP; 11426 11427 if (vid) { 11428 pr_info("%s: vlans aren't supported yet for dev_uc|mc_add()\n", dev->name); 11429 return -EINVAL; 11430 } 11431 11432 /* Hardware does not support aging addresses so if a 11433 * ndm_state is given only allow permanent addresses 11434 */ 11435 if (ndm->ndm_state && !(ndm->ndm_state & NUD_PERMANENT)) { 11436 netdev_info(dev, "FDB only supports static addresses\n"); 11437 return -EINVAL; 11438 } 11439 11440 if (is_unicast_ether_addr(addr) || is_link_local_ether_addr(addr)) 11441 err = dev_uc_add_excl(dev, addr); 11442 else if (is_multicast_ether_addr(addr)) 11443 err = dev_mc_add_excl(dev, addr); 11444 else 11445 err = -EINVAL; 11446 11447 /* Only return duplicate errors if NLM_F_EXCL is set */ 11448 if (err == -EEXIST && !(flags & NLM_F_EXCL)) 11449 err = 0; 11450 11451 return err; 11452 } 11453 11454 /** 11455 * i40e_ndo_bridge_setlink - Set the hardware bridge mode 11456 * @dev: the netdev being configured 11457 * @nlh: RTNL message 11458 * 11459 * Inserts a new hardware bridge if not already created and 11460 * enables the bridging mode requested (VEB or VEPA). If the 11461 * hardware bridge has already been inserted and the request 11462 * is to change the mode then that requires a PF reset to 11463 * allow rebuild of the components with required hardware 11464 * bridge mode enabled. 11465 * 11466 * Note: expects to be called while under rtnl_lock() 11467 **/ 11468 static int i40e_ndo_bridge_setlink(struct net_device *dev, 11469 struct nlmsghdr *nlh, 11470 u16 flags) 11471 { 11472 struct i40e_netdev_priv *np = netdev_priv(dev); 11473 struct i40e_vsi *vsi = np->vsi; 11474 struct i40e_pf *pf = vsi->back; 11475 struct i40e_veb *veb = NULL; 11476 struct nlattr *attr, *br_spec; 11477 int i, rem; 11478 11479 /* Only for PF VSI for now */ 11480 if (vsi->seid != pf->vsi[pf->lan_vsi]->seid) 11481 return -EOPNOTSUPP; 11482 11483 /* Find the HW bridge for PF VSI */ 11484 for (i = 0; i < I40E_MAX_VEB && !veb; i++) { 11485 if (pf->veb[i] && pf->veb[i]->seid == vsi->uplink_seid) 11486 veb = pf->veb[i]; 11487 } 11488 11489 br_spec = nlmsg_find_attr(nlh, sizeof(struct ifinfomsg), IFLA_AF_SPEC); 11490 11491 nla_for_each_nested(attr, br_spec, rem) { 11492 __u16 mode; 11493 11494 if (nla_type(attr) != IFLA_BRIDGE_MODE) 11495 continue; 11496 11497 mode = nla_get_u16(attr); 11498 if ((mode != BRIDGE_MODE_VEPA) && 11499 (mode != BRIDGE_MODE_VEB)) 11500 return -EINVAL; 11501 11502 /* Insert a new HW bridge */ 11503 if (!veb) { 11504 veb = i40e_veb_setup(pf, 0, vsi->uplink_seid, vsi->seid, 11505 vsi->tc_config.enabled_tc); 11506 if (veb) { 11507 veb->bridge_mode = mode; 11508 i40e_config_bridge_mode(veb); 11509 } else { 11510 /* No Bridge HW offload available */ 11511 return -ENOENT; 11512 } 11513 break; 11514 } else if (mode != veb->bridge_mode) { 11515 /* Existing HW bridge but different mode needs reset */ 11516 veb->bridge_mode = mode; 11517 /* TODO: If no VFs or VMDq VSIs, disallow VEB mode */ 11518 if (mode == BRIDGE_MODE_VEB) 11519 pf->flags |= I40E_FLAG_VEB_MODE_ENABLED; 11520 else 11521 pf->flags &= ~I40E_FLAG_VEB_MODE_ENABLED; 11522 i40e_do_reset(pf, I40E_PF_RESET_FLAG, true); 11523 break; 11524 } 11525 } 11526 11527 return 0; 11528 } 11529 11530 /** 11531 * i40e_ndo_bridge_getlink - Get the hardware bridge mode 11532 * @skb: skb buff 11533 * @pid: process id 11534 * @seq: RTNL message seq # 11535 * @dev: the netdev being configured 11536 * @filter_mask: unused 11537 * @nlflags: netlink flags passed in 11538 * 11539 * Return the mode in which the hardware bridge is operating in 11540 * i.e VEB or VEPA. 11541 **/ 11542 static int i40e_ndo_bridge_getlink(struct sk_buff *skb, u32 pid, u32 seq, 11543 struct net_device *dev, 11544 u32 __always_unused filter_mask, 11545 int nlflags) 11546 { 11547 struct i40e_netdev_priv *np = netdev_priv(dev); 11548 struct i40e_vsi *vsi = np->vsi; 11549 struct i40e_pf *pf = vsi->back; 11550 struct i40e_veb *veb = NULL; 11551 int i; 11552 11553 /* Only for PF VSI for now */ 11554 if (vsi->seid != pf->vsi[pf->lan_vsi]->seid) 11555 return -EOPNOTSUPP; 11556 11557 /* Find the HW bridge for the PF VSI */ 11558 for (i = 0; i < I40E_MAX_VEB && !veb; i++) { 11559 if (pf->veb[i] && pf->veb[i]->seid == vsi->uplink_seid) 11560 veb = pf->veb[i]; 11561 } 11562 11563 if (!veb) 11564 return 0; 11565 11566 return ndo_dflt_bridge_getlink(skb, pid, seq, dev, veb->bridge_mode, 11567 0, 0, nlflags, filter_mask, NULL); 11568 } 11569 11570 /** 11571 * i40e_features_check - Validate encapsulated packet conforms to limits 11572 * @skb: skb buff 11573 * @dev: This physical port's netdev 11574 * @features: Offload features that the stack believes apply 11575 **/ 11576 static netdev_features_t i40e_features_check(struct sk_buff *skb, 11577 struct net_device *dev, 11578 netdev_features_t features) 11579 { 11580 size_t len; 11581 11582 /* No point in doing any of this if neither checksum nor GSO are 11583 * being requested for this frame. We can rule out both by just 11584 * checking for CHECKSUM_PARTIAL 11585 */ 11586 if (skb->ip_summed != CHECKSUM_PARTIAL) 11587 return features; 11588 11589 /* We cannot support GSO if the MSS is going to be less than 11590 * 64 bytes. If it is then we need to drop support for GSO. 11591 */ 11592 if (skb_is_gso(skb) && (skb_shinfo(skb)->gso_size < 64)) 11593 features &= ~NETIF_F_GSO_MASK; 11594 11595 /* MACLEN can support at most 63 words */ 11596 len = skb_network_header(skb) - skb->data; 11597 if (len & ~(63 * 2)) 11598 goto out_err; 11599 11600 /* IPLEN and EIPLEN can support at most 127 dwords */ 11601 len = skb_transport_header(skb) - skb_network_header(skb); 11602 if (len & ~(127 * 4)) 11603 goto out_err; 11604 11605 if (skb->encapsulation) { 11606 /* L4TUNLEN can support 127 words */ 11607 len = skb_inner_network_header(skb) - skb_transport_header(skb); 11608 if (len & ~(127 * 2)) 11609 goto out_err; 11610 11611 /* IPLEN can support at most 127 dwords */ 11612 len = skb_inner_transport_header(skb) - 11613 skb_inner_network_header(skb); 11614 if (len & ~(127 * 4)) 11615 goto out_err; 11616 } 11617 11618 /* No need to validate L4LEN as TCP is the only protocol with a 11619 * a flexible value and we support all possible values supported 11620 * by TCP, which is at most 15 dwords 11621 */ 11622 11623 return features; 11624 out_err: 11625 return features & ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK); 11626 } 11627 11628 /** 11629 * i40e_xdp_setup - add/remove an XDP program 11630 * @vsi: VSI to changed 11631 * @prog: XDP program 11632 **/ 11633 static int i40e_xdp_setup(struct i40e_vsi *vsi, 11634 struct bpf_prog *prog) 11635 { 11636 int frame_size = vsi->netdev->mtu + ETH_HLEN + ETH_FCS_LEN + VLAN_HLEN; 11637 struct i40e_pf *pf = vsi->back; 11638 struct bpf_prog *old_prog; 11639 bool need_reset; 11640 int i; 11641 11642 /* Don't allow frames that span over multiple buffers */ 11643 if (frame_size > vsi->rx_buf_len) 11644 return -EINVAL; 11645 11646 if (!i40e_enabled_xdp_vsi(vsi) && !prog) 11647 return 0; 11648 11649 /* When turning XDP on->off/off->on we reset and rebuild the rings. */ 11650 need_reset = (i40e_enabled_xdp_vsi(vsi) != !!prog); 11651 11652 if (need_reset) 11653 i40e_prep_for_reset(pf, true); 11654 11655 old_prog = xchg(&vsi->xdp_prog, prog); 11656 11657 if (need_reset) 11658 i40e_reset_and_rebuild(pf, true, true); 11659 11660 for (i = 0; i < vsi->num_queue_pairs; i++) 11661 WRITE_ONCE(vsi->rx_rings[i]->xdp_prog, vsi->xdp_prog); 11662 11663 if (old_prog) 11664 bpf_prog_put(old_prog); 11665 11666 return 0; 11667 } 11668 11669 /** 11670 * i40e_xdp - implements ndo_bpf for i40e 11671 * @dev: netdevice 11672 * @xdp: XDP command 11673 **/ 11674 static int i40e_xdp(struct net_device *dev, 11675 struct netdev_bpf *xdp) 11676 { 11677 struct i40e_netdev_priv *np = netdev_priv(dev); 11678 struct i40e_vsi *vsi = np->vsi; 11679 11680 if (vsi->type != I40E_VSI_MAIN) 11681 return -EINVAL; 11682 11683 switch (xdp->command) { 11684 case XDP_SETUP_PROG: 11685 return i40e_xdp_setup(vsi, xdp->prog); 11686 case XDP_QUERY_PROG: 11687 xdp->prog_attached = i40e_enabled_xdp_vsi(vsi); 11688 xdp->prog_id = vsi->xdp_prog ? vsi->xdp_prog->aux->id : 0; 11689 return 0; 11690 default: 11691 return -EINVAL; 11692 } 11693 } 11694 11695 static const struct net_device_ops i40e_netdev_ops = { 11696 .ndo_open = i40e_open, 11697 .ndo_stop = i40e_close, 11698 .ndo_start_xmit = i40e_lan_xmit_frame, 11699 .ndo_get_stats64 = i40e_get_netdev_stats_struct, 11700 .ndo_set_rx_mode = i40e_set_rx_mode, 11701 .ndo_validate_addr = eth_validate_addr, 11702 .ndo_set_mac_address = i40e_set_mac, 11703 .ndo_change_mtu = i40e_change_mtu, 11704 .ndo_do_ioctl = i40e_ioctl, 11705 .ndo_tx_timeout = i40e_tx_timeout, 11706 .ndo_vlan_rx_add_vid = i40e_vlan_rx_add_vid, 11707 .ndo_vlan_rx_kill_vid = i40e_vlan_rx_kill_vid, 11708 #ifdef CONFIG_NET_POLL_CONTROLLER 11709 .ndo_poll_controller = i40e_netpoll, 11710 #endif 11711 .ndo_setup_tc = __i40e_setup_tc, 11712 .ndo_set_features = i40e_set_features, 11713 .ndo_set_vf_mac = i40e_ndo_set_vf_mac, 11714 .ndo_set_vf_vlan = i40e_ndo_set_vf_port_vlan, 11715 .ndo_set_vf_rate = i40e_ndo_set_vf_bw, 11716 .ndo_get_vf_config = i40e_ndo_get_vf_config, 11717 .ndo_set_vf_link_state = i40e_ndo_set_vf_link_state, 11718 .ndo_set_vf_spoofchk = i40e_ndo_set_vf_spoofchk, 11719 .ndo_set_vf_trust = i40e_ndo_set_vf_trust, 11720 .ndo_udp_tunnel_add = i40e_udp_tunnel_add, 11721 .ndo_udp_tunnel_del = i40e_udp_tunnel_del, 11722 .ndo_get_phys_port_id = i40e_get_phys_port_id, 11723 .ndo_fdb_add = i40e_ndo_fdb_add, 11724 .ndo_features_check = i40e_features_check, 11725 .ndo_bridge_getlink = i40e_ndo_bridge_getlink, 11726 .ndo_bridge_setlink = i40e_ndo_bridge_setlink, 11727 .ndo_bpf = i40e_xdp, 11728 }; 11729 11730 /** 11731 * i40e_config_netdev - Setup the netdev flags 11732 * @vsi: the VSI being configured 11733 * 11734 * Returns 0 on success, negative value on failure 11735 **/ 11736 static int i40e_config_netdev(struct i40e_vsi *vsi) 11737 { 11738 struct i40e_pf *pf = vsi->back; 11739 struct i40e_hw *hw = &pf->hw; 11740 struct i40e_netdev_priv *np; 11741 struct net_device *netdev; 11742 u8 broadcast[ETH_ALEN]; 11743 u8 mac_addr[ETH_ALEN]; 11744 int etherdev_size; 11745 netdev_features_t hw_enc_features; 11746 netdev_features_t hw_features; 11747 11748 etherdev_size = sizeof(struct i40e_netdev_priv); 11749 netdev = alloc_etherdev_mq(etherdev_size, vsi->alloc_queue_pairs); 11750 if (!netdev) 11751 return -ENOMEM; 11752 11753 vsi->netdev = netdev; 11754 np = netdev_priv(netdev); 11755 np->vsi = vsi; 11756 11757 hw_enc_features = NETIF_F_SG | 11758 NETIF_F_IP_CSUM | 11759 NETIF_F_IPV6_CSUM | 11760 NETIF_F_HIGHDMA | 11761 NETIF_F_SOFT_FEATURES | 11762 NETIF_F_TSO | 11763 NETIF_F_TSO_ECN | 11764 NETIF_F_TSO6 | 11765 NETIF_F_GSO_GRE | 11766 NETIF_F_GSO_GRE_CSUM | 11767 NETIF_F_GSO_PARTIAL | 11768 NETIF_F_GSO_UDP_TUNNEL | 11769 NETIF_F_GSO_UDP_TUNNEL_CSUM | 11770 NETIF_F_SCTP_CRC | 11771 NETIF_F_RXHASH | 11772 NETIF_F_RXCSUM | 11773 0; 11774 11775 if (!(pf->hw_features & I40E_HW_OUTER_UDP_CSUM_CAPABLE)) 11776 netdev->gso_partial_features |= NETIF_F_GSO_UDP_TUNNEL_CSUM; 11777 11778 netdev->gso_partial_features |= NETIF_F_GSO_GRE_CSUM; 11779 11780 netdev->hw_enc_features |= hw_enc_features; 11781 11782 /* record features VLANs can make use of */ 11783 netdev->vlan_features |= hw_enc_features | NETIF_F_TSO_MANGLEID; 11784 11785 if (!(pf->flags & I40E_FLAG_MFP_ENABLED)) 11786 netdev->hw_features |= NETIF_F_NTUPLE | NETIF_F_HW_TC; 11787 11788 hw_features = hw_enc_features | 11789 NETIF_F_HW_VLAN_CTAG_TX | 11790 NETIF_F_HW_VLAN_CTAG_RX; 11791 11792 netdev->hw_features |= hw_features; 11793 11794 netdev->features |= hw_features | NETIF_F_HW_VLAN_CTAG_FILTER; 11795 netdev->hw_enc_features |= NETIF_F_TSO_MANGLEID; 11796 11797 if (vsi->type == I40E_VSI_MAIN) { 11798 SET_NETDEV_DEV(netdev, &pf->pdev->dev); 11799 ether_addr_copy(mac_addr, hw->mac.perm_addr); 11800 /* The following steps are necessary for two reasons. First, 11801 * some older NVM configurations load a default MAC-VLAN 11802 * filter that will accept any tagged packet, and we want to 11803 * replace this with a normal filter. Additionally, it is 11804 * possible our MAC address was provided by the platform using 11805 * Open Firmware or similar. 11806 * 11807 * Thus, we need to remove the default filter and install one 11808 * specific to the MAC address. 11809 */ 11810 i40e_rm_default_mac_filter(vsi, mac_addr); 11811 spin_lock_bh(&vsi->mac_filter_hash_lock); 11812 i40e_add_mac_filter(vsi, mac_addr); 11813 spin_unlock_bh(&vsi->mac_filter_hash_lock); 11814 } else { 11815 /* Relate the VSI_VMDQ name to the VSI_MAIN name. Note that we 11816 * are still limited by IFNAMSIZ, but we're adding 'v%d\0' to 11817 * the end, which is 4 bytes long, so force truncation of the 11818 * original name by IFNAMSIZ - 4 11819 */ 11820 snprintf(netdev->name, IFNAMSIZ, "%.*sv%%d", 11821 IFNAMSIZ - 4, 11822 pf->vsi[pf->lan_vsi]->netdev->name); 11823 random_ether_addr(mac_addr); 11824 11825 spin_lock_bh(&vsi->mac_filter_hash_lock); 11826 i40e_add_mac_filter(vsi, mac_addr); 11827 spin_unlock_bh(&vsi->mac_filter_hash_lock); 11828 } 11829 11830 /* Add the broadcast filter so that we initially will receive 11831 * broadcast packets. Note that when a new VLAN is first added the 11832 * driver will convert all filters marked I40E_VLAN_ANY into VLAN 11833 * specific filters as part of transitioning into "vlan" operation. 11834 * When more VLANs are added, the driver will copy each existing MAC 11835 * filter and add it for the new VLAN. 11836 * 11837 * Broadcast filters are handled specially by 11838 * i40e_sync_filters_subtask, as the driver must to set the broadcast 11839 * promiscuous bit instead of adding this directly as a MAC/VLAN 11840 * filter. The subtask will update the correct broadcast promiscuous 11841 * bits as VLANs become active or inactive. 11842 */ 11843 eth_broadcast_addr(broadcast); 11844 spin_lock_bh(&vsi->mac_filter_hash_lock); 11845 i40e_add_mac_filter(vsi, broadcast); 11846 spin_unlock_bh(&vsi->mac_filter_hash_lock); 11847 11848 ether_addr_copy(netdev->dev_addr, mac_addr); 11849 ether_addr_copy(netdev->perm_addr, mac_addr); 11850 11851 netdev->priv_flags |= IFF_UNICAST_FLT; 11852 netdev->priv_flags |= IFF_SUPP_NOFCS; 11853 /* Setup netdev TC information */ 11854 i40e_vsi_config_netdev_tc(vsi, vsi->tc_config.enabled_tc); 11855 11856 netdev->netdev_ops = &i40e_netdev_ops; 11857 netdev->watchdog_timeo = 5 * HZ; 11858 i40e_set_ethtool_ops(netdev); 11859 11860 /* MTU range: 68 - 9706 */ 11861 netdev->min_mtu = ETH_MIN_MTU; 11862 netdev->max_mtu = I40E_MAX_RXBUFFER - I40E_PACKET_HDR_PAD; 11863 11864 return 0; 11865 } 11866 11867 /** 11868 * i40e_vsi_delete - Delete a VSI from the switch 11869 * @vsi: the VSI being removed 11870 * 11871 * Returns 0 on success, negative value on failure 11872 **/ 11873 static void i40e_vsi_delete(struct i40e_vsi *vsi) 11874 { 11875 /* remove default VSI is not allowed */ 11876 if (vsi == vsi->back->vsi[vsi->back->lan_vsi]) 11877 return; 11878 11879 i40e_aq_delete_element(&vsi->back->hw, vsi->seid, NULL); 11880 } 11881 11882 /** 11883 * i40e_is_vsi_uplink_mode_veb - Check if the VSI's uplink bridge mode is VEB 11884 * @vsi: the VSI being queried 11885 * 11886 * Returns 1 if HW bridge mode is VEB and return 0 in case of VEPA mode 11887 **/ 11888 int i40e_is_vsi_uplink_mode_veb(struct i40e_vsi *vsi) 11889 { 11890 struct i40e_veb *veb; 11891 struct i40e_pf *pf = vsi->back; 11892 11893 /* Uplink is not a bridge so default to VEB */ 11894 if (vsi->veb_idx == I40E_NO_VEB) 11895 return 1; 11896 11897 veb = pf->veb[vsi->veb_idx]; 11898 if (!veb) { 11899 dev_info(&pf->pdev->dev, 11900 "There is no veb associated with the bridge\n"); 11901 return -ENOENT; 11902 } 11903 11904 /* Uplink is a bridge in VEPA mode */ 11905 if (veb->bridge_mode & BRIDGE_MODE_VEPA) { 11906 return 0; 11907 } else { 11908 /* Uplink is a bridge in VEB mode */ 11909 return 1; 11910 } 11911 11912 /* VEPA is now default bridge, so return 0 */ 11913 return 0; 11914 } 11915 11916 /** 11917 * i40e_add_vsi - Add a VSI to the switch 11918 * @vsi: the VSI being configured 11919 * 11920 * This initializes a VSI context depending on the VSI type to be added and 11921 * passes it down to the add_vsi aq command. 11922 **/ 11923 static int i40e_add_vsi(struct i40e_vsi *vsi) 11924 { 11925 int ret = -ENODEV; 11926 struct i40e_pf *pf = vsi->back; 11927 struct i40e_hw *hw = &pf->hw; 11928 struct i40e_vsi_context ctxt; 11929 struct i40e_mac_filter *f; 11930 struct hlist_node *h; 11931 int bkt; 11932 11933 u8 enabled_tc = 0x1; /* TC0 enabled */ 11934 int f_count = 0; 11935 11936 memset(&ctxt, 0, sizeof(ctxt)); 11937 switch (vsi->type) { 11938 case I40E_VSI_MAIN: 11939 /* The PF's main VSI is already setup as part of the 11940 * device initialization, so we'll not bother with 11941 * the add_vsi call, but we will retrieve the current 11942 * VSI context. 11943 */ 11944 ctxt.seid = pf->main_vsi_seid; 11945 ctxt.pf_num = pf->hw.pf_id; 11946 ctxt.vf_num = 0; 11947 ret = i40e_aq_get_vsi_params(&pf->hw, &ctxt, NULL); 11948 ctxt.flags = I40E_AQ_VSI_TYPE_PF; 11949 if (ret) { 11950 dev_info(&pf->pdev->dev, 11951 "couldn't get PF vsi config, err %s aq_err %s\n", 11952 i40e_stat_str(&pf->hw, ret), 11953 i40e_aq_str(&pf->hw, 11954 pf->hw.aq.asq_last_status)); 11955 return -ENOENT; 11956 } 11957 vsi->info = ctxt.info; 11958 vsi->info.valid_sections = 0; 11959 11960 vsi->seid = ctxt.seid; 11961 vsi->id = ctxt.vsi_number; 11962 11963 enabled_tc = i40e_pf_get_tc_map(pf); 11964 11965 /* Source pruning is enabled by default, so the flag is 11966 * negative logic - if it's set, we need to fiddle with 11967 * the VSI to disable source pruning. 11968 */ 11969 if (pf->flags & I40E_FLAG_SOURCE_PRUNING_DISABLED) { 11970 memset(&ctxt, 0, sizeof(ctxt)); 11971 ctxt.seid = pf->main_vsi_seid; 11972 ctxt.pf_num = pf->hw.pf_id; 11973 ctxt.vf_num = 0; 11974 ctxt.info.valid_sections |= 11975 cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID); 11976 ctxt.info.switch_id = 11977 cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_LOCAL_LB); 11978 ret = i40e_aq_update_vsi_params(hw, &ctxt, NULL); 11979 if (ret) { 11980 dev_info(&pf->pdev->dev, 11981 "update vsi failed, err %s aq_err %s\n", 11982 i40e_stat_str(&pf->hw, ret), 11983 i40e_aq_str(&pf->hw, 11984 pf->hw.aq.asq_last_status)); 11985 ret = -ENOENT; 11986 goto err; 11987 } 11988 } 11989 11990 /* MFP mode setup queue map and update VSI */ 11991 if ((pf->flags & I40E_FLAG_MFP_ENABLED) && 11992 !(pf->hw.func_caps.iscsi)) { /* NIC type PF */ 11993 memset(&ctxt, 0, sizeof(ctxt)); 11994 ctxt.seid = pf->main_vsi_seid; 11995 ctxt.pf_num = pf->hw.pf_id; 11996 ctxt.vf_num = 0; 11997 i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, false); 11998 ret = i40e_aq_update_vsi_params(hw, &ctxt, NULL); 11999 if (ret) { 12000 dev_info(&pf->pdev->dev, 12001 "update vsi failed, err %s aq_err %s\n", 12002 i40e_stat_str(&pf->hw, ret), 12003 i40e_aq_str(&pf->hw, 12004 pf->hw.aq.asq_last_status)); 12005 ret = -ENOENT; 12006 goto err; 12007 } 12008 /* update the local VSI info queue map */ 12009 i40e_vsi_update_queue_map(vsi, &ctxt); 12010 vsi->info.valid_sections = 0; 12011 } else { 12012 /* Default/Main VSI is only enabled for TC0 12013 * reconfigure it to enable all TCs that are 12014 * available on the port in SFP mode. 12015 * For MFP case the iSCSI PF would use this 12016 * flow to enable LAN+iSCSI TC. 12017 */ 12018 ret = i40e_vsi_config_tc(vsi, enabled_tc); 12019 if (ret) { 12020 /* Single TC condition is not fatal, 12021 * message and continue 12022 */ 12023 dev_info(&pf->pdev->dev, 12024 "failed to configure TCs for main VSI tc_map 0x%08x, err %s aq_err %s\n", 12025 enabled_tc, 12026 i40e_stat_str(&pf->hw, ret), 12027 i40e_aq_str(&pf->hw, 12028 pf->hw.aq.asq_last_status)); 12029 } 12030 } 12031 break; 12032 12033 case I40E_VSI_FDIR: 12034 ctxt.pf_num = hw->pf_id; 12035 ctxt.vf_num = 0; 12036 ctxt.uplink_seid = vsi->uplink_seid; 12037 ctxt.connection_type = I40E_AQ_VSI_CONN_TYPE_NORMAL; 12038 ctxt.flags = I40E_AQ_VSI_TYPE_PF; 12039 if ((pf->flags & I40E_FLAG_VEB_MODE_ENABLED) && 12040 (i40e_is_vsi_uplink_mode_veb(vsi))) { 12041 ctxt.info.valid_sections |= 12042 cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID); 12043 ctxt.info.switch_id = 12044 cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB); 12045 } 12046 i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, true); 12047 break; 12048 12049 case I40E_VSI_VMDQ2: 12050 ctxt.pf_num = hw->pf_id; 12051 ctxt.vf_num = 0; 12052 ctxt.uplink_seid = vsi->uplink_seid; 12053 ctxt.connection_type = I40E_AQ_VSI_CONN_TYPE_NORMAL; 12054 ctxt.flags = I40E_AQ_VSI_TYPE_VMDQ2; 12055 12056 /* This VSI is connected to VEB so the switch_id 12057 * should be set to zero by default. 12058 */ 12059 if (i40e_is_vsi_uplink_mode_veb(vsi)) { 12060 ctxt.info.valid_sections |= 12061 cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID); 12062 ctxt.info.switch_id = 12063 cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB); 12064 } 12065 12066 /* Setup the VSI tx/rx queue map for TC0 only for now */ 12067 i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, true); 12068 break; 12069 12070 case I40E_VSI_SRIOV: 12071 ctxt.pf_num = hw->pf_id; 12072 ctxt.vf_num = vsi->vf_id + hw->func_caps.vf_base_id; 12073 ctxt.uplink_seid = vsi->uplink_seid; 12074 ctxt.connection_type = I40E_AQ_VSI_CONN_TYPE_NORMAL; 12075 ctxt.flags = I40E_AQ_VSI_TYPE_VF; 12076 12077 /* This VSI is connected to VEB so the switch_id 12078 * should be set to zero by default. 12079 */ 12080 if (i40e_is_vsi_uplink_mode_veb(vsi)) { 12081 ctxt.info.valid_sections |= 12082 cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID); 12083 ctxt.info.switch_id = 12084 cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB); 12085 } 12086 12087 if (vsi->back->flags & I40E_FLAG_IWARP_ENABLED) { 12088 ctxt.info.valid_sections |= 12089 cpu_to_le16(I40E_AQ_VSI_PROP_QUEUE_OPT_VALID); 12090 ctxt.info.queueing_opt_flags |= 12091 (I40E_AQ_VSI_QUE_OPT_TCP_ENA | 12092 I40E_AQ_VSI_QUE_OPT_RSS_LUT_VSI); 12093 } 12094 12095 ctxt.info.valid_sections |= cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID); 12096 ctxt.info.port_vlan_flags |= I40E_AQ_VSI_PVLAN_MODE_ALL; 12097 if (pf->vf[vsi->vf_id].spoofchk) { 12098 ctxt.info.valid_sections |= 12099 cpu_to_le16(I40E_AQ_VSI_PROP_SECURITY_VALID); 12100 ctxt.info.sec_flags |= 12101 (I40E_AQ_VSI_SEC_FLAG_ENABLE_VLAN_CHK | 12102 I40E_AQ_VSI_SEC_FLAG_ENABLE_MAC_CHK); 12103 } 12104 /* Setup the VSI tx/rx queue map for TC0 only for now */ 12105 i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, true); 12106 break; 12107 12108 case I40E_VSI_IWARP: 12109 /* send down message to iWARP */ 12110 break; 12111 12112 default: 12113 return -ENODEV; 12114 } 12115 12116 if (vsi->type != I40E_VSI_MAIN) { 12117 ret = i40e_aq_add_vsi(hw, &ctxt, NULL); 12118 if (ret) { 12119 dev_info(&vsi->back->pdev->dev, 12120 "add vsi failed, err %s aq_err %s\n", 12121 i40e_stat_str(&pf->hw, ret), 12122 i40e_aq_str(&pf->hw, 12123 pf->hw.aq.asq_last_status)); 12124 ret = -ENOENT; 12125 goto err; 12126 } 12127 vsi->info = ctxt.info; 12128 vsi->info.valid_sections = 0; 12129 vsi->seid = ctxt.seid; 12130 vsi->id = ctxt.vsi_number; 12131 } 12132 12133 vsi->active_filters = 0; 12134 clear_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state); 12135 spin_lock_bh(&vsi->mac_filter_hash_lock); 12136 /* If macvlan filters already exist, force them to get loaded */ 12137 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) { 12138 f->state = I40E_FILTER_NEW; 12139 f_count++; 12140 } 12141 spin_unlock_bh(&vsi->mac_filter_hash_lock); 12142 12143 if (f_count) { 12144 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED; 12145 pf->flags |= I40E_FLAG_FILTER_SYNC; 12146 } 12147 12148 /* Update VSI BW information */ 12149 ret = i40e_vsi_get_bw_info(vsi); 12150 if (ret) { 12151 dev_info(&pf->pdev->dev, 12152 "couldn't get vsi bw info, err %s aq_err %s\n", 12153 i40e_stat_str(&pf->hw, ret), 12154 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 12155 /* VSI is already added so not tearing that up */ 12156 ret = 0; 12157 } 12158 12159 err: 12160 return ret; 12161 } 12162 12163 /** 12164 * i40e_vsi_release - Delete a VSI and free its resources 12165 * @vsi: the VSI being removed 12166 * 12167 * Returns 0 on success or < 0 on error 12168 **/ 12169 int i40e_vsi_release(struct i40e_vsi *vsi) 12170 { 12171 struct i40e_mac_filter *f; 12172 struct hlist_node *h; 12173 struct i40e_veb *veb = NULL; 12174 struct i40e_pf *pf; 12175 u16 uplink_seid; 12176 int i, n, bkt; 12177 12178 pf = vsi->back; 12179 12180 /* release of a VEB-owner or last VSI is not allowed */ 12181 if (vsi->flags & I40E_VSI_FLAG_VEB_OWNER) { 12182 dev_info(&pf->pdev->dev, "VSI %d has existing VEB %d\n", 12183 vsi->seid, vsi->uplink_seid); 12184 return -ENODEV; 12185 } 12186 if (vsi == pf->vsi[pf->lan_vsi] && 12187 !test_bit(__I40E_DOWN, pf->state)) { 12188 dev_info(&pf->pdev->dev, "Can't remove PF VSI\n"); 12189 return -ENODEV; 12190 } 12191 12192 uplink_seid = vsi->uplink_seid; 12193 if (vsi->type != I40E_VSI_SRIOV) { 12194 if (vsi->netdev_registered) { 12195 vsi->netdev_registered = false; 12196 if (vsi->netdev) { 12197 /* results in a call to i40e_close() */ 12198 unregister_netdev(vsi->netdev); 12199 } 12200 } else { 12201 i40e_vsi_close(vsi); 12202 } 12203 i40e_vsi_disable_irq(vsi); 12204 } 12205 12206 spin_lock_bh(&vsi->mac_filter_hash_lock); 12207 12208 /* clear the sync flag on all filters */ 12209 if (vsi->netdev) { 12210 __dev_uc_unsync(vsi->netdev, NULL); 12211 __dev_mc_unsync(vsi->netdev, NULL); 12212 } 12213 12214 /* make sure any remaining filters are marked for deletion */ 12215 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) 12216 __i40e_del_filter(vsi, f); 12217 12218 spin_unlock_bh(&vsi->mac_filter_hash_lock); 12219 12220 i40e_sync_vsi_filters(vsi); 12221 12222 i40e_vsi_delete(vsi); 12223 i40e_vsi_free_q_vectors(vsi); 12224 if (vsi->netdev) { 12225 free_netdev(vsi->netdev); 12226 vsi->netdev = NULL; 12227 } 12228 i40e_vsi_clear_rings(vsi); 12229 i40e_vsi_clear(vsi); 12230 12231 /* If this was the last thing on the VEB, except for the 12232 * controlling VSI, remove the VEB, which puts the controlling 12233 * VSI onto the next level down in the switch. 12234 * 12235 * Well, okay, there's one more exception here: don't remove 12236 * the orphan VEBs yet. We'll wait for an explicit remove request 12237 * from up the network stack. 12238 */ 12239 for (n = 0, i = 0; i < pf->num_alloc_vsi; i++) { 12240 if (pf->vsi[i] && 12241 pf->vsi[i]->uplink_seid == uplink_seid && 12242 (pf->vsi[i]->flags & I40E_VSI_FLAG_VEB_OWNER) == 0) { 12243 n++; /* count the VSIs */ 12244 } 12245 } 12246 for (i = 0; i < I40E_MAX_VEB; i++) { 12247 if (!pf->veb[i]) 12248 continue; 12249 if (pf->veb[i]->uplink_seid == uplink_seid) 12250 n++; /* count the VEBs */ 12251 if (pf->veb[i]->seid == uplink_seid) 12252 veb = pf->veb[i]; 12253 } 12254 if (n == 0 && veb && veb->uplink_seid != 0) 12255 i40e_veb_release(veb); 12256 12257 return 0; 12258 } 12259 12260 /** 12261 * i40e_vsi_setup_vectors - Set up the q_vectors for the given VSI 12262 * @vsi: ptr to the VSI 12263 * 12264 * This should only be called after i40e_vsi_mem_alloc() which allocates the 12265 * corresponding SW VSI structure and initializes num_queue_pairs for the 12266 * newly allocated VSI. 12267 * 12268 * Returns 0 on success or negative on failure 12269 **/ 12270 static int i40e_vsi_setup_vectors(struct i40e_vsi *vsi) 12271 { 12272 int ret = -ENOENT; 12273 struct i40e_pf *pf = vsi->back; 12274 12275 if (vsi->q_vectors[0]) { 12276 dev_info(&pf->pdev->dev, "VSI %d has existing q_vectors\n", 12277 vsi->seid); 12278 return -EEXIST; 12279 } 12280 12281 if (vsi->base_vector) { 12282 dev_info(&pf->pdev->dev, "VSI %d has non-zero base vector %d\n", 12283 vsi->seid, vsi->base_vector); 12284 return -EEXIST; 12285 } 12286 12287 ret = i40e_vsi_alloc_q_vectors(vsi); 12288 if (ret) { 12289 dev_info(&pf->pdev->dev, 12290 "failed to allocate %d q_vector for VSI %d, ret=%d\n", 12291 vsi->num_q_vectors, vsi->seid, ret); 12292 vsi->num_q_vectors = 0; 12293 goto vector_setup_out; 12294 } 12295 12296 /* In Legacy mode, we do not have to get any other vector since we 12297 * piggyback on the misc/ICR0 for queue interrupts. 12298 */ 12299 if (!(pf->flags & I40E_FLAG_MSIX_ENABLED)) 12300 return ret; 12301 if (vsi->num_q_vectors) 12302 vsi->base_vector = i40e_get_lump(pf, pf->irq_pile, 12303 vsi->num_q_vectors, vsi->idx); 12304 if (vsi->base_vector < 0) { 12305 dev_info(&pf->pdev->dev, 12306 "failed to get tracking for %d vectors for VSI %d, err=%d\n", 12307 vsi->num_q_vectors, vsi->seid, vsi->base_vector); 12308 i40e_vsi_free_q_vectors(vsi); 12309 ret = -ENOENT; 12310 goto vector_setup_out; 12311 } 12312 12313 vector_setup_out: 12314 return ret; 12315 } 12316 12317 /** 12318 * i40e_vsi_reinit_setup - return and reallocate resources for a VSI 12319 * @vsi: pointer to the vsi. 12320 * 12321 * This re-allocates a vsi's queue resources. 12322 * 12323 * Returns pointer to the successfully allocated and configured VSI sw struct 12324 * on success, otherwise returns NULL on failure. 12325 **/ 12326 static struct i40e_vsi *i40e_vsi_reinit_setup(struct i40e_vsi *vsi) 12327 { 12328 u16 alloc_queue_pairs; 12329 struct i40e_pf *pf; 12330 u8 enabled_tc; 12331 int ret; 12332 12333 if (!vsi) 12334 return NULL; 12335 12336 pf = vsi->back; 12337 12338 i40e_put_lump(pf->qp_pile, vsi->base_queue, vsi->idx); 12339 i40e_vsi_clear_rings(vsi); 12340 12341 i40e_vsi_free_arrays(vsi, false); 12342 i40e_set_num_rings_in_vsi(vsi); 12343 ret = i40e_vsi_alloc_arrays(vsi, false); 12344 if (ret) 12345 goto err_vsi; 12346 12347 alloc_queue_pairs = vsi->alloc_queue_pairs * 12348 (i40e_enabled_xdp_vsi(vsi) ? 2 : 1); 12349 12350 ret = i40e_get_lump(pf, pf->qp_pile, alloc_queue_pairs, vsi->idx); 12351 if (ret < 0) { 12352 dev_info(&pf->pdev->dev, 12353 "failed to get tracking for %d queues for VSI %d err %d\n", 12354 alloc_queue_pairs, vsi->seid, ret); 12355 goto err_vsi; 12356 } 12357 vsi->base_queue = ret; 12358 12359 /* Update the FW view of the VSI. Force a reset of TC and queue 12360 * layout configurations. 12361 */ 12362 enabled_tc = pf->vsi[pf->lan_vsi]->tc_config.enabled_tc; 12363 pf->vsi[pf->lan_vsi]->tc_config.enabled_tc = 0; 12364 pf->vsi[pf->lan_vsi]->seid = pf->main_vsi_seid; 12365 i40e_vsi_config_tc(pf->vsi[pf->lan_vsi], enabled_tc); 12366 if (vsi->type == I40E_VSI_MAIN) 12367 i40e_rm_default_mac_filter(vsi, pf->hw.mac.perm_addr); 12368 12369 /* assign it some queues */ 12370 ret = i40e_alloc_rings(vsi); 12371 if (ret) 12372 goto err_rings; 12373 12374 /* map all of the rings to the q_vectors */ 12375 i40e_vsi_map_rings_to_vectors(vsi); 12376 return vsi; 12377 12378 err_rings: 12379 i40e_vsi_free_q_vectors(vsi); 12380 if (vsi->netdev_registered) { 12381 vsi->netdev_registered = false; 12382 unregister_netdev(vsi->netdev); 12383 free_netdev(vsi->netdev); 12384 vsi->netdev = NULL; 12385 } 12386 i40e_aq_delete_element(&pf->hw, vsi->seid, NULL); 12387 err_vsi: 12388 i40e_vsi_clear(vsi); 12389 return NULL; 12390 } 12391 12392 /** 12393 * i40e_vsi_setup - Set up a VSI by a given type 12394 * @pf: board private structure 12395 * @type: VSI type 12396 * @uplink_seid: the switch element to link to 12397 * @param1: usage depends upon VSI type. For VF types, indicates VF id 12398 * 12399 * This allocates the sw VSI structure and its queue resources, then add a VSI 12400 * to the identified VEB. 12401 * 12402 * Returns pointer to the successfully allocated and configure VSI sw struct on 12403 * success, otherwise returns NULL on failure. 12404 **/ 12405 struct i40e_vsi *i40e_vsi_setup(struct i40e_pf *pf, u8 type, 12406 u16 uplink_seid, u32 param1) 12407 { 12408 struct i40e_vsi *vsi = NULL; 12409 struct i40e_veb *veb = NULL; 12410 u16 alloc_queue_pairs; 12411 int ret, i; 12412 int v_idx; 12413 12414 /* The requested uplink_seid must be either 12415 * - the PF's port seid 12416 * no VEB is needed because this is the PF 12417 * or this is a Flow Director special case VSI 12418 * - seid of an existing VEB 12419 * - seid of a VSI that owns an existing VEB 12420 * - seid of a VSI that doesn't own a VEB 12421 * a new VEB is created and the VSI becomes the owner 12422 * - seid of the PF VSI, which is what creates the first VEB 12423 * this is a special case of the previous 12424 * 12425 * Find which uplink_seid we were given and create a new VEB if needed 12426 */ 12427 for (i = 0; i < I40E_MAX_VEB; i++) { 12428 if (pf->veb[i] && pf->veb[i]->seid == uplink_seid) { 12429 veb = pf->veb[i]; 12430 break; 12431 } 12432 } 12433 12434 if (!veb && uplink_seid != pf->mac_seid) { 12435 12436 for (i = 0; i < pf->num_alloc_vsi; i++) { 12437 if (pf->vsi[i] && pf->vsi[i]->seid == uplink_seid) { 12438 vsi = pf->vsi[i]; 12439 break; 12440 } 12441 } 12442 if (!vsi) { 12443 dev_info(&pf->pdev->dev, "no such uplink_seid %d\n", 12444 uplink_seid); 12445 return NULL; 12446 } 12447 12448 if (vsi->uplink_seid == pf->mac_seid) 12449 veb = i40e_veb_setup(pf, 0, pf->mac_seid, vsi->seid, 12450 vsi->tc_config.enabled_tc); 12451 else if ((vsi->flags & I40E_VSI_FLAG_VEB_OWNER) == 0) 12452 veb = i40e_veb_setup(pf, 0, vsi->uplink_seid, vsi->seid, 12453 vsi->tc_config.enabled_tc); 12454 if (veb) { 12455 if (vsi->seid != pf->vsi[pf->lan_vsi]->seid) { 12456 dev_info(&vsi->back->pdev->dev, 12457 "New VSI creation error, uplink seid of LAN VSI expected.\n"); 12458 return NULL; 12459 } 12460 /* We come up by default in VEPA mode if SRIOV is not 12461 * already enabled, in which case we can't force VEPA 12462 * mode. 12463 */ 12464 if (!(pf->flags & I40E_FLAG_VEB_MODE_ENABLED)) { 12465 veb->bridge_mode = BRIDGE_MODE_VEPA; 12466 pf->flags &= ~I40E_FLAG_VEB_MODE_ENABLED; 12467 } 12468 i40e_config_bridge_mode(veb); 12469 } 12470 for (i = 0; i < I40E_MAX_VEB && !veb; i++) { 12471 if (pf->veb[i] && pf->veb[i]->seid == vsi->uplink_seid) 12472 veb = pf->veb[i]; 12473 } 12474 if (!veb) { 12475 dev_info(&pf->pdev->dev, "couldn't add VEB\n"); 12476 return NULL; 12477 } 12478 12479 vsi->flags |= I40E_VSI_FLAG_VEB_OWNER; 12480 uplink_seid = veb->seid; 12481 } 12482 12483 /* get vsi sw struct */ 12484 v_idx = i40e_vsi_mem_alloc(pf, type); 12485 if (v_idx < 0) 12486 goto err_alloc; 12487 vsi = pf->vsi[v_idx]; 12488 if (!vsi) 12489 goto err_alloc; 12490 vsi->type = type; 12491 vsi->veb_idx = (veb ? veb->idx : I40E_NO_VEB); 12492 12493 if (type == I40E_VSI_MAIN) 12494 pf->lan_vsi = v_idx; 12495 else if (type == I40E_VSI_SRIOV) 12496 vsi->vf_id = param1; 12497 /* assign it some queues */ 12498 alloc_queue_pairs = vsi->alloc_queue_pairs * 12499 (i40e_enabled_xdp_vsi(vsi) ? 2 : 1); 12500 12501 ret = i40e_get_lump(pf, pf->qp_pile, alloc_queue_pairs, vsi->idx); 12502 if (ret < 0) { 12503 dev_info(&pf->pdev->dev, 12504 "failed to get tracking for %d queues for VSI %d err=%d\n", 12505 alloc_queue_pairs, vsi->seid, ret); 12506 goto err_vsi; 12507 } 12508 vsi->base_queue = ret; 12509 12510 /* get a VSI from the hardware */ 12511 vsi->uplink_seid = uplink_seid; 12512 ret = i40e_add_vsi(vsi); 12513 if (ret) 12514 goto err_vsi; 12515 12516 switch (vsi->type) { 12517 /* setup the netdev if needed */ 12518 case I40E_VSI_MAIN: 12519 case I40E_VSI_VMDQ2: 12520 ret = i40e_config_netdev(vsi); 12521 if (ret) 12522 goto err_netdev; 12523 ret = register_netdev(vsi->netdev); 12524 if (ret) 12525 goto err_netdev; 12526 vsi->netdev_registered = true; 12527 netif_carrier_off(vsi->netdev); 12528 #ifdef CONFIG_I40E_DCB 12529 /* Setup DCB netlink interface */ 12530 i40e_dcbnl_setup(vsi); 12531 #endif /* CONFIG_I40E_DCB */ 12532 /* fall through */ 12533 12534 case I40E_VSI_FDIR: 12535 /* set up vectors and rings if needed */ 12536 ret = i40e_vsi_setup_vectors(vsi); 12537 if (ret) 12538 goto err_msix; 12539 12540 ret = i40e_alloc_rings(vsi); 12541 if (ret) 12542 goto err_rings; 12543 12544 /* map all of the rings to the q_vectors */ 12545 i40e_vsi_map_rings_to_vectors(vsi); 12546 12547 i40e_vsi_reset_stats(vsi); 12548 break; 12549 12550 default: 12551 /* no netdev or rings for the other VSI types */ 12552 break; 12553 } 12554 12555 if ((pf->hw_features & I40E_HW_RSS_AQ_CAPABLE) && 12556 (vsi->type == I40E_VSI_VMDQ2)) { 12557 ret = i40e_vsi_config_rss(vsi); 12558 } 12559 return vsi; 12560 12561 err_rings: 12562 i40e_vsi_free_q_vectors(vsi); 12563 err_msix: 12564 if (vsi->netdev_registered) { 12565 vsi->netdev_registered = false; 12566 unregister_netdev(vsi->netdev); 12567 free_netdev(vsi->netdev); 12568 vsi->netdev = NULL; 12569 } 12570 err_netdev: 12571 i40e_aq_delete_element(&pf->hw, vsi->seid, NULL); 12572 err_vsi: 12573 i40e_vsi_clear(vsi); 12574 err_alloc: 12575 return NULL; 12576 } 12577 12578 /** 12579 * i40e_veb_get_bw_info - Query VEB BW information 12580 * @veb: the veb to query 12581 * 12582 * Query the Tx scheduler BW configuration data for given VEB 12583 **/ 12584 static int i40e_veb_get_bw_info(struct i40e_veb *veb) 12585 { 12586 struct i40e_aqc_query_switching_comp_ets_config_resp ets_data; 12587 struct i40e_aqc_query_switching_comp_bw_config_resp bw_data; 12588 struct i40e_pf *pf = veb->pf; 12589 struct i40e_hw *hw = &pf->hw; 12590 u32 tc_bw_max; 12591 int ret = 0; 12592 int i; 12593 12594 ret = i40e_aq_query_switch_comp_bw_config(hw, veb->seid, 12595 &bw_data, NULL); 12596 if (ret) { 12597 dev_info(&pf->pdev->dev, 12598 "query veb bw config failed, err %s aq_err %s\n", 12599 i40e_stat_str(&pf->hw, ret), 12600 i40e_aq_str(&pf->hw, hw->aq.asq_last_status)); 12601 goto out; 12602 } 12603 12604 ret = i40e_aq_query_switch_comp_ets_config(hw, veb->seid, 12605 &ets_data, NULL); 12606 if (ret) { 12607 dev_info(&pf->pdev->dev, 12608 "query veb bw ets config failed, err %s aq_err %s\n", 12609 i40e_stat_str(&pf->hw, ret), 12610 i40e_aq_str(&pf->hw, hw->aq.asq_last_status)); 12611 goto out; 12612 } 12613 12614 veb->bw_limit = le16_to_cpu(ets_data.port_bw_limit); 12615 veb->bw_max_quanta = ets_data.tc_bw_max; 12616 veb->is_abs_credits = bw_data.absolute_credits_enable; 12617 veb->enabled_tc = ets_data.tc_valid_bits; 12618 tc_bw_max = le16_to_cpu(bw_data.tc_bw_max[0]) | 12619 (le16_to_cpu(bw_data.tc_bw_max[1]) << 16); 12620 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 12621 veb->bw_tc_share_credits[i] = bw_data.tc_bw_share_credits[i]; 12622 veb->bw_tc_limit_credits[i] = 12623 le16_to_cpu(bw_data.tc_bw_limits[i]); 12624 veb->bw_tc_max_quanta[i] = ((tc_bw_max >> (i*4)) & 0x7); 12625 } 12626 12627 out: 12628 return ret; 12629 } 12630 12631 /** 12632 * i40e_veb_mem_alloc - Allocates the next available struct veb in the PF 12633 * @pf: board private structure 12634 * 12635 * On error: returns error code (negative) 12636 * On success: returns vsi index in PF (positive) 12637 **/ 12638 static int i40e_veb_mem_alloc(struct i40e_pf *pf) 12639 { 12640 int ret = -ENOENT; 12641 struct i40e_veb *veb; 12642 int i; 12643 12644 /* Need to protect the allocation of switch elements at the PF level */ 12645 mutex_lock(&pf->switch_mutex); 12646 12647 /* VEB list may be fragmented if VEB creation/destruction has 12648 * been happening. We can afford to do a quick scan to look 12649 * for any free slots in the list. 12650 * 12651 * find next empty veb slot, looping back around if necessary 12652 */ 12653 i = 0; 12654 while ((i < I40E_MAX_VEB) && (pf->veb[i] != NULL)) 12655 i++; 12656 if (i >= I40E_MAX_VEB) { 12657 ret = -ENOMEM; 12658 goto err_alloc_veb; /* out of VEB slots! */ 12659 } 12660 12661 veb = kzalloc(sizeof(*veb), GFP_KERNEL); 12662 if (!veb) { 12663 ret = -ENOMEM; 12664 goto err_alloc_veb; 12665 } 12666 veb->pf = pf; 12667 veb->idx = i; 12668 veb->enabled_tc = 1; 12669 12670 pf->veb[i] = veb; 12671 ret = i; 12672 err_alloc_veb: 12673 mutex_unlock(&pf->switch_mutex); 12674 return ret; 12675 } 12676 12677 /** 12678 * i40e_switch_branch_release - Delete a branch of the switch tree 12679 * @branch: where to start deleting 12680 * 12681 * This uses recursion to find the tips of the branch to be 12682 * removed, deleting until we get back to and can delete this VEB. 12683 **/ 12684 static void i40e_switch_branch_release(struct i40e_veb *branch) 12685 { 12686 struct i40e_pf *pf = branch->pf; 12687 u16 branch_seid = branch->seid; 12688 u16 veb_idx = branch->idx; 12689 int i; 12690 12691 /* release any VEBs on this VEB - RECURSION */ 12692 for (i = 0; i < I40E_MAX_VEB; i++) { 12693 if (!pf->veb[i]) 12694 continue; 12695 if (pf->veb[i]->uplink_seid == branch->seid) 12696 i40e_switch_branch_release(pf->veb[i]); 12697 } 12698 12699 /* Release the VSIs on this VEB, but not the owner VSI. 12700 * 12701 * NOTE: Removing the last VSI on a VEB has the SIDE EFFECT of removing 12702 * the VEB itself, so don't use (*branch) after this loop. 12703 */ 12704 for (i = 0; i < pf->num_alloc_vsi; i++) { 12705 if (!pf->vsi[i]) 12706 continue; 12707 if (pf->vsi[i]->uplink_seid == branch_seid && 12708 (pf->vsi[i]->flags & I40E_VSI_FLAG_VEB_OWNER) == 0) { 12709 i40e_vsi_release(pf->vsi[i]); 12710 } 12711 } 12712 12713 /* There's one corner case where the VEB might not have been 12714 * removed, so double check it here and remove it if needed. 12715 * This case happens if the veb was created from the debugfs 12716 * commands and no VSIs were added to it. 12717 */ 12718 if (pf->veb[veb_idx]) 12719 i40e_veb_release(pf->veb[veb_idx]); 12720 } 12721 12722 /** 12723 * i40e_veb_clear - remove veb struct 12724 * @veb: the veb to remove 12725 **/ 12726 static void i40e_veb_clear(struct i40e_veb *veb) 12727 { 12728 if (!veb) 12729 return; 12730 12731 if (veb->pf) { 12732 struct i40e_pf *pf = veb->pf; 12733 12734 mutex_lock(&pf->switch_mutex); 12735 if (pf->veb[veb->idx] == veb) 12736 pf->veb[veb->idx] = NULL; 12737 mutex_unlock(&pf->switch_mutex); 12738 } 12739 12740 kfree(veb); 12741 } 12742 12743 /** 12744 * i40e_veb_release - Delete a VEB and free its resources 12745 * @veb: the VEB being removed 12746 **/ 12747 void i40e_veb_release(struct i40e_veb *veb) 12748 { 12749 struct i40e_vsi *vsi = NULL; 12750 struct i40e_pf *pf; 12751 int i, n = 0; 12752 12753 pf = veb->pf; 12754 12755 /* find the remaining VSI and check for extras */ 12756 for (i = 0; i < pf->num_alloc_vsi; i++) { 12757 if (pf->vsi[i] && pf->vsi[i]->uplink_seid == veb->seid) { 12758 n++; 12759 vsi = pf->vsi[i]; 12760 } 12761 } 12762 if (n != 1) { 12763 dev_info(&pf->pdev->dev, 12764 "can't remove VEB %d with %d VSIs left\n", 12765 veb->seid, n); 12766 return; 12767 } 12768 12769 /* move the remaining VSI to uplink veb */ 12770 vsi->flags &= ~I40E_VSI_FLAG_VEB_OWNER; 12771 if (veb->uplink_seid) { 12772 vsi->uplink_seid = veb->uplink_seid; 12773 if (veb->uplink_seid == pf->mac_seid) 12774 vsi->veb_idx = I40E_NO_VEB; 12775 else 12776 vsi->veb_idx = veb->veb_idx; 12777 } else { 12778 /* floating VEB */ 12779 vsi->uplink_seid = pf->vsi[pf->lan_vsi]->uplink_seid; 12780 vsi->veb_idx = pf->vsi[pf->lan_vsi]->veb_idx; 12781 } 12782 12783 i40e_aq_delete_element(&pf->hw, veb->seid, NULL); 12784 i40e_veb_clear(veb); 12785 } 12786 12787 /** 12788 * i40e_add_veb - create the VEB in the switch 12789 * @veb: the VEB to be instantiated 12790 * @vsi: the controlling VSI 12791 **/ 12792 static int i40e_add_veb(struct i40e_veb *veb, struct i40e_vsi *vsi) 12793 { 12794 struct i40e_pf *pf = veb->pf; 12795 bool enable_stats = !!(pf->flags & I40E_FLAG_VEB_STATS_ENABLED); 12796 int ret; 12797 12798 ret = i40e_aq_add_veb(&pf->hw, veb->uplink_seid, vsi->seid, 12799 veb->enabled_tc, false, 12800 &veb->seid, enable_stats, NULL); 12801 12802 /* get a VEB from the hardware */ 12803 if (ret) { 12804 dev_info(&pf->pdev->dev, 12805 "couldn't add VEB, err %s aq_err %s\n", 12806 i40e_stat_str(&pf->hw, ret), 12807 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 12808 return -EPERM; 12809 } 12810 12811 /* get statistics counter */ 12812 ret = i40e_aq_get_veb_parameters(&pf->hw, veb->seid, NULL, NULL, 12813 &veb->stats_idx, NULL, NULL, NULL); 12814 if (ret) { 12815 dev_info(&pf->pdev->dev, 12816 "couldn't get VEB statistics idx, err %s aq_err %s\n", 12817 i40e_stat_str(&pf->hw, ret), 12818 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 12819 return -EPERM; 12820 } 12821 ret = i40e_veb_get_bw_info(veb); 12822 if (ret) { 12823 dev_info(&pf->pdev->dev, 12824 "couldn't get VEB bw info, err %s aq_err %s\n", 12825 i40e_stat_str(&pf->hw, ret), 12826 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 12827 i40e_aq_delete_element(&pf->hw, veb->seid, NULL); 12828 return -ENOENT; 12829 } 12830 12831 vsi->uplink_seid = veb->seid; 12832 vsi->veb_idx = veb->idx; 12833 vsi->flags |= I40E_VSI_FLAG_VEB_OWNER; 12834 12835 return 0; 12836 } 12837 12838 /** 12839 * i40e_veb_setup - Set up a VEB 12840 * @pf: board private structure 12841 * @flags: VEB setup flags 12842 * @uplink_seid: the switch element to link to 12843 * @vsi_seid: the initial VSI seid 12844 * @enabled_tc: Enabled TC bit-map 12845 * 12846 * This allocates the sw VEB structure and links it into the switch 12847 * It is possible and legal for this to be a duplicate of an already 12848 * existing VEB. It is also possible for both uplink and vsi seids 12849 * to be zero, in order to create a floating VEB. 12850 * 12851 * Returns pointer to the successfully allocated VEB sw struct on 12852 * success, otherwise returns NULL on failure. 12853 **/ 12854 struct i40e_veb *i40e_veb_setup(struct i40e_pf *pf, u16 flags, 12855 u16 uplink_seid, u16 vsi_seid, 12856 u8 enabled_tc) 12857 { 12858 struct i40e_veb *veb, *uplink_veb = NULL; 12859 int vsi_idx, veb_idx; 12860 int ret; 12861 12862 /* if one seid is 0, the other must be 0 to create a floating relay */ 12863 if ((uplink_seid == 0 || vsi_seid == 0) && 12864 (uplink_seid + vsi_seid != 0)) { 12865 dev_info(&pf->pdev->dev, 12866 "one, not both seid's are 0: uplink=%d vsi=%d\n", 12867 uplink_seid, vsi_seid); 12868 return NULL; 12869 } 12870 12871 /* make sure there is such a vsi and uplink */ 12872 for (vsi_idx = 0; vsi_idx < pf->num_alloc_vsi; vsi_idx++) 12873 if (pf->vsi[vsi_idx] && pf->vsi[vsi_idx]->seid == vsi_seid) 12874 break; 12875 if (vsi_idx >= pf->num_alloc_vsi && vsi_seid != 0) { 12876 dev_info(&pf->pdev->dev, "vsi seid %d not found\n", 12877 vsi_seid); 12878 return NULL; 12879 } 12880 12881 if (uplink_seid && uplink_seid != pf->mac_seid) { 12882 for (veb_idx = 0; veb_idx < I40E_MAX_VEB; veb_idx++) { 12883 if (pf->veb[veb_idx] && 12884 pf->veb[veb_idx]->seid == uplink_seid) { 12885 uplink_veb = pf->veb[veb_idx]; 12886 break; 12887 } 12888 } 12889 if (!uplink_veb) { 12890 dev_info(&pf->pdev->dev, 12891 "uplink seid %d not found\n", uplink_seid); 12892 return NULL; 12893 } 12894 } 12895 12896 /* get veb sw struct */ 12897 veb_idx = i40e_veb_mem_alloc(pf); 12898 if (veb_idx < 0) 12899 goto err_alloc; 12900 veb = pf->veb[veb_idx]; 12901 veb->flags = flags; 12902 veb->uplink_seid = uplink_seid; 12903 veb->veb_idx = (uplink_veb ? uplink_veb->idx : I40E_NO_VEB); 12904 veb->enabled_tc = (enabled_tc ? enabled_tc : 0x1); 12905 12906 /* create the VEB in the switch */ 12907 ret = i40e_add_veb(veb, pf->vsi[vsi_idx]); 12908 if (ret) 12909 goto err_veb; 12910 if (vsi_idx == pf->lan_vsi) 12911 pf->lan_veb = veb->idx; 12912 12913 return veb; 12914 12915 err_veb: 12916 i40e_veb_clear(veb); 12917 err_alloc: 12918 return NULL; 12919 } 12920 12921 /** 12922 * i40e_setup_pf_switch_element - set PF vars based on switch type 12923 * @pf: board private structure 12924 * @ele: element we are building info from 12925 * @num_reported: total number of elements 12926 * @printconfig: should we print the contents 12927 * 12928 * helper function to assist in extracting a few useful SEID values. 12929 **/ 12930 static void i40e_setup_pf_switch_element(struct i40e_pf *pf, 12931 struct i40e_aqc_switch_config_element_resp *ele, 12932 u16 num_reported, bool printconfig) 12933 { 12934 u16 downlink_seid = le16_to_cpu(ele->downlink_seid); 12935 u16 uplink_seid = le16_to_cpu(ele->uplink_seid); 12936 u8 element_type = ele->element_type; 12937 u16 seid = le16_to_cpu(ele->seid); 12938 12939 if (printconfig) 12940 dev_info(&pf->pdev->dev, 12941 "type=%d seid=%d uplink=%d downlink=%d\n", 12942 element_type, seid, uplink_seid, downlink_seid); 12943 12944 switch (element_type) { 12945 case I40E_SWITCH_ELEMENT_TYPE_MAC: 12946 pf->mac_seid = seid; 12947 break; 12948 case I40E_SWITCH_ELEMENT_TYPE_VEB: 12949 /* Main VEB? */ 12950 if (uplink_seid != pf->mac_seid) 12951 break; 12952 if (pf->lan_veb == I40E_NO_VEB) { 12953 int v; 12954 12955 /* find existing or else empty VEB */ 12956 for (v = 0; v < I40E_MAX_VEB; v++) { 12957 if (pf->veb[v] && (pf->veb[v]->seid == seid)) { 12958 pf->lan_veb = v; 12959 break; 12960 } 12961 } 12962 if (pf->lan_veb == I40E_NO_VEB) { 12963 v = i40e_veb_mem_alloc(pf); 12964 if (v < 0) 12965 break; 12966 pf->lan_veb = v; 12967 } 12968 } 12969 12970 pf->veb[pf->lan_veb]->seid = seid; 12971 pf->veb[pf->lan_veb]->uplink_seid = pf->mac_seid; 12972 pf->veb[pf->lan_veb]->pf = pf; 12973 pf->veb[pf->lan_veb]->veb_idx = I40E_NO_VEB; 12974 break; 12975 case I40E_SWITCH_ELEMENT_TYPE_VSI: 12976 if (num_reported != 1) 12977 break; 12978 /* This is immediately after a reset so we can assume this is 12979 * the PF's VSI 12980 */ 12981 pf->mac_seid = uplink_seid; 12982 pf->pf_seid = downlink_seid; 12983 pf->main_vsi_seid = seid; 12984 if (printconfig) 12985 dev_info(&pf->pdev->dev, 12986 "pf_seid=%d main_vsi_seid=%d\n", 12987 pf->pf_seid, pf->main_vsi_seid); 12988 break; 12989 case I40E_SWITCH_ELEMENT_TYPE_PF: 12990 case I40E_SWITCH_ELEMENT_TYPE_VF: 12991 case I40E_SWITCH_ELEMENT_TYPE_EMP: 12992 case I40E_SWITCH_ELEMENT_TYPE_BMC: 12993 case I40E_SWITCH_ELEMENT_TYPE_PE: 12994 case I40E_SWITCH_ELEMENT_TYPE_PA: 12995 /* ignore these for now */ 12996 break; 12997 default: 12998 dev_info(&pf->pdev->dev, "unknown element type=%d seid=%d\n", 12999 element_type, seid); 13000 break; 13001 } 13002 } 13003 13004 /** 13005 * i40e_fetch_switch_configuration - Get switch config from firmware 13006 * @pf: board private structure 13007 * @printconfig: should we print the contents 13008 * 13009 * Get the current switch configuration from the device and 13010 * extract a few useful SEID values. 13011 **/ 13012 int i40e_fetch_switch_configuration(struct i40e_pf *pf, bool printconfig) 13013 { 13014 struct i40e_aqc_get_switch_config_resp *sw_config; 13015 u16 next_seid = 0; 13016 int ret = 0; 13017 u8 *aq_buf; 13018 int i; 13019 13020 aq_buf = kzalloc(I40E_AQ_LARGE_BUF, GFP_KERNEL); 13021 if (!aq_buf) 13022 return -ENOMEM; 13023 13024 sw_config = (struct i40e_aqc_get_switch_config_resp *)aq_buf; 13025 do { 13026 u16 num_reported, num_total; 13027 13028 ret = i40e_aq_get_switch_config(&pf->hw, sw_config, 13029 I40E_AQ_LARGE_BUF, 13030 &next_seid, NULL); 13031 if (ret) { 13032 dev_info(&pf->pdev->dev, 13033 "get switch config failed err %s aq_err %s\n", 13034 i40e_stat_str(&pf->hw, ret), 13035 i40e_aq_str(&pf->hw, 13036 pf->hw.aq.asq_last_status)); 13037 kfree(aq_buf); 13038 return -ENOENT; 13039 } 13040 13041 num_reported = le16_to_cpu(sw_config->header.num_reported); 13042 num_total = le16_to_cpu(sw_config->header.num_total); 13043 13044 if (printconfig) 13045 dev_info(&pf->pdev->dev, 13046 "header: %d reported %d total\n", 13047 num_reported, num_total); 13048 13049 for (i = 0; i < num_reported; i++) { 13050 struct i40e_aqc_switch_config_element_resp *ele = 13051 &sw_config->element[i]; 13052 13053 i40e_setup_pf_switch_element(pf, ele, num_reported, 13054 printconfig); 13055 } 13056 } while (next_seid != 0); 13057 13058 kfree(aq_buf); 13059 return ret; 13060 } 13061 13062 /** 13063 * i40e_setup_pf_switch - Setup the HW switch on startup or after reset 13064 * @pf: board private structure 13065 * @reinit: if the Main VSI needs to re-initialized. 13066 * 13067 * Returns 0 on success, negative value on failure 13068 **/ 13069 static int i40e_setup_pf_switch(struct i40e_pf *pf, bool reinit) 13070 { 13071 u16 flags = 0; 13072 int ret; 13073 13074 /* find out what's out there already */ 13075 ret = i40e_fetch_switch_configuration(pf, false); 13076 if (ret) { 13077 dev_info(&pf->pdev->dev, 13078 "couldn't fetch switch config, err %s aq_err %s\n", 13079 i40e_stat_str(&pf->hw, ret), 13080 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 13081 return ret; 13082 } 13083 i40e_pf_reset_stats(pf); 13084 13085 /* set the switch config bit for the whole device to 13086 * support limited promisc or true promisc 13087 * when user requests promisc. The default is limited 13088 * promisc. 13089 */ 13090 13091 if ((pf->hw.pf_id == 0) && 13092 !(pf->flags & I40E_FLAG_TRUE_PROMISC_SUPPORT)) { 13093 flags = I40E_AQ_SET_SWITCH_CFG_PROMISC; 13094 pf->last_sw_conf_flags = flags; 13095 } 13096 13097 if (pf->hw.pf_id == 0) { 13098 u16 valid_flags; 13099 13100 valid_flags = I40E_AQ_SET_SWITCH_CFG_PROMISC; 13101 ret = i40e_aq_set_switch_config(&pf->hw, flags, valid_flags, 0, 13102 NULL); 13103 if (ret && pf->hw.aq.asq_last_status != I40E_AQ_RC_ESRCH) { 13104 dev_info(&pf->pdev->dev, 13105 "couldn't set switch config bits, err %s aq_err %s\n", 13106 i40e_stat_str(&pf->hw, ret), 13107 i40e_aq_str(&pf->hw, 13108 pf->hw.aq.asq_last_status)); 13109 /* not a fatal problem, just keep going */ 13110 } 13111 pf->last_sw_conf_valid_flags = valid_flags; 13112 } 13113 13114 /* first time setup */ 13115 if (pf->lan_vsi == I40E_NO_VSI || reinit) { 13116 struct i40e_vsi *vsi = NULL; 13117 u16 uplink_seid; 13118 13119 /* Set up the PF VSI associated with the PF's main VSI 13120 * that is already in the HW switch 13121 */ 13122 if (pf->lan_veb != I40E_NO_VEB && pf->veb[pf->lan_veb]) 13123 uplink_seid = pf->veb[pf->lan_veb]->seid; 13124 else 13125 uplink_seid = pf->mac_seid; 13126 if (pf->lan_vsi == I40E_NO_VSI) 13127 vsi = i40e_vsi_setup(pf, I40E_VSI_MAIN, uplink_seid, 0); 13128 else if (reinit) 13129 vsi = i40e_vsi_reinit_setup(pf->vsi[pf->lan_vsi]); 13130 if (!vsi) { 13131 dev_info(&pf->pdev->dev, "setup of MAIN VSI failed\n"); 13132 i40e_cloud_filter_exit(pf); 13133 i40e_fdir_teardown(pf); 13134 return -EAGAIN; 13135 } 13136 } else { 13137 /* force a reset of TC and queue layout configurations */ 13138 u8 enabled_tc = pf->vsi[pf->lan_vsi]->tc_config.enabled_tc; 13139 13140 pf->vsi[pf->lan_vsi]->tc_config.enabled_tc = 0; 13141 pf->vsi[pf->lan_vsi]->seid = pf->main_vsi_seid; 13142 i40e_vsi_config_tc(pf->vsi[pf->lan_vsi], enabled_tc); 13143 } 13144 i40e_vlan_stripping_disable(pf->vsi[pf->lan_vsi]); 13145 13146 i40e_fdir_sb_setup(pf); 13147 13148 /* Setup static PF queue filter control settings */ 13149 ret = i40e_setup_pf_filter_control(pf); 13150 if (ret) { 13151 dev_info(&pf->pdev->dev, "setup_pf_filter_control failed: %d\n", 13152 ret); 13153 /* Failure here should not stop continuing other steps */ 13154 } 13155 13156 /* enable RSS in the HW, even for only one queue, as the stack can use 13157 * the hash 13158 */ 13159 if ((pf->flags & I40E_FLAG_RSS_ENABLED)) 13160 i40e_pf_config_rss(pf); 13161 13162 /* fill in link information and enable LSE reporting */ 13163 i40e_link_event(pf); 13164 13165 /* Initialize user-specific link properties */ 13166 pf->fc_autoneg_status = ((pf->hw.phy.link_info.an_info & 13167 I40E_AQ_AN_COMPLETED) ? true : false); 13168 13169 i40e_ptp_init(pf); 13170 13171 /* repopulate tunnel port filters */ 13172 i40e_sync_udp_filters(pf); 13173 13174 return ret; 13175 } 13176 13177 /** 13178 * i40e_determine_queue_usage - Work out queue distribution 13179 * @pf: board private structure 13180 **/ 13181 static void i40e_determine_queue_usage(struct i40e_pf *pf) 13182 { 13183 int queues_left; 13184 int q_max; 13185 13186 pf->num_lan_qps = 0; 13187 13188 /* Find the max queues to be put into basic use. We'll always be 13189 * using TC0, whether or not DCB is running, and TC0 will get the 13190 * big RSS set. 13191 */ 13192 queues_left = pf->hw.func_caps.num_tx_qp; 13193 13194 if ((queues_left == 1) || 13195 !(pf->flags & I40E_FLAG_MSIX_ENABLED)) { 13196 /* one qp for PF, no queues for anything else */ 13197 queues_left = 0; 13198 pf->alloc_rss_size = pf->num_lan_qps = 1; 13199 13200 /* make sure all the fancies are disabled */ 13201 pf->flags &= ~(I40E_FLAG_RSS_ENABLED | 13202 I40E_FLAG_IWARP_ENABLED | 13203 I40E_FLAG_FD_SB_ENABLED | 13204 I40E_FLAG_FD_ATR_ENABLED | 13205 I40E_FLAG_DCB_CAPABLE | 13206 I40E_FLAG_DCB_ENABLED | 13207 I40E_FLAG_SRIOV_ENABLED | 13208 I40E_FLAG_VMDQ_ENABLED); 13209 pf->flags |= I40E_FLAG_FD_SB_INACTIVE; 13210 } else if (!(pf->flags & (I40E_FLAG_RSS_ENABLED | 13211 I40E_FLAG_FD_SB_ENABLED | 13212 I40E_FLAG_FD_ATR_ENABLED | 13213 I40E_FLAG_DCB_CAPABLE))) { 13214 /* one qp for PF */ 13215 pf->alloc_rss_size = pf->num_lan_qps = 1; 13216 queues_left -= pf->num_lan_qps; 13217 13218 pf->flags &= ~(I40E_FLAG_RSS_ENABLED | 13219 I40E_FLAG_IWARP_ENABLED | 13220 I40E_FLAG_FD_SB_ENABLED | 13221 I40E_FLAG_FD_ATR_ENABLED | 13222 I40E_FLAG_DCB_ENABLED | 13223 I40E_FLAG_VMDQ_ENABLED); 13224 pf->flags |= I40E_FLAG_FD_SB_INACTIVE; 13225 } else { 13226 /* Not enough queues for all TCs */ 13227 if ((pf->flags & I40E_FLAG_DCB_CAPABLE) && 13228 (queues_left < I40E_MAX_TRAFFIC_CLASS)) { 13229 pf->flags &= ~(I40E_FLAG_DCB_CAPABLE | 13230 I40E_FLAG_DCB_ENABLED); 13231 dev_info(&pf->pdev->dev, "not enough queues for DCB. DCB is disabled.\n"); 13232 } 13233 13234 /* limit lan qps to the smaller of qps, cpus or msix */ 13235 q_max = max_t(int, pf->rss_size_max, num_online_cpus()); 13236 q_max = min_t(int, q_max, pf->hw.func_caps.num_tx_qp); 13237 q_max = min_t(int, q_max, pf->hw.func_caps.num_msix_vectors); 13238 pf->num_lan_qps = q_max; 13239 13240 queues_left -= pf->num_lan_qps; 13241 } 13242 13243 if (pf->flags & I40E_FLAG_FD_SB_ENABLED) { 13244 if (queues_left > 1) { 13245 queues_left -= 1; /* save 1 queue for FD */ 13246 } else { 13247 pf->flags &= ~I40E_FLAG_FD_SB_ENABLED; 13248 pf->flags |= I40E_FLAG_FD_SB_INACTIVE; 13249 dev_info(&pf->pdev->dev, "not enough queues for Flow Director. Flow Director feature is disabled\n"); 13250 } 13251 } 13252 13253 if ((pf->flags & I40E_FLAG_SRIOV_ENABLED) && 13254 pf->num_vf_qps && pf->num_req_vfs && queues_left) { 13255 pf->num_req_vfs = min_t(int, pf->num_req_vfs, 13256 (queues_left / pf->num_vf_qps)); 13257 queues_left -= (pf->num_req_vfs * pf->num_vf_qps); 13258 } 13259 13260 if ((pf->flags & I40E_FLAG_VMDQ_ENABLED) && 13261 pf->num_vmdq_vsis && pf->num_vmdq_qps && queues_left) { 13262 pf->num_vmdq_vsis = min_t(int, pf->num_vmdq_vsis, 13263 (queues_left / pf->num_vmdq_qps)); 13264 queues_left -= (pf->num_vmdq_vsis * pf->num_vmdq_qps); 13265 } 13266 13267 pf->queues_left = queues_left; 13268 dev_dbg(&pf->pdev->dev, 13269 "qs_avail=%d FD SB=%d lan_qs=%d lan_tc0=%d vf=%d*%d vmdq=%d*%d, remaining=%d\n", 13270 pf->hw.func_caps.num_tx_qp, 13271 !!(pf->flags & I40E_FLAG_FD_SB_ENABLED), 13272 pf->num_lan_qps, pf->alloc_rss_size, pf->num_req_vfs, 13273 pf->num_vf_qps, pf->num_vmdq_vsis, pf->num_vmdq_qps, 13274 queues_left); 13275 } 13276 13277 /** 13278 * i40e_setup_pf_filter_control - Setup PF static filter control 13279 * @pf: PF to be setup 13280 * 13281 * i40e_setup_pf_filter_control sets up a PF's initial filter control 13282 * settings. If PE/FCoE are enabled then it will also set the per PF 13283 * based filter sizes required for them. It also enables Flow director, 13284 * ethertype and macvlan type filter settings for the pf. 13285 * 13286 * Returns 0 on success, negative on failure 13287 **/ 13288 static int i40e_setup_pf_filter_control(struct i40e_pf *pf) 13289 { 13290 struct i40e_filter_control_settings *settings = &pf->filter_settings; 13291 13292 settings->hash_lut_size = I40E_HASH_LUT_SIZE_128; 13293 13294 /* Flow Director is enabled */ 13295 if (pf->flags & (I40E_FLAG_FD_SB_ENABLED | I40E_FLAG_FD_ATR_ENABLED)) 13296 settings->enable_fdir = true; 13297 13298 /* Ethtype and MACVLAN filters enabled for PF */ 13299 settings->enable_ethtype = true; 13300 settings->enable_macvlan = true; 13301 13302 if (i40e_set_filter_control(&pf->hw, settings)) 13303 return -ENOENT; 13304 13305 return 0; 13306 } 13307 13308 #define INFO_STRING_LEN 255 13309 #define REMAIN(__x) (INFO_STRING_LEN - (__x)) 13310 static void i40e_print_features(struct i40e_pf *pf) 13311 { 13312 struct i40e_hw *hw = &pf->hw; 13313 char *buf; 13314 int i; 13315 13316 buf = kmalloc(INFO_STRING_LEN, GFP_KERNEL); 13317 if (!buf) 13318 return; 13319 13320 i = snprintf(buf, INFO_STRING_LEN, "Features: PF-id[%d]", hw->pf_id); 13321 #ifdef CONFIG_PCI_IOV 13322 i += snprintf(&buf[i], REMAIN(i), " VFs: %d", pf->num_req_vfs); 13323 #endif 13324 i += snprintf(&buf[i], REMAIN(i), " VSIs: %d QP: %d", 13325 pf->hw.func_caps.num_vsis, 13326 pf->vsi[pf->lan_vsi]->num_queue_pairs); 13327 if (pf->flags & I40E_FLAG_RSS_ENABLED) 13328 i += snprintf(&buf[i], REMAIN(i), " RSS"); 13329 if (pf->flags & I40E_FLAG_FD_ATR_ENABLED) 13330 i += snprintf(&buf[i], REMAIN(i), " FD_ATR"); 13331 if (pf->flags & I40E_FLAG_FD_SB_ENABLED) { 13332 i += snprintf(&buf[i], REMAIN(i), " FD_SB"); 13333 i += snprintf(&buf[i], REMAIN(i), " NTUPLE"); 13334 } 13335 if (pf->flags & I40E_FLAG_DCB_CAPABLE) 13336 i += snprintf(&buf[i], REMAIN(i), " DCB"); 13337 i += snprintf(&buf[i], REMAIN(i), " VxLAN"); 13338 i += snprintf(&buf[i], REMAIN(i), " Geneve"); 13339 if (pf->flags & I40E_FLAG_PTP) 13340 i += snprintf(&buf[i], REMAIN(i), " PTP"); 13341 if (pf->flags & I40E_FLAG_VEB_MODE_ENABLED) 13342 i += snprintf(&buf[i], REMAIN(i), " VEB"); 13343 else 13344 i += snprintf(&buf[i], REMAIN(i), " VEPA"); 13345 13346 dev_info(&pf->pdev->dev, "%s\n", buf); 13347 kfree(buf); 13348 WARN_ON(i > INFO_STRING_LEN); 13349 } 13350 13351 /** 13352 * i40e_get_platform_mac_addr - get platform-specific MAC address 13353 * @pdev: PCI device information struct 13354 * @pf: board private structure 13355 * 13356 * Look up the MAC address for the device. First we'll try 13357 * eth_platform_get_mac_address, which will check Open Firmware, or arch 13358 * specific fallback. Otherwise, we'll default to the stored value in 13359 * firmware. 13360 **/ 13361 static void i40e_get_platform_mac_addr(struct pci_dev *pdev, struct i40e_pf *pf) 13362 { 13363 if (eth_platform_get_mac_address(&pdev->dev, pf->hw.mac.addr)) 13364 i40e_get_mac_addr(&pf->hw, pf->hw.mac.addr); 13365 } 13366 13367 /** 13368 * i40e_probe - Device initialization routine 13369 * @pdev: PCI device information struct 13370 * @ent: entry in i40e_pci_tbl 13371 * 13372 * i40e_probe initializes a PF identified by a pci_dev structure. 13373 * The OS initialization, configuring of the PF private structure, 13374 * and a hardware reset occur. 13375 * 13376 * Returns 0 on success, negative on failure 13377 **/ 13378 static int i40e_probe(struct pci_dev *pdev, const struct pci_device_id *ent) 13379 { 13380 struct i40e_aq_get_phy_abilities_resp abilities; 13381 struct i40e_pf *pf; 13382 struct i40e_hw *hw; 13383 static u16 pfs_found; 13384 u16 wol_nvm_bits; 13385 u16 link_status; 13386 int err; 13387 u32 val; 13388 u32 i; 13389 u8 set_fc_aq_fail; 13390 13391 err = pci_enable_device_mem(pdev); 13392 if (err) 13393 return err; 13394 13395 /* set up for high or low dma */ 13396 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); 13397 if (err) { 13398 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)); 13399 if (err) { 13400 dev_err(&pdev->dev, 13401 "DMA configuration failed: 0x%x\n", err); 13402 goto err_dma; 13403 } 13404 } 13405 13406 /* set up pci connections */ 13407 err = pci_request_mem_regions(pdev, i40e_driver_name); 13408 if (err) { 13409 dev_info(&pdev->dev, 13410 "pci_request_selected_regions failed %d\n", err); 13411 goto err_pci_reg; 13412 } 13413 13414 pci_enable_pcie_error_reporting(pdev); 13415 pci_set_master(pdev); 13416 13417 /* Now that we have a PCI connection, we need to do the 13418 * low level device setup. This is primarily setting up 13419 * the Admin Queue structures and then querying for the 13420 * device's current profile information. 13421 */ 13422 pf = kzalloc(sizeof(*pf), GFP_KERNEL); 13423 if (!pf) { 13424 err = -ENOMEM; 13425 goto err_pf_alloc; 13426 } 13427 pf->next_vsi = 0; 13428 pf->pdev = pdev; 13429 set_bit(__I40E_DOWN, pf->state); 13430 13431 hw = &pf->hw; 13432 hw->back = pf; 13433 13434 pf->ioremap_len = min_t(int, pci_resource_len(pdev, 0), 13435 I40E_MAX_CSR_SPACE); 13436 13437 hw->hw_addr = ioremap(pci_resource_start(pdev, 0), pf->ioremap_len); 13438 if (!hw->hw_addr) { 13439 err = -EIO; 13440 dev_info(&pdev->dev, "ioremap(0x%04x, 0x%04x) failed: 0x%x\n", 13441 (unsigned int)pci_resource_start(pdev, 0), 13442 pf->ioremap_len, err); 13443 goto err_ioremap; 13444 } 13445 hw->vendor_id = pdev->vendor; 13446 hw->device_id = pdev->device; 13447 pci_read_config_byte(pdev, PCI_REVISION_ID, &hw->revision_id); 13448 hw->subsystem_vendor_id = pdev->subsystem_vendor; 13449 hw->subsystem_device_id = pdev->subsystem_device; 13450 hw->bus.device = PCI_SLOT(pdev->devfn); 13451 hw->bus.func = PCI_FUNC(pdev->devfn); 13452 hw->bus.bus_id = pdev->bus->number; 13453 pf->instance = pfs_found; 13454 13455 /* Select something other than the 802.1ad ethertype for the 13456 * switch to use internally and drop on ingress. 13457 */ 13458 hw->switch_tag = 0xffff; 13459 hw->first_tag = ETH_P_8021AD; 13460 hw->second_tag = ETH_P_8021Q; 13461 13462 INIT_LIST_HEAD(&pf->l3_flex_pit_list); 13463 INIT_LIST_HEAD(&pf->l4_flex_pit_list); 13464 13465 /* set up the locks for the AQ, do this only once in probe 13466 * and destroy them only once in remove 13467 */ 13468 mutex_init(&hw->aq.asq_mutex); 13469 mutex_init(&hw->aq.arq_mutex); 13470 13471 pf->msg_enable = netif_msg_init(debug, 13472 NETIF_MSG_DRV | 13473 NETIF_MSG_PROBE | 13474 NETIF_MSG_LINK); 13475 if (debug < -1) 13476 pf->hw.debug_mask = debug; 13477 13478 /* do a special CORER for clearing PXE mode once at init */ 13479 if (hw->revision_id == 0 && 13480 (rd32(hw, I40E_GLLAN_RCTL_0) & I40E_GLLAN_RCTL_0_PXE_MODE_MASK)) { 13481 wr32(hw, I40E_GLGEN_RTRIG, I40E_GLGEN_RTRIG_CORER_MASK); 13482 i40e_flush(hw); 13483 msleep(200); 13484 pf->corer_count++; 13485 13486 i40e_clear_pxe_mode(hw); 13487 } 13488 13489 /* Reset here to make sure all is clean and to define PF 'n' */ 13490 i40e_clear_hw(hw); 13491 err = i40e_pf_reset(hw); 13492 if (err) { 13493 dev_info(&pdev->dev, "Initial pf_reset failed: %d\n", err); 13494 goto err_pf_reset; 13495 } 13496 pf->pfr_count++; 13497 13498 hw->aq.num_arq_entries = I40E_AQ_LEN; 13499 hw->aq.num_asq_entries = I40E_AQ_LEN; 13500 hw->aq.arq_buf_size = I40E_MAX_AQ_BUF_SIZE; 13501 hw->aq.asq_buf_size = I40E_MAX_AQ_BUF_SIZE; 13502 pf->adminq_work_limit = I40E_AQ_WORK_LIMIT; 13503 13504 snprintf(pf->int_name, sizeof(pf->int_name) - 1, 13505 "%s-%s:misc", 13506 dev_driver_string(&pf->pdev->dev), dev_name(&pdev->dev)); 13507 13508 err = i40e_init_shared_code(hw); 13509 if (err) { 13510 dev_warn(&pdev->dev, "unidentified MAC or BLANK NVM: %d\n", 13511 err); 13512 goto err_pf_reset; 13513 } 13514 13515 /* set up a default setting for link flow control */ 13516 pf->hw.fc.requested_mode = I40E_FC_NONE; 13517 13518 err = i40e_init_adminq(hw); 13519 if (err) { 13520 if (err == I40E_ERR_FIRMWARE_API_VERSION) 13521 dev_info(&pdev->dev, 13522 "The driver for the device stopped because the NVM image is newer than expected. You must install the most recent version of the network driver.\n"); 13523 else 13524 dev_info(&pdev->dev, 13525 "The driver for the device stopped because the device firmware failed to init. Try updating your NVM image.\n"); 13526 13527 goto err_pf_reset; 13528 } 13529 i40e_get_oem_version(hw); 13530 13531 /* provide nvm, fw, api versions */ 13532 dev_info(&pdev->dev, "fw %d.%d.%05d api %d.%d nvm %s\n", 13533 hw->aq.fw_maj_ver, hw->aq.fw_min_ver, hw->aq.fw_build, 13534 hw->aq.api_maj_ver, hw->aq.api_min_ver, 13535 i40e_nvm_version_str(hw)); 13536 13537 if (hw->aq.api_maj_ver == I40E_FW_API_VERSION_MAJOR && 13538 hw->aq.api_min_ver > I40E_FW_MINOR_VERSION(hw)) 13539 dev_info(&pdev->dev, 13540 "The driver for the device detected a newer version of the NVM image than expected. Please install the most recent version of the network driver.\n"); 13541 else if (hw->aq.api_maj_ver == 1 && hw->aq.api_min_ver < 4) 13542 dev_info(&pdev->dev, 13543 "The driver for the device detected an older version of the NVM image than expected. Please update the NVM image.\n"); 13544 13545 i40e_verify_eeprom(pf); 13546 13547 /* Rev 0 hardware was never productized */ 13548 if (hw->revision_id < 1) 13549 dev_warn(&pdev->dev, "This device is a pre-production adapter/LOM. Please be aware there may be issues with your hardware. If you are experiencing problems please contact your Intel or hardware representative who provided you with this hardware.\n"); 13550 13551 i40e_clear_pxe_mode(hw); 13552 err = i40e_get_capabilities(pf, i40e_aqc_opc_list_func_capabilities); 13553 if (err) 13554 goto err_adminq_setup; 13555 13556 err = i40e_sw_init(pf); 13557 if (err) { 13558 dev_info(&pdev->dev, "sw_init failed: %d\n", err); 13559 goto err_sw_init; 13560 } 13561 13562 err = i40e_init_lan_hmc(hw, hw->func_caps.num_tx_qp, 13563 hw->func_caps.num_rx_qp, 0, 0); 13564 if (err) { 13565 dev_info(&pdev->dev, "init_lan_hmc failed: %d\n", err); 13566 goto err_init_lan_hmc; 13567 } 13568 13569 err = i40e_configure_lan_hmc(hw, I40E_HMC_MODEL_DIRECT_ONLY); 13570 if (err) { 13571 dev_info(&pdev->dev, "configure_lan_hmc failed: %d\n", err); 13572 err = -ENOENT; 13573 goto err_configure_lan_hmc; 13574 } 13575 13576 /* Disable LLDP for NICs that have firmware versions lower than v4.3. 13577 * Ignore error return codes because if it was already disabled via 13578 * hardware settings this will fail 13579 */ 13580 if (pf->hw_features & I40E_HW_STOP_FW_LLDP) { 13581 dev_info(&pdev->dev, "Stopping firmware LLDP agent.\n"); 13582 i40e_aq_stop_lldp(hw, true, NULL); 13583 } 13584 13585 /* allow a platform config to override the HW addr */ 13586 i40e_get_platform_mac_addr(pdev, pf); 13587 13588 if (!is_valid_ether_addr(hw->mac.addr)) { 13589 dev_info(&pdev->dev, "invalid MAC address %pM\n", hw->mac.addr); 13590 err = -EIO; 13591 goto err_mac_addr; 13592 } 13593 dev_info(&pdev->dev, "MAC address: %pM\n", hw->mac.addr); 13594 ether_addr_copy(hw->mac.perm_addr, hw->mac.addr); 13595 i40e_get_port_mac_addr(hw, hw->mac.port_addr); 13596 if (is_valid_ether_addr(hw->mac.port_addr)) 13597 pf->hw_features |= I40E_HW_PORT_ID_VALID; 13598 13599 pci_set_drvdata(pdev, pf); 13600 pci_save_state(pdev); 13601 #ifdef CONFIG_I40E_DCB 13602 err = i40e_init_pf_dcb(pf); 13603 if (err) { 13604 dev_info(&pdev->dev, "DCB init failed %d, disabled\n", err); 13605 pf->flags &= ~(I40E_FLAG_DCB_CAPABLE | I40E_FLAG_DCB_ENABLED); 13606 /* Continue without DCB enabled */ 13607 } 13608 #endif /* CONFIG_I40E_DCB */ 13609 13610 /* set up periodic task facility */ 13611 timer_setup(&pf->service_timer, i40e_service_timer, 0); 13612 pf->service_timer_period = HZ; 13613 13614 INIT_WORK(&pf->service_task, i40e_service_task); 13615 clear_bit(__I40E_SERVICE_SCHED, pf->state); 13616 13617 /* NVM bit on means WoL disabled for the port */ 13618 i40e_read_nvm_word(hw, I40E_SR_NVM_WAKE_ON_LAN, &wol_nvm_bits); 13619 if (BIT (hw->port) & wol_nvm_bits || hw->partition_id != 1) 13620 pf->wol_en = false; 13621 else 13622 pf->wol_en = true; 13623 device_set_wakeup_enable(&pf->pdev->dev, pf->wol_en); 13624 13625 /* set up the main switch operations */ 13626 i40e_determine_queue_usage(pf); 13627 err = i40e_init_interrupt_scheme(pf); 13628 if (err) 13629 goto err_switch_setup; 13630 13631 /* The number of VSIs reported by the FW is the minimum guaranteed 13632 * to us; HW supports far more and we share the remaining pool with 13633 * the other PFs. We allocate space for more than the guarantee with 13634 * the understanding that we might not get them all later. 13635 */ 13636 if (pf->hw.func_caps.num_vsis < I40E_MIN_VSI_ALLOC) 13637 pf->num_alloc_vsi = I40E_MIN_VSI_ALLOC; 13638 else 13639 pf->num_alloc_vsi = pf->hw.func_caps.num_vsis; 13640 13641 /* Set up the *vsi struct and our local tracking of the MAIN PF vsi. */ 13642 pf->vsi = kcalloc(pf->num_alloc_vsi, sizeof(struct i40e_vsi *), 13643 GFP_KERNEL); 13644 if (!pf->vsi) { 13645 err = -ENOMEM; 13646 goto err_switch_setup; 13647 } 13648 13649 #ifdef CONFIG_PCI_IOV 13650 /* prep for VF support */ 13651 if ((pf->flags & I40E_FLAG_SRIOV_ENABLED) && 13652 (pf->flags & I40E_FLAG_MSIX_ENABLED) && 13653 !test_bit(__I40E_BAD_EEPROM, pf->state)) { 13654 if (pci_num_vf(pdev)) 13655 pf->flags |= I40E_FLAG_VEB_MODE_ENABLED; 13656 } 13657 #endif 13658 err = i40e_setup_pf_switch(pf, false); 13659 if (err) { 13660 dev_info(&pdev->dev, "setup_pf_switch failed: %d\n", err); 13661 goto err_vsis; 13662 } 13663 INIT_LIST_HEAD(&pf->vsi[pf->lan_vsi]->ch_list); 13664 13665 /* Make sure flow control is set according to current settings */ 13666 err = i40e_set_fc(hw, &set_fc_aq_fail, true); 13667 if (set_fc_aq_fail & I40E_SET_FC_AQ_FAIL_GET) 13668 dev_dbg(&pf->pdev->dev, 13669 "Set fc with err %s aq_err %s on get_phy_cap\n", 13670 i40e_stat_str(hw, err), 13671 i40e_aq_str(hw, hw->aq.asq_last_status)); 13672 if (set_fc_aq_fail & I40E_SET_FC_AQ_FAIL_SET) 13673 dev_dbg(&pf->pdev->dev, 13674 "Set fc with err %s aq_err %s on set_phy_config\n", 13675 i40e_stat_str(hw, err), 13676 i40e_aq_str(hw, hw->aq.asq_last_status)); 13677 if (set_fc_aq_fail & I40E_SET_FC_AQ_FAIL_UPDATE) 13678 dev_dbg(&pf->pdev->dev, 13679 "Set fc with err %s aq_err %s on get_link_info\n", 13680 i40e_stat_str(hw, err), 13681 i40e_aq_str(hw, hw->aq.asq_last_status)); 13682 13683 /* if FDIR VSI was set up, start it now */ 13684 for (i = 0; i < pf->num_alloc_vsi; i++) { 13685 if (pf->vsi[i] && pf->vsi[i]->type == I40E_VSI_FDIR) { 13686 i40e_vsi_open(pf->vsi[i]); 13687 break; 13688 } 13689 } 13690 13691 /* The driver only wants link up/down and module qualification 13692 * reports from firmware. Note the negative logic. 13693 */ 13694 err = i40e_aq_set_phy_int_mask(&pf->hw, 13695 ~(I40E_AQ_EVENT_LINK_UPDOWN | 13696 I40E_AQ_EVENT_MEDIA_NA | 13697 I40E_AQ_EVENT_MODULE_QUAL_FAIL), NULL); 13698 if (err) 13699 dev_info(&pf->pdev->dev, "set phy mask fail, err %s aq_err %s\n", 13700 i40e_stat_str(&pf->hw, err), 13701 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 13702 13703 /* Reconfigure hardware for allowing smaller MSS in the case 13704 * of TSO, so that we avoid the MDD being fired and causing 13705 * a reset in the case of small MSS+TSO. 13706 */ 13707 val = rd32(hw, I40E_REG_MSS); 13708 if ((val & I40E_REG_MSS_MIN_MASK) > I40E_64BYTE_MSS) { 13709 val &= ~I40E_REG_MSS_MIN_MASK; 13710 val |= I40E_64BYTE_MSS; 13711 wr32(hw, I40E_REG_MSS, val); 13712 } 13713 13714 if (pf->hw_features & I40E_HW_RESTART_AUTONEG) { 13715 msleep(75); 13716 err = i40e_aq_set_link_restart_an(&pf->hw, true, NULL); 13717 if (err) 13718 dev_info(&pf->pdev->dev, "link restart failed, err %s aq_err %s\n", 13719 i40e_stat_str(&pf->hw, err), 13720 i40e_aq_str(&pf->hw, 13721 pf->hw.aq.asq_last_status)); 13722 } 13723 /* The main driver is (mostly) up and happy. We need to set this state 13724 * before setting up the misc vector or we get a race and the vector 13725 * ends up disabled forever. 13726 */ 13727 clear_bit(__I40E_DOWN, pf->state); 13728 13729 /* In case of MSIX we are going to setup the misc vector right here 13730 * to handle admin queue events etc. In case of legacy and MSI 13731 * the misc functionality and queue processing is combined in 13732 * the same vector and that gets setup at open. 13733 */ 13734 if (pf->flags & I40E_FLAG_MSIX_ENABLED) { 13735 err = i40e_setup_misc_vector(pf); 13736 if (err) { 13737 dev_info(&pdev->dev, 13738 "setup of misc vector failed: %d\n", err); 13739 goto err_vsis; 13740 } 13741 } 13742 13743 #ifdef CONFIG_PCI_IOV 13744 /* prep for VF support */ 13745 if ((pf->flags & I40E_FLAG_SRIOV_ENABLED) && 13746 (pf->flags & I40E_FLAG_MSIX_ENABLED) && 13747 !test_bit(__I40E_BAD_EEPROM, pf->state)) { 13748 /* disable link interrupts for VFs */ 13749 val = rd32(hw, I40E_PFGEN_PORTMDIO_NUM); 13750 val &= ~I40E_PFGEN_PORTMDIO_NUM_VFLINK_STAT_ENA_MASK; 13751 wr32(hw, I40E_PFGEN_PORTMDIO_NUM, val); 13752 i40e_flush(hw); 13753 13754 if (pci_num_vf(pdev)) { 13755 dev_info(&pdev->dev, 13756 "Active VFs found, allocating resources.\n"); 13757 err = i40e_alloc_vfs(pf, pci_num_vf(pdev)); 13758 if (err) 13759 dev_info(&pdev->dev, 13760 "Error %d allocating resources for existing VFs\n", 13761 err); 13762 } 13763 } 13764 #endif /* CONFIG_PCI_IOV */ 13765 13766 if (pf->flags & I40E_FLAG_IWARP_ENABLED) { 13767 pf->iwarp_base_vector = i40e_get_lump(pf, pf->irq_pile, 13768 pf->num_iwarp_msix, 13769 I40E_IWARP_IRQ_PILE_ID); 13770 if (pf->iwarp_base_vector < 0) { 13771 dev_info(&pdev->dev, 13772 "failed to get tracking for %d vectors for IWARP err=%d\n", 13773 pf->num_iwarp_msix, pf->iwarp_base_vector); 13774 pf->flags &= ~I40E_FLAG_IWARP_ENABLED; 13775 } 13776 } 13777 13778 i40e_dbg_pf_init(pf); 13779 13780 /* tell the firmware that we're starting */ 13781 i40e_send_version(pf); 13782 13783 /* since everything's happy, start the service_task timer */ 13784 mod_timer(&pf->service_timer, 13785 round_jiffies(jiffies + pf->service_timer_period)); 13786 13787 /* add this PF to client device list and launch a client service task */ 13788 if (pf->flags & I40E_FLAG_IWARP_ENABLED) { 13789 err = i40e_lan_add_device(pf); 13790 if (err) 13791 dev_info(&pdev->dev, "Failed to add PF to client API service list: %d\n", 13792 err); 13793 } 13794 13795 #define PCI_SPEED_SIZE 8 13796 #define PCI_WIDTH_SIZE 8 13797 /* Devices on the IOSF bus do not have this information 13798 * and will report PCI Gen 1 x 1 by default so don't bother 13799 * checking them. 13800 */ 13801 if (!(pf->hw_features & I40E_HW_NO_PCI_LINK_CHECK)) { 13802 char speed[PCI_SPEED_SIZE] = "Unknown"; 13803 char width[PCI_WIDTH_SIZE] = "Unknown"; 13804 13805 /* Get the negotiated link width and speed from PCI config 13806 * space 13807 */ 13808 pcie_capability_read_word(pf->pdev, PCI_EXP_LNKSTA, 13809 &link_status); 13810 13811 i40e_set_pci_config_data(hw, link_status); 13812 13813 switch (hw->bus.speed) { 13814 case i40e_bus_speed_8000: 13815 strncpy(speed, "8.0", PCI_SPEED_SIZE); break; 13816 case i40e_bus_speed_5000: 13817 strncpy(speed, "5.0", PCI_SPEED_SIZE); break; 13818 case i40e_bus_speed_2500: 13819 strncpy(speed, "2.5", PCI_SPEED_SIZE); break; 13820 default: 13821 break; 13822 } 13823 switch (hw->bus.width) { 13824 case i40e_bus_width_pcie_x8: 13825 strncpy(width, "8", PCI_WIDTH_SIZE); break; 13826 case i40e_bus_width_pcie_x4: 13827 strncpy(width, "4", PCI_WIDTH_SIZE); break; 13828 case i40e_bus_width_pcie_x2: 13829 strncpy(width, "2", PCI_WIDTH_SIZE); break; 13830 case i40e_bus_width_pcie_x1: 13831 strncpy(width, "1", PCI_WIDTH_SIZE); break; 13832 default: 13833 break; 13834 } 13835 13836 dev_info(&pdev->dev, "PCI-Express: Speed %sGT/s Width x%s\n", 13837 speed, width); 13838 13839 if (hw->bus.width < i40e_bus_width_pcie_x8 || 13840 hw->bus.speed < i40e_bus_speed_8000) { 13841 dev_warn(&pdev->dev, "PCI-Express bandwidth available for this device may be insufficient for optimal performance.\n"); 13842 dev_warn(&pdev->dev, "Please move the device to a different PCI-e link with more lanes and/or higher transfer rate.\n"); 13843 } 13844 } 13845 13846 /* get the requested speeds from the fw */ 13847 err = i40e_aq_get_phy_capabilities(hw, false, false, &abilities, NULL); 13848 if (err) 13849 dev_dbg(&pf->pdev->dev, "get requested speeds ret = %s last_status = %s\n", 13850 i40e_stat_str(&pf->hw, err), 13851 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 13852 pf->hw.phy.link_info.requested_speeds = abilities.link_speed; 13853 13854 /* get the supported phy types from the fw */ 13855 err = i40e_aq_get_phy_capabilities(hw, false, true, &abilities, NULL); 13856 if (err) 13857 dev_dbg(&pf->pdev->dev, "get supported phy types ret = %s last_status = %s\n", 13858 i40e_stat_str(&pf->hw, err), 13859 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 13860 13861 /* Add a filter to drop all Flow control frames from any VSI from being 13862 * transmitted. By doing so we stop a malicious VF from sending out 13863 * PAUSE or PFC frames and potentially controlling traffic for other 13864 * PF/VF VSIs. 13865 * The FW can still send Flow control frames if enabled. 13866 */ 13867 i40e_add_filter_to_drop_tx_flow_control_frames(&pf->hw, 13868 pf->main_vsi_seid); 13869 13870 if ((pf->hw.device_id == I40E_DEV_ID_10G_BASE_T) || 13871 (pf->hw.device_id == I40E_DEV_ID_10G_BASE_T4)) 13872 pf->hw_features |= I40E_HW_PHY_CONTROLS_LEDS; 13873 if (pf->hw.device_id == I40E_DEV_ID_SFP_I_X722) 13874 pf->hw_features |= I40E_HW_HAVE_CRT_RETIMER; 13875 /* print a string summarizing features */ 13876 i40e_print_features(pf); 13877 13878 return 0; 13879 13880 /* Unwind what we've done if something failed in the setup */ 13881 err_vsis: 13882 set_bit(__I40E_DOWN, pf->state); 13883 i40e_clear_interrupt_scheme(pf); 13884 kfree(pf->vsi); 13885 err_switch_setup: 13886 i40e_reset_interrupt_capability(pf); 13887 del_timer_sync(&pf->service_timer); 13888 err_mac_addr: 13889 err_configure_lan_hmc: 13890 (void)i40e_shutdown_lan_hmc(hw); 13891 err_init_lan_hmc: 13892 kfree(pf->qp_pile); 13893 err_sw_init: 13894 err_adminq_setup: 13895 err_pf_reset: 13896 iounmap(hw->hw_addr); 13897 err_ioremap: 13898 kfree(pf); 13899 err_pf_alloc: 13900 pci_disable_pcie_error_reporting(pdev); 13901 pci_release_mem_regions(pdev); 13902 err_pci_reg: 13903 err_dma: 13904 pci_disable_device(pdev); 13905 return err; 13906 } 13907 13908 /** 13909 * i40e_remove - Device removal routine 13910 * @pdev: PCI device information struct 13911 * 13912 * i40e_remove is called by the PCI subsystem to alert the driver 13913 * that is should release a PCI device. This could be caused by a 13914 * Hot-Plug event, or because the driver is going to be removed from 13915 * memory. 13916 **/ 13917 static void i40e_remove(struct pci_dev *pdev) 13918 { 13919 struct i40e_pf *pf = pci_get_drvdata(pdev); 13920 struct i40e_hw *hw = &pf->hw; 13921 i40e_status ret_code; 13922 int i; 13923 13924 i40e_dbg_pf_exit(pf); 13925 13926 i40e_ptp_stop(pf); 13927 13928 /* Disable RSS in hw */ 13929 i40e_write_rx_ctl(hw, I40E_PFQF_HENA(0), 0); 13930 i40e_write_rx_ctl(hw, I40E_PFQF_HENA(1), 0); 13931 13932 /* no more scheduling of any task */ 13933 set_bit(__I40E_SUSPENDED, pf->state); 13934 set_bit(__I40E_DOWN, pf->state); 13935 if (pf->service_timer.function) 13936 del_timer_sync(&pf->service_timer); 13937 if (pf->service_task.func) 13938 cancel_work_sync(&pf->service_task); 13939 13940 /* Client close must be called explicitly here because the timer 13941 * has been stopped. 13942 */ 13943 i40e_notify_client_of_netdev_close(pf->vsi[pf->lan_vsi], false); 13944 13945 if (pf->flags & I40E_FLAG_SRIOV_ENABLED) { 13946 i40e_free_vfs(pf); 13947 pf->flags &= ~I40E_FLAG_SRIOV_ENABLED; 13948 } 13949 13950 i40e_fdir_teardown(pf); 13951 13952 /* If there is a switch structure or any orphans, remove them. 13953 * This will leave only the PF's VSI remaining. 13954 */ 13955 for (i = 0; i < I40E_MAX_VEB; i++) { 13956 if (!pf->veb[i]) 13957 continue; 13958 13959 if (pf->veb[i]->uplink_seid == pf->mac_seid || 13960 pf->veb[i]->uplink_seid == 0) 13961 i40e_switch_branch_release(pf->veb[i]); 13962 } 13963 13964 /* Now we can shutdown the PF's VSI, just before we kill 13965 * adminq and hmc. 13966 */ 13967 if (pf->vsi[pf->lan_vsi]) 13968 i40e_vsi_release(pf->vsi[pf->lan_vsi]); 13969 13970 i40e_cloud_filter_exit(pf); 13971 13972 /* remove attached clients */ 13973 if (pf->flags & I40E_FLAG_IWARP_ENABLED) { 13974 ret_code = i40e_lan_del_device(pf); 13975 if (ret_code) 13976 dev_warn(&pdev->dev, "Failed to delete client device: %d\n", 13977 ret_code); 13978 } 13979 13980 /* shutdown and destroy the HMC */ 13981 if (hw->hmc.hmc_obj) { 13982 ret_code = i40e_shutdown_lan_hmc(hw); 13983 if (ret_code) 13984 dev_warn(&pdev->dev, 13985 "Failed to destroy the HMC resources: %d\n", 13986 ret_code); 13987 } 13988 13989 /* shutdown the adminq */ 13990 i40e_shutdown_adminq(hw); 13991 13992 /* destroy the locks only once, here */ 13993 mutex_destroy(&hw->aq.arq_mutex); 13994 mutex_destroy(&hw->aq.asq_mutex); 13995 13996 /* Clear all dynamic memory lists of rings, q_vectors, and VSIs */ 13997 i40e_clear_interrupt_scheme(pf); 13998 for (i = 0; i < pf->num_alloc_vsi; i++) { 13999 if (pf->vsi[i]) { 14000 i40e_vsi_clear_rings(pf->vsi[i]); 14001 i40e_vsi_clear(pf->vsi[i]); 14002 pf->vsi[i] = NULL; 14003 } 14004 } 14005 14006 for (i = 0; i < I40E_MAX_VEB; i++) { 14007 kfree(pf->veb[i]); 14008 pf->veb[i] = NULL; 14009 } 14010 14011 kfree(pf->qp_pile); 14012 kfree(pf->vsi); 14013 14014 iounmap(hw->hw_addr); 14015 kfree(pf); 14016 pci_release_mem_regions(pdev); 14017 14018 pci_disable_pcie_error_reporting(pdev); 14019 pci_disable_device(pdev); 14020 } 14021 14022 /** 14023 * i40e_pci_error_detected - warning that something funky happened in PCI land 14024 * @pdev: PCI device information struct 14025 * 14026 * Called to warn that something happened and the error handling steps 14027 * are in progress. Allows the driver to quiesce things, be ready for 14028 * remediation. 14029 **/ 14030 static pci_ers_result_t i40e_pci_error_detected(struct pci_dev *pdev, 14031 enum pci_channel_state error) 14032 { 14033 struct i40e_pf *pf = pci_get_drvdata(pdev); 14034 14035 dev_info(&pdev->dev, "%s: error %d\n", __func__, error); 14036 14037 if (!pf) { 14038 dev_info(&pdev->dev, 14039 "Cannot recover - error happened during device probe\n"); 14040 return PCI_ERS_RESULT_DISCONNECT; 14041 } 14042 14043 /* shutdown all operations */ 14044 if (!test_bit(__I40E_SUSPENDED, pf->state)) 14045 i40e_prep_for_reset(pf, false); 14046 14047 /* Request a slot reset */ 14048 return PCI_ERS_RESULT_NEED_RESET; 14049 } 14050 14051 /** 14052 * i40e_pci_error_slot_reset - a PCI slot reset just happened 14053 * @pdev: PCI device information struct 14054 * 14055 * Called to find if the driver can work with the device now that 14056 * the pci slot has been reset. If a basic connection seems good 14057 * (registers are readable and have sane content) then return a 14058 * happy little PCI_ERS_RESULT_xxx. 14059 **/ 14060 static pci_ers_result_t i40e_pci_error_slot_reset(struct pci_dev *pdev) 14061 { 14062 struct i40e_pf *pf = pci_get_drvdata(pdev); 14063 pci_ers_result_t result; 14064 int err; 14065 u32 reg; 14066 14067 dev_dbg(&pdev->dev, "%s\n", __func__); 14068 if (pci_enable_device_mem(pdev)) { 14069 dev_info(&pdev->dev, 14070 "Cannot re-enable PCI device after reset.\n"); 14071 result = PCI_ERS_RESULT_DISCONNECT; 14072 } else { 14073 pci_set_master(pdev); 14074 pci_restore_state(pdev); 14075 pci_save_state(pdev); 14076 pci_wake_from_d3(pdev, false); 14077 14078 reg = rd32(&pf->hw, I40E_GLGEN_RTRIG); 14079 if (reg == 0) 14080 result = PCI_ERS_RESULT_RECOVERED; 14081 else 14082 result = PCI_ERS_RESULT_DISCONNECT; 14083 } 14084 14085 err = pci_cleanup_aer_uncorrect_error_status(pdev); 14086 if (err) { 14087 dev_info(&pdev->dev, 14088 "pci_cleanup_aer_uncorrect_error_status failed 0x%0x\n", 14089 err); 14090 /* non-fatal, continue */ 14091 } 14092 14093 return result; 14094 } 14095 14096 /** 14097 * i40e_pci_error_reset_prepare - prepare device driver for pci reset 14098 * @pdev: PCI device information struct 14099 */ 14100 static void i40e_pci_error_reset_prepare(struct pci_dev *pdev) 14101 { 14102 struct i40e_pf *pf = pci_get_drvdata(pdev); 14103 14104 i40e_prep_for_reset(pf, false); 14105 } 14106 14107 /** 14108 * i40e_pci_error_reset_done - pci reset done, device driver reset can begin 14109 * @pdev: PCI device information struct 14110 */ 14111 static void i40e_pci_error_reset_done(struct pci_dev *pdev) 14112 { 14113 struct i40e_pf *pf = pci_get_drvdata(pdev); 14114 14115 i40e_reset_and_rebuild(pf, false, false); 14116 } 14117 14118 /** 14119 * i40e_pci_error_resume - restart operations after PCI error recovery 14120 * @pdev: PCI device information struct 14121 * 14122 * Called to allow the driver to bring things back up after PCI error 14123 * and/or reset recovery has finished. 14124 **/ 14125 static void i40e_pci_error_resume(struct pci_dev *pdev) 14126 { 14127 struct i40e_pf *pf = pci_get_drvdata(pdev); 14128 14129 dev_dbg(&pdev->dev, "%s\n", __func__); 14130 if (test_bit(__I40E_SUSPENDED, pf->state)) 14131 return; 14132 14133 i40e_handle_reset_warning(pf, false); 14134 } 14135 14136 /** 14137 * i40e_enable_mc_magic_wake - enable multicast magic packet wake up 14138 * using the mac_address_write admin q function 14139 * @pf: pointer to i40e_pf struct 14140 **/ 14141 static void i40e_enable_mc_magic_wake(struct i40e_pf *pf) 14142 { 14143 struct i40e_hw *hw = &pf->hw; 14144 i40e_status ret; 14145 u8 mac_addr[6]; 14146 u16 flags = 0; 14147 14148 /* Get current MAC address in case it's an LAA */ 14149 if (pf->vsi[pf->lan_vsi] && pf->vsi[pf->lan_vsi]->netdev) { 14150 ether_addr_copy(mac_addr, 14151 pf->vsi[pf->lan_vsi]->netdev->dev_addr); 14152 } else { 14153 dev_err(&pf->pdev->dev, 14154 "Failed to retrieve MAC address; using default\n"); 14155 ether_addr_copy(mac_addr, hw->mac.addr); 14156 } 14157 14158 /* The FW expects the mac address write cmd to first be called with 14159 * one of these flags before calling it again with the multicast 14160 * enable flags. 14161 */ 14162 flags = I40E_AQC_WRITE_TYPE_LAA_WOL; 14163 14164 if (hw->func_caps.flex10_enable && hw->partition_id != 1) 14165 flags = I40E_AQC_WRITE_TYPE_LAA_ONLY; 14166 14167 ret = i40e_aq_mac_address_write(hw, flags, mac_addr, NULL); 14168 if (ret) { 14169 dev_err(&pf->pdev->dev, 14170 "Failed to update MAC address registers; cannot enable Multicast Magic packet wake up"); 14171 return; 14172 } 14173 14174 flags = I40E_AQC_MC_MAG_EN 14175 | I40E_AQC_WOL_PRESERVE_ON_PFR 14176 | I40E_AQC_WRITE_TYPE_UPDATE_MC_MAG; 14177 ret = i40e_aq_mac_address_write(hw, flags, mac_addr, NULL); 14178 if (ret) 14179 dev_err(&pf->pdev->dev, 14180 "Failed to enable Multicast Magic Packet wake up\n"); 14181 } 14182 14183 /** 14184 * i40e_shutdown - PCI callback for shutting down 14185 * @pdev: PCI device information struct 14186 **/ 14187 static void i40e_shutdown(struct pci_dev *pdev) 14188 { 14189 struct i40e_pf *pf = pci_get_drvdata(pdev); 14190 struct i40e_hw *hw = &pf->hw; 14191 14192 set_bit(__I40E_SUSPENDED, pf->state); 14193 set_bit(__I40E_DOWN, pf->state); 14194 rtnl_lock(); 14195 i40e_prep_for_reset(pf, true); 14196 rtnl_unlock(); 14197 14198 wr32(hw, I40E_PFPM_APM, (pf->wol_en ? I40E_PFPM_APM_APME_MASK : 0)); 14199 wr32(hw, I40E_PFPM_WUFC, (pf->wol_en ? I40E_PFPM_WUFC_MAG_MASK : 0)); 14200 14201 del_timer_sync(&pf->service_timer); 14202 cancel_work_sync(&pf->service_task); 14203 i40e_cloud_filter_exit(pf); 14204 i40e_fdir_teardown(pf); 14205 14206 /* Client close must be called explicitly here because the timer 14207 * has been stopped. 14208 */ 14209 i40e_notify_client_of_netdev_close(pf->vsi[pf->lan_vsi], false); 14210 14211 if (pf->wol_en && (pf->hw_features & I40E_HW_WOL_MC_MAGIC_PKT_WAKE)) 14212 i40e_enable_mc_magic_wake(pf); 14213 14214 i40e_prep_for_reset(pf, false); 14215 14216 wr32(hw, I40E_PFPM_APM, 14217 (pf->wol_en ? I40E_PFPM_APM_APME_MASK : 0)); 14218 wr32(hw, I40E_PFPM_WUFC, 14219 (pf->wol_en ? I40E_PFPM_WUFC_MAG_MASK : 0)); 14220 14221 i40e_clear_interrupt_scheme(pf); 14222 14223 if (system_state == SYSTEM_POWER_OFF) { 14224 pci_wake_from_d3(pdev, pf->wol_en); 14225 pci_set_power_state(pdev, PCI_D3hot); 14226 } 14227 } 14228 14229 /** 14230 * i40e_suspend - PM callback for moving to D3 14231 * @dev: generic device information structure 14232 **/ 14233 static int __maybe_unused i40e_suspend(struct device *dev) 14234 { 14235 struct pci_dev *pdev = to_pci_dev(dev); 14236 struct i40e_pf *pf = pci_get_drvdata(pdev); 14237 struct i40e_hw *hw = &pf->hw; 14238 14239 /* If we're already suspended, then there is nothing to do */ 14240 if (test_and_set_bit(__I40E_SUSPENDED, pf->state)) 14241 return 0; 14242 14243 set_bit(__I40E_DOWN, pf->state); 14244 14245 /* Ensure service task will not be running */ 14246 del_timer_sync(&pf->service_timer); 14247 cancel_work_sync(&pf->service_task); 14248 14249 if (pf->wol_en && (pf->hw_features & I40E_HW_WOL_MC_MAGIC_PKT_WAKE)) 14250 i40e_enable_mc_magic_wake(pf); 14251 14252 i40e_prep_for_reset(pf, false); 14253 14254 wr32(hw, I40E_PFPM_APM, (pf->wol_en ? I40E_PFPM_APM_APME_MASK : 0)); 14255 wr32(hw, I40E_PFPM_WUFC, (pf->wol_en ? I40E_PFPM_WUFC_MAG_MASK : 0)); 14256 14257 /* Clear the interrupt scheme and release our IRQs so that the system 14258 * can safely hibernate even when there are a large number of CPUs. 14259 * Otherwise hibernation might fail when mapping all the vectors back 14260 * to CPU0. 14261 */ 14262 i40e_clear_interrupt_scheme(pf); 14263 14264 return 0; 14265 } 14266 14267 /** 14268 * i40e_resume - PM callback for waking up from D3 14269 * @dev: generic device information structure 14270 **/ 14271 static int __maybe_unused i40e_resume(struct device *dev) 14272 { 14273 struct pci_dev *pdev = to_pci_dev(dev); 14274 struct i40e_pf *pf = pci_get_drvdata(pdev); 14275 int err; 14276 14277 /* If we're not suspended, then there is nothing to do */ 14278 if (!test_bit(__I40E_SUSPENDED, pf->state)) 14279 return 0; 14280 14281 /* We cleared the interrupt scheme when we suspended, so we need to 14282 * restore it now to resume device functionality. 14283 */ 14284 err = i40e_restore_interrupt_scheme(pf); 14285 if (err) { 14286 dev_err(&pdev->dev, "Cannot restore interrupt scheme: %d\n", 14287 err); 14288 } 14289 14290 clear_bit(__I40E_DOWN, pf->state); 14291 i40e_reset_and_rebuild(pf, false, false); 14292 14293 /* Clear suspended state last after everything is recovered */ 14294 clear_bit(__I40E_SUSPENDED, pf->state); 14295 14296 /* Restart the service task */ 14297 mod_timer(&pf->service_timer, 14298 round_jiffies(jiffies + pf->service_timer_period)); 14299 14300 return 0; 14301 } 14302 14303 static const struct pci_error_handlers i40e_err_handler = { 14304 .error_detected = i40e_pci_error_detected, 14305 .slot_reset = i40e_pci_error_slot_reset, 14306 .reset_prepare = i40e_pci_error_reset_prepare, 14307 .reset_done = i40e_pci_error_reset_done, 14308 .resume = i40e_pci_error_resume, 14309 }; 14310 14311 static SIMPLE_DEV_PM_OPS(i40e_pm_ops, i40e_suspend, i40e_resume); 14312 14313 static struct pci_driver i40e_driver = { 14314 .name = i40e_driver_name, 14315 .id_table = i40e_pci_tbl, 14316 .probe = i40e_probe, 14317 .remove = i40e_remove, 14318 .driver = { 14319 .pm = &i40e_pm_ops, 14320 }, 14321 .shutdown = i40e_shutdown, 14322 .err_handler = &i40e_err_handler, 14323 .sriov_configure = i40e_pci_sriov_configure, 14324 }; 14325 14326 /** 14327 * i40e_init_module - Driver registration routine 14328 * 14329 * i40e_init_module is the first routine called when the driver is 14330 * loaded. All it does is register with the PCI subsystem. 14331 **/ 14332 static int __init i40e_init_module(void) 14333 { 14334 pr_info("%s: %s - version %s\n", i40e_driver_name, 14335 i40e_driver_string, i40e_driver_version_str); 14336 pr_info("%s: %s\n", i40e_driver_name, i40e_copyright); 14337 14338 /* There is no need to throttle the number of active tasks because 14339 * each device limits its own task using a state bit for scheduling 14340 * the service task, and the device tasks do not interfere with each 14341 * other, so we don't set a max task limit. We must set WQ_MEM_RECLAIM 14342 * since we need to be able to guarantee forward progress even under 14343 * memory pressure. 14344 */ 14345 i40e_wq = alloc_workqueue("%s", WQ_MEM_RECLAIM, 0, i40e_driver_name); 14346 if (!i40e_wq) { 14347 pr_err("%s: Failed to create workqueue\n", i40e_driver_name); 14348 return -ENOMEM; 14349 } 14350 14351 i40e_dbg_init(); 14352 return pci_register_driver(&i40e_driver); 14353 } 14354 module_init(i40e_init_module); 14355 14356 /** 14357 * i40e_exit_module - Driver exit cleanup routine 14358 * 14359 * i40e_exit_module is called just before the driver is removed 14360 * from memory. 14361 **/ 14362 static void __exit i40e_exit_module(void) 14363 { 14364 pci_unregister_driver(&i40e_driver); 14365 destroy_workqueue(i40e_wq); 14366 i40e_dbg_exit(); 14367 } 14368 module_exit(i40e_exit_module); 14369