1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright(c) 2013 - 2018 Intel Corporation. */ 3 4 #include <linux/etherdevice.h> 5 #include <linux/of_net.h> 6 #include <linux/pci.h> 7 #include <linux/bpf.h> 8 9 /* Local includes */ 10 #include "i40e.h" 11 #include "i40e_diag.h" 12 #include "i40e_xsk.h" 13 #include <net/udp_tunnel.h> 14 #include <net/xdp_sock.h> 15 /* All i40e tracepoints are defined by the include below, which 16 * must be included exactly once across the whole kernel with 17 * CREATE_TRACE_POINTS defined 18 */ 19 #define CREATE_TRACE_POINTS 20 #include "i40e_trace.h" 21 22 const char i40e_driver_name[] = "i40e"; 23 static const char i40e_driver_string[] = 24 "Intel(R) Ethernet Connection XL710 Network Driver"; 25 26 #define DRV_KERN "-k" 27 28 #define DRV_VERSION_MAJOR 2 29 #define DRV_VERSION_MINOR 3 30 #define DRV_VERSION_BUILD 2 31 #define DRV_VERSION __stringify(DRV_VERSION_MAJOR) "." \ 32 __stringify(DRV_VERSION_MINOR) "." \ 33 __stringify(DRV_VERSION_BUILD) DRV_KERN 34 const char i40e_driver_version_str[] = DRV_VERSION; 35 static const char i40e_copyright[] = "Copyright (c) 2013 - 2014 Intel Corporation."; 36 37 /* a bit of forward declarations */ 38 static void i40e_vsi_reinit_locked(struct i40e_vsi *vsi); 39 static void i40e_handle_reset_warning(struct i40e_pf *pf, bool lock_acquired); 40 static int i40e_add_vsi(struct i40e_vsi *vsi); 41 static int i40e_add_veb(struct i40e_veb *veb, struct i40e_vsi *vsi); 42 static int i40e_setup_pf_switch(struct i40e_pf *pf, bool reinit); 43 static int i40e_setup_misc_vector(struct i40e_pf *pf); 44 static void i40e_determine_queue_usage(struct i40e_pf *pf); 45 static int i40e_setup_pf_filter_control(struct i40e_pf *pf); 46 static void i40e_prep_for_reset(struct i40e_pf *pf, bool lock_acquired); 47 static int i40e_reset(struct i40e_pf *pf); 48 static void i40e_rebuild(struct i40e_pf *pf, bool reinit, bool lock_acquired); 49 static void i40e_fdir_sb_setup(struct i40e_pf *pf); 50 static int i40e_veb_get_bw_info(struct i40e_veb *veb); 51 static int i40e_get_capabilities(struct i40e_pf *pf, 52 enum i40e_admin_queue_opc list_type); 53 54 55 /* i40e_pci_tbl - PCI Device ID Table 56 * 57 * Last entry must be all 0s 58 * 59 * { Vendor ID, Device ID, SubVendor ID, SubDevice ID, 60 * Class, Class Mask, private data (not used) } 61 */ 62 static const struct pci_device_id i40e_pci_tbl[] = { 63 {PCI_VDEVICE(INTEL, I40E_DEV_ID_SFP_XL710), 0}, 64 {PCI_VDEVICE(INTEL, I40E_DEV_ID_QEMU), 0}, 65 {PCI_VDEVICE(INTEL, I40E_DEV_ID_KX_B), 0}, 66 {PCI_VDEVICE(INTEL, I40E_DEV_ID_KX_C), 0}, 67 {PCI_VDEVICE(INTEL, I40E_DEV_ID_QSFP_A), 0}, 68 {PCI_VDEVICE(INTEL, I40E_DEV_ID_QSFP_B), 0}, 69 {PCI_VDEVICE(INTEL, I40E_DEV_ID_QSFP_C), 0}, 70 {PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_BASE_T), 0}, 71 {PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_BASE_T4), 0}, 72 {PCI_VDEVICE(INTEL, I40E_DEV_ID_KX_X722), 0}, 73 {PCI_VDEVICE(INTEL, I40E_DEV_ID_QSFP_X722), 0}, 74 {PCI_VDEVICE(INTEL, I40E_DEV_ID_SFP_X722), 0}, 75 {PCI_VDEVICE(INTEL, I40E_DEV_ID_1G_BASE_T_X722), 0}, 76 {PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_BASE_T_X722), 0}, 77 {PCI_VDEVICE(INTEL, I40E_DEV_ID_SFP_I_X722), 0}, 78 {PCI_VDEVICE(INTEL, I40E_DEV_ID_20G_KR2), 0}, 79 {PCI_VDEVICE(INTEL, I40E_DEV_ID_20G_KR2_A), 0}, 80 {PCI_VDEVICE(INTEL, I40E_DEV_ID_25G_B), 0}, 81 {PCI_VDEVICE(INTEL, I40E_DEV_ID_25G_SFP28), 0}, 82 /* required last entry */ 83 {0, } 84 }; 85 MODULE_DEVICE_TABLE(pci, i40e_pci_tbl); 86 87 #define I40E_MAX_VF_COUNT 128 88 static int debug = -1; 89 module_param(debug, uint, 0); 90 MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all), Debug mask (0x8XXXXXXX)"); 91 92 MODULE_AUTHOR("Intel Corporation, <e1000-devel@lists.sourceforge.net>"); 93 MODULE_DESCRIPTION("Intel(R) Ethernet Connection XL710 Network Driver"); 94 MODULE_LICENSE("GPL"); 95 MODULE_VERSION(DRV_VERSION); 96 97 static struct workqueue_struct *i40e_wq; 98 99 /** 100 * i40e_allocate_dma_mem_d - OS specific memory alloc for shared code 101 * @hw: pointer to the HW structure 102 * @mem: ptr to mem struct to fill out 103 * @size: size of memory requested 104 * @alignment: what to align the allocation to 105 **/ 106 int i40e_allocate_dma_mem_d(struct i40e_hw *hw, struct i40e_dma_mem *mem, 107 u64 size, u32 alignment) 108 { 109 struct i40e_pf *pf = (struct i40e_pf *)hw->back; 110 111 mem->size = ALIGN(size, alignment); 112 mem->va = dma_zalloc_coherent(&pf->pdev->dev, mem->size, 113 &mem->pa, GFP_KERNEL); 114 if (!mem->va) 115 return -ENOMEM; 116 117 return 0; 118 } 119 120 /** 121 * i40e_free_dma_mem_d - OS specific memory free for shared code 122 * @hw: pointer to the HW structure 123 * @mem: ptr to mem struct to free 124 **/ 125 int i40e_free_dma_mem_d(struct i40e_hw *hw, struct i40e_dma_mem *mem) 126 { 127 struct i40e_pf *pf = (struct i40e_pf *)hw->back; 128 129 dma_free_coherent(&pf->pdev->dev, mem->size, mem->va, mem->pa); 130 mem->va = NULL; 131 mem->pa = 0; 132 mem->size = 0; 133 134 return 0; 135 } 136 137 /** 138 * i40e_allocate_virt_mem_d - OS specific memory alloc for shared code 139 * @hw: pointer to the HW structure 140 * @mem: ptr to mem struct to fill out 141 * @size: size of memory requested 142 **/ 143 int i40e_allocate_virt_mem_d(struct i40e_hw *hw, struct i40e_virt_mem *mem, 144 u32 size) 145 { 146 mem->size = size; 147 mem->va = kzalloc(size, GFP_KERNEL); 148 149 if (!mem->va) 150 return -ENOMEM; 151 152 return 0; 153 } 154 155 /** 156 * i40e_free_virt_mem_d - OS specific memory free for shared code 157 * @hw: pointer to the HW structure 158 * @mem: ptr to mem struct to free 159 **/ 160 int i40e_free_virt_mem_d(struct i40e_hw *hw, struct i40e_virt_mem *mem) 161 { 162 /* it's ok to kfree a NULL pointer */ 163 kfree(mem->va); 164 mem->va = NULL; 165 mem->size = 0; 166 167 return 0; 168 } 169 170 /** 171 * i40e_get_lump - find a lump of free generic resource 172 * @pf: board private structure 173 * @pile: the pile of resource to search 174 * @needed: the number of items needed 175 * @id: an owner id to stick on the items assigned 176 * 177 * Returns the base item index of the lump, or negative for error 178 * 179 * The search_hint trick and lack of advanced fit-finding only work 180 * because we're highly likely to have all the same size lump requests. 181 * Linear search time and any fragmentation should be minimal. 182 **/ 183 static int i40e_get_lump(struct i40e_pf *pf, struct i40e_lump_tracking *pile, 184 u16 needed, u16 id) 185 { 186 int ret = -ENOMEM; 187 int i, j; 188 189 if (!pile || needed == 0 || id >= I40E_PILE_VALID_BIT) { 190 dev_info(&pf->pdev->dev, 191 "param err: pile=%s needed=%d id=0x%04x\n", 192 pile ? "<valid>" : "<null>", needed, id); 193 return -EINVAL; 194 } 195 196 /* start the linear search with an imperfect hint */ 197 i = pile->search_hint; 198 while (i < pile->num_entries) { 199 /* skip already allocated entries */ 200 if (pile->list[i] & I40E_PILE_VALID_BIT) { 201 i++; 202 continue; 203 } 204 205 /* do we have enough in this lump? */ 206 for (j = 0; (j < needed) && ((i+j) < pile->num_entries); j++) { 207 if (pile->list[i+j] & I40E_PILE_VALID_BIT) 208 break; 209 } 210 211 if (j == needed) { 212 /* there was enough, so assign it to the requestor */ 213 for (j = 0; j < needed; j++) 214 pile->list[i+j] = id | I40E_PILE_VALID_BIT; 215 ret = i; 216 pile->search_hint = i + j; 217 break; 218 } 219 220 /* not enough, so skip over it and continue looking */ 221 i += j; 222 } 223 224 return ret; 225 } 226 227 /** 228 * i40e_put_lump - return a lump of generic resource 229 * @pile: the pile of resource to search 230 * @index: the base item index 231 * @id: the owner id of the items assigned 232 * 233 * Returns the count of items in the lump 234 **/ 235 static int i40e_put_lump(struct i40e_lump_tracking *pile, u16 index, u16 id) 236 { 237 int valid_id = (id | I40E_PILE_VALID_BIT); 238 int count = 0; 239 int i; 240 241 if (!pile || index >= pile->num_entries) 242 return -EINVAL; 243 244 for (i = index; 245 i < pile->num_entries && pile->list[i] == valid_id; 246 i++) { 247 pile->list[i] = 0; 248 count++; 249 } 250 251 if (count && index < pile->search_hint) 252 pile->search_hint = index; 253 254 return count; 255 } 256 257 /** 258 * i40e_find_vsi_from_id - searches for the vsi with the given id 259 * @pf: the pf structure to search for the vsi 260 * @id: id of the vsi it is searching for 261 **/ 262 struct i40e_vsi *i40e_find_vsi_from_id(struct i40e_pf *pf, u16 id) 263 { 264 int i; 265 266 for (i = 0; i < pf->num_alloc_vsi; i++) 267 if (pf->vsi[i] && (pf->vsi[i]->id == id)) 268 return pf->vsi[i]; 269 270 return NULL; 271 } 272 273 /** 274 * i40e_service_event_schedule - Schedule the service task to wake up 275 * @pf: board private structure 276 * 277 * If not already scheduled, this puts the task into the work queue 278 **/ 279 void i40e_service_event_schedule(struct i40e_pf *pf) 280 { 281 if (!test_bit(__I40E_DOWN, pf->state) && 282 !test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state)) 283 queue_work(i40e_wq, &pf->service_task); 284 } 285 286 /** 287 * i40e_tx_timeout - Respond to a Tx Hang 288 * @netdev: network interface device structure 289 * 290 * If any port has noticed a Tx timeout, it is likely that the whole 291 * device is munged, not just the one netdev port, so go for the full 292 * reset. 293 **/ 294 static void i40e_tx_timeout(struct net_device *netdev) 295 { 296 struct i40e_netdev_priv *np = netdev_priv(netdev); 297 struct i40e_vsi *vsi = np->vsi; 298 struct i40e_pf *pf = vsi->back; 299 struct i40e_ring *tx_ring = NULL; 300 unsigned int i, hung_queue = 0; 301 u32 head, val; 302 303 pf->tx_timeout_count++; 304 305 /* find the stopped queue the same way the stack does */ 306 for (i = 0; i < netdev->num_tx_queues; i++) { 307 struct netdev_queue *q; 308 unsigned long trans_start; 309 310 q = netdev_get_tx_queue(netdev, i); 311 trans_start = q->trans_start; 312 if (netif_xmit_stopped(q) && 313 time_after(jiffies, 314 (trans_start + netdev->watchdog_timeo))) { 315 hung_queue = i; 316 break; 317 } 318 } 319 320 if (i == netdev->num_tx_queues) { 321 netdev_info(netdev, "tx_timeout: no netdev hung queue found\n"); 322 } else { 323 /* now that we have an index, find the tx_ring struct */ 324 for (i = 0; i < vsi->num_queue_pairs; i++) { 325 if (vsi->tx_rings[i] && vsi->tx_rings[i]->desc) { 326 if (hung_queue == 327 vsi->tx_rings[i]->queue_index) { 328 tx_ring = vsi->tx_rings[i]; 329 break; 330 } 331 } 332 } 333 } 334 335 if (time_after(jiffies, (pf->tx_timeout_last_recovery + HZ*20))) 336 pf->tx_timeout_recovery_level = 1; /* reset after some time */ 337 else if (time_before(jiffies, 338 (pf->tx_timeout_last_recovery + netdev->watchdog_timeo))) 339 return; /* don't do any new action before the next timeout */ 340 341 if (tx_ring) { 342 head = i40e_get_head(tx_ring); 343 /* Read interrupt register */ 344 if (pf->flags & I40E_FLAG_MSIX_ENABLED) 345 val = rd32(&pf->hw, 346 I40E_PFINT_DYN_CTLN(tx_ring->q_vector->v_idx + 347 tx_ring->vsi->base_vector - 1)); 348 else 349 val = rd32(&pf->hw, I40E_PFINT_DYN_CTL0); 350 351 netdev_info(netdev, "tx_timeout: VSI_seid: %d, Q %d, NTC: 0x%x, HWB: 0x%x, NTU: 0x%x, TAIL: 0x%x, INT: 0x%x\n", 352 vsi->seid, hung_queue, tx_ring->next_to_clean, 353 head, tx_ring->next_to_use, 354 readl(tx_ring->tail), val); 355 } 356 357 pf->tx_timeout_last_recovery = jiffies; 358 netdev_info(netdev, "tx_timeout recovery level %d, hung_queue %d\n", 359 pf->tx_timeout_recovery_level, hung_queue); 360 361 switch (pf->tx_timeout_recovery_level) { 362 case 1: 363 set_bit(__I40E_PF_RESET_REQUESTED, pf->state); 364 break; 365 case 2: 366 set_bit(__I40E_CORE_RESET_REQUESTED, pf->state); 367 break; 368 case 3: 369 set_bit(__I40E_GLOBAL_RESET_REQUESTED, pf->state); 370 break; 371 default: 372 netdev_err(netdev, "tx_timeout recovery unsuccessful\n"); 373 break; 374 } 375 376 i40e_service_event_schedule(pf); 377 pf->tx_timeout_recovery_level++; 378 } 379 380 /** 381 * i40e_get_vsi_stats_struct - Get System Network Statistics 382 * @vsi: the VSI we care about 383 * 384 * Returns the address of the device statistics structure. 385 * The statistics are actually updated from the service task. 386 **/ 387 struct rtnl_link_stats64 *i40e_get_vsi_stats_struct(struct i40e_vsi *vsi) 388 { 389 return &vsi->net_stats; 390 } 391 392 /** 393 * i40e_get_netdev_stats_struct_tx - populate stats from a Tx ring 394 * @ring: Tx ring to get statistics from 395 * @stats: statistics entry to be updated 396 **/ 397 static void i40e_get_netdev_stats_struct_tx(struct i40e_ring *ring, 398 struct rtnl_link_stats64 *stats) 399 { 400 u64 bytes, packets; 401 unsigned int start; 402 403 do { 404 start = u64_stats_fetch_begin_irq(&ring->syncp); 405 packets = ring->stats.packets; 406 bytes = ring->stats.bytes; 407 } while (u64_stats_fetch_retry_irq(&ring->syncp, start)); 408 409 stats->tx_packets += packets; 410 stats->tx_bytes += bytes; 411 } 412 413 /** 414 * i40e_get_netdev_stats_struct - Get statistics for netdev interface 415 * @netdev: network interface device structure 416 * @stats: data structure to store statistics 417 * 418 * Returns the address of the device statistics structure. 419 * The statistics are actually updated from the service task. 420 **/ 421 static void i40e_get_netdev_stats_struct(struct net_device *netdev, 422 struct rtnl_link_stats64 *stats) 423 { 424 struct i40e_netdev_priv *np = netdev_priv(netdev); 425 struct i40e_vsi *vsi = np->vsi; 426 struct rtnl_link_stats64 *vsi_stats = i40e_get_vsi_stats_struct(vsi); 427 struct i40e_ring *ring; 428 int i; 429 430 if (test_bit(__I40E_VSI_DOWN, vsi->state)) 431 return; 432 433 if (!vsi->tx_rings) 434 return; 435 436 rcu_read_lock(); 437 for (i = 0; i < vsi->num_queue_pairs; i++) { 438 u64 bytes, packets; 439 unsigned int start; 440 441 ring = READ_ONCE(vsi->tx_rings[i]); 442 if (!ring) 443 continue; 444 i40e_get_netdev_stats_struct_tx(ring, stats); 445 446 if (i40e_enabled_xdp_vsi(vsi)) { 447 ring++; 448 i40e_get_netdev_stats_struct_tx(ring, stats); 449 } 450 451 ring++; 452 do { 453 start = u64_stats_fetch_begin_irq(&ring->syncp); 454 packets = ring->stats.packets; 455 bytes = ring->stats.bytes; 456 } while (u64_stats_fetch_retry_irq(&ring->syncp, start)); 457 458 stats->rx_packets += packets; 459 stats->rx_bytes += bytes; 460 461 } 462 rcu_read_unlock(); 463 464 /* following stats updated by i40e_watchdog_subtask() */ 465 stats->multicast = vsi_stats->multicast; 466 stats->tx_errors = vsi_stats->tx_errors; 467 stats->tx_dropped = vsi_stats->tx_dropped; 468 stats->rx_errors = vsi_stats->rx_errors; 469 stats->rx_dropped = vsi_stats->rx_dropped; 470 stats->rx_crc_errors = vsi_stats->rx_crc_errors; 471 stats->rx_length_errors = vsi_stats->rx_length_errors; 472 } 473 474 /** 475 * i40e_vsi_reset_stats - Resets all stats of the given vsi 476 * @vsi: the VSI to have its stats reset 477 **/ 478 void i40e_vsi_reset_stats(struct i40e_vsi *vsi) 479 { 480 struct rtnl_link_stats64 *ns; 481 int i; 482 483 if (!vsi) 484 return; 485 486 ns = i40e_get_vsi_stats_struct(vsi); 487 memset(ns, 0, sizeof(*ns)); 488 memset(&vsi->net_stats_offsets, 0, sizeof(vsi->net_stats_offsets)); 489 memset(&vsi->eth_stats, 0, sizeof(vsi->eth_stats)); 490 memset(&vsi->eth_stats_offsets, 0, sizeof(vsi->eth_stats_offsets)); 491 if (vsi->rx_rings && vsi->rx_rings[0]) { 492 for (i = 0; i < vsi->num_queue_pairs; i++) { 493 memset(&vsi->rx_rings[i]->stats, 0, 494 sizeof(vsi->rx_rings[i]->stats)); 495 memset(&vsi->rx_rings[i]->rx_stats, 0, 496 sizeof(vsi->rx_rings[i]->rx_stats)); 497 memset(&vsi->tx_rings[i]->stats, 0, 498 sizeof(vsi->tx_rings[i]->stats)); 499 memset(&vsi->tx_rings[i]->tx_stats, 0, 500 sizeof(vsi->tx_rings[i]->tx_stats)); 501 } 502 } 503 vsi->stat_offsets_loaded = false; 504 } 505 506 /** 507 * i40e_pf_reset_stats - Reset all of the stats for the given PF 508 * @pf: the PF to be reset 509 **/ 510 void i40e_pf_reset_stats(struct i40e_pf *pf) 511 { 512 int i; 513 514 memset(&pf->stats, 0, sizeof(pf->stats)); 515 memset(&pf->stats_offsets, 0, sizeof(pf->stats_offsets)); 516 pf->stat_offsets_loaded = false; 517 518 for (i = 0; i < I40E_MAX_VEB; i++) { 519 if (pf->veb[i]) { 520 memset(&pf->veb[i]->stats, 0, 521 sizeof(pf->veb[i]->stats)); 522 memset(&pf->veb[i]->stats_offsets, 0, 523 sizeof(pf->veb[i]->stats_offsets)); 524 pf->veb[i]->stat_offsets_loaded = false; 525 } 526 } 527 pf->hw_csum_rx_error = 0; 528 } 529 530 /** 531 * i40e_stat_update48 - read and update a 48 bit stat from the chip 532 * @hw: ptr to the hardware info 533 * @hireg: the high 32 bit reg to read 534 * @loreg: the low 32 bit reg to read 535 * @offset_loaded: has the initial offset been loaded yet 536 * @offset: ptr to current offset value 537 * @stat: ptr to the stat 538 * 539 * Since the device stats are not reset at PFReset, they likely will not 540 * be zeroed when the driver starts. We'll save the first values read 541 * and use them as offsets to be subtracted from the raw values in order 542 * to report stats that count from zero. In the process, we also manage 543 * the potential roll-over. 544 **/ 545 static void i40e_stat_update48(struct i40e_hw *hw, u32 hireg, u32 loreg, 546 bool offset_loaded, u64 *offset, u64 *stat) 547 { 548 u64 new_data; 549 550 if (hw->device_id == I40E_DEV_ID_QEMU) { 551 new_data = rd32(hw, loreg); 552 new_data |= ((u64)(rd32(hw, hireg) & 0xFFFF)) << 32; 553 } else { 554 new_data = rd64(hw, loreg); 555 } 556 if (!offset_loaded) 557 *offset = new_data; 558 if (likely(new_data >= *offset)) 559 *stat = new_data - *offset; 560 else 561 *stat = (new_data + BIT_ULL(48)) - *offset; 562 *stat &= 0xFFFFFFFFFFFFULL; 563 } 564 565 /** 566 * i40e_stat_update32 - read and update a 32 bit stat from the chip 567 * @hw: ptr to the hardware info 568 * @reg: the hw reg to read 569 * @offset_loaded: has the initial offset been loaded yet 570 * @offset: ptr to current offset value 571 * @stat: ptr to the stat 572 **/ 573 static void i40e_stat_update32(struct i40e_hw *hw, u32 reg, 574 bool offset_loaded, u64 *offset, u64 *stat) 575 { 576 u32 new_data; 577 578 new_data = rd32(hw, reg); 579 if (!offset_loaded) 580 *offset = new_data; 581 if (likely(new_data >= *offset)) 582 *stat = (u32)(new_data - *offset); 583 else 584 *stat = (u32)((new_data + BIT_ULL(32)) - *offset); 585 } 586 587 /** 588 * i40e_stat_update_and_clear32 - read and clear hw reg, update a 32 bit stat 589 * @hw: ptr to the hardware info 590 * @reg: the hw reg to read and clear 591 * @stat: ptr to the stat 592 **/ 593 static void i40e_stat_update_and_clear32(struct i40e_hw *hw, u32 reg, u64 *stat) 594 { 595 u32 new_data = rd32(hw, reg); 596 597 wr32(hw, reg, 1); /* must write a nonzero value to clear register */ 598 *stat += new_data; 599 } 600 601 /** 602 * i40e_update_eth_stats - Update VSI-specific ethernet statistics counters. 603 * @vsi: the VSI to be updated 604 **/ 605 void i40e_update_eth_stats(struct i40e_vsi *vsi) 606 { 607 int stat_idx = le16_to_cpu(vsi->info.stat_counter_idx); 608 struct i40e_pf *pf = vsi->back; 609 struct i40e_hw *hw = &pf->hw; 610 struct i40e_eth_stats *oes; 611 struct i40e_eth_stats *es; /* device's eth stats */ 612 613 es = &vsi->eth_stats; 614 oes = &vsi->eth_stats_offsets; 615 616 /* Gather up the stats that the hw collects */ 617 i40e_stat_update32(hw, I40E_GLV_TEPC(stat_idx), 618 vsi->stat_offsets_loaded, 619 &oes->tx_errors, &es->tx_errors); 620 i40e_stat_update32(hw, I40E_GLV_RDPC(stat_idx), 621 vsi->stat_offsets_loaded, 622 &oes->rx_discards, &es->rx_discards); 623 i40e_stat_update32(hw, I40E_GLV_RUPP(stat_idx), 624 vsi->stat_offsets_loaded, 625 &oes->rx_unknown_protocol, &es->rx_unknown_protocol); 626 i40e_stat_update32(hw, I40E_GLV_TEPC(stat_idx), 627 vsi->stat_offsets_loaded, 628 &oes->tx_errors, &es->tx_errors); 629 630 i40e_stat_update48(hw, I40E_GLV_GORCH(stat_idx), 631 I40E_GLV_GORCL(stat_idx), 632 vsi->stat_offsets_loaded, 633 &oes->rx_bytes, &es->rx_bytes); 634 i40e_stat_update48(hw, I40E_GLV_UPRCH(stat_idx), 635 I40E_GLV_UPRCL(stat_idx), 636 vsi->stat_offsets_loaded, 637 &oes->rx_unicast, &es->rx_unicast); 638 i40e_stat_update48(hw, I40E_GLV_MPRCH(stat_idx), 639 I40E_GLV_MPRCL(stat_idx), 640 vsi->stat_offsets_loaded, 641 &oes->rx_multicast, &es->rx_multicast); 642 i40e_stat_update48(hw, I40E_GLV_BPRCH(stat_idx), 643 I40E_GLV_BPRCL(stat_idx), 644 vsi->stat_offsets_loaded, 645 &oes->rx_broadcast, &es->rx_broadcast); 646 647 i40e_stat_update48(hw, I40E_GLV_GOTCH(stat_idx), 648 I40E_GLV_GOTCL(stat_idx), 649 vsi->stat_offsets_loaded, 650 &oes->tx_bytes, &es->tx_bytes); 651 i40e_stat_update48(hw, I40E_GLV_UPTCH(stat_idx), 652 I40E_GLV_UPTCL(stat_idx), 653 vsi->stat_offsets_loaded, 654 &oes->tx_unicast, &es->tx_unicast); 655 i40e_stat_update48(hw, I40E_GLV_MPTCH(stat_idx), 656 I40E_GLV_MPTCL(stat_idx), 657 vsi->stat_offsets_loaded, 658 &oes->tx_multicast, &es->tx_multicast); 659 i40e_stat_update48(hw, I40E_GLV_BPTCH(stat_idx), 660 I40E_GLV_BPTCL(stat_idx), 661 vsi->stat_offsets_loaded, 662 &oes->tx_broadcast, &es->tx_broadcast); 663 vsi->stat_offsets_loaded = true; 664 } 665 666 /** 667 * i40e_update_veb_stats - Update Switch component statistics 668 * @veb: the VEB being updated 669 **/ 670 static void i40e_update_veb_stats(struct i40e_veb *veb) 671 { 672 struct i40e_pf *pf = veb->pf; 673 struct i40e_hw *hw = &pf->hw; 674 struct i40e_eth_stats *oes; 675 struct i40e_eth_stats *es; /* device's eth stats */ 676 struct i40e_veb_tc_stats *veb_oes; 677 struct i40e_veb_tc_stats *veb_es; 678 int i, idx = 0; 679 680 idx = veb->stats_idx; 681 es = &veb->stats; 682 oes = &veb->stats_offsets; 683 veb_es = &veb->tc_stats; 684 veb_oes = &veb->tc_stats_offsets; 685 686 /* Gather up the stats that the hw collects */ 687 i40e_stat_update32(hw, I40E_GLSW_TDPC(idx), 688 veb->stat_offsets_loaded, 689 &oes->tx_discards, &es->tx_discards); 690 if (hw->revision_id > 0) 691 i40e_stat_update32(hw, I40E_GLSW_RUPP(idx), 692 veb->stat_offsets_loaded, 693 &oes->rx_unknown_protocol, 694 &es->rx_unknown_protocol); 695 i40e_stat_update48(hw, I40E_GLSW_GORCH(idx), I40E_GLSW_GORCL(idx), 696 veb->stat_offsets_loaded, 697 &oes->rx_bytes, &es->rx_bytes); 698 i40e_stat_update48(hw, I40E_GLSW_UPRCH(idx), I40E_GLSW_UPRCL(idx), 699 veb->stat_offsets_loaded, 700 &oes->rx_unicast, &es->rx_unicast); 701 i40e_stat_update48(hw, I40E_GLSW_MPRCH(idx), I40E_GLSW_MPRCL(idx), 702 veb->stat_offsets_loaded, 703 &oes->rx_multicast, &es->rx_multicast); 704 i40e_stat_update48(hw, I40E_GLSW_BPRCH(idx), I40E_GLSW_BPRCL(idx), 705 veb->stat_offsets_loaded, 706 &oes->rx_broadcast, &es->rx_broadcast); 707 708 i40e_stat_update48(hw, I40E_GLSW_GOTCH(idx), I40E_GLSW_GOTCL(idx), 709 veb->stat_offsets_loaded, 710 &oes->tx_bytes, &es->tx_bytes); 711 i40e_stat_update48(hw, I40E_GLSW_UPTCH(idx), I40E_GLSW_UPTCL(idx), 712 veb->stat_offsets_loaded, 713 &oes->tx_unicast, &es->tx_unicast); 714 i40e_stat_update48(hw, I40E_GLSW_MPTCH(idx), I40E_GLSW_MPTCL(idx), 715 veb->stat_offsets_loaded, 716 &oes->tx_multicast, &es->tx_multicast); 717 i40e_stat_update48(hw, I40E_GLSW_BPTCH(idx), I40E_GLSW_BPTCL(idx), 718 veb->stat_offsets_loaded, 719 &oes->tx_broadcast, &es->tx_broadcast); 720 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 721 i40e_stat_update48(hw, I40E_GLVEBTC_RPCH(i, idx), 722 I40E_GLVEBTC_RPCL(i, idx), 723 veb->stat_offsets_loaded, 724 &veb_oes->tc_rx_packets[i], 725 &veb_es->tc_rx_packets[i]); 726 i40e_stat_update48(hw, I40E_GLVEBTC_RBCH(i, idx), 727 I40E_GLVEBTC_RBCL(i, idx), 728 veb->stat_offsets_loaded, 729 &veb_oes->tc_rx_bytes[i], 730 &veb_es->tc_rx_bytes[i]); 731 i40e_stat_update48(hw, I40E_GLVEBTC_TPCH(i, idx), 732 I40E_GLVEBTC_TPCL(i, idx), 733 veb->stat_offsets_loaded, 734 &veb_oes->tc_tx_packets[i], 735 &veb_es->tc_tx_packets[i]); 736 i40e_stat_update48(hw, I40E_GLVEBTC_TBCH(i, idx), 737 I40E_GLVEBTC_TBCL(i, idx), 738 veb->stat_offsets_loaded, 739 &veb_oes->tc_tx_bytes[i], 740 &veb_es->tc_tx_bytes[i]); 741 } 742 veb->stat_offsets_loaded = true; 743 } 744 745 /** 746 * i40e_update_vsi_stats - Update the vsi statistics counters. 747 * @vsi: the VSI to be updated 748 * 749 * There are a few instances where we store the same stat in a 750 * couple of different structs. This is partly because we have 751 * the netdev stats that need to be filled out, which is slightly 752 * different from the "eth_stats" defined by the chip and used in 753 * VF communications. We sort it out here. 754 **/ 755 static void i40e_update_vsi_stats(struct i40e_vsi *vsi) 756 { 757 struct i40e_pf *pf = vsi->back; 758 struct rtnl_link_stats64 *ons; 759 struct rtnl_link_stats64 *ns; /* netdev stats */ 760 struct i40e_eth_stats *oes; 761 struct i40e_eth_stats *es; /* device's eth stats */ 762 u32 tx_restart, tx_busy; 763 struct i40e_ring *p; 764 u32 rx_page, rx_buf; 765 u64 bytes, packets; 766 unsigned int start; 767 u64 tx_linearize; 768 u64 tx_force_wb; 769 u64 rx_p, rx_b; 770 u64 tx_p, tx_b; 771 u16 q; 772 773 if (test_bit(__I40E_VSI_DOWN, vsi->state) || 774 test_bit(__I40E_CONFIG_BUSY, pf->state)) 775 return; 776 777 ns = i40e_get_vsi_stats_struct(vsi); 778 ons = &vsi->net_stats_offsets; 779 es = &vsi->eth_stats; 780 oes = &vsi->eth_stats_offsets; 781 782 /* Gather up the netdev and vsi stats that the driver collects 783 * on the fly during packet processing 784 */ 785 rx_b = rx_p = 0; 786 tx_b = tx_p = 0; 787 tx_restart = tx_busy = tx_linearize = tx_force_wb = 0; 788 rx_page = 0; 789 rx_buf = 0; 790 rcu_read_lock(); 791 for (q = 0; q < vsi->num_queue_pairs; q++) { 792 /* locate Tx ring */ 793 p = READ_ONCE(vsi->tx_rings[q]); 794 795 do { 796 start = u64_stats_fetch_begin_irq(&p->syncp); 797 packets = p->stats.packets; 798 bytes = p->stats.bytes; 799 } while (u64_stats_fetch_retry_irq(&p->syncp, start)); 800 tx_b += bytes; 801 tx_p += packets; 802 tx_restart += p->tx_stats.restart_queue; 803 tx_busy += p->tx_stats.tx_busy; 804 tx_linearize += p->tx_stats.tx_linearize; 805 tx_force_wb += p->tx_stats.tx_force_wb; 806 807 /* Rx queue is part of the same block as Tx queue */ 808 p = &p[1]; 809 do { 810 start = u64_stats_fetch_begin_irq(&p->syncp); 811 packets = p->stats.packets; 812 bytes = p->stats.bytes; 813 } while (u64_stats_fetch_retry_irq(&p->syncp, start)); 814 rx_b += bytes; 815 rx_p += packets; 816 rx_buf += p->rx_stats.alloc_buff_failed; 817 rx_page += p->rx_stats.alloc_page_failed; 818 } 819 rcu_read_unlock(); 820 vsi->tx_restart = tx_restart; 821 vsi->tx_busy = tx_busy; 822 vsi->tx_linearize = tx_linearize; 823 vsi->tx_force_wb = tx_force_wb; 824 vsi->rx_page_failed = rx_page; 825 vsi->rx_buf_failed = rx_buf; 826 827 ns->rx_packets = rx_p; 828 ns->rx_bytes = rx_b; 829 ns->tx_packets = tx_p; 830 ns->tx_bytes = tx_b; 831 832 /* update netdev stats from eth stats */ 833 i40e_update_eth_stats(vsi); 834 ons->tx_errors = oes->tx_errors; 835 ns->tx_errors = es->tx_errors; 836 ons->multicast = oes->rx_multicast; 837 ns->multicast = es->rx_multicast; 838 ons->rx_dropped = oes->rx_discards; 839 ns->rx_dropped = es->rx_discards; 840 ons->tx_dropped = oes->tx_discards; 841 ns->tx_dropped = es->tx_discards; 842 843 /* pull in a couple PF stats if this is the main vsi */ 844 if (vsi == pf->vsi[pf->lan_vsi]) { 845 ns->rx_crc_errors = pf->stats.crc_errors; 846 ns->rx_errors = pf->stats.crc_errors + pf->stats.illegal_bytes; 847 ns->rx_length_errors = pf->stats.rx_length_errors; 848 } 849 } 850 851 /** 852 * i40e_update_pf_stats - Update the PF statistics counters. 853 * @pf: the PF to be updated 854 **/ 855 static void i40e_update_pf_stats(struct i40e_pf *pf) 856 { 857 struct i40e_hw_port_stats *osd = &pf->stats_offsets; 858 struct i40e_hw_port_stats *nsd = &pf->stats; 859 struct i40e_hw *hw = &pf->hw; 860 u32 val; 861 int i; 862 863 i40e_stat_update48(hw, I40E_GLPRT_GORCH(hw->port), 864 I40E_GLPRT_GORCL(hw->port), 865 pf->stat_offsets_loaded, 866 &osd->eth.rx_bytes, &nsd->eth.rx_bytes); 867 i40e_stat_update48(hw, I40E_GLPRT_GOTCH(hw->port), 868 I40E_GLPRT_GOTCL(hw->port), 869 pf->stat_offsets_loaded, 870 &osd->eth.tx_bytes, &nsd->eth.tx_bytes); 871 i40e_stat_update32(hw, I40E_GLPRT_RDPC(hw->port), 872 pf->stat_offsets_loaded, 873 &osd->eth.rx_discards, 874 &nsd->eth.rx_discards); 875 i40e_stat_update48(hw, I40E_GLPRT_UPRCH(hw->port), 876 I40E_GLPRT_UPRCL(hw->port), 877 pf->stat_offsets_loaded, 878 &osd->eth.rx_unicast, 879 &nsd->eth.rx_unicast); 880 i40e_stat_update48(hw, I40E_GLPRT_MPRCH(hw->port), 881 I40E_GLPRT_MPRCL(hw->port), 882 pf->stat_offsets_loaded, 883 &osd->eth.rx_multicast, 884 &nsd->eth.rx_multicast); 885 i40e_stat_update48(hw, I40E_GLPRT_BPRCH(hw->port), 886 I40E_GLPRT_BPRCL(hw->port), 887 pf->stat_offsets_loaded, 888 &osd->eth.rx_broadcast, 889 &nsd->eth.rx_broadcast); 890 i40e_stat_update48(hw, I40E_GLPRT_UPTCH(hw->port), 891 I40E_GLPRT_UPTCL(hw->port), 892 pf->stat_offsets_loaded, 893 &osd->eth.tx_unicast, 894 &nsd->eth.tx_unicast); 895 i40e_stat_update48(hw, I40E_GLPRT_MPTCH(hw->port), 896 I40E_GLPRT_MPTCL(hw->port), 897 pf->stat_offsets_loaded, 898 &osd->eth.tx_multicast, 899 &nsd->eth.tx_multicast); 900 i40e_stat_update48(hw, I40E_GLPRT_BPTCH(hw->port), 901 I40E_GLPRT_BPTCL(hw->port), 902 pf->stat_offsets_loaded, 903 &osd->eth.tx_broadcast, 904 &nsd->eth.tx_broadcast); 905 906 i40e_stat_update32(hw, I40E_GLPRT_TDOLD(hw->port), 907 pf->stat_offsets_loaded, 908 &osd->tx_dropped_link_down, 909 &nsd->tx_dropped_link_down); 910 911 i40e_stat_update32(hw, I40E_GLPRT_CRCERRS(hw->port), 912 pf->stat_offsets_loaded, 913 &osd->crc_errors, &nsd->crc_errors); 914 915 i40e_stat_update32(hw, I40E_GLPRT_ILLERRC(hw->port), 916 pf->stat_offsets_loaded, 917 &osd->illegal_bytes, &nsd->illegal_bytes); 918 919 i40e_stat_update32(hw, I40E_GLPRT_MLFC(hw->port), 920 pf->stat_offsets_loaded, 921 &osd->mac_local_faults, 922 &nsd->mac_local_faults); 923 i40e_stat_update32(hw, I40E_GLPRT_MRFC(hw->port), 924 pf->stat_offsets_loaded, 925 &osd->mac_remote_faults, 926 &nsd->mac_remote_faults); 927 928 i40e_stat_update32(hw, I40E_GLPRT_RLEC(hw->port), 929 pf->stat_offsets_loaded, 930 &osd->rx_length_errors, 931 &nsd->rx_length_errors); 932 933 i40e_stat_update32(hw, I40E_GLPRT_LXONRXC(hw->port), 934 pf->stat_offsets_loaded, 935 &osd->link_xon_rx, &nsd->link_xon_rx); 936 i40e_stat_update32(hw, I40E_GLPRT_LXONTXC(hw->port), 937 pf->stat_offsets_loaded, 938 &osd->link_xon_tx, &nsd->link_xon_tx); 939 i40e_stat_update32(hw, I40E_GLPRT_LXOFFRXC(hw->port), 940 pf->stat_offsets_loaded, 941 &osd->link_xoff_rx, &nsd->link_xoff_rx); 942 i40e_stat_update32(hw, I40E_GLPRT_LXOFFTXC(hw->port), 943 pf->stat_offsets_loaded, 944 &osd->link_xoff_tx, &nsd->link_xoff_tx); 945 946 for (i = 0; i < 8; i++) { 947 i40e_stat_update32(hw, I40E_GLPRT_PXOFFRXC(hw->port, i), 948 pf->stat_offsets_loaded, 949 &osd->priority_xoff_rx[i], 950 &nsd->priority_xoff_rx[i]); 951 i40e_stat_update32(hw, I40E_GLPRT_PXONRXC(hw->port, i), 952 pf->stat_offsets_loaded, 953 &osd->priority_xon_rx[i], 954 &nsd->priority_xon_rx[i]); 955 i40e_stat_update32(hw, I40E_GLPRT_PXONTXC(hw->port, i), 956 pf->stat_offsets_loaded, 957 &osd->priority_xon_tx[i], 958 &nsd->priority_xon_tx[i]); 959 i40e_stat_update32(hw, I40E_GLPRT_PXOFFTXC(hw->port, i), 960 pf->stat_offsets_loaded, 961 &osd->priority_xoff_tx[i], 962 &nsd->priority_xoff_tx[i]); 963 i40e_stat_update32(hw, 964 I40E_GLPRT_RXON2OFFCNT(hw->port, i), 965 pf->stat_offsets_loaded, 966 &osd->priority_xon_2_xoff[i], 967 &nsd->priority_xon_2_xoff[i]); 968 } 969 970 i40e_stat_update48(hw, I40E_GLPRT_PRC64H(hw->port), 971 I40E_GLPRT_PRC64L(hw->port), 972 pf->stat_offsets_loaded, 973 &osd->rx_size_64, &nsd->rx_size_64); 974 i40e_stat_update48(hw, I40E_GLPRT_PRC127H(hw->port), 975 I40E_GLPRT_PRC127L(hw->port), 976 pf->stat_offsets_loaded, 977 &osd->rx_size_127, &nsd->rx_size_127); 978 i40e_stat_update48(hw, I40E_GLPRT_PRC255H(hw->port), 979 I40E_GLPRT_PRC255L(hw->port), 980 pf->stat_offsets_loaded, 981 &osd->rx_size_255, &nsd->rx_size_255); 982 i40e_stat_update48(hw, I40E_GLPRT_PRC511H(hw->port), 983 I40E_GLPRT_PRC511L(hw->port), 984 pf->stat_offsets_loaded, 985 &osd->rx_size_511, &nsd->rx_size_511); 986 i40e_stat_update48(hw, I40E_GLPRT_PRC1023H(hw->port), 987 I40E_GLPRT_PRC1023L(hw->port), 988 pf->stat_offsets_loaded, 989 &osd->rx_size_1023, &nsd->rx_size_1023); 990 i40e_stat_update48(hw, I40E_GLPRT_PRC1522H(hw->port), 991 I40E_GLPRT_PRC1522L(hw->port), 992 pf->stat_offsets_loaded, 993 &osd->rx_size_1522, &nsd->rx_size_1522); 994 i40e_stat_update48(hw, I40E_GLPRT_PRC9522H(hw->port), 995 I40E_GLPRT_PRC9522L(hw->port), 996 pf->stat_offsets_loaded, 997 &osd->rx_size_big, &nsd->rx_size_big); 998 999 i40e_stat_update48(hw, I40E_GLPRT_PTC64H(hw->port), 1000 I40E_GLPRT_PTC64L(hw->port), 1001 pf->stat_offsets_loaded, 1002 &osd->tx_size_64, &nsd->tx_size_64); 1003 i40e_stat_update48(hw, I40E_GLPRT_PTC127H(hw->port), 1004 I40E_GLPRT_PTC127L(hw->port), 1005 pf->stat_offsets_loaded, 1006 &osd->tx_size_127, &nsd->tx_size_127); 1007 i40e_stat_update48(hw, I40E_GLPRT_PTC255H(hw->port), 1008 I40E_GLPRT_PTC255L(hw->port), 1009 pf->stat_offsets_loaded, 1010 &osd->tx_size_255, &nsd->tx_size_255); 1011 i40e_stat_update48(hw, I40E_GLPRT_PTC511H(hw->port), 1012 I40E_GLPRT_PTC511L(hw->port), 1013 pf->stat_offsets_loaded, 1014 &osd->tx_size_511, &nsd->tx_size_511); 1015 i40e_stat_update48(hw, I40E_GLPRT_PTC1023H(hw->port), 1016 I40E_GLPRT_PTC1023L(hw->port), 1017 pf->stat_offsets_loaded, 1018 &osd->tx_size_1023, &nsd->tx_size_1023); 1019 i40e_stat_update48(hw, I40E_GLPRT_PTC1522H(hw->port), 1020 I40E_GLPRT_PTC1522L(hw->port), 1021 pf->stat_offsets_loaded, 1022 &osd->tx_size_1522, &nsd->tx_size_1522); 1023 i40e_stat_update48(hw, I40E_GLPRT_PTC9522H(hw->port), 1024 I40E_GLPRT_PTC9522L(hw->port), 1025 pf->stat_offsets_loaded, 1026 &osd->tx_size_big, &nsd->tx_size_big); 1027 1028 i40e_stat_update32(hw, I40E_GLPRT_RUC(hw->port), 1029 pf->stat_offsets_loaded, 1030 &osd->rx_undersize, &nsd->rx_undersize); 1031 i40e_stat_update32(hw, I40E_GLPRT_RFC(hw->port), 1032 pf->stat_offsets_loaded, 1033 &osd->rx_fragments, &nsd->rx_fragments); 1034 i40e_stat_update32(hw, I40E_GLPRT_ROC(hw->port), 1035 pf->stat_offsets_loaded, 1036 &osd->rx_oversize, &nsd->rx_oversize); 1037 i40e_stat_update32(hw, I40E_GLPRT_RJC(hw->port), 1038 pf->stat_offsets_loaded, 1039 &osd->rx_jabber, &nsd->rx_jabber); 1040 1041 /* FDIR stats */ 1042 i40e_stat_update_and_clear32(hw, 1043 I40E_GLQF_PCNT(I40E_FD_ATR_STAT_IDX(hw->pf_id)), 1044 &nsd->fd_atr_match); 1045 i40e_stat_update_and_clear32(hw, 1046 I40E_GLQF_PCNT(I40E_FD_SB_STAT_IDX(hw->pf_id)), 1047 &nsd->fd_sb_match); 1048 i40e_stat_update_and_clear32(hw, 1049 I40E_GLQF_PCNT(I40E_FD_ATR_TUNNEL_STAT_IDX(hw->pf_id)), 1050 &nsd->fd_atr_tunnel_match); 1051 1052 val = rd32(hw, I40E_PRTPM_EEE_STAT); 1053 nsd->tx_lpi_status = 1054 (val & I40E_PRTPM_EEE_STAT_TX_LPI_STATUS_MASK) >> 1055 I40E_PRTPM_EEE_STAT_TX_LPI_STATUS_SHIFT; 1056 nsd->rx_lpi_status = 1057 (val & I40E_PRTPM_EEE_STAT_RX_LPI_STATUS_MASK) >> 1058 I40E_PRTPM_EEE_STAT_RX_LPI_STATUS_SHIFT; 1059 i40e_stat_update32(hw, I40E_PRTPM_TLPIC, 1060 pf->stat_offsets_loaded, 1061 &osd->tx_lpi_count, &nsd->tx_lpi_count); 1062 i40e_stat_update32(hw, I40E_PRTPM_RLPIC, 1063 pf->stat_offsets_loaded, 1064 &osd->rx_lpi_count, &nsd->rx_lpi_count); 1065 1066 if (pf->flags & I40E_FLAG_FD_SB_ENABLED && 1067 !test_bit(__I40E_FD_SB_AUTO_DISABLED, pf->state)) 1068 nsd->fd_sb_status = true; 1069 else 1070 nsd->fd_sb_status = false; 1071 1072 if (pf->flags & I40E_FLAG_FD_ATR_ENABLED && 1073 !test_bit(__I40E_FD_ATR_AUTO_DISABLED, pf->state)) 1074 nsd->fd_atr_status = true; 1075 else 1076 nsd->fd_atr_status = false; 1077 1078 pf->stat_offsets_loaded = true; 1079 } 1080 1081 /** 1082 * i40e_update_stats - Update the various statistics counters. 1083 * @vsi: the VSI to be updated 1084 * 1085 * Update the various stats for this VSI and its related entities. 1086 **/ 1087 void i40e_update_stats(struct i40e_vsi *vsi) 1088 { 1089 struct i40e_pf *pf = vsi->back; 1090 1091 if (vsi == pf->vsi[pf->lan_vsi]) 1092 i40e_update_pf_stats(pf); 1093 1094 i40e_update_vsi_stats(vsi); 1095 } 1096 1097 /** 1098 * i40e_find_filter - Search VSI filter list for specific mac/vlan filter 1099 * @vsi: the VSI to be searched 1100 * @macaddr: the MAC address 1101 * @vlan: the vlan 1102 * 1103 * Returns ptr to the filter object or NULL 1104 **/ 1105 static struct i40e_mac_filter *i40e_find_filter(struct i40e_vsi *vsi, 1106 const u8 *macaddr, s16 vlan) 1107 { 1108 struct i40e_mac_filter *f; 1109 u64 key; 1110 1111 if (!vsi || !macaddr) 1112 return NULL; 1113 1114 key = i40e_addr_to_hkey(macaddr); 1115 hash_for_each_possible(vsi->mac_filter_hash, f, hlist, key) { 1116 if ((ether_addr_equal(macaddr, f->macaddr)) && 1117 (vlan == f->vlan)) 1118 return f; 1119 } 1120 return NULL; 1121 } 1122 1123 /** 1124 * i40e_find_mac - Find a mac addr in the macvlan filters list 1125 * @vsi: the VSI to be searched 1126 * @macaddr: the MAC address we are searching for 1127 * 1128 * Returns the first filter with the provided MAC address or NULL if 1129 * MAC address was not found 1130 **/ 1131 struct i40e_mac_filter *i40e_find_mac(struct i40e_vsi *vsi, const u8 *macaddr) 1132 { 1133 struct i40e_mac_filter *f; 1134 u64 key; 1135 1136 if (!vsi || !macaddr) 1137 return NULL; 1138 1139 key = i40e_addr_to_hkey(macaddr); 1140 hash_for_each_possible(vsi->mac_filter_hash, f, hlist, key) { 1141 if ((ether_addr_equal(macaddr, f->macaddr))) 1142 return f; 1143 } 1144 return NULL; 1145 } 1146 1147 /** 1148 * i40e_is_vsi_in_vlan - Check if VSI is in vlan mode 1149 * @vsi: the VSI to be searched 1150 * 1151 * Returns true if VSI is in vlan mode or false otherwise 1152 **/ 1153 bool i40e_is_vsi_in_vlan(struct i40e_vsi *vsi) 1154 { 1155 /* If we have a PVID, always operate in VLAN mode */ 1156 if (vsi->info.pvid) 1157 return true; 1158 1159 /* We need to operate in VLAN mode whenever we have any filters with 1160 * a VLAN other than I40E_VLAN_ALL. We could check the table each 1161 * time, incurring search cost repeatedly. However, we can notice two 1162 * things: 1163 * 1164 * 1) the only place where we can gain a VLAN filter is in 1165 * i40e_add_filter. 1166 * 1167 * 2) the only place where filters are actually removed is in 1168 * i40e_sync_filters_subtask. 1169 * 1170 * Thus, we can simply use a boolean value, has_vlan_filters which we 1171 * will set to true when we add a VLAN filter in i40e_add_filter. Then 1172 * we have to perform the full search after deleting filters in 1173 * i40e_sync_filters_subtask, but we already have to search 1174 * filters here and can perform the check at the same time. This 1175 * results in avoiding embedding a loop for VLAN mode inside another 1176 * loop over all the filters, and should maintain correctness as noted 1177 * above. 1178 */ 1179 return vsi->has_vlan_filter; 1180 } 1181 1182 /** 1183 * i40e_correct_mac_vlan_filters - Correct non-VLAN filters if necessary 1184 * @vsi: the VSI to configure 1185 * @tmp_add_list: list of filters ready to be added 1186 * @tmp_del_list: list of filters ready to be deleted 1187 * @vlan_filters: the number of active VLAN filters 1188 * 1189 * Update VLAN=0 and VLAN=-1 (I40E_VLAN_ANY) filters properly so that they 1190 * behave as expected. If we have any active VLAN filters remaining or about 1191 * to be added then we need to update non-VLAN filters to be marked as VLAN=0 1192 * so that they only match against untagged traffic. If we no longer have any 1193 * active VLAN filters, we need to make all non-VLAN filters marked as VLAN=-1 1194 * so that they match against both tagged and untagged traffic. In this way, 1195 * we ensure that we correctly receive the desired traffic. This ensures that 1196 * when we have an active VLAN we will receive only untagged traffic and 1197 * traffic matching active VLANs. If we have no active VLANs then we will 1198 * operate in non-VLAN mode and receive all traffic, tagged or untagged. 1199 * 1200 * Finally, in a similar fashion, this function also corrects filters when 1201 * there is an active PVID assigned to this VSI. 1202 * 1203 * In case of memory allocation failure return -ENOMEM. Otherwise, return 0. 1204 * 1205 * This function is only expected to be called from within 1206 * i40e_sync_vsi_filters. 1207 * 1208 * NOTE: This function expects to be called while under the 1209 * mac_filter_hash_lock 1210 */ 1211 static int i40e_correct_mac_vlan_filters(struct i40e_vsi *vsi, 1212 struct hlist_head *tmp_add_list, 1213 struct hlist_head *tmp_del_list, 1214 int vlan_filters) 1215 { 1216 s16 pvid = le16_to_cpu(vsi->info.pvid); 1217 struct i40e_mac_filter *f, *add_head; 1218 struct i40e_new_mac_filter *new; 1219 struct hlist_node *h; 1220 int bkt, new_vlan; 1221 1222 /* To determine if a particular filter needs to be replaced we 1223 * have the three following conditions: 1224 * 1225 * a) if we have a PVID assigned, then all filters which are 1226 * not marked as VLAN=PVID must be replaced with filters that 1227 * are. 1228 * b) otherwise, if we have any active VLANS, all filters 1229 * which are marked as VLAN=-1 must be replaced with 1230 * filters marked as VLAN=0 1231 * c) finally, if we do not have any active VLANS, all filters 1232 * which are marked as VLAN=0 must be replaced with filters 1233 * marked as VLAN=-1 1234 */ 1235 1236 /* Update the filters about to be added in place */ 1237 hlist_for_each_entry(new, tmp_add_list, hlist) { 1238 if (pvid && new->f->vlan != pvid) 1239 new->f->vlan = pvid; 1240 else if (vlan_filters && new->f->vlan == I40E_VLAN_ANY) 1241 new->f->vlan = 0; 1242 else if (!vlan_filters && new->f->vlan == 0) 1243 new->f->vlan = I40E_VLAN_ANY; 1244 } 1245 1246 /* Update the remaining active filters */ 1247 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) { 1248 /* Combine the checks for whether a filter needs to be changed 1249 * and then determine the new VLAN inside the if block, in 1250 * order to avoid duplicating code for adding the new filter 1251 * then deleting the old filter. 1252 */ 1253 if ((pvid && f->vlan != pvid) || 1254 (vlan_filters && f->vlan == I40E_VLAN_ANY) || 1255 (!vlan_filters && f->vlan == 0)) { 1256 /* Determine the new vlan we will be adding */ 1257 if (pvid) 1258 new_vlan = pvid; 1259 else if (vlan_filters) 1260 new_vlan = 0; 1261 else 1262 new_vlan = I40E_VLAN_ANY; 1263 1264 /* Create the new filter */ 1265 add_head = i40e_add_filter(vsi, f->macaddr, new_vlan); 1266 if (!add_head) 1267 return -ENOMEM; 1268 1269 /* Create a temporary i40e_new_mac_filter */ 1270 new = kzalloc(sizeof(*new), GFP_ATOMIC); 1271 if (!new) 1272 return -ENOMEM; 1273 1274 new->f = add_head; 1275 new->state = add_head->state; 1276 1277 /* Add the new filter to the tmp list */ 1278 hlist_add_head(&new->hlist, tmp_add_list); 1279 1280 /* Put the original filter into the delete list */ 1281 f->state = I40E_FILTER_REMOVE; 1282 hash_del(&f->hlist); 1283 hlist_add_head(&f->hlist, tmp_del_list); 1284 } 1285 } 1286 1287 vsi->has_vlan_filter = !!vlan_filters; 1288 1289 return 0; 1290 } 1291 1292 /** 1293 * i40e_rm_default_mac_filter - Remove the default MAC filter set by NVM 1294 * @vsi: the PF Main VSI - inappropriate for any other VSI 1295 * @macaddr: the MAC address 1296 * 1297 * Remove whatever filter the firmware set up so the driver can manage 1298 * its own filtering intelligently. 1299 **/ 1300 static void i40e_rm_default_mac_filter(struct i40e_vsi *vsi, u8 *macaddr) 1301 { 1302 struct i40e_aqc_remove_macvlan_element_data element; 1303 struct i40e_pf *pf = vsi->back; 1304 1305 /* Only appropriate for the PF main VSI */ 1306 if (vsi->type != I40E_VSI_MAIN) 1307 return; 1308 1309 memset(&element, 0, sizeof(element)); 1310 ether_addr_copy(element.mac_addr, macaddr); 1311 element.vlan_tag = 0; 1312 /* Ignore error returns, some firmware does it this way... */ 1313 element.flags = I40E_AQC_MACVLAN_DEL_PERFECT_MATCH; 1314 i40e_aq_remove_macvlan(&pf->hw, vsi->seid, &element, 1, NULL); 1315 1316 memset(&element, 0, sizeof(element)); 1317 ether_addr_copy(element.mac_addr, macaddr); 1318 element.vlan_tag = 0; 1319 /* ...and some firmware does it this way. */ 1320 element.flags = I40E_AQC_MACVLAN_DEL_PERFECT_MATCH | 1321 I40E_AQC_MACVLAN_DEL_IGNORE_VLAN; 1322 i40e_aq_remove_macvlan(&pf->hw, vsi->seid, &element, 1, NULL); 1323 } 1324 1325 /** 1326 * i40e_add_filter - Add a mac/vlan filter to the VSI 1327 * @vsi: the VSI to be searched 1328 * @macaddr: the MAC address 1329 * @vlan: the vlan 1330 * 1331 * Returns ptr to the filter object or NULL when no memory available. 1332 * 1333 * NOTE: This function is expected to be called with mac_filter_hash_lock 1334 * being held. 1335 **/ 1336 struct i40e_mac_filter *i40e_add_filter(struct i40e_vsi *vsi, 1337 const u8 *macaddr, s16 vlan) 1338 { 1339 struct i40e_mac_filter *f; 1340 u64 key; 1341 1342 if (!vsi || !macaddr) 1343 return NULL; 1344 1345 f = i40e_find_filter(vsi, macaddr, vlan); 1346 if (!f) { 1347 f = kzalloc(sizeof(*f), GFP_ATOMIC); 1348 if (!f) 1349 return NULL; 1350 1351 /* Update the boolean indicating if we need to function in 1352 * VLAN mode. 1353 */ 1354 if (vlan >= 0) 1355 vsi->has_vlan_filter = true; 1356 1357 ether_addr_copy(f->macaddr, macaddr); 1358 f->vlan = vlan; 1359 f->state = I40E_FILTER_NEW; 1360 INIT_HLIST_NODE(&f->hlist); 1361 1362 key = i40e_addr_to_hkey(macaddr); 1363 hash_add(vsi->mac_filter_hash, &f->hlist, key); 1364 1365 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED; 1366 set_bit(__I40E_MACVLAN_SYNC_PENDING, vsi->back->state); 1367 } 1368 1369 /* If we're asked to add a filter that has been marked for removal, it 1370 * is safe to simply restore it to active state. __i40e_del_filter 1371 * will have simply deleted any filters which were previously marked 1372 * NEW or FAILED, so if it is currently marked REMOVE it must have 1373 * previously been ACTIVE. Since we haven't yet run the sync filters 1374 * task, just restore this filter to the ACTIVE state so that the 1375 * sync task leaves it in place 1376 */ 1377 if (f->state == I40E_FILTER_REMOVE) 1378 f->state = I40E_FILTER_ACTIVE; 1379 1380 return f; 1381 } 1382 1383 /** 1384 * __i40e_del_filter - Remove a specific filter from the VSI 1385 * @vsi: VSI to remove from 1386 * @f: the filter to remove from the list 1387 * 1388 * This function should be called instead of i40e_del_filter only if you know 1389 * the exact filter you will remove already, such as via i40e_find_filter or 1390 * i40e_find_mac. 1391 * 1392 * NOTE: This function is expected to be called with mac_filter_hash_lock 1393 * being held. 1394 * ANOTHER NOTE: This function MUST be called from within the context of 1395 * the "safe" variants of any list iterators, e.g. list_for_each_entry_safe() 1396 * instead of list_for_each_entry(). 1397 **/ 1398 void __i40e_del_filter(struct i40e_vsi *vsi, struct i40e_mac_filter *f) 1399 { 1400 if (!f) 1401 return; 1402 1403 /* If the filter was never added to firmware then we can just delete it 1404 * directly and we don't want to set the status to remove or else an 1405 * admin queue command will unnecessarily fire. 1406 */ 1407 if ((f->state == I40E_FILTER_FAILED) || 1408 (f->state == I40E_FILTER_NEW)) { 1409 hash_del(&f->hlist); 1410 kfree(f); 1411 } else { 1412 f->state = I40E_FILTER_REMOVE; 1413 } 1414 1415 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED; 1416 set_bit(__I40E_MACVLAN_SYNC_PENDING, vsi->state); 1417 } 1418 1419 /** 1420 * i40e_del_filter - Remove a MAC/VLAN filter from the VSI 1421 * @vsi: the VSI to be searched 1422 * @macaddr: the MAC address 1423 * @vlan: the VLAN 1424 * 1425 * NOTE: This function is expected to be called with mac_filter_hash_lock 1426 * being held. 1427 * ANOTHER NOTE: This function MUST be called from within the context of 1428 * the "safe" variants of any list iterators, e.g. list_for_each_entry_safe() 1429 * instead of list_for_each_entry(). 1430 **/ 1431 void i40e_del_filter(struct i40e_vsi *vsi, const u8 *macaddr, s16 vlan) 1432 { 1433 struct i40e_mac_filter *f; 1434 1435 if (!vsi || !macaddr) 1436 return; 1437 1438 f = i40e_find_filter(vsi, macaddr, vlan); 1439 __i40e_del_filter(vsi, f); 1440 } 1441 1442 /** 1443 * i40e_add_mac_filter - Add a MAC filter for all active VLANs 1444 * @vsi: the VSI to be searched 1445 * @macaddr: the mac address to be filtered 1446 * 1447 * If we're not in VLAN mode, just add the filter to I40E_VLAN_ANY. Otherwise, 1448 * go through all the macvlan filters and add a macvlan filter for each 1449 * unique vlan that already exists. If a PVID has been assigned, instead only 1450 * add the macaddr to that VLAN. 1451 * 1452 * Returns last filter added on success, else NULL 1453 **/ 1454 struct i40e_mac_filter *i40e_add_mac_filter(struct i40e_vsi *vsi, 1455 const u8 *macaddr) 1456 { 1457 struct i40e_mac_filter *f, *add = NULL; 1458 struct hlist_node *h; 1459 int bkt; 1460 1461 if (vsi->info.pvid) 1462 return i40e_add_filter(vsi, macaddr, 1463 le16_to_cpu(vsi->info.pvid)); 1464 1465 if (!i40e_is_vsi_in_vlan(vsi)) 1466 return i40e_add_filter(vsi, macaddr, I40E_VLAN_ANY); 1467 1468 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) { 1469 if (f->state == I40E_FILTER_REMOVE) 1470 continue; 1471 add = i40e_add_filter(vsi, macaddr, f->vlan); 1472 if (!add) 1473 return NULL; 1474 } 1475 1476 return add; 1477 } 1478 1479 /** 1480 * i40e_del_mac_filter - Remove a MAC filter from all VLANs 1481 * @vsi: the VSI to be searched 1482 * @macaddr: the mac address to be removed 1483 * 1484 * Removes a given MAC address from a VSI regardless of what VLAN it has been 1485 * associated with. 1486 * 1487 * Returns 0 for success, or error 1488 **/ 1489 int i40e_del_mac_filter(struct i40e_vsi *vsi, const u8 *macaddr) 1490 { 1491 struct i40e_mac_filter *f; 1492 struct hlist_node *h; 1493 bool found = false; 1494 int bkt; 1495 1496 WARN(!spin_is_locked(&vsi->mac_filter_hash_lock), 1497 "Missing mac_filter_hash_lock\n"); 1498 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) { 1499 if (ether_addr_equal(macaddr, f->macaddr)) { 1500 __i40e_del_filter(vsi, f); 1501 found = true; 1502 } 1503 } 1504 1505 if (found) 1506 return 0; 1507 else 1508 return -ENOENT; 1509 } 1510 1511 /** 1512 * i40e_set_mac - NDO callback to set mac address 1513 * @netdev: network interface device structure 1514 * @p: pointer to an address structure 1515 * 1516 * Returns 0 on success, negative on failure 1517 **/ 1518 static int i40e_set_mac(struct net_device *netdev, void *p) 1519 { 1520 struct i40e_netdev_priv *np = netdev_priv(netdev); 1521 struct i40e_vsi *vsi = np->vsi; 1522 struct i40e_pf *pf = vsi->back; 1523 struct i40e_hw *hw = &pf->hw; 1524 struct sockaddr *addr = p; 1525 1526 if (!is_valid_ether_addr(addr->sa_data)) 1527 return -EADDRNOTAVAIL; 1528 1529 if (ether_addr_equal(netdev->dev_addr, addr->sa_data)) { 1530 netdev_info(netdev, "already using mac address %pM\n", 1531 addr->sa_data); 1532 return 0; 1533 } 1534 1535 if (test_bit(__I40E_VSI_DOWN, vsi->back->state) || 1536 test_bit(__I40E_RESET_RECOVERY_PENDING, vsi->back->state)) 1537 return -EADDRNOTAVAIL; 1538 1539 if (ether_addr_equal(hw->mac.addr, addr->sa_data)) 1540 netdev_info(netdev, "returning to hw mac address %pM\n", 1541 hw->mac.addr); 1542 else 1543 netdev_info(netdev, "set new mac address %pM\n", addr->sa_data); 1544 1545 /* Copy the address first, so that we avoid a possible race with 1546 * .set_rx_mode(). If we copy after changing the address in the filter 1547 * list, we might open ourselves to a narrow race window where 1548 * .set_rx_mode could delete our dev_addr filter and prevent traffic 1549 * from passing. 1550 */ 1551 ether_addr_copy(netdev->dev_addr, addr->sa_data); 1552 1553 spin_lock_bh(&vsi->mac_filter_hash_lock); 1554 i40e_del_mac_filter(vsi, netdev->dev_addr); 1555 i40e_add_mac_filter(vsi, addr->sa_data); 1556 spin_unlock_bh(&vsi->mac_filter_hash_lock); 1557 if (vsi->type == I40E_VSI_MAIN) { 1558 i40e_status ret; 1559 1560 ret = i40e_aq_mac_address_write(&vsi->back->hw, 1561 I40E_AQC_WRITE_TYPE_LAA_WOL, 1562 addr->sa_data, NULL); 1563 if (ret) 1564 netdev_info(netdev, "Ignoring error from firmware on LAA update, status %s, AQ ret %s\n", 1565 i40e_stat_str(hw, ret), 1566 i40e_aq_str(hw, hw->aq.asq_last_status)); 1567 } 1568 1569 /* schedule our worker thread which will take care of 1570 * applying the new filter changes 1571 */ 1572 i40e_service_event_schedule(vsi->back); 1573 return 0; 1574 } 1575 1576 /** 1577 * i40e_config_rss_aq - Prepare for RSS using AQ commands 1578 * @vsi: vsi structure 1579 * @seed: RSS hash seed 1580 **/ 1581 static int i40e_config_rss_aq(struct i40e_vsi *vsi, const u8 *seed, 1582 u8 *lut, u16 lut_size) 1583 { 1584 struct i40e_pf *pf = vsi->back; 1585 struct i40e_hw *hw = &pf->hw; 1586 int ret = 0; 1587 1588 if (seed) { 1589 struct i40e_aqc_get_set_rss_key_data *seed_dw = 1590 (struct i40e_aqc_get_set_rss_key_data *)seed; 1591 ret = i40e_aq_set_rss_key(hw, vsi->id, seed_dw); 1592 if (ret) { 1593 dev_info(&pf->pdev->dev, 1594 "Cannot set RSS key, err %s aq_err %s\n", 1595 i40e_stat_str(hw, ret), 1596 i40e_aq_str(hw, hw->aq.asq_last_status)); 1597 return ret; 1598 } 1599 } 1600 if (lut) { 1601 bool pf_lut = vsi->type == I40E_VSI_MAIN ? true : false; 1602 1603 ret = i40e_aq_set_rss_lut(hw, vsi->id, pf_lut, lut, lut_size); 1604 if (ret) { 1605 dev_info(&pf->pdev->dev, 1606 "Cannot set RSS lut, err %s aq_err %s\n", 1607 i40e_stat_str(hw, ret), 1608 i40e_aq_str(hw, hw->aq.asq_last_status)); 1609 return ret; 1610 } 1611 } 1612 return ret; 1613 } 1614 1615 /** 1616 * i40e_vsi_config_rss - Prepare for VSI(VMDq) RSS if used 1617 * @vsi: VSI structure 1618 **/ 1619 static int i40e_vsi_config_rss(struct i40e_vsi *vsi) 1620 { 1621 struct i40e_pf *pf = vsi->back; 1622 u8 seed[I40E_HKEY_ARRAY_SIZE]; 1623 u8 *lut; 1624 int ret; 1625 1626 if (!(pf->hw_features & I40E_HW_RSS_AQ_CAPABLE)) 1627 return 0; 1628 if (!vsi->rss_size) 1629 vsi->rss_size = min_t(int, pf->alloc_rss_size, 1630 vsi->num_queue_pairs); 1631 if (!vsi->rss_size) 1632 return -EINVAL; 1633 lut = kzalloc(vsi->rss_table_size, GFP_KERNEL); 1634 if (!lut) 1635 return -ENOMEM; 1636 1637 /* Use the user configured hash keys and lookup table if there is one, 1638 * otherwise use default 1639 */ 1640 if (vsi->rss_lut_user) 1641 memcpy(lut, vsi->rss_lut_user, vsi->rss_table_size); 1642 else 1643 i40e_fill_rss_lut(pf, lut, vsi->rss_table_size, vsi->rss_size); 1644 if (vsi->rss_hkey_user) 1645 memcpy(seed, vsi->rss_hkey_user, I40E_HKEY_ARRAY_SIZE); 1646 else 1647 netdev_rss_key_fill((void *)seed, I40E_HKEY_ARRAY_SIZE); 1648 ret = i40e_config_rss_aq(vsi, seed, lut, vsi->rss_table_size); 1649 kfree(lut); 1650 return ret; 1651 } 1652 1653 /** 1654 * i40e_vsi_setup_queue_map_mqprio - Prepares mqprio based tc_config 1655 * @vsi: the VSI being configured, 1656 * @ctxt: VSI context structure 1657 * @enabled_tc: number of traffic classes to enable 1658 * 1659 * Prepares VSI tc_config to have queue configurations based on MQPRIO options. 1660 **/ 1661 static int i40e_vsi_setup_queue_map_mqprio(struct i40e_vsi *vsi, 1662 struct i40e_vsi_context *ctxt, 1663 u8 enabled_tc) 1664 { 1665 u16 qcount = 0, max_qcount, qmap, sections = 0; 1666 int i, override_q, pow, num_qps, ret; 1667 u8 netdev_tc = 0, offset = 0; 1668 1669 if (vsi->type != I40E_VSI_MAIN) 1670 return -EINVAL; 1671 sections = I40E_AQ_VSI_PROP_QUEUE_MAP_VALID; 1672 sections |= I40E_AQ_VSI_PROP_SCHED_VALID; 1673 vsi->tc_config.numtc = vsi->mqprio_qopt.qopt.num_tc; 1674 vsi->tc_config.enabled_tc = enabled_tc ? enabled_tc : 1; 1675 num_qps = vsi->mqprio_qopt.qopt.count[0]; 1676 1677 /* find the next higher power-of-2 of num queue pairs */ 1678 pow = ilog2(num_qps); 1679 if (!is_power_of_2(num_qps)) 1680 pow++; 1681 qmap = (offset << I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT) | 1682 (pow << I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT); 1683 1684 /* Setup queue offset/count for all TCs for given VSI */ 1685 max_qcount = vsi->mqprio_qopt.qopt.count[0]; 1686 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 1687 /* See if the given TC is enabled for the given VSI */ 1688 if (vsi->tc_config.enabled_tc & BIT(i)) { 1689 offset = vsi->mqprio_qopt.qopt.offset[i]; 1690 qcount = vsi->mqprio_qopt.qopt.count[i]; 1691 if (qcount > max_qcount) 1692 max_qcount = qcount; 1693 vsi->tc_config.tc_info[i].qoffset = offset; 1694 vsi->tc_config.tc_info[i].qcount = qcount; 1695 vsi->tc_config.tc_info[i].netdev_tc = netdev_tc++; 1696 } else { 1697 /* TC is not enabled so set the offset to 1698 * default queue and allocate one queue 1699 * for the given TC. 1700 */ 1701 vsi->tc_config.tc_info[i].qoffset = 0; 1702 vsi->tc_config.tc_info[i].qcount = 1; 1703 vsi->tc_config.tc_info[i].netdev_tc = 0; 1704 } 1705 } 1706 1707 /* Set actual Tx/Rx queue pairs */ 1708 vsi->num_queue_pairs = offset + qcount; 1709 1710 /* Setup queue TC[0].qmap for given VSI context */ 1711 ctxt->info.tc_mapping[0] = cpu_to_le16(qmap); 1712 ctxt->info.mapping_flags |= cpu_to_le16(I40E_AQ_VSI_QUE_MAP_CONTIG); 1713 ctxt->info.queue_mapping[0] = cpu_to_le16(vsi->base_queue); 1714 ctxt->info.valid_sections |= cpu_to_le16(sections); 1715 1716 /* Reconfigure RSS for main VSI with max queue count */ 1717 vsi->rss_size = max_qcount; 1718 ret = i40e_vsi_config_rss(vsi); 1719 if (ret) { 1720 dev_info(&vsi->back->pdev->dev, 1721 "Failed to reconfig rss for num_queues (%u)\n", 1722 max_qcount); 1723 return ret; 1724 } 1725 vsi->reconfig_rss = true; 1726 dev_dbg(&vsi->back->pdev->dev, 1727 "Reconfigured rss with num_queues (%u)\n", max_qcount); 1728 1729 /* Find queue count available for channel VSIs and starting offset 1730 * for channel VSIs 1731 */ 1732 override_q = vsi->mqprio_qopt.qopt.count[0]; 1733 if (override_q && override_q < vsi->num_queue_pairs) { 1734 vsi->cnt_q_avail = vsi->num_queue_pairs - override_q; 1735 vsi->next_base_queue = override_q; 1736 } 1737 return 0; 1738 } 1739 1740 /** 1741 * i40e_vsi_setup_queue_map - Setup a VSI queue map based on enabled_tc 1742 * @vsi: the VSI being setup 1743 * @ctxt: VSI context structure 1744 * @enabled_tc: Enabled TCs bitmap 1745 * @is_add: True if called before Add VSI 1746 * 1747 * Setup VSI queue mapping for enabled traffic classes. 1748 **/ 1749 static void i40e_vsi_setup_queue_map(struct i40e_vsi *vsi, 1750 struct i40e_vsi_context *ctxt, 1751 u8 enabled_tc, 1752 bool is_add) 1753 { 1754 struct i40e_pf *pf = vsi->back; 1755 u16 sections = 0; 1756 u8 netdev_tc = 0; 1757 u16 numtc = 1; 1758 u16 qcount; 1759 u8 offset; 1760 u16 qmap; 1761 int i; 1762 u16 num_tc_qps = 0; 1763 1764 sections = I40E_AQ_VSI_PROP_QUEUE_MAP_VALID; 1765 offset = 0; 1766 1767 /* Number of queues per enabled TC */ 1768 num_tc_qps = vsi->alloc_queue_pairs; 1769 if (enabled_tc && (vsi->back->flags & I40E_FLAG_DCB_ENABLED)) { 1770 /* Find numtc from enabled TC bitmap */ 1771 for (i = 0, numtc = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 1772 if (enabled_tc & BIT(i)) /* TC is enabled */ 1773 numtc++; 1774 } 1775 if (!numtc) { 1776 dev_warn(&pf->pdev->dev, "DCB is enabled but no TC enabled, forcing TC0\n"); 1777 numtc = 1; 1778 } 1779 num_tc_qps = num_tc_qps / numtc; 1780 num_tc_qps = min_t(int, num_tc_qps, 1781 i40e_pf_get_max_q_per_tc(pf)); 1782 } 1783 1784 vsi->tc_config.numtc = numtc; 1785 vsi->tc_config.enabled_tc = enabled_tc ? enabled_tc : 1; 1786 1787 /* Do not allow use more TC queue pairs than MSI-X vectors exist */ 1788 if (pf->flags & I40E_FLAG_MSIX_ENABLED) 1789 num_tc_qps = min_t(int, num_tc_qps, pf->num_lan_msix); 1790 1791 /* Setup queue offset/count for all TCs for given VSI */ 1792 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 1793 /* See if the given TC is enabled for the given VSI */ 1794 if (vsi->tc_config.enabled_tc & BIT(i)) { 1795 /* TC is enabled */ 1796 int pow, num_qps; 1797 1798 switch (vsi->type) { 1799 case I40E_VSI_MAIN: 1800 if (!(pf->flags & (I40E_FLAG_FD_SB_ENABLED | 1801 I40E_FLAG_FD_ATR_ENABLED)) || 1802 vsi->tc_config.enabled_tc != 1) { 1803 qcount = min_t(int, pf->alloc_rss_size, 1804 num_tc_qps); 1805 break; 1806 } 1807 /* fall through */ 1808 case I40E_VSI_FDIR: 1809 case I40E_VSI_SRIOV: 1810 case I40E_VSI_VMDQ2: 1811 default: 1812 qcount = num_tc_qps; 1813 WARN_ON(i != 0); 1814 break; 1815 } 1816 vsi->tc_config.tc_info[i].qoffset = offset; 1817 vsi->tc_config.tc_info[i].qcount = qcount; 1818 1819 /* find the next higher power-of-2 of num queue pairs */ 1820 num_qps = qcount; 1821 pow = 0; 1822 while (num_qps && (BIT_ULL(pow) < qcount)) { 1823 pow++; 1824 num_qps >>= 1; 1825 } 1826 1827 vsi->tc_config.tc_info[i].netdev_tc = netdev_tc++; 1828 qmap = 1829 (offset << I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT) | 1830 (pow << I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT); 1831 1832 offset += qcount; 1833 } else { 1834 /* TC is not enabled so set the offset to 1835 * default queue and allocate one queue 1836 * for the given TC. 1837 */ 1838 vsi->tc_config.tc_info[i].qoffset = 0; 1839 vsi->tc_config.tc_info[i].qcount = 1; 1840 vsi->tc_config.tc_info[i].netdev_tc = 0; 1841 1842 qmap = 0; 1843 } 1844 ctxt->info.tc_mapping[i] = cpu_to_le16(qmap); 1845 } 1846 1847 /* Set actual Tx/Rx queue pairs */ 1848 vsi->num_queue_pairs = offset; 1849 if ((vsi->type == I40E_VSI_MAIN) && (numtc == 1)) { 1850 if (vsi->req_queue_pairs > 0) 1851 vsi->num_queue_pairs = vsi->req_queue_pairs; 1852 else if (pf->flags & I40E_FLAG_MSIX_ENABLED) 1853 vsi->num_queue_pairs = pf->num_lan_msix; 1854 } 1855 1856 /* Scheduler section valid can only be set for ADD VSI */ 1857 if (is_add) { 1858 sections |= I40E_AQ_VSI_PROP_SCHED_VALID; 1859 1860 ctxt->info.up_enable_bits = enabled_tc; 1861 } 1862 if (vsi->type == I40E_VSI_SRIOV) { 1863 ctxt->info.mapping_flags |= 1864 cpu_to_le16(I40E_AQ_VSI_QUE_MAP_NONCONTIG); 1865 for (i = 0; i < vsi->num_queue_pairs; i++) 1866 ctxt->info.queue_mapping[i] = 1867 cpu_to_le16(vsi->base_queue + i); 1868 } else { 1869 ctxt->info.mapping_flags |= 1870 cpu_to_le16(I40E_AQ_VSI_QUE_MAP_CONTIG); 1871 ctxt->info.queue_mapping[0] = cpu_to_le16(vsi->base_queue); 1872 } 1873 ctxt->info.valid_sections |= cpu_to_le16(sections); 1874 } 1875 1876 /** 1877 * i40e_addr_sync - Callback for dev_(mc|uc)_sync to add address 1878 * @netdev: the netdevice 1879 * @addr: address to add 1880 * 1881 * Called by __dev_(mc|uc)_sync when an address needs to be added. We call 1882 * __dev_(uc|mc)_sync from .set_rx_mode and guarantee to hold the hash lock. 1883 */ 1884 static int i40e_addr_sync(struct net_device *netdev, const u8 *addr) 1885 { 1886 struct i40e_netdev_priv *np = netdev_priv(netdev); 1887 struct i40e_vsi *vsi = np->vsi; 1888 1889 if (i40e_add_mac_filter(vsi, addr)) 1890 return 0; 1891 else 1892 return -ENOMEM; 1893 } 1894 1895 /** 1896 * i40e_addr_unsync - Callback for dev_(mc|uc)_sync to remove address 1897 * @netdev: the netdevice 1898 * @addr: address to add 1899 * 1900 * Called by __dev_(mc|uc)_sync when an address needs to be removed. We call 1901 * __dev_(uc|mc)_sync from .set_rx_mode and guarantee to hold the hash lock. 1902 */ 1903 static int i40e_addr_unsync(struct net_device *netdev, const u8 *addr) 1904 { 1905 struct i40e_netdev_priv *np = netdev_priv(netdev); 1906 struct i40e_vsi *vsi = np->vsi; 1907 1908 /* Under some circumstances, we might receive a request to delete 1909 * our own device address from our uc list. Because we store the 1910 * device address in the VSI's MAC/VLAN filter list, we need to ignore 1911 * such requests and not delete our device address from this list. 1912 */ 1913 if (ether_addr_equal(addr, netdev->dev_addr)) 1914 return 0; 1915 1916 i40e_del_mac_filter(vsi, addr); 1917 1918 return 0; 1919 } 1920 1921 /** 1922 * i40e_set_rx_mode - NDO callback to set the netdev filters 1923 * @netdev: network interface device structure 1924 **/ 1925 static void i40e_set_rx_mode(struct net_device *netdev) 1926 { 1927 struct i40e_netdev_priv *np = netdev_priv(netdev); 1928 struct i40e_vsi *vsi = np->vsi; 1929 1930 spin_lock_bh(&vsi->mac_filter_hash_lock); 1931 1932 __dev_uc_sync(netdev, i40e_addr_sync, i40e_addr_unsync); 1933 __dev_mc_sync(netdev, i40e_addr_sync, i40e_addr_unsync); 1934 1935 spin_unlock_bh(&vsi->mac_filter_hash_lock); 1936 1937 /* check for other flag changes */ 1938 if (vsi->current_netdev_flags != vsi->netdev->flags) { 1939 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED; 1940 set_bit(__I40E_MACVLAN_SYNC_PENDING, vsi->back->state); 1941 } 1942 } 1943 1944 /** 1945 * i40e_undo_del_filter_entries - Undo the changes made to MAC filter entries 1946 * @vsi: Pointer to VSI struct 1947 * @from: Pointer to list which contains MAC filter entries - changes to 1948 * those entries needs to be undone. 1949 * 1950 * MAC filter entries from this list were slated for deletion. 1951 **/ 1952 static void i40e_undo_del_filter_entries(struct i40e_vsi *vsi, 1953 struct hlist_head *from) 1954 { 1955 struct i40e_mac_filter *f; 1956 struct hlist_node *h; 1957 1958 hlist_for_each_entry_safe(f, h, from, hlist) { 1959 u64 key = i40e_addr_to_hkey(f->macaddr); 1960 1961 /* Move the element back into MAC filter list*/ 1962 hlist_del(&f->hlist); 1963 hash_add(vsi->mac_filter_hash, &f->hlist, key); 1964 } 1965 } 1966 1967 /** 1968 * i40e_undo_add_filter_entries - Undo the changes made to MAC filter entries 1969 * @vsi: Pointer to vsi struct 1970 * @from: Pointer to list which contains MAC filter entries - changes to 1971 * those entries needs to be undone. 1972 * 1973 * MAC filter entries from this list were slated for addition. 1974 **/ 1975 static void i40e_undo_add_filter_entries(struct i40e_vsi *vsi, 1976 struct hlist_head *from) 1977 { 1978 struct i40e_new_mac_filter *new; 1979 struct hlist_node *h; 1980 1981 hlist_for_each_entry_safe(new, h, from, hlist) { 1982 /* We can simply free the wrapper structure */ 1983 hlist_del(&new->hlist); 1984 kfree(new); 1985 } 1986 } 1987 1988 /** 1989 * i40e_next_entry - Get the next non-broadcast filter from a list 1990 * @next: pointer to filter in list 1991 * 1992 * Returns the next non-broadcast filter in the list. Required so that we 1993 * ignore broadcast filters within the list, since these are not handled via 1994 * the normal firmware update path. 1995 */ 1996 static 1997 struct i40e_new_mac_filter *i40e_next_filter(struct i40e_new_mac_filter *next) 1998 { 1999 hlist_for_each_entry_continue(next, hlist) { 2000 if (!is_broadcast_ether_addr(next->f->macaddr)) 2001 return next; 2002 } 2003 2004 return NULL; 2005 } 2006 2007 /** 2008 * i40e_update_filter_state - Update filter state based on return data 2009 * from firmware 2010 * @count: Number of filters added 2011 * @add_list: return data from fw 2012 * @add_head: pointer to first filter in current batch 2013 * 2014 * MAC filter entries from list were slated to be added to device. Returns 2015 * number of successful filters. Note that 0 does NOT mean success! 2016 **/ 2017 static int 2018 i40e_update_filter_state(int count, 2019 struct i40e_aqc_add_macvlan_element_data *add_list, 2020 struct i40e_new_mac_filter *add_head) 2021 { 2022 int retval = 0; 2023 int i; 2024 2025 for (i = 0; i < count; i++) { 2026 /* Always check status of each filter. We don't need to check 2027 * the firmware return status because we pre-set the filter 2028 * status to I40E_AQC_MM_ERR_NO_RES when sending the filter 2029 * request to the adminq. Thus, if it no longer matches then 2030 * we know the filter is active. 2031 */ 2032 if (add_list[i].match_method == I40E_AQC_MM_ERR_NO_RES) { 2033 add_head->state = I40E_FILTER_FAILED; 2034 } else { 2035 add_head->state = I40E_FILTER_ACTIVE; 2036 retval++; 2037 } 2038 2039 add_head = i40e_next_filter(add_head); 2040 if (!add_head) 2041 break; 2042 } 2043 2044 return retval; 2045 } 2046 2047 /** 2048 * i40e_aqc_del_filters - Request firmware to delete a set of filters 2049 * @vsi: ptr to the VSI 2050 * @vsi_name: name to display in messages 2051 * @list: the list of filters to send to firmware 2052 * @num_del: the number of filters to delete 2053 * @retval: Set to -EIO on failure to delete 2054 * 2055 * Send a request to firmware via AdminQ to delete a set of filters. Uses 2056 * *retval instead of a return value so that success does not force ret_val to 2057 * be set to 0. This ensures that a sequence of calls to this function 2058 * preserve the previous value of *retval on successful delete. 2059 */ 2060 static 2061 void i40e_aqc_del_filters(struct i40e_vsi *vsi, const char *vsi_name, 2062 struct i40e_aqc_remove_macvlan_element_data *list, 2063 int num_del, int *retval) 2064 { 2065 struct i40e_hw *hw = &vsi->back->hw; 2066 i40e_status aq_ret; 2067 int aq_err; 2068 2069 aq_ret = i40e_aq_remove_macvlan(hw, vsi->seid, list, num_del, NULL); 2070 aq_err = hw->aq.asq_last_status; 2071 2072 /* Explicitly ignore and do not report when firmware returns ENOENT */ 2073 if (aq_ret && !(aq_err == I40E_AQ_RC_ENOENT)) { 2074 *retval = -EIO; 2075 dev_info(&vsi->back->pdev->dev, 2076 "ignoring delete macvlan error on %s, err %s, aq_err %s\n", 2077 vsi_name, i40e_stat_str(hw, aq_ret), 2078 i40e_aq_str(hw, aq_err)); 2079 } 2080 } 2081 2082 /** 2083 * i40e_aqc_add_filters - Request firmware to add a set of filters 2084 * @vsi: ptr to the VSI 2085 * @vsi_name: name to display in messages 2086 * @list: the list of filters to send to firmware 2087 * @add_head: Position in the add hlist 2088 * @num_add: the number of filters to add 2089 * 2090 * Send a request to firmware via AdminQ to add a chunk of filters. Will set 2091 * __I40E_VSI_OVERFLOW_PROMISC bit in vsi->state if the firmware has run out of 2092 * space for more filters. 2093 */ 2094 static 2095 void i40e_aqc_add_filters(struct i40e_vsi *vsi, const char *vsi_name, 2096 struct i40e_aqc_add_macvlan_element_data *list, 2097 struct i40e_new_mac_filter *add_head, 2098 int num_add) 2099 { 2100 struct i40e_hw *hw = &vsi->back->hw; 2101 int aq_err, fcnt; 2102 2103 i40e_aq_add_macvlan(hw, vsi->seid, list, num_add, NULL); 2104 aq_err = hw->aq.asq_last_status; 2105 fcnt = i40e_update_filter_state(num_add, list, add_head); 2106 2107 if (fcnt != num_add) { 2108 set_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state); 2109 dev_warn(&vsi->back->pdev->dev, 2110 "Error %s adding RX filters on %s, promiscuous mode forced on\n", 2111 i40e_aq_str(hw, aq_err), 2112 vsi_name); 2113 } 2114 } 2115 2116 /** 2117 * i40e_aqc_broadcast_filter - Set promiscuous broadcast flags 2118 * @vsi: pointer to the VSI 2119 * @vsi_name: the VSI name 2120 * @f: filter data 2121 * 2122 * This function sets or clears the promiscuous broadcast flags for VLAN 2123 * filters in order to properly receive broadcast frames. Assumes that only 2124 * broadcast filters are passed. 2125 * 2126 * Returns status indicating success or failure; 2127 **/ 2128 static i40e_status 2129 i40e_aqc_broadcast_filter(struct i40e_vsi *vsi, const char *vsi_name, 2130 struct i40e_mac_filter *f) 2131 { 2132 bool enable = f->state == I40E_FILTER_NEW; 2133 struct i40e_hw *hw = &vsi->back->hw; 2134 i40e_status aq_ret; 2135 2136 if (f->vlan == I40E_VLAN_ANY) { 2137 aq_ret = i40e_aq_set_vsi_broadcast(hw, 2138 vsi->seid, 2139 enable, 2140 NULL); 2141 } else { 2142 aq_ret = i40e_aq_set_vsi_bc_promisc_on_vlan(hw, 2143 vsi->seid, 2144 enable, 2145 f->vlan, 2146 NULL); 2147 } 2148 2149 if (aq_ret) { 2150 set_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state); 2151 dev_warn(&vsi->back->pdev->dev, 2152 "Error %s, forcing overflow promiscuous on %s\n", 2153 i40e_aq_str(hw, hw->aq.asq_last_status), 2154 vsi_name); 2155 } 2156 2157 return aq_ret; 2158 } 2159 2160 /** 2161 * i40e_set_promiscuous - set promiscuous mode 2162 * @pf: board private structure 2163 * @promisc: promisc on or off 2164 * 2165 * There are different ways of setting promiscuous mode on a PF depending on 2166 * what state/environment we're in. This identifies and sets it appropriately. 2167 * Returns 0 on success. 2168 **/ 2169 static int i40e_set_promiscuous(struct i40e_pf *pf, bool promisc) 2170 { 2171 struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi]; 2172 struct i40e_hw *hw = &pf->hw; 2173 i40e_status aq_ret; 2174 2175 if (vsi->type == I40E_VSI_MAIN && 2176 pf->lan_veb != I40E_NO_VEB && 2177 !(pf->flags & I40E_FLAG_MFP_ENABLED)) { 2178 /* set defport ON for Main VSI instead of true promisc 2179 * this way we will get all unicast/multicast and VLAN 2180 * promisc behavior but will not get VF or VMDq traffic 2181 * replicated on the Main VSI. 2182 */ 2183 if (promisc) 2184 aq_ret = i40e_aq_set_default_vsi(hw, 2185 vsi->seid, 2186 NULL); 2187 else 2188 aq_ret = i40e_aq_clear_default_vsi(hw, 2189 vsi->seid, 2190 NULL); 2191 if (aq_ret) { 2192 dev_info(&pf->pdev->dev, 2193 "Set default VSI failed, err %s, aq_err %s\n", 2194 i40e_stat_str(hw, aq_ret), 2195 i40e_aq_str(hw, hw->aq.asq_last_status)); 2196 } 2197 } else { 2198 aq_ret = i40e_aq_set_vsi_unicast_promiscuous( 2199 hw, 2200 vsi->seid, 2201 promisc, NULL, 2202 true); 2203 if (aq_ret) { 2204 dev_info(&pf->pdev->dev, 2205 "set unicast promisc failed, err %s, aq_err %s\n", 2206 i40e_stat_str(hw, aq_ret), 2207 i40e_aq_str(hw, hw->aq.asq_last_status)); 2208 } 2209 aq_ret = i40e_aq_set_vsi_multicast_promiscuous( 2210 hw, 2211 vsi->seid, 2212 promisc, NULL); 2213 if (aq_ret) { 2214 dev_info(&pf->pdev->dev, 2215 "set multicast promisc failed, err %s, aq_err %s\n", 2216 i40e_stat_str(hw, aq_ret), 2217 i40e_aq_str(hw, hw->aq.asq_last_status)); 2218 } 2219 } 2220 2221 if (!aq_ret) 2222 pf->cur_promisc = promisc; 2223 2224 return aq_ret; 2225 } 2226 2227 /** 2228 * i40e_sync_vsi_filters - Update the VSI filter list to the HW 2229 * @vsi: ptr to the VSI 2230 * 2231 * Push any outstanding VSI filter changes through the AdminQ. 2232 * 2233 * Returns 0 or error value 2234 **/ 2235 int i40e_sync_vsi_filters(struct i40e_vsi *vsi) 2236 { 2237 struct hlist_head tmp_add_list, tmp_del_list; 2238 struct i40e_mac_filter *f; 2239 struct i40e_new_mac_filter *new, *add_head = NULL; 2240 struct i40e_hw *hw = &vsi->back->hw; 2241 bool old_overflow, new_overflow; 2242 unsigned int failed_filters = 0; 2243 unsigned int vlan_filters = 0; 2244 char vsi_name[16] = "PF"; 2245 int filter_list_len = 0; 2246 i40e_status aq_ret = 0; 2247 u32 changed_flags = 0; 2248 struct hlist_node *h; 2249 struct i40e_pf *pf; 2250 int num_add = 0; 2251 int num_del = 0; 2252 int retval = 0; 2253 u16 cmd_flags; 2254 int list_size; 2255 int bkt; 2256 2257 /* empty array typed pointers, kcalloc later */ 2258 struct i40e_aqc_add_macvlan_element_data *add_list; 2259 struct i40e_aqc_remove_macvlan_element_data *del_list; 2260 2261 while (test_and_set_bit(__I40E_VSI_SYNCING_FILTERS, vsi->state)) 2262 usleep_range(1000, 2000); 2263 pf = vsi->back; 2264 2265 old_overflow = test_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state); 2266 2267 if (vsi->netdev) { 2268 changed_flags = vsi->current_netdev_flags ^ vsi->netdev->flags; 2269 vsi->current_netdev_flags = vsi->netdev->flags; 2270 } 2271 2272 INIT_HLIST_HEAD(&tmp_add_list); 2273 INIT_HLIST_HEAD(&tmp_del_list); 2274 2275 if (vsi->type == I40E_VSI_SRIOV) 2276 snprintf(vsi_name, sizeof(vsi_name) - 1, "VF %d", vsi->vf_id); 2277 else if (vsi->type != I40E_VSI_MAIN) 2278 snprintf(vsi_name, sizeof(vsi_name) - 1, "vsi %d", vsi->seid); 2279 2280 if (vsi->flags & I40E_VSI_FLAG_FILTER_CHANGED) { 2281 vsi->flags &= ~I40E_VSI_FLAG_FILTER_CHANGED; 2282 2283 spin_lock_bh(&vsi->mac_filter_hash_lock); 2284 /* Create a list of filters to delete. */ 2285 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) { 2286 if (f->state == I40E_FILTER_REMOVE) { 2287 /* Move the element into temporary del_list */ 2288 hash_del(&f->hlist); 2289 hlist_add_head(&f->hlist, &tmp_del_list); 2290 2291 /* Avoid counting removed filters */ 2292 continue; 2293 } 2294 if (f->state == I40E_FILTER_NEW) { 2295 /* Create a temporary i40e_new_mac_filter */ 2296 new = kzalloc(sizeof(*new), GFP_ATOMIC); 2297 if (!new) 2298 goto err_no_memory_locked; 2299 2300 /* Store pointer to the real filter */ 2301 new->f = f; 2302 new->state = f->state; 2303 2304 /* Add it to the hash list */ 2305 hlist_add_head(&new->hlist, &tmp_add_list); 2306 } 2307 2308 /* Count the number of active (current and new) VLAN 2309 * filters we have now. Does not count filters which 2310 * are marked for deletion. 2311 */ 2312 if (f->vlan > 0) 2313 vlan_filters++; 2314 } 2315 2316 retval = i40e_correct_mac_vlan_filters(vsi, 2317 &tmp_add_list, 2318 &tmp_del_list, 2319 vlan_filters); 2320 if (retval) 2321 goto err_no_memory_locked; 2322 2323 spin_unlock_bh(&vsi->mac_filter_hash_lock); 2324 } 2325 2326 /* Now process 'del_list' outside the lock */ 2327 if (!hlist_empty(&tmp_del_list)) { 2328 filter_list_len = hw->aq.asq_buf_size / 2329 sizeof(struct i40e_aqc_remove_macvlan_element_data); 2330 list_size = filter_list_len * 2331 sizeof(struct i40e_aqc_remove_macvlan_element_data); 2332 del_list = kzalloc(list_size, GFP_ATOMIC); 2333 if (!del_list) 2334 goto err_no_memory; 2335 2336 hlist_for_each_entry_safe(f, h, &tmp_del_list, hlist) { 2337 cmd_flags = 0; 2338 2339 /* handle broadcast filters by updating the broadcast 2340 * promiscuous flag and release filter list. 2341 */ 2342 if (is_broadcast_ether_addr(f->macaddr)) { 2343 i40e_aqc_broadcast_filter(vsi, vsi_name, f); 2344 2345 hlist_del(&f->hlist); 2346 kfree(f); 2347 continue; 2348 } 2349 2350 /* add to delete list */ 2351 ether_addr_copy(del_list[num_del].mac_addr, f->macaddr); 2352 if (f->vlan == I40E_VLAN_ANY) { 2353 del_list[num_del].vlan_tag = 0; 2354 cmd_flags |= I40E_AQC_MACVLAN_DEL_IGNORE_VLAN; 2355 } else { 2356 del_list[num_del].vlan_tag = 2357 cpu_to_le16((u16)(f->vlan)); 2358 } 2359 2360 cmd_flags |= I40E_AQC_MACVLAN_DEL_PERFECT_MATCH; 2361 del_list[num_del].flags = cmd_flags; 2362 num_del++; 2363 2364 /* flush a full buffer */ 2365 if (num_del == filter_list_len) { 2366 i40e_aqc_del_filters(vsi, vsi_name, del_list, 2367 num_del, &retval); 2368 memset(del_list, 0, list_size); 2369 num_del = 0; 2370 } 2371 /* Release memory for MAC filter entries which were 2372 * synced up with HW. 2373 */ 2374 hlist_del(&f->hlist); 2375 kfree(f); 2376 } 2377 2378 if (num_del) { 2379 i40e_aqc_del_filters(vsi, vsi_name, del_list, 2380 num_del, &retval); 2381 } 2382 2383 kfree(del_list); 2384 del_list = NULL; 2385 } 2386 2387 if (!hlist_empty(&tmp_add_list)) { 2388 /* Do all the adds now. */ 2389 filter_list_len = hw->aq.asq_buf_size / 2390 sizeof(struct i40e_aqc_add_macvlan_element_data); 2391 list_size = filter_list_len * 2392 sizeof(struct i40e_aqc_add_macvlan_element_data); 2393 add_list = kzalloc(list_size, GFP_ATOMIC); 2394 if (!add_list) 2395 goto err_no_memory; 2396 2397 num_add = 0; 2398 hlist_for_each_entry_safe(new, h, &tmp_add_list, hlist) { 2399 /* handle broadcast filters by updating the broadcast 2400 * promiscuous flag instead of adding a MAC filter. 2401 */ 2402 if (is_broadcast_ether_addr(new->f->macaddr)) { 2403 if (i40e_aqc_broadcast_filter(vsi, vsi_name, 2404 new->f)) 2405 new->state = I40E_FILTER_FAILED; 2406 else 2407 new->state = I40E_FILTER_ACTIVE; 2408 continue; 2409 } 2410 2411 /* add to add array */ 2412 if (num_add == 0) 2413 add_head = new; 2414 cmd_flags = 0; 2415 ether_addr_copy(add_list[num_add].mac_addr, 2416 new->f->macaddr); 2417 if (new->f->vlan == I40E_VLAN_ANY) { 2418 add_list[num_add].vlan_tag = 0; 2419 cmd_flags |= I40E_AQC_MACVLAN_ADD_IGNORE_VLAN; 2420 } else { 2421 add_list[num_add].vlan_tag = 2422 cpu_to_le16((u16)(new->f->vlan)); 2423 } 2424 add_list[num_add].queue_number = 0; 2425 /* set invalid match method for later detection */ 2426 add_list[num_add].match_method = I40E_AQC_MM_ERR_NO_RES; 2427 cmd_flags |= I40E_AQC_MACVLAN_ADD_PERFECT_MATCH; 2428 add_list[num_add].flags = cpu_to_le16(cmd_flags); 2429 num_add++; 2430 2431 /* flush a full buffer */ 2432 if (num_add == filter_list_len) { 2433 i40e_aqc_add_filters(vsi, vsi_name, add_list, 2434 add_head, num_add); 2435 memset(add_list, 0, list_size); 2436 num_add = 0; 2437 } 2438 } 2439 if (num_add) { 2440 i40e_aqc_add_filters(vsi, vsi_name, add_list, add_head, 2441 num_add); 2442 } 2443 /* Now move all of the filters from the temp add list back to 2444 * the VSI's list. 2445 */ 2446 spin_lock_bh(&vsi->mac_filter_hash_lock); 2447 hlist_for_each_entry_safe(new, h, &tmp_add_list, hlist) { 2448 /* Only update the state if we're still NEW */ 2449 if (new->f->state == I40E_FILTER_NEW) 2450 new->f->state = new->state; 2451 hlist_del(&new->hlist); 2452 kfree(new); 2453 } 2454 spin_unlock_bh(&vsi->mac_filter_hash_lock); 2455 kfree(add_list); 2456 add_list = NULL; 2457 } 2458 2459 /* Determine the number of active and failed filters. */ 2460 spin_lock_bh(&vsi->mac_filter_hash_lock); 2461 vsi->active_filters = 0; 2462 hash_for_each(vsi->mac_filter_hash, bkt, f, hlist) { 2463 if (f->state == I40E_FILTER_ACTIVE) 2464 vsi->active_filters++; 2465 else if (f->state == I40E_FILTER_FAILED) 2466 failed_filters++; 2467 } 2468 spin_unlock_bh(&vsi->mac_filter_hash_lock); 2469 2470 /* Check if we are able to exit overflow promiscuous mode. We can 2471 * safely exit if we didn't just enter, we no longer have any failed 2472 * filters, and we have reduced filters below the threshold value. 2473 */ 2474 if (old_overflow && !failed_filters && 2475 vsi->active_filters < vsi->promisc_threshold) { 2476 dev_info(&pf->pdev->dev, 2477 "filter logjam cleared on %s, leaving overflow promiscuous mode\n", 2478 vsi_name); 2479 clear_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state); 2480 vsi->promisc_threshold = 0; 2481 } 2482 2483 /* if the VF is not trusted do not do promisc */ 2484 if ((vsi->type == I40E_VSI_SRIOV) && !pf->vf[vsi->vf_id].trusted) { 2485 clear_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state); 2486 goto out; 2487 } 2488 2489 new_overflow = test_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state); 2490 2491 /* If we are entering overflow promiscuous, we need to calculate a new 2492 * threshold for when we are safe to exit 2493 */ 2494 if (!old_overflow && new_overflow) 2495 vsi->promisc_threshold = (vsi->active_filters * 3) / 4; 2496 2497 /* check for changes in promiscuous modes */ 2498 if (changed_flags & IFF_ALLMULTI) { 2499 bool cur_multipromisc; 2500 2501 cur_multipromisc = !!(vsi->current_netdev_flags & IFF_ALLMULTI); 2502 aq_ret = i40e_aq_set_vsi_multicast_promiscuous(&vsi->back->hw, 2503 vsi->seid, 2504 cur_multipromisc, 2505 NULL); 2506 if (aq_ret) { 2507 retval = i40e_aq_rc_to_posix(aq_ret, 2508 hw->aq.asq_last_status); 2509 dev_info(&pf->pdev->dev, 2510 "set multi promisc failed on %s, err %s aq_err %s\n", 2511 vsi_name, 2512 i40e_stat_str(hw, aq_ret), 2513 i40e_aq_str(hw, hw->aq.asq_last_status)); 2514 } 2515 } 2516 2517 if ((changed_flags & IFF_PROMISC) || old_overflow != new_overflow) { 2518 bool cur_promisc; 2519 2520 cur_promisc = (!!(vsi->current_netdev_flags & IFF_PROMISC) || 2521 new_overflow); 2522 aq_ret = i40e_set_promiscuous(pf, cur_promisc); 2523 if (aq_ret) { 2524 retval = i40e_aq_rc_to_posix(aq_ret, 2525 hw->aq.asq_last_status); 2526 dev_info(&pf->pdev->dev, 2527 "Setting promiscuous %s failed on %s, err %s aq_err %s\n", 2528 cur_promisc ? "on" : "off", 2529 vsi_name, 2530 i40e_stat_str(hw, aq_ret), 2531 i40e_aq_str(hw, hw->aq.asq_last_status)); 2532 } 2533 } 2534 out: 2535 /* if something went wrong then set the changed flag so we try again */ 2536 if (retval) 2537 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED; 2538 2539 clear_bit(__I40E_VSI_SYNCING_FILTERS, vsi->state); 2540 return retval; 2541 2542 err_no_memory: 2543 /* Restore elements on the temporary add and delete lists */ 2544 spin_lock_bh(&vsi->mac_filter_hash_lock); 2545 err_no_memory_locked: 2546 i40e_undo_del_filter_entries(vsi, &tmp_del_list); 2547 i40e_undo_add_filter_entries(vsi, &tmp_add_list); 2548 spin_unlock_bh(&vsi->mac_filter_hash_lock); 2549 2550 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED; 2551 clear_bit(__I40E_VSI_SYNCING_FILTERS, vsi->state); 2552 return -ENOMEM; 2553 } 2554 2555 /** 2556 * i40e_sync_filters_subtask - Sync the VSI filter list with HW 2557 * @pf: board private structure 2558 **/ 2559 static void i40e_sync_filters_subtask(struct i40e_pf *pf) 2560 { 2561 int v; 2562 2563 if (!pf) 2564 return; 2565 if (!test_and_clear_bit(__I40E_MACVLAN_SYNC_PENDING, pf->state)) 2566 return; 2567 2568 for (v = 0; v < pf->num_alloc_vsi; v++) { 2569 if (pf->vsi[v] && 2570 (pf->vsi[v]->flags & I40E_VSI_FLAG_FILTER_CHANGED)) { 2571 int ret = i40e_sync_vsi_filters(pf->vsi[v]); 2572 2573 if (ret) { 2574 /* come back and try again later */ 2575 set_bit(__I40E_MACVLAN_SYNC_PENDING, 2576 pf->state); 2577 break; 2578 } 2579 } 2580 } 2581 } 2582 2583 /** 2584 * i40e_max_xdp_frame_size - returns the maximum allowed frame size for XDP 2585 * @vsi: the vsi 2586 **/ 2587 static int i40e_max_xdp_frame_size(struct i40e_vsi *vsi) 2588 { 2589 if (PAGE_SIZE >= 8192 || (vsi->back->flags & I40E_FLAG_LEGACY_RX)) 2590 return I40E_RXBUFFER_2048; 2591 else 2592 return I40E_RXBUFFER_3072; 2593 } 2594 2595 /** 2596 * i40e_change_mtu - NDO callback to change the Maximum Transfer Unit 2597 * @netdev: network interface device structure 2598 * @new_mtu: new value for maximum frame size 2599 * 2600 * Returns 0 on success, negative on failure 2601 **/ 2602 static int i40e_change_mtu(struct net_device *netdev, int new_mtu) 2603 { 2604 struct i40e_netdev_priv *np = netdev_priv(netdev); 2605 struct i40e_vsi *vsi = np->vsi; 2606 struct i40e_pf *pf = vsi->back; 2607 2608 if (i40e_enabled_xdp_vsi(vsi)) { 2609 int frame_size = new_mtu + ETH_HLEN + ETH_FCS_LEN + VLAN_HLEN; 2610 2611 if (frame_size > i40e_max_xdp_frame_size(vsi)) 2612 return -EINVAL; 2613 } 2614 2615 netdev_info(netdev, "changing MTU from %d to %d\n", 2616 netdev->mtu, new_mtu); 2617 netdev->mtu = new_mtu; 2618 if (netif_running(netdev)) 2619 i40e_vsi_reinit_locked(vsi); 2620 set_bit(__I40E_CLIENT_SERVICE_REQUESTED, pf->state); 2621 set_bit(__I40E_CLIENT_L2_CHANGE, pf->state); 2622 return 0; 2623 } 2624 2625 /** 2626 * i40e_ioctl - Access the hwtstamp interface 2627 * @netdev: network interface device structure 2628 * @ifr: interface request data 2629 * @cmd: ioctl command 2630 **/ 2631 int i40e_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd) 2632 { 2633 struct i40e_netdev_priv *np = netdev_priv(netdev); 2634 struct i40e_pf *pf = np->vsi->back; 2635 2636 switch (cmd) { 2637 case SIOCGHWTSTAMP: 2638 return i40e_ptp_get_ts_config(pf, ifr); 2639 case SIOCSHWTSTAMP: 2640 return i40e_ptp_set_ts_config(pf, ifr); 2641 default: 2642 return -EOPNOTSUPP; 2643 } 2644 } 2645 2646 /** 2647 * i40e_vlan_stripping_enable - Turn on vlan stripping for the VSI 2648 * @vsi: the vsi being adjusted 2649 **/ 2650 void i40e_vlan_stripping_enable(struct i40e_vsi *vsi) 2651 { 2652 struct i40e_vsi_context ctxt; 2653 i40e_status ret; 2654 2655 if ((vsi->info.valid_sections & 2656 cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID)) && 2657 ((vsi->info.port_vlan_flags & I40E_AQ_VSI_PVLAN_MODE_MASK) == 0)) 2658 return; /* already enabled */ 2659 2660 vsi->info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID); 2661 vsi->info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_ALL | 2662 I40E_AQ_VSI_PVLAN_EMOD_STR_BOTH; 2663 2664 ctxt.seid = vsi->seid; 2665 ctxt.info = vsi->info; 2666 ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL); 2667 if (ret) { 2668 dev_info(&vsi->back->pdev->dev, 2669 "update vlan stripping failed, err %s aq_err %s\n", 2670 i40e_stat_str(&vsi->back->hw, ret), 2671 i40e_aq_str(&vsi->back->hw, 2672 vsi->back->hw.aq.asq_last_status)); 2673 } 2674 } 2675 2676 /** 2677 * i40e_vlan_stripping_disable - Turn off vlan stripping for the VSI 2678 * @vsi: the vsi being adjusted 2679 **/ 2680 void i40e_vlan_stripping_disable(struct i40e_vsi *vsi) 2681 { 2682 struct i40e_vsi_context ctxt; 2683 i40e_status ret; 2684 2685 if ((vsi->info.valid_sections & 2686 cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID)) && 2687 ((vsi->info.port_vlan_flags & I40E_AQ_VSI_PVLAN_EMOD_MASK) == 2688 I40E_AQ_VSI_PVLAN_EMOD_MASK)) 2689 return; /* already disabled */ 2690 2691 vsi->info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID); 2692 vsi->info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_ALL | 2693 I40E_AQ_VSI_PVLAN_EMOD_NOTHING; 2694 2695 ctxt.seid = vsi->seid; 2696 ctxt.info = vsi->info; 2697 ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL); 2698 if (ret) { 2699 dev_info(&vsi->back->pdev->dev, 2700 "update vlan stripping failed, err %s aq_err %s\n", 2701 i40e_stat_str(&vsi->back->hw, ret), 2702 i40e_aq_str(&vsi->back->hw, 2703 vsi->back->hw.aq.asq_last_status)); 2704 } 2705 } 2706 2707 /** 2708 * i40e_add_vlan_all_mac - Add a MAC/VLAN filter for each existing MAC address 2709 * @vsi: the vsi being configured 2710 * @vid: vlan id to be added (0 = untagged only , -1 = any) 2711 * 2712 * This is a helper function for adding a new MAC/VLAN filter with the 2713 * specified VLAN for each existing MAC address already in the hash table. 2714 * This function does *not* perform any accounting to update filters based on 2715 * VLAN mode. 2716 * 2717 * NOTE: this function expects to be called while under the 2718 * mac_filter_hash_lock 2719 **/ 2720 int i40e_add_vlan_all_mac(struct i40e_vsi *vsi, s16 vid) 2721 { 2722 struct i40e_mac_filter *f, *add_f; 2723 struct hlist_node *h; 2724 int bkt; 2725 2726 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) { 2727 if (f->state == I40E_FILTER_REMOVE) 2728 continue; 2729 add_f = i40e_add_filter(vsi, f->macaddr, vid); 2730 if (!add_f) { 2731 dev_info(&vsi->back->pdev->dev, 2732 "Could not add vlan filter %d for %pM\n", 2733 vid, f->macaddr); 2734 return -ENOMEM; 2735 } 2736 } 2737 2738 return 0; 2739 } 2740 2741 /** 2742 * i40e_vsi_add_vlan - Add VSI membership for given VLAN 2743 * @vsi: the VSI being configured 2744 * @vid: VLAN id to be added 2745 **/ 2746 int i40e_vsi_add_vlan(struct i40e_vsi *vsi, u16 vid) 2747 { 2748 int err; 2749 2750 if (vsi->info.pvid) 2751 return -EINVAL; 2752 2753 /* The network stack will attempt to add VID=0, with the intention to 2754 * receive priority tagged packets with a VLAN of 0. Our HW receives 2755 * these packets by default when configured to receive untagged 2756 * packets, so we don't need to add a filter for this case. 2757 * Additionally, HW interprets adding a VID=0 filter as meaning to 2758 * receive *only* tagged traffic and stops receiving untagged traffic. 2759 * Thus, we do not want to actually add a filter for VID=0 2760 */ 2761 if (!vid) 2762 return 0; 2763 2764 /* Locked once because all functions invoked below iterates list*/ 2765 spin_lock_bh(&vsi->mac_filter_hash_lock); 2766 err = i40e_add_vlan_all_mac(vsi, vid); 2767 spin_unlock_bh(&vsi->mac_filter_hash_lock); 2768 if (err) 2769 return err; 2770 2771 /* schedule our worker thread which will take care of 2772 * applying the new filter changes 2773 */ 2774 i40e_service_event_schedule(vsi->back); 2775 return 0; 2776 } 2777 2778 /** 2779 * i40e_rm_vlan_all_mac - Remove MAC/VLAN pair for all MAC with the given VLAN 2780 * @vsi: the vsi being configured 2781 * @vid: vlan id to be removed (0 = untagged only , -1 = any) 2782 * 2783 * This function should be used to remove all VLAN filters which match the 2784 * given VID. It does not schedule the service event and does not take the 2785 * mac_filter_hash_lock so it may be combined with other operations under 2786 * a single invocation of the mac_filter_hash_lock. 2787 * 2788 * NOTE: this function expects to be called while under the 2789 * mac_filter_hash_lock 2790 */ 2791 void i40e_rm_vlan_all_mac(struct i40e_vsi *vsi, s16 vid) 2792 { 2793 struct i40e_mac_filter *f; 2794 struct hlist_node *h; 2795 int bkt; 2796 2797 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) { 2798 if (f->vlan == vid) 2799 __i40e_del_filter(vsi, f); 2800 } 2801 } 2802 2803 /** 2804 * i40e_vsi_kill_vlan - Remove VSI membership for given VLAN 2805 * @vsi: the VSI being configured 2806 * @vid: VLAN id to be removed 2807 **/ 2808 void i40e_vsi_kill_vlan(struct i40e_vsi *vsi, u16 vid) 2809 { 2810 if (!vid || vsi->info.pvid) 2811 return; 2812 2813 spin_lock_bh(&vsi->mac_filter_hash_lock); 2814 i40e_rm_vlan_all_mac(vsi, vid); 2815 spin_unlock_bh(&vsi->mac_filter_hash_lock); 2816 2817 /* schedule our worker thread which will take care of 2818 * applying the new filter changes 2819 */ 2820 i40e_service_event_schedule(vsi->back); 2821 } 2822 2823 /** 2824 * i40e_vlan_rx_add_vid - Add a vlan id filter to HW offload 2825 * @netdev: network interface to be adjusted 2826 * @proto: unused protocol value 2827 * @vid: vlan id to be added 2828 * 2829 * net_device_ops implementation for adding vlan ids 2830 **/ 2831 static int i40e_vlan_rx_add_vid(struct net_device *netdev, 2832 __always_unused __be16 proto, u16 vid) 2833 { 2834 struct i40e_netdev_priv *np = netdev_priv(netdev); 2835 struct i40e_vsi *vsi = np->vsi; 2836 int ret = 0; 2837 2838 if (vid >= VLAN_N_VID) 2839 return -EINVAL; 2840 2841 ret = i40e_vsi_add_vlan(vsi, vid); 2842 if (!ret) 2843 set_bit(vid, vsi->active_vlans); 2844 2845 return ret; 2846 } 2847 2848 /** 2849 * i40e_vlan_rx_add_vid_up - Add a vlan id filter to HW offload in UP path 2850 * @netdev: network interface to be adjusted 2851 * @proto: unused protocol value 2852 * @vid: vlan id to be added 2853 **/ 2854 static void i40e_vlan_rx_add_vid_up(struct net_device *netdev, 2855 __always_unused __be16 proto, u16 vid) 2856 { 2857 struct i40e_netdev_priv *np = netdev_priv(netdev); 2858 struct i40e_vsi *vsi = np->vsi; 2859 2860 if (vid >= VLAN_N_VID) 2861 return; 2862 set_bit(vid, vsi->active_vlans); 2863 } 2864 2865 /** 2866 * i40e_vlan_rx_kill_vid - Remove a vlan id filter from HW offload 2867 * @netdev: network interface to be adjusted 2868 * @proto: unused protocol value 2869 * @vid: vlan id to be removed 2870 * 2871 * net_device_ops implementation for removing vlan ids 2872 **/ 2873 static int i40e_vlan_rx_kill_vid(struct net_device *netdev, 2874 __always_unused __be16 proto, u16 vid) 2875 { 2876 struct i40e_netdev_priv *np = netdev_priv(netdev); 2877 struct i40e_vsi *vsi = np->vsi; 2878 2879 /* return code is ignored as there is nothing a user 2880 * can do about failure to remove and a log message was 2881 * already printed from the other function 2882 */ 2883 i40e_vsi_kill_vlan(vsi, vid); 2884 2885 clear_bit(vid, vsi->active_vlans); 2886 2887 return 0; 2888 } 2889 2890 /** 2891 * i40e_restore_vlan - Reinstate vlans when vsi/netdev comes back up 2892 * @vsi: the vsi being brought back up 2893 **/ 2894 static void i40e_restore_vlan(struct i40e_vsi *vsi) 2895 { 2896 u16 vid; 2897 2898 if (!vsi->netdev) 2899 return; 2900 2901 if (vsi->netdev->features & NETIF_F_HW_VLAN_CTAG_RX) 2902 i40e_vlan_stripping_enable(vsi); 2903 else 2904 i40e_vlan_stripping_disable(vsi); 2905 2906 for_each_set_bit(vid, vsi->active_vlans, VLAN_N_VID) 2907 i40e_vlan_rx_add_vid_up(vsi->netdev, htons(ETH_P_8021Q), 2908 vid); 2909 } 2910 2911 /** 2912 * i40e_vsi_add_pvid - Add pvid for the VSI 2913 * @vsi: the vsi being adjusted 2914 * @vid: the vlan id to set as a PVID 2915 **/ 2916 int i40e_vsi_add_pvid(struct i40e_vsi *vsi, u16 vid) 2917 { 2918 struct i40e_vsi_context ctxt; 2919 i40e_status ret; 2920 2921 vsi->info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID); 2922 vsi->info.pvid = cpu_to_le16(vid); 2923 vsi->info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_TAGGED | 2924 I40E_AQ_VSI_PVLAN_INSERT_PVID | 2925 I40E_AQ_VSI_PVLAN_EMOD_STR; 2926 2927 ctxt.seid = vsi->seid; 2928 ctxt.info = vsi->info; 2929 ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL); 2930 if (ret) { 2931 dev_info(&vsi->back->pdev->dev, 2932 "add pvid failed, err %s aq_err %s\n", 2933 i40e_stat_str(&vsi->back->hw, ret), 2934 i40e_aq_str(&vsi->back->hw, 2935 vsi->back->hw.aq.asq_last_status)); 2936 return -ENOENT; 2937 } 2938 2939 return 0; 2940 } 2941 2942 /** 2943 * i40e_vsi_remove_pvid - Remove the pvid from the VSI 2944 * @vsi: the vsi being adjusted 2945 * 2946 * Just use the vlan_rx_register() service to put it back to normal 2947 **/ 2948 void i40e_vsi_remove_pvid(struct i40e_vsi *vsi) 2949 { 2950 i40e_vlan_stripping_disable(vsi); 2951 2952 vsi->info.pvid = 0; 2953 } 2954 2955 /** 2956 * i40e_vsi_setup_tx_resources - Allocate VSI Tx queue resources 2957 * @vsi: ptr to the VSI 2958 * 2959 * If this function returns with an error, then it's possible one or 2960 * more of the rings is populated (while the rest are not). It is the 2961 * callers duty to clean those orphaned rings. 2962 * 2963 * Return 0 on success, negative on failure 2964 **/ 2965 static int i40e_vsi_setup_tx_resources(struct i40e_vsi *vsi) 2966 { 2967 int i, err = 0; 2968 2969 for (i = 0; i < vsi->num_queue_pairs && !err; i++) 2970 err = i40e_setup_tx_descriptors(vsi->tx_rings[i]); 2971 2972 if (!i40e_enabled_xdp_vsi(vsi)) 2973 return err; 2974 2975 for (i = 0; i < vsi->num_queue_pairs && !err; i++) 2976 err = i40e_setup_tx_descriptors(vsi->xdp_rings[i]); 2977 2978 return err; 2979 } 2980 2981 /** 2982 * i40e_vsi_free_tx_resources - Free Tx resources for VSI queues 2983 * @vsi: ptr to the VSI 2984 * 2985 * Free VSI's transmit software resources 2986 **/ 2987 static void i40e_vsi_free_tx_resources(struct i40e_vsi *vsi) 2988 { 2989 int i; 2990 2991 if (vsi->tx_rings) { 2992 for (i = 0; i < vsi->num_queue_pairs; i++) 2993 if (vsi->tx_rings[i] && vsi->tx_rings[i]->desc) 2994 i40e_free_tx_resources(vsi->tx_rings[i]); 2995 } 2996 2997 if (vsi->xdp_rings) { 2998 for (i = 0; i < vsi->num_queue_pairs; i++) 2999 if (vsi->xdp_rings[i] && vsi->xdp_rings[i]->desc) 3000 i40e_free_tx_resources(vsi->xdp_rings[i]); 3001 } 3002 } 3003 3004 /** 3005 * i40e_vsi_setup_rx_resources - Allocate VSI queues Rx resources 3006 * @vsi: ptr to the VSI 3007 * 3008 * If this function returns with an error, then it's possible one or 3009 * more of the rings is populated (while the rest are not). It is the 3010 * callers duty to clean those orphaned rings. 3011 * 3012 * Return 0 on success, negative on failure 3013 **/ 3014 static int i40e_vsi_setup_rx_resources(struct i40e_vsi *vsi) 3015 { 3016 int i, err = 0; 3017 3018 for (i = 0; i < vsi->num_queue_pairs && !err; i++) 3019 err = i40e_setup_rx_descriptors(vsi->rx_rings[i]); 3020 return err; 3021 } 3022 3023 /** 3024 * i40e_vsi_free_rx_resources - Free Rx Resources for VSI queues 3025 * @vsi: ptr to the VSI 3026 * 3027 * Free all receive software resources 3028 **/ 3029 static void i40e_vsi_free_rx_resources(struct i40e_vsi *vsi) 3030 { 3031 int i; 3032 3033 if (!vsi->rx_rings) 3034 return; 3035 3036 for (i = 0; i < vsi->num_queue_pairs; i++) 3037 if (vsi->rx_rings[i] && vsi->rx_rings[i]->desc) 3038 i40e_free_rx_resources(vsi->rx_rings[i]); 3039 } 3040 3041 /** 3042 * i40e_config_xps_tx_ring - Configure XPS for a Tx ring 3043 * @ring: The Tx ring to configure 3044 * 3045 * This enables/disables XPS for a given Tx descriptor ring 3046 * based on the TCs enabled for the VSI that ring belongs to. 3047 **/ 3048 static void i40e_config_xps_tx_ring(struct i40e_ring *ring) 3049 { 3050 int cpu; 3051 3052 if (!ring->q_vector || !ring->netdev || ring->ch) 3053 return; 3054 3055 /* We only initialize XPS once, so as not to overwrite user settings */ 3056 if (test_and_set_bit(__I40E_TX_XPS_INIT_DONE, ring->state)) 3057 return; 3058 3059 cpu = cpumask_local_spread(ring->q_vector->v_idx, -1); 3060 netif_set_xps_queue(ring->netdev, get_cpu_mask(cpu), 3061 ring->queue_index); 3062 } 3063 3064 /** 3065 * i40e_configure_tx_ring - Configure a transmit ring context and rest 3066 * @ring: The Tx ring to configure 3067 * 3068 * Configure the Tx descriptor ring in the HMC context. 3069 **/ 3070 static int i40e_configure_tx_ring(struct i40e_ring *ring) 3071 { 3072 struct i40e_vsi *vsi = ring->vsi; 3073 u16 pf_q = vsi->base_queue + ring->queue_index; 3074 struct i40e_hw *hw = &vsi->back->hw; 3075 struct i40e_hmc_obj_txq tx_ctx; 3076 i40e_status err = 0; 3077 u32 qtx_ctl = 0; 3078 3079 if (ring_is_xdp(ring)) 3080 ring->xsk_umem = i40e_xsk_umem(ring); 3081 3082 /* some ATR related tx ring init */ 3083 if (vsi->back->flags & I40E_FLAG_FD_ATR_ENABLED) { 3084 ring->atr_sample_rate = vsi->back->atr_sample_rate; 3085 ring->atr_count = 0; 3086 } else { 3087 ring->atr_sample_rate = 0; 3088 } 3089 3090 /* configure XPS */ 3091 i40e_config_xps_tx_ring(ring); 3092 3093 /* clear the context structure first */ 3094 memset(&tx_ctx, 0, sizeof(tx_ctx)); 3095 3096 tx_ctx.new_context = 1; 3097 tx_ctx.base = (ring->dma / 128); 3098 tx_ctx.qlen = ring->count; 3099 tx_ctx.fd_ena = !!(vsi->back->flags & (I40E_FLAG_FD_SB_ENABLED | 3100 I40E_FLAG_FD_ATR_ENABLED)); 3101 tx_ctx.timesync_ena = !!(vsi->back->flags & I40E_FLAG_PTP); 3102 /* FDIR VSI tx ring can still use RS bit and writebacks */ 3103 if (vsi->type != I40E_VSI_FDIR) 3104 tx_ctx.head_wb_ena = 1; 3105 tx_ctx.head_wb_addr = ring->dma + 3106 (ring->count * sizeof(struct i40e_tx_desc)); 3107 3108 /* As part of VSI creation/update, FW allocates certain 3109 * Tx arbitration queue sets for each TC enabled for 3110 * the VSI. The FW returns the handles to these queue 3111 * sets as part of the response buffer to Add VSI, 3112 * Update VSI, etc. AQ commands. It is expected that 3113 * these queue set handles be associated with the Tx 3114 * queues by the driver as part of the TX queue context 3115 * initialization. This has to be done regardless of 3116 * DCB as by default everything is mapped to TC0. 3117 */ 3118 3119 if (ring->ch) 3120 tx_ctx.rdylist = 3121 le16_to_cpu(ring->ch->info.qs_handle[ring->dcb_tc]); 3122 3123 else 3124 tx_ctx.rdylist = le16_to_cpu(vsi->info.qs_handle[ring->dcb_tc]); 3125 3126 tx_ctx.rdylist_act = 0; 3127 3128 /* clear the context in the HMC */ 3129 err = i40e_clear_lan_tx_queue_context(hw, pf_q); 3130 if (err) { 3131 dev_info(&vsi->back->pdev->dev, 3132 "Failed to clear LAN Tx queue context on Tx ring %d (pf_q %d), error: %d\n", 3133 ring->queue_index, pf_q, err); 3134 return -ENOMEM; 3135 } 3136 3137 /* set the context in the HMC */ 3138 err = i40e_set_lan_tx_queue_context(hw, pf_q, &tx_ctx); 3139 if (err) { 3140 dev_info(&vsi->back->pdev->dev, 3141 "Failed to set LAN Tx queue context on Tx ring %d (pf_q %d, error: %d\n", 3142 ring->queue_index, pf_q, err); 3143 return -ENOMEM; 3144 } 3145 3146 /* Now associate this queue with this PCI function */ 3147 if (ring->ch) { 3148 if (ring->ch->type == I40E_VSI_VMDQ2) 3149 qtx_ctl = I40E_QTX_CTL_VM_QUEUE; 3150 else 3151 return -EINVAL; 3152 3153 qtx_ctl |= (ring->ch->vsi_number << 3154 I40E_QTX_CTL_VFVM_INDX_SHIFT) & 3155 I40E_QTX_CTL_VFVM_INDX_MASK; 3156 } else { 3157 if (vsi->type == I40E_VSI_VMDQ2) { 3158 qtx_ctl = I40E_QTX_CTL_VM_QUEUE; 3159 qtx_ctl |= ((vsi->id) << I40E_QTX_CTL_VFVM_INDX_SHIFT) & 3160 I40E_QTX_CTL_VFVM_INDX_MASK; 3161 } else { 3162 qtx_ctl = I40E_QTX_CTL_PF_QUEUE; 3163 } 3164 } 3165 3166 qtx_ctl |= ((hw->pf_id << I40E_QTX_CTL_PF_INDX_SHIFT) & 3167 I40E_QTX_CTL_PF_INDX_MASK); 3168 wr32(hw, I40E_QTX_CTL(pf_q), qtx_ctl); 3169 i40e_flush(hw); 3170 3171 /* cache tail off for easier writes later */ 3172 ring->tail = hw->hw_addr + I40E_QTX_TAIL(pf_q); 3173 3174 return 0; 3175 } 3176 3177 /** 3178 * i40e_configure_rx_ring - Configure a receive ring context 3179 * @ring: The Rx ring to configure 3180 * 3181 * Configure the Rx descriptor ring in the HMC context. 3182 **/ 3183 static int i40e_configure_rx_ring(struct i40e_ring *ring) 3184 { 3185 struct i40e_vsi *vsi = ring->vsi; 3186 u32 chain_len = vsi->back->hw.func_caps.rx_buf_chain_len; 3187 u16 pf_q = vsi->base_queue + ring->queue_index; 3188 struct i40e_hw *hw = &vsi->back->hw; 3189 struct i40e_hmc_obj_rxq rx_ctx; 3190 i40e_status err = 0; 3191 bool ok; 3192 int ret; 3193 3194 bitmap_zero(ring->state, __I40E_RING_STATE_NBITS); 3195 3196 /* clear the context structure first */ 3197 memset(&rx_ctx, 0, sizeof(rx_ctx)); 3198 3199 if (ring->vsi->type == I40E_VSI_MAIN) 3200 xdp_rxq_info_unreg_mem_model(&ring->xdp_rxq); 3201 3202 ring->xsk_umem = i40e_xsk_umem(ring); 3203 if (ring->xsk_umem) { 3204 ring->rx_buf_len = ring->xsk_umem->chunk_size_nohr - 3205 XDP_PACKET_HEADROOM; 3206 /* For AF_XDP ZC, we disallow packets to span on 3207 * multiple buffers, thus letting us skip that 3208 * handling in the fast-path. 3209 */ 3210 chain_len = 1; 3211 ring->zca.free = i40e_zca_free; 3212 ret = xdp_rxq_info_reg_mem_model(&ring->xdp_rxq, 3213 MEM_TYPE_ZERO_COPY, 3214 &ring->zca); 3215 if (ret) 3216 return ret; 3217 dev_info(&vsi->back->pdev->dev, 3218 "Registered XDP mem model MEM_TYPE_ZERO_COPY on Rx ring %d\n", 3219 ring->queue_index); 3220 3221 } else { 3222 ring->rx_buf_len = vsi->rx_buf_len; 3223 if (ring->vsi->type == I40E_VSI_MAIN) { 3224 ret = xdp_rxq_info_reg_mem_model(&ring->xdp_rxq, 3225 MEM_TYPE_PAGE_SHARED, 3226 NULL); 3227 if (ret) 3228 return ret; 3229 } 3230 } 3231 3232 rx_ctx.dbuff = DIV_ROUND_UP(ring->rx_buf_len, 3233 BIT_ULL(I40E_RXQ_CTX_DBUFF_SHIFT)); 3234 3235 rx_ctx.base = (ring->dma / 128); 3236 rx_ctx.qlen = ring->count; 3237 3238 /* use 32 byte descriptors */ 3239 rx_ctx.dsize = 1; 3240 3241 /* descriptor type is always zero 3242 * rx_ctx.dtype = 0; 3243 */ 3244 rx_ctx.hsplit_0 = 0; 3245 3246 rx_ctx.rxmax = min_t(u16, vsi->max_frame, chain_len * ring->rx_buf_len); 3247 if (hw->revision_id == 0) 3248 rx_ctx.lrxqthresh = 0; 3249 else 3250 rx_ctx.lrxqthresh = 1; 3251 rx_ctx.crcstrip = 1; 3252 rx_ctx.l2tsel = 1; 3253 /* this controls whether VLAN is stripped from inner headers */ 3254 rx_ctx.showiv = 0; 3255 /* set the prefena field to 1 because the manual says to */ 3256 rx_ctx.prefena = 1; 3257 3258 /* clear the context in the HMC */ 3259 err = i40e_clear_lan_rx_queue_context(hw, pf_q); 3260 if (err) { 3261 dev_info(&vsi->back->pdev->dev, 3262 "Failed to clear LAN Rx queue context on Rx ring %d (pf_q %d), error: %d\n", 3263 ring->queue_index, pf_q, err); 3264 return -ENOMEM; 3265 } 3266 3267 /* set the context in the HMC */ 3268 err = i40e_set_lan_rx_queue_context(hw, pf_q, &rx_ctx); 3269 if (err) { 3270 dev_info(&vsi->back->pdev->dev, 3271 "Failed to set LAN Rx queue context on Rx ring %d (pf_q %d), error: %d\n", 3272 ring->queue_index, pf_q, err); 3273 return -ENOMEM; 3274 } 3275 3276 /* configure Rx buffer alignment */ 3277 if (!vsi->netdev || (vsi->back->flags & I40E_FLAG_LEGACY_RX)) 3278 clear_ring_build_skb_enabled(ring); 3279 else 3280 set_ring_build_skb_enabled(ring); 3281 3282 /* cache tail for quicker writes, and clear the reg before use */ 3283 ring->tail = hw->hw_addr + I40E_QRX_TAIL(pf_q); 3284 writel(0, ring->tail); 3285 3286 ok = ring->xsk_umem ? 3287 i40e_alloc_rx_buffers_zc(ring, I40E_DESC_UNUSED(ring)) : 3288 !i40e_alloc_rx_buffers(ring, I40E_DESC_UNUSED(ring)); 3289 if (!ok) { 3290 dev_info(&vsi->back->pdev->dev, 3291 "Failed allocate some buffers on %sRx ring %d (pf_q %d)\n", 3292 ring->xsk_umem ? "UMEM enabled " : "", 3293 ring->queue_index, pf_q); 3294 } 3295 3296 return 0; 3297 } 3298 3299 /** 3300 * i40e_vsi_configure_tx - Configure the VSI for Tx 3301 * @vsi: VSI structure describing this set of rings and resources 3302 * 3303 * Configure the Tx VSI for operation. 3304 **/ 3305 static int i40e_vsi_configure_tx(struct i40e_vsi *vsi) 3306 { 3307 int err = 0; 3308 u16 i; 3309 3310 for (i = 0; (i < vsi->num_queue_pairs) && !err; i++) 3311 err = i40e_configure_tx_ring(vsi->tx_rings[i]); 3312 3313 if (!i40e_enabled_xdp_vsi(vsi)) 3314 return err; 3315 3316 for (i = 0; (i < vsi->num_queue_pairs) && !err; i++) 3317 err = i40e_configure_tx_ring(vsi->xdp_rings[i]); 3318 3319 return err; 3320 } 3321 3322 /** 3323 * i40e_vsi_configure_rx - Configure the VSI for Rx 3324 * @vsi: the VSI being configured 3325 * 3326 * Configure the Rx VSI for operation. 3327 **/ 3328 static int i40e_vsi_configure_rx(struct i40e_vsi *vsi) 3329 { 3330 int err = 0; 3331 u16 i; 3332 3333 if (!vsi->netdev || (vsi->back->flags & I40E_FLAG_LEGACY_RX)) { 3334 vsi->max_frame = I40E_MAX_RXBUFFER; 3335 vsi->rx_buf_len = I40E_RXBUFFER_2048; 3336 #if (PAGE_SIZE < 8192) 3337 } else if (!I40E_2K_TOO_SMALL_WITH_PADDING && 3338 (vsi->netdev->mtu <= ETH_DATA_LEN)) { 3339 vsi->max_frame = I40E_RXBUFFER_1536 - NET_IP_ALIGN; 3340 vsi->rx_buf_len = I40E_RXBUFFER_1536 - NET_IP_ALIGN; 3341 #endif 3342 } else { 3343 vsi->max_frame = I40E_MAX_RXBUFFER; 3344 vsi->rx_buf_len = (PAGE_SIZE < 8192) ? I40E_RXBUFFER_3072 : 3345 I40E_RXBUFFER_2048; 3346 } 3347 3348 /* set up individual rings */ 3349 for (i = 0; i < vsi->num_queue_pairs && !err; i++) 3350 err = i40e_configure_rx_ring(vsi->rx_rings[i]); 3351 3352 return err; 3353 } 3354 3355 /** 3356 * i40e_vsi_config_dcb_rings - Update rings to reflect DCB TC 3357 * @vsi: ptr to the VSI 3358 **/ 3359 static void i40e_vsi_config_dcb_rings(struct i40e_vsi *vsi) 3360 { 3361 struct i40e_ring *tx_ring, *rx_ring; 3362 u16 qoffset, qcount; 3363 int i, n; 3364 3365 if (!(vsi->back->flags & I40E_FLAG_DCB_ENABLED)) { 3366 /* Reset the TC information */ 3367 for (i = 0; i < vsi->num_queue_pairs; i++) { 3368 rx_ring = vsi->rx_rings[i]; 3369 tx_ring = vsi->tx_rings[i]; 3370 rx_ring->dcb_tc = 0; 3371 tx_ring->dcb_tc = 0; 3372 } 3373 return; 3374 } 3375 3376 for (n = 0; n < I40E_MAX_TRAFFIC_CLASS; n++) { 3377 if (!(vsi->tc_config.enabled_tc & BIT_ULL(n))) 3378 continue; 3379 3380 qoffset = vsi->tc_config.tc_info[n].qoffset; 3381 qcount = vsi->tc_config.tc_info[n].qcount; 3382 for (i = qoffset; i < (qoffset + qcount); i++) { 3383 rx_ring = vsi->rx_rings[i]; 3384 tx_ring = vsi->tx_rings[i]; 3385 rx_ring->dcb_tc = n; 3386 tx_ring->dcb_tc = n; 3387 } 3388 } 3389 } 3390 3391 /** 3392 * i40e_set_vsi_rx_mode - Call set_rx_mode on a VSI 3393 * @vsi: ptr to the VSI 3394 **/ 3395 static void i40e_set_vsi_rx_mode(struct i40e_vsi *vsi) 3396 { 3397 if (vsi->netdev) 3398 i40e_set_rx_mode(vsi->netdev); 3399 } 3400 3401 /** 3402 * i40e_fdir_filter_restore - Restore the Sideband Flow Director filters 3403 * @vsi: Pointer to the targeted VSI 3404 * 3405 * This function replays the hlist on the hw where all the SB Flow Director 3406 * filters were saved. 3407 **/ 3408 static void i40e_fdir_filter_restore(struct i40e_vsi *vsi) 3409 { 3410 struct i40e_fdir_filter *filter; 3411 struct i40e_pf *pf = vsi->back; 3412 struct hlist_node *node; 3413 3414 if (!(pf->flags & I40E_FLAG_FD_SB_ENABLED)) 3415 return; 3416 3417 /* Reset FDir counters as we're replaying all existing filters */ 3418 pf->fd_tcp4_filter_cnt = 0; 3419 pf->fd_udp4_filter_cnt = 0; 3420 pf->fd_sctp4_filter_cnt = 0; 3421 pf->fd_ip4_filter_cnt = 0; 3422 3423 hlist_for_each_entry_safe(filter, node, 3424 &pf->fdir_filter_list, fdir_node) { 3425 i40e_add_del_fdir(vsi, filter, true); 3426 } 3427 } 3428 3429 /** 3430 * i40e_vsi_configure - Set up the VSI for action 3431 * @vsi: the VSI being configured 3432 **/ 3433 static int i40e_vsi_configure(struct i40e_vsi *vsi) 3434 { 3435 int err; 3436 3437 i40e_set_vsi_rx_mode(vsi); 3438 i40e_restore_vlan(vsi); 3439 i40e_vsi_config_dcb_rings(vsi); 3440 err = i40e_vsi_configure_tx(vsi); 3441 if (!err) 3442 err = i40e_vsi_configure_rx(vsi); 3443 3444 return err; 3445 } 3446 3447 /** 3448 * i40e_vsi_configure_msix - MSIX mode Interrupt Config in the HW 3449 * @vsi: the VSI being configured 3450 **/ 3451 static void i40e_vsi_configure_msix(struct i40e_vsi *vsi) 3452 { 3453 bool has_xdp = i40e_enabled_xdp_vsi(vsi); 3454 struct i40e_pf *pf = vsi->back; 3455 struct i40e_hw *hw = &pf->hw; 3456 u16 vector; 3457 int i, q; 3458 u32 qp; 3459 3460 /* The interrupt indexing is offset by 1 in the PFINT_ITRn 3461 * and PFINT_LNKLSTn registers, e.g.: 3462 * PFINT_ITRn[0..n-1] gets msix-1..msix-n (qpair interrupts) 3463 */ 3464 qp = vsi->base_queue; 3465 vector = vsi->base_vector; 3466 for (i = 0; i < vsi->num_q_vectors; i++, vector++) { 3467 struct i40e_q_vector *q_vector = vsi->q_vectors[i]; 3468 3469 q_vector->rx.next_update = jiffies + 1; 3470 q_vector->rx.target_itr = 3471 ITR_TO_REG(vsi->rx_rings[i]->itr_setting); 3472 wr32(hw, I40E_PFINT_ITRN(I40E_RX_ITR, vector - 1), 3473 q_vector->rx.target_itr); 3474 q_vector->rx.current_itr = q_vector->rx.target_itr; 3475 3476 q_vector->tx.next_update = jiffies + 1; 3477 q_vector->tx.target_itr = 3478 ITR_TO_REG(vsi->tx_rings[i]->itr_setting); 3479 wr32(hw, I40E_PFINT_ITRN(I40E_TX_ITR, vector - 1), 3480 q_vector->tx.target_itr); 3481 q_vector->tx.current_itr = q_vector->tx.target_itr; 3482 3483 wr32(hw, I40E_PFINT_RATEN(vector - 1), 3484 i40e_intrl_usec_to_reg(vsi->int_rate_limit)); 3485 3486 /* Linked list for the queuepairs assigned to this vector */ 3487 wr32(hw, I40E_PFINT_LNKLSTN(vector - 1), qp); 3488 for (q = 0; q < q_vector->num_ringpairs; q++) { 3489 u32 nextqp = has_xdp ? qp + vsi->alloc_queue_pairs : qp; 3490 u32 val; 3491 3492 val = I40E_QINT_RQCTL_CAUSE_ENA_MASK | 3493 (I40E_RX_ITR << I40E_QINT_RQCTL_ITR_INDX_SHIFT) | 3494 (vector << I40E_QINT_RQCTL_MSIX_INDX_SHIFT) | 3495 (nextqp << I40E_QINT_RQCTL_NEXTQ_INDX_SHIFT) | 3496 (I40E_QUEUE_TYPE_TX << 3497 I40E_QINT_RQCTL_NEXTQ_TYPE_SHIFT); 3498 3499 wr32(hw, I40E_QINT_RQCTL(qp), val); 3500 3501 if (has_xdp) { 3502 val = I40E_QINT_TQCTL_CAUSE_ENA_MASK | 3503 (I40E_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT) | 3504 (vector << I40E_QINT_TQCTL_MSIX_INDX_SHIFT) | 3505 (qp << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT) | 3506 (I40E_QUEUE_TYPE_TX << 3507 I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT); 3508 3509 wr32(hw, I40E_QINT_TQCTL(nextqp), val); 3510 } 3511 3512 val = I40E_QINT_TQCTL_CAUSE_ENA_MASK | 3513 (I40E_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT) | 3514 (vector << I40E_QINT_TQCTL_MSIX_INDX_SHIFT) | 3515 ((qp + 1) << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT) | 3516 (I40E_QUEUE_TYPE_RX << 3517 I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT); 3518 3519 /* Terminate the linked list */ 3520 if (q == (q_vector->num_ringpairs - 1)) 3521 val |= (I40E_QUEUE_END_OF_LIST << 3522 I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT); 3523 3524 wr32(hw, I40E_QINT_TQCTL(qp), val); 3525 qp++; 3526 } 3527 } 3528 3529 i40e_flush(hw); 3530 } 3531 3532 /** 3533 * i40e_enable_misc_int_causes - enable the non-queue interrupts 3534 * @pf: pointer to private device data structure 3535 **/ 3536 static void i40e_enable_misc_int_causes(struct i40e_pf *pf) 3537 { 3538 struct i40e_hw *hw = &pf->hw; 3539 u32 val; 3540 3541 /* clear things first */ 3542 wr32(hw, I40E_PFINT_ICR0_ENA, 0); /* disable all */ 3543 rd32(hw, I40E_PFINT_ICR0); /* read to clear */ 3544 3545 val = I40E_PFINT_ICR0_ENA_ECC_ERR_MASK | 3546 I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK | 3547 I40E_PFINT_ICR0_ENA_GRST_MASK | 3548 I40E_PFINT_ICR0_ENA_PCI_EXCEPTION_MASK | 3549 I40E_PFINT_ICR0_ENA_GPIO_MASK | 3550 I40E_PFINT_ICR0_ENA_HMC_ERR_MASK | 3551 I40E_PFINT_ICR0_ENA_VFLR_MASK | 3552 I40E_PFINT_ICR0_ENA_ADMINQ_MASK; 3553 3554 if (pf->flags & I40E_FLAG_IWARP_ENABLED) 3555 val |= I40E_PFINT_ICR0_ENA_PE_CRITERR_MASK; 3556 3557 if (pf->flags & I40E_FLAG_PTP) 3558 val |= I40E_PFINT_ICR0_ENA_TIMESYNC_MASK; 3559 3560 wr32(hw, I40E_PFINT_ICR0_ENA, val); 3561 3562 /* SW_ITR_IDX = 0, but don't change INTENA */ 3563 wr32(hw, I40E_PFINT_DYN_CTL0, I40E_PFINT_DYN_CTL0_SW_ITR_INDX_MASK | 3564 I40E_PFINT_DYN_CTL0_INTENA_MSK_MASK); 3565 3566 /* OTHER_ITR_IDX = 0 */ 3567 wr32(hw, I40E_PFINT_STAT_CTL0, 0); 3568 } 3569 3570 /** 3571 * i40e_configure_msi_and_legacy - Legacy mode interrupt config in the HW 3572 * @vsi: the VSI being configured 3573 **/ 3574 static void i40e_configure_msi_and_legacy(struct i40e_vsi *vsi) 3575 { 3576 u32 nextqp = i40e_enabled_xdp_vsi(vsi) ? vsi->alloc_queue_pairs : 0; 3577 struct i40e_q_vector *q_vector = vsi->q_vectors[0]; 3578 struct i40e_pf *pf = vsi->back; 3579 struct i40e_hw *hw = &pf->hw; 3580 u32 val; 3581 3582 /* set the ITR configuration */ 3583 q_vector->rx.next_update = jiffies + 1; 3584 q_vector->rx.target_itr = ITR_TO_REG(vsi->rx_rings[0]->itr_setting); 3585 wr32(hw, I40E_PFINT_ITR0(I40E_RX_ITR), q_vector->rx.target_itr); 3586 q_vector->rx.current_itr = q_vector->rx.target_itr; 3587 q_vector->tx.next_update = jiffies + 1; 3588 q_vector->tx.target_itr = ITR_TO_REG(vsi->tx_rings[0]->itr_setting); 3589 wr32(hw, I40E_PFINT_ITR0(I40E_TX_ITR), q_vector->tx.target_itr); 3590 q_vector->tx.current_itr = q_vector->tx.target_itr; 3591 3592 i40e_enable_misc_int_causes(pf); 3593 3594 /* FIRSTQ_INDX = 0, FIRSTQ_TYPE = 0 (rx) */ 3595 wr32(hw, I40E_PFINT_LNKLST0, 0); 3596 3597 /* Associate the queue pair to the vector and enable the queue int */ 3598 val = I40E_QINT_RQCTL_CAUSE_ENA_MASK | 3599 (I40E_RX_ITR << I40E_QINT_RQCTL_ITR_INDX_SHIFT) | 3600 (nextqp << I40E_QINT_RQCTL_NEXTQ_INDX_SHIFT)| 3601 (I40E_QUEUE_TYPE_TX << I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT); 3602 3603 wr32(hw, I40E_QINT_RQCTL(0), val); 3604 3605 if (i40e_enabled_xdp_vsi(vsi)) { 3606 val = I40E_QINT_TQCTL_CAUSE_ENA_MASK | 3607 (I40E_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT)| 3608 (I40E_QUEUE_TYPE_TX 3609 << I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT); 3610 3611 wr32(hw, I40E_QINT_TQCTL(nextqp), val); 3612 } 3613 3614 val = I40E_QINT_TQCTL_CAUSE_ENA_MASK | 3615 (I40E_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT) | 3616 (I40E_QUEUE_END_OF_LIST << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT); 3617 3618 wr32(hw, I40E_QINT_TQCTL(0), val); 3619 i40e_flush(hw); 3620 } 3621 3622 /** 3623 * i40e_irq_dynamic_disable_icr0 - Disable default interrupt generation for icr0 3624 * @pf: board private structure 3625 **/ 3626 void i40e_irq_dynamic_disable_icr0(struct i40e_pf *pf) 3627 { 3628 struct i40e_hw *hw = &pf->hw; 3629 3630 wr32(hw, I40E_PFINT_DYN_CTL0, 3631 I40E_ITR_NONE << I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT); 3632 i40e_flush(hw); 3633 } 3634 3635 /** 3636 * i40e_irq_dynamic_enable_icr0 - Enable default interrupt generation for icr0 3637 * @pf: board private structure 3638 **/ 3639 void i40e_irq_dynamic_enable_icr0(struct i40e_pf *pf) 3640 { 3641 struct i40e_hw *hw = &pf->hw; 3642 u32 val; 3643 3644 val = I40E_PFINT_DYN_CTL0_INTENA_MASK | 3645 I40E_PFINT_DYN_CTL0_CLEARPBA_MASK | 3646 (I40E_ITR_NONE << I40E_PFINT_DYN_CTL0_ITR_INDX_SHIFT); 3647 3648 wr32(hw, I40E_PFINT_DYN_CTL0, val); 3649 i40e_flush(hw); 3650 } 3651 3652 /** 3653 * i40e_msix_clean_rings - MSIX mode Interrupt Handler 3654 * @irq: interrupt number 3655 * @data: pointer to a q_vector 3656 **/ 3657 static irqreturn_t i40e_msix_clean_rings(int irq, void *data) 3658 { 3659 struct i40e_q_vector *q_vector = data; 3660 3661 if (!q_vector->tx.ring && !q_vector->rx.ring) 3662 return IRQ_HANDLED; 3663 3664 napi_schedule_irqoff(&q_vector->napi); 3665 3666 return IRQ_HANDLED; 3667 } 3668 3669 /** 3670 * i40e_irq_affinity_notify - Callback for affinity changes 3671 * @notify: context as to what irq was changed 3672 * @mask: the new affinity mask 3673 * 3674 * This is a callback function used by the irq_set_affinity_notifier function 3675 * so that we may register to receive changes to the irq affinity masks. 3676 **/ 3677 static void i40e_irq_affinity_notify(struct irq_affinity_notify *notify, 3678 const cpumask_t *mask) 3679 { 3680 struct i40e_q_vector *q_vector = 3681 container_of(notify, struct i40e_q_vector, affinity_notify); 3682 3683 cpumask_copy(&q_vector->affinity_mask, mask); 3684 } 3685 3686 /** 3687 * i40e_irq_affinity_release - Callback for affinity notifier release 3688 * @ref: internal core kernel usage 3689 * 3690 * This is a callback function used by the irq_set_affinity_notifier function 3691 * to inform the current notification subscriber that they will no longer 3692 * receive notifications. 3693 **/ 3694 static void i40e_irq_affinity_release(struct kref *ref) {} 3695 3696 /** 3697 * i40e_vsi_request_irq_msix - Initialize MSI-X interrupts 3698 * @vsi: the VSI being configured 3699 * @basename: name for the vector 3700 * 3701 * Allocates MSI-X vectors and requests interrupts from the kernel. 3702 **/ 3703 static int i40e_vsi_request_irq_msix(struct i40e_vsi *vsi, char *basename) 3704 { 3705 int q_vectors = vsi->num_q_vectors; 3706 struct i40e_pf *pf = vsi->back; 3707 int base = vsi->base_vector; 3708 int rx_int_idx = 0; 3709 int tx_int_idx = 0; 3710 int vector, err; 3711 int irq_num; 3712 int cpu; 3713 3714 for (vector = 0; vector < q_vectors; vector++) { 3715 struct i40e_q_vector *q_vector = vsi->q_vectors[vector]; 3716 3717 irq_num = pf->msix_entries[base + vector].vector; 3718 3719 if (q_vector->tx.ring && q_vector->rx.ring) { 3720 snprintf(q_vector->name, sizeof(q_vector->name) - 1, 3721 "%s-%s-%d", basename, "TxRx", rx_int_idx++); 3722 tx_int_idx++; 3723 } else if (q_vector->rx.ring) { 3724 snprintf(q_vector->name, sizeof(q_vector->name) - 1, 3725 "%s-%s-%d", basename, "rx", rx_int_idx++); 3726 } else if (q_vector->tx.ring) { 3727 snprintf(q_vector->name, sizeof(q_vector->name) - 1, 3728 "%s-%s-%d", basename, "tx", tx_int_idx++); 3729 } else { 3730 /* skip this unused q_vector */ 3731 continue; 3732 } 3733 err = request_irq(irq_num, 3734 vsi->irq_handler, 3735 0, 3736 q_vector->name, 3737 q_vector); 3738 if (err) { 3739 dev_info(&pf->pdev->dev, 3740 "MSIX request_irq failed, error: %d\n", err); 3741 goto free_queue_irqs; 3742 } 3743 3744 /* register for affinity change notifications */ 3745 q_vector->affinity_notify.notify = i40e_irq_affinity_notify; 3746 q_vector->affinity_notify.release = i40e_irq_affinity_release; 3747 irq_set_affinity_notifier(irq_num, &q_vector->affinity_notify); 3748 /* Spread affinity hints out across online CPUs. 3749 * 3750 * get_cpu_mask returns a static constant mask with 3751 * a permanent lifetime so it's ok to pass to 3752 * irq_set_affinity_hint without making a copy. 3753 */ 3754 cpu = cpumask_local_spread(q_vector->v_idx, -1); 3755 irq_set_affinity_hint(irq_num, get_cpu_mask(cpu)); 3756 } 3757 3758 vsi->irqs_ready = true; 3759 return 0; 3760 3761 free_queue_irqs: 3762 while (vector) { 3763 vector--; 3764 irq_num = pf->msix_entries[base + vector].vector; 3765 irq_set_affinity_notifier(irq_num, NULL); 3766 irq_set_affinity_hint(irq_num, NULL); 3767 free_irq(irq_num, &vsi->q_vectors[vector]); 3768 } 3769 return err; 3770 } 3771 3772 /** 3773 * i40e_vsi_disable_irq - Mask off queue interrupt generation on the VSI 3774 * @vsi: the VSI being un-configured 3775 **/ 3776 static void i40e_vsi_disable_irq(struct i40e_vsi *vsi) 3777 { 3778 struct i40e_pf *pf = vsi->back; 3779 struct i40e_hw *hw = &pf->hw; 3780 int base = vsi->base_vector; 3781 int i; 3782 3783 /* disable interrupt causation from each queue */ 3784 for (i = 0; i < vsi->num_queue_pairs; i++) { 3785 u32 val; 3786 3787 val = rd32(hw, I40E_QINT_TQCTL(vsi->tx_rings[i]->reg_idx)); 3788 val &= ~I40E_QINT_TQCTL_CAUSE_ENA_MASK; 3789 wr32(hw, I40E_QINT_TQCTL(vsi->tx_rings[i]->reg_idx), val); 3790 3791 val = rd32(hw, I40E_QINT_RQCTL(vsi->rx_rings[i]->reg_idx)); 3792 val &= ~I40E_QINT_RQCTL_CAUSE_ENA_MASK; 3793 wr32(hw, I40E_QINT_RQCTL(vsi->rx_rings[i]->reg_idx), val); 3794 3795 if (!i40e_enabled_xdp_vsi(vsi)) 3796 continue; 3797 wr32(hw, I40E_QINT_TQCTL(vsi->xdp_rings[i]->reg_idx), 0); 3798 } 3799 3800 /* disable each interrupt */ 3801 if (pf->flags & I40E_FLAG_MSIX_ENABLED) { 3802 for (i = vsi->base_vector; 3803 i < (vsi->num_q_vectors + vsi->base_vector); i++) 3804 wr32(hw, I40E_PFINT_DYN_CTLN(i - 1), 0); 3805 3806 i40e_flush(hw); 3807 for (i = 0; i < vsi->num_q_vectors; i++) 3808 synchronize_irq(pf->msix_entries[i + base].vector); 3809 } else { 3810 /* Legacy and MSI mode - this stops all interrupt handling */ 3811 wr32(hw, I40E_PFINT_ICR0_ENA, 0); 3812 wr32(hw, I40E_PFINT_DYN_CTL0, 0); 3813 i40e_flush(hw); 3814 synchronize_irq(pf->pdev->irq); 3815 } 3816 } 3817 3818 /** 3819 * i40e_vsi_enable_irq - Enable IRQ for the given VSI 3820 * @vsi: the VSI being configured 3821 **/ 3822 static int i40e_vsi_enable_irq(struct i40e_vsi *vsi) 3823 { 3824 struct i40e_pf *pf = vsi->back; 3825 int i; 3826 3827 if (pf->flags & I40E_FLAG_MSIX_ENABLED) { 3828 for (i = 0; i < vsi->num_q_vectors; i++) 3829 i40e_irq_dynamic_enable(vsi, i); 3830 } else { 3831 i40e_irq_dynamic_enable_icr0(pf); 3832 } 3833 3834 i40e_flush(&pf->hw); 3835 return 0; 3836 } 3837 3838 /** 3839 * i40e_free_misc_vector - Free the vector that handles non-queue events 3840 * @pf: board private structure 3841 **/ 3842 static void i40e_free_misc_vector(struct i40e_pf *pf) 3843 { 3844 /* Disable ICR 0 */ 3845 wr32(&pf->hw, I40E_PFINT_ICR0_ENA, 0); 3846 i40e_flush(&pf->hw); 3847 3848 if (pf->flags & I40E_FLAG_MSIX_ENABLED && pf->msix_entries) { 3849 synchronize_irq(pf->msix_entries[0].vector); 3850 free_irq(pf->msix_entries[0].vector, pf); 3851 clear_bit(__I40E_MISC_IRQ_REQUESTED, pf->state); 3852 } 3853 } 3854 3855 /** 3856 * i40e_intr - MSI/Legacy and non-queue interrupt handler 3857 * @irq: interrupt number 3858 * @data: pointer to a q_vector 3859 * 3860 * This is the handler used for all MSI/Legacy interrupts, and deals 3861 * with both queue and non-queue interrupts. This is also used in 3862 * MSIX mode to handle the non-queue interrupts. 3863 **/ 3864 static irqreturn_t i40e_intr(int irq, void *data) 3865 { 3866 struct i40e_pf *pf = (struct i40e_pf *)data; 3867 struct i40e_hw *hw = &pf->hw; 3868 irqreturn_t ret = IRQ_NONE; 3869 u32 icr0, icr0_remaining; 3870 u32 val, ena_mask; 3871 3872 icr0 = rd32(hw, I40E_PFINT_ICR0); 3873 ena_mask = rd32(hw, I40E_PFINT_ICR0_ENA); 3874 3875 /* if sharing a legacy IRQ, we might get called w/o an intr pending */ 3876 if ((icr0 & I40E_PFINT_ICR0_INTEVENT_MASK) == 0) 3877 goto enable_intr; 3878 3879 /* if interrupt but no bits showing, must be SWINT */ 3880 if (((icr0 & ~I40E_PFINT_ICR0_INTEVENT_MASK) == 0) || 3881 (icr0 & I40E_PFINT_ICR0_SWINT_MASK)) 3882 pf->sw_int_count++; 3883 3884 if ((pf->flags & I40E_FLAG_IWARP_ENABLED) && 3885 (icr0 & I40E_PFINT_ICR0_ENA_PE_CRITERR_MASK)) { 3886 ena_mask &= ~I40E_PFINT_ICR0_ENA_PE_CRITERR_MASK; 3887 dev_dbg(&pf->pdev->dev, "cleared PE_CRITERR\n"); 3888 set_bit(__I40E_CORE_RESET_REQUESTED, pf->state); 3889 } 3890 3891 /* only q0 is used in MSI/Legacy mode, and none are used in MSIX */ 3892 if (icr0 & I40E_PFINT_ICR0_QUEUE_0_MASK) { 3893 struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi]; 3894 struct i40e_q_vector *q_vector = vsi->q_vectors[0]; 3895 3896 /* We do not have a way to disarm Queue causes while leaving 3897 * interrupt enabled for all other causes, ideally 3898 * interrupt should be disabled while we are in NAPI but 3899 * this is not a performance path and napi_schedule() 3900 * can deal with rescheduling. 3901 */ 3902 if (!test_bit(__I40E_DOWN, pf->state)) 3903 napi_schedule_irqoff(&q_vector->napi); 3904 } 3905 3906 if (icr0 & I40E_PFINT_ICR0_ADMINQ_MASK) { 3907 ena_mask &= ~I40E_PFINT_ICR0_ENA_ADMINQ_MASK; 3908 set_bit(__I40E_ADMINQ_EVENT_PENDING, pf->state); 3909 i40e_debug(&pf->hw, I40E_DEBUG_NVM, "AdminQ event\n"); 3910 } 3911 3912 if (icr0 & I40E_PFINT_ICR0_MAL_DETECT_MASK) { 3913 ena_mask &= ~I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK; 3914 set_bit(__I40E_MDD_EVENT_PENDING, pf->state); 3915 } 3916 3917 if (icr0 & I40E_PFINT_ICR0_VFLR_MASK) { 3918 ena_mask &= ~I40E_PFINT_ICR0_ENA_VFLR_MASK; 3919 set_bit(__I40E_VFLR_EVENT_PENDING, pf->state); 3920 } 3921 3922 if (icr0 & I40E_PFINT_ICR0_GRST_MASK) { 3923 if (!test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state)) 3924 set_bit(__I40E_RESET_INTR_RECEIVED, pf->state); 3925 ena_mask &= ~I40E_PFINT_ICR0_ENA_GRST_MASK; 3926 val = rd32(hw, I40E_GLGEN_RSTAT); 3927 val = (val & I40E_GLGEN_RSTAT_RESET_TYPE_MASK) 3928 >> I40E_GLGEN_RSTAT_RESET_TYPE_SHIFT; 3929 if (val == I40E_RESET_CORER) { 3930 pf->corer_count++; 3931 } else if (val == I40E_RESET_GLOBR) { 3932 pf->globr_count++; 3933 } else if (val == I40E_RESET_EMPR) { 3934 pf->empr_count++; 3935 set_bit(__I40E_EMP_RESET_INTR_RECEIVED, pf->state); 3936 } 3937 } 3938 3939 if (icr0 & I40E_PFINT_ICR0_HMC_ERR_MASK) { 3940 icr0 &= ~I40E_PFINT_ICR0_HMC_ERR_MASK; 3941 dev_info(&pf->pdev->dev, "HMC error interrupt\n"); 3942 dev_info(&pf->pdev->dev, "HMC error info 0x%x, HMC error data 0x%x\n", 3943 rd32(hw, I40E_PFHMC_ERRORINFO), 3944 rd32(hw, I40E_PFHMC_ERRORDATA)); 3945 } 3946 3947 if (icr0 & I40E_PFINT_ICR0_TIMESYNC_MASK) { 3948 u32 prttsyn_stat = rd32(hw, I40E_PRTTSYN_STAT_0); 3949 3950 if (prttsyn_stat & I40E_PRTTSYN_STAT_0_TXTIME_MASK) { 3951 icr0 &= ~I40E_PFINT_ICR0_ENA_TIMESYNC_MASK; 3952 i40e_ptp_tx_hwtstamp(pf); 3953 } 3954 } 3955 3956 /* If a critical error is pending we have no choice but to reset the 3957 * device. 3958 * Report and mask out any remaining unexpected interrupts. 3959 */ 3960 icr0_remaining = icr0 & ena_mask; 3961 if (icr0_remaining) { 3962 dev_info(&pf->pdev->dev, "unhandled interrupt icr0=0x%08x\n", 3963 icr0_remaining); 3964 if ((icr0_remaining & I40E_PFINT_ICR0_PE_CRITERR_MASK) || 3965 (icr0_remaining & I40E_PFINT_ICR0_PCI_EXCEPTION_MASK) || 3966 (icr0_remaining & I40E_PFINT_ICR0_ECC_ERR_MASK)) { 3967 dev_info(&pf->pdev->dev, "device will be reset\n"); 3968 set_bit(__I40E_PF_RESET_REQUESTED, pf->state); 3969 i40e_service_event_schedule(pf); 3970 } 3971 ena_mask &= ~icr0_remaining; 3972 } 3973 ret = IRQ_HANDLED; 3974 3975 enable_intr: 3976 /* re-enable interrupt causes */ 3977 wr32(hw, I40E_PFINT_ICR0_ENA, ena_mask); 3978 if (!test_bit(__I40E_DOWN, pf->state)) { 3979 i40e_service_event_schedule(pf); 3980 i40e_irq_dynamic_enable_icr0(pf); 3981 } 3982 3983 return ret; 3984 } 3985 3986 /** 3987 * i40e_clean_fdir_tx_irq - Reclaim resources after transmit completes 3988 * @tx_ring: tx ring to clean 3989 * @budget: how many cleans we're allowed 3990 * 3991 * Returns true if there's any budget left (e.g. the clean is finished) 3992 **/ 3993 static bool i40e_clean_fdir_tx_irq(struct i40e_ring *tx_ring, int budget) 3994 { 3995 struct i40e_vsi *vsi = tx_ring->vsi; 3996 u16 i = tx_ring->next_to_clean; 3997 struct i40e_tx_buffer *tx_buf; 3998 struct i40e_tx_desc *tx_desc; 3999 4000 tx_buf = &tx_ring->tx_bi[i]; 4001 tx_desc = I40E_TX_DESC(tx_ring, i); 4002 i -= tx_ring->count; 4003 4004 do { 4005 struct i40e_tx_desc *eop_desc = tx_buf->next_to_watch; 4006 4007 /* if next_to_watch is not set then there is no work pending */ 4008 if (!eop_desc) 4009 break; 4010 4011 /* prevent any other reads prior to eop_desc */ 4012 smp_rmb(); 4013 4014 /* if the descriptor isn't done, no work yet to do */ 4015 if (!(eop_desc->cmd_type_offset_bsz & 4016 cpu_to_le64(I40E_TX_DESC_DTYPE_DESC_DONE))) 4017 break; 4018 4019 /* clear next_to_watch to prevent false hangs */ 4020 tx_buf->next_to_watch = NULL; 4021 4022 tx_desc->buffer_addr = 0; 4023 tx_desc->cmd_type_offset_bsz = 0; 4024 /* move past filter desc */ 4025 tx_buf++; 4026 tx_desc++; 4027 i++; 4028 if (unlikely(!i)) { 4029 i -= tx_ring->count; 4030 tx_buf = tx_ring->tx_bi; 4031 tx_desc = I40E_TX_DESC(tx_ring, 0); 4032 } 4033 /* unmap skb header data */ 4034 dma_unmap_single(tx_ring->dev, 4035 dma_unmap_addr(tx_buf, dma), 4036 dma_unmap_len(tx_buf, len), 4037 DMA_TO_DEVICE); 4038 if (tx_buf->tx_flags & I40E_TX_FLAGS_FD_SB) 4039 kfree(tx_buf->raw_buf); 4040 4041 tx_buf->raw_buf = NULL; 4042 tx_buf->tx_flags = 0; 4043 tx_buf->next_to_watch = NULL; 4044 dma_unmap_len_set(tx_buf, len, 0); 4045 tx_desc->buffer_addr = 0; 4046 tx_desc->cmd_type_offset_bsz = 0; 4047 4048 /* move us past the eop_desc for start of next FD desc */ 4049 tx_buf++; 4050 tx_desc++; 4051 i++; 4052 if (unlikely(!i)) { 4053 i -= tx_ring->count; 4054 tx_buf = tx_ring->tx_bi; 4055 tx_desc = I40E_TX_DESC(tx_ring, 0); 4056 } 4057 4058 /* update budget accounting */ 4059 budget--; 4060 } while (likely(budget)); 4061 4062 i += tx_ring->count; 4063 tx_ring->next_to_clean = i; 4064 4065 if (vsi->back->flags & I40E_FLAG_MSIX_ENABLED) 4066 i40e_irq_dynamic_enable(vsi, tx_ring->q_vector->v_idx); 4067 4068 return budget > 0; 4069 } 4070 4071 /** 4072 * i40e_fdir_clean_ring - Interrupt Handler for FDIR SB ring 4073 * @irq: interrupt number 4074 * @data: pointer to a q_vector 4075 **/ 4076 static irqreturn_t i40e_fdir_clean_ring(int irq, void *data) 4077 { 4078 struct i40e_q_vector *q_vector = data; 4079 struct i40e_vsi *vsi; 4080 4081 if (!q_vector->tx.ring) 4082 return IRQ_HANDLED; 4083 4084 vsi = q_vector->tx.ring->vsi; 4085 i40e_clean_fdir_tx_irq(q_vector->tx.ring, vsi->work_limit); 4086 4087 return IRQ_HANDLED; 4088 } 4089 4090 /** 4091 * i40e_map_vector_to_qp - Assigns the queue pair to the vector 4092 * @vsi: the VSI being configured 4093 * @v_idx: vector index 4094 * @qp_idx: queue pair index 4095 **/ 4096 static void i40e_map_vector_to_qp(struct i40e_vsi *vsi, int v_idx, int qp_idx) 4097 { 4098 struct i40e_q_vector *q_vector = vsi->q_vectors[v_idx]; 4099 struct i40e_ring *tx_ring = vsi->tx_rings[qp_idx]; 4100 struct i40e_ring *rx_ring = vsi->rx_rings[qp_idx]; 4101 4102 tx_ring->q_vector = q_vector; 4103 tx_ring->next = q_vector->tx.ring; 4104 q_vector->tx.ring = tx_ring; 4105 q_vector->tx.count++; 4106 4107 /* Place XDP Tx ring in the same q_vector ring list as regular Tx */ 4108 if (i40e_enabled_xdp_vsi(vsi)) { 4109 struct i40e_ring *xdp_ring = vsi->xdp_rings[qp_idx]; 4110 4111 xdp_ring->q_vector = q_vector; 4112 xdp_ring->next = q_vector->tx.ring; 4113 q_vector->tx.ring = xdp_ring; 4114 q_vector->tx.count++; 4115 } 4116 4117 rx_ring->q_vector = q_vector; 4118 rx_ring->next = q_vector->rx.ring; 4119 q_vector->rx.ring = rx_ring; 4120 q_vector->rx.count++; 4121 } 4122 4123 /** 4124 * i40e_vsi_map_rings_to_vectors - Maps descriptor rings to vectors 4125 * @vsi: the VSI being configured 4126 * 4127 * This function maps descriptor rings to the queue-specific vectors 4128 * we were allotted through the MSI-X enabling code. Ideally, we'd have 4129 * one vector per queue pair, but on a constrained vector budget, we 4130 * group the queue pairs as "efficiently" as possible. 4131 **/ 4132 static void i40e_vsi_map_rings_to_vectors(struct i40e_vsi *vsi) 4133 { 4134 int qp_remaining = vsi->num_queue_pairs; 4135 int q_vectors = vsi->num_q_vectors; 4136 int num_ringpairs; 4137 int v_start = 0; 4138 int qp_idx = 0; 4139 4140 /* If we don't have enough vectors for a 1-to-1 mapping, we'll have to 4141 * group them so there are multiple queues per vector. 4142 * It is also important to go through all the vectors available to be 4143 * sure that if we don't use all the vectors, that the remaining vectors 4144 * are cleared. This is especially important when decreasing the 4145 * number of queues in use. 4146 */ 4147 for (; v_start < q_vectors; v_start++) { 4148 struct i40e_q_vector *q_vector = vsi->q_vectors[v_start]; 4149 4150 num_ringpairs = DIV_ROUND_UP(qp_remaining, q_vectors - v_start); 4151 4152 q_vector->num_ringpairs = num_ringpairs; 4153 q_vector->reg_idx = q_vector->v_idx + vsi->base_vector - 1; 4154 4155 q_vector->rx.count = 0; 4156 q_vector->tx.count = 0; 4157 q_vector->rx.ring = NULL; 4158 q_vector->tx.ring = NULL; 4159 4160 while (num_ringpairs--) { 4161 i40e_map_vector_to_qp(vsi, v_start, qp_idx); 4162 qp_idx++; 4163 qp_remaining--; 4164 } 4165 } 4166 } 4167 4168 /** 4169 * i40e_vsi_request_irq - Request IRQ from the OS 4170 * @vsi: the VSI being configured 4171 * @basename: name for the vector 4172 **/ 4173 static int i40e_vsi_request_irq(struct i40e_vsi *vsi, char *basename) 4174 { 4175 struct i40e_pf *pf = vsi->back; 4176 int err; 4177 4178 if (pf->flags & I40E_FLAG_MSIX_ENABLED) 4179 err = i40e_vsi_request_irq_msix(vsi, basename); 4180 else if (pf->flags & I40E_FLAG_MSI_ENABLED) 4181 err = request_irq(pf->pdev->irq, i40e_intr, 0, 4182 pf->int_name, pf); 4183 else 4184 err = request_irq(pf->pdev->irq, i40e_intr, IRQF_SHARED, 4185 pf->int_name, pf); 4186 4187 if (err) 4188 dev_info(&pf->pdev->dev, "request_irq failed, Error %d\n", err); 4189 4190 return err; 4191 } 4192 4193 #ifdef CONFIG_NET_POLL_CONTROLLER 4194 /** 4195 * i40e_netpoll - A Polling 'interrupt' handler 4196 * @netdev: network interface device structure 4197 * 4198 * This is used by netconsole to send skbs without having to re-enable 4199 * interrupts. It's not called while the normal interrupt routine is executing. 4200 **/ 4201 static void i40e_netpoll(struct net_device *netdev) 4202 { 4203 struct i40e_netdev_priv *np = netdev_priv(netdev); 4204 struct i40e_vsi *vsi = np->vsi; 4205 struct i40e_pf *pf = vsi->back; 4206 int i; 4207 4208 /* if interface is down do nothing */ 4209 if (test_bit(__I40E_VSI_DOWN, vsi->state)) 4210 return; 4211 4212 if (pf->flags & I40E_FLAG_MSIX_ENABLED) { 4213 for (i = 0; i < vsi->num_q_vectors; i++) 4214 i40e_msix_clean_rings(0, vsi->q_vectors[i]); 4215 } else { 4216 i40e_intr(pf->pdev->irq, netdev); 4217 } 4218 } 4219 #endif 4220 4221 #define I40E_QTX_ENA_WAIT_COUNT 50 4222 4223 /** 4224 * i40e_pf_txq_wait - Wait for a PF's Tx queue to be enabled or disabled 4225 * @pf: the PF being configured 4226 * @pf_q: the PF queue 4227 * @enable: enable or disable state of the queue 4228 * 4229 * This routine will wait for the given Tx queue of the PF to reach the 4230 * enabled or disabled state. 4231 * Returns -ETIMEDOUT in case of failing to reach the requested state after 4232 * multiple retries; else will return 0 in case of success. 4233 **/ 4234 static int i40e_pf_txq_wait(struct i40e_pf *pf, int pf_q, bool enable) 4235 { 4236 int i; 4237 u32 tx_reg; 4238 4239 for (i = 0; i < I40E_QUEUE_WAIT_RETRY_LIMIT; i++) { 4240 tx_reg = rd32(&pf->hw, I40E_QTX_ENA(pf_q)); 4241 if (enable == !!(tx_reg & I40E_QTX_ENA_QENA_STAT_MASK)) 4242 break; 4243 4244 usleep_range(10, 20); 4245 } 4246 if (i >= I40E_QUEUE_WAIT_RETRY_LIMIT) 4247 return -ETIMEDOUT; 4248 4249 return 0; 4250 } 4251 4252 /** 4253 * i40e_control_tx_q - Start or stop a particular Tx queue 4254 * @pf: the PF structure 4255 * @pf_q: the PF queue to configure 4256 * @enable: start or stop the queue 4257 * 4258 * This function enables or disables a single queue. Note that any delay 4259 * required after the operation is expected to be handled by the caller of 4260 * this function. 4261 **/ 4262 static void i40e_control_tx_q(struct i40e_pf *pf, int pf_q, bool enable) 4263 { 4264 struct i40e_hw *hw = &pf->hw; 4265 u32 tx_reg; 4266 int i; 4267 4268 /* warn the TX unit of coming changes */ 4269 i40e_pre_tx_queue_cfg(&pf->hw, pf_q, enable); 4270 if (!enable) 4271 usleep_range(10, 20); 4272 4273 for (i = 0; i < I40E_QTX_ENA_WAIT_COUNT; i++) { 4274 tx_reg = rd32(hw, I40E_QTX_ENA(pf_q)); 4275 if (((tx_reg >> I40E_QTX_ENA_QENA_REQ_SHIFT) & 1) == 4276 ((tx_reg >> I40E_QTX_ENA_QENA_STAT_SHIFT) & 1)) 4277 break; 4278 usleep_range(1000, 2000); 4279 } 4280 4281 /* Skip if the queue is already in the requested state */ 4282 if (enable == !!(tx_reg & I40E_QTX_ENA_QENA_STAT_MASK)) 4283 return; 4284 4285 /* turn on/off the queue */ 4286 if (enable) { 4287 wr32(hw, I40E_QTX_HEAD(pf_q), 0); 4288 tx_reg |= I40E_QTX_ENA_QENA_REQ_MASK; 4289 } else { 4290 tx_reg &= ~I40E_QTX_ENA_QENA_REQ_MASK; 4291 } 4292 4293 wr32(hw, I40E_QTX_ENA(pf_q), tx_reg); 4294 } 4295 4296 /** 4297 * i40e_control_wait_tx_q - Start/stop Tx queue and wait for completion 4298 * @seid: VSI SEID 4299 * @pf: the PF structure 4300 * @pf_q: the PF queue to configure 4301 * @is_xdp: true if the queue is used for XDP 4302 * @enable: start or stop the queue 4303 **/ 4304 int i40e_control_wait_tx_q(int seid, struct i40e_pf *pf, int pf_q, 4305 bool is_xdp, bool enable) 4306 { 4307 int ret; 4308 4309 i40e_control_tx_q(pf, pf_q, enable); 4310 4311 /* wait for the change to finish */ 4312 ret = i40e_pf_txq_wait(pf, pf_q, enable); 4313 if (ret) { 4314 dev_info(&pf->pdev->dev, 4315 "VSI seid %d %sTx ring %d %sable timeout\n", 4316 seid, (is_xdp ? "XDP " : ""), pf_q, 4317 (enable ? "en" : "dis")); 4318 } 4319 4320 return ret; 4321 } 4322 4323 /** 4324 * i40e_vsi_control_tx - Start or stop a VSI's rings 4325 * @vsi: the VSI being configured 4326 * @enable: start or stop the rings 4327 **/ 4328 static int i40e_vsi_control_tx(struct i40e_vsi *vsi, bool enable) 4329 { 4330 struct i40e_pf *pf = vsi->back; 4331 int i, pf_q, ret = 0; 4332 4333 pf_q = vsi->base_queue; 4334 for (i = 0; i < vsi->num_queue_pairs; i++, pf_q++) { 4335 ret = i40e_control_wait_tx_q(vsi->seid, pf, 4336 pf_q, 4337 false /*is xdp*/, enable); 4338 if (ret) 4339 break; 4340 4341 if (!i40e_enabled_xdp_vsi(vsi)) 4342 continue; 4343 4344 ret = i40e_control_wait_tx_q(vsi->seid, pf, 4345 pf_q + vsi->alloc_queue_pairs, 4346 true /*is xdp*/, enable); 4347 if (ret) 4348 break; 4349 } 4350 return ret; 4351 } 4352 4353 /** 4354 * i40e_pf_rxq_wait - Wait for a PF's Rx queue to be enabled or disabled 4355 * @pf: the PF being configured 4356 * @pf_q: the PF queue 4357 * @enable: enable or disable state of the queue 4358 * 4359 * This routine will wait for the given Rx queue of the PF to reach the 4360 * enabled or disabled state. 4361 * Returns -ETIMEDOUT in case of failing to reach the requested state after 4362 * multiple retries; else will return 0 in case of success. 4363 **/ 4364 static int i40e_pf_rxq_wait(struct i40e_pf *pf, int pf_q, bool enable) 4365 { 4366 int i; 4367 u32 rx_reg; 4368 4369 for (i = 0; i < I40E_QUEUE_WAIT_RETRY_LIMIT; i++) { 4370 rx_reg = rd32(&pf->hw, I40E_QRX_ENA(pf_q)); 4371 if (enable == !!(rx_reg & I40E_QRX_ENA_QENA_STAT_MASK)) 4372 break; 4373 4374 usleep_range(10, 20); 4375 } 4376 if (i >= I40E_QUEUE_WAIT_RETRY_LIMIT) 4377 return -ETIMEDOUT; 4378 4379 return 0; 4380 } 4381 4382 /** 4383 * i40e_control_rx_q - Start or stop a particular Rx queue 4384 * @pf: the PF structure 4385 * @pf_q: the PF queue to configure 4386 * @enable: start or stop the queue 4387 * 4388 * This function enables or disables a single queue. Note that 4389 * any delay required after the operation is expected to be 4390 * handled by the caller of this function. 4391 **/ 4392 static void i40e_control_rx_q(struct i40e_pf *pf, int pf_q, bool enable) 4393 { 4394 struct i40e_hw *hw = &pf->hw; 4395 u32 rx_reg; 4396 int i; 4397 4398 for (i = 0; i < I40E_QTX_ENA_WAIT_COUNT; i++) { 4399 rx_reg = rd32(hw, I40E_QRX_ENA(pf_q)); 4400 if (((rx_reg >> I40E_QRX_ENA_QENA_REQ_SHIFT) & 1) == 4401 ((rx_reg >> I40E_QRX_ENA_QENA_STAT_SHIFT) & 1)) 4402 break; 4403 usleep_range(1000, 2000); 4404 } 4405 4406 /* Skip if the queue is already in the requested state */ 4407 if (enable == !!(rx_reg & I40E_QRX_ENA_QENA_STAT_MASK)) 4408 return; 4409 4410 /* turn on/off the queue */ 4411 if (enable) 4412 rx_reg |= I40E_QRX_ENA_QENA_REQ_MASK; 4413 else 4414 rx_reg &= ~I40E_QRX_ENA_QENA_REQ_MASK; 4415 4416 wr32(hw, I40E_QRX_ENA(pf_q), rx_reg); 4417 } 4418 4419 /** 4420 * i40e_control_wait_rx_q 4421 * @pf: the PF structure 4422 * @pf_q: queue being configured 4423 * @enable: start or stop the rings 4424 * 4425 * This function enables or disables a single queue along with waiting 4426 * for the change to finish. The caller of this function should handle 4427 * the delays needed in the case of disabling queues. 4428 **/ 4429 int i40e_control_wait_rx_q(struct i40e_pf *pf, int pf_q, bool enable) 4430 { 4431 int ret = 0; 4432 4433 i40e_control_rx_q(pf, pf_q, enable); 4434 4435 /* wait for the change to finish */ 4436 ret = i40e_pf_rxq_wait(pf, pf_q, enable); 4437 if (ret) 4438 return ret; 4439 4440 return ret; 4441 } 4442 4443 /** 4444 * i40e_vsi_control_rx - Start or stop a VSI's rings 4445 * @vsi: the VSI being configured 4446 * @enable: start or stop the rings 4447 **/ 4448 static int i40e_vsi_control_rx(struct i40e_vsi *vsi, bool enable) 4449 { 4450 struct i40e_pf *pf = vsi->back; 4451 int i, pf_q, ret = 0; 4452 4453 pf_q = vsi->base_queue; 4454 for (i = 0; i < vsi->num_queue_pairs; i++, pf_q++) { 4455 ret = i40e_control_wait_rx_q(pf, pf_q, enable); 4456 if (ret) { 4457 dev_info(&pf->pdev->dev, 4458 "VSI seid %d Rx ring %d %sable timeout\n", 4459 vsi->seid, pf_q, (enable ? "en" : "dis")); 4460 break; 4461 } 4462 } 4463 4464 /* Due to HW errata, on Rx disable only, the register can indicate done 4465 * before it really is. Needs 50ms to be sure 4466 */ 4467 if (!enable) 4468 mdelay(50); 4469 4470 return ret; 4471 } 4472 4473 /** 4474 * i40e_vsi_start_rings - Start a VSI's rings 4475 * @vsi: the VSI being configured 4476 **/ 4477 int i40e_vsi_start_rings(struct i40e_vsi *vsi) 4478 { 4479 int ret = 0; 4480 4481 /* do rx first for enable and last for disable */ 4482 ret = i40e_vsi_control_rx(vsi, true); 4483 if (ret) 4484 return ret; 4485 ret = i40e_vsi_control_tx(vsi, true); 4486 4487 return ret; 4488 } 4489 4490 /** 4491 * i40e_vsi_stop_rings - Stop a VSI's rings 4492 * @vsi: the VSI being configured 4493 **/ 4494 void i40e_vsi_stop_rings(struct i40e_vsi *vsi) 4495 { 4496 /* When port TX is suspended, don't wait */ 4497 if (test_bit(__I40E_PORT_SUSPENDED, vsi->back->state)) 4498 return i40e_vsi_stop_rings_no_wait(vsi); 4499 4500 /* do rx first for enable and last for disable 4501 * Ignore return value, we need to shutdown whatever we can 4502 */ 4503 i40e_vsi_control_tx(vsi, false); 4504 i40e_vsi_control_rx(vsi, false); 4505 } 4506 4507 /** 4508 * i40e_vsi_stop_rings_no_wait - Stop a VSI's rings and do not delay 4509 * @vsi: the VSI being shutdown 4510 * 4511 * This function stops all the rings for a VSI but does not delay to verify 4512 * that rings have been disabled. It is expected that the caller is shutting 4513 * down multiple VSIs at once and will delay together for all the VSIs after 4514 * initiating the shutdown. This is particularly useful for shutting down lots 4515 * of VFs together. Otherwise, a large delay can be incurred while configuring 4516 * each VSI in serial. 4517 **/ 4518 void i40e_vsi_stop_rings_no_wait(struct i40e_vsi *vsi) 4519 { 4520 struct i40e_pf *pf = vsi->back; 4521 int i, pf_q; 4522 4523 pf_q = vsi->base_queue; 4524 for (i = 0; i < vsi->num_queue_pairs; i++, pf_q++) { 4525 i40e_control_tx_q(pf, pf_q, false); 4526 i40e_control_rx_q(pf, pf_q, false); 4527 } 4528 } 4529 4530 /** 4531 * i40e_vsi_free_irq - Free the irq association with the OS 4532 * @vsi: the VSI being configured 4533 **/ 4534 static void i40e_vsi_free_irq(struct i40e_vsi *vsi) 4535 { 4536 struct i40e_pf *pf = vsi->back; 4537 struct i40e_hw *hw = &pf->hw; 4538 int base = vsi->base_vector; 4539 u32 val, qp; 4540 int i; 4541 4542 if (pf->flags & I40E_FLAG_MSIX_ENABLED) { 4543 if (!vsi->q_vectors) 4544 return; 4545 4546 if (!vsi->irqs_ready) 4547 return; 4548 4549 vsi->irqs_ready = false; 4550 for (i = 0; i < vsi->num_q_vectors; i++) { 4551 int irq_num; 4552 u16 vector; 4553 4554 vector = i + base; 4555 irq_num = pf->msix_entries[vector].vector; 4556 4557 /* free only the irqs that were actually requested */ 4558 if (!vsi->q_vectors[i] || 4559 !vsi->q_vectors[i]->num_ringpairs) 4560 continue; 4561 4562 /* clear the affinity notifier in the IRQ descriptor */ 4563 irq_set_affinity_notifier(irq_num, NULL); 4564 /* remove our suggested affinity mask for this IRQ */ 4565 irq_set_affinity_hint(irq_num, NULL); 4566 synchronize_irq(irq_num); 4567 free_irq(irq_num, vsi->q_vectors[i]); 4568 4569 /* Tear down the interrupt queue link list 4570 * 4571 * We know that they come in pairs and always 4572 * the Rx first, then the Tx. To clear the 4573 * link list, stick the EOL value into the 4574 * next_q field of the registers. 4575 */ 4576 val = rd32(hw, I40E_PFINT_LNKLSTN(vector - 1)); 4577 qp = (val & I40E_PFINT_LNKLSTN_FIRSTQ_INDX_MASK) 4578 >> I40E_PFINT_LNKLSTN_FIRSTQ_INDX_SHIFT; 4579 val |= I40E_QUEUE_END_OF_LIST 4580 << I40E_PFINT_LNKLSTN_FIRSTQ_INDX_SHIFT; 4581 wr32(hw, I40E_PFINT_LNKLSTN(vector - 1), val); 4582 4583 while (qp != I40E_QUEUE_END_OF_LIST) { 4584 u32 next; 4585 4586 val = rd32(hw, I40E_QINT_RQCTL(qp)); 4587 4588 val &= ~(I40E_QINT_RQCTL_MSIX_INDX_MASK | 4589 I40E_QINT_RQCTL_MSIX0_INDX_MASK | 4590 I40E_QINT_RQCTL_CAUSE_ENA_MASK | 4591 I40E_QINT_RQCTL_INTEVENT_MASK); 4592 4593 val |= (I40E_QINT_RQCTL_ITR_INDX_MASK | 4594 I40E_QINT_RQCTL_NEXTQ_INDX_MASK); 4595 4596 wr32(hw, I40E_QINT_RQCTL(qp), val); 4597 4598 val = rd32(hw, I40E_QINT_TQCTL(qp)); 4599 4600 next = (val & I40E_QINT_TQCTL_NEXTQ_INDX_MASK) 4601 >> I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT; 4602 4603 val &= ~(I40E_QINT_TQCTL_MSIX_INDX_MASK | 4604 I40E_QINT_TQCTL_MSIX0_INDX_MASK | 4605 I40E_QINT_TQCTL_CAUSE_ENA_MASK | 4606 I40E_QINT_TQCTL_INTEVENT_MASK); 4607 4608 val |= (I40E_QINT_TQCTL_ITR_INDX_MASK | 4609 I40E_QINT_TQCTL_NEXTQ_INDX_MASK); 4610 4611 wr32(hw, I40E_QINT_TQCTL(qp), val); 4612 qp = next; 4613 } 4614 } 4615 } else { 4616 free_irq(pf->pdev->irq, pf); 4617 4618 val = rd32(hw, I40E_PFINT_LNKLST0); 4619 qp = (val & I40E_PFINT_LNKLSTN_FIRSTQ_INDX_MASK) 4620 >> I40E_PFINT_LNKLSTN_FIRSTQ_INDX_SHIFT; 4621 val |= I40E_QUEUE_END_OF_LIST 4622 << I40E_PFINT_LNKLST0_FIRSTQ_INDX_SHIFT; 4623 wr32(hw, I40E_PFINT_LNKLST0, val); 4624 4625 val = rd32(hw, I40E_QINT_RQCTL(qp)); 4626 val &= ~(I40E_QINT_RQCTL_MSIX_INDX_MASK | 4627 I40E_QINT_RQCTL_MSIX0_INDX_MASK | 4628 I40E_QINT_RQCTL_CAUSE_ENA_MASK | 4629 I40E_QINT_RQCTL_INTEVENT_MASK); 4630 4631 val |= (I40E_QINT_RQCTL_ITR_INDX_MASK | 4632 I40E_QINT_RQCTL_NEXTQ_INDX_MASK); 4633 4634 wr32(hw, I40E_QINT_RQCTL(qp), val); 4635 4636 val = rd32(hw, I40E_QINT_TQCTL(qp)); 4637 4638 val &= ~(I40E_QINT_TQCTL_MSIX_INDX_MASK | 4639 I40E_QINT_TQCTL_MSIX0_INDX_MASK | 4640 I40E_QINT_TQCTL_CAUSE_ENA_MASK | 4641 I40E_QINT_TQCTL_INTEVENT_MASK); 4642 4643 val |= (I40E_QINT_TQCTL_ITR_INDX_MASK | 4644 I40E_QINT_TQCTL_NEXTQ_INDX_MASK); 4645 4646 wr32(hw, I40E_QINT_TQCTL(qp), val); 4647 } 4648 } 4649 4650 /** 4651 * i40e_free_q_vector - Free memory allocated for specific interrupt vector 4652 * @vsi: the VSI being configured 4653 * @v_idx: Index of vector to be freed 4654 * 4655 * This function frees the memory allocated to the q_vector. In addition if 4656 * NAPI is enabled it will delete any references to the NAPI struct prior 4657 * to freeing the q_vector. 4658 **/ 4659 static void i40e_free_q_vector(struct i40e_vsi *vsi, int v_idx) 4660 { 4661 struct i40e_q_vector *q_vector = vsi->q_vectors[v_idx]; 4662 struct i40e_ring *ring; 4663 4664 if (!q_vector) 4665 return; 4666 4667 /* disassociate q_vector from rings */ 4668 i40e_for_each_ring(ring, q_vector->tx) 4669 ring->q_vector = NULL; 4670 4671 i40e_for_each_ring(ring, q_vector->rx) 4672 ring->q_vector = NULL; 4673 4674 /* only VSI w/ an associated netdev is set up w/ NAPI */ 4675 if (vsi->netdev) 4676 netif_napi_del(&q_vector->napi); 4677 4678 vsi->q_vectors[v_idx] = NULL; 4679 4680 kfree_rcu(q_vector, rcu); 4681 } 4682 4683 /** 4684 * i40e_vsi_free_q_vectors - Free memory allocated for interrupt vectors 4685 * @vsi: the VSI being un-configured 4686 * 4687 * This frees the memory allocated to the q_vectors and 4688 * deletes references to the NAPI struct. 4689 **/ 4690 static void i40e_vsi_free_q_vectors(struct i40e_vsi *vsi) 4691 { 4692 int v_idx; 4693 4694 for (v_idx = 0; v_idx < vsi->num_q_vectors; v_idx++) 4695 i40e_free_q_vector(vsi, v_idx); 4696 } 4697 4698 /** 4699 * i40e_reset_interrupt_capability - Disable interrupt setup in OS 4700 * @pf: board private structure 4701 **/ 4702 static void i40e_reset_interrupt_capability(struct i40e_pf *pf) 4703 { 4704 /* If we're in Legacy mode, the interrupt was cleaned in vsi_close */ 4705 if (pf->flags & I40E_FLAG_MSIX_ENABLED) { 4706 pci_disable_msix(pf->pdev); 4707 kfree(pf->msix_entries); 4708 pf->msix_entries = NULL; 4709 kfree(pf->irq_pile); 4710 pf->irq_pile = NULL; 4711 } else if (pf->flags & I40E_FLAG_MSI_ENABLED) { 4712 pci_disable_msi(pf->pdev); 4713 } 4714 pf->flags &= ~(I40E_FLAG_MSIX_ENABLED | I40E_FLAG_MSI_ENABLED); 4715 } 4716 4717 /** 4718 * i40e_clear_interrupt_scheme - Clear the current interrupt scheme settings 4719 * @pf: board private structure 4720 * 4721 * We go through and clear interrupt specific resources and reset the structure 4722 * to pre-load conditions 4723 **/ 4724 static void i40e_clear_interrupt_scheme(struct i40e_pf *pf) 4725 { 4726 int i; 4727 4728 i40e_free_misc_vector(pf); 4729 4730 i40e_put_lump(pf->irq_pile, pf->iwarp_base_vector, 4731 I40E_IWARP_IRQ_PILE_ID); 4732 4733 i40e_put_lump(pf->irq_pile, 0, I40E_PILE_VALID_BIT-1); 4734 for (i = 0; i < pf->num_alloc_vsi; i++) 4735 if (pf->vsi[i]) 4736 i40e_vsi_free_q_vectors(pf->vsi[i]); 4737 i40e_reset_interrupt_capability(pf); 4738 } 4739 4740 /** 4741 * i40e_napi_enable_all - Enable NAPI for all q_vectors in the VSI 4742 * @vsi: the VSI being configured 4743 **/ 4744 static void i40e_napi_enable_all(struct i40e_vsi *vsi) 4745 { 4746 int q_idx; 4747 4748 if (!vsi->netdev) 4749 return; 4750 4751 for (q_idx = 0; q_idx < vsi->num_q_vectors; q_idx++) { 4752 struct i40e_q_vector *q_vector = vsi->q_vectors[q_idx]; 4753 4754 if (q_vector->rx.ring || q_vector->tx.ring) 4755 napi_enable(&q_vector->napi); 4756 } 4757 } 4758 4759 /** 4760 * i40e_napi_disable_all - Disable NAPI for all q_vectors in the VSI 4761 * @vsi: the VSI being configured 4762 **/ 4763 static void i40e_napi_disable_all(struct i40e_vsi *vsi) 4764 { 4765 int q_idx; 4766 4767 if (!vsi->netdev) 4768 return; 4769 4770 for (q_idx = 0; q_idx < vsi->num_q_vectors; q_idx++) { 4771 struct i40e_q_vector *q_vector = vsi->q_vectors[q_idx]; 4772 4773 if (q_vector->rx.ring || q_vector->tx.ring) 4774 napi_disable(&q_vector->napi); 4775 } 4776 } 4777 4778 /** 4779 * i40e_vsi_close - Shut down a VSI 4780 * @vsi: the vsi to be quelled 4781 **/ 4782 static void i40e_vsi_close(struct i40e_vsi *vsi) 4783 { 4784 struct i40e_pf *pf = vsi->back; 4785 if (!test_and_set_bit(__I40E_VSI_DOWN, vsi->state)) 4786 i40e_down(vsi); 4787 i40e_vsi_free_irq(vsi); 4788 i40e_vsi_free_tx_resources(vsi); 4789 i40e_vsi_free_rx_resources(vsi); 4790 vsi->current_netdev_flags = 0; 4791 set_bit(__I40E_CLIENT_SERVICE_REQUESTED, pf->state); 4792 if (test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state)) 4793 set_bit(__I40E_CLIENT_RESET, pf->state); 4794 } 4795 4796 /** 4797 * i40e_quiesce_vsi - Pause a given VSI 4798 * @vsi: the VSI being paused 4799 **/ 4800 static void i40e_quiesce_vsi(struct i40e_vsi *vsi) 4801 { 4802 if (test_bit(__I40E_VSI_DOWN, vsi->state)) 4803 return; 4804 4805 set_bit(__I40E_VSI_NEEDS_RESTART, vsi->state); 4806 if (vsi->netdev && netif_running(vsi->netdev)) 4807 vsi->netdev->netdev_ops->ndo_stop(vsi->netdev); 4808 else 4809 i40e_vsi_close(vsi); 4810 } 4811 4812 /** 4813 * i40e_unquiesce_vsi - Resume a given VSI 4814 * @vsi: the VSI being resumed 4815 **/ 4816 static void i40e_unquiesce_vsi(struct i40e_vsi *vsi) 4817 { 4818 if (!test_and_clear_bit(__I40E_VSI_NEEDS_RESTART, vsi->state)) 4819 return; 4820 4821 if (vsi->netdev && netif_running(vsi->netdev)) 4822 vsi->netdev->netdev_ops->ndo_open(vsi->netdev); 4823 else 4824 i40e_vsi_open(vsi); /* this clears the DOWN bit */ 4825 } 4826 4827 /** 4828 * i40e_pf_quiesce_all_vsi - Pause all VSIs on a PF 4829 * @pf: the PF 4830 **/ 4831 static void i40e_pf_quiesce_all_vsi(struct i40e_pf *pf) 4832 { 4833 int v; 4834 4835 for (v = 0; v < pf->num_alloc_vsi; v++) { 4836 if (pf->vsi[v]) 4837 i40e_quiesce_vsi(pf->vsi[v]); 4838 } 4839 } 4840 4841 /** 4842 * i40e_pf_unquiesce_all_vsi - Resume all VSIs on a PF 4843 * @pf: the PF 4844 **/ 4845 static void i40e_pf_unquiesce_all_vsi(struct i40e_pf *pf) 4846 { 4847 int v; 4848 4849 for (v = 0; v < pf->num_alloc_vsi; v++) { 4850 if (pf->vsi[v]) 4851 i40e_unquiesce_vsi(pf->vsi[v]); 4852 } 4853 } 4854 4855 /** 4856 * i40e_vsi_wait_queues_disabled - Wait for VSI's queues to be disabled 4857 * @vsi: the VSI being configured 4858 * 4859 * Wait until all queues on a given VSI have been disabled. 4860 **/ 4861 int i40e_vsi_wait_queues_disabled(struct i40e_vsi *vsi) 4862 { 4863 struct i40e_pf *pf = vsi->back; 4864 int i, pf_q, ret; 4865 4866 pf_q = vsi->base_queue; 4867 for (i = 0; i < vsi->num_queue_pairs; i++, pf_q++) { 4868 /* Check and wait for the Tx queue */ 4869 ret = i40e_pf_txq_wait(pf, pf_q, false); 4870 if (ret) { 4871 dev_info(&pf->pdev->dev, 4872 "VSI seid %d Tx ring %d disable timeout\n", 4873 vsi->seid, pf_q); 4874 return ret; 4875 } 4876 4877 if (!i40e_enabled_xdp_vsi(vsi)) 4878 goto wait_rx; 4879 4880 /* Check and wait for the XDP Tx queue */ 4881 ret = i40e_pf_txq_wait(pf, pf_q + vsi->alloc_queue_pairs, 4882 false); 4883 if (ret) { 4884 dev_info(&pf->pdev->dev, 4885 "VSI seid %d XDP Tx ring %d disable timeout\n", 4886 vsi->seid, pf_q); 4887 return ret; 4888 } 4889 wait_rx: 4890 /* Check and wait for the Rx queue */ 4891 ret = i40e_pf_rxq_wait(pf, pf_q, false); 4892 if (ret) { 4893 dev_info(&pf->pdev->dev, 4894 "VSI seid %d Rx ring %d disable timeout\n", 4895 vsi->seid, pf_q); 4896 return ret; 4897 } 4898 } 4899 4900 return 0; 4901 } 4902 4903 #ifdef CONFIG_I40E_DCB 4904 /** 4905 * i40e_pf_wait_queues_disabled - Wait for all queues of PF VSIs to be disabled 4906 * @pf: the PF 4907 * 4908 * This function waits for the queues to be in disabled state for all the 4909 * VSIs that are managed by this PF. 4910 **/ 4911 static int i40e_pf_wait_queues_disabled(struct i40e_pf *pf) 4912 { 4913 int v, ret = 0; 4914 4915 for (v = 0; v < pf->hw.func_caps.num_vsis; v++) { 4916 if (pf->vsi[v]) { 4917 ret = i40e_vsi_wait_queues_disabled(pf->vsi[v]); 4918 if (ret) 4919 break; 4920 } 4921 } 4922 4923 return ret; 4924 } 4925 4926 #endif 4927 4928 /** 4929 * i40e_get_iscsi_tc_map - Return TC map for iSCSI APP 4930 * @pf: pointer to PF 4931 * 4932 * Get TC map for ISCSI PF type that will include iSCSI TC 4933 * and LAN TC. 4934 **/ 4935 static u8 i40e_get_iscsi_tc_map(struct i40e_pf *pf) 4936 { 4937 struct i40e_dcb_app_priority_table app; 4938 struct i40e_hw *hw = &pf->hw; 4939 u8 enabled_tc = 1; /* TC0 is always enabled */ 4940 u8 tc, i; 4941 /* Get the iSCSI APP TLV */ 4942 struct i40e_dcbx_config *dcbcfg = &hw->local_dcbx_config; 4943 4944 for (i = 0; i < dcbcfg->numapps; i++) { 4945 app = dcbcfg->app[i]; 4946 if (app.selector == I40E_APP_SEL_TCPIP && 4947 app.protocolid == I40E_APP_PROTOID_ISCSI) { 4948 tc = dcbcfg->etscfg.prioritytable[app.priority]; 4949 enabled_tc |= BIT(tc); 4950 break; 4951 } 4952 } 4953 4954 return enabled_tc; 4955 } 4956 4957 /** 4958 * i40e_dcb_get_num_tc - Get the number of TCs from DCBx config 4959 * @dcbcfg: the corresponding DCBx configuration structure 4960 * 4961 * Return the number of TCs from given DCBx configuration 4962 **/ 4963 static u8 i40e_dcb_get_num_tc(struct i40e_dcbx_config *dcbcfg) 4964 { 4965 int i, tc_unused = 0; 4966 u8 num_tc = 0; 4967 u8 ret = 0; 4968 4969 /* Scan the ETS Config Priority Table to find 4970 * traffic class enabled for a given priority 4971 * and create a bitmask of enabled TCs 4972 */ 4973 for (i = 0; i < I40E_MAX_USER_PRIORITY; i++) 4974 num_tc |= BIT(dcbcfg->etscfg.prioritytable[i]); 4975 4976 /* Now scan the bitmask to check for 4977 * contiguous TCs starting with TC0 4978 */ 4979 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 4980 if (num_tc & BIT(i)) { 4981 if (!tc_unused) { 4982 ret++; 4983 } else { 4984 pr_err("Non-contiguous TC - Disabling DCB\n"); 4985 return 1; 4986 } 4987 } else { 4988 tc_unused = 1; 4989 } 4990 } 4991 4992 /* There is always at least TC0 */ 4993 if (!ret) 4994 ret = 1; 4995 4996 return ret; 4997 } 4998 4999 /** 5000 * i40e_dcb_get_enabled_tc - Get enabled traffic classes 5001 * @dcbcfg: the corresponding DCBx configuration structure 5002 * 5003 * Query the current DCB configuration and return the number of 5004 * traffic classes enabled from the given DCBX config 5005 **/ 5006 static u8 i40e_dcb_get_enabled_tc(struct i40e_dcbx_config *dcbcfg) 5007 { 5008 u8 num_tc = i40e_dcb_get_num_tc(dcbcfg); 5009 u8 enabled_tc = 1; 5010 u8 i; 5011 5012 for (i = 0; i < num_tc; i++) 5013 enabled_tc |= BIT(i); 5014 5015 return enabled_tc; 5016 } 5017 5018 /** 5019 * i40e_mqprio_get_enabled_tc - Get enabled traffic classes 5020 * @pf: PF being queried 5021 * 5022 * Query the current MQPRIO configuration and return the number of 5023 * traffic classes enabled. 5024 **/ 5025 static u8 i40e_mqprio_get_enabled_tc(struct i40e_pf *pf) 5026 { 5027 struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi]; 5028 u8 num_tc = vsi->mqprio_qopt.qopt.num_tc; 5029 u8 enabled_tc = 1, i; 5030 5031 for (i = 1; i < num_tc; i++) 5032 enabled_tc |= BIT(i); 5033 return enabled_tc; 5034 } 5035 5036 /** 5037 * i40e_pf_get_num_tc - Get enabled traffic classes for PF 5038 * @pf: PF being queried 5039 * 5040 * Return number of traffic classes enabled for the given PF 5041 **/ 5042 static u8 i40e_pf_get_num_tc(struct i40e_pf *pf) 5043 { 5044 struct i40e_hw *hw = &pf->hw; 5045 u8 i, enabled_tc = 1; 5046 u8 num_tc = 0; 5047 struct i40e_dcbx_config *dcbcfg = &hw->local_dcbx_config; 5048 5049 if (pf->flags & I40E_FLAG_TC_MQPRIO) 5050 return pf->vsi[pf->lan_vsi]->mqprio_qopt.qopt.num_tc; 5051 5052 /* If neither MQPRIO nor DCB is enabled, then always use single TC */ 5053 if (!(pf->flags & I40E_FLAG_DCB_ENABLED)) 5054 return 1; 5055 5056 /* SFP mode will be enabled for all TCs on port */ 5057 if (!(pf->flags & I40E_FLAG_MFP_ENABLED)) 5058 return i40e_dcb_get_num_tc(dcbcfg); 5059 5060 /* MFP mode return count of enabled TCs for this PF */ 5061 if (pf->hw.func_caps.iscsi) 5062 enabled_tc = i40e_get_iscsi_tc_map(pf); 5063 else 5064 return 1; /* Only TC0 */ 5065 5066 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 5067 if (enabled_tc & BIT(i)) 5068 num_tc++; 5069 } 5070 return num_tc; 5071 } 5072 5073 /** 5074 * i40e_pf_get_pf_tc_map - Get bitmap for enabled traffic classes 5075 * @pf: PF being queried 5076 * 5077 * Return a bitmap for enabled traffic classes for this PF. 5078 **/ 5079 static u8 i40e_pf_get_tc_map(struct i40e_pf *pf) 5080 { 5081 if (pf->flags & I40E_FLAG_TC_MQPRIO) 5082 return i40e_mqprio_get_enabled_tc(pf); 5083 5084 /* If neither MQPRIO nor DCB is enabled for this PF then just return 5085 * default TC 5086 */ 5087 if (!(pf->flags & I40E_FLAG_DCB_ENABLED)) 5088 return I40E_DEFAULT_TRAFFIC_CLASS; 5089 5090 /* SFP mode we want PF to be enabled for all TCs */ 5091 if (!(pf->flags & I40E_FLAG_MFP_ENABLED)) 5092 return i40e_dcb_get_enabled_tc(&pf->hw.local_dcbx_config); 5093 5094 /* MFP enabled and iSCSI PF type */ 5095 if (pf->hw.func_caps.iscsi) 5096 return i40e_get_iscsi_tc_map(pf); 5097 else 5098 return I40E_DEFAULT_TRAFFIC_CLASS; 5099 } 5100 5101 /** 5102 * i40e_vsi_get_bw_info - Query VSI BW Information 5103 * @vsi: the VSI being queried 5104 * 5105 * Returns 0 on success, negative value on failure 5106 **/ 5107 static int i40e_vsi_get_bw_info(struct i40e_vsi *vsi) 5108 { 5109 struct i40e_aqc_query_vsi_ets_sla_config_resp bw_ets_config = {0}; 5110 struct i40e_aqc_query_vsi_bw_config_resp bw_config = {0}; 5111 struct i40e_pf *pf = vsi->back; 5112 struct i40e_hw *hw = &pf->hw; 5113 i40e_status ret; 5114 u32 tc_bw_max; 5115 int i; 5116 5117 /* Get the VSI level BW configuration */ 5118 ret = i40e_aq_query_vsi_bw_config(hw, vsi->seid, &bw_config, NULL); 5119 if (ret) { 5120 dev_info(&pf->pdev->dev, 5121 "couldn't get PF vsi bw config, err %s aq_err %s\n", 5122 i40e_stat_str(&pf->hw, ret), 5123 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 5124 return -EINVAL; 5125 } 5126 5127 /* Get the VSI level BW configuration per TC */ 5128 ret = i40e_aq_query_vsi_ets_sla_config(hw, vsi->seid, &bw_ets_config, 5129 NULL); 5130 if (ret) { 5131 dev_info(&pf->pdev->dev, 5132 "couldn't get PF vsi ets bw config, err %s aq_err %s\n", 5133 i40e_stat_str(&pf->hw, ret), 5134 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 5135 return -EINVAL; 5136 } 5137 5138 if (bw_config.tc_valid_bits != bw_ets_config.tc_valid_bits) { 5139 dev_info(&pf->pdev->dev, 5140 "Enabled TCs mismatch from querying VSI BW info 0x%08x 0x%08x\n", 5141 bw_config.tc_valid_bits, 5142 bw_ets_config.tc_valid_bits); 5143 /* Still continuing */ 5144 } 5145 5146 vsi->bw_limit = le16_to_cpu(bw_config.port_bw_limit); 5147 vsi->bw_max_quanta = bw_config.max_bw; 5148 tc_bw_max = le16_to_cpu(bw_ets_config.tc_bw_max[0]) | 5149 (le16_to_cpu(bw_ets_config.tc_bw_max[1]) << 16); 5150 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 5151 vsi->bw_ets_share_credits[i] = bw_ets_config.share_credits[i]; 5152 vsi->bw_ets_limit_credits[i] = 5153 le16_to_cpu(bw_ets_config.credits[i]); 5154 /* 3 bits out of 4 for each TC */ 5155 vsi->bw_ets_max_quanta[i] = (u8)((tc_bw_max >> (i*4)) & 0x7); 5156 } 5157 5158 return 0; 5159 } 5160 5161 /** 5162 * i40e_vsi_configure_bw_alloc - Configure VSI BW allocation per TC 5163 * @vsi: the VSI being configured 5164 * @enabled_tc: TC bitmap 5165 * @bw_share: BW shared credits per TC 5166 * 5167 * Returns 0 on success, negative value on failure 5168 **/ 5169 static int i40e_vsi_configure_bw_alloc(struct i40e_vsi *vsi, u8 enabled_tc, 5170 u8 *bw_share) 5171 { 5172 struct i40e_aqc_configure_vsi_tc_bw_data bw_data; 5173 struct i40e_pf *pf = vsi->back; 5174 i40e_status ret; 5175 int i; 5176 5177 /* There is no need to reset BW when mqprio mode is on. */ 5178 if (pf->flags & I40E_FLAG_TC_MQPRIO) 5179 return 0; 5180 if (!vsi->mqprio_qopt.qopt.hw && !(pf->flags & I40E_FLAG_DCB_ENABLED)) { 5181 ret = i40e_set_bw_limit(vsi, vsi->seid, 0); 5182 if (ret) 5183 dev_info(&pf->pdev->dev, 5184 "Failed to reset tx rate for vsi->seid %u\n", 5185 vsi->seid); 5186 return ret; 5187 } 5188 bw_data.tc_valid_bits = enabled_tc; 5189 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) 5190 bw_data.tc_bw_credits[i] = bw_share[i]; 5191 5192 ret = i40e_aq_config_vsi_tc_bw(&pf->hw, vsi->seid, &bw_data, NULL); 5193 if (ret) { 5194 dev_info(&pf->pdev->dev, 5195 "AQ command Config VSI BW allocation per TC failed = %d\n", 5196 pf->hw.aq.asq_last_status); 5197 return -EINVAL; 5198 } 5199 5200 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) 5201 vsi->info.qs_handle[i] = bw_data.qs_handles[i]; 5202 5203 return 0; 5204 } 5205 5206 /** 5207 * i40e_vsi_config_netdev_tc - Setup the netdev TC configuration 5208 * @vsi: the VSI being configured 5209 * @enabled_tc: TC map to be enabled 5210 * 5211 **/ 5212 static void i40e_vsi_config_netdev_tc(struct i40e_vsi *vsi, u8 enabled_tc) 5213 { 5214 struct net_device *netdev = vsi->netdev; 5215 struct i40e_pf *pf = vsi->back; 5216 struct i40e_hw *hw = &pf->hw; 5217 u8 netdev_tc = 0; 5218 int i; 5219 struct i40e_dcbx_config *dcbcfg = &hw->local_dcbx_config; 5220 5221 if (!netdev) 5222 return; 5223 5224 if (!enabled_tc) { 5225 netdev_reset_tc(netdev); 5226 return; 5227 } 5228 5229 /* Set up actual enabled TCs on the VSI */ 5230 if (netdev_set_num_tc(netdev, vsi->tc_config.numtc)) 5231 return; 5232 5233 /* set per TC queues for the VSI */ 5234 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 5235 /* Only set TC queues for enabled tcs 5236 * 5237 * e.g. For a VSI that has TC0 and TC3 enabled the 5238 * enabled_tc bitmap would be 0x00001001; the driver 5239 * will set the numtc for netdev as 2 that will be 5240 * referenced by the netdev layer as TC 0 and 1. 5241 */ 5242 if (vsi->tc_config.enabled_tc & BIT(i)) 5243 netdev_set_tc_queue(netdev, 5244 vsi->tc_config.tc_info[i].netdev_tc, 5245 vsi->tc_config.tc_info[i].qcount, 5246 vsi->tc_config.tc_info[i].qoffset); 5247 } 5248 5249 if (pf->flags & I40E_FLAG_TC_MQPRIO) 5250 return; 5251 5252 /* Assign UP2TC map for the VSI */ 5253 for (i = 0; i < I40E_MAX_USER_PRIORITY; i++) { 5254 /* Get the actual TC# for the UP */ 5255 u8 ets_tc = dcbcfg->etscfg.prioritytable[i]; 5256 /* Get the mapped netdev TC# for the UP */ 5257 netdev_tc = vsi->tc_config.tc_info[ets_tc].netdev_tc; 5258 netdev_set_prio_tc_map(netdev, i, netdev_tc); 5259 } 5260 } 5261 5262 /** 5263 * i40e_vsi_update_queue_map - Update our copy of VSi info with new queue map 5264 * @vsi: the VSI being configured 5265 * @ctxt: the ctxt buffer returned from AQ VSI update param command 5266 **/ 5267 static void i40e_vsi_update_queue_map(struct i40e_vsi *vsi, 5268 struct i40e_vsi_context *ctxt) 5269 { 5270 /* copy just the sections touched not the entire info 5271 * since not all sections are valid as returned by 5272 * update vsi params 5273 */ 5274 vsi->info.mapping_flags = ctxt->info.mapping_flags; 5275 memcpy(&vsi->info.queue_mapping, 5276 &ctxt->info.queue_mapping, sizeof(vsi->info.queue_mapping)); 5277 memcpy(&vsi->info.tc_mapping, ctxt->info.tc_mapping, 5278 sizeof(vsi->info.tc_mapping)); 5279 } 5280 5281 /** 5282 * i40e_vsi_config_tc - Configure VSI Tx Scheduler for given TC map 5283 * @vsi: VSI to be configured 5284 * @enabled_tc: TC bitmap 5285 * 5286 * This configures a particular VSI for TCs that are mapped to the 5287 * given TC bitmap. It uses default bandwidth share for TCs across 5288 * VSIs to configure TC for a particular VSI. 5289 * 5290 * NOTE: 5291 * It is expected that the VSI queues have been quisced before calling 5292 * this function. 5293 **/ 5294 static int i40e_vsi_config_tc(struct i40e_vsi *vsi, u8 enabled_tc) 5295 { 5296 u8 bw_share[I40E_MAX_TRAFFIC_CLASS] = {0}; 5297 struct i40e_pf *pf = vsi->back; 5298 struct i40e_hw *hw = &pf->hw; 5299 struct i40e_vsi_context ctxt; 5300 int ret = 0; 5301 int i; 5302 5303 /* Check if enabled_tc is same as existing or new TCs */ 5304 if (vsi->tc_config.enabled_tc == enabled_tc && 5305 vsi->mqprio_qopt.mode != TC_MQPRIO_MODE_CHANNEL) 5306 return ret; 5307 5308 /* Enable ETS TCs with equal BW Share for now across all VSIs */ 5309 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 5310 if (enabled_tc & BIT(i)) 5311 bw_share[i] = 1; 5312 } 5313 5314 ret = i40e_vsi_configure_bw_alloc(vsi, enabled_tc, bw_share); 5315 if (ret) { 5316 struct i40e_aqc_query_vsi_bw_config_resp bw_config = {0}; 5317 5318 dev_info(&pf->pdev->dev, 5319 "Failed configuring TC map %d for VSI %d\n", 5320 enabled_tc, vsi->seid); 5321 ret = i40e_aq_query_vsi_bw_config(hw, vsi->seid, 5322 &bw_config, NULL); 5323 if (ret) { 5324 dev_info(&pf->pdev->dev, 5325 "Failed querying vsi bw info, err %s aq_err %s\n", 5326 i40e_stat_str(hw, ret), 5327 i40e_aq_str(hw, hw->aq.asq_last_status)); 5328 goto out; 5329 } 5330 if ((bw_config.tc_valid_bits & enabled_tc) != enabled_tc) { 5331 u8 valid_tc = bw_config.tc_valid_bits & enabled_tc; 5332 5333 if (!valid_tc) 5334 valid_tc = bw_config.tc_valid_bits; 5335 /* Always enable TC0, no matter what */ 5336 valid_tc |= 1; 5337 dev_info(&pf->pdev->dev, 5338 "Requested tc 0x%x, but FW reports 0x%x as valid. Attempting to use 0x%x.\n", 5339 enabled_tc, bw_config.tc_valid_bits, valid_tc); 5340 enabled_tc = valid_tc; 5341 } 5342 5343 ret = i40e_vsi_configure_bw_alloc(vsi, enabled_tc, bw_share); 5344 if (ret) { 5345 dev_err(&pf->pdev->dev, 5346 "Unable to configure TC map %d for VSI %d\n", 5347 enabled_tc, vsi->seid); 5348 goto out; 5349 } 5350 } 5351 5352 /* Update Queue Pairs Mapping for currently enabled UPs */ 5353 ctxt.seid = vsi->seid; 5354 ctxt.pf_num = vsi->back->hw.pf_id; 5355 ctxt.vf_num = 0; 5356 ctxt.uplink_seid = vsi->uplink_seid; 5357 ctxt.info = vsi->info; 5358 if (vsi->back->flags & I40E_FLAG_TC_MQPRIO) { 5359 ret = i40e_vsi_setup_queue_map_mqprio(vsi, &ctxt, enabled_tc); 5360 if (ret) 5361 goto out; 5362 } else { 5363 i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, false); 5364 } 5365 5366 /* On destroying the qdisc, reset vsi->rss_size, as number of enabled 5367 * queues changed. 5368 */ 5369 if (!vsi->mqprio_qopt.qopt.hw && vsi->reconfig_rss) { 5370 vsi->rss_size = min_t(int, vsi->back->alloc_rss_size, 5371 vsi->num_queue_pairs); 5372 ret = i40e_vsi_config_rss(vsi); 5373 if (ret) { 5374 dev_info(&vsi->back->pdev->dev, 5375 "Failed to reconfig rss for num_queues\n"); 5376 return ret; 5377 } 5378 vsi->reconfig_rss = false; 5379 } 5380 if (vsi->back->flags & I40E_FLAG_IWARP_ENABLED) { 5381 ctxt.info.valid_sections |= 5382 cpu_to_le16(I40E_AQ_VSI_PROP_QUEUE_OPT_VALID); 5383 ctxt.info.queueing_opt_flags |= I40E_AQ_VSI_QUE_OPT_TCP_ENA; 5384 } 5385 5386 /* Update the VSI after updating the VSI queue-mapping 5387 * information 5388 */ 5389 ret = i40e_aq_update_vsi_params(hw, &ctxt, NULL); 5390 if (ret) { 5391 dev_info(&pf->pdev->dev, 5392 "Update vsi tc config failed, err %s aq_err %s\n", 5393 i40e_stat_str(hw, ret), 5394 i40e_aq_str(hw, hw->aq.asq_last_status)); 5395 goto out; 5396 } 5397 /* update the local VSI info with updated queue map */ 5398 i40e_vsi_update_queue_map(vsi, &ctxt); 5399 vsi->info.valid_sections = 0; 5400 5401 /* Update current VSI BW information */ 5402 ret = i40e_vsi_get_bw_info(vsi); 5403 if (ret) { 5404 dev_info(&pf->pdev->dev, 5405 "Failed updating vsi bw info, err %s aq_err %s\n", 5406 i40e_stat_str(hw, ret), 5407 i40e_aq_str(hw, hw->aq.asq_last_status)); 5408 goto out; 5409 } 5410 5411 /* Update the netdev TC setup */ 5412 i40e_vsi_config_netdev_tc(vsi, enabled_tc); 5413 out: 5414 return ret; 5415 } 5416 5417 /** 5418 * i40e_get_link_speed - Returns link speed for the interface 5419 * @vsi: VSI to be configured 5420 * 5421 **/ 5422 static int i40e_get_link_speed(struct i40e_vsi *vsi) 5423 { 5424 struct i40e_pf *pf = vsi->back; 5425 5426 switch (pf->hw.phy.link_info.link_speed) { 5427 case I40E_LINK_SPEED_40GB: 5428 return 40000; 5429 case I40E_LINK_SPEED_25GB: 5430 return 25000; 5431 case I40E_LINK_SPEED_20GB: 5432 return 20000; 5433 case I40E_LINK_SPEED_10GB: 5434 return 10000; 5435 case I40E_LINK_SPEED_1GB: 5436 return 1000; 5437 default: 5438 return -EINVAL; 5439 } 5440 } 5441 5442 /** 5443 * i40e_set_bw_limit - setup BW limit for Tx traffic based on max_tx_rate 5444 * @vsi: VSI to be configured 5445 * @seid: seid of the channel/VSI 5446 * @max_tx_rate: max TX rate to be configured as BW limit 5447 * 5448 * Helper function to set BW limit for a given VSI 5449 **/ 5450 int i40e_set_bw_limit(struct i40e_vsi *vsi, u16 seid, u64 max_tx_rate) 5451 { 5452 struct i40e_pf *pf = vsi->back; 5453 u64 credits = 0; 5454 int speed = 0; 5455 int ret = 0; 5456 5457 speed = i40e_get_link_speed(vsi); 5458 if (max_tx_rate > speed) { 5459 dev_err(&pf->pdev->dev, 5460 "Invalid max tx rate %llu specified for VSI seid %d.", 5461 max_tx_rate, seid); 5462 return -EINVAL; 5463 } 5464 if (max_tx_rate && max_tx_rate < 50) { 5465 dev_warn(&pf->pdev->dev, 5466 "Setting max tx rate to minimum usable value of 50Mbps.\n"); 5467 max_tx_rate = 50; 5468 } 5469 5470 /* Tx rate credits are in values of 50Mbps, 0 is disabled */ 5471 credits = max_tx_rate; 5472 do_div(credits, I40E_BW_CREDIT_DIVISOR); 5473 ret = i40e_aq_config_vsi_bw_limit(&pf->hw, seid, credits, 5474 I40E_MAX_BW_INACTIVE_ACCUM, NULL); 5475 if (ret) 5476 dev_err(&pf->pdev->dev, 5477 "Failed set tx rate (%llu Mbps) for vsi->seid %u, err %s aq_err %s\n", 5478 max_tx_rate, seid, i40e_stat_str(&pf->hw, ret), 5479 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 5480 return ret; 5481 } 5482 5483 /** 5484 * i40e_remove_queue_channels - Remove queue channels for the TCs 5485 * @vsi: VSI to be configured 5486 * 5487 * Remove queue channels for the TCs 5488 **/ 5489 static void i40e_remove_queue_channels(struct i40e_vsi *vsi) 5490 { 5491 enum i40e_admin_queue_err last_aq_status; 5492 struct i40e_cloud_filter *cfilter; 5493 struct i40e_channel *ch, *ch_tmp; 5494 struct i40e_pf *pf = vsi->back; 5495 struct hlist_node *node; 5496 int ret, i; 5497 5498 /* Reset rss size that was stored when reconfiguring rss for 5499 * channel VSIs with non-power-of-2 queue count. 5500 */ 5501 vsi->current_rss_size = 0; 5502 5503 /* perform cleanup for channels if they exist */ 5504 if (list_empty(&vsi->ch_list)) 5505 return; 5506 5507 list_for_each_entry_safe(ch, ch_tmp, &vsi->ch_list, list) { 5508 struct i40e_vsi *p_vsi; 5509 5510 list_del(&ch->list); 5511 p_vsi = ch->parent_vsi; 5512 if (!p_vsi || !ch->initialized) { 5513 kfree(ch); 5514 continue; 5515 } 5516 /* Reset queue contexts */ 5517 for (i = 0; i < ch->num_queue_pairs; i++) { 5518 struct i40e_ring *tx_ring, *rx_ring; 5519 u16 pf_q; 5520 5521 pf_q = ch->base_queue + i; 5522 tx_ring = vsi->tx_rings[pf_q]; 5523 tx_ring->ch = NULL; 5524 5525 rx_ring = vsi->rx_rings[pf_q]; 5526 rx_ring->ch = NULL; 5527 } 5528 5529 /* Reset BW configured for this VSI via mqprio */ 5530 ret = i40e_set_bw_limit(vsi, ch->seid, 0); 5531 if (ret) 5532 dev_info(&vsi->back->pdev->dev, 5533 "Failed to reset tx rate for ch->seid %u\n", 5534 ch->seid); 5535 5536 /* delete cloud filters associated with this channel */ 5537 hlist_for_each_entry_safe(cfilter, node, 5538 &pf->cloud_filter_list, cloud_node) { 5539 if (cfilter->seid != ch->seid) 5540 continue; 5541 5542 hash_del(&cfilter->cloud_node); 5543 if (cfilter->dst_port) 5544 ret = i40e_add_del_cloud_filter_big_buf(vsi, 5545 cfilter, 5546 false); 5547 else 5548 ret = i40e_add_del_cloud_filter(vsi, cfilter, 5549 false); 5550 last_aq_status = pf->hw.aq.asq_last_status; 5551 if (ret) 5552 dev_info(&pf->pdev->dev, 5553 "Failed to delete cloud filter, err %s aq_err %s\n", 5554 i40e_stat_str(&pf->hw, ret), 5555 i40e_aq_str(&pf->hw, last_aq_status)); 5556 kfree(cfilter); 5557 } 5558 5559 /* delete VSI from FW */ 5560 ret = i40e_aq_delete_element(&vsi->back->hw, ch->seid, 5561 NULL); 5562 if (ret) 5563 dev_err(&vsi->back->pdev->dev, 5564 "unable to remove channel (%d) for parent VSI(%d)\n", 5565 ch->seid, p_vsi->seid); 5566 kfree(ch); 5567 } 5568 INIT_LIST_HEAD(&vsi->ch_list); 5569 } 5570 5571 /** 5572 * i40e_is_any_channel - channel exist or not 5573 * @vsi: ptr to VSI to which channels are associated with 5574 * 5575 * Returns true or false if channel(s) exist for associated VSI or not 5576 **/ 5577 static bool i40e_is_any_channel(struct i40e_vsi *vsi) 5578 { 5579 struct i40e_channel *ch, *ch_tmp; 5580 5581 list_for_each_entry_safe(ch, ch_tmp, &vsi->ch_list, list) { 5582 if (ch->initialized) 5583 return true; 5584 } 5585 5586 return false; 5587 } 5588 5589 /** 5590 * i40e_get_max_queues_for_channel 5591 * @vsi: ptr to VSI to which channels are associated with 5592 * 5593 * Helper function which returns max value among the queue counts set on the 5594 * channels/TCs created. 5595 **/ 5596 static int i40e_get_max_queues_for_channel(struct i40e_vsi *vsi) 5597 { 5598 struct i40e_channel *ch, *ch_tmp; 5599 int max = 0; 5600 5601 list_for_each_entry_safe(ch, ch_tmp, &vsi->ch_list, list) { 5602 if (!ch->initialized) 5603 continue; 5604 if (ch->num_queue_pairs > max) 5605 max = ch->num_queue_pairs; 5606 } 5607 5608 return max; 5609 } 5610 5611 /** 5612 * i40e_validate_num_queues - validate num_queues w.r.t channel 5613 * @pf: ptr to PF device 5614 * @num_queues: number of queues 5615 * @vsi: the parent VSI 5616 * @reconfig_rss: indicates should the RSS be reconfigured or not 5617 * 5618 * This function validates number of queues in the context of new channel 5619 * which is being established and determines if RSS should be reconfigured 5620 * or not for parent VSI. 5621 **/ 5622 static int i40e_validate_num_queues(struct i40e_pf *pf, int num_queues, 5623 struct i40e_vsi *vsi, bool *reconfig_rss) 5624 { 5625 int max_ch_queues; 5626 5627 if (!reconfig_rss) 5628 return -EINVAL; 5629 5630 *reconfig_rss = false; 5631 if (vsi->current_rss_size) { 5632 if (num_queues > vsi->current_rss_size) { 5633 dev_dbg(&pf->pdev->dev, 5634 "Error: num_queues (%d) > vsi's current_size(%d)\n", 5635 num_queues, vsi->current_rss_size); 5636 return -EINVAL; 5637 } else if ((num_queues < vsi->current_rss_size) && 5638 (!is_power_of_2(num_queues))) { 5639 dev_dbg(&pf->pdev->dev, 5640 "Error: num_queues (%d) < vsi's current_size(%d), but not power of 2\n", 5641 num_queues, vsi->current_rss_size); 5642 return -EINVAL; 5643 } 5644 } 5645 5646 if (!is_power_of_2(num_queues)) { 5647 /* Find the max num_queues configured for channel if channel 5648 * exist. 5649 * if channel exist, then enforce 'num_queues' to be more than 5650 * max ever queues configured for channel. 5651 */ 5652 max_ch_queues = i40e_get_max_queues_for_channel(vsi); 5653 if (num_queues < max_ch_queues) { 5654 dev_dbg(&pf->pdev->dev, 5655 "Error: num_queues (%d) < max queues configured for channel(%d)\n", 5656 num_queues, max_ch_queues); 5657 return -EINVAL; 5658 } 5659 *reconfig_rss = true; 5660 } 5661 5662 return 0; 5663 } 5664 5665 /** 5666 * i40e_vsi_reconfig_rss - reconfig RSS based on specified rss_size 5667 * @vsi: the VSI being setup 5668 * @rss_size: size of RSS, accordingly LUT gets reprogrammed 5669 * 5670 * This function reconfigures RSS by reprogramming LUTs using 'rss_size' 5671 **/ 5672 static int i40e_vsi_reconfig_rss(struct i40e_vsi *vsi, u16 rss_size) 5673 { 5674 struct i40e_pf *pf = vsi->back; 5675 u8 seed[I40E_HKEY_ARRAY_SIZE]; 5676 struct i40e_hw *hw = &pf->hw; 5677 int local_rss_size; 5678 u8 *lut; 5679 int ret; 5680 5681 if (!vsi->rss_size) 5682 return -EINVAL; 5683 5684 if (rss_size > vsi->rss_size) 5685 return -EINVAL; 5686 5687 local_rss_size = min_t(int, vsi->rss_size, rss_size); 5688 lut = kzalloc(vsi->rss_table_size, GFP_KERNEL); 5689 if (!lut) 5690 return -ENOMEM; 5691 5692 /* Ignoring user configured lut if there is one */ 5693 i40e_fill_rss_lut(pf, lut, vsi->rss_table_size, local_rss_size); 5694 5695 /* Use user configured hash key if there is one, otherwise 5696 * use default. 5697 */ 5698 if (vsi->rss_hkey_user) 5699 memcpy(seed, vsi->rss_hkey_user, I40E_HKEY_ARRAY_SIZE); 5700 else 5701 netdev_rss_key_fill((void *)seed, I40E_HKEY_ARRAY_SIZE); 5702 5703 ret = i40e_config_rss(vsi, seed, lut, vsi->rss_table_size); 5704 if (ret) { 5705 dev_info(&pf->pdev->dev, 5706 "Cannot set RSS lut, err %s aq_err %s\n", 5707 i40e_stat_str(hw, ret), 5708 i40e_aq_str(hw, hw->aq.asq_last_status)); 5709 kfree(lut); 5710 return ret; 5711 } 5712 kfree(lut); 5713 5714 /* Do the update w.r.t. storing rss_size */ 5715 if (!vsi->orig_rss_size) 5716 vsi->orig_rss_size = vsi->rss_size; 5717 vsi->current_rss_size = local_rss_size; 5718 5719 return ret; 5720 } 5721 5722 /** 5723 * i40e_channel_setup_queue_map - Setup a channel queue map 5724 * @pf: ptr to PF device 5725 * @vsi: the VSI being setup 5726 * @ctxt: VSI context structure 5727 * @ch: ptr to channel structure 5728 * 5729 * Setup queue map for a specific channel 5730 **/ 5731 static void i40e_channel_setup_queue_map(struct i40e_pf *pf, 5732 struct i40e_vsi_context *ctxt, 5733 struct i40e_channel *ch) 5734 { 5735 u16 qcount, qmap, sections = 0; 5736 u8 offset = 0; 5737 int pow; 5738 5739 sections = I40E_AQ_VSI_PROP_QUEUE_MAP_VALID; 5740 sections |= I40E_AQ_VSI_PROP_SCHED_VALID; 5741 5742 qcount = min_t(int, ch->num_queue_pairs, pf->num_lan_msix); 5743 ch->num_queue_pairs = qcount; 5744 5745 /* find the next higher power-of-2 of num queue pairs */ 5746 pow = ilog2(qcount); 5747 if (!is_power_of_2(qcount)) 5748 pow++; 5749 5750 qmap = (offset << I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT) | 5751 (pow << I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT); 5752 5753 /* Setup queue TC[0].qmap for given VSI context */ 5754 ctxt->info.tc_mapping[0] = cpu_to_le16(qmap); 5755 5756 ctxt->info.up_enable_bits = 0x1; /* TC0 enabled */ 5757 ctxt->info.mapping_flags |= cpu_to_le16(I40E_AQ_VSI_QUE_MAP_CONTIG); 5758 ctxt->info.queue_mapping[0] = cpu_to_le16(ch->base_queue); 5759 ctxt->info.valid_sections |= cpu_to_le16(sections); 5760 } 5761 5762 /** 5763 * i40e_add_channel - add a channel by adding VSI 5764 * @pf: ptr to PF device 5765 * @uplink_seid: underlying HW switching element (VEB) ID 5766 * @ch: ptr to channel structure 5767 * 5768 * Add a channel (VSI) using add_vsi and queue_map 5769 **/ 5770 static int i40e_add_channel(struct i40e_pf *pf, u16 uplink_seid, 5771 struct i40e_channel *ch) 5772 { 5773 struct i40e_hw *hw = &pf->hw; 5774 struct i40e_vsi_context ctxt; 5775 u8 enabled_tc = 0x1; /* TC0 enabled */ 5776 int ret; 5777 5778 if (ch->type != I40E_VSI_VMDQ2) { 5779 dev_info(&pf->pdev->dev, 5780 "add new vsi failed, ch->type %d\n", ch->type); 5781 return -EINVAL; 5782 } 5783 5784 memset(&ctxt, 0, sizeof(ctxt)); 5785 ctxt.pf_num = hw->pf_id; 5786 ctxt.vf_num = 0; 5787 ctxt.uplink_seid = uplink_seid; 5788 ctxt.connection_type = I40E_AQ_VSI_CONN_TYPE_NORMAL; 5789 if (ch->type == I40E_VSI_VMDQ2) 5790 ctxt.flags = I40E_AQ_VSI_TYPE_VMDQ2; 5791 5792 if (pf->flags & I40E_FLAG_VEB_MODE_ENABLED) { 5793 ctxt.info.valid_sections |= 5794 cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID); 5795 ctxt.info.switch_id = 5796 cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB); 5797 } 5798 5799 /* Set queue map for a given VSI context */ 5800 i40e_channel_setup_queue_map(pf, &ctxt, ch); 5801 5802 /* Now time to create VSI */ 5803 ret = i40e_aq_add_vsi(hw, &ctxt, NULL); 5804 if (ret) { 5805 dev_info(&pf->pdev->dev, 5806 "add new vsi failed, err %s aq_err %s\n", 5807 i40e_stat_str(&pf->hw, ret), 5808 i40e_aq_str(&pf->hw, 5809 pf->hw.aq.asq_last_status)); 5810 return -ENOENT; 5811 } 5812 5813 /* Success, update channel */ 5814 ch->enabled_tc = enabled_tc; 5815 ch->seid = ctxt.seid; 5816 ch->vsi_number = ctxt.vsi_number; 5817 ch->stat_counter_idx = cpu_to_le16(ctxt.info.stat_counter_idx); 5818 5819 /* copy just the sections touched not the entire info 5820 * since not all sections are valid as returned by 5821 * update vsi params 5822 */ 5823 ch->info.mapping_flags = ctxt.info.mapping_flags; 5824 memcpy(&ch->info.queue_mapping, 5825 &ctxt.info.queue_mapping, sizeof(ctxt.info.queue_mapping)); 5826 memcpy(&ch->info.tc_mapping, ctxt.info.tc_mapping, 5827 sizeof(ctxt.info.tc_mapping)); 5828 5829 return 0; 5830 } 5831 5832 static int i40e_channel_config_bw(struct i40e_vsi *vsi, struct i40e_channel *ch, 5833 u8 *bw_share) 5834 { 5835 struct i40e_aqc_configure_vsi_tc_bw_data bw_data; 5836 i40e_status ret; 5837 int i; 5838 5839 bw_data.tc_valid_bits = ch->enabled_tc; 5840 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) 5841 bw_data.tc_bw_credits[i] = bw_share[i]; 5842 5843 ret = i40e_aq_config_vsi_tc_bw(&vsi->back->hw, ch->seid, 5844 &bw_data, NULL); 5845 if (ret) { 5846 dev_info(&vsi->back->pdev->dev, 5847 "Config VSI BW allocation per TC failed, aq_err: %d for new_vsi->seid %u\n", 5848 vsi->back->hw.aq.asq_last_status, ch->seid); 5849 return -EINVAL; 5850 } 5851 5852 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) 5853 ch->info.qs_handle[i] = bw_data.qs_handles[i]; 5854 5855 return 0; 5856 } 5857 5858 /** 5859 * i40e_channel_config_tx_ring - config TX ring associated with new channel 5860 * @pf: ptr to PF device 5861 * @vsi: the VSI being setup 5862 * @ch: ptr to channel structure 5863 * 5864 * Configure TX rings associated with channel (VSI) since queues are being 5865 * from parent VSI. 5866 **/ 5867 static int i40e_channel_config_tx_ring(struct i40e_pf *pf, 5868 struct i40e_vsi *vsi, 5869 struct i40e_channel *ch) 5870 { 5871 i40e_status ret; 5872 int i; 5873 u8 bw_share[I40E_MAX_TRAFFIC_CLASS] = {0}; 5874 5875 /* Enable ETS TCs with equal BW Share for now across all VSIs */ 5876 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 5877 if (ch->enabled_tc & BIT(i)) 5878 bw_share[i] = 1; 5879 } 5880 5881 /* configure BW for new VSI */ 5882 ret = i40e_channel_config_bw(vsi, ch, bw_share); 5883 if (ret) { 5884 dev_info(&vsi->back->pdev->dev, 5885 "Failed configuring TC map %d for channel (seid %u)\n", 5886 ch->enabled_tc, ch->seid); 5887 return ret; 5888 } 5889 5890 for (i = 0; i < ch->num_queue_pairs; i++) { 5891 struct i40e_ring *tx_ring, *rx_ring; 5892 u16 pf_q; 5893 5894 pf_q = ch->base_queue + i; 5895 5896 /* Get to TX ring ptr of main VSI, for re-setup TX queue 5897 * context 5898 */ 5899 tx_ring = vsi->tx_rings[pf_q]; 5900 tx_ring->ch = ch; 5901 5902 /* Get the RX ring ptr */ 5903 rx_ring = vsi->rx_rings[pf_q]; 5904 rx_ring->ch = ch; 5905 } 5906 5907 return 0; 5908 } 5909 5910 /** 5911 * i40e_setup_hw_channel - setup new channel 5912 * @pf: ptr to PF device 5913 * @vsi: the VSI being setup 5914 * @ch: ptr to channel structure 5915 * @uplink_seid: underlying HW switching element (VEB) ID 5916 * @type: type of channel to be created (VMDq2/VF) 5917 * 5918 * Setup new channel (VSI) based on specified type (VMDq2/VF) 5919 * and configures TX rings accordingly 5920 **/ 5921 static inline int i40e_setup_hw_channel(struct i40e_pf *pf, 5922 struct i40e_vsi *vsi, 5923 struct i40e_channel *ch, 5924 u16 uplink_seid, u8 type) 5925 { 5926 int ret; 5927 5928 ch->initialized = false; 5929 ch->base_queue = vsi->next_base_queue; 5930 ch->type = type; 5931 5932 /* Proceed with creation of channel (VMDq2) VSI */ 5933 ret = i40e_add_channel(pf, uplink_seid, ch); 5934 if (ret) { 5935 dev_info(&pf->pdev->dev, 5936 "failed to add_channel using uplink_seid %u\n", 5937 uplink_seid); 5938 return ret; 5939 } 5940 5941 /* Mark the successful creation of channel */ 5942 ch->initialized = true; 5943 5944 /* Reconfigure TX queues using QTX_CTL register */ 5945 ret = i40e_channel_config_tx_ring(pf, vsi, ch); 5946 if (ret) { 5947 dev_info(&pf->pdev->dev, 5948 "failed to configure TX rings for channel %u\n", 5949 ch->seid); 5950 return ret; 5951 } 5952 5953 /* update 'next_base_queue' */ 5954 vsi->next_base_queue = vsi->next_base_queue + ch->num_queue_pairs; 5955 dev_dbg(&pf->pdev->dev, 5956 "Added channel: vsi_seid %u, vsi_number %u, stat_counter_idx %u, num_queue_pairs %u, pf->next_base_queue %d\n", 5957 ch->seid, ch->vsi_number, ch->stat_counter_idx, 5958 ch->num_queue_pairs, 5959 vsi->next_base_queue); 5960 return ret; 5961 } 5962 5963 /** 5964 * i40e_setup_channel - setup new channel using uplink element 5965 * @pf: ptr to PF device 5966 * @type: type of channel to be created (VMDq2/VF) 5967 * @uplink_seid: underlying HW switching element (VEB) ID 5968 * @ch: ptr to channel structure 5969 * 5970 * Setup new channel (VSI) based on specified type (VMDq2/VF) 5971 * and uplink switching element (uplink_seid) 5972 **/ 5973 static bool i40e_setup_channel(struct i40e_pf *pf, struct i40e_vsi *vsi, 5974 struct i40e_channel *ch) 5975 { 5976 u8 vsi_type; 5977 u16 seid; 5978 int ret; 5979 5980 if (vsi->type == I40E_VSI_MAIN) { 5981 vsi_type = I40E_VSI_VMDQ2; 5982 } else { 5983 dev_err(&pf->pdev->dev, "unsupported parent vsi type(%d)\n", 5984 vsi->type); 5985 return false; 5986 } 5987 5988 /* underlying switching element */ 5989 seid = pf->vsi[pf->lan_vsi]->uplink_seid; 5990 5991 /* create channel (VSI), configure TX rings */ 5992 ret = i40e_setup_hw_channel(pf, vsi, ch, seid, vsi_type); 5993 if (ret) { 5994 dev_err(&pf->pdev->dev, "failed to setup hw_channel\n"); 5995 return false; 5996 } 5997 5998 return ch->initialized ? true : false; 5999 } 6000 6001 /** 6002 * i40e_validate_and_set_switch_mode - sets up switch mode correctly 6003 * @vsi: ptr to VSI which has PF backing 6004 * 6005 * Sets up switch mode correctly if it needs to be changed and perform 6006 * what are allowed modes. 6007 **/ 6008 static int i40e_validate_and_set_switch_mode(struct i40e_vsi *vsi) 6009 { 6010 u8 mode; 6011 struct i40e_pf *pf = vsi->back; 6012 struct i40e_hw *hw = &pf->hw; 6013 int ret; 6014 6015 ret = i40e_get_capabilities(pf, i40e_aqc_opc_list_dev_capabilities); 6016 if (ret) 6017 return -EINVAL; 6018 6019 if (hw->dev_caps.switch_mode) { 6020 /* if switch mode is set, support mode2 (non-tunneled for 6021 * cloud filter) for now 6022 */ 6023 u32 switch_mode = hw->dev_caps.switch_mode & 6024 I40E_SWITCH_MODE_MASK; 6025 if (switch_mode >= I40E_CLOUD_FILTER_MODE1) { 6026 if (switch_mode == I40E_CLOUD_FILTER_MODE2) 6027 return 0; 6028 dev_err(&pf->pdev->dev, 6029 "Invalid switch_mode (%d), only non-tunneled mode for cloud filter is supported\n", 6030 hw->dev_caps.switch_mode); 6031 return -EINVAL; 6032 } 6033 } 6034 6035 /* Set Bit 7 to be valid */ 6036 mode = I40E_AQ_SET_SWITCH_BIT7_VALID; 6037 6038 /* Set L4type for TCP support */ 6039 mode |= I40E_AQ_SET_SWITCH_L4_TYPE_TCP; 6040 6041 /* Set cloud filter mode */ 6042 mode |= I40E_AQ_SET_SWITCH_MODE_NON_TUNNEL; 6043 6044 /* Prep mode field for set_switch_config */ 6045 ret = i40e_aq_set_switch_config(hw, pf->last_sw_conf_flags, 6046 pf->last_sw_conf_valid_flags, 6047 mode, NULL); 6048 if (ret && hw->aq.asq_last_status != I40E_AQ_RC_ESRCH) 6049 dev_err(&pf->pdev->dev, 6050 "couldn't set switch config bits, err %s aq_err %s\n", 6051 i40e_stat_str(hw, ret), 6052 i40e_aq_str(hw, 6053 hw->aq.asq_last_status)); 6054 6055 return ret; 6056 } 6057 6058 /** 6059 * i40e_create_queue_channel - function to create channel 6060 * @vsi: VSI to be configured 6061 * @ch: ptr to channel (it contains channel specific params) 6062 * 6063 * This function creates channel (VSI) using num_queues specified by user, 6064 * reconfigs RSS if needed. 6065 **/ 6066 int i40e_create_queue_channel(struct i40e_vsi *vsi, 6067 struct i40e_channel *ch) 6068 { 6069 struct i40e_pf *pf = vsi->back; 6070 bool reconfig_rss; 6071 int err; 6072 6073 if (!ch) 6074 return -EINVAL; 6075 6076 if (!ch->num_queue_pairs) { 6077 dev_err(&pf->pdev->dev, "Invalid num_queues requested: %d\n", 6078 ch->num_queue_pairs); 6079 return -EINVAL; 6080 } 6081 6082 /* validate user requested num_queues for channel */ 6083 err = i40e_validate_num_queues(pf, ch->num_queue_pairs, vsi, 6084 &reconfig_rss); 6085 if (err) { 6086 dev_info(&pf->pdev->dev, "Failed to validate num_queues (%d)\n", 6087 ch->num_queue_pairs); 6088 return -EINVAL; 6089 } 6090 6091 /* By default we are in VEPA mode, if this is the first VF/VMDq 6092 * VSI to be added switch to VEB mode. 6093 */ 6094 if ((!(pf->flags & I40E_FLAG_VEB_MODE_ENABLED)) || 6095 (!i40e_is_any_channel(vsi))) { 6096 if (!is_power_of_2(vsi->tc_config.tc_info[0].qcount)) { 6097 dev_dbg(&pf->pdev->dev, 6098 "Failed to create channel. Override queues (%u) not power of 2\n", 6099 vsi->tc_config.tc_info[0].qcount); 6100 return -EINVAL; 6101 } 6102 6103 if (!(pf->flags & I40E_FLAG_VEB_MODE_ENABLED)) { 6104 pf->flags |= I40E_FLAG_VEB_MODE_ENABLED; 6105 6106 if (vsi->type == I40E_VSI_MAIN) { 6107 if (pf->flags & I40E_FLAG_TC_MQPRIO) 6108 i40e_do_reset(pf, I40E_PF_RESET_FLAG, 6109 true); 6110 else 6111 i40e_do_reset_safe(pf, 6112 I40E_PF_RESET_FLAG); 6113 } 6114 } 6115 /* now onwards for main VSI, number of queues will be value 6116 * of TC0's queue count 6117 */ 6118 } 6119 6120 /* By this time, vsi->cnt_q_avail shall be set to non-zero and 6121 * it should be more than num_queues 6122 */ 6123 if (!vsi->cnt_q_avail || vsi->cnt_q_avail < ch->num_queue_pairs) { 6124 dev_dbg(&pf->pdev->dev, 6125 "Error: cnt_q_avail (%u) less than num_queues %d\n", 6126 vsi->cnt_q_avail, ch->num_queue_pairs); 6127 return -EINVAL; 6128 } 6129 6130 /* reconfig_rss only if vsi type is MAIN_VSI */ 6131 if (reconfig_rss && (vsi->type == I40E_VSI_MAIN)) { 6132 err = i40e_vsi_reconfig_rss(vsi, ch->num_queue_pairs); 6133 if (err) { 6134 dev_info(&pf->pdev->dev, 6135 "Error: unable to reconfig rss for num_queues (%u)\n", 6136 ch->num_queue_pairs); 6137 return -EINVAL; 6138 } 6139 } 6140 6141 if (!i40e_setup_channel(pf, vsi, ch)) { 6142 dev_info(&pf->pdev->dev, "Failed to setup channel\n"); 6143 return -EINVAL; 6144 } 6145 6146 dev_info(&pf->pdev->dev, 6147 "Setup channel (id:%u) utilizing num_queues %d\n", 6148 ch->seid, ch->num_queue_pairs); 6149 6150 /* configure VSI for BW limit */ 6151 if (ch->max_tx_rate) { 6152 u64 credits = ch->max_tx_rate; 6153 6154 if (i40e_set_bw_limit(vsi, ch->seid, ch->max_tx_rate)) 6155 return -EINVAL; 6156 6157 do_div(credits, I40E_BW_CREDIT_DIVISOR); 6158 dev_dbg(&pf->pdev->dev, 6159 "Set tx rate of %llu Mbps (count of 50Mbps %llu) for vsi->seid %u\n", 6160 ch->max_tx_rate, 6161 credits, 6162 ch->seid); 6163 } 6164 6165 /* in case of VF, this will be main SRIOV VSI */ 6166 ch->parent_vsi = vsi; 6167 6168 /* and update main_vsi's count for queue_available to use */ 6169 vsi->cnt_q_avail -= ch->num_queue_pairs; 6170 6171 return 0; 6172 } 6173 6174 /** 6175 * i40e_configure_queue_channels - Add queue channel for the given TCs 6176 * @vsi: VSI to be configured 6177 * 6178 * Configures queue channel mapping to the given TCs 6179 **/ 6180 static int i40e_configure_queue_channels(struct i40e_vsi *vsi) 6181 { 6182 struct i40e_channel *ch; 6183 u64 max_rate = 0; 6184 int ret = 0, i; 6185 6186 /* Create app vsi with the TCs. Main VSI with TC0 is already set up */ 6187 vsi->tc_seid_map[0] = vsi->seid; 6188 for (i = 1; i < I40E_MAX_TRAFFIC_CLASS; i++) { 6189 if (vsi->tc_config.enabled_tc & BIT(i)) { 6190 ch = kzalloc(sizeof(*ch), GFP_KERNEL); 6191 if (!ch) { 6192 ret = -ENOMEM; 6193 goto err_free; 6194 } 6195 6196 INIT_LIST_HEAD(&ch->list); 6197 ch->num_queue_pairs = 6198 vsi->tc_config.tc_info[i].qcount; 6199 ch->base_queue = 6200 vsi->tc_config.tc_info[i].qoffset; 6201 6202 /* Bandwidth limit through tc interface is in bytes/s, 6203 * change to Mbit/s 6204 */ 6205 max_rate = vsi->mqprio_qopt.max_rate[i]; 6206 do_div(max_rate, I40E_BW_MBPS_DIVISOR); 6207 ch->max_tx_rate = max_rate; 6208 6209 list_add_tail(&ch->list, &vsi->ch_list); 6210 6211 ret = i40e_create_queue_channel(vsi, ch); 6212 if (ret) { 6213 dev_err(&vsi->back->pdev->dev, 6214 "Failed creating queue channel with TC%d: queues %d\n", 6215 i, ch->num_queue_pairs); 6216 goto err_free; 6217 } 6218 vsi->tc_seid_map[i] = ch->seid; 6219 } 6220 } 6221 return ret; 6222 6223 err_free: 6224 i40e_remove_queue_channels(vsi); 6225 return ret; 6226 } 6227 6228 /** 6229 * i40e_veb_config_tc - Configure TCs for given VEB 6230 * @veb: given VEB 6231 * @enabled_tc: TC bitmap 6232 * 6233 * Configures given TC bitmap for VEB (switching) element 6234 **/ 6235 int i40e_veb_config_tc(struct i40e_veb *veb, u8 enabled_tc) 6236 { 6237 struct i40e_aqc_configure_switching_comp_bw_config_data bw_data = {0}; 6238 struct i40e_pf *pf = veb->pf; 6239 int ret = 0; 6240 int i; 6241 6242 /* No TCs or already enabled TCs just return */ 6243 if (!enabled_tc || veb->enabled_tc == enabled_tc) 6244 return ret; 6245 6246 bw_data.tc_valid_bits = enabled_tc; 6247 /* bw_data.absolute_credits is not set (relative) */ 6248 6249 /* Enable ETS TCs with equal BW Share for now */ 6250 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 6251 if (enabled_tc & BIT(i)) 6252 bw_data.tc_bw_share_credits[i] = 1; 6253 } 6254 6255 ret = i40e_aq_config_switch_comp_bw_config(&pf->hw, veb->seid, 6256 &bw_data, NULL); 6257 if (ret) { 6258 dev_info(&pf->pdev->dev, 6259 "VEB bw config failed, err %s aq_err %s\n", 6260 i40e_stat_str(&pf->hw, ret), 6261 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 6262 goto out; 6263 } 6264 6265 /* Update the BW information */ 6266 ret = i40e_veb_get_bw_info(veb); 6267 if (ret) { 6268 dev_info(&pf->pdev->dev, 6269 "Failed getting veb bw config, err %s aq_err %s\n", 6270 i40e_stat_str(&pf->hw, ret), 6271 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 6272 } 6273 6274 out: 6275 return ret; 6276 } 6277 6278 #ifdef CONFIG_I40E_DCB 6279 /** 6280 * i40e_dcb_reconfigure - Reconfigure all VEBs and VSIs 6281 * @pf: PF struct 6282 * 6283 * Reconfigure VEB/VSIs on a given PF; it is assumed that 6284 * the caller would've quiesce all the VSIs before calling 6285 * this function 6286 **/ 6287 static void i40e_dcb_reconfigure(struct i40e_pf *pf) 6288 { 6289 u8 tc_map = 0; 6290 int ret; 6291 u8 v; 6292 6293 /* Enable the TCs available on PF to all VEBs */ 6294 tc_map = i40e_pf_get_tc_map(pf); 6295 for (v = 0; v < I40E_MAX_VEB; v++) { 6296 if (!pf->veb[v]) 6297 continue; 6298 ret = i40e_veb_config_tc(pf->veb[v], tc_map); 6299 if (ret) { 6300 dev_info(&pf->pdev->dev, 6301 "Failed configuring TC for VEB seid=%d\n", 6302 pf->veb[v]->seid); 6303 /* Will try to configure as many components */ 6304 } 6305 } 6306 6307 /* Update each VSI */ 6308 for (v = 0; v < pf->num_alloc_vsi; v++) { 6309 if (!pf->vsi[v]) 6310 continue; 6311 6312 /* - Enable all TCs for the LAN VSI 6313 * - For all others keep them at TC0 for now 6314 */ 6315 if (v == pf->lan_vsi) 6316 tc_map = i40e_pf_get_tc_map(pf); 6317 else 6318 tc_map = I40E_DEFAULT_TRAFFIC_CLASS; 6319 6320 ret = i40e_vsi_config_tc(pf->vsi[v], tc_map); 6321 if (ret) { 6322 dev_info(&pf->pdev->dev, 6323 "Failed configuring TC for VSI seid=%d\n", 6324 pf->vsi[v]->seid); 6325 /* Will try to configure as many components */ 6326 } else { 6327 /* Re-configure VSI vectors based on updated TC map */ 6328 i40e_vsi_map_rings_to_vectors(pf->vsi[v]); 6329 if (pf->vsi[v]->netdev) 6330 i40e_dcbnl_set_all(pf->vsi[v]); 6331 } 6332 } 6333 } 6334 6335 /** 6336 * i40e_resume_port_tx - Resume port Tx 6337 * @pf: PF struct 6338 * 6339 * Resume a port's Tx and issue a PF reset in case of failure to 6340 * resume. 6341 **/ 6342 static int i40e_resume_port_tx(struct i40e_pf *pf) 6343 { 6344 struct i40e_hw *hw = &pf->hw; 6345 int ret; 6346 6347 ret = i40e_aq_resume_port_tx(hw, NULL); 6348 if (ret) { 6349 dev_info(&pf->pdev->dev, 6350 "Resume Port Tx failed, err %s aq_err %s\n", 6351 i40e_stat_str(&pf->hw, ret), 6352 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 6353 /* Schedule PF reset to recover */ 6354 set_bit(__I40E_PF_RESET_REQUESTED, pf->state); 6355 i40e_service_event_schedule(pf); 6356 } 6357 6358 return ret; 6359 } 6360 6361 /** 6362 * i40e_init_pf_dcb - Initialize DCB configuration 6363 * @pf: PF being configured 6364 * 6365 * Query the current DCB configuration and cache it 6366 * in the hardware structure 6367 **/ 6368 static int i40e_init_pf_dcb(struct i40e_pf *pf) 6369 { 6370 struct i40e_hw *hw = &pf->hw; 6371 int err = 0; 6372 6373 /* Do not enable DCB for SW1 and SW2 images even if the FW is capable 6374 * Also do not enable DCBx if FW LLDP agent is disabled 6375 */ 6376 if ((pf->hw_features & I40E_HW_NO_DCB_SUPPORT) || 6377 (pf->flags & I40E_FLAG_DISABLE_FW_LLDP)) 6378 goto out; 6379 6380 /* Get the initial DCB configuration */ 6381 err = i40e_init_dcb(hw); 6382 if (!err) { 6383 /* Device/Function is not DCBX capable */ 6384 if ((!hw->func_caps.dcb) || 6385 (hw->dcbx_status == I40E_DCBX_STATUS_DISABLED)) { 6386 dev_info(&pf->pdev->dev, 6387 "DCBX offload is not supported or is disabled for this PF.\n"); 6388 } else { 6389 /* When status is not DISABLED then DCBX in FW */ 6390 pf->dcbx_cap = DCB_CAP_DCBX_LLD_MANAGED | 6391 DCB_CAP_DCBX_VER_IEEE; 6392 6393 pf->flags |= I40E_FLAG_DCB_CAPABLE; 6394 /* Enable DCB tagging only when more than one TC 6395 * or explicitly disable if only one TC 6396 */ 6397 if (i40e_dcb_get_num_tc(&hw->local_dcbx_config) > 1) 6398 pf->flags |= I40E_FLAG_DCB_ENABLED; 6399 else 6400 pf->flags &= ~I40E_FLAG_DCB_ENABLED; 6401 dev_dbg(&pf->pdev->dev, 6402 "DCBX offload is supported for this PF.\n"); 6403 } 6404 } else if (pf->hw.aq.asq_last_status == I40E_AQ_RC_EPERM) { 6405 dev_info(&pf->pdev->dev, "FW LLDP disabled for this PF.\n"); 6406 pf->flags |= I40E_FLAG_DISABLE_FW_LLDP; 6407 } else { 6408 dev_info(&pf->pdev->dev, 6409 "Query for DCB configuration failed, err %s aq_err %s\n", 6410 i40e_stat_str(&pf->hw, err), 6411 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 6412 } 6413 6414 out: 6415 return err; 6416 } 6417 #endif /* CONFIG_I40E_DCB */ 6418 #define SPEED_SIZE 14 6419 #define FC_SIZE 8 6420 /** 6421 * i40e_print_link_message - print link up or down 6422 * @vsi: the VSI for which link needs a message 6423 * @isup: true of link is up, false otherwise 6424 */ 6425 void i40e_print_link_message(struct i40e_vsi *vsi, bool isup) 6426 { 6427 enum i40e_aq_link_speed new_speed; 6428 struct i40e_pf *pf = vsi->back; 6429 char *speed = "Unknown"; 6430 char *fc = "Unknown"; 6431 char *fec = ""; 6432 char *req_fec = ""; 6433 char *an = ""; 6434 6435 new_speed = pf->hw.phy.link_info.link_speed; 6436 6437 if ((vsi->current_isup == isup) && (vsi->current_speed == new_speed)) 6438 return; 6439 vsi->current_isup = isup; 6440 vsi->current_speed = new_speed; 6441 if (!isup) { 6442 netdev_info(vsi->netdev, "NIC Link is Down\n"); 6443 return; 6444 } 6445 6446 /* Warn user if link speed on NPAR enabled partition is not at 6447 * least 10GB 6448 */ 6449 if (pf->hw.func_caps.npar_enable && 6450 (pf->hw.phy.link_info.link_speed == I40E_LINK_SPEED_1GB || 6451 pf->hw.phy.link_info.link_speed == I40E_LINK_SPEED_100MB)) 6452 netdev_warn(vsi->netdev, 6453 "The partition detected link speed that is less than 10Gbps\n"); 6454 6455 switch (pf->hw.phy.link_info.link_speed) { 6456 case I40E_LINK_SPEED_40GB: 6457 speed = "40 G"; 6458 break; 6459 case I40E_LINK_SPEED_20GB: 6460 speed = "20 G"; 6461 break; 6462 case I40E_LINK_SPEED_25GB: 6463 speed = "25 G"; 6464 break; 6465 case I40E_LINK_SPEED_10GB: 6466 speed = "10 G"; 6467 break; 6468 case I40E_LINK_SPEED_1GB: 6469 speed = "1000 M"; 6470 break; 6471 case I40E_LINK_SPEED_100MB: 6472 speed = "100 M"; 6473 break; 6474 default: 6475 break; 6476 } 6477 6478 switch (pf->hw.fc.current_mode) { 6479 case I40E_FC_FULL: 6480 fc = "RX/TX"; 6481 break; 6482 case I40E_FC_TX_PAUSE: 6483 fc = "TX"; 6484 break; 6485 case I40E_FC_RX_PAUSE: 6486 fc = "RX"; 6487 break; 6488 default: 6489 fc = "None"; 6490 break; 6491 } 6492 6493 if (pf->hw.phy.link_info.link_speed == I40E_LINK_SPEED_25GB) { 6494 req_fec = ", Requested FEC: None"; 6495 fec = ", FEC: None"; 6496 an = ", Autoneg: False"; 6497 6498 if (pf->hw.phy.link_info.an_info & I40E_AQ_AN_COMPLETED) 6499 an = ", Autoneg: True"; 6500 6501 if (pf->hw.phy.link_info.fec_info & 6502 I40E_AQ_CONFIG_FEC_KR_ENA) 6503 fec = ", FEC: CL74 FC-FEC/BASE-R"; 6504 else if (pf->hw.phy.link_info.fec_info & 6505 I40E_AQ_CONFIG_FEC_RS_ENA) 6506 fec = ", FEC: CL108 RS-FEC"; 6507 6508 /* 'CL108 RS-FEC' should be displayed when RS is requested, or 6509 * both RS and FC are requested 6510 */ 6511 if (vsi->back->hw.phy.link_info.req_fec_info & 6512 (I40E_AQ_REQUEST_FEC_KR | I40E_AQ_REQUEST_FEC_RS)) { 6513 if (vsi->back->hw.phy.link_info.req_fec_info & 6514 I40E_AQ_REQUEST_FEC_RS) 6515 req_fec = ", Requested FEC: CL108 RS-FEC"; 6516 else 6517 req_fec = ", Requested FEC: CL74 FC-FEC/BASE-R"; 6518 } 6519 } 6520 6521 netdev_info(vsi->netdev, "NIC Link is Up, %sbps Full Duplex%s%s%s, Flow Control: %s\n", 6522 speed, req_fec, fec, an, fc); 6523 } 6524 6525 /** 6526 * i40e_up_complete - Finish the last steps of bringing up a connection 6527 * @vsi: the VSI being configured 6528 **/ 6529 static int i40e_up_complete(struct i40e_vsi *vsi) 6530 { 6531 struct i40e_pf *pf = vsi->back; 6532 int err; 6533 6534 if (pf->flags & I40E_FLAG_MSIX_ENABLED) 6535 i40e_vsi_configure_msix(vsi); 6536 else 6537 i40e_configure_msi_and_legacy(vsi); 6538 6539 /* start rings */ 6540 err = i40e_vsi_start_rings(vsi); 6541 if (err) 6542 return err; 6543 6544 clear_bit(__I40E_VSI_DOWN, vsi->state); 6545 i40e_napi_enable_all(vsi); 6546 i40e_vsi_enable_irq(vsi); 6547 6548 if ((pf->hw.phy.link_info.link_info & I40E_AQ_LINK_UP) && 6549 (vsi->netdev)) { 6550 i40e_print_link_message(vsi, true); 6551 netif_tx_start_all_queues(vsi->netdev); 6552 netif_carrier_on(vsi->netdev); 6553 } 6554 6555 /* replay FDIR SB filters */ 6556 if (vsi->type == I40E_VSI_FDIR) { 6557 /* reset fd counters */ 6558 pf->fd_add_err = 0; 6559 pf->fd_atr_cnt = 0; 6560 i40e_fdir_filter_restore(vsi); 6561 } 6562 6563 /* On the next run of the service_task, notify any clients of the new 6564 * opened netdev 6565 */ 6566 set_bit(__I40E_CLIENT_SERVICE_REQUESTED, pf->state); 6567 i40e_service_event_schedule(pf); 6568 6569 return 0; 6570 } 6571 6572 /** 6573 * i40e_vsi_reinit_locked - Reset the VSI 6574 * @vsi: the VSI being configured 6575 * 6576 * Rebuild the ring structs after some configuration 6577 * has changed, e.g. MTU size. 6578 **/ 6579 static void i40e_vsi_reinit_locked(struct i40e_vsi *vsi) 6580 { 6581 struct i40e_pf *pf = vsi->back; 6582 6583 WARN_ON(in_interrupt()); 6584 while (test_and_set_bit(__I40E_CONFIG_BUSY, pf->state)) 6585 usleep_range(1000, 2000); 6586 i40e_down(vsi); 6587 6588 i40e_up(vsi); 6589 clear_bit(__I40E_CONFIG_BUSY, pf->state); 6590 } 6591 6592 /** 6593 * i40e_up - Bring the connection back up after being down 6594 * @vsi: the VSI being configured 6595 **/ 6596 int i40e_up(struct i40e_vsi *vsi) 6597 { 6598 int err; 6599 6600 err = i40e_vsi_configure(vsi); 6601 if (!err) 6602 err = i40e_up_complete(vsi); 6603 6604 return err; 6605 } 6606 6607 /** 6608 * i40e_force_link_state - Force the link status 6609 * @pf: board private structure 6610 * @is_up: whether the link state should be forced up or down 6611 **/ 6612 static i40e_status i40e_force_link_state(struct i40e_pf *pf, bool is_up) 6613 { 6614 struct i40e_aq_get_phy_abilities_resp abilities; 6615 struct i40e_aq_set_phy_config config = {0}; 6616 struct i40e_hw *hw = &pf->hw; 6617 i40e_status err; 6618 u64 mask; 6619 u8 speed; 6620 6621 /* Card might've been put in an unstable state by other drivers 6622 * and applications, which causes incorrect speed values being 6623 * set on startup. In order to clear speed registers, we call 6624 * get_phy_capabilities twice, once to get initial state of 6625 * available speeds, and once to get current PHY config. 6626 */ 6627 err = i40e_aq_get_phy_capabilities(hw, false, true, &abilities, 6628 NULL); 6629 if (err) { 6630 dev_err(&pf->pdev->dev, 6631 "failed to get phy cap., ret = %s last_status = %s\n", 6632 i40e_stat_str(hw, err), 6633 i40e_aq_str(hw, hw->aq.asq_last_status)); 6634 return err; 6635 } 6636 speed = abilities.link_speed; 6637 6638 /* Get the current phy config */ 6639 err = i40e_aq_get_phy_capabilities(hw, false, false, &abilities, 6640 NULL); 6641 if (err) { 6642 dev_err(&pf->pdev->dev, 6643 "failed to get phy cap., ret = %s last_status = %s\n", 6644 i40e_stat_str(hw, err), 6645 i40e_aq_str(hw, hw->aq.asq_last_status)); 6646 return err; 6647 } 6648 6649 /* If link needs to go up, but was not forced to go down, 6650 * and its speed values are OK, no need for a flap 6651 */ 6652 if (is_up && abilities.phy_type != 0 && abilities.link_speed != 0) 6653 return I40E_SUCCESS; 6654 6655 /* To force link we need to set bits for all supported PHY types, 6656 * but there are now more than 32, so we need to split the bitmap 6657 * across two fields. 6658 */ 6659 mask = I40E_PHY_TYPES_BITMASK; 6660 config.phy_type = is_up ? cpu_to_le32((u32)(mask & 0xffffffff)) : 0; 6661 config.phy_type_ext = is_up ? (u8)((mask >> 32) & 0xff) : 0; 6662 /* Copy the old settings, except of phy_type */ 6663 config.abilities = abilities.abilities; 6664 if (abilities.link_speed != 0) 6665 config.link_speed = abilities.link_speed; 6666 else 6667 config.link_speed = speed; 6668 config.eee_capability = abilities.eee_capability; 6669 config.eeer = abilities.eeer_val; 6670 config.low_power_ctrl = abilities.d3_lpan; 6671 config.fec_config = abilities.fec_cfg_curr_mod_ext_info & 6672 I40E_AQ_PHY_FEC_CONFIG_MASK; 6673 err = i40e_aq_set_phy_config(hw, &config, NULL); 6674 6675 if (err) { 6676 dev_err(&pf->pdev->dev, 6677 "set phy config ret = %s last_status = %s\n", 6678 i40e_stat_str(&pf->hw, err), 6679 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 6680 return err; 6681 } 6682 6683 /* Update the link info */ 6684 err = i40e_update_link_info(hw); 6685 if (err) { 6686 /* Wait a little bit (on 40G cards it sometimes takes a really 6687 * long time for link to come back from the atomic reset) 6688 * and try once more 6689 */ 6690 msleep(1000); 6691 i40e_update_link_info(hw); 6692 } 6693 6694 i40e_aq_set_link_restart_an(hw, true, NULL); 6695 6696 return I40E_SUCCESS; 6697 } 6698 6699 /** 6700 * i40e_down - Shutdown the connection processing 6701 * @vsi: the VSI being stopped 6702 **/ 6703 void i40e_down(struct i40e_vsi *vsi) 6704 { 6705 int i; 6706 6707 /* It is assumed that the caller of this function 6708 * sets the vsi->state __I40E_VSI_DOWN bit. 6709 */ 6710 if (vsi->netdev) { 6711 netif_carrier_off(vsi->netdev); 6712 netif_tx_disable(vsi->netdev); 6713 } 6714 i40e_vsi_disable_irq(vsi); 6715 i40e_vsi_stop_rings(vsi); 6716 if (vsi->type == I40E_VSI_MAIN && 6717 vsi->back->flags & I40E_FLAG_LINK_DOWN_ON_CLOSE_ENABLED) 6718 i40e_force_link_state(vsi->back, false); 6719 i40e_napi_disable_all(vsi); 6720 6721 for (i = 0; i < vsi->num_queue_pairs; i++) { 6722 i40e_clean_tx_ring(vsi->tx_rings[i]); 6723 if (i40e_enabled_xdp_vsi(vsi)) 6724 i40e_clean_tx_ring(vsi->xdp_rings[i]); 6725 i40e_clean_rx_ring(vsi->rx_rings[i]); 6726 } 6727 6728 } 6729 6730 /** 6731 * i40e_validate_mqprio_qopt- validate queue mapping info 6732 * @vsi: the VSI being configured 6733 * @mqprio_qopt: queue parametrs 6734 **/ 6735 static int i40e_validate_mqprio_qopt(struct i40e_vsi *vsi, 6736 struct tc_mqprio_qopt_offload *mqprio_qopt) 6737 { 6738 u64 sum_max_rate = 0; 6739 u64 max_rate = 0; 6740 int i; 6741 6742 if (mqprio_qopt->qopt.offset[0] != 0 || 6743 mqprio_qopt->qopt.num_tc < 1 || 6744 mqprio_qopt->qopt.num_tc > I40E_MAX_TRAFFIC_CLASS) 6745 return -EINVAL; 6746 for (i = 0; ; i++) { 6747 if (!mqprio_qopt->qopt.count[i]) 6748 return -EINVAL; 6749 if (mqprio_qopt->min_rate[i]) { 6750 dev_err(&vsi->back->pdev->dev, 6751 "Invalid min tx rate (greater than 0) specified\n"); 6752 return -EINVAL; 6753 } 6754 max_rate = mqprio_qopt->max_rate[i]; 6755 do_div(max_rate, I40E_BW_MBPS_DIVISOR); 6756 sum_max_rate += max_rate; 6757 6758 if (i >= mqprio_qopt->qopt.num_tc - 1) 6759 break; 6760 if (mqprio_qopt->qopt.offset[i + 1] != 6761 (mqprio_qopt->qopt.offset[i] + mqprio_qopt->qopt.count[i])) 6762 return -EINVAL; 6763 } 6764 if (vsi->num_queue_pairs < 6765 (mqprio_qopt->qopt.offset[i] + mqprio_qopt->qopt.count[i])) { 6766 return -EINVAL; 6767 } 6768 if (sum_max_rate > i40e_get_link_speed(vsi)) { 6769 dev_err(&vsi->back->pdev->dev, 6770 "Invalid max tx rate specified\n"); 6771 return -EINVAL; 6772 } 6773 return 0; 6774 } 6775 6776 /** 6777 * i40e_vsi_set_default_tc_config - set default values for tc configuration 6778 * @vsi: the VSI being configured 6779 **/ 6780 static void i40e_vsi_set_default_tc_config(struct i40e_vsi *vsi) 6781 { 6782 u16 qcount; 6783 int i; 6784 6785 /* Only TC0 is enabled */ 6786 vsi->tc_config.numtc = 1; 6787 vsi->tc_config.enabled_tc = 1; 6788 qcount = min_t(int, vsi->alloc_queue_pairs, 6789 i40e_pf_get_max_q_per_tc(vsi->back)); 6790 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 6791 /* For the TC that is not enabled set the offset to to default 6792 * queue and allocate one queue for the given TC. 6793 */ 6794 vsi->tc_config.tc_info[i].qoffset = 0; 6795 if (i == 0) 6796 vsi->tc_config.tc_info[i].qcount = qcount; 6797 else 6798 vsi->tc_config.tc_info[i].qcount = 1; 6799 vsi->tc_config.tc_info[i].netdev_tc = 0; 6800 } 6801 } 6802 6803 /** 6804 * i40e_setup_tc - configure multiple traffic classes 6805 * @netdev: net device to configure 6806 * @type_data: tc offload data 6807 **/ 6808 static int i40e_setup_tc(struct net_device *netdev, void *type_data) 6809 { 6810 struct tc_mqprio_qopt_offload *mqprio_qopt = type_data; 6811 struct i40e_netdev_priv *np = netdev_priv(netdev); 6812 struct i40e_vsi *vsi = np->vsi; 6813 struct i40e_pf *pf = vsi->back; 6814 u8 enabled_tc = 0, num_tc, hw; 6815 bool need_reset = false; 6816 int ret = -EINVAL; 6817 u16 mode; 6818 int i; 6819 6820 num_tc = mqprio_qopt->qopt.num_tc; 6821 hw = mqprio_qopt->qopt.hw; 6822 mode = mqprio_qopt->mode; 6823 if (!hw) { 6824 pf->flags &= ~I40E_FLAG_TC_MQPRIO; 6825 memcpy(&vsi->mqprio_qopt, mqprio_qopt, sizeof(*mqprio_qopt)); 6826 goto config_tc; 6827 } 6828 6829 /* Check if MFP enabled */ 6830 if (pf->flags & I40E_FLAG_MFP_ENABLED) { 6831 netdev_info(netdev, 6832 "Configuring TC not supported in MFP mode\n"); 6833 return ret; 6834 } 6835 switch (mode) { 6836 case TC_MQPRIO_MODE_DCB: 6837 pf->flags &= ~I40E_FLAG_TC_MQPRIO; 6838 6839 /* Check if DCB enabled to continue */ 6840 if (!(pf->flags & I40E_FLAG_DCB_ENABLED)) { 6841 netdev_info(netdev, 6842 "DCB is not enabled for adapter\n"); 6843 return ret; 6844 } 6845 6846 /* Check whether tc count is within enabled limit */ 6847 if (num_tc > i40e_pf_get_num_tc(pf)) { 6848 netdev_info(netdev, 6849 "TC count greater than enabled on link for adapter\n"); 6850 return ret; 6851 } 6852 break; 6853 case TC_MQPRIO_MODE_CHANNEL: 6854 if (pf->flags & I40E_FLAG_DCB_ENABLED) { 6855 netdev_info(netdev, 6856 "Full offload of TC Mqprio options is not supported when DCB is enabled\n"); 6857 return ret; 6858 } 6859 if (!(pf->flags & I40E_FLAG_MSIX_ENABLED)) 6860 return ret; 6861 ret = i40e_validate_mqprio_qopt(vsi, mqprio_qopt); 6862 if (ret) 6863 return ret; 6864 memcpy(&vsi->mqprio_qopt, mqprio_qopt, 6865 sizeof(*mqprio_qopt)); 6866 pf->flags |= I40E_FLAG_TC_MQPRIO; 6867 pf->flags &= ~I40E_FLAG_DCB_ENABLED; 6868 break; 6869 default: 6870 return -EINVAL; 6871 } 6872 6873 config_tc: 6874 /* Generate TC map for number of tc requested */ 6875 for (i = 0; i < num_tc; i++) 6876 enabled_tc |= BIT(i); 6877 6878 /* Requesting same TC configuration as already enabled */ 6879 if (enabled_tc == vsi->tc_config.enabled_tc && 6880 mode != TC_MQPRIO_MODE_CHANNEL) 6881 return 0; 6882 6883 /* Quiesce VSI queues */ 6884 i40e_quiesce_vsi(vsi); 6885 6886 if (!hw && !(pf->flags & I40E_FLAG_TC_MQPRIO)) 6887 i40e_remove_queue_channels(vsi); 6888 6889 /* Configure VSI for enabled TCs */ 6890 ret = i40e_vsi_config_tc(vsi, enabled_tc); 6891 if (ret) { 6892 netdev_info(netdev, "Failed configuring TC for VSI seid=%d\n", 6893 vsi->seid); 6894 need_reset = true; 6895 goto exit; 6896 } 6897 6898 if (pf->flags & I40E_FLAG_TC_MQPRIO) { 6899 if (vsi->mqprio_qopt.max_rate[0]) { 6900 u64 max_tx_rate = vsi->mqprio_qopt.max_rate[0]; 6901 6902 do_div(max_tx_rate, I40E_BW_MBPS_DIVISOR); 6903 ret = i40e_set_bw_limit(vsi, vsi->seid, max_tx_rate); 6904 if (!ret) { 6905 u64 credits = max_tx_rate; 6906 6907 do_div(credits, I40E_BW_CREDIT_DIVISOR); 6908 dev_dbg(&vsi->back->pdev->dev, 6909 "Set tx rate of %llu Mbps (count of 50Mbps %llu) for vsi->seid %u\n", 6910 max_tx_rate, 6911 credits, 6912 vsi->seid); 6913 } else { 6914 need_reset = true; 6915 goto exit; 6916 } 6917 } 6918 ret = i40e_configure_queue_channels(vsi); 6919 if (ret) { 6920 netdev_info(netdev, 6921 "Failed configuring queue channels\n"); 6922 need_reset = true; 6923 goto exit; 6924 } 6925 } 6926 6927 exit: 6928 /* Reset the configuration data to defaults, only TC0 is enabled */ 6929 if (need_reset) { 6930 i40e_vsi_set_default_tc_config(vsi); 6931 need_reset = false; 6932 } 6933 6934 /* Unquiesce VSI */ 6935 i40e_unquiesce_vsi(vsi); 6936 return ret; 6937 } 6938 6939 /** 6940 * i40e_set_cld_element - sets cloud filter element data 6941 * @filter: cloud filter rule 6942 * @cld: ptr to cloud filter element data 6943 * 6944 * This is helper function to copy data into cloud filter element 6945 **/ 6946 static inline void 6947 i40e_set_cld_element(struct i40e_cloud_filter *filter, 6948 struct i40e_aqc_cloud_filters_element_data *cld) 6949 { 6950 int i, j; 6951 u32 ipa; 6952 6953 memset(cld, 0, sizeof(*cld)); 6954 ether_addr_copy(cld->outer_mac, filter->dst_mac); 6955 ether_addr_copy(cld->inner_mac, filter->src_mac); 6956 6957 if (filter->n_proto != ETH_P_IP && filter->n_proto != ETH_P_IPV6) 6958 return; 6959 6960 if (filter->n_proto == ETH_P_IPV6) { 6961 #define IPV6_MAX_INDEX (ARRAY_SIZE(filter->dst_ipv6) - 1) 6962 for (i = 0, j = 0; i < ARRAY_SIZE(filter->dst_ipv6); 6963 i++, j += 2) { 6964 ipa = be32_to_cpu(filter->dst_ipv6[IPV6_MAX_INDEX - i]); 6965 ipa = cpu_to_le32(ipa); 6966 memcpy(&cld->ipaddr.raw_v6.data[j], &ipa, sizeof(ipa)); 6967 } 6968 } else { 6969 ipa = be32_to_cpu(filter->dst_ipv4); 6970 memcpy(&cld->ipaddr.v4.data, &ipa, sizeof(ipa)); 6971 } 6972 6973 cld->inner_vlan = cpu_to_le16(ntohs(filter->vlan_id)); 6974 6975 /* tenant_id is not supported by FW now, once the support is enabled 6976 * fill the cld->tenant_id with cpu_to_le32(filter->tenant_id) 6977 */ 6978 if (filter->tenant_id) 6979 return; 6980 } 6981 6982 /** 6983 * i40e_add_del_cloud_filter - Add/del cloud filter 6984 * @vsi: pointer to VSI 6985 * @filter: cloud filter rule 6986 * @add: if true, add, if false, delete 6987 * 6988 * Add or delete a cloud filter for a specific flow spec. 6989 * Returns 0 if the filter were successfully added. 6990 **/ 6991 int i40e_add_del_cloud_filter(struct i40e_vsi *vsi, 6992 struct i40e_cloud_filter *filter, bool add) 6993 { 6994 struct i40e_aqc_cloud_filters_element_data cld_filter; 6995 struct i40e_pf *pf = vsi->back; 6996 int ret; 6997 static const u16 flag_table[128] = { 6998 [I40E_CLOUD_FILTER_FLAGS_OMAC] = 6999 I40E_AQC_ADD_CLOUD_FILTER_OMAC, 7000 [I40E_CLOUD_FILTER_FLAGS_IMAC] = 7001 I40E_AQC_ADD_CLOUD_FILTER_IMAC, 7002 [I40E_CLOUD_FILTER_FLAGS_IMAC_IVLAN] = 7003 I40E_AQC_ADD_CLOUD_FILTER_IMAC_IVLAN, 7004 [I40E_CLOUD_FILTER_FLAGS_IMAC_TEN_ID] = 7005 I40E_AQC_ADD_CLOUD_FILTER_IMAC_TEN_ID, 7006 [I40E_CLOUD_FILTER_FLAGS_OMAC_TEN_ID_IMAC] = 7007 I40E_AQC_ADD_CLOUD_FILTER_OMAC_TEN_ID_IMAC, 7008 [I40E_CLOUD_FILTER_FLAGS_IMAC_IVLAN_TEN_ID] = 7009 I40E_AQC_ADD_CLOUD_FILTER_IMAC_IVLAN_TEN_ID, 7010 [I40E_CLOUD_FILTER_FLAGS_IIP] = 7011 I40E_AQC_ADD_CLOUD_FILTER_IIP, 7012 }; 7013 7014 if (filter->flags >= ARRAY_SIZE(flag_table)) 7015 return I40E_ERR_CONFIG; 7016 7017 /* copy element needed to add cloud filter from filter */ 7018 i40e_set_cld_element(filter, &cld_filter); 7019 7020 if (filter->tunnel_type != I40E_CLOUD_TNL_TYPE_NONE) 7021 cld_filter.flags = cpu_to_le16(filter->tunnel_type << 7022 I40E_AQC_ADD_CLOUD_TNL_TYPE_SHIFT); 7023 7024 if (filter->n_proto == ETH_P_IPV6) 7025 cld_filter.flags |= cpu_to_le16(flag_table[filter->flags] | 7026 I40E_AQC_ADD_CLOUD_FLAGS_IPV6); 7027 else 7028 cld_filter.flags |= cpu_to_le16(flag_table[filter->flags] | 7029 I40E_AQC_ADD_CLOUD_FLAGS_IPV4); 7030 7031 if (add) 7032 ret = i40e_aq_add_cloud_filters(&pf->hw, filter->seid, 7033 &cld_filter, 1); 7034 else 7035 ret = i40e_aq_rem_cloud_filters(&pf->hw, filter->seid, 7036 &cld_filter, 1); 7037 if (ret) 7038 dev_dbg(&pf->pdev->dev, 7039 "Failed to %s cloud filter using l4 port %u, err %d aq_err %d\n", 7040 add ? "add" : "delete", filter->dst_port, ret, 7041 pf->hw.aq.asq_last_status); 7042 else 7043 dev_info(&pf->pdev->dev, 7044 "%s cloud filter for VSI: %d\n", 7045 add ? "Added" : "Deleted", filter->seid); 7046 return ret; 7047 } 7048 7049 /** 7050 * i40e_add_del_cloud_filter_big_buf - Add/del cloud filter using big_buf 7051 * @vsi: pointer to VSI 7052 * @filter: cloud filter rule 7053 * @add: if true, add, if false, delete 7054 * 7055 * Add or delete a cloud filter for a specific flow spec using big buffer. 7056 * Returns 0 if the filter were successfully added. 7057 **/ 7058 int i40e_add_del_cloud_filter_big_buf(struct i40e_vsi *vsi, 7059 struct i40e_cloud_filter *filter, 7060 bool add) 7061 { 7062 struct i40e_aqc_cloud_filters_element_bb cld_filter; 7063 struct i40e_pf *pf = vsi->back; 7064 int ret; 7065 7066 /* Both (src/dst) valid mac_addr are not supported */ 7067 if ((is_valid_ether_addr(filter->dst_mac) && 7068 is_valid_ether_addr(filter->src_mac)) || 7069 (is_multicast_ether_addr(filter->dst_mac) && 7070 is_multicast_ether_addr(filter->src_mac))) 7071 return -EOPNOTSUPP; 7072 7073 /* Big buffer cloud filter needs 'L4 port' to be non-zero. Also, UDP 7074 * ports are not supported via big buffer now. 7075 */ 7076 if (!filter->dst_port || filter->ip_proto == IPPROTO_UDP) 7077 return -EOPNOTSUPP; 7078 7079 /* adding filter using src_port/src_ip is not supported at this stage */ 7080 if (filter->src_port || filter->src_ipv4 || 7081 !ipv6_addr_any(&filter->ip.v6.src_ip6)) 7082 return -EOPNOTSUPP; 7083 7084 /* copy element needed to add cloud filter from filter */ 7085 i40e_set_cld_element(filter, &cld_filter.element); 7086 7087 if (is_valid_ether_addr(filter->dst_mac) || 7088 is_valid_ether_addr(filter->src_mac) || 7089 is_multicast_ether_addr(filter->dst_mac) || 7090 is_multicast_ether_addr(filter->src_mac)) { 7091 /* MAC + IP : unsupported mode */ 7092 if (filter->dst_ipv4) 7093 return -EOPNOTSUPP; 7094 7095 /* since we validated that L4 port must be valid before 7096 * we get here, start with respective "flags" value 7097 * and update if vlan is present or not 7098 */ 7099 cld_filter.element.flags = 7100 cpu_to_le16(I40E_AQC_ADD_CLOUD_FILTER_MAC_PORT); 7101 7102 if (filter->vlan_id) { 7103 cld_filter.element.flags = 7104 cpu_to_le16(I40E_AQC_ADD_CLOUD_FILTER_MAC_VLAN_PORT); 7105 } 7106 7107 } else if (filter->dst_ipv4 || 7108 !ipv6_addr_any(&filter->ip.v6.dst_ip6)) { 7109 cld_filter.element.flags = 7110 cpu_to_le16(I40E_AQC_ADD_CLOUD_FILTER_IP_PORT); 7111 if (filter->n_proto == ETH_P_IPV6) 7112 cld_filter.element.flags |= 7113 cpu_to_le16(I40E_AQC_ADD_CLOUD_FLAGS_IPV6); 7114 else 7115 cld_filter.element.flags |= 7116 cpu_to_le16(I40E_AQC_ADD_CLOUD_FLAGS_IPV4); 7117 } else { 7118 dev_err(&pf->pdev->dev, 7119 "either mac or ip has to be valid for cloud filter\n"); 7120 return -EINVAL; 7121 } 7122 7123 /* Now copy L4 port in Byte 6..7 in general fields */ 7124 cld_filter.general_fields[I40E_AQC_ADD_CLOUD_FV_FLU_0X16_WORD0] = 7125 be16_to_cpu(filter->dst_port); 7126 7127 if (add) { 7128 /* Validate current device switch mode, change if necessary */ 7129 ret = i40e_validate_and_set_switch_mode(vsi); 7130 if (ret) { 7131 dev_err(&pf->pdev->dev, 7132 "failed to set switch mode, ret %d\n", 7133 ret); 7134 return ret; 7135 } 7136 7137 ret = i40e_aq_add_cloud_filters_bb(&pf->hw, filter->seid, 7138 &cld_filter, 1); 7139 } else { 7140 ret = i40e_aq_rem_cloud_filters_bb(&pf->hw, filter->seid, 7141 &cld_filter, 1); 7142 } 7143 7144 if (ret) 7145 dev_dbg(&pf->pdev->dev, 7146 "Failed to %s cloud filter(big buffer) err %d aq_err %d\n", 7147 add ? "add" : "delete", ret, pf->hw.aq.asq_last_status); 7148 else 7149 dev_info(&pf->pdev->dev, 7150 "%s cloud filter for VSI: %d, L4 port: %d\n", 7151 add ? "add" : "delete", filter->seid, 7152 ntohs(filter->dst_port)); 7153 return ret; 7154 } 7155 7156 /** 7157 * i40e_parse_cls_flower - Parse tc flower filters provided by kernel 7158 * @vsi: Pointer to VSI 7159 * @cls_flower: Pointer to struct tc_cls_flower_offload 7160 * @filter: Pointer to cloud filter structure 7161 * 7162 **/ 7163 static int i40e_parse_cls_flower(struct i40e_vsi *vsi, 7164 struct tc_cls_flower_offload *f, 7165 struct i40e_cloud_filter *filter) 7166 { 7167 u16 n_proto_mask = 0, n_proto_key = 0, addr_type = 0; 7168 struct i40e_pf *pf = vsi->back; 7169 u8 field_flags = 0; 7170 7171 if (f->dissector->used_keys & 7172 ~(BIT(FLOW_DISSECTOR_KEY_CONTROL) | 7173 BIT(FLOW_DISSECTOR_KEY_BASIC) | 7174 BIT(FLOW_DISSECTOR_KEY_ETH_ADDRS) | 7175 BIT(FLOW_DISSECTOR_KEY_VLAN) | 7176 BIT(FLOW_DISSECTOR_KEY_IPV4_ADDRS) | 7177 BIT(FLOW_DISSECTOR_KEY_IPV6_ADDRS) | 7178 BIT(FLOW_DISSECTOR_KEY_PORTS) | 7179 BIT(FLOW_DISSECTOR_KEY_ENC_KEYID))) { 7180 dev_err(&pf->pdev->dev, "Unsupported key used: 0x%x\n", 7181 f->dissector->used_keys); 7182 return -EOPNOTSUPP; 7183 } 7184 7185 if (dissector_uses_key(f->dissector, FLOW_DISSECTOR_KEY_ENC_KEYID)) { 7186 struct flow_dissector_key_keyid *key = 7187 skb_flow_dissector_target(f->dissector, 7188 FLOW_DISSECTOR_KEY_ENC_KEYID, 7189 f->key); 7190 7191 struct flow_dissector_key_keyid *mask = 7192 skb_flow_dissector_target(f->dissector, 7193 FLOW_DISSECTOR_KEY_ENC_KEYID, 7194 f->mask); 7195 7196 if (mask->keyid != 0) 7197 field_flags |= I40E_CLOUD_FIELD_TEN_ID; 7198 7199 filter->tenant_id = be32_to_cpu(key->keyid); 7200 } 7201 7202 if (dissector_uses_key(f->dissector, FLOW_DISSECTOR_KEY_BASIC)) { 7203 struct flow_dissector_key_basic *key = 7204 skb_flow_dissector_target(f->dissector, 7205 FLOW_DISSECTOR_KEY_BASIC, 7206 f->key); 7207 7208 struct flow_dissector_key_basic *mask = 7209 skb_flow_dissector_target(f->dissector, 7210 FLOW_DISSECTOR_KEY_BASIC, 7211 f->mask); 7212 7213 n_proto_key = ntohs(key->n_proto); 7214 n_proto_mask = ntohs(mask->n_proto); 7215 7216 if (n_proto_key == ETH_P_ALL) { 7217 n_proto_key = 0; 7218 n_proto_mask = 0; 7219 } 7220 filter->n_proto = n_proto_key & n_proto_mask; 7221 filter->ip_proto = key->ip_proto; 7222 } 7223 7224 if (dissector_uses_key(f->dissector, FLOW_DISSECTOR_KEY_ETH_ADDRS)) { 7225 struct flow_dissector_key_eth_addrs *key = 7226 skb_flow_dissector_target(f->dissector, 7227 FLOW_DISSECTOR_KEY_ETH_ADDRS, 7228 f->key); 7229 7230 struct flow_dissector_key_eth_addrs *mask = 7231 skb_flow_dissector_target(f->dissector, 7232 FLOW_DISSECTOR_KEY_ETH_ADDRS, 7233 f->mask); 7234 7235 /* use is_broadcast and is_zero to check for all 0xf or 0 */ 7236 if (!is_zero_ether_addr(mask->dst)) { 7237 if (is_broadcast_ether_addr(mask->dst)) { 7238 field_flags |= I40E_CLOUD_FIELD_OMAC; 7239 } else { 7240 dev_err(&pf->pdev->dev, "Bad ether dest mask %pM\n", 7241 mask->dst); 7242 return I40E_ERR_CONFIG; 7243 } 7244 } 7245 7246 if (!is_zero_ether_addr(mask->src)) { 7247 if (is_broadcast_ether_addr(mask->src)) { 7248 field_flags |= I40E_CLOUD_FIELD_IMAC; 7249 } else { 7250 dev_err(&pf->pdev->dev, "Bad ether src mask %pM\n", 7251 mask->src); 7252 return I40E_ERR_CONFIG; 7253 } 7254 } 7255 ether_addr_copy(filter->dst_mac, key->dst); 7256 ether_addr_copy(filter->src_mac, key->src); 7257 } 7258 7259 if (dissector_uses_key(f->dissector, FLOW_DISSECTOR_KEY_VLAN)) { 7260 struct flow_dissector_key_vlan *key = 7261 skb_flow_dissector_target(f->dissector, 7262 FLOW_DISSECTOR_KEY_VLAN, 7263 f->key); 7264 struct flow_dissector_key_vlan *mask = 7265 skb_flow_dissector_target(f->dissector, 7266 FLOW_DISSECTOR_KEY_VLAN, 7267 f->mask); 7268 7269 if (mask->vlan_id) { 7270 if (mask->vlan_id == VLAN_VID_MASK) { 7271 field_flags |= I40E_CLOUD_FIELD_IVLAN; 7272 7273 } else { 7274 dev_err(&pf->pdev->dev, "Bad vlan mask 0x%04x\n", 7275 mask->vlan_id); 7276 return I40E_ERR_CONFIG; 7277 } 7278 } 7279 7280 filter->vlan_id = cpu_to_be16(key->vlan_id); 7281 } 7282 7283 if (dissector_uses_key(f->dissector, FLOW_DISSECTOR_KEY_CONTROL)) { 7284 struct flow_dissector_key_control *key = 7285 skb_flow_dissector_target(f->dissector, 7286 FLOW_DISSECTOR_KEY_CONTROL, 7287 f->key); 7288 7289 addr_type = key->addr_type; 7290 } 7291 7292 if (addr_type == FLOW_DISSECTOR_KEY_IPV4_ADDRS) { 7293 struct flow_dissector_key_ipv4_addrs *key = 7294 skb_flow_dissector_target(f->dissector, 7295 FLOW_DISSECTOR_KEY_IPV4_ADDRS, 7296 f->key); 7297 struct flow_dissector_key_ipv4_addrs *mask = 7298 skb_flow_dissector_target(f->dissector, 7299 FLOW_DISSECTOR_KEY_IPV4_ADDRS, 7300 f->mask); 7301 7302 if (mask->dst) { 7303 if (mask->dst == cpu_to_be32(0xffffffff)) { 7304 field_flags |= I40E_CLOUD_FIELD_IIP; 7305 } else { 7306 dev_err(&pf->pdev->dev, "Bad ip dst mask %pI4b\n", 7307 &mask->dst); 7308 return I40E_ERR_CONFIG; 7309 } 7310 } 7311 7312 if (mask->src) { 7313 if (mask->src == cpu_to_be32(0xffffffff)) { 7314 field_flags |= I40E_CLOUD_FIELD_IIP; 7315 } else { 7316 dev_err(&pf->pdev->dev, "Bad ip src mask %pI4b\n", 7317 &mask->src); 7318 return I40E_ERR_CONFIG; 7319 } 7320 } 7321 7322 if (field_flags & I40E_CLOUD_FIELD_TEN_ID) { 7323 dev_err(&pf->pdev->dev, "Tenant id not allowed for ip filter\n"); 7324 return I40E_ERR_CONFIG; 7325 } 7326 filter->dst_ipv4 = key->dst; 7327 filter->src_ipv4 = key->src; 7328 } 7329 7330 if (addr_type == FLOW_DISSECTOR_KEY_IPV6_ADDRS) { 7331 struct flow_dissector_key_ipv6_addrs *key = 7332 skb_flow_dissector_target(f->dissector, 7333 FLOW_DISSECTOR_KEY_IPV6_ADDRS, 7334 f->key); 7335 struct flow_dissector_key_ipv6_addrs *mask = 7336 skb_flow_dissector_target(f->dissector, 7337 FLOW_DISSECTOR_KEY_IPV6_ADDRS, 7338 f->mask); 7339 7340 /* src and dest IPV6 address should not be LOOPBACK 7341 * (0:0:0:0:0:0:0:1), which can be represented as ::1 7342 */ 7343 if (ipv6_addr_loopback(&key->dst) || 7344 ipv6_addr_loopback(&key->src)) { 7345 dev_err(&pf->pdev->dev, 7346 "Bad ipv6, addr is LOOPBACK\n"); 7347 return I40E_ERR_CONFIG; 7348 } 7349 if (!ipv6_addr_any(&mask->dst) || !ipv6_addr_any(&mask->src)) 7350 field_flags |= I40E_CLOUD_FIELD_IIP; 7351 7352 memcpy(&filter->src_ipv6, &key->src.s6_addr32, 7353 sizeof(filter->src_ipv6)); 7354 memcpy(&filter->dst_ipv6, &key->dst.s6_addr32, 7355 sizeof(filter->dst_ipv6)); 7356 } 7357 7358 if (dissector_uses_key(f->dissector, FLOW_DISSECTOR_KEY_PORTS)) { 7359 struct flow_dissector_key_ports *key = 7360 skb_flow_dissector_target(f->dissector, 7361 FLOW_DISSECTOR_KEY_PORTS, 7362 f->key); 7363 struct flow_dissector_key_ports *mask = 7364 skb_flow_dissector_target(f->dissector, 7365 FLOW_DISSECTOR_KEY_PORTS, 7366 f->mask); 7367 7368 if (mask->src) { 7369 if (mask->src == cpu_to_be16(0xffff)) { 7370 field_flags |= I40E_CLOUD_FIELD_IIP; 7371 } else { 7372 dev_err(&pf->pdev->dev, "Bad src port mask 0x%04x\n", 7373 be16_to_cpu(mask->src)); 7374 return I40E_ERR_CONFIG; 7375 } 7376 } 7377 7378 if (mask->dst) { 7379 if (mask->dst == cpu_to_be16(0xffff)) { 7380 field_flags |= I40E_CLOUD_FIELD_IIP; 7381 } else { 7382 dev_err(&pf->pdev->dev, "Bad dst port mask 0x%04x\n", 7383 be16_to_cpu(mask->dst)); 7384 return I40E_ERR_CONFIG; 7385 } 7386 } 7387 7388 filter->dst_port = key->dst; 7389 filter->src_port = key->src; 7390 7391 switch (filter->ip_proto) { 7392 case IPPROTO_TCP: 7393 case IPPROTO_UDP: 7394 break; 7395 default: 7396 dev_err(&pf->pdev->dev, 7397 "Only UDP and TCP transport are supported\n"); 7398 return -EINVAL; 7399 } 7400 } 7401 filter->flags = field_flags; 7402 return 0; 7403 } 7404 7405 /** 7406 * i40e_handle_tclass: Forward to a traffic class on the device 7407 * @vsi: Pointer to VSI 7408 * @tc: traffic class index on the device 7409 * @filter: Pointer to cloud filter structure 7410 * 7411 **/ 7412 static int i40e_handle_tclass(struct i40e_vsi *vsi, u32 tc, 7413 struct i40e_cloud_filter *filter) 7414 { 7415 struct i40e_channel *ch, *ch_tmp; 7416 7417 /* direct to a traffic class on the same device */ 7418 if (tc == 0) { 7419 filter->seid = vsi->seid; 7420 return 0; 7421 } else if (vsi->tc_config.enabled_tc & BIT(tc)) { 7422 if (!filter->dst_port) { 7423 dev_err(&vsi->back->pdev->dev, 7424 "Specify destination port to direct to traffic class that is not default\n"); 7425 return -EINVAL; 7426 } 7427 if (list_empty(&vsi->ch_list)) 7428 return -EINVAL; 7429 list_for_each_entry_safe(ch, ch_tmp, &vsi->ch_list, 7430 list) { 7431 if (ch->seid == vsi->tc_seid_map[tc]) 7432 filter->seid = ch->seid; 7433 } 7434 return 0; 7435 } 7436 dev_err(&vsi->back->pdev->dev, "TC is not enabled\n"); 7437 return -EINVAL; 7438 } 7439 7440 /** 7441 * i40e_configure_clsflower - Configure tc flower filters 7442 * @vsi: Pointer to VSI 7443 * @cls_flower: Pointer to struct tc_cls_flower_offload 7444 * 7445 **/ 7446 static int i40e_configure_clsflower(struct i40e_vsi *vsi, 7447 struct tc_cls_flower_offload *cls_flower) 7448 { 7449 int tc = tc_classid_to_hwtc(vsi->netdev, cls_flower->classid); 7450 struct i40e_cloud_filter *filter = NULL; 7451 struct i40e_pf *pf = vsi->back; 7452 int err = 0; 7453 7454 if (tc < 0) { 7455 dev_err(&vsi->back->pdev->dev, "Invalid traffic class\n"); 7456 return -EOPNOTSUPP; 7457 } 7458 7459 if (test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state) || 7460 test_bit(__I40E_RESET_INTR_RECEIVED, pf->state)) 7461 return -EBUSY; 7462 7463 if (pf->fdir_pf_active_filters || 7464 (!hlist_empty(&pf->fdir_filter_list))) { 7465 dev_err(&vsi->back->pdev->dev, 7466 "Flow Director Sideband filters exists, turn ntuple off to configure cloud filters\n"); 7467 return -EINVAL; 7468 } 7469 7470 if (vsi->back->flags & I40E_FLAG_FD_SB_ENABLED) { 7471 dev_err(&vsi->back->pdev->dev, 7472 "Disable Flow Director Sideband, configuring Cloud filters via tc-flower\n"); 7473 vsi->back->flags &= ~I40E_FLAG_FD_SB_ENABLED; 7474 vsi->back->flags |= I40E_FLAG_FD_SB_TO_CLOUD_FILTER; 7475 } 7476 7477 filter = kzalloc(sizeof(*filter), GFP_KERNEL); 7478 if (!filter) 7479 return -ENOMEM; 7480 7481 filter->cookie = cls_flower->cookie; 7482 7483 err = i40e_parse_cls_flower(vsi, cls_flower, filter); 7484 if (err < 0) 7485 goto err; 7486 7487 err = i40e_handle_tclass(vsi, tc, filter); 7488 if (err < 0) 7489 goto err; 7490 7491 /* Add cloud filter */ 7492 if (filter->dst_port) 7493 err = i40e_add_del_cloud_filter_big_buf(vsi, filter, true); 7494 else 7495 err = i40e_add_del_cloud_filter(vsi, filter, true); 7496 7497 if (err) { 7498 dev_err(&pf->pdev->dev, 7499 "Failed to add cloud filter, err %s\n", 7500 i40e_stat_str(&pf->hw, err)); 7501 goto err; 7502 } 7503 7504 /* add filter to the ordered list */ 7505 INIT_HLIST_NODE(&filter->cloud_node); 7506 7507 hlist_add_head(&filter->cloud_node, &pf->cloud_filter_list); 7508 7509 pf->num_cloud_filters++; 7510 7511 return err; 7512 err: 7513 kfree(filter); 7514 return err; 7515 } 7516 7517 /** 7518 * i40e_find_cloud_filter - Find the could filter in the list 7519 * @vsi: Pointer to VSI 7520 * @cookie: filter specific cookie 7521 * 7522 **/ 7523 static struct i40e_cloud_filter *i40e_find_cloud_filter(struct i40e_vsi *vsi, 7524 unsigned long *cookie) 7525 { 7526 struct i40e_cloud_filter *filter = NULL; 7527 struct hlist_node *node2; 7528 7529 hlist_for_each_entry_safe(filter, node2, 7530 &vsi->back->cloud_filter_list, cloud_node) 7531 if (!memcmp(cookie, &filter->cookie, sizeof(filter->cookie))) 7532 return filter; 7533 return NULL; 7534 } 7535 7536 /** 7537 * i40e_delete_clsflower - Remove tc flower filters 7538 * @vsi: Pointer to VSI 7539 * @cls_flower: Pointer to struct tc_cls_flower_offload 7540 * 7541 **/ 7542 static int i40e_delete_clsflower(struct i40e_vsi *vsi, 7543 struct tc_cls_flower_offload *cls_flower) 7544 { 7545 struct i40e_cloud_filter *filter = NULL; 7546 struct i40e_pf *pf = vsi->back; 7547 int err = 0; 7548 7549 filter = i40e_find_cloud_filter(vsi, &cls_flower->cookie); 7550 7551 if (!filter) 7552 return -EINVAL; 7553 7554 hash_del(&filter->cloud_node); 7555 7556 if (filter->dst_port) 7557 err = i40e_add_del_cloud_filter_big_buf(vsi, filter, false); 7558 else 7559 err = i40e_add_del_cloud_filter(vsi, filter, false); 7560 7561 kfree(filter); 7562 if (err) { 7563 dev_err(&pf->pdev->dev, 7564 "Failed to delete cloud filter, err %s\n", 7565 i40e_stat_str(&pf->hw, err)); 7566 return i40e_aq_rc_to_posix(err, pf->hw.aq.asq_last_status); 7567 } 7568 7569 pf->num_cloud_filters--; 7570 if (!pf->num_cloud_filters) 7571 if ((pf->flags & I40E_FLAG_FD_SB_TO_CLOUD_FILTER) && 7572 !(pf->flags & I40E_FLAG_FD_SB_INACTIVE)) { 7573 pf->flags |= I40E_FLAG_FD_SB_ENABLED; 7574 pf->flags &= ~I40E_FLAG_FD_SB_TO_CLOUD_FILTER; 7575 pf->flags &= ~I40E_FLAG_FD_SB_INACTIVE; 7576 } 7577 return 0; 7578 } 7579 7580 /** 7581 * i40e_setup_tc_cls_flower - flower classifier offloads 7582 * @netdev: net device to configure 7583 * @type_data: offload data 7584 **/ 7585 static int i40e_setup_tc_cls_flower(struct i40e_netdev_priv *np, 7586 struct tc_cls_flower_offload *cls_flower) 7587 { 7588 struct i40e_vsi *vsi = np->vsi; 7589 7590 switch (cls_flower->command) { 7591 case TC_CLSFLOWER_REPLACE: 7592 return i40e_configure_clsflower(vsi, cls_flower); 7593 case TC_CLSFLOWER_DESTROY: 7594 return i40e_delete_clsflower(vsi, cls_flower); 7595 case TC_CLSFLOWER_STATS: 7596 return -EOPNOTSUPP; 7597 default: 7598 return -EOPNOTSUPP; 7599 } 7600 } 7601 7602 static int i40e_setup_tc_block_cb(enum tc_setup_type type, void *type_data, 7603 void *cb_priv) 7604 { 7605 struct i40e_netdev_priv *np = cb_priv; 7606 7607 if (!tc_cls_can_offload_and_chain0(np->vsi->netdev, type_data)) 7608 return -EOPNOTSUPP; 7609 7610 switch (type) { 7611 case TC_SETUP_CLSFLOWER: 7612 return i40e_setup_tc_cls_flower(np, type_data); 7613 7614 default: 7615 return -EOPNOTSUPP; 7616 } 7617 } 7618 7619 static int i40e_setup_tc_block(struct net_device *dev, 7620 struct tc_block_offload *f) 7621 { 7622 struct i40e_netdev_priv *np = netdev_priv(dev); 7623 7624 if (f->binder_type != TCF_BLOCK_BINDER_TYPE_CLSACT_INGRESS) 7625 return -EOPNOTSUPP; 7626 7627 switch (f->command) { 7628 case TC_BLOCK_BIND: 7629 return tcf_block_cb_register(f->block, i40e_setup_tc_block_cb, 7630 np, np, f->extack); 7631 case TC_BLOCK_UNBIND: 7632 tcf_block_cb_unregister(f->block, i40e_setup_tc_block_cb, np); 7633 return 0; 7634 default: 7635 return -EOPNOTSUPP; 7636 } 7637 } 7638 7639 static int __i40e_setup_tc(struct net_device *netdev, enum tc_setup_type type, 7640 void *type_data) 7641 { 7642 switch (type) { 7643 case TC_SETUP_QDISC_MQPRIO: 7644 return i40e_setup_tc(netdev, type_data); 7645 case TC_SETUP_BLOCK: 7646 return i40e_setup_tc_block(netdev, type_data); 7647 default: 7648 return -EOPNOTSUPP; 7649 } 7650 } 7651 7652 /** 7653 * i40e_open - Called when a network interface is made active 7654 * @netdev: network interface device structure 7655 * 7656 * The open entry point is called when a network interface is made 7657 * active by the system (IFF_UP). At this point all resources needed 7658 * for transmit and receive operations are allocated, the interrupt 7659 * handler is registered with the OS, the netdev watchdog subtask is 7660 * enabled, and the stack is notified that the interface is ready. 7661 * 7662 * Returns 0 on success, negative value on failure 7663 **/ 7664 int i40e_open(struct net_device *netdev) 7665 { 7666 struct i40e_netdev_priv *np = netdev_priv(netdev); 7667 struct i40e_vsi *vsi = np->vsi; 7668 struct i40e_pf *pf = vsi->back; 7669 int err; 7670 7671 /* disallow open during test or if eeprom is broken */ 7672 if (test_bit(__I40E_TESTING, pf->state) || 7673 test_bit(__I40E_BAD_EEPROM, pf->state)) 7674 return -EBUSY; 7675 7676 netif_carrier_off(netdev); 7677 7678 if (i40e_force_link_state(pf, true)) 7679 return -EAGAIN; 7680 7681 err = i40e_vsi_open(vsi); 7682 if (err) 7683 return err; 7684 7685 /* configure global TSO hardware offload settings */ 7686 wr32(&pf->hw, I40E_GLLAN_TSOMSK_F, be32_to_cpu(TCP_FLAG_PSH | 7687 TCP_FLAG_FIN) >> 16); 7688 wr32(&pf->hw, I40E_GLLAN_TSOMSK_M, be32_to_cpu(TCP_FLAG_PSH | 7689 TCP_FLAG_FIN | 7690 TCP_FLAG_CWR) >> 16); 7691 wr32(&pf->hw, I40E_GLLAN_TSOMSK_L, be32_to_cpu(TCP_FLAG_CWR) >> 16); 7692 7693 udp_tunnel_get_rx_info(netdev); 7694 7695 return 0; 7696 } 7697 7698 /** 7699 * i40e_vsi_open - 7700 * @vsi: the VSI to open 7701 * 7702 * Finish initialization of the VSI. 7703 * 7704 * Returns 0 on success, negative value on failure 7705 * 7706 * Note: expects to be called while under rtnl_lock() 7707 **/ 7708 int i40e_vsi_open(struct i40e_vsi *vsi) 7709 { 7710 struct i40e_pf *pf = vsi->back; 7711 char int_name[I40E_INT_NAME_STR_LEN]; 7712 int err; 7713 7714 /* allocate descriptors */ 7715 err = i40e_vsi_setup_tx_resources(vsi); 7716 if (err) 7717 goto err_setup_tx; 7718 err = i40e_vsi_setup_rx_resources(vsi); 7719 if (err) 7720 goto err_setup_rx; 7721 7722 err = i40e_vsi_configure(vsi); 7723 if (err) 7724 goto err_setup_rx; 7725 7726 if (vsi->netdev) { 7727 snprintf(int_name, sizeof(int_name) - 1, "%s-%s", 7728 dev_driver_string(&pf->pdev->dev), vsi->netdev->name); 7729 err = i40e_vsi_request_irq(vsi, int_name); 7730 if (err) 7731 goto err_setup_rx; 7732 7733 /* Notify the stack of the actual queue counts. */ 7734 err = netif_set_real_num_tx_queues(vsi->netdev, 7735 vsi->num_queue_pairs); 7736 if (err) 7737 goto err_set_queues; 7738 7739 err = netif_set_real_num_rx_queues(vsi->netdev, 7740 vsi->num_queue_pairs); 7741 if (err) 7742 goto err_set_queues; 7743 7744 } else if (vsi->type == I40E_VSI_FDIR) { 7745 snprintf(int_name, sizeof(int_name) - 1, "%s-%s:fdir", 7746 dev_driver_string(&pf->pdev->dev), 7747 dev_name(&pf->pdev->dev)); 7748 err = i40e_vsi_request_irq(vsi, int_name); 7749 7750 } else { 7751 err = -EINVAL; 7752 goto err_setup_rx; 7753 } 7754 7755 err = i40e_up_complete(vsi); 7756 if (err) 7757 goto err_up_complete; 7758 7759 return 0; 7760 7761 err_up_complete: 7762 i40e_down(vsi); 7763 err_set_queues: 7764 i40e_vsi_free_irq(vsi); 7765 err_setup_rx: 7766 i40e_vsi_free_rx_resources(vsi); 7767 err_setup_tx: 7768 i40e_vsi_free_tx_resources(vsi); 7769 if (vsi == pf->vsi[pf->lan_vsi]) 7770 i40e_do_reset(pf, I40E_PF_RESET_FLAG, true); 7771 7772 return err; 7773 } 7774 7775 /** 7776 * i40e_fdir_filter_exit - Cleans up the Flow Director accounting 7777 * @pf: Pointer to PF 7778 * 7779 * This function destroys the hlist where all the Flow Director 7780 * filters were saved. 7781 **/ 7782 static void i40e_fdir_filter_exit(struct i40e_pf *pf) 7783 { 7784 struct i40e_fdir_filter *filter; 7785 struct i40e_flex_pit *pit_entry, *tmp; 7786 struct hlist_node *node2; 7787 7788 hlist_for_each_entry_safe(filter, node2, 7789 &pf->fdir_filter_list, fdir_node) { 7790 hlist_del(&filter->fdir_node); 7791 kfree(filter); 7792 } 7793 7794 list_for_each_entry_safe(pit_entry, tmp, &pf->l3_flex_pit_list, list) { 7795 list_del(&pit_entry->list); 7796 kfree(pit_entry); 7797 } 7798 INIT_LIST_HEAD(&pf->l3_flex_pit_list); 7799 7800 list_for_each_entry_safe(pit_entry, tmp, &pf->l4_flex_pit_list, list) { 7801 list_del(&pit_entry->list); 7802 kfree(pit_entry); 7803 } 7804 INIT_LIST_HEAD(&pf->l4_flex_pit_list); 7805 7806 pf->fdir_pf_active_filters = 0; 7807 pf->fd_tcp4_filter_cnt = 0; 7808 pf->fd_udp4_filter_cnt = 0; 7809 pf->fd_sctp4_filter_cnt = 0; 7810 pf->fd_ip4_filter_cnt = 0; 7811 7812 /* Reprogram the default input set for TCP/IPv4 */ 7813 i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV4_TCP, 7814 I40E_L3_SRC_MASK | I40E_L3_DST_MASK | 7815 I40E_L4_SRC_MASK | I40E_L4_DST_MASK); 7816 7817 /* Reprogram the default input set for UDP/IPv4 */ 7818 i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV4_UDP, 7819 I40E_L3_SRC_MASK | I40E_L3_DST_MASK | 7820 I40E_L4_SRC_MASK | I40E_L4_DST_MASK); 7821 7822 /* Reprogram the default input set for SCTP/IPv4 */ 7823 i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV4_SCTP, 7824 I40E_L3_SRC_MASK | I40E_L3_DST_MASK | 7825 I40E_L4_SRC_MASK | I40E_L4_DST_MASK); 7826 7827 /* Reprogram the default input set for Other/IPv4 */ 7828 i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV4_OTHER, 7829 I40E_L3_SRC_MASK | I40E_L3_DST_MASK); 7830 7831 i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_FRAG_IPV4, 7832 I40E_L3_SRC_MASK | I40E_L3_DST_MASK); 7833 } 7834 7835 /** 7836 * i40e_cloud_filter_exit - Cleans up the cloud filters 7837 * @pf: Pointer to PF 7838 * 7839 * This function destroys the hlist where all the cloud filters 7840 * were saved. 7841 **/ 7842 static void i40e_cloud_filter_exit(struct i40e_pf *pf) 7843 { 7844 struct i40e_cloud_filter *cfilter; 7845 struct hlist_node *node; 7846 7847 hlist_for_each_entry_safe(cfilter, node, 7848 &pf->cloud_filter_list, cloud_node) { 7849 hlist_del(&cfilter->cloud_node); 7850 kfree(cfilter); 7851 } 7852 pf->num_cloud_filters = 0; 7853 7854 if ((pf->flags & I40E_FLAG_FD_SB_TO_CLOUD_FILTER) && 7855 !(pf->flags & I40E_FLAG_FD_SB_INACTIVE)) { 7856 pf->flags |= I40E_FLAG_FD_SB_ENABLED; 7857 pf->flags &= ~I40E_FLAG_FD_SB_TO_CLOUD_FILTER; 7858 pf->flags &= ~I40E_FLAG_FD_SB_INACTIVE; 7859 } 7860 } 7861 7862 /** 7863 * i40e_close - Disables a network interface 7864 * @netdev: network interface device structure 7865 * 7866 * The close entry point is called when an interface is de-activated 7867 * by the OS. The hardware is still under the driver's control, but 7868 * this netdev interface is disabled. 7869 * 7870 * Returns 0, this is not allowed to fail 7871 **/ 7872 int i40e_close(struct net_device *netdev) 7873 { 7874 struct i40e_netdev_priv *np = netdev_priv(netdev); 7875 struct i40e_vsi *vsi = np->vsi; 7876 7877 i40e_vsi_close(vsi); 7878 7879 return 0; 7880 } 7881 7882 /** 7883 * i40e_do_reset - Start a PF or Core Reset sequence 7884 * @pf: board private structure 7885 * @reset_flags: which reset is requested 7886 * @lock_acquired: indicates whether or not the lock has been acquired 7887 * before this function was called. 7888 * 7889 * The essential difference in resets is that the PF Reset 7890 * doesn't clear the packet buffers, doesn't reset the PE 7891 * firmware, and doesn't bother the other PFs on the chip. 7892 **/ 7893 void i40e_do_reset(struct i40e_pf *pf, u32 reset_flags, bool lock_acquired) 7894 { 7895 u32 val; 7896 7897 WARN_ON(in_interrupt()); 7898 7899 7900 /* do the biggest reset indicated */ 7901 if (reset_flags & BIT_ULL(__I40E_GLOBAL_RESET_REQUESTED)) { 7902 7903 /* Request a Global Reset 7904 * 7905 * This will start the chip's countdown to the actual full 7906 * chip reset event, and a warning interrupt to be sent 7907 * to all PFs, including the requestor. Our handler 7908 * for the warning interrupt will deal with the shutdown 7909 * and recovery of the switch setup. 7910 */ 7911 dev_dbg(&pf->pdev->dev, "GlobalR requested\n"); 7912 val = rd32(&pf->hw, I40E_GLGEN_RTRIG); 7913 val |= I40E_GLGEN_RTRIG_GLOBR_MASK; 7914 wr32(&pf->hw, I40E_GLGEN_RTRIG, val); 7915 7916 } else if (reset_flags & BIT_ULL(__I40E_CORE_RESET_REQUESTED)) { 7917 7918 /* Request a Core Reset 7919 * 7920 * Same as Global Reset, except does *not* include the MAC/PHY 7921 */ 7922 dev_dbg(&pf->pdev->dev, "CoreR requested\n"); 7923 val = rd32(&pf->hw, I40E_GLGEN_RTRIG); 7924 val |= I40E_GLGEN_RTRIG_CORER_MASK; 7925 wr32(&pf->hw, I40E_GLGEN_RTRIG, val); 7926 i40e_flush(&pf->hw); 7927 7928 } else if (reset_flags & I40E_PF_RESET_FLAG) { 7929 7930 /* Request a PF Reset 7931 * 7932 * Resets only the PF-specific registers 7933 * 7934 * This goes directly to the tear-down and rebuild of 7935 * the switch, since we need to do all the recovery as 7936 * for the Core Reset. 7937 */ 7938 dev_dbg(&pf->pdev->dev, "PFR requested\n"); 7939 i40e_handle_reset_warning(pf, lock_acquired); 7940 7941 } else if (reset_flags & BIT_ULL(__I40E_REINIT_REQUESTED)) { 7942 int v; 7943 7944 /* Find the VSI(s) that requested a re-init */ 7945 dev_info(&pf->pdev->dev, 7946 "VSI reinit requested\n"); 7947 for (v = 0; v < pf->num_alloc_vsi; v++) { 7948 struct i40e_vsi *vsi = pf->vsi[v]; 7949 7950 if (vsi != NULL && 7951 test_and_clear_bit(__I40E_VSI_REINIT_REQUESTED, 7952 vsi->state)) 7953 i40e_vsi_reinit_locked(pf->vsi[v]); 7954 } 7955 } else if (reset_flags & BIT_ULL(__I40E_DOWN_REQUESTED)) { 7956 int v; 7957 7958 /* Find the VSI(s) that needs to be brought down */ 7959 dev_info(&pf->pdev->dev, "VSI down requested\n"); 7960 for (v = 0; v < pf->num_alloc_vsi; v++) { 7961 struct i40e_vsi *vsi = pf->vsi[v]; 7962 7963 if (vsi != NULL && 7964 test_and_clear_bit(__I40E_VSI_DOWN_REQUESTED, 7965 vsi->state)) { 7966 set_bit(__I40E_VSI_DOWN, vsi->state); 7967 i40e_down(vsi); 7968 } 7969 } 7970 } else { 7971 dev_info(&pf->pdev->dev, 7972 "bad reset request 0x%08x\n", reset_flags); 7973 } 7974 } 7975 7976 #ifdef CONFIG_I40E_DCB 7977 /** 7978 * i40e_dcb_need_reconfig - Check if DCB needs reconfig 7979 * @pf: board private structure 7980 * @old_cfg: current DCB config 7981 * @new_cfg: new DCB config 7982 **/ 7983 bool i40e_dcb_need_reconfig(struct i40e_pf *pf, 7984 struct i40e_dcbx_config *old_cfg, 7985 struct i40e_dcbx_config *new_cfg) 7986 { 7987 bool need_reconfig = false; 7988 7989 /* Check if ETS configuration has changed */ 7990 if (memcmp(&new_cfg->etscfg, 7991 &old_cfg->etscfg, 7992 sizeof(new_cfg->etscfg))) { 7993 /* If Priority Table has changed reconfig is needed */ 7994 if (memcmp(&new_cfg->etscfg.prioritytable, 7995 &old_cfg->etscfg.prioritytable, 7996 sizeof(new_cfg->etscfg.prioritytable))) { 7997 need_reconfig = true; 7998 dev_dbg(&pf->pdev->dev, "ETS UP2TC changed.\n"); 7999 } 8000 8001 if (memcmp(&new_cfg->etscfg.tcbwtable, 8002 &old_cfg->etscfg.tcbwtable, 8003 sizeof(new_cfg->etscfg.tcbwtable))) 8004 dev_dbg(&pf->pdev->dev, "ETS TC BW Table changed.\n"); 8005 8006 if (memcmp(&new_cfg->etscfg.tsatable, 8007 &old_cfg->etscfg.tsatable, 8008 sizeof(new_cfg->etscfg.tsatable))) 8009 dev_dbg(&pf->pdev->dev, "ETS TSA Table changed.\n"); 8010 } 8011 8012 /* Check if PFC configuration has changed */ 8013 if (memcmp(&new_cfg->pfc, 8014 &old_cfg->pfc, 8015 sizeof(new_cfg->pfc))) { 8016 need_reconfig = true; 8017 dev_dbg(&pf->pdev->dev, "PFC config change detected.\n"); 8018 } 8019 8020 /* Check if APP Table has changed */ 8021 if (memcmp(&new_cfg->app, 8022 &old_cfg->app, 8023 sizeof(new_cfg->app))) { 8024 need_reconfig = true; 8025 dev_dbg(&pf->pdev->dev, "APP Table change detected.\n"); 8026 } 8027 8028 dev_dbg(&pf->pdev->dev, "dcb need_reconfig=%d\n", need_reconfig); 8029 return need_reconfig; 8030 } 8031 8032 /** 8033 * i40e_handle_lldp_event - Handle LLDP Change MIB event 8034 * @pf: board private structure 8035 * @e: event info posted on ARQ 8036 **/ 8037 static int i40e_handle_lldp_event(struct i40e_pf *pf, 8038 struct i40e_arq_event_info *e) 8039 { 8040 struct i40e_aqc_lldp_get_mib *mib = 8041 (struct i40e_aqc_lldp_get_mib *)&e->desc.params.raw; 8042 struct i40e_hw *hw = &pf->hw; 8043 struct i40e_dcbx_config tmp_dcbx_cfg; 8044 bool need_reconfig = false; 8045 int ret = 0; 8046 u8 type; 8047 8048 /* Not DCB capable or capability disabled */ 8049 if (!(pf->flags & I40E_FLAG_DCB_CAPABLE)) 8050 return ret; 8051 8052 /* Ignore if event is not for Nearest Bridge */ 8053 type = ((mib->type >> I40E_AQ_LLDP_BRIDGE_TYPE_SHIFT) 8054 & I40E_AQ_LLDP_BRIDGE_TYPE_MASK); 8055 dev_dbg(&pf->pdev->dev, "LLDP event mib bridge type 0x%x\n", type); 8056 if (type != I40E_AQ_LLDP_BRIDGE_TYPE_NEAREST_BRIDGE) 8057 return ret; 8058 8059 /* Check MIB Type and return if event for Remote MIB update */ 8060 type = mib->type & I40E_AQ_LLDP_MIB_TYPE_MASK; 8061 dev_dbg(&pf->pdev->dev, 8062 "LLDP event mib type %s\n", type ? "remote" : "local"); 8063 if (type == I40E_AQ_LLDP_MIB_REMOTE) { 8064 /* Update the remote cached instance and return */ 8065 ret = i40e_aq_get_dcb_config(hw, I40E_AQ_LLDP_MIB_REMOTE, 8066 I40E_AQ_LLDP_BRIDGE_TYPE_NEAREST_BRIDGE, 8067 &hw->remote_dcbx_config); 8068 goto exit; 8069 } 8070 8071 /* Store the old configuration */ 8072 tmp_dcbx_cfg = hw->local_dcbx_config; 8073 8074 /* Reset the old DCBx configuration data */ 8075 memset(&hw->local_dcbx_config, 0, sizeof(hw->local_dcbx_config)); 8076 /* Get updated DCBX data from firmware */ 8077 ret = i40e_get_dcb_config(&pf->hw); 8078 if (ret) { 8079 dev_info(&pf->pdev->dev, 8080 "Failed querying DCB configuration data from firmware, err %s aq_err %s\n", 8081 i40e_stat_str(&pf->hw, ret), 8082 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 8083 goto exit; 8084 } 8085 8086 /* No change detected in DCBX configs */ 8087 if (!memcmp(&tmp_dcbx_cfg, &hw->local_dcbx_config, 8088 sizeof(tmp_dcbx_cfg))) { 8089 dev_dbg(&pf->pdev->dev, "No change detected in DCBX configuration.\n"); 8090 goto exit; 8091 } 8092 8093 need_reconfig = i40e_dcb_need_reconfig(pf, &tmp_dcbx_cfg, 8094 &hw->local_dcbx_config); 8095 8096 i40e_dcbnl_flush_apps(pf, &tmp_dcbx_cfg, &hw->local_dcbx_config); 8097 8098 if (!need_reconfig) 8099 goto exit; 8100 8101 /* Enable DCB tagging only when more than one TC */ 8102 if (i40e_dcb_get_num_tc(&hw->local_dcbx_config) > 1) 8103 pf->flags |= I40E_FLAG_DCB_ENABLED; 8104 else 8105 pf->flags &= ~I40E_FLAG_DCB_ENABLED; 8106 8107 set_bit(__I40E_PORT_SUSPENDED, pf->state); 8108 /* Reconfiguration needed quiesce all VSIs */ 8109 i40e_pf_quiesce_all_vsi(pf); 8110 8111 /* Changes in configuration update VEB/VSI */ 8112 i40e_dcb_reconfigure(pf); 8113 8114 ret = i40e_resume_port_tx(pf); 8115 8116 clear_bit(__I40E_PORT_SUSPENDED, pf->state); 8117 /* In case of error no point in resuming VSIs */ 8118 if (ret) 8119 goto exit; 8120 8121 /* Wait for the PF's queues to be disabled */ 8122 ret = i40e_pf_wait_queues_disabled(pf); 8123 if (ret) { 8124 /* Schedule PF reset to recover */ 8125 set_bit(__I40E_PF_RESET_REQUESTED, pf->state); 8126 i40e_service_event_schedule(pf); 8127 } else { 8128 i40e_pf_unquiesce_all_vsi(pf); 8129 set_bit(__I40E_CLIENT_SERVICE_REQUESTED, pf->state); 8130 set_bit(__I40E_CLIENT_L2_CHANGE, pf->state); 8131 } 8132 8133 exit: 8134 return ret; 8135 } 8136 #endif /* CONFIG_I40E_DCB */ 8137 8138 /** 8139 * i40e_do_reset_safe - Protected reset path for userland calls. 8140 * @pf: board private structure 8141 * @reset_flags: which reset is requested 8142 * 8143 **/ 8144 void i40e_do_reset_safe(struct i40e_pf *pf, u32 reset_flags) 8145 { 8146 rtnl_lock(); 8147 i40e_do_reset(pf, reset_flags, true); 8148 rtnl_unlock(); 8149 } 8150 8151 /** 8152 * i40e_handle_lan_overflow_event - Handler for LAN queue overflow event 8153 * @pf: board private structure 8154 * @e: event info posted on ARQ 8155 * 8156 * Handler for LAN Queue Overflow Event generated by the firmware for PF 8157 * and VF queues 8158 **/ 8159 static void i40e_handle_lan_overflow_event(struct i40e_pf *pf, 8160 struct i40e_arq_event_info *e) 8161 { 8162 struct i40e_aqc_lan_overflow *data = 8163 (struct i40e_aqc_lan_overflow *)&e->desc.params.raw; 8164 u32 queue = le32_to_cpu(data->prtdcb_rupto); 8165 u32 qtx_ctl = le32_to_cpu(data->otx_ctl); 8166 struct i40e_hw *hw = &pf->hw; 8167 struct i40e_vf *vf; 8168 u16 vf_id; 8169 8170 dev_dbg(&pf->pdev->dev, "overflow Rx Queue Number = %d QTX_CTL=0x%08x\n", 8171 queue, qtx_ctl); 8172 8173 /* Queue belongs to VF, find the VF and issue VF reset */ 8174 if (((qtx_ctl & I40E_QTX_CTL_PFVF_Q_MASK) 8175 >> I40E_QTX_CTL_PFVF_Q_SHIFT) == I40E_QTX_CTL_VF_QUEUE) { 8176 vf_id = (u16)((qtx_ctl & I40E_QTX_CTL_VFVM_INDX_MASK) 8177 >> I40E_QTX_CTL_VFVM_INDX_SHIFT); 8178 vf_id -= hw->func_caps.vf_base_id; 8179 vf = &pf->vf[vf_id]; 8180 i40e_vc_notify_vf_reset(vf); 8181 /* Allow VF to process pending reset notification */ 8182 msleep(20); 8183 i40e_reset_vf(vf, false); 8184 } 8185 } 8186 8187 /** 8188 * i40e_get_cur_guaranteed_fd_count - Get the consumed guaranteed FD filters 8189 * @pf: board private structure 8190 **/ 8191 u32 i40e_get_cur_guaranteed_fd_count(struct i40e_pf *pf) 8192 { 8193 u32 val, fcnt_prog; 8194 8195 val = rd32(&pf->hw, I40E_PFQF_FDSTAT); 8196 fcnt_prog = (val & I40E_PFQF_FDSTAT_GUARANT_CNT_MASK); 8197 return fcnt_prog; 8198 } 8199 8200 /** 8201 * i40e_get_current_fd_count - Get total FD filters programmed for this PF 8202 * @pf: board private structure 8203 **/ 8204 u32 i40e_get_current_fd_count(struct i40e_pf *pf) 8205 { 8206 u32 val, fcnt_prog; 8207 8208 val = rd32(&pf->hw, I40E_PFQF_FDSTAT); 8209 fcnt_prog = (val & I40E_PFQF_FDSTAT_GUARANT_CNT_MASK) + 8210 ((val & I40E_PFQF_FDSTAT_BEST_CNT_MASK) >> 8211 I40E_PFQF_FDSTAT_BEST_CNT_SHIFT); 8212 return fcnt_prog; 8213 } 8214 8215 /** 8216 * i40e_get_global_fd_count - Get total FD filters programmed on device 8217 * @pf: board private structure 8218 **/ 8219 u32 i40e_get_global_fd_count(struct i40e_pf *pf) 8220 { 8221 u32 val, fcnt_prog; 8222 8223 val = rd32(&pf->hw, I40E_GLQF_FDCNT_0); 8224 fcnt_prog = (val & I40E_GLQF_FDCNT_0_GUARANT_CNT_MASK) + 8225 ((val & I40E_GLQF_FDCNT_0_BESTCNT_MASK) >> 8226 I40E_GLQF_FDCNT_0_BESTCNT_SHIFT); 8227 return fcnt_prog; 8228 } 8229 8230 /** 8231 * i40e_reenable_fdir_sb - Restore FDir SB capability 8232 * @pf: board private structure 8233 **/ 8234 static void i40e_reenable_fdir_sb(struct i40e_pf *pf) 8235 { 8236 if (test_and_clear_bit(__I40E_FD_SB_AUTO_DISABLED, pf->state)) 8237 if ((pf->flags & I40E_FLAG_FD_SB_ENABLED) && 8238 (I40E_DEBUG_FD & pf->hw.debug_mask)) 8239 dev_info(&pf->pdev->dev, "FD Sideband/ntuple is being enabled since we have space in the table now\n"); 8240 } 8241 8242 /** 8243 * i40e_reenable_fdir_atr - Restore FDir ATR capability 8244 * @pf: board private structure 8245 **/ 8246 static void i40e_reenable_fdir_atr(struct i40e_pf *pf) 8247 { 8248 if (test_and_clear_bit(__I40E_FD_ATR_AUTO_DISABLED, pf->state)) { 8249 /* ATR uses the same filtering logic as SB rules. It only 8250 * functions properly if the input set mask is at the default 8251 * settings. It is safe to restore the default input set 8252 * because there are no active TCPv4 filter rules. 8253 */ 8254 i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV4_TCP, 8255 I40E_L3_SRC_MASK | I40E_L3_DST_MASK | 8256 I40E_L4_SRC_MASK | I40E_L4_DST_MASK); 8257 8258 if ((pf->flags & I40E_FLAG_FD_ATR_ENABLED) && 8259 (I40E_DEBUG_FD & pf->hw.debug_mask)) 8260 dev_info(&pf->pdev->dev, "ATR is being enabled since we have space in the table and there are no conflicting ntuple rules\n"); 8261 } 8262 } 8263 8264 /** 8265 * i40e_delete_invalid_filter - Delete an invalid FDIR filter 8266 * @pf: board private structure 8267 * @filter: FDir filter to remove 8268 */ 8269 static void i40e_delete_invalid_filter(struct i40e_pf *pf, 8270 struct i40e_fdir_filter *filter) 8271 { 8272 /* Update counters */ 8273 pf->fdir_pf_active_filters--; 8274 pf->fd_inv = 0; 8275 8276 switch (filter->flow_type) { 8277 case TCP_V4_FLOW: 8278 pf->fd_tcp4_filter_cnt--; 8279 break; 8280 case UDP_V4_FLOW: 8281 pf->fd_udp4_filter_cnt--; 8282 break; 8283 case SCTP_V4_FLOW: 8284 pf->fd_sctp4_filter_cnt--; 8285 break; 8286 case IP_USER_FLOW: 8287 switch (filter->ip4_proto) { 8288 case IPPROTO_TCP: 8289 pf->fd_tcp4_filter_cnt--; 8290 break; 8291 case IPPROTO_UDP: 8292 pf->fd_udp4_filter_cnt--; 8293 break; 8294 case IPPROTO_SCTP: 8295 pf->fd_sctp4_filter_cnt--; 8296 break; 8297 case IPPROTO_IP: 8298 pf->fd_ip4_filter_cnt--; 8299 break; 8300 } 8301 break; 8302 } 8303 8304 /* Remove the filter from the list and free memory */ 8305 hlist_del(&filter->fdir_node); 8306 kfree(filter); 8307 } 8308 8309 /** 8310 * i40e_fdir_check_and_reenable - Function to reenabe FD ATR or SB if disabled 8311 * @pf: board private structure 8312 **/ 8313 void i40e_fdir_check_and_reenable(struct i40e_pf *pf) 8314 { 8315 struct i40e_fdir_filter *filter; 8316 u32 fcnt_prog, fcnt_avail; 8317 struct hlist_node *node; 8318 8319 if (test_bit(__I40E_FD_FLUSH_REQUESTED, pf->state)) 8320 return; 8321 8322 /* Check if we have enough room to re-enable FDir SB capability. */ 8323 fcnt_prog = i40e_get_global_fd_count(pf); 8324 fcnt_avail = pf->fdir_pf_filter_count; 8325 if ((fcnt_prog < (fcnt_avail - I40E_FDIR_BUFFER_HEAD_ROOM)) || 8326 (pf->fd_add_err == 0) || 8327 (i40e_get_current_atr_cnt(pf) < pf->fd_atr_cnt)) 8328 i40e_reenable_fdir_sb(pf); 8329 8330 /* We should wait for even more space before re-enabling ATR. 8331 * Additionally, we cannot enable ATR as long as we still have TCP SB 8332 * rules active. 8333 */ 8334 if ((fcnt_prog < (fcnt_avail - I40E_FDIR_BUFFER_HEAD_ROOM_FOR_ATR)) && 8335 (pf->fd_tcp4_filter_cnt == 0)) 8336 i40e_reenable_fdir_atr(pf); 8337 8338 /* if hw had a problem adding a filter, delete it */ 8339 if (pf->fd_inv > 0) { 8340 hlist_for_each_entry_safe(filter, node, 8341 &pf->fdir_filter_list, fdir_node) 8342 if (filter->fd_id == pf->fd_inv) 8343 i40e_delete_invalid_filter(pf, filter); 8344 } 8345 } 8346 8347 #define I40E_MIN_FD_FLUSH_INTERVAL 10 8348 #define I40E_MIN_FD_FLUSH_SB_ATR_UNSTABLE 30 8349 /** 8350 * i40e_fdir_flush_and_replay - Function to flush all FD filters and replay SB 8351 * @pf: board private structure 8352 **/ 8353 static void i40e_fdir_flush_and_replay(struct i40e_pf *pf) 8354 { 8355 unsigned long min_flush_time; 8356 int flush_wait_retry = 50; 8357 bool disable_atr = false; 8358 int fd_room; 8359 int reg; 8360 8361 if (!time_after(jiffies, pf->fd_flush_timestamp + 8362 (I40E_MIN_FD_FLUSH_INTERVAL * HZ))) 8363 return; 8364 8365 /* If the flush is happening too quick and we have mostly SB rules we 8366 * should not re-enable ATR for some time. 8367 */ 8368 min_flush_time = pf->fd_flush_timestamp + 8369 (I40E_MIN_FD_FLUSH_SB_ATR_UNSTABLE * HZ); 8370 fd_room = pf->fdir_pf_filter_count - pf->fdir_pf_active_filters; 8371 8372 if (!(time_after(jiffies, min_flush_time)) && 8373 (fd_room < I40E_FDIR_BUFFER_HEAD_ROOM_FOR_ATR)) { 8374 if (I40E_DEBUG_FD & pf->hw.debug_mask) 8375 dev_info(&pf->pdev->dev, "ATR disabled, not enough FD filter space.\n"); 8376 disable_atr = true; 8377 } 8378 8379 pf->fd_flush_timestamp = jiffies; 8380 set_bit(__I40E_FD_ATR_AUTO_DISABLED, pf->state); 8381 /* flush all filters */ 8382 wr32(&pf->hw, I40E_PFQF_CTL_1, 8383 I40E_PFQF_CTL_1_CLEARFDTABLE_MASK); 8384 i40e_flush(&pf->hw); 8385 pf->fd_flush_cnt++; 8386 pf->fd_add_err = 0; 8387 do { 8388 /* Check FD flush status every 5-6msec */ 8389 usleep_range(5000, 6000); 8390 reg = rd32(&pf->hw, I40E_PFQF_CTL_1); 8391 if (!(reg & I40E_PFQF_CTL_1_CLEARFDTABLE_MASK)) 8392 break; 8393 } while (flush_wait_retry--); 8394 if (reg & I40E_PFQF_CTL_1_CLEARFDTABLE_MASK) { 8395 dev_warn(&pf->pdev->dev, "FD table did not flush, needs more time\n"); 8396 } else { 8397 /* replay sideband filters */ 8398 i40e_fdir_filter_restore(pf->vsi[pf->lan_vsi]); 8399 if (!disable_atr && !pf->fd_tcp4_filter_cnt) 8400 clear_bit(__I40E_FD_ATR_AUTO_DISABLED, pf->state); 8401 clear_bit(__I40E_FD_FLUSH_REQUESTED, pf->state); 8402 if (I40E_DEBUG_FD & pf->hw.debug_mask) 8403 dev_info(&pf->pdev->dev, "FD Filter table flushed and FD-SB replayed.\n"); 8404 } 8405 } 8406 8407 /** 8408 * i40e_get_current_atr_count - Get the count of total FD ATR filters programmed 8409 * @pf: board private structure 8410 **/ 8411 u32 i40e_get_current_atr_cnt(struct i40e_pf *pf) 8412 { 8413 return i40e_get_current_fd_count(pf) - pf->fdir_pf_active_filters; 8414 } 8415 8416 /* We can see up to 256 filter programming desc in transit if the filters are 8417 * being applied really fast; before we see the first 8418 * filter miss error on Rx queue 0. Accumulating enough error messages before 8419 * reacting will make sure we don't cause flush too often. 8420 */ 8421 #define I40E_MAX_FD_PROGRAM_ERROR 256 8422 8423 /** 8424 * i40e_fdir_reinit_subtask - Worker thread to reinit FDIR filter table 8425 * @pf: board private structure 8426 **/ 8427 static void i40e_fdir_reinit_subtask(struct i40e_pf *pf) 8428 { 8429 8430 /* if interface is down do nothing */ 8431 if (test_bit(__I40E_DOWN, pf->state)) 8432 return; 8433 8434 if (test_bit(__I40E_FD_FLUSH_REQUESTED, pf->state)) 8435 i40e_fdir_flush_and_replay(pf); 8436 8437 i40e_fdir_check_and_reenable(pf); 8438 8439 } 8440 8441 /** 8442 * i40e_vsi_link_event - notify VSI of a link event 8443 * @vsi: vsi to be notified 8444 * @link_up: link up or down 8445 **/ 8446 static void i40e_vsi_link_event(struct i40e_vsi *vsi, bool link_up) 8447 { 8448 if (!vsi || test_bit(__I40E_VSI_DOWN, vsi->state)) 8449 return; 8450 8451 switch (vsi->type) { 8452 case I40E_VSI_MAIN: 8453 if (!vsi->netdev || !vsi->netdev_registered) 8454 break; 8455 8456 if (link_up) { 8457 netif_carrier_on(vsi->netdev); 8458 netif_tx_wake_all_queues(vsi->netdev); 8459 } else { 8460 netif_carrier_off(vsi->netdev); 8461 netif_tx_stop_all_queues(vsi->netdev); 8462 } 8463 break; 8464 8465 case I40E_VSI_SRIOV: 8466 case I40E_VSI_VMDQ2: 8467 case I40E_VSI_CTRL: 8468 case I40E_VSI_IWARP: 8469 case I40E_VSI_MIRROR: 8470 default: 8471 /* there is no notification for other VSIs */ 8472 break; 8473 } 8474 } 8475 8476 /** 8477 * i40e_veb_link_event - notify elements on the veb of a link event 8478 * @veb: veb to be notified 8479 * @link_up: link up or down 8480 **/ 8481 static void i40e_veb_link_event(struct i40e_veb *veb, bool link_up) 8482 { 8483 struct i40e_pf *pf; 8484 int i; 8485 8486 if (!veb || !veb->pf) 8487 return; 8488 pf = veb->pf; 8489 8490 /* depth first... */ 8491 for (i = 0; i < I40E_MAX_VEB; i++) 8492 if (pf->veb[i] && (pf->veb[i]->uplink_seid == veb->seid)) 8493 i40e_veb_link_event(pf->veb[i], link_up); 8494 8495 /* ... now the local VSIs */ 8496 for (i = 0; i < pf->num_alloc_vsi; i++) 8497 if (pf->vsi[i] && (pf->vsi[i]->uplink_seid == veb->seid)) 8498 i40e_vsi_link_event(pf->vsi[i], link_up); 8499 } 8500 8501 /** 8502 * i40e_link_event - Update netif_carrier status 8503 * @pf: board private structure 8504 **/ 8505 static void i40e_link_event(struct i40e_pf *pf) 8506 { 8507 struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi]; 8508 u8 new_link_speed, old_link_speed; 8509 i40e_status status; 8510 bool new_link, old_link; 8511 8512 /* save off old link status information */ 8513 pf->hw.phy.link_info_old = pf->hw.phy.link_info; 8514 8515 /* set this to force the get_link_status call to refresh state */ 8516 pf->hw.phy.get_link_info = true; 8517 8518 old_link = (pf->hw.phy.link_info_old.link_info & I40E_AQ_LINK_UP); 8519 8520 status = i40e_get_link_status(&pf->hw, &new_link); 8521 8522 /* On success, disable temp link polling */ 8523 if (status == I40E_SUCCESS) { 8524 clear_bit(__I40E_TEMP_LINK_POLLING, pf->state); 8525 } else { 8526 /* Enable link polling temporarily until i40e_get_link_status 8527 * returns I40E_SUCCESS 8528 */ 8529 set_bit(__I40E_TEMP_LINK_POLLING, pf->state); 8530 dev_dbg(&pf->pdev->dev, "couldn't get link state, status: %d\n", 8531 status); 8532 return; 8533 } 8534 8535 old_link_speed = pf->hw.phy.link_info_old.link_speed; 8536 new_link_speed = pf->hw.phy.link_info.link_speed; 8537 8538 if (new_link == old_link && 8539 new_link_speed == old_link_speed && 8540 (test_bit(__I40E_VSI_DOWN, vsi->state) || 8541 new_link == netif_carrier_ok(vsi->netdev))) 8542 return; 8543 8544 i40e_print_link_message(vsi, new_link); 8545 8546 /* Notify the base of the switch tree connected to 8547 * the link. Floating VEBs are not notified. 8548 */ 8549 if (pf->lan_veb != I40E_NO_VEB && pf->veb[pf->lan_veb]) 8550 i40e_veb_link_event(pf->veb[pf->lan_veb], new_link); 8551 else 8552 i40e_vsi_link_event(vsi, new_link); 8553 8554 if (pf->vf) 8555 i40e_vc_notify_link_state(pf); 8556 8557 if (pf->flags & I40E_FLAG_PTP) 8558 i40e_ptp_set_increment(pf); 8559 } 8560 8561 /** 8562 * i40e_watchdog_subtask - periodic checks not using event driven response 8563 * @pf: board private structure 8564 **/ 8565 static void i40e_watchdog_subtask(struct i40e_pf *pf) 8566 { 8567 int i; 8568 8569 /* if interface is down do nothing */ 8570 if (test_bit(__I40E_DOWN, pf->state) || 8571 test_bit(__I40E_CONFIG_BUSY, pf->state)) 8572 return; 8573 8574 /* make sure we don't do these things too often */ 8575 if (time_before(jiffies, (pf->service_timer_previous + 8576 pf->service_timer_period))) 8577 return; 8578 pf->service_timer_previous = jiffies; 8579 8580 if ((pf->flags & I40E_FLAG_LINK_POLLING_ENABLED) || 8581 test_bit(__I40E_TEMP_LINK_POLLING, pf->state)) 8582 i40e_link_event(pf); 8583 8584 /* Update the stats for active netdevs so the network stack 8585 * can look at updated numbers whenever it cares to 8586 */ 8587 for (i = 0; i < pf->num_alloc_vsi; i++) 8588 if (pf->vsi[i] && pf->vsi[i]->netdev) 8589 i40e_update_stats(pf->vsi[i]); 8590 8591 if (pf->flags & I40E_FLAG_VEB_STATS_ENABLED) { 8592 /* Update the stats for the active switching components */ 8593 for (i = 0; i < I40E_MAX_VEB; i++) 8594 if (pf->veb[i]) 8595 i40e_update_veb_stats(pf->veb[i]); 8596 } 8597 8598 i40e_ptp_rx_hang(pf); 8599 i40e_ptp_tx_hang(pf); 8600 } 8601 8602 /** 8603 * i40e_reset_subtask - Set up for resetting the device and driver 8604 * @pf: board private structure 8605 **/ 8606 static void i40e_reset_subtask(struct i40e_pf *pf) 8607 { 8608 u32 reset_flags = 0; 8609 8610 if (test_bit(__I40E_REINIT_REQUESTED, pf->state)) { 8611 reset_flags |= BIT(__I40E_REINIT_REQUESTED); 8612 clear_bit(__I40E_REINIT_REQUESTED, pf->state); 8613 } 8614 if (test_bit(__I40E_PF_RESET_REQUESTED, pf->state)) { 8615 reset_flags |= BIT(__I40E_PF_RESET_REQUESTED); 8616 clear_bit(__I40E_PF_RESET_REQUESTED, pf->state); 8617 } 8618 if (test_bit(__I40E_CORE_RESET_REQUESTED, pf->state)) { 8619 reset_flags |= BIT(__I40E_CORE_RESET_REQUESTED); 8620 clear_bit(__I40E_CORE_RESET_REQUESTED, pf->state); 8621 } 8622 if (test_bit(__I40E_GLOBAL_RESET_REQUESTED, pf->state)) { 8623 reset_flags |= BIT(__I40E_GLOBAL_RESET_REQUESTED); 8624 clear_bit(__I40E_GLOBAL_RESET_REQUESTED, pf->state); 8625 } 8626 if (test_bit(__I40E_DOWN_REQUESTED, pf->state)) { 8627 reset_flags |= BIT(__I40E_DOWN_REQUESTED); 8628 clear_bit(__I40E_DOWN_REQUESTED, pf->state); 8629 } 8630 8631 /* If there's a recovery already waiting, it takes 8632 * precedence before starting a new reset sequence. 8633 */ 8634 if (test_bit(__I40E_RESET_INTR_RECEIVED, pf->state)) { 8635 i40e_prep_for_reset(pf, false); 8636 i40e_reset(pf); 8637 i40e_rebuild(pf, false, false); 8638 } 8639 8640 /* If we're already down or resetting, just bail */ 8641 if (reset_flags && 8642 !test_bit(__I40E_DOWN, pf->state) && 8643 !test_bit(__I40E_CONFIG_BUSY, pf->state)) { 8644 i40e_do_reset(pf, reset_flags, false); 8645 } 8646 } 8647 8648 /** 8649 * i40e_handle_link_event - Handle link event 8650 * @pf: board private structure 8651 * @e: event info posted on ARQ 8652 **/ 8653 static void i40e_handle_link_event(struct i40e_pf *pf, 8654 struct i40e_arq_event_info *e) 8655 { 8656 struct i40e_aqc_get_link_status *status = 8657 (struct i40e_aqc_get_link_status *)&e->desc.params.raw; 8658 8659 /* Do a new status request to re-enable LSE reporting 8660 * and load new status information into the hw struct 8661 * This completely ignores any state information 8662 * in the ARQ event info, instead choosing to always 8663 * issue the AQ update link status command. 8664 */ 8665 i40e_link_event(pf); 8666 8667 /* Check if module meets thermal requirements */ 8668 if (status->phy_type == I40E_PHY_TYPE_NOT_SUPPORTED_HIGH_TEMP) { 8669 dev_err(&pf->pdev->dev, 8670 "Rx/Tx is disabled on this device because the module does not meet thermal requirements.\n"); 8671 dev_err(&pf->pdev->dev, 8672 "Refer to the Intel(R) Ethernet Adapters and Devices User Guide for a list of supported modules.\n"); 8673 } else { 8674 /* check for unqualified module, if link is down, suppress 8675 * the message if link was forced to be down. 8676 */ 8677 if ((status->link_info & I40E_AQ_MEDIA_AVAILABLE) && 8678 (!(status->an_info & I40E_AQ_QUALIFIED_MODULE)) && 8679 (!(status->link_info & I40E_AQ_LINK_UP)) && 8680 (!(pf->flags & I40E_FLAG_LINK_DOWN_ON_CLOSE_ENABLED))) { 8681 dev_err(&pf->pdev->dev, 8682 "Rx/Tx is disabled on this device because an unsupported SFP module type was detected.\n"); 8683 dev_err(&pf->pdev->dev, 8684 "Refer to the Intel(R) Ethernet Adapters and Devices User Guide for a list of supported modules.\n"); 8685 } 8686 } 8687 } 8688 8689 /** 8690 * i40e_clean_adminq_subtask - Clean the AdminQ rings 8691 * @pf: board private structure 8692 **/ 8693 static void i40e_clean_adminq_subtask(struct i40e_pf *pf) 8694 { 8695 struct i40e_arq_event_info event; 8696 struct i40e_hw *hw = &pf->hw; 8697 u16 pending, i = 0; 8698 i40e_status ret; 8699 u16 opcode; 8700 u32 oldval; 8701 u32 val; 8702 8703 /* Do not run clean AQ when PF reset fails */ 8704 if (test_bit(__I40E_RESET_FAILED, pf->state)) 8705 return; 8706 8707 /* check for error indications */ 8708 val = rd32(&pf->hw, pf->hw.aq.arq.len); 8709 oldval = val; 8710 if (val & I40E_PF_ARQLEN_ARQVFE_MASK) { 8711 if (hw->debug_mask & I40E_DEBUG_AQ) 8712 dev_info(&pf->pdev->dev, "ARQ VF Error detected\n"); 8713 val &= ~I40E_PF_ARQLEN_ARQVFE_MASK; 8714 } 8715 if (val & I40E_PF_ARQLEN_ARQOVFL_MASK) { 8716 if (hw->debug_mask & I40E_DEBUG_AQ) 8717 dev_info(&pf->pdev->dev, "ARQ Overflow Error detected\n"); 8718 val &= ~I40E_PF_ARQLEN_ARQOVFL_MASK; 8719 pf->arq_overflows++; 8720 } 8721 if (val & I40E_PF_ARQLEN_ARQCRIT_MASK) { 8722 if (hw->debug_mask & I40E_DEBUG_AQ) 8723 dev_info(&pf->pdev->dev, "ARQ Critical Error detected\n"); 8724 val &= ~I40E_PF_ARQLEN_ARQCRIT_MASK; 8725 } 8726 if (oldval != val) 8727 wr32(&pf->hw, pf->hw.aq.arq.len, val); 8728 8729 val = rd32(&pf->hw, pf->hw.aq.asq.len); 8730 oldval = val; 8731 if (val & I40E_PF_ATQLEN_ATQVFE_MASK) { 8732 if (pf->hw.debug_mask & I40E_DEBUG_AQ) 8733 dev_info(&pf->pdev->dev, "ASQ VF Error detected\n"); 8734 val &= ~I40E_PF_ATQLEN_ATQVFE_MASK; 8735 } 8736 if (val & I40E_PF_ATQLEN_ATQOVFL_MASK) { 8737 if (pf->hw.debug_mask & I40E_DEBUG_AQ) 8738 dev_info(&pf->pdev->dev, "ASQ Overflow Error detected\n"); 8739 val &= ~I40E_PF_ATQLEN_ATQOVFL_MASK; 8740 } 8741 if (val & I40E_PF_ATQLEN_ATQCRIT_MASK) { 8742 if (pf->hw.debug_mask & I40E_DEBUG_AQ) 8743 dev_info(&pf->pdev->dev, "ASQ Critical Error detected\n"); 8744 val &= ~I40E_PF_ATQLEN_ATQCRIT_MASK; 8745 } 8746 if (oldval != val) 8747 wr32(&pf->hw, pf->hw.aq.asq.len, val); 8748 8749 event.buf_len = I40E_MAX_AQ_BUF_SIZE; 8750 event.msg_buf = kzalloc(event.buf_len, GFP_KERNEL); 8751 if (!event.msg_buf) 8752 return; 8753 8754 do { 8755 ret = i40e_clean_arq_element(hw, &event, &pending); 8756 if (ret == I40E_ERR_ADMIN_QUEUE_NO_WORK) 8757 break; 8758 else if (ret) { 8759 dev_info(&pf->pdev->dev, "ARQ event error %d\n", ret); 8760 break; 8761 } 8762 8763 opcode = le16_to_cpu(event.desc.opcode); 8764 switch (opcode) { 8765 8766 case i40e_aqc_opc_get_link_status: 8767 i40e_handle_link_event(pf, &event); 8768 break; 8769 case i40e_aqc_opc_send_msg_to_pf: 8770 ret = i40e_vc_process_vf_msg(pf, 8771 le16_to_cpu(event.desc.retval), 8772 le32_to_cpu(event.desc.cookie_high), 8773 le32_to_cpu(event.desc.cookie_low), 8774 event.msg_buf, 8775 event.msg_len); 8776 break; 8777 case i40e_aqc_opc_lldp_update_mib: 8778 dev_dbg(&pf->pdev->dev, "ARQ: Update LLDP MIB event received\n"); 8779 #ifdef CONFIG_I40E_DCB 8780 rtnl_lock(); 8781 ret = i40e_handle_lldp_event(pf, &event); 8782 rtnl_unlock(); 8783 #endif /* CONFIG_I40E_DCB */ 8784 break; 8785 case i40e_aqc_opc_event_lan_overflow: 8786 dev_dbg(&pf->pdev->dev, "ARQ LAN queue overflow event received\n"); 8787 i40e_handle_lan_overflow_event(pf, &event); 8788 break; 8789 case i40e_aqc_opc_send_msg_to_peer: 8790 dev_info(&pf->pdev->dev, "ARQ: Msg from other pf\n"); 8791 break; 8792 case i40e_aqc_opc_nvm_erase: 8793 case i40e_aqc_opc_nvm_update: 8794 case i40e_aqc_opc_oem_post_update: 8795 i40e_debug(&pf->hw, I40E_DEBUG_NVM, 8796 "ARQ NVM operation 0x%04x completed\n", 8797 opcode); 8798 break; 8799 default: 8800 dev_info(&pf->pdev->dev, 8801 "ARQ: Unknown event 0x%04x ignored\n", 8802 opcode); 8803 break; 8804 } 8805 } while (i++ < pf->adminq_work_limit); 8806 8807 if (i < pf->adminq_work_limit) 8808 clear_bit(__I40E_ADMINQ_EVENT_PENDING, pf->state); 8809 8810 /* re-enable Admin queue interrupt cause */ 8811 val = rd32(hw, I40E_PFINT_ICR0_ENA); 8812 val |= I40E_PFINT_ICR0_ENA_ADMINQ_MASK; 8813 wr32(hw, I40E_PFINT_ICR0_ENA, val); 8814 i40e_flush(hw); 8815 8816 kfree(event.msg_buf); 8817 } 8818 8819 /** 8820 * i40e_verify_eeprom - make sure eeprom is good to use 8821 * @pf: board private structure 8822 **/ 8823 static void i40e_verify_eeprom(struct i40e_pf *pf) 8824 { 8825 int err; 8826 8827 err = i40e_diag_eeprom_test(&pf->hw); 8828 if (err) { 8829 /* retry in case of garbage read */ 8830 err = i40e_diag_eeprom_test(&pf->hw); 8831 if (err) { 8832 dev_info(&pf->pdev->dev, "eeprom check failed (%d), Tx/Rx traffic disabled\n", 8833 err); 8834 set_bit(__I40E_BAD_EEPROM, pf->state); 8835 } 8836 } 8837 8838 if (!err && test_bit(__I40E_BAD_EEPROM, pf->state)) { 8839 dev_info(&pf->pdev->dev, "eeprom check passed, Tx/Rx traffic enabled\n"); 8840 clear_bit(__I40E_BAD_EEPROM, pf->state); 8841 } 8842 } 8843 8844 /** 8845 * i40e_enable_pf_switch_lb 8846 * @pf: pointer to the PF structure 8847 * 8848 * enable switch loop back or die - no point in a return value 8849 **/ 8850 static void i40e_enable_pf_switch_lb(struct i40e_pf *pf) 8851 { 8852 struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi]; 8853 struct i40e_vsi_context ctxt; 8854 int ret; 8855 8856 ctxt.seid = pf->main_vsi_seid; 8857 ctxt.pf_num = pf->hw.pf_id; 8858 ctxt.vf_num = 0; 8859 ret = i40e_aq_get_vsi_params(&pf->hw, &ctxt, NULL); 8860 if (ret) { 8861 dev_info(&pf->pdev->dev, 8862 "couldn't get PF vsi config, err %s aq_err %s\n", 8863 i40e_stat_str(&pf->hw, ret), 8864 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 8865 return; 8866 } 8867 ctxt.flags = I40E_AQ_VSI_TYPE_PF; 8868 ctxt.info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID); 8869 ctxt.info.switch_id |= cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB); 8870 8871 ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL); 8872 if (ret) { 8873 dev_info(&pf->pdev->dev, 8874 "update vsi switch failed, err %s aq_err %s\n", 8875 i40e_stat_str(&pf->hw, ret), 8876 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 8877 } 8878 } 8879 8880 /** 8881 * i40e_disable_pf_switch_lb 8882 * @pf: pointer to the PF structure 8883 * 8884 * disable switch loop back or die - no point in a return value 8885 **/ 8886 static void i40e_disable_pf_switch_lb(struct i40e_pf *pf) 8887 { 8888 struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi]; 8889 struct i40e_vsi_context ctxt; 8890 int ret; 8891 8892 ctxt.seid = pf->main_vsi_seid; 8893 ctxt.pf_num = pf->hw.pf_id; 8894 ctxt.vf_num = 0; 8895 ret = i40e_aq_get_vsi_params(&pf->hw, &ctxt, NULL); 8896 if (ret) { 8897 dev_info(&pf->pdev->dev, 8898 "couldn't get PF vsi config, err %s aq_err %s\n", 8899 i40e_stat_str(&pf->hw, ret), 8900 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 8901 return; 8902 } 8903 ctxt.flags = I40E_AQ_VSI_TYPE_PF; 8904 ctxt.info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID); 8905 ctxt.info.switch_id &= ~cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB); 8906 8907 ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL); 8908 if (ret) { 8909 dev_info(&pf->pdev->dev, 8910 "update vsi switch failed, err %s aq_err %s\n", 8911 i40e_stat_str(&pf->hw, ret), 8912 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 8913 } 8914 } 8915 8916 /** 8917 * i40e_config_bridge_mode - Configure the HW bridge mode 8918 * @veb: pointer to the bridge instance 8919 * 8920 * Configure the loop back mode for the LAN VSI that is downlink to the 8921 * specified HW bridge instance. It is expected this function is called 8922 * when a new HW bridge is instantiated. 8923 **/ 8924 static void i40e_config_bridge_mode(struct i40e_veb *veb) 8925 { 8926 struct i40e_pf *pf = veb->pf; 8927 8928 if (pf->hw.debug_mask & I40E_DEBUG_LAN) 8929 dev_info(&pf->pdev->dev, "enabling bridge mode: %s\n", 8930 veb->bridge_mode == BRIDGE_MODE_VEPA ? "VEPA" : "VEB"); 8931 if (veb->bridge_mode & BRIDGE_MODE_VEPA) 8932 i40e_disable_pf_switch_lb(pf); 8933 else 8934 i40e_enable_pf_switch_lb(pf); 8935 } 8936 8937 /** 8938 * i40e_reconstitute_veb - rebuild the VEB and anything connected to it 8939 * @veb: pointer to the VEB instance 8940 * 8941 * This is a recursive function that first builds the attached VSIs then 8942 * recurses in to build the next layer of VEB. We track the connections 8943 * through our own index numbers because the seid's from the HW could 8944 * change across the reset. 8945 **/ 8946 static int i40e_reconstitute_veb(struct i40e_veb *veb) 8947 { 8948 struct i40e_vsi *ctl_vsi = NULL; 8949 struct i40e_pf *pf = veb->pf; 8950 int v, veb_idx; 8951 int ret; 8952 8953 /* build VSI that owns this VEB, temporarily attached to base VEB */ 8954 for (v = 0; v < pf->num_alloc_vsi && !ctl_vsi; v++) { 8955 if (pf->vsi[v] && 8956 pf->vsi[v]->veb_idx == veb->idx && 8957 pf->vsi[v]->flags & I40E_VSI_FLAG_VEB_OWNER) { 8958 ctl_vsi = pf->vsi[v]; 8959 break; 8960 } 8961 } 8962 if (!ctl_vsi) { 8963 dev_info(&pf->pdev->dev, 8964 "missing owner VSI for veb_idx %d\n", veb->idx); 8965 ret = -ENOENT; 8966 goto end_reconstitute; 8967 } 8968 if (ctl_vsi != pf->vsi[pf->lan_vsi]) 8969 ctl_vsi->uplink_seid = pf->vsi[pf->lan_vsi]->uplink_seid; 8970 ret = i40e_add_vsi(ctl_vsi); 8971 if (ret) { 8972 dev_info(&pf->pdev->dev, 8973 "rebuild of veb_idx %d owner VSI failed: %d\n", 8974 veb->idx, ret); 8975 goto end_reconstitute; 8976 } 8977 i40e_vsi_reset_stats(ctl_vsi); 8978 8979 /* create the VEB in the switch and move the VSI onto the VEB */ 8980 ret = i40e_add_veb(veb, ctl_vsi); 8981 if (ret) 8982 goto end_reconstitute; 8983 8984 if (pf->flags & I40E_FLAG_VEB_MODE_ENABLED) 8985 veb->bridge_mode = BRIDGE_MODE_VEB; 8986 else 8987 veb->bridge_mode = BRIDGE_MODE_VEPA; 8988 i40e_config_bridge_mode(veb); 8989 8990 /* create the remaining VSIs attached to this VEB */ 8991 for (v = 0; v < pf->num_alloc_vsi; v++) { 8992 if (!pf->vsi[v] || pf->vsi[v] == ctl_vsi) 8993 continue; 8994 8995 if (pf->vsi[v]->veb_idx == veb->idx) { 8996 struct i40e_vsi *vsi = pf->vsi[v]; 8997 8998 vsi->uplink_seid = veb->seid; 8999 ret = i40e_add_vsi(vsi); 9000 if (ret) { 9001 dev_info(&pf->pdev->dev, 9002 "rebuild of vsi_idx %d failed: %d\n", 9003 v, ret); 9004 goto end_reconstitute; 9005 } 9006 i40e_vsi_reset_stats(vsi); 9007 } 9008 } 9009 9010 /* create any VEBs attached to this VEB - RECURSION */ 9011 for (veb_idx = 0; veb_idx < I40E_MAX_VEB; veb_idx++) { 9012 if (pf->veb[veb_idx] && pf->veb[veb_idx]->veb_idx == veb->idx) { 9013 pf->veb[veb_idx]->uplink_seid = veb->seid; 9014 ret = i40e_reconstitute_veb(pf->veb[veb_idx]); 9015 if (ret) 9016 break; 9017 } 9018 } 9019 9020 end_reconstitute: 9021 return ret; 9022 } 9023 9024 /** 9025 * i40e_get_capabilities - get info about the HW 9026 * @pf: the PF struct 9027 **/ 9028 static int i40e_get_capabilities(struct i40e_pf *pf, 9029 enum i40e_admin_queue_opc list_type) 9030 { 9031 struct i40e_aqc_list_capabilities_element_resp *cap_buf; 9032 u16 data_size; 9033 int buf_len; 9034 int err; 9035 9036 buf_len = 40 * sizeof(struct i40e_aqc_list_capabilities_element_resp); 9037 do { 9038 cap_buf = kzalloc(buf_len, GFP_KERNEL); 9039 if (!cap_buf) 9040 return -ENOMEM; 9041 9042 /* this loads the data into the hw struct for us */ 9043 err = i40e_aq_discover_capabilities(&pf->hw, cap_buf, buf_len, 9044 &data_size, list_type, 9045 NULL); 9046 /* data loaded, buffer no longer needed */ 9047 kfree(cap_buf); 9048 9049 if (pf->hw.aq.asq_last_status == I40E_AQ_RC_ENOMEM) { 9050 /* retry with a larger buffer */ 9051 buf_len = data_size; 9052 } else if (pf->hw.aq.asq_last_status != I40E_AQ_RC_OK) { 9053 dev_info(&pf->pdev->dev, 9054 "capability discovery failed, err %s aq_err %s\n", 9055 i40e_stat_str(&pf->hw, err), 9056 i40e_aq_str(&pf->hw, 9057 pf->hw.aq.asq_last_status)); 9058 return -ENODEV; 9059 } 9060 } while (err); 9061 9062 if (pf->hw.debug_mask & I40E_DEBUG_USER) { 9063 if (list_type == i40e_aqc_opc_list_func_capabilities) { 9064 dev_info(&pf->pdev->dev, 9065 "pf=%d, num_vfs=%d, msix_pf=%d, msix_vf=%d, fd_g=%d, fd_b=%d, pf_max_q=%d num_vsi=%d\n", 9066 pf->hw.pf_id, pf->hw.func_caps.num_vfs, 9067 pf->hw.func_caps.num_msix_vectors, 9068 pf->hw.func_caps.num_msix_vectors_vf, 9069 pf->hw.func_caps.fd_filters_guaranteed, 9070 pf->hw.func_caps.fd_filters_best_effort, 9071 pf->hw.func_caps.num_tx_qp, 9072 pf->hw.func_caps.num_vsis); 9073 } else if (list_type == i40e_aqc_opc_list_dev_capabilities) { 9074 dev_info(&pf->pdev->dev, 9075 "switch_mode=0x%04x, function_valid=0x%08x\n", 9076 pf->hw.dev_caps.switch_mode, 9077 pf->hw.dev_caps.valid_functions); 9078 dev_info(&pf->pdev->dev, 9079 "SR-IOV=%d, num_vfs for all function=%u\n", 9080 pf->hw.dev_caps.sr_iov_1_1, 9081 pf->hw.dev_caps.num_vfs); 9082 dev_info(&pf->pdev->dev, 9083 "num_vsis=%u, num_rx:%u, num_tx=%u\n", 9084 pf->hw.dev_caps.num_vsis, 9085 pf->hw.dev_caps.num_rx_qp, 9086 pf->hw.dev_caps.num_tx_qp); 9087 } 9088 } 9089 if (list_type == i40e_aqc_opc_list_func_capabilities) { 9090 #define DEF_NUM_VSI (1 + (pf->hw.func_caps.fcoe ? 1 : 0) \ 9091 + pf->hw.func_caps.num_vfs) 9092 if (pf->hw.revision_id == 0 && 9093 pf->hw.func_caps.num_vsis < DEF_NUM_VSI) { 9094 dev_info(&pf->pdev->dev, 9095 "got num_vsis %d, setting num_vsis to %d\n", 9096 pf->hw.func_caps.num_vsis, DEF_NUM_VSI); 9097 pf->hw.func_caps.num_vsis = DEF_NUM_VSI; 9098 } 9099 } 9100 return 0; 9101 } 9102 9103 static int i40e_vsi_clear(struct i40e_vsi *vsi); 9104 9105 /** 9106 * i40e_fdir_sb_setup - initialize the Flow Director resources for Sideband 9107 * @pf: board private structure 9108 **/ 9109 static void i40e_fdir_sb_setup(struct i40e_pf *pf) 9110 { 9111 struct i40e_vsi *vsi; 9112 9113 /* quick workaround for an NVM issue that leaves a critical register 9114 * uninitialized 9115 */ 9116 if (!rd32(&pf->hw, I40E_GLQF_HKEY(0))) { 9117 static const u32 hkey[] = { 9118 0xe640d33f, 0xcdfe98ab, 0x73fa7161, 0x0d7a7d36, 9119 0xeacb7d61, 0xaa4f05b6, 0x9c5c89ed, 0xfc425ddb, 9120 0xa4654832, 0xfc7461d4, 0x8f827619, 0xf5c63c21, 9121 0x95b3a76d}; 9122 int i; 9123 9124 for (i = 0; i <= I40E_GLQF_HKEY_MAX_INDEX; i++) 9125 wr32(&pf->hw, I40E_GLQF_HKEY(i), hkey[i]); 9126 } 9127 9128 if (!(pf->flags & I40E_FLAG_FD_SB_ENABLED)) 9129 return; 9130 9131 /* find existing VSI and see if it needs configuring */ 9132 vsi = i40e_find_vsi_by_type(pf, I40E_VSI_FDIR); 9133 9134 /* create a new VSI if none exists */ 9135 if (!vsi) { 9136 vsi = i40e_vsi_setup(pf, I40E_VSI_FDIR, 9137 pf->vsi[pf->lan_vsi]->seid, 0); 9138 if (!vsi) { 9139 dev_info(&pf->pdev->dev, "Couldn't create FDir VSI\n"); 9140 pf->flags &= ~I40E_FLAG_FD_SB_ENABLED; 9141 pf->flags |= I40E_FLAG_FD_SB_INACTIVE; 9142 return; 9143 } 9144 } 9145 9146 i40e_vsi_setup_irqhandler(vsi, i40e_fdir_clean_ring); 9147 } 9148 9149 /** 9150 * i40e_fdir_teardown - release the Flow Director resources 9151 * @pf: board private structure 9152 **/ 9153 static void i40e_fdir_teardown(struct i40e_pf *pf) 9154 { 9155 struct i40e_vsi *vsi; 9156 9157 i40e_fdir_filter_exit(pf); 9158 vsi = i40e_find_vsi_by_type(pf, I40E_VSI_FDIR); 9159 if (vsi) 9160 i40e_vsi_release(vsi); 9161 } 9162 9163 /** 9164 * i40e_rebuild_cloud_filters - Rebuilds cloud filters for VSIs 9165 * @vsi: PF main vsi 9166 * @seid: seid of main or channel VSIs 9167 * 9168 * Rebuilds cloud filters associated with main VSI and channel VSIs if they 9169 * existed before reset 9170 **/ 9171 static int i40e_rebuild_cloud_filters(struct i40e_vsi *vsi, u16 seid) 9172 { 9173 struct i40e_cloud_filter *cfilter; 9174 struct i40e_pf *pf = vsi->back; 9175 struct hlist_node *node; 9176 i40e_status ret; 9177 9178 /* Add cloud filters back if they exist */ 9179 hlist_for_each_entry_safe(cfilter, node, &pf->cloud_filter_list, 9180 cloud_node) { 9181 if (cfilter->seid != seid) 9182 continue; 9183 9184 if (cfilter->dst_port) 9185 ret = i40e_add_del_cloud_filter_big_buf(vsi, cfilter, 9186 true); 9187 else 9188 ret = i40e_add_del_cloud_filter(vsi, cfilter, true); 9189 9190 if (ret) { 9191 dev_dbg(&pf->pdev->dev, 9192 "Failed to rebuild cloud filter, err %s aq_err %s\n", 9193 i40e_stat_str(&pf->hw, ret), 9194 i40e_aq_str(&pf->hw, 9195 pf->hw.aq.asq_last_status)); 9196 return ret; 9197 } 9198 } 9199 return 0; 9200 } 9201 9202 /** 9203 * i40e_rebuild_channels - Rebuilds channel VSIs if they existed before reset 9204 * @vsi: PF main vsi 9205 * 9206 * Rebuilds channel VSIs if they existed before reset 9207 **/ 9208 static int i40e_rebuild_channels(struct i40e_vsi *vsi) 9209 { 9210 struct i40e_channel *ch, *ch_tmp; 9211 i40e_status ret; 9212 9213 if (list_empty(&vsi->ch_list)) 9214 return 0; 9215 9216 list_for_each_entry_safe(ch, ch_tmp, &vsi->ch_list, list) { 9217 if (!ch->initialized) 9218 break; 9219 /* Proceed with creation of channel (VMDq2) VSI */ 9220 ret = i40e_add_channel(vsi->back, vsi->uplink_seid, ch); 9221 if (ret) { 9222 dev_info(&vsi->back->pdev->dev, 9223 "failed to rebuild channels using uplink_seid %u\n", 9224 vsi->uplink_seid); 9225 return ret; 9226 } 9227 /* Reconfigure TX queues using QTX_CTL register */ 9228 ret = i40e_channel_config_tx_ring(vsi->back, vsi, ch); 9229 if (ret) { 9230 dev_info(&vsi->back->pdev->dev, 9231 "failed to configure TX rings for channel %u\n", 9232 ch->seid); 9233 return ret; 9234 } 9235 /* update 'next_base_queue' */ 9236 vsi->next_base_queue = vsi->next_base_queue + 9237 ch->num_queue_pairs; 9238 if (ch->max_tx_rate) { 9239 u64 credits = ch->max_tx_rate; 9240 9241 if (i40e_set_bw_limit(vsi, ch->seid, 9242 ch->max_tx_rate)) 9243 return -EINVAL; 9244 9245 do_div(credits, I40E_BW_CREDIT_DIVISOR); 9246 dev_dbg(&vsi->back->pdev->dev, 9247 "Set tx rate of %llu Mbps (count of 50Mbps %llu) for vsi->seid %u\n", 9248 ch->max_tx_rate, 9249 credits, 9250 ch->seid); 9251 } 9252 ret = i40e_rebuild_cloud_filters(vsi, ch->seid); 9253 if (ret) { 9254 dev_dbg(&vsi->back->pdev->dev, 9255 "Failed to rebuild cloud filters for channel VSI %u\n", 9256 ch->seid); 9257 return ret; 9258 } 9259 } 9260 return 0; 9261 } 9262 9263 /** 9264 * i40e_prep_for_reset - prep for the core to reset 9265 * @pf: board private structure 9266 * @lock_acquired: indicates whether or not the lock has been acquired 9267 * before this function was called. 9268 * 9269 * Close up the VFs and other things in prep for PF Reset. 9270 **/ 9271 static void i40e_prep_for_reset(struct i40e_pf *pf, bool lock_acquired) 9272 { 9273 struct i40e_hw *hw = &pf->hw; 9274 i40e_status ret = 0; 9275 u32 v; 9276 9277 clear_bit(__I40E_RESET_INTR_RECEIVED, pf->state); 9278 if (test_and_set_bit(__I40E_RESET_RECOVERY_PENDING, pf->state)) 9279 return; 9280 if (i40e_check_asq_alive(&pf->hw)) 9281 i40e_vc_notify_reset(pf); 9282 9283 dev_dbg(&pf->pdev->dev, "Tearing down internal switch for reset\n"); 9284 9285 /* quiesce the VSIs and their queues that are not already DOWN */ 9286 /* pf_quiesce_all_vsi modifies netdev structures -rtnl_lock needed */ 9287 if (!lock_acquired) 9288 rtnl_lock(); 9289 i40e_pf_quiesce_all_vsi(pf); 9290 if (!lock_acquired) 9291 rtnl_unlock(); 9292 9293 for (v = 0; v < pf->num_alloc_vsi; v++) { 9294 if (pf->vsi[v]) 9295 pf->vsi[v]->seid = 0; 9296 } 9297 9298 i40e_shutdown_adminq(&pf->hw); 9299 9300 /* call shutdown HMC */ 9301 if (hw->hmc.hmc_obj) { 9302 ret = i40e_shutdown_lan_hmc(hw); 9303 if (ret) 9304 dev_warn(&pf->pdev->dev, 9305 "shutdown_lan_hmc failed: %d\n", ret); 9306 } 9307 } 9308 9309 /** 9310 * i40e_send_version - update firmware with driver version 9311 * @pf: PF struct 9312 */ 9313 static void i40e_send_version(struct i40e_pf *pf) 9314 { 9315 struct i40e_driver_version dv; 9316 9317 dv.major_version = DRV_VERSION_MAJOR; 9318 dv.minor_version = DRV_VERSION_MINOR; 9319 dv.build_version = DRV_VERSION_BUILD; 9320 dv.subbuild_version = 0; 9321 strlcpy(dv.driver_string, DRV_VERSION, sizeof(dv.driver_string)); 9322 i40e_aq_send_driver_version(&pf->hw, &dv, NULL); 9323 } 9324 9325 /** 9326 * i40e_get_oem_version - get OEM specific version information 9327 * @hw: pointer to the hardware structure 9328 **/ 9329 static void i40e_get_oem_version(struct i40e_hw *hw) 9330 { 9331 u16 block_offset = 0xffff; 9332 u16 block_length = 0; 9333 u16 capabilities = 0; 9334 u16 gen_snap = 0; 9335 u16 release = 0; 9336 9337 #define I40E_SR_NVM_OEM_VERSION_PTR 0x1B 9338 #define I40E_NVM_OEM_LENGTH_OFFSET 0x00 9339 #define I40E_NVM_OEM_CAPABILITIES_OFFSET 0x01 9340 #define I40E_NVM_OEM_GEN_OFFSET 0x02 9341 #define I40E_NVM_OEM_RELEASE_OFFSET 0x03 9342 #define I40E_NVM_OEM_CAPABILITIES_MASK 0x000F 9343 #define I40E_NVM_OEM_LENGTH 3 9344 9345 /* Check if pointer to OEM version block is valid. */ 9346 i40e_read_nvm_word(hw, I40E_SR_NVM_OEM_VERSION_PTR, &block_offset); 9347 if (block_offset == 0xffff) 9348 return; 9349 9350 /* Check if OEM version block has correct length. */ 9351 i40e_read_nvm_word(hw, block_offset + I40E_NVM_OEM_LENGTH_OFFSET, 9352 &block_length); 9353 if (block_length < I40E_NVM_OEM_LENGTH) 9354 return; 9355 9356 /* Check if OEM version format is as expected. */ 9357 i40e_read_nvm_word(hw, block_offset + I40E_NVM_OEM_CAPABILITIES_OFFSET, 9358 &capabilities); 9359 if ((capabilities & I40E_NVM_OEM_CAPABILITIES_MASK) != 0) 9360 return; 9361 9362 i40e_read_nvm_word(hw, block_offset + I40E_NVM_OEM_GEN_OFFSET, 9363 &gen_snap); 9364 i40e_read_nvm_word(hw, block_offset + I40E_NVM_OEM_RELEASE_OFFSET, 9365 &release); 9366 hw->nvm.oem_ver = (gen_snap << I40E_OEM_SNAP_SHIFT) | release; 9367 hw->nvm.eetrack = I40E_OEM_EETRACK_ID; 9368 } 9369 9370 /** 9371 * i40e_reset - wait for core reset to finish reset, reset pf if corer not seen 9372 * @pf: board private structure 9373 **/ 9374 static int i40e_reset(struct i40e_pf *pf) 9375 { 9376 struct i40e_hw *hw = &pf->hw; 9377 i40e_status ret; 9378 9379 ret = i40e_pf_reset(hw); 9380 if (ret) { 9381 dev_info(&pf->pdev->dev, "PF reset failed, %d\n", ret); 9382 set_bit(__I40E_RESET_FAILED, pf->state); 9383 clear_bit(__I40E_RESET_RECOVERY_PENDING, pf->state); 9384 } else { 9385 pf->pfr_count++; 9386 } 9387 return ret; 9388 } 9389 9390 /** 9391 * i40e_rebuild - rebuild using a saved config 9392 * @pf: board private structure 9393 * @reinit: if the Main VSI needs to re-initialized. 9394 * @lock_acquired: indicates whether or not the lock has been acquired 9395 * before this function was called. 9396 **/ 9397 static void i40e_rebuild(struct i40e_pf *pf, bool reinit, bool lock_acquired) 9398 { 9399 struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi]; 9400 struct i40e_hw *hw = &pf->hw; 9401 u8 set_fc_aq_fail = 0; 9402 i40e_status ret; 9403 u32 val; 9404 int v; 9405 9406 if (test_bit(__I40E_DOWN, pf->state)) 9407 goto clear_recovery; 9408 dev_dbg(&pf->pdev->dev, "Rebuilding internal switch\n"); 9409 9410 /* rebuild the basics for the AdminQ, HMC, and initial HW switch */ 9411 ret = i40e_init_adminq(&pf->hw); 9412 if (ret) { 9413 dev_info(&pf->pdev->dev, "Rebuild AdminQ failed, err %s aq_err %s\n", 9414 i40e_stat_str(&pf->hw, ret), 9415 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 9416 goto clear_recovery; 9417 } 9418 i40e_get_oem_version(&pf->hw); 9419 9420 if (test_bit(__I40E_EMP_RESET_INTR_RECEIVED, pf->state) && 9421 ((hw->aq.fw_maj_ver == 4 && hw->aq.fw_min_ver <= 33) || 9422 hw->aq.fw_maj_ver < 4) && hw->mac.type == I40E_MAC_XL710) { 9423 /* The following delay is necessary for 4.33 firmware and older 9424 * to recover after EMP reset. 200 ms should suffice but we 9425 * put here 300 ms to be sure that FW is ready to operate 9426 * after reset. 9427 */ 9428 mdelay(300); 9429 } 9430 9431 /* re-verify the eeprom if we just had an EMP reset */ 9432 if (test_and_clear_bit(__I40E_EMP_RESET_INTR_RECEIVED, pf->state)) 9433 i40e_verify_eeprom(pf); 9434 9435 i40e_clear_pxe_mode(hw); 9436 ret = i40e_get_capabilities(pf, i40e_aqc_opc_list_func_capabilities); 9437 if (ret) 9438 goto end_core_reset; 9439 9440 ret = i40e_init_lan_hmc(hw, hw->func_caps.num_tx_qp, 9441 hw->func_caps.num_rx_qp, 0, 0); 9442 if (ret) { 9443 dev_info(&pf->pdev->dev, "init_lan_hmc failed: %d\n", ret); 9444 goto end_core_reset; 9445 } 9446 ret = i40e_configure_lan_hmc(hw, I40E_HMC_MODEL_DIRECT_ONLY); 9447 if (ret) { 9448 dev_info(&pf->pdev->dev, "configure_lan_hmc failed: %d\n", ret); 9449 goto end_core_reset; 9450 } 9451 9452 /* Enable FW to write a default DCB config on link-up */ 9453 i40e_aq_set_dcb_parameters(hw, true, NULL); 9454 9455 #ifdef CONFIG_I40E_DCB 9456 ret = i40e_init_pf_dcb(pf); 9457 if (ret) { 9458 dev_info(&pf->pdev->dev, "DCB init failed %d, disabled\n", ret); 9459 pf->flags &= ~I40E_FLAG_DCB_CAPABLE; 9460 /* Continue without DCB enabled */ 9461 } 9462 #endif /* CONFIG_I40E_DCB */ 9463 /* do basic switch setup */ 9464 if (!lock_acquired) 9465 rtnl_lock(); 9466 ret = i40e_setup_pf_switch(pf, reinit); 9467 if (ret) 9468 goto end_unlock; 9469 9470 /* The driver only wants link up/down and module qualification 9471 * reports from firmware. Note the negative logic. 9472 */ 9473 ret = i40e_aq_set_phy_int_mask(&pf->hw, 9474 ~(I40E_AQ_EVENT_LINK_UPDOWN | 9475 I40E_AQ_EVENT_MEDIA_NA | 9476 I40E_AQ_EVENT_MODULE_QUAL_FAIL), NULL); 9477 if (ret) 9478 dev_info(&pf->pdev->dev, "set phy mask fail, err %s aq_err %s\n", 9479 i40e_stat_str(&pf->hw, ret), 9480 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 9481 9482 /* make sure our flow control settings are restored */ 9483 ret = i40e_set_fc(&pf->hw, &set_fc_aq_fail, true); 9484 if (ret) 9485 dev_dbg(&pf->pdev->dev, "setting flow control: ret = %s last_status = %s\n", 9486 i40e_stat_str(&pf->hw, ret), 9487 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 9488 9489 /* Rebuild the VSIs and VEBs that existed before reset. 9490 * They are still in our local switch element arrays, so only 9491 * need to rebuild the switch model in the HW. 9492 * 9493 * If there were VEBs but the reconstitution failed, we'll try 9494 * try to recover minimal use by getting the basic PF VSI working. 9495 */ 9496 if (vsi->uplink_seid != pf->mac_seid) { 9497 dev_dbg(&pf->pdev->dev, "attempting to rebuild switch\n"); 9498 /* find the one VEB connected to the MAC, and find orphans */ 9499 for (v = 0; v < I40E_MAX_VEB; v++) { 9500 if (!pf->veb[v]) 9501 continue; 9502 9503 if (pf->veb[v]->uplink_seid == pf->mac_seid || 9504 pf->veb[v]->uplink_seid == 0) { 9505 ret = i40e_reconstitute_veb(pf->veb[v]); 9506 9507 if (!ret) 9508 continue; 9509 9510 /* If Main VEB failed, we're in deep doodoo, 9511 * so give up rebuilding the switch and set up 9512 * for minimal rebuild of PF VSI. 9513 * If orphan failed, we'll report the error 9514 * but try to keep going. 9515 */ 9516 if (pf->veb[v]->uplink_seid == pf->mac_seid) { 9517 dev_info(&pf->pdev->dev, 9518 "rebuild of switch failed: %d, will try to set up simple PF connection\n", 9519 ret); 9520 vsi->uplink_seid = pf->mac_seid; 9521 break; 9522 } else if (pf->veb[v]->uplink_seid == 0) { 9523 dev_info(&pf->pdev->dev, 9524 "rebuild of orphan VEB failed: %d\n", 9525 ret); 9526 } 9527 } 9528 } 9529 } 9530 9531 if (vsi->uplink_seid == pf->mac_seid) { 9532 dev_dbg(&pf->pdev->dev, "attempting to rebuild PF VSI\n"); 9533 /* no VEB, so rebuild only the Main VSI */ 9534 ret = i40e_add_vsi(vsi); 9535 if (ret) { 9536 dev_info(&pf->pdev->dev, 9537 "rebuild of Main VSI failed: %d\n", ret); 9538 goto end_unlock; 9539 } 9540 } 9541 9542 if (vsi->mqprio_qopt.max_rate[0]) { 9543 u64 max_tx_rate = vsi->mqprio_qopt.max_rate[0]; 9544 u64 credits = 0; 9545 9546 do_div(max_tx_rate, I40E_BW_MBPS_DIVISOR); 9547 ret = i40e_set_bw_limit(vsi, vsi->seid, max_tx_rate); 9548 if (ret) 9549 goto end_unlock; 9550 9551 credits = max_tx_rate; 9552 do_div(credits, I40E_BW_CREDIT_DIVISOR); 9553 dev_dbg(&vsi->back->pdev->dev, 9554 "Set tx rate of %llu Mbps (count of 50Mbps %llu) for vsi->seid %u\n", 9555 max_tx_rate, 9556 credits, 9557 vsi->seid); 9558 } 9559 9560 ret = i40e_rebuild_cloud_filters(vsi, vsi->seid); 9561 if (ret) 9562 goto end_unlock; 9563 9564 /* PF Main VSI is rebuild by now, go ahead and rebuild channel VSIs 9565 * for this main VSI if they exist 9566 */ 9567 ret = i40e_rebuild_channels(vsi); 9568 if (ret) 9569 goto end_unlock; 9570 9571 /* Reconfigure hardware for allowing smaller MSS in the case 9572 * of TSO, so that we avoid the MDD being fired and causing 9573 * a reset in the case of small MSS+TSO. 9574 */ 9575 #define I40E_REG_MSS 0x000E64DC 9576 #define I40E_REG_MSS_MIN_MASK 0x3FF0000 9577 #define I40E_64BYTE_MSS 0x400000 9578 val = rd32(hw, I40E_REG_MSS); 9579 if ((val & I40E_REG_MSS_MIN_MASK) > I40E_64BYTE_MSS) { 9580 val &= ~I40E_REG_MSS_MIN_MASK; 9581 val |= I40E_64BYTE_MSS; 9582 wr32(hw, I40E_REG_MSS, val); 9583 } 9584 9585 if (pf->hw_features & I40E_HW_RESTART_AUTONEG) { 9586 msleep(75); 9587 ret = i40e_aq_set_link_restart_an(&pf->hw, true, NULL); 9588 if (ret) 9589 dev_info(&pf->pdev->dev, "link restart failed, err %s aq_err %s\n", 9590 i40e_stat_str(&pf->hw, ret), 9591 i40e_aq_str(&pf->hw, 9592 pf->hw.aq.asq_last_status)); 9593 } 9594 /* reinit the misc interrupt */ 9595 if (pf->flags & I40E_FLAG_MSIX_ENABLED) 9596 ret = i40e_setup_misc_vector(pf); 9597 9598 /* Add a filter to drop all Flow control frames from any VSI from being 9599 * transmitted. By doing so we stop a malicious VF from sending out 9600 * PAUSE or PFC frames and potentially controlling traffic for other 9601 * PF/VF VSIs. 9602 * The FW can still send Flow control frames if enabled. 9603 */ 9604 i40e_add_filter_to_drop_tx_flow_control_frames(&pf->hw, 9605 pf->main_vsi_seid); 9606 9607 /* restart the VSIs that were rebuilt and running before the reset */ 9608 i40e_pf_unquiesce_all_vsi(pf); 9609 9610 /* Release the RTNL lock before we start resetting VFs */ 9611 if (!lock_acquired) 9612 rtnl_unlock(); 9613 9614 /* Restore promiscuous settings */ 9615 ret = i40e_set_promiscuous(pf, pf->cur_promisc); 9616 if (ret) 9617 dev_warn(&pf->pdev->dev, 9618 "Failed to restore promiscuous setting: %s, err %s aq_err %s\n", 9619 pf->cur_promisc ? "on" : "off", 9620 i40e_stat_str(&pf->hw, ret), 9621 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 9622 9623 i40e_reset_all_vfs(pf, true); 9624 9625 /* tell the firmware that we're starting */ 9626 i40e_send_version(pf); 9627 9628 /* We've already released the lock, so don't do it again */ 9629 goto end_core_reset; 9630 9631 end_unlock: 9632 if (!lock_acquired) 9633 rtnl_unlock(); 9634 end_core_reset: 9635 clear_bit(__I40E_RESET_FAILED, pf->state); 9636 clear_recovery: 9637 clear_bit(__I40E_RESET_RECOVERY_PENDING, pf->state); 9638 } 9639 9640 /** 9641 * i40e_reset_and_rebuild - reset and rebuild using a saved config 9642 * @pf: board private structure 9643 * @reinit: if the Main VSI needs to re-initialized. 9644 * @lock_acquired: indicates whether or not the lock has been acquired 9645 * before this function was called. 9646 **/ 9647 static void i40e_reset_and_rebuild(struct i40e_pf *pf, bool reinit, 9648 bool lock_acquired) 9649 { 9650 int ret; 9651 /* Now we wait for GRST to settle out. 9652 * We don't have to delete the VEBs or VSIs from the hw switch 9653 * because the reset will make them disappear. 9654 */ 9655 ret = i40e_reset(pf); 9656 if (!ret) 9657 i40e_rebuild(pf, reinit, lock_acquired); 9658 } 9659 9660 /** 9661 * i40e_handle_reset_warning - prep for the PF to reset, reset and rebuild 9662 * @pf: board private structure 9663 * 9664 * Close up the VFs and other things in prep for a Core Reset, 9665 * then get ready to rebuild the world. 9666 * @lock_acquired: indicates whether or not the lock has been acquired 9667 * before this function was called. 9668 **/ 9669 static void i40e_handle_reset_warning(struct i40e_pf *pf, bool lock_acquired) 9670 { 9671 i40e_prep_for_reset(pf, lock_acquired); 9672 i40e_reset_and_rebuild(pf, false, lock_acquired); 9673 } 9674 9675 /** 9676 * i40e_handle_mdd_event 9677 * @pf: pointer to the PF structure 9678 * 9679 * Called from the MDD irq handler to identify possibly malicious vfs 9680 **/ 9681 static void i40e_handle_mdd_event(struct i40e_pf *pf) 9682 { 9683 struct i40e_hw *hw = &pf->hw; 9684 bool mdd_detected = false; 9685 bool pf_mdd_detected = false; 9686 struct i40e_vf *vf; 9687 u32 reg; 9688 int i; 9689 9690 if (!test_bit(__I40E_MDD_EVENT_PENDING, pf->state)) 9691 return; 9692 9693 /* find what triggered the MDD event */ 9694 reg = rd32(hw, I40E_GL_MDET_TX); 9695 if (reg & I40E_GL_MDET_TX_VALID_MASK) { 9696 u8 pf_num = (reg & I40E_GL_MDET_TX_PF_NUM_MASK) >> 9697 I40E_GL_MDET_TX_PF_NUM_SHIFT; 9698 u16 vf_num = (reg & I40E_GL_MDET_TX_VF_NUM_MASK) >> 9699 I40E_GL_MDET_TX_VF_NUM_SHIFT; 9700 u8 event = (reg & I40E_GL_MDET_TX_EVENT_MASK) >> 9701 I40E_GL_MDET_TX_EVENT_SHIFT; 9702 u16 queue = ((reg & I40E_GL_MDET_TX_QUEUE_MASK) >> 9703 I40E_GL_MDET_TX_QUEUE_SHIFT) - 9704 pf->hw.func_caps.base_queue; 9705 if (netif_msg_tx_err(pf)) 9706 dev_info(&pf->pdev->dev, "Malicious Driver Detection event 0x%02x on TX queue %d PF number 0x%02x VF number 0x%02x\n", 9707 event, queue, pf_num, vf_num); 9708 wr32(hw, I40E_GL_MDET_TX, 0xffffffff); 9709 mdd_detected = true; 9710 } 9711 reg = rd32(hw, I40E_GL_MDET_RX); 9712 if (reg & I40E_GL_MDET_RX_VALID_MASK) { 9713 u8 func = (reg & I40E_GL_MDET_RX_FUNCTION_MASK) >> 9714 I40E_GL_MDET_RX_FUNCTION_SHIFT; 9715 u8 event = (reg & I40E_GL_MDET_RX_EVENT_MASK) >> 9716 I40E_GL_MDET_RX_EVENT_SHIFT; 9717 u16 queue = ((reg & I40E_GL_MDET_RX_QUEUE_MASK) >> 9718 I40E_GL_MDET_RX_QUEUE_SHIFT) - 9719 pf->hw.func_caps.base_queue; 9720 if (netif_msg_rx_err(pf)) 9721 dev_info(&pf->pdev->dev, "Malicious Driver Detection event 0x%02x on RX queue %d of function 0x%02x\n", 9722 event, queue, func); 9723 wr32(hw, I40E_GL_MDET_RX, 0xffffffff); 9724 mdd_detected = true; 9725 } 9726 9727 if (mdd_detected) { 9728 reg = rd32(hw, I40E_PF_MDET_TX); 9729 if (reg & I40E_PF_MDET_TX_VALID_MASK) { 9730 wr32(hw, I40E_PF_MDET_TX, 0xFFFF); 9731 dev_info(&pf->pdev->dev, "TX driver issue detected, PF reset issued\n"); 9732 pf_mdd_detected = true; 9733 } 9734 reg = rd32(hw, I40E_PF_MDET_RX); 9735 if (reg & I40E_PF_MDET_RX_VALID_MASK) { 9736 wr32(hw, I40E_PF_MDET_RX, 0xFFFF); 9737 dev_info(&pf->pdev->dev, "RX driver issue detected, PF reset issued\n"); 9738 pf_mdd_detected = true; 9739 } 9740 /* Queue belongs to the PF, initiate a reset */ 9741 if (pf_mdd_detected) { 9742 set_bit(__I40E_PF_RESET_REQUESTED, pf->state); 9743 i40e_service_event_schedule(pf); 9744 } 9745 } 9746 9747 /* see if one of the VFs needs its hand slapped */ 9748 for (i = 0; i < pf->num_alloc_vfs && mdd_detected; i++) { 9749 vf = &(pf->vf[i]); 9750 reg = rd32(hw, I40E_VP_MDET_TX(i)); 9751 if (reg & I40E_VP_MDET_TX_VALID_MASK) { 9752 wr32(hw, I40E_VP_MDET_TX(i), 0xFFFF); 9753 vf->num_mdd_events++; 9754 dev_info(&pf->pdev->dev, "TX driver issue detected on VF %d\n", 9755 i); 9756 } 9757 9758 reg = rd32(hw, I40E_VP_MDET_RX(i)); 9759 if (reg & I40E_VP_MDET_RX_VALID_MASK) { 9760 wr32(hw, I40E_VP_MDET_RX(i), 0xFFFF); 9761 vf->num_mdd_events++; 9762 dev_info(&pf->pdev->dev, "RX driver issue detected on VF %d\n", 9763 i); 9764 } 9765 9766 if (vf->num_mdd_events > I40E_DEFAULT_NUM_MDD_EVENTS_ALLOWED) { 9767 dev_info(&pf->pdev->dev, 9768 "Too many MDD events on VF %d, disabled\n", i); 9769 dev_info(&pf->pdev->dev, 9770 "Use PF Control I/F to re-enable the VF\n"); 9771 set_bit(I40E_VF_STATE_DISABLED, &vf->vf_states); 9772 } 9773 } 9774 9775 /* re-enable mdd interrupt cause */ 9776 clear_bit(__I40E_MDD_EVENT_PENDING, pf->state); 9777 reg = rd32(hw, I40E_PFINT_ICR0_ENA); 9778 reg |= I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK; 9779 wr32(hw, I40E_PFINT_ICR0_ENA, reg); 9780 i40e_flush(hw); 9781 } 9782 9783 static const char *i40e_tunnel_name(u8 type) 9784 { 9785 switch (type) { 9786 case UDP_TUNNEL_TYPE_VXLAN: 9787 return "vxlan"; 9788 case UDP_TUNNEL_TYPE_GENEVE: 9789 return "geneve"; 9790 default: 9791 return "unknown"; 9792 } 9793 } 9794 9795 /** 9796 * i40e_sync_udp_filters - Trigger a sync event for existing UDP filters 9797 * @pf: board private structure 9798 **/ 9799 static void i40e_sync_udp_filters(struct i40e_pf *pf) 9800 { 9801 int i; 9802 9803 /* loop through and set pending bit for all active UDP filters */ 9804 for (i = 0; i < I40E_MAX_PF_UDP_OFFLOAD_PORTS; i++) { 9805 if (pf->udp_ports[i].port) 9806 pf->pending_udp_bitmap |= BIT_ULL(i); 9807 } 9808 9809 set_bit(__I40E_UDP_FILTER_SYNC_PENDING, pf->state); 9810 } 9811 9812 /** 9813 * i40e_sync_udp_filters_subtask - Sync the VSI filter list with HW 9814 * @pf: board private structure 9815 **/ 9816 static void i40e_sync_udp_filters_subtask(struct i40e_pf *pf) 9817 { 9818 struct i40e_hw *hw = &pf->hw; 9819 u8 filter_index, type; 9820 u16 port; 9821 int i; 9822 9823 if (!test_and_clear_bit(__I40E_UDP_FILTER_SYNC_PENDING, pf->state)) 9824 return; 9825 9826 /* acquire RTNL to maintain state of flags and port requests */ 9827 rtnl_lock(); 9828 9829 for (i = 0; i < I40E_MAX_PF_UDP_OFFLOAD_PORTS; i++) { 9830 if (pf->pending_udp_bitmap & BIT_ULL(i)) { 9831 struct i40e_udp_port_config *udp_port; 9832 i40e_status ret = 0; 9833 9834 udp_port = &pf->udp_ports[i]; 9835 pf->pending_udp_bitmap &= ~BIT_ULL(i); 9836 9837 port = READ_ONCE(udp_port->port); 9838 type = READ_ONCE(udp_port->type); 9839 filter_index = READ_ONCE(udp_port->filter_index); 9840 9841 /* release RTNL while we wait on AQ command */ 9842 rtnl_unlock(); 9843 9844 if (port) 9845 ret = i40e_aq_add_udp_tunnel(hw, port, 9846 type, 9847 &filter_index, 9848 NULL); 9849 else if (filter_index != I40E_UDP_PORT_INDEX_UNUSED) 9850 ret = i40e_aq_del_udp_tunnel(hw, filter_index, 9851 NULL); 9852 9853 /* reacquire RTNL so we can update filter_index */ 9854 rtnl_lock(); 9855 9856 if (ret) { 9857 dev_info(&pf->pdev->dev, 9858 "%s %s port %d, index %d failed, err %s aq_err %s\n", 9859 i40e_tunnel_name(type), 9860 port ? "add" : "delete", 9861 port, 9862 filter_index, 9863 i40e_stat_str(&pf->hw, ret), 9864 i40e_aq_str(&pf->hw, 9865 pf->hw.aq.asq_last_status)); 9866 if (port) { 9867 /* failed to add, just reset port, 9868 * drop pending bit for any deletion 9869 */ 9870 udp_port->port = 0; 9871 pf->pending_udp_bitmap &= ~BIT_ULL(i); 9872 } 9873 } else if (port) { 9874 /* record filter index on success */ 9875 udp_port->filter_index = filter_index; 9876 } 9877 } 9878 } 9879 9880 rtnl_unlock(); 9881 } 9882 9883 /** 9884 * i40e_service_task - Run the driver's async subtasks 9885 * @work: pointer to work_struct containing our data 9886 **/ 9887 static void i40e_service_task(struct work_struct *work) 9888 { 9889 struct i40e_pf *pf = container_of(work, 9890 struct i40e_pf, 9891 service_task); 9892 unsigned long start_time = jiffies; 9893 9894 /* don't bother with service tasks if a reset is in progress */ 9895 if (test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state)) 9896 return; 9897 9898 if (test_and_set_bit(__I40E_SERVICE_SCHED, pf->state)) 9899 return; 9900 9901 i40e_detect_recover_hung(pf->vsi[pf->lan_vsi]); 9902 i40e_sync_filters_subtask(pf); 9903 i40e_reset_subtask(pf); 9904 i40e_handle_mdd_event(pf); 9905 i40e_vc_process_vflr_event(pf); 9906 i40e_watchdog_subtask(pf); 9907 i40e_fdir_reinit_subtask(pf); 9908 if (test_and_clear_bit(__I40E_CLIENT_RESET, pf->state)) { 9909 /* Client subtask will reopen next time through. */ 9910 i40e_notify_client_of_netdev_close(pf->vsi[pf->lan_vsi], true); 9911 } else { 9912 i40e_client_subtask(pf); 9913 if (test_and_clear_bit(__I40E_CLIENT_L2_CHANGE, 9914 pf->state)) 9915 i40e_notify_client_of_l2_param_changes( 9916 pf->vsi[pf->lan_vsi]); 9917 } 9918 i40e_sync_filters_subtask(pf); 9919 i40e_sync_udp_filters_subtask(pf); 9920 i40e_clean_adminq_subtask(pf); 9921 9922 /* flush memory to make sure state is correct before next watchdog */ 9923 smp_mb__before_atomic(); 9924 clear_bit(__I40E_SERVICE_SCHED, pf->state); 9925 9926 /* If the tasks have taken longer than one timer cycle or there 9927 * is more work to be done, reschedule the service task now 9928 * rather than wait for the timer to tick again. 9929 */ 9930 if (time_after(jiffies, (start_time + pf->service_timer_period)) || 9931 test_bit(__I40E_ADMINQ_EVENT_PENDING, pf->state) || 9932 test_bit(__I40E_MDD_EVENT_PENDING, pf->state) || 9933 test_bit(__I40E_VFLR_EVENT_PENDING, pf->state)) 9934 i40e_service_event_schedule(pf); 9935 } 9936 9937 /** 9938 * i40e_service_timer - timer callback 9939 * @data: pointer to PF struct 9940 **/ 9941 static void i40e_service_timer(struct timer_list *t) 9942 { 9943 struct i40e_pf *pf = from_timer(pf, t, service_timer); 9944 9945 mod_timer(&pf->service_timer, 9946 round_jiffies(jiffies + pf->service_timer_period)); 9947 i40e_service_event_schedule(pf); 9948 } 9949 9950 /** 9951 * i40e_set_num_rings_in_vsi - Determine number of rings in the VSI 9952 * @vsi: the VSI being configured 9953 **/ 9954 static int i40e_set_num_rings_in_vsi(struct i40e_vsi *vsi) 9955 { 9956 struct i40e_pf *pf = vsi->back; 9957 9958 switch (vsi->type) { 9959 case I40E_VSI_MAIN: 9960 vsi->alloc_queue_pairs = pf->num_lan_qps; 9961 vsi->num_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS, 9962 I40E_REQ_DESCRIPTOR_MULTIPLE); 9963 if (pf->flags & I40E_FLAG_MSIX_ENABLED) 9964 vsi->num_q_vectors = pf->num_lan_msix; 9965 else 9966 vsi->num_q_vectors = 1; 9967 9968 break; 9969 9970 case I40E_VSI_FDIR: 9971 vsi->alloc_queue_pairs = 1; 9972 vsi->num_desc = ALIGN(I40E_FDIR_RING_COUNT, 9973 I40E_REQ_DESCRIPTOR_MULTIPLE); 9974 vsi->num_q_vectors = pf->num_fdsb_msix; 9975 break; 9976 9977 case I40E_VSI_VMDQ2: 9978 vsi->alloc_queue_pairs = pf->num_vmdq_qps; 9979 vsi->num_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS, 9980 I40E_REQ_DESCRIPTOR_MULTIPLE); 9981 vsi->num_q_vectors = pf->num_vmdq_msix; 9982 break; 9983 9984 case I40E_VSI_SRIOV: 9985 vsi->alloc_queue_pairs = pf->num_vf_qps; 9986 vsi->num_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS, 9987 I40E_REQ_DESCRIPTOR_MULTIPLE); 9988 break; 9989 9990 default: 9991 WARN_ON(1); 9992 return -ENODATA; 9993 } 9994 9995 return 0; 9996 } 9997 9998 /** 9999 * i40e_vsi_alloc_arrays - Allocate queue and vector pointer arrays for the vsi 10000 * @vsi: VSI pointer 10001 * @alloc_qvectors: a bool to specify if q_vectors need to be allocated. 10002 * 10003 * On error: returns error code (negative) 10004 * On success: returns 0 10005 **/ 10006 static int i40e_vsi_alloc_arrays(struct i40e_vsi *vsi, bool alloc_qvectors) 10007 { 10008 struct i40e_ring **next_rings; 10009 int size; 10010 int ret = 0; 10011 10012 /* allocate memory for both Tx, XDP Tx and Rx ring pointers */ 10013 size = sizeof(struct i40e_ring *) * vsi->alloc_queue_pairs * 10014 (i40e_enabled_xdp_vsi(vsi) ? 3 : 2); 10015 vsi->tx_rings = kzalloc(size, GFP_KERNEL); 10016 if (!vsi->tx_rings) 10017 return -ENOMEM; 10018 next_rings = vsi->tx_rings + vsi->alloc_queue_pairs; 10019 if (i40e_enabled_xdp_vsi(vsi)) { 10020 vsi->xdp_rings = next_rings; 10021 next_rings += vsi->alloc_queue_pairs; 10022 } 10023 vsi->rx_rings = next_rings; 10024 10025 if (alloc_qvectors) { 10026 /* allocate memory for q_vector pointers */ 10027 size = sizeof(struct i40e_q_vector *) * vsi->num_q_vectors; 10028 vsi->q_vectors = kzalloc(size, GFP_KERNEL); 10029 if (!vsi->q_vectors) { 10030 ret = -ENOMEM; 10031 goto err_vectors; 10032 } 10033 } 10034 return ret; 10035 10036 err_vectors: 10037 kfree(vsi->tx_rings); 10038 return ret; 10039 } 10040 10041 /** 10042 * i40e_vsi_mem_alloc - Allocates the next available struct vsi in the PF 10043 * @pf: board private structure 10044 * @type: type of VSI 10045 * 10046 * On error: returns error code (negative) 10047 * On success: returns vsi index in PF (positive) 10048 **/ 10049 static int i40e_vsi_mem_alloc(struct i40e_pf *pf, enum i40e_vsi_type type) 10050 { 10051 int ret = -ENODEV; 10052 struct i40e_vsi *vsi; 10053 int vsi_idx; 10054 int i; 10055 10056 /* Need to protect the allocation of the VSIs at the PF level */ 10057 mutex_lock(&pf->switch_mutex); 10058 10059 /* VSI list may be fragmented if VSI creation/destruction has 10060 * been happening. We can afford to do a quick scan to look 10061 * for any free VSIs in the list. 10062 * 10063 * find next empty vsi slot, looping back around if necessary 10064 */ 10065 i = pf->next_vsi; 10066 while (i < pf->num_alloc_vsi && pf->vsi[i]) 10067 i++; 10068 if (i >= pf->num_alloc_vsi) { 10069 i = 0; 10070 while (i < pf->next_vsi && pf->vsi[i]) 10071 i++; 10072 } 10073 10074 if (i < pf->num_alloc_vsi && !pf->vsi[i]) { 10075 vsi_idx = i; /* Found one! */ 10076 } else { 10077 ret = -ENODEV; 10078 goto unlock_pf; /* out of VSI slots! */ 10079 } 10080 pf->next_vsi = ++i; 10081 10082 vsi = kzalloc(sizeof(*vsi), GFP_KERNEL); 10083 if (!vsi) { 10084 ret = -ENOMEM; 10085 goto unlock_pf; 10086 } 10087 vsi->type = type; 10088 vsi->back = pf; 10089 set_bit(__I40E_VSI_DOWN, vsi->state); 10090 vsi->flags = 0; 10091 vsi->idx = vsi_idx; 10092 vsi->int_rate_limit = 0; 10093 vsi->rss_table_size = (vsi->type == I40E_VSI_MAIN) ? 10094 pf->rss_table_size : 64; 10095 vsi->netdev_registered = false; 10096 vsi->work_limit = I40E_DEFAULT_IRQ_WORK; 10097 hash_init(vsi->mac_filter_hash); 10098 vsi->irqs_ready = false; 10099 10100 ret = i40e_set_num_rings_in_vsi(vsi); 10101 if (ret) 10102 goto err_rings; 10103 10104 ret = i40e_vsi_alloc_arrays(vsi, true); 10105 if (ret) 10106 goto err_rings; 10107 10108 /* Setup default MSIX irq handler for VSI */ 10109 i40e_vsi_setup_irqhandler(vsi, i40e_msix_clean_rings); 10110 10111 /* Initialize VSI lock */ 10112 spin_lock_init(&vsi->mac_filter_hash_lock); 10113 pf->vsi[vsi_idx] = vsi; 10114 ret = vsi_idx; 10115 goto unlock_pf; 10116 10117 err_rings: 10118 pf->next_vsi = i - 1; 10119 kfree(vsi); 10120 unlock_pf: 10121 mutex_unlock(&pf->switch_mutex); 10122 return ret; 10123 } 10124 10125 /** 10126 * i40e_vsi_free_arrays - Free queue and vector pointer arrays for the VSI 10127 * @vsi: VSI pointer 10128 * @free_qvectors: a bool to specify if q_vectors need to be freed. 10129 * 10130 * On error: returns error code (negative) 10131 * On success: returns 0 10132 **/ 10133 static void i40e_vsi_free_arrays(struct i40e_vsi *vsi, bool free_qvectors) 10134 { 10135 /* free the ring and vector containers */ 10136 if (free_qvectors) { 10137 kfree(vsi->q_vectors); 10138 vsi->q_vectors = NULL; 10139 } 10140 kfree(vsi->tx_rings); 10141 vsi->tx_rings = NULL; 10142 vsi->rx_rings = NULL; 10143 vsi->xdp_rings = NULL; 10144 } 10145 10146 /** 10147 * i40e_clear_rss_config_user - clear the user configured RSS hash keys 10148 * and lookup table 10149 * @vsi: Pointer to VSI structure 10150 */ 10151 static void i40e_clear_rss_config_user(struct i40e_vsi *vsi) 10152 { 10153 if (!vsi) 10154 return; 10155 10156 kfree(vsi->rss_hkey_user); 10157 vsi->rss_hkey_user = NULL; 10158 10159 kfree(vsi->rss_lut_user); 10160 vsi->rss_lut_user = NULL; 10161 } 10162 10163 /** 10164 * i40e_vsi_clear - Deallocate the VSI provided 10165 * @vsi: the VSI being un-configured 10166 **/ 10167 static int i40e_vsi_clear(struct i40e_vsi *vsi) 10168 { 10169 struct i40e_pf *pf; 10170 10171 if (!vsi) 10172 return 0; 10173 10174 if (!vsi->back) 10175 goto free_vsi; 10176 pf = vsi->back; 10177 10178 mutex_lock(&pf->switch_mutex); 10179 if (!pf->vsi[vsi->idx]) { 10180 dev_err(&pf->pdev->dev, "pf->vsi[%d] is NULL, just free vsi[%d](type %d)\n", 10181 vsi->idx, vsi->idx, vsi->type); 10182 goto unlock_vsi; 10183 } 10184 10185 if (pf->vsi[vsi->idx] != vsi) { 10186 dev_err(&pf->pdev->dev, 10187 "pf->vsi[%d](type %d) != vsi[%d](type %d): no free!\n", 10188 pf->vsi[vsi->idx]->idx, 10189 pf->vsi[vsi->idx]->type, 10190 vsi->idx, vsi->type); 10191 goto unlock_vsi; 10192 } 10193 10194 /* updates the PF for this cleared vsi */ 10195 i40e_put_lump(pf->qp_pile, vsi->base_queue, vsi->idx); 10196 i40e_put_lump(pf->irq_pile, vsi->base_vector, vsi->idx); 10197 10198 i40e_vsi_free_arrays(vsi, true); 10199 i40e_clear_rss_config_user(vsi); 10200 10201 pf->vsi[vsi->idx] = NULL; 10202 if (vsi->idx < pf->next_vsi) 10203 pf->next_vsi = vsi->idx; 10204 10205 unlock_vsi: 10206 mutex_unlock(&pf->switch_mutex); 10207 free_vsi: 10208 kfree(vsi); 10209 10210 return 0; 10211 } 10212 10213 /** 10214 * i40e_vsi_clear_rings - Deallocates the Rx and Tx rings for the provided VSI 10215 * @vsi: the VSI being cleaned 10216 **/ 10217 static void i40e_vsi_clear_rings(struct i40e_vsi *vsi) 10218 { 10219 int i; 10220 10221 if (vsi->tx_rings && vsi->tx_rings[0]) { 10222 for (i = 0; i < vsi->alloc_queue_pairs; i++) { 10223 kfree_rcu(vsi->tx_rings[i], rcu); 10224 vsi->tx_rings[i] = NULL; 10225 vsi->rx_rings[i] = NULL; 10226 if (vsi->xdp_rings) 10227 vsi->xdp_rings[i] = NULL; 10228 } 10229 } 10230 } 10231 10232 /** 10233 * i40e_alloc_rings - Allocates the Rx and Tx rings for the provided VSI 10234 * @vsi: the VSI being configured 10235 **/ 10236 static int i40e_alloc_rings(struct i40e_vsi *vsi) 10237 { 10238 int i, qpv = i40e_enabled_xdp_vsi(vsi) ? 3 : 2; 10239 struct i40e_pf *pf = vsi->back; 10240 struct i40e_ring *ring; 10241 10242 /* Set basic values in the rings to be used later during open() */ 10243 for (i = 0; i < vsi->alloc_queue_pairs; i++) { 10244 /* allocate space for both Tx and Rx in one shot */ 10245 ring = kcalloc(qpv, sizeof(struct i40e_ring), GFP_KERNEL); 10246 if (!ring) 10247 goto err_out; 10248 10249 ring->queue_index = i; 10250 ring->reg_idx = vsi->base_queue + i; 10251 ring->ring_active = false; 10252 ring->vsi = vsi; 10253 ring->netdev = vsi->netdev; 10254 ring->dev = &pf->pdev->dev; 10255 ring->count = vsi->num_desc; 10256 ring->size = 0; 10257 ring->dcb_tc = 0; 10258 if (vsi->back->hw_features & I40E_HW_WB_ON_ITR_CAPABLE) 10259 ring->flags = I40E_TXR_FLAGS_WB_ON_ITR; 10260 ring->itr_setting = pf->tx_itr_default; 10261 vsi->tx_rings[i] = ring++; 10262 10263 if (!i40e_enabled_xdp_vsi(vsi)) 10264 goto setup_rx; 10265 10266 ring->queue_index = vsi->alloc_queue_pairs + i; 10267 ring->reg_idx = vsi->base_queue + ring->queue_index; 10268 ring->ring_active = false; 10269 ring->vsi = vsi; 10270 ring->netdev = NULL; 10271 ring->dev = &pf->pdev->dev; 10272 ring->count = vsi->num_desc; 10273 ring->size = 0; 10274 ring->dcb_tc = 0; 10275 if (vsi->back->hw_features & I40E_HW_WB_ON_ITR_CAPABLE) 10276 ring->flags = I40E_TXR_FLAGS_WB_ON_ITR; 10277 set_ring_xdp(ring); 10278 ring->itr_setting = pf->tx_itr_default; 10279 vsi->xdp_rings[i] = ring++; 10280 10281 setup_rx: 10282 ring->queue_index = i; 10283 ring->reg_idx = vsi->base_queue + i; 10284 ring->ring_active = false; 10285 ring->vsi = vsi; 10286 ring->netdev = vsi->netdev; 10287 ring->dev = &pf->pdev->dev; 10288 ring->count = vsi->num_desc; 10289 ring->size = 0; 10290 ring->dcb_tc = 0; 10291 ring->itr_setting = pf->rx_itr_default; 10292 vsi->rx_rings[i] = ring; 10293 } 10294 10295 return 0; 10296 10297 err_out: 10298 i40e_vsi_clear_rings(vsi); 10299 return -ENOMEM; 10300 } 10301 10302 /** 10303 * i40e_reserve_msix_vectors - Reserve MSI-X vectors in the kernel 10304 * @pf: board private structure 10305 * @vectors: the number of MSI-X vectors to request 10306 * 10307 * Returns the number of vectors reserved, or error 10308 **/ 10309 static int i40e_reserve_msix_vectors(struct i40e_pf *pf, int vectors) 10310 { 10311 vectors = pci_enable_msix_range(pf->pdev, pf->msix_entries, 10312 I40E_MIN_MSIX, vectors); 10313 if (vectors < 0) { 10314 dev_info(&pf->pdev->dev, 10315 "MSI-X vector reservation failed: %d\n", vectors); 10316 vectors = 0; 10317 } 10318 10319 return vectors; 10320 } 10321 10322 /** 10323 * i40e_init_msix - Setup the MSIX capability 10324 * @pf: board private structure 10325 * 10326 * Work with the OS to set up the MSIX vectors needed. 10327 * 10328 * Returns the number of vectors reserved or negative on failure 10329 **/ 10330 static int i40e_init_msix(struct i40e_pf *pf) 10331 { 10332 struct i40e_hw *hw = &pf->hw; 10333 int cpus, extra_vectors; 10334 int vectors_left; 10335 int v_budget, i; 10336 int v_actual; 10337 int iwarp_requested = 0; 10338 10339 if (!(pf->flags & I40E_FLAG_MSIX_ENABLED)) 10340 return -ENODEV; 10341 10342 /* The number of vectors we'll request will be comprised of: 10343 * - Add 1 for "other" cause for Admin Queue events, etc. 10344 * - The number of LAN queue pairs 10345 * - Queues being used for RSS. 10346 * We don't need as many as max_rss_size vectors. 10347 * use rss_size instead in the calculation since that 10348 * is governed by number of cpus in the system. 10349 * - assumes symmetric Tx/Rx pairing 10350 * - The number of VMDq pairs 10351 * - The CPU count within the NUMA node if iWARP is enabled 10352 * Once we count this up, try the request. 10353 * 10354 * If we can't get what we want, we'll simplify to nearly nothing 10355 * and try again. If that still fails, we punt. 10356 */ 10357 vectors_left = hw->func_caps.num_msix_vectors; 10358 v_budget = 0; 10359 10360 /* reserve one vector for miscellaneous handler */ 10361 if (vectors_left) { 10362 v_budget++; 10363 vectors_left--; 10364 } 10365 10366 /* reserve some vectors for the main PF traffic queues. Initially we 10367 * only reserve at most 50% of the available vectors, in the case that 10368 * the number of online CPUs is large. This ensures that we can enable 10369 * extra features as well. Once we've enabled the other features, we 10370 * will use any remaining vectors to reach as close as we can to the 10371 * number of online CPUs. 10372 */ 10373 cpus = num_online_cpus(); 10374 pf->num_lan_msix = min_t(int, cpus, vectors_left / 2); 10375 vectors_left -= pf->num_lan_msix; 10376 10377 /* reserve one vector for sideband flow director */ 10378 if (pf->flags & I40E_FLAG_FD_SB_ENABLED) { 10379 if (vectors_left) { 10380 pf->num_fdsb_msix = 1; 10381 v_budget++; 10382 vectors_left--; 10383 } else { 10384 pf->num_fdsb_msix = 0; 10385 } 10386 } 10387 10388 /* can we reserve enough for iWARP? */ 10389 if (pf->flags & I40E_FLAG_IWARP_ENABLED) { 10390 iwarp_requested = pf->num_iwarp_msix; 10391 10392 if (!vectors_left) 10393 pf->num_iwarp_msix = 0; 10394 else if (vectors_left < pf->num_iwarp_msix) 10395 pf->num_iwarp_msix = 1; 10396 v_budget += pf->num_iwarp_msix; 10397 vectors_left -= pf->num_iwarp_msix; 10398 } 10399 10400 /* any vectors left over go for VMDq support */ 10401 if (pf->flags & I40E_FLAG_VMDQ_ENABLED) { 10402 if (!vectors_left) { 10403 pf->num_vmdq_msix = 0; 10404 pf->num_vmdq_qps = 0; 10405 } else { 10406 int vmdq_vecs_wanted = 10407 pf->num_vmdq_vsis * pf->num_vmdq_qps; 10408 int vmdq_vecs = 10409 min_t(int, vectors_left, vmdq_vecs_wanted); 10410 10411 /* if we're short on vectors for what's desired, we limit 10412 * the queues per vmdq. If this is still more than are 10413 * available, the user will need to change the number of 10414 * queues/vectors used by the PF later with the ethtool 10415 * channels command 10416 */ 10417 if (vectors_left < vmdq_vecs_wanted) { 10418 pf->num_vmdq_qps = 1; 10419 vmdq_vecs_wanted = pf->num_vmdq_vsis; 10420 vmdq_vecs = min_t(int, 10421 vectors_left, 10422 vmdq_vecs_wanted); 10423 } 10424 pf->num_vmdq_msix = pf->num_vmdq_qps; 10425 10426 v_budget += vmdq_vecs; 10427 vectors_left -= vmdq_vecs; 10428 } 10429 } 10430 10431 /* On systems with a large number of SMP cores, we previously limited 10432 * the number of vectors for num_lan_msix to be at most 50% of the 10433 * available vectors, to allow for other features. Now, we add back 10434 * the remaining vectors. However, we ensure that the total 10435 * num_lan_msix will not exceed num_online_cpus(). To do this, we 10436 * calculate the number of vectors we can add without going over the 10437 * cap of CPUs. For systems with a small number of CPUs this will be 10438 * zero. 10439 */ 10440 extra_vectors = min_t(int, cpus - pf->num_lan_msix, vectors_left); 10441 pf->num_lan_msix += extra_vectors; 10442 vectors_left -= extra_vectors; 10443 10444 WARN(vectors_left < 0, 10445 "Calculation of remaining vectors underflowed. This is an accounting bug when determining total MSI-X vectors.\n"); 10446 10447 v_budget += pf->num_lan_msix; 10448 pf->msix_entries = kcalloc(v_budget, sizeof(struct msix_entry), 10449 GFP_KERNEL); 10450 if (!pf->msix_entries) 10451 return -ENOMEM; 10452 10453 for (i = 0; i < v_budget; i++) 10454 pf->msix_entries[i].entry = i; 10455 v_actual = i40e_reserve_msix_vectors(pf, v_budget); 10456 10457 if (v_actual < I40E_MIN_MSIX) { 10458 pf->flags &= ~I40E_FLAG_MSIX_ENABLED; 10459 kfree(pf->msix_entries); 10460 pf->msix_entries = NULL; 10461 pci_disable_msix(pf->pdev); 10462 return -ENODEV; 10463 10464 } else if (v_actual == I40E_MIN_MSIX) { 10465 /* Adjust for minimal MSIX use */ 10466 pf->num_vmdq_vsis = 0; 10467 pf->num_vmdq_qps = 0; 10468 pf->num_lan_qps = 1; 10469 pf->num_lan_msix = 1; 10470 10471 } else if (v_actual != v_budget) { 10472 /* If we have limited resources, we will start with no vectors 10473 * for the special features and then allocate vectors to some 10474 * of these features based on the policy and at the end disable 10475 * the features that did not get any vectors. 10476 */ 10477 int vec; 10478 10479 dev_info(&pf->pdev->dev, 10480 "MSI-X vector limit reached with %d, wanted %d, attempting to redistribute vectors\n", 10481 v_actual, v_budget); 10482 /* reserve the misc vector */ 10483 vec = v_actual - 1; 10484 10485 /* Scale vector usage down */ 10486 pf->num_vmdq_msix = 1; /* force VMDqs to only one vector */ 10487 pf->num_vmdq_vsis = 1; 10488 pf->num_vmdq_qps = 1; 10489 10490 /* partition out the remaining vectors */ 10491 switch (vec) { 10492 case 2: 10493 pf->num_lan_msix = 1; 10494 break; 10495 case 3: 10496 if (pf->flags & I40E_FLAG_IWARP_ENABLED) { 10497 pf->num_lan_msix = 1; 10498 pf->num_iwarp_msix = 1; 10499 } else { 10500 pf->num_lan_msix = 2; 10501 } 10502 break; 10503 default: 10504 if (pf->flags & I40E_FLAG_IWARP_ENABLED) { 10505 pf->num_iwarp_msix = min_t(int, (vec / 3), 10506 iwarp_requested); 10507 pf->num_vmdq_vsis = min_t(int, (vec / 3), 10508 I40E_DEFAULT_NUM_VMDQ_VSI); 10509 } else { 10510 pf->num_vmdq_vsis = min_t(int, (vec / 2), 10511 I40E_DEFAULT_NUM_VMDQ_VSI); 10512 } 10513 if (pf->flags & I40E_FLAG_FD_SB_ENABLED) { 10514 pf->num_fdsb_msix = 1; 10515 vec--; 10516 } 10517 pf->num_lan_msix = min_t(int, 10518 (vec - (pf->num_iwarp_msix + pf->num_vmdq_vsis)), 10519 pf->num_lan_msix); 10520 pf->num_lan_qps = pf->num_lan_msix; 10521 break; 10522 } 10523 } 10524 10525 if ((pf->flags & I40E_FLAG_FD_SB_ENABLED) && 10526 (pf->num_fdsb_msix == 0)) { 10527 dev_info(&pf->pdev->dev, "Sideband Flowdir disabled, not enough MSI-X vectors\n"); 10528 pf->flags &= ~I40E_FLAG_FD_SB_ENABLED; 10529 pf->flags |= I40E_FLAG_FD_SB_INACTIVE; 10530 } 10531 if ((pf->flags & I40E_FLAG_VMDQ_ENABLED) && 10532 (pf->num_vmdq_msix == 0)) { 10533 dev_info(&pf->pdev->dev, "VMDq disabled, not enough MSI-X vectors\n"); 10534 pf->flags &= ~I40E_FLAG_VMDQ_ENABLED; 10535 } 10536 10537 if ((pf->flags & I40E_FLAG_IWARP_ENABLED) && 10538 (pf->num_iwarp_msix == 0)) { 10539 dev_info(&pf->pdev->dev, "IWARP disabled, not enough MSI-X vectors\n"); 10540 pf->flags &= ~I40E_FLAG_IWARP_ENABLED; 10541 } 10542 i40e_debug(&pf->hw, I40E_DEBUG_INIT, 10543 "MSI-X vector distribution: PF %d, VMDq %d, FDSB %d, iWARP %d\n", 10544 pf->num_lan_msix, 10545 pf->num_vmdq_msix * pf->num_vmdq_vsis, 10546 pf->num_fdsb_msix, 10547 pf->num_iwarp_msix); 10548 10549 return v_actual; 10550 } 10551 10552 /** 10553 * i40e_vsi_alloc_q_vector - Allocate memory for a single interrupt vector 10554 * @vsi: the VSI being configured 10555 * @v_idx: index of the vector in the vsi struct 10556 * @cpu: cpu to be used on affinity_mask 10557 * 10558 * We allocate one q_vector. If allocation fails we return -ENOMEM. 10559 **/ 10560 static int i40e_vsi_alloc_q_vector(struct i40e_vsi *vsi, int v_idx, int cpu) 10561 { 10562 struct i40e_q_vector *q_vector; 10563 10564 /* allocate q_vector */ 10565 q_vector = kzalloc(sizeof(struct i40e_q_vector), GFP_KERNEL); 10566 if (!q_vector) 10567 return -ENOMEM; 10568 10569 q_vector->vsi = vsi; 10570 q_vector->v_idx = v_idx; 10571 cpumask_copy(&q_vector->affinity_mask, cpu_possible_mask); 10572 10573 if (vsi->netdev) 10574 netif_napi_add(vsi->netdev, &q_vector->napi, 10575 i40e_napi_poll, NAPI_POLL_WEIGHT); 10576 10577 /* tie q_vector and vsi together */ 10578 vsi->q_vectors[v_idx] = q_vector; 10579 10580 return 0; 10581 } 10582 10583 /** 10584 * i40e_vsi_alloc_q_vectors - Allocate memory for interrupt vectors 10585 * @vsi: the VSI being configured 10586 * 10587 * We allocate one q_vector per queue interrupt. If allocation fails we 10588 * return -ENOMEM. 10589 **/ 10590 static int i40e_vsi_alloc_q_vectors(struct i40e_vsi *vsi) 10591 { 10592 struct i40e_pf *pf = vsi->back; 10593 int err, v_idx, num_q_vectors, current_cpu; 10594 10595 /* if not MSIX, give the one vector only to the LAN VSI */ 10596 if (pf->flags & I40E_FLAG_MSIX_ENABLED) 10597 num_q_vectors = vsi->num_q_vectors; 10598 else if (vsi == pf->vsi[pf->lan_vsi]) 10599 num_q_vectors = 1; 10600 else 10601 return -EINVAL; 10602 10603 current_cpu = cpumask_first(cpu_online_mask); 10604 10605 for (v_idx = 0; v_idx < num_q_vectors; v_idx++) { 10606 err = i40e_vsi_alloc_q_vector(vsi, v_idx, current_cpu); 10607 if (err) 10608 goto err_out; 10609 current_cpu = cpumask_next(current_cpu, cpu_online_mask); 10610 if (unlikely(current_cpu >= nr_cpu_ids)) 10611 current_cpu = cpumask_first(cpu_online_mask); 10612 } 10613 10614 return 0; 10615 10616 err_out: 10617 while (v_idx--) 10618 i40e_free_q_vector(vsi, v_idx); 10619 10620 return err; 10621 } 10622 10623 /** 10624 * i40e_init_interrupt_scheme - Determine proper interrupt scheme 10625 * @pf: board private structure to initialize 10626 **/ 10627 static int i40e_init_interrupt_scheme(struct i40e_pf *pf) 10628 { 10629 int vectors = 0; 10630 ssize_t size; 10631 10632 if (pf->flags & I40E_FLAG_MSIX_ENABLED) { 10633 vectors = i40e_init_msix(pf); 10634 if (vectors < 0) { 10635 pf->flags &= ~(I40E_FLAG_MSIX_ENABLED | 10636 I40E_FLAG_IWARP_ENABLED | 10637 I40E_FLAG_RSS_ENABLED | 10638 I40E_FLAG_DCB_CAPABLE | 10639 I40E_FLAG_DCB_ENABLED | 10640 I40E_FLAG_SRIOV_ENABLED | 10641 I40E_FLAG_FD_SB_ENABLED | 10642 I40E_FLAG_FD_ATR_ENABLED | 10643 I40E_FLAG_VMDQ_ENABLED); 10644 pf->flags |= I40E_FLAG_FD_SB_INACTIVE; 10645 10646 /* rework the queue expectations without MSIX */ 10647 i40e_determine_queue_usage(pf); 10648 } 10649 } 10650 10651 if (!(pf->flags & I40E_FLAG_MSIX_ENABLED) && 10652 (pf->flags & I40E_FLAG_MSI_ENABLED)) { 10653 dev_info(&pf->pdev->dev, "MSI-X not available, trying MSI\n"); 10654 vectors = pci_enable_msi(pf->pdev); 10655 if (vectors < 0) { 10656 dev_info(&pf->pdev->dev, "MSI init failed - %d\n", 10657 vectors); 10658 pf->flags &= ~I40E_FLAG_MSI_ENABLED; 10659 } 10660 vectors = 1; /* one MSI or Legacy vector */ 10661 } 10662 10663 if (!(pf->flags & (I40E_FLAG_MSIX_ENABLED | I40E_FLAG_MSI_ENABLED))) 10664 dev_info(&pf->pdev->dev, "MSI-X and MSI not available, falling back to Legacy IRQ\n"); 10665 10666 /* set up vector assignment tracking */ 10667 size = sizeof(struct i40e_lump_tracking) + (sizeof(u16) * vectors); 10668 pf->irq_pile = kzalloc(size, GFP_KERNEL); 10669 if (!pf->irq_pile) 10670 return -ENOMEM; 10671 10672 pf->irq_pile->num_entries = vectors; 10673 pf->irq_pile->search_hint = 0; 10674 10675 /* track first vector for misc interrupts, ignore return */ 10676 (void)i40e_get_lump(pf, pf->irq_pile, 1, I40E_PILE_VALID_BIT - 1); 10677 10678 return 0; 10679 } 10680 10681 /** 10682 * i40e_restore_interrupt_scheme - Restore the interrupt scheme 10683 * @pf: private board data structure 10684 * 10685 * Restore the interrupt scheme that was cleared when we suspended the 10686 * device. This should be called during resume to re-allocate the q_vectors 10687 * and reacquire IRQs. 10688 */ 10689 static int i40e_restore_interrupt_scheme(struct i40e_pf *pf) 10690 { 10691 int err, i; 10692 10693 /* We cleared the MSI and MSI-X flags when disabling the old interrupt 10694 * scheme. We need to re-enabled them here in order to attempt to 10695 * re-acquire the MSI or MSI-X vectors 10696 */ 10697 pf->flags |= (I40E_FLAG_MSIX_ENABLED | I40E_FLAG_MSI_ENABLED); 10698 10699 err = i40e_init_interrupt_scheme(pf); 10700 if (err) 10701 return err; 10702 10703 /* Now that we've re-acquired IRQs, we need to remap the vectors and 10704 * rings together again. 10705 */ 10706 for (i = 0; i < pf->num_alloc_vsi; i++) { 10707 if (pf->vsi[i]) { 10708 err = i40e_vsi_alloc_q_vectors(pf->vsi[i]); 10709 if (err) 10710 goto err_unwind; 10711 i40e_vsi_map_rings_to_vectors(pf->vsi[i]); 10712 } 10713 } 10714 10715 err = i40e_setup_misc_vector(pf); 10716 if (err) 10717 goto err_unwind; 10718 10719 if (pf->flags & I40E_FLAG_IWARP_ENABLED) 10720 i40e_client_update_msix_info(pf); 10721 10722 return 0; 10723 10724 err_unwind: 10725 while (i--) { 10726 if (pf->vsi[i]) 10727 i40e_vsi_free_q_vectors(pf->vsi[i]); 10728 } 10729 10730 return err; 10731 } 10732 10733 /** 10734 * i40e_setup_misc_vector - Setup the misc vector to handle non queue events 10735 * @pf: board private structure 10736 * 10737 * This sets up the handler for MSIX 0, which is used to manage the 10738 * non-queue interrupts, e.g. AdminQ and errors. This is not used 10739 * when in MSI or Legacy interrupt mode. 10740 **/ 10741 static int i40e_setup_misc_vector(struct i40e_pf *pf) 10742 { 10743 struct i40e_hw *hw = &pf->hw; 10744 int err = 0; 10745 10746 /* Only request the IRQ once, the first time through. */ 10747 if (!test_and_set_bit(__I40E_MISC_IRQ_REQUESTED, pf->state)) { 10748 err = request_irq(pf->msix_entries[0].vector, 10749 i40e_intr, 0, pf->int_name, pf); 10750 if (err) { 10751 clear_bit(__I40E_MISC_IRQ_REQUESTED, pf->state); 10752 dev_info(&pf->pdev->dev, 10753 "request_irq for %s failed: %d\n", 10754 pf->int_name, err); 10755 return -EFAULT; 10756 } 10757 } 10758 10759 i40e_enable_misc_int_causes(pf); 10760 10761 /* associate no queues to the misc vector */ 10762 wr32(hw, I40E_PFINT_LNKLST0, I40E_QUEUE_END_OF_LIST); 10763 wr32(hw, I40E_PFINT_ITR0(I40E_RX_ITR), I40E_ITR_8K); 10764 10765 i40e_flush(hw); 10766 10767 i40e_irq_dynamic_enable_icr0(pf); 10768 10769 return err; 10770 } 10771 10772 /** 10773 * i40e_get_rss_aq - Get RSS keys and lut by using AQ commands 10774 * @vsi: Pointer to vsi structure 10775 * @seed: Buffter to store the hash keys 10776 * @lut: Buffer to store the lookup table entries 10777 * @lut_size: Size of buffer to store the lookup table entries 10778 * 10779 * Return 0 on success, negative on failure 10780 */ 10781 static int i40e_get_rss_aq(struct i40e_vsi *vsi, const u8 *seed, 10782 u8 *lut, u16 lut_size) 10783 { 10784 struct i40e_pf *pf = vsi->back; 10785 struct i40e_hw *hw = &pf->hw; 10786 int ret = 0; 10787 10788 if (seed) { 10789 ret = i40e_aq_get_rss_key(hw, vsi->id, 10790 (struct i40e_aqc_get_set_rss_key_data *)seed); 10791 if (ret) { 10792 dev_info(&pf->pdev->dev, 10793 "Cannot get RSS key, err %s aq_err %s\n", 10794 i40e_stat_str(&pf->hw, ret), 10795 i40e_aq_str(&pf->hw, 10796 pf->hw.aq.asq_last_status)); 10797 return ret; 10798 } 10799 } 10800 10801 if (lut) { 10802 bool pf_lut = vsi->type == I40E_VSI_MAIN ? true : false; 10803 10804 ret = i40e_aq_get_rss_lut(hw, vsi->id, pf_lut, lut, lut_size); 10805 if (ret) { 10806 dev_info(&pf->pdev->dev, 10807 "Cannot get RSS lut, err %s aq_err %s\n", 10808 i40e_stat_str(&pf->hw, ret), 10809 i40e_aq_str(&pf->hw, 10810 pf->hw.aq.asq_last_status)); 10811 return ret; 10812 } 10813 } 10814 10815 return ret; 10816 } 10817 10818 /** 10819 * i40e_config_rss_reg - Configure RSS keys and lut by writing registers 10820 * @vsi: Pointer to vsi structure 10821 * @seed: RSS hash seed 10822 * @lut: Lookup table 10823 * @lut_size: Lookup table size 10824 * 10825 * Returns 0 on success, negative on failure 10826 **/ 10827 static int i40e_config_rss_reg(struct i40e_vsi *vsi, const u8 *seed, 10828 const u8 *lut, u16 lut_size) 10829 { 10830 struct i40e_pf *pf = vsi->back; 10831 struct i40e_hw *hw = &pf->hw; 10832 u16 vf_id = vsi->vf_id; 10833 u8 i; 10834 10835 /* Fill out hash function seed */ 10836 if (seed) { 10837 u32 *seed_dw = (u32 *)seed; 10838 10839 if (vsi->type == I40E_VSI_MAIN) { 10840 for (i = 0; i <= I40E_PFQF_HKEY_MAX_INDEX; i++) 10841 wr32(hw, I40E_PFQF_HKEY(i), seed_dw[i]); 10842 } else if (vsi->type == I40E_VSI_SRIOV) { 10843 for (i = 0; i <= I40E_VFQF_HKEY1_MAX_INDEX; i++) 10844 wr32(hw, I40E_VFQF_HKEY1(i, vf_id), seed_dw[i]); 10845 } else { 10846 dev_err(&pf->pdev->dev, "Cannot set RSS seed - invalid VSI type\n"); 10847 } 10848 } 10849 10850 if (lut) { 10851 u32 *lut_dw = (u32 *)lut; 10852 10853 if (vsi->type == I40E_VSI_MAIN) { 10854 if (lut_size != I40E_HLUT_ARRAY_SIZE) 10855 return -EINVAL; 10856 for (i = 0; i <= I40E_PFQF_HLUT_MAX_INDEX; i++) 10857 wr32(hw, I40E_PFQF_HLUT(i), lut_dw[i]); 10858 } else if (vsi->type == I40E_VSI_SRIOV) { 10859 if (lut_size != I40E_VF_HLUT_ARRAY_SIZE) 10860 return -EINVAL; 10861 for (i = 0; i <= I40E_VFQF_HLUT_MAX_INDEX; i++) 10862 wr32(hw, I40E_VFQF_HLUT1(i, vf_id), lut_dw[i]); 10863 } else { 10864 dev_err(&pf->pdev->dev, "Cannot set RSS LUT - invalid VSI type\n"); 10865 } 10866 } 10867 i40e_flush(hw); 10868 10869 return 0; 10870 } 10871 10872 /** 10873 * i40e_get_rss_reg - Get the RSS keys and lut by reading registers 10874 * @vsi: Pointer to VSI structure 10875 * @seed: Buffer to store the keys 10876 * @lut: Buffer to store the lookup table entries 10877 * @lut_size: Size of buffer to store the lookup table entries 10878 * 10879 * Returns 0 on success, negative on failure 10880 */ 10881 static int i40e_get_rss_reg(struct i40e_vsi *vsi, u8 *seed, 10882 u8 *lut, u16 lut_size) 10883 { 10884 struct i40e_pf *pf = vsi->back; 10885 struct i40e_hw *hw = &pf->hw; 10886 u16 i; 10887 10888 if (seed) { 10889 u32 *seed_dw = (u32 *)seed; 10890 10891 for (i = 0; i <= I40E_PFQF_HKEY_MAX_INDEX; i++) 10892 seed_dw[i] = i40e_read_rx_ctl(hw, I40E_PFQF_HKEY(i)); 10893 } 10894 if (lut) { 10895 u32 *lut_dw = (u32 *)lut; 10896 10897 if (lut_size != I40E_HLUT_ARRAY_SIZE) 10898 return -EINVAL; 10899 for (i = 0; i <= I40E_PFQF_HLUT_MAX_INDEX; i++) 10900 lut_dw[i] = rd32(hw, I40E_PFQF_HLUT(i)); 10901 } 10902 10903 return 0; 10904 } 10905 10906 /** 10907 * i40e_config_rss - Configure RSS keys and lut 10908 * @vsi: Pointer to VSI structure 10909 * @seed: RSS hash seed 10910 * @lut: Lookup table 10911 * @lut_size: Lookup table size 10912 * 10913 * Returns 0 on success, negative on failure 10914 */ 10915 int i40e_config_rss(struct i40e_vsi *vsi, u8 *seed, u8 *lut, u16 lut_size) 10916 { 10917 struct i40e_pf *pf = vsi->back; 10918 10919 if (pf->hw_features & I40E_HW_RSS_AQ_CAPABLE) 10920 return i40e_config_rss_aq(vsi, seed, lut, lut_size); 10921 else 10922 return i40e_config_rss_reg(vsi, seed, lut, lut_size); 10923 } 10924 10925 /** 10926 * i40e_get_rss - Get RSS keys and lut 10927 * @vsi: Pointer to VSI structure 10928 * @seed: Buffer to store the keys 10929 * @lut: Buffer to store the lookup table entries 10930 * @lut_size: Size of buffer to store the lookup table entries 10931 * 10932 * Returns 0 on success, negative on failure 10933 */ 10934 int i40e_get_rss(struct i40e_vsi *vsi, u8 *seed, u8 *lut, u16 lut_size) 10935 { 10936 struct i40e_pf *pf = vsi->back; 10937 10938 if (pf->hw_features & I40E_HW_RSS_AQ_CAPABLE) 10939 return i40e_get_rss_aq(vsi, seed, lut, lut_size); 10940 else 10941 return i40e_get_rss_reg(vsi, seed, lut, lut_size); 10942 } 10943 10944 /** 10945 * i40e_fill_rss_lut - Fill the RSS lookup table with default values 10946 * @pf: Pointer to board private structure 10947 * @lut: Lookup table 10948 * @rss_table_size: Lookup table size 10949 * @rss_size: Range of queue number for hashing 10950 */ 10951 void i40e_fill_rss_lut(struct i40e_pf *pf, u8 *lut, 10952 u16 rss_table_size, u16 rss_size) 10953 { 10954 u16 i; 10955 10956 for (i = 0; i < rss_table_size; i++) 10957 lut[i] = i % rss_size; 10958 } 10959 10960 /** 10961 * i40e_pf_config_rss - Prepare for RSS if used 10962 * @pf: board private structure 10963 **/ 10964 static int i40e_pf_config_rss(struct i40e_pf *pf) 10965 { 10966 struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi]; 10967 u8 seed[I40E_HKEY_ARRAY_SIZE]; 10968 u8 *lut; 10969 struct i40e_hw *hw = &pf->hw; 10970 u32 reg_val; 10971 u64 hena; 10972 int ret; 10973 10974 /* By default we enable TCP/UDP with IPv4/IPv6 ptypes */ 10975 hena = (u64)i40e_read_rx_ctl(hw, I40E_PFQF_HENA(0)) | 10976 ((u64)i40e_read_rx_ctl(hw, I40E_PFQF_HENA(1)) << 32); 10977 hena |= i40e_pf_get_default_rss_hena(pf); 10978 10979 i40e_write_rx_ctl(hw, I40E_PFQF_HENA(0), (u32)hena); 10980 i40e_write_rx_ctl(hw, I40E_PFQF_HENA(1), (u32)(hena >> 32)); 10981 10982 /* Determine the RSS table size based on the hardware capabilities */ 10983 reg_val = i40e_read_rx_ctl(hw, I40E_PFQF_CTL_0); 10984 reg_val = (pf->rss_table_size == 512) ? 10985 (reg_val | I40E_PFQF_CTL_0_HASHLUTSIZE_512) : 10986 (reg_val & ~I40E_PFQF_CTL_0_HASHLUTSIZE_512); 10987 i40e_write_rx_ctl(hw, I40E_PFQF_CTL_0, reg_val); 10988 10989 /* Determine the RSS size of the VSI */ 10990 if (!vsi->rss_size) { 10991 u16 qcount; 10992 /* If the firmware does something weird during VSI init, we 10993 * could end up with zero TCs. Check for that to avoid 10994 * divide-by-zero. It probably won't pass traffic, but it also 10995 * won't panic. 10996 */ 10997 qcount = vsi->num_queue_pairs / 10998 (vsi->tc_config.numtc ? vsi->tc_config.numtc : 1); 10999 vsi->rss_size = min_t(int, pf->alloc_rss_size, qcount); 11000 } 11001 if (!vsi->rss_size) 11002 return -EINVAL; 11003 11004 lut = kzalloc(vsi->rss_table_size, GFP_KERNEL); 11005 if (!lut) 11006 return -ENOMEM; 11007 11008 /* Use user configured lut if there is one, otherwise use default */ 11009 if (vsi->rss_lut_user) 11010 memcpy(lut, vsi->rss_lut_user, vsi->rss_table_size); 11011 else 11012 i40e_fill_rss_lut(pf, lut, vsi->rss_table_size, vsi->rss_size); 11013 11014 /* Use user configured hash key if there is one, otherwise 11015 * use default. 11016 */ 11017 if (vsi->rss_hkey_user) 11018 memcpy(seed, vsi->rss_hkey_user, I40E_HKEY_ARRAY_SIZE); 11019 else 11020 netdev_rss_key_fill((void *)seed, I40E_HKEY_ARRAY_SIZE); 11021 ret = i40e_config_rss(vsi, seed, lut, vsi->rss_table_size); 11022 kfree(lut); 11023 11024 return ret; 11025 } 11026 11027 /** 11028 * i40e_reconfig_rss_queues - change number of queues for rss and rebuild 11029 * @pf: board private structure 11030 * @queue_count: the requested queue count for rss. 11031 * 11032 * returns 0 if rss is not enabled, if enabled returns the final rss queue 11033 * count which may be different from the requested queue count. 11034 * Note: expects to be called while under rtnl_lock() 11035 **/ 11036 int i40e_reconfig_rss_queues(struct i40e_pf *pf, int queue_count) 11037 { 11038 struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi]; 11039 int new_rss_size; 11040 11041 if (!(pf->flags & I40E_FLAG_RSS_ENABLED)) 11042 return 0; 11043 11044 new_rss_size = min_t(int, queue_count, pf->rss_size_max); 11045 11046 if (queue_count != vsi->num_queue_pairs) { 11047 u16 qcount; 11048 11049 vsi->req_queue_pairs = queue_count; 11050 i40e_prep_for_reset(pf, true); 11051 11052 pf->alloc_rss_size = new_rss_size; 11053 11054 i40e_reset_and_rebuild(pf, true, true); 11055 11056 /* Discard the user configured hash keys and lut, if less 11057 * queues are enabled. 11058 */ 11059 if (queue_count < vsi->rss_size) { 11060 i40e_clear_rss_config_user(vsi); 11061 dev_dbg(&pf->pdev->dev, 11062 "discard user configured hash keys and lut\n"); 11063 } 11064 11065 /* Reset vsi->rss_size, as number of enabled queues changed */ 11066 qcount = vsi->num_queue_pairs / vsi->tc_config.numtc; 11067 vsi->rss_size = min_t(int, pf->alloc_rss_size, qcount); 11068 11069 i40e_pf_config_rss(pf); 11070 } 11071 dev_info(&pf->pdev->dev, "User requested queue count/HW max RSS count: %d/%d\n", 11072 vsi->req_queue_pairs, pf->rss_size_max); 11073 return pf->alloc_rss_size; 11074 } 11075 11076 /** 11077 * i40e_get_partition_bw_setting - Retrieve BW settings for this PF partition 11078 * @pf: board private structure 11079 **/ 11080 i40e_status i40e_get_partition_bw_setting(struct i40e_pf *pf) 11081 { 11082 i40e_status status; 11083 bool min_valid, max_valid; 11084 u32 max_bw, min_bw; 11085 11086 status = i40e_read_bw_from_alt_ram(&pf->hw, &max_bw, &min_bw, 11087 &min_valid, &max_valid); 11088 11089 if (!status) { 11090 if (min_valid) 11091 pf->min_bw = min_bw; 11092 if (max_valid) 11093 pf->max_bw = max_bw; 11094 } 11095 11096 return status; 11097 } 11098 11099 /** 11100 * i40e_set_partition_bw_setting - Set BW settings for this PF partition 11101 * @pf: board private structure 11102 **/ 11103 i40e_status i40e_set_partition_bw_setting(struct i40e_pf *pf) 11104 { 11105 struct i40e_aqc_configure_partition_bw_data bw_data; 11106 i40e_status status; 11107 11108 /* Set the valid bit for this PF */ 11109 bw_data.pf_valid_bits = cpu_to_le16(BIT(pf->hw.pf_id)); 11110 bw_data.max_bw[pf->hw.pf_id] = pf->max_bw & I40E_ALT_BW_VALUE_MASK; 11111 bw_data.min_bw[pf->hw.pf_id] = pf->min_bw & I40E_ALT_BW_VALUE_MASK; 11112 11113 /* Set the new bandwidths */ 11114 status = i40e_aq_configure_partition_bw(&pf->hw, &bw_data, NULL); 11115 11116 return status; 11117 } 11118 11119 /** 11120 * i40e_commit_partition_bw_setting - Commit BW settings for this PF partition 11121 * @pf: board private structure 11122 **/ 11123 i40e_status i40e_commit_partition_bw_setting(struct i40e_pf *pf) 11124 { 11125 /* Commit temporary BW setting to permanent NVM image */ 11126 enum i40e_admin_queue_err last_aq_status; 11127 i40e_status ret; 11128 u16 nvm_word; 11129 11130 if (pf->hw.partition_id != 1) { 11131 dev_info(&pf->pdev->dev, 11132 "Commit BW only works on partition 1! This is partition %d", 11133 pf->hw.partition_id); 11134 ret = I40E_NOT_SUPPORTED; 11135 goto bw_commit_out; 11136 } 11137 11138 /* Acquire NVM for read access */ 11139 ret = i40e_acquire_nvm(&pf->hw, I40E_RESOURCE_READ); 11140 last_aq_status = pf->hw.aq.asq_last_status; 11141 if (ret) { 11142 dev_info(&pf->pdev->dev, 11143 "Cannot acquire NVM for read access, err %s aq_err %s\n", 11144 i40e_stat_str(&pf->hw, ret), 11145 i40e_aq_str(&pf->hw, last_aq_status)); 11146 goto bw_commit_out; 11147 } 11148 11149 /* Read word 0x10 of NVM - SW compatibility word 1 */ 11150 ret = i40e_aq_read_nvm(&pf->hw, 11151 I40E_SR_NVM_CONTROL_WORD, 11152 0x10, sizeof(nvm_word), &nvm_word, 11153 false, NULL); 11154 /* Save off last admin queue command status before releasing 11155 * the NVM 11156 */ 11157 last_aq_status = pf->hw.aq.asq_last_status; 11158 i40e_release_nvm(&pf->hw); 11159 if (ret) { 11160 dev_info(&pf->pdev->dev, "NVM read error, err %s aq_err %s\n", 11161 i40e_stat_str(&pf->hw, ret), 11162 i40e_aq_str(&pf->hw, last_aq_status)); 11163 goto bw_commit_out; 11164 } 11165 11166 /* Wait a bit for NVM release to complete */ 11167 msleep(50); 11168 11169 /* Acquire NVM for write access */ 11170 ret = i40e_acquire_nvm(&pf->hw, I40E_RESOURCE_WRITE); 11171 last_aq_status = pf->hw.aq.asq_last_status; 11172 if (ret) { 11173 dev_info(&pf->pdev->dev, 11174 "Cannot acquire NVM for write access, err %s aq_err %s\n", 11175 i40e_stat_str(&pf->hw, ret), 11176 i40e_aq_str(&pf->hw, last_aq_status)); 11177 goto bw_commit_out; 11178 } 11179 /* Write it back out unchanged to initiate update NVM, 11180 * which will force a write of the shadow (alt) RAM to 11181 * the NVM - thus storing the bandwidth values permanently. 11182 */ 11183 ret = i40e_aq_update_nvm(&pf->hw, 11184 I40E_SR_NVM_CONTROL_WORD, 11185 0x10, sizeof(nvm_word), 11186 &nvm_word, true, 0, NULL); 11187 /* Save off last admin queue command status before releasing 11188 * the NVM 11189 */ 11190 last_aq_status = pf->hw.aq.asq_last_status; 11191 i40e_release_nvm(&pf->hw); 11192 if (ret) 11193 dev_info(&pf->pdev->dev, 11194 "BW settings NOT SAVED, err %s aq_err %s\n", 11195 i40e_stat_str(&pf->hw, ret), 11196 i40e_aq_str(&pf->hw, last_aq_status)); 11197 bw_commit_out: 11198 11199 return ret; 11200 } 11201 11202 /** 11203 * i40e_sw_init - Initialize general software structures (struct i40e_pf) 11204 * @pf: board private structure to initialize 11205 * 11206 * i40e_sw_init initializes the Adapter private data structure. 11207 * Fields are initialized based on PCI device information and 11208 * OS network device settings (MTU size). 11209 **/ 11210 static int i40e_sw_init(struct i40e_pf *pf) 11211 { 11212 int err = 0; 11213 int size; 11214 11215 /* Set default capability flags */ 11216 pf->flags = I40E_FLAG_RX_CSUM_ENABLED | 11217 I40E_FLAG_MSI_ENABLED | 11218 I40E_FLAG_MSIX_ENABLED; 11219 11220 /* Set default ITR */ 11221 pf->rx_itr_default = I40E_ITR_RX_DEF; 11222 pf->tx_itr_default = I40E_ITR_TX_DEF; 11223 11224 /* Depending on PF configurations, it is possible that the RSS 11225 * maximum might end up larger than the available queues 11226 */ 11227 pf->rss_size_max = BIT(pf->hw.func_caps.rss_table_entry_width); 11228 pf->alloc_rss_size = 1; 11229 pf->rss_table_size = pf->hw.func_caps.rss_table_size; 11230 pf->rss_size_max = min_t(int, pf->rss_size_max, 11231 pf->hw.func_caps.num_tx_qp); 11232 if (pf->hw.func_caps.rss) { 11233 pf->flags |= I40E_FLAG_RSS_ENABLED; 11234 pf->alloc_rss_size = min_t(int, pf->rss_size_max, 11235 num_online_cpus()); 11236 } 11237 11238 /* MFP mode enabled */ 11239 if (pf->hw.func_caps.npar_enable || pf->hw.func_caps.flex10_enable) { 11240 pf->flags |= I40E_FLAG_MFP_ENABLED; 11241 dev_info(&pf->pdev->dev, "MFP mode Enabled\n"); 11242 if (i40e_get_partition_bw_setting(pf)) { 11243 dev_warn(&pf->pdev->dev, 11244 "Could not get partition bw settings\n"); 11245 } else { 11246 dev_info(&pf->pdev->dev, 11247 "Partition BW Min = %8.8x, Max = %8.8x\n", 11248 pf->min_bw, pf->max_bw); 11249 11250 /* nudge the Tx scheduler */ 11251 i40e_set_partition_bw_setting(pf); 11252 } 11253 } 11254 11255 if ((pf->hw.func_caps.fd_filters_guaranteed > 0) || 11256 (pf->hw.func_caps.fd_filters_best_effort > 0)) { 11257 pf->flags |= I40E_FLAG_FD_ATR_ENABLED; 11258 pf->atr_sample_rate = I40E_DEFAULT_ATR_SAMPLE_RATE; 11259 if (pf->flags & I40E_FLAG_MFP_ENABLED && 11260 pf->hw.num_partitions > 1) 11261 dev_info(&pf->pdev->dev, 11262 "Flow Director Sideband mode Disabled in MFP mode\n"); 11263 else 11264 pf->flags |= I40E_FLAG_FD_SB_ENABLED; 11265 pf->fdir_pf_filter_count = 11266 pf->hw.func_caps.fd_filters_guaranteed; 11267 pf->hw.fdir_shared_filter_count = 11268 pf->hw.func_caps.fd_filters_best_effort; 11269 } 11270 11271 if (pf->hw.mac.type == I40E_MAC_X722) { 11272 pf->hw_features |= (I40E_HW_RSS_AQ_CAPABLE | 11273 I40E_HW_128_QP_RSS_CAPABLE | 11274 I40E_HW_ATR_EVICT_CAPABLE | 11275 I40E_HW_WB_ON_ITR_CAPABLE | 11276 I40E_HW_MULTIPLE_TCP_UDP_RSS_PCTYPE | 11277 I40E_HW_NO_PCI_LINK_CHECK | 11278 I40E_HW_USE_SET_LLDP_MIB | 11279 I40E_HW_GENEVE_OFFLOAD_CAPABLE | 11280 I40E_HW_PTP_L4_CAPABLE | 11281 I40E_HW_WOL_MC_MAGIC_PKT_WAKE | 11282 I40E_HW_OUTER_UDP_CSUM_CAPABLE); 11283 11284 #define I40E_FDEVICT_PCTYPE_DEFAULT 0xc03 11285 if (rd32(&pf->hw, I40E_GLQF_FDEVICTENA(1)) != 11286 I40E_FDEVICT_PCTYPE_DEFAULT) { 11287 dev_warn(&pf->pdev->dev, 11288 "FD EVICT PCTYPES are not right, disable FD HW EVICT\n"); 11289 pf->hw_features &= ~I40E_HW_ATR_EVICT_CAPABLE; 11290 } 11291 } else if ((pf->hw.aq.api_maj_ver > 1) || 11292 ((pf->hw.aq.api_maj_ver == 1) && 11293 (pf->hw.aq.api_min_ver > 4))) { 11294 /* Supported in FW API version higher than 1.4 */ 11295 pf->hw_features |= I40E_HW_GENEVE_OFFLOAD_CAPABLE; 11296 } 11297 11298 /* Enable HW ATR eviction if possible */ 11299 if (pf->hw_features & I40E_HW_ATR_EVICT_CAPABLE) 11300 pf->flags |= I40E_FLAG_HW_ATR_EVICT_ENABLED; 11301 11302 if ((pf->hw.mac.type == I40E_MAC_XL710) && 11303 (((pf->hw.aq.fw_maj_ver == 4) && (pf->hw.aq.fw_min_ver < 33)) || 11304 (pf->hw.aq.fw_maj_ver < 4))) { 11305 pf->hw_features |= I40E_HW_RESTART_AUTONEG; 11306 /* No DCB support for FW < v4.33 */ 11307 pf->hw_features |= I40E_HW_NO_DCB_SUPPORT; 11308 } 11309 11310 /* Disable FW LLDP if FW < v4.3 */ 11311 if ((pf->hw.mac.type == I40E_MAC_XL710) && 11312 (((pf->hw.aq.fw_maj_ver == 4) && (pf->hw.aq.fw_min_ver < 3)) || 11313 (pf->hw.aq.fw_maj_ver < 4))) 11314 pf->hw_features |= I40E_HW_STOP_FW_LLDP; 11315 11316 /* Use the FW Set LLDP MIB API if FW > v4.40 */ 11317 if ((pf->hw.mac.type == I40E_MAC_XL710) && 11318 (((pf->hw.aq.fw_maj_ver == 4) && (pf->hw.aq.fw_min_ver >= 40)) || 11319 (pf->hw.aq.fw_maj_ver >= 5))) 11320 pf->hw_features |= I40E_HW_USE_SET_LLDP_MIB; 11321 11322 /* Enable PTP L4 if FW > v6.0 */ 11323 if (pf->hw.mac.type == I40E_MAC_XL710 && 11324 pf->hw.aq.fw_maj_ver >= 6) 11325 pf->hw_features |= I40E_HW_PTP_L4_CAPABLE; 11326 11327 if (pf->hw.func_caps.vmdq && num_online_cpus() != 1) { 11328 pf->num_vmdq_vsis = I40E_DEFAULT_NUM_VMDQ_VSI; 11329 pf->flags |= I40E_FLAG_VMDQ_ENABLED; 11330 pf->num_vmdq_qps = i40e_default_queues_per_vmdq(pf); 11331 } 11332 11333 if (pf->hw.func_caps.iwarp && num_online_cpus() != 1) { 11334 pf->flags |= I40E_FLAG_IWARP_ENABLED; 11335 /* IWARP needs one extra vector for CQP just like MISC.*/ 11336 pf->num_iwarp_msix = (int)num_online_cpus() + 1; 11337 } 11338 /* Stopping the FW LLDP engine is only supported on the 11339 * XL710 with a FW ver >= 1.7. Also, stopping FW LLDP 11340 * engine is not supported if NPAR is functioning on this 11341 * part 11342 */ 11343 if (pf->hw.mac.type == I40E_MAC_XL710 && 11344 !pf->hw.func_caps.npar_enable && 11345 (pf->hw.aq.api_maj_ver > 1 || 11346 (pf->hw.aq.api_maj_ver == 1 && pf->hw.aq.api_min_ver > 6))) 11347 pf->hw_features |= I40E_HW_STOPPABLE_FW_LLDP; 11348 11349 #ifdef CONFIG_PCI_IOV 11350 if (pf->hw.func_caps.num_vfs && pf->hw.partition_id == 1) { 11351 pf->num_vf_qps = I40E_DEFAULT_QUEUES_PER_VF; 11352 pf->flags |= I40E_FLAG_SRIOV_ENABLED; 11353 pf->num_req_vfs = min_t(int, 11354 pf->hw.func_caps.num_vfs, 11355 I40E_MAX_VF_COUNT); 11356 } 11357 #endif /* CONFIG_PCI_IOV */ 11358 pf->eeprom_version = 0xDEAD; 11359 pf->lan_veb = I40E_NO_VEB; 11360 pf->lan_vsi = I40E_NO_VSI; 11361 11362 /* By default FW has this off for performance reasons */ 11363 pf->flags &= ~I40E_FLAG_VEB_STATS_ENABLED; 11364 11365 /* set up queue assignment tracking */ 11366 size = sizeof(struct i40e_lump_tracking) 11367 + (sizeof(u16) * pf->hw.func_caps.num_tx_qp); 11368 pf->qp_pile = kzalloc(size, GFP_KERNEL); 11369 if (!pf->qp_pile) { 11370 err = -ENOMEM; 11371 goto sw_init_done; 11372 } 11373 pf->qp_pile->num_entries = pf->hw.func_caps.num_tx_qp; 11374 pf->qp_pile->search_hint = 0; 11375 11376 pf->tx_timeout_recovery_level = 1; 11377 11378 mutex_init(&pf->switch_mutex); 11379 11380 sw_init_done: 11381 return err; 11382 } 11383 11384 /** 11385 * i40e_set_ntuple - set the ntuple feature flag and take action 11386 * @pf: board private structure to initialize 11387 * @features: the feature set that the stack is suggesting 11388 * 11389 * returns a bool to indicate if reset needs to happen 11390 **/ 11391 bool i40e_set_ntuple(struct i40e_pf *pf, netdev_features_t features) 11392 { 11393 bool need_reset = false; 11394 11395 /* Check if Flow Director n-tuple support was enabled or disabled. If 11396 * the state changed, we need to reset. 11397 */ 11398 if (features & NETIF_F_NTUPLE) { 11399 /* Enable filters and mark for reset */ 11400 if (!(pf->flags & I40E_FLAG_FD_SB_ENABLED)) 11401 need_reset = true; 11402 /* enable FD_SB only if there is MSI-X vector and no cloud 11403 * filters exist 11404 */ 11405 if (pf->num_fdsb_msix > 0 && !pf->num_cloud_filters) { 11406 pf->flags |= I40E_FLAG_FD_SB_ENABLED; 11407 pf->flags &= ~I40E_FLAG_FD_SB_INACTIVE; 11408 } 11409 } else { 11410 /* turn off filters, mark for reset and clear SW filter list */ 11411 if (pf->flags & I40E_FLAG_FD_SB_ENABLED) { 11412 need_reset = true; 11413 i40e_fdir_filter_exit(pf); 11414 } 11415 pf->flags &= ~I40E_FLAG_FD_SB_ENABLED; 11416 clear_bit(__I40E_FD_SB_AUTO_DISABLED, pf->state); 11417 pf->flags |= I40E_FLAG_FD_SB_INACTIVE; 11418 11419 /* reset fd counters */ 11420 pf->fd_add_err = 0; 11421 pf->fd_atr_cnt = 0; 11422 /* if ATR was auto disabled it can be re-enabled. */ 11423 if (test_and_clear_bit(__I40E_FD_ATR_AUTO_DISABLED, pf->state)) 11424 if ((pf->flags & I40E_FLAG_FD_ATR_ENABLED) && 11425 (I40E_DEBUG_FD & pf->hw.debug_mask)) 11426 dev_info(&pf->pdev->dev, "ATR re-enabled.\n"); 11427 } 11428 return need_reset; 11429 } 11430 11431 /** 11432 * i40e_clear_rss_lut - clear the rx hash lookup table 11433 * @vsi: the VSI being configured 11434 **/ 11435 static void i40e_clear_rss_lut(struct i40e_vsi *vsi) 11436 { 11437 struct i40e_pf *pf = vsi->back; 11438 struct i40e_hw *hw = &pf->hw; 11439 u16 vf_id = vsi->vf_id; 11440 u8 i; 11441 11442 if (vsi->type == I40E_VSI_MAIN) { 11443 for (i = 0; i <= I40E_PFQF_HLUT_MAX_INDEX; i++) 11444 wr32(hw, I40E_PFQF_HLUT(i), 0); 11445 } else if (vsi->type == I40E_VSI_SRIOV) { 11446 for (i = 0; i <= I40E_VFQF_HLUT_MAX_INDEX; i++) 11447 i40e_write_rx_ctl(hw, I40E_VFQF_HLUT1(i, vf_id), 0); 11448 } else { 11449 dev_err(&pf->pdev->dev, "Cannot set RSS LUT - invalid VSI type\n"); 11450 } 11451 } 11452 11453 /** 11454 * i40e_set_features - set the netdev feature flags 11455 * @netdev: ptr to the netdev being adjusted 11456 * @features: the feature set that the stack is suggesting 11457 * Note: expects to be called while under rtnl_lock() 11458 **/ 11459 static int i40e_set_features(struct net_device *netdev, 11460 netdev_features_t features) 11461 { 11462 struct i40e_netdev_priv *np = netdev_priv(netdev); 11463 struct i40e_vsi *vsi = np->vsi; 11464 struct i40e_pf *pf = vsi->back; 11465 bool need_reset; 11466 11467 if (features & NETIF_F_RXHASH && !(netdev->features & NETIF_F_RXHASH)) 11468 i40e_pf_config_rss(pf); 11469 else if (!(features & NETIF_F_RXHASH) && 11470 netdev->features & NETIF_F_RXHASH) 11471 i40e_clear_rss_lut(vsi); 11472 11473 if (features & NETIF_F_HW_VLAN_CTAG_RX) 11474 i40e_vlan_stripping_enable(vsi); 11475 else 11476 i40e_vlan_stripping_disable(vsi); 11477 11478 if (!(features & NETIF_F_HW_TC) && pf->num_cloud_filters) { 11479 dev_err(&pf->pdev->dev, 11480 "Offloaded tc filters active, can't turn hw_tc_offload off"); 11481 return -EINVAL; 11482 } 11483 11484 need_reset = i40e_set_ntuple(pf, features); 11485 11486 if (need_reset) 11487 i40e_do_reset(pf, I40E_PF_RESET_FLAG, true); 11488 11489 return 0; 11490 } 11491 11492 /** 11493 * i40e_get_udp_port_idx - Lookup a possibly offloaded for Rx UDP port 11494 * @pf: board private structure 11495 * @port: The UDP port to look up 11496 * 11497 * Returns the index number or I40E_MAX_PF_UDP_OFFLOAD_PORTS if port not found 11498 **/ 11499 static u8 i40e_get_udp_port_idx(struct i40e_pf *pf, u16 port) 11500 { 11501 u8 i; 11502 11503 for (i = 0; i < I40E_MAX_PF_UDP_OFFLOAD_PORTS; i++) { 11504 /* Do not report ports with pending deletions as 11505 * being available. 11506 */ 11507 if (!port && (pf->pending_udp_bitmap & BIT_ULL(i))) 11508 continue; 11509 if (pf->udp_ports[i].port == port) 11510 return i; 11511 } 11512 11513 return i; 11514 } 11515 11516 /** 11517 * i40e_udp_tunnel_add - Get notifications about UDP tunnel ports that come up 11518 * @netdev: This physical port's netdev 11519 * @ti: Tunnel endpoint information 11520 **/ 11521 static void i40e_udp_tunnel_add(struct net_device *netdev, 11522 struct udp_tunnel_info *ti) 11523 { 11524 struct i40e_netdev_priv *np = netdev_priv(netdev); 11525 struct i40e_vsi *vsi = np->vsi; 11526 struct i40e_pf *pf = vsi->back; 11527 u16 port = ntohs(ti->port); 11528 u8 next_idx; 11529 u8 idx; 11530 11531 idx = i40e_get_udp_port_idx(pf, port); 11532 11533 /* Check if port already exists */ 11534 if (idx < I40E_MAX_PF_UDP_OFFLOAD_PORTS) { 11535 netdev_info(netdev, "port %d already offloaded\n", port); 11536 return; 11537 } 11538 11539 /* Now check if there is space to add the new port */ 11540 next_idx = i40e_get_udp_port_idx(pf, 0); 11541 11542 if (next_idx == I40E_MAX_PF_UDP_OFFLOAD_PORTS) { 11543 netdev_info(netdev, "maximum number of offloaded UDP ports reached, not adding port %d\n", 11544 port); 11545 return; 11546 } 11547 11548 switch (ti->type) { 11549 case UDP_TUNNEL_TYPE_VXLAN: 11550 pf->udp_ports[next_idx].type = I40E_AQC_TUNNEL_TYPE_VXLAN; 11551 break; 11552 case UDP_TUNNEL_TYPE_GENEVE: 11553 if (!(pf->hw_features & I40E_HW_GENEVE_OFFLOAD_CAPABLE)) 11554 return; 11555 pf->udp_ports[next_idx].type = I40E_AQC_TUNNEL_TYPE_NGE; 11556 break; 11557 default: 11558 return; 11559 } 11560 11561 /* New port: add it and mark its index in the bitmap */ 11562 pf->udp_ports[next_idx].port = port; 11563 pf->udp_ports[next_idx].filter_index = I40E_UDP_PORT_INDEX_UNUSED; 11564 pf->pending_udp_bitmap |= BIT_ULL(next_idx); 11565 set_bit(__I40E_UDP_FILTER_SYNC_PENDING, pf->state); 11566 } 11567 11568 /** 11569 * i40e_udp_tunnel_del - Get notifications about UDP tunnel ports that go away 11570 * @netdev: This physical port's netdev 11571 * @ti: Tunnel endpoint information 11572 **/ 11573 static void i40e_udp_tunnel_del(struct net_device *netdev, 11574 struct udp_tunnel_info *ti) 11575 { 11576 struct i40e_netdev_priv *np = netdev_priv(netdev); 11577 struct i40e_vsi *vsi = np->vsi; 11578 struct i40e_pf *pf = vsi->back; 11579 u16 port = ntohs(ti->port); 11580 u8 idx; 11581 11582 idx = i40e_get_udp_port_idx(pf, port); 11583 11584 /* Check if port already exists */ 11585 if (idx >= I40E_MAX_PF_UDP_OFFLOAD_PORTS) 11586 goto not_found; 11587 11588 switch (ti->type) { 11589 case UDP_TUNNEL_TYPE_VXLAN: 11590 if (pf->udp_ports[idx].type != I40E_AQC_TUNNEL_TYPE_VXLAN) 11591 goto not_found; 11592 break; 11593 case UDP_TUNNEL_TYPE_GENEVE: 11594 if (pf->udp_ports[idx].type != I40E_AQC_TUNNEL_TYPE_NGE) 11595 goto not_found; 11596 break; 11597 default: 11598 goto not_found; 11599 } 11600 11601 /* if port exists, set it to 0 (mark for deletion) 11602 * and make it pending 11603 */ 11604 pf->udp_ports[idx].port = 0; 11605 11606 /* Toggle pending bit instead of setting it. This way if we are 11607 * deleting a port that has yet to be added we just clear the pending 11608 * bit and don't have to worry about it. 11609 */ 11610 pf->pending_udp_bitmap ^= BIT_ULL(idx); 11611 set_bit(__I40E_UDP_FILTER_SYNC_PENDING, pf->state); 11612 11613 return; 11614 not_found: 11615 netdev_warn(netdev, "UDP port %d was not found, not deleting\n", 11616 port); 11617 } 11618 11619 static int i40e_get_phys_port_id(struct net_device *netdev, 11620 struct netdev_phys_item_id *ppid) 11621 { 11622 struct i40e_netdev_priv *np = netdev_priv(netdev); 11623 struct i40e_pf *pf = np->vsi->back; 11624 struct i40e_hw *hw = &pf->hw; 11625 11626 if (!(pf->hw_features & I40E_HW_PORT_ID_VALID)) 11627 return -EOPNOTSUPP; 11628 11629 ppid->id_len = min_t(int, sizeof(hw->mac.port_addr), sizeof(ppid->id)); 11630 memcpy(ppid->id, hw->mac.port_addr, ppid->id_len); 11631 11632 return 0; 11633 } 11634 11635 /** 11636 * i40e_ndo_fdb_add - add an entry to the hardware database 11637 * @ndm: the input from the stack 11638 * @tb: pointer to array of nladdr (unused) 11639 * @dev: the net device pointer 11640 * @addr: the MAC address entry being added 11641 * @vid: VLAN ID 11642 * @flags: instructions from stack about fdb operation 11643 */ 11644 static int i40e_ndo_fdb_add(struct ndmsg *ndm, struct nlattr *tb[], 11645 struct net_device *dev, 11646 const unsigned char *addr, u16 vid, 11647 u16 flags) 11648 { 11649 struct i40e_netdev_priv *np = netdev_priv(dev); 11650 struct i40e_pf *pf = np->vsi->back; 11651 int err = 0; 11652 11653 if (!(pf->flags & I40E_FLAG_SRIOV_ENABLED)) 11654 return -EOPNOTSUPP; 11655 11656 if (vid) { 11657 pr_info("%s: vlans aren't supported yet for dev_uc|mc_add()\n", dev->name); 11658 return -EINVAL; 11659 } 11660 11661 /* Hardware does not support aging addresses so if a 11662 * ndm_state is given only allow permanent addresses 11663 */ 11664 if (ndm->ndm_state && !(ndm->ndm_state & NUD_PERMANENT)) { 11665 netdev_info(dev, "FDB only supports static addresses\n"); 11666 return -EINVAL; 11667 } 11668 11669 if (is_unicast_ether_addr(addr) || is_link_local_ether_addr(addr)) 11670 err = dev_uc_add_excl(dev, addr); 11671 else if (is_multicast_ether_addr(addr)) 11672 err = dev_mc_add_excl(dev, addr); 11673 else 11674 err = -EINVAL; 11675 11676 /* Only return duplicate errors if NLM_F_EXCL is set */ 11677 if (err == -EEXIST && !(flags & NLM_F_EXCL)) 11678 err = 0; 11679 11680 return err; 11681 } 11682 11683 /** 11684 * i40e_ndo_bridge_setlink - Set the hardware bridge mode 11685 * @dev: the netdev being configured 11686 * @nlh: RTNL message 11687 * @flags: bridge flags 11688 * 11689 * Inserts a new hardware bridge if not already created and 11690 * enables the bridging mode requested (VEB or VEPA). If the 11691 * hardware bridge has already been inserted and the request 11692 * is to change the mode then that requires a PF reset to 11693 * allow rebuild of the components with required hardware 11694 * bridge mode enabled. 11695 * 11696 * Note: expects to be called while under rtnl_lock() 11697 **/ 11698 static int i40e_ndo_bridge_setlink(struct net_device *dev, 11699 struct nlmsghdr *nlh, 11700 u16 flags) 11701 { 11702 struct i40e_netdev_priv *np = netdev_priv(dev); 11703 struct i40e_vsi *vsi = np->vsi; 11704 struct i40e_pf *pf = vsi->back; 11705 struct i40e_veb *veb = NULL; 11706 struct nlattr *attr, *br_spec; 11707 int i, rem; 11708 11709 /* Only for PF VSI for now */ 11710 if (vsi->seid != pf->vsi[pf->lan_vsi]->seid) 11711 return -EOPNOTSUPP; 11712 11713 /* Find the HW bridge for PF VSI */ 11714 for (i = 0; i < I40E_MAX_VEB && !veb; i++) { 11715 if (pf->veb[i] && pf->veb[i]->seid == vsi->uplink_seid) 11716 veb = pf->veb[i]; 11717 } 11718 11719 br_spec = nlmsg_find_attr(nlh, sizeof(struct ifinfomsg), IFLA_AF_SPEC); 11720 11721 nla_for_each_nested(attr, br_spec, rem) { 11722 __u16 mode; 11723 11724 if (nla_type(attr) != IFLA_BRIDGE_MODE) 11725 continue; 11726 11727 mode = nla_get_u16(attr); 11728 if ((mode != BRIDGE_MODE_VEPA) && 11729 (mode != BRIDGE_MODE_VEB)) 11730 return -EINVAL; 11731 11732 /* Insert a new HW bridge */ 11733 if (!veb) { 11734 veb = i40e_veb_setup(pf, 0, vsi->uplink_seid, vsi->seid, 11735 vsi->tc_config.enabled_tc); 11736 if (veb) { 11737 veb->bridge_mode = mode; 11738 i40e_config_bridge_mode(veb); 11739 } else { 11740 /* No Bridge HW offload available */ 11741 return -ENOENT; 11742 } 11743 break; 11744 } else if (mode != veb->bridge_mode) { 11745 /* Existing HW bridge but different mode needs reset */ 11746 veb->bridge_mode = mode; 11747 /* TODO: If no VFs or VMDq VSIs, disallow VEB mode */ 11748 if (mode == BRIDGE_MODE_VEB) 11749 pf->flags |= I40E_FLAG_VEB_MODE_ENABLED; 11750 else 11751 pf->flags &= ~I40E_FLAG_VEB_MODE_ENABLED; 11752 i40e_do_reset(pf, I40E_PF_RESET_FLAG, true); 11753 break; 11754 } 11755 } 11756 11757 return 0; 11758 } 11759 11760 /** 11761 * i40e_ndo_bridge_getlink - Get the hardware bridge mode 11762 * @skb: skb buff 11763 * @pid: process id 11764 * @seq: RTNL message seq # 11765 * @dev: the netdev being configured 11766 * @filter_mask: unused 11767 * @nlflags: netlink flags passed in 11768 * 11769 * Return the mode in which the hardware bridge is operating in 11770 * i.e VEB or VEPA. 11771 **/ 11772 static int i40e_ndo_bridge_getlink(struct sk_buff *skb, u32 pid, u32 seq, 11773 struct net_device *dev, 11774 u32 __always_unused filter_mask, 11775 int nlflags) 11776 { 11777 struct i40e_netdev_priv *np = netdev_priv(dev); 11778 struct i40e_vsi *vsi = np->vsi; 11779 struct i40e_pf *pf = vsi->back; 11780 struct i40e_veb *veb = NULL; 11781 int i; 11782 11783 /* Only for PF VSI for now */ 11784 if (vsi->seid != pf->vsi[pf->lan_vsi]->seid) 11785 return -EOPNOTSUPP; 11786 11787 /* Find the HW bridge for the PF VSI */ 11788 for (i = 0; i < I40E_MAX_VEB && !veb; i++) { 11789 if (pf->veb[i] && pf->veb[i]->seid == vsi->uplink_seid) 11790 veb = pf->veb[i]; 11791 } 11792 11793 if (!veb) 11794 return 0; 11795 11796 return ndo_dflt_bridge_getlink(skb, pid, seq, dev, veb->bridge_mode, 11797 0, 0, nlflags, filter_mask, NULL); 11798 } 11799 11800 /** 11801 * i40e_features_check - Validate encapsulated packet conforms to limits 11802 * @skb: skb buff 11803 * @dev: This physical port's netdev 11804 * @features: Offload features that the stack believes apply 11805 **/ 11806 static netdev_features_t i40e_features_check(struct sk_buff *skb, 11807 struct net_device *dev, 11808 netdev_features_t features) 11809 { 11810 size_t len; 11811 11812 /* No point in doing any of this if neither checksum nor GSO are 11813 * being requested for this frame. We can rule out both by just 11814 * checking for CHECKSUM_PARTIAL 11815 */ 11816 if (skb->ip_summed != CHECKSUM_PARTIAL) 11817 return features; 11818 11819 /* We cannot support GSO if the MSS is going to be less than 11820 * 64 bytes. If it is then we need to drop support for GSO. 11821 */ 11822 if (skb_is_gso(skb) && (skb_shinfo(skb)->gso_size < 64)) 11823 features &= ~NETIF_F_GSO_MASK; 11824 11825 /* MACLEN can support at most 63 words */ 11826 len = skb_network_header(skb) - skb->data; 11827 if (len & ~(63 * 2)) 11828 goto out_err; 11829 11830 /* IPLEN and EIPLEN can support at most 127 dwords */ 11831 len = skb_transport_header(skb) - skb_network_header(skb); 11832 if (len & ~(127 * 4)) 11833 goto out_err; 11834 11835 if (skb->encapsulation) { 11836 /* L4TUNLEN can support 127 words */ 11837 len = skb_inner_network_header(skb) - skb_transport_header(skb); 11838 if (len & ~(127 * 2)) 11839 goto out_err; 11840 11841 /* IPLEN can support at most 127 dwords */ 11842 len = skb_inner_transport_header(skb) - 11843 skb_inner_network_header(skb); 11844 if (len & ~(127 * 4)) 11845 goto out_err; 11846 } 11847 11848 /* No need to validate L4LEN as TCP is the only protocol with a 11849 * a flexible value and we support all possible values supported 11850 * by TCP, which is at most 15 dwords 11851 */ 11852 11853 return features; 11854 out_err: 11855 return features & ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK); 11856 } 11857 11858 /** 11859 * i40e_xdp_setup - add/remove an XDP program 11860 * @vsi: VSI to changed 11861 * @prog: XDP program 11862 **/ 11863 static int i40e_xdp_setup(struct i40e_vsi *vsi, 11864 struct bpf_prog *prog) 11865 { 11866 int frame_size = vsi->netdev->mtu + ETH_HLEN + ETH_FCS_LEN + VLAN_HLEN; 11867 struct i40e_pf *pf = vsi->back; 11868 struct bpf_prog *old_prog; 11869 bool need_reset; 11870 int i; 11871 11872 /* Don't allow frames that span over multiple buffers */ 11873 if (frame_size > vsi->rx_buf_len) 11874 return -EINVAL; 11875 11876 if (!i40e_enabled_xdp_vsi(vsi) && !prog) 11877 return 0; 11878 11879 /* When turning XDP on->off/off->on we reset and rebuild the rings. */ 11880 need_reset = (i40e_enabled_xdp_vsi(vsi) != !!prog); 11881 11882 if (need_reset) 11883 i40e_prep_for_reset(pf, true); 11884 11885 old_prog = xchg(&vsi->xdp_prog, prog); 11886 11887 if (need_reset) 11888 i40e_reset_and_rebuild(pf, true, true); 11889 11890 for (i = 0; i < vsi->num_queue_pairs; i++) 11891 WRITE_ONCE(vsi->rx_rings[i]->xdp_prog, vsi->xdp_prog); 11892 11893 if (old_prog) 11894 bpf_prog_put(old_prog); 11895 11896 return 0; 11897 } 11898 11899 /** 11900 * i40e_enter_busy_conf - Enters busy config state 11901 * @vsi: vsi 11902 * 11903 * Returns 0 on success, <0 for failure. 11904 **/ 11905 static int i40e_enter_busy_conf(struct i40e_vsi *vsi) 11906 { 11907 struct i40e_pf *pf = vsi->back; 11908 int timeout = 50; 11909 11910 while (test_and_set_bit(__I40E_CONFIG_BUSY, pf->state)) { 11911 timeout--; 11912 if (!timeout) 11913 return -EBUSY; 11914 usleep_range(1000, 2000); 11915 } 11916 11917 return 0; 11918 } 11919 11920 /** 11921 * i40e_exit_busy_conf - Exits busy config state 11922 * @vsi: vsi 11923 **/ 11924 static void i40e_exit_busy_conf(struct i40e_vsi *vsi) 11925 { 11926 struct i40e_pf *pf = vsi->back; 11927 11928 clear_bit(__I40E_CONFIG_BUSY, pf->state); 11929 } 11930 11931 /** 11932 * i40e_queue_pair_reset_stats - Resets all statistics for a queue pair 11933 * @vsi: vsi 11934 * @queue_pair: queue pair 11935 **/ 11936 static void i40e_queue_pair_reset_stats(struct i40e_vsi *vsi, int queue_pair) 11937 { 11938 memset(&vsi->rx_rings[queue_pair]->rx_stats, 0, 11939 sizeof(vsi->rx_rings[queue_pair]->rx_stats)); 11940 memset(&vsi->tx_rings[queue_pair]->stats, 0, 11941 sizeof(vsi->tx_rings[queue_pair]->stats)); 11942 if (i40e_enabled_xdp_vsi(vsi)) { 11943 memset(&vsi->xdp_rings[queue_pair]->stats, 0, 11944 sizeof(vsi->xdp_rings[queue_pair]->stats)); 11945 } 11946 } 11947 11948 /** 11949 * i40e_queue_pair_clean_rings - Cleans all the rings of a queue pair 11950 * @vsi: vsi 11951 * @queue_pair: queue pair 11952 **/ 11953 static void i40e_queue_pair_clean_rings(struct i40e_vsi *vsi, int queue_pair) 11954 { 11955 i40e_clean_tx_ring(vsi->tx_rings[queue_pair]); 11956 if (i40e_enabled_xdp_vsi(vsi)) 11957 i40e_clean_tx_ring(vsi->xdp_rings[queue_pair]); 11958 i40e_clean_rx_ring(vsi->rx_rings[queue_pair]); 11959 } 11960 11961 /** 11962 * i40e_queue_pair_toggle_napi - Enables/disables NAPI for a queue pair 11963 * @vsi: vsi 11964 * @queue_pair: queue pair 11965 * @enable: true for enable, false for disable 11966 **/ 11967 static void i40e_queue_pair_toggle_napi(struct i40e_vsi *vsi, int queue_pair, 11968 bool enable) 11969 { 11970 struct i40e_ring *rxr = vsi->rx_rings[queue_pair]; 11971 struct i40e_q_vector *q_vector = rxr->q_vector; 11972 11973 if (!vsi->netdev) 11974 return; 11975 11976 /* All rings in a qp belong to the same qvector. */ 11977 if (q_vector->rx.ring || q_vector->tx.ring) { 11978 if (enable) 11979 napi_enable(&q_vector->napi); 11980 else 11981 napi_disable(&q_vector->napi); 11982 } 11983 } 11984 11985 /** 11986 * i40e_queue_pair_toggle_rings - Enables/disables all rings for a queue pair 11987 * @vsi: vsi 11988 * @queue_pair: queue pair 11989 * @enable: true for enable, false for disable 11990 * 11991 * Returns 0 on success, <0 on failure. 11992 **/ 11993 static int i40e_queue_pair_toggle_rings(struct i40e_vsi *vsi, int queue_pair, 11994 bool enable) 11995 { 11996 struct i40e_pf *pf = vsi->back; 11997 int pf_q, ret = 0; 11998 11999 pf_q = vsi->base_queue + queue_pair; 12000 ret = i40e_control_wait_tx_q(vsi->seid, pf, pf_q, 12001 false /*is xdp*/, enable); 12002 if (ret) { 12003 dev_info(&pf->pdev->dev, 12004 "VSI seid %d Tx ring %d %sable timeout\n", 12005 vsi->seid, pf_q, (enable ? "en" : "dis")); 12006 return ret; 12007 } 12008 12009 i40e_control_rx_q(pf, pf_q, enable); 12010 ret = i40e_pf_rxq_wait(pf, pf_q, enable); 12011 if (ret) { 12012 dev_info(&pf->pdev->dev, 12013 "VSI seid %d Rx ring %d %sable timeout\n", 12014 vsi->seid, pf_q, (enable ? "en" : "dis")); 12015 return ret; 12016 } 12017 12018 /* Due to HW errata, on Rx disable only, the register can 12019 * indicate done before it really is. Needs 50ms to be sure 12020 */ 12021 if (!enable) 12022 mdelay(50); 12023 12024 if (!i40e_enabled_xdp_vsi(vsi)) 12025 return ret; 12026 12027 ret = i40e_control_wait_tx_q(vsi->seid, pf, 12028 pf_q + vsi->alloc_queue_pairs, 12029 true /*is xdp*/, enable); 12030 if (ret) { 12031 dev_info(&pf->pdev->dev, 12032 "VSI seid %d XDP Tx ring %d %sable timeout\n", 12033 vsi->seid, pf_q, (enable ? "en" : "dis")); 12034 } 12035 12036 return ret; 12037 } 12038 12039 /** 12040 * i40e_queue_pair_enable_irq - Enables interrupts for a queue pair 12041 * @vsi: vsi 12042 * @queue_pair: queue_pair 12043 **/ 12044 static void i40e_queue_pair_enable_irq(struct i40e_vsi *vsi, int queue_pair) 12045 { 12046 struct i40e_ring *rxr = vsi->rx_rings[queue_pair]; 12047 struct i40e_pf *pf = vsi->back; 12048 struct i40e_hw *hw = &pf->hw; 12049 12050 /* All rings in a qp belong to the same qvector. */ 12051 if (pf->flags & I40E_FLAG_MSIX_ENABLED) 12052 i40e_irq_dynamic_enable(vsi, rxr->q_vector->v_idx); 12053 else 12054 i40e_irq_dynamic_enable_icr0(pf); 12055 12056 i40e_flush(hw); 12057 } 12058 12059 /** 12060 * i40e_queue_pair_disable_irq - Disables interrupts for a queue pair 12061 * @vsi: vsi 12062 * @queue_pair: queue_pair 12063 **/ 12064 static void i40e_queue_pair_disable_irq(struct i40e_vsi *vsi, int queue_pair) 12065 { 12066 struct i40e_ring *rxr = vsi->rx_rings[queue_pair]; 12067 struct i40e_pf *pf = vsi->back; 12068 struct i40e_hw *hw = &pf->hw; 12069 12070 /* For simplicity, instead of removing the qp interrupt causes 12071 * from the interrupt linked list, we simply disable the interrupt, and 12072 * leave the list intact. 12073 * 12074 * All rings in a qp belong to the same qvector. 12075 */ 12076 if (pf->flags & I40E_FLAG_MSIX_ENABLED) { 12077 u32 intpf = vsi->base_vector + rxr->q_vector->v_idx; 12078 12079 wr32(hw, I40E_PFINT_DYN_CTLN(intpf - 1), 0); 12080 i40e_flush(hw); 12081 synchronize_irq(pf->msix_entries[intpf].vector); 12082 } else { 12083 /* Legacy and MSI mode - this stops all interrupt handling */ 12084 wr32(hw, I40E_PFINT_ICR0_ENA, 0); 12085 wr32(hw, I40E_PFINT_DYN_CTL0, 0); 12086 i40e_flush(hw); 12087 synchronize_irq(pf->pdev->irq); 12088 } 12089 } 12090 12091 /** 12092 * i40e_queue_pair_disable - Disables a queue pair 12093 * @vsi: vsi 12094 * @queue_pair: queue pair 12095 * 12096 * Returns 0 on success, <0 on failure. 12097 **/ 12098 int i40e_queue_pair_disable(struct i40e_vsi *vsi, int queue_pair) 12099 { 12100 int err; 12101 12102 err = i40e_enter_busy_conf(vsi); 12103 if (err) 12104 return err; 12105 12106 i40e_queue_pair_disable_irq(vsi, queue_pair); 12107 err = i40e_queue_pair_toggle_rings(vsi, queue_pair, false /* off */); 12108 i40e_queue_pair_toggle_napi(vsi, queue_pair, false /* off */); 12109 i40e_queue_pair_clean_rings(vsi, queue_pair); 12110 i40e_queue_pair_reset_stats(vsi, queue_pair); 12111 12112 return err; 12113 } 12114 12115 /** 12116 * i40e_queue_pair_enable - Enables a queue pair 12117 * @vsi: vsi 12118 * @queue_pair: queue pair 12119 * 12120 * Returns 0 on success, <0 on failure. 12121 **/ 12122 int i40e_queue_pair_enable(struct i40e_vsi *vsi, int queue_pair) 12123 { 12124 int err; 12125 12126 err = i40e_configure_tx_ring(vsi->tx_rings[queue_pair]); 12127 if (err) 12128 return err; 12129 12130 if (i40e_enabled_xdp_vsi(vsi)) { 12131 err = i40e_configure_tx_ring(vsi->xdp_rings[queue_pair]); 12132 if (err) 12133 return err; 12134 } 12135 12136 err = i40e_configure_rx_ring(vsi->rx_rings[queue_pair]); 12137 if (err) 12138 return err; 12139 12140 err = i40e_queue_pair_toggle_rings(vsi, queue_pair, true /* on */); 12141 i40e_queue_pair_toggle_napi(vsi, queue_pair, true /* on */); 12142 i40e_queue_pair_enable_irq(vsi, queue_pair); 12143 12144 i40e_exit_busy_conf(vsi); 12145 12146 return err; 12147 } 12148 12149 /** 12150 * i40e_xdp - implements ndo_bpf for i40e 12151 * @dev: netdevice 12152 * @xdp: XDP command 12153 **/ 12154 static int i40e_xdp(struct net_device *dev, 12155 struct netdev_bpf *xdp) 12156 { 12157 struct i40e_netdev_priv *np = netdev_priv(dev); 12158 struct i40e_vsi *vsi = np->vsi; 12159 12160 if (vsi->type != I40E_VSI_MAIN) 12161 return -EINVAL; 12162 12163 switch (xdp->command) { 12164 case XDP_SETUP_PROG: 12165 return i40e_xdp_setup(vsi, xdp->prog); 12166 case XDP_QUERY_PROG: 12167 xdp->prog_id = vsi->xdp_prog ? vsi->xdp_prog->aux->id : 0; 12168 return 0; 12169 case XDP_QUERY_XSK_UMEM: 12170 return i40e_xsk_umem_query(vsi, &xdp->xsk.umem, 12171 xdp->xsk.queue_id); 12172 case XDP_SETUP_XSK_UMEM: 12173 return i40e_xsk_umem_setup(vsi, xdp->xsk.umem, 12174 xdp->xsk.queue_id); 12175 default: 12176 return -EINVAL; 12177 } 12178 } 12179 12180 static const struct net_device_ops i40e_netdev_ops = { 12181 .ndo_open = i40e_open, 12182 .ndo_stop = i40e_close, 12183 .ndo_start_xmit = i40e_lan_xmit_frame, 12184 .ndo_get_stats64 = i40e_get_netdev_stats_struct, 12185 .ndo_set_rx_mode = i40e_set_rx_mode, 12186 .ndo_validate_addr = eth_validate_addr, 12187 .ndo_set_mac_address = i40e_set_mac, 12188 .ndo_change_mtu = i40e_change_mtu, 12189 .ndo_do_ioctl = i40e_ioctl, 12190 .ndo_tx_timeout = i40e_tx_timeout, 12191 .ndo_vlan_rx_add_vid = i40e_vlan_rx_add_vid, 12192 .ndo_vlan_rx_kill_vid = i40e_vlan_rx_kill_vid, 12193 #ifdef CONFIG_NET_POLL_CONTROLLER 12194 .ndo_poll_controller = i40e_netpoll, 12195 #endif 12196 .ndo_setup_tc = __i40e_setup_tc, 12197 .ndo_set_features = i40e_set_features, 12198 .ndo_set_vf_mac = i40e_ndo_set_vf_mac, 12199 .ndo_set_vf_vlan = i40e_ndo_set_vf_port_vlan, 12200 .ndo_set_vf_rate = i40e_ndo_set_vf_bw, 12201 .ndo_get_vf_config = i40e_ndo_get_vf_config, 12202 .ndo_set_vf_link_state = i40e_ndo_set_vf_link_state, 12203 .ndo_set_vf_spoofchk = i40e_ndo_set_vf_spoofchk, 12204 .ndo_set_vf_trust = i40e_ndo_set_vf_trust, 12205 .ndo_udp_tunnel_add = i40e_udp_tunnel_add, 12206 .ndo_udp_tunnel_del = i40e_udp_tunnel_del, 12207 .ndo_get_phys_port_id = i40e_get_phys_port_id, 12208 .ndo_fdb_add = i40e_ndo_fdb_add, 12209 .ndo_features_check = i40e_features_check, 12210 .ndo_bridge_getlink = i40e_ndo_bridge_getlink, 12211 .ndo_bridge_setlink = i40e_ndo_bridge_setlink, 12212 .ndo_bpf = i40e_xdp, 12213 .ndo_xdp_xmit = i40e_xdp_xmit, 12214 .ndo_xsk_async_xmit = i40e_xsk_async_xmit, 12215 }; 12216 12217 /** 12218 * i40e_config_netdev - Setup the netdev flags 12219 * @vsi: the VSI being configured 12220 * 12221 * Returns 0 on success, negative value on failure 12222 **/ 12223 static int i40e_config_netdev(struct i40e_vsi *vsi) 12224 { 12225 struct i40e_pf *pf = vsi->back; 12226 struct i40e_hw *hw = &pf->hw; 12227 struct i40e_netdev_priv *np; 12228 struct net_device *netdev; 12229 u8 broadcast[ETH_ALEN]; 12230 u8 mac_addr[ETH_ALEN]; 12231 int etherdev_size; 12232 netdev_features_t hw_enc_features; 12233 netdev_features_t hw_features; 12234 12235 etherdev_size = sizeof(struct i40e_netdev_priv); 12236 netdev = alloc_etherdev_mq(etherdev_size, vsi->alloc_queue_pairs); 12237 if (!netdev) 12238 return -ENOMEM; 12239 12240 vsi->netdev = netdev; 12241 np = netdev_priv(netdev); 12242 np->vsi = vsi; 12243 12244 hw_enc_features = NETIF_F_SG | 12245 NETIF_F_IP_CSUM | 12246 NETIF_F_IPV6_CSUM | 12247 NETIF_F_HIGHDMA | 12248 NETIF_F_SOFT_FEATURES | 12249 NETIF_F_TSO | 12250 NETIF_F_TSO_ECN | 12251 NETIF_F_TSO6 | 12252 NETIF_F_GSO_GRE | 12253 NETIF_F_GSO_GRE_CSUM | 12254 NETIF_F_GSO_PARTIAL | 12255 NETIF_F_GSO_UDP_TUNNEL | 12256 NETIF_F_GSO_UDP_TUNNEL_CSUM | 12257 NETIF_F_SCTP_CRC | 12258 NETIF_F_RXHASH | 12259 NETIF_F_RXCSUM | 12260 0; 12261 12262 if (!(pf->hw_features & I40E_HW_OUTER_UDP_CSUM_CAPABLE)) 12263 netdev->gso_partial_features |= NETIF_F_GSO_UDP_TUNNEL_CSUM; 12264 12265 netdev->gso_partial_features |= NETIF_F_GSO_GRE_CSUM; 12266 12267 netdev->hw_enc_features |= hw_enc_features; 12268 12269 /* record features VLANs can make use of */ 12270 netdev->vlan_features |= hw_enc_features | NETIF_F_TSO_MANGLEID; 12271 12272 if (!(pf->flags & I40E_FLAG_MFP_ENABLED)) 12273 netdev->hw_features |= NETIF_F_NTUPLE | NETIF_F_HW_TC; 12274 12275 hw_features = hw_enc_features | 12276 NETIF_F_HW_VLAN_CTAG_TX | 12277 NETIF_F_HW_VLAN_CTAG_RX; 12278 12279 netdev->hw_features |= hw_features; 12280 12281 netdev->features |= hw_features | NETIF_F_HW_VLAN_CTAG_FILTER; 12282 netdev->hw_enc_features |= NETIF_F_TSO_MANGLEID; 12283 12284 if (vsi->type == I40E_VSI_MAIN) { 12285 SET_NETDEV_DEV(netdev, &pf->pdev->dev); 12286 ether_addr_copy(mac_addr, hw->mac.perm_addr); 12287 /* The following steps are necessary for two reasons. First, 12288 * some older NVM configurations load a default MAC-VLAN 12289 * filter that will accept any tagged packet, and we want to 12290 * replace this with a normal filter. Additionally, it is 12291 * possible our MAC address was provided by the platform using 12292 * Open Firmware or similar. 12293 * 12294 * Thus, we need to remove the default filter and install one 12295 * specific to the MAC address. 12296 */ 12297 i40e_rm_default_mac_filter(vsi, mac_addr); 12298 spin_lock_bh(&vsi->mac_filter_hash_lock); 12299 i40e_add_mac_filter(vsi, mac_addr); 12300 spin_unlock_bh(&vsi->mac_filter_hash_lock); 12301 } else { 12302 /* Relate the VSI_VMDQ name to the VSI_MAIN name. Note that we 12303 * are still limited by IFNAMSIZ, but we're adding 'v%d\0' to 12304 * the end, which is 4 bytes long, so force truncation of the 12305 * original name by IFNAMSIZ - 4 12306 */ 12307 snprintf(netdev->name, IFNAMSIZ, "%.*sv%%d", 12308 IFNAMSIZ - 4, 12309 pf->vsi[pf->lan_vsi]->netdev->name); 12310 eth_random_addr(mac_addr); 12311 12312 spin_lock_bh(&vsi->mac_filter_hash_lock); 12313 i40e_add_mac_filter(vsi, mac_addr); 12314 spin_unlock_bh(&vsi->mac_filter_hash_lock); 12315 } 12316 12317 /* Add the broadcast filter so that we initially will receive 12318 * broadcast packets. Note that when a new VLAN is first added the 12319 * driver will convert all filters marked I40E_VLAN_ANY into VLAN 12320 * specific filters as part of transitioning into "vlan" operation. 12321 * When more VLANs are added, the driver will copy each existing MAC 12322 * filter and add it for the new VLAN. 12323 * 12324 * Broadcast filters are handled specially by 12325 * i40e_sync_filters_subtask, as the driver must to set the broadcast 12326 * promiscuous bit instead of adding this directly as a MAC/VLAN 12327 * filter. The subtask will update the correct broadcast promiscuous 12328 * bits as VLANs become active or inactive. 12329 */ 12330 eth_broadcast_addr(broadcast); 12331 spin_lock_bh(&vsi->mac_filter_hash_lock); 12332 i40e_add_mac_filter(vsi, broadcast); 12333 spin_unlock_bh(&vsi->mac_filter_hash_lock); 12334 12335 ether_addr_copy(netdev->dev_addr, mac_addr); 12336 ether_addr_copy(netdev->perm_addr, mac_addr); 12337 12338 netdev->priv_flags |= IFF_UNICAST_FLT; 12339 netdev->priv_flags |= IFF_SUPP_NOFCS; 12340 /* Setup netdev TC information */ 12341 i40e_vsi_config_netdev_tc(vsi, vsi->tc_config.enabled_tc); 12342 12343 netdev->netdev_ops = &i40e_netdev_ops; 12344 netdev->watchdog_timeo = 5 * HZ; 12345 i40e_set_ethtool_ops(netdev); 12346 12347 /* MTU range: 68 - 9706 */ 12348 netdev->min_mtu = ETH_MIN_MTU; 12349 netdev->max_mtu = I40E_MAX_RXBUFFER - I40E_PACKET_HDR_PAD; 12350 12351 return 0; 12352 } 12353 12354 /** 12355 * i40e_vsi_delete - Delete a VSI from the switch 12356 * @vsi: the VSI being removed 12357 * 12358 * Returns 0 on success, negative value on failure 12359 **/ 12360 static void i40e_vsi_delete(struct i40e_vsi *vsi) 12361 { 12362 /* remove default VSI is not allowed */ 12363 if (vsi == vsi->back->vsi[vsi->back->lan_vsi]) 12364 return; 12365 12366 i40e_aq_delete_element(&vsi->back->hw, vsi->seid, NULL); 12367 } 12368 12369 /** 12370 * i40e_is_vsi_uplink_mode_veb - Check if the VSI's uplink bridge mode is VEB 12371 * @vsi: the VSI being queried 12372 * 12373 * Returns 1 if HW bridge mode is VEB and return 0 in case of VEPA mode 12374 **/ 12375 int i40e_is_vsi_uplink_mode_veb(struct i40e_vsi *vsi) 12376 { 12377 struct i40e_veb *veb; 12378 struct i40e_pf *pf = vsi->back; 12379 12380 /* Uplink is not a bridge so default to VEB */ 12381 if (vsi->veb_idx == I40E_NO_VEB) 12382 return 1; 12383 12384 veb = pf->veb[vsi->veb_idx]; 12385 if (!veb) { 12386 dev_info(&pf->pdev->dev, 12387 "There is no veb associated with the bridge\n"); 12388 return -ENOENT; 12389 } 12390 12391 /* Uplink is a bridge in VEPA mode */ 12392 if (veb->bridge_mode & BRIDGE_MODE_VEPA) { 12393 return 0; 12394 } else { 12395 /* Uplink is a bridge in VEB mode */ 12396 return 1; 12397 } 12398 12399 /* VEPA is now default bridge, so return 0 */ 12400 return 0; 12401 } 12402 12403 /** 12404 * i40e_add_vsi - Add a VSI to the switch 12405 * @vsi: the VSI being configured 12406 * 12407 * This initializes a VSI context depending on the VSI type to be added and 12408 * passes it down to the add_vsi aq command. 12409 **/ 12410 static int i40e_add_vsi(struct i40e_vsi *vsi) 12411 { 12412 int ret = -ENODEV; 12413 struct i40e_pf *pf = vsi->back; 12414 struct i40e_hw *hw = &pf->hw; 12415 struct i40e_vsi_context ctxt; 12416 struct i40e_mac_filter *f; 12417 struct hlist_node *h; 12418 int bkt; 12419 12420 u8 enabled_tc = 0x1; /* TC0 enabled */ 12421 int f_count = 0; 12422 12423 memset(&ctxt, 0, sizeof(ctxt)); 12424 switch (vsi->type) { 12425 case I40E_VSI_MAIN: 12426 /* The PF's main VSI is already setup as part of the 12427 * device initialization, so we'll not bother with 12428 * the add_vsi call, but we will retrieve the current 12429 * VSI context. 12430 */ 12431 ctxt.seid = pf->main_vsi_seid; 12432 ctxt.pf_num = pf->hw.pf_id; 12433 ctxt.vf_num = 0; 12434 ret = i40e_aq_get_vsi_params(&pf->hw, &ctxt, NULL); 12435 ctxt.flags = I40E_AQ_VSI_TYPE_PF; 12436 if (ret) { 12437 dev_info(&pf->pdev->dev, 12438 "couldn't get PF vsi config, err %s aq_err %s\n", 12439 i40e_stat_str(&pf->hw, ret), 12440 i40e_aq_str(&pf->hw, 12441 pf->hw.aq.asq_last_status)); 12442 return -ENOENT; 12443 } 12444 vsi->info = ctxt.info; 12445 vsi->info.valid_sections = 0; 12446 12447 vsi->seid = ctxt.seid; 12448 vsi->id = ctxt.vsi_number; 12449 12450 enabled_tc = i40e_pf_get_tc_map(pf); 12451 12452 /* Source pruning is enabled by default, so the flag is 12453 * negative logic - if it's set, we need to fiddle with 12454 * the VSI to disable source pruning. 12455 */ 12456 if (pf->flags & I40E_FLAG_SOURCE_PRUNING_DISABLED) { 12457 memset(&ctxt, 0, sizeof(ctxt)); 12458 ctxt.seid = pf->main_vsi_seid; 12459 ctxt.pf_num = pf->hw.pf_id; 12460 ctxt.vf_num = 0; 12461 ctxt.info.valid_sections |= 12462 cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID); 12463 ctxt.info.switch_id = 12464 cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_LOCAL_LB); 12465 ret = i40e_aq_update_vsi_params(hw, &ctxt, NULL); 12466 if (ret) { 12467 dev_info(&pf->pdev->dev, 12468 "update vsi failed, err %s aq_err %s\n", 12469 i40e_stat_str(&pf->hw, ret), 12470 i40e_aq_str(&pf->hw, 12471 pf->hw.aq.asq_last_status)); 12472 ret = -ENOENT; 12473 goto err; 12474 } 12475 } 12476 12477 /* MFP mode setup queue map and update VSI */ 12478 if ((pf->flags & I40E_FLAG_MFP_ENABLED) && 12479 !(pf->hw.func_caps.iscsi)) { /* NIC type PF */ 12480 memset(&ctxt, 0, sizeof(ctxt)); 12481 ctxt.seid = pf->main_vsi_seid; 12482 ctxt.pf_num = pf->hw.pf_id; 12483 ctxt.vf_num = 0; 12484 i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, false); 12485 ret = i40e_aq_update_vsi_params(hw, &ctxt, NULL); 12486 if (ret) { 12487 dev_info(&pf->pdev->dev, 12488 "update vsi failed, err %s aq_err %s\n", 12489 i40e_stat_str(&pf->hw, ret), 12490 i40e_aq_str(&pf->hw, 12491 pf->hw.aq.asq_last_status)); 12492 ret = -ENOENT; 12493 goto err; 12494 } 12495 /* update the local VSI info queue map */ 12496 i40e_vsi_update_queue_map(vsi, &ctxt); 12497 vsi->info.valid_sections = 0; 12498 } else { 12499 /* Default/Main VSI is only enabled for TC0 12500 * reconfigure it to enable all TCs that are 12501 * available on the port in SFP mode. 12502 * For MFP case the iSCSI PF would use this 12503 * flow to enable LAN+iSCSI TC. 12504 */ 12505 ret = i40e_vsi_config_tc(vsi, enabled_tc); 12506 if (ret) { 12507 /* Single TC condition is not fatal, 12508 * message and continue 12509 */ 12510 dev_info(&pf->pdev->dev, 12511 "failed to configure TCs for main VSI tc_map 0x%08x, err %s aq_err %s\n", 12512 enabled_tc, 12513 i40e_stat_str(&pf->hw, ret), 12514 i40e_aq_str(&pf->hw, 12515 pf->hw.aq.asq_last_status)); 12516 } 12517 } 12518 break; 12519 12520 case I40E_VSI_FDIR: 12521 ctxt.pf_num = hw->pf_id; 12522 ctxt.vf_num = 0; 12523 ctxt.uplink_seid = vsi->uplink_seid; 12524 ctxt.connection_type = I40E_AQ_VSI_CONN_TYPE_NORMAL; 12525 ctxt.flags = I40E_AQ_VSI_TYPE_PF; 12526 if ((pf->flags & I40E_FLAG_VEB_MODE_ENABLED) && 12527 (i40e_is_vsi_uplink_mode_veb(vsi))) { 12528 ctxt.info.valid_sections |= 12529 cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID); 12530 ctxt.info.switch_id = 12531 cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB); 12532 } 12533 i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, true); 12534 break; 12535 12536 case I40E_VSI_VMDQ2: 12537 ctxt.pf_num = hw->pf_id; 12538 ctxt.vf_num = 0; 12539 ctxt.uplink_seid = vsi->uplink_seid; 12540 ctxt.connection_type = I40E_AQ_VSI_CONN_TYPE_NORMAL; 12541 ctxt.flags = I40E_AQ_VSI_TYPE_VMDQ2; 12542 12543 /* This VSI is connected to VEB so the switch_id 12544 * should be set to zero by default. 12545 */ 12546 if (i40e_is_vsi_uplink_mode_veb(vsi)) { 12547 ctxt.info.valid_sections |= 12548 cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID); 12549 ctxt.info.switch_id = 12550 cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB); 12551 } 12552 12553 /* Setup the VSI tx/rx queue map for TC0 only for now */ 12554 i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, true); 12555 break; 12556 12557 case I40E_VSI_SRIOV: 12558 ctxt.pf_num = hw->pf_id; 12559 ctxt.vf_num = vsi->vf_id + hw->func_caps.vf_base_id; 12560 ctxt.uplink_seid = vsi->uplink_seid; 12561 ctxt.connection_type = I40E_AQ_VSI_CONN_TYPE_NORMAL; 12562 ctxt.flags = I40E_AQ_VSI_TYPE_VF; 12563 12564 /* This VSI is connected to VEB so the switch_id 12565 * should be set to zero by default. 12566 */ 12567 if (i40e_is_vsi_uplink_mode_veb(vsi)) { 12568 ctxt.info.valid_sections |= 12569 cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID); 12570 ctxt.info.switch_id = 12571 cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB); 12572 } 12573 12574 if (vsi->back->flags & I40E_FLAG_IWARP_ENABLED) { 12575 ctxt.info.valid_sections |= 12576 cpu_to_le16(I40E_AQ_VSI_PROP_QUEUE_OPT_VALID); 12577 ctxt.info.queueing_opt_flags |= 12578 (I40E_AQ_VSI_QUE_OPT_TCP_ENA | 12579 I40E_AQ_VSI_QUE_OPT_RSS_LUT_VSI); 12580 } 12581 12582 ctxt.info.valid_sections |= cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID); 12583 ctxt.info.port_vlan_flags |= I40E_AQ_VSI_PVLAN_MODE_ALL; 12584 if (pf->vf[vsi->vf_id].spoofchk) { 12585 ctxt.info.valid_sections |= 12586 cpu_to_le16(I40E_AQ_VSI_PROP_SECURITY_VALID); 12587 ctxt.info.sec_flags |= 12588 (I40E_AQ_VSI_SEC_FLAG_ENABLE_VLAN_CHK | 12589 I40E_AQ_VSI_SEC_FLAG_ENABLE_MAC_CHK); 12590 } 12591 /* Setup the VSI tx/rx queue map for TC0 only for now */ 12592 i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, true); 12593 break; 12594 12595 case I40E_VSI_IWARP: 12596 /* send down message to iWARP */ 12597 break; 12598 12599 default: 12600 return -ENODEV; 12601 } 12602 12603 if (vsi->type != I40E_VSI_MAIN) { 12604 ret = i40e_aq_add_vsi(hw, &ctxt, NULL); 12605 if (ret) { 12606 dev_info(&vsi->back->pdev->dev, 12607 "add vsi failed, err %s aq_err %s\n", 12608 i40e_stat_str(&pf->hw, ret), 12609 i40e_aq_str(&pf->hw, 12610 pf->hw.aq.asq_last_status)); 12611 ret = -ENOENT; 12612 goto err; 12613 } 12614 vsi->info = ctxt.info; 12615 vsi->info.valid_sections = 0; 12616 vsi->seid = ctxt.seid; 12617 vsi->id = ctxt.vsi_number; 12618 } 12619 12620 vsi->active_filters = 0; 12621 clear_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state); 12622 spin_lock_bh(&vsi->mac_filter_hash_lock); 12623 /* If macvlan filters already exist, force them to get loaded */ 12624 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) { 12625 f->state = I40E_FILTER_NEW; 12626 f_count++; 12627 } 12628 spin_unlock_bh(&vsi->mac_filter_hash_lock); 12629 12630 if (f_count) { 12631 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED; 12632 set_bit(__I40E_MACVLAN_SYNC_PENDING, pf->state); 12633 } 12634 12635 /* Update VSI BW information */ 12636 ret = i40e_vsi_get_bw_info(vsi); 12637 if (ret) { 12638 dev_info(&pf->pdev->dev, 12639 "couldn't get vsi bw info, err %s aq_err %s\n", 12640 i40e_stat_str(&pf->hw, ret), 12641 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 12642 /* VSI is already added so not tearing that up */ 12643 ret = 0; 12644 } 12645 12646 err: 12647 return ret; 12648 } 12649 12650 /** 12651 * i40e_vsi_release - Delete a VSI and free its resources 12652 * @vsi: the VSI being removed 12653 * 12654 * Returns 0 on success or < 0 on error 12655 **/ 12656 int i40e_vsi_release(struct i40e_vsi *vsi) 12657 { 12658 struct i40e_mac_filter *f; 12659 struct hlist_node *h; 12660 struct i40e_veb *veb = NULL; 12661 struct i40e_pf *pf; 12662 u16 uplink_seid; 12663 int i, n, bkt; 12664 12665 pf = vsi->back; 12666 12667 /* release of a VEB-owner or last VSI is not allowed */ 12668 if (vsi->flags & I40E_VSI_FLAG_VEB_OWNER) { 12669 dev_info(&pf->pdev->dev, "VSI %d has existing VEB %d\n", 12670 vsi->seid, vsi->uplink_seid); 12671 return -ENODEV; 12672 } 12673 if (vsi == pf->vsi[pf->lan_vsi] && 12674 !test_bit(__I40E_DOWN, pf->state)) { 12675 dev_info(&pf->pdev->dev, "Can't remove PF VSI\n"); 12676 return -ENODEV; 12677 } 12678 12679 uplink_seid = vsi->uplink_seid; 12680 if (vsi->type != I40E_VSI_SRIOV) { 12681 if (vsi->netdev_registered) { 12682 vsi->netdev_registered = false; 12683 if (vsi->netdev) { 12684 /* results in a call to i40e_close() */ 12685 unregister_netdev(vsi->netdev); 12686 } 12687 } else { 12688 i40e_vsi_close(vsi); 12689 } 12690 i40e_vsi_disable_irq(vsi); 12691 } 12692 12693 spin_lock_bh(&vsi->mac_filter_hash_lock); 12694 12695 /* clear the sync flag on all filters */ 12696 if (vsi->netdev) { 12697 __dev_uc_unsync(vsi->netdev, NULL); 12698 __dev_mc_unsync(vsi->netdev, NULL); 12699 } 12700 12701 /* make sure any remaining filters are marked for deletion */ 12702 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) 12703 __i40e_del_filter(vsi, f); 12704 12705 spin_unlock_bh(&vsi->mac_filter_hash_lock); 12706 12707 i40e_sync_vsi_filters(vsi); 12708 12709 i40e_vsi_delete(vsi); 12710 i40e_vsi_free_q_vectors(vsi); 12711 if (vsi->netdev) { 12712 free_netdev(vsi->netdev); 12713 vsi->netdev = NULL; 12714 } 12715 i40e_vsi_clear_rings(vsi); 12716 i40e_vsi_clear(vsi); 12717 12718 /* If this was the last thing on the VEB, except for the 12719 * controlling VSI, remove the VEB, which puts the controlling 12720 * VSI onto the next level down in the switch. 12721 * 12722 * Well, okay, there's one more exception here: don't remove 12723 * the orphan VEBs yet. We'll wait for an explicit remove request 12724 * from up the network stack. 12725 */ 12726 for (n = 0, i = 0; i < pf->num_alloc_vsi; i++) { 12727 if (pf->vsi[i] && 12728 pf->vsi[i]->uplink_seid == uplink_seid && 12729 (pf->vsi[i]->flags & I40E_VSI_FLAG_VEB_OWNER) == 0) { 12730 n++; /* count the VSIs */ 12731 } 12732 } 12733 for (i = 0; i < I40E_MAX_VEB; i++) { 12734 if (!pf->veb[i]) 12735 continue; 12736 if (pf->veb[i]->uplink_seid == uplink_seid) 12737 n++; /* count the VEBs */ 12738 if (pf->veb[i]->seid == uplink_seid) 12739 veb = pf->veb[i]; 12740 } 12741 if (n == 0 && veb && veb->uplink_seid != 0) 12742 i40e_veb_release(veb); 12743 12744 return 0; 12745 } 12746 12747 /** 12748 * i40e_vsi_setup_vectors - Set up the q_vectors for the given VSI 12749 * @vsi: ptr to the VSI 12750 * 12751 * This should only be called after i40e_vsi_mem_alloc() which allocates the 12752 * corresponding SW VSI structure and initializes num_queue_pairs for the 12753 * newly allocated VSI. 12754 * 12755 * Returns 0 on success or negative on failure 12756 **/ 12757 static int i40e_vsi_setup_vectors(struct i40e_vsi *vsi) 12758 { 12759 int ret = -ENOENT; 12760 struct i40e_pf *pf = vsi->back; 12761 12762 if (vsi->q_vectors[0]) { 12763 dev_info(&pf->pdev->dev, "VSI %d has existing q_vectors\n", 12764 vsi->seid); 12765 return -EEXIST; 12766 } 12767 12768 if (vsi->base_vector) { 12769 dev_info(&pf->pdev->dev, "VSI %d has non-zero base vector %d\n", 12770 vsi->seid, vsi->base_vector); 12771 return -EEXIST; 12772 } 12773 12774 ret = i40e_vsi_alloc_q_vectors(vsi); 12775 if (ret) { 12776 dev_info(&pf->pdev->dev, 12777 "failed to allocate %d q_vector for VSI %d, ret=%d\n", 12778 vsi->num_q_vectors, vsi->seid, ret); 12779 vsi->num_q_vectors = 0; 12780 goto vector_setup_out; 12781 } 12782 12783 /* In Legacy mode, we do not have to get any other vector since we 12784 * piggyback on the misc/ICR0 for queue interrupts. 12785 */ 12786 if (!(pf->flags & I40E_FLAG_MSIX_ENABLED)) 12787 return ret; 12788 if (vsi->num_q_vectors) 12789 vsi->base_vector = i40e_get_lump(pf, pf->irq_pile, 12790 vsi->num_q_vectors, vsi->idx); 12791 if (vsi->base_vector < 0) { 12792 dev_info(&pf->pdev->dev, 12793 "failed to get tracking for %d vectors for VSI %d, err=%d\n", 12794 vsi->num_q_vectors, vsi->seid, vsi->base_vector); 12795 i40e_vsi_free_q_vectors(vsi); 12796 ret = -ENOENT; 12797 goto vector_setup_out; 12798 } 12799 12800 vector_setup_out: 12801 return ret; 12802 } 12803 12804 /** 12805 * i40e_vsi_reinit_setup - return and reallocate resources for a VSI 12806 * @vsi: pointer to the vsi. 12807 * 12808 * This re-allocates a vsi's queue resources. 12809 * 12810 * Returns pointer to the successfully allocated and configured VSI sw struct 12811 * on success, otherwise returns NULL on failure. 12812 **/ 12813 static struct i40e_vsi *i40e_vsi_reinit_setup(struct i40e_vsi *vsi) 12814 { 12815 u16 alloc_queue_pairs; 12816 struct i40e_pf *pf; 12817 u8 enabled_tc; 12818 int ret; 12819 12820 if (!vsi) 12821 return NULL; 12822 12823 pf = vsi->back; 12824 12825 i40e_put_lump(pf->qp_pile, vsi->base_queue, vsi->idx); 12826 i40e_vsi_clear_rings(vsi); 12827 12828 i40e_vsi_free_arrays(vsi, false); 12829 i40e_set_num_rings_in_vsi(vsi); 12830 ret = i40e_vsi_alloc_arrays(vsi, false); 12831 if (ret) 12832 goto err_vsi; 12833 12834 alloc_queue_pairs = vsi->alloc_queue_pairs * 12835 (i40e_enabled_xdp_vsi(vsi) ? 2 : 1); 12836 12837 ret = i40e_get_lump(pf, pf->qp_pile, alloc_queue_pairs, vsi->idx); 12838 if (ret < 0) { 12839 dev_info(&pf->pdev->dev, 12840 "failed to get tracking for %d queues for VSI %d err %d\n", 12841 alloc_queue_pairs, vsi->seid, ret); 12842 goto err_vsi; 12843 } 12844 vsi->base_queue = ret; 12845 12846 /* Update the FW view of the VSI. Force a reset of TC and queue 12847 * layout configurations. 12848 */ 12849 enabled_tc = pf->vsi[pf->lan_vsi]->tc_config.enabled_tc; 12850 pf->vsi[pf->lan_vsi]->tc_config.enabled_tc = 0; 12851 pf->vsi[pf->lan_vsi]->seid = pf->main_vsi_seid; 12852 i40e_vsi_config_tc(pf->vsi[pf->lan_vsi], enabled_tc); 12853 if (vsi->type == I40E_VSI_MAIN) 12854 i40e_rm_default_mac_filter(vsi, pf->hw.mac.perm_addr); 12855 12856 /* assign it some queues */ 12857 ret = i40e_alloc_rings(vsi); 12858 if (ret) 12859 goto err_rings; 12860 12861 /* map all of the rings to the q_vectors */ 12862 i40e_vsi_map_rings_to_vectors(vsi); 12863 return vsi; 12864 12865 err_rings: 12866 i40e_vsi_free_q_vectors(vsi); 12867 if (vsi->netdev_registered) { 12868 vsi->netdev_registered = false; 12869 unregister_netdev(vsi->netdev); 12870 free_netdev(vsi->netdev); 12871 vsi->netdev = NULL; 12872 } 12873 i40e_aq_delete_element(&pf->hw, vsi->seid, NULL); 12874 err_vsi: 12875 i40e_vsi_clear(vsi); 12876 return NULL; 12877 } 12878 12879 /** 12880 * i40e_vsi_setup - Set up a VSI by a given type 12881 * @pf: board private structure 12882 * @type: VSI type 12883 * @uplink_seid: the switch element to link to 12884 * @param1: usage depends upon VSI type. For VF types, indicates VF id 12885 * 12886 * This allocates the sw VSI structure and its queue resources, then add a VSI 12887 * to the identified VEB. 12888 * 12889 * Returns pointer to the successfully allocated and configure VSI sw struct on 12890 * success, otherwise returns NULL on failure. 12891 **/ 12892 struct i40e_vsi *i40e_vsi_setup(struct i40e_pf *pf, u8 type, 12893 u16 uplink_seid, u32 param1) 12894 { 12895 struct i40e_vsi *vsi = NULL; 12896 struct i40e_veb *veb = NULL; 12897 u16 alloc_queue_pairs; 12898 int ret, i; 12899 int v_idx; 12900 12901 /* The requested uplink_seid must be either 12902 * - the PF's port seid 12903 * no VEB is needed because this is the PF 12904 * or this is a Flow Director special case VSI 12905 * - seid of an existing VEB 12906 * - seid of a VSI that owns an existing VEB 12907 * - seid of a VSI that doesn't own a VEB 12908 * a new VEB is created and the VSI becomes the owner 12909 * - seid of the PF VSI, which is what creates the first VEB 12910 * this is a special case of the previous 12911 * 12912 * Find which uplink_seid we were given and create a new VEB if needed 12913 */ 12914 for (i = 0; i < I40E_MAX_VEB; i++) { 12915 if (pf->veb[i] && pf->veb[i]->seid == uplink_seid) { 12916 veb = pf->veb[i]; 12917 break; 12918 } 12919 } 12920 12921 if (!veb && uplink_seid != pf->mac_seid) { 12922 12923 for (i = 0; i < pf->num_alloc_vsi; i++) { 12924 if (pf->vsi[i] && pf->vsi[i]->seid == uplink_seid) { 12925 vsi = pf->vsi[i]; 12926 break; 12927 } 12928 } 12929 if (!vsi) { 12930 dev_info(&pf->pdev->dev, "no such uplink_seid %d\n", 12931 uplink_seid); 12932 return NULL; 12933 } 12934 12935 if (vsi->uplink_seid == pf->mac_seid) 12936 veb = i40e_veb_setup(pf, 0, pf->mac_seid, vsi->seid, 12937 vsi->tc_config.enabled_tc); 12938 else if ((vsi->flags & I40E_VSI_FLAG_VEB_OWNER) == 0) 12939 veb = i40e_veb_setup(pf, 0, vsi->uplink_seid, vsi->seid, 12940 vsi->tc_config.enabled_tc); 12941 if (veb) { 12942 if (vsi->seid != pf->vsi[pf->lan_vsi]->seid) { 12943 dev_info(&vsi->back->pdev->dev, 12944 "New VSI creation error, uplink seid of LAN VSI expected.\n"); 12945 return NULL; 12946 } 12947 /* We come up by default in VEPA mode if SRIOV is not 12948 * already enabled, in which case we can't force VEPA 12949 * mode. 12950 */ 12951 if (!(pf->flags & I40E_FLAG_VEB_MODE_ENABLED)) { 12952 veb->bridge_mode = BRIDGE_MODE_VEPA; 12953 pf->flags &= ~I40E_FLAG_VEB_MODE_ENABLED; 12954 } 12955 i40e_config_bridge_mode(veb); 12956 } 12957 for (i = 0; i < I40E_MAX_VEB && !veb; i++) { 12958 if (pf->veb[i] && pf->veb[i]->seid == vsi->uplink_seid) 12959 veb = pf->veb[i]; 12960 } 12961 if (!veb) { 12962 dev_info(&pf->pdev->dev, "couldn't add VEB\n"); 12963 return NULL; 12964 } 12965 12966 vsi->flags |= I40E_VSI_FLAG_VEB_OWNER; 12967 uplink_seid = veb->seid; 12968 } 12969 12970 /* get vsi sw struct */ 12971 v_idx = i40e_vsi_mem_alloc(pf, type); 12972 if (v_idx < 0) 12973 goto err_alloc; 12974 vsi = pf->vsi[v_idx]; 12975 if (!vsi) 12976 goto err_alloc; 12977 vsi->type = type; 12978 vsi->veb_idx = (veb ? veb->idx : I40E_NO_VEB); 12979 12980 if (type == I40E_VSI_MAIN) 12981 pf->lan_vsi = v_idx; 12982 else if (type == I40E_VSI_SRIOV) 12983 vsi->vf_id = param1; 12984 /* assign it some queues */ 12985 alloc_queue_pairs = vsi->alloc_queue_pairs * 12986 (i40e_enabled_xdp_vsi(vsi) ? 2 : 1); 12987 12988 ret = i40e_get_lump(pf, pf->qp_pile, alloc_queue_pairs, vsi->idx); 12989 if (ret < 0) { 12990 dev_info(&pf->pdev->dev, 12991 "failed to get tracking for %d queues for VSI %d err=%d\n", 12992 alloc_queue_pairs, vsi->seid, ret); 12993 goto err_vsi; 12994 } 12995 vsi->base_queue = ret; 12996 12997 /* get a VSI from the hardware */ 12998 vsi->uplink_seid = uplink_seid; 12999 ret = i40e_add_vsi(vsi); 13000 if (ret) 13001 goto err_vsi; 13002 13003 switch (vsi->type) { 13004 /* setup the netdev if needed */ 13005 case I40E_VSI_MAIN: 13006 case I40E_VSI_VMDQ2: 13007 ret = i40e_config_netdev(vsi); 13008 if (ret) 13009 goto err_netdev; 13010 ret = register_netdev(vsi->netdev); 13011 if (ret) 13012 goto err_netdev; 13013 vsi->netdev_registered = true; 13014 netif_carrier_off(vsi->netdev); 13015 #ifdef CONFIG_I40E_DCB 13016 /* Setup DCB netlink interface */ 13017 i40e_dcbnl_setup(vsi); 13018 #endif /* CONFIG_I40E_DCB */ 13019 /* fall through */ 13020 13021 case I40E_VSI_FDIR: 13022 /* set up vectors and rings if needed */ 13023 ret = i40e_vsi_setup_vectors(vsi); 13024 if (ret) 13025 goto err_msix; 13026 13027 ret = i40e_alloc_rings(vsi); 13028 if (ret) 13029 goto err_rings; 13030 13031 /* map all of the rings to the q_vectors */ 13032 i40e_vsi_map_rings_to_vectors(vsi); 13033 13034 i40e_vsi_reset_stats(vsi); 13035 break; 13036 13037 default: 13038 /* no netdev or rings for the other VSI types */ 13039 break; 13040 } 13041 13042 if ((pf->hw_features & I40E_HW_RSS_AQ_CAPABLE) && 13043 (vsi->type == I40E_VSI_VMDQ2)) { 13044 ret = i40e_vsi_config_rss(vsi); 13045 } 13046 return vsi; 13047 13048 err_rings: 13049 i40e_vsi_free_q_vectors(vsi); 13050 err_msix: 13051 if (vsi->netdev_registered) { 13052 vsi->netdev_registered = false; 13053 unregister_netdev(vsi->netdev); 13054 free_netdev(vsi->netdev); 13055 vsi->netdev = NULL; 13056 } 13057 err_netdev: 13058 i40e_aq_delete_element(&pf->hw, vsi->seid, NULL); 13059 err_vsi: 13060 i40e_vsi_clear(vsi); 13061 err_alloc: 13062 return NULL; 13063 } 13064 13065 /** 13066 * i40e_veb_get_bw_info - Query VEB BW information 13067 * @veb: the veb to query 13068 * 13069 * Query the Tx scheduler BW configuration data for given VEB 13070 **/ 13071 static int i40e_veb_get_bw_info(struct i40e_veb *veb) 13072 { 13073 struct i40e_aqc_query_switching_comp_ets_config_resp ets_data; 13074 struct i40e_aqc_query_switching_comp_bw_config_resp bw_data; 13075 struct i40e_pf *pf = veb->pf; 13076 struct i40e_hw *hw = &pf->hw; 13077 u32 tc_bw_max; 13078 int ret = 0; 13079 int i; 13080 13081 ret = i40e_aq_query_switch_comp_bw_config(hw, veb->seid, 13082 &bw_data, NULL); 13083 if (ret) { 13084 dev_info(&pf->pdev->dev, 13085 "query veb bw config failed, err %s aq_err %s\n", 13086 i40e_stat_str(&pf->hw, ret), 13087 i40e_aq_str(&pf->hw, hw->aq.asq_last_status)); 13088 goto out; 13089 } 13090 13091 ret = i40e_aq_query_switch_comp_ets_config(hw, veb->seid, 13092 &ets_data, NULL); 13093 if (ret) { 13094 dev_info(&pf->pdev->dev, 13095 "query veb bw ets config failed, err %s aq_err %s\n", 13096 i40e_stat_str(&pf->hw, ret), 13097 i40e_aq_str(&pf->hw, hw->aq.asq_last_status)); 13098 goto out; 13099 } 13100 13101 veb->bw_limit = le16_to_cpu(ets_data.port_bw_limit); 13102 veb->bw_max_quanta = ets_data.tc_bw_max; 13103 veb->is_abs_credits = bw_data.absolute_credits_enable; 13104 veb->enabled_tc = ets_data.tc_valid_bits; 13105 tc_bw_max = le16_to_cpu(bw_data.tc_bw_max[0]) | 13106 (le16_to_cpu(bw_data.tc_bw_max[1]) << 16); 13107 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 13108 veb->bw_tc_share_credits[i] = bw_data.tc_bw_share_credits[i]; 13109 veb->bw_tc_limit_credits[i] = 13110 le16_to_cpu(bw_data.tc_bw_limits[i]); 13111 veb->bw_tc_max_quanta[i] = ((tc_bw_max >> (i*4)) & 0x7); 13112 } 13113 13114 out: 13115 return ret; 13116 } 13117 13118 /** 13119 * i40e_veb_mem_alloc - Allocates the next available struct veb in the PF 13120 * @pf: board private structure 13121 * 13122 * On error: returns error code (negative) 13123 * On success: returns vsi index in PF (positive) 13124 **/ 13125 static int i40e_veb_mem_alloc(struct i40e_pf *pf) 13126 { 13127 int ret = -ENOENT; 13128 struct i40e_veb *veb; 13129 int i; 13130 13131 /* Need to protect the allocation of switch elements at the PF level */ 13132 mutex_lock(&pf->switch_mutex); 13133 13134 /* VEB list may be fragmented if VEB creation/destruction has 13135 * been happening. We can afford to do a quick scan to look 13136 * for any free slots in the list. 13137 * 13138 * find next empty veb slot, looping back around if necessary 13139 */ 13140 i = 0; 13141 while ((i < I40E_MAX_VEB) && (pf->veb[i] != NULL)) 13142 i++; 13143 if (i >= I40E_MAX_VEB) { 13144 ret = -ENOMEM; 13145 goto err_alloc_veb; /* out of VEB slots! */ 13146 } 13147 13148 veb = kzalloc(sizeof(*veb), GFP_KERNEL); 13149 if (!veb) { 13150 ret = -ENOMEM; 13151 goto err_alloc_veb; 13152 } 13153 veb->pf = pf; 13154 veb->idx = i; 13155 veb->enabled_tc = 1; 13156 13157 pf->veb[i] = veb; 13158 ret = i; 13159 err_alloc_veb: 13160 mutex_unlock(&pf->switch_mutex); 13161 return ret; 13162 } 13163 13164 /** 13165 * i40e_switch_branch_release - Delete a branch of the switch tree 13166 * @branch: where to start deleting 13167 * 13168 * This uses recursion to find the tips of the branch to be 13169 * removed, deleting until we get back to and can delete this VEB. 13170 **/ 13171 static void i40e_switch_branch_release(struct i40e_veb *branch) 13172 { 13173 struct i40e_pf *pf = branch->pf; 13174 u16 branch_seid = branch->seid; 13175 u16 veb_idx = branch->idx; 13176 int i; 13177 13178 /* release any VEBs on this VEB - RECURSION */ 13179 for (i = 0; i < I40E_MAX_VEB; i++) { 13180 if (!pf->veb[i]) 13181 continue; 13182 if (pf->veb[i]->uplink_seid == branch->seid) 13183 i40e_switch_branch_release(pf->veb[i]); 13184 } 13185 13186 /* Release the VSIs on this VEB, but not the owner VSI. 13187 * 13188 * NOTE: Removing the last VSI on a VEB has the SIDE EFFECT of removing 13189 * the VEB itself, so don't use (*branch) after this loop. 13190 */ 13191 for (i = 0; i < pf->num_alloc_vsi; i++) { 13192 if (!pf->vsi[i]) 13193 continue; 13194 if (pf->vsi[i]->uplink_seid == branch_seid && 13195 (pf->vsi[i]->flags & I40E_VSI_FLAG_VEB_OWNER) == 0) { 13196 i40e_vsi_release(pf->vsi[i]); 13197 } 13198 } 13199 13200 /* There's one corner case where the VEB might not have been 13201 * removed, so double check it here and remove it if needed. 13202 * This case happens if the veb was created from the debugfs 13203 * commands and no VSIs were added to it. 13204 */ 13205 if (pf->veb[veb_idx]) 13206 i40e_veb_release(pf->veb[veb_idx]); 13207 } 13208 13209 /** 13210 * i40e_veb_clear - remove veb struct 13211 * @veb: the veb to remove 13212 **/ 13213 static void i40e_veb_clear(struct i40e_veb *veb) 13214 { 13215 if (!veb) 13216 return; 13217 13218 if (veb->pf) { 13219 struct i40e_pf *pf = veb->pf; 13220 13221 mutex_lock(&pf->switch_mutex); 13222 if (pf->veb[veb->idx] == veb) 13223 pf->veb[veb->idx] = NULL; 13224 mutex_unlock(&pf->switch_mutex); 13225 } 13226 13227 kfree(veb); 13228 } 13229 13230 /** 13231 * i40e_veb_release - Delete a VEB and free its resources 13232 * @veb: the VEB being removed 13233 **/ 13234 void i40e_veb_release(struct i40e_veb *veb) 13235 { 13236 struct i40e_vsi *vsi = NULL; 13237 struct i40e_pf *pf; 13238 int i, n = 0; 13239 13240 pf = veb->pf; 13241 13242 /* find the remaining VSI and check for extras */ 13243 for (i = 0; i < pf->num_alloc_vsi; i++) { 13244 if (pf->vsi[i] && pf->vsi[i]->uplink_seid == veb->seid) { 13245 n++; 13246 vsi = pf->vsi[i]; 13247 } 13248 } 13249 if (n != 1) { 13250 dev_info(&pf->pdev->dev, 13251 "can't remove VEB %d with %d VSIs left\n", 13252 veb->seid, n); 13253 return; 13254 } 13255 13256 /* move the remaining VSI to uplink veb */ 13257 vsi->flags &= ~I40E_VSI_FLAG_VEB_OWNER; 13258 if (veb->uplink_seid) { 13259 vsi->uplink_seid = veb->uplink_seid; 13260 if (veb->uplink_seid == pf->mac_seid) 13261 vsi->veb_idx = I40E_NO_VEB; 13262 else 13263 vsi->veb_idx = veb->veb_idx; 13264 } else { 13265 /* floating VEB */ 13266 vsi->uplink_seid = pf->vsi[pf->lan_vsi]->uplink_seid; 13267 vsi->veb_idx = pf->vsi[pf->lan_vsi]->veb_idx; 13268 } 13269 13270 i40e_aq_delete_element(&pf->hw, veb->seid, NULL); 13271 i40e_veb_clear(veb); 13272 } 13273 13274 /** 13275 * i40e_add_veb - create the VEB in the switch 13276 * @veb: the VEB to be instantiated 13277 * @vsi: the controlling VSI 13278 **/ 13279 static int i40e_add_veb(struct i40e_veb *veb, struct i40e_vsi *vsi) 13280 { 13281 struct i40e_pf *pf = veb->pf; 13282 bool enable_stats = !!(pf->flags & I40E_FLAG_VEB_STATS_ENABLED); 13283 int ret; 13284 13285 ret = i40e_aq_add_veb(&pf->hw, veb->uplink_seid, vsi->seid, 13286 veb->enabled_tc, false, 13287 &veb->seid, enable_stats, NULL); 13288 13289 /* get a VEB from the hardware */ 13290 if (ret) { 13291 dev_info(&pf->pdev->dev, 13292 "couldn't add VEB, err %s aq_err %s\n", 13293 i40e_stat_str(&pf->hw, ret), 13294 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 13295 return -EPERM; 13296 } 13297 13298 /* get statistics counter */ 13299 ret = i40e_aq_get_veb_parameters(&pf->hw, veb->seid, NULL, NULL, 13300 &veb->stats_idx, NULL, NULL, NULL); 13301 if (ret) { 13302 dev_info(&pf->pdev->dev, 13303 "couldn't get VEB statistics idx, err %s aq_err %s\n", 13304 i40e_stat_str(&pf->hw, ret), 13305 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 13306 return -EPERM; 13307 } 13308 ret = i40e_veb_get_bw_info(veb); 13309 if (ret) { 13310 dev_info(&pf->pdev->dev, 13311 "couldn't get VEB bw info, err %s aq_err %s\n", 13312 i40e_stat_str(&pf->hw, ret), 13313 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 13314 i40e_aq_delete_element(&pf->hw, veb->seid, NULL); 13315 return -ENOENT; 13316 } 13317 13318 vsi->uplink_seid = veb->seid; 13319 vsi->veb_idx = veb->idx; 13320 vsi->flags |= I40E_VSI_FLAG_VEB_OWNER; 13321 13322 return 0; 13323 } 13324 13325 /** 13326 * i40e_veb_setup - Set up a VEB 13327 * @pf: board private structure 13328 * @flags: VEB setup flags 13329 * @uplink_seid: the switch element to link to 13330 * @vsi_seid: the initial VSI seid 13331 * @enabled_tc: Enabled TC bit-map 13332 * 13333 * This allocates the sw VEB structure and links it into the switch 13334 * It is possible and legal for this to be a duplicate of an already 13335 * existing VEB. It is also possible for both uplink and vsi seids 13336 * to be zero, in order to create a floating VEB. 13337 * 13338 * Returns pointer to the successfully allocated VEB sw struct on 13339 * success, otherwise returns NULL on failure. 13340 **/ 13341 struct i40e_veb *i40e_veb_setup(struct i40e_pf *pf, u16 flags, 13342 u16 uplink_seid, u16 vsi_seid, 13343 u8 enabled_tc) 13344 { 13345 struct i40e_veb *veb, *uplink_veb = NULL; 13346 int vsi_idx, veb_idx; 13347 int ret; 13348 13349 /* if one seid is 0, the other must be 0 to create a floating relay */ 13350 if ((uplink_seid == 0 || vsi_seid == 0) && 13351 (uplink_seid + vsi_seid != 0)) { 13352 dev_info(&pf->pdev->dev, 13353 "one, not both seid's are 0: uplink=%d vsi=%d\n", 13354 uplink_seid, vsi_seid); 13355 return NULL; 13356 } 13357 13358 /* make sure there is such a vsi and uplink */ 13359 for (vsi_idx = 0; vsi_idx < pf->num_alloc_vsi; vsi_idx++) 13360 if (pf->vsi[vsi_idx] && pf->vsi[vsi_idx]->seid == vsi_seid) 13361 break; 13362 if (vsi_idx == pf->num_alloc_vsi && vsi_seid != 0) { 13363 dev_info(&pf->pdev->dev, "vsi seid %d not found\n", 13364 vsi_seid); 13365 return NULL; 13366 } 13367 13368 if (uplink_seid && uplink_seid != pf->mac_seid) { 13369 for (veb_idx = 0; veb_idx < I40E_MAX_VEB; veb_idx++) { 13370 if (pf->veb[veb_idx] && 13371 pf->veb[veb_idx]->seid == uplink_seid) { 13372 uplink_veb = pf->veb[veb_idx]; 13373 break; 13374 } 13375 } 13376 if (!uplink_veb) { 13377 dev_info(&pf->pdev->dev, 13378 "uplink seid %d not found\n", uplink_seid); 13379 return NULL; 13380 } 13381 } 13382 13383 /* get veb sw struct */ 13384 veb_idx = i40e_veb_mem_alloc(pf); 13385 if (veb_idx < 0) 13386 goto err_alloc; 13387 veb = pf->veb[veb_idx]; 13388 veb->flags = flags; 13389 veb->uplink_seid = uplink_seid; 13390 veb->veb_idx = (uplink_veb ? uplink_veb->idx : I40E_NO_VEB); 13391 veb->enabled_tc = (enabled_tc ? enabled_tc : 0x1); 13392 13393 /* create the VEB in the switch */ 13394 ret = i40e_add_veb(veb, pf->vsi[vsi_idx]); 13395 if (ret) 13396 goto err_veb; 13397 if (vsi_idx == pf->lan_vsi) 13398 pf->lan_veb = veb->idx; 13399 13400 return veb; 13401 13402 err_veb: 13403 i40e_veb_clear(veb); 13404 err_alloc: 13405 return NULL; 13406 } 13407 13408 /** 13409 * i40e_setup_pf_switch_element - set PF vars based on switch type 13410 * @pf: board private structure 13411 * @ele: element we are building info from 13412 * @num_reported: total number of elements 13413 * @printconfig: should we print the contents 13414 * 13415 * helper function to assist in extracting a few useful SEID values. 13416 **/ 13417 static void i40e_setup_pf_switch_element(struct i40e_pf *pf, 13418 struct i40e_aqc_switch_config_element_resp *ele, 13419 u16 num_reported, bool printconfig) 13420 { 13421 u16 downlink_seid = le16_to_cpu(ele->downlink_seid); 13422 u16 uplink_seid = le16_to_cpu(ele->uplink_seid); 13423 u8 element_type = ele->element_type; 13424 u16 seid = le16_to_cpu(ele->seid); 13425 13426 if (printconfig) 13427 dev_info(&pf->pdev->dev, 13428 "type=%d seid=%d uplink=%d downlink=%d\n", 13429 element_type, seid, uplink_seid, downlink_seid); 13430 13431 switch (element_type) { 13432 case I40E_SWITCH_ELEMENT_TYPE_MAC: 13433 pf->mac_seid = seid; 13434 break; 13435 case I40E_SWITCH_ELEMENT_TYPE_VEB: 13436 /* Main VEB? */ 13437 if (uplink_seid != pf->mac_seid) 13438 break; 13439 if (pf->lan_veb == I40E_NO_VEB) { 13440 int v; 13441 13442 /* find existing or else empty VEB */ 13443 for (v = 0; v < I40E_MAX_VEB; v++) { 13444 if (pf->veb[v] && (pf->veb[v]->seid == seid)) { 13445 pf->lan_veb = v; 13446 break; 13447 } 13448 } 13449 if (pf->lan_veb == I40E_NO_VEB) { 13450 v = i40e_veb_mem_alloc(pf); 13451 if (v < 0) 13452 break; 13453 pf->lan_veb = v; 13454 } 13455 } 13456 13457 pf->veb[pf->lan_veb]->seid = seid; 13458 pf->veb[pf->lan_veb]->uplink_seid = pf->mac_seid; 13459 pf->veb[pf->lan_veb]->pf = pf; 13460 pf->veb[pf->lan_veb]->veb_idx = I40E_NO_VEB; 13461 break; 13462 case I40E_SWITCH_ELEMENT_TYPE_VSI: 13463 if (num_reported != 1) 13464 break; 13465 /* This is immediately after a reset so we can assume this is 13466 * the PF's VSI 13467 */ 13468 pf->mac_seid = uplink_seid; 13469 pf->pf_seid = downlink_seid; 13470 pf->main_vsi_seid = seid; 13471 if (printconfig) 13472 dev_info(&pf->pdev->dev, 13473 "pf_seid=%d main_vsi_seid=%d\n", 13474 pf->pf_seid, pf->main_vsi_seid); 13475 break; 13476 case I40E_SWITCH_ELEMENT_TYPE_PF: 13477 case I40E_SWITCH_ELEMENT_TYPE_VF: 13478 case I40E_SWITCH_ELEMENT_TYPE_EMP: 13479 case I40E_SWITCH_ELEMENT_TYPE_BMC: 13480 case I40E_SWITCH_ELEMENT_TYPE_PE: 13481 case I40E_SWITCH_ELEMENT_TYPE_PA: 13482 /* ignore these for now */ 13483 break; 13484 default: 13485 dev_info(&pf->pdev->dev, "unknown element type=%d seid=%d\n", 13486 element_type, seid); 13487 break; 13488 } 13489 } 13490 13491 /** 13492 * i40e_fetch_switch_configuration - Get switch config from firmware 13493 * @pf: board private structure 13494 * @printconfig: should we print the contents 13495 * 13496 * Get the current switch configuration from the device and 13497 * extract a few useful SEID values. 13498 **/ 13499 int i40e_fetch_switch_configuration(struct i40e_pf *pf, bool printconfig) 13500 { 13501 struct i40e_aqc_get_switch_config_resp *sw_config; 13502 u16 next_seid = 0; 13503 int ret = 0; 13504 u8 *aq_buf; 13505 int i; 13506 13507 aq_buf = kzalloc(I40E_AQ_LARGE_BUF, GFP_KERNEL); 13508 if (!aq_buf) 13509 return -ENOMEM; 13510 13511 sw_config = (struct i40e_aqc_get_switch_config_resp *)aq_buf; 13512 do { 13513 u16 num_reported, num_total; 13514 13515 ret = i40e_aq_get_switch_config(&pf->hw, sw_config, 13516 I40E_AQ_LARGE_BUF, 13517 &next_seid, NULL); 13518 if (ret) { 13519 dev_info(&pf->pdev->dev, 13520 "get switch config failed err %s aq_err %s\n", 13521 i40e_stat_str(&pf->hw, ret), 13522 i40e_aq_str(&pf->hw, 13523 pf->hw.aq.asq_last_status)); 13524 kfree(aq_buf); 13525 return -ENOENT; 13526 } 13527 13528 num_reported = le16_to_cpu(sw_config->header.num_reported); 13529 num_total = le16_to_cpu(sw_config->header.num_total); 13530 13531 if (printconfig) 13532 dev_info(&pf->pdev->dev, 13533 "header: %d reported %d total\n", 13534 num_reported, num_total); 13535 13536 for (i = 0; i < num_reported; i++) { 13537 struct i40e_aqc_switch_config_element_resp *ele = 13538 &sw_config->element[i]; 13539 13540 i40e_setup_pf_switch_element(pf, ele, num_reported, 13541 printconfig); 13542 } 13543 } while (next_seid != 0); 13544 13545 kfree(aq_buf); 13546 return ret; 13547 } 13548 13549 /** 13550 * i40e_setup_pf_switch - Setup the HW switch on startup or after reset 13551 * @pf: board private structure 13552 * @reinit: if the Main VSI needs to re-initialized. 13553 * 13554 * Returns 0 on success, negative value on failure 13555 **/ 13556 static int i40e_setup_pf_switch(struct i40e_pf *pf, bool reinit) 13557 { 13558 u16 flags = 0; 13559 int ret; 13560 13561 /* find out what's out there already */ 13562 ret = i40e_fetch_switch_configuration(pf, false); 13563 if (ret) { 13564 dev_info(&pf->pdev->dev, 13565 "couldn't fetch switch config, err %s aq_err %s\n", 13566 i40e_stat_str(&pf->hw, ret), 13567 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 13568 return ret; 13569 } 13570 i40e_pf_reset_stats(pf); 13571 13572 /* set the switch config bit for the whole device to 13573 * support limited promisc or true promisc 13574 * when user requests promisc. The default is limited 13575 * promisc. 13576 */ 13577 13578 if ((pf->hw.pf_id == 0) && 13579 !(pf->flags & I40E_FLAG_TRUE_PROMISC_SUPPORT)) { 13580 flags = I40E_AQ_SET_SWITCH_CFG_PROMISC; 13581 pf->last_sw_conf_flags = flags; 13582 } 13583 13584 if (pf->hw.pf_id == 0) { 13585 u16 valid_flags; 13586 13587 valid_flags = I40E_AQ_SET_SWITCH_CFG_PROMISC; 13588 ret = i40e_aq_set_switch_config(&pf->hw, flags, valid_flags, 0, 13589 NULL); 13590 if (ret && pf->hw.aq.asq_last_status != I40E_AQ_RC_ESRCH) { 13591 dev_info(&pf->pdev->dev, 13592 "couldn't set switch config bits, err %s aq_err %s\n", 13593 i40e_stat_str(&pf->hw, ret), 13594 i40e_aq_str(&pf->hw, 13595 pf->hw.aq.asq_last_status)); 13596 /* not a fatal problem, just keep going */ 13597 } 13598 pf->last_sw_conf_valid_flags = valid_flags; 13599 } 13600 13601 /* first time setup */ 13602 if (pf->lan_vsi == I40E_NO_VSI || reinit) { 13603 struct i40e_vsi *vsi = NULL; 13604 u16 uplink_seid; 13605 13606 /* Set up the PF VSI associated with the PF's main VSI 13607 * that is already in the HW switch 13608 */ 13609 if (pf->lan_veb != I40E_NO_VEB && pf->veb[pf->lan_veb]) 13610 uplink_seid = pf->veb[pf->lan_veb]->seid; 13611 else 13612 uplink_seid = pf->mac_seid; 13613 if (pf->lan_vsi == I40E_NO_VSI) 13614 vsi = i40e_vsi_setup(pf, I40E_VSI_MAIN, uplink_seid, 0); 13615 else if (reinit) 13616 vsi = i40e_vsi_reinit_setup(pf->vsi[pf->lan_vsi]); 13617 if (!vsi) { 13618 dev_info(&pf->pdev->dev, "setup of MAIN VSI failed\n"); 13619 i40e_cloud_filter_exit(pf); 13620 i40e_fdir_teardown(pf); 13621 return -EAGAIN; 13622 } 13623 } else { 13624 /* force a reset of TC and queue layout configurations */ 13625 u8 enabled_tc = pf->vsi[pf->lan_vsi]->tc_config.enabled_tc; 13626 13627 pf->vsi[pf->lan_vsi]->tc_config.enabled_tc = 0; 13628 pf->vsi[pf->lan_vsi]->seid = pf->main_vsi_seid; 13629 i40e_vsi_config_tc(pf->vsi[pf->lan_vsi], enabled_tc); 13630 } 13631 i40e_vlan_stripping_disable(pf->vsi[pf->lan_vsi]); 13632 13633 i40e_fdir_sb_setup(pf); 13634 13635 /* Setup static PF queue filter control settings */ 13636 ret = i40e_setup_pf_filter_control(pf); 13637 if (ret) { 13638 dev_info(&pf->pdev->dev, "setup_pf_filter_control failed: %d\n", 13639 ret); 13640 /* Failure here should not stop continuing other steps */ 13641 } 13642 13643 /* enable RSS in the HW, even for only one queue, as the stack can use 13644 * the hash 13645 */ 13646 if ((pf->flags & I40E_FLAG_RSS_ENABLED)) 13647 i40e_pf_config_rss(pf); 13648 13649 /* fill in link information and enable LSE reporting */ 13650 i40e_link_event(pf); 13651 13652 /* Initialize user-specific link properties */ 13653 pf->fc_autoneg_status = ((pf->hw.phy.link_info.an_info & 13654 I40E_AQ_AN_COMPLETED) ? true : false); 13655 13656 i40e_ptp_init(pf); 13657 13658 /* repopulate tunnel port filters */ 13659 i40e_sync_udp_filters(pf); 13660 13661 return ret; 13662 } 13663 13664 /** 13665 * i40e_determine_queue_usage - Work out queue distribution 13666 * @pf: board private structure 13667 **/ 13668 static void i40e_determine_queue_usage(struct i40e_pf *pf) 13669 { 13670 int queues_left; 13671 int q_max; 13672 13673 pf->num_lan_qps = 0; 13674 13675 /* Find the max queues to be put into basic use. We'll always be 13676 * using TC0, whether or not DCB is running, and TC0 will get the 13677 * big RSS set. 13678 */ 13679 queues_left = pf->hw.func_caps.num_tx_qp; 13680 13681 if ((queues_left == 1) || 13682 !(pf->flags & I40E_FLAG_MSIX_ENABLED)) { 13683 /* one qp for PF, no queues for anything else */ 13684 queues_left = 0; 13685 pf->alloc_rss_size = pf->num_lan_qps = 1; 13686 13687 /* make sure all the fancies are disabled */ 13688 pf->flags &= ~(I40E_FLAG_RSS_ENABLED | 13689 I40E_FLAG_IWARP_ENABLED | 13690 I40E_FLAG_FD_SB_ENABLED | 13691 I40E_FLAG_FD_ATR_ENABLED | 13692 I40E_FLAG_DCB_CAPABLE | 13693 I40E_FLAG_DCB_ENABLED | 13694 I40E_FLAG_SRIOV_ENABLED | 13695 I40E_FLAG_VMDQ_ENABLED); 13696 pf->flags |= I40E_FLAG_FD_SB_INACTIVE; 13697 } else if (!(pf->flags & (I40E_FLAG_RSS_ENABLED | 13698 I40E_FLAG_FD_SB_ENABLED | 13699 I40E_FLAG_FD_ATR_ENABLED | 13700 I40E_FLAG_DCB_CAPABLE))) { 13701 /* one qp for PF */ 13702 pf->alloc_rss_size = pf->num_lan_qps = 1; 13703 queues_left -= pf->num_lan_qps; 13704 13705 pf->flags &= ~(I40E_FLAG_RSS_ENABLED | 13706 I40E_FLAG_IWARP_ENABLED | 13707 I40E_FLAG_FD_SB_ENABLED | 13708 I40E_FLAG_FD_ATR_ENABLED | 13709 I40E_FLAG_DCB_ENABLED | 13710 I40E_FLAG_VMDQ_ENABLED); 13711 pf->flags |= I40E_FLAG_FD_SB_INACTIVE; 13712 } else { 13713 /* Not enough queues for all TCs */ 13714 if ((pf->flags & I40E_FLAG_DCB_CAPABLE) && 13715 (queues_left < I40E_MAX_TRAFFIC_CLASS)) { 13716 pf->flags &= ~(I40E_FLAG_DCB_CAPABLE | 13717 I40E_FLAG_DCB_ENABLED); 13718 dev_info(&pf->pdev->dev, "not enough queues for DCB. DCB is disabled.\n"); 13719 } 13720 13721 /* limit lan qps to the smaller of qps, cpus or msix */ 13722 q_max = max_t(int, pf->rss_size_max, num_online_cpus()); 13723 q_max = min_t(int, q_max, pf->hw.func_caps.num_tx_qp); 13724 q_max = min_t(int, q_max, pf->hw.func_caps.num_msix_vectors); 13725 pf->num_lan_qps = q_max; 13726 13727 queues_left -= pf->num_lan_qps; 13728 } 13729 13730 if (pf->flags & I40E_FLAG_FD_SB_ENABLED) { 13731 if (queues_left > 1) { 13732 queues_left -= 1; /* save 1 queue for FD */ 13733 } else { 13734 pf->flags &= ~I40E_FLAG_FD_SB_ENABLED; 13735 pf->flags |= I40E_FLAG_FD_SB_INACTIVE; 13736 dev_info(&pf->pdev->dev, "not enough queues for Flow Director. Flow Director feature is disabled\n"); 13737 } 13738 } 13739 13740 if ((pf->flags & I40E_FLAG_SRIOV_ENABLED) && 13741 pf->num_vf_qps && pf->num_req_vfs && queues_left) { 13742 pf->num_req_vfs = min_t(int, pf->num_req_vfs, 13743 (queues_left / pf->num_vf_qps)); 13744 queues_left -= (pf->num_req_vfs * pf->num_vf_qps); 13745 } 13746 13747 if ((pf->flags & I40E_FLAG_VMDQ_ENABLED) && 13748 pf->num_vmdq_vsis && pf->num_vmdq_qps && queues_left) { 13749 pf->num_vmdq_vsis = min_t(int, pf->num_vmdq_vsis, 13750 (queues_left / pf->num_vmdq_qps)); 13751 queues_left -= (pf->num_vmdq_vsis * pf->num_vmdq_qps); 13752 } 13753 13754 pf->queues_left = queues_left; 13755 dev_dbg(&pf->pdev->dev, 13756 "qs_avail=%d FD SB=%d lan_qs=%d lan_tc0=%d vf=%d*%d vmdq=%d*%d, remaining=%d\n", 13757 pf->hw.func_caps.num_tx_qp, 13758 !!(pf->flags & I40E_FLAG_FD_SB_ENABLED), 13759 pf->num_lan_qps, pf->alloc_rss_size, pf->num_req_vfs, 13760 pf->num_vf_qps, pf->num_vmdq_vsis, pf->num_vmdq_qps, 13761 queues_left); 13762 } 13763 13764 /** 13765 * i40e_setup_pf_filter_control - Setup PF static filter control 13766 * @pf: PF to be setup 13767 * 13768 * i40e_setup_pf_filter_control sets up a PF's initial filter control 13769 * settings. If PE/FCoE are enabled then it will also set the per PF 13770 * based filter sizes required for them. It also enables Flow director, 13771 * ethertype and macvlan type filter settings for the pf. 13772 * 13773 * Returns 0 on success, negative on failure 13774 **/ 13775 static int i40e_setup_pf_filter_control(struct i40e_pf *pf) 13776 { 13777 struct i40e_filter_control_settings *settings = &pf->filter_settings; 13778 13779 settings->hash_lut_size = I40E_HASH_LUT_SIZE_128; 13780 13781 /* Flow Director is enabled */ 13782 if (pf->flags & (I40E_FLAG_FD_SB_ENABLED | I40E_FLAG_FD_ATR_ENABLED)) 13783 settings->enable_fdir = true; 13784 13785 /* Ethtype and MACVLAN filters enabled for PF */ 13786 settings->enable_ethtype = true; 13787 settings->enable_macvlan = true; 13788 13789 if (i40e_set_filter_control(&pf->hw, settings)) 13790 return -ENOENT; 13791 13792 return 0; 13793 } 13794 13795 #define INFO_STRING_LEN 255 13796 #define REMAIN(__x) (INFO_STRING_LEN - (__x)) 13797 static void i40e_print_features(struct i40e_pf *pf) 13798 { 13799 struct i40e_hw *hw = &pf->hw; 13800 char *buf; 13801 int i; 13802 13803 buf = kmalloc(INFO_STRING_LEN, GFP_KERNEL); 13804 if (!buf) 13805 return; 13806 13807 i = snprintf(buf, INFO_STRING_LEN, "Features: PF-id[%d]", hw->pf_id); 13808 #ifdef CONFIG_PCI_IOV 13809 i += snprintf(&buf[i], REMAIN(i), " VFs: %d", pf->num_req_vfs); 13810 #endif 13811 i += snprintf(&buf[i], REMAIN(i), " VSIs: %d QP: %d", 13812 pf->hw.func_caps.num_vsis, 13813 pf->vsi[pf->lan_vsi]->num_queue_pairs); 13814 if (pf->flags & I40E_FLAG_RSS_ENABLED) 13815 i += snprintf(&buf[i], REMAIN(i), " RSS"); 13816 if (pf->flags & I40E_FLAG_FD_ATR_ENABLED) 13817 i += snprintf(&buf[i], REMAIN(i), " FD_ATR"); 13818 if (pf->flags & I40E_FLAG_FD_SB_ENABLED) { 13819 i += snprintf(&buf[i], REMAIN(i), " FD_SB"); 13820 i += snprintf(&buf[i], REMAIN(i), " NTUPLE"); 13821 } 13822 if (pf->flags & I40E_FLAG_DCB_CAPABLE) 13823 i += snprintf(&buf[i], REMAIN(i), " DCB"); 13824 i += snprintf(&buf[i], REMAIN(i), " VxLAN"); 13825 i += snprintf(&buf[i], REMAIN(i), " Geneve"); 13826 if (pf->flags & I40E_FLAG_PTP) 13827 i += snprintf(&buf[i], REMAIN(i), " PTP"); 13828 if (pf->flags & I40E_FLAG_VEB_MODE_ENABLED) 13829 i += snprintf(&buf[i], REMAIN(i), " VEB"); 13830 else 13831 i += snprintf(&buf[i], REMAIN(i), " VEPA"); 13832 13833 dev_info(&pf->pdev->dev, "%s\n", buf); 13834 kfree(buf); 13835 WARN_ON(i > INFO_STRING_LEN); 13836 } 13837 13838 /** 13839 * i40e_get_platform_mac_addr - get platform-specific MAC address 13840 * @pdev: PCI device information struct 13841 * @pf: board private structure 13842 * 13843 * Look up the MAC address for the device. First we'll try 13844 * eth_platform_get_mac_address, which will check Open Firmware, or arch 13845 * specific fallback. Otherwise, we'll default to the stored value in 13846 * firmware. 13847 **/ 13848 static void i40e_get_platform_mac_addr(struct pci_dev *pdev, struct i40e_pf *pf) 13849 { 13850 if (eth_platform_get_mac_address(&pdev->dev, pf->hw.mac.addr)) 13851 i40e_get_mac_addr(&pf->hw, pf->hw.mac.addr); 13852 } 13853 13854 /** 13855 * i40e_probe - Device initialization routine 13856 * @pdev: PCI device information struct 13857 * @ent: entry in i40e_pci_tbl 13858 * 13859 * i40e_probe initializes a PF identified by a pci_dev structure. 13860 * The OS initialization, configuring of the PF private structure, 13861 * and a hardware reset occur. 13862 * 13863 * Returns 0 on success, negative on failure 13864 **/ 13865 static int i40e_probe(struct pci_dev *pdev, const struct pci_device_id *ent) 13866 { 13867 struct i40e_aq_get_phy_abilities_resp abilities; 13868 struct i40e_pf *pf; 13869 struct i40e_hw *hw; 13870 static u16 pfs_found; 13871 u16 wol_nvm_bits; 13872 u16 link_status; 13873 int err; 13874 u32 val; 13875 u32 i; 13876 u8 set_fc_aq_fail; 13877 13878 err = pci_enable_device_mem(pdev); 13879 if (err) 13880 return err; 13881 13882 /* set up for high or low dma */ 13883 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); 13884 if (err) { 13885 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)); 13886 if (err) { 13887 dev_err(&pdev->dev, 13888 "DMA configuration failed: 0x%x\n", err); 13889 goto err_dma; 13890 } 13891 } 13892 13893 /* set up pci connections */ 13894 err = pci_request_mem_regions(pdev, i40e_driver_name); 13895 if (err) { 13896 dev_info(&pdev->dev, 13897 "pci_request_selected_regions failed %d\n", err); 13898 goto err_pci_reg; 13899 } 13900 13901 pci_enable_pcie_error_reporting(pdev); 13902 pci_set_master(pdev); 13903 13904 /* Now that we have a PCI connection, we need to do the 13905 * low level device setup. This is primarily setting up 13906 * the Admin Queue structures and then querying for the 13907 * device's current profile information. 13908 */ 13909 pf = kzalloc(sizeof(*pf), GFP_KERNEL); 13910 if (!pf) { 13911 err = -ENOMEM; 13912 goto err_pf_alloc; 13913 } 13914 pf->next_vsi = 0; 13915 pf->pdev = pdev; 13916 set_bit(__I40E_DOWN, pf->state); 13917 13918 hw = &pf->hw; 13919 hw->back = pf; 13920 13921 pf->ioremap_len = min_t(int, pci_resource_len(pdev, 0), 13922 I40E_MAX_CSR_SPACE); 13923 13924 hw->hw_addr = ioremap(pci_resource_start(pdev, 0), pf->ioremap_len); 13925 if (!hw->hw_addr) { 13926 err = -EIO; 13927 dev_info(&pdev->dev, "ioremap(0x%04x, 0x%04x) failed: 0x%x\n", 13928 (unsigned int)pci_resource_start(pdev, 0), 13929 pf->ioremap_len, err); 13930 goto err_ioremap; 13931 } 13932 hw->vendor_id = pdev->vendor; 13933 hw->device_id = pdev->device; 13934 pci_read_config_byte(pdev, PCI_REVISION_ID, &hw->revision_id); 13935 hw->subsystem_vendor_id = pdev->subsystem_vendor; 13936 hw->subsystem_device_id = pdev->subsystem_device; 13937 hw->bus.device = PCI_SLOT(pdev->devfn); 13938 hw->bus.func = PCI_FUNC(pdev->devfn); 13939 hw->bus.bus_id = pdev->bus->number; 13940 pf->instance = pfs_found; 13941 13942 /* Select something other than the 802.1ad ethertype for the 13943 * switch to use internally and drop on ingress. 13944 */ 13945 hw->switch_tag = 0xffff; 13946 hw->first_tag = ETH_P_8021AD; 13947 hw->second_tag = ETH_P_8021Q; 13948 13949 INIT_LIST_HEAD(&pf->l3_flex_pit_list); 13950 INIT_LIST_HEAD(&pf->l4_flex_pit_list); 13951 13952 /* set up the locks for the AQ, do this only once in probe 13953 * and destroy them only once in remove 13954 */ 13955 mutex_init(&hw->aq.asq_mutex); 13956 mutex_init(&hw->aq.arq_mutex); 13957 13958 pf->msg_enable = netif_msg_init(debug, 13959 NETIF_MSG_DRV | 13960 NETIF_MSG_PROBE | 13961 NETIF_MSG_LINK); 13962 if (debug < -1) 13963 pf->hw.debug_mask = debug; 13964 13965 /* do a special CORER for clearing PXE mode once at init */ 13966 if (hw->revision_id == 0 && 13967 (rd32(hw, I40E_GLLAN_RCTL_0) & I40E_GLLAN_RCTL_0_PXE_MODE_MASK)) { 13968 wr32(hw, I40E_GLGEN_RTRIG, I40E_GLGEN_RTRIG_CORER_MASK); 13969 i40e_flush(hw); 13970 msleep(200); 13971 pf->corer_count++; 13972 13973 i40e_clear_pxe_mode(hw); 13974 } 13975 13976 /* Reset here to make sure all is clean and to define PF 'n' */ 13977 i40e_clear_hw(hw); 13978 err = i40e_pf_reset(hw); 13979 if (err) { 13980 dev_info(&pdev->dev, "Initial pf_reset failed: %d\n", err); 13981 goto err_pf_reset; 13982 } 13983 pf->pfr_count++; 13984 13985 hw->aq.num_arq_entries = I40E_AQ_LEN; 13986 hw->aq.num_asq_entries = I40E_AQ_LEN; 13987 hw->aq.arq_buf_size = I40E_MAX_AQ_BUF_SIZE; 13988 hw->aq.asq_buf_size = I40E_MAX_AQ_BUF_SIZE; 13989 pf->adminq_work_limit = I40E_AQ_WORK_LIMIT; 13990 13991 snprintf(pf->int_name, sizeof(pf->int_name) - 1, 13992 "%s-%s:misc", 13993 dev_driver_string(&pf->pdev->dev), dev_name(&pdev->dev)); 13994 13995 err = i40e_init_shared_code(hw); 13996 if (err) { 13997 dev_warn(&pdev->dev, "unidentified MAC or BLANK NVM: %d\n", 13998 err); 13999 goto err_pf_reset; 14000 } 14001 14002 /* set up a default setting for link flow control */ 14003 pf->hw.fc.requested_mode = I40E_FC_NONE; 14004 14005 err = i40e_init_adminq(hw); 14006 if (err) { 14007 if (err == I40E_ERR_FIRMWARE_API_VERSION) 14008 dev_info(&pdev->dev, 14009 "The driver for the device stopped because the NVM image is newer than expected. You must install the most recent version of the network driver.\n"); 14010 else 14011 dev_info(&pdev->dev, 14012 "The driver for the device stopped because the device firmware failed to init. Try updating your NVM image.\n"); 14013 14014 goto err_pf_reset; 14015 } 14016 i40e_get_oem_version(hw); 14017 14018 /* provide nvm, fw, api versions */ 14019 dev_info(&pdev->dev, "fw %d.%d.%05d api %d.%d nvm %s\n", 14020 hw->aq.fw_maj_ver, hw->aq.fw_min_ver, hw->aq.fw_build, 14021 hw->aq.api_maj_ver, hw->aq.api_min_ver, 14022 i40e_nvm_version_str(hw)); 14023 14024 if (hw->aq.api_maj_ver == I40E_FW_API_VERSION_MAJOR && 14025 hw->aq.api_min_ver > I40E_FW_MINOR_VERSION(hw)) 14026 dev_info(&pdev->dev, 14027 "The driver for the device detected a newer version of the NVM image than expected. Please install the most recent version of the network driver.\n"); 14028 else if (hw->aq.api_maj_ver == 1 && hw->aq.api_min_ver < 4) 14029 dev_info(&pdev->dev, 14030 "The driver for the device detected an older version of the NVM image than expected. Please update the NVM image.\n"); 14031 14032 i40e_verify_eeprom(pf); 14033 14034 /* Rev 0 hardware was never productized */ 14035 if (hw->revision_id < 1) 14036 dev_warn(&pdev->dev, "This device is a pre-production adapter/LOM. Please be aware there may be issues with your hardware. If you are experiencing problems please contact your Intel or hardware representative who provided you with this hardware.\n"); 14037 14038 i40e_clear_pxe_mode(hw); 14039 err = i40e_get_capabilities(pf, i40e_aqc_opc_list_func_capabilities); 14040 if (err) 14041 goto err_adminq_setup; 14042 14043 err = i40e_sw_init(pf); 14044 if (err) { 14045 dev_info(&pdev->dev, "sw_init failed: %d\n", err); 14046 goto err_sw_init; 14047 } 14048 14049 err = i40e_init_lan_hmc(hw, hw->func_caps.num_tx_qp, 14050 hw->func_caps.num_rx_qp, 0, 0); 14051 if (err) { 14052 dev_info(&pdev->dev, "init_lan_hmc failed: %d\n", err); 14053 goto err_init_lan_hmc; 14054 } 14055 14056 err = i40e_configure_lan_hmc(hw, I40E_HMC_MODEL_DIRECT_ONLY); 14057 if (err) { 14058 dev_info(&pdev->dev, "configure_lan_hmc failed: %d\n", err); 14059 err = -ENOENT; 14060 goto err_configure_lan_hmc; 14061 } 14062 14063 /* Disable LLDP for NICs that have firmware versions lower than v4.3. 14064 * Ignore error return codes because if it was already disabled via 14065 * hardware settings this will fail 14066 */ 14067 if (pf->hw_features & I40E_HW_STOP_FW_LLDP) { 14068 dev_info(&pdev->dev, "Stopping firmware LLDP agent.\n"); 14069 i40e_aq_stop_lldp(hw, true, NULL); 14070 } 14071 14072 /* allow a platform config to override the HW addr */ 14073 i40e_get_platform_mac_addr(pdev, pf); 14074 14075 if (!is_valid_ether_addr(hw->mac.addr)) { 14076 dev_info(&pdev->dev, "invalid MAC address %pM\n", hw->mac.addr); 14077 err = -EIO; 14078 goto err_mac_addr; 14079 } 14080 dev_info(&pdev->dev, "MAC address: %pM\n", hw->mac.addr); 14081 ether_addr_copy(hw->mac.perm_addr, hw->mac.addr); 14082 i40e_get_port_mac_addr(hw, hw->mac.port_addr); 14083 if (is_valid_ether_addr(hw->mac.port_addr)) 14084 pf->hw_features |= I40E_HW_PORT_ID_VALID; 14085 14086 pci_set_drvdata(pdev, pf); 14087 pci_save_state(pdev); 14088 14089 /* Enable FW to write default DCB config on link-up */ 14090 i40e_aq_set_dcb_parameters(hw, true, NULL); 14091 14092 #ifdef CONFIG_I40E_DCB 14093 err = i40e_init_pf_dcb(pf); 14094 if (err) { 14095 dev_info(&pdev->dev, "DCB init failed %d, disabled\n", err); 14096 pf->flags &= ~(I40E_FLAG_DCB_CAPABLE | I40E_FLAG_DCB_ENABLED); 14097 /* Continue without DCB enabled */ 14098 } 14099 #endif /* CONFIG_I40E_DCB */ 14100 14101 /* set up periodic task facility */ 14102 timer_setup(&pf->service_timer, i40e_service_timer, 0); 14103 pf->service_timer_period = HZ; 14104 14105 INIT_WORK(&pf->service_task, i40e_service_task); 14106 clear_bit(__I40E_SERVICE_SCHED, pf->state); 14107 14108 /* NVM bit on means WoL disabled for the port */ 14109 i40e_read_nvm_word(hw, I40E_SR_NVM_WAKE_ON_LAN, &wol_nvm_bits); 14110 if (BIT (hw->port) & wol_nvm_bits || hw->partition_id != 1) 14111 pf->wol_en = false; 14112 else 14113 pf->wol_en = true; 14114 device_set_wakeup_enable(&pf->pdev->dev, pf->wol_en); 14115 14116 /* set up the main switch operations */ 14117 i40e_determine_queue_usage(pf); 14118 err = i40e_init_interrupt_scheme(pf); 14119 if (err) 14120 goto err_switch_setup; 14121 14122 /* The number of VSIs reported by the FW is the minimum guaranteed 14123 * to us; HW supports far more and we share the remaining pool with 14124 * the other PFs. We allocate space for more than the guarantee with 14125 * the understanding that we might not get them all later. 14126 */ 14127 if (pf->hw.func_caps.num_vsis < I40E_MIN_VSI_ALLOC) 14128 pf->num_alloc_vsi = I40E_MIN_VSI_ALLOC; 14129 else 14130 pf->num_alloc_vsi = pf->hw.func_caps.num_vsis; 14131 14132 /* Set up the *vsi struct and our local tracking of the MAIN PF vsi. */ 14133 pf->vsi = kcalloc(pf->num_alloc_vsi, sizeof(struct i40e_vsi *), 14134 GFP_KERNEL); 14135 if (!pf->vsi) { 14136 err = -ENOMEM; 14137 goto err_switch_setup; 14138 } 14139 14140 #ifdef CONFIG_PCI_IOV 14141 /* prep for VF support */ 14142 if ((pf->flags & I40E_FLAG_SRIOV_ENABLED) && 14143 (pf->flags & I40E_FLAG_MSIX_ENABLED) && 14144 !test_bit(__I40E_BAD_EEPROM, pf->state)) { 14145 if (pci_num_vf(pdev)) 14146 pf->flags |= I40E_FLAG_VEB_MODE_ENABLED; 14147 } 14148 #endif 14149 err = i40e_setup_pf_switch(pf, false); 14150 if (err) { 14151 dev_info(&pdev->dev, "setup_pf_switch failed: %d\n", err); 14152 goto err_vsis; 14153 } 14154 INIT_LIST_HEAD(&pf->vsi[pf->lan_vsi]->ch_list); 14155 14156 /* Make sure flow control is set according to current settings */ 14157 err = i40e_set_fc(hw, &set_fc_aq_fail, true); 14158 if (set_fc_aq_fail & I40E_SET_FC_AQ_FAIL_GET) 14159 dev_dbg(&pf->pdev->dev, 14160 "Set fc with err %s aq_err %s on get_phy_cap\n", 14161 i40e_stat_str(hw, err), 14162 i40e_aq_str(hw, hw->aq.asq_last_status)); 14163 if (set_fc_aq_fail & I40E_SET_FC_AQ_FAIL_SET) 14164 dev_dbg(&pf->pdev->dev, 14165 "Set fc with err %s aq_err %s on set_phy_config\n", 14166 i40e_stat_str(hw, err), 14167 i40e_aq_str(hw, hw->aq.asq_last_status)); 14168 if (set_fc_aq_fail & I40E_SET_FC_AQ_FAIL_UPDATE) 14169 dev_dbg(&pf->pdev->dev, 14170 "Set fc with err %s aq_err %s on get_link_info\n", 14171 i40e_stat_str(hw, err), 14172 i40e_aq_str(hw, hw->aq.asq_last_status)); 14173 14174 /* if FDIR VSI was set up, start it now */ 14175 for (i = 0; i < pf->num_alloc_vsi; i++) { 14176 if (pf->vsi[i] && pf->vsi[i]->type == I40E_VSI_FDIR) { 14177 i40e_vsi_open(pf->vsi[i]); 14178 break; 14179 } 14180 } 14181 14182 /* The driver only wants link up/down and module qualification 14183 * reports from firmware. Note the negative logic. 14184 */ 14185 err = i40e_aq_set_phy_int_mask(&pf->hw, 14186 ~(I40E_AQ_EVENT_LINK_UPDOWN | 14187 I40E_AQ_EVENT_MEDIA_NA | 14188 I40E_AQ_EVENT_MODULE_QUAL_FAIL), NULL); 14189 if (err) 14190 dev_info(&pf->pdev->dev, "set phy mask fail, err %s aq_err %s\n", 14191 i40e_stat_str(&pf->hw, err), 14192 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 14193 14194 /* Reconfigure hardware for allowing smaller MSS in the case 14195 * of TSO, so that we avoid the MDD being fired and causing 14196 * a reset in the case of small MSS+TSO. 14197 */ 14198 val = rd32(hw, I40E_REG_MSS); 14199 if ((val & I40E_REG_MSS_MIN_MASK) > I40E_64BYTE_MSS) { 14200 val &= ~I40E_REG_MSS_MIN_MASK; 14201 val |= I40E_64BYTE_MSS; 14202 wr32(hw, I40E_REG_MSS, val); 14203 } 14204 14205 if (pf->hw_features & I40E_HW_RESTART_AUTONEG) { 14206 msleep(75); 14207 err = i40e_aq_set_link_restart_an(&pf->hw, true, NULL); 14208 if (err) 14209 dev_info(&pf->pdev->dev, "link restart failed, err %s aq_err %s\n", 14210 i40e_stat_str(&pf->hw, err), 14211 i40e_aq_str(&pf->hw, 14212 pf->hw.aq.asq_last_status)); 14213 } 14214 /* The main driver is (mostly) up and happy. We need to set this state 14215 * before setting up the misc vector or we get a race and the vector 14216 * ends up disabled forever. 14217 */ 14218 clear_bit(__I40E_DOWN, pf->state); 14219 14220 /* In case of MSIX we are going to setup the misc vector right here 14221 * to handle admin queue events etc. In case of legacy and MSI 14222 * the misc functionality and queue processing is combined in 14223 * the same vector and that gets setup at open. 14224 */ 14225 if (pf->flags & I40E_FLAG_MSIX_ENABLED) { 14226 err = i40e_setup_misc_vector(pf); 14227 if (err) { 14228 dev_info(&pdev->dev, 14229 "setup of misc vector failed: %d\n", err); 14230 goto err_vsis; 14231 } 14232 } 14233 14234 #ifdef CONFIG_PCI_IOV 14235 /* prep for VF support */ 14236 if ((pf->flags & I40E_FLAG_SRIOV_ENABLED) && 14237 (pf->flags & I40E_FLAG_MSIX_ENABLED) && 14238 !test_bit(__I40E_BAD_EEPROM, pf->state)) { 14239 /* disable link interrupts for VFs */ 14240 val = rd32(hw, I40E_PFGEN_PORTMDIO_NUM); 14241 val &= ~I40E_PFGEN_PORTMDIO_NUM_VFLINK_STAT_ENA_MASK; 14242 wr32(hw, I40E_PFGEN_PORTMDIO_NUM, val); 14243 i40e_flush(hw); 14244 14245 if (pci_num_vf(pdev)) { 14246 dev_info(&pdev->dev, 14247 "Active VFs found, allocating resources.\n"); 14248 err = i40e_alloc_vfs(pf, pci_num_vf(pdev)); 14249 if (err) 14250 dev_info(&pdev->dev, 14251 "Error %d allocating resources for existing VFs\n", 14252 err); 14253 } 14254 } 14255 #endif /* CONFIG_PCI_IOV */ 14256 14257 if (pf->flags & I40E_FLAG_IWARP_ENABLED) { 14258 pf->iwarp_base_vector = i40e_get_lump(pf, pf->irq_pile, 14259 pf->num_iwarp_msix, 14260 I40E_IWARP_IRQ_PILE_ID); 14261 if (pf->iwarp_base_vector < 0) { 14262 dev_info(&pdev->dev, 14263 "failed to get tracking for %d vectors for IWARP err=%d\n", 14264 pf->num_iwarp_msix, pf->iwarp_base_vector); 14265 pf->flags &= ~I40E_FLAG_IWARP_ENABLED; 14266 } 14267 } 14268 14269 i40e_dbg_pf_init(pf); 14270 14271 /* tell the firmware that we're starting */ 14272 i40e_send_version(pf); 14273 14274 /* since everything's happy, start the service_task timer */ 14275 mod_timer(&pf->service_timer, 14276 round_jiffies(jiffies + pf->service_timer_period)); 14277 14278 /* add this PF to client device list and launch a client service task */ 14279 if (pf->flags & I40E_FLAG_IWARP_ENABLED) { 14280 err = i40e_lan_add_device(pf); 14281 if (err) 14282 dev_info(&pdev->dev, "Failed to add PF to client API service list: %d\n", 14283 err); 14284 } 14285 14286 #define PCI_SPEED_SIZE 8 14287 #define PCI_WIDTH_SIZE 8 14288 /* Devices on the IOSF bus do not have this information 14289 * and will report PCI Gen 1 x 1 by default so don't bother 14290 * checking them. 14291 */ 14292 if (!(pf->hw_features & I40E_HW_NO_PCI_LINK_CHECK)) { 14293 char speed[PCI_SPEED_SIZE] = "Unknown"; 14294 char width[PCI_WIDTH_SIZE] = "Unknown"; 14295 14296 /* Get the negotiated link width and speed from PCI config 14297 * space 14298 */ 14299 pcie_capability_read_word(pf->pdev, PCI_EXP_LNKSTA, 14300 &link_status); 14301 14302 i40e_set_pci_config_data(hw, link_status); 14303 14304 switch (hw->bus.speed) { 14305 case i40e_bus_speed_8000: 14306 strncpy(speed, "8.0", PCI_SPEED_SIZE); break; 14307 case i40e_bus_speed_5000: 14308 strncpy(speed, "5.0", PCI_SPEED_SIZE); break; 14309 case i40e_bus_speed_2500: 14310 strncpy(speed, "2.5", PCI_SPEED_SIZE); break; 14311 default: 14312 break; 14313 } 14314 switch (hw->bus.width) { 14315 case i40e_bus_width_pcie_x8: 14316 strncpy(width, "8", PCI_WIDTH_SIZE); break; 14317 case i40e_bus_width_pcie_x4: 14318 strncpy(width, "4", PCI_WIDTH_SIZE); break; 14319 case i40e_bus_width_pcie_x2: 14320 strncpy(width, "2", PCI_WIDTH_SIZE); break; 14321 case i40e_bus_width_pcie_x1: 14322 strncpy(width, "1", PCI_WIDTH_SIZE); break; 14323 default: 14324 break; 14325 } 14326 14327 dev_info(&pdev->dev, "PCI-Express: Speed %sGT/s Width x%s\n", 14328 speed, width); 14329 14330 if (hw->bus.width < i40e_bus_width_pcie_x8 || 14331 hw->bus.speed < i40e_bus_speed_8000) { 14332 dev_warn(&pdev->dev, "PCI-Express bandwidth available for this device may be insufficient for optimal performance.\n"); 14333 dev_warn(&pdev->dev, "Please move the device to a different PCI-e link with more lanes and/or higher transfer rate.\n"); 14334 } 14335 } 14336 14337 /* get the requested speeds from the fw */ 14338 err = i40e_aq_get_phy_capabilities(hw, false, false, &abilities, NULL); 14339 if (err) 14340 dev_dbg(&pf->pdev->dev, "get requested speeds ret = %s last_status = %s\n", 14341 i40e_stat_str(&pf->hw, err), 14342 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 14343 pf->hw.phy.link_info.requested_speeds = abilities.link_speed; 14344 14345 /* get the supported phy types from the fw */ 14346 err = i40e_aq_get_phy_capabilities(hw, false, true, &abilities, NULL); 14347 if (err) 14348 dev_dbg(&pf->pdev->dev, "get supported phy types ret = %s last_status = %s\n", 14349 i40e_stat_str(&pf->hw, err), 14350 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 14351 14352 /* Add a filter to drop all Flow control frames from any VSI from being 14353 * transmitted. By doing so we stop a malicious VF from sending out 14354 * PAUSE or PFC frames and potentially controlling traffic for other 14355 * PF/VF VSIs. 14356 * The FW can still send Flow control frames if enabled. 14357 */ 14358 i40e_add_filter_to_drop_tx_flow_control_frames(&pf->hw, 14359 pf->main_vsi_seid); 14360 14361 if ((pf->hw.device_id == I40E_DEV_ID_10G_BASE_T) || 14362 (pf->hw.device_id == I40E_DEV_ID_10G_BASE_T4)) 14363 pf->hw_features |= I40E_HW_PHY_CONTROLS_LEDS; 14364 if (pf->hw.device_id == I40E_DEV_ID_SFP_I_X722) 14365 pf->hw_features |= I40E_HW_HAVE_CRT_RETIMER; 14366 /* print a string summarizing features */ 14367 i40e_print_features(pf); 14368 14369 return 0; 14370 14371 /* Unwind what we've done if something failed in the setup */ 14372 err_vsis: 14373 set_bit(__I40E_DOWN, pf->state); 14374 i40e_clear_interrupt_scheme(pf); 14375 kfree(pf->vsi); 14376 err_switch_setup: 14377 i40e_reset_interrupt_capability(pf); 14378 del_timer_sync(&pf->service_timer); 14379 err_mac_addr: 14380 err_configure_lan_hmc: 14381 (void)i40e_shutdown_lan_hmc(hw); 14382 err_init_lan_hmc: 14383 kfree(pf->qp_pile); 14384 err_sw_init: 14385 err_adminq_setup: 14386 err_pf_reset: 14387 iounmap(hw->hw_addr); 14388 err_ioremap: 14389 kfree(pf); 14390 err_pf_alloc: 14391 pci_disable_pcie_error_reporting(pdev); 14392 pci_release_mem_regions(pdev); 14393 err_pci_reg: 14394 err_dma: 14395 pci_disable_device(pdev); 14396 return err; 14397 } 14398 14399 /** 14400 * i40e_remove - Device removal routine 14401 * @pdev: PCI device information struct 14402 * 14403 * i40e_remove is called by the PCI subsystem to alert the driver 14404 * that is should release a PCI device. This could be caused by a 14405 * Hot-Plug event, or because the driver is going to be removed from 14406 * memory. 14407 **/ 14408 static void i40e_remove(struct pci_dev *pdev) 14409 { 14410 struct i40e_pf *pf = pci_get_drvdata(pdev); 14411 struct i40e_hw *hw = &pf->hw; 14412 i40e_status ret_code; 14413 int i; 14414 14415 i40e_dbg_pf_exit(pf); 14416 14417 i40e_ptp_stop(pf); 14418 14419 /* Disable RSS in hw */ 14420 i40e_write_rx_ctl(hw, I40E_PFQF_HENA(0), 0); 14421 i40e_write_rx_ctl(hw, I40E_PFQF_HENA(1), 0); 14422 14423 /* no more scheduling of any task */ 14424 set_bit(__I40E_SUSPENDED, pf->state); 14425 set_bit(__I40E_DOWN, pf->state); 14426 if (pf->service_timer.function) 14427 del_timer_sync(&pf->service_timer); 14428 if (pf->service_task.func) 14429 cancel_work_sync(&pf->service_task); 14430 14431 /* Client close must be called explicitly here because the timer 14432 * has been stopped. 14433 */ 14434 i40e_notify_client_of_netdev_close(pf->vsi[pf->lan_vsi], false); 14435 14436 if (pf->flags & I40E_FLAG_SRIOV_ENABLED) { 14437 i40e_free_vfs(pf); 14438 pf->flags &= ~I40E_FLAG_SRIOV_ENABLED; 14439 } 14440 14441 i40e_fdir_teardown(pf); 14442 14443 /* If there is a switch structure or any orphans, remove them. 14444 * This will leave only the PF's VSI remaining. 14445 */ 14446 for (i = 0; i < I40E_MAX_VEB; i++) { 14447 if (!pf->veb[i]) 14448 continue; 14449 14450 if (pf->veb[i]->uplink_seid == pf->mac_seid || 14451 pf->veb[i]->uplink_seid == 0) 14452 i40e_switch_branch_release(pf->veb[i]); 14453 } 14454 14455 /* Now we can shutdown the PF's VSI, just before we kill 14456 * adminq and hmc. 14457 */ 14458 if (pf->vsi[pf->lan_vsi]) 14459 i40e_vsi_release(pf->vsi[pf->lan_vsi]); 14460 14461 i40e_cloud_filter_exit(pf); 14462 14463 /* remove attached clients */ 14464 if (pf->flags & I40E_FLAG_IWARP_ENABLED) { 14465 ret_code = i40e_lan_del_device(pf); 14466 if (ret_code) 14467 dev_warn(&pdev->dev, "Failed to delete client device: %d\n", 14468 ret_code); 14469 } 14470 14471 /* shutdown and destroy the HMC */ 14472 if (hw->hmc.hmc_obj) { 14473 ret_code = i40e_shutdown_lan_hmc(hw); 14474 if (ret_code) 14475 dev_warn(&pdev->dev, 14476 "Failed to destroy the HMC resources: %d\n", 14477 ret_code); 14478 } 14479 14480 /* shutdown the adminq */ 14481 i40e_shutdown_adminq(hw); 14482 14483 /* destroy the locks only once, here */ 14484 mutex_destroy(&hw->aq.arq_mutex); 14485 mutex_destroy(&hw->aq.asq_mutex); 14486 14487 /* Clear all dynamic memory lists of rings, q_vectors, and VSIs */ 14488 rtnl_lock(); 14489 i40e_clear_interrupt_scheme(pf); 14490 for (i = 0; i < pf->num_alloc_vsi; i++) { 14491 if (pf->vsi[i]) { 14492 i40e_vsi_clear_rings(pf->vsi[i]); 14493 i40e_vsi_clear(pf->vsi[i]); 14494 pf->vsi[i] = NULL; 14495 } 14496 } 14497 rtnl_unlock(); 14498 14499 for (i = 0; i < I40E_MAX_VEB; i++) { 14500 kfree(pf->veb[i]); 14501 pf->veb[i] = NULL; 14502 } 14503 14504 kfree(pf->qp_pile); 14505 kfree(pf->vsi); 14506 14507 iounmap(hw->hw_addr); 14508 kfree(pf); 14509 pci_release_mem_regions(pdev); 14510 14511 pci_disable_pcie_error_reporting(pdev); 14512 pci_disable_device(pdev); 14513 } 14514 14515 /** 14516 * i40e_pci_error_detected - warning that something funky happened in PCI land 14517 * @pdev: PCI device information struct 14518 * @error: the type of PCI error 14519 * 14520 * Called to warn that something happened and the error handling steps 14521 * are in progress. Allows the driver to quiesce things, be ready for 14522 * remediation. 14523 **/ 14524 static pci_ers_result_t i40e_pci_error_detected(struct pci_dev *pdev, 14525 enum pci_channel_state error) 14526 { 14527 struct i40e_pf *pf = pci_get_drvdata(pdev); 14528 14529 dev_info(&pdev->dev, "%s: error %d\n", __func__, error); 14530 14531 if (!pf) { 14532 dev_info(&pdev->dev, 14533 "Cannot recover - error happened during device probe\n"); 14534 return PCI_ERS_RESULT_DISCONNECT; 14535 } 14536 14537 /* shutdown all operations */ 14538 if (!test_bit(__I40E_SUSPENDED, pf->state)) 14539 i40e_prep_for_reset(pf, false); 14540 14541 /* Request a slot reset */ 14542 return PCI_ERS_RESULT_NEED_RESET; 14543 } 14544 14545 /** 14546 * i40e_pci_error_slot_reset - a PCI slot reset just happened 14547 * @pdev: PCI device information struct 14548 * 14549 * Called to find if the driver can work with the device now that 14550 * the pci slot has been reset. If a basic connection seems good 14551 * (registers are readable and have sane content) then return a 14552 * happy little PCI_ERS_RESULT_xxx. 14553 **/ 14554 static pci_ers_result_t i40e_pci_error_slot_reset(struct pci_dev *pdev) 14555 { 14556 struct i40e_pf *pf = pci_get_drvdata(pdev); 14557 pci_ers_result_t result; 14558 int err; 14559 u32 reg; 14560 14561 dev_dbg(&pdev->dev, "%s\n", __func__); 14562 if (pci_enable_device_mem(pdev)) { 14563 dev_info(&pdev->dev, 14564 "Cannot re-enable PCI device after reset.\n"); 14565 result = PCI_ERS_RESULT_DISCONNECT; 14566 } else { 14567 pci_set_master(pdev); 14568 pci_restore_state(pdev); 14569 pci_save_state(pdev); 14570 pci_wake_from_d3(pdev, false); 14571 14572 reg = rd32(&pf->hw, I40E_GLGEN_RTRIG); 14573 if (reg == 0) 14574 result = PCI_ERS_RESULT_RECOVERED; 14575 else 14576 result = PCI_ERS_RESULT_DISCONNECT; 14577 } 14578 14579 err = pci_cleanup_aer_uncorrect_error_status(pdev); 14580 if (err) { 14581 dev_info(&pdev->dev, 14582 "pci_cleanup_aer_uncorrect_error_status failed 0x%0x\n", 14583 err); 14584 /* non-fatal, continue */ 14585 } 14586 14587 return result; 14588 } 14589 14590 /** 14591 * i40e_pci_error_reset_prepare - prepare device driver for pci reset 14592 * @pdev: PCI device information struct 14593 */ 14594 static void i40e_pci_error_reset_prepare(struct pci_dev *pdev) 14595 { 14596 struct i40e_pf *pf = pci_get_drvdata(pdev); 14597 14598 i40e_prep_for_reset(pf, false); 14599 } 14600 14601 /** 14602 * i40e_pci_error_reset_done - pci reset done, device driver reset can begin 14603 * @pdev: PCI device information struct 14604 */ 14605 static void i40e_pci_error_reset_done(struct pci_dev *pdev) 14606 { 14607 struct i40e_pf *pf = pci_get_drvdata(pdev); 14608 14609 i40e_reset_and_rebuild(pf, false, false); 14610 } 14611 14612 /** 14613 * i40e_pci_error_resume - restart operations after PCI error recovery 14614 * @pdev: PCI device information struct 14615 * 14616 * Called to allow the driver to bring things back up after PCI error 14617 * and/or reset recovery has finished. 14618 **/ 14619 static void i40e_pci_error_resume(struct pci_dev *pdev) 14620 { 14621 struct i40e_pf *pf = pci_get_drvdata(pdev); 14622 14623 dev_dbg(&pdev->dev, "%s\n", __func__); 14624 if (test_bit(__I40E_SUSPENDED, pf->state)) 14625 return; 14626 14627 i40e_handle_reset_warning(pf, false); 14628 } 14629 14630 /** 14631 * i40e_enable_mc_magic_wake - enable multicast magic packet wake up 14632 * using the mac_address_write admin q function 14633 * @pf: pointer to i40e_pf struct 14634 **/ 14635 static void i40e_enable_mc_magic_wake(struct i40e_pf *pf) 14636 { 14637 struct i40e_hw *hw = &pf->hw; 14638 i40e_status ret; 14639 u8 mac_addr[6]; 14640 u16 flags = 0; 14641 14642 /* Get current MAC address in case it's an LAA */ 14643 if (pf->vsi[pf->lan_vsi] && pf->vsi[pf->lan_vsi]->netdev) { 14644 ether_addr_copy(mac_addr, 14645 pf->vsi[pf->lan_vsi]->netdev->dev_addr); 14646 } else { 14647 dev_err(&pf->pdev->dev, 14648 "Failed to retrieve MAC address; using default\n"); 14649 ether_addr_copy(mac_addr, hw->mac.addr); 14650 } 14651 14652 /* The FW expects the mac address write cmd to first be called with 14653 * one of these flags before calling it again with the multicast 14654 * enable flags. 14655 */ 14656 flags = I40E_AQC_WRITE_TYPE_LAA_WOL; 14657 14658 if (hw->func_caps.flex10_enable && hw->partition_id != 1) 14659 flags = I40E_AQC_WRITE_TYPE_LAA_ONLY; 14660 14661 ret = i40e_aq_mac_address_write(hw, flags, mac_addr, NULL); 14662 if (ret) { 14663 dev_err(&pf->pdev->dev, 14664 "Failed to update MAC address registers; cannot enable Multicast Magic packet wake up"); 14665 return; 14666 } 14667 14668 flags = I40E_AQC_MC_MAG_EN 14669 | I40E_AQC_WOL_PRESERVE_ON_PFR 14670 | I40E_AQC_WRITE_TYPE_UPDATE_MC_MAG; 14671 ret = i40e_aq_mac_address_write(hw, flags, mac_addr, NULL); 14672 if (ret) 14673 dev_err(&pf->pdev->dev, 14674 "Failed to enable Multicast Magic Packet wake up\n"); 14675 } 14676 14677 /** 14678 * i40e_shutdown - PCI callback for shutting down 14679 * @pdev: PCI device information struct 14680 **/ 14681 static void i40e_shutdown(struct pci_dev *pdev) 14682 { 14683 struct i40e_pf *pf = pci_get_drvdata(pdev); 14684 struct i40e_hw *hw = &pf->hw; 14685 14686 set_bit(__I40E_SUSPENDED, pf->state); 14687 set_bit(__I40E_DOWN, pf->state); 14688 14689 del_timer_sync(&pf->service_timer); 14690 cancel_work_sync(&pf->service_task); 14691 i40e_cloud_filter_exit(pf); 14692 i40e_fdir_teardown(pf); 14693 14694 /* Client close must be called explicitly here because the timer 14695 * has been stopped. 14696 */ 14697 i40e_notify_client_of_netdev_close(pf->vsi[pf->lan_vsi], false); 14698 14699 if (pf->wol_en && (pf->hw_features & I40E_HW_WOL_MC_MAGIC_PKT_WAKE)) 14700 i40e_enable_mc_magic_wake(pf); 14701 14702 i40e_prep_for_reset(pf, false); 14703 14704 wr32(hw, I40E_PFPM_APM, 14705 (pf->wol_en ? I40E_PFPM_APM_APME_MASK : 0)); 14706 wr32(hw, I40E_PFPM_WUFC, 14707 (pf->wol_en ? I40E_PFPM_WUFC_MAG_MASK : 0)); 14708 14709 /* Since we're going to destroy queues during the 14710 * i40e_clear_interrupt_scheme() we should hold the RTNL lock for this 14711 * whole section 14712 */ 14713 rtnl_lock(); 14714 i40e_clear_interrupt_scheme(pf); 14715 rtnl_unlock(); 14716 14717 if (system_state == SYSTEM_POWER_OFF) { 14718 pci_wake_from_d3(pdev, pf->wol_en); 14719 pci_set_power_state(pdev, PCI_D3hot); 14720 } 14721 } 14722 14723 /** 14724 * i40e_suspend - PM callback for moving to D3 14725 * @dev: generic device information structure 14726 **/ 14727 static int __maybe_unused i40e_suspend(struct device *dev) 14728 { 14729 struct pci_dev *pdev = to_pci_dev(dev); 14730 struct i40e_pf *pf = pci_get_drvdata(pdev); 14731 struct i40e_hw *hw = &pf->hw; 14732 14733 /* If we're already suspended, then there is nothing to do */ 14734 if (test_and_set_bit(__I40E_SUSPENDED, pf->state)) 14735 return 0; 14736 14737 set_bit(__I40E_DOWN, pf->state); 14738 14739 /* Ensure service task will not be running */ 14740 del_timer_sync(&pf->service_timer); 14741 cancel_work_sync(&pf->service_task); 14742 14743 /* Client close must be called explicitly here because the timer 14744 * has been stopped. 14745 */ 14746 i40e_notify_client_of_netdev_close(pf->vsi[pf->lan_vsi], false); 14747 14748 if (pf->wol_en && (pf->hw_features & I40E_HW_WOL_MC_MAGIC_PKT_WAKE)) 14749 i40e_enable_mc_magic_wake(pf); 14750 14751 /* Since we're going to destroy queues during the 14752 * i40e_clear_interrupt_scheme() we should hold the RTNL lock for this 14753 * whole section 14754 */ 14755 rtnl_lock(); 14756 14757 i40e_prep_for_reset(pf, true); 14758 14759 wr32(hw, I40E_PFPM_APM, (pf->wol_en ? I40E_PFPM_APM_APME_MASK : 0)); 14760 wr32(hw, I40E_PFPM_WUFC, (pf->wol_en ? I40E_PFPM_WUFC_MAG_MASK : 0)); 14761 14762 /* Clear the interrupt scheme and release our IRQs so that the system 14763 * can safely hibernate even when there are a large number of CPUs. 14764 * Otherwise hibernation might fail when mapping all the vectors back 14765 * to CPU0. 14766 */ 14767 i40e_clear_interrupt_scheme(pf); 14768 14769 rtnl_unlock(); 14770 14771 return 0; 14772 } 14773 14774 /** 14775 * i40e_resume - PM callback for waking up from D3 14776 * @dev: generic device information structure 14777 **/ 14778 static int __maybe_unused i40e_resume(struct device *dev) 14779 { 14780 struct pci_dev *pdev = to_pci_dev(dev); 14781 struct i40e_pf *pf = pci_get_drvdata(pdev); 14782 int err; 14783 14784 /* If we're not suspended, then there is nothing to do */ 14785 if (!test_bit(__I40E_SUSPENDED, pf->state)) 14786 return 0; 14787 14788 /* We need to hold the RTNL lock prior to restoring interrupt schemes, 14789 * since we're going to be restoring queues 14790 */ 14791 rtnl_lock(); 14792 14793 /* We cleared the interrupt scheme when we suspended, so we need to 14794 * restore it now to resume device functionality. 14795 */ 14796 err = i40e_restore_interrupt_scheme(pf); 14797 if (err) { 14798 dev_err(&pdev->dev, "Cannot restore interrupt scheme: %d\n", 14799 err); 14800 } 14801 14802 clear_bit(__I40E_DOWN, pf->state); 14803 i40e_reset_and_rebuild(pf, false, true); 14804 14805 rtnl_unlock(); 14806 14807 /* Clear suspended state last after everything is recovered */ 14808 clear_bit(__I40E_SUSPENDED, pf->state); 14809 14810 /* Restart the service task */ 14811 mod_timer(&pf->service_timer, 14812 round_jiffies(jiffies + pf->service_timer_period)); 14813 14814 return 0; 14815 } 14816 14817 static const struct pci_error_handlers i40e_err_handler = { 14818 .error_detected = i40e_pci_error_detected, 14819 .slot_reset = i40e_pci_error_slot_reset, 14820 .reset_prepare = i40e_pci_error_reset_prepare, 14821 .reset_done = i40e_pci_error_reset_done, 14822 .resume = i40e_pci_error_resume, 14823 }; 14824 14825 static SIMPLE_DEV_PM_OPS(i40e_pm_ops, i40e_suspend, i40e_resume); 14826 14827 static struct pci_driver i40e_driver = { 14828 .name = i40e_driver_name, 14829 .id_table = i40e_pci_tbl, 14830 .probe = i40e_probe, 14831 .remove = i40e_remove, 14832 .driver = { 14833 .pm = &i40e_pm_ops, 14834 }, 14835 .shutdown = i40e_shutdown, 14836 .err_handler = &i40e_err_handler, 14837 .sriov_configure = i40e_pci_sriov_configure, 14838 }; 14839 14840 /** 14841 * i40e_init_module - Driver registration routine 14842 * 14843 * i40e_init_module is the first routine called when the driver is 14844 * loaded. All it does is register with the PCI subsystem. 14845 **/ 14846 static int __init i40e_init_module(void) 14847 { 14848 pr_info("%s: %s - version %s\n", i40e_driver_name, 14849 i40e_driver_string, i40e_driver_version_str); 14850 pr_info("%s: %s\n", i40e_driver_name, i40e_copyright); 14851 14852 /* There is no need to throttle the number of active tasks because 14853 * each device limits its own task using a state bit for scheduling 14854 * the service task, and the device tasks do not interfere with each 14855 * other, so we don't set a max task limit. We must set WQ_MEM_RECLAIM 14856 * since we need to be able to guarantee forward progress even under 14857 * memory pressure. 14858 */ 14859 i40e_wq = alloc_workqueue("%s", WQ_MEM_RECLAIM, 0, i40e_driver_name); 14860 if (!i40e_wq) { 14861 pr_err("%s: Failed to create workqueue\n", i40e_driver_name); 14862 return -ENOMEM; 14863 } 14864 14865 i40e_dbg_init(); 14866 return pci_register_driver(&i40e_driver); 14867 } 14868 module_init(i40e_init_module); 14869 14870 /** 14871 * i40e_exit_module - Driver exit cleanup routine 14872 * 14873 * i40e_exit_module is called just before the driver is removed 14874 * from memory. 14875 **/ 14876 static void __exit i40e_exit_module(void) 14877 { 14878 pci_unregister_driver(&i40e_driver); 14879 destroy_workqueue(i40e_wq); 14880 i40e_dbg_exit(); 14881 } 14882 module_exit(i40e_exit_module); 14883