1 /******************************************************************************* 2 * 3 * Intel Ethernet Controller XL710 Family Linux Driver 4 * Copyright(c) 2013 - 2016 Intel Corporation. 5 * 6 * This program is free software; you can redistribute it and/or modify it 7 * under the terms and conditions of the GNU General Public License, 8 * version 2, as published by the Free Software Foundation. 9 * 10 * This program is distributed in the hope it will be useful, but WITHOUT 11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 13 * more details. 14 * 15 * You should have received a copy of the GNU General Public License along 16 * with this program. If not, see <http://www.gnu.org/licenses/>. 17 * 18 * The full GNU General Public License is included in this distribution in 19 * the file called "COPYING". 20 * 21 * Contact Information: 22 * e1000-devel Mailing List <e1000-devel@lists.sourceforge.net> 23 * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497 24 * 25 ******************************************************************************/ 26 27 #include <linux/etherdevice.h> 28 #include <linux/of_net.h> 29 #include <linux/pci.h> 30 31 /* Local includes */ 32 #include "i40e.h" 33 #include "i40e_diag.h" 34 #include <net/udp_tunnel.h> 35 36 const char i40e_driver_name[] = "i40e"; 37 static const char i40e_driver_string[] = 38 "Intel(R) Ethernet Connection XL710 Network Driver"; 39 40 #define DRV_KERN "-k" 41 42 #define DRV_VERSION_MAJOR 1 43 #define DRV_VERSION_MINOR 6 44 #define DRV_VERSION_BUILD 25 45 #define DRV_VERSION __stringify(DRV_VERSION_MAJOR) "." \ 46 __stringify(DRV_VERSION_MINOR) "." \ 47 __stringify(DRV_VERSION_BUILD) DRV_KERN 48 const char i40e_driver_version_str[] = DRV_VERSION; 49 static const char i40e_copyright[] = "Copyright (c) 2013 - 2014 Intel Corporation."; 50 51 /* a bit of forward declarations */ 52 static void i40e_vsi_reinit_locked(struct i40e_vsi *vsi); 53 static void i40e_handle_reset_warning(struct i40e_pf *pf); 54 static int i40e_add_vsi(struct i40e_vsi *vsi); 55 static int i40e_add_veb(struct i40e_veb *veb, struct i40e_vsi *vsi); 56 static int i40e_setup_pf_switch(struct i40e_pf *pf, bool reinit); 57 static int i40e_setup_misc_vector(struct i40e_pf *pf); 58 static void i40e_determine_queue_usage(struct i40e_pf *pf); 59 static int i40e_setup_pf_filter_control(struct i40e_pf *pf); 60 static void i40e_fdir_sb_setup(struct i40e_pf *pf); 61 static int i40e_veb_get_bw_info(struct i40e_veb *veb); 62 63 /* i40e_pci_tbl - PCI Device ID Table 64 * 65 * Last entry must be all 0s 66 * 67 * { Vendor ID, Device ID, SubVendor ID, SubDevice ID, 68 * Class, Class Mask, private data (not used) } 69 */ 70 static const struct pci_device_id i40e_pci_tbl[] = { 71 {PCI_VDEVICE(INTEL, I40E_DEV_ID_SFP_XL710), 0}, 72 {PCI_VDEVICE(INTEL, I40E_DEV_ID_QEMU), 0}, 73 {PCI_VDEVICE(INTEL, I40E_DEV_ID_KX_B), 0}, 74 {PCI_VDEVICE(INTEL, I40E_DEV_ID_KX_C), 0}, 75 {PCI_VDEVICE(INTEL, I40E_DEV_ID_QSFP_A), 0}, 76 {PCI_VDEVICE(INTEL, I40E_DEV_ID_QSFP_B), 0}, 77 {PCI_VDEVICE(INTEL, I40E_DEV_ID_QSFP_C), 0}, 78 {PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_BASE_T), 0}, 79 {PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_BASE_T4), 0}, 80 {PCI_VDEVICE(INTEL, I40E_DEV_ID_20G_KR2), 0}, 81 {PCI_VDEVICE(INTEL, I40E_DEV_ID_KX_X722), 0}, 82 {PCI_VDEVICE(INTEL, I40E_DEV_ID_QSFP_X722), 0}, 83 {PCI_VDEVICE(INTEL, I40E_DEV_ID_SFP_X722), 0}, 84 {PCI_VDEVICE(INTEL, I40E_DEV_ID_1G_BASE_T_X722), 0}, 85 {PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_BASE_T_X722), 0}, 86 {PCI_VDEVICE(INTEL, I40E_DEV_ID_SFP_I_X722), 0}, 87 {PCI_VDEVICE(INTEL, I40E_DEV_ID_20G_KR2), 0}, 88 {PCI_VDEVICE(INTEL, I40E_DEV_ID_20G_KR2_A), 0}, 89 {PCI_VDEVICE(INTEL, I40E_DEV_ID_25G_B), 0}, 90 {PCI_VDEVICE(INTEL, I40E_DEV_ID_25G_SFP28), 0}, 91 /* required last entry */ 92 {0, } 93 }; 94 MODULE_DEVICE_TABLE(pci, i40e_pci_tbl); 95 96 #define I40E_MAX_VF_COUNT 128 97 static int debug = -1; 98 module_param(debug, uint, 0); 99 MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all), Debug mask (0x8XXXXXXX)"); 100 101 MODULE_AUTHOR("Intel Corporation, <e1000-devel@lists.sourceforge.net>"); 102 MODULE_DESCRIPTION("Intel(R) Ethernet Connection XL710 Network Driver"); 103 MODULE_LICENSE("GPL"); 104 MODULE_VERSION(DRV_VERSION); 105 106 static struct workqueue_struct *i40e_wq; 107 108 /** 109 * i40e_allocate_dma_mem_d - OS specific memory alloc for shared code 110 * @hw: pointer to the HW structure 111 * @mem: ptr to mem struct to fill out 112 * @size: size of memory requested 113 * @alignment: what to align the allocation to 114 **/ 115 int i40e_allocate_dma_mem_d(struct i40e_hw *hw, struct i40e_dma_mem *mem, 116 u64 size, u32 alignment) 117 { 118 struct i40e_pf *pf = (struct i40e_pf *)hw->back; 119 120 mem->size = ALIGN(size, alignment); 121 mem->va = dma_zalloc_coherent(&pf->pdev->dev, mem->size, 122 &mem->pa, GFP_KERNEL); 123 if (!mem->va) 124 return -ENOMEM; 125 126 return 0; 127 } 128 129 /** 130 * i40e_free_dma_mem_d - OS specific memory free for shared code 131 * @hw: pointer to the HW structure 132 * @mem: ptr to mem struct to free 133 **/ 134 int i40e_free_dma_mem_d(struct i40e_hw *hw, struct i40e_dma_mem *mem) 135 { 136 struct i40e_pf *pf = (struct i40e_pf *)hw->back; 137 138 dma_free_coherent(&pf->pdev->dev, mem->size, mem->va, mem->pa); 139 mem->va = NULL; 140 mem->pa = 0; 141 mem->size = 0; 142 143 return 0; 144 } 145 146 /** 147 * i40e_allocate_virt_mem_d - OS specific memory alloc for shared code 148 * @hw: pointer to the HW structure 149 * @mem: ptr to mem struct to fill out 150 * @size: size of memory requested 151 **/ 152 int i40e_allocate_virt_mem_d(struct i40e_hw *hw, struct i40e_virt_mem *mem, 153 u32 size) 154 { 155 mem->size = size; 156 mem->va = kzalloc(size, GFP_KERNEL); 157 158 if (!mem->va) 159 return -ENOMEM; 160 161 return 0; 162 } 163 164 /** 165 * i40e_free_virt_mem_d - OS specific memory free for shared code 166 * @hw: pointer to the HW structure 167 * @mem: ptr to mem struct to free 168 **/ 169 int i40e_free_virt_mem_d(struct i40e_hw *hw, struct i40e_virt_mem *mem) 170 { 171 /* it's ok to kfree a NULL pointer */ 172 kfree(mem->va); 173 mem->va = NULL; 174 mem->size = 0; 175 176 return 0; 177 } 178 179 /** 180 * i40e_get_lump - find a lump of free generic resource 181 * @pf: board private structure 182 * @pile: the pile of resource to search 183 * @needed: the number of items needed 184 * @id: an owner id to stick on the items assigned 185 * 186 * Returns the base item index of the lump, or negative for error 187 * 188 * The search_hint trick and lack of advanced fit-finding only work 189 * because we're highly likely to have all the same size lump requests. 190 * Linear search time and any fragmentation should be minimal. 191 **/ 192 static int i40e_get_lump(struct i40e_pf *pf, struct i40e_lump_tracking *pile, 193 u16 needed, u16 id) 194 { 195 int ret = -ENOMEM; 196 int i, j; 197 198 if (!pile || needed == 0 || id >= I40E_PILE_VALID_BIT) { 199 dev_info(&pf->pdev->dev, 200 "param err: pile=%p needed=%d id=0x%04x\n", 201 pile, needed, id); 202 return -EINVAL; 203 } 204 205 /* start the linear search with an imperfect hint */ 206 i = pile->search_hint; 207 while (i < pile->num_entries) { 208 /* skip already allocated entries */ 209 if (pile->list[i] & I40E_PILE_VALID_BIT) { 210 i++; 211 continue; 212 } 213 214 /* do we have enough in this lump? */ 215 for (j = 0; (j < needed) && ((i+j) < pile->num_entries); j++) { 216 if (pile->list[i+j] & I40E_PILE_VALID_BIT) 217 break; 218 } 219 220 if (j == needed) { 221 /* there was enough, so assign it to the requestor */ 222 for (j = 0; j < needed; j++) 223 pile->list[i+j] = id | I40E_PILE_VALID_BIT; 224 ret = i; 225 pile->search_hint = i + j; 226 break; 227 } 228 229 /* not enough, so skip over it and continue looking */ 230 i += j; 231 } 232 233 return ret; 234 } 235 236 /** 237 * i40e_put_lump - return a lump of generic resource 238 * @pile: the pile of resource to search 239 * @index: the base item index 240 * @id: the owner id of the items assigned 241 * 242 * Returns the count of items in the lump 243 **/ 244 static int i40e_put_lump(struct i40e_lump_tracking *pile, u16 index, u16 id) 245 { 246 int valid_id = (id | I40E_PILE_VALID_BIT); 247 int count = 0; 248 int i; 249 250 if (!pile || index >= pile->num_entries) 251 return -EINVAL; 252 253 for (i = index; 254 i < pile->num_entries && pile->list[i] == valid_id; 255 i++) { 256 pile->list[i] = 0; 257 count++; 258 } 259 260 if (count && index < pile->search_hint) 261 pile->search_hint = index; 262 263 return count; 264 } 265 266 /** 267 * i40e_find_vsi_from_id - searches for the vsi with the given id 268 * @pf - the pf structure to search for the vsi 269 * @id - id of the vsi it is searching for 270 **/ 271 struct i40e_vsi *i40e_find_vsi_from_id(struct i40e_pf *pf, u16 id) 272 { 273 int i; 274 275 for (i = 0; i < pf->num_alloc_vsi; i++) 276 if (pf->vsi[i] && (pf->vsi[i]->id == id)) 277 return pf->vsi[i]; 278 279 return NULL; 280 } 281 282 /** 283 * i40e_service_event_schedule - Schedule the service task to wake up 284 * @pf: board private structure 285 * 286 * If not already scheduled, this puts the task into the work queue 287 **/ 288 void i40e_service_event_schedule(struct i40e_pf *pf) 289 { 290 if (!test_bit(__I40E_DOWN, &pf->state) && 291 !test_bit(__I40E_RESET_RECOVERY_PENDING, &pf->state)) 292 queue_work(i40e_wq, &pf->service_task); 293 } 294 295 /** 296 * i40e_tx_timeout - Respond to a Tx Hang 297 * @netdev: network interface device structure 298 * 299 * If any port has noticed a Tx timeout, it is likely that the whole 300 * device is munged, not just the one netdev port, so go for the full 301 * reset. 302 **/ 303 #ifdef I40E_FCOE 304 void i40e_tx_timeout(struct net_device *netdev) 305 #else 306 static void i40e_tx_timeout(struct net_device *netdev) 307 #endif 308 { 309 struct i40e_netdev_priv *np = netdev_priv(netdev); 310 struct i40e_vsi *vsi = np->vsi; 311 struct i40e_pf *pf = vsi->back; 312 struct i40e_ring *tx_ring = NULL; 313 unsigned int i, hung_queue = 0; 314 u32 head, val; 315 316 pf->tx_timeout_count++; 317 318 /* find the stopped queue the same way the stack does */ 319 for (i = 0; i < netdev->num_tx_queues; i++) { 320 struct netdev_queue *q; 321 unsigned long trans_start; 322 323 q = netdev_get_tx_queue(netdev, i); 324 trans_start = q->trans_start; 325 if (netif_xmit_stopped(q) && 326 time_after(jiffies, 327 (trans_start + netdev->watchdog_timeo))) { 328 hung_queue = i; 329 break; 330 } 331 } 332 333 if (i == netdev->num_tx_queues) { 334 netdev_info(netdev, "tx_timeout: no netdev hung queue found\n"); 335 } else { 336 /* now that we have an index, find the tx_ring struct */ 337 for (i = 0; i < vsi->num_queue_pairs; i++) { 338 if (vsi->tx_rings[i] && vsi->tx_rings[i]->desc) { 339 if (hung_queue == 340 vsi->tx_rings[i]->queue_index) { 341 tx_ring = vsi->tx_rings[i]; 342 break; 343 } 344 } 345 } 346 } 347 348 if (time_after(jiffies, (pf->tx_timeout_last_recovery + HZ*20))) 349 pf->tx_timeout_recovery_level = 1; /* reset after some time */ 350 else if (time_before(jiffies, 351 (pf->tx_timeout_last_recovery + netdev->watchdog_timeo))) 352 return; /* don't do any new action before the next timeout */ 353 354 if (tx_ring) { 355 head = i40e_get_head(tx_ring); 356 /* Read interrupt register */ 357 if (pf->flags & I40E_FLAG_MSIX_ENABLED) 358 val = rd32(&pf->hw, 359 I40E_PFINT_DYN_CTLN(tx_ring->q_vector->v_idx + 360 tx_ring->vsi->base_vector - 1)); 361 else 362 val = rd32(&pf->hw, I40E_PFINT_DYN_CTL0); 363 364 netdev_info(netdev, "tx_timeout: VSI_seid: %d, Q %d, NTC: 0x%x, HWB: 0x%x, NTU: 0x%x, TAIL: 0x%x, INT: 0x%x\n", 365 vsi->seid, hung_queue, tx_ring->next_to_clean, 366 head, tx_ring->next_to_use, 367 readl(tx_ring->tail), val); 368 } 369 370 pf->tx_timeout_last_recovery = jiffies; 371 netdev_info(netdev, "tx_timeout recovery level %d, hung_queue %d\n", 372 pf->tx_timeout_recovery_level, hung_queue); 373 374 switch (pf->tx_timeout_recovery_level) { 375 case 1: 376 set_bit(__I40E_PF_RESET_REQUESTED, &pf->state); 377 break; 378 case 2: 379 set_bit(__I40E_CORE_RESET_REQUESTED, &pf->state); 380 break; 381 case 3: 382 set_bit(__I40E_GLOBAL_RESET_REQUESTED, &pf->state); 383 break; 384 default: 385 netdev_err(netdev, "tx_timeout recovery unsuccessful\n"); 386 break; 387 } 388 389 i40e_service_event_schedule(pf); 390 pf->tx_timeout_recovery_level++; 391 } 392 393 /** 394 * i40e_get_vsi_stats_struct - Get System Network Statistics 395 * @vsi: the VSI we care about 396 * 397 * Returns the address of the device statistics structure. 398 * The statistics are actually updated from the service task. 399 **/ 400 struct rtnl_link_stats64 *i40e_get_vsi_stats_struct(struct i40e_vsi *vsi) 401 { 402 return &vsi->net_stats; 403 } 404 405 /** 406 * i40e_get_netdev_stats_struct - Get statistics for netdev interface 407 * @netdev: network interface device structure 408 * 409 * Returns the address of the device statistics structure. 410 * The statistics are actually updated from the service task. 411 **/ 412 #ifndef I40E_FCOE 413 static 414 #endif 415 void i40e_get_netdev_stats_struct(struct net_device *netdev, 416 struct rtnl_link_stats64 *stats) 417 { 418 struct i40e_netdev_priv *np = netdev_priv(netdev); 419 struct i40e_ring *tx_ring, *rx_ring; 420 struct i40e_vsi *vsi = np->vsi; 421 struct rtnl_link_stats64 *vsi_stats = i40e_get_vsi_stats_struct(vsi); 422 int i; 423 424 if (test_bit(__I40E_DOWN, &vsi->state)) 425 return; 426 427 if (!vsi->tx_rings) 428 return; 429 430 rcu_read_lock(); 431 for (i = 0; i < vsi->num_queue_pairs; i++) { 432 u64 bytes, packets; 433 unsigned int start; 434 435 tx_ring = ACCESS_ONCE(vsi->tx_rings[i]); 436 if (!tx_ring) 437 continue; 438 439 do { 440 start = u64_stats_fetch_begin_irq(&tx_ring->syncp); 441 packets = tx_ring->stats.packets; 442 bytes = tx_ring->stats.bytes; 443 } while (u64_stats_fetch_retry_irq(&tx_ring->syncp, start)); 444 445 stats->tx_packets += packets; 446 stats->tx_bytes += bytes; 447 rx_ring = &tx_ring[1]; 448 449 do { 450 start = u64_stats_fetch_begin_irq(&rx_ring->syncp); 451 packets = rx_ring->stats.packets; 452 bytes = rx_ring->stats.bytes; 453 } while (u64_stats_fetch_retry_irq(&rx_ring->syncp, start)); 454 455 stats->rx_packets += packets; 456 stats->rx_bytes += bytes; 457 } 458 rcu_read_unlock(); 459 460 /* following stats updated by i40e_watchdog_subtask() */ 461 stats->multicast = vsi_stats->multicast; 462 stats->tx_errors = vsi_stats->tx_errors; 463 stats->tx_dropped = vsi_stats->tx_dropped; 464 stats->rx_errors = vsi_stats->rx_errors; 465 stats->rx_dropped = vsi_stats->rx_dropped; 466 stats->rx_crc_errors = vsi_stats->rx_crc_errors; 467 stats->rx_length_errors = vsi_stats->rx_length_errors; 468 } 469 470 /** 471 * i40e_vsi_reset_stats - Resets all stats of the given vsi 472 * @vsi: the VSI to have its stats reset 473 **/ 474 void i40e_vsi_reset_stats(struct i40e_vsi *vsi) 475 { 476 struct rtnl_link_stats64 *ns; 477 int i; 478 479 if (!vsi) 480 return; 481 482 ns = i40e_get_vsi_stats_struct(vsi); 483 memset(ns, 0, sizeof(*ns)); 484 memset(&vsi->net_stats_offsets, 0, sizeof(vsi->net_stats_offsets)); 485 memset(&vsi->eth_stats, 0, sizeof(vsi->eth_stats)); 486 memset(&vsi->eth_stats_offsets, 0, sizeof(vsi->eth_stats_offsets)); 487 if (vsi->rx_rings && vsi->rx_rings[0]) { 488 for (i = 0; i < vsi->num_queue_pairs; i++) { 489 memset(&vsi->rx_rings[i]->stats, 0, 490 sizeof(vsi->rx_rings[i]->stats)); 491 memset(&vsi->rx_rings[i]->rx_stats, 0, 492 sizeof(vsi->rx_rings[i]->rx_stats)); 493 memset(&vsi->tx_rings[i]->stats, 0, 494 sizeof(vsi->tx_rings[i]->stats)); 495 memset(&vsi->tx_rings[i]->tx_stats, 0, 496 sizeof(vsi->tx_rings[i]->tx_stats)); 497 } 498 } 499 vsi->stat_offsets_loaded = false; 500 } 501 502 /** 503 * i40e_pf_reset_stats - Reset all of the stats for the given PF 504 * @pf: the PF to be reset 505 **/ 506 void i40e_pf_reset_stats(struct i40e_pf *pf) 507 { 508 int i; 509 510 memset(&pf->stats, 0, sizeof(pf->stats)); 511 memset(&pf->stats_offsets, 0, sizeof(pf->stats_offsets)); 512 pf->stat_offsets_loaded = false; 513 514 for (i = 0; i < I40E_MAX_VEB; i++) { 515 if (pf->veb[i]) { 516 memset(&pf->veb[i]->stats, 0, 517 sizeof(pf->veb[i]->stats)); 518 memset(&pf->veb[i]->stats_offsets, 0, 519 sizeof(pf->veb[i]->stats_offsets)); 520 pf->veb[i]->stat_offsets_loaded = false; 521 } 522 } 523 pf->hw_csum_rx_error = 0; 524 } 525 526 /** 527 * i40e_stat_update48 - read and update a 48 bit stat from the chip 528 * @hw: ptr to the hardware info 529 * @hireg: the high 32 bit reg to read 530 * @loreg: the low 32 bit reg to read 531 * @offset_loaded: has the initial offset been loaded yet 532 * @offset: ptr to current offset value 533 * @stat: ptr to the stat 534 * 535 * Since the device stats are not reset at PFReset, they likely will not 536 * be zeroed when the driver starts. We'll save the first values read 537 * and use them as offsets to be subtracted from the raw values in order 538 * to report stats that count from zero. In the process, we also manage 539 * the potential roll-over. 540 **/ 541 static void i40e_stat_update48(struct i40e_hw *hw, u32 hireg, u32 loreg, 542 bool offset_loaded, u64 *offset, u64 *stat) 543 { 544 u64 new_data; 545 546 if (hw->device_id == I40E_DEV_ID_QEMU) { 547 new_data = rd32(hw, loreg); 548 new_data |= ((u64)(rd32(hw, hireg) & 0xFFFF)) << 32; 549 } else { 550 new_data = rd64(hw, loreg); 551 } 552 if (!offset_loaded) 553 *offset = new_data; 554 if (likely(new_data >= *offset)) 555 *stat = new_data - *offset; 556 else 557 *stat = (new_data + BIT_ULL(48)) - *offset; 558 *stat &= 0xFFFFFFFFFFFFULL; 559 } 560 561 /** 562 * i40e_stat_update32 - read and update a 32 bit stat from the chip 563 * @hw: ptr to the hardware info 564 * @reg: the hw reg to read 565 * @offset_loaded: has the initial offset been loaded yet 566 * @offset: ptr to current offset value 567 * @stat: ptr to the stat 568 **/ 569 static void i40e_stat_update32(struct i40e_hw *hw, u32 reg, 570 bool offset_loaded, u64 *offset, u64 *stat) 571 { 572 u32 new_data; 573 574 new_data = rd32(hw, reg); 575 if (!offset_loaded) 576 *offset = new_data; 577 if (likely(new_data >= *offset)) 578 *stat = (u32)(new_data - *offset); 579 else 580 *stat = (u32)((new_data + BIT_ULL(32)) - *offset); 581 } 582 583 /** 584 * i40e_update_eth_stats - Update VSI-specific ethernet statistics counters. 585 * @vsi: the VSI to be updated 586 **/ 587 void i40e_update_eth_stats(struct i40e_vsi *vsi) 588 { 589 int stat_idx = le16_to_cpu(vsi->info.stat_counter_idx); 590 struct i40e_pf *pf = vsi->back; 591 struct i40e_hw *hw = &pf->hw; 592 struct i40e_eth_stats *oes; 593 struct i40e_eth_stats *es; /* device's eth stats */ 594 595 es = &vsi->eth_stats; 596 oes = &vsi->eth_stats_offsets; 597 598 /* Gather up the stats that the hw collects */ 599 i40e_stat_update32(hw, I40E_GLV_TEPC(stat_idx), 600 vsi->stat_offsets_loaded, 601 &oes->tx_errors, &es->tx_errors); 602 i40e_stat_update32(hw, I40E_GLV_RDPC(stat_idx), 603 vsi->stat_offsets_loaded, 604 &oes->rx_discards, &es->rx_discards); 605 i40e_stat_update32(hw, I40E_GLV_RUPP(stat_idx), 606 vsi->stat_offsets_loaded, 607 &oes->rx_unknown_protocol, &es->rx_unknown_protocol); 608 i40e_stat_update32(hw, I40E_GLV_TEPC(stat_idx), 609 vsi->stat_offsets_loaded, 610 &oes->tx_errors, &es->tx_errors); 611 612 i40e_stat_update48(hw, I40E_GLV_GORCH(stat_idx), 613 I40E_GLV_GORCL(stat_idx), 614 vsi->stat_offsets_loaded, 615 &oes->rx_bytes, &es->rx_bytes); 616 i40e_stat_update48(hw, I40E_GLV_UPRCH(stat_idx), 617 I40E_GLV_UPRCL(stat_idx), 618 vsi->stat_offsets_loaded, 619 &oes->rx_unicast, &es->rx_unicast); 620 i40e_stat_update48(hw, I40E_GLV_MPRCH(stat_idx), 621 I40E_GLV_MPRCL(stat_idx), 622 vsi->stat_offsets_loaded, 623 &oes->rx_multicast, &es->rx_multicast); 624 i40e_stat_update48(hw, I40E_GLV_BPRCH(stat_idx), 625 I40E_GLV_BPRCL(stat_idx), 626 vsi->stat_offsets_loaded, 627 &oes->rx_broadcast, &es->rx_broadcast); 628 629 i40e_stat_update48(hw, I40E_GLV_GOTCH(stat_idx), 630 I40E_GLV_GOTCL(stat_idx), 631 vsi->stat_offsets_loaded, 632 &oes->tx_bytes, &es->tx_bytes); 633 i40e_stat_update48(hw, I40E_GLV_UPTCH(stat_idx), 634 I40E_GLV_UPTCL(stat_idx), 635 vsi->stat_offsets_loaded, 636 &oes->tx_unicast, &es->tx_unicast); 637 i40e_stat_update48(hw, I40E_GLV_MPTCH(stat_idx), 638 I40E_GLV_MPTCL(stat_idx), 639 vsi->stat_offsets_loaded, 640 &oes->tx_multicast, &es->tx_multicast); 641 i40e_stat_update48(hw, I40E_GLV_BPTCH(stat_idx), 642 I40E_GLV_BPTCL(stat_idx), 643 vsi->stat_offsets_loaded, 644 &oes->tx_broadcast, &es->tx_broadcast); 645 vsi->stat_offsets_loaded = true; 646 } 647 648 /** 649 * i40e_update_veb_stats - Update Switch component statistics 650 * @veb: the VEB being updated 651 **/ 652 static void i40e_update_veb_stats(struct i40e_veb *veb) 653 { 654 struct i40e_pf *pf = veb->pf; 655 struct i40e_hw *hw = &pf->hw; 656 struct i40e_eth_stats *oes; 657 struct i40e_eth_stats *es; /* device's eth stats */ 658 struct i40e_veb_tc_stats *veb_oes; 659 struct i40e_veb_tc_stats *veb_es; 660 int i, idx = 0; 661 662 idx = veb->stats_idx; 663 es = &veb->stats; 664 oes = &veb->stats_offsets; 665 veb_es = &veb->tc_stats; 666 veb_oes = &veb->tc_stats_offsets; 667 668 /* Gather up the stats that the hw collects */ 669 i40e_stat_update32(hw, I40E_GLSW_TDPC(idx), 670 veb->stat_offsets_loaded, 671 &oes->tx_discards, &es->tx_discards); 672 if (hw->revision_id > 0) 673 i40e_stat_update32(hw, I40E_GLSW_RUPP(idx), 674 veb->stat_offsets_loaded, 675 &oes->rx_unknown_protocol, 676 &es->rx_unknown_protocol); 677 i40e_stat_update48(hw, I40E_GLSW_GORCH(idx), I40E_GLSW_GORCL(idx), 678 veb->stat_offsets_loaded, 679 &oes->rx_bytes, &es->rx_bytes); 680 i40e_stat_update48(hw, I40E_GLSW_UPRCH(idx), I40E_GLSW_UPRCL(idx), 681 veb->stat_offsets_loaded, 682 &oes->rx_unicast, &es->rx_unicast); 683 i40e_stat_update48(hw, I40E_GLSW_MPRCH(idx), I40E_GLSW_MPRCL(idx), 684 veb->stat_offsets_loaded, 685 &oes->rx_multicast, &es->rx_multicast); 686 i40e_stat_update48(hw, I40E_GLSW_BPRCH(idx), I40E_GLSW_BPRCL(idx), 687 veb->stat_offsets_loaded, 688 &oes->rx_broadcast, &es->rx_broadcast); 689 690 i40e_stat_update48(hw, I40E_GLSW_GOTCH(idx), I40E_GLSW_GOTCL(idx), 691 veb->stat_offsets_loaded, 692 &oes->tx_bytes, &es->tx_bytes); 693 i40e_stat_update48(hw, I40E_GLSW_UPTCH(idx), I40E_GLSW_UPTCL(idx), 694 veb->stat_offsets_loaded, 695 &oes->tx_unicast, &es->tx_unicast); 696 i40e_stat_update48(hw, I40E_GLSW_MPTCH(idx), I40E_GLSW_MPTCL(idx), 697 veb->stat_offsets_loaded, 698 &oes->tx_multicast, &es->tx_multicast); 699 i40e_stat_update48(hw, I40E_GLSW_BPTCH(idx), I40E_GLSW_BPTCL(idx), 700 veb->stat_offsets_loaded, 701 &oes->tx_broadcast, &es->tx_broadcast); 702 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 703 i40e_stat_update48(hw, I40E_GLVEBTC_RPCH(i, idx), 704 I40E_GLVEBTC_RPCL(i, idx), 705 veb->stat_offsets_loaded, 706 &veb_oes->tc_rx_packets[i], 707 &veb_es->tc_rx_packets[i]); 708 i40e_stat_update48(hw, I40E_GLVEBTC_RBCH(i, idx), 709 I40E_GLVEBTC_RBCL(i, idx), 710 veb->stat_offsets_loaded, 711 &veb_oes->tc_rx_bytes[i], 712 &veb_es->tc_rx_bytes[i]); 713 i40e_stat_update48(hw, I40E_GLVEBTC_TPCH(i, idx), 714 I40E_GLVEBTC_TPCL(i, idx), 715 veb->stat_offsets_loaded, 716 &veb_oes->tc_tx_packets[i], 717 &veb_es->tc_tx_packets[i]); 718 i40e_stat_update48(hw, I40E_GLVEBTC_TBCH(i, idx), 719 I40E_GLVEBTC_TBCL(i, idx), 720 veb->stat_offsets_loaded, 721 &veb_oes->tc_tx_bytes[i], 722 &veb_es->tc_tx_bytes[i]); 723 } 724 veb->stat_offsets_loaded = true; 725 } 726 727 #ifdef I40E_FCOE 728 /** 729 * i40e_update_fcoe_stats - Update FCoE-specific ethernet statistics counters. 730 * @vsi: the VSI that is capable of doing FCoE 731 **/ 732 static void i40e_update_fcoe_stats(struct i40e_vsi *vsi) 733 { 734 struct i40e_pf *pf = vsi->back; 735 struct i40e_hw *hw = &pf->hw; 736 struct i40e_fcoe_stats *ofs; 737 struct i40e_fcoe_stats *fs; /* device's eth stats */ 738 int idx; 739 740 if (vsi->type != I40E_VSI_FCOE) 741 return; 742 743 idx = hw->pf_id + I40E_FCOE_PF_STAT_OFFSET; 744 fs = &vsi->fcoe_stats; 745 ofs = &vsi->fcoe_stats_offsets; 746 747 i40e_stat_update32(hw, I40E_GL_FCOEPRC(idx), 748 vsi->fcoe_stat_offsets_loaded, 749 &ofs->rx_fcoe_packets, &fs->rx_fcoe_packets); 750 i40e_stat_update48(hw, I40E_GL_FCOEDWRCH(idx), I40E_GL_FCOEDWRCL(idx), 751 vsi->fcoe_stat_offsets_loaded, 752 &ofs->rx_fcoe_dwords, &fs->rx_fcoe_dwords); 753 i40e_stat_update32(hw, I40E_GL_FCOERPDC(idx), 754 vsi->fcoe_stat_offsets_loaded, 755 &ofs->rx_fcoe_dropped, &fs->rx_fcoe_dropped); 756 i40e_stat_update32(hw, I40E_GL_FCOEPTC(idx), 757 vsi->fcoe_stat_offsets_loaded, 758 &ofs->tx_fcoe_packets, &fs->tx_fcoe_packets); 759 i40e_stat_update48(hw, I40E_GL_FCOEDWTCH(idx), I40E_GL_FCOEDWTCL(idx), 760 vsi->fcoe_stat_offsets_loaded, 761 &ofs->tx_fcoe_dwords, &fs->tx_fcoe_dwords); 762 i40e_stat_update32(hw, I40E_GL_FCOECRC(idx), 763 vsi->fcoe_stat_offsets_loaded, 764 &ofs->fcoe_bad_fccrc, &fs->fcoe_bad_fccrc); 765 i40e_stat_update32(hw, I40E_GL_FCOELAST(idx), 766 vsi->fcoe_stat_offsets_loaded, 767 &ofs->fcoe_last_error, &fs->fcoe_last_error); 768 i40e_stat_update32(hw, I40E_GL_FCOEDDPC(idx), 769 vsi->fcoe_stat_offsets_loaded, 770 &ofs->fcoe_ddp_count, &fs->fcoe_ddp_count); 771 772 vsi->fcoe_stat_offsets_loaded = true; 773 } 774 775 #endif 776 /** 777 * i40e_update_vsi_stats - Update the vsi statistics counters. 778 * @vsi: the VSI to be updated 779 * 780 * There are a few instances where we store the same stat in a 781 * couple of different structs. This is partly because we have 782 * the netdev stats that need to be filled out, which is slightly 783 * different from the "eth_stats" defined by the chip and used in 784 * VF communications. We sort it out here. 785 **/ 786 static void i40e_update_vsi_stats(struct i40e_vsi *vsi) 787 { 788 struct i40e_pf *pf = vsi->back; 789 struct rtnl_link_stats64 *ons; 790 struct rtnl_link_stats64 *ns; /* netdev stats */ 791 struct i40e_eth_stats *oes; 792 struct i40e_eth_stats *es; /* device's eth stats */ 793 u32 tx_restart, tx_busy; 794 u64 tx_lost_interrupt; 795 struct i40e_ring *p; 796 u32 rx_page, rx_buf; 797 u64 bytes, packets; 798 unsigned int start; 799 u64 tx_linearize; 800 u64 tx_force_wb; 801 u64 rx_p, rx_b; 802 u64 tx_p, tx_b; 803 u16 q; 804 805 if (test_bit(__I40E_DOWN, &vsi->state) || 806 test_bit(__I40E_CONFIG_BUSY, &pf->state)) 807 return; 808 809 ns = i40e_get_vsi_stats_struct(vsi); 810 ons = &vsi->net_stats_offsets; 811 es = &vsi->eth_stats; 812 oes = &vsi->eth_stats_offsets; 813 814 /* Gather up the netdev and vsi stats that the driver collects 815 * on the fly during packet processing 816 */ 817 rx_b = rx_p = 0; 818 tx_b = tx_p = 0; 819 tx_restart = tx_busy = tx_linearize = tx_force_wb = 0; 820 tx_lost_interrupt = 0; 821 rx_page = 0; 822 rx_buf = 0; 823 rcu_read_lock(); 824 for (q = 0; q < vsi->num_queue_pairs; q++) { 825 /* locate Tx ring */ 826 p = ACCESS_ONCE(vsi->tx_rings[q]); 827 828 do { 829 start = u64_stats_fetch_begin_irq(&p->syncp); 830 packets = p->stats.packets; 831 bytes = p->stats.bytes; 832 } while (u64_stats_fetch_retry_irq(&p->syncp, start)); 833 tx_b += bytes; 834 tx_p += packets; 835 tx_restart += p->tx_stats.restart_queue; 836 tx_busy += p->tx_stats.tx_busy; 837 tx_linearize += p->tx_stats.tx_linearize; 838 tx_force_wb += p->tx_stats.tx_force_wb; 839 tx_lost_interrupt += p->tx_stats.tx_lost_interrupt; 840 841 /* Rx queue is part of the same block as Tx queue */ 842 p = &p[1]; 843 do { 844 start = u64_stats_fetch_begin_irq(&p->syncp); 845 packets = p->stats.packets; 846 bytes = p->stats.bytes; 847 } while (u64_stats_fetch_retry_irq(&p->syncp, start)); 848 rx_b += bytes; 849 rx_p += packets; 850 rx_buf += p->rx_stats.alloc_buff_failed; 851 rx_page += p->rx_stats.alloc_page_failed; 852 } 853 rcu_read_unlock(); 854 vsi->tx_restart = tx_restart; 855 vsi->tx_busy = tx_busy; 856 vsi->tx_linearize = tx_linearize; 857 vsi->tx_force_wb = tx_force_wb; 858 vsi->tx_lost_interrupt = tx_lost_interrupt; 859 vsi->rx_page_failed = rx_page; 860 vsi->rx_buf_failed = rx_buf; 861 862 ns->rx_packets = rx_p; 863 ns->rx_bytes = rx_b; 864 ns->tx_packets = tx_p; 865 ns->tx_bytes = tx_b; 866 867 /* update netdev stats from eth stats */ 868 i40e_update_eth_stats(vsi); 869 ons->tx_errors = oes->tx_errors; 870 ns->tx_errors = es->tx_errors; 871 ons->multicast = oes->rx_multicast; 872 ns->multicast = es->rx_multicast; 873 ons->rx_dropped = oes->rx_discards; 874 ns->rx_dropped = es->rx_discards; 875 ons->tx_dropped = oes->tx_discards; 876 ns->tx_dropped = es->tx_discards; 877 878 /* pull in a couple PF stats if this is the main vsi */ 879 if (vsi == pf->vsi[pf->lan_vsi]) { 880 ns->rx_crc_errors = pf->stats.crc_errors; 881 ns->rx_errors = pf->stats.crc_errors + pf->stats.illegal_bytes; 882 ns->rx_length_errors = pf->stats.rx_length_errors; 883 } 884 } 885 886 /** 887 * i40e_update_pf_stats - Update the PF statistics counters. 888 * @pf: the PF to be updated 889 **/ 890 static void i40e_update_pf_stats(struct i40e_pf *pf) 891 { 892 struct i40e_hw_port_stats *osd = &pf->stats_offsets; 893 struct i40e_hw_port_stats *nsd = &pf->stats; 894 struct i40e_hw *hw = &pf->hw; 895 u32 val; 896 int i; 897 898 i40e_stat_update48(hw, I40E_GLPRT_GORCH(hw->port), 899 I40E_GLPRT_GORCL(hw->port), 900 pf->stat_offsets_loaded, 901 &osd->eth.rx_bytes, &nsd->eth.rx_bytes); 902 i40e_stat_update48(hw, I40E_GLPRT_GOTCH(hw->port), 903 I40E_GLPRT_GOTCL(hw->port), 904 pf->stat_offsets_loaded, 905 &osd->eth.tx_bytes, &nsd->eth.tx_bytes); 906 i40e_stat_update32(hw, I40E_GLPRT_RDPC(hw->port), 907 pf->stat_offsets_loaded, 908 &osd->eth.rx_discards, 909 &nsd->eth.rx_discards); 910 i40e_stat_update48(hw, I40E_GLPRT_UPRCH(hw->port), 911 I40E_GLPRT_UPRCL(hw->port), 912 pf->stat_offsets_loaded, 913 &osd->eth.rx_unicast, 914 &nsd->eth.rx_unicast); 915 i40e_stat_update48(hw, I40E_GLPRT_MPRCH(hw->port), 916 I40E_GLPRT_MPRCL(hw->port), 917 pf->stat_offsets_loaded, 918 &osd->eth.rx_multicast, 919 &nsd->eth.rx_multicast); 920 i40e_stat_update48(hw, I40E_GLPRT_BPRCH(hw->port), 921 I40E_GLPRT_BPRCL(hw->port), 922 pf->stat_offsets_loaded, 923 &osd->eth.rx_broadcast, 924 &nsd->eth.rx_broadcast); 925 i40e_stat_update48(hw, I40E_GLPRT_UPTCH(hw->port), 926 I40E_GLPRT_UPTCL(hw->port), 927 pf->stat_offsets_loaded, 928 &osd->eth.tx_unicast, 929 &nsd->eth.tx_unicast); 930 i40e_stat_update48(hw, I40E_GLPRT_MPTCH(hw->port), 931 I40E_GLPRT_MPTCL(hw->port), 932 pf->stat_offsets_loaded, 933 &osd->eth.tx_multicast, 934 &nsd->eth.tx_multicast); 935 i40e_stat_update48(hw, I40E_GLPRT_BPTCH(hw->port), 936 I40E_GLPRT_BPTCL(hw->port), 937 pf->stat_offsets_loaded, 938 &osd->eth.tx_broadcast, 939 &nsd->eth.tx_broadcast); 940 941 i40e_stat_update32(hw, I40E_GLPRT_TDOLD(hw->port), 942 pf->stat_offsets_loaded, 943 &osd->tx_dropped_link_down, 944 &nsd->tx_dropped_link_down); 945 946 i40e_stat_update32(hw, I40E_GLPRT_CRCERRS(hw->port), 947 pf->stat_offsets_loaded, 948 &osd->crc_errors, &nsd->crc_errors); 949 950 i40e_stat_update32(hw, I40E_GLPRT_ILLERRC(hw->port), 951 pf->stat_offsets_loaded, 952 &osd->illegal_bytes, &nsd->illegal_bytes); 953 954 i40e_stat_update32(hw, I40E_GLPRT_MLFC(hw->port), 955 pf->stat_offsets_loaded, 956 &osd->mac_local_faults, 957 &nsd->mac_local_faults); 958 i40e_stat_update32(hw, I40E_GLPRT_MRFC(hw->port), 959 pf->stat_offsets_loaded, 960 &osd->mac_remote_faults, 961 &nsd->mac_remote_faults); 962 963 i40e_stat_update32(hw, I40E_GLPRT_RLEC(hw->port), 964 pf->stat_offsets_loaded, 965 &osd->rx_length_errors, 966 &nsd->rx_length_errors); 967 968 i40e_stat_update32(hw, I40E_GLPRT_LXONRXC(hw->port), 969 pf->stat_offsets_loaded, 970 &osd->link_xon_rx, &nsd->link_xon_rx); 971 i40e_stat_update32(hw, I40E_GLPRT_LXONTXC(hw->port), 972 pf->stat_offsets_loaded, 973 &osd->link_xon_tx, &nsd->link_xon_tx); 974 i40e_stat_update32(hw, I40E_GLPRT_LXOFFRXC(hw->port), 975 pf->stat_offsets_loaded, 976 &osd->link_xoff_rx, &nsd->link_xoff_rx); 977 i40e_stat_update32(hw, I40E_GLPRT_LXOFFTXC(hw->port), 978 pf->stat_offsets_loaded, 979 &osd->link_xoff_tx, &nsd->link_xoff_tx); 980 981 for (i = 0; i < 8; i++) { 982 i40e_stat_update32(hw, I40E_GLPRT_PXOFFRXC(hw->port, i), 983 pf->stat_offsets_loaded, 984 &osd->priority_xoff_rx[i], 985 &nsd->priority_xoff_rx[i]); 986 i40e_stat_update32(hw, I40E_GLPRT_PXONRXC(hw->port, i), 987 pf->stat_offsets_loaded, 988 &osd->priority_xon_rx[i], 989 &nsd->priority_xon_rx[i]); 990 i40e_stat_update32(hw, I40E_GLPRT_PXONTXC(hw->port, i), 991 pf->stat_offsets_loaded, 992 &osd->priority_xon_tx[i], 993 &nsd->priority_xon_tx[i]); 994 i40e_stat_update32(hw, I40E_GLPRT_PXOFFTXC(hw->port, i), 995 pf->stat_offsets_loaded, 996 &osd->priority_xoff_tx[i], 997 &nsd->priority_xoff_tx[i]); 998 i40e_stat_update32(hw, 999 I40E_GLPRT_RXON2OFFCNT(hw->port, i), 1000 pf->stat_offsets_loaded, 1001 &osd->priority_xon_2_xoff[i], 1002 &nsd->priority_xon_2_xoff[i]); 1003 } 1004 1005 i40e_stat_update48(hw, I40E_GLPRT_PRC64H(hw->port), 1006 I40E_GLPRT_PRC64L(hw->port), 1007 pf->stat_offsets_loaded, 1008 &osd->rx_size_64, &nsd->rx_size_64); 1009 i40e_stat_update48(hw, I40E_GLPRT_PRC127H(hw->port), 1010 I40E_GLPRT_PRC127L(hw->port), 1011 pf->stat_offsets_loaded, 1012 &osd->rx_size_127, &nsd->rx_size_127); 1013 i40e_stat_update48(hw, I40E_GLPRT_PRC255H(hw->port), 1014 I40E_GLPRT_PRC255L(hw->port), 1015 pf->stat_offsets_loaded, 1016 &osd->rx_size_255, &nsd->rx_size_255); 1017 i40e_stat_update48(hw, I40E_GLPRT_PRC511H(hw->port), 1018 I40E_GLPRT_PRC511L(hw->port), 1019 pf->stat_offsets_loaded, 1020 &osd->rx_size_511, &nsd->rx_size_511); 1021 i40e_stat_update48(hw, I40E_GLPRT_PRC1023H(hw->port), 1022 I40E_GLPRT_PRC1023L(hw->port), 1023 pf->stat_offsets_loaded, 1024 &osd->rx_size_1023, &nsd->rx_size_1023); 1025 i40e_stat_update48(hw, I40E_GLPRT_PRC1522H(hw->port), 1026 I40E_GLPRT_PRC1522L(hw->port), 1027 pf->stat_offsets_loaded, 1028 &osd->rx_size_1522, &nsd->rx_size_1522); 1029 i40e_stat_update48(hw, I40E_GLPRT_PRC9522H(hw->port), 1030 I40E_GLPRT_PRC9522L(hw->port), 1031 pf->stat_offsets_loaded, 1032 &osd->rx_size_big, &nsd->rx_size_big); 1033 1034 i40e_stat_update48(hw, I40E_GLPRT_PTC64H(hw->port), 1035 I40E_GLPRT_PTC64L(hw->port), 1036 pf->stat_offsets_loaded, 1037 &osd->tx_size_64, &nsd->tx_size_64); 1038 i40e_stat_update48(hw, I40E_GLPRT_PTC127H(hw->port), 1039 I40E_GLPRT_PTC127L(hw->port), 1040 pf->stat_offsets_loaded, 1041 &osd->tx_size_127, &nsd->tx_size_127); 1042 i40e_stat_update48(hw, I40E_GLPRT_PTC255H(hw->port), 1043 I40E_GLPRT_PTC255L(hw->port), 1044 pf->stat_offsets_loaded, 1045 &osd->tx_size_255, &nsd->tx_size_255); 1046 i40e_stat_update48(hw, I40E_GLPRT_PTC511H(hw->port), 1047 I40E_GLPRT_PTC511L(hw->port), 1048 pf->stat_offsets_loaded, 1049 &osd->tx_size_511, &nsd->tx_size_511); 1050 i40e_stat_update48(hw, I40E_GLPRT_PTC1023H(hw->port), 1051 I40E_GLPRT_PTC1023L(hw->port), 1052 pf->stat_offsets_loaded, 1053 &osd->tx_size_1023, &nsd->tx_size_1023); 1054 i40e_stat_update48(hw, I40E_GLPRT_PTC1522H(hw->port), 1055 I40E_GLPRT_PTC1522L(hw->port), 1056 pf->stat_offsets_loaded, 1057 &osd->tx_size_1522, &nsd->tx_size_1522); 1058 i40e_stat_update48(hw, I40E_GLPRT_PTC9522H(hw->port), 1059 I40E_GLPRT_PTC9522L(hw->port), 1060 pf->stat_offsets_loaded, 1061 &osd->tx_size_big, &nsd->tx_size_big); 1062 1063 i40e_stat_update32(hw, I40E_GLPRT_RUC(hw->port), 1064 pf->stat_offsets_loaded, 1065 &osd->rx_undersize, &nsd->rx_undersize); 1066 i40e_stat_update32(hw, I40E_GLPRT_RFC(hw->port), 1067 pf->stat_offsets_loaded, 1068 &osd->rx_fragments, &nsd->rx_fragments); 1069 i40e_stat_update32(hw, I40E_GLPRT_ROC(hw->port), 1070 pf->stat_offsets_loaded, 1071 &osd->rx_oversize, &nsd->rx_oversize); 1072 i40e_stat_update32(hw, I40E_GLPRT_RJC(hw->port), 1073 pf->stat_offsets_loaded, 1074 &osd->rx_jabber, &nsd->rx_jabber); 1075 1076 /* FDIR stats */ 1077 i40e_stat_update32(hw, 1078 I40E_GLQF_PCNT(I40E_FD_ATR_STAT_IDX(pf->hw.pf_id)), 1079 pf->stat_offsets_loaded, 1080 &osd->fd_atr_match, &nsd->fd_atr_match); 1081 i40e_stat_update32(hw, 1082 I40E_GLQF_PCNT(I40E_FD_SB_STAT_IDX(pf->hw.pf_id)), 1083 pf->stat_offsets_loaded, 1084 &osd->fd_sb_match, &nsd->fd_sb_match); 1085 i40e_stat_update32(hw, 1086 I40E_GLQF_PCNT(I40E_FD_ATR_TUNNEL_STAT_IDX(pf->hw.pf_id)), 1087 pf->stat_offsets_loaded, 1088 &osd->fd_atr_tunnel_match, &nsd->fd_atr_tunnel_match); 1089 1090 val = rd32(hw, I40E_PRTPM_EEE_STAT); 1091 nsd->tx_lpi_status = 1092 (val & I40E_PRTPM_EEE_STAT_TX_LPI_STATUS_MASK) >> 1093 I40E_PRTPM_EEE_STAT_TX_LPI_STATUS_SHIFT; 1094 nsd->rx_lpi_status = 1095 (val & I40E_PRTPM_EEE_STAT_RX_LPI_STATUS_MASK) >> 1096 I40E_PRTPM_EEE_STAT_RX_LPI_STATUS_SHIFT; 1097 i40e_stat_update32(hw, I40E_PRTPM_TLPIC, 1098 pf->stat_offsets_loaded, 1099 &osd->tx_lpi_count, &nsd->tx_lpi_count); 1100 i40e_stat_update32(hw, I40E_PRTPM_RLPIC, 1101 pf->stat_offsets_loaded, 1102 &osd->rx_lpi_count, &nsd->rx_lpi_count); 1103 1104 if (pf->flags & I40E_FLAG_FD_SB_ENABLED && 1105 !(pf->auto_disable_flags & I40E_FLAG_FD_SB_ENABLED)) 1106 nsd->fd_sb_status = true; 1107 else 1108 nsd->fd_sb_status = false; 1109 1110 if (pf->flags & I40E_FLAG_FD_ATR_ENABLED && 1111 !(pf->auto_disable_flags & I40E_FLAG_FD_ATR_ENABLED)) 1112 nsd->fd_atr_status = true; 1113 else 1114 nsd->fd_atr_status = false; 1115 1116 pf->stat_offsets_loaded = true; 1117 } 1118 1119 /** 1120 * i40e_update_stats - Update the various statistics counters. 1121 * @vsi: the VSI to be updated 1122 * 1123 * Update the various stats for this VSI and its related entities. 1124 **/ 1125 void i40e_update_stats(struct i40e_vsi *vsi) 1126 { 1127 struct i40e_pf *pf = vsi->back; 1128 1129 if (vsi == pf->vsi[pf->lan_vsi]) 1130 i40e_update_pf_stats(pf); 1131 1132 i40e_update_vsi_stats(vsi); 1133 #ifdef I40E_FCOE 1134 i40e_update_fcoe_stats(vsi); 1135 #endif 1136 } 1137 1138 /** 1139 * i40e_find_filter - Search VSI filter list for specific mac/vlan filter 1140 * @vsi: the VSI to be searched 1141 * @macaddr: the MAC address 1142 * @vlan: the vlan 1143 * 1144 * Returns ptr to the filter object or NULL 1145 **/ 1146 static struct i40e_mac_filter *i40e_find_filter(struct i40e_vsi *vsi, 1147 const u8 *macaddr, s16 vlan) 1148 { 1149 struct i40e_mac_filter *f; 1150 u64 key; 1151 1152 if (!vsi || !macaddr) 1153 return NULL; 1154 1155 key = i40e_addr_to_hkey(macaddr); 1156 hash_for_each_possible(vsi->mac_filter_hash, f, hlist, key) { 1157 if ((ether_addr_equal(macaddr, f->macaddr)) && 1158 (vlan == f->vlan)) 1159 return f; 1160 } 1161 return NULL; 1162 } 1163 1164 /** 1165 * i40e_find_mac - Find a mac addr in the macvlan filters list 1166 * @vsi: the VSI to be searched 1167 * @macaddr: the MAC address we are searching for 1168 * 1169 * Returns the first filter with the provided MAC address or NULL if 1170 * MAC address was not found 1171 **/ 1172 struct i40e_mac_filter *i40e_find_mac(struct i40e_vsi *vsi, const u8 *macaddr) 1173 { 1174 struct i40e_mac_filter *f; 1175 u64 key; 1176 1177 if (!vsi || !macaddr) 1178 return NULL; 1179 1180 key = i40e_addr_to_hkey(macaddr); 1181 hash_for_each_possible(vsi->mac_filter_hash, f, hlist, key) { 1182 if ((ether_addr_equal(macaddr, f->macaddr))) 1183 return f; 1184 } 1185 return NULL; 1186 } 1187 1188 /** 1189 * i40e_is_vsi_in_vlan - Check if VSI is in vlan mode 1190 * @vsi: the VSI to be searched 1191 * 1192 * Returns true if VSI is in vlan mode or false otherwise 1193 **/ 1194 bool i40e_is_vsi_in_vlan(struct i40e_vsi *vsi) 1195 { 1196 /* If we have a PVID, always operate in VLAN mode */ 1197 if (vsi->info.pvid) 1198 return true; 1199 1200 /* We need to operate in VLAN mode whenever we have any filters with 1201 * a VLAN other than I40E_VLAN_ALL. We could check the table each 1202 * time, incurring search cost repeatedly. However, we can notice two 1203 * things: 1204 * 1205 * 1) the only place where we can gain a VLAN filter is in 1206 * i40e_add_filter. 1207 * 1208 * 2) the only place where filters are actually removed is in 1209 * i40e_sync_filters_subtask. 1210 * 1211 * Thus, we can simply use a boolean value, has_vlan_filters which we 1212 * will set to true when we add a VLAN filter in i40e_add_filter. Then 1213 * we have to perform the full search after deleting filters in 1214 * i40e_sync_filters_subtask, but we already have to search 1215 * filters here and can perform the check at the same time. This 1216 * results in avoiding embedding a loop for VLAN mode inside another 1217 * loop over all the filters, and should maintain correctness as noted 1218 * above. 1219 */ 1220 return vsi->has_vlan_filter; 1221 } 1222 1223 /** 1224 * i40e_correct_mac_vlan_filters - Correct non-VLAN filters if necessary 1225 * @vsi: the VSI to configure 1226 * @tmp_add_list: list of filters ready to be added 1227 * @tmp_del_list: list of filters ready to be deleted 1228 * @vlan_filters: the number of active VLAN filters 1229 * 1230 * Update VLAN=0 and VLAN=-1 (I40E_VLAN_ANY) filters properly so that they 1231 * behave as expected. If we have any active VLAN filters remaining or about 1232 * to be added then we need to update non-VLAN filters to be marked as VLAN=0 1233 * so that they only match against untagged traffic. If we no longer have any 1234 * active VLAN filters, we need to make all non-VLAN filters marked as VLAN=-1 1235 * so that they match against both tagged and untagged traffic. In this way, 1236 * we ensure that we correctly receive the desired traffic. This ensures that 1237 * when we have an active VLAN we will receive only untagged traffic and 1238 * traffic matching active VLANs. If we have no active VLANs then we will 1239 * operate in non-VLAN mode and receive all traffic, tagged or untagged. 1240 * 1241 * Finally, in a similar fashion, this function also corrects filters when 1242 * there is an active PVID assigned to this VSI. 1243 * 1244 * In case of memory allocation failure return -ENOMEM. Otherwise, return 0. 1245 * 1246 * This function is only expected to be called from within 1247 * i40e_sync_vsi_filters. 1248 * 1249 * NOTE: This function expects to be called while under the 1250 * mac_filter_hash_lock 1251 */ 1252 static int i40e_correct_mac_vlan_filters(struct i40e_vsi *vsi, 1253 struct hlist_head *tmp_add_list, 1254 struct hlist_head *tmp_del_list, 1255 int vlan_filters) 1256 { 1257 struct i40e_mac_filter *f, *add_head; 1258 struct hlist_node *h; 1259 int bkt, new_vlan; 1260 1261 /* To determine if a particular filter needs to be replaced we 1262 * have the three following conditions: 1263 * 1264 * a) if we have a PVID assigned, then all filters which are 1265 * not marked as VLAN=PVID must be replaced with filters that 1266 * are. 1267 * b) otherwise, if we have any active VLANS, all filters 1268 * which are marked as VLAN=-1 must be replaced with 1269 * filters marked as VLAN=0 1270 * c) finally, if we do not have any active VLANS, all filters 1271 * which are marked as VLAN=0 must be replaced with filters 1272 * marked as VLAN=-1 1273 */ 1274 1275 /* Update the filters about to be added in place */ 1276 hlist_for_each_entry(f, tmp_add_list, hlist) { 1277 if (vsi->info.pvid && f->vlan != vsi->info.pvid) 1278 f->vlan = vsi->info.pvid; 1279 else if (vlan_filters && f->vlan == I40E_VLAN_ANY) 1280 f->vlan = 0; 1281 else if (!vlan_filters && f->vlan == 0) 1282 f->vlan = I40E_VLAN_ANY; 1283 } 1284 1285 /* Update the remaining active filters */ 1286 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) { 1287 /* Combine the checks for whether a filter needs to be changed 1288 * and then determine the new VLAN inside the if block, in 1289 * order to avoid duplicating code for adding the new filter 1290 * then deleting the old filter. 1291 */ 1292 if ((vsi->info.pvid && f->vlan != vsi->info.pvid) || 1293 (vlan_filters && f->vlan == I40E_VLAN_ANY) || 1294 (!vlan_filters && f->vlan == 0)) { 1295 /* Determine the new vlan we will be adding */ 1296 if (vsi->info.pvid) 1297 new_vlan = vsi->info.pvid; 1298 else if (vlan_filters) 1299 new_vlan = 0; 1300 else 1301 new_vlan = I40E_VLAN_ANY; 1302 1303 /* Create the new filter */ 1304 add_head = i40e_add_filter(vsi, f->macaddr, new_vlan); 1305 if (!add_head) 1306 return -ENOMEM; 1307 1308 /* Put the replacement filter into the add list */ 1309 hash_del(&add_head->hlist); 1310 hlist_add_head(&add_head->hlist, tmp_add_list); 1311 1312 /* Put the original filter into the delete list */ 1313 f->state = I40E_FILTER_REMOVE; 1314 hash_del(&f->hlist); 1315 hlist_add_head(&f->hlist, tmp_del_list); 1316 } 1317 } 1318 1319 vsi->has_vlan_filter = !!vlan_filters; 1320 1321 return 0; 1322 } 1323 1324 /** 1325 * i40e_rm_default_mac_filter - Remove the default MAC filter set by NVM 1326 * @vsi: the PF Main VSI - inappropriate for any other VSI 1327 * @macaddr: the MAC address 1328 * 1329 * Remove whatever filter the firmware set up so the driver can manage 1330 * its own filtering intelligently. 1331 **/ 1332 static void i40e_rm_default_mac_filter(struct i40e_vsi *vsi, u8 *macaddr) 1333 { 1334 struct i40e_aqc_remove_macvlan_element_data element; 1335 struct i40e_pf *pf = vsi->back; 1336 1337 /* Only appropriate for the PF main VSI */ 1338 if (vsi->type != I40E_VSI_MAIN) 1339 return; 1340 1341 memset(&element, 0, sizeof(element)); 1342 ether_addr_copy(element.mac_addr, macaddr); 1343 element.vlan_tag = 0; 1344 /* Ignore error returns, some firmware does it this way... */ 1345 element.flags = I40E_AQC_MACVLAN_DEL_PERFECT_MATCH; 1346 i40e_aq_remove_macvlan(&pf->hw, vsi->seid, &element, 1, NULL); 1347 1348 memset(&element, 0, sizeof(element)); 1349 ether_addr_copy(element.mac_addr, macaddr); 1350 element.vlan_tag = 0; 1351 /* ...and some firmware does it this way. */ 1352 element.flags = I40E_AQC_MACVLAN_DEL_PERFECT_MATCH | 1353 I40E_AQC_MACVLAN_DEL_IGNORE_VLAN; 1354 i40e_aq_remove_macvlan(&pf->hw, vsi->seid, &element, 1, NULL); 1355 } 1356 1357 /** 1358 * i40e_add_filter - Add a mac/vlan filter to the VSI 1359 * @vsi: the VSI to be searched 1360 * @macaddr: the MAC address 1361 * @vlan: the vlan 1362 * 1363 * Returns ptr to the filter object or NULL when no memory available. 1364 * 1365 * NOTE: This function is expected to be called with mac_filter_hash_lock 1366 * being held. 1367 **/ 1368 struct i40e_mac_filter *i40e_add_filter(struct i40e_vsi *vsi, 1369 const u8 *macaddr, s16 vlan) 1370 { 1371 struct i40e_mac_filter *f; 1372 u64 key; 1373 1374 if (!vsi || !macaddr) 1375 return NULL; 1376 1377 f = i40e_find_filter(vsi, macaddr, vlan); 1378 if (!f) { 1379 f = kzalloc(sizeof(*f), GFP_ATOMIC); 1380 if (!f) 1381 return NULL; 1382 1383 /* Update the boolean indicating if we need to function in 1384 * VLAN mode. 1385 */ 1386 if (vlan >= 0) 1387 vsi->has_vlan_filter = true; 1388 1389 ether_addr_copy(f->macaddr, macaddr); 1390 f->vlan = vlan; 1391 /* If we're in overflow promisc mode, set the state directly 1392 * to failed, so we don't bother to try sending the filter 1393 * to the hardware. 1394 */ 1395 if (test_bit(__I40E_FILTER_OVERFLOW_PROMISC, &vsi->state)) 1396 f->state = I40E_FILTER_FAILED; 1397 else 1398 f->state = I40E_FILTER_NEW; 1399 INIT_HLIST_NODE(&f->hlist); 1400 1401 key = i40e_addr_to_hkey(macaddr); 1402 hash_add(vsi->mac_filter_hash, &f->hlist, key); 1403 1404 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED; 1405 vsi->back->flags |= I40E_FLAG_FILTER_SYNC; 1406 } 1407 1408 /* If we're asked to add a filter that has been marked for removal, it 1409 * is safe to simply restore it to active state. __i40e_del_filter 1410 * will have simply deleted any filters which were previously marked 1411 * NEW or FAILED, so if it is currently marked REMOVE it must have 1412 * previously been ACTIVE. Since we haven't yet run the sync filters 1413 * task, just restore this filter to the ACTIVE state so that the 1414 * sync task leaves it in place 1415 */ 1416 if (f->state == I40E_FILTER_REMOVE) 1417 f->state = I40E_FILTER_ACTIVE; 1418 1419 return f; 1420 } 1421 1422 /** 1423 * __i40e_del_filter - Remove a specific filter from the VSI 1424 * @vsi: VSI to remove from 1425 * @f: the filter to remove from the list 1426 * 1427 * This function should be called instead of i40e_del_filter only if you know 1428 * the exact filter you will remove already, such as via i40e_find_filter or 1429 * i40e_find_mac. 1430 * 1431 * NOTE: This function is expected to be called with mac_filter_hash_lock 1432 * being held. 1433 * ANOTHER NOTE: This function MUST be called from within the context of 1434 * the "safe" variants of any list iterators, e.g. list_for_each_entry_safe() 1435 * instead of list_for_each_entry(). 1436 **/ 1437 void __i40e_del_filter(struct i40e_vsi *vsi, struct i40e_mac_filter *f) 1438 { 1439 if (!f) 1440 return; 1441 1442 if ((f->state == I40E_FILTER_FAILED) || 1443 (f->state == I40E_FILTER_NEW)) { 1444 /* this one never got added by the FW. Just remove it, 1445 * no need to sync anything. 1446 */ 1447 hash_del(&f->hlist); 1448 kfree(f); 1449 } else { 1450 f->state = I40E_FILTER_REMOVE; 1451 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED; 1452 vsi->back->flags |= I40E_FLAG_FILTER_SYNC; 1453 } 1454 } 1455 1456 /** 1457 * i40e_del_filter - Remove a MAC/VLAN filter from the VSI 1458 * @vsi: the VSI to be searched 1459 * @macaddr: the MAC address 1460 * @vlan: the VLAN 1461 * 1462 * NOTE: This function is expected to be called with mac_filter_hash_lock 1463 * being held. 1464 * ANOTHER NOTE: This function MUST be called from within the context of 1465 * the "safe" variants of any list iterators, e.g. list_for_each_entry_safe() 1466 * instead of list_for_each_entry(). 1467 **/ 1468 void i40e_del_filter(struct i40e_vsi *vsi, const u8 *macaddr, s16 vlan) 1469 { 1470 struct i40e_mac_filter *f; 1471 1472 if (!vsi || !macaddr) 1473 return; 1474 1475 f = i40e_find_filter(vsi, macaddr, vlan); 1476 __i40e_del_filter(vsi, f); 1477 } 1478 1479 /** 1480 * i40e_add_mac_filter - Add a MAC filter for all active VLANs 1481 * @vsi: the VSI to be searched 1482 * @macaddr: the mac address to be filtered 1483 * 1484 * If we're not in VLAN mode, just add the filter to I40E_VLAN_ANY. Otherwise, 1485 * go through all the macvlan filters and add a macvlan filter for each 1486 * unique vlan that already exists. If a PVID has been assigned, instead only 1487 * add the macaddr to that VLAN. 1488 * 1489 * Returns last filter added on success, else NULL 1490 **/ 1491 struct i40e_mac_filter *i40e_add_mac_filter(struct i40e_vsi *vsi, 1492 const u8 *macaddr) 1493 { 1494 struct i40e_mac_filter *f, *add = NULL; 1495 struct hlist_node *h; 1496 int bkt; 1497 1498 if (vsi->info.pvid) 1499 return i40e_add_filter(vsi, macaddr, 1500 le16_to_cpu(vsi->info.pvid)); 1501 1502 if (!i40e_is_vsi_in_vlan(vsi)) 1503 return i40e_add_filter(vsi, macaddr, I40E_VLAN_ANY); 1504 1505 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) { 1506 if (f->state == I40E_FILTER_REMOVE) 1507 continue; 1508 add = i40e_add_filter(vsi, macaddr, f->vlan); 1509 if (!add) 1510 return NULL; 1511 } 1512 1513 return add; 1514 } 1515 1516 /** 1517 * i40e_del_mac_filter - Remove a MAC filter from all VLANs 1518 * @vsi: the VSI to be searched 1519 * @macaddr: the mac address to be removed 1520 * 1521 * Removes a given MAC address from a VSI regardless of what VLAN it has been 1522 * associated with. 1523 * 1524 * Returns 0 for success, or error 1525 **/ 1526 int i40e_del_mac_filter(struct i40e_vsi *vsi, const u8 *macaddr) 1527 { 1528 struct i40e_mac_filter *f; 1529 struct hlist_node *h; 1530 bool found = false; 1531 int bkt; 1532 1533 WARN(!spin_is_locked(&vsi->mac_filter_hash_lock), 1534 "Missing mac_filter_hash_lock\n"); 1535 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) { 1536 if (ether_addr_equal(macaddr, f->macaddr)) { 1537 __i40e_del_filter(vsi, f); 1538 found = true; 1539 } 1540 } 1541 1542 if (found) 1543 return 0; 1544 else 1545 return -ENOENT; 1546 } 1547 1548 /** 1549 * i40e_set_mac - NDO callback to set mac address 1550 * @netdev: network interface device structure 1551 * @p: pointer to an address structure 1552 * 1553 * Returns 0 on success, negative on failure 1554 **/ 1555 #ifdef I40E_FCOE 1556 int i40e_set_mac(struct net_device *netdev, void *p) 1557 #else 1558 static int i40e_set_mac(struct net_device *netdev, void *p) 1559 #endif 1560 { 1561 struct i40e_netdev_priv *np = netdev_priv(netdev); 1562 struct i40e_vsi *vsi = np->vsi; 1563 struct i40e_pf *pf = vsi->back; 1564 struct i40e_hw *hw = &pf->hw; 1565 struct sockaddr *addr = p; 1566 1567 if (!is_valid_ether_addr(addr->sa_data)) 1568 return -EADDRNOTAVAIL; 1569 1570 if (ether_addr_equal(netdev->dev_addr, addr->sa_data)) { 1571 netdev_info(netdev, "already using mac address %pM\n", 1572 addr->sa_data); 1573 return 0; 1574 } 1575 1576 if (test_bit(__I40E_DOWN, &vsi->back->state) || 1577 test_bit(__I40E_RESET_RECOVERY_PENDING, &vsi->back->state)) 1578 return -EADDRNOTAVAIL; 1579 1580 if (ether_addr_equal(hw->mac.addr, addr->sa_data)) 1581 netdev_info(netdev, "returning to hw mac address %pM\n", 1582 hw->mac.addr); 1583 else 1584 netdev_info(netdev, "set new mac address %pM\n", addr->sa_data); 1585 1586 spin_lock_bh(&vsi->mac_filter_hash_lock); 1587 i40e_del_mac_filter(vsi, netdev->dev_addr); 1588 i40e_add_mac_filter(vsi, addr->sa_data); 1589 spin_unlock_bh(&vsi->mac_filter_hash_lock); 1590 ether_addr_copy(netdev->dev_addr, addr->sa_data); 1591 if (vsi->type == I40E_VSI_MAIN) { 1592 i40e_status ret; 1593 1594 ret = i40e_aq_mac_address_write(&vsi->back->hw, 1595 I40E_AQC_WRITE_TYPE_LAA_WOL, 1596 addr->sa_data, NULL); 1597 if (ret) 1598 netdev_info(netdev, "Ignoring error from firmware on LAA update, status %s, AQ ret %s\n", 1599 i40e_stat_str(hw, ret), 1600 i40e_aq_str(hw, hw->aq.asq_last_status)); 1601 } 1602 1603 /* schedule our worker thread which will take care of 1604 * applying the new filter changes 1605 */ 1606 i40e_service_event_schedule(vsi->back); 1607 return 0; 1608 } 1609 1610 /** 1611 * i40e_vsi_setup_queue_map - Setup a VSI queue map based on enabled_tc 1612 * @vsi: the VSI being setup 1613 * @ctxt: VSI context structure 1614 * @enabled_tc: Enabled TCs bitmap 1615 * @is_add: True if called before Add VSI 1616 * 1617 * Setup VSI queue mapping for enabled traffic classes. 1618 **/ 1619 #ifdef I40E_FCOE 1620 void i40e_vsi_setup_queue_map(struct i40e_vsi *vsi, 1621 struct i40e_vsi_context *ctxt, 1622 u8 enabled_tc, 1623 bool is_add) 1624 #else 1625 static void i40e_vsi_setup_queue_map(struct i40e_vsi *vsi, 1626 struct i40e_vsi_context *ctxt, 1627 u8 enabled_tc, 1628 bool is_add) 1629 #endif 1630 { 1631 struct i40e_pf *pf = vsi->back; 1632 u16 sections = 0; 1633 u8 netdev_tc = 0; 1634 u16 numtc = 0; 1635 u16 qcount; 1636 u8 offset; 1637 u16 qmap; 1638 int i; 1639 u16 num_tc_qps = 0; 1640 1641 sections = I40E_AQ_VSI_PROP_QUEUE_MAP_VALID; 1642 offset = 0; 1643 1644 if (enabled_tc && (vsi->back->flags & I40E_FLAG_DCB_ENABLED)) { 1645 /* Find numtc from enabled TC bitmap */ 1646 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 1647 if (enabled_tc & BIT(i)) /* TC is enabled */ 1648 numtc++; 1649 } 1650 if (!numtc) { 1651 dev_warn(&pf->pdev->dev, "DCB is enabled but no TC enabled, forcing TC0\n"); 1652 numtc = 1; 1653 } 1654 } else { 1655 /* At least TC0 is enabled in case of non-DCB case */ 1656 numtc = 1; 1657 } 1658 1659 vsi->tc_config.numtc = numtc; 1660 vsi->tc_config.enabled_tc = enabled_tc ? enabled_tc : 1; 1661 /* Number of queues per enabled TC */ 1662 qcount = vsi->alloc_queue_pairs; 1663 1664 num_tc_qps = qcount / numtc; 1665 num_tc_qps = min_t(int, num_tc_qps, i40e_pf_get_max_q_per_tc(pf)); 1666 1667 /* Setup queue offset/count for all TCs for given VSI */ 1668 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 1669 /* See if the given TC is enabled for the given VSI */ 1670 if (vsi->tc_config.enabled_tc & BIT(i)) { 1671 /* TC is enabled */ 1672 int pow, num_qps; 1673 1674 switch (vsi->type) { 1675 case I40E_VSI_MAIN: 1676 qcount = min_t(int, pf->alloc_rss_size, 1677 num_tc_qps); 1678 break; 1679 #ifdef I40E_FCOE 1680 case I40E_VSI_FCOE: 1681 qcount = num_tc_qps; 1682 break; 1683 #endif 1684 case I40E_VSI_FDIR: 1685 case I40E_VSI_SRIOV: 1686 case I40E_VSI_VMDQ2: 1687 default: 1688 qcount = num_tc_qps; 1689 WARN_ON(i != 0); 1690 break; 1691 } 1692 vsi->tc_config.tc_info[i].qoffset = offset; 1693 vsi->tc_config.tc_info[i].qcount = qcount; 1694 1695 /* find the next higher power-of-2 of num queue pairs */ 1696 num_qps = qcount; 1697 pow = 0; 1698 while (num_qps && (BIT_ULL(pow) < qcount)) { 1699 pow++; 1700 num_qps >>= 1; 1701 } 1702 1703 vsi->tc_config.tc_info[i].netdev_tc = netdev_tc++; 1704 qmap = 1705 (offset << I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT) | 1706 (pow << I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT); 1707 1708 offset += qcount; 1709 } else { 1710 /* TC is not enabled so set the offset to 1711 * default queue and allocate one queue 1712 * for the given TC. 1713 */ 1714 vsi->tc_config.tc_info[i].qoffset = 0; 1715 vsi->tc_config.tc_info[i].qcount = 1; 1716 vsi->tc_config.tc_info[i].netdev_tc = 0; 1717 1718 qmap = 0; 1719 } 1720 ctxt->info.tc_mapping[i] = cpu_to_le16(qmap); 1721 } 1722 1723 /* Set actual Tx/Rx queue pairs */ 1724 vsi->num_queue_pairs = offset; 1725 if ((vsi->type == I40E_VSI_MAIN) && (numtc == 1)) { 1726 if (vsi->req_queue_pairs > 0) 1727 vsi->num_queue_pairs = vsi->req_queue_pairs; 1728 else if (pf->flags & I40E_FLAG_MSIX_ENABLED) 1729 vsi->num_queue_pairs = pf->num_lan_msix; 1730 } 1731 1732 /* Scheduler section valid can only be set for ADD VSI */ 1733 if (is_add) { 1734 sections |= I40E_AQ_VSI_PROP_SCHED_VALID; 1735 1736 ctxt->info.up_enable_bits = enabled_tc; 1737 } 1738 if (vsi->type == I40E_VSI_SRIOV) { 1739 ctxt->info.mapping_flags |= 1740 cpu_to_le16(I40E_AQ_VSI_QUE_MAP_NONCONTIG); 1741 for (i = 0; i < vsi->num_queue_pairs; i++) 1742 ctxt->info.queue_mapping[i] = 1743 cpu_to_le16(vsi->base_queue + i); 1744 } else { 1745 ctxt->info.mapping_flags |= 1746 cpu_to_le16(I40E_AQ_VSI_QUE_MAP_CONTIG); 1747 ctxt->info.queue_mapping[0] = cpu_to_le16(vsi->base_queue); 1748 } 1749 ctxt->info.valid_sections |= cpu_to_le16(sections); 1750 } 1751 1752 /** 1753 * i40e_addr_sync - Callback for dev_(mc|uc)_sync to add address 1754 * @netdev: the netdevice 1755 * @addr: address to add 1756 * 1757 * Called by __dev_(mc|uc)_sync when an address needs to be added. We call 1758 * __dev_(uc|mc)_sync from .set_rx_mode and guarantee to hold the hash lock. 1759 */ 1760 static int i40e_addr_sync(struct net_device *netdev, const u8 *addr) 1761 { 1762 struct i40e_netdev_priv *np = netdev_priv(netdev); 1763 struct i40e_vsi *vsi = np->vsi; 1764 1765 if (i40e_add_mac_filter(vsi, addr)) 1766 return 0; 1767 else 1768 return -ENOMEM; 1769 } 1770 1771 /** 1772 * i40e_addr_unsync - Callback for dev_(mc|uc)_sync to remove address 1773 * @netdev: the netdevice 1774 * @addr: address to add 1775 * 1776 * Called by __dev_(mc|uc)_sync when an address needs to be removed. We call 1777 * __dev_(uc|mc)_sync from .set_rx_mode and guarantee to hold the hash lock. 1778 */ 1779 static int i40e_addr_unsync(struct net_device *netdev, const u8 *addr) 1780 { 1781 struct i40e_netdev_priv *np = netdev_priv(netdev); 1782 struct i40e_vsi *vsi = np->vsi; 1783 1784 i40e_del_mac_filter(vsi, addr); 1785 1786 return 0; 1787 } 1788 1789 /** 1790 * i40e_set_rx_mode - NDO callback to set the netdev filters 1791 * @netdev: network interface device structure 1792 **/ 1793 #ifdef I40E_FCOE 1794 void i40e_set_rx_mode(struct net_device *netdev) 1795 #else 1796 static void i40e_set_rx_mode(struct net_device *netdev) 1797 #endif 1798 { 1799 struct i40e_netdev_priv *np = netdev_priv(netdev); 1800 struct i40e_vsi *vsi = np->vsi; 1801 1802 spin_lock_bh(&vsi->mac_filter_hash_lock); 1803 1804 __dev_uc_sync(netdev, i40e_addr_sync, i40e_addr_unsync); 1805 __dev_mc_sync(netdev, i40e_addr_sync, i40e_addr_unsync); 1806 1807 spin_unlock_bh(&vsi->mac_filter_hash_lock); 1808 1809 /* check for other flag changes */ 1810 if (vsi->current_netdev_flags != vsi->netdev->flags) { 1811 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED; 1812 vsi->back->flags |= I40E_FLAG_FILTER_SYNC; 1813 } 1814 1815 /* schedule our worker thread which will take care of 1816 * applying the new filter changes 1817 */ 1818 i40e_service_event_schedule(vsi->back); 1819 } 1820 1821 /** 1822 * i40e_undo_filter_entries - Undo the changes made to MAC filter entries 1823 * @vsi: Pointer to VSI struct 1824 * @from: Pointer to list which contains MAC filter entries - changes to 1825 * those entries needs to be undone. 1826 * 1827 * MAC filter entries from list were slated to be sent to firmware, either for 1828 * addition or deletion. 1829 **/ 1830 static void i40e_undo_filter_entries(struct i40e_vsi *vsi, 1831 struct hlist_head *from) 1832 { 1833 struct i40e_mac_filter *f; 1834 struct hlist_node *h; 1835 1836 hlist_for_each_entry_safe(f, h, from, hlist) { 1837 u64 key = i40e_addr_to_hkey(f->macaddr); 1838 1839 /* Move the element back into MAC filter list*/ 1840 hlist_del(&f->hlist); 1841 hash_add(vsi->mac_filter_hash, &f->hlist, key); 1842 } 1843 } 1844 1845 /** 1846 * i40e_update_filter_state - Update filter state based on return data 1847 * from firmware 1848 * @count: Number of filters added 1849 * @add_list: return data from fw 1850 * @head: pointer to first filter in current batch 1851 * 1852 * MAC filter entries from list were slated to be added to device. Returns 1853 * number of successful filters. Note that 0 does NOT mean success! 1854 **/ 1855 static int 1856 i40e_update_filter_state(int count, 1857 struct i40e_aqc_add_macvlan_element_data *add_list, 1858 struct i40e_mac_filter *add_head) 1859 { 1860 int retval = 0; 1861 int i; 1862 1863 for (i = 0; i < count; i++) { 1864 /* Always check status of each filter. We don't need to check 1865 * the firmware return status because we pre-set the filter 1866 * status to I40E_AQC_MM_ERR_NO_RES when sending the filter 1867 * request to the adminq. Thus, if it no longer matches then 1868 * we know the filter is active. 1869 */ 1870 if (add_list[i].match_method == I40E_AQC_MM_ERR_NO_RES) { 1871 add_head->state = I40E_FILTER_FAILED; 1872 } else { 1873 add_head->state = I40E_FILTER_ACTIVE; 1874 retval++; 1875 } 1876 1877 add_head = hlist_entry(add_head->hlist.next, 1878 typeof(struct i40e_mac_filter), 1879 hlist); 1880 } 1881 1882 return retval; 1883 } 1884 1885 /** 1886 * i40e_aqc_del_filters - Request firmware to delete a set of filters 1887 * @vsi: ptr to the VSI 1888 * @vsi_name: name to display in messages 1889 * @list: the list of filters to send to firmware 1890 * @num_del: the number of filters to delete 1891 * @retval: Set to -EIO on failure to delete 1892 * 1893 * Send a request to firmware via AdminQ to delete a set of filters. Uses 1894 * *retval instead of a return value so that success does not force ret_val to 1895 * be set to 0. This ensures that a sequence of calls to this function 1896 * preserve the previous value of *retval on successful delete. 1897 */ 1898 static 1899 void i40e_aqc_del_filters(struct i40e_vsi *vsi, const char *vsi_name, 1900 struct i40e_aqc_remove_macvlan_element_data *list, 1901 int num_del, int *retval) 1902 { 1903 struct i40e_hw *hw = &vsi->back->hw; 1904 i40e_status aq_ret; 1905 int aq_err; 1906 1907 aq_ret = i40e_aq_remove_macvlan(hw, vsi->seid, list, num_del, NULL); 1908 aq_err = hw->aq.asq_last_status; 1909 1910 /* Explicitly ignore and do not report when firmware returns ENOENT */ 1911 if (aq_ret && !(aq_err == I40E_AQ_RC_ENOENT)) { 1912 *retval = -EIO; 1913 dev_info(&vsi->back->pdev->dev, 1914 "ignoring delete macvlan error on %s, err %s, aq_err %s\n", 1915 vsi_name, i40e_stat_str(hw, aq_ret), 1916 i40e_aq_str(hw, aq_err)); 1917 } 1918 } 1919 1920 /** 1921 * i40e_aqc_add_filters - Request firmware to add a set of filters 1922 * @vsi: ptr to the VSI 1923 * @vsi_name: name to display in messages 1924 * @list: the list of filters to send to firmware 1925 * @add_head: Position in the add hlist 1926 * @num_add: the number of filters to add 1927 * @promisc_change: set to true on exit if promiscuous mode was forced on 1928 * 1929 * Send a request to firmware via AdminQ to add a chunk of filters. Will set 1930 * promisc_changed to true if the firmware has run out of space for more 1931 * filters. 1932 */ 1933 static 1934 void i40e_aqc_add_filters(struct i40e_vsi *vsi, const char *vsi_name, 1935 struct i40e_aqc_add_macvlan_element_data *list, 1936 struct i40e_mac_filter *add_head, 1937 int num_add, bool *promisc_changed) 1938 { 1939 struct i40e_hw *hw = &vsi->back->hw; 1940 int aq_err, fcnt; 1941 1942 i40e_aq_add_macvlan(hw, vsi->seid, list, num_add, NULL); 1943 aq_err = hw->aq.asq_last_status; 1944 fcnt = i40e_update_filter_state(num_add, list, add_head); 1945 1946 if (fcnt != num_add) { 1947 *promisc_changed = true; 1948 set_bit(__I40E_FILTER_OVERFLOW_PROMISC, &vsi->state); 1949 dev_warn(&vsi->back->pdev->dev, 1950 "Error %s adding RX filters on %s, promiscuous mode forced on\n", 1951 i40e_aq_str(hw, aq_err), 1952 vsi_name); 1953 } 1954 } 1955 1956 /** 1957 * i40e_aqc_broadcast_filter - Set promiscuous broadcast flags 1958 * @vsi: pointer to the VSI 1959 * @f: filter data 1960 * 1961 * This function sets or clears the promiscuous broadcast flags for VLAN 1962 * filters in order to properly receive broadcast frames. Assumes that only 1963 * broadcast filters are passed. 1964 **/ 1965 static 1966 void i40e_aqc_broadcast_filter(struct i40e_vsi *vsi, const char *vsi_name, 1967 struct i40e_mac_filter *f) 1968 { 1969 bool enable = f->state == I40E_FILTER_NEW; 1970 struct i40e_hw *hw = &vsi->back->hw; 1971 i40e_status aq_ret; 1972 1973 if (f->vlan == I40E_VLAN_ANY) { 1974 aq_ret = i40e_aq_set_vsi_broadcast(hw, 1975 vsi->seid, 1976 enable, 1977 NULL); 1978 } else { 1979 aq_ret = i40e_aq_set_vsi_bc_promisc_on_vlan(hw, 1980 vsi->seid, 1981 enable, 1982 f->vlan, 1983 NULL); 1984 } 1985 1986 if (aq_ret) { 1987 dev_warn(&vsi->back->pdev->dev, 1988 "Error %s setting broadcast promiscuous mode on %s\n", 1989 i40e_aq_str(hw, hw->aq.asq_last_status), 1990 vsi_name); 1991 f->state = I40E_FILTER_FAILED; 1992 } else if (enable) { 1993 f->state = I40E_FILTER_ACTIVE; 1994 } 1995 } 1996 1997 /** 1998 * i40e_sync_vsi_filters - Update the VSI filter list to the HW 1999 * @vsi: ptr to the VSI 2000 * 2001 * Push any outstanding VSI filter changes through the AdminQ. 2002 * 2003 * Returns 0 or error value 2004 **/ 2005 int i40e_sync_vsi_filters(struct i40e_vsi *vsi) 2006 { 2007 struct hlist_head tmp_add_list, tmp_del_list; 2008 struct i40e_mac_filter *f, *add_head = NULL; 2009 struct i40e_hw *hw = &vsi->back->hw; 2010 unsigned int failed_filters = 0; 2011 unsigned int vlan_filters = 0; 2012 bool promisc_changed = false; 2013 char vsi_name[16] = "PF"; 2014 int filter_list_len = 0; 2015 i40e_status aq_ret = 0; 2016 u32 changed_flags = 0; 2017 struct hlist_node *h; 2018 struct i40e_pf *pf; 2019 int num_add = 0; 2020 int num_del = 0; 2021 int retval = 0; 2022 u16 cmd_flags; 2023 int list_size; 2024 int bkt; 2025 2026 /* empty array typed pointers, kcalloc later */ 2027 struct i40e_aqc_add_macvlan_element_data *add_list; 2028 struct i40e_aqc_remove_macvlan_element_data *del_list; 2029 2030 while (test_and_set_bit(__I40E_CONFIG_BUSY, &vsi->state)) 2031 usleep_range(1000, 2000); 2032 pf = vsi->back; 2033 2034 if (vsi->netdev) { 2035 changed_flags = vsi->current_netdev_flags ^ vsi->netdev->flags; 2036 vsi->current_netdev_flags = vsi->netdev->flags; 2037 } 2038 2039 INIT_HLIST_HEAD(&tmp_add_list); 2040 INIT_HLIST_HEAD(&tmp_del_list); 2041 2042 if (vsi->type == I40E_VSI_SRIOV) 2043 snprintf(vsi_name, sizeof(vsi_name) - 1, "VF %d", vsi->vf_id); 2044 else if (vsi->type != I40E_VSI_MAIN) 2045 snprintf(vsi_name, sizeof(vsi_name) - 1, "vsi %d", vsi->seid); 2046 2047 if (vsi->flags & I40E_VSI_FLAG_FILTER_CHANGED) { 2048 vsi->flags &= ~I40E_VSI_FLAG_FILTER_CHANGED; 2049 2050 spin_lock_bh(&vsi->mac_filter_hash_lock); 2051 /* Create a list of filters to delete. */ 2052 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) { 2053 if (f->state == I40E_FILTER_REMOVE) { 2054 /* Move the element into temporary del_list */ 2055 hash_del(&f->hlist); 2056 hlist_add_head(&f->hlist, &tmp_del_list); 2057 2058 /* Avoid counting removed filters */ 2059 continue; 2060 } 2061 if (f->state == I40E_FILTER_NEW) { 2062 hash_del(&f->hlist); 2063 hlist_add_head(&f->hlist, &tmp_add_list); 2064 } 2065 2066 /* Count the number of active (current and new) VLAN 2067 * filters we have now. Does not count filters which 2068 * are marked for deletion. 2069 */ 2070 if (f->vlan > 0) 2071 vlan_filters++; 2072 } 2073 2074 retval = i40e_correct_mac_vlan_filters(vsi, 2075 &tmp_add_list, 2076 &tmp_del_list, 2077 vlan_filters); 2078 if (retval) 2079 goto err_no_memory_locked; 2080 2081 spin_unlock_bh(&vsi->mac_filter_hash_lock); 2082 } 2083 2084 /* Now process 'del_list' outside the lock */ 2085 if (!hlist_empty(&tmp_del_list)) { 2086 filter_list_len = hw->aq.asq_buf_size / 2087 sizeof(struct i40e_aqc_remove_macvlan_element_data); 2088 list_size = filter_list_len * 2089 sizeof(struct i40e_aqc_remove_macvlan_element_data); 2090 del_list = kzalloc(list_size, GFP_ATOMIC); 2091 if (!del_list) 2092 goto err_no_memory; 2093 2094 hlist_for_each_entry_safe(f, h, &tmp_del_list, hlist) { 2095 cmd_flags = 0; 2096 2097 /* handle broadcast filters by updating the broadcast 2098 * promiscuous flag instead of deleting a MAC filter. 2099 */ 2100 if (is_broadcast_ether_addr(f->macaddr)) { 2101 i40e_aqc_broadcast_filter(vsi, vsi_name, f); 2102 2103 hlist_del(&f->hlist); 2104 kfree(f); 2105 continue; 2106 } 2107 2108 /* add to delete list */ 2109 ether_addr_copy(del_list[num_del].mac_addr, f->macaddr); 2110 if (f->vlan == I40E_VLAN_ANY) { 2111 del_list[num_del].vlan_tag = 0; 2112 cmd_flags |= I40E_AQC_MACVLAN_DEL_IGNORE_VLAN; 2113 } else { 2114 del_list[num_del].vlan_tag = 2115 cpu_to_le16((u16)(f->vlan)); 2116 } 2117 2118 cmd_flags |= I40E_AQC_MACVLAN_DEL_PERFECT_MATCH; 2119 del_list[num_del].flags = cmd_flags; 2120 num_del++; 2121 2122 /* flush a full buffer */ 2123 if (num_del == filter_list_len) { 2124 i40e_aqc_del_filters(vsi, vsi_name, del_list, 2125 num_del, &retval); 2126 memset(del_list, 0, list_size); 2127 num_del = 0; 2128 } 2129 /* Release memory for MAC filter entries which were 2130 * synced up with HW. 2131 */ 2132 hlist_del(&f->hlist); 2133 kfree(f); 2134 } 2135 2136 if (num_del) { 2137 i40e_aqc_del_filters(vsi, vsi_name, del_list, 2138 num_del, &retval); 2139 } 2140 2141 kfree(del_list); 2142 del_list = NULL; 2143 } 2144 2145 if (!hlist_empty(&tmp_add_list)) { 2146 /* Do all the adds now. */ 2147 filter_list_len = hw->aq.asq_buf_size / 2148 sizeof(struct i40e_aqc_add_macvlan_element_data); 2149 list_size = filter_list_len * 2150 sizeof(struct i40e_aqc_add_macvlan_element_data); 2151 add_list = kzalloc(list_size, GFP_ATOMIC); 2152 if (!add_list) 2153 goto err_no_memory; 2154 2155 num_add = 0; 2156 hlist_for_each_entry_safe(f, h, &tmp_add_list, hlist) { 2157 if (test_bit(__I40E_FILTER_OVERFLOW_PROMISC, 2158 &vsi->state)) { 2159 f->state = I40E_FILTER_FAILED; 2160 continue; 2161 } 2162 2163 /* handle broadcast filters by updating the broadcast 2164 * promiscuous flag instead of adding a MAC filter. 2165 */ 2166 if (is_broadcast_ether_addr(f->macaddr)) { 2167 u64 key = i40e_addr_to_hkey(f->macaddr); 2168 i40e_aqc_broadcast_filter(vsi, vsi_name, f); 2169 2170 hlist_del(&f->hlist); 2171 hash_add(vsi->mac_filter_hash, &f->hlist, key); 2172 continue; 2173 } 2174 2175 /* add to add array */ 2176 if (num_add == 0) 2177 add_head = f; 2178 cmd_flags = 0; 2179 ether_addr_copy(add_list[num_add].mac_addr, f->macaddr); 2180 if (f->vlan == I40E_VLAN_ANY) { 2181 add_list[num_add].vlan_tag = 0; 2182 cmd_flags |= I40E_AQC_MACVLAN_ADD_IGNORE_VLAN; 2183 } else { 2184 add_list[num_add].vlan_tag = 2185 cpu_to_le16((u16)(f->vlan)); 2186 } 2187 add_list[num_add].queue_number = 0; 2188 /* set invalid match method for later detection */ 2189 add_list[num_add].match_method = I40E_AQC_MM_ERR_NO_RES; 2190 cmd_flags |= I40E_AQC_MACVLAN_ADD_PERFECT_MATCH; 2191 add_list[num_add].flags = cpu_to_le16(cmd_flags); 2192 num_add++; 2193 2194 /* flush a full buffer */ 2195 if (num_add == filter_list_len) { 2196 i40e_aqc_add_filters(vsi, vsi_name, add_list, 2197 add_head, num_add, 2198 &promisc_changed); 2199 memset(add_list, 0, list_size); 2200 num_add = 0; 2201 } 2202 } 2203 if (num_add) { 2204 i40e_aqc_add_filters(vsi, vsi_name, add_list, add_head, 2205 num_add, &promisc_changed); 2206 } 2207 /* Now move all of the filters from the temp add list back to 2208 * the VSI's list. 2209 */ 2210 spin_lock_bh(&vsi->mac_filter_hash_lock); 2211 hlist_for_each_entry_safe(f, h, &tmp_add_list, hlist) { 2212 u64 key = i40e_addr_to_hkey(f->macaddr); 2213 2214 hlist_del(&f->hlist); 2215 hash_add(vsi->mac_filter_hash, &f->hlist, key); 2216 } 2217 spin_unlock_bh(&vsi->mac_filter_hash_lock); 2218 kfree(add_list); 2219 add_list = NULL; 2220 } 2221 2222 /* Determine the number of active and failed filters. */ 2223 spin_lock_bh(&vsi->mac_filter_hash_lock); 2224 vsi->active_filters = 0; 2225 hash_for_each(vsi->mac_filter_hash, bkt, f, hlist) { 2226 if (f->state == I40E_FILTER_ACTIVE) 2227 vsi->active_filters++; 2228 else if (f->state == I40E_FILTER_FAILED) 2229 failed_filters++; 2230 } 2231 spin_unlock_bh(&vsi->mac_filter_hash_lock); 2232 2233 /* If promiscuous mode has changed, we need to calculate a new 2234 * threshold for when we are safe to exit 2235 */ 2236 if (promisc_changed) 2237 vsi->promisc_threshold = (vsi->active_filters * 3) / 4; 2238 2239 /* Check if we are able to exit overflow promiscuous mode. We can 2240 * safely exit if we didn't just enter, we no longer have any failed 2241 * filters, and we have reduced filters below the threshold value. 2242 */ 2243 if (test_bit(__I40E_FILTER_OVERFLOW_PROMISC, &vsi->state) && 2244 !promisc_changed && !failed_filters && 2245 (vsi->active_filters < vsi->promisc_threshold)) { 2246 dev_info(&pf->pdev->dev, 2247 "filter logjam cleared on %s, leaving overflow promiscuous mode\n", 2248 vsi_name); 2249 clear_bit(__I40E_FILTER_OVERFLOW_PROMISC, &vsi->state); 2250 promisc_changed = true; 2251 vsi->promisc_threshold = 0; 2252 } 2253 2254 /* if the VF is not trusted do not do promisc */ 2255 if ((vsi->type == I40E_VSI_SRIOV) && !pf->vf[vsi->vf_id].trusted) { 2256 clear_bit(__I40E_FILTER_OVERFLOW_PROMISC, &vsi->state); 2257 goto out; 2258 } 2259 2260 /* check for changes in promiscuous modes */ 2261 if (changed_flags & IFF_ALLMULTI) { 2262 bool cur_multipromisc; 2263 2264 cur_multipromisc = !!(vsi->current_netdev_flags & IFF_ALLMULTI); 2265 aq_ret = i40e_aq_set_vsi_multicast_promiscuous(&vsi->back->hw, 2266 vsi->seid, 2267 cur_multipromisc, 2268 NULL); 2269 if (aq_ret) { 2270 retval = i40e_aq_rc_to_posix(aq_ret, 2271 hw->aq.asq_last_status); 2272 dev_info(&pf->pdev->dev, 2273 "set multi promisc failed on %s, err %s aq_err %s\n", 2274 vsi_name, 2275 i40e_stat_str(hw, aq_ret), 2276 i40e_aq_str(hw, hw->aq.asq_last_status)); 2277 } 2278 } 2279 if ((changed_flags & IFF_PROMISC) || 2280 (promisc_changed && 2281 test_bit(__I40E_FILTER_OVERFLOW_PROMISC, &vsi->state))) { 2282 bool cur_promisc; 2283 2284 cur_promisc = (!!(vsi->current_netdev_flags & IFF_PROMISC) || 2285 test_bit(__I40E_FILTER_OVERFLOW_PROMISC, 2286 &vsi->state)); 2287 if ((vsi->type == I40E_VSI_MAIN) && 2288 (pf->lan_veb != I40E_NO_VEB) && 2289 !(pf->flags & I40E_FLAG_MFP_ENABLED)) { 2290 /* set defport ON for Main VSI instead of true promisc 2291 * this way we will get all unicast/multicast and VLAN 2292 * promisc behavior but will not get VF or VMDq traffic 2293 * replicated on the Main VSI. 2294 */ 2295 if (pf->cur_promisc != cur_promisc) { 2296 pf->cur_promisc = cur_promisc; 2297 if (cur_promisc) 2298 aq_ret = 2299 i40e_aq_set_default_vsi(hw, 2300 vsi->seid, 2301 NULL); 2302 else 2303 aq_ret = 2304 i40e_aq_clear_default_vsi(hw, 2305 vsi->seid, 2306 NULL); 2307 if (aq_ret) { 2308 retval = i40e_aq_rc_to_posix(aq_ret, 2309 hw->aq.asq_last_status); 2310 dev_info(&pf->pdev->dev, 2311 "Set default VSI failed on %s, err %s, aq_err %s\n", 2312 vsi_name, 2313 i40e_stat_str(hw, aq_ret), 2314 i40e_aq_str(hw, 2315 hw->aq.asq_last_status)); 2316 } 2317 } 2318 } else { 2319 aq_ret = i40e_aq_set_vsi_unicast_promiscuous( 2320 hw, 2321 vsi->seid, 2322 cur_promisc, NULL, 2323 true); 2324 if (aq_ret) { 2325 retval = 2326 i40e_aq_rc_to_posix(aq_ret, 2327 hw->aq.asq_last_status); 2328 dev_info(&pf->pdev->dev, 2329 "set unicast promisc failed on %s, err %s, aq_err %s\n", 2330 vsi_name, 2331 i40e_stat_str(hw, aq_ret), 2332 i40e_aq_str(hw, 2333 hw->aq.asq_last_status)); 2334 } 2335 aq_ret = i40e_aq_set_vsi_multicast_promiscuous( 2336 hw, 2337 vsi->seid, 2338 cur_promisc, NULL); 2339 if (aq_ret) { 2340 retval = 2341 i40e_aq_rc_to_posix(aq_ret, 2342 hw->aq.asq_last_status); 2343 dev_info(&pf->pdev->dev, 2344 "set multicast promisc failed on %s, err %s, aq_err %s\n", 2345 vsi_name, 2346 i40e_stat_str(hw, aq_ret), 2347 i40e_aq_str(hw, 2348 hw->aq.asq_last_status)); 2349 } 2350 } 2351 aq_ret = i40e_aq_set_vsi_broadcast(&vsi->back->hw, 2352 vsi->seid, 2353 cur_promisc, NULL); 2354 if (aq_ret) { 2355 retval = i40e_aq_rc_to_posix(aq_ret, 2356 pf->hw.aq.asq_last_status); 2357 dev_info(&pf->pdev->dev, 2358 "set brdcast promisc failed, err %s, aq_err %s\n", 2359 i40e_stat_str(hw, aq_ret), 2360 i40e_aq_str(hw, 2361 hw->aq.asq_last_status)); 2362 } 2363 } 2364 out: 2365 /* if something went wrong then set the changed flag so we try again */ 2366 if (retval) 2367 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED; 2368 2369 clear_bit(__I40E_CONFIG_BUSY, &vsi->state); 2370 return retval; 2371 2372 err_no_memory: 2373 /* Restore elements on the temporary add and delete lists */ 2374 spin_lock_bh(&vsi->mac_filter_hash_lock); 2375 err_no_memory_locked: 2376 i40e_undo_filter_entries(vsi, &tmp_del_list); 2377 i40e_undo_filter_entries(vsi, &tmp_add_list); 2378 spin_unlock_bh(&vsi->mac_filter_hash_lock); 2379 2380 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED; 2381 clear_bit(__I40E_CONFIG_BUSY, &vsi->state); 2382 return -ENOMEM; 2383 } 2384 2385 /** 2386 * i40e_sync_filters_subtask - Sync the VSI filter list with HW 2387 * @pf: board private structure 2388 **/ 2389 static void i40e_sync_filters_subtask(struct i40e_pf *pf) 2390 { 2391 int v; 2392 2393 if (!pf || !(pf->flags & I40E_FLAG_FILTER_SYNC)) 2394 return; 2395 pf->flags &= ~I40E_FLAG_FILTER_SYNC; 2396 2397 for (v = 0; v < pf->num_alloc_vsi; v++) { 2398 if (pf->vsi[v] && 2399 (pf->vsi[v]->flags & I40E_VSI_FLAG_FILTER_CHANGED)) { 2400 int ret = i40e_sync_vsi_filters(pf->vsi[v]); 2401 2402 if (ret) { 2403 /* come back and try again later */ 2404 pf->flags |= I40E_FLAG_FILTER_SYNC; 2405 break; 2406 } 2407 } 2408 } 2409 } 2410 2411 /** 2412 * i40e_change_mtu - NDO callback to change the Maximum Transfer Unit 2413 * @netdev: network interface device structure 2414 * @new_mtu: new value for maximum frame size 2415 * 2416 * Returns 0 on success, negative on failure 2417 **/ 2418 static int i40e_change_mtu(struct net_device *netdev, int new_mtu) 2419 { 2420 struct i40e_netdev_priv *np = netdev_priv(netdev); 2421 struct i40e_vsi *vsi = np->vsi; 2422 2423 netdev_info(netdev, "changing MTU from %d to %d\n", 2424 netdev->mtu, new_mtu); 2425 netdev->mtu = new_mtu; 2426 if (netif_running(netdev)) 2427 i40e_vsi_reinit_locked(vsi); 2428 i40e_notify_client_of_l2_param_changes(vsi); 2429 return 0; 2430 } 2431 2432 /** 2433 * i40e_ioctl - Access the hwtstamp interface 2434 * @netdev: network interface device structure 2435 * @ifr: interface request data 2436 * @cmd: ioctl command 2437 **/ 2438 int i40e_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd) 2439 { 2440 struct i40e_netdev_priv *np = netdev_priv(netdev); 2441 struct i40e_pf *pf = np->vsi->back; 2442 2443 switch (cmd) { 2444 case SIOCGHWTSTAMP: 2445 return i40e_ptp_get_ts_config(pf, ifr); 2446 case SIOCSHWTSTAMP: 2447 return i40e_ptp_set_ts_config(pf, ifr); 2448 default: 2449 return -EOPNOTSUPP; 2450 } 2451 } 2452 2453 /** 2454 * i40e_vlan_stripping_enable - Turn on vlan stripping for the VSI 2455 * @vsi: the vsi being adjusted 2456 **/ 2457 void i40e_vlan_stripping_enable(struct i40e_vsi *vsi) 2458 { 2459 struct i40e_vsi_context ctxt; 2460 i40e_status ret; 2461 2462 if ((vsi->info.valid_sections & 2463 cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID)) && 2464 ((vsi->info.port_vlan_flags & I40E_AQ_VSI_PVLAN_MODE_MASK) == 0)) 2465 return; /* already enabled */ 2466 2467 vsi->info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID); 2468 vsi->info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_ALL | 2469 I40E_AQ_VSI_PVLAN_EMOD_STR_BOTH; 2470 2471 ctxt.seid = vsi->seid; 2472 ctxt.info = vsi->info; 2473 ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL); 2474 if (ret) { 2475 dev_info(&vsi->back->pdev->dev, 2476 "update vlan stripping failed, err %s aq_err %s\n", 2477 i40e_stat_str(&vsi->back->hw, ret), 2478 i40e_aq_str(&vsi->back->hw, 2479 vsi->back->hw.aq.asq_last_status)); 2480 } 2481 } 2482 2483 /** 2484 * i40e_vlan_stripping_disable - Turn off vlan stripping for the VSI 2485 * @vsi: the vsi being adjusted 2486 **/ 2487 void i40e_vlan_stripping_disable(struct i40e_vsi *vsi) 2488 { 2489 struct i40e_vsi_context ctxt; 2490 i40e_status ret; 2491 2492 if ((vsi->info.valid_sections & 2493 cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID)) && 2494 ((vsi->info.port_vlan_flags & I40E_AQ_VSI_PVLAN_EMOD_MASK) == 2495 I40E_AQ_VSI_PVLAN_EMOD_MASK)) 2496 return; /* already disabled */ 2497 2498 vsi->info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID); 2499 vsi->info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_ALL | 2500 I40E_AQ_VSI_PVLAN_EMOD_NOTHING; 2501 2502 ctxt.seid = vsi->seid; 2503 ctxt.info = vsi->info; 2504 ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL); 2505 if (ret) { 2506 dev_info(&vsi->back->pdev->dev, 2507 "update vlan stripping failed, err %s aq_err %s\n", 2508 i40e_stat_str(&vsi->back->hw, ret), 2509 i40e_aq_str(&vsi->back->hw, 2510 vsi->back->hw.aq.asq_last_status)); 2511 } 2512 } 2513 2514 /** 2515 * i40e_vlan_rx_register - Setup or shutdown vlan offload 2516 * @netdev: network interface to be adjusted 2517 * @features: netdev features to test if VLAN offload is enabled or not 2518 **/ 2519 static void i40e_vlan_rx_register(struct net_device *netdev, u32 features) 2520 { 2521 struct i40e_netdev_priv *np = netdev_priv(netdev); 2522 struct i40e_vsi *vsi = np->vsi; 2523 2524 if (features & NETIF_F_HW_VLAN_CTAG_RX) 2525 i40e_vlan_stripping_enable(vsi); 2526 else 2527 i40e_vlan_stripping_disable(vsi); 2528 } 2529 2530 /** 2531 * i40e_add_vlan_all_mac - Add a MAC/VLAN filter for each existing MAC address 2532 * @vsi: the vsi being configured 2533 * @vid: vlan id to be added (0 = untagged only , -1 = any) 2534 * 2535 * This is a helper function for adding a new MAC/VLAN filter with the 2536 * specified VLAN for each existing MAC address already in the hash table. 2537 * This function does *not* perform any accounting to update filters based on 2538 * VLAN mode. 2539 * 2540 * NOTE: this function expects to be called while under the 2541 * mac_filter_hash_lock 2542 **/ 2543 int i40e_add_vlan_all_mac(struct i40e_vsi *vsi, s16 vid) 2544 { 2545 struct i40e_mac_filter *f, *add_f; 2546 struct hlist_node *h; 2547 int bkt; 2548 2549 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) { 2550 if (f->state == I40E_FILTER_REMOVE) 2551 continue; 2552 add_f = i40e_add_filter(vsi, f->macaddr, vid); 2553 if (!add_f) { 2554 dev_info(&vsi->back->pdev->dev, 2555 "Could not add vlan filter %d for %pM\n", 2556 vid, f->macaddr); 2557 return -ENOMEM; 2558 } 2559 } 2560 2561 return 0; 2562 } 2563 2564 /** 2565 * i40e_vsi_add_vlan - Add VSI membership for given VLAN 2566 * @vsi: the VSI being configured 2567 * @vid: VLAN id to be added 2568 **/ 2569 int i40e_vsi_add_vlan(struct i40e_vsi *vsi, u16 vid) 2570 { 2571 int err; 2572 2573 if (!vid || vsi->info.pvid) 2574 return -EINVAL; 2575 2576 /* Locked once because all functions invoked below iterates list*/ 2577 spin_lock_bh(&vsi->mac_filter_hash_lock); 2578 err = i40e_add_vlan_all_mac(vsi, vid); 2579 spin_unlock_bh(&vsi->mac_filter_hash_lock); 2580 if (err) 2581 return err; 2582 2583 /* schedule our worker thread which will take care of 2584 * applying the new filter changes 2585 */ 2586 i40e_service_event_schedule(vsi->back); 2587 return 0; 2588 } 2589 2590 /** 2591 * i40e_rm_vlan_all_mac - Remove MAC/VLAN pair for all MAC with the given VLAN 2592 * @vsi: the vsi being configured 2593 * @vid: vlan id to be removed (0 = untagged only , -1 = any) 2594 * 2595 * This function should be used to remove all VLAN filters which match the 2596 * given VID. It does not schedule the service event and does not take the 2597 * mac_filter_hash_lock so it may be combined with other operations under 2598 * a single invocation of the mac_filter_hash_lock. 2599 * 2600 * NOTE: this function expects to be called while under the 2601 * mac_filter_hash_lock 2602 */ 2603 void i40e_rm_vlan_all_mac(struct i40e_vsi *vsi, s16 vid) 2604 { 2605 struct i40e_mac_filter *f; 2606 struct hlist_node *h; 2607 int bkt; 2608 2609 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) { 2610 if (f->vlan == vid) 2611 __i40e_del_filter(vsi, f); 2612 } 2613 } 2614 2615 /** 2616 * i40e_vsi_kill_vlan - Remove VSI membership for given VLAN 2617 * @vsi: the VSI being configured 2618 * @vid: VLAN id to be removed 2619 **/ 2620 void i40e_vsi_kill_vlan(struct i40e_vsi *vsi, u16 vid) 2621 { 2622 if (!vid || vsi->info.pvid) 2623 return; 2624 2625 spin_lock_bh(&vsi->mac_filter_hash_lock); 2626 i40e_rm_vlan_all_mac(vsi, vid); 2627 spin_unlock_bh(&vsi->mac_filter_hash_lock); 2628 2629 /* schedule our worker thread which will take care of 2630 * applying the new filter changes 2631 */ 2632 i40e_service_event_schedule(vsi->back); 2633 } 2634 2635 /** 2636 * i40e_vlan_rx_add_vid - Add a vlan id filter to HW offload 2637 * @netdev: network interface to be adjusted 2638 * @vid: vlan id to be added 2639 * 2640 * net_device_ops implementation for adding vlan ids 2641 **/ 2642 #ifdef I40E_FCOE 2643 int i40e_vlan_rx_add_vid(struct net_device *netdev, 2644 __always_unused __be16 proto, u16 vid) 2645 #else 2646 static int i40e_vlan_rx_add_vid(struct net_device *netdev, 2647 __always_unused __be16 proto, u16 vid) 2648 #endif 2649 { 2650 struct i40e_netdev_priv *np = netdev_priv(netdev); 2651 struct i40e_vsi *vsi = np->vsi; 2652 int ret = 0; 2653 2654 if (vid >= VLAN_N_VID) 2655 return -EINVAL; 2656 2657 /* If the network stack called us with vid = 0 then 2658 * it is asking to receive priority tagged packets with 2659 * vlan id 0. Our HW receives them by default when configured 2660 * to receive untagged packets so there is no need to add an 2661 * extra filter for vlan 0 tagged packets. 2662 */ 2663 if (vid) 2664 ret = i40e_vsi_add_vlan(vsi, vid); 2665 2666 if (!ret) 2667 set_bit(vid, vsi->active_vlans); 2668 2669 return ret; 2670 } 2671 2672 /** 2673 * i40e_vlan_rx_kill_vid - Remove a vlan id filter from HW offload 2674 * @netdev: network interface to be adjusted 2675 * @vid: vlan id to be removed 2676 * 2677 * net_device_ops implementation for removing vlan ids 2678 **/ 2679 #ifdef I40E_FCOE 2680 int i40e_vlan_rx_kill_vid(struct net_device *netdev, 2681 __always_unused __be16 proto, u16 vid) 2682 #else 2683 static int i40e_vlan_rx_kill_vid(struct net_device *netdev, 2684 __always_unused __be16 proto, u16 vid) 2685 #endif 2686 { 2687 struct i40e_netdev_priv *np = netdev_priv(netdev); 2688 struct i40e_vsi *vsi = np->vsi; 2689 2690 /* return code is ignored as there is nothing a user 2691 * can do about failure to remove and a log message was 2692 * already printed from the other function 2693 */ 2694 i40e_vsi_kill_vlan(vsi, vid); 2695 2696 clear_bit(vid, vsi->active_vlans); 2697 2698 return 0; 2699 } 2700 2701 /** 2702 * i40e_macaddr_init - explicitly write the mac address filters 2703 * 2704 * @vsi: pointer to the vsi 2705 * @macaddr: the MAC address 2706 * 2707 * This is needed when the macaddr has been obtained by other 2708 * means than the default, e.g., from Open Firmware or IDPROM. 2709 * Returns 0 on success, negative on failure 2710 **/ 2711 static int i40e_macaddr_init(struct i40e_vsi *vsi, u8 *macaddr) 2712 { 2713 int ret; 2714 struct i40e_aqc_add_macvlan_element_data element; 2715 2716 ret = i40e_aq_mac_address_write(&vsi->back->hw, 2717 I40E_AQC_WRITE_TYPE_LAA_WOL, 2718 macaddr, NULL); 2719 if (ret) { 2720 dev_info(&vsi->back->pdev->dev, 2721 "Addr change for VSI failed: %d\n", ret); 2722 return -EADDRNOTAVAIL; 2723 } 2724 2725 memset(&element, 0, sizeof(element)); 2726 ether_addr_copy(element.mac_addr, macaddr); 2727 element.flags = cpu_to_le16(I40E_AQC_MACVLAN_ADD_PERFECT_MATCH); 2728 ret = i40e_aq_add_macvlan(&vsi->back->hw, vsi->seid, &element, 1, NULL); 2729 if (ret) { 2730 dev_info(&vsi->back->pdev->dev, 2731 "add filter failed err %s aq_err %s\n", 2732 i40e_stat_str(&vsi->back->hw, ret), 2733 i40e_aq_str(&vsi->back->hw, 2734 vsi->back->hw.aq.asq_last_status)); 2735 } 2736 return ret; 2737 } 2738 2739 /** 2740 * i40e_restore_vlan - Reinstate vlans when vsi/netdev comes back up 2741 * @vsi: the vsi being brought back up 2742 **/ 2743 static void i40e_restore_vlan(struct i40e_vsi *vsi) 2744 { 2745 u16 vid; 2746 2747 if (!vsi->netdev) 2748 return; 2749 2750 i40e_vlan_rx_register(vsi->netdev, vsi->netdev->features); 2751 2752 for_each_set_bit(vid, vsi->active_vlans, VLAN_N_VID) 2753 i40e_vlan_rx_add_vid(vsi->netdev, htons(ETH_P_8021Q), 2754 vid); 2755 } 2756 2757 /** 2758 * i40e_vsi_add_pvid - Add pvid for the VSI 2759 * @vsi: the vsi being adjusted 2760 * @vid: the vlan id to set as a PVID 2761 **/ 2762 int i40e_vsi_add_pvid(struct i40e_vsi *vsi, u16 vid) 2763 { 2764 struct i40e_vsi_context ctxt; 2765 i40e_status ret; 2766 2767 vsi->info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID); 2768 vsi->info.pvid = cpu_to_le16(vid); 2769 vsi->info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_TAGGED | 2770 I40E_AQ_VSI_PVLAN_INSERT_PVID | 2771 I40E_AQ_VSI_PVLAN_EMOD_STR; 2772 2773 ctxt.seid = vsi->seid; 2774 ctxt.info = vsi->info; 2775 ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL); 2776 if (ret) { 2777 dev_info(&vsi->back->pdev->dev, 2778 "add pvid failed, err %s aq_err %s\n", 2779 i40e_stat_str(&vsi->back->hw, ret), 2780 i40e_aq_str(&vsi->back->hw, 2781 vsi->back->hw.aq.asq_last_status)); 2782 return -ENOENT; 2783 } 2784 2785 return 0; 2786 } 2787 2788 /** 2789 * i40e_vsi_remove_pvid - Remove the pvid from the VSI 2790 * @vsi: the vsi being adjusted 2791 * 2792 * Just use the vlan_rx_register() service to put it back to normal 2793 **/ 2794 void i40e_vsi_remove_pvid(struct i40e_vsi *vsi) 2795 { 2796 i40e_vlan_stripping_disable(vsi); 2797 2798 vsi->info.pvid = 0; 2799 } 2800 2801 /** 2802 * i40e_vsi_setup_tx_resources - Allocate VSI Tx queue resources 2803 * @vsi: ptr to the VSI 2804 * 2805 * If this function returns with an error, then it's possible one or 2806 * more of the rings is populated (while the rest are not). It is the 2807 * callers duty to clean those orphaned rings. 2808 * 2809 * Return 0 on success, negative on failure 2810 **/ 2811 static int i40e_vsi_setup_tx_resources(struct i40e_vsi *vsi) 2812 { 2813 int i, err = 0; 2814 2815 for (i = 0; i < vsi->num_queue_pairs && !err; i++) 2816 err = i40e_setup_tx_descriptors(vsi->tx_rings[i]); 2817 2818 return err; 2819 } 2820 2821 /** 2822 * i40e_vsi_free_tx_resources - Free Tx resources for VSI queues 2823 * @vsi: ptr to the VSI 2824 * 2825 * Free VSI's transmit software resources 2826 **/ 2827 static void i40e_vsi_free_tx_resources(struct i40e_vsi *vsi) 2828 { 2829 int i; 2830 2831 if (!vsi->tx_rings) 2832 return; 2833 2834 for (i = 0; i < vsi->num_queue_pairs; i++) 2835 if (vsi->tx_rings[i] && vsi->tx_rings[i]->desc) 2836 i40e_free_tx_resources(vsi->tx_rings[i]); 2837 } 2838 2839 /** 2840 * i40e_vsi_setup_rx_resources - Allocate VSI queues Rx resources 2841 * @vsi: ptr to the VSI 2842 * 2843 * If this function returns with an error, then it's possible one or 2844 * more of the rings is populated (while the rest are not). It is the 2845 * callers duty to clean those orphaned rings. 2846 * 2847 * Return 0 on success, negative on failure 2848 **/ 2849 static int i40e_vsi_setup_rx_resources(struct i40e_vsi *vsi) 2850 { 2851 int i, err = 0; 2852 2853 for (i = 0; i < vsi->num_queue_pairs && !err; i++) 2854 err = i40e_setup_rx_descriptors(vsi->rx_rings[i]); 2855 #ifdef I40E_FCOE 2856 i40e_fcoe_setup_ddp_resources(vsi); 2857 #endif 2858 return err; 2859 } 2860 2861 /** 2862 * i40e_vsi_free_rx_resources - Free Rx Resources for VSI queues 2863 * @vsi: ptr to the VSI 2864 * 2865 * Free all receive software resources 2866 **/ 2867 static void i40e_vsi_free_rx_resources(struct i40e_vsi *vsi) 2868 { 2869 int i; 2870 2871 if (!vsi->rx_rings) 2872 return; 2873 2874 for (i = 0; i < vsi->num_queue_pairs; i++) 2875 if (vsi->rx_rings[i] && vsi->rx_rings[i]->desc) 2876 i40e_free_rx_resources(vsi->rx_rings[i]); 2877 #ifdef I40E_FCOE 2878 i40e_fcoe_free_ddp_resources(vsi); 2879 #endif 2880 } 2881 2882 /** 2883 * i40e_config_xps_tx_ring - Configure XPS for a Tx ring 2884 * @ring: The Tx ring to configure 2885 * 2886 * This enables/disables XPS for a given Tx descriptor ring 2887 * based on the TCs enabled for the VSI that ring belongs to. 2888 **/ 2889 static void i40e_config_xps_tx_ring(struct i40e_ring *ring) 2890 { 2891 struct i40e_vsi *vsi = ring->vsi; 2892 cpumask_var_t mask; 2893 2894 if (!ring->q_vector || !ring->netdev) 2895 return; 2896 2897 /* Single TC mode enable XPS */ 2898 if (vsi->tc_config.numtc <= 1) { 2899 if (!test_and_set_bit(__I40E_TX_XPS_INIT_DONE, &ring->state)) 2900 netif_set_xps_queue(ring->netdev, 2901 &ring->q_vector->affinity_mask, 2902 ring->queue_index); 2903 } else if (alloc_cpumask_var(&mask, GFP_KERNEL)) { 2904 /* Disable XPS to allow selection based on TC */ 2905 bitmap_zero(cpumask_bits(mask), nr_cpumask_bits); 2906 netif_set_xps_queue(ring->netdev, mask, ring->queue_index); 2907 free_cpumask_var(mask); 2908 } 2909 2910 /* schedule our worker thread which will take care of 2911 * applying the new filter changes 2912 */ 2913 i40e_service_event_schedule(vsi->back); 2914 } 2915 2916 /** 2917 * i40e_configure_tx_ring - Configure a transmit ring context and rest 2918 * @ring: The Tx ring to configure 2919 * 2920 * Configure the Tx descriptor ring in the HMC context. 2921 **/ 2922 static int i40e_configure_tx_ring(struct i40e_ring *ring) 2923 { 2924 struct i40e_vsi *vsi = ring->vsi; 2925 u16 pf_q = vsi->base_queue + ring->queue_index; 2926 struct i40e_hw *hw = &vsi->back->hw; 2927 struct i40e_hmc_obj_txq tx_ctx; 2928 i40e_status err = 0; 2929 u32 qtx_ctl = 0; 2930 2931 /* some ATR related tx ring init */ 2932 if (vsi->back->flags & I40E_FLAG_FD_ATR_ENABLED) { 2933 ring->atr_sample_rate = vsi->back->atr_sample_rate; 2934 ring->atr_count = 0; 2935 } else { 2936 ring->atr_sample_rate = 0; 2937 } 2938 2939 /* configure XPS */ 2940 i40e_config_xps_tx_ring(ring); 2941 2942 /* clear the context structure first */ 2943 memset(&tx_ctx, 0, sizeof(tx_ctx)); 2944 2945 tx_ctx.new_context = 1; 2946 tx_ctx.base = (ring->dma / 128); 2947 tx_ctx.qlen = ring->count; 2948 tx_ctx.fd_ena = !!(vsi->back->flags & (I40E_FLAG_FD_SB_ENABLED | 2949 I40E_FLAG_FD_ATR_ENABLED)); 2950 #ifdef I40E_FCOE 2951 tx_ctx.fc_ena = (vsi->type == I40E_VSI_FCOE); 2952 #endif 2953 tx_ctx.timesync_ena = !!(vsi->back->flags & I40E_FLAG_PTP); 2954 /* FDIR VSI tx ring can still use RS bit and writebacks */ 2955 if (vsi->type != I40E_VSI_FDIR) 2956 tx_ctx.head_wb_ena = 1; 2957 tx_ctx.head_wb_addr = ring->dma + 2958 (ring->count * sizeof(struct i40e_tx_desc)); 2959 2960 /* As part of VSI creation/update, FW allocates certain 2961 * Tx arbitration queue sets for each TC enabled for 2962 * the VSI. The FW returns the handles to these queue 2963 * sets as part of the response buffer to Add VSI, 2964 * Update VSI, etc. AQ commands. It is expected that 2965 * these queue set handles be associated with the Tx 2966 * queues by the driver as part of the TX queue context 2967 * initialization. This has to be done regardless of 2968 * DCB as by default everything is mapped to TC0. 2969 */ 2970 tx_ctx.rdylist = le16_to_cpu(vsi->info.qs_handle[ring->dcb_tc]); 2971 tx_ctx.rdylist_act = 0; 2972 2973 /* clear the context in the HMC */ 2974 err = i40e_clear_lan_tx_queue_context(hw, pf_q); 2975 if (err) { 2976 dev_info(&vsi->back->pdev->dev, 2977 "Failed to clear LAN Tx queue context on Tx ring %d (pf_q %d), error: %d\n", 2978 ring->queue_index, pf_q, err); 2979 return -ENOMEM; 2980 } 2981 2982 /* set the context in the HMC */ 2983 err = i40e_set_lan_tx_queue_context(hw, pf_q, &tx_ctx); 2984 if (err) { 2985 dev_info(&vsi->back->pdev->dev, 2986 "Failed to set LAN Tx queue context on Tx ring %d (pf_q %d, error: %d\n", 2987 ring->queue_index, pf_q, err); 2988 return -ENOMEM; 2989 } 2990 2991 /* Now associate this queue with this PCI function */ 2992 if (vsi->type == I40E_VSI_VMDQ2) { 2993 qtx_ctl = I40E_QTX_CTL_VM_QUEUE; 2994 qtx_ctl |= ((vsi->id) << I40E_QTX_CTL_VFVM_INDX_SHIFT) & 2995 I40E_QTX_CTL_VFVM_INDX_MASK; 2996 } else { 2997 qtx_ctl = I40E_QTX_CTL_PF_QUEUE; 2998 } 2999 3000 qtx_ctl |= ((hw->pf_id << I40E_QTX_CTL_PF_INDX_SHIFT) & 3001 I40E_QTX_CTL_PF_INDX_MASK); 3002 wr32(hw, I40E_QTX_CTL(pf_q), qtx_ctl); 3003 i40e_flush(hw); 3004 3005 /* cache tail off for easier writes later */ 3006 ring->tail = hw->hw_addr + I40E_QTX_TAIL(pf_q); 3007 3008 return 0; 3009 } 3010 3011 /** 3012 * i40e_configure_rx_ring - Configure a receive ring context 3013 * @ring: The Rx ring to configure 3014 * 3015 * Configure the Rx descriptor ring in the HMC context. 3016 **/ 3017 static int i40e_configure_rx_ring(struct i40e_ring *ring) 3018 { 3019 struct i40e_vsi *vsi = ring->vsi; 3020 u32 chain_len = vsi->back->hw.func_caps.rx_buf_chain_len; 3021 u16 pf_q = vsi->base_queue + ring->queue_index; 3022 struct i40e_hw *hw = &vsi->back->hw; 3023 struct i40e_hmc_obj_rxq rx_ctx; 3024 i40e_status err = 0; 3025 3026 ring->state = 0; 3027 3028 /* clear the context structure first */ 3029 memset(&rx_ctx, 0, sizeof(rx_ctx)); 3030 3031 ring->rx_buf_len = vsi->rx_buf_len; 3032 3033 rx_ctx.dbuff = ring->rx_buf_len >> I40E_RXQ_CTX_DBUFF_SHIFT; 3034 3035 rx_ctx.base = (ring->dma / 128); 3036 rx_ctx.qlen = ring->count; 3037 3038 /* use 32 byte descriptors */ 3039 rx_ctx.dsize = 1; 3040 3041 /* descriptor type is always zero 3042 * rx_ctx.dtype = 0; 3043 */ 3044 rx_ctx.hsplit_0 = 0; 3045 3046 rx_ctx.rxmax = min_t(u16, vsi->max_frame, chain_len * ring->rx_buf_len); 3047 if (hw->revision_id == 0) 3048 rx_ctx.lrxqthresh = 0; 3049 else 3050 rx_ctx.lrxqthresh = 2; 3051 rx_ctx.crcstrip = 1; 3052 rx_ctx.l2tsel = 1; 3053 /* this controls whether VLAN is stripped from inner headers */ 3054 rx_ctx.showiv = 0; 3055 #ifdef I40E_FCOE 3056 rx_ctx.fc_ena = (vsi->type == I40E_VSI_FCOE); 3057 #endif 3058 /* set the prefena field to 1 because the manual says to */ 3059 rx_ctx.prefena = 1; 3060 3061 /* clear the context in the HMC */ 3062 err = i40e_clear_lan_rx_queue_context(hw, pf_q); 3063 if (err) { 3064 dev_info(&vsi->back->pdev->dev, 3065 "Failed to clear LAN Rx queue context on Rx ring %d (pf_q %d), error: %d\n", 3066 ring->queue_index, pf_q, err); 3067 return -ENOMEM; 3068 } 3069 3070 /* set the context in the HMC */ 3071 err = i40e_set_lan_rx_queue_context(hw, pf_q, &rx_ctx); 3072 if (err) { 3073 dev_info(&vsi->back->pdev->dev, 3074 "Failed to set LAN Rx queue context on Rx ring %d (pf_q %d), error: %d\n", 3075 ring->queue_index, pf_q, err); 3076 return -ENOMEM; 3077 } 3078 3079 /* cache tail for quicker writes, and clear the reg before use */ 3080 ring->tail = hw->hw_addr + I40E_QRX_TAIL(pf_q); 3081 writel(0, ring->tail); 3082 3083 i40e_alloc_rx_buffers(ring, I40E_DESC_UNUSED(ring)); 3084 3085 return 0; 3086 } 3087 3088 /** 3089 * i40e_vsi_configure_tx - Configure the VSI for Tx 3090 * @vsi: VSI structure describing this set of rings and resources 3091 * 3092 * Configure the Tx VSI for operation. 3093 **/ 3094 static int i40e_vsi_configure_tx(struct i40e_vsi *vsi) 3095 { 3096 int err = 0; 3097 u16 i; 3098 3099 for (i = 0; (i < vsi->num_queue_pairs) && !err; i++) 3100 err = i40e_configure_tx_ring(vsi->tx_rings[i]); 3101 3102 return err; 3103 } 3104 3105 /** 3106 * i40e_vsi_configure_rx - Configure the VSI for Rx 3107 * @vsi: the VSI being configured 3108 * 3109 * Configure the Rx VSI for operation. 3110 **/ 3111 static int i40e_vsi_configure_rx(struct i40e_vsi *vsi) 3112 { 3113 int err = 0; 3114 u16 i; 3115 3116 if (vsi->netdev && (vsi->netdev->mtu > ETH_DATA_LEN)) 3117 vsi->max_frame = vsi->netdev->mtu + ETH_HLEN 3118 + ETH_FCS_LEN + VLAN_HLEN; 3119 else 3120 vsi->max_frame = I40E_RXBUFFER_2048; 3121 3122 vsi->rx_buf_len = I40E_RXBUFFER_2048; 3123 3124 #ifdef I40E_FCOE 3125 /* setup rx buffer for FCoE */ 3126 if ((vsi->type == I40E_VSI_FCOE) && 3127 (vsi->back->flags & I40E_FLAG_FCOE_ENABLED)) { 3128 vsi->rx_buf_len = I40E_RXBUFFER_3072; 3129 vsi->max_frame = I40E_RXBUFFER_3072; 3130 } 3131 3132 #endif /* I40E_FCOE */ 3133 /* round up for the chip's needs */ 3134 vsi->rx_buf_len = ALIGN(vsi->rx_buf_len, 3135 BIT_ULL(I40E_RXQ_CTX_DBUFF_SHIFT)); 3136 3137 /* set up individual rings */ 3138 for (i = 0; i < vsi->num_queue_pairs && !err; i++) 3139 err = i40e_configure_rx_ring(vsi->rx_rings[i]); 3140 3141 return err; 3142 } 3143 3144 /** 3145 * i40e_vsi_config_dcb_rings - Update rings to reflect DCB TC 3146 * @vsi: ptr to the VSI 3147 **/ 3148 static void i40e_vsi_config_dcb_rings(struct i40e_vsi *vsi) 3149 { 3150 struct i40e_ring *tx_ring, *rx_ring; 3151 u16 qoffset, qcount; 3152 int i, n; 3153 3154 if (!(vsi->back->flags & I40E_FLAG_DCB_ENABLED)) { 3155 /* Reset the TC information */ 3156 for (i = 0; i < vsi->num_queue_pairs; i++) { 3157 rx_ring = vsi->rx_rings[i]; 3158 tx_ring = vsi->tx_rings[i]; 3159 rx_ring->dcb_tc = 0; 3160 tx_ring->dcb_tc = 0; 3161 } 3162 } 3163 3164 for (n = 0; n < I40E_MAX_TRAFFIC_CLASS; n++) { 3165 if (!(vsi->tc_config.enabled_tc & BIT_ULL(n))) 3166 continue; 3167 3168 qoffset = vsi->tc_config.tc_info[n].qoffset; 3169 qcount = vsi->tc_config.tc_info[n].qcount; 3170 for (i = qoffset; i < (qoffset + qcount); i++) { 3171 rx_ring = vsi->rx_rings[i]; 3172 tx_ring = vsi->tx_rings[i]; 3173 rx_ring->dcb_tc = n; 3174 tx_ring->dcb_tc = n; 3175 } 3176 } 3177 } 3178 3179 /** 3180 * i40e_set_vsi_rx_mode - Call set_rx_mode on a VSI 3181 * @vsi: ptr to the VSI 3182 **/ 3183 static void i40e_set_vsi_rx_mode(struct i40e_vsi *vsi) 3184 { 3185 struct i40e_pf *pf = vsi->back; 3186 int err; 3187 3188 if (vsi->netdev) 3189 i40e_set_rx_mode(vsi->netdev); 3190 3191 if (!!(pf->flags & I40E_FLAG_PF_MAC)) { 3192 err = i40e_macaddr_init(vsi, pf->hw.mac.addr); 3193 if (err) { 3194 dev_warn(&pf->pdev->dev, 3195 "could not set up macaddr; err %d\n", err); 3196 } 3197 } 3198 } 3199 3200 /** 3201 * i40e_fdir_filter_restore - Restore the Sideband Flow Director filters 3202 * @vsi: Pointer to the targeted VSI 3203 * 3204 * This function replays the hlist on the hw where all the SB Flow Director 3205 * filters were saved. 3206 **/ 3207 static void i40e_fdir_filter_restore(struct i40e_vsi *vsi) 3208 { 3209 struct i40e_fdir_filter *filter; 3210 struct i40e_pf *pf = vsi->back; 3211 struct hlist_node *node; 3212 3213 if (!(pf->flags & I40E_FLAG_FD_SB_ENABLED)) 3214 return; 3215 3216 hlist_for_each_entry_safe(filter, node, 3217 &pf->fdir_filter_list, fdir_node) { 3218 i40e_add_del_fdir(vsi, filter, true); 3219 } 3220 } 3221 3222 /** 3223 * i40e_vsi_configure - Set up the VSI for action 3224 * @vsi: the VSI being configured 3225 **/ 3226 static int i40e_vsi_configure(struct i40e_vsi *vsi) 3227 { 3228 int err; 3229 3230 i40e_set_vsi_rx_mode(vsi); 3231 i40e_restore_vlan(vsi); 3232 i40e_vsi_config_dcb_rings(vsi); 3233 err = i40e_vsi_configure_tx(vsi); 3234 if (!err) 3235 err = i40e_vsi_configure_rx(vsi); 3236 3237 return err; 3238 } 3239 3240 /** 3241 * i40e_vsi_configure_msix - MSIX mode Interrupt Config in the HW 3242 * @vsi: the VSI being configured 3243 **/ 3244 static void i40e_vsi_configure_msix(struct i40e_vsi *vsi) 3245 { 3246 struct i40e_pf *pf = vsi->back; 3247 struct i40e_hw *hw = &pf->hw; 3248 u16 vector; 3249 int i, q; 3250 u32 qp; 3251 3252 /* The interrupt indexing is offset by 1 in the PFINT_ITRn 3253 * and PFINT_LNKLSTn registers, e.g.: 3254 * PFINT_ITRn[0..n-1] gets msix-1..msix-n (qpair interrupts) 3255 */ 3256 qp = vsi->base_queue; 3257 vector = vsi->base_vector; 3258 for (i = 0; i < vsi->num_q_vectors; i++, vector++) { 3259 struct i40e_q_vector *q_vector = vsi->q_vectors[i]; 3260 3261 q_vector->itr_countdown = ITR_COUNTDOWN_START; 3262 q_vector->rx.itr = ITR_TO_REG(vsi->rx_rings[i]->rx_itr_setting); 3263 q_vector->rx.latency_range = I40E_LOW_LATENCY; 3264 wr32(hw, I40E_PFINT_ITRN(I40E_RX_ITR, vector - 1), 3265 q_vector->rx.itr); 3266 q_vector->tx.itr = ITR_TO_REG(vsi->tx_rings[i]->tx_itr_setting); 3267 q_vector->tx.latency_range = I40E_LOW_LATENCY; 3268 wr32(hw, I40E_PFINT_ITRN(I40E_TX_ITR, vector - 1), 3269 q_vector->tx.itr); 3270 wr32(hw, I40E_PFINT_RATEN(vector - 1), 3271 i40e_intrl_usec_to_reg(vsi->int_rate_limit)); 3272 3273 /* Linked list for the queuepairs assigned to this vector */ 3274 wr32(hw, I40E_PFINT_LNKLSTN(vector - 1), qp); 3275 for (q = 0; q < q_vector->num_ringpairs; q++) { 3276 u32 val; 3277 3278 val = I40E_QINT_RQCTL_CAUSE_ENA_MASK | 3279 (I40E_RX_ITR << I40E_QINT_RQCTL_ITR_INDX_SHIFT) | 3280 (vector << I40E_QINT_RQCTL_MSIX_INDX_SHIFT) | 3281 (qp << I40E_QINT_RQCTL_NEXTQ_INDX_SHIFT)| 3282 (I40E_QUEUE_TYPE_TX 3283 << I40E_QINT_RQCTL_NEXTQ_TYPE_SHIFT); 3284 3285 wr32(hw, I40E_QINT_RQCTL(qp), val); 3286 3287 val = I40E_QINT_TQCTL_CAUSE_ENA_MASK | 3288 (I40E_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT) | 3289 (vector << I40E_QINT_TQCTL_MSIX_INDX_SHIFT) | 3290 ((qp+1) << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT)| 3291 (I40E_QUEUE_TYPE_RX 3292 << I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT); 3293 3294 /* Terminate the linked list */ 3295 if (q == (q_vector->num_ringpairs - 1)) 3296 val |= (I40E_QUEUE_END_OF_LIST 3297 << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT); 3298 3299 wr32(hw, I40E_QINT_TQCTL(qp), val); 3300 qp++; 3301 } 3302 } 3303 3304 i40e_flush(hw); 3305 } 3306 3307 /** 3308 * i40e_enable_misc_int_causes - enable the non-queue interrupts 3309 * @hw: ptr to the hardware info 3310 **/ 3311 static void i40e_enable_misc_int_causes(struct i40e_pf *pf) 3312 { 3313 struct i40e_hw *hw = &pf->hw; 3314 u32 val; 3315 3316 /* clear things first */ 3317 wr32(hw, I40E_PFINT_ICR0_ENA, 0); /* disable all */ 3318 rd32(hw, I40E_PFINT_ICR0); /* read to clear */ 3319 3320 val = I40E_PFINT_ICR0_ENA_ECC_ERR_MASK | 3321 I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK | 3322 I40E_PFINT_ICR0_ENA_GRST_MASK | 3323 I40E_PFINT_ICR0_ENA_PCI_EXCEPTION_MASK | 3324 I40E_PFINT_ICR0_ENA_GPIO_MASK | 3325 I40E_PFINT_ICR0_ENA_HMC_ERR_MASK | 3326 I40E_PFINT_ICR0_ENA_VFLR_MASK | 3327 I40E_PFINT_ICR0_ENA_ADMINQ_MASK; 3328 3329 if (pf->flags & I40E_FLAG_IWARP_ENABLED) 3330 val |= I40E_PFINT_ICR0_ENA_PE_CRITERR_MASK; 3331 3332 if (pf->flags & I40E_FLAG_PTP) 3333 val |= I40E_PFINT_ICR0_ENA_TIMESYNC_MASK; 3334 3335 wr32(hw, I40E_PFINT_ICR0_ENA, val); 3336 3337 /* SW_ITR_IDX = 0, but don't change INTENA */ 3338 wr32(hw, I40E_PFINT_DYN_CTL0, I40E_PFINT_DYN_CTL0_SW_ITR_INDX_MASK | 3339 I40E_PFINT_DYN_CTL0_INTENA_MSK_MASK); 3340 3341 /* OTHER_ITR_IDX = 0 */ 3342 wr32(hw, I40E_PFINT_STAT_CTL0, 0); 3343 } 3344 3345 /** 3346 * i40e_configure_msi_and_legacy - Legacy mode interrupt config in the HW 3347 * @vsi: the VSI being configured 3348 **/ 3349 static void i40e_configure_msi_and_legacy(struct i40e_vsi *vsi) 3350 { 3351 struct i40e_q_vector *q_vector = vsi->q_vectors[0]; 3352 struct i40e_pf *pf = vsi->back; 3353 struct i40e_hw *hw = &pf->hw; 3354 u32 val; 3355 3356 /* set the ITR configuration */ 3357 q_vector->itr_countdown = ITR_COUNTDOWN_START; 3358 q_vector->rx.itr = ITR_TO_REG(vsi->rx_rings[0]->rx_itr_setting); 3359 q_vector->rx.latency_range = I40E_LOW_LATENCY; 3360 wr32(hw, I40E_PFINT_ITR0(I40E_RX_ITR), q_vector->rx.itr); 3361 q_vector->tx.itr = ITR_TO_REG(vsi->tx_rings[0]->tx_itr_setting); 3362 q_vector->tx.latency_range = I40E_LOW_LATENCY; 3363 wr32(hw, I40E_PFINT_ITR0(I40E_TX_ITR), q_vector->tx.itr); 3364 3365 i40e_enable_misc_int_causes(pf); 3366 3367 /* FIRSTQ_INDX = 0, FIRSTQ_TYPE = 0 (rx) */ 3368 wr32(hw, I40E_PFINT_LNKLST0, 0); 3369 3370 /* Associate the queue pair to the vector and enable the queue int */ 3371 val = I40E_QINT_RQCTL_CAUSE_ENA_MASK | 3372 (I40E_RX_ITR << I40E_QINT_RQCTL_ITR_INDX_SHIFT) | 3373 (I40E_QUEUE_TYPE_TX << I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT); 3374 3375 wr32(hw, I40E_QINT_RQCTL(0), val); 3376 3377 val = I40E_QINT_TQCTL_CAUSE_ENA_MASK | 3378 (I40E_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT) | 3379 (I40E_QUEUE_END_OF_LIST << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT); 3380 3381 wr32(hw, I40E_QINT_TQCTL(0), val); 3382 i40e_flush(hw); 3383 } 3384 3385 /** 3386 * i40e_irq_dynamic_disable_icr0 - Disable default interrupt generation for icr0 3387 * @pf: board private structure 3388 **/ 3389 void i40e_irq_dynamic_disable_icr0(struct i40e_pf *pf) 3390 { 3391 struct i40e_hw *hw = &pf->hw; 3392 3393 wr32(hw, I40E_PFINT_DYN_CTL0, 3394 I40E_ITR_NONE << I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT); 3395 i40e_flush(hw); 3396 } 3397 3398 /** 3399 * i40e_irq_dynamic_enable_icr0 - Enable default interrupt generation for icr0 3400 * @pf: board private structure 3401 * @clearpba: true when all pending interrupt events should be cleared 3402 **/ 3403 void i40e_irq_dynamic_enable_icr0(struct i40e_pf *pf, bool clearpba) 3404 { 3405 struct i40e_hw *hw = &pf->hw; 3406 u32 val; 3407 3408 val = I40E_PFINT_DYN_CTL0_INTENA_MASK | 3409 (clearpba ? I40E_PFINT_DYN_CTL0_CLEARPBA_MASK : 0) | 3410 (I40E_ITR_NONE << I40E_PFINT_DYN_CTL0_ITR_INDX_SHIFT); 3411 3412 wr32(hw, I40E_PFINT_DYN_CTL0, val); 3413 i40e_flush(hw); 3414 } 3415 3416 /** 3417 * i40e_msix_clean_rings - MSIX mode Interrupt Handler 3418 * @irq: interrupt number 3419 * @data: pointer to a q_vector 3420 **/ 3421 static irqreturn_t i40e_msix_clean_rings(int irq, void *data) 3422 { 3423 struct i40e_q_vector *q_vector = data; 3424 3425 if (!q_vector->tx.ring && !q_vector->rx.ring) 3426 return IRQ_HANDLED; 3427 3428 napi_schedule_irqoff(&q_vector->napi); 3429 3430 return IRQ_HANDLED; 3431 } 3432 3433 /** 3434 * i40e_irq_affinity_notify - Callback for affinity changes 3435 * @notify: context as to what irq was changed 3436 * @mask: the new affinity mask 3437 * 3438 * This is a callback function used by the irq_set_affinity_notifier function 3439 * so that we may register to receive changes to the irq affinity masks. 3440 **/ 3441 static void i40e_irq_affinity_notify(struct irq_affinity_notify *notify, 3442 const cpumask_t *mask) 3443 { 3444 struct i40e_q_vector *q_vector = 3445 container_of(notify, struct i40e_q_vector, affinity_notify); 3446 3447 q_vector->affinity_mask = *mask; 3448 } 3449 3450 /** 3451 * i40e_irq_affinity_release - Callback for affinity notifier release 3452 * @ref: internal core kernel usage 3453 * 3454 * This is a callback function used by the irq_set_affinity_notifier function 3455 * to inform the current notification subscriber that they will no longer 3456 * receive notifications. 3457 **/ 3458 static void i40e_irq_affinity_release(struct kref *ref) {} 3459 3460 /** 3461 * i40e_vsi_request_irq_msix - Initialize MSI-X interrupts 3462 * @vsi: the VSI being configured 3463 * @basename: name for the vector 3464 * 3465 * Allocates MSI-X vectors and requests interrupts from the kernel. 3466 **/ 3467 static int i40e_vsi_request_irq_msix(struct i40e_vsi *vsi, char *basename) 3468 { 3469 int q_vectors = vsi->num_q_vectors; 3470 struct i40e_pf *pf = vsi->back; 3471 int base = vsi->base_vector; 3472 int rx_int_idx = 0; 3473 int tx_int_idx = 0; 3474 int vector, err; 3475 int irq_num; 3476 3477 for (vector = 0; vector < q_vectors; vector++) { 3478 struct i40e_q_vector *q_vector = vsi->q_vectors[vector]; 3479 3480 irq_num = pf->msix_entries[base + vector].vector; 3481 3482 if (q_vector->tx.ring && q_vector->rx.ring) { 3483 snprintf(q_vector->name, sizeof(q_vector->name) - 1, 3484 "%s-%s-%d", basename, "TxRx", rx_int_idx++); 3485 tx_int_idx++; 3486 } else if (q_vector->rx.ring) { 3487 snprintf(q_vector->name, sizeof(q_vector->name) - 1, 3488 "%s-%s-%d", basename, "rx", rx_int_idx++); 3489 } else if (q_vector->tx.ring) { 3490 snprintf(q_vector->name, sizeof(q_vector->name) - 1, 3491 "%s-%s-%d", basename, "tx", tx_int_idx++); 3492 } else { 3493 /* skip this unused q_vector */ 3494 continue; 3495 } 3496 err = request_irq(irq_num, 3497 vsi->irq_handler, 3498 0, 3499 q_vector->name, 3500 q_vector); 3501 if (err) { 3502 dev_info(&pf->pdev->dev, 3503 "MSIX request_irq failed, error: %d\n", err); 3504 goto free_queue_irqs; 3505 } 3506 3507 /* register for affinity change notifications */ 3508 q_vector->affinity_notify.notify = i40e_irq_affinity_notify; 3509 q_vector->affinity_notify.release = i40e_irq_affinity_release; 3510 irq_set_affinity_notifier(irq_num, &q_vector->affinity_notify); 3511 /* assign the mask for this irq */ 3512 irq_set_affinity_hint(irq_num, &q_vector->affinity_mask); 3513 } 3514 3515 vsi->irqs_ready = true; 3516 return 0; 3517 3518 free_queue_irqs: 3519 while (vector) { 3520 vector--; 3521 irq_num = pf->msix_entries[base + vector].vector; 3522 irq_set_affinity_notifier(irq_num, NULL); 3523 irq_set_affinity_hint(irq_num, NULL); 3524 free_irq(irq_num, &vsi->q_vectors[vector]); 3525 } 3526 return err; 3527 } 3528 3529 /** 3530 * i40e_vsi_disable_irq - Mask off queue interrupt generation on the VSI 3531 * @vsi: the VSI being un-configured 3532 **/ 3533 static void i40e_vsi_disable_irq(struct i40e_vsi *vsi) 3534 { 3535 struct i40e_pf *pf = vsi->back; 3536 struct i40e_hw *hw = &pf->hw; 3537 int base = vsi->base_vector; 3538 int i; 3539 3540 for (i = 0; i < vsi->num_queue_pairs; i++) { 3541 wr32(hw, I40E_QINT_TQCTL(vsi->tx_rings[i]->reg_idx), 0); 3542 wr32(hw, I40E_QINT_RQCTL(vsi->rx_rings[i]->reg_idx), 0); 3543 } 3544 3545 if (pf->flags & I40E_FLAG_MSIX_ENABLED) { 3546 for (i = vsi->base_vector; 3547 i < (vsi->num_q_vectors + vsi->base_vector); i++) 3548 wr32(hw, I40E_PFINT_DYN_CTLN(i - 1), 0); 3549 3550 i40e_flush(hw); 3551 for (i = 0; i < vsi->num_q_vectors; i++) 3552 synchronize_irq(pf->msix_entries[i + base].vector); 3553 } else { 3554 /* Legacy and MSI mode - this stops all interrupt handling */ 3555 wr32(hw, I40E_PFINT_ICR0_ENA, 0); 3556 wr32(hw, I40E_PFINT_DYN_CTL0, 0); 3557 i40e_flush(hw); 3558 synchronize_irq(pf->pdev->irq); 3559 } 3560 } 3561 3562 /** 3563 * i40e_vsi_enable_irq - Enable IRQ for the given VSI 3564 * @vsi: the VSI being configured 3565 **/ 3566 static int i40e_vsi_enable_irq(struct i40e_vsi *vsi) 3567 { 3568 struct i40e_pf *pf = vsi->back; 3569 int i; 3570 3571 if (pf->flags & I40E_FLAG_MSIX_ENABLED) { 3572 for (i = 0; i < vsi->num_q_vectors; i++) 3573 i40e_irq_dynamic_enable(vsi, i); 3574 } else { 3575 i40e_irq_dynamic_enable_icr0(pf, true); 3576 } 3577 3578 i40e_flush(&pf->hw); 3579 return 0; 3580 } 3581 3582 /** 3583 * i40e_stop_misc_vector - Stop the vector that handles non-queue events 3584 * @pf: board private structure 3585 **/ 3586 static void i40e_stop_misc_vector(struct i40e_pf *pf) 3587 { 3588 /* Disable ICR 0 */ 3589 wr32(&pf->hw, I40E_PFINT_ICR0_ENA, 0); 3590 i40e_flush(&pf->hw); 3591 } 3592 3593 /** 3594 * i40e_intr - MSI/Legacy and non-queue interrupt handler 3595 * @irq: interrupt number 3596 * @data: pointer to a q_vector 3597 * 3598 * This is the handler used for all MSI/Legacy interrupts, and deals 3599 * with both queue and non-queue interrupts. This is also used in 3600 * MSIX mode to handle the non-queue interrupts. 3601 **/ 3602 static irqreturn_t i40e_intr(int irq, void *data) 3603 { 3604 struct i40e_pf *pf = (struct i40e_pf *)data; 3605 struct i40e_hw *hw = &pf->hw; 3606 irqreturn_t ret = IRQ_NONE; 3607 u32 icr0, icr0_remaining; 3608 u32 val, ena_mask; 3609 3610 icr0 = rd32(hw, I40E_PFINT_ICR0); 3611 ena_mask = rd32(hw, I40E_PFINT_ICR0_ENA); 3612 3613 /* if sharing a legacy IRQ, we might get called w/o an intr pending */ 3614 if ((icr0 & I40E_PFINT_ICR0_INTEVENT_MASK) == 0) 3615 goto enable_intr; 3616 3617 /* if interrupt but no bits showing, must be SWINT */ 3618 if (((icr0 & ~I40E_PFINT_ICR0_INTEVENT_MASK) == 0) || 3619 (icr0 & I40E_PFINT_ICR0_SWINT_MASK)) 3620 pf->sw_int_count++; 3621 3622 if ((pf->flags & I40E_FLAG_IWARP_ENABLED) && 3623 (ena_mask & I40E_PFINT_ICR0_ENA_PE_CRITERR_MASK)) { 3624 ena_mask &= ~I40E_PFINT_ICR0_ENA_PE_CRITERR_MASK; 3625 icr0 &= ~I40E_PFINT_ICR0_ENA_PE_CRITERR_MASK; 3626 dev_dbg(&pf->pdev->dev, "cleared PE_CRITERR\n"); 3627 } 3628 3629 /* only q0 is used in MSI/Legacy mode, and none are used in MSIX */ 3630 if (icr0 & I40E_PFINT_ICR0_QUEUE_0_MASK) { 3631 struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi]; 3632 struct i40e_q_vector *q_vector = vsi->q_vectors[0]; 3633 3634 /* We do not have a way to disarm Queue causes while leaving 3635 * interrupt enabled for all other causes, ideally 3636 * interrupt should be disabled while we are in NAPI but 3637 * this is not a performance path and napi_schedule() 3638 * can deal with rescheduling. 3639 */ 3640 if (!test_bit(__I40E_DOWN, &pf->state)) 3641 napi_schedule_irqoff(&q_vector->napi); 3642 } 3643 3644 if (icr0 & I40E_PFINT_ICR0_ADMINQ_MASK) { 3645 ena_mask &= ~I40E_PFINT_ICR0_ENA_ADMINQ_MASK; 3646 set_bit(__I40E_ADMINQ_EVENT_PENDING, &pf->state); 3647 i40e_debug(&pf->hw, I40E_DEBUG_NVM, "AdminQ event\n"); 3648 } 3649 3650 if (icr0 & I40E_PFINT_ICR0_MAL_DETECT_MASK) { 3651 ena_mask &= ~I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK; 3652 set_bit(__I40E_MDD_EVENT_PENDING, &pf->state); 3653 } 3654 3655 if (icr0 & I40E_PFINT_ICR0_VFLR_MASK) { 3656 ena_mask &= ~I40E_PFINT_ICR0_ENA_VFLR_MASK; 3657 set_bit(__I40E_VFLR_EVENT_PENDING, &pf->state); 3658 } 3659 3660 if (icr0 & I40E_PFINT_ICR0_GRST_MASK) { 3661 if (!test_bit(__I40E_RESET_RECOVERY_PENDING, &pf->state)) 3662 set_bit(__I40E_RESET_INTR_RECEIVED, &pf->state); 3663 ena_mask &= ~I40E_PFINT_ICR0_ENA_GRST_MASK; 3664 val = rd32(hw, I40E_GLGEN_RSTAT); 3665 val = (val & I40E_GLGEN_RSTAT_RESET_TYPE_MASK) 3666 >> I40E_GLGEN_RSTAT_RESET_TYPE_SHIFT; 3667 if (val == I40E_RESET_CORER) { 3668 pf->corer_count++; 3669 } else if (val == I40E_RESET_GLOBR) { 3670 pf->globr_count++; 3671 } else if (val == I40E_RESET_EMPR) { 3672 pf->empr_count++; 3673 set_bit(__I40E_EMP_RESET_INTR_RECEIVED, &pf->state); 3674 } 3675 } 3676 3677 if (icr0 & I40E_PFINT_ICR0_HMC_ERR_MASK) { 3678 icr0 &= ~I40E_PFINT_ICR0_HMC_ERR_MASK; 3679 dev_info(&pf->pdev->dev, "HMC error interrupt\n"); 3680 dev_info(&pf->pdev->dev, "HMC error info 0x%x, HMC error data 0x%x\n", 3681 rd32(hw, I40E_PFHMC_ERRORINFO), 3682 rd32(hw, I40E_PFHMC_ERRORDATA)); 3683 } 3684 3685 if (icr0 & I40E_PFINT_ICR0_TIMESYNC_MASK) { 3686 u32 prttsyn_stat = rd32(hw, I40E_PRTTSYN_STAT_0); 3687 3688 if (prttsyn_stat & I40E_PRTTSYN_STAT_0_TXTIME_MASK) { 3689 icr0 &= ~I40E_PFINT_ICR0_ENA_TIMESYNC_MASK; 3690 i40e_ptp_tx_hwtstamp(pf); 3691 } 3692 } 3693 3694 /* If a critical error is pending we have no choice but to reset the 3695 * device. 3696 * Report and mask out any remaining unexpected interrupts. 3697 */ 3698 icr0_remaining = icr0 & ena_mask; 3699 if (icr0_remaining) { 3700 dev_info(&pf->pdev->dev, "unhandled interrupt icr0=0x%08x\n", 3701 icr0_remaining); 3702 if ((icr0_remaining & I40E_PFINT_ICR0_PE_CRITERR_MASK) || 3703 (icr0_remaining & I40E_PFINT_ICR0_PCI_EXCEPTION_MASK) || 3704 (icr0_remaining & I40E_PFINT_ICR0_ECC_ERR_MASK)) { 3705 dev_info(&pf->pdev->dev, "device will be reset\n"); 3706 set_bit(__I40E_PF_RESET_REQUESTED, &pf->state); 3707 i40e_service_event_schedule(pf); 3708 } 3709 ena_mask &= ~icr0_remaining; 3710 } 3711 ret = IRQ_HANDLED; 3712 3713 enable_intr: 3714 /* re-enable interrupt causes */ 3715 wr32(hw, I40E_PFINT_ICR0_ENA, ena_mask); 3716 if (!test_bit(__I40E_DOWN, &pf->state)) { 3717 i40e_service_event_schedule(pf); 3718 i40e_irq_dynamic_enable_icr0(pf, false); 3719 } 3720 3721 return ret; 3722 } 3723 3724 /** 3725 * i40e_clean_fdir_tx_irq - Reclaim resources after transmit completes 3726 * @tx_ring: tx ring to clean 3727 * @budget: how many cleans we're allowed 3728 * 3729 * Returns true if there's any budget left (e.g. the clean is finished) 3730 **/ 3731 static bool i40e_clean_fdir_tx_irq(struct i40e_ring *tx_ring, int budget) 3732 { 3733 struct i40e_vsi *vsi = tx_ring->vsi; 3734 u16 i = tx_ring->next_to_clean; 3735 struct i40e_tx_buffer *tx_buf; 3736 struct i40e_tx_desc *tx_desc; 3737 3738 tx_buf = &tx_ring->tx_bi[i]; 3739 tx_desc = I40E_TX_DESC(tx_ring, i); 3740 i -= tx_ring->count; 3741 3742 do { 3743 struct i40e_tx_desc *eop_desc = tx_buf->next_to_watch; 3744 3745 /* if next_to_watch is not set then there is no work pending */ 3746 if (!eop_desc) 3747 break; 3748 3749 /* prevent any other reads prior to eop_desc */ 3750 read_barrier_depends(); 3751 3752 /* if the descriptor isn't done, no work yet to do */ 3753 if (!(eop_desc->cmd_type_offset_bsz & 3754 cpu_to_le64(I40E_TX_DESC_DTYPE_DESC_DONE))) 3755 break; 3756 3757 /* clear next_to_watch to prevent false hangs */ 3758 tx_buf->next_to_watch = NULL; 3759 3760 tx_desc->buffer_addr = 0; 3761 tx_desc->cmd_type_offset_bsz = 0; 3762 /* move past filter desc */ 3763 tx_buf++; 3764 tx_desc++; 3765 i++; 3766 if (unlikely(!i)) { 3767 i -= tx_ring->count; 3768 tx_buf = tx_ring->tx_bi; 3769 tx_desc = I40E_TX_DESC(tx_ring, 0); 3770 } 3771 /* unmap skb header data */ 3772 dma_unmap_single(tx_ring->dev, 3773 dma_unmap_addr(tx_buf, dma), 3774 dma_unmap_len(tx_buf, len), 3775 DMA_TO_DEVICE); 3776 if (tx_buf->tx_flags & I40E_TX_FLAGS_FD_SB) 3777 kfree(tx_buf->raw_buf); 3778 3779 tx_buf->raw_buf = NULL; 3780 tx_buf->tx_flags = 0; 3781 tx_buf->next_to_watch = NULL; 3782 dma_unmap_len_set(tx_buf, len, 0); 3783 tx_desc->buffer_addr = 0; 3784 tx_desc->cmd_type_offset_bsz = 0; 3785 3786 /* move us past the eop_desc for start of next FD desc */ 3787 tx_buf++; 3788 tx_desc++; 3789 i++; 3790 if (unlikely(!i)) { 3791 i -= tx_ring->count; 3792 tx_buf = tx_ring->tx_bi; 3793 tx_desc = I40E_TX_DESC(tx_ring, 0); 3794 } 3795 3796 /* update budget accounting */ 3797 budget--; 3798 } while (likely(budget)); 3799 3800 i += tx_ring->count; 3801 tx_ring->next_to_clean = i; 3802 3803 if (vsi->back->flags & I40E_FLAG_MSIX_ENABLED) 3804 i40e_irq_dynamic_enable(vsi, tx_ring->q_vector->v_idx); 3805 3806 return budget > 0; 3807 } 3808 3809 /** 3810 * i40e_fdir_clean_ring - Interrupt Handler for FDIR SB ring 3811 * @irq: interrupt number 3812 * @data: pointer to a q_vector 3813 **/ 3814 static irqreturn_t i40e_fdir_clean_ring(int irq, void *data) 3815 { 3816 struct i40e_q_vector *q_vector = data; 3817 struct i40e_vsi *vsi; 3818 3819 if (!q_vector->tx.ring) 3820 return IRQ_HANDLED; 3821 3822 vsi = q_vector->tx.ring->vsi; 3823 i40e_clean_fdir_tx_irq(q_vector->tx.ring, vsi->work_limit); 3824 3825 return IRQ_HANDLED; 3826 } 3827 3828 /** 3829 * i40e_map_vector_to_qp - Assigns the queue pair to the vector 3830 * @vsi: the VSI being configured 3831 * @v_idx: vector index 3832 * @qp_idx: queue pair index 3833 **/ 3834 static void i40e_map_vector_to_qp(struct i40e_vsi *vsi, int v_idx, int qp_idx) 3835 { 3836 struct i40e_q_vector *q_vector = vsi->q_vectors[v_idx]; 3837 struct i40e_ring *tx_ring = vsi->tx_rings[qp_idx]; 3838 struct i40e_ring *rx_ring = vsi->rx_rings[qp_idx]; 3839 3840 tx_ring->q_vector = q_vector; 3841 tx_ring->next = q_vector->tx.ring; 3842 q_vector->tx.ring = tx_ring; 3843 q_vector->tx.count++; 3844 3845 rx_ring->q_vector = q_vector; 3846 rx_ring->next = q_vector->rx.ring; 3847 q_vector->rx.ring = rx_ring; 3848 q_vector->rx.count++; 3849 } 3850 3851 /** 3852 * i40e_vsi_map_rings_to_vectors - Maps descriptor rings to vectors 3853 * @vsi: the VSI being configured 3854 * 3855 * This function maps descriptor rings to the queue-specific vectors 3856 * we were allotted through the MSI-X enabling code. Ideally, we'd have 3857 * one vector per queue pair, but on a constrained vector budget, we 3858 * group the queue pairs as "efficiently" as possible. 3859 **/ 3860 static void i40e_vsi_map_rings_to_vectors(struct i40e_vsi *vsi) 3861 { 3862 int qp_remaining = vsi->num_queue_pairs; 3863 int q_vectors = vsi->num_q_vectors; 3864 int num_ringpairs; 3865 int v_start = 0; 3866 int qp_idx = 0; 3867 3868 /* If we don't have enough vectors for a 1-to-1 mapping, we'll have to 3869 * group them so there are multiple queues per vector. 3870 * It is also important to go through all the vectors available to be 3871 * sure that if we don't use all the vectors, that the remaining vectors 3872 * are cleared. This is especially important when decreasing the 3873 * number of queues in use. 3874 */ 3875 for (; v_start < q_vectors; v_start++) { 3876 struct i40e_q_vector *q_vector = vsi->q_vectors[v_start]; 3877 3878 num_ringpairs = DIV_ROUND_UP(qp_remaining, q_vectors - v_start); 3879 3880 q_vector->num_ringpairs = num_ringpairs; 3881 3882 q_vector->rx.count = 0; 3883 q_vector->tx.count = 0; 3884 q_vector->rx.ring = NULL; 3885 q_vector->tx.ring = NULL; 3886 3887 while (num_ringpairs--) { 3888 i40e_map_vector_to_qp(vsi, v_start, qp_idx); 3889 qp_idx++; 3890 qp_remaining--; 3891 } 3892 } 3893 } 3894 3895 /** 3896 * i40e_vsi_request_irq - Request IRQ from the OS 3897 * @vsi: the VSI being configured 3898 * @basename: name for the vector 3899 **/ 3900 static int i40e_vsi_request_irq(struct i40e_vsi *vsi, char *basename) 3901 { 3902 struct i40e_pf *pf = vsi->back; 3903 int err; 3904 3905 if (pf->flags & I40E_FLAG_MSIX_ENABLED) 3906 err = i40e_vsi_request_irq_msix(vsi, basename); 3907 else if (pf->flags & I40E_FLAG_MSI_ENABLED) 3908 err = request_irq(pf->pdev->irq, i40e_intr, 0, 3909 pf->int_name, pf); 3910 else 3911 err = request_irq(pf->pdev->irq, i40e_intr, IRQF_SHARED, 3912 pf->int_name, pf); 3913 3914 if (err) 3915 dev_info(&pf->pdev->dev, "request_irq failed, Error %d\n", err); 3916 3917 return err; 3918 } 3919 3920 #ifdef CONFIG_NET_POLL_CONTROLLER 3921 /** 3922 * i40e_netpoll - A Polling 'interrupt' handler 3923 * @netdev: network interface device structure 3924 * 3925 * This is used by netconsole to send skbs without having to re-enable 3926 * interrupts. It's not called while the normal interrupt routine is executing. 3927 **/ 3928 #ifdef I40E_FCOE 3929 void i40e_netpoll(struct net_device *netdev) 3930 #else 3931 static void i40e_netpoll(struct net_device *netdev) 3932 #endif 3933 { 3934 struct i40e_netdev_priv *np = netdev_priv(netdev); 3935 struct i40e_vsi *vsi = np->vsi; 3936 struct i40e_pf *pf = vsi->back; 3937 int i; 3938 3939 /* if interface is down do nothing */ 3940 if (test_bit(__I40E_DOWN, &vsi->state)) 3941 return; 3942 3943 if (pf->flags & I40E_FLAG_MSIX_ENABLED) { 3944 for (i = 0; i < vsi->num_q_vectors; i++) 3945 i40e_msix_clean_rings(0, vsi->q_vectors[i]); 3946 } else { 3947 i40e_intr(pf->pdev->irq, netdev); 3948 } 3949 } 3950 #endif 3951 3952 /** 3953 * i40e_pf_txq_wait - Wait for a PF's Tx queue to be enabled or disabled 3954 * @pf: the PF being configured 3955 * @pf_q: the PF queue 3956 * @enable: enable or disable state of the queue 3957 * 3958 * This routine will wait for the given Tx queue of the PF to reach the 3959 * enabled or disabled state. 3960 * Returns -ETIMEDOUT in case of failing to reach the requested state after 3961 * multiple retries; else will return 0 in case of success. 3962 **/ 3963 static int i40e_pf_txq_wait(struct i40e_pf *pf, int pf_q, bool enable) 3964 { 3965 int i; 3966 u32 tx_reg; 3967 3968 for (i = 0; i < I40E_QUEUE_WAIT_RETRY_LIMIT; i++) { 3969 tx_reg = rd32(&pf->hw, I40E_QTX_ENA(pf_q)); 3970 if (enable == !!(tx_reg & I40E_QTX_ENA_QENA_STAT_MASK)) 3971 break; 3972 3973 usleep_range(10, 20); 3974 } 3975 if (i >= I40E_QUEUE_WAIT_RETRY_LIMIT) 3976 return -ETIMEDOUT; 3977 3978 return 0; 3979 } 3980 3981 /** 3982 * i40e_vsi_control_tx - Start or stop a VSI's rings 3983 * @vsi: the VSI being configured 3984 * @enable: start or stop the rings 3985 **/ 3986 static int i40e_vsi_control_tx(struct i40e_vsi *vsi, bool enable) 3987 { 3988 struct i40e_pf *pf = vsi->back; 3989 struct i40e_hw *hw = &pf->hw; 3990 int i, j, pf_q, ret = 0; 3991 u32 tx_reg; 3992 3993 pf_q = vsi->base_queue; 3994 for (i = 0; i < vsi->num_queue_pairs; i++, pf_q++) { 3995 3996 /* warn the TX unit of coming changes */ 3997 i40e_pre_tx_queue_cfg(&pf->hw, pf_q, enable); 3998 if (!enable) 3999 usleep_range(10, 20); 4000 4001 for (j = 0; j < 50; j++) { 4002 tx_reg = rd32(hw, I40E_QTX_ENA(pf_q)); 4003 if (((tx_reg >> I40E_QTX_ENA_QENA_REQ_SHIFT) & 1) == 4004 ((tx_reg >> I40E_QTX_ENA_QENA_STAT_SHIFT) & 1)) 4005 break; 4006 usleep_range(1000, 2000); 4007 } 4008 /* Skip if the queue is already in the requested state */ 4009 if (enable == !!(tx_reg & I40E_QTX_ENA_QENA_STAT_MASK)) 4010 continue; 4011 4012 /* turn on/off the queue */ 4013 if (enable) { 4014 wr32(hw, I40E_QTX_HEAD(pf_q), 0); 4015 tx_reg |= I40E_QTX_ENA_QENA_REQ_MASK; 4016 } else { 4017 tx_reg &= ~I40E_QTX_ENA_QENA_REQ_MASK; 4018 } 4019 4020 wr32(hw, I40E_QTX_ENA(pf_q), tx_reg); 4021 /* No waiting for the Tx queue to disable */ 4022 if (!enable && test_bit(__I40E_PORT_TX_SUSPENDED, &pf->state)) 4023 continue; 4024 4025 /* wait for the change to finish */ 4026 ret = i40e_pf_txq_wait(pf, pf_q, enable); 4027 if (ret) { 4028 dev_info(&pf->pdev->dev, 4029 "VSI seid %d Tx ring %d %sable timeout\n", 4030 vsi->seid, pf_q, (enable ? "en" : "dis")); 4031 break; 4032 } 4033 } 4034 4035 if (hw->revision_id == 0) 4036 mdelay(50); 4037 return ret; 4038 } 4039 4040 /** 4041 * i40e_pf_rxq_wait - Wait for a PF's Rx queue to be enabled or disabled 4042 * @pf: the PF being configured 4043 * @pf_q: the PF queue 4044 * @enable: enable or disable state of the queue 4045 * 4046 * This routine will wait for the given Rx queue of the PF to reach the 4047 * enabled or disabled state. 4048 * Returns -ETIMEDOUT in case of failing to reach the requested state after 4049 * multiple retries; else will return 0 in case of success. 4050 **/ 4051 static int i40e_pf_rxq_wait(struct i40e_pf *pf, int pf_q, bool enable) 4052 { 4053 int i; 4054 u32 rx_reg; 4055 4056 for (i = 0; i < I40E_QUEUE_WAIT_RETRY_LIMIT; i++) { 4057 rx_reg = rd32(&pf->hw, I40E_QRX_ENA(pf_q)); 4058 if (enable == !!(rx_reg & I40E_QRX_ENA_QENA_STAT_MASK)) 4059 break; 4060 4061 usleep_range(10, 20); 4062 } 4063 if (i >= I40E_QUEUE_WAIT_RETRY_LIMIT) 4064 return -ETIMEDOUT; 4065 4066 return 0; 4067 } 4068 4069 /** 4070 * i40e_vsi_control_rx - Start or stop a VSI's rings 4071 * @vsi: the VSI being configured 4072 * @enable: start or stop the rings 4073 **/ 4074 static int i40e_vsi_control_rx(struct i40e_vsi *vsi, bool enable) 4075 { 4076 struct i40e_pf *pf = vsi->back; 4077 struct i40e_hw *hw = &pf->hw; 4078 int i, j, pf_q, ret = 0; 4079 u32 rx_reg; 4080 4081 pf_q = vsi->base_queue; 4082 for (i = 0; i < vsi->num_queue_pairs; i++, pf_q++) { 4083 for (j = 0; j < 50; j++) { 4084 rx_reg = rd32(hw, I40E_QRX_ENA(pf_q)); 4085 if (((rx_reg >> I40E_QRX_ENA_QENA_REQ_SHIFT) & 1) == 4086 ((rx_reg >> I40E_QRX_ENA_QENA_STAT_SHIFT) & 1)) 4087 break; 4088 usleep_range(1000, 2000); 4089 } 4090 4091 /* Skip if the queue is already in the requested state */ 4092 if (enable == !!(rx_reg & I40E_QRX_ENA_QENA_STAT_MASK)) 4093 continue; 4094 4095 /* turn on/off the queue */ 4096 if (enable) 4097 rx_reg |= I40E_QRX_ENA_QENA_REQ_MASK; 4098 else 4099 rx_reg &= ~I40E_QRX_ENA_QENA_REQ_MASK; 4100 wr32(hw, I40E_QRX_ENA(pf_q), rx_reg); 4101 /* No waiting for the Tx queue to disable */ 4102 if (!enable && test_bit(__I40E_PORT_TX_SUSPENDED, &pf->state)) 4103 continue; 4104 4105 /* wait for the change to finish */ 4106 ret = i40e_pf_rxq_wait(pf, pf_q, enable); 4107 if (ret) { 4108 dev_info(&pf->pdev->dev, 4109 "VSI seid %d Rx ring %d %sable timeout\n", 4110 vsi->seid, pf_q, (enable ? "en" : "dis")); 4111 break; 4112 } 4113 } 4114 4115 return ret; 4116 } 4117 4118 /** 4119 * i40e_vsi_start_rings - Start a VSI's rings 4120 * @vsi: the VSI being configured 4121 **/ 4122 int i40e_vsi_start_rings(struct i40e_vsi *vsi) 4123 { 4124 int ret = 0; 4125 4126 /* do rx first for enable and last for disable */ 4127 ret = i40e_vsi_control_rx(vsi, true); 4128 if (ret) 4129 return ret; 4130 ret = i40e_vsi_control_tx(vsi, true); 4131 4132 return ret; 4133 } 4134 4135 /** 4136 * i40e_vsi_stop_rings - Stop a VSI's rings 4137 * @vsi: the VSI being configured 4138 **/ 4139 void i40e_vsi_stop_rings(struct i40e_vsi *vsi) 4140 { 4141 /* do rx first for enable and last for disable 4142 * Ignore return value, we need to shutdown whatever we can 4143 */ 4144 i40e_vsi_control_tx(vsi, false); 4145 i40e_vsi_control_rx(vsi, false); 4146 } 4147 4148 /** 4149 * i40e_vsi_free_irq - Free the irq association with the OS 4150 * @vsi: the VSI being configured 4151 **/ 4152 static void i40e_vsi_free_irq(struct i40e_vsi *vsi) 4153 { 4154 struct i40e_pf *pf = vsi->back; 4155 struct i40e_hw *hw = &pf->hw; 4156 int base = vsi->base_vector; 4157 u32 val, qp; 4158 int i; 4159 4160 if (pf->flags & I40E_FLAG_MSIX_ENABLED) { 4161 if (!vsi->q_vectors) 4162 return; 4163 4164 if (!vsi->irqs_ready) 4165 return; 4166 4167 vsi->irqs_ready = false; 4168 for (i = 0; i < vsi->num_q_vectors; i++) { 4169 int irq_num; 4170 u16 vector; 4171 4172 vector = i + base; 4173 irq_num = pf->msix_entries[vector].vector; 4174 4175 /* free only the irqs that were actually requested */ 4176 if (!vsi->q_vectors[i] || 4177 !vsi->q_vectors[i]->num_ringpairs) 4178 continue; 4179 4180 /* clear the affinity notifier in the IRQ descriptor */ 4181 irq_set_affinity_notifier(irq_num, NULL); 4182 /* clear the affinity_mask in the IRQ descriptor */ 4183 irq_set_affinity_hint(irq_num, NULL); 4184 synchronize_irq(irq_num); 4185 free_irq(irq_num, vsi->q_vectors[i]); 4186 4187 /* Tear down the interrupt queue link list 4188 * 4189 * We know that they come in pairs and always 4190 * the Rx first, then the Tx. To clear the 4191 * link list, stick the EOL value into the 4192 * next_q field of the registers. 4193 */ 4194 val = rd32(hw, I40E_PFINT_LNKLSTN(vector - 1)); 4195 qp = (val & I40E_PFINT_LNKLSTN_FIRSTQ_INDX_MASK) 4196 >> I40E_PFINT_LNKLSTN_FIRSTQ_INDX_SHIFT; 4197 val |= I40E_QUEUE_END_OF_LIST 4198 << I40E_PFINT_LNKLSTN_FIRSTQ_INDX_SHIFT; 4199 wr32(hw, I40E_PFINT_LNKLSTN(vector - 1), val); 4200 4201 while (qp != I40E_QUEUE_END_OF_LIST) { 4202 u32 next; 4203 4204 val = rd32(hw, I40E_QINT_RQCTL(qp)); 4205 4206 val &= ~(I40E_QINT_RQCTL_MSIX_INDX_MASK | 4207 I40E_QINT_RQCTL_MSIX0_INDX_MASK | 4208 I40E_QINT_RQCTL_CAUSE_ENA_MASK | 4209 I40E_QINT_RQCTL_INTEVENT_MASK); 4210 4211 val |= (I40E_QINT_RQCTL_ITR_INDX_MASK | 4212 I40E_QINT_RQCTL_NEXTQ_INDX_MASK); 4213 4214 wr32(hw, I40E_QINT_RQCTL(qp), val); 4215 4216 val = rd32(hw, I40E_QINT_TQCTL(qp)); 4217 4218 next = (val & I40E_QINT_TQCTL_NEXTQ_INDX_MASK) 4219 >> I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT; 4220 4221 val &= ~(I40E_QINT_TQCTL_MSIX_INDX_MASK | 4222 I40E_QINT_TQCTL_MSIX0_INDX_MASK | 4223 I40E_QINT_TQCTL_CAUSE_ENA_MASK | 4224 I40E_QINT_TQCTL_INTEVENT_MASK); 4225 4226 val |= (I40E_QINT_TQCTL_ITR_INDX_MASK | 4227 I40E_QINT_TQCTL_NEXTQ_INDX_MASK); 4228 4229 wr32(hw, I40E_QINT_TQCTL(qp), val); 4230 qp = next; 4231 } 4232 } 4233 } else { 4234 free_irq(pf->pdev->irq, pf); 4235 4236 val = rd32(hw, I40E_PFINT_LNKLST0); 4237 qp = (val & I40E_PFINT_LNKLSTN_FIRSTQ_INDX_MASK) 4238 >> I40E_PFINT_LNKLSTN_FIRSTQ_INDX_SHIFT; 4239 val |= I40E_QUEUE_END_OF_LIST 4240 << I40E_PFINT_LNKLST0_FIRSTQ_INDX_SHIFT; 4241 wr32(hw, I40E_PFINT_LNKLST0, val); 4242 4243 val = rd32(hw, I40E_QINT_RQCTL(qp)); 4244 val &= ~(I40E_QINT_RQCTL_MSIX_INDX_MASK | 4245 I40E_QINT_RQCTL_MSIX0_INDX_MASK | 4246 I40E_QINT_RQCTL_CAUSE_ENA_MASK | 4247 I40E_QINT_RQCTL_INTEVENT_MASK); 4248 4249 val |= (I40E_QINT_RQCTL_ITR_INDX_MASK | 4250 I40E_QINT_RQCTL_NEXTQ_INDX_MASK); 4251 4252 wr32(hw, I40E_QINT_RQCTL(qp), val); 4253 4254 val = rd32(hw, I40E_QINT_TQCTL(qp)); 4255 4256 val &= ~(I40E_QINT_TQCTL_MSIX_INDX_MASK | 4257 I40E_QINT_TQCTL_MSIX0_INDX_MASK | 4258 I40E_QINT_TQCTL_CAUSE_ENA_MASK | 4259 I40E_QINT_TQCTL_INTEVENT_MASK); 4260 4261 val |= (I40E_QINT_TQCTL_ITR_INDX_MASK | 4262 I40E_QINT_TQCTL_NEXTQ_INDX_MASK); 4263 4264 wr32(hw, I40E_QINT_TQCTL(qp), val); 4265 } 4266 } 4267 4268 /** 4269 * i40e_free_q_vector - Free memory allocated for specific interrupt vector 4270 * @vsi: the VSI being configured 4271 * @v_idx: Index of vector to be freed 4272 * 4273 * This function frees the memory allocated to the q_vector. In addition if 4274 * NAPI is enabled it will delete any references to the NAPI struct prior 4275 * to freeing the q_vector. 4276 **/ 4277 static void i40e_free_q_vector(struct i40e_vsi *vsi, int v_idx) 4278 { 4279 struct i40e_q_vector *q_vector = vsi->q_vectors[v_idx]; 4280 struct i40e_ring *ring; 4281 4282 if (!q_vector) 4283 return; 4284 4285 /* disassociate q_vector from rings */ 4286 i40e_for_each_ring(ring, q_vector->tx) 4287 ring->q_vector = NULL; 4288 4289 i40e_for_each_ring(ring, q_vector->rx) 4290 ring->q_vector = NULL; 4291 4292 /* only VSI w/ an associated netdev is set up w/ NAPI */ 4293 if (vsi->netdev) 4294 netif_napi_del(&q_vector->napi); 4295 4296 vsi->q_vectors[v_idx] = NULL; 4297 4298 kfree_rcu(q_vector, rcu); 4299 } 4300 4301 /** 4302 * i40e_vsi_free_q_vectors - Free memory allocated for interrupt vectors 4303 * @vsi: the VSI being un-configured 4304 * 4305 * This frees the memory allocated to the q_vectors and 4306 * deletes references to the NAPI struct. 4307 **/ 4308 static void i40e_vsi_free_q_vectors(struct i40e_vsi *vsi) 4309 { 4310 int v_idx; 4311 4312 for (v_idx = 0; v_idx < vsi->num_q_vectors; v_idx++) 4313 i40e_free_q_vector(vsi, v_idx); 4314 } 4315 4316 /** 4317 * i40e_reset_interrupt_capability - Disable interrupt setup in OS 4318 * @pf: board private structure 4319 **/ 4320 static void i40e_reset_interrupt_capability(struct i40e_pf *pf) 4321 { 4322 /* If we're in Legacy mode, the interrupt was cleaned in vsi_close */ 4323 if (pf->flags & I40E_FLAG_MSIX_ENABLED) { 4324 pci_disable_msix(pf->pdev); 4325 kfree(pf->msix_entries); 4326 pf->msix_entries = NULL; 4327 kfree(pf->irq_pile); 4328 pf->irq_pile = NULL; 4329 } else if (pf->flags & I40E_FLAG_MSI_ENABLED) { 4330 pci_disable_msi(pf->pdev); 4331 } 4332 pf->flags &= ~(I40E_FLAG_MSIX_ENABLED | I40E_FLAG_MSI_ENABLED); 4333 } 4334 4335 /** 4336 * i40e_clear_interrupt_scheme - Clear the current interrupt scheme settings 4337 * @pf: board private structure 4338 * 4339 * We go through and clear interrupt specific resources and reset the structure 4340 * to pre-load conditions 4341 **/ 4342 static void i40e_clear_interrupt_scheme(struct i40e_pf *pf) 4343 { 4344 int i; 4345 4346 i40e_stop_misc_vector(pf); 4347 if (pf->flags & I40E_FLAG_MSIX_ENABLED && pf->msix_entries) { 4348 synchronize_irq(pf->msix_entries[0].vector); 4349 free_irq(pf->msix_entries[0].vector, pf); 4350 } 4351 4352 i40e_put_lump(pf->irq_pile, pf->iwarp_base_vector, 4353 I40E_IWARP_IRQ_PILE_ID); 4354 4355 i40e_put_lump(pf->irq_pile, 0, I40E_PILE_VALID_BIT-1); 4356 for (i = 0; i < pf->num_alloc_vsi; i++) 4357 if (pf->vsi[i]) 4358 i40e_vsi_free_q_vectors(pf->vsi[i]); 4359 i40e_reset_interrupt_capability(pf); 4360 } 4361 4362 /** 4363 * i40e_napi_enable_all - Enable NAPI for all q_vectors in the VSI 4364 * @vsi: the VSI being configured 4365 **/ 4366 static void i40e_napi_enable_all(struct i40e_vsi *vsi) 4367 { 4368 int q_idx; 4369 4370 if (!vsi->netdev) 4371 return; 4372 4373 for (q_idx = 0; q_idx < vsi->num_q_vectors; q_idx++) 4374 napi_enable(&vsi->q_vectors[q_idx]->napi); 4375 } 4376 4377 /** 4378 * i40e_napi_disable_all - Disable NAPI for all q_vectors in the VSI 4379 * @vsi: the VSI being configured 4380 **/ 4381 static void i40e_napi_disable_all(struct i40e_vsi *vsi) 4382 { 4383 int q_idx; 4384 4385 if (!vsi->netdev) 4386 return; 4387 4388 for (q_idx = 0; q_idx < vsi->num_q_vectors; q_idx++) 4389 napi_disable(&vsi->q_vectors[q_idx]->napi); 4390 } 4391 4392 /** 4393 * i40e_vsi_close - Shut down a VSI 4394 * @vsi: the vsi to be quelled 4395 **/ 4396 static void i40e_vsi_close(struct i40e_vsi *vsi) 4397 { 4398 bool reset = false; 4399 4400 if (!test_and_set_bit(__I40E_DOWN, &vsi->state)) 4401 i40e_down(vsi); 4402 i40e_vsi_free_irq(vsi); 4403 i40e_vsi_free_tx_resources(vsi); 4404 i40e_vsi_free_rx_resources(vsi); 4405 vsi->current_netdev_flags = 0; 4406 if (test_bit(__I40E_RESET_RECOVERY_PENDING, &vsi->back->state)) 4407 reset = true; 4408 i40e_notify_client_of_netdev_close(vsi, reset); 4409 } 4410 4411 /** 4412 * i40e_quiesce_vsi - Pause a given VSI 4413 * @vsi: the VSI being paused 4414 **/ 4415 static void i40e_quiesce_vsi(struct i40e_vsi *vsi) 4416 { 4417 if (test_bit(__I40E_DOWN, &vsi->state)) 4418 return; 4419 4420 /* No need to disable FCoE VSI when Tx suspended */ 4421 if ((test_bit(__I40E_PORT_TX_SUSPENDED, &vsi->back->state)) && 4422 vsi->type == I40E_VSI_FCOE) { 4423 dev_dbg(&vsi->back->pdev->dev, 4424 "VSI seid %d skipping FCoE VSI disable\n", vsi->seid); 4425 return; 4426 } 4427 4428 set_bit(__I40E_NEEDS_RESTART, &vsi->state); 4429 if (vsi->netdev && netif_running(vsi->netdev)) 4430 vsi->netdev->netdev_ops->ndo_stop(vsi->netdev); 4431 else 4432 i40e_vsi_close(vsi); 4433 } 4434 4435 /** 4436 * i40e_unquiesce_vsi - Resume a given VSI 4437 * @vsi: the VSI being resumed 4438 **/ 4439 static void i40e_unquiesce_vsi(struct i40e_vsi *vsi) 4440 { 4441 if (!test_bit(__I40E_NEEDS_RESTART, &vsi->state)) 4442 return; 4443 4444 clear_bit(__I40E_NEEDS_RESTART, &vsi->state); 4445 if (vsi->netdev && netif_running(vsi->netdev)) 4446 vsi->netdev->netdev_ops->ndo_open(vsi->netdev); 4447 else 4448 i40e_vsi_open(vsi); /* this clears the DOWN bit */ 4449 } 4450 4451 /** 4452 * i40e_pf_quiesce_all_vsi - Pause all VSIs on a PF 4453 * @pf: the PF 4454 **/ 4455 static void i40e_pf_quiesce_all_vsi(struct i40e_pf *pf) 4456 { 4457 int v; 4458 4459 for (v = 0; v < pf->num_alloc_vsi; v++) { 4460 if (pf->vsi[v]) 4461 i40e_quiesce_vsi(pf->vsi[v]); 4462 } 4463 } 4464 4465 /** 4466 * i40e_pf_unquiesce_all_vsi - Resume all VSIs on a PF 4467 * @pf: the PF 4468 **/ 4469 static void i40e_pf_unquiesce_all_vsi(struct i40e_pf *pf) 4470 { 4471 int v; 4472 4473 for (v = 0; v < pf->num_alloc_vsi; v++) { 4474 if (pf->vsi[v]) 4475 i40e_unquiesce_vsi(pf->vsi[v]); 4476 } 4477 } 4478 4479 #ifdef CONFIG_I40E_DCB 4480 /** 4481 * i40e_vsi_wait_queues_disabled - Wait for VSI's queues to be disabled 4482 * @vsi: the VSI being configured 4483 * 4484 * This function waits for the given VSI's queues to be disabled. 4485 **/ 4486 static int i40e_vsi_wait_queues_disabled(struct i40e_vsi *vsi) 4487 { 4488 struct i40e_pf *pf = vsi->back; 4489 int i, pf_q, ret; 4490 4491 pf_q = vsi->base_queue; 4492 for (i = 0; i < vsi->num_queue_pairs; i++, pf_q++) { 4493 /* Check and wait for the disable status of the queue */ 4494 ret = i40e_pf_txq_wait(pf, pf_q, false); 4495 if (ret) { 4496 dev_info(&pf->pdev->dev, 4497 "VSI seid %d Tx ring %d disable timeout\n", 4498 vsi->seid, pf_q); 4499 return ret; 4500 } 4501 } 4502 4503 pf_q = vsi->base_queue; 4504 for (i = 0; i < vsi->num_queue_pairs; i++, pf_q++) { 4505 /* Check and wait for the disable status of the queue */ 4506 ret = i40e_pf_rxq_wait(pf, pf_q, false); 4507 if (ret) { 4508 dev_info(&pf->pdev->dev, 4509 "VSI seid %d Rx ring %d disable timeout\n", 4510 vsi->seid, pf_q); 4511 return ret; 4512 } 4513 } 4514 4515 return 0; 4516 } 4517 4518 /** 4519 * i40e_pf_wait_queues_disabled - Wait for all queues of PF VSIs to be disabled 4520 * @pf: the PF 4521 * 4522 * This function waits for the queues to be in disabled state for all the 4523 * VSIs that are managed by this PF. 4524 **/ 4525 static int i40e_pf_wait_queues_disabled(struct i40e_pf *pf) 4526 { 4527 int v, ret = 0; 4528 4529 for (v = 0; v < pf->hw.func_caps.num_vsis; v++) { 4530 /* No need to wait for FCoE VSI queues */ 4531 if (pf->vsi[v] && pf->vsi[v]->type != I40E_VSI_FCOE) { 4532 ret = i40e_vsi_wait_queues_disabled(pf->vsi[v]); 4533 if (ret) 4534 break; 4535 } 4536 } 4537 4538 return ret; 4539 } 4540 4541 #endif 4542 4543 /** 4544 * i40e_detect_recover_hung_queue - Function to detect and recover hung_queue 4545 * @q_idx: TX queue number 4546 * @vsi: Pointer to VSI struct 4547 * 4548 * This function checks specified queue for given VSI. Detects hung condition. 4549 * Sets hung bit since it is two step process. Before next run of service task 4550 * if napi_poll runs, it reset 'hung' bit for respective q_vector. If not, 4551 * hung condition remain unchanged and during subsequent run, this function 4552 * issues SW interrupt to recover from hung condition. 4553 **/ 4554 static void i40e_detect_recover_hung_queue(int q_idx, struct i40e_vsi *vsi) 4555 { 4556 struct i40e_ring *tx_ring = NULL; 4557 struct i40e_pf *pf; 4558 u32 head, val, tx_pending_hw; 4559 int i; 4560 4561 pf = vsi->back; 4562 4563 /* now that we have an index, find the tx_ring struct */ 4564 for (i = 0; i < vsi->num_queue_pairs; i++) { 4565 if (vsi->tx_rings[i] && vsi->tx_rings[i]->desc) { 4566 if (q_idx == vsi->tx_rings[i]->queue_index) { 4567 tx_ring = vsi->tx_rings[i]; 4568 break; 4569 } 4570 } 4571 } 4572 4573 if (!tx_ring) 4574 return; 4575 4576 /* Read interrupt register */ 4577 if (pf->flags & I40E_FLAG_MSIX_ENABLED) 4578 val = rd32(&pf->hw, 4579 I40E_PFINT_DYN_CTLN(tx_ring->q_vector->v_idx + 4580 tx_ring->vsi->base_vector - 1)); 4581 else 4582 val = rd32(&pf->hw, I40E_PFINT_DYN_CTL0); 4583 4584 head = i40e_get_head(tx_ring); 4585 4586 tx_pending_hw = i40e_get_tx_pending(tx_ring, false); 4587 4588 /* HW is done executing descriptors, updated HEAD write back, 4589 * but SW hasn't processed those descriptors. If interrupt is 4590 * not generated from this point ON, it could result into 4591 * dev_watchdog detecting timeout on those netdev_queue, 4592 * hence proactively trigger SW interrupt. 4593 */ 4594 if (tx_pending_hw && (!(val & I40E_PFINT_DYN_CTLN_INTENA_MASK))) { 4595 /* NAPI Poll didn't run and clear since it was set */ 4596 if (test_and_clear_bit(I40E_Q_VECTOR_HUNG_DETECT, 4597 &tx_ring->q_vector->hung_detected)) { 4598 netdev_info(vsi->netdev, "VSI_seid %d, Hung TX queue %d, tx_pending_hw: %d, NTC:0x%x, HWB: 0x%x, NTU: 0x%x, TAIL: 0x%x\n", 4599 vsi->seid, q_idx, tx_pending_hw, 4600 tx_ring->next_to_clean, head, 4601 tx_ring->next_to_use, 4602 readl(tx_ring->tail)); 4603 netdev_info(vsi->netdev, "VSI_seid %d, Issuing force_wb for TX queue %d, Interrupt Reg: 0x%x\n", 4604 vsi->seid, q_idx, val); 4605 i40e_force_wb(vsi, tx_ring->q_vector); 4606 } else { 4607 /* First Chance - detected possible hung */ 4608 set_bit(I40E_Q_VECTOR_HUNG_DETECT, 4609 &tx_ring->q_vector->hung_detected); 4610 } 4611 } 4612 4613 /* This is the case where we have interrupts missing, 4614 * so the tx_pending in HW will most likely be 0, but we 4615 * will have tx_pending in SW since the WB happened but the 4616 * interrupt got lost. 4617 */ 4618 if ((!tx_pending_hw) && i40e_get_tx_pending(tx_ring, true) && 4619 (!(val & I40E_PFINT_DYN_CTLN_INTENA_MASK))) { 4620 if (napi_reschedule(&tx_ring->q_vector->napi)) 4621 tx_ring->tx_stats.tx_lost_interrupt++; 4622 } 4623 } 4624 4625 /** 4626 * i40e_detect_recover_hung - Function to detect and recover hung_queues 4627 * @pf: pointer to PF struct 4628 * 4629 * LAN VSI has netdev and netdev has TX queues. This function is to check 4630 * each of those TX queues if they are hung, trigger recovery by issuing 4631 * SW interrupt. 4632 **/ 4633 static void i40e_detect_recover_hung(struct i40e_pf *pf) 4634 { 4635 struct net_device *netdev; 4636 struct i40e_vsi *vsi; 4637 int i; 4638 4639 /* Only for LAN VSI */ 4640 vsi = pf->vsi[pf->lan_vsi]; 4641 4642 if (!vsi) 4643 return; 4644 4645 /* Make sure, VSI state is not DOWN/RECOVERY_PENDING */ 4646 if (test_bit(__I40E_DOWN, &vsi->back->state) || 4647 test_bit(__I40E_RESET_RECOVERY_PENDING, &vsi->back->state)) 4648 return; 4649 4650 /* Make sure type is MAIN VSI */ 4651 if (vsi->type != I40E_VSI_MAIN) 4652 return; 4653 4654 netdev = vsi->netdev; 4655 if (!netdev) 4656 return; 4657 4658 /* Bail out if netif_carrier is not OK */ 4659 if (!netif_carrier_ok(netdev)) 4660 return; 4661 4662 /* Go thru' TX queues for netdev */ 4663 for (i = 0; i < netdev->num_tx_queues; i++) { 4664 struct netdev_queue *q; 4665 4666 q = netdev_get_tx_queue(netdev, i); 4667 if (q) 4668 i40e_detect_recover_hung_queue(i, vsi); 4669 } 4670 } 4671 4672 /** 4673 * i40e_get_iscsi_tc_map - Return TC map for iSCSI APP 4674 * @pf: pointer to PF 4675 * 4676 * Get TC map for ISCSI PF type that will include iSCSI TC 4677 * and LAN TC. 4678 **/ 4679 static u8 i40e_get_iscsi_tc_map(struct i40e_pf *pf) 4680 { 4681 struct i40e_dcb_app_priority_table app; 4682 struct i40e_hw *hw = &pf->hw; 4683 u8 enabled_tc = 1; /* TC0 is always enabled */ 4684 u8 tc, i; 4685 /* Get the iSCSI APP TLV */ 4686 struct i40e_dcbx_config *dcbcfg = &hw->local_dcbx_config; 4687 4688 for (i = 0; i < dcbcfg->numapps; i++) { 4689 app = dcbcfg->app[i]; 4690 if (app.selector == I40E_APP_SEL_TCPIP && 4691 app.protocolid == I40E_APP_PROTOID_ISCSI) { 4692 tc = dcbcfg->etscfg.prioritytable[app.priority]; 4693 enabled_tc |= BIT(tc); 4694 break; 4695 } 4696 } 4697 4698 return enabled_tc; 4699 } 4700 4701 /** 4702 * i40e_dcb_get_num_tc - Get the number of TCs from DCBx config 4703 * @dcbcfg: the corresponding DCBx configuration structure 4704 * 4705 * Return the number of TCs from given DCBx configuration 4706 **/ 4707 static u8 i40e_dcb_get_num_tc(struct i40e_dcbx_config *dcbcfg) 4708 { 4709 int i, tc_unused = 0; 4710 u8 num_tc = 0; 4711 u8 ret = 0; 4712 4713 /* Scan the ETS Config Priority Table to find 4714 * traffic class enabled for a given priority 4715 * and create a bitmask of enabled TCs 4716 */ 4717 for (i = 0; i < I40E_MAX_USER_PRIORITY; i++) 4718 num_tc |= BIT(dcbcfg->etscfg.prioritytable[i]); 4719 4720 /* Now scan the bitmask to check for 4721 * contiguous TCs starting with TC0 4722 */ 4723 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 4724 if (num_tc & BIT(i)) { 4725 if (!tc_unused) { 4726 ret++; 4727 } else { 4728 pr_err("Non-contiguous TC - Disabling DCB\n"); 4729 return 1; 4730 } 4731 } else { 4732 tc_unused = 1; 4733 } 4734 } 4735 4736 /* There is always at least TC0 */ 4737 if (!ret) 4738 ret = 1; 4739 4740 return ret; 4741 } 4742 4743 /** 4744 * i40e_dcb_get_enabled_tc - Get enabled traffic classes 4745 * @dcbcfg: the corresponding DCBx configuration structure 4746 * 4747 * Query the current DCB configuration and return the number of 4748 * traffic classes enabled from the given DCBX config 4749 **/ 4750 static u8 i40e_dcb_get_enabled_tc(struct i40e_dcbx_config *dcbcfg) 4751 { 4752 u8 num_tc = i40e_dcb_get_num_tc(dcbcfg); 4753 u8 enabled_tc = 1; 4754 u8 i; 4755 4756 for (i = 0; i < num_tc; i++) 4757 enabled_tc |= BIT(i); 4758 4759 return enabled_tc; 4760 } 4761 4762 /** 4763 * i40e_pf_get_num_tc - Get enabled traffic classes for PF 4764 * @pf: PF being queried 4765 * 4766 * Return number of traffic classes enabled for the given PF 4767 **/ 4768 static u8 i40e_pf_get_num_tc(struct i40e_pf *pf) 4769 { 4770 struct i40e_hw *hw = &pf->hw; 4771 u8 i, enabled_tc = 1; 4772 u8 num_tc = 0; 4773 struct i40e_dcbx_config *dcbcfg = &hw->local_dcbx_config; 4774 4775 /* If DCB is not enabled then always in single TC */ 4776 if (!(pf->flags & I40E_FLAG_DCB_ENABLED)) 4777 return 1; 4778 4779 /* SFP mode will be enabled for all TCs on port */ 4780 if (!(pf->flags & I40E_FLAG_MFP_ENABLED)) 4781 return i40e_dcb_get_num_tc(dcbcfg); 4782 4783 /* MFP mode return count of enabled TCs for this PF */ 4784 if (pf->hw.func_caps.iscsi) 4785 enabled_tc = i40e_get_iscsi_tc_map(pf); 4786 else 4787 return 1; /* Only TC0 */ 4788 4789 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 4790 if (enabled_tc & BIT(i)) 4791 num_tc++; 4792 } 4793 return num_tc; 4794 } 4795 4796 /** 4797 * i40e_pf_get_pf_tc_map - Get bitmap for enabled traffic classes 4798 * @pf: PF being queried 4799 * 4800 * Return a bitmap for enabled traffic classes for this PF. 4801 **/ 4802 static u8 i40e_pf_get_tc_map(struct i40e_pf *pf) 4803 { 4804 /* If DCB is not enabled for this PF then just return default TC */ 4805 if (!(pf->flags & I40E_FLAG_DCB_ENABLED)) 4806 return I40E_DEFAULT_TRAFFIC_CLASS; 4807 4808 /* SFP mode we want PF to be enabled for all TCs */ 4809 if (!(pf->flags & I40E_FLAG_MFP_ENABLED)) 4810 return i40e_dcb_get_enabled_tc(&pf->hw.local_dcbx_config); 4811 4812 /* MFP enabled and iSCSI PF type */ 4813 if (pf->hw.func_caps.iscsi) 4814 return i40e_get_iscsi_tc_map(pf); 4815 else 4816 return I40E_DEFAULT_TRAFFIC_CLASS; 4817 } 4818 4819 /** 4820 * i40e_vsi_get_bw_info - Query VSI BW Information 4821 * @vsi: the VSI being queried 4822 * 4823 * Returns 0 on success, negative value on failure 4824 **/ 4825 static int i40e_vsi_get_bw_info(struct i40e_vsi *vsi) 4826 { 4827 struct i40e_aqc_query_vsi_ets_sla_config_resp bw_ets_config = {0}; 4828 struct i40e_aqc_query_vsi_bw_config_resp bw_config = {0}; 4829 struct i40e_pf *pf = vsi->back; 4830 struct i40e_hw *hw = &pf->hw; 4831 i40e_status ret; 4832 u32 tc_bw_max; 4833 int i; 4834 4835 /* Get the VSI level BW configuration */ 4836 ret = i40e_aq_query_vsi_bw_config(hw, vsi->seid, &bw_config, NULL); 4837 if (ret) { 4838 dev_info(&pf->pdev->dev, 4839 "couldn't get PF vsi bw config, err %s aq_err %s\n", 4840 i40e_stat_str(&pf->hw, ret), 4841 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 4842 return -EINVAL; 4843 } 4844 4845 /* Get the VSI level BW configuration per TC */ 4846 ret = i40e_aq_query_vsi_ets_sla_config(hw, vsi->seid, &bw_ets_config, 4847 NULL); 4848 if (ret) { 4849 dev_info(&pf->pdev->dev, 4850 "couldn't get PF vsi ets bw config, err %s aq_err %s\n", 4851 i40e_stat_str(&pf->hw, ret), 4852 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 4853 return -EINVAL; 4854 } 4855 4856 if (bw_config.tc_valid_bits != bw_ets_config.tc_valid_bits) { 4857 dev_info(&pf->pdev->dev, 4858 "Enabled TCs mismatch from querying VSI BW info 0x%08x 0x%08x\n", 4859 bw_config.tc_valid_bits, 4860 bw_ets_config.tc_valid_bits); 4861 /* Still continuing */ 4862 } 4863 4864 vsi->bw_limit = le16_to_cpu(bw_config.port_bw_limit); 4865 vsi->bw_max_quanta = bw_config.max_bw; 4866 tc_bw_max = le16_to_cpu(bw_ets_config.tc_bw_max[0]) | 4867 (le16_to_cpu(bw_ets_config.tc_bw_max[1]) << 16); 4868 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 4869 vsi->bw_ets_share_credits[i] = bw_ets_config.share_credits[i]; 4870 vsi->bw_ets_limit_credits[i] = 4871 le16_to_cpu(bw_ets_config.credits[i]); 4872 /* 3 bits out of 4 for each TC */ 4873 vsi->bw_ets_max_quanta[i] = (u8)((tc_bw_max >> (i*4)) & 0x7); 4874 } 4875 4876 return 0; 4877 } 4878 4879 /** 4880 * i40e_vsi_configure_bw_alloc - Configure VSI BW allocation per TC 4881 * @vsi: the VSI being configured 4882 * @enabled_tc: TC bitmap 4883 * @bw_credits: BW shared credits per TC 4884 * 4885 * Returns 0 on success, negative value on failure 4886 **/ 4887 static int i40e_vsi_configure_bw_alloc(struct i40e_vsi *vsi, u8 enabled_tc, 4888 u8 *bw_share) 4889 { 4890 struct i40e_aqc_configure_vsi_tc_bw_data bw_data; 4891 i40e_status ret; 4892 int i; 4893 4894 bw_data.tc_valid_bits = enabled_tc; 4895 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) 4896 bw_data.tc_bw_credits[i] = bw_share[i]; 4897 4898 ret = i40e_aq_config_vsi_tc_bw(&vsi->back->hw, vsi->seid, &bw_data, 4899 NULL); 4900 if (ret) { 4901 dev_info(&vsi->back->pdev->dev, 4902 "AQ command Config VSI BW allocation per TC failed = %d\n", 4903 vsi->back->hw.aq.asq_last_status); 4904 return -EINVAL; 4905 } 4906 4907 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) 4908 vsi->info.qs_handle[i] = bw_data.qs_handles[i]; 4909 4910 return 0; 4911 } 4912 4913 /** 4914 * i40e_vsi_config_netdev_tc - Setup the netdev TC configuration 4915 * @vsi: the VSI being configured 4916 * @enabled_tc: TC map to be enabled 4917 * 4918 **/ 4919 static void i40e_vsi_config_netdev_tc(struct i40e_vsi *vsi, u8 enabled_tc) 4920 { 4921 struct net_device *netdev = vsi->netdev; 4922 struct i40e_pf *pf = vsi->back; 4923 struct i40e_hw *hw = &pf->hw; 4924 u8 netdev_tc = 0; 4925 int i; 4926 struct i40e_dcbx_config *dcbcfg = &hw->local_dcbx_config; 4927 4928 if (!netdev) 4929 return; 4930 4931 if (!enabled_tc) { 4932 netdev_reset_tc(netdev); 4933 return; 4934 } 4935 4936 /* Set up actual enabled TCs on the VSI */ 4937 if (netdev_set_num_tc(netdev, vsi->tc_config.numtc)) 4938 return; 4939 4940 /* set per TC queues for the VSI */ 4941 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 4942 /* Only set TC queues for enabled tcs 4943 * 4944 * e.g. For a VSI that has TC0 and TC3 enabled the 4945 * enabled_tc bitmap would be 0x00001001; the driver 4946 * will set the numtc for netdev as 2 that will be 4947 * referenced by the netdev layer as TC 0 and 1. 4948 */ 4949 if (vsi->tc_config.enabled_tc & BIT(i)) 4950 netdev_set_tc_queue(netdev, 4951 vsi->tc_config.tc_info[i].netdev_tc, 4952 vsi->tc_config.tc_info[i].qcount, 4953 vsi->tc_config.tc_info[i].qoffset); 4954 } 4955 4956 /* Assign UP2TC map for the VSI */ 4957 for (i = 0; i < I40E_MAX_USER_PRIORITY; i++) { 4958 /* Get the actual TC# for the UP */ 4959 u8 ets_tc = dcbcfg->etscfg.prioritytable[i]; 4960 /* Get the mapped netdev TC# for the UP */ 4961 netdev_tc = vsi->tc_config.tc_info[ets_tc].netdev_tc; 4962 netdev_set_prio_tc_map(netdev, i, netdev_tc); 4963 } 4964 } 4965 4966 /** 4967 * i40e_vsi_update_queue_map - Update our copy of VSi info with new queue map 4968 * @vsi: the VSI being configured 4969 * @ctxt: the ctxt buffer returned from AQ VSI update param command 4970 **/ 4971 static void i40e_vsi_update_queue_map(struct i40e_vsi *vsi, 4972 struct i40e_vsi_context *ctxt) 4973 { 4974 /* copy just the sections touched not the entire info 4975 * since not all sections are valid as returned by 4976 * update vsi params 4977 */ 4978 vsi->info.mapping_flags = ctxt->info.mapping_flags; 4979 memcpy(&vsi->info.queue_mapping, 4980 &ctxt->info.queue_mapping, sizeof(vsi->info.queue_mapping)); 4981 memcpy(&vsi->info.tc_mapping, ctxt->info.tc_mapping, 4982 sizeof(vsi->info.tc_mapping)); 4983 } 4984 4985 /** 4986 * i40e_vsi_config_tc - Configure VSI Tx Scheduler for given TC map 4987 * @vsi: VSI to be configured 4988 * @enabled_tc: TC bitmap 4989 * 4990 * This configures a particular VSI for TCs that are mapped to the 4991 * given TC bitmap. It uses default bandwidth share for TCs across 4992 * VSIs to configure TC for a particular VSI. 4993 * 4994 * NOTE: 4995 * It is expected that the VSI queues have been quisced before calling 4996 * this function. 4997 **/ 4998 static int i40e_vsi_config_tc(struct i40e_vsi *vsi, u8 enabled_tc) 4999 { 5000 u8 bw_share[I40E_MAX_TRAFFIC_CLASS] = {0}; 5001 struct i40e_vsi_context ctxt; 5002 int ret = 0; 5003 int i; 5004 5005 /* Check if enabled_tc is same as existing or new TCs */ 5006 if (vsi->tc_config.enabled_tc == enabled_tc) 5007 return ret; 5008 5009 /* Enable ETS TCs with equal BW Share for now across all VSIs */ 5010 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 5011 if (enabled_tc & BIT(i)) 5012 bw_share[i] = 1; 5013 } 5014 5015 ret = i40e_vsi_configure_bw_alloc(vsi, enabled_tc, bw_share); 5016 if (ret) { 5017 dev_info(&vsi->back->pdev->dev, 5018 "Failed configuring TC map %d for VSI %d\n", 5019 enabled_tc, vsi->seid); 5020 goto out; 5021 } 5022 5023 /* Update Queue Pairs Mapping for currently enabled UPs */ 5024 ctxt.seid = vsi->seid; 5025 ctxt.pf_num = vsi->back->hw.pf_id; 5026 ctxt.vf_num = 0; 5027 ctxt.uplink_seid = vsi->uplink_seid; 5028 ctxt.info = vsi->info; 5029 i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, false); 5030 5031 if (vsi->back->flags & I40E_FLAG_IWARP_ENABLED) { 5032 ctxt.info.valid_sections |= 5033 cpu_to_le16(I40E_AQ_VSI_PROP_QUEUE_OPT_VALID); 5034 ctxt.info.queueing_opt_flags |= I40E_AQ_VSI_QUE_OPT_TCP_ENA; 5035 } 5036 5037 /* Update the VSI after updating the VSI queue-mapping information */ 5038 ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL); 5039 if (ret) { 5040 dev_info(&vsi->back->pdev->dev, 5041 "Update vsi tc config failed, err %s aq_err %s\n", 5042 i40e_stat_str(&vsi->back->hw, ret), 5043 i40e_aq_str(&vsi->back->hw, 5044 vsi->back->hw.aq.asq_last_status)); 5045 goto out; 5046 } 5047 /* update the local VSI info with updated queue map */ 5048 i40e_vsi_update_queue_map(vsi, &ctxt); 5049 vsi->info.valid_sections = 0; 5050 5051 /* Update current VSI BW information */ 5052 ret = i40e_vsi_get_bw_info(vsi); 5053 if (ret) { 5054 dev_info(&vsi->back->pdev->dev, 5055 "Failed updating vsi bw info, err %s aq_err %s\n", 5056 i40e_stat_str(&vsi->back->hw, ret), 5057 i40e_aq_str(&vsi->back->hw, 5058 vsi->back->hw.aq.asq_last_status)); 5059 goto out; 5060 } 5061 5062 /* Update the netdev TC setup */ 5063 i40e_vsi_config_netdev_tc(vsi, enabled_tc); 5064 out: 5065 return ret; 5066 } 5067 5068 /** 5069 * i40e_veb_config_tc - Configure TCs for given VEB 5070 * @veb: given VEB 5071 * @enabled_tc: TC bitmap 5072 * 5073 * Configures given TC bitmap for VEB (switching) element 5074 **/ 5075 int i40e_veb_config_tc(struct i40e_veb *veb, u8 enabled_tc) 5076 { 5077 struct i40e_aqc_configure_switching_comp_bw_config_data bw_data = {0}; 5078 struct i40e_pf *pf = veb->pf; 5079 int ret = 0; 5080 int i; 5081 5082 /* No TCs or already enabled TCs just return */ 5083 if (!enabled_tc || veb->enabled_tc == enabled_tc) 5084 return ret; 5085 5086 bw_data.tc_valid_bits = enabled_tc; 5087 /* bw_data.absolute_credits is not set (relative) */ 5088 5089 /* Enable ETS TCs with equal BW Share for now */ 5090 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 5091 if (enabled_tc & BIT(i)) 5092 bw_data.tc_bw_share_credits[i] = 1; 5093 } 5094 5095 ret = i40e_aq_config_switch_comp_bw_config(&pf->hw, veb->seid, 5096 &bw_data, NULL); 5097 if (ret) { 5098 dev_info(&pf->pdev->dev, 5099 "VEB bw config failed, err %s aq_err %s\n", 5100 i40e_stat_str(&pf->hw, ret), 5101 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 5102 goto out; 5103 } 5104 5105 /* Update the BW information */ 5106 ret = i40e_veb_get_bw_info(veb); 5107 if (ret) { 5108 dev_info(&pf->pdev->dev, 5109 "Failed getting veb bw config, err %s aq_err %s\n", 5110 i40e_stat_str(&pf->hw, ret), 5111 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 5112 } 5113 5114 out: 5115 return ret; 5116 } 5117 5118 #ifdef CONFIG_I40E_DCB 5119 /** 5120 * i40e_dcb_reconfigure - Reconfigure all VEBs and VSIs 5121 * @pf: PF struct 5122 * 5123 * Reconfigure VEB/VSIs on a given PF; it is assumed that 5124 * the caller would've quiesce all the VSIs before calling 5125 * this function 5126 **/ 5127 static void i40e_dcb_reconfigure(struct i40e_pf *pf) 5128 { 5129 u8 tc_map = 0; 5130 int ret; 5131 u8 v; 5132 5133 /* Enable the TCs available on PF to all VEBs */ 5134 tc_map = i40e_pf_get_tc_map(pf); 5135 for (v = 0; v < I40E_MAX_VEB; v++) { 5136 if (!pf->veb[v]) 5137 continue; 5138 ret = i40e_veb_config_tc(pf->veb[v], tc_map); 5139 if (ret) { 5140 dev_info(&pf->pdev->dev, 5141 "Failed configuring TC for VEB seid=%d\n", 5142 pf->veb[v]->seid); 5143 /* Will try to configure as many components */ 5144 } 5145 } 5146 5147 /* Update each VSI */ 5148 for (v = 0; v < pf->num_alloc_vsi; v++) { 5149 if (!pf->vsi[v]) 5150 continue; 5151 5152 /* - Enable all TCs for the LAN VSI 5153 #ifdef I40E_FCOE 5154 * - For FCoE VSI only enable the TC configured 5155 * as per the APP TLV 5156 #endif 5157 * - For all others keep them at TC0 for now 5158 */ 5159 if (v == pf->lan_vsi) 5160 tc_map = i40e_pf_get_tc_map(pf); 5161 else 5162 tc_map = I40E_DEFAULT_TRAFFIC_CLASS; 5163 #ifdef I40E_FCOE 5164 if (pf->vsi[v]->type == I40E_VSI_FCOE) 5165 tc_map = i40e_get_fcoe_tc_map(pf); 5166 #endif /* #ifdef I40E_FCOE */ 5167 5168 ret = i40e_vsi_config_tc(pf->vsi[v], tc_map); 5169 if (ret) { 5170 dev_info(&pf->pdev->dev, 5171 "Failed configuring TC for VSI seid=%d\n", 5172 pf->vsi[v]->seid); 5173 /* Will try to configure as many components */ 5174 } else { 5175 /* Re-configure VSI vectors based on updated TC map */ 5176 i40e_vsi_map_rings_to_vectors(pf->vsi[v]); 5177 if (pf->vsi[v]->netdev) 5178 i40e_dcbnl_set_all(pf->vsi[v]); 5179 } 5180 } 5181 } 5182 5183 /** 5184 * i40e_resume_port_tx - Resume port Tx 5185 * @pf: PF struct 5186 * 5187 * Resume a port's Tx and issue a PF reset in case of failure to 5188 * resume. 5189 **/ 5190 static int i40e_resume_port_tx(struct i40e_pf *pf) 5191 { 5192 struct i40e_hw *hw = &pf->hw; 5193 int ret; 5194 5195 ret = i40e_aq_resume_port_tx(hw, NULL); 5196 if (ret) { 5197 dev_info(&pf->pdev->dev, 5198 "Resume Port Tx failed, err %s aq_err %s\n", 5199 i40e_stat_str(&pf->hw, ret), 5200 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 5201 /* Schedule PF reset to recover */ 5202 set_bit(__I40E_PF_RESET_REQUESTED, &pf->state); 5203 i40e_service_event_schedule(pf); 5204 } 5205 5206 return ret; 5207 } 5208 5209 /** 5210 * i40e_init_pf_dcb - Initialize DCB configuration 5211 * @pf: PF being configured 5212 * 5213 * Query the current DCB configuration and cache it 5214 * in the hardware structure 5215 **/ 5216 static int i40e_init_pf_dcb(struct i40e_pf *pf) 5217 { 5218 struct i40e_hw *hw = &pf->hw; 5219 int err = 0; 5220 5221 /* Do not enable DCB for SW1 and SW2 images even if the FW is capable */ 5222 if (pf->flags & I40E_FLAG_NO_DCB_SUPPORT) 5223 goto out; 5224 5225 /* Get the initial DCB configuration */ 5226 err = i40e_init_dcb(hw); 5227 if (!err) { 5228 /* Device/Function is not DCBX capable */ 5229 if ((!hw->func_caps.dcb) || 5230 (hw->dcbx_status == I40E_DCBX_STATUS_DISABLED)) { 5231 dev_info(&pf->pdev->dev, 5232 "DCBX offload is not supported or is disabled for this PF.\n"); 5233 5234 if (pf->flags & I40E_FLAG_MFP_ENABLED) 5235 goto out; 5236 5237 } else { 5238 /* When status is not DISABLED then DCBX in FW */ 5239 pf->dcbx_cap = DCB_CAP_DCBX_LLD_MANAGED | 5240 DCB_CAP_DCBX_VER_IEEE; 5241 5242 pf->flags |= I40E_FLAG_DCB_CAPABLE; 5243 /* Enable DCB tagging only when more than one TC 5244 * or explicitly disable if only one TC 5245 */ 5246 if (i40e_dcb_get_num_tc(&hw->local_dcbx_config) > 1) 5247 pf->flags |= I40E_FLAG_DCB_ENABLED; 5248 else 5249 pf->flags &= ~I40E_FLAG_DCB_ENABLED; 5250 dev_dbg(&pf->pdev->dev, 5251 "DCBX offload is supported for this PF.\n"); 5252 } 5253 } else { 5254 dev_info(&pf->pdev->dev, 5255 "Query for DCB configuration failed, err %s aq_err %s\n", 5256 i40e_stat_str(&pf->hw, err), 5257 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 5258 } 5259 5260 out: 5261 return err; 5262 } 5263 #endif /* CONFIG_I40E_DCB */ 5264 #define SPEED_SIZE 14 5265 #define FC_SIZE 8 5266 /** 5267 * i40e_print_link_message - print link up or down 5268 * @vsi: the VSI for which link needs a message 5269 */ 5270 void i40e_print_link_message(struct i40e_vsi *vsi, bool isup) 5271 { 5272 enum i40e_aq_link_speed new_speed; 5273 char *speed = "Unknown"; 5274 char *fc = "Unknown"; 5275 5276 new_speed = vsi->back->hw.phy.link_info.link_speed; 5277 5278 if ((vsi->current_isup == isup) && (vsi->current_speed == new_speed)) 5279 return; 5280 vsi->current_isup = isup; 5281 vsi->current_speed = new_speed; 5282 if (!isup) { 5283 netdev_info(vsi->netdev, "NIC Link is Down\n"); 5284 return; 5285 } 5286 5287 /* Warn user if link speed on NPAR enabled partition is not at 5288 * least 10GB 5289 */ 5290 if (vsi->back->hw.func_caps.npar_enable && 5291 (vsi->back->hw.phy.link_info.link_speed == I40E_LINK_SPEED_1GB || 5292 vsi->back->hw.phy.link_info.link_speed == I40E_LINK_SPEED_100MB)) 5293 netdev_warn(vsi->netdev, 5294 "The partition detected link speed that is less than 10Gbps\n"); 5295 5296 switch (vsi->back->hw.phy.link_info.link_speed) { 5297 case I40E_LINK_SPEED_40GB: 5298 speed = "40 G"; 5299 break; 5300 case I40E_LINK_SPEED_20GB: 5301 speed = "20 G"; 5302 break; 5303 case I40E_LINK_SPEED_25GB: 5304 speed = "25 G"; 5305 break; 5306 case I40E_LINK_SPEED_10GB: 5307 speed = "10 G"; 5308 break; 5309 case I40E_LINK_SPEED_1GB: 5310 speed = "1000 M"; 5311 break; 5312 case I40E_LINK_SPEED_100MB: 5313 speed = "100 M"; 5314 break; 5315 default: 5316 break; 5317 } 5318 5319 switch (vsi->back->hw.fc.current_mode) { 5320 case I40E_FC_FULL: 5321 fc = "RX/TX"; 5322 break; 5323 case I40E_FC_TX_PAUSE: 5324 fc = "TX"; 5325 break; 5326 case I40E_FC_RX_PAUSE: 5327 fc = "RX"; 5328 break; 5329 default: 5330 fc = "None"; 5331 break; 5332 } 5333 5334 netdev_info(vsi->netdev, "NIC Link is Up %sbps Full Duplex, Flow Control: %s\n", 5335 speed, fc); 5336 } 5337 5338 /** 5339 * i40e_up_complete - Finish the last steps of bringing up a connection 5340 * @vsi: the VSI being configured 5341 **/ 5342 static int i40e_up_complete(struct i40e_vsi *vsi) 5343 { 5344 struct i40e_pf *pf = vsi->back; 5345 int err; 5346 5347 if (pf->flags & I40E_FLAG_MSIX_ENABLED) 5348 i40e_vsi_configure_msix(vsi); 5349 else 5350 i40e_configure_msi_and_legacy(vsi); 5351 5352 /* start rings */ 5353 err = i40e_vsi_start_rings(vsi); 5354 if (err) 5355 return err; 5356 5357 clear_bit(__I40E_DOWN, &vsi->state); 5358 i40e_napi_enable_all(vsi); 5359 i40e_vsi_enable_irq(vsi); 5360 5361 if ((pf->hw.phy.link_info.link_info & I40E_AQ_LINK_UP) && 5362 (vsi->netdev)) { 5363 i40e_print_link_message(vsi, true); 5364 netif_tx_start_all_queues(vsi->netdev); 5365 netif_carrier_on(vsi->netdev); 5366 } else if (vsi->netdev) { 5367 i40e_print_link_message(vsi, false); 5368 /* need to check for qualified module here*/ 5369 if ((pf->hw.phy.link_info.link_info & 5370 I40E_AQ_MEDIA_AVAILABLE) && 5371 (!(pf->hw.phy.link_info.an_info & 5372 I40E_AQ_QUALIFIED_MODULE))) 5373 netdev_err(vsi->netdev, 5374 "the driver failed to link because an unqualified module was detected."); 5375 } 5376 5377 /* replay FDIR SB filters */ 5378 if (vsi->type == I40E_VSI_FDIR) { 5379 /* reset fd counters */ 5380 pf->fd_add_err = pf->fd_atr_cnt = 0; 5381 if (pf->fd_tcp_rule > 0) { 5382 pf->auto_disable_flags |= I40E_FLAG_FD_ATR_ENABLED; 5383 if (I40E_DEBUG_FD & pf->hw.debug_mask) 5384 dev_info(&pf->pdev->dev, "Forcing ATR off, sideband rules for TCP/IPv4 exist\n"); 5385 pf->fd_tcp_rule = 0; 5386 } 5387 i40e_fdir_filter_restore(vsi); 5388 } 5389 5390 /* On the next run of the service_task, notify any clients of the new 5391 * opened netdev 5392 */ 5393 pf->flags |= I40E_FLAG_SERVICE_CLIENT_REQUESTED; 5394 i40e_service_event_schedule(pf); 5395 5396 return 0; 5397 } 5398 5399 /** 5400 * i40e_vsi_reinit_locked - Reset the VSI 5401 * @vsi: the VSI being configured 5402 * 5403 * Rebuild the ring structs after some configuration 5404 * has changed, e.g. MTU size. 5405 **/ 5406 static void i40e_vsi_reinit_locked(struct i40e_vsi *vsi) 5407 { 5408 struct i40e_pf *pf = vsi->back; 5409 5410 WARN_ON(in_interrupt()); 5411 while (test_and_set_bit(__I40E_CONFIG_BUSY, &pf->state)) 5412 usleep_range(1000, 2000); 5413 i40e_down(vsi); 5414 5415 i40e_up(vsi); 5416 clear_bit(__I40E_CONFIG_BUSY, &pf->state); 5417 } 5418 5419 /** 5420 * i40e_up - Bring the connection back up after being down 5421 * @vsi: the VSI being configured 5422 **/ 5423 int i40e_up(struct i40e_vsi *vsi) 5424 { 5425 int err; 5426 5427 err = i40e_vsi_configure(vsi); 5428 if (!err) 5429 err = i40e_up_complete(vsi); 5430 5431 return err; 5432 } 5433 5434 /** 5435 * i40e_down - Shutdown the connection processing 5436 * @vsi: the VSI being stopped 5437 **/ 5438 void i40e_down(struct i40e_vsi *vsi) 5439 { 5440 int i; 5441 5442 /* It is assumed that the caller of this function 5443 * sets the vsi->state __I40E_DOWN bit. 5444 */ 5445 if (vsi->netdev) { 5446 netif_carrier_off(vsi->netdev); 5447 netif_tx_disable(vsi->netdev); 5448 } 5449 i40e_vsi_disable_irq(vsi); 5450 i40e_vsi_stop_rings(vsi); 5451 i40e_napi_disable_all(vsi); 5452 5453 for (i = 0; i < vsi->num_queue_pairs; i++) { 5454 i40e_clean_tx_ring(vsi->tx_rings[i]); 5455 i40e_clean_rx_ring(vsi->rx_rings[i]); 5456 } 5457 5458 i40e_notify_client_of_netdev_close(vsi, false); 5459 5460 } 5461 5462 /** 5463 * i40e_setup_tc - configure multiple traffic classes 5464 * @netdev: net device to configure 5465 * @tc: number of traffic classes to enable 5466 **/ 5467 static int i40e_setup_tc(struct net_device *netdev, u8 tc) 5468 { 5469 struct i40e_netdev_priv *np = netdev_priv(netdev); 5470 struct i40e_vsi *vsi = np->vsi; 5471 struct i40e_pf *pf = vsi->back; 5472 u8 enabled_tc = 0; 5473 int ret = -EINVAL; 5474 int i; 5475 5476 /* Check if DCB enabled to continue */ 5477 if (!(pf->flags & I40E_FLAG_DCB_ENABLED)) { 5478 netdev_info(netdev, "DCB is not enabled for adapter\n"); 5479 goto exit; 5480 } 5481 5482 /* Check if MFP enabled */ 5483 if (pf->flags & I40E_FLAG_MFP_ENABLED) { 5484 netdev_info(netdev, "Configuring TC not supported in MFP mode\n"); 5485 goto exit; 5486 } 5487 5488 /* Check whether tc count is within enabled limit */ 5489 if (tc > i40e_pf_get_num_tc(pf)) { 5490 netdev_info(netdev, "TC count greater than enabled on link for adapter\n"); 5491 goto exit; 5492 } 5493 5494 /* Generate TC map for number of tc requested */ 5495 for (i = 0; i < tc; i++) 5496 enabled_tc |= BIT(i); 5497 5498 /* Requesting same TC configuration as already enabled */ 5499 if (enabled_tc == vsi->tc_config.enabled_tc) 5500 return 0; 5501 5502 /* Quiesce VSI queues */ 5503 i40e_quiesce_vsi(vsi); 5504 5505 /* Configure VSI for enabled TCs */ 5506 ret = i40e_vsi_config_tc(vsi, enabled_tc); 5507 if (ret) { 5508 netdev_info(netdev, "Failed configuring TC for VSI seid=%d\n", 5509 vsi->seid); 5510 goto exit; 5511 } 5512 5513 /* Unquiesce VSI */ 5514 i40e_unquiesce_vsi(vsi); 5515 5516 exit: 5517 return ret; 5518 } 5519 5520 #ifdef I40E_FCOE 5521 int __i40e_setup_tc(struct net_device *netdev, u32 handle, __be16 proto, 5522 struct tc_to_netdev *tc) 5523 #else 5524 static int __i40e_setup_tc(struct net_device *netdev, u32 handle, __be16 proto, 5525 struct tc_to_netdev *tc) 5526 #endif 5527 { 5528 if (handle != TC_H_ROOT || tc->type != TC_SETUP_MQPRIO) 5529 return -EINVAL; 5530 return i40e_setup_tc(netdev, tc->tc); 5531 } 5532 5533 /** 5534 * i40e_open - Called when a network interface is made active 5535 * @netdev: network interface device structure 5536 * 5537 * The open entry point is called when a network interface is made 5538 * active by the system (IFF_UP). At this point all resources needed 5539 * for transmit and receive operations are allocated, the interrupt 5540 * handler is registered with the OS, the netdev watchdog subtask is 5541 * enabled, and the stack is notified that the interface is ready. 5542 * 5543 * Returns 0 on success, negative value on failure 5544 **/ 5545 int i40e_open(struct net_device *netdev) 5546 { 5547 struct i40e_netdev_priv *np = netdev_priv(netdev); 5548 struct i40e_vsi *vsi = np->vsi; 5549 struct i40e_pf *pf = vsi->back; 5550 int err; 5551 5552 /* disallow open during test or if eeprom is broken */ 5553 if (test_bit(__I40E_TESTING, &pf->state) || 5554 test_bit(__I40E_BAD_EEPROM, &pf->state)) 5555 return -EBUSY; 5556 5557 netif_carrier_off(netdev); 5558 5559 err = i40e_vsi_open(vsi); 5560 if (err) 5561 return err; 5562 5563 /* configure global TSO hardware offload settings */ 5564 wr32(&pf->hw, I40E_GLLAN_TSOMSK_F, be32_to_cpu(TCP_FLAG_PSH | 5565 TCP_FLAG_FIN) >> 16); 5566 wr32(&pf->hw, I40E_GLLAN_TSOMSK_M, be32_to_cpu(TCP_FLAG_PSH | 5567 TCP_FLAG_FIN | 5568 TCP_FLAG_CWR) >> 16); 5569 wr32(&pf->hw, I40E_GLLAN_TSOMSK_L, be32_to_cpu(TCP_FLAG_CWR) >> 16); 5570 5571 udp_tunnel_get_rx_info(netdev); 5572 5573 return 0; 5574 } 5575 5576 /** 5577 * i40e_vsi_open - 5578 * @vsi: the VSI to open 5579 * 5580 * Finish initialization of the VSI. 5581 * 5582 * Returns 0 on success, negative value on failure 5583 **/ 5584 int i40e_vsi_open(struct i40e_vsi *vsi) 5585 { 5586 struct i40e_pf *pf = vsi->back; 5587 char int_name[I40E_INT_NAME_STR_LEN]; 5588 int err; 5589 5590 /* allocate descriptors */ 5591 err = i40e_vsi_setup_tx_resources(vsi); 5592 if (err) 5593 goto err_setup_tx; 5594 err = i40e_vsi_setup_rx_resources(vsi); 5595 if (err) 5596 goto err_setup_rx; 5597 5598 err = i40e_vsi_configure(vsi); 5599 if (err) 5600 goto err_setup_rx; 5601 5602 if (vsi->netdev) { 5603 snprintf(int_name, sizeof(int_name) - 1, "%s-%s", 5604 dev_driver_string(&pf->pdev->dev), vsi->netdev->name); 5605 err = i40e_vsi_request_irq(vsi, int_name); 5606 if (err) 5607 goto err_setup_rx; 5608 5609 /* Notify the stack of the actual queue counts. */ 5610 err = netif_set_real_num_tx_queues(vsi->netdev, 5611 vsi->num_queue_pairs); 5612 if (err) 5613 goto err_set_queues; 5614 5615 err = netif_set_real_num_rx_queues(vsi->netdev, 5616 vsi->num_queue_pairs); 5617 if (err) 5618 goto err_set_queues; 5619 5620 } else if (vsi->type == I40E_VSI_FDIR) { 5621 snprintf(int_name, sizeof(int_name) - 1, "%s-%s:fdir", 5622 dev_driver_string(&pf->pdev->dev), 5623 dev_name(&pf->pdev->dev)); 5624 err = i40e_vsi_request_irq(vsi, int_name); 5625 5626 } else { 5627 err = -EINVAL; 5628 goto err_setup_rx; 5629 } 5630 5631 err = i40e_up_complete(vsi); 5632 if (err) 5633 goto err_up_complete; 5634 5635 return 0; 5636 5637 err_up_complete: 5638 i40e_down(vsi); 5639 err_set_queues: 5640 i40e_vsi_free_irq(vsi); 5641 err_setup_rx: 5642 i40e_vsi_free_rx_resources(vsi); 5643 err_setup_tx: 5644 i40e_vsi_free_tx_resources(vsi); 5645 if (vsi == pf->vsi[pf->lan_vsi]) 5646 i40e_do_reset(pf, BIT_ULL(__I40E_PF_RESET_REQUESTED)); 5647 5648 return err; 5649 } 5650 5651 /** 5652 * i40e_fdir_filter_exit - Cleans up the Flow Director accounting 5653 * @pf: Pointer to PF 5654 * 5655 * This function destroys the hlist where all the Flow Director 5656 * filters were saved. 5657 **/ 5658 static void i40e_fdir_filter_exit(struct i40e_pf *pf) 5659 { 5660 struct i40e_fdir_filter *filter; 5661 struct hlist_node *node2; 5662 5663 hlist_for_each_entry_safe(filter, node2, 5664 &pf->fdir_filter_list, fdir_node) { 5665 hlist_del(&filter->fdir_node); 5666 kfree(filter); 5667 } 5668 pf->fdir_pf_active_filters = 0; 5669 } 5670 5671 /** 5672 * i40e_close - Disables a network interface 5673 * @netdev: network interface device structure 5674 * 5675 * The close entry point is called when an interface is de-activated 5676 * by the OS. The hardware is still under the driver's control, but 5677 * this netdev interface is disabled. 5678 * 5679 * Returns 0, this is not allowed to fail 5680 **/ 5681 int i40e_close(struct net_device *netdev) 5682 { 5683 struct i40e_netdev_priv *np = netdev_priv(netdev); 5684 struct i40e_vsi *vsi = np->vsi; 5685 5686 i40e_vsi_close(vsi); 5687 5688 return 0; 5689 } 5690 5691 /** 5692 * i40e_do_reset - Start a PF or Core Reset sequence 5693 * @pf: board private structure 5694 * @reset_flags: which reset is requested 5695 * 5696 * The essential difference in resets is that the PF Reset 5697 * doesn't clear the packet buffers, doesn't reset the PE 5698 * firmware, and doesn't bother the other PFs on the chip. 5699 **/ 5700 void i40e_do_reset(struct i40e_pf *pf, u32 reset_flags) 5701 { 5702 u32 val; 5703 5704 WARN_ON(in_interrupt()); 5705 5706 5707 /* do the biggest reset indicated */ 5708 if (reset_flags & BIT_ULL(__I40E_GLOBAL_RESET_REQUESTED)) { 5709 5710 /* Request a Global Reset 5711 * 5712 * This will start the chip's countdown to the actual full 5713 * chip reset event, and a warning interrupt to be sent 5714 * to all PFs, including the requestor. Our handler 5715 * for the warning interrupt will deal with the shutdown 5716 * and recovery of the switch setup. 5717 */ 5718 dev_dbg(&pf->pdev->dev, "GlobalR requested\n"); 5719 val = rd32(&pf->hw, I40E_GLGEN_RTRIG); 5720 val |= I40E_GLGEN_RTRIG_GLOBR_MASK; 5721 wr32(&pf->hw, I40E_GLGEN_RTRIG, val); 5722 5723 } else if (reset_flags & BIT_ULL(__I40E_CORE_RESET_REQUESTED)) { 5724 5725 /* Request a Core Reset 5726 * 5727 * Same as Global Reset, except does *not* include the MAC/PHY 5728 */ 5729 dev_dbg(&pf->pdev->dev, "CoreR requested\n"); 5730 val = rd32(&pf->hw, I40E_GLGEN_RTRIG); 5731 val |= I40E_GLGEN_RTRIG_CORER_MASK; 5732 wr32(&pf->hw, I40E_GLGEN_RTRIG, val); 5733 i40e_flush(&pf->hw); 5734 5735 } else if (reset_flags & BIT_ULL(__I40E_PF_RESET_REQUESTED)) { 5736 5737 /* Request a PF Reset 5738 * 5739 * Resets only the PF-specific registers 5740 * 5741 * This goes directly to the tear-down and rebuild of 5742 * the switch, since we need to do all the recovery as 5743 * for the Core Reset. 5744 */ 5745 dev_dbg(&pf->pdev->dev, "PFR requested\n"); 5746 i40e_handle_reset_warning(pf); 5747 5748 } else if (reset_flags & BIT_ULL(__I40E_REINIT_REQUESTED)) { 5749 int v; 5750 5751 /* Find the VSI(s) that requested a re-init */ 5752 dev_info(&pf->pdev->dev, 5753 "VSI reinit requested\n"); 5754 for (v = 0; v < pf->num_alloc_vsi; v++) { 5755 struct i40e_vsi *vsi = pf->vsi[v]; 5756 5757 if (vsi != NULL && 5758 test_bit(__I40E_REINIT_REQUESTED, &vsi->state)) { 5759 i40e_vsi_reinit_locked(pf->vsi[v]); 5760 clear_bit(__I40E_REINIT_REQUESTED, &vsi->state); 5761 } 5762 } 5763 } else if (reset_flags & BIT_ULL(__I40E_DOWN_REQUESTED)) { 5764 int v; 5765 5766 /* Find the VSI(s) that needs to be brought down */ 5767 dev_info(&pf->pdev->dev, "VSI down requested\n"); 5768 for (v = 0; v < pf->num_alloc_vsi; v++) { 5769 struct i40e_vsi *vsi = pf->vsi[v]; 5770 5771 if (vsi != NULL && 5772 test_bit(__I40E_DOWN_REQUESTED, &vsi->state)) { 5773 set_bit(__I40E_DOWN, &vsi->state); 5774 i40e_down(vsi); 5775 clear_bit(__I40E_DOWN_REQUESTED, &vsi->state); 5776 } 5777 } 5778 } else { 5779 dev_info(&pf->pdev->dev, 5780 "bad reset request 0x%08x\n", reset_flags); 5781 } 5782 } 5783 5784 #ifdef CONFIG_I40E_DCB 5785 /** 5786 * i40e_dcb_need_reconfig - Check if DCB needs reconfig 5787 * @pf: board private structure 5788 * @old_cfg: current DCB config 5789 * @new_cfg: new DCB config 5790 **/ 5791 bool i40e_dcb_need_reconfig(struct i40e_pf *pf, 5792 struct i40e_dcbx_config *old_cfg, 5793 struct i40e_dcbx_config *new_cfg) 5794 { 5795 bool need_reconfig = false; 5796 5797 /* Check if ETS configuration has changed */ 5798 if (memcmp(&new_cfg->etscfg, 5799 &old_cfg->etscfg, 5800 sizeof(new_cfg->etscfg))) { 5801 /* If Priority Table has changed reconfig is needed */ 5802 if (memcmp(&new_cfg->etscfg.prioritytable, 5803 &old_cfg->etscfg.prioritytable, 5804 sizeof(new_cfg->etscfg.prioritytable))) { 5805 need_reconfig = true; 5806 dev_dbg(&pf->pdev->dev, "ETS UP2TC changed.\n"); 5807 } 5808 5809 if (memcmp(&new_cfg->etscfg.tcbwtable, 5810 &old_cfg->etscfg.tcbwtable, 5811 sizeof(new_cfg->etscfg.tcbwtable))) 5812 dev_dbg(&pf->pdev->dev, "ETS TC BW Table changed.\n"); 5813 5814 if (memcmp(&new_cfg->etscfg.tsatable, 5815 &old_cfg->etscfg.tsatable, 5816 sizeof(new_cfg->etscfg.tsatable))) 5817 dev_dbg(&pf->pdev->dev, "ETS TSA Table changed.\n"); 5818 } 5819 5820 /* Check if PFC configuration has changed */ 5821 if (memcmp(&new_cfg->pfc, 5822 &old_cfg->pfc, 5823 sizeof(new_cfg->pfc))) { 5824 need_reconfig = true; 5825 dev_dbg(&pf->pdev->dev, "PFC config change detected.\n"); 5826 } 5827 5828 /* Check if APP Table has changed */ 5829 if (memcmp(&new_cfg->app, 5830 &old_cfg->app, 5831 sizeof(new_cfg->app))) { 5832 need_reconfig = true; 5833 dev_dbg(&pf->pdev->dev, "APP Table change detected.\n"); 5834 } 5835 5836 dev_dbg(&pf->pdev->dev, "dcb need_reconfig=%d\n", need_reconfig); 5837 return need_reconfig; 5838 } 5839 5840 /** 5841 * i40e_handle_lldp_event - Handle LLDP Change MIB event 5842 * @pf: board private structure 5843 * @e: event info posted on ARQ 5844 **/ 5845 static int i40e_handle_lldp_event(struct i40e_pf *pf, 5846 struct i40e_arq_event_info *e) 5847 { 5848 struct i40e_aqc_lldp_get_mib *mib = 5849 (struct i40e_aqc_lldp_get_mib *)&e->desc.params.raw; 5850 struct i40e_hw *hw = &pf->hw; 5851 struct i40e_dcbx_config tmp_dcbx_cfg; 5852 bool need_reconfig = false; 5853 int ret = 0; 5854 u8 type; 5855 5856 /* Not DCB capable or capability disabled */ 5857 if (!(pf->flags & I40E_FLAG_DCB_CAPABLE)) 5858 return ret; 5859 5860 /* Ignore if event is not for Nearest Bridge */ 5861 type = ((mib->type >> I40E_AQ_LLDP_BRIDGE_TYPE_SHIFT) 5862 & I40E_AQ_LLDP_BRIDGE_TYPE_MASK); 5863 dev_dbg(&pf->pdev->dev, "LLDP event mib bridge type 0x%x\n", type); 5864 if (type != I40E_AQ_LLDP_BRIDGE_TYPE_NEAREST_BRIDGE) 5865 return ret; 5866 5867 /* Check MIB Type and return if event for Remote MIB update */ 5868 type = mib->type & I40E_AQ_LLDP_MIB_TYPE_MASK; 5869 dev_dbg(&pf->pdev->dev, 5870 "LLDP event mib type %s\n", type ? "remote" : "local"); 5871 if (type == I40E_AQ_LLDP_MIB_REMOTE) { 5872 /* Update the remote cached instance and return */ 5873 ret = i40e_aq_get_dcb_config(hw, I40E_AQ_LLDP_MIB_REMOTE, 5874 I40E_AQ_LLDP_BRIDGE_TYPE_NEAREST_BRIDGE, 5875 &hw->remote_dcbx_config); 5876 goto exit; 5877 } 5878 5879 /* Store the old configuration */ 5880 tmp_dcbx_cfg = hw->local_dcbx_config; 5881 5882 /* Reset the old DCBx configuration data */ 5883 memset(&hw->local_dcbx_config, 0, sizeof(hw->local_dcbx_config)); 5884 /* Get updated DCBX data from firmware */ 5885 ret = i40e_get_dcb_config(&pf->hw); 5886 if (ret) { 5887 dev_info(&pf->pdev->dev, 5888 "Failed querying DCB configuration data from firmware, err %s aq_err %s\n", 5889 i40e_stat_str(&pf->hw, ret), 5890 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 5891 goto exit; 5892 } 5893 5894 /* No change detected in DCBX configs */ 5895 if (!memcmp(&tmp_dcbx_cfg, &hw->local_dcbx_config, 5896 sizeof(tmp_dcbx_cfg))) { 5897 dev_dbg(&pf->pdev->dev, "No change detected in DCBX configuration.\n"); 5898 goto exit; 5899 } 5900 5901 need_reconfig = i40e_dcb_need_reconfig(pf, &tmp_dcbx_cfg, 5902 &hw->local_dcbx_config); 5903 5904 i40e_dcbnl_flush_apps(pf, &tmp_dcbx_cfg, &hw->local_dcbx_config); 5905 5906 if (!need_reconfig) 5907 goto exit; 5908 5909 /* Enable DCB tagging only when more than one TC */ 5910 if (i40e_dcb_get_num_tc(&hw->local_dcbx_config) > 1) 5911 pf->flags |= I40E_FLAG_DCB_ENABLED; 5912 else 5913 pf->flags &= ~I40E_FLAG_DCB_ENABLED; 5914 5915 set_bit(__I40E_PORT_TX_SUSPENDED, &pf->state); 5916 /* Reconfiguration needed quiesce all VSIs */ 5917 i40e_pf_quiesce_all_vsi(pf); 5918 5919 /* Changes in configuration update VEB/VSI */ 5920 i40e_dcb_reconfigure(pf); 5921 5922 ret = i40e_resume_port_tx(pf); 5923 5924 clear_bit(__I40E_PORT_TX_SUSPENDED, &pf->state); 5925 /* In case of error no point in resuming VSIs */ 5926 if (ret) 5927 goto exit; 5928 5929 /* Wait for the PF's queues to be disabled */ 5930 ret = i40e_pf_wait_queues_disabled(pf); 5931 if (ret) { 5932 /* Schedule PF reset to recover */ 5933 set_bit(__I40E_PF_RESET_REQUESTED, &pf->state); 5934 i40e_service_event_schedule(pf); 5935 } else { 5936 i40e_pf_unquiesce_all_vsi(pf); 5937 /* Notify the client for the DCB changes */ 5938 i40e_notify_client_of_l2_param_changes(pf->vsi[pf->lan_vsi]); 5939 } 5940 5941 exit: 5942 return ret; 5943 } 5944 #endif /* CONFIG_I40E_DCB */ 5945 5946 /** 5947 * i40e_do_reset_safe - Protected reset path for userland calls. 5948 * @pf: board private structure 5949 * @reset_flags: which reset is requested 5950 * 5951 **/ 5952 void i40e_do_reset_safe(struct i40e_pf *pf, u32 reset_flags) 5953 { 5954 rtnl_lock(); 5955 i40e_do_reset(pf, reset_flags); 5956 rtnl_unlock(); 5957 } 5958 5959 /** 5960 * i40e_handle_lan_overflow_event - Handler for LAN queue overflow event 5961 * @pf: board private structure 5962 * @e: event info posted on ARQ 5963 * 5964 * Handler for LAN Queue Overflow Event generated by the firmware for PF 5965 * and VF queues 5966 **/ 5967 static void i40e_handle_lan_overflow_event(struct i40e_pf *pf, 5968 struct i40e_arq_event_info *e) 5969 { 5970 struct i40e_aqc_lan_overflow *data = 5971 (struct i40e_aqc_lan_overflow *)&e->desc.params.raw; 5972 u32 queue = le32_to_cpu(data->prtdcb_rupto); 5973 u32 qtx_ctl = le32_to_cpu(data->otx_ctl); 5974 struct i40e_hw *hw = &pf->hw; 5975 struct i40e_vf *vf; 5976 u16 vf_id; 5977 5978 dev_dbg(&pf->pdev->dev, "overflow Rx Queue Number = %d QTX_CTL=0x%08x\n", 5979 queue, qtx_ctl); 5980 5981 /* Queue belongs to VF, find the VF and issue VF reset */ 5982 if (((qtx_ctl & I40E_QTX_CTL_PFVF_Q_MASK) 5983 >> I40E_QTX_CTL_PFVF_Q_SHIFT) == I40E_QTX_CTL_VF_QUEUE) { 5984 vf_id = (u16)((qtx_ctl & I40E_QTX_CTL_VFVM_INDX_MASK) 5985 >> I40E_QTX_CTL_VFVM_INDX_SHIFT); 5986 vf_id -= hw->func_caps.vf_base_id; 5987 vf = &pf->vf[vf_id]; 5988 i40e_vc_notify_vf_reset(vf); 5989 /* Allow VF to process pending reset notification */ 5990 msleep(20); 5991 i40e_reset_vf(vf, false); 5992 } 5993 } 5994 5995 /** 5996 * i40e_get_cur_guaranteed_fd_count - Get the consumed guaranteed FD filters 5997 * @pf: board private structure 5998 **/ 5999 u32 i40e_get_cur_guaranteed_fd_count(struct i40e_pf *pf) 6000 { 6001 u32 val, fcnt_prog; 6002 6003 val = rd32(&pf->hw, I40E_PFQF_FDSTAT); 6004 fcnt_prog = (val & I40E_PFQF_FDSTAT_GUARANT_CNT_MASK); 6005 return fcnt_prog; 6006 } 6007 6008 /** 6009 * i40e_get_current_fd_count - Get total FD filters programmed for this PF 6010 * @pf: board private structure 6011 **/ 6012 u32 i40e_get_current_fd_count(struct i40e_pf *pf) 6013 { 6014 u32 val, fcnt_prog; 6015 6016 val = rd32(&pf->hw, I40E_PFQF_FDSTAT); 6017 fcnt_prog = (val & I40E_PFQF_FDSTAT_GUARANT_CNT_MASK) + 6018 ((val & I40E_PFQF_FDSTAT_BEST_CNT_MASK) >> 6019 I40E_PFQF_FDSTAT_BEST_CNT_SHIFT); 6020 return fcnt_prog; 6021 } 6022 6023 /** 6024 * i40e_get_global_fd_count - Get total FD filters programmed on device 6025 * @pf: board private structure 6026 **/ 6027 u32 i40e_get_global_fd_count(struct i40e_pf *pf) 6028 { 6029 u32 val, fcnt_prog; 6030 6031 val = rd32(&pf->hw, I40E_GLQF_FDCNT_0); 6032 fcnt_prog = (val & I40E_GLQF_FDCNT_0_GUARANT_CNT_MASK) + 6033 ((val & I40E_GLQF_FDCNT_0_BESTCNT_MASK) >> 6034 I40E_GLQF_FDCNT_0_BESTCNT_SHIFT); 6035 return fcnt_prog; 6036 } 6037 6038 /** 6039 * i40e_fdir_check_and_reenable - Function to reenabe FD ATR or SB if disabled 6040 * @pf: board private structure 6041 **/ 6042 void i40e_fdir_check_and_reenable(struct i40e_pf *pf) 6043 { 6044 struct i40e_fdir_filter *filter; 6045 u32 fcnt_prog, fcnt_avail; 6046 struct hlist_node *node; 6047 6048 if (test_bit(__I40E_FD_FLUSH_REQUESTED, &pf->state)) 6049 return; 6050 6051 /* Check if, FD SB or ATR was auto disabled and if there is enough room 6052 * to re-enable 6053 */ 6054 fcnt_prog = i40e_get_global_fd_count(pf); 6055 fcnt_avail = pf->fdir_pf_filter_count; 6056 if ((fcnt_prog < (fcnt_avail - I40E_FDIR_BUFFER_HEAD_ROOM)) || 6057 (pf->fd_add_err == 0) || 6058 (i40e_get_current_atr_cnt(pf) < pf->fd_atr_cnt)) { 6059 if ((pf->flags & I40E_FLAG_FD_SB_ENABLED) && 6060 (pf->auto_disable_flags & I40E_FLAG_FD_SB_ENABLED)) { 6061 pf->auto_disable_flags &= ~I40E_FLAG_FD_SB_ENABLED; 6062 if (I40E_DEBUG_FD & pf->hw.debug_mask) 6063 dev_info(&pf->pdev->dev, "FD Sideband/ntuple is being enabled since we have space in the table now\n"); 6064 } 6065 } 6066 6067 /* Wait for some more space to be available to turn on ATR. We also 6068 * must check that no existing ntuple rules for TCP are in effect 6069 */ 6070 if (fcnt_prog < (fcnt_avail - I40E_FDIR_BUFFER_HEAD_ROOM * 2)) { 6071 if ((pf->flags & I40E_FLAG_FD_ATR_ENABLED) && 6072 (pf->auto_disable_flags & I40E_FLAG_FD_ATR_ENABLED) && 6073 (pf->fd_tcp_rule == 0)) { 6074 pf->auto_disable_flags &= ~I40E_FLAG_FD_ATR_ENABLED; 6075 if (I40E_DEBUG_FD & pf->hw.debug_mask) 6076 dev_info(&pf->pdev->dev, "ATR is being enabled since we have space in the table and there are no conflicting ntuple rules\n"); 6077 } 6078 } 6079 6080 /* if hw had a problem adding a filter, delete it */ 6081 if (pf->fd_inv > 0) { 6082 hlist_for_each_entry_safe(filter, node, 6083 &pf->fdir_filter_list, fdir_node) { 6084 if (filter->fd_id == pf->fd_inv) { 6085 hlist_del(&filter->fdir_node); 6086 kfree(filter); 6087 pf->fdir_pf_active_filters--; 6088 } 6089 } 6090 } 6091 } 6092 6093 #define I40E_MIN_FD_FLUSH_INTERVAL 10 6094 #define I40E_MIN_FD_FLUSH_SB_ATR_UNSTABLE 30 6095 /** 6096 * i40e_fdir_flush_and_replay - Function to flush all FD filters and replay SB 6097 * @pf: board private structure 6098 **/ 6099 static void i40e_fdir_flush_and_replay(struct i40e_pf *pf) 6100 { 6101 unsigned long min_flush_time; 6102 int flush_wait_retry = 50; 6103 bool disable_atr = false; 6104 int fd_room; 6105 int reg; 6106 6107 if (!time_after(jiffies, pf->fd_flush_timestamp + 6108 (I40E_MIN_FD_FLUSH_INTERVAL * HZ))) 6109 return; 6110 6111 /* If the flush is happening too quick and we have mostly SB rules we 6112 * should not re-enable ATR for some time. 6113 */ 6114 min_flush_time = pf->fd_flush_timestamp + 6115 (I40E_MIN_FD_FLUSH_SB_ATR_UNSTABLE * HZ); 6116 fd_room = pf->fdir_pf_filter_count - pf->fdir_pf_active_filters; 6117 6118 if (!(time_after(jiffies, min_flush_time)) && 6119 (fd_room < I40E_FDIR_BUFFER_HEAD_ROOM_FOR_ATR)) { 6120 if (I40E_DEBUG_FD & pf->hw.debug_mask) 6121 dev_info(&pf->pdev->dev, "ATR disabled, not enough FD filter space.\n"); 6122 disable_atr = true; 6123 } 6124 6125 pf->fd_flush_timestamp = jiffies; 6126 pf->auto_disable_flags |= I40E_FLAG_FD_ATR_ENABLED; 6127 /* flush all filters */ 6128 wr32(&pf->hw, I40E_PFQF_CTL_1, 6129 I40E_PFQF_CTL_1_CLEARFDTABLE_MASK); 6130 i40e_flush(&pf->hw); 6131 pf->fd_flush_cnt++; 6132 pf->fd_add_err = 0; 6133 do { 6134 /* Check FD flush status every 5-6msec */ 6135 usleep_range(5000, 6000); 6136 reg = rd32(&pf->hw, I40E_PFQF_CTL_1); 6137 if (!(reg & I40E_PFQF_CTL_1_CLEARFDTABLE_MASK)) 6138 break; 6139 } while (flush_wait_retry--); 6140 if (reg & I40E_PFQF_CTL_1_CLEARFDTABLE_MASK) { 6141 dev_warn(&pf->pdev->dev, "FD table did not flush, needs more time\n"); 6142 } else { 6143 /* replay sideband filters */ 6144 i40e_fdir_filter_restore(pf->vsi[pf->lan_vsi]); 6145 if (!disable_atr) 6146 pf->auto_disable_flags &= ~I40E_FLAG_FD_ATR_ENABLED; 6147 clear_bit(__I40E_FD_FLUSH_REQUESTED, &pf->state); 6148 if (I40E_DEBUG_FD & pf->hw.debug_mask) 6149 dev_info(&pf->pdev->dev, "FD Filter table flushed and FD-SB replayed.\n"); 6150 } 6151 } 6152 6153 /** 6154 * i40e_get_current_atr_count - Get the count of total FD ATR filters programmed 6155 * @pf: board private structure 6156 **/ 6157 u32 i40e_get_current_atr_cnt(struct i40e_pf *pf) 6158 { 6159 return i40e_get_current_fd_count(pf) - pf->fdir_pf_active_filters; 6160 } 6161 6162 /* We can see up to 256 filter programming desc in transit if the filters are 6163 * being applied really fast; before we see the first 6164 * filter miss error on Rx queue 0. Accumulating enough error messages before 6165 * reacting will make sure we don't cause flush too often. 6166 */ 6167 #define I40E_MAX_FD_PROGRAM_ERROR 256 6168 6169 /** 6170 * i40e_fdir_reinit_subtask - Worker thread to reinit FDIR filter table 6171 * @pf: board private structure 6172 **/ 6173 static void i40e_fdir_reinit_subtask(struct i40e_pf *pf) 6174 { 6175 6176 /* if interface is down do nothing */ 6177 if (test_bit(__I40E_DOWN, &pf->state)) 6178 return; 6179 6180 if (test_bit(__I40E_FD_FLUSH_REQUESTED, &pf->state)) 6181 i40e_fdir_flush_and_replay(pf); 6182 6183 i40e_fdir_check_and_reenable(pf); 6184 6185 } 6186 6187 /** 6188 * i40e_vsi_link_event - notify VSI of a link event 6189 * @vsi: vsi to be notified 6190 * @link_up: link up or down 6191 **/ 6192 static void i40e_vsi_link_event(struct i40e_vsi *vsi, bool link_up) 6193 { 6194 if (!vsi || test_bit(__I40E_DOWN, &vsi->state)) 6195 return; 6196 6197 switch (vsi->type) { 6198 case I40E_VSI_MAIN: 6199 #ifdef I40E_FCOE 6200 case I40E_VSI_FCOE: 6201 #endif 6202 if (!vsi->netdev || !vsi->netdev_registered) 6203 break; 6204 6205 if (link_up) { 6206 netif_carrier_on(vsi->netdev); 6207 netif_tx_wake_all_queues(vsi->netdev); 6208 } else { 6209 netif_carrier_off(vsi->netdev); 6210 netif_tx_stop_all_queues(vsi->netdev); 6211 } 6212 break; 6213 6214 case I40E_VSI_SRIOV: 6215 case I40E_VSI_VMDQ2: 6216 case I40E_VSI_CTRL: 6217 case I40E_VSI_IWARP: 6218 case I40E_VSI_MIRROR: 6219 default: 6220 /* there is no notification for other VSIs */ 6221 break; 6222 } 6223 } 6224 6225 /** 6226 * i40e_veb_link_event - notify elements on the veb of a link event 6227 * @veb: veb to be notified 6228 * @link_up: link up or down 6229 **/ 6230 static void i40e_veb_link_event(struct i40e_veb *veb, bool link_up) 6231 { 6232 struct i40e_pf *pf; 6233 int i; 6234 6235 if (!veb || !veb->pf) 6236 return; 6237 pf = veb->pf; 6238 6239 /* depth first... */ 6240 for (i = 0; i < I40E_MAX_VEB; i++) 6241 if (pf->veb[i] && (pf->veb[i]->uplink_seid == veb->seid)) 6242 i40e_veb_link_event(pf->veb[i], link_up); 6243 6244 /* ... now the local VSIs */ 6245 for (i = 0; i < pf->num_alloc_vsi; i++) 6246 if (pf->vsi[i] && (pf->vsi[i]->uplink_seid == veb->seid)) 6247 i40e_vsi_link_event(pf->vsi[i], link_up); 6248 } 6249 6250 /** 6251 * i40e_link_event - Update netif_carrier status 6252 * @pf: board private structure 6253 **/ 6254 static void i40e_link_event(struct i40e_pf *pf) 6255 { 6256 struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi]; 6257 u8 new_link_speed, old_link_speed; 6258 i40e_status status; 6259 bool new_link, old_link; 6260 6261 /* save off old link status information */ 6262 pf->hw.phy.link_info_old = pf->hw.phy.link_info; 6263 6264 /* set this to force the get_link_status call to refresh state */ 6265 pf->hw.phy.get_link_info = true; 6266 6267 old_link = (pf->hw.phy.link_info_old.link_info & I40E_AQ_LINK_UP); 6268 6269 status = i40e_get_link_status(&pf->hw, &new_link); 6270 if (status) { 6271 dev_dbg(&pf->pdev->dev, "couldn't get link state, status: %d\n", 6272 status); 6273 return; 6274 } 6275 6276 old_link_speed = pf->hw.phy.link_info_old.link_speed; 6277 new_link_speed = pf->hw.phy.link_info.link_speed; 6278 6279 if (new_link == old_link && 6280 new_link_speed == old_link_speed && 6281 (test_bit(__I40E_DOWN, &vsi->state) || 6282 new_link == netif_carrier_ok(vsi->netdev))) 6283 return; 6284 6285 if (!test_bit(__I40E_DOWN, &vsi->state)) 6286 i40e_print_link_message(vsi, new_link); 6287 6288 /* Notify the base of the switch tree connected to 6289 * the link. Floating VEBs are not notified. 6290 */ 6291 if (pf->lan_veb != I40E_NO_VEB && pf->veb[pf->lan_veb]) 6292 i40e_veb_link_event(pf->veb[pf->lan_veb], new_link); 6293 else 6294 i40e_vsi_link_event(vsi, new_link); 6295 6296 if (pf->vf) 6297 i40e_vc_notify_link_state(pf); 6298 6299 if (pf->flags & I40E_FLAG_PTP) 6300 i40e_ptp_set_increment(pf); 6301 } 6302 6303 /** 6304 * i40e_watchdog_subtask - periodic checks not using event driven response 6305 * @pf: board private structure 6306 **/ 6307 static void i40e_watchdog_subtask(struct i40e_pf *pf) 6308 { 6309 int i; 6310 6311 /* if interface is down do nothing */ 6312 if (test_bit(__I40E_DOWN, &pf->state) || 6313 test_bit(__I40E_CONFIG_BUSY, &pf->state)) 6314 return; 6315 6316 /* make sure we don't do these things too often */ 6317 if (time_before(jiffies, (pf->service_timer_previous + 6318 pf->service_timer_period))) 6319 return; 6320 pf->service_timer_previous = jiffies; 6321 6322 if (pf->flags & I40E_FLAG_LINK_POLLING_ENABLED) 6323 i40e_link_event(pf); 6324 6325 /* Update the stats for active netdevs so the network stack 6326 * can look at updated numbers whenever it cares to 6327 */ 6328 for (i = 0; i < pf->num_alloc_vsi; i++) 6329 if (pf->vsi[i] && pf->vsi[i]->netdev) 6330 i40e_update_stats(pf->vsi[i]); 6331 6332 if (pf->flags & I40E_FLAG_VEB_STATS_ENABLED) { 6333 /* Update the stats for the active switching components */ 6334 for (i = 0; i < I40E_MAX_VEB; i++) 6335 if (pf->veb[i]) 6336 i40e_update_veb_stats(pf->veb[i]); 6337 } 6338 6339 i40e_ptp_rx_hang(pf->vsi[pf->lan_vsi]); 6340 } 6341 6342 /** 6343 * i40e_reset_subtask - Set up for resetting the device and driver 6344 * @pf: board private structure 6345 **/ 6346 static void i40e_reset_subtask(struct i40e_pf *pf) 6347 { 6348 u32 reset_flags = 0; 6349 6350 rtnl_lock(); 6351 if (test_bit(__I40E_REINIT_REQUESTED, &pf->state)) { 6352 reset_flags |= BIT(__I40E_REINIT_REQUESTED); 6353 clear_bit(__I40E_REINIT_REQUESTED, &pf->state); 6354 } 6355 if (test_bit(__I40E_PF_RESET_REQUESTED, &pf->state)) { 6356 reset_flags |= BIT(__I40E_PF_RESET_REQUESTED); 6357 clear_bit(__I40E_PF_RESET_REQUESTED, &pf->state); 6358 } 6359 if (test_bit(__I40E_CORE_RESET_REQUESTED, &pf->state)) { 6360 reset_flags |= BIT(__I40E_CORE_RESET_REQUESTED); 6361 clear_bit(__I40E_CORE_RESET_REQUESTED, &pf->state); 6362 } 6363 if (test_bit(__I40E_GLOBAL_RESET_REQUESTED, &pf->state)) { 6364 reset_flags |= BIT(__I40E_GLOBAL_RESET_REQUESTED); 6365 clear_bit(__I40E_GLOBAL_RESET_REQUESTED, &pf->state); 6366 } 6367 if (test_bit(__I40E_DOWN_REQUESTED, &pf->state)) { 6368 reset_flags |= BIT(__I40E_DOWN_REQUESTED); 6369 clear_bit(__I40E_DOWN_REQUESTED, &pf->state); 6370 } 6371 6372 /* If there's a recovery already waiting, it takes 6373 * precedence before starting a new reset sequence. 6374 */ 6375 if (test_bit(__I40E_RESET_INTR_RECEIVED, &pf->state)) { 6376 i40e_handle_reset_warning(pf); 6377 goto unlock; 6378 } 6379 6380 /* If we're already down or resetting, just bail */ 6381 if (reset_flags && 6382 !test_bit(__I40E_DOWN, &pf->state) && 6383 !test_bit(__I40E_CONFIG_BUSY, &pf->state)) 6384 i40e_do_reset(pf, reset_flags); 6385 6386 unlock: 6387 rtnl_unlock(); 6388 } 6389 6390 /** 6391 * i40e_handle_link_event - Handle link event 6392 * @pf: board private structure 6393 * @e: event info posted on ARQ 6394 **/ 6395 static void i40e_handle_link_event(struct i40e_pf *pf, 6396 struct i40e_arq_event_info *e) 6397 { 6398 struct i40e_aqc_get_link_status *status = 6399 (struct i40e_aqc_get_link_status *)&e->desc.params.raw; 6400 6401 /* Do a new status request to re-enable LSE reporting 6402 * and load new status information into the hw struct 6403 * This completely ignores any state information 6404 * in the ARQ event info, instead choosing to always 6405 * issue the AQ update link status command. 6406 */ 6407 i40e_link_event(pf); 6408 6409 /* check for unqualified module, if link is down */ 6410 if ((status->link_info & I40E_AQ_MEDIA_AVAILABLE) && 6411 (!(status->an_info & I40E_AQ_QUALIFIED_MODULE)) && 6412 (!(status->link_info & I40E_AQ_LINK_UP))) 6413 dev_err(&pf->pdev->dev, 6414 "The driver failed to link because an unqualified module was detected.\n"); 6415 } 6416 6417 /** 6418 * i40e_clean_adminq_subtask - Clean the AdminQ rings 6419 * @pf: board private structure 6420 **/ 6421 static void i40e_clean_adminq_subtask(struct i40e_pf *pf) 6422 { 6423 struct i40e_arq_event_info event; 6424 struct i40e_hw *hw = &pf->hw; 6425 u16 pending, i = 0; 6426 i40e_status ret; 6427 u16 opcode; 6428 u32 oldval; 6429 u32 val; 6430 6431 /* Do not run clean AQ when PF reset fails */ 6432 if (test_bit(__I40E_RESET_FAILED, &pf->state)) 6433 return; 6434 6435 /* check for error indications */ 6436 val = rd32(&pf->hw, pf->hw.aq.arq.len); 6437 oldval = val; 6438 if (val & I40E_PF_ARQLEN_ARQVFE_MASK) { 6439 if (hw->debug_mask & I40E_DEBUG_AQ) 6440 dev_info(&pf->pdev->dev, "ARQ VF Error detected\n"); 6441 val &= ~I40E_PF_ARQLEN_ARQVFE_MASK; 6442 } 6443 if (val & I40E_PF_ARQLEN_ARQOVFL_MASK) { 6444 if (hw->debug_mask & I40E_DEBUG_AQ) 6445 dev_info(&pf->pdev->dev, "ARQ Overflow Error detected\n"); 6446 val &= ~I40E_PF_ARQLEN_ARQOVFL_MASK; 6447 pf->arq_overflows++; 6448 } 6449 if (val & I40E_PF_ARQLEN_ARQCRIT_MASK) { 6450 if (hw->debug_mask & I40E_DEBUG_AQ) 6451 dev_info(&pf->pdev->dev, "ARQ Critical Error detected\n"); 6452 val &= ~I40E_PF_ARQLEN_ARQCRIT_MASK; 6453 } 6454 if (oldval != val) 6455 wr32(&pf->hw, pf->hw.aq.arq.len, val); 6456 6457 val = rd32(&pf->hw, pf->hw.aq.asq.len); 6458 oldval = val; 6459 if (val & I40E_PF_ATQLEN_ATQVFE_MASK) { 6460 if (pf->hw.debug_mask & I40E_DEBUG_AQ) 6461 dev_info(&pf->pdev->dev, "ASQ VF Error detected\n"); 6462 val &= ~I40E_PF_ATQLEN_ATQVFE_MASK; 6463 } 6464 if (val & I40E_PF_ATQLEN_ATQOVFL_MASK) { 6465 if (pf->hw.debug_mask & I40E_DEBUG_AQ) 6466 dev_info(&pf->pdev->dev, "ASQ Overflow Error detected\n"); 6467 val &= ~I40E_PF_ATQLEN_ATQOVFL_MASK; 6468 } 6469 if (val & I40E_PF_ATQLEN_ATQCRIT_MASK) { 6470 if (pf->hw.debug_mask & I40E_DEBUG_AQ) 6471 dev_info(&pf->pdev->dev, "ASQ Critical Error detected\n"); 6472 val &= ~I40E_PF_ATQLEN_ATQCRIT_MASK; 6473 } 6474 if (oldval != val) 6475 wr32(&pf->hw, pf->hw.aq.asq.len, val); 6476 6477 event.buf_len = I40E_MAX_AQ_BUF_SIZE; 6478 event.msg_buf = kzalloc(event.buf_len, GFP_KERNEL); 6479 if (!event.msg_buf) 6480 return; 6481 6482 do { 6483 ret = i40e_clean_arq_element(hw, &event, &pending); 6484 if (ret == I40E_ERR_ADMIN_QUEUE_NO_WORK) 6485 break; 6486 else if (ret) { 6487 dev_info(&pf->pdev->dev, "ARQ event error %d\n", ret); 6488 break; 6489 } 6490 6491 opcode = le16_to_cpu(event.desc.opcode); 6492 switch (opcode) { 6493 6494 case i40e_aqc_opc_get_link_status: 6495 i40e_handle_link_event(pf, &event); 6496 break; 6497 case i40e_aqc_opc_send_msg_to_pf: 6498 ret = i40e_vc_process_vf_msg(pf, 6499 le16_to_cpu(event.desc.retval), 6500 le32_to_cpu(event.desc.cookie_high), 6501 le32_to_cpu(event.desc.cookie_low), 6502 event.msg_buf, 6503 event.msg_len); 6504 break; 6505 case i40e_aqc_opc_lldp_update_mib: 6506 dev_dbg(&pf->pdev->dev, "ARQ: Update LLDP MIB event received\n"); 6507 #ifdef CONFIG_I40E_DCB 6508 rtnl_lock(); 6509 ret = i40e_handle_lldp_event(pf, &event); 6510 rtnl_unlock(); 6511 #endif /* CONFIG_I40E_DCB */ 6512 break; 6513 case i40e_aqc_opc_event_lan_overflow: 6514 dev_dbg(&pf->pdev->dev, "ARQ LAN queue overflow event received\n"); 6515 i40e_handle_lan_overflow_event(pf, &event); 6516 break; 6517 case i40e_aqc_opc_send_msg_to_peer: 6518 dev_info(&pf->pdev->dev, "ARQ: Msg from other pf\n"); 6519 break; 6520 case i40e_aqc_opc_nvm_erase: 6521 case i40e_aqc_opc_nvm_update: 6522 case i40e_aqc_opc_oem_post_update: 6523 i40e_debug(&pf->hw, I40E_DEBUG_NVM, 6524 "ARQ NVM operation 0x%04x completed\n", 6525 opcode); 6526 break; 6527 default: 6528 dev_info(&pf->pdev->dev, 6529 "ARQ: Unknown event 0x%04x ignored\n", 6530 opcode); 6531 break; 6532 } 6533 } while (pending && (i++ < pf->adminq_work_limit)); 6534 6535 clear_bit(__I40E_ADMINQ_EVENT_PENDING, &pf->state); 6536 /* re-enable Admin queue interrupt cause */ 6537 val = rd32(hw, I40E_PFINT_ICR0_ENA); 6538 val |= I40E_PFINT_ICR0_ENA_ADMINQ_MASK; 6539 wr32(hw, I40E_PFINT_ICR0_ENA, val); 6540 i40e_flush(hw); 6541 6542 kfree(event.msg_buf); 6543 } 6544 6545 /** 6546 * i40e_verify_eeprom - make sure eeprom is good to use 6547 * @pf: board private structure 6548 **/ 6549 static void i40e_verify_eeprom(struct i40e_pf *pf) 6550 { 6551 int err; 6552 6553 err = i40e_diag_eeprom_test(&pf->hw); 6554 if (err) { 6555 /* retry in case of garbage read */ 6556 err = i40e_diag_eeprom_test(&pf->hw); 6557 if (err) { 6558 dev_info(&pf->pdev->dev, "eeprom check failed (%d), Tx/Rx traffic disabled\n", 6559 err); 6560 set_bit(__I40E_BAD_EEPROM, &pf->state); 6561 } 6562 } 6563 6564 if (!err && test_bit(__I40E_BAD_EEPROM, &pf->state)) { 6565 dev_info(&pf->pdev->dev, "eeprom check passed, Tx/Rx traffic enabled\n"); 6566 clear_bit(__I40E_BAD_EEPROM, &pf->state); 6567 } 6568 } 6569 6570 /** 6571 * i40e_enable_pf_switch_lb 6572 * @pf: pointer to the PF structure 6573 * 6574 * enable switch loop back or die - no point in a return value 6575 **/ 6576 static void i40e_enable_pf_switch_lb(struct i40e_pf *pf) 6577 { 6578 struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi]; 6579 struct i40e_vsi_context ctxt; 6580 int ret; 6581 6582 ctxt.seid = pf->main_vsi_seid; 6583 ctxt.pf_num = pf->hw.pf_id; 6584 ctxt.vf_num = 0; 6585 ret = i40e_aq_get_vsi_params(&pf->hw, &ctxt, NULL); 6586 if (ret) { 6587 dev_info(&pf->pdev->dev, 6588 "couldn't get PF vsi config, err %s aq_err %s\n", 6589 i40e_stat_str(&pf->hw, ret), 6590 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 6591 return; 6592 } 6593 ctxt.flags = I40E_AQ_VSI_TYPE_PF; 6594 ctxt.info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID); 6595 ctxt.info.switch_id |= cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB); 6596 6597 ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL); 6598 if (ret) { 6599 dev_info(&pf->pdev->dev, 6600 "update vsi switch failed, err %s aq_err %s\n", 6601 i40e_stat_str(&pf->hw, ret), 6602 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 6603 } 6604 } 6605 6606 /** 6607 * i40e_disable_pf_switch_lb 6608 * @pf: pointer to the PF structure 6609 * 6610 * disable switch loop back or die - no point in a return value 6611 **/ 6612 static void i40e_disable_pf_switch_lb(struct i40e_pf *pf) 6613 { 6614 struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi]; 6615 struct i40e_vsi_context ctxt; 6616 int ret; 6617 6618 ctxt.seid = pf->main_vsi_seid; 6619 ctxt.pf_num = pf->hw.pf_id; 6620 ctxt.vf_num = 0; 6621 ret = i40e_aq_get_vsi_params(&pf->hw, &ctxt, NULL); 6622 if (ret) { 6623 dev_info(&pf->pdev->dev, 6624 "couldn't get PF vsi config, err %s aq_err %s\n", 6625 i40e_stat_str(&pf->hw, ret), 6626 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 6627 return; 6628 } 6629 ctxt.flags = I40E_AQ_VSI_TYPE_PF; 6630 ctxt.info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID); 6631 ctxt.info.switch_id &= ~cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB); 6632 6633 ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL); 6634 if (ret) { 6635 dev_info(&pf->pdev->dev, 6636 "update vsi switch failed, err %s aq_err %s\n", 6637 i40e_stat_str(&pf->hw, ret), 6638 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 6639 } 6640 } 6641 6642 /** 6643 * i40e_config_bridge_mode - Configure the HW bridge mode 6644 * @veb: pointer to the bridge instance 6645 * 6646 * Configure the loop back mode for the LAN VSI that is downlink to the 6647 * specified HW bridge instance. It is expected this function is called 6648 * when a new HW bridge is instantiated. 6649 **/ 6650 static void i40e_config_bridge_mode(struct i40e_veb *veb) 6651 { 6652 struct i40e_pf *pf = veb->pf; 6653 6654 if (pf->hw.debug_mask & I40E_DEBUG_LAN) 6655 dev_info(&pf->pdev->dev, "enabling bridge mode: %s\n", 6656 veb->bridge_mode == BRIDGE_MODE_VEPA ? "VEPA" : "VEB"); 6657 if (veb->bridge_mode & BRIDGE_MODE_VEPA) 6658 i40e_disable_pf_switch_lb(pf); 6659 else 6660 i40e_enable_pf_switch_lb(pf); 6661 } 6662 6663 /** 6664 * i40e_reconstitute_veb - rebuild the VEB and anything connected to it 6665 * @veb: pointer to the VEB instance 6666 * 6667 * This is a recursive function that first builds the attached VSIs then 6668 * recurses in to build the next layer of VEB. We track the connections 6669 * through our own index numbers because the seid's from the HW could 6670 * change across the reset. 6671 **/ 6672 static int i40e_reconstitute_veb(struct i40e_veb *veb) 6673 { 6674 struct i40e_vsi *ctl_vsi = NULL; 6675 struct i40e_pf *pf = veb->pf; 6676 int v, veb_idx; 6677 int ret; 6678 6679 /* build VSI that owns this VEB, temporarily attached to base VEB */ 6680 for (v = 0; v < pf->num_alloc_vsi && !ctl_vsi; v++) { 6681 if (pf->vsi[v] && 6682 pf->vsi[v]->veb_idx == veb->idx && 6683 pf->vsi[v]->flags & I40E_VSI_FLAG_VEB_OWNER) { 6684 ctl_vsi = pf->vsi[v]; 6685 break; 6686 } 6687 } 6688 if (!ctl_vsi) { 6689 dev_info(&pf->pdev->dev, 6690 "missing owner VSI for veb_idx %d\n", veb->idx); 6691 ret = -ENOENT; 6692 goto end_reconstitute; 6693 } 6694 if (ctl_vsi != pf->vsi[pf->lan_vsi]) 6695 ctl_vsi->uplink_seid = pf->vsi[pf->lan_vsi]->uplink_seid; 6696 ret = i40e_add_vsi(ctl_vsi); 6697 if (ret) { 6698 dev_info(&pf->pdev->dev, 6699 "rebuild of veb_idx %d owner VSI failed: %d\n", 6700 veb->idx, ret); 6701 goto end_reconstitute; 6702 } 6703 i40e_vsi_reset_stats(ctl_vsi); 6704 6705 /* create the VEB in the switch and move the VSI onto the VEB */ 6706 ret = i40e_add_veb(veb, ctl_vsi); 6707 if (ret) 6708 goto end_reconstitute; 6709 6710 if (pf->flags & I40E_FLAG_VEB_MODE_ENABLED) 6711 veb->bridge_mode = BRIDGE_MODE_VEB; 6712 else 6713 veb->bridge_mode = BRIDGE_MODE_VEPA; 6714 i40e_config_bridge_mode(veb); 6715 6716 /* create the remaining VSIs attached to this VEB */ 6717 for (v = 0; v < pf->num_alloc_vsi; v++) { 6718 if (!pf->vsi[v] || pf->vsi[v] == ctl_vsi) 6719 continue; 6720 6721 if (pf->vsi[v]->veb_idx == veb->idx) { 6722 struct i40e_vsi *vsi = pf->vsi[v]; 6723 6724 vsi->uplink_seid = veb->seid; 6725 ret = i40e_add_vsi(vsi); 6726 if (ret) { 6727 dev_info(&pf->pdev->dev, 6728 "rebuild of vsi_idx %d failed: %d\n", 6729 v, ret); 6730 goto end_reconstitute; 6731 } 6732 i40e_vsi_reset_stats(vsi); 6733 } 6734 } 6735 6736 /* create any VEBs attached to this VEB - RECURSION */ 6737 for (veb_idx = 0; veb_idx < I40E_MAX_VEB; veb_idx++) { 6738 if (pf->veb[veb_idx] && pf->veb[veb_idx]->veb_idx == veb->idx) { 6739 pf->veb[veb_idx]->uplink_seid = veb->seid; 6740 ret = i40e_reconstitute_veb(pf->veb[veb_idx]); 6741 if (ret) 6742 break; 6743 } 6744 } 6745 6746 end_reconstitute: 6747 return ret; 6748 } 6749 6750 /** 6751 * i40e_get_capabilities - get info about the HW 6752 * @pf: the PF struct 6753 **/ 6754 static int i40e_get_capabilities(struct i40e_pf *pf) 6755 { 6756 struct i40e_aqc_list_capabilities_element_resp *cap_buf; 6757 u16 data_size; 6758 int buf_len; 6759 int err; 6760 6761 buf_len = 40 * sizeof(struct i40e_aqc_list_capabilities_element_resp); 6762 do { 6763 cap_buf = kzalloc(buf_len, GFP_KERNEL); 6764 if (!cap_buf) 6765 return -ENOMEM; 6766 6767 /* this loads the data into the hw struct for us */ 6768 err = i40e_aq_discover_capabilities(&pf->hw, cap_buf, buf_len, 6769 &data_size, 6770 i40e_aqc_opc_list_func_capabilities, 6771 NULL); 6772 /* data loaded, buffer no longer needed */ 6773 kfree(cap_buf); 6774 6775 if (pf->hw.aq.asq_last_status == I40E_AQ_RC_ENOMEM) { 6776 /* retry with a larger buffer */ 6777 buf_len = data_size; 6778 } else if (pf->hw.aq.asq_last_status != I40E_AQ_RC_OK) { 6779 dev_info(&pf->pdev->dev, 6780 "capability discovery failed, err %s aq_err %s\n", 6781 i40e_stat_str(&pf->hw, err), 6782 i40e_aq_str(&pf->hw, 6783 pf->hw.aq.asq_last_status)); 6784 return -ENODEV; 6785 } 6786 } while (err); 6787 6788 if (pf->hw.debug_mask & I40E_DEBUG_USER) 6789 dev_info(&pf->pdev->dev, 6790 "pf=%d, num_vfs=%d, msix_pf=%d, msix_vf=%d, fd_g=%d, fd_b=%d, pf_max_q=%d num_vsi=%d\n", 6791 pf->hw.pf_id, pf->hw.func_caps.num_vfs, 6792 pf->hw.func_caps.num_msix_vectors, 6793 pf->hw.func_caps.num_msix_vectors_vf, 6794 pf->hw.func_caps.fd_filters_guaranteed, 6795 pf->hw.func_caps.fd_filters_best_effort, 6796 pf->hw.func_caps.num_tx_qp, 6797 pf->hw.func_caps.num_vsis); 6798 6799 #define DEF_NUM_VSI (1 + (pf->hw.func_caps.fcoe ? 1 : 0) \ 6800 + pf->hw.func_caps.num_vfs) 6801 if (pf->hw.revision_id == 0 && (DEF_NUM_VSI > pf->hw.func_caps.num_vsis)) { 6802 dev_info(&pf->pdev->dev, 6803 "got num_vsis %d, setting num_vsis to %d\n", 6804 pf->hw.func_caps.num_vsis, DEF_NUM_VSI); 6805 pf->hw.func_caps.num_vsis = DEF_NUM_VSI; 6806 } 6807 6808 return 0; 6809 } 6810 6811 static int i40e_vsi_clear(struct i40e_vsi *vsi); 6812 6813 /** 6814 * i40e_fdir_sb_setup - initialize the Flow Director resources for Sideband 6815 * @pf: board private structure 6816 **/ 6817 static void i40e_fdir_sb_setup(struct i40e_pf *pf) 6818 { 6819 struct i40e_vsi *vsi; 6820 6821 /* quick workaround for an NVM issue that leaves a critical register 6822 * uninitialized 6823 */ 6824 if (!rd32(&pf->hw, I40E_GLQF_HKEY(0))) { 6825 static const u32 hkey[] = { 6826 0xe640d33f, 0xcdfe98ab, 0x73fa7161, 0x0d7a7d36, 6827 0xeacb7d61, 0xaa4f05b6, 0x9c5c89ed, 0xfc425ddb, 6828 0xa4654832, 0xfc7461d4, 0x8f827619, 0xf5c63c21, 6829 0x95b3a76d}; 6830 int i; 6831 6832 for (i = 0; i <= I40E_GLQF_HKEY_MAX_INDEX; i++) 6833 wr32(&pf->hw, I40E_GLQF_HKEY(i), hkey[i]); 6834 } 6835 6836 if (!(pf->flags & I40E_FLAG_FD_SB_ENABLED)) 6837 return; 6838 6839 /* find existing VSI and see if it needs configuring */ 6840 vsi = i40e_find_vsi_by_type(pf, I40E_VSI_FDIR); 6841 6842 /* create a new VSI if none exists */ 6843 if (!vsi) { 6844 vsi = i40e_vsi_setup(pf, I40E_VSI_FDIR, 6845 pf->vsi[pf->lan_vsi]->seid, 0); 6846 if (!vsi) { 6847 dev_info(&pf->pdev->dev, "Couldn't create FDir VSI\n"); 6848 pf->flags &= ~I40E_FLAG_FD_SB_ENABLED; 6849 return; 6850 } 6851 } 6852 6853 i40e_vsi_setup_irqhandler(vsi, i40e_fdir_clean_ring); 6854 } 6855 6856 /** 6857 * i40e_fdir_teardown - release the Flow Director resources 6858 * @pf: board private structure 6859 **/ 6860 static void i40e_fdir_teardown(struct i40e_pf *pf) 6861 { 6862 struct i40e_vsi *vsi; 6863 6864 i40e_fdir_filter_exit(pf); 6865 vsi = i40e_find_vsi_by_type(pf, I40E_VSI_FDIR); 6866 if (vsi) 6867 i40e_vsi_release(vsi); 6868 } 6869 6870 /** 6871 * i40e_prep_for_reset - prep for the core to reset 6872 * @pf: board private structure 6873 * 6874 * Close up the VFs and other things in prep for PF Reset. 6875 **/ 6876 static void i40e_prep_for_reset(struct i40e_pf *pf) 6877 { 6878 struct i40e_hw *hw = &pf->hw; 6879 i40e_status ret = 0; 6880 u32 v; 6881 6882 clear_bit(__I40E_RESET_INTR_RECEIVED, &pf->state); 6883 if (test_and_set_bit(__I40E_RESET_RECOVERY_PENDING, &pf->state)) 6884 return; 6885 if (i40e_check_asq_alive(&pf->hw)) 6886 i40e_vc_notify_reset(pf); 6887 6888 dev_dbg(&pf->pdev->dev, "Tearing down internal switch for reset\n"); 6889 6890 /* quiesce the VSIs and their queues that are not already DOWN */ 6891 i40e_pf_quiesce_all_vsi(pf); 6892 6893 for (v = 0; v < pf->num_alloc_vsi; v++) { 6894 if (pf->vsi[v]) 6895 pf->vsi[v]->seid = 0; 6896 } 6897 6898 i40e_shutdown_adminq(&pf->hw); 6899 6900 /* call shutdown HMC */ 6901 if (hw->hmc.hmc_obj) { 6902 ret = i40e_shutdown_lan_hmc(hw); 6903 if (ret) 6904 dev_warn(&pf->pdev->dev, 6905 "shutdown_lan_hmc failed: %d\n", ret); 6906 } 6907 } 6908 6909 /** 6910 * i40e_send_version - update firmware with driver version 6911 * @pf: PF struct 6912 */ 6913 static void i40e_send_version(struct i40e_pf *pf) 6914 { 6915 struct i40e_driver_version dv; 6916 6917 dv.major_version = DRV_VERSION_MAJOR; 6918 dv.minor_version = DRV_VERSION_MINOR; 6919 dv.build_version = DRV_VERSION_BUILD; 6920 dv.subbuild_version = 0; 6921 strlcpy(dv.driver_string, DRV_VERSION, sizeof(dv.driver_string)); 6922 i40e_aq_send_driver_version(&pf->hw, &dv, NULL); 6923 } 6924 6925 /** 6926 * i40e_reset_and_rebuild - reset and rebuild using a saved config 6927 * @pf: board private structure 6928 * @reinit: if the Main VSI needs to re-initialized. 6929 **/ 6930 static void i40e_reset_and_rebuild(struct i40e_pf *pf, bool reinit) 6931 { 6932 struct i40e_hw *hw = &pf->hw; 6933 u8 set_fc_aq_fail = 0; 6934 i40e_status ret; 6935 u32 val; 6936 u32 v; 6937 6938 /* Now we wait for GRST to settle out. 6939 * We don't have to delete the VEBs or VSIs from the hw switch 6940 * because the reset will make them disappear. 6941 */ 6942 ret = i40e_pf_reset(hw); 6943 if (ret) { 6944 dev_info(&pf->pdev->dev, "PF reset failed, %d\n", ret); 6945 set_bit(__I40E_RESET_FAILED, &pf->state); 6946 goto clear_recovery; 6947 } 6948 pf->pfr_count++; 6949 6950 if (test_bit(__I40E_DOWN, &pf->state)) 6951 goto clear_recovery; 6952 dev_dbg(&pf->pdev->dev, "Rebuilding internal switch\n"); 6953 6954 /* rebuild the basics for the AdminQ, HMC, and initial HW switch */ 6955 ret = i40e_init_adminq(&pf->hw); 6956 if (ret) { 6957 dev_info(&pf->pdev->dev, "Rebuild AdminQ failed, err %s aq_err %s\n", 6958 i40e_stat_str(&pf->hw, ret), 6959 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 6960 goto clear_recovery; 6961 } 6962 6963 /* re-verify the eeprom if we just had an EMP reset */ 6964 if (test_and_clear_bit(__I40E_EMP_RESET_INTR_RECEIVED, &pf->state)) 6965 i40e_verify_eeprom(pf); 6966 6967 i40e_clear_pxe_mode(hw); 6968 ret = i40e_get_capabilities(pf); 6969 if (ret) 6970 goto end_core_reset; 6971 6972 ret = i40e_init_lan_hmc(hw, hw->func_caps.num_tx_qp, 6973 hw->func_caps.num_rx_qp, 6974 pf->fcoe_hmc_cntx_num, pf->fcoe_hmc_filt_num); 6975 if (ret) { 6976 dev_info(&pf->pdev->dev, "init_lan_hmc failed: %d\n", ret); 6977 goto end_core_reset; 6978 } 6979 ret = i40e_configure_lan_hmc(hw, I40E_HMC_MODEL_DIRECT_ONLY); 6980 if (ret) { 6981 dev_info(&pf->pdev->dev, "configure_lan_hmc failed: %d\n", ret); 6982 goto end_core_reset; 6983 } 6984 6985 #ifdef CONFIG_I40E_DCB 6986 ret = i40e_init_pf_dcb(pf); 6987 if (ret) { 6988 dev_info(&pf->pdev->dev, "DCB init failed %d, disabled\n", ret); 6989 pf->flags &= ~I40E_FLAG_DCB_CAPABLE; 6990 /* Continue without DCB enabled */ 6991 } 6992 #endif /* CONFIG_I40E_DCB */ 6993 #ifdef I40E_FCOE 6994 i40e_init_pf_fcoe(pf); 6995 6996 #endif 6997 /* do basic switch setup */ 6998 ret = i40e_setup_pf_switch(pf, reinit); 6999 if (ret) 7000 goto end_core_reset; 7001 7002 /* The driver only wants link up/down and module qualification 7003 * reports from firmware. Note the negative logic. 7004 */ 7005 ret = i40e_aq_set_phy_int_mask(&pf->hw, 7006 ~(I40E_AQ_EVENT_LINK_UPDOWN | 7007 I40E_AQ_EVENT_MEDIA_NA | 7008 I40E_AQ_EVENT_MODULE_QUAL_FAIL), NULL); 7009 if (ret) 7010 dev_info(&pf->pdev->dev, "set phy mask fail, err %s aq_err %s\n", 7011 i40e_stat_str(&pf->hw, ret), 7012 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 7013 7014 /* make sure our flow control settings are restored */ 7015 ret = i40e_set_fc(&pf->hw, &set_fc_aq_fail, true); 7016 if (ret) 7017 dev_dbg(&pf->pdev->dev, "setting flow control: ret = %s last_status = %s\n", 7018 i40e_stat_str(&pf->hw, ret), 7019 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 7020 7021 /* Rebuild the VSIs and VEBs that existed before reset. 7022 * They are still in our local switch element arrays, so only 7023 * need to rebuild the switch model in the HW. 7024 * 7025 * If there were VEBs but the reconstitution failed, we'll try 7026 * try to recover minimal use by getting the basic PF VSI working. 7027 */ 7028 if (pf->vsi[pf->lan_vsi]->uplink_seid != pf->mac_seid) { 7029 dev_dbg(&pf->pdev->dev, "attempting to rebuild switch\n"); 7030 /* find the one VEB connected to the MAC, and find orphans */ 7031 for (v = 0; v < I40E_MAX_VEB; v++) { 7032 if (!pf->veb[v]) 7033 continue; 7034 7035 if (pf->veb[v]->uplink_seid == pf->mac_seid || 7036 pf->veb[v]->uplink_seid == 0) { 7037 ret = i40e_reconstitute_veb(pf->veb[v]); 7038 7039 if (!ret) 7040 continue; 7041 7042 /* If Main VEB failed, we're in deep doodoo, 7043 * so give up rebuilding the switch and set up 7044 * for minimal rebuild of PF VSI. 7045 * If orphan failed, we'll report the error 7046 * but try to keep going. 7047 */ 7048 if (pf->veb[v]->uplink_seid == pf->mac_seid) { 7049 dev_info(&pf->pdev->dev, 7050 "rebuild of switch failed: %d, will try to set up simple PF connection\n", 7051 ret); 7052 pf->vsi[pf->lan_vsi]->uplink_seid 7053 = pf->mac_seid; 7054 break; 7055 } else if (pf->veb[v]->uplink_seid == 0) { 7056 dev_info(&pf->pdev->dev, 7057 "rebuild of orphan VEB failed: %d\n", 7058 ret); 7059 } 7060 } 7061 } 7062 } 7063 7064 if (pf->vsi[pf->lan_vsi]->uplink_seid == pf->mac_seid) { 7065 dev_dbg(&pf->pdev->dev, "attempting to rebuild PF VSI\n"); 7066 /* no VEB, so rebuild only the Main VSI */ 7067 ret = i40e_add_vsi(pf->vsi[pf->lan_vsi]); 7068 if (ret) { 7069 dev_info(&pf->pdev->dev, 7070 "rebuild of Main VSI failed: %d\n", ret); 7071 goto end_core_reset; 7072 } 7073 } 7074 7075 /* Reconfigure hardware for allowing smaller MSS in the case 7076 * of TSO, so that we avoid the MDD being fired and causing 7077 * a reset in the case of small MSS+TSO. 7078 */ 7079 #define I40E_REG_MSS 0x000E64DC 7080 #define I40E_REG_MSS_MIN_MASK 0x3FF0000 7081 #define I40E_64BYTE_MSS 0x400000 7082 val = rd32(hw, I40E_REG_MSS); 7083 if ((val & I40E_REG_MSS_MIN_MASK) > I40E_64BYTE_MSS) { 7084 val &= ~I40E_REG_MSS_MIN_MASK; 7085 val |= I40E_64BYTE_MSS; 7086 wr32(hw, I40E_REG_MSS, val); 7087 } 7088 7089 if (pf->flags & I40E_FLAG_RESTART_AUTONEG) { 7090 msleep(75); 7091 ret = i40e_aq_set_link_restart_an(&pf->hw, true, NULL); 7092 if (ret) 7093 dev_info(&pf->pdev->dev, "link restart failed, err %s aq_err %s\n", 7094 i40e_stat_str(&pf->hw, ret), 7095 i40e_aq_str(&pf->hw, 7096 pf->hw.aq.asq_last_status)); 7097 } 7098 /* reinit the misc interrupt */ 7099 if (pf->flags & I40E_FLAG_MSIX_ENABLED) 7100 ret = i40e_setup_misc_vector(pf); 7101 7102 /* Add a filter to drop all Flow control frames from any VSI from being 7103 * transmitted. By doing so we stop a malicious VF from sending out 7104 * PAUSE or PFC frames and potentially controlling traffic for other 7105 * PF/VF VSIs. 7106 * The FW can still send Flow control frames if enabled. 7107 */ 7108 i40e_add_filter_to_drop_tx_flow_control_frames(&pf->hw, 7109 pf->main_vsi_seid); 7110 7111 /* restart the VSIs that were rebuilt and running before the reset */ 7112 i40e_pf_unquiesce_all_vsi(pf); 7113 7114 if (pf->num_alloc_vfs) { 7115 for (v = 0; v < pf->num_alloc_vfs; v++) 7116 i40e_reset_vf(&pf->vf[v], true); 7117 } 7118 7119 /* tell the firmware that we're starting */ 7120 i40e_send_version(pf); 7121 7122 end_core_reset: 7123 clear_bit(__I40E_RESET_FAILED, &pf->state); 7124 clear_recovery: 7125 clear_bit(__I40E_RESET_RECOVERY_PENDING, &pf->state); 7126 } 7127 7128 /** 7129 * i40e_handle_reset_warning - prep for the PF to reset, reset and rebuild 7130 * @pf: board private structure 7131 * 7132 * Close up the VFs and other things in prep for a Core Reset, 7133 * then get ready to rebuild the world. 7134 **/ 7135 static void i40e_handle_reset_warning(struct i40e_pf *pf) 7136 { 7137 i40e_prep_for_reset(pf); 7138 i40e_reset_and_rebuild(pf, false); 7139 } 7140 7141 /** 7142 * i40e_handle_mdd_event 7143 * @pf: pointer to the PF structure 7144 * 7145 * Called from the MDD irq handler to identify possibly malicious vfs 7146 **/ 7147 static void i40e_handle_mdd_event(struct i40e_pf *pf) 7148 { 7149 struct i40e_hw *hw = &pf->hw; 7150 bool mdd_detected = false; 7151 bool pf_mdd_detected = false; 7152 struct i40e_vf *vf; 7153 u32 reg; 7154 int i; 7155 7156 if (!test_bit(__I40E_MDD_EVENT_PENDING, &pf->state)) 7157 return; 7158 7159 /* find what triggered the MDD event */ 7160 reg = rd32(hw, I40E_GL_MDET_TX); 7161 if (reg & I40E_GL_MDET_TX_VALID_MASK) { 7162 u8 pf_num = (reg & I40E_GL_MDET_TX_PF_NUM_MASK) >> 7163 I40E_GL_MDET_TX_PF_NUM_SHIFT; 7164 u16 vf_num = (reg & I40E_GL_MDET_TX_VF_NUM_MASK) >> 7165 I40E_GL_MDET_TX_VF_NUM_SHIFT; 7166 u8 event = (reg & I40E_GL_MDET_TX_EVENT_MASK) >> 7167 I40E_GL_MDET_TX_EVENT_SHIFT; 7168 u16 queue = ((reg & I40E_GL_MDET_TX_QUEUE_MASK) >> 7169 I40E_GL_MDET_TX_QUEUE_SHIFT) - 7170 pf->hw.func_caps.base_queue; 7171 if (netif_msg_tx_err(pf)) 7172 dev_info(&pf->pdev->dev, "Malicious Driver Detection event 0x%02x on TX queue %d PF number 0x%02x VF number 0x%02x\n", 7173 event, queue, pf_num, vf_num); 7174 wr32(hw, I40E_GL_MDET_TX, 0xffffffff); 7175 mdd_detected = true; 7176 } 7177 reg = rd32(hw, I40E_GL_MDET_RX); 7178 if (reg & I40E_GL_MDET_RX_VALID_MASK) { 7179 u8 func = (reg & I40E_GL_MDET_RX_FUNCTION_MASK) >> 7180 I40E_GL_MDET_RX_FUNCTION_SHIFT; 7181 u8 event = (reg & I40E_GL_MDET_RX_EVENT_MASK) >> 7182 I40E_GL_MDET_RX_EVENT_SHIFT; 7183 u16 queue = ((reg & I40E_GL_MDET_RX_QUEUE_MASK) >> 7184 I40E_GL_MDET_RX_QUEUE_SHIFT) - 7185 pf->hw.func_caps.base_queue; 7186 if (netif_msg_rx_err(pf)) 7187 dev_info(&pf->pdev->dev, "Malicious Driver Detection event 0x%02x on RX queue %d of function 0x%02x\n", 7188 event, queue, func); 7189 wr32(hw, I40E_GL_MDET_RX, 0xffffffff); 7190 mdd_detected = true; 7191 } 7192 7193 if (mdd_detected) { 7194 reg = rd32(hw, I40E_PF_MDET_TX); 7195 if (reg & I40E_PF_MDET_TX_VALID_MASK) { 7196 wr32(hw, I40E_PF_MDET_TX, 0xFFFF); 7197 dev_info(&pf->pdev->dev, "TX driver issue detected, PF reset issued\n"); 7198 pf_mdd_detected = true; 7199 } 7200 reg = rd32(hw, I40E_PF_MDET_RX); 7201 if (reg & I40E_PF_MDET_RX_VALID_MASK) { 7202 wr32(hw, I40E_PF_MDET_RX, 0xFFFF); 7203 dev_info(&pf->pdev->dev, "RX driver issue detected, PF reset issued\n"); 7204 pf_mdd_detected = true; 7205 } 7206 /* Queue belongs to the PF, initiate a reset */ 7207 if (pf_mdd_detected) { 7208 set_bit(__I40E_PF_RESET_REQUESTED, &pf->state); 7209 i40e_service_event_schedule(pf); 7210 } 7211 } 7212 7213 /* see if one of the VFs needs its hand slapped */ 7214 for (i = 0; i < pf->num_alloc_vfs && mdd_detected; i++) { 7215 vf = &(pf->vf[i]); 7216 reg = rd32(hw, I40E_VP_MDET_TX(i)); 7217 if (reg & I40E_VP_MDET_TX_VALID_MASK) { 7218 wr32(hw, I40E_VP_MDET_TX(i), 0xFFFF); 7219 vf->num_mdd_events++; 7220 dev_info(&pf->pdev->dev, "TX driver issue detected on VF %d\n", 7221 i); 7222 } 7223 7224 reg = rd32(hw, I40E_VP_MDET_RX(i)); 7225 if (reg & I40E_VP_MDET_RX_VALID_MASK) { 7226 wr32(hw, I40E_VP_MDET_RX(i), 0xFFFF); 7227 vf->num_mdd_events++; 7228 dev_info(&pf->pdev->dev, "RX driver issue detected on VF %d\n", 7229 i); 7230 } 7231 7232 if (vf->num_mdd_events > I40E_DEFAULT_NUM_MDD_EVENTS_ALLOWED) { 7233 dev_info(&pf->pdev->dev, 7234 "Too many MDD events on VF %d, disabled\n", i); 7235 dev_info(&pf->pdev->dev, 7236 "Use PF Control I/F to re-enable the VF\n"); 7237 set_bit(I40E_VF_STAT_DISABLED, &vf->vf_states); 7238 } 7239 } 7240 7241 /* re-enable mdd interrupt cause */ 7242 clear_bit(__I40E_MDD_EVENT_PENDING, &pf->state); 7243 reg = rd32(hw, I40E_PFINT_ICR0_ENA); 7244 reg |= I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK; 7245 wr32(hw, I40E_PFINT_ICR0_ENA, reg); 7246 i40e_flush(hw); 7247 } 7248 7249 /** 7250 * i40e_sync_udp_filters_subtask - Sync the VSI filter list with HW 7251 * @pf: board private structure 7252 **/ 7253 static void i40e_sync_udp_filters_subtask(struct i40e_pf *pf) 7254 { 7255 struct i40e_hw *hw = &pf->hw; 7256 i40e_status ret; 7257 __be16 port; 7258 int i; 7259 7260 if (!(pf->flags & I40E_FLAG_UDP_FILTER_SYNC)) 7261 return; 7262 7263 pf->flags &= ~I40E_FLAG_UDP_FILTER_SYNC; 7264 7265 for (i = 0; i < I40E_MAX_PF_UDP_OFFLOAD_PORTS; i++) { 7266 if (pf->pending_udp_bitmap & BIT_ULL(i)) { 7267 pf->pending_udp_bitmap &= ~BIT_ULL(i); 7268 port = pf->udp_ports[i].index; 7269 if (port) 7270 ret = i40e_aq_add_udp_tunnel(hw, port, 7271 pf->udp_ports[i].type, 7272 NULL, NULL); 7273 else 7274 ret = i40e_aq_del_udp_tunnel(hw, i, NULL); 7275 7276 if (ret) { 7277 dev_dbg(&pf->pdev->dev, 7278 "%s %s port %d, index %d failed, err %s aq_err %s\n", 7279 pf->udp_ports[i].type ? "vxlan" : "geneve", 7280 port ? "add" : "delete", 7281 ntohs(port), i, 7282 i40e_stat_str(&pf->hw, ret), 7283 i40e_aq_str(&pf->hw, 7284 pf->hw.aq.asq_last_status)); 7285 pf->udp_ports[i].index = 0; 7286 } 7287 } 7288 } 7289 } 7290 7291 /** 7292 * i40e_service_task - Run the driver's async subtasks 7293 * @work: pointer to work_struct containing our data 7294 **/ 7295 static void i40e_service_task(struct work_struct *work) 7296 { 7297 struct i40e_pf *pf = container_of(work, 7298 struct i40e_pf, 7299 service_task); 7300 unsigned long start_time = jiffies; 7301 7302 /* don't bother with service tasks if a reset is in progress */ 7303 if (test_bit(__I40E_RESET_RECOVERY_PENDING, &pf->state)) { 7304 return; 7305 } 7306 7307 if (test_and_set_bit(__I40E_SERVICE_SCHED, &pf->state)) 7308 return; 7309 7310 i40e_detect_recover_hung(pf); 7311 i40e_sync_filters_subtask(pf); 7312 i40e_reset_subtask(pf); 7313 i40e_handle_mdd_event(pf); 7314 i40e_vc_process_vflr_event(pf); 7315 i40e_watchdog_subtask(pf); 7316 i40e_fdir_reinit_subtask(pf); 7317 i40e_client_subtask(pf); 7318 i40e_sync_filters_subtask(pf); 7319 i40e_sync_udp_filters_subtask(pf); 7320 i40e_clean_adminq_subtask(pf); 7321 7322 /* flush memory to make sure state is correct before next watchdog */ 7323 smp_mb__before_atomic(); 7324 clear_bit(__I40E_SERVICE_SCHED, &pf->state); 7325 7326 /* If the tasks have taken longer than one timer cycle or there 7327 * is more work to be done, reschedule the service task now 7328 * rather than wait for the timer to tick again. 7329 */ 7330 if (time_after(jiffies, (start_time + pf->service_timer_period)) || 7331 test_bit(__I40E_ADMINQ_EVENT_PENDING, &pf->state) || 7332 test_bit(__I40E_MDD_EVENT_PENDING, &pf->state) || 7333 test_bit(__I40E_VFLR_EVENT_PENDING, &pf->state)) 7334 i40e_service_event_schedule(pf); 7335 } 7336 7337 /** 7338 * i40e_service_timer - timer callback 7339 * @data: pointer to PF struct 7340 **/ 7341 static void i40e_service_timer(unsigned long data) 7342 { 7343 struct i40e_pf *pf = (struct i40e_pf *)data; 7344 7345 mod_timer(&pf->service_timer, 7346 round_jiffies(jiffies + pf->service_timer_period)); 7347 i40e_service_event_schedule(pf); 7348 } 7349 7350 /** 7351 * i40e_set_num_rings_in_vsi - Determine number of rings in the VSI 7352 * @vsi: the VSI being configured 7353 **/ 7354 static int i40e_set_num_rings_in_vsi(struct i40e_vsi *vsi) 7355 { 7356 struct i40e_pf *pf = vsi->back; 7357 7358 switch (vsi->type) { 7359 case I40E_VSI_MAIN: 7360 vsi->alloc_queue_pairs = pf->num_lan_qps; 7361 vsi->num_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS, 7362 I40E_REQ_DESCRIPTOR_MULTIPLE); 7363 if (pf->flags & I40E_FLAG_MSIX_ENABLED) 7364 vsi->num_q_vectors = pf->num_lan_msix; 7365 else 7366 vsi->num_q_vectors = 1; 7367 7368 break; 7369 7370 case I40E_VSI_FDIR: 7371 vsi->alloc_queue_pairs = 1; 7372 vsi->num_desc = ALIGN(I40E_FDIR_RING_COUNT, 7373 I40E_REQ_DESCRIPTOR_MULTIPLE); 7374 vsi->num_q_vectors = pf->num_fdsb_msix; 7375 break; 7376 7377 case I40E_VSI_VMDQ2: 7378 vsi->alloc_queue_pairs = pf->num_vmdq_qps; 7379 vsi->num_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS, 7380 I40E_REQ_DESCRIPTOR_MULTIPLE); 7381 vsi->num_q_vectors = pf->num_vmdq_msix; 7382 break; 7383 7384 case I40E_VSI_SRIOV: 7385 vsi->alloc_queue_pairs = pf->num_vf_qps; 7386 vsi->num_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS, 7387 I40E_REQ_DESCRIPTOR_MULTIPLE); 7388 break; 7389 7390 #ifdef I40E_FCOE 7391 case I40E_VSI_FCOE: 7392 vsi->alloc_queue_pairs = pf->num_fcoe_qps; 7393 vsi->num_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS, 7394 I40E_REQ_DESCRIPTOR_MULTIPLE); 7395 vsi->num_q_vectors = pf->num_fcoe_msix; 7396 break; 7397 7398 #endif /* I40E_FCOE */ 7399 default: 7400 WARN_ON(1); 7401 return -ENODATA; 7402 } 7403 7404 return 0; 7405 } 7406 7407 /** 7408 * i40e_vsi_alloc_arrays - Allocate queue and vector pointer arrays for the vsi 7409 * @type: VSI pointer 7410 * @alloc_qvectors: a bool to specify if q_vectors need to be allocated. 7411 * 7412 * On error: returns error code (negative) 7413 * On success: returns 0 7414 **/ 7415 static int i40e_vsi_alloc_arrays(struct i40e_vsi *vsi, bool alloc_qvectors) 7416 { 7417 int size; 7418 int ret = 0; 7419 7420 /* allocate memory for both Tx and Rx ring pointers */ 7421 size = sizeof(struct i40e_ring *) * vsi->alloc_queue_pairs * 2; 7422 vsi->tx_rings = kzalloc(size, GFP_KERNEL); 7423 if (!vsi->tx_rings) 7424 return -ENOMEM; 7425 vsi->rx_rings = &vsi->tx_rings[vsi->alloc_queue_pairs]; 7426 7427 if (alloc_qvectors) { 7428 /* allocate memory for q_vector pointers */ 7429 size = sizeof(struct i40e_q_vector *) * vsi->num_q_vectors; 7430 vsi->q_vectors = kzalloc(size, GFP_KERNEL); 7431 if (!vsi->q_vectors) { 7432 ret = -ENOMEM; 7433 goto err_vectors; 7434 } 7435 } 7436 return ret; 7437 7438 err_vectors: 7439 kfree(vsi->tx_rings); 7440 return ret; 7441 } 7442 7443 /** 7444 * i40e_vsi_mem_alloc - Allocates the next available struct vsi in the PF 7445 * @pf: board private structure 7446 * @type: type of VSI 7447 * 7448 * On error: returns error code (negative) 7449 * On success: returns vsi index in PF (positive) 7450 **/ 7451 static int i40e_vsi_mem_alloc(struct i40e_pf *pf, enum i40e_vsi_type type) 7452 { 7453 int ret = -ENODEV; 7454 struct i40e_vsi *vsi; 7455 int vsi_idx; 7456 int i; 7457 7458 /* Need to protect the allocation of the VSIs at the PF level */ 7459 mutex_lock(&pf->switch_mutex); 7460 7461 /* VSI list may be fragmented if VSI creation/destruction has 7462 * been happening. We can afford to do a quick scan to look 7463 * for any free VSIs in the list. 7464 * 7465 * find next empty vsi slot, looping back around if necessary 7466 */ 7467 i = pf->next_vsi; 7468 while (i < pf->num_alloc_vsi && pf->vsi[i]) 7469 i++; 7470 if (i >= pf->num_alloc_vsi) { 7471 i = 0; 7472 while (i < pf->next_vsi && pf->vsi[i]) 7473 i++; 7474 } 7475 7476 if (i < pf->num_alloc_vsi && !pf->vsi[i]) { 7477 vsi_idx = i; /* Found one! */ 7478 } else { 7479 ret = -ENODEV; 7480 goto unlock_pf; /* out of VSI slots! */ 7481 } 7482 pf->next_vsi = ++i; 7483 7484 vsi = kzalloc(sizeof(*vsi), GFP_KERNEL); 7485 if (!vsi) { 7486 ret = -ENOMEM; 7487 goto unlock_pf; 7488 } 7489 vsi->type = type; 7490 vsi->back = pf; 7491 set_bit(__I40E_DOWN, &vsi->state); 7492 vsi->flags = 0; 7493 vsi->idx = vsi_idx; 7494 vsi->int_rate_limit = 0; 7495 vsi->rss_table_size = (vsi->type == I40E_VSI_MAIN) ? 7496 pf->rss_table_size : 64; 7497 vsi->netdev_registered = false; 7498 vsi->work_limit = I40E_DEFAULT_IRQ_WORK; 7499 hash_init(vsi->mac_filter_hash); 7500 vsi->irqs_ready = false; 7501 7502 ret = i40e_set_num_rings_in_vsi(vsi); 7503 if (ret) 7504 goto err_rings; 7505 7506 ret = i40e_vsi_alloc_arrays(vsi, true); 7507 if (ret) 7508 goto err_rings; 7509 7510 /* Setup default MSIX irq handler for VSI */ 7511 i40e_vsi_setup_irqhandler(vsi, i40e_msix_clean_rings); 7512 7513 /* Initialize VSI lock */ 7514 spin_lock_init(&vsi->mac_filter_hash_lock); 7515 pf->vsi[vsi_idx] = vsi; 7516 ret = vsi_idx; 7517 goto unlock_pf; 7518 7519 err_rings: 7520 pf->next_vsi = i - 1; 7521 kfree(vsi); 7522 unlock_pf: 7523 mutex_unlock(&pf->switch_mutex); 7524 return ret; 7525 } 7526 7527 /** 7528 * i40e_vsi_free_arrays - Free queue and vector pointer arrays for the VSI 7529 * @type: VSI pointer 7530 * @free_qvectors: a bool to specify if q_vectors need to be freed. 7531 * 7532 * On error: returns error code (negative) 7533 * On success: returns 0 7534 **/ 7535 static void i40e_vsi_free_arrays(struct i40e_vsi *vsi, bool free_qvectors) 7536 { 7537 /* free the ring and vector containers */ 7538 if (free_qvectors) { 7539 kfree(vsi->q_vectors); 7540 vsi->q_vectors = NULL; 7541 } 7542 kfree(vsi->tx_rings); 7543 vsi->tx_rings = NULL; 7544 vsi->rx_rings = NULL; 7545 } 7546 7547 /** 7548 * i40e_clear_rss_config_user - clear the user configured RSS hash keys 7549 * and lookup table 7550 * @vsi: Pointer to VSI structure 7551 */ 7552 static void i40e_clear_rss_config_user(struct i40e_vsi *vsi) 7553 { 7554 if (!vsi) 7555 return; 7556 7557 kfree(vsi->rss_hkey_user); 7558 vsi->rss_hkey_user = NULL; 7559 7560 kfree(vsi->rss_lut_user); 7561 vsi->rss_lut_user = NULL; 7562 } 7563 7564 /** 7565 * i40e_vsi_clear - Deallocate the VSI provided 7566 * @vsi: the VSI being un-configured 7567 **/ 7568 static int i40e_vsi_clear(struct i40e_vsi *vsi) 7569 { 7570 struct i40e_pf *pf; 7571 7572 if (!vsi) 7573 return 0; 7574 7575 if (!vsi->back) 7576 goto free_vsi; 7577 pf = vsi->back; 7578 7579 mutex_lock(&pf->switch_mutex); 7580 if (!pf->vsi[vsi->idx]) { 7581 dev_err(&pf->pdev->dev, "pf->vsi[%d] is NULL, just free vsi[%d](%p,type %d)\n", 7582 vsi->idx, vsi->idx, vsi, vsi->type); 7583 goto unlock_vsi; 7584 } 7585 7586 if (pf->vsi[vsi->idx] != vsi) { 7587 dev_err(&pf->pdev->dev, 7588 "pf->vsi[%d](%p, type %d) != vsi[%d](%p,type %d): no free!\n", 7589 pf->vsi[vsi->idx]->idx, 7590 pf->vsi[vsi->idx], 7591 pf->vsi[vsi->idx]->type, 7592 vsi->idx, vsi, vsi->type); 7593 goto unlock_vsi; 7594 } 7595 7596 /* updates the PF for this cleared vsi */ 7597 i40e_put_lump(pf->qp_pile, vsi->base_queue, vsi->idx); 7598 i40e_put_lump(pf->irq_pile, vsi->base_vector, vsi->idx); 7599 7600 i40e_vsi_free_arrays(vsi, true); 7601 i40e_clear_rss_config_user(vsi); 7602 7603 pf->vsi[vsi->idx] = NULL; 7604 if (vsi->idx < pf->next_vsi) 7605 pf->next_vsi = vsi->idx; 7606 7607 unlock_vsi: 7608 mutex_unlock(&pf->switch_mutex); 7609 free_vsi: 7610 kfree(vsi); 7611 7612 return 0; 7613 } 7614 7615 /** 7616 * i40e_vsi_clear_rings - Deallocates the Rx and Tx rings for the provided VSI 7617 * @vsi: the VSI being cleaned 7618 **/ 7619 static void i40e_vsi_clear_rings(struct i40e_vsi *vsi) 7620 { 7621 int i; 7622 7623 if (vsi->tx_rings && vsi->tx_rings[0]) { 7624 for (i = 0; i < vsi->alloc_queue_pairs; i++) { 7625 kfree_rcu(vsi->tx_rings[i], rcu); 7626 vsi->tx_rings[i] = NULL; 7627 vsi->rx_rings[i] = NULL; 7628 } 7629 } 7630 } 7631 7632 /** 7633 * i40e_alloc_rings - Allocates the Rx and Tx rings for the provided VSI 7634 * @vsi: the VSI being configured 7635 **/ 7636 static int i40e_alloc_rings(struct i40e_vsi *vsi) 7637 { 7638 struct i40e_ring *tx_ring, *rx_ring; 7639 struct i40e_pf *pf = vsi->back; 7640 int i; 7641 7642 /* Set basic values in the rings to be used later during open() */ 7643 for (i = 0; i < vsi->alloc_queue_pairs; i++) { 7644 /* allocate space for both Tx and Rx in one shot */ 7645 tx_ring = kzalloc(sizeof(struct i40e_ring) * 2, GFP_KERNEL); 7646 if (!tx_ring) 7647 goto err_out; 7648 7649 tx_ring->queue_index = i; 7650 tx_ring->reg_idx = vsi->base_queue + i; 7651 tx_ring->ring_active = false; 7652 tx_ring->vsi = vsi; 7653 tx_ring->netdev = vsi->netdev; 7654 tx_ring->dev = &pf->pdev->dev; 7655 tx_ring->count = vsi->num_desc; 7656 tx_ring->size = 0; 7657 tx_ring->dcb_tc = 0; 7658 if (vsi->back->flags & I40E_FLAG_WB_ON_ITR_CAPABLE) 7659 tx_ring->flags = I40E_TXR_FLAGS_WB_ON_ITR; 7660 tx_ring->tx_itr_setting = pf->tx_itr_default; 7661 vsi->tx_rings[i] = tx_ring; 7662 7663 rx_ring = &tx_ring[1]; 7664 rx_ring->queue_index = i; 7665 rx_ring->reg_idx = vsi->base_queue + i; 7666 rx_ring->ring_active = false; 7667 rx_ring->vsi = vsi; 7668 rx_ring->netdev = vsi->netdev; 7669 rx_ring->dev = &pf->pdev->dev; 7670 rx_ring->count = vsi->num_desc; 7671 rx_ring->size = 0; 7672 rx_ring->dcb_tc = 0; 7673 rx_ring->rx_itr_setting = pf->rx_itr_default; 7674 vsi->rx_rings[i] = rx_ring; 7675 } 7676 7677 return 0; 7678 7679 err_out: 7680 i40e_vsi_clear_rings(vsi); 7681 return -ENOMEM; 7682 } 7683 7684 /** 7685 * i40e_reserve_msix_vectors - Reserve MSI-X vectors in the kernel 7686 * @pf: board private structure 7687 * @vectors: the number of MSI-X vectors to request 7688 * 7689 * Returns the number of vectors reserved, or error 7690 **/ 7691 static int i40e_reserve_msix_vectors(struct i40e_pf *pf, int vectors) 7692 { 7693 vectors = pci_enable_msix_range(pf->pdev, pf->msix_entries, 7694 I40E_MIN_MSIX, vectors); 7695 if (vectors < 0) { 7696 dev_info(&pf->pdev->dev, 7697 "MSI-X vector reservation failed: %d\n", vectors); 7698 vectors = 0; 7699 } 7700 7701 return vectors; 7702 } 7703 7704 /** 7705 * i40e_init_msix - Setup the MSIX capability 7706 * @pf: board private structure 7707 * 7708 * Work with the OS to set up the MSIX vectors needed. 7709 * 7710 * Returns the number of vectors reserved or negative on failure 7711 **/ 7712 static int i40e_init_msix(struct i40e_pf *pf) 7713 { 7714 struct i40e_hw *hw = &pf->hw; 7715 int vectors_left; 7716 int v_budget, i; 7717 int v_actual; 7718 int iwarp_requested = 0; 7719 7720 if (!(pf->flags & I40E_FLAG_MSIX_ENABLED)) 7721 return -ENODEV; 7722 7723 /* The number of vectors we'll request will be comprised of: 7724 * - Add 1 for "other" cause for Admin Queue events, etc. 7725 * - The number of LAN queue pairs 7726 * - Queues being used for RSS. 7727 * We don't need as many as max_rss_size vectors. 7728 * use rss_size instead in the calculation since that 7729 * is governed by number of cpus in the system. 7730 * - assumes symmetric Tx/Rx pairing 7731 * - The number of VMDq pairs 7732 * - The CPU count within the NUMA node if iWARP is enabled 7733 #ifdef I40E_FCOE 7734 * - The number of FCOE qps. 7735 #endif 7736 * Once we count this up, try the request. 7737 * 7738 * If we can't get what we want, we'll simplify to nearly nothing 7739 * and try again. If that still fails, we punt. 7740 */ 7741 vectors_left = hw->func_caps.num_msix_vectors; 7742 v_budget = 0; 7743 7744 /* reserve one vector for miscellaneous handler */ 7745 if (vectors_left) { 7746 v_budget++; 7747 vectors_left--; 7748 } 7749 7750 /* reserve vectors for the main PF traffic queues */ 7751 pf->num_lan_msix = min_t(int, num_online_cpus(), vectors_left); 7752 vectors_left -= pf->num_lan_msix; 7753 v_budget += pf->num_lan_msix; 7754 7755 /* reserve one vector for sideband flow director */ 7756 if (pf->flags & I40E_FLAG_FD_SB_ENABLED) { 7757 if (vectors_left) { 7758 pf->num_fdsb_msix = 1; 7759 v_budget++; 7760 vectors_left--; 7761 } else { 7762 pf->num_fdsb_msix = 0; 7763 } 7764 } 7765 7766 #ifdef I40E_FCOE 7767 /* can we reserve enough for FCoE? */ 7768 if (pf->flags & I40E_FLAG_FCOE_ENABLED) { 7769 if (!vectors_left) 7770 pf->num_fcoe_msix = 0; 7771 else if (vectors_left >= pf->num_fcoe_qps) 7772 pf->num_fcoe_msix = pf->num_fcoe_qps; 7773 else 7774 pf->num_fcoe_msix = 1; 7775 v_budget += pf->num_fcoe_msix; 7776 vectors_left -= pf->num_fcoe_msix; 7777 } 7778 7779 #endif 7780 /* can we reserve enough for iWARP? */ 7781 if (pf->flags & I40E_FLAG_IWARP_ENABLED) { 7782 iwarp_requested = pf->num_iwarp_msix; 7783 7784 if (!vectors_left) 7785 pf->num_iwarp_msix = 0; 7786 else if (vectors_left < pf->num_iwarp_msix) 7787 pf->num_iwarp_msix = 1; 7788 v_budget += pf->num_iwarp_msix; 7789 vectors_left -= pf->num_iwarp_msix; 7790 } 7791 7792 /* any vectors left over go for VMDq support */ 7793 if (pf->flags & I40E_FLAG_VMDQ_ENABLED) { 7794 int vmdq_vecs_wanted = pf->num_vmdq_vsis * pf->num_vmdq_qps; 7795 int vmdq_vecs = min_t(int, vectors_left, vmdq_vecs_wanted); 7796 7797 if (!vectors_left) { 7798 pf->num_vmdq_msix = 0; 7799 pf->num_vmdq_qps = 0; 7800 } else { 7801 /* if we're short on vectors for what's desired, we limit 7802 * the queues per vmdq. If this is still more than are 7803 * available, the user will need to change the number of 7804 * queues/vectors used by the PF later with the ethtool 7805 * channels command 7806 */ 7807 if (vmdq_vecs < vmdq_vecs_wanted) 7808 pf->num_vmdq_qps = 1; 7809 pf->num_vmdq_msix = pf->num_vmdq_qps; 7810 7811 v_budget += vmdq_vecs; 7812 vectors_left -= vmdq_vecs; 7813 } 7814 } 7815 7816 pf->msix_entries = kcalloc(v_budget, sizeof(struct msix_entry), 7817 GFP_KERNEL); 7818 if (!pf->msix_entries) 7819 return -ENOMEM; 7820 7821 for (i = 0; i < v_budget; i++) 7822 pf->msix_entries[i].entry = i; 7823 v_actual = i40e_reserve_msix_vectors(pf, v_budget); 7824 7825 if (v_actual < I40E_MIN_MSIX) { 7826 pf->flags &= ~I40E_FLAG_MSIX_ENABLED; 7827 kfree(pf->msix_entries); 7828 pf->msix_entries = NULL; 7829 pci_disable_msix(pf->pdev); 7830 return -ENODEV; 7831 7832 } else if (v_actual == I40E_MIN_MSIX) { 7833 /* Adjust for minimal MSIX use */ 7834 pf->num_vmdq_vsis = 0; 7835 pf->num_vmdq_qps = 0; 7836 pf->num_lan_qps = 1; 7837 pf->num_lan_msix = 1; 7838 7839 } else if (!vectors_left) { 7840 /* If we have limited resources, we will start with no vectors 7841 * for the special features and then allocate vectors to some 7842 * of these features based on the policy and at the end disable 7843 * the features that did not get any vectors. 7844 */ 7845 int vec; 7846 7847 dev_info(&pf->pdev->dev, 7848 "MSI-X vector limit reached, attempting to redistribute vectors\n"); 7849 /* reserve the misc vector */ 7850 vec = v_actual - 1; 7851 7852 /* Scale vector usage down */ 7853 pf->num_vmdq_msix = 1; /* force VMDqs to only one vector */ 7854 pf->num_vmdq_vsis = 1; 7855 pf->num_vmdq_qps = 1; 7856 #ifdef I40E_FCOE 7857 pf->num_fcoe_qps = 0; 7858 pf->num_fcoe_msix = 0; 7859 #endif 7860 7861 /* partition out the remaining vectors */ 7862 switch (vec) { 7863 case 2: 7864 pf->num_lan_msix = 1; 7865 break; 7866 case 3: 7867 if (pf->flags & I40E_FLAG_IWARP_ENABLED) { 7868 pf->num_lan_msix = 1; 7869 pf->num_iwarp_msix = 1; 7870 } else { 7871 pf->num_lan_msix = 2; 7872 } 7873 #ifdef I40E_FCOE 7874 /* give one vector to FCoE */ 7875 if (pf->flags & I40E_FLAG_FCOE_ENABLED) { 7876 pf->num_lan_msix = 1; 7877 pf->num_fcoe_msix = 1; 7878 } 7879 #endif 7880 break; 7881 default: 7882 if (pf->flags & I40E_FLAG_IWARP_ENABLED) { 7883 pf->num_iwarp_msix = min_t(int, (vec / 3), 7884 iwarp_requested); 7885 pf->num_vmdq_vsis = min_t(int, (vec / 3), 7886 I40E_DEFAULT_NUM_VMDQ_VSI); 7887 } else { 7888 pf->num_vmdq_vsis = min_t(int, (vec / 2), 7889 I40E_DEFAULT_NUM_VMDQ_VSI); 7890 } 7891 if (pf->flags & I40E_FLAG_FD_SB_ENABLED) { 7892 pf->num_fdsb_msix = 1; 7893 vec--; 7894 } 7895 pf->num_lan_msix = min_t(int, 7896 (vec - (pf->num_iwarp_msix + pf->num_vmdq_vsis)), 7897 pf->num_lan_msix); 7898 pf->num_lan_qps = pf->num_lan_msix; 7899 #ifdef I40E_FCOE 7900 /* give one vector to FCoE */ 7901 if (pf->flags & I40E_FLAG_FCOE_ENABLED) { 7902 pf->num_fcoe_msix = 1; 7903 vec--; 7904 } 7905 #endif 7906 break; 7907 } 7908 } 7909 7910 if ((pf->flags & I40E_FLAG_FD_SB_ENABLED) && 7911 (pf->num_fdsb_msix == 0)) { 7912 dev_info(&pf->pdev->dev, "Sideband Flowdir disabled, not enough MSI-X vectors\n"); 7913 pf->flags &= ~I40E_FLAG_FD_SB_ENABLED; 7914 } 7915 if ((pf->flags & I40E_FLAG_VMDQ_ENABLED) && 7916 (pf->num_vmdq_msix == 0)) { 7917 dev_info(&pf->pdev->dev, "VMDq disabled, not enough MSI-X vectors\n"); 7918 pf->flags &= ~I40E_FLAG_VMDQ_ENABLED; 7919 } 7920 7921 if ((pf->flags & I40E_FLAG_IWARP_ENABLED) && 7922 (pf->num_iwarp_msix == 0)) { 7923 dev_info(&pf->pdev->dev, "IWARP disabled, not enough MSI-X vectors\n"); 7924 pf->flags &= ~I40E_FLAG_IWARP_ENABLED; 7925 } 7926 #ifdef I40E_FCOE 7927 7928 if ((pf->flags & I40E_FLAG_FCOE_ENABLED) && (pf->num_fcoe_msix == 0)) { 7929 dev_info(&pf->pdev->dev, "FCOE disabled, not enough MSI-X vectors\n"); 7930 pf->flags &= ~I40E_FLAG_FCOE_ENABLED; 7931 } 7932 #endif 7933 i40e_debug(&pf->hw, I40E_DEBUG_INIT, 7934 "MSI-X vector distribution: PF %d, VMDq %d, FDSB %d, iWARP %d\n", 7935 pf->num_lan_msix, 7936 pf->num_vmdq_msix * pf->num_vmdq_vsis, 7937 pf->num_fdsb_msix, 7938 pf->num_iwarp_msix); 7939 7940 return v_actual; 7941 } 7942 7943 /** 7944 * i40e_vsi_alloc_q_vector - Allocate memory for a single interrupt vector 7945 * @vsi: the VSI being configured 7946 * @v_idx: index of the vector in the vsi struct 7947 * @cpu: cpu to be used on affinity_mask 7948 * 7949 * We allocate one q_vector. If allocation fails we return -ENOMEM. 7950 **/ 7951 static int i40e_vsi_alloc_q_vector(struct i40e_vsi *vsi, int v_idx, int cpu) 7952 { 7953 struct i40e_q_vector *q_vector; 7954 7955 /* allocate q_vector */ 7956 q_vector = kzalloc(sizeof(struct i40e_q_vector), GFP_KERNEL); 7957 if (!q_vector) 7958 return -ENOMEM; 7959 7960 q_vector->vsi = vsi; 7961 q_vector->v_idx = v_idx; 7962 cpumask_set_cpu(cpu, &q_vector->affinity_mask); 7963 7964 if (vsi->netdev) 7965 netif_napi_add(vsi->netdev, &q_vector->napi, 7966 i40e_napi_poll, NAPI_POLL_WEIGHT); 7967 7968 q_vector->rx.latency_range = I40E_LOW_LATENCY; 7969 q_vector->tx.latency_range = I40E_LOW_LATENCY; 7970 7971 /* tie q_vector and vsi together */ 7972 vsi->q_vectors[v_idx] = q_vector; 7973 7974 return 0; 7975 } 7976 7977 /** 7978 * i40e_vsi_alloc_q_vectors - Allocate memory for interrupt vectors 7979 * @vsi: the VSI being configured 7980 * 7981 * We allocate one q_vector per queue interrupt. If allocation fails we 7982 * return -ENOMEM. 7983 **/ 7984 static int i40e_vsi_alloc_q_vectors(struct i40e_vsi *vsi) 7985 { 7986 struct i40e_pf *pf = vsi->back; 7987 int err, v_idx, num_q_vectors, current_cpu; 7988 7989 /* if not MSIX, give the one vector only to the LAN VSI */ 7990 if (pf->flags & I40E_FLAG_MSIX_ENABLED) 7991 num_q_vectors = vsi->num_q_vectors; 7992 else if (vsi == pf->vsi[pf->lan_vsi]) 7993 num_q_vectors = 1; 7994 else 7995 return -EINVAL; 7996 7997 current_cpu = cpumask_first(cpu_online_mask); 7998 7999 for (v_idx = 0; v_idx < num_q_vectors; v_idx++) { 8000 err = i40e_vsi_alloc_q_vector(vsi, v_idx, current_cpu); 8001 if (err) 8002 goto err_out; 8003 current_cpu = cpumask_next(current_cpu, cpu_online_mask); 8004 if (unlikely(current_cpu >= nr_cpu_ids)) 8005 current_cpu = cpumask_first(cpu_online_mask); 8006 } 8007 8008 return 0; 8009 8010 err_out: 8011 while (v_idx--) 8012 i40e_free_q_vector(vsi, v_idx); 8013 8014 return err; 8015 } 8016 8017 /** 8018 * i40e_init_interrupt_scheme - Determine proper interrupt scheme 8019 * @pf: board private structure to initialize 8020 **/ 8021 static int i40e_init_interrupt_scheme(struct i40e_pf *pf) 8022 { 8023 int vectors = 0; 8024 ssize_t size; 8025 8026 if (pf->flags & I40E_FLAG_MSIX_ENABLED) { 8027 vectors = i40e_init_msix(pf); 8028 if (vectors < 0) { 8029 pf->flags &= ~(I40E_FLAG_MSIX_ENABLED | 8030 I40E_FLAG_IWARP_ENABLED | 8031 #ifdef I40E_FCOE 8032 I40E_FLAG_FCOE_ENABLED | 8033 #endif 8034 I40E_FLAG_RSS_ENABLED | 8035 I40E_FLAG_DCB_CAPABLE | 8036 I40E_FLAG_DCB_ENABLED | 8037 I40E_FLAG_SRIOV_ENABLED | 8038 I40E_FLAG_FD_SB_ENABLED | 8039 I40E_FLAG_FD_ATR_ENABLED | 8040 I40E_FLAG_VMDQ_ENABLED); 8041 8042 /* rework the queue expectations without MSIX */ 8043 i40e_determine_queue_usage(pf); 8044 } 8045 } 8046 8047 if (!(pf->flags & I40E_FLAG_MSIX_ENABLED) && 8048 (pf->flags & I40E_FLAG_MSI_ENABLED)) { 8049 dev_info(&pf->pdev->dev, "MSI-X not available, trying MSI\n"); 8050 vectors = pci_enable_msi(pf->pdev); 8051 if (vectors < 0) { 8052 dev_info(&pf->pdev->dev, "MSI init failed - %d\n", 8053 vectors); 8054 pf->flags &= ~I40E_FLAG_MSI_ENABLED; 8055 } 8056 vectors = 1; /* one MSI or Legacy vector */ 8057 } 8058 8059 if (!(pf->flags & (I40E_FLAG_MSIX_ENABLED | I40E_FLAG_MSI_ENABLED))) 8060 dev_info(&pf->pdev->dev, "MSI-X and MSI not available, falling back to Legacy IRQ\n"); 8061 8062 /* set up vector assignment tracking */ 8063 size = sizeof(struct i40e_lump_tracking) + (sizeof(u16) * vectors); 8064 pf->irq_pile = kzalloc(size, GFP_KERNEL); 8065 if (!pf->irq_pile) { 8066 dev_err(&pf->pdev->dev, "error allocating irq_pile memory\n"); 8067 return -ENOMEM; 8068 } 8069 pf->irq_pile->num_entries = vectors; 8070 pf->irq_pile->search_hint = 0; 8071 8072 /* track first vector for misc interrupts, ignore return */ 8073 (void)i40e_get_lump(pf, pf->irq_pile, 1, I40E_PILE_VALID_BIT - 1); 8074 8075 return 0; 8076 } 8077 8078 /** 8079 * i40e_setup_misc_vector - Setup the misc vector to handle non queue events 8080 * @pf: board private structure 8081 * 8082 * This sets up the handler for MSIX 0, which is used to manage the 8083 * non-queue interrupts, e.g. AdminQ and errors. This is not used 8084 * when in MSI or Legacy interrupt mode. 8085 **/ 8086 static int i40e_setup_misc_vector(struct i40e_pf *pf) 8087 { 8088 struct i40e_hw *hw = &pf->hw; 8089 int err = 0; 8090 8091 /* Only request the irq if this is the first time through, and 8092 * not when we're rebuilding after a Reset 8093 */ 8094 if (!test_bit(__I40E_RESET_RECOVERY_PENDING, &pf->state)) { 8095 err = request_irq(pf->msix_entries[0].vector, 8096 i40e_intr, 0, pf->int_name, pf); 8097 if (err) { 8098 dev_info(&pf->pdev->dev, 8099 "request_irq for %s failed: %d\n", 8100 pf->int_name, err); 8101 return -EFAULT; 8102 } 8103 } 8104 8105 i40e_enable_misc_int_causes(pf); 8106 8107 /* associate no queues to the misc vector */ 8108 wr32(hw, I40E_PFINT_LNKLST0, I40E_QUEUE_END_OF_LIST); 8109 wr32(hw, I40E_PFINT_ITR0(I40E_RX_ITR), I40E_ITR_8K); 8110 8111 i40e_flush(hw); 8112 8113 i40e_irq_dynamic_enable_icr0(pf, true); 8114 8115 return err; 8116 } 8117 8118 /** 8119 * i40e_config_rss_aq - Prepare for RSS using AQ commands 8120 * @vsi: vsi structure 8121 * @seed: RSS hash seed 8122 **/ 8123 static int i40e_config_rss_aq(struct i40e_vsi *vsi, const u8 *seed, 8124 u8 *lut, u16 lut_size) 8125 { 8126 struct i40e_pf *pf = vsi->back; 8127 struct i40e_hw *hw = &pf->hw; 8128 int ret = 0; 8129 8130 if (seed) { 8131 struct i40e_aqc_get_set_rss_key_data *seed_dw = 8132 (struct i40e_aqc_get_set_rss_key_data *)seed; 8133 ret = i40e_aq_set_rss_key(hw, vsi->id, seed_dw); 8134 if (ret) { 8135 dev_info(&pf->pdev->dev, 8136 "Cannot set RSS key, err %s aq_err %s\n", 8137 i40e_stat_str(hw, ret), 8138 i40e_aq_str(hw, hw->aq.asq_last_status)); 8139 return ret; 8140 } 8141 } 8142 if (lut) { 8143 bool pf_lut = vsi->type == I40E_VSI_MAIN ? true : false; 8144 8145 ret = i40e_aq_set_rss_lut(hw, vsi->id, pf_lut, lut, lut_size); 8146 if (ret) { 8147 dev_info(&pf->pdev->dev, 8148 "Cannot set RSS lut, err %s aq_err %s\n", 8149 i40e_stat_str(hw, ret), 8150 i40e_aq_str(hw, hw->aq.asq_last_status)); 8151 return ret; 8152 } 8153 } 8154 return ret; 8155 } 8156 8157 /** 8158 * i40e_get_rss_aq - Get RSS keys and lut by using AQ commands 8159 * @vsi: Pointer to vsi structure 8160 * @seed: Buffter to store the hash keys 8161 * @lut: Buffer to store the lookup table entries 8162 * @lut_size: Size of buffer to store the lookup table entries 8163 * 8164 * Return 0 on success, negative on failure 8165 */ 8166 static int i40e_get_rss_aq(struct i40e_vsi *vsi, const u8 *seed, 8167 u8 *lut, u16 lut_size) 8168 { 8169 struct i40e_pf *pf = vsi->back; 8170 struct i40e_hw *hw = &pf->hw; 8171 int ret = 0; 8172 8173 if (seed) { 8174 ret = i40e_aq_get_rss_key(hw, vsi->id, 8175 (struct i40e_aqc_get_set_rss_key_data *)seed); 8176 if (ret) { 8177 dev_info(&pf->pdev->dev, 8178 "Cannot get RSS key, err %s aq_err %s\n", 8179 i40e_stat_str(&pf->hw, ret), 8180 i40e_aq_str(&pf->hw, 8181 pf->hw.aq.asq_last_status)); 8182 return ret; 8183 } 8184 } 8185 8186 if (lut) { 8187 bool pf_lut = vsi->type == I40E_VSI_MAIN ? true : false; 8188 8189 ret = i40e_aq_get_rss_lut(hw, vsi->id, pf_lut, lut, lut_size); 8190 if (ret) { 8191 dev_info(&pf->pdev->dev, 8192 "Cannot get RSS lut, err %s aq_err %s\n", 8193 i40e_stat_str(&pf->hw, ret), 8194 i40e_aq_str(&pf->hw, 8195 pf->hw.aq.asq_last_status)); 8196 return ret; 8197 } 8198 } 8199 8200 return ret; 8201 } 8202 8203 /** 8204 * i40e_vsi_config_rss - Prepare for VSI(VMDq) RSS if used 8205 * @vsi: VSI structure 8206 **/ 8207 static int i40e_vsi_config_rss(struct i40e_vsi *vsi) 8208 { 8209 u8 seed[I40E_HKEY_ARRAY_SIZE]; 8210 struct i40e_pf *pf = vsi->back; 8211 u8 *lut; 8212 int ret; 8213 8214 if (!(pf->flags & I40E_FLAG_RSS_AQ_CAPABLE)) 8215 return 0; 8216 8217 if (!vsi->rss_size) 8218 vsi->rss_size = min_t(int, pf->alloc_rss_size, 8219 vsi->num_queue_pairs); 8220 if (!vsi->rss_size) 8221 return -EINVAL; 8222 8223 lut = kzalloc(vsi->rss_table_size, GFP_KERNEL); 8224 if (!lut) 8225 return -ENOMEM; 8226 /* Use the user configured hash keys and lookup table if there is one, 8227 * otherwise use default 8228 */ 8229 if (vsi->rss_lut_user) 8230 memcpy(lut, vsi->rss_lut_user, vsi->rss_table_size); 8231 else 8232 i40e_fill_rss_lut(pf, lut, vsi->rss_table_size, vsi->rss_size); 8233 if (vsi->rss_hkey_user) 8234 memcpy(seed, vsi->rss_hkey_user, I40E_HKEY_ARRAY_SIZE); 8235 else 8236 netdev_rss_key_fill((void *)seed, I40E_HKEY_ARRAY_SIZE); 8237 ret = i40e_config_rss_aq(vsi, seed, lut, vsi->rss_table_size); 8238 kfree(lut); 8239 8240 return ret; 8241 } 8242 8243 /** 8244 * i40e_config_rss_reg - Configure RSS keys and lut by writing registers 8245 * @vsi: Pointer to vsi structure 8246 * @seed: RSS hash seed 8247 * @lut: Lookup table 8248 * @lut_size: Lookup table size 8249 * 8250 * Returns 0 on success, negative on failure 8251 **/ 8252 static int i40e_config_rss_reg(struct i40e_vsi *vsi, const u8 *seed, 8253 const u8 *lut, u16 lut_size) 8254 { 8255 struct i40e_pf *pf = vsi->back; 8256 struct i40e_hw *hw = &pf->hw; 8257 u16 vf_id = vsi->vf_id; 8258 u8 i; 8259 8260 /* Fill out hash function seed */ 8261 if (seed) { 8262 u32 *seed_dw = (u32 *)seed; 8263 8264 if (vsi->type == I40E_VSI_MAIN) { 8265 for (i = 0; i <= I40E_PFQF_HKEY_MAX_INDEX; i++) 8266 i40e_write_rx_ctl(hw, I40E_PFQF_HKEY(i), 8267 seed_dw[i]); 8268 } else if (vsi->type == I40E_VSI_SRIOV) { 8269 for (i = 0; i <= I40E_VFQF_HKEY1_MAX_INDEX; i++) 8270 i40e_write_rx_ctl(hw, 8271 I40E_VFQF_HKEY1(i, vf_id), 8272 seed_dw[i]); 8273 } else { 8274 dev_err(&pf->pdev->dev, "Cannot set RSS seed - invalid VSI type\n"); 8275 } 8276 } 8277 8278 if (lut) { 8279 u32 *lut_dw = (u32 *)lut; 8280 8281 if (vsi->type == I40E_VSI_MAIN) { 8282 if (lut_size != I40E_HLUT_ARRAY_SIZE) 8283 return -EINVAL; 8284 for (i = 0; i <= I40E_PFQF_HLUT_MAX_INDEX; i++) 8285 wr32(hw, I40E_PFQF_HLUT(i), lut_dw[i]); 8286 } else if (vsi->type == I40E_VSI_SRIOV) { 8287 if (lut_size != I40E_VF_HLUT_ARRAY_SIZE) 8288 return -EINVAL; 8289 for (i = 0; i <= I40E_VFQF_HLUT_MAX_INDEX; i++) 8290 i40e_write_rx_ctl(hw, 8291 I40E_VFQF_HLUT1(i, vf_id), 8292 lut_dw[i]); 8293 } else { 8294 dev_err(&pf->pdev->dev, "Cannot set RSS LUT - invalid VSI type\n"); 8295 } 8296 } 8297 i40e_flush(hw); 8298 8299 return 0; 8300 } 8301 8302 /** 8303 * i40e_get_rss_reg - Get the RSS keys and lut by reading registers 8304 * @vsi: Pointer to VSI structure 8305 * @seed: Buffer to store the keys 8306 * @lut: Buffer to store the lookup table entries 8307 * @lut_size: Size of buffer to store the lookup table entries 8308 * 8309 * Returns 0 on success, negative on failure 8310 */ 8311 static int i40e_get_rss_reg(struct i40e_vsi *vsi, u8 *seed, 8312 u8 *lut, u16 lut_size) 8313 { 8314 struct i40e_pf *pf = vsi->back; 8315 struct i40e_hw *hw = &pf->hw; 8316 u16 i; 8317 8318 if (seed) { 8319 u32 *seed_dw = (u32 *)seed; 8320 8321 for (i = 0; i <= I40E_PFQF_HKEY_MAX_INDEX; i++) 8322 seed_dw[i] = i40e_read_rx_ctl(hw, I40E_PFQF_HKEY(i)); 8323 } 8324 if (lut) { 8325 u32 *lut_dw = (u32 *)lut; 8326 8327 if (lut_size != I40E_HLUT_ARRAY_SIZE) 8328 return -EINVAL; 8329 for (i = 0; i <= I40E_PFQF_HLUT_MAX_INDEX; i++) 8330 lut_dw[i] = rd32(hw, I40E_PFQF_HLUT(i)); 8331 } 8332 8333 return 0; 8334 } 8335 8336 /** 8337 * i40e_config_rss - Configure RSS keys and lut 8338 * @vsi: Pointer to VSI structure 8339 * @seed: RSS hash seed 8340 * @lut: Lookup table 8341 * @lut_size: Lookup table size 8342 * 8343 * Returns 0 on success, negative on failure 8344 */ 8345 int i40e_config_rss(struct i40e_vsi *vsi, u8 *seed, u8 *lut, u16 lut_size) 8346 { 8347 struct i40e_pf *pf = vsi->back; 8348 8349 if (pf->flags & I40E_FLAG_RSS_AQ_CAPABLE) 8350 return i40e_config_rss_aq(vsi, seed, lut, lut_size); 8351 else 8352 return i40e_config_rss_reg(vsi, seed, lut, lut_size); 8353 } 8354 8355 /** 8356 * i40e_get_rss - Get RSS keys and lut 8357 * @vsi: Pointer to VSI structure 8358 * @seed: Buffer to store the keys 8359 * @lut: Buffer to store the lookup table entries 8360 * lut_size: Size of buffer to store the lookup table entries 8361 * 8362 * Returns 0 on success, negative on failure 8363 */ 8364 int i40e_get_rss(struct i40e_vsi *vsi, u8 *seed, u8 *lut, u16 lut_size) 8365 { 8366 struct i40e_pf *pf = vsi->back; 8367 8368 if (pf->flags & I40E_FLAG_RSS_AQ_CAPABLE) 8369 return i40e_get_rss_aq(vsi, seed, lut, lut_size); 8370 else 8371 return i40e_get_rss_reg(vsi, seed, lut, lut_size); 8372 } 8373 8374 /** 8375 * i40e_fill_rss_lut - Fill the RSS lookup table with default values 8376 * @pf: Pointer to board private structure 8377 * @lut: Lookup table 8378 * @rss_table_size: Lookup table size 8379 * @rss_size: Range of queue number for hashing 8380 */ 8381 void i40e_fill_rss_lut(struct i40e_pf *pf, u8 *lut, 8382 u16 rss_table_size, u16 rss_size) 8383 { 8384 u16 i; 8385 8386 for (i = 0; i < rss_table_size; i++) 8387 lut[i] = i % rss_size; 8388 } 8389 8390 /** 8391 * i40e_pf_config_rss - Prepare for RSS if used 8392 * @pf: board private structure 8393 **/ 8394 static int i40e_pf_config_rss(struct i40e_pf *pf) 8395 { 8396 struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi]; 8397 u8 seed[I40E_HKEY_ARRAY_SIZE]; 8398 u8 *lut; 8399 struct i40e_hw *hw = &pf->hw; 8400 u32 reg_val; 8401 u64 hena; 8402 int ret; 8403 8404 /* By default we enable TCP/UDP with IPv4/IPv6 ptypes */ 8405 hena = (u64)i40e_read_rx_ctl(hw, I40E_PFQF_HENA(0)) | 8406 ((u64)i40e_read_rx_ctl(hw, I40E_PFQF_HENA(1)) << 32); 8407 hena |= i40e_pf_get_default_rss_hena(pf); 8408 8409 i40e_write_rx_ctl(hw, I40E_PFQF_HENA(0), (u32)hena); 8410 i40e_write_rx_ctl(hw, I40E_PFQF_HENA(1), (u32)(hena >> 32)); 8411 8412 /* Determine the RSS table size based on the hardware capabilities */ 8413 reg_val = i40e_read_rx_ctl(hw, I40E_PFQF_CTL_0); 8414 reg_val = (pf->rss_table_size == 512) ? 8415 (reg_val | I40E_PFQF_CTL_0_HASHLUTSIZE_512) : 8416 (reg_val & ~I40E_PFQF_CTL_0_HASHLUTSIZE_512); 8417 i40e_write_rx_ctl(hw, I40E_PFQF_CTL_0, reg_val); 8418 8419 /* Determine the RSS size of the VSI */ 8420 if (!vsi->rss_size) 8421 vsi->rss_size = min_t(int, pf->alloc_rss_size, 8422 vsi->num_queue_pairs); 8423 if (!vsi->rss_size) 8424 return -EINVAL; 8425 8426 lut = kzalloc(vsi->rss_table_size, GFP_KERNEL); 8427 if (!lut) 8428 return -ENOMEM; 8429 8430 /* Use user configured lut if there is one, otherwise use default */ 8431 if (vsi->rss_lut_user) 8432 memcpy(lut, vsi->rss_lut_user, vsi->rss_table_size); 8433 else 8434 i40e_fill_rss_lut(pf, lut, vsi->rss_table_size, vsi->rss_size); 8435 8436 /* Use user configured hash key if there is one, otherwise 8437 * use default. 8438 */ 8439 if (vsi->rss_hkey_user) 8440 memcpy(seed, vsi->rss_hkey_user, I40E_HKEY_ARRAY_SIZE); 8441 else 8442 netdev_rss_key_fill((void *)seed, I40E_HKEY_ARRAY_SIZE); 8443 ret = i40e_config_rss(vsi, seed, lut, vsi->rss_table_size); 8444 kfree(lut); 8445 8446 return ret; 8447 } 8448 8449 /** 8450 * i40e_reconfig_rss_queues - change number of queues for rss and rebuild 8451 * @pf: board private structure 8452 * @queue_count: the requested queue count for rss. 8453 * 8454 * returns 0 if rss is not enabled, if enabled returns the final rss queue 8455 * count which may be different from the requested queue count. 8456 **/ 8457 int i40e_reconfig_rss_queues(struct i40e_pf *pf, int queue_count) 8458 { 8459 struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi]; 8460 int new_rss_size; 8461 8462 if (!(pf->flags & I40E_FLAG_RSS_ENABLED)) 8463 return 0; 8464 8465 new_rss_size = min_t(int, queue_count, pf->rss_size_max); 8466 8467 if (queue_count != vsi->num_queue_pairs) { 8468 vsi->req_queue_pairs = queue_count; 8469 i40e_prep_for_reset(pf); 8470 8471 pf->alloc_rss_size = new_rss_size; 8472 8473 i40e_reset_and_rebuild(pf, true); 8474 8475 /* Discard the user configured hash keys and lut, if less 8476 * queues are enabled. 8477 */ 8478 if (queue_count < vsi->rss_size) { 8479 i40e_clear_rss_config_user(vsi); 8480 dev_dbg(&pf->pdev->dev, 8481 "discard user configured hash keys and lut\n"); 8482 } 8483 8484 /* Reset vsi->rss_size, as number of enabled queues changed */ 8485 vsi->rss_size = min_t(int, pf->alloc_rss_size, 8486 vsi->num_queue_pairs); 8487 8488 i40e_pf_config_rss(pf); 8489 } 8490 dev_info(&pf->pdev->dev, "User requested queue count/HW max RSS count: %d/%d\n", 8491 vsi->req_queue_pairs, pf->rss_size_max); 8492 return pf->alloc_rss_size; 8493 } 8494 8495 /** 8496 * i40e_get_npar_bw_setting - Retrieve BW settings for this PF partition 8497 * @pf: board private structure 8498 **/ 8499 i40e_status i40e_get_npar_bw_setting(struct i40e_pf *pf) 8500 { 8501 i40e_status status; 8502 bool min_valid, max_valid; 8503 u32 max_bw, min_bw; 8504 8505 status = i40e_read_bw_from_alt_ram(&pf->hw, &max_bw, &min_bw, 8506 &min_valid, &max_valid); 8507 8508 if (!status) { 8509 if (min_valid) 8510 pf->npar_min_bw = min_bw; 8511 if (max_valid) 8512 pf->npar_max_bw = max_bw; 8513 } 8514 8515 return status; 8516 } 8517 8518 /** 8519 * i40e_set_npar_bw_setting - Set BW settings for this PF partition 8520 * @pf: board private structure 8521 **/ 8522 i40e_status i40e_set_npar_bw_setting(struct i40e_pf *pf) 8523 { 8524 struct i40e_aqc_configure_partition_bw_data bw_data; 8525 i40e_status status; 8526 8527 /* Set the valid bit for this PF */ 8528 bw_data.pf_valid_bits = cpu_to_le16(BIT(pf->hw.pf_id)); 8529 bw_data.max_bw[pf->hw.pf_id] = pf->npar_max_bw & I40E_ALT_BW_VALUE_MASK; 8530 bw_data.min_bw[pf->hw.pf_id] = pf->npar_min_bw & I40E_ALT_BW_VALUE_MASK; 8531 8532 /* Set the new bandwidths */ 8533 status = i40e_aq_configure_partition_bw(&pf->hw, &bw_data, NULL); 8534 8535 return status; 8536 } 8537 8538 /** 8539 * i40e_commit_npar_bw_setting - Commit BW settings for this PF partition 8540 * @pf: board private structure 8541 **/ 8542 i40e_status i40e_commit_npar_bw_setting(struct i40e_pf *pf) 8543 { 8544 /* Commit temporary BW setting to permanent NVM image */ 8545 enum i40e_admin_queue_err last_aq_status; 8546 i40e_status ret; 8547 u16 nvm_word; 8548 8549 if (pf->hw.partition_id != 1) { 8550 dev_info(&pf->pdev->dev, 8551 "Commit BW only works on partition 1! This is partition %d", 8552 pf->hw.partition_id); 8553 ret = I40E_NOT_SUPPORTED; 8554 goto bw_commit_out; 8555 } 8556 8557 /* Acquire NVM for read access */ 8558 ret = i40e_acquire_nvm(&pf->hw, I40E_RESOURCE_READ); 8559 last_aq_status = pf->hw.aq.asq_last_status; 8560 if (ret) { 8561 dev_info(&pf->pdev->dev, 8562 "Cannot acquire NVM for read access, err %s aq_err %s\n", 8563 i40e_stat_str(&pf->hw, ret), 8564 i40e_aq_str(&pf->hw, last_aq_status)); 8565 goto bw_commit_out; 8566 } 8567 8568 /* Read word 0x10 of NVM - SW compatibility word 1 */ 8569 ret = i40e_aq_read_nvm(&pf->hw, 8570 I40E_SR_NVM_CONTROL_WORD, 8571 0x10, sizeof(nvm_word), &nvm_word, 8572 false, NULL); 8573 /* Save off last admin queue command status before releasing 8574 * the NVM 8575 */ 8576 last_aq_status = pf->hw.aq.asq_last_status; 8577 i40e_release_nvm(&pf->hw); 8578 if (ret) { 8579 dev_info(&pf->pdev->dev, "NVM read error, err %s aq_err %s\n", 8580 i40e_stat_str(&pf->hw, ret), 8581 i40e_aq_str(&pf->hw, last_aq_status)); 8582 goto bw_commit_out; 8583 } 8584 8585 /* Wait a bit for NVM release to complete */ 8586 msleep(50); 8587 8588 /* Acquire NVM for write access */ 8589 ret = i40e_acquire_nvm(&pf->hw, I40E_RESOURCE_WRITE); 8590 last_aq_status = pf->hw.aq.asq_last_status; 8591 if (ret) { 8592 dev_info(&pf->pdev->dev, 8593 "Cannot acquire NVM for write access, err %s aq_err %s\n", 8594 i40e_stat_str(&pf->hw, ret), 8595 i40e_aq_str(&pf->hw, last_aq_status)); 8596 goto bw_commit_out; 8597 } 8598 /* Write it back out unchanged to initiate update NVM, 8599 * which will force a write of the shadow (alt) RAM to 8600 * the NVM - thus storing the bandwidth values permanently. 8601 */ 8602 ret = i40e_aq_update_nvm(&pf->hw, 8603 I40E_SR_NVM_CONTROL_WORD, 8604 0x10, sizeof(nvm_word), 8605 &nvm_word, true, NULL); 8606 /* Save off last admin queue command status before releasing 8607 * the NVM 8608 */ 8609 last_aq_status = pf->hw.aq.asq_last_status; 8610 i40e_release_nvm(&pf->hw); 8611 if (ret) 8612 dev_info(&pf->pdev->dev, 8613 "BW settings NOT SAVED, err %s aq_err %s\n", 8614 i40e_stat_str(&pf->hw, ret), 8615 i40e_aq_str(&pf->hw, last_aq_status)); 8616 bw_commit_out: 8617 8618 return ret; 8619 } 8620 8621 /** 8622 * i40e_sw_init - Initialize general software structures (struct i40e_pf) 8623 * @pf: board private structure to initialize 8624 * 8625 * i40e_sw_init initializes the Adapter private data structure. 8626 * Fields are initialized based on PCI device information and 8627 * OS network device settings (MTU size). 8628 **/ 8629 static int i40e_sw_init(struct i40e_pf *pf) 8630 { 8631 int err = 0; 8632 int size; 8633 8634 /* Set default capability flags */ 8635 pf->flags = I40E_FLAG_RX_CSUM_ENABLED | 8636 I40E_FLAG_MSI_ENABLED | 8637 I40E_FLAG_MSIX_ENABLED; 8638 8639 /* Set default ITR */ 8640 pf->rx_itr_default = I40E_ITR_DYNAMIC | I40E_ITR_RX_DEF; 8641 pf->tx_itr_default = I40E_ITR_DYNAMIC | I40E_ITR_TX_DEF; 8642 8643 /* Depending on PF configurations, it is possible that the RSS 8644 * maximum might end up larger than the available queues 8645 */ 8646 pf->rss_size_max = BIT(pf->hw.func_caps.rss_table_entry_width); 8647 pf->alloc_rss_size = 1; 8648 pf->rss_table_size = pf->hw.func_caps.rss_table_size; 8649 pf->rss_size_max = min_t(int, pf->rss_size_max, 8650 pf->hw.func_caps.num_tx_qp); 8651 if (pf->hw.func_caps.rss) { 8652 pf->flags |= I40E_FLAG_RSS_ENABLED; 8653 pf->alloc_rss_size = min_t(int, pf->rss_size_max, 8654 num_online_cpus()); 8655 } 8656 8657 /* MFP mode enabled */ 8658 if (pf->hw.func_caps.npar_enable || pf->hw.func_caps.flex10_enable) { 8659 pf->flags |= I40E_FLAG_MFP_ENABLED; 8660 dev_info(&pf->pdev->dev, "MFP mode Enabled\n"); 8661 if (i40e_get_npar_bw_setting(pf)) 8662 dev_warn(&pf->pdev->dev, 8663 "Could not get NPAR bw settings\n"); 8664 else 8665 dev_info(&pf->pdev->dev, 8666 "Min BW = %8.8x, Max BW = %8.8x\n", 8667 pf->npar_min_bw, pf->npar_max_bw); 8668 } 8669 8670 /* FW/NVM is not yet fixed in this regard */ 8671 if ((pf->hw.func_caps.fd_filters_guaranteed > 0) || 8672 (pf->hw.func_caps.fd_filters_best_effort > 0)) { 8673 pf->flags |= I40E_FLAG_FD_ATR_ENABLED; 8674 pf->atr_sample_rate = I40E_DEFAULT_ATR_SAMPLE_RATE; 8675 if (pf->flags & I40E_FLAG_MFP_ENABLED && 8676 pf->hw.num_partitions > 1) 8677 dev_info(&pf->pdev->dev, 8678 "Flow Director Sideband mode Disabled in MFP mode\n"); 8679 else 8680 pf->flags |= I40E_FLAG_FD_SB_ENABLED; 8681 pf->fdir_pf_filter_count = 8682 pf->hw.func_caps.fd_filters_guaranteed; 8683 pf->hw.fdir_shared_filter_count = 8684 pf->hw.func_caps.fd_filters_best_effort; 8685 } 8686 8687 if ((pf->hw.mac.type == I40E_MAC_XL710) && 8688 (((pf->hw.aq.fw_maj_ver == 4) && (pf->hw.aq.fw_min_ver < 33)) || 8689 (pf->hw.aq.fw_maj_ver < 4))) { 8690 pf->flags |= I40E_FLAG_RESTART_AUTONEG; 8691 /* No DCB support for FW < v4.33 */ 8692 pf->flags |= I40E_FLAG_NO_DCB_SUPPORT; 8693 } 8694 8695 /* Disable FW LLDP if FW < v4.3 */ 8696 if ((pf->hw.mac.type == I40E_MAC_XL710) && 8697 (((pf->hw.aq.fw_maj_ver == 4) && (pf->hw.aq.fw_min_ver < 3)) || 8698 (pf->hw.aq.fw_maj_ver < 4))) 8699 pf->flags |= I40E_FLAG_STOP_FW_LLDP; 8700 8701 /* Use the FW Set LLDP MIB API if FW > v4.40 */ 8702 if ((pf->hw.mac.type == I40E_MAC_XL710) && 8703 (((pf->hw.aq.fw_maj_ver == 4) && (pf->hw.aq.fw_min_ver >= 40)) || 8704 (pf->hw.aq.fw_maj_ver >= 5))) 8705 pf->flags |= I40E_FLAG_USE_SET_LLDP_MIB; 8706 8707 if (pf->hw.func_caps.vmdq) { 8708 pf->num_vmdq_vsis = I40E_DEFAULT_NUM_VMDQ_VSI; 8709 pf->flags |= I40E_FLAG_VMDQ_ENABLED; 8710 pf->num_vmdq_qps = i40e_default_queues_per_vmdq(pf); 8711 } 8712 8713 if (pf->hw.func_caps.iwarp) { 8714 pf->flags |= I40E_FLAG_IWARP_ENABLED; 8715 /* IWARP needs one extra vector for CQP just like MISC.*/ 8716 pf->num_iwarp_msix = (int)num_online_cpus() + 1; 8717 } 8718 8719 #ifdef I40E_FCOE 8720 i40e_init_pf_fcoe(pf); 8721 8722 #endif /* I40E_FCOE */ 8723 #ifdef CONFIG_PCI_IOV 8724 if (pf->hw.func_caps.num_vfs && pf->hw.partition_id == 1) { 8725 pf->num_vf_qps = I40E_DEFAULT_QUEUES_PER_VF; 8726 pf->flags |= I40E_FLAG_SRIOV_ENABLED; 8727 pf->num_req_vfs = min_t(int, 8728 pf->hw.func_caps.num_vfs, 8729 I40E_MAX_VF_COUNT); 8730 } 8731 #endif /* CONFIG_PCI_IOV */ 8732 if (pf->hw.mac.type == I40E_MAC_X722) { 8733 pf->flags |= I40E_FLAG_RSS_AQ_CAPABLE | 8734 I40E_FLAG_128_QP_RSS_CAPABLE | 8735 I40E_FLAG_HW_ATR_EVICT_CAPABLE | 8736 I40E_FLAG_OUTER_UDP_CSUM_CAPABLE | 8737 I40E_FLAG_WB_ON_ITR_CAPABLE | 8738 I40E_FLAG_MULTIPLE_TCP_UDP_RSS_PCTYPE | 8739 I40E_FLAG_NO_PCI_LINK_CHECK | 8740 I40E_FLAG_USE_SET_LLDP_MIB | 8741 I40E_FLAG_GENEVE_OFFLOAD_CAPABLE | 8742 I40E_FLAG_PTP_L4_CAPABLE; 8743 } else if ((pf->hw.aq.api_maj_ver > 1) || 8744 ((pf->hw.aq.api_maj_ver == 1) && 8745 (pf->hw.aq.api_min_ver > 4))) { 8746 /* Supported in FW API version higher than 1.4 */ 8747 pf->flags |= I40E_FLAG_GENEVE_OFFLOAD_CAPABLE; 8748 pf->auto_disable_flags = I40E_FLAG_HW_ATR_EVICT_CAPABLE; 8749 } else { 8750 pf->auto_disable_flags = I40E_FLAG_HW_ATR_EVICT_CAPABLE; 8751 } 8752 8753 pf->eeprom_version = 0xDEAD; 8754 pf->lan_veb = I40E_NO_VEB; 8755 pf->lan_vsi = I40E_NO_VSI; 8756 8757 /* By default FW has this off for performance reasons */ 8758 pf->flags &= ~I40E_FLAG_VEB_STATS_ENABLED; 8759 8760 /* set up queue assignment tracking */ 8761 size = sizeof(struct i40e_lump_tracking) 8762 + (sizeof(u16) * pf->hw.func_caps.num_tx_qp); 8763 pf->qp_pile = kzalloc(size, GFP_KERNEL); 8764 if (!pf->qp_pile) { 8765 err = -ENOMEM; 8766 goto sw_init_done; 8767 } 8768 pf->qp_pile->num_entries = pf->hw.func_caps.num_tx_qp; 8769 pf->qp_pile->search_hint = 0; 8770 8771 pf->tx_timeout_recovery_level = 1; 8772 8773 mutex_init(&pf->switch_mutex); 8774 8775 /* If NPAR is enabled nudge the Tx scheduler */ 8776 if (pf->hw.func_caps.npar_enable && (!i40e_get_npar_bw_setting(pf))) 8777 i40e_set_npar_bw_setting(pf); 8778 8779 sw_init_done: 8780 return err; 8781 } 8782 8783 /** 8784 * i40e_set_ntuple - set the ntuple feature flag and take action 8785 * @pf: board private structure to initialize 8786 * @features: the feature set that the stack is suggesting 8787 * 8788 * returns a bool to indicate if reset needs to happen 8789 **/ 8790 bool i40e_set_ntuple(struct i40e_pf *pf, netdev_features_t features) 8791 { 8792 bool need_reset = false; 8793 8794 /* Check if Flow Director n-tuple support was enabled or disabled. If 8795 * the state changed, we need to reset. 8796 */ 8797 if (features & NETIF_F_NTUPLE) { 8798 /* Enable filters and mark for reset */ 8799 if (!(pf->flags & I40E_FLAG_FD_SB_ENABLED)) 8800 need_reset = true; 8801 /* enable FD_SB only if there is MSI-X vector */ 8802 if (pf->num_fdsb_msix > 0) 8803 pf->flags |= I40E_FLAG_FD_SB_ENABLED; 8804 } else { 8805 /* turn off filters, mark for reset and clear SW filter list */ 8806 if (pf->flags & I40E_FLAG_FD_SB_ENABLED) { 8807 need_reset = true; 8808 i40e_fdir_filter_exit(pf); 8809 } 8810 pf->flags &= ~I40E_FLAG_FD_SB_ENABLED; 8811 pf->auto_disable_flags &= ~I40E_FLAG_FD_SB_ENABLED; 8812 /* reset fd counters */ 8813 pf->fd_add_err = pf->fd_atr_cnt = pf->fd_tcp_rule = 0; 8814 pf->fdir_pf_active_filters = 0; 8815 /* if ATR was auto disabled it can be re-enabled. */ 8816 if ((pf->flags & I40E_FLAG_FD_ATR_ENABLED) && 8817 (pf->auto_disable_flags & I40E_FLAG_FD_ATR_ENABLED)) { 8818 pf->auto_disable_flags &= ~I40E_FLAG_FD_ATR_ENABLED; 8819 if (I40E_DEBUG_FD & pf->hw.debug_mask) 8820 dev_info(&pf->pdev->dev, "ATR re-enabled.\n"); 8821 } 8822 } 8823 return need_reset; 8824 } 8825 8826 /** 8827 * i40e_clear_rss_lut - clear the rx hash lookup table 8828 * @vsi: the VSI being configured 8829 **/ 8830 static void i40e_clear_rss_lut(struct i40e_vsi *vsi) 8831 { 8832 struct i40e_pf *pf = vsi->back; 8833 struct i40e_hw *hw = &pf->hw; 8834 u16 vf_id = vsi->vf_id; 8835 u8 i; 8836 8837 if (vsi->type == I40E_VSI_MAIN) { 8838 for (i = 0; i <= I40E_PFQF_HLUT_MAX_INDEX; i++) 8839 wr32(hw, I40E_PFQF_HLUT(i), 0); 8840 } else if (vsi->type == I40E_VSI_SRIOV) { 8841 for (i = 0; i <= I40E_VFQF_HLUT_MAX_INDEX; i++) 8842 i40e_write_rx_ctl(hw, I40E_VFQF_HLUT1(i, vf_id), 0); 8843 } else { 8844 dev_err(&pf->pdev->dev, "Cannot set RSS LUT - invalid VSI type\n"); 8845 } 8846 } 8847 8848 /** 8849 * i40e_set_features - set the netdev feature flags 8850 * @netdev: ptr to the netdev being adjusted 8851 * @features: the feature set that the stack is suggesting 8852 **/ 8853 static int i40e_set_features(struct net_device *netdev, 8854 netdev_features_t features) 8855 { 8856 struct i40e_netdev_priv *np = netdev_priv(netdev); 8857 struct i40e_vsi *vsi = np->vsi; 8858 struct i40e_pf *pf = vsi->back; 8859 bool need_reset; 8860 8861 if (features & NETIF_F_RXHASH && !(netdev->features & NETIF_F_RXHASH)) 8862 i40e_pf_config_rss(pf); 8863 else if (!(features & NETIF_F_RXHASH) && 8864 netdev->features & NETIF_F_RXHASH) 8865 i40e_clear_rss_lut(vsi); 8866 8867 if (features & NETIF_F_HW_VLAN_CTAG_RX) 8868 i40e_vlan_stripping_enable(vsi); 8869 else 8870 i40e_vlan_stripping_disable(vsi); 8871 8872 need_reset = i40e_set_ntuple(pf, features); 8873 8874 if (need_reset) 8875 i40e_do_reset(pf, BIT_ULL(__I40E_PF_RESET_REQUESTED)); 8876 8877 return 0; 8878 } 8879 8880 /** 8881 * i40e_get_udp_port_idx - Lookup a possibly offloaded for Rx UDP port 8882 * @pf: board private structure 8883 * @port: The UDP port to look up 8884 * 8885 * Returns the index number or I40E_MAX_PF_UDP_OFFLOAD_PORTS if port not found 8886 **/ 8887 static u8 i40e_get_udp_port_idx(struct i40e_pf *pf, __be16 port) 8888 { 8889 u8 i; 8890 8891 for (i = 0; i < I40E_MAX_PF_UDP_OFFLOAD_PORTS; i++) { 8892 if (pf->udp_ports[i].index == port) 8893 return i; 8894 } 8895 8896 return i; 8897 } 8898 8899 /** 8900 * i40e_udp_tunnel_add - Get notifications about UDP tunnel ports that come up 8901 * @netdev: This physical port's netdev 8902 * @ti: Tunnel endpoint information 8903 **/ 8904 static void i40e_udp_tunnel_add(struct net_device *netdev, 8905 struct udp_tunnel_info *ti) 8906 { 8907 struct i40e_netdev_priv *np = netdev_priv(netdev); 8908 struct i40e_vsi *vsi = np->vsi; 8909 struct i40e_pf *pf = vsi->back; 8910 __be16 port = ti->port; 8911 u8 next_idx; 8912 u8 idx; 8913 8914 idx = i40e_get_udp_port_idx(pf, port); 8915 8916 /* Check if port already exists */ 8917 if (idx < I40E_MAX_PF_UDP_OFFLOAD_PORTS) { 8918 netdev_info(netdev, "port %d already offloaded\n", 8919 ntohs(port)); 8920 return; 8921 } 8922 8923 /* Now check if there is space to add the new port */ 8924 next_idx = i40e_get_udp_port_idx(pf, 0); 8925 8926 if (next_idx == I40E_MAX_PF_UDP_OFFLOAD_PORTS) { 8927 netdev_info(netdev, "maximum number of offloaded UDP ports reached, not adding port %d\n", 8928 ntohs(port)); 8929 return; 8930 } 8931 8932 switch (ti->type) { 8933 case UDP_TUNNEL_TYPE_VXLAN: 8934 pf->udp_ports[next_idx].type = I40E_AQC_TUNNEL_TYPE_VXLAN; 8935 break; 8936 case UDP_TUNNEL_TYPE_GENEVE: 8937 if (!(pf->flags & I40E_FLAG_GENEVE_OFFLOAD_CAPABLE)) 8938 return; 8939 pf->udp_ports[next_idx].type = I40E_AQC_TUNNEL_TYPE_NGE; 8940 break; 8941 default: 8942 return; 8943 } 8944 8945 /* New port: add it and mark its index in the bitmap */ 8946 pf->udp_ports[next_idx].index = port; 8947 pf->pending_udp_bitmap |= BIT_ULL(next_idx); 8948 pf->flags |= I40E_FLAG_UDP_FILTER_SYNC; 8949 } 8950 8951 /** 8952 * i40e_udp_tunnel_del - Get notifications about UDP tunnel ports that go away 8953 * @netdev: This physical port's netdev 8954 * @ti: Tunnel endpoint information 8955 **/ 8956 static void i40e_udp_tunnel_del(struct net_device *netdev, 8957 struct udp_tunnel_info *ti) 8958 { 8959 struct i40e_netdev_priv *np = netdev_priv(netdev); 8960 struct i40e_vsi *vsi = np->vsi; 8961 struct i40e_pf *pf = vsi->back; 8962 __be16 port = ti->port; 8963 u8 idx; 8964 8965 idx = i40e_get_udp_port_idx(pf, port); 8966 8967 /* Check if port already exists */ 8968 if (idx >= I40E_MAX_PF_UDP_OFFLOAD_PORTS) 8969 goto not_found; 8970 8971 switch (ti->type) { 8972 case UDP_TUNNEL_TYPE_VXLAN: 8973 if (pf->udp_ports[idx].type != I40E_AQC_TUNNEL_TYPE_VXLAN) 8974 goto not_found; 8975 break; 8976 case UDP_TUNNEL_TYPE_GENEVE: 8977 if (pf->udp_ports[idx].type != I40E_AQC_TUNNEL_TYPE_NGE) 8978 goto not_found; 8979 break; 8980 default: 8981 goto not_found; 8982 } 8983 8984 /* if port exists, set it to 0 (mark for deletion) 8985 * and make it pending 8986 */ 8987 pf->udp_ports[idx].index = 0; 8988 pf->pending_udp_bitmap |= BIT_ULL(idx); 8989 pf->flags |= I40E_FLAG_UDP_FILTER_SYNC; 8990 8991 return; 8992 not_found: 8993 netdev_warn(netdev, "UDP port %d was not found, not deleting\n", 8994 ntohs(port)); 8995 } 8996 8997 static int i40e_get_phys_port_id(struct net_device *netdev, 8998 struct netdev_phys_item_id *ppid) 8999 { 9000 struct i40e_netdev_priv *np = netdev_priv(netdev); 9001 struct i40e_pf *pf = np->vsi->back; 9002 struct i40e_hw *hw = &pf->hw; 9003 9004 if (!(pf->flags & I40E_FLAG_PORT_ID_VALID)) 9005 return -EOPNOTSUPP; 9006 9007 ppid->id_len = min_t(int, sizeof(hw->mac.port_addr), sizeof(ppid->id)); 9008 memcpy(ppid->id, hw->mac.port_addr, ppid->id_len); 9009 9010 return 0; 9011 } 9012 9013 /** 9014 * i40e_ndo_fdb_add - add an entry to the hardware database 9015 * @ndm: the input from the stack 9016 * @tb: pointer to array of nladdr (unused) 9017 * @dev: the net device pointer 9018 * @addr: the MAC address entry being added 9019 * @flags: instructions from stack about fdb operation 9020 */ 9021 static int i40e_ndo_fdb_add(struct ndmsg *ndm, struct nlattr *tb[], 9022 struct net_device *dev, 9023 const unsigned char *addr, u16 vid, 9024 u16 flags) 9025 { 9026 struct i40e_netdev_priv *np = netdev_priv(dev); 9027 struct i40e_pf *pf = np->vsi->back; 9028 int err = 0; 9029 9030 if (!(pf->flags & I40E_FLAG_SRIOV_ENABLED)) 9031 return -EOPNOTSUPP; 9032 9033 if (vid) { 9034 pr_info("%s: vlans aren't supported yet for dev_uc|mc_add()\n", dev->name); 9035 return -EINVAL; 9036 } 9037 9038 /* Hardware does not support aging addresses so if a 9039 * ndm_state is given only allow permanent addresses 9040 */ 9041 if (ndm->ndm_state && !(ndm->ndm_state & NUD_PERMANENT)) { 9042 netdev_info(dev, "FDB only supports static addresses\n"); 9043 return -EINVAL; 9044 } 9045 9046 if (is_unicast_ether_addr(addr) || is_link_local_ether_addr(addr)) 9047 err = dev_uc_add_excl(dev, addr); 9048 else if (is_multicast_ether_addr(addr)) 9049 err = dev_mc_add_excl(dev, addr); 9050 else 9051 err = -EINVAL; 9052 9053 /* Only return duplicate errors if NLM_F_EXCL is set */ 9054 if (err == -EEXIST && !(flags & NLM_F_EXCL)) 9055 err = 0; 9056 9057 return err; 9058 } 9059 9060 /** 9061 * i40e_ndo_bridge_setlink - Set the hardware bridge mode 9062 * @dev: the netdev being configured 9063 * @nlh: RTNL message 9064 * 9065 * Inserts a new hardware bridge if not already created and 9066 * enables the bridging mode requested (VEB or VEPA). If the 9067 * hardware bridge has already been inserted and the request 9068 * is to change the mode then that requires a PF reset to 9069 * allow rebuild of the components with required hardware 9070 * bridge mode enabled. 9071 **/ 9072 static int i40e_ndo_bridge_setlink(struct net_device *dev, 9073 struct nlmsghdr *nlh, 9074 u16 flags) 9075 { 9076 struct i40e_netdev_priv *np = netdev_priv(dev); 9077 struct i40e_vsi *vsi = np->vsi; 9078 struct i40e_pf *pf = vsi->back; 9079 struct i40e_veb *veb = NULL; 9080 struct nlattr *attr, *br_spec; 9081 int i, rem; 9082 9083 /* Only for PF VSI for now */ 9084 if (vsi->seid != pf->vsi[pf->lan_vsi]->seid) 9085 return -EOPNOTSUPP; 9086 9087 /* Find the HW bridge for PF VSI */ 9088 for (i = 0; i < I40E_MAX_VEB && !veb; i++) { 9089 if (pf->veb[i] && pf->veb[i]->seid == vsi->uplink_seid) 9090 veb = pf->veb[i]; 9091 } 9092 9093 br_spec = nlmsg_find_attr(nlh, sizeof(struct ifinfomsg), IFLA_AF_SPEC); 9094 9095 nla_for_each_nested(attr, br_spec, rem) { 9096 __u16 mode; 9097 9098 if (nla_type(attr) != IFLA_BRIDGE_MODE) 9099 continue; 9100 9101 mode = nla_get_u16(attr); 9102 if ((mode != BRIDGE_MODE_VEPA) && 9103 (mode != BRIDGE_MODE_VEB)) 9104 return -EINVAL; 9105 9106 /* Insert a new HW bridge */ 9107 if (!veb) { 9108 veb = i40e_veb_setup(pf, 0, vsi->uplink_seid, vsi->seid, 9109 vsi->tc_config.enabled_tc); 9110 if (veb) { 9111 veb->bridge_mode = mode; 9112 i40e_config_bridge_mode(veb); 9113 } else { 9114 /* No Bridge HW offload available */ 9115 return -ENOENT; 9116 } 9117 break; 9118 } else if (mode != veb->bridge_mode) { 9119 /* Existing HW bridge but different mode needs reset */ 9120 veb->bridge_mode = mode; 9121 /* TODO: If no VFs or VMDq VSIs, disallow VEB mode */ 9122 if (mode == BRIDGE_MODE_VEB) 9123 pf->flags |= I40E_FLAG_VEB_MODE_ENABLED; 9124 else 9125 pf->flags &= ~I40E_FLAG_VEB_MODE_ENABLED; 9126 i40e_do_reset(pf, BIT_ULL(__I40E_PF_RESET_REQUESTED)); 9127 break; 9128 } 9129 } 9130 9131 return 0; 9132 } 9133 9134 /** 9135 * i40e_ndo_bridge_getlink - Get the hardware bridge mode 9136 * @skb: skb buff 9137 * @pid: process id 9138 * @seq: RTNL message seq # 9139 * @dev: the netdev being configured 9140 * @filter_mask: unused 9141 * @nlflags: netlink flags passed in 9142 * 9143 * Return the mode in which the hardware bridge is operating in 9144 * i.e VEB or VEPA. 9145 **/ 9146 static int i40e_ndo_bridge_getlink(struct sk_buff *skb, u32 pid, u32 seq, 9147 struct net_device *dev, 9148 u32 __always_unused filter_mask, 9149 int nlflags) 9150 { 9151 struct i40e_netdev_priv *np = netdev_priv(dev); 9152 struct i40e_vsi *vsi = np->vsi; 9153 struct i40e_pf *pf = vsi->back; 9154 struct i40e_veb *veb = NULL; 9155 int i; 9156 9157 /* Only for PF VSI for now */ 9158 if (vsi->seid != pf->vsi[pf->lan_vsi]->seid) 9159 return -EOPNOTSUPP; 9160 9161 /* Find the HW bridge for the PF VSI */ 9162 for (i = 0; i < I40E_MAX_VEB && !veb; i++) { 9163 if (pf->veb[i] && pf->veb[i]->seid == vsi->uplink_seid) 9164 veb = pf->veb[i]; 9165 } 9166 9167 if (!veb) 9168 return 0; 9169 9170 return ndo_dflt_bridge_getlink(skb, pid, seq, dev, veb->bridge_mode, 9171 0, 0, nlflags, filter_mask, NULL); 9172 } 9173 9174 /** 9175 * i40e_features_check - Validate encapsulated packet conforms to limits 9176 * @skb: skb buff 9177 * @dev: This physical port's netdev 9178 * @features: Offload features that the stack believes apply 9179 **/ 9180 static netdev_features_t i40e_features_check(struct sk_buff *skb, 9181 struct net_device *dev, 9182 netdev_features_t features) 9183 { 9184 size_t len; 9185 9186 /* No point in doing any of this if neither checksum nor GSO are 9187 * being requested for this frame. We can rule out both by just 9188 * checking for CHECKSUM_PARTIAL 9189 */ 9190 if (skb->ip_summed != CHECKSUM_PARTIAL) 9191 return features; 9192 9193 /* We cannot support GSO if the MSS is going to be less than 9194 * 64 bytes. If it is then we need to drop support for GSO. 9195 */ 9196 if (skb_is_gso(skb) && (skb_shinfo(skb)->gso_size < 64)) 9197 features &= ~NETIF_F_GSO_MASK; 9198 9199 /* MACLEN can support at most 63 words */ 9200 len = skb_network_header(skb) - skb->data; 9201 if (len & ~(63 * 2)) 9202 goto out_err; 9203 9204 /* IPLEN and EIPLEN can support at most 127 dwords */ 9205 len = skb_transport_header(skb) - skb_network_header(skb); 9206 if (len & ~(127 * 4)) 9207 goto out_err; 9208 9209 if (skb->encapsulation) { 9210 /* L4TUNLEN can support 127 words */ 9211 len = skb_inner_network_header(skb) - skb_transport_header(skb); 9212 if (len & ~(127 * 2)) 9213 goto out_err; 9214 9215 /* IPLEN can support at most 127 dwords */ 9216 len = skb_inner_transport_header(skb) - 9217 skb_inner_network_header(skb); 9218 if (len & ~(127 * 4)) 9219 goto out_err; 9220 } 9221 9222 /* No need to validate L4LEN as TCP is the only protocol with a 9223 * a flexible value and we support all possible values supported 9224 * by TCP, which is at most 15 dwords 9225 */ 9226 9227 return features; 9228 out_err: 9229 return features & ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK); 9230 } 9231 9232 static const struct net_device_ops i40e_netdev_ops = { 9233 .ndo_open = i40e_open, 9234 .ndo_stop = i40e_close, 9235 .ndo_start_xmit = i40e_lan_xmit_frame, 9236 .ndo_get_stats64 = i40e_get_netdev_stats_struct, 9237 .ndo_set_rx_mode = i40e_set_rx_mode, 9238 .ndo_validate_addr = eth_validate_addr, 9239 .ndo_set_mac_address = i40e_set_mac, 9240 .ndo_change_mtu = i40e_change_mtu, 9241 .ndo_do_ioctl = i40e_ioctl, 9242 .ndo_tx_timeout = i40e_tx_timeout, 9243 .ndo_vlan_rx_add_vid = i40e_vlan_rx_add_vid, 9244 .ndo_vlan_rx_kill_vid = i40e_vlan_rx_kill_vid, 9245 #ifdef CONFIG_NET_POLL_CONTROLLER 9246 .ndo_poll_controller = i40e_netpoll, 9247 #endif 9248 .ndo_setup_tc = __i40e_setup_tc, 9249 #ifdef I40E_FCOE 9250 .ndo_fcoe_enable = i40e_fcoe_enable, 9251 .ndo_fcoe_disable = i40e_fcoe_disable, 9252 #endif 9253 .ndo_set_features = i40e_set_features, 9254 .ndo_set_vf_mac = i40e_ndo_set_vf_mac, 9255 .ndo_set_vf_vlan = i40e_ndo_set_vf_port_vlan, 9256 .ndo_set_vf_rate = i40e_ndo_set_vf_bw, 9257 .ndo_get_vf_config = i40e_ndo_get_vf_config, 9258 .ndo_set_vf_link_state = i40e_ndo_set_vf_link_state, 9259 .ndo_set_vf_spoofchk = i40e_ndo_set_vf_spoofchk, 9260 .ndo_set_vf_trust = i40e_ndo_set_vf_trust, 9261 .ndo_udp_tunnel_add = i40e_udp_tunnel_add, 9262 .ndo_udp_tunnel_del = i40e_udp_tunnel_del, 9263 .ndo_get_phys_port_id = i40e_get_phys_port_id, 9264 .ndo_fdb_add = i40e_ndo_fdb_add, 9265 .ndo_features_check = i40e_features_check, 9266 .ndo_bridge_getlink = i40e_ndo_bridge_getlink, 9267 .ndo_bridge_setlink = i40e_ndo_bridge_setlink, 9268 }; 9269 9270 /** 9271 * i40e_config_netdev - Setup the netdev flags 9272 * @vsi: the VSI being configured 9273 * 9274 * Returns 0 on success, negative value on failure 9275 **/ 9276 static int i40e_config_netdev(struct i40e_vsi *vsi) 9277 { 9278 struct i40e_pf *pf = vsi->back; 9279 struct i40e_hw *hw = &pf->hw; 9280 struct i40e_netdev_priv *np; 9281 struct net_device *netdev; 9282 u8 broadcast[ETH_ALEN]; 9283 u8 mac_addr[ETH_ALEN]; 9284 int etherdev_size; 9285 9286 etherdev_size = sizeof(struct i40e_netdev_priv); 9287 netdev = alloc_etherdev_mq(etherdev_size, vsi->alloc_queue_pairs); 9288 if (!netdev) 9289 return -ENOMEM; 9290 9291 vsi->netdev = netdev; 9292 np = netdev_priv(netdev); 9293 np->vsi = vsi; 9294 9295 netdev->hw_enc_features |= NETIF_F_SG | 9296 NETIF_F_IP_CSUM | 9297 NETIF_F_IPV6_CSUM | 9298 NETIF_F_HIGHDMA | 9299 NETIF_F_SOFT_FEATURES | 9300 NETIF_F_TSO | 9301 NETIF_F_TSO_ECN | 9302 NETIF_F_TSO6 | 9303 NETIF_F_GSO_GRE | 9304 NETIF_F_GSO_GRE_CSUM | 9305 NETIF_F_GSO_IPXIP4 | 9306 NETIF_F_GSO_IPXIP6 | 9307 NETIF_F_GSO_UDP_TUNNEL | 9308 NETIF_F_GSO_UDP_TUNNEL_CSUM | 9309 NETIF_F_GSO_PARTIAL | 9310 NETIF_F_SCTP_CRC | 9311 NETIF_F_RXHASH | 9312 NETIF_F_RXCSUM | 9313 0; 9314 9315 if (!(pf->flags & I40E_FLAG_OUTER_UDP_CSUM_CAPABLE)) 9316 netdev->gso_partial_features |= NETIF_F_GSO_UDP_TUNNEL_CSUM; 9317 9318 netdev->gso_partial_features |= NETIF_F_GSO_GRE_CSUM; 9319 9320 /* record features VLANs can make use of */ 9321 netdev->vlan_features |= netdev->hw_enc_features | 9322 NETIF_F_TSO_MANGLEID; 9323 9324 if (!(pf->flags & I40E_FLAG_MFP_ENABLED)) 9325 netdev->hw_features |= NETIF_F_NTUPLE; 9326 9327 netdev->hw_features |= netdev->hw_enc_features | 9328 NETIF_F_HW_VLAN_CTAG_TX | 9329 NETIF_F_HW_VLAN_CTAG_RX; 9330 9331 netdev->features |= netdev->hw_features | NETIF_F_HW_VLAN_CTAG_FILTER; 9332 netdev->hw_enc_features |= NETIF_F_TSO_MANGLEID; 9333 9334 if (vsi->type == I40E_VSI_MAIN) { 9335 SET_NETDEV_DEV(netdev, &pf->pdev->dev); 9336 ether_addr_copy(mac_addr, hw->mac.perm_addr); 9337 /* The following steps are necessary to prevent reception 9338 * of tagged packets - some older NVM configurations load a 9339 * default a MAC-VLAN filter that accepts any tagged packet 9340 * which must be replaced by a normal filter. 9341 */ 9342 i40e_rm_default_mac_filter(vsi, mac_addr); 9343 spin_lock_bh(&vsi->mac_filter_hash_lock); 9344 i40e_add_mac_filter(vsi, mac_addr); 9345 spin_unlock_bh(&vsi->mac_filter_hash_lock); 9346 } else { 9347 /* relate the VSI_VMDQ name to the VSI_MAIN name */ 9348 snprintf(netdev->name, IFNAMSIZ, "%sv%%d", 9349 pf->vsi[pf->lan_vsi]->netdev->name); 9350 random_ether_addr(mac_addr); 9351 9352 spin_lock_bh(&vsi->mac_filter_hash_lock); 9353 i40e_add_mac_filter(vsi, mac_addr); 9354 spin_unlock_bh(&vsi->mac_filter_hash_lock); 9355 } 9356 9357 /* Add the broadcast filter so that we initially will receive 9358 * broadcast packets. Note that when a new VLAN is first added the 9359 * driver will convert all filters marked I40E_VLAN_ANY into VLAN 9360 * specific filters as part of transitioning into "vlan" operation. 9361 * When more VLANs are added, the driver will copy each existing MAC 9362 * filter and add it for the new VLAN. 9363 * 9364 * Broadcast filters are handled specially by 9365 * i40e_sync_filters_subtask, as the driver must to set the broadcast 9366 * promiscuous bit instead of adding this directly as a MAC/VLAN 9367 * filter. The subtask will update the correct broadcast promiscuous 9368 * bits as VLANs become active or inactive. 9369 */ 9370 eth_broadcast_addr(broadcast); 9371 spin_lock_bh(&vsi->mac_filter_hash_lock); 9372 i40e_add_mac_filter(vsi, broadcast); 9373 spin_unlock_bh(&vsi->mac_filter_hash_lock); 9374 9375 ether_addr_copy(netdev->dev_addr, mac_addr); 9376 ether_addr_copy(netdev->perm_addr, mac_addr); 9377 9378 netdev->priv_flags |= IFF_UNICAST_FLT; 9379 netdev->priv_flags |= IFF_SUPP_NOFCS; 9380 /* Setup netdev TC information */ 9381 i40e_vsi_config_netdev_tc(vsi, vsi->tc_config.enabled_tc); 9382 9383 netdev->netdev_ops = &i40e_netdev_ops; 9384 netdev->watchdog_timeo = 5 * HZ; 9385 i40e_set_ethtool_ops(netdev); 9386 #ifdef I40E_FCOE 9387 i40e_fcoe_config_netdev(netdev, vsi); 9388 #endif 9389 9390 /* MTU range: 68 - 9706 */ 9391 netdev->min_mtu = ETH_MIN_MTU; 9392 netdev->max_mtu = I40E_MAX_RXBUFFER - 9393 (ETH_HLEN + ETH_FCS_LEN + VLAN_HLEN); 9394 9395 return 0; 9396 } 9397 9398 /** 9399 * i40e_vsi_delete - Delete a VSI from the switch 9400 * @vsi: the VSI being removed 9401 * 9402 * Returns 0 on success, negative value on failure 9403 **/ 9404 static void i40e_vsi_delete(struct i40e_vsi *vsi) 9405 { 9406 /* remove default VSI is not allowed */ 9407 if (vsi == vsi->back->vsi[vsi->back->lan_vsi]) 9408 return; 9409 9410 i40e_aq_delete_element(&vsi->back->hw, vsi->seid, NULL); 9411 } 9412 9413 /** 9414 * i40e_is_vsi_uplink_mode_veb - Check if the VSI's uplink bridge mode is VEB 9415 * @vsi: the VSI being queried 9416 * 9417 * Returns 1 if HW bridge mode is VEB and return 0 in case of VEPA mode 9418 **/ 9419 int i40e_is_vsi_uplink_mode_veb(struct i40e_vsi *vsi) 9420 { 9421 struct i40e_veb *veb; 9422 struct i40e_pf *pf = vsi->back; 9423 9424 /* Uplink is not a bridge so default to VEB */ 9425 if (vsi->veb_idx == I40E_NO_VEB) 9426 return 1; 9427 9428 veb = pf->veb[vsi->veb_idx]; 9429 if (!veb) { 9430 dev_info(&pf->pdev->dev, 9431 "There is no veb associated with the bridge\n"); 9432 return -ENOENT; 9433 } 9434 9435 /* Uplink is a bridge in VEPA mode */ 9436 if (veb->bridge_mode & BRIDGE_MODE_VEPA) { 9437 return 0; 9438 } else { 9439 /* Uplink is a bridge in VEB mode */ 9440 return 1; 9441 } 9442 9443 /* VEPA is now default bridge, so return 0 */ 9444 return 0; 9445 } 9446 9447 /** 9448 * i40e_add_vsi - Add a VSI to the switch 9449 * @vsi: the VSI being configured 9450 * 9451 * This initializes a VSI context depending on the VSI type to be added and 9452 * passes it down to the add_vsi aq command. 9453 **/ 9454 static int i40e_add_vsi(struct i40e_vsi *vsi) 9455 { 9456 int ret = -ENODEV; 9457 struct i40e_pf *pf = vsi->back; 9458 struct i40e_hw *hw = &pf->hw; 9459 struct i40e_vsi_context ctxt; 9460 struct i40e_mac_filter *f; 9461 struct hlist_node *h; 9462 int bkt; 9463 9464 u8 enabled_tc = 0x1; /* TC0 enabled */ 9465 int f_count = 0; 9466 9467 memset(&ctxt, 0, sizeof(ctxt)); 9468 switch (vsi->type) { 9469 case I40E_VSI_MAIN: 9470 /* The PF's main VSI is already setup as part of the 9471 * device initialization, so we'll not bother with 9472 * the add_vsi call, but we will retrieve the current 9473 * VSI context. 9474 */ 9475 ctxt.seid = pf->main_vsi_seid; 9476 ctxt.pf_num = pf->hw.pf_id; 9477 ctxt.vf_num = 0; 9478 ret = i40e_aq_get_vsi_params(&pf->hw, &ctxt, NULL); 9479 ctxt.flags = I40E_AQ_VSI_TYPE_PF; 9480 if (ret) { 9481 dev_info(&pf->pdev->dev, 9482 "couldn't get PF vsi config, err %s aq_err %s\n", 9483 i40e_stat_str(&pf->hw, ret), 9484 i40e_aq_str(&pf->hw, 9485 pf->hw.aq.asq_last_status)); 9486 return -ENOENT; 9487 } 9488 vsi->info = ctxt.info; 9489 vsi->info.valid_sections = 0; 9490 9491 vsi->seid = ctxt.seid; 9492 vsi->id = ctxt.vsi_number; 9493 9494 enabled_tc = i40e_pf_get_tc_map(pf); 9495 9496 /* MFP mode setup queue map and update VSI */ 9497 if ((pf->flags & I40E_FLAG_MFP_ENABLED) && 9498 !(pf->hw.func_caps.iscsi)) { /* NIC type PF */ 9499 memset(&ctxt, 0, sizeof(ctxt)); 9500 ctxt.seid = pf->main_vsi_seid; 9501 ctxt.pf_num = pf->hw.pf_id; 9502 ctxt.vf_num = 0; 9503 i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, false); 9504 ret = i40e_aq_update_vsi_params(hw, &ctxt, NULL); 9505 if (ret) { 9506 dev_info(&pf->pdev->dev, 9507 "update vsi failed, err %s aq_err %s\n", 9508 i40e_stat_str(&pf->hw, ret), 9509 i40e_aq_str(&pf->hw, 9510 pf->hw.aq.asq_last_status)); 9511 ret = -ENOENT; 9512 goto err; 9513 } 9514 /* update the local VSI info queue map */ 9515 i40e_vsi_update_queue_map(vsi, &ctxt); 9516 vsi->info.valid_sections = 0; 9517 } else { 9518 /* Default/Main VSI is only enabled for TC0 9519 * reconfigure it to enable all TCs that are 9520 * available on the port in SFP mode. 9521 * For MFP case the iSCSI PF would use this 9522 * flow to enable LAN+iSCSI TC. 9523 */ 9524 ret = i40e_vsi_config_tc(vsi, enabled_tc); 9525 if (ret) { 9526 dev_info(&pf->pdev->dev, 9527 "failed to configure TCs for main VSI tc_map 0x%08x, err %s aq_err %s\n", 9528 enabled_tc, 9529 i40e_stat_str(&pf->hw, ret), 9530 i40e_aq_str(&pf->hw, 9531 pf->hw.aq.asq_last_status)); 9532 ret = -ENOENT; 9533 } 9534 } 9535 break; 9536 9537 case I40E_VSI_FDIR: 9538 ctxt.pf_num = hw->pf_id; 9539 ctxt.vf_num = 0; 9540 ctxt.uplink_seid = vsi->uplink_seid; 9541 ctxt.connection_type = I40E_AQ_VSI_CONN_TYPE_NORMAL; 9542 ctxt.flags = I40E_AQ_VSI_TYPE_PF; 9543 if ((pf->flags & I40E_FLAG_VEB_MODE_ENABLED) && 9544 (i40e_is_vsi_uplink_mode_veb(vsi))) { 9545 ctxt.info.valid_sections |= 9546 cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID); 9547 ctxt.info.switch_id = 9548 cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB); 9549 } 9550 i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, true); 9551 break; 9552 9553 case I40E_VSI_VMDQ2: 9554 ctxt.pf_num = hw->pf_id; 9555 ctxt.vf_num = 0; 9556 ctxt.uplink_seid = vsi->uplink_seid; 9557 ctxt.connection_type = I40E_AQ_VSI_CONN_TYPE_NORMAL; 9558 ctxt.flags = I40E_AQ_VSI_TYPE_VMDQ2; 9559 9560 /* This VSI is connected to VEB so the switch_id 9561 * should be set to zero by default. 9562 */ 9563 if (i40e_is_vsi_uplink_mode_veb(vsi)) { 9564 ctxt.info.valid_sections |= 9565 cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID); 9566 ctxt.info.switch_id = 9567 cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB); 9568 } 9569 9570 /* Setup the VSI tx/rx queue map for TC0 only for now */ 9571 i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, true); 9572 break; 9573 9574 case I40E_VSI_SRIOV: 9575 ctxt.pf_num = hw->pf_id; 9576 ctxt.vf_num = vsi->vf_id + hw->func_caps.vf_base_id; 9577 ctxt.uplink_seid = vsi->uplink_seid; 9578 ctxt.connection_type = I40E_AQ_VSI_CONN_TYPE_NORMAL; 9579 ctxt.flags = I40E_AQ_VSI_TYPE_VF; 9580 9581 /* This VSI is connected to VEB so the switch_id 9582 * should be set to zero by default. 9583 */ 9584 if (i40e_is_vsi_uplink_mode_veb(vsi)) { 9585 ctxt.info.valid_sections |= 9586 cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID); 9587 ctxt.info.switch_id = 9588 cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB); 9589 } 9590 9591 if (vsi->back->flags & I40E_FLAG_IWARP_ENABLED) { 9592 ctxt.info.valid_sections |= 9593 cpu_to_le16(I40E_AQ_VSI_PROP_QUEUE_OPT_VALID); 9594 ctxt.info.queueing_opt_flags |= 9595 (I40E_AQ_VSI_QUE_OPT_TCP_ENA | 9596 I40E_AQ_VSI_QUE_OPT_RSS_LUT_VSI); 9597 } 9598 9599 ctxt.info.valid_sections |= cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID); 9600 ctxt.info.port_vlan_flags |= I40E_AQ_VSI_PVLAN_MODE_ALL; 9601 if (pf->vf[vsi->vf_id].spoofchk) { 9602 ctxt.info.valid_sections |= 9603 cpu_to_le16(I40E_AQ_VSI_PROP_SECURITY_VALID); 9604 ctxt.info.sec_flags |= 9605 (I40E_AQ_VSI_SEC_FLAG_ENABLE_VLAN_CHK | 9606 I40E_AQ_VSI_SEC_FLAG_ENABLE_MAC_CHK); 9607 } 9608 /* Setup the VSI tx/rx queue map for TC0 only for now */ 9609 i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, true); 9610 break; 9611 9612 #ifdef I40E_FCOE 9613 case I40E_VSI_FCOE: 9614 ret = i40e_fcoe_vsi_init(vsi, &ctxt); 9615 if (ret) { 9616 dev_info(&pf->pdev->dev, "failed to initialize FCoE VSI\n"); 9617 return ret; 9618 } 9619 break; 9620 9621 #endif /* I40E_FCOE */ 9622 case I40E_VSI_IWARP: 9623 /* send down message to iWARP */ 9624 break; 9625 9626 default: 9627 return -ENODEV; 9628 } 9629 9630 if (vsi->type != I40E_VSI_MAIN) { 9631 ret = i40e_aq_add_vsi(hw, &ctxt, NULL); 9632 if (ret) { 9633 dev_info(&vsi->back->pdev->dev, 9634 "add vsi failed, err %s aq_err %s\n", 9635 i40e_stat_str(&pf->hw, ret), 9636 i40e_aq_str(&pf->hw, 9637 pf->hw.aq.asq_last_status)); 9638 ret = -ENOENT; 9639 goto err; 9640 } 9641 vsi->info = ctxt.info; 9642 vsi->info.valid_sections = 0; 9643 vsi->seid = ctxt.seid; 9644 vsi->id = ctxt.vsi_number; 9645 } 9646 9647 vsi->active_filters = 0; 9648 clear_bit(__I40E_FILTER_OVERFLOW_PROMISC, &vsi->state); 9649 spin_lock_bh(&vsi->mac_filter_hash_lock); 9650 /* If macvlan filters already exist, force them to get loaded */ 9651 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) { 9652 f->state = I40E_FILTER_NEW; 9653 f_count++; 9654 } 9655 spin_unlock_bh(&vsi->mac_filter_hash_lock); 9656 9657 if (f_count) { 9658 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED; 9659 pf->flags |= I40E_FLAG_FILTER_SYNC; 9660 } 9661 9662 /* Update VSI BW information */ 9663 ret = i40e_vsi_get_bw_info(vsi); 9664 if (ret) { 9665 dev_info(&pf->pdev->dev, 9666 "couldn't get vsi bw info, err %s aq_err %s\n", 9667 i40e_stat_str(&pf->hw, ret), 9668 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 9669 /* VSI is already added so not tearing that up */ 9670 ret = 0; 9671 } 9672 9673 err: 9674 return ret; 9675 } 9676 9677 /** 9678 * i40e_vsi_release - Delete a VSI and free its resources 9679 * @vsi: the VSI being removed 9680 * 9681 * Returns 0 on success or < 0 on error 9682 **/ 9683 int i40e_vsi_release(struct i40e_vsi *vsi) 9684 { 9685 struct i40e_mac_filter *f; 9686 struct hlist_node *h; 9687 struct i40e_veb *veb = NULL; 9688 struct i40e_pf *pf; 9689 u16 uplink_seid; 9690 int i, n, bkt; 9691 9692 pf = vsi->back; 9693 9694 /* release of a VEB-owner or last VSI is not allowed */ 9695 if (vsi->flags & I40E_VSI_FLAG_VEB_OWNER) { 9696 dev_info(&pf->pdev->dev, "VSI %d has existing VEB %d\n", 9697 vsi->seid, vsi->uplink_seid); 9698 return -ENODEV; 9699 } 9700 if (vsi == pf->vsi[pf->lan_vsi] && 9701 !test_bit(__I40E_DOWN, &pf->state)) { 9702 dev_info(&pf->pdev->dev, "Can't remove PF VSI\n"); 9703 return -ENODEV; 9704 } 9705 9706 uplink_seid = vsi->uplink_seid; 9707 if (vsi->type != I40E_VSI_SRIOV) { 9708 if (vsi->netdev_registered) { 9709 vsi->netdev_registered = false; 9710 if (vsi->netdev) { 9711 /* results in a call to i40e_close() */ 9712 unregister_netdev(vsi->netdev); 9713 } 9714 } else { 9715 i40e_vsi_close(vsi); 9716 } 9717 i40e_vsi_disable_irq(vsi); 9718 } 9719 9720 spin_lock_bh(&vsi->mac_filter_hash_lock); 9721 9722 /* clear the sync flag on all filters */ 9723 if (vsi->netdev) { 9724 __dev_uc_unsync(vsi->netdev, NULL); 9725 __dev_mc_unsync(vsi->netdev, NULL); 9726 } 9727 9728 /* make sure any remaining filters are marked for deletion */ 9729 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) 9730 __i40e_del_filter(vsi, f); 9731 9732 spin_unlock_bh(&vsi->mac_filter_hash_lock); 9733 9734 i40e_sync_vsi_filters(vsi); 9735 9736 i40e_vsi_delete(vsi); 9737 i40e_vsi_free_q_vectors(vsi); 9738 if (vsi->netdev) { 9739 free_netdev(vsi->netdev); 9740 vsi->netdev = NULL; 9741 } 9742 i40e_vsi_clear_rings(vsi); 9743 i40e_vsi_clear(vsi); 9744 9745 /* If this was the last thing on the VEB, except for the 9746 * controlling VSI, remove the VEB, which puts the controlling 9747 * VSI onto the next level down in the switch. 9748 * 9749 * Well, okay, there's one more exception here: don't remove 9750 * the orphan VEBs yet. We'll wait for an explicit remove request 9751 * from up the network stack. 9752 */ 9753 for (n = 0, i = 0; i < pf->num_alloc_vsi; i++) { 9754 if (pf->vsi[i] && 9755 pf->vsi[i]->uplink_seid == uplink_seid && 9756 (pf->vsi[i]->flags & I40E_VSI_FLAG_VEB_OWNER) == 0) { 9757 n++; /* count the VSIs */ 9758 } 9759 } 9760 for (i = 0; i < I40E_MAX_VEB; i++) { 9761 if (!pf->veb[i]) 9762 continue; 9763 if (pf->veb[i]->uplink_seid == uplink_seid) 9764 n++; /* count the VEBs */ 9765 if (pf->veb[i]->seid == uplink_seid) 9766 veb = pf->veb[i]; 9767 } 9768 if (n == 0 && veb && veb->uplink_seid != 0) 9769 i40e_veb_release(veb); 9770 9771 return 0; 9772 } 9773 9774 /** 9775 * i40e_vsi_setup_vectors - Set up the q_vectors for the given VSI 9776 * @vsi: ptr to the VSI 9777 * 9778 * This should only be called after i40e_vsi_mem_alloc() which allocates the 9779 * corresponding SW VSI structure and initializes num_queue_pairs for the 9780 * newly allocated VSI. 9781 * 9782 * Returns 0 on success or negative on failure 9783 **/ 9784 static int i40e_vsi_setup_vectors(struct i40e_vsi *vsi) 9785 { 9786 int ret = -ENOENT; 9787 struct i40e_pf *pf = vsi->back; 9788 9789 if (vsi->q_vectors[0]) { 9790 dev_info(&pf->pdev->dev, "VSI %d has existing q_vectors\n", 9791 vsi->seid); 9792 return -EEXIST; 9793 } 9794 9795 if (vsi->base_vector) { 9796 dev_info(&pf->pdev->dev, "VSI %d has non-zero base vector %d\n", 9797 vsi->seid, vsi->base_vector); 9798 return -EEXIST; 9799 } 9800 9801 ret = i40e_vsi_alloc_q_vectors(vsi); 9802 if (ret) { 9803 dev_info(&pf->pdev->dev, 9804 "failed to allocate %d q_vector for VSI %d, ret=%d\n", 9805 vsi->num_q_vectors, vsi->seid, ret); 9806 vsi->num_q_vectors = 0; 9807 goto vector_setup_out; 9808 } 9809 9810 /* In Legacy mode, we do not have to get any other vector since we 9811 * piggyback on the misc/ICR0 for queue interrupts. 9812 */ 9813 if (!(pf->flags & I40E_FLAG_MSIX_ENABLED)) 9814 return ret; 9815 if (vsi->num_q_vectors) 9816 vsi->base_vector = i40e_get_lump(pf, pf->irq_pile, 9817 vsi->num_q_vectors, vsi->idx); 9818 if (vsi->base_vector < 0) { 9819 dev_info(&pf->pdev->dev, 9820 "failed to get tracking for %d vectors for VSI %d, err=%d\n", 9821 vsi->num_q_vectors, vsi->seid, vsi->base_vector); 9822 i40e_vsi_free_q_vectors(vsi); 9823 ret = -ENOENT; 9824 goto vector_setup_out; 9825 } 9826 9827 vector_setup_out: 9828 return ret; 9829 } 9830 9831 /** 9832 * i40e_vsi_reinit_setup - return and reallocate resources for a VSI 9833 * @vsi: pointer to the vsi. 9834 * 9835 * This re-allocates a vsi's queue resources. 9836 * 9837 * Returns pointer to the successfully allocated and configured VSI sw struct 9838 * on success, otherwise returns NULL on failure. 9839 **/ 9840 static struct i40e_vsi *i40e_vsi_reinit_setup(struct i40e_vsi *vsi) 9841 { 9842 struct i40e_pf *pf; 9843 u8 enabled_tc; 9844 int ret; 9845 9846 if (!vsi) 9847 return NULL; 9848 9849 pf = vsi->back; 9850 9851 i40e_put_lump(pf->qp_pile, vsi->base_queue, vsi->idx); 9852 i40e_vsi_clear_rings(vsi); 9853 9854 i40e_vsi_free_arrays(vsi, false); 9855 i40e_set_num_rings_in_vsi(vsi); 9856 ret = i40e_vsi_alloc_arrays(vsi, false); 9857 if (ret) 9858 goto err_vsi; 9859 9860 ret = i40e_get_lump(pf, pf->qp_pile, vsi->alloc_queue_pairs, vsi->idx); 9861 if (ret < 0) { 9862 dev_info(&pf->pdev->dev, 9863 "failed to get tracking for %d queues for VSI %d err %d\n", 9864 vsi->alloc_queue_pairs, vsi->seid, ret); 9865 goto err_vsi; 9866 } 9867 vsi->base_queue = ret; 9868 9869 /* Update the FW view of the VSI. Force a reset of TC and queue 9870 * layout configurations. 9871 */ 9872 enabled_tc = pf->vsi[pf->lan_vsi]->tc_config.enabled_tc; 9873 pf->vsi[pf->lan_vsi]->tc_config.enabled_tc = 0; 9874 pf->vsi[pf->lan_vsi]->seid = pf->main_vsi_seid; 9875 i40e_vsi_config_tc(pf->vsi[pf->lan_vsi], enabled_tc); 9876 if (vsi->type == I40E_VSI_MAIN) 9877 i40e_rm_default_mac_filter(vsi, pf->hw.mac.perm_addr); 9878 9879 /* assign it some queues */ 9880 ret = i40e_alloc_rings(vsi); 9881 if (ret) 9882 goto err_rings; 9883 9884 /* map all of the rings to the q_vectors */ 9885 i40e_vsi_map_rings_to_vectors(vsi); 9886 return vsi; 9887 9888 err_rings: 9889 i40e_vsi_free_q_vectors(vsi); 9890 if (vsi->netdev_registered) { 9891 vsi->netdev_registered = false; 9892 unregister_netdev(vsi->netdev); 9893 free_netdev(vsi->netdev); 9894 vsi->netdev = NULL; 9895 } 9896 i40e_aq_delete_element(&pf->hw, vsi->seid, NULL); 9897 err_vsi: 9898 i40e_vsi_clear(vsi); 9899 return NULL; 9900 } 9901 9902 /** 9903 * i40e_vsi_setup - Set up a VSI by a given type 9904 * @pf: board private structure 9905 * @type: VSI type 9906 * @uplink_seid: the switch element to link to 9907 * @param1: usage depends upon VSI type. For VF types, indicates VF id 9908 * 9909 * This allocates the sw VSI structure and its queue resources, then add a VSI 9910 * to the identified VEB. 9911 * 9912 * Returns pointer to the successfully allocated and configure VSI sw struct on 9913 * success, otherwise returns NULL on failure. 9914 **/ 9915 struct i40e_vsi *i40e_vsi_setup(struct i40e_pf *pf, u8 type, 9916 u16 uplink_seid, u32 param1) 9917 { 9918 struct i40e_vsi *vsi = NULL; 9919 struct i40e_veb *veb = NULL; 9920 int ret, i; 9921 int v_idx; 9922 9923 /* The requested uplink_seid must be either 9924 * - the PF's port seid 9925 * no VEB is needed because this is the PF 9926 * or this is a Flow Director special case VSI 9927 * - seid of an existing VEB 9928 * - seid of a VSI that owns an existing VEB 9929 * - seid of a VSI that doesn't own a VEB 9930 * a new VEB is created and the VSI becomes the owner 9931 * - seid of the PF VSI, which is what creates the first VEB 9932 * this is a special case of the previous 9933 * 9934 * Find which uplink_seid we were given and create a new VEB if needed 9935 */ 9936 for (i = 0; i < I40E_MAX_VEB; i++) { 9937 if (pf->veb[i] && pf->veb[i]->seid == uplink_seid) { 9938 veb = pf->veb[i]; 9939 break; 9940 } 9941 } 9942 9943 if (!veb && uplink_seid != pf->mac_seid) { 9944 9945 for (i = 0; i < pf->num_alloc_vsi; i++) { 9946 if (pf->vsi[i] && pf->vsi[i]->seid == uplink_seid) { 9947 vsi = pf->vsi[i]; 9948 break; 9949 } 9950 } 9951 if (!vsi) { 9952 dev_info(&pf->pdev->dev, "no such uplink_seid %d\n", 9953 uplink_seid); 9954 return NULL; 9955 } 9956 9957 if (vsi->uplink_seid == pf->mac_seid) 9958 veb = i40e_veb_setup(pf, 0, pf->mac_seid, vsi->seid, 9959 vsi->tc_config.enabled_tc); 9960 else if ((vsi->flags & I40E_VSI_FLAG_VEB_OWNER) == 0) 9961 veb = i40e_veb_setup(pf, 0, vsi->uplink_seid, vsi->seid, 9962 vsi->tc_config.enabled_tc); 9963 if (veb) { 9964 if (vsi->seid != pf->vsi[pf->lan_vsi]->seid) { 9965 dev_info(&vsi->back->pdev->dev, 9966 "New VSI creation error, uplink seid of LAN VSI expected.\n"); 9967 return NULL; 9968 } 9969 /* We come up by default in VEPA mode if SRIOV is not 9970 * already enabled, in which case we can't force VEPA 9971 * mode. 9972 */ 9973 if (!(pf->flags & I40E_FLAG_VEB_MODE_ENABLED)) { 9974 veb->bridge_mode = BRIDGE_MODE_VEPA; 9975 pf->flags &= ~I40E_FLAG_VEB_MODE_ENABLED; 9976 } 9977 i40e_config_bridge_mode(veb); 9978 } 9979 for (i = 0; i < I40E_MAX_VEB && !veb; i++) { 9980 if (pf->veb[i] && pf->veb[i]->seid == vsi->uplink_seid) 9981 veb = pf->veb[i]; 9982 } 9983 if (!veb) { 9984 dev_info(&pf->pdev->dev, "couldn't add VEB\n"); 9985 return NULL; 9986 } 9987 9988 vsi->flags |= I40E_VSI_FLAG_VEB_OWNER; 9989 uplink_seid = veb->seid; 9990 } 9991 9992 /* get vsi sw struct */ 9993 v_idx = i40e_vsi_mem_alloc(pf, type); 9994 if (v_idx < 0) 9995 goto err_alloc; 9996 vsi = pf->vsi[v_idx]; 9997 if (!vsi) 9998 goto err_alloc; 9999 vsi->type = type; 10000 vsi->veb_idx = (veb ? veb->idx : I40E_NO_VEB); 10001 10002 if (type == I40E_VSI_MAIN) 10003 pf->lan_vsi = v_idx; 10004 else if (type == I40E_VSI_SRIOV) 10005 vsi->vf_id = param1; 10006 /* assign it some queues */ 10007 ret = i40e_get_lump(pf, pf->qp_pile, vsi->alloc_queue_pairs, 10008 vsi->idx); 10009 if (ret < 0) { 10010 dev_info(&pf->pdev->dev, 10011 "failed to get tracking for %d queues for VSI %d err=%d\n", 10012 vsi->alloc_queue_pairs, vsi->seid, ret); 10013 goto err_vsi; 10014 } 10015 vsi->base_queue = ret; 10016 10017 /* get a VSI from the hardware */ 10018 vsi->uplink_seid = uplink_seid; 10019 ret = i40e_add_vsi(vsi); 10020 if (ret) 10021 goto err_vsi; 10022 10023 switch (vsi->type) { 10024 /* setup the netdev if needed */ 10025 case I40E_VSI_MAIN: 10026 /* Apply relevant filters if a platform-specific mac 10027 * address was selected. 10028 */ 10029 if (!!(pf->flags & I40E_FLAG_PF_MAC)) { 10030 ret = i40e_macaddr_init(vsi, pf->hw.mac.addr); 10031 if (ret) { 10032 dev_warn(&pf->pdev->dev, 10033 "could not set up macaddr; err %d\n", 10034 ret); 10035 } 10036 } 10037 case I40E_VSI_VMDQ2: 10038 case I40E_VSI_FCOE: 10039 ret = i40e_config_netdev(vsi); 10040 if (ret) 10041 goto err_netdev; 10042 ret = register_netdev(vsi->netdev); 10043 if (ret) 10044 goto err_netdev; 10045 vsi->netdev_registered = true; 10046 netif_carrier_off(vsi->netdev); 10047 #ifdef CONFIG_I40E_DCB 10048 /* Setup DCB netlink interface */ 10049 i40e_dcbnl_setup(vsi); 10050 #endif /* CONFIG_I40E_DCB */ 10051 /* fall through */ 10052 10053 case I40E_VSI_FDIR: 10054 /* set up vectors and rings if needed */ 10055 ret = i40e_vsi_setup_vectors(vsi); 10056 if (ret) 10057 goto err_msix; 10058 10059 ret = i40e_alloc_rings(vsi); 10060 if (ret) 10061 goto err_rings; 10062 10063 /* map all of the rings to the q_vectors */ 10064 i40e_vsi_map_rings_to_vectors(vsi); 10065 10066 i40e_vsi_reset_stats(vsi); 10067 break; 10068 10069 default: 10070 /* no netdev or rings for the other VSI types */ 10071 break; 10072 } 10073 10074 if ((pf->flags & I40E_FLAG_RSS_AQ_CAPABLE) && 10075 (vsi->type == I40E_VSI_VMDQ2)) { 10076 ret = i40e_vsi_config_rss(vsi); 10077 } 10078 return vsi; 10079 10080 err_rings: 10081 i40e_vsi_free_q_vectors(vsi); 10082 err_msix: 10083 if (vsi->netdev_registered) { 10084 vsi->netdev_registered = false; 10085 unregister_netdev(vsi->netdev); 10086 free_netdev(vsi->netdev); 10087 vsi->netdev = NULL; 10088 } 10089 err_netdev: 10090 i40e_aq_delete_element(&pf->hw, vsi->seid, NULL); 10091 err_vsi: 10092 i40e_vsi_clear(vsi); 10093 err_alloc: 10094 return NULL; 10095 } 10096 10097 /** 10098 * i40e_veb_get_bw_info - Query VEB BW information 10099 * @veb: the veb to query 10100 * 10101 * Query the Tx scheduler BW configuration data for given VEB 10102 **/ 10103 static int i40e_veb_get_bw_info(struct i40e_veb *veb) 10104 { 10105 struct i40e_aqc_query_switching_comp_ets_config_resp ets_data; 10106 struct i40e_aqc_query_switching_comp_bw_config_resp bw_data; 10107 struct i40e_pf *pf = veb->pf; 10108 struct i40e_hw *hw = &pf->hw; 10109 u32 tc_bw_max; 10110 int ret = 0; 10111 int i; 10112 10113 ret = i40e_aq_query_switch_comp_bw_config(hw, veb->seid, 10114 &bw_data, NULL); 10115 if (ret) { 10116 dev_info(&pf->pdev->dev, 10117 "query veb bw config failed, err %s aq_err %s\n", 10118 i40e_stat_str(&pf->hw, ret), 10119 i40e_aq_str(&pf->hw, hw->aq.asq_last_status)); 10120 goto out; 10121 } 10122 10123 ret = i40e_aq_query_switch_comp_ets_config(hw, veb->seid, 10124 &ets_data, NULL); 10125 if (ret) { 10126 dev_info(&pf->pdev->dev, 10127 "query veb bw ets config failed, err %s aq_err %s\n", 10128 i40e_stat_str(&pf->hw, ret), 10129 i40e_aq_str(&pf->hw, hw->aq.asq_last_status)); 10130 goto out; 10131 } 10132 10133 veb->bw_limit = le16_to_cpu(ets_data.port_bw_limit); 10134 veb->bw_max_quanta = ets_data.tc_bw_max; 10135 veb->is_abs_credits = bw_data.absolute_credits_enable; 10136 veb->enabled_tc = ets_data.tc_valid_bits; 10137 tc_bw_max = le16_to_cpu(bw_data.tc_bw_max[0]) | 10138 (le16_to_cpu(bw_data.tc_bw_max[1]) << 16); 10139 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 10140 veb->bw_tc_share_credits[i] = bw_data.tc_bw_share_credits[i]; 10141 veb->bw_tc_limit_credits[i] = 10142 le16_to_cpu(bw_data.tc_bw_limits[i]); 10143 veb->bw_tc_max_quanta[i] = ((tc_bw_max >> (i*4)) & 0x7); 10144 } 10145 10146 out: 10147 return ret; 10148 } 10149 10150 /** 10151 * i40e_veb_mem_alloc - Allocates the next available struct veb in the PF 10152 * @pf: board private structure 10153 * 10154 * On error: returns error code (negative) 10155 * On success: returns vsi index in PF (positive) 10156 **/ 10157 static int i40e_veb_mem_alloc(struct i40e_pf *pf) 10158 { 10159 int ret = -ENOENT; 10160 struct i40e_veb *veb; 10161 int i; 10162 10163 /* Need to protect the allocation of switch elements at the PF level */ 10164 mutex_lock(&pf->switch_mutex); 10165 10166 /* VEB list may be fragmented if VEB creation/destruction has 10167 * been happening. We can afford to do a quick scan to look 10168 * for any free slots in the list. 10169 * 10170 * find next empty veb slot, looping back around if necessary 10171 */ 10172 i = 0; 10173 while ((i < I40E_MAX_VEB) && (pf->veb[i] != NULL)) 10174 i++; 10175 if (i >= I40E_MAX_VEB) { 10176 ret = -ENOMEM; 10177 goto err_alloc_veb; /* out of VEB slots! */ 10178 } 10179 10180 veb = kzalloc(sizeof(*veb), GFP_KERNEL); 10181 if (!veb) { 10182 ret = -ENOMEM; 10183 goto err_alloc_veb; 10184 } 10185 veb->pf = pf; 10186 veb->idx = i; 10187 veb->enabled_tc = 1; 10188 10189 pf->veb[i] = veb; 10190 ret = i; 10191 err_alloc_veb: 10192 mutex_unlock(&pf->switch_mutex); 10193 return ret; 10194 } 10195 10196 /** 10197 * i40e_switch_branch_release - Delete a branch of the switch tree 10198 * @branch: where to start deleting 10199 * 10200 * This uses recursion to find the tips of the branch to be 10201 * removed, deleting until we get back to and can delete this VEB. 10202 **/ 10203 static void i40e_switch_branch_release(struct i40e_veb *branch) 10204 { 10205 struct i40e_pf *pf = branch->pf; 10206 u16 branch_seid = branch->seid; 10207 u16 veb_idx = branch->idx; 10208 int i; 10209 10210 /* release any VEBs on this VEB - RECURSION */ 10211 for (i = 0; i < I40E_MAX_VEB; i++) { 10212 if (!pf->veb[i]) 10213 continue; 10214 if (pf->veb[i]->uplink_seid == branch->seid) 10215 i40e_switch_branch_release(pf->veb[i]); 10216 } 10217 10218 /* Release the VSIs on this VEB, but not the owner VSI. 10219 * 10220 * NOTE: Removing the last VSI on a VEB has the SIDE EFFECT of removing 10221 * the VEB itself, so don't use (*branch) after this loop. 10222 */ 10223 for (i = 0; i < pf->num_alloc_vsi; i++) { 10224 if (!pf->vsi[i]) 10225 continue; 10226 if (pf->vsi[i]->uplink_seid == branch_seid && 10227 (pf->vsi[i]->flags & I40E_VSI_FLAG_VEB_OWNER) == 0) { 10228 i40e_vsi_release(pf->vsi[i]); 10229 } 10230 } 10231 10232 /* There's one corner case where the VEB might not have been 10233 * removed, so double check it here and remove it if needed. 10234 * This case happens if the veb was created from the debugfs 10235 * commands and no VSIs were added to it. 10236 */ 10237 if (pf->veb[veb_idx]) 10238 i40e_veb_release(pf->veb[veb_idx]); 10239 } 10240 10241 /** 10242 * i40e_veb_clear - remove veb struct 10243 * @veb: the veb to remove 10244 **/ 10245 static void i40e_veb_clear(struct i40e_veb *veb) 10246 { 10247 if (!veb) 10248 return; 10249 10250 if (veb->pf) { 10251 struct i40e_pf *pf = veb->pf; 10252 10253 mutex_lock(&pf->switch_mutex); 10254 if (pf->veb[veb->idx] == veb) 10255 pf->veb[veb->idx] = NULL; 10256 mutex_unlock(&pf->switch_mutex); 10257 } 10258 10259 kfree(veb); 10260 } 10261 10262 /** 10263 * i40e_veb_release - Delete a VEB and free its resources 10264 * @veb: the VEB being removed 10265 **/ 10266 void i40e_veb_release(struct i40e_veb *veb) 10267 { 10268 struct i40e_vsi *vsi = NULL; 10269 struct i40e_pf *pf; 10270 int i, n = 0; 10271 10272 pf = veb->pf; 10273 10274 /* find the remaining VSI and check for extras */ 10275 for (i = 0; i < pf->num_alloc_vsi; i++) { 10276 if (pf->vsi[i] && pf->vsi[i]->uplink_seid == veb->seid) { 10277 n++; 10278 vsi = pf->vsi[i]; 10279 } 10280 } 10281 if (n != 1) { 10282 dev_info(&pf->pdev->dev, 10283 "can't remove VEB %d with %d VSIs left\n", 10284 veb->seid, n); 10285 return; 10286 } 10287 10288 /* move the remaining VSI to uplink veb */ 10289 vsi->flags &= ~I40E_VSI_FLAG_VEB_OWNER; 10290 if (veb->uplink_seid) { 10291 vsi->uplink_seid = veb->uplink_seid; 10292 if (veb->uplink_seid == pf->mac_seid) 10293 vsi->veb_idx = I40E_NO_VEB; 10294 else 10295 vsi->veb_idx = veb->veb_idx; 10296 } else { 10297 /* floating VEB */ 10298 vsi->uplink_seid = pf->vsi[pf->lan_vsi]->uplink_seid; 10299 vsi->veb_idx = pf->vsi[pf->lan_vsi]->veb_idx; 10300 } 10301 10302 i40e_aq_delete_element(&pf->hw, veb->seid, NULL); 10303 i40e_veb_clear(veb); 10304 } 10305 10306 /** 10307 * i40e_add_veb - create the VEB in the switch 10308 * @veb: the VEB to be instantiated 10309 * @vsi: the controlling VSI 10310 **/ 10311 static int i40e_add_veb(struct i40e_veb *veb, struct i40e_vsi *vsi) 10312 { 10313 struct i40e_pf *pf = veb->pf; 10314 bool enable_stats = !!(pf->flags & I40E_FLAG_VEB_STATS_ENABLED); 10315 int ret; 10316 10317 ret = i40e_aq_add_veb(&pf->hw, veb->uplink_seid, vsi->seid, 10318 veb->enabled_tc, false, 10319 &veb->seid, enable_stats, NULL); 10320 10321 /* get a VEB from the hardware */ 10322 if (ret) { 10323 dev_info(&pf->pdev->dev, 10324 "couldn't add VEB, err %s aq_err %s\n", 10325 i40e_stat_str(&pf->hw, ret), 10326 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 10327 return -EPERM; 10328 } 10329 10330 /* get statistics counter */ 10331 ret = i40e_aq_get_veb_parameters(&pf->hw, veb->seid, NULL, NULL, 10332 &veb->stats_idx, NULL, NULL, NULL); 10333 if (ret) { 10334 dev_info(&pf->pdev->dev, 10335 "couldn't get VEB statistics idx, err %s aq_err %s\n", 10336 i40e_stat_str(&pf->hw, ret), 10337 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 10338 return -EPERM; 10339 } 10340 ret = i40e_veb_get_bw_info(veb); 10341 if (ret) { 10342 dev_info(&pf->pdev->dev, 10343 "couldn't get VEB bw info, err %s aq_err %s\n", 10344 i40e_stat_str(&pf->hw, ret), 10345 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 10346 i40e_aq_delete_element(&pf->hw, veb->seid, NULL); 10347 return -ENOENT; 10348 } 10349 10350 vsi->uplink_seid = veb->seid; 10351 vsi->veb_idx = veb->idx; 10352 vsi->flags |= I40E_VSI_FLAG_VEB_OWNER; 10353 10354 return 0; 10355 } 10356 10357 /** 10358 * i40e_veb_setup - Set up a VEB 10359 * @pf: board private structure 10360 * @flags: VEB setup flags 10361 * @uplink_seid: the switch element to link to 10362 * @vsi_seid: the initial VSI seid 10363 * @enabled_tc: Enabled TC bit-map 10364 * 10365 * This allocates the sw VEB structure and links it into the switch 10366 * It is possible and legal for this to be a duplicate of an already 10367 * existing VEB. It is also possible for both uplink and vsi seids 10368 * to be zero, in order to create a floating VEB. 10369 * 10370 * Returns pointer to the successfully allocated VEB sw struct on 10371 * success, otherwise returns NULL on failure. 10372 **/ 10373 struct i40e_veb *i40e_veb_setup(struct i40e_pf *pf, u16 flags, 10374 u16 uplink_seid, u16 vsi_seid, 10375 u8 enabled_tc) 10376 { 10377 struct i40e_veb *veb, *uplink_veb = NULL; 10378 int vsi_idx, veb_idx; 10379 int ret; 10380 10381 /* if one seid is 0, the other must be 0 to create a floating relay */ 10382 if ((uplink_seid == 0 || vsi_seid == 0) && 10383 (uplink_seid + vsi_seid != 0)) { 10384 dev_info(&pf->pdev->dev, 10385 "one, not both seid's are 0: uplink=%d vsi=%d\n", 10386 uplink_seid, vsi_seid); 10387 return NULL; 10388 } 10389 10390 /* make sure there is such a vsi and uplink */ 10391 for (vsi_idx = 0; vsi_idx < pf->num_alloc_vsi; vsi_idx++) 10392 if (pf->vsi[vsi_idx] && pf->vsi[vsi_idx]->seid == vsi_seid) 10393 break; 10394 if (vsi_idx >= pf->num_alloc_vsi && vsi_seid != 0) { 10395 dev_info(&pf->pdev->dev, "vsi seid %d not found\n", 10396 vsi_seid); 10397 return NULL; 10398 } 10399 10400 if (uplink_seid && uplink_seid != pf->mac_seid) { 10401 for (veb_idx = 0; veb_idx < I40E_MAX_VEB; veb_idx++) { 10402 if (pf->veb[veb_idx] && 10403 pf->veb[veb_idx]->seid == uplink_seid) { 10404 uplink_veb = pf->veb[veb_idx]; 10405 break; 10406 } 10407 } 10408 if (!uplink_veb) { 10409 dev_info(&pf->pdev->dev, 10410 "uplink seid %d not found\n", uplink_seid); 10411 return NULL; 10412 } 10413 } 10414 10415 /* get veb sw struct */ 10416 veb_idx = i40e_veb_mem_alloc(pf); 10417 if (veb_idx < 0) 10418 goto err_alloc; 10419 veb = pf->veb[veb_idx]; 10420 veb->flags = flags; 10421 veb->uplink_seid = uplink_seid; 10422 veb->veb_idx = (uplink_veb ? uplink_veb->idx : I40E_NO_VEB); 10423 veb->enabled_tc = (enabled_tc ? enabled_tc : 0x1); 10424 10425 /* create the VEB in the switch */ 10426 ret = i40e_add_veb(veb, pf->vsi[vsi_idx]); 10427 if (ret) 10428 goto err_veb; 10429 if (vsi_idx == pf->lan_vsi) 10430 pf->lan_veb = veb->idx; 10431 10432 return veb; 10433 10434 err_veb: 10435 i40e_veb_clear(veb); 10436 err_alloc: 10437 return NULL; 10438 } 10439 10440 /** 10441 * i40e_setup_pf_switch_element - set PF vars based on switch type 10442 * @pf: board private structure 10443 * @ele: element we are building info from 10444 * @num_reported: total number of elements 10445 * @printconfig: should we print the contents 10446 * 10447 * helper function to assist in extracting a few useful SEID values. 10448 **/ 10449 static void i40e_setup_pf_switch_element(struct i40e_pf *pf, 10450 struct i40e_aqc_switch_config_element_resp *ele, 10451 u16 num_reported, bool printconfig) 10452 { 10453 u16 downlink_seid = le16_to_cpu(ele->downlink_seid); 10454 u16 uplink_seid = le16_to_cpu(ele->uplink_seid); 10455 u8 element_type = ele->element_type; 10456 u16 seid = le16_to_cpu(ele->seid); 10457 10458 if (printconfig) 10459 dev_info(&pf->pdev->dev, 10460 "type=%d seid=%d uplink=%d downlink=%d\n", 10461 element_type, seid, uplink_seid, downlink_seid); 10462 10463 switch (element_type) { 10464 case I40E_SWITCH_ELEMENT_TYPE_MAC: 10465 pf->mac_seid = seid; 10466 break; 10467 case I40E_SWITCH_ELEMENT_TYPE_VEB: 10468 /* Main VEB? */ 10469 if (uplink_seid != pf->mac_seid) 10470 break; 10471 if (pf->lan_veb == I40E_NO_VEB) { 10472 int v; 10473 10474 /* find existing or else empty VEB */ 10475 for (v = 0; v < I40E_MAX_VEB; v++) { 10476 if (pf->veb[v] && (pf->veb[v]->seid == seid)) { 10477 pf->lan_veb = v; 10478 break; 10479 } 10480 } 10481 if (pf->lan_veb == I40E_NO_VEB) { 10482 v = i40e_veb_mem_alloc(pf); 10483 if (v < 0) 10484 break; 10485 pf->lan_veb = v; 10486 } 10487 } 10488 10489 pf->veb[pf->lan_veb]->seid = seid; 10490 pf->veb[pf->lan_veb]->uplink_seid = pf->mac_seid; 10491 pf->veb[pf->lan_veb]->pf = pf; 10492 pf->veb[pf->lan_veb]->veb_idx = I40E_NO_VEB; 10493 break; 10494 case I40E_SWITCH_ELEMENT_TYPE_VSI: 10495 if (num_reported != 1) 10496 break; 10497 /* This is immediately after a reset so we can assume this is 10498 * the PF's VSI 10499 */ 10500 pf->mac_seid = uplink_seid; 10501 pf->pf_seid = downlink_seid; 10502 pf->main_vsi_seid = seid; 10503 if (printconfig) 10504 dev_info(&pf->pdev->dev, 10505 "pf_seid=%d main_vsi_seid=%d\n", 10506 pf->pf_seid, pf->main_vsi_seid); 10507 break; 10508 case I40E_SWITCH_ELEMENT_TYPE_PF: 10509 case I40E_SWITCH_ELEMENT_TYPE_VF: 10510 case I40E_SWITCH_ELEMENT_TYPE_EMP: 10511 case I40E_SWITCH_ELEMENT_TYPE_BMC: 10512 case I40E_SWITCH_ELEMENT_TYPE_PE: 10513 case I40E_SWITCH_ELEMENT_TYPE_PA: 10514 /* ignore these for now */ 10515 break; 10516 default: 10517 dev_info(&pf->pdev->dev, "unknown element type=%d seid=%d\n", 10518 element_type, seid); 10519 break; 10520 } 10521 } 10522 10523 /** 10524 * i40e_fetch_switch_configuration - Get switch config from firmware 10525 * @pf: board private structure 10526 * @printconfig: should we print the contents 10527 * 10528 * Get the current switch configuration from the device and 10529 * extract a few useful SEID values. 10530 **/ 10531 int i40e_fetch_switch_configuration(struct i40e_pf *pf, bool printconfig) 10532 { 10533 struct i40e_aqc_get_switch_config_resp *sw_config; 10534 u16 next_seid = 0; 10535 int ret = 0; 10536 u8 *aq_buf; 10537 int i; 10538 10539 aq_buf = kzalloc(I40E_AQ_LARGE_BUF, GFP_KERNEL); 10540 if (!aq_buf) 10541 return -ENOMEM; 10542 10543 sw_config = (struct i40e_aqc_get_switch_config_resp *)aq_buf; 10544 do { 10545 u16 num_reported, num_total; 10546 10547 ret = i40e_aq_get_switch_config(&pf->hw, sw_config, 10548 I40E_AQ_LARGE_BUF, 10549 &next_seid, NULL); 10550 if (ret) { 10551 dev_info(&pf->pdev->dev, 10552 "get switch config failed err %s aq_err %s\n", 10553 i40e_stat_str(&pf->hw, ret), 10554 i40e_aq_str(&pf->hw, 10555 pf->hw.aq.asq_last_status)); 10556 kfree(aq_buf); 10557 return -ENOENT; 10558 } 10559 10560 num_reported = le16_to_cpu(sw_config->header.num_reported); 10561 num_total = le16_to_cpu(sw_config->header.num_total); 10562 10563 if (printconfig) 10564 dev_info(&pf->pdev->dev, 10565 "header: %d reported %d total\n", 10566 num_reported, num_total); 10567 10568 for (i = 0; i < num_reported; i++) { 10569 struct i40e_aqc_switch_config_element_resp *ele = 10570 &sw_config->element[i]; 10571 10572 i40e_setup_pf_switch_element(pf, ele, num_reported, 10573 printconfig); 10574 } 10575 } while (next_seid != 0); 10576 10577 kfree(aq_buf); 10578 return ret; 10579 } 10580 10581 /** 10582 * i40e_setup_pf_switch - Setup the HW switch on startup or after reset 10583 * @pf: board private structure 10584 * @reinit: if the Main VSI needs to re-initialized. 10585 * 10586 * Returns 0 on success, negative value on failure 10587 **/ 10588 static int i40e_setup_pf_switch(struct i40e_pf *pf, bool reinit) 10589 { 10590 u16 flags = 0; 10591 int ret; 10592 10593 /* find out what's out there already */ 10594 ret = i40e_fetch_switch_configuration(pf, false); 10595 if (ret) { 10596 dev_info(&pf->pdev->dev, 10597 "couldn't fetch switch config, err %s aq_err %s\n", 10598 i40e_stat_str(&pf->hw, ret), 10599 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 10600 return ret; 10601 } 10602 i40e_pf_reset_stats(pf); 10603 10604 /* set the switch config bit for the whole device to 10605 * support limited promisc or true promisc 10606 * when user requests promisc. The default is limited 10607 * promisc. 10608 */ 10609 10610 if ((pf->hw.pf_id == 0) && 10611 !(pf->flags & I40E_FLAG_TRUE_PROMISC_SUPPORT)) 10612 flags = I40E_AQ_SET_SWITCH_CFG_PROMISC; 10613 10614 if (pf->hw.pf_id == 0) { 10615 u16 valid_flags; 10616 10617 valid_flags = I40E_AQ_SET_SWITCH_CFG_PROMISC; 10618 ret = i40e_aq_set_switch_config(&pf->hw, flags, valid_flags, 10619 NULL); 10620 if (ret && pf->hw.aq.asq_last_status != I40E_AQ_RC_ESRCH) { 10621 dev_info(&pf->pdev->dev, 10622 "couldn't set switch config bits, err %s aq_err %s\n", 10623 i40e_stat_str(&pf->hw, ret), 10624 i40e_aq_str(&pf->hw, 10625 pf->hw.aq.asq_last_status)); 10626 /* not a fatal problem, just keep going */ 10627 } 10628 } 10629 10630 /* first time setup */ 10631 if (pf->lan_vsi == I40E_NO_VSI || reinit) { 10632 struct i40e_vsi *vsi = NULL; 10633 u16 uplink_seid; 10634 10635 /* Set up the PF VSI associated with the PF's main VSI 10636 * that is already in the HW switch 10637 */ 10638 if (pf->lan_veb != I40E_NO_VEB && pf->veb[pf->lan_veb]) 10639 uplink_seid = pf->veb[pf->lan_veb]->seid; 10640 else 10641 uplink_seid = pf->mac_seid; 10642 if (pf->lan_vsi == I40E_NO_VSI) 10643 vsi = i40e_vsi_setup(pf, I40E_VSI_MAIN, uplink_seid, 0); 10644 else if (reinit) 10645 vsi = i40e_vsi_reinit_setup(pf->vsi[pf->lan_vsi]); 10646 if (!vsi) { 10647 dev_info(&pf->pdev->dev, "setup of MAIN VSI failed\n"); 10648 i40e_fdir_teardown(pf); 10649 return -EAGAIN; 10650 } 10651 } else { 10652 /* force a reset of TC and queue layout configurations */ 10653 u8 enabled_tc = pf->vsi[pf->lan_vsi]->tc_config.enabled_tc; 10654 10655 pf->vsi[pf->lan_vsi]->tc_config.enabled_tc = 0; 10656 pf->vsi[pf->lan_vsi]->seid = pf->main_vsi_seid; 10657 i40e_vsi_config_tc(pf->vsi[pf->lan_vsi], enabled_tc); 10658 } 10659 i40e_vlan_stripping_disable(pf->vsi[pf->lan_vsi]); 10660 10661 i40e_fdir_sb_setup(pf); 10662 10663 /* Setup static PF queue filter control settings */ 10664 ret = i40e_setup_pf_filter_control(pf); 10665 if (ret) { 10666 dev_info(&pf->pdev->dev, "setup_pf_filter_control failed: %d\n", 10667 ret); 10668 /* Failure here should not stop continuing other steps */ 10669 } 10670 10671 /* enable RSS in the HW, even for only one queue, as the stack can use 10672 * the hash 10673 */ 10674 if ((pf->flags & I40E_FLAG_RSS_ENABLED)) 10675 i40e_pf_config_rss(pf); 10676 10677 /* fill in link information and enable LSE reporting */ 10678 i40e_update_link_info(&pf->hw); 10679 i40e_link_event(pf); 10680 10681 /* Initialize user-specific link properties */ 10682 pf->fc_autoneg_status = ((pf->hw.phy.link_info.an_info & 10683 I40E_AQ_AN_COMPLETED) ? true : false); 10684 10685 i40e_ptp_init(pf); 10686 10687 return ret; 10688 } 10689 10690 /** 10691 * i40e_determine_queue_usage - Work out queue distribution 10692 * @pf: board private structure 10693 **/ 10694 static void i40e_determine_queue_usage(struct i40e_pf *pf) 10695 { 10696 int queues_left; 10697 10698 pf->num_lan_qps = 0; 10699 #ifdef I40E_FCOE 10700 pf->num_fcoe_qps = 0; 10701 #endif 10702 10703 /* Find the max queues to be put into basic use. We'll always be 10704 * using TC0, whether or not DCB is running, and TC0 will get the 10705 * big RSS set. 10706 */ 10707 queues_left = pf->hw.func_caps.num_tx_qp; 10708 10709 if ((queues_left == 1) || 10710 !(pf->flags & I40E_FLAG_MSIX_ENABLED)) { 10711 /* one qp for PF, no queues for anything else */ 10712 queues_left = 0; 10713 pf->alloc_rss_size = pf->num_lan_qps = 1; 10714 10715 /* make sure all the fancies are disabled */ 10716 pf->flags &= ~(I40E_FLAG_RSS_ENABLED | 10717 I40E_FLAG_IWARP_ENABLED | 10718 #ifdef I40E_FCOE 10719 I40E_FLAG_FCOE_ENABLED | 10720 #endif 10721 I40E_FLAG_FD_SB_ENABLED | 10722 I40E_FLAG_FD_ATR_ENABLED | 10723 I40E_FLAG_DCB_CAPABLE | 10724 I40E_FLAG_DCB_ENABLED | 10725 I40E_FLAG_SRIOV_ENABLED | 10726 I40E_FLAG_VMDQ_ENABLED); 10727 } else if (!(pf->flags & (I40E_FLAG_RSS_ENABLED | 10728 I40E_FLAG_FD_SB_ENABLED | 10729 I40E_FLAG_FD_ATR_ENABLED | 10730 I40E_FLAG_DCB_CAPABLE))) { 10731 /* one qp for PF */ 10732 pf->alloc_rss_size = pf->num_lan_qps = 1; 10733 queues_left -= pf->num_lan_qps; 10734 10735 pf->flags &= ~(I40E_FLAG_RSS_ENABLED | 10736 I40E_FLAG_IWARP_ENABLED | 10737 #ifdef I40E_FCOE 10738 I40E_FLAG_FCOE_ENABLED | 10739 #endif 10740 I40E_FLAG_FD_SB_ENABLED | 10741 I40E_FLAG_FD_ATR_ENABLED | 10742 I40E_FLAG_DCB_ENABLED | 10743 I40E_FLAG_VMDQ_ENABLED); 10744 } else { 10745 /* Not enough queues for all TCs */ 10746 if ((pf->flags & I40E_FLAG_DCB_CAPABLE) && 10747 (queues_left < I40E_MAX_TRAFFIC_CLASS)) { 10748 pf->flags &= ~(I40E_FLAG_DCB_CAPABLE | 10749 I40E_FLAG_DCB_ENABLED); 10750 dev_info(&pf->pdev->dev, "not enough queues for DCB. DCB is disabled.\n"); 10751 } 10752 pf->num_lan_qps = max_t(int, pf->rss_size_max, 10753 num_online_cpus()); 10754 pf->num_lan_qps = min_t(int, pf->num_lan_qps, 10755 pf->hw.func_caps.num_tx_qp); 10756 10757 queues_left -= pf->num_lan_qps; 10758 } 10759 10760 #ifdef I40E_FCOE 10761 if (pf->flags & I40E_FLAG_FCOE_ENABLED) { 10762 if (I40E_DEFAULT_FCOE <= queues_left) { 10763 pf->num_fcoe_qps = I40E_DEFAULT_FCOE; 10764 } else if (I40E_MINIMUM_FCOE <= queues_left) { 10765 pf->num_fcoe_qps = I40E_MINIMUM_FCOE; 10766 } else { 10767 pf->num_fcoe_qps = 0; 10768 pf->flags &= ~I40E_FLAG_FCOE_ENABLED; 10769 dev_info(&pf->pdev->dev, "not enough queues for FCoE. FCoE feature will be disabled\n"); 10770 } 10771 10772 queues_left -= pf->num_fcoe_qps; 10773 } 10774 10775 #endif 10776 if (pf->flags & I40E_FLAG_FD_SB_ENABLED) { 10777 if (queues_left > 1) { 10778 queues_left -= 1; /* save 1 queue for FD */ 10779 } else { 10780 pf->flags &= ~I40E_FLAG_FD_SB_ENABLED; 10781 dev_info(&pf->pdev->dev, "not enough queues for Flow Director. Flow Director feature is disabled\n"); 10782 } 10783 } 10784 10785 if ((pf->flags & I40E_FLAG_SRIOV_ENABLED) && 10786 pf->num_vf_qps && pf->num_req_vfs && queues_left) { 10787 pf->num_req_vfs = min_t(int, pf->num_req_vfs, 10788 (queues_left / pf->num_vf_qps)); 10789 queues_left -= (pf->num_req_vfs * pf->num_vf_qps); 10790 } 10791 10792 if ((pf->flags & I40E_FLAG_VMDQ_ENABLED) && 10793 pf->num_vmdq_vsis && pf->num_vmdq_qps && queues_left) { 10794 pf->num_vmdq_vsis = min_t(int, pf->num_vmdq_vsis, 10795 (queues_left / pf->num_vmdq_qps)); 10796 queues_left -= (pf->num_vmdq_vsis * pf->num_vmdq_qps); 10797 } 10798 10799 pf->queues_left = queues_left; 10800 dev_dbg(&pf->pdev->dev, 10801 "qs_avail=%d FD SB=%d lan_qs=%d lan_tc0=%d vf=%d*%d vmdq=%d*%d, remaining=%d\n", 10802 pf->hw.func_caps.num_tx_qp, 10803 !!(pf->flags & I40E_FLAG_FD_SB_ENABLED), 10804 pf->num_lan_qps, pf->alloc_rss_size, pf->num_req_vfs, 10805 pf->num_vf_qps, pf->num_vmdq_vsis, pf->num_vmdq_qps, 10806 queues_left); 10807 #ifdef I40E_FCOE 10808 dev_dbg(&pf->pdev->dev, "fcoe queues = %d\n", pf->num_fcoe_qps); 10809 #endif 10810 } 10811 10812 /** 10813 * i40e_setup_pf_filter_control - Setup PF static filter control 10814 * @pf: PF to be setup 10815 * 10816 * i40e_setup_pf_filter_control sets up a PF's initial filter control 10817 * settings. If PE/FCoE are enabled then it will also set the per PF 10818 * based filter sizes required for them. It also enables Flow director, 10819 * ethertype and macvlan type filter settings for the pf. 10820 * 10821 * Returns 0 on success, negative on failure 10822 **/ 10823 static int i40e_setup_pf_filter_control(struct i40e_pf *pf) 10824 { 10825 struct i40e_filter_control_settings *settings = &pf->filter_settings; 10826 10827 settings->hash_lut_size = I40E_HASH_LUT_SIZE_128; 10828 10829 /* Flow Director is enabled */ 10830 if (pf->flags & (I40E_FLAG_FD_SB_ENABLED | I40E_FLAG_FD_ATR_ENABLED)) 10831 settings->enable_fdir = true; 10832 10833 /* Ethtype and MACVLAN filters enabled for PF */ 10834 settings->enable_ethtype = true; 10835 settings->enable_macvlan = true; 10836 10837 if (i40e_set_filter_control(&pf->hw, settings)) 10838 return -ENOENT; 10839 10840 return 0; 10841 } 10842 10843 #define INFO_STRING_LEN 255 10844 #define REMAIN(__x) (INFO_STRING_LEN - (__x)) 10845 static void i40e_print_features(struct i40e_pf *pf) 10846 { 10847 struct i40e_hw *hw = &pf->hw; 10848 char *buf; 10849 int i; 10850 10851 buf = kmalloc(INFO_STRING_LEN, GFP_KERNEL); 10852 if (!buf) 10853 return; 10854 10855 i = snprintf(buf, INFO_STRING_LEN, "Features: PF-id[%d]", hw->pf_id); 10856 #ifdef CONFIG_PCI_IOV 10857 i += snprintf(&buf[i], REMAIN(i), " VFs: %d", pf->num_req_vfs); 10858 #endif 10859 i += snprintf(&buf[i], REMAIN(i), " VSIs: %d QP: %d", 10860 pf->hw.func_caps.num_vsis, 10861 pf->vsi[pf->lan_vsi]->num_queue_pairs); 10862 if (pf->flags & I40E_FLAG_RSS_ENABLED) 10863 i += snprintf(&buf[i], REMAIN(i), " RSS"); 10864 if (pf->flags & I40E_FLAG_FD_ATR_ENABLED) 10865 i += snprintf(&buf[i], REMAIN(i), " FD_ATR"); 10866 if (pf->flags & I40E_FLAG_FD_SB_ENABLED) { 10867 i += snprintf(&buf[i], REMAIN(i), " FD_SB"); 10868 i += snprintf(&buf[i], REMAIN(i), " NTUPLE"); 10869 } 10870 if (pf->flags & I40E_FLAG_DCB_CAPABLE) 10871 i += snprintf(&buf[i], REMAIN(i), " DCB"); 10872 i += snprintf(&buf[i], REMAIN(i), " VxLAN"); 10873 i += snprintf(&buf[i], REMAIN(i), " Geneve"); 10874 if (pf->flags & I40E_FLAG_PTP) 10875 i += snprintf(&buf[i], REMAIN(i), " PTP"); 10876 #ifdef I40E_FCOE 10877 if (pf->flags & I40E_FLAG_FCOE_ENABLED) 10878 i += snprintf(&buf[i], REMAIN(i), " FCOE"); 10879 #endif 10880 if (pf->flags & I40E_FLAG_VEB_MODE_ENABLED) 10881 i += snprintf(&buf[i], REMAIN(i), " VEB"); 10882 else 10883 i += snprintf(&buf[i], REMAIN(i), " VEPA"); 10884 10885 dev_info(&pf->pdev->dev, "%s\n", buf); 10886 kfree(buf); 10887 WARN_ON(i > INFO_STRING_LEN); 10888 } 10889 10890 /** 10891 * i40e_get_platform_mac_addr - get platform-specific MAC address 10892 * 10893 * @pdev: PCI device information struct 10894 * @pf: board private structure 10895 * 10896 * Look up the MAC address in Open Firmware on systems that support it, 10897 * and use IDPROM on SPARC if no OF address is found. On return, the 10898 * I40E_FLAG_PF_MAC will be wset in pf->flags if a platform-specific value 10899 * has been selected. 10900 **/ 10901 static void i40e_get_platform_mac_addr(struct pci_dev *pdev, struct i40e_pf *pf) 10902 { 10903 pf->flags &= ~I40E_FLAG_PF_MAC; 10904 if (!eth_platform_get_mac_address(&pdev->dev, pf->hw.mac.addr)) 10905 pf->flags |= I40E_FLAG_PF_MAC; 10906 } 10907 10908 /** 10909 * i40e_probe - Device initialization routine 10910 * @pdev: PCI device information struct 10911 * @ent: entry in i40e_pci_tbl 10912 * 10913 * i40e_probe initializes a PF identified by a pci_dev structure. 10914 * The OS initialization, configuring of the PF private structure, 10915 * and a hardware reset occur. 10916 * 10917 * Returns 0 on success, negative on failure 10918 **/ 10919 static int i40e_probe(struct pci_dev *pdev, const struct pci_device_id *ent) 10920 { 10921 struct i40e_aq_get_phy_abilities_resp abilities; 10922 struct i40e_pf *pf; 10923 struct i40e_hw *hw; 10924 static u16 pfs_found; 10925 u16 wol_nvm_bits; 10926 u16 link_status; 10927 int err; 10928 u32 val; 10929 u32 i; 10930 u8 set_fc_aq_fail; 10931 10932 err = pci_enable_device_mem(pdev); 10933 if (err) 10934 return err; 10935 10936 /* set up for high or low dma */ 10937 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); 10938 if (err) { 10939 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)); 10940 if (err) { 10941 dev_err(&pdev->dev, 10942 "DMA configuration failed: 0x%x\n", err); 10943 goto err_dma; 10944 } 10945 } 10946 10947 /* set up pci connections */ 10948 err = pci_request_mem_regions(pdev, i40e_driver_name); 10949 if (err) { 10950 dev_info(&pdev->dev, 10951 "pci_request_selected_regions failed %d\n", err); 10952 goto err_pci_reg; 10953 } 10954 10955 pci_enable_pcie_error_reporting(pdev); 10956 pci_set_master(pdev); 10957 10958 /* Now that we have a PCI connection, we need to do the 10959 * low level device setup. This is primarily setting up 10960 * the Admin Queue structures and then querying for the 10961 * device's current profile information. 10962 */ 10963 pf = kzalloc(sizeof(*pf), GFP_KERNEL); 10964 if (!pf) { 10965 err = -ENOMEM; 10966 goto err_pf_alloc; 10967 } 10968 pf->next_vsi = 0; 10969 pf->pdev = pdev; 10970 set_bit(__I40E_DOWN, &pf->state); 10971 10972 hw = &pf->hw; 10973 hw->back = pf; 10974 10975 pf->ioremap_len = min_t(int, pci_resource_len(pdev, 0), 10976 I40E_MAX_CSR_SPACE); 10977 10978 hw->hw_addr = ioremap(pci_resource_start(pdev, 0), pf->ioremap_len); 10979 if (!hw->hw_addr) { 10980 err = -EIO; 10981 dev_info(&pdev->dev, "ioremap(0x%04x, 0x%04x) failed: 0x%x\n", 10982 (unsigned int)pci_resource_start(pdev, 0), 10983 pf->ioremap_len, err); 10984 goto err_ioremap; 10985 } 10986 hw->vendor_id = pdev->vendor; 10987 hw->device_id = pdev->device; 10988 pci_read_config_byte(pdev, PCI_REVISION_ID, &hw->revision_id); 10989 hw->subsystem_vendor_id = pdev->subsystem_vendor; 10990 hw->subsystem_device_id = pdev->subsystem_device; 10991 hw->bus.device = PCI_SLOT(pdev->devfn); 10992 hw->bus.func = PCI_FUNC(pdev->devfn); 10993 pf->instance = pfs_found; 10994 10995 /* set up the locks for the AQ, do this only once in probe 10996 * and destroy them only once in remove 10997 */ 10998 mutex_init(&hw->aq.asq_mutex); 10999 mutex_init(&hw->aq.arq_mutex); 11000 11001 pf->msg_enable = netif_msg_init(debug, 11002 NETIF_MSG_DRV | 11003 NETIF_MSG_PROBE | 11004 NETIF_MSG_LINK); 11005 if (debug < -1) 11006 pf->hw.debug_mask = debug; 11007 11008 /* do a special CORER for clearing PXE mode once at init */ 11009 if (hw->revision_id == 0 && 11010 (rd32(hw, I40E_GLLAN_RCTL_0) & I40E_GLLAN_RCTL_0_PXE_MODE_MASK)) { 11011 wr32(hw, I40E_GLGEN_RTRIG, I40E_GLGEN_RTRIG_CORER_MASK); 11012 i40e_flush(hw); 11013 msleep(200); 11014 pf->corer_count++; 11015 11016 i40e_clear_pxe_mode(hw); 11017 } 11018 11019 /* Reset here to make sure all is clean and to define PF 'n' */ 11020 i40e_clear_hw(hw); 11021 err = i40e_pf_reset(hw); 11022 if (err) { 11023 dev_info(&pdev->dev, "Initial pf_reset failed: %d\n", err); 11024 goto err_pf_reset; 11025 } 11026 pf->pfr_count++; 11027 11028 hw->aq.num_arq_entries = I40E_AQ_LEN; 11029 hw->aq.num_asq_entries = I40E_AQ_LEN; 11030 hw->aq.arq_buf_size = I40E_MAX_AQ_BUF_SIZE; 11031 hw->aq.asq_buf_size = I40E_MAX_AQ_BUF_SIZE; 11032 pf->adminq_work_limit = I40E_AQ_WORK_LIMIT; 11033 11034 snprintf(pf->int_name, sizeof(pf->int_name) - 1, 11035 "%s-%s:misc", 11036 dev_driver_string(&pf->pdev->dev), dev_name(&pdev->dev)); 11037 11038 err = i40e_init_shared_code(hw); 11039 if (err) { 11040 dev_warn(&pdev->dev, "unidentified MAC or BLANK NVM: %d\n", 11041 err); 11042 goto err_pf_reset; 11043 } 11044 11045 /* set up a default setting for link flow control */ 11046 pf->hw.fc.requested_mode = I40E_FC_NONE; 11047 11048 err = i40e_init_adminq(hw); 11049 if (err) { 11050 if (err == I40E_ERR_FIRMWARE_API_VERSION) 11051 dev_info(&pdev->dev, 11052 "The driver for the device stopped because the NVM image is newer than expected. You must install the most recent version of the network driver.\n"); 11053 else 11054 dev_info(&pdev->dev, 11055 "The driver for the device stopped because the device firmware failed to init. Try updating your NVM image.\n"); 11056 11057 goto err_pf_reset; 11058 } 11059 11060 /* provide nvm, fw, api versions */ 11061 dev_info(&pdev->dev, "fw %d.%d.%05d api %d.%d nvm %s\n", 11062 hw->aq.fw_maj_ver, hw->aq.fw_min_ver, hw->aq.fw_build, 11063 hw->aq.api_maj_ver, hw->aq.api_min_ver, 11064 i40e_nvm_version_str(hw)); 11065 11066 if (hw->aq.api_maj_ver == I40E_FW_API_VERSION_MAJOR && 11067 hw->aq.api_min_ver > I40E_FW_API_VERSION_MINOR) 11068 dev_info(&pdev->dev, 11069 "The driver for the device detected a newer version of the NVM image than expected. Please install the most recent version of the network driver.\n"); 11070 else if (hw->aq.api_maj_ver < I40E_FW_API_VERSION_MAJOR || 11071 hw->aq.api_min_ver < (I40E_FW_API_VERSION_MINOR - 1)) 11072 dev_info(&pdev->dev, 11073 "The driver for the device detected an older version of the NVM image than expected. Please update the NVM image.\n"); 11074 11075 i40e_verify_eeprom(pf); 11076 11077 /* Rev 0 hardware was never productized */ 11078 if (hw->revision_id < 1) 11079 dev_warn(&pdev->dev, "This device is a pre-production adapter/LOM. Please be aware there may be issues with your hardware. If you are experiencing problems please contact your Intel or hardware representative who provided you with this hardware.\n"); 11080 11081 i40e_clear_pxe_mode(hw); 11082 err = i40e_get_capabilities(pf); 11083 if (err) 11084 goto err_adminq_setup; 11085 11086 err = i40e_sw_init(pf); 11087 if (err) { 11088 dev_info(&pdev->dev, "sw_init failed: %d\n", err); 11089 goto err_sw_init; 11090 } 11091 11092 err = i40e_init_lan_hmc(hw, hw->func_caps.num_tx_qp, 11093 hw->func_caps.num_rx_qp, 11094 pf->fcoe_hmc_cntx_num, pf->fcoe_hmc_filt_num); 11095 if (err) { 11096 dev_info(&pdev->dev, "init_lan_hmc failed: %d\n", err); 11097 goto err_init_lan_hmc; 11098 } 11099 11100 err = i40e_configure_lan_hmc(hw, I40E_HMC_MODEL_DIRECT_ONLY); 11101 if (err) { 11102 dev_info(&pdev->dev, "configure_lan_hmc failed: %d\n", err); 11103 err = -ENOENT; 11104 goto err_configure_lan_hmc; 11105 } 11106 11107 /* Disable LLDP for NICs that have firmware versions lower than v4.3. 11108 * Ignore error return codes because if it was already disabled via 11109 * hardware settings this will fail 11110 */ 11111 if (pf->flags & I40E_FLAG_STOP_FW_LLDP) { 11112 dev_info(&pdev->dev, "Stopping firmware LLDP agent.\n"); 11113 i40e_aq_stop_lldp(hw, true, NULL); 11114 } 11115 11116 i40e_get_mac_addr(hw, hw->mac.addr); 11117 /* allow a platform config to override the HW addr */ 11118 i40e_get_platform_mac_addr(pdev, pf); 11119 if (!is_valid_ether_addr(hw->mac.addr)) { 11120 dev_info(&pdev->dev, "invalid MAC address %pM\n", hw->mac.addr); 11121 err = -EIO; 11122 goto err_mac_addr; 11123 } 11124 dev_info(&pdev->dev, "MAC address: %pM\n", hw->mac.addr); 11125 ether_addr_copy(hw->mac.perm_addr, hw->mac.addr); 11126 i40e_get_port_mac_addr(hw, hw->mac.port_addr); 11127 if (is_valid_ether_addr(hw->mac.port_addr)) 11128 pf->flags |= I40E_FLAG_PORT_ID_VALID; 11129 #ifdef I40E_FCOE 11130 err = i40e_get_san_mac_addr(hw, hw->mac.san_addr); 11131 if (err) 11132 dev_info(&pdev->dev, 11133 "(non-fatal) SAN MAC retrieval failed: %d\n", err); 11134 if (!is_valid_ether_addr(hw->mac.san_addr)) { 11135 dev_warn(&pdev->dev, "invalid SAN MAC address %pM, falling back to LAN MAC\n", 11136 hw->mac.san_addr); 11137 ether_addr_copy(hw->mac.san_addr, hw->mac.addr); 11138 } 11139 dev_info(&pf->pdev->dev, "SAN MAC: %pM\n", hw->mac.san_addr); 11140 #endif /* I40E_FCOE */ 11141 11142 pci_set_drvdata(pdev, pf); 11143 pci_save_state(pdev); 11144 #ifdef CONFIG_I40E_DCB 11145 err = i40e_init_pf_dcb(pf); 11146 if (err) { 11147 dev_info(&pdev->dev, "DCB init failed %d, disabled\n", err); 11148 pf->flags &= ~(I40E_FLAG_DCB_CAPABLE | I40E_FLAG_DCB_ENABLED); 11149 /* Continue without DCB enabled */ 11150 } 11151 #endif /* CONFIG_I40E_DCB */ 11152 11153 /* set up periodic task facility */ 11154 setup_timer(&pf->service_timer, i40e_service_timer, (unsigned long)pf); 11155 pf->service_timer_period = HZ; 11156 11157 INIT_WORK(&pf->service_task, i40e_service_task); 11158 clear_bit(__I40E_SERVICE_SCHED, &pf->state); 11159 pf->flags |= I40E_FLAG_NEED_LINK_UPDATE; 11160 11161 /* NVM bit on means WoL disabled for the port */ 11162 i40e_read_nvm_word(hw, I40E_SR_NVM_WAKE_ON_LAN, &wol_nvm_bits); 11163 if (BIT (hw->port) & wol_nvm_bits || hw->partition_id != 1) 11164 pf->wol_en = false; 11165 else 11166 pf->wol_en = true; 11167 device_set_wakeup_enable(&pf->pdev->dev, pf->wol_en); 11168 11169 /* set up the main switch operations */ 11170 i40e_determine_queue_usage(pf); 11171 err = i40e_init_interrupt_scheme(pf); 11172 if (err) 11173 goto err_switch_setup; 11174 11175 /* The number of VSIs reported by the FW is the minimum guaranteed 11176 * to us; HW supports far more and we share the remaining pool with 11177 * the other PFs. We allocate space for more than the guarantee with 11178 * the understanding that we might not get them all later. 11179 */ 11180 if (pf->hw.func_caps.num_vsis < I40E_MIN_VSI_ALLOC) 11181 pf->num_alloc_vsi = I40E_MIN_VSI_ALLOC; 11182 else 11183 pf->num_alloc_vsi = pf->hw.func_caps.num_vsis; 11184 11185 /* Set up the *vsi struct and our local tracking of the MAIN PF vsi. */ 11186 pf->vsi = kcalloc(pf->num_alloc_vsi, sizeof(struct i40e_vsi *), 11187 GFP_KERNEL); 11188 if (!pf->vsi) { 11189 err = -ENOMEM; 11190 goto err_switch_setup; 11191 } 11192 11193 #ifdef CONFIG_PCI_IOV 11194 /* prep for VF support */ 11195 if ((pf->flags & I40E_FLAG_SRIOV_ENABLED) && 11196 (pf->flags & I40E_FLAG_MSIX_ENABLED) && 11197 !test_bit(__I40E_BAD_EEPROM, &pf->state)) { 11198 if (pci_num_vf(pdev)) 11199 pf->flags |= I40E_FLAG_VEB_MODE_ENABLED; 11200 } 11201 #endif 11202 err = i40e_setup_pf_switch(pf, false); 11203 if (err) { 11204 dev_info(&pdev->dev, "setup_pf_switch failed: %d\n", err); 11205 goto err_vsis; 11206 } 11207 11208 /* Make sure flow control is set according to current settings */ 11209 err = i40e_set_fc(hw, &set_fc_aq_fail, true); 11210 if (set_fc_aq_fail & I40E_SET_FC_AQ_FAIL_GET) 11211 dev_dbg(&pf->pdev->dev, 11212 "Set fc with err %s aq_err %s on get_phy_cap\n", 11213 i40e_stat_str(hw, err), 11214 i40e_aq_str(hw, hw->aq.asq_last_status)); 11215 if (set_fc_aq_fail & I40E_SET_FC_AQ_FAIL_SET) 11216 dev_dbg(&pf->pdev->dev, 11217 "Set fc with err %s aq_err %s on set_phy_config\n", 11218 i40e_stat_str(hw, err), 11219 i40e_aq_str(hw, hw->aq.asq_last_status)); 11220 if (set_fc_aq_fail & I40E_SET_FC_AQ_FAIL_UPDATE) 11221 dev_dbg(&pf->pdev->dev, 11222 "Set fc with err %s aq_err %s on get_link_info\n", 11223 i40e_stat_str(hw, err), 11224 i40e_aq_str(hw, hw->aq.asq_last_status)); 11225 11226 /* if FDIR VSI was set up, start it now */ 11227 for (i = 0; i < pf->num_alloc_vsi; i++) { 11228 if (pf->vsi[i] && pf->vsi[i]->type == I40E_VSI_FDIR) { 11229 i40e_vsi_open(pf->vsi[i]); 11230 break; 11231 } 11232 } 11233 11234 /* The driver only wants link up/down and module qualification 11235 * reports from firmware. Note the negative logic. 11236 */ 11237 err = i40e_aq_set_phy_int_mask(&pf->hw, 11238 ~(I40E_AQ_EVENT_LINK_UPDOWN | 11239 I40E_AQ_EVENT_MEDIA_NA | 11240 I40E_AQ_EVENT_MODULE_QUAL_FAIL), NULL); 11241 if (err) 11242 dev_info(&pf->pdev->dev, "set phy mask fail, err %s aq_err %s\n", 11243 i40e_stat_str(&pf->hw, err), 11244 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 11245 11246 /* Reconfigure hardware for allowing smaller MSS in the case 11247 * of TSO, so that we avoid the MDD being fired and causing 11248 * a reset in the case of small MSS+TSO. 11249 */ 11250 val = rd32(hw, I40E_REG_MSS); 11251 if ((val & I40E_REG_MSS_MIN_MASK) > I40E_64BYTE_MSS) { 11252 val &= ~I40E_REG_MSS_MIN_MASK; 11253 val |= I40E_64BYTE_MSS; 11254 wr32(hw, I40E_REG_MSS, val); 11255 } 11256 11257 if (pf->flags & I40E_FLAG_RESTART_AUTONEG) { 11258 msleep(75); 11259 err = i40e_aq_set_link_restart_an(&pf->hw, true, NULL); 11260 if (err) 11261 dev_info(&pf->pdev->dev, "link restart failed, err %s aq_err %s\n", 11262 i40e_stat_str(&pf->hw, err), 11263 i40e_aq_str(&pf->hw, 11264 pf->hw.aq.asq_last_status)); 11265 } 11266 /* The main driver is (mostly) up and happy. We need to set this state 11267 * before setting up the misc vector or we get a race and the vector 11268 * ends up disabled forever. 11269 */ 11270 clear_bit(__I40E_DOWN, &pf->state); 11271 11272 /* In case of MSIX we are going to setup the misc vector right here 11273 * to handle admin queue events etc. In case of legacy and MSI 11274 * the misc functionality and queue processing is combined in 11275 * the same vector and that gets setup at open. 11276 */ 11277 if (pf->flags & I40E_FLAG_MSIX_ENABLED) { 11278 err = i40e_setup_misc_vector(pf); 11279 if (err) { 11280 dev_info(&pdev->dev, 11281 "setup of misc vector failed: %d\n", err); 11282 goto err_vsis; 11283 } 11284 } 11285 11286 #ifdef CONFIG_PCI_IOV 11287 /* prep for VF support */ 11288 if ((pf->flags & I40E_FLAG_SRIOV_ENABLED) && 11289 (pf->flags & I40E_FLAG_MSIX_ENABLED) && 11290 !test_bit(__I40E_BAD_EEPROM, &pf->state)) { 11291 /* disable link interrupts for VFs */ 11292 val = rd32(hw, I40E_PFGEN_PORTMDIO_NUM); 11293 val &= ~I40E_PFGEN_PORTMDIO_NUM_VFLINK_STAT_ENA_MASK; 11294 wr32(hw, I40E_PFGEN_PORTMDIO_NUM, val); 11295 i40e_flush(hw); 11296 11297 if (pci_num_vf(pdev)) { 11298 dev_info(&pdev->dev, 11299 "Active VFs found, allocating resources.\n"); 11300 err = i40e_alloc_vfs(pf, pci_num_vf(pdev)); 11301 if (err) 11302 dev_info(&pdev->dev, 11303 "Error %d allocating resources for existing VFs\n", 11304 err); 11305 } 11306 } 11307 #endif /* CONFIG_PCI_IOV */ 11308 11309 if (pf->flags & I40E_FLAG_IWARP_ENABLED) { 11310 pf->iwarp_base_vector = i40e_get_lump(pf, pf->irq_pile, 11311 pf->num_iwarp_msix, 11312 I40E_IWARP_IRQ_PILE_ID); 11313 if (pf->iwarp_base_vector < 0) { 11314 dev_info(&pdev->dev, 11315 "failed to get tracking for %d vectors for IWARP err=%d\n", 11316 pf->num_iwarp_msix, pf->iwarp_base_vector); 11317 pf->flags &= ~I40E_FLAG_IWARP_ENABLED; 11318 } 11319 } 11320 11321 i40e_dbg_pf_init(pf); 11322 11323 /* tell the firmware that we're starting */ 11324 i40e_send_version(pf); 11325 11326 /* since everything's happy, start the service_task timer */ 11327 mod_timer(&pf->service_timer, 11328 round_jiffies(jiffies + pf->service_timer_period)); 11329 11330 /* add this PF to client device list and launch a client service task */ 11331 err = i40e_lan_add_device(pf); 11332 if (err) 11333 dev_info(&pdev->dev, "Failed to add PF to client API service list: %d\n", 11334 err); 11335 11336 #ifdef I40E_FCOE 11337 /* create FCoE interface */ 11338 i40e_fcoe_vsi_setup(pf); 11339 11340 #endif 11341 #define PCI_SPEED_SIZE 8 11342 #define PCI_WIDTH_SIZE 8 11343 /* Devices on the IOSF bus do not have this information 11344 * and will report PCI Gen 1 x 1 by default so don't bother 11345 * checking them. 11346 */ 11347 if (!(pf->flags & I40E_FLAG_NO_PCI_LINK_CHECK)) { 11348 char speed[PCI_SPEED_SIZE] = "Unknown"; 11349 char width[PCI_WIDTH_SIZE] = "Unknown"; 11350 11351 /* Get the negotiated link width and speed from PCI config 11352 * space 11353 */ 11354 pcie_capability_read_word(pf->pdev, PCI_EXP_LNKSTA, 11355 &link_status); 11356 11357 i40e_set_pci_config_data(hw, link_status); 11358 11359 switch (hw->bus.speed) { 11360 case i40e_bus_speed_8000: 11361 strncpy(speed, "8.0", PCI_SPEED_SIZE); break; 11362 case i40e_bus_speed_5000: 11363 strncpy(speed, "5.0", PCI_SPEED_SIZE); break; 11364 case i40e_bus_speed_2500: 11365 strncpy(speed, "2.5", PCI_SPEED_SIZE); break; 11366 default: 11367 break; 11368 } 11369 switch (hw->bus.width) { 11370 case i40e_bus_width_pcie_x8: 11371 strncpy(width, "8", PCI_WIDTH_SIZE); break; 11372 case i40e_bus_width_pcie_x4: 11373 strncpy(width, "4", PCI_WIDTH_SIZE); break; 11374 case i40e_bus_width_pcie_x2: 11375 strncpy(width, "2", PCI_WIDTH_SIZE); break; 11376 case i40e_bus_width_pcie_x1: 11377 strncpy(width, "1", PCI_WIDTH_SIZE); break; 11378 default: 11379 break; 11380 } 11381 11382 dev_info(&pdev->dev, "PCI-Express: Speed %sGT/s Width x%s\n", 11383 speed, width); 11384 11385 if (hw->bus.width < i40e_bus_width_pcie_x8 || 11386 hw->bus.speed < i40e_bus_speed_8000) { 11387 dev_warn(&pdev->dev, "PCI-Express bandwidth available for this device may be insufficient for optimal performance.\n"); 11388 dev_warn(&pdev->dev, "Please move the device to a different PCI-e link with more lanes and/or higher transfer rate.\n"); 11389 } 11390 } 11391 11392 /* get the requested speeds from the fw */ 11393 err = i40e_aq_get_phy_capabilities(hw, false, false, &abilities, NULL); 11394 if (err) 11395 dev_dbg(&pf->pdev->dev, "get requested speeds ret = %s last_status = %s\n", 11396 i40e_stat_str(&pf->hw, err), 11397 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 11398 pf->hw.phy.link_info.requested_speeds = abilities.link_speed; 11399 11400 /* get the supported phy types from the fw */ 11401 err = i40e_aq_get_phy_capabilities(hw, false, true, &abilities, NULL); 11402 if (err) 11403 dev_dbg(&pf->pdev->dev, "get supported phy types ret = %s last_status = %s\n", 11404 i40e_stat_str(&pf->hw, err), 11405 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 11406 11407 /* Add a filter to drop all Flow control frames from any VSI from being 11408 * transmitted. By doing so we stop a malicious VF from sending out 11409 * PAUSE or PFC frames and potentially controlling traffic for other 11410 * PF/VF VSIs. 11411 * The FW can still send Flow control frames if enabled. 11412 */ 11413 i40e_add_filter_to_drop_tx_flow_control_frames(&pf->hw, 11414 pf->main_vsi_seid); 11415 11416 if ((pf->hw.device_id == I40E_DEV_ID_10G_BASE_T) || 11417 (pf->hw.device_id == I40E_DEV_ID_10G_BASE_T4)) 11418 pf->flags |= I40E_FLAG_PHY_CONTROLS_LEDS; 11419 if (pf->hw.device_id == I40E_DEV_ID_SFP_I_X722) 11420 pf->flags |= I40E_FLAG_HAVE_CRT_RETIMER; 11421 /* print a string summarizing features */ 11422 i40e_print_features(pf); 11423 11424 return 0; 11425 11426 /* Unwind what we've done if something failed in the setup */ 11427 err_vsis: 11428 set_bit(__I40E_DOWN, &pf->state); 11429 i40e_clear_interrupt_scheme(pf); 11430 kfree(pf->vsi); 11431 err_switch_setup: 11432 i40e_reset_interrupt_capability(pf); 11433 del_timer_sync(&pf->service_timer); 11434 err_mac_addr: 11435 err_configure_lan_hmc: 11436 (void)i40e_shutdown_lan_hmc(hw); 11437 err_init_lan_hmc: 11438 kfree(pf->qp_pile); 11439 err_sw_init: 11440 err_adminq_setup: 11441 err_pf_reset: 11442 iounmap(hw->hw_addr); 11443 err_ioremap: 11444 kfree(pf); 11445 err_pf_alloc: 11446 pci_disable_pcie_error_reporting(pdev); 11447 pci_release_mem_regions(pdev); 11448 err_pci_reg: 11449 err_dma: 11450 pci_disable_device(pdev); 11451 return err; 11452 } 11453 11454 /** 11455 * i40e_remove - Device removal routine 11456 * @pdev: PCI device information struct 11457 * 11458 * i40e_remove is called by the PCI subsystem to alert the driver 11459 * that is should release a PCI device. This could be caused by a 11460 * Hot-Plug event, or because the driver is going to be removed from 11461 * memory. 11462 **/ 11463 static void i40e_remove(struct pci_dev *pdev) 11464 { 11465 struct i40e_pf *pf = pci_get_drvdata(pdev); 11466 struct i40e_hw *hw = &pf->hw; 11467 i40e_status ret_code; 11468 int i; 11469 11470 i40e_dbg_pf_exit(pf); 11471 11472 i40e_ptp_stop(pf); 11473 11474 /* Disable RSS in hw */ 11475 i40e_write_rx_ctl(hw, I40E_PFQF_HENA(0), 0); 11476 i40e_write_rx_ctl(hw, I40E_PFQF_HENA(1), 0); 11477 11478 /* no more scheduling of any task */ 11479 set_bit(__I40E_SUSPENDED, &pf->state); 11480 set_bit(__I40E_DOWN, &pf->state); 11481 if (pf->service_timer.data) 11482 del_timer_sync(&pf->service_timer); 11483 if (pf->service_task.func) 11484 cancel_work_sync(&pf->service_task); 11485 11486 if (pf->flags & I40E_FLAG_SRIOV_ENABLED) { 11487 i40e_free_vfs(pf); 11488 pf->flags &= ~I40E_FLAG_SRIOV_ENABLED; 11489 } 11490 11491 i40e_fdir_teardown(pf); 11492 11493 /* If there is a switch structure or any orphans, remove them. 11494 * This will leave only the PF's VSI remaining. 11495 */ 11496 for (i = 0; i < I40E_MAX_VEB; i++) { 11497 if (!pf->veb[i]) 11498 continue; 11499 11500 if (pf->veb[i]->uplink_seid == pf->mac_seid || 11501 pf->veb[i]->uplink_seid == 0) 11502 i40e_switch_branch_release(pf->veb[i]); 11503 } 11504 11505 /* Now we can shutdown the PF's VSI, just before we kill 11506 * adminq and hmc. 11507 */ 11508 if (pf->vsi[pf->lan_vsi]) 11509 i40e_vsi_release(pf->vsi[pf->lan_vsi]); 11510 11511 /* remove attached clients */ 11512 ret_code = i40e_lan_del_device(pf); 11513 if (ret_code) { 11514 dev_warn(&pdev->dev, "Failed to delete client device: %d\n", 11515 ret_code); 11516 } 11517 11518 /* shutdown and destroy the HMC */ 11519 if (hw->hmc.hmc_obj) { 11520 ret_code = i40e_shutdown_lan_hmc(hw); 11521 if (ret_code) 11522 dev_warn(&pdev->dev, 11523 "Failed to destroy the HMC resources: %d\n", 11524 ret_code); 11525 } 11526 11527 /* shutdown the adminq */ 11528 i40e_shutdown_adminq(hw); 11529 11530 /* destroy the locks only once, here */ 11531 mutex_destroy(&hw->aq.arq_mutex); 11532 mutex_destroy(&hw->aq.asq_mutex); 11533 11534 /* Clear all dynamic memory lists of rings, q_vectors, and VSIs */ 11535 i40e_clear_interrupt_scheme(pf); 11536 for (i = 0; i < pf->num_alloc_vsi; i++) { 11537 if (pf->vsi[i]) { 11538 i40e_vsi_clear_rings(pf->vsi[i]); 11539 i40e_vsi_clear(pf->vsi[i]); 11540 pf->vsi[i] = NULL; 11541 } 11542 } 11543 11544 for (i = 0; i < I40E_MAX_VEB; i++) { 11545 kfree(pf->veb[i]); 11546 pf->veb[i] = NULL; 11547 } 11548 11549 kfree(pf->qp_pile); 11550 kfree(pf->vsi); 11551 11552 iounmap(hw->hw_addr); 11553 kfree(pf); 11554 pci_release_mem_regions(pdev); 11555 11556 pci_disable_pcie_error_reporting(pdev); 11557 pci_disable_device(pdev); 11558 } 11559 11560 /** 11561 * i40e_pci_error_detected - warning that something funky happened in PCI land 11562 * @pdev: PCI device information struct 11563 * 11564 * Called to warn that something happened and the error handling steps 11565 * are in progress. Allows the driver to quiesce things, be ready for 11566 * remediation. 11567 **/ 11568 static pci_ers_result_t i40e_pci_error_detected(struct pci_dev *pdev, 11569 enum pci_channel_state error) 11570 { 11571 struct i40e_pf *pf = pci_get_drvdata(pdev); 11572 11573 dev_info(&pdev->dev, "%s: error %d\n", __func__, error); 11574 11575 if (!pf) { 11576 dev_info(&pdev->dev, 11577 "Cannot recover - error happened during device probe\n"); 11578 return PCI_ERS_RESULT_DISCONNECT; 11579 } 11580 11581 /* shutdown all operations */ 11582 if (!test_bit(__I40E_SUSPENDED, &pf->state)) { 11583 rtnl_lock(); 11584 i40e_prep_for_reset(pf); 11585 rtnl_unlock(); 11586 } 11587 11588 /* Request a slot reset */ 11589 return PCI_ERS_RESULT_NEED_RESET; 11590 } 11591 11592 /** 11593 * i40e_pci_error_slot_reset - a PCI slot reset just happened 11594 * @pdev: PCI device information struct 11595 * 11596 * Called to find if the driver can work with the device now that 11597 * the pci slot has been reset. If a basic connection seems good 11598 * (registers are readable and have sane content) then return a 11599 * happy little PCI_ERS_RESULT_xxx. 11600 **/ 11601 static pci_ers_result_t i40e_pci_error_slot_reset(struct pci_dev *pdev) 11602 { 11603 struct i40e_pf *pf = pci_get_drvdata(pdev); 11604 pci_ers_result_t result; 11605 int err; 11606 u32 reg; 11607 11608 dev_dbg(&pdev->dev, "%s\n", __func__); 11609 if (pci_enable_device_mem(pdev)) { 11610 dev_info(&pdev->dev, 11611 "Cannot re-enable PCI device after reset.\n"); 11612 result = PCI_ERS_RESULT_DISCONNECT; 11613 } else { 11614 pci_set_master(pdev); 11615 pci_restore_state(pdev); 11616 pci_save_state(pdev); 11617 pci_wake_from_d3(pdev, false); 11618 11619 reg = rd32(&pf->hw, I40E_GLGEN_RTRIG); 11620 if (reg == 0) 11621 result = PCI_ERS_RESULT_RECOVERED; 11622 else 11623 result = PCI_ERS_RESULT_DISCONNECT; 11624 } 11625 11626 err = pci_cleanup_aer_uncorrect_error_status(pdev); 11627 if (err) { 11628 dev_info(&pdev->dev, 11629 "pci_cleanup_aer_uncorrect_error_status failed 0x%0x\n", 11630 err); 11631 /* non-fatal, continue */ 11632 } 11633 11634 return result; 11635 } 11636 11637 /** 11638 * i40e_pci_error_resume - restart operations after PCI error recovery 11639 * @pdev: PCI device information struct 11640 * 11641 * Called to allow the driver to bring things back up after PCI error 11642 * and/or reset recovery has finished. 11643 **/ 11644 static void i40e_pci_error_resume(struct pci_dev *pdev) 11645 { 11646 struct i40e_pf *pf = pci_get_drvdata(pdev); 11647 11648 dev_dbg(&pdev->dev, "%s\n", __func__); 11649 if (test_bit(__I40E_SUSPENDED, &pf->state)) 11650 return; 11651 11652 rtnl_lock(); 11653 i40e_handle_reset_warning(pf); 11654 rtnl_unlock(); 11655 } 11656 11657 /** 11658 * i40e_shutdown - PCI callback for shutting down 11659 * @pdev: PCI device information struct 11660 **/ 11661 static void i40e_shutdown(struct pci_dev *pdev) 11662 { 11663 struct i40e_pf *pf = pci_get_drvdata(pdev); 11664 struct i40e_hw *hw = &pf->hw; 11665 11666 set_bit(__I40E_SUSPENDED, &pf->state); 11667 set_bit(__I40E_DOWN, &pf->state); 11668 rtnl_lock(); 11669 i40e_prep_for_reset(pf); 11670 rtnl_unlock(); 11671 11672 wr32(hw, I40E_PFPM_APM, (pf->wol_en ? I40E_PFPM_APM_APME_MASK : 0)); 11673 wr32(hw, I40E_PFPM_WUFC, (pf->wol_en ? I40E_PFPM_WUFC_MAG_MASK : 0)); 11674 11675 del_timer_sync(&pf->service_timer); 11676 cancel_work_sync(&pf->service_task); 11677 i40e_fdir_teardown(pf); 11678 11679 rtnl_lock(); 11680 i40e_prep_for_reset(pf); 11681 rtnl_unlock(); 11682 11683 wr32(hw, I40E_PFPM_APM, 11684 (pf->wol_en ? I40E_PFPM_APM_APME_MASK : 0)); 11685 wr32(hw, I40E_PFPM_WUFC, 11686 (pf->wol_en ? I40E_PFPM_WUFC_MAG_MASK : 0)); 11687 11688 i40e_clear_interrupt_scheme(pf); 11689 11690 if (system_state == SYSTEM_POWER_OFF) { 11691 pci_wake_from_d3(pdev, pf->wol_en); 11692 pci_set_power_state(pdev, PCI_D3hot); 11693 } 11694 } 11695 11696 #ifdef CONFIG_PM 11697 /** 11698 * i40e_suspend - PCI callback for moving to D3 11699 * @pdev: PCI device information struct 11700 **/ 11701 static int i40e_suspend(struct pci_dev *pdev, pm_message_t state) 11702 { 11703 struct i40e_pf *pf = pci_get_drvdata(pdev); 11704 struct i40e_hw *hw = &pf->hw; 11705 int retval = 0; 11706 11707 set_bit(__I40E_SUSPENDED, &pf->state); 11708 set_bit(__I40E_DOWN, &pf->state); 11709 11710 rtnl_lock(); 11711 i40e_prep_for_reset(pf); 11712 rtnl_unlock(); 11713 11714 wr32(hw, I40E_PFPM_APM, (pf->wol_en ? I40E_PFPM_APM_APME_MASK : 0)); 11715 wr32(hw, I40E_PFPM_WUFC, (pf->wol_en ? I40E_PFPM_WUFC_MAG_MASK : 0)); 11716 11717 i40e_stop_misc_vector(pf); 11718 11719 retval = pci_save_state(pdev); 11720 if (retval) 11721 return retval; 11722 11723 pci_wake_from_d3(pdev, pf->wol_en); 11724 pci_set_power_state(pdev, PCI_D3hot); 11725 11726 return retval; 11727 } 11728 11729 /** 11730 * i40e_resume - PCI callback for waking up from D3 11731 * @pdev: PCI device information struct 11732 **/ 11733 static int i40e_resume(struct pci_dev *pdev) 11734 { 11735 struct i40e_pf *pf = pci_get_drvdata(pdev); 11736 u32 err; 11737 11738 pci_set_power_state(pdev, PCI_D0); 11739 pci_restore_state(pdev); 11740 /* pci_restore_state() clears dev->state_saves, so 11741 * call pci_save_state() again to restore it. 11742 */ 11743 pci_save_state(pdev); 11744 11745 err = pci_enable_device_mem(pdev); 11746 if (err) { 11747 dev_err(&pdev->dev, "Cannot enable PCI device from suspend\n"); 11748 return err; 11749 } 11750 pci_set_master(pdev); 11751 11752 /* no wakeup events while running */ 11753 pci_wake_from_d3(pdev, false); 11754 11755 /* handling the reset will rebuild the device state */ 11756 if (test_and_clear_bit(__I40E_SUSPENDED, &pf->state)) { 11757 clear_bit(__I40E_DOWN, &pf->state); 11758 rtnl_lock(); 11759 i40e_reset_and_rebuild(pf, false); 11760 rtnl_unlock(); 11761 } 11762 11763 return 0; 11764 } 11765 11766 #endif 11767 static const struct pci_error_handlers i40e_err_handler = { 11768 .error_detected = i40e_pci_error_detected, 11769 .slot_reset = i40e_pci_error_slot_reset, 11770 .resume = i40e_pci_error_resume, 11771 }; 11772 11773 static struct pci_driver i40e_driver = { 11774 .name = i40e_driver_name, 11775 .id_table = i40e_pci_tbl, 11776 .probe = i40e_probe, 11777 .remove = i40e_remove, 11778 #ifdef CONFIG_PM 11779 .suspend = i40e_suspend, 11780 .resume = i40e_resume, 11781 #endif 11782 .shutdown = i40e_shutdown, 11783 .err_handler = &i40e_err_handler, 11784 .sriov_configure = i40e_pci_sriov_configure, 11785 }; 11786 11787 /** 11788 * i40e_init_module - Driver registration routine 11789 * 11790 * i40e_init_module is the first routine called when the driver is 11791 * loaded. All it does is register with the PCI subsystem. 11792 **/ 11793 static int __init i40e_init_module(void) 11794 { 11795 pr_info("%s: %s - version %s\n", i40e_driver_name, 11796 i40e_driver_string, i40e_driver_version_str); 11797 pr_info("%s: %s\n", i40e_driver_name, i40e_copyright); 11798 11799 /* we will see if single thread per module is enough for now, 11800 * it can't be any worse than using the system workqueue which 11801 * was already single threaded 11802 */ 11803 i40e_wq = alloc_workqueue("%s", WQ_UNBOUND | WQ_MEM_RECLAIM, 1, 11804 i40e_driver_name); 11805 if (!i40e_wq) { 11806 pr_err("%s: Failed to create workqueue\n", i40e_driver_name); 11807 return -ENOMEM; 11808 } 11809 11810 i40e_dbg_init(); 11811 return pci_register_driver(&i40e_driver); 11812 } 11813 module_init(i40e_init_module); 11814 11815 /** 11816 * i40e_exit_module - Driver exit cleanup routine 11817 * 11818 * i40e_exit_module is called just before the driver is removed 11819 * from memory. 11820 **/ 11821 static void __exit i40e_exit_module(void) 11822 { 11823 pci_unregister_driver(&i40e_driver); 11824 destroy_workqueue(i40e_wq); 11825 i40e_dbg_exit(); 11826 } 11827 module_exit(i40e_exit_module); 11828